net/smc/af_smc.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  *  Shared Memory Communications over RDMA (SMC-R) and RoCE
   4  *
   5  *  AF_SMC protocol family socket handler keeping the AF_INET sock address type
   6  *  applies to SOCK_STREAM sockets only
   7  *  offers an alternative communication option for TCP-protocol sockets
   8  *  applicable with RoCE-cards only
   9  *
  10  *  Initial restrictions:
  11  *    - support for alternate links postponed
  12  *
  13  *  Copyright IBM Corp. 2016, 2018
  14  *
  15  *  Author(s):  Ursula Braun <ubraun@linux.vnet.ibm.com>
  16  *              based on prototype from Frank Blaschka
  17  */
  18
  19 #define KMSG_COMPONENT "smc"
  20 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  21
  22 #include <linux/module.h>
  23 #include <linux/socket.h>
  24 #include <linux/workqueue.h>
  25 #include <linux/in.h>
  26 #include <linux/sched/signal.h>
  27 #include <linux/if_vlan.h>
  28 #include <linux/rcupdate_wait.h>
  29
  30 #include <net/sock.h>
  31 #include <net/tcp.h>
  32 #include <net/smc.h>
  33 #include <asm/ioctls.h>
  34
  35 #include <net/net_namespace.h>
  36 #include <net/netns/generic.h>
  37 #include "smc_netns.h"
  38
  39 #include "smc.h"
  40 #include "smc_clc.h"
  41 #include "smc_llc.h"
  42 #include "smc_cdc.h"
  43 #include "smc_core.h"
  44 #include "smc_ib.h"
  45 #include "smc_ism.h"
  46 #include "smc_pnet.h"
  47 #include "smc_tx.h"
  48 #include "smc_rx.h"
  49 #include "smc_close.h"
  50
  51 static DEFINE_MUTEX(smc_server_lgr_pending);    /* serialize link group
  52                                                  * creation on server
  53                                                  */
  54 static DEFINE_MUTEX(smc_client_lgr_pending);    /* serialize link group
  55                                                  * creation on client
  56                                                  */
  57
  58 static void smc_tcp_listen_work(struct work_struct *);
  59 static void smc_connect_work(struct work_struct *);
  60
  61 static void smc_set_keepalive(struct sock *sk, int val)
  62 {
  63         struct smc_sock *smc = smc_sk(sk);
  64
  65         smc->clcsock->sk->sk_prot->keepalive(smc->clcsock->sk, val);
  66 }
  67
  68 static struct smc_hashinfo smc_v4_hashinfo = {
  69         .lock = __RW_LOCK_UNLOCKED(smc_v4_hashinfo.lock),
  70 };
  71
  72 static struct smc_hashinfo smc_v6_hashinfo = {
  73         .lock = __RW_LOCK_UNLOCKED(smc_v6_hashinfo.lock),
  74 };
  75
  76 int smc_hash_sk(struct sock *sk)
  77 {
  78         struct smc_hashinfo *h = sk->sk_prot->h.smc_hash;
  79         struct hlist_head *head;
  80
  81         head = &h->ht;
  82
  83         write_lock_bh(&h->lock);
  84         sk_add_node(sk, head);
  85         sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
  86         write_unlock_bh(&h->lock);
  87
  88         return 0;
  89 }
  90 EXPORT_SYMBOL_GPL(smc_hash_sk);
  91
  92 void smc_unhash_sk(struct sock *sk)
  93 {
  94         struct smc_hashinfo *h = sk->sk_prot->h.smc_hash;
  95
  96         write_lock_bh(&h->lock);
  97         if (sk_del_node_init(sk))
  98                 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
  99         write_unlock_bh(&h->lock);
 100 }
 101 EXPORT_SYMBOL_GPL(smc_unhash_sk);
 102
 103 struct proto smc_proto = {
 104         .name           = "SMC",
 105         .owner          = THIS_MODULE,
 106         .keepalive      = smc_set_keepalive,
 107         .hash           = smc_hash_sk,
 108         .unhash         = smc_unhash_sk,
 109         .obj_size       = sizeof(struct smc_sock),
 110         .h.smc_hash     = &smc_v4_hashinfo,
 111         .slab_flags     = SLAB_TYPESAFE_BY_RCU,
 112 };
 113 EXPORT_SYMBOL_GPL(smc_proto);
 114
 115 struct proto smc_proto6 = {
 116         .name           = "SMC6",
 117         .owner          = THIS_MODULE,
 118         .keepalive      = smc_set_keepalive,
 119         .hash           = smc_hash_sk,
 120         .unhash         = smc_unhash_sk,
 121         .obj_size       = sizeof(struct smc_sock),
 122         .h.smc_hash     = &smc_v6_hashinfo,
 123         .slab_flags     = SLAB_TYPESAFE_BY_RCU,
 124 };
 125 EXPORT_SYMBOL_GPL(smc_proto6);
 126
 127 static void smc_restore_fallback_changes(struct smc_sock *smc)
 128 {
 129         smc->clcsock->file->private_data = smc->sk.sk_socket;
 130         smc->clcsock->file = NULL;
 131 }
 132
 133 static int __smc_release(struct smc_sock *smc)
 134 {
 135         struct sock *sk = &smc->sk;
 136         int rc = 0;
 137
 138         if (!smc->use_fallback) {
 139                 rc = smc_close_active(smc);
 140                 sock_set_flag(sk, SOCK_DEAD);
 141                 sk->sk_shutdown |= SHUTDOWN_MASK;
 142         } else {
 143                 if (sk->sk_state != SMC_LISTEN && sk->sk_state != SMC_INIT)
 144                         sock_put(sk); /* passive closing */
 145                 if (sk->sk_state == SMC_LISTEN) {
 146                         /* wake up clcsock accept */
 147                         rc = kernel_sock_shutdown(smc->clcsock, SHUT_RDWR);
 148                 }
 149                 sk->sk_state = SMC_CLOSED;
 150                 sk->sk_state_change(sk);
 151                 smc_restore_fallback_changes(smc);
 152         }
 153
 154         sk->sk_prot->unhash(sk);
 155
 156         if (sk->sk_state == SMC_CLOSED) {
 157                 if (smc->clcsock) {
 158                         release_sock(sk);
 159                         smc_clcsock_release(smc);
 160                         lock_sock(sk);
 161                 }
 162                 if (!smc->use_fallback)
 163                         smc_conn_free(&smc->conn);
 164         }
 165
 166         return rc;
 167 }
 168
 169 static int smc_release(struct socket *sock)
 170 {
 171         struct sock *sk = sock->sk;
 172         struct smc_sock *smc;
 173         int rc = 0;
 174
 175         if (!sk)
 176                 goto out;
 177
 178         sock_hold(sk); /* sock_put below */
 179         smc = smc_sk(sk);
 180
 181         /* cleanup for a dangling non-blocking connect */
 182         if (smc->connect_nonblock && sk->sk_state == SMC_INIT)
 183                 tcp_abort(smc->clcsock->sk, ECONNABORTED);
 184         flush_work(&smc->connect_work);
 185
 186         if (sk->sk_state == SMC_LISTEN)
 187                 /* smc_close_non_accepted() is called and acquires
 188                  * sock lock for child sockets again
 189                  */
 190                 lock_sock_nested(sk, SINGLE_DEPTH_NESTING);
 191         else
 192                 lock_sock(sk);
 193
 194         rc = __smc_release(smc);
 195
 196         /* detach socket */
 197         sock_orphan(sk);
 198         sock->sk = NULL;
 199         release_sock(sk);
 200
 201         sock_put(sk); /* sock_hold above */
 202         sock_put(sk); /* final sock_put */
 203 out:
 204         return rc;
 205 }
 206
 207 static void smc_destruct(struct sock *sk)
 208 {
 209         if (sk->sk_state != SMC_CLOSED)
 210                 return;
 211         if (!sock_flag(sk, SOCK_DEAD))
 212                 return;
 213
 214         sk_refcnt_debug_dec(sk);
 215 }
 216
 217 static struct sock *smc_sock_alloc(struct net *net, struct socket *sock,
 218                                    int protocol)
 219 {
 220         struct smc_sock *smc;
 221         struct proto *prot;
 222         struct sock *sk;
 223
 224         prot = (protocol == SMCPROTO_SMC6) ? &smc_proto6 : &smc_proto;
 225         sk = sk_alloc(net, PF_SMC, GFP_KERNEL, prot, 0);
 226         if (!sk)
 227                 return NULL;
 228
 229         sock_init_data(sock, sk); /* sets sk_refcnt to 1 */
 230         sk->sk_state = SMC_INIT;
 231         sk->sk_destruct = smc_destruct;
 232         sk->sk_protocol = protocol;
 233         smc = smc_sk(sk);
 234         INIT_WORK(&smc->tcp_listen_work, smc_tcp_listen_work);
 235         INIT_WORK(&smc->connect_work, smc_connect_work);
 236         INIT_DELAYED_WORK(&smc->conn.tx_work, smc_tx_work);
 237         INIT_LIST_HEAD(&smc->accept_q);
 238         spin_lock_init(&smc->accept_q_lock);
 239         spin_lock_init(&smc->conn.send_lock);
 240         sk->sk_prot->hash(sk);
 241         sk_refcnt_debug_inc(sk);
 242         mutex_init(&smc->clcsock_release_lock);
 243
 244         return sk;
 245 }
 246
 247 static int smc_bind(struct socket *sock, struct sockaddr *uaddr,
 248                     int addr_len)
 249 {
 250         struct sockaddr_in *addr = (struct sockaddr_in *)uaddr;
 251         struct sock *sk = sock->sk;
 252         struct smc_sock *smc;
 253         int rc;
 254
 255         smc = smc_sk(sk);
 256
 257         /* replicate tests from inet_bind(), to be safe wrt. future changes */
 258         rc = -EINVAL;
 259         if (addr_len < sizeof(struct sockaddr_in))
 260                 goto out;
 261
 262         rc = -EAFNOSUPPORT;
 263         if (addr->sin_family != AF_INET &&
 264             addr->sin_family != AF_INET6 &&
 265             addr->sin_family != AF_UNSPEC)
 266                 goto out;
 267         /* accept AF_UNSPEC (mapped to AF_INET) only if s_addr is INADDR_ANY */
 268         if (addr->sin_family == AF_UNSPEC &&
 269             addr->sin_addr.s_addr != htonl(INADDR_ANY))
 270                 goto out;
 271
 272         lock_sock(sk);
 273
 274         /* Check if socket is already active */
 275         rc = -EINVAL;
 276         if (sk->sk_state != SMC_INIT || smc->connect_nonblock)
 277                 goto out_rel;
 278
 279         smc->clcsock->sk->sk_reuse = sk->sk_reuse;
 280         rc = kernel_bind(smc->clcsock, uaddr, addr_len);
 281
 282 out_rel:
 283         release_sock(sk);
 284 out:
 285         return rc;
 286 }
 287
 288 static void smc_copy_sock_settings(struct sock *nsk, struct sock *osk,
 289                                    unsigned long mask)
 290 {
 291         /* options we don't get control via setsockopt for */
 292         nsk->sk_type = osk->sk_type;
 293         nsk->sk_sndbuf = osk->sk_sndbuf;
 294         nsk->sk_rcvbuf = osk->sk_rcvbuf;
 295         nsk->sk_sndtimeo = osk->sk_sndtimeo;
 296         nsk->sk_rcvtimeo = osk->sk_rcvtimeo;
 297         nsk->sk_mark = osk->sk_mark;
 298         nsk->sk_priority = osk->sk_priority;
 299         nsk->sk_rcvlowat = osk->sk_rcvlowat;
 300         nsk->sk_bound_dev_if = osk->sk_bound_dev_if;
 301         nsk->sk_err = osk->sk_err;
 302
 303         nsk->sk_flags &= ~mask;
 304         nsk->sk_flags |= osk->sk_flags & mask;
 305 }
 306
 307 #define SK_FLAGS_SMC_TO_CLC ((1UL << SOCK_URGINLINE) | \
 308                              (1UL << SOCK_KEEPOPEN) | \
 309                              (1UL << SOCK_LINGER) | \
 310                              (1UL << SOCK_BROADCAST) | \
 311                              (1UL << SOCK_TIMESTAMP) | \
 312                              (1UL << SOCK_DBG) | \
 313                              (1UL << SOCK_RCVTSTAMP) | \
 314                              (1UL << SOCK_RCVTSTAMPNS) | \
 315                              (1UL << SOCK_LOCALROUTE) | \
 316                              (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE) | \
 317                              (1UL << SOCK_RXQ_OVFL) | \
 318                              (1UL << SOCK_WIFI_STATUS) | \
 319                              (1UL << SOCK_NOFCS) | \
 320                              (1UL << SOCK_FILTER_LOCKED) | \
 321                              (1UL << SOCK_TSTAMP_NEW))
 322 /* copy only relevant settings and flags of SOL_SOCKET level from smc to
 323  * clc socket (since smc is not called for these options from net/core)
 324  */
 325 static void smc_copy_sock_settings_to_clc(struct smc_sock *smc)
 326 {
 327         smc_copy_sock_settings(smc->clcsock->sk, &smc->sk, SK_FLAGS_SMC_TO_CLC);
 328 }
 329
 330 #define SK_FLAGS_CLC_TO_SMC ((1UL << SOCK_URGINLINE) | \
 331                              (1UL << SOCK_KEEPOPEN) | \
 332                              (1UL << SOCK_LINGER) | \
 333                              (1UL << SOCK_DBG))
 334 /* copy only settings and flags relevant for smc from clc to smc socket */
 335 static void smc_copy_sock_settings_to_smc(struct smc_sock *smc)
 336 {
 337         smc_copy_sock_settings(&smc->sk, smc->clcsock->sk, SK_FLAGS_CLC_TO_SMC);
 338 }
 339
 340 /* register a new rmb, send confirm_rkey msg to register with peer */
 341 static int smcr_link_reg_rmb(struct smc_link *link,
 342                              struct smc_buf_desc *rmb_desc, bool conf_rkey)
 343 {
 344         if (!rmb_desc->is_reg_mr[link->link_idx]) {
 345                 /* register memory region for new rmb */
 346                 if (smc_wr_reg_send(link, rmb_desc->mr_rx[link->link_idx])) {
 347                         rmb_desc->is_reg_err = true;
 348                         return -EFAULT;
 349                 }
 350                 rmb_desc->is_reg_mr[link->link_idx] = true;
 351         }
 352         if (!conf_rkey)
 353                 return 0;
 354
 355         /* exchange confirm_rkey msg with peer */
 356         if (!rmb_desc->is_conf_rkey) {
 357                 if (smc_llc_do_confirm_rkey(link, rmb_desc)) {
 358                         rmb_desc->is_reg_err = true;
 359                         return -EFAULT;
 360                 }
 361                 rmb_desc->is_conf_rkey = true;
 362         }
 363         return 0;
 364 }
 365
 366 /* register the new rmb on all links */
 367 static int smcr_lgr_reg_rmbs(struct smc_link_group *lgr,
 368                              struct smc_buf_desc *rmb_desc)
 369 {
 370         int i, rc;
 371
 372         for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
 373                 if (lgr->lnk[i].state != SMC_LNK_ACTIVE)
 374                         continue;
 375                 rc = smcr_link_reg_rmb(&lgr->lnk[i], rmb_desc, true);
 376                 if (rc)
 377                         return rc;
 378         }
 379         return 0;
 380 }
 381
 382 static int smcr_clnt_conf_first_link(struct smc_sock *smc)
 383 {
 384         struct net *net = sock_net(smc->clcsock->sk);
 385         struct smc_link *link = smc->conn.lnk;
 386         int rest;
 387         int rc;
 388
 389         /* receive CONFIRM LINK request from server over RoCE fabric */
 390         rest = wait_for_completion_interruptible_timeout(
 391                 &link->llc_confirm,
 392                 SMC_LLC_WAIT_FIRST_TIME);
 393         if (rest <= 0) {
 394                 struct smc_clc_msg_decline dclc;
 395
 396                 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
 397                                       SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
 398                 return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_CL : rc;
 399         }
 400
 401         if (link->llc_confirm_rc)
 402                 return SMC_CLC_DECL_RMBE_EC;
 403
 404         rc = smc_ib_modify_qp_rts(link);
 405         if (rc)
 406                 return SMC_CLC_DECL_ERR_RDYLNK;
 407
 408         smc_wr_remember_qp_attr(link);
 409
 410         if (smcr_link_reg_rmb(link, smc->conn.rmb_desc, false))
 411                 return SMC_CLC_DECL_ERR_REGRMB;
 412
 413         /* send CONFIRM LINK response over RoCE fabric */
 414         rc = smc_llc_send_confirm_link(link, SMC_LLC_RESP);
 415         if (rc < 0)
 416                 return SMC_CLC_DECL_TIMEOUT_CL;
 417
 418         /* receive ADD LINK request from server over RoCE fabric */
 419         rest = wait_for_completion_interruptible_timeout(&link->llc_add,
 420                                                          SMC_LLC_WAIT_TIME);
 421         if (rest <= 0) {
 422                 struct smc_clc_msg_decline dclc;
 423
 424                 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
 425                                       SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
 426                 return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_AL : rc;
 427         }
 428
 429         /* send add link reject message, only one link supported for now */
 430         rc = smc_llc_send_add_link(link,
 431                                    link->smcibdev->mac[link->ibport - 1],
 432                                    link->gid, SMC_LLC_RESP);
 433         if (rc < 0)
 434                 return SMC_CLC_DECL_TIMEOUT_AL;
 435
 436         smc_llc_link_active(link, net->ipv4.sysctl_tcp_keepalive_time);
 437
 438         return 0;
 439 }
 440
 441 static void smcr_conn_save_peer_info(struct smc_sock *smc,
 442                                      struct smc_clc_msg_accept_confirm *clc)
 443 {
 444         int bufsize = smc_uncompress_bufsize(clc->rmbe_size);
 445
 446         smc->conn.peer_rmbe_idx = clc->rmbe_idx;
 447         smc->conn.local_tx_ctrl.token = ntohl(clc->rmbe_alert_token);
 448         smc->conn.peer_rmbe_size = bufsize;
 449         atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size);
 450         smc->conn.tx_off = bufsize * (smc->conn.peer_rmbe_idx - 1);
 451 }
 452
 453 static void smcd_conn_save_peer_info(struct smc_sock *smc,
 454                                      struct smc_clc_msg_accept_confirm *clc)
 455 {
 456         int bufsize = smc_uncompress_bufsize(clc->dmbe_size);
 457
 458         smc->conn.peer_rmbe_idx = clc->dmbe_idx;
 459         smc->conn.peer_token = clc->token;
 460         /* msg header takes up space in the buffer */
 461         smc->conn.peer_rmbe_size = bufsize - sizeof(struct smcd_cdc_msg);
 462         atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size);
 463         smc->conn.tx_off = bufsize * smc->conn.peer_rmbe_idx;
 464 }
 465
 466 static void smc_conn_save_peer_info(struct smc_sock *smc,
 467                                     struct smc_clc_msg_accept_confirm *clc)
 468 {
 469         if (smc->conn.lgr->is_smcd)
 470                 smcd_conn_save_peer_info(smc, clc);
 471         else
 472                 smcr_conn_save_peer_info(smc, clc);
 473 }
 474
 475 static void smc_link_save_peer_info(struct smc_link *link,
 476                                     struct smc_clc_msg_accept_confirm *clc)
 477 {
 478         link->peer_qpn = ntoh24(clc->qpn);
 479         memcpy(link->peer_gid, clc->lcl.gid, SMC_GID_SIZE);
 480         memcpy(link->peer_mac, clc->lcl.mac, sizeof(link->peer_mac));
 481         link->peer_psn = ntoh24(clc->psn);
 482         link->peer_mtu = clc->qp_mtu;
 483 }
 484
 485 static void smc_switch_to_fallback(struct smc_sock *smc)
 486 {
 487         smc->use_fallback = true;
 488         if (smc->sk.sk_socket && smc->sk.sk_socket->file) {
 489                 smc->clcsock->file = smc->sk.sk_socket->file;
 490                 smc->clcsock->file->private_data = smc->clcsock;
 491                 smc->clcsock->wq.fasync_list =
 492                         smc->sk.sk_socket->wq.fasync_list;
 493         }
 494 }
 495
 496 /* fall back during connect */
 497 static int smc_connect_fallback(struct smc_sock *smc, int reason_code)
 498 {
 499         smc_switch_to_fallback(smc);
 500         smc->fallback_rsn = reason_code;
 501         smc_copy_sock_settings_to_clc(smc);
 502         smc->connect_nonblock = 0;
 503         if (smc->sk.sk_state == SMC_INIT)
 504                 smc->sk.sk_state = SMC_ACTIVE;
 505         return 0;
 506 }
 507
 508 /* decline and fall back during connect */
 509 static int smc_connect_decline_fallback(struct smc_sock *smc, int reason_code)
 510 {
 511         int rc;
 512
 513         if (reason_code < 0) { /* error, fallback is not possible */
 514                 if (smc->sk.sk_state == SMC_INIT)
 515                         sock_put(&smc->sk); /* passive closing */
 516                 return reason_code;
 517         }
 518         if (reason_code != SMC_CLC_DECL_PEERDECL) {
 519                 rc = smc_clc_send_decline(smc, reason_code);
 520                 if (rc < 0) {
 521                         if (smc->sk.sk_state == SMC_INIT)
 522                                 sock_put(&smc->sk); /* passive closing */
 523                         return rc;
 524                 }
 525         }
 526         return smc_connect_fallback(smc, reason_code);
 527 }
 528
 529 /* abort connecting */
 530 static int smc_connect_abort(struct smc_sock *smc, int reason_code,
 531                              int local_contact)
 532 {
 533         bool is_smcd = smc->conn.lgr->is_smcd;
 534
 535         if (local_contact == SMC_FIRST_CONTACT)
 536                 smc_lgr_cleanup_early(&smc->conn);
 537         else
 538                 smc_conn_free(&smc->conn);
 539         if (is_smcd)
 540                 /* there is only one lgr role for SMC-D; use server lock */
 541                 mutex_unlock(&smc_server_lgr_pending);
 542         else
 543                 mutex_unlock(&smc_client_lgr_pending);
 544
 545         smc->connect_nonblock = 0;
 546         return reason_code;
 547 }
 548
 549 /* check if there is a rdma device available for this connection. */
 550 /* called for connect and listen */
 551 static int smc_find_rdma_device(struct smc_sock *smc, struct smc_init_info *ini)
 552 {
 553         /* PNET table look up: search active ib_device and port
 554          * within same PNETID that also contains the ethernet device
 555          * used for the internal TCP socket
 556          */
 557         smc_pnet_find_roce_resource(smc->clcsock->sk, ini);
 558         if (!ini->ib_dev)
 559                 return SMC_CLC_DECL_NOSMCRDEV;
 560         return 0;
 561 }
 562
 563 /* check if there is an ISM device available for this connection. */
 564 /* called for connect and listen */
 565 static int smc_find_ism_device(struct smc_sock *smc, struct smc_init_info *ini)
 566 {
 567         /* Find ISM device with same PNETID as connecting interface  */
 568         smc_pnet_find_ism_resource(smc->clcsock->sk, ini);
 569         if (!ini->ism_dev)
 570                 return SMC_CLC_DECL_NOSMCDDEV;
 571         return 0;
 572 }
 573
 574 /* Check for VLAN ID and register it on ISM device just for CLC handshake */
 575 static int smc_connect_ism_vlan_setup(struct smc_sock *smc,
 576                                       struct smc_init_info *ini)
 577 {
 578         if (ini->vlan_id && smc_ism_get_vlan(ini->ism_dev, ini->vlan_id))
 579                 return SMC_CLC_DECL_ISMVLANERR;
 580         return 0;
 581 }
 582
 583 /* cleanup temporary VLAN ID registration used for CLC handshake. If ISM is
 584  * used, the VLAN ID will be registered again during the connection setup.
 585  */
 586 static int smc_connect_ism_vlan_cleanup(struct smc_sock *smc, bool is_smcd,
 587                                         struct smc_init_info *ini)
 588 {
 589         if (!is_smcd)
 590                 return 0;
 591         if (ini->vlan_id && smc_ism_put_vlan(ini->ism_dev, ini->vlan_id))
 592                 return SMC_CLC_DECL_CNFERR;
 593         return 0;
 594 }
 595
 596 /* CLC handshake during connect */
 597 static int smc_connect_clc(struct smc_sock *smc, int smc_type,
 598                            struct smc_clc_msg_accept_confirm *aclc,
 599                            struct smc_init_info *ini)
 600 {
 601         int rc = 0;
 602
 603         /* do inband token exchange */
 604         rc = smc_clc_send_proposal(smc, smc_type, ini);
 605         if (rc)
 606                 return rc;
 607         /* receive SMC Accept CLC message */
 608         return smc_clc_wait_msg(smc, aclc, sizeof(*aclc), SMC_CLC_ACCEPT,
 609                                 CLC_WAIT_TIME);
 610 }
 611
 612 /* setup for RDMA connection of client */
 613 static int smc_connect_rdma(struct smc_sock *smc,
 614                             struct smc_clc_msg_accept_confirm *aclc,
 615                             struct smc_init_info *ini)
 616 {
 617         struct smc_link *link;
 618         int reason_code = 0;
 619
 620         ini->is_smcd = false;
 621         ini->ib_lcl = &aclc->lcl;
 622         ini->ib_clcqpn = ntoh24(aclc->qpn);
 623         ini->srv_first_contact = aclc->hdr.flag;
 624
 625         mutex_lock(&smc_client_lgr_pending);
 626         reason_code = smc_conn_create(smc, ini);
 627         if (reason_code) {
 628                 mutex_unlock(&smc_client_lgr_pending);
 629                 return reason_code;
 630         }
 631         link = smc->conn.lnk;
 632
 633         smc_conn_save_peer_info(smc, aclc);
 634
 635         /* create send buffer and rmb */
 636         if (smc_buf_create(smc, false))
 637                 return smc_connect_abort(smc, SMC_CLC_DECL_MEM,
 638                                          ini->cln_first_contact);
 639
 640         if (ini->cln_first_contact == SMC_FIRST_CONTACT)
 641                 smc_link_save_peer_info(link, aclc);
 642
 643         if (smc_rmb_rtoken_handling(&smc->conn, aclc))
 644                 return smc_connect_abort(smc, SMC_CLC_DECL_ERR_RTOK,
 645                                          ini->cln_first_contact);
 646
 647         smc_close_init(smc);
 648         smc_rx_init(smc);
 649
 650         if (ini->cln_first_contact == SMC_FIRST_CONTACT) {
 651                 if (smc_ib_ready_link(link))
 652                         return smc_connect_abort(smc, SMC_CLC_DECL_ERR_RDYLNK,
 653                                                  ini->cln_first_contact);
 654         } else {
 655                 if (smcr_lgr_reg_rmbs(smc->conn.lgr, smc->conn.rmb_desc))
 656                         return smc_connect_abort(smc, SMC_CLC_DECL_ERR_REGRMB,
 657                                                  ini->cln_first_contact);
 658         }
 659         smc_rmb_sync_sg_for_device(&smc->conn);
 660
 661         reason_code = smc_clc_send_confirm(smc);
 662         if (reason_code)
 663                 return smc_connect_abort(smc, reason_code,
 664                                          ini->cln_first_contact);
 665
 666         smc_tx_init(smc);
 667
 668         if (ini->cln_first_contact == SMC_FIRST_CONTACT) {
 669                 /* QP confirmation over RoCE fabric */
 670                 reason_code = smcr_clnt_conf_first_link(smc);
 671                 if (reason_code)
 672                         return smc_connect_abort(smc, reason_code,
 673                                                  ini->cln_first_contact);
 674         }
 675         mutex_unlock(&smc_client_lgr_pending);
 676
 677         smc_copy_sock_settings_to_clc(smc);
 678         smc->connect_nonblock = 0;
 679         if (smc->sk.sk_state == SMC_INIT)
 680                 smc->sk.sk_state = SMC_ACTIVE;
 681
 682         return 0;
 683 }
 684
 685 /* setup for ISM connection of client */
 686 static int smc_connect_ism(struct smc_sock *smc,
 687                            struct smc_clc_msg_accept_confirm *aclc,
 688                            struct smc_init_info *ini)
 689 {
 690         int rc = 0;
 691
 692         ini->is_smcd = true;
 693         ini->ism_gid = aclc->gid;
 694         ini->srv_first_contact = aclc->hdr.flag;
 695
 696         /* there is only one lgr role for SMC-D; use server lock */
 697         mutex_lock(&smc_server_lgr_pending);
 698         rc = smc_conn_create(smc, ini);
 699         if (rc) {
 700                 mutex_unlock(&smc_server_lgr_pending);
 701                 return rc;
 702         }
 703
 704         /* Create send and receive buffers */
 705         if (smc_buf_create(smc, true))
 706                 return smc_connect_abort(smc, SMC_CLC_DECL_MEM,
 707                                          ini->cln_first_contact);
 708
 709         smc_conn_save_peer_info(smc, aclc);
 710         smc_close_init(smc);
 711         smc_rx_init(smc);
 712         smc_tx_init(smc);
 713
 714         rc = smc_clc_send_confirm(smc);
 715         if (rc)
 716                 return smc_connect_abort(smc, rc, ini->cln_first_contact);
 717         mutex_unlock(&smc_server_lgr_pending);
 718
 719         smc_copy_sock_settings_to_clc(smc);
 720         smc->connect_nonblock = 0;
 721         if (smc->sk.sk_state == SMC_INIT)
 722                 smc->sk.sk_state = SMC_ACTIVE;
 723
 724         return 0;
 725 }
 726
 727 /* perform steps before actually connecting */
 728 static int __smc_connect(struct smc_sock *smc)
 729 {
 730         bool ism_supported = false, rdma_supported = false;
 731         struct smc_clc_msg_accept_confirm aclc;
 732         struct smc_init_info ini = {0};
 733         int smc_type;
 734         int rc = 0;
 735
 736         if (smc->use_fallback)
 737                 return smc_connect_fallback(smc, smc->fallback_rsn);
 738
 739         /* if peer has not signalled SMC-capability, fall back */
 740         if (!tcp_sk(smc->clcsock->sk)->syn_smc)
 741                 return smc_connect_fallback(smc, SMC_CLC_DECL_PEERNOSMC);
 742
 743         /* IPSec connections opt out of SMC-R optimizations */
 744         if (using_ipsec(smc))
 745                 return smc_connect_decline_fallback(smc, SMC_CLC_DECL_IPSEC);
 746
 747         /* get vlan id from IP device */
 748         if (smc_vlan_by_tcpsk(smc->clcsock, &ini))
 749                 return smc_connect_decline_fallback(smc,
 750                                                     SMC_CLC_DECL_GETVLANERR);
 751
 752         /* check if there is an ism device available */
 753         if (!smc_find_ism_device(smc, &ini) &&
 754             !smc_connect_ism_vlan_setup(smc, &ini)) {
 755                 /* ISM is supported for this connection */
 756                 ism_supported = true;
 757                 smc_type = SMC_TYPE_D;
 758         }
 759
 760         /* check if there is a rdma device available */
 761         if (!smc_find_rdma_device(smc, &ini)) {
 762                 /* RDMA is supported for this connection */
 763                 rdma_supported = true;
 764                 if (ism_supported)
 765                         smc_type = SMC_TYPE_B; /* both */
 766                 else
 767                         smc_type = SMC_TYPE_R; /* only RDMA */
 768         }
 769
 770         /* if neither ISM nor RDMA are supported, fallback */
 771         if (!rdma_supported && !ism_supported)
 772                 return smc_connect_decline_fallback(smc, SMC_CLC_DECL_NOSMCDEV);
 773
 774         /* perform CLC handshake */
 775         rc = smc_connect_clc(smc, smc_type, &aclc, &ini);
 776         if (rc) {
 777                 smc_connect_ism_vlan_cleanup(smc, ism_supported, &ini);
 778                 return smc_connect_decline_fallback(smc, rc);
 779         }
 780
 781         /* depending on previous steps, connect using rdma or ism */
 782         if (rdma_supported && aclc.hdr.path == SMC_TYPE_R)
 783                 rc = smc_connect_rdma(smc, &aclc, &ini);
 784         else if (ism_supported && aclc.hdr.path == SMC_TYPE_D)
 785                 rc = smc_connect_ism(smc, &aclc, &ini);
 786         else
 787                 rc = SMC_CLC_DECL_MODEUNSUPP;
 788         if (rc) {
 789                 smc_connect_ism_vlan_cleanup(smc, ism_supported, &ini);
 790                 return smc_connect_decline_fallback(smc, rc);
 791         }
 792
 793         smc_connect_ism_vlan_cleanup(smc, ism_supported, &ini);
 794         return 0;
 795 }
 796
 797 static void smc_connect_work(struct work_struct *work)
 798 {
 799         struct smc_sock *smc = container_of(work, struct smc_sock,
 800                                             connect_work);
 801         long timeo = smc->sk.sk_sndtimeo;
 802         int rc = 0;
 803
 804         if (!timeo)
 805                 timeo = MAX_SCHEDULE_TIMEOUT;
 806         lock_sock(smc->clcsock->sk);
 807         if (smc->clcsock->sk->sk_err) {
 808                 smc->sk.sk_err = smc->clcsock->sk->sk_err;
 809         } else if ((1 << smc->clcsock->sk->sk_state) &
 810                                         (TCPF_SYN_SENT | TCP_SYN_RECV)) {
 811                 rc = sk_stream_wait_connect(smc->clcsock->sk, &timeo);
 812                 if ((rc == -EPIPE) &&
 813                     ((1 << smc->clcsock->sk->sk_state) &
 814                                         (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)))
 815                         rc = 0;
 816         }
 817         release_sock(smc->clcsock->sk);
 818         lock_sock(&smc->sk);
 819         if (rc != 0 || smc->sk.sk_err) {
 820                 smc->sk.sk_state = SMC_CLOSED;
 821                 if (rc == -EPIPE || rc == -EAGAIN)
 822                         smc->sk.sk_err = EPIPE;
 823                 else if (signal_pending(current))
 824                         smc->sk.sk_err = -sock_intr_errno(timeo);
 825                 sock_put(&smc->sk); /* passive closing */
 826                 goto out;
 827         }
 828
 829         rc = __smc_connect(smc);
 830         if (rc < 0)
 831                 smc->sk.sk_err = -rc;
 832
 833 out:
 834         if (!sock_flag(&smc->sk, SOCK_DEAD)) {
 835                 if (smc->sk.sk_err) {
 836                         smc->sk.sk_state_change(&smc->sk);
 837                 } else { /* allow polling before and after fallback decision */
 838                         smc->clcsock->sk->sk_write_space(smc->clcsock->sk);
 839                         smc->sk.sk_write_space(&smc->sk);
 840                 }
 841         }
 842         release_sock(&smc->sk);
 843 }
 844
 845 static int smc_connect(struct socket *sock, struct sockaddr *addr,
 846                        int alen, int flags)
 847 {
 848         struct sock *sk = sock->sk;
 849         struct smc_sock *smc;
 850         int rc = -EINVAL;
 851
 852         smc = smc_sk(sk);
 853
 854         /* separate smc parameter checking to be safe */
 855         if (alen < sizeof(addr->sa_family))
 856                 goto out_err;
 857         if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6)
 858                 goto out_err;
 859
 860         lock_sock(sk);
 861         switch (sk->sk_state) {
 862         default:
 863                 goto out;
 864         case SMC_ACTIVE:
 865                 rc = -EISCONN;
 866                 goto out;
 867         case SMC_INIT:
 868                 rc = 0;
 869                 break;
 870         }
 871
 872         smc_copy_sock_settings_to_clc(smc);
 873         tcp_sk(smc->clcsock->sk)->syn_smc = 1;
 874         if (smc->connect_nonblock) {
 875                 rc = -EALREADY;
 876                 goto out;
 877         }
 878         rc = kernel_connect(smc->clcsock, addr, alen, flags);
 879         if (rc && rc != -EINPROGRESS)
 880                 goto out;
 881
 882         sock_hold(&smc->sk); /* sock put in passive closing */
 883         if (smc->use_fallback)
 884                 goto out;
 885         if (flags & O_NONBLOCK) {
 886                 if (schedule_work(&smc->connect_work))
 887                         smc->connect_nonblock = 1;
 888                 rc = -EINPROGRESS;
 889         } else {
 890                 rc = __smc_connect(smc);
 891                 if (rc < 0)
 892                         goto out;
 893                 else
 894                         rc = 0; /* success cases including fallback */
 895         }
 896
 897 out:
 898         release_sock(sk);
 899 out_err:
 900         return rc;
 901 }
 902
 903 static int smc_clcsock_accept(struct smc_sock *lsmc, struct smc_sock **new_smc)
 904 {
 905         struct socket *new_clcsock = NULL;
 906         struct sock *lsk = &lsmc->sk;
 907         struct sock *new_sk;
 908         int rc = -EINVAL;
 909
 910         release_sock(lsk);
 911         new_sk = smc_sock_alloc(sock_net(lsk), NULL, lsk->sk_protocol);
 912         if (!new_sk) {
 913                 rc = -ENOMEM;
 914                 lsk->sk_err = ENOMEM;
 915                 *new_smc = NULL;
 916                 lock_sock(lsk);
 917                 goto out;
 918         }
 919         *new_smc = smc_sk(new_sk);
 920
 921         mutex_lock(&lsmc->clcsock_release_lock);
 922         if (lsmc->clcsock)
 923                 rc = kernel_accept(lsmc->clcsock, &new_clcsock, 0);
 924         mutex_unlock(&lsmc->clcsock_release_lock);
 925         lock_sock(lsk);
 926         if  (rc < 0)
 927                 lsk->sk_err = -rc;
 928         if (rc < 0 || lsk->sk_state == SMC_CLOSED) {
 929                 new_sk->sk_prot->unhash(new_sk);
 930                 if (new_clcsock)
 931                         sock_release(new_clcsock);
 932                 new_sk->sk_state = SMC_CLOSED;
 933                 sock_set_flag(new_sk, SOCK_DEAD);
 934                 sock_put(new_sk); /* final */
 935                 *new_smc = NULL;
 936                 goto out;
 937         }
 938
 939         (*new_smc)->clcsock = new_clcsock;
 940 out:
 941         return rc;
 942 }
 943
 944 /* add a just created sock to the accept queue of the listen sock as
 945  * candidate for a following socket accept call from user space
 946  */
 947 static void smc_accept_enqueue(struct sock *parent, struct sock *sk)
 948 {
 949         struct smc_sock *par = smc_sk(parent);
 950
 951         sock_hold(sk); /* sock_put in smc_accept_unlink () */
 952         spin_lock(&par->accept_q_lock);
 953         list_add_tail(&smc_sk(sk)->accept_q, &par->accept_q);
 954         spin_unlock(&par->accept_q_lock);
 955         sk_acceptq_added(parent);
 956 }
 957
 958 /* remove a socket from the accept queue of its parental listening socket */
 959 static void smc_accept_unlink(struct sock *sk)
 960 {
 961         struct smc_sock *par = smc_sk(sk)->listen_smc;
 962
 963         spin_lock(&par->accept_q_lock);
 964         list_del_init(&smc_sk(sk)->accept_q);
 965         spin_unlock(&par->accept_q_lock);
 966         sk_acceptq_removed(&smc_sk(sk)->listen_smc->sk);
 967         sock_put(sk); /* sock_hold in smc_accept_enqueue */
 968 }
 969
 970 /* remove a sock from the accept queue to bind it to a new socket created
 971  * for a socket accept call from user space
 972  */
 973 struct sock *smc_accept_dequeue(struct sock *parent,
 974                                 struct socket *new_sock)
 975 {
 976         struct smc_sock *isk, *n;
 977         struct sock *new_sk;
 978
 979         list_for_each_entry_safe(isk, n, &smc_sk(parent)->accept_q, accept_q) {
 980                 new_sk = (struct sock *)isk;
 981
 982                 smc_accept_unlink(new_sk);
 983                 if (new_sk->sk_state == SMC_CLOSED) {
 984                         new_sk->sk_prot->unhash(new_sk);
 985                         if (isk->clcsock) {
 986                                 sock_release(isk->clcsock);
 987                                 isk->clcsock = NULL;
 988                         }
 989                         sock_put(new_sk); /* final */
 990                         continue;
 991                 }
 992                 if (new_sock) {
 993                         sock_graft(new_sk, new_sock);
 994                         if (isk->use_fallback) {
 995                                 smc_sk(new_sk)->clcsock->file = new_sock->file;
 996                                 isk->clcsock->file->private_data = isk->clcsock;
 997                         }
 998                 }
 999                 return new_sk;
1000         }
1001         return NULL;
1002 }
1003
1004 /* clean up for a created but never accepted sock */
1005 void smc_close_non_accepted(struct sock *sk)
1006 {
1007         struct smc_sock *smc = smc_sk(sk);
1008
1009         sock_hold(sk); /* sock_put below */
1010         lock_sock(sk);
1011         if (!sk->sk_lingertime)
1012                 /* wait for peer closing */
1013                 sk->sk_lingertime = SMC_MAX_STREAM_WAIT_TIMEOUT;
1014         __smc_release(smc);
1015         release_sock(sk);
1016         sock_put(sk); /* sock_hold above */
1017         sock_put(sk); /* final sock_put */
1018 }
1019
1020 static int smcr_serv_conf_first_link(struct smc_sock *smc)
1021 {
1022         struct net *net = sock_net(smc->clcsock->sk);
1023         struct smc_link *link = smc->conn.lnk;
1024         int rest;
1025         int rc;
1026
1027         if (smcr_link_reg_rmb(link, smc->conn.rmb_desc, false))
1028                 return SMC_CLC_DECL_ERR_REGRMB;
1029
1030         /* send CONFIRM LINK request to client over the RoCE fabric */
1031         rc = smc_llc_send_confirm_link(link, SMC_LLC_REQ);
1032         if (rc < 0)
1033                 return SMC_CLC_DECL_TIMEOUT_CL;
1034
1035         /* receive CONFIRM LINK response from client over the RoCE fabric */
1036         rest = wait_for_completion_interruptible_timeout(
1037                 &link->llc_confirm_resp,
1038                 SMC_LLC_WAIT_FIRST_TIME);
1039         if (rest <= 0) {
1040                 struct smc_clc_msg_decline dclc;
1041
1042                 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
1043                                       SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
1044                 return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_CL : rc;
1045         }
1046
1047         if (link->llc_confirm_resp_rc)
1048                 return SMC_CLC_DECL_RMBE_EC;
1049
1050         /* send ADD LINK request to client over the RoCE fabric */
1051         rc = smc_llc_send_add_link(link,
1052                                    link->smcibdev->mac[link->ibport - 1],
1053                                    link->gid, SMC_LLC_REQ);
1054         if (rc < 0)
1055                 return SMC_CLC_DECL_TIMEOUT_AL;
1056
1057         /* receive ADD LINK response from client over the RoCE fabric */
1058         rest = wait_for_completion_interruptible_timeout(&link->llc_add_resp,
1059                                                          SMC_LLC_WAIT_TIME);
1060         if (rest <= 0) {
1061                 struct smc_clc_msg_decline dclc;
1062
1063                 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
1064                                       SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
1065                 return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_AL : rc;
1066         }
1067
1068         smc_llc_link_active(link, net->ipv4.sysctl_tcp_keepalive_time);
1069
1070         return 0;
1071 }
1072
1073 /* listen worker: finish */
1074 static void smc_listen_out(struct smc_sock *new_smc)
1075 {
1076         struct smc_sock *lsmc = new_smc->listen_smc;
1077         struct sock *newsmcsk = &new_smc->sk;
1078
1079         if (lsmc->sk.sk_state == SMC_LISTEN) {
1080                 lock_sock_nested(&lsmc->sk, SINGLE_DEPTH_NESTING);
1081                 smc_accept_enqueue(&lsmc->sk, newsmcsk);
1082                 release_sock(&lsmc->sk);
1083         } else { /* no longer listening */
1084                 smc_close_non_accepted(newsmcsk);
1085         }
1086
1087         /* Wake up accept */
1088         lsmc->sk.sk_data_ready(&lsmc->sk);
1089         sock_put(&lsmc->sk); /* sock_hold in smc_tcp_listen_work */
1090 }
1091
1092 /* listen worker: finish in state connected */
1093 static void smc_listen_out_connected(struct smc_sock *new_smc)
1094 {
1095         struct sock *newsmcsk = &new_smc->sk;
1096
1097         sk_refcnt_debug_inc(newsmcsk);
1098         if (newsmcsk->sk_state == SMC_INIT)
1099                 newsmcsk->sk_state = SMC_ACTIVE;
1100
1101         smc_listen_out(new_smc);
1102 }
1103
1104 /* listen worker: finish in error state */
1105 static void smc_listen_out_err(struct smc_sock *new_smc)
1106 {
1107         struct sock *newsmcsk = &new_smc->sk;
1108
1109         if (newsmcsk->sk_state == SMC_INIT)
1110                 sock_put(&new_smc->sk); /* passive closing */
1111         newsmcsk->sk_state = SMC_CLOSED;
1112
1113         smc_listen_out(new_smc);
1114 }
1115
1116 /* listen worker: decline and fall back if possible */
1117 static void smc_listen_decline(struct smc_sock *new_smc, int reason_code,
1118                                int local_contact)
1119 {
1120         /* RDMA setup failed, switch back to TCP */
1121         if (local_contact == SMC_FIRST_CONTACT)
1122                 smc_lgr_cleanup_early(&new_smc->conn);
1123         else
1124                 smc_conn_free(&new_smc->conn);
1125         if (reason_code < 0) { /* error, no fallback possible */
1126                 smc_listen_out_err(new_smc);
1127                 return;
1128         }
1129         smc_switch_to_fallback(new_smc);
1130         new_smc->fallback_rsn = reason_code;
1131         if (reason_code && reason_code != SMC_CLC_DECL_PEERDECL) {
1132                 if (smc_clc_send_decline(new_smc, reason_code) < 0) {
1133                         smc_listen_out_err(new_smc);
1134                         return;
1135                 }
1136         }
1137         smc_listen_out_connected(new_smc);
1138 }
1139
1140 /* listen worker: check prefixes */
1141 static int smc_listen_prfx_check(struct smc_sock *new_smc,
1142                                  struct smc_clc_msg_proposal *pclc)
1143 {
1144         struct smc_clc_msg_proposal_prefix *pclc_prfx;
1145         struct socket *newclcsock = new_smc->clcsock;
1146
1147         pclc_prfx = smc_clc_proposal_get_prefix(pclc);
1148         if (smc_clc_prfx_match(newclcsock, pclc_prfx))
1149                 return SMC_CLC_DECL_DIFFPREFIX;
1150
1151         return 0;
1152 }
1153
1154 /* listen worker: initialize connection and buffers */
1155 static int smc_listen_rdma_init(struct smc_sock *new_smc,
1156                                 struct smc_init_info *ini)
1157 {
1158         int rc;
1159
1160         /* allocate connection / link group */
1161         rc = smc_conn_create(new_smc, ini);
1162         if (rc)
1163                 return rc;
1164
1165         /* create send buffer and rmb */
1166         if (smc_buf_create(new_smc, false))
1167                 return SMC_CLC_DECL_MEM;
1168
1169         return 0;
1170 }
1171
1172 /* listen worker: initialize connection and buffers for SMC-D */
1173 static int smc_listen_ism_init(struct smc_sock *new_smc,
1174                                struct smc_clc_msg_proposal *pclc,
1175                                struct smc_init_info *ini)
1176 {
1177         struct smc_clc_msg_smcd *pclc_smcd;
1178         int rc;
1179
1180         pclc_smcd = smc_get_clc_msg_smcd(pclc);
1181         ini->ism_gid = pclc_smcd->gid;
1182         rc = smc_conn_create(new_smc, ini);
1183         if (rc)
1184                 return rc;
1185
1186         /* Check if peer can be reached via ISM device */
1187         if (smc_ism_cantalk(new_smc->conn.lgr->peer_gid,
1188                             new_smc->conn.lgr->vlan_id,
1189                             new_smc->conn.lgr->smcd)) {
1190                 if (ini->cln_first_contact == SMC_FIRST_CONTACT)
1191                         smc_lgr_cleanup_early(&new_smc->conn);
1192                 else
1193                         smc_conn_free(&new_smc->conn);
1194                 return SMC_CLC_DECL_SMCDNOTALK;
1195         }
1196
1197         /* Create send and receive buffers */
1198         if (smc_buf_create(new_smc, true)) {
1199                 if (ini->cln_first_contact == SMC_FIRST_CONTACT)
1200                         smc_lgr_cleanup_early(&new_smc->conn);
1201                 else
1202                         smc_conn_free(&new_smc->conn);
1203                 return SMC_CLC_DECL_MEM;
1204         }
1205
1206         return 0;
1207 }
1208
1209 /* listen worker: register buffers */
1210 static int smc_listen_rdma_reg(struct smc_sock *new_smc, int local_contact)
1211 {
1212         struct smc_connection *conn = &new_smc->conn;
1213
1214         if (local_contact != SMC_FIRST_CONTACT) {
1215                 if (smcr_lgr_reg_rmbs(conn->lgr, conn->rmb_desc))
1216                         return SMC_CLC_DECL_ERR_REGRMB;
1217         }
1218         smc_rmb_sync_sg_for_device(&new_smc->conn);
1219
1220         return 0;
1221 }
1222
1223 /* listen worker: finish RDMA setup */
1224 static int smc_listen_rdma_finish(struct smc_sock *new_smc,
1225                                   struct smc_clc_msg_accept_confirm *cclc,
1226                                   int local_contact)
1227 {
1228         struct smc_link *link = new_smc->conn.lnk;
1229         int reason_code = 0;
1230
1231         if (local_contact == SMC_FIRST_CONTACT)
1232                 smc_link_save_peer_info(link, cclc);
1233
1234         if (smc_rmb_rtoken_handling(&new_smc->conn, cclc)) {
1235                 reason_code = SMC_CLC_DECL_ERR_RTOK;
1236                 goto decline;
1237         }
1238
1239         if (local_contact == SMC_FIRST_CONTACT) {
1240                 if (smc_ib_ready_link(link)) {
1241                         reason_code = SMC_CLC_DECL_ERR_RDYLNK;
1242                         goto decline;
1243                 }
1244                 /* QP confirmation over RoCE fabric */
1245                 reason_code = smcr_serv_conf_first_link(new_smc);
1246                 if (reason_code)
1247                         goto decline;
1248         }
1249         return 0;
1250
1251 decline:
1252         smc_listen_decline(new_smc, reason_code, local_contact);
1253         return reason_code;
1254 }
1255
1256 /* setup for RDMA connection of server */
1257 static void smc_listen_work(struct work_struct *work)
1258 {
1259         struct smc_sock *new_smc = container_of(work, struct smc_sock,
1260                                                 smc_listen_work);
1261         struct socket *newclcsock = new_smc->clcsock;
1262         struct smc_clc_msg_accept_confirm cclc;
1263         struct smc_clc_msg_proposal *pclc;
1264         struct smc_init_info ini = {0};
1265         bool ism_supported = false;
1266         u8 buf[SMC_CLC_MAX_LEN];
1267         int rc = 0;
1268
1269         if (new_smc->listen_smc->sk.sk_state != SMC_LISTEN)
1270                 return smc_listen_out_err(new_smc);
1271
1272         if (new_smc->use_fallback) {
1273                 smc_listen_out_connected(new_smc);
1274                 return;
1275         }
1276
1277         /* check if peer is smc capable */
1278         if (!tcp_sk(newclcsock->sk)->syn_smc) {
1279                 smc_switch_to_fallback(new_smc);
1280                 new_smc->fallback_rsn = SMC_CLC_DECL_PEERNOSMC;
1281                 smc_listen_out_connected(new_smc);
1282                 return;
1283         }
1284
1285         /* do inband token exchange -
1286          * wait for and receive SMC Proposal CLC message
1287          */
1288         pclc = (struct smc_clc_msg_proposal *)&buf;
1289         rc = smc_clc_wait_msg(new_smc, pclc, SMC_CLC_MAX_LEN,
1290                               SMC_CLC_PROPOSAL, CLC_WAIT_TIME);
1291         if (rc)
1292                 goto out_decl;
1293
1294         /* IPSec connections opt out of SMC-R optimizations */
1295         if (using_ipsec(new_smc)) {
1296                 rc = SMC_CLC_DECL_IPSEC;
1297                 goto out_decl;
1298         }
1299
1300         /* check for matching IP prefix and subnet length */
1301         rc = smc_listen_prfx_check(new_smc, pclc);
1302         if (rc)
1303                 goto out_decl;
1304
1305         /* get vlan id from IP device */
1306         if (smc_vlan_by_tcpsk(new_smc->clcsock, &ini)) {
1307                 rc = SMC_CLC_DECL_GETVLANERR;
1308                 goto out_decl;
1309         }
1310
1311         mutex_lock(&smc_server_lgr_pending);
1312         smc_close_init(new_smc);
1313         smc_rx_init(new_smc);
1314         smc_tx_init(new_smc);
1315
1316         /* check if ISM is available */
1317         if (pclc->hdr.path == SMC_TYPE_D || pclc->hdr.path == SMC_TYPE_B) {
1318                 ini.is_smcd = true; /* prepare ISM check */
1319                 rc = smc_find_ism_device(new_smc, &ini);
1320                 if (!rc)
1321                         rc = smc_listen_ism_init(new_smc, pclc, &ini);
1322                 if (!rc)
1323                         ism_supported = true;
1324                 else if (pclc->hdr.path == SMC_TYPE_D)
1325                         goto out_unlock; /* skip RDMA and decline */
1326         }
1327
1328         /* check if RDMA is available */
1329         if (!ism_supported) { /* SMC_TYPE_R or SMC_TYPE_B */
1330                 /* prepare RDMA check */
1331                 ini.is_smcd = false;
1332                 ini.ism_dev = NULL;
1333                 ini.ib_lcl = &pclc->lcl;
1334                 rc = smc_find_rdma_device(new_smc, &ini);
1335                 if (rc) {
1336                         /* no RDMA device found */
1337                         if (pclc->hdr.path == SMC_TYPE_B)
1338                                 /* neither ISM nor RDMA device found */
1339                                 rc = SMC_CLC_DECL_NOSMCDEV;
1340                         goto out_unlock;
1341                 }
1342                 rc = smc_listen_rdma_init(new_smc, &ini);
1343                 if (rc)
1344                         goto out_unlock;
1345                 rc = smc_listen_rdma_reg(new_smc, ini.cln_first_contact);
1346                 if (rc)
1347                         goto out_unlock;
1348         }
1349
1350         /* send SMC Accept CLC message */
1351         rc = smc_clc_send_accept(new_smc, ini.cln_first_contact);
1352         if (rc)
1353                 goto out_unlock;
1354
1355         /* SMC-D does not need this lock any more */
1356         if (ism_supported)
1357                 mutex_unlock(&smc_server_lgr_pending);
1358
1359         /* receive SMC Confirm CLC message */
1360         rc = smc_clc_wait_msg(new_smc, &cclc, sizeof(cclc),
1361                               SMC_CLC_CONFIRM, CLC_WAIT_TIME);
1362         if (rc) {
1363                 if (!ism_supported)
1364                         goto out_unlock;
1365                 goto out_decl;
1366         }
1367
1368         /* finish worker */
1369         if (!ism_supported) {
1370                 rc = smc_listen_rdma_finish(new_smc, &cclc,
1371                                             ini.cln_first_contact);
1372                 mutex_unlock(&smc_server_lgr_pending);
1373                 if (rc)
1374                         return;
1375         }
1376         smc_conn_save_peer_info(new_smc, &cclc);
1377         smc_listen_out_connected(new_smc);
1378         return;
1379
1380 out_unlock:
1381         mutex_unlock(&smc_server_lgr_pending);
1382 out_decl:
1383         smc_listen_decline(new_smc, rc, ini.cln_first_contact);
1384 }
1385
1386 static void smc_tcp_listen_work(struct work_struct *work)
1387 {
1388         struct smc_sock *lsmc = container_of(work, struct smc_sock,
1389                                              tcp_listen_work);
1390         struct sock *lsk = &lsmc->sk;
1391         struct smc_sock *new_smc;
1392         int rc = 0;
1393
1394         lock_sock(lsk);
1395         while (lsk->sk_state == SMC_LISTEN) {
1396                 rc = smc_clcsock_accept(lsmc, &new_smc);
1397                 if (rc)
1398                         goto out;
1399                 if (!new_smc)
1400                         continue;
1401
1402                 new_smc->listen_smc = lsmc;
1403                 new_smc->use_fallback = lsmc->use_fallback;
1404                 new_smc->fallback_rsn = lsmc->fallback_rsn;
1405                 sock_hold(lsk); /* sock_put in smc_listen_work */
1406                 INIT_WORK(&new_smc->smc_listen_work, smc_listen_work);
1407                 smc_copy_sock_settings_to_smc(new_smc);
1408                 new_smc->sk.sk_sndbuf = lsmc->sk.sk_sndbuf;
1409                 new_smc->sk.sk_rcvbuf = lsmc->sk.sk_rcvbuf;
1410                 sock_hold(&new_smc->sk); /* sock_put in passive closing */
1411                 if (!schedule_work(&new_smc->smc_listen_work))
1412                         sock_put(&new_smc->sk);
1413         }
1414
1415 out:
1416         release_sock(lsk);
1417         sock_put(&lsmc->sk); /* sock_hold in smc_listen */
1418 }
1419
1420 static int smc_listen(struct socket *sock, int backlog)
1421 {
1422         struct sock *sk = sock->sk;
1423         struct smc_sock *smc;
1424         int rc;
1425
1426         smc = smc_sk(sk);
1427         lock_sock(sk);
1428
1429         rc = -EINVAL;
1430         if ((sk->sk_state != SMC_INIT && sk->sk_state != SMC_LISTEN) ||
1431             smc->connect_nonblock)
1432                 goto out;
1433
1434         rc = 0;
1435         if (sk->sk_state == SMC_LISTEN) {
1436                 sk->sk_max_ack_backlog = backlog;
1437                 goto out;
1438         }
1439         /* some socket options are handled in core, so we could not apply
1440          * them to the clc socket -- copy smc socket options to clc socket
1441          */
1442         smc_copy_sock_settings_to_clc(smc);
1443         if (!smc->use_fallback)
1444                 tcp_sk(smc->clcsock->sk)->syn_smc = 1;
1445
1446         rc = kernel_listen(smc->clcsock, backlog);
1447         if (rc)
1448                 goto out;
1449         sk->sk_max_ack_backlog = backlog;
1450         sk->sk_ack_backlog = 0;
1451         sk->sk_state = SMC_LISTEN;
1452         sock_hold(sk); /* sock_hold in tcp_listen_worker */
1453         if (!schedule_work(&smc->tcp_listen_work))
1454                 sock_put(sk);
1455
1456 out:
1457         release_sock(sk);
1458         return rc;
1459 }
1460
1461 static int smc_accept(struct socket *sock, struct socket *new_sock,
1462                       int flags, bool kern)
1463 {
1464         struct sock *sk = sock->sk, *nsk;
1465         DECLARE_WAITQUEUE(wait, current);
1466         struct smc_sock *lsmc;
1467         long timeo;
1468         int rc = 0;
1469
1470         lsmc = smc_sk(sk);
1471         sock_hold(sk); /* sock_put below */
1472         lock_sock(sk);
1473
1474         if (lsmc->sk.sk_state != SMC_LISTEN) {
1475                 rc = -EINVAL;
1476                 release_sock(sk);
1477                 goto out;
1478         }
1479
1480         /* Wait for an incoming connection */
1481         timeo = sock_rcvtimeo(sk, flags & O_NONBLOCK);
1482         add_wait_queue_exclusive(sk_sleep(sk), &wait);
1483         while (!(nsk = smc_accept_dequeue(sk, new_sock))) {
1484                 set_current_state(TASK_INTERRUPTIBLE);
1485                 if (!timeo) {
1486                         rc = -EAGAIN;
1487                         break;
1488                 }
1489                 release_sock(sk);
1490                 timeo = schedule_timeout(timeo);
1491                 /* wakeup by sk_data_ready in smc_listen_work() */
1492                 sched_annotate_sleep();
1493                 lock_sock(sk);
1494                 if (signal_pending(current)) {
1495                         rc = sock_intr_errno(timeo);
1496                         break;
1497                 }
1498         }
1499         set_current_state(TASK_RUNNING);
1500         remove_wait_queue(sk_sleep(sk), &wait);
1501
1502         if (!rc)
1503                 rc = sock_error(nsk);
1504         release_sock(sk);
1505         if (rc)
1506                 goto out;
1507
1508         if (lsmc->sockopt_defer_accept && !(flags & O_NONBLOCK)) {
1509                 /* wait till data arrives on the socket */
1510                 timeo = msecs_to_jiffies(lsmc->sockopt_defer_accept *
1511                                                                 MSEC_PER_SEC);
1512                 if (smc_sk(nsk)->use_fallback) {
1513                         struct sock *clcsk = smc_sk(nsk)->clcsock->sk;
1514
1515                         lock_sock(clcsk);
1516                         if (skb_queue_empty(&clcsk->sk_receive_queue))
1517                                 sk_wait_data(clcsk, &timeo, NULL);
1518                         release_sock(clcsk);
1519                 } else if (!atomic_read(&smc_sk(nsk)->conn.bytes_to_rcv)) {
1520                         lock_sock(nsk);
1521                         smc_rx_wait(smc_sk(nsk), &timeo, smc_rx_data_available);
1522                         release_sock(nsk);
1523                 }
1524         }
1525
1526 out:
1527         sock_put(sk); /* sock_hold above */
1528         return rc;
1529 }
1530
1531 static int smc_getname(struct socket *sock, struct sockaddr *addr,
1532                        int peer)
1533 {
1534         struct smc_sock *smc;
1535
1536         if (peer && (sock->sk->sk_state != SMC_ACTIVE) &&
1537             (sock->sk->sk_state != SMC_APPCLOSEWAIT1))
1538                 return -ENOTCONN;
1539
1540         smc = smc_sk(sock->sk);
1541
1542         return smc->clcsock->ops->getname(smc->clcsock, addr, peer);
1543 }
1544
1545 static int smc_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
1546 {
1547         struct sock *sk = sock->sk;
1548         struct smc_sock *smc;
1549         int rc = -EPIPE;
1550
1551         smc = smc_sk(sk);
1552         lock_sock(sk);
1553         if ((sk->sk_state != SMC_ACTIVE) &&
1554             (sk->sk_state != SMC_APPCLOSEWAIT1) &&
1555             (sk->sk_state != SMC_INIT))
1556                 goto out;
1557
1558         if (msg->msg_flags & MSG_FASTOPEN) {
1559                 if (sk->sk_state == SMC_INIT && !smc->connect_nonblock) {
1560                         smc_switch_to_fallback(smc);
1561                         smc->fallback_rsn = SMC_CLC_DECL_OPTUNSUPP;
1562                 } else {
1563                         rc = -EINVAL;
1564                         goto out;
1565                 }
1566         }
1567
1568         if (smc->use_fallback)
1569                 rc = smc->clcsock->ops->sendmsg(smc->clcsock, msg, len);
1570         else
1571                 rc = smc_tx_sendmsg(smc, msg, len);
1572 out:
1573         release_sock(sk);
1574         return rc;
1575 }
1576
1577 static int smc_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
1578                        int flags)
1579 {
1580         struct sock *sk = sock->sk;
1581         struct smc_sock *smc;
1582         int rc = -ENOTCONN;
1583
1584         smc = smc_sk(sk);
1585         lock_sock(sk);
1586         if (sk->sk_state == SMC_CLOSED && (sk->sk_shutdown & RCV_SHUTDOWN)) {
1587                 /* socket was connected before, no more data to read */
1588                 rc = 0;
1589                 goto out;
1590         }
1591         if ((sk->sk_state == SMC_INIT) ||
1592             (sk->sk_state == SMC_LISTEN) ||
1593             (sk->sk_state == SMC_CLOSED))
1594                 goto out;
1595
1596         if (sk->sk_state == SMC_PEERFINCLOSEWAIT) {
1597                 rc = 0;
1598                 goto out;
1599         }
1600
1601         if (smc->use_fallback) {
1602                 rc = smc->clcsock->ops->recvmsg(smc->clcsock, msg, len, flags);
1603         } else {
1604                 msg->msg_namelen = 0;
1605                 rc = smc_rx_recvmsg(smc, msg, NULL, len, flags);
1606         }
1607
1608 out:
1609         release_sock(sk);
1610         return rc;
1611 }
1612
1613 static __poll_t smc_accept_poll(struct sock *parent)
1614 {
1615         struct smc_sock *isk = smc_sk(parent);
1616         __poll_t mask = 0;
1617
1618         spin_lock(&isk->accept_q_lock);
1619         if (!list_empty(&isk->accept_q))
1620                 mask = EPOLLIN | EPOLLRDNORM;
1621         spin_unlock(&isk->accept_q_lock);
1622
1623         return mask;
1624 }
1625
1626 static __poll_t smc_poll(struct file *file, struct socket *sock,
1627                              poll_table *wait)
1628 {
1629         struct sock *sk = sock->sk;
1630         struct smc_sock *smc;
1631         __poll_t mask = 0;
1632
1633         if (!sk)
1634                 return EPOLLNVAL;
1635
1636         smc = smc_sk(sock->sk);
1637         if (smc->use_fallback) {
1638                 /* delegate to CLC child sock */
1639                 mask = smc->clcsock->ops->poll(file, smc->clcsock, wait);
1640                 sk->sk_err = smc->clcsock->sk->sk_err;
1641         } else {
1642                 if (sk->sk_state != SMC_CLOSED)
1643                         sock_poll_wait(file, sock, wait);
1644                 if (sk->sk_err)
1645                         mask |= EPOLLERR;
1646                 if ((sk->sk_shutdown == SHUTDOWN_MASK) ||
1647                     (sk->sk_state == SMC_CLOSED))
1648                         mask |= EPOLLHUP;
1649                 if (sk->sk_state == SMC_LISTEN) {
1650                         /* woken up by sk_data_ready in smc_listen_work() */
1651                         mask |= smc_accept_poll(sk);
1652                 } else if (smc->use_fallback) { /* as result of connect_work()*/
1653                         mask |= smc->clcsock->ops->poll(file, smc->clcsock,
1654                                                            wait);
1655                         sk->sk_err = smc->clcsock->sk->sk_err;
1656                 } else {
1657                         if ((sk->sk_state != SMC_INIT &&
1658                              atomic_read(&smc->conn.sndbuf_space)) ||
1659                             sk->sk_shutdown & SEND_SHUTDOWN) {
1660                                 mask |= EPOLLOUT | EPOLLWRNORM;
1661                         } else {
1662                                 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
1663                                 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
1664                         }
1665                         if (atomic_read(&smc->conn.bytes_to_rcv))
1666                                 mask |= EPOLLIN | EPOLLRDNORM;
1667                         if (sk->sk_shutdown & RCV_SHUTDOWN)
1668                                 mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
1669                         if (sk->sk_state == SMC_APPCLOSEWAIT1)
1670                                 mask |= EPOLLIN;
1671                         if (smc->conn.urg_state == SMC_URG_VALID)
1672                                 mask |= EPOLLPRI;
1673                 }
1674         }
1675
1676         return mask;
1677 }
1678
1679 static int smc_shutdown(struct socket *sock, int how)
1680 {
1681         struct sock *sk = sock->sk;
1682         struct smc_sock *smc;
1683         int rc = -EINVAL;
1684         int rc1 = 0;
1685
1686         smc = smc_sk(sk);
1687
1688         if ((how < SHUT_RD) || (how > SHUT_RDWR))
1689                 return rc;
1690
1691         lock_sock(sk);
1692
1693         rc = -ENOTCONN;
1694         if ((sk->sk_state != SMC_ACTIVE) &&
1695             (sk->sk_state != SMC_PEERCLOSEWAIT1) &&
1696             (sk->sk_state != SMC_PEERCLOSEWAIT2) &&
1697             (sk->sk_state != SMC_APPCLOSEWAIT1) &&
1698             (sk->sk_state != SMC_APPCLOSEWAIT2) &&
1699             (sk->sk_state != SMC_APPFINCLOSEWAIT))
1700                 goto out;
1701         if (smc->use_fallback) {
1702                 rc = kernel_sock_shutdown(smc->clcsock, how);
1703                 sk->sk_shutdown = smc->clcsock->sk->sk_shutdown;
1704                 if (sk->sk_shutdown == SHUTDOWN_MASK)
1705                         sk->sk_state = SMC_CLOSED;
1706                 goto out;
1707         }
1708         switch (how) {
1709         case SHUT_RDWR:         /* shutdown in both directions */
1710                 rc = smc_close_active(smc);
1711                 break;
1712         case SHUT_WR:
1713                 rc = smc_close_shutdown_write(smc);
1714                 break;
1715         case SHUT_RD:
1716                 rc = 0;
1717                 /* nothing more to do because peer is not involved */
1718                 break;
1719         }
1720         if (smc->clcsock)
1721                 rc1 = kernel_sock_shutdown(smc->clcsock, how);
1722         /* map sock_shutdown_cmd constants to sk_shutdown value range */
1723         sk->sk_shutdown |= how + 1;
1724
1725 out:
1726         release_sock(sk);
1727         return rc ? rc : rc1;
1728 }
1729
1730 static int smc_setsockopt(struct socket *sock, int level, int optname,
1731                           char __user *optval, unsigned int optlen)
1732 {
1733         struct sock *sk = sock->sk;
1734         struct smc_sock *smc;
1735         int val, rc;
1736
1737         smc = smc_sk(sk);
1738
1739         /* generic setsockopts reaching us here always apply to the
1740          * CLC socket
1741          */
1742         rc = smc->clcsock->ops->setsockopt(smc->clcsock, level, optname,
1743                                            optval, optlen);
1744         if (smc->clcsock->sk->sk_err) {
1745                 sk->sk_err = smc->clcsock->sk->sk_err;
1746                 sk->sk_error_report(sk);
1747         }
1748
1749         if (optlen < sizeof(int))
1750                 return -EINVAL;
1751         if (get_user(val, (int __user *)optval))
1752                 return -EFAULT;
1753
1754         lock_sock(sk);
1755         if (rc || smc->use_fallback)
1756                 goto out;
1757         switch (optname) {
1758         case TCP_ULP:
1759         case TCP_FASTOPEN:
1760         case TCP_FASTOPEN_CONNECT:
1761         case TCP_FASTOPEN_KEY:
1762         case TCP_FASTOPEN_NO_COOKIE:
1763                 /* option not supported by SMC */
1764                 if (sk->sk_state == SMC_INIT && !smc->connect_nonblock) {
1765                         smc_switch_to_fallback(smc);
1766                         smc->fallback_rsn = SMC_CLC_DECL_OPTUNSUPP;
1767                 } else {
1768                         rc = -EINVAL;
1769                 }
1770                 break;
1771         case TCP_NODELAY:
1772                 if (sk->sk_state != SMC_INIT &&
1773                     sk->sk_state != SMC_LISTEN &&
1774                     sk->sk_state != SMC_CLOSED) {
1775                         if (val)
1776                                 mod_delayed_work(system_wq, &smc->conn.tx_work,
1777                                                  0);
1778                 }
1779                 break;
1780         case TCP_CORK:
1781                 if (sk->sk_state != SMC_INIT &&
1782                     sk->sk_state != SMC_LISTEN &&
1783                     sk->sk_state != SMC_CLOSED) {
1784                         if (!val)
1785                                 mod_delayed_work(system_wq, &smc->conn.tx_work,
1786                                                  0);
1787                 }
1788                 break;
1789         case TCP_DEFER_ACCEPT:
1790                 smc->sockopt_defer_accept = val;
1791                 break;
1792         default:
1793                 break;
1794         }
1795 out:
1796         release_sock(sk);
1797
1798         return rc;
1799 }
1800
1801 static int smc_getsockopt(struct socket *sock, int level, int optname,
1802                           char __user *optval, int __user *optlen)
1803 {
1804         struct smc_sock *smc;
1805
1806         smc = smc_sk(sock->sk);
1807         /* socket options apply to the CLC socket */
1808         return smc->clcsock->ops->getsockopt(smc->clcsock, level, optname,
1809                                              optval, optlen);
1810 }
1811
1812 static int smc_ioctl(struct socket *sock, unsigned int cmd,
1813                      unsigned long arg)
1814 {
1815         union smc_host_cursor cons, urg;
1816         struct smc_connection *conn;
1817         struct smc_sock *smc;
1818         int answ;
1819
1820         smc = smc_sk(sock->sk);
1821         conn = &smc->conn;
1822         lock_sock(&smc->sk);
1823         if (smc->use_fallback) {
1824                 if (!smc->clcsock) {
1825                         release_sock(&smc->sk);
1826                         return -EBADF;
1827                 }
1828                 answ = smc->clcsock->ops->ioctl(smc->clcsock, cmd, arg);
1829                 release_sock(&smc->sk);
1830                 return answ;
1831         }
1832         switch (cmd) {
1833         case SIOCINQ: /* same as FIONREAD */
1834                 if (smc->sk.sk_state == SMC_LISTEN) {
1835                         release_sock(&smc->sk);
1836                         return -EINVAL;
1837                 }
1838                 if (smc->sk.sk_state == SMC_INIT ||
1839                     smc->sk.sk_state == SMC_CLOSED)
1840                         answ = 0;
1841                 else
1842                         answ = atomic_read(&smc->conn.bytes_to_rcv);
1843                 break;
1844         case SIOCOUTQ:
1845                 /* output queue size (not send + not acked) */
1846                 if (smc->sk.sk_state == SMC_LISTEN) {
1847                         release_sock(&smc->sk);
1848                         return -EINVAL;
1849                 }
1850                 if (smc->sk.sk_state == SMC_INIT ||
1851                     smc->sk.sk_state == SMC_CLOSED)
1852                         answ = 0;
1853                 else
1854                         answ = smc->conn.sndbuf_desc->len -
1855                                         atomic_read(&smc->conn.sndbuf_space);
1856                 break;
1857         case SIOCOUTQNSD:
1858                 /* output queue size (not send only) */
1859                 if (smc->sk.sk_state == SMC_LISTEN) {
1860                         release_sock(&smc->sk);
1861                         return -EINVAL;
1862                 }
1863                 if (smc->sk.sk_state == SMC_INIT ||
1864                     smc->sk.sk_state == SMC_CLOSED)
1865                         answ = 0;
1866                 else
1867                         answ = smc_tx_prepared_sends(&smc->conn);
1868                 break;
1869         case SIOCATMARK:
1870                 if (smc->sk.sk_state == SMC_LISTEN) {
1871                         release_sock(&smc->sk);
1872                         return -EINVAL;
1873                 }
1874                 if (smc->sk.sk_state == SMC_INIT ||
1875                     smc->sk.sk_state == SMC_CLOSED) {
1876                         answ = 0;
1877                 } else {
1878                         smc_curs_copy(&cons, &conn->local_tx_ctrl.cons, conn);
1879                         smc_curs_copy(&urg, &conn->urg_curs, conn);
1880                         answ = smc_curs_diff(conn->rmb_desc->len,
1881                                              &cons, &urg) == 1;
1882                 }
1883                 break;
1884         default:
1885                 release_sock(&smc->sk);
1886                 return -ENOIOCTLCMD;
1887         }
1888         release_sock(&smc->sk);
1889
1890         return put_user(answ, (int __user *)arg);
1891 }
1892
1893 static ssize_t smc_sendpage(struct socket *sock, struct page *page,
1894                             int offset, size_t size, int flags)
1895 {
1896         struct sock *sk = sock->sk;
1897         struct smc_sock *smc;
1898         int rc = -EPIPE;
1899
1900         smc = smc_sk(sk);
1901         lock_sock(sk);
1902         if (sk->sk_state != SMC_ACTIVE) {
1903                 release_sock(sk);
1904                 goto out;
1905         }
1906         release_sock(sk);
1907         if (smc->use_fallback)
1908                 rc = kernel_sendpage(smc->clcsock, page, offset,
1909                                      size, flags);
1910         else
1911                 rc = sock_no_sendpage(sock, page, offset, size, flags);
1912
1913 out:
1914         return rc;
1915 }
1916
1917 /* Map the affected portions of the rmbe into an spd, note the number of bytes
1918  * to splice in conn->splice_pending, and press 'go'. Delays consumer cursor
1919  * updates till whenever a respective page has been fully processed.
1920  * Note that subsequent recv() calls have to wait till all splice() processing
1921  * completed.
1922  */
1923 static ssize_t smc_splice_read(struct socket *sock, loff_t *ppos,
1924                                struct pipe_inode_info *pipe, size_t len,
1925                                unsigned int flags)
1926 {
1927         struct sock *sk = sock->sk;
1928         struct smc_sock *smc;
1929         int rc = -ENOTCONN;
1930
1931         smc = smc_sk(sk);
1932         lock_sock(sk);
1933         if (sk->sk_state == SMC_CLOSED && (sk->sk_shutdown & RCV_SHUTDOWN)) {
1934                 /* socket was connected before, no more data to read */
1935                 rc = 0;
1936                 goto out;
1937         }
1938         if (sk->sk_state == SMC_INIT ||
1939             sk->sk_state == SMC_LISTEN ||
1940             sk->sk_state == SMC_CLOSED)
1941                 goto out;
1942
1943         if (sk->sk_state == SMC_PEERFINCLOSEWAIT) {
1944                 rc = 0;
1945                 goto out;
1946         }
1947
1948         if (smc->use_fallback) {
1949                 rc = smc->clcsock->ops->splice_read(smc->clcsock, ppos,
1950                                                     pipe, len, flags);
1951         } else {
1952                 if (*ppos) {
1953                         rc = -ESPIPE;
1954                         goto out;
1955                 }
1956                 if (flags & SPLICE_F_NONBLOCK)
1957                         flags = MSG_DONTWAIT;
1958                 else
1959                         flags = 0;
1960                 rc = smc_rx_recvmsg(smc, NULL, pipe, len, flags);
1961         }
1962 out:
1963         release_sock(sk);
1964
1965         return rc;
1966 }
1967
1968 /* must look like tcp */
1969 static const struct proto_ops smc_sock_ops = {
1970         .family         = PF_SMC,
1971         .owner          = THIS_MODULE,
1972         .release        = smc_release,
1973         .bind           = smc_bind,
1974         .connect        = smc_connect,
1975         .socketpair     = sock_no_socketpair,
1976         .accept         = smc_accept,
1977         .getname        = smc_getname,
1978         .poll           = smc_poll,
1979         .ioctl          = smc_ioctl,
1980         .listen         = smc_listen,
1981         .shutdown       = smc_shutdown,
1982         .setsockopt     = smc_setsockopt,
1983         .getsockopt     = smc_getsockopt,
1984         .sendmsg        = smc_sendmsg,
1985         .recvmsg        = smc_recvmsg,
1986         .mmap           = sock_no_mmap,
1987         .sendpage       = smc_sendpage,
1988         .splice_read    = smc_splice_read,
1989 };
1990
1991 static int smc_create(struct net *net, struct socket *sock, int protocol,
1992                       int kern)
1993 {
1994         int family = (protocol == SMCPROTO_SMC6) ? PF_INET6 : PF_INET;
1995         struct smc_sock *smc;
1996         struct sock *sk;
1997         int rc;
1998
1999         rc = -ESOCKTNOSUPPORT;
2000         if (sock->type != SOCK_STREAM)
2001                 goto out;
2002
2003         rc = -EPROTONOSUPPORT;
2004         if (protocol != SMCPROTO_SMC && protocol != SMCPROTO_SMC6)
2005                 goto out;
2006
2007         rc = -ENOBUFS;
2008         sock->ops = &smc_sock_ops;
2009         sk = smc_sock_alloc(net, sock, protocol);
2010         if (!sk)
2011                 goto out;
2012
2013         /* create internal TCP socket for CLC handshake and fallback */
2014         smc = smc_sk(sk);
2015         smc->use_fallback = false; /* assume rdma capability first */
2016         smc->fallback_rsn = 0;
2017         rc = sock_create_kern(net, family, SOCK_STREAM, IPPROTO_TCP,
2018                               &smc->clcsock);
2019         if (rc) {
2020                 sk_common_release(sk);
2021                 goto out;
2022         }
2023         smc->sk.sk_sndbuf = max(smc->clcsock->sk->sk_sndbuf, SMC_BUF_MIN_SIZE);
2024         smc->sk.sk_rcvbuf = max(smc->clcsock->sk->sk_rcvbuf, SMC_BUF_MIN_SIZE);
2025
2026 out:
2027         return rc;
2028 }
2029
2030 static const struct net_proto_family smc_sock_family_ops = {
2031         .family = PF_SMC,
2032         .owner  = THIS_MODULE,
2033         .create = smc_create,
2034 };
2035
2036 unsigned int smc_net_id;
2037
2038 static __net_init int smc_net_init(struct net *net)
2039 {
2040         return smc_pnet_net_init(net);
2041 }
2042
2043 static void __net_exit smc_net_exit(struct net *net)
2044 {
2045         smc_pnet_net_exit(net);
2046 }
2047
2048 static struct pernet_operations smc_net_ops = {
2049         .init = smc_net_init,
2050         .exit = smc_net_exit,
2051         .id   = &smc_net_id,
2052         .size = sizeof(struct smc_net),
2053 };
2054
2055 static int __init smc_init(void)
2056 {
2057         int rc;
2058
2059         rc = register_pernet_subsys(&smc_net_ops);
2060         if (rc)
2061                 return rc;
2062
2063         rc = smc_pnet_init();
2064         if (rc)
2065                 goto out_pernet_subsys;
2066
2067         rc = smc_core_init();
2068         if (rc) {
2069                 pr_err("%s: smc_core_init fails with %d\n", __func__, rc);
2070                 goto out_pnet;
2071         }
2072
2073         rc = smc_llc_init();
2074         if (rc) {
2075                 pr_err("%s: smc_llc_init fails with %d\n", __func__, rc);
2076                 goto out_core;
2077         }
2078
2079         rc = smc_cdc_init();
2080         if (rc) {
2081                 pr_err("%s: smc_cdc_init fails with %d\n", __func__, rc);
2082                 goto out_core;
2083         }
2084
2085         rc = proto_register(&smc_proto, 1);
2086         if (rc) {
2087                 pr_err("%s: proto_register(v4) fails with %d\n", __func__, rc);
2088                 goto out_core;
2089         }
2090
2091         rc = proto_register(&smc_proto6, 1);
2092         if (rc) {
2093                 pr_err("%s: proto_register(v6) fails with %d\n", __func__, rc);
2094                 goto out_proto;
2095         }
2096
2097         rc = sock_register(&smc_sock_family_ops);
2098         if (rc) {
2099                 pr_err("%s: sock_register fails with %d\n", __func__, rc);
2100                 goto out_proto6;
2101         }
2102         INIT_HLIST_HEAD(&smc_v4_hashinfo.ht);
2103         INIT_HLIST_HEAD(&smc_v6_hashinfo.ht);
2104
2105         rc = smc_ib_register_client();
2106         if (rc) {
2107                 pr_err("%s: ib_register fails with %d\n", __func__, rc);
2108                 goto out_sock;
2109         }
2110
2111         static_branch_enable(&tcp_have_smc);
2112         return 0;
2113
2114 out_sock:
2115         sock_unregister(PF_SMC);
2116 out_proto6:
2117         proto_unregister(&smc_proto6);
2118 out_proto:
2119         proto_unregister(&smc_proto);
2120 out_core:
2121         smc_core_exit();
2122 out_pnet:
2123         smc_pnet_exit();
2124 out_pernet_subsys:
2125         unregister_pernet_subsys(&smc_net_ops);
2126
2127         return rc;
2128 }
2129
2130 static void __exit smc_exit(void)
2131 {
2132         static_branch_disable(&tcp_have_smc);
2133         sock_unregister(PF_SMC);
2134         smc_core_exit();
2135         smc_ib_unregister_client();
2136         proto_unregister(&smc_proto6);
2137         proto_unregister(&smc_proto);
2138         smc_pnet_exit();
2139         unregister_pernet_subsys(&smc_net_ops);
2140         rcu_barrier();
2141 }
2142
2143 module_init(smc_init);
2144 module_exit(smc_exit);
2145
2146 MODULE_AUTHOR("Ursula Braun <ubraun@linux.vnet.ibm.com>");
2147 MODULE_DESCRIPTION("smc socket address family");
2148 MODULE_LICENSE("GPL");
2149 MODULE_ALIAS_NETPROTO(PF_SMC);