net/smc/af_smc.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  *  Shared Memory Communications over RDMA (SMC-R) and RoCE
   4  *
   5  *  AF_SMC protocol family socket handler keeping the AF_INET sock address type
   6  *  applies to SOCK_STREAM sockets only
   7  *  offers an alternative communication option for TCP-protocol sockets
   8  *  applicable with RoCE-cards only
   9  *
  10  *  Initial restrictions:
  11  *    - support for alternate links postponed
  12  *
  13  *  Copyright IBM Corp. 2016, 2018
  14  *
  15  *  Author(s):  Ursula Braun <ubraun@linux.vnet.ibm.com>
  16  *              based on prototype from Frank Blaschka
  17  */
  18
  19 #define KMSG_COMPONENT "smc"
  20 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  21
  22 #include <linux/module.h>
  23 #include <linux/socket.h>
  24 #include <linux/workqueue.h>
  25 #include <linux/in.h>
  26 #include <linux/sched/signal.h>
  27 #include <linux/if_vlan.h>
  28 #include <linux/rcupdate_wait.h>
  29 #include <linux/ctype.h>
  30
  31 #include <net/sock.h>
  32 #include <net/tcp.h>
  33 #include <net/smc.h>
  34 #include <asm/ioctls.h>
  35
  36 #include <net/net_namespace.h>
  37 #include <net/netns/generic.h>
  38 #include "smc_netns.h"
  39
  40 #include "smc.h"
  41 #include "smc_clc.h"
  42 #include "smc_llc.h"
  43 #include "smc_cdc.h"
  44 #include "smc_core.h"
  45 #include "smc_ib.h"
  46 #include "smc_ism.h"
  47 #include "smc_pnet.h"
  48 #include "smc_netlink.h"
  49 #include "smc_tx.h"
  50 #include "smc_rx.h"
  51 #include "smc_close.h"
  52 #include "smc_stats.h"
  53 #include "smc_tracepoint.h"
  54
  55 static DEFINE_MUTEX(smc_server_lgr_pending);    /* serialize link group
  56                                                  * creation on server
  57                                                  */
  58 static DEFINE_MUTEX(smc_client_lgr_pending);    /* serialize link group
  59                                                  * creation on client
  60                                                  */
  61
  62 struct workqueue_struct *smc_hs_wq;     /* wq for handshake work */
  63 struct workqueue_struct *smc_close_wq;  /* wq for close work */
  64
  65 static void smc_tcp_listen_work(struct work_struct *);
  66 static void smc_connect_work(struct work_struct *);
  67
  68 static void smc_set_keepalive(struct sock *sk, int val)
  69 {
  70         struct smc_sock *smc = smc_sk(sk);
  71
  72         smc->clcsock->sk->sk_prot->keepalive(smc->clcsock->sk, val);
  73 }
  74
  75 static struct smc_hashinfo smc_v4_hashinfo = {
  76         .lock = __RW_LOCK_UNLOCKED(smc_v4_hashinfo.lock),
  77 };
  78
  79 static struct smc_hashinfo smc_v6_hashinfo = {
  80         .lock = __RW_LOCK_UNLOCKED(smc_v6_hashinfo.lock),
  81 };
  82
  83 int smc_hash_sk(struct sock *sk)
  84 {
  85         struct smc_hashinfo *h = sk->sk_prot->h.smc_hash;
  86         struct hlist_head *head;
  87
  88         head = &h->ht;
  89
  90         write_lock_bh(&h->lock);
  91         sk_add_node(sk, head);
  92         sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
  93         write_unlock_bh(&h->lock);
  94
  95         return 0;
  96 }
  97 EXPORT_SYMBOL_GPL(smc_hash_sk);
  98
  99 void smc_unhash_sk(struct sock *sk)
 100 {
 101         struct smc_hashinfo *h = sk->sk_prot->h.smc_hash;
 102
 103         write_lock_bh(&h->lock);
 104         if (sk_del_node_init(sk))
 105                 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
 106         write_unlock_bh(&h->lock);
 107 }
 108 EXPORT_SYMBOL_GPL(smc_unhash_sk);
 109
 110 struct proto smc_proto = {
 111         .name           = "SMC",
 112         .owner          = THIS_MODULE,
 113         .keepalive      = smc_set_keepalive,
 114         .hash           = smc_hash_sk,
 115         .unhash         = smc_unhash_sk,
 116         .obj_size       = sizeof(struct smc_sock),
 117         .h.smc_hash     = &smc_v4_hashinfo,
 118         .slab_flags     = SLAB_TYPESAFE_BY_RCU,
 119 };
 120 EXPORT_SYMBOL_GPL(smc_proto);
 121
 122 struct proto smc_proto6 = {
 123         .name           = "SMC6",
 124         .owner          = THIS_MODULE,
 125         .keepalive      = smc_set_keepalive,
 126         .hash           = smc_hash_sk,
 127         .unhash         = smc_unhash_sk,
 128         .obj_size       = sizeof(struct smc_sock),
 129         .h.smc_hash     = &smc_v6_hashinfo,
 130         .slab_flags     = SLAB_TYPESAFE_BY_RCU,
 131 };
 132 EXPORT_SYMBOL_GPL(smc_proto6);
 133
 134 static void smc_restore_fallback_changes(struct smc_sock *smc)
 135 {
 136         if (smc->clcsock->file) { /* non-accepted sockets have no file yet */
 137                 smc->clcsock->file->private_data = smc->sk.sk_socket;
 138                 smc->clcsock->file = NULL;
 139         }
 140 }
 141
 142 static int __smc_release(struct smc_sock *smc)
 143 {
 144         struct sock *sk = &smc->sk;
 145         int rc = 0;
 146
 147         if (!smc->use_fallback) {
 148                 rc = smc_close_active(smc);
 149                 sock_set_flag(sk, SOCK_DEAD);
 150                 sk->sk_shutdown |= SHUTDOWN_MASK;
 151         } else {
 152                 if (sk->sk_state != SMC_CLOSED) {
 153                         if (sk->sk_state != SMC_LISTEN &&
 154                             sk->sk_state != SMC_INIT)
 155                                 sock_put(sk); /* passive closing */
 156                         if (sk->sk_state == SMC_LISTEN) {
 157                                 /* wake up clcsock accept */
 158                                 rc = kernel_sock_shutdown(smc->clcsock,
 159                                                           SHUT_RDWR);
 160                         }
 161                         sk->sk_state = SMC_CLOSED;
 162                         sk->sk_state_change(sk);
 163                 }
 164                 smc_restore_fallback_changes(smc);
 165         }
 166
 167         sk->sk_prot->unhash(sk);
 168
 169         if (sk->sk_state == SMC_CLOSED) {
 170                 if (smc->clcsock) {
 171                         release_sock(sk);
 172                         smc_clcsock_release(smc);
 173                         lock_sock(sk);
 174                 }
 175                 if (!smc->use_fallback)
 176                         smc_conn_free(&smc->conn);
 177         }
 178
 179         return rc;
 180 }
 181
 182 static int smc_release(struct socket *sock)
 183 {
 184         struct sock *sk = sock->sk;
 185         struct smc_sock *smc;
 186         int rc = 0;
 187
 188         if (!sk)
 189                 goto out;
 190
 191         sock_hold(sk); /* sock_put below */
 192         smc = smc_sk(sk);
 193
 194         /* cleanup for a dangling non-blocking connect */
 195         if (smc->connect_nonblock && sk->sk_state == SMC_INIT)
 196                 tcp_abort(smc->clcsock->sk, ECONNABORTED);
 197         flush_work(&smc->connect_work);
 198
 199         if (sk->sk_state == SMC_LISTEN)
 200                 /* smc_close_non_accepted() is called and acquires
 201                  * sock lock for child sockets again
 202                  */
 203                 lock_sock_nested(sk, SINGLE_DEPTH_NESTING);
 204         else
 205                 lock_sock(sk);
 206
 207         rc = __smc_release(smc);
 208
 209         /* detach socket */
 210         sock_orphan(sk);
 211         sock->sk = NULL;
 212         release_sock(sk);
 213
 214         sock_put(sk); /* sock_hold above */
 215         sock_put(sk); /* final sock_put */
 216 out:
 217         return rc;
 218 }
 219
 220 static void smc_destruct(struct sock *sk)
 221 {
 222         if (sk->sk_state != SMC_CLOSED)
 223                 return;
 224         if (!sock_flag(sk, SOCK_DEAD))
 225                 return;
 226
 227         sk_refcnt_debug_dec(sk);
 228 }
 229
 230 static struct sock *smc_sock_alloc(struct net *net, struct socket *sock,
 231                                    int protocol)
 232 {
 233         struct smc_sock *smc;
 234         struct proto *prot;
 235         struct sock *sk;
 236
 237         prot = (protocol == SMCPROTO_SMC6) ? &smc_proto6 : &smc_proto;
 238         sk = sk_alloc(net, PF_SMC, GFP_KERNEL, prot, 0);
 239         if (!sk)
 240                 return NULL;
 241
 242         sock_init_data(sock, sk); /* sets sk_refcnt to 1 */
 243         sk->sk_state = SMC_INIT;
 244         sk->sk_destruct = smc_destruct;
 245         sk->sk_protocol = protocol;
 246         smc = smc_sk(sk);
 247         INIT_WORK(&smc->tcp_listen_work, smc_tcp_listen_work);
 248         INIT_WORK(&smc->connect_work, smc_connect_work);
 249         INIT_DELAYED_WORK(&smc->conn.tx_work, smc_tx_work);
 250         INIT_LIST_HEAD(&smc->accept_q);
 251         spin_lock_init(&smc->accept_q_lock);
 252         spin_lock_init(&smc->conn.send_lock);
 253         sk->sk_prot->hash(sk);
 254         sk_refcnt_debug_inc(sk);
 255         mutex_init(&smc->clcsock_release_lock);
 256
 257         return sk;
 258 }
 259
 260 static int smc_bind(struct socket *sock, struct sockaddr *uaddr,
 261                     int addr_len)
 262 {
 263         struct sockaddr_in *addr = (struct sockaddr_in *)uaddr;
 264         struct sock *sk = sock->sk;
 265         struct smc_sock *smc;
 266         int rc;
 267
 268         smc = smc_sk(sk);
 269
 270         /* replicate tests from inet_bind(), to be safe wrt. future changes */
 271         rc = -EINVAL;
 272         if (addr_len < sizeof(struct sockaddr_in))
 273                 goto out;
 274
 275         rc = -EAFNOSUPPORT;
 276         if (addr->sin_family != AF_INET &&
 277             addr->sin_family != AF_INET6 &&
 278             addr->sin_family != AF_UNSPEC)
 279                 goto out;
 280         /* accept AF_UNSPEC (mapped to AF_INET) only if s_addr is INADDR_ANY */
 281         if (addr->sin_family == AF_UNSPEC &&
 282             addr->sin_addr.s_addr != htonl(INADDR_ANY))
 283                 goto out;
 284
 285         lock_sock(sk);
 286
 287         /* Check if socket is already active */
 288         rc = -EINVAL;
 289         if (sk->sk_state != SMC_INIT || smc->connect_nonblock)
 290                 goto out_rel;
 291
 292         smc->clcsock->sk->sk_reuse = sk->sk_reuse;
 293         rc = kernel_bind(smc->clcsock, uaddr, addr_len);
 294
 295 out_rel:
 296         release_sock(sk);
 297 out:
 298         return rc;
 299 }
 300
 301 static void smc_copy_sock_settings(struct sock *nsk, struct sock *osk,
 302                                    unsigned long mask)
 303 {
 304         /* options we don't get control via setsockopt for */
 305         nsk->sk_type = osk->sk_type;
 306         nsk->sk_sndbuf = osk->sk_sndbuf;
 307         nsk->sk_rcvbuf = osk->sk_rcvbuf;
 308         nsk->sk_sndtimeo = osk->sk_sndtimeo;
 309         nsk->sk_rcvtimeo = osk->sk_rcvtimeo;
 310         nsk->sk_mark = osk->sk_mark;
 311         nsk->sk_priority = osk->sk_priority;
 312         nsk->sk_rcvlowat = osk->sk_rcvlowat;
 313         nsk->sk_bound_dev_if = osk->sk_bound_dev_if;
 314         nsk->sk_err = osk->sk_err;
 315
 316         nsk->sk_flags &= ~mask;
 317         nsk->sk_flags |= osk->sk_flags & mask;
 318 }
 319
 320 #define SK_FLAGS_SMC_TO_CLC ((1UL << SOCK_URGINLINE) | \
 321                              (1UL << SOCK_KEEPOPEN) | \
 322                              (1UL << SOCK_LINGER) | \
 323                              (1UL << SOCK_BROADCAST) | \
 324                              (1UL << SOCK_TIMESTAMP) | \
 325                              (1UL << SOCK_DBG) | \
 326                              (1UL << SOCK_RCVTSTAMP) | \
 327                              (1UL << SOCK_RCVTSTAMPNS) | \
 328                              (1UL << SOCK_LOCALROUTE) | \
 329                              (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE) | \
 330                              (1UL << SOCK_RXQ_OVFL) | \
 331                              (1UL << SOCK_WIFI_STATUS) | \
 332                              (1UL << SOCK_NOFCS) | \
 333                              (1UL << SOCK_FILTER_LOCKED) | \
 334                              (1UL << SOCK_TSTAMP_NEW))
 335 /* copy only relevant settings and flags of SOL_SOCKET level from smc to
 336  * clc socket (since smc is not called for these options from net/core)
 337  */
 338 static void smc_copy_sock_settings_to_clc(struct smc_sock *smc)
 339 {
 340         smc_copy_sock_settings(smc->clcsock->sk, &smc->sk, SK_FLAGS_SMC_TO_CLC);
 341 }
 342
 343 #define SK_FLAGS_CLC_TO_SMC ((1UL << SOCK_URGINLINE) | \
 344                              (1UL << SOCK_KEEPOPEN) | \
 345                              (1UL << SOCK_LINGER) | \
 346                              (1UL << SOCK_DBG))
 347 /* copy only settings and flags relevant for smc from clc to smc socket */
 348 static void smc_copy_sock_settings_to_smc(struct smc_sock *smc)
 349 {
 350         smc_copy_sock_settings(&smc->sk, smc->clcsock->sk, SK_FLAGS_CLC_TO_SMC);
 351 }
 352
 353 /* register the new rmb on all links */
 354 static int smcr_lgr_reg_rmbs(struct smc_link *link,
 355                              struct smc_buf_desc *rmb_desc)
 356 {
 357         struct smc_link_group *lgr = link->lgr;
 358         int i, rc = 0;
 359
 360         rc = smc_llc_flow_initiate(lgr, SMC_LLC_FLOW_RKEY);
 361         if (rc)
 362                 return rc;
 363         /* protect against parallel smc_llc_cli_rkey_exchange() and
 364          * parallel smcr_link_reg_rmb()
 365          */
 366         mutex_lock(&lgr->llc_conf_mutex);
 367         for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
 368                 if (!smc_link_active(&lgr->lnk[i]))
 369                         continue;
 370                 rc = smcr_link_reg_rmb(&lgr->lnk[i], rmb_desc);
 371                 if (rc)
 372                         goto out;
 373         }
 374
 375         /* exchange confirm_rkey msg with peer */
 376         rc = smc_llc_do_confirm_rkey(link, rmb_desc);
 377         if (rc) {
 378                 rc = -EFAULT;
 379                 goto out;
 380         }
 381         rmb_desc->is_conf_rkey = true;
 382 out:
 383         mutex_unlock(&lgr->llc_conf_mutex);
 384         smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl);
 385         return rc;
 386 }
 387
 388 static int smcr_clnt_conf_first_link(struct smc_sock *smc)
 389 {
 390         struct smc_link *link = smc->conn.lnk;
 391         struct smc_llc_qentry *qentry;
 392         int rc;
 393
 394         /* receive CONFIRM LINK request from server over RoCE fabric */
 395         qentry = smc_llc_wait(link->lgr, NULL, SMC_LLC_WAIT_TIME,
 396                               SMC_LLC_CONFIRM_LINK);
 397         if (!qentry) {
 398                 struct smc_clc_msg_decline dclc;
 399
 400                 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
 401                                       SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
 402                 return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_CL : rc;
 403         }
 404         smc_llc_save_peer_uid(qentry);
 405         rc = smc_llc_eval_conf_link(qentry, SMC_LLC_REQ);
 406         smc_llc_flow_qentry_del(&link->lgr->llc_flow_lcl);
 407         if (rc)
 408                 return SMC_CLC_DECL_RMBE_EC;
 409
 410         rc = smc_ib_modify_qp_rts(link);
 411         if (rc)
 412                 return SMC_CLC_DECL_ERR_RDYLNK;
 413
 414         smc_wr_remember_qp_attr(link);
 415
 416         if (smcr_link_reg_rmb(link, smc->conn.rmb_desc))
 417                 return SMC_CLC_DECL_ERR_REGRMB;
 418
 419         /* confirm_rkey is implicit on 1st contact */
 420         smc->conn.rmb_desc->is_conf_rkey = true;
 421
 422         /* send CONFIRM LINK response over RoCE fabric */
 423         rc = smc_llc_send_confirm_link(link, SMC_LLC_RESP);
 424         if (rc < 0)
 425                 return SMC_CLC_DECL_TIMEOUT_CL;
 426
 427         smc_llc_link_active(link);
 428         smcr_lgr_set_type(link->lgr, SMC_LGR_SINGLE);
 429
 430         /* optional 2nd link, receive ADD LINK request from server */
 431         qentry = smc_llc_wait(link->lgr, NULL, SMC_LLC_WAIT_TIME,
 432                               SMC_LLC_ADD_LINK);
 433         if (!qentry) {
 434                 struct smc_clc_msg_decline dclc;
 435
 436                 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
 437                                       SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
 438                 if (rc == -EAGAIN)
 439                         rc = 0; /* no DECLINE received, go with one link */
 440                 return rc;
 441         }
 442         smc_llc_flow_qentry_clr(&link->lgr->llc_flow_lcl);
 443         smc_llc_cli_add_link(link, qentry);
 444         return 0;
 445 }
 446
 447 static bool smc_isascii(char *hostname)
 448 {
 449         int i;
 450
 451         for (i = 0; i < SMC_MAX_HOSTNAME_LEN; i++)
 452                 if (!isascii(hostname[i]))
 453                         return false;
 454         return true;
 455 }
 456
 457 static void smc_conn_save_peer_info_fce(struct smc_sock *smc,
 458                                         struct smc_clc_msg_accept_confirm *clc)
 459 {
 460         struct smc_clc_msg_accept_confirm_v2 *clc_v2 =
 461                 (struct smc_clc_msg_accept_confirm_v2 *)clc;
 462         struct smc_clc_first_contact_ext *fce;
 463         int clc_v2_len;
 464
 465         if (clc->hdr.version == SMC_V1 ||
 466             !(clc->hdr.typev2 & SMC_FIRST_CONTACT_MASK))
 467                 return;
 468
 469         if (smc->conn.lgr->is_smcd) {
 470                 memcpy(smc->conn.lgr->negotiated_eid, clc_v2->d1.eid,
 471                        SMC_MAX_EID_LEN);
 472                 clc_v2_len = offsetofend(struct smc_clc_msg_accept_confirm_v2,
 473                                          d1);
 474         } else {
 475                 memcpy(smc->conn.lgr->negotiated_eid, clc_v2->r1.eid,
 476                        SMC_MAX_EID_LEN);
 477                 clc_v2_len = offsetofend(struct smc_clc_msg_accept_confirm_v2,
 478                                          r1);
 479         }
 480         fce = (struct smc_clc_first_contact_ext *)(((u8 *)clc_v2) + clc_v2_len);
 481         smc->conn.lgr->peer_os = fce->os_type;
 482         smc->conn.lgr->peer_smc_release = fce->release;
 483         if (smc_isascii(fce->hostname))
 484                 memcpy(smc->conn.lgr->peer_hostname, fce->hostname,
 485                        SMC_MAX_HOSTNAME_LEN);
 486 }
 487
 488 static void smcr_conn_save_peer_info(struct smc_sock *smc,
 489                                      struct smc_clc_msg_accept_confirm *clc)
 490 {
 491         int bufsize = smc_uncompress_bufsize(clc->r0.rmbe_size);
 492
 493         smc->conn.peer_rmbe_idx = clc->r0.rmbe_idx;
 494         smc->conn.local_tx_ctrl.token = ntohl(clc->r0.rmbe_alert_token);
 495         smc->conn.peer_rmbe_size = bufsize;
 496         atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size);
 497         smc->conn.tx_off = bufsize * (smc->conn.peer_rmbe_idx - 1);
 498 }
 499
 500 static void smcd_conn_save_peer_info(struct smc_sock *smc,
 501                                      struct smc_clc_msg_accept_confirm *clc)
 502 {
 503         int bufsize = smc_uncompress_bufsize(clc->d0.dmbe_size);
 504
 505         smc->conn.peer_rmbe_idx = clc->d0.dmbe_idx;
 506         smc->conn.peer_token = clc->d0.token;
 507         /* msg header takes up space in the buffer */
 508         smc->conn.peer_rmbe_size = bufsize - sizeof(struct smcd_cdc_msg);
 509         atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size);
 510         smc->conn.tx_off = bufsize * smc->conn.peer_rmbe_idx;
 511 }
 512
 513 static void smc_conn_save_peer_info(struct smc_sock *smc,
 514                                     struct smc_clc_msg_accept_confirm *clc)
 515 {
 516         if (smc->conn.lgr->is_smcd)
 517                 smcd_conn_save_peer_info(smc, clc);
 518         else
 519                 smcr_conn_save_peer_info(smc, clc);
 520         smc_conn_save_peer_info_fce(smc, clc);
 521 }
 522
 523 static void smc_link_save_peer_info(struct smc_link *link,
 524                                     struct smc_clc_msg_accept_confirm *clc,
 525                                     struct smc_init_info *ini)
 526 {
 527         link->peer_qpn = ntoh24(clc->r0.qpn);
 528         memcpy(link->peer_gid, ini->peer_gid, SMC_GID_SIZE);
 529         memcpy(link->peer_mac, ini->peer_mac, sizeof(link->peer_mac));
 530         link->peer_psn = ntoh24(clc->r0.psn);
 531         link->peer_mtu = clc->r0.qp_mtu;
 532 }
 533
 534 static void smc_stat_inc_fback_rsn_cnt(struct smc_sock *smc,
 535                                        struct smc_stats_fback *fback_arr)
 536 {
 537         int cnt;
 538
 539         for (cnt = 0; cnt < SMC_MAX_FBACK_RSN_CNT; cnt++) {
 540                 if (fback_arr[cnt].fback_code == smc->fallback_rsn) {
 541                         fback_arr[cnt].count++;
 542                         break;
 543                 }
 544                 if (!fback_arr[cnt].fback_code) {
 545                         fback_arr[cnt].fback_code = smc->fallback_rsn;
 546                         fback_arr[cnt].count++;
 547                         break;
 548                 }
 549         }
 550 }
 551
 552 static void smc_stat_fallback(struct smc_sock *smc)
 553 {
 554         struct net *net = sock_net(&smc->sk);
 555
 556         mutex_lock(&net->smc.mutex_fback_rsn);
 557         if (smc->listen_smc) {
 558                 smc_stat_inc_fback_rsn_cnt(smc, net->smc.fback_rsn->srv);
 559                 net->smc.fback_rsn->srv_fback_cnt++;
 560         } else {
 561                 smc_stat_inc_fback_rsn_cnt(smc, net->smc.fback_rsn->clnt);
 562                 net->smc.fback_rsn->clnt_fback_cnt++;
 563         }
 564         mutex_unlock(&net->smc.mutex_fback_rsn);
 565 }
 566
 567 static void smc_switch_to_fallback(struct smc_sock *smc, int reason_code)
 568 {
 569         wait_queue_head_t *smc_wait = sk_sleep(&smc->sk);
 570         wait_queue_head_t *clc_wait = sk_sleep(smc->clcsock->sk);
 571         unsigned long flags;
 572
 573         smc->use_fallback = true;
 574         smc->fallback_rsn = reason_code;
 575         smc_stat_fallback(smc);
 576         trace_smc_switch_to_fallback(smc, reason_code);
 577         if (smc->sk.sk_socket && smc->sk.sk_socket->file) {
 578                 smc->clcsock->file = smc->sk.sk_socket->file;
 579                 smc->clcsock->file->private_data = smc->clcsock;
 580                 smc->clcsock->wq.fasync_list =
 581                         smc->sk.sk_socket->wq.fasync_list;
 582
 583                 /* There may be some entries remaining in
 584                  * smc socket->wq, which should be removed
 585                  * to clcsocket->wq during the fallback.
 586                  */
 587                 spin_lock_irqsave(&smc_wait->lock, flags);
 588                 spin_lock(&clc_wait->lock);
 589                 list_splice_init(&smc_wait->head, &clc_wait->head);
 590                 spin_unlock(&clc_wait->lock);
 591                 spin_unlock_irqrestore(&smc_wait->lock, flags);
 592         }
 593 }
 594
 595 /* fall back during connect */
 596 static int smc_connect_fallback(struct smc_sock *smc, int reason_code)
 597 {
 598         smc_switch_to_fallback(smc, reason_code);
 599         smc_copy_sock_settings_to_clc(smc);
 600         smc->connect_nonblock = 0;
 601         if (smc->sk.sk_state == SMC_INIT)
 602                 smc->sk.sk_state = SMC_ACTIVE;
 603         return 0;
 604 }
 605
 606 /* decline and fall back during connect */
 607 static int smc_connect_decline_fallback(struct smc_sock *smc, int reason_code,
 608                                         u8 version)
 609 {
 610         struct net *net = sock_net(&smc->sk);
 611         int rc;
 612
 613         if (reason_code < 0) { /* error, fallback is not possible */
 614                 this_cpu_inc(net->smc.smc_stats->clnt_hshake_err_cnt);
 615                 if (smc->sk.sk_state == SMC_INIT)
 616                         sock_put(&smc->sk); /* passive closing */
 617                 return reason_code;
 618         }
 619         if (reason_code != SMC_CLC_DECL_PEERDECL) {
 620                 rc = smc_clc_send_decline(smc, reason_code, version);
 621                 if (rc < 0) {
 622                         this_cpu_inc(net->smc.smc_stats->clnt_hshake_err_cnt);
 623                         if (smc->sk.sk_state == SMC_INIT)
 624                                 sock_put(&smc->sk); /* passive closing */
 625                         return rc;
 626                 }
 627         }
 628         return smc_connect_fallback(smc, reason_code);
 629 }
 630
 631 static void smc_conn_abort(struct smc_sock *smc, int local_first)
 632 {
 633         if (local_first)
 634                 smc_lgr_cleanup_early(&smc->conn);
 635         else
 636                 smc_conn_free(&smc->conn);
 637 }
 638
 639 /* check if there is a rdma device available for this connection. */
 640 /* called for connect and listen */
 641 static int smc_find_rdma_device(struct smc_sock *smc, struct smc_init_info *ini)
 642 {
 643         /* PNET table look up: search active ib_device and port
 644          * within same PNETID that also contains the ethernet device
 645          * used for the internal TCP socket
 646          */
 647         smc_pnet_find_roce_resource(smc->clcsock->sk, ini);
 648         if (!ini->check_smcrv2 && !ini->ib_dev)
 649                 return SMC_CLC_DECL_NOSMCRDEV;
 650         if (ini->check_smcrv2 && !ini->smcrv2.ib_dev_v2)
 651                 return SMC_CLC_DECL_NOSMCRDEV;
 652         return 0;
 653 }
 654
 655 /* check if there is an ISM device available for this connection. */
 656 /* called for connect and listen */
 657 static int smc_find_ism_device(struct smc_sock *smc, struct smc_init_info *ini)
 658 {
 659         /* Find ISM device with same PNETID as connecting interface  */
 660         smc_pnet_find_ism_resource(smc->clcsock->sk, ini);
 661         if (!ini->ism_dev[0])
 662                 return SMC_CLC_DECL_NOSMCDDEV;
 663         else
 664                 ini->ism_chid[0] = smc_ism_get_chid(ini->ism_dev[0]);
 665         return 0;
 666 }
 667
 668 /* is chid unique for the ism devices that are already determined? */
 669 static bool smc_find_ism_v2_is_unique_chid(u16 chid, struct smc_init_info *ini,
 670                                            int cnt)
 671 {
 672         int i = (!ini->ism_dev[0]) ? 1 : 0;
 673
 674         for (; i < cnt; i++)
 675                 if (ini->ism_chid[i] == chid)
 676                         return false;
 677         return true;
 678 }
 679
 680 /* determine possible V2 ISM devices (either without PNETID or with PNETID plus
 681  * PNETID matching net_device)
 682  */
 683 static int smc_find_ism_v2_device_clnt(struct smc_sock *smc,
 684                                        struct smc_init_info *ini)
 685 {
 686         int rc = SMC_CLC_DECL_NOSMCDDEV;
 687         struct smcd_dev *smcd;
 688         int i = 1;
 689         u16 chid;
 690
 691         if (smcd_indicated(ini->smc_type_v1))
 692                 rc = 0;         /* already initialized for V1 */
 693         mutex_lock(&smcd_dev_list.mutex);
 694         list_for_each_entry(smcd, &smcd_dev_list.list, list) {
 695                 if (smcd->going_away || smcd == ini->ism_dev[0])
 696                         continue;
 697                 chid = smc_ism_get_chid(smcd);
 698                 if (!smc_find_ism_v2_is_unique_chid(chid, ini, i))
 699                         continue;
 700                 if (!smc_pnet_is_pnetid_set(smcd->pnetid) ||
 701                     smc_pnet_is_ndev_pnetid(sock_net(&smc->sk), smcd->pnetid)) {
 702                         ini->ism_dev[i] = smcd;
 703                         ini->ism_chid[i] = chid;
 704                         ini->is_smcd = true;
 705                         rc = 0;
 706                         i++;
 707                         if (i > SMC_MAX_ISM_DEVS)
 708                                 break;
 709                 }
 710         }
 711         mutex_unlock(&smcd_dev_list.mutex);
 712         ini->ism_offered_cnt = i - 1;
 713         if (!ini->ism_dev[0] && !ini->ism_dev[1])
 714                 ini->smcd_version = 0;
 715
 716         return rc;
 717 }
 718
 719 /* Check for VLAN ID and register it on ISM device just for CLC handshake */
 720 static int smc_connect_ism_vlan_setup(struct smc_sock *smc,
 721                                       struct smc_init_info *ini)
 722 {
 723         if (ini->vlan_id && smc_ism_get_vlan(ini->ism_dev[0], ini->vlan_id))
 724                 return SMC_CLC_DECL_ISMVLANERR;
 725         return 0;
 726 }
 727
 728 static int smc_find_proposal_devices(struct smc_sock *smc,
 729                                      struct smc_init_info *ini)
 730 {
 731         int rc = 0;
 732
 733         /* check if there is an ism device available */
 734         if (!(ini->smcd_version & SMC_V1) ||
 735             smc_find_ism_device(smc, ini) ||
 736             smc_connect_ism_vlan_setup(smc, ini))
 737                 ini->smcd_version &= ~SMC_V1;
 738         /* else ISM V1 is supported for this connection */
 739
 740         /* check if there is an rdma device available */
 741         if (!(ini->smcr_version & SMC_V1) ||
 742             smc_find_rdma_device(smc, ini))
 743                 ini->smcr_version &= ~SMC_V1;
 744         /* else RDMA is supported for this connection */
 745
 746         ini->smc_type_v1 = smc_indicated_type(ini->smcd_version & SMC_V1,
 747                                               ini->smcr_version & SMC_V1);
 748
 749         /* check if there is an ism v2 device available */
 750         if (!(ini->smcd_version & SMC_V2) ||
 751             !smc_ism_is_v2_capable() ||
 752             smc_find_ism_v2_device_clnt(smc, ini))
 753                 ini->smcd_version &= ~SMC_V2;
 754
 755         /* check if there is an rdma v2 device available */
 756         ini->check_smcrv2 = true;
 757         ini->smcrv2.saddr = smc->clcsock->sk->sk_rcv_saddr;
 758         if (!(ini->smcr_version & SMC_V2) ||
 759             smc->clcsock->sk->sk_family != AF_INET ||
 760             !smc_clc_ueid_count() ||
 761             smc_find_rdma_device(smc, ini))
 762                 ini->smcr_version &= ~SMC_V2;
 763         ini->check_smcrv2 = false;
 764
 765         ini->smc_type_v2 = smc_indicated_type(ini->smcd_version & SMC_V2,
 766                                               ini->smcr_version & SMC_V2);
 767
 768         /* if neither ISM nor RDMA are supported, fallback */
 769         if (ini->smc_type_v1 == SMC_TYPE_N && ini->smc_type_v2 == SMC_TYPE_N)
 770                 rc = SMC_CLC_DECL_NOSMCDEV;
 771
 772         return rc;
 773 }
 774
 775 /* cleanup temporary VLAN ID registration used for CLC handshake. If ISM is
 776  * used, the VLAN ID will be registered again during the connection setup.
 777  */
 778 static int smc_connect_ism_vlan_cleanup(struct smc_sock *smc,
 779                                         struct smc_init_info *ini)
 780 {
 781         if (!smcd_indicated(ini->smc_type_v1))
 782                 return 0;
 783         if (ini->vlan_id && smc_ism_put_vlan(ini->ism_dev[0], ini->vlan_id))
 784                 return SMC_CLC_DECL_CNFERR;
 785         return 0;
 786 }
 787
 788 #define SMC_CLC_MAX_ACCEPT_LEN \
 789         (sizeof(struct smc_clc_msg_accept_confirm_v2) + \
 790          sizeof(struct smc_clc_first_contact_ext) + \
 791          sizeof(struct smc_clc_msg_trail))
 792
 793 /* CLC handshake during connect */
 794 static int smc_connect_clc(struct smc_sock *smc,
 795                            struct smc_clc_msg_accept_confirm_v2 *aclc2,
 796                            struct smc_init_info *ini)
 797 {
 798         int rc = 0;
 799
 800         /* do inband token exchange */
 801         rc = smc_clc_send_proposal(smc, ini);
 802         if (rc)
 803                 return rc;
 804         /* receive SMC Accept CLC message */
 805         return smc_clc_wait_msg(smc, aclc2, SMC_CLC_MAX_ACCEPT_LEN,
 806                                 SMC_CLC_ACCEPT, CLC_WAIT_TIME);
 807 }
 808
 809 void smc_fill_gid_list(struct smc_link_group *lgr,
 810                        struct smc_gidlist *gidlist,
 811                        struct smc_ib_device *known_dev, u8 *known_gid)
 812 {
 813         struct smc_init_info *alt_ini = NULL;
 814
 815         memset(gidlist, 0, sizeof(*gidlist));
 816         memcpy(gidlist->list[gidlist->len++], known_gid, SMC_GID_SIZE);
 817
 818         alt_ini = kzalloc(sizeof(*alt_ini), GFP_KERNEL);
 819         if (!alt_ini)
 820                 goto out;
 821
 822         alt_ini->vlan_id = lgr->vlan_id;
 823         alt_ini->check_smcrv2 = true;
 824         alt_ini->smcrv2.saddr = lgr->saddr;
 825         smc_pnet_find_alt_roce(lgr, alt_ini, known_dev);
 826
 827         if (!alt_ini->smcrv2.ib_dev_v2)
 828                 goto out;
 829
 830         memcpy(gidlist->list[gidlist->len++], alt_ini->smcrv2.ib_gid_v2,
 831                SMC_GID_SIZE);
 832
 833 out:
 834         kfree(alt_ini);
 835 }
 836
 837 static int smc_connect_rdma_v2_prepare(struct smc_sock *smc,
 838                                        struct smc_clc_msg_accept_confirm *aclc,
 839                                        struct smc_init_info *ini)
 840 {
 841         struct smc_clc_msg_accept_confirm_v2 *clc_v2 =
 842                 (struct smc_clc_msg_accept_confirm_v2 *)aclc;
 843         struct smc_clc_first_contact_ext *fce =
 844                 (struct smc_clc_first_contact_ext *)
 845                         (((u8 *)clc_v2) + sizeof(*clc_v2));
 846
 847         if (!ini->first_contact_peer || aclc->hdr.version == SMC_V1)
 848                 return 0;
 849
 850         if (fce->v2_direct) {
 851                 memcpy(ini->smcrv2.nexthop_mac, &aclc->r0.lcl.mac, ETH_ALEN);
 852                 ini->smcrv2.uses_gateway = false;
 853         } else {
 854                 if (smc_ib_find_route(smc->clcsock->sk->sk_rcv_saddr,
 855                                       smc_ib_gid_to_ipv4(aclc->r0.lcl.gid),
 856                                       ini->smcrv2.nexthop_mac,
 857                                       &ini->smcrv2.uses_gateway))
 858                         return SMC_CLC_DECL_NOROUTE;
 859                 if (!ini->smcrv2.uses_gateway) {
 860                         /* mismatch: peer claims indirect, but its direct */
 861                         return SMC_CLC_DECL_NOINDIRECT;
 862                 }
 863         }
 864         return 0;
 865 }
 866
 867 /* setup for RDMA connection of client */
 868 static int smc_connect_rdma(struct smc_sock *smc,
 869                             struct smc_clc_msg_accept_confirm *aclc,
 870                             struct smc_init_info *ini)
 871 {
 872         int i, reason_code = 0;
 873         struct smc_link *link;
 874         u8 *eid = NULL;
 875
 876         ini->is_smcd = false;
 877         ini->ib_clcqpn = ntoh24(aclc->r0.qpn);
 878         ini->first_contact_peer = aclc->hdr.typev2 & SMC_FIRST_CONTACT_MASK;
 879         memcpy(ini->peer_systemid, aclc->r0.lcl.id_for_peer, SMC_SYSTEMID_LEN);
 880         memcpy(ini->peer_gid, aclc->r0.lcl.gid, SMC_GID_SIZE);
 881         memcpy(ini->peer_mac, aclc->r0.lcl.mac, ETH_ALEN);
 882
 883         reason_code = smc_connect_rdma_v2_prepare(smc, aclc, ini);
 884         if (reason_code)
 885                 return reason_code;
 886
 887         mutex_lock(&smc_client_lgr_pending);
 888         reason_code = smc_conn_create(smc, ini);
 889         if (reason_code) {
 890                 mutex_unlock(&smc_client_lgr_pending);
 891                 return reason_code;
 892         }
 893
 894         smc_conn_save_peer_info(smc, aclc);
 895
 896         if (ini->first_contact_local) {
 897                 link = smc->conn.lnk;
 898         } else {
 899                 /* set link that was assigned by server */
 900                 link = NULL;
 901                 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
 902                         struct smc_link *l = &smc->conn.lgr->lnk[i];
 903
 904                         if (l->peer_qpn == ntoh24(aclc->r0.qpn) &&
 905                             !memcmp(l->peer_gid, &aclc->r0.lcl.gid,
 906                                     SMC_GID_SIZE) &&
 907                             (aclc->hdr.version > SMC_V1 ||
 908                              !memcmp(l->peer_mac, &aclc->r0.lcl.mac,
 909                                      sizeof(l->peer_mac)))) {
 910                                 link = l;
 911                                 break;
 912                         }
 913                 }
 914                 if (!link) {
 915                         reason_code = SMC_CLC_DECL_NOSRVLINK;
 916                         goto connect_abort;
 917                 }
 918                 smc_switch_link_and_count(&smc->conn, link);
 919         }
 920
 921         /* create send buffer and rmb */
 922         if (smc_buf_create(smc, false)) {
 923                 reason_code = SMC_CLC_DECL_MEM;
 924                 goto connect_abort;
 925         }
 926
 927         if (ini->first_contact_local)
 928                 smc_link_save_peer_info(link, aclc, ini);
 929
 930         if (smc_rmb_rtoken_handling(&smc->conn, link, aclc)) {
 931                 reason_code = SMC_CLC_DECL_ERR_RTOK;
 932                 goto connect_abort;
 933         }
 934
 935         smc_close_init(smc);
 936         smc_rx_init(smc);
 937
 938         if (ini->first_contact_local) {
 939                 if (smc_ib_ready_link(link)) {
 940                         reason_code = SMC_CLC_DECL_ERR_RDYLNK;
 941                         goto connect_abort;
 942                 }
 943         } else {
 944                 if (smcr_lgr_reg_rmbs(link, smc->conn.rmb_desc)) {
 945                         reason_code = SMC_CLC_DECL_ERR_REGRMB;
 946                         goto connect_abort;
 947                 }
 948         }
 949         smc_rmb_sync_sg_for_device(&smc->conn);
 950
 951         if (aclc->hdr.version > SMC_V1) {
 952                 struct smc_clc_msg_accept_confirm_v2 *clc_v2 =
 953                         (struct smc_clc_msg_accept_confirm_v2 *)aclc;
 954
 955                 eid = clc_v2->r1.eid;
 956                 if (ini->first_contact_local)
 957                         smc_fill_gid_list(link->lgr, &ini->smcrv2.gidlist,
 958                                           link->smcibdev, link->gid);
 959         }
 960
 961         reason_code = smc_clc_send_confirm(smc, ini->first_contact_local,
 962                                            aclc->hdr.version, eid, ini);
 963         if (reason_code)
 964                 goto connect_abort;
 965
 966         smc_tx_init(smc);
 967
 968         if (ini->first_contact_local) {
 969                 /* QP confirmation over RoCE fabric */
 970                 smc_llc_flow_initiate(link->lgr, SMC_LLC_FLOW_ADD_LINK);
 971                 reason_code = smcr_clnt_conf_first_link(smc);
 972                 smc_llc_flow_stop(link->lgr, &link->lgr->llc_flow_lcl);
 973                 if (reason_code)
 974                         goto connect_abort;
 975         }
 976         mutex_unlock(&smc_client_lgr_pending);
 977
 978         smc_copy_sock_settings_to_clc(smc);
 979         smc->connect_nonblock = 0;
 980         if (smc->sk.sk_state == SMC_INIT)
 981                 smc->sk.sk_state = SMC_ACTIVE;
 982
 983         return 0;
 984 connect_abort:
 985         smc_conn_abort(smc, ini->first_contact_local);
 986         mutex_unlock(&smc_client_lgr_pending);
 987         smc->connect_nonblock = 0;
 988
 989         return reason_code;
 990 }
 991
 992 /* The server has chosen one of the proposed ISM devices for the communication.
 993  * Determine from the CHID of the received CLC ACCEPT the ISM device chosen.
 994  */
 995 static int
 996 smc_v2_determine_accepted_chid(struct smc_clc_msg_accept_confirm_v2 *aclc,
 997                                struct smc_init_info *ini)
 998 {
 999         int i;
1000
1001         for (i = 0; i < ini->ism_offered_cnt + 1; i++) {
1002                 if (ini->ism_chid[i] == ntohs(aclc->d1.chid)) {
1003                         ini->ism_selected = i;
1004                         return 0;
1005                 }
1006         }
1007
1008         return -EPROTO;
1009 }
1010
1011 /* setup for ISM connection of client */
1012 static int smc_connect_ism(struct smc_sock *smc,
1013                            struct smc_clc_msg_accept_confirm *aclc,
1014                            struct smc_init_info *ini)
1015 {
1016         u8 *eid = NULL;
1017         int rc = 0;
1018
1019         ini->is_smcd = true;
1020         ini->first_contact_peer = aclc->hdr.typev2 & SMC_FIRST_CONTACT_MASK;
1021
1022         if (aclc->hdr.version == SMC_V2) {
1023                 struct smc_clc_msg_accept_confirm_v2 *aclc_v2 =
1024                         (struct smc_clc_msg_accept_confirm_v2 *)aclc;
1025
1026                 rc = smc_v2_determine_accepted_chid(aclc_v2, ini);
1027                 if (rc)
1028                         return rc;
1029         }
1030         ini->ism_peer_gid[ini->ism_selected] = aclc->d0.gid;
1031
1032         /* there is only one lgr role for SMC-D; use server lock */
1033         mutex_lock(&smc_server_lgr_pending);
1034         rc = smc_conn_create(smc, ini);
1035         if (rc) {
1036                 mutex_unlock(&smc_server_lgr_pending);
1037                 return rc;
1038         }
1039
1040         /* Create send and receive buffers */
1041         rc = smc_buf_create(smc, true);
1042         if (rc) {
1043                 rc = (rc == -ENOSPC) ? SMC_CLC_DECL_MAX_DMB : SMC_CLC_DECL_MEM;
1044                 goto connect_abort;
1045         }
1046
1047         smc_conn_save_peer_info(smc, aclc);
1048         smc_close_init(smc);
1049         smc_rx_init(smc);
1050         smc_tx_init(smc);
1051
1052         if (aclc->hdr.version > SMC_V1) {
1053                 struct smc_clc_msg_accept_confirm_v2 *clc_v2 =
1054                         (struct smc_clc_msg_accept_confirm_v2 *)aclc;
1055
1056                 eid = clc_v2->d1.eid;
1057         }
1058
1059         rc = smc_clc_send_confirm(smc, ini->first_contact_local,
1060                                   aclc->hdr.version, eid, NULL);
1061         if (rc)
1062                 goto connect_abort;
1063         mutex_unlock(&smc_server_lgr_pending);
1064
1065         smc_copy_sock_settings_to_clc(smc);
1066         smc->connect_nonblock = 0;
1067         if (smc->sk.sk_state == SMC_INIT)
1068                 smc->sk.sk_state = SMC_ACTIVE;
1069
1070         return 0;
1071 connect_abort:
1072         smc_conn_abort(smc, ini->first_contact_local);
1073         mutex_unlock(&smc_server_lgr_pending);
1074         smc->connect_nonblock = 0;
1075
1076         return rc;
1077 }
1078
1079 /* check if received accept type and version matches a proposed one */
1080 static int smc_connect_check_aclc(struct smc_init_info *ini,
1081                                   struct smc_clc_msg_accept_confirm *aclc)
1082 {
1083         if (aclc->hdr.typev1 != SMC_TYPE_R &&
1084             aclc->hdr.typev1 != SMC_TYPE_D)
1085                 return SMC_CLC_DECL_MODEUNSUPP;
1086
1087         if (aclc->hdr.version >= SMC_V2) {
1088                 if ((aclc->hdr.typev1 == SMC_TYPE_R &&
1089                      !smcr_indicated(ini->smc_type_v2)) ||
1090                     (aclc->hdr.typev1 == SMC_TYPE_D &&
1091                      !smcd_indicated(ini->smc_type_v2)))
1092                         return SMC_CLC_DECL_MODEUNSUPP;
1093         } else {
1094                 if ((aclc->hdr.typev1 == SMC_TYPE_R &&
1095                      !smcr_indicated(ini->smc_type_v1)) ||
1096                     (aclc->hdr.typev1 == SMC_TYPE_D &&
1097                      !smcd_indicated(ini->smc_type_v1)))
1098                         return SMC_CLC_DECL_MODEUNSUPP;
1099         }
1100
1101         return 0;
1102 }
1103
1104 /* perform steps before actually connecting */
1105 static int __smc_connect(struct smc_sock *smc)
1106 {
1107         u8 version = smc_ism_is_v2_capable() ? SMC_V2 : SMC_V1;
1108         struct smc_clc_msg_accept_confirm_v2 *aclc2;
1109         struct smc_clc_msg_accept_confirm *aclc;
1110         struct smc_init_info *ini = NULL;
1111         u8 *buf = NULL;
1112         int rc = 0;
1113
1114         if (smc->use_fallback)
1115                 return smc_connect_fallback(smc, smc->fallback_rsn);
1116
1117         /* if peer has not signalled SMC-capability, fall back */
1118         if (!tcp_sk(smc->clcsock->sk)->syn_smc)
1119                 return smc_connect_fallback(smc, SMC_CLC_DECL_PEERNOSMC);
1120
1121         /* IPSec connections opt out of SMC optimizations */
1122         if (using_ipsec(smc))
1123                 return smc_connect_decline_fallback(smc, SMC_CLC_DECL_IPSEC,
1124                                                     version);
1125
1126         ini = kzalloc(sizeof(*ini), GFP_KERNEL);
1127         if (!ini)
1128                 return smc_connect_decline_fallback(smc, SMC_CLC_DECL_MEM,
1129                                                     version);
1130
1131         ini->smcd_version = SMC_V1 | SMC_V2;
1132         ini->smcr_version = SMC_V1 | SMC_V2;
1133         ini->smc_type_v1 = SMC_TYPE_B;
1134         ini->smc_type_v2 = SMC_TYPE_B;
1135
1136         /* get vlan id from IP device */
1137         if (smc_vlan_by_tcpsk(smc->clcsock, ini)) {
1138                 ini->smcd_version &= ~SMC_V1;
1139                 ini->smcr_version = 0;
1140                 ini->smc_type_v1 = SMC_TYPE_N;
1141                 if (!ini->smcd_version) {
1142                         rc = SMC_CLC_DECL_GETVLANERR;
1143                         goto fallback;
1144                 }
1145         }
1146
1147         rc = smc_find_proposal_devices(smc, ini);
1148         if (rc)
1149                 goto fallback;
1150
1151         buf = kzalloc(SMC_CLC_MAX_ACCEPT_LEN, GFP_KERNEL);
1152         if (!buf) {
1153                 rc = SMC_CLC_DECL_MEM;
1154                 goto fallback;
1155         }
1156         aclc2 = (struct smc_clc_msg_accept_confirm_v2 *)buf;
1157         aclc = (struct smc_clc_msg_accept_confirm *)aclc2;
1158
1159         /* perform CLC handshake */
1160         rc = smc_connect_clc(smc, aclc2, ini);
1161         if (rc)
1162                 goto vlan_cleanup;
1163
1164         /* check if smc modes and versions of CLC proposal and accept match */
1165         rc = smc_connect_check_aclc(ini, aclc);
1166         version = aclc->hdr.version == SMC_V1 ? SMC_V1 : SMC_V2;
1167         if (rc)
1168                 goto vlan_cleanup;
1169
1170         /* depending on previous steps, connect using rdma or ism */
1171         if (aclc->hdr.typev1 == SMC_TYPE_R) {
1172                 ini->smcr_version = version;
1173                 rc = smc_connect_rdma(smc, aclc, ini);
1174         } else if (aclc->hdr.typev1 == SMC_TYPE_D) {
1175                 ini->smcd_version = version;
1176                 rc = smc_connect_ism(smc, aclc, ini);
1177         }
1178         if (rc)
1179                 goto vlan_cleanup;
1180
1181         SMC_STAT_CLNT_SUCC_INC(sock_net(smc->clcsock->sk), aclc);
1182         smc_connect_ism_vlan_cleanup(smc, ini);
1183         kfree(buf);
1184         kfree(ini);
1185         return 0;
1186
1187 vlan_cleanup:
1188         smc_connect_ism_vlan_cleanup(smc, ini);
1189         kfree(buf);
1190 fallback:
1191         kfree(ini);
1192         return smc_connect_decline_fallback(smc, rc, version);
1193 }
1194
1195 static void smc_connect_work(struct work_struct *work)
1196 {
1197         struct smc_sock *smc = container_of(work, struct smc_sock,
1198                                             connect_work);
1199         long timeo = smc->sk.sk_sndtimeo;
1200         int rc = 0;
1201
1202         if (!timeo)
1203                 timeo = MAX_SCHEDULE_TIMEOUT;
1204         lock_sock(smc->clcsock->sk);
1205         if (smc->clcsock->sk->sk_err) {
1206                 smc->sk.sk_err = smc->clcsock->sk->sk_err;
1207         } else if ((1 << smc->clcsock->sk->sk_state) &
1208                                         (TCPF_SYN_SENT | TCPF_SYN_RECV)) {
1209                 rc = sk_stream_wait_connect(smc->clcsock->sk, &timeo);
1210                 if ((rc == -EPIPE) &&
1211                     ((1 << smc->clcsock->sk->sk_state) &
1212                                         (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)))
1213                         rc = 0;
1214         }
1215         release_sock(smc->clcsock->sk);
1216         lock_sock(&smc->sk);
1217         if (rc != 0 || smc->sk.sk_err) {
1218                 smc->sk.sk_state = SMC_CLOSED;
1219                 if (rc == -EPIPE || rc == -EAGAIN)
1220                         smc->sk.sk_err = EPIPE;
1221                 else if (signal_pending(current))
1222                         smc->sk.sk_err = -sock_intr_errno(timeo);
1223                 sock_put(&smc->sk); /* passive closing */
1224                 goto out;
1225         }
1226
1227         rc = __smc_connect(smc);
1228         if (rc < 0)
1229                 smc->sk.sk_err = -rc;
1230
1231 out:
1232         if (!sock_flag(&smc->sk, SOCK_DEAD)) {
1233                 if (smc->sk.sk_err) {
1234                         smc->sk.sk_state_change(&smc->sk);
1235                 } else { /* allow polling before and after fallback decision */
1236                         smc->clcsock->sk->sk_write_space(smc->clcsock->sk);
1237                         smc->sk.sk_write_space(&smc->sk);
1238                 }
1239         }
1240         release_sock(&smc->sk);
1241 }
1242
1243 static int smc_connect(struct socket *sock, struct sockaddr *addr,
1244                        int alen, int flags)
1245 {
1246         struct sock *sk = sock->sk;
1247         struct smc_sock *smc;
1248         int rc = -EINVAL;
1249
1250         smc = smc_sk(sk);
1251
1252         /* separate smc parameter checking to be safe */
1253         if (alen < sizeof(addr->sa_family))
1254                 goto out_err;
1255         if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6)
1256                 goto out_err;
1257
1258         lock_sock(sk);
1259         switch (sk->sk_state) {
1260         default:
1261                 goto out;
1262         case SMC_ACTIVE:
1263                 rc = -EISCONN;
1264                 goto out;
1265         case SMC_INIT:
1266                 break;
1267         }
1268
1269         smc_copy_sock_settings_to_clc(smc);
1270         tcp_sk(smc->clcsock->sk)->syn_smc = 1;
1271         if (smc->connect_nonblock) {
1272                 rc = -EALREADY;
1273                 goto out;
1274         }
1275         rc = kernel_connect(smc->clcsock, addr, alen, flags);
1276         if (rc && rc != -EINPROGRESS)
1277                 goto out;
1278
1279         sock_hold(&smc->sk); /* sock put in passive closing */
1280         if (smc->use_fallback)
1281                 goto out;
1282         if (flags & O_NONBLOCK) {
1283                 if (queue_work(smc_hs_wq, &smc->connect_work))
1284                         smc->connect_nonblock = 1;
1285                 rc = -EINPROGRESS;
1286         } else {
1287                 rc = __smc_connect(smc);
1288                 if (rc < 0)
1289                         goto out;
1290                 else
1291                         rc = 0; /* success cases including fallback */
1292         }
1293
1294 out:
1295         release_sock(sk);
1296 out_err:
1297         return rc;
1298 }
1299
1300 static int smc_clcsock_accept(struct smc_sock *lsmc, struct smc_sock **new_smc)
1301 {
1302         struct socket *new_clcsock = NULL;
1303         struct sock *lsk = &lsmc->sk;
1304         struct sock *new_sk;
1305         int rc = -EINVAL;
1306
1307         release_sock(lsk);
1308         new_sk = smc_sock_alloc(sock_net(lsk), NULL, lsk->sk_protocol);
1309         if (!new_sk) {
1310                 rc = -ENOMEM;
1311                 lsk->sk_err = ENOMEM;
1312                 *new_smc = NULL;
1313                 lock_sock(lsk);
1314                 goto out;
1315         }
1316         *new_smc = smc_sk(new_sk);
1317
1318         mutex_lock(&lsmc->clcsock_release_lock);
1319         if (lsmc->clcsock)
1320                 rc = kernel_accept(lsmc->clcsock, &new_clcsock, SOCK_NONBLOCK);
1321         mutex_unlock(&lsmc->clcsock_release_lock);
1322         lock_sock(lsk);
1323         if  (rc < 0 && rc != -EAGAIN)
1324                 lsk->sk_err = -rc;
1325         if (rc < 0 || lsk->sk_state == SMC_CLOSED) {
1326                 new_sk->sk_prot->unhash(new_sk);
1327                 if (new_clcsock)
1328                         sock_release(new_clcsock);
1329                 new_sk->sk_state = SMC_CLOSED;
1330                 sock_set_flag(new_sk, SOCK_DEAD);
1331                 sock_put(new_sk); /* final */
1332                 *new_smc = NULL;
1333                 goto out;
1334         }
1335
1336         /* new clcsock has inherited the smc listen-specific sk_data_ready
1337          * function; switch it back to the original sk_data_ready function
1338          */
1339         new_clcsock->sk->sk_data_ready = lsmc->clcsk_data_ready;
1340         (*new_smc)->clcsock = new_clcsock;
1341 out:
1342         return rc;
1343 }
1344
1345 /* add a just created sock to the accept queue of the listen sock as
1346  * candidate for a following socket accept call from user space
1347  */
1348 static void smc_accept_enqueue(struct sock *parent, struct sock *sk)
1349 {
1350         struct smc_sock *par = smc_sk(parent);
1351
1352         sock_hold(sk); /* sock_put in smc_accept_unlink () */
1353         spin_lock(&par->accept_q_lock);
1354         list_add_tail(&smc_sk(sk)->accept_q, &par->accept_q);
1355         spin_unlock(&par->accept_q_lock);
1356         sk_acceptq_added(parent);
1357 }
1358
1359 /* remove a socket from the accept queue of its parental listening socket */
1360 static void smc_accept_unlink(struct sock *sk)
1361 {
1362         struct smc_sock *par = smc_sk(sk)->listen_smc;
1363
1364         spin_lock(&par->accept_q_lock);
1365         list_del_init(&smc_sk(sk)->accept_q);
1366         spin_unlock(&par->accept_q_lock);
1367         sk_acceptq_removed(&smc_sk(sk)->listen_smc->sk);
1368         sock_put(sk); /* sock_hold in smc_accept_enqueue */
1369 }
1370
1371 /* remove a sock from the accept queue to bind it to a new socket created
1372  * for a socket accept call from user space
1373  */
1374 struct sock *smc_accept_dequeue(struct sock *parent,
1375                                 struct socket *new_sock)
1376 {
1377         struct smc_sock *isk, *n;
1378         struct sock *new_sk;
1379
1380         list_for_each_entry_safe(isk, n, &smc_sk(parent)->accept_q, accept_q) {
1381                 new_sk = (struct sock *)isk;
1382
1383                 smc_accept_unlink(new_sk);
1384                 if (new_sk->sk_state == SMC_CLOSED) {
1385                         new_sk->sk_prot->unhash(new_sk);
1386                         if (isk->clcsock) {
1387                                 sock_release(isk->clcsock);
1388                                 isk->clcsock = NULL;
1389                         }
1390                         sock_put(new_sk); /* final */
1391                         continue;
1392                 }
1393                 if (new_sock) {
1394                         sock_graft(new_sk, new_sock);
1395                         if (isk->use_fallback) {
1396                                 smc_sk(new_sk)->clcsock->file = new_sock->file;
1397                                 isk->clcsock->file->private_data = isk->clcsock;
1398                         }
1399                 }
1400                 return new_sk;
1401         }
1402         return NULL;
1403 }
1404
1405 /* clean up for a created but never accepted sock */
1406 void smc_close_non_accepted(struct sock *sk)
1407 {
1408         struct smc_sock *smc = smc_sk(sk);
1409
1410         sock_hold(sk); /* sock_put below */
1411         lock_sock(sk);
1412         if (!sk->sk_lingertime)
1413                 /* wait for peer closing */
1414                 sk->sk_lingertime = SMC_MAX_STREAM_WAIT_TIMEOUT;
1415         __smc_release(smc);
1416         release_sock(sk);
1417         sock_put(sk); /* sock_hold above */
1418         sock_put(sk); /* final sock_put */
1419 }
1420
1421 static int smcr_serv_conf_first_link(struct smc_sock *smc)
1422 {
1423         struct smc_link *link = smc->conn.lnk;
1424         struct smc_llc_qentry *qentry;
1425         int rc;
1426
1427         if (smcr_link_reg_rmb(link, smc->conn.rmb_desc))
1428                 return SMC_CLC_DECL_ERR_REGRMB;
1429
1430         /* send CONFIRM LINK request to client over the RoCE fabric */
1431         rc = smc_llc_send_confirm_link(link, SMC_LLC_REQ);
1432         if (rc < 0)
1433                 return SMC_CLC_DECL_TIMEOUT_CL;
1434
1435         /* receive CONFIRM LINK response from client over the RoCE fabric */
1436         qentry = smc_llc_wait(link->lgr, link, SMC_LLC_WAIT_TIME,
1437                               SMC_LLC_CONFIRM_LINK);
1438         if (!qentry) {
1439                 struct smc_clc_msg_decline dclc;
1440
1441                 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
1442                                       SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
1443                 return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_CL : rc;
1444         }
1445         smc_llc_save_peer_uid(qentry);
1446         rc = smc_llc_eval_conf_link(qentry, SMC_LLC_RESP);
1447         smc_llc_flow_qentry_del(&link->lgr->llc_flow_lcl);
1448         if (rc)
1449                 return SMC_CLC_DECL_RMBE_EC;
1450
1451         /* confirm_rkey is implicit on 1st contact */
1452         smc->conn.rmb_desc->is_conf_rkey = true;
1453
1454         smc_llc_link_active(link);
1455         smcr_lgr_set_type(link->lgr, SMC_LGR_SINGLE);
1456
1457         /* initial contact - try to establish second link */
1458         smc_llc_srv_add_link(link, NULL);
1459         return 0;
1460 }
1461
1462 /* listen worker: finish */
1463 static void smc_listen_out(struct smc_sock *new_smc)
1464 {
1465         struct smc_sock *lsmc = new_smc->listen_smc;
1466         struct sock *newsmcsk = &new_smc->sk;
1467
1468         if (lsmc->sk.sk_state == SMC_LISTEN) {
1469                 lock_sock_nested(&lsmc->sk, SINGLE_DEPTH_NESTING);
1470                 smc_accept_enqueue(&lsmc->sk, newsmcsk);
1471                 release_sock(&lsmc->sk);
1472         } else { /* no longer listening */
1473                 smc_close_non_accepted(newsmcsk);
1474         }
1475
1476         /* Wake up accept */
1477         lsmc->sk.sk_data_ready(&lsmc->sk);
1478         sock_put(&lsmc->sk); /* sock_hold in smc_tcp_listen_work */
1479 }
1480
1481 /* listen worker: finish in state connected */
1482 static void smc_listen_out_connected(struct smc_sock *new_smc)
1483 {
1484         struct sock *newsmcsk = &new_smc->sk;
1485
1486         sk_refcnt_debug_inc(newsmcsk);
1487         if (newsmcsk->sk_state == SMC_INIT)
1488                 newsmcsk->sk_state = SMC_ACTIVE;
1489
1490         smc_listen_out(new_smc);
1491 }
1492
1493 /* listen worker: finish in error state */
1494 static void smc_listen_out_err(struct smc_sock *new_smc)
1495 {
1496         struct sock *newsmcsk = &new_smc->sk;
1497         struct net *net = sock_net(newsmcsk);
1498
1499         this_cpu_inc(net->smc.smc_stats->srv_hshake_err_cnt);
1500         if (newsmcsk->sk_state == SMC_INIT)
1501                 sock_put(&new_smc->sk); /* passive closing */
1502         newsmcsk->sk_state = SMC_CLOSED;
1503
1504         smc_listen_out(new_smc);
1505 }
1506
1507 /* listen worker: decline and fall back if possible */
1508 static void smc_listen_decline(struct smc_sock *new_smc, int reason_code,
1509                                int local_first, u8 version)
1510 {
1511         /* RDMA setup failed, switch back to TCP */
1512         smc_conn_abort(new_smc, local_first);
1513         if (reason_code < 0) { /* error, no fallback possible */
1514                 smc_listen_out_err(new_smc);
1515                 return;
1516         }
1517         smc_switch_to_fallback(new_smc, reason_code);
1518         if (reason_code && reason_code != SMC_CLC_DECL_PEERDECL) {
1519                 if (smc_clc_send_decline(new_smc, reason_code, version) < 0) {
1520                         smc_listen_out_err(new_smc);
1521                         return;
1522                 }
1523         }
1524         smc_listen_out_connected(new_smc);
1525 }
1526
1527 /* listen worker: version checking */
1528 static int smc_listen_v2_check(struct smc_sock *new_smc,
1529                                struct smc_clc_msg_proposal *pclc,
1530                                struct smc_init_info *ini)
1531 {
1532         struct smc_clc_smcd_v2_extension *pclc_smcd_v2_ext;
1533         struct smc_clc_v2_extension *pclc_v2_ext;
1534         int rc = SMC_CLC_DECL_PEERNOSMC;
1535
1536         ini->smc_type_v1 = pclc->hdr.typev1;
1537         ini->smc_type_v2 = pclc->hdr.typev2;
1538         ini->smcd_version = smcd_indicated(ini->smc_type_v1) ? SMC_V1 : 0;
1539         ini->smcr_version = smcr_indicated(ini->smc_type_v1) ? SMC_V1 : 0;
1540         if (pclc->hdr.version > SMC_V1) {
1541                 if (smcd_indicated(ini->smc_type_v2))
1542                         ini->smcd_version |= SMC_V2;
1543                 if (smcr_indicated(ini->smc_type_v2))
1544                         ini->smcr_version |= SMC_V2;
1545         }
1546         if (!(ini->smcd_version & SMC_V2) && !(ini->smcr_version & SMC_V2)) {
1547                 rc = SMC_CLC_DECL_PEERNOSMC;
1548                 goto out;
1549         }
1550         pclc_v2_ext = smc_get_clc_v2_ext(pclc);
1551         if (!pclc_v2_ext) {
1552                 ini->smcd_version &= ~SMC_V2;
1553                 ini->smcr_version &= ~SMC_V2;
1554                 rc = SMC_CLC_DECL_NOV2EXT;
1555                 goto out;
1556         }
1557         pclc_smcd_v2_ext = smc_get_clc_smcd_v2_ext(pclc_v2_ext);
1558         if (ini->smcd_version & SMC_V2) {
1559                 if (!smc_ism_is_v2_capable()) {
1560                         ini->smcd_version &= ~SMC_V2;
1561                         rc = SMC_CLC_DECL_NOISM2SUPP;
1562                 } else if (!pclc_smcd_v2_ext) {
1563                         ini->smcd_version &= ~SMC_V2;
1564                         rc = SMC_CLC_DECL_NOV2DEXT;
1565                 } else if (!pclc_v2_ext->hdr.eid_cnt &&
1566                            !pclc_v2_ext->hdr.flag.seid) {
1567                         ini->smcd_version &= ~SMC_V2;
1568                         rc = SMC_CLC_DECL_NOUEID;
1569                 }
1570         }
1571         if (ini->smcr_version & SMC_V2) {
1572                 if (!pclc_v2_ext->hdr.eid_cnt) {
1573                         ini->smcr_version &= ~SMC_V2;
1574                         rc = SMC_CLC_DECL_NOUEID;
1575                 }
1576         }
1577
1578 out:
1579         if (!ini->smcd_version && !ini->smcr_version)
1580                 return rc;
1581
1582         return 0;
1583 }
1584
1585 /* listen worker: check prefixes */
1586 static int smc_listen_prfx_check(struct smc_sock *new_smc,
1587                                  struct smc_clc_msg_proposal *pclc)
1588 {
1589         struct smc_clc_msg_proposal_prefix *pclc_prfx;
1590         struct socket *newclcsock = new_smc->clcsock;
1591
1592         if (pclc->hdr.typev1 == SMC_TYPE_N)
1593                 return 0;
1594         pclc_prfx = smc_clc_proposal_get_prefix(pclc);
1595         if (smc_clc_prfx_match(newclcsock, pclc_prfx))
1596                 return SMC_CLC_DECL_DIFFPREFIX;
1597
1598         return 0;
1599 }
1600
1601 /* listen worker: initialize connection and buffers */
1602 static int smc_listen_rdma_init(struct smc_sock *new_smc,
1603                                 struct smc_init_info *ini)
1604 {
1605         int rc;
1606
1607         /* allocate connection / link group */
1608         rc = smc_conn_create(new_smc, ini);
1609         if (rc)
1610                 return rc;
1611
1612         /* create send buffer and rmb */
1613         if (smc_buf_create(new_smc, false))
1614                 return SMC_CLC_DECL_MEM;
1615
1616         return 0;
1617 }
1618
1619 /* listen worker: initialize connection and buffers for SMC-D */
1620 static int smc_listen_ism_init(struct smc_sock *new_smc,
1621                                struct smc_init_info *ini)
1622 {
1623         int rc;
1624
1625         rc = smc_conn_create(new_smc, ini);
1626         if (rc)
1627                 return rc;
1628
1629         /* Create send and receive buffers */
1630         rc = smc_buf_create(new_smc, true);
1631         if (rc) {
1632                 smc_conn_abort(new_smc, ini->first_contact_local);
1633                 return (rc == -ENOSPC) ? SMC_CLC_DECL_MAX_DMB :
1634                                          SMC_CLC_DECL_MEM;
1635         }
1636
1637         return 0;
1638 }
1639
1640 static bool smc_is_already_selected(struct smcd_dev *smcd,
1641                                     struct smc_init_info *ini,
1642                                     int matches)
1643 {
1644         int i;
1645
1646         for (i = 0; i < matches; i++)
1647                 if (smcd == ini->ism_dev[i])
1648                         return true;
1649
1650         return false;
1651 }
1652
1653 /* check for ISM devices matching proposed ISM devices */
1654 static void smc_check_ism_v2_match(struct smc_init_info *ini,
1655                                    u16 proposed_chid, u64 proposed_gid,
1656                                    unsigned int *matches)
1657 {
1658         struct smcd_dev *smcd;
1659
1660         list_for_each_entry(smcd, &smcd_dev_list.list, list) {
1661                 if (smcd->going_away)
1662                         continue;
1663                 if (smc_is_already_selected(smcd, ini, *matches))
1664                         continue;
1665                 if (smc_ism_get_chid(smcd) == proposed_chid &&
1666                     !smc_ism_cantalk(proposed_gid, ISM_RESERVED_VLANID, smcd)) {
1667                         ini->ism_peer_gid[*matches] = proposed_gid;
1668                         ini->ism_dev[*matches] = smcd;
1669                         (*matches)++;
1670                         break;
1671                 }
1672         }
1673 }
1674
1675 static void smc_find_ism_store_rc(u32 rc, struct smc_init_info *ini)
1676 {
1677         if (!ini->rc)
1678                 ini->rc = rc;
1679 }
1680
1681 static void smc_find_ism_v2_device_serv(struct smc_sock *new_smc,
1682                                         struct smc_clc_msg_proposal *pclc,
1683                                         struct smc_init_info *ini)
1684 {
1685         struct smc_clc_smcd_v2_extension *smcd_v2_ext;
1686         struct smc_clc_v2_extension *smc_v2_ext;
1687         struct smc_clc_msg_smcd *pclc_smcd;
1688         unsigned int matches = 0;
1689         u8 smcd_version;
1690         u8 *eid = NULL;
1691         int i, rc;
1692
1693         if (!(ini->smcd_version & SMC_V2) || !smcd_indicated(ini->smc_type_v2))
1694                 goto not_found;
1695
1696         pclc_smcd = smc_get_clc_msg_smcd(pclc);
1697         smc_v2_ext = smc_get_clc_v2_ext(pclc);
1698         smcd_v2_ext = smc_get_clc_smcd_v2_ext(smc_v2_ext);
1699
1700         mutex_lock(&smcd_dev_list.mutex);
1701         if (pclc_smcd->ism.chid)
1702                 /* check for ISM device matching proposed native ISM device */
1703                 smc_check_ism_v2_match(ini, ntohs(pclc_smcd->ism.chid),
1704                                        ntohll(pclc_smcd->ism.gid), &matches);
1705         for (i = 1; i <= smc_v2_ext->hdr.ism_gid_cnt; i++) {
1706                 /* check for ISM devices matching proposed non-native ISM
1707                  * devices
1708                  */
1709                 smc_check_ism_v2_match(ini,
1710                                        ntohs(smcd_v2_ext->gidchid[i - 1].chid),
1711                                        ntohll(smcd_v2_ext->gidchid[i - 1].gid),
1712                                        &matches);
1713         }
1714         mutex_unlock(&smcd_dev_list.mutex);
1715
1716         if (!ini->ism_dev[0]) {
1717                 smc_find_ism_store_rc(SMC_CLC_DECL_NOSMCD2DEV, ini);
1718                 goto not_found;
1719         }
1720
1721         smc_ism_get_system_eid(&eid);
1722         if (!smc_clc_match_eid(ini->negotiated_eid, smc_v2_ext,
1723                                smcd_v2_ext->system_eid, eid))
1724                 goto not_found;
1725
1726         /* separate - outside the smcd_dev_list.lock */
1727         smcd_version = ini->smcd_version;
1728         for (i = 0; i < matches; i++) {
1729                 ini->smcd_version = SMC_V2;
1730                 ini->is_smcd = true;
1731                 ini->ism_selected = i;
1732                 rc = smc_listen_ism_init(new_smc, ini);
1733                 if (rc) {
1734                         smc_find_ism_store_rc(rc, ini);
1735                         /* try next active ISM device */
1736                         continue;
1737                 }
1738                 return; /* matching and usable V2 ISM device found */
1739         }
1740         /* no V2 ISM device could be initialized */
1741         ini->smcd_version = smcd_version;       /* restore original value */
1742         ini->negotiated_eid[0] = 0;
1743
1744 not_found:
1745         ini->smcd_version &= ~SMC_V2;
1746         ini->ism_dev[0] = NULL;
1747         ini->is_smcd = false;
1748 }
1749
1750 static void smc_find_ism_v1_device_serv(struct smc_sock *new_smc,
1751                                         struct smc_clc_msg_proposal *pclc,
1752                                         struct smc_init_info *ini)
1753 {
1754         struct smc_clc_msg_smcd *pclc_smcd = smc_get_clc_msg_smcd(pclc);
1755         int rc = 0;
1756
1757         /* check if ISM V1 is available */
1758         if (!(ini->smcd_version & SMC_V1) || !smcd_indicated(ini->smc_type_v1))
1759                 goto not_found;
1760         ini->is_smcd = true; /* prepare ISM check */
1761         ini->ism_peer_gid[0] = ntohll(pclc_smcd->ism.gid);
1762         rc = smc_find_ism_device(new_smc, ini);
1763         if (rc)
1764                 goto not_found;
1765         ini->ism_selected = 0;
1766         rc = smc_listen_ism_init(new_smc, ini);
1767         if (!rc)
1768                 return;         /* V1 ISM device found */
1769
1770 not_found:
1771         smc_find_ism_store_rc(rc, ini);
1772         ini->smcd_version &= ~SMC_V1;
1773         ini->ism_dev[0] = NULL;
1774         ini->is_smcd = false;
1775 }
1776
1777 /* listen worker: register buffers */
1778 static int smc_listen_rdma_reg(struct smc_sock *new_smc, bool local_first)
1779 {
1780         struct smc_connection *conn = &new_smc->conn;
1781
1782         if (!local_first) {
1783                 if (smcr_lgr_reg_rmbs(conn->lnk, conn->rmb_desc))
1784                         return SMC_CLC_DECL_ERR_REGRMB;
1785         }
1786         smc_rmb_sync_sg_for_device(&new_smc->conn);
1787
1788         return 0;
1789 }
1790
1791 static void smc_find_rdma_v2_device_serv(struct smc_sock *new_smc,
1792                                          struct smc_clc_msg_proposal *pclc,
1793                                          struct smc_init_info *ini)
1794 {
1795         struct smc_clc_v2_extension *smc_v2_ext;
1796         u8 smcr_version;
1797         int rc;
1798
1799         if (!(ini->smcr_version & SMC_V2) || !smcr_indicated(ini->smc_type_v2))
1800                 goto not_found;
1801
1802         smc_v2_ext = smc_get_clc_v2_ext(pclc);
1803         if (!smc_clc_match_eid(ini->negotiated_eid, smc_v2_ext, NULL, NULL))
1804                 goto not_found;
1805
1806         /* prepare RDMA check */
1807         memcpy(ini->peer_systemid, pclc->lcl.id_for_peer, SMC_SYSTEMID_LEN);
1808         memcpy(ini->peer_gid, smc_v2_ext->roce, SMC_GID_SIZE);
1809         memcpy(ini->peer_mac, pclc->lcl.mac, ETH_ALEN);
1810         ini->check_smcrv2 = true;
1811         ini->smcrv2.clc_sk = new_smc->clcsock->sk;
1812         ini->smcrv2.saddr = new_smc->clcsock->sk->sk_rcv_saddr;
1813         ini->smcrv2.daddr = smc_ib_gid_to_ipv4(smc_v2_ext->roce);
1814         rc = smc_find_rdma_device(new_smc, ini);
1815         if (rc) {
1816                 smc_find_ism_store_rc(rc, ini);
1817                 goto not_found;
1818         }
1819         if (!ini->smcrv2.uses_gateway)
1820                 memcpy(ini->smcrv2.nexthop_mac, pclc->lcl.mac, ETH_ALEN);
1821
1822         smcr_version = ini->smcr_version;
1823         ini->smcr_version = SMC_V2;
1824         rc = smc_listen_rdma_init(new_smc, ini);
1825         if (!rc)
1826                 rc = smc_listen_rdma_reg(new_smc, ini->first_contact_local);
1827         if (!rc)
1828                 return;
1829         ini->smcr_version = smcr_version;
1830         smc_find_ism_store_rc(rc, ini);
1831
1832 not_found:
1833         ini->smcr_version &= ~SMC_V2;
1834         ini->check_smcrv2 = false;
1835 }
1836
1837 static int smc_find_rdma_v1_device_serv(struct smc_sock *new_smc,
1838                                         struct smc_clc_msg_proposal *pclc,
1839                                         struct smc_init_info *ini)
1840 {
1841         int rc;
1842
1843         if (!(ini->smcr_version & SMC_V1) || !smcr_indicated(ini->smc_type_v1))
1844                 return SMC_CLC_DECL_NOSMCDEV;
1845
1846         /* prepare RDMA check */
1847         memcpy(ini->peer_systemid, pclc->lcl.id_for_peer, SMC_SYSTEMID_LEN);
1848         memcpy(ini->peer_gid, pclc->lcl.gid, SMC_GID_SIZE);
1849         memcpy(ini->peer_mac, pclc->lcl.mac, ETH_ALEN);
1850         rc = smc_find_rdma_device(new_smc, ini);
1851         if (rc) {
1852                 /* no RDMA device found */
1853                 return SMC_CLC_DECL_NOSMCDEV;
1854         }
1855         rc = smc_listen_rdma_init(new_smc, ini);
1856         if (rc)
1857                 return rc;
1858         return smc_listen_rdma_reg(new_smc, ini->first_contact_local);
1859 }
1860
1861 /* determine the local device matching to proposal */
1862 static int smc_listen_find_device(struct smc_sock *new_smc,
1863                                   struct smc_clc_msg_proposal *pclc,
1864                                   struct smc_init_info *ini)
1865 {
1866         int prfx_rc;
1867
1868         /* check for ISM device matching V2 proposed device */
1869         smc_find_ism_v2_device_serv(new_smc, pclc, ini);
1870         if (ini->ism_dev[0])
1871                 return 0;
1872
1873         /* check for matching IP prefix and subnet length (V1) */
1874         prfx_rc = smc_listen_prfx_check(new_smc, pclc);
1875         if (prfx_rc)
1876                 smc_find_ism_store_rc(prfx_rc, ini);
1877
1878         /* get vlan id from IP device */
1879         if (smc_vlan_by_tcpsk(new_smc->clcsock, ini))
1880                 return ini->rc ?: SMC_CLC_DECL_GETVLANERR;
1881
1882         /* check for ISM device matching V1 proposed device */
1883         if (!prfx_rc)
1884                 smc_find_ism_v1_device_serv(new_smc, pclc, ini);
1885         if (ini->ism_dev[0])
1886                 return 0;
1887
1888         if (!smcr_indicated(pclc->hdr.typev1) &&
1889             !smcr_indicated(pclc->hdr.typev2))
1890                 /* skip RDMA and decline */
1891                 return ini->rc ?: SMC_CLC_DECL_NOSMCDDEV;
1892
1893         /* check if RDMA V2 is available */
1894         smc_find_rdma_v2_device_serv(new_smc, pclc, ini);
1895         if (ini->smcrv2.ib_dev_v2)
1896                 return 0;
1897
1898         /* check if RDMA V1 is available */
1899         if (!prfx_rc) {
1900                 int rc;
1901
1902                 rc = smc_find_rdma_v1_device_serv(new_smc, pclc, ini);
1903                 smc_find_ism_store_rc(rc, ini);
1904                 return (!rc) ? 0 : ini->rc;
1905         }
1906         return SMC_CLC_DECL_NOSMCDEV;
1907 }
1908
1909 /* listen worker: finish RDMA setup */
1910 static int smc_listen_rdma_finish(struct smc_sock *new_smc,
1911                                   struct smc_clc_msg_accept_confirm *cclc,
1912                                   bool local_first,
1913                                   struct smc_init_info *ini)
1914 {
1915         struct smc_link *link = new_smc->conn.lnk;
1916         int reason_code = 0;
1917
1918         if (local_first)
1919                 smc_link_save_peer_info(link, cclc, ini);
1920
1921         if (smc_rmb_rtoken_handling(&new_smc->conn, link, cclc))
1922                 return SMC_CLC_DECL_ERR_RTOK;
1923
1924         if (local_first) {
1925                 if (smc_ib_ready_link(link))
1926                         return SMC_CLC_DECL_ERR_RDYLNK;
1927                 /* QP confirmation over RoCE fabric */
1928                 smc_llc_flow_initiate(link->lgr, SMC_LLC_FLOW_ADD_LINK);
1929                 reason_code = smcr_serv_conf_first_link(new_smc);
1930                 smc_llc_flow_stop(link->lgr, &link->lgr->llc_flow_lcl);
1931         }
1932         return reason_code;
1933 }
1934
1935 /* setup for connection of server */
1936 static void smc_listen_work(struct work_struct *work)
1937 {
1938         struct smc_sock *new_smc = container_of(work, struct smc_sock,
1939                                                 smc_listen_work);
1940         struct socket *newclcsock = new_smc->clcsock;
1941         struct smc_clc_msg_accept_confirm *cclc;
1942         struct smc_clc_msg_proposal_area *buf;
1943         struct smc_clc_msg_proposal *pclc;
1944         struct smc_init_info *ini = NULL;
1945         u8 proposal_version = SMC_V1;
1946         u8 accept_version;
1947         int rc = 0;
1948
1949         if (new_smc->listen_smc->sk.sk_state != SMC_LISTEN)
1950                 return smc_listen_out_err(new_smc);
1951
1952         if (new_smc->use_fallback) {
1953                 smc_listen_out_connected(new_smc);
1954                 return;
1955         }
1956
1957         /* check if peer is smc capable */
1958         if (!tcp_sk(newclcsock->sk)->syn_smc) {
1959                 smc_switch_to_fallback(new_smc, SMC_CLC_DECL_PEERNOSMC);
1960                 smc_listen_out_connected(new_smc);
1961                 return;
1962         }
1963
1964         /* do inband token exchange -
1965          * wait for and receive SMC Proposal CLC message
1966          */
1967         buf = kzalloc(sizeof(*buf), GFP_KERNEL);
1968         if (!buf) {
1969                 rc = SMC_CLC_DECL_MEM;
1970                 goto out_decl;
1971         }
1972         pclc = (struct smc_clc_msg_proposal *)buf;
1973         rc = smc_clc_wait_msg(new_smc, pclc, sizeof(*buf),
1974                               SMC_CLC_PROPOSAL, CLC_WAIT_TIME);
1975         if (rc)
1976                 goto out_decl;
1977
1978         if (pclc->hdr.version > SMC_V1)
1979                 proposal_version = SMC_V2;
1980
1981         /* IPSec connections opt out of SMC optimizations */
1982         if (using_ipsec(new_smc)) {
1983                 rc = SMC_CLC_DECL_IPSEC;
1984                 goto out_decl;
1985         }
1986
1987         ini = kzalloc(sizeof(*ini), GFP_KERNEL);
1988         if (!ini) {
1989                 rc = SMC_CLC_DECL_MEM;
1990                 goto out_decl;
1991         }
1992
1993         /* initial version checking */
1994         rc = smc_listen_v2_check(new_smc, pclc, ini);
1995         if (rc)
1996                 goto out_decl;
1997
1998         mutex_lock(&smc_server_lgr_pending);
1999         smc_close_init(new_smc);
2000         smc_rx_init(new_smc);
2001         smc_tx_init(new_smc);
2002
2003         /* determine ISM or RoCE device used for connection */
2004         rc = smc_listen_find_device(new_smc, pclc, ini);
2005         if (rc)
2006                 goto out_unlock;
2007
2008         /* send SMC Accept CLC message */
2009         accept_version = ini->is_smcd ? ini->smcd_version : ini->smcr_version;
2010         rc = smc_clc_send_accept(new_smc, ini->first_contact_local,
2011                                  accept_version, ini->negotiated_eid);
2012         if (rc)
2013                 goto out_unlock;
2014
2015         /* SMC-D does not need this lock any more */
2016         if (ini->is_smcd)
2017                 mutex_unlock(&smc_server_lgr_pending);
2018
2019         /* receive SMC Confirm CLC message */
2020         memset(buf, 0, sizeof(*buf));
2021         cclc = (struct smc_clc_msg_accept_confirm *)buf;
2022         rc = smc_clc_wait_msg(new_smc, cclc, sizeof(*buf),
2023                               SMC_CLC_CONFIRM, CLC_WAIT_TIME);
2024         if (rc) {
2025                 if (!ini->is_smcd)
2026                         goto out_unlock;
2027                 goto out_decl;
2028         }
2029
2030         /* finish worker */
2031         if (!ini->is_smcd) {
2032                 rc = smc_listen_rdma_finish(new_smc, cclc,
2033                                             ini->first_contact_local, ini);
2034                 if (rc)
2035                         goto out_unlock;
2036                 mutex_unlock(&smc_server_lgr_pending);
2037         }
2038         smc_conn_save_peer_info(new_smc, cclc);
2039         smc_listen_out_connected(new_smc);
2040         SMC_STAT_SERV_SUCC_INC(sock_net(newclcsock->sk), ini);
2041         goto out_free;
2042
2043 out_unlock:
2044         mutex_unlock(&smc_server_lgr_pending);
2045 out_decl:
2046         smc_listen_decline(new_smc, rc, ini ? ini->first_contact_local : 0,
2047                            proposal_version);
2048 out_free:
2049         kfree(ini);
2050         kfree(buf);
2051 }
2052
2053 static void smc_tcp_listen_work(struct work_struct *work)
2054 {
2055         struct smc_sock *lsmc = container_of(work, struct smc_sock,
2056                                              tcp_listen_work);
2057         struct sock *lsk = &lsmc->sk;
2058         struct smc_sock *new_smc;
2059         int rc = 0;
2060
2061         lock_sock(lsk);
2062         while (lsk->sk_state == SMC_LISTEN) {
2063                 rc = smc_clcsock_accept(lsmc, &new_smc);
2064                 if (rc) /* clcsock accept queue empty or error */
2065                         goto out;
2066                 if (!new_smc)
2067                         continue;
2068
2069                 new_smc->listen_smc = lsmc;
2070                 new_smc->use_fallback = lsmc->use_fallback;
2071                 new_smc->fallback_rsn = lsmc->fallback_rsn;
2072                 sock_hold(lsk); /* sock_put in smc_listen_work */
2073                 INIT_WORK(&new_smc->smc_listen_work, smc_listen_work);
2074                 smc_copy_sock_settings_to_smc(new_smc);
2075                 new_smc->sk.sk_sndbuf = lsmc->sk.sk_sndbuf;
2076                 new_smc->sk.sk_rcvbuf = lsmc->sk.sk_rcvbuf;
2077                 sock_hold(&new_smc->sk); /* sock_put in passive closing */
2078                 if (!queue_work(smc_hs_wq, &new_smc->smc_listen_work))
2079                         sock_put(&new_smc->sk);
2080         }
2081
2082 out:
2083         release_sock(lsk);
2084         sock_put(&lsmc->sk); /* sock_hold in smc_clcsock_data_ready() */
2085 }
2086
2087 static void smc_clcsock_data_ready(struct sock *listen_clcsock)
2088 {
2089         struct smc_sock *lsmc;
2090
2091         lsmc = (struct smc_sock *)
2092                ((uintptr_t)listen_clcsock->sk_user_data & ~SK_USER_DATA_NOCOPY);
2093         if (!lsmc)
2094                 return;
2095         lsmc->clcsk_data_ready(listen_clcsock);
2096         if (lsmc->sk.sk_state == SMC_LISTEN) {
2097                 sock_hold(&lsmc->sk); /* sock_put in smc_tcp_listen_work() */
2098                 if (!queue_work(smc_hs_wq, &lsmc->tcp_listen_work))
2099                         sock_put(&lsmc->sk);
2100         }
2101 }
2102
2103 static int smc_listen(struct socket *sock, int backlog)
2104 {
2105         struct sock *sk = sock->sk;
2106         struct smc_sock *smc;
2107         int rc;
2108
2109         smc = smc_sk(sk);
2110         lock_sock(sk);
2111
2112         rc = -EINVAL;
2113         if ((sk->sk_state != SMC_INIT && sk->sk_state != SMC_LISTEN) ||
2114             smc->connect_nonblock)
2115                 goto out;
2116
2117         rc = 0;
2118         if (sk->sk_state == SMC_LISTEN) {
2119                 sk->sk_max_ack_backlog = backlog;
2120                 goto out;
2121         }
2122         /* some socket options are handled in core, so we could not apply
2123          * them to the clc socket -- copy smc socket options to clc socket
2124          */
2125         smc_copy_sock_settings_to_clc(smc);
2126         if (!smc->use_fallback)
2127                 tcp_sk(smc->clcsock->sk)->syn_smc = 1;
2128
2129         /* save original sk_data_ready function and establish
2130          * smc-specific sk_data_ready function
2131          */
2132         smc->clcsk_data_ready = smc->clcsock->sk->sk_data_ready;
2133         smc->clcsock->sk->sk_data_ready = smc_clcsock_data_ready;
2134         smc->clcsock->sk->sk_user_data =
2135                 (void *)((uintptr_t)smc | SK_USER_DATA_NOCOPY);
2136         rc = kernel_listen(smc->clcsock, backlog);
2137         if (rc)
2138                 goto out;
2139         sk->sk_max_ack_backlog = backlog;
2140         sk->sk_ack_backlog = 0;
2141         sk->sk_state = SMC_LISTEN;
2142
2143 out:
2144         release_sock(sk);
2145         return rc;
2146 }
2147
2148 static int smc_accept(struct socket *sock, struct socket *new_sock,
2149                       int flags, bool kern)
2150 {
2151         struct sock *sk = sock->sk, *nsk;
2152         DECLARE_WAITQUEUE(wait, current);
2153         struct smc_sock *lsmc;
2154         long timeo;
2155         int rc = 0;
2156
2157         lsmc = smc_sk(sk);
2158         sock_hold(sk); /* sock_put below */
2159         lock_sock(sk);
2160
2161         if (lsmc->sk.sk_state != SMC_LISTEN) {
2162                 rc = -EINVAL;
2163                 release_sock(sk);
2164                 goto out;
2165         }
2166
2167         /* Wait for an incoming connection */
2168         timeo = sock_rcvtimeo(sk, flags & O_NONBLOCK);
2169         add_wait_queue_exclusive(sk_sleep(sk), &wait);
2170         while (!(nsk = smc_accept_dequeue(sk, new_sock))) {
2171                 set_current_state(TASK_INTERRUPTIBLE);
2172                 if (!timeo) {
2173                         rc = -EAGAIN;
2174                         break;
2175                 }
2176                 release_sock(sk);
2177                 timeo = schedule_timeout(timeo);
2178                 /* wakeup by sk_data_ready in smc_listen_work() */
2179                 sched_annotate_sleep();
2180                 lock_sock(sk);
2181                 if (signal_pending(current)) {
2182                         rc = sock_intr_errno(timeo);
2183                         break;
2184                 }
2185         }
2186         set_current_state(TASK_RUNNING);
2187         remove_wait_queue(sk_sleep(sk), &wait);
2188
2189         if (!rc)
2190                 rc = sock_error(nsk);
2191         release_sock(sk);
2192         if (rc)
2193                 goto out;
2194
2195         if (lsmc->sockopt_defer_accept && !(flags & O_NONBLOCK)) {
2196                 /* wait till data arrives on the socket */
2197                 timeo = msecs_to_jiffies(lsmc->sockopt_defer_accept *
2198                                                                 MSEC_PER_SEC);
2199                 if (smc_sk(nsk)->use_fallback) {
2200                         struct sock *clcsk = smc_sk(nsk)->clcsock->sk;
2201
2202                         lock_sock(clcsk);
2203                         if (skb_queue_empty(&clcsk->sk_receive_queue))
2204                                 sk_wait_data(clcsk, &timeo, NULL);
2205                         release_sock(clcsk);
2206                 } else if (!atomic_read(&smc_sk(nsk)->conn.bytes_to_rcv)) {
2207                         lock_sock(nsk);
2208                         smc_rx_wait(smc_sk(nsk), &timeo, smc_rx_data_available);
2209                         release_sock(nsk);
2210                 }
2211         }
2212
2213 out:
2214         sock_put(sk); /* sock_hold above */
2215         return rc;
2216 }
2217
2218 static int smc_getname(struct socket *sock, struct sockaddr *addr,
2219                        int peer)
2220 {
2221         struct smc_sock *smc;
2222
2223         if (peer && (sock->sk->sk_state != SMC_ACTIVE) &&
2224             (sock->sk->sk_state != SMC_APPCLOSEWAIT1))
2225                 return -ENOTCONN;
2226
2227         smc = smc_sk(sock->sk);
2228
2229         return smc->clcsock->ops->getname(smc->clcsock, addr, peer);
2230 }
2231
2232 static int smc_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
2233 {
2234         struct sock *sk = sock->sk;
2235         struct smc_sock *smc;
2236         int rc = -EPIPE;
2237
2238         smc = smc_sk(sk);
2239         lock_sock(sk);
2240         if ((sk->sk_state != SMC_ACTIVE) &&
2241             (sk->sk_state != SMC_APPCLOSEWAIT1) &&
2242             (sk->sk_state != SMC_INIT))
2243                 goto out;
2244
2245         if (msg->msg_flags & MSG_FASTOPEN) {
2246                 if (sk->sk_state == SMC_INIT && !smc->connect_nonblock) {
2247                         smc_switch_to_fallback(smc, SMC_CLC_DECL_OPTUNSUPP);
2248                 } else {
2249                         rc = -EINVAL;
2250                         goto out;
2251                 }
2252         }
2253
2254         if (smc->use_fallback) {
2255                 rc = smc->clcsock->ops->sendmsg(smc->clcsock, msg, len);
2256         } else {
2257                 rc = smc_tx_sendmsg(smc, msg, len);
2258                 SMC_STAT_TX_PAYLOAD(smc, len, rc);
2259         }
2260 out:
2261         release_sock(sk);
2262         return rc;
2263 }
2264
2265 static int smc_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
2266                        int flags)
2267 {
2268         struct sock *sk = sock->sk;
2269         struct smc_sock *smc;
2270         int rc = -ENOTCONN;
2271
2272         smc = smc_sk(sk);
2273         lock_sock(sk);
2274         if (sk->sk_state == SMC_CLOSED && (sk->sk_shutdown & RCV_SHUTDOWN)) {
2275                 /* socket was connected before, no more data to read */
2276                 rc = 0;
2277                 goto out;
2278         }
2279         if ((sk->sk_state == SMC_INIT) ||
2280             (sk->sk_state == SMC_LISTEN) ||
2281             (sk->sk_state == SMC_CLOSED))
2282                 goto out;
2283
2284         if (sk->sk_state == SMC_PEERFINCLOSEWAIT) {
2285                 rc = 0;
2286                 goto out;
2287         }
2288
2289         if (smc->use_fallback) {
2290                 rc = smc->clcsock->ops->recvmsg(smc->clcsock, msg, len, flags);
2291         } else {
2292                 msg->msg_namelen = 0;
2293                 rc = smc_rx_recvmsg(smc, msg, NULL, len, flags);
2294                 SMC_STAT_RX_PAYLOAD(smc, rc, rc);
2295         }
2296
2297 out:
2298         release_sock(sk);
2299         return rc;
2300 }
2301
2302 static __poll_t smc_accept_poll(struct sock *parent)
2303 {
2304         struct smc_sock *isk = smc_sk(parent);
2305         __poll_t mask = 0;
2306
2307         spin_lock(&isk->accept_q_lock);
2308         if (!list_empty(&isk->accept_q))
2309                 mask = EPOLLIN | EPOLLRDNORM;
2310         spin_unlock(&isk->accept_q_lock);
2311
2312         return mask;
2313 }
2314
2315 static __poll_t smc_poll(struct file *file, struct socket *sock,
2316                              poll_table *wait)
2317 {
2318         struct sock *sk = sock->sk;
2319         struct smc_sock *smc;
2320         __poll_t mask = 0;
2321
2322         if (!sk)
2323                 return EPOLLNVAL;
2324
2325         smc = smc_sk(sock->sk);
2326         if (smc->use_fallback) {
2327                 /* delegate to CLC child sock */
2328                 mask = smc->clcsock->ops->poll(file, smc->clcsock, wait);
2329                 sk->sk_err = smc->clcsock->sk->sk_err;
2330         } else {
2331                 if (sk->sk_state != SMC_CLOSED)
2332                         sock_poll_wait(file, sock, wait);
2333                 if (sk->sk_err)
2334                         mask |= EPOLLERR;
2335                 if ((sk->sk_shutdown == SHUTDOWN_MASK) ||
2336                     (sk->sk_state == SMC_CLOSED))
2337                         mask |= EPOLLHUP;
2338                 if (sk->sk_state == SMC_LISTEN) {
2339                         /* woken up by sk_data_ready in smc_listen_work() */
2340                         mask |= smc_accept_poll(sk);
2341                 } else if (smc->use_fallback) { /* as result of connect_work()*/
2342                         mask |= smc->clcsock->ops->poll(file, smc->clcsock,
2343                                                            wait);
2344                         sk->sk_err = smc->clcsock->sk->sk_err;
2345                 } else {
2346                         if ((sk->sk_state != SMC_INIT &&
2347                              atomic_read(&smc->conn.sndbuf_space)) ||
2348                             sk->sk_shutdown & SEND_SHUTDOWN) {
2349                                 mask |= EPOLLOUT | EPOLLWRNORM;
2350                         } else {
2351                                 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
2352                                 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
2353                         }
2354                         if (atomic_read(&smc->conn.bytes_to_rcv))
2355                                 mask |= EPOLLIN | EPOLLRDNORM;
2356                         if (sk->sk_shutdown & RCV_SHUTDOWN)
2357                                 mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
2358                         if (sk->sk_state == SMC_APPCLOSEWAIT1)
2359                                 mask |= EPOLLIN;
2360                         if (smc->conn.urg_state == SMC_URG_VALID)
2361                                 mask |= EPOLLPRI;
2362                 }
2363         }
2364
2365         return mask;
2366 }
2367
2368 static int smc_shutdown(struct socket *sock, int how)
2369 {
2370         struct sock *sk = sock->sk;
2371         struct smc_sock *smc;
2372         int rc = -EINVAL;
2373         int rc1 = 0;
2374
2375         smc = smc_sk(sk);
2376
2377         if ((how < SHUT_RD) || (how > SHUT_RDWR))
2378                 return rc;
2379
2380         lock_sock(sk);
2381
2382         rc = -ENOTCONN;
2383         if ((sk->sk_state != SMC_ACTIVE) &&
2384             (sk->sk_state != SMC_PEERCLOSEWAIT1) &&
2385             (sk->sk_state != SMC_PEERCLOSEWAIT2) &&
2386             (sk->sk_state != SMC_APPCLOSEWAIT1) &&
2387             (sk->sk_state != SMC_APPCLOSEWAIT2) &&
2388             (sk->sk_state != SMC_APPFINCLOSEWAIT))
2389                 goto out;
2390         if (smc->use_fallback) {
2391                 rc = kernel_sock_shutdown(smc->clcsock, how);
2392                 sk->sk_shutdown = smc->clcsock->sk->sk_shutdown;
2393                 if (sk->sk_shutdown == SHUTDOWN_MASK)
2394                         sk->sk_state = SMC_CLOSED;
2395                 goto out;
2396         }
2397         switch (how) {
2398         case SHUT_RDWR:         /* shutdown in both directions */
2399                 rc = smc_close_active(smc);
2400                 break;
2401         case SHUT_WR:
2402                 rc = smc_close_shutdown_write(smc);
2403                 break;
2404         case SHUT_RD:
2405                 rc = 0;
2406                 /* nothing more to do because peer is not involved */
2407                 break;
2408         }
2409         if (smc->clcsock)
2410                 rc1 = kernel_sock_shutdown(smc->clcsock, how);
2411         /* map sock_shutdown_cmd constants to sk_shutdown value range */
2412         sk->sk_shutdown |= how + 1;
2413
2414 out:
2415         release_sock(sk);
2416         return rc ? rc : rc1;
2417 }
2418
2419 static int smc_setsockopt(struct socket *sock, int level, int optname,
2420                           sockptr_t optval, unsigned int optlen)
2421 {
2422         struct sock *sk = sock->sk;
2423         struct smc_sock *smc;
2424         int val, rc;
2425
2426         if (level == SOL_TCP && optname == TCP_ULP)
2427                 return -EOPNOTSUPP;
2428
2429         smc = smc_sk(sk);
2430
2431         /* generic setsockopts reaching us here always apply to the
2432          * CLC socket
2433          */
2434         if (unlikely(!smc->clcsock->ops->setsockopt))
2435                 rc = -EOPNOTSUPP;
2436         else
2437                 rc = smc->clcsock->ops->setsockopt(smc->clcsock, level, optname,
2438                                                    optval, optlen);
2439         if (smc->clcsock->sk->sk_err) {
2440                 sk->sk_err = smc->clcsock->sk->sk_err;
2441                 sk_error_report(sk);
2442         }
2443
2444         if (optlen < sizeof(int))
2445                 return -EINVAL;
2446         if (copy_from_sockptr(&val, optval, sizeof(int)))
2447                 return -EFAULT;
2448
2449         lock_sock(sk);
2450         if (rc || smc->use_fallback)
2451                 goto out;
2452         switch (optname) {
2453         case TCP_FASTOPEN:
2454         case TCP_FASTOPEN_CONNECT:
2455         case TCP_FASTOPEN_KEY:
2456         case TCP_FASTOPEN_NO_COOKIE:
2457                 /* option not supported by SMC */
2458                 if (sk->sk_state == SMC_INIT && !smc->connect_nonblock) {
2459                         smc_switch_to_fallback(smc, SMC_CLC_DECL_OPTUNSUPP);
2460                 } else {
2461                         rc = -EINVAL;
2462                 }
2463                 break;
2464         case TCP_NODELAY:
2465                 if (sk->sk_state != SMC_INIT &&
2466                     sk->sk_state != SMC_LISTEN &&
2467                     sk->sk_state != SMC_CLOSED) {
2468                         if (val) {
2469                                 SMC_STAT_INC(smc, ndly_cnt);
2470                                 mod_delayed_work(smc->conn.lgr->tx_wq,
2471                                                  &smc->conn.tx_work, 0);
2472                         }
2473                 }
2474                 break;
2475         case TCP_CORK:
2476                 if (sk->sk_state != SMC_INIT &&
2477                     sk->sk_state != SMC_LISTEN &&
2478                     sk->sk_state != SMC_CLOSED) {
2479                         if (!val) {
2480                                 SMC_STAT_INC(smc, cork_cnt);
2481                                 mod_delayed_work(smc->conn.lgr->tx_wq,
2482                                                  &smc->conn.tx_work, 0);
2483                         }
2484                 }
2485                 break;
2486         case TCP_DEFER_ACCEPT:
2487                 smc->sockopt_defer_accept = val;
2488                 break;
2489         default:
2490                 break;
2491         }
2492 out:
2493         release_sock(sk);
2494
2495         return rc;
2496 }
2497
2498 static int smc_getsockopt(struct socket *sock, int level, int optname,
2499                           char __user *optval, int __user *optlen)
2500 {
2501         struct smc_sock *smc;
2502
2503         smc = smc_sk(sock->sk);
2504         /* socket options apply to the CLC socket */
2505         if (unlikely(!smc->clcsock->ops->getsockopt))
2506                 return -EOPNOTSUPP;
2507         return smc->clcsock->ops->getsockopt(smc->clcsock, level, optname,
2508                                              optval, optlen);
2509 }
2510
2511 static int smc_ioctl(struct socket *sock, unsigned int cmd,
2512                      unsigned long arg)
2513 {
2514         union smc_host_cursor cons, urg;
2515         struct smc_connection *conn;
2516         struct smc_sock *smc;
2517         int answ;
2518
2519         smc = smc_sk(sock->sk);
2520         conn = &smc->conn;
2521         lock_sock(&smc->sk);
2522         if (smc->use_fallback) {
2523                 if (!smc->clcsock) {
2524                         release_sock(&smc->sk);
2525                         return -EBADF;
2526                 }
2527                 answ = smc->clcsock->ops->ioctl(smc->clcsock, cmd, arg);
2528                 release_sock(&smc->sk);
2529                 return answ;
2530         }
2531         switch (cmd) {
2532         case SIOCINQ: /* same as FIONREAD */
2533                 if (smc->sk.sk_state == SMC_LISTEN) {
2534                         release_sock(&smc->sk);
2535                         return -EINVAL;
2536                 }
2537                 if (smc->sk.sk_state == SMC_INIT ||
2538                     smc->sk.sk_state == SMC_CLOSED)
2539                         answ = 0;
2540                 else
2541                         answ = atomic_read(&smc->conn.bytes_to_rcv);
2542                 break;
2543         case SIOCOUTQ:
2544                 /* output queue size (not send + not acked) */
2545                 if (smc->sk.sk_state == SMC_LISTEN) {
2546                         release_sock(&smc->sk);
2547                         return -EINVAL;
2548                 }
2549                 if (smc->sk.sk_state == SMC_INIT ||
2550                     smc->sk.sk_state == SMC_CLOSED)
2551                         answ = 0;
2552                 else
2553                         answ = smc->conn.sndbuf_desc->len -
2554                                         atomic_read(&smc->conn.sndbuf_space);
2555                 break;
2556         case SIOCOUTQNSD:
2557                 /* output queue size (not send only) */
2558                 if (smc->sk.sk_state == SMC_LISTEN) {
2559                         release_sock(&smc->sk);
2560                         return -EINVAL;
2561                 }
2562                 if (smc->sk.sk_state == SMC_INIT ||
2563                     smc->sk.sk_state == SMC_CLOSED)
2564                         answ = 0;
2565                 else
2566                         answ = smc_tx_prepared_sends(&smc->conn);
2567                 break;
2568         case SIOCATMARK:
2569                 if (smc->sk.sk_state == SMC_LISTEN) {
2570                         release_sock(&smc->sk);
2571                         return -EINVAL;
2572                 }
2573                 if (smc->sk.sk_state == SMC_INIT ||
2574                     smc->sk.sk_state == SMC_CLOSED) {
2575                         answ = 0;
2576                 } else {
2577                         smc_curs_copy(&cons, &conn->local_tx_ctrl.cons, conn);
2578                         smc_curs_copy(&urg, &conn->urg_curs, conn);
2579                         answ = smc_curs_diff(conn->rmb_desc->len,
2580                                              &cons, &urg) == 1;
2581                 }
2582                 break;
2583         default:
2584                 release_sock(&smc->sk);
2585                 return -ENOIOCTLCMD;
2586         }
2587         release_sock(&smc->sk);
2588
2589         return put_user(answ, (int __user *)arg);
2590 }
2591
2592 static ssize_t smc_sendpage(struct socket *sock, struct page *page,
2593                             int offset, size_t size, int flags)
2594 {
2595         struct sock *sk = sock->sk;
2596         struct smc_sock *smc;
2597         int rc = -EPIPE;
2598
2599         smc = smc_sk(sk);
2600         lock_sock(sk);
2601         if (sk->sk_state != SMC_ACTIVE) {
2602                 release_sock(sk);
2603                 goto out;
2604         }
2605         release_sock(sk);
2606         if (smc->use_fallback) {
2607                 rc = kernel_sendpage(smc->clcsock, page, offset,
2608                                      size, flags);
2609         } else {
2610                 SMC_STAT_INC(smc, sendpage_cnt);
2611                 rc = sock_no_sendpage(sock, page, offset, size, flags);
2612         }
2613
2614 out:
2615         return rc;
2616 }
2617
2618 /* Map the affected portions of the rmbe into an spd, note the number of bytes
2619  * to splice in conn->splice_pending, and press 'go'. Delays consumer cursor
2620  * updates till whenever a respective page has been fully processed.
2621  * Note that subsequent recv() calls have to wait till all splice() processing
2622  * completed.
2623  */
2624 static ssize_t smc_splice_read(struct socket *sock, loff_t *ppos,
2625                                struct pipe_inode_info *pipe, size_t len,
2626                                unsigned int flags)
2627 {
2628         struct sock *sk = sock->sk;
2629         struct smc_sock *smc;
2630         int rc = -ENOTCONN;
2631
2632         smc = smc_sk(sk);
2633         lock_sock(sk);
2634         if (sk->sk_state == SMC_CLOSED && (sk->sk_shutdown & RCV_SHUTDOWN)) {
2635                 /* socket was connected before, no more data to read */
2636                 rc = 0;
2637                 goto out;
2638         }
2639         if (sk->sk_state == SMC_INIT ||
2640             sk->sk_state == SMC_LISTEN ||
2641             sk->sk_state == SMC_CLOSED)
2642                 goto out;
2643
2644         if (sk->sk_state == SMC_PEERFINCLOSEWAIT) {
2645                 rc = 0;
2646                 goto out;
2647         }
2648
2649         if (smc->use_fallback) {
2650                 rc = smc->clcsock->ops->splice_read(smc->clcsock, ppos,
2651                                                     pipe, len, flags);
2652         } else {
2653                 if (*ppos) {
2654                         rc = -ESPIPE;
2655                         goto out;
2656                 }
2657                 if (flags & SPLICE_F_NONBLOCK)
2658                         flags = MSG_DONTWAIT;
2659                 else
2660                         flags = 0;
2661                 SMC_STAT_INC(smc, splice_cnt);
2662                 rc = smc_rx_recvmsg(smc, NULL, pipe, len, flags);
2663         }
2664 out:
2665         release_sock(sk);
2666
2667         return rc;
2668 }
2669
2670 /* must look like tcp */
2671 static const struct proto_ops smc_sock_ops = {
2672         .family         = PF_SMC,
2673         .owner          = THIS_MODULE,
2674         .release        = smc_release,
2675         .bind           = smc_bind,
2676         .connect        = smc_connect,
2677         .socketpair     = sock_no_socketpair,
2678         .accept         = smc_accept,
2679         .getname        = smc_getname,
2680         .poll           = smc_poll,
2681         .ioctl          = smc_ioctl,
2682         .listen         = smc_listen,
2683         .shutdown       = smc_shutdown,
2684         .setsockopt     = smc_setsockopt,
2685         .getsockopt     = smc_getsockopt,
2686         .sendmsg        = smc_sendmsg,
2687         .recvmsg        = smc_recvmsg,
2688         .mmap           = sock_no_mmap,
2689         .sendpage       = smc_sendpage,
2690         .splice_read    = smc_splice_read,
2691 };
2692
2693 static int smc_create(struct net *net, struct socket *sock, int protocol,
2694                       int kern)
2695 {
2696         int family = (protocol == SMCPROTO_SMC6) ? PF_INET6 : PF_INET;
2697         struct smc_sock *smc;
2698         struct sock *sk;
2699         int rc;
2700
2701         rc = -ESOCKTNOSUPPORT;
2702         if (sock->type != SOCK_STREAM)
2703                 goto out;
2704
2705         rc = -EPROTONOSUPPORT;
2706         if (protocol != SMCPROTO_SMC && protocol != SMCPROTO_SMC6)
2707                 goto out;
2708
2709         rc = -ENOBUFS;
2710         sock->ops = &smc_sock_ops;
2711         sk = smc_sock_alloc(net, sock, protocol);
2712         if (!sk)
2713                 goto out;
2714
2715         /* create internal TCP socket for CLC handshake and fallback */
2716         smc = smc_sk(sk);
2717         smc->use_fallback = false; /* assume rdma capability first */
2718         smc->fallback_rsn = 0;
2719         rc = sock_create_kern(net, family, SOCK_STREAM, IPPROTO_TCP,
2720                               &smc->clcsock);
2721         if (rc) {
2722                 sk_common_release(sk);
2723                 goto out;
2724         }
2725         smc->sk.sk_sndbuf = max(smc->clcsock->sk->sk_sndbuf, SMC_BUF_MIN_SIZE);
2726         smc->sk.sk_rcvbuf = max(smc->clcsock->sk->sk_rcvbuf, SMC_BUF_MIN_SIZE);
2727
2728 out:
2729         return rc;
2730 }
2731
2732 static const struct net_proto_family smc_sock_family_ops = {
2733         .family = PF_SMC,
2734         .owner  = THIS_MODULE,
2735         .create = smc_create,
2736 };
2737
2738 unsigned int smc_net_id;
2739
2740 static __net_init int smc_net_init(struct net *net)
2741 {
2742         return smc_pnet_net_init(net);
2743 }
2744
2745 static void __net_exit smc_net_exit(struct net *net)
2746 {
2747         smc_pnet_net_exit(net);
2748 }
2749
2750 static __net_init int smc_net_stat_init(struct net *net)
2751 {
2752         return smc_stats_init(net);
2753 }
2754
2755 static void __net_exit smc_net_stat_exit(struct net *net)
2756 {
2757         smc_stats_exit(net);
2758 }
2759
2760 static struct pernet_operations smc_net_ops = {
2761         .init = smc_net_init,
2762         .exit = smc_net_exit,
2763         .id   = &smc_net_id,
2764         .size = sizeof(struct smc_net),
2765 };
2766
2767 static struct pernet_operations smc_net_stat_ops = {
2768         .init = smc_net_stat_init,
2769         .exit = smc_net_stat_exit,
2770 };
2771
2772 static int __init smc_init(void)
2773 {
2774         int rc;
2775
2776         rc = register_pernet_subsys(&smc_net_ops);
2777         if (rc)
2778                 return rc;
2779
2780         rc = register_pernet_subsys(&smc_net_stat_ops);
2781         if (rc)
2782                 return rc;
2783
2784         smc_ism_init();
2785         smc_clc_init();
2786
2787         rc = smc_nl_init();
2788         if (rc)
2789                 goto out_pernet_subsys;
2790
2791         rc = smc_pnet_init();
2792         if (rc)
2793                 goto out_nl;
2794
2795         rc = -ENOMEM;
2796         smc_hs_wq = alloc_workqueue("smc_hs_wq", 0, 0);
2797         if (!smc_hs_wq)
2798                 goto out_pnet;
2799
2800         smc_close_wq = alloc_workqueue("smc_close_wq", 0, 0);
2801         if (!smc_close_wq)
2802                 goto out_alloc_hs_wq;
2803
2804         rc = smc_core_init();
2805         if (rc) {
2806                 pr_err("%s: smc_core_init fails with %d\n", __func__, rc);
2807                 goto out_alloc_wqs;
2808         }
2809
2810         rc = smc_llc_init();
2811         if (rc) {
2812                 pr_err("%s: smc_llc_init fails with %d\n", __func__, rc);
2813                 goto out_core;
2814         }
2815
2816         rc = smc_cdc_init();
2817         if (rc) {
2818                 pr_err("%s: smc_cdc_init fails with %d\n", __func__, rc);
2819                 goto out_core;
2820         }
2821
2822         rc = proto_register(&smc_proto, 1);
2823         if (rc) {
2824                 pr_err("%s: proto_register(v4) fails with %d\n", __func__, rc);
2825                 goto out_core;
2826         }
2827
2828         rc = proto_register(&smc_proto6, 1);
2829         if (rc) {
2830                 pr_err("%s: proto_register(v6) fails with %d\n", __func__, rc);
2831                 goto out_proto;
2832         }
2833
2834         rc = sock_register(&smc_sock_family_ops);
2835         if (rc) {
2836                 pr_err("%s: sock_register fails with %d\n", __func__, rc);
2837                 goto out_proto6;
2838         }
2839         INIT_HLIST_HEAD(&smc_v4_hashinfo.ht);
2840         INIT_HLIST_HEAD(&smc_v6_hashinfo.ht);
2841
2842         rc = smc_ib_register_client();
2843         if (rc) {
2844                 pr_err("%s: ib_register fails with %d\n", __func__, rc);
2845                 goto out_sock;
2846         }
2847
2848         static_branch_enable(&tcp_have_smc);
2849         return 0;
2850
2851 out_sock:
2852         sock_unregister(PF_SMC);
2853 out_proto6:
2854         proto_unregister(&smc_proto6);
2855 out_proto:
2856         proto_unregister(&smc_proto);
2857 out_core:
2858         smc_core_exit();
2859 out_alloc_wqs:
2860         destroy_workqueue(smc_close_wq);
2861 out_alloc_hs_wq:
2862         destroy_workqueue(smc_hs_wq);
2863 out_pnet:
2864         smc_pnet_exit();
2865 out_nl:
2866         smc_nl_exit();
2867 out_pernet_subsys:
2868         unregister_pernet_subsys(&smc_net_ops);
2869
2870         return rc;
2871 }
2872
2873 static void __exit smc_exit(void)
2874 {
2875         static_branch_disable(&tcp_have_smc);
2876         sock_unregister(PF_SMC);
2877         smc_core_exit();
2878         smc_ib_unregister_client();
2879         destroy_workqueue(smc_close_wq);
2880         destroy_workqueue(smc_hs_wq);
2881         proto_unregister(&smc_proto6);
2882         proto_unregister(&smc_proto);
2883         smc_pnet_exit();
2884         smc_nl_exit();
2885         smc_clc_exit();
2886         unregister_pernet_subsys(&smc_net_stat_ops);
2887         unregister_pernet_subsys(&smc_net_ops);
2888         rcu_barrier();
2889 }
2890
2891 module_init(smc_init);
2892 module_exit(smc_exit);
2893
2894 MODULE_AUTHOR("Ursula Braun <ubraun@linux.vnet.ibm.com>");
2895 MODULE_DESCRIPTION("smc socket address family");
2896 MODULE_LICENSE("GPL");
2897 MODULE_ALIAS_NETPROTO(PF_SMC);