2 * Shared Memory Communications over RDMA (SMC-R) and RoCE
4 * AF_SMC protocol family socket handler keeping the AF_INET sock address type
5 * applies to SOCK_STREAM sockets only
6 * offers an alternative communication option for TCP-protocol sockets
7 * applicable with RoCE-cards only
9 * Initial restrictions:
10 * - support for alternate links postponed
12 * Copyright IBM Corp. 2016, 2018
14 * Author(s): Ursula Braun <ubraun@linux.vnet.ibm.com>
15 * based on prototype from Frank Blaschka
18 #define KMSG_COMPONENT "smc"
19 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
21 #include <linux/module.h>
22 #include <linux/socket.h>
23 #include <linux/workqueue.h>
25 #include <linux/sched/signal.h>
30 #include <asm/ioctls.h>
41 #include "smc_close.h"
43 static DEFINE_MUTEX(smc_create_lgr_pending); /* serialize link group
47 static void smc_tcp_listen_work(struct work_struct *);
49 static void smc_set_keepalive(struct sock *sk, int val)
51 struct smc_sock *smc = smc_sk(sk);
53 smc->clcsock->sk->sk_prot->keepalive(smc->clcsock->sk, val);
56 static struct smc_hashinfo smc_v4_hashinfo = {
57 .lock = __RW_LOCK_UNLOCKED(smc_v4_hashinfo.lock),
60 static struct smc_hashinfo smc_v6_hashinfo = {
61 .lock = __RW_LOCK_UNLOCKED(smc_v6_hashinfo.lock),
64 int smc_hash_sk(struct sock *sk)
66 struct smc_hashinfo *h = sk->sk_prot->h.smc_hash;
67 struct hlist_head *head;
71 write_lock_bh(&h->lock);
72 sk_add_node(sk, head);
73 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
74 write_unlock_bh(&h->lock);
78 EXPORT_SYMBOL_GPL(smc_hash_sk);
80 void smc_unhash_sk(struct sock *sk)
82 struct smc_hashinfo *h = sk->sk_prot->h.smc_hash;
84 write_lock_bh(&h->lock);
85 if (sk_del_node_init(sk))
86 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
87 write_unlock_bh(&h->lock);
89 EXPORT_SYMBOL_GPL(smc_unhash_sk);
91 struct proto smc_proto = {
94 .keepalive = smc_set_keepalive,
96 .unhash = smc_unhash_sk,
97 .obj_size = sizeof(struct smc_sock),
98 .h.smc_hash = &smc_v4_hashinfo,
99 .slab_flags = SLAB_TYPESAFE_BY_RCU,
101 EXPORT_SYMBOL_GPL(smc_proto);
103 struct proto smc_proto6 = {
105 .owner = THIS_MODULE,
106 .keepalive = smc_set_keepalive,
108 .unhash = smc_unhash_sk,
109 .obj_size = sizeof(struct smc_sock),
110 .h.smc_hash = &smc_v6_hashinfo,
111 .slab_flags = SLAB_TYPESAFE_BY_RCU,
113 EXPORT_SYMBOL_GPL(smc_proto6);
115 static int smc_release(struct socket *sock)
117 struct sock *sk = sock->sk;
118 struct smc_sock *smc;
125 if (sk->sk_state == SMC_LISTEN)
126 /* smc_close_non_accepted() is called and acquires
127 * sock lock for child sockets again
129 lock_sock_nested(sk, SINGLE_DEPTH_NESTING);
133 if (!smc->use_fallback) {
134 rc = smc_close_active(smc);
135 sock_set_flag(sk, SOCK_DEAD);
136 sk->sk_shutdown |= SHUTDOWN_MASK;
139 sock_release(smc->clcsock);
142 if (smc->use_fallback) {
143 sock_put(sk); /* passive closing */
144 sk->sk_state = SMC_CLOSED;
145 sk->sk_state_change(sk);
151 if (!smc->use_fallback && sk->sk_state == SMC_CLOSED)
152 smc_conn_free(&smc->conn);
155 sk->sk_prot->unhash(sk);
156 sock_put(sk); /* final sock_put */
161 static void smc_destruct(struct sock *sk)
163 if (sk->sk_state != SMC_CLOSED)
165 if (!sock_flag(sk, SOCK_DEAD))
168 sk_refcnt_debug_dec(sk);
171 static struct sock *smc_sock_alloc(struct net *net, struct socket *sock,
174 struct smc_sock *smc;
178 prot = (protocol == SMCPROTO_SMC6) ? &smc_proto6 : &smc_proto;
179 sk = sk_alloc(net, PF_SMC, GFP_KERNEL, prot, 0);
183 sock_init_data(sock, sk); /* sets sk_refcnt to 1 */
184 sk->sk_state = SMC_INIT;
185 sk->sk_destruct = smc_destruct;
186 sk->sk_protocol = protocol;
188 INIT_WORK(&smc->tcp_listen_work, smc_tcp_listen_work);
189 INIT_DELAYED_WORK(&smc->conn.tx_work, smc_tx_work);
190 INIT_LIST_HEAD(&smc->accept_q);
191 spin_lock_init(&smc->accept_q_lock);
192 spin_lock_init(&smc->conn.send_lock);
193 sk->sk_prot->hash(sk);
194 sk_refcnt_debug_inc(sk);
199 static int smc_bind(struct socket *sock, struct sockaddr *uaddr,
202 struct sockaddr_in *addr = (struct sockaddr_in *)uaddr;
203 struct sock *sk = sock->sk;
204 struct smc_sock *smc;
209 /* replicate tests from inet_bind(), to be safe wrt. future changes */
211 if (addr_len < sizeof(struct sockaddr_in))
215 if (addr->sin_family != AF_INET &&
216 addr->sin_family != AF_INET6 &&
217 addr->sin_family != AF_UNSPEC)
219 /* accept AF_UNSPEC (mapped to AF_INET) only if s_addr is INADDR_ANY */
220 if (addr->sin_family == AF_UNSPEC &&
221 addr->sin_addr.s_addr != htonl(INADDR_ANY))
226 /* Check if socket is already active */
228 if (sk->sk_state != SMC_INIT)
231 smc->clcsock->sk->sk_reuse = sk->sk_reuse;
232 rc = kernel_bind(smc->clcsock, uaddr, addr_len);
240 static void smc_copy_sock_settings(struct sock *nsk, struct sock *osk,
243 /* options we don't get control via setsockopt for */
244 nsk->sk_type = osk->sk_type;
245 nsk->sk_sndbuf = osk->sk_sndbuf;
246 nsk->sk_rcvbuf = osk->sk_rcvbuf;
247 nsk->sk_sndtimeo = osk->sk_sndtimeo;
248 nsk->sk_rcvtimeo = osk->sk_rcvtimeo;
249 nsk->sk_mark = osk->sk_mark;
250 nsk->sk_priority = osk->sk_priority;
251 nsk->sk_rcvlowat = osk->sk_rcvlowat;
252 nsk->sk_bound_dev_if = osk->sk_bound_dev_if;
253 nsk->sk_err = osk->sk_err;
255 nsk->sk_flags &= ~mask;
256 nsk->sk_flags |= osk->sk_flags & mask;
259 #define SK_FLAGS_SMC_TO_CLC ((1UL << SOCK_URGINLINE) | \
260 (1UL << SOCK_KEEPOPEN) | \
261 (1UL << SOCK_LINGER) | \
262 (1UL << SOCK_BROADCAST) | \
263 (1UL << SOCK_TIMESTAMP) | \
264 (1UL << SOCK_DBG) | \
265 (1UL << SOCK_RCVTSTAMP) | \
266 (1UL << SOCK_RCVTSTAMPNS) | \
267 (1UL << SOCK_LOCALROUTE) | \
268 (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE) | \
269 (1UL << SOCK_RXQ_OVFL) | \
270 (1UL << SOCK_WIFI_STATUS) | \
271 (1UL << SOCK_NOFCS) | \
272 (1UL << SOCK_FILTER_LOCKED))
273 /* copy only relevant settings and flags of SOL_SOCKET level from smc to
274 * clc socket (since smc is not called for these options from net/core)
276 static void smc_copy_sock_settings_to_clc(struct smc_sock *smc)
278 smc_copy_sock_settings(smc->clcsock->sk, &smc->sk, SK_FLAGS_SMC_TO_CLC);
281 #define SK_FLAGS_CLC_TO_SMC ((1UL << SOCK_URGINLINE) | \
282 (1UL << SOCK_KEEPOPEN) | \
283 (1UL << SOCK_LINGER) | \
285 /* copy only settings and flags relevant for smc from clc to smc socket */
286 static void smc_copy_sock_settings_to_smc(struct smc_sock *smc)
288 smc_copy_sock_settings(&smc->sk, smc->clcsock->sk, SK_FLAGS_CLC_TO_SMC);
291 /* register a new rmb, optionally send confirm_rkey msg to register with peer */
292 static int smc_reg_rmb(struct smc_link *link, struct smc_buf_desc *rmb_desc,
295 /* register memory region for new rmb */
296 if (smc_wr_reg_send(link, rmb_desc->mr_rx[SMC_SINGLE_LINK])) {
297 rmb_desc->regerr = 1;
302 /* exchange confirm_rkey msg with peer */
303 if (smc_llc_do_confirm_rkey(link, rmb_desc)) {
304 rmb_desc->regerr = 1;
310 static int smc_clnt_conf_first_link(struct smc_sock *smc)
312 struct net *net = sock_net(smc->clcsock->sk);
313 struct smc_link_group *lgr = smc->conn.lgr;
314 struct smc_link *link;
318 link = &lgr->lnk[SMC_SINGLE_LINK];
319 /* receive CONFIRM LINK request from server over RoCE fabric */
320 rest = wait_for_completion_interruptible_timeout(
322 SMC_LLC_WAIT_FIRST_TIME);
324 struct smc_clc_msg_decline dclc;
326 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
331 if (link->llc_confirm_rc)
332 return SMC_CLC_DECL_RMBE_EC;
334 rc = smc_ib_modify_qp_rts(link);
336 return SMC_CLC_DECL_INTERR;
338 smc_wr_remember_qp_attr(link);
340 if (smc_reg_rmb(link, smc->conn.rmb_desc, false))
341 return SMC_CLC_DECL_INTERR;
343 /* send CONFIRM LINK response over RoCE fabric */
344 rc = smc_llc_send_confirm_link(link,
345 link->smcibdev->mac[link->ibport - 1],
346 &link->smcibdev->gid[link->ibport - 1],
349 return SMC_CLC_DECL_TCL;
351 /* receive ADD LINK request from server over RoCE fabric */
352 rest = wait_for_completion_interruptible_timeout(&link->llc_add,
355 struct smc_clc_msg_decline dclc;
357 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
362 /* send add link reject message, only one link supported for now */
363 rc = smc_llc_send_add_link(link,
364 link->smcibdev->mac[link->ibport - 1],
365 &link->smcibdev->gid[link->ibport - 1],
368 return SMC_CLC_DECL_TCL;
370 smc_llc_link_active(link, net->ipv4.sysctl_tcp_keepalive_time);
375 static void smc_conn_save_peer_info(struct smc_sock *smc,
376 struct smc_clc_msg_accept_confirm *clc)
378 int bufsize = smc_uncompress_bufsize(clc->rmbe_size);
380 smc->conn.peer_rmbe_idx = clc->rmbe_idx;
381 smc->conn.local_tx_ctrl.token = ntohl(clc->rmbe_alert_token);
382 smc->conn.peer_rmbe_size = bufsize;
383 atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size);
384 smc->conn.tx_off = bufsize * (smc->conn.peer_rmbe_idx - 1);
387 static void smc_link_save_peer_info(struct smc_link *link,
388 struct smc_clc_msg_accept_confirm *clc)
390 link->peer_qpn = ntoh24(clc->qpn);
391 memcpy(link->peer_gid, clc->lcl.gid, SMC_GID_SIZE);
392 memcpy(link->peer_mac, clc->lcl.mac, sizeof(link->peer_mac));
393 link->peer_psn = ntoh24(clc->psn);
394 link->peer_mtu = clc->qp_mtu;
397 /* fall back during connect */
398 static int smc_connect_fallback(struct smc_sock *smc)
400 smc->use_fallback = true;
401 smc_copy_sock_settings_to_clc(smc);
402 if (smc->sk.sk_state == SMC_INIT)
403 smc->sk.sk_state = SMC_ACTIVE;
407 /* decline and fall back during connect */
408 static int smc_connect_decline_fallback(struct smc_sock *smc, int reason_code)
412 if (reason_code < 0) /* error, fallback is not possible */
414 if (reason_code != SMC_CLC_DECL_REPLY) {
415 rc = smc_clc_send_decline(smc, reason_code);
419 return smc_connect_fallback(smc);
422 /* abort connecting */
423 static int smc_connect_abort(struct smc_sock *smc, int reason_code,
426 if (local_contact == SMC_FIRST_CONTACT)
427 smc_lgr_forget(smc->conn.lgr);
428 mutex_unlock(&smc_create_lgr_pending);
429 smc_conn_free(&smc->conn);
430 if (reason_code < 0 && smc->sk.sk_state == SMC_INIT)
431 sock_put(&smc->sk); /* passive closing */
435 /* check if there is a rdma device available for this connection. */
436 /* called for connect and listen */
437 static int smc_check_rdma(struct smc_sock *smc, struct smc_ib_device **ibdev,
442 /* PNET table look up: search active ib_device and port
443 * within same PNETID that also contains the ethernet device
444 * used for the internal TCP socket
446 smc_pnet_find_roce_resource(smc->clcsock->sk, ibdev, ibport);
448 reason_code = SMC_CLC_DECL_CNFERR; /* configuration error */
453 /* CLC handshake during connect */
454 static int smc_connect_clc(struct smc_sock *smc,
455 struct smc_clc_msg_accept_confirm *aclc,
456 struct smc_ib_device *ibdev, u8 ibport)
460 /* do inband token exchange */
461 rc = smc_clc_send_proposal(smc, ibdev, ibport);
464 /* receive SMC Accept CLC message */
465 return smc_clc_wait_msg(smc, aclc, sizeof(*aclc), SMC_CLC_ACCEPT);
468 /* setup for RDMA connection of client */
469 static int smc_connect_rdma(struct smc_sock *smc,
470 struct smc_clc_msg_accept_confirm *aclc,
471 struct smc_ib_device *ibdev, u8 ibport)
473 int local_contact = SMC_FIRST_CONTACT;
474 struct smc_link *link;
477 mutex_lock(&smc_create_lgr_pending);
478 local_contact = smc_conn_create(smc, ibdev, ibport, &aclc->lcl,
480 if (local_contact < 0) {
481 if (local_contact == -ENOMEM)
482 reason_code = SMC_CLC_DECL_MEM;/* insufficient memory*/
483 else if (local_contact == -ENOLINK)
484 reason_code = SMC_CLC_DECL_SYNCERR; /* synchr. error */
486 reason_code = SMC_CLC_DECL_INTERR; /* other error */
487 return smc_connect_abort(smc, reason_code, 0);
489 link = &smc->conn.lgr->lnk[SMC_SINGLE_LINK];
491 smc_conn_save_peer_info(smc, aclc);
493 /* create send buffer and rmb */
494 if (smc_buf_create(smc))
495 return smc_connect_abort(smc, SMC_CLC_DECL_MEM, local_contact);
497 if (local_contact == SMC_FIRST_CONTACT)
498 smc_link_save_peer_info(link, aclc);
500 if (smc_rmb_rtoken_handling(&smc->conn, aclc))
501 return smc_connect_abort(smc, SMC_CLC_DECL_INTERR,
507 if (local_contact == SMC_FIRST_CONTACT) {
508 if (smc_ib_ready_link(link))
509 return smc_connect_abort(smc, SMC_CLC_DECL_INTERR,
512 if (!smc->conn.rmb_desc->reused &&
513 smc_reg_rmb(link, smc->conn.rmb_desc, true))
514 return smc_connect_abort(smc, SMC_CLC_DECL_INTERR,
517 smc_rmb_sync_sg_for_device(&smc->conn);
519 reason_code = smc_clc_send_confirm(smc);
521 return smc_connect_abort(smc, reason_code, local_contact);
525 if (local_contact == SMC_FIRST_CONTACT) {
526 /* QP confirmation over RoCE fabric */
527 reason_code = smc_clnt_conf_first_link(smc);
529 return smc_connect_abort(smc, reason_code,
532 mutex_unlock(&smc_create_lgr_pending);
534 smc_copy_sock_settings_to_clc(smc);
535 if (smc->sk.sk_state == SMC_INIT)
536 smc->sk.sk_state = SMC_ACTIVE;
541 /* perform steps before actually connecting */
542 static int __smc_connect(struct smc_sock *smc)
544 struct smc_clc_msg_accept_confirm aclc;
545 struct smc_ib_device *ibdev;
549 sock_hold(&smc->sk); /* sock put in passive closing */
551 if (smc->use_fallback)
552 return smc_connect_fallback(smc);
554 /* if peer has not signalled SMC-capability, fall back */
555 if (!tcp_sk(smc->clcsock->sk)->syn_smc)
556 return smc_connect_fallback(smc);
558 /* IPSec connections opt out of SMC-R optimizations */
559 if (using_ipsec(smc))
560 return smc_connect_decline_fallback(smc, SMC_CLC_DECL_IPSEC);
562 /* check if a RDMA device is available; if not, fall back */
563 if (smc_check_rdma(smc, &ibdev, &ibport))
564 return smc_connect_decline_fallback(smc, SMC_CLC_DECL_CNFERR);
566 /* perform CLC handshake */
567 rc = smc_connect_clc(smc, &aclc, ibdev, ibport);
569 return smc_connect_decline_fallback(smc, rc);
571 /* connect using rdma */
572 rc = smc_connect_rdma(smc, &aclc, ibdev, ibport);
574 return smc_connect_decline_fallback(smc, rc);
579 static int smc_connect(struct socket *sock, struct sockaddr *addr,
582 struct sock *sk = sock->sk;
583 struct smc_sock *smc;
588 /* separate smc parameter checking to be safe */
589 if (alen < sizeof(addr->sa_family))
591 if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6)
595 switch (sk->sk_state) {
606 smc_copy_sock_settings_to_clc(smc);
607 tcp_sk(smc->clcsock->sk)->syn_smc = 1;
608 rc = kernel_connect(smc->clcsock, addr, alen, flags);
612 rc = __smc_connect(smc);
616 rc = 0; /* success cases including fallback */
624 static int smc_clcsock_accept(struct smc_sock *lsmc, struct smc_sock **new_smc)
626 struct socket *new_clcsock = NULL;
627 struct sock *lsk = &lsmc->sk;
632 new_sk = smc_sock_alloc(sock_net(lsk), NULL, lsk->sk_protocol);
635 lsk->sk_err = ENOMEM;
640 *new_smc = smc_sk(new_sk);
642 rc = kernel_accept(lsmc->clcsock, &new_clcsock, 0);
646 if (rc < 0 || lsk->sk_state == SMC_CLOSED) {
648 sock_release(new_clcsock);
649 new_sk->sk_state = SMC_CLOSED;
650 sock_set_flag(new_sk, SOCK_DEAD);
651 new_sk->sk_prot->unhash(new_sk);
652 sock_put(new_sk); /* final */
657 (*new_smc)->clcsock = new_clcsock;
662 /* add a just created sock to the accept queue of the listen sock as
663 * candidate for a following socket accept call from user space
665 static void smc_accept_enqueue(struct sock *parent, struct sock *sk)
667 struct smc_sock *par = smc_sk(parent);
669 sock_hold(sk); /* sock_put in smc_accept_unlink () */
670 spin_lock(&par->accept_q_lock);
671 list_add_tail(&smc_sk(sk)->accept_q, &par->accept_q);
672 spin_unlock(&par->accept_q_lock);
673 sk_acceptq_added(parent);
676 /* remove a socket from the accept queue of its parental listening socket */
677 static void smc_accept_unlink(struct sock *sk)
679 struct smc_sock *par = smc_sk(sk)->listen_smc;
681 spin_lock(&par->accept_q_lock);
682 list_del_init(&smc_sk(sk)->accept_q);
683 spin_unlock(&par->accept_q_lock);
684 sk_acceptq_removed(&smc_sk(sk)->listen_smc->sk);
685 sock_put(sk); /* sock_hold in smc_accept_enqueue */
688 /* remove a sock from the accept queue to bind it to a new socket created
689 * for a socket accept call from user space
691 struct sock *smc_accept_dequeue(struct sock *parent,
692 struct socket *new_sock)
694 struct smc_sock *isk, *n;
697 list_for_each_entry_safe(isk, n, &smc_sk(parent)->accept_q, accept_q) {
698 new_sk = (struct sock *)isk;
700 smc_accept_unlink(new_sk);
701 if (new_sk->sk_state == SMC_CLOSED) {
703 sock_release(isk->clcsock);
706 new_sk->sk_prot->unhash(new_sk);
707 sock_put(new_sk); /* final */
711 sock_graft(new_sk, new_sock);
717 /* clean up for a created but never accepted sock */
718 void smc_close_non_accepted(struct sock *sk)
720 struct smc_sock *smc = smc_sk(sk);
723 if (!sk->sk_lingertime)
724 /* wait for peer closing */
725 sk->sk_lingertime = SMC_MAX_STREAM_WAIT_TIMEOUT;
726 if (!smc->use_fallback) {
727 smc_close_active(smc);
728 sock_set_flag(sk, SOCK_DEAD);
729 sk->sk_shutdown |= SHUTDOWN_MASK;
738 if (smc->use_fallback) {
739 sock_put(sk); /* passive closing */
740 sk->sk_state = SMC_CLOSED;
742 if (sk->sk_state == SMC_CLOSED)
743 smc_conn_free(&smc->conn);
746 sk->sk_prot->unhash(sk);
747 sock_put(sk); /* final sock_put */
750 static int smc_serv_conf_first_link(struct smc_sock *smc)
752 struct net *net = sock_net(smc->clcsock->sk);
753 struct smc_link_group *lgr = smc->conn.lgr;
754 struct smc_link *link;
758 link = &lgr->lnk[SMC_SINGLE_LINK];
760 if (smc_reg_rmb(link, smc->conn.rmb_desc, false))
761 return SMC_CLC_DECL_INTERR;
763 /* send CONFIRM LINK request to client over the RoCE fabric */
764 rc = smc_llc_send_confirm_link(link,
765 link->smcibdev->mac[link->ibport - 1],
766 &link->smcibdev->gid[link->ibport - 1],
769 return SMC_CLC_DECL_TCL;
771 /* receive CONFIRM LINK response from client over the RoCE fabric */
772 rest = wait_for_completion_interruptible_timeout(
773 &link->llc_confirm_resp,
774 SMC_LLC_WAIT_FIRST_TIME);
776 struct smc_clc_msg_decline dclc;
778 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
783 if (link->llc_confirm_resp_rc)
784 return SMC_CLC_DECL_RMBE_EC;
786 /* send ADD LINK request to client over the RoCE fabric */
787 rc = smc_llc_send_add_link(link,
788 link->smcibdev->mac[link->ibport - 1],
789 &link->smcibdev->gid[link->ibport - 1],
792 return SMC_CLC_DECL_TCL;
794 /* receive ADD LINK response from client over the RoCE fabric */
795 rest = wait_for_completion_interruptible_timeout(&link->llc_add_resp,
798 struct smc_clc_msg_decline dclc;
800 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
805 smc_llc_link_active(link, net->ipv4.sysctl_tcp_keepalive_time);
810 /* listen worker: finish */
811 static void smc_listen_out(struct smc_sock *new_smc)
813 struct smc_sock *lsmc = new_smc->listen_smc;
814 struct sock *newsmcsk = &new_smc->sk;
816 lock_sock_nested(&lsmc->sk, SINGLE_DEPTH_NESTING);
817 if (lsmc->sk.sk_state == SMC_LISTEN) {
818 smc_accept_enqueue(&lsmc->sk, newsmcsk);
819 } else { /* no longer listening */
820 smc_close_non_accepted(newsmcsk);
822 release_sock(&lsmc->sk);
825 lsmc->sk.sk_data_ready(&lsmc->sk);
826 sock_put(&lsmc->sk); /* sock_hold in smc_tcp_listen_work */
829 /* listen worker: finish in state connected */
830 static void smc_listen_out_connected(struct smc_sock *new_smc)
832 struct sock *newsmcsk = &new_smc->sk;
834 sk_refcnt_debug_inc(newsmcsk);
835 if (newsmcsk->sk_state == SMC_INIT)
836 newsmcsk->sk_state = SMC_ACTIVE;
838 smc_listen_out(new_smc);
841 /* listen worker: finish in error state */
842 static void smc_listen_out_err(struct smc_sock *new_smc)
844 struct sock *newsmcsk = &new_smc->sk;
846 if (newsmcsk->sk_state == SMC_INIT)
847 sock_put(&new_smc->sk); /* passive closing */
848 newsmcsk->sk_state = SMC_CLOSED;
849 smc_conn_free(&new_smc->conn);
851 smc_listen_out(new_smc);
854 /* listen worker: decline and fall back if possible */
855 static void smc_listen_decline(struct smc_sock *new_smc, int reason_code,
858 /* RDMA setup failed, switch back to TCP */
859 if (local_contact == SMC_FIRST_CONTACT)
860 smc_lgr_forget(new_smc->conn.lgr);
861 if (reason_code < 0) { /* error, no fallback possible */
862 smc_listen_out_err(new_smc);
865 smc_conn_free(&new_smc->conn);
866 new_smc->use_fallback = true;
867 if (reason_code && reason_code != SMC_CLC_DECL_REPLY) {
868 if (smc_clc_send_decline(new_smc, reason_code) < 0) {
869 smc_listen_out_err(new_smc);
873 smc_listen_out_connected(new_smc);
876 /* listen worker: check prefixes */
877 static int smc_listen_rdma_check(struct smc_sock *new_smc,
878 struct smc_clc_msg_proposal *pclc)
880 struct smc_clc_msg_proposal_prefix *pclc_prfx;
881 struct socket *newclcsock = new_smc->clcsock;
883 pclc_prfx = smc_clc_proposal_get_prefix(pclc);
884 if (smc_clc_prfx_match(newclcsock, pclc_prfx))
885 return SMC_CLC_DECL_CNFERR;
890 /* listen worker: initialize connection and buffers */
891 static int smc_listen_rdma_init(struct smc_sock *new_smc,
892 struct smc_clc_msg_proposal *pclc,
893 struct smc_ib_device *ibdev, u8 ibport,
896 /* allocate connection / link group */
897 *local_contact = smc_conn_create(new_smc, ibdev, ibport, &pclc->lcl, 0);
898 if (*local_contact < 0) {
899 if (*local_contact == -ENOMEM)
900 return SMC_CLC_DECL_MEM;/* insufficient memory*/
901 return SMC_CLC_DECL_INTERR; /* other error */
904 /* create send buffer and rmb */
905 if (smc_buf_create(new_smc))
906 return SMC_CLC_DECL_MEM;
911 /* listen worker: register buffers */
912 static int smc_listen_rdma_reg(struct smc_sock *new_smc, int local_contact)
914 struct smc_link *link = &new_smc->conn.lgr->lnk[SMC_SINGLE_LINK];
916 if (local_contact != SMC_FIRST_CONTACT) {
917 if (!new_smc->conn.rmb_desc->reused) {
918 if (smc_reg_rmb(link, new_smc->conn.rmb_desc, true))
919 return SMC_CLC_DECL_INTERR;
922 smc_rmb_sync_sg_for_device(&new_smc->conn);
927 /* listen worker: finish RDMA setup */
928 static void smc_listen_rdma_finish(struct smc_sock *new_smc,
929 struct smc_clc_msg_accept_confirm *cclc,
932 struct smc_link *link = &new_smc->conn.lgr->lnk[SMC_SINGLE_LINK];
935 if (local_contact == SMC_FIRST_CONTACT)
936 smc_link_save_peer_info(link, cclc);
938 if (smc_rmb_rtoken_handling(&new_smc->conn, cclc)) {
939 reason_code = SMC_CLC_DECL_INTERR;
943 if (local_contact == SMC_FIRST_CONTACT) {
944 if (smc_ib_ready_link(link)) {
945 reason_code = SMC_CLC_DECL_INTERR;
948 /* QP confirmation over RoCE fabric */
949 reason_code = smc_serv_conf_first_link(new_smc);
956 mutex_unlock(&smc_create_lgr_pending);
957 smc_listen_decline(new_smc, reason_code, local_contact);
960 /* setup for RDMA connection of server */
961 static void smc_listen_work(struct work_struct *work)
963 struct smc_sock *new_smc = container_of(work, struct smc_sock,
965 struct socket *newclcsock = new_smc->clcsock;
966 struct smc_clc_msg_accept_confirm cclc;
967 struct smc_clc_msg_proposal *pclc;
968 struct smc_ib_device *ibdev;
969 u8 buf[SMC_CLC_MAX_LEN];
970 int local_contact = 0;
975 if (new_smc->use_fallback) {
976 smc_listen_out_connected(new_smc);
980 /* check if peer is smc capable */
981 if (!tcp_sk(newclcsock->sk)->syn_smc) {
982 new_smc->use_fallback = true;
983 smc_listen_out_connected(new_smc);
987 /* do inband token exchange -
988 * wait for and receive SMC Proposal CLC message
990 pclc = (struct smc_clc_msg_proposal *)&buf;
991 reason_code = smc_clc_wait_msg(new_smc, pclc, SMC_CLC_MAX_LEN,
994 smc_listen_decline(new_smc, reason_code, 0);
998 /* IPSec connections opt out of SMC-R optimizations */
999 if (using_ipsec(new_smc)) {
1000 smc_listen_decline(new_smc, SMC_CLC_DECL_IPSEC, 0);
1004 mutex_lock(&smc_create_lgr_pending);
1005 smc_close_init(new_smc);
1006 smc_rx_init(new_smc);
1007 smc_tx_init(new_smc);
1009 /* check if RDMA is available */
1010 if (smc_check_rdma(new_smc, &ibdev, &ibport) ||
1011 smc_listen_rdma_check(new_smc, pclc) ||
1012 smc_listen_rdma_init(new_smc, pclc, ibdev, ibport,
1014 smc_listen_rdma_reg(new_smc, local_contact)) {
1015 /* SMC not supported, decline */
1016 mutex_unlock(&smc_create_lgr_pending);
1017 smc_listen_decline(new_smc, SMC_CLC_DECL_CNFERR, local_contact);
1021 /* send SMC Accept CLC message */
1022 rc = smc_clc_send_accept(new_smc, local_contact);
1024 mutex_unlock(&smc_create_lgr_pending);
1025 smc_listen_decline(new_smc, rc, local_contact);
1029 /* receive SMC Confirm CLC message */
1030 reason_code = smc_clc_wait_msg(new_smc, &cclc, sizeof(cclc),
1033 mutex_unlock(&smc_create_lgr_pending);
1034 smc_listen_decline(new_smc, reason_code, local_contact);
1039 smc_listen_rdma_finish(new_smc, &cclc, local_contact);
1040 smc_conn_save_peer_info(new_smc, &cclc);
1041 mutex_unlock(&smc_create_lgr_pending);
1042 smc_listen_out_connected(new_smc);
1045 static void smc_tcp_listen_work(struct work_struct *work)
1047 struct smc_sock *lsmc = container_of(work, struct smc_sock,
1049 struct sock *lsk = &lsmc->sk;
1050 struct smc_sock *new_smc;
1054 while (lsk->sk_state == SMC_LISTEN) {
1055 rc = smc_clcsock_accept(lsmc, &new_smc);
1061 new_smc->listen_smc = lsmc;
1062 new_smc->use_fallback = lsmc->use_fallback;
1063 sock_hold(lsk); /* sock_put in smc_listen_work */
1064 INIT_WORK(&new_smc->smc_listen_work, smc_listen_work);
1065 smc_copy_sock_settings_to_smc(new_smc);
1066 sock_hold(&new_smc->sk); /* sock_put in passive closing */
1067 if (!schedule_work(&new_smc->smc_listen_work))
1068 sock_put(&new_smc->sk);
1073 sock_put(&lsmc->sk); /* sock_hold in smc_listen */
1076 static int smc_listen(struct socket *sock, int backlog)
1078 struct sock *sk = sock->sk;
1079 struct smc_sock *smc;
1086 if ((sk->sk_state != SMC_INIT) && (sk->sk_state != SMC_LISTEN))
1090 if (sk->sk_state == SMC_LISTEN) {
1091 sk->sk_max_ack_backlog = backlog;
1094 /* some socket options are handled in core, so we could not apply
1095 * them to the clc socket -- copy smc socket options to clc socket
1097 smc_copy_sock_settings_to_clc(smc);
1098 if (!smc->use_fallback)
1099 tcp_sk(smc->clcsock->sk)->syn_smc = 1;
1101 rc = kernel_listen(smc->clcsock, backlog);
1104 sk->sk_max_ack_backlog = backlog;
1105 sk->sk_ack_backlog = 0;
1106 sk->sk_state = SMC_LISTEN;
1107 INIT_WORK(&smc->tcp_listen_work, smc_tcp_listen_work);
1108 sock_hold(sk); /* sock_hold in tcp_listen_worker */
1109 if (!schedule_work(&smc->tcp_listen_work))
1117 static int smc_accept(struct socket *sock, struct socket *new_sock,
1118 int flags, bool kern)
1120 struct sock *sk = sock->sk, *nsk;
1121 DECLARE_WAITQUEUE(wait, current);
1122 struct smc_sock *lsmc;
1127 sock_hold(sk); /* sock_put below */
1130 if (lsmc->sk.sk_state != SMC_LISTEN) {
1136 /* Wait for an incoming connection */
1137 timeo = sock_rcvtimeo(sk, flags & O_NONBLOCK);
1138 add_wait_queue_exclusive(sk_sleep(sk), &wait);
1139 while (!(nsk = smc_accept_dequeue(sk, new_sock))) {
1140 set_current_state(TASK_INTERRUPTIBLE);
1146 timeo = schedule_timeout(timeo);
1147 /* wakeup by sk_data_ready in smc_listen_work() */
1148 sched_annotate_sleep();
1150 if (signal_pending(current)) {
1151 rc = sock_intr_errno(timeo);
1155 set_current_state(TASK_RUNNING);
1156 remove_wait_queue(sk_sleep(sk), &wait);
1159 rc = sock_error(nsk);
1164 if (lsmc->sockopt_defer_accept && !(flags & O_NONBLOCK)) {
1165 /* wait till data arrives on the socket */
1166 timeo = msecs_to_jiffies(lsmc->sockopt_defer_accept *
1168 if (smc_sk(nsk)->use_fallback) {
1169 struct sock *clcsk = smc_sk(nsk)->clcsock->sk;
1172 if (skb_queue_empty(&clcsk->sk_receive_queue))
1173 sk_wait_data(clcsk, &timeo, NULL);
1174 release_sock(clcsk);
1175 } else if (!atomic_read(&smc_sk(nsk)->conn.bytes_to_rcv)) {
1177 smc_rx_wait(smc_sk(nsk), &timeo, smc_rx_data_available);
1183 sock_put(sk); /* sock_hold above */
1187 static int smc_getname(struct socket *sock, struct sockaddr *addr,
1190 struct smc_sock *smc;
1192 if (peer && (sock->sk->sk_state != SMC_ACTIVE) &&
1193 (sock->sk->sk_state != SMC_APPCLOSEWAIT1))
1196 smc = smc_sk(sock->sk);
1198 return smc->clcsock->ops->getname(smc->clcsock, addr, peer);
1201 static int smc_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
1203 struct sock *sk = sock->sk;
1204 struct smc_sock *smc;
1209 if ((sk->sk_state != SMC_ACTIVE) &&
1210 (sk->sk_state != SMC_APPCLOSEWAIT1) &&
1211 (sk->sk_state != SMC_INIT))
1214 if (msg->msg_flags & MSG_FASTOPEN) {
1215 if (sk->sk_state == SMC_INIT) {
1216 smc->use_fallback = true;
1223 if (smc->use_fallback)
1224 rc = smc->clcsock->ops->sendmsg(smc->clcsock, msg, len);
1226 rc = smc_tx_sendmsg(smc, msg, len);
1232 static int smc_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
1235 struct sock *sk = sock->sk;
1236 struct smc_sock *smc;
1241 if ((sk->sk_state == SMC_INIT) ||
1242 (sk->sk_state == SMC_LISTEN) ||
1243 (sk->sk_state == SMC_CLOSED))
1246 if (sk->sk_state == SMC_PEERFINCLOSEWAIT) {
1251 if (smc->use_fallback) {
1252 rc = smc->clcsock->ops->recvmsg(smc->clcsock, msg, len, flags);
1254 msg->msg_namelen = 0;
1255 rc = smc_rx_recvmsg(smc, msg, NULL, len, flags);
1263 static __poll_t smc_accept_poll(struct sock *parent)
1265 struct smc_sock *isk = smc_sk(parent);
1268 spin_lock(&isk->accept_q_lock);
1269 if (!list_empty(&isk->accept_q))
1270 mask = EPOLLIN | EPOLLRDNORM;
1271 spin_unlock(&isk->accept_q_lock);
1276 static __poll_t smc_poll(struct file *file, struct socket *sock,
1279 struct sock *sk = sock->sk;
1281 struct smc_sock *smc;
1287 smc = smc_sk(sock->sk);
1290 if ((sk->sk_state == SMC_INIT) || smc->use_fallback) {
1291 /* delegate to CLC child sock */
1293 mask = smc->clcsock->ops->poll(file, smc->clcsock, wait);
1295 sk->sk_err = smc->clcsock->sk->sk_err;
1299 /* if non-blocking connect finished ... */
1300 if (sk->sk_state == SMC_INIT &&
1302 smc->clcsock->sk->sk_state != TCP_CLOSE) {
1303 rc = __smc_connect(smc);
1306 /* success cases including fallback */
1307 mask |= EPOLLOUT | EPOLLWRNORM;
1311 if (sk->sk_state != SMC_CLOSED) {
1313 sock_poll_wait(file, sk_sleep(sk), wait);
1318 if ((sk->sk_shutdown == SHUTDOWN_MASK) ||
1319 (sk->sk_state == SMC_CLOSED))
1321 if (sk->sk_state == SMC_LISTEN) {
1322 /* woken up by sk_data_ready in smc_listen_work() */
1323 mask = smc_accept_poll(sk);
1325 if (atomic_read(&smc->conn.sndbuf_space) ||
1326 sk->sk_shutdown & SEND_SHUTDOWN) {
1327 mask |= EPOLLOUT | EPOLLWRNORM;
1329 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
1330 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
1332 if (atomic_read(&smc->conn.bytes_to_rcv))
1333 mask |= EPOLLIN | EPOLLRDNORM;
1334 if (sk->sk_shutdown & RCV_SHUTDOWN)
1335 mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
1336 if (sk->sk_state == SMC_APPCLOSEWAIT1)
1339 if (smc->conn.urg_state == SMC_URG_VALID)
1349 static int smc_shutdown(struct socket *sock, int how)
1351 struct sock *sk = sock->sk;
1352 struct smc_sock *smc;
1358 if ((how < SHUT_RD) || (how > SHUT_RDWR))
1364 if ((sk->sk_state != SMC_LISTEN) &&
1365 (sk->sk_state != SMC_ACTIVE) &&
1366 (sk->sk_state != SMC_PEERCLOSEWAIT1) &&
1367 (sk->sk_state != SMC_PEERCLOSEWAIT2) &&
1368 (sk->sk_state != SMC_APPCLOSEWAIT1) &&
1369 (sk->sk_state != SMC_APPCLOSEWAIT2) &&
1370 (sk->sk_state != SMC_APPFINCLOSEWAIT))
1372 if (smc->use_fallback) {
1373 rc = kernel_sock_shutdown(smc->clcsock, how);
1374 sk->sk_shutdown = smc->clcsock->sk->sk_shutdown;
1375 if (sk->sk_shutdown == SHUTDOWN_MASK)
1376 sk->sk_state = SMC_CLOSED;
1380 case SHUT_RDWR: /* shutdown in both directions */
1381 rc = smc_close_active(smc);
1384 rc = smc_close_shutdown_write(smc);
1388 /* nothing more to do because peer is not involved */
1392 rc1 = kernel_sock_shutdown(smc->clcsock, how);
1393 /* map sock_shutdown_cmd constants to sk_shutdown value range */
1394 sk->sk_shutdown |= how + 1;
1398 return rc ? rc : rc1;
1401 static int smc_setsockopt(struct socket *sock, int level, int optname,
1402 char __user *optval, unsigned int optlen)
1404 struct sock *sk = sock->sk;
1405 struct smc_sock *smc;
1410 /* generic setsockopts reaching us here always apply to the
1413 rc = smc->clcsock->ops->setsockopt(smc->clcsock, level, optname,
1415 if (smc->clcsock->sk->sk_err) {
1416 sk->sk_err = smc->clcsock->sk->sk_err;
1417 sk->sk_error_report(sk);
1422 if (optlen < sizeof(int))
1424 get_user(val, (int __user *)optval);
1430 case TCP_FASTOPEN_CONNECT:
1431 case TCP_FASTOPEN_KEY:
1432 case TCP_FASTOPEN_NO_COOKIE:
1433 /* option not supported by SMC */
1434 if (sk->sk_state == SMC_INIT) {
1435 smc->use_fallback = true;
1437 if (!smc->use_fallback)
1442 if (sk->sk_state != SMC_INIT && sk->sk_state != SMC_LISTEN) {
1443 if (val && !smc->use_fallback)
1444 mod_delayed_work(system_wq, &smc->conn.tx_work,
1449 if (sk->sk_state != SMC_INIT && sk->sk_state != SMC_LISTEN) {
1450 if (!val && !smc->use_fallback)
1451 mod_delayed_work(system_wq, &smc->conn.tx_work,
1455 case TCP_DEFER_ACCEPT:
1456 smc->sockopt_defer_accept = val;
1466 static int smc_getsockopt(struct socket *sock, int level, int optname,
1467 char __user *optval, int __user *optlen)
1469 struct smc_sock *smc;
1471 smc = smc_sk(sock->sk);
1472 /* socket options apply to the CLC socket */
1473 return smc->clcsock->ops->getsockopt(smc->clcsock, level, optname,
1477 static int smc_ioctl(struct socket *sock, unsigned int cmd,
1480 union smc_host_cursor cons, urg;
1481 struct smc_connection *conn;
1482 struct smc_sock *smc;
1485 smc = smc_sk(sock->sk);
1487 if (smc->use_fallback) {
1490 return smc->clcsock->ops->ioctl(smc->clcsock, cmd, arg);
1493 case SIOCINQ: /* same as FIONREAD */
1494 if (smc->sk.sk_state == SMC_LISTEN)
1496 if (smc->sk.sk_state == SMC_INIT ||
1497 smc->sk.sk_state == SMC_CLOSED)
1500 answ = atomic_read(&smc->conn.bytes_to_rcv);
1503 /* output queue size (not send + not acked) */
1504 if (smc->sk.sk_state == SMC_LISTEN)
1506 if (smc->sk.sk_state == SMC_INIT ||
1507 smc->sk.sk_state == SMC_CLOSED)
1510 answ = smc->conn.sndbuf_desc->len -
1511 atomic_read(&smc->conn.sndbuf_space);
1514 /* output queue size (not send only) */
1515 if (smc->sk.sk_state == SMC_LISTEN)
1517 if (smc->sk.sk_state == SMC_INIT ||
1518 smc->sk.sk_state == SMC_CLOSED)
1521 answ = smc_tx_prepared_sends(&smc->conn);
1524 if (smc->sk.sk_state == SMC_LISTEN)
1526 if (smc->sk.sk_state == SMC_INIT ||
1527 smc->sk.sk_state == SMC_CLOSED) {
1530 smc_curs_write(&cons,
1531 smc_curs_read(&conn->local_tx_ctrl.cons, conn),
1533 smc_curs_write(&urg,
1534 smc_curs_read(&conn->urg_curs, conn),
1536 answ = smc_curs_diff(conn->rmb_desc->len,
1541 return -ENOIOCTLCMD;
1544 return put_user(answ, (int __user *)arg);
1547 static ssize_t smc_sendpage(struct socket *sock, struct page *page,
1548 int offset, size_t size, int flags)
1550 struct sock *sk = sock->sk;
1551 struct smc_sock *smc;
1556 if (sk->sk_state != SMC_ACTIVE) {
1561 if (smc->use_fallback)
1562 rc = kernel_sendpage(smc->clcsock, page, offset,
1565 rc = sock_no_sendpage(sock, page, offset, size, flags);
1571 /* Map the affected portions of the rmbe into an spd, note the number of bytes
1572 * to splice in conn->splice_pending, and press 'go'. Delays consumer cursor
1573 * updates till whenever a respective page has been fully processed.
1574 * Note that subsequent recv() calls have to wait till all splice() processing
1577 static ssize_t smc_splice_read(struct socket *sock, loff_t *ppos,
1578 struct pipe_inode_info *pipe, size_t len,
1581 struct sock *sk = sock->sk;
1582 struct smc_sock *smc;
1588 if (sk->sk_state == SMC_INIT ||
1589 sk->sk_state == SMC_LISTEN ||
1590 sk->sk_state == SMC_CLOSED)
1593 if (sk->sk_state == SMC_PEERFINCLOSEWAIT) {
1598 if (smc->use_fallback) {
1599 rc = smc->clcsock->ops->splice_read(smc->clcsock, ppos,
1606 if (flags & SPLICE_F_NONBLOCK)
1607 flags = MSG_DONTWAIT;
1610 rc = smc_rx_recvmsg(smc, NULL, pipe, len, flags);
1618 /* must look like tcp */
1619 static const struct proto_ops smc_sock_ops = {
1621 .owner = THIS_MODULE,
1622 .release = smc_release,
1624 .connect = smc_connect,
1625 .socketpair = sock_no_socketpair,
1626 .accept = smc_accept,
1627 .getname = smc_getname,
1630 .listen = smc_listen,
1631 .shutdown = smc_shutdown,
1632 .setsockopt = smc_setsockopt,
1633 .getsockopt = smc_getsockopt,
1634 .sendmsg = smc_sendmsg,
1635 .recvmsg = smc_recvmsg,
1636 .mmap = sock_no_mmap,
1637 .sendpage = smc_sendpage,
1638 .splice_read = smc_splice_read,
1641 static int smc_create(struct net *net, struct socket *sock, int protocol,
1644 int family = (protocol == SMCPROTO_SMC6) ? PF_INET6 : PF_INET;
1645 struct smc_sock *smc;
1649 rc = -ESOCKTNOSUPPORT;
1650 if (sock->type != SOCK_STREAM)
1653 rc = -EPROTONOSUPPORT;
1654 if (protocol != SMCPROTO_SMC && protocol != SMCPROTO_SMC6)
1658 sock->ops = &smc_sock_ops;
1659 sk = smc_sock_alloc(net, sock, protocol);
1663 /* create internal TCP socket for CLC handshake and fallback */
1665 smc->use_fallback = false; /* assume rdma capability first */
1666 rc = sock_create_kern(net, family, SOCK_STREAM, IPPROTO_TCP,
1669 sk_common_release(sk);
1672 smc->sk.sk_sndbuf = max(smc->clcsock->sk->sk_sndbuf, SMC_BUF_MIN_SIZE);
1673 smc->sk.sk_rcvbuf = max(smc->clcsock->sk->sk_rcvbuf, SMC_BUF_MIN_SIZE);
1679 static const struct net_proto_family smc_sock_family_ops = {
1681 .owner = THIS_MODULE,
1682 .create = smc_create,
1685 static int __init smc_init(void)
1689 rc = smc_pnet_init();
1693 rc = smc_llc_init();
1695 pr_err("%s: smc_llc_init fails with %d\n", __func__, rc);
1699 rc = smc_cdc_init();
1701 pr_err("%s: smc_cdc_init fails with %d\n", __func__, rc);
1705 rc = proto_register(&smc_proto, 1);
1707 pr_err("%s: proto_register(v4) fails with %d\n", __func__, rc);
1711 rc = proto_register(&smc_proto6, 1);
1713 pr_err("%s: proto_register(v6) fails with %d\n", __func__, rc);
1717 rc = sock_register(&smc_sock_family_ops);
1719 pr_err("%s: sock_register fails with %d\n", __func__, rc);
1722 INIT_HLIST_HEAD(&smc_v4_hashinfo.ht);
1723 INIT_HLIST_HEAD(&smc_v6_hashinfo.ht);
1725 rc = smc_ib_register_client();
1727 pr_err("%s: ib_register fails with %d\n", __func__, rc);
1731 static_branch_enable(&tcp_have_smc);
1735 sock_unregister(PF_SMC);
1737 proto_unregister(&smc_proto6);
1739 proto_unregister(&smc_proto);
1745 static void __exit smc_exit(void)
1748 static_branch_disable(&tcp_have_smc);
1749 smc_ib_unregister_client();
1750 sock_unregister(PF_SMC);
1751 proto_unregister(&smc_proto6);
1752 proto_unregister(&smc_proto);
1756 module_init(smc_init);
1757 module_exit(smc_exit);
1759 MODULE_AUTHOR("Ursula Braun <ubraun@linux.vnet.ibm.com>");
1760 MODULE_DESCRIPTION("smc socket address family");
1761 MODULE_LICENSE("GPL");
1762 MODULE_ALIAS_NETPROTO(PF_SMC);