2 * Shared Memory Communications over RDMA (SMC-R) and RoCE
4 * AF_SMC protocol family socket handler keeping the AF_INET sock address type
5 * applies to SOCK_STREAM sockets only
6 * offers an alternative communication option for TCP-protocol sockets
7 * applicable with RoCE-cards only
9 * Initial restrictions:
10 * - support for alternate links postponed
12 * Copyright IBM Corp. 2016, 2018
14 * Author(s): Ursula Braun <ubraun@linux.vnet.ibm.com>
15 * based on prototype from Frank Blaschka
18 #define KMSG_COMPONENT "smc"
19 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
21 #include <linux/module.h>
22 #include <linux/socket.h>
23 #include <linux/workqueue.h>
25 #include <linux/sched/signal.h>
30 #include <asm/ioctls.h>
41 #include "smc_close.h"
43 static DEFINE_MUTEX(smc_create_lgr_pending); /* serialize link group
47 static void smc_tcp_listen_work(struct work_struct *);
48 static void smc_connect_work(struct work_struct *);
50 static void smc_set_keepalive(struct sock *sk, int val)
52 struct smc_sock *smc = smc_sk(sk);
54 smc->clcsock->sk->sk_prot->keepalive(smc->clcsock->sk, val);
57 static struct smc_hashinfo smc_v4_hashinfo = {
58 .lock = __RW_LOCK_UNLOCKED(smc_v4_hashinfo.lock),
61 static struct smc_hashinfo smc_v6_hashinfo = {
62 .lock = __RW_LOCK_UNLOCKED(smc_v6_hashinfo.lock),
65 int smc_hash_sk(struct sock *sk)
67 struct smc_hashinfo *h = sk->sk_prot->h.smc_hash;
68 struct hlist_head *head;
72 write_lock_bh(&h->lock);
73 sk_add_node(sk, head);
74 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
75 write_unlock_bh(&h->lock);
79 EXPORT_SYMBOL_GPL(smc_hash_sk);
81 void smc_unhash_sk(struct sock *sk)
83 struct smc_hashinfo *h = sk->sk_prot->h.smc_hash;
85 write_lock_bh(&h->lock);
86 if (sk_del_node_init(sk))
87 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
88 write_unlock_bh(&h->lock);
90 EXPORT_SYMBOL_GPL(smc_unhash_sk);
92 struct proto smc_proto = {
95 .keepalive = smc_set_keepalive,
97 .unhash = smc_unhash_sk,
98 .obj_size = sizeof(struct smc_sock),
99 .h.smc_hash = &smc_v4_hashinfo,
100 .slab_flags = SLAB_TYPESAFE_BY_RCU,
102 EXPORT_SYMBOL_GPL(smc_proto);
104 struct proto smc_proto6 = {
106 .owner = THIS_MODULE,
107 .keepalive = smc_set_keepalive,
109 .unhash = smc_unhash_sk,
110 .obj_size = sizeof(struct smc_sock),
111 .h.smc_hash = &smc_v6_hashinfo,
112 .slab_flags = SLAB_TYPESAFE_BY_RCU,
114 EXPORT_SYMBOL_GPL(smc_proto6);
116 static int smc_release(struct socket *sock)
118 struct sock *sk = sock->sk;
119 struct smc_sock *smc;
127 /* cleanup for a dangling non-blocking connect */
128 flush_work(&smc->connect_work);
129 kfree(smc->connect_info);
130 smc->connect_info = NULL;
132 if (sk->sk_state == SMC_LISTEN)
133 /* smc_close_non_accepted() is called and acquires
134 * sock lock for child sockets again
136 lock_sock_nested(sk, SINGLE_DEPTH_NESTING);
140 if (!smc->use_fallback) {
141 rc = smc_close_active(smc);
142 sock_set_flag(sk, SOCK_DEAD);
143 sk->sk_shutdown |= SHUTDOWN_MASK;
146 sock_release(smc->clcsock);
149 if (smc->use_fallback) {
150 sock_put(sk); /* passive closing */
151 sk->sk_state = SMC_CLOSED;
152 sk->sk_state_change(sk);
158 if (!smc->use_fallback && sk->sk_state == SMC_CLOSED)
159 smc_conn_free(&smc->conn);
162 sk->sk_prot->unhash(sk);
163 sock_put(sk); /* final sock_put */
168 static void smc_destruct(struct sock *sk)
170 if (sk->sk_state != SMC_CLOSED)
172 if (!sock_flag(sk, SOCK_DEAD))
175 sk_refcnt_debug_dec(sk);
178 static struct sock *smc_sock_alloc(struct net *net, struct socket *sock,
181 struct smc_sock *smc;
185 prot = (protocol == SMCPROTO_SMC6) ? &smc_proto6 : &smc_proto;
186 sk = sk_alloc(net, PF_SMC, GFP_KERNEL, prot, 0);
190 sock_init_data(sock, sk); /* sets sk_refcnt to 1 */
191 sk->sk_state = SMC_INIT;
192 sk->sk_destruct = smc_destruct;
193 sk->sk_protocol = protocol;
195 INIT_WORK(&smc->tcp_listen_work, smc_tcp_listen_work);
196 INIT_WORK(&smc->connect_work, smc_connect_work);
197 INIT_DELAYED_WORK(&smc->conn.tx_work, smc_tx_work);
198 INIT_LIST_HEAD(&smc->accept_q);
199 spin_lock_init(&smc->accept_q_lock);
200 spin_lock_init(&smc->conn.send_lock);
201 sk->sk_prot->hash(sk);
202 sk_refcnt_debug_inc(sk);
207 static int smc_bind(struct socket *sock, struct sockaddr *uaddr,
210 struct sockaddr_in *addr = (struct sockaddr_in *)uaddr;
211 struct sock *sk = sock->sk;
212 struct smc_sock *smc;
217 /* replicate tests from inet_bind(), to be safe wrt. future changes */
219 if (addr_len < sizeof(struct sockaddr_in))
223 if (addr->sin_family != AF_INET &&
224 addr->sin_family != AF_INET6 &&
225 addr->sin_family != AF_UNSPEC)
227 /* accept AF_UNSPEC (mapped to AF_INET) only if s_addr is INADDR_ANY */
228 if (addr->sin_family == AF_UNSPEC &&
229 addr->sin_addr.s_addr != htonl(INADDR_ANY))
234 /* Check if socket is already active */
236 if (sk->sk_state != SMC_INIT)
239 smc->clcsock->sk->sk_reuse = sk->sk_reuse;
240 rc = kernel_bind(smc->clcsock, uaddr, addr_len);
248 static void smc_copy_sock_settings(struct sock *nsk, struct sock *osk,
251 /* options we don't get control via setsockopt for */
252 nsk->sk_type = osk->sk_type;
253 nsk->sk_sndbuf = osk->sk_sndbuf;
254 nsk->sk_rcvbuf = osk->sk_rcvbuf;
255 nsk->sk_sndtimeo = osk->sk_sndtimeo;
256 nsk->sk_rcvtimeo = osk->sk_rcvtimeo;
257 nsk->sk_mark = osk->sk_mark;
258 nsk->sk_priority = osk->sk_priority;
259 nsk->sk_rcvlowat = osk->sk_rcvlowat;
260 nsk->sk_bound_dev_if = osk->sk_bound_dev_if;
261 nsk->sk_err = osk->sk_err;
263 nsk->sk_flags &= ~mask;
264 nsk->sk_flags |= osk->sk_flags & mask;
267 #define SK_FLAGS_SMC_TO_CLC ((1UL << SOCK_URGINLINE) | \
268 (1UL << SOCK_KEEPOPEN) | \
269 (1UL << SOCK_LINGER) | \
270 (1UL << SOCK_BROADCAST) | \
271 (1UL << SOCK_TIMESTAMP) | \
272 (1UL << SOCK_DBG) | \
273 (1UL << SOCK_RCVTSTAMP) | \
274 (1UL << SOCK_RCVTSTAMPNS) | \
275 (1UL << SOCK_LOCALROUTE) | \
276 (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE) | \
277 (1UL << SOCK_RXQ_OVFL) | \
278 (1UL << SOCK_WIFI_STATUS) | \
279 (1UL << SOCK_NOFCS) | \
280 (1UL << SOCK_FILTER_LOCKED))
281 /* copy only relevant settings and flags of SOL_SOCKET level from smc to
282 * clc socket (since smc is not called for these options from net/core)
284 static void smc_copy_sock_settings_to_clc(struct smc_sock *smc)
286 smc_copy_sock_settings(smc->clcsock->sk, &smc->sk, SK_FLAGS_SMC_TO_CLC);
289 #define SK_FLAGS_CLC_TO_SMC ((1UL << SOCK_URGINLINE) | \
290 (1UL << SOCK_KEEPOPEN) | \
291 (1UL << SOCK_LINGER) | \
293 /* copy only settings and flags relevant for smc from clc to smc socket */
294 static void smc_copy_sock_settings_to_smc(struct smc_sock *smc)
296 smc_copy_sock_settings(&smc->sk, smc->clcsock->sk, SK_FLAGS_CLC_TO_SMC);
299 /* register a new rmb, optionally send confirm_rkey msg to register with peer */
300 static int smc_reg_rmb(struct smc_link *link, struct smc_buf_desc *rmb_desc,
303 /* register memory region for new rmb */
304 if (smc_wr_reg_send(link, rmb_desc->mr_rx[SMC_SINGLE_LINK])) {
305 rmb_desc->regerr = 1;
310 /* exchange confirm_rkey msg with peer */
311 if (smc_llc_do_confirm_rkey(link, rmb_desc)) {
312 rmb_desc->regerr = 1;
318 static int smc_clnt_conf_first_link(struct smc_sock *smc)
320 struct net *net = sock_net(smc->clcsock->sk);
321 struct smc_link_group *lgr = smc->conn.lgr;
322 struct smc_link *link;
326 link = &lgr->lnk[SMC_SINGLE_LINK];
327 /* receive CONFIRM LINK request from server over RoCE fabric */
328 rest = wait_for_completion_interruptible_timeout(
330 SMC_LLC_WAIT_FIRST_TIME);
332 struct smc_clc_msg_decline dclc;
334 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
339 if (link->llc_confirm_rc)
340 return SMC_CLC_DECL_RMBE_EC;
342 rc = smc_ib_modify_qp_rts(link);
344 return SMC_CLC_DECL_INTERR;
346 smc_wr_remember_qp_attr(link);
348 if (smc_reg_rmb(link, smc->conn.rmb_desc, false))
349 return SMC_CLC_DECL_INTERR;
351 /* send CONFIRM LINK response over RoCE fabric */
352 rc = smc_llc_send_confirm_link(link,
353 link->smcibdev->mac[link->ibport - 1],
354 &link->smcibdev->gid[link->ibport - 1],
357 return SMC_CLC_DECL_TCL;
359 /* receive ADD LINK request from server over RoCE fabric */
360 rest = wait_for_completion_interruptible_timeout(&link->llc_add,
363 struct smc_clc_msg_decline dclc;
365 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
370 /* send add link reject message, only one link supported for now */
371 rc = smc_llc_send_add_link(link,
372 link->smcibdev->mac[link->ibport - 1],
373 &link->smcibdev->gid[link->ibport - 1],
376 return SMC_CLC_DECL_TCL;
378 smc_llc_link_active(link, net->ipv4.sysctl_tcp_keepalive_time);
383 static void smc_conn_save_peer_info(struct smc_sock *smc,
384 struct smc_clc_msg_accept_confirm *clc)
386 int bufsize = smc_uncompress_bufsize(clc->rmbe_size);
388 smc->conn.peer_rmbe_idx = clc->rmbe_idx;
389 smc->conn.local_tx_ctrl.token = ntohl(clc->rmbe_alert_token);
390 smc->conn.peer_rmbe_size = bufsize;
391 atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size);
392 smc->conn.tx_off = bufsize * (smc->conn.peer_rmbe_idx - 1);
395 static void smc_link_save_peer_info(struct smc_link *link,
396 struct smc_clc_msg_accept_confirm *clc)
398 link->peer_qpn = ntoh24(clc->qpn);
399 memcpy(link->peer_gid, clc->lcl.gid, SMC_GID_SIZE);
400 memcpy(link->peer_mac, clc->lcl.mac, sizeof(link->peer_mac));
401 link->peer_psn = ntoh24(clc->psn);
402 link->peer_mtu = clc->qp_mtu;
405 /* fall back during connect */
406 static int smc_connect_fallback(struct smc_sock *smc)
408 smc->use_fallback = true;
409 smc_copy_sock_settings_to_clc(smc);
410 if (smc->sk.sk_state == SMC_INIT)
411 smc->sk.sk_state = SMC_ACTIVE;
415 /* decline and fall back during connect */
416 static int smc_connect_decline_fallback(struct smc_sock *smc, int reason_code)
420 if (reason_code < 0) /* error, fallback is not possible */
422 if (reason_code != SMC_CLC_DECL_REPLY) {
423 rc = smc_clc_send_decline(smc, reason_code);
427 return smc_connect_fallback(smc);
430 /* abort connecting */
431 static int smc_connect_abort(struct smc_sock *smc, int reason_code,
434 if (local_contact == SMC_FIRST_CONTACT)
435 smc_lgr_forget(smc->conn.lgr);
436 mutex_unlock(&smc_create_lgr_pending);
437 smc_conn_free(&smc->conn);
438 if (reason_code < 0 && smc->sk.sk_state == SMC_INIT)
439 sock_put(&smc->sk); /* passive closing */
443 /* check if there is a rdma device available for this connection. */
444 /* called for connect and listen */
445 static int smc_check_rdma(struct smc_sock *smc, struct smc_ib_device **ibdev,
450 /* PNET table look up: search active ib_device and port
451 * within same PNETID that also contains the ethernet device
452 * used for the internal TCP socket
454 smc_pnet_find_roce_resource(smc->clcsock->sk, ibdev, ibport);
456 reason_code = SMC_CLC_DECL_CNFERR; /* configuration error */
461 /* CLC handshake during connect */
462 static int smc_connect_clc(struct smc_sock *smc,
463 struct smc_clc_msg_accept_confirm *aclc,
464 struct smc_ib_device *ibdev, u8 ibport)
468 /* do inband token exchange */
469 rc = smc_clc_send_proposal(smc, ibdev, ibport);
472 /* receive SMC Accept CLC message */
473 return smc_clc_wait_msg(smc, aclc, sizeof(*aclc), SMC_CLC_ACCEPT);
476 /* setup for RDMA connection of client */
477 static int smc_connect_rdma(struct smc_sock *smc,
478 struct smc_clc_msg_accept_confirm *aclc,
479 struct smc_ib_device *ibdev, u8 ibport)
481 int local_contact = SMC_FIRST_CONTACT;
482 struct smc_link *link;
485 mutex_lock(&smc_create_lgr_pending);
486 local_contact = smc_conn_create(smc, ibdev, ibport, &aclc->lcl,
488 if (local_contact < 0) {
489 if (local_contact == -ENOMEM)
490 reason_code = SMC_CLC_DECL_MEM;/* insufficient memory*/
491 else if (local_contact == -ENOLINK)
492 reason_code = SMC_CLC_DECL_SYNCERR; /* synchr. error */
494 reason_code = SMC_CLC_DECL_INTERR; /* other error */
495 return smc_connect_abort(smc, reason_code, 0);
497 link = &smc->conn.lgr->lnk[SMC_SINGLE_LINK];
499 smc_conn_save_peer_info(smc, aclc);
501 /* create send buffer and rmb */
502 if (smc_buf_create(smc))
503 return smc_connect_abort(smc, SMC_CLC_DECL_MEM, local_contact);
505 if (local_contact == SMC_FIRST_CONTACT)
506 smc_link_save_peer_info(link, aclc);
508 if (smc_rmb_rtoken_handling(&smc->conn, aclc))
509 return smc_connect_abort(smc, SMC_CLC_DECL_INTERR,
515 if (local_contact == SMC_FIRST_CONTACT) {
516 if (smc_ib_ready_link(link))
517 return smc_connect_abort(smc, SMC_CLC_DECL_INTERR,
520 if (!smc->conn.rmb_desc->reused &&
521 smc_reg_rmb(link, smc->conn.rmb_desc, true))
522 return smc_connect_abort(smc, SMC_CLC_DECL_INTERR,
525 smc_rmb_sync_sg_for_device(&smc->conn);
527 reason_code = smc_clc_send_confirm(smc);
529 return smc_connect_abort(smc, reason_code, local_contact);
533 if (local_contact == SMC_FIRST_CONTACT) {
534 /* QP confirmation over RoCE fabric */
535 reason_code = smc_clnt_conf_first_link(smc);
537 return smc_connect_abort(smc, reason_code,
540 mutex_unlock(&smc_create_lgr_pending);
542 smc_copy_sock_settings_to_clc(smc);
543 if (smc->sk.sk_state == SMC_INIT)
544 smc->sk.sk_state = SMC_ACTIVE;
549 /* perform steps before actually connecting */
550 static int __smc_connect(struct smc_sock *smc)
552 struct smc_clc_msg_accept_confirm aclc;
553 struct smc_ib_device *ibdev;
557 sock_hold(&smc->sk); /* sock put in passive closing */
559 if (smc->use_fallback)
560 return smc_connect_fallback(smc);
562 /* if peer has not signalled SMC-capability, fall back */
563 if (!tcp_sk(smc->clcsock->sk)->syn_smc)
564 return smc_connect_fallback(smc);
566 /* IPSec connections opt out of SMC-R optimizations */
567 if (using_ipsec(smc))
568 return smc_connect_decline_fallback(smc, SMC_CLC_DECL_IPSEC);
570 /* check if a RDMA device is available; if not, fall back */
571 if (smc_check_rdma(smc, &ibdev, &ibport))
572 return smc_connect_decline_fallback(smc, SMC_CLC_DECL_CNFERR);
574 /* perform CLC handshake */
575 rc = smc_connect_clc(smc, &aclc, ibdev, ibport);
577 return smc_connect_decline_fallback(smc, rc);
579 /* connect using rdma */
580 rc = smc_connect_rdma(smc, &aclc, ibdev, ibport);
582 return smc_connect_decline_fallback(smc, rc);
587 static void smc_connect_work(struct work_struct *work)
589 struct smc_sock *smc = container_of(work, struct smc_sock,
594 rc = kernel_connect(smc->clcsock, &smc->connect_info->addr,
595 smc->connect_info->alen, smc->connect_info->flags);
596 if (smc->clcsock->sk->sk_err) {
597 smc->sk.sk_err = smc->clcsock->sk->sk_err;
601 smc->sk.sk_err = -rc;
605 rc = __smc_connect(smc);
607 smc->sk.sk_err = -rc;
610 smc->sk.sk_state_change(&smc->sk);
611 kfree(smc->connect_info);
612 smc->connect_info = NULL;
613 release_sock(&smc->sk);
616 static int smc_connect(struct socket *sock, struct sockaddr *addr,
619 struct sock *sk = sock->sk;
620 struct smc_sock *smc;
625 /* separate smc parameter checking to be safe */
626 if (alen < sizeof(addr->sa_family))
628 if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6)
632 switch (sk->sk_state) {
643 smc_copy_sock_settings_to_clc(smc);
644 tcp_sk(smc->clcsock->sk)->syn_smc = 1;
645 if (flags & O_NONBLOCK) {
646 if (smc->connect_info) {
650 smc->connect_info = kzalloc(alen + 2 * sizeof(int), GFP_KERNEL);
651 if (!smc->connect_info) {
655 smc->connect_info->alen = alen;
656 smc->connect_info->flags = flags ^ O_NONBLOCK;
657 memcpy(&smc->connect_info->addr, addr, alen);
658 schedule_work(&smc->connect_work);
661 rc = kernel_connect(smc->clcsock, addr, alen, flags);
665 rc = __smc_connect(smc);
669 rc = 0; /* success cases including fallback */
678 static int smc_clcsock_accept(struct smc_sock *lsmc, struct smc_sock **new_smc)
680 struct socket *new_clcsock = NULL;
681 struct sock *lsk = &lsmc->sk;
686 new_sk = smc_sock_alloc(sock_net(lsk), NULL, lsk->sk_protocol);
689 lsk->sk_err = ENOMEM;
694 *new_smc = smc_sk(new_sk);
696 rc = kernel_accept(lsmc->clcsock, &new_clcsock, 0);
700 if (rc < 0 || lsk->sk_state == SMC_CLOSED) {
702 sock_release(new_clcsock);
703 new_sk->sk_state = SMC_CLOSED;
704 sock_set_flag(new_sk, SOCK_DEAD);
705 new_sk->sk_prot->unhash(new_sk);
706 sock_put(new_sk); /* final */
711 (*new_smc)->clcsock = new_clcsock;
716 /* add a just created sock to the accept queue of the listen sock as
717 * candidate for a following socket accept call from user space
719 static void smc_accept_enqueue(struct sock *parent, struct sock *sk)
721 struct smc_sock *par = smc_sk(parent);
723 sock_hold(sk); /* sock_put in smc_accept_unlink () */
724 spin_lock(&par->accept_q_lock);
725 list_add_tail(&smc_sk(sk)->accept_q, &par->accept_q);
726 spin_unlock(&par->accept_q_lock);
727 sk_acceptq_added(parent);
730 /* remove a socket from the accept queue of its parental listening socket */
731 static void smc_accept_unlink(struct sock *sk)
733 struct smc_sock *par = smc_sk(sk)->listen_smc;
735 spin_lock(&par->accept_q_lock);
736 list_del_init(&smc_sk(sk)->accept_q);
737 spin_unlock(&par->accept_q_lock);
738 sk_acceptq_removed(&smc_sk(sk)->listen_smc->sk);
739 sock_put(sk); /* sock_hold in smc_accept_enqueue */
742 /* remove a sock from the accept queue to bind it to a new socket created
743 * for a socket accept call from user space
745 struct sock *smc_accept_dequeue(struct sock *parent,
746 struct socket *new_sock)
748 struct smc_sock *isk, *n;
751 list_for_each_entry_safe(isk, n, &smc_sk(parent)->accept_q, accept_q) {
752 new_sk = (struct sock *)isk;
754 smc_accept_unlink(new_sk);
755 if (new_sk->sk_state == SMC_CLOSED) {
757 sock_release(isk->clcsock);
760 new_sk->sk_prot->unhash(new_sk);
761 sock_put(new_sk); /* final */
765 sock_graft(new_sk, new_sock);
771 /* clean up for a created but never accepted sock */
772 void smc_close_non_accepted(struct sock *sk)
774 struct smc_sock *smc = smc_sk(sk);
777 if (!sk->sk_lingertime)
778 /* wait for peer closing */
779 sk->sk_lingertime = SMC_MAX_STREAM_WAIT_TIMEOUT;
780 if (!smc->use_fallback) {
781 smc_close_active(smc);
782 sock_set_flag(sk, SOCK_DEAD);
783 sk->sk_shutdown |= SHUTDOWN_MASK;
792 if (smc->use_fallback) {
793 sock_put(sk); /* passive closing */
794 sk->sk_state = SMC_CLOSED;
796 if (sk->sk_state == SMC_CLOSED)
797 smc_conn_free(&smc->conn);
800 sk->sk_prot->unhash(sk);
801 sock_put(sk); /* final sock_put */
804 static int smc_serv_conf_first_link(struct smc_sock *smc)
806 struct net *net = sock_net(smc->clcsock->sk);
807 struct smc_link_group *lgr = smc->conn.lgr;
808 struct smc_link *link;
812 link = &lgr->lnk[SMC_SINGLE_LINK];
814 if (smc_reg_rmb(link, smc->conn.rmb_desc, false))
815 return SMC_CLC_DECL_INTERR;
817 /* send CONFIRM LINK request to client over the RoCE fabric */
818 rc = smc_llc_send_confirm_link(link,
819 link->smcibdev->mac[link->ibport - 1],
820 &link->smcibdev->gid[link->ibport - 1],
823 return SMC_CLC_DECL_TCL;
825 /* receive CONFIRM LINK response from client over the RoCE fabric */
826 rest = wait_for_completion_interruptible_timeout(
827 &link->llc_confirm_resp,
828 SMC_LLC_WAIT_FIRST_TIME);
830 struct smc_clc_msg_decline dclc;
832 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
837 if (link->llc_confirm_resp_rc)
838 return SMC_CLC_DECL_RMBE_EC;
840 /* send ADD LINK request to client over the RoCE fabric */
841 rc = smc_llc_send_add_link(link,
842 link->smcibdev->mac[link->ibport - 1],
843 &link->smcibdev->gid[link->ibport - 1],
846 return SMC_CLC_DECL_TCL;
848 /* receive ADD LINK response from client over the RoCE fabric */
849 rest = wait_for_completion_interruptible_timeout(&link->llc_add_resp,
852 struct smc_clc_msg_decline dclc;
854 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
859 smc_llc_link_active(link, net->ipv4.sysctl_tcp_keepalive_time);
864 /* listen worker: finish */
865 static void smc_listen_out(struct smc_sock *new_smc)
867 struct smc_sock *lsmc = new_smc->listen_smc;
868 struct sock *newsmcsk = &new_smc->sk;
870 lock_sock_nested(&lsmc->sk, SINGLE_DEPTH_NESTING);
871 if (lsmc->sk.sk_state == SMC_LISTEN) {
872 smc_accept_enqueue(&lsmc->sk, newsmcsk);
873 } else { /* no longer listening */
874 smc_close_non_accepted(newsmcsk);
876 release_sock(&lsmc->sk);
879 lsmc->sk.sk_data_ready(&lsmc->sk);
880 sock_put(&lsmc->sk); /* sock_hold in smc_tcp_listen_work */
883 /* listen worker: finish in state connected */
884 static void smc_listen_out_connected(struct smc_sock *new_smc)
886 struct sock *newsmcsk = &new_smc->sk;
888 sk_refcnt_debug_inc(newsmcsk);
889 if (newsmcsk->sk_state == SMC_INIT)
890 newsmcsk->sk_state = SMC_ACTIVE;
892 smc_listen_out(new_smc);
895 /* listen worker: finish in error state */
896 static void smc_listen_out_err(struct smc_sock *new_smc)
898 struct sock *newsmcsk = &new_smc->sk;
900 if (newsmcsk->sk_state == SMC_INIT)
901 sock_put(&new_smc->sk); /* passive closing */
902 newsmcsk->sk_state = SMC_CLOSED;
903 smc_conn_free(&new_smc->conn);
905 smc_listen_out(new_smc);
908 /* listen worker: decline and fall back if possible */
909 static void smc_listen_decline(struct smc_sock *new_smc, int reason_code,
912 /* RDMA setup failed, switch back to TCP */
913 if (local_contact == SMC_FIRST_CONTACT)
914 smc_lgr_forget(new_smc->conn.lgr);
915 if (reason_code < 0) { /* error, no fallback possible */
916 smc_listen_out_err(new_smc);
919 smc_conn_free(&new_smc->conn);
920 new_smc->use_fallback = true;
921 if (reason_code && reason_code != SMC_CLC_DECL_REPLY) {
922 if (smc_clc_send_decline(new_smc, reason_code) < 0) {
923 smc_listen_out_err(new_smc);
927 smc_listen_out_connected(new_smc);
930 /* listen worker: check prefixes */
931 static int smc_listen_rdma_check(struct smc_sock *new_smc,
932 struct smc_clc_msg_proposal *pclc)
934 struct smc_clc_msg_proposal_prefix *pclc_prfx;
935 struct socket *newclcsock = new_smc->clcsock;
937 pclc_prfx = smc_clc_proposal_get_prefix(pclc);
938 if (smc_clc_prfx_match(newclcsock, pclc_prfx))
939 return SMC_CLC_DECL_CNFERR;
944 /* listen worker: initialize connection and buffers */
945 static int smc_listen_rdma_init(struct smc_sock *new_smc,
946 struct smc_clc_msg_proposal *pclc,
947 struct smc_ib_device *ibdev, u8 ibport,
950 /* allocate connection / link group */
951 *local_contact = smc_conn_create(new_smc, ibdev, ibport, &pclc->lcl, 0);
952 if (*local_contact < 0) {
953 if (*local_contact == -ENOMEM)
954 return SMC_CLC_DECL_MEM;/* insufficient memory*/
955 return SMC_CLC_DECL_INTERR; /* other error */
958 /* create send buffer and rmb */
959 if (smc_buf_create(new_smc))
960 return SMC_CLC_DECL_MEM;
965 /* listen worker: register buffers */
966 static int smc_listen_rdma_reg(struct smc_sock *new_smc, int local_contact)
968 struct smc_link *link = &new_smc->conn.lgr->lnk[SMC_SINGLE_LINK];
970 if (local_contact != SMC_FIRST_CONTACT) {
971 if (!new_smc->conn.rmb_desc->reused) {
972 if (smc_reg_rmb(link, new_smc->conn.rmb_desc, true))
973 return SMC_CLC_DECL_INTERR;
976 smc_rmb_sync_sg_for_device(&new_smc->conn);
981 /* listen worker: finish RDMA setup */
982 static void smc_listen_rdma_finish(struct smc_sock *new_smc,
983 struct smc_clc_msg_accept_confirm *cclc,
986 struct smc_link *link = &new_smc->conn.lgr->lnk[SMC_SINGLE_LINK];
989 if (local_contact == SMC_FIRST_CONTACT)
990 smc_link_save_peer_info(link, cclc);
992 if (smc_rmb_rtoken_handling(&new_smc->conn, cclc)) {
993 reason_code = SMC_CLC_DECL_INTERR;
997 if (local_contact == SMC_FIRST_CONTACT) {
998 if (smc_ib_ready_link(link)) {
999 reason_code = SMC_CLC_DECL_INTERR;
1002 /* QP confirmation over RoCE fabric */
1003 reason_code = smc_serv_conf_first_link(new_smc);
1010 mutex_unlock(&smc_create_lgr_pending);
1011 smc_listen_decline(new_smc, reason_code, local_contact);
1014 /* setup for RDMA connection of server */
1015 static void smc_listen_work(struct work_struct *work)
1017 struct smc_sock *new_smc = container_of(work, struct smc_sock,
1019 struct socket *newclcsock = new_smc->clcsock;
1020 struct smc_clc_msg_accept_confirm cclc;
1021 struct smc_clc_msg_proposal *pclc;
1022 struct smc_ib_device *ibdev;
1023 u8 buf[SMC_CLC_MAX_LEN];
1024 int local_contact = 0;
1025 int reason_code = 0;
1029 if (new_smc->use_fallback) {
1030 smc_listen_out_connected(new_smc);
1034 /* check if peer is smc capable */
1035 if (!tcp_sk(newclcsock->sk)->syn_smc) {
1036 new_smc->use_fallback = true;
1037 smc_listen_out_connected(new_smc);
1041 /* do inband token exchange -
1042 * wait for and receive SMC Proposal CLC message
1044 pclc = (struct smc_clc_msg_proposal *)&buf;
1045 reason_code = smc_clc_wait_msg(new_smc, pclc, SMC_CLC_MAX_LEN,
1048 smc_listen_decline(new_smc, reason_code, 0);
1052 /* IPSec connections opt out of SMC-R optimizations */
1053 if (using_ipsec(new_smc)) {
1054 smc_listen_decline(new_smc, SMC_CLC_DECL_IPSEC, 0);
1058 mutex_lock(&smc_create_lgr_pending);
1059 smc_close_init(new_smc);
1060 smc_rx_init(new_smc);
1061 smc_tx_init(new_smc);
1063 /* check if RDMA is available */
1064 if (smc_check_rdma(new_smc, &ibdev, &ibport) ||
1065 smc_listen_rdma_check(new_smc, pclc) ||
1066 smc_listen_rdma_init(new_smc, pclc, ibdev, ibport,
1068 smc_listen_rdma_reg(new_smc, local_contact)) {
1069 /* SMC not supported, decline */
1070 mutex_unlock(&smc_create_lgr_pending);
1071 smc_listen_decline(new_smc, SMC_CLC_DECL_CNFERR, local_contact);
1075 /* send SMC Accept CLC message */
1076 rc = smc_clc_send_accept(new_smc, local_contact);
1078 mutex_unlock(&smc_create_lgr_pending);
1079 smc_listen_decline(new_smc, rc, local_contact);
1083 /* receive SMC Confirm CLC message */
1084 reason_code = smc_clc_wait_msg(new_smc, &cclc, sizeof(cclc),
1087 mutex_unlock(&smc_create_lgr_pending);
1088 smc_listen_decline(new_smc, reason_code, local_contact);
1093 smc_listen_rdma_finish(new_smc, &cclc, local_contact);
1094 smc_conn_save_peer_info(new_smc, &cclc);
1095 mutex_unlock(&smc_create_lgr_pending);
1096 smc_listen_out_connected(new_smc);
1099 static void smc_tcp_listen_work(struct work_struct *work)
1101 struct smc_sock *lsmc = container_of(work, struct smc_sock,
1103 struct sock *lsk = &lsmc->sk;
1104 struct smc_sock *new_smc;
1108 while (lsk->sk_state == SMC_LISTEN) {
1109 rc = smc_clcsock_accept(lsmc, &new_smc);
1115 new_smc->listen_smc = lsmc;
1116 new_smc->use_fallback = lsmc->use_fallback;
1117 sock_hold(lsk); /* sock_put in smc_listen_work */
1118 INIT_WORK(&new_smc->smc_listen_work, smc_listen_work);
1119 smc_copy_sock_settings_to_smc(new_smc);
1120 sock_hold(&new_smc->sk); /* sock_put in passive closing */
1121 if (!schedule_work(&new_smc->smc_listen_work))
1122 sock_put(&new_smc->sk);
1127 sock_put(&lsmc->sk); /* sock_hold in smc_listen */
1130 static int smc_listen(struct socket *sock, int backlog)
1132 struct sock *sk = sock->sk;
1133 struct smc_sock *smc;
1140 if ((sk->sk_state != SMC_INIT) && (sk->sk_state != SMC_LISTEN))
1144 if (sk->sk_state == SMC_LISTEN) {
1145 sk->sk_max_ack_backlog = backlog;
1148 /* some socket options are handled in core, so we could not apply
1149 * them to the clc socket -- copy smc socket options to clc socket
1151 smc_copy_sock_settings_to_clc(smc);
1152 if (!smc->use_fallback)
1153 tcp_sk(smc->clcsock->sk)->syn_smc = 1;
1155 rc = kernel_listen(smc->clcsock, backlog);
1158 sk->sk_max_ack_backlog = backlog;
1159 sk->sk_ack_backlog = 0;
1160 sk->sk_state = SMC_LISTEN;
1161 INIT_WORK(&smc->tcp_listen_work, smc_tcp_listen_work);
1162 sock_hold(sk); /* sock_hold in tcp_listen_worker */
1163 if (!schedule_work(&smc->tcp_listen_work))
1171 static int smc_accept(struct socket *sock, struct socket *new_sock,
1172 int flags, bool kern)
1174 struct sock *sk = sock->sk, *nsk;
1175 DECLARE_WAITQUEUE(wait, current);
1176 struct smc_sock *lsmc;
1181 sock_hold(sk); /* sock_put below */
1184 if (lsmc->sk.sk_state != SMC_LISTEN) {
1190 /* Wait for an incoming connection */
1191 timeo = sock_rcvtimeo(sk, flags & O_NONBLOCK);
1192 add_wait_queue_exclusive(sk_sleep(sk), &wait);
1193 while (!(nsk = smc_accept_dequeue(sk, new_sock))) {
1194 set_current_state(TASK_INTERRUPTIBLE);
1200 timeo = schedule_timeout(timeo);
1201 /* wakeup by sk_data_ready in smc_listen_work() */
1202 sched_annotate_sleep();
1204 if (signal_pending(current)) {
1205 rc = sock_intr_errno(timeo);
1209 set_current_state(TASK_RUNNING);
1210 remove_wait_queue(sk_sleep(sk), &wait);
1213 rc = sock_error(nsk);
1218 if (lsmc->sockopt_defer_accept && !(flags & O_NONBLOCK)) {
1219 /* wait till data arrives on the socket */
1220 timeo = msecs_to_jiffies(lsmc->sockopt_defer_accept *
1222 if (smc_sk(nsk)->use_fallback) {
1223 struct sock *clcsk = smc_sk(nsk)->clcsock->sk;
1226 if (skb_queue_empty(&clcsk->sk_receive_queue))
1227 sk_wait_data(clcsk, &timeo, NULL);
1228 release_sock(clcsk);
1229 } else if (!atomic_read(&smc_sk(nsk)->conn.bytes_to_rcv)) {
1231 smc_rx_wait(smc_sk(nsk), &timeo, smc_rx_data_available);
1237 sock_put(sk); /* sock_hold above */
1241 static int smc_getname(struct socket *sock, struct sockaddr *addr,
1244 struct smc_sock *smc;
1246 if (peer && (sock->sk->sk_state != SMC_ACTIVE) &&
1247 (sock->sk->sk_state != SMC_APPCLOSEWAIT1))
1250 smc = smc_sk(sock->sk);
1252 return smc->clcsock->ops->getname(smc->clcsock, addr, peer);
1255 static int smc_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
1257 struct sock *sk = sock->sk;
1258 struct smc_sock *smc;
1263 if ((sk->sk_state != SMC_ACTIVE) &&
1264 (sk->sk_state != SMC_APPCLOSEWAIT1) &&
1265 (sk->sk_state != SMC_INIT))
1268 if (msg->msg_flags & MSG_FASTOPEN) {
1269 if (sk->sk_state == SMC_INIT) {
1270 smc->use_fallback = true;
1277 if (smc->use_fallback)
1278 rc = smc->clcsock->ops->sendmsg(smc->clcsock, msg, len);
1280 rc = smc_tx_sendmsg(smc, msg, len);
1286 static int smc_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
1289 struct sock *sk = sock->sk;
1290 struct smc_sock *smc;
1295 if ((sk->sk_state == SMC_INIT) ||
1296 (sk->sk_state == SMC_LISTEN) ||
1297 (sk->sk_state == SMC_CLOSED))
1300 if (sk->sk_state == SMC_PEERFINCLOSEWAIT) {
1305 if (smc->use_fallback) {
1306 rc = smc->clcsock->ops->recvmsg(smc->clcsock, msg, len, flags);
1308 msg->msg_namelen = 0;
1309 rc = smc_rx_recvmsg(smc, msg, NULL, len, flags);
1317 static __poll_t smc_accept_poll(struct sock *parent)
1319 struct smc_sock *isk = smc_sk(parent);
1322 spin_lock(&isk->accept_q_lock);
1323 if (!list_empty(&isk->accept_q))
1324 mask = EPOLLIN | EPOLLRDNORM;
1325 spin_unlock(&isk->accept_q_lock);
1330 static __poll_t smc_poll(struct file *file, struct socket *sock,
1333 struct sock *sk = sock->sk;
1335 struct smc_sock *smc;
1340 smc = smc_sk(sock->sk);
1341 if ((sk->sk_state == SMC_INIT) || smc->use_fallback) {
1342 /* delegate to CLC child sock */
1343 mask = smc->clcsock->ops->poll(file, smc->clcsock, wait);
1344 sk->sk_err = smc->clcsock->sk->sk_err;
1350 if ((sk->sk_shutdown == SHUTDOWN_MASK) ||
1351 (sk->sk_state == SMC_CLOSED))
1353 if (sk->sk_state == SMC_LISTEN) {
1354 /* woken up by sk_data_ready in smc_listen_work() */
1355 mask = smc_accept_poll(sk);
1357 if (atomic_read(&smc->conn.sndbuf_space) ||
1358 sk->sk_shutdown & SEND_SHUTDOWN) {
1359 mask |= EPOLLOUT | EPOLLWRNORM;
1361 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
1362 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
1364 if (atomic_read(&smc->conn.bytes_to_rcv))
1365 mask |= EPOLLIN | EPOLLRDNORM;
1366 if (sk->sk_shutdown & RCV_SHUTDOWN)
1367 mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
1368 if (sk->sk_state == SMC_APPCLOSEWAIT1)
1371 if (smc->conn.urg_state == SMC_URG_VALID)
1379 static int smc_shutdown(struct socket *sock, int how)
1381 struct sock *sk = sock->sk;
1382 struct smc_sock *smc;
1388 if ((how < SHUT_RD) || (how > SHUT_RDWR))
1394 if ((sk->sk_state != SMC_LISTEN) &&
1395 (sk->sk_state != SMC_ACTIVE) &&
1396 (sk->sk_state != SMC_PEERCLOSEWAIT1) &&
1397 (sk->sk_state != SMC_PEERCLOSEWAIT2) &&
1398 (sk->sk_state != SMC_APPCLOSEWAIT1) &&
1399 (sk->sk_state != SMC_APPCLOSEWAIT2) &&
1400 (sk->sk_state != SMC_APPFINCLOSEWAIT))
1402 if (smc->use_fallback) {
1403 rc = kernel_sock_shutdown(smc->clcsock, how);
1404 sk->sk_shutdown = smc->clcsock->sk->sk_shutdown;
1405 if (sk->sk_shutdown == SHUTDOWN_MASK)
1406 sk->sk_state = SMC_CLOSED;
1410 case SHUT_RDWR: /* shutdown in both directions */
1411 rc = smc_close_active(smc);
1414 rc = smc_close_shutdown_write(smc);
1418 /* nothing more to do because peer is not involved */
1422 rc1 = kernel_sock_shutdown(smc->clcsock, how);
1423 /* map sock_shutdown_cmd constants to sk_shutdown value range */
1424 sk->sk_shutdown |= how + 1;
1428 return rc ? rc : rc1;
1431 static int smc_setsockopt(struct socket *sock, int level, int optname,
1432 char __user *optval, unsigned int optlen)
1434 struct sock *sk = sock->sk;
1435 struct smc_sock *smc;
1440 /* generic setsockopts reaching us here always apply to the
1443 rc = smc->clcsock->ops->setsockopt(smc->clcsock, level, optname,
1445 if (smc->clcsock->sk->sk_err) {
1446 sk->sk_err = smc->clcsock->sk->sk_err;
1447 sk->sk_error_report(sk);
1452 if (optlen < sizeof(int))
1454 get_user(val, (int __user *)optval);
1460 case TCP_FASTOPEN_CONNECT:
1461 case TCP_FASTOPEN_KEY:
1462 case TCP_FASTOPEN_NO_COOKIE:
1463 /* option not supported by SMC */
1464 if (sk->sk_state == SMC_INIT) {
1465 smc->use_fallback = true;
1467 if (!smc->use_fallback)
1472 if (sk->sk_state != SMC_INIT && sk->sk_state != SMC_LISTEN) {
1473 if (val && !smc->use_fallback)
1474 mod_delayed_work(system_wq, &smc->conn.tx_work,
1479 if (sk->sk_state != SMC_INIT && sk->sk_state != SMC_LISTEN) {
1480 if (!val && !smc->use_fallback)
1481 mod_delayed_work(system_wq, &smc->conn.tx_work,
1485 case TCP_DEFER_ACCEPT:
1486 smc->sockopt_defer_accept = val;
1496 static int smc_getsockopt(struct socket *sock, int level, int optname,
1497 char __user *optval, int __user *optlen)
1499 struct smc_sock *smc;
1501 smc = smc_sk(sock->sk);
1502 /* socket options apply to the CLC socket */
1503 return smc->clcsock->ops->getsockopt(smc->clcsock, level, optname,
1507 static int smc_ioctl(struct socket *sock, unsigned int cmd,
1510 union smc_host_cursor cons, urg;
1511 struct smc_connection *conn;
1512 struct smc_sock *smc;
1515 smc = smc_sk(sock->sk);
1517 if (smc->use_fallback) {
1520 return smc->clcsock->ops->ioctl(smc->clcsock, cmd, arg);
1523 case SIOCINQ: /* same as FIONREAD */
1524 if (smc->sk.sk_state == SMC_LISTEN)
1526 if (smc->sk.sk_state == SMC_INIT ||
1527 smc->sk.sk_state == SMC_CLOSED)
1530 answ = atomic_read(&smc->conn.bytes_to_rcv);
1533 /* output queue size (not send + not acked) */
1534 if (smc->sk.sk_state == SMC_LISTEN)
1536 if (smc->sk.sk_state == SMC_INIT ||
1537 smc->sk.sk_state == SMC_CLOSED)
1540 answ = smc->conn.sndbuf_desc->len -
1541 atomic_read(&smc->conn.sndbuf_space);
1544 /* output queue size (not send only) */
1545 if (smc->sk.sk_state == SMC_LISTEN)
1547 if (smc->sk.sk_state == SMC_INIT ||
1548 smc->sk.sk_state == SMC_CLOSED)
1551 answ = smc_tx_prepared_sends(&smc->conn);
1554 if (smc->sk.sk_state == SMC_LISTEN)
1556 if (smc->sk.sk_state == SMC_INIT ||
1557 smc->sk.sk_state == SMC_CLOSED) {
1560 smc_curs_write(&cons,
1561 smc_curs_read(&conn->local_tx_ctrl.cons, conn),
1563 smc_curs_write(&urg,
1564 smc_curs_read(&conn->urg_curs, conn),
1566 answ = smc_curs_diff(conn->rmb_desc->len,
1571 return -ENOIOCTLCMD;
1574 return put_user(answ, (int __user *)arg);
1577 static ssize_t smc_sendpage(struct socket *sock, struct page *page,
1578 int offset, size_t size, int flags)
1580 struct sock *sk = sock->sk;
1581 struct smc_sock *smc;
1586 if (sk->sk_state != SMC_ACTIVE) {
1591 if (smc->use_fallback)
1592 rc = kernel_sendpage(smc->clcsock, page, offset,
1595 rc = sock_no_sendpage(sock, page, offset, size, flags);
1601 /* Map the affected portions of the rmbe into an spd, note the number of bytes
1602 * to splice in conn->splice_pending, and press 'go'. Delays consumer cursor
1603 * updates till whenever a respective page has been fully processed.
1604 * Note that subsequent recv() calls have to wait till all splice() processing
1607 static ssize_t smc_splice_read(struct socket *sock, loff_t *ppos,
1608 struct pipe_inode_info *pipe, size_t len,
1611 struct sock *sk = sock->sk;
1612 struct smc_sock *smc;
1618 if (sk->sk_state == SMC_INIT ||
1619 sk->sk_state == SMC_LISTEN ||
1620 sk->sk_state == SMC_CLOSED)
1623 if (sk->sk_state == SMC_PEERFINCLOSEWAIT) {
1628 if (smc->use_fallback) {
1629 rc = smc->clcsock->ops->splice_read(smc->clcsock, ppos,
1636 if (flags & SPLICE_F_NONBLOCK)
1637 flags = MSG_DONTWAIT;
1640 rc = smc_rx_recvmsg(smc, NULL, pipe, len, flags);
1648 /* must look like tcp */
1649 static const struct proto_ops smc_sock_ops = {
1651 .owner = THIS_MODULE,
1652 .release = smc_release,
1654 .connect = smc_connect,
1655 .socketpair = sock_no_socketpair,
1656 .accept = smc_accept,
1657 .getname = smc_getname,
1660 .listen = smc_listen,
1661 .shutdown = smc_shutdown,
1662 .setsockopt = smc_setsockopt,
1663 .getsockopt = smc_getsockopt,
1664 .sendmsg = smc_sendmsg,
1665 .recvmsg = smc_recvmsg,
1666 .mmap = sock_no_mmap,
1667 .sendpage = smc_sendpage,
1668 .splice_read = smc_splice_read,
1671 static int smc_create(struct net *net, struct socket *sock, int protocol,
1674 int family = (protocol == SMCPROTO_SMC6) ? PF_INET6 : PF_INET;
1675 struct smc_sock *smc;
1679 rc = -ESOCKTNOSUPPORT;
1680 if (sock->type != SOCK_STREAM)
1683 rc = -EPROTONOSUPPORT;
1684 if (protocol != SMCPROTO_SMC && protocol != SMCPROTO_SMC6)
1688 sock->ops = &smc_sock_ops;
1689 sk = smc_sock_alloc(net, sock, protocol);
1693 /* create internal TCP socket for CLC handshake and fallback */
1695 smc->use_fallback = false; /* assume rdma capability first */
1696 rc = sock_create_kern(net, family, SOCK_STREAM, IPPROTO_TCP,
1699 sk_common_release(sk);
1702 smc->sk.sk_sndbuf = max(smc->clcsock->sk->sk_sndbuf, SMC_BUF_MIN_SIZE);
1703 smc->sk.sk_rcvbuf = max(smc->clcsock->sk->sk_rcvbuf, SMC_BUF_MIN_SIZE);
1709 static const struct net_proto_family smc_sock_family_ops = {
1711 .owner = THIS_MODULE,
1712 .create = smc_create,
1715 static int __init smc_init(void)
1719 rc = smc_pnet_init();
1723 rc = smc_llc_init();
1725 pr_err("%s: smc_llc_init fails with %d\n", __func__, rc);
1729 rc = smc_cdc_init();
1731 pr_err("%s: smc_cdc_init fails with %d\n", __func__, rc);
1735 rc = proto_register(&smc_proto, 1);
1737 pr_err("%s: proto_register(v4) fails with %d\n", __func__, rc);
1741 rc = proto_register(&smc_proto6, 1);
1743 pr_err("%s: proto_register(v6) fails with %d\n", __func__, rc);
1747 rc = sock_register(&smc_sock_family_ops);
1749 pr_err("%s: sock_register fails with %d\n", __func__, rc);
1752 INIT_HLIST_HEAD(&smc_v4_hashinfo.ht);
1753 INIT_HLIST_HEAD(&smc_v6_hashinfo.ht);
1755 rc = smc_ib_register_client();
1757 pr_err("%s: ib_register fails with %d\n", __func__, rc);
1761 static_branch_enable(&tcp_have_smc);
1765 sock_unregister(PF_SMC);
1767 proto_unregister(&smc_proto6);
1769 proto_unregister(&smc_proto);
1775 static void __exit smc_exit(void)
1778 static_branch_disable(&tcp_have_smc);
1779 smc_ib_unregister_client();
1780 sock_unregister(PF_SMC);
1781 proto_unregister(&smc_proto6);
1782 proto_unregister(&smc_proto);
1786 module_init(smc_init);
1787 module_exit(smc_exit);
1789 MODULE_AUTHOR("Ursula Braun <ubraun@linux.vnet.ibm.com>");
1790 MODULE_DESCRIPTION("smc socket address family");
1791 MODULE_LICENSE("GPL");
1792 MODULE_ALIAS_NETPROTO(PF_SMC);