1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * INET An implementation of the TCP/IP protocol suite for the LINUX
4 * operating system. INET is implemented using the BSD Socket
5 * interface as the means of communication with the user level.
7 * Implementation of the Transmission Control Protocol(TCP).
9 * IPv4 specific functions
13 * linux/ipv4/tcp_input.c
14 * linux/ipv4/tcp_output.c
16 * See tcp.c for author information
21 * David S. Miller : New socket lookup architecture.
22 * This code is dedicated to John Dyson.
23 * David S. Miller : Change semantics of established hash,
24 * half is devoted to TIME_WAIT sockets
25 * and the rest go in the other half.
26 * Andi Kleen : Add support for syncookies and fixed
27 * some bugs: ip options weren't passed to
28 * the TCP layer, missed a check for an
30 * Andi Kleen : Implemented fast path mtu discovery.
31 * Fixed many serious bugs in the
32 * request_sock handling and moved
33 * most of it into the af independent code.
34 * Added tail drop and some other bugfixes.
35 * Added new listen semantics.
36 * Mike McLagan : Routing by source
37 * Juan Jose Ciarlante: ip_dynaddr bits
38 * Andi Kleen: various fixes.
39 * Vitaly E. Lavrov : Transparent proxy revived after year
41 * Andi Kleen : Fix new listen.
42 * Andi Kleen : Fix accept error reporting.
43 * YOSHIFUJI Hideaki @USAGI and: Support IPV6_V6ONLY socket option, which
44 * Alexey Kuznetsov allow both IPv4 and IPv6 sockets to bind
45 * a single port at the same time.
48 #define pr_fmt(fmt) "TCP: " fmt
50 #include <linux/bottom_half.h>
51 #include <linux/types.h>
52 #include <linux/fcntl.h>
53 #include <linux/module.h>
54 #include <linux/random.h>
55 #include <linux/cache.h>
56 #include <linux/jhash.h>
57 #include <linux/init.h>
58 #include <linux/times.h>
59 #include <linux/slab.h>
61 #include <net/net_namespace.h>
63 #include <net/inet_hashtables.h>
65 #include <net/transp_v6.h>
67 #include <net/inet_common.h>
68 #include <net/timewait_sock.h>
70 #include <net/secure_seq.h>
71 #include <net/busy_poll.h>
73 #include <linux/inet.h>
74 #include <linux/ipv6.h>
75 #include <linux/stddef.h>
76 #include <linux/proc_fs.h>
77 #include <linux/seq_file.h>
78 #include <linux/inetdevice.h>
79 #include <linux/btf_ids.h>
81 #include <crypto/hash.h>
82 #include <linux/scatterlist.h>
84 #include <trace/events/tcp.h>
86 #ifdef CONFIG_TCP_MD5SIG
87 static int tcp_v4_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key *key,
88 __be32 daddr, __be32 saddr, const struct tcphdr *th);
91 struct inet_hashinfo tcp_hashinfo;
92 EXPORT_SYMBOL(tcp_hashinfo);
94 static DEFINE_PER_CPU(struct sock *, ipv4_tcp_sk);
96 static u32 tcp_v4_init_seq(const struct sk_buff *skb)
98 return secure_tcp_seq(ip_hdr(skb)->daddr,
101 tcp_hdr(skb)->source);
104 static u32 tcp_v4_init_ts_off(const struct net *net, const struct sk_buff *skb)
106 return secure_tcp_ts_off(net, ip_hdr(skb)->daddr, ip_hdr(skb)->saddr);
109 int tcp_twsk_unique(struct sock *sk, struct sock *sktw, void *twp)
111 int reuse = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_tw_reuse);
112 const struct inet_timewait_sock *tw = inet_twsk(sktw);
113 const struct tcp_timewait_sock *tcptw = tcp_twsk(sktw);
114 struct tcp_sock *tp = tcp_sk(sk);
117 /* Still does not detect *everything* that goes through
118 * lo, since we require a loopback src or dst address
119 * or direct binding to 'lo' interface.
121 bool loopback = false;
122 if (tw->tw_bound_dev_if == LOOPBACK_IFINDEX)
124 #if IS_ENABLED(CONFIG_IPV6)
125 if (tw->tw_family == AF_INET6) {
126 if (ipv6_addr_loopback(&tw->tw_v6_daddr) ||
127 ipv6_addr_v4mapped_loopback(&tw->tw_v6_daddr) ||
128 ipv6_addr_loopback(&tw->tw_v6_rcv_saddr) ||
129 ipv6_addr_v4mapped_loopback(&tw->tw_v6_rcv_saddr))
134 if (ipv4_is_loopback(tw->tw_daddr) ||
135 ipv4_is_loopback(tw->tw_rcv_saddr))
142 /* With PAWS, it is safe from the viewpoint
143 of data integrity. Even without PAWS it is safe provided sequence
144 spaces do not overlap i.e. at data rates <= 80Mbit/sec.
146 Actually, the idea is close to VJ's one, only timestamp cache is
147 held not per host, but per port pair and TW bucket is used as state
150 If TW bucket has been already destroyed we fall back to VJ's scheme
151 and use initial timestamp retrieved from peer table.
153 if (tcptw->tw_ts_recent_stamp &&
154 (!twp || (reuse && time_after32(ktime_get_seconds(),
155 tcptw->tw_ts_recent_stamp)))) {
156 /* In case of repair and re-using TIME-WAIT sockets we still
157 * want to be sure that it is safe as above but honor the
158 * sequence numbers and time stamps set as part of the repair
161 * Without this check re-using a TIME-WAIT socket with TCP
162 * repair would accumulate a -1 on the repair assigned
163 * sequence number. The first time it is reused the sequence
164 * is -1, the second time -2, etc. This fixes that issue
165 * without appearing to create any others.
167 if (likely(!tp->repair)) {
168 u32 seq = tcptw->tw_snd_nxt + 65535 + 2;
172 WRITE_ONCE(tp->write_seq, seq);
173 tp->rx_opt.ts_recent = tcptw->tw_ts_recent;
174 tp->rx_opt.ts_recent_stamp = tcptw->tw_ts_recent_stamp;
182 EXPORT_SYMBOL_GPL(tcp_twsk_unique);
184 static int tcp_v4_pre_connect(struct sock *sk, struct sockaddr *uaddr,
187 /* This check is replicated from tcp_v4_connect() and intended to
188 * prevent BPF program called below from accessing bytes that are out
189 * of the bound specified by user in addr_len.
191 if (addr_len < sizeof(struct sockaddr_in))
194 sock_owned_by_me(sk);
196 return BPF_CGROUP_RUN_PROG_INET4_CONNECT(sk, uaddr);
199 /* This will initiate an outgoing connection. */
200 int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)
202 struct sockaddr_in *usin = (struct sockaddr_in *)uaddr;
203 struct inet_timewait_death_row *tcp_death_row;
204 struct inet_sock *inet = inet_sk(sk);
205 struct tcp_sock *tp = tcp_sk(sk);
206 struct ip_options_rcu *inet_opt;
207 struct net *net = sock_net(sk);
208 __be16 orig_sport, orig_dport;
209 __be32 daddr, nexthop;
214 if (addr_len < sizeof(struct sockaddr_in))
217 if (usin->sin_family != AF_INET)
218 return -EAFNOSUPPORT;
220 nexthop = daddr = usin->sin_addr.s_addr;
221 inet_opt = rcu_dereference_protected(inet->inet_opt,
222 lockdep_sock_is_held(sk));
223 if (inet_opt && inet_opt->opt.srr) {
226 nexthop = inet_opt->opt.faddr;
229 orig_sport = inet->inet_sport;
230 orig_dport = usin->sin_port;
231 fl4 = &inet->cork.fl.u.ip4;
232 rt = ip_route_connect(fl4, nexthop, inet->inet_saddr,
233 sk->sk_bound_dev_if, IPPROTO_TCP, orig_sport,
237 if (err == -ENETUNREACH)
238 IP_INC_STATS(net, IPSTATS_MIB_OUTNOROUTES);
242 if (rt->rt_flags & (RTCF_MULTICAST | RTCF_BROADCAST)) {
247 if (!inet_opt || !inet_opt->opt.srr)
250 tcp_death_row = &sock_net(sk)->ipv4.tcp_death_row;
252 if (!inet->inet_saddr) {
253 err = inet_bhash2_update_saddr(sk, &fl4->saddr, AF_INET);
259 sk_rcv_saddr_set(sk, inet->inet_saddr);
262 if (tp->rx_opt.ts_recent_stamp && inet->inet_daddr != daddr) {
263 /* Reset inherited state */
264 tp->rx_opt.ts_recent = 0;
265 tp->rx_opt.ts_recent_stamp = 0;
266 if (likely(!tp->repair))
267 WRITE_ONCE(tp->write_seq, 0);
270 inet->inet_dport = usin->sin_port;
271 sk_daddr_set(sk, daddr);
273 inet_csk(sk)->icsk_ext_hdr_len = 0;
275 inet_csk(sk)->icsk_ext_hdr_len = inet_opt->opt.optlen;
277 tp->rx_opt.mss_clamp = TCP_MSS_DEFAULT;
279 /* Socket identity is still unknown (sport may be zero).
280 * However we set state to SYN-SENT and not releasing socket
281 * lock select source port, enter ourselves into the hash tables and
282 * complete initialization after this.
284 tcp_set_state(sk, TCP_SYN_SENT);
285 err = inet_hash_connect(tcp_death_row, sk);
291 rt = ip_route_newports(fl4, rt, orig_sport, orig_dport,
292 inet->inet_sport, inet->inet_dport, sk);
298 /* OK, now commit destination to socket. */
299 sk->sk_gso_type = SKB_GSO_TCPV4;
300 sk_setup_caps(sk, &rt->dst);
303 if (likely(!tp->repair)) {
305 WRITE_ONCE(tp->write_seq,
306 secure_tcp_seq(inet->inet_saddr,
310 tp->tsoffset = secure_tcp_ts_off(net, inet->inet_saddr,
314 inet->inet_id = get_random_u16();
316 if (tcp_fastopen_defer_connect(sk, &err))
321 err = tcp_connect(sk);
330 * This unhashes the socket and releases the local port,
333 tcp_set_state(sk, TCP_CLOSE);
334 inet_bhash2_reset_saddr(sk);
336 sk->sk_route_caps = 0;
337 inet->inet_dport = 0;
340 EXPORT_SYMBOL(tcp_v4_connect);
343 * This routine reacts to ICMP_FRAG_NEEDED mtu indications as defined in RFC1191.
344 * It can be called through tcp_release_cb() if socket was owned by user
345 * at the time tcp_v4_err() was called to handle ICMP message.
347 void tcp_v4_mtu_reduced(struct sock *sk)
349 struct inet_sock *inet = inet_sk(sk);
350 struct dst_entry *dst;
353 if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE))
355 mtu = READ_ONCE(tcp_sk(sk)->mtu_info);
356 dst = inet_csk_update_pmtu(sk, mtu);
360 /* Something is about to be wrong... Remember soft error
361 * for the case, if this connection will not able to recover.
363 if (mtu < dst_mtu(dst) && ip_dont_fragment(sk, dst))
364 sk->sk_err_soft = EMSGSIZE;
368 if (inet->pmtudisc != IP_PMTUDISC_DONT &&
369 ip_sk_accept_pmtu(sk) &&
370 inet_csk(sk)->icsk_pmtu_cookie > mtu) {
371 tcp_sync_mss(sk, mtu);
373 /* Resend the TCP packet because it's
374 * clear that the old packet has been
375 * dropped. This is the new "fast" path mtu
378 tcp_simple_retransmit(sk);
379 } /* else let the usual retransmit timer handle it */
381 EXPORT_SYMBOL(tcp_v4_mtu_reduced);
383 static void do_redirect(struct sk_buff *skb, struct sock *sk)
385 struct dst_entry *dst = __sk_dst_check(sk, 0);
388 dst->ops->redirect(dst, sk, skb);
392 /* handle ICMP messages on TCP_NEW_SYN_RECV request sockets */
393 void tcp_req_err(struct sock *sk, u32 seq, bool abort)
395 struct request_sock *req = inet_reqsk(sk);
396 struct net *net = sock_net(sk);
398 /* ICMPs are not backlogged, hence we cannot get
399 * an established socket here.
401 if (seq != tcp_rsk(req)->snt_isn) {
402 __NET_INC_STATS(net, LINUX_MIB_OUTOFWINDOWICMPS);
405 * Still in SYN_RECV, just remove it silently.
406 * There is no good way to pass the error to the newly
407 * created socket, and POSIX does not want network
408 * errors returned from accept().
410 inet_csk_reqsk_queue_drop(req->rsk_listener, req);
411 tcp_listendrop(req->rsk_listener);
415 EXPORT_SYMBOL(tcp_req_err);
417 /* TCP-LD (RFC 6069) logic */
418 void tcp_ld_RTO_revert(struct sock *sk, u32 seq)
420 struct inet_connection_sock *icsk = inet_csk(sk);
421 struct tcp_sock *tp = tcp_sk(sk);
426 if (sock_owned_by_user(sk))
429 if (seq != tp->snd_una || !icsk->icsk_retransmits ||
433 skb = tcp_rtx_queue_head(sk);
434 if (WARN_ON_ONCE(!skb))
437 icsk->icsk_backoff--;
438 icsk->icsk_rto = tp->srtt_us ? __tcp_set_rto(tp) : TCP_TIMEOUT_INIT;
439 icsk->icsk_rto = inet_csk_rto_backoff(icsk, TCP_RTO_MAX);
441 tcp_mstamp_refresh(tp);
442 delta_us = (u32)(tp->tcp_mstamp - tcp_skb_timestamp_us(skb));
443 remaining = icsk->icsk_rto - usecs_to_jiffies(delta_us);
446 inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
447 remaining, TCP_RTO_MAX);
449 /* RTO revert clocked out retransmission.
450 * Will retransmit now.
452 tcp_retransmit_timer(sk);
455 EXPORT_SYMBOL(tcp_ld_RTO_revert);
458 * This routine is called by the ICMP module when it gets some
459 * sort of error condition. If err < 0 then the socket should
460 * be closed and the error returned to the user. If err > 0
461 * it's just the icmp type << 8 | icmp code. After adjustment
462 * header points to the first 8 bytes of the tcp header. We need
463 * to find the appropriate port.
465 * The locking strategy used here is very "optimistic". When
466 * someone else accesses the socket the ICMP is just dropped
467 * and for some paths there is no check at all.
468 * A more general error queue to queue errors for later handling
469 * is probably better.
473 int tcp_v4_err(struct sk_buff *skb, u32 info)
475 const struct iphdr *iph = (const struct iphdr *)skb->data;
476 struct tcphdr *th = (struct tcphdr *)(skb->data + (iph->ihl << 2));
478 struct inet_sock *inet;
479 const int type = icmp_hdr(skb)->type;
480 const int code = icmp_hdr(skb)->code;
482 struct request_sock *fastopen;
485 struct net *net = dev_net(skb->dev);
487 sk = __inet_lookup_established(net, net->ipv4.tcp_death_row.hashinfo,
488 iph->daddr, th->dest, iph->saddr,
489 ntohs(th->source), inet_iif(skb), 0);
491 __ICMP_INC_STATS(net, ICMP_MIB_INERRORS);
494 if (sk->sk_state == TCP_TIME_WAIT) {
495 inet_twsk_put(inet_twsk(sk));
498 seq = ntohl(th->seq);
499 if (sk->sk_state == TCP_NEW_SYN_RECV) {
500 tcp_req_err(sk, seq, type == ICMP_PARAMETERPROB ||
501 type == ICMP_TIME_EXCEEDED ||
502 (type == ICMP_DEST_UNREACH &&
503 (code == ICMP_NET_UNREACH ||
504 code == ICMP_HOST_UNREACH)));
509 /* If too many ICMPs get dropped on busy
510 * servers this needs to be solved differently.
511 * We do take care of PMTU discovery (RFC1191) special case :
512 * we can receive locally generated ICMP messages while socket is held.
514 if (sock_owned_by_user(sk)) {
515 if (!(type == ICMP_DEST_UNREACH && code == ICMP_FRAG_NEEDED))
516 __NET_INC_STATS(net, LINUX_MIB_LOCKDROPPEDICMPS);
518 if (sk->sk_state == TCP_CLOSE)
521 if (static_branch_unlikely(&ip4_min_ttl)) {
522 /* min_ttl can be changed concurrently from do_ip_setsockopt() */
523 if (unlikely(iph->ttl < READ_ONCE(inet_sk(sk)->min_ttl))) {
524 __NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP);
530 /* XXX (TFO) - tp->snd_una should be ISN (tcp_create_openreq_child() */
531 fastopen = rcu_dereference(tp->fastopen_rsk);
532 snd_una = fastopen ? tcp_rsk(fastopen)->snt_isn : tp->snd_una;
533 if (sk->sk_state != TCP_LISTEN &&
534 !between(seq, snd_una, tp->snd_nxt)) {
535 __NET_INC_STATS(net, LINUX_MIB_OUTOFWINDOWICMPS);
541 if (!sock_owned_by_user(sk))
542 do_redirect(skb, sk);
544 case ICMP_SOURCE_QUENCH:
545 /* Just silently ignore these. */
547 case ICMP_PARAMETERPROB:
550 case ICMP_DEST_UNREACH:
551 if (code > NR_ICMP_UNREACH)
554 if (code == ICMP_FRAG_NEEDED) { /* PMTU discovery (RFC1191) */
555 /* We are not interested in TCP_LISTEN and open_requests
556 * (SYN-ACKs send out by Linux are always <576bytes so
557 * they should go through unfragmented).
559 if (sk->sk_state == TCP_LISTEN)
562 WRITE_ONCE(tp->mtu_info, info);
563 if (!sock_owned_by_user(sk)) {
564 tcp_v4_mtu_reduced(sk);
566 if (!test_and_set_bit(TCP_MTU_REDUCED_DEFERRED, &sk->sk_tsq_flags))
572 err = icmp_err_convert[code].errno;
573 /* check if this ICMP message allows revert of backoff.
577 (code == ICMP_NET_UNREACH || code == ICMP_HOST_UNREACH))
578 tcp_ld_RTO_revert(sk, seq);
580 case ICMP_TIME_EXCEEDED:
587 switch (sk->sk_state) {
590 /* Only in fast or simultaneous open. If a fast open socket is
591 * already accepted it is treated as a connected one below.
593 if (fastopen && !fastopen->sk)
596 ip_icmp_error(sk, skb, err, th->dest, info, (u8 *)th);
598 if (!sock_owned_by_user(sk)) {
605 sk->sk_err_soft = err;
610 /* If we've already connected we will keep trying
611 * until we time out, or the user gives up.
613 * rfc1122 4.2.3.9 allows to consider as hard errors
614 * only PROTO_UNREACH and PORT_UNREACH (well, FRAG_FAILED too,
615 * but it is obsoleted by pmtu discovery).
617 * Note, that in modern internet, where routing is unreliable
618 * and in each dark corner broken firewalls sit, sending random
619 * errors ordered by their masters even this two messages finally lose
620 * their original sense (even Linux sends invalid PORT_UNREACHs)
622 * Now we are in compliance with RFCs.
627 if (!sock_owned_by_user(sk) && inet->recverr) {
630 } else { /* Only an error on timeout */
631 sk->sk_err_soft = err;
640 void __tcp_v4_send_check(struct sk_buff *skb, __be32 saddr, __be32 daddr)
642 struct tcphdr *th = tcp_hdr(skb);
644 th->check = ~tcp_v4_check(skb->len, saddr, daddr, 0);
645 skb->csum_start = skb_transport_header(skb) - skb->head;
646 skb->csum_offset = offsetof(struct tcphdr, check);
649 /* This routine computes an IPv4 TCP checksum. */
650 void tcp_v4_send_check(struct sock *sk, struct sk_buff *skb)
652 const struct inet_sock *inet = inet_sk(sk);
654 __tcp_v4_send_check(skb, inet->inet_saddr, inet->inet_daddr);
656 EXPORT_SYMBOL(tcp_v4_send_check);
659 * This routine will send an RST to the other tcp.
661 * Someone asks: why I NEVER use socket parameters (TOS, TTL etc.)
663 * Answer: if a packet caused RST, it is not for a socket
664 * existing in our system, if it is matched to a socket,
665 * it is just duplicate segment or bug in other side's TCP.
666 * So that we build reply only basing on parameters
667 * arrived with segment.
668 * Exception: precedence violation. We do not implement it in any case.
671 #ifdef CONFIG_TCP_MD5SIG
672 #define OPTION_BYTES TCPOLEN_MD5SIG_ALIGNED
674 #define OPTION_BYTES sizeof(__be32)
677 static void tcp_v4_send_reset(const struct sock *sk, struct sk_buff *skb)
679 const struct tcphdr *th = tcp_hdr(skb);
682 __be32 opt[OPTION_BYTES / sizeof(__be32)];
684 struct ip_reply_arg arg;
685 #ifdef CONFIG_TCP_MD5SIG
686 struct tcp_md5sig_key *key = NULL;
687 const __u8 *hash_location = NULL;
688 unsigned char newhash[16];
690 struct sock *sk1 = NULL;
692 u64 transmit_time = 0;
696 /* Never send a reset in response to a reset. */
700 /* If sk not NULL, it means we did a successful lookup and incoming
701 * route had to be correct. prequeue might have dropped our dst.
703 if (!sk && skb_rtable(skb)->rt_type != RTN_LOCAL)
706 /* Swap the send and the receive. */
707 memset(&rep, 0, sizeof(rep));
708 rep.th.dest = th->source;
709 rep.th.source = th->dest;
710 rep.th.doff = sizeof(struct tcphdr) / 4;
714 rep.th.seq = th->ack_seq;
717 rep.th.ack_seq = htonl(ntohl(th->seq) + th->syn + th->fin +
718 skb->len - (th->doff << 2));
721 memset(&arg, 0, sizeof(arg));
722 arg.iov[0].iov_base = (unsigned char *)&rep;
723 arg.iov[0].iov_len = sizeof(rep.th);
725 net = sk ? sock_net(sk) : dev_net(skb_dst(skb)->dev);
726 #ifdef CONFIG_TCP_MD5SIG
728 hash_location = tcp_parse_md5sig_option(th);
729 if (sk && sk_fullsock(sk)) {
730 const union tcp_md5_addr *addr;
733 /* sdif set, means packet ingressed via a device
734 * in an L3 domain and inet_iif is set to it.
736 l3index = tcp_v4_sdif(skb) ? inet_iif(skb) : 0;
737 addr = (union tcp_md5_addr *)&ip_hdr(skb)->saddr;
738 key = tcp_md5_do_lookup(sk, l3index, addr, AF_INET);
739 } else if (hash_location) {
740 const union tcp_md5_addr *addr;
741 int sdif = tcp_v4_sdif(skb);
742 int dif = inet_iif(skb);
746 * active side is lost. Try to find listening socket through
747 * source port, and then find md5 key through listening socket.
748 * we are not loose security here:
749 * Incoming packet is checked with md5 hash with finding key,
750 * no RST generated if md5 hash doesn't match.
752 sk1 = __inet_lookup_listener(net, net->ipv4.tcp_death_row.hashinfo,
753 NULL, 0, ip_hdr(skb)->saddr,
754 th->source, ip_hdr(skb)->daddr,
755 ntohs(th->source), dif, sdif);
756 /* don't send rst if it can't find key */
760 /* sdif set, means packet ingressed via a device
761 * in an L3 domain and dif is set to it.
763 l3index = sdif ? dif : 0;
764 addr = (union tcp_md5_addr *)&ip_hdr(skb)->saddr;
765 key = tcp_md5_do_lookup(sk1, l3index, addr, AF_INET);
770 genhash = tcp_v4_md5_hash_skb(newhash, key, NULL, skb);
771 if (genhash || memcmp(hash_location, newhash, 16) != 0)
777 rep.opt[0] = htonl((TCPOPT_NOP << 24) |
779 (TCPOPT_MD5SIG << 8) |
781 /* Update length and the length the header thinks exists */
782 arg.iov[0].iov_len += TCPOLEN_MD5SIG_ALIGNED;
783 rep.th.doff = arg.iov[0].iov_len / 4;
785 tcp_v4_md5_hash_hdr((__u8 *) &rep.opt[1],
786 key, ip_hdr(skb)->saddr,
787 ip_hdr(skb)->daddr, &rep.th);
790 /* Can't co-exist with TCPMD5, hence check rep.opt[0] */
791 if (rep.opt[0] == 0) {
792 __be32 mrst = mptcp_reset_option(skb);
796 arg.iov[0].iov_len += sizeof(mrst);
797 rep.th.doff = arg.iov[0].iov_len / 4;
801 arg.csum = csum_tcpudp_nofold(ip_hdr(skb)->daddr,
802 ip_hdr(skb)->saddr, /* XXX */
803 arg.iov[0].iov_len, IPPROTO_TCP, 0);
804 arg.csumoffset = offsetof(struct tcphdr, check) / 2;
805 arg.flags = (sk && inet_sk_transparent(sk)) ? IP_REPLY_ARG_NOSRCCHECK : 0;
807 /* When socket is gone, all binding information is lost.
808 * routing might fail in this case. No choice here, if we choose to force
809 * input interface, we will misroute in case of asymmetric route.
812 arg.bound_dev_if = sk->sk_bound_dev_if;
814 trace_tcp_send_reset(sk, skb);
817 BUILD_BUG_ON(offsetof(struct sock, sk_bound_dev_if) !=
818 offsetof(struct inet_timewait_sock, tw_bound_dev_if));
820 arg.tos = ip_hdr(skb)->tos;
821 arg.uid = sock_net_uid(net, sk && sk_fullsock(sk) ? sk : NULL);
823 ctl_sk = this_cpu_read(ipv4_tcp_sk);
824 sock_net_set(ctl_sk, net);
826 ctl_sk->sk_mark = (sk->sk_state == TCP_TIME_WAIT) ?
827 inet_twsk(sk)->tw_mark : sk->sk_mark;
828 ctl_sk->sk_priority = (sk->sk_state == TCP_TIME_WAIT) ?
829 inet_twsk(sk)->tw_priority : sk->sk_priority;
830 transmit_time = tcp_transmit_time(sk);
831 xfrm_sk_clone_policy(ctl_sk, sk);
833 ip_send_unicast_reply(ctl_sk,
834 skb, &TCP_SKB_CB(skb)->header.h4.opt,
835 ip_hdr(skb)->saddr, ip_hdr(skb)->daddr,
836 &arg, arg.iov[0].iov_len,
840 xfrm_sk_free_policy(ctl_sk);
841 sock_net_set(ctl_sk, &init_net);
842 __TCP_INC_STATS(net, TCP_MIB_OUTSEGS);
843 __TCP_INC_STATS(net, TCP_MIB_OUTRSTS);
846 #ifdef CONFIG_TCP_MD5SIG
852 /* The code following below sending ACKs in SYN-RECV and TIME-WAIT states
853 outside socket context is ugly, certainly. What can I do?
856 static void tcp_v4_send_ack(const struct sock *sk,
857 struct sk_buff *skb, u32 seq, u32 ack,
858 u32 win, u32 tsval, u32 tsecr, int oif,
859 struct tcp_md5sig_key *key,
860 int reply_flags, u8 tos)
862 const struct tcphdr *th = tcp_hdr(skb);
865 __be32 opt[(TCPOLEN_TSTAMP_ALIGNED >> 2)
866 #ifdef CONFIG_TCP_MD5SIG
867 + (TCPOLEN_MD5SIG_ALIGNED >> 2)
871 struct net *net = sock_net(sk);
872 struct ip_reply_arg arg;
876 memset(&rep.th, 0, sizeof(struct tcphdr));
877 memset(&arg, 0, sizeof(arg));
879 arg.iov[0].iov_base = (unsigned char *)&rep;
880 arg.iov[0].iov_len = sizeof(rep.th);
882 rep.opt[0] = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) |
883 (TCPOPT_TIMESTAMP << 8) |
885 rep.opt[1] = htonl(tsval);
886 rep.opt[2] = htonl(tsecr);
887 arg.iov[0].iov_len += TCPOLEN_TSTAMP_ALIGNED;
890 /* Swap the send and the receive. */
891 rep.th.dest = th->source;
892 rep.th.source = th->dest;
893 rep.th.doff = arg.iov[0].iov_len / 4;
894 rep.th.seq = htonl(seq);
895 rep.th.ack_seq = htonl(ack);
897 rep.th.window = htons(win);
899 #ifdef CONFIG_TCP_MD5SIG
901 int offset = (tsecr) ? 3 : 0;
903 rep.opt[offset++] = htonl((TCPOPT_NOP << 24) |
905 (TCPOPT_MD5SIG << 8) |
907 arg.iov[0].iov_len += TCPOLEN_MD5SIG_ALIGNED;
908 rep.th.doff = arg.iov[0].iov_len/4;
910 tcp_v4_md5_hash_hdr((__u8 *) &rep.opt[offset],
911 key, ip_hdr(skb)->saddr,
912 ip_hdr(skb)->daddr, &rep.th);
915 arg.flags = reply_flags;
916 arg.csum = csum_tcpudp_nofold(ip_hdr(skb)->daddr,
917 ip_hdr(skb)->saddr, /* XXX */
918 arg.iov[0].iov_len, IPPROTO_TCP, 0);
919 arg.csumoffset = offsetof(struct tcphdr, check) / 2;
921 arg.bound_dev_if = oif;
923 arg.uid = sock_net_uid(net, sk_fullsock(sk) ? sk : NULL);
925 ctl_sk = this_cpu_read(ipv4_tcp_sk);
926 sock_net_set(ctl_sk, net);
927 ctl_sk->sk_mark = (sk->sk_state == TCP_TIME_WAIT) ?
928 inet_twsk(sk)->tw_mark : sk->sk_mark;
929 ctl_sk->sk_priority = (sk->sk_state == TCP_TIME_WAIT) ?
930 inet_twsk(sk)->tw_priority : sk->sk_priority;
931 transmit_time = tcp_transmit_time(sk);
932 ip_send_unicast_reply(ctl_sk,
933 skb, &TCP_SKB_CB(skb)->header.h4.opt,
934 ip_hdr(skb)->saddr, ip_hdr(skb)->daddr,
935 &arg, arg.iov[0].iov_len,
939 sock_net_set(ctl_sk, &init_net);
940 __TCP_INC_STATS(net, TCP_MIB_OUTSEGS);
944 static void tcp_v4_timewait_ack(struct sock *sk, struct sk_buff *skb)
946 struct inet_timewait_sock *tw = inet_twsk(sk);
947 struct tcp_timewait_sock *tcptw = tcp_twsk(sk);
949 tcp_v4_send_ack(sk, skb,
950 tcptw->tw_snd_nxt, tcptw->tw_rcv_nxt,
951 tcptw->tw_rcv_wnd >> tw->tw_rcv_wscale,
952 tcp_time_stamp_raw() + tcptw->tw_ts_offset,
955 tcp_twsk_md5_key(tcptw),
956 tw->tw_transparent ? IP_REPLY_ARG_NOSRCCHECK : 0,
963 static void tcp_v4_reqsk_send_ack(const struct sock *sk, struct sk_buff *skb,
964 struct request_sock *req)
966 const union tcp_md5_addr *addr;
969 /* sk->sk_state == TCP_LISTEN -> for regular TCP_SYN_RECV
970 * sk->sk_state == TCP_SYN_RECV -> for Fast Open.
972 u32 seq = (sk->sk_state == TCP_LISTEN) ? tcp_rsk(req)->snt_isn + 1 :
976 * The window field (SEG.WND) of every outgoing segment, with the
977 * exception of <SYN> segments, MUST be right-shifted by
978 * Rcv.Wind.Shift bits:
980 addr = (union tcp_md5_addr *)&ip_hdr(skb)->saddr;
981 l3index = tcp_v4_sdif(skb) ? inet_iif(skb) : 0;
982 tcp_v4_send_ack(sk, skb, seq,
983 tcp_rsk(req)->rcv_nxt,
984 req->rsk_rcv_wnd >> inet_rsk(req)->rcv_wscale,
985 tcp_time_stamp_raw() + tcp_rsk(req)->ts_off,
988 tcp_md5_do_lookup(sk, l3index, addr, AF_INET),
989 inet_rsk(req)->no_srccheck ? IP_REPLY_ARG_NOSRCCHECK : 0,
994 * Send a SYN-ACK after having received a SYN.
995 * This still operates on a request_sock only, not on a big
998 static int tcp_v4_send_synack(const struct sock *sk, struct dst_entry *dst,
1000 struct request_sock *req,
1001 struct tcp_fastopen_cookie *foc,
1002 enum tcp_synack_type synack_type,
1003 struct sk_buff *syn_skb)
1005 const struct inet_request_sock *ireq = inet_rsk(req);
1008 struct sk_buff *skb;
1011 /* First, grab a route. */
1012 if (!dst && (dst = inet_csk_route_req(sk, &fl4, req)) == NULL)
1015 skb = tcp_make_synack(sk, dst, req, foc, synack_type, syn_skb);
1018 __tcp_v4_send_check(skb, ireq->ir_loc_addr, ireq->ir_rmt_addr);
1020 tos = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reflect_tos) ?
1021 (tcp_rsk(req)->syn_tos & ~INET_ECN_MASK) |
1022 (inet_sk(sk)->tos & INET_ECN_MASK) :
1025 if (!INET_ECN_is_capable(tos) &&
1026 tcp_bpf_ca_needs_ecn((struct sock *)req))
1027 tos |= INET_ECN_ECT_0;
1030 err = ip_build_and_send_pkt(skb, sk, ireq->ir_loc_addr,
1032 rcu_dereference(ireq->ireq_opt),
1035 err = net_xmit_eval(err);
1042 * IPv4 request_sock destructor.
1044 static void tcp_v4_reqsk_destructor(struct request_sock *req)
1046 kfree(rcu_dereference_protected(inet_rsk(req)->ireq_opt, 1));
1049 #ifdef CONFIG_TCP_MD5SIG
1051 * RFC2385 MD5 checksumming requires a mapping of
1052 * IP address->MD5 Key.
1053 * We need to maintain these in the sk structure.
1056 DEFINE_STATIC_KEY_DEFERRED_FALSE(tcp_md5_needed, HZ);
1057 EXPORT_SYMBOL(tcp_md5_needed);
1059 static bool better_md5_match(struct tcp_md5sig_key *old, struct tcp_md5sig_key *new)
1064 /* l3index always overrides non-l3index */
1065 if (old->l3index && new->l3index == 0)
1067 if (old->l3index == 0 && new->l3index)
1070 return old->prefixlen < new->prefixlen;
1073 /* Find the Key structure for an address. */
1074 struct tcp_md5sig_key *__tcp_md5_do_lookup(const struct sock *sk, int l3index,
1075 const union tcp_md5_addr *addr,
1078 const struct tcp_sock *tp = tcp_sk(sk);
1079 struct tcp_md5sig_key *key;
1080 const struct tcp_md5sig_info *md5sig;
1082 struct tcp_md5sig_key *best_match = NULL;
1085 /* caller either holds rcu_read_lock() or socket lock */
1086 md5sig = rcu_dereference_check(tp->md5sig_info,
1087 lockdep_sock_is_held(sk));
1091 hlist_for_each_entry_rcu(key, &md5sig->head, node,
1092 lockdep_sock_is_held(sk)) {
1093 if (key->family != family)
1095 if (key->flags & TCP_MD5SIG_FLAG_IFINDEX && key->l3index != l3index)
1097 if (family == AF_INET) {
1098 mask = inet_make_mask(key->prefixlen);
1099 match = (key->addr.a4.s_addr & mask) ==
1100 (addr->a4.s_addr & mask);
1101 #if IS_ENABLED(CONFIG_IPV6)
1102 } else if (family == AF_INET6) {
1103 match = ipv6_prefix_equal(&key->addr.a6, &addr->a6,
1110 if (match && better_md5_match(best_match, key))
1115 EXPORT_SYMBOL(__tcp_md5_do_lookup);
1117 static struct tcp_md5sig_key *tcp_md5_do_lookup_exact(const struct sock *sk,
1118 const union tcp_md5_addr *addr,
1119 int family, u8 prefixlen,
1120 int l3index, u8 flags)
1122 const struct tcp_sock *tp = tcp_sk(sk);
1123 struct tcp_md5sig_key *key;
1124 unsigned int size = sizeof(struct in_addr);
1125 const struct tcp_md5sig_info *md5sig;
1127 /* caller either holds rcu_read_lock() or socket lock */
1128 md5sig = rcu_dereference_check(tp->md5sig_info,
1129 lockdep_sock_is_held(sk));
1132 #if IS_ENABLED(CONFIG_IPV6)
1133 if (family == AF_INET6)
1134 size = sizeof(struct in6_addr);
1136 hlist_for_each_entry_rcu(key, &md5sig->head, node,
1137 lockdep_sock_is_held(sk)) {
1138 if (key->family != family)
1140 if ((key->flags & TCP_MD5SIG_FLAG_IFINDEX) != (flags & TCP_MD5SIG_FLAG_IFINDEX))
1142 if (key->l3index != l3index)
1144 if (!memcmp(&key->addr, addr, size) &&
1145 key->prefixlen == prefixlen)
1151 struct tcp_md5sig_key *tcp_v4_md5_lookup(const struct sock *sk,
1152 const struct sock *addr_sk)
1154 const union tcp_md5_addr *addr;
1157 l3index = l3mdev_master_ifindex_by_index(sock_net(sk),
1158 addr_sk->sk_bound_dev_if);
1159 addr = (const union tcp_md5_addr *)&addr_sk->sk_daddr;
1160 return tcp_md5_do_lookup(sk, l3index, addr, AF_INET);
1162 EXPORT_SYMBOL(tcp_v4_md5_lookup);
1164 static int tcp_md5sig_info_add(struct sock *sk, gfp_t gfp)
1166 struct tcp_sock *tp = tcp_sk(sk);
1167 struct tcp_md5sig_info *md5sig;
1169 md5sig = kmalloc(sizeof(*md5sig), gfp);
1174 INIT_HLIST_HEAD(&md5sig->head);
1175 rcu_assign_pointer(tp->md5sig_info, md5sig);
1179 /* This can be called on a newly created socket, from other files */
1180 static int __tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr,
1181 int family, u8 prefixlen, int l3index, u8 flags,
1182 const u8 *newkey, u8 newkeylen, gfp_t gfp)
1184 /* Add Key to the list */
1185 struct tcp_md5sig_key *key;
1186 struct tcp_sock *tp = tcp_sk(sk);
1187 struct tcp_md5sig_info *md5sig;
1189 key = tcp_md5_do_lookup_exact(sk, addr, family, prefixlen, l3index, flags);
1191 /* Pre-existing entry - just update that one.
1192 * Note that the key might be used concurrently.
1193 * data_race() is telling kcsan that we do not care of
1194 * key mismatches, since changing MD5 key on live flows
1195 * can lead to packet drops.
1197 data_race(memcpy(key->key, newkey, newkeylen));
1199 /* Pairs with READ_ONCE() in tcp_md5_hash_key().
1200 * Also note that a reader could catch new key->keylen value
1201 * but old key->key[], this is the reason we use __GFP_ZERO
1202 * at sock_kmalloc() time below these lines.
1204 WRITE_ONCE(key->keylen, newkeylen);
1209 md5sig = rcu_dereference_protected(tp->md5sig_info,
1210 lockdep_sock_is_held(sk));
1212 key = sock_kmalloc(sk, sizeof(*key), gfp | __GFP_ZERO);
1215 if (!tcp_alloc_md5sig_pool()) {
1216 sock_kfree_s(sk, key, sizeof(*key));
1220 memcpy(key->key, newkey, newkeylen);
1221 key->keylen = newkeylen;
1222 key->family = family;
1223 key->prefixlen = prefixlen;
1224 key->l3index = l3index;
1226 memcpy(&key->addr, addr,
1227 (IS_ENABLED(CONFIG_IPV6) && family == AF_INET6) ? sizeof(struct in6_addr) :
1228 sizeof(struct in_addr));
1229 hlist_add_head_rcu(&key->node, &md5sig->head);
1233 int tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr,
1234 int family, u8 prefixlen, int l3index, u8 flags,
1235 const u8 *newkey, u8 newkeylen)
1237 struct tcp_sock *tp = tcp_sk(sk);
1239 if (!rcu_dereference_protected(tp->md5sig_info, lockdep_sock_is_held(sk))) {
1240 if (tcp_md5sig_info_add(sk, GFP_KERNEL))
1243 if (!static_branch_inc(&tcp_md5_needed.key)) {
1244 struct tcp_md5sig_info *md5sig;
1246 md5sig = rcu_dereference_protected(tp->md5sig_info, lockdep_sock_is_held(sk));
1247 rcu_assign_pointer(tp->md5sig_info, NULL);
1248 kfree_rcu(md5sig, rcu);
1253 return __tcp_md5_do_add(sk, addr, family, prefixlen, l3index, flags,
1254 newkey, newkeylen, GFP_KERNEL);
1256 EXPORT_SYMBOL(tcp_md5_do_add);
1258 int tcp_md5_key_copy(struct sock *sk, const union tcp_md5_addr *addr,
1259 int family, u8 prefixlen, int l3index,
1260 struct tcp_md5sig_key *key)
1262 struct tcp_sock *tp = tcp_sk(sk);
1264 if (!rcu_dereference_protected(tp->md5sig_info, lockdep_sock_is_held(sk))) {
1265 if (tcp_md5sig_info_add(sk, sk_gfp_mask(sk, GFP_ATOMIC)))
1268 if (!static_key_fast_inc_not_disabled(&tcp_md5_needed.key.key)) {
1269 struct tcp_md5sig_info *md5sig;
1271 md5sig = rcu_dereference_protected(tp->md5sig_info, lockdep_sock_is_held(sk));
1272 net_warn_ratelimited("Too many TCP-MD5 keys in the system\n");
1273 rcu_assign_pointer(tp->md5sig_info, NULL);
1274 kfree_rcu(md5sig, rcu);
1279 return __tcp_md5_do_add(sk, addr, family, prefixlen, l3index,
1280 key->flags, key->key, key->keylen,
1281 sk_gfp_mask(sk, GFP_ATOMIC));
1283 EXPORT_SYMBOL(tcp_md5_key_copy);
1285 int tcp_md5_do_del(struct sock *sk, const union tcp_md5_addr *addr, int family,
1286 u8 prefixlen, int l3index, u8 flags)
1288 struct tcp_md5sig_key *key;
1290 key = tcp_md5_do_lookup_exact(sk, addr, family, prefixlen, l3index, flags);
1293 hlist_del_rcu(&key->node);
1294 atomic_sub(sizeof(*key), &sk->sk_omem_alloc);
1295 kfree_rcu(key, rcu);
1298 EXPORT_SYMBOL(tcp_md5_do_del);
1300 static void tcp_clear_md5_list(struct sock *sk)
1302 struct tcp_sock *tp = tcp_sk(sk);
1303 struct tcp_md5sig_key *key;
1304 struct hlist_node *n;
1305 struct tcp_md5sig_info *md5sig;
1307 md5sig = rcu_dereference_protected(tp->md5sig_info, 1);
1309 hlist_for_each_entry_safe(key, n, &md5sig->head, node) {
1310 hlist_del_rcu(&key->node);
1311 atomic_sub(sizeof(*key), &sk->sk_omem_alloc);
1312 kfree_rcu(key, rcu);
1316 static int tcp_v4_parse_md5_keys(struct sock *sk, int optname,
1317 sockptr_t optval, int optlen)
1319 struct tcp_md5sig cmd;
1320 struct sockaddr_in *sin = (struct sockaddr_in *)&cmd.tcpm_addr;
1321 const union tcp_md5_addr *addr;
1326 if (optlen < sizeof(cmd))
1329 if (copy_from_sockptr(&cmd, optval, sizeof(cmd)))
1332 if (sin->sin_family != AF_INET)
1335 flags = cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX;
1337 if (optname == TCP_MD5SIG_EXT &&
1338 cmd.tcpm_flags & TCP_MD5SIG_FLAG_PREFIX) {
1339 prefixlen = cmd.tcpm_prefixlen;
1344 if (optname == TCP_MD5SIG_EXT && cmd.tcpm_ifindex &&
1345 cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX) {
1346 struct net_device *dev;
1349 dev = dev_get_by_index_rcu(sock_net(sk), cmd.tcpm_ifindex);
1350 if (dev && netif_is_l3_master(dev))
1351 l3index = dev->ifindex;
1355 /* ok to reference set/not set outside of rcu;
1356 * right now device MUST be an L3 master
1358 if (!dev || !l3index)
1362 addr = (union tcp_md5_addr *)&sin->sin_addr.s_addr;
1364 if (!cmd.tcpm_keylen)
1365 return tcp_md5_do_del(sk, addr, AF_INET, prefixlen, l3index, flags);
1367 if (cmd.tcpm_keylen > TCP_MD5SIG_MAXKEYLEN)
1370 return tcp_md5_do_add(sk, addr, AF_INET, prefixlen, l3index, flags,
1371 cmd.tcpm_key, cmd.tcpm_keylen);
1374 static int tcp_v4_md5_hash_headers(struct tcp_md5sig_pool *hp,
1375 __be32 daddr, __be32 saddr,
1376 const struct tcphdr *th, int nbytes)
1378 struct tcp4_pseudohdr *bp;
1379 struct scatterlist sg;
1386 bp->protocol = IPPROTO_TCP;
1387 bp->len = cpu_to_be16(nbytes);
1389 _th = (struct tcphdr *)(bp + 1);
1390 memcpy(_th, th, sizeof(*th));
1393 sg_init_one(&sg, bp, sizeof(*bp) + sizeof(*th));
1394 ahash_request_set_crypt(hp->md5_req, &sg, NULL,
1395 sizeof(*bp) + sizeof(*th));
1396 return crypto_ahash_update(hp->md5_req);
1399 static int tcp_v4_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key *key,
1400 __be32 daddr, __be32 saddr, const struct tcphdr *th)
1402 struct tcp_md5sig_pool *hp;
1403 struct ahash_request *req;
1405 hp = tcp_get_md5sig_pool();
1407 goto clear_hash_noput;
1410 if (crypto_ahash_init(req))
1412 if (tcp_v4_md5_hash_headers(hp, daddr, saddr, th, th->doff << 2))
1414 if (tcp_md5_hash_key(hp, key))
1416 ahash_request_set_crypt(req, NULL, md5_hash, 0);
1417 if (crypto_ahash_final(req))
1420 tcp_put_md5sig_pool();
1424 tcp_put_md5sig_pool();
1426 memset(md5_hash, 0, 16);
1430 int tcp_v4_md5_hash_skb(char *md5_hash, const struct tcp_md5sig_key *key,
1431 const struct sock *sk,
1432 const struct sk_buff *skb)
1434 struct tcp_md5sig_pool *hp;
1435 struct ahash_request *req;
1436 const struct tcphdr *th = tcp_hdr(skb);
1437 __be32 saddr, daddr;
1439 if (sk) { /* valid for establish/request sockets */
1440 saddr = sk->sk_rcv_saddr;
1441 daddr = sk->sk_daddr;
1443 const struct iphdr *iph = ip_hdr(skb);
1448 hp = tcp_get_md5sig_pool();
1450 goto clear_hash_noput;
1453 if (crypto_ahash_init(req))
1456 if (tcp_v4_md5_hash_headers(hp, daddr, saddr, th, skb->len))
1458 if (tcp_md5_hash_skb_data(hp, skb, th->doff << 2))
1460 if (tcp_md5_hash_key(hp, key))
1462 ahash_request_set_crypt(req, NULL, md5_hash, 0);
1463 if (crypto_ahash_final(req))
1466 tcp_put_md5sig_pool();
1470 tcp_put_md5sig_pool();
1472 memset(md5_hash, 0, 16);
1475 EXPORT_SYMBOL(tcp_v4_md5_hash_skb);
1479 static void tcp_v4_init_req(struct request_sock *req,
1480 const struct sock *sk_listener,
1481 struct sk_buff *skb)
1483 struct inet_request_sock *ireq = inet_rsk(req);
1484 struct net *net = sock_net(sk_listener);
1486 sk_rcv_saddr_set(req_to_sk(req), ip_hdr(skb)->daddr);
1487 sk_daddr_set(req_to_sk(req), ip_hdr(skb)->saddr);
1488 RCU_INIT_POINTER(ireq->ireq_opt, tcp_v4_save_options(net, skb));
1491 static struct dst_entry *tcp_v4_route_req(const struct sock *sk,
1492 struct sk_buff *skb,
1494 struct request_sock *req)
1496 tcp_v4_init_req(req, sk, skb);
1498 if (security_inet_conn_request(sk, skb, req))
1501 return inet_csk_route_req(sk, &fl->u.ip4, req);
1504 struct request_sock_ops tcp_request_sock_ops __read_mostly = {
1506 .obj_size = sizeof(struct tcp_request_sock),
1507 .rtx_syn_ack = tcp_rtx_synack,
1508 .send_ack = tcp_v4_reqsk_send_ack,
1509 .destructor = tcp_v4_reqsk_destructor,
1510 .send_reset = tcp_v4_send_reset,
1511 .syn_ack_timeout = tcp_syn_ack_timeout,
1514 const struct tcp_request_sock_ops tcp_request_sock_ipv4_ops = {
1515 .mss_clamp = TCP_MSS_DEFAULT,
1516 #ifdef CONFIG_TCP_MD5SIG
1517 .req_md5_lookup = tcp_v4_md5_lookup,
1518 .calc_md5_hash = tcp_v4_md5_hash_skb,
1520 #ifdef CONFIG_SYN_COOKIES
1521 .cookie_init_seq = cookie_v4_init_sequence,
1523 .route_req = tcp_v4_route_req,
1524 .init_seq = tcp_v4_init_seq,
1525 .init_ts_off = tcp_v4_init_ts_off,
1526 .send_synack = tcp_v4_send_synack,
1529 int tcp_v4_conn_request(struct sock *sk, struct sk_buff *skb)
1531 /* Never answer to SYNs send to broadcast or multicast */
1532 if (skb_rtable(skb)->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST))
1535 return tcp_conn_request(&tcp_request_sock_ops,
1536 &tcp_request_sock_ipv4_ops, sk, skb);
1542 EXPORT_SYMBOL(tcp_v4_conn_request);
1546 * The three way handshake has completed - we got a valid synack -
1547 * now create the new socket.
1549 struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb,
1550 struct request_sock *req,
1551 struct dst_entry *dst,
1552 struct request_sock *req_unhash,
1555 struct inet_request_sock *ireq;
1556 bool found_dup_sk = false;
1557 struct inet_sock *newinet;
1558 struct tcp_sock *newtp;
1560 #ifdef CONFIG_TCP_MD5SIG
1561 const union tcp_md5_addr *addr;
1562 struct tcp_md5sig_key *key;
1565 struct ip_options_rcu *inet_opt;
1567 if (sk_acceptq_is_full(sk))
1570 newsk = tcp_create_openreq_child(sk, req, skb);
1574 newsk->sk_gso_type = SKB_GSO_TCPV4;
1575 inet_sk_rx_dst_set(newsk, skb);
1577 newtp = tcp_sk(newsk);
1578 newinet = inet_sk(newsk);
1579 ireq = inet_rsk(req);
1580 sk_daddr_set(newsk, ireq->ir_rmt_addr);
1581 sk_rcv_saddr_set(newsk, ireq->ir_loc_addr);
1582 newsk->sk_bound_dev_if = ireq->ir_iif;
1583 newinet->inet_saddr = ireq->ir_loc_addr;
1584 inet_opt = rcu_dereference(ireq->ireq_opt);
1585 RCU_INIT_POINTER(newinet->inet_opt, inet_opt);
1586 newinet->mc_index = inet_iif(skb);
1587 newinet->mc_ttl = ip_hdr(skb)->ttl;
1588 newinet->rcv_tos = ip_hdr(skb)->tos;
1589 inet_csk(newsk)->icsk_ext_hdr_len = 0;
1591 inet_csk(newsk)->icsk_ext_hdr_len = inet_opt->opt.optlen;
1592 newinet->inet_id = get_random_u16();
1594 /* Set ToS of the new socket based upon the value of incoming SYN.
1595 * ECT bits are set later in tcp_init_transfer().
1597 if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reflect_tos))
1598 newinet->tos = tcp_rsk(req)->syn_tos & ~INET_ECN_MASK;
1601 dst = inet_csk_route_child_sock(sk, newsk, req);
1605 /* syncookie case : see end of cookie_v4_check() */
1607 sk_setup_caps(newsk, dst);
1609 tcp_ca_openreq_child(newsk, dst);
1611 tcp_sync_mss(newsk, dst_mtu(dst));
1612 newtp->advmss = tcp_mss_clamp(tcp_sk(sk), dst_metric_advmss(dst));
1614 tcp_initialize_rcv_mss(newsk);
1616 #ifdef CONFIG_TCP_MD5SIG
1617 l3index = l3mdev_master_ifindex_by_index(sock_net(sk), ireq->ir_iif);
1618 /* Copy over the MD5 key from the original socket */
1619 addr = (union tcp_md5_addr *)&newinet->inet_daddr;
1620 key = tcp_md5_do_lookup(sk, l3index, addr, AF_INET);
1622 if (tcp_md5_key_copy(newsk, addr, AF_INET, 32, l3index, key))
1624 sk_gso_disable(newsk);
1628 if (__inet_inherit_port(sk, newsk) < 0)
1630 *own_req = inet_ehash_nolisten(newsk, req_to_sk(req_unhash),
1632 if (likely(*own_req)) {
1633 tcp_move_syn(newtp, req);
1634 ireq->ireq_opt = NULL;
1636 newinet->inet_opt = NULL;
1638 if (!req_unhash && found_dup_sk) {
1639 /* This code path should only be executed in the
1640 * syncookie case only
1642 bh_unlock_sock(newsk);
1650 NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENOVERFLOWS);
1657 newinet->inet_opt = NULL;
1658 inet_csk_prepare_forced_close(newsk);
1662 EXPORT_SYMBOL(tcp_v4_syn_recv_sock);
1664 static struct sock *tcp_v4_cookie_check(struct sock *sk, struct sk_buff *skb)
1666 #ifdef CONFIG_SYN_COOKIES
1667 const struct tcphdr *th = tcp_hdr(skb);
1670 sk = cookie_v4_check(sk, skb);
1675 u16 tcp_v4_get_syncookie(struct sock *sk, struct iphdr *iph,
1676 struct tcphdr *th, u32 *cookie)
1679 #ifdef CONFIG_SYN_COOKIES
1680 mss = tcp_get_syncookie_mss(&tcp_request_sock_ops,
1681 &tcp_request_sock_ipv4_ops, sk, th);
1683 *cookie = __cookie_v4_init_sequence(iph, th, &mss);
1684 tcp_synq_overflow(sk);
1690 INDIRECT_CALLABLE_DECLARE(struct dst_entry *ipv4_dst_check(struct dst_entry *,
1692 /* The socket must have it's spinlock held when we get
1693 * here, unless it is a TCP_LISTEN socket.
1695 * We have a potential double-lock case here, so even when
1696 * doing backlog processing we use the BH locking scheme.
1697 * This is because we cannot sleep with the original spinlock
1700 int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb)
1702 enum skb_drop_reason reason;
1705 if (sk->sk_state == TCP_ESTABLISHED) { /* Fast path */
1706 struct dst_entry *dst;
1708 dst = rcu_dereference_protected(sk->sk_rx_dst,
1709 lockdep_sock_is_held(sk));
1711 sock_rps_save_rxhash(sk, skb);
1712 sk_mark_napi_id(sk, skb);
1714 if (sk->sk_rx_dst_ifindex != skb->skb_iif ||
1715 !INDIRECT_CALL_1(dst->ops->check, ipv4_dst_check,
1717 RCU_INIT_POINTER(sk->sk_rx_dst, NULL);
1721 tcp_rcv_established(sk, skb);
1725 reason = SKB_DROP_REASON_NOT_SPECIFIED;
1726 if (tcp_checksum_complete(skb))
1729 if (sk->sk_state == TCP_LISTEN) {
1730 struct sock *nsk = tcp_v4_cookie_check(sk, skb);
1735 if (tcp_child_process(sk, nsk, skb)) {
1742 sock_rps_save_rxhash(sk, skb);
1744 if (tcp_rcv_state_process(sk, skb)) {
1751 tcp_v4_send_reset(rsk, skb);
1753 kfree_skb_reason(skb, reason);
1754 /* Be careful here. If this function gets more complicated and
1755 * gcc suffers from register pressure on the x86, sk (in %ebx)
1756 * might be destroyed here. This current version compiles correctly,
1757 * but you have been warned.
1762 reason = SKB_DROP_REASON_TCP_CSUM;
1763 trace_tcp_bad_csum(skb);
1764 TCP_INC_STATS(sock_net(sk), TCP_MIB_CSUMERRORS);
1765 TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
1768 EXPORT_SYMBOL(tcp_v4_do_rcv);
1770 int tcp_v4_early_demux(struct sk_buff *skb)
1772 struct net *net = dev_net(skb->dev);
1773 const struct iphdr *iph;
1774 const struct tcphdr *th;
1777 if (skb->pkt_type != PACKET_HOST)
1780 if (!pskb_may_pull(skb, skb_transport_offset(skb) + sizeof(struct tcphdr)))
1786 if (th->doff < sizeof(struct tcphdr) / 4)
1789 sk = __inet_lookup_established(net, net->ipv4.tcp_death_row.hashinfo,
1790 iph->saddr, th->source,
1791 iph->daddr, ntohs(th->dest),
1792 skb->skb_iif, inet_sdif(skb));
1795 skb->destructor = sock_edemux;
1796 if (sk_fullsock(sk)) {
1797 struct dst_entry *dst = rcu_dereference(sk->sk_rx_dst);
1800 dst = dst_check(dst, 0);
1802 sk->sk_rx_dst_ifindex == skb->skb_iif)
1803 skb_dst_set_noref(skb, dst);
1809 bool tcp_add_backlog(struct sock *sk, struct sk_buff *skb,
1810 enum skb_drop_reason *reason)
1812 u32 limit, tail_gso_size, tail_gso_segs;
1813 struct skb_shared_info *shinfo;
1814 const struct tcphdr *th;
1815 struct tcphdr *thtail;
1816 struct sk_buff *tail;
1817 unsigned int hdrlen;
1823 /* In case all data was pulled from skb frags (in __pskb_pull_tail()),
1824 * we can fix skb->truesize to its real value to avoid future drops.
1825 * This is valid because skb is not yet charged to the socket.
1826 * It has been noticed pure SACK packets were sometimes dropped
1827 * (if cooked by drivers without copybreak feature).
1833 if (unlikely(tcp_checksum_complete(skb))) {
1835 trace_tcp_bad_csum(skb);
1836 *reason = SKB_DROP_REASON_TCP_CSUM;
1837 __TCP_INC_STATS(sock_net(sk), TCP_MIB_CSUMERRORS);
1838 __TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
1842 /* Attempt coalescing to last skb in backlog, even if we are
1844 * This is okay because skb capacity is limited to MAX_SKB_FRAGS.
1846 th = (const struct tcphdr *)skb->data;
1847 hdrlen = th->doff * 4;
1849 tail = sk->sk_backlog.tail;
1852 thtail = (struct tcphdr *)tail->data;
1854 if (TCP_SKB_CB(tail)->end_seq != TCP_SKB_CB(skb)->seq ||
1855 TCP_SKB_CB(tail)->ip_dsfield != TCP_SKB_CB(skb)->ip_dsfield ||
1856 ((TCP_SKB_CB(tail)->tcp_flags |
1857 TCP_SKB_CB(skb)->tcp_flags) & (TCPHDR_SYN | TCPHDR_RST | TCPHDR_URG)) ||
1858 !((TCP_SKB_CB(tail)->tcp_flags &
1859 TCP_SKB_CB(skb)->tcp_flags) & TCPHDR_ACK) ||
1860 ((TCP_SKB_CB(tail)->tcp_flags ^
1861 TCP_SKB_CB(skb)->tcp_flags) & (TCPHDR_ECE | TCPHDR_CWR)) ||
1862 #ifdef CONFIG_TLS_DEVICE
1863 tail->decrypted != skb->decrypted ||
1865 thtail->doff != th->doff ||
1866 memcmp(thtail + 1, th + 1, hdrlen - sizeof(*th)))
1869 __skb_pull(skb, hdrlen);
1871 shinfo = skb_shinfo(skb);
1872 gso_size = shinfo->gso_size ?: skb->len;
1873 gso_segs = shinfo->gso_segs ?: 1;
1875 shinfo = skb_shinfo(tail);
1876 tail_gso_size = shinfo->gso_size ?: (tail->len - hdrlen);
1877 tail_gso_segs = shinfo->gso_segs ?: 1;
1879 if (skb_try_coalesce(tail, skb, &fragstolen, &delta)) {
1880 TCP_SKB_CB(tail)->end_seq = TCP_SKB_CB(skb)->end_seq;
1882 if (likely(!before(TCP_SKB_CB(skb)->ack_seq, TCP_SKB_CB(tail)->ack_seq))) {
1883 TCP_SKB_CB(tail)->ack_seq = TCP_SKB_CB(skb)->ack_seq;
1884 thtail->window = th->window;
1887 /* We have to update both TCP_SKB_CB(tail)->tcp_flags and
1888 * thtail->fin, so that the fast path in tcp_rcv_established()
1889 * is not entered if we append a packet with a FIN.
1890 * SYN, RST, URG are not present.
1891 * ACK is set on both packets.
1892 * PSH : we do not really care in TCP stack,
1893 * at least for 'GRO' packets.
1895 thtail->fin |= th->fin;
1896 TCP_SKB_CB(tail)->tcp_flags |= TCP_SKB_CB(skb)->tcp_flags;
1898 if (TCP_SKB_CB(skb)->has_rxtstamp) {
1899 TCP_SKB_CB(tail)->has_rxtstamp = true;
1900 tail->tstamp = skb->tstamp;
1901 skb_hwtstamps(tail)->hwtstamp = skb_hwtstamps(skb)->hwtstamp;
1904 /* Not as strict as GRO. We only need to carry mss max value */
1905 shinfo->gso_size = max(gso_size, tail_gso_size);
1906 shinfo->gso_segs = min_t(u32, gso_segs + tail_gso_segs, 0xFFFF);
1908 sk->sk_backlog.len += delta;
1909 __NET_INC_STATS(sock_net(sk),
1910 LINUX_MIB_TCPBACKLOGCOALESCE);
1911 kfree_skb_partial(skb, fragstolen);
1914 __skb_push(skb, hdrlen);
1917 limit = (u32)READ_ONCE(sk->sk_rcvbuf) + (u32)(READ_ONCE(sk->sk_sndbuf) >> 1);
1919 /* Only socket owner can try to collapse/prune rx queues
1920 * to reduce memory overhead, so add a little headroom here.
1921 * Few sockets backlog are possibly concurrently non empty.
1925 if (unlikely(sk_add_backlog(sk, skb, limit))) {
1927 *reason = SKB_DROP_REASON_SOCKET_BACKLOG;
1928 __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPBACKLOGDROP);
1933 EXPORT_SYMBOL(tcp_add_backlog);
1935 int tcp_filter(struct sock *sk, struct sk_buff *skb)
1937 struct tcphdr *th = (struct tcphdr *)skb->data;
1939 return sk_filter_trim_cap(sk, skb, th->doff * 4);
1941 EXPORT_SYMBOL(tcp_filter);
1943 static void tcp_v4_restore_cb(struct sk_buff *skb)
1945 memmove(IPCB(skb), &TCP_SKB_CB(skb)->header.h4,
1946 sizeof(struct inet_skb_parm));
1949 static void tcp_v4_fill_cb(struct sk_buff *skb, const struct iphdr *iph,
1950 const struct tcphdr *th)
1952 /* This is tricky : We move IPCB at its correct location into TCP_SKB_CB()
1953 * barrier() makes sure compiler wont play fool^Waliasing games.
1955 memmove(&TCP_SKB_CB(skb)->header.h4, IPCB(skb),
1956 sizeof(struct inet_skb_parm));
1959 TCP_SKB_CB(skb)->seq = ntohl(th->seq);
1960 TCP_SKB_CB(skb)->end_seq = (TCP_SKB_CB(skb)->seq + th->syn + th->fin +
1961 skb->len - th->doff * 4);
1962 TCP_SKB_CB(skb)->ack_seq = ntohl(th->ack_seq);
1963 TCP_SKB_CB(skb)->tcp_flags = tcp_flag_byte(th);
1964 TCP_SKB_CB(skb)->tcp_tw_isn = 0;
1965 TCP_SKB_CB(skb)->ip_dsfield = ipv4_get_dsfield(iph);
1966 TCP_SKB_CB(skb)->sacked = 0;
1967 TCP_SKB_CB(skb)->has_rxtstamp =
1968 skb->tstamp || skb_hwtstamps(skb)->hwtstamp;
1975 int tcp_v4_rcv(struct sk_buff *skb)
1977 struct net *net = dev_net(skb->dev);
1978 enum skb_drop_reason drop_reason;
1979 int sdif = inet_sdif(skb);
1980 int dif = inet_iif(skb);
1981 const struct iphdr *iph;
1982 const struct tcphdr *th;
1987 drop_reason = SKB_DROP_REASON_NOT_SPECIFIED;
1988 if (skb->pkt_type != PACKET_HOST)
1991 /* Count it even if it's bad */
1992 __TCP_INC_STATS(net, TCP_MIB_INSEGS);
1994 if (!pskb_may_pull(skb, sizeof(struct tcphdr)))
1997 th = (const struct tcphdr *)skb->data;
1999 if (unlikely(th->doff < sizeof(struct tcphdr) / 4)) {
2000 drop_reason = SKB_DROP_REASON_PKT_TOO_SMALL;
2003 if (!pskb_may_pull(skb, th->doff * 4))
2006 /* An explanation is required here, I think.
2007 * Packet length and doff are validated by header prediction,
2008 * provided case of th->doff==0 is eliminated.
2009 * So, we defer the checks. */
2011 if (skb_checksum_init(skb, IPPROTO_TCP, inet_compute_pseudo))
2014 th = (const struct tcphdr *)skb->data;
2017 sk = __inet_lookup_skb(net->ipv4.tcp_death_row.hashinfo,
2018 skb, __tcp_hdrlen(th), th->source,
2019 th->dest, sdif, &refcounted);
2024 if (sk->sk_state == TCP_TIME_WAIT)
2027 if (sk->sk_state == TCP_NEW_SYN_RECV) {
2028 struct request_sock *req = inet_reqsk(sk);
2029 bool req_stolen = false;
2032 sk = req->rsk_listener;
2033 if (!xfrm4_policy_check(sk, XFRM_POLICY_IN, skb))
2034 drop_reason = SKB_DROP_REASON_XFRM_POLICY;
2036 drop_reason = tcp_inbound_md5_hash(sk, skb,
2037 &iph->saddr, &iph->daddr,
2038 AF_INET, dif, sdif);
2039 if (unlikely(drop_reason)) {
2040 sk_drops_add(sk, skb);
2044 if (tcp_checksum_complete(skb)) {
2048 if (unlikely(sk->sk_state != TCP_LISTEN)) {
2049 nsk = reuseport_migrate_sock(sk, req_to_sk(req), skb);
2051 inet_csk_reqsk_queue_drop_and_put(sk, req);
2055 /* reuseport_migrate_sock() has already held one sk_refcnt
2059 /* We own a reference on the listener, increase it again
2060 * as we might lose it too soon.
2066 if (!tcp_filter(sk, skb)) {
2067 th = (const struct tcphdr *)skb->data;
2069 tcp_v4_fill_cb(skb, iph, th);
2070 nsk = tcp_check_req(sk, skb, req, false, &req_stolen);
2072 drop_reason = SKB_DROP_REASON_SOCKET_FILTER;
2077 /* Another cpu got exclusive access to req
2078 * and created a full blown socket.
2079 * Try to feed this packet to this socket
2080 * instead of discarding it.
2082 tcp_v4_restore_cb(skb);
2086 goto discard_and_relse;
2091 tcp_v4_restore_cb(skb);
2092 } else if (tcp_child_process(sk, nsk, skb)) {
2093 tcp_v4_send_reset(nsk, skb);
2094 goto discard_and_relse;
2101 if (static_branch_unlikely(&ip4_min_ttl)) {
2102 /* min_ttl can be changed concurrently from do_ip_setsockopt() */
2103 if (unlikely(iph->ttl < READ_ONCE(inet_sk(sk)->min_ttl))) {
2104 __NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP);
2105 drop_reason = SKB_DROP_REASON_TCP_MINTTL;
2106 goto discard_and_relse;
2110 if (!xfrm4_policy_check(sk, XFRM_POLICY_IN, skb)) {
2111 drop_reason = SKB_DROP_REASON_XFRM_POLICY;
2112 goto discard_and_relse;
2115 drop_reason = tcp_inbound_md5_hash(sk, skb, &iph->saddr,
2116 &iph->daddr, AF_INET, dif, sdif);
2118 goto discard_and_relse;
2122 if (tcp_filter(sk, skb)) {
2123 drop_reason = SKB_DROP_REASON_SOCKET_FILTER;
2124 goto discard_and_relse;
2126 th = (const struct tcphdr *)skb->data;
2128 tcp_v4_fill_cb(skb, iph, th);
2132 if (sk->sk_state == TCP_LISTEN) {
2133 ret = tcp_v4_do_rcv(sk, skb);
2134 goto put_and_return;
2137 sk_incoming_cpu_update(sk);
2139 bh_lock_sock_nested(sk);
2140 tcp_segs_in(tcp_sk(sk), skb);
2142 if (!sock_owned_by_user(sk)) {
2143 ret = tcp_v4_do_rcv(sk, skb);
2145 if (tcp_add_backlog(sk, skb, &drop_reason))
2146 goto discard_and_relse;
2157 drop_reason = SKB_DROP_REASON_NO_SOCKET;
2158 if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb))
2161 tcp_v4_fill_cb(skb, iph, th);
2163 if (tcp_checksum_complete(skb)) {
2165 drop_reason = SKB_DROP_REASON_TCP_CSUM;
2166 trace_tcp_bad_csum(skb);
2167 __TCP_INC_STATS(net, TCP_MIB_CSUMERRORS);
2169 __TCP_INC_STATS(net, TCP_MIB_INERRS);
2171 tcp_v4_send_reset(NULL, skb);
2175 SKB_DR_OR(drop_reason, NOT_SPECIFIED);
2176 /* Discard frame. */
2177 kfree_skb_reason(skb, drop_reason);
2181 sk_drops_add(sk, skb);
2187 if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb)) {
2188 drop_reason = SKB_DROP_REASON_XFRM_POLICY;
2189 inet_twsk_put(inet_twsk(sk));
2193 tcp_v4_fill_cb(skb, iph, th);
2195 if (tcp_checksum_complete(skb)) {
2196 inet_twsk_put(inet_twsk(sk));
2199 switch (tcp_timewait_state_process(inet_twsk(sk), skb, th)) {
2201 struct sock *sk2 = inet_lookup_listener(net,
2202 net->ipv4.tcp_death_row.hashinfo,
2203 skb, __tcp_hdrlen(th),
2204 iph->saddr, th->source,
2205 iph->daddr, th->dest,
2209 inet_twsk_deschedule_put(inet_twsk(sk));
2211 tcp_v4_restore_cb(skb);
2219 tcp_v4_timewait_ack(sk, skb);
2222 tcp_v4_send_reset(sk, skb);
2223 inet_twsk_deschedule_put(inet_twsk(sk));
2225 case TCP_TW_SUCCESS:;
2230 static struct timewait_sock_ops tcp_timewait_sock_ops = {
2231 .twsk_obj_size = sizeof(struct tcp_timewait_sock),
2232 .twsk_unique = tcp_twsk_unique,
2233 .twsk_destructor= tcp_twsk_destructor,
2236 void inet_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb)
2238 struct dst_entry *dst = skb_dst(skb);
2240 if (dst && dst_hold_safe(dst)) {
2241 rcu_assign_pointer(sk->sk_rx_dst, dst);
2242 sk->sk_rx_dst_ifindex = skb->skb_iif;
2245 EXPORT_SYMBOL(inet_sk_rx_dst_set);
2247 const struct inet_connection_sock_af_ops ipv4_specific = {
2248 .queue_xmit = ip_queue_xmit,
2249 .send_check = tcp_v4_send_check,
2250 .rebuild_header = inet_sk_rebuild_header,
2251 .sk_rx_dst_set = inet_sk_rx_dst_set,
2252 .conn_request = tcp_v4_conn_request,
2253 .syn_recv_sock = tcp_v4_syn_recv_sock,
2254 .net_header_len = sizeof(struct iphdr),
2255 .setsockopt = ip_setsockopt,
2256 .getsockopt = ip_getsockopt,
2257 .addr2sockaddr = inet_csk_addr2sockaddr,
2258 .sockaddr_len = sizeof(struct sockaddr_in),
2259 .mtu_reduced = tcp_v4_mtu_reduced,
2261 EXPORT_SYMBOL(ipv4_specific);
2263 #ifdef CONFIG_TCP_MD5SIG
2264 static const struct tcp_sock_af_ops tcp_sock_ipv4_specific = {
2265 .md5_lookup = tcp_v4_md5_lookup,
2266 .calc_md5_hash = tcp_v4_md5_hash_skb,
2267 .md5_parse = tcp_v4_parse_md5_keys,
2271 /* NOTE: A lot of things set to zero explicitly by call to
2272 * sk_alloc() so need not be done here.
2274 static int tcp_v4_init_sock(struct sock *sk)
2276 struct inet_connection_sock *icsk = inet_csk(sk);
2280 icsk->icsk_af_ops = &ipv4_specific;
2282 #ifdef CONFIG_TCP_MD5SIG
2283 tcp_sk(sk)->af_specific = &tcp_sock_ipv4_specific;
2289 void tcp_v4_destroy_sock(struct sock *sk)
2291 struct tcp_sock *tp = tcp_sk(sk);
2293 trace_tcp_destroy_sock(sk);
2295 tcp_clear_xmit_timers(sk);
2297 tcp_cleanup_congestion_control(sk);
2299 tcp_cleanup_ulp(sk);
2301 /* Cleanup up the write buffer. */
2302 tcp_write_queue_purge(sk);
2304 /* Check if we want to disable active TFO */
2305 tcp_fastopen_active_disable_ofo_check(sk);
2307 /* Cleans up our, hopefully empty, out_of_order_queue. */
2308 skb_rbtree_purge(&tp->out_of_order_queue);
2310 #ifdef CONFIG_TCP_MD5SIG
2311 /* Clean up the MD5 key list, if any */
2312 if (tp->md5sig_info) {
2313 tcp_clear_md5_list(sk);
2314 kfree_rcu(rcu_dereference_protected(tp->md5sig_info, 1), rcu);
2315 tp->md5sig_info = NULL;
2316 static_branch_slow_dec_deferred(&tcp_md5_needed);
2320 /* Clean up a referenced TCP bind bucket. */
2321 if (inet_csk(sk)->icsk_bind_hash)
2324 BUG_ON(rcu_access_pointer(tp->fastopen_rsk));
2326 /* If socket is aborted during connect operation */
2327 tcp_free_fastopen_req(tp);
2328 tcp_fastopen_destroy_cipher(sk);
2329 tcp_saved_syn_free(tp);
2331 sk_sockets_allocated_dec(sk);
2333 EXPORT_SYMBOL(tcp_v4_destroy_sock);
2335 #ifdef CONFIG_PROC_FS
2336 /* Proc filesystem TCP sock list dumping. */
2338 static unsigned short seq_file_family(const struct seq_file *seq);
2340 static bool seq_sk_match(struct seq_file *seq, const struct sock *sk)
2342 unsigned short family = seq_file_family(seq);
2344 /* AF_UNSPEC is used as a match all */
2345 return ((family == AF_UNSPEC || family == sk->sk_family) &&
2346 net_eq(sock_net(sk), seq_file_net(seq)));
2349 /* Find a non empty bucket (starting from st->bucket)
2350 * and return the first sk from it.
2352 static void *listening_get_first(struct seq_file *seq)
2354 struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
2355 struct tcp_iter_state *st = seq->private;
2358 for (; st->bucket <= hinfo->lhash2_mask; st->bucket++) {
2359 struct inet_listen_hashbucket *ilb2;
2360 struct hlist_nulls_node *node;
2363 ilb2 = &hinfo->lhash2[st->bucket];
2364 if (hlist_nulls_empty(&ilb2->nulls_head))
2367 spin_lock(&ilb2->lock);
2368 sk_nulls_for_each(sk, node, &ilb2->nulls_head) {
2369 if (seq_sk_match(seq, sk))
2372 spin_unlock(&ilb2->lock);
2378 /* Find the next sk of "cur" within the same bucket (i.e. st->bucket).
2379 * If "cur" is the last one in the st->bucket,
2380 * call listening_get_first() to return the first sk of the next
2383 static void *listening_get_next(struct seq_file *seq, void *cur)
2385 struct tcp_iter_state *st = seq->private;
2386 struct inet_listen_hashbucket *ilb2;
2387 struct hlist_nulls_node *node;
2388 struct inet_hashinfo *hinfo;
2389 struct sock *sk = cur;
2394 sk = sk_nulls_next(sk);
2395 sk_nulls_for_each_from(sk, node) {
2396 if (seq_sk_match(seq, sk))
2400 hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
2401 ilb2 = &hinfo->lhash2[st->bucket];
2402 spin_unlock(&ilb2->lock);
2404 return listening_get_first(seq);
2407 static void *listening_get_idx(struct seq_file *seq, loff_t *pos)
2409 struct tcp_iter_state *st = seq->private;
2414 rc = listening_get_first(seq);
2416 while (rc && *pos) {
2417 rc = listening_get_next(seq, rc);
2423 static inline bool empty_bucket(struct inet_hashinfo *hinfo,
2424 const struct tcp_iter_state *st)
2426 return hlist_nulls_empty(&hinfo->ehash[st->bucket].chain);
2430 * Get first established socket starting from bucket given in st->bucket.
2431 * If st->bucket is zero, the very first socket in the hash is returned.
2433 static void *established_get_first(struct seq_file *seq)
2435 struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
2436 struct tcp_iter_state *st = seq->private;
2439 for (; st->bucket <= hinfo->ehash_mask; ++st->bucket) {
2441 struct hlist_nulls_node *node;
2442 spinlock_t *lock = inet_ehash_lockp(hinfo, st->bucket);
2444 /* Lockless fast path for the common case of empty buckets */
2445 if (empty_bucket(hinfo, st))
2449 sk_nulls_for_each(sk, node, &hinfo->ehash[st->bucket].chain) {
2450 if (seq_sk_match(seq, sk))
2453 spin_unlock_bh(lock);
2459 static void *established_get_next(struct seq_file *seq, void *cur)
2461 struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
2462 struct tcp_iter_state *st = seq->private;
2463 struct hlist_nulls_node *node;
2464 struct sock *sk = cur;
2469 sk = sk_nulls_next(sk);
2471 sk_nulls_for_each_from(sk, node) {
2472 if (seq_sk_match(seq, sk))
2476 spin_unlock_bh(inet_ehash_lockp(hinfo, st->bucket));
2478 return established_get_first(seq);
2481 static void *established_get_idx(struct seq_file *seq, loff_t pos)
2483 struct tcp_iter_state *st = seq->private;
2487 rc = established_get_first(seq);
2490 rc = established_get_next(seq, rc);
2496 static void *tcp_get_idx(struct seq_file *seq, loff_t pos)
2499 struct tcp_iter_state *st = seq->private;
2501 st->state = TCP_SEQ_STATE_LISTENING;
2502 rc = listening_get_idx(seq, &pos);
2505 st->state = TCP_SEQ_STATE_ESTABLISHED;
2506 rc = established_get_idx(seq, pos);
2512 static void *tcp_seek_last_pos(struct seq_file *seq)
2514 struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
2515 struct tcp_iter_state *st = seq->private;
2516 int bucket = st->bucket;
2517 int offset = st->offset;
2518 int orig_num = st->num;
2521 switch (st->state) {
2522 case TCP_SEQ_STATE_LISTENING:
2523 if (st->bucket > hinfo->lhash2_mask)
2525 rc = listening_get_first(seq);
2526 while (offset-- && rc && bucket == st->bucket)
2527 rc = listening_get_next(seq, rc);
2531 st->state = TCP_SEQ_STATE_ESTABLISHED;
2533 case TCP_SEQ_STATE_ESTABLISHED:
2534 if (st->bucket > hinfo->ehash_mask)
2536 rc = established_get_first(seq);
2537 while (offset-- && rc && bucket == st->bucket)
2538 rc = established_get_next(seq, rc);
2546 void *tcp_seq_start(struct seq_file *seq, loff_t *pos)
2548 struct tcp_iter_state *st = seq->private;
2551 if (*pos && *pos == st->last_pos) {
2552 rc = tcp_seek_last_pos(seq);
2557 st->state = TCP_SEQ_STATE_LISTENING;
2561 rc = *pos ? tcp_get_idx(seq, *pos - 1) : SEQ_START_TOKEN;
2564 st->last_pos = *pos;
2567 EXPORT_SYMBOL(tcp_seq_start);
2569 void *tcp_seq_next(struct seq_file *seq, void *v, loff_t *pos)
2571 struct tcp_iter_state *st = seq->private;
2574 if (v == SEQ_START_TOKEN) {
2575 rc = tcp_get_idx(seq, 0);
2579 switch (st->state) {
2580 case TCP_SEQ_STATE_LISTENING:
2581 rc = listening_get_next(seq, v);
2583 st->state = TCP_SEQ_STATE_ESTABLISHED;
2586 rc = established_get_first(seq);
2589 case TCP_SEQ_STATE_ESTABLISHED:
2590 rc = established_get_next(seq, v);
2595 st->last_pos = *pos;
2598 EXPORT_SYMBOL(tcp_seq_next);
2600 void tcp_seq_stop(struct seq_file *seq, void *v)
2602 struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
2603 struct tcp_iter_state *st = seq->private;
2605 switch (st->state) {
2606 case TCP_SEQ_STATE_LISTENING:
2607 if (v != SEQ_START_TOKEN)
2608 spin_unlock(&hinfo->lhash2[st->bucket].lock);
2610 case TCP_SEQ_STATE_ESTABLISHED:
2612 spin_unlock_bh(inet_ehash_lockp(hinfo, st->bucket));
2616 EXPORT_SYMBOL(tcp_seq_stop);
2618 static void get_openreq4(const struct request_sock *req,
2619 struct seq_file *f, int i)
2621 const struct inet_request_sock *ireq = inet_rsk(req);
2622 long delta = req->rsk_timer.expires - jiffies;
2624 seq_printf(f, "%4d: %08X:%04X %08X:%04X"
2625 " %02X %08X:%08X %02X:%08lX %08X %5u %8d %u %d %pK",
2630 ntohs(ireq->ir_rmt_port),
2632 0, 0, /* could print option size, but that is af dependent. */
2633 1, /* timers active (only the expire timer) */
2634 jiffies_delta_to_clock_t(delta),
2636 from_kuid_munged(seq_user_ns(f),
2637 sock_i_uid(req->rsk_listener)),
2638 0, /* non standard timer */
2639 0, /* open_requests have no inode */
2644 static void get_tcp4_sock(struct sock *sk, struct seq_file *f, int i)
2647 unsigned long timer_expires;
2648 const struct tcp_sock *tp = tcp_sk(sk);
2649 const struct inet_connection_sock *icsk = inet_csk(sk);
2650 const struct inet_sock *inet = inet_sk(sk);
2651 const struct fastopen_queue *fastopenq = &icsk->icsk_accept_queue.fastopenq;
2652 __be32 dest = inet->inet_daddr;
2653 __be32 src = inet->inet_rcv_saddr;
2654 __u16 destp = ntohs(inet->inet_dport);
2655 __u16 srcp = ntohs(inet->inet_sport);
2659 if (icsk->icsk_pending == ICSK_TIME_RETRANS ||
2660 icsk->icsk_pending == ICSK_TIME_REO_TIMEOUT ||
2661 icsk->icsk_pending == ICSK_TIME_LOSS_PROBE) {
2663 timer_expires = icsk->icsk_timeout;
2664 } else if (icsk->icsk_pending == ICSK_TIME_PROBE0) {
2666 timer_expires = icsk->icsk_timeout;
2667 } else if (timer_pending(&sk->sk_timer)) {
2669 timer_expires = sk->sk_timer.expires;
2672 timer_expires = jiffies;
2675 state = inet_sk_state_load(sk);
2676 if (state == TCP_LISTEN)
2677 rx_queue = READ_ONCE(sk->sk_ack_backlog);
2679 /* Because we don't lock the socket,
2680 * we might find a transient negative value.
2682 rx_queue = max_t(int, READ_ONCE(tp->rcv_nxt) -
2683 READ_ONCE(tp->copied_seq), 0);
2685 seq_printf(f, "%4d: %08X:%04X %08X:%04X %02X %08X:%08X %02X:%08lX "
2686 "%08X %5u %8d %lu %d %pK %lu %lu %u %u %d",
2687 i, src, srcp, dest, destp, state,
2688 READ_ONCE(tp->write_seq) - tp->snd_una,
2691 jiffies_delta_to_clock_t(timer_expires - jiffies),
2692 icsk->icsk_retransmits,
2693 from_kuid_munged(seq_user_ns(f), sock_i_uid(sk)),
2694 icsk->icsk_probes_out,
2696 refcount_read(&sk->sk_refcnt), sk,
2697 jiffies_to_clock_t(icsk->icsk_rto),
2698 jiffies_to_clock_t(icsk->icsk_ack.ato),
2699 (icsk->icsk_ack.quick << 1) | inet_csk_in_pingpong_mode(sk),
2701 state == TCP_LISTEN ?
2702 fastopenq->max_qlen :
2703 (tcp_in_initial_slowstart(tp) ? -1 : tp->snd_ssthresh));
2706 static void get_timewait4_sock(const struct inet_timewait_sock *tw,
2707 struct seq_file *f, int i)
2709 long delta = tw->tw_timer.expires - jiffies;
2713 dest = tw->tw_daddr;
2714 src = tw->tw_rcv_saddr;
2715 destp = ntohs(tw->tw_dport);
2716 srcp = ntohs(tw->tw_sport);
2718 seq_printf(f, "%4d: %08X:%04X %08X:%04X"
2719 " %02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d %pK",
2720 i, src, srcp, dest, destp, tw->tw_substate, 0, 0,
2721 3, jiffies_delta_to_clock_t(delta), 0, 0, 0, 0,
2722 refcount_read(&tw->tw_refcnt), tw);
2727 static int tcp4_seq_show(struct seq_file *seq, void *v)
2729 struct tcp_iter_state *st;
2730 struct sock *sk = v;
2732 seq_setwidth(seq, TMPSZ - 1);
2733 if (v == SEQ_START_TOKEN) {
2734 seq_puts(seq, " sl local_address rem_address st tx_queue "
2735 "rx_queue tr tm->when retrnsmt uid timeout "
2741 if (sk->sk_state == TCP_TIME_WAIT)
2742 get_timewait4_sock(v, seq, st->num);
2743 else if (sk->sk_state == TCP_NEW_SYN_RECV)
2744 get_openreq4(v, seq, st->num);
2746 get_tcp4_sock(v, seq, st->num);
2752 #ifdef CONFIG_BPF_SYSCALL
2753 struct bpf_tcp_iter_state {
2754 struct tcp_iter_state state;
2755 unsigned int cur_sk;
2756 unsigned int end_sk;
2757 unsigned int max_sk;
2758 struct sock **batch;
2759 bool st_bucket_done;
2762 struct bpf_iter__tcp {
2763 __bpf_md_ptr(struct bpf_iter_meta *, meta);
2764 __bpf_md_ptr(struct sock_common *, sk_common);
2765 uid_t uid __aligned(8);
2768 static int tcp_prog_seq_show(struct bpf_prog *prog, struct bpf_iter_meta *meta,
2769 struct sock_common *sk_common, uid_t uid)
2771 struct bpf_iter__tcp ctx;
2773 meta->seq_num--; /* skip SEQ_START_TOKEN */
2775 ctx.sk_common = sk_common;
2777 return bpf_iter_run_prog(prog, &ctx);
2780 static void bpf_iter_tcp_put_batch(struct bpf_tcp_iter_state *iter)
2782 while (iter->cur_sk < iter->end_sk)
2783 sock_put(iter->batch[iter->cur_sk++]);
2786 static int bpf_iter_tcp_realloc_batch(struct bpf_tcp_iter_state *iter,
2787 unsigned int new_batch_sz)
2789 struct sock **new_batch;
2791 new_batch = kvmalloc(sizeof(*new_batch) * new_batch_sz,
2792 GFP_USER | __GFP_NOWARN);
2796 bpf_iter_tcp_put_batch(iter);
2797 kvfree(iter->batch);
2798 iter->batch = new_batch;
2799 iter->max_sk = new_batch_sz;
2804 static unsigned int bpf_iter_tcp_listening_batch(struct seq_file *seq,
2805 struct sock *start_sk)
2807 struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
2808 struct bpf_tcp_iter_state *iter = seq->private;
2809 struct tcp_iter_state *st = &iter->state;
2810 struct hlist_nulls_node *node;
2811 unsigned int expected = 1;
2814 sock_hold(start_sk);
2815 iter->batch[iter->end_sk++] = start_sk;
2817 sk = sk_nulls_next(start_sk);
2818 sk_nulls_for_each_from(sk, node) {
2819 if (seq_sk_match(seq, sk)) {
2820 if (iter->end_sk < iter->max_sk) {
2822 iter->batch[iter->end_sk++] = sk;
2827 spin_unlock(&hinfo->lhash2[st->bucket].lock);
2832 static unsigned int bpf_iter_tcp_established_batch(struct seq_file *seq,
2833 struct sock *start_sk)
2835 struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
2836 struct bpf_tcp_iter_state *iter = seq->private;
2837 struct tcp_iter_state *st = &iter->state;
2838 struct hlist_nulls_node *node;
2839 unsigned int expected = 1;
2842 sock_hold(start_sk);
2843 iter->batch[iter->end_sk++] = start_sk;
2845 sk = sk_nulls_next(start_sk);
2846 sk_nulls_for_each_from(sk, node) {
2847 if (seq_sk_match(seq, sk)) {
2848 if (iter->end_sk < iter->max_sk) {
2850 iter->batch[iter->end_sk++] = sk;
2855 spin_unlock_bh(inet_ehash_lockp(hinfo, st->bucket));
2860 static struct sock *bpf_iter_tcp_batch(struct seq_file *seq)
2862 struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
2863 struct bpf_tcp_iter_state *iter = seq->private;
2864 struct tcp_iter_state *st = &iter->state;
2865 unsigned int expected;
2866 bool resized = false;
2869 /* The st->bucket is done. Directly advance to the next
2870 * bucket instead of having the tcp_seek_last_pos() to skip
2871 * one by one in the current bucket and eventually find out
2872 * it has to advance to the next bucket.
2874 if (iter->st_bucket_done) {
2877 if (st->state == TCP_SEQ_STATE_LISTENING &&
2878 st->bucket > hinfo->lhash2_mask) {
2879 st->state = TCP_SEQ_STATE_ESTABLISHED;
2885 /* Get a new batch */
2888 iter->st_bucket_done = false;
2890 sk = tcp_seek_last_pos(seq);
2892 return NULL; /* Done */
2894 if (st->state == TCP_SEQ_STATE_LISTENING)
2895 expected = bpf_iter_tcp_listening_batch(seq, sk);
2897 expected = bpf_iter_tcp_established_batch(seq, sk);
2899 if (iter->end_sk == expected) {
2900 iter->st_bucket_done = true;
2904 if (!resized && !bpf_iter_tcp_realloc_batch(iter, expected * 3 / 2)) {
2912 static void *bpf_iter_tcp_seq_start(struct seq_file *seq, loff_t *pos)
2914 /* bpf iter does not support lseek, so it always
2915 * continue from where it was stop()-ped.
2918 return bpf_iter_tcp_batch(seq);
2920 return SEQ_START_TOKEN;
2923 static void *bpf_iter_tcp_seq_next(struct seq_file *seq, void *v, loff_t *pos)
2925 struct bpf_tcp_iter_state *iter = seq->private;
2926 struct tcp_iter_state *st = &iter->state;
2929 /* Whenever seq_next() is called, the iter->cur_sk is
2930 * done with seq_show(), so advance to the next sk in
2933 if (iter->cur_sk < iter->end_sk) {
2934 /* Keeping st->num consistent in tcp_iter_state.
2935 * bpf_iter_tcp does not use st->num.
2936 * meta.seq_num is used instead.
2939 /* Move st->offset to the next sk in the bucket such that
2940 * the future start() will resume at st->offset in
2941 * st->bucket. See tcp_seek_last_pos().
2944 sock_put(iter->batch[iter->cur_sk++]);
2947 if (iter->cur_sk < iter->end_sk)
2948 sk = iter->batch[iter->cur_sk];
2950 sk = bpf_iter_tcp_batch(seq);
2953 /* Keeping st->last_pos consistent in tcp_iter_state.
2954 * bpf iter does not do lseek, so st->last_pos always equals to *pos.
2956 st->last_pos = *pos;
2960 static int bpf_iter_tcp_seq_show(struct seq_file *seq, void *v)
2962 struct bpf_iter_meta meta;
2963 struct bpf_prog *prog;
2964 struct sock *sk = v;
2969 if (v == SEQ_START_TOKEN)
2972 if (sk_fullsock(sk))
2973 slow = lock_sock_fast(sk);
2975 if (unlikely(sk_unhashed(sk))) {
2980 if (sk->sk_state == TCP_TIME_WAIT) {
2982 } else if (sk->sk_state == TCP_NEW_SYN_RECV) {
2983 const struct request_sock *req = v;
2985 uid = from_kuid_munged(seq_user_ns(seq),
2986 sock_i_uid(req->rsk_listener));
2988 uid = from_kuid_munged(seq_user_ns(seq), sock_i_uid(sk));
2992 prog = bpf_iter_get_info(&meta, false);
2993 ret = tcp_prog_seq_show(prog, &meta, v, uid);
2996 if (sk_fullsock(sk))
2997 unlock_sock_fast(sk, slow);
3002 static void bpf_iter_tcp_seq_stop(struct seq_file *seq, void *v)
3004 struct bpf_tcp_iter_state *iter = seq->private;
3005 struct bpf_iter_meta meta;
3006 struct bpf_prog *prog;
3010 prog = bpf_iter_get_info(&meta, true);
3012 (void)tcp_prog_seq_show(prog, &meta, v, 0);
3015 if (iter->cur_sk < iter->end_sk) {
3016 bpf_iter_tcp_put_batch(iter);
3017 iter->st_bucket_done = false;
3021 static const struct seq_operations bpf_iter_tcp_seq_ops = {
3022 .show = bpf_iter_tcp_seq_show,
3023 .start = bpf_iter_tcp_seq_start,
3024 .next = bpf_iter_tcp_seq_next,
3025 .stop = bpf_iter_tcp_seq_stop,
3028 static unsigned short seq_file_family(const struct seq_file *seq)
3030 const struct tcp_seq_afinfo *afinfo;
3032 #ifdef CONFIG_BPF_SYSCALL
3033 /* Iterated from bpf_iter. Let the bpf prog to filter instead. */
3034 if (seq->op == &bpf_iter_tcp_seq_ops)
3038 /* Iterated from proc fs */
3039 afinfo = pde_data(file_inode(seq->file));
3040 return afinfo->family;
3043 static const struct seq_operations tcp4_seq_ops = {
3044 .show = tcp4_seq_show,
3045 .start = tcp_seq_start,
3046 .next = tcp_seq_next,
3047 .stop = tcp_seq_stop,
3050 static struct tcp_seq_afinfo tcp4_seq_afinfo = {
3054 static int __net_init tcp4_proc_init_net(struct net *net)
3056 if (!proc_create_net_data("tcp", 0444, net->proc_net, &tcp4_seq_ops,
3057 sizeof(struct tcp_iter_state), &tcp4_seq_afinfo))
3062 static void __net_exit tcp4_proc_exit_net(struct net *net)
3064 remove_proc_entry("tcp", net->proc_net);
3067 static struct pernet_operations tcp4_net_ops = {
3068 .init = tcp4_proc_init_net,
3069 .exit = tcp4_proc_exit_net,
3072 int __init tcp4_proc_init(void)
3074 return register_pernet_subsys(&tcp4_net_ops);
3077 void tcp4_proc_exit(void)
3079 unregister_pernet_subsys(&tcp4_net_ops);
3081 #endif /* CONFIG_PROC_FS */
3083 /* @wake is one when sk_stream_write_space() calls us.
3084 * This sends EPOLLOUT only if notsent_bytes is half the limit.
3085 * This mimics the strategy used in sock_def_write_space().
3087 bool tcp_stream_memory_free(const struct sock *sk, int wake)
3089 const struct tcp_sock *tp = tcp_sk(sk);
3090 u32 notsent_bytes = READ_ONCE(tp->write_seq) -
3091 READ_ONCE(tp->snd_nxt);
3093 return (notsent_bytes << wake) < tcp_notsent_lowat(tp);
3095 EXPORT_SYMBOL(tcp_stream_memory_free);
3097 struct proto tcp_prot = {
3099 .owner = THIS_MODULE,
3101 .pre_connect = tcp_v4_pre_connect,
3102 .connect = tcp_v4_connect,
3103 .disconnect = tcp_disconnect,
3104 .accept = inet_csk_accept,
3106 .init = tcp_v4_init_sock,
3107 .destroy = tcp_v4_destroy_sock,
3108 .shutdown = tcp_shutdown,
3109 .setsockopt = tcp_setsockopt,
3110 .getsockopt = tcp_getsockopt,
3111 .bpf_bypass_getsockopt = tcp_bpf_bypass_getsockopt,
3112 .keepalive = tcp_set_keepalive,
3113 .recvmsg = tcp_recvmsg,
3114 .sendmsg = tcp_sendmsg,
3115 .sendpage = tcp_sendpage,
3116 .backlog_rcv = tcp_v4_do_rcv,
3117 .release_cb = tcp_release_cb,
3119 .unhash = inet_unhash,
3120 .get_port = inet_csk_get_port,
3121 .put_port = inet_put_port,
3122 #ifdef CONFIG_BPF_SYSCALL
3123 .psock_update_sk_prot = tcp_bpf_update_proto,
3125 .enter_memory_pressure = tcp_enter_memory_pressure,
3126 .leave_memory_pressure = tcp_leave_memory_pressure,
3127 .stream_memory_free = tcp_stream_memory_free,
3128 .sockets_allocated = &tcp_sockets_allocated,
3129 .orphan_count = &tcp_orphan_count,
3131 .memory_allocated = &tcp_memory_allocated,
3132 .per_cpu_fw_alloc = &tcp_memory_per_cpu_fw_alloc,
3134 .memory_pressure = &tcp_memory_pressure,
3135 .sysctl_mem = sysctl_tcp_mem,
3136 .sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_tcp_wmem),
3137 .sysctl_rmem_offset = offsetof(struct net, ipv4.sysctl_tcp_rmem),
3138 .max_header = MAX_TCP_HEADER,
3139 .obj_size = sizeof(struct tcp_sock),
3140 .slab_flags = SLAB_TYPESAFE_BY_RCU,
3141 .twsk_prot = &tcp_timewait_sock_ops,
3142 .rsk_prot = &tcp_request_sock_ops,
3144 .no_autobind = true,
3145 .diag_destroy = tcp_abort,
3147 EXPORT_SYMBOL(tcp_prot);
3149 static void __net_exit tcp_sk_exit(struct net *net)
3151 if (net->ipv4.tcp_congestion_control)
3152 bpf_module_put(net->ipv4.tcp_congestion_control,
3153 net->ipv4.tcp_congestion_control->owner);
3156 static void __net_init tcp_set_hashinfo(struct net *net)
3158 struct inet_hashinfo *hinfo;
3159 unsigned int ehash_entries;
3160 struct net *old_net;
3162 if (net_eq(net, &init_net))
3165 old_net = current->nsproxy->net_ns;
3166 ehash_entries = READ_ONCE(old_net->ipv4.sysctl_tcp_child_ehash_entries);
3170 ehash_entries = roundup_pow_of_two(ehash_entries);
3171 hinfo = inet_pernet_hashinfo_alloc(&tcp_hashinfo, ehash_entries);
3173 pr_warn("Failed to allocate TCP ehash (entries: %u) "
3174 "for a netns, fallback to the global one\n",
3177 hinfo = &tcp_hashinfo;
3178 ehash_entries = tcp_hashinfo.ehash_mask + 1;
3181 net->ipv4.tcp_death_row.hashinfo = hinfo;
3182 net->ipv4.tcp_death_row.sysctl_max_tw_buckets = ehash_entries / 2;
3183 net->ipv4.sysctl_max_syn_backlog = max(128U, ehash_entries / 128);
3186 static int __net_init tcp_sk_init(struct net *net)
3188 net->ipv4.sysctl_tcp_ecn = 2;
3189 net->ipv4.sysctl_tcp_ecn_fallback = 1;
3191 net->ipv4.sysctl_tcp_base_mss = TCP_BASE_MSS;
3192 net->ipv4.sysctl_tcp_min_snd_mss = TCP_MIN_SND_MSS;
3193 net->ipv4.sysctl_tcp_probe_threshold = TCP_PROBE_THRESHOLD;
3194 net->ipv4.sysctl_tcp_probe_interval = TCP_PROBE_INTERVAL;
3195 net->ipv4.sysctl_tcp_mtu_probe_floor = TCP_MIN_SND_MSS;
3197 net->ipv4.sysctl_tcp_keepalive_time = TCP_KEEPALIVE_TIME;
3198 net->ipv4.sysctl_tcp_keepalive_probes = TCP_KEEPALIVE_PROBES;
3199 net->ipv4.sysctl_tcp_keepalive_intvl = TCP_KEEPALIVE_INTVL;
3201 net->ipv4.sysctl_tcp_syn_retries = TCP_SYN_RETRIES;
3202 net->ipv4.sysctl_tcp_synack_retries = TCP_SYNACK_RETRIES;
3203 net->ipv4.sysctl_tcp_syncookies = 1;
3204 net->ipv4.sysctl_tcp_reordering = TCP_FASTRETRANS_THRESH;
3205 net->ipv4.sysctl_tcp_retries1 = TCP_RETR1;
3206 net->ipv4.sysctl_tcp_retries2 = TCP_RETR2;
3207 net->ipv4.sysctl_tcp_orphan_retries = 0;
3208 net->ipv4.sysctl_tcp_fin_timeout = TCP_FIN_TIMEOUT;
3209 net->ipv4.sysctl_tcp_notsent_lowat = UINT_MAX;
3210 net->ipv4.sysctl_tcp_tw_reuse = 2;
3211 net->ipv4.sysctl_tcp_no_ssthresh_metrics_save = 1;
3213 refcount_set(&net->ipv4.tcp_death_row.tw_refcount, 1);
3214 tcp_set_hashinfo(net);
3216 net->ipv4.sysctl_tcp_sack = 1;
3217 net->ipv4.sysctl_tcp_window_scaling = 1;
3218 net->ipv4.sysctl_tcp_timestamps = 1;
3219 net->ipv4.sysctl_tcp_early_retrans = 3;
3220 net->ipv4.sysctl_tcp_recovery = TCP_RACK_LOSS_DETECTION;
3221 net->ipv4.sysctl_tcp_slow_start_after_idle = 1; /* By default, RFC2861 behavior. */
3222 net->ipv4.sysctl_tcp_retrans_collapse = 1;
3223 net->ipv4.sysctl_tcp_max_reordering = 300;
3224 net->ipv4.sysctl_tcp_dsack = 1;
3225 net->ipv4.sysctl_tcp_app_win = 31;
3226 net->ipv4.sysctl_tcp_adv_win_scale = 1;
3227 net->ipv4.sysctl_tcp_frto = 2;
3228 net->ipv4.sysctl_tcp_moderate_rcvbuf = 1;
3229 /* This limits the percentage of the congestion window which we
3230 * will allow a single TSO frame to consume. Building TSO frames
3231 * which are too large can cause TCP streams to be bursty.
3233 net->ipv4.sysctl_tcp_tso_win_divisor = 3;
3234 /* Default TSQ limit of 16 TSO segments */
3235 net->ipv4.sysctl_tcp_limit_output_bytes = 16 * 65536;
3237 /* rfc5961 challenge ack rate limiting, per net-ns, disabled by default. */
3238 net->ipv4.sysctl_tcp_challenge_ack_limit = INT_MAX;
3240 net->ipv4.sysctl_tcp_min_tso_segs = 2;
3241 net->ipv4.sysctl_tcp_tso_rtt_log = 9; /* 2^9 = 512 usec */
3242 net->ipv4.sysctl_tcp_min_rtt_wlen = 300;
3243 net->ipv4.sysctl_tcp_autocorking = 1;
3244 net->ipv4.sysctl_tcp_invalid_ratelimit = HZ/2;
3245 net->ipv4.sysctl_tcp_pacing_ss_ratio = 200;
3246 net->ipv4.sysctl_tcp_pacing_ca_ratio = 120;
3247 if (net != &init_net) {
3248 memcpy(net->ipv4.sysctl_tcp_rmem,
3249 init_net.ipv4.sysctl_tcp_rmem,
3250 sizeof(init_net.ipv4.sysctl_tcp_rmem));
3251 memcpy(net->ipv4.sysctl_tcp_wmem,
3252 init_net.ipv4.sysctl_tcp_wmem,
3253 sizeof(init_net.ipv4.sysctl_tcp_wmem));
3255 net->ipv4.sysctl_tcp_comp_sack_delay_ns = NSEC_PER_MSEC;
3256 net->ipv4.sysctl_tcp_comp_sack_slack_ns = 100 * NSEC_PER_USEC;
3257 net->ipv4.sysctl_tcp_comp_sack_nr = 44;
3258 net->ipv4.sysctl_tcp_fastopen = TFO_CLIENT_ENABLE;
3259 net->ipv4.sysctl_tcp_fastopen_blackhole_timeout = 0;
3260 atomic_set(&net->ipv4.tfo_active_disable_times, 0);
3262 /* Set default values for PLB */
3263 net->ipv4.sysctl_tcp_plb_enabled = 0; /* Disabled by default */
3264 net->ipv4.sysctl_tcp_plb_idle_rehash_rounds = 3;
3265 net->ipv4.sysctl_tcp_plb_rehash_rounds = 12;
3266 net->ipv4.sysctl_tcp_plb_suspend_rto_sec = 60;
3267 /* Default congestion threshold for PLB to mark a round is 50% */
3268 net->ipv4.sysctl_tcp_plb_cong_thresh = (1 << TCP_PLB_SCALE) / 2;
3270 /* Reno is always built in */
3271 if (!net_eq(net, &init_net) &&
3272 bpf_try_module_get(init_net.ipv4.tcp_congestion_control,
3273 init_net.ipv4.tcp_congestion_control->owner))
3274 net->ipv4.tcp_congestion_control = init_net.ipv4.tcp_congestion_control;
3276 net->ipv4.tcp_congestion_control = &tcp_reno;
3281 static void __net_exit tcp_sk_exit_batch(struct list_head *net_exit_list)
3285 tcp_twsk_purge(net_exit_list, AF_INET);
3287 list_for_each_entry(net, net_exit_list, exit_list) {
3288 inet_pernet_hashinfo_free(net->ipv4.tcp_death_row.hashinfo);
3289 WARN_ON_ONCE(!refcount_dec_and_test(&net->ipv4.tcp_death_row.tw_refcount));
3290 tcp_fastopen_ctx_destroy(net);
3294 static struct pernet_operations __net_initdata tcp_sk_ops = {
3295 .init = tcp_sk_init,
3296 .exit = tcp_sk_exit,
3297 .exit_batch = tcp_sk_exit_batch,
3300 #if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
3301 DEFINE_BPF_ITER_FUNC(tcp, struct bpf_iter_meta *meta,
3302 struct sock_common *sk_common, uid_t uid)
3304 #define INIT_BATCH_SZ 16
3306 static int bpf_iter_init_tcp(void *priv_data, struct bpf_iter_aux_info *aux)
3308 struct bpf_tcp_iter_state *iter = priv_data;
3311 err = bpf_iter_init_seq_net(priv_data, aux);
3315 err = bpf_iter_tcp_realloc_batch(iter, INIT_BATCH_SZ);
3317 bpf_iter_fini_seq_net(priv_data);
3324 static void bpf_iter_fini_tcp(void *priv_data)
3326 struct bpf_tcp_iter_state *iter = priv_data;
3328 bpf_iter_fini_seq_net(priv_data);
3329 kvfree(iter->batch);
3332 static const struct bpf_iter_seq_info tcp_seq_info = {
3333 .seq_ops = &bpf_iter_tcp_seq_ops,
3334 .init_seq_private = bpf_iter_init_tcp,
3335 .fini_seq_private = bpf_iter_fini_tcp,
3336 .seq_priv_size = sizeof(struct bpf_tcp_iter_state),
3339 static const struct bpf_func_proto *
3340 bpf_iter_tcp_get_func_proto(enum bpf_func_id func_id,
3341 const struct bpf_prog *prog)
3344 case BPF_FUNC_setsockopt:
3345 return &bpf_sk_setsockopt_proto;
3346 case BPF_FUNC_getsockopt:
3347 return &bpf_sk_getsockopt_proto;
3353 static struct bpf_iter_reg tcp_reg_info = {
3355 .ctx_arg_info_size = 1,
3357 { offsetof(struct bpf_iter__tcp, sk_common),
3358 PTR_TO_BTF_ID_OR_NULL },
3360 .get_func_proto = bpf_iter_tcp_get_func_proto,
3361 .seq_info = &tcp_seq_info,
3364 static void __init bpf_iter_register(void)
3366 tcp_reg_info.ctx_arg_info[0].btf_id = btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON];
3367 if (bpf_iter_reg_target(&tcp_reg_info))
3368 pr_warn("Warning: could not register bpf iterator tcp\n");
3373 void __init tcp_v4_init(void)
3377 for_each_possible_cpu(cpu) {
3380 res = inet_ctl_sock_create(&sk, PF_INET, SOCK_RAW,
3381 IPPROTO_TCP, &init_net);
3383 panic("Failed to create the TCP control socket.\n");
3384 sock_set_flag(sk, SOCK_USE_WRITE_QUEUE);
3386 /* Please enforce IP_DF and IPID==0 for RST and
3387 * ACK sent in SYN-RECV and TIME-WAIT state.
3389 inet_sk(sk)->pmtudisc = IP_PMTUDISC_DO;
3391 per_cpu(ipv4_tcp_sk, cpu) = sk;
3393 if (register_pernet_subsys(&tcp_sk_ops))
3394 panic("Failed to create the TCP control socket.\n");
3396 #if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
3397 bpf_iter_register();