1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * INET An implementation of the TCP/IP protocol suite for the LINUX
4 * operating system. INET is implemented using the BSD Socket
5 * interface as the means of communication with the user level.
7 * Implementation of the Transmission Control Protocol(TCP).
9 * IPv4 specific functions
13 * linux/ipv4/tcp_input.c
14 * linux/ipv4/tcp_output.c
16 * See tcp.c for author information
21 * David S. Miller : New socket lookup architecture.
22 * This code is dedicated to John Dyson.
23 * David S. Miller : Change semantics of established hash,
24 * half is devoted to TIME_WAIT sockets
25 * and the rest go in the other half.
26 * Andi Kleen : Add support for syncookies and fixed
27 * some bugs: ip options weren't passed to
28 * the TCP layer, missed a check for an
30 * Andi Kleen : Implemented fast path mtu discovery.
31 * Fixed many serious bugs in the
32 * request_sock handling and moved
33 * most of it into the af independent code.
34 * Added tail drop and some other bugfixes.
35 * Added new listen semantics.
36 * Mike McLagan : Routing by source
37 * Juan Jose Ciarlante: ip_dynaddr bits
38 * Andi Kleen: various fixes.
39 * Vitaly E. Lavrov : Transparent proxy revived after year
41 * Andi Kleen : Fix new listen.
42 * Andi Kleen : Fix accept error reporting.
43 * YOSHIFUJI Hideaki @USAGI and: Support IPV6_V6ONLY socket option, which
44 * Alexey Kuznetsov allow both IPv4 and IPv6 sockets to bind
45 * a single port at the same time.
48 #define pr_fmt(fmt) "TCP: " fmt
50 #include <linux/bottom_half.h>
51 #include <linux/types.h>
52 #include <linux/fcntl.h>
53 #include <linux/module.h>
54 #include <linux/random.h>
55 #include <linux/cache.h>
56 #include <linux/jhash.h>
57 #include <linux/init.h>
58 #include <linux/times.h>
59 #include <linux/slab.h>
61 #include <net/net_namespace.h>
63 #include <net/inet_hashtables.h>
65 #include <net/transp_v6.h>
67 #include <net/inet_common.h>
68 #include <net/timewait_sock.h>
70 #include <net/secure_seq.h>
71 #include <net/busy_poll.h>
73 #include <linux/inet.h>
74 #include <linux/ipv6.h>
75 #include <linux/stddef.h>
76 #include <linux/proc_fs.h>
77 #include <linux/seq_file.h>
78 #include <linux/inetdevice.h>
79 #include <linux/btf_ids.h>
81 #include <crypto/hash.h>
82 #include <linux/scatterlist.h>
84 #include <trace/events/tcp.h>
86 #ifdef CONFIG_TCP_MD5SIG
87 static int tcp_v4_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key *key,
88 __be32 daddr, __be32 saddr, const struct tcphdr *th);
91 struct inet_hashinfo tcp_hashinfo;
92 EXPORT_SYMBOL(tcp_hashinfo);
94 static u32 tcp_v4_init_seq(const struct sk_buff *skb)
96 return secure_tcp_seq(ip_hdr(skb)->daddr,
99 tcp_hdr(skb)->source);
102 static u32 tcp_v4_init_ts_off(const struct net *net, const struct sk_buff *skb)
104 return secure_tcp_ts_off(net, ip_hdr(skb)->daddr, ip_hdr(skb)->saddr);
107 int tcp_twsk_unique(struct sock *sk, struct sock *sktw, void *twp)
109 const struct inet_timewait_sock *tw = inet_twsk(sktw);
110 const struct tcp_timewait_sock *tcptw = tcp_twsk(sktw);
111 struct tcp_sock *tp = tcp_sk(sk);
112 int reuse = sock_net(sk)->ipv4.sysctl_tcp_tw_reuse;
115 /* Still does not detect *everything* that goes through
116 * lo, since we require a loopback src or dst address
117 * or direct binding to 'lo' interface.
119 bool loopback = false;
120 if (tw->tw_bound_dev_if == LOOPBACK_IFINDEX)
122 #if IS_ENABLED(CONFIG_IPV6)
123 if (tw->tw_family == AF_INET6) {
124 if (ipv6_addr_loopback(&tw->tw_v6_daddr) ||
125 ipv6_addr_v4mapped_loopback(&tw->tw_v6_daddr) ||
126 ipv6_addr_loopback(&tw->tw_v6_rcv_saddr) ||
127 ipv6_addr_v4mapped_loopback(&tw->tw_v6_rcv_saddr))
132 if (ipv4_is_loopback(tw->tw_daddr) ||
133 ipv4_is_loopback(tw->tw_rcv_saddr))
140 /* With PAWS, it is safe from the viewpoint
141 of data integrity. Even without PAWS it is safe provided sequence
142 spaces do not overlap i.e. at data rates <= 80Mbit/sec.
144 Actually, the idea is close to VJ's one, only timestamp cache is
145 held not per host, but per port pair and TW bucket is used as state
148 If TW bucket has been already destroyed we fall back to VJ's scheme
149 and use initial timestamp retrieved from peer table.
151 if (tcptw->tw_ts_recent_stamp &&
152 (!twp || (reuse && time_after32(ktime_get_seconds(),
153 tcptw->tw_ts_recent_stamp)))) {
154 /* In case of repair and re-using TIME-WAIT sockets we still
155 * want to be sure that it is safe as above but honor the
156 * sequence numbers and time stamps set as part of the repair
159 * Without this check re-using a TIME-WAIT socket with TCP
160 * repair would accumulate a -1 on the repair assigned
161 * sequence number. The first time it is reused the sequence
162 * is -1, the second time -2, etc. This fixes that issue
163 * without appearing to create any others.
165 if (likely(!tp->repair)) {
166 u32 seq = tcptw->tw_snd_nxt + 65535 + 2;
170 WRITE_ONCE(tp->write_seq, seq);
171 tp->rx_opt.ts_recent = tcptw->tw_ts_recent;
172 tp->rx_opt.ts_recent_stamp = tcptw->tw_ts_recent_stamp;
180 EXPORT_SYMBOL_GPL(tcp_twsk_unique);
182 static int tcp_v4_pre_connect(struct sock *sk, struct sockaddr *uaddr,
185 /* This check is replicated from tcp_v4_connect() and intended to
186 * prevent BPF program called below from accessing bytes that are out
187 * of the bound specified by user in addr_len.
189 if (addr_len < sizeof(struct sockaddr_in))
192 sock_owned_by_me(sk);
194 return BPF_CGROUP_RUN_PROG_INET4_CONNECT(sk, uaddr);
197 /* This will initiate an outgoing connection. */
198 int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)
200 struct sockaddr_in *usin = (struct sockaddr_in *)uaddr;
201 struct inet_sock *inet = inet_sk(sk);
202 struct tcp_sock *tp = tcp_sk(sk);
203 __be16 orig_sport, orig_dport;
204 __be32 daddr, nexthop;
208 struct ip_options_rcu *inet_opt;
209 struct inet_timewait_death_row *tcp_death_row = &sock_net(sk)->ipv4.tcp_death_row;
211 if (addr_len < sizeof(struct sockaddr_in))
214 if (usin->sin_family != AF_INET)
215 return -EAFNOSUPPORT;
217 nexthop = daddr = usin->sin_addr.s_addr;
218 inet_opt = rcu_dereference_protected(inet->inet_opt,
219 lockdep_sock_is_held(sk));
220 if (inet_opt && inet_opt->opt.srr) {
223 nexthop = inet_opt->opt.faddr;
226 orig_sport = inet->inet_sport;
227 orig_dport = usin->sin_port;
228 fl4 = &inet->cork.fl.u.ip4;
229 rt = ip_route_connect(fl4, nexthop, inet->inet_saddr,
230 RT_CONN_FLAGS(sk), sk->sk_bound_dev_if,
232 orig_sport, orig_dport, sk);
235 if (err == -ENETUNREACH)
236 IP_INC_STATS(sock_net(sk), IPSTATS_MIB_OUTNOROUTES);
240 if (rt->rt_flags & (RTCF_MULTICAST | RTCF_BROADCAST)) {
245 if (!inet_opt || !inet_opt->opt.srr)
248 if (!inet->inet_saddr)
249 inet->inet_saddr = fl4->saddr;
250 sk_rcv_saddr_set(sk, inet->inet_saddr);
252 if (tp->rx_opt.ts_recent_stamp && inet->inet_daddr != daddr) {
253 /* Reset inherited state */
254 tp->rx_opt.ts_recent = 0;
255 tp->rx_opt.ts_recent_stamp = 0;
256 if (likely(!tp->repair))
257 WRITE_ONCE(tp->write_seq, 0);
260 inet->inet_dport = usin->sin_port;
261 sk_daddr_set(sk, daddr);
263 inet_csk(sk)->icsk_ext_hdr_len = 0;
265 inet_csk(sk)->icsk_ext_hdr_len = inet_opt->opt.optlen;
267 tp->rx_opt.mss_clamp = TCP_MSS_DEFAULT;
269 /* Socket identity is still unknown (sport may be zero).
270 * However we set state to SYN-SENT and not releasing socket
271 * lock select source port, enter ourselves into the hash tables and
272 * complete initialization after this.
274 tcp_set_state(sk, TCP_SYN_SENT);
275 err = inet_hash_connect(tcp_death_row, sk);
281 rt = ip_route_newports(fl4, rt, orig_sport, orig_dport,
282 inet->inet_sport, inet->inet_dport, sk);
288 /* OK, now commit destination to socket. */
289 sk->sk_gso_type = SKB_GSO_TCPV4;
290 sk_setup_caps(sk, &rt->dst);
293 if (likely(!tp->repair)) {
295 WRITE_ONCE(tp->write_seq,
296 secure_tcp_seq(inet->inet_saddr,
300 tp->tsoffset = secure_tcp_ts_off(sock_net(sk),
305 inet->inet_id = prandom_u32();
307 if (tcp_fastopen_defer_connect(sk, &err))
312 err = tcp_connect(sk);
321 * This unhashes the socket and releases the local port,
324 tcp_set_state(sk, TCP_CLOSE);
326 sk->sk_route_caps = 0;
327 inet->inet_dport = 0;
330 EXPORT_SYMBOL(tcp_v4_connect);
333 * This routine reacts to ICMP_FRAG_NEEDED mtu indications as defined in RFC1191.
334 * It can be called through tcp_release_cb() if socket was owned by user
335 * at the time tcp_v4_err() was called to handle ICMP message.
337 void tcp_v4_mtu_reduced(struct sock *sk)
339 struct inet_sock *inet = inet_sk(sk);
340 struct dst_entry *dst;
343 if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE))
345 mtu = tcp_sk(sk)->mtu_info;
346 dst = inet_csk_update_pmtu(sk, mtu);
350 /* Something is about to be wrong... Remember soft error
351 * for the case, if this connection will not able to recover.
353 if (mtu < dst_mtu(dst) && ip_dont_fragment(sk, dst))
354 sk->sk_err_soft = EMSGSIZE;
358 if (inet->pmtudisc != IP_PMTUDISC_DONT &&
359 ip_sk_accept_pmtu(sk) &&
360 inet_csk(sk)->icsk_pmtu_cookie > mtu) {
361 tcp_sync_mss(sk, mtu);
363 /* Resend the TCP packet because it's
364 * clear that the old packet has been
365 * dropped. This is the new "fast" path mtu
368 tcp_simple_retransmit(sk);
369 } /* else let the usual retransmit timer handle it */
371 EXPORT_SYMBOL(tcp_v4_mtu_reduced);
373 static void do_redirect(struct sk_buff *skb, struct sock *sk)
375 struct dst_entry *dst = __sk_dst_check(sk, 0);
378 dst->ops->redirect(dst, sk, skb);
382 /* handle ICMP messages on TCP_NEW_SYN_RECV request sockets */
383 void tcp_req_err(struct sock *sk, u32 seq, bool abort)
385 struct request_sock *req = inet_reqsk(sk);
386 struct net *net = sock_net(sk);
388 /* ICMPs are not backlogged, hence we cannot get
389 * an established socket here.
391 if (seq != tcp_rsk(req)->snt_isn) {
392 __NET_INC_STATS(net, LINUX_MIB_OUTOFWINDOWICMPS);
395 * Still in SYN_RECV, just remove it silently.
396 * There is no good way to pass the error to the newly
397 * created socket, and POSIX does not want network
398 * errors returned from accept().
400 inet_csk_reqsk_queue_drop(req->rsk_listener, req);
401 tcp_listendrop(req->rsk_listener);
405 EXPORT_SYMBOL(tcp_req_err);
407 /* TCP-LD (RFC 6069) logic */
408 void tcp_ld_RTO_revert(struct sock *sk, u32 seq)
410 struct inet_connection_sock *icsk = inet_csk(sk);
411 struct tcp_sock *tp = tcp_sk(sk);
416 if (sock_owned_by_user(sk))
419 if (seq != tp->snd_una || !icsk->icsk_retransmits ||
423 skb = tcp_rtx_queue_head(sk);
424 if (WARN_ON_ONCE(!skb))
427 icsk->icsk_backoff--;
428 icsk->icsk_rto = tp->srtt_us ? __tcp_set_rto(tp) : TCP_TIMEOUT_INIT;
429 icsk->icsk_rto = inet_csk_rto_backoff(icsk, TCP_RTO_MAX);
431 tcp_mstamp_refresh(tp);
432 delta_us = (u32)(tp->tcp_mstamp - tcp_skb_timestamp_us(skb));
433 remaining = icsk->icsk_rto - usecs_to_jiffies(delta_us);
436 inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
437 remaining, TCP_RTO_MAX);
439 /* RTO revert clocked out retransmission.
440 * Will retransmit now.
442 tcp_retransmit_timer(sk);
445 EXPORT_SYMBOL(tcp_ld_RTO_revert);
448 * This routine is called by the ICMP module when it gets some
449 * sort of error condition. If err < 0 then the socket should
450 * be closed and the error returned to the user. If err > 0
451 * it's just the icmp type << 8 | icmp code. After adjustment
452 * header points to the first 8 bytes of the tcp header. We need
453 * to find the appropriate port.
455 * The locking strategy used here is very "optimistic". When
456 * someone else accesses the socket the ICMP is just dropped
457 * and for some paths there is no check at all.
458 * A more general error queue to queue errors for later handling
459 * is probably better.
463 int tcp_v4_err(struct sk_buff *skb, u32 info)
465 const struct iphdr *iph = (const struct iphdr *)skb->data;
466 struct tcphdr *th = (struct tcphdr *)(skb->data + (iph->ihl << 2));
468 struct inet_sock *inet;
469 const int type = icmp_hdr(skb)->type;
470 const int code = icmp_hdr(skb)->code;
472 struct request_sock *fastopen;
475 struct net *net = dev_net(skb->dev);
477 sk = __inet_lookup_established(net, &tcp_hashinfo, iph->daddr,
478 th->dest, iph->saddr, ntohs(th->source),
481 __ICMP_INC_STATS(net, ICMP_MIB_INERRORS);
484 if (sk->sk_state == TCP_TIME_WAIT) {
485 inet_twsk_put(inet_twsk(sk));
488 seq = ntohl(th->seq);
489 if (sk->sk_state == TCP_NEW_SYN_RECV) {
490 tcp_req_err(sk, seq, type == ICMP_PARAMETERPROB ||
491 type == ICMP_TIME_EXCEEDED ||
492 (type == ICMP_DEST_UNREACH &&
493 (code == ICMP_NET_UNREACH ||
494 code == ICMP_HOST_UNREACH)));
499 /* If too many ICMPs get dropped on busy
500 * servers this needs to be solved differently.
501 * We do take care of PMTU discovery (RFC1191) special case :
502 * we can receive locally generated ICMP messages while socket is held.
504 if (sock_owned_by_user(sk)) {
505 if (!(type == ICMP_DEST_UNREACH && code == ICMP_FRAG_NEEDED))
506 __NET_INC_STATS(net, LINUX_MIB_LOCKDROPPEDICMPS);
508 if (sk->sk_state == TCP_CLOSE)
511 if (unlikely(iph->ttl < inet_sk(sk)->min_ttl)) {
512 __NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP);
517 /* XXX (TFO) - tp->snd_una should be ISN (tcp_create_openreq_child() */
518 fastopen = rcu_dereference(tp->fastopen_rsk);
519 snd_una = fastopen ? tcp_rsk(fastopen)->snt_isn : tp->snd_una;
520 if (sk->sk_state != TCP_LISTEN &&
521 !between(seq, snd_una, tp->snd_nxt)) {
522 __NET_INC_STATS(net, LINUX_MIB_OUTOFWINDOWICMPS);
528 if (!sock_owned_by_user(sk))
529 do_redirect(skb, sk);
531 case ICMP_SOURCE_QUENCH:
532 /* Just silently ignore these. */
534 case ICMP_PARAMETERPROB:
537 case ICMP_DEST_UNREACH:
538 if (code > NR_ICMP_UNREACH)
541 if (code == ICMP_FRAG_NEEDED) { /* PMTU discovery (RFC1191) */
542 /* We are not interested in TCP_LISTEN and open_requests
543 * (SYN-ACKs send out by Linux are always <576bytes so
544 * they should go through unfragmented).
546 if (sk->sk_state == TCP_LISTEN)
550 if (!sock_owned_by_user(sk)) {
551 tcp_v4_mtu_reduced(sk);
553 if (!test_and_set_bit(TCP_MTU_REDUCED_DEFERRED, &sk->sk_tsq_flags))
559 err = icmp_err_convert[code].errno;
560 /* check if this ICMP message allows revert of backoff.
564 (code == ICMP_NET_UNREACH || code == ICMP_HOST_UNREACH))
565 tcp_ld_RTO_revert(sk, seq);
567 case ICMP_TIME_EXCEEDED:
574 switch (sk->sk_state) {
577 /* Only in fast or simultaneous open. If a fast open socket is
578 * already accepted it is treated as a connected one below.
580 if (fastopen && !fastopen->sk)
583 ip_icmp_error(sk, skb, err, th->dest, info, (u8 *)th);
585 if (!sock_owned_by_user(sk)) {
588 sk->sk_error_report(sk);
592 sk->sk_err_soft = err;
597 /* If we've already connected we will keep trying
598 * until we time out, or the user gives up.
600 * rfc1122 4.2.3.9 allows to consider as hard errors
601 * only PROTO_UNREACH and PORT_UNREACH (well, FRAG_FAILED too,
602 * but it is obsoleted by pmtu discovery).
604 * Note, that in modern internet, where routing is unreliable
605 * and in each dark corner broken firewalls sit, sending random
606 * errors ordered by their masters even this two messages finally lose
607 * their original sense (even Linux sends invalid PORT_UNREACHs)
609 * Now we are in compliance with RFCs.
614 if (!sock_owned_by_user(sk) && inet->recverr) {
616 sk->sk_error_report(sk);
617 } else { /* Only an error on timeout */
618 sk->sk_err_soft = err;
627 void __tcp_v4_send_check(struct sk_buff *skb, __be32 saddr, __be32 daddr)
629 struct tcphdr *th = tcp_hdr(skb);
631 th->check = ~tcp_v4_check(skb->len, saddr, daddr, 0);
632 skb->csum_start = skb_transport_header(skb) - skb->head;
633 skb->csum_offset = offsetof(struct tcphdr, check);
636 /* This routine computes an IPv4 TCP checksum. */
637 void tcp_v4_send_check(struct sock *sk, struct sk_buff *skb)
639 const struct inet_sock *inet = inet_sk(sk);
641 __tcp_v4_send_check(skb, inet->inet_saddr, inet->inet_daddr);
643 EXPORT_SYMBOL(tcp_v4_send_check);
646 * This routine will send an RST to the other tcp.
648 * Someone asks: why I NEVER use socket parameters (TOS, TTL etc.)
650 * Answer: if a packet caused RST, it is not for a socket
651 * existing in our system, if it is matched to a socket,
652 * it is just duplicate segment or bug in other side's TCP.
653 * So that we build reply only basing on parameters
654 * arrived with segment.
655 * Exception: precedence violation. We do not implement it in any case.
658 static void tcp_v4_send_reset(const struct sock *sk, struct sk_buff *skb)
660 const struct tcphdr *th = tcp_hdr(skb);
663 #ifdef CONFIG_TCP_MD5SIG
664 __be32 opt[(TCPOLEN_MD5SIG_ALIGNED >> 2)];
667 struct ip_reply_arg arg;
668 #ifdef CONFIG_TCP_MD5SIG
669 struct tcp_md5sig_key *key = NULL;
670 const __u8 *hash_location = NULL;
671 unsigned char newhash[16];
673 struct sock *sk1 = NULL;
675 u64 transmit_time = 0;
679 /* Never send a reset in response to a reset. */
683 /* If sk not NULL, it means we did a successful lookup and incoming
684 * route had to be correct. prequeue might have dropped our dst.
686 if (!sk && skb_rtable(skb)->rt_type != RTN_LOCAL)
689 /* Swap the send and the receive. */
690 memset(&rep, 0, sizeof(rep));
691 rep.th.dest = th->source;
692 rep.th.source = th->dest;
693 rep.th.doff = sizeof(struct tcphdr) / 4;
697 rep.th.seq = th->ack_seq;
700 rep.th.ack_seq = htonl(ntohl(th->seq) + th->syn + th->fin +
701 skb->len - (th->doff << 2));
704 memset(&arg, 0, sizeof(arg));
705 arg.iov[0].iov_base = (unsigned char *)&rep;
706 arg.iov[0].iov_len = sizeof(rep.th);
708 net = sk ? sock_net(sk) : dev_net(skb_dst(skb)->dev);
709 #ifdef CONFIG_TCP_MD5SIG
711 hash_location = tcp_parse_md5sig_option(th);
712 if (sk && sk_fullsock(sk)) {
713 const union tcp_md5_addr *addr;
716 /* sdif set, means packet ingressed via a device
717 * in an L3 domain and inet_iif is set to it.
719 l3index = tcp_v4_sdif(skb) ? inet_iif(skb) : 0;
720 addr = (union tcp_md5_addr *)&ip_hdr(skb)->saddr;
721 key = tcp_md5_do_lookup(sk, l3index, addr, AF_INET);
722 } else if (hash_location) {
723 const union tcp_md5_addr *addr;
724 int sdif = tcp_v4_sdif(skb);
725 int dif = inet_iif(skb);
729 * active side is lost. Try to find listening socket through
730 * source port, and then find md5 key through listening socket.
731 * we are not loose security here:
732 * Incoming packet is checked with md5 hash with finding key,
733 * no RST generated if md5 hash doesn't match.
735 sk1 = __inet_lookup_listener(net, &tcp_hashinfo, NULL, 0,
737 th->source, ip_hdr(skb)->daddr,
738 ntohs(th->source), dif, sdif);
739 /* don't send rst if it can't find key */
743 /* sdif set, means packet ingressed via a device
744 * in an L3 domain and dif is set to it.
746 l3index = sdif ? dif : 0;
747 addr = (union tcp_md5_addr *)&ip_hdr(skb)->saddr;
748 key = tcp_md5_do_lookup(sk1, l3index, addr, AF_INET);
753 genhash = tcp_v4_md5_hash_skb(newhash, key, NULL, skb);
754 if (genhash || memcmp(hash_location, newhash, 16) != 0)
760 rep.opt[0] = htonl((TCPOPT_NOP << 24) |
762 (TCPOPT_MD5SIG << 8) |
764 /* Update length and the length the header thinks exists */
765 arg.iov[0].iov_len += TCPOLEN_MD5SIG_ALIGNED;
766 rep.th.doff = arg.iov[0].iov_len / 4;
768 tcp_v4_md5_hash_hdr((__u8 *) &rep.opt[1],
769 key, ip_hdr(skb)->saddr,
770 ip_hdr(skb)->daddr, &rep.th);
773 arg.csum = csum_tcpudp_nofold(ip_hdr(skb)->daddr,
774 ip_hdr(skb)->saddr, /* XXX */
775 arg.iov[0].iov_len, IPPROTO_TCP, 0);
776 arg.csumoffset = offsetof(struct tcphdr, check) / 2;
777 arg.flags = (sk && inet_sk_transparent(sk)) ? IP_REPLY_ARG_NOSRCCHECK : 0;
779 /* When socket is gone, all binding information is lost.
780 * routing might fail in this case. No choice here, if we choose to force
781 * input interface, we will misroute in case of asymmetric route.
784 arg.bound_dev_if = sk->sk_bound_dev_if;
786 trace_tcp_send_reset(sk, skb);
789 BUILD_BUG_ON(offsetof(struct sock, sk_bound_dev_if) !=
790 offsetof(struct inet_timewait_sock, tw_bound_dev_if));
792 arg.tos = ip_hdr(skb)->tos;
793 arg.uid = sock_net_uid(net, sk && sk_fullsock(sk) ? sk : NULL);
795 ctl_sk = this_cpu_read(*net->ipv4.tcp_sk);
797 ctl_sk->sk_mark = (sk->sk_state == TCP_TIME_WAIT) ?
798 inet_twsk(sk)->tw_mark : sk->sk_mark;
799 ctl_sk->sk_priority = (sk->sk_state == TCP_TIME_WAIT) ?
800 inet_twsk(sk)->tw_priority : sk->sk_priority;
801 transmit_time = tcp_transmit_time(sk);
803 ip_send_unicast_reply(ctl_sk,
804 skb, &TCP_SKB_CB(skb)->header.h4.opt,
805 ip_hdr(skb)->saddr, ip_hdr(skb)->daddr,
806 &arg, arg.iov[0].iov_len,
810 __TCP_INC_STATS(net, TCP_MIB_OUTSEGS);
811 __TCP_INC_STATS(net, TCP_MIB_OUTRSTS);
814 #ifdef CONFIG_TCP_MD5SIG
820 /* The code following below sending ACKs in SYN-RECV and TIME-WAIT states
821 outside socket context is ugly, certainly. What can I do?
824 static void tcp_v4_send_ack(const struct sock *sk,
825 struct sk_buff *skb, u32 seq, u32 ack,
826 u32 win, u32 tsval, u32 tsecr, int oif,
827 struct tcp_md5sig_key *key,
828 int reply_flags, u8 tos)
830 const struct tcphdr *th = tcp_hdr(skb);
833 __be32 opt[(TCPOLEN_TSTAMP_ALIGNED >> 2)
834 #ifdef CONFIG_TCP_MD5SIG
835 + (TCPOLEN_MD5SIG_ALIGNED >> 2)
839 struct net *net = sock_net(sk);
840 struct ip_reply_arg arg;
844 memset(&rep.th, 0, sizeof(struct tcphdr));
845 memset(&arg, 0, sizeof(arg));
847 arg.iov[0].iov_base = (unsigned char *)&rep;
848 arg.iov[0].iov_len = sizeof(rep.th);
850 rep.opt[0] = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) |
851 (TCPOPT_TIMESTAMP << 8) |
853 rep.opt[1] = htonl(tsval);
854 rep.opt[2] = htonl(tsecr);
855 arg.iov[0].iov_len += TCPOLEN_TSTAMP_ALIGNED;
858 /* Swap the send and the receive. */
859 rep.th.dest = th->source;
860 rep.th.source = th->dest;
861 rep.th.doff = arg.iov[0].iov_len / 4;
862 rep.th.seq = htonl(seq);
863 rep.th.ack_seq = htonl(ack);
865 rep.th.window = htons(win);
867 #ifdef CONFIG_TCP_MD5SIG
869 int offset = (tsecr) ? 3 : 0;
871 rep.opt[offset++] = htonl((TCPOPT_NOP << 24) |
873 (TCPOPT_MD5SIG << 8) |
875 arg.iov[0].iov_len += TCPOLEN_MD5SIG_ALIGNED;
876 rep.th.doff = arg.iov[0].iov_len/4;
878 tcp_v4_md5_hash_hdr((__u8 *) &rep.opt[offset],
879 key, ip_hdr(skb)->saddr,
880 ip_hdr(skb)->daddr, &rep.th);
883 arg.flags = reply_flags;
884 arg.csum = csum_tcpudp_nofold(ip_hdr(skb)->daddr,
885 ip_hdr(skb)->saddr, /* XXX */
886 arg.iov[0].iov_len, IPPROTO_TCP, 0);
887 arg.csumoffset = offsetof(struct tcphdr, check) / 2;
889 arg.bound_dev_if = oif;
891 arg.uid = sock_net_uid(net, sk_fullsock(sk) ? sk : NULL);
893 ctl_sk = this_cpu_read(*net->ipv4.tcp_sk);
894 ctl_sk->sk_mark = (sk->sk_state == TCP_TIME_WAIT) ?
895 inet_twsk(sk)->tw_mark : sk->sk_mark;
896 ctl_sk->sk_priority = (sk->sk_state == TCP_TIME_WAIT) ?
897 inet_twsk(sk)->tw_priority : sk->sk_priority;
898 transmit_time = tcp_transmit_time(sk);
899 ip_send_unicast_reply(ctl_sk,
900 skb, &TCP_SKB_CB(skb)->header.h4.opt,
901 ip_hdr(skb)->saddr, ip_hdr(skb)->daddr,
902 &arg, arg.iov[0].iov_len,
906 __TCP_INC_STATS(net, TCP_MIB_OUTSEGS);
910 static void tcp_v4_timewait_ack(struct sock *sk, struct sk_buff *skb)
912 struct inet_timewait_sock *tw = inet_twsk(sk);
913 struct tcp_timewait_sock *tcptw = tcp_twsk(sk);
915 tcp_v4_send_ack(sk, skb,
916 tcptw->tw_snd_nxt, tcptw->tw_rcv_nxt,
917 tcptw->tw_rcv_wnd >> tw->tw_rcv_wscale,
918 tcp_time_stamp_raw() + tcptw->tw_ts_offset,
921 tcp_twsk_md5_key(tcptw),
922 tw->tw_transparent ? IP_REPLY_ARG_NOSRCCHECK : 0,
929 static void tcp_v4_reqsk_send_ack(const struct sock *sk, struct sk_buff *skb,
930 struct request_sock *req)
932 const union tcp_md5_addr *addr;
935 /* sk->sk_state == TCP_LISTEN -> for regular TCP_SYN_RECV
936 * sk->sk_state == TCP_SYN_RECV -> for Fast Open.
938 u32 seq = (sk->sk_state == TCP_LISTEN) ? tcp_rsk(req)->snt_isn + 1 :
942 * The window field (SEG.WND) of every outgoing segment, with the
943 * exception of <SYN> segments, MUST be right-shifted by
944 * Rcv.Wind.Shift bits:
946 addr = (union tcp_md5_addr *)&ip_hdr(skb)->saddr;
947 l3index = tcp_v4_sdif(skb) ? inet_iif(skb) : 0;
948 tcp_v4_send_ack(sk, skb, seq,
949 tcp_rsk(req)->rcv_nxt,
950 req->rsk_rcv_wnd >> inet_rsk(req)->rcv_wscale,
951 tcp_time_stamp_raw() + tcp_rsk(req)->ts_off,
954 tcp_md5_do_lookup(sk, l3index, addr, AF_INET),
955 inet_rsk(req)->no_srccheck ? IP_REPLY_ARG_NOSRCCHECK : 0,
960 * Send a SYN-ACK after having received a SYN.
961 * This still operates on a request_sock only, not on a big
964 static int tcp_v4_send_synack(const struct sock *sk, struct dst_entry *dst,
966 struct request_sock *req,
967 struct tcp_fastopen_cookie *foc,
968 enum tcp_synack_type synack_type,
969 struct sk_buff *syn_skb)
971 const struct inet_request_sock *ireq = inet_rsk(req);
977 /* First, grab a route. */
978 if (!dst && (dst = inet_csk_route_req(sk, &fl4, req)) == NULL)
981 skb = tcp_make_synack(sk, dst, req, foc, synack_type, syn_skb);
984 __tcp_v4_send_check(skb, ireq->ir_loc_addr, ireq->ir_rmt_addr);
986 tos = sock_net(sk)->ipv4.sysctl_tcp_reflect_tos ?
987 (tcp_rsk(req)->syn_tos & ~INET_ECN_MASK) |
988 (inet_sk(sk)->tos & INET_ECN_MASK) :
991 if (!INET_ECN_is_capable(tos) &&
992 tcp_bpf_ca_needs_ecn((struct sock *)req))
993 tos |= INET_ECN_ECT_0;
996 err = ip_build_and_send_pkt(skb, sk, ireq->ir_loc_addr,
998 rcu_dereference(ireq->ireq_opt),
1001 err = net_xmit_eval(err);
1008 * IPv4 request_sock destructor.
1010 static void tcp_v4_reqsk_destructor(struct request_sock *req)
1012 kfree(rcu_dereference_protected(inet_rsk(req)->ireq_opt, 1));
1015 #ifdef CONFIG_TCP_MD5SIG
1017 * RFC2385 MD5 checksumming requires a mapping of
1018 * IP address->MD5 Key.
1019 * We need to maintain these in the sk structure.
1022 DEFINE_STATIC_KEY_FALSE(tcp_md5_needed);
1023 EXPORT_SYMBOL(tcp_md5_needed);
1025 /* Find the Key structure for an address. */
1026 struct tcp_md5sig_key *__tcp_md5_do_lookup(const struct sock *sk, int l3index,
1027 const union tcp_md5_addr *addr,
1030 const struct tcp_sock *tp = tcp_sk(sk);
1031 struct tcp_md5sig_key *key;
1032 const struct tcp_md5sig_info *md5sig;
1034 struct tcp_md5sig_key *best_match = NULL;
1037 /* caller either holds rcu_read_lock() or socket lock */
1038 md5sig = rcu_dereference_check(tp->md5sig_info,
1039 lockdep_sock_is_held(sk));
1043 hlist_for_each_entry_rcu(key, &md5sig->head, node,
1044 lockdep_sock_is_held(sk)) {
1045 if (key->family != family)
1047 if (key->l3index && key->l3index != l3index)
1049 if (family == AF_INET) {
1050 mask = inet_make_mask(key->prefixlen);
1051 match = (key->addr.a4.s_addr & mask) ==
1052 (addr->a4.s_addr & mask);
1053 #if IS_ENABLED(CONFIG_IPV6)
1054 } else if (family == AF_INET6) {
1055 match = ipv6_prefix_equal(&key->addr.a6, &addr->a6,
1062 if (match && (!best_match ||
1063 key->prefixlen > best_match->prefixlen))
1068 EXPORT_SYMBOL(__tcp_md5_do_lookup);
1070 static struct tcp_md5sig_key *tcp_md5_do_lookup_exact(const struct sock *sk,
1071 const union tcp_md5_addr *addr,
1072 int family, u8 prefixlen,
1075 const struct tcp_sock *tp = tcp_sk(sk);
1076 struct tcp_md5sig_key *key;
1077 unsigned int size = sizeof(struct in_addr);
1078 const struct tcp_md5sig_info *md5sig;
1080 /* caller either holds rcu_read_lock() or socket lock */
1081 md5sig = rcu_dereference_check(tp->md5sig_info,
1082 lockdep_sock_is_held(sk));
1085 #if IS_ENABLED(CONFIG_IPV6)
1086 if (family == AF_INET6)
1087 size = sizeof(struct in6_addr);
1089 hlist_for_each_entry_rcu(key, &md5sig->head, node,
1090 lockdep_sock_is_held(sk)) {
1091 if (key->family != family)
1093 if (key->l3index && key->l3index != l3index)
1095 if (!memcmp(&key->addr, addr, size) &&
1096 key->prefixlen == prefixlen)
1102 struct tcp_md5sig_key *tcp_v4_md5_lookup(const struct sock *sk,
1103 const struct sock *addr_sk)
1105 const union tcp_md5_addr *addr;
1108 l3index = l3mdev_master_ifindex_by_index(sock_net(sk),
1109 addr_sk->sk_bound_dev_if);
1110 addr = (const union tcp_md5_addr *)&addr_sk->sk_daddr;
1111 return tcp_md5_do_lookup(sk, l3index, addr, AF_INET);
1113 EXPORT_SYMBOL(tcp_v4_md5_lookup);
1115 /* This can be called on a newly created socket, from other files */
1116 int tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr,
1117 int family, u8 prefixlen, int l3index,
1118 const u8 *newkey, u8 newkeylen, gfp_t gfp)
1120 /* Add Key to the list */
1121 struct tcp_md5sig_key *key;
1122 struct tcp_sock *tp = tcp_sk(sk);
1123 struct tcp_md5sig_info *md5sig;
1125 key = tcp_md5_do_lookup_exact(sk, addr, family, prefixlen, l3index);
1127 /* Pre-existing entry - just update that one.
1128 * Note that the key might be used concurrently.
1129 * data_race() is telling kcsan that we do not care of
1130 * key mismatches, since changing MD5 key on live flows
1131 * can lead to packet drops.
1133 data_race(memcpy(key->key, newkey, newkeylen));
1135 /* Pairs with READ_ONCE() in tcp_md5_hash_key().
1136 * Also note that a reader could catch new key->keylen value
1137 * but old key->key[], this is the reason we use __GFP_ZERO
1138 * at sock_kmalloc() time below these lines.
1140 WRITE_ONCE(key->keylen, newkeylen);
1145 md5sig = rcu_dereference_protected(tp->md5sig_info,
1146 lockdep_sock_is_held(sk));
1148 md5sig = kmalloc(sizeof(*md5sig), gfp);
1152 sk_nocaps_add(sk, NETIF_F_GSO_MASK);
1153 INIT_HLIST_HEAD(&md5sig->head);
1154 rcu_assign_pointer(tp->md5sig_info, md5sig);
1157 key = sock_kmalloc(sk, sizeof(*key), gfp | __GFP_ZERO);
1160 if (!tcp_alloc_md5sig_pool()) {
1161 sock_kfree_s(sk, key, sizeof(*key));
1165 memcpy(key->key, newkey, newkeylen);
1166 key->keylen = newkeylen;
1167 key->family = family;
1168 key->prefixlen = prefixlen;
1169 key->l3index = l3index;
1170 memcpy(&key->addr, addr,
1171 (family == AF_INET6) ? sizeof(struct in6_addr) :
1172 sizeof(struct in_addr));
1173 hlist_add_head_rcu(&key->node, &md5sig->head);
1176 EXPORT_SYMBOL(tcp_md5_do_add);
1178 int tcp_md5_do_del(struct sock *sk, const union tcp_md5_addr *addr, int family,
1179 u8 prefixlen, int l3index)
1181 struct tcp_md5sig_key *key;
1183 key = tcp_md5_do_lookup_exact(sk, addr, family, prefixlen, l3index);
1186 hlist_del_rcu(&key->node);
1187 atomic_sub(sizeof(*key), &sk->sk_omem_alloc);
1188 kfree_rcu(key, rcu);
1191 EXPORT_SYMBOL(tcp_md5_do_del);
1193 static void tcp_clear_md5_list(struct sock *sk)
1195 struct tcp_sock *tp = tcp_sk(sk);
1196 struct tcp_md5sig_key *key;
1197 struct hlist_node *n;
1198 struct tcp_md5sig_info *md5sig;
1200 md5sig = rcu_dereference_protected(tp->md5sig_info, 1);
1202 hlist_for_each_entry_safe(key, n, &md5sig->head, node) {
1203 hlist_del_rcu(&key->node);
1204 atomic_sub(sizeof(*key), &sk->sk_omem_alloc);
1205 kfree_rcu(key, rcu);
1209 static int tcp_v4_parse_md5_keys(struct sock *sk, int optname,
1210 sockptr_t optval, int optlen)
1212 struct tcp_md5sig cmd;
1213 struct sockaddr_in *sin = (struct sockaddr_in *)&cmd.tcpm_addr;
1214 const union tcp_md5_addr *addr;
1218 if (optlen < sizeof(cmd))
1221 if (copy_from_sockptr(&cmd, optval, sizeof(cmd)))
1224 if (sin->sin_family != AF_INET)
1227 if (optname == TCP_MD5SIG_EXT &&
1228 cmd.tcpm_flags & TCP_MD5SIG_FLAG_PREFIX) {
1229 prefixlen = cmd.tcpm_prefixlen;
1234 if (optname == TCP_MD5SIG_EXT &&
1235 cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX) {
1236 struct net_device *dev;
1239 dev = dev_get_by_index_rcu(sock_net(sk), cmd.tcpm_ifindex);
1240 if (dev && netif_is_l3_master(dev))
1241 l3index = dev->ifindex;
1245 /* ok to reference set/not set outside of rcu;
1246 * right now device MUST be an L3 master
1248 if (!dev || !l3index)
1252 addr = (union tcp_md5_addr *)&sin->sin_addr.s_addr;
1254 if (!cmd.tcpm_keylen)
1255 return tcp_md5_do_del(sk, addr, AF_INET, prefixlen, l3index);
1257 if (cmd.tcpm_keylen > TCP_MD5SIG_MAXKEYLEN)
1260 return tcp_md5_do_add(sk, addr, AF_INET, prefixlen, l3index,
1261 cmd.tcpm_key, cmd.tcpm_keylen, GFP_KERNEL);
1264 static int tcp_v4_md5_hash_headers(struct tcp_md5sig_pool *hp,
1265 __be32 daddr, __be32 saddr,
1266 const struct tcphdr *th, int nbytes)
1268 struct tcp4_pseudohdr *bp;
1269 struct scatterlist sg;
1276 bp->protocol = IPPROTO_TCP;
1277 bp->len = cpu_to_be16(nbytes);
1279 _th = (struct tcphdr *)(bp + 1);
1280 memcpy(_th, th, sizeof(*th));
1283 sg_init_one(&sg, bp, sizeof(*bp) + sizeof(*th));
1284 ahash_request_set_crypt(hp->md5_req, &sg, NULL,
1285 sizeof(*bp) + sizeof(*th));
1286 return crypto_ahash_update(hp->md5_req);
1289 static int tcp_v4_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key *key,
1290 __be32 daddr, __be32 saddr, const struct tcphdr *th)
1292 struct tcp_md5sig_pool *hp;
1293 struct ahash_request *req;
1295 hp = tcp_get_md5sig_pool();
1297 goto clear_hash_noput;
1300 if (crypto_ahash_init(req))
1302 if (tcp_v4_md5_hash_headers(hp, daddr, saddr, th, th->doff << 2))
1304 if (tcp_md5_hash_key(hp, key))
1306 ahash_request_set_crypt(req, NULL, md5_hash, 0);
1307 if (crypto_ahash_final(req))
1310 tcp_put_md5sig_pool();
1314 tcp_put_md5sig_pool();
1316 memset(md5_hash, 0, 16);
1320 int tcp_v4_md5_hash_skb(char *md5_hash, const struct tcp_md5sig_key *key,
1321 const struct sock *sk,
1322 const struct sk_buff *skb)
1324 struct tcp_md5sig_pool *hp;
1325 struct ahash_request *req;
1326 const struct tcphdr *th = tcp_hdr(skb);
1327 __be32 saddr, daddr;
1329 if (sk) { /* valid for establish/request sockets */
1330 saddr = sk->sk_rcv_saddr;
1331 daddr = sk->sk_daddr;
1333 const struct iphdr *iph = ip_hdr(skb);
1338 hp = tcp_get_md5sig_pool();
1340 goto clear_hash_noput;
1343 if (crypto_ahash_init(req))
1346 if (tcp_v4_md5_hash_headers(hp, daddr, saddr, th, skb->len))
1348 if (tcp_md5_hash_skb_data(hp, skb, th->doff << 2))
1350 if (tcp_md5_hash_key(hp, key))
1352 ahash_request_set_crypt(req, NULL, md5_hash, 0);
1353 if (crypto_ahash_final(req))
1356 tcp_put_md5sig_pool();
1360 tcp_put_md5sig_pool();
1362 memset(md5_hash, 0, 16);
1365 EXPORT_SYMBOL(tcp_v4_md5_hash_skb);
1369 /* Called with rcu_read_lock() */
1370 static bool tcp_v4_inbound_md5_hash(const struct sock *sk,
1371 const struct sk_buff *skb,
1374 #ifdef CONFIG_TCP_MD5SIG
1376 * This gets called for each TCP segment that arrives
1377 * so we want to be efficient.
1378 * We have 3 drop cases:
1379 * o No MD5 hash and one expected.
1380 * o MD5 hash and we're not expecting one.
1381 * o MD5 hash and its wrong.
1383 const __u8 *hash_location = NULL;
1384 struct tcp_md5sig_key *hash_expected;
1385 const struct iphdr *iph = ip_hdr(skb);
1386 const struct tcphdr *th = tcp_hdr(skb);
1387 const union tcp_md5_addr *addr;
1388 unsigned char newhash[16];
1389 int genhash, l3index;
1391 /* sdif set, means packet ingressed via a device
1392 * in an L3 domain and dif is set to the l3mdev
1394 l3index = sdif ? dif : 0;
1396 addr = (union tcp_md5_addr *)&iph->saddr;
1397 hash_expected = tcp_md5_do_lookup(sk, l3index, addr, AF_INET);
1398 hash_location = tcp_parse_md5sig_option(th);
1400 /* We've parsed the options - do we have a hash? */
1401 if (!hash_expected && !hash_location)
1404 if (hash_expected && !hash_location) {
1405 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMD5NOTFOUND);
1409 if (!hash_expected && hash_location) {
1410 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMD5UNEXPECTED);
1414 /* Okay, so this is hash_expected and hash_location -
1415 * so we need to calculate the checksum.
1417 genhash = tcp_v4_md5_hash_skb(newhash,
1421 if (genhash || memcmp(hash_location, newhash, 16) != 0) {
1422 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMD5FAILURE);
1423 net_info_ratelimited("MD5 Hash failed for (%pI4, %d)->(%pI4, %d)%s L3 index %d\n",
1424 &iph->saddr, ntohs(th->source),
1425 &iph->daddr, ntohs(th->dest),
1426 genhash ? " tcp_v4_calc_md5_hash failed"
1435 static void tcp_v4_init_req(struct request_sock *req,
1436 const struct sock *sk_listener,
1437 struct sk_buff *skb)
1439 struct inet_request_sock *ireq = inet_rsk(req);
1440 struct net *net = sock_net(sk_listener);
1442 sk_rcv_saddr_set(req_to_sk(req), ip_hdr(skb)->daddr);
1443 sk_daddr_set(req_to_sk(req), ip_hdr(skb)->saddr);
1444 RCU_INIT_POINTER(ireq->ireq_opt, tcp_v4_save_options(net, skb));
1447 static struct dst_entry *tcp_v4_route_req(const struct sock *sk,
1448 struct sk_buff *skb,
1450 struct request_sock *req)
1452 tcp_v4_init_req(req, sk, skb);
1454 if (security_inet_conn_request(sk, skb, req))
1457 return inet_csk_route_req(sk, &fl->u.ip4, req);
1460 struct request_sock_ops tcp_request_sock_ops __read_mostly = {
1462 .obj_size = sizeof(struct tcp_request_sock),
1463 .rtx_syn_ack = tcp_rtx_synack,
1464 .send_ack = tcp_v4_reqsk_send_ack,
1465 .destructor = tcp_v4_reqsk_destructor,
1466 .send_reset = tcp_v4_send_reset,
1467 .syn_ack_timeout = tcp_syn_ack_timeout,
1470 const struct tcp_request_sock_ops tcp_request_sock_ipv4_ops = {
1471 .mss_clamp = TCP_MSS_DEFAULT,
1472 #ifdef CONFIG_TCP_MD5SIG
1473 .req_md5_lookup = tcp_v4_md5_lookup,
1474 .calc_md5_hash = tcp_v4_md5_hash_skb,
1476 #ifdef CONFIG_SYN_COOKIES
1477 .cookie_init_seq = cookie_v4_init_sequence,
1479 .route_req = tcp_v4_route_req,
1480 .init_seq = tcp_v4_init_seq,
1481 .init_ts_off = tcp_v4_init_ts_off,
1482 .send_synack = tcp_v4_send_synack,
1485 int tcp_v4_conn_request(struct sock *sk, struct sk_buff *skb)
1487 /* Never answer to SYNs send to broadcast or multicast */
1488 if (skb_rtable(skb)->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST))
1491 return tcp_conn_request(&tcp_request_sock_ops,
1492 &tcp_request_sock_ipv4_ops, sk, skb);
1498 EXPORT_SYMBOL(tcp_v4_conn_request);
1502 * The three way handshake has completed - we got a valid synack -
1503 * now create the new socket.
1505 struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb,
1506 struct request_sock *req,
1507 struct dst_entry *dst,
1508 struct request_sock *req_unhash,
1511 struct inet_request_sock *ireq;
1512 bool found_dup_sk = false;
1513 struct inet_sock *newinet;
1514 struct tcp_sock *newtp;
1516 #ifdef CONFIG_TCP_MD5SIG
1517 const union tcp_md5_addr *addr;
1518 struct tcp_md5sig_key *key;
1521 struct ip_options_rcu *inet_opt;
1523 if (sk_acceptq_is_full(sk))
1526 newsk = tcp_create_openreq_child(sk, req, skb);
1530 newsk->sk_gso_type = SKB_GSO_TCPV4;
1531 inet_sk_rx_dst_set(newsk, skb);
1533 newtp = tcp_sk(newsk);
1534 newinet = inet_sk(newsk);
1535 ireq = inet_rsk(req);
1536 sk_daddr_set(newsk, ireq->ir_rmt_addr);
1537 sk_rcv_saddr_set(newsk, ireq->ir_loc_addr);
1538 newsk->sk_bound_dev_if = ireq->ir_iif;
1539 newinet->inet_saddr = ireq->ir_loc_addr;
1540 inet_opt = rcu_dereference(ireq->ireq_opt);
1541 RCU_INIT_POINTER(newinet->inet_opt, inet_opt);
1542 newinet->mc_index = inet_iif(skb);
1543 newinet->mc_ttl = ip_hdr(skb)->ttl;
1544 newinet->rcv_tos = ip_hdr(skb)->tos;
1545 inet_csk(newsk)->icsk_ext_hdr_len = 0;
1547 inet_csk(newsk)->icsk_ext_hdr_len = inet_opt->opt.optlen;
1548 newinet->inet_id = prandom_u32();
1550 /* Set ToS of the new socket based upon the value of incoming SYN.
1551 * ECT bits are set later in tcp_init_transfer().
1553 if (sock_net(sk)->ipv4.sysctl_tcp_reflect_tos)
1554 newinet->tos = tcp_rsk(req)->syn_tos & ~INET_ECN_MASK;
1557 dst = inet_csk_route_child_sock(sk, newsk, req);
1561 /* syncookie case : see end of cookie_v4_check() */
1563 sk_setup_caps(newsk, dst);
1565 tcp_ca_openreq_child(newsk, dst);
1567 tcp_sync_mss(newsk, dst_mtu(dst));
1568 newtp->advmss = tcp_mss_clamp(tcp_sk(sk), dst_metric_advmss(dst));
1570 tcp_initialize_rcv_mss(newsk);
1572 #ifdef CONFIG_TCP_MD5SIG
1573 l3index = l3mdev_master_ifindex_by_index(sock_net(sk), ireq->ir_iif);
1574 /* Copy over the MD5 key from the original socket */
1575 addr = (union tcp_md5_addr *)&newinet->inet_daddr;
1576 key = tcp_md5_do_lookup(sk, l3index, addr, AF_INET);
1579 * We're using one, so create a matching key
1580 * on the newsk structure. If we fail to get
1581 * memory, then we end up not copying the key
1584 tcp_md5_do_add(newsk, addr, AF_INET, 32, l3index,
1585 key->key, key->keylen, GFP_ATOMIC);
1586 sk_nocaps_add(newsk, NETIF_F_GSO_MASK);
1590 if (__inet_inherit_port(sk, newsk) < 0)
1592 *own_req = inet_ehash_nolisten(newsk, req_to_sk(req_unhash),
1594 if (likely(*own_req)) {
1595 tcp_move_syn(newtp, req);
1596 ireq->ireq_opt = NULL;
1598 if (!req_unhash && found_dup_sk) {
1599 /* This code path should only be executed in the
1600 * syncookie case only
1602 bh_unlock_sock(newsk);
1606 newinet->inet_opt = NULL;
1612 NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENOVERFLOWS);
1619 newinet->inet_opt = NULL;
1620 inet_csk_prepare_forced_close(newsk);
1624 EXPORT_SYMBOL(tcp_v4_syn_recv_sock);
1626 static struct sock *tcp_v4_cookie_check(struct sock *sk, struct sk_buff *skb)
1628 #ifdef CONFIG_SYN_COOKIES
1629 const struct tcphdr *th = tcp_hdr(skb);
1632 sk = cookie_v4_check(sk, skb);
1637 u16 tcp_v4_get_syncookie(struct sock *sk, struct iphdr *iph,
1638 struct tcphdr *th, u32 *cookie)
1641 #ifdef CONFIG_SYN_COOKIES
1642 mss = tcp_get_syncookie_mss(&tcp_request_sock_ops,
1643 &tcp_request_sock_ipv4_ops, sk, th);
1645 *cookie = __cookie_v4_init_sequence(iph, th, &mss);
1646 tcp_synq_overflow(sk);
1652 /* The socket must have it's spinlock held when we get
1653 * here, unless it is a TCP_LISTEN socket.
1655 * We have a potential double-lock case here, so even when
1656 * doing backlog processing we use the BH locking scheme.
1657 * This is because we cannot sleep with the original spinlock
1660 int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb)
1664 if (sk->sk_state == TCP_ESTABLISHED) { /* Fast path */
1665 struct dst_entry *dst = sk->sk_rx_dst;
1667 sock_rps_save_rxhash(sk, skb);
1668 sk_mark_napi_id(sk, skb);
1670 if (inet_sk(sk)->rx_dst_ifindex != skb->skb_iif ||
1671 !dst->ops->check(dst, 0)) {
1673 sk->sk_rx_dst = NULL;
1676 tcp_rcv_established(sk, skb);
1680 if (tcp_checksum_complete(skb))
1683 if (sk->sk_state == TCP_LISTEN) {
1684 struct sock *nsk = tcp_v4_cookie_check(sk, skb);
1689 if (tcp_child_process(sk, nsk, skb)) {
1696 sock_rps_save_rxhash(sk, skb);
1698 if (tcp_rcv_state_process(sk, skb)) {
1705 tcp_v4_send_reset(rsk, skb);
1708 /* Be careful here. If this function gets more complicated and
1709 * gcc suffers from register pressure on the x86, sk (in %ebx)
1710 * might be destroyed here. This current version compiles correctly,
1711 * but you have been warned.
1716 TCP_INC_STATS(sock_net(sk), TCP_MIB_CSUMERRORS);
1717 TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
1720 EXPORT_SYMBOL(tcp_v4_do_rcv);
1722 int tcp_v4_early_demux(struct sk_buff *skb)
1724 const struct iphdr *iph;
1725 const struct tcphdr *th;
1728 if (skb->pkt_type != PACKET_HOST)
1731 if (!pskb_may_pull(skb, skb_transport_offset(skb) + sizeof(struct tcphdr)))
1737 if (th->doff < sizeof(struct tcphdr) / 4)
1740 sk = __inet_lookup_established(dev_net(skb->dev), &tcp_hashinfo,
1741 iph->saddr, th->source,
1742 iph->daddr, ntohs(th->dest),
1743 skb->skb_iif, inet_sdif(skb));
1746 skb->destructor = sock_edemux;
1747 if (sk_fullsock(sk)) {
1748 struct dst_entry *dst = READ_ONCE(sk->sk_rx_dst);
1751 dst = dst_check(dst, 0);
1753 inet_sk(sk)->rx_dst_ifindex == skb->skb_iif)
1754 skb_dst_set_noref(skb, dst);
1760 bool tcp_add_backlog(struct sock *sk, struct sk_buff *skb)
1762 u32 limit = READ_ONCE(sk->sk_rcvbuf) + READ_ONCE(sk->sk_sndbuf);
1763 u32 tail_gso_size, tail_gso_segs;
1764 struct skb_shared_info *shinfo;
1765 const struct tcphdr *th;
1766 struct tcphdr *thtail;
1767 struct sk_buff *tail;
1768 unsigned int hdrlen;
1774 /* In case all data was pulled from skb frags (in __pskb_pull_tail()),
1775 * we can fix skb->truesize to its real value to avoid future drops.
1776 * This is valid because skb is not yet charged to the socket.
1777 * It has been noticed pure SACK packets were sometimes dropped
1778 * (if cooked by drivers without copybreak feature).
1784 if (unlikely(tcp_checksum_complete(skb))) {
1786 __TCP_INC_STATS(sock_net(sk), TCP_MIB_CSUMERRORS);
1787 __TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
1791 /* Attempt coalescing to last skb in backlog, even if we are
1793 * This is okay because skb capacity is limited to MAX_SKB_FRAGS.
1795 th = (const struct tcphdr *)skb->data;
1796 hdrlen = th->doff * 4;
1798 tail = sk->sk_backlog.tail;
1801 thtail = (struct tcphdr *)tail->data;
1803 if (TCP_SKB_CB(tail)->end_seq != TCP_SKB_CB(skb)->seq ||
1804 TCP_SKB_CB(tail)->ip_dsfield != TCP_SKB_CB(skb)->ip_dsfield ||
1805 ((TCP_SKB_CB(tail)->tcp_flags |
1806 TCP_SKB_CB(skb)->tcp_flags) & (TCPHDR_SYN | TCPHDR_RST | TCPHDR_URG)) ||
1807 !((TCP_SKB_CB(tail)->tcp_flags &
1808 TCP_SKB_CB(skb)->tcp_flags) & TCPHDR_ACK) ||
1809 ((TCP_SKB_CB(tail)->tcp_flags ^
1810 TCP_SKB_CB(skb)->tcp_flags) & (TCPHDR_ECE | TCPHDR_CWR)) ||
1811 #ifdef CONFIG_TLS_DEVICE
1812 tail->decrypted != skb->decrypted ||
1814 thtail->doff != th->doff ||
1815 memcmp(thtail + 1, th + 1, hdrlen - sizeof(*th)))
1818 __skb_pull(skb, hdrlen);
1820 shinfo = skb_shinfo(skb);
1821 gso_size = shinfo->gso_size ?: skb->len;
1822 gso_segs = shinfo->gso_segs ?: 1;
1824 shinfo = skb_shinfo(tail);
1825 tail_gso_size = shinfo->gso_size ?: (tail->len - hdrlen);
1826 tail_gso_segs = shinfo->gso_segs ?: 1;
1828 if (skb_try_coalesce(tail, skb, &fragstolen, &delta)) {
1829 TCP_SKB_CB(tail)->end_seq = TCP_SKB_CB(skb)->end_seq;
1831 if (likely(!before(TCP_SKB_CB(skb)->ack_seq, TCP_SKB_CB(tail)->ack_seq))) {
1832 TCP_SKB_CB(tail)->ack_seq = TCP_SKB_CB(skb)->ack_seq;
1833 thtail->window = th->window;
1836 /* We have to update both TCP_SKB_CB(tail)->tcp_flags and
1837 * thtail->fin, so that the fast path in tcp_rcv_established()
1838 * is not entered if we append a packet with a FIN.
1839 * SYN, RST, URG are not present.
1840 * ACK is set on both packets.
1841 * PSH : we do not really care in TCP stack,
1842 * at least for 'GRO' packets.
1844 thtail->fin |= th->fin;
1845 TCP_SKB_CB(tail)->tcp_flags |= TCP_SKB_CB(skb)->tcp_flags;
1847 if (TCP_SKB_CB(skb)->has_rxtstamp) {
1848 TCP_SKB_CB(tail)->has_rxtstamp = true;
1849 tail->tstamp = skb->tstamp;
1850 skb_hwtstamps(tail)->hwtstamp = skb_hwtstamps(skb)->hwtstamp;
1853 /* Not as strict as GRO. We only need to carry mss max value */
1854 shinfo->gso_size = max(gso_size, tail_gso_size);
1855 shinfo->gso_segs = min_t(u32, gso_segs + tail_gso_segs, 0xFFFF);
1857 sk->sk_backlog.len += delta;
1858 __NET_INC_STATS(sock_net(sk),
1859 LINUX_MIB_TCPBACKLOGCOALESCE);
1860 kfree_skb_partial(skb, fragstolen);
1863 __skb_push(skb, hdrlen);
1866 /* Only socket owner can try to collapse/prune rx queues
1867 * to reduce memory overhead, so add a little headroom here.
1868 * Few sockets backlog are possibly concurrently non empty.
1872 if (unlikely(sk_add_backlog(sk, skb, limit))) {
1874 __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPBACKLOGDROP);
1879 EXPORT_SYMBOL(tcp_add_backlog);
1881 int tcp_filter(struct sock *sk, struct sk_buff *skb)
1883 struct tcphdr *th = (struct tcphdr *)skb->data;
1885 return sk_filter_trim_cap(sk, skb, th->doff * 4);
1887 EXPORT_SYMBOL(tcp_filter);
1889 static void tcp_v4_restore_cb(struct sk_buff *skb)
1891 memmove(IPCB(skb), &TCP_SKB_CB(skb)->header.h4,
1892 sizeof(struct inet_skb_parm));
1895 static void tcp_v4_fill_cb(struct sk_buff *skb, const struct iphdr *iph,
1896 const struct tcphdr *th)
1898 /* This is tricky : We move IPCB at its correct location into TCP_SKB_CB()
1899 * barrier() makes sure compiler wont play fool^Waliasing games.
1901 memmove(&TCP_SKB_CB(skb)->header.h4, IPCB(skb),
1902 sizeof(struct inet_skb_parm));
1905 TCP_SKB_CB(skb)->seq = ntohl(th->seq);
1906 TCP_SKB_CB(skb)->end_seq = (TCP_SKB_CB(skb)->seq + th->syn + th->fin +
1907 skb->len - th->doff * 4);
1908 TCP_SKB_CB(skb)->ack_seq = ntohl(th->ack_seq);
1909 TCP_SKB_CB(skb)->tcp_flags = tcp_flag_byte(th);
1910 TCP_SKB_CB(skb)->tcp_tw_isn = 0;
1911 TCP_SKB_CB(skb)->ip_dsfield = ipv4_get_dsfield(iph);
1912 TCP_SKB_CB(skb)->sacked = 0;
1913 TCP_SKB_CB(skb)->has_rxtstamp =
1914 skb->tstamp || skb_hwtstamps(skb)->hwtstamp;
1921 int tcp_v4_rcv(struct sk_buff *skb)
1923 struct net *net = dev_net(skb->dev);
1924 struct sk_buff *skb_to_free;
1925 int sdif = inet_sdif(skb);
1926 int dif = inet_iif(skb);
1927 const struct iphdr *iph;
1928 const struct tcphdr *th;
1933 if (skb->pkt_type != PACKET_HOST)
1936 /* Count it even if it's bad */
1937 __TCP_INC_STATS(net, TCP_MIB_INSEGS);
1939 if (!pskb_may_pull(skb, sizeof(struct tcphdr)))
1942 th = (const struct tcphdr *)skb->data;
1944 if (unlikely(th->doff < sizeof(struct tcphdr) / 4))
1946 if (!pskb_may_pull(skb, th->doff * 4))
1949 /* An explanation is required here, I think.
1950 * Packet length and doff are validated by header prediction,
1951 * provided case of th->doff==0 is eliminated.
1952 * So, we defer the checks. */
1954 if (skb_checksum_init(skb, IPPROTO_TCP, inet_compute_pseudo))
1957 th = (const struct tcphdr *)skb->data;
1960 sk = __inet_lookup_skb(&tcp_hashinfo, skb, __tcp_hdrlen(th), th->source,
1961 th->dest, sdif, &refcounted);
1966 if (sk->sk_state == TCP_TIME_WAIT)
1969 if (sk->sk_state == TCP_NEW_SYN_RECV) {
1970 struct request_sock *req = inet_reqsk(sk);
1971 bool req_stolen = false;
1974 sk = req->rsk_listener;
1975 if (unlikely(tcp_v4_inbound_md5_hash(sk, skb, dif, sdif))) {
1976 sk_drops_add(sk, skb);
1980 if (tcp_checksum_complete(skb)) {
1984 if (unlikely(sk->sk_state != TCP_LISTEN)) {
1985 inet_csk_reqsk_queue_drop_and_put(sk, req);
1988 /* We own a reference on the listener, increase it again
1989 * as we might lose it too soon.
1994 if (!tcp_filter(sk, skb)) {
1995 th = (const struct tcphdr *)skb->data;
1997 tcp_v4_fill_cb(skb, iph, th);
1998 nsk = tcp_check_req(sk, skb, req, false, &req_stolen);
2003 /* Another cpu got exclusive access to req
2004 * and created a full blown socket.
2005 * Try to feed this packet to this socket
2006 * instead of discarding it.
2008 tcp_v4_restore_cb(skb);
2012 goto discard_and_relse;
2016 tcp_v4_restore_cb(skb);
2017 } else if (tcp_child_process(sk, nsk, skb)) {
2018 tcp_v4_send_reset(nsk, skb);
2019 goto discard_and_relse;
2025 if (unlikely(iph->ttl < inet_sk(sk)->min_ttl)) {
2026 __NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP);
2027 goto discard_and_relse;
2030 if (!xfrm4_policy_check(sk, XFRM_POLICY_IN, skb))
2031 goto discard_and_relse;
2033 if (tcp_v4_inbound_md5_hash(sk, skb, dif, sdif))
2034 goto discard_and_relse;
2038 if (tcp_filter(sk, skb))
2039 goto discard_and_relse;
2040 th = (const struct tcphdr *)skb->data;
2042 tcp_v4_fill_cb(skb, iph, th);
2046 if (sk->sk_state == TCP_LISTEN) {
2047 ret = tcp_v4_do_rcv(sk, skb);
2048 goto put_and_return;
2051 sk_incoming_cpu_update(sk);
2053 bh_lock_sock_nested(sk);
2054 tcp_segs_in(tcp_sk(sk), skb);
2056 if (!sock_owned_by_user(sk)) {
2057 skb_to_free = sk->sk_rx_skb_cache;
2058 sk->sk_rx_skb_cache = NULL;
2059 ret = tcp_v4_do_rcv(sk, skb);
2061 if (tcp_add_backlog(sk, skb))
2062 goto discard_and_relse;
2067 __kfree_skb(skb_to_free);
2076 if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb))
2079 tcp_v4_fill_cb(skb, iph, th);
2081 if (tcp_checksum_complete(skb)) {
2083 __TCP_INC_STATS(net, TCP_MIB_CSUMERRORS);
2085 __TCP_INC_STATS(net, TCP_MIB_INERRS);
2087 tcp_v4_send_reset(NULL, skb);
2091 /* Discard frame. */
2096 sk_drops_add(sk, skb);
2102 if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb)) {
2103 inet_twsk_put(inet_twsk(sk));
2107 tcp_v4_fill_cb(skb, iph, th);
2109 if (tcp_checksum_complete(skb)) {
2110 inet_twsk_put(inet_twsk(sk));
2113 switch (tcp_timewait_state_process(inet_twsk(sk), skb, th)) {
2115 struct sock *sk2 = inet_lookup_listener(dev_net(skb->dev),
2118 iph->saddr, th->source,
2119 iph->daddr, th->dest,
2123 inet_twsk_deschedule_put(inet_twsk(sk));
2125 tcp_v4_restore_cb(skb);
2133 tcp_v4_timewait_ack(sk, skb);
2136 tcp_v4_send_reset(sk, skb);
2137 inet_twsk_deschedule_put(inet_twsk(sk));
2139 case TCP_TW_SUCCESS:;
2144 static struct timewait_sock_ops tcp_timewait_sock_ops = {
2145 .twsk_obj_size = sizeof(struct tcp_timewait_sock),
2146 .twsk_unique = tcp_twsk_unique,
2147 .twsk_destructor= tcp_twsk_destructor,
2150 void inet_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb)
2152 struct dst_entry *dst = skb_dst(skb);
2154 if (dst && dst_hold_safe(dst)) {
2155 sk->sk_rx_dst = dst;
2156 inet_sk(sk)->rx_dst_ifindex = skb->skb_iif;
2159 EXPORT_SYMBOL(inet_sk_rx_dst_set);
2161 const struct inet_connection_sock_af_ops ipv4_specific = {
2162 .queue_xmit = ip_queue_xmit,
2163 .send_check = tcp_v4_send_check,
2164 .rebuild_header = inet_sk_rebuild_header,
2165 .sk_rx_dst_set = inet_sk_rx_dst_set,
2166 .conn_request = tcp_v4_conn_request,
2167 .syn_recv_sock = tcp_v4_syn_recv_sock,
2168 .net_header_len = sizeof(struct iphdr),
2169 .setsockopt = ip_setsockopt,
2170 .getsockopt = ip_getsockopt,
2171 .addr2sockaddr = inet_csk_addr2sockaddr,
2172 .sockaddr_len = sizeof(struct sockaddr_in),
2173 .mtu_reduced = tcp_v4_mtu_reduced,
2175 EXPORT_SYMBOL(ipv4_specific);
2177 #ifdef CONFIG_TCP_MD5SIG
2178 static const struct tcp_sock_af_ops tcp_sock_ipv4_specific = {
2179 .md5_lookup = tcp_v4_md5_lookup,
2180 .calc_md5_hash = tcp_v4_md5_hash_skb,
2181 .md5_parse = tcp_v4_parse_md5_keys,
2185 /* NOTE: A lot of things set to zero explicitly by call to
2186 * sk_alloc() so need not be done here.
2188 static int tcp_v4_init_sock(struct sock *sk)
2190 struct inet_connection_sock *icsk = inet_csk(sk);
2194 icsk->icsk_af_ops = &ipv4_specific;
2196 #ifdef CONFIG_TCP_MD5SIG
2197 tcp_sk(sk)->af_specific = &tcp_sock_ipv4_specific;
2203 void tcp_v4_destroy_sock(struct sock *sk)
2205 struct tcp_sock *tp = tcp_sk(sk);
2207 trace_tcp_destroy_sock(sk);
2209 tcp_clear_xmit_timers(sk);
2211 tcp_cleanup_congestion_control(sk);
2213 tcp_cleanup_ulp(sk);
2215 /* Cleanup up the write buffer. */
2216 tcp_write_queue_purge(sk);
2218 /* Check if we want to disable active TFO */
2219 tcp_fastopen_active_disable_ofo_check(sk);
2221 /* Cleans up our, hopefully empty, out_of_order_queue. */
2222 skb_rbtree_purge(&tp->out_of_order_queue);
2224 #ifdef CONFIG_TCP_MD5SIG
2225 /* Clean up the MD5 key list, if any */
2226 if (tp->md5sig_info) {
2227 tcp_clear_md5_list(sk);
2228 kfree_rcu(rcu_dereference_protected(tp->md5sig_info, 1), rcu);
2229 tp->md5sig_info = NULL;
2233 /* Clean up a referenced TCP bind bucket. */
2234 if (inet_csk(sk)->icsk_bind_hash)
2237 BUG_ON(rcu_access_pointer(tp->fastopen_rsk));
2239 /* If socket is aborted during connect operation */
2240 tcp_free_fastopen_req(tp);
2241 tcp_fastopen_destroy_cipher(sk);
2242 tcp_saved_syn_free(tp);
2244 sk_sockets_allocated_dec(sk);
2246 EXPORT_SYMBOL(tcp_v4_destroy_sock);
2248 #ifdef CONFIG_PROC_FS
2249 /* Proc filesystem TCP sock list dumping. */
2252 * Get next listener socket follow cur. If cur is NULL, get first socket
2253 * starting from bucket given in st->bucket; when st->bucket is zero the
2254 * very first socket in the hash table is returned.
2256 static void *listening_get_next(struct seq_file *seq, void *cur)
2258 struct tcp_seq_afinfo *afinfo;
2259 struct tcp_iter_state *st = seq->private;
2260 struct net *net = seq_file_net(seq);
2261 struct inet_listen_hashbucket *ilb;
2262 struct hlist_nulls_node *node;
2263 struct sock *sk = cur;
2265 if (st->bpf_seq_afinfo)
2266 afinfo = st->bpf_seq_afinfo;
2268 afinfo = PDE_DATA(file_inode(seq->file));
2272 ilb = &tcp_hashinfo.listening_hash[st->bucket];
2273 spin_lock(&ilb->lock);
2274 sk = sk_nulls_head(&ilb->nulls_head);
2278 ilb = &tcp_hashinfo.listening_hash[st->bucket];
2282 sk = sk_nulls_next(sk);
2284 sk_nulls_for_each_from(sk, node) {
2285 if (!net_eq(sock_net(sk), net))
2287 if (afinfo->family == AF_UNSPEC ||
2288 sk->sk_family == afinfo->family)
2291 spin_unlock(&ilb->lock);
2293 if (++st->bucket < INET_LHTABLE_SIZE)
2298 static void *listening_get_idx(struct seq_file *seq, loff_t *pos)
2300 struct tcp_iter_state *st = seq->private;
2305 rc = listening_get_next(seq, NULL);
2307 while (rc && *pos) {
2308 rc = listening_get_next(seq, rc);
2314 static inline bool empty_bucket(const struct tcp_iter_state *st)
2316 return hlist_nulls_empty(&tcp_hashinfo.ehash[st->bucket].chain);
2320 * Get first established socket starting from bucket given in st->bucket.
2321 * If st->bucket is zero, the very first socket in the hash is returned.
2323 static void *established_get_first(struct seq_file *seq)
2325 struct tcp_seq_afinfo *afinfo;
2326 struct tcp_iter_state *st = seq->private;
2327 struct net *net = seq_file_net(seq);
2330 if (st->bpf_seq_afinfo)
2331 afinfo = st->bpf_seq_afinfo;
2333 afinfo = PDE_DATA(file_inode(seq->file));
2336 for (; st->bucket <= tcp_hashinfo.ehash_mask; ++st->bucket) {
2338 struct hlist_nulls_node *node;
2339 spinlock_t *lock = inet_ehash_lockp(&tcp_hashinfo, st->bucket);
2341 /* Lockless fast path for the common case of empty buckets */
2342 if (empty_bucket(st))
2346 sk_nulls_for_each(sk, node, &tcp_hashinfo.ehash[st->bucket].chain) {
2347 if ((afinfo->family != AF_UNSPEC &&
2348 sk->sk_family != afinfo->family) ||
2349 !net_eq(sock_net(sk), net)) {
2355 spin_unlock_bh(lock);
2361 static void *established_get_next(struct seq_file *seq, void *cur)
2363 struct tcp_seq_afinfo *afinfo;
2364 struct sock *sk = cur;
2365 struct hlist_nulls_node *node;
2366 struct tcp_iter_state *st = seq->private;
2367 struct net *net = seq_file_net(seq);
2369 if (st->bpf_seq_afinfo)
2370 afinfo = st->bpf_seq_afinfo;
2372 afinfo = PDE_DATA(file_inode(seq->file));
2377 sk = sk_nulls_next(sk);
2379 sk_nulls_for_each_from(sk, node) {
2380 if ((afinfo->family == AF_UNSPEC ||
2381 sk->sk_family == afinfo->family) &&
2382 net_eq(sock_net(sk), net))
2386 spin_unlock_bh(inet_ehash_lockp(&tcp_hashinfo, st->bucket));
2388 return established_get_first(seq);
2391 static void *established_get_idx(struct seq_file *seq, loff_t pos)
2393 struct tcp_iter_state *st = seq->private;
2397 rc = established_get_first(seq);
2400 rc = established_get_next(seq, rc);
2406 static void *tcp_get_idx(struct seq_file *seq, loff_t pos)
2409 struct tcp_iter_state *st = seq->private;
2411 st->state = TCP_SEQ_STATE_LISTENING;
2412 rc = listening_get_idx(seq, &pos);
2415 st->state = TCP_SEQ_STATE_ESTABLISHED;
2416 rc = established_get_idx(seq, pos);
2422 static void *tcp_seek_last_pos(struct seq_file *seq)
2424 struct tcp_iter_state *st = seq->private;
2425 int offset = st->offset;
2426 int orig_num = st->num;
2429 switch (st->state) {
2430 case TCP_SEQ_STATE_LISTENING:
2431 if (st->bucket >= INET_LHTABLE_SIZE)
2433 st->state = TCP_SEQ_STATE_LISTENING;
2434 rc = listening_get_next(seq, NULL);
2435 while (offset-- && rc)
2436 rc = listening_get_next(seq, rc);
2440 st->state = TCP_SEQ_STATE_ESTABLISHED;
2442 case TCP_SEQ_STATE_ESTABLISHED:
2443 if (st->bucket > tcp_hashinfo.ehash_mask)
2445 rc = established_get_first(seq);
2446 while (offset-- && rc)
2447 rc = established_get_next(seq, rc);
2455 void *tcp_seq_start(struct seq_file *seq, loff_t *pos)
2457 struct tcp_iter_state *st = seq->private;
2460 if (*pos && *pos == st->last_pos) {
2461 rc = tcp_seek_last_pos(seq);
2466 st->state = TCP_SEQ_STATE_LISTENING;
2470 rc = *pos ? tcp_get_idx(seq, *pos - 1) : SEQ_START_TOKEN;
2473 st->last_pos = *pos;
2476 EXPORT_SYMBOL(tcp_seq_start);
2478 void *tcp_seq_next(struct seq_file *seq, void *v, loff_t *pos)
2480 struct tcp_iter_state *st = seq->private;
2483 if (v == SEQ_START_TOKEN) {
2484 rc = tcp_get_idx(seq, 0);
2488 switch (st->state) {
2489 case TCP_SEQ_STATE_LISTENING:
2490 rc = listening_get_next(seq, v);
2492 st->state = TCP_SEQ_STATE_ESTABLISHED;
2495 rc = established_get_first(seq);
2498 case TCP_SEQ_STATE_ESTABLISHED:
2499 rc = established_get_next(seq, v);
2504 st->last_pos = *pos;
2507 EXPORT_SYMBOL(tcp_seq_next);
2509 void tcp_seq_stop(struct seq_file *seq, void *v)
2511 struct tcp_iter_state *st = seq->private;
2513 switch (st->state) {
2514 case TCP_SEQ_STATE_LISTENING:
2515 if (v != SEQ_START_TOKEN)
2516 spin_unlock(&tcp_hashinfo.listening_hash[st->bucket].lock);
2518 case TCP_SEQ_STATE_ESTABLISHED:
2520 spin_unlock_bh(inet_ehash_lockp(&tcp_hashinfo, st->bucket));
2524 EXPORT_SYMBOL(tcp_seq_stop);
2526 static void get_openreq4(const struct request_sock *req,
2527 struct seq_file *f, int i)
2529 const struct inet_request_sock *ireq = inet_rsk(req);
2530 long delta = req->rsk_timer.expires - jiffies;
2532 seq_printf(f, "%4d: %08X:%04X %08X:%04X"
2533 " %02X %08X:%08X %02X:%08lX %08X %5u %8d %u %d %pK",
2538 ntohs(ireq->ir_rmt_port),
2540 0, 0, /* could print option size, but that is af dependent. */
2541 1, /* timers active (only the expire timer) */
2542 jiffies_delta_to_clock_t(delta),
2544 from_kuid_munged(seq_user_ns(f),
2545 sock_i_uid(req->rsk_listener)),
2546 0, /* non standard timer */
2547 0, /* open_requests have no inode */
2552 static void get_tcp4_sock(struct sock *sk, struct seq_file *f, int i)
2555 unsigned long timer_expires;
2556 const struct tcp_sock *tp = tcp_sk(sk);
2557 const struct inet_connection_sock *icsk = inet_csk(sk);
2558 const struct inet_sock *inet = inet_sk(sk);
2559 const struct fastopen_queue *fastopenq = &icsk->icsk_accept_queue.fastopenq;
2560 __be32 dest = inet->inet_daddr;
2561 __be32 src = inet->inet_rcv_saddr;
2562 __u16 destp = ntohs(inet->inet_dport);
2563 __u16 srcp = ntohs(inet->inet_sport);
2567 if (icsk->icsk_pending == ICSK_TIME_RETRANS ||
2568 icsk->icsk_pending == ICSK_TIME_REO_TIMEOUT ||
2569 icsk->icsk_pending == ICSK_TIME_LOSS_PROBE) {
2571 timer_expires = icsk->icsk_timeout;
2572 } else if (icsk->icsk_pending == ICSK_TIME_PROBE0) {
2574 timer_expires = icsk->icsk_timeout;
2575 } else if (timer_pending(&sk->sk_timer)) {
2577 timer_expires = sk->sk_timer.expires;
2580 timer_expires = jiffies;
2583 state = inet_sk_state_load(sk);
2584 if (state == TCP_LISTEN)
2585 rx_queue = READ_ONCE(sk->sk_ack_backlog);
2587 /* Because we don't lock the socket,
2588 * we might find a transient negative value.
2590 rx_queue = max_t(int, READ_ONCE(tp->rcv_nxt) -
2591 READ_ONCE(tp->copied_seq), 0);
2593 seq_printf(f, "%4d: %08X:%04X %08X:%04X %02X %08X:%08X %02X:%08lX "
2594 "%08X %5u %8d %lu %d %pK %lu %lu %u %u %d",
2595 i, src, srcp, dest, destp, state,
2596 READ_ONCE(tp->write_seq) - tp->snd_una,
2599 jiffies_delta_to_clock_t(timer_expires - jiffies),
2600 icsk->icsk_retransmits,
2601 from_kuid_munged(seq_user_ns(f), sock_i_uid(sk)),
2602 icsk->icsk_probes_out,
2604 refcount_read(&sk->sk_refcnt), sk,
2605 jiffies_to_clock_t(icsk->icsk_rto),
2606 jiffies_to_clock_t(icsk->icsk_ack.ato),
2607 (icsk->icsk_ack.quick << 1) | inet_csk_in_pingpong_mode(sk),
2609 state == TCP_LISTEN ?
2610 fastopenq->max_qlen :
2611 (tcp_in_initial_slowstart(tp) ? -1 : tp->snd_ssthresh));
2614 static void get_timewait4_sock(const struct inet_timewait_sock *tw,
2615 struct seq_file *f, int i)
2617 long delta = tw->tw_timer.expires - jiffies;
2621 dest = tw->tw_daddr;
2622 src = tw->tw_rcv_saddr;
2623 destp = ntohs(tw->tw_dport);
2624 srcp = ntohs(tw->tw_sport);
2626 seq_printf(f, "%4d: %08X:%04X %08X:%04X"
2627 " %02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d %pK",
2628 i, src, srcp, dest, destp, tw->tw_substate, 0, 0,
2629 3, jiffies_delta_to_clock_t(delta), 0, 0, 0, 0,
2630 refcount_read(&tw->tw_refcnt), tw);
2635 static int tcp4_seq_show(struct seq_file *seq, void *v)
2637 struct tcp_iter_state *st;
2638 struct sock *sk = v;
2640 seq_setwidth(seq, TMPSZ - 1);
2641 if (v == SEQ_START_TOKEN) {
2642 seq_puts(seq, " sl local_address rem_address st tx_queue "
2643 "rx_queue tr tm->when retrnsmt uid timeout "
2649 if (sk->sk_state == TCP_TIME_WAIT)
2650 get_timewait4_sock(v, seq, st->num);
2651 else if (sk->sk_state == TCP_NEW_SYN_RECV)
2652 get_openreq4(v, seq, st->num);
2654 get_tcp4_sock(v, seq, st->num);
2660 #ifdef CONFIG_BPF_SYSCALL
2661 struct bpf_iter__tcp {
2662 __bpf_md_ptr(struct bpf_iter_meta *, meta);
2663 __bpf_md_ptr(struct sock_common *, sk_common);
2664 uid_t uid __aligned(8);
2667 static int tcp_prog_seq_show(struct bpf_prog *prog, struct bpf_iter_meta *meta,
2668 struct sock_common *sk_common, uid_t uid)
2670 struct bpf_iter__tcp ctx;
2672 meta->seq_num--; /* skip SEQ_START_TOKEN */
2674 ctx.sk_common = sk_common;
2676 return bpf_iter_run_prog(prog, &ctx);
2679 static int bpf_iter_tcp_seq_show(struct seq_file *seq, void *v)
2681 struct bpf_iter_meta meta;
2682 struct bpf_prog *prog;
2683 struct sock *sk = v;
2686 if (v == SEQ_START_TOKEN)
2689 if (sk->sk_state == TCP_TIME_WAIT) {
2691 } else if (sk->sk_state == TCP_NEW_SYN_RECV) {
2692 const struct request_sock *req = v;
2694 uid = from_kuid_munged(seq_user_ns(seq),
2695 sock_i_uid(req->rsk_listener));
2697 uid = from_kuid_munged(seq_user_ns(seq), sock_i_uid(sk));
2701 prog = bpf_iter_get_info(&meta, false);
2702 return tcp_prog_seq_show(prog, &meta, v, uid);
2705 static void bpf_iter_tcp_seq_stop(struct seq_file *seq, void *v)
2707 struct bpf_iter_meta meta;
2708 struct bpf_prog *prog;
2712 prog = bpf_iter_get_info(&meta, true);
2714 (void)tcp_prog_seq_show(prog, &meta, v, 0);
2717 tcp_seq_stop(seq, v);
2720 static const struct seq_operations bpf_iter_tcp_seq_ops = {
2721 .show = bpf_iter_tcp_seq_show,
2722 .start = tcp_seq_start,
2723 .next = tcp_seq_next,
2724 .stop = bpf_iter_tcp_seq_stop,
2728 static const struct seq_operations tcp4_seq_ops = {
2729 .show = tcp4_seq_show,
2730 .start = tcp_seq_start,
2731 .next = tcp_seq_next,
2732 .stop = tcp_seq_stop,
2735 static struct tcp_seq_afinfo tcp4_seq_afinfo = {
2739 static int __net_init tcp4_proc_init_net(struct net *net)
2741 if (!proc_create_net_data("tcp", 0444, net->proc_net, &tcp4_seq_ops,
2742 sizeof(struct tcp_iter_state), &tcp4_seq_afinfo))
2747 static void __net_exit tcp4_proc_exit_net(struct net *net)
2749 remove_proc_entry("tcp", net->proc_net);
2752 static struct pernet_operations tcp4_net_ops = {
2753 .init = tcp4_proc_init_net,
2754 .exit = tcp4_proc_exit_net,
2757 int __init tcp4_proc_init(void)
2759 return register_pernet_subsys(&tcp4_net_ops);
2762 void tcp4_proc_exit(void)
2764 unregister_pernet_subsys(&tcp4_net_ops);
2766 #endif /* CONFIG_PROC_FS */
2768 /* @wake is one when sk_stream_write_space() calls us.
2769 * This sends EPOLLOUT only if notsent_bytes is half the limit.
2770 * This mimics the strategy used in sock_def_write_space().
2772 bool tcp_stream_memory_free(const struct sock *sk, int wake)
2774 const struct tcp_sock *tp = tcp_sk(sk);
2775 u32 notsent_bytes = READ_ONCE(tp->write_seq) -
2776 READ_ONCE(tp->snd_nxt);
2778 return (notsent_bytes << wake) < tcp_notsent_lowat(tp);
2780 EXPORT_SYMBOL(tcp_stream_memory_free);
2782 struct proto tcp_prot = {
2784 .owner = THIS_MODULE,
2786 .pre_connect = tcp_v4_pre_connect,
2787 .connect = tcp_v4_connect,
2788 .disconnect = tcp_disconnect,
2789 .accept = inet_csk_accept,
2791 .init = tcp_v4_init_sock,
2792 .destroy = tcp_v4_destroy_sock,
2793 .shutdown = tcp_shutdown,
2794 .setsockopt = tcp_setsockopt,
2795 .getsockopt = tcp_getsockopt,
2796 .keepalive = tcp_set_keepalive,
2797 .recvmsg = tcp_recvmsg,
2798 .sendmsg = tcp_sendmsg,
2799 .sendpage = tcp_sendpage,
2800 .backlog_rcv = tcp_v4_do_rcv,
2801 .release_cb = tcp_release_cb,
2803 .unhash = inet_unhash,
2804 .get_port = inet_csk_get_port,
2805 .enter_memory_pressure = tcp_enter_memory_pressure,
2806 .leave_memory_pressure = tcp_leave_memory_pressure,
2807 .stream_memory_free = tcp_stream_memory_free,
2808 .sockets_allocated = &tcp_sockets_allocated,
2809 .orphan_count = &tcp_orphan_count,
2810 .memory_allocated = &tcp_memory_allocated,
2811 .memory_pressure = &tcp_memory_pressure,
2812 .sysctl_mem = sysctl_tcp_mem,
2813 .sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_tcp_wmem),
2814 .sysctl_rmem_offset = offsetof(struct net, ipv4.sysctl_tcp_rmem),
2815 .max_header = MAX_TCP_HEADER,
2816 .obj_size = sizeof(struct tcp_sock),
2817 .slab_flags = SLAB_TYPESAFE_BY_RCU,
2818 .twsk_prot = &tcp_timewait_sock_ops,
2819 .rsk_prot = &tcp_request_sock_ops,
2820 .h.hashinfo = &tcp_hashinfo,
2821 .no_autobind = true,
2822 .diag_destroy = tcp_abort,
2824 EXPORT_SYMBOL(tcp_prot);
2826 static void __net_exit tcp_sk_exit(struct net *net)
2830 if (net->ipv4.tcp_congestion_control)
2831 bpf_module_put(net->ipv4.tcp_congestion_control,
2832 net->ipv4.tcp_congestion_control->owner);
2834 for_each_possible_cpu(cpu)
2835 inet_ctl_sock_destroy(*per_cpu_ptr(net->ipv4.tcp_sk, cpu));
2836 free_percpu(net->ipv4.tcp_sk);
2839 static int __net_init tcp_sk_init(struct net *net)
2843 net->ipv4.tcp_sk = alloc_percpu(struct sock *);
2844 if (!net->ipv4.tcp_sk)
2847 for_each_possible_cpu(cpu) {
2850 res = inet_ctl_sock_create(&sk, PF_INET, SOCK_RAW,
2854 sock_set_flag(sk, SOCK_USE_WRITE_QUEUE);
2856 /* Please enforce IP_DF and IPID==0 for RST and
2857 * ACK sent in SYN-RECV and TIME-WAIT state.
2859 inet_sk(sk)->pmtudisc = IP_PMTUDISC_DO;
2861 *per_cpu_ptr(net->ipv4.tcp_sk, cpu) = sk;
2864 net->ipv4.sysctl_tcp_ecn = 2;
2865 net->ipv4.sysctl_tcp_ecn_fallback = 1;
2867 net->ipv4.sysctl_tcp_base_mss = TCP_BASE_MSS;
2868 net->ipv4.sysctl_tcp_min_snd_mss = TCP_MIN_SND_MSS;
2869 net->ipv4.sysctl_tcp_probe_threshold = TCP_PROBE_THRESHOLD;
2870 net->ipv4.sysctl_tcp_probe_interval = TCP_PROBE_INTERVAL;
2871 net->ipv4.sysctl_tcp_mtu_probe_floor = TCP_MIN_SND_MSS;
2873 net->ipv4.sysctl_tcp_keepalive_time = TCP_KEEPALIVE_TIME;
2874 net->ipv4.sysctl_tcp_keepalive_probes = TCP_KEEPALIVE_PROBES;
2875 net->ipv4.sysctl_tcp_keepalive_intvl = TCP_KEEPALIVE_INTVL;
2877 net->ipv4.sysctl_tcp_syn_retries = TCP_SYN_RETRIES;
2878 net->ipv4.sysctl_tcp_synack_retries = TCP_SYNACK_RETRIES;
2879 net->ipv4.sysctl_tcp_syncookies = 1;
2880 net->ipv4.sysctl_tcp_reordering = TCP_FASTRETRANS_THRESH;
2881 net->ipv4.sysctl_tcp_retries1 = TCP_RETR1;
2882 net->ipv4.sysctl_tcp_retries2 = TCP_RETR2;
2883 net->ipv4.sysctl_tcp_orphan_retries = 0;
2884 net->ipv4.sysctl_tcp_fin_timeout = TCP_FIN_TIMEOUT;
2885 net->ipv4.sysctl_tcp_notsent_lowat = UINT_MAX;
2886 net->ipv4.sysctl_tcp_tw_reuse = 2;
2887 net->ipv4.sysctl_tcp_no_ssthresh_metrics_save = 1;
2889 cnt = tcp_hashinfo.ehash_mask + 1;
2890 net->ipv4.tcp_death_row.sysctl_max_tw_buckets = cnt / 2;
2891 net->ipv4.tcp_death_row.hashinfo = &tcp_hashinfo;
2893 net->ipv4.sysctl_max_syn_backlog = max(128, cnt / 128);
2894 net->ipv4.sysctl_tcp_sack = 1;
2895 net->ipv4.sysctl_tcp_window_scaling = 1;
2896 net->ipv4.sysctl_tcp_timestamps = 1;
2897 net->ipv4.sysctl_tcp_early_retrans = 3;
2898 net->ipv4.sysctl_tcp_recovery = TCP_RACK_LOSS_DETECTION;
2899 net->ipv4.sysctl_tcp_slow_start_after_idle = 1; /* By default, RFC2861 behavior. */
2900 net->ipv4.sysctl_tcp_retrans_collapse = 1;
2901 net->ipv4.sysctl_tcp_max_reordering = 300;
2902 net->ipv4.sysctl_tcp_dsack = 1;
2903 net->ipv4.sysctl_tcp_app_win = 31;
2904 net->ipv4.sysctl_tcp_adv_win_scale = 1;
2905 net->ipv4.sysctl_tcp_frto = 2;
2906 net->ipv4.sysctl_tcp_moderate_rcvbuf = 1;
2907 /* This limits the percentage of the congestion window which we
2908 * will allow a single TSO frame to consume. Building TSO frames
2909 * which are too large can cause TCP streams to be bursty.
2911 net->ipv4.sysctl_tcp_tso_win_divisor = 3;
2912 /* Default TSQ limit of 16 TSO segments */
2913 net->ipv4.sysctl_tcp_limit_output_bytes = 16 * 65536;
2914 /* rfc5961 challenge ack rate limiting */
2915 net->ipv4.sysctl_tcp_challenge_ack_limit = 1000;
2916 net->ipv4.sysctl_tcp_min_tso_segs = 2;
2917 net->ipv4.sysctl_tcp_min_rtt_wlen = 300;
2918 net->ipv4.sysctl_tcp_autocorking = 1;
2919 net->ipv4.sysctl_tcp_invalid_ratelimit = HZ/2;
2920 net->ipv4.sysctl_tcp_pacing_ss_ratio = 200;
2921 net->ipv4.sysctl_tcp_pacing_ca_ratio = 120;
2922 if (net != &init_net) {
2923 memcpy(net->ipv4.sysctl_tcp_rmem,
2924 init_net.ipv4.sysctl_tcp_rmem,
2925 sizeof(init_net.ipv4.sysctl_tcp_rmem));
2926 memcpy(net->ipv4.sysctl_tcp_wmem,
2927 init_net.ipv4.sysctl_tcp_wmem,
2928 sizeof(init_net.ipv4.sysctl_tcp_wmem));
2930 net->ipv4.sysctl_tcp_comp_sack_delay_ns = NSEC_PER_MSEC;
2931 net->ipv4.sysctl_tcp_comp_sack_slack_ns = 100 * NSEC_PER_USEC;
2932 net->ipv4.sysctl_tcp_comp_sack_nr = 44;
2933 net->ipv4.sysctl_tcp_fastopen = TFO_CLIENT_ENABLE;
2934 spin_lock_init(&net->ipv4.tcp_fastopen_ctx_lock);
2935 net->ipv4.sysctl_tcp_fastopen_blackhole_timeout = 60 * 60;
2936 atomic_set(&net->ipv4.tfo_active_disable_times, 0);
2938 /* Reno is always built in */
2939 if (!net_eq(net, &init_net) &&
2940 bpf_try_module_get(init_net.ipv4.tcp_congestion_control,
2941 init_net.ipv4.tcp_congestion_control->owner))
2942 net->ipv4.tcp_congestion_control = init_net.ipv4.tcp_congestion_control;
2944 net->ipv4.tcp_congestion_control = &tcp_reno;
2953 static void __net_exit tcp_sk_exit_batch(struct list_head *net_exit_list)
2957 inet_twsk_purge(&tcp_hashinfo, AF_INET);
2959 list_for_each_entry(net, net_exit_list, exit_list)
2960 tcp_fastopen_ctx_destroy(net);
2963 static struct pernet_operations __net_initdata tcp_sk_ops = {
2964 .init = tcp_sk_init,
2965 .exit = tcp_sk_exit,
2966 .exit_batch = tcp_sk_exit_batch,
2969 #if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
2970 DEFINE_BPF_ITER_FUNC(tcp, struct bpf_iter_meta *meta,
2971 struct sock_common *sk_common, uid_t uid)
2973 static int bpf_iter_init_tcp(void *priv_data, struct bpf_iter_aux_info *aux)
2975 struct tcp_iter_state *st = priv_data;
2976 struct tcp_seq_afinfo *afinfo;
2979 afinfo = kmalloc(sizeof(*afinfo), GFP_USER | __GFP_NOWARN);
2983 afinfo->family = AF_UNSPEC;
2984 st->bpf_seq_afinfo = afinfo;
2985 ret = bpf_iter_init_seq_net(priv_data, aux);
2991 static void bpf_iter_fini_tcp(void *priv_data)
2993 struct tcp_iter_state *st = priv_data;
2995 kfree(st->bpf_seq_afinfo);
2996 bpf_iter_fini_seq_net(priv_data);
2999 static const struct bpf_iter_seq_info tcp_seq_info = {
3000 .seq_ops = &bpf_iter_tcp_seq_ops,
3001 .init_seq_private = bpf_iter_init_tcp,
3002 .fini_seq_private = bpf_iter_fini_tcp,
3003 .seq_priv_size = sizeof(struct tcp_iter_state),
3006 static struct bpf_iter_reg tcp_reg_info = {
3008 .ctx_arg_info_size = 1,
3010 { offsetof(struct bpf_iter__tcp, sk_common),
3011 PTR_TO_BTF_ID_OR_NULL },
3013 .seq_info = &tcp_seq_info,
3016 static void __init bpf_iter_register(void)
3018 tcp_reg_info.ctx_arg_info[0].btf_id = btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON];
3019 if (bpf_iter_reg_target(&tcp_reg_info))
3020 pr_warn("Warning: could not register bpf iterator tcp\n");
3025 void __init tcp_v4_init(void)
3027 if (register_pernet_subsys(&tcp_sk_ops))
3028 panic("Failed to create the TCP control socket.\n");
3030 #if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
3031 bpf_iter_register();