2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
6 * Implementation of the Transmission Control Protocol(TCP).
8 * IPv4 specific functions
13 * linux/ipv4/tcp_input.c
14 * linux/ipv4/tcp_output.c
16 * See tcp.c for author information
18 * This program is free software; you can redistribute it and/or
19 * modify it under the terms of the GNU General Public License
20 * as published by the Free Software Foundation; either version
21 * 2 of the License, or (at your option) any later version.
26 * David S. Miller : New socket lookup architecture.
27 * This code is dedicated to John Dyson.
28 * David S. Miller : Change semantics of established hash,
29 * half is devoted to TIME_WAIT sockets
30 * and the rest go in the other half.
31 * Andi Kleen : Add support for syncookies and fixed
32 * some bugs: ip options weren't passed to
33 * the TCP layer, missed a check for an
35 * Andi Kleen : Implemented fast path mtu discovery.
36 * Fixed many serious bugs in the
37 * request_sock handling and moved
38 * most of it into the af independent code.
39 * Added tail drop and some other bugfixes.
40 * Added new listen semantics.
41 * Mike McLagan : Routing by source
42 * Juan Jose Ciarlante: ip_dynaddr bits
43 * Andi Kleen: various fixes.
44 * Vitaly E. Lavrov : Transparent proxy revived after year
46 * Andi Kleen : Fix new listen.
47 * Andi Kleen : Fix accept error reporting.
48 * YOSHIFUJI Hideaki @USAGI and: Support IPV6_V6ONLY socket option, which
49 * Alexey Kuznetsov allow both IPv4 and IPv6 sockets to bind
50 * a single port at the same time.
53 #define pr_fmt(fmt) "TCP: " fmt
55 #include <linux/bottom_half.h>
56 #include <linux/types.h>
57 #include <linux/fcntl.h>
58 #include <linux/module.h>
59 #include <linux/random.h>
60 #include <linux/cache.h>
61 #include <linux/jhash.h>
62 #include <linux/init.h>
63 #include <linux/times.h>
64 #include <linux/slab.h>
66 #include <net/net_namespace.h>
68 #include <net/inet_hashtables.h>
70 #include <net/transp_v6.h>
72 #include <net/inet_common.h>
73 #include <net/timewait_sock.h>
75 #include <net/secure_seq.h>
76 #include <net/busy_poll.h>
78 #include <linux/inet.h>
79 #include <linux/ipv6.h>
80 #include <linux/stddef.h>
81 #include <linux/proc_fs.h>
82 #include <linux/seq_file.h>
83 #include <linux/inetdevice.h>
85 #include <crypto/hash.h>
86 #include <linux/scatterlist.h>
88 #include <trace/events/tcp.h>
90 #ifdef CONFIG_TCP_MD5SIG
91 static int tcp_v4_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key *key,
92 __be32 daddr, __be32 saddr, const struct tcphdr *th);
95 struct inet_hashinfo tcp_hashinfo;
96 EXPORT_SYMBOL(tcp_hashinfo);
98 static u32 tcp_v4_init_seq(const struct sk_buff *skb)
100 return secure_tcp_seq(ip_hdr(skb)->daddr,
103 tcp_hdr(skb)->source);
106 static u32 tcp_v4_init_ts_off(const struct net *net, const struct sk_buff *skb)
108 return secure_tcp_ts_off(net, ip_hdr(skb)->daddr, ip_hdr(skb)->saddr);
111 int tcp_twsk_unique(struct sock *sk, struct sock *sktw, void *twp)
113 const struct inet_timewait_sock *tw = inet_twsk(sktw);
114 const struct tcp_timewait_sock *tcptw = tcp_twsk(sktw);
115 struct tcp_sock *tp = tcp_sk(sk);
116 int reuse = sock_net(sk)->ipv4.sysctl_tcp_tw_reuse;
119 /* Still does not detect *everything* that goes through
120 * lo, since we require a loopback src or dst address
121 * or direct binding to 'lo' interface.
123 bool loopback = false;
124 if (tw->tw_bound_dev_if == LOOPBACK_IFINDEX)
126 #if IS_ENABLED(CONFIG_IPV6)
127 if (tw->tw_family == AF_INET6) {
128 if (ipv6_addr_loopback(&tw->tw_v6_daddr) ||
129 (ipv6_addr_v4mapped(&tw->tw_v6_daddr) &&
130 (tw->tw_v6_daddr.s6_addr[12] == 127)) ||
131 ipv6_addr_loopback(&tw->tw_v6_rcv_saddr) ||
132 (ipv6_addr_v4mapped(&tw->tw_v6_rcv_saddr) &&
133 (tw->tw_v6_rcv_saddr.s6_addr[12] == 127)))
138 if (ipv4_is_loopback(tw->tw_daddr) ||
139 ipv4_is_loopback(tw->tw_rcv_saddr))
146 /* With PAWS, it is safe from the viewpoint
147 of data integrity. Even without PAWS it is safe provided sequence
148 spaces do not overlap i.e. at data rates <= 80Mbit/sec.
150 Actually, the idea is close to VJ's one, only timestamp cache is
151 held not per host, but per port pair and TW bucket is used as state
154 If TW bucket has been already destroyed we fall back to VJ's scheme
155 and use initial timestamp retrieved from peer table.
157 if (tcptw->tw_ts_recent_stamp &&
158 (!twp || (reuse && time_after32(ktime_get_seconds(),
159 tcptw->tw_ts_recent_stamp)))) {
160 /* In case of repair and re-using TIME-WAIT sockets we still
161 * want to be sure that it is safe as above but honor the
162 * sequence numbers and time stamps set as part of the repair
165 * Without this check re-using a TIME-WAIT socket with TCP
166 * repair would accumulate a -1 on the repair assigned
167 * sequence number. The first time it is reused the sequence
168 * is -1, the second time -2, etc. This fixes that issue
169 * without appearing to create any others.
171 if (likely(!tp->repair)) {
172 tp->write_seq = tcptw->tw_snd_nxt + 65535 + 2;
173 if (tp->write_seq == 0)
175 tp->rx_opt.ts_recent = tcptw->tw_ts_recent;
176 tp->rx_opt.ts_recent_stamp = tcptw->tw_ts_recent_stamp;
184 EXPORT_SYMBOL_GPL(tcp_twsk_unique);
186 static int tcp_v4_pre_connect(struct sock *sk, struct sockaddr *uaddr,
189 /* This check is replicated from tcp_v4_connect() and intended to
190 * prevent BPF program called below from accessing bytes that are out
191 * of the bound specified by user in addr_len.
193 if (addr_len < sizeof(struct sockaddr_in))
196 sock_owned_by_me(sk);
198 return BPF_CGROUP_RUN_PROG_INET4_CONNECT(sk, uaddr);
201 /* This will initiate an outgoing connection. */
202 int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)
204 struct sockaddr_in *usin = (struct sockaddr_in *)uaddr;
205 struct inet_sock *inet = inet_sk(sk);
206 struct tcp_sock *tp = tcp_sk(sk);
207 __be16 orig_sport, orig_dport;
208 __be32 daddr, nexthop;
212 struct ip_options_rcu *inet_opt;
213 struct inet_timewait_death_row *tcp_death_row = &sock_net(sk)->ipv4.tcp_death_row;
215 if (addr_len < sizeof(struct sockaddr_in))
218 if (usin->sin_family != AF_INET)
219 return -EAFNOSUPPORT;
221 nexthop = daddr = usin->sin_addr.s_addr;
222 inet_opt = rcu_dereference_protected(inet->inet_opt,
223 lockdep_sock_is_held(sk));
224 if (inet_opt && inet_opt->opt.srr) {
227 nexthop = inet_opt->opt.faddr;
230 orig_sport = inet->inet_sport;
231 orig_dport = usin->sin_port;
232 fl4 = &inet->cork.fl.u.ip4;
233 rt = ip_route_connect(fl4, nexthop, inet->inet_saddr,
234 RT_CONN_FLAGS(sk), sk->sk_bound_dev_if,
236 orig_sport, orig_dport, sk);
239 if (err == -ENETUNREACH)
240 IP_INC_STATS(sock_net(sk), IPSTATS_MIB_OUTNOROUTES);
244 if (rt->rt_flags & (RTCF_MULTICAST | RTCF_BROADCAST)) {
249 if (!inet_opt || !inet_opt->opt.srr)
252 if (!inet->inet_saddr)
253 inet->inet_saddr = fl4->saddr;
254 sk_rcv_saddr_set(sk, inet->inet_saddr);
256 if (tp->rx_opt.ts_recent_stamp && inet->inet_daddr != daddr) {
257 /* Reset inherited state */
258 tp->rx_opt.ts_recent = 0;
259 tp->rx_opt.ts_recent_stamp = 0;
260 if (likely(!tp->repair))
264 inet->inet_dport = usin->sin_port;
265 sk_daddr_set(sk, daddr);
267 inet_csk(sk)->icsk_ext_hdr_len = 0;
269 inet_csk(sk)->icsk_ext_hdr_len = inet_opt->opt.optlen;
271 tp->rx_opt.mss_clamp = TCP_MSS_DEFAULT;
273 /* Socket identity is still unknown (sport may be zero).
274 * However we set state to SYN-SENT and not releasing socket
275 * lock select source port, enter ourselves into the hash tables and
276 * complete initialization after this.
278 tcp_set_state(sk, TCP_SYN_SENT);
279 err = inet_hash_connect(tcp_death_row, sk);
285 rt = ip_route_newports(fl4, rt, orig_sport, orig_dport,
286 inet->inet_sport, inet->inet_dport, sk);
292 /* OK, now commit destination to socket. */
293 sk->sk_gso_type = SKB_GSO_TCPV4;
294 sk_setup_caps(sk, &rt->dst);
297 if (likely(!tp->repair)) {
299 tp->write_seq = secure_tcp_seq(inet->inet_saddr,
303 tp->tsoffset = secure_tcp_ts_off(sock_net(sk),
308 inet->inet_id = tp->write_seq ^ jiffies;
310 if (tcp_fastopen_defer_connect(sk, &err))
315 err = tcp_connect(sk);
324 * This unhashes the socket and releases the local port,
327 tcp_set_state(sk, TCP_CLOSE);
329 sk->sk_route_caps = 0;
330 inet->inet_dport = 0;
333 EXPORT_SYMBOL(tcp_v4_connect);
336 * This routine reacts to ICMP_FRAG_NEEDED mtu indications as defined in RFC1191.
337 * It can be called through tcp_release_cb() if socket was owned by user
338 * at the time tcp_v4_err() was called to handle ICMP message.
340 void tcp_v4_mtu_reduced(struct sock *sk)
342 struct inet_sock *inet = inet_sk(sk);
343 struct dst_entry *dst;
346 if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE))
348 mtu = tcp_sk(sk)->mtu_info;
349 dst = inet_csk_update_pmtu(sk, mtu);
353 /* Something is about to be wrong... Remember soft error
354 * for the case, if this connection will not able to recover.
356 if (mtu < dst_mtu(dst) && ip_dont_fragment(sk, dst))
357 sk->sk_err_soft = EMSGSIZE;
361 if (inet->pmtudisc != IP_PMTUDISC_DONT &&
362 ip_sk_accept_pmtu(sk) &&
363 inet_csk(sk)->icsk_pmtu_cookie > mtu) {
364 tcp_sync_mss(sk, mtu);
366 /* Resend the TCP packet because it's
367 * clear that the old packet has been
368 * dropped. This is the new "fast" path mtu
371 tcp_simple_retransmit(sk);
372 } /* else let the usual retransmit timer handle it */
374 EXPORT_SYMBOL(tcp_v4_mtu_reduced);
376 static void do_redirect(struct sk_buff *skb, struct sock *sk)
378 struct dst_entry *dst = __sk_dst_check(sk, 0);
381 dst->ops->redirect(dst, sk, skb);
385 /* handle ICMP messages on TCP_NEW_SYN_RECV request sockets */
386 void tcp_req_err(struct sock *sk, u32 seq, bool abort)
388 struct request_sock *req = inet_reqsk(sk);
389 struct net *net = sock_net(sk);
391 /* ICMPs are not backlogged, hence we cannot get
392 * an established socket here.
394 if (seq != tcp_rsk(req)->snt_isn) {
395 __NET_INC_STATS(net, LINUX_MIB_OUTOFWINDOWICMPS);
398 * Still in SYN_RECV, just remove it silently.
399 * There is no good way to pass the error to the newly
400 * created socket, and POSIX does not want network
401 * errors returned from accept().
403 inet_csk_reqsk_queue_drop(req->rsk_listener, req);
404 tcp_listendrop(req->rsk_listener);
408 EXPORT_SYMBOL(tcp_req_err);
411 * This routine is called by the ICMP module when it gets some
412 * sort of error condition. If err < 0 then the socket should
413 * be closed and the error returned to the user. If err > 0
414 * it's just the icmp type << 8 | icmp code. After adjustment
415 * header points to the first 8 bytes of the tcp header. We need
416 * to find the appropriate port.
418 * The locking strategy used here is very "optimistic". When
419 * someone else accesses the socket the ICMP is just dropped
420 * and for some paths there is no check at all.
421 * A more general error queue to queue errors for later handling
422 * is probably better.
426 int tcp_v4_err(struct sk_buff *icmp_skb, u32 info)
428 const struct iphdr *iph = (const struct iphdr *)icmp_skb->data;
429 struct tcphdr *th = (struct tcphdr *)(icmp_skb->data + (iph->ihl << 2));
430 struct inet_connection_sock *icsk;
432 struct inet_sock *inet;
433 const int type = icmp_hdr(icmp_skb)->type;
434 const int code = icmp_hdr(icmp_skb)->code;
437 struct request_sock *fastopen;
442 struct net *net = dev_net(icmp_skb->dev);
444 sk = __inet_lookup_established(net, &tcp_hashinfo, iph->daddr,
445 th->dest, iph->saddr, ntohs(th->source),
446 inet_iif(icmp_skb), 0);
448 __ICMP_INC_STATS(net, ICMP_MIB_INERRORS);
451 if (sk->sk_state == TCP_TIME_WAIT) {
452 inet_twsk_put(inet_twsk(sk));
455 seq = ntohl(th->seq);
456 if (sk->sk_state == TCP_NEW_SYN_RECV) {
457 tcp_req_err(sk, seq, type == ICMP_PARAMETERPROB ||
458 type == ICMP_TIME_EXCEEDED ||
459 (type == ICMP_DEST_UNREACH &&
460 (code == ICMP_NET_UNREACH ||
461 code == ICMP_HOST_UNREACH)));
466 /* If too many ICMPs get dropped on busy
467 * servers this needs to be solved differently.
468 * We do take care of PMTU discovery (RFC1191) special case :
469 * we can receive locally generated ICMP messages while socket is held.
471 if (sock_owned_by_user(sk)) {
472 if (!(type == ICMP_DEST_UNREACH && code == ICMP_FRAG_NEEDED))
473 __NET_INC_STATS(net, LINUX_MIB_LOCKDROPPEDICMPS);
475 if (sk->sk_state == TCP_CLOSE)
478 if (unlikely(iph->ttl < inet_sk(sk)->min_ttl)) {
479 __NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP);
485 /* XXX (TFO) - tp->snd_una should be ISN (tcp_create_openreq_child() */
486 fastopen = tp->fastopen_rsk;
487 snd_una = fastopen ? tcp_rsk(fastopen)->snt_isn : tp->snd_una;
488 if (sk->sk_state != TCP_LISTEN &&
489 !between(seq, snd_una, tp->snd_nxt)) {
490 __NET_INC_STATS(net, LINUX_MIB_OUTOFWINDOWICMPS);
496 if (!sock_owned_by_user(sk))
497 do_redirect(icmp_skb, sk);
499 case ICMP_SOURCE_QUENCH:
500 /* Just silently ignore these. */
502 case ICMP_PARAMETERPROB:
505 case ICMP_DEST_UNREACH:
506 if (code > NR_ICMP_UNREACH)
509 if (code == ICMP_FRAG_NEEDED) { /* PMTU discovery (RFC1191) */
510 /* We are not interested in TCP_LISTEN and open_requests
511 * (SYN-ACKs send out by Linux are always <576bytes so
512 * they should go through unfragmented).
514 if (sk->sk_state == TCP_LISTEN)
518 if (!sock_owned_by_user(sk)) {
519 tcp_v4_mtu_reduced(sk);
521 if (!test_and_set_bit(TCP_MTU_REDUCED_DEFERRED, &sk->sk_tsq_flags))
527 err = icmp_err_convert[code].errno;
528 /* check if icmp_skb allows revert of backoff
529 * (see draft-zimmermann-tcp-lcd) */
530 if (code != ICMP_NET_UNREACH && code != ICMP_HOST_UNREACH)
532 if (seq != tp->snd_una || !icsk->icsk_retransmits ||
533 !icsk->icsk_backoff || fastopen)
536 if (sock_owned_by_user(sk))
539 icsk->icsk_backoff--;
540 icsk->icsk_rto = tp->srtt_us ? __tcp_set_rto(tp) :
542 icsk->icsk_rto = inet_csk_rto_backoff(icsk, TCP_RTO_MAX);
544 skb = tcp_rtx_queue_head(sk);
546 tcp_mstamp_refresh(tp);
547 delta_us = (u32)(tp->tcp_mstamp - tcp_skb_timestamp_us(skb));
548 remaining = icsk->icsk_rto -
549 usecs_to_jiffies(delta_us);
552 inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
553 remaining, TCP_RTO_MAX);
555 /* RTO revert clocked out retransmission.
556 * Will retransmit now */
557 tcp_retransmit_timer(sk);
561 case ICMP_TIME_EXCEEDED:
568 switch (sk->sk_state) {
571 /* Only in fast or simultaneous open. If a fast open socket is
572 * is already accepted it is treated as a connected one below.
574 if (fastopen && !fastopen->sk)
577 if (!sock_owned_by_user(sk)) {
580 sk->sk_error_report(sk);
584 sk->sk_err_soft = err;
589 /* If we've already connected we will keep trying
590 * until we time out, or the user gives up.
592 * rfc1122 4.2.3.9 allows to consider as hard errors
593 * only PROTO_UNREACH and PORT_UNREACH (well, FRAG_FAILED too,
594 * but it is obsoleted by pmtu discovery).
596 * Note, that in modern internet, where routing is unreliable
597 * and in each dark corner broken firewalls sit, sending random
598 * errors ordered by their masters even this two messages finally lose
599 * their original sense (even Linux sends invalid PORT_UNREACHs)
601 * Now we are in compliance with RFCs.
606 if (!sock_owned_by_user(sk) && inet->recverr) {
608 sk->sk_error_report(sk);
609 } else { /* Only an error on timeout */
610 sk->sk_err_soft = err;
619 void __tcp_v4_send_check(struct sk_buff *skb, __be32 saddr, __be32 daddr)
621 struct tcphdr *th = tcp_hdr(skb);
623 th->check = ~tcp_v4_check(skb->len, saddr, daddr, 0);
624 skb->csum_start = skb_transport_header(skb) - skb->head;
625 skb->csum_offset = offsetof(struct tcphdr, check);
628 /* This routine computes an IPv4 TCP checksum. */
629 void tcp_v4_send_check(struct sock *sk, struct sk_buff *skb)
631 const struct inet_sock *inet = inet_sk(sk);
633 __tcp_v4_send_check(skb, inet->inet_saddr, inet->inet_daddr);
635 EXPORT_SYMBOL(tcp_v4_send_check);
638 * This routine will send an RST to the other tcp.
640 * Someone asks: why I NEVER use socket parameters (TOS, TTL etc.)
642 * Answer: if a packet caused RST, it is not for a socket
643 * existing in our system, if it is matched to a socket,
644 * it is just duplicate segment or bug in other side's TCP.
645 * So that we build reply only basing on parameters
646 * arrived with segment.
647 * Exception: precedence violation. We do not implement it in any case.
650 static void tcp_v4_send_reset(const struct sock *sk, struct sk_buff *skb)
652 const struct tcphdr *th = tcp_hdr(skb);
655 #ifdef CONFIG_TCP_MD5SIG
656 __be32 opt[(TCPOLEN_MD5SIG_ALIGNED >> 2)];
659 struct ip_reply_arg arg;
660 #ifdef CONFIG_TCP_MD5SIG
661 struct tcp_md5sig_key *key = NULL;
662 const __u8 *hash_location = NULL;
663 unsigned char newhash[16];
665 struct sock *sk1 = NULL;
670 /* Never send a reset in response to a reset. */
674 /* If sk not NULL, it means we did a successful lookup and incoming
675 * route had to be correct. prequeue might have dropped our dst.
677 if (!sk && skb_rtable(skb)->rt_type != RTN_LOCAL)
680 /* Swap the send and the receive. */
681 memset(&rep, 0, sizeof(rep));
682 rep.th.dest = th->source;
683 rep.th.source = th->dest;
684 rep.th.doff = sizeof(struct tcphdr) / 4;
688 rep.th.seq = th->ack_seq;
691 rep.th.ack_seq = htonl(ntohl(th->seq) + th->syn + th->fin +
692 skb->len - (th->doff << 2));
695 memset(&arg, 0, sizeof(arg));
696 arg.iov[0].iov_base = (unsigned char *)&rep;
697 arg.iov[0].iov_len = sizeof(rep.th);
699 net = sk ? sock_net(sk) : dev_net(skb_dst(skb)->dev);
700 #ifdef CONFIG_TCP_MD5SIG
702 hash_location = tcp_parse_md5sig_option(th);
703 if (sk && sk_fullsock(sk)) {
704 key = tcp_md5_do_lookup(sk, (union tcp_md5_addr *)
705 &ip_hdr(skb)->saddr, AF_INET);
706 } else if (hash_location) {
708 * active side is lost. Try to find listening socket through
709 * source port, and then find md5 key through listening socket.
710 * we are not loose security here:
711 * Incoming packet is checked with md5 hash with finding key,
712 * no RST generated if md5 hash doesn't match.
714 sk1 = __inet_lookup_listener(net, &tcp_hashinfo, NULL, 0,
716 th->source, ip_hdr(skb)->daddr,
717 ntohs(th->source), inet_iif(skb),
719 /* don't send rst if it can't find key */
723 key = tcp_md5_do_lookup(sk1, (union tcp_md5_addr *)
724 &ip_hdr(skb)->saddr, AF_INET);
729 genhash = tcp_v4_md5_hash_skb(newhash, key, NULL, skb);
730 if (genhash || memcmp(hash_location, newhash, 16) != 0)
736 rep.opt[0] = htonl((TCPOPT_NOP << 24) |
738 (TCPOPT_MD5SIG << 8) |
740 /* Update length and the length the header thinks exists */
741 arg.iov[0].iov_len += TCPOLEN_MD5SIG_ALIGNED;
742 rep.th.doff = arg.iov[0].iov_len / 4;
744 tcp_v4_md5_hash_hdr((__u8 *) &rep.opt[1],
745 key, ip_hdr(skb)->saddr,
746 ip_hdr(skb)->daddr, &rep.th);
749 arg.csum = csum_tcpudp_nofold(ip_hdr(skb)->daddr,
750 ip_hdr(skb)->saddr, /* XXX */
751 arg.iov[0].iov_len, IPPROTO_TCP, 0);
752 arg.csumoffset = offsetof(struct tcphdr, check) / 2;
753 arg.flags = (sk && inet_sk_transparent(sk)) ? IP_REPLY_ARG_NOSRCCHECK : 0;
755 /* When socket is gone, all binding information is lost.
756 * routing might fail in this case. No choice here, if we choose to force
757 * input interface, we will misroute in case of asymmetric route.
760 arg.bound_dev_if = sk->sk_bound_dev_if;
762 trace_tcp_send_reset(sk, skb);
765 BUILD_BUG_ON(offsetof(struct sock, sk_bound_dev_if) !=
766 offsetof(struct inet_timewait_sock, tw_bound_dev_if));
768 arg.tos = ip_hdr(skb)->tos;
769 arg.uid = sock_net_uid(net, sk && sk_fullsock(sk) ? sk : NULL);
771 ctl_sk = *this_cpu_ptr(net->ipv4.tcp_sk);
773 ctl_sk->sk_mark = (sk->sk_state == TCP_TIME_WAIT) ?
774 inet_twsk(sk)->tw_mark : sk->sk_mark;
775 ip_send_unicast_reply(ctl_sk,
776 skb, &TCP_SKB_CB(skb)->header.h4.opt,
777 ip_hdr(skb)->saddr, ip_hdr(skb)->daddr,
778 &arg, arg.iov[0].iov_len);
781 __TCP_INC_STATS(net, TCP_MIB_OUTSEGS);
782 __TCP_INC_STATS(net, TCP_MIB_OUTRSTS);
785 #ifdef CONFIG_TCP_MD5SIG
791 /* The code following below sending ACKs in SYN-RECV and TIME-WAIT states
792 outside socket context is ugly, certainly. What can I do?
795 static void tcp_v4_send_ack(const struct sock *sk,
796 struct sk_buff *skb, u32 seq, u32 ack,
797 u32 win, u32 tsval, u32 tsecr, int oif,
798 struct tcp_md5sig_key *key,
799 int reply_flags, u8 tos)
801 const struct tcphdr *th = tcp_hdr(skb);
804 __be32 opt[(TCPOLEN_TSTAMP_ALIGNED >> 2)
805 #ifdef CONFIG_TCP_MD5SIG
806 + (TCPOLEN_MD5SIG_ALIGNED >> 2)
810 struct net *net = sock_net(sk);
811 struct ip_reply_arg arg;
814 memset(&rep.th, 0, sizeof(struct tcphdr));
815 memset(&arg, 0, sizeof(arg));
817 arg.iov[0].iov_base = (unsigned char *)&rep;
818 arg.iov[0].iov_len = sizeof(rep.th);
820 rep.opt[0] = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) |
821 (TCPOPT_TIMESTAMP << 8) |
823 rep.opt[1] = htonl(tsval);
824 rep.opt[2] = htonl(tsecr);
825 arg.iov[0].iov_len += TCPOLEN_TSTAMP_ALIGNED;
828 /* Swap the send and the receive. */
829 rep.th.dest = th->source;
830 rep.th.source = th->dest;
831 rep.th.doff = arg.iov[0].iov_len / 4;
832 rep.th.seq = htonl(seq);
833 rep.th.ack_seq = htonl(ack);
835 rep.th.window = htons(win);
837 #ifdef CONFIG_TCP_MD5SIG
839 int offset = (tsecr) ? 3 : 0;
841 rep.opt[offset++] = htonl((TCPOPT_NOP << 24) |
843 (TCPOPT_MD5SIG << 8) |
845 arg.iov[0].iov_len += TCPOLEN_MD5SIG_ALIGNED;
846 rep.th.doff = arg.iov[0].iov_len/4;
848 tcp_v4_md5_hash_hdr((__u8 *) &rep.opt[offset],
849 key, ip_hdr(skb)->saddr,
850 ip_hdr(skb)->daddr, &rep.th);
853 arg.flags = reply_flags;
854 arg.csum = csum_tcpudp_nofold(ip_hdr(skb)->daddr,
855 ip_hdr(skb)->saddr, /* XXX */
856 arg.iov[0].iov_len, IPPROTO_TCP, 0);
857 arg.csumoffset = offsetof(struct tcphdr, check) / 2;
859 arg.bound_dev_if = oif;
861 arg.uid = sock_net_uid(net, sk_fullsock(sk) ? sk : NULL);
863 ctl_sk = *this_cpu_ptr(net->ipv4.tcp_sk);
865 ctl_sk->sk_mark = (sk->sk_state == TCP_TIME_WAIT) ?
866 inet_twsk(sk)->tw_mark : sk->sk_mark;
867 ip_send_unicast_reply(ctl_sk,
868 skb, &TCP_SKB_CB(skb)->header.h4.opt,
869 ip_hdr(skb)->saddr, ip_hdr(skb)->daddr,
870 &arg, arg.iov[0].iov_len);
873 __TCP_INC_STATS(net, TCP_MIB_OUTSEGS);
877 static void tcp_v4_timewait_ack(struct sock *sk, struct sk_buff *skb)
879 struct inet_timewait_sock *tw = inet_twsk(sk);
880 struct tcp_timewait_sock *tcptw = tcp_twsk(sk);
882 tcp_v4_send_ack(sk, skb,
883 tcptw->tw_snd_nxt, tcptw->tw_rcv_nxt,
884 tcptw->tw_rcv_wnd >> tw->tw_rcv_wscale,
885 tcp_time_stamp_raw() + tcptw->tw_ts_offset,
888 tcp_twsk_md5_key(tcptw),
889 tw->tw_transparent ? IP_REPLY_ARG_NOSRCCHECK : 0,
896 static void tcp_v4_reqsk_send_ack(const struct sock *sk, struct sk_buff *skb,
897 struct request_sock *req)
899 /* sk->sk_state == TCP_LISTEN -> for regular TCP_SYN_RECV
900 * sk->sk_state == TCP_SYN_RECV -> for Fast Open.
902 u32 seq = (sk->sk_state == TCP_LISTEN) ? tcp_rsk(req)->snt_isn + 1 :
906 * The window field (SEG.WND) of every outgoing segment, with the
907 * exception of <SYN> segments, MUST be right-shifted by
908 * Rcv.Wind.Shift bits:
910 tcp_v4_send_ack(sk, skb, seq,
911 tcp_rsk(req)->rcv_nxt,
912 req->rsk_rcv_wnd >> inet_rsk(req)->rcv_wscale,
913 tcp_time_stamp_raw() + tcp_rsk(req)->ts_off,
916 tcp_md5_do_lookup(sk, (union tcp_md5_addr *)&ip_hdr(skb)->saddr,
918 inet_rsk(req)->no_srccheck ? IP_REPLY_ARG_NOSRCCHECK : 0,
923 * Send a SYN-ACK after having received a SYN.
924 * This still operates on a request_sock only, not on a big
927 static int tcp_v4_send_synack(const struct sock *sk, struct dst_entry *dst,
929 struct request_sock *req,
930 struct tcp_fastopen_cookie *foc,
931 enum tcp_synack_type synack_type)
933 const struct inet_request_sock *ireq = inet_rsk(req);
938 /* First, grab a route. */
939 if (!dst && (dst = inet_csk_route_req(sk, &fl4, req)) == NULL)
942 skb = tcp_make_synack(sk, dst, req, foc, synack_type);
945 __tcp_v4_send_check(skb, ireq->ir_loc_addr, ireq->ir_rmt_addr);
948 err = ip_build_and_send_pkt(skb, sk, ireq->ir_loc_addr,
950 rcu_dereference(ireq->ireq_opt));
952 err = net_xmit_eval(err);
959 * IPv4 request_sock destructor.
961 static void tcp_v4_reqsk_destructor(struct request_sock *req)
963 kfree(rcu_dereference_protected(inet_rsk(req)->ireq_opt, 1));
966 #ifdef CONFIG_TCP_MD5SIG
968 * RFC2385 MD5 checksumming requires a mapping of
969 * IP address->MD5 Key.
970 * We need to maintain these in the sk structure.
973 struct static_key tcp_md5_needed __read_mostly;
974 EXPORT_SYMBOL(tcp_md5_needed);
976 /* Find the Key structure for an address. */
977 struct tcp_md5sig_key *__tcp_md5_do_lookup(const struct sock *sk,
978 const union tcp_md5_addr *addr,
981 const struct tcp_sock *tp = tcp_sk(sk);
982 struct tcp_md5sig_key *key;
983 const struct tcp_md5sig_info *md5sig;
985 struct tcp_md5sig_key *best_match = NULL;
988 /* caller either holds rcu_read_lock() or socket lock */
989 md5sig = rcu_dereference_check(tp->md5sig_info,
990 lockdep_sock_is_held(sk));
994 hlist_for_each_entry_rcu(key, &md5sig->head, node) {
995 if (key->family != family)
998 if (family == AF_INET) {
999 mask = inet_make_mask(key->prefixlen);
1000 match = (key->addr.a4.s_addr & mask) ==
1001 (addr->a4.s_addr & mask);
1002 #if IS_ENABLED(CONFIG_IPV6)
1003 } else if (family == AF_INET6) {
1004 match = ipv6_prefix_equal(&key->addr.a6, &addr->a6,
1011 if (match && (!best_match ||
1012 key->prefixlen > best_match->prefixlen))
1017 EXPORT_SYMBOL(__tcp_md5_do_lookup);
1019 static struct tcp_md5sig_key *tcp_md5_do_lookup_exact(const struct sock *sk,
1020 const union tcp_md5_addr *addr,
1021 int family, u8 prefixlen)
1023 const struct tcp_sock *tp = tcp_sk(sk);
1024 struct tcp_md5sig_key *key;
1025 unsigned int size = sizeof(struct in_addr);
1026 const struct tcp_md5sig_info *md5sig;
1028 /* caller either holds rcu_read_lock() or socket lock */
1029 md5sig = rcu_dereference_check(tp->md5sig_info,
1030 lockdep_sock_is_held(sk));
1033 #if IS_ENABLED(CONFIG_IPV6)
1034 if (family == AF_INET6)
1035 size = sizeof(struct in6_addr);
1037 hlist_for_each_entry_rcu(key, &md5sig->head, node) {
1038 if (key->family != family)
1040 if (!memcmp(&key->addr, addr, size) &&
1041 key->prefixlen == prefixlen)
1047 struct tcp_md5sig_key *tcp_v4_md5_lookup(const struct sock *sk,
1048 const struct sock *addr_sk)
1050 const union tcp_md5_addr *addr;
1052 addr = (const union tcp_md5_addr *)&addr_sk->sk_daddr;
1053 return tcp_md5_do_lookup(sk, addr, AF_INET);
1055 EXPORT_SYMBOL(tcp_v4_md5_lookup);
1057 /* This can be called on a newly created socket, from other files */
1058 int tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr,
1059 int family, u8 prefixlen, const u8 *newkey, u8 newkeylen,
1062 /* Add Key to the list */
1063 struct tcp_md5sig_key *key;
1064 struct tcp_sock *tp = tcp_sk(sk);
1065 struct tcp_md5sig_info *md5sig;
1067 key = tcp_md5_do_lookup_exact(sk, addr, family, prefixlen);
1069 /* Pre-existing entry - just update that one. */
1070 memcpy(key->key, newkey, newkeylen);
1071 key->keylen = newkeylen;
1075 md5sig = rcu_dereference_protected(tp->md5sig_info,
1076 lockdep_sock_is_held(sk));
1078 md5sig = kmalloc(sizeof(*md5sig), gfp);
1082 sk_nocaps_add(sk, NETIF_F_GSO_MASK);
1083 INIT_HLIST_HEAD(&md5sig->head);
1084 rcu_assign_pointer(tp->md5sig_info, md5sig);
1087 key = sock_kmalloc(sk, sizeof(*key), gfp);
1090 if (!tcp_alloc_md5sig_pool()) {
1091 sock_kfree_s(sk, key, sizeof(*key));
1095 memcpy(key->key, newkey, newkeylen);
1096 key->keylen = newkeylen;
1097 key->family = family;
1098 key->prefixlen = prefixlen;
1099 memcpy(&key->addr, addr,
1100 (family == AF_INET6) ? sizeof(struct in6_addr) :
1101 sizeof(struct in_addr));
1102 hlist_add_head_rcu(&key->node, &md5sig->head);
1105 EXPORT_SYMBOL(tcp_md5_do_add);
1107 int tcp_md5_do_del(struct sock *sk, const union tcp_md5_addr *addr, int family,
1110 struct tcp_md5sig_key *key;
1112 key = tcp_md5_do_lookup_exact(sk, addr, family, prefixlen);
1115 hlist_del_rcu(&key->node);
1116 atomic_sub(sizeof(*key), &sk->sk_omem_alloc);
1117 kfree_rcu(key, rcu);
1120 EXPORT_SYMBOL(tcp_md5_do_del);
1122 static void tcp_clear_md5_list(struct sock *sk)
1124 struct tcp_sock *tp = tcp_sk(sk);
1125 struct tcp_md5sig_key *key;
1126 struct hlist_node *n;
1127 struct tcp_md5sig_info *md5sig;
1129 md5sig = rcu_dereference_protected(tp->md5sig_info, 1);
1131 hlist_for_each_entry_safe(key, n, &md5sig->head, node) {
1132 hlist_del_rcu(&key->node);
1133 atomic_sub(sizeof(*key), &sk->sk_omem_alloc);
1134 kfree_rcu(key, rcu);
1138 static int tcp_v4_parse_md5_keys(struct sock *sk, int optname,
1139 char __user *optval, int optlen)
1141 struct tcp_md5sig cmd;
1142 struct sockaddr_in *sin = (struct sockaddr_in *)&cmd.tcpm_addr;
1145 if (optlen < sizeof(cmd))
1148 if (copy_from_user(&cmd, optval, sizeof(cmd)))
1151 if (sin->sin_family != AF_INET)
1154 if (optname == TCP_MD5SIG_EXT &&
1155 cmd.tcpm_flags & TCP_MD5SIG_FLAG_PREFIX) {
1156 prefixlen = cmd.tcpm_prefixlen;
1161 if (!cmd.tcpm_keylen)
1162 return tcp_md5_do_del(sk, (union tcp_md5_addr *)&sin->sin_addr.s_addr,
1163 AF_INET, prefixlen);
1165 if (cmd.tcpm_keylen > TCP_MD5SIG_MAXKEYLEN)
1168 return tcp_md5_do_add(sk, (union tcp_md5_addr *)&sin->sin_addr.s_addr,
1169 AF_INET, prefixlen, cmd.tcpm_key, cmd.tcpm_keylen,
1173 static int tcp_v4_md5_hash_headers(struct tcp_md5sig_pool *hp,
1174 __be32 daddr, __be32 saddr,
1175 const struct tcphdr *th, int nbytes)
1177 struct tcp4_pseudohdr *bp;
1178 struct scatterlist sg;
1185 bp->protocol = IPPROTO_TCP;
1186 bp->len = cpu_to_be16(nbytes);
1188 _th = (struct tcphdr *)(bp + 1);
1189 memcpy(_th, th, sizeof(*th));
1192 sg_init_one(&sg, bp, sizeof(*bp) + sizeof(*th));
1193 ahash_request_set_crypt(hp->md5_req, &sg, NULL,
1194 sizeof(*bp) + sizeof(*th));
1195 return crypto_ahash_update(hp->md5_req);
1198 static int tcp_v4_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key *key,
1199 __be32 daddr, __be32 saddr, const struct tcphdr *th)
1201 struct tcp_md5sig_pool *hp;
1202 struct ahash_request *req;
1204 hp = tcp_get_md5sig_pool();
1206 goto clear_hash_noput;
1209 if (crypto_ahash_init(req))
1211 if (tcp_v4_md5_hash_headers(hp, daddr, saddr, th, th->doff << 2))
1213 if (tcp_md5_hash_key(hp, key))
1215 ahash_request_set_crypt(req, NULL, md5_hash, 0);
1216 if (crypto_ahash_final(req))
1219 tcp_put_md5sig_pool();
1223 tcp_put_md5sig_pool();
1225 memset(md5_hash, 0, 16);
1229 int tcp_v4_md5_hash_skb(char *md5_hash, const struct tcp_md5sig_key *key,
1230 const struct sock *sk,
1231 const struct sk_buff *skb)
1233 struct tcp_md5sig_pool *hp;
1234 struct ahash_request *req;
1235 const struct tcphdr *th = tcp_hdr(skb);
1236 __be32 saddr, daddr;
1238 if (sk) { /* valid for establish/request sockets */
1239 saddr = sk->sk_rcv_saddr;
1240 daddr = sk->sk_daddr;
1242 const struct iphdr *iph = ip_hdr(skb);
1247 hp = tcp_get_md5sig_pool();
1249 goto clear_hash_noput;
1252 if (crypto_ahash_init(req))
1255 if (tcp_v4_md5_hash_headers(hp, daddr, saddr, th, skb->len))
1257 if (tcp_md5_hash_skb_data(hp, skb, th->doff << 2))
1259 if (tcp_md5_hash_key(hp, key))
1261 ahash_request_set_crypt(req, NULL, md5_hash, 0);
1262 if (crypto_ahash_final(req))
1265 tcp_put_md5sig_pool();
1269 tcp_put_md5sig_pool();
1271 memset(md5_hash, 0, 16);
1274 EXPORT_SYMBOL(tcp_v4_md5_hash_skb);
1278 /* Called with rcu_read_lock() */
1279 static bool tcp_v4_inbound_md5_hash(const struct sock *sk,
1280 const struct sk_buff *skb)
1282 #ifdef CONFIG_TCP_MD5SIG
1284 * This gets called for each TCP segment that arrives
1285 * so we want to be efficient.
1286 * We have 3 drop cases:
1287 * o No MD5 hash and one expected.
1288 * o MD5 hash and we're not expecting one.
1289 * o MD5 hash and its wrong.
1291 const __u8 *hash_location = NULL;
1292 struct tcp_md5sig_key *hash_expected;
1293 const struct iphdr *iph = ip_hdr(skb);
1294 const struct tcphdr *th = tcp_hdr(skb);
1296 unsigned char newhash[16];
1298 hash_expected = tcp_md5_do_lookup(sk, (union tcp_md5_addr *)&iph->saddr,
1300 hash_location = tcp_parse_md5sig_option(th);
1302 /* We've parsed the options - do we have a hash? */
1303 if (!hash_expected && !hash_location)
1306 if (hash_expected && !hash_location) {
1307 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMD5NOTFOUND);
1311 if (!hash_expected && hash_location) {
1312 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMD5UNEXPECTED);
1316 /* Okay, so this is hash_expected and hash_location -
1317 * so we need to calculate the checksum.
1319 genhash = tcp_v4_md5_hash_skb(newhash,
1323 if (genhash || memcmp(hash_location, newhash, 16) != 0) {
1324 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMD5FAILURE);
1325 net_info_ratelimited("MD5 Hash failed for (%pI4, %d)->(%pI4, %d)%s\n",
1326 &iph->saddr, ntohs(th->source),
1327 &iph->daddr, ntohs(th->dest),
1328 genhash ? " tcp_v4_calc_md5_hash failed"
1337 static void tcp_v4_init_req(struct request_sock *req,
1338 const struct sock *sk_listener,
1339 struct sk_buff *skb)
1341 struct inet_request_sock *ireq = inet_rsk(req);
1342 struct net *net = sock_net(sk_listener);
1344 sk_rcv_saddr_set(req_to_sk(req), ip_hdr(skb)->daddr);
1345 sk_daddr_set(req_to_sk(req), ip_hdr(skb)->saddr);
1346 RCU_INIT_POINTER(ireq->ireq_opt, tcp_v4_save_options(net, skb));
1349 static struct dst_entry *tcp_v4_route_req(const struct sock *sk,
1351 const struct request_sock *req)
1353 return inet_csk_route_req(sk, &fl->u.ip4, req);
1356 struct request_sock_ops tcp_request_sock_ops __read_mostly = {
1358 .obj_size = sizeof(struct tcp_request_sock),
1359 .rtx_syn_ack = tcp_rtx_synack,
1360 .send_ack = tcp_v4_reqsk_send_ack,
1361 .destructor = tcp_v4_reqsk_destructor,
1362 .send_reset = tcp_v4_send_reset,
1363 .syn_ack_timeout = tcp_syn_ack_timeout,
1366 static const struct tcp_request_sock_ops tcp_request_sock_ipv4_ops = {
1367 .mss_clamp = TCP_MSS_DEFAULT,
1368 #ifdef CONFIG_TCP_MD5SIG
1369 .req_md5_lookup = tcp_v4_md5_lookup,
1370 .calc_md5_hash = tcp_v4_md5_hash_skb,
1372 .init_req = tcp_v4_init_req,
1373 #ifdef CONFIG_SYN_COOKIES
1374 .cookie_init_seq = cookie_v4_init_sequence,
1376 .route_req = tcp_v4_route_req,
1377 .init_seq = tcp_v4_init_seq,
1378 .init_ts_off = tcp_v4_init_ts_off,
1379 .send_synack = tcp_v4_send_synack,
1382 int tcp_v4_conn_request(struct sock *sk, struct sk_buff *skb)
1384 /* Never answer to SYNs send to broadcast or multicast */
1385 if (skb_rtable(skb)->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST))
1388 return tcp_conn_request(&tcp_request_sock_ops,
1389 &tcp_request_sock_ipv4_ops, sk, skb);
1395 EXPORT_SYMBOL(tcp_v4_conn_request);
1399 * The three way handshake has completed - we got a valid synack -
1400 * now create the new socket.
1402 struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb,
1403 struct request_sock *req,
1404 struct dst_entry *dst,
1405 struct request_sock *req_unhash,
1408 struct inet_request_sock *ireq;
1409 struct inet_sock *newinet;
1410 struct tcp_sock *newtp;
1412 #ifdef CONFIG_TCP_MD5SIG
1413 struct tcp_md5sig_key *key;
1415 struct ip_options_rcu *inet_opt;
1417 if (sk_acceptq_is_full(sk))
1420 newsk = tcp_create_openreq_child(sk, req, skb);
1424 newsk->sk_gso_type = SKB_GSO_TCPV4;
1425 inet_sk_rx_dst_set(newsk, skb);
1427 newtp = tcp_sk(newsk);
1428 newinet = inet_sk(newsk);
1429 ireq = inet_rsk(req);
1430 sk_daddr_set(newsk, ireq->ir_rmt_addr);
1431 sk_rcv_saddr_set(newsk, ireq->ir_loc_addr);
1432 newsk->sk_bound_dev_if = ireq->ir_iif;
1433 newinet->inet_saddr = ireq->ir_loc_addr;
1434 inet_opt = rcu_dereference(ireq->ireq_opt);
1435 RCU_INIT_POINTER(newinet->inet_opt, inet_opt);
1436 newinet->mc_index = inet_iif(skb);
1437 newinet->mc_ttl = ip_hdr(skb)->ttl;
1438 newinet->rcv_tos = ip_hdr(skb)->tos;
1439 inet_csk(newsk)->icsk_ext_hdr_len = 0;
1441 inet_csk(newsk)->icsk_ext_hdr_len = inet_opt->opt.optlen;
1442 newinet->inet_id = newtp->write_seq ^ jiffies;
1445 dst = inet_csk_route_child_sock(sk, newsk, req);
1449 /* syncookie case : see end of cookie_v4_check() */
1451 sk_setup_caps(newsk, dst);
1453 tcp_ca_openreq_child(newsk, dst);
1455 tcp_sync_mss(newsk, dst_mtu(dst));
1456 newtp->advmss = tcp_mss_clamp(tcp_sk(sk), dst_metric_advmss(dst));
1458 tcp_initialize_rcv_mss(newsk);
1460 #ifdef CONFIG_TCP_MD5SIG
1461 /* Copy over the MD5 key from the original socket */
1462 key = tcp_md5_do_lookup(sk, (union tcp_md5_addr *)&newinet->inet_daddr,
1466 * We're using one, so create a matching key
1467 * on the newsk structure. If we fail to get
1468 * memory, then we end up not copying the key
1471 tcp_md5_do_add(newsk, (union tcp_md5_addr *)&newinet->inet_daddr,
1472 AF_INET, 32, key->key, key->keylen, GFP_ATOMIC);
1473 sk_nocaps_add(newsk, NETIF_F_GSO_MASK);
1477 if (__inet_inherit_port(sk, newsk) < 0)
1479 *own_req = inet_ehash_nolisten(newsk, req_to_sk(req_unhash));
1480 if (likely(*own_req)) {
1481 tcp_move_syn(newtp, req);
1482 ireq->ireq_opt = NULL;
1484 newinet->inet_opt = NULL;
1489 NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENOVERFLOWS);
1496 newinet->inet_opt = NULL;
1497 inet_csk_prepare_forced_close(newsk);
1501 EXPORT_SYMBOL(tcp_v4_syn_recv_sock);
1503 static struct sock *tcp_v4_cookie_check(struct sock *sk, struct sk_buff *skb)
1505 #ifdef CONFIG_SYN_COOKIES
1506 const struct tcphdr *th = tcp_hdr(skb);
1509 sk = cookie_v4_check(sk, skb);
1514 /* The socket must have it's spinlock held when we get
1515 * here, unless it is a TCP_LISTEN socket.
1517 * We have a potential double-lock case here, so even when
1518 * doing backlog processing we use the BH locking scheme.
1519 * This is because we cannot sleep with the original spinlock
1522 int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb)
1526 if (sk->sk_state == TCP_ESTABLISHED) { /* Fast path */
1527 struct dst_entry *dst = sk->sk_rx_dst;
1529 sock_rps_save_rxhash(sk, skb);
1530 sk_mark_napi_id(sk, skb);
1532 if (inet_sk(sk)->rx_dst_ifindex != skb->skb_iif ||
1533 !dst->ops->check(dst, 0)) {
1535 sk->sk_rx_dst = NULL;
1538 tcp_rcv_established(sk, skb);
1542 if (tcp_checksum_complete(skb))
1545 if (sk->sk_state == TCP_LISTEN) {
1546 struct sock *nsk = tcp_v4_cookie_check(sk, skb);
1551 if (tcp_child_process(sk, nsk, skb)) {
1558 sock_rps_save_rxhash(sk, skb);
1560 if (tcp_rcv_state_process(sk, skb)) {
1567 tcp_v4_send_reset(rsk, skb);
1570 /* Be careful here. If this function gets more complicated and
1571 * gcc suffers from register pressure on the x86, sk (in %ebx)
1572 * might be destroyed here. This current version compiles correctly,
1573 * but you have been warned.
1578 TCP_INC_STATS(sock_net(sk), TCP_MIB_CSUMERRORS);
1579 TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
1582 EXPORT_SYMBOL(tcp_v4_do_rcv);
1584 int tcp_v4_early_demux(struct sk_buff *skb)
1586 const struct iphdr *iph;
1587 const struct tcphdr *th;
1590 if (skb->pkt_type != PACKET_HOST)
1593 if (!pskb_may_pull(skb, skb_transport_offset(skb) + sizeof(struct tcphdr)))
1599 if (th->doff < sizeof(struct tcphdr) / 4)
1602 sk = __inet_lookup_established(dev_net(skb->dev), &tcp_hashinfo,
1603 iph->saddr, th->source,
1604 iph->daddr, ntohs(th->dest),
1605 skb->skb_iif, inet_sdif(skb));
1608 skb->destructor = sock_edemux;
1609 if (sk_fullsock(sk)) {
1610 struct dst_entry *dst = READ_ONCE(sk->sk_rx_dst);
1613 dst = dst_check(dst, 0);
1615 inet_sk(sk)->rx_dst_ifindex == skb->skb_iif)
1616 skb_dst_set_noref(skb, dst);
1622 bool tcp_add_backlog(struct sock *sk, struct sk_buff *skb)
1624 u32 limit = sk->sk_rcvbuf + sk->sk_sndbuf;
1625 struct skb_shared_info *shinfo;
1626 const struct tcphdr *th;
1627 struct tcphdr *thtail;
1628 struct sk_buff *tail;
1629 unsigned int hdrlen;
1634 /* In case all data was pulled from skb frags (in __pskb_pull_tail()),
1635 * we can fix skb->truesize to its real value to avoid future drops.
1636 * This is valid because skb is not yet charged to the socket.
1637 * It has been noticed pure SACK packets were sometimes dropped
1638 * (if cooked by drivers without copybreak feature).
1644 if (unlikely(tcp_checksum_complete(skb))) {
1646 __TCP_INC_STATS(sock_net(sk), TCP_MIB_CSUMERRORS);
1647 __TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
1651 /* Attempt coalescing to last skb in backlog, even if we are
1653 * This is okay because skb capacity is limited to MAX_SKB_FRAGS.
1655 th = (const struct tcphdr *)skb->data;
1656 hdrlen = th->doff * 4;
1657 shinfo = skb_shinfo(skb);
1659 if (!shinfo->gso_size)
1660 shinfo->gso_size = skb->len - hdrlen;
1662 if (!shinfo->gso_segs)
1663 shinfo->gso_segs = 1;
1665 tail = sk->sk_backlog.tail;
1668 thtail = (struct tcphdr *)tail->data;
1670 if (TCP_SKB_CB(tail)->end_seq != TCP_SKB_CB(skb)->seq ||
1671 TCP_SKB_CB(tail)->ip_dsfield != TCP_SKB_CB(skb)->ip_dsfield ||
1672 ((TCP_SKB_CB(tail)->tcp_flags |
1673 TCP_SKB_CB(skb)->tcp_flags) & TCPHDR_URG) ||
1674 ((TCP_SKB_CB(tail)->tcp_flags ^
1675 TCP_SKB_CB(skb)->tcp_flags) & (TCPHDR_ECE | TCPHDR_CWR)) ||
1676 #ifdef CONFIG_TLS_DEVICE
1677 tail->decrypted != skb->decrypted ||
1679 thtail->doff != th->doff ||
1680 memcmp(thtail + 1, th + 1, hdrlen - sizeof(*th)))
1683 __skb_pull(skb, hdrlen);
1684 if (skb_try_coalesce(tail, skb, &fragstolen, &delta)) {
1685 thtail->window = th->window;
1687 TCP_SKB_CB(tail)->end_seq = TCP_SKB_CB(skb)->end_seq;
1689 if (after(TCP_SKB_CB(skb)->ack_seq, TCP_SKB_CB(tail)->ack_seq))
1690 TCP_SKB_CB(tail)->ack_seq = TCP_SKB_CB(skb)->ack_seq;
1692 TCP_SKB_CB(tail)->tcp_flags |= TCP_SKB_CB(skb)->tcp_flags;
1694 if (TCP_SKB_CB(skb)->has_rxtstamp) {
1695 TCP_SKB_CB(tail)->has_rxtstamp = true;
1696 tail->tstamp = skb->tstamp;
1697 skb_hwtstamps(tail)->hwtstamp = skb_hwtstamps(skb)->hwtstamp;
1700 /* Not as strict as GRO. We only need to carry mss max value */
1701 skb_shinfo(tail)->gso_size = max(shinfo->gso_size,
1702 skb_shinfo(tail)->gso_size);
1704 gso_segs = skb_shinfo(tail)->gso_segs + shinfo->gso_segs;
1705 skb_shinfo(tail)->gso_segs = min_t(u32, gso_segs, 0xFFFF);
1707 sk->sk_backlog.len += delta;
1708 __NET_INC_STATS(sock_net(sk),
1709 LINUX_MIB_TCPBACKLOGCOALESCE);
1710 kfree_skb_partial(skb, fragstolen);
1713 __skb_push(skb, hdrlen);
1716 /* Only socket owner can try to collapse/prune rx queues
1717 * to reduce memory overhead, so add a little headroom here.
1718 * Few sockets backlog are possibly concurrently non empty.
1722 if (unlikely(sk_add_backlog(sk, skb, limit))) {
1724 __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPBACKLOGDROP);
1729 EXPORT_SYMBOL(tcp_add_backlog);
1731 int tcp_filter(struct sock *sk, struct sk_buff *skb)
1733 struct tcphdr *th = (struct tcphdr *)skb->data;
1734 unsigned int eaten = skb->len;
1737 err = sk_filter_trim_cap(sk, skb, th->doff * 4);
1740 TCP_SKB_CB(skb)->end_seq -= eaten;
1744 EXPORT_SYMBOL(tcp_filter);
1746 static void tcp_v4_restore_cb(struct sk_buff *skb)
1748 memmove(IPCB(skb), &TCP_SKB_CB(skb)->header.h4,
1749 sizeof(struct inet_skb_parm));
1752 static void tcp_v4_fill_cb(struct sk_buff *skb, const struct iphdr *iph,
1753 const struct tcphdr *th)
1755 /* This is tricky : We move IPCB at its correct location into TCP_SKB_CB()
1756 * barrier() makes sure compiler wont play fool^Waliasing games.
1758 memmove(&TCP_SKB_CB(skb)->header.h4, IPCB(skb),
1759 sizeof(struct inet_skb_parm));
1762 TCP_SKB_CB(skb)->seq = ntohl(th->seq);
1763 TCP_SKB_CB(skb)->end_seq = (TCP_SKB_CB(skb)->seq + th->syn + th->fin +
1764 skb->len - th->doff * 4);
1765 TCP_SKB_CB(skb)->ack_seq = ntohl(th->ack_seq);
1766 TCP_SKB_CB(skb)->tcp_flags = tcp_flag_byte(th);
1767 TCP_SKB_CB(skb)->tcp_tw_isn = 0;
1768 TCP_SKB_CB(skb)->ip_dsfield = ipv4_get_dsfield(iph);
1769 TCP_SKB_CB(skb)->sacked = 0;
1770 TCP_SKB_CB(skb)->has_rxtstamp =
1771 skb->tstamp || skb_hwtstamps(skb)->hwtstamp;
1778 int tcp_v4_rcv(struct sk_buff *skb)
1780 struct net *net = dev_net(skb->dev);
1781 int sdif = inet_sdif(skb);
1782 const struct iphdr *iph;
1783 const struct tcphdr *th;
1788 if (skb->pkt_type != PACKET_HOST)
1791 /* Count it even if it's bad */
1792 __TCP_INC_STATS(net, TCP_MIB_INSEGS);
1794 if (!pskb_may_pull(skb, sizeof(struct tcphdr)))
1797 th = (const struct tcphdr *)skb->data;
1799 if (unlikely(th->doff < sizeof(struct tcphdr) / 4))
1801 if (!pskb_may_pull(skb, th->doff * 4))
1804 /* An explanation is required here, I think.
1805 * Packet length and doff are validated by header prediction,
1806 * provided case of th->doff==0 is eliminated.
1807 * So, we defer the checks. */
1809 if (skb_checksum_init(skb, IPPROTO_TCP, inet_compute_pseudo))
1812 th = (const struct tcphdr *)skb->data;
1815 sk = __inet_lookup_skb(&tcp_hashinfo, skb, __tcp_hdrlen(th), th->source,
1816 th->dest, sdif, &refcounted);
1821 if (sk->sk_state == TCP_TIME_WAIT)
1824 if (sk->sk_state == TCP_NEW_SYN_RECV) {
1825 struct request_sock *req = inet_reqsk(sk);
1826 bool req_stolen = false;
1829 sk = req->rsk_listener;
1830 if (unlikely(tcp_v4_inbound_md5_hash(sk, skb))) {
1831 sk_drops_add(sk, skb);
1835 if (tcp_checksum_complete(skb)) {
1839 if (unlikely(sk->sk_state != TCP_LISTEN)) {
1840 inet_csk_reqsk_queue_drop_and_put(sk, req);
1843 /* We own a reference on the listener, increase it again
1844 * as we might lose it too soon.
1849 if (!tcp_filter(sk, skb)) {
1850 th = (const struct tcphdr *)skb->data;
1852 tcp_v4_fill_cb(skb, iph, th);
1853 nsk = tcp_check_req(sk, skb, req, false, &req_stolen);
1858 /* Another cpu got exclusive access to req
1859 * and created a full blown socket.
1860 * Try to feed this packet to this socket
1861 * instead of discarding it.
1863 tcp_v4_restore_cb(skb);
1867 goto discard_and_relse;
1871 tcp_v4_restore_cb(skb);
1872 } else if (tcp_child_process(sk, nsk, skb)) {
1873 tcp_v4_send_reset(nsk, skb);
1874 goto discard_and_relse;
1880 if (unlikely(iph->ttl < inet_sk(sk)->min_ttl)) {
1881 __NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP);
1882 goto discard_and_relse;
1885 if (!xfrm4_policy_check(sk, XFRM_POLICY_IN, skb))
1886 goto discard_and_relse;
1888 if (tcp_v4_inbound_md5_hash(sk, skb))
1889 goto discard_and_relse;
1893 if (tcp_filter(sk, skb))
1894 goto discard_and_relse;
1895 th = (const struct tcphdr *)skb->data;
1897 tcp_v4_fill_cb(skb, iph, th);
1901 if (sk->sk_state == TCP_LISTEN) {
1902 ret = tcp_v4_do_rcv(sk, skb);
1903 goto put_and_return;
1906 sk_incoming_cpu_update(sk);
1908 bh_lock_sock_nested(sk);
1909 tcp_segs_in(tcp_sk(sk), skb);
1911 if (!sock_owned_by_user(sk)) {
1912 ret = tcp_v4_do_rcv(sk, skb);
1913 } else if (tcp_add_backlog(sk, skb)) {
1914 goto discard_and_relse;
1925 if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb))
1928 tcp_v4_fill_cb(skb, iph, th);
1930 if (tcp_checksum_complete(skb)) {
1932 __TCP_INC_STATS(net, TCP_MIB_CSUMERRORS);
1934 __TCP_INC_STATS(net, TCP_MIB_INERRS);
1936 tcp_v4_send_reset(NULL, skb);
1940 /* Discard frame. */
1945 sk_drops_add(sk, skb);
1951 if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb)) {
1952 inet_twsk_put(inet_twsk(sk));
1956 tcp_v4_fill_cb(skb, iph, th);
1958 if (tcp_checksum_complete(skb)) {
1959 inet_twsk_put(inet_twsk(sk));
1962 switch (tcp_timewait_state_process(inet_twsk(sk), skb, th)) {
1964 struct sock *sk2 = inet_lookup_listener(dev_net(skb->dev),
1967 iph->saddr, th->source,
1968 iph->daddr, th->dest,
1972 inet_twsk_deschedule_put(inet_twsk(sk));
1974 tcp_v4_restore_cb(skb);
1982 tcp_v4_timewait_ack(sk, skb);
1985 tcp_v4_send_reset(sk, skb);
1986 inet_twsk_deschedule_put(inet_twsk(sk));
1988 case TCP_TW_SUCCESS:;
1993 static struct timewait_sock_ops tcp_timewait_sock_ops = {
1994 .twsk_obj_size = sizeof(struct tcp_timewait_sock),
1995 .twsk_unique = tcp_twsk_unique,
1996 .twsk_destructor= tcp_twsk_destructor,
1999 void inet_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb)
2001 struct dst_entry *dst = skb_dst(skb);
2003 if (dst && dst_hold_safe(dst)) {
2004 sk->sk_rx_dst = dst;
2005 inet_sk(sk)->rx_dst_ifindex = skb->skb_iif;
2008 EXPORT_SYMBOL(inet_sk_rx_dst_set);
2010 const struct inet_connection_sock_af_ops ipv4_specific = {
2011 .queue_xmit = ip_queue_xmit,
2012 .send_check = tcp_v4_send_check,
2013 .rebuild_header = inet_sk_rebuild_header,
2014 .sk_rx_dst_set = inet_sk_rx_dst_set,
2015 .conn_request = tcp_v4_conn_request,
2016 .syn_recv_sock = tcp_v4_syn_recv_sock,
2017 .net_header_len = sizeof(struct iphdr),
2018 .setsockopt = ip_setsockopt,
2019 .getsockopt = ip_getsockopt,
2020 .addr2sockaddr = inet_csk_addr2sockaddr,
2021 .sockaddr_len = sizeof(struct sockaddr_in),
2022 #ifdef CONFIG_COMPAT
2023 .compat_setsockopt = compat_ip_setsockopt,
2024 .compat_getsockopt = compat_ip_getsockopt,
2026 .mtu_reduced = tcp_v4_mtu_reduced,
2028 EXPORT_SYMBOL(ipv4_specific);
2030 #ifdef CONFIG_TCP_MD5SIG
2031 static const struct tcp_sock_af_ops tcp_sock_ipv4_specific = {
2032 .md5_lookup = tcp_v4_md5_lookup,
2033 .calc_md5_hash = tcp_v4_md5_hash_skb,
2034 .md5_parse = tcp_v4_parse_md5_keys,
2038 /* NOTE: A lot of things set to zero explicitly by call to
2039 * sk_alloc() so need not be done here.
2041 static int tcp_v4_init_sock(struct sock *sk)
2043 struct inet_connection_sock *icsk = inet_csk(sk);
2047 icsk->icsk_af_ops = &ipv4_specific;
2049 #ifdef CONFIG_TCP_MD5SIG
2050 tcp_sk(sk)->af_specific = &tcp_sock_ipv4_specific;
2056 void tcp_v4_destroy_sock(struct sock *sk)
2058 struct tcp_sock *tp = tcp_sk(sk);
2060 trace_tcp_destroy_sock(sk);
2062 tcp_clear_xmit_timers(sk);
2064 tcp_cleanup_congestion_control(sk);
2066 tcp_cleanup_ulp(sk);
2068 /* Cleanup up the write buffer. */
2069 tcp_write_queue_purge(sk);
2071 /* Check if we want to disable active TFO */
2072 tcp_fastopen_active_disable_ofo_check(sk);
2074 /* Cleans up our, hopefully empty, out_of_order_queue. */
2075 skb_rbtree_purge(&tp->out_of_order_queue);
2077 #ifdef CONFIG_TCP_MD5SIG
2078 /* Clean up the MD5 key list, if any */
2079 if (tp->md5sig_info) {
2080 tcp_clear_md5_list(sk);
2081 kfree_rcu(rcu_dereference_protected(tp->md5sig_info, 1), rcu);
2082 tp->md5sig_info = NULL;
2086 /* Clean up a referenced TCP bind bucket. */
2087 if (inet_csk(sk)->icsk_bind_hash)
2090 BUG_ON(tp->fastopen_rsk);
2092 /* If socket is aborted during connect operation */
2093 tcp_free_fastopen_req(tp);
2094 tcp_fastopen_destroy_cipher(sk);
2095 tcp_saved_syn_free(tp);
2097 sk_sockets_allocated_dec(sk);
2099 EXPORT_SYMBOL(tcp_v4_destroy_sock);
2101 #ifdef CONFIG_PROC_FS
2102 /* Proc filesystem TCP sock list dumping. */
2105 * Get next listener socket follow cur. If cur is NULL, get first socket
2106 * starting from bucket given in st->bucket; when st->bucket is zero the
2107 * very first socket in the hash table is returned.
2109 static void *listening_get_next(struct seq_file *seq, void *cur)
2111 struct tcp_seq_afinfo *afinfo = PDE_DATA(file_inode(seq->file));
2112 struct tcp_iter_state *st = seq->private;
2113 struct net *net = seq_file_net(seq);
2114 struct inet_listen_hashbucket *ilb;
2115 struct sock *sk = cur;
2119 ilb = &tcp_hashinfo.listening_hash[st->bucket];
2120 spin_lock(&ilb->lock);
2121 sk = sk_head(&ilb->head);
2125 ilb = &tcp_hashinfo.listening_hash[st->bucket];
2131 sk_for_each_from(sk) {
2132 if (!net_eq(sock_net(sk), net))
2134 if (sk->sk_family == afinfo->family)
2137 spin_unlock(&ilb->lock);
2139 if (++st->bucket < INET_LHTABLE_SIZE)
2144 static void *listening_get_idx(struct seq_file *seq, loff_t *pos)
2146 struct tcp_iter_state *st = seq->private;
2151 rc = listening_get_next(seq, NULL);
2153 while (rc && *pos) {
2154 rc = listening_get_next(seq, rc);
2160 static inline bool empty_bucket(const struct tcp_iter_state *st)
2162 return hlist_nulls_empty(&tcp_hashinfo.ehash[st->bucket].chain);
2166 * Get first established socket starting from bucket given in st->bucket.
2167 * If st->bucket is zero, the very first socket in the hash is returned.
2169 static void *established_get_first(struct seq_file *seq)
2171 struct tcp_seq_afinfo *afinfo = PDE_DATA(file_inode(seq->file));
2172 struct tcp_iter_state *st = seq->private;
2173 struct net *net = seq_file_net(seq);
2177 for (; st->bucket <= tcp_hashinfo.ehash_mask; ++st->bucket) {
2179 struct hlist_nulls_node *node;
2180 spinlock_t *lock = inet_ehash_lockp(&tcp_hashinfo, st->bucket);
2182 /* Lockless fast path for the common case of empty buckets */
2183 if (empty_bucket(st))
2187 sk_nulls_for_each(sk, node, &tcp_hashinfo.ehash[st->bucket].chain) {
2188 if (sk->sk_family != afinfo->family ||
2189 !net_eq(sock_net(sk), net)) {
2195 spin_unlock_bh(lock);
2201 static void *established_get_next(struct seq_file *seq, void *cur)
2203 struct tcp_seq_afinfo *afinfo = PDE_DATA(file_inode(seq->file));
2204 struct sock *sk = cur;
2205 struct hlist_nulls_node *node;
2206 struct tcp_iter_state *st = seq->private;
2207 struct net *net = seq_file_net(seq);
2212 sk = sk_nulls_next(sk);
2214 sk_nulls_for_each_from(sk, node) {
2215 if (sk->sk_family == afinfo->family &&
2216 net_eq(sock_net(sk), net))
2220 spin_unlock_bh(inet_ehash_lockp(&tcp_hashinfo, st->bucket));
2222 return established_get_first(seq);
2225 static void *established_get_idx(struct seq_file *seq, loff_t pos)
2227 struct tcp_iter_state *st = seq->private;
2231 rc = established_get_first(seq);
2234 rc = established_get_next(seq, rc);
2240 static void *tcp_get_idx(struct seq_file *seq, loff_t pos)
2243 struct tcp_iter_state *st = seq->private;
2245 st->state = TCP_SEQ_STATE_LISTENING;
2246 rc = listening_get_idx(seq, &pos);
2249 st->state = TCP_SEQ_STATE_ESTABLISHED;
2250 rc = established_get_idx(seq, pos);
2256 static void *tcp_seek_last_pos(struct seq_file *seq)
2258 struct tcp_iter_state *st = seq->private;
2259 int offset = st->offset;
2260 int orig_num = st->num;
2263 switch (st->state) {
2264 case TCP_SEQ_STATE_LISTENING:
2265 if (st->bucket >= INET_LHTABLE_SIZE)
2267 st->state = TCP_SEQ_STATE_LISTENING;
2268 rc = listening_get_next(seq, NULL);
2269 while (offset-- && rc)
2270 rc = listening_get_next(seq, rc);
2274 st->state = TCP_SEQ_STATE_ESTABLISHED;
2276 case TCP_SEQ_STATE_ESTABLISHED:
2277 if (st->bucket > tcp_hashinfo.ehash_mask)
2279 rc = established_get_first(seq);
2280 while (offset-- && rc)
2281 rc = established_get_next(seq, rc);
2289 void *tcp_seq_start(struct seq_file *seq, loff_t *pos)
2291 struct tcp_iter_state *st = seq->private;
2294 if (*pos && *pos == st->last_pos) {
2295 rc = tcp_seek_last_pos(seq);
2300 st->state = TCP_SEQ_STATE_LISTENING;
2304 rc = *pos ? tcp_get_idx(seq, *pos - 1) : SEQ_START_TOKEN;
2307 st->last_pos = *pos;
2310 EXPORT_SYMBOL(tcp_seq_start);
2312 void *tcp_seq_next(struct seq_file *seq, void *v, loff_t *pos)
2314 struct tcp_iter_state *st = seq->private;
2317 if (v == SEQ_START_TOKEN) {
2318 rc = tcp_get_idx(seq, 0);
2322 switch (st->state) {
2323 case TCP_SEQ_STATE_LISTENING:
2324 rc = listening_get_next(seq, v);
2326 st->state = TCP_SEQ_STATE_ESTABLISHED;
2329 rc = established_get_first(seq);
2332 case TCP_SEQ_STATE_ESTABLISHED:
2333 rc = established_get_next(seq, v);
2338 st->last_pos = *pos;
2341 EXPORT_SYMBOL(tcp_seq_next);
2343 void tcp_seq_stop(struct seq_file *seq, void *v)
2345 struct tcp_iter_state *st = seq->private;
2347 switch (st->state) {
2348 case TCP_SEQ_STATE_LISTENING:
2349 if (v != SEQ_START_TOKEN)
2350 spin_unlock(&tcp_hashinfo.listening_hash[st->bucket].lock);
2352 case TCP_SEQ_STATE_ESTABLISHED:
2354 spin_unlock_bh(inet_ehash_lockp(&tcp_hashinfo, st->bucket));
2358 EXPORT_SYMBOL(tcp_seq_stop);
2360 static void get_openreq4(const struct request_sock *req,
2361 struct seq_file *f, int i)
2363 const struct inet_request_sock *ireq = inet_rsk(req);
2364 long delta = req->rsk_timer.expires - jiffies;
2366 seq_printf(f, "%4d: %08X:%04X %08X:%04X"
2367 " %02X %08X:%08X %02X:%08lX %08X %5u %8d %u %d %pK",
2372 ntohs(ireq->ir_rmt_port),
2374 0, 0, /* could print option size, but that is af dependent. */
2375 1, /* timers active (only the expire timer) */
2376 jiffies_delta_to_clock_t(delta),
2378 from_kuid_munged(seq_user_ns(f),
2379 sock_i_uid(req->rsk_listener)),
2380 0, /* non standard timer */
2381 0, /* open_requests have no inode */
2386 static void get_tcp4_sock(struct sock *sk, struct seq_file *f, int i)
2389 unsigned long timer_expires;
2390 const struct tcp_sock *tp = tcp_sk(sk);
2391 const struct inet_connection_sock *icsk = inet_csk(sk);
2392 const struct inet_sock *inet = inet_sk(sk);
2393 const struct fastopen_queue *fastopenq = &icsk->icsk_accept_queue.fastopenq;
2394 __be32 dest = inet->inet_daddr;
2395 __be32 src = inet->inet_rcv_saddr;
2396 __u16 destp = ntohs(inet->inet_dport);
2397 __u16 srcp = ntohs(inet->inet_sport);
2401 if (icsk->icsk_pending == ICSK_TIME_RETRANS ||
2402 icsk->icsk_pending == ICSK_TIME_REO_TIMEOUT ||
2403 icsk->icsk_pending == ICSK_TIME_LOSS_PROBE) {
2405 timer_expires = icsk->icsk_timeout;
2406 } else if (icsk->icsk_pending == ICSK_TIME_PROBE0) {
2408 timer_expires = icsk->icsk_timeout;
2409 } else if (timer_pending(&sk->sk_timer)) {
2411 timer_expires = sk->sk_timer.expires;
2414 timer_expires = jiffies;
2417 state = inet_sk_state_load(sk);
2418 if (state == TCP_LISTEN)
2419 rx_queue = sk->sk_ack_backlog;
2421 /* Because we don't lock the socket,
2422 * we might find a transient negative value.
2424 rx_queue = max_t(int, tp->rcv_nxt - tp->copied_seq, 0);
2426 seq_printf(f, "%4d: %08X:%04X %08X:%04X %02X %08X:%08X %02X:%08lX "
2427 "%08X %5u %8d %lu %d %pK %lu %lu %u %u %d",
2428 i, src, srcp, dest, destp, state,
2429 tp->write_seq - tp->snd_una,
2432 jiffies_delta_to_clock_t(timer_expires - jiffies),
2433 icsk->icsk_retransmits,
2434 from_kuid_munged(seq_user_ns(f), sock_i_uid(sk)),
2435 icsk->icsk_probes_out,
2437 refcount_read(&sk->sk_refcnt), sk,
2438 jiffies_to_clock_t(icsk->icsk_rto),
2439 jiffies_to_clock_t(icsk->icsk_ack.ato),
2440 (icsk->icsk_ack.quick << 1) | icsk->icsk_ack.pingpong,
2442 state == TCP_LISTEN ?
2443 fastopenq->max_qlen :
2444 (tcp_in_initial_slowstart(tp) ? -1 : tp->snd_ssthresh));
2447 static void get_timewait4_sock(const struct inet_timewait_sock *tw,
2448 struct seq_file *f, int i)
2450 long delta = tw->tw_timer.expires - jiffies;
2454 dest = tw->tw_daddr;
2455 src = tw->tw_rcv_saddr;
2456 destp = ntohs(tw->tw_dport);
2457 srcp = ntohs(tw->tw_sport);
2459 seq_printf(f, "%4d: %08X:%04X %08X:%04X"
2460 " %02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d %pK",
2461 i, src, srcp, dest, destp, tw->tw_substate, 0, 0,
2462 3, jiffies_delta_to_clock_t(delta), 0, 0, 0, 0,
2463 refcount_read(&tw->tw_refcnt), tw);
2468 static int tcp4_seq_show(struct seq_file *seq, void *v)
2470 struct tcp_iter_state *st;
2471 struct sock *sk = v;
2473 seq_setwidth(seq, TMPSZ - 1);
2474 if (v == SEQ_START_TOKEN) {
2475 seq_puts(seq, " sl local_address rem_address st tx_queue "
2476 "rx_queue tr tm->when retrnsmt uid timeout "
2482 if (sk->sk_state == TCP_TIME_WAIT)
2483 get_timewait4_sock(v, seq, st->num);
2484 else if (sk->sk_state == TCP_NEW_SYN_RECV)
2485 get_openreq4(v, seq, st->num);
2487 get_tcp4_sock(v, seq, st->num);
2493 static const struct seq_operations tcp4_seq_ops = {
2494 .show = tcp4_seq_show,
2495 .start = tcp_seq_start,
2496 .next = tcp_seq_next,
2497 .stop = tcp_seq_stop,
2500 static struct tcp_seq_afinfo tcp4_seq_afinfo = {
2504 static int __net_init tcp4_proc_init_net(struct net *net)
2506 if (!proc_create_net_data("tcp", 0444, net->proc_net, &tcp4_seq_ops,
2507 sizeof(struct tcp_iter_state), &tcp4_seq_afinfo))
2512 static void __net_exit tcp4_proc_exit_net(struct net *net)
2514 remove_proc_entry("tcp", net->proc_net);
2517 static struct pernet_operations tcp4_net_ops = {
2518 .init = tcp4_proc_init_net,
2519 .exit = tcp4_proc_exit_net,
2522 int __init tcp4_proc_init(void)
2524 return register_pernet_subsys(&tcp4_net_ops);
2527 void tcp4_proc_exit(void)
2529 unregister_pernet_subsys(&tcp4_net_ops);
2531 #endif /* CONFIG_PROC_FS */
2533 struct proto tcp_prot = {
2535 .owner = THIS_MODULE,
2537 .pre_connect = tcp_v4_pre_connect,
2538 .connect = tcp_v4_connect,
2539 .disconnect = tcp_disconnect,
2540 .accept = inet_csk_accept,
2542 .init = tcp_v4_init_sock,
2543 .destroy = tcp_v4_destroy_sock,
2544 .shutdown = tcp_shutdown,
2545 .setsockopt = tcp_setsockopt,
2546 .getsockopt = tcp_getsockopt,
2547 .keepalive = tcp_set_keepalive,
2548 .recvmsg = tcp_recvmsg,
2549 .sendmsg = tcp_sendmsg,
2550 .sendpage = tcp_sendpage,
2551 .backlog_rcv = tcp_v4_do_rcv,
2552 .release_cb = tcp_release_cb,
2554 .unhash = inet_unhash,
2555 .get_port = inet_csk_get_port,
2556 .enter_memory_pressure = tcp_enter_memory_pressure,
2557 .leave_memory_pressure = tcp_leave_memory_pressure,
2558 .stream_memory_free = tcp_stream_memory_free,
2559 .sockets_allocated = &tcp_sockets_allocated,
2560 .orphan_count = &tcp_orphan_count,
2561 .memory_allocated = &tcp_memory_allocated,
2562 .memory_pressure = &tcp_memory_pressure,
2563 .sysctl_mem = sysctl_tcp_mem,
2564 .sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_tcp_wmem),
2565 .sysctl_rmem_offset = offsetof(struct net, ipv4.sysctl_tcp_rmem),
2566 .max_header = MAX_TCP_HEADER,
2567 .obj_size = sizeof(struct tcp_sock),
2568 .slab_flags = SLAB_TYPESAFE_BY_RCU,
2569 .twsk_prot = &tcp_timewait_sock_ops,
2570 .rsk_prot = &tcp_request_sock_ops,
2571 .h.hashinfo = &tcp_hashinfo,
2572 .no_autobind = true,
2573 #ifdef CONFIG_COMPAT
2574 .compat_setsockopt = compat_tcp_setsockopt,
2575 .compat_getsockopt = compat_tcp_getsockopt,
2577 .diag_destroy = tcp_abort,
2579 EXPORT_SYMBOL(tcp_prot);
2581 static void __net_exit tcp_sk_exit(struct net *net)
2585 module_put(net->ipv4.tcp_congestion_control->owner);
2587 for_each_possible_cpu(cpu)
2588 inet_ctl_sock_destroy(*per_cpu_ptr(net->ipv4.tcp_sk, cpu));
2589 free_percpu(net->ipv4.tcp_sk);
2592 static int __net_init tcp_sk_init(struct net *net)
2596 net->ipv4.tcp_sk = alloc_percpu(struct sock *);
2597 if (!net->ipv4.tcp_sk)
2600 for_each_possible_cpu(cpu) {
2603 res = inet_ctl_sock_create(&sk, PF_INET, SOCK_RAW,
2607 sock_set_flag(sk, SOCK_USE_WRITE_QUEUE);
2609 /* Please enforce IP_DF and IPID==0 for RST and
2610 * ACK sent in SYN-RECV and TIME-WAIT state.
2612 inet_sk(sk)->pmtudisc = IP_PMTUDISC_DO;
2614 *per_cpu_ptr(net->ipv4.tcp_sk, cpu) = sk;
2617 net->ipv4.sysctl_tcp_ecn = 2;
2618 net->ipv4.sysctl_tcp_ecn_fallback = 1;
2620 net->ipv4.sysctl_tcp_base_mss = TCP_BASE_MSS;
2621 net->ipv4.sysctl_tcp_probe_threshold = TCP_PROBE_THRESHOLD;
2622 net->ipv4.sysctl_tcp_probe_interval = TCP_PROBE_INTERVAL;
2624 net->ipv4.sysctl_tcp_keepalive_time = TCP_KEEPALIVE_TIME;
2625 net->ipv4.sysctl_tcp_keepalive_probes = TCP_KEEPALIVE_PROBES;
2626 net->ipv4.sysctl_tcp_keepalive_intvl = TCP_KEEPALIVE_INTVL;
2628 net->ipv4.sysctl_tcp_syn_retries = TCP_SYN_RETRIES;
2629 net->ipv4.sysctl_tcp_synack_retries = TCP_SYNACK_RETRIES;
2630 net->ipv4.sysctl_tcp_syncookies = 1;
2631 net->ipv4.sysctl_tcp_reordering = TCP_FASTRETRANS_THRESH;
2632 net->ipv4.sysctl_tcp_retries1 = TCP_RETR1;
2633 net->ipv4.sysctl_tcp_retries2 = TCP_RETR2;
2634 net->ipv4.sysctl_tcp_orphan_retries = 0;
2635 net->ipv4.sysctl_tcp_fin_timeout = TCP_FIN_TIMEOUT;
2636 net->ipv4.sysctl_tcp_notsent_lowat = UINT_MAX;
2637 net->ipv4.sysctl_tcp_tw_reuse = 2;
2639 cnt = tcp_hashinfo.ehash_mask + 1;
2640 net->ipv4.tcp_death_row.sysctl_max_tw_buckets = cnt / 2;
2641 net->ipv4.tcp_death_row.hashinfo = &tcp_hashinfo;
2643 net->ipv4.sysctl_max_syn_backlog = max(128, cnt / 256);
2644 net->ipv4.sysctl_tcp_sack = 1;
2645 net->ipv4.sysctl_tcp_window_scaling = 1;
2646 net->ipv4.sysctl_tcp_timestamps = 1;
2647 net->ipv4.sysctl_tcp_early_retrans = 3;
2648 net->ipv4.sysctl_tcp_recovery = TCP_RACK_LOSS_DETECTION;
2649 net->ipv4.sysctl_tcp_slow_start_after_idle = 1; /* By default, RFC2861 behavior. */
2650 net->ipv4.sysctl_tcp_retrans_collapse = 1;
2651 net->ipv4.sysctl_tcp_max_reordering = 300;
2652 net->ipv4.sysctl_tcp_dsack = 1;
2653 net->ipv4.sysctl_tcp_app_win = 31;
2654 net->ipv4.sysctl_tcp_adv_win_scale = 1;
2655 net->ipv4.sysctl_tcp_frto = 2;
2656 net->ipv4.sysctl_tcp_moderate_rcvbuf = 1;
2657 /* This limits the percentage of the congestion window which we
2658 * will allow a single TSO frame to consume. Building TSO frames
2659 * which are too large can cause TCP streams to be bursty.
2661 net->ipv4.sysctl_tcp_tso_win_divisor = 3;
2662 /* Default TSQ limit of 16 TSO segments */
2663 net->ipv4.sysctl_tcp_limit_output_bytes = 16 * 65536;
2664 /* rfc5961 challenge ack rate limiting */
2665 net->ipv4.sysctl_tcp_challenge_ack_limit = 1000;
2666 net->ipv4.sysctl_tcp_min_tso_segs = 2;
2667 net->ipv4.sysctl_tcp_min_rtt_wlen = 300;
2668 net->ipv4.sysctl_tcp_autocorking = 1;
2669 net->ipv4.sysctl_tcp_invalid_ratelimit = HZ/2;
2670 net->ipv4.sysctl_tcp_pacing_ss_ratio = 200;
2671 net->ipv4.sysctl_tcp_pacing_ca_ratio = 120;
2672 if (net != &init_net) {
2673 memcpy(net->ipv4.sysctl_tcp_rmem,
2674 init_net.ipv4.sysctl_tcp_rmem,
2675 sizeof(init_net.ipv4.sysctl_tcp_rmem));
2676 memcpy(net->ipv4.sysctl_tcp_wmem,
2677 init_net.ipv4.sysctl_tcp_wmem,
2678 sizeof(init_net.ipv4.sysctl_tcp_wmem));
2680 net->ipv4.sysctl_tcp_comp_sack_delay_ns = NSEC_PER_MSEC;
2681 net->ipv4.sysctl_tcp_comp_sack_nr = 44;
2682 net->ipv4.sysctl_tcp_fastopen = TFO_CLIENT_ENABLE;
2683 spin_lock_init(&net->ipv4.tcp_fastopen_ctx_lock);
2684 net->ipv4.sysctl_tcp_fastopen_blackhole_timeout = 60 * 60;
2685 atomic_set(&net->ipv4.tfo_active_disable_times, 0);
2687 /* Reno is always built in */
2688 if (!net_eq(net, &init_net) &&
2689 try_module_get(init_net.ipv4.tcp_congestion_control->owner))
2690 net->ipv4.tcp_congestion_control = init_net.ipv4.tcp_congestion_control;
2692 net->ipv4.tcp_congestion_control = &tcp_reno;
2701 static void __net_exit tcp_sk_exit_batch(struct list_head *net_exit_list)
2705 inet_twsk_purge(&tcp_hashinfo, AF_INET);
2707 list_for_each_entry(net, net_exit_list, exit_list)
2708 tcp_fastopen_ctx_destroy(net);
2711 static struct pernet_operations __net_initdata tcp_sk_ops = {
2712 .init = tcp_sk_init,
2713 .exit = tcp_sk_exit,
2714 .exit_batch = tcp_sk_exit_batch,
2717 void __init tcp_v4_init(void)
2719 if (register_pernet_subsys(&tcp_sk_ops))
2720 panic("Failed to create the TCP control socket.\n");