net/ipv6/tcp_ipv6.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  *      TCP over IPv6
   4  *      Linux INET6 implementation
   5  *
   6  *      Authors:
   7  *      Pedro Roque             <roque@di.fc.ul.pt>
   8  *
   9  *      Based on:
  10  *      linux/net/ipv4/tcp.c
  11  *      linux/net/ipv4/tcp_input.c
  12  *      linux/net/ipv4/tcp_output.c
  13  *
  14  *      Fixes:
  15  *      Hideaki YOSHIFUJI       :       sin6_scope_id support
  16  *      YOSHIFUJI Hideaki @USAGI and:   Support IPV6_V6ONLY socket option, which
  17  *      Alexey Kuznetsov                allow both IPv4 and IPv6 sockets to bind
  18  *                                      a single port at the same time.
  19  *      YOSHIFUJI Hideaki @USAGI:       convert /proc/net/tcp6 to seq_file.
  20  */
  21
  22 #include <linux/bottom_half.h>
  23 #include <linux/module.h>
  24 #include <linux/errno.h>
  25 #include <linux/types.h>
  26 #include <linux/socket.h>
  27 #include <linux/sockios.h>
  28 #include <linux/net.h>
  29 #include <linux/jiffies.h>
  30 #include <linux/in.h>
  31 #include <linux/in6.h>
  32 #include <linux/netdevice.h>
  33 #include <linux/init.h>
  34 #include <linux/jhash.h>
  35 #include <linux/ipsec.h>
  36 #include <linux/times.h>
  37 #include <linux/slab.h>
  38 #include <linux/uaccess.h>
  39 #include <linux/ipv6.h>
  40 #include <linux/icmpv6.h>
  41 #include <linux/random.h>
  42 #include <linux/indirect_call_wrapper.h>
  43
  44 #include <net/tcp.h>
  45 #include <net/ndisc.h>
  46 #include <net/inet6_hashtables.h>
  47 #include <net/inet6_connection_sock.h>
  48 #include <net/ipv6.h>
  49 #include <net/transp_v6.h>
  50 #include <net/addrconf.h>
  51 #include <net/ip6_route.h>
  52 #include <net/ip6_checksum.h>
  53 #include <net/inet_ecn.h>
  54 #include <net/protocol.h>
  55 #include <net/xfrm.h>
  56 #include <net/snmp.h>
  57 #include <net/dsfield.h>
  58 #include <net/timewait_sock.h>
  59 #include <net/inet_common.h>
  60 #include <net/secure_seq.h>
  61 #include <net/busy_poll.h>
  62
  63 #include <linux/proc_fs.h>
  64 #include <linux/seq_file.h>
  65
  66 #include <crypto/hash.h>
  67 #include <linux/scatterlist.h>
  68
  69 #include <trace/events/tcp.h>
  70
  71 static void     tcp_v6_send_reset(const struct sock *sk, struct sk_buff *skb);
  72 static void     tcp_v6_reqsk_send_ack(const struct sock *sk, struct sk_buff *skb,
  73                                       struct request_sock *req);
  74
  75 static int      tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb);
  76
  77 static const struct inet_connection_sock_af_ops ipv6_mapped;
  78 const struct inet_connection_sock_af_ops ipv6_specific;
  79 #ifdef CONFIG_TCP_MD5SIG
  80 static const struct tcp_sock_af_ops tcp_sock_ipv6_specific;
  81 static const struct tcp_sock_af_ops tcp_sock_ipv6_mapped_specific;
  82 #else
  83 static struct tcp_md5sig_key *tcp_v6_md5_do_lookup(const struct sock *sk,
  84                                                    const struct in6_addr *addr,
  85                                                    int l3index)
  86 {
  87         return NULL;
  88 }
  89 #endif
  90
  91 /* Helper returning the inet6 address from a given tcp socket.
  92  * It can be used in TCP stack instead of inet6_sk(sk).
  93  * This avoids a dereference and allow compiler optimizations.
  94  * It is a specialized version of inet6_sk_generic().
  95  */
  96 static struct ipv6_pinfo *tcp_inet6_sk(const struct sock *sk)
  97 {
  98         unsigned int offset = sizeof(struct tcp6_sock) - sizeof(struct ipv6_pinfo);
  99
 100         return (struct ipv6_pinfo *)(((u8 *)sk) + offset);
 101 }
 102
 103 static void inet6_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb)
 104 {
 105         struct dst_entry *dst = skb_dst(skb);
 106
 107         if (dst && dst_hold_safe(dst)) {
 108                 const struct rt6_info *rt = (const struct rt6_info *)dst;
 109
 110                 sk->sk_rx_dst = dst;
 111                 inet_sk(sk)->rx_dst_ifindex = skb->skb_iif;
 112                 tcp_inet6_sk(sk)->rx_dst_cookie = rt6_get_cookie(rt);
 113         }
 114 }
 115
 116 static u32 tcp_v6_init_seq(const struct sk_buff *skb)
 117 {
 118         return secure_tcpv6_seq(ipv6_hdr(skb)->daddr.s6_addr32,
 119                                 ipv6_hdr(skb)->saddr.s6_addr32,
 120                                 tcp_hdr(skb)->dest,
 121                                 tcp_hdr(skb)->source);
 122 }
 123
 124 static u32 tcp_v6_init_ts_off(const struct net *net, const struct sk_buff *skb)
 125 {
 126         return secure_tcpv6_ts_off(net, ipv6_hdr(skb)->daddr.s6_addr32,
 127                                    ipv6_hdr(skb)->saddr.s6_addr32);
 128 }
 129
 130 static int tcp_v6_pre_connect(struct sock *sk, struct sockaddr *uaddr,
 131                               int addr_len)
 132 {
 133         /* This check is replicated from tcp_v6_connect() and intended to
 134          * prevent BPF program called below from accessing bytes that are out
 135          * of the bound specified by user in addr_len.
 136          */
 137         if (addr_len < SIN6_LEN_RFC2133)
 138                 return -EINVAL;
 139
 140         sock_owned_by_me(sk);
 141
 142         return BPF_CGROUP_RUN_PROG_INET6_CONNECT(sk, uaddr);
 143 }
 144
 145 static int tcp_v6_connect(struct sock *sk, struct sockaddr *uaddr,
 146                           int addr_len)
 147 {
 148         struct sockaddr_in6 *usin = (struct sockaddr_in6 *) uaddr;
 149         struct inet_sock *inet = inet_sk(sk);
 150         struct inet_connection_sock *icsk = inet_csk(sk);
 151         struct ipv6_pinfo *np = tcp_inet6_sk(sk);
 152         struct tcp_sock *tp = tcp_sk(sk);
 153         struct in6_addr *saddr = NULL, *final_p, final;
 154         struct ipv6_txoptions *opt;
 155         struct flowi6 fl6;
 156         struct dst_entry *dst;
 157         int addr_type;
 158         int err;
 159         struct inet_timewait_death_row *tcp_death_row = &sock_net(sk)->ipv4.tcp_death_row;
 160
 161         if (addr_len < SIN6_LEN_RFC2133)
 162                 return -EINVAL;
 163
 164         if (usin->sin6_family != AF_INET6)
 165                 return -EAFNOSUPPORT;
 166
 167         memset(&fl6, 0, sizeof(fl6));
 168
 169         if (np->sndflow) {
 170                 fl6.flowlabel = usin->sin6_flowinfo&IPV6_FLOWINFO_MASK;
 171                 IP6_ECN_flow_init(fl6.flowlabel);
 172                 if (fl6.flowlabel&IPV6_FLOWLABEL_MASK) {
 173                         struct ip6_flowlabel *flowlabel;
 174                         flowlabel = fl6_sock_lookup(sk, fl6.flowlabel);
 175                         if (IS_ERR(flowlabel))
 176                                 return -EINVAL;
 177                         fl6_sock_release(flowlabel);
 178                 }
 179         }
 180
 181         /*
 182          *      connect() to INADDR_ANY means loopback (BSD'ism).
 183          */
 184
 185         if (ipv6_addr_any(&usin->sin6_addr)) {
 186                 if (ipv6_addr_v4mapped(&sk->sk_v6_rcv_saddr))
 187                         ipv6_addr_set_v4mapped(htonl(INADDR_LOOPBACK),
 188                                                &usin->sin6_addr);
 189                 else
 190                         usin->sin6_addr = in6addr_loopback;
 191         }
 192
 193         addr_type = ipv6_addr_type(&usin->sin6_addr);
 194
 195         if (addr_type & IPV6_ADDR_MULTICAST)
 196                 return -ENETUNREACH;
 197
 198         if (addr_type&IPV6_ADDR_LINKLOCAL) {
 199                 if (addr_len >= sizeof(struct sockaddr_in6) &&
 200                     usin->sin6_scope_id) {
 201                         /* If interface is set while binding, indices
 202                          * must coincide.
 203                          */
 204                         if (!sk_dev_equal_l3scope(sk, usin->sin6_scope_id))
 205                                 return -EINVAL;
 206
 207                         sk->sk_bound_dev_if = usin->sin6_scope_id;
 208                 }
 209
 210                 /* Connect to link-local address requires an interface */
 211                 if (!sk->sk_bound_dev_if)
 212                         return -EINVAL;
 213         }
 214
 215         if (tp->rx_opt.ts_recent_stamp &&
 216             !ipv6_addr_equal(&sk->sk_v6_daddr, &usin->sin6_addr)) {
 217                 tp->rx_opt.ts_recent = 0;
 218                 tp->rx_opt.ts_recent_stamp = 0;
 219                 WRITE_ONCE(tp->write_seq, 0);
 220         }
 221
 222         sk->sk_v6_daddr = usin->sin6_addr;
 223         np->flow_label = fl6.flowlabel;
 224
 225         /*
 226          *      TCP over IPv4
 227          */
 228
 229         if (addr_type & IPV6_ADDR_MAPPED) {
 230                 u32 exthdrlen = icsk->icsk_ext_hdr_len;
 231                 struct sockaddr_in sin;
 232
 233                 if (__ipv6_only_sock(sk))
 234                         return -ENETUNREACH;
 235
 236                 sin.sin_family = AF_INET;
 237                 sin.sin_port = usin->sin6_port;
 238                 sin.sin_addr.s_addr = usin->sin6_addr.s6_addr32[3];
 239
 240                 icsk->icsk_af_ops = &ipv6_mapped;
 241                 if (sk_is_mptcp(sk))
 242                         mptcpv6_handle_mapped(sk, true);
 243                 sk->sk_backlog_rcv = tcp_v4_do_rcv;
 244 #ifdef CONFIG_TCP_MD5SIG
 245                 tp->af_specific = &tcp_sock_ipv6_mapped_specific;
 246 #endif
 247
 248                 err = tcp_v4_connect(sk, (struct sockaddr *)&sin, sizeof(sin));
 249
 250                 if (err) {
 251                         icsk->icsk_ext_hdr_len = exthdrlen;
 252                         icsk->icsk_af_ops = &ipv6_specific;
 253                         if (sk_is_mptcp(sk))
 254                                 mptcpv6_handle_mapped(sk, false);
 255                         sk->sk_backlog_rcv = tcp_v6_do_rcv;
 256 #ifdef CONFIG_TCP_MD5SIG
 257                         tp->af_specific = &tcp_sock_ipv6_specific;
 258 #endif
 259                         goto failure;
 260                 }
 261                 np->saddr = sk->sk_v6_rcv_saddr;
 262
 263                 return err;
 264         }
 265
 266         if (!ipv6_addr_any(&sk->sk_v6_rcv_saddr))
 267                 saddr = &sk->sk_v6_rcv_saddr;
 268
 269         fl6.flowi6_proto = IPPROTO_TCP;
 270         fl6.daddr = sk->sk_v6_daddr;
 271         fl6.saddr = saddr ? *saddr : np->saddr;
 272         fl6.flowi6_oif = sk->sk_bound_dev_if;
 273         fl6.flowi6_mark = sk->sk_mark;
 274         fl6.fl6_dport = usin->sin6_port;
 275         fl6.fl6_sport = inet->inet_sport;
 276         fl6.flowi6_uid = sk->sk_uid;
 277
 278         opt = rcu_dereference_protected(np->opt, lockdep_sock_is_held(sk));
 279         final_p = fl6_update_dst(&fl6, opt, &final);
 280
 281         security_sk_classify_flow(sk, flowi6_to_flowi_common(&fl6));
 282
 283         dst = ip6_dst_lookup_flow(sock_net(sk), sk, &fl6, final_p);
 284         if (IS_ERR(dst)) {
 285                 err = PTR_ERR(dst);
 286                 goto failure;
 287         }
 288
 289         if (!saddr) {
 290                 saddr = &fl6.saddr;
 291                 sk->sk_v6_rcv_saddr = *saddr;
 292         }
 293
 294         /* set the source address */
 295         np->saddr = *saddr;
 296         inet->inet_rcv_saddr = LOOPBACK4_IPV6;
 297
 298         sk->sk_gso_type = SKB_GSO_TCPV6;
 299         ip6_dst_store(sk, dst, NULL, NULL);
 300
 301         icsk->icsk_ext_hdr_len = 0;
 302         if (opt)
 303                 icsk->icsk_ext_hdr_len = opt->opt_flen +
 304                                          opt->opt_nflen;
 305
 306         tp->rx_opt.mss_clamp = IPV6_MIN_MTU - sizeof(struct tcphdr) - sizeof(struct ipv6hdr);
 307
 308         inet->inet_dport = usin->sin6_port;
 309
 310         tcp_set_state(sk, TCP_SYN_SENT);
 311         err = inet6_hash_connect(tcp_death_row, sk);
 312         if (err)
 313                 goto late_failure;
 314
 315         sk_set_txhash(sk);
 316
 317         if (likely(!tp->repair)) {
 318                 if (!tp->write_seq)
 319                         WRITE_ONCE(tp->write_seq,
 320                                    secure_tcpv6_seq(np->saddr.s6_addr32,
 321                                                     sk->sk_v6_daddr.s6_addr32,
 322                                                     inet->inet_sport,
 323                                                     inet->inet_dport));
 324                 tp->tsoffset = secure_tcpv6_ts_off(sock_net(sk),
 325                                                    np->saddr.s6_addr32,
 326                                                    sk->sk_v6_daddr.s6_addr32);
 327         }
 328
 329         if (tcp_fastopen_defer_connect(sk, &err))
 330                 return err;
 331         if (err)
 332                 goto late_failure;
 333
 334         err = tcp_connect(sk);
 335         if (err)
 336                 goto late_failure;
 337
 338         return 0;
 339
 340 late_failure:
 341         tcp_set_state(sk, TCP_CLOSE);
 342 failure:
 343         inet->inet_dport = 0;
 344         sk->sk_route_caps = 0;
 345         return err;
 346 }
 347
 348 static void tcp_v6_mtu_reduced(struct sock *sk)
 349 {
 350         struct dst_entry *dst;
 351
 352         if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE))
 353                 return;
 354
 355         dst = inet6_csk_update_pmtu(sk, tcp_sk(sk)->mtu_info);
 356         if (!dst)
 357                 return;
 358
 359         if (inet_csk(sk)->icsk_pmtu_cookie > dst_mtu(dst)) {
 360                 tcp_sync_mss(sk, dst_mtu(dst));
 361                 tcp_simple_retransmit(sk);
 362         }
 363 }
 364
 365 static int tcp_v6_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
 366                 u8 type, u8 code, int offset, __be32 info)
 367 {
 368         const struct ipv6hdr *hdr = (const struct ipv6hdr *)skb->data;
 369         const struct tcphdr *th = (struct tcphdr *)(skb->data+offset);
 370         struct net *net = dev_net(skb->dev);
 371         struct request_sock *fastopen;
 372         struct ipv6_pinfo *np;
 373         struct tcp_sock *tp;
 374         __u32 seq, snd_una;
 375         struct sock *sk;
 376         bool fatal;
 377         int err;
 378
 379         sk = __inet6_lookup_established(net, &tcp_hashinfo,
 380                                         &hdr->daddr, th->dest,
 381                                         &hdr->saddr, ntohs(th->source),
 382                                         skb->dev->ifindex, inet6_sdif(skb));
 383
 384         if (!sk) {
 385                 __ICMP6_INC_STATS(net, __in6_dev_get(skb->dev),
 386                                   ICMP6_MIB_INERRORS);
 387                 return -ENOENT;
 388         }
 389
 390         if (sk->sk_state == TCP_TIME_WAIT) {
 391                 inet_twsk_put(inet_twsk(sk));
 392                 return 0;
 393         }
 394         seq = ntohl(th->seq);
 395         fatal = icmpv6_err_convert(type, code, &err);
 396         if (sk->sk_state == TCP_NEW_SYN_RECV) {
 397                 tcp_req_err(sk, seq, fatal);
 398                 return 0;
 399         }
 400
 401         bh_lock_sock(sk);
 402         if (sock_owned_by_user(sk) && type != ICMPV6_PKT_TOOBIG)
 403                 __NET_INC_STATS(net, LINUX_MIB_LOCKDROPPEDICMPS);
 404
 405         if (sk->sk_state == TCP_CLOSE)
 406                 goto out;
 407
 408         if (ipv6_hdr(skb)->hop_limit < tcp_inet6_sk(sk)->min_hopcount) {
 409                 __NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP);
 410                 goto out;
 411         }
 412
 413         tp = tcp_sk(sk);
 414         /* XXX (TFO) - tp->snd_una should be ISN (tcp_create_openreq_child() */
 415         fastopen = rcu_dereference(tp->fastopen_rsk);
 416         snd_una = fastopen ? tcp_rsk(fastopen)->snt_isn : tp->snd_una;
 417         if (sk->sk_state != TCP_LISTEN &&
 418             !between(seq, snd_una, tp->snd_nxt)) {
 419                 __NET_INC_STATS(net, LINUX_MIB_OUTOFWINDOWICMPS);
 420                 goto out;
 421         }
 422
 423         np = tcp_inet6_sk(sk);
 424
 425         if (type == NDISC_REDIRECT) {
 426                 if (!sock_owned_by_user(sk)) {
 427                         struct dst_entry *dst = __sk_dst_check(sk, np->dst_cookie);
 428
 429                         if (dst)
 430                                 dst->ops->redirect(dst, sk, skb);
 431                 }
 432                 goto out;
 433         }
 434
 435         if (type == ICMPV6_PKT_TOOBIG) {
 436                 /* We are not interested in TCP_LISTEN and open_requests
 437                  * (SYN-ACKs send out by Linux are always <576bytes so
 438                  * they should go through unfragmented).
 439                  */
 440                 if (sk->sk_state == TCP_LISTEN)
 441                         goto out;
 442
 443                 if (!ip6_sk_accept_pmtu(sk))
 444                         goto out;
 445
 446                 tp->mtu_info = ntohl(info);
 447                 if (!sock_owned_by_user(sk))
 448                         tcp_v6_mtu_reduced(sk);
 449                 else if (!test_and_set_bit(TCP_MTU_REDUCED_DEFERRED,
 450                                            &sk->sk_tsq_flags))
 451                         sock_hold(sk);
 452                 goto out;
 453         }
 454
 455
 456         /* Might be for an request_sock */
 457         switch (sk->sk_state) {
 458         case TCP_SYN_SENT:
 459         case TCP_SYN_RECV:
 460                 /* Only in fast or simultaneous open. If a fast open socket is
 461                  * already accepted it is treated as a connected one below.
 462                  */
 463                 if (fastopen && !fastopen->sk)
 464                         break;
 465
 466                 ipv6_icmp_error(sk, skb, err, th->dest, ntohl(info), (u8 *)th);
 467
 468                 if (!sock_owned_by_user(sk)) {
 469                         sk->sk_err = err;
 470                         sk_error_report(sk);            /* Wake people up to see the error (see connect in sock.c) */
 471
 472                         tcp_done(sk);
 473                 } else
 474                         sk->sk_err_soft = err;
 475                 goto out;
 476         case TCP_LISTEN:
 477                 break;
 478         default:
 479                 /* check if this ICMP message allows revert of backoff.
 480                  * (see RFC 6069)
 481                  */
 482                 if (!fastopen && type == ICMPV6_DEST_UNREACH &&
 483                     code == ICMPV6_NOROUTE)
 484                         tcp_ld_RTO_revert(sk, seq);
 485         }
 486
 487         if (!sock_owned_by_user(sk) && np->recverr) {
 488                 sk->sk_err = err;
 489                 sk_error_report(sk);
 490         } else
 491                 sk->sk_err_soft = err;
 492
 493 out:
 494         bh_unlock_sock(sk);
 495         sock_put(sk);
 496         return 0;
 497 }
 498
 499
 500 static int tcp_v6_send_synack(const struct sock *sk, struct dst_entry *dst,
 501                               struct flowi *fl,
 502                               struct request_sock *req,
 503                               struct tcp_fastopen_cookie *foc,
 504                               enum tcp_synack_type synack_type,
 505                               struct sk_buff *syn_skb)
 506 {
 507         struct inet_request_sock *ireq = inet_rsk(req);
 508         struct ipv6_pinfo *np = tcp_inet6_sk(sk);
 509         struct ipv6_txoptions *opt;
 510         struct flowi6 *fl6 = &fl->u.ip6;
 511         struct sk_buff *skb;
 512         int err = -ENOMEM;
 513         u8 tclass;
 514
 515         /* First, grab a route. */
 516         if (!dst && (dst = inet6_csk_route_req(sk, fl6, req,
 517                                                IPPROTO_TCP)) == NULL)
 518                 goto done;
 519
 520         skb = tcp_make_synack(sk, dst, req, foc, synack_type, syn_skb);
 521
 522         if (skb) {
 523                 __tcp_v6_send_check(skb, &ireq->ir_v6_loc_addr,
 524                                     &ireq->ir_v6_rmt_addr);
 525
 526                 fl6->daddr = ireq->ir_v6_rmt_addr;
 527                 if (np->repflow && ireq->pktopts)
 528                         fl6->flowlabel = ip6_flowlabel(ipv6_hdr(ireq->pktopts));
 529
 530                 tclass = sock_net(sk)->ipv4.sysctl_tcp_reflect_tos ?
 531                                 (tcp_rsk(req)->syn_tos & ~INET_ECN_MASK) |
 532                                 (np->tclass & INET_ECN_MASK) :
 533                                 np->tclass;
 534
 535                 if (!INET_ECN_is_capable(tclass) &&
 536                     tcp_bpf_ca_needs_ecn((struct sock *)req))
 537                         tclass |= INET_ECN_ECT_0;
 538
 539                 rcu_read_lock();
 540                 opt = ireq->ipv6_opt;
 541                 if (!opt)
 542                         opt = rcu_dereference(np->opt);
 543                 err = ip6_xmit(sk, skb, fl6, sk->sk_mark, opt,
 544                                tclass, sk->sk_priority);
 545                 rcu_read_unlock();
 546                 err = net_xmit_eval(err);
 547         }
 548
 549 done:
 550         return err;
 551 }
 552
 553
 554 static void tcp_v6_reqsk_destructor(struct request_sock *req)
 555 {
 556         kfree(inet_rsk(req)->ipv6_opt);
 557         kfree_skb(inet_rsk(req)->pktopts);
 558 }
 559
 560 #ifdef CONFIG_TCP_MD5SIG
 561 static struct tcp_md5sig_key *tcp_v6_md5_do_lookup(const struct sock *sk,
 562                                                    const struct in6_addr *addr,
 563                                                    int l3index)
 564 {
 565         return tcp_md5_do_lookup(sk, l3index,
 566                                  (union tcp_md5_addr *)addr, AF_INET6);
 567 }
 568
 569 static struct tcp_md5sig_key *tcp_v6_md5_lookup(const struct sock *sk,
 570                                                 const struct sock *addr_sk)
 571 {
 572         int l3index;
 573
 574         l3index = l3mdev_master_ifindex_by_index(sock_net(sk),
 575                                                  addr_sk->sk_bound_dev_if);
 576         return tcp_v6_md5_do_lookup(sk, &addr_sk->sk_v6_daddr,
 577                                     l3index);
 578 }
 579
 580 static int tcp_v6_parse_md5_keys(struct sock *sk, int optname,
 581                                  sockptr_t optval, int optlen)
 582 {
 583         struct tcp_md5sig cmd;
 584         struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&cmd.tcpm_addr;
 585         int l3index = 0;
 586         u8 prefixlen;
 587
 588         if (optlen < sizeof(cmd))
 589                 return -EINVAL;
 590
 591         if (copy_from_sockptr(&cmd, optval, sizeof(cmd)))
 592                 return -EFAULT;
 593
 594         if (sin6->sin6_family != AF_INET6)
 595                 return -EINVAL;
 596
 597         if (optname == TCP_MD5SIG_EXT &&
 598             cmd.tcpm_flags & TCP_MD5SIG_FLAG_PREFIX) {
 599                 prefixlen = cmd.tcpm_prefixlen;
 600                 if (prefixlen > 128 || (ipv6_addr_v4mapped(&sin6->sin6_addr) &&
 601                                         prefixlen > 32))
 602                         return -EINVAL;
 603         } else {
 604                 prefixlen = ipv6_addr_v4mapped(&sin6->sin6_addr) ? 32 : 128;
 605         }
 606
 607         if (optname == TCP_MD5SIG_EXT &&
 608             cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX) {
 609                 struct net_device *dev;
 610
 611                 rcu_read_lock();
 612                 dev = dev_get_by_index_rcu(sock_net(sk), cmd.tcpm_ifindex);
 613                 if (dev && netif_is_l3_master(dev))
 614                         l3index = dev->ifindex;
 615                 rcu_read_unlock();
 616
 617                 /* ok to reference set/not set outside of rcu;
 618                  * right now device MUST be an L3 master
 619                  */
 620                 if (!dev || !l3index)
 621                         return -EINVAL;
 622         }
 623
 624         if (!cmd.tcpm_keylen) {
 625                 if (ipv6_addr_v4mapped(&sin6->sin6_addr))
 626                         return tcp_md5_do_del(sk, (union tcp_md5_addr *)&sin6->sin6_addr.s6_addr32[3],
 627                                               AF_INET, prefixlen,
 628                                               l3index);
 629                 return tcp_md5_do_del(sk, (union tcp_md5_addr *)&sin6->sin6_addr,
 630                                       AF_INET6, prefixlen, l3index);
 631         }
 632
 633         if (cmd.tcpm_keylen > TCP_MD5SIG_MAXKEYLEN)
 634                 return -EINVAL;
 635
 636         if (ipv6_addr_v4mapped(&sin6->sin6_addr))
 637                 return tcp_md5_do_add(sk, (union tcp_md5_addr *)&sin6->sin6_addr.s6_addr32[3],
 638                                       AF_INET, prefixlen, l3index,
 639                                       cmd.tcpm_key, cmd.tcpm_keylen,
 640                                       GFP_KERNEL);
 641
 642         return tcp_md5_do_add(sk, (union tcp_md5_addr *)&sin6->sin6_addr,
 643                               AF_INET6, prefixlen, l3index,
 644                               cmd.tcpm_key, cmd.tcpm_keylen, GFP_KERNEL);
 645 }
 646
 647 static int tcp_v6_md5_hash_headers(struct tcp_md5sig_pool *hp,
 648                                    const struct in6_addr *daddr,
 649                                    const struct in6_addr *saddr,
 650                                    const struct tcphdr *th, int nbytes)
 651 {
 652         struct tcp6_pseudohdr *bp;
 653         struct scatterlist sg;
 654         struct tcphdr *_th;
 655
 656         bp = hp->scratch;
 657         /* 1. TCP pseudo-header (RFC2460) */
 658         bp->saddr = *saddr;
 659         bp->daddr = *daddr;
 660         bp->protocol = cpu_to_be32(IPPROTO_TCP);
 661         bp->len = cpu_to_be32(nbytes);
 662
 663         _th = (struct tcphdr *)(bp + 1);
 664         memcpy(_th, th, sizeof(*th));
 665         _th->check = 0;
 666
 667         sg_init_one(&sg, bp, sizeof(*bp) + sizeof(*th));
 668         ahash_request_set_crypt(hp->md5_req, &sg, NULL,
 669                                 sizeof(*bp) + sizeof(*th));
 670         return crypto_ahash_update(hp->md5_req);
 671 }
 672
 673 static int tcp_v6_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key *key,
 674                                const struct in6_addr *daddr, struct in6_addr *saddr,
 675                                const struct tcphdr *th)
 676 {
 677         struct tcp_md5sig_pool *hp;
 678         struct ahash_request *req;
 679
 680         hp = tcp_get_md5sig_pool();
 681         if (!hp)
 682                 goto clear_hash_noput;
 683         req = hp->md5_req;
 684
 685         if (crypto_ahash_init(req))
 686                 goto clear_hash;
 687         if (tcp_v6_md5_hash_headers(hp, daddr, saddr, th, th->doff << 2))
 688                 goto clear_hash;
 689         if (tcp_md5_hash_key(hp, key))
 690                 goto clear_hash;
 691         ahash_request_set_crypt(req, NULL, md5_hash, 0);
 692         if (crypto_ahash_final(req))
 693                 goto clear_hash;
 694
 695         tcp_put_md5sig_pool();
 696         return 0;
 697
 698 clear_hash:
 699         tcp_put_md5sig_pool();
 700 clear_hash_noput:
 701         memset(md5_hash, 0, 16);
 702         return 1;
 703 }
 704
 705 static int tcp_v6_md5_hash_skb(char *md5_hash,
 706                                const struct tcp_md5sig_key *key,
 707                                const struct sock *sk,
 708                                const struct sk_buff *skb)
 709 {
 710         const struct in6_addr *saddr, *daddr;
 711         struct tcp_md5sig_pool *hp;
 712         struct ahash_request *req;
 713         const struct tcphdr *th = tcp_hdr(skb);
 714
 715         if (sk) { /* valid for establish/request sockets */
 716                 saddr = &sk->sk_v6_rcv_saddr;
 717                 daddr = &sk->sk_v6_daddr;
 718         } else {
 719                 const struct ipv6hdr *ip6h = ipv6_hdr(skb);
 720                 saddr = &ip6h->saddr;
 721                 daddr = &ip6h->daddr;
 722         }
 723
 724         hp = tcp_get_md5sig_pool();
 725         if (!hp)
 726                 goto clear_hash_noput;
 727         req = hp->md5_req;
 728
 729         if (crypto_ahash_init(req))
 730                 goto clear_hash;
 731
 732         if (tcp_v6_md5_hash_headers(hp, daddr, saddr, th, skb->len))
 733                 goto clear_hash;
 734         if (tcp_md5_hash_skb_data(hp, skb, th->doff << 2))
 735                 goto clear_hash;
 736         if (tcp_md5_hash_key(hp, key))
 737                 goto clear_hash;
 738         ahash_request_set_crypt(req, NULL, md5_hash, 0);
 739         if (crypto_ahash_final(req))
 740                 goto clear_hash;
 741
 742         tcp_put_md5sig_pool();
 743         return 0;
 744
 745 clear_hash:
 746         tcp_put_md5sig_pool();
 747 clear_hash_noput:
 748         memset(md5_hash, 0, 16);
 749         return 1;
 750 }
 751
 752 #endif
 753
 754 static bool tcp_v6_inbound_md5_hash(const struct sock *sk,
 755                                     const struct sk_buff *skb,
 756                                     int dif, int sdif)
 757 {
 758 #ifdef CONFIG_TCP_MD5SIG
 759         const __u8 *hash_location = NULL;
 760         struct tcp_md5sig_key *hash_expected;
 761         const struct ipv6hdr *ip6h = ipv6_hdr(skb);
 762         const struct tcphdr *th = tcp_hdr(skb);
 763         int genhash, l3index;
 764         u8 newhash[16];
 765
 766         /* sdif set, means packet ingressed via a device
 767          * in an L3 domain and dif is set to the l3mdev
 768          */
 769         l3index = sdif ? dif : 0;
 770
 771         hash_expected = tcp_v6_md5_do_lookup(sk, &ip6h->saddr, l3index);
 772         hash_location = tcp_parse_md5sig_option(th);
 773
 774         /* We've parsed the options - do we have a hash? */
 775         if (!hash_expected && !hash_location)
 776                 return false;
 777
 778         if (hash_expected && !hash_location) {
 779                 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMD5NOTFOUND);
 780                 return true;
 781         }
 782
 783         if (!hash_expected && hash_location) {
 784                 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMD5UNEXPECTED);
 785                 return true;
 786         }
 787
 788         /* check the signature */
 789         genhash = tcp_v6_md5_hash_skb(newhash,
 790                                       hash_expected,
 791                                       NULL, skb);
 792
 793         if (genhash || memcmp(hash_location, newhash, 16) != 0) {
 794                 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMD5FAILURE);
 795                 net_info_ratelimited("MD5 Hash %s for [%pI6c]:%u->[%pI6c]:%u L3 index %d\n",
 796                                      genhash ? "failed" : "mismatch",
 797                                      &ip6h->saddr, ntohs(th->source),
 798                                      &ip6h->daddr, ntohs(th->dest), l3index);
 799                 return true;
 800         }
 801 #endif
 802         return false;
 803 }
 804
 805 static void tcp_v6_init_req(struct request_sock *req,
 806                             const struct sock *sk_listener,
 807                             struct sk_buff *skb)
 808 {
 809         bool l3_slave = ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags);
 810         struct inet_request_sock *ireq = inet_rsk(req);
 811         const struct ipv6_pinfo *np = tcp_inet6_sk(sk_listener);
 812
 813         ireq->ir_v6_rmt_addr = ipv6_hdr(skb)->saddr;
 814         ireq->ir_v6_loc_addr = ipv6_hdr(skb)->daddr;
 815
 816         /* So that link locals have meaning */
 817         if ((!sk_listener->sk_bound_dev_if || l3_slave) &&
 818             ipv6_addr_type(&ireq->ir_v6_rmt_addr) & IPV6_ADDR_LINKLOCAL)
 819                 ireq->ir_iif = tcp_v6_iif(skb);
 820
 821         if (!TCP_SKB_CB(skb)->tcp_tw_isn &&
 822             (ipv6_opt_accepted(sk_listener, skb, &TCP_SKB_CB(skb)->header.h6) ||
 823              np->rxopt.bits.rxinfo ||
 824              np->rxopt.bits.rxoinfo || np->rxopt.bits.rxhlim ||
 825              np->rxopt.bits.rxohlim || np->repflow)) {
 826                 refcount_inc(&skb->users);
 827                 ireq->pktopts = skb;
 828         }
 829 }
 830
 831 static struct dst_entry *tcp_v6_route_req(const struct sock *sk,
 832                                           struct sk_buff *skb,
 833                                           struct flowi *fl,
 834                                           struct request_sock *req)
 835 {
 836         tcp_v6_init_req(req, sk, skb);
 837
 838         if (security_inet_conn_request(sk, skb, req))
 839                 return NULL;
 840
 841         return inet6_csk_route_req(sk, &fl->u.ip6, req, IPPROTO_TCP);
 842 }
 843
 844 struct request_sock_ops tcp6_request_sock_ops __read_mostly = {
 845         .family         =       AF_INET6,
 846         .obj_size       =       sizeof(struct tcp6_request_sock),
 847         .rtx_syn_ack    =       tcp_rtx_synack,
 848         .send_ack       =       tcp_v6_reqsk_send_ack,
 849         .destructor     =       tcp_v6_reqsk_destructor,
 850         .send_reset     =       tcp_v6_send_reset,
 851         .syn_ack_timeout =      tcp_syn_ack_timeout,
 852 };
 853
 854 const struct tcp_request_sock_ops tcp_request_sock_ipv6_ops = {
 855         .mss_clamp      =       IPV6_MIN_MTU - sizeof(struct tcphdr) -
 856                                 sizeof(struct ipv6hdr),
 857 #ifdef CONFIG_TCP_MD5SIG
 858         .req_md5_lookup =       tcp_v6_md5_lookup,
 859         .calc_md5_hash  =       tcp_v6_md5_hash_skb,
 860 #endif
 861 #ifdef CONFIG_SYN_COOKIES
 862         .cookie_init_seq =      cookie_v6_init_sequence,
 863 #endif
 864         .route_req      =       tcp_v6_route_req,
 865         .init_seq       =       tcp_v6_init_seq,
 866         .init_ts_off    =       tcp_v6_init_ts_off,
 867         .send_synack    =       tcp_v6_send_synack,
 868 };
 869
 870 static void tcp_v6_send_response(const struct sock *sk, struct sk_buff *skb, u32 seq,
 871                                  u32 ack, u32 win, u32 tsval, u32 tsecr,
 872                                  int oif, struct tcp_md5sig_key *key, int rst,
 873                                  u8 tclass, __be32 label, u32 priority)
 874 {
 875         const struct tcphdr *th = tcp_hdr(skb);
 876         struct tcphdr *t1;
 877         struct sk_buff *buff;
 878         struct flowi6 fl6;
 879         struct net *net = sk ? sock_net(sk) : dev_net(skb_dst(skb)->dev);
 880         struct sock *ctl_sk = net->ipv6.tcp_sk;
 881         unsigned int tot_len = sizeof(struct tcphdr);
 882         __be32 mrst = 0, *topt;
 883         struct dst_entry *dst;
 884         __u32 mark = 0;
 885
 886         if (tsecr)
 887                 tot_len += TCPOLEN_TSTAMP_ALIGNED;
 888 #ifdef CONFIG_TCP_MD5SIG
 889         if (key)
 890                 tot_len += TCPOLEN_MD5SIG_ALIGNED;
 891 #endif
 892
 893 #ifdef CONFIG_MPTCP
 894         if (rst && !key) {
 895                 mrst = mptcp_reset_option(skb);
 896
 897                 if (mrst)
 898                         tot_len += sizeof(__be32);
 899         }
 900 #endif
 901
 902         buff = alloc_skb(MAX_HEADER + sizeof(struct ipv6hdr) + tot_len,
 903                          GFP_ATOMIC);
 904         if (!buff)
 905                 return;
 906
 907         skb_reserve(buff, MAX_HEADER + sizeof(struct ipv6hdr) + tot_len);
 908
 909         t1 = skb_push(buff, tot_len);
 910         skb_reset_transport_header(buff);
 911
 912         /* Swap the send and the receive. */
 913         memset(t1, 0, sizeof(*t1));
 914         t1->dest = th->source;
 915         t1->source = th->dest;
 916         t1->doff = tot_len / 4;
 917         t1->seq = htonl(seq);
 918         t1->ack_seq = htonl(ack);
 919         t1->ack = !rst || !th->ack;
 920         t1->rst = rst;
 921         t1->window = htons(win);
 922
 923         topt = (__be32 *)(t1 + 1);
 924
 925         if (tsecr) {
 926                 *topt++ = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) |
 927                                 (TCPOPT_TIMESTAMP << 8) | TCPOLEN_TIMESTAMP);
 928                 *topt++ = htonl(tsval);
 929                 *topt++ = htonl(tsecr);
 930         }
 931
 932         if (mrst)
 933                 *topt++ = mrst;
 934
 935 #ifdef CONFIG_TCP_MD5SIG
 936         if (key) {
 937                 *topt++ = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) |
 938                                 (TCPOPT_MD5SIG << 8) | TCPOLEN_MD5SIG);
 939                 tcp_v6_md5_hash_hdr((__u8 *)topt, key,
 940                                     &ipv6_hdr(skb)->saddr,
 941                                     &ipv6_hdr(skb)->daddr, t1);
 942         }
 943 #endif
 944
 945         memset(&fl6, 0, sizeof(fl6));
 946         fl6.daddr = ipv6_hdr(skb)->saddr;
 947         fl6.saddr = ipv6_hdr(skb)->daddr;
 948         fl6.flowlabel = label;
 949
 950         buff->ip_summed = CHECKSUM_PARTIAL;
 951         buff->csum = 0;
 952
 953         __tcp_v6_send_check(buff, &fl6.saddr, &fl6.daddr);
 954
 955         fl6.flowi6_proto = IPPROTO_TCP;
 956         if (rt6_need_strict(&fl6.daddr) && !oif)
 957                 fl6.flowi6_oif = tcp_v6_iif(skb);
 958         else {
 959                 if (!oif && netif_index_is_l3_master(net, skb->skb_iif))
 960                         oif = skb->skb_iif;
 961
 962                 fl6.flowi6_oif = oif;
 963         }
 964
 965         if (sk) {
 966                 if (sk->sk_state == TCP_TIME_WAIT) {
 967                         mark = inet_twsk(sk)->tw_mark;
 968                         /* autoflowlabel relies on buff->hash */
 969                         skb_set_hash(buff, inet_twsk(sk)->tw_txhash,
 970                                      PKT_HASH_TYPE_L4);
 971                 } else {
 972                         mark = sk->sk_mark;
 973                 }
 974                 buff->tstamp = tcp_transmit_time(sk);
 975         }
 976         fl6.flowi6_mark = IP6_REPLY_MARK(net, skb->mark) ?: mark;
 977         fl6.fl6_dport = t1->dest;
 978         fl6.fl6_sport = t1->source;
 979         fl6.flowi6_uid = sock_net_uid(net, sk && sk_fullsock(sk) ? sk : NULL);
 980         security_skb_classify_flow(skb, flowi6_to_flowi_common(&fl6));
 981
 982         /* Pass a socket to ip6_dst_lookup either it is for RST
 983          * Underlying function will use this to retrieve the network
 984          * namespace
 985          */
 986         dst = ip6_dst_lookup_flow(sock_net(ctl_sk), ctl_sk, &fl6, NULL);
 987         if (!IS_ERR(dst)) {
 988                 skb_dst_set(buff, dst);
 989                 ip6_xmit(ctl_sk, buff, &fl6, fl6.flowi6_mark, NULL,
 990                          tclass & ~INET_ECN_MASK, priority);
 991                 TCP_INC_STATS(net, TCP_MIB_OUTSEGS);
 992                 if (rst)
 993                         TCP_INC_STATS(net, TCP_MIB_OUTRSTS);
 994                 return;
 995         }
 996
 997         kfree_skb(buff);
 998 }
 999
1000 static void tcp_v6_send_reset(const struct sock *sk, struct sk_buff *skb)
1001 {
1002         const struct tcphdr *th = tcp_hdr(skb);
1003         struct ipv6hdr *ipv6h = ipv6_hdr(skb);
1004         u32 seq = 0, ack_seq = 0;
1005         struct tcp_md5sig_key *key = NULL;
1006 #ifdef CONFIG_TCP_MD5SIG
1007         const __u8 *hash_location = NULL;
1008         unsigned char newhash[16];
1009         int genhash;
1010         struct sock *sk1 = NULL;
1011 #endif
1012         __be32 label = 0;
1013         u32 priority = 0;
1014         struct net *net;
1015         int oif = 0;
1016
1017         if (th->rst)
1018                 return;
1019
1020         /* If sk not NULL, it means we did a successful lookup and incoming
1021          * route had to be correct. prequeue might have dropped our dst.
1022          */
1023         if (!sk && !ipv6_unicast_destination(skb))
1024                 return;
1025
1026         net = sk ? sock_net(sk) : dev_net(skb_dst(skb)->dev);
1027 #ifdef CONFIG_TCP_MD5SIG
1028         rcu_read_lock();
1029         hash_location = tcp_parse_md5sig_option(th);
1030         if (sk && sk_fullsock(sk)) {
1031                 int l3index;
1032
1033                 /* sdif set, means packet ingressed via a device
1034                  * in an L3 domain and inet_iif is set to it.
1035                  */
1036                 l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0;
1037                 key = tcp_v6_md5_do_lookup(sk, &ipv6h->saddr, l3index);
1038         } else if (hash_location) {
1039                 int dif = tcp_v6_iif_l3_slave(skb);
1040                 int sdif = tcp_v6_sdif(skb);
1041                 int l3index;
1042
1043                 /*
1044                  * active side is lost. Try to find listening socket through
1045                  * source port, and then find md5 key through listening socket.
1046                  * we are not loose security here:
1047                  * Incoming packet is checked with md5 hash with finding key,
1048                  * no RST generated if md5 hash doesn't match.
1049                  */
1050                 sk1 = inet6_lookup_listener(net,
1051                                            &tcp_hashinfo, NULL, 0,
1052                                            &ipv6h->saddr,
1053                                            th->source, &ipv6h->daddr,
1054                                            ntohs(th->source), dif, sdif);
1055                 if (!sk1)
1056                         goto out;
1057
1058                 /* sdif set, means packet ingressed via a device
1059                  * in an L3 domain and dif is set to it.
1060                  */
1061                 l3index = tcp_v6_sdif(skb) ? dif : 0;
1062
1063                 key = tcp_v6_md5_do_lookup(sk1, &ipv6h->saddr, l3index);
1064                 if (!key)
1065                         goto out;
1066
1067                 genhash = tcp_v6_md5_hash_skb(newhash, key, NULL, skb);
1068                 if (genhash || memcmp(hash_location, newhash, 16) != 0)
1069                         goto out;
1070         }
1071 #endif
1072
1073         if (th->ack)
1074                 seq = ntohl(th->ack_seq);
1075         else
1076                 ack_seq = ntohl(th->seq) + th->syn + th->fin + skb->len -
1077                           (th->doff << 2);
1078
1079         if (sk) {
1080                 oif = sk->sk_bound_dev_if;
1081                 if (sk_fullsock(sk)) {
1082                         const struct ipv6_pinfo *np = tcp_inet6_sk(sk);
1083
1084                         trace_tcp_send_reset(sk, skb);
1085                         if (np->repflow)
1086                                 label = ip6_flowlabel(ipv6h);
1087                         priority = sk->sk_priority;
1088                 }
1089                 if (sk->sk_state == TCP_TIME_WAIT) {
1090                         label = cpu_to_be32(inet_twsk(sk)->tw_flowlabel);
1091                         priority = inet_twsk(sk)->tw_priority;
1092                 }
1093         } else {
1094                 if (net->ipv6.sysctl.flowlabel_reflect & FLOWLABEL_REFLECT_TCP_RESET)
1095                         label = ip6_flowlabel(ipv6h);
1096         }
1097
1098         tcp_v6_send_response(sk, skb, seq, ack_seq, 0, 0, 0, oif, key, 1,
1099                              ipv6_get_dsfield(ipv6h), label, priority);
1100
1101 #ifdef CONFIG_TCP_MD5SIG
1102 out:
1103         rcu_read_unlock();
1104 #endif
1105 }
1106
1107 static void tcp_v6_send_ack(const struct sock *sk, struct sk_buff *skb, u32 seq,
1108                             u32 ack, u32 win, u32 tsval, u32 tsecr, int oif,
1109                             struct tcp_md5sig_key *key, u8 tclass,
1110                             __be32 label, u32 priority)
1111 {
1112         tcp_v6_send_response(sk, skb, seq, ack, win, tsval, tsecr, oif, key, 0,
1113                              tclass, label, priority);
1114 }
1115
1116 static void tcp_v6_timewait_ack(struct sock *sk, struct sk_buff *skb)
1117 {
1118         struct inet_timewait_sock *tw = inet_twsk(sk);
1119         struct tcp_timewait_sock *tcptw = tcp_twsk(sk);
1120
1121         tcp_v6_send_ack(sk, skb, tcptw->tw_snd_nxt, tcptw->tw_rcv_nxt,
1122                         tcptw->tw_rcv_wnd >> tw->tw_rcv_wscale,
1123                         tcp_time_stamp_raw() + tcptw->tw_ts_offset,
1124                         tcptw->tw_ts_recent, tw->tw_bound_dev_if, tcp_twsk_md5_key(tcptw),
1125                         tw->tw_tclass, cpu_to_be32(tw->tw_flowlabel), tw->tw_priority);
1126
1127         inet_twsk_put(tw);
1128 }
1129
1130 static void tcp_v6_reqsk_send_ack(const struct sock *sk, struct sk_buff *skb,
1131                                   struct request_sock *req)
1132 {
1133         int l3index;
1134
1135         l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0;
1136
1137         /* sk->sk_state == TCP_LISTEN -> for regular TCP_SYN_RECV
1138          * sk->sk_state == TCP_SYN_RECV -> for Fast Open.
1139          */
1140         /* RFC 7323 2.3
1141          * The window field (SEG.WND) of every outgoing segment, with the
1142          * exception of <SYN> segments, MUST be right-shifted by
1143          * Rcv.Wind.Shift bits:
1144          */
1145         tcp_v6_send_ack(sk, skb, (sk->sk_state == TCP_LISTEN) ?
1146                         tcp_rsk(req)->snt_isn + 1 : tcp_sk(sk)->snd_nxt,
1147                         tcp_rsk(req)->rcv_nxt,
1148                         req->rsk_rcv_wnd >> inet_rsk(req)->rcv_wscale,
1149                         tcp_time_stamp_raw() + tcp_rsk(req)->ts_off,
1150                         req->ts_recent, sk->sk_bound_dev_if,
1151                         tcp_v6_md5_do_lookup(sk, &ipv6_hdr(skb)->saddr, l3index),
1152                         ipv6_get_dsfield(ipv6_hdr(skb)), 0, sk->sk_priority);
1153 }
1154
1155
1156 static struct sock *tcp_v6_cookie_check(struct sock *sk, struct sk_buff *skb)
1157 {
1158 #ifdef CONFIG_SYN_COOKIES
1159         const struct tcphdr *th = tcp_hdr(skb);
1160
1161         if (!th->syn)
1162                 sk = cookie_v6_check(sk, skb);
1163 #endif
1164         return sk;
1165 }
1166
1167 u16 tcp_v6_get_syncookie(struct sock *sk, struct ipv6hdr *iph,
1168                          struct tcphdr *th, u32 *cookie)
1169 {
1170         u16 mss = 0;
1171 #ifdef CONFIG_SYN_COOKIES
1172         mss = tcp_get_syncookie_mss(&tcp6_request_sock_ops,
1173                                     &tcp_request_sock_ipv6_ops, sk, th);
1174         if (mss) {
1175                 *cookie = __cookie_v6_init_sequence(iph, th, &mss);
1176                 tcp_synq_overflow(sk);
1177         }
1178 #endif
1179         return mss;
1180 }
1181
1182 static int tcp_v6_conn_request(struct sock *sk, struct sk_buff *skb)
1183 {
1184         if (skb->protocol == htons(ETH_P_IP))
1185                 return tcp_v4_conn_request(sk, skb);
1186
1187         if (!ipv6_unicast_destination(skb))
1188                 goto drop;
1189
1190         if (ipv6_addr_v4mapped(&ipv6_hdr(skb)->saddr)) {
1191                 __IP6_INC_STATS(sock_net(sk), NULL, IPSTATS_MIB_INHDRERRORS);
1192                 return 0;
1193         }
1194
1195         return tcp_conn_request(&tcp6_request_sock_ops,
1196                                 &tcp_request_sock_ipv6_ops, sk, skb);
1197
1198 drop:
1199         tcp_listendrop(sk);
1200         return 0; /* don't send reset */
1201 }
1202
1203 static void tcp_v6_restore_cb(struct sk_buff *skb)
1204 {
1205         /* We need to move header back to the beginning if xfrm6_policy_check()
1206          * and tcp_v6_fill_cb() are going to be called again.
1207          * ip6_datagram_recv_specific_ctl() also expects IP6CB to be there.
1208          */
1209         memmove(IP6CB(skb), &TCP_SKB_CB(skb)->header.h6,
1210                 sizeof(struct inet6_skb_parm));
1211 }
1212
1213 static struct sock *tcp_v6_syn_recv_sock(const struct sock *sk, struct sk_buff *skb,
1214                                          struct request_sock *req,
1215                                          struct dst_entry *dst,
1216                                          struct request_sock *req_unhash,
1217                                          bool *own_req)
1218 {
1219         struct inet_request_sock *ireq;
1220         struct ipv6_pinfo *newnp;
1221         const struct ipv6_pinfo *np = tcp_inet6_sk(sk);
1222         struct ipv6_txoptions *opt;
1223         struct inet_sock *newinet;
1224         bool found_dup_sk = false;
1225         struct tcp_sock *newtp;
1226         struct sock *newsk;
1227 #ifdef CONFIG_TCP_MD5SIG
1228         struct tcp_md5sig_key *key;
1229         int l3index;
1230 #endif
1231         struct flowi6 fl6;
1232
1233         if (skb->protocol == htons(ETH_P_IP)) {
1234                 /*
1235                  *      v6 mapped
1236                  */
1237
1238                 newsk = tcp_v4_syn_recv_sock(sk, skb, req, dst,
1239                                              req_unhash, own_req);
1240
1241                 if (!newsk)
1242                         return NULL;
1243
1244                 inet_sk(newsk)->pinet6 = tcp_inet6_sk(newsk);
1245
1246                 newinet = inet_sk(newsk);
1247                 newnp = tcp_inet6_sk(newsk);
1248                 newtp = tcp_sk(newsk);
1249
1250                 memcpy(newnp, np, sizeof(struct ipv6_pinfo));
1251
1252                 newnp->saddr = newsk->sk_v6_rcv_saddr;
1253
1254                 inet_csk(newsk)->icsk_af_ops = &ipv6_mapped;
1255                 if (sk_is_mptcp(newsk))
1256                         mptcpv6_handle_mapped(newsk, true);
1257                 newsk->sk_backlog_rcv = tcp_v4_do_rcv;
1258 #ifdef CONFIG_TCP_MD5SIG
1259                 newtp->af_specific = &tcp_sock_ipv6_mapped_specific;
1260 #endif
1261
1262                 newnp->ipv6_mc_list = NULL;
1263                 newnp->ipv6_ac_list = NULL;
1264                 newnp->ipv6_fl_list = NULL;
1265                 newnp->pktoptions  = NULL;
1266                 newnp->opt         = NULL;
1267                 newnp->mcast_oif   = inet_iif(skb);
1268                 newnp->mcast_hops  = ip_hdr(skb)->ttl;
1269                 newnp->rcv_flowinfo = 0;
1270                 if (np->repflow)
1271                         newnp->flow_label = 0;
1272
1273                 /*
1274                  * No need to charge this sock to the relevant IPv6 refcnt debug socks count
1275                  * here, tcp_create_openreq_child now does this for us, see the comment in
1276                  * that function for the gory details. -acme
1277                  */
1278
1279                 /* It is tricky place. Until this moment IPv4 tcp
1280                    worked with IPv6 icsk.icsk_af_ops.
1281                    Sync it now.
1282                  */
1283                 tcp_sync_mss(newsk, inet_csk(newsk)->icsk_pmtu_cookie);
1284
1285                 return newsk;
1286         }
1287
1288         ireq = inet_rsk(req);
1289
1290         if (sk_acceptq_is_full(sk))
1291                 goto out_overflow;
1292
1293         if (!dst) {
1294                 dst = inet6_csk_route_req(sk, &fl6, req, IPPROTO_TCP);
1295                 if (!dst)
1296                         goto out;
1297         }
1298
1299         newsk = tcp_create_openreq_child(sk, req, skb);
1300         if (!newsk)
1301                 goto out_nonewsk;
1302
1303         /*
1304          * No need to charge this sock to the relevant IPv6 refcnt debug socks
1305          * count here, tcp_create_openreq_child now does this for us, see the
1306          * comment in that function for the gory details. -acme
1307          */
1308
1309         newsk->sk_gso_type = SKB_GSO_TCPV6;
1310         ip6_dst_store(newsk, dst, NULL, NULL);
1311         inet6_sk_rx_dst_set(newsk, skb);
1312
1313         inet_sk(newsk)->pinet6 = tcp_inet6_sk(newsk);
1314
1315         newtp = tcp_sk(newsk);
1316         newinet = inet_sk(newsk);
1317         newnp = tcp_inet6_sk(newsk);
1318
1319         memcpy(newnp, np, sizeof(struct ipv6_pinfo));
1320
1321         newsk->sk_v6_daddr = ireq->ir_v6_rmt_addr;
1322         newnp->saddr = ireq->ir_v6_loc_addr;
1323         newsk->sk_v6_rcv_saddr = ireq->ir_v6_loc_addr;
1324         newsk->sk_bound_dev_if = ireq->ir_iif;
1325
1326         /* Now IPv6 options...
1327
1328            First: no IPv4 options.
1329          */
1330         newinet->inet_opt = NULL;
1331         newnp->ipv6_mc_list = NULL;
1332         newnp->ipv6_ac_list = NULL;
1333         newnp->ipv6_fl_list = NULL;
1334
1335         /* Clone RX bits */
1336         newnp->rxopt.all = np->rxopt.all;
1337
1338         newnp->pktoptions = NULL;
1339         newnp->opt        = NULL;
1340         newnp->mcast_oif  = tcp_v6_iif(skb);
1341         newnp->mcast_hops = ipv6_hdr(skb)->hop_limit;
1342         newnp->rcv_flowinfo = ip6_flowinfo(ipv6_hdr(skb));
1343         if (np->repflow)
1344                 newnp->flow_label = ip6_flowlabel(ipv6_hdr(skb));
1345
1346         /* Set ToS of the new socket based upon the value of incoming SYN.
1347          * ECT bits are set later in tcp_init_transfer().
1348          */
1349         if (sock_net(sk)->ipv4.sysctl_tcp_reflect_tos)
1350                 newnp->tclass = tcp_rsk(req)->syn_tos & ~INET_ECN_MASK;
1351
1352         /* Clone native IPv6 options from listening socket (if any)
1353
1354            Yes, keeping reference count would be much more clever,
1355            but we make one more one thing there: reattach optmem
1356            to newsk.
1357          */
1358         opt = ireq->ipv6_opt;
1359         if (!opt)
1360                 opt = rcu_dereference(np->opt);
1361         if (opt) {
1362                 opt = ipv6_dup_options(newsk, opt);
1363                 RCU_INIT_POINTER(newnp->opt, opt);
1364         }
1365         inet_csk(newsk)->icsk_ext_hdr_len = 0;
1366         if (opt)
1367                 inet_csk(newsk)->icsk_ext_hdr_len = opt->opt_nflen +
1368                                                     opt->opt_flen;
1369
1370         tcp_ca_openreq_child(newsk, dst);
1371
1372         tcp_sync_mss(newsk, dst_mtu(dst));
1373         newtp->advmss = tcp_mss_clamp(tcp_sk(sk), dst_metric_advmss(dst));
1374
1375         tcp_initialize_rcv_mss(newsk);
1376
1377         newinet->inet_daddr = newinet->inet_saddr = LOOPBACK4_IPV6;
1378         newinet->inet_rcv_saddr = LOOPBACK4_IPV6;
1379
1380 #ifdef CONFIG_TCP_MD5SIG
1381         l3index = l3mdev_master_ifindex_by_index(sock_net(sk), ireq->ir_iif);
1382
1383         /* Copy over the MD5 key from the original socket */
1384         key = tcp_v6_md5_do_lookup(sk, &newsk->sk_v6_daddr, l3index);
1385         if (key) {
1386                 /* We're using one, so create a matching key
1387                  * on the newsk structure. If we fail to get
1388                  * memory, then we end up not copying the key
1389                  * across. Shucks.
1390                  */
1391                 tcp_md5_do_add(newsk, (union tcp_md5_addr *)&newsk->sk_v6_daddr,
1392                                AF_INET6, 128, l3index, key->key, key->keylen,
1393                                sk_gfp_mask(sk, GFP_ATOMIC));
1394         }
1395 #endif
1396
1397         if (__inet_inherit_port(sk, newsk) < 0) {
1398                 inet_csk_prepare_forced_close(newsk);
1399                 tcp_done(newsk);
1400                 goto out;
1401         }
1402         *own_req = inet_ehash_nolisten(newsk, req_to_sk(req_unhash),
1403                                        &found_dup_sk);
1404         if (*own_req) {
1405                 tcp_move_syn(newtp, req);
1406
1407                 /* Clone pktoptions received with SYN, if we own the req */
1408                 if (ireq->pktopts) {
1409                         newnp->pktoptions = skb_clone(ireq->pktopts,
1410                                                       sk_gfp_mask(sk, GFP_ATOMIC));
1411                         consume_skb(ireq->pktopts);
1412                         ireq->pktopts = NULL;
1413                         if (newnp->pktoptions) {
1414                                 tcp_v6_restore_cb(newnp->pktoptions);
1415                                 skb_set_owner_r(newnp->pktoptions, newsk);
1416                         }
1417                 }
1418         } else {
1419                 if (!req_unhash && found_dup_sk) {
1420                         /* This code path should only be executed in the
1421                          * syncookie case only
1422                          */
1423                         bh_unlock_sock(newsk);
1424                         sock_put(newsk);
1425                         newsk = NULL;
1426                 }
1427         }
1428
1429         return newsk;
1430
1431 out_overflow:
1432         __NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENOVERFLOWS);
1433 out_nonewsk:
1434         dst_release(dst);
1435 out:
1436         tcp_listendrop(sk);
1437         return NULL;
1438 }
1439
1440 INDIRECT_CALLABLE_DECLARE(struct dst_entry *ipv4_dst_check(struct dst_entry *,
1441                                                            u32));
1442 /* The socket must have it's spinlock held when we get
1443  * here, unless it is a TCP_LISTEN socket.
1444  *
1445  * We have a potential double-lock case here, so even when
1446  * doing backlog processing we use the BH locking scheme.
1447  * This is because we cannot sleep with the original spinlock
1448  * held.
1449  */
1450 static int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb)
1451 {
1452         struct ipv6_pinfo *np = tcp_inet6_sk(sk);
1453         struct sk_buff *opt_skb = NULL;
1454         struct tcp_sock *tp;
1455
1456         /* Imagine: socket is IPv6. IPv4 packet arrives,
1457            goes to IPv4 receive handler and backlogged.
1458            From backlog it always goes here. Kerboom...
1459            Fortunately, tcp_rcv_established and rcv_established
1460            handle them correctly, but it is not case with
1461            tcp_v6_hnd_req and tcp_v6_send_reset().   --ANK
1462          */
1463
1464         if (skb->protocol == htons(ETH_P_IP))
1465                 return tcp_v4_do_rcv(sk, skb);
1466
1467         /*
1468          *      socket locking is here for SMP purposes as backlog rcv
1469          *      is currently called with bh processing disabled.
1470          */
1471
1472         /* Do Stevens' IPV6_PKTOPTIONS.
1473
1474            Yes, guys, it is the only place in our code, where we
1475            may make it not affecting IPv4.
1476            The rest of code is protocol independent,
1477            and I do not like idea to uglify IPv4.
1478
1479            Actually, all the idea behind IPV6_PKTOPTIONS
1480            looks not very well thought. For now we latch
1481            options, received in the last packet, enqueued
1482            by tcp. Feel free to propose better solution.
1483                                                --ANK (980728)
1484          */
1485         if (np->rxopt.all)
1486                 opt_skb = skb_clone(skb, sk_gfp_mask(sk, GFP_ATOMIC));
1487
1488         if (sk->sk_state == TCP_ESTABLISHED) { /* Fast path */
1489                 struct dst_entry *dst = sk->sk_rx_dst;
1490
1491                 sock_rps_save_rxhash(sk, skb);
1492                 sk_mark_napi_id(sk, skb);
1493                 if (dst) {
1494                         if (inet_sk(sk)->rx_dst_ifindex != skb->skb_iif ||
1495                             INDIRECT_CALL_1(dst->ops->check, ip6_dst_check,
1496                                             dst, np->rx_dst_cookie) == NULL) {
1497                                 dst_release(dst);
1498                                 sk->sk_rx_dst = NULL;
1499                         }
1500                 }
1501
1502                 tcp_rcv_established(sk, skb);
1503                 if (opt_skb)
1504                         goto ipv6_pktoptions;
1505                 return 0;
1506         }
1507
1508         if (tcp_checksum_complete(skb))
1509                 goto csum_err;
1510
1511         if (sk->sk_state == TCP_LISTEN) {
1512                 struct sock *nsk = tcp_v6_cookie_check(sk, skb);
1513
1514                 if (!nsk)
1515                         goto discard;
1516
1517                 if (nsk != sk) {
1518                         if (tcp_child_process(sk, nsk, skb))
1519                                 goto reset;
1520                         if (opt_skb)
1521                                 __kfree_skb(opt_skb);
1522                         return 0;
1523                 }
1524         } else
1525                 sock_rps_save_rxhash(sk, skb);
1526
1527         if (tcp_rcv_state_process(sk, skb))
1528                 goto reset;
1529         if (opt_skb)
1530                 goto ipv6_pktoptions;
1531         return 0;
1532
1533 reset:
1534         tcp_v6_send_reset(sk, skb);
1535 discard:
1536         if (opt_skb)
1537                 __kfree_skb(opt_skb);
1538         kfree_skb(skb);
1539         return 0;
1540 csum_err:
1541         trace_tcp_bad_csum(skb);
1542         TCP_INC_STATS(sock_net(sk), TCP_MIB_CSUMERRORS);
1543         TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
1544         goto discard;
1545
1546
1547 ipv6_pktoptions:
1548         /* Do you ask, what is it?
1549
1550            1. skb was enqueued by tcp.
1551            2. skb is added to tail of read queue, rather than out of order.
1552            3. socket is not in passive state.
1553            4. Finally, it really contains options, which user wants to receive.
1554          */
1555         tp = tcp_sk(sk);
1556         if (TCP_SKB_CB(opt_skb)->end_seq == tp->rcv_nxt &&
1557             !((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN))) {
1558                 if (np->rxopt.bits.rxinfo || np->rxopt.bits.rxoinfo)
1559                         np->mcast_oif = tcp_v6_iif(opt_skb);
1560                 if (np->rxopt.bits.rxhlim || np->rxopt.bits.rxohlim)
1561                         np->mcast_hops = ipv6_hdr(opt_skb)->hop_limit;
1562                 if (np->rxopt.bits.rxflow || np->rxopt.bits.rxtclass)
1563                         np->rcv_flowinfo = ip6_flowinfo(ipv6_hdr(opt_skb));
1564                 if (np->repflow)
1565                         np->flow_label = ip6_flowlabel(ipv6_hdr(opt_skb));
1566                 if (ipv6_opt_accepted(sk, opt_skb, &TCP_SKB_CB(opt_skb)->header.h6)) {
1567                         skb_set_owner_r(opt_skb, sk);
1568                         tcp_v6_restore_cb(opt_skb);
1569                         opt_skb = xchg(&np->pktoptions, opt_skb);
1570                 } else {
1571                         __kfree_skb(opt_skb);
1572                         opt_skb = xchg(&np->pktoptions, NULL);
1573                 }
1574         }
1575
1576         kfree_skb(opt_skb);
1577         return 0;
1578 }
1579
1580 static void tcp_v6_fill_cb(struct sk_buff *skb, const struct ipv6hdr *hdr,
1581                            const struct tcphdr *th)
1582 {
1583         /* This is tricky: we move IP6CB at its correct location into
1584          * TCP_SKB_CB(). It must be done after xfrm6_policy_check(), because
1585          * _decode_session6() uses IP6CB().
1586          * barrier() makes sure compiler won't play aliasing games.
1587          */
1588         memmove(&TCP_SKB_CB(skb)->header.h6, IP6CB(skb),
1589                 sizeof(struct inet6_skb_parm));
1590         barrier();
1591
1592         TCP_SKB_CB(skb)->seq = ntohl(th->seq);
1593         TCP_SKB_CB(skb)->end_seq = (TCP_SKB_CB(skb)->seq + th->syn + th->fin +
1594                                     skb->len - th->doff*4);
1595         TCP_SKB_CB(skb)->ack_seq = ntohl(th->ack_seq);
1596         TCP_SKB_CB(skb)->tcp_flags = tcp_flag_byte(th);
1597         TCP_SKB_CB(skb)->tcp_tw_isn = 0;
1598         TCP_SKB_CB(skb)->ip_dsfield = ipv6_get_dsfield(hdr);
1599         TCP_SKB_CB(skb)->sacked = 0;
1600         TCP_SKB_CB(skb)->has_rxtstamp =
1601                         skb->tstamp || skb_hwtstamps(skb)->hwtstamp;
1602 }
1603
1604 INDIRECT_CALLABLE_SCOPE int tcp_v6_rcv(struct sk_buff *skb)
1605 {
1606         struct sk_buff *skb_to_free;
1607         int sdif = inet6_sdif(skb);
1608         int dif = inet6_iif(skb);
1609         const struct tcphdr *th;
1610         const struct ipv6hdr *hdr;
1611         bool refcounted;
1612         struct sock *sk;
1613         int ret;
1614         struct net *net = dev_net(skb->dev);
1615
1616         if (skb->pkt_type != PACKET_HOST)
1617                 goto discard_it;
1618
1619         /*
1620          *      Count it even if it's bad.
1621          */
1622         __TCP_INC_STATS(net, TCP_MIB_INSEGS);
1623
1624         if (!pskb_may_pull(skb, sizeof(struct tcphdr)))
1625                 goto discard_it;
1626
1627         th = (const struct tcphdr *)skb->data;
1628
1629         if (unlikely(th->doff < sizeof(struct tcphdr)/4))
1630                 goto bad_packet;
1631         if (!pskb_may_pull(skb, th->doff*4))
1632                 goto discard_it;
1633
1634         if (skb_checksum_init(skb, IPPROTO_TCP, ip6_compute_pseudo))
1635                 goto csum_error;
1636
1637         th = (const struct tcphdr *)skb->data;
1638         hdr = ipv6_hdr(skb);
1639
1640 lookup:
1641         sk = __inet6_lookup_skb(&tcp_hashinfo, skb, __tcp_hdrlen(th),
1642                                 th->source, th->dest, inet6_iif(skb), sdif,
1643                                 &refcounted);
1644         if (!sk)
1645                 goto no_tcp_socket;
1646
1647 process:
1648         if (sk->sk_state == TCP_TIME_WAIT)
1649                 goto do_time_wait;
1650
1651         if (sk->sk_state == TCP_NEW_SYN_RECV) {
1652                 struct request_sock *req = inet_reqsk(sk);
1653                 bool req_stolen = false;
1654                 struct sock *nsk;
1655
1656                 sk = req->rsk_listener;
1657                 if (tcp_v6_inbound_md5_hash(sk, skb, dif, sdif)) {
1658                         sk_drops_add(sk, skb);
1659                         reqsk_put(req);
1660                         goto discard_it;
1661                 }
1662                 if (tcp_checksum_complete(skb)) {
1663                         reqsk_put(req);
1664                         goto csum_error;
1665                 }
1666                 if (unlikely(sk->sk_state != TCP_LISTEN)) {
1667                         nsk = reuseport_migrate_sock(sk, req_to_sk(req), skb);
1668                         if (!nsk) {
1669                                 inet_csk_reqsk_queue_drop_and_put(sk, req);
1670                                 goto lookup;
1671                         }
1672                         sk = nsk;
1673                         /* reuseport_migrate_sock() has already held one sk_refcnt
1674                          * before returning.
1675                          */
1676                 } else {
1677                         sock_hold(sk);
1678                 }
1679                 refcounted = true;
1680                 nsk = NULL;
1681                 if (!tcp_filter(sk, skb)) {
1682                         th = (const struct tcphdr *)skb->data;
1683                         hdr = ipv6_hdr(skb);
1684                         tcp_v6_fill_cb(skb, hdr, th);
1685                         nsk = tcp_check_req(sk, skb, req, false, &req_stolen);
1686                 }
1687                 if (!nsk) {
1688                         reqsk_put(req);
1689                         if (req_stolen) {
1690                                 /* Another cpu got exclusive access to req
1691                                  * and created a full blown socket.
1692                                  * Try to feed this packet to this socket
1693                                  * instead of discarding it.
1694                                  */
1695                                 tcp_v6_restore_cb(skb);
1696                                 sock_put(sk);
1697                                 goto lookup;
1698                         }
1699                         goto discard_and_relse;
1700                 }
1701                 if (nsk == sk) {
1702                         reqsk_put(req);
1703                         tcp_v6_restore_cb(skb);
1704                 } else if (tcp_child_process(sk, nsk, skb)) {
1705                         tcp_v6_send_reset(nsk, skb);
1706                         goto discard_and_relse;
1707                 } else {
1708                         sock_put(sk);
1709                         return 0;
1710                 }
1711         }
1712         if (hdr->hop_limit < tcp_inet6_sk(sk)->min_hopcount) {
1713                 __NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP);
1714                 goto discard_and_relse;
1715         }
1716
1717         if (!xfrm6_policy_check(sk, XFRM_POLICY_IN, skb))
1718                 goto discard_and_relse;
1719
1720         if (tcp_v6_inbound_md5_hash(sk, skb, dif, sdif))
1721                 goto discard_and_relse;
1722
1723         if (tcp_filter(sk, skb))
1724                 goto discard_and_relse;
1725         th = (const struct tcphdr *)skb->data;
1726         hdr = ipv6_hdr(skb);
1727         tcp_v6_fill_cb(skb, hdr, th);
1728
1729         skb->dev = NULL;
1730
1731         if (sk->sk_state == TCP_LISTEN) {
1732                 ret = tcp_v6_do_rcv(sk, skb);
1733                 goto put_and_return;
1734         }
1735
1736         sk_incoming_cpu_update(sk);
1737
1738         bh_lock_sock_nested(sk);
1739         tcp_segs_in(tcp_sk(sk), skb);
1740         ret = 0;
1741         if (!sock_owned_by_user(sk)) {
1742                 skb_to_free = sk->sk_rx_skb_cache;
1743                 sk->sk_rx_skb_cache = NULL;
1744                 ret = tcp_v6_do_rcv(sk, skb);
1745         } else {
1746                 if (tcp_add_backlog(sk, skb))
1747                         goto discard_and_relse;
1748                 skb_to_free = NULL;
1749         }
1750         bh_unlock_sock(sk);
1751         if (skb_to_free)
1752                 __kfree_skb(skb_to_free);
1753 put_and_return:
1754         if (refcounted)
1755                 sock_put(sk);
1756         return ret ? -1 : 0;
1757
1758 no_tcp_socket:
1759         if (!xfrm6_policy_check(NULL, XFRM_POLICY_IN, skb))
1760                 goto discard_it;
1761
1762         tcp_v6_fill_cb(skb, hdr, th);
1763
1764         if (tcp_checksum_complete(skb)) {
1765 csum_error:
1766                 trace_tcp_bad_csum(skb);
1767                 __TCP_INC_STATS(net, TCP_MIB_CSUMERRORS);
1768 bad_packet:
1769                 __TCP_INC_STATS(net, TCP_MIB_INERRS);
1770         } else {
1771                 tcp_v6_send_reset(NULL, skb);
1772         }
1773
1774 discard_it:
1775         kfree_skb(skb);
1776         return 0;
1777
1778 discard_and_relse:
1779         sk_drops_add(sk, skb);
1780         if (refcounted)
1781                 sock_put(sk);
1782         goto discard_it;
1783
1784 do_time_wait:
1785         if (!xfrm6_policy_check(NULL, XFRM_POLICY_IN, skb)) {
1786                 inet_twsk_put(inet_twsk(sk));
1787                 goto discard_it;
1788         }
1789
1790         tcp_v6_fill_cb(skb, hdr, th);
1791
1792         if (tcp_checksum_complete(skb)) {
1793                 inet_twsk_put(inet_twsk(sk));
1794                 goto csum_error;
1795         }
1796
1797         switch (tcp_timewait_state_process(inet_twsk(sk), skb, th)) {
1798         case TCP_TW_SYN:
1799         {
1800                 struct sock *sk2;
1801
1802                 sk2 = inet6_lookup_listener(dev_net(skb->dev), &tcp_hashinfo,
1803                                             skb, __tcp_hdrlen(th),
1804                                             &ipv6_hdr(skb)->saddr, th->source,
1805                                             &ipv6_hdr(skb)->daddr,
1806                                             ntohs(th->dest),
1807                                             tcp_v6_iif_l3_slave(skb),
1808                                             sdif);
1809                 if (sk2) {
1810                         struct inet_timewait_sock *tw = inet_twsk(sk);
1811                         inet_twsk_deschedule_put(tw);
1812                         sk = sk2;
1813                         tcp_v6_restore_cb(skb);
1814                         refcounted = false;
1815                         goto process;
1816                 }
1817         }
1818                 /* to ACK */
1819                 fallthrough;
1820         case TCP_TW_ACK:
1821                 tcp_v6_timewait_ack(sk, skb);
1822                 break;
1823         case TCP_TW_RST:
1824                 tcp_v6_send_reset(sk, skb);
1825                 inet_twsk_deschedule_put(inet_twsk(sk));
1826                 goto discard_it;
1827         case TCP_TW_SUCCESS:
1828                 ;
1829         }
1830         goto discard_it;
1831 }
1832
1833 INDIRECT_CALLABLE_SCOPE void tcp_v6_early_demux(struct sk_buff *skb)
1834 {
1835         const struct ipv6hdr *hdr;
1836         const struct tcphdr *th;
1837         struct sock *sk;
1838
1839         if (skb->pkt_type != PACKET_HOST)
1840                 return;
1841
1842         if (!pskb_may_pull(skb, skb_transport_offset(skb) + sizeof(struct tcphdr)))
1843                 return;
1844
1845         hdr = ipv6_hdr(skb);
1846         th = tcp_hdr(skb);
1847
1848         if (th->doff < sizeof(struct tcphdr) / 4)
1849                 return;
1850
1851         /* Note : We use inet6_iif() here, not tcp_v6_iif() */
1852         sk = __inet6_lookup_established(dev_net(skb->dev), &tcp_hashinfo,
1853                                         &hdr->saddr, th->source,
1854                                         &hdr->daddr, ntohs(th->dest),
1855                                         inet6_iif(skb), inet6_sdif(skb));
1856         if (sk) {
1857                 skb->sk = sk;
1858                 skb->destructor = sock_edemux;
1859                 if (sk_fullsock(sk)) {
1860                         struct dst_entry *dst = READ_ONCE(sk->sk_rx_dst);
1861
1862                         if (dst)
1863                                 dst = dst_check(dst, tcp_inet6_sk(sk)->rx_dst_cookie);
1864                         if (dst &&
1865                             inet_sk(sk)->rx_dst_ifindex == skb->skb_iif)
1866                                 skb_dst_set_noref(skb, dst);
1867                 }
1868         }
1869 }
1870
1871 static struct timewait_sock_ops tcp6_timewait_sock_ops = {
1872         .twsk_obj_size  = sizeof(struct tcp6_timewait_sock),
1873         .twsk_unique    = tcp_twsk_unique,
1874         .twsk_destructor = tcp_twsk_destructor,
1875 };
1876
1877 INDIRECT_CALLABLE_SCOPE void tcp_v6_send_check(struct sock *sk, struct sk_buff *skb)
1878 {
1879         struct ipv6_pinfo *np = inet6_sk(sk);
1880
1881         __tcp_v6_send_check(skb, &np->saddr, &sk->sk_v6_daddr);
1882 }
1883
1884 const struct inet_connection_sock_af_ops ipv6_specific = {
1885         .queue_xmit        = inet6_csk_xmit,
1886         .send_check        = tcp_v6_send_check,
1887         .rebuild_header    = inet6_sk_rebuild_header,
1888         .sk_rx_dst_set     = inet6_sk_rx_dst_set,
1889         .conn_request      = tcp_v6_conn_request,
1890         .syn_recv_sock     = tcp_v6_syn_recv_sock,
1891         .net_header_len    = sizeof(struct ipv6hdr),
1892         .net_frag_header_len = sizeof(struct frag_hdr),
1893         .setsockopt        = ipv6_setsockopt,
1894         .getsockopt        = ipv6_getsockopt,
1895         .addr2sockaddr     = inet6_csk_addr2sockaddr,
1896         .sockaddr_len      = sizeof(struct sockaddr_in6),
1897         .mtu_reduced       = tcp_v6_mtu_reduced,
1898 };
1899
1900 #ifdef CONFIG_TCP_MD5SIG
1901 static const struct tcp_sock_af_ops tcp_sock_ipv6_specific = {
1902         .md5_lookup     =       tcp_v6_md5_lookup,
1903         .calc_md5_hash  =       tcp_v6_md5_hash_skb,
1904         .md5_parse      =       tcp_v6_parse_md5_keys,
1905 };
1906 #endif
1907
1908 /*
1909  *      TCP over IPv4 via INET6 API
1910  */
1911 static const struct inet_connection_sock_af_ops ipv6_mapped = {
1912         .queue_xmit        = ip_queue_xmit,
1913         .send_check        = tcp_v4_send_check,
1914         .rebuild_header    = inet_sk_rebuild_header,
1915         .sk_rx_dst_set     = inet_sk_rx_dst_set,
1916         .conn_request      = tcp_v6_conn_request,
1917         .syn_recv_sock     = tcp_v6_syn_recv_sock,
1918         .net_header_len    = sizeof(struct iphdr),
1919         .setsockopt        = ipv6_setsockopt,
1920         .getsockopt        = ipv6_getsockopt,
1921         .addr2sockaddr     = inet6_csk_addr2sockaddr,
1922         .sockaddr_len      = sizeof(struct sockaddr_in6),
1923         .mtu_reduced       = tcp_v4_mtu_reduced,
1924 };
1925
1926 #ifdef CONFIG_TCP_MD5SIG
1927 static const struct tcp_sock_af_ops tcp_sock_ipv6_mapped_specific = {
1928         .md5_lookup     =       tcp_v4_md5_lookup,
1929         .calc_md5_hash  =       tcp_v4_md5_hash_skb,
1930         .md5_parse      =       tcp_v6_parse_md5_keys,
1931 };
1932 #endif
1933
1934 /* NOTE: A lot of things set to zero explicitly by call to
1935  *       sk_alloc() so need not be done here.
1936  */
1937 static int tcp_v6_init_sock(struct sock *sk)
1938 {
1939         struct inet_connection_sock *icsk = inet_csk(sk);
1940
1941         tcp_init_sock(sk);
1942
1943         icsk->icsk_af_ops = &ipv6_specific;
1944
1945 #ifdef CONFIG_TCP_MD5SIG
1946         tcp_sk(sk)->af_specific = &tcp_sock_ipv6_specific;
1947 #endif
1948
1949         return 0;
1950 }
1951
1952 static void tcp_v6_destroy_sock(struct sock *sk)
1953 {
1954         tcp_v4_destroy_sock(sk);
1955         inet6_destroy_sock(sk);
1956 }
1957
1958 #ifdef CONFIG_PROC_FS
1959 /* Proc filesystem TCPv6 sock list dumping. */
1960 static void get_openreq6(struct seq_file *seq,
1961                          const struct request_sock *req, int i)
1962 {
1963         long ttd = req->rsk_timer.expires - jiffies;
1964         const struct in6_addr *src = &inet_rsk(req)->ir_v6_loc_addr;
1965         const struct in6_addr *dest = &inet_rsk(req)->ir_v6_rmt_addr;
1966
1967         if (ttd < 0)
1968                 ttd = 0;
1969
1970         seq_printf(seq,
1971                    "%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X "
1972                    "%02X %08X:%08X %02X:%08lX %08X %5u %8d %d %d %pK\n",
1973                    i,
1974                    src->s6_addr32[0], src->s6_addr32[1],
1975                    src->s6_addr32[2], src->s6_addr32[3],
1976                    inet_rsk(req)->ir_num,
1977                    dest->s6_addr32[0], dest->s6_addr32[1],
1978                    dest->s6_addr32[2], dest->s6_addr32[3],
1979                    ntohs(inet_rsk(req)->ir_rmt_port),
1980                    TCP_SYN_RECV,
1981                    0, 0, /* could print option size, but that is af dependent. */
1982                    1,   /* timers active (only the expire timer) */
1983                    jiffies_to_clock_t(ttd),
1984                    req->num_timeout,
1985                    from_kuid_munged(seq_user_ns(seq),
1986                                     sock_i_uid(req->rsk_listener)),
1987                    0,  /* non standard timer */
1988                    0, /* open_requests have no inode */
1989                    0, req);
1990 }
1991
1992 static void get_tcp6_sock(struct seq_file *seq, struct sock *sp, int i)
1993 {
1994         const struct in6_addr *dest, *src;
1995         __u16 destp, srcp;
1996         int timer_active;
1997         unsigned long timer_expires;
1998         const struct inet_sock *inet = inet_sk(sp);
1999         const struct tcp_sock *tp = tcp_sk(sp);
2000         const struct inet_connection_sock *icsk = inet_csk(sp);
2001         const struct fastopen_queue *fastopenq = &icsk->icsk_accept_queue.fastopenq;
2002         int rx_queue;
2003         int state;
2004
2005         dest  = &sp->sk_v6_daddr;
2006         src   = &sp->sk_v6_rcv_saddr;
2007         destp = ntohs(inet->inet_dport);
2008         srcp  = ntohs(inet->inet_sport);
2009
2010         if (icsk->icsk_pending == ICSK_TIME_RETRANS ||
2011             icsk->icsk_pending == ICSK_TIME_REO_TIMEOUT ||
2012             icsk->icsk_pending == ICSK_TIME_LOSS_PROBE) {
2013                 timer_active    = 1;
2014                 timer_expires   = icsk->icsk_timeout;
2015         } else if (icsk->icsk_pending == ICSK_TIME_PROBE0) {
2016                 timer_active    = 4;
2017                 timer_expires   = icsk->icsk_timeout;
2018         } else if (timer_pending(&sp->sk_timer)) {
2019                 timer_active    = 2;
2020                 timer_expires   = sp->sk_timer.expires;
2021         } else {
2022                 timer_active    = 0;
2023                 timer_expires = jiffies;
2024         }
2025
2026         state = inet_sk_state_load(sp);
2027         if (state == TCP_LISTEN)
2028                 rx_queue = READ_ONCE(sp->sk_ack_backlog);
2029         else
2030                 /* Because we don't lock the socket,
2031                  * we might find a transient negative value.
2032                  */
2033                 rx_queue = max_t(int, READ_ONCE(tp->rcv_nxt) -
2034                                       READ_ONCE(tp->copied_seq), 0);
2035
2036         seq_printf(seq,
2037                    "%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X "
2038                    "%02X %08X:%08X %02X:%08lX %08X %5u %8d %lu %d %pK %lu %lu %u %u %d\n",
2039                    i,
2040                    src->s6_addr32[0], src->s6_addr32[1],
2041                    src->s6_addr32[2], src->s6_addr32[3], srcp,
2042                    dest->s6_addr32[0], dest->s6_addr32[1],
2043                    dest->s6_addr32[2], dest->s6_addr32[3], destp,
2044                    state,
2045                    READ_ONCE(tp->write_seq) - tp->snd_una,
2046                    rx_queue,
2047                    timer_active,
2048                    jiffies_delta_to_clock_t(timer_expires - jiffies),
2049                    icsk->icsk_retransmits,
2050                    from_kuid_munged(seq_user_ns(seq), sock_i_uid(sp)),
2051                    icsk->icsk_probes_out,
2052                    sock_i_ino(sp),
2053                    refcount_read(&sp->sk_refcnt), sp,
2054                    jiffies_to_clock_t(icsk->icsk_rto),
2055                    jiffies_to_clock_t(icsk->icsk_ack.ato),
2056                    (icsk->icsk_ack.quick << 1) | inet_csk_in_pingpong_mode(sp),
2057                    tp->snd_cwnd,
2058                    state == TCP_LISTEN ?
2059                         fastopenq->max_qlen :
2060                         (tcp_in_initial_slowstart(tp) ? -1 : tp->snd_ssthresh)
2061                    );
2062 }
2063
2064 static void get_timewait6_sock(struct seq_file *seq,
2065                                struct inet_timewait_sock *tw, int i)
2066 {
2067         long delta = tw->tw_timer.expires - jiffies;
2068         const struct in6_addr *dest, *src;
2069         __u16 destp, srcp;
2070
2071         dest = &tw->tw_v6_daddr;
2072         src  = &tw->tw_v6_rcv_saddr;
2073         destp = ntohs(tw->tw_dport);
2074         srcp  = ntohs(tw->tw_sport);
2075
2076         seq_printf(seq,
2077                    "%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X "
2078                    "%02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d %pK\n",
2079                    i,
2080                    src->s6_addr32[0], src->s6_addr32[1],
2081                    src->s6_addr32[2], src->s6_addr32[3], srcp,
2082                    dest->s6_addr32[0], dest->s6_addr32[1],
2083                    dest->s6_addr32[2], dest->s6_addr32[3], destp,
2084                    tw->tw_substate, 0, 0,
2085                    3, jiffies_delta_to_clock_t(delta), 0, 0, 0, 0,
2086                    refcount_read(&tw->tw_refcnt), tw);
2087 }
2088
2089 static int tcp6_seq_show(struct seq_file *seq, void *v)
2090 {
2091         struct tcp_iter_state *st;
2092         struct sock *sk = v;
2093
2094         if (v == SEQ_START_TOKEN) {
2095                 seq_puts(seq,
2096                          "  sl  "
2097                          "local_address                         "
2098                          "remote_address                        "
2099                          "st tx_queue rx_queue tr tm->when retrnsmt"
2100                          "   uid  timeout inode\n");
2101                 goto out;
2102         }
2103         st = seq->private;
2104
2105         if (sk->sk_state == TCP_TIME_WAIT)
2106                 get_timewait6_sock(seq, v, st->num);
2107         else if (sk->sk_state == TCP_NEW_SYN_RECV)
2108                 get_openreq6(seq, v, st->num);
2109         else
2110                 get_tcp6_sock(seq, v, st->num);
2111 out:
2112         return 0;
2113 }
2114
2115 static const struct seq_operations tcp6_seq_ops = {
2116         .show           = tcp6_seq_show,
2117         .start          = tcp_seq_start,
2118         .next           = tcp_seq_next,
2119         .stop           = tcp_seq_stop,
2120 };
2121
2122 static struct tcp_seq_afinfo tcp6_seq_afinfo = {
2123         .family         = AF_INET6,
2124 };
2125
2126 int __net_init tcp6_proc_init(struct net *net)
2127 {
2128         if (!proc_create_net_data("tcp6", 0444, net->proc_net, &tcp6_seq_ops,
2129                         sizeof(struct tcp_iter_state), &tcp6_seq_afinfo))
2130                 return -ENOMEM;
2131         return 0;
2132 }
2133
2134 void tcp6_proc_exit(struct net *net)
2135 {
2136         remove_proc_entry("tcp6", net->proc_net);
2137 }
2138 #endif
2139
2140 struct proto tcpv6_prot = {
2141         .name                   = "TCPv6",
2142         .owner                  = THIS_MODULE,
2143         .close                  = tcp_close,
2144         .pre_connect            = tcp_v6_pre_connect,
2145         .connect                = tcp_v6_connect,
2146         .disconnect             = tcp_disconnect,
2147         .accept                 = inet_csk_accept,
2148         .ioctl                  = tcp_ioctl,
2149         .init                   = tcp_v6_init_sock,
2150         .destroy                = tcp_v6_destroy_sock,
2151         .shutdown               = tcp_shutdown,
2152         .setsockopt             = tcp_setsockopt,
2153         .getsockopt             = tcp_getsockopt,
2154         .bpf_bypass_getsockopt  = tcp_bpf_bypass_getsockopt,
2155         .keepalive              = tcp_set_keepalive,
2156         .recvmsg                = tcp_recvmsg,
2157         .sendmsg                = tcp_sendmsg,
2158         .sendpage               = tcp_sendpage,
2159         .backlog_rcv            = tcp_v6_do_rcv,
2160         .release_cb             = tcp_release_cb,
2161         .hash                   = inet6_hash,
2162         .unhash                 = inet_unhash,
2163         .get_port               = inet_csk_get_port,
2164 #ifdef CONFIG_BPF_SYSCALL
2165         .psock_update_sk_prot   = tcp_bpf_update_proto,
2166 #endif
2167         .enter_memory_pressure  = tcp_enter_memory_pressure,
2168         .leave_memory_pressure  = tcp_leave_memory_pressure,
2169         .stream_memory_free     = tcp_stream_memory_free,
2170         .sockets_allocated      = &tcp_sockets_allocated,
2171         .memory_allocated       = &tcp_memory_allocated,
2172         .memory_pressure        = &tcp_memory_pressure,
2173         .orphan_count           = &tcp_orphan_count,
2174         .sysctl_mem             = sysctl_tcp_mem,
2175         .sysctl_wmem_offset     = offsetof(struct net, ipv4.sysctl_tcp_wmem),
2176         .sysctl_rmem_offset     = offsetof(struct net, ipv4.sysctl_tcp_rmem),
2177         .max_header             = MAX_TCP_HEADER,
2178         .obj_size               = sizeof(struct tcp6_sock),
2179         .slab_flags             = SLAB_TYPESAFE_BY_RCU,
2180         .twsk_prot              = &tcp6_timewait_sock_ops,
2181         .rsk_prot               = &tcp6_request_sock_ops,
2182         .h.hashinfo             = &tcp_hashinfo,
2183         .no_autobind            = true,
2184         .diag_destroy           = tcp_abort,
2185 };
2186 EXPORT_SYMBOL_GPL(tcpv6_prot);
2187
2188 /* thinking of making this const? Don't.
2189  * early_demux can change based on sysctl.
2190  */
2191 static struct inet6_protocol tcpv6_protocol = {
2192         .early_demux    =       tcp_v6_early_demux,
2193         .early_demux_handler =  tcp_v6_early_demux,
2194         .handler        =       tcp_v6_rcv,
2195         .err_handler    =       tcp_v6_err,
2196         .flags          =       INET6_PROTO_NOPOLICY|INET6_PROTO_FINAL,
2197 };
2198
2199 static struct inet_protosw tcpv6_protosw = {
2200         .type           =       SOCK_STREAM,
2201         .protocol       =       IPPROTO_TCP,
2202         .prot           =       &tcpv6_prot,
2203         .ops            =       &inet6_stream_ops,
2204         .flags          =       INET_PROTOSW_PERMANENT |
2205                                 INET_PROTOSW_ICSK,
2206 };
2207
2208 static int __net_init tcpv6_net_init(struct net *net)
2209 {
2210         return inet_ctl_sock_create(&net->ipv6.tcp_sk, PF_INET6,
2211                                     SOCK_RAW, IPPROTO_TCP, net);
2212 }
2213
2214 static void __net_exit tcpv6_net_exit(struct net *net)
2215 {
2216         inet_ctl_sock_destroy(net->ipv6.tcp_sk);
2217 }
2218
2219 static void __net_exit tcpv6_net_exit_batch(struct list_head *net_exit_list)
2220 {
2221         inet_twsk_purge(&tcp_hashinfo, AF_INET6);
2222 }
2223
2224 static struct pernet_operations tcpv6_net_ops = {
2225         .init       = tcpv6_net_init,
2226         .exit       = tcpv6_net_exit,
2227         .exit_batch = tcpv6_net_exit_batch,
2228 };
2229
2230 int __init tcpv6_init(void)
2231 {
2232         int ret;
2233
2234         ret = inet6_add_protocol(&tcpv6_protocol, IPPROTO_TCP);
2235         if (ret)
2236                 goto out;
2237
2238         /* register inet6 protocol */
2239         ret = inet6_register_protosw(&tcpv6_protosw);
2240         if (ret)
2241                 goto out_tcpv6_protocol;
2242
2243         ret = register_pernet_subsys(&tcpv6_net_ops);
2244         if (ret)
2245                 goto out_tcpv6_protosw;
2246
2247         ret = mptcpv6_init();
2248         if (ret)
2249                 goto out_tcpv6_pernet_subsys;
2250
2251 out:
2252         return ret;
2253
2254 out_tcpv6_pernet_subsys:
2255         unregister_pernet_subsys(&tcpv6_net_ops);
2256 out_tcpv6_protosw:
2257         inet6_unregister_protosw(&tcpv6_protosw);
2258 out_tcpv6_protocol:
2259         inet6_del_protocol(&tcpv6_protocol, IPPROTO_TCP);
2260         goto out;
2261 }
2262
2263 void tcpv6_exit(void)
2264 {
2265         unregister_pernet_subsys(&tcpv6_net_ops);
2266         inet6_unregister_protosw(&tcpv6_protosw);
2267         inet6_del_protocol(&tcpv6_protocol, IPPROTO_TCP);
2268 }