net/netfilter/nf_conntrack_core.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Connection state tracking for netfilter.  This is separated from,
   3    but required by, the NAT layer; it can also be used by an iptables
   4    extension. */
   5
   6 /* (C) 1999-2001 Paul `Rusty' Russell
   7  * (C) 2002-2006 Netfilter Core Team <coreteam@netfilter.org>
   8  * (C) 2003,2004 USAGI/WIDE Project <http://www.linux-ipv6.org>
   9  * (C) 2005-2012 Patrick McHardy <kaber@trash.net>
  10  */
  11
  12 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  13
  14 #include <linux/types.h>
  15 #include <linux/netfilter.h>
  16 #include <linux/module.h>
  17 #include <linux/sched.h>
  18 #include <linux/skbuff.h>
  19 #include <linux/proc_fs.h>
  20 #include <linux/vmalloc.h>
  21 #include <linux/stddef.h>
  22 #include <linux/slab.h>
  23 #include <linux/random.h>
  24 #include <linux/jhash.h>
  25 #include <linux/siphash.h>
  26 #include <linux/err.h>
  27 #include <linux/percpu.h>
  28 #include <linux/moduleparam.h>
  29 #include <linux/notifier.h>
  30 #include <linux/kernel.h>
  31 #include <linux/netdevice.h>
  32 #include <linux/socket.h>
  33 #include <linux/mm.h>
  34 #include <linux/nsproxy.h>
  35 #include <linux/rculist_nulls.h>
  36
  37 #include <net/netfilter/nf_conntrack.h>
  38 #include <net/netfilter/nf_conntrack_l4proto.h>
  39 #include <net/netfilter/nf_conntrack_expect.h>
  40 #include <net/netfilter/nf_conntrack_helper.h>
  41 #include <net/netfilter/nf_conntrack_seqadj.h>
  42 #include <net/netfilter/nf_conntrack_core.h>
  43 #include <net/netfilter/nf_conntrack_extend.h>
  44 #include <net/netfilter/nf_conntrack_acct.h>
  45 #include <net/netfilter/nf_conntrack_ecache.h>
  46 #include <net/netfilter/nf_conntrack_zones.h>
  47 #include <net/netfilter/nf_conntrack_timestamp.h>
  48 #include <net/netfilter/nf_conntrack_timeout.h>
  49 #include <net/netfilter/nf_conntrack_labels.h>
  50 #include <net/netfilter/nf_conntrack_synproxy.h>
  51 #include <net/netfilter/nf_nat.h>
  52 #include <net/netfilter/nf_nat_helper.h>
  53 #include <net/netns/hash.h>
  54 #include <net/ip.h>
  55
  56 #include "nf_internals.h"
  57
  58 __cacheline_aligned_in_smp spinlock_t nf_conntrack_locks[CONNTRACK_LOCKS];
  59 EXPORT_SYMBOL_GPL(nf_conntrack_locks);
  60
  61 __cacheline_aligned_in_smp DEFINE_SPINLOCK(nf_conntrack_expect_lock);
  62 EXPORT_SYMBOL_GPL(nf_conntrack_expect_lock);
  63
  64 struct hlist_nulls_head *nf_conntrack_hash __read_mostly;
  65 EXPORT_SYMBOL_GPL(nf_conntrack_hash);
  66
  67 struct conntrack_gc_work {
  68         struct delayed_work     dwork;
  69         u32                     last_bucket;
  70         bool                    exiting;
  71         bool                    early_drop;
  72         long                    next_gc_run;
  73 };
  74
  75 static __read_mostly struct kmem_cache *nf_conntrack_cachep;
  76 static DEFINE_SPINLOCK(nf_conntrack_locks_all_lock);
  77 static __read_mostly bool nf_conntrack_locks_all;
  78
  79 /* every gc cycle scans at most 1/GC_MAX_BUCKETS_DIV part of table */
  80 #define GC_MAX_BUCKETS_DIV      128u
  81 /* upper bound of full table scan */
  82 #define GC_MAX_SCAN_JIFFIES     (16u * HZ)
  83 /* desired ratio of entries found to be expired */
  84 #define GC_EVICT_RATIO  50u
  85
  86 static struct conntrack_gc_work conntrack_gc_work;
  87
  88 void nf_conntrack_lock(spinlock_t *lock) __acquires(lock)
  89 {
  90         /* 1) Acquire the lock */
  91         spin_lock(lock);
  92
  93         /* 2) read nf_conntrack_locks_all, with ACQUIRE semantics
  94          * It pairs with the smp_store_release() in nf_conntrack_all_unlock()
  95          */
  96         if (likely(smp_load_acquire(&nf_conntrack_locks_all) == false))
  97                 return;
  98
  99         /* fast path failed, unlock */
 100         spin_unlock(lock);
 101
 102         /* Slow path 1) get global lock */
 103         spin_lock(&nf_conntrack_locks_all_lock);
 104
 105         /* Slow path 2) get the lock we want */
 106         spin_lock(lock);
 107
 108         /* Slow path 3) release the global lock */
 109         spin_unlock(&nf_conntrack_locks_all_lock);
 110 }
 111 EXPORT_SYMBOL_GPL(nf_conntrack_lock);
 112
 113 static void nf_conntrack_double_unlock(unsigned int h1, unsigned int h2)
 114 {
 115         h1 %= CONNTRACK_LOCKS;
 116         h2 %= CONNTRACK_LOCKS;
 117         spin_unlock(&nf_conntrack_locks[h1]);
 118         if (h1 != h2)
 119                 spin_unlock(&nf_conntrack_locks[h2]);
 120 }
 121
 122 /* return true if we need to recompute hashes (in case hash table was resized) */
 123 static bool nf_conntrack_double_lock(struct net *net, unsigned int h1,
 124                                      unsigned int h2, unsigned int sequence)
 125 {
 126         h1 %= CONNTRACK_LOCKS;
 127         h2 %= CONNTRACK_LOCKS;
 128         if (h1 <= h2) {
 129                 nf_conntrack_lock(&nf_conntrack_locks[h1]);
 130                 if (h1 != h2)
 131                         spin_lock_nested(&nf_conntrack_locks[h2],
 132                                          SINGLE_DEPTH_NESTING);
 133         } else {
 134                 nf_conntrack_lock(&nf_conntrack_locks[h2]);
 135                 spin_lock_nested(&nf_conntrack_locks[h1],
 136                                  SINGLE_DEPTH_NESTING);
 137         }
 138         if (read_seqcount_retry(&nf_conntrack_generation, sequence)) {
 139                 nf_conntrack_double_unlock(h1, h2);
 140                 return true;
 141         }
 142         return false;
 143 }
 144
 145 static void nf_conntrack_all_lock(void)
 146         __acquires(&nf_conntrack_locks_all_lock)
 147 {
 148         int i;
 149
 150         spin_lock(&nf_conntrack_locks_all_lock);
 151
 152         /* For nf_contrack_locks_all, only the latest time when another
 153          * CPU will see an update is controlled, by the "release" of the
 154          * spin_lock below.
 155          * The earliest time is not controlled, an thus KCSAN could detect
 156          * a race when nf_conntract_lock() reads the variable.
 157          * WRITE_ONCE() is used to ensure the compiler will not
 158          * optimize the write.
 159          */
 160         WRITE_ONCE(nf_conntrack_locks_all, true);
 161
 162         for (i = 0; i < CONNTRACK_LOCKS; i++) {
 163                 spin_lock(&nf_conntrack_locks[i]);
 164
 165                 /* This spin_unlock provides the "release" to ensure that
 166                  * nf_conntrack_locks_all==true is visible to everyone that
 167                  * acquired spin_lock(&nf_conntrack_locks[]).
 168                  */
 169                 spin_unlock(&nf_conntrack_locks[i]);
 170         }
 171 }
 172
 173 static void nf_conntrack_all_unlock(void)
 174         __releases(&nf_conntrack_locks_all_lock)
 175 {
 176         /* All prior stores must be complete before we clear
 177          * 'nf_conntrack_locks_all'. Otherwise nf_conntrack_lock()
 178          * might observe the false value but not the entire
 179          * critical section.
 180          * It pairs with the smp_load_acquire() in nf_conntrack_lock()
 181          */
 182         smp_store_release(&nf_conntrack_locks_all, false);
 183         spin_unlock(&nf_conntrack_locks_all_lock);
 184 }
 185
 186 unsigned int nf_conntrack_htable_size __read_mostly;
 187 EXPORT_SYMBOL_GPL(nf_conntrack_htable_size);
 188
 189 unsigned int nf_conntrack_max __read_mostly;
 190 EXPORT_SYMBOL_GPL(nf_conntrack_max);
 191 seqcount_spinlock_t nf_conntrack_generation __read_mostly;
 192 static unsigned int nf_conntrack_hash_rnd __read_mostly;
 193
 194 static u32 hash_conntrack_raw(const struct nf_conntrack_tuple *tuple,
 195                               const struct net *net)
 196 {
 197         unsigned int n;
 198         u32 seed;
 199
 200         get_random_once(&nf_conntrack_hash_rnd, sizeof(nf_conntrack_hash_rnd));
 201
 202         /* The direction must be ignored, so we hash everything up to the
 203          * destination ports (which is a multiple of 4) and treat the last
 204          * three bytes manually.
 205          */
 206         seed = nf_conntrack_hash_rnd ^ net_hash_mix(net);
 207         n = (sizeof(tuple->src) + sizeof(tuple->dst.u3)) / sizeof(u32);
 208         return jhash2((u32 *)tuple, n, seed ^
 209                       (((__force __u16)tuple->dst.u.all << 16) |
 210                       tuple->dst.protonum));
 211 }
 212
 213 static u32 scale_hash(u32 hash)
 214 {
 215         return reciprocal_scale(hash, nf_conntrack_htable_size);
 216 }
 217
 218 static u32 __hash_conntrack(const struct net *net,
 219                             const struct nf_conntrack_tuple *tuple,
 220                             unsigned int size)
 221 {
 222         return reciprocal_scale(hash_conntrack_raw(tuple, net), size);
 223 }
 224
 225 static u32 hash_conntrack(const struct net *net,
 226                           const struct nf_conntrack_tuple *tuple)
 227 {
 228         return scale_hash(hash_conntrack_raw(tuple, net));
 229 }
 230
 231 static bool nf_ct_get_tuple_ports(const struct sk_buff *skb,
 232                                   unsigned int dataoff,
 233                                   struct nf_conntrack_tuple *tuple)
 234 {       struct {
 235                 __be16 sport;
 236                 __be16 dport;
 237         } _inet_hdr, *inet_hdr;
 238
 239         /* Actually only need first 4 bytes to get ports. */
 240         inet_hdr = skb_header_pointer(skb, dataoff, sizeof(_inet_hdr), &_inet_hdr);
 241         if (!inet_hdr)
 242                 return false;
 243
 244         tuple->src.u.udp.port = inet_hdr->sport;
 245         tuple->dst.u.udp.port = inet_hdr->dport;
 246         return true;
 247 }
 248
 249 static bool
 250 nf_ct_get_tuple(const struct sk_buff *skb,
 251                 unsigned int nhoff,
 252                 unsigned int dataoff,
 253                 u_int16_t l3num,
 254                 u_int8_t protonum,
 255                 struct net *net,
 256                 struct nf_conntrack_tuple *tuple)
 257 {
 258         unsigned int size;
 259         const __be32 *ap;
 260         __be32 _addrs[8];
 261
 262         memset(tuple, 0, sizeof(*tuple));
 263
 264         tuple->src.l3num = l3num;
 265         switch (l3num) {
 266         case NFPROTO_IPV4:
 267                 nhoff += offsetof(struct iphdr, saddr);
 268                 size = 2 * sizeof(__be32);
 269                 break;
 270         case NFPROTO_IPV6:
 271                 nhoff += offsetof(struct ipv6hdr, saddr);
 272                 size = sizeof(_addrs);
 273                 break;
 274         default:
 275                 return true;
 276         }
 277
 278         ap = skb_header_pointer(skb, nhoff, size, _addrs);
 279         if (!ap)
 280                 return false;
 281
 282         switch (l3num) {
 283         case NFPROTO_IPV4:
 284                 tuple->src.u3.ip = ap[0];
 285                 tuple->dst.u3.ip = ap[1];
 286                 break;
 287         case NFPROTO_IPV6:
 288                 memcpy(tuple->src.u3.ip6, ap, sizeof(tuple->src.u3.ip6));
 289                 memcpy(tuple->dst.u3.ip6, ap + 4, sizeof(tuple->dst.u3.ip6));
 290                 break;
 291         }
 292
 293         tuple->dst.protonum = protonum;
 294         tuple->dst.dir = IP_CT_DIR_ORIGINAL;
 295
 296         switch (protonum) {
 297 #if IS_ENABLED(CONFIG_IPV6)
 298         case IPPROTO_ICMPV6:
 299                 return icmpv6_pkt_to_tuple(skb, dataoff, net, tuple);
 300 #endif
 301         case IPPROTO_ICMP:
 302                 return icmp_pkt_to_tuple(skb, dataoff, net, tuple);
 303 #ifdef CONFIG_NF_CT_PROTO_GRE
 304         case IPPROTO_GRE:
 305                 return gre_pkt_to_tuple(skb, dataoff, net, tuple);
 306 #endif
 307         case IPPROTO_TCP:
 308         case IPPROTO_UDP: /* fallthrough */
 309                 return nf_ct_get_tuple_ports(skb, dataoff, tuple);
 310 #ifdef CONFIG_NF_CT_PROTO_UDPLITE
 311         case IPPROTO_UDPLITE:
 312                 return nf_ct_get_tuple_ports(skb, dataoff, tuple);
 313 #endif
 314 #ifdef CONFIG_NF_CT_PROTO_SCTP
 315         case IPPROTO_SCTP:
 316                 return nf_ct_get_tuple_ports(skb, dataoff, tuple);
 317 #endif
 318 #ifdef CONFIG_NF_CT_PROTO_DCCP
 319         case IPPROTO_DCCP:
 320                 return nf_ct_get_tuple_ports(skb, dataoff, tuple);
 321 #endif
 322         default:
 323                 break;
 324         }
 325
 326         return true;
 327 }
 328
 329 static int ipv4_get_l4proto(const struct sk_buff *skb, unsigned int nhoff,
 330                             u_int8_t *protonum)
 331 {
 332         int dataoff = -1;
 333         const struct iphdr *iph;
 334         struct iphdr _iph;
 335
 336         iph = skb_header_pointer(skb, nhoff, sizeof(_iph), &_iph);
 337         if (!iph)
 338                 return -1;
 339
 340         /* Conntrack defragments packets, we might still see fragments
 341          * inside ICMP packets though.
 342          */
 343         if (iph->frag_off & htons(IP_OFFSET))
 344                 return -1;
 345
 346         dataoff = nhoff + (iph->ihl << 2);
 347         *protonum = iph->protocol;
 348
 349         /* Check bogus IP headers */
 350         if (dataoff > skb->len) {
 351                 pr_debug("bogus IPv4 packet: nhoff %u, ihl %u, skblen %u\n",
 352                          nhoff, iph->ihl << 2, skb->len);
 353                 return -1;
 354         }
 355         return dataoff;
 356 }
 357
 358 #if IS_ENABLED(CONFIG_IPV6)
 359 static int ipv6_get_l4proto(const struct sk_buff *skb, unsigned int nhoff,
 360                             u8 *protonum)
 361 {
 362         int protoff = -1;
 363         unsigned int extoff = nhoff + sizeof(struct ipv6hdr);
 364         __be16 frag_off;
 365         u8 nexthdr;
 366
 367         if (skb_copy_bits(skb, nhoff + offsetof(struct ipv6hdr, nexthdr),
 368                           &nexthdr, sizeof(nexthdr)) != 0) {
 369                 pr_debug("can't get nexthdr\n");
 370                 return -1;
 371         }
 372         protoff = ipv6_skip_exthdr(skb, extoff, &nexthdr, &frag_off);
 373         /*
 374          * (protoff == skb->len) means the packet has not data, just
 375          * IPv6 and possibly extensions headers, but it is tracked anyway
 376          */
 377         if (protoff < 0 || (frag_off & htons(~0x7)) != 0) {
 378                 pr_debug("can't find proto in pkt\n");
 379                 return -1;
 380         }
 381
 382         *protonum = nexthdr;
 383         return protoff;
 384 }
 385 #endif
 386
 387 static int get_l4proto(const struct sk_buff *skb,
 388                        unsigned int nhoff, u8 pf, u8 *l4num)
 389 {
 390         switch (pf) {
 391         case NFPROTO_IPV4:
 392                 return ipv4_get_l4proto(skb, nhoff, l4num);
 393 #if IS_ENABLED(CONFIG_IPV6)
 394         case NFPROTO_IPV6:
 395                 return ipv6_get_l4proto(skb, nhoff, l4num);
 396 #endif
 397         default:
 398                 *l4num = 0;
 399                 break;
 400         }
 401         return -1;
 402 }
 403
 404 bool nf_ct_get_tuplepr(const struct sk_buff *skb, unsigned int nhoff,
 405                        u_int16_t l3num,
 406                        struct net *net, struct nf_conntrack_tuple *tuple)
 407 {
 408         u8 protonum;
 409         int protoff;
 410
 411         protoff = get_l4proto(skb, nhoff, l3num, &protonum);
 412         if (protoff <= 0)
 413                 return false;
 414
 415         return nf_ct_get_tuple(skb, nhoff, protoff, l3num, protonum, net, tuple);
 416 }
 417 EXPORT_SYMBOL_GPL(nf_ct_get_tuplepr);
 418
 419 bool
 420 nf_ct_invert_tuple(struct nf_conntrack_tuple *inverse,
 421                    const struct nf_conntrack_tuple *orig)
 422 {
 423         memset(inverse, 0, sizeof(*inverse));
 424
 425         inverse->src.l3num = orig->src.l3num;
 426
 427         switch (orig->src.l3num) {
 428         case NFPROTO_IPV4:
 429                 inverse->src.u3.ip = orig->dst.u3.ip;
 430                 inverse->dst.u3.ip = orig->src.u3.ip;
 431                 break;
 432         case NFPROTO_IPV6:
 433                 inverse->src.u3.in6 = orig->dst.u3.in6;
 434                 inverse->dst.u3.in6 = orig->src.u3.in6;
 435                 break;
 436         default:
 437                 break;
 438         }
 439
 440         inverse->dst.dir = !orig->dst.dir;
 441
 442         inverse->dst.protonum = orig->dst.protonum;
 443
 444         switch (orig->dst.protonum) {
 445         case IPPROTO_ICMP:
 446                 return nf_conntrack_invert_icmp_tuple(inverse, orig);
 447 #if IS_ENABLED(CONFIG_IPV6)
 448         case IPPROTO_ICMPV6:
 449                 return nf_conntrack_invert_icmpv6_tuple(inverse, orig);
 450 #endif
 451         }
 452
 453         inverse->src.u.all = orig->dst.u.all;
 454         inverse->dst.u.all = orig->src.u.all;
 455         return true;
 456 }
 457 EXPORT_SYMBOL_GPL(nf_ct_invert_tuple);
 458
 459 /* Generate a almost-unique pseudo-id for a given conntrack.
 460  *
 461  * intentionally doesn't re-use any of the seeds used for hash
 462  * table location, we assume id gets exposed to userspace.
 463  *
 464  * Following nf_conn items do not change throughout lifetime
 465  * of the nf_conn:
 466  *
 467  * 1. nf_conn address
 468  * 2. nf_conn->master address (normally NULL)
 469  * 3. the associated net namespace
 470  * 4. the original direction tuple
 471  */
 472 u32 nf_ct_get_id(const struct nf_conn *ct)
 473 {
 474         static __read_mostly siphash_key_t ct_id_seed;
 475         unsigned long a, b, c, d;
 476
 477         net_get_random_once(&ct_id_seed, sizeof(ct_id_seed));
 478
 479         a = (unsigned long)ct;
 480         b = (unsigned long)ct->master;
 481         c = (unsigned long)nf_ct_net(ct);
 482         d = (unsigned long)siphash(&ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple,
 483                                    sizeof(ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple),
 484                                    &ct_id_seed);
 485 #ifdef CONFIG_64BIT
 486         return siphash_4u64((u64)a, (u64)b, (u64)c, (u64)d, &ct_id_seed);
 487 #else
 488         return siphash_4u32((u32)a, (u32)b, (u32)c, (u32)d, &ct_id_seed);
 489 #endif
 490 }
 491 EXPORT_SYMBOL_GPL(nf_ct_get_id);
 492
 493 static void
 494 clean_from_lists(struct nf_conn *ct)
 495 {
 496         pr_debug("clean_from_lists(%p)\n", ct);
 497         hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);
 498         hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode);
 499
 500         /* Destroy all pending expectations */
 501         nf_ct_remove_expectations(ct);
 502 }
 503
 504 /* must be called with local_bh_disable */
 505 static void nf_ct_add_to_dying_list(struct nf_conn *ct)
 506 {
 507         struct ct_pcpu *pcpu;
 508
 509         /* add this conntrack to the (per cpu) dying list */
 510         ct->cpu = smp_processor_id();
 511         pcpu = per_cpu_ptr(nf_ct_net(ct)->ct.pcpu_lists, ct->cpu);
 512
 513         spin_lock(&pcpu->lock);
 514         hlist_nulls_add_head(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,
 515                              &pcpu->dying);
 516         spin_unlock(&pcpu->lock);
 517 }
 518
 519 /* must be called with local_bh_disable */
 520 static void nf_ct_add_to_unconfirmed_list(struct nf_conn *ct)
 521 {
 522         struct ct_pcpu *pcpu;
 523
 524         /* add this conntrack to the (per cpu) unconfirmed list */
 525         ct->cpu = smp_processor_id();
 526         pcpu = per_cpu_ptr(nf_ct_net(ct)->ct.pcpu_lists, ct->cpu);
 527
 528         spin_lock(&pcpu->lock);
 529         hlist_nulls_add_head(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,
 530                              &pcpu->unconfirmed);
 531         spin_unlock(&pcpu->lock);
 532 }
 533
 534 /* must be called with local_bh_disable */
 535 static void nf_ct_del_from_dying_or_unconfirmed_list(struct nf_conn *ct)
 536 {
 537         struct ct_pcpu *pcpu;
 538
 539         /* We overload first tuple to link into unconfirmed or dying list.*/
 540         pcpu = per_cpu_ptr(nf_ct_net(ct)->ct.pcpu_lists, ct->cpu);
 541
 542         spin_lock(&pcpu->lock);
 543         BUG_ON(hlist_nulls_unhashed(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode));
 544         hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);
 545         spin_unlock(&pcpu->lock);
 546 }
 547
 548 #define NFCT_ALIGN(len) (((len) + NFCT_INFOMASK) & ~NFCT_INFOMASK)
 549
 550 /* Released via destroy_conntrack() */
 551 struct nf_conn *nf_ct_tmpl_alloc(struct net *net,
 552                                  const struct nf_conntrack_zone *zone,
 553                                  gfp_t flags)
 554 {
 555         struct nf_conn *tmpl, *p;
 556
 557         if (ARCH_KMALLOC_MINALIGN <= NFCT_INFOMASK) {
 558                 tmpl = kzalloc(sizeof(*tmpl) + NFCT_INFOMASK, flags);
 559                 if (!tmpl)
 560                         return NULL;
 561
 562                 p = tmpl;
 563                 tmpl = (struct nf_conn *)NFCT_ALIGN((unsigned long)p);
 564                 if (tmpl != p) {
 565                         tmpl = (struct nf_conn *)NFCT_ALIGN((unsigned long)p);
 566                         tmpl->proto.tmpl_padto = (char *)tmpl - (char *)p;
 567                 }
 568         } else {
 569                 tmpl = kzalloc(sizeof(*tmpl), flags);
 570                 if (!tmpl)
 571                         return NULL;
 572         }
 573
 574         tmpl->status = IPS_TEMPLATE;
 575         write_pnet(&tmpl->ct_net, net);
 576         nf_ct_zone_add(tmpl, zone);
 577         atomic_set(&tmpl->ct_general.use, 0);
 578
 579         return tmpl;
 580 }
 581 EXPORT_SYMBOL_GPL(nf_ct_tmpl_alloc);
 582
 583 void nf_ct_tmpl_free(struct nf_conn *tmpl)
 584 {
 585         nf_ct_ext_destroy(tmpl);
 586
 587         if (ARCH_KMALLOC_MINALIGN <= NFCT_INFOMASK)
 588                 kfree((char *)tmpl - tmpl->proto.tmpl_padto);
 589         else
 590                 kfree(tmpl);
 591 }
 592 EXPORT_SYMBOL_GPL(nf_ct_tmpl_free);
 593
 594 static void destroy_gre_conntrack(struct nf_conn *ct)
 595 {
 596 #ifdef CONFIG_NF_CT_PROTO_GRE
 597         struct nf_conn *master = ct->master;
 598
 599         if (master)
 600                 nf_ct_gre_keymap_destroy(master);
 601 #endif
 602 }
 603
 604 static void
 605 destroy_conntrack(struct nf_conntrack *nfct)
 606 {
 607         struct nf_conn *ct = (struct nf_conn *)nfct;
 608
 609         pr_debug("destroy_conntrack(%p)\n", ct);
 610         WARN_ON(atomic_read(&nfct->use) != 0);
 611
 612         if (unlikely(nf_ct_is_template(ct))) {
 613                 nf_ct_tmpl_free(ct);
 614                 return;
 615         }
 616
 617         if (unlikely(nf_ct_protonum(ct) == IPPROTO_GRE))
 618                 destroy_gre_conntrack(ct);
 619
 620         local_bh_disable();
 621         /* Expectations will have been removed in clean_from_lists,
 622          * except TFTP can create an expectation on the first packet,
 623          * before connection is in the list, so we need to clean here,
 624          * too.
 625          */
 626         nf_ct_remove_expectations(ct);
 627
 628         nf_ct_del_from_dying_or_unconfirmed_list(ct);
 629
 630         local_bh_enable();
 631
 632         if (ct->master)
 633                 nf_ct_put(ct->master);
 634
 635         pr_debug("destroy_conntrack: returning ct=%p to slab\n", ct);
 636         nf_conntrack_free(ct);
 637 }
 638
 639 static void nf_ct_delete_from_lists(struct nf_conn *ct)
 640 {
 641         struct net *net = nf_ct_net(ct);
 642         unsigned int hash, reply_hash;
 643         unsigned int sequence;
 644
 645         nf_ct_helper_destroy(ct);
 646
 647         local_bh_disable();
 648         do {
 649                 sequence = read_seqcount_begin(&nf_conntrack_generation);
 650                 hash = hash_conntrack(net,
 651                                       &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple);
 652                 reply_hash = hash_conntrack(net,
 653                                            &ct->tuplehash[IP_CT_DIR_REPLY].tuple);
 654         } while (nf_conntrack_double_lock(net, hash, reply_hash, sequence));
 655
 656         clean_from_lists(ct);
 657         nf_conntrack_double_unlock(hash, reply_hash);
 658
 659         nf_ct_add_to_dying_list(ct);
 660
 661         local_bh_enable();
 662 }
 663
 664 bool nf_ct_delete(struct nf_conn *ct, u32 portid, int report)
 665 {
 666         struct nf_conn_tstamp *tstamp;
 667         struct net *net;
 668
 669         if (test_and_set_bit(IPS_DYING_BIT, &ct->status))
 670                 return false;
 671
 672         tstamp = nf_conn_tstamp_find(ct);
 673         if (tstamp && tstamp->stop == 0)
 674                 tstamp->stop = ktime_get_real_ns();
 675
 676         if (nf_conntrack_event_report(IPCT_DESTROY, ct,
 677                                     portid, report) < 0) {
 678                 /* destroy event was not delivered. nf_ct_put will
 679                  * be done by event cache worker on redelivery.
 680                  */
 681                 nf_ct_delete_from_lists(ct);
 682                 nf_conntrack_ecache_work(nf_ct_net(ct), NFCT_ECACHE_DESTROY_FAIL);
 683                 return false;
 684         }
 685
 686         net = nf_ct_net(ct);
 687         if (nf_conntrack_ecache_dwork_pending(net))
 688                 nf_conntrack_ecache_work(net, NFCT_ECACHE_DESTROY_SENT);
 689         nf_ct_delete_from_lists(ct);
 690         nf_ct_put(ct);
 691         return true;
 692 }
 693 EXPORT_SYMBOL_GPL(nf_ct_delete);
 694
 695 static inline bool
 696 nf_ct_key_equal(struct nf_conntrack_tuple_hash *h,
 697                 const struct nf_conntrack_tuple *tuple,
 698                 const struct nf_conntrack_zone *zone,
 699                 const struct net *net)
 700 {
 701         struct nf_conn *ct = nf_ct_tuplehash_to_ctrack(h);
 702
 703         /* A conntrack can be recreated with the equal tuple,
 704          * so we need to check that the conntrack is confirmed
 705          */
 706         return nf_ct_tuple_equal(tuple, &h->tuple) &&
 707                nf_ct_zone_equal(ct, zone, NF_CT_DIRECTION(h)) &&
 708                nf_ct_is_confirmed(ct) &&
 709                net_eq(net, nf_ct_net(ct));
 710 }
 711
 712 static inline bool
 713 nf_ct_match(const struct nf_conn *ct1, const struct nf_conn *ct2)
 714 {
 715         return nf_ct_tuple_equal(&ct1->tuplehash[IP_CT_DIR_ORIGINAL].tuple,
 716                                  &ct2->tuplehash[IP_CT_DIR_ORIGINAL].tuple) &&
 717                nf_ct_tuple_equal(&ct1->tuplehash[IP_CT_DIR_REPLY].tuple,
 718                                  &ct2->tuplehash[IP_CT_DIR_REPLY].tuple) &&
 719                nf_ct_zone_equal(ct1, nf_ct_zone(ct2), IP_CT_DIR_ORIGINAL) &&
 720                nf_ct_zone_equal(ct1, nf_ct_zone(ct2), IP_CT_DIR_REPLY) &&
 721                net_eq(nf_ct_net(ct1), nf_ct_net(ct2));
 722 }
 723
 724 /* caller must hold rcu readlock and none of the nf_conntrack_locks */
 725 static void nf_ct_gc_expired(struct nf_conn *ct)
 726 {
 727         if (!atomic_inc_not_zero(&ct->ct_general.use))
 728                 return;
 729
 730         if (nf_ct_should_gc(ct))
 731                 nf_ct_kill(ct);
 732
 733         nf_ct_put(ct);
 734 }
 735
 736 /*
 737  * Warning :
 738  * - Caller must take a reference on returned object
 739  *   and recheck nf_ct_tuple_equal(tuple, &h->tuple)
 740  */
 741 static struct nf_conntrack_tuple_hash *
 742 ____nf_conntrack_find(struct net *net, const struct nf_conntrack_zone *zone,
 743                       const struct nf_conntrack_tuple *tuple, u32 hash)
 744 {
 745         struct nf_conntrack_tuple_hash *h;
 746         struct hlist_nulls_head *ct_hash;
 747         struct hlist_nulls_node *n;
 748         unsigned int bucket, hsize;
 749
 750 begin:
 751         nf_conntrack_get_ht(&ct_hash, &hsize);
 752         bucket = reciprocal_scale(hash, hsize);
 753
 754         hlist_nulls_for_each_entry_rcu(h, n, &ct_hash[bucket], hnnode) {
 755                 struct nf_conn *ct;
 756
 757                 ct = nf_ct_tuplehash_to_ctrack(h);
 758                 if (nf_ct_is_expired(ct)) {
 759                         nf_ct_gc_expired(ct);
 760                         continue;
 761                 }
 762
 763                 if (nf_ct_key_equal(h, tuple, zone, net))
 764                         return h;
 765         }
 766         /*
 767          * if the nulls value we got at the end of this lookup is
 768          * not the expected one, we must restart lookup.
 769          * We probably met an item that was moved to another chain.
 770          */
 771         if (get_nulls_value(n) != bucket) {
 772                 NF_CT_STAT_INC_ATOMIC(net, search_restart);
 773                 goto begin;
 774         }
 775
 776         return NULL;
 777 }
 778
 779 /* Find a connection corresponding to a tuple. */
 780 static struct nf_conntrack_tuple_hash *
 781 __nf_conntrack_find_get(struct net *net, const struct nf_conntrack_zone *zone,
 782                         const struct nf_conntrack_tuple *tuple, u32 hash)
 783 {
 784         struct nf_conntrack_tuple_hash *h;
 785         struct nf_conn *ct;
 786
 787         rcu_read_lock();
 788
 789         h = ____nf_conntrack_find(net, zone, tuple, hash);
 790         if (h) {
 791                 /* We have a candidate that matches the tuple we're interested
 792                  * in, try to obtain a reference and re-check tuple
 793                  */
 794                 ct = nf_ct_tuplehash_to_ctrack(h);
 795                 if (likely(atomic_inc_not_zero(&ct->ct_general.use))) {
 796                         if (likely(nf_ct_key_equal(h, tuple, zone, net)))
 797                                 goto found;
 798
 799                         /* TYPESAFE_BY_RCU recycled the candidate */
 800                         nf_ct_put(ct);
 801                 }
 802
 803                 h = NULL;
 804         }
 805 found:
 806         rcu_read_unlock();
 807
 808         return h;
 809 }
 810
 811 struct nf_conntrack_tuple_hash *
 812 nf_conntrack_find_get(struct net *net, const struct nf_conntrack_zone *zone,
 813                       const struct nf_conntrack_tuple *tuple)
 814 {
 815         return __nf_conntrack_find_get(net, zone, tuple,
 816                                        hash_conntrack_raw(tuple, net));
 817 }
 818 EXPORT_SYMBOL_GPL(nf_conntrack_find_get);
 819
 820 static void __nf_conntrack_hash_insert(struct nf_conn *ct,
 821                                        unsigned int hash,
 822                                        unsigned int reply_hash)
 823 {
 824         hlist_nulls_add_head_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,
 825                            &nf_conntrack_hash[hash]);
 826         hlist_nulls_add_head_rcu(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode,
 827                            &nf_conntrack_hash[reply_hash]);
 828 }
 829
 830 int
 831 nf_conntrack_hash_check_insert(struct nf_conn *ct)
 832 {
 833         const struct nf_conntrack_zone *zone;
 834         struct net *net = nf_ct_net(ct);
 835         unsigned int hash, reply_hash;
 836         struct nf_conntrack_tuple_hash *h;
 837         struct hlist_nulls_node *n;
 838         unsigned int sequence;
 839
 840         zone = nf_ct_zone(ct);
 841
 842         local_bh_disable();
 843         do {
 844                 sequence = read_seqcount_begin(&nf_conntrack_generation);
 845                 hash = hash_conntrack(net,
 846                                       &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple);
 847                 reply_hash = hash_conntrack(net,
 848                                            &ct->tuplehash[IP_CT_DIR_REPLY].tuple);
 849         } while (nf_conntrack_double_lock(net, hash, reply_hash, sequence));
 850
 851         /* See if there's one in the list already, including reverse */
 852         hlist_nulls_for_each_entry(h, n, &nf_conntrack_hash[hash], hnnode)
 853                 if (nf_ct_key_equal(h, &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple,
 854                                     zone, net))
 855                         goto out;
 856
 857         hlist_nulls_for_each_entry(h, n, &nf_conntrack_hash[reply_hash], hnnode)
 858                 if (nf_ct_key_equal(h, &ct->tuplehash[IP_CT_DIR_REPLY].tuple,
 859                                     zone, net))
 860                         goto out;
 861
 862         smp_wmb();
 863         /* The caller holds a reference to this object */
 864         atomic_set(&ct->ct_general.use, 2);
 865         __nf_conntrack_hash_insert(ct, hash, reply_hash);
 866         nf_conntrack_double_unlock(hash, reply_hash);
 867         NF_CT_STAT_INC(net, insert);
 868         local_bh_enable();
 869         return 0;
 870
 871 out:
 872         nf_conntrack_double_unlock(hash, reply_hash);
 873         local_bh_enable();
 874         return -EEXIST;
 875 }
 876 EXPORT_SYMBOL_GPL(nf_conntrack_hash_check_insert);
 877
 878 void nf_ct_acct_add(struct nf_conn *ct, u32 dir, unsigned int packets,
 879                     unsigned int bytes)
 880 {
 881         struct nf_conn_acct *acct;
 882
 883         acct = nf_conn_acct_find(ct);
 884         if (acct) {
 885                 struct nf_conn_counter *counter = acct->counter;
 886
 887                 atomic64_add(packets, &counter[dir].packets);
 888                 atomic64_add(bytes, &counter[dir].bytes);
 889         }
 890 }
 891 EXPORT_SYMBOL_GPL(nf_ct_acct_add);
 892
 893 static void nf_ct_acct_merge(struct nf_conn *ct, enum ip_conntrack_info ctinfo,
 894                              const struct nf_conn *loser_ct)
 895 {
 896         struct nf_conn_acct *acct;
 897
 898         acct = nf_conn_acct_find(loser_ct);
 899         if (acct) {
 900                 struct nf_conn_counter *counter = acct->counter;
 901                 unsigned int bytes;
 902
 903                 /* u32 should be fine since we must have seen one packet. */
 904                 bytes = atomic64_read(&counter[CTINFO2DIR(ctinfo)].bytes);
 905                 nf_ct_acct_update(ct, CTINFO2DIR(ctinfo), bytes);
 906         }
 907 }
 908
 909 static void __nf_conntrack_insert_prepare(struct nf_conn *ct)
 910 {
 911         struct nf_conn_tstamp *tstamp;
 912
 913         atomic_inc(&ct->ct_general.use);
 914         ct->status |= IPS_CONFIRMED;
 915
 916         /* set conntrack timestamp, if enabled. */
 917         tstamp = nf_conn_tstamp_find(ct);
 918         if (tstamp)
 919                 tstamp->start = ktime_get_real_ns();
 920 }
 921
 922 /* caller must hold locks to prevent concurrent changes */
 923 static int __nf_ct_resolve_clash(struct sk_buff *skb,
 924                                  struct nf_conntrack_tuple_hash *h)
 925 {
 926         /* This is the conntrack entry already in hashes that won race. */
 927         struct nf_conn *ct = nf_ct_tuplehash_to_ctrack(h);
 928         enum ip_conntrack_info ctinfo;
 929         struct nf_conn *loser_ct;
 930
 931         loser_ct = nf_ct_get(skb, &ctinfo);
 932
 933         if (nf_ct_is_dying(ct))
 934                 return NF_DROP;
 935
 936         if (((ct->status & IPS_NAT_DONE_MASK) == 0) ||
 937             nf_ct_match(ct, loser_ct)) {
 938                 struct net *net = nf_ct_net(ct);
 939
 940                 nf_conntrack_get(&ct->ct_general);
 941
 942                 nf_ct_acct_merge(ct, ctinfo, loser_ct);
 943                 nf_ct_add_to_dying_list(loser_ct);
 944                 nf_conntrack_put(&loser_ct->ct_general);
 945                 nf_ct_set(skb, ct, ctinfo);
 946
 947                 NF_CT_STAT_INC(net, clash_resolve);
 948                 return NF_ACCEPT;
 949         }
 950
 951         return NF_DROP;
 952 }
 953
 954 /**
 955  * nf_ct_resolve_clash_harder - attempt to insert clashing conntrack entry
 956  *
 957  * @skb: skb that causes the collision
 958  * @repl_idx: hash slot for reply direction
 959  *
 960  * Called when origin or reply direction had a clash.
 961  * The skb can be handled without packet drop provided the reply direction
 962  * is unique or there the existing entry has the identical tuple in both
 963  * directions.
 964  *
 965  * Caller must hold conntrack table locks to prevent concurrent updates.
 966  *
 967  * Returns NF_DROP if the clash could not be handled.
 968  */
 969 static int nf_ct_resolve_clash_harder(struct sk_buff *skb, u32 repl_idx)
 970 {
 971         struct nf_conn *loser_ct = (struct nf_conn *)skb_nfct(skb);
 972         const struct nf_conntrack_zone *zone;
 973         struct nf_conntrack_tuple_hash *h;
 974         struct hlist_nulls_node *n;
 975         struct net *net;
 976
 977         zone = nf_ct_zone(loser_ct);
 978         net = nf_ct_net(loser_ct);
 979
 980         /* Reply direction must never result in a clash, unless both origin
 981          * and reply tuples are identical.
 982          */
 983         hlist_nulls_for_each_entry(h, n, &nf_conntrack_hash[repl_idx], hnnode) {
 984                 if (nf_ct_key_equal(h,
 985                                     &loser_ct->tuplehash[IP_CT_DIR_REPLY].tuple,
 986                                     zone, net))
 987                         return __nf_ct_resolve_clash(skb, h);
 988         }
 989
 990         /* We want the clashing entry to go away real soon: 1 second timeout. */
 991         loser_ct->timeout = nfct_time_stamp + HZ;
 992
 993         /* IPS_NAT_CLASH removes the entry automatically on the first
 994          * reply.  Also prevents UDP tracker from moving the entry to
 995          * ASSURED state, i.e. the entry can always be evicted under
 996          * pressure.
 997          */
 998         loser_ct->status |= IPS_FIXED_TIMEOUT | IPS_NAT_CLASH;
 999
1000         __nf_conntrack_insert_prepare(loser_ct);
1001
1002         /* fake add for ORIGINAL dir: we want lookups to only find the entry
1003          * already in the table.  This also hides the clashing entry from
1004          * ctnetlink iteration, i.e. conntrack -L won't show them.
1005          */
1006         hlist_nulls_add_fake(&loser_ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);
1007
1008         hlist_nulls_add_head_rcu(&loser_ct->tuplehash[IP_CT_DIR_REPLY].hnnode,
1009                                  &nf_conntrack_hash[repl_idx]);
1010
1011         NF_CT_STAT_INC(net, clash_resolve);
1012         return NF_ACCEPT;
1013 }
1014
1015 /**
1016  * nf_ct_resolve_clash - attempt to handle clash without packet drop
1017  *
1018  * @skb: skb that causes the clash
1019  * @h: tuplehash of the clashing entry already in table
1020  * @reply_hash: hash slot for reply direction
1021  *
1022  * A conntrack entry can be inserted to the connection tracking table
1023  * if there is no existing entry with an identical tuple.
1024  *
1025  * If there is one, @skb (and the assocated, unconfirmed conntrack) has
1026  * to be dropped.  In case @skb is retransmitted, next conntrack lookup
1027  * will find the already-existing entry.
1028  *
1029  * The major problem with such packet drop is the extra delay added by
1030  * the packet loss -- it will take some time for a retransmit to occur
1031  * (or the sender to time out when waiting for a reply).
1032  *
1033  * This function attempts to handle the situation without packet drop.
1034  *
1035  * If @skb has no NAT transformation or if the colliding entries are
1036  * exactly the same, only the to-be-confirmed conntrack entry is discarded
1037  * and @skb is associated with the conntrack entry already in the table.
1038  *
1039  * Failing that, the new, unconfirmed conntrack is still added to the table
1040  * provided that the collision only occurs in the ORIGINAL direction.
1041  * The new entry will be added only in the non-clashing REPLY direction,
1042  * so packets in the ORIGINAL direction will continue to match the existing
1043  * entry.  The new entry will also have a fixed timeout so it expires --
1044  * due to the collision, it will only see reply traffic.
1045  *
1046  * Returns NF_DROP if the clash could not be resolved.
1047  */
1048 static __cold noinline int
1049 nf_ct_resolve_clash(struct sk_buff *skb, struct nf_conntrack_tuple_hash *h,
1050                     u32 reply_hash)
1051 {
1052         /* This is the conntrack entry already in hashes that won race. */
1053         struct nf_conn *ct = nf_ct_tuplehash_to_ctrack(h);
1054         const struct nf_conntrack_l4proto *l4proto;
1055         enum ip_conntrack_info ctinfo;
1056         struct nf_conn *loser_ct;
1057         struct net *net;
1058         int ret;
1059
1060         loser_ct = nf_ct_get(skb, &ctinfo);
1061         net = nf_ct_net(loser_ct);
1062
1063         l4proto = nf_ct_l4proto_find(nf_ct_protonum(ct));
1064         if (!l4proto->allow_clash)
1065                 goto drop;
1066
1067         ret = __nf_ct_resolve_clash(skb, h);
1068         if (ret == NF_ACCEPT)
1069                 return ret;
1070
1071         ret = nf_ct_resolve_clash_harder(skb, reply_hash);
1072         if (ret == NF_ACCEPT)
1073                 return ret;
1074
1075 drop:
1076         nf_ct_add_to_dying_list(loser_ct);
1077         NF_CT_STAT_INC(net, drop);
1078         NF_CT_STAT_INC(net, insert_failed);
1079         return NF_DROP;
1080 }
1081
1082 /* Confirm a connection given skb; places it in hash table */
1083 int
1084 __nf_conntrack_confirm(struct sk_buff *skb)
1085 {
1086         const struct nf_conntrack_zone *zone;
1087         unsigned int hash, reply_hash;
1088         struct nf_conntrack_tuple_hash *h;
1089         struct nf_conn *ct;
1090         struct nf_conn_help *help;
1091         struct hlist_nulls_node *n;
1092         enum ip_conntrack_info ctinfo;
1093         struct net *net;
1094         unsigned int sequence;
1095         int ret = NF_DROP;
1096
1097         ct = nf_ct_get(skb, &ctinfo);
1098         net = nf_ct_net(ct);
1099
1100         /* ipt_REJECT uses nf_conntrack_attach to attach related
1101            ICMP/TCP RST packets in other direction.  Actual packet
1102            which created connection will be IP_CT_NEW or for an
1103            expected connection, IP_CT_RELATED. */
1104         if (CTINFO2DIR(ctinfo) != IP_CT_DIR_ORIGINAL)
1105                 return NF_ACCEPT;
1106
1107         zone = nf_ct_zone(ct);
1108         local_bh_disable();
1109
1110         do {
1111                 sequence = read_seqcount_begin(&nf_conntrack_generation);
1112                 /* reuse the hash saved before */
1113                 hash = *(unsigned long *)&ct->tuplehash[IP_CT_DIR_REPLY].hnnode.pprev;
1114                 hash = scale_hash(hash);
1115                 reply_hash = hash_conntrack(net,
1116                                            &ct->tuplehash[IP_CT_DIR_REPLY].tuple);
1117
1118         } while (nf_conntrack_double_lock(net, hash, reply_hash, sequence));
1119
1120         /* We're not in hash table, and we refuse to set up related
1121          * connections for unconfirmed conns.  But packet copies and
1122          * REJECT will give spurious warnings here.
1123          */
1124
1125         /* Another skb with the same unconfirmed conntrack may
1126          * win the race. This may happen for bridge(br_flood)
1127          * or broadcast/multicast packets do skb_clone with
1128          * unconfirmed conntrack.
1129          */
1130         if (unlikely(nf_ct_is_confirmed(ct))) {
1131                 WARN_ON_ONCE(1);
1132                 nf_conntrack_double_unlock(hash, reply_hash);
1133                 local_bh_enable();
1134                 return NF_DROP;
1135         }
1136
1137         pr_debug("Confirming conntrack %p\n", ct);
1138         /* We have to check the DYING flag after unlink to prevent
1139          * a race against nf_ct_get_next_corpse() possibly called from
1140          * user context, else we insert an already 'dead' hash, blocking
1141          * further use of that particular connection -JM.
1142          */
1143         nf_ct_del_from_dying_or_unconfirmed_list(ct);
1144
1145         if (unlikely(nf_ct_is_dying(ct))) {
1146                 nf_ct_add_to_dying_list(ct);
1147                 NF_CT_STAT_INC(net, insert_failed);
1148                 goto dying;
1149         }
1150
1151         /* See if there's one in the list already, including reverse:
1152            NAT could have grabbed it without realizing, since we're
1153            not in the hash.  If there is, we lost race. */
1154         hlist_nulls_for_each_entry(h, n, &nf_conntrack_hash[hash], hnnode)
1155                 if (nf_ct_key_equal(h, &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple,
1156                                     zone, net))
1157                         goto out;
1158
1159         hlist_nulls_for_each_entry(h, n, &nf_conntrack_hash[reply_hash], hnnode)
1160                 if (nf_ct_key_equal(h, &ct->tuplehash[IP_CT_DIR_REPLY].tuple,
1161                                     zone, net))
1162                         goto out;
1163
1164         /* Timer relative to confirmation time, not original
1165            setting time, otherwise we'd get timer wrap in
1166            weird delay cases. */
1167         ct->timeout += nfct_time_stamp;
1168
1169         __nf_conntrack_insert_prepare(ct);
1170
1171         /* Since the lookup is lockless, hash insertion must be done after
1172          * starting the timer and setting the CONFIRMED bit. The RCU barriers
1173          * guarantee that no other CPU can find the conntrack before the above
1174          * stores are visible.
1175          */
1176         __nf_conntrack_hash_insert(ct, hash, reply_hash);
1177         nf_conntrack_double_unlock(hash, reply_hash);
1178         local_bh_enable();
1179
1180         help = nfct_help(ct);
1181         if (help && help->helper)
1182                 nf_conntrack_event_cache(IPCT_HELPER, ct);
1183
1184         nf_conntrack_event_cache(master_ct(ct) ?
1185                                  IPCT_RELATED : IPCT_NEW, ct);
1186         return NF_ACCEPT;
1187
1188 out:
1189         ret = nf_ct_resolve_clash(skb, h, reply_hash);
1190 dying:
1191         nf_conntrack_double_unlock(hash, reply_hash);
1192         local_bh_enable();
1193         return ret;
1194 }
1195 EXPORT_SYMBOL_GPL(__nf_conntrack_confirm);
1196
1197 /* Returns true if a connection correspondings to the tuple (required
1198    for NAT). */
1199 int
1200 nf_conntrack_tuple_taken(const struct nf_conntrack_tuple *tuple,
1201                          const struct nf_conn *ignored_conntrack)
1202 {
1203         struct net *net = nf_ct_net(ignored_conntrack);
1204         const struct nf_conntrack_zone *zone;
1205         struct nf_conntrack_tuple_hash *h;
1206         struct hlist_nulls_head *ct_hash;
1207         unsigned int hash, hsize;
1208         struct hlist_nulls_node *n;
1209         struct nf_conn *ct;
1210
1211         zone = nf_ct_zone(ignored_conntrack);
1212
1213         rcu_read_lock();
1214  begin:
1215         nf_conntrack_get_ht(&ct_hash, &hsize);
1216         hash = __hash_conntrack(net, tuple, hsize);
1217
1218         hlist_nulls_for_each_entry_rcu(h, n, &ct_hash[hash], hnnode) {
1219                 ct = nf_ct_tuplehash_to_ctrack(h);
1220
1221                 if (ct == ignored_conntrack)
1222                         continue;
1223
1224                 if (nf_ct_is_expired(ct)) {
1225                         nf_ct_gc_expired(ct);
1226                         continue;
1227                 }
1228
1229                 if (nf_ct_key_equal(h, tuple, zone, net)) {
1230                         /* Tuple is taken already, so caller will need to find
1231                          * a new source port to use.
1232                          *
1233                          * Only exception:
1234                          * If the *original tuples* are identical, then both
1235                          * conntracks refer to the same flow.
1236                          * This is a rare situation, it can occur e.g. when
1237                          * more than one UDP packet is sent from same socket
1238                          * in different threads.
1239                          *
1240                          * Let nf_ct_resolve_clash() deal with this later.
1241                          */
1242                         if (nf_ct_tuple_equal(&ignored_conntrack->tuplehash[IP_CT_DIR_ORIGINAL].tuple,
1243                                               &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple) &&
1244                                               nf_ct_zone_equal(ct, zone, IP_CT_DIR_ORIGINAL))
1245                                 continue;
1246
1247                         NF_CT_STAT_INC_ATOMIC(net, found);
1248                         rcu_read_unlock();
1249                         return 1;
1250                 }
1251         }
1252
1253         if (get_nulls_value(n) != hash) {
1254                 NF_CT_STAT_INC_ATOMIC(net, search_restart);
1255                 goto begin;
1256         }
1257
1258         rcu_read_unlock();
1259
1260         return 0;
1261 }
1262 EXPORT_SYMBOL_GPL(nf_conntrack_tuple_taken);
1263
1264 #define NF_CT_EVICTION_RANGE    8
1265
1266 /* There's a small race here where we may free a just-assured
1267    connection.  Too bad: we're in trouble anyway. */
1268 static unsigned int early_drop_list(struct net *net,
1269                                     struct hlist_nulls_head *head)
1270 {
1271         struct nf_conntrack_tuple_hash *h;
1272         struct hlist_nulls_node *n;
1273         unsigned int drops = 0;
1274         struct nf_conn *tmp;
1275
1276         hlist_nulls_for_each_entry_rcu(h, n, head, hnnode) {
1277                 tmp = nf_ct_tuplehash_to_ctrack(h);
1278
1279                 if (test_bit(IPS_OFFLOAD_BIT, &tmp->status))
1280                         continue;
1281
1282                 if (nf_ct_is_expired(tmp)) {
1283                         nf_ct_gc_expired(tmp);
1284                         continue;
1285                 }
1286
1287                 if (test_bit(IPS_ASSURED_BIT, &tmp->status) ||
1288                     !net_eq(nf_ct_net(tmp), net) ||
1289                     nf_ct_is_dying(tmp))
1290                         continue;
1291
1292                 if (!atomic_inc_not_zero(&tmp->ct_general.use))
1293                         continue;
1294
1295                 /* kill only if still in same netns -- might have moved due to
1296                  * SLAB_TYPESAFE_BY_RCU rules.
1297                  *
1298                  * We steal the timer reference.  If that fails timer has
1299                  * already fired or someone else deleted it. Just drop ref
1300                  * and move to next entry.
1301                  */
1302                 if (net_eq(nf_ct_net(tmp), net) &&
1303                     nf_ct_is_confirmed(tmp) &&
1304                     nf_ct_delete(tmp, 0, 0))
1305                         drops++;
1306
1307                 nf_ct_put(tmp);
1308         }
1309
1310         return drops;
1311 }
1312
1313 static noinline int early_drop(struct net *net, unsigned int hash)
1314 {
1315         unsigned int i, bucket;
1316
1317         for (i = 0; i < NF_CT_EVICTION_RANGE; i++) {
1318                 struct hlist_nulls_head *ct_hash;
1319                 unsigned int hsize, drops;
1320
1321                 rcu_read_lock();
1322                 nf_conntrack_get_ht(&ct_hash, &hsize);
1323                 if (!i)
1324                         bucket = reciprocal_scale(hash, hsize);
1325                 else
1326                         bucket = (bucket + 1) % hsize;
1327
1328                 drops = early_drop_list(net, &ct_hash[bucket]);
1329                 rcu_read_unlock();
1330
1331                 if (drops) {
1332                         NF_CT_STAT_ADD_ATOMIC(net, early_drop, drops);
1333                         return true;
1334                 }
1335         }
1336
1337         return false;
1338 }
1339
1340 static bool gc_worker_skip_ct(const struct nf_conn *ct)
1341 {
1342         return !nf_ct_is_confirmed(ct) || nf_ct_is_dying(ct);
1343 }
1344
1345 static bool gc_worker_can_early_drop(const struct nf_conn *ct)
1346 {
1347         const struct nf_conntrack_l4proto *l4proto;
1348
1349         if (!test_bit(IPS_ASSURED_BIT, &ct->status))
1350                 return true;
1351
1352         l4proto = nf_ct_l4proto_find(nf_ct_protonum(ct));
1353         if (l4proto->can_early_drop && l4proto->can_early_drop(ct))
1354                 return true;
1355
1356         return false;
1357 }
1358
1359 static void gc_worker(struct work_struct *work)
1360 {
1361         unsigned int min_interval = max(HZ / GC_MAX_BUCKETS_DIV, 1u);
1362         unsigned int i, goal, buckets = 0, expired_count = 0;
1363         unsigned int nf_conntrack_max95 = 0;
1364         struct conntrack_gc_work *gc_work;
1365         unsigned int ratio, scanned = 0;
1366         unsigned long next_run;
1367
1368         gc_work = container_of(work, struct conntrack_gc_work, dwork.work);
1369
1370         goal = nf_conntrack_htable_size / GC_MAX_BUCKETS_DIV;
1371         i = gc_work->last_bucket;
1372         if (gc_work->early_drop)
1373                 nf_conntrack_max95 = nf_conntrack_max / 100u * 95u;
1374
1375         do {
1376                 struct nf_conntrack_tuple_hash *h;
1377                 struct hlist_nulls_head *ct_hash;
1378                 struct hlist_nulls_node *n;
1379                 unsigned int hashsz;
1380                 struct nf_conn *tmp;
1381
1382                 i++;
1383                 rcu_read_lock();
1384
1385                 nf_conntrack_get_ht(&ct_hash, &hashsz);
1386                 if (i >= hashsz)
1387                         i = 0;
1388
1389                 hlist_nulls_for_each_entry_rcu(h, n, &ct_hash[i], hnnode) {
1390                         struct nf_conntrack_net *cnet;
1391                         struct net *net;
1392
1393                         tmp = nf_ct_tuplehash_to_ctrack(h);
1394
1395                         scanned++;
1396                         if (test_bit(IPS_OFFLOAD_BIT, &tmp->status)) {
1397                                 nf_ct_offload_timeout(tmp);
1398                                 continue;
1399                         }
1400
1401                         if (nf_ct_is_expired(tmp)) {
1402                                 nf_ct_gc_expired(tmp);
1403                                 expired_count++;
1404                                 continue;
1405                         }
1406
1407                         if (nf_conntrack_max95 == 0 || gc_worker_skip_ct(tmp))
1408                                 continue;
1409
1410                         net = nf_ct_net(tmp);
1411                         cnet = nf_ct_pernet(net);
1412                         if (atomic_read(&cnet->count) < nf_conntrack_max95)
1413                                 continue;
1414
1415                         /* need to take reference to avoid possible races */
1416                         if (!atomic_inc_not_zero(&tmp->ct_general.use))
1417                                 continue;
1418
1419                         if (gc_worker_skip_ct(tmp)) {
1420                                 nf_ct_put(tmp);
1421                                 continue;
1422                         }
1423
1424                         if (gc_worker_can_early_drop(tmp))
1425                                 nf_ct_kill(tmp);
1426
1427                         nf_ct_put(tmp);
1428                 }
1429
1430                 /* could check get_nulls_value() here and restart if ct
1431                  * was moved to another chain.  But given gc is best-effort
1432                  * we will just continue with next hash slot.
1433                  */
1434                 rcu_read_unlock();
1435                 cond_resched();
1436         } while (++buckets < goal);
1437
1438         if (gc_work->exiting)
1439                 return;
1440
1441         /*
1442          * Eviction will normally happen from the packet path, and not
1443          * from this gc worker.
1444          *
1445          * This worker is only here to reap expired entries when system went
1446          * idle after a busy period.
1447          *
1448          * The heuristics below are supposed to balance conflicting goals:
1449          *
1450          * 1. Minimize time until we notice a stale entry
1451          * 2. Maximize scan intervals to not waste cycles
1452          *
1453          * Normally, expire ratio will be close to 0.
1454          *
1455          * As soon as a sizeable fraction of the entries have expired
1456          * increase scan frequency.
1457          */
1458         ratio = scanned ? expired_count * 100 / scanned : 0;
1459         if (ratio > GC_EVICT_RATIO) {
1460                 gc_work->next_gc_run = min_interval;
1461         } else {
1462                 unsigned int max = GC_MAX_SCAN_JIFFIES / GC_MAX_BUCKETS_DIV;
1463
1464                 BUILD_BUG_ON((GC_MAX_SCAN_JIFFIES / GC_MAX_BUCKETS_DIV) == 0);
1465
1466                 gc_work->next_gc_run += min_interval;
1467                 if (gc_work->next_gc_run > max)
1468                         gc_work->next_gc_run = max;
1469         }
1470
1471         next_run = gc_work->next_gc_run;
1472         gc_work->last_bucket = i;
1473         gc_work->early_drop = false;
1474         queue_delayed_work(system_power_efficient_wq, &gc_work->dwork, next_run);
1475 }
1476
1477 static void conntrack_gc_work_init(struct conntrack_gc_work *gc_work)
1478 {
1479         INIT_DEFERRABLE_WORK(&gc_work->dwork, gc_worker);
1480         gc_work->next_gc_run = HZ;
1481         gc_work->exiting = false;
1482 }
1483
1484 static struct nf_conn *
1485 __nf_conntrack_alloc(struct net *net,
1486                      const struct nf_conntrack_zone *zone,
1487                      const struct nf_conntrack_tuple *orig,
1488                      const struct nf_conntrack_tuple *repl,
1489                      gfp_t gfp, u32 hash)
1490 {
1491         struct nf_conntrack_net *cnet = nf_ct_pernet(net);
1492         unsigned int ct_count;
1493         struct nf_conn *ct;
1494
1495         /* We don't want any race condition at early drop stage */
1496         ct_count = atomic_inc_return(&cnet->count);
1497
1498         if (nf_conntrack_max && unlikely(ct_count > nf_conntrack_max)) {
1499                 if (!early_drop(net, hash)) {
1500                         if (!conntrack_gc_work.early_drop)
1501                                 conntrack_gc_work.early_drop = true;
1502                         atomic_dec(&cnet->count);
1503                         net_warn_ratelimited("nf_conntrack: table full, dropping packet\n");
1504                         return ERR_PTR(-ENOMEM);
1505                 }
1506         }
1507
1508         /*
1509          * Do not use kmem_cache_zalloc(), as this cache uses
1510          * SLAB_TYPESAFE_BY_RCU.
1511          */
1512         ct = kmem_cache_alloc(nf_conntrack_cachep, gfp);
1513         if (ct == NULL)
1514                 goto out;
1515
1516         spin_lock_init(&ct->lock);
1517         ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple = *orig;
1518         ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode.pprev = NULL;
1519         ct->tuplehash[IP_CT_DIR_REPLY].tuple = *repl;
1520         /* save hash for reusing when confirming */
1521         *(unsigned long *)(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode.pprev) = hash;
1522         ct->status = 0;
1523         ct->timeout = 0;
1524         write_pnet(&ct->ct_net, net);
1525         memset(&ct->__nfct_init_offset, 0,
1526                offsetof(struct nf_conn, proto) -
1527                offsetof(struct nf_conn, __nfct_init_offset));
1528
1529         nf_ct_zone_add(ct, zone);
1530
1531         /* Because we use RCU lookups, we set ct_general.use to zero before
1532          * this is inserted in any list.
1533          */
1534         atomic_set(&ct->ct_general.use, 0);
1535         return ct;
1536 out:
1537         atomic_dec(&cnet->count);
1538         return ERR_PTR(-ENOMEM);
1539 }
1540
1541 struct nf_conn *nf_conntrack_alloc(struct net *net,
1542                                    const struct nf_conntrack_zone *zone,
1543                                    const struct nf_conntrack_tuple *orig,
1544                                    const struct nf_conntrack_tuple *repl,
1545                                    gfp_t gfp)
1546 {
1547         return __nf_conntrack_alloc(net, zone, orig, repl, gfp, 0);
1548 }
1549 EXPORT_SYMBOL_GPL(nf_conntrack_alloc);
1550
1551 void nf_conntrack_free(struct nf_conn *ct)
1552 {
1553         struct net *net = nf_ct_net(ct);
1554         struct nf_conntrack_net *cnet;
1555
1556         /* A freed object has refcnt == 0, that's
1557          * the golden rule for SLAB_TYPESAFE_BY_RCU
1558          */
1559         WARN_ON(atomic_read(&ct->ct_general.use) != 0);
1560
1561         nf_ct_ext_destroy(ct);
1562         kmem_cache_free(nf_conntrack_cachep, ct);
1563         cnet = nf_ct_pernet(net);
1564
1565         smp_mb__before_atomic();
1566         atomic_dec(&cnet->count);
1567 }
1568 EXPORT_SYMBOL_GPL(nf_conntrack_free);
1569
1570
1571 /* Allocate a new conntrack: we return -ENOMEM if classification
1572    failed due to stress.  Otherwise it really is unclassifiable. */
1573 static noinline struct nf_conntrack_tuple_hash *
1574 init_conntrack(struct net *net, struct nf_conn *tmpl,
1575                const struct nf_conntrack_tuple *tuple,
1576                struct sk_buff *skb,
1577                unsigned int dataoff, u32 hash)
1578 {
1579         struct nf_conn *ct;
1580         struct nf_conn_help *help;
1581         struct nf_conntrack_tuple repl_tuple;
1582         struct nf_conntrack_ecache *ecache;
1583         struct nf_conntrack_expect *exp = NULL;
1584         const struct nf_conntrack_zone *zone;
1585         struct nf_conn_timeout *timeout_ext;
1586         struct nf_conntrack_zone tmp;
1587         struct nf_conntrack_net *cnet;
1588
1589         if (!nf_ct_invert_tuple(&repl_tuple, tuple)) {
1590                 pr_debug("Can't invert tuple.\n");
1591                 return NULL;
1592         }
1593
1594         zone = nf_ct_zone_tmpl(tmpl, skb, &tmp);
1595         ct = __nf_conntrack_alloc(net, zone, tuple, &repl_tuple, GFP_ATOMIC,
1596                                   hash);
1597         if (IS_ERR(ct))
1598                 return (struct nf_conntrack_tuple_hash *)ct;
1599
1600         if (!nf_ct_add_synproxy(ct, tmpl)) {
1601                 nf_conntrack_free(ct);
1602                 return ERR_PTR(-ENOMEM);
1603         }
1604
1605         timeout_ext = tmpl ? nf_ct_timeout_find(tmpl) : NULL;
1606
1607         if (timeout_ext)
1608                 nf_ct_timeout_ext_add(ct, rcu_dereference(timeout_ext->timeout),
1609                                       GFP_ATOMIC);
1610
1611         nf_ct_acct_ext_add(ct, GFP_ATOMIC);
1612         nf_ct_tstamp_ext_add(ct, GFP_ATOMIC);
1613         nf_ct_labels_ext_add(ct);
1614
1615         ecache = tmpl ? nf_ct_ecache_find(tmpl) : NULL;
1616         nf_ct_ecache_ext_add(ct, ecache ? ecache->ctmask : 0,
1617                                  ecache ? ecache->expmask : 0,
1618                              GFP_ATOMIC);
1619
1620         local_bh_disable();
1621         cnet = nf_ct_pernet(net);
1622         if (cnet->expect_count) {
1623                 spin_lock(&nf_conntrack_expect_lock);
1624                 exp = nf_ct_find_expectation(net, zone, tuple);
1625                 if (exp) {
1626                         pr_debug("expectation arrives ct=%p exp=%p\n",
1627                                  ct, exp);
1628                         /* Welcome, Mr. Bond.  We've been expecting you... */
1629                         __set_bit(IPS_EXPECTED_BIT, &ct->status);
1630                         /* exp->master safe, refcnt bumped in nf_ct_find_expectation */
1631                         ct->master = exp->master;
1632                         if (exp->helper) {
1633                                 help = nf_ct_helper_ext_add(ct, GFP_ATOMIC);
1634                                 if (help)
1635                                         rcu_assign_pointer(help->helper, exp->helper);
1636                         }
1637
1638 #ifdef CONFIG_NF_CONNTRACK_MARK
1639                         ct->mark = exp->master->mark;
1640 #endif
1641 #ifdef CONFIG_NF_CONNTRACK_SECMARK
1642                         ct->secmark = exp->master->secmark;
1643 #endif
1644                         NF_CT_STAT_INC(net, expect_new);
1645                 }
1646                 spin_unlock(&nf_conntrack_expect_lock);
1647         }
1648         if (!exp)
1649                 __nf_ct_try_assign_helper(ct, tmpl, GFP_ATOMIC);
1650
1651         /* Now it is inserted into the unconfirmed list, bump refcount */
1652         nf_conntrack_get(&ct->ct_general);
1653         nf_ct_add_to_unconfirmed_list(ct);
1654
1655         local_bh_enable();
1656
1657         if (exp) {
1658                 if (exp->expectfn)
1659                         exp->expectfn(ct, exp);
1660                 nf_ct_expect_put(exp);
1661         }
1662
1663         return &ct->tuplehash[IP_CT_DIR_ORIGINAL];
1664 }
1665
1666 /* On success, returns 0, sets skb->_nfct | ctinfo */
1667 static int
1668 resolve_normal_ct(struct nf_conn *tmpl,
1669                   struct sk_buff *skb,
1670                   unsigned int dataoff,
1671                   u_int8_t protonum,
1672                   const struct nf_hook_state *state)
1673 {
1674         const struct nf_conntrack_zone *zone;
1675         struct nf_conntrack_tuple tuple;
1676         struct nf_conntrack_tuple_hash *h;
1677         enum ip_conntrack_info ctinfo;
1678         struct nf_conntrack_zone tmp;
1679         struct nf_conn *ct;
1680         u32 hash;
1681
1682         if (!nf_ct_get_tuple(skb, skb_network_offset(skb),
1683                              dataoff, state->pf, protonum, state->net,
1684                              &tuple)) {
1685                 pr_debug("Can't get tuple\n");
1686                 return 0;
1687         }
1688
1689         /* look for tuple match */
1690         zone = nf_ct_zone_tmpl(tmpl, skb, &tmp);
1691         hash = hash_conntrack_raw(&tuple, state->net);
1692         h = __nf_conntrack_find_get(state->net, zone, &tuple, hash);
1693         if (!h) {
1694                 h = init_conntrack(state->net, tmpl, &tuple,
1695                                    skb, dataoff, hash);
1696                 if (!h)
1697                         return 0;
1698                 if (IS_ERR(h))
1699                         return PTR_ERR(h);
1700         }
1701         ct = nf_ct_tuplehash_to_ctrack(h);
1702
1703         /* It exists; we have (non-exclusive) reference. */
1704         if (NF_CT_DIRECTION(h) == IP_CT_DIR_REPLY) {
1705                 ctinfo = IP_CT_ESTABLISHED_REPLY;
1706         } else {
1707                 /* Once we've had two way comms, always ESTABLISHED. */
1708                 if (test_bit(IPS_SEEN_REPLY_BIT, &ct->status)) {
1709                         pr_debug("normal packet for %p\n", ct);
1710                         ctinfo = IP_CT_ESTABLISHED;
1711                 } else if (test_bit(IPS_EXPECTED_BIT, &ct->status)) {
1712                         pr_debug("related packet for %p\n", ct);
1713                         ctinfo = IP_CT_RELATED;
1714                 } else {
1715                         pr_debug("new packet for %p\n", ct);
1716                         ctinfo = IP_CT_NEW;
1717                 }
1718         }
1719         nf_ct_set(skb, ct, ctinfo);
1720         return 0;
1721 }
1722
1723 /*
1724  * icmp packets need special treatment to handle error messages that are
1725  * related to a connection.
1726  *
1727  * Callers need to check if skb has a conntrack assigned when this
1728  * helper returns; in such case skb belongs to an already known connection.
1729  */
1730 static unsigned int __cold
1731 nf_conntrack_handle_icmp(struct nf_conn *tmpl,
1732                          struct sk_buff *skb,
1733                          unsigned int dataoff,
1734                          u8 protonum,
1735                          const struct nf_hook_state *state)
1736 {
1737         int ret;
1738
1739         if (state->pf == NFPROTO_IPV4 && protonum == IPPROTO_ICMP)
1740                 ret = nf_conntrack_icmpv4_error(tmpl, skb, dataoff, state);
1741 #if IS_ENABLED(CONFIG_IPV6)
1742         else if (state->pf == NFPROTO_IPV6 && protonum == IPPROTO_ICMPV6)
1743                 ret = nf_conntrack_icmpv6_error(tmpl, skb, dataoff, state);
1744 #endif
1745         else
1746                 return NF_ACCEPT;
1747
1748         if (ret <= 0)
1749                 NF_CT_STAT_INC_ATOMIC(state->net, error);
1750
1751         return ret;
1752 }
1753
1754 static int generic_packet(struct nf_conn *ct, struct sk_buff *skb,
1755                           enum ip_conntrack_info ctinfo)
1756 {
1757         const unsigned int *timeout = nf_ct_timeout_lookup(ct);
1758
1759         if (!timeout)
1760                 timeout = &nf_generic_pernet(nf_ct_net(ct))->timeout;
1761
1762         nf_ct_refresh_acct(ct, ctinfo, skb, *timeout);
1763         return NF_ACCEPT;
1764 }
1765
1766 /* Returns verdict for packet, or -1 for invalid. */
1767 static int nf_conntrack_handle_packet(struct nf_conn *ct,
1768                                       struct sk_buff *skb,
1769                                       unsigned int dataoff,
1770                                       enum ip_conntrack_info ctinfo,
1771                                       const struct nf_hook_state *state)
1772 {
1773         switch (nf_ct_protonum(ct)) {
1774         case IPPROTO_TCP:
1775                 return nf_conntrack_tcp_packet(ct, skb, dataoff,
1776                                                ctinfo, state);
1777         case IPPROTO_UDP:
1778                 return nf_conntrack_udp_packet(ct, skb, dataoff,
1779                                                ctinfo, state);
1780         case IPPROTO_ICMP:
1781                 return nf_conntrack_icmp_packet(ct, skb, ctinfo, state);
1782 #if IS_ENABLED(CONFIG_IPV6)
1783         case IPPROTO_ICMPV6:
1784                 return nf_conntrack_icmpv6_packet(ct, skb, ctinfo, state);
1785 #endif
1786 #ifdef CONFIG_NF_CT_PROTO_UDPLITE
1787         case IPPROTO_UDPLITE:
1788                 return nf_conntrack_udplite_packet(ct, skb, dataoff,
1789                                                    ctinfo, state);
1790 #endif
1791 #ifdef CONFIG_NF_CT_PROTO_SCTP
1792         case IPPROTO_SCTP:
1793                 return nf_conntrack_sctp_packet(ct, skb, dataoff,
1794                                                 ctinfo, state);
1795 #endif
1796 #ifdef CONFIG_NF_CT_PROTO_DCCP
1797         case IPPROTO_DCCP:
1798                 return nf_conntrack_dccp_packet(ct, skb, dataoff,
1799                                                 ctinfo, state);
1800 #endif
1801 #ifdef CONFIG_NF_CT_PROTO_GRE
1802         case IPPROTO_GRE:
1803                 return nf_conntrack_gre_packet(ct, skb, dataoff,
1804                                                ctinfo, state);
1805 #endif
1806         }
1807
1808         return generic_packet(ct, skb, ctinfo);
1809 }
1810
1811 unsigned int
1812 nf_conntrack_in(struct sk_buff *skb, const struct nf_hook_state *state)
1813 {
1814         enum ip_conntrack_info ctinfo;
1815         struct nf_conn *ct, *tmpl;
1816         u_int8_t protonum;
1817         int dataoff, ret;
1818
1819         tmpl = nf_ct_get(skb, &ctinfo);
1820         if (tmpl || ctinfo == IP_CT_UNTRACKED) {
1821                 /* Previously seen (loopback or untracked)?  Ignore. */
1822                 if ((tmpl && !nf_ct_is_template(tmpl)) ||
1823                      ctinfo == IP_CT_UNTRACKED)
1824                         return NF_ACCEPT;
1825                 skb->_nfct = 0;
1826         }
1827
1828         /* rcu_read_lock()ed by nf_hook_thresh */
1829         dataoff = get_l4proto(skb, skb_network_offset(skb), state->pf, &protonum);
1830         if (dataoff <= 0) {
1831                 pr_debug("not prepared to track yet or error occurred\n");
1832                 NF_CT_STAT_INC_ATOMIC(state->net, invalid);
1833                 ret = NF_ACCEPT;
1834                 goto out;
1835         }
1836
1837         if (protonum == IPPROTO_ICMP || protonum == IPPROTO_ICMPV6) {
1838                 ret = nf_conntrack_handle_icmp(tmpl, skb, dataoff,
1839                                                protonum, state);
1840                 if (ret <= 0) {
1841                         ret = -ret;
1842                         goto out;
1843                 }
1844                 /* ICMP[v6] protocol trackers may assign one conntrack. */
1845                 if (skb->_nfct)
1846                         goto out;
1847         }
1848 repeat:
1849         ret = resolve_normal_ct(tmpl, skb, dataoff,
1850                                 protonum, state);
1851         if (ret < 0) {
1852                 /* Too stressed to deal. */
1853                 NF_CT_STAT_INC_ATOMIC(state->net, drop);
1854                 ret = NF_DROP;
1855                 goto out;
1856         }
1857
1858         ct = nf_ct_get(skb, &ctinfo);
1859         if (!ct) {
1860                 /* Not valid part of a connection */
1861                 NF_CT_STAT_INC_ATOMIC(state->net, invalid);
1862                 ret = NF_ACCEPT;
1863                 goto out;
1864         }
1865
1866         ret = nf_conntrack_handle_packet(ct, skb, dataoff, ctinfo, state);
1867         if (ret <= 0) {
1868                 /* Invalid: inverse of the return code tells
1869                  * the netfilter core what to do */
1870                 pr_debug("nf_conntrack_in: Can't track with proto module\n");
1871                 nf_conntrack_put(&ct->ct_general);
1872                 skb->_nfct = 0;
1873                 NF_CT_STAT_INC_ATOMIC(state->net, invalid);
1874                 if (ret == -NF_DROP)
1875                         NF_CT_STAT_INC_ATOMIC(state->net, drop);
1876                 /* Special case: TCP tracker reports an attempt to reopen a
1877                  * closed/aborted connection. We have to go back and create a
1878                  * fresh conntrack.
1879                  */
1880                 if (ret == -NF_REPEAT)
1881                         goto repeat;
1882                 ret = -ret;
1883                 goto out;
1884         }
1885
1886         if (ctinfo == IP_CT_ESTABLISHED_REPLY &&
1887             !test_and_set_bit(IPS_SEEN_REPLY_BIT, &ct->status))
1888                 nf_conntrack_event_cache(IPCT_REPLY, ct);
1889 out:
1890         if (tmpl)
1891                 nf_ct_put(tmpl);
1892
1893         return ret;
1894 }
1895 EXPORT_SYMBOL_GPL(nf_conntrack_in);
1896
1897 /* Alter reply tuple (maybe alter helper).  This is for NAT, and is
1898    implicitly racy: see __nf_conntrack_confirm */
1899 void nf_conntrack_alter_reply(struct nf_conn *ct,
1900                               const struct nf_conntrack_tuple *newreply)
1901 {
1902         struct nf_conn_help *help = nfct_help(ct);
1903
1904         /* Should be unconfirmed, so not in hash table yet */
1905         WARN_ON(nf_ct_is_confirmed(ct));
1906
1907         pr_debug("Altering reply tuple of %p to ", ct);
1908         nf_ct_dump_tuple(newreply);
1909
1910         ct->tuplehash[IP_CT_DIR_REPLY].tuple = *newreply;
1911         if (ct->master || (help && !hlist_empty(&help->expectations)))
1912                 return;
1913
1914         rcu_read_lock();
1915         __nf_ct_try_assign_helper(ct, NULL, GFP_ATOMIC);
1916         rcu_read_unlock();
1917 }
1918 EXPORT_SYMBOL_GPL(nf_conntrack_alter_reply);
1919
1920 /* Refresh conntrack for this many jiffies and do accounting if do_acct is 1 */
1921 void __nf_ct_refresh_acct(struct nf_conn *ct,
1922                           enum ip_conntrack_info ctinfo,
1923                           const struct sk_buff *skb,
1924                           u32 extra_jiffies,
1925                           bool do_acct)
1926 {
1927         /* Only update if this is not a fixed timeout */
1928         if (test_bit(IPS_FIXED_TIMEOUT_BIT, &ct->status))
1929                 goto acct;
1930
1931         /* If not in hash table, timer will not be active yet */
1932         if (nf_ct_is_confirmed(ct))
1933                 extra_jiffies += nfct_time_stamp;
1934
1935         if (READ_ONCE(ct->timeout) != extra_jiffies)
1936                 WRITE_ONCE(ct->timeout, extra_jiffies);
1937 acct:
1938         if (do_acct)
1939                 nf_ct_acct_update(ct, CTINFO2DIR(ctinfo), skb->len);
1940 }
1941 EXPORT_SYMBOL_GPL(__nf_ct_refresh_acct);
1942
1943 bool nf_ct_kill_acct(struct nf_conn *ct,
1944                      enum ip_conntrack_info ctinfo,
1945                      const struct sk_buff *skb)
1946 {
1947         nf_ct_acct_update(ct, CTINFO2DIR(ctinfo), skb->len);
1948
1949         return nf_ct_delete(ct, 0, 0);
1950 }
1951 EXPORT_SYMBOL_GPL(nf_ct_kill_acct);
1952
1953 #if IS_ENABLED(CONFIG_NF_CT_NETLINK)
1954
1955 #include <linux/netfilter/nfnetlink.h>
1956 #include <linux/netfilter/nfnetlink_conntrack.h>
1957 #include <linux/mutex.h>
1958
1959 /* Generic function for tcp/udp/sctp/dccp and alike. */
1960 int nf_ct_port_tuple_to_nlattr(struct sk_buff *skb,
1961                                const struct nf_conntrack_tuple *tuple)
1962 {
1963         if (nla_put_be16(skb, CTA_PROTO_SRC_PORT, tuple->src.u.tcp.port) ||
1964             nla_put_be16(skb, CTA_PROTO_DST_PORT, tuple->dst.u.tcp.port))
1965                 goto nla_put_failure;
1966         return 0;
1967
1968 nla_put_failure:
1969         return -1;
1970 }
1971 EXPORT_SYMBOL_GPL(nf_ct_port_tuple_to_nlattr);
1972
1973 const struct nla_policy nf_ct_port_nla_policy[CTA_PROTO_MAX+1] = {
1974         [CTA_PROTO_SRC_PORT]  = { .type = NLA_U16 },
1975         [CTA_PROTO_DST_PORT]  = { .type = NLA_U16 },
1976 };
1977 EXPORT_SYMBOL_GPL(nf_ct_port_nla_policy);
1978
1979 int nf_ct_port_nlattr_to_tuple(struct nlattr *tb[],
1980                                struct nf_conntrack_tuple *t,
1981                                u_int32_t flags)
1982 {
1983         if (flags & CTA_FILTER_FLAG(CTA_PROTO_SRC_PORT)) {
1984                 if (!tb[CTA_PROTO_SRC_PORT])
1985                         return -EINVAL;
1986
1987                 t->src.u.tcp.port = nla_get_be16(tb[CTA_PROTO_SRC_PORT]);
1988         }
1989
1990         if (flags & CTA_FILTER_FLAG(CTA_PROTO_DST_PORT)) {
1991                 if (!tb[CTA_PROTO_DST_PORT])
1992                         return -EINVAL;
1993
1994                 t->dst.u.tcp.port = nla_get_be16(tb[CTA_PROTO_DST_PORT]);
1995         }
1996
1997         return 0;
1998 }
1999 EXPORT_SYMBOL_GPL(nf_ct_port_nlattr_to_tuple);
2000
2001 unsigned int nf_ct_port_nlattr_tuple_size(void)
2002 {
2003         static unsigned int size __read_mostly;
2004
2005         if (!size)
2006                 size = nla_policy_len(nf_ct_port_nla_policy, CTA_PROTO_MAX + 1);
2007
2008         return size;
2009 }
2010 EXPORT_SYMBOL_GPL(nf_ct_port_nlattr_tuple_size);
2011 #endif
2012
2013 /* Used by ipt_REJECT and ip6t_REJECT. */
2014 static void nf_conntrack_attach(struct sk_buff *nskb, const struct sk_buff *skb)
2015 {
2016         struct nf_conn *ct;
2017         enum ip_conntrack_info ctinfo;
2018
2019         /* This ICMP is in reverse direction to the packet which caused it */
2020         ct = nf_ct_get(skb, &ctinfo);
2021         if (CTINFO2DIR(ctinfo) == IP_CT_DIR_ORIGINAL)
2022                 ctinfo = IP_CT_RELATED_REPLY;
2023         else
2024                 ctinfo = IP_CT_RELATED;
2025
2026         /* Attach to new skbuff, and increment count */
2027         nf_ct_set(nskb, ct, ctinfo);
2028         nf_conntrack_get(skb_nfct(nskb));
2029 }
2030
2031 static int __nf_conntrack_update(struct net *net, struct sk_buff *skb,
2032                                  struct nf_conn *ct,
2033                                  enum ip_conntrack_info ctinfo)
2034 {
2035         struct nf_conntrack_tuple_hash *h;
2036         struct nf_conntrack_tuple tuple;
2037         struct nf_nat_hook *nat_hook;
2038         unsigned int status;
2039         int dataoff;
2040         u16 l3num;
2041         u8 l4num;
2042
2043         l3num = nf_ct_l3num(ct);
2044
2045         dataoff = get_l4proto(skb, skb_network_offset(skb), l3num, &l4num);
2046         if (dataoff <= 0)
2047                 return -1;
2048
2049         if (!nf_ct_get_tuple(skb, skb_network_offset(skb), dataoff, l3num,
2050                              l4num, net, &tuple))
2051                 return -1;
2052
2053         if (ct->status & IPS_SRC_NAT) {
2054                 memcpy(tuple.src.u3.all,
2055                        ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple.src.u3.all,
2056                        sizeof(tuple.src.u3.all));
2057                 tuple.src.u.all =
2058                         ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple.src.u.all;
2059         }
2060
2061         if (ct->status & IPS_DST_NAT) {
2062                 memcpy(tuple.dst.u3.all,
2063                        ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple.dst.u3.all,
2064                        sizeof(tuple.dst.u3.all));
2065                 tuple.dst.u.all =
2066                         ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple.dst.u.all;
2067         }
2068
2069         h = nf_conntrack_find_get(net, nf_ct_zone(ct), &tuple);
2070         if (!h)
2071                 return 0;
2072
2073         /* Store status bits of the conntrack that is clashing to re-do NAT
2074          * mangling according to what it has been done already to this packet.
2075          */
2076         status = ct->status;
2077
2078         nf_ct_put(ct);
2079         ct = nf_ct_tuplehash_to_ctrack(h);
2080         nf_ct_set(skb, ct, ctinfo);
2081
2082         nat_hook = rcu_dereference(nf_nat_hook);
2083         if (!nat_hook)
2084                 return 0;
2085
2086         if (status & IPS_SRC_NAT &&
2087             nat_hook->manip_pkt(skb, ct, NF_NAT_MANIP_SRC,
2088                                 IP_CT_DIR_ORIGINAL) == NF_DROP)
2089                 return -1;
2090
2091         if (status & IPS_DST_NAT &&
2092             nat_hook->manip_pkt(skb, ct, NF_NAT_MANIP_DST,
2093                                 IP_CT_DIR_ORIGINAL) == NF_DROP)
2094                 return -1;
2095
2096         return 0;
2097 }
2098
2099 /* This packet is coming from userspace via nf_queue, complete the packet
2100  * processing after the helper invocation in nf_confirm().
2101  */
2102 static int nf_confirm_cthelper(struct sk_buff *skb, struct nf_conn *ct,
2103                                enum ip_conntrack_info ctinfo)
2104 {
2105         const struct nf_conntrack_helper *helper;
2106         const struct nf_conn_help *help;
2107         int protoff;
2108
2109         help = nfct_help(ct);
2110         if (!help)
2111                 return 0;
2112
2113         helper = rcu_dereference(help->helper);
2114         if (!(helper->flags & NF_CT_HELPER_F_USERSPACE))
2115                 return 0;
2116
2117         switch (nf_ct_l3num(ct)) {
2118         case NFPROTO_IPV4:
2119                 protoff = skb_network_offset(skb) + ip_hdrlen(skb);
2120                 break;
2121 #if IS_ENABLED(CONFIG_IPV6)
2122         case NFPROTO_IPV6: {
2123                 __be16 frag_off;
2124                 u8 pnum;
2125
2126                 pnum = ipv6_hdr(skb)->nexthdr;
2127                 protoff = ipv6_skip_exthdr(skb, sizeof(struct ipv6hdr), &pnum,
2128                                            &frag_off);
2129                 if (protoff < 0 || (frag_off & htons(~0x7)) != 0)
2130                         return 0;
2131                 break;
2132         }
2133 #endif
2134         default:
2135                 return 0;
2136         }
2137
2138         if (test_bit(IPS_SEQ_ADJUST_BIT, &ct->status) &&
2139             !nf_is_loopback_packet(skb)) {
2140                 if (!nf_ct_seq_adjust(skb, ct, ctinfo, protoff)) {
2141                         NF_CT_STAT_INC_ATOMIC(nf_ct_net(ct), drop);
2142                         return -1;
2143                 }
2144         }
2145
2146         /* We've seen it coming out the other side: confirm it */
2147         return nf_conntrack_confirm(skb) == NF_DROP ? - 1 : 0;
2148 }
2149
2150 static int nf_conntrack_update(struct net *net, struct sk_buff *skb)
2151 {
2152         enum ip_conntrack_info ctinfo;
2153         struct nf_conn *ct;
2154         int err;
2155
2156         ct = nf_ct_get(skb, &ctinfo);
2157         if (!ct)
2158                 return 0;
2159
2160         if (!nf_ct_is_confirmed(ct)) {
2161                 err = __nf_conntrack_update(net, skb, ct, ctinfo);
2162                 if (err < 0)
2163                         return err;
2164
2165                 ct = nf_ct_get(skb, &ctinfo);
2166         }
2167
2168         return nf_confirm_cthelper(skb, ct, ctinfo);
2169 }
2170
2171 static bool nf_conntrack_get_tuple_skb(struct nf_conntrack_tuple *dst_tuple,
2172                                        const struct sk_buff *skb)
2173 {
2174         const struct nf_conntrack_tuple *src_tuple;
2175         const struct nf_conntrack_tuple_hash *hash;
2176         struct nf_conntrack_tuple srctuple;
2177         enum ip_conntrack_info ctinfo;
2178         struct nf_conn *ct;
2179
2180         ct = nf_ct_get(skb, &ctinfo);
2181         if (ct) {
2182                 src_tuple = nf_ct_tuple(ct, CTINFO2DIR(ctinfo));
2183                 memcpy(dst_tuple, src_tuple, sizeof(*dst_tuple));
2184                 return true;
2185         }
2186
2187         if (!nf_ct_get_tuplepr(skb, skb_network_offset(skb),
2188                                NFPROTO_IPV4, dev_net(skb->dev),
2189                                &srctuple))
2190                 return false;
2191
2192         hash = nf_conntrack_find_get(dev_net(skb->dev),
2193                                      &nf_ct_zone_dflt,
2194                                      &srctuple);
2195         if (!hash)
2196                 return false;
2197
2198         ct = nf_ct_tuplehash_to_ctrack(hash);
2199         src_tuple = nf_ct_tuple(ct, !hash->tuple.dst.dir);
2200         memcpy(dst_tuple, src_tuple, sizeof(*dst_tuple));
2201         nf_ct_put(ct);
2202
2203         return true;
2204 }
2205
2206 /* Bring out ya dead! */
2207 static struct nf_conn *
2208 get_next_corpse(int (*iter)(struct nf_conn *i, void *data),
2209                 void *data, unsigned int *bucket)
2210 {
2211         struct nf_conntrack_tuple_hash *h;
2212         struct nf_conn *ct;
2213         struct hlist_nulls_node *n;
2214         spinlock_t *lockp;
2215
2216         for (; *bucket < nf_conntrack_htable_size; (*bucket)++) {
2217                 lockp = &nf_conntrack_locks[*bucket % CONNTRACK_LOCKS];
2218                 local_bh_disable();
2219                 nf_conntrack_lock(lockp);
2220                 if (*bucket < nf_conntrack_htable_size) {
2221                         hlist_nulls_for_each_entry(h, n, &nf_conntrack_hash[*bucket], hnnode) {
2222                                 if (NF_CT_DIRECTION(h) != IP_CT_DIR_REPLY)
2223                                         continue;
2224                                 /* All nf_conn objects are added to hash table twice, one
2225                                  * for original direction tuple, once for the reply tuple.
2226                                  *
2227                                  * Exception: In the IPS_NAT_CLASH case, only the reply
2228                                  * tuple is added (the original tuple already existed for
2229                                  * a different object).
2230                                  *
2231                                  * We only need to call the iterator once for each
2232                                  * conntrack, so we just use the 'reply' direction
2233                                  * tuple while iterating.
2234                                  */
2235                                 ct = nf_ct_tuplehash_to_ctrack(h);
2236                                 if (iter(ct, data))
2237                                         goto found;
2238                         }
2239                 }
2240                 spin_unlock(lockp);
2241                 local_bh_enable();
2242                 cond_resched();
2243         }
2244
2245         return NULL;
2246 found:
2247         atomic_inc(&ct->ct_general.use);
2248         spin_unlock(lockp);
2249         local_bh_enable();
2250         return ct;
2251 }
2252
2253 static void nf_ct_iterate_cleanup(int (*iter)(struct nf_conn *i, void *data),
2254                                   void *data, u32 portid, int report)
2255 {
2256         unsigned int bucket = 0, sequence;
2257         struct nf_conn *ct;
2258
2259         might_sleep();
2260
2261         for (;;) {
2262                 sequence = read_seqcount_begin(&nf_conntrack_generation);
2263
2264                 while ((ct = get_next_corpse(iter, data, &bucket)) != NULL) {
2265                         /* Time to push up daises... */
2266
2267                         nf_ct_delete(ct, portid, report);
2268                         nf_ct_put(ct);
2269                         cond_resched();
2270                 }
2271
2272                 if (!read_seqcount_retry(&nf_conntrack_generation, sequence))
2273                         break;
2274                 bucket = 0;
2275         }
2276 }
2277
2278 struct iter_data {
2279         int (*iter)(struct nf_conn *i, void *data);
2280         void *data;
2281         struct net *net;
2282 };
2283
2284 static int iter_net_only(struct nf_conn *i, void *data)
2285 {
2286         struct iter_data *d = data;
2287
2288         if (!net_eq(d->net, nf_ct_net(i)))
2289                 return 0;
2290
2291         return d->iter(i, d->data);
2292 }
2293
2294 static void
2295 __nf_ct_unconfirmed_destroy(struct net *net)
2296 {
2297         int cpu;
2298
2299         for_each_possible_cpu(cpu) {
2300                 struct nf_conntrack_tuple_hash *h;
2301                 struct hlist_nulls_node *n;
2302                 struct ct_pcpu *pcpu;
2303
2304                 pcpu = per_cpu_ptr(net->ct.pcpu_lists, cpu);
2305
2306                 spin_lock_bh(&pcpu->lock);
2307                 hlist_nulls_for_each_entry(h, n, &pcpu->unconfirmed, hnnode) {
2308                         struct nf_conn *ct;
2309
2310                         ct = nf_ct_tuplehash_to_ctrack(h);
2311
2312                         /* we cannot call iter() on unconfirmed list, the
2313                          * owning cpu can reallocate ct->ext at any time.
2314                          */
2315                         set_bit(IPS_DYING_BIT, &ct->status);
2316                 }
2317                 spin_unlock_bh(&pcpu->lock);
2318                 cond_resched();
2319         }
2320 }
2321
2322 void nf_ct_unconfirmed_destroy(struct net *net)
2323 {
2324         struct nf_conntrack_net *cnet = nf_ct_pernet(net);
2325
2326         might_sleep();
2327
2328         if (atomic_read(&cnet->count) > 0) {
2329                 __nf_ct_unconfirmed_destroy(net);
2330                 nf_queue_nf_hook_drop(net);
2331                 synchronize_net();
2332         }
2333 }
2334 EXPORT_SYMBOL_GPL(nf_ct_unconfirmed_destroy);
2335
2336 void nf_ct_iterate_cleanup_net(struct net *net,
2337                                int (*iter)(struct nf_conn *i, void *data),
2338                                void *data, u32 portid, int report)
2339 {
2340         struct nf_conntrack_net *cnet = nf_ct_pernet(net);
2341         struct iter_data d;
2342
2343         might_sleep();
2344
2345         if (atomic_read(&cnet->count) == 0)
2346                 return;
2347
2348         d.iter = iter;
2349         d.data = data;
2350         d.net = net;
2351
2352         nf_ct_iterate_cleanup(iter_net_only, &d, portid, report);
2353 }
2354 EXPORT_SYMBOL_GPL(nf_ct_iterate_cleanup_net);
2355
2356 /**
2357  * nf_ct_iterate_destroy - destroy unconfirmed conntracks and iterate table
2358  * @iter: callback to invoke for each conntrack
2359  * @data: data to pass to @iter
2360  *
2361  * Like nf_ct_iterate_cleanup, but first marks conntracks on the
2362  * unconfirmed list as dying (so they will not be inserted into
2363  * main table).
2364  *
2365  * Can only be called in module exit path.
2366  */
2367 void
2368 nf_ct_iterate_destroy(int (*iter)(struct nf_conn *i, void *data), void *data)
2369 {
2370         struct net *net;
2371
2372         down_read(&net_rwsem);
2373         for_each_net(net) {
2374                 struct nf_conntrack_net *cnet = nf_ct_pernet(net);
2375
2376                 if (atomic_read(&cnet->count) == 0)
2377                         continue;
2378                 __nf_ct_unconfirmed_destroy(net);
2379                 nf_queue_nf_hook_drop(net);
2380         }
2381         up_read(&net_rwsem);
2382
2383         /* Need to wait for netns cleanup worker to finish, if its
2384          * running -- it might have deleted a net namespace from
2385          * the global list, so our __nf_ct_unconfirmed_destroy() might
2386          * not have affected all namespaces.
2387          */
2388         net_ns_barrier();
2389
2390         /* a conntrack could have been unlinked from unconfirmed list
2391          * before we grabbed pcpu lock in __nf_ct_unconfirmed_destroy().
2392          * This makes sure its inserted into conntrack table.
2393          */
2394         synchronize_net();
2395
2396         nf_ct_iterate_cleanup(iter, data, 0, 0);
2397 }
2398 EXPORT_SYMBOL_GPL(nf_ct_iterate_destroy);
2399
2400 static int kill_all(struct nf_conn *i, void *data)
2401 {
2402         return net_eq(nf_ct_net(i), data);
2403 }
2404
2405 void nf_conntrack_cleanup_start(void)
2406 {
2407         conntrack_gc_work.exiting = true;
2408         RCU_INIT_POINTER(ip_ct_attach, NULL);
2409 }
2410
2411 void nf_conntrack_cleanup_end(void)
2412 {
2413         RCU_INIT_POINTER(nf_ct_hook, NULL);
2414         cancel_delayed_work_sync(&conntrack_gc_work.dwork);
2415         kvfree(nf_conntrack_hash);
2416
2417         nf_conntrack_proto_fini();
2418         nf_conntrack_seqadj_fini();
2419         nf_conntrack_labels_fini();
2420         nf_conntrack_helper_fini();
2421         nf_conntrack_timeout_fini();
2422         nf_conntrack_ecache_fini();
2423         nf_conntrack_tstamp_fini();
2424         nf_conntrack_acct_fini();
2425         nf_conntrack_expect_fini();
2426
2427         kmem_cache_destroy(nf_conntrack_cachep);
2428 }
2429
2430 /*
2431  * Mishearing the voices in his head, our hero wonders how he's
2432  * supposed to kill the mall.
2433  */
2434 void nf_conntrack_cleanup_net(struct net *net)
2435 {
2436         LIST_HEAD(single);
2437
2438         list_add(&net->exit_list, &single);
2439         nf_conntrack_cleanup_net_list(&single);
2440 }
2441
2442 void nf_conntrack_cleanup_net_list(struct list_head *net_exit_list)
2443 {
2444         int busy;
2445         struct net *net;
2446
2447         /*
2448          * This makes sure all current packets have passed through
2449          *  netfilter framework.  Roll on, two-stage module
2450          *  delete...
2451          */
2452         synchronize_net();
2453 i_see_dead_people:
2454         busy = 0;
2455         list_for_each_entry(net, net_exit_list, exit_list) {
2456                 struct nf_conntrack_net *cnet = nf_ct_pernet(net);
2457
2458                 nf_ct_iterate_cleanup(kill_all, net, 0, 0);
2459                 if (atomic_read(&cnet->count) != 0)
2460                         busy = 1;
2461         }
2462         if (busy) {
2463                 schedule();
2464                 goto i_see_dead_people;
2465         }
2466
2467         list_for_each_entry(net, net_exit_list, exit_list) {
2468                 nf_conntrack_ecache_pernet_fini(net);
2469                 nf_conntrack_expect_pernet_fini(net);
2470                 free_percpu(net->ct.stat);
2471                 free_percpu(net->ct.pcpu_lists);
2472         }
2473 }
2474
2475 void *nf_ct_alloc_hashtable(unsigned int *sizep, int nulls)
2476 {
2477         struct hlist_nulls_head *hash;
2478         unsigned int nr_slots, i;
2479
2480         if (*sizep > (UINT_MAX / sizeof(struct hlist_nulls_head)))
2481                 return NULL;
2482
2483         BUILD_BUG_ON(sizeof(struct hlist_nulls_head) != sizeof(struct hlist_head));
2484         nr_slots = *sizep = roundup(*sizep, PAGE_SIZE / sizeof(struct hlist_nulls_head));
2485
2486         hash = kvcalloc(nr_slots, sizeof(struct hlist_nulls_head), GFP_KERNEL);
2487
2488         if (hash && nulls)
2489                 for (i = 0; i < nr_slots; i++)
2490                         INIT_HLIST_NULLS_HEAD(&hash[i], i);
2491
2492         return hash;
2493 }
2494 EXPORT_SYMBOL_GPL(nf_ct_alloc_hashtable);
2495
2496 int nf_conntrack_hash_resize(unsigned int hashsize)
2497 {
2498         int i, bucket;
2499         unsigned int old_size;
2500         struct hlist_nulls_head *hash, *old_hash;
2501         struct nf_conntrack_tuple_hash *h;
2502         struct nf_conn *ct;
2503
2504         if (!hashsize)
2505                 return -EINVAL;
2506
2507         hash = nf_ct_alloc_hashtable(&hashsize, 1);
2508         if (!hash)
2509                 return -ENOMEM;
2510
2511         old_size = nf_conntrack_htable_size;
2512         if (old_size == hashsize) {
2513                 kvfree(hash);
2514                 return 0;
2515         }
2516
2517         local_bh_disable();
2518         nf_conntrack_all_lock();
2519         write_seqcount_begin(&nf_conntrack_generation);
2520
2521         /* Lookups in the old hash might happen in parallel, which means we
2522          * might get false negatives during connection lookup. New connections
2523          * created because of a false negative won't make it into the hash
2524          * though since that required taking the locks.
2525          */
2526
2527         for (i = 0; i < nf_conntrack_htable_size; i++) {
2528                 while (!hlist_nulls_empty(&nf_conntrack_hash[i])) {
2529                         h = hlist_nulls_entry(nf_conntrack_hash[i].first,
2530                                               struct nf_conntrack_tuple_hash, hnnode);
2531                         ct = nf_ct_tuplehash_to_ctrack(h);
2532                         hlist_nulls_del_rcu(&h->hnnode);
2533                         bucket = __hash_conntrack(nf_ct_net(ct),
2534                                                   &h->tuple, hashsize);
2535                         hlist_nulls_add_head_rcu(&h->hnnode, &hash[bucket]);
2536                 }
2537         }
2538         old_size = nf_conntrack_htable_size;
2539         old_hash = nf_conntrack_hash;
2540
2541         nf_conntrack_hash = hash;
2542         nf_conntrack_htable_size = hashsize;
2543
2544         write_seqcount_end(&nf_conntrack_generation);
2545         nf_conntrack_all_unlock();
2546         local_bh_enable();
2547
2548         synchronize_net();
2549         kvfree(old_hash);
2550         return 0;
2551 }
2552
2553 int nf_conntrack_set_hashsize(const char *val, const struct kernel_param *kp)
2554 {
2555         unsigned int hashsize;
2556         int rc;
2557
2558         if (current->nsproxy->net_ns != &init_net)
2559                 return -EOPNOTSUPP;
2560
2561         /* On boot, we can set this without any fancy locking. */
2562         if (!nf_conntrack_hash)
2563                 return param_set_uint(val, kp);
2564
2565         rc = kstrtouint(val, 0, &hashsize);
2566         if (rc)
2567                 return rc;
2568
2569         return nf_conntrack_hash_resize(hashsize);
2570 }
2571
2572 static __always_inline unsigned int total_extension_size(void)
2573 {
2574         /* remember to add new extensions below */
2575         BUILD_BUG_ON(NF_CT_EXT_NUM > 9);
2576
2577         return sizeof(struct nf_ct_ext) +
2578                sizeof(struct nf_conn_help)
2579 #if IS_ENABLED(CONFIG_NF_NAT)
2580                 + sizeof(struct nf_conn_nat)
2581 #endif
2582                 + sizeof(struct nf_conn_seqadj)
2583                 + sizeof(struct nf_conn_acct)
2584 #ifdef CONFIG_NF_CONNTRACK_EVENTS
2585                 + sizeof(struct nf_conntrack_ecache)
2586 #endif
2587 #ifdef CONFIG_NF_CONNTRACK_TIMESTAMP
2588                 + sizeof(struct nf_conn_tstamp)
2589 #endif
2590 #ifdef CONFIG_NF_CONNTRACK_TIMEOUT
2591                 + sizeof(struct nf_conn_timeout)
2592 #endif
2593 #ifdef CONFIG_NF_CONNTRACK_LABELS
2594                 + sizeof(struct nf_conn_labels)
2595 #endif
2596 #if IS_ENABLED(CONFIG_NETFILTER_SYNPROXY)
2597                 + sizeof(struct nf_conn_synproxy)
2598 #endif
2599         ;
2600 };
2601
2602 int nf_conntrack_init_start(void)
2603 {
2604         unsigned long nr_pages = totalram_pages();
2605         int max_factor = 8;
2606         int ret = -ENOMEM;
2607         int i;
2608
2609         /* struct nf_ct_ext uses u8 to store offsets/size */
2610         BUILD_BUG_ON(total_extension_size() > 255u);
2611
2612         seqcount_spinlock_init(&nf_conntrack_generation,
2613                                &nf_conntrack_locks_all_lock);
2614
2615         for (i = 0; i < CONNTRACK_LOCKS; i++)
2616                 spin_lock_init(&nf_conntrack_locks[i]);
2617
2618         if (!nf_conntrack_htable_size) {
2619                 /* Idea from tcp.c: use 1/16384 of memory.
2620                  * On i386: 32MB machine has 512 buckets.
2621                  * >= 1GB machines have 16384 buckets.
2622                  * >= 4GB machines have 65536 buckets.
2623                  */
2624                 nf_conntrack_htable_size
2625                         = (((nr_pages << PAGE_SHIFT) / 16384)
2626                            / sizeof(struct hlist_head));
2627                 if (nr_pages > (4 * (1024 * 1024 * 1024 / PAGE_SIZE)))
2628                         nf_conntrack_htable_size = 65536;
2629                 else if (nr_pages > (1024 * 1024 * 1024 / PAGE_SIZE))
2630                         nf_conntrack_htable_size = 16384;
2631                 if (nf_conntrack_htable_size < 32)
2632                         nf_conntrack_htable_size = 32;
2633
2634                 /* Use a max. factor of four by default to get the same max as
2635                  * with the old struct list_heads. When a table size is given
2636                  * we use the old value of 8 to avoid reducing the max.
2637                  * entries. */
2638                 max_factor = 4;
2639         }
2640
2641         nf_conntrack_hash = nf_ct_alloc_hashtable(&nf_conntrack_htable_size, 1);
2642         if (!nf_conntrack_hash)
2643                 return -ENOMEM;
2644
2645         nf_conntrack_max = max_factor * nf_conntrack_htable_size;
2646
2647         nf_conntrack_cachep = kmem_cache_create("nf_conntrack",
2648                                                 sizeof(struct nf_conn),
2649                                                 NFCT_INFOMASK + 1,
2650                                                 SLAB_TYPESAFE_BY_RCU | SLAB_HWCACHE_ALIGN, NULL);
2651         if (!nf_conntrack_cachep)
2652                 goto err_cachep;
2653
2654         ret = nf_conntrack_expect_init();
2655         if (ret < 0)
2656                 goto err_expect;
2657
2658         ret = nf_conntrack_acct_init();
2659         if (ret < 0)
2660                 goto err_acct;
2661
2662         ret = nf_conntrack_tstamp_init();
2663         if (ret < 0)
2664                 goto err_tstamp;
2665
2666         ret = nf_conntrack_ecache_init();
2667         if (ret < 0)
2668                 goto err_ecache;
2669
2670         ret = nf_conntrack_timeout_init();
2671         if (ret < 0)
2672                 goto err_timeout;
2673
2674         ret = nf_conntrack_helper_init();
2675         if (ret < 0)
2676                 goto err_helper;
2677
2678         ret = nf_conntrack_labels_init();
2679         if (ret < 0)
2680                 goto err_labels;
2681
2682         ret = nf_conntrack_seqadj_init();
2683         if (ret < 0)
2684                 goto err_seqadj;
2685
2686         ret = nf_conntrack_proto_init();
2687         if (ret < 0)
2688                 goto err_proto;
2689
2690         conntrack_gc_work_init(&conntrack_gc_work);
2691         queue_delayed_work(system_power_efficient_wq, &conntrack_gc_work.dwork, HZ);
2692
2693         return 0;
2694
2695 err_proto:
2696         nf_conntrack_seqadj_fini();
2697 err_seqadj:
2698         nf_conntrack_labels_fini();
2699 err_labels:
2700         nf_conntrack_helper_fini();
2701 err_helper:
2702         nf_conntrack_timeout_fini();
2703 err_timeout:
2704         nf_conntrack_ecache_fini();
2705 err_ecache:
2706         nf_conntrack_tstamp_fini();
2707 err_tstamp:
2708         nf_conntrack_acct_fini();
2709 err_acct:
2710         nf_conntrack_expect_fini();
2711 err_expect:
2712         kmem_cache_destroy(nf_conntrack_cachep);
2713 err_cachep:
2714         kvfree(nf_conntrack_hash);
2715         return ret;
2716 }
2717
2718 static struct nf_ct_hook nf_conntrack_hook = {
2719         .update         = nf_conntrack_update,
2720         .destroy        = destroy_conntrack,
2721         .get_tuple_skb  = nf_conntrack_get_tuple_skb,
2722 };
2723
2724 void nf_conntrack_init_end(void)
2725 {
2726         /* For use by REJECT target */
2727         RCU_INIT_POINTER(ip_ct_attach, nf_conntrack_attach);
2728         RCU_INIT_POINTER(nf_ct_hook, &nf_conntrack_hook);
2729 }
2730
2731 /*
2732  * We need to use special "null" values, not used in hash table
2733  */
2734 #define UNCONFIRMED_NULLS_VAL   ((1<<30)+0)
2735 #define DYING_NULLS_VAL         ((1<<30)+1)
2736
2737 int nf_conntrack_init_net(struct net *net)
2738 {
2739         struct nf_conntrack_net *cnet = nf_ct_pernet(net);
2740         int ret = -ENOMEM;
2741         int cpu;
2742
2743         BUILD_BUG_ON(IP_CT_UNTRACKED == IP_CT_NUMBER);
2744         BUILD_BUG_ON_NOT_POWER_OF_2(CONNTRACK_LOCKS);
2745         atomic_set(&cnet->count, 0);
2746
2747         net->ct.pcpu_lists = alloc_percpu(struct ct_pcpu);
2748         if (!net->ct.pcpu_lists)
2749                 goto err_stat;
2750
2751         for_each_possible_cpu(cpu) {
2752                 struct ct_pcpu *pcpu = per_cpu_ptr(net->ct.pcpu_lists, cpu);
2753
2754                 spin_lock_init(&pcpu->lock);
2755                 INIT_HLIST_NULLS_HEAD(&pcpu->unconfirmed, UNCONFIRMED_NULLS_VAL);
2756                 INIT_HLIST_NULLS_HEAD(&pcpu->dying, DYING_NULLS_VAL);
2757         }
2758
2759         net->ct.stat = alloc_percpu(struct ip_conntrack_stat);
2760         if (!net->ct.stat)
2761                 goto err_pcpu_lists;
2762
2763         ret = nf_conntrack_expect_pernet_init(net);
2764         if (ret < 0)
2765                 goto err_expect;
2766
2767         nf_conntrack_acct_pernet_init(net);
2768         nf_conntrack_tstamp_pernet_init(net);
2769         nf_conntrack_ecache_pernet_init(net);
2770         nf_conntrack_helper_pernet_init(net);
2771         nf_conntrack_proto_pernet_init(net);
2772
2773         return 0;
2774
2775 err_expect:
2776         free_percpu(net->ct.stat);
2777 err_pcpu_lists:
2778         free_percpu(net->ct.pcpu_lists);
2779 err_stat:
2780         return ret;
2781 }