net/netfilter/nf_conntrack_core.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Connection state tracking for netfilter.  This is separated from,
   3    but required by, the NAT layer; it can also be used by an iptables
   4    extension. */
   5
   6 /* (C) 1999-2001 Paul `Rusty' Russell
   7  * (C) 2002-2006 Netfilter Core Team <coreteam@netfilter.org>
   8  * (C) 2003,2004 USAGI/WIDE Project <http://www.linux-ipv6.org>
   9  * (C) 2005-2012 Patrick McHardy <kaber@trash.net>
  10  */
  11
  12 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  13
  14 #include <linux/types.h>
  15 #include <linux/netfilter.h>
  16 #include <linux/module.h>
  17 #include <linux/sched.h>
  18 #include <linux/skbuff.h>
  19 #include <linux/proc_fs.h>
  20 #include <linux/vmalloc.h>
  21 #include <linux/stddef.h>
  22 #include <linux/slab.h>
  23 #include <linux/random.h>
  24 #include <linux/siphash.h>
  25 #include <linux/err.h>
  26 #include <linux/percpu.h>
  27 #include <linux/moduleparam.h>
  28 #include <linux/notifier.h>
  29 #include <linux/kernel.h>
  30 #include <linux/netdevice.h>
  31 #include <linux/socket.h>
  32 #include <linux/mm.h>
  33 #include <linux/nsproxy.h>
  34 #include <linux/rculist_nulls.h>
  35
  36 #include <net/netfilter/nf_conntrack.h>
  37 #include <net/netfilter/nf_conntrack_bpf.h>
  38 #include <net/netfilter/nf_conntrack_l4proto.h>
  39 #include <net/netfilter/nf_conntrack_expect.h>
  40 #include <net/netfilter/nf_conntrack_helper.h>
  41 #include <net/netfilter/nf_conntrack_core.h>
  42 #include <net/netfilter/nf_conntrack_extend.h>
  43 #include <net/netfilter/nf_conntrack_acct.h>
  44 #include <net/netfilter/nf_conntrack_ecache.h>
  45 #include <net/netfilter/nf_conntrack_zones.h>
  46 #include <net/netfilter/nf_conntrack_timestamp.h>
  47 #include <net/netfilter/nf_conntrack_timeout.h>
  48 #include <net/netfilter/nf_conntrack_labels.h>
  49 #include <net/netfilter/nf_conntrack_synproxy.h>
  50 #include <net/netfilter/nf_nat.h>
  51 #include <net/netfilter/nf_nat_helper.h>
  52 #include <net/netns/hash.h>
  53 #include <net/ip.h>
  54
  55 #include "nf_internals.h"
  56
  57 __cacheline_aligned_in_smp spinlock_t nf_conntrack_locks[CONNTRACK_LOCKS];
  58 EXPORT_SYMBOL_GPL(nf_conntrack_locks);
  59
  60 __cacheline_aligned_in_smp DEFINE_SPINLOCK(nf_conntrack_expect_lock);
  61 EXPORT_SYMBOL_GPL(nf_conntrack_expect_lock);
  62
  63 struct hlist_nulls_head *nf_conntrack_hash __read_mostly;
  64 EXPORT_SYMBOL_GPL(nf_conntrack_hash);
  65
  66 struct conntrack_gc_work {
  67         struct delayed_work     dwork;
  68         u32                     next_bucket;
  69         u32                     avg_timeout;
  70         u32                     start_time;
  71         bool                    exiting;
  72         bool                    early_drop;
  73 };
  74
  75 static __read_mostly struct kmem_cache *nf_conntrack_cachep;
  76 static DEFINE_SPINLOCK(nf_conntrack_locks_all_lock);
  77 static __read_mostly bool nf_conntrack_locks_all;
  78
  79 /* serialize hash resizes and nf_ct_iterate_cleanup */
  80 static DEFINE_MUTEX(nf_conntrack_mutex);
  81
  82 #define GC_SCAN_INTERVAL_MAX    (60ul * HZ)
  83 #define GC_SCAN_INTERVAL_MIN    (1ul * HZ)
  84
  85 /* clamp timeouts to this value (TCP unacked) */
  86 #define GC_SCAN_INTERVAL_CLAMP  (300ul * HZ)
  87
  88 /* large initial bias so that we don't scan often just because we have
  89  * three entries with a 1s timeout.
  90  */
  91 #define GC_SCAN_INTERVAL_INIT   INT_MAX
  92
  93 #define GC_SCAN_MAX_DURATION    msecs_to_jiffies(10)
  94 #define GC_SCAN_EXPIRED_MAX     (64000u / HZ)
  95
  96 #define MIN_CHAINLEN    8u
  97 #define MAX_CHAINLEN    (32u - MIN_CHAINLEN)
  98
  99 static struct conntrack_gc_work conntrack_gc_work;
 100
 101 void nf_conntrack_lock(spinlock_t *lock) __acquires(lock)
 102 {
 103         /* 1) Acquire the lock */
 104         spin_lock(lock);
 105
 106         /* 2) read nf_conntrack_locks_all, with ACQUIRE semantics
 107          * It pairs with the smp_store_release() in nf_conntrack_all_unlock()
 108          */
 109         if (likely(smp_load_acquire(&nf_conntrack_locks_all) == false))
 110                 return;
 111
 112         /* fast path failed, unlock */
 113         spin_unlock(lock);
 114
 115         /* Slow path 1) get global lock */
 116         spin_lock(&nf_conntrack_locks_all_lock);
 117
 118         /* Slow path 2) get the lock we want */
 119         spin_lock(lock);
 120
 121         /* Slow path 3) release the global lock */
 122         spin_unlock(&nf_conntrack_locks_all_lock);
 123 }
 124 EXPORT_SYMBOL_GPL(nf_conntrack_lock);
 125
 126 static void nf_conntrack_double_unlock(unsigned int h1, unsigned int h2)
 127 {
 128         h1 %= CONNTRACK_LOCKS;
 129         h2 %= CONNTRACK_LOCKS;
 130         spin_unlock(&nf_conntrack_locks[h1]);
 131         if (h1 != h2)
 132                 spin_unlock(&nf_conntrack_locks[h2]);
 133 }
 134
 135 /* return true if we need to recompute hashes (in case hash table was resized) */
 136 static bool nf_conntrack_double_lock(struct net *net, unsigned int h1,
 137                                      unsigned int h2, unsigned int sequence)
 138 {
 139         h1 %= CONNTRACK_LOCKS;
 140         h2 %= CONNTRACK_LOCKS;
 141         if (h1 <= h2) {
 142                 nf_conntrack_lock(&nf_conntrack_locks[h1]);
 143                 if (h1 != h2)
 144                         spin_lock_nested(&nf_conntrack_locks[h2],
 145                                          SINGLE_DEPTH_NESTING);
 146         } else {
 147                 nf_conntrack_lock(&nf_conntrack_locks[h2]);
 148                 spin_lock_nested(&nf_conntrack_locks[h1],
 149                                  SINGLE_DEPTH_NESTING);
 150         }
 151         if (read_seqcount_retry(&nf_conntrack_generation, sequence)) {
 152                 nf_conntrack_double_unlock(h1, h2);
 153                 return true;
 154         }
 155         return false;
 156 }
 157
 158 static void nf_conntrack_all_lock(void)
 159         __acquires(&nf_conntrack_locks_all_lock)
 160 {
 161         int i;
 162
 163         spin_lock(&nf_conntrack_locks_all_lock);
 164
 165         /* For nf_contrack_locks_all, only the latest time when another
 166          * CPU will see an update is controlled, by the "release" of the
 167          * spin_lock below.
 168          * The earliest time is not controlled, an thus KCSAN could detect
 169          * a race when nf_conntract_lock() reads the variable.
 170          * WRITE_ONCE() is used to ensure the compiler will not
 171          * optimize the write.
 172          */
 173         WRITE_ONCE(nf_conntrack_locks_all, true);
 174
 175         for (i = 0; i < CONNTRACK_LOCKS; i++) {
 176                 spin_lock(&nf_conntrack_locks[i]);
 177
 178                 /* This spin_unlock provides the "release" to ensure that
 179                  * nf_conntrack_locks_all==true is visible to everyone that
 180                  * acquired spin_lock(&nf_conntrack_locks[]).
 181                  */
 182                 spin_unlock(&nf_conntrack_locks[i]);
 183         }
 184 }
 185
 186 static void nf_conntrack_all_unlock(void)
 187         __releases(&nf_conntrack_locks_all_lock)
 188 {
 189         /* All prior stores must be complete before we clear
 190          * 'nf_conntrack_locks_all'. Otherwise nf_conntrack_lock()
 191          * might observe the false value but not the entire
 192          * critical section.
 193          * It pairs with the smp_load_acquire() in nf_conntrack_lock()
 194          */
 195         smp_store_release(&nf_conntrack_locks_all, false);
 196         spin_unlock(&nf_conntrack_locks_all_lock);
 197 }
 198
 199 unsigned int nf_conntrack_htable_size __read_mostly;
 200 EXPORT_SYMBOL_GPL(nf_conntrack_htable_size);
 201
 202 unsigned int nf_conntrack_max __read_mostly;
 203 EXPORT_SYMBOL_GPL(nf_conntrack_max);
 204 seqcount_spinlock_t nf_conntrack_generation __read_mostly;
 205 static siphash_aligned_key_t nf_conntrack_hash_rnd;
 206
 207 static u32 hash_conntrack_raw(const struct nf_conntrack_tuple *tuple,
 208                               unsigned int zoneid,
 209                               const struct net *net)
 210 {
 211         struct {
 212                 struct nf_conntrack_man src;
 213                 union nf_inet_addr dst_addr;
 214                 unsigned int zone;
 215                 u32 net_mix;
 216                 u16 dport;
 217                 u16 proto;
 218         } __aligned(SIPHASH_ALIGNMENT) combined;
 219
 220         get_random_once(&nf_conntrack_hash_rnd, sizeof(nf_conntrack_hash_rnd));
 221
 222         memset(&combined, 0, sizeof(combined));
 223
 224         /* The direction must be ignored, so handle usable members manually. */
 225         combined.src = tuple->src;
 226         combined.dst_addr = tuple->dst.u3;
 227         combined.zone = zoneid;
 228         combined.net_mix = net_hash_mix(net);
 229         combined.dport = (__force __u16)tuple->dst.u.all;
 230         combined.proto = tuple->dst.protonum;
 231
 232         return (u32)siphash(&combined, sizeof(combined), &nf_conntrack_hash_rnd);
 233 }
 234
 235 static u32 scale_hash(u32 hash)
 236 {
 237         return reciprocal_scale(hash, nf_conntrack_htable_size);
 238 }
 239
 240 static u32 __hash_conntrack(const struct net *net,
 241                             const struct nf_conntrack_tuple *tuple,
 242                             unsigned int zoneid,
 243                             unsigned int size)
 244 {
 245         return reciprocal_scale(hash_conntrack_raw(tuple, zoneid, net), size);
 246 }
 247
 248 static u32 hash_conntrack(const struct net *net,
 249                           const struct nf_conntrack_tuple *tuple,
 250                           unsigned int zoneid)
 251 {
 252         return scale_hash(hash_conntrack_raw(tuple, zoneid, net));
 253 }
 254
 255 static bool nf_ct_get_tuple_ports(const struct sk_buff *skb,
 256                                   unsigned int dataoff,
 257                                   struct nf_conntrack_tuple *tuple)
 258 {       struct {
 259                 __be16 sport;
 260                 __be16 dport;
 261         } _inet_hdr, *inet_hdr;
 262
 263         /* Actually only need first 4 bytes to get ports. */
 264         inet_hdr = skb_header_pointer(skb, dataoff, sizeof(_inet_hdr), &_inet_hdr);
 265         if (!inet_hdr)
 266                 return false;
 267
 268         tuple->src.u.udp.port = inet_hdr->sport;
 269         tuple->dst.u.udp.port = inet_hdr->dport;
 270         return true;
 271 }
 272
 273 static bool
 274 nf_ct_get_tuple(const struct sk_buff *skb,
 275                 unsigned int nhoff,
 276                 unsigned int dataoff,
 277                 u_int16_t l3num,
 278                 u_int8_t protonum,
 279                 struct net *net,
 280                 struct nf_conntrack_tuple *tuple)
 281 {
 282         unsigned int size;
 283         const __be32 *ap;
 284         __be32 _addrs[8];
 285
 286         memset(tuple, 0, sizeof(*tuple));
 287
 288         tuple->src.l3num = l3num;
 289         switch (l3num) {
 290         case NFPROTO_IPV4:
 291                 nhoff += offsetof(struct iphdr, saddr);
 292                 size = 2 * sizeof(__be32);
 293                 break;
 294         case NFPROTO_IPV6:
 295                 nhoff += offsetof(struct ipv6hdr, saddr);
 296                 size = sizeof(_addrs);
 297                 break;
 298         default:
 299                 return true;
 300         }
 301
 302         ap = skb_header_pointer(skb, nhoff, size, _addrs);
 303         if (!ap)
 304                 return false;
 305
 306         switch (l3num) {
 307         case NFPROTO_IPV4:
 308                 tuple->src.u3.ip = ap[0];
 309                 tuple->dst.u3.ip = ap[1];
 310                 break;
 311         case NFPROTO_IPV6:
 312                 memcpy(tuple->src.u3.ip6, ap, sizeof(tuple->src.u3.ip6));
 313                 memcpy(tuple->dst.u3.ip6, ap + 4, sizeof(tuple->dst.u3.ip6));
 314                 break;
 315         }
 316
 317         tuple->dst.protonum = protonum;
 318         tuple->dst.dir = IP_CT_DIR_ORIGINAL;
 319
 320         switch (protonum) {
 321 #if IS_ENABLED(CONFIG_IPV6)
 322         case IPPROTO_ICMPV6:
 323                 return icmpv6_pkt_to_tuple(skb, dataoff, net, tuple);
 324 #endif
 325         case IPPROTO_ICMP:
 326                 return icmp_pkt_to_tuple(skb, dataoff, net, tuple);
 327 #ifdef CONFIG_NF_CT_PROTO_GRE
 328         case IPPROTO_GRE:
 329                 return gre_pkt_to_tuple(skb, dataoff, net, tuple);
 330 #endif
 331         case IPPROTO_TCP:
 332         case IPPROTO_UDP: /* fallthrough */
 333                 return nf_ct_get_tuple_ports(skb, dataoff, tuple);
 334 #ifdef CONFIG_NF_CT_PROTO_UDPLITE
 335         case IPPROTO_UDPLITE:
 336                 return nf_ct_get_tuple_ports(skb, dataoff, tuple);
 337 #endif
 338 #ifdef CONFIG_NF_CT_PROTO_SCTP
 339         case IPPROTO_SCTP:
 340                 return nf_ct_get_tuple_ports(skb, dataoff, tuple);
 341 #endif
 342 #ifdef CONFIG_NF_CT_PROTO_DCCP
 343         case IPPROTO_DCCP:
 344                 return nf_ct_get_tuple_ports(skb, dataoff, tuple);
 345 #endif
 346         default:
 347                 break;
 348         }
 349
 350         return true;
 351 }
 352
 353 static int ipv4_get_l4proto(const struct sk_buff *skb, unsigned int nhoff,
 354                             u_int8_t *protonum)
 355 {
 356         int dataoff = -1;
 357         const struct iphdr *iph;
 358         struct iphdr _iph;
 359
 360         iph = skb_header_pointer(skb, nhoff, sizeof(_iph), &_iph);
 361         if (!iph)
 362                 return -1;
 363
 364         /* Conntrack defragments packets, we might still see fragments
 365          * inside ICMP packets though.
 366          */
 367         if (iph->frag_off & htons(IP_OFFSET))
 368                 return -1;
 369
 370         dataoff = nhoff + (iph->ihl << 2);
 371         *protonum = iph->protocol;
 372
 373         /* Check bogus IP headers */
 374         if (dataoff > skb->len) {
 375                 pr_debug("bogus IPv4 packet: nhoff %u, ihl %u, skblen %u\n",
 376                          nhoff, iph->ihl << 2, skb->len);
 377                 return -1;
 378         }
 379         return dataoff;
 380 }
 381
 382 #if IS_ENABLED(CONFIG_IPV6)
 383 static int ipv6_get_l4proto(const struct sk_buff *skb, unsigned int nhoff,
 384                             u8 *protonum)
 385 {
 386         int protoff = -1;
 387         unsigned int extoff = nhoff + sizeof(struct ipv6hdr);
 388         __be16 frag_off;
 389         u8 nexthdr;
 390
 391         if (skb_copy_bits(skb, nhoff + offsetof(struct ipv6hdr, nexthdr),
 392                           &nexthdr, sizeof(nexthdr)) != 0) {
 393                 pr_debug("can't get nexthdr\n");
 394                 return -1;
 395         }
 396         protoff = ipv6_skip_exthdr(skb, extoff, &nexthdr, &frag_off);
 397         /*
 398          * (protoff == skb->len) means the packet has not data, just
 399          * IPv6 and possibly extensions headers, but it is tracked anyway
 400          */
 401         if (protoff < 0 || (frag_off & htons(~0x7)) != 0) {
 402                 pr_debug("can't find proto in pkt\n");
 403                 return -1;
 404         }
 405
 406         *protonum = nexthdr;
 407         return protoff;
 408 }
 409 #endif
 410
 411 static int get_l4proto(const struct sk_buff *skb,
 412                        unsigned int nhoff, u8 pf, u8 *l4num)
 413 {
 414         switch (pf) {
 415         case NFPROTO_IPV4:
 416                 return ipv4_get_l4proto(skb, nhoff, l4num);
 417 #if IS_ENABLED(CONFIG_IPV6)
 418         case NFPROTO_IPV6:
 419                 return ipv6_get_l4proto(skb, nhoff, l4num);
 420 #endif
 421         default:
 422                 *l4num = 0;
 423                 break;
 424         }
 425         return -1;
 426 }
 427
 428 bool nf_ct_get_tuplepr(const struct sk_buff *skb, unsigned int nhoff,
 429                        u_int16_t l3num,
 430                        struct net *net, struct nf_conntrack_tuple *tuple)
 431 {
 432         u8 protonum;
 433         int protoff;
 434
 435         protoff = get_l4proto(skb, nhoff, l3num, &protonum);
 436         if (protoff <= 0)
 437                 return false;
 438
 439         return nf_ct_get_tuple(skb, nhoff, protoff, l3num, protonum, net, tuple);
 440 }
 441 EXPORT_SYMBOL_GPL(nf_ct_get_tuplepr);
 442
 443 bool
 444 nf_ct_invert_tuple(struct nf_conntrack_tuple *inverse,
 445                    const struct nf_conntrack_tuple *orig)
 446 {
 447         memset(inverse, 0, sizeof(*inverse));
 448
 449         inverse->src.l3num = orig->src.l3num;
 450
 451         switch (orig->src.l3num) {
 452         case NFPROTO_IPV4:
 453                 inverse->src.u3.ip = orig->dst.u3.ip;
 454                 inverse->dst.u3.ip = orig->src.u3.ip;
 455                 break;
 456         case NFPROTO_IPV6:
 457                 inverse->src.u3.in6 = orig->dst.u3.in6;
 458                 inverse->dst.u3.in6 = orig->src.u3.in6;
 459                 break;
 460         default:
 461                 break;
 462         }
 463
 464         inverse->dst.dir = !orig->dst.dir;
 465
 466         inverse->dst.protonum = orig->dst.protonum;
 467
 468         switch (orig->dst.protonum) {
 469         case IPPROTO_ICMP:
 470                 return nf_conntrack_invert_icmp_tuple(inverse, orig);
 471 #if IS_ENABLED(CONFIG_IPV6)
 472         case IPPROTO_ICMPV6:
 473                 return nf_conntrack_invert_icmpv6_tuple(inverse, orig);
 474 #endif
 475         }
 476
 477         inverse->src.u.all = orig->dst.u.all;
 478         inverse->dst.u.all = orig->src.u.all;
 479         return true;
 480 }
 481 EXPORT_SYMBOL_GPL(nf_ct_invert_tuple);
 482
 483 /* Generate a almost-unique pseudo-id for a given conntrack.
 484  *
 485  * intentionally doesn't re-use any of the seeds used for hash
 486  * table location, we assume id gets exposed to userspace.
 487  *
 488  * Following nf_conn items do not change throughout lifetime
 489  * of the nf_conn:
 490  *
 491  * 1. nf_conn address
 492  * 2. nf_conn->master address (normally NULL)
 493  * 3. the associated net namespace
 494  * 4. the original direction tuple
 495  */
 496 u32 nf_ct_get_id(const struct nf_conn *ct)
 497 {
 498         static siphash_aligned_key_t ct_id_seed;
 499         unsigned long a, b, c, d;
 500
 501         net_get_random_once(&ct_id_seed, sizeof(ct_id_seed));
 502
 503         a = (unsigned long)ct;
 504         b = (unsigned long)ct->master;
 505         c = (unsigned long)nf_ct_net(ct);
 506         d = (unsigned long)siphash(&ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple,
 507                                    sizeof(ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple),
 508                                    &ct_id_seed);
 509 #ifdef CONFIG_64BIT
 510         return siphash_4u64((u64)a, (u64)b, (u64)c, (u64)d, &ct_id_seed);
 511 #else
 512         return siphash_4u32((u32)a, (u32)b, (u32)c, (u32)d, &ct_id_seed);
 513 #endif
 514 }
 515 EXPORT_SYMBOL_GPL(nf_ct_get_id);
 516
 517 static void
 518 clean_from_lists(struct nf_conn *ct)
 519 {
 520         pr_debug("clean_from_lists(%p)\n", ct);
 521         hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);
 522         hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode);
 523
 524         /* Destroy all pending expectations */
 525         nf_ct_remove_expectations(ct);
 526 }
 527
 528 #define NFCT_ALIGN(len) (((len) + NFCT_INFOMASK) & ~NFCT_INFOMASK)
 529
 530 /* Released via nf_ct_destroy() */
 531 struct nf_conn *nf_ct_tmpl_alloc(struct net *net,
 532                                  const struct nf_conntrack_zone *zone,
 533                                  gfp_t flags)
 534 {
 535         struct nf_conn *tmpl, *p;
 536
 537         if (ARCH_KMALLOC_MINALIGN <= NFCT_INFOMASK) {
 538                 tmpl = kzalloc(sizeof(*tmpl) + NFCT_INFOMASK, flags);
 539                 if (!tmpl)
 540                         return NULL;
 541
 542                 p = tmpl;
 543                 tmpl = (struct nf_conn *)NFCT_ALIGN((unsigned long)p);
 544                 if (tmpl != p) {
 545                         tmpl = (struct nf_conn *)NFCT_ALIGN((unsigned long)p);
 546                         tmpl->proto.tmpl_padto = (char *)tmpl - (char *)p;
 547                 }
 548         } else {
 549                 tmpl = kzalloc(sizeof(*tmpl), flags);
 550                 if (!tmpl)
 551                         return NULL;
 552         }
 553
 554         tmpl->status = IPS_TEMPLATE;
 555         write_pnet(&tmpl->ct_net, net);
 556         nf_ct_zone_add(tmpl, zone);
 557         refcount_set(&tmpl->ct_general.use, 1);
 558
 559         return tmpl;
 560 }
 561 EXPORT_SYMBOL_GPL(nf_ct_tmpl_alloc);
 562
 563 void nf_ct_tmpl_free(struct nf_conn *tmpl)
 564 {
 565         kfree(tmpl->ext);
 566
 567         if (ARCH_KMALLOC_MINALIGN <= NFCT_INFOMASK)
 568                 kfree((char *)tmpl - tmpl->proto.tmpl_padto);
 569         else
 570                 kfree(tmpl);
 571 }
 572 EXPORT_SYMBOL_GPL(nf_ct_tmpl_free);
 573
 574 static void destroy_gre_conntrack(struct nf_conn *ct)
 575 {
 576 #ifdef CONFIG_NF_CT_PROTO_GRE
 577         struct nf_conn *master = ct->master;
 578
 579         if (master)
 580                 nf_ct_gre_keymap_destroy(master);
 581 #endif
 582 }
 583
 584 void nf_ct_destroy(struct nf_conntrack *nfct)
 585 {
 586         struct nf_conn *ct = (struct nf_conn *)nfct;
 587
 588         pr_debug("%s(%p)\n", __func__, ct);
 589         WARN_ON(refcount_read(&nfct->use) != 0);
 590
 591         if (unlikely(nf_ct_is_template(ct))) {
 592                 nf_ct_tmpl_free(ct);
 593                 return;
 594         }
 595
 596         if (unlikely(nf_ct_protonum(ct) == IPPROTO_GRE))
 597                 destroy_gre_conntrack(ct);
 598
 599         /* Expectations will have been removed in clean_from_lists,
 600          * except TFTP can create an expectation on the first packet,
 601          * before connection is in the list, so we need to clean here,
 602          * too.
 603          */
 604         nf_ct_remove_expectations(ct);
 605
 606         if (ct->master)
 607                 nf_ct_put(ct->master);
 608
 609         pr_debug("%s: returning ct=%p to slab\n", __func__, ct);
 610         nf_conntrack_free(ct);
 611 }
 612 EXPORT_SYMBOL(nf_ct_destroy);
 613
 614 static void __nf_ct_delete_from_lists(struct nf_conn *ct)
 615 {
 616         struct net *net = nf_ct_net(ct);
 617         unsigned int hash, reply_hash;
 618         unsigned int sequence;
 619
 620         do {
 621                 sequence = read_seqcount_begin(&nf_conntrack_generation);
 622                 hash = hash_conntrack(net,
 623                                       &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple,
 624                                       nf_ct_zone_id(nf_ct_zone(ct), IP_CT_DIR_ORIGINAL));
 625                 reply_hash = hash_conntrack(net,
 626                                            &ct->tuplehash[IP_CT_DIR_REPLY].tuple,
 627                                            nf_ct_zone_id(nf_ct_zone(ct), IP_CT_DIR_REPLY));
 628         } while (nf_conntrack_double_lock(net, hash, reply_hash, sequence));
 629
 630         clean_from_lists(ct);
 631         nf_conntrack_double_unlock(hash, reply_hash);
 632 }
 633
 634 static void nf_ct_delete_from_lists(struct nf_conn *ct)
 635 {
 636         nf_ct_helper_destroy(ct);
 637         local_bh_disable();
 638
 639         __nf_ct_delete_from_lists(ct);
 640
 641         local_bh_enable();
 642 }
 643
 644 static void nf_ct_add_to_ecache_list(struct nf_conn *ct)
 645 {
 646 #ifdef CONFIG_NF_CONNTRACK_EVENTS
 647         struct nf_conntrack_net *cnet = nf_ct_pernet(nf_ct_net(ct));
 648
 649         spin_lock(&cnet->ecache.dying_lock);
 650         hlist_nulls_add_head_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,
 651                                  &cnet->ecache.dying_list);
 652         spin_unlock(&cnet->ecache.dying_lock);
 653 #endif
 654 }
 655
 656 bool nf_ct_delete(struct nf_conn *ct, u32 portid, int report)
 657 {
 658         struct nf_conn_tstamp *tstamp;
 659         struct net *net;
 660
 661         if (test_and_set_bit(IPS_DYING_BIT, &ct->status))
 662                 return false;
 663
 664         tstamp = nf_conn_tstamp_find(ct);
 665         if (tstamp) {
 666                 s32 timeout = READ_ONCE(ct->timeout) - nfct_time_stamp;
 667
 668                 tstamp->stop = ktime_get_real_ns();
 669                 if (timeout < 0)
 670                         tstamp->stop -= jiffies_to_nsecs(-timeout);
 671         }
 672
 673         if (nf_conntrack_event_report(IPCT_DESTROY, ct,
 674                                     portid, report) < 0) {
 675                 /* destroy event was not delivered. nf_ct_put will
 676                  * be done by event cache worker on redelivery.
 677                  */
 678                 nf_ct_helper_destroy(ct);
 679                 local_bh_disable();
 680                 __nf_ct_delete_from_lists(ct);
 681                 nf_ct_add_to_ecache_list(ct);
 682                 local_bh_enable();
 683
 684                 nf_conntrack_ecache_work(nf_ct_net(ct), NFCT_ECACHE_DESTROY_FAIL);
 685                 return false;
 686         }
 687
 688         net = nf_ct_net(ct);
 689         if (nf_conntrack_ecache_dwork_pending(net))
 690                 nf_conntrack_ecache_work(net, NFCT_ECACHE_DESTROY_SENT);
 691         nf_ct_delete_from_lists(ct);
 692         nf_ct_put(ct);
 693         return true;
 694 }
 695 EXPORT_SYMBOL_GPL(nf_ct_delete);
 696
 697 static inline bool
 698 nf_ct_key_equal(struct nf_conntrack_tuple_hash *h,
 699                 const struct nf_conntrack_tuple *tuple,
 700                 const struct nf_conntrack_zone *zone,
 701                 const struct net *net)
 702 {
 703         struct nf_conn *ct = nf_ct_tuplehash_to_ctrack(h);
 704
 705         /* A conntrack can be recreated with the equal tuple,
 706          * so we need to check that the conntrack is confirmed
 707          */
 708         return nf_ct_tuple_equal(tuple, &h->tuple) &&
 709                nf_ct_zone_equal(ct, zone, NF_CT_DIRECTION(h)) &&
 710                nf_ct_is_confirmed(ct) &&
 711                net_eq(net, nf_ct_net(ct));
 712 }
 713
 714 static inline bool
 715 nf_ct_match(const struct nf_conn *ct1, const struct nf_conn *ct2)
 716 {
 717         return nf_ct_tuple_equal(&ct1->tuplehash[IP_CT_DIR_ORIGINAL].tuple,
 718                                  &ct2->tuplehash[IP_CT_DIR_ORIGINAL].tuple) &&
 719                nf_ct_tuple_equal(&ct1->tuplehash[IP_CT_DIR_REPLY].tuple,
 720                                  &ct2->tuplehash[IP_CT_DIR_REPLY].tuple) &&
 721                nf_ct_zone_equal(ct1, nf_ct_zone(ct2), IP_CT_DIR_ORIGINAL) &&
 722                nf_ct_zone_equal(ct1, nf_ct_zone(ct2), IP_CT_DIR_REPLY) &&
 723                net_eq(nf_ct_net(ct1), nf_ct_net(ct2));
 724 }
 725
 726 /* caller must hold rcu readlock and none of the nf_conntrack_locks */
 727 static void nf_ct_gc_expired(struct nf_conn *ct)
 728 {
 729         if (!refcount_inc_not_zero(&ct->ct_general.use))
 730                 return;
 731
 732         if (nf_ct_should_gc(ct))
 733                 nf_ct_kill(ct);
 734
 735         nf_ct_put(ct);
 736 }
 737
 738 /*
 739  * Warning :
 740  * - Caller must take a reference on returned object
 741  *   and recheck nf_ct_tuple_equal(tuple, &h->tuple)
 742  */
 743 static struct nf_conntrack_tuple_hash *
 744 ____nf_conntrack_find(struct net *net, const struct nf_conntrack_zone *zone,
 745                       const struct nf_conntrack_tuple *tuple, u32 hash)
 746 {
 747         struct nf_conntrack_tuple_hash *h;
 748         struct hlist_nulls_head *ct_hash;
 749         struct hlist_nulls_node *n;
 750         unsigned int bucket, hsize;
 751
 752 begin:
 753         nf_conntrack_get_ht(&ct_hash, &hsize);
 754         bucket = reciprocal_scale(hash, hsize);
 755
 756         hlist_nulls_for_each_entry_rcu(h, n, &ct_hash[bucket], hnnode) {
 757                 struct nf_conn *ct;
 758
 759                 ct = nf_ct_tuplehash_to_ctrack(h);
 760                 if (nf_ct_is_expired(ct)) {
 761                         nf_ct_gc_expired(ct);
 762                         continue;
 763                 }
 764
 765                 if (nf_ct_key_equal(h, tuple, zone, net))
 766                         return h;
 767         }
 768         /*
 769          * if the nulls value we got at the end of this lookup is
 770          * not the expected one, we must restart lookup.
 771          * We probably met an item that was moved to another chain.
 772          */
 773         if (get_nulls_value(n) != bucket) {
 774                 NF_CT_STAT_INC_ATOMIC(net, search_restart);
 775                 goto begin;
 776         }
 777
 778         return NULL;
 779 }
 780
 781 /* Find a connection corresponding to a tuple. */
 782 static struct nf_conntrack_tuple_hash *
 783 __nf_conntrack_find_get(struct net *net, const struct nf_conntrack_zone *zone,
 784                         const struct nf_conntrack_tuple *tuple, u32 hash)
 785 {
 786         struct nf_conntrack_tuple_hash *h;
 787         struct nf_conn *ct;
 788
 789         rcu_read_lock();
 790
 791         h = ____nf_conntrack_find(net, zone, tuple, hash);
 792         if (h) {
 793                 /* We have a candidate that matches the tuple we're interested
 794                  * in, try to obtain a reference and re-check tuple
 795                  */
 796                 ct = nf_ct_tuplehash_to_ctrack(h);
 797                 if (likely(refcount_inc_not_zero(&ct->ct_general.use))) {
 798                         if (likely(nf_ct_key_equal(h, tuple, zone, net)))
 799                                 goto found;
 800
 801                         /* TYPESAFE_BY_RCU recycled the candidate */
 802                         nf_ct_put(ct);
 803                 }
 804
 805                 h = NULL;
 806         }
 807 found:
 808         rcu_read_unlock();
 809
 810         return h;
 811 }
 812
 813 struct nf_conntrack_tuple_hash *
 814 nf_conntrack_find_get(struct net *net, const struct nf_conntrack_zone *zone,
 815                       const struct nf_conntrack_tuple *tuple)
 816 {
 817         unsigned int rid, zone_id = nf_ct_zone_id(zone, IP_CT_DIR_ORIGINAL);
 818         struct nf_conntrack_tuple_hash *thash;
 819
 820         thash = __nf_conntrack_find_get(net, zone, tuple,
 821                                         hash_conntrack_raw(tuple, zone_id, net));
 822
 823         if (thash)
 824                 return thash;
 825
 826         rid = nf_ct_zone_id(zone, IP_CT_DIR_REPLY);
 827         if (rid != zone_id)
 828                 return __nf_conntrack_find_get(net, zone, tuple,
 829                                                hash_conntrack_raw(tuple, rid, net));
 830         return thash;
 831 }
 832 EXPORT_SYMBOL_GPL(nf_conntrack_find_get);
 833
 834 static void __nf_conntrack_hash_insert(struct nf_conn *ct,
 835                                        unsigned int hash,
 836                                        unsigned int reply_hash)
 837 {
 838         hlist_nulls_add_head_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,
 839                            &nf_conntrack_hash[hash]);
 840         hlist_nulls_add_head_rcu(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode,
 841                            &nf_conntrack_hash[reply_hash]);
 842 }
 843
 844 static bool nf_ct_ext_valid_pre(const struct nf_ct_ext *ext)
 845 {
 846         /* if ext->gen_id is not equal to nf_conntrack_ext_genid, some extensions
 847          * may contain stale pointers to e.g. helper that has been removed.
 848          *
 849          * The helper can't clear this because the nf_conn object isn't in
 850          * any hash and synchronize_rcu() isn't enough because associated skb
 851          * might sit in a queue.
 852          */
 853         return !ext || ext->gen_id == atomic_read(&nf_conntrack_ext_genid);
 854 }
 855
 856 static bool nf_ct_ext_valid_post(struct nf_ct_ext *ext)
 857 {
 858         if (!ext)
 859                 return true;
 860
 861         if (ext->gen_id != atomic_read(&nf_conntrack_ext_genid))
 862                 return false;
 863
 864         /* inserted into conntrack table, nf_ct_iterate_cleanup()
 865          * will find it.  Disable nf_ct_ext_find() id check.
 866          */
 867         WRITE_ONCE(ext->gen_id, 0);
 868         return true;
 869 }
 870
 871 int
 872 nf_conntrack_hash_check_insert(struct nf_conn *ct)
 873 {
 874         const struct nf_conntrack_zone *zone;
 875         struct net *net = nf_ct_net(ct);
 876         unsigned int hash, reply_hash;
 877         struct nf_conntrack_tuple_hash *h;
 878         struct hlist_nulls_node *n;
 879         unsigned int max_chainlen;
 880         unsigned int chainlen = 0;
 881         unsigned int sequence;
 882         int err = -EEXIST;
 883
 884         zone = nf_ct_zone(ct);
 885
 886         if (!nf_ct_ext_valid_pre(ct->ext)) {
 887                 NF_CT_STAT_INC(net, insert_failed);
 888                 return -ETIMEDOUT;
 889         }
 890
 891         local_bh_disable();
 892         do {
 893                 sequence = read_seqcount_begin(&nf_conntrack_generation);
 894                 hash = hash_conntrack(net,
 895                                       &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple,
 896                                       nf_ct_zone_id(nf_ct_zone(ct), IP_CT_DIR_ORIGINAL));
 897                 reply_hash = hash_conntrack(net,
 898                                            &ct->tuplehash[IP_CT_DIR_REPLY].tuple,
 899                                            nf_ct_zone_id(nf_ct_zone(ct), IP_CT_DIR_REPLY));
 900         } while (nf_conntrack_double_lock(net, hash, reply_hash, sequence));
 901
 902         max_chainlen = MIN_CHAINLEN + prandom_u32_max(MAX_CHAINLEN);
 903
 904         /* See if there's one in the list already, including reverse */
 905         hlist_nulls_for_each_entry(h, n, &nf_conntrack_hash[hash], hnnode) {
 906                 if (nf_ct_key_equal(h, &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple,
 907                                     zone, net))
 908                         goto out;
 909
 910                 if (chainlen++ > max_chainlen)
 911                         goto chaintoolong;
 912         }
 913
 914         chainlen = 0;
 915
 916         hlist_nulls_for_each_entry(h, n, &nf_conntrack_hash[reply_hash], hnnode) {
 917                 if (nf_ct_key_equal(h, &ct->tuplehash[IP_CT_DIR_REPLY].tuple,
 918                                     zone, net))
 919                         goto out;
 920                 if (chainlen++ > max_chainlen)
 921                         goto chaintoolong;
 922         }
 923
 924         smp_wmb();
 925         /* The caller holds a reference to this object */
 926         refcount_set(&ct->ct_general.use, 2);
 927         __nf_conntrack_hash_insert(ct, hash, reply_hash);
 928         nf_conntrack_double_unlock(hash, reply_hash);
 929         NF_CT_STAT_INC(net, insert);
 930         local_bh_enable();
 931
 932         if (!nf_ct_ext_valid_post(ct->ext)) {
 933                 nf_ct_kill(ct);
 934                 NF_CT_STAT_INC(net, drop);
 935                 return -ETIMEDOUT;
 936         }
 937
 938         return 0;
 939 chaintoolong:
 940         NF_CT_STAT_INC(net, chaintoolong);
 941         err = -ENOSPC;
 942 out:
 943         nf_conntrack_double_unlock(hash, reply_hash);
 944         local_bh_enable();
 945         return err;
 946 }
 947 EXPORT_SYMBOL_GPL(nf_conntrack_hash_check_insert);
 948
 949 void nf_ct_acct_add(struct nf_conn *ct, u32 dir, unsigned int packets,
 950                     unsigned int bytes)
 951 {
 952         struct nf_conn_acct *acct;
 953
 954         acct = nf_conn_acct_find(ct);
 955         if (acct) {
 956                 struct nf_conn_counter *counter = acct->counter;
 957
 958                 atomic64_add(packets, &counter[dir].packets);
 959                 atomic64_add(bytes, &counter[dir].bytes);
 960         }
 961 }
 962 EXPORT_SYMBOL_GPL(nf_ct_acct_add);
 963
 964 static void nf_ct_acct_merge(struct nf_conn *ct, enum ip_conntrack_info ctinfo,
 965                              const struct nf_conn *loser_ct)
 966 {
 967         struct nf_conn_acct *acct;
 968
 969         acct = nf_conn_acct_find(loser_ct);
 970         if (acct) {
 971                 struct nf_conn_counter *counter = acct->counter;
 972                 unsigned int bytes;
 973
 974                 /* u32 should be fine since we must have seen one packet. */
 975                 bytes = atomic64_read(&counter[CTINFO2DIR(ctinfo)].bytes);
 976                 nf_ct_acct_update(ct, CTINFO2DIR(ctinfo), bytes);
 977         }
 978 }
 979
 980 static void __nf_conntrack_insert_prepare(struct nf_conn *ct)
 981 {
 982         struct nf_conn_tstamp *tstamp;
 983
 984         refcount_inc(&ct->ct_general.use);
 985
 986         /* set conntrack timestamp, if enabled. */
 987         tstamp = nf_conn_tstamp_find(ct);
 988         if (tstamp)
 989                 tstamp->start = ktime_get_real_ns();
 990 }
 991
 992 /* caller must hold locks to prevent concurrent changes */
 993 static int __nf_ct_resolve_clash(struct sk_buff *skb,
 994                                  struct nf_conntrack_tuple_hash *h)
 995 {
 996         /* This is the conntrack entry already in hashes that won race. */
 997         struct nf_conn *ct = nf_ct_tuplehash_to_ctrack(h);
 998         enum ip_conntrack_info ctinfo;
 999         struct nf_conn *loser_ct;
1000
1001         loser_ct = nf_ct_get(skb, &ctinfo);
1002
1003         if (nf_ct_is_dying(ct))
1004                 return NF_DROP;
1005
1006         if (((ct->status & IPS_NAT_DONE_MASK) == 0) ||
1007             nf_ct_match(ct, loser_ct)) {
1008                 struct net *net = nf_ct_net(ct);
1009
1010                 nf_conntrack_get(&ct->ct_general);
1011
1012                 nf_ct_acct_merge(ct, ctinfo, loser_ct);
1013                 nf_ct_put(loser_ct);
1014                 nf_ct_set(skb, ct, ctinfo);
1015
1016                 NF_CT_STAT_INC(net, clash_resolve);
1017                 return NF_ACCEPT;
1018         }
1019
1020         return NF_DROP;
1021 }
1022
1023 /**
1024  * nf_ct_resolve_clash_harder - attempt to insert clashing conntrack entry
1025  *
1026  * @skb: skb that causes the collision
1027  * @repl_idx: hash slot for reply direction
1028  *
1029  * Called when origin or reply direction had a clash.
1030  * The skb can be handled without packet drop provided the reply direction
1031  * is unique or there the existing entry has the identical tuple in both
1032  * directions.
1033  *
1034  * Caller must hold conntrack table locks to prevent concurrent updates.
1035  *
1036  * Returns NF_DROP if the clash could not be handled.
1037  */
1038 static int nf_ct_resolve_clash_harder(struct sk_buff *skb, u32 repl_idx)
1039 {
1040         struct nf_conn *loser_ct = (struct nf_conn *)skb_nfct(skb);
1041         const struct nf_conntrack_zone *zone;
1042         struct nf_conntrack_tuple_hash *h;
1043         struct hlist_nulls_node *n;
1044         struct net *net;
1045
1046         zone = nf_ct_zone(loser_ct);
1047         net = nf_ct_net(loser_ct);
1048
1049         /* Reply direction must never result in a clash, unless both origin
1050          * and reply tuples are identical.
1051          */
1052         hlist_nulls_for_each_entry(h, n, &nf_conntrack_hash[repl_idx], hnnode) {
1053                 if (nf_ct_key_equal(h,
1054                                     &loser_ct->tuplehash[IP_CT_DIR_REPLY].tuple,
1055                                     zone, net))
1056                         return __nf_ct_resolve_clash(skb, h);
1057         }
1058
1059         /* We want the clashing entry to go away real soon: 1 second timeout. */
1060         WRITE_ONCE(loser_ct->timeout, nfct_time_stamp + HZ);
1061
1062         /* IPS_NAT_CLASH removes the entry automatically on the first
1063          * reply.  Also prevents UDP tracker from moving the entry to
1064          * ASSURED state, i.e. the entry can always be evicted under
1065          * pressure.
1066          */
1067         loser_ct->status |= IPS_FIXED_TIMEOUT | IPS_NAT_CLASH;
1068
1069         __nf_conntrack_insert_prepare(loser_ct);
1070
1071         /* fake add for ORIGINAL dir: we want lookups to only find the entry
1072          * already in the table.  This also hides the clashing entry from
1073          * ctnetlink iteration, i.e. conntrack -L won't show them.
1074          */
1075         hlist_nulls_add_fake(&loser_ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);
1076
1077         hlist_nulls_add_head_rcu(&loser_ct->tuplehash[IP_CT_DIR_REPLY].hnnode,
1078                                  &nf_conntrack_hash[repl_idx]);
1079
1080         NF_CT_STAT_INC(net, clash_resolve);
1081         return NF_ACCEPT;
1082 }
1083
1084 /**
1085  * nf_ct_resolve_clash - attempt to handle clash without packet drop
1086  *
1087  * @skb: skb that causes the clash
1088  * @h: tuplehash of the clashing entry already in table
1089  * @reply_hash: hash slot for reply direction
1090  *
1091  * A conntrack entry can be inserted to the connection tracking table
1092  * if there is no existing entry with an identical tuple.
1093  *
1094  * If there is one, @skb (and the assocated, unconfirmed conntrack) has
1095  * to be dropped.  In case @skb is retransmitted, next conntrack lookup
1096  * will find the already-existing entry.
1097  *
1098  * The major problem with such packet drop is the extra delay added by
1099  * the packet loss -- it will take some time for a retransmit to occur
1100  * (or the sender to time out when waiting for a reply).
1101  *
1102  * This function attempts to handle the situation without packet drop.
1103  *
1104  * If @skb has no NAT transformation or if the colliding entries are
1105  * exactly the same, only the to-be-confirmed conntrack entry is discarded
1106  * and @skb is associated with the conntrack entry already in the table.
1107  *
1108  * Failing that, the new, unconfirmed conntrack is still added to the table
1109  * provided that the collision only occurs in the ORIGINAL direction.
1110  * The new entry will be added only in the non-clashing REPLY direction,
1111  * so packets in the ORIGINAL direction will continue to match the existing
1112  * entry.  The new entry will also have a fixed timeout so it expires --
1113  * due to the collision, it will only see reply traffic.
1114  *
1115  * Returns NF_DROP if the clash could not be resolved.
1116  */
1117 static __cold noinline int
1118 nf_ct_resolve_clash(struct sk_buff *skb, struct nf_conntrack_tuple_hash *h,
1119                     u32 reply_hash)
1120 {
1121         /* This is the conntrack entry already in hashes that won race. */
1122         struct nf_conn *ct = nf_ct_tuplehash_to_ctrack(h);
1123         const struct nf_conntrack_l4proto *l4proto;
1124         enum ip_conntrack_info ctinfo;
1125         struct nf_conn *loser_ct;
1126         struct net *net;
1127         int ret;
1128
1129         loser_ct = nf_ct_get(skb, &ctinfo);
1130         net = nf_ct_net(loser_ct);
1131
1132         l4proto = nf_ct_l4proto_find(nf_ct_protonum(ct));
1133         if (!l4proto->allow_clash)
1134                 goto drop;
1135
1136         ret = __nf_ct_resolve_clash(skb, h);
1137         if (ret == NF_ACCEPT)
1138                 return ret;
1139
1140         ret = nf_ct_resolve_clash_harder(skb, reply_hash);
1141         if (ret == NF_ACCEPT)
1142                 return ret;
1143
1144 drop:
1145         NF_CT_STAT_INC(net, drop);
1146         NF_CT_STAT_INC(net, insert_failed);
1147         return NF_DROP;
1148 }
1149
1150 /* Confirm a connection given skb; places it in hash table */
1151 int
1152 __nf_conntrack_confirm(struct sk_buff *skb)
1153 {
1154         unsigned int chainlen = 0, sequence, max_chainlen;
1155         const struct nf_conntrack_zone *zone;
1156         unsigned int hash, reply_hash;
1157         struct nf_conntrack_tuple_hash *h;
1158         struct nf_conn *ct;
1159         struct nf_conn_help *help;
1160         struct hlist_nulls_node *n;
1161         enum ip_conntrack_info ctinfo;
1162         struct net *net;
1163         int ret = NF_DROP;
1164
1165         ct = nf_ct_get(skb, &ctinfo);
1166         net = nf_ct_net(ct);
1167
1168         /* ipt_REJECT uses nf_conntrack_attach to attach related
1169            ICMP/TCP RST packets in other direction.  Actual packet
1170            which created connection will be IP_CT_NEW or for an
1171            expected connection, IP_CT_RELATED. */
1172         if (CTINFO2DIR(ctinfo) != IP_CT_DIR_ORIGINAL)
1173                 return NF_ACCEPT;
1174
1175         zone = nf_ct_zone(ct);
1176         local_bh_disable();
1177
1178         do {
1179                 sequence = read_seqcount_begin(&nf_conntrack_generation);
1180                 /* reuse the hash saved before */
1181                 hash = *(unsigned long *)&ct->tuplehash[IP_CT_DIR_REPLY].hnnode.pprev;
1182                 hash = scale_hash(hash);
1183                 reply_hash = hash_conntrack(net,
1184                                            &ct->tuplehash[IP_CT_DIR_REPLY].tuple,
1185                                            nf_ct_zone_id(nf_ct_zone(ct), IP_CT_DIR_REPLY));
1186         } while (nf_conntrack_double_lock(net, hash, reply_hash, sequence));
1187
1188         /* We're not in hash table, and we refuse to set up related
1189          * connections for unconfirmed conns.  But packet copies and
1190          * REJECT will give spurious warnings here.
1191          */
1192
1193         /* Another skb with the same unconfirmed conntrack may
1194          * win the race. This may happen for bridge(br_flood)
1195          * or broadcast/multicast packets do skb_clone with
1196          * unconfirmed conntrack.
1197          */
1198         if (unlikely(nf_ct_is_confirmed(ct))) {
1199                 WARN_ON_ONCE(1);
1200                 nf_conntrack_double_unlock(hash, reply_hash);
1201                 local_bh_enable();
1202                 return NF_DROP;
1203         }
1204
1205         if (!nf_ct_ext_valid_pre(ct->ext)) {
1206                 NF_CT_STAT_INC(net, insert_failed);
1207                 goto dying;
1208         }
1209
1210         pr_debug("Confirming conntrack %p\n", ct);
1211         /* We have to check the DYING flag after unlink to prevent
1212          * a race against nf_ct_get_next_corpse() possibly called from
1213          * user context, else we insert an already 'dead' hash, blocking
1214          * further use of that particular connection -JM.
1215          */
1216         ct->status |= IPS_CONFIRMED;
1217
1218         if (unlikely(nf_ct_is_dying(ct))) {
1219                 NF_CT_STAT_INC(net, insert_failed);
1220                 goto dying;
1221         }
1222
1223         max_chainlen = MIN_CHAINLEN + prandom_u32_max(MAX_CHAINLEN);
1224         /* See if there's one in the list already, including reverse:
1225            NAT could have grabbed it without realizing, since we're
1226            not in the hash.  If there is, we lost race. */
1227         hlist_nulls_for_each_entry(h, n, &nf_conntrack_hash[hash], hnnode) {
1228                 if (nf_ct_key_equal(h, &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple,
1229                                     zone, net))
1230                         goto out;
1231                 if (chainlen++ > max_chainlen)
1232                         goto chaintoolong;
1233         }
1234
1235         chainlen = 0;
1236         hlist_nulls_for_each_entry(h, n, &nf_conntrack_hash[reply_hash], hnnode) {
1237                 if (nf_ct_key_equal(h, &ct->tuplehash[IP_CT_DIR_REPLY].tuple,
1238                                     zone, net))
1239                         goto out;
1240                 if (chainlen++ > max_chainlen) {
1241 chaintoolong:
1242                         NF_CT_STAT_INC(net, chaintoolong);
1243                         NF_CT_STAT_INC(net, insert_failed);
1244                         ret = NF_DROP;
1245                         goto dying;
1246                 }
1247         }
1248
1249         /* Timer relative to confirmation time, not original
1250            setting time, otherwise we'd get timer wrap in
1251            weird delay cases. */
1252         ct->timeout += nfct_time_stamp;
1253
1254         __nf_conntrack_insert_prepare(ct);
1255
1256         /* Since the lookup is lockless, hash insertion must be done after
1257          * starting the timer and setting the CONFIRMED bit. The RCU barriers
1258          * guarantee that no other CPU can find the conntrack before the above
1259          * stores are visible.
1260          */
1261         __nf_conntrack_hash_insert(ct, hash, reply_hash);
1262         nf_conntrack_double_unlock(hash, reply_hash);
1263         local_bh_enable();
1264
1265         /* ext area is still valid (rcu read lock is held,
1266          * but will go out of scope soon, we need to remove
1267          * this conntrack again.
1268          */
1269         if (!nf_ct_ext_valid_post(ct->ext)) {
1270                 nf_ct_kill(ct);
1271                 NF_CT_STAT_INC(net, drop);
1272                 return NF_DROP;
1273         }
1274
1275         help = nfct_help(ct);
1276         if (help && help->helper)
1277                 nf_conntrack_event_cache(IPCT_HELPER, ct);
1278
1279         nf_conntrack_event_cache(master_ct(ct) ?
1280                                  IPCT_RELATED : IPCT_NEW, ct);
1281         return NF_ACCEPT;
1282
1283 out:
1284         ret = nf_ct_resolve_clash(skb, h, reply_hash);
1285 dying:
1286         nf_conntrack_double_unlock(hash, reply_hash);
1287         local_bh_enable();
1288         return ret;
1289 }
1290 EXPORT_SYMBOL_GPL(__nf_conntrack_confirm);
1291
1292 /* Returns true if a connection correspondings to the tuple (required
1293    for NAT). */
1294 int
1295 nf_conntrack_tuple_taken(const struct nf_conntrack_tuple *tuple,
1296                          const struct nf_conn *ignored_conntrack)
1297 {
1298         struct net *net = nf_ct_net(ignored_conntrack);
1299         const struct nf_conntrack_zone *zone;
1300         struct nf_conntrack_tuple_hash *h;
1301         struct hlist_nulls_head *ct_hash;
1302         unsigned int hash, hsize;
1303         struct hlist_nulls_node *n;
1304         struct nf_conn *ct;
1305
1306         zone = nf_ct_zone(ignored_conntrack);
1307
1308         rcu_read_lock();
1309  begin:
1310         nf_conntrack_get_ht(&ct_hash, &hsize);
1311         hash = __hash_conntrack(net, tuple, nf_ct_zone_id(zone, IP_CT_DIR_REPLY), hsize);
1312
1313         hlist_nulls_for_each_entry_rcu(h, n, &ct_hash[hash], hnnode) {
1314                 ct = nf_ct_tuplehash_to_ctrack(h);
1315
1316                 if (ct == ignored_conntrack)
1317                         continue;
1318
1319                 if (nf_ct_is_expired(ct)) {
1320                         nf_ct_gc_expired(ct);
1321                         continue;
1322                 }
1323
1324                 if (nf_ct_key_equal(h, tuple, zone, net)) {
1325                         /* Tuple is taken already, so caller will need to find
1326                          * a new source port to use.
1327                          *
1328                          * Only exception:
1329                          * If the *original tuples* are identical, then both
1330                          * conntracks refer to the same flow.
1331                          * This is a rare situation, it can occur e.g. when
1332                          * more than one UDP packet is sent from same socket
1333                          * in different threads.
1334                          *
1335                          * Let nf_ct_resolve_clash() deal with this later.
1336                          */
1337                         if (nf_ct_tuple_equal(&ignored_conntrack->tuplehash[IP_CT_DIR_ORIGINAL].tuple,
1338                                               &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple) &&
1339                                               nf_ct_zone_equal(ct, zone, IP_CT_DIR_ORIGINAL))
1340                                 continue;
1341
1342                         NF_CT_STAT_INC_ATOMIC(net, found);
1343                         rcu_read_unlock();
1344                         return 1;
1345                 }
1346         }
1347
1348         if (get_nulls_value(n) != hash) {
1349                 NF_CT_STAT_INC_ATOMIC(net, search_restart);
1350                 goto begin;
1351         }
1352
1353         rcu_read_unlock();
1354
1355         return 0;
1356 }
1357 EXPORT_SYMBOL_GPL(nf_conntrack_tuple_taken);
1358
1359 #define NF_CT_EVICTION_RANGE    8
1360
1361 /* There's a small race here where we may free a just-assured
1362    connection.  Too bad: we're in trouble anyway. */
1363 static unsigned int early_drop_list(struct net *net,
1364                                     struct hlist_nulls_head *head)
1365 {
1366         struct nf_conntrack_tuple_hash *h;
1367         struct hlist_nulls_node *n;
1368         unsigned int drops = 0;
1369         struct nf_conn *tmp;
1370
1371         hlist_nulls_for_each_entry_rcu(h, n, head, hnnode) {
1372                 tmp = nf_ct_tuplehash_to_ctrack(h);
1373
1374                 if (test_bit(IPS_OFFLOAD_BIT, &tmp->status))
1375                         continue;
1376
1377                 if (nf_ct_is_expired(tmp)) {
1378                         nf_ct_gc_expired(tmp);
1379                         continue;
1380                 }
1381
1382                 if (test_bit(IPS_ASSURED_BIT, &tmp->status) ||
1383                     !net_eq(nf_ct_net(tmp), net) ||
1384                     nf_ct_is_dying(tmp))
1385                         continue;
1386
1387                 if (!refcount_inc_not_zero(&tmp->ct_general.use))
1388                         continue;
1389
1390                 /* kill only if still in same netns -- might have moved due to
1391                  * SLAB_TYPESAFE_BY_RCU rules.
1392                  *
1393                  * We steal the timer reference.  If that fails timer has
1394                  * already fired or someone else deleted it. Just drop ref
1395                  * and move to next entry.
1396                  */
1397                 if (net_eq(nf_ct_net(tmp), net) &&
1398                     nf_ct_is_confirmed(tmp) &&
1399                     nf_ct_delete(tmp, 0, 0))
1400                         drops++;
1401
1402                 nf_ct_put(tmp);
1403         }
1404
1405         return drops;
1406 }
1407
1408 static noinline int early_drop(struct net *net, unsigned int hash)
1409 {
1410         unsigned int i, bucket;
1411
1412         for (i = 0; i < NF_CT_EVICTION_RANGE; i++) {
1413                 struct hlist_nulls_head *ct_hash;
1414                 unsigned int hsize, drops;
1415
1416                 rcu_read_lock();
1417                 nf_conntrack_get_ht(&ct_hash, &hsize);
1418                 if (!i)
1419                         bucket = reciprocal_scale(hash, hsize);
1420                 else
1421                         bucket = (bucket + 1) % hsize;
1422
1423                 drops = early_drop_list(net, &ct_hash[bucket]);
1424                 rcu_read_unlock();
1425
1426                 if (drops) {
1427                         NF_CT_STAT_ADD_ATOMIC(net, early_drop, drops);
1428                         return true;
1429                 }
1430         }
1431
1432         return false;
1433 }
1434
1435 static bool gc_worker_skip_ct(const struct nf_conn *ct)
1436 {
1437         return !nf_ct_is_confirmed(ct) || nf_ct_is_dying(ct);
1438 }
1439
1440 static bool gc_worker_can_early_drop(const struct nf_conn *ct)
1441 {
1442         const struct nf_conntrack_l4proto *l4proto;
1443
1444         if (!test_bit(IPS_ASSURED_BIT, &ct->status))
1445                 return true;
1446
1447         l4proto = nf_ct_l4proto_find(nf_ct_protonum(ct));
1448         if (l4proto->can_early_drop && l4proto->can_early_drop(ct))
1449                 return true;
1450
1451         return false;
1452 }
1453
1454 static void gc_worker(struct work_struct *work)
1455 {
1456         unsigned int i, hashsz, nf_conntrack_max95 = 0;
1457         u32 end_time, start_time = nfct_time_stamp;
1458         struct conntrack_gc_work *gc_work;
1459         unsigned int expired_count = 0;
1460         unsigned long next_run;
1461         s32 delta_time;
1462
1463         gc_work = container_of(work, struct conntrack_gc_work, dwork.work);
1464
1465         i = gc_work->next_bucket;
1466         if (gc_work->early_drop)
1467                 nf_conntrack_max95 = nf_conntrack_max / 100u * 95u;
1468
1469         if (i == 0) {
1470                 gc_work->avg_timeout = GC_SCAN_INTERVAL_INIT;
1471                 gc_work->start_time = start_time;
1472         }
1473
1474         next_run = gc_work->avg_timeout;
1475
1476         end_time = start_time + GC_SCAN_MAX_DURATION;
1477
1478         do {
1479                 struct nf_conntrack_tuple_hash *h;
1480                 struct hlist_nulls_head *ct_hash;
1481                 struct hlist_nulls_node *n;
1482                 struct nf_conn *tmp;
1483
1484                 rcu_read_lock();
1485
1486                 nf_conntrack_get_ht(&ct_hash, &hashsz);
1487                 if (i >= hashsz) {
1488                         rcu_read_unlock();
1489                         break;
1490                 }
1491
1492                 hlist_nulls_for_each_entry_rcu(h, n, &ct_hash[i], hnnode) {
1493                         struct nf_conntrack_net *cnet;
1494                         unsigned long expires;
1495                         struct net *net;
1496
1497                         tmp = nf_ct_tuplehash_to_ctrack(h);
1498
1499                         if (test_bit(IPS_OFFLOAD_BIT, &tmp->status)) {
1500                                 nf_ct_offload_timeout(tmp);
1501                                 continue;
1502                         }
1503
1504                         if (expired_count > GC_SCAN_EXPIRED_MAX) {
1505                                 rcu_read_unlock();
1506
1507                                 gc_work->next_bucket = i;
1508                                 gc_work->avg_timeout = next_run;
1509
1510                                 delta_time = nfct_time_stamp - gc_work->start_time;
1511
1512                                 /* re-sched immediately if total cycle time is exceeded */
1513                                 next_run = delta_time < (s32)GC_SCAN_INTERVAL_MAX;
1514                                 goto early_exit;
1515                         }
1516
1517                         if (nf_ct_is_expired(tmp)) {
1518                                 nf_ct_gc_expired(tmp);
1519                                 expired_count++;
1520                                 continue;
1521                         }
1522
1523                         expires = clamp(nf_ct_expires(tmp), GC_SCAN_INTERVAL_MIN, GC_SCAN_INTERVAL_CLAMP);
1524                         next_run += expires;
1525                         next_run /= 2u;
1526
1527                         if (nf_conntrack_max95 == 0 || gc_worker_skip_ct(tmp))
1528                                 continue;
1529
1530                         net = nf_ct_net(tmp);
1531                         cnet = nf_ct_pernet(net);
1532                         if (atomic_read(&cnet->count) < nf_conntrack_max95)
1533                                 continue;
1534
1535                         /* need to take reference to avoid possible races */
1536                         if (!refcount_inc_not_zero(&tmp->ct_general.use))
1537                                 continue;
1538
1539                         if (gc_worker_skip_ct(tmp)) {
1540                                 nf_ct_put(tmp);
1541                                 continue;
1542                         }
1543
1544                         if (gc_worker_can_early_drop(tmp)) {
1545                                 nf_ct_kill(tmp);
1546                                 expired_count++;
1547                         }
1548
1549                         nf_ct_put(tmp);
1550                 }
1551
1552                 /* could check get_nulls_value() here and restart if ct
1553                  * was moved to another chain.  But given gc is best-effort
1554                  * we will just continue with next hash slot.
1555                  */
1556                 rcu_read_unlock();
1557                 cond_resched();
1558                 i++;
1559
1560                 delta_time = nfct_time_stamp - end_time;
1561                 if (delta_time > 0 && i < hashsz) {
1562                         gc_work->avg_timeout = next_run;
1563                         gc_work->next_bucket = i;
1564                         next_run = 0;
1565                         goto early_exit;
1566                 }
1567         } while (i < hashsz);
1568
1569         gc_work->next_bucket = 0;
1570
1571         next_run = clamp(next_run, GC_SCAN_INTERVAL_MIN, GC_SCAN_INTERVAL_MAX);
1572
1573         delta_time = max_t(s32, nfct_time_stamp - gc_work->start_time, 1);
1574         if (next_run > (unsigned long)delta_time)
1575                 next_run -= delta_time;
1576         else
1577                 next_run = 1;
1578
1579 early_exit:
1580         if (gc_work->exiting)
1581                 return;
1582
1583         if (next_run)
1584                 gc_work->early_drop = false;
1585
1586         queue_delayed_work(system_power_efficient_wq, &gc_work->dwork, next_run);
1587 }
1588
1589 static void conntrack_gc_work_init(struct conntrack_gc_work *gc_work)
1590 {
1591         INIT_DELAYED_WORK(&gc_work->dwork, gc_worker);
1592         gc_work->exiting = false;
1593 }
1594
1595 static struct nf_conn *
1596 __nf_conntrack_alloc(struct net *net,
1597                      const struct nf_conntrack_zone *zone,
1598                      const struct nf_conntrack_tuple *orig,
1599                      const struct nf_conntrack_tuple *repl,
1600                      gfp_t gfp, u32 hash)
1601 {
1602         struct nf_conntrack_net *cnet = nf_ct_pernet(net);
1603         unsigned int ct_count;
1604         struct nf_conn *ct;
1605
1606         /* We don't want any race condition at early drop stage */
1607         ct_count = atomic_inc_return(&cnet->count);
1608
1609         if (nf_conntrack_max && unlikely(ct_count > nf_conntrack_max)) {
1610                 if (!early_drop(net, hash)) {
1611                         if (!conntrack_gc_work.early_drop)
1612                                 conntrack_gc_work.early_drop = true;
1613                         atomic_dec(&cnet->count);
1614                         net_warn_ratelimited("nf_conntrack: table full, dropping packet\n");
1615                         return ERR_PTR(-ENOMEM);
1616                 }
1617         }
1618
1619         /*
1620          * Do not use kmem_cache_zalloc(), as this cache uses
1621          * SLAB_TYPESAFE_BY_RCU.
1622          */
1623         ct = kmem_cache_alloc(nf_conntrack_cachep, gfp);
1624         if (ct == NULL)
1625                 goto out;
1626
1627         spin_lock_init(&ct->lock);
1628         ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple = *orig;
1629         ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode.pprev = NULL;
1630         ct->tuplehash[IP_CT_DIR_REPLY].tuple = *repl;
1631         /* save hash for reusing when confirming */
1632         *(unsigned long *)(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode.pprev) = hash;
1633         ct->status = 0;
1634         WRITE_ONCE(ct->timeout, 0);
1635         write_pnet(&ct->ct_net, net);
1636         memset_after(ct, 0, __nfct_init_offset);
1637
1638         nf_ct_zone_add(ct, zone);
1639
1640         /* Because we use RCU lookups, we set ct_general.use to zero before
1641          * this is inserted in any list.
1642          */
1643         refcount_set(&ct->ct_general.use, 0);
1644         return ct;
1645 out:
1646         atomic_dec(&cnet->count);
1647         return ERR_PTR(-ENOMEM);
1648 }
1649
1650 struct nf_conn *nf_conntrack_alloc(struct net *net,
1651                                    const struct nf_conntrack_zone *zone,
1652                                    const struct nf_conntrack_tuple *orig,
1653                                    const struct nf_conntrack_tuple *repl,
1654                                    gfp_t gfp)
1655 {
1656         return __nf_conntrack_alloc(net, zone, orig, repl, gfp, 0);
1657 }
1658 EXPORT_SYMBOL_GPL(nf_conntrack_alloc);
1659
1660 void nf_conntrack_free(struct nf_conn *ct)
1661 {
1662         struct net *net = nf_ct_net(ct);
1663         struct nf_conntrack_net *cnet;
1664
1665         /* A freed object has refcnt == 0, that's
1666          * the golden rule for SLAB_TYPESAFE_BY_RCU
1667          */
1668         WARN_ON(refcount_read(&ct->ct_general.use) != 0);
1669
1670         if (ct->status & IPS_SRC_NAT_DONE) {
1671                 const struct nf_nat_hook *nat_hook;
1672
1673                 rcu_read_lock();
1674                 nat_hook = rcu_dereference(nf_nat_hook);
1675                 if (nat_hook)
1676                         nat_hook->remove_nat_bysrc(ct);
1677                 rcu_read_unlock();
1678         }
1679
1680         kfree(ct->ext);
1681         kmem_cache_free(nf_conntrack_cachep, ct);
1682         cnet = nf_ct_pernet(net);
1683
1684         smp_mb__before_atomic();
1685         atomic_dec(&cnet->count);
1686 }
1687 EXPORT_SYMBOL_GPL(nf_conntrack_free);
1688
1689
1690 /* Allocate a new conntrack: we return -ENOMEM if classification
1691    failed due to stress.  Otherwise it really is unclassifiable. */
1692 static noinline struct nf_conntrack_tuple_hash *
1693 init_conntrack(struct net *net, struct nf_conn *tmpl,
1694                const struct nf_conntrack_tuple *tuple,
1695                struct sk_buff *skb,
1696                unsigned int dataoff, u32 hash)
1697 {
1698         struct nf_conn *ct;
1699         struct nf_conn_help *help;
1700         struct nf_conntrack_tuple repl_tuple;
1701 #ifdef CONFIG_NF_CONNTRACK_EVENTS
1702         struct nf_conntrack_ecache *ecache;
1703 #endif
1704         struct nf_conntrack_expect *exp = NULL;
1705         const struct nf_conntrack_zone *zone;
1706         struct nf_conn_timeout *timeout_ext;
1707         struct nf_conntrack_zone tmp;
1708         struct nf_conntrack_net *cnet;
1709
1710         if (!nf_ct_invert_tuple(&repl_tuple, tuple)) {
1711                 pr_debug("Can't invert tuple.\n");
1712                 return NULL;
1713         }
1714
1715         zone = nf_ct_zone_tmpl(tmpl, skb, &tmp);
1716         ct = __nf_conntrack_alloc(net, zone, tuple, &repl_tuple, GFP_ATOMIC,
1717                                   hash);
1718         if (IS_ERR(ct))
1719                 return (struct nf_conntrack_tuple_hash *)ct;
1720
1721         if (!nf_ct_add_synproxy(ct, tmpl)) {
1722                 nf_conntrack_free(ct);
1723                 return ERR_PTR(-ENOMEM);
1724         }
1725
1726         timeout_ext = tmpl ? nf_ct_timeout_find(tmpl) : NULL;
1727
1728         if (timeout_ext)
1729                 nf_ct_timeout_ext_add(ct, rcu_dereference(timeout_ext->timeout),
1730                                       GFP_ATOMIC);
1731
1732         nf_ct_acct_ext_add(ct, GFP_ATOMIC);
1733         nf_ct_tstamp_ext_add(ct, GFP_ATOMIC);
1734         nf_ct_labels_ext_add(ct);
1735
1736 #ifdef CONFIG_NF_CONNTRACK_EVENTS
1737         ecache = tmpl ? nf_ct_ecache_find(tmpl) : NULL;
1738
1739         if ((ecache || net->ct.sysctl_events) &&
1740             !nf_ct_ecache_ext_add(ct, ecache ? ecache->ctmask : 0,
1741                                   ecache ? ecache->expmask : 0,
1742                                   GFP_ATOMIC)) {
1743                 nf_conntrack_free(ct);
1744                 return ERR_PTR(-ENOMEM);
1745         }
1746 #endif
1747
1748         cnet = nf_ct_pernet(net);
1749         if (cnet->expect_count) {
1750                 spin_lock_bh(&nf_conntrack_expect_lock);
1751                 exp = nf_ct_find_expectation(net, zone, tuple);
1752                 if (exp) {
1753                         pr_debug("expectation arrives ct=%p exp=%p\n",
1754                                  ct, exp);
1755                         /* Welcome, Mr. Bond.  We've been expecting you... */
1756                         __set_bit(IPS_EXPECTED_BIT, &ct->status);
1757                         /* exp->master safe, refcnt bumped in nf_ct_find_expectation */
1758                         ct->master = exp->master;
1759                         if (exp->helper) {
1760                                 help = nf_ct_helper_ext_add(ct, GFP_ATOMIC);
1761                                 if (help)
1762                                         rcu_assign_pointer(help->helper, exp->helper);
1763                         }
1764
1765 #ifdef CONFIG_NF_CONNTRACK_MARK
1766                         ct->mark = exp->master->mark;
1767 #endif
1768 #ifdef CONFIG_NF_CONNTRACK_SECMARK
1769                         ct->secmark = exp->master->secmark;
1770 #endif
1771                         NF_CT_STAT_INC(net, expect_new);
1772                 }
1773                 spin_unlock_bh(&nf_conntrack_expect_lock);
1774         }
1775         if (!exp)
1776                 __nf_ct_try_assign_helper(ct, tmpl, GFP_ATOMIC);
1777
1778         /* Now it is going to be associated with an sk_buff, set refcount to 1. */
1779         refcount_set(&ct->ct_general.use, 1);
1780
1781         if (exp) {
1782                 if (exp->expectfn)
1783                         exp->expectfn(ct, exp);
1784                 nf_ct_expect_put(exp);
1785         }
1786
1787         return &ct->tuplehash[IP_CT_DIR_ORIGINAL];
1788 }
1789
1790 /* On success, returns 0, sets skb->_nfct | ctinfo */
1791 static int
1792 resolve_normal_ct(struct nf_conn *tmpl,
1793                   struct sk_buff *skb,
1794                   unsigned int dataoff,
1795                   u_int8_t protonum,
1796                   const struct nf_hook_state *state)
1797 {
1798         const struct nf_conntrack_zone *zone;
1799         struct nf_conntrack_tuple tuple;
1800         struct nf_conntrack_tuple_hash *h;
1801         enum ip_conntrack_info ctinfo;
1802         struct nf_conntrack_zone tmp;
1803         u32 hash, zone_id, rid;
1804         struct nf_conn *ct;
1805
1806         if (!nf_ct_get_tuple(skb, skb_network_offset(skb),
1807                              dataoff, state->pf, protonum, state->net,
1808                              &tuple)) {
1809                 pr_debug("Can't get tuple\n");
1810                 return 0;
1811         }
1812
1813         /* look for tuple match */
1814         zone = nf_ct_zone_tmpl(tmpl, skb, &tmp);
1815
1816         zone_id = nf_ct_zone_id(zone, IP_CT_DIR_ORIGINAL);
1817         hash = hash_conntrack_raw(&tuple, zone_id, state->net);
1818         h = __nf_conntrack_find_get(state->net, zone, &tuple, hash);
1819
1820         if (!h) {
1821                 rid = nf_ct_zone_id(zone, IP_CT_DIR_REPLY);
1822                 if (zone_id != rid) {
1823                         u32 tmp = hash_conntrack_raw(&tuple, rid, state->net);
1824
1825                         h = __nf_conntrack_find_get(state->net, zone, &tuple, tmp);
1826                 }
1827         }
1828
1829         if (!h) {
1830                 h = init_conntrack(state->net, tmpl, &tuple,
1831                                    skb, dataoff, hash);
1832                 if (!h)
1833                         return 0;
1834                 if (IS_ERR(h))
1835                         return PTR_ERR(h);
1836         }
1837         ct = nf_ct_tuplehash_to_ctrack(h);
1838
1839         /* It exists; we have (non-exclusive) reference. */
1840         if (NF_CT_DIRECTION(h) == IP_CT_DIR_REPLY) {
1841                 ctinfo = IP_CT_ESTABLISHED_REPLY;
1842         } else {
1843                 /* Once we've had two way comms, always ESTABLISHED. */
1844                 if (test_bit(IPS_SEEN_REPLY_BIT, &ct->status)) {
1845                         pr_debug("normal packet for %p\n", ct);
1846                         ctinfo = IP_CT_ESTABLISHED;
1847                 } else if (test_bit(IPS_EXPECTED_BIT, &ct->status)) {
1848                         pr_debug("related packet for %p\n", ct);
1849                         ctinfo = IP_CT_RELATED;
1850                 } else {
1851                         pr_debug("new packet for %p\n", ct);
1852                         ctinfo = IP_CT_NEW;
1853                 }
1854         }
1855         nf_ct_set(skb, ct, ctinfo);
1856         return 0;
1857 }
1858
1859 /*
1860  * icmp packets need special treatment to handle error messages that are
1861  * related to a connection.
1862  *
1863  * Callers need to check if skb has a conntrack assigned when this
1864  * helper returns; in such case skb belongs to an already known connection.
1865  */
1866 static unsigned int __cold
1867 nf_conntrack_handle_icmp(struct nf_conn *tmpl,
1868                          struct sk_buff *skb,
1869                          unsigned int dataoff,
1870                          u8 protonum,
1871                          const struct nf_hook_state *state)
1872 {
1873         int ret;
1874
1875         if (state->pf == NFPROTO_IPV4 && protonum == IPPROTO_ICMP)
1876                 ret = nf_conntrack_icmpv4_error(tmpl, skb, dataoff, state);
1877 #if IS_ENABLED(CONFIG_IPV6)
1878         else if (state->pf == NFPROTO_IPV6 && protonum == IPPROTO_ICMPV6)
1879                 ret = nf_conntrack_icmpv6_error(tmpl, skb, dataoff, state);
1880 #endif
1881         else
1882                 return NF_ACCEPT;
1883
1884         if (ret <= 0)
1885                 NF_CT_STAT_INC_ATOMIC(state->net, error);
1886
1887         return ret;
1888 }
1889
1890 static int generic_packet(struct nf_conn *ct, struct sk_buff *skb,
1891                           enum ip_conntrack_info ctinfo)
1892 {
1893         const unsigned int *timeout = nf_ct_timeout_lookup(ct);
1894
1895         if (!timeout)
1896                 timeout = &nf_generic_pernet(nf_ct_net(ct))->timeout;
1897
1898         nf_ct_refresh_acct(ct, ctinfo, skb, *timeout);
1899         return NF_ACCEPT;
1900 }
1901
1902 /* Returns verdict for packet, or -1 for invalid. */
1903 static int nf_conntrack_handle_packet(struct nf_conn *ct,
1904                                       struct sk_buff *skb,
1905                                       unsigned int dataoff,
1906                                       enum ip_conntrack_info ctinfo,
1907                                       const struct nf_hook_state *state)
1908 {
1909         switch (nf_ct_protonum(ct)) {
1910         case IPPROTO_TCP:
1911                 return nf_conntrack_tcp_packet(ct, skb, dataoff,
1912                                                ctinfo, state);
1913         case IPPROTO_UDP:
1914                 return nf_conntrack_udp_packet(ct, skb, dataoff,
1915                                                ctinfo, state);
1916         case IPPROTO_ICMP:
1917                 return nf_conntrack_icmp_packet(ct, skb, ctinfo, state);
1918 #if IS_ENABLED(CONFIG_IPV6)
1919         case IPPROTO_ICMPV6:
1920                 return nf_conntrack_icmpv6_packet(ct, skb, ctinfo, state);
1921 #endif
1922 #ifdef CONFIG_NF_CT_PROTO_UDPLITE
1923         case IPPROTO_UDPLITE:
1924                 return nf_conntrack_udplite_packet(ct, skb, dataoff,
1925                                                    ctinfo, state);
1926 #endif
1927 #ifdef CONFIG_NF_CT_PROTO_SCTP
1928         case IPPROTO_SCTP:
1929                 return nf_conntrack_sctp_packet(ct, skb, dataoff,
1930                                                 ctinfo, state);
1931 #endif
1932 #ifdef CONFIG_NF_CT_PROTO_DCCP
1933         case IPPROTO_DCCP:
1934                 return nf_conntrack_dccp_packet(ct, skb, dataoff,
1935                                                 ctinfo, state);
1936 #endif
1937 #ifdef CONFIG_NF_CT_PROTO_GRE
1938         case IPPROTO_GRE:
1939                 return nf_conntrack_gre_packet(ct, skb, dataoff,
1940                                                ctinfo, state);
1941 #endif
1942         }
1943
1944         return generic_packet(ct, skb, ctinfo);
1945 }
1946
1947 unsigned int
1948 nf_conntrack_in(struct sk_buff *skb, const struct nf_hook_state *state)
1949 {
1950         enum ip_conntrack_info ctinfo;
1951         struct nf_conn *ct, *tmpl;
1952         u_int8_t protonum;
1953         int dataoff, ret;
1954
1955         tmpl = nf_ct_get(skb, &ctinfo);
1956         if (tmpl || ctinfo == IP_CT_UNTRACKED) {
1957                 /* Previously seen (loopback or untracked)?  Ignore. */
1958                 if ((tmpl && !nf_ct_is_template(tmpl)) ||
1959                      ctinfo == IP_CT_UNTRACKED)
1960                         return NF_ACCEPT;
1961                 skb->_nfct = 0;
1962         }
1963
1964         /* rcu_read_lock()ed by nf_hook_thresh */
1965         dataoff = get_l4proto(skb, skb_network_offset(skb), state->pf, &protonum);
1966         if (dataoff <= 0) {
1967                 pr_debug("not prepared to track yet or error occurred\n");
1968                 NF_CT_STAT_INC_ATOMIC(state->net, invalid);
1969                 ret = NF_ACCEPT;
1970                 goto out;
1971         }
1972
1973         if (protonum == IPPROTO_ICMP || protonum == IPPROTO_ICMPV6) {
1974                 ret = nf_conntrack_handle_icmp(tmpl, skb, dataoff,
1975                                                protonum, state);
1976                 if (ret <= 0) {
1977                         ret = -ret;
1978                         goto out;
1979                 }
1980                 /* ICMP[v6] protocol trackers may assign one conntrack. */
1981                 if (skb->_nfct)
1982                         goto out;
1983         }
1984 repeat:
1985         ret = resolve_normal_ct(tmpl, skb, dataoff,
1986                                 protonum, state);
1987         if (ret < 0) {
1988                 /* Too stressed to deal. */
1989                 NF_CT_STAT_INC_ATOMIC(state->net, drop);
1990                 ret = NF_DROP;
1991                 goto out;
1992         }
1993
1994         ct = nf_ct_get(skb, &ctinfo);
1995         if (!ct) {
1996                 /* Not valid part of a connection */
1997                 NF_CT_STAT_INC_ATOMIC(state->net, invalid);
1998                 ret = NF_ACCEPT;
1999                 goto out;
2000         }
2001
2002         ret = nf_conntrack_handle_packet(ct, skb, dataoff, ctinfo, state);
2003         if (ret <= 0) {
2004                 /* Invalid: inverse of the return code tells
2005                  * the netfilter core what to do */
2006                 pr_debug("nf_conntrack_in: Can't track with proto module\n");
2007                 nf_ct_put(ct);
2008                 skb->_nfct = 0;
2009                 /* Special case: TCP tracker reports an attempt to reopen a
2010                  * closed/aborted connection. We have to go back and create a
2011                  * fresh conntrack.
2012                  */
2013                 if (ret == -NF_REPEAT)
2014                         goto repeat;
2015
2016                 NF_CT_STAT_INC_ATOMIC(state->net, invalid);
2017                 if (ret == -NF_DROP)
2018                         NF_CT_STAT_INC_ATOMIC(state->net, drop);
2019
2020                 ret = -ret;
2021                 goto out;
2022         }
2023
2024         if (ctinfo == IP_CT_ESTABLISHED_REPLY &&
2025             !test_and_set_bit(IPS_SEEN_REPLY_BIT, &ct->status))
2026                 nf_conntrack_event_cache(IPCT_REPLY, ct);
2027 out:
2028         if (tmpl)
2029                 nf_ct_put(tmpl);
2030
2031         return ret;
2032 }
2033 EXPORT_SYMBOL_GPL(nf_conntrack_in);
2034
2035 /* Alter reply tuple (maybe alter helper).  This is for NAT, and is
2036    implicitly racy: see __nf_conntrack_confirm */
2037 void nf_conntrack_alter_reply(struct nf_conn *ct,
2038                               const struct nf_conntrack_tuple *newreply)
2039 {
2040         struct nf_conn_help *help = nfct_help(ct);
2041
2042         /* Should be unconfirmed, so not in hash table yet */
2043         WARN_ON(nf_ct_is_confirmed(ct));
2044
2045         pr_debug("Altering reply tuple of %p to ", ct);
2046         nf_ct_dump_tuple(newreply);
2047
2048         ct->tuplehash[IP_CT_DIR_REPLY].tuple = *newreply;
2049         if (ct->master || (help && !hlist_empty(&help->expectations)))
2050                 return;
2051
2052         rcu_read_lock();
2053         __nf_ct_try_assign_helper(ct, NULL, GFP_ATOMIC);
2054         rcu_read_unlock();
2055 }
2056 EXPORT_SYMBOL_GPL(nf_conntrack_alter_reply);
2057
2058 /* Refresh conntrack for this many jiffies and do accounting if do_acct is 1 */
2059 void __nf_ct_refresh_acct(struct nf_conn *ct,
2060                           enum ip_conntrack_info ctinfo,
2061                           const struct sk_buff *skb,
2062                           u32 extra_jiffies,
2063                           bool do_acct)
2064 {
2065         /* Only update if this is not a fixed timeout */
2066         if (test_bit(IPS_FIXED_TIMEOUT_BIT, &ct->status))
2067                 goto acct;
2068
2069         /* If not in hash table, timer will not be active yet */
2070         if (nf_ct_is_confirmed(ct))
2071                 extra_jiffies += nfct_time_stamp;
2072
2073         if (READ_ONCE(ct->timeout) != extra_jiffies)
2074                 WRITE_ONCE(ct->timeout, extra_jiffies);
2075 acct:
2076         if (do_acct)
2077                 nf_ct_acct_update(ct, CTINFO2DIR(ctinfo), skb->len);
2078 }
2079 EXPORT_SYMBOL_GPL(__nf_ct_refresh_acct);
2080
2081 bool nf_ct_kill_acct(struct nf_conn *ct,
2082                      enum ip_conntrack_info ctinfo,
2083                      const struct sk_buff *skb)
2084 {
2085         nf_ct_acct_update(ct, CTINFO2DIR(ctinfo), skb->len);
2086
2087         return nf_ct_delete(ct, 0, 0);
2088 }
2089 EXPORT_SYMBOL_GPL(nf_ct_kill_acct);
2090
2091 #if IS_ENABLED(CONFIG_NF_CT_NETLINK)
2092
2093 #include <linux/netfilter/nfnetlink.h>
2094 #include <linux/netfilter/nfnetlink_conntrack.h>
2095 #include <linux/mutex.h>
2096
2097 /* Generic function for tcp/udp/sctp/dccp and alike. */
2098 int nf_ct_port_tuple_to_nlattr(struct sk_buff *skb,
2099                                const struct nf_conntrack_tuple *tuple)
2100 {
2101         if (nla_put_be16(skb, CTA_PROTO_SRC_PORT, tuple->src.u.tcp.port) ||
2102             nla_put_be16(skb, CTA_PROTO_DST_PORT, tuple->dst.u.tcp.port))
2103                 goto nla_put_failure;
2104         return 0;
2105
2106 nla_put_failure:
2107         return -1;
2108 }
2109 EXPORT_SYMBOL_GPL(nf_ct_port_tuple_to_nlattr);
2110
2111 const struct nla_policy nf_ct_port_nla_policy[CTA_PROTO_MAX+1] = {
2112         [CTA_PROTO_SRC_PORT]  = { .type = NLA_U16 },
2113         [CTA_PROTO_DST_PORT]  = { .type = NLA_U16 },
2114 };
2115 EXPORT_SYMBOL_GPL(nf_ct_port_nla_policy);
2116
2117 int nf_ct_port_nlattr_to_tuple(struct nlattr *tb[],
2118                                struct nf_conntrack_tuple *t,
2119                                u_int32_t flags)
2120 {
2121         if (flags & CTA_FILTER_FLAG(CTA_PROTO_SRC_PORT)) {
2122                 if (!tb[CTA_PROTO_SRC_PORT])
2123                         return -EINVAL;
2124
2125                 t->src.u.tcp.port = nla_get_be16(tb[CTA_PROTO_SRC_PORT]);
2126         }
2127
2128         if (flags & CTA_FILTER_FLAG(CTA_PROTO_DST_PORT)) {
2129                 if (!tb[CTA_PROTO_DST_PORT])
2130                         return -EINVAL;
2131
2132                 t->dst.u.tcp.port = nla_get_be16(tb[CTA_PROTO_DST_PORT]);
2133         }
2134
2135         return 0;
2136 }
2137 EXPORT_SYMBOL_GPL(nf_ct_port_nlattr_to_tuple);
2138
2139 unsigned int nf_ct_port_nlattr_tuple_size(void)
2140 {
2141         static unsigned int size __read_mostly;
2142
2143         if (!size)
2144                 size = nla_policy_len(nf_ct_port_nla_policy, CTA_PROTO_MAX + 1);
2145
2146         return size;
2147 }
2148 EXPORT_SYMBOL_GPL(nf_ct_port_nlattr_tuple_size);
2149 #endif
2150
2151 /* Used by ipt_REJECT and ip6t_REJECT. */
2152 static void nf_conntrack_attach(struct sk_buff *nskb, const struct sk_buff *skb)
2153 {
2154         struct nf_conn *ct;
2155         enum ip_conntrack_info ctinfo;
2156
2157         /* This ICMP is in reverse direction to the packet which caused it */
2158         ct = nf_ct_get(skb, &ctinfo);
2159         if (CTINFO2DIR(ctinfo) == IP_CT_DIR_ORIGINAL)
2160                 ctinfo = IP_CT_RELATED_REPLY;
2161         else
2162                 ctinfo = IP_CT_RELATED;
2163
2164         /* Attach to new skbuff, and increment count */
2165         nf_ct_set(nskb, ct, ctinfo);
2166         nf_conntrack_get(skb_nfct(nskb));
2167 }
2168
2169 static int __nf_conntrack_update(struct net *net, struct sk_buff *skb,
2170                                  struct nf_conn *ct,
2171                                  enum ip_conntrack_info ctinfo)
2172 {
2173         const struct nf_nat_hook *nat_hook;
2174         struct nf_conntrack_tuple_hash *h;
2175         struct nf_conntrack_tuple tuple;
2176         unsigned int status;
2177         int dataoff;
2178         u16 l3num;
2179         u8 l4num;
2180
2181         l3num = nf_ct_l3num(ct);
2182
2183         dataoff = get_l4proto(skb, skb_network_offset(skb), l3num, &l4num);
2184         if (dataoff <= 0)
2185                 return -1;
2186
2187         if (!nf_ct_get_tuple(skb, skb_network_offset(skb), dataoff, l3num,
2188                              l4num, net, &tuple))
2189                 return -1;
2190
2191         if (ct->status & IPS_SRC_NAT) {
2192                 memcpy(tuple.src.u3.all,
2193                        ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple.src.u3.all,
2194                        sizeof(tuple.src.u3.all));
2195                 tuple.src.u.all =
2196                         ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple.src.u.all;
2197         }
2198
2199         if (ct->status & IPS_DST_NAT) {
2200                 memcpy(tuple.dst.u3.all,
2201                        ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple.dst.u3.all,
2202                        sizeof(tuple.dst.u3.all));
2203                 tuple.dst.u.all =
2204                         ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple.dst.u.all;
2205         }
2206
2207         h = nf_conntrack_find_get(net, nf_ct_zone(ct), &tuple);
2208         if (!h)
2209                 return 0;
2210
2211         /* Store status bits of the conntrack that is clashing to re-do NAT
2212          * mangling according to what it has been done already to this packet.
2213          */
2214         status = ct->status;
2215
2216         nf_ct_put(ct);
2217         ct = nf_ct_tuplehash_to_ctrack(h);
2218         nf_ct_set(skb, ct, ctinfo);
2219
2220         nat_hook = rcu_dereference(nf_nat_hook);
2221         if (!nat_hook)
2222                 return 0;
2223
2224         if (status & IPS_SRC_NAT &&
2225             nat_hook->manip_pkt(skb, ct, NF_NAT_MANIP_SRC,
2226                                 IP_CT_DIR_ORIGINAL) == NF_DROP)
2227                 return -1;
2228
2229         if (status & IPS_DST_NAT &&
2230             nat_hook->manip_pkt(skb, ct, NF_NAT_MANIP_DST,
2231                                 IP_CT_DIR_ORIGINAL) == NF_DROP)
2232                 return -1;
2233
2234         return 0;
2235 }
2236
2237 /* This packet is coming from userspace via nf_queue, complete the packet
2238  * processing after the helper invocation in nf_confirm().
2239  */
2240 static int nf_confirm_cthelper(struct sk_buff *skb, struct nf_conn *ct,
2241                                enum ip_conntrack_info ctinfo)
2242 {
2243         const struct nf_conntrack_helper *helper;
2244         const struct nf_conn_help *help;
2245         int protoff;
2246
2247         help = nfct_help(ct);
2248         if (!help)
2249                 return 0;
2250
2251         helper = rcu_dereference(help->helper);
2252         if (!(helper->flags & NF_CT_HELPER_F_USERSPACE))
2253                 return 0;
2254
2255         switch (nf_ct_l3num(ct)) {
2256         case NFPROTO_IPV4:
2257                 protoff = skb_network_offset(skb) + ip_hdrlen(skb);
2258                 break;
2259 #if IS_ENABLED(CONFIG_IPV6)
2260         case NFPROTO_IPV6: {
2261                 __be16 frag_off;
2262                 u8 pnum;
2263
2264                 pnum = ipv6_hdr(skb)->nexthdr;
2265                 protoff = ipv6_skip_exthdr(skb, sizeof(struct ipv6hdr), &pnum,
2266                                            &frag_off);
2267                 if (protoff < 0 || (frag_off & htons(~0x7)) != 0)
2268                         return 0;
2269                 break;
2270         }
2271 #endif
2272         default:
2273                 return 0;
2274         }
2275
2276         if (test_bit(IPS_SEQ_ADJUST_BIT, &ct->status) &&
2277             !nf_is_loopback_packet(skb)) {
2278                 if (!nf_ct_seq_adjust(skb, ct, ctinfo, protoff)) {
2279                         NF_CT_STAT_INC_ATOMIC(nf_ct_net(ct), drop);
2280                         return -1;
2281                 }
2282         }
2283
2284         /* We've seen it coming out the other side: confirm it */
2285         return nf_conntrack_confirm(skb) == NF_DROP ? - 1 : 0;
2286 }
2287
2288 static int nf_conntrack_update(struct net *net, struct sk_buff *skb)
2289 {
2290         enum ip_conntrack_info ctinfo;
2291         struct nf_conn *ct;
2292         int err;
2293
2294         ct = nf_ct_get(skb, &ctinfo);
2295         if (!ct)
2296                 return 0;
2297
2298         if (!nf_ct_is_confirmed(ct)) {
2299                 err = __nf_conntrack_update(net, skb, ct, ctinfo);
2300                 if (err < 0)
2301                         return err;
2302
2303                 ct = nf_ct_get(skb, &ctinfo);
2304         }
2305
2306         return nf_confirm_cthelper(skb, ct, ctinfo);
2307 }
2308
2309 static bool nf_conntrack_get_tuple_skb(struct nf_conntrack_tuple *dst_tuple,
2310                                        const struct sk_buff *skb)
2311 {
2312         const struct nf_conntrack_tuple *src_tuple;
2313         const struct nf_conntrack_tuple_hash *hash;
2314         struct nf_conntrack_tuple srctuple;
2315         enum ip_conntrack_info ctinfo;
2316         struct nf_conn *ct;
2317
2318         ct = nf_ct_get(skb, &ctinfo);
2319         if (ct) {
2320                 src_tuple = nf_ct_tuple(ct, CTINFO2DIR(ctinfo));
2321                 memcpy(dst_tuple, src_tuple, sizeof(*dst_tuple));
2322                 return true;
2323         }
2324
2325         if (!nf_ct_get_tuplepr(skb, skb_network_offset(skb),
2326                                NFPROTO_IPV4, dev_net(skb->dev),
2327                                &srctuple))
2328                 return false;
2329
2330         hash = nf_conntrack_find_get(dev_net(skb->dev),
2331                                      &nf_ct_zone_dflt,
2332                                      &srctuple);
2333         if (!hash)
2334                 return false;
2335
2336         ct = nf_ct_tuplehash_to_ctrack(hash);
2337         src_tuple = nf_ct_tuple(ct, !hash->tuple.dst.dir);
2338         memcpy(dst_tuple, src_tuple, sizeof(*dst_tuple));
2339         nf_ct_put(ct);
2340
2341         return true;
2342 }
2343
2344 /* Bring out ya dead! */
2345 static struct nf_conn *
2346 get_next_corpse(int (*iter)(struct nf_conn *i, void *data),
2347                 const struct nf_ct_iter_data *iter_data, unsigned int *bucket)
2348 {
2349         struct nf_conntrack_tuple_hash *h;
2350         struct nf_conn *ct;
2351         struct hlist_nulls_node *n;
2352         spinlock_t *lockp;
2353
2354         for (; *bucket < nf_conntrack_htable_size; (*bucket)++) {
2355                 struct hlist_nulls_head *hslot = &nf_conntrack_hash[*bucket];
2356
2357                 if (hlist_nulls_empty(hslot))
2358                         continue;
2359
2360                 lockp = &nf_conntrack_locks[*bucket % CONNTRACK_LOCKS];
2361                 local_bh_disable();
2362                 nf_conntrack_lock(lockp);
2363                 hlist_nulls_for_each_entry(h, n, hslot, hnnode) {
2364                         if (NF_CT_DIRECTION(h) != IP_CT_DIR_REPLY)
2365                                 continue;
2366                         /* All nf_conn objects are added to hash table twice, one
2367                          * for original direction tuple, once for the reply tuple.
2368                          *
2369                          * Exception: In the IPS_NAT_CLASH case, only the reply
2370                          * tuple is added (the original tuple already existed for
2371                          * a different object).
2372                          *
2373                          * We only need to call the iterator once for each
2374                          * conntrack, so we just use the 'reply' direction
2375                          * tuple while iterating.
2376                          */
2377                         ct = nf_ct_tuplehash_to_ctrack(h);
2378
2379                         if (iter_data->net &&
2380                             !net_eq(iter_data->net, nf_ct_net(ct)))
2381                                 continue;
2382
2383                         if (iter(ct, iter_data->data))
2384                                 goto found;
2385                 }
2386                 spin_unlock(lockp);
2387                 local_bh_enable();
2388                 cond_resched();
2389         }
2390
2391         return NULL;
2392 found:
2393         refcount_inc(&ct->ct_general.use);
2394         spin_unlock(lockp);
2395         local_bh_enable();
2396         return ct;
2397 }
2398
2399 static void nf_ct_iterate_cleanup(int (*iter)(struct nf_conn *i, void *data),
2400                                   const struct nf_ct_iter_data *iter_data)
2401 {
2402         unsigned int bucket = 0;
2403         struct nf_conn *ct;
2404
2405         might_sleep();
2406
2407         mutex_lock(&nf_conntrack_mutex);
2408         while ((ct = get_next_corpse(iter, iter_data, &bucket)) != NULL) {
2409                 /* Time to push up daises... */
2410
2411                 nf_ct_delete(ct, iter_data->portid, iter_data->report);
2412                 nf_ct_put(ct);
2413                 cond_resched();
2414         }
2415         mutex_unlock(&nf_conntrack_mutex);
2416 }
2417
2418 void nf_ct_iterate_cleanup_net(int (*iter)(struct nf_conn *i, void *data),
2419                                const struct nf_ct_iter_data *iter_data)
2420 {
2421         struct net *net = iter_data->net;
2422         struct nf_conntrack_net *cnet = nf_ct_pernet(net);
2423
2424         might_sleep();
2425
2426         if (atomic_read(&cnet->count) == 0)
2427                 return;
2428
2429         nf_ct_iterate_cleanup(iter, iter_data);
2430 }
2431 EXPORT_SYMBOL_GPL(nf_ct_iterate_cleanup_net);
2432
2433 /**
2434  * nf_ct_iterate_destroy - destroy unconfirmed conntracks and iterate table
2435  * @iter: callback to invoke for each conntrack
2436  * @data: data to pass to @iter
2437  *
2438  * Like nf_ct_iterate_cleanup, but first marks conntracks on the
2439  * unconfirmed list as dying (so they will not be inserted into
2440  * main table).
2441  *
2442  * Can only be called in module exit path.
2443  */
2444 void
2445 nf_ct_iterate_destroy(int (*iter)(struct nf_conn *i, void *data), void *data)
2446 {
2447         struct nf_ct_iter_data iter_data = {};
2448         struct net *net;
2449
2450         down_read(&net_rwsem);
2451         for_each_net(net) {
2452                 struct nf_conntrack_net *cnet = nf_ct_pernet(net);
2453
2454                 if (atomic_read(&cnet->count) == 0)
2455                         continue;
2456                 nf_queue_nf_hook_drop(net);
2457         }
2458         up_read(&net_rwsem);
2459
2460         /* Need to wait for netns cleanup worker to finish, if its
2461          * running -- it might have deleted a net namespace from
2462          * the global list, so hook drop above might not have
2463          * affected all namespaces.
2464          */
2465         net_ns_barrier();
2466
2467         /* a skb w. unconfirmed conntrack could have been reinjected just
2468          * before we called nf_queue_nf_hook_drop().
2469          *
2470          * This makes sure its inserted into conntrack table.
2471          */
2472         synchronize_net();
2473
2474         nf_ct_ext_bump_genid();
2475         iter_data.data = data;
2476         nf_ct_iterate_cleanup(iter, &iter_data);
2477
2478         /* Another cpu might be in a rcu read section with
2479          * rcu protected pointer cleared in iter callback
2480          * or hidden via nf_ct_ext_bump_genid() above.
2481          *
2482          * Wait until those are done.
2483          */
2484         synchronize_rcu();
2485 }
2486 EXPORT_SYMBOL_GPL(nf_ct_iterate_destroy);
2487
2488 static int kill_all(struct nf_conn *i, void *data)
2489 {
2490         return 1;
2491 }
2492
2493 void nf_conntrack_cleanup_start(void)
2494 {
2495         conntrack_gc_work.exiting = true;
2496 }
2497
2498 void nf_conntrack_cleanup_end(void)
2499 {
2500         RCU_INIT_POINTER(nf_ct_hook, NULL);
2501         cancel_delayed_work_sync(&conntrack_gc_work.dwork);
2502         kvfree(nf_conntrack_hash);
2503
2504         nf_conntrack_proto_fini();
2505         nf_conntrack_helper_fini();
2506         nf_conntrack_expect_fini();
2507
2508         kmem_cache_destroy(nf_conntrack_cachep);
2509 }
2510
2511 /*
2512  * Mishearing the voices in his head, our hero wonders how he's
2513  * supposed to kill the mall.
2514  */
2515 void nf_conntrack_cleanup_net(struct net *net)
2516 {
2517         LIST_HEAD(single);
2518
2519         list_add(&net->exit_list, &single);
2520         nf_conntrack_cleanup_net_list(&single);
2521 }
2522
2523 void nf_conntrack_cleanup_net_list(struct list_head *net_exit_list)
2524 {
2525         struct nf_ct_iter_data iter_data = {};
2526         struct net *net;
2527         int busy;
2528
2529         /*
2530          * This makes sure all current packets have passed through
2531          *  netfilter framework.  Roll on, two-stage module
2532          *  delete...
2533          */
2534         synchronize_net();
2535 i_see_dead_people:
2536         busy = 0;
2537         list_for_each_entry(net, net_exit_list, exit_list) {
2538                 struct nf_conntrack_net *cnet = nf_ct_pernet(net);
2539
2540                 iter_data.net = net;
2541                 nf_ct_iterate_cleanup_net(kill_all, &iter_data);
2542                 if (atomic_read(&cnet->count) != 0)
2543                         busy = 1;
2544         }
2545         if (busy) {
2546                 schedule();
2547                 goto i_see_dead_people;
2548         }
2549
2550         list_for_each_entry(net, net_exit_list, exit_list) {
2551                 nf_conntrack_ecache_pernet_fini(net);
2552                 nf_conntrack_expect_pernet_fini(net);
2553                 free_percpu(net->ct.stat);
2554         }
2555 }
2556
2557 void *nf_ct_alloc_hashtable(unsigned int *sizep, int nulls)
2558 {
2559         struct hlist_nulls_head *hash;
2560         unsigned int nr_slots, i;
2561
2562         if (*sizep > (UINT_MAX / sizeof(struct hlist_nulls_head)))
2563                 return NULL;
2564
2565         BUILD_BUG_ON(sizeof(struct hlist_nulls_head) != sizeof(struct hlist_head));
2566         nr_slots = *sizep = roundup(*sizep, PAGE_SIZE / sizeof(struct hlist_nulls_head));
2567
2568         hash = kvcalloc(nr_slots, sizeof(struct hlist_nulls_head), GFP_KERNEL);
2569
2570         if (hash && nulls)
2571                 for (i = 0; i < nr_slots; i++)
2572                         INIT_HLIST_NULLS_HEAD(&hash[i], i);
2573
2574         return hash;
2575 }
2576 EXPORT_SYMBOL_GPL(nf_ct_alloc_hashtable);
2577
2578 int nf_conntrack_hash_resize(unsigned int hashsize)
2579 {
2580         int i, bucket;
2581         unsigned int old_size;
2582         struct hlist_nulls_head *hash, *old_hash;
2583         struct nf_conntrack_tuple_hash *h;
2584         struct nf_conn *ct;
2585
2586         if (!hashsize)
2587                 return -EINVAL;
2588
2589         hash = nf_ct_alloc_hashtable(&hashsize, 1);
2590         if (!hash)
2591                 return -ENOMEM;
2592
2593         mutex_lock(&nf_conntrack_mutex);
2594         old_size = nf_conntrack_htable_size;
2595         if (old_size == hashsize) {
2596                 mutex_unlock(&nf_conntrack_mutex);
2597                 kvfree(hash);
2598                 return 0;
2599         }
2600
2601         local_bh_disable();
2602         nf_conntrack_all_lock();
2603         write_seqcount_begin(&nf_conntrack_generation);
2604
2605         /* Lookups in the old hash might happen in parallel, which means we
2606          * might get false negatives during connection lookup. New connections
2607          * created because of a false negative won't make it into the hash
2608          * though since that required taking the locks.
2609          */
2610
2611         for (i = 0; i < nf_conntrack_htable_size; i++) {
2612                 while (!hlist_nulls_empty(&nf_conntrack_hash[i])) {
2613                         unsigned int zone_id;
2614
2615                         h = hlist_nulls_entry(nf_conntrack_hash[i].first,
2616                                               struct nf_conntrack_tuple_hash, hnnode);
2617                         ct = nf_ct_tuplehash_to_ctrack(h);
2618                         hlist_nulls_del_rcu(&h->hnnode);
2619
2620                         zone_id = nf_ct_zone_id(nf_ct_zone(ct), NF_CT_DIRECTION(h));
2621                         bucket = __hash_conntrack(nf_ct_net(ct),
2622                                                   &h->tuple, zone_id, hashsize);
2623                         hlist_nulls_add_head_rcu(&h->hnnode, &hash[bucket]);
2624                 }
2625         }
2626         old_hash = nf_conntrack_hash;
2627
2628         nf_conntrack_hash = hash;
2629         nf_conntrack_htable_size = hashsize;
2630
2631         write_seqcount_end(&nf_conntrack_generation);
2632         nf_conntrack_all_unlock();
2633         local_bh_enable();
2634
2635         mutex_unlock(&nf_conntrack_mutex);
2636
2637         synchronize_net();
2638         kvfree(old_hash);
2639         return 0;
2640 }
2641
2642 int nf_conntrack_set_hashsize(const char *val, const struct kernel_param *kp)
2643 {
2644         unsigned int hashsize;
2645         int rc;
2646
2647         if (current->nsproxy->net_ns != &init_net)
2648                 return -EOPNOTSUPP;
2649
2650         /* On boot, we can set this without any fancy locking. */
2651         if (!nf_conntrack_hash)
2652                 return param_set_uint(val, kp);
2653
2654         rc = kstrtouint(val, 0, &hashsize);
2655         if (rc)
2656                 return rc;
2657
2658         return nf_conntrack_hash_resize(hashsize);
2659 }
2660
2661 int nf_conntrack_init_start(void)
2662 {
2663         unsigned long nr_pages = totalram_pages();
2664         int max_factor = 8;
2665         int ret = -ENOMEM;
2666         int i;
2667
2668         seqcount_spinlock_init(&nf_conntrack_generation,
2669                                &nf_conntrack_locks_all_lock);
2670
2671         for (i = 0; i < CONNTRACK_LOCKS; i++)
2672                 spin_lock_init(&nf_conntrack_locks[i]);
2673
2674         if (!nf_conntrack_htable_size) {
2675                 nf_conntrack_htable_size
2676                         = (((nr_pages << PAGE_SHIFT) / 16384)
2677                            / sizeof(struct hlist_head));
2678                 if (BITS_PER_LONG >= 64 &&
2679                     nr_pages > (4 * (1024 * 1024 * 1024 / PAGE_SIZE)))
2680                         nf_conntrack_htable_size = 262144;
2681                 else if (nr_pages > (1024 * 1024 * 1024 / PAGE_SIZE))
2682                         nf_conntrack_htable_size = 65536;
2683
2684                 if (nf_conntrack_htable_size < 1024)
2685                         nf_conntrack_htable_size = 1024;
2686                 /* Use a max. factor of one by default to keep the average
2687                  * hash chain length at 2 entries.  Each entry has to be added
2688                  * twice (once for original direction, once for reply).
2689                  * When a table size is given we use the old value of 8 to
2690                  * avoid implicit reduction of the max entries setting.
2691                  */
2692                 max_factor = 1;
2693         }
2694
2695         nf_conntrack_hash = nf_ct_alloc_hashtable(&nf_conntrack_htable_size, 1);
2696         if (!nf_conntrack_hash)
2697                 return -ENOMEM;
2698
2699         nf_conntrack_max = max_factor * nf_conntrack_htable_size;
2700
2701         nf_conntrack_cachep = kmem_cache_create("nf_conntrack",
2702                                                 sizeof(struct nf_conn),
2703                                                 NFCT_INFOMASK + 1,
2704                                                 SLAB_TYPESAFE_BY_RCU | SLAB_HWCACHE_ALIGN, NULL);
2705         if (!nf_conntrack_cachep)
2706                 goto err_cachep;
2707
2708         ret = nf_conntrack_expect_init();
2709         if (ret < 0)
2710                 goto err_expect;
2711
2712         ret = nf_conntrack_helper_init();
2713         if (ret < 0)
2714                 goto err_helper;
2715
2716         ret = nf_conntrack_proto_init();
2717         if (ret < 0)
2718                 goto err_proto;
2719
2720         conntrack_gc_work_init(&conntrack_gc_work);
2721         queue_delayed_work(system_power_efficient_wq, &conntrack_gc_work.dwork, HZ);
2722
2723         ret = register_nf_conntrack_bpf();
2724         if (ret < 0)
2725                 goto err_kfunc;
2726
2727         return 0;
2728
2729 err_kfunc:
2730         cancel_delayed_work_sync(&conntrack_gc_work.dwork);
2731         nf_conntrack_proto_fini();
2732 err_proto:
2733         nf_conntrack_helper_fini();
2734 err_helper:
2735         nf_conntrack_expect_fini();
2736 err_expect:
2737         kmem_cache_destroy(nf_conntrack_cachep);
2738 err_cachep:
2739         kvfree(nf_conntrack_hash);
2740         return ret;
2741 }
2742
2743 static const struct nf_ct_hook nf_conntrack_hook = {
2744         .update         = nf_conntrack_update,
2745         .destroy        = nf_ct_destroy,
2746         .get_tuple_skb  = nf_conntrack_get_tuple_skb,
2747         .attach         = nf_conntrack_attach,
2748 };
2749
2750 void nf_conntrack_init_end(void)
2751 {
2752         RCU_INIT_POINTER(nf_ct_hook, &nf_conntrack_hook);
2753 }
2754
2755 /*
2756  * We need to use special "null" values, not used in hash table
2757  */
2758 #define UNCONFIRMED_NULLS_VAL   ((1<<30)+0)
2759
2760 int nf_conntrack_init_net(struct net *net)
2761 {
2762         struct nf_conntrack_net *cnet = nf_ct_pernet(net);
2763         int ret = -ENOMEM;
2764
2765         BUILD_BUG_ON(IP_CT_UNTRACKED == IP_CT_NUMBER);
2766         BUILD_BUG_ON_NOT_POWER_OF_2(CONNTRACK_LOCKS);
2767         atomic_set(&cnet->count, 0);
2768
2769         net->ct.stat = alloc_percpu(struct ip_conntrack_stat);
2770         if (!net->ct.stat)
2771                 return ret;
2772
2773         ret = nf_conntrack_expect_pernet_init(net);
2774         if (ret < 0)
2775                 goto err_expect;
2776
2777         nf_conntrack_acct_pernet_init(net);
2778         nf_conntrack_tstamp_pernet_init(net);
2779         nf_conntrack_ecache_pernet_init(net);
2780         nf_conntrack_helper_pernet_init(net);
2781         nf_conntrack_proto_pernet_init(net);
2782
2783         return 0;
2784
2785 err_expect:
2786         free_percpu(net->ct.stat);
2787         return ret;
2788 }