net/ipv4/route.c

   1 /*
   2  * INET         An implementation of the TCP/IP protocol suite for the LINUX
   3  *              operating system.  INET is implemented using the  BSD Socket
   4  *              interface as the means of communication with the user level.
   5  *
   6  *              ROUTE - implementation of the IP router.
   7  *
   8  * Authors:     Ross Biro
   9  *              Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
  10  *              Alan Cox, <gw4pts@gw4pts.ampr.org>
  11  *              Linus Torvalds, <Linus.Torvalds@helsinki.fi>
  12  *              Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
  13  *
  14  * Fixes:
  15  *              Alan Cox        :       Verify area fixes.
  16  *              Alan Cox        :       cli() protects routing changes
  17  *              Rui Oliveira    :       ICMP routing table updates
  18  *              (rco@di.uminho.pt)      Routing table insertion and update
  19  *              Linus Torvalds  :       Rewrote bits to be sensible
  20  *              Alan Cox        :       Added BSD route gw semantics
  21  *              Alan Cox        :       Super /proc >4K
  22  *              Alan Cox        :       MTU in route table
  23  *              Alan Cox        :       MSS actually. Also added the window
  24  *                                      clamper.
  25  *              Sam Lantinga    :       Fixed route matching in rt_del()
  26  *              Alan Cox        :       Routing cache support.
  27  *              Alan Cox        :       Removed compatibility cruft.
  28  *              Alan Cox        :       RTF_REJECT support.
  29  *              Alan Cox        :       TCP irtt support.
  30  *              Jonathan Naylor :       Added Metric support.
  31  *      Miquel van Smoorenburg  :       BSD API fixes.
  32  *      Miquel van Smoorenburg  :       Metrics.
  33  *              Alan Cox        :       Use __u32 properly
  34  *              Alan Cox        :       Aligned routing errors more closely with BSD
  35  *                                      our system is still very different.
  36  *              Alan Cox        :       Faster /proc handling
  37  *      Alexey Kuznetsov        :       Massive rework to support tree based routing,
  38  *                                      routing caches and better behaviour.
  39  *
  40  *              Olaf Erb        :       irtt wasn't being copied right.
  41  *              Bjorn Ekwall    :       Kerneld route support.
  42  *              Alan Cox        :       Multicast fixed (I hope)
  43  *              Pavel Krauz     :       Limited broadcast fixed
  44  *              Mike McLagan    :       Routing by source
  45  *      Alexey Kuznetsov        :       End of old history. Split to fib.c and
  46  *                                      route.c and rewritten from scratch.
  47  *              Andi Kleen      :       Load-limit warning messages.
  48  *      Vitaly E. Lavrov        :       Transparent proxy revived after year coma.
  49  *      Vitaly E. Lavrov        :       Race condition in ip_route_input_slow.
  50  *      Tobias Ringstrom        :       Uninitialized res.type in ip_route_output_slow.
  51  *      Vladimir V. Ivanov      :       IP rule info (flowid) is really useful.
  52  *              Marc Boucher    :       routing by fwmark
  53  *      Robert Olsson           :       Added rt_cache statistics
  54  *      Arnaldo C. Melo         :       Convert proc stuff to seq_file
  55  *      Eric Dumazet            :       hashed spinlocks and rt_check_expire() fixes.
  56  *      Ilia Sotnikov           :       Ignore TOS on PMTUD and Redirect
  57  *      Ilia Sotnikov           :       Removed TOS from hash calculations
  58  *
  59  *              This program is free software; you can redistribute it and/or
  60  *              modify it under the terms of the GNU General Public License
  61  *              as published by the Free Software Foundation; either version
  62  *              2 of the License, or (at your option) any later version.
  63  */
  64
  65 #define pr_fmt(fmt) "IPv4: " fmt
  66
  67 #include <linux/module.h>
  68 #include <linux/uaccess.h>
  69 #include <linux/bitops.h>
  70 #include <linux/types.h>
  71 #include <linux/kernel.h>
  72 #include <linux/mm.h>
  73 #include <linux/string.h>
  74 #include <linux/socket.h>
  75 #include <linux/sockios.h>
  76 #include <linux/errno.h>
  77 #include <linux/in.h>
  78 #include <linux/inet.h>
  79 #include <linux/netdevice.h>
  80 #include <linux/proc_fs.h>
  81 #include <linux/init.h>
  82 #include <linux/skbuff.h>
  83 #include <linux/inetdevice.h>
  84 #include <linux/igmp.h>
  85 #include <linux/pkt_sched.h>
  86 #include <linux/mroute.h>
  87 #include <linux/netfilter_ipv4.h>
  88 #include <linux/random.h>
  89 #include <linux/rcupdate.h>
  90 #include <linux/times.h>
  91 #include <linux/slab.h>
  92 #include <linux/jhash.h>
  93 #include <net/dst.h>
  94 #include <net/dst_metadata.h>
  95 #include <net/net_namespace.h>
  96 #include <net/protocol.h>
  97 #include <net/ip.h>
  98 #include <net/route.h>
  99 #include <net/inetpeer.h>
 100 #include <net/sock.h>
 101 #include <net/ip_fib.h>
 102 #include <net/arp.h>
 103 #include <net/tcp.h>
 104 #include <net/icmp.h>
 105 #include <net/xfrm.h>
 106 #include <net/lwtunnel.h>
 107 #include <net/netevent.h>
 108 #include <net/rtnetlink.h>
 109 #ifdef CONFIG_SYSCTL
 110 #include <linux/sysctl.h>
 111 #endif
 112 #include <net/secure_seq.h>
 113 #include <net/ip_tunnels.h>
 114 #include <net/l3mdev.h>
 115
 116 #include "fib_lookup.h"
 117
 118 #define RT_FL_TOS(oldflp4) \
 119         ((oldflp4)->flowi4_tos & (IPTOS_RT_MASK | RTO_ONLINK))
 120
 121 #define RT_GC_TIMEOUT (300*HZ)
 122
 123 static int ip_rt_max_size;
 124 static int ip_rt_redirect_number __read_mostly  = 9;
 125 static int ip_rt_redirect_load __read_mostly    = HZ / 50;
 126 static int ip_rt_redirect_silence __read_mostly = ((HZ / 50) << (9 + 1));
 127 static int ip_rt_error_cost __read_mostly       = HZ;
 128 static int ip_rt_error_burst __read_mostly      = 5 * HZ;
 129 static int ip_rt_mtu_expires __read_mostly      = 10 * 60 * HZ;
 130 static u32 ip_rt_min_pmtu __read_mostly         = 512 + 20 + 20;
 131 static int ip_rt_min_advmss __read_mostly       = 256;
 132
 133 static int ip_rt_gc_timeout __read_mostly       = RT_GC_TIMEOUT;
 134
 135 /*
 136  *      Interface to generic destination cache.
 137  */
 138
 139 static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie);
 140 static unsigned int      ipv4_default_advmss(const struct dst_entry *dst);
 141 static unsigned int      ipv4_mtu(const struct dst_entry *dst);
 142 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst);
 143 static void              ipv4_link_failure(struct sk_buff *skb);
 144 static void              ip_rt_update_pmtu(struct dst_entry *dst, struct sock *sk,
 145                                            struct sk_buff *skb, u32 mtu);
 146 static void              ip_do_redirect(struct dst_entry *dst, struct sock *sk,
 147                                         struct sk_buff *skb);
 148 static void             ipv4_dst_destroy(struct dst_entry *dst);
 149
 150 static u32 *ipv4_cow_metrics(struct dst_entry *dst, unsigned long old)
 151 {
 152         WARN_ON(1);
 153         return NULL;
 154 }
 155
 156 static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst,
 157                                            struct sk_buff *skb,
 158                                            const void *daddr);
 159 static void ipv4_confirm_neigh(const struct dst_entry *dst, const void *daddr);
 160
 161 static struct dst_ops ipv4_dst_ops = {
 162         .family =               AF_INET,
 163         .check =                ipv4_dst_check,
 164         .default_advmss =       ipv4_default_advmss,
 165         .mtu =                  ipv4_mtu,
 166         .cow_metrics =          ipv4_cow_metrics,
 167         .destroy =              ipv4_dst_destroy,
 168         .negative_advice =      ipv4_negative_advice,
 169         .link_failure =         ipv4_link_failure,
 170         .update_pmtu =          ip_rt_update_pmtu,
 171         .redirect =             ip_do_redirect,
 172         .local_out =            __ip_local_out,
 173         .neigh_lookup =         ipv4_neigh_lookup,
 174         .confirm_neigh =        ipv4_confirm_neigh,
 175 };
 176
 177 #define ECN_OR_COST(class)      TC_PRIO_##class
 178
 179 const __u8 ip_tos2prio[16] = {
 180         TC_PRIO_BESTEFFORT,
 181         ECN_OR_COST(BESTEFFORT),
 182         TC_PRIO_BESTEFFORT,
 183         ECN_OR_COST(BESTEFFORT),
 184         TC_PRIO_BULK,
 185         ECN_OR_COST(BULK),
 186         TC_PRIO_BULK,
 187         ECN_OR_COST(BULK),
 188         TC_PRIO_INTERACTIVE,
 189         ECN_OR_COST(INTERACTIVE),
 190         TC_PRIO_INTERACTIVE,
 191         ECN_OR_COST(INTERACTIVE),
 192         TC_PRIO_INTERACTIVE_BULK,
 193         ECN_OR_COST(INTERACTIVE_BULK),
 194         TC_PRIO_INTERACTIVE_BULK,
 195         ECN_OR_COST(INTERACTIVE_BULK)
 196 };
 197 EXPORT_SYMBOL(ip_tos2prio);
 198
 199 static DEFINE_PER_CPU(struct rt_cache_stat, rt_cache_stat);
 200 #define RT_CACHE_STAT_INC(field) raw_cpu_inc(rt_cache_stat.field)
 201
 202 #ifdef CONFIG_PROC_FS
 203 static void *rt_cache_seq_start(struct seq_file *seq, loff_t *pos)
 204 {
 205         if (*pos)
 206                 return NULL;
 207         return SEQ_START_TOKEN;
 208 }
 209
 210 static void *rt_cache_seq_next(struct seq_file *seq, void *v, loff_t *pos)
 211 {
 212         ++*pos;
 213         return NULL;
 214 }
 215
 216 static void rt_cache_seq_stop(struct seq_file *seq, void *v)
 217 {
 218 }
 219
 220 static int rt_cache_seq_show(struct seq_file *seq, void *v)
 221 {
 222         if (v == SEQ_START_TOKEN)
 223                 seq_printf(seq, "%-127s\n",
 224                            "Iface\tDestination\tGateway \tFlags\t\tRefCnt\tUse\t"
 225                            "Metric\tSource\t\tMTU\tWindow\tIRTT\tTOS\tHHRef\t"
 226                            "HHUptod\tSpecDst");
 227         return 0;
 228 }
 229
 230 static const struct seq_operations rt_cache_seq_ops = {
 231         .start  = rt_cache_seq_start,
 232         .next   = rt_cache_seq_next,
 233         .stop   = rt_cache_seq_stop,
 234         .show   = rt_cache_seq_show,
 235 };
 236
 237 static int rt_cache_seq_open(struct inode *inode, struct file *file)
 238 {
 239         return seq_open(file, &rt_cache_seq_ops);
 240 }
 241
 242 static const struct file_operations rt_cache_seq_fops = {
 243         .open    = rt_cache_seq_open,
 244         .read    = seq_read,
 245         .llseek  = seq_lseek,
 246         .release = seq_release,
 247 };
 248
 249
 250 static void *rt_cpu_seq_start(struct seq_file *seq, loff_t *pos)
 251 {
 252         int cpu;
 253
 254         if (*pos == 0)
 255                 return SEQ_START_TOKEN;
 256
 257         for (cpu = *pos-1; cpu < nr_cpu_ids; ++cpu) {
 258                 if (!cpu_possible(cpu))
 259                         continue;
 260                 *pos = cpu+1;
 261                 return &per_cpu(rt_cache_stat, cpu);
 262         }
 263         return NULL;
 264 }
 265
 266 static void *rt_cpu_seq_next(struct seq_file *seq, void *v, loff_t *pos)
 267 {
 268         int cpu;
 269
 270         for (cpu = *pos; cpu < nr_cpu_ids; ++cpu) {
 271                 if (!cpu_possible(cpu))
 272                         continue;
 273                 *pos = cpu+1;
 274                 return &per_cpu(rt_cache_stat, cpu);
 275         }
 276         return NULL;
 277
 278 }
 279
 280 static void rt_cpu_seq_stop(struct seq_file *seq, void *v)
 281 {
 282
 283 }
 284
 285 static int rt_cpu_seq_show(struct seq_file *seq, void *v)
 286 {
 287         struct rt_cache_stat *st = v;
 288
 289         if (v == SEQ_START_TOKEN) {
 290                 seq_printf(seq, "entries  in_hit in_slow_tot in_slow_mc in_no_route in_brd in_martian_dst in_martian_src  out_hit out_slow_tot out_slow_mc  gc_total gc_ignored gc_goal_miss gc_dst_overflow in_hlist_search out_hlist_search\n");
 291                 return 0;
 292         }
 293
 294         seq_printf(seq,"%08x  %08x %08x %08x %08x %08x %08x %08x "
 295                    " %08x %08x %08x %08x %08x %08x %08x %08x %08x \n",
 296                    dst_entries_get_slow(&ipv4_dst_ops),
 297                    0, /* st->in_hit */
 298                    st->in_slow_tot,
 299                    st->in_slow_mc,
 300                    st->in_no_route,
 301                    st->in_brd,
 302                    st->in_martian_dst,
 303                    st->in_martian_src,
 304
 305                    0, /* st->out_hit */
 306                    st->out_slow_tot,
 307                    st->out_slow_mc,
 308
 309                    0, /* st->gc_total */
 310                    0, /* st->gc_ignored */
 311                    0, /* st->gc_goal_miss */
 312                    0, /* st->gc_dst_overflow */
 313                    0, /* st->in_hlist_search */
 314                    0  /* st->out_hlist_search */
 315                 );
 316         return 0;
 317 }
 318
 319 static const struct seq_operations rt_cpu_seq_ops = {
 320         .start  = rt_cpu_seq_start,
 321         .next   = rt_cpu_seq_next,
 322         .stop   = rt_cpu_seq_stop,
 323         .show   = rt_cpu_seq_show,
 324 };
 325
 326
 327 static int rt_cpu_seq_open(struct inode *inode, struct file *file)
 328 {
 329         return seq_open(file, &rt_cpu_seq_ops);
 330 }
 331
 332 static const struct file_operations rt_cpu_seq_fops = {
 333         .open    = rt_cpu_seq_open,
 334         .read    = seq_read,
 335         .llseek  = seq_lseek,
 336         .release = seq_release,
 337 };
 338
 339 #ifdef CONFIG_IP_ROUTE_CLASSID
 340 static int rt_acct_proc_show(struct seq_file *m, void *v)
 341 {
 342         struct ip_rt_acct *dst, *src;
 343         unsigned int i, j;
 344
 345         dst = kcalloc(256, sizeof(struct ip_rt_acct), GFP_KERNEL);
 346         if (!dst)
 347                 return -ENOMEM;
 348
 349         for_each_possible_cpu(i) {
 350                 src = (struct ip_rt_acct *)per_cpu_ptr(ip_rt_acct, i);
 351                 for (j = 0; j < 256; j++) {
 352                         dst[j].o_bytes   += src[j].o_bytes;
 353                         dst[j].o_packets += src[j].o_packets;
 354                         dst[j].i_bytes   += src[j].i_bytes;
 355                         dst[j].i_packets += src[j].i_packets;
 356                 }
 357         }
 358
 359         seq_write(m, dst, 256 * sizeof(struct ip_rt_acct));
 360         kfree(dst);
 361         return 0;
 362 }
 363 #endif
 364
 365 static int __net_init ip_rt_do_proc_init(struct net *net)
 366 {
 367         struct proc_dir_entry *pde;
 368
 369         pde = proc_create("rt_cache", 0444, net->proc_net,
 370                           &rt_cache_seq_fops);
 371         if (!pde)
 372                 goto err1;
 373
 374         pde = proc_create("rt_cache", 0444,
 375                           net->proc_net_stat, &rt_cpu_seq_fops);
 376         if (!pde)
 377                 goto err2;
 378
 379 #ifdef CONFIG_IP_ROUTE_CLASSID
 380         pde = proc_create_single("rt_acct", 0, net->proc_net,
 381                         rt_acct_proc_show);
 382         if (!pde)
 383                 goto err3;
 384 #endif
 385         return 0;
 386
 387 #ifdef CONFIG_IP_ROUTE_CLASSID
 388 err3:
 389         remove_proc_entry("rt_cache", net->proc_net_stat);
 390 #endif
 391 err2:
 392         remove_proc_entry("rt_cache", net->proc_net);
 393 err1:
 394         return -ENOMEM;
 395 }
 396
 397 static void __net_exit ip_rt_do_proc_exit(struct net *net)
 398 {
 399         remove_proc_entry("rt_cache", net->proc_net_stat);
 400         remove_proc_entry("rt_cache", net->proc_net);
 401 #ifdef CONFIG_IP_ROUTE_CLASSID
 402         remove_proc_entry("rt_acct", net->proc_net);
 403 #endif
 404 }
 405
 406 static struct pernet_operations ip_rt_proc_ops __net_initdata =  {
 407         .init = ip_rt_do_proc_init,
 408         .exit = ip_rt_do_proc_exit,
 409 };
 410
 411 static int __init ip_rt_proc_init(void)
 412 {
 413         return register_pernet_subsys(&ip_rt_proc_ops);
 414 }
 415
 416 #else
 417 static inline int ip_rt_proc_init(void)
 418 {
 419         return 0;
 420 }
 421 #endif /* CONFIG_PROC_FS */
 422
 423 static inline bool rt_is_expired(const struct rtable *rth)
 424 {
 425         return rth->rt_genid != rt_genid_ipv4(dev_net(rth->dst.dev));
 426 }
 427
 428 void rt_cache_flush(struct net *net)
 429 {
 430         rt_genid_bump_ipv4(net);
 431 }
 432
 433 static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst,
 434                                            struct sk_buff *skb,
 435                                            const void *daddr)
 436 {
 437         struct net_device *dev = dst->dev;
 438         const __be32 *pkey = daddr;
 439         const struct rtable *rt;
 440         struct neighbour *n;
 441
 442         rt = (const struct rtable *) dst;
 443         if (rt->rt_gateway)
 444                 pkey = (const __be32 *) &rt->rt_gateway;
 445         else if (skb)
 446                 pkey = &ip_hdr(skb)->daddr;
 447
 448         n = __ipv4_neigh_lookup(dev, *(__force u32 *)pkey);
 449         if (n)
 450                 return n;
 451         return neigh_create(&arp_tbl, pkey, dev);
 452 }
 453
 454 static void ipv4_confirm_neigh(const struct dst_entry *dst, const void *daddr)
 455 {
 456         struct net_device *dev = dst->dev;
 457         const __be32 *pkey = daddr;
 458         const struct rtable *rt;
 459
 460         rt = (const struct rtable *)dst;
 461         if (rt->rt_gateway)
 462                 pkey = (const __be32 *)&rt->rt_gateway;
 463         else if (!daddr ||
 464                  (rt->rt_flags &
 465                   (RTCF_MULTICAST | RTCF_BROADCAST | RTCF_LOCAL)))
 466                 return;
 467
 468         __ipv4_confirm_neigh(dev, *(__force u32 *)pkey);
 469 }
 470
 471 #define IP_IDENTS_SZ 2048u
 472
 473 static atomic_t *ip_idents __read_mostly;
 474 static u32 *ip_tstamps __read_mostly;
 475
 476 /* In order to protect privacy, we add a perturbation to identifiers
 477  * if one generator is seldom used. This makes hard for an attacker
 478  * to infer how many packets were sent between two points in time.
 479  */
 480 u32 ip_idents_reserve(u32 hash, int segs)
 481 {
 482         u32 *p_tstamp = ip_tstamps + hash % IP_IDENTS_SZ;
 483         atomic_t *p_id = ip_idents + hash % IP_IDENTS_SZ;
 484         u32 old = READ_ONCE(*p_tstamp);
 485         u32 now = (u32)jiffies;
 486         u32 new, delta = 0;
 487
 488         if (old != now && cmpxchg(p_tstamp, old, now) == old)
 489                 delta = prandom_u32_max(now - old);
 490
 491         /* Do not use atomic_add_return() as it makes UBSAN unhappy */
 492         do {
 493                 old = (u32)atomic_read(p_id);
 494                 new = old + delta + segs;
 495         } while (atomic_cmpxchg(p_id, old, new) != old);
 496
 497         return new - segs;
 498 }
 499 EXPORT_SYMBOL(ip_idents_reserve);
 500
 501 void __ip_select_ident(struct net *net, struct iphdr *iph, int segs)
 502 {
 503         static u32 ip_idents_hashrnd __read_mostly;
 504         u32 hash, id;
 505
 506         net_get_random_once(&ip_idents_hashrnd, sizeof(ip_idents_hashrnd));
 507
 508         hash = jhash_3words((__force u32)iph->daddr,
 509                             (__force u32)iph->saddr,
 510                             iph->protocol ^ net_hash_mix(net),
 511                             ip_idents_hashrnd);
 512         id = ip_idents_reserve(hash, segs);
 513         iph->id = htons(id);
 514 }
 515 EXPORT_SYMBOL(__ip_select_ident);
 516
 517 static void __build_flow_key(const struct net *net, struct flowi4 *fl4,
 518                              const struct sock *sk,
 519                              const struct iphdr *iph,
 520                              int oif, u8 tos,
 521                              u8 prot, u32 mark, int flow_flags)
 522 {
 523         if (sk) {
 524                 const struct inet_sock *inet = inet_sk(sk);
 525
 526                 oif = sk->sk_bound_dev_if;
 527                 mark = sk->sk_mark;
 528                 tos = RT_CONN_FLAGS(sk);
 529                 prot = inet->hdrincl ? IPPROTO_RAW : sk->sk_protocol;
 530         }
 531         flowi4_init_output(fl4, oif, mark, tos,
 532                            RT_SCOPE_UNIVERSE, prot,
 533                            flow_flags,
 534                            iph->daddr, iph->saddr, 0, 0,
 535                            sock_net_uid(net, sk));
 536 }
 537
 538 static void build_skb_flow_key(struct flowi4 *fl4, const struct sk_buff *skb,
 539                                const struct sock *sk)
 540 {
 541         const struct net *net = dev_net(skb->dev);
 542         const struct iphdr *iph = ip_hdr(skb);
 543         int oif = skb->dev->ifindex;
 544         u8 tos = RT_TOS(iph->tos);
 545         u8 prot = iph->protocol;
 546         u32 mark = skb->mark;
 547
 548         __build_flow_key(net, fl4, sk, iph, oif, tos, prot, mark, 0);
 549 }
 550
 551 static void build_sk_flow_key(struct flowi4 *fl4, const struct sock *sk)
 552 {
 553         const struct inet_sock *inet = inet_sk(sk);
 554         const struct ip_options_rcu *inet_opt;
 555         __be32 daddr = inet->inet_daddr;
 556
 557         rcu_read_lock();
 558         inet_opt = rcu_dereference(inet->inet_opt);
 559         if (inet_opt && inet_opt->opt.srr)
 560                 daddr = inet_opt->opt.faddr;
 561         flowi4_init_output(fl4, sk->sk_bound_dev_if, sk->sk_mark,
 562                            RT_CONN_FLAGS(sk), RT_SCOPE_UNIVERSE,
 563                            inet->hdrincl ? IPPROTO_RAW : sk->sk_protocol,
 564                            inet_sk_flowi_flags(sk),
 565                            daddr, inet->inet_saddr, 0, 0, sk->sk_uid);
 566         rcu_read_unlock();
 567 }
 568
 569 static void ip_rt_build_flow_key(struct flowi4 *fl4, const struct sock *sk,
 570                                  const struct sk_buff *skb)
 571 {
 572         if (skb)
 573                 build_skb_flow_key(fl4, skb, sk);
 574         else
 575                 build_sk_flow_key(fl4, sk);
 576 }
 577
 578 static DEFINE_SPINLOCK(fnhe_lock);
 579
 580 static void fnhe_flush_routes(struct fib_nh_exception *fnhe)
 581 {
 582         struct rtable *rt;
 583
 584         rt = rcu_dereference(fnhe->fnhe_rth_input);
 585         if (rt) {
 586                 RCU_INIT_POINTER(fnhe->fnhe_rth_input, NULL);
 587                 dst_dev_put(&rt->dst);
 588                 dst_release(&rt->dst);
 589         }
 590         rt = rcu_dereference(fnhe->fnhe_rth_output);
 591         if (rt) {
 592                 RCU_INIT_POINTER(fnhe->fnhe_rth_output, NULL);
 593                 dst_dev_put(&rt->dst);
 594                 dst_release(&rt->dst);
 595         }
 596 }
 597
 598 static struct fib_nh_exception *fnhe_oldest(struct fnhe_hash_bucket *hash)
 599 {
 600         struct fib_nh_exception *fnhe, *oldest;
 601
 602         oldest = rcu_dereference(hash->chain);
 603         for (fnhe = rcu_dereference(oldest->fnhe_next); fnhe;
 604              fnhe = rcu_dereference(fnhe->fnhe_next)) {
 605                 if (time_before(fnhe->fnhe_stamp, oldest->fnhe_stamp))
 606                         oldest = fnhe;
 607         }
 608         fnhe_flush_routes(oldest);
 609         return oldest;
 610 }
 611
 612 static inline u32 fnhe_hashfun(__be32 daddr)
 613 {
 614         static u32 fnhe_hashrnd __read_mostly;
 615         u32 hval;
 616
 617         net_get_random_once(&fnhe_hashrnd, sizeof(fnhe_hashrnd));
 618         hval = jhash_1word((__force u32) daddr, fnhe_hashrnd);
 619         return hash_32(hval, FNHE_HASH_SHIFT);
 620 }
 621
 622 static void fill_route_from_fnhe(struct rtable *rt, struct fib_nh_exception *fnhe)
 623 {
 624         rt->rt_pmtu = fnhe->fnhe_pmtu;
 625         rt->rt_mtu_locked = fnhe->fnhe_mtu_locked;
 626         rt->dst.expires = fnhe->fnhe_expires;
 627
 628         if (fnhe->fnhe_gw) {
 629                 rt->rt_flags |= RTCF_REDIRECTED;
 630                 rt->rt_gateway = fnhe->fnhe_gw;
 631                 rt->rt_uses_gateway = 1;
 632         }
 633 }
 634
 635 static void update_or_create_fnhe(struct fib_nh *nh, __be32 daddr, __be32 gw,
 636                                   u32 pmtu, bool lock, unsigned long expires)
 637 {
 638         struct fnhe_hash_bucket *hash;
 639         struct fib_nh_exception *fnhe;
 640         struct rtable *rt;
 641         u32 genid, hval;
 642         unsigned int i;
 643         int depth;
 644
 645         genid = fnhe_genid(dev_net(nh->nh_dev));
 646         hval = fnhe_hashfun(daddr);
 647
 648         spin_lock_bh(&fnhe_lock);
 649
 650         hash = rcu_dereference(nh->nh_exceptions);
 651         if (!hash) {
 652                 hash = kcalloc(FNHE_HASH_SIZE, sizeof(*hash), GFP_ATOMIC);
 653                 if (!hash)
 654                         goto out_unlock;
 655                 rcu_assign_pointer(nh->nh_exceptions, hash);
 656         }
 657
 658         hash += hval;
 659
 660         depth = 0;
 661         for (fnhe = rcu_dereference(hash->chain); fnhe;
 662              fnhe = rcu_dereference(fnhe->fnhe_next)) {
 663                 if (fnhe->fnhe_daddr == daddr)
 664                         break;
 665                 depth++;
 666         }
 667
 668         if (fnhe) {
 669                 if (fnhe->fnhe_genid != genid)
 670                         fnhe->fnhe_genid = genid;
 671                 if (gw)
 672                         fnhe->fnhe_gw = gw;
 673                 if (pmtu) {
 674                         fnhe->fnhe_pmtu = pmtu;
 675                         fnhe->fnhe_mtu_locked = lock;
 676                 }
 677                 fnhe->fnhe_expires = max(1UL, expires);
 678                 /* Update all cached dsts too */
 679                 rt = rcu_dereference(fnhe->fnhe_rth_input);
 680                 if (rt)
 681                         fill_route_from_fnhe(rt, fnhe);
 682                 rt = rcu_dereference(fnhe->fnhe_rth_output);
 683                 if (rt)
 684                         fill_route_from_fnhe(rt, fnhe);
 685         } else {
 686                 if (depth > FNHE_RECLAIM_DEPTH)
 687                         fnhe = fnhe_oldest(hash);
 688                 else {
 689                         fnhe = kzalloc(sizeof(*fnhe), GFP_ATOMIC);
 690                         if (!fnhe)
 691                                 goto out_unlock;
 692
 693                         fnhe->fnhe_next = hash->chain;
 694                         rcu_assign_pointer(hash->chain, fnhe);
 695                 }
 696                 fnhe->fnhe_genid = genid;
 697                 fnhe->fnhe_daddr = daddr;
 698                 fnhe->fnhe_gw = gw;
 699                 fnhe->fnhe_pmtu = pmtu;
 700                 fnhe->fnhe_mtu_locked = lock;
 701                 fnhe->fnhe_expires = max(1UL, expires);
 702
 703                 /* Exception created; mark the cached routes for the nexthop
 704                  * stale, so anyone caching it rechecks if this exception
 705                  * applies to them.
 706                  */
 707                 rt = rcu_dereference(nh->nh_rth_input);
 708                 if (rt)
 709                         rt->dst.obsolete = DST_OBSOLETE_KILL;
 710
 711                 for_each_possible_cpu(i) {
 712                         struct rtable __rcu **prt;
 713                         prt = per_cpu_ptr(nh->nh_pcpu_rth_output, i);
 714                         rt = rcu_dereference(*prt);
 715                         if (rt)
 716                                 rt->dst.obsolete = DST_OBSOLETE_KILL;
 717                 }
 718         }
 719
 720         fnhe->fnhe_stamp = jiffies;
 721
 722 out_unlock:
 723         spin_unlock_bh(&fnhe_lock);
 724 }
 725
 726 static void __ip_do_redirect(struct rtable *rt, struct sk_buff *skb, struct flowi4 *fl4,
 727                              bool kill_route)
 728 {
 729         __be32 new_gw = icmp_hdr(skb)->un.gateway;
 730         __be32 old_gw = ip_hdr(skb)->saddr;
 731         struct net_device *dev = skb->dev;
 732         struct in_device *in_dev;
 733         struct fib_result res;
 734         struct neighbour *n;
 735         struct net *net;
 736
 737         switch (icmp_hdr(skb)->code & 7) {
 738         case ICMP_REDIR_NET:
 739         case ICMP_REDIR_NETTOS:
 740         case ICMP_REDIR_HOST:
 741         case ICMP_REDIR_HOSTTOS:
 742                 break;
 743
 744         default:
 745                 return;
 746         }
 747
 748         if (rt->rt_gateway != old_gw)
 749                 return;
 750
 751         in_dev = __in_dev_get_rcu(dev);
 752         if (!in_dev)
 753                 return;
 754
 755         net = dev_net(dev);
 756         if (new_gw == old_gw || !IN_DEV_RX_REDIRECTS(in_dev) ||
 757             ipv4_is_multicast(new_gw) || ipv4_is_lbcast(new_gw) ||
 758             ipv4_is_zeronet(new_gw))
 759                 goto reject_redirect;
 760
 761         if (!IN_DEV_SHARED_MEDIA(in_dev)) {
 762                 if (!inet_addr_onlink(in_dev, new_gw, old_gw))
 763                         goto reject_redirect;
 764                 if (IN_DEV_SEC_REDIRECTS(in_dev) && ip_fib_check_default(new_gw, dev))
 765                         goto reject_redirect;
 766         } else {
 767                 if (inet_addr_type(net, new_gw) != RTN_UNICAST)
 768                         goto reject_redirect;
 769         }
 770
 771         n = __ipv4_neigh_lookup(rt->dst.dev, new_gw);
 772         if (!n)
 773                 n = neigh_create(&arp_tbl, &new_gw, rt->dst.dev);
 774         if (!IS_ERR(n)) {
 775                 if (!(n->nud_state & NUD_VALID)) {
 776                         neigh_event_send(n, NULL);
 777                 } else {
 778                         if (fib_lookup(net, fl4, &res, 0) == 0) {
 779                                 struct fib_nh *nh = &FIB_RES_NH(res);
 780
 781                                 update_or_create_fnhe(nh, fl4->daddr, new_gw,
 782                                                 0, false,
 783                                                 jiffies + ip_rt_gc_timeout);
 784                         }
 785                         if (kill_route)
 786                                 rt->dst.obsolete = DST_OBSOLETE_KILL;
 787                         call_netevent_notifiers(NETEVENT_NEIGH_UPDATE, n);
 788                 }
 789                 neigh_release(n);
 790         }
 791         return;
 792
 793 reject_redirect:
 794 #ifdef CONFIG_IP_ROUTE_VERBOSE
 795         if (IN_DEV_LOG_MARTIANS(in_dev)) {
 796                 const struct iphdr *iph = (const struct iphdr *) skb->data;
 797                 __be32 daddr = iph->daddr;
 798                 __be32 saddr = iph->saddr;
 799
 800                 net_info_ratelimited("Redirect from %pI4 on %s about %pI4 ignored\n"
 801                                      "  Advised path = %pI4 -> %pI4\n",
 802                                      &old_gw, dev->name, &new_gw,
 803                                      &saddr, &daddr);
 804         }
 805 #endif
 806         ;
 807 }
 808
 809 static void ip_do_redirect(struct dst_entry *dst, struct sock *sk, struct sk_buff *skb)
 810 {
 811         struct rtable *rt;
 812         struct flowi4 fl4;
 813         const struct iphdr *iph = (const struct iphdr *) skb->data;
 814         struct net *net = dev_net(skb->dev);
 815         int oif = skb->dev->ifindex;
 816         u8 tos = RT_TOS(iph->tos);
 817         u8 prot = iph->protocol;
 818         u32 mark = skb->mark;
 819
 820         rt = (struct rtable *) dst;
 821
 822         __build_flow_key(net, &fl4, sk, iph, oif, tos, prot, mark, 0);
 823         __ip_do_redirect(rt, skb, &fl4, true);
 824 }
 825
 826 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst)
 827 {
 828         struct rtable *rt = (struct rtable *)dst;
 829         struct dst_entry *ret = dst;
 830
 831         if (rt) {
 832                 if (dst->obsolete > 0) {
 833                         ip_rt_put(rt);
 834                         ret = NULL;
 835                 } else if ((rt->rt_flags & RTCF_REDIRECTED) ||
 836                            rt->dst.expires) {
 837                         ip_rt_put(rt);
 838                         ret = NULL;
 839                 }
 840         }
 841         return ret;
 842 }
 843
 844 /*
 845  * Algorithm:
 846  *      1. The first ip_rt_redirect_number redirects are sent
 847  *         with exponential backoff, then we stop sending them at all,
 848  *         assuming that the host ignores our redirects.
 849  *      2. If we did not see packets requiring redirects
 850  *         during ip_rt_redirect_silence, we assume that the host
 851  *         forgot redirected route and start to send redirects again.
 852  *
 853  * This algorithm is much cheaper and more intelligent than dumb load limiting
 854  * in icmp.c.
 855  *
 856  * NOTE. Do not forget to inhibit load limiting for redirects (redundant)
 857  * and "frag. need" (breaks PMTU discovery) in icmp.c.
 858  */
 859
 860 void ip_rt_send_redirect(struct sk_buff *skb)
 861 {
 862         struct rtable *rt = skb_rtable(skb);
 863         struct in_device *in_dev;
 864         struct inet_peer *peer;
 865         struct net *net;
 866         int log_martians;
 867         int vif;
 868
 869         rcu_read_lock();
 870         in_dev = __in_dev_get_rcu(rt->dst.dev);
 871         if (!in_dev || !IN_DEV_TX_REDIRECTS(in_dev)) {
 872                 rcu_read_unlock();
 873                 return;
 874         }
 875         log_martians = IN_DEV_LOG_MARTIANS(in_dev);
 876         vif = l3mdev_master_ifindex_rcu(rt->dst.dev);
 877         rcu_read_unlock();
 878
 879         net = dev_net(rt->dst.dev);
 880         peer = inet_getpeer_v4(net->ipv4.peers, ip_hdr(skb)->saddr, vif, 1);
 881         if (!peer) {
 882                 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST,
 883                           rt_nexthop(rt, ip_hdr(skb)->daddr));
 884                 return;
 885         }
 886
 887         /* No redirected packets during ip_rt_redirect_silence;
 888          * reset the algorithm.
 889          */
 890         if (time_after(jiffies, peer->rate_last + ip_rt_redirect_silence)) {
 891                 peer->rate_tokens = 0;
 892                 peer->n_redirects = 0;
 893         }
 894
 895         /* Too many ignored redirects; do not send anything
 896          * set dst.rate_last to the last seen redirected packet.
 897          */
 898         if (peer->n_redirects >= ip_rt_redirect_number) {
 899                 peer->rate_last = jiffies;
 900                 goto out_put_peer;
 901         }
 902
 903         /* Check for load limit; set rate_last to the latest sent
 904          * redirect.
 905          */
 906         if (peer->rate_tokens == 0 ||
 907             time_after(jiffies,
 908                        (peer->rate_last +
 909                         (ip_rt_redirect_load << peer->rate_tokens)))) {
 910                 __be32 gw = rt_nexthop(rt, ip_hdr(skb)->daddr);
 911
 912                 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST, gw);
 913                 peer->rate_last = jiffies;
 914                 ++peer->rate_tokens;
 915                 ++peer->n_redirects;
 916 #ifdef CONFIG_IP_ROUTE_VERBOSE
 917                 if (log_martians &&
 918                     peer->rate_tokens == ip_rt_redirect_number)
 919                         net_warn_ratelimited("host %pI4/if%d ignores redirects for %pI4 to %pI4\n",
 920                                              &ip_hdr(skb)->saddr, inet_iif(skb),
 921                                              &ip_hdr(skb)->daddr, &gw);
 922 #endif
 923         }
 924 out_put_peer:
 925         inet_putpeer(peer);
 926 }
 927
 928 static int ip_error(struct sk_buff *skb)
 929 {
 930         struct rtable *rt = skb_rtable(skb);
 931         struct net_device *dev = skb->dev;
 932         struct in_device *in_dev;
 933         struct inet_peer *peer;
 934         unsigned long now;
 935         struct net *net;
 936         bool send;
 937         int code;
 938
 939         if (netif_is_l3_master(skb->dev)) {
 940                 dev = __dev_get_by_index(dev_net(skb->dev), IPCB(skb)->iif);
 941                 if (!dev)
 942                         goto out;
 943         }
 944
 945         in_dev = __in_dev_get_rcu(dev);
 946
 947         /* IP on this device is disabled. */
 948         if (!in_dev)
 949                 goto out;
 950
 951         net = dev_net(rt->dst.dev);
 952         if (!IN_DEV_FORWARD(in_dev)) {
 953                 switch (rt->dst.error) {
 954                 case EHOSTUNREACH:
 955                         __IP_INC_STATS(net, IPSTATS_MIB_INADDRERRORS);
 956                         break;
 957
 958                 case ENETUNREACH:
 959                         __IP_INC_STATS(net, IPSTATS_MIB_INNOROUTES);
 960                         break;
 961                 }
 962                 goto out;
 963         }
 964
 965         switch (rt->dst.error) {
 966         case EINVAL:
 967         default:
 968                 goto out;
 969         case EHOSTUNREACH:
 970                 code = ICMP_HOST_UNREACH;
 971                 break;
 972         case ENETUNREACH:
 973                 code = ICMP_NET_UNREACH;
 974                 __IP_INC_STATS(net, IPSTATS_MIB_INNOROUTES);
 975                 break;
 976         case EACCES:
 977                 code = ICMP_PKT_FILTERED;
 978                 break;
 979         }
 980
 981         peer = inet_getpeer_v4(net->ipv4.peers, ip_hdr(skb)->saddr,
 982                                l3mdev_master_ifindex(skb->dev), 1);
 983
 984         send = true;
 985         if (peer) {
 986                 now = jiffies;
 987                 peer->rate_tokens += now - peer->rate_last;
 988                 if (peer->rate_tokens > ip_rt_error_burst)
 989                         peer->rate_tokens = ip_rt_error_burst;
 990                 peer->rate_last = now;
 991                 if (peer->rate_tokens >= ip_rt_error_cost)
 992                         peer->rate_tokens -= ip_rt_error_cost;
 993                 else
 994                         send = false;
 995                 inet_putpeer(peer);
 996         }
 997         if (send)
 998                 icmp_send(skb, ICMP_DEST_UNREACH, code, 0);
 999
1000 out:    kfree_skb(skb);
1001         return 0;
1002 }
1003
1004 static void __ip_rt_update_pmtu(struct rtable *rt, struct flowi4 *fl4, u32 mtu)
1005 {
1006         struct dst_entry *dst = &rt->dst;
1007         u32 old_mtu = ipv4_mtu(dst);
1008         struct fib_result res;
1009         bool lock = false;
1010
1011         if (ip_mtu_locked(dst))
1012                 return;
1013
1014         if (old_mtu < mtu)
1015                 return;
1016
1017         if (mtu < ip_rt_min_pmtu) {
1018                 lock = true;
1019                 mtu = min(old_mtu, ip_rt_min_pmtu);
1020         }
1021
1022         if (rt->rt_pmtu == mtu && !lock &&
1023             time_before(jiffies, dst->expires - ip_rt_mtu_expires / 2))
1024                 return;
1025
1026         rcu_read_lock();
1027         if (fib_lookup(dev_net(dst->dev), fl4, &res, 0) == 0) {
1028                 struct fib_nh *nh = &FIB_RES_NH(res);
1029
1030                 update_or_create_fnhe(nh, fl4->daddr, 0, mtu, lock,
1031                                       jiffies + ip_rt_mtu_expires);
1032         }
1033         rcu_read_unlock();
1034 }
1035
1036 static void ip_rt_update_pmtu(struct dst_entry *dst, struct sock *sk,
1037                               struct sk_buff *skb, u32 mtu)
1038 {
1039         struct rtable *rt = (struct rtable *) dst;
1040         struct flowi4 fl4;
1041
1042         ip_rt_build_flow_key(&fl4, sk, skb);
1043         __ip_rt_update_pmtu(rt, &fl4, mtu);
1044 }
1045
1046 void ipv4_update_pmtu(struct sk_buff *skb, struct net *net, u32 mtu,
1047                       int oif, u8 protocol)
1048 {
1049         const struct iphdr *iph = (const struct iphdr *) skb->data;
1050         struct flowi4 fl4;
1051         struct rtable *rt;
1052         u32 mark = IP4_REPLY_MARK(net, skb->mark);
1053
1054         __build_flow_key(net, &fl4, NULL, iph, oif,
1055                          RT_TOS(iph->tos), protocol, mark, 0);
1056         rt = __ip_route_output_key(net, &fl4);
1057         if (!IS_ERR(rt)) {
1058                 __ip_rt_update_pmtu(rt, &fl4, mtu);
1059                 ip_rt_put(rt);
1060         }
1061 }
1062 EXPORT_SYMBOL_GPL(ipv4_update_pmtu);
1063
1064 static void __ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu)
1065 {
1066         const struct iphdr *iph = (const struct iphdr *) skb->data;
1067         struct flowi4 fl4;
1068         struct rtable *rt;
1069
1070         __build_flow_key(sock_net(sk), &fl4, sk, iph, 0, 0, 0, 0, 0);
1071
1072         if (!fl4.flowi4_mark)
1073                 fl4.flowi4_mark = IP4_REPLY_MARK(sock_net(sk), skb->mark);
1074
1075         rt = __ip_route_output_key(sock_net(sk), &fl4);
1076         if (!IS_ERR(rt)) {
1077                 __ip_rt_update_pmtu(rt, &fl4, mtu);
1078                 ip_rt_put(rt);
1079         }
1080 }
1081
1082 void ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu)
1083 {
1084         const struct iphdr *iph = (const struct iphdr *) skb->data;
1085         struct flowi4 fl4;
1086         struct rtable *rt;
1087         struct dst_entry *odst = NULL;
1088         bool new = false;
1089         struct net *net = sock_net(sk);
1090
1091         bh_lock_sock(sk);
1092
1093         if (!ip_sk_accept_pmtu(sk))
1094                 goto out;
1095
1096         odst = sk_dst_get(sk);
1097
1098         if (sock_owned_by_user(sk) || !odst) {
1099                 __ipv4_sk_update_pmtu(skb, sk, mtu);
1100                 goto out;
1101         }
1102
1103         __build_flow_key(net, &fl4, sk, iph, 0, 0, 0, 0, 0);
1104
1105         rt = (struct rtable *)odst;
1106         if (odst->obsolete && !odst->ops->check(odst, 0)) {
1107                 rt = ip_route_output_flow(sock_net(sk), &fl4, sk);
1108                 if (IS_ERR(rt))
1109                         goto out;
1110
1111                 new = true;
1112         }
1113
1114         __ip_rt_update_pmtu((struct rtable *) xfrm_dst_path(&rt->dst), &fl4, mtu);
1115
1116         if (!dst_check(&rt->dst, 0)) {
1117                 if (new)
1118                         dst_release(&rt->dst);
1119
1120                 rt = ip_route_output_flow(sock_net(sk), &fl4, sk);
1121                 if (IS_ERR(rt))
1122                         goto out;
1123
1124                 new = true;
1125         }
1126
1127         if (new)
1128                 sk_dst_set(sk, &rt->dst);
1129
1130 out:
1131         bh_unlock_sock(sk);
1132         dst_release(odst);
1133 }
1134 EXPORT_SYMBOL_GPL(ipv4_sk_update_pmtu);
1135
1136 void ipv4_redirect(struct sk_buff *skb, struct net *net,
1137                    int oif, u8 protocol)
1138 {
1139         const struct iphdr *iph = (const struct iphdr *) skb->data;
1140         struct flowi4 fl4;
1141         struct rtable *rt;
1142
1143         __build_flow_key(net, &fl4, NULL, iph, oif,
1144                          RT_TOS(iph->tos), protocol, 0, 0);
1145         rt = __ip_route_output_key(net, &fl4);
1146         if (!IS_ERR(rt)) {
1147                 __ip_do_redirect(rt, skb, &fl4, false);
1148                 ip_rt_put(rt);
1149         }
1150 }
1151 EXPORT_SYMBOL_GPL(ipv4_redirect);
1152
1153 void ipv4_sk_redirect(struct sk_buff *skb, struct sock *sk)
1154 {
1155         const struct iphdr *iph = (const struct iphdr *) skb->data;
1156         struct flowi4 fl4;
1157         struct rtable *rt;
1158         struct net *net = sock_net(sk);
1159
1160         __build_flow_key(net, &fl4, sk, iph, 0, 0, 0, 0, 0);
1161         rt = __ip_route_output_key(net, &fl4);
1162         if (!IS_ERR(rt)) {
1163                 __ip_do_redirect(rt, skb, &fl4, false);
1164                 ip_rt_put(rt);
1165         }
1166 }
1167 EXPORT_SYMBOL_GPL(ipv4_sk_redirect);
1168
1169 static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie)
1170 {
1171         struct rtable *rt = (struct rtable *) dst;
1172
1173         /* All IPV4 dsts are created with ->obsolete set to the value
1174          * DST_OBSOLETE_FORCE_CHK which forces validation calls down
1175          * into this function always.
1176          *
1177          * When a PMTU/redirect information update invalidates a route,
1178          * this is indicated by setting obsolete to DST_OBSOLETE_KILL or
1179          * DST_OBSOLETE_DEAD by dst_free().
1180          */
1181         if (dst->obsolete != DST_OBSOLETE_FORCE_CHK || rt_is_expired(rt))
1182                 return NULL;
1183         return dst;
1184 }
1185
1186 static void ipv4_link_failure(struct sk_buff *skb)
1187 {
1188         struct rtable *rt;
1189
1190         icmp_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0);
1191
1192         rt = skb_rtable(skb);
1193         if (rt)
1194                 dst_set_expires(&rt->dst, 0);
1195 }
1196
1197 static int ip_rt_bug(struct net *net, struct sock *sk, struct sk_buff *skb)
1198 {
1199         pr_debug("%s: %pI4 -> %pI4, %s\n",
1200                  __func__, &ip_hdr(skb)->saddr, &ip_hdr(skb)->daddr,
1201                  skb->dev ? skb->dev->name : "?");
1202         kfree_skb(skb);
1203         WARN_ON(1);
1204         return 0;
1205 }
1206
1207 /*
1208    We do not cache source address of outgoing interface,
1209    because it is used only by IP RR, TS and SRR options,
1210    so that it out of fast path.
1211
1212    BTW remember: "addr" is allowed to be not aligned
1213    in IP options!
1214  */
1215
1216 void ip_rt_get_source(u8 *addr, struct sk_buff *skb, struct rtable *rt)
1217 {
1218         __be32 src;
1219
1220         if (rt_is_output_route(rt))
1221                 src = ip_hdr(skb)->saddr;
1222         else {
1223                 struct fib_result res;
1224                 struct iphdr *iph = ip_hdr(skb);
1225                 struct flowi4 fl4 = {
1226                         .daddr = iph->daddr,
1227                         .saddr = iph->saddr,
1228                         .flowi4_tos = RT_TOS(iph->tos),
1229                         .flowi4_oif = rt->dst.dev->ifindex,
1230                         .flowi4_iif = skb->dev->ifindex,
1231                         .flowi4_mark = skb->mark,
1232                 };
1233
1234                 rcu_read_lock();
1235                 if (fib_lookup(dev_net(rt->dst.dev), &fl4, &res, 0) == 0)
1236                         src = FIB_RES_PREFSRC(dev_net(rt->dst.dev), res);
1237                 else
1238                         src = inet_select_addr(rt->dst.dev,
1239                                                rt_nexthop(rt, iph->daddr),
1240                                                RT_SCOPE_UNIVERSE);
1241                 rcu_read_unlock();
1242         }
1243         memcpy(addr, &src, 4);
1244 }
1245
1246 #ifdef CONFIG_IP_ROUTE_CLASSID
1247 static void set_class_tag(struct rtable *rt, u32 tag)
1248 {
1249         if (!(rt->dst.tclassid & 0xFFFF))
1250                 rt->dst.tclassid |= tag & 0xFFFF;
1251         if (!(rt->dst.tclassid & 0xFFFF0000))
1252                 rt->dst.tclassid |= tag & 0xFFFF0000;
1253 }
1254 #endif
1255
1256 static unsigned int ipv4_default_advmss(const struct dst_entry *dst)
1257 {
1258         unsigned int header_size = sizeof(struct tcphdr) + sizeof(struct iphdr);
1259         unsigned int advmss = max_t(unsigned int, ipv4_mtu(dst) - header_size,
1260                                     ip_rt_min_advmss);
1261
1262         return min(advmss, IPV4_MAX_PMTU - header_size);
1263 }
1264
1265 static unsigned int ipv4_mtu(const struct dst_entry *dst)
1266 {
1267         const struct rtable *rt = (const struct rtable *) dst;
1268         unsigned int mtu = rt->rt_pmtu;
1269
1270         if (!mtu || time_after_eq(jiffies, rt->dst.expires))
1271                 mtu = dst_metric_raw(dst, RTAX_MTU);
1272
1273         if (mtu)
1274                 return mtu;
1275
1276         mtu = READ_ONCE(dst->dev->mtu);
1277
1278         if (unlikely(ip_mtu_locked(dst))) {
1279                 if (rt->rt_uses_gateway && mtu > 576)
1280                         mtu = 576;
1281         }
1282
1283         mtu = min_t(unsigned int, mtu, IP_MAX_MTU);
1284
1285         return mtu - lwtunnel_headroom(dst->lwtstate, mtu);
1286 }
1287
1288 static void ip_del_fnhe(struct fib_nh *nh, __be32 daddr)
1289 {
1290         struct fnhe_hash_bucket *hash;
1291         struct fib_nh_exception *fnhe, __rcu **fnhe_p;
1292         u32 hval = fnhe_hashfun(daddr);
1293
1294         spin_lock_bh(&fnhe_lock);
1295
1296         hash = rcu_dereference_protected(nh->nh_exceptions,
1297                                          lockdep_is_held(&fnhe_lock));
1298         hash += hval;
1299
1300         fnhe_p = &hash->chain;
1301         fnhe = rcu_dereference_protected(*fnhe_p, lockdep_is_held(&fnhe_lock));
1302         while (fnhe) {
1303                 if (fnhe->fnhe_daddr == daddr) {
1304                         rcu_assign_pointer(*fnhe_p, rcu_dereference_protected(
1305                                 fnhe->fnhe_next, lockdep_is_held(&fnhe_lock)));
1306                         /* set fnhe_daddr to 0 to ensure it won't bind with
1307                          * new dsts in rt_bind_exception().
1308                          */
1309                         fnhe->fnhe_daddr = 0;
1310                         fnhe_flush_routes(fnhe);
1311                         kfree_rcu(fnhe, rcu);
1312                         break;
1313                 }
1314                 fnhe_p = &fnhe->fnhe_next;
1315                 fnhe = rcu_dereference_protected(fnhe->fnhe_next,
1316                                                  lockdep_is_held(&fnhe_lock));
1317         }
1318
1319         spin_unlock_bh(&fnhe_lock);
1320 }
1321
1322 static struct fib_nh_exception *find_exception(struct fib_nh *nh, __be32 daddr)
1323 {
1324         struct fnhe_hash_bucket *hash = rcu_dereference(nh->nh_exceptions);
1325         struct fib_nh_exception *fnhe;
1326         u32 hval;
1327
1328         if (!hash)
1329                 return NULL;
1330
1331         hval = fnhe_hashfun(daddr);
1332
1333         for (fnhe = rcu_dereference(hash[hval].chain); fnhe;
1334              fnhe = rcu_dereference(fnhe->fnhe_next)) {
1335                 if (fnhe->fnhe_daddr == daddr) {
1336                         if (fnhe->fnhe_expires &&
1337                             time_after(jiffies, fnhe->fnhe_expires)) {
1338                                 ip_del_fnhe(nh, daddr);
1339                                 break;
1340                         }
1341                         return fnhe;
1342                 }
1343         }
1344         return NULL;
1345 }
1346
1347 /* MTU selection:
1348  * 1. mtu on route is locked - use it
1349  * 2. mtu from nexthop exception
1350  * 3. mtu from egress device
1351  */
1352
1353 u32 ip_mtu_from_fib_result(struct fib_result *res, __be32 daddr)
1354 {
1355         struct fib_info *fi = res->fi;
1356         struct fib_nh *nh = &fi->fib_nh[res->nh_sel];
1357         struct net_device *dev = nh->nh_dev;
1358         u32 mtu = 0;
1359
1360         if (dev_net(dev)->ipv4.sysctl_ip_fwd_use_pmtu ||
1361             fi->fib_metrics->metrics[RTAX_LOCK - 1] & (1 << RTAX_MTU))
1362                 mtu = fi->fib_mtu;
1363
1364         if (likely(!mtu)) {
1365                 struct fib_nh_exception *fnhe;
1366
1367                 fnhe = find_exception(nh, daddr);
1368                 if (fnhe && !time_after_eq(jiffies, fnhe->fnhe_expires))
1369                         mtu = fnhe->fnhe_pmtu;
1370         }
1371
1372         if (likely(!mtu))
1373                 mtu = min(READ_ONCE(dev->mtu), IP_MAX_MTU);
1374
1375         return mtu - lwtunnel_headroom(nh->nh_lwtstate, mtu);
1376 }
1377
1378 static bool rt_bind_exception(struct rtable *rt, struct fib_nh_exception *fnhe,
1379                               __be32 daddr, const bool do_cache)
1380 {
1381         bool ret = false;
1382
1383         spin_lock_bh(&fnhe_lock);
1384
1385         if (daddr == fnhe->fnhe_daddr) {
1386                 struct rtable __rcu **porig;
1387                 struct rtable *orig;
1388                 int genid = fnhe_genid(dev_net(rt->dst.dev));
1389
1390                 if (rt_is_input_route(rt))
1391                         porig = &fnhe->fnhe_rth_input;
1392                 else
1393                         porig = &fnhe->fnhe_rth_output;
1394                 orig = rcu_dereference(*porig);
1395
1396                 if (fnhe->fnhe_genid != genid) {
1397                         fnhe->fnhe_genid = genid;
1398                         fnhe->fnhe_gw = 0;
1399                         fnhe->fnhe_pmtu = 0;
1400                         fnhe->fnhe_expires = 0;
1401                         fnhe->fnhe_mtu_locked = false;
1402                         fnhe_flush_routes(fnhe);
1403                         orig = NULL;
1404                 }
1405                 fill_route_from_fnhe(rt, fnhe);
1406                 if (!rt->rt_gateway)
1407                         rt->rt_gateway = daddr;
1408
1409                 if (do_cache) {
1410                         dst_hold(&rt->dst);
1411                         rcu_assign_pointer(*porig, rt);
1412                         if (orig) {
1413                                 dst_dev_put(&orig->dst);
1414                                 dst_release(&orig->dst);
1415                         }
1416                         ret = true;
1417                 }
1418
1419                 fnhe->fnhe_stamp = jiffies;
1420         }
1421         spin_unlock_bh(&fnhe_lock);
1422
1423         return ret;
1424 }
1425
1426 static bool rt_cache_route(struct fib_nh *nh, struct rtable *rt)
1427 {
1428         struct rtable *orig, *prev, **p;
1429         bool ret = true;
1430
1431         if (rt_is_input_route(rt)) {
1432                 p = (struct rtable **)&nh->nh_rth_input;
1433         } else {
1434                 p = (struct rtable **)raw_cpu_ptr(nh->nh_pcpu_rth_output);
1435         }
1436         orig = *p;
1437
1438         /* hold dst before doing cmpxchg() to avoid race condition
1439          * on this dst
1440          */
1441         dst_hold(&rt->dst);
1442         prev = cmpxchg(p, orig, rt);
1443         if (prev == orig) {
1444                 if (orig) {
1445                         dst_dev_put(&orig->dst);
1446                         dst_release(&orig->dst);
1447                 }
1448         } else {
1449                 dst_release(&rt->dst);
1450                 ret = false;
1451         }
1452
1453         return ret;
1454 }
1455
1456 struct uncached_list {
1457         spinlock_t              lock;
1458         struct list_head        head;
1459 };
1460
1461 static DEFINE_PER_CPU_ALIGNED(struct uncached_list, rt_uncached_list);
1462
1463 void rt_add_uncached_list(struct rtable *rt)
1464 {
1465         struct uncached_list *ul = raw_cpu_ptr(&rt_uncached_list);
1466
1467         rt->rt_uncached_list = ul;
1468
1469         spin_lock_bh(&ul->lock);
1470         list_add_tail(&rt->rt_uncached, &ul->head);
1471         spin_unlock_bh(&ul->lock);
1472 }
1473
1474 void rt_del_uncached_list(struct rtable *rt)
1475 {
1476         if (!list_empty(&rt->rt_uncached)) {
1477                 struct uncached_list *ul = rt->rt_uncached_list;
1478
1479                 spin_lock_bh(&ul->lock);
1480                 list_del(&rt->rt_uncached);
1481                 spin_unlock_bh(&ul->lock);
1482         }
1483 }
1484
1485 static void ipv4_dst_destroy(struct dst_entry *dst)
1486 {
1487         struct rtable *rt = (struct rtable *)dst;
1488
1489         ip_dst_metrics_put(dst);
1490         rt_del_uncached_list(rt);
1491 }
1492
1493 void rt_flush_dev(struct net_device *dev)
1494 {
1495         struct net *net = dev_net(dev);
1496         struct rtable *rt;
1497         int cpu;
1498
1499         for_each_possible_cpu(cpu) {
1500                 struct uncached_list *ul = &per_cpu(rt_uncached_list, cpu);
1501
1502                 spin_lock_bh(&ul->lock);
1503                 list_for_each_entry(rt, &ul->head, rt_uncached) {
1504                         if (rt->dst.dev != dev)
1505                                 continue;
1506                         rt->dst.dev = net->loopback_dev;
1507                         dev_hold(rt->dst.dev);
1508                         dev_put(dev);
1509                 }
1510                 spin_unlock_bh(&ul->lock);
1511         }
1512 }
1513
1514 static bool rt_cache_valid(const struct rtable *rt)
1515 {
1516         return  rt &&
1517                 rt->dst.obsolete == DST_OBSOLETE_FORCE_CHK &&
1518                 !rt_is_expired(rt);
1519 }
1520
1521 static void rt_set_nexthop(struct rtable *rt, __be32 daddr,
1522                            const struct fib_result *res,
1523                            struct fib_nh_exception *fnhe,
1524                            struct fib_info *fi, u16 type, u32 itag,
1525                            const bool do_cache)
1526 {
1527         bool cached = false;
1528
1529         if (fi) {
1530                 struct fib_nh *nh = &FIB_RES_NH(*res);
1531
1532                 if (nh->nh_gw && nh->nh_scope == RT_SCOPE_LINK) {
1533                         rt->rt_gateway = nh->nh_gw;
1534                         rt->rt_uses_gateway = 1;
1535                 }
1536                 ip_dst_init_metrics(&rt->dst, fi->fib_metrics);
1537
1538 #ifdef CONFIG_IP_ROUTE_CLASSID
1539                 rt->dst.tclassid = nh->nh_tclassid;
1540 #endif
1541                 rt->dst.lwtstate = lwtstate_get(nh->nh_lwtstate);
1542                 if (unlikely(fnhe))
1543                         cached = rt_bind_exception(rt, fnhe, daddr, do_cache);
1544                 else if (do_cache)
1545                         cached = rt_cache_route(nh, rt);
1546                 if (unlikely(!cached)) {
1547                         /* Routes we intend to cache in nexthop exception or
1548                          * FIB nexthop have the DST_NOCACHE bit clear.
1549                          * However, if we are unsuccessful at storing this
1550                          * route into the cache we really need to set it.
1551                          */
1552                         if (!rt->rt_gateway)
1553                                 rt->rt_gateway = daddr;
1554                         rt_add_uncached_list(rt);
1555                 }
1556         } else
1557                 rt_add_uncached_list(rt);
1558
1559 #ifdef CONFIG_IP_ROUTE_CLASSID
1560 #ifdef CONFIG_IP_MULTIPLE_TABLES
1561         set_class_tag(rt, res->tclassid);
1562 #endif
1563         set_class_tag(rt, itag);
1564 #endif
1565 }
1566
1567 struct rtable *rt_dst_alloc(struct net_device *dev,
1568                             unsigned int flags, u16 type,
1569                             bool nopolicy, bool noxfrm, bool will_cache)
1570 {
1571         struct rtable *rt;
1572
1573         rt = dst_alloc(&ipv4_dst_ops, dev, 1, DST_OBSOLETE_FORCE_CHK,
1574                        (will_cache ? 0 : DST_HOST) |
1575                        (nopolicy ? DST_NOPOLICY : 0) |
1576                        (noxfrm ? DST_NOXFRM : 0));
1577
1578         if (rt) {
1579                 rt->rt_genid = rt_genid_ipv4(dev_net(dev));
1580                 rt->rt_flags = flags;
1581                 rt->rt_type = type;
1582                 rt->rt_is_input = 0;
1583                 rt->rt_iif = 0;
1584                 rt->rt_pmtu = 0;
1585                 rt->rt_mtu_locked = 0;
1586                 rt->rt_gateway = 0;
1587                 rt->rt_uses_gateway = 0;
1588                 INIT_LIST_HEAD(&rt->rt_uncached);
1589
1590                 rt->dst.output = ip_output;
1591                 if (flags & RTCF_LOCAL)
1592                         rt->dst.input = ip_local_deliver;
1593         }
1594
1595         return rt;
1596 }
1597 EXPORT_SYMBOL(rt_dst_alloc);
1598
1599 /* called in rcu_read_lock() section */
1600 int ip_mc_validate_source(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1601                           u8 tos, struct net_device *dev,
1602                           struct in_device *in_dev, u32 *itag)
1603 {
1604         int err;
1605
1606         /* Primary sanity checks. */
1607         if (!in_dev)
1608                 return -EINVAL;
1609
1610         if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) ||
1611             skb->protocol != htons(ETH_P_IP))
1612                 return -EINVAL;
1613
1614         if (ipv4_is_loopback(saddr) && !IN_DEV_ROUTE_LOCALNET(in_dev))
1615                 return -EINVAL;
1616
1617         if (ipv4_is_zeronet(saddr)) {
1618                 if (!ipv4_is_local_multicast(daddr) &&
1619                     ip_hdr(skb)->protocol != IPPROTO_IGMP)
1620                         return -EINVAL;
1621         } else {
1622                 err = fib_validate_source(skb, saddr, 0, tos, 0, dev,
1623                                           in_dev, itag);
1624                 if (err < 0)
1625                         return err;
1626         }
1627         return 0;
1628 }
1629
1630 /* called in rcu_read_lock() section */
1631 static int ip_route_input_mc(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1632                              u8 tos, struct net_device *dev, int our)
1633 {
1634         struct in_device *in_dev = __in_dev_get_rcu(dev);
1635         unsigned int flags = RTCF_MULTICAST;
1636         struct rtable *rth;
1637         u32 itag = 0;
1638         int err;
1639
1640         err = ip_mc_validate_source(skb, daddr, saddr, tos, dev, in_dev, &itag);
1641         if (err)
1642                 return err;
1643
1644         if (our)
1645                 flags |= RTCF_LOCAL;
1646
1647         rth = rt_dst_alloc(dev_net(dev)->loopback_dev, flags, RTN_MULTICAST,
1648                            IN_DEV_CONF_GET(in_dev, NOPOLICY), false, false);
1649         if (!rth)
1650                 return -ENOBUFS;
1651
1652 #ifdef CONFIG_IP_ROUTE_CLASSID
1653         rth->dst.tclassid = itag;
1654 #endif
1655         rth->dst.output = ip_rt_bug;
1656         rth->rt_is_input= 1;
1657
1658 #ifdef CONFIG_IP_MROUTE
1659         if (!ipv4_is_local_multicast(daddr) && IN_DEV_MFORWARD(in_dev))
1660                 rth->dst.input = ip_mr_input;
1661 #endif
1662         RT_CACHE_STAT_INC(in_slow_mc);
1663
1664         skb_dst_set(skb, &rth->dst);
1665         return 0;
1666 }
1667
1668
1669 static void ip_handle_martian_source(struct net_device *dev,
1670                                      struct in_device *in_dev,
1671                                      struct sk_buff *skb,
1672                                      __be32 daddr,
1673                                      __be32 saddr)
1674 {
1675         RT_CACHE_STAT_INC(in_martian_src);
1676 #ifdef CONFIG_IP_ROUTE_VERBOSE
1677         if (IN_DEV_LOG_MARTIANS(in_dev) && net_ratelimit()) {
1678                 /*
1679                  *      RFC1812 recommendation, if source is martian,
1680                  *      the only hint is MAC header.
1681                  */
1682                 pr_warn("martian source %pI4 from %pI4, on dev %s\n",
1683                         &daddr, &saddr, dev->name);
1684                 if (dev->hard_header_len && skb_mac_header_was_set(skb)) {
1685                         print_hex_dump(KERN_WARNING, "ll header: ",
1686                                        DUMP_PREFIX_OFFSET, 16, 1,
1687                                        skb_mac_header(skb),
1688                                        dev->hard_header_len, false);
1689                 }
1690         }
1691 #endif
1692 }
1693
1694 /* called in rcu_read_lock() section */
1695 static int __mkroute_input(struct sk_buff *skb,
1696                            const struct fib_result *res,
1697                            struct in_device *in_dev,
1698                            __be32 daddr, __be32 saddr, u32 tos)
1699 {
1700         struct fib_nh_exception *fnhe;
1701         struct rtable *rth;
1702         int err;
1703         struct in_device *out_dev;
1704         bool do_cache;
1705         u32 itag = 0;
1706
1707         /* get a working reference to the output device */
1708         out_dev = __in_dev_get_rcu(FIB_RES_DEV(*res));
1709         if (!out_dev) {
1710                 net_crit_ratelimited("Bug in ip_route_input_slow(). Please report.\n");
1711                 return -EINVAL;
1712         }
1713
1714         err = fib_validate_source(skb, saddr, daddr, tos, FIB_RES_OIF(*res),
1715                                   in_dev->dev, in_dev, &itag);
1716         if (err < 0) {
1717                 ip_handle_martian_source(in_dev->dev, in_dev, skb, daddr,
1718                                          saddr);
1719
1720                 goto cleanup;
1721         }
1722
1723         do_cache = res->fi && !itag;
1724         if (out_dev == in_dev && err && IN_DEV_TX_REDIRECTS(out_dev) &&
1725             skb->protocol == htons(ETH_P_IP) &&
1726             (IN_DEV_SHARED_MEDIA(out_dev) ||
1727              inet_addr_onlink(out_dev, saddr, FIB_RES_GW(*res))))
1728                 IPCB(skb)->flags |= IPSKB_DOREDIRECT;
1729
1730         if (skb->protocol != htons(ETH_P_IP)) {
1731                 /* Not IP (i.e. ARP). Do not create route, if it is
1732                  * invalid for proxy arp. DNAT routes are always valid.
1733                  *
1734                  * Proxy arp feature have been extended to allow, ARP
1735                  * replies back to the same interface, to support
1736                  * Private VLAN switch technologies. See arp.c.
1737                  */
1738                 if (out_dev == in_dev &&
1739                     IN_DEV_PROXY_ARP_PVLAN(in_dev) == 0) {
1740                         err = -EINVAL;
1741                         goto cleanup;
1742                 }
1743         }
1744
1745         fnhe = find_exception(&FIB_RES_NH(*res), daddr);
1746         if (do_cache) {
1747                 if (fnhe)
1748                         rth = rcu_dereference(fnhe->fnhe_rth_input);
1749                 else
1750                         rth = rcu_dereference(FIB_RES_NH(*res).nh_rth_input);
1751                 if (rt_cache_valid(rth)) {
1752                         skb_dst_set_noref(skb, &rth->dst);
1753                         goto out;
1754                 }
1755         }
1756
1757         rth = rt_dst_alloc(out_dev->dev, 0, res->type,
1758                            IN_DEV_CONF_GET(in_dev, NOPOLICY),
1759                            IN_DEV_CONF_GET(out_dev, NOXFRM), do_cache);
1760         if (!rth) {
1761                 err = -ENOBUFS;
1762                 goto cleanup;
1763         }
1764
1765         rth->rt_is_input = 1;
1766         RT_CACHE_STAT_INC(in_slow_tot);
1767
1768         rth->dst.input = ip_forward;
1769
1770         rt_set_nexthop(rth, daddr, res, fnhe, res->fi, res->type, itag,
1771                        do_cache);
1772         lwtunnel_set_redirect(&rth->dst);
1773         skb_dst_set(skb, &rth->dst);
1774 out:
1775         err = 0;
1776  cleanup:
1777         return err;
1778 }
1779
1780 #ifdef CONFIG_IP_ROUTE_MULTIPATH
1781 /* To make ICMP packets follow the right flow, the multipath hash is
1782  * calculated from the inner IP addresses.
1783  */
1784 static void ip_multipath_l3_keys(const struct sk_buff *skb,
1785                                  struct flow_keys *hash_keys)
1786 {
1787         const struct iphdr *outer_iph = ip_hdr(skb);
1788         const struct iphdr *key_iph = outer_iph;
1789         const struct iphdr *inner_iph;
1790         const struct icmphdr *icmph;
1791         struct iphdr _inner_iph;
1792         struct icmphdr _icmph;
1793
1794         if (likely(outer_iph->protocol != IPPROTO_ICMP))
1795                 goto out;
1796
1797         if (unlikely((outer_iph->frag_off & htons(IP_OFFSET)) != 0))
1798                 goto out;
1799
1800         icmph = skb_header_pointer(skb, outer_iph->ihl * 4, sizeof(_icmph),
1801                                    &_icmph);
1802         if (!icmph)
1803                 goto out;
1804
1805         if (icmph->type != ICMP_DEST_UNREACH &&
1806             icmph->type != ICMP_REDIRECT &&
1807             icmph->type != ICMP_TIME_EXCEEDED &&
1808             icmph->type != ICMP_PARAMETERPROB)
1809                 goto out;
1810
1811         inner_iph = skb_header_pointer(skb,
1812                                        outer_iph->ihl * 4 + sizeof(_icmph),
1813                                        sizeof(_inner_iph), &_inner_iph);
1814         if (!inner_iph)
1815                 goto out;
1816
1817         key_iph = inner_iph;
1818 out:
1819         hash_keys->addrs.v4addrs.src = key_iph->saddr;
1820         hash_keys->addrs.v4addrs.dst = key_iph->daddr;
1821 }
1822
1823 /* if skb is set it will be used and fl4 can be NULL */
1824 int fib_multipath_hash(const struct net *net, const struct flowi4 *fl4,
1825                        const struct sk_buff *skb, struct flow_keys *flkeys)
1826 {
1827         u32 multipath_hash = fl4 ? fl4->flowi4_multipath_hash : 0;
1828         struct flow_keys hash_keys;
1829         u32 mhash;
1830
1831         switch (net->ipv4.sysctl_fib_multipath_hash_policy) {
1832         case 0:
1833                 memset(&hash_keys, 0, sizeof(hash_keys));
1834                 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
1835                 if (skb) {
1836                         ip_multipath_l3_keys(skb, &hash_keys);
1837                 } else {
1838                         hash_keys.addrs.v4addrs.src = fl4->saddr;
1839                         hash_keys.addrs.v4addrs.dst = fl4->daddr;
1840                 }
1841                 break;
1842         case 1:
1843                 /* skb is currently provided only when forwarding */
1844                 if (skb) {
1845                         unsigned int flag = FLOW_DISSECTOR_F_STOP_AT_ENCAP;
1846                         struct flow_keys keys;
1847
1848                         /* short-circuit if we already have L4 hash present */
1849                         if (skb->l4_hash)
1850                                 return skb_get_hash_raw(skb) >> 1;
1851
1852                         memset(&hash_keys, 0, sizeof(hash_keys));
1853
1854                         if (!flkeys) {
1855                                 skb_flow_dissect_flow_keys(skb, &keys, flag);
1856                                 flkeys = &keys;
1857                         }
1858
1859                         hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
1860                         hash_keys.addrs.v4addrs.src = flkeys->addrs.v4addrs.src;
1861                         hash_keys.addrs.v4addrs.dst = flkeys->addrs.v4addrs.dst;
1862                         hash_keys.ports.src = flkeys->ports.src;
1863                         hash_keys.ports.dst = flkeys->ports.dst;
1864                         hash_keys.basic.ip_proto = flkeys->basic.ip_proto;
1865                 } else {
1866                         memset(&hash_keys, 0, sizeof(hash_keys));
1867                         hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
1868                         hash_keys.addrs.v4addrs.src = fl4->saddr;
1869                         hash_keys.addrs.v4addrs.dst = fl4->daddr;
1870                         hash_keys.ports.src = fl4->fl4_sport;
1871                         hash_keys.ports.dst = fl4->fl4_dport;
1872                         hash_keys.basic.ip_proto = fl4->flowi4_proto;
1873                 }
1874                 break;
1875         }
1876         mhash = flow_hash_from_keys(&hash_keys);
1877
1878         if (multipath_hash)
1879                 mhash = jhash_2words(mhash, multipath_hash, 0);
1880
1881         return mhash >> 1;
1882 }
1883 #endif /* CONFIG_IP_ROUTE_MULTIPATH */
1884
1885 static int ip_mkroute_input(struct sk_buff *skb,
1886                             struct fib_result *res,
1887                             struct in_device *in_dev,
1888                             __be32 daddr, __be32 saddr, u32 tos,
1889                             struct flow_keys *hkeys)
1890 {
1891 #ifdef CONFIG_IP_ROUTE_MULTIPATH
1892         if (res->fi && res->fi->fib_nhs > 1) {
1893                 int h = fib_multipath_hash(res->fi->fib_net, NULL, skb, hkeys);
1894
1895                 fib_select_multipath(res, h);
1896         }
1897 #endif
1898
1899         /* create a routing cache entry */
1900         return __mkroute_input(skb, res, in_dev, daddr, saddr, tos);
1901 }
1902
1903 /*
1904  *      NOTE. We drop all the packets that has local source
1905  *      addresses, because every properly looped back packet
1906  *      must have correct destination already attached by output routine.
1907  *
1908  *      Such approach solves two big problems:
1909  *      1. Not simplex devices are handled properly.
1910  *      2. IP spoofing attempts are filtered with 100% of guarantee.
1911  *      called with rcu_read_lock()
1912  */
1913
1914 static int ip_route_input_slow(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1915                                u8 tos, struct net_device *dev,
1916                                struct fib_result *res)
1917 {
1918         struct in_device *in_dev = __in_dev_get_rcu(dev);
1919         struct flow_keys *flkeys = NULL, _flkeys;
1920         struct net    *net = dev_net(dev);
1921         struct ip_tunnel_info *tun_info;
1922         int             err = -EINVAL;
1923         unsigned int    flags = 0;
1924         u32             itag = 0;
1925         struct rtable   *rth;
1926         struct flowi4   fl4;
1927         bool do_cache;
1928
1929         /* IP on this device is disabled. */
1930
1931         if (!in_dev)
1932                 goto out;
1933
1934         /* Check for the most weird martians, which can be not detected
1935            by fib_lookup.
1936          */
1937
1938         tun_info = skb_tunnel_info(skb);
1939         if (tun_info && !(tun_info->mode & IP_TUNNEL_INFO_TX))
1940                 fl4.flowi4_tun_key.tun_id = tun_info->key.tun_id;
1941         else
1942                 fl4.flowi4_tun_key.tun_id = 0;
1943         skb_dst_drop(skb);
1944
1945         if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr))
1946                 goto martian_source;
1947
1948         res->fi = NULL;
1949         res->table = NULL;
1950         if (ipv4_is_lbcast(daddr) || (saddr == 0 && daddr == 0))
1951                 goto brd_input;
1952
1953         /* Accept zero addresses only to limited broadcast;
1954          * I even do not know to fix it or not. Waiting for complains :-)
1955          */
1956         if (ipv4_is_zeronet(saddr))
1957                 goto martian_source;
1958
1959         if (ipv4_is_zeronet(daddr))
1960                 goto martian_destination;
1961
1962         /* Following code try to avoid calling IN_DEV_NET_ROUTE_LOCALNET(),
1963          * and call it once if daddr or/and saddr are loopback addresses
1964          */
1965         if (ipv4_is_loopback(daddr)) {
1966                 if (!IN_DEV_NET_ROUTE_LOCALNET(in_dev, net))
1967                         goto martian_destination;
1968         } else if (ipv4_is_loopback(saddr)) {
1969                 if (!IN_DEV_NET_ROUTE_LOCALNET(in_dev, net))
1970                         goto martian_source;
1971         }
1972
1973         /*
1974          *      Now we are ready to route packet.
1975          */
1976         fl4.flowi4_oif = 0;
1977         fl4.flowi4_iif = dev->ifindex;
1978         fl4.flowi4_mark = skb->mark;
1979         fl4.flowi4_tos = tos;
1980         fl4.flowi4_scope = RT_SCOPE_UNIVERSE;
1981         fl4.flowi4_flags = 0;
1982         fl4.daddr = daddr;
1983         fl4.saddr = saddr;
1984         fl4.flowi4_uid = sock_net_uid(net, NULL);
1985
1986         if (fib4_rules_early_flow_dissect(net, skb, &fl4, &_flkeys)) {
1987                 flkeys = &_flkeys;
1988         } else {
1989                 fl4.flowi4_proto = 0;
1990                 fl4.fl4_sport = 0;
1991                 fl4.fl4_dport = 0;
1992         }
1993
1994         err = fib_lookup(net, &fl4, res, 0);
1995         if (err != 0) {
1996                 if (!IN_DEV_FORWARD(in_dev))
1997                         err = -EHOSTUNREACH;
1998                 goto no_route;
1999         }
2000
2001         if (res->type == RTN_BROADCAST) {
2002                 if (IN_DEV_BFORWARD(in_dev))
2003                         goto make_route;
2004                 goto brd_input;
2005         }
2006
2007         if (res->type == RTN_LOCAL) {
2008                 err = fib_validate_source(skb, saddr, daddr, tos,
2009                                           0, dev, in_dev, &itag);
2010                 if (err < 0)
2011                         goto martian_source;
2012                 goto local_input;
2013         }
2014
2015         if (!IN_DEV_FORWARD(in_dev)) {
2016                 err = -EHOSTUNREACH;
2017                 goto no_route;
2018         }
2019         if (res->type != RTN_UNICAST)
2020                 goto martian_destination;
2021
2022 make_route:
2023         err = ip_mkroute_input(skb, res, in_dev, daddr, saddr, tos, flkeys);
2024 out:    return err;
2025
2026 brd_input:
2027         if (skb->protocol != htons(ETH_P_IP))
2028                 goto e_inval;
2029
2030         if (!ipv4_is_zeronet(saddr)) {
2031                 err = fib_validate_source(skb, saddr, 0, tos, 0, dev,
2032                                           in_dev, &itag);
2033                 if (err < 0)
2034                         goto martian_source;
2035         }
2036         flags |= RTCF_BROADCAST;
2037         res->type = RTN_BROADCAST;
2038         RT_CACHE_STAT_INC(in_brd);
2039
2040 local_input:
2041         do_cache = false;
2042         if (res->fi) {
2043                 if (!itag) {
2044                         rth = rcu_dereference(FIB_RES_NH(*res).nh_rth_input);
2045                         if (rt_cache_valid(rth)) {
2046                                 skb_dst_set_noref(skb, &rth->dst);
2047                                 err = 0;
2048                                 goto out;
2049                         }
2050                         do_cache = true;
2051                 }
2052         }
2053
2054         rth = rt_dst_alloc(l3mdev_master_dev_rcu(dev) ? : net->loopback_dev,
2055                            flags | RTCF_LOCAL, res->type,
2056                            IN_DEV_CONF_GET(in_dev, NOPOLICY), false, do_cache);
2057         if (!rth)
2058                 goto e_nobufs;
2059
2060         rth->dst.output= ip_rt_bug;
2061 #ifdef CONFIG_IP_ROUTE_CLASSID
2062         rth->dst.tclassid = itag;
2063 #endif
2064         rth->rt_is_input = 1;
2065
2066         RT_CACHE_STAT_INC(in_slow_tot);
2067         if (res->type == RTN_UNREACHABLE) {
2068                 rth->dst.input= ip_error;
2069                 rth->dst.error= -err;
2070                 rth->rt_flags   &= ~RTCF_LOCAL;
2071         }
2072
2073         if (do_cache) {
2074                 struct fib_nh *nh = &FIB_RES_NH(*res);
2075
2076                 rth->dst.lwtstate = lwtstate_get(nh->nh_lwtstate);
2077                 if (lwtunnel_input_redirect(rth->dst.lwtstate)) {
2078                         WARN_ON(rth->dst.input == lwtunnel_input);
2079                         rth->dst.lwtstate->orig_input = rth->dst.input;
2080                         rth->dst.input = lwtunnel_input;
2081                 }
2082
2083                 if (unlikely(!rt_cache_route(nh, rth)))
2084                         rt_add_uncached_list(rth);
2085         }
2086         skb_dst_set(skb, &rth->dst);
2087         err = 0;
2088         goto out;
2089
2090 no_route:
2091         RT_CACHE_STAT_INC(in_no_route);
2092         res->type = RTN_UNREACHABLE;
2093         res->fi = NULL;
2094         res->table = NULL;
2095         goto local_input;
2096
2097         /*
2098          *      Do not cache martian addresses: they should be logged (RFC1812)
2099          */
2100 martian_destination:
2101         RT_CACHE_STAT_INC(in_martian_dst);
2102 #ifdef CONFIG_IP_ROUTE_VERBOSE
2103         if (IN_DEV_LOG_MARTIANS(in_dev))
2104                 net_warn_ratelimited("martian destination %pI4 from %pI4, dev %s\n",
2105                                      &daddr, &saddr, dev->name);
2106 #endif
2107
2108 e_inval:
2109         err = -EINVAL;
2110         goto out;
2111
2112 e_nobufs:
2113         err = -ENOBUFS;
2114         goto out;
2115
2116 martian_source:
2117         ip_handle_martian_source(dev, in_dev, skb, daddr, saddr);
2118         goto out;
2119 }
2120
2121 int ip_route_input_noref(struct sk_buff *skb, __be32 daddr, __be32 saddr,
2122                          u8 tos, struct net_device *dev)
2123 {
2124         struct fib_result res;
2125         int err;
2126
2127         tos &= IPTOS_RT_MASK;
2128         rcu_read_lock();
2129         err = ip_route_input_rcu(skb, daddr, saddr, tos, dev, &res);
2130         rcu_read_unlock();
2131
2132         return err;
2133 }
2134 EXPORT_SYMBOL(ip_route_input_noref);
2135
2136 /* called with rcu_read_lock held */
2137 int ip_route_input_rcu(struct sk_buff *skb, __be32 daddr, __be32 saddr,
2138                        u8 tos, struct net_device *dev, struct fib_result *res)
2139 {
2140         /* Multicast recognition logic is moved from route cache to here.
2141            The problem was that too many Ethernet cards have broken/missing
2142            hardware multicast filters :-( As result the host on multicasting
2143            network acquires a lot of useless route cache entries, sort of
2144            SDR messages from all the world. Now we try to get rid of them.
2145            Really, provided software IP multicast filter is organized
2146            reasonably (at least, hashed), it does not result in a slowdown
2147            comparing with route cache reject entries.
2148            Note, that multicast routers are not affected, because
2149            route cache entry is created eventually.
2150          */
2151         if (ipv4_is_multicast(daddr)) {
2152                 struct in_device *in_dev = __in_dev_get_rcu(dev);
2153                 int our = 0;
2154                 int err = -EINVAL;
2155
2156                 if (!in_dev)
2157                         return err;
2158                 our = ip_check_mc_rcu(in_dev, daddr, saddr,
2159                                       ip_hdr(skb)->protocol);
2160
2161                 /* check l3 master if no match yet */
2162                 if (!our && netif_is_l3_slave(dev)) {
2163                         struct in_device *l3_in_dev;
2164
2165                         l3_in_dev = __in_dev_get_rcu(skb->dev);
2166                         if (l3_in_dev)
2167                                 our = ip_check_mc_rcu(l3_in_dev, daddr, saddr,
2168                                                       ip_hdr(skb)->protocol);
2169                 }
2170
2171                 if (our
2172 #ifdef CONFIG_IP_MROUTE
2173                         ||
2174                     (!ipv4_is_local_multicast(daddr) &&
2175                      IN_DEV_MFORWARD(in_dev))
2176 #endif
2177                    ) {
2178                         err = ip_route_input_mc(skb, daddr, saddr,
2179                                                 tos, dev, our);
2180                 }
2181                 return err;
2182         }
2183
2184         return ip_route_input_slow(skb, daddr, saddr, tos, dev, res);
2185 }
2186
2187 /* called with rcu_read_lock() */
2188 static struct rtable *__mkroute_output(const struct fib_result *res,
2189                                        const struct flowi4 *fl4, int orig_oif,
2190                                        struct net_device *dev_out,
2191                                        unsigned int flags)
2192 {
2193         struct fib_info *fi = res->fi;
2194         struct fib_nh_exception *fnhe;
2195         struct in_device *in_dev;
2196         u16 type = res->type;
2197         struct rtable *rth;
2198         bool do_cache;
2199
2200         in_dev = __in_dev_get_rcu(dev_out);
2201         if (!in_dev)
2202                 return ERR_PTR(-EINVAL);
2203
2204         if (likely(!IN_DEV_ROUTE_LOCALNET(in_dev)))
2205                 if (ipv4_is_loopback(fl4->saddr) &&
2206                     !(dev_out->flags & IFF_LOOPBACK) &&
2207                     !netif_is_l3_master(dev_out))
2208                         return ERR_PTR(-EINVAL);
2209
2210         if (ipv4_is_lbcast(fl4->daddr))
2211                 type = RTN_BROADCAST;
2212         else if (ipv4_is_multicast(fl4->daddr))
2213                 type = RTN_MULTICAST;
2214         else if (ipv4_is_zeronet(fl4->daddr))
2215                 return ERR_PTR(-EINVAL);
2216
2217         if (dev_out->flags & IFF_LOOPBACK)
2218                 flags |= RTCF_LOCAL;
2219
2220         do_cache = true;
2221         if (type == RTN_BROADCAST) {
2222                 flags |= RTCF_BROADCAST | RTCF_LOCAL;
2223                 fi = NULL;
2224         } else if (type == RTN_MULTICAST) {
2225                 flags |= RTCF_MULTICAST | RTCF_LOCAL;
2226                 if (!ip_check_mc_rcu(in_dev, fl4->daddr, fl4->saddr,
2227                                      fl4->flowi4_proto))
2228                         flags &= ~RTCF_LOCAL;
2229                 else
2230                         do_cache = false;
2231                 /* If multicast route do not exist use
2232                  * default one, but do not gateway in this case.
2233                  * Yes, it is hack.
2234                  */
2235                 if (fi && res->prefixlen < 4)
2236                         fi = NULL;
2237         } else if ((type == RTN_LOCAL) && (orig_oif != 0) &&
2238                    (orig_oif != dev_out->ifindex)) {
2239                 /* For local routes that require a particular output interface
2240                  * we do not want to cache the result.  Caching the result
2241                  * causes incorrect behaviour when there are multiple source
2242                  * addresses on the interface, the end result being that if the
2243                  * intended recipient is waiting on that interface for the
2244                  * packet he won't receive it because it will be delivered on
2245                  * the loopback interface and the IP_PKTINFO ipi_ifindex will
2246                  * be set to the loopback interface as well.
2247                  */
2248                 do_cache = false;
2249         }
2250
2251         fnhe = NULL;
2252         do_cache &= fi != NULL;
2253         if (fi) {
2254                 struct rtable __rcu **prth;
2255                 struct fib_nh *nh = &FIB_RES_NH(*res);
2256
2257                 fnhe = find_exception(nh, fl4->daddr);
2258                 if (!do_cache)
2259                         goto add;
2260                 if (fnhe) {
2261                         prth = &fnhe->fnhe_rth_output;
2262                 } else {
2263                         if (unlikely(fl4->flowi4_flags &
2264                                      FLOWI_FLAG_KNOWN_NH &&
2265                                      !(nh->nh_gw &&
2266                                        nh->nh_scope == RT_SCOPE_LINK))) {
2267                                 do_cache = false;
2268                                 goto add;
2269                         }
2270                         prth = raw_cpu_ptr(nh->nh_pcpu_rth_output);
2271                 }
2272                 rth = rcu_dereference(*prth);
2273                 if (rt_cache_valid(rth) && dst_hold_safe(&rth->dst))
2274                         return rth;
2275         }
2276
2277 add:
2278         rth = rt_dst_alloc(dev_out, flags, type,
2279                            IN_DEV_CONF_GET(in_dev, NOPOLICY),
2280                            IN_DEV_CONF_GET(in_dev, NOXFRM),
2281                            do_cache);
2282         if (!rth)
2283                 return ERR_PTR(-ENOBUFS);
2284
2285         rth->rt_iif = orig_oif;
2286
2287         RT_CACHE_STAT_INC(out_slow_tot);
2288
2289         if (flags & (RTCF_BROADCAST | RTCF_MULTICAST)) {
2290                 if (flags & RTCF_LOCAL &&
2291                     !(dev_out->flags & IFF_LOOPBACK)) {
2292                         rth->dst.output = ip_mc_output;
2293                         RT_CACHE_STAT_INC(out_slow_mc);
2294                 }
2295 #ifdef CONFIG_IP_MROUTE
2296                 if (type == RTN_MULTICAST) {
2297                         if (IN_DEV_MFORWARD(in_dev) &&
2298                             !ipv4_is_local_multicast(fl4->daddr)) {
2299                                 rth->dst.input = ip_mr_input;
2300                                 rth->dst.output = ip_mc_output;
2301                         }
2302                 }
2303 #endif
2304         }
2305
2306         rt_set_nexthop(rth, fl4->daddr, res, fnhe, fi, type, 0, do_cache);
2307         lwtunnel_set_redirect(&rth->dst);
2308
2309         return rth;
2310 }
2311
2312 /*
2313  * Major route resolver routine.
2314  */
2315
2316 struct rtable *ip_route_output_key_hash(struct net *net, struct flowi4 *fl4,
2317                                         const struct sk_buff *skb)
2318 {
2319         __u8 tos = RT_FL_TOS(fl4);
2320         struct fib_result res = {
2321                 .type           = RTN_UNSPEC,
2322                 .fi             = NULL,
2323                 .table          = NULL,
2324                 .tclassid       = 0,
2325         };
2326         struct rtable *rth;
2327
2328         fl4->flowi4_iif = LOOPBACK_IFINDEX;
2329         fl4->flowi4_tos = tos & IPTOS_RT_MASK;
2330         fl4->flowi4_scope = ((tos & RTO_ONLINK) ?
2331                          RT_SCOPE_LINK : RT_SCOPE_UNIVERSE);
2332
2333         rcu_read_lock();
2334         rth = ip_route_output_key_hash_rcu(net, fl4, &res, skb);
2335         rcu_read_unlock();
2336
2337         return rth;
2338 }
2339 EXPORT_SYMBOL_GPL(ip_route_output_key_hash);
2340
2341 struct rtable *ip_route_output_key_hash_rcu(struct net *net, struct flowi4 *fl4,
2342                                             struct fib_result *res,
2343                                             const struct sk_buff *skb)
2344 {
2345         struct net_device *dev_out = NULL;
2346         int orig_oif = fl4->flowi4_oif;
2347         unsigned int flags = 0;
2348         struct rtable *rth;
2349         int err = -ENETUNREACH;
2350
2351         if (fl4->saddr) {
2352                 rth = ERR_PTR(-EINVAL);
2353                 if (ipv4_is_multicast(fl4->saddr) ||
2354                     ipv4_is_lbcast(fl4->saddr) ||
2355                     ipv4_is_zeronet(fl4->saddr))
2356                         goto out;
2357
2358                 /* I removed check for oif == dev_out->oif here.
2359                    It was wrong for two reasons:
2360                    1. ip_dev_find(net, saddr) can return wrong iface, if saddr
2361                       is assigned to multiple interfaces.
2362                    2. Moreover, we are allowed to send packets with saddr
2363                       of another iface. --ANK
2364                  */
2365
2366                 if (fl4->flowi4_oif == 0 &&
2367                     (ipv4_is_multicast(fl4->daddr) ||
2368                      ipv4_is_lbcast(fl4->daddr))) {
2369                         /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
2370                         dev_out = __ip_dev_find(net, fl4->saddr, false);
2371                         if (!dev_out)
2372                                 goto out;
2373
2374                         /* Special hack: user can direct multicasts
2375                            and limited broadcast via necessary interface
2376                            without fiddling with IP_MULTICAST_IF or IP_PKTINFO.
2377                            This hack is not just for fun, it allows
2378                            vic,vat and friends to work.
2379                            They bind socket to loopback, set ttl to zero
2380                            and expect that it will work.
2381                            From the viewpoint of routing cache they are broken,
2382                            because we are not allowed to build multicast path
2383                            with loopback source addr (look, routing cache
2384                            cannot know, that ttl is zero, so that packet
2385                            will not leave this host and route is valid).
2386                            Luckily, this hack is good workaround.
2387                          */
2388
2389                         fl4->flowi4_oif = dev_out->ifindex;
2390                         goto make_route;
2391                 }
2392
2393                 if (!(fl4->flowi4_flags & FLOWI_FLAG_ANYSRC)) {
2394                         /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
2395                         if (!__ip_dev_find(net, fl4->saddr, false))
2396                                 goto out;
2397                 }
2398         }
2399
2400
2401         if (fl4->flowi4_oif) {
2402                 dev_out = dev_get_by_index_rcu(net, fl4->flowi4_oif);
2403                 rth = ERR_PTR(-ENODEV);
2404                 if (!dev_out)
2405                         goto out;
2406
2407                 /* RACE: Check return value of inet_select_addr instead. */
2408                 if (!(dev_out->flags & IFF_UP) || !__in_dev_get_rcu(dev_out)) {
2409                         rth = ERR_PTR(-ENETUNREACH);
2410                         goto out;
2411                 }
2412                 if (ipv4_is_local_multicast(fl4->daddr) ||
2413                     ipv4_is_lbcast(fl4->daddr) ||
2414                     fl4->flowi4_proto == IPPROTO_IGMP) {
2415                         if (!fl4->saddr)
2416                                 fl4->saddr = inet_select_addr(dev_out, 0,
2417                                                               RT_SCOPE_LINK);
2418                         goto make_route;
2419                 }
2420                 if (!fl4->saddr) {
2421                         if (ipv4_is_multicast(fl4->daddr))
2422                                 fl4->saddr = inet_select_addr(dev_out, 0,
2423                                                               fl4->flowi4_scope);
2424                         else if (!fl4->daddr)
2425                                 fl4->saddr = inet_select_addr(dev_out, 0,
2426                                                               RT_SCOPE_HOST);
2427                 }
2428         }
2429
2430         if (!fl4->daddr) {
2431                 fl4->daddr = fl4->saddr;
2432                 if (!fl4->daddr)
2433                         fl4->daddr = fl4->saddr = htonl(INADDR_LOOPBACK);
2434                 dev_out = net->loopback_dev;
2435                 fl4->flowi4_oif = LOOPBACK_IFINDEX;
2436                 res->type = RTN_LOCAL;
2437                 flags |= RTCF_LOCAL;
2438                 goto make_route;
2439         }
2440
2441         err = fib_lookup(net, fl4, res, 0);
2442         if (err) {
2443                 res->fi = NULL;
2444                 res->table = NULL;
2445                 if (fl4->flowi4_oif &&
2446                     (ipv4_is_multicast(fl4->daddr) ||
2447                     !netif_index_is_l3_master(net, fl4->flowi4_oif))) {
2448                         /* Apparently, routing tables are wrong. Assume,
2449                            that the destination is on link.
2450
2451                            WHY? DW.
2452                            Because we are allowed to send to iface
2453                            even if it has NO routes and NO assigned
2454                            addresses. When oif is specified, routing
2455                            tables are looked up with only one purpose:
2456                            to catch if destination is gatewayed, rather than
2457                            direct. Moreover, if MSG_DONTROUTE is set,
2458                            we send packet, ignoring both routing tables
2459                            and ifaddr state. --ANK
2460
2461
2462                            We could make it even if oif is unknown,
2463                            likely IPv6, but we do not.
2464                          */
2465
2466                         if (fl4->saddr == 0)
2467                                 fl4->saddr = inet_select_addr(dev_out, 0,
2468                                                               RT_SCOPE_LINK);
2469                         res->type = RTN_UNICAST;
2470                         goto make_route;
2471                 }
2472                 rth = ERR_PTR(err);
2473                 goto out;
2474         }
2475
2476         if (res->type == RTN_LOCAL) {
2477                 if (!fl4->saddr) {
2478                         if (res->fi->fib_prefsrc)
2479                                 fl4->saddr = res->fi->fib_prefsrc;
2480                         else
2481                                 fl4->saddr = fl4->daddr;
2482                 }
2483
2484                 /* L3 master device is the loopback for that domain */
2485                 dev_out = l3mdev_master_dev_rcu(FIB_RES_DEV(*res)) ? :
2486                         net->loopback_dev;
2487
2488                 /* make sure orig_oif points to fib result device even
2489                  * though packet rx/tx happens over loopback or l3mdev
2490                  */
2491                 orig_oif = FIB_RES_OIF(*res);
2492
2493                 fl4->flowi4_oif = dev_out->ifindex;
2494                 flags |= RTCF_LOCAL;
2495                 goto make_route;
2496         }
2497
2498         fib_select_path(net, res, fl4, skb);
2499
2500         dev_out = FIB_RES_DEV(*res);
2501         fl4->flowi4_oif = dev_out->ifindex;
2502
2503
2504 make_route:
2505         rth = __mkroute_output(res, fl4, orig_oif, dev_out, flags);
2506
2507 out:
2508         return rth;
2509 }
2510
2511 static struct dst_entry *ipv4_blackhole_dst_check(struct dst_entry *dst, u32 cookie)
2512 {
2513         return NULL;
2514 }
2515
2516 static unsigned int ipv4_blackhole_mtu(const struct dst_entry *dst)
2517 {
2518         unsigned int mtu = dst_metric_raw(dst, RTAX_MTU);
2519
2520         return mtu ? : dst->dev->mtu;
2521 }
2522
2523 static void ipv4_rt_blackhole_update_pmtu(struct dst_entry *dst, struct sock *sk,
2524                                           struct sk_buff *skb, u32 mtu)
2525 {
2526 }
2527
2528 static void ipv4_rt_blackhole_redirect(struct dst_entry *dst, struct sock *sk,
2529                                        struct sk_buff *skb)
2530 {
2531 }
2532
2533 static u32 *ipv4_rt_blackhole_cow_metrics(struct dst_entry *dst,
2534                                           unsigned long old)
2535 {
2536         return NULL;
2537 }
2538
2539 static struct dst_ops ipv4_dst_blackhole_ops = {
2540         .family                 =       AF_INET,
2541         .check                  =       ipv4_blackhole_dst_check,
2542         .mtu                    =       ipv4_blackhole_mtu,
2543         .default_advmss         =       ipv4_default_advmss,
2544         .update_pmtu            =       ipv4_rt_blackhole_update_pmtu,
2545         .redirect               =       ipv4_rt_blackhole_redirect,
2546         .cow_metrics            =       ipv4_rt_blackhole_cow_metrics,
2547         .neigh_lookup           =       ipv4_neigh_lookup,
2548 };
2549
2550 struct dst_entry *ipv4_blackhole_route(struct net *net, struct dst_entry *dst_orig)
2551 {
2552         struct rtable *ort = (struct rtable *) dst_orig;
2553         struct rtable *rt;
2554
2555         rt = dst_alloc(&ipv4_dst_blackhole_ops, NULL, 1, DST_OBSOLETE_DEAD, 0);
2556         if (rt) {
2557                 struct dst_entry *new = &rt->dst;
2558
2559                 new->__use = 1;
2560                 new->input = dst_discard;
2561                 new->output = dst_discard_out;
2562
2563                 new->dev = net->loopback_dev;
2564                 if (new->dev)
2565                         dev_hold(new->dev);
2566
2567                 rt->rt_is_input = ort->rt_is_input;
2568                 rt->rt_iif = ort->rt_iif;
2569                 rt->rt_pmtu = ort->rt_pmtu;
2570                 rt->rt_mtu_locked = ort->rt_mtu_locked;
2571
2572                 rt->rt_genid = rt_genid_ipv4(net);
2573                 rt->rt_flags = ort->rt_flags;
2574                 rt->rt_type = ort->rt_type;
2575                 rt->rt_gateway = ort->rt_gateway;
2576                 rt->rt_uses_gateway = ort->rt_uses_gateway;
2577
2578                 INIT_LIST_HEAD(&rt->rt_uncached);
2579         }
2580
2581         dst_release(dst_orig);
2582
2583         return rt ? &rt->dst : ERR_PTR(-ENOMEM);
2584 }
2585
2586 struct rtable *ip_route_output_flow(struct net *net, struct flowi4 *flp4,
2587                                     const struct sock *sk)
2588 {
2589         struct rtable *rt = __ip_route_output_key(net, flp4);
2590
2591         if (IS_ERR(rt))
2592                 return rt;
2593
2594         if (flp4->flowi4_proto)
2595                 rt = (struct rtable *)xfrm_lookup_route(net, &rt->dst,
2596                                                         flowi4_to_flowi(flp4),
2597                                                         sk, 0);
2598
2599         return rt;
2600 }
2601 EXPORT_SYMBOL_GPL(ip_route_output_flow);
2602
2603 /* called with rcu_read_lock held */
2604 static int rt_fill_info(struct net *net, __be32 dst, __be32 src,
2605                         struct rtable *rt, u32 table_id, struct flowi4 *fl4,
2606                         struct sk_buff *skb, u32 portid, u32 seq)
2607 {
2608         struct rtmsg *r;
2609         struct nlmsghdr *nlh;
2610         unsigned long expires = 0;
2611         u32 error;
2612         u32 metrics[RTAX_MAX];
2613
2614         nlh = nlmsg_put(skb, portid, seq, RTM_NEWROUTE, sizeof(*r), 0);
2615         if (!nlh)
2616                 return -EMSGSIZE;
2617
2618         r = nlmsg_data(nlh);
2619         r->rtm_family    = AF_INET;
2620         r->rtm_dst_len  = 32;
2621         r->rtm_src_len  = 0;
2622         r->rtm_tos      = fl4->flowi4_tos;
2623         r->rtm_table    = table_id < 256 ? table_id : RT_TABLE_COMPAT;
2624         if (nla_put_u32(skb, RTA_TABLE, table_id))
2625                 goto nla_put_failure;
2626         r->rtm_type     = rt->rt_type;
2627         r->rtm_scope    = RT_SCOPE_UNIVERSE;
2628         r->rtm_protocol = RTPROT_UNSPEC;
2629         r->rtm_flags    = (rt->rt_flags & ~0xFFFF) | RTM_F_CLONED;
2630         if (rt->rt_flags & RTCF_NOTIFY)
2631                 r->rtm_flags |= RTM_F_NOTIFY;
2632         if (IPCB(skb)->flags & IPSKB_DOREDIRECT)
2633                 r->rtm_flags |= RTCF_DOREDIRECT;
2634
2635         if (nla_put_in_addr(skb, RTA_DST, dst))
2636                 goto nla_put_failure;
2637         if (src) {
2638                 r->rtm_src_len = 32;
2639                 if (nla_put_in_addr(skb, RTA_SRC, src))
2640                         goto nla_put_failure;
2641         }
2642         if (rt->dst.dev &&
2643             nla_put_u32(skb, RTA_OIF, rt->dst.dev->ifindex))
2644                 goto nla_put_failure;
2645 #ifdef CONFIG_IP_ROUTE_CLASSID
2646         if (rt->dst.tclassid &&
2647             nla_put_u32(skb, RTA_FLOW, rt->dst.tclassid))
2648                 goto nla_put_failure;
2649 #endif
2650         if (!rt_is_input_route(rt) &&
2651             fl4->saddr != src) {
2652                 if (nla_put_in_addr(skb, RTA_PREFSRC, fl4->saddr))
2653                         goto nla_put_failure;
2654         }
2655         if (rt->rt_uses_gateway &&
2656             nla_put_in_addr(skb, RTA_GATEWAY, rt->rt_gateway))
2657                 goto nla_put_failure;
2658
2659         expires = rt->dst.expires;
2660         if (expires) {
2661                 unsigned long now = jiffies;
2662
2663                 if (time_before(now, expires))
2664                         expires -= now;
2665                 else
2666                         expires = 0;
2667         }
2668
2669         memcpy(metrics, dst_metrics_ptr(&rt->dst), sizeof(metrics));
2670         if (rt->rt_pmtu && expires)
2671                 metrics[RTAX_MTU - 1] = rt->rt_pmtu;
2672         if (rt->rt_mtu_locked && expires)
2673                 metrics[RTAX_LOCK - 1] |= BIT(RTAX_MTU);
2674         if (rtnetlink_put_metrics(skb, metrics) < 0)
2675                 goto nla_put_failure;
2676
2677         if (fl4->flowi4_mark &&
2678             nla_put_u32(skb, RTA_MARK, fl4->flowi4_mark))
2679                 goto nla_put_failure;
2680
2681         if (!uid_eq(fl4->flowi4_uid, INVALID_UID) &&
2682             nla_put_u32(skb, RTA_UID,
2683                         from_kuid_munged(current_user_ns(), fl4->flowi4_uid)))
2684                 goto nla_put_failure;
2685
2686         error = rt->dst.error;
2687
2688         if (rt_is_input_route(rt)) {
2689 #ifdef CONFIG_IP_MROUTE
2690                 if (ipv4_is_multicast(dst) && !ipv4_is_local_multicast(dst) &&
2691                     IPV4_DEVCONF_ALL(net, MC_FORWARDING)) {
2692                         int err = ipmr_get_route(net, skb,
2693                                                  fl4->saddr, fl4->daddr,
2694                                                  r, portid);
2695
2696                         if (err <= 0) {
2697                                 if (err == 0)
2698                                         return 0;
2699                                 goto nla_put_failure;
2700                         }
2701                 } else
2702 #endif
2703                         if (nla_put_u32(skb, RTA_IIF, fl4->flowi4_iif))
2704                                 goto nla_put_failure;
2705         }
2706
2707         if (rtnl_put_cacheinfo(skb, &rt->dst, 0, expires, error) < 0)
2708                 goto nla_put_failure;
2709
2710         nlmsg_end(skb, nlh);
2711         return 0;
2712
2713 nla_put_failure:
2714         nlmsg_cancel(skb, nlh);
2715         return -EMSGSIZE;
2716 }
2717
2718 static struct sk_buff *inet_rtm_getroute_build_skb(__be32 src, __be32 dst,
2719                                                    u8 ip_proto, __be16 sport,
2720                                                    __be16 dport)
2721 {
2722         struct sk_buff *skb;
2723         struct iphdr *iph;
2724
2725         skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
2726         if (!skb)
2727                 return NULL;
2728
2729         /* Reserve room for dummy headers, this skb can pass
2730          * through good chunk of routing engine.
2731          */
2732         skb_reset_mac_header(skb);
2733         skb_reset_network_header(skb);
2734         skb->protocol = htons(ETH_P_IP);
2735         iph = skb_put(skb, sizeof(struct iphdr));
2736         iph->protocol = ip_proto;
2737         iph->saddr = src;
2738         iph->daddr = dst;
2739         iph->version = 0x4;
2740         iph->frag_off = 0;
2741         iph->ihl = 0x5;
2742         skb_set_transport_header(skb, skb->len);
2743
2744         switch (iph->protocol) {
2745         case IPPROTO_UDP: {
2746                 struct udphdr *udph;
2747
2748                 udph = skb_put_zero(skb, sizeof(struct udphdr));
2749                 udph->source = sport;
2750                 udph->dest = dport;
2751                 udph->len = sizeof(struct udphdr);
2752                 udph->check = 0;
2753                 break;
2754         }
2755         case IPPROTO_TCP: {
2756                 struct tcphdr *tcph;
2757
2758                 tcph = skb_put_zero(skb, sizeof(struct tcphdr));
2759                 tcph->source    = sport;
2760                 tcph->dest      = dport;
2761                 tcph->doff      = sizeof(struct tcphdr) / 4;
2762                 tcph->rst = 1;
2763                 tcph->check = ~tcp_v4_check(sizeof(struct tcphdr),
2764                                             src, dst, 0);
2765                 break;
2766         }
2767         case IPPROTO_ICMP: {
2768                 struct icmphdr *icmph;
2769
2770                 icmph = skb_put_zero(skb, sizeof(struct icmphdr));
2771                 icmph->type = ICMP_ECHO;
2772                 icmph->code = 0;
2773         }
2774         }
2775
2776         return skb;
2777 }
2778
2779 static int inet_rtm_valid_getroute_req(struct sk_buff *skb,
2780                                        const struct nlmsghdr *nlh,
2781                                        struct nlattr **tb,
2782                                        struct netlink_ext_ack *extack)
2783 {
2784         struct rtmsg *rtm;
2785         int i, err;
2786
2787         if (nlh->nlmsg_len < nlmsg_msg_size(sizeof(*rtm))) {
2788                 NL_SET_ERR_MSG(extack,
2789                                "ipv4: Invalid header for route get request");
2790                 return -EINVAL;
2791         }
2792
2793         if (!netlink_strict_get_check(skb))
2794                 return nlmsg_parse(nlh, sizeof(*rtm), tb, RTA_MAX,
2795                                    rtm_ipv4_policy, extack);
2796
2797         rtm = nlmsg_data(nlh);
2798         if ((rtm->rtm_src_len && rtm->rtm_src_len != 32) ||
2799             (rtm->rtm_dst_len && rtm->rtm_dst_len != 32) ||
2800             rtm->rtm_table || rtm->rtm_protocol ||
2801             rtm->rtm_scope || rtm->rtm_type) {
2802                 NL_SET_ERR_MSG(extack, "ipv4: Invalid values in header for route get request");
2803                 return -EINVAL;
2804         }
2805
2806         if (rtm->rtm_flags & ~(RTM_F_NOTIFY |
2807                                RTM_F_LOOKUP_TABLE |
2808                                RTM_F_FIB_MATCH)) {
2809                 NL_SET_ERR_MSG(extack, "ipv4: Unsupported rtm_flags for route get request");
2810                 return -EINVAL;
2811         }
2812
2813         err = nlmsg_parse_strict(nlh, sizeof(*rtm), tb, RTA_MAX,
2814                                  rtm_ipv4_policy, extack);
2815         if (err)
2816                 return err;
2817
2818         if ((tb[RTA_SRC] && !rtm->rtm_src_len) ||
2819             (tb[RTA_DST] && !rtm->rtm_dst_len)) {
2820                 NL_SET_ERR_MSG(extack, "ipv4: rtm_src_len and rtm_dst_len must be 32 for IPv4");
2821                 return -EINVAL;
2822         }
2823
2824         for (i = 0; i <= RTA_MAX; i++) {
2825                 if (!tb[i])
2826                         continue;
2827
2828                 switch (i) {
2829                 case RTA_IIF:
2830                 case RTA_OIF:
2831                 case RTA_SRC:
2832                 case RTA_DST:
2833                 case RTA_IP_PROTO:
2834                 case RTA_SPORT:
2835                 case RTA_DPORT:
2836                 case RTA_MARK:
2837                 case RTA_UID:
2838                         break;
2839                 default:
2840                         NL_SET_ERR_MSG(extack, "ipv4: Unsupported attribute in route get request");
2841                         return -EINVAL;
2842                 }
2843         }
2844
2845         return 0;
2846 }
2847
2848 static int inet_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr *nlh,
2849                              struct netlink_ext_ack *extack)
2850 {
2851         struct net *net = sock_net(in_skb->sk);
2852         struct nlattr *tb[RTA_MAX+1];
2853         u32 table_id = RT_TABLE_MAIN;
2854         __be16 sport = 0, dport = 0;
2855         struct fib_result res = {};
2856         u8 ip_proto = IPPROTO_UDP;
2857         struct rtable *rt = NULL;
2858         struct sk_buff *skb;
2859         struct rtmsg *rtm;
2860         struct flowi4 fl4 = {};
2861         __be32 dst = 0;
2862         __be32 src = 0;
2863         kuid_t uid;
2864         u32 iif;
2865         int err;
2866         int mark;
2867
2868         err = inet_rtm_valid_getroute_req(in_skb, nlh, tb, extack);
2869         if (err < 0)
2870                 return err;
2871
2872         rtm = nlmsg_data(nlh);
2873         src = tb[RTA_SRC] ? nla_get_in_addr(tb[RTA_SRC]) : 0;
2874         dst = tb[RTA_DST] ? nla_get_in_addr(tb[RTA_DST]) : 0;
2875         iif = tb[RTA_IIF] ? nla_get_u32(tb[RTA_IIF]) : 0;
2876         mark = tb[RTA_MARK] ? nla_get_u32(tb[RTA_MARK]) : 0;
2877         if (tb[RTA_UID])
2878                 uid = make_kuid(current_user_ns(), nla_get_u32(tb[RTA_UID]));
2879         else
2880                 uid = (iif ? INVALID_UID : current_uid());
2881
2882         if (tb[RTA_IP_PROTO]) {
2883                 err = rtm_getroute_parse_ip_proto(tb[RTA_IP_PROTO],
2884                                                   &ip_proto, AF_INET, extack);
2885                 if (err)
2886                         return err;
2887         }
2888
2889         if (tb[RTA_SPORT])
2890                 sport = nla_get_be16(tb[RTA_SPORT]);
2891
2892         if (tb[RTA_DPORT])
2893                 dport = nla_get_be16(tb[RTA_DPORT]);
2894
2895         skb = inet_rtm_getroute_build_skb(src, dst, ip_proto, sport, dport);
2896         if (!skb)
2897                 return -ENOBUFS;
2898
2899         fl4.daddr = dst;
2900         fl4.saddr = src;
2901         fl4.flowi4_tos = rtm->rtm_tos;
2902         fl4.flowi4_oif = tb[RTA_OIF] ? nla_get_u32(tb[RTA_OIF]) : 0;
2903         fl4.flowi4_mark = mark;
2904         fl4.flowi4_uid = uid;
2905         if (sport)
2906                 fl4.fl4_sport = sport;
2907         if (dport)
2908                 fl4.fl4_dport = dport;
2909         fl4.flowi4_proto = ip_proto;
2910
2911         rcu_read_lock();
2912
2913         if (iif) {
2914                 struct net_device *dev;
2915
2916                 dev = dev_get_by_index_rcu(net, iif);
2917                 if (!dev) {
2918                         err = -ENODEV;
2919                         goto errout_rcu;
2920                 }
2921
2922                 fl4.flowi4_iif = iif; /* for rt_fill_info */
2923                 skb->dev        = dev;
2924                 skb->mark       = mark;
2925                 err = ip_route_input_rcu(skb, dst, src, rtm->rtm_tos,
2926                                          dev, &res);
2927
2928                 rt = skb_rtable(skb);
2929                 if (err == 0 && rt->dst.error)
2930                         err = -rt->dst.error;
2931         } else {
2932                 fl4.flowi4_iif = LOOPBACK_IFINDEX;
2933                 skb->dev = net->loopback_dev;
2934                 rt = ip_route_output_key_hash_rcu(net, &fl4, &res, skb);
2935                 err = 0;
2936                 if (IS_ERR(rt))
2937                         err = PTR_ERR(rt);
2938                 else
2939                         skb_dst_set(skb, &rt->dst);
2940         }
2941
2942         if (err)
2943                 goto errout_rcu;
2944
2945         if (rtm->rtm_flags & RTM_F_NOTIFY)
2946                 rt->rt_flags |= RTCF_NOTIFY;
2947
2948         if (rtm->rtm_flags & RTM_F_LOOKUP_TABLE)
2949                 table_id = res.table ? res.table->tb_id : 0;
2950
2951         /* reset skb for netlink reply msg */
2952         skb_trim(skb, 0);
2953         skb_reset_network_header(skb);
2954         skb_reset_transport_header(skb);
2955         skb_reset_mac_header(skb);
2956
2957         if (rtm->rtm_flags & RTM_F_FIB_MATCH) {
2958                 if (!res.fi) {
2959                         err = fib_props[res.type].error;
2960                         if (!err)
2961                                 err = -EHOSTUNREACH;
2962                         goto errout_rcu;
2963                 }
2964                 err = fib_dump_info(skb, NETLINK_CB(in_skb).portid,
2965                                     nlh->nlmsg_seq, RTM_NEWROUTE, table_id,
2966                                     rt->rt_type, res.prefix, res.prefixlen,
2967                                     fl4.flowi4_tos, res.fi, 0);
2968         } else {
2969                 err = rt_fill_info(net, dst, src, rt, table_id, &fl4, skb,
2970                                    NETLINK_CB(in_skb).portid, nlh->nlmsg_seq);
2971         }
2972         if (err < 0)
2973                 goto errout_rcu;
2974
2975         rcu_read_unlock();
2976
2977         err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).portid);
2978
2979 errout_free:
2980         return err;
2981 errout_rcu:
2982         rcu_read_unlock();
2983         kfree_skb(skb);
2984         goto errout_free;
2985 }
2986
2987 void ip_rt_multicast_event(struct in_device *in_dev)
2988 {
2989         rt_cache_flush(dev_net(in_dev->dev));
2990 }
2991
2992 #ifdef CONFIG_SYSCTL
2993 static int ip_rt_gc_interval __read_mostly  = 60 * HZ;
2994 static int ip_rt_gc_min_interval __read_mostly  = HZ / 2;
2995 static int ip_rt_gc_elasticity __read_mostly    = 8;
2996 static int ip_min_valid_pmtu __read_mostly      = IPV4_MIN_MTU;
2997
2998 static int ipv4_sysctl_rtcache_flush(struct ctl_table *__ctl, int write,
2999                                         void __user *buffer,
3000                                         size_t *lenp, loff_t *ppos)
3001 {
3002         struct net *net = (struct net *)__ctl->extra1;
3003
3004         if (write) {
3005                 rt_cache_flush(net);
3006                 fnhe_genid_bump(net);
3007                 return 0;
3008         }
3009
3010         return -EINVAL;
3011 }
3012
3013 static struct ctl_table ipv4_route_table[] = {
3014         {
3015                 .procname       = "gc_thresh",
3016                 .data           = &ipv4_dst_ops.gc_thresh,
3017                 .maxlen         = sizeof(int),
3018                 .mode           = 0644,
3019                 .proc_handler   = proc_dointvec,
3020         },
3021         {
3022                 .procname       = "max_size",
3023                 .data           = &ip_rt_max_size,
3024                 .maxlen         = sizeof(int),
3025                 .mode           = 0644,
3026                 .proc_handler   = proc_dointvec,
3027         },
3028         {
3029                 /*  Deprecated. Use gc_min_interval_ms */
3030
3031                 .procname       = "gc_min_interval",
3032                 .data           = &ip_rt_gc_min_interval,
3033                 .maxlen         = sizeof(int),
3034                 .mode           = 0644,
3035                 .proc_handler   = proc_dointvec_jiffies,
3036         },
3037         {
3038                 .procname       = "gc_min_interval_ms",
3039                 .data           = &ip_rt_gc_min_interval,
3040                 .maxlen         = sizeof(int),
3041                 .mode           = 0644,
3042                 .proc_handler   = proc_dointvec_ms_jiffies,
3043         },
3044         {
3045                 .procname       = "gc_timeout",
3046                 .data           = &ip_rt_gc_timeout,
3047                 .maxlen         = sizeof(int),
3048                 .mode           = 0644,
3049                 .proc_handler   = proc_dointvec_jiffies,
3050         },
3051         {
3052                 .procname       = "gc_interval",
3053                 .data           = &ip_rt_gc_interval,
3054                 .maxlen         = sizeof(int),
3055                 .mode           = 0644,
3056                 .proc_handler   = proc_dointvec_jiffies,
3057         },
3058         {
3059                 .procname       = "redirect_load",
3060                 .data           = &ip_rt_redirect_load,
3061                 .maxlen         = sizeof(int),
3062                 .mode           = 0644,
3063                 .proc_handler   = proc_dointvec,
3064         },
3065         {
3066                 .procname       = "redirect_number",
3067                 .data           = &ip_rt_redirect_number,
3068                 .maxlen         = sizeof(int),
3069                 .mode           = 0644,
3070                 .proc_handler   = proc_dointvec,
3071         },
3072         {
3073                 .procname       = "redirect_silence",
3074                 .data           = &ip_rt_redirect_silence,
3075                 .maxlen         = sizeof(int),
3076                 .mode           = 0644,
3077                 .proc_handler   = proc_dointvec,
3078         },
3079         {
3080                 .procname       = "error_cost",
3081                 .data           = &ip_rt_error_cost,
3082                 .maxlen         = sizeof(int),
3083                 .mode           = 0644,
3084                 .proc_handler   = proc_dointvec,
3085         },
3086         {
3087                 .procname       = "error_burst",
3088                 .data           = &ip_rt_error_burst,
3089                 .maxlen         = sizeof(int),
3090                 .mode           = 0644,
3091                 .proc_handler   = proc_dointvec,
3092         },
3093         {
3094                 .procname       = "gc_elasticity",
3095                 .data           = &ip_rt_gc_elasticity,
3096                 .maxlen         = sizeof(int),
3097                 .mode           = 0644,
3098                 .proc_handler   = proc_dointvec,
3099         },
3100         {
3101                 .procname       = "mtu_expires",
3102                 .data           = &ip_rt_mtu_expires,
3103                 .maxlen         = sizeof(int),
3104                 .mode           = 0644,
3105                 .proc_handler   = proc_dointvec_jiffies,
3106         },
3107         {
3108                 .procname       = "min_pmtu",
3109                 .data           = &ip_rt_min_pmtu,
3110                 .maxlen         = sizeof(int),
3111                 .mode           = 0644,
3112                 .proc_handler   = proc_dointvec_minmax,
3113                 .extra1         = &ip_min_valid_pmtu,
3114         },
3115         {
3116                 .procname       = "min_adv_mss",
3117                 .data           = &ip_rt_min_advmss,
3118                 .maxlen         = sizeof(int),
3119                 .mode           = 0644,
3120                 .proc_handler   = proc_dointvec,
3121         },
3122         { }
3123 };
3124
3125 static struct ctl_table ipv4_route_flush_table[] = {
3126         {
3127                 .procname       = "flush",
3128                 .maxlen         = sizeof(int),
3129                 .mode           = 0200,
3130                 .proc_handler   = ipv4_sysctl_rtcache_flush,
3131         },
3132         { },
3133 };
3134
3135 static __net_init int sysctl_route_net_init(struct net *net)
3136 {
3137         struct ctl_table *tbl;
3138
3139         tbl = ipv4_route_flush_table;
3140         if (!net_eq(net, &init_net)) {
3141                 tbl = kmemdup(tbl, sizeof(ipv4_route_flush_table), GFP_KERNEL);
3142                 if (!tbl)
3143                         goto err_dup;
3144
3145                 /* Don't export sysctls to unprivileged users */
3146                 if (net->user_ns != &init_user_ns)
3147                         tbl[0].procname = NULL;
3148         }
3149         tbl[0].extra1 = net;
3150
3151         net->ipv4.route_hdr = register_net_sysctl(net, "net/ipv4/route", tbl);
3152         if (!net->ipv4.route_hdr)
3153                 goto err_reg;
3154         return 0;
3155
3156 err_reg:
3157         if (tbl != ipv4_route_flush_table)
3158                 kfree(tbl);
3159 err_dup:
3160         return -ENOMEM;
3161 }
3162
3163 static __net_exit void sysctl_route_net_exit(struct net *net)
3164 {
3165         struct ctl_table *tbl;
3166
3167         tbl = net->ipv4.route_hdr->ctl_table_arg;
3168         unregister_net_sysctl_table(net->ipv4.route_hdr);
3169         BUG_ON(tbl == ipv4_route_flush_table);
3170         kfree(tbl);
3171 }
3172
3173 static __net_initdata struct pernet_operations sysctl_route_ops = {
3174         .init = sysctl_route_net_init,
3175         .exit = sysctl_route_net_exit,
3176 };
3177 #endif
3178
3179 static __net_init int rt_genid_init(struct net *net)
3180 {
3181         atomic_set(&net->ipv4.rt_genid, 0);
3182         atomic_set(&net->fnhe_genid, 0);
3183         atomic_set(&net->ipv4.dev_addr_genid, get_random_int());
3184         return 0;
3185 }
3186
3187 static __net_initdata struct pernet_operations rt_genid_ops = {
3188         .init = rt_genid_init,
3189 };
3190
3191 static int __net_init ipv4_inetpeer_init(struct net *net)
3192 {
3193         struct inet_peer_base *bp = kmalloc(sizeof(*bp), GFP_KERNEL);
3194
3195         if (!bp)
3196                 return -ENOMEM;
3197         inet_peer_base_init(bp);
3198         net->ipv4.peers = bp;
3199         return 0;
3200 }
3201
3202 static void __net_exit ipv4_inetpeer_exit(struct net *net)
3203 {
3204         struct inet_peer_base *bp = net->ipv4.peers;
3205
3206         net->ipv4.peers = NULL;
3207         inetpeer_invalidate_tree(bp);
3208         kfree(bp);
3209 }
3210
3211 static __net_initdata struct pernet_operations ipv4_inetpeer_ops = {
3212         .init   =       ipv4_inetpeer_init,
3213         .exit   =       ipv4_inetpeer_exit,
3214 };
3215
3216 #ifdef CONFIG_IP_ROUTE_CLASSID
3217 struct ip_rt_acct __percpu *ip_rt_acct __read_mostly;
3218 #endif /* CONFIG_IP_ROUTE_CLASSID */
3219
3220 int __init ip_rt_init(void)
3221 {
3222         int cpu;
3223
3224         ip_idents = kmalloc_array(IP_IDENTS_SZ, sizeof(*ip_idents),
3225                                   GFP_KERNEL);
3226         if (!ip_idents)
3227                 panic("IP: failed to allocate ip_idents\n");
3228
3229         prandom_bytes(ip_idents, IP_IDENTS_SZ * sizeof(*ip_idents));
3230
3231         ip_tstamps = kcalloc(IP_IDENTS_SZ, sizeof(*ip_tstamps), GFP_KERNEL);
3232         if (!ip_tstamps)
3233                 panic("IP: failed to allocate ip_tstamps\n");
3234
3235         for_each_possible_cpu(cpu) {
3236                 struct uncached_list *ul = &per_cpu(rt_uncached_list, cpu);
3237
3238                 INIT_LIST_HEAD(&ul->head);
3239                 spin_lock_init(&ul->lock);
3240         }
3241 #ifdef CONFIG_IP_ROUTE_CLASSID
3242         ip_rt_acct = __alloc_percpu(256 * sizeof(struct ip_rt_acct), __alignof__(struct ip_rt_acct));
3243         if (!ip_rt_acct)
3244                 panic("IP: failed to allocate ip_rt_acct\n");
3245 #endif
3246
3247         ipv4_dst_ops.kmem_cachep =
3248                 kmem_cache_create("ip_dst_cache", sizeof(struct rtable), 0,
3249                                   SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
3250
3251         ipv4_dst_blackhole_ops.kmem_cachep = ipv4_dst_ops.kmem_cachep;
3252
3253         if (dst_entries_init(&ipv4_dst_ops) < 0)
3254                 panic("IP: failed to allocate ipv4_dst_ops counter\n");
3255
3256         if (dst_entries_init(&ipv4_dst_blackhole_ops) < 0)
3257                 panic("IP: failed to allocate ipv4_dst_blackhole_ops counter\n");
3258
3259         ipv4_dst_ops.gc_thresh = ~0;
3260         ip_rt_max_size = INT_MAX;
3261
3262         devinet_init();
3263         ip_fib_init();
3264
3265         if (ip_rt_proc_init())
3266                 pr_err("Unable to create route proc files\n");
3267 #ifdef CONFIG_XFRM
3268         xfrm_init();
3269         xfrm4_init();
3270 #endif
3271         rtnl_register(PF_INET, RTM_GETROUTE, inet_rtm_getroute, NULL,
3272                       RTNL_FLAG_DOIT_UNLOCKED);
3273
3274 #ifdef CONFIG_SYSCTL
3275         register_pernet_subsys(&sysctl_route_ops);
3276 #endif
3277         register_pernet_subsys(&rt_genid_ops);
3278         register_pernet_subsys(&ipv4_inetpeer_ops);
3279         return 0;
3280 }
3281
3282 #ifdef CONFIG_SYSCTL
3283 /*
3284  * We really need to sanitize the damn ipv4 init order, then all
3285  * this nonsense will go away.
3286  */
3287 void __init ip_static_sysctl_init(void)
3288 {
3289         register_net_sysctl(&init_net, "net/ipv4/route", ipv4_route_table);
3290 }
3291 #endif