net/ipv4/route.c

   1 /*
   2  * INET         An implementation of the TCP/IP protocol suite for the LINUX
   3  *              operating system.  INET is implemented using the  BSD Socket
   4  *              interface as the means of communication with the user level.
   5  *
   6  *              ROUTE - implementation of the IP router.
   7  *
   8  * Authors:     Ross Biro
   9  *              Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
  10  *              Alan Cox, <gw4pts@gw4pts.ampr.org>
  11  *              Linus Torvalds, <Linus.Torvalds@helsinki.fi>
  12  *              Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
  13  *
  14  * Fixes:
  15  *              Alan Cox        :       Verify area fixes.
  16  *              Alan Cox        :       cli() protects routing changes
  17  *              Rui Oliveira    :       ICMP routing table updates
  18  *              (rco@di.uminho.pt)      Routing table insertion and update
  19  *              Linus Torvalds  :       Rewrote bits to be sensible
  20  *              Alan Cox        :       Added BSD route gw semantics
  21  *              Alan Cox        :       Super /proc >4K
  22  *              Alan Cox        :       MTU in route table
  23  *              Alan Cox        :       MSS actually. Also added the window
  24  *                                      clamper.
  25  *              Sam Lantinga    :       Fixed route matching in rt_del()
  26  *              Alan Cox        :       Routing cache support.
  27  *              Alan Cox        :       Removed compatibility cruft.
  28  *              Alan Cox        :       RTF_REJECT support.
  29  *              Alan Cox        :       TCP irtt support.
  30  *              Jonathan Naylor :       Added Metric support.
  31  *      Miquel van Smoorenburg  :       BSD API fixes.
  32  *      Miquel van Smoorenburg  :       Metrics.
  33  *              Alan Cox        :       Use __u32 properly
  34  *              Alan Cox        :       Aligned routing errors more closely with BSD
  35  *                                      our system is still very different.
  36  *              Alan Cox        :       Faster /proc handling
  37  *      Alexey Kuznetsov        :       Massive rework to support tree based routing,
  38  *                                      routing caches and better behaviour.
  39  *
  40  *              Olaf Erb        :       irtt wasn't being copied right.
  41  *              Bjorn Ekwall    :       Kerneld route support.
  42  *              Alan Cox        :       Multicast fixed (I hope)
  43  *              Pavel Krauz     :       Limited broadcast fixed
  44  *              Mike McLagan    :       Routing by source
  45  *      Alexey Kuznetsov        :       End of old history. Split to fib.c and
  46  *                                      route.c and rewritten from scratch.
  47  *              Andi Kleen      :       Load-limit warning messages.
  48  *      Vitaly E. Lavrov        :       Transparent proxy revived after year coma.
  49  *      Vitaly E. Lavrov        :       Race condition in ip_route_input_slow.
  50  *      Tobias Ringstrom        :       Uninitialized res.type in ip_route_output_slow.
  51  *      Vladimir V. Ivanov      :       IP rule info (flowid) is really useful.
  52  *              Marc Boucher    :       routing by fwmark
  53  *      Robert Olsson           :       Added rt_cache statistics
  54  *      Arnaldo C. Melo         :       Convert proc stuff to seq_file
  55  *      Eric Dumazet            :       hashed spinlocks and rt_check_expire() fixes.
  56  *      Ilia Sotnikov           :       Ignore TOS on PMTUD and Redirect
  57  *      Ilia Sotnikov           :       Removed TOS from hash calculations
  58  *
  59  *              This program is free software; you can redistribute it and/or
  60  *              modify it under the terms of the GNU General Public License
  61  *              as published by the Free Software Foundation; either version
  62  *              2 of the License, or (at your option) any later version.
  63  */
  64
  65 #define pr_fmt(fmt) "IPv4: " fmt
  66
  67 #include <linux/module.h>
  68 #include <linux/uaccess.h>
  69 #include <linux/bitops.h>
  70 #include <linux/types.h>
  71 #include <linux/kernel.h>
  72 #include <linux/mm.h>
  73 #include <linux/string.h>
  74 #include <linux/socket.h>
  75 #include <linux/sockios.h>
  76 #include <linux/errno.h>
  77 #include <linux/in.h>
  78 #include <linux/inet.h>
  79 #include <linux/netdevice.h>
  80 #include <linux/proc_fs.h>
  81 #include <linux/init.h>
  82 #include <linux/skbuff.h>
  83 #include <linux/inetdevice.h>
  84 #include <linux/igmp.h>
  85 #include <linux/pkt_sched.h>
  86 #include <linux/mroute.h>
  87 #include <linux/netfilter_ipv4.h>
  88 #include <linux/random.h>
  89 #include <linux/rcupdate.h>
  90 #include <linux/times.h>
  91 #include <linux/slab.h>
  92 #include <linux/jhash.h>
  93 #include <net/dst.h>
  94 #include <net/dst_metadata.h>
  95 #include <net/net_namespace.h>
  96 #include <net/protocol.h>
  97 #include <net/ip.h>
  98 #include <net/route.h>
  99 #include <net/inetpeer.h>
 100 #include <net/sock.h>
 101 #include <net/ip_fib.h>
 102 #include <net/arp.h>
 103 #include <net/tcp.h>
 104 #include <net/icmp.h>
 105 #include <net/xfrm.h>
 106 #include <net/lwtunnel.h>
 107 #include <net/netevent.h>
 108 #include <net/rtnetlink.h>
 109 #ifdef CONFIG_SYSCTL
 110 #include <linux/sysctl.h>
 111 #endif
 112 #include <net/secure_seq.h>
 113 #include <net/ip_tunnels.h>
 114 #include <net/l3mdev.h>
 115
 116 #include "fib_lookup.h"
 117
 118 #define RT_FL_TOS(oldflp4) \
 119         ((oldflp4)->flowi4_tos & (IPTOS_RT_MASK | RTO_ONLINK))
 120
 121 #define RT_GC_TIMEOUT (300*HZ)
 122
 123 static int ip_rt_max_size;
 124 static int ip_rt_redirect_number __read_mostly  = 9;
 125 static int ip_rt_redirect_load __read_mostly    = HZ / 50;
 126 static int ip_rt_redirect_silence __read_mostly = ((HZ / 50) << (9 + 1));
 127 static int ip_rt_error_cost __read_mostly       = HZ;
 128 static int ip_rt_error_burst __read_mostly      = 5 * HZ;
 129 static int ip_rt_mtu_expires __read_mostly      = 10 * 60 * HZ;
 130 static u32 ip_rt_min_pmtu __read_mostly         = 512 + 20 + 20;
 131 static int ip_rt_min_advmss __read_mostly       = 256;
 132
 133 static int ip_rt_gc_timeout __read_mostly       = RT_GC_TIMEOUT;
 134
 135 /*
 136  *      Interface to generic destination cache.
 137  */
 138
 139 static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie);
 140 static unsigned int      ipv4_default_advmss(const struct dst_entry *dst);
 141 static unsigned int      ipv4_mtu(const struct dst_entry *dst);
 142 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst);
 143 static void              ipv4_link_failure(struct sk_buff *skb);
 144 static void              ip_rt_update_pmtu(struct dst_entry *dst, struct sock *sk,
 145                                            struct sk_buff *skb, u32 mtu);
 146 static void              ip_do_redirect(struct dst_entry *dst, struct sock *sk,
 147                                         struct sk_buff *skb);
 148 static void             ipv4_dst_destroy(struct dst_entry *dst);
 149
 150 static u32 *ipv4_cow_metrics(struct dst_entry *dst, unsigned long old)
 151 {
 152         WARN_ON(1);
 153         return NULL;
 154 }
 155
 156 static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst,
 157                                            struct sk_buff *skb,
 158                                            const void *daddr);
 159 static void ipv4_confirm_neigh(const struct dst_entry *dst, const void *daddr);
 160
 161 static struct dst_ops ipv4_dst_ops = {
 162         .family =               AF_INET,
 163         .check =                ipv4_dst_check,
 164         .default_advmss =       ipv4_default_advmss,
 165         .mtu =                  ipv4_mtu,
 166         .cow_metrics =          ipv4_cow_metrics,
 167         .destroy =              ipv4_dst_destroy,
 168         .negative_advice =      ipv4_negative_advice,
 169         .link_failure =         ipv4_link_failure,
 170         .update_pmtu =          ip_rt_update_pmtu,
 171         .redirect =             ip_do_redirect,
 172         .local_out =            __ip_local_out,
 173         .neigh_lookup =         ipv4_neigh_lookup,
 174         .confirm_neigh =        ipv4_confirm_neigh,
 175 };
 176
 177 #define ECN_OR_COST(class)      TC_PRIO_##class
 178
 179 const __u8 ip_tos2prio[16] = {
 180         TC_PRIO_BESTEFFORT,
 181         ECN_OR_COST(BESTEFFORT),
 182         TC_PRIO_BESTEFFORT,
 183         ECN_OR_COST(BESTEFFORT),
 184         TC_PRIO_BULK,
 185         ECN_OR_COST(BULK),
 186         TC_PRIO_BULK,
 187         ECN_OR_COST(BULK),
 188         TC_PRIO_INTERACTIVE,
 189         ECN_OR_COST(INTERACTIVE),
 190         TC_PRIO_INTERACTIVE,
 191         ECN_OR_COST(INTERACTIVE),
 192         TC_PRIO_INTERACTIVE_BULK,
 193         ECN_OR_COST(INTERACTIVE_BULK),
 194         TC_PRIO_INTERACTIVE_BULK,
 195         ECN_OR_COST(INTERACTIVE_BULK)
 196 };
 197 EXPORT_SYMBOL(ip_tos2prio);
 198
 199 static DEFINE_PER_CPU(struct rt_cache_stat, rt_cache_stat);
 200 #define RT_CACHE_STAT_INC(field) raw_cpu_inc(rt_cache_stat.field)
 201
 202 #ifdef CONFIG_PROC_FS
 203 static void *rt_cache_seq_start(struct seq_file *seq, loff_t *pos)
 204 {
 205         if (*pos)
 206                 return NULL;
 207         return SEQ_START_TOKEN;
 208 }
 209
 210 static void *rt_cache_seq_next(struct seq_file *seq, void *v, loff_t *pos)
 211 {
 212         ++*pos;
 213         return NULL;
 214 }
 215
 216 static void rt_cache_seq_stop(struct seq_file *seq, void *v)
 217 {
 218 }
 219
 220 static int rt_cache_seq_show(struct seq_file *seq, void *v)
 221 {
 222         if (v == SEQ_START_TOKEN)
 223                 seq_printf(seq, "%-127s\n",
 224                            "Iface\tDestination\tGateway \tFlags\t\tRefCnt\tUse\t"
 225                            "Metric\tSource\t\tMTU\tWindow\tIRTT\tTOS\tHHRef\t"
 226                            "HHUptod\tSpecDst");
 227         return 0;
 228 }
 229
 230 static const struct seq_operations rt_cache_seq_ops = {
 231         .start  = rt_cache_seq_start,
 232         .next   = rt_cache_seq_next,
 233         .stop   = rt_cache_seq_stop,
 234         .show   = rt_cache_seq_show,
 235 };
 236
 237 static int rt_cache_seq_open(struct inode *inode, struct file *file)
 238 {
 239         return seq_open(file, &rt_cache_seq_ops);
 240 }
 241
 242 static const struct file_operations rt_cache_seq_fops = {
 243         .open    = rt_cache_seq_open,
 244         .read    = seq_read,
 245         .llseek  = seq_lseek,
 246         .release = seq_release,
 247 };
 248
 249
 250 static void *rt_cpu_seq_start(struct seq_file *seq, loff_t *pos)
 251 {
 252         int cpu;
 253
 254         if (*pos == 0)
 255                 return SEQ_START_TOKEN;
 256
 257         for (cpu = *pos-1; cpu < nr_cpu_ids; ++cpu) {
 258                 if (!cpu_possible(cpu))
 259                         continue;
 260                 *pos = cpu+1;
 261                 return &per_cpu(rt_cache_stat, cpu);
 262         }
 263         return NULL;
 264 }
 265
 266 static void *rt_cpu_seq_next(struct seq_file *seq, void *v, loff_t *pos)
 267 {
 268         int cpu;
 269
 270         for (cpu = *pos; cpu < nr_cpu_ids; ++cpu) {
 271                 if (!cpu_possible(cpu))
 272                         continue;
 273                 *pos = cpu+1;
 274                 return &per_cpu(rt_cache_stat, cpu);
 275         }
 276         return NULL;
 277
 278 }
 279
 280 static void rt_cpu_seq_stop(struct seq_file *seq, void *v)
 281 {
 282
 283 }
 284
 285 static int rt_cpu_seq_show(struct seq_file *seq, void *v)
 286 {
 287         struct rt_cache_stat *st = v;
 288
 289         if (v == SEQ_START_TOKEN) {
 290                 seq_printf(seq, "entries  in_hit in_slow_tot in_slow_mc in_no_route in_brd in_martian_dst in_martian_src  out_hit out_slow_tot out_slow_mc  gc_total gc_ignored gc_goal_miss gc_dst_overflow in_hlist_search out_hlist_search\n");
 291                 return 0;
 292         }
 293
 294         seq_printf(seq,"%08x  %08x %08x %08x %08x %08x %08x %08x "
 295                    " %08x %08x %08x %08x %08x %08x %08x %08x %08x \n",
 296                    dst_entries_get_slow(&ipv4_dst_ops),
 297                    0, /* st->in_hit */
 298                    st->in_slow_tot,
 299                    st->in_slow_mc,
 300                    st->in_no_route,
 301                    st->in_brd,
 302                    st->in_martian_dst,
 303                    st->in_martian_src,
 304
 305                    0, /* st->out_hit */
 306                    st->out_slow_tot,
 307                    st->out_slow_mc,
 308
 309                    0, /* st->gc_total */
 310                    0, /* st->gc_ignored */
 311                    0, /* st->gc_goal_miss */
 312                    0, /* st->gc_dst_overflow */
 313                    0, /* st->in_hlist_search */
 314                    0  /* st->out_hlist_search */
 315                 );
 316         return 0;
 317 }
 318
 319 static const struct seq_operations rt_cpu_seq_ops = {
 320         .start  = rt_cpu_seq_start,
 321         .next   = rt_cpu_seq_next,
 322         .stop   = rt_cpu_seq_stop,
 323         .show   = rt_cpu_seq_show,
 324 };
 325
 326
 327 static int rt_cpu_seq_open(struct inode *inode, struct file *file)
 328 {
 329         return seq_open(file, &rt_cpu_seq_ops);
 330 }
 331
 332 static const struct file_operations rt_cpu_seq_fops = {
 333         .open    = rt_cpu_seq_open,
 334         .read    = seq_read,
 335         .llseek  = seq_lseek,
 336         .release = seq_release,
 337 };
 338
 339 #ifdef CONFIG_IP_ROUTE_CLASSID
 340 static int rt_acct_proc_show(struct seq_file *m, void *v)
 341 {
 342         struct ip_rt_acct *dst, *src;
 343         unsigned int i, j;
 344
 345         dst = kcalloc(256, sizeof(struct ip_rt_acct), GFP_KERNEL);
 346         if (!dst)
 347                 return -ENOMEM;
 348
 349         for_each_possible_cpu(i) {
 350                 src = (struct ip_rt_acct *)per_cpu_ptr(ip_rt_acct, i);
 351                 for (j = 0; j < 256; j++) {
 352                         dst[j].o_bytes   += src[j].o_bytes;
 353                         dst[j].o_packets += src[j].o_packets;
 354                         dst[j].i_bytes   += src[j].i_bytes;
 355                         dst[j].i_packets += src[j].i_packets;
 356                 }
 357         }
 358
 359         seq_write(m, dst, 256 * sizeof(struct ip_rt_acct));
 360         kfree(dst);
 361         return 0;
 362 }
 363 #endif
 364
 365 static int __net_init ip_rt_do_proc_init(struct net *net)
 366 {
 367         struct proc_dir_entry *pde;
 368
 369         pde = proc_create("rt_cache", 0444, net->proc_net,
 370                           &rt_cache_seq_fops);
 371         if (!pde)
 372                 goto err1;
 373
 374         pde = proc_create("rt_cache", 0444,
 375                           net->proc_net_stat, &rt_cpu_seq_fops);
 376         if (!pde)
 377                 goto err2;
 378
 379 #ifdef CONFIG_IP_ROUTE_CLASSID
 380         pde = proc_create_single("rt_acct", 0, net->proc_net,
 381                         rt_acct_proc_show);
 382         if (!pde)
 383                 goto err3;
 384 #endif
 385         return 0;
 386
 387 #ifdef CONFIG_IP_ROUTE_CLASSID
 388 err3:
 389         remove_proc_entry("rt_cache", net->proc_net_stat);
 390 #endif
 391 err2:
 392         remove_proc_entry("rt_cache", net->proc_net);
 393 err1:
 394         return -ENOMEM;
 395 }
 396
 397 static void __net_exit ip_rt_do_proc_exit(struct net *net)
 398 {
 399         remove_proc_entry("rt_cache", net->proc_net_stat);
 400         remove_proc_entry("rt_cache", net->proc_net);
 401 #ifdef CONFIG_IP_ROUTE_CLASSID
 402         remove_proc_entry("rt_acct", net->proc_net);
 403 #endif
 404 }
 405
 406 static struct pernet_operations ip_rt_proc_ops __net_initdata =  {
 407         .init = ip_rt_do_proc_init,
 408         .exit = ip_rt_do_proc_exit,
 409 };
 410
 411 static int __init ip_rt_proc_init(void)
 412 {
 413         return register_pernet_subsys(&ip_rt_proc_ops);
 414 }
 415
 416 #else
 417 static inline int ip_rt_proc_init(void)
 418 {
 419         return 0;
 420 }
 421 #endif /* CONFIG_PROC_FS */
 422
 423 static inline bool rt_is_expired(const struct rtable *rth)
 424 {
 425         return rth->rt_genid != rt_genid_ipv4(dev_net(rth->dst.dev));
 426 }
 427
 428 void rt_cache_flush(struct net *net)
 429 {
 430         rt_genid_bump_ipv4(net);
 431 }
 432
 433 static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst,
 434                                            struct sk_buff *skb,
 435                                            const void *daddr)
 436 {
 437         struct net_device *dev = dst->dev;
 438         const __be32 *pkey = daddr;
 439         const struct rtable *rt;
 440         struct neighbour *n;
 441
 442         rt = (const struct rtable *) dst;
 443         if (rt->rt_gateway)
 444                 pkey = (const __be32 *) &rt->rt_gateway;
 445         else if (skb)
 446                 pkey = &ip_hdr(skb)->daddr;
 447
 448         n = __ipv4_neigh_lookup(dev, *(__force u32 *)pkey);
 449         if (n)
 450                 return n;
 451         return neigh_create(&arp_tbl, pkey, dev);
 452 }
 453
 454 static void ipv4_confirm_neigh(const struct dst_entry *dst, const void *daddr)
 455 {
 456         struct net_device *dev = dst->dev;
 457         const __be32 *pkey = daddr;
 458         const struct rtable *rt;
 459
 460         rt = (const struct rtable *)dst;
 461         if (rt->rt_gateway)
 462                 pkey = (const __be32 *)&rt->rt_gateway;
 463         else if (!daddr ||
 464                  (rt->rt_flags &
 465                   (RTCF_MULTICAST | RTCF_BROADCAST | RTCF_LOCAL)))
 466                 return;
 467
 468         __ipv4_confirm_neigh(dev, *(__force u32 *)pkey);
 469 }
 470
 471 #define IP_IDENTS_SZ 2048u
 472
 473 static atomic_t *ip_idents __read_mostly;
 474 static u32 *ip_tstamps __read_mostly;
 475
 476 /* In order to protect privacy, we add a perturbation to identifiers
 477  * if one generator is seldom used. This makes hard for an attacker
 478  * to infer how many packets were sent between two points in time.
 479  */
 480 u32 ip_idents_reserve(u32 hash, int segs)
 481 {
 482         u32 *p_tstamp = ip_tstamps + hash % IP_IDENTS_SZ;
 483         atomic_t *p_id = ip_idents + hash % IP_IDENTS_SZ;
 484         u32 old = READ_ONCE(*p_tstamp);
 485         u32 now = (u32)jiffies;
 486         u32 new, delta = 0;
 487
 488         if (old != now && cmpxchg(p_tstamp, old, now) == old)
 489                 delta = prandom_u32_max(now - old);
 490
 491         /* Do not use atomic_add_return() as it makes UBSAN unhappy */
 492         do {
 493                 old = (u32)atomic_read(p_id);
 494                 new = old + delta + segs;
 495         } while (atomic_cmpxchg(p_id, old, new) != old);
 496
 497         return new - segs;
 498 }
 499 EXPORT_SYMBOL(ip_idents_reserve);
 500
 501 void __ip_select_ident(struct net *net, struct iphdr *iph, int segs)
 502 {
 503         static u32 ip_idents_hashrnd __read_mostly;
 504         u32 hash, id;
 505
 506         net_get_random_once(&ip_idents_hashrnd, sizeof(ip_idents_hashrnd));
 507
 508         hash = jhash_3words((__force u32)iph->daddr,
 509                             (__force u32)iph->saddr,
 510                             iph->protocol ^ net_hash_mix(net),
 511                             ip_idents_hashrnd);
 512         id = ip_idents_reserve(hash, segs);
 513         iph->id = htons(id);
 514 }
 515 EXPORT_SYMBOL(__ip_select_ident);
 516
 517 static void __build_flow_key(const struct net *net, struct flowi4 *fl4,
 518                              const struct sock *sk,
 519                              const struct iphdr *iph,
 520                              int oif, u8 tos,
 521                              u8 prot, u32 mark, int flow_flags)
 522 {
 523         if (sk) {
 524                 const struct inet_sock *inet = inet_sk(sk);
 525
 526                 oif = sk->sk_bound_dev_if;
 527                 mark = sk->sk_mark;
 528                 tos = RT_CONN_FLAGS(sk);
 529                 prot = inet->hdrincl ? IPPROTO_RAW : sk->sk_protocol;
 530         }
 531         flowi4_init_output(fl4, oif, mark, tos,
 532                            RT_SCOPE_UNIVERSE, prot,
 533                            flow_flags,
 534                            iph->daddr, iph->saddr, 0, 0,
 535                            sock_net_uid(net, sk));
 536 }
 537
 538 static void build_skb_flow_key(struct flowi4 *fl4, const struct sk_buff *skb,
 539                                const struct sock *sk)
 540 {
 541         const struct net *net = dev_net(skb->dev);
 542         const struct iphdr *iph = ip_hdr(skb);
 543         int oif = skb->dev->ifindex;
 544         u8 tos = RT_TOS(iph->tos);
 545         u8 prot = iph->protocol;
 546         u32 mark = skb->mark;
 547
 548         __build_flow_key(net, fl4, sk, iph, oif, tos, prot, mark, 0);
 549 }
 550
 551 static void build_sk_flow_key(struct flowi4 *fl4, const struct sock *sk)
 552 {
 553         const struct inet_sock *inet = inet_sk(sk);
 554         const struct ip_options_rcu *inet_opt;
 555         __be32 daddr = inet->inet_daddr;
 556
 557         rcu_read_lock();
 558         inet_opt = rcu_dereference(inet->inet_opt);
 559         if (inet_opt && inet_opt->opt.srr)
 560                 daddr = inet_opt->opt.faddr;
 561         flowi4_init_output(fl4, sk->sk_bound_dev_if, sk->sk_mark,
 562                            RT_CONN_FLAGS(sk), RT_SCOPE_UNIVERSE,
 563                            inet->hdrincl ? IPPROTO_RAW : sk->sk_protocol,
 564                            inet_sk_flowi_flags(sk),
 565                            daddr, inet->inet_saddr, 0, 0, sk->sk_uid);
 566         rcu_read_unlock();
 567 }
 568
 569 static void ip_rt_build_flow_key(struct flowi4 *fl4, const struct sock *sk,
 570                                  const struct sk_buff *skb)
 571 {
 572         if (skb)
 573                 build_skb_flow_key(fl4, skb, sk);
 574         else
 575                 build_sk_flow_key(fl4, sk);
 576 }
 577
 578 static DEFINE_SPINLOCK(fnhe_lock);
 579
 580 static void fnhe_flush_routes(struct fib_nh_exception *fnhe)
 581 {
 582         struct rtable *rt;
 583
 584         rt = rcu_dereference(fnhe->fnhe_rth_input);
 585         if (rt) {
 586                 RCU_INIT_POINTER(fnhe->fnhe_rth_input, NULL);
 587                 dst_dev_put(&rt->dst);
 588                 dst_release(&rt->dst);
 589         }
 590         rt = rcu_dereference(fnhe->fnhe_rth_output);
 591         if (rt) {
 592                 RCU_INIT_POINTER(fnhe->fnhe_rth_output, NULL);
 593                 dst_dev_put(&rt->dst);
 594                 dst_release(&rt->dst);
 595         }
 596 }
 597
 598 static struct fib_nh_exception *fnhe_oldest(struct fnhe_hash_bucket *hash)
 599 {
 600         struct fib_nh_exception *fnhe, *oldest;
 601
 602         oldest = rcu_dereference(hash->chain);
 603         for (fnhe = rcu_dereference(oldest->fnhe_next); fnhe;
 604              fnhe = rcu_dereference(fnhe->fnhe_next)) {
 605                 if (time_before(fnhe->fnhe_stamp, oldest->fnhe_stamp))
 606                         oldest = fnhe;
 607         }
 608         fnhe_flush_routes(oldest);
 609         return oldest;
 610 }
 611
 612 static inline u32 fnhe_hashfun(__be32 daddr)
 613 {
 614         static u32 fnhe_hashrnd __read_mostly;
 615         u32 hval;
 616
 617         net_get_random_once(&fnhe_hashrnd, sizeof(fnhe_hashrnd));
 618         hval = jhash_1word((__force u32) daddr, fnhe_hashrnd);
 619         return hash_32(hval, FNHE_HASH_SHIFT);
 620 }
 621
 622 static void fill_route_from_fnhe(struct rtable *rt, struct fib_nh_exception *fnhe)
 623 {
 624         rt->rt_pmtu = fnhe->fnhe_pmtu;
 625         rt->rt_mtu_locked = fnhe->fnhe_mtu_locked;
 626         rt->dst.expires = fnhe->fnhe_expires;
 627
 628         if (fnhe->fnhe_gw) {
 629                 rt->rt_flags |= RTCF_REDIRECTED;
 630                 rt->rt_gateway = fnhe->fnhe_gw;
 631                 rt->rt_uses_gateway = 1;
 632         }
 633 }
 634
 635 static void update_or_create_fnhe(struct fib_nh *nh, __be32 daddr, __be32 gw,
 636                                   u32 pmtu, bool lock, unsigned long expires)
 637 {
 638         struct fnhe_hash_bucket *hash;
 639         struct fib_nh_exception *fnhe;
 640         struct rtable *rt;
 641         u32 genid, hval;
 642         unsigned int i;
 643         int depth;
 644
 645         genid = fnhe_genid(dev_net(nh->nh_dev));
 646         hval = fnhe_hashfun(daddr);
 647
 648         spin_lock_bh(&fnhe_lock);
 649
 650         hash = rcu_dereference(nh->nh_exceptions);
 651         if (!hash) {
 652                 hash = kcalloc(FNHE_HASH_SIZE, sizeof(*hash), GFP_ATOMIC);
 653                 if (!hash)
 654                         goto out_unlock;
 655                 rcu_assign_pointer(nh->nh_exceptions, hash);
 656         }
 657
 658         hash += hval;
 659
 660         depth = 0;
 661         for (fnhe = rcu_dereference(hash->chain); fnhe;
 662              fnhe = rcu_dereference(fnhe->fnhe_next)) {
 663                 if (fnhe->fnhe_daddr == daddr)
 664                         break;
 665                 depth++;
 666         }
 667
 668         if (fnhe) {
 669                 if (fnhe->fnhe_genid != genid)
 670                         fnhe->fnhe_genid = genid;
 671                 if (gw)
 672                         fnhe->fnhe_gw = gw;
 673                 if (pmtu) {
 674                         fnhe->fnhe_pmtu = pmtu;
 675                         fnhe->fnhe_mtu_locked = lock;
 676                 }
 677                 fnhe->fnhe_expires = max(1UL, expires);
 678                 /* Update all cached dsts too */
 679                 rt = rcu_dereference(fnhe->fnhe_rth_input);
 680                 if (rt)
 681                         fill_route_from_fnhe(rt, fnhe);
 682                 rt = rcu_dereference(fnhe->fnhe_rth_output);
 683                 if (rt)
 684                         fill_route_from_fnhe(rt, fnhe);
 685         } else {
 686                 if (depth > FNHE_RECLAIM_DEPTH)
 687                         fnhe = fnhe_oldest(hash);
 688                 else {
 689                         fnhe = kzalloc(sizeof(*fnhe), GFP_ATOMIC);
 690                         if (!fnhe)
 691                                 goto out_unlock;
 692
 693                         fnhe->fnhe_next = hash->chain;
 694                         rcu_assign_pointer(hash->chain, fnhe);
 695                 }
 696                 fnhe->fnhe_genid = genid;
 697                 fnhe->fnhe_daddr = daddr;
 698                 fnhe->fnhe_gw = gw;
 699                 fnhe->fnhe_pmtu = pmtu;
 700                 fnhe->fnhe_mtu_locked = lock;
 701                 fnhe->fnhe_expires = max(1UL, expires);
 702
 703                 /* Exception created; mark the cached routes for the nexthop
 704                  * stale, so anyone caching it rechecks if this exception
 705                  * applies to them.
 706                  */
 707                 rt = rcu_dereference(nh->nh_rth_input);
 708                 if (rt)
 709                         rt->dst.obsolete = DST_OBSOLETE_KILL;
 710
 711                 for_each_possible_cpu(i) {
 712                         struct rtable __rcu **prt;
 713                         prt = per_cpu_ptr(nh->nh_pcpu_rth_output, i);
 714                         rt = rcu_dereference(*prt);
 715                         if (rt)
 716                                 rt->dst.obsolete = DST_OBSOLETE_KILL;
 717                 }
 718         }
 719
 720         fnhe->fnhe_stamp = jiffies;
 721
 722 out_unlock:
 723         spin_unlock_bh(&fnhe_lock);
 724 }
 725
 726 static void __ip_do_redirect(struct rtable *rt, struct sk_buff *skb, struct flowi4 *fl4,
 727                              bool kill_route)
 728 {
 729         __be32 new_gw = icmp_hdr(skb)->un.gateway;
 730         __be32 old_gw = ip_hdr(skb)->saddr;
 731         struct net_device *dev = skb->dev;
 732         struct in_device *in_dev;
 733         struct fib_result res;
 734         struct neighbour *n;
 735         struct net *net;
 736
 737         switch (icmp_hdr(skb)->code & 7) {
 738         case ICMP_REDIR_NET:
 739         case ICMP_REDIR_NETTOS:
 740         case ICMP_REDIR_HOST:
 741         case ICMP_REDIR_HOSTTOS:
 742                 break;
 743
 744         default:
 745                 return;
 746         }
 747
 748         if (rt->rt_gateway != old_gw)
 749                 return;
 750
 751         in_dev = __in_dev_get_rcu(dev);
 752         if (!in_dev)
 753                 return;
 754
 755         net = dev_net(dev);
 756         if (new_gw == old_gw || !IN_DEV_RX_REDIRECTS(in_dev) ||
 757             ipv4_is_multicast(new_gw) || ipv4_is_lbcast(new_gw) ||
 758             ipv4_is_zeronet(new_gw))
 759                 goto reject_redirect;
 760
 761         if (!IN_DEV_SHARED_MEDIA(in_dev)) {
 762                 if (!inet_addr_onlink(in_dev, new_gw, old_gw))
 763                         goto reject_redirect;
 764                 if (IN_DEV_SEC_REDIRECTS(in_dev) && ip_fib_check_default(new_gw, dev))
 765                         goto reject_redirect;
 766         } else {
 767                 if (inet_addr_type(net, new_gw) != RTN_UNICAST)
 768                         goto reject_redirect;
 769         }
 770
 771         n = __ipv4_neigh_lookup(rt->dst.dev, new_gw);
 772         if (!n)
 773                 n = neigh_create(&arp_tbl, &new_gw, rt->dst.dev);
 774         if (!IS_ERR(n)) {
 775                 if (!(n->nud_state & NUD_VALID)) {
 776                         neigh_event_send(n, NULL);
 777                 } else {
 778                         if (fib_lookup(net, fl4, &res, 0) == 0) {
 779                                 struct fib_nh *nh = &FIB_RES_NH(res);
 780
 781                                 update_or_create_fnhe(nh, fl4->daddr, new_gw,
 782                                                 0, false,
 783                                                 jiffies + ip_rt_gc_timeout);
 784                         }
 785                         if (kill_route)
 786                                 rt->dst.obsolete = DST_OBSOLETE_KILL;
 787                         call_netevent_notifiers(NETEVENT_NEIGH_UPDATE, n);
 788                 }
 789                 neigh_release(n);
 790         }
 791         return;
 792
 793 reject_redirect:
 794 #ifdef CONFIG_IP_ROUTE_VERBOSE
 795         if (IN_DEV_LOG_MARTIANS(in_dev)) {
 796                 const struct iphdr *iph = (const struct iphdr *) skb->data;
 797                 __be32 daddr = iph->daddr;
 798                 __be32 saddr = iph->saddr;
 799
 800                 net_info_ratelimited("Redirect from %pI4 on %s about %pI4 ignored\n"
 801                                      "  Advised path = %pI4 -> %pI4\n",
 802                                      &old_gw, dev->name, &new_gw,
 803                                      &saddr, &daddr);
 804         }
 805 #endif
 806         ;
 807 }
 808
 809 static void ip_do_redirect(struct dst_entry *dst, struct sock *sk, struct sk_buff *skb)
 810 {
 811         struct rtable *rt;
 812         struct flowi4 fl4;
 813         const struct iphdr *iph = (const struct iphdr *) skb->data;
 814         struct net *net = dev_net(skb->dev);
 815         int oif = skb->dev->ifindex;
 816         u8 tos = RT_TOS(iph->tos);
 817         u8 prot = iph->protocol;
 818         u32 mark = skb->mark;
 819
 820         rt = (struct rtable *) dst;
 821
 822         __build_flow_key(net, &fl4, sk, iph, oif, tos, prot, mark, 0);
 823         __ip_do_redirect(rt, skb, &fl4, true);
 824 }
 825
 826 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst)
 827 {
 828         struct rtable *rt = (struct rtable *)dst;
 829         struct dst_entry *ret = dst;
 830
 831         if (rt) {
 832                 if (dst->obsolete > 0) {
 833                         ip_rt_put(rt);
 834                         ret = NULL;
 835                 } else if ((rt->rt_flags & RTCF_REDIRECTED) ||
 836                            rt->dst.expires) {
 837                         ip_rt_put(rt);
 838                         ret = NULL;
 839                 }
 840         }
 841         return ret;
 842 }
 843
 844 /*
 845  * Algorithm:
 846  *      1. The first ip_rt_redirect_number redirects are sent
 847  *         with exponential backoff, then we stop sending them at all,
 848  *         assuming that the host ignores our redirects.
 849  *      2. If we did not see packets requiring redirects
 850  *         during ip_rt_redirect_silence, we assume that the host
 851  *         forgot redirected route and start to send redirects again.
 852  *
 853  * This algorithm is much cheaper and more intelligent than dumb load limiting
 854  * in icmp.c.
 855  *
 856  * NOTE. Do not forget to inhibit load limiting for redirects (redundant)
 857  * and "frag. need" (breaks PMTU discovery) in icmp.c.
 858  */
 859
 860 void ip_rt_send_redirect(struct sk_buff *skb)
 861 {
 862         struct rtable *rt = skb_rtable(skb);
 863         struct in_device *in_dev;
 864         struct inet_peer *peer;
 865         struct net *net;
 866         int log_martians;
 867         int vif;
 868
 869         rcu_read_lock();
 870         in_dev = __in_dev_get_rcu(rt->dst.dev);
 871         if (!in_dev || !IN_DEV_TX_REDIRECTS(in_dev)) {
 872                 rcu_read_unlock();
 873                 return;
 874         }
 875         log_martians = IN_DEV_LOG_MARTIANS(in_dev);
 876         vif = l3mdev_master_ifindex_rcu(rt->dst.dev);
 877         rcu_read_unlock();
 878
 879         net = dev_net(rt->dst.dev);
 880         peer = inet_getpeer_v4(net->ipv4.peers, ip_hdr(skb)->saddr, vif, 1);
 881         if (!peer) {
 882                 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST,
 883                           rt_nexthop(rt, ip_hdr(skb)->daddr));
 884                 return;
 885         }
 886
 887         /* No redirected packets during ip_rt_redirect_silence;
 888          * reset the algorithm.
 889          */
 890         if (time_after(jiffies, peer->rate_last + ip_rt_redirect_silence)) {
 891                 peer->rate_tokens = 0;
 892                 peer->n_redirects = 0;
 893         }
 894
 895         /* Too many ignored redirects; do not send anything
 896          * set dst.rate_last to the last seen redirected packet.
 897          */
 898         if (peer->n_redirects >= ip_rt_redirect_number) {
 899                 peer->rate_last = jiffies;
 900                 goto out_put_peer;
 901         }
 902
 903         /* Check for load limit; set rate_last to the latest sent
 904          * redirect.
 905          */
 906         if (peer->rate_tokens == 0 ||
 907             time_after(jiffies,
 908                        (peer->rate_last +
 909                         (ip_rt_redirect_load << peer->rate_tokens)))) {
 910                 __be32 gw = rt_nexthop(rt, ip_hdr(skb)->daddr);
 911
 912                 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST, gw);
 913                 peer->rate_last = jiffies;
 914                 ++peer->rate_tokens;
 915                 ++peer->n_redirects;
 916 #ifdef CONFIG_IP_ROUTE_VERBOSE
 917                 if (log_martians &&
 918                     peer->rate_tokens == ip_rt_redirect_number)
 919                         net_warn_ratelimited("host %pI4/if%d ignores redirects for %pI4 to %pI4\n",
 920                                              &ip_hdr(skb)->saddr, inet_iif(skb),
 921                                              &ip_hdr(skb)->daddr, &gw);
 922 #endif
 923         }
 924 out_put_peer:
 925         inet_putpeer(peer);
 926 }
 927
 928 static int ip_error(struct sk_buff *skb)
 929 {
 930         struct rtable *rt = skb_rtable(skb);
 931         struct net_device *dev = skb->dev;
 932         struct in_device *in_dev;
 933         struct inet_peer *peer;
 934         unsigned long now;
 935         struct net *net;
 936         bool send;
 937         int code;
 938
 939         if (netif_is_l3_master(skb->dev)) {
 940                 dev = __dev_get_by_index(dev_net(skb->dev), IPCB(skb)->iif);
 941                 if (!dev)
 942                         goto out;
 943         }
 944
 945         in_dev = __in_dev_get_rcu(dev);
 946
 947         /* IP on this device is disabled. */
 948         if (!in_dev)
 949                 goto out;
 950
 951         net = dev_net(rt->dst.dev);
 952         if (!IN_DEV_FORWARD(in_dev)) {
 953                 switch (rt->dst.error) {
 954                 case EHOSTUNREACH:
 955                         __IP_INC_STATS(net, IPSTATS_MIB_INADDRERRORS);
 956                         break;
 957
 958                 case ENETUNREACH:
 959                         __IP_INC_STATS(net, IPSTATS_MIB_INNOROUTES);
 960                         break;
 961                 }
 962                 goto out;
 963         }
 964
 965         switch (rt->dst.error) {
 966         case EINVAL:
 967         default:
 968                 goto out;
 969         case EHOSTUNREACH:
 970                 code = ICMP_HOST_UNREACH;
 971                 break;
 972         case ENETUNREACH:
 973                 code = ICMP_NET_UNREACH;
 974                 __IP_INC_STATS(net, IPSTATS_MIB_INNOROUTES);
 975                 break;
 976         case EACCES:
 977                 code = ICMP_PKT_FILTERED;
 978                 break;
 979         }
 980
 981         peer = inet_getpeer_v4(net->ipv4.peers, ip_hdr(skb)->saddr,
 982                                l3mdev_master_ifindex(skb->dev), 1);
 983
 984         send = true;
 985         if (peer) {
 986                 now = jiffies;
 987                 peer->rate_tokens += now - peer->rate_last;
 988                 if (peer->rate_tokens > ip_rt_error_burst)
 989                         peer->rate_tokens = ip_rt_error_burst;
 990                 peer->rate_last = now;
 991                 if (peer->rate_tokens >= ip_rt_error_cost)
 992                         peer->rate_tokens -= ip_rt_error_cost;
 993                 else
 994                         send = false;
 995                 inet_putpeer(peer);
 996         }
 997         if (send)
 998                 icmp_send(skb, ICMP_DEST_UNREACH, code, 0);
 999
1000 out:    kfree_skb(skb);
1001         return 0;
1002 }
1003
1004 static void __ip_rt_update_pmtu(struct rtable *rt, struct flowi4 *fl4, u32 mtu)
1005 {
1006         struct dst_entry *dst = &rt->dst;
1007         u32 old_mtu = ipv4_mtu(dst);
1008         struct fib_result res;
1009         bool lock = false;
1010
1011         if (ip_mtu_locked(dst))
1012                 return;
1013
1014         if (old_mtu < mtu)
1015                 return;
1016
1017         if (mtu < ip_rt_min_pmtu) {
1018                 lock = true;
1019                 mtu = min(old_mtu, ip_rt_min_pmtu);
1020         }
1021
1022         if (rt->rt_pmtu == mtu && !lock &&
1023             time_before(jiffies, dst->expires - ip_rt_mtu_expires / 2))
1024                 return;
1025
1026         rcu_read_lock();
1027         if (fib_lookup(dev_net(dst->dev), fl4, &res, 0) == 0) {
1028                 struct fib_nh *nh = &FIB_RES_NH(res);
1029
1030                 update_or_create_fnhe(nh, fl4->daddr, 0, mtu, lock,
1031                                       jiffies + ip_rt_mtu_expires);
1032         }
1033         rcu_read_unlock();
1034 }
1035
1036 static void ip_rt_update_pmtu(struct dst_entry *dst, struct sock *sk,
1037                               struct sk_buff *skb, u32 mtu)
1038 {
1039         struct rtable *rt = (struct rtable *) dst;
1040         struct flowi4 fl4;
1041
1042         ip_rt_build_flow_key(&fl4, sk, skb);
1043         __ip_rt_update_pmtu(rt, &fl4, mtu);
1044 }
1045
1046 void ipv4_update_pmtu(struct sk_buff *skb, struct net *net, u32 mtu,
1047                       int oif, u8 protocol)
1048 {
1049         const struct iphdr *iph = (const struct iphdr *) skb->data;
1050         struct flowi4 fl4;
1051         struct rtable *rt;
1052         u32 mark = IP4_REPLY_MARK(net, skb->mark);
1053
1054         __build_flow_key(net, &fl4, NULL, iph, oif,
1055                          RT_TOS(iph->tos), protocol, mark, 0);
1056         rt = __ip_route_output_key(net, &fl4);
1057         if (!IS_ERR(rt)) {
1058                 __ip_rt_update_pmtu(rt, &fl4, mtu);
1059                 ip_rt_put(rt);
1060         }
1061 }
1062 EXPORT_SYMBOL_GPL(ipv4_update_pmtu);
1063
1064 static void __ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu)
1065 {
1066         const struct iphdr *iph = (const struct iphdr *) skb->data;
1067         struct flowi4 fl4;
1068         struct rtable *rt;
1069
1070         __build_flow_key(sock_net(sk), &fl4, sk, iph, 0, 0, 0, 0, 0);
1071
1072         if (!fl4.flowi4_mark)
1073                 fl4.flowi4_mark = IP4_REPLY_MARK(sock_net(sk), skb->mark);
1074
1075         rt = __ip_route_output_key(sock_net(sk), &fl4);
1076         if (!IS_ERR(rt)) {
1077                 __ip_rt_update_pmtu(rt, &fl4, mtu);
1078                 ip_rt_put(rt);
1079         }
1080 }
1081
1082 void ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu)
1083 {
1084         const struct iphdr *iph = (const struct iphdr *) skb->data;
1085         struct flowi4 fl4;
1086         struct rtable *rt;
1087         struct dst_entry *odst = NULL;
1088         bool new = false;
1089         struct net *net = sock_net(sk);
1090
1091         bh_lock_sock(sk);
1092
1093         if (!ip_sk_accept_pmtu(sk))
1094                 goto out;
1095
1096         odst = sk_dst_get(sk);
1097
1098         if (sock_owned_by_user(sk) || !odst) {
1099                 __ipv4_sk_update_pmtu(skb, sk, mtu);
1100                 goto out;
1101         }
1102
1103         __build_flow_key(net, &fl4, sk, iph, 0, 0, 0, 0, 0);
1104
1105         rt = (struct rtable *)odst;
1106         if (odst->obsolete && !odst->ops->check(odst, 0)) {
1107                 rt = ip_route_output_flow(sock_net(sk), &fl4, sk);
1108                 if (IS_ERR(rt))
1109                         goto out;
1110
1111                 new = true;
1112         }
1113
1114         __ip_rt_update_pmtu((struct rtable *) xfrm_dst_path(&rt->dst), &fl4, mtu);
1115
1116         if (!dst_check(&rt->dst, 0)) {
1117                 if (new)
1118                         dst_release(&rt->dst);
1119
1120                 rt = ip_route_output_flow(sock_net(sk), &fl4, sk);
1121                 if (IS_ERR(rt))
1122                         goto out;
1123
1124                 new = true;
1125         }
1126
1127         if (new)
1128                 sk_dst_set(sk, &rt->dst);
1129
1130 out:
1131         bh_unlock_sock(sk);
1132         dst_release(odst);
1133 }
1134 EXPORT_SYMBOL_GPL(ipv4_sk_update_pmtu);
1135
1136 void ipv4_redirect(struct sk_buff *skb, struct net *net,
1137                    int oif, u8 protocol)
1138 {
1139         const struct iphdr *iph = (const struct iphdr *) skb->data;
1140         struct flowi4 fl4;
1141         struct rtable *rt;
1142
1143         __build_flow_key(net, &fl4, NULL, iph, oif,
1144                          RT_TOS(iph->tos), protocol, 0, 0);
1145         rt = __ip_route_output_key(net, &fl4);
1146         if (!IS_ERR(rt)) {
1147                 __ip_do_redirect(rt, skb, &fl4, false);
1148                 ip_rt_put(rt);
1149         }
1150 }
1151 EXPORT_SYMBOL_GPL(ipv4_redirect);
1152
1153 void ipv4_sk_redirect(struct sk_buff *skb, struct sock *sk)
1154 {
1155         const struct iphdr *iph = (const struct iphdr *) skb->data;
1156         struct flowi4 fl4;
1157         struct rtable *rt;
1158         struct net *net = sock_net(sk);
1159
1160         __build_flow_key(net, &fl4, sk, iph, 0, 0, 0, 0, 0);
1161         rt = __ip_route_output_key(net, &fl4);
1162         if (!IS_ERR(rt)) {
1163                 __ip_do_redirect(rt, skb, &fl4, false);
1164                 ip_rt_put(rt);
1165         }
1166 }
1167 EXPORT_SYMBOL_GPL(ipv4_sk_redirect);
1168
1169 static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie)
1170 {
1171         struct rtable *rt = (struct rtable *) dst;
1172
1173         /* All IPV4 dsts are created with ->obsolete set to the value
1174          * DST_OBSOLETE_FORCE_CHK which forces validation calls down
1175          * into this function always.
1176          *
1177          * When a PMTU/redirect information update invalidates a route,
1178          * this is indicated by setting obsolete to DST_OBSOLETE_KILL or
1179          * DST_OBSOLETE_DEAD by dst_free().
1180          */
1181         if (dst->obsolete != DST_OBSOLETE_FORCE_CHK || rt_is_expired(rt))
1182                 return NULL;
1183         return dst;
1184 }
1185
1186 static void ipv4_send_dest_unreach(struct sk_buff *skb)
1187 {
1188         struct ip_options opt;
1189         int res;
1190
1191         /* Recompile ip options since IPCB may not be valid anymore.
1192          * Also check we have a reasonable ipv4 header.
1193          */
1194         if (!pskb_network_may_pull(skb, sizeof(struct iphdr)) ||
1195             ip_hdr(skb)->version != 4 || ip_hdr(skb)->ihl < 5)
1196                 return;
1197
1198         memset(&opt, 0, sizeof(opt));
1199         if (ip_hdr(skb)->ihl > 5) {
1200                 if (!pskb_network_may_pull(skb, ip_hdr(skb)->ihl * 4))
1201                         return;
1202                 opt.optlen = ip_hdr(skb)->ihl * 4 - sizeof(struct iphdr);
1203
1204                 rcu_read_lock();
1205                 res = __ip_options_compile(dev_net(skb->dev), &opt, skb, NULL);
1206                 rcu_read_unlock();
1207
1208                 if (res)
1209                         return;
1210         }
1211         __icmp_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0, &opt);
1212 }
1213
1214 static void ipv4_link_failure(struct sk_buff *skb)
1215 {
1216         struct rtable *rt;
1217
1218         ipv4_send_dest_unreach(skb);
1219
1220         rt = skb_rtable(skb);
1221         if (rt)
1222                 dst_set_expires(&rt->dst, 0);
1223 }
1224
1225 static int ip_rt_bug(struct net *net, struct sock *sk, struct sk_buff *skb)
1226 {
1227         pr_debug("%s: %pI4 -> %pI4, %s\n",
1228                  __func__, &ip_hdr(skb)->saddr, &ip_hdr(skb)->daddr,
1229                  skb->dev ? skb->dev->name : "?");
1230         kfree_skb(skb);
1231         WARN_ON(1);
1232         return 0;
1233 }
1234
1235 /*
1236    We do not cache source address of outgoing interface,
1237    because it is used only by IP RR, TS and SRR options,
1238    so that it out of fast path.
1239
1240    BTW remember: "addr" is allowed to be not aligned
1241    in IP options!
1242  */
1243
1244 void ip_rt_get_source(u8 *addr, struct sk_buff *skb, struct rtable *rt)
1245 {
1246         __be32 src;
1247
1248         if (rt_is_output_route(rt))
1249                 src = ip_hdr(skb)->saddr;
1250         else {
1251                 struct fib_result res;
1252                 struct iphdr *iph = ip_hdr(skb);
1253                 struct flowi4 fl4 = {
1254                         .daddr = iph->daddr,
1255                         .saddr = iph->saddr,
1256                         .flowi4_tos = RT_TOS(iph->tos),
1257                         .flowi4_oif = rt->dst.dev->ifindex,
1258                         .flowi4_iif = skb->dev->ifindex,
1259                         .flowi4_mark = skb->mark,
1260                 };
1261
1262                 rcu_read_lock();
1263                 if (fib_lookup(dev_net(rt->dst.dev), &fl4, &res, 0) == 0)
1264                         src = FIB_RES_PREFSRC(dev_net(rt->dst.dev), res);
1265                 else
1266                         src = inet_select_addr(rt->dst.dev,
1267                                                rt_nexthop(rt, iph->daddr),
1268                                                RT_SCOPE_UNIVERSE);
1269                 rcu_read_unlock();
1270         }
1271         memcpy(addr, &src, 4);
1272 }
1273
1274 #ifdef CONFIG_IP_ROUTE_CLASSID
1275 static void set_class_tag(struct rtable *rt, u32 tag)
1276 {
1277         if (!(rt->dst.tclassid & 0xFFFF))
1278                 rt->dst.tclassid |= tag & 0xFFFF;
1279         if (!(rt->dst.tclassid & 0xFFFF0000))
1280                 rt->dst.tclassid |= tag & 0xFFFF0000;
1281 }
1282 #endif
1283
1284 static unsigned int ipv4_default_advmss(const struct dst_entry *dst)
1285 {
1286         unsigned int header_size = sizeof(struct tcphdr) + sizeof(struct iphdr);
1287         unsigned int advmss = max_t(unsigned int, ipv4_mtu(dst) - header_size,
1288                                     ip_rt_min_advmss);
1289
1290         return min(advmss, IPV4_MAX_PMTU - header_size);
1291 }
1292
1293 static unsigned int ipv4_mtu(const struct dst_entry *dst)
1294 {
1295         const struct rtable *rt = (const struct rtable *) dst;
1296         unsigned int mtu = rt->rt_pmtu;
1297
1298         if (!mtu || time_after_eq(jiffies, rt->dst.expires))
1299                 mtu = dst_metric_raw(dst, RTAX_MTU);
1300
1301         if (mtu)
1302                 return mtu;
1303
1304         mtu = READ_ONCE(dst->dev->mtu);
1305
1306         if (unlikely(ip_mtu_locked(dst))) {
1307                 if (rt->rt_uses_gateway && mtu > 576)
1308                         mtu = 576;
1309         }
1310
1311         mtu = min_t(unsigned int, mtu, IP_MAX_MTU);
1312
1313         return mtu - lwtunnel_headroom(dst->lwtstate, mtu);
1314 }
1315
1316 static void ip_del_fnhe(struct fib_nh *nh, __be32 daddr)
1317 {
1318         struct fnhe_hash_bucket *hash;
1319         struct fib_nh_exception *fnhe, __rcu **fnhe_p;
1320         u32 hval = fnhe_hashfun(daddr);
1321
1322         spin_lock_bh(&fnhe_lock);
1323
1324         hash = rcu_dereference_protected(nh->nh_exceptions,
1325                                          lockdep_is_held(&fnhe_lock));
1326         hash += hval;
1327
1328         fnhe_p = &hash->chain;
1329         fnhe = rcu_dereference_protected(*fnhe_p, lockdep_is_held(&fnhe_lock));
1330         while (fnhe) {
1331                 if (fnhe->fnhe_daddr == daddr) {
1332                         rcu_assign_pointer(*fnhe_p, rcu_dereference_protected(
1333                                 fnhe->fnhe_next, lockdep_is_held(&fnhe_lock)));
1334                         /* set fnhe_daddr to 0 to ensure it won't bind with
1335                          * new dsts in rt_bind_exception().
1336                          */
1337                         fnhe->fnhe_daddr = 0;
1338                         fnhe_flush_routes(fnhe);
1339                         kfree_rcu(fnhe, rcu);
1340                         break;
1341                 }
1342                 fnhe_p = &fnhe->fnhe_next;
1343                 fnhe = rcu_dereference_protected(fnhe->fnhe_next,
1344                                                  lockdep_is_held(&fnhe_lock));
1345         }
1346
1347         spin_unlock_bh(&fnhe_lock);
1348 }
1349
1350 static struct fib_nh_exception *find_exception(struct fib_nh *nh, __be32 daddr)
1351 {
1352         struct fnhe_hash_bucket *hash = rcu_dereference(nh->nh_exceptions);
1353         struct fib_nh_exception *fnhe;
1354         u32 hval;
1355
1356         if (!hash)
1357                 return NULL;
1358
1359         hval = fnhe_hashfun(daddr);
1360
1361         for (fnhe = rcu_dereference(hash[hval].chain); fnhe;
1362              fnhe = rcu_dereference(fnhe->fnhe_next)) {
1363                 if (fnhe->fnhe_daddr == daddr) {
1364                         if (fnhe->fnhe_expires &&
1365                             time_after(jiffies, fnhe->fnhe_expires)) {
1366                                 ip_del_fnhe(nh, daddr);
1367                                 break;
1368                         }
1369                         return fnhe;
1370                 }
1371         }
1372         return NULL;
1373 }
1374
1375 /* MTU selection:
1376  * 1. mtu on route is locked - use it
1377  * 2. mtu from nexthop exception
1378  * 3. mtu from egress device
1379  */
1380
1381 u32 ip_mtu_from_fib_result(struct fib_result *res, __be32 daddr)
1382 {
1383         struct fib_info *fi = res->fi;
1384         struct fib_nh *nh = &fi->fib_nh[res->nh_sel];
1385         struct net_device *dev = nh->nh_dev;
1386         u32 mtu = 0;
1387
1388         if (dev_net(dev)->ipv4.sysctl_ip_fwd_use_pmtu ||
1389             fi->fib_metrics->metrics[RTAX_LOCK - 1] & (1 << RTAX_MTU))
1390                 mtu = fi->fib_mtu;
1391
1392         if (likely(!mtu)) {
1393                 struct fib_nh_exception *fnhe;
1394
1395                 fnhe = find_exception(nh, daddr);
1396                 if (fnhe && !time_after_eq(jiffies, fnhe->fnhe_expires))
1397                         mtu = fnhe->fnhe_pmtu;
1398         }
1399
1400         if (likely(!mtu))
1401                 mtu = min(READ_ONCE(dev->mtu), IP_MAX_MTU);
1402
1403         return mtu - lwtunnel_headroom(nh->nh_lwtstate, mtu);
1404 }
1405
1406 static bool rt_bind_exception(struct rtable *rt, struct fib_nh_exception *fnhe,
1407                               __be32 daddr, const bool do_cache)
1408 {
1409         bool ret = false;
1410
1411         spin_lock_bh(&fnhe_lock);
1412
1413         if (daddr == fnhe->fnhe_daddr) {
1414                 struct rtable __rcu **porig;
1415                 struct rtable *orig;
1416                 int genid = fnhe_genid(dev_net(rt->dst.dev));
1417
1418                 if (rt_is_input_route(rt))
1419                         porig = &fnhe->fnhe_rth_input;
1420                 else
1421                         porig = &fnhe->fnhe_rth_output;
1422                 orig = rcu_dereference(*porig);
1423
1424                 if (fnhe->fnhe_genid != genid) {
1425                         fnhe->fnhe_genid = genid;
1426                         fnhe->fnhe_gw = 0;
1427                         fnhe->fnhe_pmtu = 0;
1428                         fnhe->fnhe_expires = 0;
1429                         fnhe->fnhe_mtu_locked = false;
1430                         fnhe_flush_routes(fnhe);
1431                         orig = NULL;
1432                 }
1433                 fill_route_from_fnhe(rt, fnhe);
1434                 if (!rt->rt_gateway)
1435                         rt->rt_gateway = daddr;
1436
1437                 if (do_cache) {
1438                         dst_hold(&rt->dst);
1439                         rcu_assign_pointer(*porig, rt);
1440                         if (orig) {
1441                                 dst_dev_put(&orig->dst);
1442                                 dst_release(&orig->dst);
1443                         }
1444                         ret = true;
1445                 }
1446
1447                 fnhe->fnhe_stamp = jiffies;
1448         }
1449         spin_unlock_bh(&fnhe_lock);
1450
1451         return ret;
1452 }
1453
1454 static bool rt_cache_route(struct fib_nh *nh, struct rtable *rt)
1455 {
1456         struct rtable *orig, *prev, **p;
1457         bool ret = true;
1458
1459         if (rt_is_input_route(rt)) {
1460                 p = (struct rtable **)&nh->nh_rth_input;
1461         } else {
1462                 p = (struct rtable **)raw_cpu_ptr(nh->nh_pcpu_rth_output);
1463         }
1464         orig = *p;
1465
1466         /* hold dst before doing cmpxchg() to avoid race condition
1467          * on this dst
1468          */
1469         dst_hold(&rt->dst);
1470         prev = cmpxchg(p, orig, rt);
1471         if (prev == orig) {
1472                 if (orig) {
1473                         dst_dev_put(&orig->dst);
1474                         dst_release(&orig->dst);
1475                 }
1476         } else {
1477                 dst_release(&rt->dst);
1478                 ret = false;
1479         }
1480
1481         return ret;
1482 }
1483
1484 struct uncached_list {
1485         spinlock_t              lock;
1486         struct list_head        head;
1487 };
1488
1489 static DEFINE_PER_CPU_ALIGNED(struct uncached_list, rt_uncached_list);
1490
1491 void rt_add_uncached_list(struct rtable *rt)
1492 {
1493         struct uncached_list *ul = raw_cpu_ptr(&rt_uncached_list);
1494
1495         rt->rt_uncached_list = ul;
1496
1497         spin_lock_bh(&ul->lock);
1498         list_add_tail(&rt->rt_uncached, &ul->head);
1499         spin_unlock_bh(&ul->lock);
1500 }
1501
1502 void rt_del_uncached_list(struct rtable *rt)
1503 {
1504         if (!list_empty(&rt->rt_uncached)) {
1505                 struct uncached_list *ul = rt->rt_uncached_list;
1506
1507                 spin_lock_bh(&ul->lock);
1508                 list_del(&rt->rt_uncached);
1509                 spin_unlock_bh(&ul->lock);
1510         }
1511 }
1512
1513 static void ipv4_dst_destroy(struct dst_entry *dst)
1514 {
1515         struct rtable *rt = (struct rtable *)dst;
1516
1517         ip_dst_metrics_put(dst);
1518         rt_del_uncached_list(rt);
1519 }
1520
1521 void rt_flush_dev(struct net_device *dev)
1522 {
1523         struct net *net = dev_net(dev);
1524         struct rtable *rt;
1525         int cpu;
1526
1527         for_each_possible_cpu(cpu) {
1528                 struct uncached_list *ul = &per_cpu(rt_uncached_list, cpu);
1529
1530                 spin_lock_bh(&ul->lock);
1531                 list_for_each_entry(rt, &ul->head, rt_uncached) {
1532                         if (rt->dst.dev != dev)
1533                                 continue;
1534                         rt->dst.dev = net->loopback_dev;
1535                         dev_hold(rt->dst.dev);
1536                         dev_put(dev);
1537                 }
1538                 spin_unlock_bh(&ul->lock);
1539         }
1540 }
1541
1542 static bool rt_cache_valid(const struct rtable *rt)
1543 {
1544         return  rt &&
1545                 rt->dst.obsolete == DST_OBSOLETE_FORCE_CHK &&
1546                 !rt_is_expired(rt);
1547 }
1548
1549 static void rt_set_nexthop(struct rtable *rt, __be32 daddr,
1550                            const struct fib_result *res,
1551                            struct fib_nh_exception *fnhe,
1552                            struct fib_info *fi, u16 type, u32 itag,
1553                            const bool do_cache)
1554 {
1555         bool cached = false;
1556
1557         if (fi) {
1558                 struct fib_nh *nh = &FIB_RES_NH(*res);
1559
1560                 if (nh->nh_gw && nh->nh_scope == RT_SCOPE_LINK) {
1561                         rt->rt_gateway = nh->nh_gw;
1562                         rt->rt_uses_gateway = 1;
1563                 }
1564                 ip_dst_init_metrics(&rt->dst, fi->fib_metrics);
1565
1566 #ifdef CONFIG_IP_ROUTE_CLASSID
1567                 rt->dst.tclassid = nh->nh_tclassid;
1568 #endif
1569                 rt->dst.lwtstate = lwtstate_get(nh->nh_lwtstate);
1570                 if (unlikely(fnhe))
1571                         cached = rt_bind_exception(rt, fnhe, daddr, do_cache);
1572                 else if (do_cache)
1573                         cached = rt_cache_route(nh, rt);
1574                 if (unlikely(!cached)) {
1575                         /* Routes we intend to cache in nexthop exception or
1576                          * FIB nexthop have the DST_NOCACHE bit clear.
1577                          * However, if we are unsuccessful at storing this
1578                          * route into the cache we really need to set it.
1579                          */
1580                         if (!rt->rt_gateway)
1581                                 rt->rt_gateway = daddr;
1582                         rt_add_uncached_list(rt);
1583                 }
1584         } else
1585                 rt_add_uncached_list(rt);
1586
1587 #ifdef CONFIG_IP_ROUTE_CLASSID
1588 #ifdef CONFIG_IP_MULTIPLE_TABLES
1589         set_class_tag(rt, res->tclassid);
1590 #endif
1591         set_class_tag(rt, itag);
1592 #endif
1593 }
1594
1595 struct rtable *rt_dst_alloc(struct net_device *dev,
1596                             unsigned int flags, u16 type,
1597                             bool nopolicy, bool noxfrm, bool will_cache)
1598 {
1599         struct rtable *rt;
1600
1601         rt = dst_alloc(&ipv4_dst_ops, dev, 1, DST_OBSOLETE_FORCE_CHK,
1602                        (will_cache ? 0 : DST_HOST) |
1603                        (nopolicy ? DST_NOPOLICY : 0) |
1604                        (noxfrm ? DST_NOXFRM : 0));
1605
1606         if (rt) {
1607                 rt->rt_genid = rt_genid_ipv4(dev_net(dev));
1608                 rt->rt_flags = flags;
1609                 rt->rt_type = type;
1610                 rt->rt_is_input = 0;
1611                 rt->rt_iif = 0;
1612                 rt->rt_pmtu = 0;
1613                 rt->rt_mtu_locked = 0;
1614                 rt->rt_gateway = 0;
1615                 rt->rt_uses_gateway = 0;
1616                 INIT_LIST_HEAD(&rt->rt_uncached);
1617
1618                 rt->dst.output = ip_output;
1619                 if (flags & RTCF_LOCAL)
1620                         rt->dst.input = ip_local_deliver;
1621         }
1622
1623         return rt;
1624 }
1625 EXPORT_SYMBOL(rt_dst_alloc);
1626
1627 /* called in rcu_read_lock() section */
1628 int ip_mc_validate_source(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1629                           u8 tos, struct net_device *dev,
1630                           struct in_device *in_dev, u32 *itag)
1631 {
1632         int err;
1633
1634         /* Primary sanity checks. */
1635         if (!in_dev)
1636                 return -EINVAL;
1637
1638         if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) ||
1639             skb->protocol != htons(ETH_P_IP))
1640                 return -EINVAL;
1641
1642         if (ipv4_is_loopback(saddr) && !IN_DEV_ROUTE_LOCALNET(in_dev))
1643                 return -EINVAL;
1644
1645         if (ipv4_is_zeronet(saddr)) {
1646                 if (!ipv4_is_local_multicast(daddr) &&
1647                     ip_hdr(skb)->protocol != IPPROTO_IGMP)
1648                         return -EINVAL;
1649         } else {
1650                 err = fib_validate_source(skb, saddr, 0, tos, 0, dev,
1651                                           in_dev, itag);
1652                 if (err < 0)
1653                         return err;
1654         }
1655         return 0;
1656 }
1657
1658 /* called in rcu_read_lock() section */
1659 static int ip_route_input_mc(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1660                              u8 tos, struct net_device *dev, int our)
1661 {
1662         struct in_device *in_dev = __in_dev_get_rcu(dev);
1663         unsigned int flags = RTCF_MULTICAST;
1664         struct rtable *rth;
1665         u32 itag = 0;
1666         int err;
1667
1668         err = ip_mc_validate_source(skb, daddr, saddr, tos, dev, in_dev, &itag);
1669         if (err)
1670                 return err;
1671
1672         if (our)
1673                 flags |= RTCF_LOCAL;
1674
1675         rth = rt_dst_alloc(dev_net(dev)->loopback_dev, flags, RTN_MULTICAST,
1676                            IN_DEV_CONF_GET(in_dev, NOPOLICY), false, false);
1677         if (!rth)
1678                 return -ENOBUFS;
1679
1680 #ifdef CONFIG_IP_ROUTE_CLASSID
1681         rth->dst.tclassid = itag;
1682 #endif
1683         rth->dst.output = ip_rt_bug;
1684         rth->rt_is_input= 1;
1685
1686 #ifdef CONFIG_IP_MROUTE
1687         if (!ipv4_is_local_multicast(daddr) && IN_DEV_MFORWARD(in_dev))
1688                 rth->dst.input = ip_mr_input;
1689 #endif
1690         RT_CACHE_STAT_INC(in_slow_mc);
1691
1692         skb_dst_set(skb, &rth->dst);
1693         return 0;
1694 }
1695
1696
1697 static void ip_handle_martian_source(struct net_device *dev,
1698                                      struct in_device *in_dev,
1699                                      struct sk_buff *skb,
1700                                      __be32 daddr,
1701                                      __be32 saddr)
1702 {
1703         RT_CACHE_STAT_INC(in_martian_src);
1704 #ifdef CONFIG_IP_ROUTE_VERBOSE
1705         if (IN_DEV_LOG_MARTIANS(in_dev) && net_ratelimit()) {
1706                 /*
1707                  *      RFC1812 recommendation, if source is martian,
1708                  *      the only hint is MAC header.
1709                  */
1710                 pr_warn("martian source %pI4 from %pI4, on dev %s\n",
1711                         &daddr, &saddr, dev->name);
1712                 if (dev->hard_header_len && skb_mac_header_was_set(skb)) {
1713                         print_hex_dump(KERN_WARNING, "ll header: ",
1714                                        DUMP_PREFIX_OFFSET, 16, 1,
1715                                        skb_mac_header(skb),
1716                                        dev->hard_header_len, false);
1717                 }
1718         }
1719 #endif
1720 }
1721
1722 /* called in rcu_read_lock() section */
1723 static int __mkroute_input(struct sk_buff *skb,
1724                            const struct fib_result *res,
1725                            struct in_device *in_dev,
1726                            __be32 daddr, __be32 saddr, u32 tos)
1727 {
1728         struct fib_nh_exception *fnhe;
1729         struct rtable *rth;
1730         int err;
1731         struct in_device *out_dev;
1732         bool do_cache;
1733         u32 itag = 0;
1734
1735         /* get a working reference to the output device */
1736         out_dev = __in_dev_get_rcu(FIB_RES_DEV(*res));
1737         if (!out_dev) {
1738                 net_crit_ratelimited("Bug in ip_route_input_slow(). Please report.\n");
1739                 return -EINVAL;
1740         }
1741
1742         err = fib_validate_source(skb, saddr, daddr, tos, FIB_RES_OIF(*res),
1743                                   in_dev->dev, in_dev, &itag);
1744         if (err < 0) {
1745                 ip_handle_martian_source(in_dev->dev, in_dev, skb, daddr,
1746                                          saddr);
1747
1748                 goto cleanup;
1749         }
1750
1751         do_cache = res->fi && !itag;
1752         if (out_dev == in_dev && err && IN_DEV_TX_REDIRECTS(out_dev) &&
1753             skb->protocol == htons(ETH_P_IP) &&
1754             (IN_DEV_SHARED_MEDIA(out_dev) ||
1755              inet_addr_onlink(out_dev, saddr, FIB_RES_GW(*res))))
1756                 IPCB(skb)->flags |= IPSKB_DOREDIRECT;
1757
1758         if (skb->protocol != htons(ETH_P_IP)) {
1759                 /* Not IP (i.e. ARP). Do not create route, if it is
1760                  * invalid for proxy arp. DNAT routes are always valid.
1761                  *
1762                  * Proxy arp feature have been extended to allow, ARP
1763                  * replies back to the same interface, to support
1764                  * Private VLAN switch technologies. See arp.c.
1765                  */
1766                 if (out_dev == in_dev &&
1767                     IN_DEV_PROXY_ARP_PVLAN(in_dev) == 0) {
1768                         err = -EINVAL;
1769                         goto cleanup;
1770                 }
1771         }
1772
1773         fnhe = find_exception(&FIB_RES_NH(*res), daddr);
1774         if (do_cache) {
1775                 if (fnhe)
1776                         rth = rcu_dereference(fnhe->fnhe_rth_input);
1777                 else
1778                         rth = rcu_dereference(FIB_RES_NH(*res).nh_rth_input);
1779                 if (rt_cache_valid(rth)) {
1780                         skb_dst_set_noref(skb, &rth->dst);
1781                         goto out;
1782                 }
1783         }
1784
1785         rth = rt_dst_alloc(out_dev->dev, 0, res->type,
1786                            IN_DEV_CONF_GET(in_dev, NOPOLICY),
1787                            IN_DEV_CONF_GET(out_dev, NOXFRM), do_cache);
1788         if (!rth) {
1789                 err = -ENOBUFS;
1790                 goto cleanup;
1791         }
1792
1793         rth->rt_is_input = 1;
1794         RT_CACHE_STAT_INC(in_slow_tot);
1795
1796         rth->dst.input = ip_forward;
1797
1798         rt_set_nexthop(rth, daddr, res, fnhe, res->fi, res->type, itag,
1799                        do_cache);
1800         lwtunnel_set_redirect(&rth->dst);
1801         skb_dst_set(skb, &rth->dst);
1802 out:
1803         err = 0;
1804  cleanup:
1805         return err;
1806 }
1807
1808 #ifdef CONFIG_IP_ROUTE_MULTIPATH
1809 /* To make ICMP packets follow the right flow, the multipath hash is
1810  * calculated from the inner IP addresses.
1811  */
1812 static void ip_multipath_l3_keys(const struct sk_buff *skb,
1813                                  struct flow_keys *hash_keys)
1814 {
1815         const struct iphdr *outer_iph = ip_hdr(skb);
1816         const struct iphdr *key_iph = outer_iph;
1817         const struct iphdr *inner_iph;
1818         const struct icmphdr *icmph;
1819         struct iphdr _inner_iph;
1820         struct icmphdr _icmph;
1821
1822         if (likely(outer_iph->protocol != IPPROTO_ICMP))
1823                 goto out;
1824
1825         if (unlikely((outer_iph->frag_off & htons(IP_OFFSET)) != 0))
1826                 goto out;
1827
1828         icmph = skb_header_pointer(skb, outer_iph->ihl * 4, sizeof(_icmph),
1829                                    &_icmph);
1830         if (!icmph)
1831                 goto out;
1832
1833         if (icmph->type != ICMP_DEST_UNREACH &&
1834             icmph->type != ICMP_REDIRECT &&
1835             icmph->type != ICMP_TIME_EXCEEDED &&
1836             icmph->type != ICMP_PARAMETERPROB)
1837                 goto out;
1838
1839         inner_iph = skb_header_pointer(skb,
1840                                        outer_iph->ihl * 4 + sizeof(_icmph),
1841                                        sizeof(_inner_iph), &_inner_iph);
1842         if (!inner_iph)
1843                 goto out;
1844
1845         key_iph = inner_iph;
1846 out:
1847         hash_keys->addrs.v4addrs.src = key_iph->saddr;
1848         hash_keys->addrs.v4addrs.dst = key_iph->daddr;
1849 }
1850
1851 /* if skb is set it will be used and fl4 can be NULL */
1852 int fib_multipath_hash(const struct net *net, const struct flowi4 *fl4,
1853                        const struct sk_buff *skb, struct flow_keys *flkeys)
1854 {
1855         u32 multipath_hash = fl4 ? fl4->flowi4_multipath_hash : 0;
1856         struct flow_keys hash_keys;
1857         u32 mhash;
1858
1859         switch (net->ipv4.sysctl_fib_multipath_hash_policy) {
1860         case 0:
1861                 memset(&hash_keys, 0, sizeof(hash_keys));
1862                 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
1863                 if (skb) {
1864                         ip_multipath_l3_keys(skb, &hash_keys);
1865                 } else {
1866                         hash_keys.addrs.v4addrs.src = fl4->saddr;
1867                         hash_keys.addrs.v4addrs.dst = fl4->daddr;
1868                 }
1869                 break;
1870         case 1:
1871                 /* skb is currently provided only when forwarding */
1872                 if (skb) {
1873                         unsigned int flag = FLOW_DISSECTOR_F_STOP_AT_ENCAP;
1874                         struct flow_keys keys;
1875
1876                         /* short-circuit if we already have L4 hash present */
1877                         if (skb->l4_hash)
1878                                 return skb_get_hash_raw(skb) >> 1;
1879
1880                         memset(&hash_keys, 0, sizeof(hash_keys));
1881
1882                         if (!flkeys) {
1883                                 skb_flow_dissect_flow_keys(skb, &keys, flag);
1884                                 flkeys = &keys;
1885                         }
1886
1887                         hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
1888                         hash_keys.addrs.v4addrs.src = flkeys->addrs.v4addrs.src;
1889                         hash_keys.addrs.v4addrs.dst = flkeys->addrs.v4addrs.dst;
1890                         hash_keys.ports.src = flkeys->ports.src;
1891                         hash_keys.ports.dst = flkeys->ports.dst;
1892                         hash_keys.basic.ip_proto = flkeys->basic.ip_proto;
1893                 } else {
1894                         memset(&hash_keys, 0, sizeof(hash_keys));
1895                         hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
1896                         hash_keys.addrs.v4addrs.src = fl4->saddr;
1897                         hash_keys.addrs.v4addrs.dst = fl4->daddr;
1898                         hash_keys.ports.src = fl4->fl4_sport;
1899                         hash_keys.ports.dst = fl4->fl4_dport;
1900                         hash_keys.basic.ip_proto = fl4->flowi4_proto;
1901                 }
1902                 break;
1903         }
1904         mhash = flow_hash_from_keys(&hash_keys);
1905
1906         if (multipath_hash)
1907                 mhash = jhash_2words(mhash, multipath_hash, 0);
1908
1909         return mhash >> 1;
1910 }
1911 #endif /* CONFIG_IP_ROUTE_MULTIPATH */
1912
1913 static int ip_mkroute_input(struct sk_buff *skb,
1914                             struct fib_result *res,
1915                             struct in_device *in_dev,
1916                             __be32 daddr, __be32 saddr, u32 tos,
1917                             struct flow_keys *hkeys)
1918 {
1919 #ifdef CONFIG_IP_ROUTE_MULTIPATH
1920         if (res->fi && res->fi->fib_nhs > 1) {
1921                 int h = fib_multipath_hash(res->fi->fib_net, NULL, skb, hkeys);
1922
1923                 fib_select_multipath(res, h);
1924         }
1925 #endif
1926
1927         /* create a routing cache entry */
1928         return __mkroute_input(skb, res, in_dev, daddr, saddr, tos);
1929 }
1930
1931 /*
1932  *      NOTE. We drop all the packets that has local source
1933  *      addresses, because every properly looped back packet
1934  *      must have correct destination already attached by output routine.
1935  *
1936  *      Such approach solves two big problems:
1937  *      1. Not simplex devices are handled properly.
1938  *      2. IP spoofing attempts are filtered with 100% of guarantee.
1939  *      called with rcu_read_lock()
1940  */
1941
1942 static int ip_route_input_slow(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1943                                u8 tos, struct net_device *dev,
1944                                struct fib_result *res)
1945 {
1946         struct in_device *in_dev = __in_dev_get_rcu(dev);
1947         struct flow_keys *flkeys = NULL, _flkeys;
1948         struct net    *net = dev_net(dev);
1949         struct ip_tunnel_info *tun_info;
1950         int             err = -EINVAL;
1951         unsigned int    flags = 0;
1952         u32             itag = 0;
1953         struct rtable   *rth;
1954         struct flowi4   fl4;
1955         bool do_cache;
1956
1957         /* IP on this device is disabled. */
1958
1959         if (!in_dev)
1960                 goto out;
1961
1962         /* Check for the most weird martians, which can be not detected
1963            by fib_lookup.
1964          */
1965
1966         tun_info = skb_tunnel_info(skb);
1967         if (tun_info && !(tun_info->mode & IP_TUNNEL_INFO_TX))
1968                 fl4.flowi4_tun_key.tun_id = tun_info->key.tun_id;
1969         else
1970                 fl4.flowi4_tun_key.tun_id = 0;
1971         skb_dst_drop(skb);
1972
1973         if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr))
1974                 goto martian_source;
1975
1976         res->fi = NULL;
1977         res->table = NULL;
1978         if (ipv4_is_lbcast(daddr) || (saddr == 0 && daddr == 0))
1979                 goto brd_input;
1980
1981         /* Accept zero addresses only to limited broadcast;
1982          * I even do not know to fix it or not. Waiting for complains :-)
1983          */
1984         if (ipv4_is_zeronet(saddr))
1985                 goto martian_source;
1986
1987         if (ipv4_is_zeronet(daddr))
1988                 goto martian_destination;
1989
1990         /* Following code try to avoid calling IN_DEV_NET_ROUTE_LOCALNET(),
1991          * and call it once if daddr or/and saddr are loopback addresses
1992          */
1993         if (ipv4_is_loopback(daddr)) {
1994                 if (!IN_DEV_NET_ROUTE_LOCALNET(in_dev, net))
1995                         goto martian_destination;
1996         } else if (ipv4_is_loopback(saddr)) {
1997                 if (!IN_DEV_NET_ROUTE_LOCALNET(in_dev, net))
1998                         goto martian_source;
1999         }
2000
2001         /*
2002          *      Now we are ready to route packet.
2003          */
2004         fl4.flowi4_oif = 0;
2005         fl4.flowi4_iif = dev->ifindex;
2006         fl4.flowi4_mark = skb->mark;
2007         fl4.flowi4_tos = tos;
2008         fl4.flowi4_scope = RT_SCOPE_UNIVERSE;
2009         fl4.flowi4_flags = 0;
2010         fl4.daddr = daddr;
2011         fl4.saddr = saddr;
2012         fl4.flowi4_uid = sock_net_uid(net, NULL);
2013
2014         if (fib4_rules_early_flow_dissect(net, skb, &fl4, &_flkeys)) {
2015                 flkeys = &_flkeys;
2016         } else {
2017                 fl4.flowi4_proto = 0;
2018                 fl4.fl4_sport = 0;
2019                 fl4.fl4_dport = 0;
2020         }
2021
2022         err = fib_lookup(net, &fl4, res, 0);
2023         if (err != 0) {
2024                 if (!IN_DEV_FORWARD(in_dev))
2025                         err = -EHOSTUNREACH;
2026                 goto no_route;
2027         }
2028
2029         if (res->type == RTN_BROADCAST) {
2030                 if (IN_DEV_BFORWARD(in_dev))
2031                         goto make_route;
2032                 goto brd_input;
2033         }
2034
2035         if (res->type == RTN_LOCAL) {
2036                 err = fib_validate_source(skb, saddr, daddr, tos,
2037                                           0, dev, in_dev, &itag);
2038                 if (err < 0)
2039                         goto martian_source;
2040                 goto local_input;
2041         }
2042
2043         if (!IN_DEV_FORWARD(in_dev)) {
2044                 err = -EHOSTUNREACH;
2045                 goto no_route;
2046         }
2047         if (res->type != RTN_UNICAST)
2048                 goto martian_destination;
2049
2050 make_route:
2051         err = ip_mkroute_input(skb, res, in_dev, daddr, saddr, tos, flkeys);
2052 out:    return err;
2053
2054 brd_input:
2055         if (skb->protocol != htons(ETH_P_IP))
2056                 goto e_inval;
2057
2058         if (!ipv4_is_zeronet(saddr)) {
2059                 err = fib_validate_source(skb, saddr, 0, tos, 0, dev,
2060                                           in_dev, &itag);
2061                 if (err < 0)
2062                         goto martian_source;
2063         }
2064         flags |= RTCF_BROADCAST;
2065         res->type = RTN_BROADCAST;
2066         RT_CACHE_STAT_INC(in_brd);
2067
2068 local_input:
2069         do_cache = false;
2070         if (res->fi) {
2071                 if (!itag) {
2072                         rth = rcu_dereference(FIB_RES_NH(*res).nh_rth_input);
2073                         if (rt_cache_valid(rth)) {
2074                                 skb_dst_set_noref(skb, &rth->dst);
2075                                 err = 0;
2076                                 goto out;
2077                         }
2078                         do_cache = true;
2079                 }
2080         }
2081
2082         rth = rt_dst_alloc(l3mdev_master_dev_rcu(dev) ? : net->loopback_dev,
2083                            flags | RTCF_LOCAL, res->type,
2084                            IN_DEV_CONF_GET(in_dev, NOPOLICY), false, do_cache);
2085         if (!rth)
2086                 goto e_nobufs;
2087
2088         rth->dst.output= ip_rt_bug;
2089 #ifdef CONFIG_IP_ROUTE_CLASSID
2090         rth->dst.tclassid = itag;
2091 #endif
2092         rth->rt_is_input = 1;
2093
2094         RT_CACHE_STAT_INC(in_slow_tot);
2095         if (res->type == RTN_UNREACHABLE) {
2096                 rth->dst.input= ip_error;
2097                 rth->dst.error= -err;
2098                 rth->rt_flags   &= ~RTCF_LOCAL;
2099         }
2100
2101         if (do_cache) {
2102                 struct fib_nh *nh = &FIB_RES_NH(*res);
2103
2104                 rth->dst.lwtstate = lwtstate_get(nh->nh_lwtstate);
2105                 if (lwtunnel_input_redirect(rth->dst.lwtstate)) {
2106                         WARN_ON(rth->dst.input == lwtunnel_input);
2107                         rth->dst.lwtstate->orig_input = rth->dst.input;
2108                         rth->dst.input = lwtunnel_input;
2109                 }
2110
2111                 if (unlikely(!rt_cache_route(nh, rth)))
2112                         rt_add_uncached_list(rth);
2113         }
2114         skb_dst_set(skb, &rth->dst);
2115         err = 0;
2116         goto out;
2117
2118 no_route:
2119         RT_CACHE_STAT_INC(in_no_route);
2120         res->type = RTN_UNREACHABLE;
2121         res->fi = NULL;
2122         res->table = NULL;
2123         goto local_input;
2124
2125         /*
2126          *      Do not cache martian addresses: they should be logged (RFC1812)
2127          */
2128 martian_destination:
2129         RT_CACHE_STAT_INC(in_martian_dst);
2130 #ifdef CONFIG_IP_ROUTE_VERBOSE
2131         if (IN_DEV_LOG_MARTIANS(in_dev))
2132                 net_warn_ratelimited("martian destination %pI4 from %pI4, dev %s\n",
2133                                      &daddr, &saddr, dev->name);
2134 #endif
2135
2136 e_inval:
2137         err = -EINVAL;
2138         goto out;
2139
2140 e_nobufs:
2141         err = -ENOBUFS;
2142         goto out;
2143
2144 martian_source:
2145         ip_handle_martian_source(dev, in_dev, skb, daddr, saddr);
2146         goto out;
2147 }
2148
2149 int ip_route_input_noref(struct sk_buff *skb, __be32 daddr, __be32 saddr,
2150                          u8 tos, struct net_device *dev)
2151 {
2152         struct fib_result res;
2153         int err;
2154
2155         tos &= IPTOS_RT_MASK;
2156         rcu_read_lock();
2157         err = ip_route_input_rcu(skb, daddr, saddr, tos, dev, &res);
2158         rcu_read_unlock();
2159
2160         return err;
2161 }
2162 EXPORT_SYMBOL(ip_route_input_noref);
2163
2164 /* called with rcu_read_lock held */
2165 int ip_route_input_rcu(struct sk_buff *skb, __be32 daddr, __be32 saddr,
2166                        u8 tos, struct net_device *dev, struct fib_result *res)
2167 {
2168         /* Multicast recognition logic is moved from route cache to here.
2169            The problem was that too many Ethernet cards have broken/missing
2170            hardware multicast filters :-( As result the host on multicasting
2171            network acquires a lot of useless route cache entries, sort of
2172            SDR messages from all the world. Now we try to get rid of them.
2173            Really, provided software IP multicast filter is organized
2174            reasonably (at least, hashed), it does not result in a slowdown
2175            comparing with route cache reject entries.
2176            Note, that multicast routers are not affected, because
2177            route cache entry is created eventually.
2178          */
2179         if (ipv4_is_multicast(daddr)) {
2180                 struct in_device *in_dev = __in_dev_get_rcu(dev);
2181                 int our = 0;
2182                 int err = -EINVAL;
2183
2184                 if (!in_dev)
2185                         return err;
2186                 our = ip_check_mc_rcu(in_dev, daddr, saddr,
2187                                       ip_hdr(skb)->protocol);
2188
2189                 /* check l3 master if no match yet */
2190                 if (!our && netif_is_l3_slave(dev)) {
2191                         struct in_device *l3_in_dev;
2192
2193                         l3_in_dev = __in_dev_get_rcu(skb->dev);
2194                         if (l3_in_dev)
2195                                 our = ip_check_mc_rcu(l3_in_dev, daddr, saddr,
2196                                                       ip_hdr(skb)->protocol);
2197                 }
2198
2199                 if (our
2200 #ifdef CONFIG_IP_MROUTE
2201                         ||
2202                     (!ipv4_is_local_multicast(daddr) &&
2203                      IN_DEV_MFORWARD(in_dev))
2204 #endif
2205                    ) {
2206                         err = ip_route_input_mc(skb, daddr, saddr,
2207                                                 tos, dev, our);
2208                 }
2209                 return err;
2210         }
2211
2212         return ip_route_input_slow(skb, daddr, saddr, tos, dev, res);
2213 }
2214
2215 /* called with rcu_read_lock() */
2216 static struct rtable *__mkroute_output(const struct fib_result *res,
2217                                        const struct flowi4 *fl4, int orig_oif,
2218                                        struct net_device *dev_out,
2219                                        unsigned int flags)
2220 {
2221         struct fib_info *fi = res->fi;
2222         struct fib_nh_exception *fnhe;
2223         struct in_device *in_dev;
2224         u16 type = res->type;
2225         struct rtable *rth;
2226         bool do_cache;
2227
2228         in_dev = __in_dev_get_rcu(dev_out);
2229         if (!in_dev)
2230                 return ERR_PTR(-EINVAL);
2231
2232         if (likely(!IN_DEV_ROUTE_LOCALNET(in_dev)))
2233                 if (ipv4_is_loopback(fl4->saddr) &&
2234                     !(dev_out->flags & IFF_LOOPBACK) &&
2235                     !netif_is_l3_master(dev_out))
2236                         return ERR_PTR(-EINVAL);
2237
2238         if (ipv4_is_lbcast(fl4->daddr))
2239                 type = RTN_BROADCAST;
2240         else if (ipv4_is_multicast(fl4->daddr))
2241                 type = RTN_MULTICAST;
2242         else if (ipv4_is_zeronet(fl4->daddr))
2243                 return ERR_PTR(-EINVAL);
2244
2245         if (dev_out->flags & IFF_LOOPBACK)
2246                 flags |= RTCF_LOCAL;
2247
2248         do_cache = true;
2249         if (type == RTN_BROADCAST) {
2250                 flags |= RTCF_BROADCAST | RTCF_LOCAL;
2251                 fi = NULL;
2252         } else if (type == RTN_MULTICAST) {
2253                 flags |= RTCF_MULTICAST | RTCF_LOCAL;
2254                 if (!ip_check_mc_rcu(in_dev, fl4->daddr, fl4->saddr,
2255                                      fl4->flowi4_proto))
2256                         flags &= ~RTCF_LOCAL;
2257                 else
2258                         do_cache = false;
2259                 /* If multicast route do not exist use
2260                  * default one, but do not gateway in this case.
2261                  * Yes, it is hack.
2262                  */
2263                 if (fi && res->prefixlen < 4)
2264                         fi = NULL;
2265         } else if ((type == RTN_LOCAL) && (orig_oif != 0) &&
2266                    (orig_oif != dev_out->ifindex)) {
2267                 /* For local routes that require a particular output interface
2268                  * we do not want to cache the result.  Caching the result
2269                  * causes incorrect behaviour when there are multiple source
2270                  * addresses on the interface, the end result being that if the
2271                  * intended recipient is waiting on that interface for the
2272                  * packet he won't receive it because it will be delivered on
2273                  * the loopback interface and the IP_PKTINFO ipi_ifindex will
2274                  * be set to the loopback interface as well.
2275                  */
2276                 do_cache = false;
2277         }
2278
2279         fnhe = NULL;
2280         do_cache &= fi != NULL;
2281         if (fi) {
2282                 struct rtable __rcu **prth;
2283                 struct fib_nh *nh = &FIB_RES_NH(*res);
2284
2285                 fnhe = find_exception(nh, fl4->daddr);
2286                 if (!do_cache)
2287                         goto add;
2288                 if (fnhe) {
2289                         prth = &fnhe->fnhe_rth_output;
2290                 } else {
2291                         if (unlikely(fl4->flowi4_flags &
2292                                      FLOWI_FLAG_KNOWN_NH &&
2293                                      !(nh->nh_gw &&
2294                                        nh->nh_scope == RT_SCOPE_LINK))) {
2295                                 do_cache = false;
2296                                 goto add;
2297                         }
2298                         prth = raw_cpu_ptr(nh->nh_pcpu_rth_output);
2299                 }
2300                 rth = rcu_dereference(*prth);
2301                 if (rt_cache_valid(rth) && dst_hold_safe(&rth->dst))
2302                         return rth;
2303         }
2304
2305 add:
2306         rth = rt_dst_alloc(dev_out, flags, type,
2307                            IN_DEV_CONF_GET(in_dev, NOPOLICY),
2308                            IN_DEV_CONF_GET(in_dev, NOXFRM),
2309                            do_cache);
2310         if (!rth)
2311                 return ERR_PTR(-ENOBUFS);
2312
2313         rth->rt_iif = orig_oif;
2314
2315         RT_CACHE_STAT_INC(out_slow_tot);
2316
2317         if (flags & (RTCF_BROADCAST | RTCF_MULTICAST)) {
2318                 if (flags & RTCF_LOCAL &&
2319                     !(dev_out->flags & IFF_LOOPBACK)) {
2320                         rth->dst.output = ip_mc_output;
2321                         RT_CACHE_STAT_INC(out_slow_mc);
2322                 }
2323 #ifdef CONFIG_IP_MROUTE
2324                 if (type == RTN_MULTICAST) {
2325                         if (IN_DEV_MFORWARD(in_dev) &&
2326                             !ipv4_is_local_multicast(fl4->daddr)) {
2327                                 rth->dst.input = ip_mr_input;
2328                                 rth->dst.output = ip_mc_output;
2329                         }
2330                 }
2331 #endif
2332         }
2333
2334         rt_set_nexthop(rth, fl4->daddr, res, fnhe, fi, type, 0, do_cache);
2335         lwtunnel_set_redirect(&rth->dst);
2336
2337         return rth;
2338 }
2339
2340 /*
2341  * Major route resolver routine.
2342  */
2343
2344 struct rtable *ip_route_output_key_hash(struct net *net, struct flowi4 *fl4,
2345                                         const struct sk_buff *skb)
2346 {
2347         __u8 tos = RT_FL_TOS(fl4);
2348         struct fib_result res = {
2349                 .type           = RTN_UNSPEC,
2350                 .fi             = NULL,
2351                 .table          = NULL,
2352                 .tclassid       = 0,
2353         };
2354         struct rtable *rth;
2355
2356         fl4->flowi4_iif = LOOPBACK_IFINDEX;
2357         fl4->flowi4_tos = tos & IPTOS_RT_MASK;
2358         fl4->flowi4_scope = ((tos & RTO_ONLINK) ?
2359                          RT_SCOPE_LINK : RT_SCOPE_UNIVERSE);
2360
2361         rcu_read_lock();
2362         rth = ip_route_output_key_hash_rcu(net, fl4, &res, skb);
2363         rcu_read_unlock();
2364
2365         return rth;
2366 }
2367 EXPORT_SYMBOL_GPL(ip_route_output_key_hash);
2368
2369 struct rtable *ip_route_output_key_hash_rcu(struct net *net, struct flowi4 *fl4,
2370                                             struct fib_result *res,
2371                                             const struct sk_buff *skb)
2372 {
2373         struct net_device *dev_out = NULL;
2374         int orig_oif = fl4->flowi4_oif;
2375         unsigned int flags = 0;
2376         struct rtable *rth;
2377         int err = -ENETUNREACH;
2378
2379         if (fl4->saddr) {
2380                 rth = ERR_PTR(-EINVAL);
2381                 if (ipv4_is_multicast(fl4->saddr) ||
2382                     ipv4_is_lbcast(fl4->saddr) ||
2383                     ipv4_is_zeronet(fl4->saddr))
2384                         goto out;
2385
2386                 /* I removed check for oif == dev_out->oif here.
2387                    It was wrong for two reasons:
2388                    1. ip_dev_find(net, saddr) can return wrong iface, if saddr
2389                       is assigned to multiple interfaces.
2390                    2. Moreover, we are allowed to send packets with saddr
2391                       of another iface. --ANK
2392                  */
2393
2394                 if (fl4->flowi4_oif == 0 &&
2395                     (ipv4_is_multicast(fl4->daddr) ||
2396                      ipv4_is_lbcast(fl4->daddr))) {
2397                         /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
2398                         dev_out = __ip_dev_find(net, fl4->saddr, false);
2399                         if (!dev_out)
2400                                 goto out;
2401
2402                         /* Special hack: user can direct multicasts
2403                            and limited broadcast via necessary interface
2404                            without fiddling with IP_MULTICAST_IF or IP_PKTINFO.
2405                            This hack is not just for fun, it allows
2406                            vic,vat and friends to work.
2407                            They bind socket to loopback, set ttl to zero
2408                            and expect that it will work.
2409                            From the viewpoint of routing cache they are broken,
2410                            because we are not allowed to build multicast path
2411                            with loopback source addr (look, routing cache
2412                            cannot know, that ttl is zero, so that packet
2413                            will not leave this host and route is valid).
2414                            Luckily, this hack is good workaround.
2415                          */
2416
2417                         fl4->flowi4_oif = dev_out->ifindex;
2418                         goto make_route;
2419                 }
2420
2421                 if (!(fl4->flowi4_flags & FLOWI_FLAG_ANYSRC)) {
2422                         /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
2423                         if (!__ip_dev_find(net, fl4->saddr, false))
2424                                 goto out;
2425                 }
2426         }
2427
2428
2429         if (fl4->flowi4_oif) {
2430                 dev_out = dev_get_by_index_rcu(net, fl4->flowi4_oif);
2431                 rth = ERR_PTR(-ENODEV);
2432                 if (!dev_out)
2433                         goto out;
2434
2435                 /* RACE: Check return value of inet_select_addr instead. */
2436                 if (!(dev_out->flags & IFF_UP) || !__in_dev_get_rcu(dev_out)) {
2437                         rth = ERR_PTR(-ENETUNREACH);
2438                         goto out;
2439                 }
2440                 if (ipv4_is_local_multicast(fl4->daddr) ||
2441                     ipv4_is_lbcast(fl4->daddr) ||
2442                     fl4->flowi4_proto == IPPROTO_IGMP) {
2443                         if (!fl4->saddr)
2444                                 fl4->saddr = inet_select_addr(dev_out, 0,
2445                                                               RT_SCOPE_LINK);
2446                         goto make_route;
2447                 }
2448                 if (!fl4->saddr) {
2449                         if (ipv4_is_multicast(fl4->daddr))
2450                                 fl4->saddr = inet_select_addr(dev_out, 0,
2451                                                               fl4->flowi4_scope);
2452                         else if (!fl4->daddr)
2453                                 fl4->saddr = inet_select_addr(dev_out, 0,
2454                                                               RT_SCOPE_HOST);
2455                 }
2456         }
2457
2458         if (!fl4->daddr) {
2459                 fl4->daddr = fl4->saddr;
2460                 if (!fl4->daddr)
2461                         fl4->daddr = fl4->saddr = htonl(INADDR_LOOPBACK);
2462                 dev_out = net->loopback_dev;
2463                 fl4->flowi4_oif = LOOPBACK_IFINDEX;
2464                 res->type = RTN_LOCAL;
2465                 flags |= RTCF_LOCAL;
2466                 goto make_route;
2467         }
2468
2469         err = fib_lookup(net, fl4, res, 0);
2470         if (err) {
2471                 res->fi = NULL;
2472                 res->table = NULL;
2473                 if (fl4->flowi4_oif &&
2474                     (ipv4_is_multicast(fl4->daddr) ||
2475                     !netif_index_is_l3_master(net, fl4->flowi4_oif))) {
2476                         /* Apparently, routing tables are wrong. Assume,
2477                            that the destination is on link.
2478
2479                            WHY? DW.
2480                            Because we are allowed to send to iface
2481                            even if it has NO routes and NO assigned
2482                            addresses. When oif is specified, routing
2483                            tables are looked up with only one purpose:
2484                            to catch if destination is gatewayed, rather than
2485                            direct. Moreover, if MSG_DONTROUTE is set,
2486                            we send packet, ignoring both routing tables
2487                            and ifaddr state. --ANK
2488
2489
2490                            We could make it even if oif is unknown,
2491                            likely IPv6, but we do not.
2492                          */
2493
2494                         if (fl4->saddr == 0)
2495                                 fl4->saddr = inet_select_addr(dev_out, 0,
2496                                                               RT_SCOPE_LINK);
2497                         res->type = RTN_UNICAST;
2498                         goto make_route;
2499                 }
2500                 rth = ERR_PTR(err);
2501                 goto out;
2502         }
2503
2504         if (res->type == RTN_LOCAL) {
2505                 if (!fl4->saddr) {
2506                         if (res->fi->fib_prefsrc)
2507                                 fl4->saddr = res->fi->fib_prefsrc;
2508                         else
2509                                 fl4->saddr = fl4->daddr;
2510                 }
2511
2512                 /* L3 master device is the loopback for that domain */
2513                 dev_out = l3mdev_master_dev_rcu(FIB_RES_DEV(*res)) ? :
2514                         net->loopback_dev;
2515
2516                 /* make sure orig_oif points to fib result device even
2517                  * though packet rx/tx happens over loopback or l3mdev
2518                  */
2519                 orig_oif = FIB_RES_OIF(*res);
2520
2521                 fl4->flowi4_oif = dev_out->ifindex;
2522                 flags |= RTCF_LOCAL;
2523                 goto make_route;
2524         }
2525
2526         fib_select_path(net, res, fl4, skb);
2527
2528         dev_out = FIB_RES_DEV(*res);
2529         fl4->flowi4_oif = dev_out->ifindex;
2530
2531
2532 make_route:
2533         rth = __mkroute_output(res, fl4, orig_oif, dev_out, flags);
2534
2535 out:
2536         return rth;
2537 }
2538
2539 static struct dst_entry *ipv4_blackhole_dst_check(struct dst_entry *dst, u32 cookie)
2540 {
2541         return NULL;
2542 }
2543
2544 static unsigned int ipv4_blackhole_mtu(const struct dst_entry *dst)
2545 {
2546         unsigned int mtu = dst_metric_raw(dst, RTAX_MTU);
2547
2548         return mtu ? : dst->dev->mtu;
2549 }
2550
2551 static void ipv4_rt_blackhole_update_pmtu(struct dst_entry *dst, struct sock *sk,
2552                                           struct sk_buff *skb, u32 mtu)
2553 {
2554 }
2555
2556 static void ipv4_rt_blackhole_redirect(struct dst_entry *dst, struct sock *sk,
2557                                        struct sk_buff *skb)
2558 {
2559 }
2560
2561 static u32 *ipv4_rt_blackhole_cow_metrics(struct dst_entry *dst,
2562                                           unsigned long old)
2563 {
2564         return NULL;
2565 }
2566
2567 static struct dst_ops ipv4_dst_blackhole_ops = {
2568         .family                 =       AF_INET,
2569         .check                  =       ipv4_blackhole_dst_check,
2570         .mtu                    =       ipv4_blackhole_mtu,
2571         .default_advmss         =       ipv4_default_advmss,
2572         .update_pmtu            =       ipv4_rt_blackhole_update_pmtu,
2573         .redirect               =       ipv4_rt_blackhole_redirect,
2574         .cow_metrics            =       ipv4_rt_blackhole_cow_metrics,
2575         .neigh_lookup           =       ipv4_neigh_lookup,
2576 };
2577
2578 struct dst_entry *ipv4_blackhole_route(struct net *net, struct dst_entry *dst_orig)
2579 {
2580         struct rtable *ort = (struct rtable *) dst_orig;
2581         struct rtable *rt;
2582
2583         rt = dst_alloc(&ipv4_dst_blackhole_ops, NULL, 1, DST_OBSOLETE_DEAD, 0);
2584         if (rt) {
2585                 struct dst_entry *new = &rt->dst;
2586
2587                 new->__use = 1;
2588                 new->input = dst_discard;
2589                 new->output = dst_discard_out;
2590
2591                 new->dev = net->loopback_dev;
2592                 if (new->dev)
2593                         dev_hold(new->dev);
2594
2595                 rt->rt_is_input = ort->rt_is_input;
2596                 rt->rt_iif = ort->rt_iif;
2597                 rt->rt_pmtu = ort->rt_pmtu;
2598                 rt->rt_mtu_locked = ort->rt_mtu_locked;
2599
2600                 rt->rt_genid = rt_genid_ipv4(net);
2601                 rt->rt_flags = ort->rt_flags;
2602                 rt->rt_type = ort->rt_type;
2603                 rt->rt_gateway = ort->rt_gateway;
2604                 rt->rt_uses_gateway = ort->rt_uses_gateway;
2605
2606                 INIT_LIST_HEAD(&rt->rt_uncached);
2607         }
2608
2609         dst_release(dst_orig);
2610
2611         return rt ? &rt->dst : ERR_PTR(-ENOMEM);
2612 }
2613
2614 struct rtable *ip_route_output_flow(struct net *net, struct flowi4 *flp4,
2615                                     const struct sock *sk)
2616 {
2617         struct rtable *rt = __ip_route_output_key(net, flp4);
2618
2619         if (IS_ERR(rt))
2620                 return rt;
2621
2622         if (flp4->flowi4_proto)
2623                 rt = (struct rtable *)xfrm_lookup_route(net, &rt->dst,
2624                                                         flowi4_to_flowi(flp4),
2625                                                         sk, 0);
2626
2627         return rt;
2628 }
2629 EXPORT_SYMBOL_GPL(ip_route_output_flow);
2630
2631 /* called with rcu_read_lock held */
2632 static int rt_fill_info(struct net *net, __be32 dst, __be32 src,
2633                         struct rtable *rt, u32 table_id, struct flowi4 *fl4,
2634                         struct sk_buff *skb, u32 portid, u32 seq)
2635 {
2636         struct rtmsg *r;
2637         struct nlmsghdr *nlh;
2638         unsigned long expires = 0;
2639         u32 error;
2640         u32 metrics[RTAX_MAX];
2641
2642         nlh = nlmsg_put(skb, portid, seq, RTM_NEWROUTE, sizeof(*r), 0);
2643         if (!nlh)
2644                 return -EMSGSIZE;
2645
2646         r = nlmsg_data(nlh);
2647         r->rtm_family    = AF_INET;
2648         r->rtm_dst_len  = 32;
2649         r->rtm_src_len  = 0;
2650         r->rtm_tos      = fl4->flowi4_tos;
2651         r->rtm_table    = table_id < 256 ? table_id : RT_TABLE_COMPAT;
2652         if (nla_put_u32(skb, RTA_TABLE, table_id))
2653                 goto nla_put_failure;
2654         r->rtm_type     = rt->rt_type;
2655         r->rtm_scope    = RT_SCOPE_UNIVERSE;
2656         r->rtm_protocol = RTPROT_UNSPEC;
2657         r->rtm_flags    = (rt->rt_flags & ~0xFFFF) | RTM_F_CLONED;
2658         if (rt->rt_flags & RTCF_NOTIFY)
2659                 r->rtm_flags |= RTM_F_NOTIFY;
2660         if (IPCB(skb)->flags & IPSKB_DOREDIRECT)
2661                 r->rtm_flags |= RTCF_DOREDIRECT;
2662
2663         if (nla_put_in_addr(skb, RTA_DST, dst))
2664                 goto nla_put_failure;
2665         if (src) {
2666                 r->rtm_src_len = 32;
2667                 if (nla_put_in_addr(skb, RTA_SRC, src))
2668                         goto nla_put_failure;
2669         }
2670         if (rt->dst.dev &&
2671             nla_put_u32(skb, RTA_OIF, rt->dst.dev->ifindex))
2672                 goto nla_put_failure;
2673 #ifdef CONFIG_IP_ROUTE_CLASSID
2674         if (rt->dst.tclassid &&
2675             nla_put_u32(skb, RTA_FLOW, rt->dst.tclassid))
2676                 goto nla_put_failure;
2677 #endif
2678         if (!rt_is_input_route(rt) &&
2679             fl4->saddr != src) {
2680                 if (nla_put_in_addr(skb, RTA_PREFSRC, fl4->saddr))
2681                         goto nla_put_failure;
2682         }
2683         if (rt->rt_uses_gateway &&
2684             nla_put_in_addr(skb, RTA_GATEWAY, rt->rt_gateway))
2685                 goto nla_put_failure;
2686
2687         expires = rt->dst.expires;
2688         if (expires) {
2689                 unsigned long now = jiffies;
2690
2691                 if (time_before(now, expires))
2692                         expires -= now;
2693                 else
2694                         expires = 0;
2695         }
2696
2697         memcpy(metrics, dst_metrics_ptr(&rt->dst), sizeof(metrics));
2698         if (rt->rt_pmtu && expires)
2699                 metrics[RTAX_MTU - 1] = rt->rt_pmtu;
2700         if (rt->rt_mtu_locked && expires)
2701                 metrics[RTAX_LOCK - 1] |= BIT(RTAX_MTU);
2702         if (rtnetlink_put_metrics(skb, metrics) < 0)
2703                 goto nla_put_failure;
2704
2705         if (fl4->flowi4_mark &&
2706             nla_put_u32(skb, RTA_MARK, fl4->flowi4_mark))
2707                 goto nla_put_failure;
2708
2709         if (!uid_eq(fl4->flowi4_uid, INVALID_UID) &&
2710             nla_put_u32(skb, RTA_UID,
2711                         from_kuid_munged(current_user_ns(), fl4->flowi4_uid)))
2712                 goto nla_put_failure;
2713
2714         error = rt->dst.error;
2715
2716         if (rt_is_input_route(rt)) {
2717 #ifdef CONFIG_IP_MROUTE
2718                 if (ipv4_is_multicast(dst) && !ipv4_is_local_multicast(dst) &&
2719                     IPV4_DEVCONF_ALL(net, MC_FORWARDING)) {
2720                         int err = ipmr_get_route(net, skb,
2721                                                  fl4->saddr, fl4->daddr,
2722                                                  r, portid);
2723
2724                         if (err <= 0) {
2725                                 if (err == 0)
2726                                         return 0;
2727                                 goto nla_put_failure;
2728                         }
2729                 } else
2730 #endif
2731                         if (nla_put_u32(skb, RTA_IIF, fl4->flowi4_iif))
2732                                 goto nla_put_failure;
2733         }
2734
2735         if (rtnl_put_cacheinfo(skb, &rt->dst, 0, expires, error) < 0)
2736                 goto nla_put_failure;
2737
2738         nlmsg_end(skb, nlh);
2739         return 0;
2740
2741 nla_put_failure:
2742         nlmsg_cancel(skb, nlh);
2743         return -EMSGSIZE;
2744 }
2745
2746 static struct sk_buff *inet_rtm_getroute_build_skb(__be32 src, __be32 dst,
2747                                                    u8 ip_proto, __be16 sport,
2748                                                    __be16 dport)
2749 {
2750         struct sk_buff *skb;
2751         struct iphdr *iph;
2752
2753         skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
2754         if (!skb)
2755                 return NULL;
2756
2757         /* Reserve room for dummy headers, this skb can pass
2758          * through good chunk of routing engine.
2759          */
2760         skb_reset_mac_header(skb);
2761         skb_reset_network_header(skb);
2762         skb->protocol = htons(ETH_P_IP);
2763         iph = skb_put(skb, sizeof(struct iphdr));
2764         iph->protocol = ip_proto;
2765         iph->saddr = src;
2766         iph->daddr = dst;
2767         iph->version = 0x4;
2768         iph->frag_off = 0;
2769         iph->ihl = 0x5;
2770         skb_set_transport_header(skb, skb->len);
2771
2772         switch (iph->protocol) {
2773         case IPPROTO_UDP: {
2774                 struct udphdr *udph;
2775
2776                 udph = skb_put_zero(skb, sizeof(struct udphdr));
2777                 udph->source = sport;
2778                 udph->dest = dport;
2779                 udph->len = sizeof(struct udphdr);
2780                 udph->check = 0;
2781                 break;
2782         }
2783         case IPPROTO_TCP: {
2784                 struct tcphdr *tcph;
2785
2786                 tcph = skb_put_zero(skb, sizeof(struct tcphdr));
2787                 tcph->source    = sport;
2788                 tcph->dest      = dport;
2789                 tcph->doff      = sizeof(struct tcphdr) / 4;
2790                 tcph->rst = 1;
2791                 tcph->check = ~tcp_v4_check(sizeof(struct tcphdr),
2792                                             src, dst, 0);
2793                 break;
2794         }
2795         case IPPROTO_ICMP: {
2796                 struct icmphdr *icmph;
2797
2798                 icmph = skb_put_zero(skb, sizeof(struct icmphdr));
2799                 icmph->type = ICMP_ECHO;
2800                 icmph->code = 0;
2801         }
2802         }
2803
2804         return skb;
2805 }
2806
2807 static int inet_rtm_valid_getroute_req(struct sk_buff *skb,
2808                                        const struct nlmsghdr *nlh,
2809                                        struct nlattr **tb,
2810                                        struct netlink_ext_ack *extack)
2811 {
2812         struct rtmsg *rtm;
2813         int i, err;
2814
2815         if (nlh->nlmsg_len < nlmsg_msg_size(sizeof(*rtm))) {
2816                 NL_SET_ERR_MSG(extack,
2817                                "ipv4: Invalid header for route get request");
2818                 return -EINVAL;
2819         }
2820
2821         if (!netlink_strict_get_check(skb))
2822                 return nlmsg_parse(nlh, sizeof(*rtm), tb, RTA_MAX,
2823                                    rtm_ipv4_policy, extack);
2824
2825         rtm = nlmsg_data(nlh);
2826         if ((rtm->rtm_src_len && rtm->rtm_src_len != 32) ||
2827             (rtm->rtm_dst_len && rtm->rtm_dst_len != 32) ||
2828             rtm->rtm_table || rtm->rtm_protocol ||
2829             rtm->rtm_scope || rtm->rtm_type) {
2830                 NL_SET_ERR_MSG(extack, "ipv4: Invalid values in header for route get request");
2831                 return -EINVAL;
2832         }
2833
2834         if (rtm->rtm_flags & ~(RTM_F_NOTIFY |
2835                                RTM_F_LOOKUP_TABLE |
2836                                RTM_F_FIB_MATCH)) {
2837                 NL_SET_ERR_MSG(extack, "ipv4: Unsupported rtm_flags for route get request");
2838                 return -EINVAL;
2839         }
2840
2841         err = nlmsg_parse_strict(nlh, sizeof(*rtm), tb, RTA_MAX,
2842                                  rtm_ipv4_policy, extack);
2843         if (err)
2844                 return err;
2845
2846         if ((tb[RTA_SRC] && !rtm->rtm_src_len) ||
2847             (tb[RTA_DST] && !rtm->rtm_dst_len)) {
2848                 NL_SET_ERR_MSG(extack, "ipv4: rtm_src_len and rtm_dst_len must be 32 for IPv4");
2849                 return -EINVAL;
2850         }
2851
2852         for (i = 0; i <= RTA_MAX; i++) {
2853                 if (!tb[i])
2854                         continue;
2855
2856                 switch (i) {
2857                 case RTA_IIF:
2858                 case RTA_OIF:
2859                 case RTA_SRC:
2860                 case RTA_DST:
2861                 case RTA_IP_PROTO:
2862                 case RTA_SPORT:
2863                 case RTA_DPORT:
2864                 case RTA_MARK:
2865                 case RTA_UID:
2866                         break;
2867                 default:
2868                         NL_SET_ERR_MSG(extack, "ipv4: Unsupported attribute in route get request");
2869                         return -EINVAL;
2870                 }
2871         }
2872
2873         return 0;
2874 }
2875
2876 static int inet_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr *nlh,
2877                              struct netlink_ext_ack *extack)
2878 {
2879         struct net *net = sock_net(in_skb->sk);
2880         struct nlattr *tb[RTA_MAX+1];
2881         u32 table_id = RT_TABLE_MAIN;
2882         __be16 sport = 0, dport = 0;
2883         struct fib_result res = {};
2884         u8 ip_proto = IPPROTO_UDP;
2885         struct rtable *rt = NULL;
2886         struct sk_buff *skb;
2887         struct rtmsg *rtm;
2888         struct flowi4 fl4 = {};
2889         __be32 dst = 0;
2890         __be32 src = 0;
2891         kuid_t uid;
2892         u32 iif;
2893         int err;
2894         int mark;
2895
2896         err = inet_rtm_valid_getroute_req(in_skb, nlh, tb, extack);
2897         if (err < 0)
2898                 return err;
2899
2900         rtm = nlmsg_data(nlh);
2901         src = tb[RTA_SRC] ? nla_get_in_addr(tb[RTA_SRC]) : 0;
2902         dst = tb[RTA_DST] ? nla_get_in_addr(tb[RTA_DST]) : 0;
2903         iif = tb[RTA_IIF] ? nla_get_u32(tb[RTA_IIF]) : 0;
2904         mark = tb[RTA_MARK] ? nla_get_u32(tb[RTA_MARK]) : 0;
2905         if (tb[RTA_UID])
2906                 uid = make_kuid(current_user_ns(), nla_get_u32(tb[RTA_UID]));
2907         else
2908                 uid = (iif ? INVALID_UID : current_uid());
2909
2910         if (tb[RTA_IP_PROTO]) {
2911                 err = rtm_getroute_parse_ip_proto(tb[RTA_IP_PROTO],
2912                                                   &ip_proto, AF_INET, extack);
2913                 if (err)
2914                         return err;
2915         }
2916
2917         if (tb[RTA_SPORT])
2918                 sport = nla_get_be16(tb[RTA_SPORT]);
2919
2920         if (tb[RTA_DPORT])
2921                 dport = nla_get_be16(tb[RTA_DPORT]);
2922
2923         skb = inet_rtm_getroute_build_skb(src, dst, ip_proto, sport, dport);
2924         if (!skb)
2925                 return -ENOBUFS;
2926
2927         fl4.daddr = dst;
2928         fl4.saddr = src;
2929         fl4.flowi4_tos = rtm->rtm_tos;
2930         fl4.flowi4_oif = tb[RTA_OIF] ? nla_get_u32(tb[RTA_OIF]) : 0;
2931         fl4.flowi4_mark = mark;
2932         fl4.flowi4_uid = uid;
2933         if (sport)
2934                 fl4.fl4_sport = sport;
2935         if (dport)
2936                 fl4.fl4_dport = dport;
2937         fl4.flowi4_proto = ip_proto;
2938
2939         rcu_read_lock();
2940
2941         if (iif) {
2942                 struct net_device *dev;
2943
2944                 dev = dev_get_by_index_rcu(net, iif);
2945                 if (!dev) {
2946                         err = -ENODEV;
2947                         goto errout_rcu;
2948                 }
2949
2950                 fl4.flowi4_iif = iif; /* for rt_fill_info */
2951                 skb->dev        = dev;
2952                 skb->mark       = mark;
2953                 err = ip_route_input_rcu(skb, dst, src, rtm->rtm_tos,
2954                                          dev, &res);
2955
2956                 rt = skb_rtable(skb);
2957                 if (err == 0 && rt->dst.error)
2958                         err = -rt->dst.error;
2959         } else {
2960                 fl4.flowi4_iif = LOOPBACK_IFINDEX;
2961                 skb->dev = net->loopback_dev;
2962                 rt = ip_route_output_key_hash_rcu(net, &fl4, &res, skb);
2963                 err = 0;
2964                 if (IS_ERR(rt))
2965                         err = PTR_ERR(rt);
2966                 else
2967                         skb_dst_set(skb, &rt->dst);
2968         }
2969
2970         if (err)
2971                 goto errout_rcu;
2972
2973         if (rtm->rtm_flags & RTM_F_NOTIFY)
2974                 rt->rt_flags |= RTCF_NOTIFY;
2975
2976         if (rtm->rtm_flags & RTM_F_LOOKUP_TABLE)
2977                 table_id = res.table ? res.table->tb_id : 0;
2978
2979         /* reset skb for netlink reply msg */
2980         skb_trim(skb, 0);
2981         skb_reset_network_header(skb);
2982         skb_reset_transport_header(skb);
2983         skb_reset_mac_header(skb);
2984
2985         if (rtm->rtm_flags & RTM_F_FIB_MATCH) {
2986                 if (!res.fi) {
2987                         err = fib_props[res.type].error;
2988                         if (!err)
2989                                 err = -EHOSTUNREACH;
2990                         goto errout_rcu;
2991                 }
2992                 err = fib_dump_info(skb, NETLINK_CB(in_skb).portid,
2993                                     nlh->nlmsg_seq, RTM_NEWROUTE, table_id,
2994                                     rt->rt_type, res.prefix, res.prefixlen,
2995                                     fl4.flowi4_tos, res.fi, 0);
2996         } else {
2997                 err = rt_fill_info(net, dst, src, rt, table_id, &fl4, skb,
2998                                    NETLINK_CB(in_skb).portid, nlh->nlmsg_seq);
2999         }
3000         if (err < 0)
3001                 goto errout_rcu;
3002
3003         rcu_read_unlock();
3004
3005         err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).portid);
3006
3007 errout_free:
3008         return err;
3009 errout_rcu:
3010         rcu_read_unlock();
3011         kfree_skb(skb);
3012         goto errout_free;
3013 }
3014
3015 void ip_rt_multicast_event(struct in_device *in_dev)
3016 {
3017         rt_cache_flush(dev_net(in_dev->dev));
3018 }
3019
3020 #ifdef CONFIG_SYSCTL
3021 static int ip_rt_gc_interval __read_mostly  = 60 * HZ;
3022 static int ip_rt_gc_min_interval __read_mostly  = HZ / 2;
3023 static int ip_rt_gc_elasticity __read_mostly    = 8;
3024 static int ip_min_valid_pmtu __read_mostly      = IPV4_MIN_MTU;
3025
3026 static int ipv4_sysctl_rtcache_flush(struct ctl_table *__ctl, int write,
3027                                         void __user *buffer,
3028                                         size_t *lenp, loff_t *ppos)
3029 {
3030         struct net *net = (struct net *)__ctl->extra1;
3031
3032         if (write) {
3033                 rt_cache_flush(net);
3034                 fnhe_genid_bump(net);
3035                 return 0;
3036         }
3037
3038         return -EINVAL;
3039 }
3040
3041 static struct ctl_table ipv4_route_table[] = {
3042         {
3043                 .procname       = "gc_thresh",
3044                 .data           = &ipv4_dst_ops.gc_thresh,
3045                 .maxlen         = sizeof(int),
3046                 .mode           = 0644,
3047                 .proc_handler   = proc_dointvec,
3048         },
3049         {
3050                 .procname       = "max_size",
3051                 .data           = &ip_rt_max_size,
3052                 .maxlen         = sizeof(int),
3053                 .mode           = 0644,
3054                 .proc_handler   = proc_dointvec,
3055         },
3056         {
3057                 /*  Deprecated. Use gc_min_interval_ms */
3058
3059                 .procname       = "gc_min_interval",
3060                 .data           = &ip_rt_gc_min_interval,
3061                 .maxlen         = sizeof(int),
3062                 .mode           = 0644,
3063                 .proc_handler   = proc_dointvec_jiffies,
3064         },
3065         {
3066                 .procname       = "gc_min_interval_ms",
3067                 .data           = &ip_rt_gc_min_interval,
3068                 .maxlen         = sizeof(int),
3069                 .mode           = 0644,
3070                 .proc_handler   = proc_dointvec_ms_jiffies,
3071         },
3072         {
3073                 .procname       = "gc_timeout",
3074                 .data           = &ip_rt_gc_timeout,
3075                 .maxlen         = sizeof(int),
3076                 .mode           = 0644,
3077                 .proc_handler   = proc_dointvec_jiffies,
3078         },
3079         {
3080                 .procname       = "gc_interval",
3081                 .data           = &ip_rt_gc_interval,
3082                 .maxlen         = sizeof(int),
3083                 .mode           = 0644,
3084                 .proc_handler   = proc_dointvec_jiffies,
3085         },
3086         {
3087                 .procname       = "redirect_load",
3088                 .data           = &ip_rt_redirect_load,
3089                 .maxlen         = sizeof(int),
3090                 .mode           = 0644,
3091                 .proc_handler   = proc_dointvec,
3092         },
3093         {
3094                 .procname       = "redirect_number",
3095                 .data           = &ip_rt_redirect_number,
3096                 .maxlen         = sizeof(int),
3097                 .mode           = 0644,
3098                 .proc_handler   = proc_dointvec,
3099         },
3100         {
3101                 .procname       = "redirect_silence",
3102                 .data           = &ip_rt_redirect_silence,
3103                 .maxlen         = sizeof(int),
3104                 .mode           = 0644,
3105                 .proc_handler   = proc_dointvec,
3106         },
3107         {
3108                 .procname       = "error_cost",
3109                 .data           = &ip_rt_error_cost,
3110                 .maxlen         = sizeof(int),
3111                 .mode           = 0644,
3112                 .proc_handler   = proc_dointvec,
3113         },
3114         {
3115                 .procname       = "error_burst",
3116                 .data           = &ip_rt_error_burst,
3117                 .maxlen         = sizeof(int),
3118                 .mode           = 0644,
3119                 .proc_handler   = proc_dointvec,
3120         },
3121         {
3122                 .procname       = "gc_elasticity",
3123                 .data           = &ip_rt_gc_elasticity,
3124                 .maxlen         = sizeof(int),
3125                 .mode           = 0644,
3126                 .proc_handler   = proc_dointvec,
3127         },
3128         {
3129                 .procname       = "mtu_expires",
3130                 .data           = &ip_rt_mtu_expires,
3131                 .maxlen         = sizeof(int),
3132                 .mode           = 0644,
3133                 .proc_handler   = proc_dointvec_jiffies,
3134         },
3135         {
3136                 .procname       = "min_pmtu",
3137                 .data           = &ip_rt_min_pmtu,
3138                 .maxlen         = sizeof(int),
3139                 .mode           = 0644,
3140                 .proc_handler   = proc_dointvec_minmax,
3141                 .extra1         = &ip_min_valid_pmtu,
3142         },
3143         {
3144                 .procname       = "min_adv_mss",
3145                 .data           = &ip_rt_min_advmss,
3146                 .maxlen         = sizeof(int),
3147                 .mode           = 0644,
3148                 .proc_handler   = proc_dointvec,
3149         },
3150         { }
3151 };
3152
3153 static struct ctl_table ipv4_route_flush_table[] = {
3154         {
3155                 .procname       = "flush",
3156                 .maxlen         = sizeof(int),
3157                 .mode           = 0200,
3158                 .proc_handler   = ipv4_sysctl_rtcache_flush,
3159         },
3160         { },
3161 };
3162
3163 static __net_init int sysctl_route_net_init(struct net *net)
3164 {
3165         struct ctl_table *tbl;
3166
3167         tbl = ipv4_route_flush_table;
3168         if (!net_eq(net, &init_net)) {
3169                 tbl = kmemdup(tbl, sizeof(ipv4_route_flush_table), GFP_KERNEL);
3170                 if (!tbl)
3171                         goto err_dup;
3172
3173                 /* Don't export sysctls to unprivileged users */
3174                 if (net->user_ns != &init_user_ns)
3175                         tbl[0].procname = NULL;
3176         }
3177         tbl[0].extra1 = net;
3178
3179         net->ipv4.route_hdr = register_net_sysctl(net, "net/ipv4/route", tbl);
3180         if (!net->ipv4.route_hdr)
3181                 goto err_reg;
3182         return 0;
3183
3184 err_reg:
3185         if (tbl != ipv4_route_flush_table)
3186                 kfree(tbl);
3187 err_dup:
3188         return -ENOMEM;
3189 }
3190
3191 static __net_exit void sysctl_route_net_exit(struct net *net)
3192 {
3193         struct ctl_table *tbl;
3194
3195         tbl = net->ipv4.route_hdr->ctl_table_arg;
3196         unregister_net_sysctl_table(net->ipv4.route_hdr);
3197         BUG_ON(tbl == ipv4_route_flush_table);
3198         kfree(tbl);
3199 }
3200
3201 static __net_initdata struct pernet_operations sysctl_route_ops = {
3202         .init = sysctl_route_net_init,
3203         .exit = sysctl_route_net_exit,
3204 };
3205 #endif
3206
3207 static __net_init int rt_genid_init(struct net *net)
3208 {
3209         atomic_set(&net->ipv4.rt_genid, 0);
3210         atomic_set(&net->fnhe_genid, 0);
3211         atomic_set(&net->ipv4.dev_addr_genid, get_random_int());
3212         return 0;
3213 }
3214
3215 static __net_initdata struct pernet_operations rt_genid_ops = {
3216         .init = rt_genid_init,
3217 };
3218
3219 static int __net_init ipv4_inetpeer_init(struct net *net)
3220 {
3221         struct inet_peer_base *bp = kmalloc(sizeof(*bp), GFP_KERNEL);
3222
3223         if (!bp)
3224                 return -ENOMEM;
3225         inet_peer_base_init(bp);
3226         net->ipv4.peers = bp;
3227         return 0;
3228 }
3229
3230 static void __net_exit ipv4_inetpeer_exit(struct net *net)
3231 {
3232         struct inet_peer_base *bp = net->ipv4.peers;
3233
3234         net->ipv4.peers = NULL;
3235         inetpeer_invalidate_tree(bp);
3236         kfree(bp);
3237 }
3238
3239 static __net_initdata struct pernet_operations ipv4_inetpeer_ops = {
3240         .init   =       ipv4_inetpeer_init,
3241         .exit   =       ipv4_inetpeer_exit,
3242 };
3243
3244 #ifdef CONFIG_IP_ROUTE_CLASSID
3245 struct ip_rt_acct __percpu *ip_rt_acct __read_mostly;
3246 #endif /* CONFIG_IP_ROUTE_CLASSID */
3247
3248 int __init ip_rt_init(void)
3249 {
3250         int cpu;
3251
3252         ip_idents = kmalloc_array(IP_IDENTS_SZ, sizeof(*ip_idents),
3253                                   GFP_KERNEL);
3254         if (!ip_idents)
3255                 panic("IP: failed to allocate ip_idents\n");
3256
3257         prandom_bytes(ip_idents, IP_IDENTS_SZ * sizeof(*ip_idents));
3258
3259         ip_tstamps = kcalloc(IP_IDENTS_SZ, sizeof(*ip_tstamps), GFP_KERNEL);
3260         if (!ip_tstamps)
3261                 panic("IP: failed to allocate ip_tstamps\n");
3262
3263         for_each_possible_cpu(cpu) {
3264                 struct uncached_list *ul = &per_cpu(rt_uncached_list, cpu);
3265
3266                 INIT_LIST_HEAD(&ul->head);
3267                 spin_lock_init(&ul->lock);
3268         }
3269 #ifdef CONFIG_IP_ROUTE_CLASSID
3270         ip_rt_acct = __alloc_percpu(256 * sizeof(struct ip_rt_acct), __alignof__(struct ip_rt_acct));
3271         if (!ip_rt_acct)
3272                 panic("IP: failed to allocate ip_rt_acct\n");
3273 #endif
3274
3275         ipv4_dst_ops.kmem_cachep =
3276                 kmem_cache_create("ip_dst_cache", sizeof(struct rtable), 0,
3277                                   SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
3278
3279         ipv4_dst_blackhole_ops.kmem_cachep = ipv4_dst_ops.kmem_cachep;
3280
3281         if (dst_entries_init(&ipv4_dst_ops) < 0)
3282                 panic("IP: failed to allocate ipv4_dst_ops counter\n");
3283
3284         if (dst_entries_init(&ipv4_dst_blackhole_ops) < 0)
3285                 panic("IP: failed to allocate ipv4_dst_blackhole_ops counter\n");
3286
3287         ipv4_dst_ops.gc_thresh = ~0;
3288         ip_rt_max_size = INT_MAX;
3289
3290         devinet_init();
3291         ip_fib_init();
3292
3293         if (ip_rt_proc_init())
3294                 pr_err("Unable to create route proc files\n");
3295 #ifdef CONFIG_XFRM
3296         xfrm_init();
3297         xfrm4_init();
3298 #endif
3299         rtnl_register(PF_INET, RTM_GETROUTE, inet_rtm_getroute, NULL,
3300                       RTNL_FLAG_DOIT_UNLOCKED);
3301
3302 #ifdef CONFIG_SYSCTL
3303         register_pernet_subsys(&sysctl_route_ops);
3304 #endif
3305         register_pernet_subsys(&rt_genid_ops);
3306         register_pernet_subsys(&ipv4_inetpeer_ops);
3307         return 0;
3308 }
3309
3310 #ifdef CONFIG_SYSCTL
3311 /*
3312  * We really need to sanitize the damn ipv4 init order, then all
3313  * this nonsense will go away.
3314  */
3315 void __init ip_static_sysctl_init(void)
3316 {
3317         register_net_sysctl(&init_net, "net/ipv4/route", ipv4_route_table);
3318 }
3319 #endif