net/ipv4/route.c

   1 /*
   2  * INET         An implementation of the TCP/IP protocol suite for the LINUX
   3  *              operating system.  INET is implemented using the  BSD Socket
   4  *              interface as the means of communication with the user level.
   5  *
   6  *              ROUTE - implementation of the IP router.
   7  *
   8  * Authors:     Ross Biro
   9  *              Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
  10  *              Alan Cox, <gw4pts@gw4pts.ampr.org>
  11  *              Linus Torvalds, <Linus.Torvalds@helsinki.fi>
  12  *              Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
  13  *
  14  * Fixes:
  15  *              Alan Cox        :       Verify area fixes.
  16  *              Alan Cox        :       cli() protects routing changes
  17  *              Rui Oliveira    :       ICMP routing table updates
  18  *              (rco@di.uminho.pt)      Routing table insertion and update
  19  *              Linus Torvalds  :       Rewrote bits to be sensible
  20  *              Alan Cox        :       Added BSD route gw semantics
  21  *              Alan Cox        :       Super /proc >4K
  22  *              Alan Cox        :       MTU in route table
  23  *              Alan Cox        :       MSS actually. Also added the window
  24  *                                      clamper.
  25  *              Sam Lantinga    :       Fixed route matching in rt_del()
  26  *              Alan Cox        :       Routing cache support.
  27  *              Alan Cox        :       Removed compatibility cruft.
  28  *              Alan Cox        :       RTF_REJECT support.
  29  *              Alan Cox        :       TCP irtt support.
  30  *              Jonathan Naylor :       Added Metric support.
  31  *      Miquel van Smoorenburg  :       BSD API fixes.
  32  *      Miquel van Smoorenburg  :       Metrics.
  33  *              Alan Cox        :       Use __u32 properly
  34  *              Alan Cox        :       Aligned routing errors more closely with BSD
  35  *                                      our system is still very different.
  36  *              Alan Cox        :       Faster /proc handling
  37  *      Alexey Kuznetsov        :       Massive rework to support tree based routing,
  38  *                                      routing caches and better behaviour.
  39  *
  40  *              Olaf Erb        :       irtt wasn't being copied right.
  41  *              Bjorn Ekwall    :       Kerneld route support.
  42  *              Alan Cox        :       Multicast fixed (I hope)
  43  *              Pavel Krauz     :       Limited broadcast fixed
  44  *              Mike McLagan    :       Routing by source
  45  *      Alexey Kuznetsov        :       End of old history. Split to fib.c and
  46  *                                      route.c and rewritten from scratch.
  47  *              Andi Kleen      :       Load-limit warning messages.
  48  *      Vitaly E. Lavrov        :       Transparent proxy revived after year coma.
  49  *      Vitaly E. Lavrov        :       Race condition in ip_route_input_slow.
  50  *      Tobias Ringstrom        :       Uninitialized res.type in ip_route_output_slow.
  51  *      Vladimir V. Ivanov      :       IP rule info (flowid) is really useful.
  52  *              Marc Boucher    :       routing by fwmark
  53  *      Robert Olsson           :       Added rt_cache statistics
  54  *      Arnaldo C. Melo         :       Convert proc stuff to seq_file
  55  *      Eric Dumazet            :       hashed spinlocks and rt_check_expire() fixes.
  56  *      Ilia Sotnikov           :       Ignore TOS on PMTUD and Redirect
  57  *      Ilia Sotnikov           :       Removed TOS from hash calculations
  58  *
  59  *              This program is free software; you can redistribute it and/or
  60  *              modify it under the terms of the GNU General Public License
  61  *              as published by the Free Software Foundation; either version
  62  *              2 of the License, or (at your option) any later version.
  63  */
  64
  65 #define pr_fmt(fmt) "IPv4: " fmt
  66
  67 #include <linux/module.h>
  68 #include <linux/uaccess.h>
  69 #include <linux/bitops.h>
  70 #include <linux/types.h>
  71 #include <linux/kernel.h>
  72 #include <linux/mm.h>
  73 #include <linux/string.h>
  74 #include <linux/socket.h>
  75 #include <linux/sockios.h>
  76 #include <linux/errno.h>
  77 #include <linux/in.h>
  78 #include <linux/inet.h>
  79 #include <linux/netdevice.h>
  80 #include <linux/proc_fs.h>
  81 #include <linux/init.h>
  82 #include <linux/skbuff.h>
  83 #include <linux/inetdevice.h>
  84 #include <linux/igmp.h>
  85 #include <linux/pkt_sched.h>
  86 #include <linux/mroute.h>
  87 #include <linux/netfilter_ipv4.h>
  88 #include <linux/random.h>
  89 #include <linux/rcupdate.h>
  90 #include <linux/times.h>
  91 #include <linux/slab.h>
  92 #include <linux/jhash.h>
  93 #include <net/dst.h>
  94 #include <net/dst_metadata.h>
  95 #include <net/net_namespace.h>
  96 #include <net/protocol.h>
  97 #include <net/ip.h>
  98 #include <net/route.h>
  99 #include <net/inetpeer.h>
 100 #include <net/sock.h>
 101 #include <net/ip_fib.h>
 102 #include <net/arp.h>
 103 #include <net/tcp.h>
 104 #include <net/icmp.h>
 105 #include <net/xfrm.h>
 106 #include <net/lwtunnel.h>
 107 #include <net/netevent.h>
 108 #include <net/rtnetlink.h>
 109 #ifdef CONFIG_SYSCTL
 110 #include <linux/sysctl.h>
 111 #endif
 112 #include <net/secure_seq.h>
 113 #include <net/ip_tunnels.h>
 114 #include <net/l3mdev.h>
 115
 116 #include "fib_lookup.h"
 117
 118 #define RT_FL_TOS(oldflp4) \
 119         ((oldflp4)->flowi4_tos & (IPTOS_RT_MASK | RTO_ONLINK))
 120
 121 #define RT_GC_TIMEOUT (300*HZ)
 122
 123 static int ip_rt_max_size;
 124 static int ip_rt_redirect_number __read_mostly  = 9;
 125 static int ip_rt_redirect_load __read_mostly    = HZ / 50;
 126 static int ip_rt_redirect_silence __read_mostly = ((HZ / 50) << (9 + 1));
 127 static int ip_rt_error_cost __read_mostly       = HZ;
 128 static int ip_rt_error_burst __read_mostly      = 5 * HZ;
 129 static int ip_rt_mtu_expires __read_mostly      = 10 * 60 * HZ;
 130 static u32 ip_rt_min_pmtu __read_mostly         = 512 + 20 + 20;
 131 static int ip_rt_min_advmss __read_mostly       = 256;
 132
 133 static int ip_rt_gc_timeout __read_mostly       = RT_GC_TIMEOUT;
 134
 135 /*
 136  *      Interface to generic destination cache.
 137  */
 138
 139 static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie);
 140 static unsigned int      ipv4_default_advmss(const struct dst_entry *dst);
 141 static unsigned int      ipv4_mtu(const struct dst_entry *dst);
 142 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst);
 143 static void              ipv4_link_failure(struct sk_buff *skb);
 144 static void              ip_rt_update_pmtu(struct dst_entry *dst, struct sock *sk,
 145                                            struct sk_buff *skb, u32 mtu);
 146 static void              ip_do_redirect(struct dst_entry *dst, struct sock *sk,
 147                                         struct sk_buff *skb);
 148 static void             ipv4_dst_destroy(struct dst_entry *dst);
 149
 150 static u32 *ipv4_cow_metrics(struct dst_entry *dst, unsigned long old)
 151 {
 152         WARN_ON(1);
 153         return NULL;
 154 }
 155
 156 static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst,
 157                                            struct sk_buff *skb,
 158                                            const void *daddr);
 159 static void ipv4_confirm_neigh(const struct dst_entry *dst, const void *daddr);
 160
 161 static struct dst_ops ipv4_dst_ops = {
 162         .family =               AF_INET,
 163         .check =                ipv4_dst_check,
 164         .default_advmss =       ipv4_default_advmss,
 165         .mtu =                  ipv4_mtu,
 166         .cow_metrics =          ipv4_cow_metrics,
 167         .destroy =              ipv4_dst_destroy,
 168         .negative_advice =      ipv4_negative_advice,
 169         .link_failure =         ipv4_link_failure,
 170         .update_pmtu =          ip_rt_update_pmtu,
 171         .redirect =             ip_do_redirect,
 172         .local_out =            __ip_local_out,
 173         .neigh_lookup =         ipv4_neigh_lookup,
 174         .confirm_neigh =        ipv4_confirm_neigh,
 175 };
 176
 177 #define ECN_OR_COST(class)      TC_PRIO_##class
 178
 179 const __u8 ip_tos2prio[16] = {
 180         TC_PRIO_BESTEFFORT,
 181         ECN_OR_COST(BESTEFFORT),
 182         TC_PRIO_BESTEFFORT,
 183         ECN_OR_COST(BESTEFFORT),
 184         TC_PRIO_BULK,
 185         ECN_OR_COST(BULK),
 186         TC_PRIO_BULK,
 187         ECN_OR_COST(BULK),
 188         TC_PRIO_INTERACTIVE,
 189         ECN_OR_COST(INTERACTIVE),
 190         TC_PRIO_INTERACTIVE,
 191         ECN_OR_COST(INTERACTIVE),
 192         TC_PRIO_INTERACTIVE_BULK,
 193         ECN_OR_COST(INTERACTIVE_BULK),
 194         TC_PRIO_INTERACTIVE_BULK,
 195         ECN_OR_COST(INTERACTIVE_BULK)
 196 };
 197 EXPORT_SYMBOL(ip_tos2prio);
 198
 199 static DEFINE_PER_CPU(struct rt_cache_stat, rt_cache_stat);
 200 #define RT_CACHE_STAT_INC(field) raw_cpu_inc(rt_cache_stat.field)
 201
 202 #ifdef CONFIG_PROC_FS
 203 static void *rt_cache_seq_start(struct seq_file *seq, loff_t *pos)
 204 {
 205         if (*pos)
 206                 return NULL;
 207         return SEQ_START_TOKEN;
 208 }
 209
 210 static void *rt_cache_seq_next(struct seq_file *seq, void *v, loff_t *pos)
 211 {
 212         ++*pos;
 213         return NULL;
 214 }
 215
 216 static void rt_cache_seq_stop(struct seq_file *seq, void *v)
 217 {
 218 }
 219
 220 static int rt_cache_seq_show(struct seq_file *seq, void *v)
 221 {
 222         if (v == SEQ_START_TOKEN)
 223                 seq_printf(seq, "%-127s\n",
 224                            "Iface\tDestination\tGateway \tFlags\t\tRefCnt\tUse\t"
 225                            "Metric\tSource\t\tMTU\tWindow\tIRTT\tTOS\tHHRef\t"
 226                            "HHUptod\tSpecDst");
 227         return 0;
 228 }
 229
 230 static const struct seq_operations rt_cache_seq_ops = {
 231         .start  = rt_cache_seq_start,
 232         .next   = rt_cache_seq_next,
 233         .stop   = rt_cache_seq_stop,
 234         .show   = rt_cache_seq_show,
 235 };
 236
 237 static int rt_cache_seq_open(struct inode *inode, struct file *file)
 238 {
 239         return seq_open(file, &rt_cache_seq_ops);
 240 }
 241
 242 static const struct file_operations rt_cache_seq_fops = {
 243         .open    = rt_cache_seq_open,
 244         .read    = seq_read,
 245         .llseek  = seq_lseek,
 246         .release = seq_release,
 247 };
 248
 249
 250 static void *rt_cpu_seq_start(struct seq_file *seq, loff_t *pos)
 251 {
 252         int cpu;
 253
 254         if (*pos == 0)
 255                 return SEQ_START_TOKEN;
 256
 257         for (cpu = *pos-1; cpu < nr_cpu_ids; ++cpu) {
 258                 if (!cpu_possible(cpu))
 259                         continue;
 260                 *pos = cpu+1;
 261                 return &per_cpu(rt_cache_stat, cpu);
 262         }
 263         return NULL;
 264 }
 265
 266 static void *rt_cpu_seq_next(struct seq_file *seq, void *v, loff_t *pos)
 267 {
 268         int cpu;
 269
 270         for (cpu = *pos; cpu < nr_cpu_ids; ++cpu) {
 271                 if (!cpu_possible(cpu))
 272                         continue;
 273                 *pos = cpu+1;
 274                 return &per_cpu(rt_cache_stat, cpu);
 275         }
 276         return NULL;
 277
 278 }
 279
 280 static void rt_cpu_seq_stop(struct seq_file *seq, void *v)
 281 {
 282
 283 }
 284
 285 static int rt_cpu_seq_show(struct seq_file *seq, void *v)
 286 {
 287         struct rt_cache_stat *st = v;
 288
 289         if (v == SEQ_START_TOKEN) {
 290                 seq_printf(seq, "entries  in_hit in_slow_tot in_slow_mc in_no_route in_brd in_martian_dst in_martian_src  out_hit out_slow_tot out_slow_mc  gc_total gc_ignored gc_goal_miss gc_dst_overflow in_hlist_search out_hlist_search\n");
 291                 return 0;
 292         }
 293
 294         seq_printf(seq,"%08x  %08x %08x %08x %08x %08x %08x %08x "
 295                    " %08x %08x %08x %08x %08x %08x %08x %08x %08x \n",
 296                    dst_entries_get_slow(&ipv4_dst_ops),
 297                    0, /* st->in_hit */
 298                    st->in_slow_tot,
 299                    st->in_slow_mc,
 300                    st->in_no_route,
 301                    st->in_brd,
 302                    st->in_martian_dst,
 303                    st->in_martian_src,
 304
 305                    0, /* st->out_hit */
 306                    st->out_slow_tot,
 307                    st->out_slow_mc,
 308
 309                    0, /* st->gc_total */
 310                    0, /* st->gc_ignored */
 311                    0, /* st->gc_goal_miss */
 312                    0, /* st->gc_dst_overflow */
 313                    0, /* st->in_hlist_search */
 314                    0  /* st->out_hlist_search */
 315                 );
 316         return 0;
 317 }
 318
 319 static const struct seq_operations rt_cpu_seq_ops = {
 320         .start  = rt_cpu_seq_start,
 321         .next   = rt_cpu_seq_next,
 322         .stop   = rt_cpu_seq_stop,
 323         .show   = rt_cpu_seq_show,
 324 };
 325
 326
 327 static int rt_cpu_seq_open(struct inode *inode, struct file *file)
 328 {
 329         return seq_open(file, &rt_cpu_seq_ops);
 330 }
 331
 332 static const struct file_operations rt_cpu_seq_fops = {
 333         .open    = rt_cpu_seq_open,
 334         .read    = seq_read,
 335         .llseek  = seq_lseek,
 336         .release = seq_release,
 337 };
 338
 339 #ifdef CONFIG_IP_ROUTE_CLASSID
 340 static int rt_acct_proc_show(struct seq_file *m, void *v)
 341 {
 342         struct ip_rt_acct *dst, *src;
 343         unsigned int i, j;
 344
 345         dst = kcalloc(256, sizeof(struct ip_rt_acct), GFP_KERNEL);
 346         if (!dst)
 347                 return -ENOMEM;
 348
 349         for_each_possible_cpu(i) {
 350                 src = (struct ip_rt_acct *)per_cpu_ptr(ip_rt_acct, i);
 351                 for (j = 0; j < 256; j++) {
 352                         dst[j].o_bytes   += src[j].o_bytes;
 353                         dst[j].o_packets += src[j].o_packets;
 354                         dst[j].i_bytes   += src[j].i_bytes;
 355                         dst[j].i_packets += src[j].i_packets;
 356                 }
 357         }
 358
 359         seq_write(m, dst, 256 * sizeof(struct ip_rt_acct));
 360         kfree(dst);
 361         return 0;
 362 }
 363 #endif
 364
 365 static int __net_init ip_rt_do_proc_init(struct net *net)
 366 {
 367         struct proc_dir_entry *pde;
 368
 369         pde = proc_create("rt_cache", 0444, net->proc_net,
 370                           &rt_cache_seq_fops);
 371         if (!pde)
 372                 goto err1;
 373
 374         pde = proc_create("rt_cache", 0444,
 375                           net->proc_net_stat, &rt_cpu_seq_fops);
 376         if (!pde)
 377                 goto err2;
 378
 379 #ifdef CONFIG_IP_ROUTE_CLASSID
 380         pde = proc_create_single("rt_acct", 0, net->proc_net,
 381                         rt_acct_proc_show);
 382         if (!pde)
 383                 goto err3;
 384 #endif
 385         return 0;
 386
 387 #ifdef CONFIG_IP_ROUTE_CLASSID
 388 err3:
 389         remove_proc_entry("rt_cache", net->proc_net_stat);
 390 #endif
 391 err2:
 392         remove_proc_entry("rt_cache", net->proc_net);
 393 err1:
 394         return -ENOMEM;
 395 }
 396
 397 static void __net_exit ip_rt_do_proc_exit(struct net *net)
 398 {
 399         remove_proc_entry("rt_cache", net->proc_net_stat);
 400         remove_proc_entry("rt_cache", net->proc_net);
 401 #ifdef CONFIG_IP_ROUTE_CLASSID
 402         remove_proc_entry("rt_acct", net->proc_net);
 403 #endif
 404 }
 405
 406 static struct pernet_operations ip_rt_proc_ops __net_initdata =  {
 407         .init = ip_rt_do_proc_init,
 408         .exit = ip_rt_do_proc_exit,
 409 };
 410
 411 static int __init ip_rt_proc_init(void)
 412 {
 413         return register_pernet_subsys(&ip_rt_proc_ops);
 414 }
 415
 416 #else
 417 static inline int ip_rt_proc_init(void)
 418 {
 419         return 0;
 420 }
 421 #endif /* CONFIG_PROC_FS */
 422
 423 static inline bool rt_is_expired(const struct rtable *rth)
 424 {
 425         return rth->rt_genid != rt_genid_ipv4(dev_net(rth->dst.dev));
 426 }
 427
 428 void rt_cache_flush(struct net *net)
 429 {
 430         rt_genid_bump_ipv4(net);
 431 }
 432
 433 static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst,
 434                                            struct sk_buff *skb,
 435                                            const void *daddr)
 436 {
 437         struct net_device *dev = dst->dev;
 438         const __be32 *pkey = daddr;
 439         const struct rtable *rt;
 440         struct neighbour *n;
 441
 442         rt = (const struct rtable *) dst;
 443         if (rt->rt_gateway)
 444                 pkey = (const __be32 *) &rt->rt_gateway;
 445         else if (skb)
 446                 pkey = &ip_hdr(skb)->daddr;
 447
 448         n = __ipv4_neigh_lookup(dev, *(__force u32 *)pkey);
 449         if (n)
 450                 return n;
 451         return neigh_create(&arp_tbl, pkey, dev);
 452 }
 453
 454 static void ipv4_confirm_neigh(const struct dst_entry *dst, const void *daddr)
 455 {
 456         struct net_device *dev = dst->dev;
 457         const __be32 *pkey = daddr;
 458         const struct rtable *rt;
 459
 460         rt = (const struct rtable *)dst;
 461         if (rt->rt_gateway)
 462                 pkey = (const __be32 *)&rt->rt_gateway;
 463         else if (!daddr ||
 464                  (rt->rt_flags &
 465                   (RTCF_MULTICAST | RTCF_BROADCAST | RTCF_LOCAL)))
 466                 return;
 467
 468         __ipv4_confirm_neigh(dev, *(__force u32 *)pkey);
 469 }
 470
 471 #define IP_IDENTS_SZ 2048u
 472
 473 static atomic_t *ip_idents __read_mostly;
 474 static u32 *ip_tstamps __read_mostly;
 475
 476 /* In order to protect privacy, we add a perturbation to identifiers
 477  * if one generator is seldom used. This makes hard for an attacker
 478  * to infer how many packets were sent between two points in time.
 479  */
 480 u32 ip_idents_reserve(u32 hash, int segs)
 481 {
 482         u32 *p_tstamp = ip_tstamps + hash % IP_IDENTS_SZ;
 483         atomic_t *p_id = ip_idents + hash % IP_IDENTS_SZ;
 484         u32 old = READ_ONCE(*p_tstamp);
 485         u32 now = (u32)jiffies;
 486         u32 new, delta = 0;
 487
 488         if (old != now && cmpxchg(p_tstamp, old, now) == old)
 489                 delta = prandom_u32_max(now - old);
 490
 491         /* Do not use atomic_add_return() as it makes UBSAN unhappy */
 492         do {
 493                 old = (u32)atomic_read(p_id);
 494                 new = old + delta + segs;
 495         } while (atomic_cmpxchg(p_id, old, new) != old);
 496
 497         return new - segs;
 498 }
 499 EXPORT_SYMBOL(ip_idents_reserve);
 500
 501 void __ip_select_ident(struct net *net, struct iphdr *iph, int segs)
 502 {
 503         static u32 ip_idents_hashrnd __read_mostly;
 504         u32 hash, id;
 505
 506         net_get_random_once(&ip_idents_hashrnd, sizeof(ip_idents_hashrnd));
 507
 508         hash = jhash_3words((__force u32)iph->daddr,
 509                             (__force u32)iph->saddr,
 510                             iph->protocol ^ net_hash_mix(net),
 511                             ip_idents_hashrnd);
 512         id = ip_idents_reserve(hash, segs);
 513         iph->id = htons(id);
 514 }
 515 EXPORT_SYMBOL(__ip_select_ident);
 516
 517 static void __build_flow_key(const struct net *net, struct flowi4 *fl4,
 518                              const struct sock *sk,
 519                              const struct iphdr *iph,
 520                              int oif, u8 tos,
 521                              u8 prot, u32 mark, int flow_flags)
 522 {
 523         if (sk) {
 524                 const struct inet_sock *inet = inet_sk(sk);
 525
 526                 oif = sk->sk_bound_dev_if;
 527                 mark = sk->sk_mark;
 528                 tos = RT_CONN_FLAGS(sk);
 529                 prot = inet->hdrincl ? IPPROTO_RAW : sk->sk_protocol;
 530         }
 531         flowi4_init_output(fl4, oif, mark, tos,
 532                            RT_SCOPE_UNIVERSE, prot,
 533                            flow_flags,
 534                            iph->daddr, iph->saddr, 0, 0,
 535                            sock_net_uid(net, sk));
 536 }
 537
 538 static void build_skb_flow_key(struct flowi4 *fl4, const struct sk_buff *skb,
 539                                const struct sock *sk)
 540 {
 541         const struct net *net = dev_net(skb->dev);
 542         const struct iphdr *iph = ip_hdr(skb);
 543         int oif = skb->dev->ifindex;
 544         u8 tos = RT_TOS(iph->tos);
 545         u8 prot = iph->protocol;
 546         u32 mark = skb->mark;
 547
 548         __build_flow_key(net, fl4, sk, iph, oif, tos, prot, mark, 0);
 549 }
 550
 551 static void build_sk_flow_key(struct flowi4 *fl4, const struct sock *sk)
 552 {
 553         const struct inet_sock *inet = inet_sk(sk);
 554         const struct ip_options_rcu *inet_opt;
 555         __be32 daddr = inet->inet_daddr;
 556
 557         rcu_read_lock();
 558         inet_opt = rcu_dereference(inet->inet_opt);
 559         if (inet_opt && inet_opt->opt.srr)
 560                 daddr = inet_opt->opt.faddr;
 561         flowi4_init_output(fl4, sk->sk_bound_dev_if, sk->sk_mark,
 562                            RT_CONN_FLAGS(sk), RT_SCOPE_UNIVERSE,
 563                            inet->hdrincl ? IPPROTO_RAW : sk->sk_protocol,
 564                            inet_sk_flowi_flags(sk),
 565                            daddr, inet->inet_saddr, 0, 0, sk->sk_uid);
 566         rcu_read_unlock();
 567 }
 568
 569 static void ip_rt_build_flow_key(struct flowi4 *fl4, const struct sock *sk,
 570                                  const struct sk_buff *skb)
 571 {
 572         if (skb)
 573                 build_skb_flow_key(fl4, skb, sk);
 574         else
 575                 build_sk_flow_key(fl4, sk);
 576 }
 577
 578 static DEFINE_SPINLOCK(fnhe_lock);
 579
 580 static void fnhe_flush_routes(struct fib_nh_exception *fnhe)
 581 {
 582         struct rtable *rt;
 583
 584         rt = rcu_dereference(fnhe->fnhe_rth_input);
 585         if (rt) {
 586                 RCU_INIT_POINTER(fnhe->fnhe_rth_input, NULL);
 587                 dst_dev_put(&rt->dst);
 588                 dst_release(&rt->dst);
 589         }
 590         rt = rcu_dereference(fnhe->fnhe_rth_output);
 591         if (rt) {
 592                 RCU_INIT_POINTER(fnhe->fnhe_rth_output, NULL);
 593                 dst_dev_put(&rt->dst);
 594                 dst_release(&rt->dst);
 595         }
 596 }
 597
 598 static struct fib_nh_exception *fnhe_oldest(struct fnhe_hash_bucket *hash)
 599 {
 600         struct fib_nh_exception *fnhe, *oldest;
 601
 602         oldest = rcu_dereference(hash->chain);
 603         for (fnhe = rcu_dereference(oldest->fnhe_next); fnhe;
 604              fnhe = rcu_dereference(fnhe->fnhe_next)) {
 605                 if (time_before(fnhe->fnhe_stamp, oldest->fnhe_stamp))
 606                         oldest = fnhe;
 607         }
 608         fnhe_flush_routes(oldest);
 609         return oldest;
 610 }
 611
 612 static inline u32 fnhe_hashfun(__be32 daddr)
 613 {
 614         static u32 fnhe_hashrnd __read_mostly;
 615         u32 hval;
 616
 617         net_get_random_once(&fnhe_hashrnd, sizeof(fnhe_hashrnd));
 618         hval = jhash_1word((__force u32) daddr, fnhe_hashrnd);
 619         return hash_32(hval, FNHE_HASH_SHIFT);
 620 }
 621
 622 static void fill_route_from_fnhe(struct rtable *rt, struct fib_nh_exception *fnhe)
 623 {
 624         rt->rt_pmtu = fnhe->fnhe_pmtu;
 625         rt->rt_mtu_locked = fnhe->fnhe_mtu_locked;
 626         rt->dst.expires = fnhe->fnhe_expires;
 627
 628         if (fnhe->fnhe_gw) {
 629                 rt->rt_flags |= RTCF_REDIRECTED;
 630                 rt->rt_gateway = fnhe->fnhe_gw;
 631                 rt->rt_uses_gateway = 1;
 632         }
 633 }
 634
 635 static void update_or_create_fnhe(struct fib_nh *nh, __be32 daddr, __be32 gw,
 636                                   u32 pmtu, bool lock, unsigned long expires)
 637 {
 638         struct fnhe_hash_bucket *hash;
 639         struct fib_nh_exception *fnhe;
 640         struct rtable *rt;
 641         u32 genid, hval;
 642         unsigned int i;
 643         int depth;
 644
 645         genid = fnhe_genid(dev_net(nh->nh_dev));
 646         hval = fnhe_hashfun(daddr);
 647
 648         spin_lock_bh(&fnhe_lock);
 649
 650         hash = rcu_dereference(nh->nh_exceptions);
 651         if (!hash) {
 652                 hash = kcalloc(FNHE_HASH_SIZE, sizeof(*hash), GFP_ATOMIC);
 653                 if (!hash)
 654                         goto out_unlock;
 655                 rcu_assign_pointer(nh->nh_exceptions, hash);
 656         }
 657
 658         hash += hval;
 659
 660         depth = 0;
 661         for (fnhe = rcu_dereference(hash->chain); fnhe;
 662              fnhe = rcu_dereference(fnhe->fnhe_next)) {
 663                 if (fnhe->fnhe_daddr == daddr)
 664                         break;
 665                 depth++;
 666         }
 667
 668         if (fnhe) {
 669                 if (fnhe->fnhe_genid != genid)
 670                         fnhe->fnhe_genid = genid;
 671                 if (gw)
 672                         fnhe->fnhe_gw = gw;
 673                 if (pmtu) {
 674                         fnhe->fnhe_pmtu = pmtu;
 675                         fnhe->fnhe_mtu_locked = lock;
 676                 }
 677                 fnhe->fnhe_expires = max(1UL, expires);
 678                 /* Update all cached dsts too */
 679                 rt = rcu_dereference(fnhe->fnhe_rth_input);
 680                 if (rt)
 681                         fill_route_from_fnhe(rt, fnhe);
 682                 rt = rcu_dereference(fnhe->fnhe_rth_output);
 683                 if (rt)
 684                         fill_route_from_fnhe(rt, fnhe);
 685         } else {
 686                 if (depth > FNHE_RECLAIM_DEPTH)
 687                         fnhe = fnhe_oldest(hash);
 688                 else {
 689                         fnhe = kzalloc(sizeof(*fnhe), GFP_ATOMIC);
 690                         if (!fnhe)
 691                                 goto out_unlock;
 692
 693                         fnhe->fnhe_next = hash->chain;
 694                         rcu_assign_pointer(hash->chain, fnhe);
 695                 }
 696                 fnhe->fnhe_genid = genid;
 697                 fnhe->fnhe_daddr = daddr;
 698                 fnhe->fnhe_gw = gw;
 699                 fnhe->fnhe_pmtu = pmtu;
 700                 fnhe->fnhe_mtu_locked = lock;
 701                 fnhe->fnhe_expires = max(1UL, expires);
 702
 703                 /* Exception created; mark the cached routes for the nexthop
 704                  * stale, so anyone caching it rechecks if this exception
 705                  * applies to them.
 706                  */
 707                 rt = rcu_dereference(nh->nh_rth_input);
 708                 if (rt)
 709                         rt->dst.obsolete = DST_OBSOLETE_KILL;
 710
 711                 for_each_possible_cpu(i) {
 712                         struct rtable __rcu **prt;
 713                         prt = per_cpu_ptr(nh->nh_pcpu_rth_output, i);
 714                         rt = rcu_dereference(*prt);
 715                         if (rt)
 716                                 rt->dst.obsolete = DST_OBSOLETE_KILL;
 717                 }
 718         }
 719
 720         fnhe->fnhe_stamp = jiffies;
 721
 722 out_unlock:
 723         spin_unlock_bh(&fnhe_lock);
 724 }
 725
 726 static void __ip_do_redirect(struct rtable *rt, struct sk_buff *skb, struct flowi4 *fl4,
 727                              bool kill_route)
 728 {
 729         __be32 new_gw = icmp_hdr(skb)->un.gateway;
 730         __be32 old_gw = ip_hdr(skb)->saddr;
 731         struct net_device *dev = skb->dev;
 732         struct in_device *in_dev;
 733         struct fib_result res;
 734         struct neighbour *n;
 735         struct net *net;
 736
 737         switch (icmp_hdr(skb)->code & 7) {
 738         case ICMP_REDIR_NET:
 739         case ICMP_REDIR_NETTOS:
 740         case ICMP_REDIR_HOST:
 741         case ICMP_REDIR_HOSTTOS:
 742                 break;
 743
 744         default:
 745                 return;
 746         }
 747
 748         if (rt->rt_gateway != old_gw)
 749                 return;
 750
 751         in_dev = __in_dev_get_rcu(dev);
 752         if (!in_dev)
 753                 return;
 754
 755         net = dev_net(dev);
 756         if (new_gw == old_gw || !IN_DEV_RX_REDIRECTS(in_dev) ||
 757             ipv4_is_multicast(new_gw) || ipv4_is_lbcast(new_gw) ||
 758             ipv4_is_zeronet(new_gw))
 759                 goto reject_redirect;
 760
 761         if (!IN_DEV_SHARED_MEDIA(in_dev)) {
 762                 if (!inet_addr_onlink(in_dev, new_gw, old_gw))
 763                         goto reject_redirect;
 764                 if (IN_DEV_SEC_REDIRECTS(in_dev) && ip_fib_check_default(new_gw, dev))
 765                         goto reject_redirect;
 766         } else {
 767                 if (inet_addr_type(net, new_gw) != RTN_UNICAST)
 768                         goto reject_redirect;
 769         }
 770
 771         n = __ipv4_neigh_lookup(rt->dst.dev, new_gw);
 772         if (!n)
 773                 n = neigh_create(&arp_tbl, &new_gw, rt->dst.dev);
 774         if (!IS_ERR(n)) {
 775                 if (!(n->nud_state & NUD_VALID)) {
 776                         neigh_event_send(n, NULL);
 777                 } else {
 778                         if (fib_lookup(net, fl4, &res, 0) == 0) {
 779                                 struct fib_nh *nh = &FIB_RES_NH(res);
 780
 781                                 update_or_create_fnhe(nh, fl4->daddr, new_gw,
 782                                                 0, false,
 783                                                 jiffies + ip_rt_gc_timeout);
 784                         }
 785                         if (kill_route)
 786                                 rt->dst.obsolete = DST_OBSOLETE_KILL;
 787                         call_netevent_notifiers(NETEVENT_NEIGH_UPDATE, n);
 788                 }
 789                 neigh_release(n);
 790         }
 791         return;
 792
 793 reject_redirect:
 794 #ifdef CONFIG_IP_ROUTE_VERBOSE
 795         if (IN_DEV_LOG_MARTIANS(in_dev)) {
 796                 const struct iphdr *iph = (const struct iphdr *) skb->data;
 797                 __be32 daddr = iph->daddr;
 798                 __be32 saddr = iph->saddr;
 799
 800                 net_info_ratelimited("Redirect from %pI4 on %s about %pI4 ignored\n"
 801                                      "  Advised path = %pI4 -> %pI4\n",
 802                                      &old_gw, dev->name, &new_gw,
 803                                      &saddr, &daddr);
 804         }
 805 #endif
 806         ;
 807 }
 808
 809 static void ip_do_redirect(struct dst_entry *dst, struct sock *sk, struct sk_buff *skb)
 810 {
 811         struct rtable *rt;
 812         struct flowi4 fl4;
 813         const struct iphdr *iph = (const struct iphdr *) skb->data;
 814         struct net *net = dev_net(skb->dev);
 815         int oif = skb->dev->ifindex;
 816         u8 tos = RT_TOS(iph->tos);
 817         u8 prot = iph->protocol;
 818         u32 mark = skb->mark;
 819
 820         rt = (struct rtable *) dst;
 821
 822         __build_flow_key(net, &fl4, sk, iph, oif, tos, prot, mark, 0);
 823         __ip_do_redirect(rt, skb, &fl4, true);
 824 }
 825
 826 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst)
 827 {
 828         struct rtable *rt = (struct rtable *)dst;
 829         struct dst_entry *ret = dst;
 830
 831         if (rt) {
 832                 if (dst->obsolete > 0) {
 833                         ip_rt_put(rt);
 834                         ret = NULL;
 835                 } else if ((rt->rt_flags & RTCF_REDIRECTED) ||
 836                            rt->dst.expires) {
 837                         ip_rt_put(rt);
 838                         ret = NULL;
 839                 }
 840         }
 841         return ret;
 842 }
 843
 844 /*
 845  * Algorithm:
 846  *      1. The first ip_rt_redirect_number redirects are sent
 847  *         with exponential backoff, then we stop sending them at all,
 848  *         assuming that the host ignores our redirects.
 849  *      2. If we did not see packets requiring redirects
 850  *         during ip_rt_redirect_silence, we assume that the host
 851  *         forgot redirected route and start to send redirects again.
 852  *
 853  * This algorithm is much cheaper and more intelligent than dumb load limiting
 854  * in icmp.c.
 855  *
 856  * NOTE. Do not forget to inhibit load limiting for redirects (redundant)
 857  * and "frag. need" (breaks PMTU discovery) in icmp.c.
 858  */
 859
 860 void ip_rt_send_redirect(struct sk_buff *skb)
 861 {
 862         struct rtable *rt = skb_rtable(skb);
 863         struct in_device *in_dev;
 864         struct inet_peer *peer;
 865         struct net *net;
 866         int log_martians;
 867         int vif;
 868
 869         rcu_read_lock();
 870         in_dev = __in_dev_get_rcu(rt->dst.dev);
 871         if (!in_dev || !IN_DEV_TX_REDIRECTS(in_dev)) {
 872                 rcu_read_unlock();
 873                 return;
 874         }
 875         log_martians = IN_DEV_LOG_MARTIANS(in_dev);
 876         vif = l3mdev_master_ifindex_rcu(rt->dst.dev);
 877         rcu_read_unlock();
 878
 879         net = dev_net(rt->dst.dev);
 880         peer = inet_getpeer_v4(net->ipv4.peers, ip_hdr(skb)->saddr, vif, 1);
 881         if (!peer) {
 882                 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST,
 883                           rt_nexthop(rt, ip_hdr(skb)->daddr));
 884                 return;
 885         }
 886
 887         /* No redirected packets during ip_rt_redirect_silence;
 888          * reset the algorithm.
 889          */
 890         if (time_after(jiffies, peer->rate_last + ip_rt_redirect_silence)) {
 891                 peer->rate_tokens = 0;
 892                 peer->n_redirects = 0;
 893         }
 894
 895         /* Too many ignored redirects; do not send anything
 896          * set dst.rate_last to the last seen redirected packet.
 897          */
 898         if (peer->n_redirects >= ip_rt_redirect_number) {
 899                 peer->rate_last = jiffies;
 900                 goto out_put_peer;
 901         }
 902
 903         /* Check for load limit; set rate_last to the latest sent
 904          * redirect.
 905          */
 906         if (peer->rate_tokens == 0 ||
 907             time_after(jiffies,
 908                        (peer->rate_last +
 909                         (ip_rt_redirect_load << peer->rate_tokens)))) {
 910                 __be32 gw = rt_nexthop(rt, ip_hdr(skb)->daddr);
 911
 912                 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST, gw);
 913                 peer->rate_last = jiffies;
 914                 ++peer->rate_tokens;
 915                 ++peer->n_redirects;
 916 #ifdef CONFIG_IP_ROUTE_VERBOSE
 917                 if (log_martians &&
 918                     peer->rate_tokens == ip_rt_redirect_number)
 919                         net_warn_ratelimited("host %pI4/if%d ignores redirects for %pI4 to %pI4\n",
 920                                              &ip_hdr(skb)->saddr, inet_iif(skb),
 921                                              &ip_hdr(skb)->daddr, &gw);
 922 #endif
 923         }
 924 out_put_peer:
 925         inet_putpeer(peer);
 926 }
 927
 928 static int ip_error(struct sk_buff *skb)
 929 {
 930         struct rtable *rt = skb_rtable(skb);
 931         struct net_device *dev = skb->dev;
 932         struct in_device *in_dev;
 933         struct inet_peer *peer;
 934         unsigned long now;
 935         struct net *net;
 936         bool send;
 937         int code;
 938
 939         if (netif_is_l3_master(skb->dev)) {
 940                 dev = __dev_get_by_index(dev_net(skb->dev), IPCB(skb)->iif);
 941                 if (!dev)
 942                         goto out;
 943         }
 944
 945         in_dev = __in_dev_get_rcu(dev);
 946
 947         /* IP on this device is disabled. */
 948         if (!in_dev)
 949                 goto out;
 950
 951         net = dev_net(rt->dst.dev);
 952         if (!IN_DEV_FORWARD(in_dev)) {
 953                 switch (rt->dst.error) {
 954                 case EHOSTUNREACH:
 955                         __IP_INC_STATS(net, IPSTATS_MIB_INADDRERRORS);
 956                         break;
 957
 958                 case ENETUNREACH:
 959                         __IP_INC_STATS(net, IPSTATS_MIB_INNOROUTES);
 960                         break;
 961                 }
 962                 goto out;
 963         }
 964
 965         switch (rt->dst.error) {
 966         case EINVAL:
 967         default:
 968                 goto out;
 969         case EHOSTUNREACH:
 970                 code = ICMP_HOST_UNREACH;
 971                 break;
 972         case ENETUNREACH:
 973                 code = ICMP_NET_UNREACH;
 974                 __IP_INC_STATS(net, IPSTATS_MIB_INNOROUTES);
 975                 break;
 976         case EACCES:
 977                 code = ICMP_PKT_FILTERED;
 978                 break;
 979         }
 980
 981         peer = inet_getpeer_v4(net->ipv4.peers, ip_hdr(skb)->saddr,
 982                                l3mdev_master_ifindex(skb->dev), 1);
 983
 984         send = true;
 985         if (peer) {
 986                 now = jiffies;
 987                 peer->rate_tokens += now - peer->rate_last;
 988                 if (peer->rate_tokens > ip_rt_error_burst)
 989                         peer->rate_tokens = ip_rt_error_burst;
 990                 peer->rate_last = now;
 991                 if (peer->rate_tokens >= ip_rt_error_cost)
 992                         peer->rate_tokens -= ip_rt_error_cost;
 993                 else
 994                         send = false;
 995                 inet_putpeer(peer);
 996         }
 997         if (send)
 998                 icmp_send(skb, ICMP_DEST_UNREACH, code, 0);
 999
1000 out:    kfree_skb(skb);
1001         return 0;
1002 }
1003
1004 static void __ip_rt_update_pmtu(struct rtable *rt, struct flowi4 *fl4, u32 mtu)
1005 {
1006         struct dst_entry *dst = &rt->dst;
1007         u32 old_mtu = ipv4_mtu(dst);
1008         struct fib_result res;
1009         bool lock = false;
1010
1011         if (ip_mtu_locked(dst))
1012                 return;
1013
1014         if (old_mtu < mtu)
1015                 return;
1016
1017         if (mtu < ip_rt_min_pmtu) {
1018                 lock = true;
1019                 mtu = min(old_mtu, ip_rt_min_pmtu);
1020         }
1021
1022         if (rt->rt_pmtu == mtu && !lock &&
1023             time_before(jiffies, dst->expires - ip_rt_mtu_expires / 2))
1024                 return;
1025
1026         rcu_read_lock();
1027         if (fib_lookup(dev_net(dst->dev), fl4, &res, 0) == 0) {
1028                 struct fib_nh *nh = &FIB_RES_NH(res);
1029
1030                 update_or_create_fnhe(nh, fl4->daddr, 0, mtu, lock,
1031                                       jiffies + ip_rt_mtu_expires);
1032         }
1033         rcu_read_unlock();
1034 }
1035
1036 static void ip_rt_update_pmtu(struct dst_entry *dst, struct sock *sk,
1037                               struct sk_buff *skb, u32 mtu)
1038 {
1039         struct rtable *rt = (struct rtable *) dst;
1040         struct flowi4 fl4;
1041
1042         ip_rt_build_flow_key(&fl4, sk, skb);
1043         __ip_rt_update_pmtu(rt, &fl4, mtu);
1044 }
1045
1046 void ipv4_update_pmtu(struct sk_buff *skb, struct net *net, u32 mtu,
1047                       int oif, u8 protocol)
1048 {
1049         const struct iphdr *iph = (const struct iphdr *) skb->data;
1050         struct flowi4 fl4;
1051         struct rtable *rt;
1052         u32 mark = IP4_REPLY_MARK(net, skb->mark);
1053
1054         __build_flow_key(net, &fl4, NULL, iph, oif,
1055                          RT_TOS(iph->tos), protocol, mark, 0);
1056         rt = __ip_route_output_key(net, &fl4);
1057         if (!IS_ERR(rt)) {
1058                 __ip_rt_update_pmtu(rt, &fl4, mtu);
1059                 ip_rt_put(rt);
1060         }
1061 }
1062 EXPORT_SYMBOL_GPL(ipv4_update_pmtu);
1063
1064 static void __ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu)
1065 {
1066         const struct iphdr *iph = (const struct iphdr *) skb->data;
1067         struct flowi4 fl4;
1068         struct rtable *rt;
1069
1070         __build_flow_key(sock_net(sk), &fl4, sk, iph, 0, 0, 0, 0, 0);
1071
1072         if (!fl4.flowi4_mark)
1073                 fl4.flowi4_mark = IP4_REPLY_MARK(sock_net(sk), skb->mark);
1074
1075         rt = __ip_route_output_key(sock_net(sk), &fl4);
1076         if (!IS_ERR(rt)) {
1077                 __ip_rt_update_pmtu(rt, &fl4, mtu);
1078                 ip_rt_put(rt);
1079         }
1080 }
1081
1082 void ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu)
1083 {
1084         const struct iphdr *iph = (const struct iphdr *) skb->data;
1085         struct flowi4 fl4;
1086         struct rtable *rt;
1087         struct dst_entry *odst = NULL;
1088         bool new = false;
1089         struct net *net = sock_net(sk);
1090
1091         bh_lock_sock(sk);
1092
1093         if (!ip_sk_accept_pmtu(sk))
1094                 goto out;
1095
1096         odst = sk_dst_get(sk);
1097
1098         if (sock_owned_by_user(sk) || !odst) {
1099                 __ipv4_sk_update_pmtu(skb, sk, mtu);
1100                 goto out;
1101         }
1102
1103         __build_flow_key(net, &fl4, sk, iph, 0, 0, 0, 0, 0);
1104
1105         rt = (struct rtable *)odst;
1106         if (odst->obsolete && !odst->ops->check(odst, 0)) {
1107                 rt = ip_route_output_flow(sock_net(sk), &fl4, sk);
1108                 if (IS_ERR(rt))
1109                         goto out;
1110
1111                 new = true;
1112         }
1113
1114         __ip_rt_update_pmtu((struct rtable *) xfrm_dst_path(&rt->dst), &fl4, mtu);
1115
1116         if (!dst_check(&rt->dst, 0)) {
1117                 if (new)
1118                         dst_release(&rt->dst);
1119
1120                 rt = ip_route_output_flow(sock_net(sk), &fl4, sk);
1121                 if (IS_ERR(rt))
1122                         goto out;
1123
1124                 new = true;
1125         }
1126
1127         if (new)
1128                 sk_dst_set(sk, &rt->dst);
1129
1130 out:
1131         bh_unlock_sock(sk);
1132         dst_release(odst);
1133 }
1134 EXPORT_SYMBOL_GPL(ipv4_sk_update_pmtu);
1135
1136 void ipv4_redirect(struct sk_buff *skb, struct net *net,
1137                    int oif, u8 protocol)
1138 {
1139         const struct iphdr *iph = (const struct iphdr *) skb->data;
1140         struct flowi4 fl4;
1141         struct rtable *rt;
1142
1143         __build_flow_key(net, &fl4, NULL, iph, oif,
1144                          RT_TOS(iph->tos), protocol, 0, 0);
1145         rt = __ip_route_output_key(net, &fl4);
1146         if (!IS_ERR(rt)) {
1147                 __ip_do_redirect(rt, skb, &fl4, false);
1148                 ip_rt_put(rt);
1149         }
1150 }
1151 EXPORT_SYMBOL_GPL(ipv4_redirect);
1152
1153 void ipv4_sk_redirect(struct sk_buff *skb, struct sock *sk)
1154 {
1155         const struct iphdr *iph = (const struct iphdr *) skb->data;
1156         struct flowi4 fl4;
1157         struct rtable *rt;
1158         struct net *net = sock_net(sk);
1159
1160         __build_flow_key(net, &fl4, sk, iph, 0, 0, 0, 0, 0);
1161         rt = __ip_route_output_key(net, &fl4);
1162         if (!IS_ERR(rt)) {
1163                 __ip_do_redirect(rt, skb, &fl4, false);
1164                 ip_rt_put(rt);
1165         }
1166 }
1167 EXPORT_SYMBOL_GPL(ipv4_sk_redirect);
1168
1169 static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie)
1170 {
1171         struct rtable *rt = (struct rtable *) dst;
1172
1173         /* All IPV4 dsts are created with ->obsolete set to the value
1174          * DST_OBSOLETE_FORCE_CHK which forces validation calls down
1175          * into this function always.
1176          *
1177          * When a PMTU/redirect information update invalidates a route,
1178          * this is indicated by setting obsolete to DST_OBSOLETE_KILL or
1179          * DST_OBSOLETE_DEAD by dst_free().
1180          */
1181         if (dst->obsolete != DST_OBSOLETE_FORCE_CHK || rt_is_expired(rt))
1182                 return NULL;
1183         return dst;
1184 }
1185
1186 static void ipv4_link_failure(struct sk_buff *skb)
1187 {
1188         struct rtable *rt;
1189
1190         icmp_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0);
1191
1192         rt = skb_rtable(skb);
1193         if (rt)
1194                 dst_set_expires(&rt->dst, 0);
1195 }
1196
1197 static int ip_rt_bug(struct net *net, struct sock *sk, struct sk_buff *skb)
1198 {
1199         pr_debug("%s: %pI4 -> %pI4, %s\n",
1200                  __func__, &ip_hdr(skb)->saddr, &ip_hdr(skb)->daddr,
1201                  skb->dev ? skb->dev->name : "?");
1202         kfree_skb(skb);
1203         WARN_ON(1);
1204         return 0;
1205 }
1206
1207 /*
1208    We do not cache source address of outgoing interface,
1209    because it is used only by IP RR, TS and SRR options,
1210    so that it out of fast path.
1211
1212    BTW remember: "addr" is allowed to be not aligned
1213    in IP options!
1214  */
1215
1216 void ip_rt_get_source(u8 *addr, struct sk_buff *skb, struct rtable *rt)
1217 {
1218         __be32 src;
1219
1220         if (rt_is_output_route(rt))
1221                 src = ip_hdr(skb)->saddr;
1222         else {
1223                 struct fib_result res;
1224                 struct iphdr *iph = ip_hdr(skb);
1225                 struct flowi4 fl4 = {
1226                         .daddr = iph->daddr,
1227                         .saddr = iph->saddr,
1228                         .flowi4_tos = RT_TOS(iph->tos),
1229                         .flowi4_oif = rt->dst.dev->ifindex,
1230                         .flowi4_iif = skb->dev->ifindex,
1231                         .flowi4_mark = skb->mark,
1232                 };
1233
1234                 rcu_read_lock();
1235                 if (fib_lookup(dev_net(rt->dst.dev), &fl4, &res, 0) == 0)
1236                         src = FIB_RES_PREFSRC(dev_net(rt->dst.dev), res);
1237                 else
1238                         src = inet_select_addr(rt->dst.dev,
1239                                                rt_nexthop(rt, iph->daddr),
1240                                                RT_SCOPE_UNIVERSE);
1241                 rcu_read_unlock();
1242         }
1243         memcpy(addr, &src, 4);
1244 }
1245
1246 #ifdef CONFIG_IP_ROUTE_CLASSID
1247 static void set_class_tag(struct rtable *rt, u32 tag)
1248 {
1249         if (!(rt->dst.tclassid & 0xFFFF))
1250                 rt->dst.tclassid |= tag & 0xFFFF;
1251         if (!(rt->dst.tclassid & 0xFFFF0000))
1252                 rt->dst.tclassid |= tag & 0xFFFF0000;
1253 }
1254 #endif
1255
1256 static unsigned int ipv4_default_advmss(const struct dst_entry *dst)
1257 {
1258         unsigned int header_size = sizeof(struct tcphdr) + sizeof(struct iphdr);
1259         unsigned int advmss = max_t(unsigned int, ipv4_mtu(dst) - header_size,
1260                                     ip_rt_min_advmss);
1261
1262         return min(advmss, IPV4_MAX_PMTU - header_size);
1263 }
1264
1265 static unsigned int ipv4_mtu(const struct dst_entry *dst)
1266 {
1267         const struct rtable *rt = (const struct rtable *) dst;
1268         unsigned int mtu = rt->rt_pmtu;
1269
1270         if (!mtu || time_after_eq(jiffies, rt->dst.expires))
1271                 mtu = dst_metric_raw(dst, RTAX_MTU);
1272
1273         if (mtu)
1274                 return mtu;
1275
1276         mtu = READ_ONCE(dst->dev->mtu);
1277
1278         if (unlikely(ip_mtu_locked(dst))) {
1279                 if (rt->rt_uses_gateway && mtu > 576)
1280                         mtu = 576;
1281         }
1282
1283         mtu = min_t(unsigned int, mtu, IP_MAX_MTU);
1284
1285         return mtu - lwtunnel_headroom(dst->lwtstate, mtu);
1286 }
1287
1288 static void ip_del_fnhe(struct fib_nh *nh, __be32 daddr)
1289 {
1290         struct fnhe_hash_bucket *hash;
1291         struct fib_nh_exception *fnhe, __rcu **fnhe_p;
1292         u32 hval = fnhe_hashfun(daddr);
1293
1294         spin_lock_bh(&fnhe_lock);
1295
1296         hash = rcu_dereference_protected(nh->nh_exceptions,
1297                                          lockdep_is_held(&fnhe_lock));
1298         hash += hval;
1299
1300         fnhe_p = &hash->chain;
1301         fnhe = rcu_dereference_protected(*fnhe_p, lockdep_is_held(&fnhe_lock));
1302         while (fnhe) {
1303                 if (fnhe->fnhe_daddr == daddr) {
1304                         rcu_assign_pointer(*fnhe_p, rcu_dereference_protected(
1305                                 fnhe->fnhe_next, lockdep_is_held(&fnhe_lock)));
1306                         fnhe_flush_routes(fnhe);
1307                         kfree_rcu(fnhe, rcu);
1308                         break;
1309                 }
1310                 fnhe_p = &fnhe->fnhe_next;
1311                 fnhe = rcu_dereference_protected(fnhe->fnhe_next,
1312                                                  lockdep_is_held(&fnhe_lock));
1313         }
1314
1315         spin_unlock_bh(&fnhe_lock);
1316 }
1317
1318 static struct fib_nh_exception *find_exception(struct fib_nh *nh, __be32 daddr)
1319 {
1320         struct fnhe_hash_bucket *hash = rcu_dereference(nh->nh_exceptions);
1321         struct fib_nh_exception *fnhe;
1322         u32 hval;
1323
1324         if (!hash)
1325                 return NULL;
1326
1327         hval = fnhe_hashfun(daddr);
1328
1329         for (fnhe = rcu_dereference(hash[hval].chain); fnhe;
1330              fnhe = rcu_dereference(fnhe->fnhe_next)) {
1331                 if (fnhe->fnhe_daddr == daddr) {
1332                         if (fnhe->fnhe_expires &&
1333                             time_after(jiffies, fnhe->fnhe_expires)) {
1334                                 ip_del_fnhe(nh, daddr);
1335                                 break;
1336                         }
1337                         return fnhe;
1338                 }
1339         }
1340         return NULL;
1341 }
1342
1343 /* MTU selection:
1344  * 1. mtu on route is locked - use it
1345  * 2. mtu from nexthop exception
1346  * 3. mtu from egress device
1347  */
1348
1349 u32 ip_mtu_from_fib_result(struct fib_result *res, __be32 daddr)
1350 {
1351         struct fib_info *fi = res->fi;
1352         struct fib_nh *nh = &fi->fib_nh[res->nh_sel];
1353         struct net_device *dev = nh->nh_dev;
1354         u32 mtu = 0;
1355
1356         if (dev_net(dev)->ipv4.sysctl_ip_fwd_use_pmtu ||
1357             fi->fib_metrics->metrics[RTAX_LOCK - 1] & (1 << RTAX_MTU))
1358                 mtu = fi->fib_mtu;
1359
1360         if (likely(!mtu)) {
1361                 struct fib_nh_exception *fnhe;
1362
1363                 fnhe = find_exception(nh, daddr);
1364                 if (fnhe && !time_after_eq(jiffies, fnhe->fnhe_expires))
1365                         mtu = fnhe->fnhe_pmtu;
1366         }
1367
1368         if (likely(!mtu))
1369                 mtu = min(READ_ONCE(dev->mtu), IP_MAX_MTU);
1370
1371         return mtu - lwtunnel_headroom(nh->nh_lwtstate, mtu);
1372 }
1373
1374 static bool rt_bind_exception(struct rtable *rt, struct fib_nh_exception *fnhe,
1375                               __be32 daddr, const bool do_cache)
1376 {
1377         bool ret = false;
1378
1379         spin_lock_bh(&fnhe_lock);
1380
1381         if (daddr == fnhe->fnhe_daddr) {
1382                 struct rtable __rcu **porig;
1383                 struct rtable *orig;
1384                 int genid = fnhe_genid(dev_net(rt->dst.dev));
1385
1386                 if (rt_is_input_route(rt))
1387                         porig = &fnhe->fnhe_rth_input;
1388                 else
1389                         porig = &fnhe->fnhe_rth_output;
1390                 orig = rcu_dereference(*porig);
1391
1392                 if (fnhe->fnhe_genid != genid) {
1393                         fnhe->fnhe_genid = genid;
1394                         fnhe->fnhe_gw = 0;
1395                         fnhe->fnhe_pmtu = 0;
1396                         fnhe->fnhe_expires = 0;
1397                         fnhe->fnhe_mtu_locked = false;
1398                         fnhe_flush_routes(fnhe);
1399                         orig = NULL;
1400                 }
1401                 fill_route_from_fnhe(rt, fnhe);
1402                 if (!rt->rt_gateway)
1403                         rt->rt_gateway = daddr;
1404
1405                 if (do_cache) {
1406                         dst_hold(&rt->dst);
1407                         rcu_assign_pointer(*porig, rt);
1408                         if (orig) {
1409                                 dst_dev_put(&orig->dst);
1410                                 dst_release(&orig->dst);
1411                         }
1412                         ret = true;
1413                 }
1414
1415                 fnhe->fnhe_stamp = jiffies;
1416         }
1417         spin_unlock_bh(&fnhe_lock);
1418
1419         return ret;
1420 }
1421
1422 static bool rt_cache_route(struct fib_nh *nh, struct rtable *rt)
1423 {
1424         struct rtable *orig, *prev, **p;
1425         bool ret = true;
1426
1427         if (rt_is_input_route(rt)) {
1428                 p = (struct rtable **)&nh->nh_rth_input;
1429         } else {
1430                 p = (struct rtable **)raw_cpu_ptr(nh->nh_pcpu_rth_output);
1431         }
1432         orig = *p;
1433
1434         /* hold dst before doing cmpxchg() to avoid race condition
1435          * on this dst
1436          */
1437         dst_hold(&rt->dst);
1438         prev = cmpxchg(p, orig, rt);
1439         if (prev == orig) {
1440                 if (orig) {
1441                         dst_dev_put(&orig->dst);
1442                         dst_release(&orig->dst);
1443                 }
1444         } else {
1445                 dst_release(&rt->dst);
1446                 ret = false;
1447         }
1448
1449         return ret;
1450 }
1451
1452 struct uncached_list {
1453         spinlock_t              lock;
1454         struct list_head        head;
1455 };
1456
1457 static DEFINE_PER_CPU_ALIGNED(struct uncached_list, rt_uncached_list);
1458
1459 void rt_add_uncached_list(struct rtable *rt)
1460 {
1461         struct uncached_list *ul = raw_cpu_ptr(&rt_uncached_list);
1462
1463         rt->rt_uncached_list = ul;
1464
1465         spin_lock_bh(&ul->lock);
1466         list_add_tail(&rt->rt_uncached, &ul->head);
1467         spin_unlock_bh(&ul->lock);
1468 }
1469
1470 void rt_del_uncached_list(struct rtable *rt)
1471 {
1472         if (!list_empty(&rt->rt_uncached)) {
1473                 struct uncached_list *ul = rt->rt_uncached_list;
1474
1475                 spin_lock_bh(&ul->lock);
1476                 list_del(&rt->rt_uncached);
1477                 spin_unlock_bh(&ul->lock);
1478         }
1479 }
1480
1481 static void ipv4_dst_destroy(struct dst_entry *dst)
1482 {
1483         struct rtable *rt = (struct rtable *)dst;
1484
1485         ip_dst_metrics_put(dst);
1486         rt_del_uncached_list(rt);
1487 }
1488
1489 void rt_flush_dev(struct net_device *dev)
1490 {
1491         struct net *net = dev_net(dev);
1492         struct rtable *rt;
1493         int cpu;
1494
1495         for_each_possible_cpu(cpu) {
1496                 struct uncached_list *ul = &per_cpu(rt_uncached_list, cpu);
1497
1498                 spin_lock_bh(&ul->lock);
1499                 list_for_each_entry(rt, &ul->head, rt_uncached) {
1500                         if (rt->dst.dev != dev)
1501                                 continue;
1502                         rt->dst.dev = net->loopback_dev;
1503                         dev_hold(rt->dst.dev);
1504                         dev_put(dev);
1505                 }
1506                 spin_unlock_bh(&ul->lock);
1507         }
1508 }
1509
1510 static bool rt_cache_valid(const struct rtable *rt)
1511 {
1512         return  rt &&
1513                 rt->dst.obsolete == DST_OBSOLETE_FORCE_CHK &&
1514                 !rt_is_expired(rt);
1515 }
1516
1517 static void rt_set_nexthop(struct rtable *rt, __be32 daddr,
1518                            const struct fib_result *res,
1519                            struct fib_nh_exception *fnhe,
1520                            struct fib_info *fi, u16 type, u32 itag,
1521                            const bool do_cache)
1522 {
1523         bool cached = false;
1524
1525         if (fi) {
1526                 struct fib_nh *nh = &FIB_RES_NH(*res);
1527
1528                 if (nh->nh_gw && nh->nh_scope == RT_SCOPE_LINK) {
1529                         rt->rt_gateway = nh->nh_gw;
1530                         rt->rt_uses_gateway = 1;
1531                 }
1532                 ip_dst_init_metrics(&rt->dst, fi->fib_metrics);
1533
1534 #ifdef CONFIG_IP_ROUTE_CLASSID
1535                 rt->dst.tclassid = nh->nh_tclassid;
1536 #endif
1537                 rt->dst.lwtstate = lwtstate_get(nh->nh_lwtstate);
1538                 if (unlikely(fnhe))
1539                         cached = rt_bind_exception(rt, fnhe, daddr, do_cache);
1540                 else if (do_cache)
1541                         cached = rt_cache_route(nh, rt);
1542                 if (unlikely(!cached)) {
1543                         /* Routes we intend to cache in nexthop exception or
1544                          * FIB nexthop have the DST_NOCACHE bit clear.
1545                          * However, if we are unsuccessful at storing this
1546                          * route into the cache we really need to set it.
1547                          */
1548                         if (!rt->rt_gateway)
1549                                 rt->rt_gateway = daddr;
1550                         rt_add_uncached_list(rt);
1551                 }
1552         } else
1553                 rt_add_uncached_list(rt);
1554
1555 #ifdef CONFIG_IP_ROUTE_CLASSID
1556 #ifdef CONFIG_IP_MULTIPLE_TABLES
1557         set_class_tag(rt, res->tclassid);
1558 #endif
1559         set_class_tag(rt, itag);
1560 #endif
1561 }
1562
1563 struct rtable *rt_dst_alloc(struct net_device *dev,
1564                             unsigned int flags, u16 type,
1565                             bool nopolicy, bool noxfrm, bool will_cache)
1566 {
1567         struct rtable *rt;
1568
1569         rt = dst_alloc(&ipv4_dst_ops, dev, 1, DST_OBSOLETE_FORCE_CHK,
1570                        (will_cache ? 0 : DST_HOST) |
1571                        (nopolicy ? DST_NOPOLICY : 0) |
1572                        (noxfrm ? DST_NOXFRM : 0));
1573
1574         if (rt) {
1575                 rt->rt_genid = rt_genid_ipv4(dev_net(dev));
1576                 rt->rt_flags = flags;
1577                 rt->rt_type = type;
1578                 rt->rt_is_input = 0;
1579                 rt->rt_iif = 0;
1580                 rt->rt_pmtu = 0;
1581                 rt->rt_mtu_locked = 0;
1582                 rt->rt_gateway = 0;
1583                 rt->rt_uses_gateway = 0;
1584                 INIT_LIST_HEAD(&rt->rt_uncached);
1585
1586                 rt->dst.output = ip_output;
1587                 if (flags & RTCF_LOCAL)
1588                         rt->dst.input = ip_local_deliver;
1589         }
1590
1591         return rt;
1592 }
1593 EXPORT_SYMBOL(rt_dst_alloc);
1594
1595 /* called in rcu_read_lock() section */
1596 int ip_mc_validate_source(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1597                           u8 tos, struct net_device *dev,
1598                           struct in_device *in_dev, u32 *itag)
1599 {
1600         int err;
1601
1602         /* Primary sanity checks. */
1603         if (!in_dev)
1604                 return -EINVAL;
1605
1606         if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) ||
1607             skb->protocol != htons(ETH_P_IP))
1608                 return -EINVAL;
1609
1610         if (ipv4_is_loopback(saddr) && !IN_DEV_ROUTE_LOCALNET(in_dev))
1611                 return -EINVAL;
1612
1613         if (ipv4_is_zeronet(saddr)) {
1614                 if (!ipv4_is_local_multicast(daddr))
1615                         return -EINVAL;
1616         } else {
1617                 err = fib_validate_source(skb, saddr, 0, tos, 0, dev,
1618                                           in_dev, itag);
1619                 if (err < 0)
1620                         return err;
1621         }
1622         return 0;
1623 }
1624
1625 /* called in rcu_read_lock() section */
1626 static int ip_route_input_mc(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1627                              u8 tos, struct net_device *dev, int our)
1628 {
1629         struct in_device *in_dev = __in_dev_get_rcu(dev);
1630         unsigned int flags = RTCF_MULTICAST;
1631         struct rtable *rth;
1632         u32 itag = 0;
1633         int err;
1634
1635         err = ip_mc_validate_source(skb, daddr, saddr, tos, dev, in_dev, &itag);
1636         if (err)
1637                 return err;
1638
1639         if (our)
1640                 flags |= RTCF_LOCAL;
1641
1642         rth = rt_dst_alloc(dev_net(dev)->loopback_dev, flags, RTN_MULTICAST,
1643                            IN_DEV_CONF_GET(in_dev, NOPOLICY), false, false);
1644         if (!rth)
1645                 return -ENOBUFS;
1646
1647 #ifdef CONFIG_IP_ROUTE_CLASSID
1648         rth->dst.tclassid = itag;
1649 #endif
1650         rth->dst.output = ip_rt_bug;
1651         rth->rt_is_input= 1;
1652
1653 #ifdef CONFIG_IP_MROUTE
1654         if (!ipv4_is_local_multicast(daddr) && IN_DEV_MFORWARD(in_dev))
1655                 rth->dst.input = ip_mr_input;
1656 #endif
1657         RT_CACHE_STAT_INC(in_slow_mc);
1658
1659         skb_dst_set(skb, &rth->dst);
1660         return 0;
1661 }
1662
1663
1664 static void ip_handle_martian_source(struct net_device *dev,
1665                                      struct in_device *in_dev,
1666                                      struct sk_buff *skb,
1667                                      __be32 daddr,
1668                                      __be32 saddr)
1669 {
1670         RT_CACHE_STAT_INC(in_martian_src);
1671 #ifdef CONFIG_IP_ROUTE_VERBOSE
1672         if (IN_DEV_LOG_MARTIANS(in_dev) && net_ratelimit()) {
1673                 /*
1674                  *      RFC1812 recommendation, if source is martian,
1675                  *      the only hint is MAC header.
1676                  */
1677                 pr_warn("martian source %pI4 from %pI4, on dev %s\n",
1678                         &daddr, &saddr, dev->name);
1679                 if (dev->hard_header_len && skb_mac_header_was_set(skb)) {
1680                         print_hex_dump(KERN_WARNING, "ll header: ",
1681                                        DUMP_PREFIX_OFFSET, 16, 1,
1682                                        skb_mac_header(skb),
1683                                        dev->hard_header_len, false);
1684                 }
1685         }
1686 #endif
1687 }
1688
1689 /* called in rcu_read_lock() section */
1690 static int __mkroute_input(struct sk_buff *skb,
1691                            const struct fib_result *res,
1692                            struct in_device *in_dev,
1693                            __be32 daddr, __be32 saddr, u32 tos)
1694 {
1695         struct fib_nh_exception *fnhe;
1696         struct rtable *rth;
1697         int err;
1698         struct in_device *out_dev;
1699         bool do_cache;
1700         u32 itag = 0;
1701
1702         /* get a working reference to the output device */
1703         out_dev = __in_dev_get_rcu(FIB_RES_DEV(*res));
1704         if (!out_dev) {
1705                 net_crit_ratelimited("Bug in ip_route_input_slow(). Please report.\n");
1706                 return -EINVAL;
1707         }
1708
1709         err = fib_validate_source(skb, saddr, daddr, tos, FIB_RES_OIF(*res),
1710                                   in_dev->dev, in_dev, &itag);
1711         if (err < 0) {
1712                 ip_handle_martian_source(in_dev->dev, in_dev, skb, daddr,
1713                                          saddr);
1714
1715                 goto cleanup;
1716         }
1717
1718         do_cache = res->fi && !itag;
1719         if (out_dev == in_dev && err && IN_DEV_TX_REDIRECTS(out_dev) &&
1720             skb->protocol == htons(ETH_P_IP) &&
1721             (IN_DEV_SHARED_MEDIA(out_dev) ||
1722              inet_addr_onlink(out_dev, saddr, FIB_RES_GW(*res))))
1723                 IPCB(skb)->flags |= IPSKB_DOREDIRECT;
1724
1725         if (skb->protocol != htons(ETH_P_IP)) {
1726                 /* Not IP (i.e. ARP). Do not create route, if it is
1727                  * invalid for proxy arp. DNAT routes are always valid.
1728                  *
1729                  * Proxy arp feature have been extended to allow, ARP
1730                  * replies back to the same interface, to support
1731                  * Private VLAN switch technologies. See arp.c.
1732                  */
1733                 if (out_dev == in_dev &&
1734                     IN_DEV_PROXY_ARP_PVLAN(in_dev) == 0) {
1735                         err = -EINVAL;
1736                         goto cleanup;
1737                 }
1738         }
1739
1740         fnhe = find_exception(&FIB_RES_NH(*res), daddr);
1741         if (do_cache) {
1742                 if (fnhe)
1743                         rth = rcu_dereference(fnhe->fnhe_rth_input);
1744                 else
1745                         rth = rcu_dereference(FIB_RES_NH(*res).nh_rth_input);
1746                 if (rt_cache_valid(rth)) {
1747                         skb_dst_set_noref(skb, &rth->dst);
1748                         goto out;
1749                 }
1750         }
1751
1752         rth = rt_dst_alloc(out_dev->dev, 0, res->type,
1753                            IN_DEV_CONF_GET(in_dev, NOPOLICY),
1754                            IN_DEV_CONF_GET(out_dev, NOXFRM), do_cache);
1755         if (!rth) {
1756                 err = -ENOBUFS;
1757                 goto cleanup;
1758         }
1759
1760         rth->rt_is_input = 1;
1761         RT_CACHE_STAT_INC(in_slow_tot);
1762
1763         rth->dst.input = ip_forward;
1764
1765         rt_set_nexthop(rth, daddr, res, fnhe, res->fi, res->type, itag,
1766                        do_cache);
1767         lwtunnel_set_redirect(&rth->dst);
1768         skb_dst_set(skb, &rth->dst);
1769 out:
1770         err = 0;
1771  cleanup:
1772         return err;
1773 }
1774
1775 #ifdef CONFIG_IP_ROUTE_MULTIPATH
1776 /* To make ICMP packets follow the right flow, the multipath hash is
1777  * calculated from the inner IP addresses.
1778  */
1779 static void ip_multipath_l3_keys(const struct sk_buff *skb,
1780                                  struct flow_keys *hash_keys)
1781 {
1782         const struct iphdr *outer_iph = ip_hdr(skb);
1783         const struct iphdr *key_iph = outer_iph;
1784         const struct iphdr *inner_iph;
1785         const struct icmphdr *icmph;
1786         struct iphdr _inner_iph;
1787         struct icmphdr _icmph;
1788
1789         if (likely(outer_iph->protocol != IPPROTO_ICMP))
1790                 goto out;
1791
1792         if (unlikely((outer_iph->frag_off & htons(IP_OFFSET)) != 0))
1793                 goto out;
1794
1795         icmph = skb_header_pointer(skb, outer_iph->ihl * 4, sizeof(_icmph),
1796                                    &_icmph);
1797         if (!icmph)
1798                 goto out;
1799
1800         if (icmph->type != ICMP_DEST_UNREACH &&
1801             icmph->type != ICMP_REDIRECT &&
1802             icmph->type != ICMP_TIME_EXCEEDED &&
1803             icmph->type != ICMP_PARAMETERPROB)
1804                 goto out;
1805
1806         inner_iph = skb_header_pointer(skb,
1807                                        outer_iph->ihl * 4 + sizeof(_icmph),
1808                                        sizeof(_inner_iph), &_inner_iph);
1809         if (!inner_iph)
1810                 goto out;
1811
1812         key_iph = inner_iph;
1813 out:
1814         hash_keys->addrs.v4addrs.src = key_iph->saddr;
1815         hash_keys->addrs.v4addrs.dst = key_iph->daddr;
1816 }
1817
1818 /* if skb is set it will be used and fl4 can be NULL */
1819 int fib_multipath_hash(const struct net *net, const struct flowi4 *fl4,
1820                        const struct sk_buff *skb, struct flow_keys *flkeys)
1821 {
1822         struct flow_keys hash_keys;
1823         u32 mhash;
1824
1825         switch (net->ipv4.sysctl_fib_multipath_hash_policy) {
1826         case 0:
1827                 memset(&hash_keys, 0, sizeof(hash_keys));
1828                 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
1829                 if (skb) {
1830                         ip_multipath_l3_keys(skb, &hash_keys);
1831                 } else {
1832                         hash_keys.addrs.v4addrs.src = fl4->saddr;
1833                         hash_keys.addrs.v4addrs.dst = fl4->daddr;
1834                 }
1835                 break;
1836         case 1:
1837                 /* skb is currently provided only when forwarding */
1838                 if (skb) {
1839                         unsigned int flag = FLOW_DISSECTOR_F_STOP_AT_ENCAP;
1840                         struct flow_keys keys;
1841
1842                         /* short-circuit if we already have L4 hash present */
1843                         if (skb->l4_hash)
1844                                 return skb_get_hash_raw(skb) >> 1;
1845
1846                         memset(&hash_keys, 0, sizeof(hash_keys));
1847
1848                         if (!flkeys) {
1849                                 skb_flow_dissect_flow_keys(skb, &keys, flag);
1850                                 flkeys = &keys;
1851                         }
1852
1853                         hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
1854                         hash_keys.addrs.v4addrs.src = flkeys->addrs.v4addrs.src;
1855                         hash_keys.addrs.v4addrs.dst = flkeys->addrs.v4addrs.dst;
1856                         hash_keys.ports.src = flkeys->ports.src;
1857                         hash_keys.ports.dst = flkeys->ports.dst;
1858                         hash_keys.basic.ip_proto = flkeys->basic.ip_proto;
1859                 } else {
1860                         memset(&hash_keys, 0, sizeof(hash_keys));
1861                         hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
1862                         hash_keys.addrs.v4addrs.src = fl4->saddr;
1863                         hash_keys.addrs.v4addrs.dst = fl4->daddr;
1864                         hash_keys.ports.src = fl4->fl4_sport;
1865                         hash_keys.ports.dst = fl4->fl4_dport;
1866                         hash_keys.basic.ip_proto = fl4->flowi4_proto;
1867                 }
1868                 break;
1869         }
1870         mhash = flow_hash_from_keys(&hash_keys);
1871
1872         return mhash >> 1;
1873 }
1874 #endif /* CONFIG_IP_ROUTE_MULTIPATH */
1875
1876 static int ip_mkroute_input(struct sk_buff *skb,
1877                             struct fib_result *res,
1878                             struct in_device *in_dev,
1879                             __be32 daddr, __be32 saddr, u32 tos,
1880                             struct flow_keys *hkeys)
1881 {
1882 #ifdef CONFIG_IP_ROUTE_MULTIPATH
1883         if (res->fi && res->fi->fib_nhs > 1) {
1884                 int h = fib_multipath_hash(res->fi->fib_net, NULL, skb, hkeys);
1885
1886                 fib_select_multipath(res, h);
1887         }
1888 #endif
1889
1890         /* create a routing cache entry */
1891         return __mkroute_input(skb, res, in_dev, daddr, saddr, tos);
1892 }
1893
1894 /*
1895  *      NOTE. We drop all the packets that has local source
1896  *      addresses, because every properly looped back packet
1897  *      must have correct destination already attached by output routine.
1898  *
1899  *      Such approach solves two big problems:
1900  *      1. Not simplex devices are handled properly.
1901  *      2. IP spoofing attempts are filtered with 100% of guarantee.
1902  *      called with rcu_read_lock()
1903  */
1904
1905 static int ip_route_input_slow(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1906                                u8 tos, struct net_device *dev,
1907                                struct fib_result *res)
1908 {
1909         struct in_device *in_dev = __in_dev_get_rcu(dev);
1910         struct flow_keys *flkeys = NULL, _flkeys;
1911         struct net    *net = dev_net(dev);
1912         struct ip_tunnel_info *tun_info;
1913         int             err = -EINVAL;
1914         unsigned int    flags = 0;
1915         u32             itag = 0;
1916         struct rtable   *rth;
1917         struct flowi4   fl4;
1918         bool do_cache;
1919
1920         /* IP on this device is disabled. */
1921
1922         if (!in_dev)
1923                 goto out;
1924
1925         /* Check for the most weird martians, which can be not detected
1926            by fib_lookup.
1927          */
1928
1929         tun_info = skb_tunnel_info(skb);
1930         if (tun_info && !(tun_info->mode & IP_TUNNEL_INFO_TX))
1931                 fl4.flowi4_tun_key.tun_id = tun_info->key.tun_id;
1932         else
1933                 fl4.flowi4_tun_key.tun_id = 0;
1934         skb_dst_drop(skb);
1935
1936         if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr))
1937                 goto martian_source;
1938
1939         res->fi = NULL;
1940         res->table = NULL;
1941         if (ipv4_is_lbcast(daddr) || (saddr == 0 && daddr == 0))
1942                 goto brd_input;
1943
1944         /* Accept zero addresses only to limited broadcast;
1945          * I even do not know to fix it or not. Waiting for complains :-)
1946          */
1947         if (ipv4_is_zeronet(saddr))
1948                 goto martian_source;
1949
1950         if (ipv4_is_zeronet(daddr))
1951                 goto martian_destination;
1952
1953         /* Following code try to avoid calling IN_DEV_NET_ROUTE_LOCALNET(),
1954          * and call it once if daddr or/and saddr are loopback addresses
1955          */
1956         if (ipv4_is_loopback(daddr)) {
1957                 if (!IN_DEV_NET_ROUTE_LOCALNET(in_dev, net))
1958                         goto martian_destination;
1959         } else if (ipv4_is_loopback(saddr)) {
1960                 if (!IN_DEV_NET_ROUTE_LOCALNET(in_dev, net))
1961                         goto martian_source;
1962         }
1963
1964         /*
1965          *      Now we are ready to route packet.
1966          */
1967         fl4.flowi4_oif = 0;
1968         fl4.flowi4_iif = dev->ifindex;
1969         fl4.flowi4_mark = skb->mark;
1970         fl4.flowi4_tos = tos;
1971         fl4.flowi4_scope = RT_SCOPE_UNIVERSE;
1972         fl4.flowi4_flags = 0;
1973         fl4.daddr = daddr;
1974         fl4.saddr = saddr;
1975         fl4.flowi4_uid = sock_net_uid(net, NULL);
1976
1977         if (fib4_rules_early_flow_dissect(net, skb, &fl4, &_flkeys)) {
1978                 flkeys = &_flkeys;
1979         } else {
1980                 fl4.flowi4_proto = 0;
1981                 fl4.fl4_sport = 0;
1982                 fl4.fl4_dport = 0;
1983         }
1984
1985         err = fib_lookup(net, &fl4, res, 0);
1986         if (err != 0) {
1987                 if (!IN_DEV_FORWARD(in_dev))
1988                         err = -EHOSTUNREACH;
1989                 goto no_route;
1990         }
1991
1992         if (res->type == RTN_BROADCAST) {
1993                 if (IN_DEV_BFORWARD(in_dev))
1994                         goto make_route;
1995                 goto brd_input;
1996         }
1997
1998         if (res->type == RTN_LOCAL) {
1999                 err = fib_validate_source(skb, saddr, daddr, tos,
2000                                           0, dev, in_dev, &itag);
2001                 if (err < 0)
2002                         goto martian_source;
2003                 goto local_input;
2004         }
2005
2006         if (!IN_DEV_FORWARD(in_dev)) {
2007                 err = -EHOSTUNREACH;
2008                 goto no_route;
2009         }
2010         if (res->type != RTN_UNICAST)
2011                 goto martian_destination;
2012
2013 make_route:
2014         err = ip_mkroute_input(skb, res, in_dev, daddr, saddr, tos, flkeys);
2015 out:    return err;
2016
2017 brd_input:
2018         if (skb->protocol != htons(ETH_P_IP))
2019                 goto e_inval;
2020
2021         if (!ipv4_is_zeronet(saddr)) {
2022                 err = fib_validate_source(skb, saddr, 0, tos, 0, dev,
2023                                           in_dev, &itag);
2024                 if (err < 0)
2025                         goto martian_source;
2026         }
2027         flags |= RTCF_BROADCAST;
2028         res->type = RTN_BROADCAST;
2029         RT_CACHE_STAT_INC(in_brd);
2030
2031 local_input:
2032         do_cache = false;
2033         if (res->fi) {
2034                 if (!itag) {
2035                         rth = rcu_dereference(FIB_RES_NH(*res).nh_rth_input);
2036                         if (rt_cache_valid(rth)) {
2037                                 skb_dst_set_noref(skb, &rth->dst);
2038                                 err = 0;
2039                                 goto out;
2040                         }
2041                         do_cache = true;
2042                 }
2043         }
2044
2045         rth = rt_dst_alloc(l3mdev_master_dev_rcu(dev) ? : net->loopback_dev,
2046                            flags | RTCF_LOCAL, res->type,
2047                            IN_DEV_CONF_GET(in_dev, NOPOLICY), false, do_cache);
2048         if (!rth)
2049                 goto e_nobufs;
2050
2051         rth->dst.output= ip_rt_bug;
2052 #ifdef CONFIG_IP_ROUTE_CLASSID
2053         rth->dst.tclassid = itag;
2054 #endif
2055         rth->rt_is_input = 1;
2056
2057         RT_CACHE_STAT_INC(in_slow_tot);
2058         if (res->type == RTN_UNREACHABLE) {
2059                 rth->dst.input= ip_error;
2060                 rth->dst.error= -err;
2061                 rth->rt_flags   &= ~RTCF_LOCAL;
2062         }
2063
2064         if (do_cache) {
2065                 struct fib_nh *nh = &FIB_RES_NH(*res);
2066
2067                 rth->dst.lwtstate = lwtstate_get(nh->nh_lwtstate);
2068                 if (lwtunnel_input_redirect(rth->dst.lwtstate)) {
2069                         WARN_ON(rth->dst.input == lwtunnel_input);
2070                         rth->dst.lwtstate->orig_input = rth->dst.input;
2071                         rth->dst.input = lwtunnel_input;
2072                 }
2073
2074                 if (unlikely(!rt_cache_route(nh, rth)))
2075                         rt_add_uncached_list(rth);
2076         }
2077         skb_dst_set(skb, &rth->dst);
2078         err = 0;
2079         goto out;
2080
2081 no_route:
2082         RT_CACHE_STAT_INC(in_no_route);
2083         res->type = RTN_UNREACHABLE;
2084         res->fi = NULL;
2085         res->table = NULL;
2086         goto local_input;
2087
2088         /*
2089          *      Do not cache martian addresses: they should be logged (RFC1812)
2090          */
2091 martian_destination:
2092         RT_CACHE_STAT_INC(in_martian_dst);
2093 #ifdef CONFIG_IP_ROUTE_VERBOSE
2094         if (IN_DEV_LOG_MARTIANS(in_dev))
2095                 net_warn_ratelimited("martian destination %pI4 from %pI4, dev %s\n",
2096                                      &daddr, &saddr, dev->name);
2097 #endif
2098
2099 e_inval:
2100         err = -EINVAL;
2101         goto out;
2102
2103 e_nobufs:
2104         err = -ENOBUFS;
2105         goto out;
2106
2107 martian_source:
2108         ip_handle_martian_source(dev, in_dev, skb, daddr, saddr);
2109         goto out;
2110 }
2111
2112 int ip_route_input_noref(struct sk_buff *skb, __be32 daddr, __be32 saddr,
2113                          u8 tos, struct net_device *dev)
2114 {
2115         struct fib_result res;
2116         int err;
2117
2118         tos &= IPTOS_RT_MASK;
2119         rcu_read_lock();
2120         err = ip_route_input_rcu(skb, daddr, saddr, tos, dev, &res);
2121         rcu_read_unlock();
2122
2123         return err;
2124 }
2125 EXPORT_SYMBOL(ip_route_input_noref);
2126
2127 /* called with rcu_read_lock held */
2128 int ip_route_input_rcu(struct sk_buff *skb, __be32 daddr, __be32 saddr,
2129                        u8 tos, struct net_device *dev, struct fib_result *res)
2130 {
2131         /* Multicast recognition logic is moved from route cache to here.
2132            The problem was that too many Ethernet cards have broken/missing
2133            hardware multicast filters :-( As result the host on multicasting
2134            network acquires a lot of useless route cache entries, sort of
2135            SDR messages from all the world. Now we try to get rid of them.
2136            Really, provided software IP multicast filter is organized
2137            reasonably (at least, hashed), it does not result in a slowdown
2138            comparing with route cache reject entries.
2139            Note, that multicast routers are not affected, because
2140            route cache entry is created eventually.
2141          */
2142         if (ipv4_is_multicast(daddr)) {
2143                 struct in_device *in_dev = __in_dev_get_rcu(dev);
2144                 int our = 0;
2145                 int err = -EINVAL;
2146
2147                 if (in_dev)
2148                         our = ip_check_mc_rcu(in_dev, daddr, saddr,
2149                                               ip_hdr(skb)->protocol);
2150
2151                 /* check l3 master if no match yet */
2152                 if ((!in_dev || !our) && netif_is_l3_slave(dev)) {
2153                         struct in_device *l3_in_dev;
2154
2155                         l3_in_dev = __in_dev_get_rcu(skb->dev);
2156                         if (l3_in_dev)
2157                                 our = ip_check_mc_rcu(l3_in_dev, daddr, saddr,
2158                                                       ip_hdr(skb)->protocol);
2159                 }
2160
2161                 if (our
2162 #ifdef CONFIG_IP_MROUTE
2163                         ||
2164                     (!ipv4_is_local_multicast(daddr) &&
2165                      IN_DEV_MFORWARD(in_dev))
2166 #endif
2167                    ) {
2168                         err = ip_route_input_mc(skb, daddr, saddr,
2169                                                 tos, dev, our);
2170                 }
2171                 return err;
2172         }
2173
2174         return ip_route_input_slow(skb, daddr, saddr, tos, dev, res);
2175 }
2176
2177 /* called with rcu_read_lock() */
2178 static struct rtable *__mkroute_output(const struct fib_result *res,
2179                                        const struct flowi4 *fl4, int orig_oif,
2180                                        struct net_device *dev_out,
2181                                        unsigned int flags)
2182 {
2183         struct fib_info *fi = res->fi;
2184         struct fib_nh_exception *fnhe;
2185         struct in_device *in_dev;
2186         u16 type = res->type;
2187         struct rtable *rth;
2188         bool do_cache;
2189
2190         in_dev = __in_dev_get_rcu(dev_out);
2191         if (!in_dev)
2192                 return ERR_PTR(-EINVAL);
2193
2194         if (likely(!IN_DEV_ROUTE_LOCALNET(in_dev)))
2195                 if (ipv4_is_loopback(fl4->saddr) &&
2196                     !(dev_out->flags & IFF_LOOPBACK) &&
2197                     !netif_is_l3_master(dev_out))
2198                         return ERR_PTR(-EINVAL);
2199
2200         if (ipv4_is_lbcast(fl4->daddr))
2201                 type = RTN_BROADCAST;
2202         else if (ipv4_is_multicast(fl4->daddr))
2203                 type = RTN_MULTICAST;
2204         else if (ipv4_is_zeronet(fl4->daddr))
2205                 return ERR_PTR(-EINVAL);
2206
2207         if (dev_out->flags & IFF_LOOPBACK)
2208                 flags |= RTCF_LOCAL;
2209
2210         do_cache = true;
2211         if (type == RTN_BROADCAST) {
2212                 flags |= RTCF_BROADCAST | RTCF_LOCAL;
2213                 fi = NULL;
2214         } else if (type == RTN_MULTICAST) {
2215                 flags |= RTCF_MULTICAST | RTCF_LOCAL;
2216                 if (!ip_check_mc_rcu(in_dev, fl4->daddr, fl4->saddr,
2217                                      fl4->flowi4_proto))
2218                         flags &= ~RTCF_LOCAL;
2219                 else
2220                         do_cache = false;
2221                 /* If multicast route do not exist use
2222                  * default one, but do not gateway in this case.
2223                  * Yes, it is hack.
2224                  */
2225                 if (fi && res->prefixlen < 4)
2226                         fi = NULL;
2227         } else if ((type == RTN_LOCAL) && (orig_oif != 0) &&
2228                    (orig_oif != dev_out->ifindex)) {
2229                 /* For local routes that require a particular output interface
2230                  * we do not want to cache the result.  Caching the result
2231                  * causes incorrect behaviour when there are multiple source
2232                  * addresses on the interface, the end result being that if the
2233                  * intended recipient is waiting on that interface for the
2234                  * packet he won't receive it because it will be delivered on
2235                  * the loopback interface and the IP_PKTINFO ipi_ifindex will
2236                  * be set to the loopback interface as well.
2237                  */
2238                 do_cache = false;
2239         }
2240
2241         fnhe = NULL;
2242         do_cache &= fi != NULL;
2243         if (fi) {
2244                 struct rtable __rcu **prth;
2245                 struct fib_nh *nh = &FIB_RES_NH(*res);
2246
2247                 fnhe = find_exception(nh, fl4->daddr);
2248                 if (!do_cache)
2249                         goto add;
2250                 if (fnhe) {
2251                         prth = &fnhe->fnhe_rth_output;
2252                 } else {
2253                         if (unlikely(fl4->flowi4_flags &
2254                                      FLOWI_FLAG_KNOWN_NH &&
2255                                      !(nh->nh_gw &&
2256                                        nh->nh_scope == RT_SCOPE_LINK))) {
2257                                 do_cache = false;
2258                                 goto add;
2259                         }
2260                         prth = raw_cpu_ptr(nh->nh_pcpu_rth_output);
2261                 }
2262                 rth = rcu_dereference(*prth);
2263                 if (rt_cache_valid(rth) && dst_hold_safe(&rth->dst))
2264                         return rth;
2265         }
2266
2267 add:
2268         rth = rt_dst_alloc(dev_out, flags, type,
2269                            IN_DEV_CONF_GET(in_dev, NOPOLICY),
2270                            IN_DEV_CONF_GET(in_dev, NOXFRM),
2271                            do_cache);
2272         if (!rth)
2273                 return ERR_PTR(-ENOBUFS);
2274
2275         rth->rt_iif = orig_oif;
2276
2277         RT_CACHE_STAT_INC(out_slow_tot);
2278
2279         if (flags & (RTCF_BROADCAST | RTCF_MULTICAST)) {
2280                 if (flags & RTCF_LOCAL &&
2281                     !(dev_out->flags & IFF_LOOPBACK)) {
2282                         rth->dst.output = ip_mc_output;
2283                         RT_CACHE_STAT_INC(out_slow_mc);
2284                 }
2285 #ifdef CONFIG_IP_MROUTE
2286                 if (type == RTN_MULTICAST) {
2287                         if (IN_DEV_MFORWARD(in_dev) &&
2288                             !ipv4_is_local_multicast(fl4->daddr)) {
2289                                 rth->dst.input = ip_mr_input;
2290                                 rth->dst.output = ip_mc_output;
2291                         }
2292                 }
2293 #endif
2294         }
2295
2296         rt_set_nexthop(rth, fl4->daddr, res, fnhe, fi, type, 0, do_cache);
2297         lwtunnel_set_redirect(&rth->dst);
2298
2299         return rth;
2300 }
2301
2302 /*
2303  * Major route resolver routine.
2304  */
2305
2306 struct rtable *ip_route_output_key_hash(struct net *net, struct flowi4 *fl4,
2307                                         const struct sk_buff *skb)
2308 {
2309         __u8 tos = RT_FL_TOS(fl4);
2310         struct fib_result res = {
2311                 .type           = RTN_UNSPEC,
2312                 .fi             = NULL,
2313                 .table          = NULL,
2314                 .tclassid       = 0,
2315         };
2316         struct rtable *rth;
2317
2318         fl4->flowi4_iif = LOOPBACK_IFINDEX;
2319         fl4->flowi4_tos = tos & IPTOS_RT_MASK;
2320         fl4->flowi4_scope = ((tos & RTO_ONLINK) ?
2321                          RT_SCOPE_LINK : RT_SCOPE_UNIVERSE);
2322
2323         rcu_read_lock();
2324         rth = ip_route_output_key_hash_rcu(net, fl4, &res, skb);
2325         rcu_read_unlock();
2326
2327         return rth;
2328 }
2329 EXPORT_SYMBOL_GPL(ip_route_output_key_hash);
2330
2331 struct rtable *ip_route_output_key_hash_rcu(struct net *net, struct flowi4 *fl4,
2332                                             struct fib_result *res,
2333                                             const struct sk_buff *skb)
2334 {
2335         struct net_device *dev_out = NULL;
2336         int orig_oif = fl4->flowi4_oif;
2337         unsigned int flags = 0;
2338         struct rtable *rth;
2339         int err = -ENETUNREACH;
2340
2341         if (fl4->saddr) {
2342                 rth = ERR_PTR(-EINVAL);
2343                 if (ipv4_is_multicast(fl4->saddr) ||
2344                     ipv4_is_lbcast(fl4->saddr) ||
2345                     ipv4_is_zeronet(fl4->saddr))
2346                         goto out;
2347
2348                 /* I removed check for oif == dev_out->oif here.
2349                    It was wrong for two reasons:
2350                    1. ip_dev_find(net, saddr) can return wrong iface, if saddr
2351                       is assigned to multiple interfaces.
2352                    2. Moreover, we are allowed to send packets with saddr
2353                       of another iface. --ANK
2354                  */
2355
2356                 if (fl4->flowi4_oif == 0 &&
2357                     (ipv4_is_multicast(fl4->daddr) ||
2358                      ipv4_is_lbcast(fl4->daddr))) {
2359                         /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
2360                         dev_out = __ip_dev_find(net, fl4->saddr, false);
2361                         if (!dev_out)
2362                                 goto out;
2363
2364                         /* Special hack: user can direct multicasts
2365                            and limited broadcast via necessary interface
2366                            without fiddling with IP_MULTICAST_IF or IP_PKTINFO.
2367                            This hack is not just for fun, it allows
2368                            vic,vat and friends to work.
2369                            They bind socket to loopback, set ttl to zero
2370                            and expect that it will work.
2371                            From the viewpoint of routing cache they are broken,
2372                            because we are not allowed to build multicast path
2373                            with loopback source addr (look, routing cache
2374                            cannot know, that ttl is zero, so that packet
2375                            will not leave this host and route is valid).
2376                            Luckily, this hack is good workaround.
2377                          */
2378
2379                         fl4->flowi4_oif = dev_out->ifindex;
2380                         goto make_route;
2381                 }
2382
2383                 if (!(fl4->flowi4_flags & FLOWI_FLAG_ANYSRC)) {
2384                         /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
2385                         if (!__ip_dev_find(net, fl4->saddr, false))
2386                                 goto out;
2387                 }
2388         }
2389
2390
2391         if (fl4->flowi4_oif) {
2392                 dev_out = dev_get_by_index_rcu(net, fl4->flowi4_oif);
2393                 rth = ERR_PTR(-ENODEV);
2394                 if (!dev_out)
2395                         goto out;
2396
2397                 /* RACE: Check return value of inet_select_addr instead. */
2398                 if (!(dev_out->flags & IFF_UP) || !__in_dev_get_rcu(dev_out)) {
2399                         rth = ERR_PTR(-ENETUNREACH);
2400                         goto out;
2401                 }
2402                 if (ipv4_is_local_multicast(fl4->daddr) ||
2403                     ipv4_is_lbcast(fl4->daddr) ||
2404                     fl4->flowi4_proto == IPPROTO_IGMP) {
2405                         if (!fl4->saddr)
2406                                 fl4->saddr = inet_select_addr(dev_out, 0,
2407                                                               RT_SCOPE_LINK);
2408                         goto make_route;
2409                 }
2410                 if (!fl4->saddr) {
2411                         if (ipv4_is_multicast(fl4->daddr))
2412                                 fl4->saddr = inet_select_addr(dev_out, 0,
2413                                                               fl4->flowi4_scope);
2414                         else if (!fl4->daddr)
2415                                 fl4->saddr = inet_select_addr(dev_out, 0,
2416                                                               RT_SCOPE_HOST);
2417                 }
2418         }
2419
2420         if (!fl4->daddr) {
2421                 fl4->daddr = fl4->saddr;
2422                 if (!fl4->daddr)
2423                         fl4->daddr = fl4->saddr = htonl(INADDR_LOOPBACK);
2424                 dev_out = net->loopback_dev;
2425                 fl4->flowi4_oif = LOOPBACK_IFINDEX;
2426                 res->type = RTN_LOCAL;
2427                 flags |= RTCF_LOCAL;
2428                 goto make_route;
2429         }
2430
2431         err = fib_lookup(net, fl4, res, 0);
2432         if (err) {
2433                 res->fi = NULL;
2434                 res->table = NULL;
2435                 if (fl4->flowi4_oif &&
2436                     (ipv4_is_multicast(fl4->daddr) ||
2437                     !netif_index_is_l3_master(net, fl4->flowi4_oif))) {
2438                         /* Apparently, routing tables are wrong. Assume,
2439                            that the destination is on link.
2440
2441                            WHY? DW.
2442                            Because we are allowed to send to iface
2443                            even if it has NO routes and NO assigned
2444                            addresses. When oif is specified, routing
2445                            tables are looked up with only one purpose:
2446                            to catch if destination is gatewayed, rather than
2447                            direct. Moreover, if MSG_DONTROUTE is set,
2448                            we send packet, ignoring both routing tables
2449                            and ifaddr state. --ANK
2450
2451
2452                            We could make it even if oif is unknown,
2453                            likely IPv6, but we do not.
2454                          */
2455
2456                         if (fl4->saddr == 0)
2457                                 fl4->saddr = inet_select_addr(dev_out, 0,
2458                                                               RT_SCOPE_LINK);
2459                         res->type = RTN_UNICAST;
2460                         goto make_route;
2461                 }
2462                 rth = ERR_PTR(err);
2463                 goto out;
2464         }
2465
2466         if (res->type == RTN_LOCAL) {
2467                 if (!fl4->saddr) {
2468                         if (res->fi->fib_prefsrc)
2469                                 fl4->saddr = res->fi->fib_prefsrc;
2470                         else
2471                                 fl4->saddr = fl4->daddr;
2472                 }
2473
2474                 /* L3 master device is the loopback for that domain */
2475                 dev_out = l3mdev_master_dev_rcu(FIB_RES_DEV(*res)) ? :
2476                         net->loopback_dev;
2477
2478                 /* make sure orig_oif points to fib result device even
2479                  * though packet rx/tx happens over loopback or l3mdev
2480                  */
2481                 orig_oif = FIB_RES_OIF(*res);
2482
2483                 fl4->flowi4_oif = dev_out->ifindex;
2484                 flags |= RTCF_LOCAL;
2485                 goto make_route;
2486         }
2487
2488         fib_select_path(net, res, fl4, skb);
2489
2490         dev_out = FIB_RES_DEV(*res);
2491         fl4->flowi4_oif = dev_out->ifindex;
2492
2493
2494 make_route:
2495         rth = __mkroute_output(res, fl4, orig_oif, dev_out, flags);
2496
2497 out:
2498         return rth;
2499 }
2500
2501 static struct dst_entry *ipv4_blackhole_dst_check(struct dst_entry *dst, u32 cookie)
2502 {
2503         return NULL;
2504 }
2505
2506 static unsigned int ipv4_blackhole_mtu(const struct dst_entry *dst)
2507 {
2508         unsigned int mtu = dst_metric_raw(dst, RTAX_MTU);
2509
2510         return mtu ? : dst->dev->mtu;
2511 }
2512
2513 static void ipv4_rt_blackhole_update_pmtu(struct dst_entry *dst, struct sock *sk,
2514                                           struct sk_buff *skb, u32 mtu)
2515 {
2516 }
2517
2518 static void ipv4_rt_blackhole_redirect(struct dst_entry *dst, struct sock *sk,
2519                                        struct sk_buff *skb)
2520 {
2521 }
2522
2523 static u32 *ipv4_rt_blackhole_cow_metrics(struct dst_entry *dst,
2524                                           unsigned long old)
2525 {
2526         return NULL;
2527 }
2528
2529 static struct dst_ops ipv4_dst_blackhole_ops = {
2530         .family                 =       AF_INET,
2531         .check                  =       ipv4_blackhole_dst_check,
2532         .mtu                    =       ipv4_blackhole_mtu,
2533         .default_advmss         =       ipv4_default_advmss,
2534         .update_pmtu            =       ipv4_rt_blackhole_update_pmtu,
2535         .redirect               =       ipv4_rt_blackhole_redirect,
2536         .cow_metrics            =       ipv4_rt_blackhole_cow_metrics,
2537         .neigh_lookup           =       ipv4_neigh_lookup,
2538 };
2539
2540 struct dst_entry *ipv4_blackhole_route(struct net *net, struct dst_entry *dst_orig)
2541 {
2542         struct rtable *ort = (struct rtable *) dst_orig;
2543         struct rtable *rt;
2544
2545         rt = dst_alloc(&ipv4_dst_blackhole_ops, NULL, 1, DST_OBSOLETE_DEAD, 0);
2546         if (rt) {
2547                 struct dst_entry *new = &rt->dst;
2548
2549                 new->__use = 1;
2550                 new->input = dst_discard;
2551                 new->output = dst_discard_out;
2552
2553                 new->dev = net->loopback_dev;
2554                 if (new->dev)
2555                         dev_hold(new->dev);
2556
2557                 rt->rt_is_input = ort->rt_is_input;
2558                 rt->rt_iif = ort->rt_iif;
2559                 rt->rt_pmtu = ort->rt_pmtu;
2560                 rt->rt_mtu_locked = ort->rt_mtu_locked;
2561
2562                 rt->rt_genid = rt_genid_ipv4(net);
2563                 rt->rt_flags = ort->rt_flags;
2564                 rt->rt_type = ort->rt_type;
2565                 rt->rt_gateway = ort->rt_gateway;
2566                 rt->rt_uses_gateway = ort->rt_uses_gateway;
2567
2568                 INIT_LIST_HEAD(&rt->rt_uncached);
2569         }
2570
2571         dst_release(dst_orig);
2572
2573         return rt ? &rt->dst : ERR_PTR(-ENOMEM);
2574 }
2575
2576 struct rtable *ip_route_output_flow(struct net *net, struct flowi4 *flp4,
2577                                     const struct sock *sk)
2578 {
2579         struct rtable *rt = __ip_route_output_key(net, flp4);
2580
2581         if (IS_ERR(rt))
2582                 return rt;
2583
2584         if (flp4->flowi4_proto)
2585                 rt = (struct rtable *)xfrm_lookup_route(net, &rt->dst,
2586                                                         flowi4_to_flowi(flp4),
2587                                                         sk, 0);
2588
2589         return rt;
2590 }
2591 EXPORT_SYMBOL_GPL(ip_route_output_flow);
2592
2593 /* called with rcu_read_lock held */
2594 static int rt_fill_info(struct net *net, __be32 dst, __be32 src,
2595                         struct rtable *rt, u32 table_id, struct flowi4 *fl4,
2596                         struct sk_buff *skb, u32 portid, u32 seq)
2597 {
2598         struct rtmsg *r;
2599         struct nlmsghdr *nlh;
2600         unsigned long expires = 0;
2601         u32 error;
2602         u32 metrics[RTAX_MAX];
2603
2604         nlh = nlmsg_put(skb, portid, seq, RTM_NEWROUTE, sizeof(*r), 0);
2605         if (!nlh)
2606                 return -EMSGSIZE;
2607
2608         r = nlmsg_data(nlh);
2609         r->rtm_family    = AF_INET;
2610         r->rtm_dst_len  = 32;
2611         r->rtm_src_len  = 0;
2612         r->rtm_tos      = fl4->flowi4_tos;
2613         r->rtm_table    = table_id < 256 ? table_id : RT_TABLE_COMPAT;
2614         if (nla_put_u32(skb, RTA_TABLE, table_id))
2615                 goto nla_put_failure;
2616         r->rtm_type     = rt->rt_type;
2617         r->rtm_scope    = RT_SCOPE_UNIVERSE;
2618         r->rtm_protocol = RTPROT_UNSPEC;
2619         r->rtm_flags    = (rt->rt_flags & ~0xFFFF) | RTM_F_CLONED;
2620         if (rt->rt_flags & RTCF_NOTIFY)
2621                 r->rtm_flags |= RTM_F_NOTIFY;
2622         if (IPCB(skb)->flags & IPSKB_DOREDIRECT)
2623                 r->rtm_flags |= RTCF_DOREDIRECT;
2624
2625         if (nla_put_in_addr(skb, RTA_DST, dst))
2626                 goto nla_put_failure;
2627         if (src) {
2628                 r->rtm_src_len = 32;
2629                 if (nla_put_in_addr(skb, RTA_SRC, src))
2630                         goto nla_put_failure;
2631         }
2632         if (rt->dst.dev &&
2633             nla_put_u32(skb, RTA_OIF, rt->dst.dev->ifindex))
2634                 goto nla_put_failure;
2635 #ifdef CONFIG_IP_ROUTE_CLASSID
2636         if (rt->dst.tclassid &&
2637             nla_put_u32(skb, RTA_FLOW, rt->dst.tclassid))
2638                 goto nla_put_failure;
2639 #endif
2640         if (!rt_is_input_route(rt) &&
2641             fl4->saddr != src) {
2642                 if (nla_put_in_addr(skb, RTA_PREFSRC, fl4->saddr))
2643                         goto nla_put_failure;
2644         }
2645         if (rt->rt_uses_gateway &&
2646             nla_put_in_addr(skb, RTA_GATEWAY, rt->rt_gateway))
2647                 goto nla_put_failure;
2648
2649         expires = rt->dst.expires;
2650         if (expires) {
2651                 unsigned long now = jiffies;
2652
2653                 if (time_before(now, expires))
2654                         expires -= now;
2655                 else
2656                         expires = 0;
2657         }
2658
2659         memcpy(metrics, dst_metrics_ptr(&rt->dst), sizeof(metrics));
2660         if (rt->rt_pmtu && expires)
2661                 metrics[RTAX_MTU - 1] = rt->rt_pmtu;
2662         if (rt->rt_mtu_locked && expires)
2663                 metrics[RTAX_LOCK - 1] |= BIT(RTAX_MTU);
2664         if (rtnetlink_put_metrics(skb, metrics) < 0)
2665                 goto nla_put_failure;
2666
2667         if (fl4->flowi4_mark &&
2668             nla_put_u32(skb, RTA_MARK, fl4->flowi4_mark))
2669                 goto nla_put_failure;
2670
2671         if (!uid_eq(fl4->flowi4_uid, INVALID_UID) &&
2672             nla_put_u32(skb, RTA_UID,
2673                         from_kuid_munged(current_user_ns(), fl4->flowi4_uid)))
2674                 goto nla_put_failure;
2675
2676         error = rt->dst.error;
2677
2678         if (rt_is_input_route(rt)) {
2679 #ifdef CONFIG_IP_MROUTE
2680                 if (ipv4_is_multicast(dst) && !ipv4_is_local_multicast(dst) &&
2681                     IPV4_DEVCONF_ALL(net, MC_FORWARDING)) {
2682                         int err = ipmr_get_route(net, skb,
2683                                                  fl4->saddr, fl4->daddr,
2684                                                  r, portid);
2685
2686                         if (err <= 0) {
2687                                 if (err == 0)
2688                                         return 0;
2689                                 goto nla_put_failure;
2690                         }
2691                 } else
2692 #endif
2693                         if (nla_put_u32(skb, RTA_IIF, fl4->flowi4_iif))
2694                                 goto nla_put_failure;
2695         }
2696
2697         if (rtnl_put_cacheinfo(skb, &rt->dst, 0, expires, error) < 0)
2698                 goto nla_put_failure;
2699
2700         nlmsg_end(skb, nlh);
2701         return 0;
2702
2703 nla_put_failure:
2704         nlmsg_cancel(skb, nlh);
2705         return -EMSGSIZE;
2706 }
2707
2708 static struct sk_buff *inet_rtm_getroute_build_skb(__be32 src, __be32 dst,
2709                                                    u8 ip_proto, __be16 sport,
2710                                                    __be16 dport)
2711 {
2712         struct sk_buff *skb;
2713         struct iphdr *iph;
2714
2715         skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
2716         if (!skb)
2717                 return NULL;
2718
2719         /* Reserve room for dummy headers, this skb can pass
2720          * through good chunk of routing engine.
2721          */
2722         skb_reset_mac_header(skb);
2723         skb_reset_network_header(skb);
2724         skb->protocol = htons(ETH_P_IP);
2725         iph = skb_put(skb, sizeof(struct iphdr));
2726         iph->protocol = ip_proto;
2727         iph->saddr = src;
2728         iph->daddr = dst;
2729         iph->version = 0x4;
2730         iph->frag_off = 0;
2731         iph->ihl = 0x5;
2732         skb_set_transport_header(skb, skb->len);
2733
2734         switch (iph->protocol) {
2735         case IPPROTO_UDP: {
2736                 struct udphdr *udph;
2737
2738                 udph = skb_put_zero(skb, sizeof(struct udphdr));
2739                 udph->source = sport;
2740                 udph->dest = dport;
2741                 udph->len = sizeof(struct udphdr);
2742                 udph->check = 0;
2743                 break;
2744         }
2745         case IPPROTO_TCP: {
2746                 struct tcphdr *tcph;
2747
2748                 tcph = skb_put_zero(skb, sizeof(struct tcphdr));
2749                 tcph->source    = sport;
2750                 tcph->dest      = dport;
2751                 tcph->doff      = sizeof(struct tcphdr) / 4;
2752                 tcph->rst = 1;
2753                 tcph->check = ~tcp_v4_check(sizeof(struct tcphdr),
2754                                             src, dst, 0);
2755                 break;
2756         }
2757         case IPPROTO_ICMP: {
2758                 struct icmphdr *icmph;
2759
2760                 icmph = skb_put_zero(skb, sizeof(struct icmphdr));
2761                 icmph->type = ICMP_ECHO;
2762                 icmph->code = 0;
2763         }
2764         }
2765
2766         return skb;
2767 }
2768
2769 static int inet_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr *nlh,
2770                              struct netlink_ext_ack *extack)
2771 {
2772         struct net *net = sock_net(in_skb->sk);
2773         struct nlattr *tb[RTA_MAX+1];
2774         u32 table_id = RT_TABLE_MAIN;
2775         __be16 sport = 0, dport = 0;
2776         struct fib_result res = {};
2777         u8 ip_proto = IPPROTO_UDP;
2778         struct rtable *rt = NULL;
2779         struct sk_buff *skb;
2780         struct rtmsg *rtm;
2781         struct flowi4 fl4 = {};
2782         __be32 dst = 0;
2783         __be32 src = 0;
2784         kuid_t uid;
2785         u32 iif;
2786         int err;
2787         int mark;
2788
2789         err = nlmsg_parse(nlh, sizeof(*rtm), tb, RTA_MAX, rtm_ipv4_policy,
2790                           extack);
2791         if (err < 0)
2792                 return err;
2793
2794         rtm = nlmsg_data(nlh);
2795         src = tb[RTA_SRC] ? nla_get_in_addr(tb[RTA_SRC]) : 0;
2796         dst = tb[RTA_DST] ? nla_get_in_addr(tb[RTA_DST]) : 0;
2797         iif = tb[RTA_IIF] ? nla_get_u32(tb[RTA_IIF]) : 0;
2798         mark = tb[RTA_MARK] ? nla_get_u32(tb[RTA_MARK]) : 0;
2799         if (tb[RTA_UID])
2800                 uid = make_kuid(current_user_ns(), nla_get_u32(tb[RTA_UID]));
2801         else
2802                 uid = (iif ? INVALID_UID : current_uid());
2803
2804         if (tb[RTA_IP_PROTO]) {
2805                 err = rtm_getroute_parse_ip_proto(tb[RTA_IP_PROTO],
2806                                                   &ip_proto, extack);
2807                 if (err)
2808                         return err;
2809         }
2810
2811         if (tb[RTA_SPORT])
2812                 sport = nla_get_be16(tb[RTA_SPORT]);
2813
2814         if (tb[RTA_DPORT])
2815                 dport = nla_get_be16(tb[RTA_DPORT]);
2816
2817         skb = inet_rtm_getroute_build_skb(src, dst, ip_proto, sport, dport);
2818         if (!skb)
2819                 return -ENOBUFS;
2820
2821         fl4.daddr = dst;
2822         fl4.saddr = src;
2823         fl4.flowi4_tos = rtm->rtm_tos;
2824         fl4.flowi4_oif = tb[RTA_OIF] ? nla_get_u32(tb[RTA_OIF]) : 0;
2825         fl4.flowi4_mark = mark;
2826         fl4.flowi4_uid = uid;
2827         if (sport)
2828                 fl4.fl4_sport = sport;
2829         if (dport)
2830                 fl4.fl4_dport = dport;
2831         fl4.flowi4_proto = ip_proto;
2832
2833         rcu_read_lock();
2834
2835         if (iif) {
2836                 struct net_device *dev;
2837
2838                 dev = dev_get_by_index_rcu(net, iif);
2839                 if (!dev) {
2840                         err = -ENODEV;
2841                         goto errout_rcu;
2842                 }
2843
2844                 fl4.flowi4_iif = iif; /* for rt_fill_info */
2845                 skb->dev        = dev;
2846                 skb->mark       = mark;
2847                 err = ip_route_input_rcu(skb, dst, src, rtm->rtm_tos,
2848                                          dev, &res);
2849
2850                 rt = skb_rtable(skb);
2851                 if (err == 0 && rt->dst.error)
2852                         err = -rt->dst.error;
2853         } else {
2854                 fl4.flowi4_iif = LOOPBACK_IFINDEX;
2855                 skb->dev = net->loopback_dev;
2856                 rt = ip_route_output_key_hash_rcu(net, &fl4, &res, skb);
2857                 err = 0;
2858                 if (IS_ERR(rt))
2859                         err = PTR_ERR(rt);
2860                 else
2861                         skb_dst_set(skb, &rt->dst);
2862         }
2863
2864         if (err)
2865                 goto errout_rcu;
2866
2867         if (rtm->rtm_flags & RTM_F_NOTIFY)
2868                 rt->rt_flags |= RTCF_NOTIFY;
2869
2870         if (rtm->rtm_flags & RTM_F_LOOKUP_TABLE)
2871                 table_id = res.table ? res.table->tb_id : 0;
2872
2873         /* reset skb for netlink reply msg */
2874         skb_trim(skb, 0);
2875         skb_reset_network_header(skb);
2876         skb_reset_transport_header(skb);
2877         skb_reset_mac_header(skb);
2878
2879         if (rtm->rtm_flags & RTM_F_FIB_MATCH) {
2880                 if (!res.fi) {
2881                         err = fib_props[res.type].error;
2882                         if (!err)
2883                                 err = -EHOSTUNREACH;
2884                         goto errout_rcu;
2885                 }
2886                 err = fib_dump_info(skb, NETLINK_CB(in_skb).portid,
2887                                     nlh->nlmsg_seq, RTM_NEWROUTE, table_id,
2888                                     rt->rt_type, res.prefix, res.prefixlen,
2889                                     fl4.flowi4_tos, res.fi, 0);
2890         } else {
2891                 err = rt_fill_info(net, dst, src, rt, table_id, &fl4, skb,
2892                                    NETLINK_CB(in_skb).portid, nlh->nlmsg_seq);
2893         }
2894         if (err < 0)
2895                 goto errout_rcu;
2896
2897         rcu_read_unlock();
2898
2899         err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).portid);
2900
2901 errout_free:
2902         return err;
2903 errout_rcu:
2904         rcu_read_unlock();
2905         kfree_skb(skb);
2906         goto errout_free;
2907 }
2908
2909 void ip_rt_multicast_event(struct in_device *in_dev)
2910 {
2911         rt_cache_flush(dev_net(in_dev->dev));
2912 }
2913
2914 #ifdef CONFIG_SYSCTL
2915 static int ip_rt_gc_interval __read_mostly  = 60 * HZ;
2916 static int ip_rt_gc_min_interval __read_mostly  = HZ / 2;
2917 static int ip_rt_gc_elasticity __read_mostly    = 8;
2918 static int ip_min_valid_pmtu __read_mostly      = IPV4_MIN_MTU;
2919
2920 static int ipv4_sysctl_rtcache_flush(struct ctl_table *__ctl, int write,
2921                                         void __user *buffer,
2922                                         size_t *lenp, loff_t *ppos)
2923 {
2924         struct net *net = (struct net *)__ctl->extra1;
2925
2926         if (write) {
2927                 rt_cache_flush(net);
2928                 fnhe_genid_bump(net);
2929                 return 0;
2930         }
2931
2932         return -EINVAL;
2933 }
2934
2935 static struct ctl_table ipv4_route_table[] = {
2936         {
2937                 .procname       = "gc_thresh",
2938                 .data           = &ipv4_dst_ops.gc_thresh,
2939                 .maxlen         = sizeof(int),
2940                 .mode           = 0644,
2941                 .proc_handler   = proc_dointvec,
2942         },
2943         {
2944                 .procname       = "max_size",
2945                 .data           = &ip_rt_max_size,
2946                 .maxlen         = sizeof(int),
2947                 .mode           = 0644,
2948                 .proc_handler   = proc_dointvec,
2949         },
2950         {
2951                 /*  Deprecated. Use gc_min_interval_ms */
2952
2953                 .procname       = "gc_min_interval",
2954                 .data           = &ip_rt_gc_min_interval,
2955                 .maxlen         = sizeof(int),
2956                 .mode           = 0644,
2957                 .proc_handler   = proc_dointvec_jiffies,
2958         },
2959         {
2960                 .procname       = "gc_min_interval_ms",
2961                 .data           = &ip_rt_gc_min_interval,
2962                 .maxlen         = sizeof(int),
2963                 .mode           = 0644,
2964                 .proc_handler   = proc_dointvec_ms_jiffies,
2965         },
2966         {
2967                 .procname       = "gc_timeout",
2968                 .data           = &ip_rt_gc_timeout,
2969                 .maxlen         = sizeof(int),
2970                 .mode           = 0644,
2971                 .proc_handler   = proc_dointvec_jiffies,
2972         },
2973         {
2974                 .procname       = "gc_interval",
2975                 .data           = &ip_rt_gc_interval,
2976                 .maxlen         = sizeof(int),
2977                 .mode           = 0644,
2978                 .proc_handler   = proc_dointvec_jiffies,
2979         },
2980         {
2981                 .procname       = "redirect_load",
2982                 .data           = &ip_rt_redirect_load,
2983                 .maxlen         = sizeof(int),
2984                 .mode           = 0644,
2985                 .proc_handler   = proc_dointvec,
2986         },
2987         {
2988                 .procname       = "redirect_number",
2989                 .data           = &ip_rt_redirect_number,
2990                 .maxlen         = sizeof(int),
2991                 .mode           = 0644,
2992                 .proc_handler   = proc_dointvec,
2993         },
2994         {
2995                 .procname       = "redirect_silence",
2996                 .data           = &ip_rt_redirect_silence,
2997                 .maxlen         = sizeof(int),
2998                 .mode           = 0644,
2999                 .proc_handler   = proc_dointvec,
3000         },
3001         {
3002                 .procname       = "error_cost",
3003                 .data           = &ip_rt_error_cost,
3004                 .maxlen         = sizeof(int),
3005                 .mode           = 0644,
3006                 .proc_handler   = proc_dointvec,
3007         },
3008         {
3009                 .procname       = "error_burst",
3010                 .data           = &ip_rt_error_burst,
3011                 .maxlen         = sizeof(int),
3012                 .mode           = 0644,
3013                 .proc_handler   = proc_dointvec,
3014         },
3015         {
3016                 .procname       = "gc_elasticity",
3017                 .data           = &ip_rt_gc_elasticity,
3018                 .maxlen         = sizeof(int),
3019                 .mode           = 0644,
3020                 .proc_handler   = proc_dointvec,
3021         },
3022         {
3023                 .procname       = "mtu_expires",
3024                 .data           = &ip_rt_mtu_expires,
3025                 .maxlen         = sizeof(int),
3026                 .mode           = 0644,
3027                 .proc_handler   = proc_dointvec_jiffies,
3028         },
3029         {
3030                 .procname       = "min_pmtu",
3031                 .data           = &ip_rt_min_pmtu,
3032                 .maxlen         = sizeof(int),
3033                 .mode           = 0644,
3034                 .proc_handler   = proc_dointvec_minmax,
3035                 .extra1         = &ip_min_valid_pmtu,
3036         },
3037         {
3038                 .procname       = "min_adv_mss",
3039                 .data           = &ip_rt_min_advmss,
3040                 .maxlen         = sizeof(int),
3041                 .mode           = 0644,
3042                 .proc_handler   = proc_dointvec,
3043         },
3044         { }
3045 };
3046
3047 static struct ctl_table ipv4_route_flush_table[] = {
3048         {
3049                 .procname       = "flush",
3050                 .maxlen         = sizeof(int),
3051                 .mode           = 0200,
3052                 .proc_handler   = ipv4_sysctl_rtcache_flush,
3053         },
3054         { },
3055 };
3056
3057 static __net_init int sysctl_route_net_init(struct net *net)
3058 {
3059         struct ctl_table *tbl;
3060
3061         tbl = ipv4_route_flush_table;
3062         if (!net_eq(net, &init_net)) {
3063                 tbl = kmemdup(tbl, sizeof(ipv4_route_flush_table), GFP_KERNEL);
3064                 if (!tbl)
3065                         goto err_dup;
3066
3067                 /* Don't export sysctls to unprivileged users */
3068                 if (net->user_ns != &init_user_ns)
3069                         tbl[0].procname = NULL;
3070         }
3071         tbl[0].extra1 = net;
3072
3073         net->ipv4.route_hdr = register_net_sysctl(net, "net/ipv4/route", tbl);
3074         if (!net->ipv4.route_hdr)
3075                 goto err_reg;
3076         return 0;
3077
3078 err_reg:
3079         if (tbl != ipv4_route_flush_table)
3080                 kfree(tbl);
3081 err_dup:
3082         return -ENOMEM;
3083 }
3084
3085 static __net_exit void sysctl_route_net_exit(struct net *net)
3086 {
3087         struct ctl_table *tbl;
3088
3089         tbl = net->ipv4.route_hdr->ctl_table_arg;
3090         unregister_net_sysctl_table(net->ipv4.route_hdr);
3091         BUG_ON(tbl == ipv4_route_flush_table);
3092         kfree(tbl);
3093 }
3094
3095 static __net_initdata struct pernet_operations sysctl_route_ops = {
3096         .init = sysctl_route_net_init,
3097         .exit = sysctl_route_net_exit,
3098 };
3099 #endif
3100
3101 static __net_init int rt_genid_init(struct net *net)
3102 {
3103         atomic_set(&net->ipv4.rt_genid, 0);
3104         atomic_set(&net->fnhe_genid, 0);
3105         atomic_set(&net->ipv4.dev_addr_genid, get_random_int());
3106         return 0;
3107 }
3108
3109 static __net_initdata struct pernet_operations rt_genid_ops = {
3110         .init = rt_genid_init,
3111 };
3112
3113 static int __net_init ipv4_inetpeer_init(struct net *net)
3114 {
3115         struct inet_peer_base *bp = kmalloc(sizeof(*bp), GFP_KERNEL);
3116
3117         if (!bp)
3118                 return -ENOMEM;
3119         inet_peer_base_init(bp);
3120         net->ipv4.peers = bp;
3121         return 0;
3122 }
3123
3124 static void __net_exit ipv4_inetpeer_exit(struct net *net)
3125 {
3126         struct inet_peer_base *bp = net->ipv4.peers;
3127
3128         net->ipv4.peers = NULL;
3129         inetpeer_invalidate_tree(bp);
3130         kfree(bp);
3131 }
3132
3133 static __net_initdata struct pernet_operations ipv4_inetpeer_ops = {
3134         .init   =       ipv4_inetpeer_init,
3135         .exit   =       ipv4_inetpeer_exit,
3136 };
3137
3138 #ifdef CONFIG_IP_ROUTE_CLASSID
3139 struct ip_rt_acct __percpu *ip_rt_acct __read_mostly;
3140 #endif /* CONFIG_IP_ROUTE_CLASSID */
3141
3142 int __init ip_rt_init(void)
3143 {
3144         int cpu;
3145
3146         ip_idents = kmalloc_array(IP_IDENTS_SZ, sizeof(*ip_idents),
3147                                   GFP_KERNEL);
3148         if (!ip_idents)
3149                 panic("IP: failed to allocate ip_idents\n");
3150
3151         prandom_bytes(ip_idents, IP_IDENTS_SZ * sizeof(*ip_idents));
3152
3153         ip_tstamps = kcalloc(IP_IDENTS_SZ, sizeof(*ip_tstamps), GFP_KERNEL);
3154         if (!ip_tstamps)
3155                 panic("IP: failed to allocate ip_tstamps\n");
3156
3157         for_each_possible_cpu(cpu) {
3158                 struct uncached_list *ul = &per_cpu(rt_uncached_list, cpu);
3159
3160                 INIT_LIST_HEAD(&ul->head);
3161                 spin_lock_init(&ul->lock);
3162         }
3163 #ifdef CONFIG_IP_ROUTE_CLASSID
3164         ip_rt_acct = __alloc_percpu(256 * sizeof(struct ip_rt_acct), __alignof__(struct ip_rt_acct));
3165         if (!ip_rt_acct)
3166                 panic("IP: failed to allocate ip_rt_acct\n");
3167 #endif
3168
3169         ipv4_dst_ops.kmem_cachep =
3170                 kmem_cache_create("ip_dst_cache", sizeof(struct rtable), 0,
3171                                   SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
3172
3173         ipv4_dst_blackhole_ops.kmem_cachep = ipv4_dst_ops.kmem_cachep;
3174
3175         if (dst_entries_init(&ipv4_dst_ops) < 0)
3176                 panic("IP: failed to allocate ipv4_dst_ops counter\n");
3177
3178         if (dst_entries_init(&ipv4_dst_blackhole_ops) < 0)
3179                 panic("IP: failed to allocate ipv4_dst_blackhole_ops counter\n");
3180
3181         ipv4_dst_ops.gc_thresh = ~0;
3182         ip_rt_max_size = INT_MAX;
3183
3184         devinet_init();
3185         ip_fib_init();
3186
3187         if (ip_rt_proc_init())
3188                 pr_err("Unable to create route proc files\n");
3189 #ifdef CONFIG_XFRM
3190         xfrm_init();
3191         xfrm4_init();
3192 #endif
3193         rtnl_register(PF_INET, RTM_GETROUTE, inet_rtm_getroute, NULL,
3194                       RTNL_FLAG_DOIT_UNLOCKED);
3195
3196 #ifdef CONFIG_SYSCTL
3197         register_pernet_subsys(&sysctl_route_ops);
3198 #endif
3199         register_pernet_subsys(&rt_genid_ops);
3200         register_pernet_subsys(&ipv4_inetpeer_ops);
3201         return 0;
3202 }
3203
3204 #ifdef CONFIG_SYSCTL
3205 /*
3206  * We really need to sanitize the damn ipv4 init order, then all
3207  * this nonsense will go away.
3208  */
3209 void __init ip_static_sysctl_init(void)
3210 {
3211         register_net_sysctl(&init_net, "net/ipv4/route", ipv4_route_table);
3212 }
3213 #endif