netfilter: fix Kconfig dependencies
[linux-2.6.git] / net / ipv4 / route.c
1 /*
2  * INET         An implementation of the TCP/IP protocol suite for the LINUX
3  *              operating system.  INET is implemented using the  BSD Socket
4  *              interface as the means of communication with the user level.
5  *
6  *              ROUTE - implementation of the IP router.
7  *
8  * Authors:     Ross Biro
9  *              Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10  *              Alan Cox, <gw4pts@gw4pts.ampr.org>
11  *              Linus Torvalds, <Linus.Torvalds@helsinki.fi>
12  *              Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
13  *
14  * Fixes:
15  *              Alan Cox        :       Verify area fixes.
16  *              Alan Cox        :       cli() protects routing changes
17  *              Rui Oliveira    :       ICMP routing table updates
18  *              (rco@di.uminho.pt)      Routing table insertion and update
19  *              Linus Torvalds  :       Rewrote bits to be sensible
20  *              Alan Cox        :       Added BSD route gw semantics
21  *              Alan Cox        :       Super /proc >4K
22  *              Alan Cox        :       MTU in route table
23  *              Alan Cox        :       MSS actually. Also added the window
24  *                                      clamper.
25  *              Sam Lantinga    :       Fixed route matching in rt_del()
26  *              Alan Cox        :       Routing cache support.
27  *              Alan Cox        :       Removed compatibility cruft.
28  *              Alan Cox        :       RTF_REJECT support.
29  *              Alan Cox        :       TCP irtt support.
30  *              Jonathan Naylor :       Added Metric support.
31  *      Miquel van Smoorenburg  :       BSD API fixes.
32  *      Miquel van Smoorenburg  :       Metrics.
33  *              Alan Cox        :       Use __u32 properly
34  *              Alan Cox        :       Aligned routing errors more closely with BSD
35  *                                      our system is still very different.
36  *              Alan Cox        :       Faster /proc handling
37  *      Alexey Kuznetsov        :       Massive rework to support tree based routing,
38  *                                      routing caches and better behaviour.
39  *
40  *              Olaf Erb        :       irtt wasn't being copied right.
41  *              Bjorn Ekwall    :       Kerneld route support.
42  *              Alan Cox        :       Multicast fixed (I hope)
43  *              Pavel Krauz     :       Limited broadcast fixed
44  *              Mike McLagan    :       Routing by source
45  *      Alexey Kuznetsov        :       End of old history. Split to fib.c and
46  *                                      route.c and rewritten from scratch.
47  *              Andi Kleen      :       Load-limit warning messages.
48  *      Vitaly E. Lavrov        :       Transparent proxy revived after year coma.
49  *      Vitaly E. Lavrov        :       Race condition in ip_route_input_slow.
50  *      Tobias Ringstrom        :       Uninitialized res.type in ip_route_output_slow.
51  *      Vladimir V. Ivanov      :       IP rule info (flowid) is really useful.
52  *              Marc Boucher    :       routing by fwmark
53  *      Robert Olsson           :       Added rt_cache statistics
54  *      Arnaldo C. Melo         :       Convert proc stuff to seq_file
55  *      Eric Dumazet            :       hashed spinlocks and rt_check_expire() fixes.
56  *      Ilia Sotnikov           :       Ignore TOS on PMTUD and Redirect
57  *      Ilia Sotnikov           :       Removed TOS from hash calculations
58  *
59  *              This program is free software; you can redistribute it and/or
60  *              modify it under the terms of the GNU General Public License
61  *              as published by the Free Software Foundation; either version
62  *              2 of the License, or (at your option) any later version.
63  */
64
65 #include <linux/module.h>
66 #include <asm/uaccess.h>
67 #include <asm/system.h>
68 #include <linux/bitops.h>
69 #include <linux/types.h>
70 #include <linux/kernel.h>
71 #include <linux/mm.h>
72 #include <linux/bootmem.h>
73 #include <linux/string.h>
74 #include <linux/socket.h>
75 #include <linux/sockios.h>
76 #include <linux/errno.h>
77 #include <linux/in.h>
78 #include <linux/inet.h>
79 #include <linux/netdevice.h>
80 #include <linux/proc_fs.h>
81 #include <linux/init.h>
82 #include <linux/workqueue.h>
83 #include <linux/skbuff.h>
84 #include <linux/inetdevice.h>
85 #include <linux/igmp.h>
86 #include <linux/pkt_sched.h>
87 #include <linux/mroute.h>
88 #include <linux/netfilter_ipv4.h>
89 #include <linux/random.h>
90 #include <linux/jhash.h>
91 #include <linux/rcupdate.h>
92 #include <linux/times.h>
93 #include <linux/slab.h>
94 #include <net/dst.h>
95 #include <net/net_namespace.h>
96 #include <net/protocol.h>
97 #include <net/ip.h>
98 #include <net/route.h>
99 #include <net/inetpeer.h>
100 #include <net/sock.h>
101 #include <net/ip_fib.h>
102 #include <net/arp.h>
103 #include <net/tcp.h>
104 #include <net/icmp.h>
105 #include <net/xfrm.h>
106 #include <net/netevent.h>
107 #include <net/rtnetlink.h>
108 #ifdef CONFIG_SYSCTL
109 #include <linux/sysctl.h>
110 #endif
111
112 #define RT_FL_TOS(oldflp) \
113     ((u32)(oldflp->fl4_tos & (IPTOS_RT_MASK | RTO_ONLINK)))
114
115 #define IP_MAX_MTU      0xFFF0
116
117 #define RT_GC_TIMEOUT (300*HZ)
118
119 static int ip_rt_max_size;
120 static int ip_rt_gc_timeout __read_mostly       = RT_GC_TIMEOUT;
121 static int ip_rt_gc_interval __read_mostly      = 60 * HZ;
122 static int ip_rt_gc_min_interval __read_mostly  = HZ / 2;
123 static int ip_rt_redirect_number __read_mostly  = 9;
124 static int ip_rt_redirect_load __read_mostly    = HZ / 50;
125 static int ip_rt_redirect_silence __read_mostly = ((HZ / 50) << (9 + 1));
126 static int ip_rt_error_cost __read_mostly       = HZ;
127 static int ip_rt_error_burst __read_mostly      = 5 * HZ;
128 static int ip_rt_gc_elasticity __read_mostly    = 8;
129 static int ip_rt_mtu_expires __read_mostly      = 10 * 60 * HZ;
130 static int ip_rt_min_pmtu __read_mostly         = 512 + 20 + 20;
131 static int ip_rt_min_advmss __read_mostly       = 256;
132 static int rt_chain_length_max __read_mostly    = 20;
133
134 static struct delayed_work expires_work;
135 static unsigned long expires_ljiffies;
136
137 /*
138  *      Interface to generic destination cache.
139  */
140
141 static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie);
142 static void              ipv4_dst_destroy(struct dst_entry *dst);
143 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst);
144 static void              ipv4_link_failure(struct sk_buff *skb);
145 static void              ip_rt_update_pmtu(struct dst_entry *dst, u32 mtu);
146 static int rt_garbage_collect(struct dst_ops *ops);
147
148 static void ipv4_dst_ifdown(struct dst_entry *dst, struct net_device *dev,
149                             int how)
150 {
151 }
152
153 static struct dst_ops ipv4_dst_ops = {
154         .family =               AF_INET,
155         .protocol =             cpu_to_be16(ETH_P_IP),
156         .gc =                   rt_garbage_collect,
157         .check =                ipv4_dst_check,
158         .destroy =              ipv4_dst_destroy,
159         .ifdown =               ipv4_dst_ifdown,
160         .negative_advice =      ipv4_negative_advice,
161         .link_failure =         ipv4_link_failure,
162         .update_pmtu =          ip_rt_update_pmtu,
163         .local_out =            __ip_local_out,
164 };
165
166 #define ECN_OR_COST(class)      TC_PRIO_##class
167
168 const __u8 ip_tos2prio[16] = {
169         TC_PRIO_BESTEFFORT,
170         ECN_OR_COST(FILLER),
171         TC_PRIO_BESTEFFORT,
172         ECN_OR_COST(BESTEFFORT),
173         TC_PRIO_BULK,
174         ECN_OR_COST(BULK),
175         TC_PRIO_BULK,
176         ECN_OR_COST(BULK),
177         TC_PRIO_INTERACTIVE,
178         ECN_OR_COST(INTERACTIVE),
179         TC_PRIO_INTERACTIVE,
180         ECN_OR_COST(INTERACTIVE),
181         TC_PRIO_INTERACTIVE_BULK,
182         ECN_OR_COST(INTERACTIVE_BULK),
183         TC_PRIO_INTERACTIVE_BULK,
184         ECN_OR_COST(INTERACTIVE_BULK)
185 };
186
187
188 /*
189  * Route cache.
190  */
191
192 /* The locking scheme is rather straight forward:
193  *
194  * 1) Read-Copy Update protects the buckets of the central route hash.
195  * 2) Only writers remove entries, and they hold the lock
196  *    as they look at rtable reference counts.
197  * 3) Only readers acquire references to rtable entries,
198  *    they do so with atomic increments and with the
199  *    lock held.
200  */
201
202 struct rt_hash_bucket {
203         struct rtable __rcu     *chain;
204 };
205
206 #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK) || \
207         defined(CONFIG_PROVE_LOCKING)
208 /*
209  * Instead of using one spinlock for each rt_hash_bucket, we use a table of spinlocks
210  * The size of this table is a power of two and depends on the number of CPUS.
211  * (on lockdep we have a quite big spinlock_t, so keep the size down there)
212  */
213 #ifdef CONFIG_LOCKDEP
214 # define RT_HASH_LOCK_SZ        256
215 #else
216 # if NR_CPUS >= 32
217 #  define RT_HASH_LOCK_SZ       4096
218 # elif NR_CPUS >= 16
219 #  define RT_HASH_LOCK_SZ       2048
220 # elif NR_CPUS >= 8
221 #  define RT_HASH_LOCK_SZ       1024
222 # elif NR_CPUS >= 4
223 #  define RT_HASH_LOCK_SZ       512
224 # else
225 #  define RT_HASH_LOCK_SZ       256
226 # endif
227 #endif
228
229 static spinlock_t       *rt_hash_locks;
230 # define rt_hash_lock_addr(slot) &rt_hash_locks[(slot) & (RT_HASH_LOCK_SZ - 1)]
231
232 static __init void rt_hash_lock_init(void)
233 {
234         int i;
235
236         rt_hash_locks = kmalloc(sizeof(spinlock_t) * RT_HASH_LOCK_SZ,
237                         GFP_KERNEL);
238         if (!rt_hash_locks)
239                 panic("IP: failed to allocate rt_hash_locks\n");
240
241         for (i = 0; i < RT_HASH_LOCK_SZ; i++)
242                 spin_lock_init(&rt_hash_locks[i]);
243 }
244 #else
245 # define rt_hash_lock_addr(slot) NULL
246
247 static inline void rt_hash_lock_init(void)
248 {
249 }
250 #endif
251
252 static struct rt_hash_bucket    *rt_hash_table __read_mostly;
253 static unsigned                 rt_hash_mask __read_mostly;
254 static unsigned int             rt_hash_log  __read_mostly;
255
256 static DEFINE_PER_CPU(struct rt_cache_stat, rt_cache_stat);
257 #define RT_CACHE_STAT_INC(field) __this_cpu_inc(rt_cache_stat.field)
258
259 static inline unsigned int rt_hash(__be32 daddr, __be32 saddr, int idx,
260                                    int genid)
261 {
262         return jhash_3words((__force u32)daddr, (__force u32)saddr,
263                             idx, genid)
264                 & rt_hash_mask;
265 }
266
267 static inline int rt_genid(struct net *net)
268 {
269         return atomic_read(&net->ipv4.rt_genid);
270 }
271
272 #ifdef CONFIG_PROC_FS
273 struct rt_cache_iter_state {
274         struct seq_net_private p;
275         int bucket;
276         int genid;
277 };
278
279 static struct rtable *rt_cache_get_first(struct seq_file *seq)
280 {
281         struct rt_cache_iter_state *st = seq->private;
282         struct rtable *r = NULL;
283
284         for (st->bucket = rt_hash_mask; st->bucket >= 0; --st->bucket) {
285                 if (!rcu_dereference_raw(rt_hash_table[st->bucket].chain))
286                         continue;
287                 rcu_read_lock_bh();
288                 r = rcu_dereference_bh(rt_hash_table[st->bucket].chain);
289                 while (r) {
290                         if (dev_net(r->dst.dev) == seq_file_net(seq) &&
291                             r->rt_genid == st->genid)
292                                 return r;
293                         r = rcu_dereference_bh(r->dst.rt_next);
294                 }
295                 rcu_read_unlock_bh();
296         }
297         return r;
298 }
299
300 static struct rtable *__rt_cache_get_next(struct seq_file *seq,
301                                           struct rtable *r)
302 {
303         struct rt_cache_iter_state *st = seq->private;
304
305         r = rcu_dereference_bh(r->dst.rt_next);
306         while (!r) {
307                 rcu_read_unlock_bh();
308                 do {
309                         if (--st->bucket < 0)
310                                 return NULL;
311                 } while (!rcu_dereference_raw(rt_hash_table[st->bucket].chain));
312                 rcu_read_lock_bh();
313                 r = rcu_dereference_bh(rt_hash_table[st->bucket].chain);
314         }
315         return r;
316 }
317
318 static struct rtable *rt_cache_get_next(struct seq_file *seq,
319                                         struct rtable *r)
320 {
321         struct rt_cache_iter_state *st = seq->private;
322         while ((r = __rt_cache_get_next(seq, r)) != NULL) {
323                 if (dev_net(r->dst.dev) != seq_file_net(seq))
324                         continue;
325                 if (r->rt_genid == st->genid)
326                         break;
327         }
328         return r;
329 }
330
331 static struct rtable *rt_cache_get_idx(struct seq_file *seq, loff_t pos)
332 {
333         struct rtable *r = rt_cache_get_first(seq);
334
335         if (r)
336                 while (pos && (r = rt_cache_get_next(seq, r)))
337                         --pos;
338         return pos ? NULL : r;
339 }
340
341 static void *rt_cache_seq_start(struct seq_file *seq, loff_t *pos)
342 {
343         struct rt_cache_iter_state *st = seq->private;
344         if (*pos)
345                 return rt_cache_get_idx(seq, *pos - 1);
346         st->genid = rt_genid(seq_file_net(seq));
347         return SEQ_START_TOKEN;
348 }
349
350 static void *rt_cache_seq_next(struct seq_file *seq, void *v, loff_t *pos)
351 {
352         struct rtable *r;
353
354         if (v == SEQ_START_TOKEN)
355                 r = rt_cache_get_first(seq);
356         else
357                 r = rt_cache_get_next(seq, v);
358         ++*pos;
359         return r;
360 }
361
362 static void rt_cache_seq_stop(struct seq_file *seq, void *v)
363 {
364         if (v && v != SEQ_START_TOKEN)
365                 rcu_read_unlock_bh();
366 }
367
368 static int rt_cache_seq_show(struct seq_file *seq, void *v)
369 {
370         if (v == SEQ_START_TOKEN)
371                 seq_printf(seq, "%-127s\n",
372                            "Iface\tDestination\tGateway \tFlags\t\tRefCnt\tUse\t"
373                            "Metric\tSource\t\tMTU\tWindow\tIRTT\tTOS\tHHRef\t"
374                            "HHUptod\tSpecDst");
375         else {
376                 struct rtable *r = v;
377                 int len;
378
379                 seq_printf(seq, "%s\t%08X\t%08X\t%8X\t%d\t%u\t%d\t"
380                               "%08X\t%d\t%u\t%u\t%02X\t%d\t%1d\t%08X%n",
381                         r->dst.dev ? r->dst.dev->name : "*",
382                         (__force u32)r->rt_dst,
383                         (__force u32)r->rt_gateway,
384                         r->rt_flags, atomic_read(&r->dst.__refcnt),
385                         r->dst.__use, 0, (__force u32)r->rt_src,
386                         (dst_metric(&r->dst, RTAX_ADVMSS) ?
387                              (int)dst_metric(&r->dst, RTAX_ADVMSS) + 40 : 0),
388                         dst_metric(&r->dst, RTAX_WINDOW),
389                         (int)((dst_metric(&r->dst, RTAX_RTT) >> 3) +
390                               dst_metric(&r->dst, RTAX_RTTVAR)),
391                         r->fl.fl4_tos,
392                         r->dst.hh ? atomic_read(&r->dst.hh->hh_refcnt) : -1,
393                         r->dst.hh ? (r->dst.hh->hh_output ==
394                                        dev_queue_xmit) : 0,
395                         r->rt_spec_dst, &len);
396
397                 seq_printf(seq, "%*s\n", 127 - len, "");
398         }
399         return 0;
400 }
401
402 static const struct seq_operations rt_cache_seq_ops = {
403         .start  = rt_cache_seq_start,
404         .next   = rt_cache_seq_next,
405         .stop   = rt_cache_seq_stop,
406         .show   = rt_cache_seq_show,
407 };
408
409 static int rt_cache_seq_open(struct inode *inode, struct file *file)
410 {
411         return seq_open_net(inode, file, &rt_cache_seq_ops,
412                         sizeof(struct rt_cache_iter_state));
413 }
414
415 static const struct file_operations rt_cache_seq_fops = {
416         .owner   = THIS_MODULE,
417         .open    = rt_cache_seq_open,
418         .read    = seq_read,
419         .llseek  = seq_lseek,
420         .release = seq_release_net,
421 };
422
423
424 static void *rt_cpu_seq_start(struct seq_file *seq, loff_t *pos)
425 {
426         int cpu;
427
428         if (*pos == 0)
429                 return SEQ_START_TOKEN;
430
431         for (cpu = *pos-1; cpu < nr_cpu_ids; ++cpu) {
432                 if (!cpu_possible(cpu))
433                         continue;
434                 *pos = cpu+1;
435                 return &per_cpu(rt_cache_stat, cpu);
436         }
437         return NULL;
438 }
439
440 static void *rt_cpu_seq_next(struct seq_file *seq, void *v, loff_t *pos)
441 {
442         int cpu;
443
444         for (cpu = *pos; cpu < nr_cpu_ids; ++cpu) {
445                 if (!cpu_possible(cpu))
446                         continue;
447                 *pos = cpu+1;
448                 return &per_cpu(rt_cache_stat, cpu);
449         }
450         return NULL;
451
452 }
453
454 static void rt_cpu_seq_stop(struct seq_file *seq, void *v)
455 {
456
457 }
458
459 static int rt_cpu_seq_show(struct seq_file *seq, void *v)
460 {
461         struct rt_cache_stat *st = v;
462
463         if (v == SEQ_START_TOKEN) {
464                 seq_printf(seq, "entries  in_hit in_slow_tot in_slow_mc in_no_route in_brd in_martian_dst in_martian_src  out_hit out_slow_tot out_slow_mc  gc_total gc_ignored gc_goal_miss gc_dst_overflow in_hlist_search out_hlist_search\n");
465                 return 0;
466         }
467
468         seq_printf(seq,"%08x  %08x %08x %08x %08x %08x %08x %08x "
469                    " %08x %08x %08x %08x %08x %08x %08x %08x %08x \n",
470                    dst_entries_get_slow(&ipv4_dst_ops),
471                    st->in_hit,
472                    st->in_slow_tot,
473                    st->in_slow_mc,
474                    st->in_no_route,
475                    st->in_brd,
476                    st->in_martian_dst,
477                    st->in_martian_src,
478
479                    st->out_hit,
480                    st->out_slow_tot,
481                    st->out_slow_mc,
482
483                    st->gc_total,
484                    st->gc_ignored,
485                    st->gc_goal_miss,
486                    st->gc_dst_overflow,
487                    st->in_hlist_search,
488                    st->out_hlist_search
489                 );
490         return 0;
491 }
492
493 static const struct seq_operations rt_cpu_seq_ops = {
494         .start  = rt_cpu_seq_start,
495         .next   = rt_cpu_seq_next,
496         .stop   = rt_cpu_seq_stop,
497         .show   = rt_cpu_seq_show,
498 };
499
500
501 static int rt_cpu_seq_open(struct inode *inode, struct file *file)
502 {
503         return seq_open(file, &rt_cpu_seq_ops);
504 }
505
506 static const struct file_operations rt_cpu_seq_fops = {
507         .owner   = THIS_MODULE,
508         .open    = rt_cpu_seq_open,
509         .read    = seq_read,
510         .llseek  = seq_lseek,
511         .release = seq_release,
512 };
513
514 #ifdef CONFIG_IP_ROUTE_CLASSID
515 static int rt_acct_proc_show(struct seq_file *m, void *v)
516 {
517         struct ip_rt_acct *dst, *src;
518         unsigned int i, j;
519
520         dst = kcalloc(256, sizeof(struct ip_rt_acct), GFP_KERNEL);
521         if (!dst)
522                 return -ENOMEM;
523
524         for_each_possible_cpu(i) {
525                 src = (struct ip_rt_acct *)per_cpu_ptr(ip_rt_acct, i);
526                 for (j = 0; j < 256; j++) {
527                         dst[j].o_bytes   += src[j].o_bytes;
528                         dst[j].o_packets += src[j].o_packets;
529                         dst[j].i_bytes   += src[j].i_bytes;
530                         dst[j].i_packets += src[j].i_packets;
531                 }
532         }
533
534         seq_write(m, dst, 256 * sizeof(struct ip_rt_acct));
535         kfree(dst);
536         return 0;
537 }
538
539 static int rt_acct_proc_open(struct inode *inode, struct file *file)
540 {
541         return single_open(file, rt_acct_proc_show, NULL);
542 }
543
544 static const struct file_operations rt_acct_proc_fops = {
545         .owner          = THIS_MODULE,
546         .open           = rt_acct_proc_open,
547         .read           = seq_read,
548         .llseek         = seq_lseek,
549         .release        = single_release,
550 };
551 #endif
552
553 static int __net_init ip_rt_do_proc_init(struct net *net)
554 {
555         struct proc_dir_entry *pde;
556
557         pde = proc_net_fops_create(net, "rt_cache", S_IRUGO,
558                         &rt_cache_seq_fops);
559         if (!pde)
560                 goto err1;
561
562         pde = proc_create("rt_cache", S_IRUGO,
563                           net->proc_net_stat, &rt_cpu_seq_fops);
564         if (!pde)
565                 goto err2;
566
567 #ifdef CONFIG_IP_ROUTE_CLASSID
568         pde = proc_create("rt_acct", 0, net->proc_net, &rt_acct_proc_fops);
569         if (!pde)
570                 goto err3;
571 #endif
572         return 0;
573
574 #ifdef CONFIG_IP_ROUTE_CLASSID
575 err3:
576         remove_proc_entry("rt_cache", net->proc_net_stat);
577 #endif
578 err2:
579         remove_proc_entry("rt_cache", net->proc_net);
580 err1:
581         return -ENOMEM;
582 }
583
584 static void __net_exit ip_rt_do_proc_exit(struct net *net)
585 {
586         remove_proc_entry("rt_cache", net->proc_net_stat);
587         remove_proc_entry("rt_cache", net->proc_net);
588 #ifdef CONFIG_IP_ROUTE_CLASSID
589         remove_proc_entry("rt_acct", net->proc_net);
590 #endif
591 }
592
593 static struct pernet_operations ip_rt_proc_ops __net_initdata =  {
594         .init = ip_rt_do_proc_init,
595         .exit = ip_rt_do_proc_exit,
596 };
597
598 static int __init ip_rt_proc_init(void)
599 {
600         return register_pernet_subsys(&ip_rt_proc_ops);
601 }
602
603 #else
604 static inline int ip_rt_proc_init(void)
605 {
606         return 0;
607 }
608 #endif /* CONFIG_PROC_FS */
609
610 static inline void rt_free(struct rtable *rt)
611 {
612         call_rcu_bh(&rt->dst.rcu_head, dst_rcu_free);
613 }
614
615 static inline void rt_drop(struct rtable *rt)
616 {
617         ip_rt_put(rt);
618         call_rcu_bh(&rt->dst.rcu_head, dst_rcu_free);
619 }
620
621 static inline int rt_fast_clean(struct rtable *rth)
622 {
623         /* Kill broadcast/multicast entries very aggresively, if they
624            collide in hash table with more useful entries */
625         return (rth->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST)) &&
626                 rt_is_input_route(rth) && rth->dst.rt_next;
627 }
628
629 static inline int rt_valuable(struct rtable *rth)
630 {
631         return (rth->rt_flags & (RTCF_REDIRECTED | RTCF_NOTIFY)) ||
632                 rth->dst.expires;
633 }
634
635 static int rt_may_expire(struct rtable *rth, unsigned long tmo1, unsigned long tmo2)
636 {
637         unsigned long age;
638         int ret = 0;
639
640         if (atomic_read(&rth->dst.__refcnt))
641                 goto out;
642
643         ret = 1;
644         if (rth->dst.expires &&
645             time_after_eq(jiffies, rth->dst.expires))
646                 goto out;
647
648         age = jiffies - rth->dst.lastuse;
649         ret = 0;
650         if ((age <= tmo1 && !rt_fast_clean(rth)) ||
651             (age <= tmo2 && rt_valuable(rth)))
652                 goto out;
653         ret = 1;
654 out:    return ret;
655 }
656
657 /* Bits of score are:
658  * 31: very valuable
659  * 30: not quite useless
660  * 29..0: usage counter
661  */
662 static inline u32 rt_score(struct rtable *rt)
663 {
664         u32 score = jiffies - rt->dst.lastuse;
665
666         score = ~score & ~(3<<30);
667
668         if (rt_valuable(rt))
669                 score |= (1<<31);
670
671         if (rt_is_output_route(rt) ||
672             !(rt->rt_flags & (RTCF_BROADCAST|RTCF_MULTICAST|RTCF_LOCAL)))
673                 score |= (1<<30);
674
675         return score;
676 }
677
678 static inline bool rt_caching(const struct net *net)
679 {
680         return net->ipv4.current_rt_cache_rebuild_count <=
681                 net->ipv4.sysctl_rt_cache_rebuild_count;
682 }
683
684 static inline bool compare_hash_inputs(const struct flowi *fl1,
685                                         const struct flowi *fl2)
686 {
687         return ((((__force u32)fl1->nl_u.ip4_u.daddr ^ (__force u32)fl2->nl_u.ip4_u.daddr) |
688                 ((__force u32)fl1->nl_u.ip4_u.saddr ^ (__force u32)fl2->nl_u.ip4_u.saddr) |
689                 (fl1->iif ^ fl2->iif)) == 0);
690 }
691
692 static inline int compare_keys(struct flowi *fl1, struct flowi *fl2)
693 {
694         return (((__force u32)fl1->nl_u.ip4_u.daddr ^ (__force u32)fl2->nl_u.ip4_u.daddr) |
695                 ((__force u32)fl1->nl_u.ip4_u.saddr ^ (__force u32)fl2->nl_u.ip4_u.saddr) |
696                 (fl1->mark ^ fl2->mark) |
697                 (*(u16 *)&fl1->nl_u.ip4_u.tos ^ *(u16 *)&fl2->nl_u.ip4_u.tos) |
698                 (fl1->oif ^ fl2->oif) |
699                 (fl1->iif ^ fl2->iif)) == 0;
700 }
701
702 static inline int compare_netns(struct rtable *rt1, struct rtable *rt2)
703 {
704         return net_eq(dev_net(rt1->dst.dev), dev_net(rt2->dst.dev));
705 }
706
707 static inline int rt_is_expired(struct rtable *rth)
708 {
709         return rth->rt_genid != rt_genid(dev_net(rth->dst.dev));
710 }
711
712 /*
713  * Perform a full scan of hash table and free all entries.
714  * Can be called by a softirq or a process.
715  * In the later case, we want to be reschedule if necessary
716  */
717 static void rt_do_flush(int process_context)
718 {
719         unsigned int i;
720         struct rtable *rth, *next;
721         struct rtable * tail;
722
723         for (i = 0; i <= rt_hash_mask; i++) {
724                 if (process_context && need_resched())
725                         cond_resched();
726                 rth = rcu_dereference_raw(rt_hash_table[i].chain);
727                 if (!rth)
728                         continue;
729
730                 spin_lock_bh(rt_hash_lock_addr(i));
731 #ifdef CONFIG_NET_NS
732                 {
733                 struct rtable __rcu **prev;
734                 struct rtable *p;
735
736                 rth = rcu_dereference_protected(rt_hash_table[i].chain,
737                         lockdep_is_held(rt_hash_lock_addr(i)));
738
739                 /* defer releasing the head of the list after spin_unlock */
740                 for (tail = rth; tail;
741                      tail = rcu_dereference_protected(tail->dst.rt_next,
742                                 lockdep_is_held(rt_hash_lock_addr(i))))
743                         if (!rt_is_expired(tail))
744                                 break;
745                 if (rth != tail)
746                         rt_hash_table[i].chain = tail;
747
748                 /* call rt_free on entries after the tail requiring flush */
749                 prev = &rt_hash_table[i].chain;
750                 for (p = rcu_dereference_protected(*prev,
751                                 lockdep_is_held(rt_hash_lock_addr(i)));
752                      p != NULL;
753                      p = next) {
754                         next = rcu_dereference_protected(p->dst.rt_next,
755                                 lockdep_is_held(rt_hash_lock_addr(i)));
756                         if (!rt_is_expired(p)) {
757                                 prev = &p->dst.rt_next;
758                         } else {
759                                 *prev = next;
760                                 rt_free(p);
761                         }
762                 }
763                 }
764 #else
765                 rth = rcu_dereference_protected(rt_hash_table[i].chain,
766                         lockdep_is_held(rt_hash_lock_addr(i)));
767                 rcu_assign_pointer(rt_hash_table[i].chain, NULL);
768                 tail = NULL;
769 #endif
770                 spin_unlock_bh(rt_hash_lock_addr(i));
771
772                 for (; rth != tail; rth = next) {
773                         next = rcu_dereference_protected(rth->dst.rt_next, 1);
774                         rt_free(rth);
775                 }
776         }
777 }
778
779 /*
780  * While freeing expired entries, we compute average chain length
781  * and standard deviation, using fixed-point arithmetic.
782  * This to have an estimation of rt_chain_length_max
783  *  rt_chain_length_max = max(elasticity, AVG + 4*SD)
784  * We use 3 bits for frational part, and 29 (or 61) for magnitude.
785  */
786
787 #define FRACT_BITS 3
788 #define ONE (1UL << FRACT_BITS)
789
790 /*
791  * Given a hash chain and an item in this hash chain,
792  * find if a previous entry has the same hash_inputs
793  * (but differs on tos, mark or oif)
794  * Returns 0 if an alias is found.
795  * Returns ONE if rth has no alias before itself.
796  */
797 static int has_noalias(const struct rtable *head, const struct rtable *rth)
798 {
799         const struct rtable *aux = head;
800
801         while (aux != rth) {
802                 if (compare_hash_inputs(&aux->fl, &rth->fl))
803                         return 0;
804                 aux = rcu_dereference_protected(aux->dst.rt_next, 1);
805         }
806         return ONE;
807 }
808
809 static void rt_check_expire(void)
810 {
811         static unsigned int rover;
812         unsigned int i = rover, goal;
813         struct rtable *rth;
814         struct rtable __rcu **rthp;
815         unsigned long samples = 0;
816         unsigned long sum = 0, sum2 = 0;
817         unsigned long delta;
818         u64 mult;
819
820         delta = jiffies - expires_ljiffies;
821         expires_ljiffies = jiffies;
822         mult = ((u64)delta) << rt_hash_log;
823         if (ip_rt_gc_timeout > 1)
824                 do_div(mult, ip_rt_gc_timeout);
825         goal = (unsigned int)mult;
826         if (goal > rt_hash_mask)
827                 goal = rt_hash_mask + 1;
828         for (; goal > 0; goal--) {
829                 unsigned long tmo = ip_rt_gc_timeout;
830                 unsigned long length;
831
832                 i = (i + 1) & rt_hash_mask;
833                 rthp = &rt_hash_table[i].chain;
834
835                 if (need_resched())
836                         cond_resched();
837
838                 samples++;
839
840                 if (rcu_dereference_raw(*rthp) == NULL)
841                         continue;
842                 length = 0;
843                 spin_lock_bh(rt_hash_lock_addr(i));
844                 while ((rth = rcu_dereference_protected(*rthp,
845                                         lockdep_is_held(rt_hash_lock_addr(i)))) != NULL) {
846                         prefetch(rth->dst.rt_next);
847                         if (rt_is_expired(rth)) {
848                                 *rthp = rth->dst.rt_next;
849                                 rt_free(rth);
850                                 continue;
851                         }
852                         if (rth->dst.expires) {
853                                 /* Entry is expired even if it is in use */
854                                 if (time_before_eq(jiffies, rth->dst.expires)) {
855 nofree:
856                                         tmo >>= 1;
857                                         rthp = &rth->dst.rt_next;
858                                         /*
859                                          * We only count entries on
860                                          * a chain with equal hash inputs once
861                                          * so that entries for different QOS
862                                          * levels, and other non-hash input
863                                          * attributes don't unfairly skew
864                                          * the length computation
865                                          */
866                                         length += has_noalias(rt_hash_table[i].chain, rth);
867                                         continue;
868                                 }
869                         } else if (!rt_may_expire(rth, tmo, ip_rt_gc_timeout))
870                                 goto nofree;
871
872                         /* Cleanup aged off entries. */
873                         *rthp = rth->dst.rt_next;
874                         rt_free(rth);
875                 }
876                 spin_unlock_bh(rt_hash_lock_addr(i));
877                 sum += length;
878                 sum2 += length*length;
879         }
880         if (samples) {
881                 unsigned long avg = sum / samples;
882                 unsigned long sd = int_sqrt(sum2 / samples - avg*avg);
883                 rt_chain_length_max = max_t(unsigned long,
884                                         ip_rt_gc_elasticity,
885                                         (avg + 4*sd) >> FRACT_BITS);
886         }
887         rover = i;
888 }
889
890 /*
891  * rt_worker_func() is run in process context.
892  * we call rt_check_expire() to scan part of the hash table
893  */
894 static void rt_worker_func(struct work_struct *work)
895 {
896         rt_check_expire();
897         schedule_delayed_work(&expires_work, ip_rt_gc_interval);
898 }
899
900 /*
901  * Pertubation of rt_genid by a small quantity [1..256]
902  * Using 8 bits of shuffling ensure we can call rt_cache_invalidate()
903  * many times (2^24) without giving recent rt_genid.
904  * Jenkins hash is strong enough that litle changes of rt_genid are OK.
905  */
906 static void rt_cache_invalidate(struct net *net)
907 {
908         unsigned char shuffle;
909
910         get_random_bytes(&shuffle, sizeof(shuffle));
911         atomic_add(shuffle + 1U, &net->ipv4.rt_genid);
912 }
913
914 /*
915  * delay < 0  : invalidate cache (fast : entries will be deleted later)
916  * delay >= 0 : invalidate & flush cache (can be long)
917  */
918 void rt_cache_flush(struct net *net, int delay)
919 {
920         rt_cache_invalidate(net);
921         if (delay >= 0)
922                 rt_do_flush(!in_softirq());
923 }
924
925 /* Flush previous cache invalidated entries from the cache */
926 void rt_cache_flush_batch(void)
927 {
928         rt_do_flush(!in_softirq());
929 }
930
931 static void rt_emergency_hash_rebuild(struct net *net)
932 {
933         if (net_ratelimit())
934                 printk(KERN_WARNING "Route hash chain too long!\n");
935         rt_cache_invalidate(net);
936 }
937
938 /*
939    Short description of GC goals.
940
941    We want to build algorithm, which will keep routing cache
942    at some equilibrium point, when number of aged off entries
943    is kept approximately equal to newly generated ones.
944
945    Current expiration strength is variable "expire".
946    We try to adjust it dynamically, so that if networking
947    is idle expires is large enough to keep enough of warm entries,
948    and when load increases it reduces to limit cache size.
949  */
950
951 static int rt_garbage_collect(struct dst_ops *ops)
952 {
953         static unsigned long expire = RT_GC_TIMEOUT;
954         static unsigned long last_gc;
955         static int rover;
956         static int equilibrium;
957         struct rtable *rth;
958         struct rtable __rcu **rthp;
959         unsigned long now = jiffies;
960         int goal;
961         int entries = dst_entries_get_fast(&ipv4_dst_ops);
962
963         /*
964          * Garbage collection is pretty expensive,
965          * do not make it too frequently.
966          */
967
968         RT_CACHE_STAT_INC(gc_total);
969
970         if (now - last_gc < ip_rt_gc_min_interval &&
971             entries < ip_rt_max_size) {
972                 RT_CACHE_STAT_INC(gc_ignored);
973                 goto out;
974         }
975
976         entries = dst_entries_get_slow(&ipv4_dst_ops);
977         /* Calculate number of entries, which we want to expire now. */
978         goal = entries - (ip_rt_gc_elasticity << rt_hash_log);
979         if (goal <= 0) {
980                 if (equilibrium < ipv4_dst_ops.gc_thresh)
981                         equilibrium = ipv4_dst_ops.gc_thresh;
982                 goal = entries - equilibrium;
983                 if (goal > 0) {
984                         equilibrium += min_t(unsigned int, goal >> 1, rt_hash_mask + 1);
985                         goal = entries - equilibrium;
986                 }
987         } else {
988                 /* We are in dangerous area. Try to reduce cache really
989                  * aggressively.
990                  */
991                 goal = max_t(unsigned int, goal >> 1, rt_hash_mask + 1);
992                 equilibrium = entries - goal;
993         }
994
995         if (now - last_gc >= ip_rt_gc_min_interval)
996                 last_gc = now;
997
998         if (goal <= 0) {
999                 equilibrium += goal;
1000                 goto work_done;
1001         }
1002
1003         do {
1004                 int i, k;
1005
1006                 for (i = rt_hash_mask, k = rover; i >= 0; i--) {
1007                         unsigned long tmo = expire;
1008
1009                         k = (k + 1) & rt_hash_mask;
1010                         rthp = &rt_hash_table[k].chain;
1011                         spin_lock_bh(rt_hash_lock_addr(k));
1012                         while ((rth = rcu_dereference_protected(*rthp,
1013                                         lockdep_is_held(rt_hash_lock_addr(k)))) != NULL) {
1014                                 if (!rt_is_expired(rth) &&
1015                                         !rt_may_expire(rth, tmo, expire)) {
1016                                         tmo >>= 1;
1017                                         rthp = &rth->dst.rt_next;
1018                                         continue;
1019                                 }
1020                                 *rthp = rth->dst.rt_next;
1021                                 rt_free(rth);
1022                                 goal--;
1023                         }
1024                         spin_unlock_bh(rt_hash_lock_addr(k));
1025                         if (goal <= 0)
1026                                 break;
1027                 }
1028                 rover = k;
1029
1030                 if (goal <= 0)
1031                         goto work_done;
1032
1033                 /* Goal is not achieved. We stop process if:
1034
1035                    - if expire reduced to zero. Otherwise, expire is halfed.
1036                    - if table is not full.
1037                    - if we are called from interrupt.
1038                    - jiffies check is just fallback/debug loop breaker.
1039                      We will not spin here for long time in any case.
1040                  */
1041
1042                 RT_CACHE_STAT_INC(gc_goal_miss);
1043
1044                 if (expire == 0)
1045                         break;
1046
1047                 expire >>= 1;
1048 #if RT_CACHE_DEBUG >= 2
1049                 printk(KERN_DEBUG "expire>> %u %d %d %d\n", expire,
1050                                 dst_entries_get_fast(&ipv4_dst_ops), goal, i);
1051 #endif
1052
1053                 if (dst_entries_get_fast(&ipv4_dst_ops) < ip_rt_max_size)
1054                         goto out;
1055         } while (!in_softirq() && time_before_eq(jiffies, now));
1056
1057         if (dst_entries_get_fast(&ipv4_dst_ops) < ip_rt_max_size)
1058                 goto out;
1059         if (dst_entries_get_slow(&ipv4_dst_ops) < ip_rt_max_size)
1060                 goto out;
1061         if (net_ratelimit())
1062                 printk(KERN_WARNING "dst cache overflow\n");
1063         RT_CACHE_STAT_INC(gc_dst_overflow);
1064         return 1;
1065
1066 work_done:
1067         expire += ip_rt_gc_min_interval;
1068         if (expire > ip_rt_gc_timeout ||
1069             dst_entries_get_fast(&ipv4_dst_ops) < ipv4_dst_ops.gc_thresh ||
1070             dst_entries_get_slow(&ipv4_dst_ops) < ipv4_dst_ops.gc_thresh)
1071                 expire = ip_rt_gc_timeout;
1072 #if RT_CACHE_DEBUG >= 2
1073         printk(KERN_DEBUG "expire++ %u %d %d %d\n", expire,
1074                         dst_entries_get_fast(&ipv4_dst_ops), goal, rover);
1075 #endif
1076 out:    return 0;
1077 }
1078
1079 /*
1080  * Returns number of entries in a hash chain that have different hash_inputs
1081  */
1082 static int slow_chain_length(const struct rtable *head)
1083 {
1084         int length = 0;
1085         const struct rtable *rth = head;
1086
1087         while (rth) {
1088                 length += has_noalias(head, rth);
1089                 rth = rcu_dereference_protected(rth->dst.rt_next, 1);
1090         }
1091         return length >> FRACT_BITS;
1092 }
1093
1094 static int rt_intern_hash(unsigned hash, struct rtable *rt,
1095                           struct rtable **rp, struct sk_buff *skb, int ifindex)
1096 {
1097         struct rtable   *rth, *cand;
1098         struct rtable __rcu **rthp, **candp;
1099         unsigned long   now;
1100         u32             min_score;
1101         int             chain_length;
1102         int attempts = !in_softirq();
1103
1104 restart:
1105         chain_length = 0;
1106         min_score = ~(u32)0;
1107         cand = NULL;
1108         candp = NULL;
1109         now = jiffies;
1110
1111         if (!rt_caching(dev_net(rt->dst.dev))) {
1112                 /*
1113                  * If we're not caching, just tell the caller we
1114                  * were successful and don't touch the route.  The
1115                  * caller hold the sole reference to the cache entry, and
1116                  * it will be released when the caller is done with it.
1117                  * If we drop it here, the callers have no way to resolve routes
1118                  * when we're not caching.  Instead, just point *rp at rt, so
1119                  * the caller gets a single use out of the route
1120                  * Note that we do rt_free on this new route entry, so that
1121                  * once its refcount hits zero, we are still able to reap it
1122                  * (Thanks Alexey)
1123                  * Note: To avoid expensive rcu stuff for this uncached dst,
1124                  * we set DST_NOCACHE so that dst_release() can free dst without
1125                  * waiting a grace period.
1126                  */
1127
1128                 rt->dst.flags |= DST_NOCACHE;
1129                 if (rt->rt_type == RTN_UNICAST || rt_is_output_route(rt)) {
1130                         int err = arp_bind_neighbour(&rt->dst);
1131                         if (err) {
1132                                 if (net_ratelimit())
1133                                         printk(KERN_WARNING
1134                                             "Neighbour table failure & not caching routes.\n");
1135                                 ip_rt_put(rt);
1136                                 return err;
1137                         }
1138                 }
1139
1140                 goto skip_hashing;
1141         }
1142
1143         rthp = &rt_hash_table[hash].chain;
1144
1145         spin_lock_bh(rt_hash_lock_addr(hash));
1146         while ((rth = rcu_dereference_protected(*rthp,
1147                         lockdep_is_held(rt_hash_lock_addr(hash)))) != NULL) {
1148                 if (rt_is_expired(rth)) {
1149                         *rthp = rth->dst.rt_next;
1150                         rt_free(rth);
1151                         continue;
1152                 }
1153                 if (compare_keys(&rth->fl, &rt->fl) && compare_netns(rth, rt)) {
1154                         /* Put it first */
1155                         *rthp = rth->dst.rt_next;
1156                         /*
1157                          * Since lookup is lockfree, the deletion
1158                          * must be visible to another weakly ordered CPU before
1159                          * the insertion at the start of the hash chain.
1160                          */
1161                         rcu_assign_pointer(rth->dst.rt_next,
1162                                            rt_hash_table[hash].chain);
1163                         /*
1164                          * Since lookup is lockfree, the update writes
1165                          * must be ordered for consistency on SMP.
1166                          */
1167                         rcu_assign_pointer(rt_hash_table[hash].chain, rth);
1168
1169                         dst_use(&rth->dst, now);
1170                         spin_unlock_bh(rt_hash_lock_addr(hash));
1171
1172                         rt_drop(rt);
1173                         if (rp)
1174                                 *rp = rth;
1175                         else
1176                                 skb_dst_set(skb, &rth->dst);
1177                         return 0;
1178                 }
1179
1180                 if (!atomic_read(&rth->dst.__refcnt)) {
1181                         u32 score = rt_score(rth);
1182
1183                         if (score <= min_score) {
1184                                 cand = rth;
1185                                 candp = rthp;
1186                                 min_score = score;
1187                         }
1188                 }
1189
1190                 chain_length++;
1191
1192                 rthp = &rth->dst.rt_next;
1193         }
1194
1195         if (cand) {
1196                 /* ip_rt_gc_elasticity used to be average length of chain
1197                  * length, when exceeded gc becomes really aggressive.
1198                  *
1199                  * The second limit is less certain. At the moment it allows
1200                  * only 2 entries per bucket. We will see.
1201                  */
1202                 if (chain_length > ip_rt_gc_elasticity) {
1203                         *candp = cand->dst.rt_next;
1204                         rt_free(cand);
1205                 }
1206         } else {
1207                 if (chain_length > rt_chain_length_max &&
1208                     slow_chain_length(rt_hash_table[hash].chain) > rt_chain_length_max) {
1209                         struct net *net = dev_net(rt->dst.dev);
1210                         int num = ++net->ipv4.current_rt_cache_rebuild_count;
1211                         if (!rt_caching(net)) {
1212                                 printk(KERN_WARNING "%s: %d rebuilds is over limit, route caching disabled\n",
1213                                         rt->dst.dev->name, num);
1214                         }
1215                         rt_emergency_hash_rebuild(net);
1216                         spin_unlock_bh(rt_hash_lock_addr(hash));
1217
1218                         hash = rt_hash(rt->fl.fl4_dst, rt->fl.fl4_src,
1219                                         ifindex, rt_genid(net));
1220                         goto restart;
1221                 }
1222         }
1223
1224         /* Try to bind route to arp only if it is output
1225            route or unicast forwarding path.
1226          */
1227         if (rt->rt_type == RTN_UNICAST || rt_is_output_route(rt)) {
1228                 int err = arp_bind_neighbour(&rt->dst);
1229                 if (err) {
1230                         spin_unlock_bh(rt_hash_lock_addr(hash));
1231
1232                         if (err != -ENOBUFS) {
1233                                 rt_drop(rt);
1234                                 return err;
1235                         }
1236
1237                         /* Neighbour tables are full and nothing
1238                            can be released. Try to shrink route cache,
1239                            it is most likely it holds some neighbour records.
1240                          */
1241                         if (attempts-- > 0) {
1242                                 int saved_elasticity = ip_rt_gc_elasticity;
1243                                 int saved_int = ip_rt_gc_min_interval;
1244                                 ip_rt_gc_elasticity     = 1;
1245                                 ip_rt_gc_min_interval   = 0;
1246                                 rt_garbage_collect(&ipv4_dst_ops);
1247                                 ip_rt_gc_min_interval   = saved_int;
1248                                 ip_rt_gc_elasticity     = saved_elasticity;
1249                                 goto restart;
1250                         }
1251
1252                         if (net_ratelimit())
1253                                 printk(KERN_WARNING "ipv4: Neighbour table overflow.\n");
1254                         rt_drop(rt);
1255                         return -ENOBUFS;
1256                 }
1257         }
1258
1259         rt->dst.rt_next = rt_hash_table[hash].chain;
1260
1261 #if RT_CACHE_DEBUG >= 2
1262         if (rt->dst.rt_next) {
1263                 struct rtable *trt;
1264                 printk(KERN_DEBUG "rt_cache @%02x: %pI4",
1265                        hash, &rt->rt_dst);
1266                 for (trt = rt->dst.rt_next; trt; trt = trt->dst.rt_next)
1267                         printk(" . %pI4", &trt->rt_dst);
1268                 printk("\n");
1269         }
1270 #endif
1271         /*
1272          * Since lookup is lockfree, we must make sure
1273          * previous writes to rt are comitted to memory
1274          * before making rt visible to other CPUS.
1275          */
1276         rcu_assign_pointer(rt_hash_table[hash].chain, rt);
1277
1278         spin_unlock_bh(rt_hash_lock_addr(hash));
1279
1280 skip_hashing:
1281         if (rp)
1282                 *rp = rt;
1283         else
1284                 skb_dst_set(skb, &rt->dst);
1285         return 0;
1286 }
1287
1288 void rt_bind_peer(struct rtable *rt, int create)
1289 {
1290         struct inet_peer *peer;
1291
1292         peer = inet_getpeer(rt->rt_dst, create);
1293
1294         if (peer && cmpxchg(&rt->peer, NULL, peer) != NULL)
1295                 inet_putpeer(peer);
1296 }
1297
1298 /*
1299  * Peer allocation may fail only in serious out-of-memory conditions.  However
1300  * we still can generate some output.
1301  * Random ID selection looks a bit dangerous because we have no chances to
1302  * select ID being unique in a reasonable period of time.
1303  * But broken packet identifier may be better than no packet at all.
1304  */
1305 static void ip_select_fb_ident(struct iphdr *iph)
1306 {
1307         static DEFINE_SPINLOCK(ip_fb_id_lock);
1308         static u32 ip_fallback_id;
1309         u32 salt;
1310
1311         spin_lock_bh(&ip_fb_id_lock);
1312         salt = secure_ip_id((__force __be32)ip_fallback_id ^ iph->daddr);
1313         iph->id = htons(salt & 0xFFFF);
1314         ip_fallback_id = salt;
1315         spin_unlock_bh(&ip_fb_id_lock);
1316 }
1317
1318 void __ip_select_ident(struct iphdr *iph, struct dst_entry *dst, int more)
1319 {
1320         struct rtable *rt = (struct rtable *) dst;
1321
1322         if (rt) {
1323                 if (rt->peer == NULL)
1324                         rt_bind_peer(rt, 1);
1325
1326                 /* If peer is attached to destination, it is never detached,
1327                    so that we need not to grab a lock to dereference it.
1328                  */
1329                 if (rt->peer) {
1330                         iph->id = htons(inet_getid(rt->peer, more));
1331                         return;
1332                 }
1333         } else
1334                 printk(KERN_DEBUG "rt_bind_peer(0) @%p\n",
1335                        __builtin_return_address(0));
1336
1337         ip_select_fb_ident(iph);
1338 }
1339 EXPORT_SYMBOL(__ip_select_ident);
1340
1341 static void rt_del(unsigned hash, struct rtable *rt)
1342 {
1343         struct rtable __rcu **rthp;
1344         struct rtable *aux;
1345
1346         rthp = &rt_hash_table[hash].chain;
1347         spin_lock_bh(rt_hash_lock_addr(hash));
1348         ip_rt_put(rt);
1349         while ((aux = rcu_dereference_protected(*rthp,
1350                         lockdep_is_held(rt_hash_lock_addr(hash)))) != NULL) {
1351                 if (aux == rt || rt_is_expired(aux)) {
1352                         *rthp = aux->dst.rt_next;
1353                         rt_free(aux);
1354                         continue;
1355                 }
1356                 rthp = &aux->dst.rt_next;
1357         }
1358         spin_unlock_bh(rt_hash_lock_addr(hash));
1359 }
1360
1361 /* called in rcu_read_lock() section */
1362 void ip_rt_redirect(__be32 old_gw, __be32 daddr, __be32 new_gw,
1363                     __be32 saddr, struct net_device *dev)
1364 {
1365         int i, k;
1366         struct in_device *in_dev = __in_dev_get_rcu(dev);
1367         struct rtable *rth;
1368         struct rtable __rcu **rthp;
1369         __be32  skeys[2] = { saddr, 0 };
1370         int  ikeys[2] = { dev->ifindex, 0 };
1371         struct netevent_redirect netevent;
1372         struct net *net;
1373
1374         if (!in_dev)
1375                 return;
1376
1377         net = dev_net(dev);
1378         if (new_gw == old_gw || !IN_DEV_RX_REDIRECTS(in_dev) ||
1379             ipv4_is_multicast(new_gw) || ipv4_is_lbcast(new_gw) ||
1380             ipv4_is_zeronet(new_gw))
1381                 goto reject_redirect;
1382
1383         if (!rt_caching(net))
1384                 goto reject_redirect;
1385
1386         if (!IN_DEV_SHARED_MEDIA(in_dev)) {
1387                 if (!inet_addr_onlink(in_dev, new_gw, old_gw))
1388                         goto reject_redirect;
1389                 if (IN_DEV_SEC_REDIRECTS(in_dev) && ip_fib_check_default(new_gw, dev))
1390                         goto reject_redirect;
1391         } else {
1392                 if (inet_addr_type(net, new_gw) != RTN_UNICAST)
1393                         goto reject_redirect;
1394         }
1395
1396         for (i = 0; i < 2; i++) {
1397                 for (k = 0; k < 2; k++) {
1398                         unsigned hash = rt_hash(daddr, skeys[i], ikeys[k],
1399                                                 rt_genid(net));
1400
1401                         rthp = &rt_hash_table[hash].chain;
1402
1403                         while ((rth = rcu_dereference(*rthp)) != NULL) {
1404                                 struct rtable *rt;
1405
1406                                 if (rth->fl.fl4_dst != daddr ||
1407                                     rth->fl.fl4_src != skeys[i] ||
1408                                     rth->fl.oif != ikeys[k] ||
1409                                     rt_is_input_route(rth) ||
1410                                     rt_is_expired(rth) ||
1411                                     !net_eq(dev_net(rth->dst.dev), net)) {
1412                                         rthp = &rth->dst.rt_next;
1413                                         continue;
1414                                 }
1415
1416                                 if (rth->rt_dst != daddr ||
1417                                     rth->rt_src != saddr ||
1418                                     rth->dst.error ||
1419                                     rth->rt_gateway != old_gw ||
1420                                     rth->dst.dev != dev)
1421                                         break;
1422
1423                                 dst_hold(&rth->dst);
1424
1425                                 rt = dst_alloc(&ipv4_dst_ops);
1426                                 if (rt == NULL) {
1427                                         ip_rt_put(rth);
1428                                         return;
1429                                 }
1430
1431                                 /* Copy all the information. */
1432                                 *rt = *rth;
1433                                 rt->dst.__use           = 1;
1434                                 atomic_set(&rt->dst.__refcnt, 1);
1435                                 rt->dst.child           = NULL;
1436                                 if (rt->dst.dev)
1437                                         dev_hold(rt->dst.dev);
1438                                 rt->dst.obsolete        = -1;
1439                                 rt->dst.lastuse = jiffies;
1440                                 rt->dst.path            = &rt->dst;
1441                                 rt->dst.neighbour       = NULL;
1442                                 rt->dst.hh              = NULL;
1443 #ifdef CONFIG_XFRM
1444                                 rt->dst.xfrm            = NULL;
1445 #endif
1446                                 rt->rt_genid            = rt_genid(net);
1447                                 rt->rt_flags            |= RTCF_REDIRECTED;
1448
1449                                 /* Gateway is different ... */
1450                                 rt->rt_gateway          = new_gw;
1451
1452                                 /* Redirect received -> path was valid */
1453                                 dst_confirm(&rth->dst);
1454
1455                                 if (rt->peer)
1456                                         atomic_inc(&rt->peer->refcnt);
1457
1458                                 if (arp_bind_neighbour(&rt->dst) ||
1459                                     !(rt->dst.neighbour->nud_state &
1460                                             NUD_VALID)) {
1461                                         if (rt->dst.neighbour)
1462                                                 neigh_event_send(rt->dst.neighbour, NULL);
1463                                         ip_rt_put(rth);
1464                                         rt_drop(rt);
1465                                         goto do_next;
1466                                 }
1467
1468                                 netevent.old = &rth->dst;
1469                                 netevent.new = &rt->dst;
1470                                 call_netevent_notifiers(NETEVENT_REDIRECT,
1471                                                         &netevent);
1472
1473                                 rt_del(hash, rth);
1474                                 if (!rt_intern_hash(hash, rt, &rt, NULL, rt->fl.oif))
1475                                         ip_rt_put(rt);
1476                                 goto do_next;
1477                         }
1478                 do_next:
1479                         ;
1480                 }
1481         }
1482         return;
1483
1484 reject_redirect:
1485 #ifdef CONFIG_IP_ROUTE_VERBOSE
1486         if (IN_DEV_LOG_MARTIANS(in_dev) && net_ratelimit())
1487                 printk(KERN_INFO "Redirect from %pI4 on %s about %pI4 ignored.\n"
1488                         "  Advised path = %pI4 -> %pI4\n",
1489                        &old_gw, dev->name, &new_gw,
1490                        &saddr, &daddr);
1491 #endif
1492         ;
1493 }
1494
1495 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst)
1496 {
1497         struct rtable *rt = (struct rtable *)dst;
1498         struct dst_entry *ret = dst;
1499
1500         if (rt) {
1501                 if (dst->obsolete > 0) {
1502                         ip_rt_put(rt);
1503                         ret = NULL;
1504                 } else if ((rt->rt_flags & RTCF_REDIRECTED) ||
1505                            (rt->dst.expires &&
1506                             time_after_eq(jiffies, rt->dst.expires))) {
1507                         unsigned hash = rt_hash(rt->fl.fl4_dst, rt->fl.fl4_src,
1508                                                 rt->fl.oif,
1509                                                 rt_genid(dev_net(dst->dev)));
1510 #if RT_CACHE_DEBUG >= 1
1511                         printk(KERN_DEBUG "ipv4_negative_advice: redirect to %pI4/%02x dropped\n",
1512                                 &rt->rt_dst, rt->fl.fl4_tos);
1513 #endif
1514                         rt_del(hash, rt);
1515                         ret = NULL;
1516                 }
1517         }
1518         return ret;
1519 }
1520
1521 /*
1522  * Algorithm:
1523  *      1. The first ip_rt_redirect_number redirects are sent
1524  *         with exponential backoff, then we stop sending them at all,
1525  *         assuming that the host ignores our redirects.
1526  *      2. If we did not see packets requiring redirects
1527  *         during ip_rt_redirect_silence, we assume that the host
1528  *         forgot redirected route and start to send redirects again.
1529  *
1530  * This algorithm is much cheaper and more intelligent than dumb load limiting
1531  * in icmp.c.
1532  *
1533  * NOTE. Do not forget to inhibit load limiting for redirects (redundant)
1534  * and "frag. need" (breaks PMTU discovery) in icmp.c.
1535  */
1536
1537 void ip_rt_send_redirect(struct sk_buff *skb)
1538 {
1539         struct rtable *rt = skb_rtable(skb);
1540         struct in_device *in_dev;
1541         int log_martians;
1542
1543         rcu_read_lock();
1544         in_dev = __in_dev_get_rcu(rt->dst.dev);
1545         if (!in_dev || !IN_DEV_TX_REDIRECTS(in_dev)) {
1546                 rcu_read_unlock();
1547                 return;
1548         }
1549         log_martians = IN_DEV_LOG_MARTIANS(in_dev);
1550         rcu_read_unlock();
1551
1552         /* No redirected packets during ip_rt_redirect_silence;
1553          * reset the algorithm.
1554          */
1555         if (time_after(jiffies, rt->dst.rate_last + ip_rt_redirect_silence))
1556                 rt->dst.rate_tokens = 0;
1557
1558         /* Too many ignored redirects; do not send anything
1559          * set dst.rate_last to the last seen redirected packet.
1560          */
1561         if (rt->dst.rate_tokens >= ip_rt_redirect_number) {
1562                 rt->dst.rate_last = jiffies;
1563                 return;
1564         }
1565
1566         /* Check for load limit; set rate_last to the latest sent
1567          * redirect.
1568          */
1569         if (rt->dst.rate_tokens == 0 ||
1570             time_after(jiffies,
1571                        (rt->dst.rate_last +
1572                         (ip_rt_redirect_load << rt->dst.rate_tokens)))) {
1573                 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST, rt->rt_gateway);
1574                 rt->dst.rate_last = jiffies;
1575                 ++rt->dst.rate_tokens;
1576 #ifdef CONFIG_IP_ROUTE_VERBOSE
1577                 if (log_martians &&
1578                     rt->dst.rate_tokens == ip_rt_redirect_number &&
1579                     net_ratelimit())
1580                         printk(KERN_WARNING "host %pI4/if%d ignores redirects for %pI4 to %pI4.\n",
1581                                 &rt->rt_src, rt->rt_iif,
1582                                 &rt->rt_dst, &rt->rt_gateway);
1583 #endif
1584         }
1585 }
1586
1587 static int ip_error(struct sk_buff *skb)
1588 {
1589         struct rtable *rt = skb_rtable(skb);
1590         unsigned long now;
1591         int code;
1592
1593         switch (rt->dst.error) {
1594                 case EINVAL:
1595                 default:
1596                         goto out;
1597                 case EHOSTUNREACH:
1598                         code = ICMP_HOST_UNREACH;
1599                         break;
1600                 case ENETUNREACH:
1601                         code = ICMP_NET_UNREACH;
1602                         IP_INC_STATS_BH(dev_net(rt->dst.dev),
1603                                         IPSTATS_MIB_INNOROUTES);
1604                         break;
1605                 case EACCES:
1606                         code = ICMP_PKT_FILTERED;
1607                         break;
1608         }
1609
1610         now = jiffies;
1611         rt->dst.rate_tokens += now - rt->dst.rate_last;
1612         if (rt->dst.rate_tokens > ip_rt_error_burst)
1613                 rt->dst.rate_tokens = ip_rt_error_burst;
1614         rt->dst.rate_last = now;
1615         if (rt->dst.rate_tokens >= ip_rt_error_cost) {
1616                 rt->dst.rate_tokens -= ip_rt_error_cost;
1617                 icmp_send(skb, ICMP_DEST_UNREACH, code, 0);
1618         }
1619
1620 out:    kfree_skb(skb);
1621         return 0;
1622 }
1623
1624 /*
1625  *      The last two values are not from the RFC but
1626  *      are needed for AMPRnet AX.25 paths.
1627  */
1628
1629 static const unsigned short mtu_plateau[] =
1630 {32000, 17914, 8166, 4352, 2002, 1492, 576, 296, 216, 128 };
1631
1632 static inline unsigned short guess_mtu(unsigned short old_mtu)
1633 {
1634         int i;
1635
1636         for (i = 0; i < ARRAY_SIZE(mtu_plateau); i++)
1637                 if (old_mtu > mtu_plateau[i])
1638                         return mtu_plateau[i];
1639         return 68;
1640 }
1641
1642 unsigned short ip_rt_frag_needed(struct net *net, struct iphdr *iph,
1643                                  unsigned short new_mtu,
1644                                  struct net_device *dev)
1645 {
1646         int i, k;
1647         unsigned short old_mtu = ntohs(iph->tot_len);
1648         struct rtable *rth;
1649         int  ikeys[2] = { dev->ifindex, 0 };
1650         __be32  skeys[2] = { iph->saddr, 0, };
1651         __be32  daddr = iph->daddr;
1652         unsigned short est_mtu = 0;
1653
1654         for (k = 0; k < 2; k++) {
1655                 for (i = 0; i < 2; i++) {
1656                         unsigned hash = rt_hash(daddr, skeys[i], ikeys[k],
1657                                                 rt_genid(net));
1658
1659                         rcu_read_lock();
1660                         for (rth = rcu_dereference(rt_hash_table[hash].chain); rth;
1661                              rth = rcu_dereference(rth->dst.rt_next)) {
1662                                 unsigned short mtu = new_mtu;
1663
1664                                 if (rth->fl.fl4_dst != daddr ||
1665                                     rth->fl.fl4_src != skeys[i] ||
1666                                     rth->rt_dst != daddr ||
1667                                     rth->rt_src != iph->saddr ||
1668                                     rth->fl.oif != ikeys[k] ||
1669                                     rt_is_input_route(rth) ||
1670                                     dst_metric_locked(&rth->dst, RTAX_MTU) ||
1671                                     !net_eq(dev_net(rth->dst.dev), net) ||
1672                                     rt_is_expired(rth))
1673                                         continue;
1674
1675                                 if (new_mtu < 68 || new_mtu >= old_mtu) {
1676
1677                                         /* BSD 4.2 compatibility hack :-( */
1678                                         if (mtu == 0 &&
1679                                             old_mtu >= dst_mtu(&rth->dst) &&
1680                                             old_mtu >= 68 + (iph->ihl << 2))
1681                                                 old_mtu -= iph->ihl << 2;
1682
1683                                         mtu = guess_mtu(old_mtu);
1684                                 }
1685                                 if (mtu <= dst_mtu(&rth->dst)) {
1686                                         if (mtu < dst_mtu(&rth->dst)) {
1687                                                 dst_confirm(&rth->dst);
1688                                                 if (mtu < ip_rt_min_pmtu) {
1689                                                         mtu = ip_rt_min_pmtu;
1690                                                         rth->dst.metrics[RTAX_LOCK-1] |=
1691                                                                 (1 << RTAX_MTU);
1692                                                 }
1693                                                 rth->dst.metrics[RTAX_MTU-1] = mtu;
1694                                                 dst_set_expires(&rth->dst,
1695                                                         ip_rt_mtu_expires);
1696                                         }
1697                                         est_mtu = mtu;
1698                                 }
1699                         }
1700                         rcu_read_unlock();
1701                 }
1702         }
1703         return est_mtu ? : new_mtu;
1704 }
1705
1706 static void ip_rt_update_pmtu(struct dst_entry *dst, u32 mtu)
1707 {
1708         if (dst_mtu(dst) > mtu && mtu >= 68 &&
1709             !(dst_metric_locked(dst, RTAX_MTU))) {
1710                 if (mtu < ip_rt_min_pmtu) {
1711                         mtu = ip_rt_min_pmtu;
1712                         dst->metrics[RTAX_LOCK-1] |= (1 << RTAX_MTU);
1713                 }
1714                 dst->metrics[RTAX_MTU-1] = mtu;
1715                 dst_set_expires(dst, ip_rt_mtu_expires);
1716                 call_netevent_notifiers(NETEVENT_PMTU_UPDATE, dst);
1717         }
1718 }
1719
1720 static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie)
1721 {
1722         if (rt_is_expired((struct rtable *)dst))
1723                 return NULL;
1724         return dst;
1725 }
1726
1727 static void ipv4_dst_destroy(struct dst_entry *dst)
1728 {
1729         struct rtable *rt = (struct rtable *) dst;
1730         struct inet_peer *peer = rt->peer;
1731
1732         if (peer) {
1733                 rt->peer = NULL;
1734                 inet_putpeer(peer);
1735         }
1736 }
1737
1738
1739 static void ipv4_link_failure(struct sk_buff *skb)
1740 {
1741         struct rtable *rt;
1742
1743         icmp_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0);
1744
1745         rt = skb_rtable(skb);
1746         if (rt)
1747                 dst_set_expires(&rt->dst, 0);
1748 }
1749
1750 static int ip_rt_bug(struct sk_buff *skb)
1751 {
1752         printk(KERN_DEBUG "ip_rt_bug: %pI4 -> %pI4, %s\n",
1753                 &ip_hdr(skb)->saddr, &ip_hdr(skb)->daddr,
1754                 skb->dev ? skb->dev->name : "?");
1755         kfree_skb(skb);
1756         return 0;
1757 }
1758
1759 /*
1760    We do not cache source address of outgoing interface,
1761    because it is used only by IP RR, TS and SRR options,
1762    so that it out of fast path.
1763
1764    BTW remember: "addr" is allowed to be not aligned
1765    in IP options!
1766  */
1767
1768 void ip_rt_get_source(u8 *addr, struct rtable *rt)
1769 {
1770         __be32 src;
1771         struct fib_result res;
1772
1773         if (rt_is_output_route(rt))
1774                 src = rt->rt_src;
1775         else {
1776                 rcu_read_lock();
1777                 if (fib_lookup(dev_net(rt->dst.dev), &rt->fl, &res) == 0)
1778                         src = FIB_RES_PREFSRC(res);
1779                 else
1780                         src = inet_select_addr(rt->dst.dev, rt->rt_gateway,
1781                                         RT_SCOPE_UNIVERSE);
1782                 rcu_read_unlock();
1783         }
1784         memcpy(addr, &src, 4);
1785 }
1786
1787 #ifdef CONFIG_IP_ROUTE_CLASSID
1788 static void set_class_tag(struct rtable *rt, u32 tag)
1789 {
1790         if (!(rt->dst.tclassid & 0xFFFF))
1791                 rt->dst.tclassid |= tag & 0xFFFF;
1792         if (!(rt->dst.tclassid & 0xFFFF0000))
1793                 rt->dst.tclassid |= tag & 0xFFFF0000;
1794 }
1795 #endif
1796
1797 static void rt_set_nexthop(struct rtable *rt, struct fib_result *res, u32 itag)
1798 {
1799         struct fib_info *fi = res->fi;
1800
1801         if (fi) {
1802                 if (FIB_RES_GW(*res) &&
1803                     FIB_RES_NH(*res).nh_scope == RT_SCOPE_LINK)
1804                         rt->rt_gateway = FIB_RES_GW(*res);
1805                 memcpy(rt->dst.metrics, fi->fib_metrics,
1806                        sizeof(rt->dst.metrics));
1807                 if (fi->fib_mtu == 0) {
1808                         rt->dst.metrics[RTAX_MTU-1] = rt->dst.dev->mtu;
1809                         if (dst_metric_locked(&rt->dst, RTAX_MTU) &&
1810                             rt->rt_gateway != rt->rt_dst &&
1811                             rt->dst.dev->mtu > 576)
1812                                 rt->dst.metrics[RTAX_MTU-1] = 576;
1813                 }
1814 #ifdef CONFIG_IP_ROUTE_CLASSID
1815                 rt->dst.tclassid = FIB_RES_NH(*res).nh_tclassid;
1816 #endif
1817         } else
1818                 rt->dst.metrics[RTAX_MTU-1]= rt->dst.dev->mtu;
1819
1820         if (dst_metric(&rt->dst, RTAX_HOPLIMIT) == 0)
1821                 rt->dst.metrics[RTAX_HOPLIMIT-1] = sysctl_ip_default_ttl;
1822         if (dst_mtu(&rt->dst) > IP_MAX_MTU)
1823                 rt->dst.metrics[RTAX_MTU-1] = IP_MAX_MTU;
1824         if (dst_metric(&rt->dst, RTAX_ADVMSS) == 0)
1825                 rt->dst.metrics[RTAX_ADVMSS-1] = max_t(unsigned int, rt->dst.dev->mtu - 40,
1826                                        ip_rt_min_advmss);
1827         if (dst_metric(&rt->dst, RTAX_ADVMSS) > 65535 - 40)
1828                 rt->dst.metrics[RTAX_ADVMSS-1] = 65535 - 40;
1829
1830 #ifdef CONFIG_IP_ROUTE_CLASSID
1831 #ifdef CONFIG_IP_MULTIPLE_TABLES
1832         set_class_tag(rt, fib_rules_tclass(res));
1833 #endif
1834         set_class_tag(rt, itag);
1835 #endif
1836         rt->rt_type = res->type;
1837 }
1838
1839 /* called in rcu_read_lock() section */
1840 static int ip_route_input_mc(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1841                                 u8 tos, struct net_device *dev, int our)
1842 {
1843         unsigned int hash;
1844         struct rtable *rth;
1845         __be32 spec_dst;
1846         struct in_device *in_dev = __in_dev_get_rcu(dev);
1847         u32 itag = 0;
1848         int err;
1849
1850         /* Primary sanity checks. */
1851
1852         if (in_dev == NULL)
1853                 return -EINVAL;
1854
1855         if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) ||
1856             ipv4_is_loopback(saddr) || skb->protocol != htons(ETH_P_IP))
1857                 goto e_inval;
1858
1859         if (ipv4_is_zeronet(saddr)) {
1860                 if (!ipv4_is_local_multicast(daddr))
1861                         goto e_inval;
1862                 spec_dst = inet_select_addr(dev, 0, RT_SCOPE_LINK);
1863         } else {
1864                 err = fib_validate_source(saddr, 0, tos, 0, dev, &spec_dst,
1865                                           &itag, 0);
1866                 if (err < 0)
1867                         goto e_err;
1868         }
1869         rth = dst_alloc(&ipv4_dst_ops);
1870         if (!rth)
1871                 goto e_nobufs;
1872
1873         rth->dst.output = ip_rt_bug;
1874         rth->dst.obsolete = -1;
1875
1876         atomic_set(&rth->dst.__refcnt, 1);
1877         rth->dst.flags= DST_HOST;
1878         if (IN_DEV_CONF_GET(in_dev, NOPOLICY))
1879                 rth->dst.flags |= DST_NOPOLICY;
1880         rth->fl.fl4_dst = daddr;
1881         rth->rt_dst     = daddr;
1882         rth->fl.fl4_tos = tos;
1883         rth->fl.mark    = skb->mark;
1884         rth->fl.fl4_src = saddr;
1885         rth->rt_src     = saddr;
1886 #ifdef CONFIG_IP_ROUTE_CLASSID
1887         rth->dst.tclassid = itag;
1888 #endif
1889         rth->rt_iif     =
1890         rth->fl.iif     = dev->ifindex;
1891         rth->dst.dev    = init_net.loopback_dev;
1892         dev_hold(rth->dst.dev);
1893         rth->fl.oif     = 0;
1894         rth->rt_gateway = daddr;
1895         rth->rt_spec_dst= spec_dst;
1896         rth->rt_genid   = rt_genid(dev_net(dev));
1897         rth->rt_flags   = RTCF_MULTICAST;
1898         rth->rt_type    = RTN_MULTICAST;
1899         if (our) {
1900                 rth->dst.input= ip_local_deliver;
1901                 rth->rt_flags |= RTCF_LOCAL;
1902         }
1903
1904 #ifdef CONFIG_IP_MROUTE
1905         if (!ipv4_is_local_multicast(daddr) && IN_DEV_MFORWARD(in_dev))
1906                 rth->dst.input = ip_mr_input;
1907 #endif
1908         RT_CACHE_STAT_INC(in_slow_mc);
1909
1910         hash = rt_hash(daddr, saddr, dev->ifindex, rt_genid(dev_net(dev)));
1911         return rt_intern_hash(hash, rth, NULL, skb, dev->ifindex);
1912
1913 e_nobufs:
1914         return -ENOBUFS;
1915 e_inval:
1916         return -EINVAL;
1917 e_err:
1918         return err;
1919 }
1920
1921
1922 static void ip_handle_martian_source(struct net_device *dev,
1923                                      struct in_device *in_dev,
1924                                      struct sk_buff *skb,
1925                                      __be32 daddr,
1926                                      __be32 saddr)
1927 {
1928         RT_CACHE_STAT_INC(in_martian_src);
1929 #ifdef CONFIG_IP_ROUTE_VERBOSE
1930         if (IN_DEV_LOG_MARTIANS(in_dev) && net_ratelimit()) {
1931                 /*
1932                  *      RFC1812 recommendation, if source is martian,
1933                  *      the only hint is MAC header.
1934                  */
1935                 printk(KERN_WARNING "martian source %pI4 from %pI4, on dev %s\n",
1936                         &daddr, &saddr, dev->name);
1937                 if (dev->hard_header_len && skb_mac_header_was_set(skb)) {
1938                         int i;
1939                         const unsigned char *p = skb_mac_header(skb);
1940                         printk(KERN_WARNING "ll header: ");
1941                         for (i = 0; i < dev->hard_header_len; i++, p++) {
1942                                 printk("%02x", *p);
1943                                 if (i < (dev->hard_header_len - 1))
1944                                         printk(":");
1945                         }
1946                         printk("\n");
1947                 }
1948         }
1949 #endif
1950 }
1951
1952 /* called in rcu_read_lock() section */
1953 static int __mkroute_input(struct sk_buff *skb,
1954                            struct fib_result *res,
1955                            struct in_device *in_dev,
1956                            __be32 daddr, __be32 saddr, u32 tos,
1957                            struct rtable **result)
1958 {
1959         struct rtable *rth;
1960         int err;
1961         struct in_device *out_dev;
1962         unsigned int flags = 0;
1963         __be32 spec_dst;
1964         u32 itag;
1965
1966         /* get a working reference to the output device */
1967         out_dev = __in_dev_get_rcu(FIB_RES_DEV(*res));
1968         if (out_dev == NULL) {
1969                 if (net_ratelimit())
1970                         printk(KERN_CRIT "Bug in ip_route_input" \
1971                                "_slow(). Please, report\n");
1972                 return -EINVAL;
1973         }
1974
1975
1976         err = fib_validate_source(saddr, daddr, tos, FIB_RES_OIF(*res),
1977                                   in_dev->dev, &spec_dst, &itag, skb->mark);
1978         if (err < 0) {
1979                 ip_handle_martian_source(in_dev->dev, in_dev, skb, daddr,
1980                                          saddr);
1981
1982                 goto cleanup;
1983         }
1984
1985         if (err)
1986                 flags |= RTCF_DIRECTSRC;
1987
1988         if (out_dev == in_dev && err &&
1989             (IN_DEV_SHARED_MEDIA(out_dev) ||
1990              inet_addr_onlink(out_dev, saddr, FIB_RES_GW(*res))))
1991                 flags |= RTCF_DOREDIRECT;
1992
1993         if (skb->protocol != htons(ETH_P_IP)) {
1994                 /* Not IP (i.e. ARP). Do not create route, if it is
1995                  * invalid for proxy arp. DNAT routes are always valid.
1996                  *
1997                  * Proxy arp feature have been extended to allow, ARP
1998                  * replies back to the same interface, to support
1999                  * Private VLAN switch technologies. See arp.c.
2000                  */
2001                 if (out_dev == in_dev &&
2002                     IN_DEV_PROXY_ARP_PVLAN(in_dev) == 0) {
2003                         err = -EINVAL;
2004                         goto cleanup;
2005                 }
2006         }
2007
2008
2009         rth = dst_alloc(&ipv4_dst_ops);
2010         if (!rth) {
2011                 err = -ENOBUFS;
2012                 goto cleanup;
2013         }
2014
2015         atomic_set(&rth->dst.__refcnt, 1);
2016         rth->dst.flags= DST_HOST;
2017         if (IN_DEV_CONF_GET(in_dev, NOPOLICY))
2018                 rth->dst.flags |= DST_NOPOLICY;
2019         if (IN_DEV_CONF_GET(out_dev, NOXFRM))
2020                 rth->dst.flags |= DST_NOXFRM;
2021         rth->fl.fl4_dst = daddr;
2022         rth->rt_dst     = daddr;
2023         rth->fl.fl4_tos = tos;
2024         rth->fl.mark    = skb->mark;
2025         rth->fl.fl4_src = saddr;
2026         rth->rt_src     = saddr;
2027         rth->rt_gateway = daddr;
2028         rth->rt_iif     =
2029                 rth->fl.iif     = in_dev->dev->ifindex;
2030         rth->dst.dev    = (out_dev)->dev;
2031         dev_hold(rth->dst.dev);
2032         rth->fl.oif     = 0;
2033         rth->rt_spec_dst= spec_dst;
2034
2035         rth->dst.obsolete = -1;
2036         rth->dst.input = ip_forward;
2037         rth->dst.output = ip_output;
2038         rth->rt_genid = rt_genid(dev_net(rth->dst.dev));
2039
2040         rt_set_nexthop(rth, res, itag);
2041
2042         rth->rt_flags = flags;
2043
2044         *result = rth;
2045         err = 0;
2046  cleanup:
2047         return err;
2048 }
2049
2050 static int ip_mkroute_input(struct sk_buff *skb,
2051                             struct fib_result *res,
2052                             const struct flowi *fl,
2053                             struct in_device *in_dev,
2054                             __be32 daddr, __be32 saddr, u32 tos)
2055 {
2056         struct rtable* rth = NULL;
2057         int err;
2058         unsigned hash;
2059
2060 #ifdef CONFIG_IP_ROUTE_MULTIPATH
2061         if (res->fi && res->fi->fib_nhs > 1 && fl->oif == 0)
2062                 fib_select_multipath(fl, res);
2063 #endif
2064
2065         /* create a routing cache entry */
2066         err = __mkroute_input(skb, res, in_dev, daddr, saddr, tos, &rth);
2067         if (err)
2068                 return err;
2069
2070         /* put it into the cache */
2071         hash = rt_hash(daddr, saddr, fl->iif,
2072                        rt_genid(dev_net(rth->dst.dev)));
2073         return rt_intern_hash(hash, rth, NULL, skb, fl->iif);
2074 }
2075
2076 /*
2077  *      NOTE. We drop all the packets that has local source
2078  *      addresses, because every properly looped back packet
2079  *      must have correct destination already attached by output routine.
2080  *
2081  *      Such approach solves two big problems:
2082  *      1. Not simplex devices are handled properly.
2083  *      2. IP spoofing attempts are filtered with 100% of guarantee.
2084  *      called with rcu_read_lock()
2085  */
2086
2087 static int ip_route_input_slow(struct sk_buff *skb, __be32 daddr, __be32 saddr,
2088                                u8 tos, struct net_device *dev)
2089 {
2090         struct fib_result res;
2091         struct in_device *in_dev = __in_dev_get_rcu(dev);
2092         struct flowi fl = { .nl_u = { .ip4_u =
2093                                       { .daddr = daddr,
2094                                         .saddr = saddr,
2095                                         .tos = tos,
2096                                         .scope = RT_SCOPE_UNIVERSE,
2097                                       } },
2098                             .mark = skb->mark,
2099                             .iif = dev->ifindex };
2100         unsigned        flags = 0;
2101         u32             itag = 0;
2102         struct rtable * rth;
2103         unsigned        hash;
2104         __be32          spec_dst;
2105         int             err = -EINVAL;
2106         struct net    * net = dev_net(dev);
2107
2108         /* IP on this device is disabled. */
2109
2110         if (!in_dev)
2111                 goto out;
2112
2113         /* Check for the most weird martians, which can be not detected
2114            by fib_lookup.
2115          */
2116
2117         if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) ||
2118             ipv4_is_loopback(saddr))
2119                 goto martian_source;
2120
2121         if (ipv4_is_lbcast(daddr) || (saddr == 0 && daddr == 0))
2122                 goto brd_input;
2123
2124         /* Accept zero addresses only to limited broadcast;
2125          * I even do not know to fix it or not. Waiting for complains :-)
2126          */
2127         if (ipv4_is_zeronet(saddr))
2128                 goto martian_source;
2129
2130         if (ipv4_is_zeronet(daddr) || ipv4_is_loopback(daddr))
2131                 goto martian_destination;
2132
2133         /*
2134          *      Now we are ready to route packet.
2135          */
2136         err = fib_lookup(net, &fl, &res);
2137         if (err != 0) {
2138                 if (!IN_DEV_FORWARD(in_dev))
2139                         goto e_hostunreach;
2140                 goto no_route;
2141         }
2142
2143         RT_CACHE_STAT_INC(in_slow_tot);
2144
2145         if (res.type == RTN_BROADCAST)
2146                 goto brd_input;
2147
2148         if (res.type == RTN_LOCAL) {
2149                 err = fib_validate_source(saddr, daddr, tos,
2150                                           net->loopback_dev->ifindex,
2151                                           dev, &spec_dst, &itag, skb->mark);
2152                 if (err < 0)
2153                         goto martian_source_keep_err;
2154                 if (err)
2155                         flags |= RTCF_DIRECTSRC;
2156                 spec_dst = daddr;
2157                 goto local_input;
2158         }
2159
2160         if (!IN_DEV_FORWARD(in_dev))
2161                 goto e_hostunreach;
2162         if (res.type != RTN_UNICAST)
2163                 goto martian_destination;
2164
2165         err = ip_mkroute_input(skb, &res, &fl, in_dev, daddr, saddr, tos);
2166 out:    return err;
2167
2168 brd_input:
2169         if (skb->protocol != htons(ETH_P_IP))
2170                 goto e_inval;
2171
2172         if (ipv4_is_zeronet(saddr))
2173                 spec_dst = inet_select_addr(dev, 0, RT_SCOPE_LINK);
2174         else {
2175                 err = fib_validate_source(saddr, 0, tos, 0, dev, &spec_dst,
2176                                           &itag, skb->mark);
2177                 if (err < 0)
2178                         goto martian_source_keep_err;
2179                 if (err)
2180                         flags |= RTCF_DIRECTSRC;
2181         }
2182         flags |= RTCF_BROADCAST;
2183         res.type = RTN_BROADCAST;
2184         RT_CACHE_STAT_INC(in_brd);
2185
2186 local_input:
2187         rth = dst_alloc(&ipv4_dst_ops);
2188         if (!rth)
2189                 goto e_nobufs;
2190
2191         rth->dst.output= ip_rt_bug;
2192         rth->dst.obsolete = -1;
2193         rth->rt_genid = rt_genid(net);
2194
2195         atomic_set(&rth->dst.__refcnt, 1);
2196         rth->dst.flags= DST_HOST;
2197         if (IN_DEV_CONF_GET(in_dev, NOPOLICY))
2198                 rth->dst.flags |= DST_NOPOLICY;
2199         rth->fl.fl4_dst = daddr;
2200         rth->rt_dst     = daddr;
2201         rth->fl.fl4_tos = tos;
2202         rth->fl.mark    = skb->mark;
2203         rth->fl.fl4_src = saddr;
2204         rth->rt_src     = saddr;
2205 #ifdef CONFIG_IP_ROUTE_CLASSID
2206         rth->dst.tclassid = itag;
2207 #endif
2208         rth->rt_iif     =
2209         rth->fl.iif     = dev->ifindex;
2210         rth->dst.dev    = net->loopback_dev;
2211         dev_hold(rth->dst.dev);
2212         rth->rt_gateway = daddr;
2213         rth->rt_spec_dst= spec_dst;
2214         rth->dst.input= ip_local_deliver;
2215         rth->rt_flags   = flags|RTCF_LOCAL;
2216         if (res.type == RTN_UNREACHABLE) {
2217                 rth->dst.input= ip_error;
2218                 rth->dst.error= -err;
2219                 rth->rt_flags   &= ~RTCF_LOCAL;
2220         }
2221         rth->rt_type    = res.type;
2222         hash = rt_hash(daddr, saddr, fl.iif, rt_genid(net));
2223         err = rt_intern_hash(hash, rth, NULL, skb, fl.iif);
2224         goto out;
2225
2226 no_route:
2227         RT_CACHE_STAT_INC(in_no_route);
2228         spec_dst = inet_select_addr(dev, 0, RT_SCOPE_UNIVERSE);
2229         res.type = RTN_UNREACHABLE;
2230         if (err == -ESRCH)
2231                 err = -ENETUNREACH;
2232         goto local_input;
2233
2234         /*
2235          *      Do not cache martian addresses: they should be logged (RFC1812)
2236          */
2237 martian_destination:
2238         RT_CACHE_STAT_INC(in_martian_dst);
2239 #ifdef CONFIG_IP_ROUTE_VERBOSE
2240         if (IN_DEV_LOG_MARTIANS(in_dev) && net_ratelimit())
2241                 printk(KERN_WARNING "martian destination %pI4 from %pI4, dev %s\n",
2242                         &daddr, &saddr, dev->name);
2243 #endif
2244
2245 e_hostunreach:
2246         err = -EHOSTUNREACH;
2247         goto out;
2248
2249 e_inval:
2250         err = -EINVAL;
2251         goto out;
2252
2253 e_nobufs:
2254         err = -ENOBUFS;
2255         goto out;
2256
2257 martian_source:
2258         err = -EINVAL;
2259 martian_source_keep_err:
2260         ip_handle_martian_source(dev, in_dev, skb, daddr, saddr);
2261         goto out;
2262 }
2263
2264 int ip_route_input_common(struct sk_buff *skb, __be32 daddr, __be32 saddr,
2265                            u8 tos, struct net_device *dev, bool noref)
2266 {
2267         struct rtable * rth;
2268         unsigned        hash;
2269         int iif = dev->ifindex;
2270         struct net *net;
2271         int res;
2272
2273         net = dev_net(dev);
2274
2275         rcu_read_lock();
2276
2277         if (!rt_caching(net))
2278                 goto skip_cache;
2279
2280         tos &= IPTOS_RT_MASK;
2281         hash = rt_hash(daddr, saddr, iif, rt_genid(net));
2282
2283         for (rth = rcu_dereference(rt_hash_table[hash].chain); rth;
2284              rth = rcu_dereference(rth->dst.rt_next)) {
2285                 if ((((__force u32)rth->fl.fl4_dst ^ (__force u32)daddr) |
2286                      ((__force u32)rth->fl.fl4_src ^ (__force u32)saddr) |
2287                      (rth->fl.iif ^ iif) |
2288                      rth->fl.oif |
2289                      (rth->fl.fl4_tos ^ tos)) == 0 &&
2290                     rth->fl.mark == skb->mark &&
2291                     net_eq(dev_net(rth->dst.dev), net) &&
2292                     !rt_is_expired(rth)) {
2293                         if (noref) {
2294                                 dst_use_noref(&rth->dst, jiffies);
2295                                 skb_dst_set_noref(skb, &rth->dst);
2296                         } else {
2297                                 dst_use(&rth->dst, jiffies);
2298                                 skb_dst_set(skb, &rth->dst);
2299                         }
2300                         RT_CACHE_STAT_INC(in_hit);
2301                         rcu_read_unlock();
2302                         return 0;
2303                 }
2304                 RT_CACHE_STAT_INC(in_hlist_search);
2305         }
2306
2307 skip_cache:
2308         /* Multicast recognition logic is moved from route cache to here.
2309            The problem was that too many Ethernet cards have broken/missing
2310            hardware multicast filters :-( As result the host on multicasting
2311            network acquires a lot of useless route cache entries, sort of
2312            SDR messages from all the world. Now we try to get rid of them.
2313            Really, provided software IP multicast filter is organized
2314            reasonably (at least, hashed), it does not result in a slowdown
2315            comparing with route cache reject entries.
2316            Note, that multicast routers are not affected, because
2317            route cache entry is created eventually.
2318          */
2319         if (ipv4_is_multicast(daddr)) {
2320                 struct in_device *in_dev = __in_dev_get_rcu(dev);
2321
2322                 if (in_dev) {
2323                         int our = ip_check_mc(in_dev, daddr, saddr,
2324                                               ip_hdr(skb)->protocol);
2325                         if (our
2326 #ifdef CONFIG_IP_MROUTE
2327                                 ||
2328                             (!ipv4_is_local_multicast(daddr) &&
2329                              IN_DEV_MFORWARD(in_dev))
2330 #endif
2331                            ) {
2332                                 int res = ip_route_input_mc(skb, daddr, saddr,
2333                                                             tos, dev, our);
2334                                 rcu_read_unlock();
2335                                 return res;
2336                         }
2337                 }
2338                 rcu_read_unlock();
2339                 return -EINVAL;
2340         }
2341         res = ip_route_input_slow(skb, daddr, saddr, tos, dev);
2342         rcu_read_unlock();
2343         return res;
2344 }
2345 EXPORT_SYMBOL(ip_route_input_common);
2346
2347 /* called with rcu_read_lock() */
2348 static int __mkroute_output(struct rtable **result,
2349                             struct fib_result *res,
2350                             const struct flowi *fl,
2351                             const struct flowi *oldflp,
2352                             struct net_device *dev_out,
2353                             unsigned flags)
2354 {
2355         struct rtable *rth;
2356         struct in_device *in_dev;
2357         u32 tos = RT_FL_TOS(oldflp);
2358
2359         if (ipv4_is_loopback(fl->fl4_src) && !(dev_out->flags & IFF_LOOPBACK))
2360                 return -EINVAL;
2361
2362         if (ipv4_is_lbcast(fl->fl4_dst))
2363                 res->type = RTN_BROADCAST;
2364         else if (ipv4_is_multicast(fl->fl4_dst))
2365                 res->type = RTN_MULTICAST;
2366         else if (ipv4_is_zeronet(fl->fl4_dst))
2367                 return -EINVAL;
2368
2369         if (dev_out->flags & IFF_LOOPBACK)
2370                 flags |= RTCF_LOCAL;
2371
2372         in_dev = __in_dev_get_rcu(dev_out);
2373         if (!in_dev)
2374                 return -EINVAL;
2375
2376         if (res->type == RTN_BROADCAST) {
2377                 flags |= RTCF_BROADCAST | RTCF_LOCAL;
2378                 res->fi = NULL;
2379         } else if (res->type == RTN_MULTICAST) {
2380                 flags |= RTCF_MULTICAST | RTCF_LOCAL;
2381                 if (!ip_check_mc(in_dev, oldflp->fl4_dst, oldflp->fl4_src,
2382                                  oldflp->proto))
2383                         flags &= ~RTCF_LOCAL;
2384                 /* If multicast route do not exist use
2385                  * default one, but do not gateway in this case.
2386                  * Yes, it is hack.
2387                  */
2388                 if (res->fi && res->prefixlen < 4)
2389                         res->fi = NULL;
2390         }
2391
2392
2393         rth = dst_alloc(&ipv4_dst_ops);
2394         if (!rth)
2395                 return -ENOBUFS;
2396
2397         atomic_set(&rth->dst.__refcnt, 1);
2398         rth->dst.flags= DST_HOST;
2399         if (IN_DEV_CONF_GET(in_dev, NOXFRM))
2400                 rth->dst.flags |= DST_NOXFRM;
2401         if (IN_DEV_CONF_GET(in_dev, NOPOLICY))
2402                 rth->dst.flags |= DST_NOPOLICY;
2403
2404         rth->fl.fl4_dst = oldflp->fl4_dst;
2405         rth->fl.fl4_tos = tos;
2406         rth->fl.fl4_src = oldflp->fl4_src;
2407         rth->fl.oif     = oldflp->oif;
2408         rth->fl.mark    = oldflp->mark;
2409         rth->rt_dst     = fl->fl4_dst;
2410         rth->rt_src     = fl->fl4_src;
2411         rth->rt_iif     = oldflp->oif ? : dev_out->ifindex;
2412         /* get references to the devices that are to be hold by the routing
2413            cache entry */
2414         rth->dst.dev    = dev_out;
2415         dev_hold(dev_out);
2416         rth->rt_gateway = fl->fl4_dst;
2417         rth->rt_spec_dst= fl->fl4_src;
2418
2419         rth->dst.output=ip_output;
2420         rth->dst.obsolete = -1;
2421         rth->rt_genid = rt_genid(dev_net(dev_out));
2422
2423         RT_CACHE_STAT_INC(out_slow_tot);
2424
2425         if (flags & RTCF_LOCAL) {
2426                 rth->dst.input = ip_local_deliver;
2427                 rth->rt_spec_dst = fl->fl4_dst;
2428         }
2429         if (flags & (RTCF_BROADCAST | RTCF_MULTICAST)) {
2430                 rth->rt_spec_dst = fl->fl4_src;
2431                 if (flags & RTCF_LOCAL &&
2432                     !(dev_out->flags & IFF_LOOPBACK)) {
2433                         rth->dst.output = ip_mc_output;
2434                         RT_CACHE_STAT_INC(out_slow_mc);
2435                 }
2436 #ifdef CONFIG_IP_MROUTE
2437                 if (res->type == RTN_MULTICAST) {
2438                         if (IN_DEV_MFORWARD(in_dev) &&
2439                             !ipv4_is_local_multicast(oldflp->fl4_dst)) {
2440                                 rth->dst.input = ip_mr_input;
2441                                 rth->dst.output = ip_mc_output;
2442                         }
2443                 }
2444 #endif
2445         }
2446
2447         rt_set_nexthop(rth, res, 0);
2448
2449         rth->rt_flags = flags;
2450         *result = rth;
2451         return 0;
2452 }
2453
2454 /* called with rcu_read_lock() */
2455 static int ip_mkroute_output(struct rtable **rp,
2456                              struct fib_result *res,
2457                              const struct flowi *fl,
2458                              const struct flowi *oldflp,
2459                              struct net_device *dev_out,
2460                              unsigned flags)
2461 {
2462         struct rtable *rth = NULL;
2463         int err = __mkroute_output(&rth, res, fl, oldflp, dev_out, flags);
2464         unsigned hash;
2465         if (err == 0) {
2466                 hash = rt_hash(oldflp->fl4_dst, oldflp->fl4_src, oldflp->oif,
2467                                rt_genid(dev_net(dev_out)));
2468                 err = rt_intern_hash(hash, rth, rp, NULL, oldflp->oif);
2469         }
2470
2471         return err;
2472 }
2473
2474 /*
2475  * Major route resolver routine.
2476  * called with rcu_read_lock();
2477  */
2478
2479 static int ip_route_output_slow(struct net *net, struct rtable **rp,
2480                                 const struct flowi *oldflp)
2481 {
2482         u32 tos = RT_FL_TOS(oldflp);
2483         struct flowi fl = { .nl_u = { .ip4_u =
2484                                       { .daddr = oldflp->fl4_dst,
2485                                         .saddr = oldflp->fl4_src,
2486                                         .tos = tos & IPTOS_RT_MASK,
2487                                         .scope = ((tos & RTO_ONLINK) ?
2488                                                   RT_SCOPE_LINK :
2489                                                   RT_SCOPE_UNIVERSE),
2490                                       } },
2491                             .mark = oldflp->mark,
2492                             .iif = net->loopback_dev->ifindex,
2493                             .oif = oldflp->oif };
2494         struct fib_result res;
2495         unsigned int flags = 0;
2496         struct net_device *dev_out = NULL;
2497         int err;
2498
2499
2500         res.fi          = NULL;
2501 #ifdef CONFIG_IP_MULTIPLE_TABLES
2502         res.r           = NULL;
2503 #endif
2504
2505         if (oldflp->fl4_src) {
2506                 err = -EINVAL;
2507                 if (ipv4_is_multicast(oldflp->fl4_src) ||
2508                     ipv4_is_lbcast(oldflp->fl4_src) ||
2509                     ipv4_is_zeronet(oldflp->fl4_src))
2510                         goto out;
2511
2512                 /* I removed check for oif == dev_out->oif here.
2513                    It was wrong for two reasons:
2514                    1. ip_dev_find(net, saddr) can return wrong iface, if saddr
2515                       is assigned to multiple interfaces.
2516                    2. Moreover, we are allowed to send packets with saddr
2517                       of another iface. --ANK
2518                  */
2519
2520                 if (oldflp->oif == 0 &&
2521                     (ipv4_is_multicast(oldflp->fl4_dst) ||
2522                      ipv4_is_lbcast(oldflp->fl4_dst))) {
2523                         /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
2524                         dev_out = __ip_dev_find(net, oldflp->fl4_src, false);
2525                         if (dev_out == NULL)
2526                                 goto out;
2527
2528                         /* Special hack: user can direct multicasts
2529                            and limited broadcast via necessary interface
2530                            without fiddling with IP_MULTICAST_IF or IP_PKTINFO.
2531                            This hack is not just for fun, it allows
2532                            vic,vat and friends to work.
2533                            They bind socket to loopback, set ttl to zero
2534                            and expect that it will work.
2535                            From the viewpoint of routing cache they are broken,
2536                            because we are not allowed to build multicast path
2537                            with loopback source addr (look, routing cache
2538                            cannot know, that ttl is zero, so that packet
2539                            will not leave this host and route is valid).
2540                            Luckily, this hack is good workaround.
2541                          */
2542
2543                         fl.oif = dev_out->ifindex;
2544                         goto make_route;
2545                 }
2546
2547                 if (!(oldflp->flags & FLOWI_FLAG_ANYSRC)) {
2548                         /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
2549                         if (!__ip_dev_find(net, oldflp->fl4_src, false))
2550                                 goto out;
2551                 }
2552         }
2553
2554
2555         if (oldflp->oif) {
2556                 dev_out = dev_get_by_index_rcu(net, oldflp->oif);
2557                 err = -ENODEV;
2558                 if (dev_out == NULL)
2559                         goto out;
2560
2561                 /* RACE: Check return value of inet_select_addr instead. */
2562                 if (rcu_dereference(dev_out->ip_ptr) == NULL)
2563                         goto out;       /* Wrong error code */
2564
2565                 if (ipv4_is_local_multicast(oldflp->fl4_dst) ||
2566                     ipv4_is_lbcast(oldflp->fl4_dst)) {
2567                         if (!fl.fl4_src)
2568                                 fl.fl4_src = inet_select_addr(dev_out, 0,
2569                                                               RT_SCOPE_LINK);
2570                         goto make_route;
2571                 }
2572                 if (!fl.fl4_src) {
2573                         if (ipv4_is_multicast(oldflp->fl4_dst))
2574                                 fl.fl4_src = inet_select_addr(dev_out, 0,
2575                                                               fl.fl4_scope);
2576                         else if (!oldflp->fl4_dst)
2577                                 fl.fl4_src = inet_select_addr(dev_out, 0,
2578                                                               RT_SCOPE_HOST);
2579                 }
2580         }
2581
2582         if (!fl.fl4_dst) {
2583                 fl.fl4_dst = fl.fl4_src;
2584                 if (!fl.fl4_dst)
2585                         fl.fl4_dst = fl.fl4_src = htonl(INADDR_LOOPBACK);
2586                 dev_out = net->loopback_dev;
2587                 fl.oif = net->loopback_dev->ifindex;
2588                 res.type = RTN_LOCAL;
2589                 flags |= RTCF_LOCAL;
2590                 goto make_route;
2591         }
2592
2593         if (fib_lookup(net, &fl, &res)) {
2594                 res.fi = NULL;
2595                 if (oldflp->oif) {
2596                         /* Apparently, routing tables are wrong. Assume,
2597                            that the destination is on link.
2598
2599                            WHY? DW.
2600                            Because we are allowed to send to iface
2601                            even if it has NO routes and NO assigned
2602                            addresses. When oif is specified, routing
2603                            tables are looked up with only one purpose:
2604                            to catch if destination is gatewayed, rather than
2605                            direct. Moreover, if MSG_DONTROUTE is set,
2606                            we send packet, ignoring both routing tables
2607                            and ifaddr state. --ANK
2608
2609
2610                            We could make it even if oif is unknown,
2611                            likely IPv6, but we do not.
2612                          */
2613
2614                         if (fl.fl4_src == 0)
2615                                 fl.fl4_src = inet_select_addr(dev_out, 0,
2616                                                               RT_SCOPE_LINK);
2617                         res.type = RTN_UNICAST;
2618                         goto make_route;
2619                 }
2620                 err = -ENETUNREACH;
2621                 goto out;
2622         }
2623
2624         if (res.type == RTN_LOCAL) {
2625                 if (!fl.fl4_src)
2626                         fl.fl4_src = fl.fl4_dst;
2627                 dev_out = net->loopback_dev;
2628                 fl.oif = dev_out->ifindex;
2629                 res.fi = NULL;
2630                 flags |= RTCF_LOCAL;
2631                 goto make_route;
2632         }
2633
2634 #ifdef CONFIG_IP_ROUTE_MULTIPATH
2635         if (res.fi->fib_nhs > 1 && fl.oif == 0)
2636                 fib_select_multipath(&fl, &res);
2637         else
2638 #endif
2639         if (!res.prefixlen && res.type == RTN_UNICAST && !fl.oif)
2640                 fib_select_default(net, &fl, &res);
2641
2642         if (!fl.fl4_src)
2643                 fl.fl4_src = FIB_RES_PREFSRC(res);
2644
2645         dev_out = FIB_RES_DEV(res);
2646         fl.oif = dev_out->ifindex;
2647
2648
2649 make_route:
2650         err = ip_mkroute_output(rp, &res, &fl, oldflp, dev_out, flags);
2651
2652 out:    return err;
2653 }
2654
2655 int __ip_route_output_key(struct net *net, struct rtable **rp,
2656                           const struct flowi *flp)
2657 {
2658         unsigned int hash;
2659         int res;
2660         struct rtable *rth;
2661
2662         if (!rt_caching(net))
2663                 goto slow_output;
2664
2665         hash = rt_hash(flp->fl4_dst, flp->fl4_src, flp->oif, rt_genid(net));
2666
2667         rcu_read_lock_bh();
2668         for (rth = rcu_dereference_bh(rt_hash_table[hash].chain); rth;
2669                 rth = rcu_dereference_bh(rth->dst.rt_next)) {
2670                 if (rth->fl.fl4_dst == flp->fl4_dst &&
2671                     rth->fl.fl4_src == flp->fl4_src &&
2672                     rt_is_output_route(rth) &&
2673                     rth->fl.oif == flp->oif &&
2674                     rth->fl.mark == flp->mark &&
2675                     !((rth->fl.fl4_tos ^ flp->fl4_tos) &
2676                             (IPTOS_RT_MASK | RTO_ONLINK)) &&
2677                     net_eq(dev_net(rth->dst.dev), net) &&
2678                     !rt_is_expired(rth)) {
2679                         dst_use(&rth->dst, jiffies);
2680                         RT_CACHE_STAT_INC(out_hit);
2681                         rcu_read_unlock_bh();
2682                         *rp = rth;
2683                         return 0;
2684                 }
2685                 RT_CACHE_STAT_INC(out_hlist_search);
2686         }
2687         rcu_read_unlock_bh();
2688
2689 slow_output:
2690         rcu_read_lock();
2691         res = ip_route_output_slow(net, rp, flp);
2692         rcu_read_unlock();
2693         return res;
2694 }
2695 EXPORT_SYMBOL_GPL(__ip_route_output_key);
2696
2697 static struct dst_entry *ipv4_blackhole_dst_check(struct dst_entry *dst, u32 cookie)
2698 {
2699         return NULL;
2700 }
2701
2702 static void ipv4_rt_blackhole_update_pmtu(struct dst_entry *dst, u32 mtu)
2703 {
2704 }
2705
2706 static struct dst_ops ipv4_dst_blackhole_ops = {
2707         .family                 =       AF_INET,
2708         .protocol               =       cpu_to_be16(ETH_P_IP),
2709         .destroy                =       ipv4_dst_destroy,
2710         .check                  =       ipv4_blackhole_dst_check,
2711         .update_pmtu            =       ipv4_rt_blackhole_update_pmtu,
2712 };
2713
2714
2715 static int ipv4_dst_blackhole(struct net *net, struct rtable **rp, struct flowi *flp)
2716 {
2717         struct rtable *ort = *rp;
2718         struct rtable *rt = (struct rtable *)
2719                 dst_alloc(&ipv4_dst_blackhole_ops);
2720
2721         if (rt) {
2722                 struct dst_entry *new = &rt->dst;
2723
2724                 atomic_set(&new->__refcnt, 1);
2725                 new->__use = 1;
2726                 new->input = dst_discard;
2727                 new->output = dst_discard;
2728                 memcpy(new->metrics, ort->dst.metrics, RTAX_MAX*sizeof(u32));
2729
2730                 new->dev = ort->dst.dev;
2731                 if (new->dev)
2732                         dev_hold(new->dev);
2733
2734                 rt->fl = ort->fl;
2735
2736                 rt->rt_genid = rt_genid(net);
2737                 rt->rt_flags = ort->rt_flags;
2738                 rt->rt_type = ort->rt_type;
2739                 rt->rt_dst = ort->rt_dst;
2740                 rt->rt_src = ort->rt_src;
2741                 rt->rt_iif = ort->rt_iif;
2742                 rt->rt_gateway = ort->rt_gateway;
2743                 rt->rt_spec_dst = ort->rt_spec_dst;
2744                 rt->peer = ort->peer;
2745                 if (rt->peer)
2746                         atomic_inc(&rt->peer->refcnt);
2747
2748                 dst_free(new);
2749         }
2750
2751         dst_release(&(*rp)->dst);
2752         *rp = rt;
2753         return rt ? 0 : -ENOMEM;
2754 }
2755
2756 int ip_route_output_flow(struct net *net, struct rtable **rp, struct flowi *flp,
2757                          struct sock *sk, int flags)
2758 {
2759         int err;
2760
2761         if ((err = __ip_route_output_key(net, rp, flp)) != 0)
2762                 return err;
2763
2764         if (flp->proto) {
2765                 if (!flp->fl4_src)
2766                         flp->fl4_src = (*rp)->rt_src;
2767                 if (!flp->fl4_dst)
2768                         flp->fl4_dst = (*rp)->rt_dst;
2769                 err = __xfrm_lookup(net, (struct dst_entry **)rp, flp, sk,
2770                                     flags ? XFRM_LOOKUP_WAIT : 0);
2771                 if (err == -EREMOTE)
2772                         err = ipv4_dst_blackhole(net, rp, flp);
2773
2774                 return err;
2775         }
2776
2777         return 0;
2778 }
2779 EXPORT_SYMBOL_GPL(ip_route_output_flow);
2780
2781 int ip_route_output_key(struct net *net, struct rtable **rp, struct flowi *flp)
2782 {
2783         return ip_route_output_flow(net, rp, flp, NULL, 0);
2784 }
2785 EXPORT_SYMBOL(ip_route_output_key);
2786
2787 static int rt_fill_info(struct net *net,
2788                         struct sk_buff *skb, u32 pid, u32 seq, int event,
2789                         int nowait, unsigned int flags)
2790 {
2791         struct rtable *rt = skb_rtable(skb);
2792         struct rtmsg *r;
2793         struct nlmsghdr *nlh;
2794         long expires;
2795         u32 id = 0, ts = 0, tsage = 0, error;
2796
2797         nlh = nlmsg_put(skb, pid, seq, event, sizeof(*r), flags);
2798         if (nlh == NULL)
2799                 return -EMSGSIZE;
2800
2801         r = nlmsg_data(nlh);
2802         r->rtm_family    = AF_INET;
2803         r->rtm_dst_len  = 32;
2804         r->rtm_src_len  = 0;
2805         r->rtm_tos      = rt->fl.fl4_tos;
2806         r->rtm_table    = RT_TABLE_MAIN;
2807         NLA_PUT_U32(skb, RTA_TABLE, RT_TABLE_MAIN);
2808         r->rtm_type     = rt->rt_type;
2809         r->rtm_scope    = RT_SCOPE_UNIVERSE;
2810         r->rtm_protocol = RTPROT_UNSPEC;
2811         r->rtm_flags    = (rt->rt_flags & ~0xFFFF) | RTM_F_CLONED;
2812         if (rt->rt_flags & RTCF_NOTIFY)
2813                 r->rtm_flags |= RTM_F_NOTIFY;
2814
2815         NLA_PUT_BE32(skb, RTA_DST, rt->rt_dst);
2816
2817         if (rt->fl.fl4_src) {
2818                 r->rtm_src_len = 32;
2819                 NLA_PUT_BE32(skb, RTA_SRC, rt->fl.fl4_src);
2820         }
2821         if (rt->dst.dev)
2822                 NLA_PUT_U32(skb, RTA_OIF, rt->dst.dev->ifindex);
2823 #ifdef CONFIG_IP_ROUTE_CLASSID
2824         if (rt->dst.tclassid)
2825                 NLA_PUT_U32(skb, RTA_FLOW, rt->dst.tclassid);
2826 #endif
2827         if (rt_is_input_route(rt))
2828                 NLA_PUT_BE32(skb, RTA_PREFSRC, rt->rt_spec_dst);
2829         else if (rt->rt_src != rt->fl.fl4_src)
2830                 NLA_PUT_BE32(skb, RTA_PREFSRC, rt->rt_src);
2831
2832         if (rt->rt_dst != rt->rt_gateway)
2833                 NLA_PUT_BE32(skb, RTA_GATEWAY, rt->rt_gateway);
2834
2835         if (rtnetlink_put_metrics(skb, rt->dst.metrics) < 0)
2836                 goto nla_put_failure;
2837
2838         if (rt->fl.mark)
2839                 NLA_PUT_BE32(skb, RTA_MARK, rt->fl.mark);
2840
2841         error = rt->dst.error;
2842         expires = rt->dst.expires ? rt->dst.expires - jiffies : 0;
2843         if (rt->peer) {
2844                 inet_peer_refcheck(rt->peer);
2845                 id = atomic_read(&rt->peer->ip_id_count) & 0xffff;
2846                 if (rt->peer->tcp_ts_stamp) {
2847                         ts = rt->peer->tcp_ts;
2848                         tsage = get_seconds() - rt->peer->tcp_ts_stamp;
2849                 }
2850         }
2851
2852         if (rt_is_input_route(rt)) {
2853 #ifdef CONFIG_IP_MROUTE
2854                 __be32 dst = rt->rt_dst;
2855
2856                 if (ipv4_is_multicast(dst) && !ipv4_is_local_multicast(dst) &&
2857                     IPV4_DEVCONF_ALL(net, MC_FORWARDING)) {
2858                         int err = ipmr_get_route(net, skb, r, nowait);
2859                         if (err <= 0) {
2860                                 if (!nowait) {
2861                                         if (err == 0)
2862                                                 return 0;
2863                                         goto nla_put_failure;
2864                                 } else {
2865                                         if (err == -EMSGSIZE)
2866                                                 goto nla_put_failure;
2867                                         error = err;
2868                                 }
2869                         }
2870                 } else
2871 #endif
2872                         NLA_PUT_U32(skb, RTA_IIF, rt->fl.iif);
2873         }
2874
2875         if (rtnl_put_cacheinfo(skb, &rt->dst, id, ts, tsage,
2876                                expires, error) < 0)
2877                 goto nla_put_failure;
2878
2879         return nlmsg_end(skb, nlh);
2880
2881 nla_put_failure:
2882         nlmsg_cancel(skb, nlh);
2883         return -EMSGSIZE;
2884 }
2885
2886 static int inet_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr* nlh, void *arg)
2887 {
2888         struct net *net = sock_net(in_skb->sk);
2889         struct rtmsg *rtm;
2890         struct nlattr *tb[RTA_MAX+1];
2891         struct rtable *rt = NULL;
2892         __be32 dst = 0;
2893         __be32 src = 0;
2894         u32 iif;
2895         int err;
2896         int mark;
2897         struct sk_buff *skb;
2898
2899         err = nlmsg_parse(nlh, sizeof(*rtm), tb, RTA_MAX, rtm_ipv4_policy);
2900         if (err < 0)
2901                 goto errout;
2902
2903         rtm = nlmsg_data(nlh);
2904
2905         skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
2906         if (skb == NULL) {
2907                 err = -ENOBUFS;
2908                 goto errout;
2909         }
2910
2911         /* Reserve room for dummy headers, this skb can pass
2912            through good chunk of routing engine.
2913          */
2914         skb_reset_mac_header(skb);
2915         skb_reset_network_header(skb);
2916
2917         /* Bugfix: need to give ip_route_input enough of an IP header to not gag. */
2918         ip_hdr(skb)->protocol = IPPROTO_ICMP;
2919         skb_reserve(skb, MAX_HEADER + sizeof(struct iphdr));
2920
2921         src = tb[RTA_SRC] ? nla_get_be32(tb[RTA_SRC]) : 0;
2922         dst = tb[RTA_DST] ? nla_get_be32(tb[RTA_DST]) : 0;
2923         iif = tb[RTA_IIF] ? nla_get_u32(tb[RTA_IIF]) : 0;
2924         mark = tb[RTA_MARK] ? nla_get_u32(tb[RTA_MARK]) : 0;
2925
2926         if (iif) {
2927                 struct net_device *dev;
2928
2929                 dev = __dev_get_by_index(net, iif);
2930                 if (dev == NULL) {
2931                         err = -ENODEV;
2932                         goto errout_free;
2933                 }
2934
2935                 skb->protocol   = htons(ETH_P_IP);
2936                 skb->dev        = dev;
2937                 skb->mark       = mark;
2938                 local_bh_disable();
2939                 err = ip_route_input(skb, dst, src, rtm->rtm_tos, dev);
2940                 local_bh_enable();
2941
2942                 rt = skb_rtable(skb);
2943                 if (err == 0 && rt->dst.error)
2944                         err = -rt->dst.error;
2945         } else {
2946                 struct flowi fl = {
2947                         .nl_u = {
2948                                 .ip4_u = {
2949                                         .daddr = dst,
2950                                         .saddr = src,
2951                                         .tos = rtm->rtm_tos,
2952                                 },
2953                         },
2954                         .oif = tb[RTA_OIF] ? nla_get_u32(tb[RTA_OIF]) : 0,
2955                         .mark = mark,
2956                 };
2957                 err = ip_route_output_key(net, &rt, &fl);
2958         }
2959
2960         if (err)
2961                 goto errout_free;
2962
2963         skb_dst_set(skb, &rt->dst);
2964         if (rtm->rtm_flags & RTM_F_NOTIFY)
2965                 rt->rt_flags |= RTCF_NOTIFY;
2966
2967         err = rt_fill_info(net, skb, NETLINK_CB(in_skb).pid, nlh->nlmsg_seq,
2968                            RTM_NEWROUTE, 0, 0);
2969         if (err <= 0)
2970                 goto errout_free;
2971
2972         err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).pid);
2973 errout:
2974         return err;
2975
2976 errout_free:
2977         kfree_skb(skb);
2978         goto errout;
2979 }
2980
2981 int ip_rt_dump(struct sk_buff *skb,  struct netlink_callback *cb)
2982 {
2983         struct rtable *rt;
2984         int h, s_h;
2985         int idx, s_idx;
2986         struct net *net;
2987
2988         net = sock_net(skb->sk);
2989
2990         s_h = cb->args[0];
2991         if (s_h < 0)
2992                 s_h = 0;
2993         s_idx = idx = cb->args[1];
2994         for (h = s_h; h <= rt_hash_mask; h++, s_idx = 0) {
2995                 if (!rt_hash_table[h].chain)
2996                         continue;
2997                 rcu_read_lock_bh();
2998                 for (rt = rcu_dereference_bh(rt_hash_table[h].chain), idx = 0; rt;
2999                      rt = rcu_dereference_bh(rt->dst.rt_next), idx++) {
3000                         if (!net_eq(dev_net(rt->dst.dev), net) || idx < s_idx)
3001                                 continue;
3002                         if (rt_is_expired(rt))
3003                                 continue;
3004                         skb_dst_set_noref(skb, &rt->dst);
3005                         if (rt_fill_info(net, skb, NETLINK_CB(cb->skb).pid,
3006                                          cb->nlh->nlmsg_seq, RTM_NEWROUTE,
3007                                          1, NLM_F_MULTI) <= 0) {
3008                                 skb_dst_drop(skb);
3009                                 rcu_read_unlock_bh();
3010                                 goto done;
3011                         }
3012                         skb_dst_drop(skb);
3013                 }
3014                 rcu_read_unlock_bh();
3015         }
3016
3017 done:
3018         cb->args[0] = h;
3019         cb->args[1] = idx;
3020         return skb->len;
3021 }
3022
3023 void ip_rt_multicast_event(struct in_device *in_dev)
3024 {
3025         rt_cache_flush(dev_net(in_dev->dev), 0);
3026 }
3027
3028 #ifdef CONFIG_SYSCTL
3029 static int ipv4_sysctl_rtcache_flush(ctl_table *__ctl, int write,
3030                                         void __user *buffer,
3031                                         size_t *lenp, loff_t *ppos)
3032 {
3033         if (write) {
3034                 int flush_delay;
3035                 ctl_table ctl;
3036                 struct net *net;
3037
3038                 memcpy(&ctl, __ctl, sizeof(ctl));
3039                 ctl.data = &flush_delay;
3040                 proc_dointvec(&ctl, write, buffer, lenp, ppos);
3041
3042                 net = (struct net *)__ctl->extra1;
3043                 rt_cache_flush(net, flush_delay);
3044                 return 0;
3045         }
3046
3047         return -EINVAL;
3048 }
3049
3050 static ctl_table ipv4_route_table[] = {
3051         {
3052                 .procname       = "gc_thresh",
3053                 .data           = &ipv4_dst_ops.gc_thresh,
3054                 .maxlen         = sizeof(int),
3055                 .mode           = 0644,
3056                 .proc_handler   = proc_dointvec,
3057         },
3058         {
3059                 .procname       = "max_size",
3060                 .data           = &ip_rt_max_size,
3061                 .maxlen         = sizeof(int),
3062                 .mode           = 0644,
3063                 .proc_handler   = proc_dointvec,
3064         },
3065         {
3066                 /*  Deprecated. Use gc_min_interval_ms */
3067
3068                 .procname       = "gc_min_interval",
3069                 .data           = &ip_rt_gc_min_interval,
3070                 .maxlen         = sizeof(int),
3071                 .mode           = 0644,
3072                 .proc_handler   = proc_dointvec_jiffies,
3073         },
3074         {
3075                 .procname       = "gc_min_interval_ms",
3076                 .data           = &ip_rt_gc_min_interval,
3077                 .maxlen         = sizeof(int),
3078                 .mode           = 0644,
3079                 .proc_handler   = proc_dointvec_ms_jiffies,
3080         },
3081         {
3082                 .procname       = "gc_timeout",
3083                 .data           = &ip_rt_gc_timeout,
3084                 .maxlen         = sizeof(int),
3085                 .mode           = 0644,
3086                 .proc_handler   = proc_dointvec_jiffies,
3087         },
3088         {
3089                 .procname       = "gc_interval",
3090                 .data           = &ip_rt_gc_interval,
3091                 .maxlen         = sizeof(int),
3092                 .mode           = 0644,
3093                 .proc_handler   = proc_dointvec_jiffies,
3094         },
3095         {
3096                 .procname       = "redirect_load",
3097                 .data           = &ip_rt_redirect_load,
3098                 .maxlen         = sizeof(int),
3099                 .mode           = 0644,
3100                 .proc_handler   = proc_dointvec,
3101         },
3102         {
3103                 .procname       = "redirect_number",
3104                 .data           = &ip_rt_redirect_number,
3105                 .maxlen         = sizeof(int),
3106                 .mode           = 0644,
3107                 .proc_handler   = proc_dointvec,
3108         },
3109         {
3110                 .procname       = "redirect_silence",
3111                 .data           = &ip_rt_redirect_silence,
3112                 .maxlen         = sizeof(int),
3113                 .mode           = 0644,
3114                 .proc_handler   = proc_dointvec,
3115         },
3116         {
3117                 .procname       = "error_cost",
3118                 .data           = &ip_rt_error_cost,
3119                 .maxlen         = sizeof(int),
3120                 .mode           = 0644,
3121                 .proc_handler   = proc_dointvec,
3122         },
3123         {
3124                 .procname       = "error_burst",
3125                 .data           = &ip_rt_error_burst,
3126                 .maxlen         = sizeof(int),
3127                 .mode           = 0644,
3128                 .proc_handler   = proc_dointvec,
3129         },
3130         {
3131                 .procname       = "gc_elasticity",
3132                 .data           = &ip_rt_gc_elasticity,
3133                 .maxlen         = sizeof(int),
3134                 .mode           = 0644,
3135                 .proc_handler   = proc_dointvec,
3136         },
3137         {
3138                 .procname       = "mtu_expires",
3139                 .data           = &ip_rt_mtu_expires,
3140                 .maxlen         = sizeof(int),
3141                 .mode           = 0644,
3142                 .proc_handler   = proc_dointvec_jiffies,
3143         },
3144         {
3145                 .procname       = "min_pmtu",
3146                 .data           = &ip_rt_min_pmtu,
3147                 .maxlen         = sizeof(int),
3148                 .mode           = 0644,
3149                 .proc_handler   = proc_dointvec,
3150         },
3151         {
3152                 .procname       = "min_adv_mss",
3153                 .data           = &ip_rt_min_advmss,
3154                 .maxlen         = sizeof(int),
3155                 .mode           = 0644,
3156                 .proc_handler   = proc_dointvec,
3157         },
3158         { }
3159 };
3160
3161 static struct ctl_table empty[1];
3162
3163 static struct ctl_table ipv4_skeleton[] =
3164 {
3165         { .procname = "route", 
3166           .mode = 0555, .child = ipv4_route_table},
3167         { .procname = "neigh", 
3168           .mode = 0555, .child = empty},
3169         { }
3170 };
3171
3172 static __net_initdata struct ctl_path ipv4_path[] = {
3173         { .procname = "net", },
3174         { .procname = "ipv4", },
3175         { },
3176 };
3177
3178 static struct ctl_table ipv4_route_flush_table[] = {
3179         {
3180                 .procname       = "flush",
3181                 .maxlen         = sizeof(int),
3182                 .mode           = 0200,
3183                 .proc_handler   = ipv4_sysctl_rtcache_flush,
3184         },
3185         { },
3186 };
3187
3188 static __net_initdata struct ctl_path ipv4_route_path[] = {
3189         { .procname = "net", },
3190         { .procname = "ipv4", },
3191         { .procname = "route", },
3192         { },
3193 };
3194
3195 static __net_init int sysctl_route_net_init(struct net *net)
3196 {
3197         struct ctl_table *tbl;
3198
3199         tbl = ipv4_route_flush_table;
3200         if (!net_eq(net, &init_net)) {
3201                 tbl = kmemdup(tbl, sizeof(ipv4_route_flush_table), GFP_KERNEL);
3202                 if (tbl == NULL)
3203                         goto err_dup;
3204         }
3205         tbl[0].extra1 = net;
3206
3207         net->ipv4.route_hdr =
3208                 register_net_sysctl_table(net, ipv4_route_path, tbl);
3209         if (net->ipv4.route_hdr == NULL)
3210                 goto err_reg;
3211         return 0;
3212
3213 err_reg:
3214         if (tbl != ipv4_route_flush_table)
3215                 kfree(tbl);
3216 err_dup:
3217         return -ENOMEM;
3218 }
3219
3220 static __net_exit void sysctl_route_net_exit(struct net *net)
3221 {
3222         struct ctl_table *tbl;
3223
3224         tbl = net->ipv4.route_hdr->ctl_table_arg;
3225         unregister_net_sysctl_table(net->ipv4.route_hdr);
3226         BUG_ON(tbl == ipv4_route_flush_table);
3227         kfree(tbl);
3228 }
3229
3230 static __net_initdata struct pernet_operations sysctl_route_ops = {
3231         .init = sysctl_route_net_init,
3232         .exit = sysctl_route_net_exit,
3233 };
3234 #endif
3235
3236 static __net_init int rt_genid_init(struct net *net)
3237 {
3238         get_random_bytes(&net->ipv4.rt_genid,
3239                          sizeof(net->ipv4.rt_genid));
3240         return 0;
3241 }
3242
3243 static __net_initdata struct pernet_operations rt_genid_ops = {
3244         .init = rt_genid_init,
3245 };
3246
3247
3248 #ifdef CONFIG_IP_ROUTE_CLASSID
3249 struct ip_rt_acct __percpu *ip_rt_acct __read_mostly;
3250 #endif /* CONFIG_IP_ROUTE_CLASSID */
3251
3252 static __initdata unsigned long rhash_entries;
3253 static int __init set_rhash_entries(char *str)
3254 {
3255         if (!str)
3256                 return 0;
3257         rhash_entries = simple_strtoul(str, &str, 0);
3258         return 1;
3259 }
3260 __setup("rhash_entries=", set_rhash_entries);
3261
3262 int __init ip_rt_init(void)
3263 {
3264         int rc = 0;
3265
3266 #ifdef CONFIG_IP_ROUTE_CLASSID
3267         ip_rt_acct = __alloc_percpu(256 * sizeof(struct ip_rt_acct), __alignof__(struct ip_rt_acct));
3268         if (!ip_rt_acct)
3269                 panic("IP: failed to allocate ip_rt_acct\n");
3270 #endif
3271
3272         ipv4_dst_ops.kmem_cachep =
3273                 kmem_cache_create("ip_dst_cache", sizeof(struct rtable), 0,
3274                                   SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
3275
3276         ipv4_dst_blackhole_ops.kmem_cachep = ipv4_dst_ops.kmem_cachep;
3277
3278         if (dst_entries_init(&ipv4_dst_ops) < 0)
3279                 panic("IP: failed to allocate ipv4_dst_ops counter\n");
3280
3281         if (dst_entries_init(&ipv4_dst_blackhole_ops) < 0)
3282                 panic("IP: failed to allocate ipv4_dst_blackhole_ops counter\n");
3283
3284         rt_hash_table = (struct rt_hash_bucket *)
3285                 alloc_large_system_hash("IP route cache",
3286                                         sizeof(struct rt_hash_bucket),
3287                                         rhash_entries,
3288                                         (totalram_pages >= 128 * 1024) ?
3289                                         15 : 17,
3290                                         0,
3291                                         &rt_hash_log,
3292                                         &rt_hash_mask,
3293                                         rhash_entries ? 0 : 512 * 1024);
3294         memset(rt_hash_table, 0, (rt_hash_mask + 1) * sizeof(struct rt_hash_bucket));
3295         rt_hash_lock_init();
3296
3297         ipv4_dst_ops.gc_thresh = (rt_hash_mask + 1);
3298         ip_rt_max_size = (rt_hash_mask + 1) * 16;
3299
3300         devinet_init();
3301         ip_fib_init();
3302
3303         /* All the timers, started at system startup tend
3304            to synchronize. Perturb it a bit.
3305          */
3306         INIT_DELAYED_WORK_DEFERRABLE(&expires_work, rt_worker_func);
3307         expires_ljiffies = jiffies;
3308         schedule_delayed_work(&expires_work,
3309                 net_random() % ip_rt_gc_interval + ip_rt_gc_interval);
3310
3311         if (ip_rt_proc_init())
3312                 printk(KERN_ERR "Unable to create route proc files\n");
3313 #ifdef CONFIG_XFRM
3314         xfrm_init();
3315         xfrm4_init(ip_rt_max_size);
3316 #endif
3317         rtnl_register(PF_INET, RTM_GETROUTE, inet_rtm_getroute, NULL);
3318
3319 #ifdef CONFIG_SYSCTL
3320         register_pernet_subsys(&sysctl_route_ops);
3321 #endif
3322         register_pernet_subsys(&rt_genid_ops);
3323         return rc;
3324 }
3325
3326 #ifdef CONFIG_SYSCTL
3327 /*
3328  * We really need to sanitize the damn ipv4 init order, then all
3329  * this nonsense will go away.
3330  */
3331 void __init ip_static_sysctl_init(void)
3332 {
3333         register_sysctl_paths(ipv4_path, ipv4_skeleton);
3334 }
3335 #endif