ipv4: Make output route lookup return rtable directly.
[linux-2.6.git] / net / ipv4 / route.c
1 /*
2  * INET         An implementation of the TCP/IP protocol suite for the LINUX
3  *              operating system.  INET is implemented using the  BSD Socket
4  *              interface as the means of communication with the user level.
5  *
6  *              ROUTE - implementation of the IP router.
7  *
8  * Authors:     Ross Biro
9  *              Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10  *              Alan Cox, <gw4pts@gw4pts.ampr.org>
11  *              Linus Torvalds, <Linus.Torvalds@helsinki.fi>
12  *              Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
13  *
14  * Fixes:
15  *              Alan Cox        :       Verify area fixes.
16  *              Alan Cox        :       cli() protects routing changes
17  *              Rui Oliveira    :       ICMP routing table updates
18  *              (rco@di.uminho.pt)      Routing table insertion and update
19  *              Linus Torvalds  :       Rewrote bits to be sensible
20  *              Alan Cox        :       Added BSD route gw semantics
21  *              Alan Cox        :       Super /proc >4K
22  *              Alan Cox        :       MTU in route table
23  *              Alan Cox        :       MSS actually. Also added the window
24  *                                      clamper.
25  *              Sam Lantinga    :       Fixed route matching in rt_del()
26  *              Alan Cox        :       Routing cache support.
27  *              Alan Cox        :       Removed compatibility cruft.
28  *              Alan Cox        :       RTF_REJECT support.
29  *              Alan Cox        :       TCP irtt support.
30  *              Jonathan Naylor :       Added Metric support.
31  *      Miquel van Smoorenburg  :       BSD API fixes.
32  *      Miquel van Smoorenburg  :       Metrics.
33  *              Alan Cox        :       Use __u32 properly
34  *              Alan Cox        :       Aligned routing errors more closely with BSD
35  *                                      our system is still very different.
36  *              Alan Cox        :       Faster /proc handling
37  *      Alexey Kuznetsov        :       Massive rework to support tree based routing,
38  *                                      routing caches and better behaviour.
39  *
40  *              Olaf Erb        :       irtt wasn't being copied right.
41  *              Bjorn Ekwall    :       Kerneld route support.
42  *              Alan Cox        :       Multicast fixed (I hope)
43  *              Pavel Krauz     :       Limited broadcast fixed
44  *              Mike McLagan    :       Routing by source
45  *      Alexey Kuznetsov        :       End of old history. Split to fib.c and
46  *                                      route.c and rewritten from scratch.
47  *              Andi Kleen      :       Load-limit warning messages.
48  *      Vitaly E. Lavrov        :       Transparent proxy revived after year coma.
49  *      Vitaly E. Lavrov        :       Race condition in ip_route_input_slow.
50  *      Tobias Ringstrom        :       Uninitialized res.type in ip_route_output_slow.
51  *      Vladimir V. Ivanov      :       IP rule info (flowid) is really useful.
52  *              Marc Boucher    :       routing by fwmark
53  *      Robert Olsson           :       Added rt_cache statistics
54  *      Arnaldo C. Melo         :       Convert proc stuff to seq_file
55  *      Eric Dumazet            :       hashed spinlocks and rt_check_expire() fixes.
56  *      Ilia Sotnikov           :       Ignore TOS on PMTUD and Redirect
57  *      Ilia Sotnikov           :       Removed TOS from hash calculations
58  *
59  *              This program is free software; you can redistribute it and/or
60  *              modify it under the terms of the GNU General Public License
61  *              as published by the Free Software Foundation; either version
62  *              2 of the License, or (at your option) any later version.
63  */
64
65 #include <linux/module.h>
66 #include <asm/uaccess.h>
67 #include <asm/system.h>
68 #include <linux/bitops.h>
69 #include <linux/types.h>
70 #include <linux/kernel.h>
71 #include <linux/mm.h>
72 #include <linux/bootmem.h>
73 #include <linux/string.h>
74 #include <linux/socket.h>
75 #include <linux/sockios.h>
76 #include <linux/errno.h>
77 #include <linux/in.h>
78 #include <linux/inet.h>
79 #include <linux/netdevice.h>
80 #include <linux/proc_fs.h>
81 #include <linux/init.h>
82 #include <linux/workqueue.h>
83 #include <linux/skbuff.h>
84 #include <linux/inetdevice.h>
85 #include <linux/igmp.h>
86 #include <linux/pkt_sched.h>
87 #include <linux/mroute.h>
88 #include <linux/netfilter_ipv4.h>
89 #include <linux/random.h>
90 #include <linux/jhash.h>
91 #include <linux/rcupdate.h>
92 #include <linux/times.h>
93 #include <linux/slab.h>
94 #include <net/dst.h>
95 #include <net/net_namespace.h>
96 #include <net/protocol.h>
97 #include <net/ip.h>
98 #include <net/route.h>
99 #include <net/inetpeer.h>
100 #include <net/sock.h>
101 #include <net/ip_fib.h>
102 #include <net/arp.h>
103 #include <net/tcp.h>
104 #include <net/icmp.h>
105 #include <net/xfrm.h>
106 #include <net/netevent.h>
107 #include <net/rtnetlink.h>
108 #ifdef CONFIG_SYSCTL
109 #include <linux/sysctl.h>
110 #endif
111
112 #define RT_FL_TOS(oldflp) \
113     ((u32)(oldflp->fl4_tos & (IPTOS_RT_MASK | RTO_ONLINK)))
114
115 #define IP_MAX_MTU      0xFFF0
116
117 #define RT_GC_TIMEOUT (300*HZ)
118
119 static int ip_rt_max_size;
120 static int ip_rt_gc_timeout __read_mostly       = RT_GC_TIMEOUT;
121 static int ip_rt_gc_interval __read_mostly      = 60 * HZ;
122 static int ip_rt_gc_min_interval __read_mostly  = HZ / 2;
123 static int ip_rt_redirect_number __read_mostly  = 9;
124 static int ip_rt_redirect_load __read_mostly    = HZ / 50;
125 static int ip_rt_redirect_silence __read_mostly = ((HZ / 50) << (9 + 1));
126 static int ip_rt_error_cost __read_mostly       = HZ;
127 static int ip_rt_error_burst __read_mostly      = 5 * HZ;
128 static int ip_rt_gc_elasticity __read_mostly    = 8;
129 static int ip_rt_mtu_expires __read_mostly      = 10 * 60 * HZ;
130 static int ip_rt_min_pmtu __read_mostly         = 512 + 20 + 20;
131 static int ip_rt_min_advmss __read_mostly       = 256;
132 static int rt_chain_length_max __read_mostly    = 20;
133
134 /*
135  *      Interface to generic destination cache.
136  */
137
138 static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie);
139 static unsigned int      ipv4_default_advmss(const struct dst_entry *dst);
140 static unsigned int      ipv4_default_mtu(const struct dst_entry *dst);
141 static void              ipv4_dst_destroy(struct dst_entry *dst);
142 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst);
143 static void              ipv4_link_failure(struct sk_buff *skb);
144 static void              ip_rt_update_pmtu(struct dst_entry *dst, u32 mtu);
145 static int rt_garbage_collect(struct dst_ops *ops);
146
147 static void ipv4_dst_ifdown(struct dst_entry *dst, struct net_device *dev,
148                             int how)
149 {
150 }
151
152 static u32 *ipv4_cow_metrics(struct dst_entry *dst, unsigned long old)
153 {
154         struct rtable *rt = (struct rtable *) dst;
155         struct inet_peer *peer;
156         u32 *p = NULL;
157
158         if (!rt->peer)
159                 rt_bind_peer(rt, 1);
160
161         peer = rt->peer;
162         if (peer) {
163                 u32 *old_p = __DST_METRICS_PTR(old);
164                 unsigned long prev, new;
165
166                 p = peer->metrics;
167                 if (inet_metrics_new(peer))
168                         memcpy(p, old_p, sizeof(u32) * RTAX_MAX);
169
170                 new = (unsigned long) p;
171                 prev = cmpxchg(&dst->_metrics, old, new);
172
173                 if (prev != old) {
174                         p = __DST_METRICS_PTR(prev);
175                         if (prev & DST_METRICS_READ_ONLY)
176                                 p = NULL;
177                 } else {
178                         if (rt->fi) {
179                                 fib_info_put(rt->fi);
180                                 rt->fi = NULL;
181                         }
182                 }
183         }
184         return p;
185 }
186
187 static struct dst_ops ipv4_dst_ops = {
188         .family =               AF_INET,
189         .protocol =             cpu_to_be16(ETH_P_IP),
190         .gc =                   rt_garbage_collect,
191         .check =                ipv4_dst_check,
192         .default_advmss =       ipv4_default_advmss,
193         .default_mtu =          ipv4_default_mtu,
194         .cow_metrics =          ipv4_cow_metrics,
195         .destroy =              ipv4_dst_destroy,
196         .ifdown =               ipv4_dst_ifdown,
197         .negative_advice =      ipv4_negative_advice,
198         .link_failure =         ipv4_link_failure,
199         .update_pmtu =          ip_rt_update_pmtu,
200         .local_out =            __ip_local_out,
201 };
202
203 #define ECN_OR_COST(class)      TC_PRIO_##class
204
205 const __u8 ip_tos2prio[16] = {
206         TC_PRIO_BESTEFFORT,
207         ECN_OR_COST(FILLER),
208         TC_PRIO_BESTEFFORT,
209         ECN_OR_COST(BESTEFFORT),
210         TC_PRIO_BULK,
211         ECN_OR_COST(BULK),
212         TC_PRIO_BULK,
213         ECN_OR_COST(BULK),
214         TC_PRIO_INTERACTIVE,
215         ECN_OR_COST(INTERACTIVE),
216         TC_PRIO_INTERACTIVE,
217         ECN_OR_COST(INTERACTIVE),
218         TC_PRIO_INTERACTIVE_BULK,
219         ECN_OR_COST(INTERACTIVE_BULK),
220         TC_PRIO_INTERACTIVE_BULK,
221         ECN_OR_COST(INTERACTIVE_BULK)
222 };
223
224
225 /*
226  * Route cache.
227  */
228
229 /* The locking scheme is rather straight forward:
230  *
231  * 1) Read-Copy Update protects the buckets of the central route hash.
232  * 2) Only writers remove entries, and they hold the lock
233  *    as they look at rtable reference counts.
234  * 3) Only readers acquire references to rtable entries,
235  *    they do so with atomic increments and with the
236  *    lock held.
237  */
238
239 struct rt_hash_bucket {
240         struct rtable __rcu     *chain;
241 };
242
243 #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK) || \
244         defined(CONFIG_PROVE_LOCKING)
245 /*
246  * Instead of using one spinlock for each rt_hash_bucket, we use a table of spinlocks
247  * The size of this table is a power of two and depends on the number of CPUS.
248  * (on lockdep we have a quite big spinlock_t, so keep the size down there)
249  */
250 #ifdef CONFIG_LOCKDEP
251 # define RT_HASH_LOCK_SZ        256
252 #else
253 # if NR_CPUS >= 32
254 #  define RT_HASH_LOCK_SZ       4096
255 # elif NR_CPUS >= 16
256 #  define RT_HASH_LOCK_SZ       2048
257 # elif NR_CPUS >= 8
258 #  define RT_HASH_LOCK_SZ       1024
259 # elif NR_CPUS >= 4
260 #  define RT_HASH_LOCK_SZ       512
261 # else
262 #  define RT_HASH_LOCK_SZ       256
263 # endif
264 #endif
265
266 static spinlock_t       *rt_hash_locks;
267 # define rt_hash_lock_addr(slot) &rt_hash_locks[(slot) & (RT_HASH_LOCK_SZ - 1)]
268
269 static __init void rt_hash_lock_init(void)
270 {
271         int i;
272
273         rt_hash_locks = kmalloc(sizeof(spinlock_t) * RT_HASH_LOCK_SZ,
274                         GFP_KERNEL);
275         if (!rt_hash_locks)
276                 panic("IP: failed to allocate rt_hash_locks\n");
277
278         for (i = 0; i < RT_HASH_LOCK_SZ; i++)
279                 spin_lock_init(&rt_hash_locks[i]);
280 }
281 #else
282 # define rt_hash_lock_addr(slot) NULL
283
284 static inline void rt_hash_lock_init(void)
285 {
286 }
287 #endif
288
289 static struct rt_hash_bucket    *rt_hash_table __read_mostly;
290 static unsigned                 rt_hash_mask __read_mostly;
291 static unsigned int             rt_hash_log  __read_mostly;
292
293 static DEFINE_PER_CPU(struct rt_cache_stat, rt_cache_stat);
294 #define RT_CACHE_STAT_INC(field) __this_cpu_inc(rt_cache_stat.field)
295
296 static inline unsigned int rt_hash(__be32 daddr, __be32 saddr, int idx,
297                                    int genid)
298 {
299         return jhash_3words((__force u32)daddr, (__force u32)saddr,
300                             idx, genid)
301                 & rt_hash_mask;
302 }
303
304 static inline int rt_genid(struct net *net)
305 {
306         return atomic_read(&net->ipv4.rt_genid);
307 }
308
309 #ifdef CONFIG_PROC_FS
310 struct rt_cache_iter_state {
311         struct seq_net_private p;
312         int bucket;
313         int genid;
314 };
315
316 static struct rtable *rt_cache_get_first(struct seq_file *seq)
317 {
318         struct rt_cache_iter_state *st = seq->private;
319         struct rtable *r = NULL;
320
321         for (st->bucket = rt_hash_mask; st->bucket >= 0; --st->bucket) {
322                 if (!rcu_dereference_raw(rt_hash_table[st->bucket].chain))
323                         continue;
324                 rcu_read_lock_bh();
325                 r = rcu_dereference_bh(rt_hash_table[st->bucket].chain);
326                 while (r) {
327                         if (dev_net(r->dst.dev) == seq_file_net(seq) &&
328                             r->rt_genid == st->genid)
329                                 return r;
330                         r = rcu_dereference_bh(r->dst.rt_next);
331                 }
332                 rcu_read_unlock_bh();
333         }
334         return r;
335 }
336
337 static struct rtable *__rt_cache_get_next(struct seq_file *seq,
338                                           struct rtable *r)
339 {
340         struct rt_cache_iter_state *st = seq->private;
341
342         r = rcu_dereference_bh(r->dst.rt_next);
343         while (!r) {
344                 rcu_read_unlock_bh();
345                 do {
346                         if (--st->bucket < 0)
347                                 return NULL;
348                 } while (!rcu_dereference_raw(rt_hash_table[st->bucket].chain));
349                 rcu_read_lock_bh();
350                 r = rcu_dereference_bh(rt_hash_table[st->bucket].chain);
351         }
352         return r;
353 }
354
355 static struct rtable *rt_cache_get_next(struct seq_file *seq,
356                                         struct rtable *r)
357 {
358         struct rt_cache_iter_state *st = seq->private;
359         while ((r = __rt_cache_get_next(seq, r)) != NULL) {
360                 if (dev_net(r->dst.dev) != seq_file_net(seq))
361                         continue;
362                 if (r->rt_genid == st->genid)
363                         break;
364         }
365         return r;
366 }
367
368 static struct rtable *rt_cache_get_idx(struct seq_file *seq, loff_t pos)
369 {
370         struct rtable *r = rt_cache_get_first(seq);
371
372         if (r)
373                 while (pos && (r = rt_cache_get_next(seq, r)))
374                         --pos;
375         return pos ? NULL : r;
376 }
377
378 static void *rt_cache_seq_start(struct seq_file *seq, loff_t *pos)
379 {
380         struct rt_cache_iter_state *st = seq->private;
381         if (*pos)
382                 return rt_cache_get_idx(seq, *pos - 1);
383         st->genid = rt_genid(seq_file_net(seq));
384         return SEQ_START_TOKEN;
385 }
386
387 static void *rt_cache_seq_next(struct seq_file *seq, void *v, loff_t *pos)
388 {
389         struct rtable *r;
390
391         if (v == SEQ_START_TOKEN)
392                 r = rt_cache_get_first(seq);
393         else
394                 r = rt_cache_get_next(seq, v);
395         ++*pos;
396         return r;
397 }
398
399 static void rt_cache_seq_stop(struct seq_file *seq, void *v)
400 {
401         if (v && v != SEQ_START_TOKEN)
402                 rcu_read_unlock_bh();
403 }
404
405 static int rt_cache_seq_show(struct seq_file *seq, void *v)
406 {
407         if (v == SEQ_START_TOKEN)
408                 seq_printf(seq, "%-127s\n",
409                            "Iface\tDestination\tGateway \tFlags\t\tRefCnt\tUse\t"
410                            "Metric\tSource\t\tMTU\tWindow\tIRTT\tTOS\tHHRef\t"
411                            "HHUptod\tSpecDst");
412         else {
413                 struct rtable *r = v;
414                 int len;
415
416                 seq_printf(seq, "%s\t%08X\t%08X\t%8X\t%d\t%u\t%d\t"
417                               "%08X\t%d\t%u\t%u\t%02X\t%d\t%1d\t%08X%n",
418                         r->dst.dev ? r->dst.dev->name : "*",
419                         (__force u32)r->rt_dst,
420                         (__force u32)r->rt_gateway,
421                         r->rt_flags, atomic_read(&r->dst.__refcnt),
422                         r->dst.__use, 0, (__force u32)r->rt_src,
423                         dst_metric_advmss(&r->dst) + 40,
424                         dst_metric(&r->dst, RTAX_WINDOW),
425                         (int)((dst_metric(&r->dst, RTAX_RTT) >> 3) +
426                               dst_metric(&r->dst, RTAX_RTTVAR)),
427                         r->fl.fl4_tos,
428                         r->dst.hh ? atomic_read(&r->dst.hh->hh_refcnt) : -1,
429                         r->dst.hh ? (r->dst.hh->hh_output ==
430                                        dev_queue_xmit) : 0,
431                         r->rt_spec_dst, &len);
432
433                 seq_printf(seq, "%*s\n", 127 - len, "");
434         }
435         return 0;
436 }
437
438 static const struct seq_operations rt_cache_seq_ops = {
439         .start  = rt_cache_seq_start,
440         .next   = rt_cache_seq_next,
441         .stop   = rt_cache_seq_stop,
442         .show   = rt_cache_seq_show,
443 };
444
445 static int rt_cache_seq_open(struct inode *inode, struct file *file)
446 {
447         return seq_open_net(inode, file, &rt_cache_seq_ops,
448                         sizeof(struct rt_cache_iter_state));
449 }
450
451 static const struct file_operations rt_cache_seq_fops = {
452         .owner   = THIS_MODULE,
453         .open    = rt_cache_seq_open,
454         .read    = seq_read,
455         .llseek  = seq_lseek,
456         .release = seq_release_net,
457 };
458
459
460 static void *rt_cpu_seq_start(struct seq_file *seq, loff_t *pos)
461 {
462         int cpu;
463
464         if (*pos == 0)
465                 return SEQ_START_TOKEN;
466
467         for (cpu = *pos-1; cpu < nr_cpu_ids; ++cpu) {
468                 if (!cpu_possible(cpu))
469                         continue;
470                 *pos = cpu+1;
471                 return &per_cpu(rt_cache_stat, cpu);
472         }
473         return NULL;
474 }
475
476 static void *rt_cpu_seq_next(struct seq_file *seq, void *v, loff_t *pos)
477 {
478         int cpu;
479
480         for (cpu = *pos; cpu < nr_cpu_ids; ++cpu) {
481                 if (!cpu_possible(cpu))
482                         continue;
483                 *pos = cpu+1;
484                 return &per_cpu(rt_cache_stat, cpu);
485         }
486         return NULL;
487
488 }
489
490 static void rt_cpu_seq_stop(struct seq_file *seq, void *v)
491 {
492
493 }
494
495 static int rt_cpu_seq_show(struct seq_file *seq, void *v)
496 {
497         struct rt_cache_stat *st = v;
498
499         if (v == SEQ_START_TOKEN) {
500                 seq_printf(seq, "entries  in_hit in_slow_tot in_slow_mc in_no_route in_brd in_martian_dst in_martian_src  out_hit out_slow_tot out_slow_mc  gc_total gc_ignored gc_goal_miss gc_dst_overflow in_hlist_search out_hlist_search\n");
501                 return 0;
502         }
503
504         seq_printf(seq,"%08x  %08x %08x %08x %08x %08x %08x %08x "
505                    " %08x %08x %08x %08x %08x %08x %08x %08x %08x \n",
506                    dst_entries_get_slow(&ipv4_dst_ops),
507                    st->in_hit,
508                    st->in_slow_tot,
509                    st->in_slow_mc,
510                    st->in_no_route,
511                    st->in_brd,
512                    st->in_martian_dst,
513                    st->in_martian_src,
514
515                    st->out_hit,
516                    st->out_slow_tot,
517                    st->out_slow_mc,
518
519                    st->gc_total,
520                    st->gc_ignored,
521                    st->gc_goal_miss,
522                    st->gc_dst_overflow,
523                    st->in_hlist_search,
524                    st->out_hlist_search
525                 );
526         return 0;
527 }
528
529 static const struct seq_operations rt_cpu_seq_ops = {
530         .start  = rt_cpu_seq_start,
531         .next   = rt_cpu_seq_next,
532         .stop   = rt_cpu_seq_stop,
533         .show   = rt_cpu_seq_show,
534 };
535
536
537 static int rt_cpu_seq_open(struct inode *inode, struct file *file)
538 {
539         return seq_open(file, &rt_cpu_seq_ops);
540 }
541
542 static const struct file_operations rt_cpu_seq_fops = {
543         .owner   = THIS_MODULE,
544         .open    = rt_cpu_seq_open,
545         .read    = seq_read,
546         .llseek  = seq_lseek,
547         .release = seq_release,
548 };
549
550 #ifdef CONFIG_IP_ROUTE_CLASSID
551 static int rt_acct_proc_show(struct seq_file *m, void *v)
552 {
553         struct ip_rt_acct *dst, *src;
554         unsigned int i, j;
555
556         dst = kcalloc(256, sizeof(struct ip_rt_acct), GFP_KERNEL);
557         if (!dst)
558                 return -ENOMEM;
559
560         for_each_possible_cpu(i) {
561                 src = (struct ip_rt_acct *)per_cpu_ptr(ip_rt_acct, i);
562                 for (j = 0; j < 256; j++) {
563                         dst[j].o_bytes   += src[j].o_bytes;
564                         dst[j].o_packets += src[j].o_packets;
565                         dst[j].i_bytes   += src[j].i_bytes;
566                         dst[j].i_packets += src[j].i_packets;
567                 }
568         }
569
570         seq_write(m, dst, 256 * sizeof(struct ip_rt_acct));
571         kfree(dst);
572         return 0;
573 }
574
575 static int rt_acct_proc_open(struct inode *inode, struct file *file)
576 {
577         return single_open(file, rt_acct_proc_show, NULL);
578 }
579
580 static const struct file_operations rt_acct_proc_fops = {
581         .owner          = THIS_MODULE,
582         .open           = rt_acct_proc_open,
583         .read           = seq_read,
584         .llseek         = seq_lseek,
585         .release        = single_release,
586 };
587 #endif
588
589 static int __net_init ip_rt_do_proc_init(struct net *net)
590 {
591         struct proc_dir_entry *pde;
592
593         pde = proc_net_fops_create(net, "rt_cache", S_IRUGO,
594                         &rt_cache_seq_fops);
595         if (!pde)
596                 goto err1;
597
598         pde = proc_create("rt_cache", S_IRUGO,
599                           net->proc_net_stat, &rt_cpu_seq_fops);
600         if (!pde)
601                 goto err2;
602
603 #ifdef CONFIG_IP_ROUTE_CLASSID
604         pde = proc_create("rt_acct", 0, net->proc_net, &rt_acct_proc_fops);
605         if (!pde)
606                 goto err3;
607 #endif
608         return 0;
609
610 #ifdef CONFIG_IP_ROUTE_CLASSID
611 err3:
612         remove_proc_entry("rt_cache", net->proc_net_stat);
613 #endif
614 err2:
615         remove_proc_entry("rt_cache", net->proc_net);
616 err1:
617         return -ENOMEM;
618 }
619
620 static void __net_exit ip_rt_do_proc_exit(struct net *net)
621 {
622         remove_proc_entry("rt_cache", net->proc_net_stat);
623         remove_proc_entry("rt_cache", net->proc_net);
624 #ifdef CONFIG_IP_ROUTE_CLASSID
625         remove_proc_entry("rt_acct", net->proc_net);
626 #endif
627 }
628
629 static struct pernet_operations ip_rt_proc_ops __net_initdata =  {
630         .init = ip_rt_do_proc_init,
631         .exit = ip_rt_do_proc_exit,
632 };
633
634 static int __init ip_rt_proc_init(void)
635 {
636         return register_pernet_subsys(&ip_rt_proc_ops);
637 }
638
639 #else
640 static inline int ip_rt_proc_init(void)
641 {
642         return 0;
643 }
644 #endif /* CONFIG_PROC_FS */
645
646 static inline void rt_free(struct rtable *rt)
647 {
648         call_rcu_bh(&rt->dst.rcu_head, dst_rcu_free);
649 }
650
651 static inline void rt_drop(struct rtable *rt)
652 {
653         ip_rt_put(rt);
654         call_rcu_bh(&rt->dst.rcu_head, dst_rcu_free);
655 }
656
657 static inline int rt_fast_clean(struct rtable *rth)
658 {
659         /* Kill broadcast/multicast entries very aggresively, if they
660            collide in hash table with more useful entries */
661         return (rth->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST)) &&
662                 rt_is_input_route(rth) && rth->dst.rt_next;
663 }
664
665 static inline int rt_valuable(struct rtable *rth)
666 {
667         return (rth->rt_flags & (RTCF_REDIRECTED | RTCF_NOTIFY)) ||
668                 (rth->peer && rth->peer->pmtu_expires);
669 }
670
671 static int rt_may_expire(struct rtable *rth, unsigned long tmo1, unsigned long tmo2)
672 {
673         unsigned long age;
674         int ret = 0;
675
676         if (atomic_read(&rth->dst.__refcnt))
677                 goto out;
678
679         age = jiffies - rth->dst.lastuse;
680         if ((age <= tmo1 && !rt_fast_clean(rth)) ||
681             (age <= tmo2 && rt_valuable(rth)))
682                 goto out;
683         ret = 1;
684 out:    return ret;
685 }
686
687 /* Bits of score are:
688  * 31: very valuable
689  * 30: not quite useless
690  * 29..0: usage counter
691  */
692 static inline u32 rt_score(struct rtable *rt)
693 {
694         u32 score = jiffies - rt->dst.lastuse;
695
696         score = ~score & ~(3<<30);
697
698         if (rt_valuable(rt))
699                 score |= (1<<31);
700
701         if (rt_is_output_route(rt) ||
702             !(rt->rt_flags & (RTCF_BROADCAST|RTCF_MULTICAST|RTCF_LOCAL)))
703                 score |= (1<<30);
704
705         return score;
706 }
707
708 static inline bool rt_caching(const struct net *net)
709 {
710         return net->ipv4.current_rt_cache_rebuild_count <=
711                 net->ipv4.sysctl_rt_cache_rebuild_count;
712 }
713
714 static inline bool compare_hash_inputs(const struct flowi *fl1,
715                                         const struct flowi *fl2)
716 {
717         return ((((__force u32)fl1->fl4_dst ^ (__force u32)fl2->fl4_dst) |
718                 ((__force u32)fl1->fl4_src ^ (__force u32)fl2->fl4_src) |
719                 (fl1->iif ^ fl2->iif)) == 0);
720 }
721
722 static inline int compare_keys(struct flowi *fl1, struct flowi *fl2)
723 {
724         return (((__force u32)fl1->fl4_dst ^ (__force u32)fl2->fl4_dst) |
725                 ((__force u32)fl1->fl4_src ^ (__force u32)fl2->fl4_src) |
726                 (fl1->mark ^ fl2->mark) |
727                 (*(u16 *)&fl1->fl4_tos ^ *(u16 *)&fl2->fl4_tos) |
728                 (fl1->oif ^ fl2->oif) |
729                 (fl1->iif ^ fl2->iif)) == 0;
730 }
731
732 static inline int compare_netns(struct rtable *rt1, struct rtable *rt2)
733 {
734         return net_eq(dev_net(rt1->dst.dev), dev_net(rt2->dst.dev));
735 }
736
737 static inline int rt_is_expired(struct rtable *rth)
738 {
739         return rth->rt_genid != rt_genid(dev_net(rth->dst.dev));
740 }
741
742 /*
743  * Perform a full scan of hash table and free all entries.
744  * Can be called by a softirq or a process.
745  * In the later case, we want to be reschedule if necessary
746  */
747 static void rt_do_flush(struct net *net, int process_context)
748 {
749         unsigned int i;
750         struct rtable *rth, *next;
751
752         for (i = 0; i <= rt_hash_mask; i++) {
753                 struct rtable __rcu **pprev;
754                 struct rtable *list;
755
756                 if (process_context && need_resched())
757                         cond_resched();
758                 rth = rcu_dereference_raw(rt_hash_table[i].chain);
759                 if (!rth)
760                         continue;
761
762                 spin_lock_bh(rt_hash_lock_addr(i));
763
764                 list = NULL;
765                 pprev = &rt_hash_table[i].chain;
766                 rth = rcu_dereference_protected(*pprev,
767                         lockdep_is_held(rt_hash_lock_addr(i)));
768
769                 while (rth) {
770                         next = rcu_dereference_protected(rth->dst.rt_next,
771                                 lockdep_is_held(rt_hash_lock_addr(i)));
772
773                         if (!net ||
774                             net_eq(dev_net(rth->dst.dev), net)) {
775                                 rcu_assign_pointer(*pprev, next);
776                                 rcu_assign_pointer(rth->dst.rt_next, list);
777                                 list = rth;
778                         } else {
779                                 pprev = &rth->dst.rt_next;
780                         }
781                         rth = next;
782                 }
783
784                 spin_unlock_bh(rt_hash_lock_addr(i));
785
786                 for (; list; list = next) {
787                         next = rcu_dereference_protected(list->dst.rt_next, 1);
788                         rt_free(list);
789                 }
790         }
791 }
792
793 /*
794  * While freeing expired entries, we compute average chain length
795  * and standard deviation, using fixed-point arithmetic.
796  * This to have an estimation of rt_chain_length_max
797  *  rt_chain_length_max = max(elasticity, AVG + 4*SD)
798  * We use 3 bits for frational part, and 29 (or 61) for magnitude.
799  */
800
801 #define FRACT_BITS 3
802 #define ONE (1UL << FRACT_BITS)
803
804 /*
805  * Given a hash chain and an item in this hash chain,
806  * find if a previous entry has the same hash_inputs
807  * (but differs on tos, mark or oif)
808  * Returns 0 if an alias is found.
809  * Returns ONE if rth has no alias before itself.
810  */
811 static int has_noalias(const struct rtable *head, const struct rtable *rth)
812 {
813         const struct rtable *aux = head;
814
815         while (aux != rth) {
816                 if (compare_hash_inputs(&aux->fl, &rth->fl))
817                         return 0;
818                 aux = rcu_dereference_protected(aux->dst.rt_next, 1);
819         }
820         return ONE;
821 }
822
823 /*
824  * Pertubation of rt_genid by a small quantity [1..256]
825  * Using 8 bits of shuffling ensure we can call rt_cache_invalidate()
826  * many times (2^24) without giving recent rt_genid.
827  * Jenkins hash is strong enough that litle changes of rt_genid are OK.
828  */
829 static void rt_cache_invalidate(struct net *net)
830 {
831         unsigned char shuffle;
832
833         get_random_bytes(&shuffle, sizeof(shuffle));
834         atomic_add(shuffle + 1U, &net->ipv4.rt_genid);
835 }
836
837 /*
838  * delay < 0  : invalidate cache (fast : entries will be deleted later)
839  * delay >= 0 : invalidate & flush cache (can be long)
840  */
841 void rt_cache_flush(struct net *net, int delay)
842 {
843         rt_cache_invalidate(net);
844         if (delay >= 0)
845                 rt_do_flush(net, !in_softirq());
846 }
847
848 /* Flush previous cache invalidated entries from the cache */
849 void rt_cache_flush_batch(struct net *net)
850 {
851         rt_do_flush(net, !in_softirq());
852 }
853
854 static void rt_emergency_hash_rebuild(struct net *net)
855 {
856         if (net_ratelimit())
857                 printk(KERN_WARNING "Route hash chain too long!\n");
858         rt_cache_invalidate(net);
859 }
860
861 /*
862    Short description of GC goals.
863
864    We want to build algorithm, which will keep routing cache
865    at some equilibrium point, when number of aged off entries
866    is kept approximately equal to newly generated ones.
867
868    Current expiration strength is variable "expire".
869    We try to adjust it dynamically, so that if networking
870    is idle expires is large enough to keep enough of warm entries,
871    and when load increases it reduces to limit cache size.
872  */
873
874 static int rt_garbage_collect(struct dst_ops *ops)
875 {
876         static unsigned long expire = RT_GC_TIMEOUT;
877         static unsigned long last_gc;
878         static int rover;
879         static int equilibrium;
880         struct rtable *rth;
881         struct rtable __rcu **rthp;
882         unsigned long now = jiffies;
883         int goal;
884         int entries = dst_entries_get_fast(&ipv4_dst_ops);
885
886         /*
887          * Garbage collection is pretty expensive,
888          * do not make it too frequently.
889          */
890
891         RT_CACHE_STAT_INC(gc_total);
892
893         if (now - last_gc < ip_rt_gc_min_interval &&
894             entries < ip_rt_max_size) {
895                 RT_CACHE_STAT_INC(gc_ignored);
896                 goto out;
897         }
898
899         entries = dst_entries_get_slow(&ipv4_dst_ops);
900         /* Calculate number of entries, which we want to expire now. */
901         goal = entries - (ip_rt_gc_elasticity << rt_hash_log);
902         if (goal <= 0) {
903                 if (equilibrium < ipv4_dst_ops.gc_thresh)
904                         equilibrium = ipv4_dst_ops.gc_thresh;
905                 goal = entries - equilibrium;
906                 if (goal > 0) {
907                         equilibrium += min_t(unsigned int, goal >> 1, rt_hash_mask + 1);
908                         goal = entries - equilibrium;
909                 }
910         } else {
911                 /* We are in dangerous area. Try to reduce cache really
912                  * aggressively.
913                  */
914                 goal = max_t(unsigned int, goal >> 1, rt_hash_mask + 1);
915                 equilibrium = entries - goal;
916         }
917
918         if (now - last_gc >= ip_rt_gc_min_interval)
919                 last_gc = now;
920
921         if (goal <= 0) {
922                 equilibrium += goal;
923                 goto work_done;
924         }
925
926         do {
927                 int i, k;
928
929                 for (i = rt_hash_mask, k = rover; i >= 0; i--) {
930                         unsigned long tmo = expire;
931
932                         k = (k + 1) & rt_hash_mask;
933                         rthp = &rt_hash_table[k].chain;
934                         spin_lock_bh(rt_hash_lock_addr(k));
935                         while ((rth = rcu_dereference_protected(*rthp,
936                                         lockdep_is_held(rt_hash_lock_addr(k)))) != NULL) {
937                                 if (!rt_is_expired(rth) &&
938                                         !rt_may_expire(rth, tmo, expire)) {
939                                         tmo >>= 1;
940                                         rthp = &rth->dst.rt_next;
941                                         continue;
942                                 }
943                                 *rthp = rth->dst.rt_next;
944                                 rt_free(rth);
945                                 goal--;
946                         }
947                         spin_unlock_bh(rt_hash_lock_addr(k));
948                         if (goal <= 0)
949                                 break;
950                 }
951                 rover = k;
952
953                 if (goal <= 0)
954                         goto work_done;
955
956                 /* Goal is not achieved. We stop process if:
957
958                    - if expire reduced to zero. Otherwise, expire is halfed.
959                    - if table is not full.
960                    - if we are called from interrupt.
961                    - jiffies check is just fallback/debug loop breaker.
962                      We will not spin here for long time in any case.
963                  */
964
965                 RT_CACHE_STAT_INC(gc_goal_miss);
966
967                 if (expire == 0)
968                         break;
969
970                 expire >>= 1;
971 #if RT_CACHE_DEBUG >= 2
972                 printk(KERN_DEBUG "expire>> %u %d %d %d\n", expire,
973                                 dst_entries_get_fast(&ipv4_dst_ops), goal, i);
974 #endif
975
976                 if (dst_entries_get_fast(&ipv4_dst_ops) < ip_rt_max_size)
977                         goto out;
978         } while (!in_softirq() && time_before_eq(jiffies, now));
979
980         if (dst_entries_get_fast(&ipv4_dst_ops) < ip_rt_max_size)
981                 goto out;
982         if (dst_entries_get_slow(&ipv4_dst_ops) < ip_rt_max_size)
983                 goto out;
984         if (net_ratelimit())
985                 printk(KERN_WARNING "dst cache overflow\n");
986         RT_CACHE_STAT_INC(gc_dst_overflow);
987         return 1;
988
989 work_done:
990         expire += ip_rt_gc_min_interval;
991         if (expire > ip_rt_gc_timeout ||
992             dst_entries_get_fast(&ipv4_dst_ops) < ipv4_dst_ops.gc_thresh ||
993             dst_entries_get_slow(&ipv4_dst_ops) < ipv4_dst_ops.gc_thresh)
994                 expire = ip_rt_gc_timeout;
995 #if RT_CACHE_DEBUG >= 2
996         printk(KERN_DEBUG "expire++ %u %d %d %d\n", expire,
997                         dst_entries_get_fast(&ipv4_dst_ops), goal, rover);
998 #endif
999 out:    return 0;
1000 }
1001
1002 /*
1003  * Returns number of entries in a hash chain that have different hash_inputs
1004  */
1005 static int slow_chain_length(const struct rtable *head)
1006 {
1007         int length = 0;
1008         const struct rtable *rth = head;
1009
1010         while (rth) {
1011                 length += has_noalias(head, rth);
1012                 rth = rcu_dereference_protected(rth->dst.rt_next, 1);
1013         }
1014         return length >> FRACT_BITS;
1015 }
1016
1017 static struct rtable *rt_intern_hash(unsigned hash, struct rtable *rt,
1018                                      struct sk_buff *skb, int ifindex)
1019 {
1020         struct rtable   *rth, *cand;
1021         struct rtable __rcu **rthp, **candp;
1022         unsigned long   now;
1023         u32             min_score;
1024         int             chain_length;
1025         int attempts = !in_softirq();
1026
1027 restart:
1028         chain_length = 0;
1029         min_score = ~(u32)0;
1030         cand = NULL;
1031         candp = NULL;
1032         now = jiffies;
1033
1034         if (!rt_caching(dev_net(rt->dst.dev))) {
1035                 /*
1036                  * If we're not caching, just tell the caller we
1037                  * were successful and don't touch the route.  The
1038                  * caller hold the sole reference to the cache entry, and
1039                  * it will be released when the caller is done with it.
1040                  * If we drop it here, the callers have no way to resolve routes
1041                  * when we're not caching.  Instead, just point *rp at rt, so
1042                  * the caller gets a single use out of the route
1043                  * Note that we do rt_free on this new route entry, so that
1044                  * once its refcount hits zero, we are still able to reap it
1045                  * (Thanks Alexey)
1046                  * Note: To avoid expensive rcu stuff for this uncached dst,
1047                  * we set DST_NOCACHE so that dst_release() can free dst without
1048                  * waiting a grace period.
1049                  */
1050
1051                 rt->dst.flags |= DST_NOCACHE;
1052                 if (rt->rt_type == RTN_UNICAST || rt_is_output_route(rt)) {
1053                         int err = arp_bind_neighbour(&rt->dst);
1054                         if (err) {
1055                                 if (net_ratelimit())
1056                                         printk(KERN_WARNING
1057                                             "Neighbour table failure & not caching routes.\n");
1058                                 ip_rt_put(rt);
1059                                 return ERR_PTR(err);
1060                         }
1061                 }
1062
1063                 goto skip_hashing;
1064         }
1065
1066         rthp = &rt_hash_table[hash].chain;
1067
1068         spin_lock_bh(rt_hash_lock_addr(hash));
1069         while ((rth = rcu_dereference_protected(*rthp,
1070                         lockdep_is_held(rt_hash_lock_addr(hash)))) != NULL) {
1071                 if (rt_is_expired(rth)) {
1072                         *rthp = rth->dst.rt_next;
1073                         rt_free(rth);
1074                         continue;
1075                 }
1076                 if (compare_keys(&rth->fl, &rt->fl) && compare_netns(rth, rt)) {
1077                         /* Put it first */
1078                         *rthp = rth->dst.rt_next;
1079                         /*
1080                          * Since lookup is lockfree, the deletion
1081                          * must be visible to another weakly ordered CPU before
1082                          * the insertion at the start of the hash chain.
1083                          */
1084                         rcu_assign_pointer(rth->dst.rt_next,
1085                                            rt_hash_table[hash].chain);
1086                         /*
1087                          * Since lookup is lockfree, the update writes
1088                          * must be ordered for consistency on SMP.
1089                          */
1090                         rcu_assign_pointer(rt_hash_table[hash].chain, rth);
1091
1092                         dst_use(&rth->dst, now);
1093                         spin_unlock_bh(rt_hash_lock_addr(hash));
1094
1095                         rt_drop(rt);
1096                         if (skb)
1097                                 skb_dst_set(skb, &rth->dst);
1098                         return rth;
1099                 }
1100
1101                 if (!atomic_read(&rth->dst.__refcnt)) {
1102                         u32 score = rt_score(rth);
1103
1104                         if (score <= min_score) {
1105                                 cand = rth;
1106                                 candp = rthp;
1107                                 min_score = score;
1108                         }
1109                 }
1110
1111                 chain_length++;
1112
1113                 rthp = &rth->dst.rt_next;
1114         }
1115
1116         if (cand) {
1117                 /* ip_rt_gc_elasticity used to be average length of chain
1118                  * length, when exceeded gc becomes really aggressive.
1119                  *
1120                  * The second limit is less certain. At the moment it allows
1121                  * only 2 entries per bucket. We will see.
1122                  */
1123                 if (chain_length > ip_rt_gc_elasticity) {
1124                         *candp = cand->dst.rt_next;
1125                         rt_free(cand);
1126                 }
1127         } else {
1128                 if (chain_length > rt_chain_length_max &&
1129                     slow_chain_length(rt_hash_table[hash].chain) > rt_chain_length_max) {
1130                         struct net *net = dev_net(rt->dst.dev);
1131                         int num = ++net->ipv4.current_rt_cache_rebuild_count;
1132                         if (!rt_caching(net)) {
1133                                 printk(KERN_WARNING "%s: %d rebuilds is over limit, route caching disabled\n",
1134                                         rt->dst.dev->name, num);
1135                         }
1136                         rt_emergency_hash_rebuild(net);
1137                         spin_unlock_bh(rt_hash_lock_addr(hash));
1138
1139                         hash = rt_hash(rt->fl.fl4_dst, rt->fl.fl4_src,
1140                                         ifindex, rt_genid(net));
1141                         goto restart;
1142                 }
1143         }
1144
1145         /* Try to bind route to arp only if it is output
1146            route or unicast forwarding path.
1147          */
1148         if (rt->rt_type == RTN_UNICAST || rt_is_output_route(rt)) {
1149                 int err = arp_bind_neighbour(&rt->dst);
1150                 if (err) {
1151                         spin_unlock_bh(rt_hash_lock_addr(hash));
1152
1153                         if (err != -ENOBUFS) {
1154                                 rt_drop(rt);
1155                                 return ERR_PTR(err);
1156                         }
1157
1158                         /* Neighbour tables are full and nothing
1159                            can be released. Try to shrink route cache,
1160                            it is most likely it holds some neighbour records.
1161                          */
1162                         if (attempts-- > 0) {
1163                                 int saved_elasticity = ip_rt_gc_elasticity;
1164                                 int saved_int = ip_rt_gc_min_interval;
1165                                 ip_rt_gc_elasticity     = 1;
1166                                 ip_rt_gc_min_interval   = 0;
1167                                 rt_garbage_collect(&ipv4_dst_ops);
1168                                 ip_rt_gc_min_interval   = saved_int;
1169                                 ip_rt_gc_elasticity     = saved_elasticity;
1170                                 goto restart;
1171                         }
1172
1173                         if (net_ratelimit())
1174                                 printk(KERN_WARNING "ipv4: Neighbour table overflow.\n");
1175                         rt_drop(rt);
1176                         return ERR_PTR(-ENOBUFS);
1177                 }
1178         }
1179
1180         rt->dst.rt_next = rt_hash_table[hash].chain;
1181
1182 #if RT_CACHE_DEBUG >= 2
1183         if (rt->dst.rt_next) {
1184                 struct rtable *trt;
1185                 printk(KERN_DEBUG "rt_cache @%02x: %pI4",
1186                        hash, &rt->rt_dst);
1187                 for (trt = rt->dst.rt_next; trt; trt = trt->dst.rt_next)
1188                         printk(" . %pI4", &trt->rt_dst);
1189                 printk("\n");
1190         }
1191 #endif
1192         /*
1193          * Since lookup is lockfree, we must make sure
1194          * previous writes to rt are comitted to memory
1195          * before making rt visible to other CPUS.
1196          */
1197         rcu_assign_pointer(rt_hash_table[hash].chain, rt);
1198
1199         spin_unlock_bh(rt_hash_lock_addr(hash));
1200
1201 skip_hashing:
1202         if (skb)
1203                 skb_dst_set(skb, &rt->dst);
1204         return rt;
1205 }
1206
1207 static atomic_t __rt_peer_genid = ATOMIC_INIT(0);
1208
1209 static u32 rt_peer_genid(void)
1210 {
1211         return atomic_read(&__rt_peer_genid);
1212 }
1213
1214 void rt_bind_peer(struct rtable *rt, int create)
1215 {
1216         struct inet_peer *peer;
1217
1218         peer = inet_getpeer_v4(rt->rt_dst, create);
1219
1220         if (peer && cmpxchg(&rt->peer, NULL, peer) != NULL)
1221                 inet_putpeer(peer);
1222         else
1223                 rt->rt_peer_genid = rt_peer_genid();
1224 }
1225
1226 /*
1227  * Peer allocation may fail only in serious out-of-memory conditions.  However
1228  * we still can generate some output.
1229  * Random ID selection looks a bit dangerous because we have no chances to
1230  * select ID being unique in a reasonable period of time.
1231  * But broken packet identifier may be better than no packet at all.
1232  */
1233 static void ip_select_fb_ident(struct iphdr *iph)
1234 {
1235         static DEFINE_SPINLOCK(ip_fb_id_lock);
1236         static u32 ip_fallback_id;
1237         u32 salt;
1238
1239         spin_lock_bh(&ip_fb_id_lock);
1240         salt = secure_ip_id((__force __be32)ip_fallback_id ^ iph->daddr);
1241         iph->id = htons(salt & 0xFFFF);
1242         ip_fallback_id = salt;
1243         spin_unlock_bh(&ip_fb_id_lock);
1244 }
1245
1246 void __ip_select_ident(struct iphdr *iph, struct dst_entry *dst, int more)
1247 {
1248         struct rtable *rt = (struct rtable *) dst;
1249
1250         if (rt) {
1251                 if (rt->peer == NULL)
1252                         rt_bind_peer(rt, 1);
1253
1254                 /* If peer is attached to destination, it is never detached,
1255                    so that we need not to grab a lock to dereference it.
1256                  */
1257                 if (rt->peer) {
1258                         iph->id = htons(inet_getid(rt->peer, more));
1259                         return;
1260                 }
1261         } else
1262                 printk(KERN_DEBUG "rt_bind_peer(0) @%p\n",
1263                        __builtin_return_address(0));
1264
1265         ip_select_fb_ident(iph);
1266 }
1267 EXPORT_SYMBOL(__ip_select_ident);
1268
1269 static void rt_del(unsigned hash, struct rtable *rt)
1270 {
1271         struct rtable __rcu **rthp;
1272         struct rtable *aux;
1273
1274         rthp = &rt_hash_table[hash].chain;
1275         spin_lock_bh(rt_hash_lock_addr(hash));
1276         ip_rt_put(rt);
1277         while ((aux = rcu_dereference_protected(*rthp,
1278                         lockdep_is_held(rt_hash_lock_addr(hash)))) != NULL) {
1279                 if (aux == rt || rt_is_expired(aux)) {
1280                         *rthp = aux->dst.rt_next;
1281                         rt_free(aux);
1282                         continue;
1283                 }
1284                 rthp = &aux->dst.rt_next;
1285         }
1286         spin_unlock_bh(rt_hash_lock_addr(hash));
1287 }
1288
1289 /* called in rcu_read_lock() section */
1290 void ip_rt_redirect(__be32 old_gw, __be32 daddr, __be32 new_gw,
1291                     __be32 saddr, struct net_device *dev)
1292 {
1293         struct in_device *in_dev = __in_dev_get_rcu(dev);
1294         struct inet_peer *peer;
1295         struct net *net;
1296
1297         if (!in_dev)
1298                 return;
1299
1300         net = dev_net(dev);
1301         if (new_gw == old_gw || !IN_DEV_RX_REDIRECTS(in_dev) ||
1302             ipv4_is_multicast(new_gw) || ipv4_is_lbcast(new_gw) ||
1303             ipv4_is_zeronet(new_gw))
1304                 goto reject_redirect;
1305
1306         if (!IN_DEV_SHARED_MEDIA(in_dev)) {
1307                 if (!inet_addr_onlink(in_dev, new_gw, old_gw))
1308                         goto reject_redirect;
1309                 if (IN_DEV_SEC_REDIRECTS(in_dev) && ip_fib_check_default(new_gw, dev))
1310                         goto reject_redirect;
1311         } else {
1312                 if (inet_addr_type(net, new_gw) != RTN_UNICAST)
1313                         goto reject_redirect;
1314         }
1315
1316         peer = inet_getpeer_v4(daddr, 1);
1317         if (peer) {
1318                 peer->redirect_learned.a4 = new_gw;
1319
1320                 inet_putpeer(peer);
1321
1322                 atomic_inc(&__rt_peer_genid);
1323         }
1324         return;
1325
1326 reject_redirect:
1327 #ifdef CONFIG_IP_ROUTE_VERBOSE
1328         if (IN_DEV_LOG_MARTIANS(in_dev) && net_ratelimit())
1329                 printk(KERN_INFO "Redirect from %pI4 on %s about %pI4 ignored.\n"
1330                         "  Advised path = %pI4 -> %pI4\n",
1331                        &old_gw, dev->name, &new_gw,
1332                        &saddr, &daddr);
1333 #endif
1334         ;
1335 }
1336
1337 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst)
1338 {
1339         struct rtable *rt = (struct rtable *)dst;
1340         struct dst_entry *ret = dst;
1341
1342         if (rt) {
1343                 if (dst->obsolete > 0) {
1344                         ip_rt_put(rt);
1345                         ret = NULL;
1346                 } else if (rt->rt_flags & RTCF_REDIRECTED) {
1347                         unsigned hash = rt_hash(rt->fl.fl4_dst, rt->fl.fl4_src,
1348                                                 rt->fl.oif,
1349                                                 rt_genid(dev_net(dst->dev)));
1350 #if RT_CACHE_DEBUG >= 1
1351                         printk(KERN_DEBUG "ipv4_negative_advice: redirect to %pI4/%02x dropped\n",
1352                                 &rt->rt_dst, rt->fl.fl4_tos);
1353 #endif
1354                         rt_del(hash, rt);
1355                         ret = NULL;
1356                 } else if (rt->peer &&
1357                            rt->peer->pmtu_expires &&
1358                            time_after_eq(jiffies, rt->peer->pmtu_expires)) {
1359                         unsigned long orig = rt->peer->pmtu_expires;
1360
1361                         if (cmpxchg(&rt->peer->pmtu_expires, orig, 0) == orig)
1362                                 dst_metric_set(dst, RTAX_MTU,
1363                                                rt->peer->pmtu_orig);
1364                 }
1365         }
1366         return ret;
1367 }
1368
1369 /*
1370  * Algorithm:
1371  *      1. The first ip_rt_redirect_number redirects are sent
1372  *         with exponential backoff, then we stop sending them at all,
1373  *         assuming that the host ignores our redirects.
1374  *      2. If we did not see packets requiring redirects
1375  *         during ip_rt_redirect_silence, we assume that the host
1376  *         forgot redirected route and start to send redirects again.
1377  *
1378  * This algorithm is much cheaper and more intelligent than dumb load limiting
1379  * in icmp.c.
1380  *
1381  * NOTE. Do not forget to inhibit load limiting for redirects (redundant)
1382  * and "frag. need" (breaks PMTU discovery) in icmp.c.
1383  */
1384
1385 void ip_rt_send_redirect(struct sk_buff *skb)
1386 {
1387         struct rtable *rt = skb_rtable(skb);
1388         struct in_device *in_dev;
1389         struct inet_peer *peer;
1390         int log_martians;
1391
1392         rcu_read_lock();
1393         in_dev = __in_dev_get_rcu(rt->dst.dev);
1394         if (!in_dev || !IN_DEV_TX_REDIRECTS(in_dev)) {
1395                 rcu_read_unlock();
1396                 return;
1397         }
1398         log_martians = IN_DEV_LOG_MARTIANS(in_dev);
1399         rcu_read_unlock();
1400
1401         if (!rt->peer)
1402                 rt_bind_peer(rt, 1);
1403         peer = rt->peer;
1404         if (!peer) {
1405                 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST, rt->rt_gateway);
1406                 return;
1407         }
1408
1409         /* No redirected packets during ip_rt_redirect_silence;
1410          * reset the algorithm.
1411          */
1412         if (time_after(jiffies, peer->rate_last + ip_rt_redirect_silence))
1413                 peer->rate_tokens = 0;
1414
1415         /* Too many ignored redirects; do not send anything
1416          * set dst.rate_last to the last seen redirected packet.
1417          */
1418         if (peer->rate_tokens >= ip_rt_redirect_number) {
1419                 peer->rate_last = jiffies;
1420                 return;
1421         }
1422
1423         /* Check for load limit; set rate_last to the latest sent
1424          * redirect.
1425          */
1426         if (peer->rate_tokens == 0 ||
1427             time_after(jiffies,
1428                        (peer->rate_last +
1429                         (ip_rt_redirect_load << peer->rate_tokens)))) {
1430                 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST, rt->rt_gateway);
1431                 peer->rate_last = jiffies;
1432                 ++peer->rate_tokens;
1433 #ifdef CONFIG_IP_ROUTE_VERBOSE
1434                 if (log_martians &&
1435                     peer->rate_tokens == ip_rt_redirect_number &&
1436                     net_ratelimit())
1437                         printk(KERN_WARNING "host %pI4/if%d ignores redirects for %pI4 to %pI4.\n",
1438                                 &rt->rt_src, rt->rt_iif,
1439                                 &rt->rt_dst, &rt->rt_gateway);
1440 #endif
1441         }
1442 }
1443
1444 static int ip_error(struct sk_buff *skb)
1445 {
1446         struct rtable *rt = skb_rtable(skb);
1447         struct inet_peer *peer;
1448         unsigned long now;
1449         bool send;
1450         int code;
1451
1452         switch (rt->dst.error) {
1453                 case EINVAL:
1454                 default:
1455                         goto out;
1456                 case EHOSTUNREACH:
1457                         code = ICMP_HOST_UNREACH;
1458                         break;
1459                 case ENETUNREACH:
1460                         code = ICMP_NET_UNREACH;
1461                         IP_INC_STATS_BH(dev_net(rt->dst.dev),
1462                                         IPSTATS_MIB_INNOROUTES);
1463                         break;
1464                 case EACCES:
1465                         code = ICMP_PKT_FILTERED;
1466                         break;
1467         }
1468
1469         if (!rt->peer)
1470                 rt_bind_peer(rt, 1);
1471         peer = rt->peer;
1472
1473         send = true;
1474         if (peer) {
1475                 now = jiffies;
1476                 peer->rate_tokens += now - peer->rate_last;
1477                 if (peer->rate_tokens > ip_rt_error_burst)
1478                         peer->rate_tokens = ip_rt_error_burst;
1479                 peer->rate_last = now;
1480                 if (peer->rate_tokens >= ip_rt_error_cost)
1481                         peer->rate_tokens -= ip_rt_error_cost;
1482                 else
1483                         send = false;
1484         }
1485         if (send)
1486                 icmp_send(skb, ICMP_DEST_UNREACH, code, 0);
1487
1488 out:    kfree_skb(skb);
1489         return 0;
1490 }
1491
1492 /*
1493  *      The last two values are not from the RFC but
1494  *      are needed for AMPRnet AX.25 paths.
1495  */
1496
1497 static const unsigned short mtu_plateau[] =
1498 {32000, 17914, 8166, 4352, 2002, 1492, 576, 296, 216, 128 };
1499
1500 static inline unsigned short guess_mtu(unsigned short old_mtu)
1501 {
1502         int i;
1503
1504         for (i = 0; i < ARRAY_SIZE(mtu_plateau); i++)
1505                 if (old_mtu > mtu_plateau[i])
1506                         return mtu_plateau[i];
1507         return 68;
1508 }
1509
1510 unsigned short ip_rt_frag_needed(struct net *net, struct iphdr *iph,
1511                                  unsigned short new_mtu,
1512                                  struct net_device *dev)
1513 {
1514         unsigned short old_mtu = ntohs(iph->tot_len);
1515         unsigned short est_mtu = 0;
1516         struct inet_peer *peer;
1517
1518         peer = inet_getpeer_v4(iph->daddr, 1);
1519         if (peer) {
1520                 unsigned short mtu = new_mtu;
1521
1522                 if (new_mtu < 68 || new_mtu >= old_mtu) {
1523                         /* BSD 4.2 derived systems incorrectly adjust
1524                          * tot_len by the IP header length, and report
1525                          * a zero MTU in the ICMP message.
1526                          */
1527                         if (mtu == 0 &&
1528                             old_mtu >= 68 + (iph->ihl << 2))
1529                                 old_mtu -= iph->ihl << 2;
1530                         mtu = guess_mtu(old_mtu);
1531                 }
1532
1533                 if (mtu < ip_rt_min_pmtu)
1534                         mtu = ip_rt_min_pmtu;
1535                 if (!peer->pmtu_expires || mtu < peer->pmtu_learned) {
1536                         est_mtu = mtu;
1537                         peer->pmtu_learned = mtu;
1538                         peer->pmtu_expires = jiffies + ip_rt_mtu_expires;
1539                 }
1540
1541                 inet_putpeer(peer);
1542
1543                 atomic_inc(&__rt_peer_genid);
1544         }
1545         return est_mtu ? : new_mtu;
1546 }
1547
1548 static void check_peer_pmtu(struct dst_entry *dst, struct inet_peer *peer)
1549 {
1550         unsigned long expires = peer->pmtu_expires;
1551
1552         if (time_before(expires, jiffies)) {
1553                 u32 orig_dst_mtu = dst_mtu(dst);
1554                 if (peer->pmtu_learned < orig_dst_mtu) {
1555                         if (!peer->pmtu_orig)
1556                                 peer->pmtu_orig = dst_metric_raw(dst, RTAX_MTU);
1557                         dst_metric_set(dst, RTAX_MTU, peer->pmtu_learned);
1558                 }
1559         } else if (cmpxchg(&peer->pmtu_expires, expires, 0) == expires)
1560                 dst_metric_set(dst, RTAX_MTU, peer->pmtu_orig);
1561 }
1562
1563 static void ip_rt_update_pmtu(struct dst_entry *dst, u32 mtu)
1564 {
1565         struct rtable *rt = (struct rtable *) dst;
1566         struct inet_peer *peer;
1567
1568         dst_confirm(dst);
1569
1570         if (!rt->peer)
1571                 rt_bind_peer(rt, 1);
1572         peer = rt->peer;
1573         if (peer) {
1574                 if (mtu < ip_rt_min_pmtu)
1575                         mtu = ip_rt_min_pmtu;
1576                 if (!peer->pmtu_expires || mtu < peer->pmtu_learned) {
1577                         peer->pmtu_learned = mtu;
1578                         peer->pmtu_expires = jiffies + ip_rt_mtu_expires;
1579
1580                         atomic_inc(&__rt_peer_genid);
1581                         rt->rt_peer_genid = rt_peer_genid();
1582
1583                         check_peer_pmtu(dst, peer);
1584                 }
1585                 inet_putpeer(peer);
1586         }
1587 }
1588
1589 static int check_peer_redir(struct dst_entry *dst, struct inet_peer *peer)
1590 {
1591         struct rtable *rt = (struct rtable *) dst;
1592         __be32 orig_gw = rt->rt_gateway;
1593
1594         dst_confirm(&rt->dst);
1595
1596         neigh_release(rt->dst.neighbour);
1597         rt->dst.neighbour = NULL;
1598
1599         rt->rt_gateway = peer->redirect_learned.a4;
1600         if (arp_bind_neighbour(&rt->dst) ||
1601             !(rt->dst.neighbour->nud_state & NUD_VALID)) {
1602                 if (rt->dst.neighbour)
1603                         neigh_event_send(rt->dst.neighbour, NULL);
1604                 rt->rt_gateway = orig_gw;
1605                 return -EAGAIN;
1606         } else {
1607                 rt->rt_flags |= RTCF_REDIRECTED;
1608                 call_netevent_notifiers(NETEVENT_NEIGH_UPDATE,
1609                                         rt->dst.neighbour);
1610         }
1611         return 0;
1612 }
1613
1614 static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie)
1615 {
1616         struct rtable *rt = (struct rtable *) dst;
1617
1618         if (rt_is_expired(rt))
1619                 return NULL;
1620         if (rt->rt_peer_genid != rt_peer_genid()) {
1621                 struct inet_peer *peer;
1622
1623                 if (!rt->peer)
1624                         rt_bind_peer(rt, 0);
1625
1626                 peer = rt->peer;
1627                 if (peer && peer->pmtu_expires)
1628                         check_peer_pmtu(dst, peer);
1629
1630                 if (peer && peer->redirect_learned.a4 &&
1631                     peer->redirect_learned.a4 != rt->rt_gateway) {
1632                         if (check_peer_redir(dst, peer))
1633                                 return NULL;
1634                 }
1635
1636                 rt->rt_peer_genid = rt_peer_genid();
1637         }
1638         return dst;
1639 }
1640
1641 static void ipv4_dst_destroy(struct dst_entry *dst)
1642 {
1643         struct rtable *rt = (struct rtable *) dst;
1644         struct inet_peer *peer = rt->peer;
1645
1646         if (rt->fi) {
1647                 fib_info_put(rt->fi);
1648                 rt->fi = NULL;
1649         }
1650         if (peer) {
1651                 rt->peer = NULL;
1652                 inet_putpeer(peer);
1653         }
1654 }
1655
1656
1657 static void ipv4_link_failure(struct sk_buff *skb)
1658 {
1659         struct rtable *rt;
1660
1661         icmp_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0);
1662
1663         rt = skb_rtable(skb);
1664         if (rt &&
1665             rt->peer &&
1666             rt->peer->pmtu_expires) {
1667                 unsigned long orig = rt->peer->pmtu_expires;
1668
1669                 if (cmpxchg(&rt->peer->pmtu_expires, orig, 0) == orig)
1670                         dst_metric_set(&rt->dst, RTAX_MTU, rt->peer->pmtu_orig);
1671         }
1672 }
1673
1674 static int ip_rt_bug(struct sk_buff *skb)
1675 {
1676         printk(KERN_DEBUG "ip_rt_bug: %pI4 -> %pI4, %s\n",
1677                 &ip_hdr(skb)->saddr, &ip_hdr(skb)->daddr,
1678                 skb->dev ? skb->dev->name : "?");
1679         kfree_skb(skb);
1680         return 0;
1681 }
1682
1683 /*
1684    We do not cache source address of outgoing interface,
1685    because it is used only by IP RR, TS and SRR options,
1686    so that it out of fast path.
1687
1688    BTW remember: "addr" is allowed to be not aligned
1689    in IP options!
1690  */
1691
1692 void ip_rt_get_source(u8 *addr, struct rtable *rt)
1693 {
1694         __be32 src;
1695         struct fib_result res;
1696
1697         if (rt_is_output_route(rt))
1698                 src = rt->rt_src;
1699         else {
1700                 rcu_read_lock();
1701                 if (fib_lookup(dev_net(rt->dst.dev), &rt->fl, &res) == 0)
1702                         src = FIB_RES_PREFSRC(res);
1703                 else
1704                         src = inet_select_addr(rt->dst.dev, rt->rt_gateway,
1705                                         RT_SCOPE_UNIVERSE);
1706                 rcu_read_unlock();
1707         }
1708         memcpy(addr, &src, 4);
1709 }
1710
1711 #ifdef CONFIG_IP_ROUTE_CLASSID
1712 static void set_class_tag(struct rtable *rt, u32 tag)
1713 {
1714         if (!(rt->dst.tclassid & 0xFFFF))
1715                 rt->dst.tclassid |= tag & 0xFFFF;
1716         if (!(rt->dst.tclassid & 0xFFFF0000))
1717                 rt->dst.tclassid |= tag & 0xFFFF0000;
1718 }
1719 #endif
1720
1721 static unsigned int ipv4_default_advmss(const struct dst_entry *dst)
1722 {
1723         unsigned int advmss = dst_metric_raw(dst, RTAX_ADVMSS);
1724
1725         if (advmss == 0) {
1726                 advmss = max_t(unsigned int, dst->dev->mtu - 40,
1727                                ip_rt_min_advmss);
1728                 if (advmss > 65535 - 40)
1729                         advmss = 65535 - 40;
1730         }
1731         return advmss;
1732 }
1733
1734 static unsigned int ipv4_default_mtu(const struct dst_entry *dst)
1735 {
1736         unsigned int mtu = dst->dev->mtu;
1737
1738         if (unlikely(dst_metric_locked(dst, RTAX_MTU))) {
1739                 const struct rtable *rt = (const struct rtable *) dst;
1740
1741                 if (rt->rt_gateway != rt->rt_dst && mtu > 576)
1742                         mtu = 576;
1743         }
1744
1745         if (mtu > IP_MAX_MTU)
1746                 mtu = IP_MAX_MTU;
1747
1748         return mtu;
1749 }
1750
1751 static void rt_init_metrics(struct rtable *rt, struct fib_info *fi)
1752 {
1753         struct inet_peer *peer;
1754         int create = 0;
1755
1756         /* If a peer entry exists for this destination, we must hook
1757          * it up in order to get at cached metrics.
1758          */
1759         if (rt->fl.flags & FLOWI_FLAG_PRECOW_METRICS)
1760                 create = 1;
1761
1762         rt_bind_peer(rt, create);
1763         peer = rt->peer;
1764         if (peer) {
1765                 if (inet_metrics_new(peer))
1766                         memcpy(peer->metrics, fi->fib_metrics,
1767                                sizeof(u32) * RTAX_MAX);
1768                 dst_init_metrics(&rt->dst, peer->metrics, false);
1769
1770                 if (peer->pmtu_expires)
1771                         check_peer_pmtu(&rt->dst, peer);
1772                 if (peer->redirect_learned.a4 &&
1773                     peer->redirect_learned.a4 != rt->rt_gateway) {
1774                         rt->rt_gateway = peer->redirect_learned.a4;
1775                         rt->rt_flags |= RTCF_REDIRECTED;
1776                 }
1777         } else {
1778                 if (fi->fib_metrics != (u32 *) dst_default_metrics) {
1779                         rt->fi = fi;
1780                         atomic_inc(&fi->fib_clntref);
1781                 }
1782                 dst_init_metrics(&rt->dst, fi->fib_metrics, true);
1783         }
1784 }
1785
1786 static void rt_set_nexthop(struct rtable *rt, const struct fib_result *res,
1787                            struct fib_info *fi, u16 type, u32 itag)
1788 {
1789         struct dst_entry *dst = &rt->dst;
1790
1791         if (fi) {
1792                 if (FIB_RES_GW(*res) &&
1793                     FIB_RES_NH(*res).nh_scope == RT_SCOPE_LINK)
1794                         rt->rt_gateway = FIB_RES_GW(*res);
1795                 rt_init_metrics(rt, fi);
1796 #ifdef CONFIG_IP_ROUTE_CLASSID
1797                 dst->tclassid = FIB_RES_NH(*res).nh_tclassid;
1798 #endif
1799         }
1800
1801         if (dst_mtu(dst) > IP_MAX_MTU)
1802                 dst_metric_set(dst, RTAX_MTU, IP_MAX_MTU);
1803         if (dst_metric_raw(dst, RTAX_ADVMSS) > 65535 - 40)
1804                 dst_metric_set(dst, RTAX_ADVMSS, 65535 - 40);
1805
1806 #ifdef CONFIG_IP_ROUTE_CLASSID
1807 #ifdef CONFIG_IP_MULTIPLE_TABLES
1808         set_class_tag(rt, fib_rules_tclass(res));
1809 #endif
1810         set_class_tag(rt, itag);
1811 #endif
1812         rt->rt_type = type;
1813 }
1814
1815 static struct rtable *rt_dst_alloc(bool nopolicy, bool noxfrm)
1816 {
1817         struct rtable *rt = dst_alloc(&ipv4_dst_ops, 1);
1818         if (rt) {
1819                 rt->dst.obsolete = -1;
1820
1821                 rt->dst.flags = DST_HOST |
1822                         (nopolicy ? DST_NOPOLICY : 0) |
1823                         (noxfrm ? DST_NOXFRM : 0);
1824         }
1825         return rt;
1826 }
1827
1828 /* called in rcu_read_lock() section */
1829 static int ip_route_input_mc(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1830                                 u8 tos, struct net_device *dev, int our)
1831 {
1832         unsigned int hash;
1833         struct rtable *rth;
1834         __be32 spec_dst;
1835         struct in_device *in_dev = __in_dev_get_rcu(dev);
1836         u32 itag = 0;
1837         int err;
1838
1839         /* Primary sanity checks. */
1840
1841         if (in_dev == NULL)
1842                 return -EINVAL;
1843
1844         if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) ||
1845             ipv4_is_loopback(saddr) || skb->protocol != htons(ETH_P_IP))
1846                 goto e_inval;
1847
1848         if (ipv4_is_zeronet(saddr)) {
1849                 if (!ipv4_is_local_multicast(daddr))
1850                         goto e_inval;
1851                 spec_dst = inet_select_addr(dev, 0, RT_SCOPE_LINK);
1852         } else {
1853                 err = fib_validate_source(saddr, 0, tos, 0, dev, &spec_dst,
1854                                           &itag, 0);
1855                 if (err < 0)
1856                         goto e_err;
1857         }
1858         rth = rt_dst_alloc(IN_DEV_CONF_GET(in_dev, NOPOLICY), false);
1859         if (!rth)
1860                 goto e_nobufs;
1861
1862         rth->dst.output = ip_rt_bug;
1863
1864         rth->fl.fl4_dst = daddr;
1865         rth->rt_dst     = daddr;
1866         rth->fl.fl4_tos = tos;
1867         rth->fl.mark    = skb->mark;
1868         rth->fl.fl4_src = saddr;
1869         rth->rt_src     = saddr;
1870 #ifdef CONFIG_IP_ROUTE_CLASSID
1871         rth->dst.tclassid = itag;
1872 #endif
1873         rth->rt_iif     =
1874         rth->fl.iif     = dev->ifindex;
1875         rth->dst.dev    = init_net.loopback_dev;
1876         dev_hold(rth->dst.dev);
1877         rth->fl.oif     = 0;
1878         rth->rt_gateway = daddr;
1879         rth->rt_spec_dst= spec_dst;
1880         rth->rt_genid   = rt_genid(dev_net(dev));
1881         rth->rt_flags   = RTCF_MULTICAST;
1882         rth->rt_type    = RTN_MULTICAST;
1883         if (our) {
1884                 rth->dst.input= ip_local_deliver;
1885                 rth->rt_flags |= RTCF_LOCAL;
1886         }
1887
1888 #ifdef CONFIG_IP_MROUTE
1889         if (!ipv4_is_local_multicast(daddr) && IN_DEV_MFORWARD(in_dev))
1890                 rth->dst.input = ip_mr_input;
1891 #endif
1892         RT_CACHE_STAT_INC(in_slow_mc);
1893
1894         hash = rt_hash(daddr, saddr, dev->ifindex, rt_genid(dev_net(dev)));
1895         rth = rt_intern_hash(hash, rth, skb, dev->ifindex);
1896         err = 0;
1897         if (IS_ERR(rth))
1898                 err = PTR_ERR(rth);
1899
1900 e_nobufs:
1901         return -ENOBUFS;
1902 e_inval:
1903         return -EINVAL;
1904 e_err:
1905         return err;
1906 }
1907
1908
1909 static void ip_handle_martian_source(struct net_device *dev,
1910                                      struct in_device *in_dev,
1911                                      struct sk_buff *skb,
1912                                      __be32 daddr,
1913                                      __be32 saddr)
1914 {
1915         RT_CACHE_STAT_INC(in_martian_src);
1916 #ifdef CONFIG_IP_ROUTE_VERBOSE
1917         if (IN_DEV_LOG_MARTIANS(in_dev) && net_ratelimit()) {
1918                 /*
1919                  *      RFC1812 recommendation, if source is martian,
1920                  *      the only hint is MAC header.
1921                  */
1922                 printk(KERN_WARNING "martian source %pI4 from %pI4, on dev %s\n",
1923                         &daddr, &saddr, dev->name);
1924                 if (dev->hard_header_len && skb_mac_header_was_set(skb)) {
1925                         int i;
1926                         const unsigned char *p = skb_mac_header(skb);
1927                         printk(KERN_WARNING "ll header: ");
1928                         for (i = 0; i < dev->hard_header_len; i++, p++) {
1929                                 printk("%02x", *p);
1930                                 if (i < (dev->hard_header_len - 1))
1931                                         printk(":");
1932                         }
1933                         printk("\n");
1934                 }
1935         }
1936 #endif
1937 }
1938
1939 /* called in rcu_read_lock() section */
1940 static int __mkroute_input(struct sk_buff *skb,
1941                            const struct fib_result *res,
1942                            struct in_device *in_dev,
1943                            __be32 daddr, __be32 saddr, u32 tos,
1944                            struct rtable **result)
1945 {
1946         struct rtable *rth;
1947         int err;
1948         struct in_device *out_dev;
1949         unsigned int flags = 0;
1950         __be32 spec_dst;
1951         u32 itag;
1952
1953         /* get a working reference to the output device */
1954         out_dev = __in_dev_get_rcu(FIB_RES_DEV(*res));
1955         if (out_dev == NULL) {
1956                 if (net_ratelimit())
1957                         printk(KERN_CRIT "Bug in ip_route_input" \
1958                                "_slow(). Please, report\n");
1959                 return -EINVAL;
1960         }
1961
1962
1963         err = fib_validate_source(saddr, daddr, tos, FIB_RES_OIF(*res),
1964                                   in_dev->dev, &spec_dst, &itag, skb->mark);
1965         if (err < 0) {
1966                 ip_handle_martian_source(in_dev->dev, in_dev, skb, daddr,
1967                                          saddr);
1968
1969                 goto cleanup;
1970         }
1971
1972         if (err)
1973                 flags |= RTCF_DIRECTSRC;
1974
1975         if (out_dev == in_dev && err &&
1976             (IN_DEV_SHARED_MEDIA(out_dev) ||
1977              inet_addr_onlink(out_dev, saddr, FIB_RES_GW(*res))))
1978                 flags |= RTCF_DOREDIRECT;
1979
1980         if (skb->protocol != htons(ETH_P_IP)) {
1981                 /* Not IP (i.e. ARP). Do not create route, if it is
1982                  * invalid for proxy arp. DNAT routes are always valid.
1983                  *
1984                  * Proxy arp feature have been extended to allow, ARP
1985                  * replies back to the same interface, to support
1986                  * Private VLAN switch technologies. See arp.c.
1987                  */
1988                 if (out_dev == in_dev &&
1989                     IN_DEV_PROXY_ARP_PVLAN(in_dev) == 0) {
1990                         err = -EINVAL;
1991                         goto cleanup;
1992                 }
1993         }
1994
1995         rth = rt_dst_alloc(IN_DEV_CONF_GET(in_dev, NOPOLICY),
1996                            IN_DEV_CONF_GET(out_dev, NOXFRM));
1997         if (!rth) {
1998                 err = -ENOBUFS;
1999                 goto cleanup;
2000         }
2001
2002         rth->fl.fl4_dst = daddr;
2003         rth->rt_dst     = daddr;
2004         rth->fl.fl4_tos = tos;
2005         rth->fl.mark    = skb->mark;
2006         rth->fl.fl4_src = saddr;
2007         rth->rt_src     = saddr;
2008         rth->rt_gateway = daddr;
2009         rth->rt_iif     =
2010                 rth->fl.iif     = in_dev->dev->ifindex;
2011         rth->dst.dev    = (out_dev)->dev;
2012         dev_hold(rth->dst.dev);
2013         rth->fl.oif     = 0;
2014         rth->rt_spec_dst= spec_dst;
2015
2016         rth->dst.input = ip_forward;
2017         rth->dst.output = ip_output;
2018         rth->rt_genid = rt_genid(dev_net(rth->dst.dev));
2019
2020         rt_set_nexthop(rth, res, res->fi, res->type, itag);
2021
2022         rth->rt_flags = flags;
2023
2024         *result = rth;
2025         err = 0;
2026  cleanup:
2027         return err;
2028 }
2029
2030 static int ip_mkroute_input(struct sk_buff *skb,
2031                             struct fib_result *res,
2032                             const struct flowi *fl,
2033                             struct in_device *in_dev,
2034                             __be32 daddr, __be32 saddr, u32 tos)
2035 {
2036         struct rtable* rth = NULL;
2037         int err;
2038         unsigned hash;
2039
2040 #ifdef CONFIG_IP_ROUTE_MULTIPATH
2041         if (res->fi && res->fi->fib_nhs > 1 && fl->oif == 0)
2042                 fib_select_multipath(fl, res);
2043 #endif
2044
2045         /* create a routing cache entry */
2046         err = __mkroute_input(skb, res, in_dev, daddr, saddr, tos, &rth);
2047         if (err)
2048                 return err;
2049
2050         /* put it into the cache */
2051         hash = rt_hash(daddr, saddr, fl->iif,
2052                        rt_genid(dev_net(rth->dst.dev)));
2053         rth = rt_intern_hash(hash, rth, skb, fl->iif);
2054         if (IS_ERR(rth))
2055                 return PTR_ERR(rth);
2056         return 0;
2057 }
2058
2059 /*
2060  *      NOTE. We drop all the packets that has local source
2061  *      addresses, because every properly looped back packet
2062  *      must have correct destination already attached by output routine.
2063  *
2064  *      Such approach solves two big problems:
2065  *      1. Not simplex devices are handled properly.
2066  *      2. IP spoofing attempts are filtered with 100% of guarantee.
2067  *      called with rcu_read_lock()
2068  */
2069
2070 static int ip_route_input_slow(struct sk_buff *skb, __be32 daddr, __be32 saddr,
2071                                u8 tos, struct net_device *dev)
2072 {
2073         struct fib_result res;
2074         struct in_device *in_dev = __in_dev_get_rcu(dev);
2075         struct flowi fl = { .fl4_dst    = daddr,
2076                             .fl4_src    = saddr,
2077                             .fl4_tos    = tos,
2078                             .fl4_scope  = RT_SCOPE_UNIVERSE,
2079                             .mark = skb->mark,
2080                             .iif = dev->ifindex };
2081         unsigned        flags = 0;
2082         u32             itag = 0;
2083         struct rtable * rth;
2084         unsigned        hash;
2085         __be32          spec_dst;
2086         int             err = -EINVAL;
2087         struct net    * net = dev_net(dev);
2088
2089         /* IP on this device is disabled. */
2090
2091         if (!in_dev)
2092                 goto out;
2093
2094         /* Check for the most weird martians, which can be not detected
2095            by fib_lookup.
2096          */
2097
2098         if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) ||
2099             ipv4_is_loopback(saddr))
2100                 goto martian_source;
2101
2102         if (ipv4_is_lbcast(daddr) || (saddr == 0 && daddr == 0))
2103                 goto brd_input;
2104
2105         /* Accept zero addresses only to limited broadcast;
2106          * I even do not know to fix it or not. Waiting for complains :-)
2107          */
2108         if (ipv4_is_zeronet(saddr))
2109                 goto martian_source;
2110
2111         if (ipv4_is_zeronet(daddr) || ipv4_is_loopback(daddr))
2112                 goto martian_destination;
2113
2114         /*
2115          *      Now we are ready to route packet.
2116          */
2117         err = fib_lookup(net, &fl, &res);
2118         if (err != 0) {
2119                 if (!IN_DEV_FORWARD(in_dev))
2120                         goto e_hostunreach;
2121                 goto no_route;
2122         }
2123
2124         RT_CACHE_STAT_INC(in_slow_tot);
2125
2126         if (res.type == RTN_BROADCAST)
2127                 goto brd_input;
2128
2129         if (res.type == RTN_LOCAL) {
2130                 err = fib_validate_source(saddr, daddr, tos,
2131                                           net->loopback_dev->ifindex,
2132                                           dev, &spec_dst, &itag, skb->mark);
2133                 if (err < 0)
2134                         goto martian_source_keep_err;
2135                 if (err)
2136                         flags |= RTCF_DIRECTSRC;
2137                 spec_dst = daddr;
2138                 goto local_input;
2139         }
2140
2141         if (!IN_DEV_FORWARD(in_dev))
2142                 goto e_hostunreach;
2143         if (res.type != RTN_UNICAST)
2144                 goto martian_destination;
2145
2146         err = ip_mkroute_input(skb, &res, &fl, in_dev, daddr, saddr, tos);
2147 out:    return err;
2148
2149 brd_input:
2150         if (skb->protocol != htons(ETH_P_IP))
2151                 goto e_inval;
2152
2153         if (ipv4_is_zeronet(saddr))
2154                 spec_dst = inet_select_addr(dev, 0, RT_SCOPE_LINK);
2155         else {
2156                 err = fib_validate_source(saddr, 0, tos, 0, dev, &spec_dst,
2157                                           &itag, skb->mark);
2158                 if (err < 0)
2159                         goto martian_source_keep_err;
2160                 if (err)
2161                         flags |= RTCF_DIRECTSRC;
2162         }
2163         flags |= RTCF_BROADCAST;
2164         res.type = RTN_BROADCAST;
2165         RT_CACHE_STAT_INC(in_brd);
2166
2167 local_input:
2168         rth = rt_dst_alloc(IN_DEV_CONF_GET(in_dev, NOPOLICY), false);
2169         if (!rth)
2170                 goto e_nobufs;
2171
2172         rth->dst.output= ip_rt_bug;
2173         rth->rt_genid = rt_genid(net);
2174
2175         rth->fl.fl4_dst = daddr;
2176         rth->rt_dst     = daddr;
2177         rth->fl.fl4_tos = tos;
2178         rth->fl.mark    = skb->mark;
2179         rth->fl.fl4_src = saddr;
2180         rth->rt_src     = saddr;
2181 #ifdef CONFIG_IP_ROUTE_CLASSID
2182         rth->dst.tclassid = itag;
2183 #endif
2184         rth->rt_iif     =
2185         rth->fl.iif     = dev->ifindex;
2186         rth->dst.dev    = net->loopback_dev;
2187         dev_hold(rth->dst.dev);
2188         rth->rt_gateway = daddr;
2189         rth->rt_spec_dst= spec_dst;
2190         rth->dst.input= ip_local_deliver;
2191         rth->rt_flags   = flags|RTCF_LOCAL;
2192         if (res.type == RTN_UNREACHABLE) {
2193                 rth->dst.input= ip_error;
2194                 rth->dst.error= -err;
2195                 rth->rt_flags   &= ~RTCF_LOCAL;
2196         }
2197         rth->rt_type    = res.type;
2198         hash = rt_hash(daddr, saddr, fl.iif, rt_genid(net));
2199         rth = rt_intern_hash(hash, rth, skb, fl.iif);
2200         err = 0;
2201         if (IS_ERR(rth))
2202                 err = PTR_ERR(rth);
2203         goto out;
2204
2205 no_route:
2206         RT_CACHE_STAT_INC(in_no_route);
2207         spec_dst = inet_select_addr(dev, 0, RT_SCOPE_UNIVERSE);
2208         res.type = RTN_UNREACHABLE;
2209         if (err == -ESRCH)
2210                 err = -ENETUNREACH;
2211         goto local_input;
2212
2213         /*
2214          *      Do not cache martian addresses: they should be logged (RFC1812)
2215          */
2216 martian_destination:
2217         RT_CACHE_STAT_INC(in_martian_dst);
2218 #ifdef CONFIG_IP_ROUTE_VERBOSE
2219         if (IN_DEV_LOG_MARTIANS(in_dev) && net_ratelimit())
2220                 printk(KERN_WARNING "martian destination %pI4 from %pI4, dev %s\n",
2221                         &daddr, &saddr, dev->name);
2222 #endif
2223
2224 e_hostunreach:
2225         err = -EHOSTUNREACH;
2226         goto out;
2227
2228 e_inval:
2229         err = -EINVAL;
2230         goto out;
2231
2232 e_nobufs:
2233         err = -ENOBUFS;
2234         goto out;
2235
2236 martian_source:
2237         err = -EINVAL;
2238 martian_source_keep_err:
2239         ip_handle_martian_source(dev, in_dev, skb, daddr, saddr);
2240         goto out;
2241 }
2242
2243 int ip_route_input_common(struct sk_buff *skb, __be32 daddr, __be32 saddr,
2244                            u8 tos, struct net_device *dev, bool noref)
2245 {
2246         struct rtable * rth;
2247         unsigned        hash;
2248         int iif = dev->ifindex;
2249         struct net *net;
2250         int res;
2251
2252         net = dev_net(dev);
2253
2254         rcu_read_lock();
2255
2256         if (!rt_caching(net))
2257                 goto skip_cache;
2258
2259         tos &= IPTOS_RT_MASK;
2260         hash = rt_hash(daddr, saddr, iif, rt_genid(net));
2261
2262         for (rth = rcu_dereference(rt_hash_table[hash].chain); rth;
2263              rth = rcu_dereference(rth->dst.rt_next)) {
2264                 if ((((__force u32)rth->fl.fl4_dst ^ (__force u32)daddr) |
2265                      ((__force u32)rth->fl.fl4_src ^ (__force u32)saddr) |
2266                      (rth->fl.iif ^ iif) |
2267                      rth->fl.oif |
2268                      (rth->fl.fl4_tos ^ tos)) == 0 &&
2269                     rth->fl.mark == skb->mark &&
2270                     net_eq(dev_net(rth->dst.dev), net) &&
2271                     !rt_is_expired(rth)) {
2272                         if (noref) {
2273                                 dst_use_noref(&rth->dst, jiffies);
2274                                 skb_dst_set_noref(skb, &rth->dst);
2275                         } else {
2276                                 dst_use(&rth->dst, jiffies);
2277                                 skb_dst_set(skb, &rth->dst);
2278                         }
2279                         RT_CACHE_STAT_INC(in_hit);
2280                         rcu_read_unlock();
2281                         return 0;
2282                 }
2283                 RT_CACHE_STAT_INC(in_hlist_search);
2284         }
2285
2286 skip_cache:
2287         /* Multicast recognition logic is moved from route cache to here.
2288            The problem was that too many Ethernet cards have broken/missing
2289            hardware multicast filters :-( As result the host on multicasting
2290            network acquires a lot of useless route cache entries, sort of
2291            SDR messages from all the world. Now we try to get rid of them.
2292            Really, provided software IP multicast filter is organized
2293            reasonably (at least, hashed), it does not result in a slowdown
2294            comparing with route cache reject entries.
2295            Note, that multicast routers are not affected, because
2296            route cache entry is created eventually.
2297          */
2298         if (ipv4_is_multicast(daddr)) {
2299                 struct in_device *in_dev = __in_dev_get_rcu(dev);
2300
2301                 if (in_dev) {
2302                         int our = ip_check_mc(in_dev, daddr, saddr,
2303                                               ip_hdr(skb)->protocol);
2304                         if (our
2305 #ifdef CONFIG_IP_MROUTE
2306                                 ||
2307                             (!ipv4_is_local_multicast(daddr) &&
2308                              IN_DEV_MFORWARD(in_dev))
2309 #endif
2310                            ) {
2311                                 int res = ip_route_input_mc(skb, daddr, saddr,
2312                                                             tos, dev, our);
2313                                 rcu_read_unlock();
2314                                 return res;
2315                         }
2316                 }
2317                 rcu_read_unlock();
2318                 return -EINVAL;
2319         }
2320         res = ip_route_input_slow(skb, daddr, saddr, tos, dev);
2321         rcu_read_unlock();
2322         return res;
2323 }
2324 EXPORT_SYMBOL(ip_route_input_common);
2325
2326 /* called with rcu_read_lock() */
2327 static struct rtable *__mkroute_output(const struct fib_result *res,
2328                                        const struct flowi *fl,
2329                                        const struct flowi *oldflp,
2330                                        struct net_device *dev_out,
2331                                        unsigned int flags)
2332 {
2333         struct fib_info *fi = res->fi;
2334         u32 tos = RT_FL_TOS(oldflp);
2335         struct in_device *in_dev;
2336         u16 type = res->type;
2337         struct rtable *rth;
2338
2339         if (ipv4_is_loopback(fl->fl4_src) && !(dev_out->flags & IFF_LOOPBACK))
2340                 return ERR_PTR(-EINVAL);
2341
2342         if (ipv4_is_lbcast(fl->fl4_dst))
2343                 type = RTN_BROADCAST;
2344         else if (ipv4_is_multicast(fl->fl4_dst))
2345                 type = RTN_MULTICAST;
2346         else if (ipv4_is_zeronet(fl->fl4_dst))
2347                 return ERR_PTR(-EINVAL);
2348
2349         if (dev_out->flags & IFF_LOOPBACK)
2350                 flags |= RTCF_LOCAL;
2351
2352         in_dev = __in_dev_get_rcu(dev_out);
2353         if (!in_dev)
2354                 return ERR_PTR(-EINVAL);
2355
2356         if (type == RTN_BROADCAST) {
2357                 flags |= RTCF_BROADCAST | RTCF_LOCAL;
2358                 fi = NULL;
2359         } else if (type == RTN_MULTICAST) {
2360                 flags |= RTCF_MULTICAST | RTCF_LOCAL;
2361                 if (!ip_check_mc(in_dev, oldflp->fl4_dst, oldflp->fl4_src,
2362                                  oldflp->proto))
2363                         flags &= ~RTCF_LOCAL;
2364                 /* If multicast route do not exist use
2365                  * default one, but do not gateway in this case.
2366                  * Yes, it is hack.
2367                  */
2368                 if (fi && res->prefixlen < 4)
2369                         fi = NULL;
2370         }
2371
2372         rth = rt_dst_alloc(IN_DEV_CONF_GET(in_dev, NOPOLICY),
2373                            IN_DEV_CONF_GET(in_dev, NOXFRM));
2374         if (!rth)
2375                 return ERR_PTR(-ENOBUFS);
2376
2377         rth->fl.fl4_dst = oldflp->fl4_dst;
2378         rth->fl.fl4_tos = tos;
2379         rth->fl.fl4_src = oldflp->fl4_src;
2380         rth->fl.oif     = oldflp->oif;
2381         rth->fl.mark    = oldflp->mark;
2382         rth->rt_dst     = fl->fl4_dst;
2383         rth->rt_src     = fl->fl4_src;
2384         rth->rt_iif     = oldflp->oif ? : dev_out->ifindex;
2385         /* get references to the devices that are to be hold by the routing
2386            cache entry */
2387         rth->dst.dev    = dev_out;
2388         dev_hold(dev_out);
2389         rth->rt_gateway = fl->fl4_dst;
2390         rth->rt_spec_dst= fl->fl4_src;
2391
2392         rth->dst.output=ip_output;
2393         rth->rt_genid = rt_genid(dev_net(dev_out));
2394
2395         RT_CACHE_STAT_INC(out_slow_tot);
2396
2397         if (flags & RTCF_LOCAL) {
2398                 rth->dst.input = ip_local_deliver;
2399                 rth->rt_spec_dst = fl->fl4_dst;
2400         }
2401         if (flags & (RTCF_BROADCAST | RTCF_MULTICAST)) {
2402                 rth->rt_spec_dst = fl->fl4_src;
2403                 if (flags & RTCF_LOCAL &&
2404                     !(dev_out->flags & IFF_LOOPBACK)) {
2405                         rth->dst.output = ip_mc_output;
2406                         RT_CACHE_STAT_INC(out_slow_mc);
2407                 }
2408 #ifdef CONFIG_IP_MROUTE
2409                 if (type == RTN_MULTICAST) {
2410                         if (IN_DEV_MFORWARD(in_dev) &&
2411                             !ipv4_is_local_multicast(oldflp->fl4_dst)) {
2412                                 rth->dst.input = ip_mr_input;
2413                                 rth->dst.output = ip_mc_output;
2414                         }
2415                 }
2416 #endif
2417         }
2418
2419         rt_set_nexthop(rth, res, fi, type, 0);
2420
2421         rth->rt_flags = flags;
2422         return rth;
2423 }
2424
2425 /*
2426  * Major route resolver routine.
2427  * called with rcu_read_lock();
2428  */
2429
2430 static struct rtable *ip_route_output_slow(struct net *net,
2431                                            const struct flowi *oldflp)
2432 {
2433         u32 tos = RT_FL_TOS(oldflp);
2434         struct flowi fl = { .fl4_dst = oldflp->fl4_dst,
2435                             .fl4_src = oldflp->fl4_src,
2436                             .fl4_tos = tos & IPTOS_RT_MASK,
2437                             .fl4_scope = ((tos & RTO_ONLINK) ?
2438                                           RT_SCOPE_LINK : RT_SCOPE_UNIVERSE),
2439                             .mark = oldflp->mark,
2440                             .iif = net->loopback_dev->ifindex,
2441                             .oif = oldflp->oif };
2442         struct fib_result res;
2443         unsigned int flags = 0;
2444         struct net_device *dev_out = NULL;
2445         struct rtable *rth;
2446
2447         res.fi          = NULL;
2448 #ifdef CONFIG_IP_MULTIPLE_TABLES
2449         res.r           = NULL;
2450 #endif
2451
2452         rcu_read_lock();
2453         if (oldflp->fl4_src) {
2454                 rth = ERR_PTR(-EINVAL);
2455                 if (ipv4_is_multicast(oldflp->fl4_src) ||
2456                     ipv4_is_lbcast(oldflp->fl4_src) ||
2457                     ipv4_is_zeronet(oldflp->fl4_src))
2458                         goto out;
2459
2460                 /* I removed check for oif == dev_out->oif here.
2461                    It was wrong for two reasons:
2462                    1. ip_dev_find(net, saddr) can return wrong iface, if saddr
2463                       is assigned to multiple interfaces.
2464                    2. Moreover, we are allowed to send packets with saddr
2465                       of another iface. --ANK
2466                  */
2467
2468                 if (oldflp->oif == 0 &&
2469                     (ipv4_is_multicast(oldflp->fl4_dst) ||
2470                      ipv4_is_lbcast(oldflp->fl4_dst))) {
2471                         /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
2472                         dev_out = __ip_dev_find(net, oldflp->fl4_src, false);
2473                         if (dev_out == NULL)
2474                                 goto out;
2475
2476                         /* Special hack: user can direct multicasts
2477                            and limited broadcast via necessary interface
2478                            without fiddling with IP_MULTICAST_IF or IP_PKTINFO.
2479                            This hack is not just for fun, it allows
2480                            vic,vat and friends to work.
2481                            They bind socket to loopback, set ttl to zero
2482                            and expect that it will work.
2483                            From the viewpoint of routing cache they are broken,
2484                            because we are not allowed to build multicast path
2485                            with loopback source addr (look, routing cache
2486                            cannot know, that ttl is zero, so that packet
2487                            will not leave this host and route is valid).
2488                            Luckily, this hack is good workaround.
2489                          */
2490
2491                         fl.oif = dev_out->ifindex;
2492                         goto make_route;
2493                 }
2494
2495                 if (!(oldflp->flags & FLOWI_FLAG_ANYSRC)) {
2496                         /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
2497                         if (!__ip_dev_find(net, oldflp->fl4_src, false))
2498                                 goto out;
2499                 }
2500         }
2501
2502
2503         if (oldflp->oif) {
2504                 dev_out = dev_get_by_index_rcu(net, oldflp->oif);
2505                 rth = ERR_PTR(-ENODEV);
2506                 if (dev_out == NULL)
2507                         goto out;
2508
2509                 /* RACE: Check return value of inet_select_addr instead. */
2510                 if (!(dev_out->flags & IFF_UP) || !__in_dev_get_rcu(dev_out)) {
2511                         rth = ERR_PTR(-ENETUNREACH);
2512                         goto out;
2513                 }
2514                 if (ipv4_is_local_multicast(oldflp->fl4_dst) ||
2515                     ipv4_is_lbcast(oldflp->fl4_dst)) {
2516                         if (!fl.fl4_src)
2517                                 fl.fl4_src = inet_select_addr(dev_out, 0,
2518                                                               RT_SCOPE_LINK);
2519                         goto make_route;
2520                 }
2521                 if (!fl.fl4_src) {
2522                         if (ipv4_is_multicast(oldflp->fl4_dst))
2523                                 fl.fl4_src = inet_select_addr(dev_out, 0,
2524                                                               fl.fl4_scope);
2525                         else if (!oldflp->fl4_dst)
2526                                 fl.fl4_src = inet_select_addr(dev_out, 0,
2527                                                               RT_SCOPE_HOST);
2528                 }
2529         }
2530
2531         if (!fl.fl4_dst) {
2532                 fl.fl4_dst = fl.fl4_src;
2533                 if (!fl.fl4_dst)
2534                         fl.fl4_dst = fl.fl4_src = htonl(INADDR_LOOPBACK);
2535                 dev_out = net->loopback_dev;
2536                 fl.oif = net->loopback_dev->ifindex;
2537                 res.type = RTN_LOCAL;
2538                 flags |= RTCF_LOCAL;
2539                 goto make_route;
2540         }
2541
2542         if (fib_lookup(net, &fl, &res)) {
2543                 res.fi = NULL;
2544                 if (oldflp->oif) {
2545                         /* Apparently, routing tables are wrong. Assume,
2546                            that the destination is on link.
2547
2548                            WHY? DW.
2549                            Because we are allowed to send to iface
2550                            even if it has NO routes and NO assigned
2551                            addresses. When oif is specified, routing
2552                            tables are looked up with only one purpose:
2553                            to catch if destination is gatewayed, rather than
2554                            direct. Moreover, if MSG_DONTROUTE is set,
2555                            we send packet, ignoring both routing tables
2556                            and ifaddr state. --ANK
2557
2558
2559                            We could make it even if oif is unknown,
2560                            likely IPv6, but we do not.
2561                          */
2562
2563                         if (fl.fl4_src == 0)
2564                                 fl.fl4_src = inet_select_addr(dev_out, 0,
2565                                                               RT_SCOPE_LINK);
2566                         res.type = RTN_UNICAST;
2567                         goto make_route;
2568                 }
2569                 rth = ERR_PTR(-ENETUNREACH);
2570                 goto out;
2571         }
2572
2573         if (res.type == RTN_LOCAL) {
2574                 if (!fl.fl4_src) {
2575                         if (res.fi->fib_prefsrc)
2576                                 fl.fl4_src = res.fi->fib_prefsrc;
2577                         else
2578                                 fl.fl4_src = fl.fl4_dst;
2579                 }
2580                 dev_out = net->loopback_dev;
2581                 fl.oif = dev_out->ifindex;
2582                 res.fi = NULL;
2583                 flags |= RTCF_LOCAL;
2584                 goto make_route;
2585         }
2586
2587 #ifdef CONFIG_IP_ROUTE_MULTIPATH
2588         if (res.fi->fib_nhs > 1 && fl.oif == 0)
2589                 fib_select_multipath(&fl, &res);
2590         else
2591 #endif
2592         if (!res.prefixlen && res.type == RTN_UNICAST && !fl.oif)
2593                 fib_select_default(&res);
2594
2595         if (!fl.fl4_src)
2596                 fl.fl4_src = FIB_RES_PREFSRC(res);
2597
2598         dev_out = FIB_RES_DEV(res);
2599         fl.oif = dev_out->ifindex;
2600
2601
2602 make_route:
2603         rth = __mkroute_output(&res, &fl, oldflp, dev_out, flags);
2604         if (!IS_ERR(rth)) {
2605                 unsigned int hash;
2606
2607                 hash = rt_hash(oldflp->fl4_dst, oldflp->fl4_src, oldflp->oif,
2608                                rt_genid(dev_net(dev_out)));
2609                 rth = rt_intern_hash(hash, rth, NULL, oldflp->oif);
2610         }
2611
2612 out:
2613         rcu_read_unlock();
2614         return rth;
2615 }
2616
2617 struct rtable *__ip_route_output_key(struct net *net, const struct flowi *flp)
2618 {
2619         struct rtable *rth;
2620         unsigned int hash;
2621
2622         if (!rt_caching(net))
2623                 goto slow_output;
2624
2625         hash = rt_hash(flp->fl4_dst, flp->fl4_src, flp->oif, rt_genid(net));
2626
2627         rcu_read_lock_bh();
2628         for (rth = rcu_dereference_bh(rt_hash_table[hash].chain); rth;
2629                 rth = rcu_dereference_bh(rth->dst.rt_next)) {
2630                 if (rth->fl.fl4_dst == flp->fl4_dst &&
2631                     rth->fl.fl4_src == flp->fl4_src &&
2632                     rt_is_output_route(rth) &&
2633                     rth->fl.oif == flp->oif &&
2634                     rth->fl.mark == flp->mark &&
2635                     !((rth->fl.fl4_tos ^ flp->fl4_tos) &
2636                             (IPTOS_RT_MASK | RTO_ONLINK)) &&
2637                     net_eq(dev_net(rth->dst.dev), net) &&
2638                     !rt_is_expired(rth)) {
2639                         dst_use(&rth->dst, jiffies);
2640                         RT_CACHE_STAT_INC(out_hit);
2641                         rcu_read_unlock_bh();
2642                         return rth;
2643                 }
2644                 RT_CACHE_STAT_INC(out_hlist_search);
2645         }
2646         rcu_read_unlock_bh();
2647
2648 slow_output:
2649         return ip_route_output_slow(net, flp);
2650 }
2651 EXPORT_SYMBOL_GPL(__ip_route_output_key);
2652
2653 static struct dst_entry *ipv4_blackhole_dst_check(struct dst_entry *dst, u32 cookie)
2654 {
2655         return NULL;
2656 }
2657
2658 static unsigned int ipv4_blackhole_default_mtu(const struct dst_entry *dst)
2659 {
2660         return 0;
2661 }
2662
2663 static void ipv4_rt_blackhole_update_pmtu(struct dst_entry *dst, u32 mtu)
2664 {
2665 }
2666
2667 static struct dst_ops ipv4_dst_blackhole_ops = {
2668         .family                 =       AF_INET,
2669         .protocol               =       cpu_to_be16(ETH_P_IP),
2670         .destroy                =       ipv4_dst_destroy,
2671         .check                  =       ipv4_blackhole_dst_check,
2672         .default_mtu            =       ipv4_blackhole_default_mtu,
2673         .default_advmss         =       ipv4_default_advmss,
2674         .update_pmtu            =       ipv4_rt_blackhole_update_pmtu,
2675 };
2676
2677 struct dst_entry *ipv4_blackhole_route(struct net *net, struct dst_entry *dst_orig)
2678 {
2679         struct rtable *rt = dst_alloc(&ipv4_dst_blackhole_ops, 1);
2680         struct rtable *ort = (struct rtable *) dst_orig;
2681
2682         if (rt) {
2683                 struct dst_entry *new = &rt->dst;
2684
2685                 new->__use = 1;
2686                 new->input = dst_discard;
2687                 new->output = dst_discard;
2688                 dst_copy_metrics(new, &ort->dst);
2689
2690                 new->dev = ort->dst.dev;
2691                 if (new->dev)
2692                         dev_hold(new->dev);
2693
2694                 rt->fl = ort->fl;
2695
2696                 rt->rt_genid = rt_genid(net);
2697                 rt->rt_flags = ort->rt_flags;
2698                 rt->rt_type = ort->rt_type;
2699                 rt->rt_dst = ort->rt_dst;
2700                 rt->rt_src = ort->rt_src;
2701                 rt->rt_iif = ort->rt_iif;
2702                 rt->rt_gateway = ort->rt_gateway;
2703                 rt->rt_spec_dst = ort->rt_spec_dst;
2704                 rt->peer = ort->peer;
2705                 if (rt->peer)
2706                         atomic_inc(&rt->peer->refcnt);
2707                 rt->fi = ort->fi;
2708                 if (rt->fi)
2709                         atomic_inc(&rt->fi->fib_clntref);
2710
2711                 dst_free(new);
2712         }
2713
2714         dst_release(dst_orig);
2715
2716         return rt ? &rt->dst : ERR_PTR(-ENOMEM);
2717 }
2718
2719 struct rtable *ip_route_output_flow(struct net *net, struct flowi *flp,
2720                                     struct sock *sk)
2721 {
2722         struct rtable *rt = __ip_route_output_key(net, flp);
2723
2724         if (IS_ERR(rt))
2725                 return rt;
2726
2727         if (flp->proto) {
2728                 if (!flp->fl4_src)
2729                         flp->fl4_src = rt->rt_src;
2730                 if (!flp->fl4_dst)
2731                         flp->fl4_dst = rt->rt_dst;
2732                 rt = (struct rtable *) xfrm_lookup(net, &rt->dst, flp, sk, 0);
2733         }
2734
2735         return rt;
2736 }
2737 EXPORT_SYMBOL_GPL(ip_route_output_flow);
2738
2739 struct rtable *ip_route_output_key(struct net *net, struct flowi *flp)
2740 {
2741         return ip_route_output_flow(net, flp, NULL);
2742 }
2743 EXPORT_SYMBOL(ip_route_output_key);
2744
2745 static int rt_fill_info(struct net *net,
2746                         struct sk_buff *skb, u32 pid, u32 seq, int event,
2747                         int nowait, unsigned int flags)
2748 {
2749         struct rtable *rt = skb_rtable(skb);
2750         struct rtmsg *r;
2751         struct nlmsghdr *nlh;
2752         long expires;
2753         u32 id = 0, ts = 0, tsage = 0, error;
2754
2755         nlh = nlmsg_put(skb, pid, seq, event, sizeof(*r), flags);
2756         if (nlh == NULL)
2757                 return -EMSGSIZE;
2758
2759         r = nlmsg_data(nlh);
2760         r->rtm_family    = AF_INET;
2761         r->rtm_dst_len  = 32;
2762         r->rtm_src_len  = 0;
2763         r->rtm_tos      = rt->fl.fl4_tos;
2764         r->rtm_table    = RT_TABLE_MAIN;
2765         NLA_PUT_U32(skb, RTA_TABLE, RT_TABLE_MAIN);
2766         r->rtm_type     = rt->rt_type;
2767         r->rtm_scope    = RT_SCOPE_UNIVERSE;
2768         r->rtm_protocol = RTPROT_UNSPEC;
2769         r->rtm_flags    = (rt->rt_flags & ~0xFFFF) | RTM_F_CLONED;
2770         if (rt->rt_flags & RTCF_NOTIFY)
2771                 r->rtm_flags |= RTM_F_NOTIFY;
2772
2773         NLA_PUT_BE32(skb, RTA_DST, rt->rt_dst);
2774
2775         if (rt->fl.fl4_src) {
2776                 r->rtm_src_len = 32;
2777                 NLA_PUT_BE32(skb, RTA_SRC, rt->fl.fl4_src);
2778         }
2779         if (rt->dst.dev)
2780                 NLA_PUT_U32(skb, RTA_OIF, rt->dst.dev->ifindex);
2781 #ifdef CONFIG_IP_ROUTE_CLASSID
2782         if (rt->dst.tclassid)
2783                 NLA_PUT_U32(skb, RTA_FLOW, rt->dst.tclassid);
2784 #endif
2785         if (rt_is_input_route(rt))
2786                 NLA_PUT_BE32(skb, RTA_PREFSRC, rt->rt_spec_dst);
2787         else if (rt->rt_src != rt->fl.fl4_src)
2788                 NLA_PUT_BE32(skb, RTA_PREFSRC, rt->rt_src);
2789
2790         if (rt->rt_dst != rt->rt_gateway)
2791                 NLA_PUT_BE32(skb, RTA_GATEWAY, rt->rt_gateway);
2792
2793         if (rtnetlink_put_metrics(skb, dst_metrics_ptr(&rt->dst)) < 0)
2794                 goto nla_put_failure;
2795
2796         if (rt->fl.mark)
2797                 NLA_PUT_BE32(skb, RTA_MARK, rt->fl.mark);
2798
2799         error = rt->dst.error;
2800         expires = (rt->peer && rt->peer->pmtu_expires) ?
2801                 rt->peer->pmtu_expires - jiffies : 0;
2802         if (rt->peer) {
2803                 inet_peer_refcheck(rt->peer);
2804                 id = atomic_read(&rt->peer->ip_id_count) & 0xffff;
2805                 if (rt->peer->tcp_ts_stamp) {
2806                         ts = rt->peer->tcp_ts;
2807                         tsage = get_seconds() - rt->peer->tcp_ts_stamp;
2808                 }
2809         }
2810
2811         if (rt_is_input_route(rt)) {
2812 #ifdef CONFIG_IP_MROUTE
2813                 __be32 dst = rt->rt_dst;
2814
2815                 if (ipv4_is_multicast(dst) && !ipv4_is_local_multicast(dst) &&
2816                     IPV4_DEVCONF_ALL(net, MC_FORWARDING)) {
2817                         int err = ipmr_get_route(net, skb, r, nowait);
2818                         if (err <= 0) {
2819                                 if (!nowait) {
2820                                         if (err == 0)
2821                                                 return 0;
2822                                         goto nla_put_failure;
2823                                 } else {
2824                                         if (err == -EMSGSIZE)
2825                                                 goto nla_put_failure;
2826                                         error = err;
2827                                 }
2828                         }
2829                 } else
2830 #endif
2831                         NLA_PUT_U32(skb, RTA_IIF, rt->fl.iif);
2832         }
2833
2834         if (rtnl_put_cacheinfo(skb, &rt->dst, id, ts, tsage,
2835                                expires, error) < 0)
2836                 goto nla_put_failure;
2837
2838         return nlmsg_end(skb, nlh);
2839
2840 nla_put_failure:
2841         nlmsg_cancel(skb, nlh);
2842         return -EMSGSIZE;
2843 }
2844
2845 static int inet_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr* nlh, void *arg)
2846 {
2847         struct net *net = sock_net(in_skb->sk);
2848         struct rtmsg *rtm;
2849         struct nlattr *tb[RTA_MAX+1];
2850         struct rtable *rt = NULL;
2851         __be32 dst = 0;
2852         __be32 src = 0;
2853         u32 iif;
2854         int err;
2855         int mark;
2856         struct sk_buff *skb;
2857
2858         err = nlmsg_parse(nlh, sizeof(*rtm), tb, RTA_MAX, rtm_ipv4_policy);
2859         if (err < 0)
2860                 goto errout;
2861
2862         rtm = nlmsg_data(nlh);
2863
2864         skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
2865         if (skb == NULL) {
2866                 err = -ENOBUFS;
2867                 goto errout;
2868         }
2869
2870         /* Reserve room for dummy headers, this skb can pass
2871            through good chunk of routing engine.
2872          */
2873         skb_reset_mac_header(skb);
2874         skb_reset_network_header(skb);
2875
2876         /* Bugfix: need to give ip_route_input enough of an IP header to not gag. */
2877         ip_hdr(skb)->protocol = IPPROTO_ICMP;
2878         skb_reserve(skb, MAX_HEADER + sizeof(struct iphdr));
2879
2880         src = tb[RTA_SRC] ? nla_get_be32(tb[RTA_SRC]) : 0;
2881         dst = tb[RTA_DST] ? nla_get_be32(tb[RTA_DST]) : 0;
2882         iif = tb[RTA_IIF] ? nla_get_u32(tb[RTA_IIF]) : 0;
2883         mark = tb[RTA_MARK] ? nla_get_u32(tb[RTA_MARK]) : 0;
2884
2885         if (iif) {
2886                 struct net_device *dev;
2887
2888                 dev = __dev_get_by_index(net, iif);
2889                 if (dev == NULL) {
2890                         err = -ENODEV;
2891                         goto errout_free;
2892                 }
2893
2894                 skb->protocol   = htons(ETH_P_IP);
2895                 skb->dev        = dev;
2896                 skb->mark       = mark;
2897                 local_bh_disable();
2898                 err = ip_route_input(skb, dst, src, rtm->rtm_tos, dev);
2899                 local_bh_enable();
2900
2901                 rt = skb_rtable(skb);
2902                 if (err == 0 && rt->dst.error)
2903                         err = -rt->dst.error;
2904         } else {
2905                 struct flowi fl = {
2906                         .fl4_dst = dst,
2907                         .fl4_src = src,
2908                         .fl4_tos = rtm->rtm_tos,
2909                         .oif = tb[RTA_OIF] ? nla_get_u32(tb[RTA_OIF]) : 0,
2910                         .mark = mark,
2911                 };
2912                 rt = ip_route_output_key(net, &fl);
2913
2914                 err = 0;
2915                 if (IS_ERR(rt))
2916                         err = PTR_ERR(rt);
2917         }
2918
2919         if (err)
2920                 goto errout_free;
2921
2922         skb_dst_set(skb, &rt->dst);
2923         if (rtm->rtm_flags & RTM_F_NOTIFY)
2924                 rt->rt_flags |= RTCF_NOTIFY;
2925
2926         err = rt_fill_info(net, skb, NETLINK_CB(in_skb).pid, nlh->nlmsg_seq,
2927                            RTM_NEWROUTE, 0, 0);
2928         if (err <= 0)
2929                 goto errout_free;
2930
2931         err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).pid);
2932 errout:
2933         return err;
2934
2935 errout_free:
2936         kfree_skb(skb);
2937         goto errout;
2938 }
2939
2940 int ip_rt_dump(struct sk_buff *skb,  struct netlink_callback *cb)
2941 {
2942         struct rtable *rt;
2943         int h, s_h;
2944         int idx, s_idx;
2945         struct net *net;
2946
2947         net = sock_net(skb->sk);
2948
2949         s_h = cb->args[0];
2950         if (s_h < 0)
2951                 s_h = 0;
2952         s_idx = idx = cb->args[1];
2953         for (h = s_h; h <= rt_hash_mask; h++, s_idx = 0) {
2954                 if (!rt_hash_table[h].chain)
2955                         continue;
2956                 rcu_read_lock_bh();
2957                 for (rt = rcu_dereference_bh(rt_hash_table[h].chain), idx = 0; rt;
2958                      rt = rcu_dereference_bh(rt->dst.rt_next), idx++) {
2959                         if (!net_eq(dev_net(rt->dst.dev), net) || idx < s_idx)
2960                                 continue;
2961                         if (rt_is_expired(rt))
2962                                 continue;
2963                         skb_dst_set_noref(skb, &rt->dst);
2964                         if (rt_fill_info(net, skb, NETLINK_CB(cb->skb).pid,
2965                                          cb->nlh->nlmsg_seq, RTM_NEWROUTE,
2966                                          1, NLM_F_MULTI) <= 0) {
2967                                 skb_dst_drop(skb);
2968                                 rcu_read_unlock_bh();
2969                                 goto done;
2970                         }
2971                         skb_dst_drop(skb);
2972                 }
2973                 rcu_read_unlock_bh();
2974         }
2975
2976 done:
2977         cb->args[0] = h;
2978         cb->args[1] = idx;
2979         return skb->len;
2980 }
2981
2982 void ip_rt_multicast_event(struct in_device *in_dev)
2983 {
2984         rt_cache_flush(dev_net(in_dev->dev), 0);
2985 }
2986
2987 #ifdef CONFIG_SYSCTL
2988 static int ipv4_sysctl_rtcache_flush(ctl_table *__ctl, int write,
2989                                         void __user *buffer,
2990                                         size_t *lenp, loff_t *ppos)
2991 {
2992         if (write) {
2993                 int flush_delay;
2994                 ctl_table ctl;
2995                 struct net *net;
2996
2997                 memcpy(&ctl, __ctl, sizeof(ctl));
2998                 ctl.data = &flush_delay;
2999                 proc_dointvec(&ctl, write, buffer, lenp, ppos);
3000
3001                 net = (struct net *)__ctl->extra1;
3002                 rt_cache_flush(net, flush_delay);
3003                 return 0;
3004         }
3005
3006         return -EINVAL;
3007 }
3008
3009 static ctl_table ipv4_route_table[] = {
3010         {
3011                 .procname       = "gc_thresh",
3012                 .data           = &ipv4_dst_ops.gc_thresh,
3013                 .maxlen         = sizeof(int),
3014                 .mode           = 0644,
3015                 .proc_handler   = proc_dointvec,
3016         },
3017         {
3018                 .procname       = "max_size",
3019                 .data           = &ip_rt_max_size,
3020                 .maxlen         = sizeof(int),
3021                 .mode           = 0644,
3022                 .proc_handler   = proc_dointvec,
3023         },
3024         {
3025                 /*  Deprecated. Use gc_min_interval_ms */
3026
3027                 .procname       = "gc_min_interval",
3028                 .data           = &ip_rt_gc_min_interval,
3029                 .maxlen         = sizeof(int),
3030                 .mode           = 0644,
3031                 .proc_handler   = proc_dointvec_jiffies,
3032         },
3033         {
3034                 .procname       = "gc_min_interval_ms",
3035                 .data           = &ip_rt_gc_min_interval,
3036                 .maxlen         = sizeof(int),
3037                 .mode           = 0644,
3038                 .proc_handler   = proc_dointvec_ms_jiffies,
3039         },
3040         {
3041                 .procname       = "gc_timeout",
3042                 .data           = &ip_rt_gc_timeout,
3043                 .maxlen         = sizeof(int),
3044                 .mode           = 0644,
3045                 .proc_handler   = proc_dointvec_jiffies,
3046         },
3047         {
3048                 .procname       = "gc_interval",
3049                 .data           = &ip_rt_gc_interval,
3050                 .maxlen         = sizeof(int),
3051                 .mode           = 0644,
3052                 .proc_handler   = proc_dointvec_jiffies,
3053         },
3054         {
3055                 .procname       = "redirect_load",
3056                 .data           = &ip_rt_redirect_load,
3057                 .maxlen         = sizeof(int),
3058                 .mode           = 0644,
3059                 .proc_handler   = proc_dointvec,
3060         },
3061         {
3062                 .procname       = "redirect_number",
3063                 .data           = &ip_rt_redirect_number,
3064                 .maxlen         = sizeof(int),
3065                 .mode           = 0644,
3066                 .proc_handler   = proc_dointvec,
3067         },
3068         {
3069                 .procname       = "redirect_silence",
3070                 .data           = &ip_rt_redirect_silence,
3071                 .maxlen         = sizeof(int),
3072                 .mode           = 0644,
3073                 .proc_handler   = proc_dointvec,
3074         },
3075         {
3076                 .procname       = "error_cost",
3077                 .data           = &ip_rt_error_cost,
3078                 .maxlen         = sizeof(int),
3079                 .mode           = 0644,
3080                 .proc_handler   = proc_dointvec,
3081         },
3082         {
3083                 .procname       = "error_burst",
3084                 .data           = &ip_rt_error_burst,
3085                 .maxlen         = sizeof(int),
3086                 .mode           = 0644,
3087                 .proc_handler   = proc_dointvec,
3088         },
3089         {
3090                 .procname       = "gc_elasticity",
3091                 .data           = &ip_rt_gc_elasticity,
3092                 .maxlen         = sizeof(int),
3093                 .mode           = 0644,
3094                 .proc_handler   = proc_dointvec,
3095         },
3096         {
3097                 .procname       = "mtu_expires",
3098                 .data           = &ip_rt_mtu_expires,
3099                 .maxlen         = sizeof(int),
3100                 .mode           = 0644,
3101                 .proc_handler   = proc_dointvec_jiffies,
3102         },
3103         {
3104                 .procname       = "min_pmtu",
3105                 .data           = &ip_rt_min_pmtu,
3106                 .maxlen         = sizeof(int),
3107                 .mode           = 0644,
3108                 .proc_handler   = proc_dointvec,
3109         },
3110         {
3111                 .procname       = "min_adv_mss",
3112                 .data           = &ip_rt_min_advmss,
3113                 .maxlen         = sizeof(int),
3114                 .mode           = 0644,
3115                 .proc_handler   = proc_dointvec,
3116         },
3117         { }
3118 };
3119
3120 static struct ctl_table empty[1];
3121
3122 static struct ctl_table ipv4_skeleton[] =
3123 {
3124         { .procname = "route", 
3125           .mode = 0555, .child = ipv4_route_table},
3126         { .procname = "neigh", 
3127           .mode = 0555, .child = empty},
3128         { }
3129 };
3130
3131 static __net_initdata struct ctl_path ipv4_path[] = {
3132         { .procname = "net", },
3133         { .procname = "ipv4", },
3134         { },
3135 };
3136
3137 static struct ctl_table ipv4_route_flush_table[] = {
3138         {
3139                 .procname       = "flush",
3140                 .maxlen         = sizeof(int),
3141                 .mode           = 0200,
3142                 .proc_handler   = ipv4_sysctl_rtcache_flush,
3143         },
3144         { },
3145 };
3146
3147 static __net_initdata struct ctl_path ipv4_route_path[] = {
3148         { .procname = "net", },
3149         { .procname = "ipv4", },
3150         { .procname = "route", },
3151         { },
3152 };
3153
3154 static __net_init int sysctl_route_net_init(struct net *net)
3155 {
3156         struct ctl_table *tbl;
3157
3158         tbl = ipv4_route_flush_table;
3159         if (!net_eq(net, &init_net)) {
3160                 tbl = kmemdup(tbl, sizeof(ipv4_route_flush_table), GFP_KERNEL);
3161                 if (tbl == NULL)
3162                         goto err_dup;
3163         }
3164         tbl[0].extra1 = net;
3165
3166         net->ipv4.route_hdr =
3167                 register_net_sysctl_table(net, ipv4_route_path, tbl);
3168         if (net->ipv4.route_hdr == NULL)
3169                 goto err_reg;
3170         return 0;
3171
3172 err_reg:
3173         if (tbl != ipv4_route_flush_table)
3174                 kfree(tbl);
3175 err_dup:
3176         return -ENOMEM;
3177 }
3178
3179 static __net_exit void sysctl_route_net_exit(struct net *net)
3180 {
3181         struct ctl_table *tbl;
3182
3183         tbl = net->ipv4.route_hdr->ctl_table_arg;
3184         unregister_net_sysctl_table(net->ipv4.route_hdr);
3185         BUG_ON(tbl == ipv4_route_flush_table);
3186         kfree(tbl);
3187 }
3188
3189 static __net_initdata struct pernet_operations sysctl_route_ops = {
3190         .init = sysctl_route_net_init,
3191         .exit = sysctl_route_net_exit,
3192 };
3193 #endif
3194
3195 static __net_init int rt_genid_init(struct net *net)
3196 {
3197         get_random_bytes(&net->ipv4.rt_genid,
3198                          sizeof(net->ipv4.rt_genid));
3199         return 0;
3200 }
3201
3202 static __net_initdata struct pernet_operations rt_genid_ops = {
3203         .init = rt_genid_init,
3204 };
3205
3206
3207 #ifdef CONFIG_IP_ROUTE_CLASSID
3208 struct ip_rt_acct __percpu *ip_rt_acct __read_mostly;
3209 #endif /* CONFIG_IP_ROUTE_CLASSID */
3210
3211 static __initdata unsigned long rhash_entries;
3212 static int __init set_rhash_entries(char *str)
3213 {
3214         if (!str)
3215                 return 0;
3216         rhash_entries = simple_strtoul(str, &str, 0);
3217         return 1;
3218 }
3219 __setup("rhash_entries=", set_rhash_entries);
3220
3221 int __init ip_rt_init(void)
3222 {
3223         int rc = 0;
3224
3225 #ifdef CONFIG_IP_ROUTE_CLASSID
3226         ip_rt_acct = __alloc_percpu(256 * sizeof(struct ip_rt_acct), __alignof__(struct ip_rt_acct));
3227         if (!ip_rt_acct)
3228                 panic("IP: failed to allocate ip_rt_acct\n");
3229 #endif
3230
3231         ipv4_dst_ops.kmem_cachep =
3232                 kmem_cache_create("ip_dst_cache", sizeof(struct rtable), 0,
3233                                   SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
3234
3235         ipv4_dst_blackhole_ops.kmem_cachep = ipv4_dst_ops.kmem_cachep;
3236
3237         if (dst_entries_init(&ipv4_dst_ops) < 0)
3238                 panic("IP: failed to allocate ipv4_dst_ops counter\n");
3239
3240         if (dst_entries_init(&ipv4_dst_blackhole_ops) < 0)
3241                 panic("IP: failed to allocate ipv4_dst_blackhole_ops counter\n");
3242
3243         rt_hash_table = (struct rt_hash_bucket *)
3244                 alloc_large_system_hash("IP route cache",
3245                                         sizeof(struct rt_hash_bucket),
3246                                         rhash_entries,
3247                                         (totalram_pages >= 128 * 1024) ?
3248                                         15 : 17,
3249                                         0,
3250                                         &rt_hash_log,
3251                                         &rt_hash_mask,
3252                                         rhash_entries ? 0 : 512 * 1024);
3253         memset(rt_hash_table, 0, (rt_hash_mask + 1) * sizeof(struct rt_hash_bucket));
3254         rt_hash_lock_init();
3255
3256         ipv4_dst_ops.gc_thresh = (rt_hash_mask + 1);
3257         ip_rt_max_size = (rt_hash_mask + 1) * 16;
3258
3259         devinet_init();
3260         ip_fib_init();
3261
3262         if (ip_rt_proc_init())
3263                 printk(KERN_ERR "Unable to create route proc files\n");
3264 #ifdef CONFIG_XFRM
3265         xfrm_init();
3266         xfrm4_init(ip_rt_max_size);
3267 #endif
3268         rtnl_register(PF_INET, RTM_GETROUTE, inet_rtm_getroute, NULL);
3269
3270 #ifdef CONFIG_SYSCTL
3271         register_pernet_subsys(&sysctl_route_ops);
3272 #endif
3273         register_pernet_subsys(&rt_genid_ops);
3274         return rc;
3275 }
3276
3277 #ifdef CONFIG_SYSCTL
3278 /*
3279  * We really need to sanitize the damn ipv4 init order, then all
3280  * this nonsense will go away.
3281  */
3282 void __init ip_static_sysctl_init(void)
3283 {
3284         register_sysctl_paths(ipv4_path, ipv4_skeleton);
3285 }
3286 #endif