ipv4: fix the reusing of routing cache entries
[linux-2.6.git] / net / ipv4 / route.c
1 /*
2  * INET         An implementation of the TCP/IP protocol suite for the LINUX
3  *              operating system.  INET is implemented using the  BSD Socket
4  *              interface as the means of communication with the user level.
5  *
6  *              ROUTE - implementation of the IP router.
7  *
8  * Authors:     Ross Biro
9  *              Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10  *              Alan Cox, <gw4pts@gw4pts.ampr.org>
11  *              Linus Torvalds, <Linus.Torvalds@helsinki.fi>
12  *              Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
13  *
14  * Fixes:
15  *              Alan Cox        :       Verify area fixes.
16  *              Alan Cox        :       cli() protects routing changes
17  *              Rui Oliveira    :       ICMP routing table updates
18  *              (rco@di.uminho.pt)      Routing table insertion and update
19  *              Linus Torvalds  :       Rewrote bits to be sensible
20  *              Alan Cox        :       Added BSD route gw semantics
21  *              Alan Cox        :       Super /proc >4K
22  *              Alan Cox        :       MTU in route table
23  *              Alan Cox        :       MSS actually. Also added the window
24  *                                      clamper.
25  *              Sam Lantinga    :       Fixed route matching in rt_del()
26  *              Alan Cox        :       Routing cache support.
27  *              Alan Cox        :       Removed compatibility cruft.
28  *              Alan Cox        :       RTF_REJECT support.
29  *              Alan Cox        :       TCP irtt support.
30  *              Jonathan Naylor :       Added Metric support.
31  *      Miquel van Smoorenburg  :       BSD API fixes.
32  *      Miquel van Smoorenburg  :       Metrics.
33  *              Alan Cox        :       Use __u32 properly
34  *              Alan Cox        :       Aligned routing errors more closely with BSD
35  *                                      our system is still very different.
36  *              Alan Cox        :       Faster /proc handling
37  *      Alexey Kuznetsov        :       Massive rework to support tree based routing,
38  *                                      routing caches and better behaviour.
39  *
40  *              Olaf Erb        :       irtt wasn't being copied right.
41  *              Bjorn Ekwall    :       Kerneld route support.
42  *              Alan Cox        :       Multicast fixed (I hope)
43  *              Pavel Krauz     :       Limited broadcast fixed
44  *              Mike McLagan    :       Routing by source
45  *      Alexey Kuznetsov        :       End of old history. Split to fib.c and
46  *                                      route.c and rewritten from scratch.
47  *              Andi Kleen      :       Load-limit warning messages.
48  *      Vitaly E. Lavrov        :       Transparent proxy revived after year coma.
49  *      Vitaly E. Lavrov        :       Race condition in ip_route_input_slow.
50  *      Tobias Ringstrom        :       Uninitialized res.type in ip_route_output_slow.
51  *      Vladimir V. Ivanov      :       IP rule info (flowid) is really useful.
52  *              Marc Boucher    :       routing by fwmark
53  *      Robert Olsson           :       Added rt_cache statistics
54  *      Arnaldo C. Melo         :       Convert proc stuff to seq_file
55  *      Eric Dumazet            :       hashed spinlocks and rt_check_expire() fixes.
56  *      Ilia Sotnikov           :       Ignore TOS on PMTUD and Redirect
57  *      Ilia Sotnikov           :       Removed TOS from hash calculations
58  *
59  *              This program is free software; you can redistribute it and/or
60  *              modify it under the terms of the GNU General Public License
61  *              as published by the Free Software Foundation; either version
62  *              2 of the License, or (at your option) any later version.
63  */
64
65 #include <linux/module.h>
66 #include <asm/uaccess.h>
67 #include <asm/system.h>
68 #include <linux/bitops.h>
69 #include <linux/types.h>
70 #include <linux/kernel.h>
71 #include <linux/mm.h>
72 #include <linux/bootmem.h>
73 #include <linux/string.h>
74 #include <linux/socket.h>
75 #include <linux/sockios.h>
76 #include <linux/errno.h>
77 #include <linux/in.h>
78 #include <linux/inet.h>
79 #include <linux/netdevice.h>
80 #include <linux/proc_fs.h>
81 #include <linux/init.h>
82 #include <linux/workqueue.h>
83 #include <linux/skbuff.h>
84 #include <linux/inetdevice.h>
85 #include <linux/igmp.h>
86 #include <linux/pkt_sched.h>
87 #include <linux/mroute.h>
88 #include <linux/netfilter_ipv4.h>
89 #include <linux/random.h>
90 #include <linux/jhash.h>
91 #include <linux/rcupdate.h>
92 #include <linux/times.h>
93 #include <linux/slab.h>
94 #include <net/dst.h>
95 #include <net/net_namespace.h>
96 #include <net/protocol.h>
97 #include <net/ip.h>
98 #include <net/route.h>
99 #include <net/inetpeer.h>
100 #include <net/sock.h>
101 #include <net/ip_fib.h>
102 #include <net/arp.h>
103 #include <net/tcp.h>
104 #include <net/icmp.h>
105 #include <net/xfrm.h>
106 #include <net/netevent.h>
107 #include <net/rtnetlink.h>
108 #ifdef CONFIG_SYSCTL
109 #include <linux/sysctl.h>
110 #endif
111 #include <net/atmclip.h>
112 #include <net/secure_seq.h>
113
114 #define RT_FL_TOS(oldflp4) \
115     ((u32)(oldflp4->flowi4_tos & (IPTOS_RT_MASK | RTO_ONLINK)))
116
117 #define IP_MAX_MTU      0xFFF0
118
119 #define RT_GC_TIMEOUT (300*HZ)
120
121 static int ip_rt_max_size;
122 static int ip_rt_gc_timeout __read_mostly       = RT_GC_TIMEOUT;
123 static int ip_rt_gc_interval __read_mostly      = 60 * HZ;
124 static int ip_rt_gc_min_interval __read_mostly  = HZ / 2;
125 static int ip_rt_redirect_number __read_mostly  = 9;
126 static int ip_rt_redirect_load __read_mostly    = HZ / 50;
127 static int ip_rt_redirect_silence __read_mostly = ((HZ / 50) << (9 + 1));
128 static int ip_rt_error_cost __read_mostly       = HZ;
129 static int ip_rt_error_burst __read_mostly      = 5 * HZ;
130 static int ip_rt_gc_elasticity __read_mostly    = 8;
131 static int ip_rt_mtu_expires __read_mostly      = 10 * 60 * HZ;
132 static int ip_rt_min_pmtu __read_mostly         = 512 + 20 + 20;
133 static int ip_rt_min_advmss __read_mostly       = 256;
134 static int rt_chain_length_max __read_mostly    = 20;
135
136 /*
137  *      Interface to generic destination cache.
138  */
139
140 static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie);
141 static unsigned int      ipv4_default_advmss(const struct dst_entry *dst);
142 static unsigned int      ipv4_default_mtu(const struct dst_entry *dst);
143 static void              ipv4_dst_destroy(struct dst_entry *dst);
144 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst);
145 static void              ipv4_link_failure(struct sk_buff *skb);
146 static void              ip_rt_update_pmtu(struct dst_entry *dst, u32 mtu);
147 static int rt_garbage_collect(struct dst_ops *ops);
148
149 static void ipv4_dst_ifdown(struct dst_entry *dst, struct net_device *dev,
150                             int how)
151 {
152 }
153
154 static u32 *ipv4_cow_metrics(struct dst_entry *dst, unsigned long old)
155 {
156         struct rtable *rt = (struct rtable *) dst;
157         struct inet_peer *peer;
158         u32 *p = NULL;
159
160         if (!rt->peer)
161                 rt_bind_peer(rt, rt->rt_dst, 1);
162
163         peer = rt->peer;
164         if (peer) {
165                 u32 *old_p = __DST_METRICS_PTR(old);
166                 unsigned long prev, new;
167
168                 p = peer->metrics;
169                 if (inet_metrics_new(peer))
170                         memcpy(p, old_p, sizeof(u32) * RTAX_MAX);
171
172                 new = (unsigned long) p;
173                 prev = cmpxchg(&dst->_metrics, old, new);
174
175                 if (prev != old) {
176                         p = __DST_METRICS_PTR(prev);
177                         if (prev & DST_METRICS_READ_ONLY)
178                                 p = NULL;
179                 } else {
180                         if (rt->fi) {
181                                 fib_info_put(rt->fi);
182                                 rt->fi = NULL;
183                         }
184                 }
185         }
186         return p;
187 }
188
189 static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst, const void *daddr);
190
191 static struct dst_ops ipv4_dst_ops = {
192         .family =               AF_INET,
193         .protocol =             cpu_to_be16(ETH_P_IP),
194         .gc =                   rt_garbage_collect,
195         .check =                ipv4_dst_check,
196         .default_advmss =       ipv4_default_advmss,
197         .default_mtu =          ipv4_default_mtu,
198         .cow_metrics =          ipv4_cow_metrics,
199         .destroy =              ipv4_dst_destroy,
200         .ifdown =               ipv4_dst_ifdown,
201         .negative_advice =      ipv4_negative_advice,
202         .link_failure =         ipv4_link_failure,
203         .update_pmtu =          ip_rt_update_pmtu,
204         .local_out =            __ip_local_out,
205         .neigh_lookup =         ipv4_neigh_lookup,
206 };
207
208 #define ECN_OR_COST(class)      TC_PRIO_##class
209
210 const __u8 ip_tos2prio[16] = {
211         TC_PRIO_BESTEFFORT,
212         ECN_OR_COST(BESTEFFORT),
213         TC_PRIO_BESTEFFORT,
214         ECN_OR_COST(BESTEFFORT),
215         TC_PRIO_BULK,
216         ECN_OR_COST(BULK),
217         TC_PRIO_BULK,
218         ECN_OR_COST(BULK),
219         TC_PRIO_INTERACTIVE,
220         ECN_OR_COST(INTERACTIVE),
221         TC_PRIO_INTERACTIVE,
222         ECN_OR_COST(INTERACTIVE),
223         TC_PRIO_INTERACTIVE_BULK,
224         ECN_OR_COST(INTERACTIVE_BULK),
225         TC_PRIO_INTERACTIVE_BULK,
226         ECN_OR_COST(INTERACTIVE_BULK)
227 };
228
229
230 /*
231  * Route cache.
232  */
233
234 /* The locking scheme is rather straight forward:
235  *
236  * 1) Read-Copy Update protects the buckets of the central route hash.
237  * 2) Only writers remove entries, and they hold the lock
238  *    as they look at rtable reference counts.
239  * 3) Only readers acquire references to rtable entries,
240  *    they do so with atomic increments and with the
241  *    lock held.
242  */
243
244 struct rt_hash_bucket {
245         struct rtable __rcu     *chain;
246 };
247
248 #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK) || \
249         defined(CONFIG_PROVE_LOCKING)
250 /*
251  * Instead of using one spinlock for each rt_hash_bucket, we use a table of spinlocks
252  * The size of this table is a power of two and depends on the number of CPUS.
253  * (on lockdep we have a quite big spinlock_t, so keep the size down there)
254  */
255 #ifdef CONFIG_LOCKDEP
256 # define RT_HASH_LOCK_SZ        256
257 #else
258 # if NR_CPUS >= 32
259 #  define RT_HASH_LOCK_SZ       4096
260 # elif NR_CPUS >= 16
261 #  define RT_HASH_LOCK_SZ       2048
262 # elif NR_CPUS >= 8
263 #  define RT_HASH_LOCK_SZ       1024
264 # elif NR_CPUS >= 4
265 #  define RT_HASH_LOCK_SZ       512
266 # else
267 #  define RT_HASH_LOCK_SZ       256
268 # endif
269 #endif
270
271 static spinlock_t       *rt_hash_locks;
272 # define rt_hash_lock_addr(slot) &rt_hash_locks[(slot) & (RT_HASH_LOCK_SZ - 1)]
273
274 static __init void rt_hash_lock_init(void)
275 {
276         int i;
277
278         rt_hash_locks = kmalloc(sizeof(spinlock_t) * RT_HASH_LOCK_SZ,
279                         GFP_KERNEL);
280         if (!rt_hash_locks)
281                 panic("IP: failed to allocate rt_hash_locks\n");
282
283         for (i = 0; i < RT_HASH_LOCK_SZ; i++)
284                 spin_lock_init(&rt_hash_locks[i]);
285 }
286 #else
287 # define rt_hash_lock_addr(slot) NULL
288
289 static inline void rt_hash_lock_init(void)
290 {
291 }
292 #endif
293
294 static struct rt_hash_bucket    *rt_hash_table __read_mostly;
295 static unsigned                 rt_hash_mask __read_mostly;
296 static unsigned int             rt_hash_log  __read_mostly;
297
298 static DEFINE_PER_CPU(struct rt_cache_stat, rt_cache_stat);
299 #define RT_CACHE_STAT_INC(field) __this_cpu_inc(rt_cache_stat.field)
300
301 static inline unsigned int rt_hash(__be32 daddr, __be32 saddr, int idx,
302                                    int genid)
303 {
304         return jhash_3words((__force u32)daddr, (__force u32)saddr,
305                             idx, genid)
306                 & rt_hash_mask;
307 }
308
309 static inline int rt_genid(struct net *net)
310 {
311         return atomic_read(&net->ipv4.rt_genid);
312 }
313
314 #ifdef CONFIG_PROC_FS
315 struct rt_cache_iter_state {
316         struct seq_net_private p;
317         int bucket;
318         int genid;
319 };
320
321 static struct rtable *rt_cache_get_first(struct seq_file *seq)
322 {
323         struct rt_cache_iter_state *st = seq->private;
324         struct rtable *r = NULL;
325
326         for (st->bucket = rt_hash_mask; st->bucket >= 0; --st->bucket) {
327                 if (!rcu_dereference_raw(rt_hash_table[st->bucket].chain))
328                         continue;
329                 rcu_read_lock_bh();
330                 r = rcu_dereference_bh(rt_hash_table[st->bucket].chain);
331                 while (r) {
332                         if (dev_net(r->dst.dev) == seq_file_net(seq) &&
333                             r->rt_genid == st->genid)
334                                 return r;
335                         r = rcu_dereference_bh(r->dst.rt_next);
336                 }
337                 rcu_read_unlock_bh();
338         }
339         return r;
340 }
341
342 static struct rtable *__rt_cache_get_next(struct seq_file *seq,
343                                           struct rtable *r)
344 {
345         struct rt_cache_iter_state *st = seq->private;
346
347         r = rcu_dereference_bh(r->dst.rt_next);
348         while (!r) {
349                 rcu_read_unlock_bh();
350                 do {
351                         if (--st->bucket < 0)
352                                 return NULL;
353                 } while (!rcu_dereference_raw(rt_hash_table[st->bucket].chain));
354                 rcu_read_lock_bh();
355                 r = rcu_dereference_bh(rt_hash_table[st->bucket].chain);
356         }
357         return r;
358 }
359
360 static struct rtable *rt_cache_get_next(struct seq_file *seq,
361                                         struct rtable *r)
362 {
363         struct rt_cache_iter_state *st = seq->private;
364         while ((r = __rt_cache_get_next(seq, r)) != NULL) {
365                 if (dev_net(r->dst.dev) != seq_file_net(seq))
366                         continue;
367                 if (r->rt_genid == st->genid)
368                         break;
369         }
370         return r;
371 }
372
373 static struct rtable *rt_cache_get_idx(struct seq_file *seq, loff_t pos)
374 {
375         struct rtable *r = rt_cache_get_first(seq);
376
377         if (r)
378                 while (pos && (r = rt_cache_get_next(seq, r)))
379                         --pos;
380         return pos ? NULL : r;
381 }
382
383 static void *rt_cache_seq_start(struct seq_file *seq, loff_t *pos)
384 {
385         struct rt_cache_iter_state *st = seq->private;
386         if (*pos)
387                 return rt_cache_get_idx(seq, *pos - 1);
388         st->genid = rt_genid(seq_file_net(seq));
389         return SEQ_START_TOKEN;
390 }
391
392 static void *rt_cache_seq_next(struct seq_file *seq, void *v, loff_t *pos)
393 {
394         struct rtable *r;
395
396         if (v == SEQ_START_TOKEN)
397                 r = rt_cache_get_first(seq);
398         else
399                 r = rt_cache_get_next(seq, v);
400         ++*pos;
401         return r;
402 }
403
404 static void rt_cache_seq_stop(struct seq_file *seq, void *v)
405 {
406         if (v && v != SEQ_START_TOKEN)
407                 rcu_read_unlock_bh();
408 }
409
410 static int rt_cache_seq_show(struct seq_file *seq, void *v)
411 {
412         if (v == SEQ_START_TOKEN)
413                 seq_printf(seq, "%-127s\n",
414                            "Iface\tDestination\tGateway \tFlags\t\tRefCnt\tUse\t"
415                            "Metric\tSource\t\tMTU\tWindow\tIRTT\tTOS\tHHRef\t"
416                            "HHUptod\tSpecDst");
417         else {
418                 struct rtable *r = v;
419                 struct neighbour *n;
420                 int len;
421
422                 n = dst_get_neighbour(&r->dst);
423                 seq_printf(seq, "%s\t%08X\t%08X\t%8X\t%d\t%u\t%d\t"
424                               "%08X\t%d\t%u\t%u\t%02X\t%d\t%1d\t%08X%n",
425                         r->dst.dev ? r->dst.dev->name : "*",
426                         (__force u32)r->rt_dst,
427                         (__force u32)r->rt_gateway,
428                         r->rt_flags, atomic_read(&r->dst.__refcnt),
429                         r->dst.__use, 0, (__force u32)r->rt_src,
430                         dst_metric_advmss(&r->dst) + 40,
431                         dst_metric(&r->dst, RTAX_WINDOW),
432                         (int)((dst_metric(&r->dst, RTAX_RTT) >> 3) +
433                               dst_metric(&r->dst, RTAX_RTTVAR)),
434                         r->rt_key_tos,
435                         -1,
436                         (n && (n->nud_state & NUD_CONNECTED)) ? 1 : 0,
437                         r->rt_spec_dst, &len);
438
439                 seq_printf(seq, "%*s\n", 127 - len, "");
440         }
441         return 0;
442 }
443
444 static const struct seq_operations rt_cache_seq_ops = {
445         .start  = rt_cache_seq_start,
446         .next   = rt_cache_seq_next,
447         .stop   = rt_cache_seq_stop,
448         .show   = rt_cache_seq_show,
449 };
450
451 static int rt_cache_seq_open(struct inode *inode, struct file *file)
452 {
453         return seq_open_net(inode, file, &rt_cache_seq_ops,
454                         sizeof(struct rt_cache_iter_state));
455 }
456
457 static const struct file_operations rt_cache_seq_fops = {
458         .owner   = THIS_MODULE,
459         .open    = rt_cache_seq_open,
460         .read    = seq_read,
461         .llseek  = seq_lseek,
462         .release = seq_release_net,
463 };
464
465
466 static void *rt_cpu_seq_start(struct seq_file *seq, loff_t *pos)
467 {
468         int cpu;
469
470         if (*pos == 0)
471                 return SEQ_START_TOKEN;
472
473         for (cpu = *pos-1; cpu < nr_cpu_ids; ++cpu) {
474                 if (!cpu_possible(cpu))
475                         continue;
476                 *pos = cpu+1;
477                 return &per_cpu(rt_cache_stat, cpu);
478         }
479         return NULL;
480 }
481
482 static void *rt_cpu_seq_next(struct seq_file *seq, void *v, loff_t *pos)
483 {
484         int cpu;
485
486         for (cpu = *pos; cpu < nr_cpu_ids; ++cpu) {
487                 if (!cpu_possible(cpu))
488                         continue;
489                 *pos = cpu+1;
490                 return &per_cpu(rt_cache_stat, cpu);
491         }
492         return NULL;
493
494 }
495
496 static void rt_cpu_seq_stop(struct seq_file *seq, void *v)
497 {
498
499 }
500
501 static int rt_cpu_seq_show(struct seq_file *seq, void *v)
502 {
503         struct rt_cache_stat *st = v;
504
505         if (v == SEQ_START_TOKEN) {
506                 seq_printf(seq, "entries  in_hit in_slow_tot in_slow_mc in_no_route in_brd in_martian_dst in_martian_src  out_hit out_slow_tot out_slow_mc  gc_total gc_ignored gc_goal_miss gc_dst_overflow in_hlist_search out_hlist_search\n");
507                 return 0;
508         }
509
510         seq_printf(seq,"%08x  %08x %08x %08x %08x %08x %08x %08x "
511                    " %08x %08x %08x %08x %08x %08x %08x %08x %08x \n",
512                    dst_entries_get_slow(&ipv4_dst_ops),
513                    st->in_hit,
514                    st->in_slow_tot,
515                    st->in_slow_mc,
516                    st->in_no_route,
517                    st->in_brd,
518                    st->in_martian_dst,
519                    st->in_martian_src,
520
521                    st->out_hit,
522                    st->out_slow_tot,
523                    st->out_slow_mc,
524
525                    st->gc_total,
526                    st->gc_ignored,
527                    st->gc_goal_miss,
528                    st->gc_dst_overflow,
529                    st->in_hlist_search,
530                    st->out_hlist_search
531                 );
532         return 0;
533 }
534
535 static const struct seq_operations rt_cpu_seq_ops = {
536         .start  = rt_cpu_seq_start,
537         .next   = rt_cpu_seq_next,
538         .stop   = rt_cpu_seq_stop,
539         .show   = rt_cpu_seq_show,
540 };
541
542
543 static int rt_cpu_seq_open(struct inode *inode, struct file *file)
544 {
545         return seq_open(file, &rt_cpu_seq_ops);
546 }
547
548 static const struct file_operations rt_cpu_seq_fops = {
549         .owner   = THIS_MODULE,
550         .open    = rt_cpu_seq_open,
551         .read    = seq_read,
552         .llseek  = seq_lseek,
553         .release = seq_release,
554 };
555
556 #ifdef CONFIG_IP_ROUTE_CLASSID
557 static int rt_acct_proc_show(struct seq_file *m, void *v)
558 {
559         struct ip_rt_acct *dst, *src;
560         unsigned int i, j;
561
562         dst = kcalloc(256, sizeof(struct ip_rt_acct), GFP_KERNEL);
563         if (!dst)
564                 return -ENOMEM;
565
566         for_each_possible_cpu(i) {
567                 src = (struct ip_rt_acct *)per_cpu_ptr(ip_rt_acct, i);
568                 for (j = 0; j < 256; j++) {
569                         dst[j].o_bytes   += src[j].o_bytes;
570                         dst[j].o_packets += src[j].o_packets;
571                         dst[j].i_bytes   += src[j].i_bytes;
572                         dst[j].i_packets += src[j].i_packets;
573                 }
574         }
575
576         seq_write(m, dst, 256 * sizeof(struct ip_rt_acct));
577         kfree(dst);
578         return 0;
579 }
580
581 static int rt_acct_proc_open(struct inode *inode, struct file *file)
582 {
583         return single_open(file, rt_acct_proc_show, NULL);
584 }
585
586 static const struct file_operations rt_acct_proc_fops = {
587         .owner          = THIS_MODULE,
588         .open           = rt_acct_proc_open,
589         .read           = seq_read,
590         .llseek         = seq_lseek,
591         .release        = single_release,
592 };
593 #endif
594
595 static int __net_init ip_rt_do_proc_init(struct net *net)
596 {
597         struct proc_dir_entry *pde;
598
599         pde = proc_net_fops_create(net, "rt_cache", S_IRUGO,
600                         &rt_cache_seq_fops);
601         if (!pde)
602                 goto err1;
603
604         pde = proc_create("rt_cache", S_IRUGO,
605                           net->proc_net_stat, &rt_cpu_seq_fops);
606         if (!pde)
607                 goto err2;
608
609 #ifdef CONFIG_IP_ROUTE_CLASSID
610         pde = proc_create("rt_acct", 0, net->proc_net, &rt_acct_proc_fops);
611         if (!pde)
612                 goto err3;
613 #endif
614         return 0;
615
616 #ifdef CONFIG_IP_ROUTE_CLASSID
617 err3:
618         remove_proc_entry("rt_cache", net->proc_net_stat);
619 #endif
620 err2:
621         remove_proc_entry("rt_cache", net->proc_net);
622 err1:
623         return -ENOMEM;
624 }
625
626 static void __net_exit ip_rt_do_proc_exit(struct net *net)
627 {
628         remove_proc_entry("rt_cache", net->proc_net_stat);
629         remove_proc_entry("rt_cache", net->proc_net);
630 #ifdef CONFIG_IP_ROUTE_CLASSID
631         remove_proc_entry("rt_acct", net->proc_net);
632 #endif
633 }
634
635 static struct pernet_operations ip_rt_proc_ops __net_initdata =  {
636         .init = ip_rt_do_proc_init,
637         .exit = ip_rt_do_proc_exit,
638 };
639
640 static int __init ip_rt_proc_init(void)
641 {
642         return register_pernet_subsys(&ip_rt_proc_ops);
643 }
644
645 #else
646 static inline int ip_rt_proc_init(void)
647 {
648         return 0;
649 }
650 #endif /* CONFIG_PROC_FS */
651
652 static inline void rt_free(struct rtable *rt)
653 {
654         call_rcu_bh(&rt->dst.rcu_head, dst_rcu_free);
655 }
656
657 static inline void rt_drop(struct rtable *rt)
658 {
659         ip_rt_put(rt);
660         call_rcu_bh(&rt->dst.rcu_head, dst_rcu_free);
661 }
662
663 static inline int rt_fast_clean(struct rtable *rth)
664 {
665         /* Kill broadcast/multicast entries very aggresively, if they
666            collide in hash table with more useful entries */
667         return (rth->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST)) &&
668                 rt_is_input_route(rth) && rth->dst.rt_next;
669 }
670
671 static inline int rt_valuable(struct rtable *rth)
672 {
673         return (rth->rt_flags & (RTCF_REDIRECTED | RTCF_NOTIFY)) ||
674                 (rth->peer && rth->peer->pmtu_expires);
675 }
676
677 static int rt_may_expire(struct rtable *rth, unsigned long tmo1, unsigned long tmo2)
678 {
679         unsigned long age;
680         int ret = 0;
681
682         if (atomic_read(&rth->dst.__refcnt))
683                 goto out;
684
685         age = jiffies - rth->dst.lastuse;
686         if ((age <= tmo1 && !rt_fast_clean(rth)) ||
687             (age <= tmo2 && rt_valuable(rth)))
688                 goto out;
689         ret = 1;
690 out:    return ret;
691 }
692
693 /* Bits of score are:
694  * 31: very valuable
695  * 30: not quite useless
696  * 29..0: usage counter
697  */
698 static inline u32 rt_score(struct rtable *rt)
699 {
700         u32 score = jiffies - rt->dst.lastuse;
701
702         score = ~score & ~(3<<30);
703
704         if (rt_valuable(rt))
705                 score |= (1<<31);
706
707         if (rt_is_output_route(rt) ||
708             !(rt->rt_flags & (RTCF_BROADCAST|RTCF_MULTICAST|RTCF_LOCAL)))
709                 score |= (1<<30);
710
711         return score;
712 }
713
714 static inline bool rt_caching(const struct net *net)
715 {
716         return net->ipv4.current_rt_cache_rebuild_count <=
717                 net->ipv4.sysctl_rt_cache_rebuild_count;
718 }
719
720 static inline bool compare_hash_inputs(const struct rtable *rt1,
721                                        const struct rtable *rt2)
722 {
723         return ((((__force u32)rt1->rt_key_dst ^ (__force u32)rt2->rt_key_dst) |
724                 ((__force u32)rt1->rt_key_src ^ (__force u32)rt2->rt_key_src) |
725                 (rt1->rt_iif ^ rt2->rt_iif)) == 0);
726 }
727
728 static inline int compare_keys(struct rtable *rt1, struct rtable *rt2)
729 {
730         return (((__force u32)rt1->rt_key_dst ^ (__force u32)rt2->rt_key_dst) |
731                 ((__force u32)rt1->rt_key_src ^ (__force u32)rt2->rt_key_src) |
732                 (rt1->rt_mark ^ rt2->rt_mark) |
733                 (rt1->rt_key_tos ^ rt2->rt_key_tos) |
734                 (rt1->rt_route_iif ^ rt2->rt_route_iif) |
735                 (rt1->rt_oif ^ rt2->rt_oif) |
736                 (rt1->rt_iif ^ rt2->rt_iif)) == 0;
737 }
738
739 static inline int compare_netns(struct rtable *rt1, struct rtable *rt2)
740 {
741         return net_eq(dev_net(rt1->dst.dev), dev_net(rt2->dst.dev));
742 }
743
744 static inline int rt_is_expired(struct rtable *rth)
745 {
746         return rth->rt_genid != rt_genid(dev_net(rth->dst.dev));
747 }
748
749 /*
750  * Perform a full scan of hash table and free all entries.
751  * Can be called by a softirq or a process.
752  * In the later case, we want to be reschedule if necessary
753  */
754 static void rt_do_flush(struct net *net, int process_context)
755 {
756         unsigned int i;
757         struct rtable *rth, *next;
758
759         for (i = 0; i <= rt_hash_mask; i++) {
760                 struct rtable __rcu **pprev;
761                 struct rtable *list;
762
763                 if (process_context && need_resched())
764                         cond_resched();
765                 rth = rcu_dereference_raw(rt_hash_table[i].chain);
766                 if (!rth)
767                         continue;
768
769                 spin_lock_bh(rt_hash_lock_addr(i));
770
771                 list = NULL;
772                 pprev = &rt_hash_table[i].chain;
773                 rth = rcu_dereference_protected(*pprev,
774                         lockdep_is_held(rt_hash_lock_addr(i)));
775
776                 while (rth) {
777                         next = rcu_dereference_protected(rth->dst.rt_next,
778                                 lockdep_is_held(rt_hash_lock_addr(i)));
779
780                         if (!net ||
781                             net_eq(dev_net(rth->dst.dev), net)) {
782                                 rcu_assign_pointer(*pprev, next);
783                                 rcu_assign_pointer(rth->dst.rt_next, list);
784                                 list = rth;
785                         } else {
786                                 pprev = &rth->dst.rt_next;
787                         }
788                         rth = next;
789                 }
790
791                 spin_unlock_bh(rt_hash_lock_addr(i));
792
793                 for (; list; list = next) {
794                         next = rcu_dereference_protected(list->dst.rt_next, 1);
795                         rt_free(list);
796                 }
797         }
798 }
799
800 /*
801  * While freeing expired entries, we compute average chain length
802  * and standard deviation, using fixed-point arithmetic.
803  * This to have an estimation of rt_chain_length_max
804  *  rt_chain_length_max = max(elasticity, AVG + 4*SD)
805  * We use 3 bits for frational part, and 29 (or 61) for magnitude.
806  */
807
808 #define FRACT_BITS 3
809 #define ONE (1UL << FRACT_BITS)
810
811 /*
812  * Given a hash chain and an item in this hash chain,
813  * find if a previous entry has the same hash_inputs
814  * (but differs on tos, mark or oif)
815  * Returns 0 if an alias is found.
816  * Returns ONE if rth has no alias before itself.
817  */
818 static int has_noalias(const struct rtable *head, const struct rtable *rth)
819 {
820         const struct rtable *aux = head;
821
822         while (aux != rth) {
823                 if (compare_hash_inputs(aux, rth))
824                         return 0;
825                 aux = rcu_dereference_protected(aux->dst.rt_next, 1);
826         }
827         return ONE;
828 }
829
830 /*
831  * Perturbation of rt_genid by a small quantity [1..256]
832  * Using 8 bits of shuffling ensure we can call rt_cache_invalidate()
833  * many times (2^24) without giving recent rt_genid.
834  * Jenkins hash is strong enough that litle changes of rt_genid are OK.
835  */
836 static void rt_cache_invalidate(struct net *net)
837 {
838         unsigned char shuffle;
839
840         get_random_bytes(&shuffle, sizeof(shuffle));
841         atomic_add(shuffle + 1U, &net->ipv4.rt_genid);
842 }
843
844 /*
845  * delay < 0  : invalidate cache (fast : entries will be deleted later)
846  * delay >= 0 : invalidate & flush cache (can be long)
847  */
848 void rt_cache_flush(struct net *net, int delay)
849 {
850         rt_cache_invalidate(net);
851         if (delay >= 0)
852                 rt_do_flush(net, !in_softirq());
853 }
854
855 /* Flush previous cache invalidated entries from the cache */
856 void rt_cache_flush_batch(struct net *net)
857 {
858         rt_do_flush(net, !in_softirq());
859 }
860
861 static void rt_emergency_hash_rebuild(struct net *net)
862 {
863         if (net_ratelimit())
864                 printk(KERN_WARNING "Route hash chain too long!\n");
865         rt_cache_invalidate(net);
866 }
867
868 /*
869    Short description of GC goals.
870
871    We want to build algorithm, which will keep routing cache
872    at some equilibrium point, when number of aged off entries
873    is kept approximately equal to newly generated ones.
874
875    Current expiration strength is variable "expire".
876    We try to adjust it dynamically, so that if networking
877    is idle expires is large enough to keep enough of warm entries,
878    and when load increases it reduces to limit cache size.
879  */
880
881 static int rt_garbage_collect(struct dst_ops *ops)
882 {
883         static unsigned long expire = RT_GC_TIMEOUT;
884         static unsigned long last_gc;
885         static int rover;
886         static int equilibrium;
887         struct rtable *rth;
888         struct rtable __rcu **rthp;
889         unsigned long now = jiffies;
890         int goal;
891         int entries = dst_entries_get_fast(&ipv4_dst_ops);
892
893         /*
894          * Garbage collection is pretty expensive,
895          * do not make it too frequently.
896          */
897
898         RT_CACHE_STAT_INC(gc_total);
899
900         if (now - last_gc < ip_rt_gc_min_interval &&
901             entries < ip_rt_max_size) {
902                 RT_CACHE_STAT_INC(gc_ignored);
903                 goto out;
904         }
905
906         entries = dst_entries_get_slow(&ipv4_dst_ops);
907         /* Calculate number of entries, which we want to expire now. */
908         goal = entries - (ip_rt_gc_elasticity << rt_hash_log);
909         if (goal <= 0) {
910                 if (equilibrium < ipv4_dst_ops.gc_thresh)
911                         equilibrium = ipv4_dst_ops.gc_thresh;
912                 goal = entries - equilibrium;
913                 if (goal > 0) {
914                         equilibrium += min_t(unsigned int, goal >> 1, rt_hash_mask + 1);
915                         goal = entries - equilibrium;
916                 }
917         } else {
918                 /* We are in dangerous area. Try to reduce cache really
919                  * aggressively.
920                  */
921                 goal = max_t(unsigned int, goal >> 1, rt_hash_mask + 1);
922                 equilibrium = entries - goal;
923         }
924
925         if (now - last_gc >= ip_rt_gc_min_interval)
926                 last_gc = now;
927
928         if (goal <= 0) {
929                 equilibrium += goal;
930                 goto work_done;
931         }
932
933         do {
934                 int i, k;
935
936                 for (i = rt_hash_mask, k = rover; i >= 0; i--) {
937                         unsigned long tmo = expire;
938
939                         k = (k + 1) & rt_hash_mask;
940                         rthp = &rt_hash_table[k].chain;
941                         spin_lock_bh(rt_hash_lock_addr(k));
942                         while ((rth = rcu_dereference_protected(*rthp,
943                                         lockdep_is_held(rt_hash_lock_addr(k)))) != NULL) {
944                                 if (!rt_is_expired(rth) &&
945                                         !rt_may_expire(rth, tmo, expire)) {
946                                         tmo >>= 1;
947                                         rthp = &rth->dst.rt_next;
948                                         continue;
949                                 }
950                                 *rthp = rth->dst.rt_next;
951                                 rt_free(rth);
952                                 goal--;
953                         }
954                         spin_unlock_bh(rt_hash_lock_addr(k));
955                         if (goal <= 0)
956                                 break;
957                 }
958                 rover = k;
959
960                 if (goal <= 0)
961                         goto work_done;
962
963                 /* Goal is not achieved. We stop process if:
964
965                    - if expire reduced to zero. Otherwise, expire is halfed.
966                    - if table is not full.
967                    - if we are called from interrupt.
968                    - jiffies check is just fallback/debug loop breaker.
969                      We will not spin here for long time in any case.
970                  */
971
972                 RT_CACHE_STAT_INC(gc_goal_miss);
973
974                 if (expire == 0)
975                         break;
976
977                 expire >>= 1;
978
979                 if (dst_entries_get_fast(&ipv4_dst_ops) < ip_rt_max_size)
980                         goto out;
981         } while (!in_softirq() && time_before_eq(jiffies, now));
982
983         if (dst_entries_get_fast(&ipv4_dst_ops) < ip_rt_max_size)
984                 goto out;
985         if (dst_entries_get_slow(&ipv4_dst_ops) < ip_rt_max_size)
986                 goto out;
987         if (net_ratelimit())
988                 printk(KERN_WARNING "dst cache overflow\n");
989         RT_CACHE_STAT_INC(gc_dst_overflow);
990         return 1;
991
992 work_done:
993         expire += ip_rt_gc_min_interval;
994         if (expire > ip_rt_gc_timeout ||
995             dst_entries_get_fast(&ipv4_dst_ops) < ipv4_dst_ops.gc_thresh ||
996             dst_entries_get_slow(&ipv4_dst_ops) < ipv4_dst_ops.gc_thresh)
997                 expire = ip_rt_gc_timeout;
998 out:    return 0;
999 }
1000
1001 /*
1002  * Returns number of entries in a hash chain that have different hash_inputs
1003  */
1004 static int slow_chain_length(const struct rtable *head)
1005 {
1006         int length = 0;
1007         const struct rtable *rth = head;
1008
1009         while (rth) {
1010                 length += has_noalias(head, rth);
1011                 rth = rcu_dereference_protected(rth->dst.rt_next, 1);
1012         }
1013         return length >> FRACT_BITS;
1014 }
1015
1016 static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst, const void *daddr)
1017 {
1018         struct neigh_table *tbl = &arp_tbl;
1019         static const __be32 inaddr_any = 0;
1020         struct net_device *dev = dst->dev;
1021         const __be32 *pkey = daddr;
1022         struct neighbour *n;
1023
1024 #if defined(CONFIG_ATM_CLIP) || defined(CONFIG_ATM_CLIP_MODULE)
1025         if (dev->type == ARPHRD_ATM)
1026                 tbl = clip_tbl_hook;
1027 #endif
1028         if (dev->flags & (IFF_LOOPBACK | IFF_POINTOPOINT))
1029                 pkey = &inaddr_any;
1030
1031         n = __ipv4_neigh_lookup(tbl, dev, *(__force u32 *)pkey);
1032         if (n)
1033                 return n;
1034         return neigh_create(tbl, pkey, dev);
1035 }
1036
1037 static int rt_bind_neighbour(struct rtable *rt)
1038 {
1039         struct neighbour *n = ipv4_neigh_lookup(&rt->dst, &rt->rt_gateway);
1040         if (IS_ERR(n))
1041                 return PTR_ERR(n);
1042         dst_set_neighbour(&rt->dst, n);
1043
1044         return 0;
1045 }
1046
1047 static struct rtable *rt_intern_hash(unsigned hash, struct rtable *rt,
1048                                      struct sk_buff *skb, int ifindex)
1049 {
1050         struct rtable   *rth, *cand;
1051         struct rtable __rcu **rthp, **candp;
1052         unsigned long   now;
1053         u32             min_score;
1054         int             chain_length;
1055         int attempts = !in_softirq();
1056
1057 restart:
1058         chain_length = 0;
1059         min_score = ~(u32)0;
1060         cand = NULL;
1061         candp = NULL;
1062         now = jiffies;
1063
1064         if (!rt_caching(dev_net(rt->dst.dev))) {
1065                 /*
1066                  * If we're not caching, just tell the caller we
1067                  * were successful and don't touch the route.  The
1068                  * caller hold the sole reference to the cache entry, and
1069                  * it will be released when the caller is done with it.
1070                  * If we drop it here, the callers have no way to resolve routes
1071                  * when we're not caching.  Instead, just point *rp at rt, so
1072                  * the caller gets a single use out of the route
1073                  * Note that we do rt_free on this new route entry, so that
1074                  * once its refcount hits zero, we are still able to reap it
1075                  * (Thanks Alexey)
1076                  * Note: To avoid expensive rcu stuff for this uncached dst,
1077                  * we set DST_NOCACHE so that dst_release() can free dst without
1078                  * waiting a grace period.
1079                  */
1080
1081                 rt->dst.flags |= DST_NOCACHE;
1082                 if (rt->rt_type == RTN_UNICAST || rt_is_output_route(rt)) {
1083                         int err = rt_bind_neighbour(rt);
1084                         if (err) {
1085                                 if (net_ratelimit())
1086                                         printk(KERN_WARNING
1087                                             "Neighbour table failure & not caching routes.\n");
1088                                 ip_rt_put(rt);
1089                                 return ERR_PTR(err);
1090                         }
1091                 }
1092
1093                 goto skip_hashing;
1094         }
1095
1096         rthp = &rt_hash_table[hash].chain;
1097
1098         spin_lock_bh(rt_hash_lock_addr(hash));
1099         while ((rth = rcu_dereference_protected(*rthp,
1100                         lockdep_is_held(rt_hash_lock_addr(hash)))) != NULL) {
1101                 if (rt_is_expired(rth)) {
1102                         *rthp = rth->dst.rt_next;
1103                         rt_free(rth);
1104                         continue;
1105                 }
1106                 if (compare_keys(rth, rt) && compare_netns(rth, rt)) {
1107                         /* Put it first */
1108                         *rthp = rth->dst.rt_next;
1109                         /*
1110                          * Since lookup is lockfree, the deletion
1111                          * must be visible to another weakly ordered CPU before
1112                          * the insertion at the start of the hash chain.
1113                          */
1114                         rcu_assign_pointer(rth->dst.rt_next,
1115                                            rt_hash_table[hash].chain);
1116                         /*
1117                          * Since lookup is lockfree, the update writes
1118                          * must be ordered for consistency on SMP.
1119                          */
1120                         rcu_assign_pointer(rt_hash_table[hash].chain, rth);
1121
1122                         dst_use(&rth->dst, now);
1123                         spin_unlock_bh(rt_hash_lock_addr(hash));
1124
1125                         rt_drop(rt);
1126                         if (skb)
1127                                 skb_dst_set(skb, &rth->dst);
1128                         return rth;
1129                 }
1130
1131                 if (!atomic_read(&rth->dst.__refcnt)) {
1132                         u32 score = rt_score(rth);
1133
1134                         if (score <= min_score) {
1135                                 cand = rth;
1136                                 candp = rthp;
1137                                 min_score = score;
1138                         }
1139                 }
1140
1141                 chain_length++;
1142
1143                 rthp = &rth->dst.rt_next;
1144         }
1145
1146         if (cand) {
1147                 /* ip_rt_gc_elasticity used to be average length of chain
1148                  * length, when exceeded gc becomes really aggressive.
1149                  *
1150                  * The second limit is less certain. At the moment it allows
1151                  * only 2 entries per bucket. We will see.
1152                  */
1153                 if (chain_length > ip_rt_gc_elasticity) {
1154                         *candp = cand->dst.rt_next;
1155                         rt_free(cand);
1156                 }
1157         } else {
1158                 if (chain_length > rt_chain_length_max &&
1159                     slow_chain_length(rt_hash_table[hash].chain) > rt_chain_length_max) {
1160                         struct net *net = dev_net(rt->dst.dev);
1161                         int num = ++net->ipv4.current_rt_cache_rebuild_count;
1162                         if (!rt_caching(net)) {
1163                                 printk(KERN_WARNING "%s: %d rebuilds is over limit, route caching disabled\n",
1164                                         rt->dst.dev->name, num);
1165                         }
1166                         rt_emergency_hash_rebuild(net);
1167                         spin_unlock_bh(rt_hash_lock_addr(hash));
1168
1169                         hash = rt_hash(rt->rt_key_dst, rt->rt_key_src,
1170                                         ifindex, rt_genid(net));
1171                         goto restart;
1172                 }
1173         }
1174
1175         /* Try to bind route to arp only if it is output
1176            route or unicast forwarding path.
1177          */
1178         if (rt->rt_type == RTN_UNICAST || rt_is_output_route(rt)) {
1179                 int err = rt_bind_neighbour(rt);
1180                 if (err) {
1181                         spin_unlock_bh(rt_hash_lock_addr(hash));
1182
1183                         if (err != -ENOBUFS) {
1184                                 rt_drop(rt);
1185                                 return ERR_PTR(err);
1186                         }
1187
1188                         /* Neighbour tables are full and nothing
1189                            can be released. Try to shrink route cache,
1190                            it is most likely it holds some neighbour records.
1191                          */
1192                         if (attempts-- > 0) {
1193                                 int saved_elasticity = ip_rt_gc_elasticity;
1194                                 int saved_int = ip_rt_gc_min_interval;
1195                                 ip_rt_gc_elasticity     = 1;
1196                                 ip_rt_gc_min_interval   = 0;
1197                                 rt_garbage_collect(&ipv4_dst_ops);
1198                                 ip_rt_gc_min_interval   = saved_int;
1199                                 ip_rt_gc_elasticity     = saved_elasticity;
1200                                 goto restart;
1201                         }
1202
1203                         if (net_ratelimit())
1204                                 printk(KERN_WARNING "ipv4: Neighbour table overflow.\n");
1205                         rt_drop(rt);
1206                         return ERR_PTR(-ENOBUFS);
1207                 }
1208         }
1209
1210         rt->dst.rt_next = rt_hash_table[hash].chain;
1211
1212         /*
1213          * Since lookup is lockfree, we must make sure
1214          * previous writes to rt are committed to memory
1215          * before making rt visible to other CPUS.
1216          */
1217         rcu_assign_pointer(rt_hash_table[hash].chain, rt);
1218
1219         spin_unlock_bh(rt_hash_lock_addr(hash));
1220
1221 skip_hashing:
1222         if (skb)
1223                 skb_dst_set(skb, &rt->dst);
1224         return rt;
1225 }
1226
1227 static atomic_t __rt_peer_genid = ATOMIC_INIT(0);
1228
1229 static u32 rt_peer_genid(void)
1230 {
1231         return atomic_read(&__rt_peer_genid);
1232 }
1233
1234 void rt_bind_peer(struct rtable *rt, __be32 daddr, int create)
1235 {
1236         struct inet_peer *peer;
1237
1238         peer = inet_getpeer_v4(daddr, create);
1239
1240         if (peer && cmpxchg(&rt->peer, NULL, peer) != NULL)
1241                 inet_putpeer(peer);
1242         else
1243                 rt->rt_peer_genid = rt_peer_genid();
1244 }
1245
1246 /*
1247  * Peer allocation may fail only in serious out-of-memory conditions.  However
1248  * we still can generate some output.
1249  * Random ID selection looks a bit dangerous because we have no chances to
1250  * select ID being unique in a reasonable period of time.
1251  * But broken packet identifier may be better than no packet at all.
1252  */
1253 static void ip_select_fb_ident(struct iphdr *iph)
1254 {
1255         static DEFINE_SPINLOCK(ip_fb_id_lock);
1256         static u32 ip_fallback_id;
1257         u32 salt;
1258
1259         spin_lock_bh(&ip_fb_id_lock);
1260         salt = secure_ip_id((__force __be32)ip_fallback_id ^ iph->daddr);
1261         iph->id = htons(salt & 0xFFFF);
1262         ip_fallback_id = salt;
1263         spin_unlock_bh(&ip_fb_id_lock);
1264 }
1265
1266 void __ip_select_ident(struct iphdr *iph, struct dst_entry *dst, int more)
1267 {
1268         struct rtable *rt = (struct rtable *) dst;
1269
1270         if (rt) {
1271                 if (rt->peer == NULL)
1272                         rt_bind_peer(rt, rt->rt_dst, 1);
1273
1274                 /* If peer is attached to destination, it is never detached,
1275                    so that we need not to grab a lock to dereference it.
1276                  */
1277                 if (rt->peer) {
1278                         iph->id = htons(inet_getid(rt->peer, more));
1279                         return;
1280                 }
1281         } else
1282                 printk(KERN_DEBUG "rt_bind_peer(0) @%p\n",
1283                        __builtin_return_address(0));
1284
1285         ip_select_fb_ident(iph);
1286 }
1287 EXPORT_SYMBOL(__ip_select_ident);
1288
1289 static void rt_del(unsigned hash, struct rtable *rt)
1290 {
1291         struct rtable __rcu **rthp;
1292         struct rtable *aux;
1293
1294         rthp = &rt_hash_table[hash].chain;
1295         spin_lock_bh(rt_hash_lock_addr(hash));
1296         ip_rt_put(rt);
1297         while ((aux = rcu_dereference_protected(*rthp,
1298                         lockdep_is_held(rt_hash_lock_addr(hash)))) != NULL) {
1299                 if (aux == rt || rt_is_expired(aux)) {
1300                         *rthp = aux->dst.rt_next;
1301                         rt_free(aux);
1302                         continue;
1303                 }
1304                 rthp = &aux->dst.rt_next;
1305         }
1306         spin_unlock_bh(rt_hash_lock_addr(hash));
1307 }
1308
1309 /* called in rcu_read_lock() section */
1310 void ip_rt_redirect(__be32 old_gw, __be32 daddr, __be32 new_gw,
1311                     __be32 saddr, struct net_device *dev)
1312 {
1313         struct in_device *in_dev = __in_dev_get_rcu(dev);
1314         struct inet_peer *peer;
1315         struct net *net;
1316
1317         if (!in_dev)
1318                 return;
1319
1320         net = dev_net(dev);
1321         if (new_gw == old_gw || !IN_DEV_RX_REDIRECTS(in_dev) ||
1322             ipv4_is_multicast(new_gw) || ipv4_is_lbcast(new_gw) ||
1323             ipv4_is_zeronet(new_gw))
1324                 goto reject_redirect;
1325
1326         if (!IN_DEV_SHARED_MEDIA(in_dev)) {
1327                 if (!inet_addr_onlink(in_dev, new_gw, old_gw))
1328                         goto reject_redirect;
1329                 if (IN_DEV_SEC_REDIRECTS(in_dev) && ip_fib_check_default(new_gw, dev))
1330                         goto reject_redirect;
1331         } else {
1332                 if (inet_addr_type(net, new_gw) != RTN_UNICAST)
1333                         goto reject_redirect;
1334         }
1335
1336         peer = inet_getpeer_v4(daddr, 1);
1337         if (peer) {
1338                 peer->redirect_learned.a4 = new_gw;
1339
1340                 inet_putpeer(peer);
1341
1342                 atomic_inc(&__rt_peer_genid);
1343         }
1344         return;
1345
1346 reject_redirect:
1347 #ifdef CONFIG_IP_ROUTE_VERBOSE
1348         if (IN_DEV_LOG_MARTIANS(in_dev) && net_ratelimit())
1349                 printk(KERN_INFO "Redirect from %pI4 on %s about %pI4 ignored.\n"
1350                         "  Advised path = %pI4 -> %pI4\n",
1351                        &old_gw, dev->name, &new_gw,
1352                        &saddr, &daddr);
1353 #endif
1354         ;
1355 }
1356
1357 static bool peer_pmtu_expired(struct inet_peer *peer)
1358 {
1359         unsigned long orig = ACCESS_ONCE(peer->pmtu_expires);
1360
1361         return orig &&
1362                time_after_eq(jiffies, orig) &&
1363                cmpxchg(&peer->pmtu_expires, orig, 0) == orig;
1364 }
1365
1366 static bool peer_pmtu_cleaned(struct inet_peer *peer)
1367 {
1368         unsigned long orig = ACCESS_ONCE(peer->pmtu_expires);
1369
1370         return orig &&
1371                cmpxchg(&peer->pmtu_expires, orig, 0) == orig;
1372 }
1373
1374 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst)
1375 {
1376         struct rtable *rt = (struct rtable *)dst;
1377         struct dst_entry *ret = dst;
1378
1379         if (rt) {
1380                 if (dst->obsolete > 0) {
1381                         ip_rt_put(rt);
1382                         ret = NULL;
1383                 } else if (rt->rt_flags & RTCF_REDIRECTED) {
1384                         unsigned hash = rt_hash(rt->rt_key_dst, rt->rt_key_src,
1385                                                 rt->rt_oif,
1386                                                 rt_genid(dev_net(dst->dev)));
1387                         rt_del(hash, rt);
1388                         ret = NULL;
1389                 } else if (rt->peer && peer_pmtu_expired(rt->peer)) {
1390                         dst_metric_set(dst, RTAX_MTU, rt->peer->pmtu_orig);
1391                 }
1392         }
1393         return ret;
1394 }
1395
1396 /*
1397  * Algorithm:
1398  *      1. The first ip_rt_redirect_number redirects are sent
1399  *         with exponential backoff, then we stop sending them at all,
1400  *         assuming that the host ignores our redirects.
1401  *      2. If we did not see packets requiring redirects
1402  *         during ip_rt_redirect_silence, we assume that the host
1403  *         forgot redirected route and start to send redirects again.
1404  *
1405  * This algorithm is much cheaper and more intelligent than dumb load limiting
1406  * in icmp.c.
1407  *
1408  * NOTE. Do not forget to inhibit load limiting for redirects (redundant)
1409  * and "frag. need" (breaks PMTU discovery) in icmp.c.
1410  */
1411
1412 void ip_rt_send_redirect(struct sk_buff *skb)
1413 {
1414         struct rtable *rt = skb_rtable(skb);
1415         struct in_device *in_dev;
1416         struct inet_peer *peer;
1417         int log_martians;
1418
1419         rcu_read_lock();
1420         in_dev = __in_dev_get_rcu(rt->dst.dev);
1421         if (!in_dev || !IN_DEV_TX_REDIRECTS(in_dev)) {
1422                 rcu_read_unlock();
1423                 return;
1424         }
1425         log_martians = IN_DEV_LOG_MARTIANS(in_dev);
1426         rcu_read_unlock();
1427
1428         if (!rt->peer)
1429                 rt_bind_peer(rt, rt->rt_dst, 1);
1430         peer = rt->peer;
1431         if (!peer) {
1432                 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST, rt->rt_gateway);
1433                 return;
1434         }
1435
1436         /* No redirected packets during ip_rt_redirect_silence;
1437          * reset the algorithm.
1438          */
1439         if (time_after(jiffies, peer->rate_last + ip_rt_redirect_silence))
1440                 peer->rate_tokens = 0;
1441
1442         /* Too many ignored redirects; do not send anything
1443          * set dst.rate_last to the last seen redirected packet.
1444          */
1445         if (peer->rate_tokens >= ip_rt_redirect_number) {
1446                 peer->rate_last = jiffies;
1447                 return;
1448         }
1449
1450         /* Check for load limit; set rate_last to the latest sent
1451          * redirect.
1452          */
1453         if (peer->rate_tokens == 0 ||
1454             time_after(jiffies,
1455                        (peer->rate_last +
1456                         (ip_rt_redirect_load << peer->rate_tokens)))) {
1457                 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST, rt->rt_gateway);
1458                 peer->rate_last = jiffies;
1459                 ++peer->rate_tokens;
1460 #ifdef CONFIG_IP_ROUTE_VERBOSE
1461                 if (log_martians &&
1462                     peer->rate_tokens == ip_rt_redirect_number &&
1463                     net_ratelimit())
1464                         printk(KERN_WARNING "host %pI4/if%d ignores redirects for %pI4 to %pI4.\n",
1465                                &ip_hdr(skb)->saddr, rt->rt_iif,
1466                                 &rt->rt_dst, &rt->rt_gateway);
1467 #endif
1468         }
1469 }
1470
1471 static int ip_error(struct sk_buff *skb)
1472 {
1473         struct rtable *rt = skb_rtable(skb);
1474         struct inet_peer *peer;
1475         unsigned long now;
1476         bool send;
1477         int code;
1478
1479         switch (rt->dst.error) {
1480         case EINVAL:
1481         default:
1482                 goto out;
1483         case EHOSTUNREACH:
1484                 code = ICMP_HOST_UNREACH;
1485                 break;
1486         case ENETUNREACH:
1487                 code = ICMP_NET_UNREACH;
1488                 IP_INC_STATS_BH(dev_net(rt->dst.dev),
1489                                 IPSTATS_MIB_INNOROUTES);
1490                 break;
1491         case EACCES:
1492                 code = ICMP_PKT_FILTERED;
1493                 break;
1494         }
1495
1496         if (!rt->peer)
1497                 rt_bind_peer(rt, rt->rt_dst, 1);
1498         peer = rt->peer;
1499
1500         send = true;
1501         if (peer) {
1502                 now = jiffies;
1503                 peer->rate_tokens += now - peer->rate_last;
1504                 if (peer->rate_tokens > ip_rt_error_burst)
1505                         peer->rate_tokens = ip_rt_error_burst;
1506                 peer->rate_last = now;
1507                 if (peer->rate_tokens >= ip_rt_error_cost)
1508                         peer->rate_tokens -= ip_rt_error_cost;
1509                 else
1510                         send = false;
1511         }
1512         if (send)
1513                 icmp_send(skb, ICMP_DEST_UNREACH, code, 0);
1514
1515 out:    kfree_skb(skb);
1516         return 0;
1517 }
1518
1519 /*
1520  *      The last two values are not from the RFC but
1521  *      are needed for AMPRnet AX.25 paths.
1522  */
1523
1524 static const unsigned short mtu_plateau[] =
1525 {32000, 17914, 8166, 4352, 2002, 1492, 576, 296, 216, 128 };
1526
1527 static inline unsigned short guess_mtu(unsigned short old_mtu)
1528 {
1529         int i;
1530
1531         for (i = 0; i < ARRAY_SIZE(mtu_plateau); i++)
1532                 if (old_mtu > mtu_plateau[i])
1533                         return mtu_plateau[i];
1534         return 68;
1535 }
1536
1537 unsigned short ip_rt_frag_needed(struct net *net, const struct iphdr *iph,
1538                                  unsigned short new_mtu,
1539                                  struct net_device *dev)
1540 {
1541         unsigned short old_mtu = ntohs(iph->tot_len);
1542         unsigned short est_mtu = 0;
1543         struct inet_peer *peer;
1544
1545         peer = inet_getpeer_v4(iph->daddr, 1);
1546         if (peer) {
1547                 unsigned short mtu = new_mtu;
1548
1549                 if (new_mtu < 68 || new_mtu >= old_mtu) {
1550                         /* BSD 4.2 derived systems incorrectly adjust
1551                          * tot_len by the IP header length, and report
1552                          * a zero MTU in the ICMP message.
1553                          */
1554                         if (mtu == 0 &&
1555                             old_mtu >= 68 + (iph->ihl << 2))
1556                                 old_mtu -= iph->ihl << 2;
1557                         mtu = guess_mtu(old_mtu);
1558                 }
1559
1560                 if (mtu < ip_rt_min_pmtu)
1561                         mtu = ip_rt_min_pmtu;
1562                 if (!peer->pmtu_expires || mtu < peer->pmtu_learned) {
1563                         unsigned long pmtu_expires;
1564
1565                         pmtu_expires = jiffies + ip_rt_mtu_expires;
1566                         if (!pmtu_expires)
1567                                 pmtu_expires = 1UL;
1568
1569                         est_mtu = mtu;
1570                         peer->pmtu_learned = mtu;
1571                         peer->pmtu_expires = pmtu_expires;
1572                 }
1573
1574                 inet_putpeer(peer);
1575
1576                 atomic_inc(&__rt_peer_genid);
1577         }
1578         return est_mtu ? : new_mtu;
1579 }
1580
1581 static void check_peer_pmtu(struct dst_entry *dst, struct inet_peer *peer)
1582 {
1583         unsigned long expires = ACCESS_ONCE(peer->pmtu_expires);
1584
1585         if (!expires)
1586                 return;
1587         if (time_before(jiffies, expires)) {
1588                 u32 orig_dst_mtu = dst_mtu(dst);
1589                 if (peer->pmtu_learned < orig_dst_mtu) {
1590                         if (!peer->pmtu_orig)
1591                                 peer->pmtu_orig = dst_metric_raw(dst, RTAX_MTU);
1592                         dst_metric_set(dst, RTAX_MTU, peer->pmtu_learned);
1593                 }
1594         } else if (cmpxchg(&peer->pmtu_expires, expires, 0) == expires)
1595                 dst_metric_set(dst, RTAX_MTU, peer->pmtu_orig);
1596 }
1597
1598 static void ip_rt_update_pmtu(struct dst_entry *dst, u32 mtu)
1599 {
1600         struct rtable *rt = (struct rtable *) dst;
1601         struct inet_peer *peer;
1602
1603         dst_confirm(dst);
1604
1605         if (!rt->peer)
1606                 rt_bind_peer(rt, rt->rt_dst, 1);
1607         peer = rt->peer;
1608         if (peer) {
1609                 unsigned long pmtu_expires = ACCESS_ONCE(peer->pmtu_expires);
1610
1611                 if (mtu < ip_rt_min_pmtu)
1612                         mtu = ip_rt_min_pmtu;
1613                 if (!pmtu_expires || mtu < peer->pmtu_learned) {
1614
1615                         pmtu_expires = jiffies + ip_rt_mtu_expires;
1616                         if (!pmtu_expires)
1617                                 pmtu_expires = 1UL;
1618
1619                         peer->pmtu_learned = mtu;
1620                         peer->pmtu_expires = pmtu_expires;
1621
1622                         atomic_inc(&__rt_peer_genid);
1623                         rt->rt_peer_genid = rt_peer_genid();
1624                 }
1625                 check_peer_pmtu(dst, peer);
1626         }
1627 }
1628
1629 static int check_peer_redir(struct dst_entry *dst, struct inet_peer *peer)
1630 {
1631         struct rtable *rt = (struct rtable *) dst;
1632         __be32 orig_gw = rt->rt_gateway;
1633         struct neighbour *n, *old_n;
1634
1635         dst_confirm(&rt->dst);
1636
1637         rt->rt_gateway = peer->redirect_learned.a4;
1638
1639         n = ipv4_neigh_lookup(&rt->dst, &rt->rt_gateway);
1640         if (IS_ERR(n))
1641                 return PTR_ERR(n);
1642         old_n = xchg(&rt->dst._neighbour, n);
1643         if (old_n)
1644                 neigh_release(old_n);
1645         if (!n || !(n->nud_state & NUD_VALID)) {
1646                 if (n)
1647                         neigh_event_send(n, NULL);
1648                 rt->rt_gateway = orig_gw;
1649                 return -EAGAIN;
1650         } else {
1651                 rt->rt_flags |= RTCF_REDIRECTED;
1652                 call_netevent_notifiers(NETEVENT_NEIGH_UPDATE, n);
1653         }
1654         return 0;
1655 }
1656
1657 static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie)
1658 {
1659         struct rtable *rt = (struct rtable *) dst;
1660
1661         if (rt_is_expired(rt))
1662                 return NULL;
1663         if (rt->rt_peer_genid != rt_peer_genid()) {
1664                 struct inet_peer *peer;
1665
1666                 if (!rt->peer)
1667                         rt_bind_peer(rt, rt->rt_dst, 0);
1668
1669                 peer = rt->peer;
1670                 if (peer) {
1671                         check_peer_pmtu(dst, peer);
1672
1673                         if (peer->redirect_learned.a4 &&
1674                             peer->redirect_learned.a4 != rt->rt_gateway) {
1675                                 if (check_peer_redir(dst, peer))
1676                                         return NULL;
1677                         }
1678                 }
1679
1680                 rt->rt_peer_genid = rt_peer_genid();
1681         }
1682         return dst;
1683 }
1684
1685 static void ipv4_dst_destroy(struct dst_entry *dst)
1686 {
1687         struct rtable *rt = (struct rtable *) dst;
1688         struct inet_peer *peer = rt->peer;
1689
1690         if (rt->fi) {
1691                 fib_info_put(rt->fi);
1692                 rt->fi = NULL;
1693         }
1694         if (peer) {
1695                 rt->peer = NULL;
1696                 inet_putpeer(peer);
1697         }
1698 }
1699
1700
1701 static void ipv4_link_failure(struct sk_buff *skb)
1702 {
1703         struct rtable *rt;
1704
1705         icmp_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0);
1706
1707         rt = skb_rtable(skb);
1708         if (rt && rt->peer && peer_pmtu_cleaned(rt->peer))
1709                 dst_metric_set(&rt->dst, RTAX_MTU, rt->peer->pmtu_orig);
1710 }
1711
1712 static int ip_rt_bug(struct sk_buff *skb)
1713 {
1714         printk(KERN_DEBUG "ip_rt_bug: %pI4 -> %pI4, %s\n",
1715                 &ip_hdr(skb)->saddr, &ip_hdr(skb)->daddr,
1716                 skb->dev ? skb->dev->name : "?");
1717         kfree_skb(skb);
1718         WARN_ON(1);
1719         return 0;
1720 }
1721
1722 /*
1723    We do not cache source address of outgoing interface,
1724    because it is used only by IP RR, TS and SRR options,
1725    so that it out of fast path.
1726
1727    BTW remember: "addr" is allowed to be not aligned
1728    in IP options!
1729  */
1730
1731 void ip_rt_get_source(u8 *addr, struct sk_buff *skb, struct rtable *rt)
1732 {
1733         __be32 src;
1734
1735         if (rt_is_output_route(rt))
1736                 src = ip_hdr(skb)->saddr;
1737         else {
1738                 struct fib_result res;
1739                 struct flowi4 fl4;
1740                 struct iphdr *iph;
1741
1742                 iph = ip_hdr(skb);
1743
1744                 memset(&fl4, 0, sizeof(fl4));
1745                 fl4.daddr = iph->daddr;
1746                 fl4.saddr = iph->saddr;
1747                 fl4.flowi4_tos = RT_TOS(iph->tos);
1748                 fl4.flowi4_oif = rt->dst.dev->ifindex;
1749                 fl4.flowi4_iif = skb->dev->ifindex;
1750                 fl4.flowi4_mark = skb->mark;
1751
1752                 rcu_read_lock();
1753                 if (fib_lookup(dev_net(rt->dst.dev), &fl4, &res) == 0)
1754                         src = FIB_RES_PREFSRC(dev_net(rt->dst.dev), res);
1755                 else
1756                         src = inet_select_addr(rt->dst.dev, rt->rt_gateway,
1757                                         RT_SCOPE_UNIVERSE);
1758                 rcu_read_unlock();
1759         }
1760         memcpy(addr, &src, 4);
1761 }
1762
1763 #ifdef CONFIG_IP_ROUTE_CLASSID
1764 static void set_class_tag(struct rtable *rt, u32 tag)
1765 {
1766         if (!(rt->dst.tclassid & 0xFFFF))
1767                 rt->dst.tclassid |= tag & 0xFFFF;
1768         if (!(rt->dst.tclassid & 0xFFFF0000))
1769                 rt->dst.tclassid |= tag & 0xFFFF0000;
1770 }
1771 #endif
1772
1773 static unsigned int ipv4_default_advmss(const struct dst_entry *dst)
1774 {
1775         unsigned int advmss = dst_metric_raw(dst, RTAX_ADVMSS);
1776
1777         if (advmss == 0) {
1778                 advmss = max_t(unsigned int, dst->dev->mtu - 40,
1779                                ip_rt_min_advmss);
1780                 if (advmss > 65535 - 40)
1781                         advmss = 65535 - 40;
1782         }
1783         return advmss;
1784 }
1785
1786 static unsigned int ipv4_default_mtu(const struct dst_entry *dst)
1787 {
1788         unsigned int mtu = dst->dev->mtu;
1789
1790         if (unlikely(dst_metric_locked(dst, RTAX_MTU))) {
1791                 const struct rtable *rt = (const struct rtable *) dst;
1792
1793                 if (rt->rt_gateway != rt->rt_dst && mtu > 576)
1794                         mtu = 576;
1795         }
1796
1797         if (mtu > IP_MAX_MTU)
1798                 mtu = IP_MAX_MTU;
1799
1800         return mtu;
1801 }
1802
1803 static void rt_init_metrics(struct rtable *rt, const struct flowi4 *fl4,
1804                             struct fib_info *fi)
1805 {
1806         struct inet_peer *peer;
1807         int create = 0;
1808
1809         /* If a peer entry exists for this destination, we must hook
1810          * it up in order to get at cached metrics.
1811          */
1812         if (fl4 && (fl4->flowi4_flags & FLOWI_FLAG_PRECOW_METRICS))
1813                 create = 1;
1814
1815         rt->peer = peer = inet_getpeer_v4(rt->rt_dst, create);
1816         if (peer) {
1817                 rt->rt_peer_genid = rt_peer_genid();
1818                 if (inet_metrics_new(peer))
1819                         memcpy(peer->metrics, fi->fib_metrics,
1820                                sizeof(u32) * RTAX_MAX);
1821                 dst_init_metrics(&rt->dst, peer->metrics, false);
1822
1823                 check_peer_pmtu(&rt->dst, peer);
1824                 if (peer->redirect_learned.a4 &&
1825                     peer->redirect_learned.a4 != rt->rt_gateway) {
1826                         rt->rt_gateway = peer->redirect_learned.a4;
1827                         rt->rt_flags |= RTCF_REDIRECTED;
1828                 }
1829         } else {
1830                 if (fi->fib_metrics != (u32 *) dst_default_metrics) {
1831                         rt->fi = fi;
1832                         atomic_inc(&fi->fib_clntref);
1833                 }
1834                 dst_init_metrics(&rt->dst, fi->fib_metrics, true);
1835         }
1836 }
1837
1838 static void rt_set_nexthop(struct rtable *rt, const struct flowi4 *fl4,
1839                            const struct fib_result *res,
1840                            struct fib_info *fi, u16 type, u32 itag)
1841 {
1842         struct dst_entry *dst = &rt->dst;
1843
1844         if (fi) {
1845                 if (FIB_RES_GW(*res) &&
1846                     FIB_RES_NH(*res).nh_scope == RT_SCOPE_LINK)
1847                         rt->rt_gateway = FIB_RES_GW(*res);
1848                 rt_init_metrics(rt, fl4, fi);
1849 #ifdef CONFIG_IP_ROUTE_CLASSID
1850                 dst->tclassid = FIB_RES_NH(*res).nh_tclassid;
1851 #endif
1852         }
1853
1854         if (dst_mtu(dst) > IP_MAX_MTU)
1855                 dst_metric_set(dst, RTAX_MTU, IP_MAX_MTU);
1856         if (dst_metric_raw(dst, RTAX_ADVMSS) > 65535 - 40)
1857                 dst_metric_set(dst, RTAX_ADVMSS, 65535 - 40);
1858
1859 #ifdef CONFIG_IP_ROUTE_CLASSID
1860 #ifdef CONFIG_IP_MULTIPLE_TABLES
1861         set_class_tag(rt, fib_rules_tclass(res));
1862 #endif
1863         set_class_tag(rt, itag);
1864 #endif
1865 }
1866
1867 static struct rtable *rt_dst_alloc(struct net_device *dev,
1868                                    bool nopolicy, bool noxfrm)
1869 {
1870         return dst_alloc(&ipv4_dst_ops, dev, 1, -1,
1871                          DST_HOST |
1872                          (nopolicy ? DST_NOPOLICY : 0) |
1873                          (noxfrm ? DST_NOXFRM : 0));
1874 }
1875
1876 /* called in rcu_read_lock() section */
1877 static int ip_route_input_mc(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1878                                 u8 tos, struct net_device *dev, int our)
1879 {
1880         unsigned int hash;
1881         struct rtable *rth;
1882         __be32 spec_dst;
1883         struct in_device *in_dev = __in_dev_get_rcu(dev);
1884         u32 itag = 0;
1885         int err;
1886
1887         /* Primary sanity checks. */
1888
1889         if (in_dev == NULL)
1890                 return -EINVAL;
1891
1892         if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) ||
1893             ipv4_is_loopback(saddr) || skb->protocol != htons(ETH_P_IP))
1894                 goto e_inval;
1895
1896         if (ipv4_is_zeronet(saddr)) {
1897                 if (!ipv4_is_local_multicast(daddr))
1898                         goto e_inval;
1899                 spec_dst = inet_select_addr(dev, 0, RT_SCOPE_LINK);
1900         } else {
1901                 err = fib_validate_source(skb, saddr, 0, tos, 0, dev, &spec_dst,
1902                                           &itag);
1903                 if (err < 0)
1904                         goto e_err;
1905         }
1906         rth = rt_dst_alloc(init_net.loopback_dev,
1907                            IN_DEV_CONF_GET(in_dev, NOPOLICY), false);
1908         if (!rth)
1909                 goto e_nobufs;
1910
1911 #ifdef CONFIG_IP_ROUTE_CLASSID
1912         rth->dst.tclassid = itag;
1913 #endif
1914         rth->dst.output = ip_rt_bug;
1915
1916         rth->rt_key_dst = daddr;
1917         rth->rt_key_src = saddr;
1918         rth->rt_genid   = rt_genid(dev_net(dev));
1919         rth->rt_flags   = RTCF_MULTICAST;
1920         rth->rt_type    = RTN_MULTICAST;
1921         rth->rt_key_tos = tos;
1922         rth->rt_dst     = daddr;
1923         rth->rt_src     = saddr;
1924         rth->rt_route_iif = dev->ifindex;
1925         rth->rt_iif     = dev->ifindex;
1926         rth->rt_oif     = 0;
1927         rth->rt_mark    = skb->mark;
1928         rth->rt_gateway = daddr;
1929         rth->rt_spec_dst= spec_dst;
1930         rth->rt_peer_genid = 0;
1931         rth->peer = NULL;
1932         rth->fi = NULL;
1933         if (our) {
1934                 rth->dst.input= ip_local_deliver;
1935                 rth->rt_flags |= RTCF_LOCAL;
1936         }
1937
1938 #ifdef CONFIG_IP_MROUTE
1939         if (!ipv4_is_local_multicast(daddr) && IN_DEV_MFORWARD(in_dev))
1940                 rth->dst.input = ip_mr_input;
1941 #endif
1942         RT_CACHE_STAT_INC(in_slow_mc);
1943
1944         hash = rt_hash(daddr, saddr, dev->ifindex, rt_genid(dev_net(dev)));
1945         rth = rt_intern_hash(hash, rth, skb, dev->ifindex);
1946         return IS_ERR(rth) ? PTR_ERR(rth) : 0;
1947
1948 e_nobufs:
1949         return -ENOBUFS;
1950 e_inval:
1951         return -EINVAL;
1952 e_err:
1953         return err;
1954 }
1955
1956
1957 static void ip_handle_martian_source(struct net_device *dev,
1958                                      struct in_device *in_dev,
1959                                      struct sk_buff *skb,
1960                                      __be32 daddr,
1961                                      __be32 saddr)
1962 {
1963         RT_CACHE_STAT_INC(in_martian_src);
1964 #ifdef CONFIG_IP_ROUTE_VERBOSE
1965         if (IN_DEV_LOG_MARTIANS(in_dev) && net_ratelimit()) {
1966                 /*
1967                  *      RFC1812 recommendation, if source is martian,
1968                  *      the only hint is MAC header.
1969                  */
1970                 printk(KERN_WARNING "martian source %pI4 from %pI4, on dev %s\n",
1971                         &daddr, &saddr, dev->name);
1972                 if (dev->hard_header_len && skb_mac_header_was_set(skb)) {
1973                         int i;
1974                         const unsigned char *p = skb_mac_header(skb);
1975                         printk(KERN_WARNING "ll header: ");
1976                         for (i = 0; i < dev->hard_header_len; i++, p++) {
1977                                 printk("%02x", *p);
1978                                 if (i < (dev->hard_header_len - 1))
1979                                         printk(":");
1980                         }
1981                         printk("\n");
1982                 }
1983         }
1984 #endif
1985 }
1986
1987 /* called in rcu_read_lock() section */
1988 static int __mkroute_input(struct sk_buff *skb,
1989                            const struct fib_result *res,
1990                            struct in_device *in_dev,
1991                            __be32 daddr, __be32 saddr, u32 tos,
1992                            struct rtable **result)
1993 {
1994         struct rtable *rth;
1995         int err;
1996         struct in_device *out_dev;
1997         unsigned int flags = 0;
1998         __be32 spec_dst;
1999         u32 itag;
2000
2001         /* get a working reference to the output device */
2002         out_dev = __in_dev_get_rcu(FIB_RES_DEV(*res));
2003         if (out_dev == NULL) {
2004                 if (net_ratelimit())
2005                         printk(KERN_CRIT "Bug in ip_route_input" \
2006                                "_slow(). Please, report\n");
2007                 return -EINVAL;
2008         }
2009
2010
2011         err = fib_validate_source(skb, saddr, daddr, tos, FIB_RES_OIF(*res),
2012                                   in_dev->dev, &spec_dst, &itag);
2013         if (err < 0) {
2014                 ip_handle_martian_source(in_dev->dev, in_dev, skb, daddr,
2015                                          saddr);
2016
2017                 goto cleanup;
2018         }
2019
2020         if (err)
2021                 flags |= RTCF_DIRECTSRC;
2022
2023         if (out_dev == in_dev && err &&
2024             (IN_DEV_SHARED_MEDIA(out_dev) ||
2025              inet_addr_onlink(out_dev, saddr, FIB_RES_GW(*res))))
2026                 flags |= RTCF_DOREDIRECT;
2027
2028         if (skb->protocol != htons(ETH_P_IP)) {
2029                 /* Not IP (i.e. ARP). Do not create route, if it is
2030                  * invalid for proxy arp. DNAT routes are always valid.
2031                  *
2032                  * Proxy arp feature have been extended to allow, ARP
2033                  * replies back to the same interface, to support
2034                  * Private VLAN switch technologies. See arp.c.
2035                  */
2036                 if (out_dev == in_dev &&
2037                     IN_DEV_PROXY_ARP_PVLAN(in_dev) == 0) {
2038                         err = -EINVAL;
2039                         goto cleanup;
2040                 }
2041         }
2042
2043         rth = rt_dst_alloc(out_dev->dev,
2044                            IN_DEV_CONF_GET(in_dev, NOPOLICY),
2045                            IN_DEV_CONF_GET(out_dev, NOXFRM));
2046         if (!rth) {
2047                 err = -ENOBUFS;
2048                 goto cleanup;
2049         }
2050
2051         rth->rt_key_dst = daddr;
2052         rth->rt_key_src = saddr;
2053         rth->rt_genid = rt_genid(dev_net(rth->dst.dev));
2054         rth->rt_flags = flags;
2055         rth->rt_type = res->type;
2056         rth->rt_key_tos = tos;
2057         rth->rt_dst     = daddr;
2058         rth->rt_src     = saddr;
2059         rth->rt_route_iif = in_dev->dev->ifindex;
2060         rth->rt_iif     = in_dev->dev->ifindex;
2061         rth->rt_oif     = 0;
2062         rth->rt_mark    = skb->mark;
2063         rth->rt_gateway = daddr;
2064         rth->rt_spec_dst= spec_dst;
2065         rth->rt_peer_genid = 0;
2066         rth->peer = NULL;
2067         rth->fi = NULL;
2068
2069         rth->dst.input = ip_forward;
2070         rth->dst.output = ip_output;
2071
2072         rt_set_nexthop(rth, NULL, res, res->fi, res->type, itag);
2073
2074         *result = rth;
2075         err = 0;
2076  cleanup:
2077         return err;
2078 }
2079
2080 static int ip_mkroute_input(struct sk_buff *skb,
2081                             struct fib_result *res,
2082                             const struct flowi4 *fl4,
2083                             struct in_device *in_dev,
2084                             __be32 daddr, __be32 saddr, u32 tos)
2085 {
2086         struct rtable* rth = NULL;
2087         int err;
2088         unsigned hash;
2089
2090 #ifdef CONFIG_IP_ROUTE_MULTIPATH
2091         if (res->fi && res->fi->fib_nhs > 1)
2092                 fib_select_multipath(res);
2093 #endif
2094
2095         /* create a routing cache entry */
2096         err = __mkroute_input(skb, res, in_dev, daddr, saddr, tos, &rth);
2097         if (err)
2098                 return err;
2099
2100         /* put it into the cache */
2101         hash = rt_hash(daddr, saddr, fl4->flowi4_iif,
2102                        rt_genid(dev_net(rth->dst.dev)));
2103         rth = rt_intern_hash(hash, rth, skb, fl4->flowi4_iif);
2104         if (IS_ERR(rth))
2105                 return PTR_ERR(rth);
2106         return 0;
2107 }
2108
2109 /*
2110  *      NOTE. We drop all the packets that has local source
2111  *      addresses, because every properly looped back packet
2112  *      must have correct destination already attached by output routine.
2113  *
2114  *      Such approach solves two big problems:
2115  *      1. Not simplex devices are handled properly.
2116  *      2. IP spoofing attempts are filtered with 100% of guarantee.
2117  *      called with rcu_read_lock()
2118  */
2119
2120 static int ip_route_input_slow(struct sk_buff *skb, __be32 daddr, __be32 saddr,
2121                                u8 tos, struct net_device *dev)
2122 {
2123         struct fib_result res;
2124         struct in_device *in_dev = __in_dev_get_rcu(dev);
2125         struct flowi4   fl4;
2126         unsigned        flags = 0;
2127         u32             itag = 0;
2128         struct rtable * rth;
2129         unsigned        hash;
2130         __be32          spec_dst;
2131         int             err = -EINVAL;
2132         struct net    * net = dev_net(dev);
2133
2134         /* IP on this device is disabled. */
2135
2136         if (!in_dev)
2137                 goto out;
2138
2139         /* Check for the most weird martians, which can be not detected
2140            by fib_lookup.
2141          */
2142
2143         if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) ||
2144             ipv4_is_loopback(saddr))
2145                 goto martian_source;
2146
2147         if (ipv4_is_lbcast(daddr) || (saddr == 0 && daddr == 0))
2148                 goto brd_input;
2149
2150         /* Accept zero addresses only to limited broadcast;
2151          * I even do not know to fix it or not. Waiting for complains :-)
2152          */
2153         if (ipv4_is_zeronet(saddr))
2154                 goto martian_source;
2155
2156         if (ipv4_is_zeronet(daddr) || ipv4_is_loopback(daddr))
2157                 goto martian_destination;
2158
2159         /*
2160          *      Now we are ready to route packet.
2161          */
2162         fl4.flowi4_oif = 0;
2163         fl4.flowi4_iif = dev->ifindex;
2164         fl4.flowi4_mark = skb->mark;
2165         fl4.flowi4_tos = tos;
2166         fl4.flowi4_scope = RT_SCOPE_UNIVERSE;
2167         fl4.daddr = daddr;
2168         fl4.saddr = saddr;
2169         err = fib_lookup(net, &fl4, &res);
2170         if (err != 0) {
2171                 if (!IN_DEV_FORWARD(in_dev))
2172                         goto e_hostunreach;
2173                 goto no_route;
2174         }
2175
2176         RT_CACHE_STAT_INC(in_slow_tot);
2177
2178         if (res.type == RTN_BROADCAST)
2179                 goto brd_input;
2180
2181         if (res.type == RTN_LOCAL) {
2182                 err = fib_validate_source(skb, saddr, daddr, tos,
2183                                           net->loopback_dev->ifindex,
2184                                           dev, &spec_dst, &itag);
2185                 if (err < 0)
2186                         goto martian_source_keep_err;
2187                 if (err)
2188                         flags |= RTCF_DIRECTSRC;
2189                 spec_dst = daddr;
2190                 goto local_input;
2191         }
2192
2193         if (!IN_DEV_FORWARD(in_dev))
2194                 goto e_hostunreach;
2195         if (res.type != RTN_UNICAST)
2196                 goto martian_destination;
2197
2198         err = ip_mkroute_input(skb, &res, &fl4, in_dev, daddr, saddr, tos);
2199 out:    return err;
2200
2201 brd_input:
2202         if (skb->protocol != htons(ETH_P_IP))
2203                 goto e_inval;
2204
2205         if (ipv4_is_zeronet(saddr))
2206                 spec_dst = inet_select_addr(dev, 0, RT_SCOPE_LINK);
2207         else {
2208                 err = fib_validate_source(skb, saddr, 0, tos, 0, dev, &spec_dst,
2209                                           &itag);
2210                 if (err < 0)
2211                         goto martian_source_keep_err;
2212                 if (err)
2213                         flags |= RTCF_DIRECTSRC;
2214         }
2215         flags |= RTCF_BROADCAST;
2216         res.type = RTN_BROADCAST;
2217         RT_CACHE_STAT_INC(in_brd);
2218
2219 local_input:
2220         rth = rt_dst_alloc(net->loopback_dev,
2221                            IN_DEV_CONF_GET(in_dev, NOPOLICY), false);
2222         if (!rth)
2223                 goto e_nobufs;
2224
2225         rth->dst.input= ip_local_deliver;
2226         rth->dst.output= ip_rt_bug;
2227 #ifdef CONFIG_IP_ROUTE_CLASSID
2228         rth->dst.tclassid = itag;
2229 #endif
2230
2231         rth->rt_key_dst = daddr;
2232         rth->rt_key_src = saddr;
2233         rth->rt_genid = rt_genid(net);
2234         rth->rt_flags   = flags|RTCF_LOCAL;
2235         rth->rt_type    = res.type;
2236         rth->rt_key_tos = tos;
2237         rth->rt_dst     = daddr;
2238         rth->rt_src     = saddr;
2239 #ifdef CONFIG_IP_ROUTE_CLASSID
2240         rth->dst.tclassid = itag;
2241 #endif
2242         rth->rt_route_iif = dev->ifindex;
2243         rth->rt_iif     = dev->ifindex;
2244         rth->rt_oif     = 0;
2245         rth->rt_mark    = skb->mark;
2246         rth->rt_gateway = daddr;
2247         rth->rt_spec_dst= spec_dst;
2248         rth->rt_peer_genid = 0;
2249         rth->peer = NULL;
2250         rth->fi = NULL;
2251         if (res.type == RTN_UNREACHABLE) {
2252                 rth->dst.input= ip_error;
2253                 rth->dst.error= -err;
2254                 rth->rt_flags   &= ~RTCF_LOCAL;
2255         }
2256         hash = rt_hash(daddr, saddr, fl4.flowi4_iif, rt_genid(net));
2257         rth = rt_intern_hash(hash, rth, skb, fl4.flowi4_iif);
2258         err = 0;
2259         if (IS_ERR(rth))
2260                 err = PTR_ERR(rth);
2261         goto out;
2262
2263 no_route:
2264         RT_CACHE_STAT_INC(in_no_route);
2265         spec_dst = inet_select_addr(dev, 0, RT_SCOPE_UNIVERSE);
2266         res.type = RTN_UNREACHABLE;
2267         if (err == -ESRCH)
2268                 err = -ENETUNREACH;
2269         goto local_input;
2270
2271         /*
2272          *      Do not cache martian addresses: they should be logged (RFC1812)
2273          */
2274 martian_destination:
2275         RT_CACHE_STAT_INC(in_martian_dst);
2276 #ifdef CONFIG_IP_ROUTE_VERBOSE
2277         if (IN_DEV_LOG_MARTIANS(in_dev) && net_ratelimit())
2278                 printk(KERN_WARNING "martian destination %pI4 from %pI4, dev %s\n",
2279                         &daddr, &saddr, dev->name);
2280 #endif
2281
2282 e_hostunreach:
2283         err = -EHOSTUNREACH;
2284         goto out;
2285
2286 e_inval:
2287         err = -EINVAL;
2288         goto out;
2289
2290 e_nobufs:
2291         err = -ENOBUFS;
2292         goto out;
2293
2294 martian_source:
2295         err = -EINVAL;
2296 martian_source_keep_err:
2297         ip_handle_martian_source(dev, in_dev, skb, daddr, saddr);
2298         goto out;
2299 }
2300
2301 int ip_route_input_common(struct sk_buff *skb, __be32 daddr, __be32 saddr,
2302                            u8 tos, struct net_device *dev, bool noref)
2303 {
2304         struct rtable * rth;
2305         unsigned        hash;
2306         int iif = dev->ifindex;
2307         struct net *net;
2308         int res;
2309
2310         net = dev_net(dev);
2311
2312         rcu_read_lock();
2313
2314         if (!rt_caching(net))
2315                 goto skip_cache;
2316
2317         tos &= IPTOS_RT_MASK;
2318         hash = rt_hash(daddr, saddr, iif, rt_genid(net));
2319
2320         for (rth = rcu_dereference(rt_hash_table[hash].chain); rth;
2321              rth = rcu_dereference(rth->dst.rt_next)) {
2322                 if ((((__force u32)rth->rt_key_dst ^ (__force u32)daddr) |
2323                      ((__force u32)rth->rt_key_src ^ (__force u32)saddr) |
2324                      (rth->rt_iif ^ iif) |
2325                      (rth->rt_key_tos ^ tos)) == 0 &&
2326                     rt_is_input_route(rth) &&
2327                     rth->rt_mark == skb->mark &&
2328                     net_eq(dev_net(rth->dst.dev), net) &&
2329                     !rt_is_expired(rth)) {
2330                         if (noref) {
2331                                 dst_use_noref(&rth->dst, jiffies);
2332                                 skb_dst_set_noref(skb, &rth->dst);
2333                         } else {
2334                                 dst_use(&rth->dst, jiffies);
2335                                 skb_dst_set(skb, &rth->dst);
2336                         }
2337                         RT_CACHE_STAT_INC(in_hit);
2338                         rcu_read_unlock();
2339                         return 0;
2340                 }
2341                 RT_CACHE_STAT_INC(in_hlist_search);
2342         }
2343
2344 skip_cache:
2345         /* Multicast recognition logic is moved from route cache to here.
2346            The problem was that too many Ethernet cards have broken/missing
2347            hardware multicast filters :-( As result the host on multicasting
2348            network acquires a lot of useless route cache entries, sort of
2349            SDR messages from all the world. Now we try to get rid of them.
2350            Really, provided software IP multicast filter is organized
2351            reasonably (at least, hashed), it does not result in a slowdown
2352            comparing with route cache reject entries.
2353            Note, that multicast routers are not affected, because
2354            route cache entry is created eventually.
2355          */
2356         if (ipv4_is_multicast(daddr)) {
2357                 struct in_device *in_dev = __in_dev_get_rcu(dev);
2358
2359                 if (in_dev) {
2360                         int our = ip_check_mc_rcu(in_dev, daddr, saddr,
2361                                                   ip_hdr(skb)->protocol);
2362                         if (our
2363 #ifdef CONFIG_IP_MROUTE
2364                                 ||
2365                             (!ipv4_is_local_multicast(daddr) &&
2366                              IN_DEV_MFORWARD(in_dev))
2367 #endif
2368                            ) {
2369                                 int res = ip_route_input_mc(skb, daddr, saddr,
2370                                                             tos, dev, our);
2371                                 rcu_read_unlock();
2372                                 return res;
2373                         }
2374                 }
2375                 rcu_read_unlock();
2376                 return -EINVAL;
2377         }
2378         res = ip_route_input_slow(skb, daddr, saddr, tos, dev);
2379         rcu_read_unlock();
2380         return res;
2381 }
2382 EXPORT_SYMBOL(ip_route_input_common);
2383
2384 /* called with rcu_read_lock() */
2385 static struct rtable *__mkroute_output(const struct fib_result *res,
2386                                        const struct flowi4 *fl4,
2387                                        __be32 orig_daddr, __be32 orig_saddr,
2388                                        int orig_oif, struct net_device *dev_out,
2389                                        unsigned int flags)
2390 {
2391         struct fib_info *fi = res->fi;
2392         u32 tos = RT_FL_TOS(fl4);
2393         struct in_device *in_dev;
2394         u16 type = res->type;
2395         struct rtable *rth;
2396
2397         if (ipv4_is_loopback(fl4->saddr) && !(dev_out->flags & IFF_LOOPBACK))
2398                 return ERR_PTR(-EINVAL);
2399
2400         if (ipv4_is_lbcast(fl4->daddr))
2401                 type = RTN_BROADCAST;
2402         else if (ipv4_is_multicast(fl4->daddr))
2403                 type = RTN_MULTICAST;
2404         else if (ipv4_is_zeronet(fl4->daddr))
2405                 return ERR_PTR(-EINVAL);
2406
2407         if (dev_out->flags & IFF_LOOPBACK)
2408                 flags |= RTCF_LOCAL;
2409
2410         in_dev = __in_dev_get_rcu(dev_out);
2411         if (!in_dev)
2412                 return ERR_PTR(-EINVAL);
2413
2414         if (type == RTN_BROADCAST) {
2415                 flags |= RTCF_BROADCAST | RTCF_LOCAL;
2416                 fi = NULL;
2417         } else if (type == RTN_MULTICAST) {
2418                 flags |= RTCF_MULTICAST | RTCF_LOCAL;
2419                 if (!ip_check_mc_rcu(in_dev, fl4->daddr, fl4->saddr,
2420                                      fl4->flowi4_proto))
2421                         flags &= ~RTCF_LOCAL;
2422                 /* If multicast route do not exist use
2423                  * default one, but do not gateway in this case.
2424                  * Yes, it is hack.
2425                  */
2426                 if (fi && res->prefixlen < 4)
2427                         fi = NULL;
2428         }
2429
2430         rth = rt_dst_alloc(dev_out,
2431                            IN_DEV_CONF_GET(in_dev, NOPOLICY),
2432                            IN_DEV_CONF_GET(in_dev, NOXFRM));
2433         if (!rth)
2434                 return ERR_PTR(-ENOBUFS);
2435
2436         rth->dst.output = ip_output;
2437
2438         rth->rt_key_dst = orig_daddr;
2439         rth->rt_key_src = orig_saddr;
2440         rth->rt_genid = rt_genid(dev_net(dev_out));
2441         rth->rt_flags   = flags;
2442         rth->rt_type    = type;
2443         rth->rt_key_tos = tos;
2444         rth->rt_dst     = fl4->daddr;
2445         rth->rt_src     = fl4->saddr;
2446         rth->rt_route_iif = 0;
2447         rth->rt_iif     = orig_oif ? : dev_out->ifindex;
2448         rth->rt_oif     = orig_oif;
2449         rth->rt_mark    = fl4->flowi4_mark;
2450         rth->rt_gateway = fl4->daddr;
2451         rth->rt_spec_dst= fl4->saddr;
2452         rth->rt_peer_genid = 0;
2453         rth->peer = NULL;
2454         rth->fi = NULL;
2455
2456         RT_CACHE_STAT_INC(out_slow_tot);
2457
2458         if (flags & RTCF_LOCAL) {
2459                 rth->dst.input = ip_local_deliver;
2460                 rth->rt_spec_dst = fl4->daddr;
2461         }
2462         if (flags & (RTCF_BROADCAST | RTCF_MULTICAST)) {
2463                 rth->rt_spec_dst = fl4->saddr;
2464                 if (flags & RTCF_LOCAL &&
2465                     !(dev_out->flags & IFF_LOOPBACK)) {
2466                         rth->dst.output = ip_mc_output;
2467                         RT_CACHE_STAT_INC(out_slow_mc);
2468                 }
2469 #ifdef CONFIG_IP_MROUTE
2470                 if (type == RTN_MULTICAST) {
2471                         if (IN_DEV_MFORWARD(in_dev) &&
2472                             !ipv4_is_local_multicast(fl4->daddr)) {
2473                                 rth->dst.input = ip_mr_input;
2474                                 rth->dst.output = ip_mc_output;
2475                         }
2476                 }
2477 #endif
2478         }
2479
2480         rt_set_nexthop(rth, fl4, res, fi, type, 0);
2481
2482         return rth;
2483 }
2484
2485 /*
2486  * Major route resolver routine.
2487  * called with rcu_read_lock();
2488  */
2489
2490 static struct rtable *ip_route_output_slow(struct net *net, struct flowi4 *fl4)
2491 {
2492         struct net_device *dev_out = NULL;
2493         u32 tos = RT_FL_TOS(fl4);
2494         unsigned int flags = 0;
2495         struct fib_result res;
2496         struct rtable *rth;
2497         __be32 orig_daddr;
2498         __be32 orig_saddr;
2499         int orig_oif;
2500
2501         res.fi          = NULL;
2502 #ifdef CONFIG_IP_MULTIPLE_TABLES
2503         res.r           = NULL;
2504 #endif
2505
2506         orig_daddr = fl4->daddr;
2507         orig_saddr = fl4->saddr;
2508         orig_oif = fl4->flowi4_oif;
2509
2510         fl4->flowi4_iif = net->loopback_dev->ifindex;
2511         fl4->flowi4_tos = tos & IPTOS_RT_MASK;
2512         fl4->flowi4_scope = ((tos & RTO_ONLINK) ?
2513                          RT_SCOPE_LINK : RT_SCOPE_UNIVERSE);
2514
2515         rcu_read_lock();
2516         if (fl4->saddr) {
2517                 rth = ERR_PTR(-EINVAL);
2518                 if (ipv4_is_multicast(fl4->saddr) ||
2519                     ipv4_is_lbcast(fl4->saddr) ||
2520                     ipv4_is_zeronet(fl4->saddr))
2521                         goto out;
2522
2523                 /* I removed check for oif == dev_out->oif here.
2524                    It was wrong for two reasons:
2525                    1. ip_dev_find(net, saddr) can return wrong iface, if saddr
2526                       is assigned to multiple interfaces.
2527                    2. Moreover, we are allowed to send packets with saddr
2528                       of another iface. --ANK
2529                  */
2530
2531                 if (fl4->flowi4_oif == 0 &&
2532                     (ipv4_is_multicast(fl4->daddr) ||
2533                      ipv4_is_lbcast(fl4->daddr))) {
2534                         /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
2535                         dev_out = __ip_dev_find(net, fl4->saddr, false);
2536                         if (dev_out == NULL)
2537                                 goto out;
2538
2539                         /* Special hack: user can direct multicasts
2540                            and limited broadcast via necessary interface
2541                            without fiddling with IP_MULTICAST_IF or IP_PKTINFO.
2542                            This hack is not just for fun, it allows
2543                            vic,vat and friends to work.
2544                            They bind socket to loopback, set ttl to zero
2545                            and expect that it will work.
2546                            From the viewpoint of routing cache they are broken,
2547                            because we are not allowed to build multicast path
2548                            with loopback source addr (look, routing cache
2549                            cannot know, that ttl is zero, so that packet
2550                            will not leave this host and route is valid).
2551                            Luckily, this hack is good workaround.
2552                          */
2553
2554                         fl4->flowi4_oif = dev_out->ifindex;
2555                         goto make_route;
2556                 }
2557
2558                 if (!(fl4->flowi4_flags & FLOWI_FLAG_ANYSRC)) {
2559                         /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
2560                         if (!__ip_dev_find(net, fl4->saddr, false))
2561                                 goto out;
2562                 }
2563         }
2564
2565
2566         if (fl4->flowi4_oif) {
2567                 dev_out = dev_get_by_index_rcu(net, fl4->flowi4_oif);
2568                 rth = ERR_PTR(-ENODEV);
2569                 if (dev_out == NULL)
2570                         goto out;
2571
2572                 /* RACE: Check return value of inet_select_addr instead. */
2573                 if (!(dev_out->flags & IFF_UP) || !__in_dev_get_rcu(dev_out)) {
2574                         rth = ERR_PTR(-ENETUNREACH);
2575                         goto out;
2576                 }
2577                 if (ipv4_is_local_multicast(fl4->daddr) ||
2578                     ipv4_is_lbcast(fl4->daddr)) {
2579                         if (!fl4->saddr)
2580                                 fl4->saddr = inet_select_addr(dev_out, 0,
2581                                                               RT_SCOPE_LINK);
2582                         goto make_route;
2583                 }
2584                 if (fl4->saddr) {
2585                         if (ipv4_is_multicast(fl4->daddr))
2586                                 fl4->saddr = inet_select_addr(dev_out, 0,
2587                                                               fl4->flowi4_scope);
2588                         else if (!fl4->daddr)
2589                                 fl4->saddr = inet_select_addr(dev_out, 0,
2590                                                               RT_SCOPE_HOST);
2591                 }
2592         }
2593
2594         if (!fl4->daddr) {
2595                 fl4->daddr = fl4->saddr;
2596                 if (!fl4->daddr)
2597                         fl4->daddr = fl4->saddr = htonl(INADDR_LOOPBACK);
2598                 dev_out = net->loopback_dev;
2599                 fl4->flowi4_oif = net->loopback_dev->ifindex;
2600                 res.type = RTN_LOCAL;
2601                 flags |= RTCF_LOCAL;
2602                 goto make_route;
2603         }
2604
2605         if (fib_lookup(net, fl4, &res)) {
2606                 res.fi = NULL;
2607                 if (fl4->flowi4_oif) {
2608                         /* Apparently, routing tables are wrong. Assume,
2609                            that the destination is on link.
2610
2611                            WHY? DW.
2612                            Because we are allowed to send to iface
2613                            even if it has NO routes and NO assigned
2614                            addresses. When oif is specified, routing
2615                            tables are looked up with only one purpose:
2616                            to catch if destination is gatewayed, rather than
2617                            direct. Moreover, if MSG_DONTROUTE is set,
2618                            we send packet, ignoring both routing tables
2619                            and ifaddr state. --ANK
2620
2621
2622                            We could make it even if oif is unknown,
2623                            likely IPv6, but we do not.
2624                          */
2625
2626                         if (fl4->saddr == 0)
2627                                 fl4->saddr = inet_select_addr(dev_out, 0,
2628                                                               RT_SCOPE_LINK);
2629                         res.type = RTN_UNICAST;
2630                         goto make_route;
2631                 }
2632                 rth = ERR_PTR(-ENETUNREACH);
2633                 goto out;
2634         }
2635
2636         if (res.type == RTN_LOCAL) {
2637                 if (!fl4->saddr) {
2638                         if (res.fi->fib_prefsrc)
2639                                 fl4->saddr = res.fi->fib_prefsrc;
2640                         else
2641                                 fl4->saddr = fl4->daddr;
2642                 }
2643                 dev_out = net->loopback_dev;
2644                 fl4->flowi4_oif = dev_out->ifindex;
2645                 res.fi = NULL;
2646                 flags |= RTCF_LOCAL;
2647                 goto make_route;
2648         }
2649
2650 #ifdef CONFIG_IP_ROUTE_MULTIPATH
2651         if (res.fi->fib_nhs > 1 && fl4->flowi4_oif == 0)
2652                 fib_select_multipath(&res);
2653         else
2654 #endif
2655         if (!res.prefixlen &&
2656             res.table->tb_num_default > 1 &&
2657             res.type == RTN_UNICAST && !fl4->flowi4_oif)
2658                 fib_select_default(&res);
2659
2660         if (!fl4->saddr)
2661                 fl4->saddr = FIB_RES_PREFSRC(net, res);
2662
2663         dev_out = FIB_RES_DEV(res);
2664         fl4->flowi4_oif = dev_out->ifindex;
2665
2666
2667 make_route:
2668         rth = __mkroute_output(&res, fl4, orig_daddr, orig_saddr, orig_oif,
2669                                dev_out, flags);
2670         if (!IS_ERR(rth)) {
2671                 unsigned int hash;
2672
2673                 hash = rt_hash(orig_daddr, orig_saddr, orig_oif,
2674                                rt_genid(dev_net(dev_out)));
2675                 rth = rt_intern_hash(hash, rth, NULL, orig_oif);
2676         }
2677
2678 out:
2679         rcu_read_unlock();
2680         return rth;
2681 }
2682
2683 struct rtable *__ip_route_output_key(struct net *net, struct flowi4 *flp4)
2684 {
2685         struct rtable *rth;
2686         unsigned int hash;
2687
2688         if (!rt_caching(net))
2689                 goto slow_output;
2690
2691         hash = rt_hash(flp4->daddr, flp4->saddr, flp4->flowi4_oif, rt_genid(net));
2692
2693         rcu_read_lock_bh();
2694         for (rth = rcu_dereference_bh(rt_hash_table[hash].chain); rth;
2695                 rth = rcu_dereference_bh(rth->dst.rt_next)) {
2696                 if (rth->rt_key_dst == flp4->daddr &&
2697                     rth->rt_key_src == flp4->saddr &&
2698                     rt_is_output_route(rth) &&
2699                     rth->rt_oif == flp4->flowi4_oif &&
2700                     rth->rt_mark == flp4->flowi4_mark &&
2701                     !((rth->rt_key_tos ^ flp4->flowi4_tos) &
2702                             (IPTOS_RT_MASK | RTO_ONLINK)) &&
2703                     net_eq(dev_net(rth->dst.dev), net) &&
2704                     !rt_is_expired(rth)) {
2705                         dst_use(&rth->dst, jiffies);
2706                         RT_CACHE_STAT_INC(out_hit);
2707                         rcu_read_unlock_bh();
2708                         if (!flp4->saddr)
2709                                 flp4->saddr = rth->rt_src;
2710                         if (!flp4->daddr)
2711                                 flp4->daddr = rth->rt_dst;
2712                         return rth;
2713                 }
2714                 RT_CACHE_STAT_INC(out_hlist_search);
2715         }
2716         rcu_read_unlock_bh();
2717
2718 slow_output:
2719         return ip_route_output_slow(net, flp4);
2720 }
2721 EXPORT_SYMBOL_GPL(__ip_route_output_key);
2722
2723 static struct dst_entry *ipv4_blackhole_dst_check(struct dst_entry *dst, u32 cookie)
2724 {
2725         return NULL;
2726 }
2727
2728 static unsigned int ipv4_blackhole_default_mtu(const struct dst_entry *dst)
2729 {
2730         return 0;
2731 }
2732
2733 static void ipv4_rt_blackhole_update_pmtu(struct dst_entry *dst, u32 mtu)
2734 {
2735 }
2736
2737 static u32 *ipv4_rt_blackhole_cow_metrics(struct dst_entry *dst,
2738                                           unsigned long old)
2739 {
2740         return NULL;
2741 }
2742
2743 static struct dst_ops ipv4_dst_blackhole_ops = {
2744         .family                 =       AF_INET,
2745         .protocol               =       cpu_to_be16(ETH_P_IP),
2746         .destroy                =       ipv4_dst_destroy,
2747         .check                  =       ipv4_blackhole_dst_check,
2748         .default_mtu            =       ipv4_blackhole_default_mtu,
2749         .default_advmss         =       ipv4_default_advmss,
2750         .update_pmtu            =       ipv4_rt_blackhole_update_pmtu,
2751         .cow_metrics            =       ipv4_rt_blackhole_cow_metrics,
2752         .neigh_lookup           =       ipv4_neigh_lookup,
2753 };
2754
2755 struct dst_entry *ipv4_blackhole_route(struct net *net, struct dst_entry *dst_orig)
2756 {
2757         struct rtable *rt = dst_alloc(&ipv4_dst_blackhole_ops, NULL, 1, 0, 0);
2758         struct rtable *ort = (struct rtable *) dst_orig;
2759
2760         if (rt) {
2761                 struct dst_entry *new = &rt->dst;
2762
2763                 new->__use = 1;
2764                 new->input = dst_discard;
2765                 new->output = dst_discard;
2766                 dst_copy_metrics(new, &ort->dst);
2767
2768                 new->dev = ort->dst.dev;
2769                 if (new->dev)
2770                         dev_hold(new->dev);
2771
2772                 rt->rt_key_dst = ort->rt_key_dst;
2773                 rt->rt_key_src = ort->rt_key_src;
2774                 rt->rt_key_tos = ort->rt_key_tos;
2775                 rt->rt_route_iif = ort->rt_route_iif;
2776                 rt->rt_iif = ort->rt_iif;
2777                 rt->rt_oif = ort->rt_oif;
2778                 rt->rt_mark = ort->rt_mark;
2779
2780                 rt->rt_genid = rt_genid(net);
2781                 rt->rt_flags = ort->rt_flags;
2782                 rt->rt_type = ort->rt_type;
2783                 rt->rt_dst = ort->rt_dst;
2784                 rt->rt_src = ort->rt_src;
2785                 rt->rt_gateway = ort->rt_gateway;
2786                 rt->rt_spec_dst = ort->rt_spec_dst;
2787                 rt->peer = ort->peer;
2788                 if (rt->peer)
2789                         atomic_inc(&rt->peer->refcnt);
2790                 rt->fi = ort->fi;
2791                 if (rt->fi)
2792                         atomic_inc(&rt->fi->fib_clntref);
2793
2794                 dst_free(new);
2795         }
2796
2797         dst_release(dst_orig);
2798
2799         return rt ? &rt->dst : ERR_PTR(-ENOMEM);
2800 }
2801
2802 struct rtable *ip_route_output_flow(struct net *net, struct flowi4 *flp4,
2803                                     struct sock *sk)
2804 {
2805         struct rtable *rt = __ip_route_output_key(net, flp4);
2806
2807         if (IS_ERR(rt))
2808                 return rt;
2809
2810         if (flp4->flowi4_proto)
2811                 rt = (struct rtable *) xfrm_lookup(net, &rt->dst,
2812                                                    flowi4_to_flowi(flp4),
2813                                                    sk, 0);
2814
2815         return rt;
2816 }
2817 EXPORT_SYMBOL_GPL(ip_route_output_flow);
2818
2819 static int rt_fill_info(struct net *net,
2820                         struct sk_buff *skb, u32 pid, u32 seq, int event,
2821                         int nowait, unsigned int flags)
2822 {
2823         struct rtable *rt = skb_rtable(skb);
2824         struct rtmsg *r;
2825         struct nlmsghdr *nlh;
2826         long expires = 0;
2827         const struct inet_peer *peer = rt->peer;
2828         u32 id = 0, ts = 0, tsage = 0, error;
2829
2830         nlh = nlmsg_put(skb, pid, seq, event, sizeof(*r), flags);
2831         if (nlh == NULL)
2832                 return -EMSGSIZE;
2833
2834         r = nlmsg_data(nlh);
2835         r->rtm_family    = AF_INET;
2836         r->rtm_dst_len  = 32;
2837         r->rtm_src_len  = 0;
2838         r->rtm_tos      = rt->rt_key_tos;
2839         r->rtm_table    = RT_TABLE_MAIN;
2840         NLA_PUT_U32(skb, RTA_TABLE, RT_TABLE_MAIN);
2841         r->rtm_type     = rt->rt_type;
2842         r->rtm_scope    = RT_SCOPE_UNIVERSE;
2843         r->rtm_protocol = RTPROT_UNSPEC;
2844         r->rtm_flags    = (rt->rt_flags & ~0xFFFF) | RTM_F_CLONED;
2845         if (rt->rt_flags & RTCF_NOTIFY)
2846                 r->rtm_flags |= RTM_F_NOTIFY;
2847
2848         NLA_PUT_BE32(skb, RTA_DST, rt->rt_dst);
2849
2850         if (rt->rt_key_src) {
2851                 r->rtm_src_len = 32;
2852                 NLA_PUT_BE32(skb, RTA_SRC, rt->rt_key_src);
2853         }
2854         if (rt->dst.dev)
2855                 NLA_PUT_U32(skb, RTA_OIF, rt->dst.dev->ifindex);
2856 #ifdef CONFIG_IP_ROUTE_CLASSID
2857         if (rt->dst.tclassid)
2858                 NLA_PUT_U32(skb, RTA_FLOW, rt->dst.tclassid);
2859 #endif
2860         if (rt_is_input_route(rt))
2861                 NLA_PUT_BE32(skb, RTA_PREFSRC, rt->rt_spec_dst);
2862         else if (rt->rt_src != rt->rt_key_src)
2863                 NLA_PUT_BE32(skb, RTA_PREFSRC, rt->rt_src);
2864
2865         if (rt->rt_dst != rt->rt_gateway)
2866                 NLA_PUT_BE32(skb, RTA_GATEWAY, rt->rt_gateway);
2867
2868         if (rtnetlink_put_metrics(skb, dst_metrics_ptr(&rt->dst)) < 0)
2869                 goto nla_put_failure;
2870
2871         if (rt->rt_mark)
2872                 NLA_PUT_BE32(skb, RTA_MARK, rt->rt_mark);
2873
2874         error = rt->dst.error;
2875         if (peer) {
2876                 inet_peer_refcheck(rt->peer);
2877                 id = atomic_read(&peer->ip_id_count) & 0xffff;
2878                 if (peer->tcp_ts_stamp) {
2879                         ts = peer->tcp_ts;
2880                         tsage = get_seconds() - peer->tcp_ts_stamp;
2881                 }
2882                 expires = ACCESS_ONCE(peer->pmtu_expires);
2883                 if (expires)
2884                         expires -= jiffies;
2885         }
2886
2887         if (rt_is_input_route(rt)) {
2888 #ifdef CONFIG_IP_MROUTE
2889                 __be32 dst = rt->rt_dst;
2890
2891                 if (ipv4_is_multicast(dst) && !ipv4_is_local_multicast(dst) &&
2892                     IPV4_DEVCONF_ALL(net, MC_FORWARDING)) {
2893                         int err = ipmr_get_route(net, skb,
2894                                                  rt->rt_src, rt->rt_dst,
2895                                                  r, nowait);
2896                         if (err <= 0) {
2897                                 if (!nowait) {
2898                                         if (err == 0)
2899                                                 return 0;
2900                                         goto nla_put_failure;
2901                                 } else {
2902                                         if (err == -EMSGSIZE)
2903                                                 goto nla_put_failure;
2904                                         error = err;
2905                                 }
2906                         }
2907                 } else
2908 #endif
2909                         NLA_PUT_U32(skb, RTA_IIF, rt->rt_iif);
2910         }
2911
2912         if (rtnl_put_cacheinfo(skb, &rt->dst, id, ts, tsage,
2913                                expires, error) < 0)
2914                 goto nla_put_failure;
2915
2916         return nlmsg_end(skb, nlh);
2917
2918 nla_put_failure:
2919         nlmsg_cancel(skb, nlh);
2920         return -EMSGSIZE;
2921 }
2922
2923 static int inet_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr* nlh, void *arg)
2924 {
2925         struct net *net = sock_net(in_skb->sk);
2926         struct rtmsg *rtm;
2927         struct nlattr *tb[RTA_MAX+1];
2928         struct rtable *rt = NULL;
2929         __be32 dst = 0;
2930         __be32 src = 0;
2931         u32 iif;
2932         int err;
2933         int mark;
2934         struct sk_buff *skb;
2935
2936         err = nlmsg_parse(nlh, sizeof(*rtm), tb, RTA_MAX, rtm_ipv4_policy);
2937         if (err < 0)
2938                 goto errout;
2939
2940         rtm = nlmsg_data(nlh);
2941
2942         skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
2943         if (skb == NULL) {
2944                 err = -ENOBUFS;
2945                 goto errout;
2946         }
2947
2948         /* Reserve room for dummy headers, this skb can pass
2949            through good chunk of routing engine.
2950          */
2951         skb_reset_mac_header(skb);
2952         skb_reset_network_header(skb);
2953
2954         /* Bugfix: need to give ip_route_input enough of an IP header to not gag. */
2955         ip_hdr(skb)->protocol = IPPROTO_ICMP;
2956         skb_reserve(skb, MAX_HEADER + sizeof(struct iphdr));
2957
2958         src = tb[RTA_SRC] ? nla_get_be32(tb[RTA_SRC]) : 0;
2959         dst = tb[RTA_DST] ? nla_get_be32(tb[RTA_DST]) : 0;
2960         iif = tb[RTA_IIF] ? nla_get_u32(tb[RTA_IIF]) : 0;
2961         mark = tb[RTA_MARK] ? nla_get_u32(tb[RTA_MARK]) : 0;
2962
2963         if (iif) {
2964                 struct net_device *dev;
2965
2966                 dev = __dev_get_by_index(net, iif);
2967                 if (dev == NULL) {
2968                         err = -ENODEV;
2969                         goto errout_free;
2970                 }
2971
2972                 skb->protocol   = htons(ETH_P_IP);
2973                 skb->dev        = dev;
2974                 skb->mark       = mark;
2975                 local_bh_disable();
2976                 err = ip_route_input(skb, dst, src, rtm->rtm_tos, dev);
2977                 local_bh_enable();
2978
2979                 rt = skb_rtable(skb);
2980                 if (err == 0 && rt->dst.error)
2981                         err = -rt->dst.error;
2982         } else {
2983                 struct flowi4 fl4 = {
2984                         .daddr = dst,
2985                         .saddr = src,
2986                         .flowi4_tos = rtm->rtm_tos,
2987                         .flowi4_oif = tb[RTA_OIF] ? nla_get_u32(tb[RTA_OIF]) : 0,
2988                         .flowi4_mark = mark,
2989                 };
2990                 rt = ip_route_output_key(net, &fl4);
2991
2992                 err = 0;
2993                 if (IS_ERR(rt))
2994                         err = PTR_ERR(rt);
2995         }
2996
2997         if (err)
2998                 goto errout_free;
2999
3000         skb_dst_set(skb, &rt->dst);
3001         if (rtm->rtm_flags & RTM_F_NOTIFY)
3002                 rt->rt_flags |= RTCF_NOTIFY;
3003
3004         err = rt_fill_info(net, skb, NETLINK_CB(in_skb).pid, nlh->nlmsg_seq,
3005                            RTM_NEWROUTE, 0, 0);
3006         if (err <= 0)
3007                 goto errout_free;
3008
3009         err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).pid);
3010 errout:
3011         return err;
3012
3013 errout_free:
3014         kfree_skb(skb);
3015         goto errout;
3016 }
3017
3018 int ip_rt_dump(struct sk_buff *skb,  struct netlink_callback *cb)
3019 {
3020         struct rtable *rt;
3021         int h, s_h;
3022         int idx, s_idx;
3023         struct net *net;
3024
3025         net = sock_net(skb->sk);
3026
3027         s_h = cb->args[0];
3028         if (s_h < 0)
3029                 s_h = 0;
3030         s_idx = idx = cb->args[1];
3031         for (h = s_h; h <= rt_hash_mask; h++, s_idx = 0) {
3032                 if (!rt_hash_table[h].chain)
3033                         continue;
3034                 rcu_read_lock_bh();
3035                 for (rt = rcu_dereference_bh(rt_hash_table[h].chain), idx = 0; rt;
3036                      rt = rcu_dereference_bh(rt->dst.rt_next), idx++) {
3037                         if (!net_eq(dev_net(rt->dst.dev), net) || idx < s_idx)
3038                                 continue;
3039                         if (rt_is_expired(rt))
3040                                 continue;
3041                         skb_dst_set_noref(skb, &rt->dst);
3042                         if (rt_fill_info(net, skb, NETLINK_CB(cb->skb).pid,
3043                                          cb->nlh->nlmsg_seq, RTM_NEWROUTE,
3044                                          1, NLM_F_MULTI) <= 0) {
3045                                 skb_dst_drop(skb);
3046                                 rcu_read_unlock_bh();
3047                                 goto done;
3048                         }
3049                         skb_dst_drop(skb);
3050                 }
3051                 rcu_read_unlock_bh();
3052         }
3053
3054 done:
3055         cb->args[0] = h;
3056         cb->args[1] = idx;
3057         return skb->len;
3058 }
3059
3060 void ip_rt_multicast_event(struct in_device *in_dev)
3061 {
3062         rt_cache_flush(dev_net(in_dev->dev), 0);
3063 }
3064
3065 #ifdef CONFIG_SYSCTL
3066 static int ipv4_sysctl_rtcache_flush(ctl_table *__ctl, int write,
3067                                         void __user *buffer,
3068                                         size_t *lenp, loff_t *ppos)
3069 {
3070         if (write) {
3071                 int flush_delay;
3072                 ctl_table ctl;
3073                 struct net *net;
3074
3075                 memcpy(&ctl, __ctl, sizeof(ctl));
3076                 ctl.data = &flush_delay;
3077                 proc_dointvec(&ctl, write, buffer, lenp, ppos);
3078
3079                 net = (struct net *)__ctl->extra1;
3080                 rt_cache_flush(net, flush_delay);
3081                 return 0;
3082         }
3083
3084         return -EINVAL;
3085 }
3086
3087 static ctl_table ipv4_route_table[] = {
3088         {
3089                 .procname       = "gc_thresh",
3090                 .data           = &ipv4_dst_ops.gc_thresh,
3091                 .maxlen         = sizeof(int),
3092                 .mode           = 0644,
3093                 .proc_handler   = proc_dointvec,
3094         },
3095         {
3096                 .procname       = "max_size",
3097                 .data           = &ip_rt_max_size,
3098                 .maxlen         = sizeof(int),
3099                 .mode           = 0644,
3100                 .proc_handler   = proc_dointvec,
3101         },
3102         {
3103                 /*  Deprecated. Use gc_min_interval_ms */
3104
3105                 .procname       = "gc_min_interval",
3106                 .data           = &ip_rt_gc_min_interval,
3107                 .maxlen         = sizeof(int),
3108                 .mode           = 0644,
3109                 .proc_handler   = proc_dointvec_jiffies,
3110         },
3111         {
3112                 .procname       = "gc_min_interval_ms",
3113                 .data           = &ip_rt_gc_min_interval,
3114                 .maxlen         = sizeof(int),
3115                 .mode           = 0644,
3116                 .proc_handler   = proc_dointvec_ms_jiffies,
3117         },
3118         {
3119                 .procname       = "gc_timeout",
3120                 .data           = &ip_rt_gc_timeout,
3121                 .maxlen         = sizeof(int),
3122                 .mode           = 0644,
3123                 .proc_handler   = proc_dointvec_jiffies,
3124         },
3125         {
3126                 .procname       = "gc_interval",
3127                 .data           = &ip_rt_gc_interval,
3128                 .maxlen         = sizeof(int),
3129                 .mode           = 0644,
3130                 .proc_handler   = proc_dointvec_jiffies,
3131         },
3132         {
3133                 .procname       = "redirect_load",
3134                 .data           = &ip_rt_redirect_load,
3135                 .maxlen         = sizeof(int),
3136                 .mode           = 0644,
3137                 .proc_handler   = proc_dointvec,
3138         },
3139         {
3140                 .procname       = "redirect_number",
3141                 .data           = &ip_rt_redirect_number,
3142                 .maxlen         = sizeof(int),
3143                 .mode           = 0644,
3144                 .proc_handler   = proc_dointvec,
3145         },
3146         {
3147                 .procname       = "redirect_silence",
3148                 .data           = &ip_rt_redirect_silence,
3149                 .maxlen         = sizeof(int),
3150                 .mode           = 0644,
3151                 .proc_handler   = proc_dointvec,
3152         },
3153         {
3154                 .procname       = "error_cost",
3155                 .data           = &ip_rt_error_cost,
3156                 .maxlen         = sizeof(int),
3157                 .mode           = 0644,
3158                 .proc_handler   = proc_dointvec,
3159         },
3160         {
3161                 .procname       = "error_burst",
3162                 .data           = &ip_rt_error_burst,
3163                 .maxlen         = sizeof(int),
3164                 .mode           = 0644,
3165                 .proc_handler   = proc_dointvec,
3166         },
3167         {
3168                 .procname       = "gc_elasticity",
3169                 .data           = &ip_rt_gc_elasticity,
3170                 .maxlen         = sizeof(int),
3171                 .mode           = 0644,
3172                 .proc_handler   = proc_dointvec,
3173         },
3174         {
3175                 .procname       = "mtu_expires",
3176                 .data           = &ip_rt_mtu_expires,
3177                 .maxlen         = sizeof(int),
3178                 .mode           = 0644,
3179                 .proc_handler   = proc_dointvec_jiffies,
3180         },
3181         {
3182                 .procname       = "min_pmtu",
3183                 .data           = &ip_rt_min_pmtu,
3184                 .maxlen         = sizeof(int),
3185                 .mode           = 0644,
3186                 .proc_handler   = proc_dointvec,
3187         },
3188         {
3189                 .procname       = "min_adv_mss",
3190                 .data           = &ip_rt_min_advmss,
3191                 .maxlen         = sizeof(int),
3192                 .mode           = 0644,
3193                 .proc_handler   = proc_dointvec,
3194         },
3195         { }
3196 };
3197
3198 static struct ctl_table empty[1];
3199
3200 static struct ctl_table ipv4_skeleton[] =
3201 {
3202         { .procname = "route", 
3203           .mode = 0555, .child = ipv4_route_table},
3204         { .procname = "neigh", 
3205           .mode = 0555, .child = empty},
3206         { }
3207 };
3208
3209 static __net_initdata struct ctl_path ipv4_path[] = {
3210         { .procname = "net", },
3211         { .procname = "ipv4", },
3212         { },
3213 };
3214
3215 static struct ctl_table ipv4_route_flush_table[] = {
3216         {
3217                 .procname       = "flush",
3218                 .maxlen         = sizeof(int),
3219                 .mode           = 0200,
3220                 .proc_handler   = ipv4_sysctl_rtcache_flush,
3221         },
3222         { },
3223 };
3224
3225 static __net_initdata struct ctl_path ipv4_route_path[] = {
3226         { .procname = "net", },
3227         { .procname = "ipv4", },
3228         { .procname = "route", },
3229         { },
3230 };
3231
3232 static __net_init int sysctl_route_net_init(struct net *net)
3233 {
3234         struct ctl_table *tbl;
3235
3236         tbl = ipv4_route_flush_table;
3237         if (!net_eq(net, &init_net)) {
3238                 tbl = kmemdup(tbl, sizeof(ipv4_route_flush_table), GFP_KERNEL);
3239                 if (tbl == NULL)
3240                         goto err_dup;
3241         }
3242         tbl[0].extra1 = net;
3243
3244         net->ipv4.route_hdr =
3245                 register_net_sysctl_table(net, ipv4_route_path, tbl);
3246         if (net->ipv4.route_hdr == NULL)
3247                 goto err_reg;
3248         return 0;
3249
3250 err_reg:
3251         if (tbl != ipv4_route_flush_table)
3252                 kfree(tbl);
3253 err_dup:
3254         return -ENOMEM;
3255 }
3256
3257 static __net_exit void sysctl_route_net_exit(struct net *net)
3258 {
3259         struct ctl_table *tbl;
3260
3261         tbl = net->ipv4.route_hdr->ctl_table_arg;
3262         unregister_net_sysctl_table(net->ipv4.route_hdr);
3263         BUG_ON(tbl == ipv4_route_flush_table);
3264         kfree(tbl);
3265 }
3266
3267 static __net_initdata struct pernet_operations sysctl_route_ops = {
3268         .init = sysctl_route_net_init,
3269         .exit = sysctl_route_net_exit,
3270 };
3271 #endif
3272
3273 static __net_init int rt_genid_init(struct net *net)
3274 {
3275         get_random_bytes(&net->ipv4.rt_genid,
3276                          sizeof(net->ipv4.rt_genid));
3277         get_random_bytes(&net->ipv4.dev_addr_genid,
3278                          sizeof(net->ipv4.dev_addr_genid));
3279         return 0;
3280 }
3281
3282 static __net_initdata struct pernet_operations rt_genid_ops = {
3283         .init = rt_genid_init,
3284 };
3285
3286
3287 #ifdef CONFIG_IP_ROUTE_CLASSID
3288 struct ip_rt_acct __percpu *ip_rt_acct __read_mostly;
3289 #endif /* CONFIG_IP_ROUTE_CLASSID */
3290
3291 static __initdata unsigned long rhash_entries;
3292 static int __init set_rhash_entries(char *str)
3293 {
3294         if (!str)
3295                 return 0;
3296         rhash_entries = simple_strtoul(str, &str, 0);
3297         return 1;
3298 }
3299 __setup("rhash_entries=", set_rhash_entries);
3300
3301 int __init ip_rt_init(void)
3302 {
3303         int rc = 0;
3304
3305 #ifdef CONFIG_IP_ROUTE_CLASSID
3306         ip_rt_acct = __alloc_percpu(256 * sizeof(struct ip_rt_acct), __alignof__(struct ip_rt_acct));
3307         if (!ip_rt_acct)
3308                 panic("IP: failed to allocate ip_rt_acct\n");
3309 #endif
3310
3311         ipv4_dst_ops.kmem_cachep =
3312                 kmem_cache_create("ip_dst_cache", sizeof(struct rtable), 0,
3313                                   SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
3314
3315         ipv4_dst_blackhole_ops.kmem_cachep = ipv4_dst_ops.kmem_cachep;
3316
3317         if (dst_entries_init(&ipv4_dst_ops) < 0)
3318                 panic("IP: failed to allocate ipv4_dst_ops counter\n");
3319
3320         if (dst_entries_init(&ipv4_dst_blackhole_ops) < 0)
3321                 panic("IP: failed to allocate ipv4_dst_blackhole_ops counter\n");
3322
3323         rt_hash_table = (struct rt_hash_bucket *)
3324                 alloc_large_system_hash("IP route cache",
3325                                         sizeof(struct rt_hash_bucket),
3326                                         rhash_entries,
3327                                         (totalram_pages >= 128 * 1024) ?
3328                                         15 : 17,
3329                                         0,
3330                                         &rt_hash_log,
3331                                         &rt_hash_mask,
3332                                         rhash_entries ? 0 : 512 * 1024);
3333         memset(rt_hash_table, 0, (rt_hash_mask + 1) * sizeof(struct rt_hash_bucket));
3334         rt_hash_lock_init();
3335
3336         ipv4_dst_ops.gc_thresh = (rt_hash_mask + 1);
3337         ip_rt_max_size = (rt_hash_mask + 1) * 16;
3338
3339         devinet_init();
3340         ip_fib_init();
3341
3342         if (ip_rt_proc_init())
3343                 printk(KERN_ERR "Unable to create route proc files\n");
3344 #ifdef CONFIG_XFRM
3345         xfrm_init();
3346         xfrm4_init(ip_rt_max_size);
3347 #endif
3348         rtnl_register(PF_INET, RTM_GETROUTE, inet_rtm_getroute, NULL, NULL);
3349
3350 #ifdef CONFIG_SYSCTL
3351         register_pernet_subsys(&sysctl_route_ops);
3352 #endif
3353         register_pernet_subsys(&rt_genid_ops);
3354         return rc;
3355 }
3356
3357 #ifdef CONFIG_SYSCTL
3358 /*
3359  * We really need to sanitize the damn ipv4 init order, then all
3360  * this nonsense will go away.
3361  */
3362 void __init ip_static_sysctl_init(void)
3363 {
3364         register_sysctl_paths(ipv4_path, ipv4_skeleton);
3365 }
3366 #endif