net: Don't report route RTT metric value in cache dumps.
[linux-3.10.git] / net / ipv4 / route.c
1 /*
2  * INET         An implementation of the TCP/IP protocol suite for the LINUX
3  *              operating system.  INET is implemented using the  BSD Socket
4  *              interface as the means of communication with the user level.
5  *
6  *              ROUTE - implementation of the IP router.
7  *
8  * Authors:     Ross Biro
9  *              Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10  *              Alan Cox, <gw4pts@gw4pts.ampr.org>
11  *              Linus Torvalds, <Linus.Torvalds@helsinki.fi>
12  *              Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
13  *
14  * Fixes:
15  *              Alan Cox        :       Verify area fixes.
16  *              Alan Cox        :       cli() protects routing changes
17  *              Rui Oliveira    :       ICMP routing table updates
18  *              (rco@di.uminho.pt)      Routing table insertion and update
19  *              Linus Torvalds  :       Rewrote bits to be sensible
20  *              Alan Cox        :       Added BSD route gw semantics
21  *              Alan Cox        :       Super /proc >4K
22  *              Alan Cox        :       MTU in route table
23  *              Alan Cox        :       MSS actually. Also added the window
24  *                                      clamper.
25  *              Sam Lantinga    :       Fixed route matching in rt_del()
26  *              Alan Cox        :       Routing cache support.
27  *              Alan Cox        :       Removed compatibility cruft.
28  *              Alan Cox        :       RTF_REJECT support.
29  *              Alan Cox        :       TCP irtt support.
30  *              Jonathan Naylor :       Added Metric support.
31  *      Miquel van Smoorenburg  :       BSD API fixes.
32  *      Miquel van Smoorenburg  :       Metrics.
33  *              Alan Cox        :       Use __u32 properly
34  *              Alan Cox        :       Aligned routing errors more closely with BSD
35  *                                      our system is still very different.
36  *              Alan Cox        :       Faster /proc handling
37  *      Alexey Kuznetsov        :       Massive rework to support tree based routing,
38  *                                      routing caches and better behaviour.
39  *
40  *              Olaf Erb        :       irtt wasn't being copied right.
41  *              Bjorn Ekwall    :       Kerneld route support.
42  *              Alan Cox        :       Multicast fixed (I hope)
43  *              Pavel Krauz     :       Limited broadcast fixed
44  *              Mike McLagan    :       Routing by source
45  *      Alexey Kuznetsov        :       End of old history. Split to fib.c and
46  *                                      route.c and rewritten from scratch.
47  *              Andi Kleen      :       Load-limit warning messages.
48  *      Vitaly E. Lavrov        :       Transparent proxy revived after year coma.
49  *      Vitaly E. Lavrov        :       Race condition in ip_route_input_slow.
50  *      Tobias Ringstrom        :       Uninitialized res.type in ip_route_output_slow.
51  *      Vladimir V. Ivanov      :       IP rule info (flowid) is really useful.
52  *              Marc Boucher    :       routing by fwmark
53  *      Robert Olsson           :       Added rt_cache statistics
54  *      Arnaldo C. Melo         :       Convert proc stuff to seq_file
55  *      Eric Dumazet            :       hashed spinlocks and rt_check_expire() fixes.
56  *      Ilia Sotnikov           :       Ignore TOS on PMTUD and Redirect
57  *      Ilia Sotnikov           :       Removed TOS from hash calculations
58  *
59  *              This program is free software; you can redistribute it and/or
60  *              modify it under the terms of the GNU General Public License
61  *              as published by the Free Software Foundation; either version
62  *              2 of the License, or (at your option) any later version.
63  */
64
65 #define pr_fmt(fmt) "IPv4: " fmt
66
67 #include <linux/module.h>
68 #include <asm/uaccess.h>
69 #include <linux/bitops.h>
70 #include <linux/types.h>
71 #include <linux/kernel.h>
72 #include <linux/mm.h>
73 #include <linux/bootmem.h>
74 #include <linux/string.h>
75 #include <linux/socket.h>
76 #include <linux/sockios.h>
77 #include <linux/errno.h>
78 #include <linux/in.h>
79 #include <linux/inet.h>
80 #include <linux/netdevice.h>
81 #include <linux/proc_fs.h>
82 #include <linux/init.h>
83 #include <linux/workqueue.h>
84 #include <linux/skbuff.h>
85 #include <linux/inetdevice.h>
86 #include <linux/igmp.h>
87 #include <linux/pkt_sched.h>
88 #include <linux/mroute.h>
89 #include <linux/netfilter_ipv4.h>
90 #include <linux/random.h>
91 #include <linux/jhash.h>
92 #include <linux/rcupdate.h>
93 #include <linux/times.h>
94 #include <linux/slab.h>
95 #include <linux/prefetch.h>
96 #include <net/dst.h>
97 #include <net/net_namespace.h>
98 #include <net/protocol.h>
99 #include <net/ip.h>
100 #include <net/route.h>
101 #include <net/inetpeer.h>
102 #include <net/sock.h>
103 #include <net/ip_fib.h>
104 #include <net/arp.h>
105 #include <net/tcp.h>
106 #include <net/icmp.h>
107 #include <net/xfrm.h>
108 #include <net/netevent.h>
109 #include <net/rtnetlink.h>
110 #ifdef CONFIG_SYSCTL
111 #include <linux/sysctl.h>
112 #include <linux/kmemleak.h>
113 #endif
114 #include <net/secure_seq.h>
115
116 #define RT_FL_TOS(oldflp4) \
117         ((oldflp4)->flowi4_tos & (IPTOS_RT_MASK | RTO_ONLINK))
118
119 #define IP_MAX_MTU      0xFFF0
120
121 #define RT_GC_TIMEOUT (300*HZ)
122
123 static int ip_rt_max_size;
124 static int ip_rt_gc_timeout __read_mostly       = RT_GC_TIMEOUT;
125 static int ip_rt_gc_interval __read_mostly  = 60 * HZ;
126 static int ip_rt_gc_min_interval __read_mostly  = HZ / 2;
127 static int ip_rt_redirect_number __read_mostly  = 9;
128 static int ip_rt_redirect_load __read_mostly    = HZ / 50;
129 static int ip_rt_redirect_silence __read_mostly = ((HZ / 50) << (9 + 1));
130 static int ip_rt_error_cost __read_mostly       = HZ;
131 static int ip_rt_error_burst __read_mostly      = 5 * HZ;
132 static int ip_rt_gc_elasticity __read_mostly    = 8;
133 static int ip_rt_mtu_expires __read_mostly      = 10 * 60 * HZ;
134 static int ip_rt_min_pmtu __read_mostly         = 512 + 20 + 20;
135 static int ip_rt_min_advmss __read_mostly       = 256;
136 static int rt_chain_length_max __read_mostly    = 20;
137
138 static struct delayed_work expires_work;
139 static unsigned long expires_ljiffies;
140
141 /*
142  *      Interface to generic destination cache.
143  */
144
145 static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie);
146 static unsigned int      ipv4_default_advmss(const struct dst_entry *dst);
147 static unsigned int      ipv4_mtu(const struct dst_entry *dst);
148 static void              ipv4_dst_destroy(struct dst_entry *dst);
149 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst);
150 static void              ipv4_link_failure(struct sk_buff *skb);
151 static void              ip_rt_update_pmtu(struct dst_entry *dst, u32 mtu);
152 static int rt_garbage_collect(struct dst_ops *ops);
153
154 static void ipv4_dst_ifdown(struct dst_entry *dst, struct net_device *dev,
155                             int how)
156 {
157 }
158
159 static u32 *ipv4_cow_metrics(struct dst_entry *dst, unsigned long old)
160 {
161         struct rtable *rt = (struct rtable *) dst;
162         struct inet_peer *peer;
163         u32 *p = NULL;
164
165         peer = rt_get_peer_create(rt, rt->rt_dst);
166         if (peer) {
167                 u32 *old_p = __DST_METRICS_PTR(old);
168                 unsigned long prev, new;
169
170                 p = peer->metrics;
171                 if (inet_metrics_new(peer))
172                         memcpy(p, old_p, sizeof(u32) * RTAX_MAX);
173
174                 new = (unsigned long) p;
175                 prev = cmpxchg(&dst->_metrics, old, new);
176
177                 if (prev != old) {
178                         p = __DST_METRICS_PTR(prev);
179                         if (prev & DST_METRICS_READ_ONLY)
180                                 p = NULL;
181                 } else {
182                         if (rt->fi) {
183                                 fib_info_put(rt->fi);
184                                 rt->fi = NULL;
185                         }
186                 }
187         }
188         return p;
189 }
190
191 static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst,
192                                            struct sk_buff *skb,
193                                            const void *daddr);
194
195 static struct dst_ops ipv4_dst_ops = {
196         .family =               AF_INET,
197         .protocol =             cpu_to_be16(ETH_P_IP),
198         .gc =                   rt_garbage_collect,
199         .check =                ipv4_dst_check,
200         .default_advmss =       ipv4_default_advmss,
201         .mtu =                  ipv4_mtu,
202         .cow_metrics =          ipv4_cow_metrics,
203         .destroy =              ipv4_dst_destroy,
204         .ifdown =               ipv4_dst_ifdown,
205         .negative_advice =      ipv4_negative_advice,
206         .link_failure =         ipv4_link_failure,
207         .update_pmtu =          ip_rt_update_pmtu,
208         .local_out =            __ip_local_out,
209         .neigh_lookup =         ipv4_neigh_lookup,
210 };
211
212 #define ECN_OR_COST(class)      TC_PRIO_##class
213
214 const __u8 ip_tos2prio[16] = {
215         TC_PRIO_BESTEFFORT,
216         ECN_OR_COST(BESTEFFORT),
217         TC_PRIO_BESTEFFORT,
218         ECN_OR_COST(BESTEFFORT),
219         TC_PRIO_BULK,
220         ECN_OR_COST(BULK),
221         TC_PRIO_BULK,
222         ECN_OR_COST(BULK),
223         TC_PRIO_INTERACTIVE,
224         ECN_OR_COST(INTERACTIVE),
225         TC_PRIO_INTERACTIVE,
226         ECN_OR_COST(INTERACTIVE),
227         TC_PRIO_INTERACTIVE_BULK,
228         ECN_OR_COST(INTERACTIVE_BULK),
229         TC_PRIO_INTERACTIVE_BULK,
230         ECN_OR_COST(INTERACTIVE_BULK)
231 };
232 EXPORT_SYMBOL(ip_tos2prio);
233
234 /*
235  * Route cache.
236  */
237
238 /* The locking scheme is rather straight forward:
239  *
240  * 1) Read-Copy Update protects the buckets of the central route hash.
241  * 2) Only writers remove entries, and they hold the lock
242  *    as they look at rtable reference counts.
243  * 3) Only readers acquire references to rtable entries,
244  *    they do so with atomic increments and with the
245  *    lock held.
246  */
247
248 struct rt_hash_bucket {
249         struct rtable __rcu     *chain;
250 };
251
252 #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK) || \
253         defined(CONFIG_PROVE_LOCKING)
254 /*
255  * Instead of using one spinlock for each rt_hash_bucket, we use a table of spinlocks
256  * The size of this table is a power of two and depends on the number of CPUS.
257  * (on lockdep we have a quite big spinlock_t, so keep the size down there)
258  */
259 #ifdef CONFIG_LOCKDEP
260 # define RT_HASH_LOCK_SZ        256
261 #else
262 # if NR_CPUS >= 32
263 #  define RT_HASH_LOCK_SZ       4096
264 # elif NR_CPUS >= 16
265 #  define RT_HASH_LOCK_SZ       2048
266 # elif NR_CPUS >= 8
267 #  define RT_HASH_LOCK_SZ       1024
268 # elif NR_CPUS >= 4
269 #  define RT_HASH_LOCK_SZ       512
270 # else
271 #  define RT_HASH_LOCK_SZ       256
272 # endif
273 #endif
274
275 static spinlock_t       *rt_hash_locks;
276 # define rt_hash_lock_addr(slot) &rt_hash_locks[(slot) & (RT_HASH_LOCK_SZ - 1)]
277
278 static __init void rt_hash_lock_init(void)
279 {
280         int i;
281
282         rt_hash_locks = kmalloc(sizeof(spinlock_t) * RT_HASH_LOCK_SZ,
283                         GFP_KERNEL);
284         if (!rt_hash_locks)
285                 panic("IP: failed to allocate rt_hash_locks\n");
286
287         for (i = 0; i < RT_HASH_LOCK_SZ; i++)
288                 spin_lock_init(&rt_hash_locks[i]);
289 }
290 #else
291 # define rt_hash_lock_addr(slot) NULL
292
293 static inline void rt_hash_lock_init(void)
294 {
295 }
296 #endif
297
298 static struct rt_hash_bucket    *rt_hash_table __read_mostly;
299 static unsigned int             rt_hash_mask __read_mostly;
300 static unsigned int             rt_hash_log  __read_mostly;
301
302 static DEFINE_PER_CPU(struct rt_cache_stat, rt_cache_stat);
303 #define RT_CACHE_STAT_INC(field) __this_cpu_inc(rt_cache_stat.field)
304
305 static inline unsigned int rt_hash(__be32 daddr, __be32 saddr, int idx,
306                                    int genid)
307 {
308         return jhash_3words((__force u32)daddr, (__force u32)saddr,
309                             idx, genid)
310                 & rt_hash_mask;
311 }
312
313 static inline int rt_genid(struct net *net)
314 {
315         return atomic_read(&net->ipv4.rt_genid);
316 }
317
318 #ifdef CONFIG_PROC_FS
319 struct rt_cache_iter_state {
320         struct seq_net_private p;
321         int bucket;
322         int genid;
323 };
324
325 static struct rtable *rt_cache_get_first(struct seq_file *seq)
326 {
327         struct rt_cache_iter_state *st = seq->private;
328         struct rtable *r = NULL;
329
330         for (st->bucket = rt_hash_mask; st->bucket >= 0; --st->bucket) {
331                 if (!rcu_access_pointer(rt_hash_table[st->bucket].chain))
332                         continue;
333                 rcu_read_lock_bh();
334                 r = rcu_dereference_bh(rt_hash_table[st->bucket].chain);
335                 while (r) {
336                         if (dev_net(r->dst.dev) == seq_file_net(seq) &&
337                             r->rt_genid == st->genid)
338                                 return r;
339                         r = rcu_dereference_bh(r->dst.rt_next);
340                 }
341                 rcu_read_unlock_bh();
342         }
343         return r;
344 }
345
346 static struct rtable *__rt_cache_get_next(struct seq_file *seq,
347                                           struct rtable *r)
348 {
349         struct rt_cache_iter_state *st = seq->private;
350
351         r = rcu_dereference_bh(r->dst.rt_next);
352         while (!r) {
353                 rcu_read_unlock_bh();
354                 do {
355                         if (--st->bucket < 0)
356                                 return NULL;
357                 } while (!rcu_access_pointer(rt_hash_table[st->bucket].chain));
358                 rcu_read_lock_bh();
359                 r = rcu_dereference_bh(rt_hash_table[st->bucket].chain);
360         }
361         return r;
362 }
363
364 static struct rtable *rt_cache_get_next(struct seq_file *seq,
365                                         struct rtable *r)
366 {
367         struct rt_cache_iter_state *st = seq->private;
368         while ((r = __rt_cache_get_next(seq, r)) != NULL) {
369                 if (dev_net(r->dst.dev) != seq_file_net(seq))
370                         continue;
371                 if (r->rt_genid == st->genid)
372                         break;
373         }
374         return r;
375 }
376
377 static struct rtable *rt_cache_get_idx(struct seq_file *seq, loff_t pos)
378 {
379         struct rtable *r = rt_cache_get_first(seq);
380
381         if (r)
382                 while (pos && (r = rt_cache_get_next(seq, r)))
383                         --pos;
384         return pos ? NULL : r;
385 }
386
387 static void *rt_cache_seq_start(struct seq_file *seq, loff_t *pos)
388 {
389         struct rt_cache_iter_state *st = seq->private;
390         if (*pos)
391                 return rt_cache_get_idx(seq, *pos - 1);
392         st->genid = rt_genid(seq_file_net(seq));
393         return SEQ_START_TOKEN;
394 }
395
396 static void *rt_cache_seq_next(struct seq_file *seq, void *v, loff_t *pos)
397 {
398         struct rtable *r;
399
400         if (v == SEQ_START_TOKEN)
401                 r = rt_cache_get_first(seq);
402         else
403                 r = rt_cache_get_next(seq, v);
404         ++*pos;
405         return r;
406 }
407
408 static void rt_cache_seq_stop(struct seq_file *seq, void *v)
409 {
410         if (v && v != SEQ_START_TOKEN)
411                 rcu_read_unlock_bh();
412 }
413
414 static int rt_cache_seq_show(struct seq_file *seq, void *v)
415 {
416         if (v == SEQ_START_TOKEN)
417                 seq_printf(seq, "%-127s\n",
418                            "Iface\tDestination\tGateway \tFlags\t\tRefCnt\tUse\t"
419                            "Metric\tSource\t\tMTU\tWindow\tIRTT\tTOS\tHHRef\t"
420                            "HHUptod\tSpecDst");
421         else {
422                 struct rtable *r = v;
423                 int len;
424
425                 seq_printf(seq, "%s\t%08X\t%08X\t%8X\t%d\t%u\t%d\t"
426                            "%08X\t%d\t%u\t%u\t%02X\t%d\t%1d\t%08X%n",
427                            r->dst.dev ? r->dst.dev->name : "*",
428                            (__force u32)r->rt_dst,
429                            (__force u32)r->rt_gateway,
430                            r->rt_flags, atomic_read(&r->dst.__refcnt),
431                            r->dst.__use, 0, (__force u32)r->rt_src,
432                            dst_metric_advmss(&r->dst) + 40,
433                            dst_metric(&r->dst, RTAX_WINDOW), 0,
434                            r->rt_key_tos,
435                            -1, 0, 0, &len);
436
437                 seq_printf(seq, "%*s\n", 127 - len, "");
438         }
439         return 0;
440 }
441
442 static const struct seq_operations rt_cache_seq_ops = {
443         .start  = rt_cache_seq_start,
444         .next   = rt_cache_seq_next,
445         .stop   = rt_cache_seq_stop,
446         .show   = rt_cache_seq_show,
447 };
448
449 static int rt_cache_seq_open(struct inode *inode, struct file *file)
450 {
451         return seq_open_net(inode, file, &rt_cache_seq_ops,
452                         sizeof(struct rt_cache_iter_state));
453 }
454
455 static const struct file_operations rt_cache_seq_fops = {
456         .owner   = THIS_MODULE,
457         .open    = rt_cache_seq_open,
458         .read    = seq_read,
459         .llseek  = seq_lseek,
460         .release = seq_release_net,
461 };
462
463
464 static void *rt_cpu_seq_start(struct seq_file *seq, loff_t *pos)
465 {
466         int cpu;
467
468         if (*pos == 0)
469                 return SEQ_START_TOKEN;
470
471         for (cpu = *pos-1; cpu < nr_cpu_ids; ++cpu) {
472                 if (!cpu_possible(cpu))
473                         continue;
474                 *pos = cpu+1;
475                 return &per_cpu(rt_cache_stat, cpu);
476         }
477         return NULL;
478 }
479
480 static void *rt_cpu_seq_next(struct seq_file *seq, void *v, loff_t *pos)
481 {
482         int cpu;
483
484         for (cpu = *pos; cpu < nr_cpu_ids; ++cpu) {
485                 if (!cpu_possible(cpu))
486                         continue;
487                 *pos = cpu+1;
488                 return &per_cpu(rt_cache_stat, cpu);
489         }
490         return NULL;
491
492 }
493
494 static void rt_cpu_seq_stop(struct seq_file *seq, void *v)
495 {
496
497 }
498
499 static int rt_cpu_seq_show(struct seq_file *seq, void *v)
500 {
501         struct rt_cache_stat *st = v;
502
503         if (v == SEQ_START_TOKEN) {
504                 seq_printf(seq, "entries  in_hit in_slow_tot in_slow_mc in_no_route in_brd in_martian_dst in_martian_src  out_hit out_slow_tot out_slow_mc  gc_total gc_ignored gc_goal_miss gc_dst_overflow in_hlist_search out_hlist_search\n");
505                 return 0;
506         }
507
508         seq_printf(seq,"%08x  %08x %08x %08x %08x %08x %08x %08x "
509                    " %08x %08x %08x %08x %08x %08x %08x %08x %08x \n",
510                    dst_entries_get_slow(&ipv4_dst_ops),
511                    st->in_hit,
512                    st->in_slow_tot,
513                    st->in_slow_mc,
514                    st->in_no_route,
515                    st->in_brd,
516                    st->in_martian_dst,
517                    st->in_martian_src,
518
519                    st->out_hit,
520                    st->out_slow_tot,
521                    st->out_slow_mc,
522
523                    st->gc_total,
524                    st->gc_ignored,
525                    st->gc_goal_miss,
526                    st->gc_dst_overflow,
527                    st->in_hlist_search,
528                    st->out_hlist_search
529                 );
530         return 0;
531 }
532
533 static const struct seq_operations rt_cpu_seq_ops = {
534         .start  = rt_cpu_seq_start,
535         .next   = rt_cpu_seq_next,
536         .stop   = rt_cpu_seq_stop,
537         .show   = rt_cpu_seq_show,
538 };
539
540
541 static int rt_cpu_seq_open(struct inode *inode, struct file *file)
542 {
543         return seq_open(file, &rt_cpu_seq_ops);
544 }
545
546 static const struct file_operations rt_cpu_seq_fops = {
547         .owner   = THIS_MODULE,
548         .open    = rt_cpu_seq_open,
549         .read    = seq_read,
550         .llseek  = seq_lseek,
551         .release = seq_release,
552 };
553
554 #ifdef CONFIG_IP_ROUTE_CLASSID
555 static int rt_acct_proc_show(struct seq_file *m, void *v)
556 {
557         struct ip_rt_acct *dst, *src;
558         unsigned int i, j;
559
560         dst = kcalloc(256, sizeof(struct ip_rt_acct), GFP_KERNEL);
561         if (!dst)
562                 return -ENOMEM;
563
564         for_each_possible_cpu(i) {
565                 src = (struct ip_rt_acct *)per_cpu_ptr(ip_rt_acct, i);
566                 for (j = 0; j < 256; j++) {
567                         dst[j].o_bytes   += src[j].o_bytes;
568                         dst[j].o_packets += src[j].o_packets;
569                         dst[j].i_bytes   += src[j].i_bytes;
570                         dst[j].i_packets += src[j].i_packets;
571                 }
572         }
573
574         seq_write(m, dst, 256 * sizeof(struct ip_rt_acct));
575         kfree(dst);
576         return 0;
577 }
578
579 static int rt_acct_proc_open(struct inode *inode, struct file *file)
580 {
581         return single_open(file, rt_acct_proc_show, NULL);
582 }
583
584 static const struct file_operations rt_acct_proc_fops = {
585         .owner          = THIS_MODULE,
586         .open           = rt_acct_proc_open,
587         .read           = seq_read,
588         .llseek         = seq_lseek,
589         .release        = single_release,
590 };
591 #endif
592
593 static int __net_init ip_rt_do_proc_init(struct net *net)
594 {
595         struct proc_dir_entry *pde;
596
597         pde = proc_net_fops_create(net, "rt_cache", S_IRUGO,
598                         &rt_cache_seq_fops);
599         if (!pde)
600                 goto err1;
601
602         pde = proc_create("rt_cache", S_IRUGO,
603                           net->proc_net_stat, &rt_cpu_seq_fops);
604         if (!pde)
605                 goto err2;
606
607 #ifdef CONFIG_IP_ROUTE_CLASSID
608         pde = proc_create("rt_acct", 0, net->proc_net, &rt_acct_proc_fops);
609         if (!pde)
610                 goto err3;
611 #endif
612         return 0;
613
614 #ifdef CONFIG_IP_ROUTE_CLASSID
615 err3:
616         remove_proc_entry("rt_cache", net->proc_net_stat);
617 #endif
618 err2:
619         remove_proc_entry("rt_cache", net->proc_net);
620 err1:
621         return -ENOMEM;
622 }
623
624 static void __net_exit ip_rt_do_proc_exit(struct net *net)
625 {
626         remove_proc_entry("rt_cache", net->proc_net_stat);
627         remove_proc_entry("rt_cache", net->proc_net);
628 #ifdef CONFIG_IP_ROUTE_CLASSID
629         remove_proc_entry("rt_acct", net->proc_net);
630 #endif
631 }
632
633 static struct pernet_operations ip_rt_proc_ops __net_initdata =  {
634         .init = ip_rt_do_proc_init,
635         .exit = ip_rt_do_proc_exit,
636 };
637
638 static int __init ip_rt_proc_init(void)
639 {
640         return register_pernet_subsys(&ip_rt_proc_ops);
641 }
642
643 #else
644 static inline int ip_rt_proc_init(void)
645 {
646         return 0;
647 }
648 #endif /* CONFIG_PROC_FS */
649
650 static inline void rt_free(struct rtable *rt)
651 {
652         call_rcu_bh(&rt->dst.rcu_head, dst_rcu_free);
653 }
654
655 static inline void rt_drop(struct rtable *rt)
656 {
657         ip_rt_put(rt);
658         call_rcu_bh(&rt->dst.rcu_head, dst_rcu_free);
659 }
660
661 static inline int rt_fast_clean(struct rtable *rth)
662 {
663         /* Kill broadcast/multicast entries very aggresively, if they
664            collide in hash table with more useful entries */
665         return (rth->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST)) &&
666                 rt_is_input_route(rth) && rth->dst.rt_next;
667 }
668
669 static inline int rt_valuable(struct rtable *rth)
670 {
671         return (rth->rt_flags & (RTCF_REDIRECTED | RTCF_NOTIFY)) ||
672                 (rt_has_peer(rth) && rt_peer_ptr(rth)->pmtu_expires);
673 }
674
675 static int rt_may_expire(struct rtable *rth, unsigned long tmo1, unsigned long tmo2)
676 {
677         unsigned long age;
678         int ret = 0;
679
680         if (atomic_read(&rth->dst.__refcnt))
681                 goto out;
682
683         age = jiffies - rth->dst.lastuse;
684         if ((age <= tmo1 && !rt_fast_clean(rth)) ||
685             (age <= tmo2 && rt_valuable(rth)))
686                 goto out;
687         ret = 1;
688 out:    return ret;
689 }
690
691 /* Bits of score are:
692  * 31: very valuable
693  * 30: not quite useless
694  * 29..0: usage counter
695  */
696 static inline u32 rt_score(struct rtable *rt)
697 {
698         u32 score = jiffies - rt->dst.lastuse;
699
700         score = ~score & ~(3<<30);
701
702         if (rt_valuable(rt))
703                 score |= (1<<31);
704
705         if (rt_is_output_route(rt) ||
706             !(rt->rt_flags & (RTCF_BROADCAST|RTCF_MULTICAST|RTCF_LOCAL)))
707                 score |= (1<<30);
708
709         return score;
710 }
711
712 static inline bool rt_caching(const struct net *net)
713 {
714         return net->ipv4.current_rt_cache_rebuild_count <=
715                 net->ipv4.sysctl_rt_cache_rebuild_count;
716 }
717
718 static inline bool compare_hash_inputs(const struct rtable *rt1,
719                                        const struct rtable *rt2)
720 {
721         return ((((__force u32)rt1->rt_key_dst ^ (__force u32)rt2->rt_key_dst) |
722                 ((__force u32)rt1->rt_key_src ^ (__force u32)rt2->rt_key_src) |
723                 (rt1->rt_route_iif ^ rt2->rt_route_iif)) == 0);
724 }
725
726 static inline int compare_keys(struct rtable *rt1, struct rtable *rt2)
727 {
728         return (((__force u32)rt1->rt_key_dst ^ (__force u32)rt2->rt_key_dst) |
729                 ((__force u32)rt1->rt_key_src ^ (__force u32)rt2->rt_key_src) |
730                 (rt1->rt_mark ^ rt2->rt_mark) |
731                 (rt1->rt_key_tos ^ rt2->rt_key_tos) |
732                 (rt1->rt_route_iif ^ rt2->rt_route_iif) |
733                 (rt1->rt_oif ^ rt2->rt_oif)) == 0;
734 }
735
736 static inline int compare_netns(struct rtable *rt1, struct rtable *rt2)
737 {
738         return net_eq(dev_net(rt1->dst.dev), dev_net(rt2->dst.dev));
739 }
740
741 static inline int rt_is_expired(struct rtable *rth)
742 {
743         return rth->rt_genid != rt_genid(dev_net(rth->dst.dev));
744 }
745
746 /*
747  * Perform a full scan of hash table and free all entries.
748  * Can be called by a softirq or a process.
749  * In the later case, we want to be reschedule if necessary
750  */
751 static void rt_do_flush(struct net *net, int process_context)
752 {
753         unsigned int i;
754         struct rtable *rth, *next;
755
756         for (i = 0; i <= rt_hash_mask; i++) {
757                 struct rtable __rcu **pprev;
758                 struct rtable *list;
759
760                 if (process_context && need_resched())
761                         cond_resched();
762                 rth = rcu_access_pointer(rt_hash_table[i].chain);
763                 if (!rth)
764                         continue;
765
766                 spin_lock_bh(rt_hash_lock_addr(i));
767
768                 list = NULL;
769                 pprev = &rt_hash_table[i].chain;
770                 rth = rcu_dereference_protected(*pprev,
771                         lockdep_is_held(rt_hash_lock_addr(i)));
772
773                 while (rth) {
774                         next = rcu_dereference_protected(rth->dst.rt_next,
775                                 lockdep_is_held(rt_hash_lock_addr(i)));
776
777                         if (!net ||
778                             net_eq(dev_net(rth->dst.dev), net)) {
779                                 rcu_assign_pointer(*pprev, next);
780                                 rcu_assign_pointer(rth->dst.rt_next, list);
781                                 list = rth;
782                         } else {
783                                 pprev = &rth->dst.rt_next;
784                         }
785                         rth = next;
786                 }
787
788                 spin_unlock_bh(rt_hash_lock_addr(i));
789
790                 for (; list; list = next) {
791                         next = rcu_dereference_protected(list->dst.rt_next, 1);
792                         rt_free(list);
793                 }
794         }
795 }
796
797 /*
798  * While freeing expired entries, we compute average chain length
799  * and standard deviation, using fixed-point arithmetic.
800  * This to have an estimation of rt_chain_length_max
801  *  rt_chain_length_max = max(elasticity, AVG + 4*SD)
802  * We use 3 bits for frational part, and 29 (or 61) for magnitude.
803  */
804
805 #define FRACT_BITS 3
806 #define ONE (1UL << FRACT_BITS)
807
808 /*
809  * Given a hash chain and an item in this hash chain,
810  * find if a previous entry has the same hash_inputs
811  * (but differs on tos, mark or oif)
812  * Returns 0 if an alias is found.
813  * Returns ONE if rth has no alias before itself.
814  */
815 static int has_noalias(const struct rtable *head, const struct rtable *rth)
816 {
817         const struct rtable *aux = head;
818
819         while (aux != rth) {
820                 if (compare_hash_inputs(aux, rth))
821                         return 0;
822                 aux = rcu_dereference_protected(aux->dst.rt_next, 1);
823         }
824         return ONE;
825 }
826
827 static void rt_check_expire(void)
828 {
829         static unsigned int rover;
830         unsigned int i = rover, goal;
831         struct rtable *rth;
832         struct rtable __rcu **rthp;
833         unsigned long samples = 0;
834         unsigned long sum = 0, sum2 = 0;
835         unsigned long delta;
836         u64 mult;
837
838         delta = jiffies - expires_ljiffies;
839         expires_ljiffies = jiffies;
840         mult = ((u64)delta) << rt_hash_log;
841         if (ip_rt_gc_timeout > 1)
842                 do_div(mult, ip_rt_gc_timeout);
843         goal = (unsigned int)mult;
844         if (goal > rt_hash_mask)
845                 goal = rt_hash_mask + 1;
846         for (; goal > 0; goal--) {
847                 unsigned long tmo = ip_rt_gc_timeout;
848                 unsigned long length;
849
850                 i = (i + 1) & rt_hash_mask;
851                 rthp = &rt_hash_table[i].chain;
852
853                 if (need_resched())
854                         cond_resched();
855
856                 samples++;
857
858                 if (rcu_dereference_raw(*rthp) == NULL)
859                         continue;
860                 length = 0;
861                 spin_lock_bh(rt_hash_lock_addr(i));
862                 while ((rth = rcu_dereference_protected(*rthp,
863                                         lockdep_is_held(rt_hash_lock_addr(i)))) != NULL) {
864                         prefetch(rth->dst.rt_next);
865                         if (rt_is_expired(rth) ||
866                             rt_may_expire(rth, tmo, ip_rt_gc_timeout)) {
867                                 *rthp = rth->dst.rt_next;
868                                 rt_free(rth);
869                                 continue;
870                         }
871
872                         /* We only count entries on a chain with equal
873                          * hash inputs once so that entries for
874                          * different QOS levels, and other non-hash
875                          * input attributes don't unfairly skew the
876                          * length computation
877                          */
878                         tmo >>= 1;
879                         rthp = &rth->dst.rt_next;
880                         length += has_noalias(rt_hash_table[i].chain, rth);
881                 }
882                 spin_unlock_bh(rt_hash_lock_addr(i));
883                 sum += length;
884                 sum2 += length*length;
885         }
886         if (samples) {
887                 unsigned long avg = sum / samples;
888                 unsigned long sd = int_sqrt(sum2 / samples - avg*avg);
889                 rt_chain_length_max = max_t(unsigned long,
890                                         ip_rt_gc_elasticity,
891                                         (avg + 4*sd) >> FRACT_BITS);
892         }
893         rover = i;
894 }
895
896 /*
897  * rt_worker_func() is run in process context.
898  * we call rt_check_expire() to scan part of the hash table
899  */
900 static void rt_worker_func(struct work_struct *work)
901 {
902         rt_check_expire();
903         schedule_delayed_work(&expires_work, ip_rt_gc_interval);
904 }
905
906 /*
907  * Perturbation of rt_genid by a small quantity [1..256]
908  * Using 8 bits of shuffling ensure we can call rt_cache_invalidate()
909  * many times (2^24) without giving recent rt_genid.
910  * Jenkins hash is strong enough that litle changes of rt_genid are OK.
911  */
912 static void rt_cache_invalidate(struct net *net)
913 {
914         unsigned char shuffle;
915
916         get_random_bytes(&shuffle, sizeof(shuffle));
917         atomic_add(shuffle + 1U, &net->ipv4.rt_genid);
918         inetpeer_invalidate_family(AF_INET);
919 }
920
921 /*
922  * delay < 0  : invalidate cache (fast : entries will be deleted later)
923  * delay >= 0 : invalidate & flush cache (can be long)
924  */
925 void rt_cache_flush(struct net *net, int delay)
926 {
927         rt_cache_invalidate(net);
928         if (delay >= 0)
929                 rt_do_flush(net, !in_softirq());
930 }
931
932 /* Flush previous cache invalidated entries from the cache */
933 void rt_cache_flush_batch(struct net *net)
934 {
935         rt_do_flush(net, !in_softirq());
936 }
937
938 static void rt_emergency_hash_rebuild(struct net *net)
939 {
940         net_warn_ratelimited("Route hash chain too long!\n");
941         rt_cache_invalidate(net);
942 }
943
944 /*
945    Short description of GC goals.
946
947    We want to build algorithm, which will keep routing cache
948    at some equilibrium point, when number of aged off entries
949    is kept approximately equal to newly generated ones.
950
951    Current expiration strength is variable "expire".
952    We try to adjust it dynamically, so that if networking
953    is idle expires is large enough to keep enough of warm entries,
954    and when load increases it reduces to limit cache size.
955  */
956
957 static int rt_garbage_collect(struct dst_ops *ops)
958 {
959         static unsigned long expire = RT_GC_TIMEOUT;
960         static unsigned long last_gc;
961         static int rover;
962         static int equilibrium;
963         struct rtable *rth;
964         struct rtable __rcu **rthp;
965         unsigned long now = jiffies;
966         int goal;
967         int entries = dst_entries_get_fast(&ipv4_dst_ops);
968
969         /*
970          * Garbage collection is pretty expensive,
971          * do not make it too frequently.
972          */
973
974         RT_CACHE_STAT_INC(gc_total);
975
976         if (now - last_gc < ip_rt_gc_min_interval &&
977             entries < ip_rt_max_size) {
978                 RT_CACHE_STAT_INC(gc_ignored);
979                 goto out;
980         }
981
982         entries = dst_entries_get_slow(&ipv4_dst_ops);
983         /* Calculate number of entries, which we want to expire now. */
984         goal = entries - (ip_rt_gc_elasticity << rt_hash_log);
985         if (goal <= 0) {
986                 if (equilibrium < ipv4_dst_ops.gc_thresh)
987                         equilibrium = ipv4_dst_ops.gc_thresh;
988                 goal = entries - equilibrium;
989                 if (goal > 0) {
990                         equilibrium += min_t(unsigned int, goal >> 1, rt_hash_mask + 1);
991                         goal = entries - equilibrium;
992                 }
993         } else {
994                 /* We are in dangerous area. Try to reduce cache really
995                  * aggressively.
996                  */
997                 goal = max_t(unsigned int, goal >> 1, rt_hash_mask + 1);
998                 equilibrium = entries - goal;
999         }
1000
1001         if (now - last_gc >= ip_rt_gc_min_interval)
1002                 last_gc = now;
1003
1004         if (goal <= 0) {
1005                 equilibrium += goal;
1006                 goto work_done;
1007         }
1008
1009         do {
1010                 int i, k;
1011
1012                 for (i = rt_hash_mask, k = rover; i >= 0; i--) {
1013                         unsigned long tmo = expire;
1014
1015                         k = (k + 1) & rt_hash_mask;
1016                         rthp = &rt_hash_table[k].chain;
1017                         spin_lock_bh(rt_hash_lock_addr(k));
1018                         while ((rth = rcu_dereference_protected(*rthp,
1019                                         lockdep_is_held(rt_hash_lock_addr(k)))) != NULL) {
1020                                 if (!rt_is_expired(rth) &&
1021                                         !rt_may_expire(rth, tmo, expire)) {
1022                                         tmo >>= 1;
1023                                         rthp = &rth->dst.rt_next;
1024                                         continue;
1025                                 }
1026                                 *rthp = rth->dst.rt_next;
1027                                 rt_free(rth);
1028                                 goal--;
1029                         }
1030                         spin_unlock_bh(rt_hash_lock_addr(k));
1031                         if (goal <= 0)
1032                                 break;
1033                 }
1034                 rover = k;
1035
1036                 if (goal <= 0)
1037                         goto work_done;
1038
1039                 /* Goal is not achieved. We stop process if:
1040
1041                    - if expire reduced to zero. Otherwise, expire is halfed.
1042                    - if table is not full.
1043                    - if we are called from interrupt.
1044                    - jiffies check is just fallback/debug loop breaker.
1045                      We will not spin here for long time in any case.
1046                  */
1047
1048                 RT_CACHE_STAT_INC(gc_goal_miss);
1049
1050                 if (expire == 0)
1051                         break;
1052
1053                 expire >>= 1;
1054
1055                 if (dst_entries_get_fast(&ipv4_dst_ops) < ip_rt_max_size)
1056                         goto out;
1057         } while (!in_softirq() && time_before_eq(jiffies, now));
1058
1059         if (dst_entries_get_fast(&ipv4_dst_ops) < ip_rt_max_size)
1060                 goto out;
1061         if (dst_entries_get_slow(&ipv4_dst_ops) < ip_rt_max_size)
1062                 goto out;
1063         net_warn_ratelimited("dst cache overflow\n");
1064         RT_CACHE_STAT_INC(gc_dst_overflow);
1065         return 1;
1066
1067 work_done:
1068         expire += ip_rt_gc_min_interval;
1069         if (expire > ip_rt_gc_timeout ||
1070             dst_entries_get_fast(&ipv4_dst_ops) < ipv4_dst_ops.gc_thresh ||
1071             dst_entries_get_slow(&ipv4_dst_ops) < ipv4_dst_ops.gc_thresh)
1072                 expire = ip_rt_gc_timeout;
1073 out:    return 0;
1074 }
1075
1076 /*
1077  * Returns number of entries in a hash chain that have different hash_inputs
1078  */
1079 static int slow_chain_length(const struct rtable *head)
1080 {
1081         int length = 0;
1082         const struct rtable *rth = head;
1083
1084         while (rth) {
1085                 length += has_noalias(head, rth);
1086                 rth = rcu_dereference_protected(rth->dst.rt_next, 1);
1087         }
1088         return length >> FRACT_BITS;
1089 }
1090
1091 static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst,
1092                                            struct sk_buff *skb,
1093                                            const void *daddr)
1094 {
1095         struct net_device *dev = dst->dev;
1096         const __be32 *pkey = daddr;
1097         const struct rtable *rt;
1098         struct neighbour *n;
1099
1100         rt = (const struct rtable *) dst;
1101         if (rt->rt_gateway)
1102                 pkey = (const __be32 *) &rt->rt_gateway;
1103         else if (skb)
1104                 pkey = &ip_hdr(skb)->daddr;
1105
1106         n = __ipv4_neigh_lookup(dev, *(__force u32 *)pkey);
1107         if (n)
1108                 return n;
1109         return neigh_create(&arp_tbl, pkey, dev);
1110 }
1111
1112 static struct rtable *rt_intern_hash(unsigned int hash, struct rtable *rt,
1113                                      struct sk_buff *skb, int ifindex)
1114 {
1115         struct rtable   *rth, *cand;
1116         struct rtable __rcu **rthp, **candp;
1117         unsigned long   now;
1118         u32             min_score;
1119         int             chain_length;
1120
1121 restart:
1122         chain_length = 0;
1123         min_score = ~(u32)0;
1124         cand = NULL;
1125         candp = NULL;
1126         now = jiffies;
1127
1128         if (!rt_caching(dev_net(rt->dst.dev)) || (rt->dst.flags & DST_NOCACHE)) {
1129                 /*
1130                  * If we're not caching, just tell the caller we
1131                  * were successful and don't touch the route.  The
1132                  * caller hold the sole reference to the cache entry, and
1133                  * it will be released when the caller is done with it.
1134                  * If we drop it here, the callers have no way to resolve routes
1135                  * when we're not caching.  Instead, just point *rp at rt, so
1136                  * the caller gets a single use out of the route
1137                  * Note that we do rt_free on this new route entry, so that
1138                  * once its refcount hits zero, we are still able to reap it
1139                  * (Thanks Alexey)
1140                  * Note: To avoid expensive rcu stuff for this uncached dst,
1141                  * we set DST_NOCACHE so that dst_release() can free dst without
1142                  * waiting a grace period.
1143                  */
1144
1145                 rt->dst.flags |= DST_NOCACHE;
1146                 goto skip_hashing;
1147         }
1148
1149         rthp = &rt_hash_table[hash].chain;
1150
1151         spin_lock_bh(rt_hash_lock_addr(hash));
1152         while ((rth = rcu_dereference_protected(*rthp,
1153                         lockdep_is_held(rt_hash_lock_addr(hash)))) != NULL) {
1154                 if (rt_is_expired(rth)) {
1155                         *rthp = rth->dst.rt_next;
1156                         rt_free(rth);
1157                         continue;
1158                 }
1159                 if (compare_keys(rth, rt) && compare_netns(rth, rt)) {
1160                         /* Put it first */
1161                         *rthp = rth->dst.rt_next;
1162                         /*
1163                          * Since lookup is lockfree, the deletion
1164                          * must be visible to another weakly ordered CPU before
1165                          * the insertion at the start of the hash chain.
1166                          */
1167                         rcu_assign_pointer(rth->dst.rt_next,
1168                                            rt_hash_table[hash].chain);
1169                         /*
1170                          * Since lookup is lockfree, the update writes
1171                          * must be ordered for consistency on SMP.
1172                          */
1173                         rcu_assign_pointer(rt_hash_table[hash].chain, rth);
1174
1175                         dst_use(&rth->dst, now);
1176                         spin_unlock_bh(rt_hash_lock_addr(hash));
1177
1178                         rt_drop(rt);
1179                         if (skb)
1180                                 skb_dst_set(skb, &rth->dst);
1181                         return rth;
1182                 }
1183
1184                 if (!atomic_read(&rth->dst.__refcnt)) {
1185                         u32 score = rt_score(rth);
1186
1187                         if (score <= min_score) {
1188                                 cand = rth;
1189                                 candp = rthp;
1190                                 min_score = score;
1191                         }
1192                 }
1193
1194                 chain_length++;
1195
1196                 rthp = &rth->dst.rt_next;
1197         }
1198
1199         if (cand) {
1200                 /* ip_rt_gc_elasticity used to be average length of chain
1201                  * length, when exceeded gc becomes really aggressive.
1202                  *
1203                  * The second limit is less certain. At the moment it allows
1204                  * only 2 entries per bucket. We will see.
1205                  */
1206                 if (chain_length > ip_rt_gc_elasticity) {
1207                         *candp = cand->dst.rt_next;
1208                         rt_free(cand);
1209                 }
1210         } else {
1211                 if (chain_length > rt_chain_length_max &&
1212                     slow_chain_length(rt_hash_table[hash].chain) > rt_chain_length_max) {
1213                         struct net *net = dev_net(rt->dst.dev);
1214                         int num = ++net->ipv4.current_rt_cache_rebuild_count;
1215                         if (!rt_caching(net)) {
1216                                 pr_warn("%s: %d rebuilds is over limit, route caching disabled\n",
1217                                         rt->dst.dev->name, num);
1218                         }
1219                         rt_emergency_hash_rebuild(net);
1220                         spin_unlock_bh(rt_hash_lock_addr(hash));
1221
1222                         hash = rt_hash(rt->rt_key_dst, rt->rt_key_src,
1223                                         ifindex, rt_genid(net));
1224                         goto restart;
1225                 }
1226         }
1227
1228         rt->dst.rt_next = rt_hash_table[hash].chain;
1229
1230         /*
1231          * Since lookup is lockfree, we must make sure
1232          * previous writes to rt are committed to memory
1233          * before making rt visible to other CPUS.
1234          */
1235         rcu_assign_pointer(rt_hash_table[hash].chain, rt);
1236
1237         spin_unlock_bh(rt_hash_lock_addr(hash));
1238
1239 skip_hashing:
1240         if (skb)
1241                 skb_dst_set(skb, &rt->dst);
1242         return rt;
1243 }
1244
1245 static atomic_t __rt_peer_genid = ATOMIC_INIT(0);
1246
1247 static u32 rt_peer_genid(void)
1248 {
1249         return atomic_read(&__rt_peer_genid);
1250 }
1251
1252 void rt_bind_peer(struct rtable *rt, __be32 daddr, int create)
1253 {
1254         struct inet_peer_base *base;
1255         struct inet_peer *peer;
1256
1257         base = inetpeer_base_ptr(rt->_peer);
1258         if (!base)
1259                 return;
1260
1261         peer = inet_getpeer_v4(base, daddr, create);
1262         if (peer) {
1263                 if (!rt_set_peer(rt, peer))
1264                         inet_putpeer(peer);
1265                 else
1266                         rt->rt_peer_genid = rt_peer_genid();
1267         }
1268 }
1269
1270 /*
1271  * Peer allocation may fail only in serious out-of-memory conditions.  However
1272  * we still can generate some output.
1273  * Random ID selection looks a bit dangerous because we have no chances to
1274  * select ID being unique in a reasonable period of time.
1275  * But broken packet identifier may be better than no packet at all.
1276  */
1277 static void ip_select_fb_ident(struct iphdr *iph)
1278 {
1279         static DEFINE_SPINLOCK(ip_fb_id_lock);
1280         static u32 ip_fallback_id;
1281         u32 salt;
1282
1283         spin_lock_bh(&ip_fb_id_lock);
1284         salt = secure_ip_id((__force __be32)ip_fallback_id ^ iph->daddr);
1285         iph->id = htons(salt & 0xFFFF);
1286         ip_fallback_id = salt;
1287         spin_unlock_bh(&ip_fb_id_lock);
1288 }
1289
1290 void __ip_select_ident(struct iphdr *iph, struct dst_entry *dst, int more)
1291 {
1292         struct rtable *rt = (struct rtable *) dst;
1293
1294         if (rt && !(rt->dst.flags & DST_NOPEER)) {
1295                 struct inet_peer *peer = rt_get_peer_create(rt, rt->rt_dst);
1296
1297                 /* If peer is attached to destination, it is never detached,
1298                    so that we need not to grab a lock to dereference it.
1299                  */
1300                 if (peer) {
1301                         iph->id = htons(inet_getid(peer, more));
1302                         return;
1303                 }
1304         } else if (!rt)
1305                 pr_debug("rt_bind_peer(0) @%p\n", __builtin_return_address(0));
1306
1307         ip_select_fb_ident(iph);
1308 }
1309 EXPORT_SYMBOL(__ip_select_ident);
1310
1311 static void rt_del(unsigned int hash, struct rtable *rt)
1312 {
1313         struct rtable __rcu **rthp;
1314         struct rtable *aux;
1315
1316         rthp = &rt_hash_table[hash].chain;
1317         spin_lock_bh(rt_hash_lock_addr(hash));
1318         ip_rt_put(rt);
1319         while ((aux = rcu_dereference_protected(*rthp,
1320                         lockdep_is_held(rt_hash_lock_addr(hash)))) != NULL) {
1321                 if (aux == rt || rt_is_expired(aux)) {
1322                         *rthp = aux->dst.rt_next;
1323                         rt_free(aux);
1324                         continue;
1325                 }
1326                 rthp = &aux->dst.rt_next;
1327         }
1328         spin_unlock_bh(rt_hash_lock_addr(hash));
1329 }
1330
1331 static void check_peer_redir(struct dst_entry *dst, struct inet_peer *peer)
1332 {
1333         struct rtable *rt = (struct rtable *) dst;
1334         __be32 orig_gw = rt->rt_gateway;
1335         struct neighbour *n;
1336
1337         dst_confirm(&rt->dst);
1338
1339         rt->rt_gateway = peer->redirect_learned.a4;
1340
1341         n = ipv4_neigh_lookup(&rt->dst, NULL, &rt->rt_gateway);
1342         if (!n) {
1343                 rt->rt_gateway = orig_gw;
1344                 return;
1345         }
1346         if (!(n->nud_state & NUD_VALID)) {
1347                 neigh_event_send(n, NULL);
1348         } else {
1349                 rt->rt_flags |= RTCF_REDIRECTED;
1350                 call_netevent_notifiers(NETEVENT_NEIGH_UPDATE, n);
1351         }
1352         neigh_release(n);
1353 }
1354
1355 /* called in rcu_read_lock() section */
1356 void ip_rt_redirect(__be32 old_gw, __be32 daddr, __be32 new_gw,
1357                     __be32 saddr, struct net_device *dev)
1358 {
1359         int s, i;
1360         struct in_device *in_dev = __in_dev_get_rcu(dev);
1361         __be32 skeys[2] = { saddr, 0 };
1362         int    ikeys[2] = { dev->ifindex, 0 };
1363         struct inet_peer *peer;
1364         struct net *net;
1365
1366         if (!in_dev)
1367                 return;
1368
1369         net = dev_net(dev);
1370         if (new_gw == old_gw || !IN_DEV_RX_REDIRECTS(in_dev) ||
1371             ipv4_is_multicast(new_gw) || ipv4_is_lbcast(new_gw) ||
1372             ipv4_is_zeronet(new_gw))
1373                 goto reject_redirect;
1374
1375         if (!IN_DEV_SHARED_MEDIA(in_dev)) {
1376                 if (!inet_addr_onlink(in_dev, new_gw, old_gw))
1377                         goto reject_redirect;
1378                 if (IN_DEV_SEC_REDIRECTS(in_dev) && ip_fib_check_default(new_gw, dev))
1379                         goto reject_redirect;
1380         } else {
1381                 if (inet_addr_type(net, new_gw) != RTN_UNICAST)
1382                         goto reject_redirect;
1383         }
1384
1385         for (s = 0; s < 2; s++) {
1386                 for (i = 0; i < 2; i++) {
1387                         unsigned int hash;
1388                         struct rtable __rcu **rthp;
1389                         struct rtable *rt;
1390
1391                         hash = rt_hash(daddr, skeys[s], ikeys[i], rt_genid(net));
1392
1393                         rthp = &rt_hash_table[hash].chain;
1394
1395                         while ((rt = rcu_dereference(*rthp)) != NULL) {
1396                                 rthp = &rt->dst.rt_next;
1397
1398                                 if (rt->rt_key_dst != daddr ||
1399                                     rt->rt_key_src != skeys[s] ||
1400                                     rt->rt_oif != ikeys[i] ||
1401                                     rt_is_input_route(rt) ||
1402                                     rt_is_expired(rt) ||
1403                                     !net_eq(dev_net(rt->dst.dev), net) ||
1404                                     rt->dst.error ||
1405                                     rt->dst.dev != dev ||
1406                                     rt->rt_gateway != old_gw)
1407                                         continue;
1408
1409                                 peer = rt_get_peer_create(rt, rt->rt_dst);
1410                                 if (peer) {
1411                                         if (peer->redirect_learned.a4 != new_gw) {
1412                                                 peer->redirect_learned.a4 = new_gw;
1413                                                 atomic_inc(&__rt_peer_genid);
1414                                         }
1415                                         check_peer_redir(&rt->dst, peer);
1416                                 }
1417                         }
1418                 }
1419         }
1420         return;
1421
1422 reject_redirect:
1423 #ifdef CONFIG_IP_ROUTE_VERBOSE
1424         if (IN_DEV_LOG_MARTIANS(in_dev))
1425                 net_info_ratelimited("Redirect from %pI4 on %s about %pI4 ignored\n"
1426                                      "  Advised path = %pI4 -> %pI4\n",
1427                                      &old_gw, dev->name, &new_gw,
1428                                      &saddr, &daddr);
1429 #endif
1430         ;
1431 }
1432
1433 static bool peer_pmtu_expired(struct inet_peer *peer)
1434 {
1435         unsigned long orig = ACCESS_ONCE(peer->pmtu_expires);
1436
1437         return orig &&
1438                time_after_eq(jiffies, orig) &&
1439                cmpxchg(&peer->pmtu_expires, orig, 0) == orig;
1440 }
1441
1442 static bool peer_pmtu_cleaned(struct inet_peer *peer)
1443 {
1444         unsigned long orig = ACCESS_ONCE(peer->pmtu_expires);
1445
1446         return orig &&
1447                cmpxchg(&peer->pmtu_expires, orig, 0) == orig;
1448 }
1449
1450 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst)
1451 {
1452         struct rtable *rt = (struct rtable *)dst;
1453         struct dst_entry *ret = dst;
1454
1455         if (rt) {
1456                 if (dst->obsolete > 0) {
1457                         ip_rt_put(rt);
1458                         ret = NULL;
1459                 } else if (rt->rt_flags & RTCF_REDIRECTED) {
1460                         unsigned int hash = rt_hash(rt->rt_key_dst, rt->rt_key_src,
1461                                                 rt->rt_oif,
1462                                                 rt_genid(dev_net(dst->dev)));
1463                         rt_del(hash, rt);
1464                         ret = NULL;
1465                 } else if (rt_has_peer(rt)) {
1466                         struct inet_peer *peer = rt_peer_ptr(rt);
1467                         if (peer_pmtu_expired(peer))
1468                                 dst_metric_set(dst, RTAX_MTU, peer->pmtu_orig);
1469                 }
1470         }
1471         return ret;
1472 }
1473
1474 /*
1475  * Algorithm:
1476  *      1. The first ip_rt_redirect_number redirects are sent
1477  *         with exponential backoff, then we stop sending them at all,
1478  *         assuming that the host ignores our redirects.
1479  *      2. If we did not see packets requiring redirects
1480  *         during ip_rt_redirect_silence, we assume that the host
1481  *         forgot redirected route and start to send redirects again.
1482  *
1483  * This algorithm is much cheaper and more intelligent than dumb load limiting
1484  * in icmp.c.
1485  *
1486  * NOTE. Do not forget to inhibit load limiting for redirects (redundant)
1487  * and "frag. need" (breaks PMTU discovery) in icmp.c.
1488  */
1489
1490 void ip_rt_send_redirect(struct sk_buff *skb)
1491 {
1492         struct rtable *rt = skb_rtable(skb);
1493         struct in_device *in_dev;
1494         struct inet_peer *peer;
1495         int log_martians;
1496
1497         rcu_read_lock();
1498         in_dev = __in_dev_get_rcu(rt->dst.dev);
1499         if (!in_dev || !IN_DEV_TX_REDIRECTS(in_dev)) {
1500                 rcu_read_unlock();
1501                 return;
1502         }
1503         log_martians = IN_DEV_LOG_MARTIANS(in_dev);
1504         rcu_read_unlock();
1505
1506         peer = rt_get_peer_create(rt, rt->rt_dst);
1507         if (!peer) {
1508                 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST, rt->rt_gateway);
1509                 return;
1510         }
1511
1512         /* No redirected packets during ip_rt_redirect_silence;
1513          * reset the algorithm.
1514          */
1515         if (time_after(jiffies, peer->rate_last + ip_rt_redirect_silence))
1516                 peer->rate_tokens = 0;
1517
1518         /* Too many ignored redirects; do not send anything
1519          * set dst.rate_last to the last seen redirected packet.
1520          */
1521         if (peer->rate_tokens >= ip_rt_redirect_number) {
1522                 peer->rate_last = jiffies;
1523                 return;
1524         }
1525
1526         /* Check for load limit; set rate_last to the latest sent
1527          * redirect.
1528          */
1529         if (peer->rate_tokens == 0 ||
1530             time_after(jiffies,
1531                        (peer->rate_last +
1532                         (ip_rt_redirect_load << peer->rate_tokens)))) {
1533                 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST, rt->rt_gateway);
1534                 peer->rate_last = jiffies;
1535                 ++peer->rate_tokens;
1536 #ifdef CONFIG_IP_ROUTE_VERBOSE
1537                 if (log_martians &&
1538                     peer->rate_tokens == ip_rt_redirect_number)
1539                         net_warn_ratelimited("host %pI4/if%d ignores redirects for %pI4 to %pI4\n",
1540                                              &ip_hdr(skb)->saddr, rt->rt_iif,
1541                                              &rt->rt_dst, &rt->rt_gateway);
1542 #endif
1543         }
1544 }
1545
1546 static int ip_error(struct sk_buff *skb)
1547 {
1548         struct in_device *in_dev = __in_dev_get_rcu(skb->dev);
1549         struct rtable *rt = skb_rtable(skb);
1550         struct inet_peer *peer;
1551         unsigned long now;
1552         struct net *net;
1553         bool send;
1554         int code;
1555
1556         net = dev_net(rt->dst.dev);
1557         if (!IN_DEV_FORWARD(in_dev)) {
1558                 switch (rt->dst.error) {
1559                 case EHOSTUNREACH:
1560                         IP_INC_STATS_BH(net, IPSTATS_MIB_INADDRERRORS);
1561                         break;
1562
1563                 case ENETUNREACH:
1564                         IP_INC_STATS_BH(net, IPSTATS_MIB_INNOROUTES);
1565                         break;
1566                 }
1567                 goto out;
1568         }
1569
1570         switch (rt->dst.error) {
1571         case EINVAL:
1572         default:
1573                 goto out;
1574         case EHOSTUNREACH:
1575                 code = ICMP_HOST_UNREACH;
1576                 break;
1577         case ENETUNREACH:
1578                 code = ICMP_NET_UNREACH;
1579                 IP_INC_STATS_BH(net, IPSTATS_MIB_INNOROUTES);
1580                 break;
1581         case EACCES:
1582                 code = ICMP_PKT_FILTERED;
1583                 break;
1584         }
1585
1586         peer = rt_get_peer_create(rt, rt->rt_dst);
1587
1588         send = true;
1589         if (peer) {
1590                 now = jiffies;
1591                 peer->rate_tokens += now - peer->rate_last;
1592                 if (peer->rate_tokens > ip_rt_error_burst)
1593                         peer->rate_tokens = ip_rt_error_burst;
1594                 peer->rate_last = now;
1595                 if (peer->rate_tokens >= ip_rt_error_cost)
1596                         peer->rate_tokens -= ip_rt_error_cost;
1597                 else
1598                         send = false;
1599         }
1600         if (send)
1601                 icmp_send(skb, ICMP_DEST_UNREACH, code, 0);
1602
1603 out:    kfree_skb(skb);
1604         return 0;
1605 }
1606
1607 static void check_peer_pmtu(struct dst_entry *dst, struct inet_peer *peer)
1608 {
1609         unsigned long expires = ACCESS_ONCE(peer->pmtu_expires);
1610
1611         if (!expires)
1612                 return;
1613         if (time_before(jiffies, expires)) {
1614                 u32 orig_dst_mtu = dst_mtu(dst);
1615                 if (peer->pmtu_learned < orig_dst_mtu) {
1616                         if (!peer->pmtu_orig)
1617                                 peer->pmtu_orig = dst_metric_raw(dst, RTAX_MTU);
1618                         dst_metric_set(dst, RTAX_MTU, peer->pmtu_learned);
1619                 }
1620         } else if (cmpxchg(&peer->pmtu_expires, expires, 0) == expires)
1621                 dst_metric_set(dst, RTAX_MTU, peer->pmtu_orig);
1622 }
1623
1624 static void ip_rt_update_pmtu(struct dst_entry *dst, u32 mtu)
1625 {
1626         struct rtable *rt = (struct rtable *) dst;
1627         struct inet_peer *peer;
1628
1629         dst_confirm(dst);
1630
1631         peer = rt_get_peer_create(rt, rt->rt_dst);
1632         if (peer) {
1633                 unsigned long pmtu_expires = ACCESS_ONCE(peer->pmtu_expires);
1634
1635                 if (mtu < ip_rt_min_pmtu)
1636                         mtu = ip_rt_min_pmtu;
1637                 if (!pmtu_expires || mtu < peer->pmtu_learned) {
1638
1639                         pmtu_expires = jiffies + ip_rt_mtu_expires;
1640                         if (!pmtu_expires)
1641                                 pmtu_expires = 1UL;
1642
1643                         peer->pmtu_learned = mtu;
1644                         peer->pmtu_expires = pmtu_expires;
1645
1646                         atomic_inc(&__rt_peer_genid);
1647                         rt->rt_peer_genid = rt_peer_genid();
1648                 }
1649                 check_peer_pmtu(dst, peer);
1650         }
1651 }
1652
1653 void ipv4_update_pmtu(struct sk_buff *skb, struct net *net, u32 mtu,
1654                       int oif, u32 mark, u8 protocol, int flow_flags)
1655 {
1656         const struct iphdr *iph = (const struct iphdr *)skb->data;
1657         struct flowi4 fl4;
1658         struct rtable *rt;
1659
1660         flowi4_init_output(&fl4, oif, mark, RT_TOS(iph->tos), RT_SCOPE_UNIVERSE,
1661                            protocol, flow_flags | FLOWI_FLAG_PRECOW_METRICS,
1662                            iph->daddr, iph->saddr, 0, 0);
1663         rt = __ip_route_output_key(net, &fl4);
1664         if (!IS_ERR(rt)) {
1665                 ip_rt_update_pmtu(&rt->dst, mtu);
1666                 ip_rt_put(rt);
1667         }
1668 }
1669 EXPORT_SYMBOL_GPL(ipv4_update_pmtu);
1670
1671 void ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu)
1672 {
1673         const struct inet_sock *inet = inet_sk(sk);
1674
1675         return ipv4_update_pmtu(skb, sock_net(sk), mtu,
1676                                 sk->sk_bound_dev_if, sk->sk_mark,
1677                                 inet->hdrincl ? IPPROTO_RAW : sk->sk_protocol,
1678                                 inet_sk_flowi_flags(sk));
1679 }
1680 EXPORT_SYMBOL_GPL(ipv4_sk_update_pmtu);
1681
1682 static void ipv4_validate_peer(struct rtable *rt)
1683 {
1684         if (rt->rt_peer_genid != rt_peer_genid()) {
1685                 struct inet_peer *peer = rt_get_peer(rt, rt->rt_dst);
1686
1687                 if (peer) {
1688                         check_peer_pmtu(&rt->dst, peer);
1689
1690                         if (peer->redirect_learned.a4 &&
1691                             peer->redirect_learned.a4 != rt->rt_gateway)
1692                                 check_peer_redir(&rt->dst, peer);
1693                 }
1694
1695                 rt->rt_peer_genid = rt_peer_genid();
1696         }
1697 }
1698
1699 static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie)
1700 {
1701         struct rtable *rt = (struct rtable *) dst;
1702
1703         if (rt_is_expired(rt))
1704                 return NULL;
1705         ipv4_validate_peer(rt);
1706         return dst;
1707 }
1708
1709 static void ipv4_dst_destroy(struct dst_entry *dst)
1710 {
1711         struct rtable *rt = (struct rtable *) dst;
1712
1713         if (rt->fi) {
1714                 fib_info_put(rt->fi);
1715                 rt->fi = NULL;
1716         }
1717         if (rt_has_peer(rt)) {
1718                 struct inet_peer *peer = rt_peer_ptr(rt);
1719                 inet_putpeer(peer);
1720         }
1721 }
1722
1723
1724 static void ipv4_link_failure(struct sk_buff *skb)
1725 {
1726         struct rtable *rt;
1727
1728         icmp_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0);
1729
1730         rt = skb_rtable(skb);
1731         if (rt && rt_has_peer(rt)) {
1732                 struct inet_peer *peer = rt_peer_ptr(rt);
1733                 if (peer_pmtu_cleaned(peer))
1734                         dst_metric_set(&rt->dst, RTAX_MTU, peer->pmtu_orig);
1735         }
1736 }
1737
1738 static int ip_rt_bug(struct sk_buff *skb)
1739 {
1740         pr_debug("%s: %pI4 -> %pI4, %s\n",
1741                  __func__, &ip_hdr(skb)->saddr, &ip_hdr(skb)->daddr,
1742                  skb->dev ? skb->dev->name : "?");
1743         kfree_skb(skb);
1744         WARN_ON(1);
1745         return 0;
1746 }
1747
1748 /*
1749    We do not cache source address of outgoing interface,
1750    because it is used only by IP RR, TS and SRR options,
1751    so that it out of fast path.
1752
1753    BTW remember: "addr" is allowed to be not aligned
1754    in IP options!
1755  */
1756
1757 void ip_rt_get_source(u8 *addr, struct sk_buff *skb, struct rtable *rt)
1758 {
1759         __be32 src;
1760
1761         if (rt_is_output_route(rt))
1762                 src = ip_hdr(skb)->saddr;
1763         else {
1764                 struct fib_result res;
1765                 struct flowi4 fl4;
1766                 struct iphdr *iph;
1767
1768                 iph = ip_hdr(skb);
1769
1770                 memset(&fl4, 0, sizeof(fl4));
1771                 fl4.daddr = iph->daddr;
1772                 fl4.saddr = iph->saddr;
1773                 fl4.flowi4_tos = RT_TOS(iph->tos);
1774                 fl4.flowi4_oif = rt->dst.dev->ifindex;
1775                 fl4.flowi4_iif = skb->dev->ifindex;
1776                 fl4.flowi4_mark = skb->mark;
1777
1778                 rcu_read_lock();
1779                 if (fib_lookup(dev_net(rt->dst.dev), &fl4, &res) == 0)
1780                         src = FIB_RES_PREFSRC(dev_net(rt->dst.dev), res);
1781                 else
1782                         src = inet_select_addr(rt->dst.dev, rt->rt_gateway,
1783                                         RT_SCOPE_UNIVERSE);
1784                 rcu_read_unlock();
1785         }
1786         memcpy(addr, &src, 4);
1787 }
1788
1789 #ifdef CONFIG_IP_ROUTE_CLASSID
1790 static void set_class_tag(struct rtable *rt, u32 tag)
1791 {
1792         if (!(rt->dst.tclassid & 0xFFFF))
1793                 rt->dst.tclassid |= tag & 0xFFFF;
1794         if (!(rt->dst.tclassid & 0xFFFF0000))
1795                 rt->dst.tclassid |= tag & 0xFFFF0000;
1796 }
1797 #endif
1798
1799 static unsigned int ipv4_default_advmss(const struct dst_entry *dst)
1800 {
1801         unsigned int advmss = dst_metric_raw(dst, RTAX_ADVMSS);
1802
1803         if (advmss == 0) {
1804                 advmss = max_t(unsigned int, dst->dev->mtu - 40,
1805                                ip_rt_min_advmss);
1806                 if (advmss > 65535 - 40)
1807                         advmss = 65535 - 40;
1808         }
1809         return advmss;
1810 }
1811
1812 static unsigned int ipv4_mtu(const struct dst_entry *dst)
1813 {
1814         const struct rtable *rt = (const struct rtable *) dst;
1815         unsigned int mtu = dst_metric_raw(dst, RTAX_MTU);
1816
1817         if (mtu && rt_is_output_route(rt))
1818                 return mtu;
1819
1820         mtu = dst->dev->mtu;
1821
1822         if (unlikely(dst_metric_locked(dst, RTAX_MTU))) {
1823
1824                 if (rt->rt_gateway != rt->rt_dst && mtu > 576)
1825                         mtu = 576;
1826         }
1827
1828         if (mtu > IP_MAX_MTU)
1829                 mtu = IP_MAX_MTU;
1830
1831         return mtu;
1832 }
1833
1834 static void rt_init_metrics(struct rtable *rt, const struct flowi4 *fl4,
1835                             struct fib_info *fi)
1836 {
1837         struct inet_peer_base *base;
1838         struct inet_peer *peer;
1839         int create = 0;
1840
1841         /* If a peer entry exists for this destination, we must hook
1842          * it up in order to get at cached metrics.
1843          */
1844         if (fl4 && (fl4->flowi4_flags & FLOWI_FLAG_PRECOW_METRICS))
1845                 create = 1;
1846
1847         base = inetpeer_base_ptr(rt->_peer);
1848         BUG_ON(!base);
1849
1850         peer = inet_getpeer_v4(base, rt->rt_dst, create);
1851         if (peer) {
1852                 __rt_set_peer(rt, peer);
1853                 rt->rt_peer_genid = rt_peer_genid();
1854                 if (inet_metrics_new(peer))
1855                         memcpy(peer->metrics, fi->fib_metrics,
1856                                sizeof(u32) * RTAX_MAX);
1857                 dst_init_metrics(&rt->dst, peer->metrics, false);
1858
1859                 check_peer_pmtu(&rt->dst, peer);
1860
1861                 if (peer->redirect_learned.a4 &&
1862                     peer->redirect_learned.a4 != rt->rt_gateway) {
1863                         rt->rt_gateway = peer->redirect_learned.a4;
1864                         rt->rt_flags |= RTCF_REDIRECTED;
1865                 }
1866         } else {
1867                 if (fi->fib_metrics != (u32 *) dst_default_metrics) {
1868                         rt->fi = fi;
1869                         atomic_inc(&fi->fib_clntref);
1870                 }
1871                 dst_init_metrics(&rt->dst, fi->fib_metrics, true);
1872         }
1873 }
1874
1875 static void rt_set_nexthop(struct rtable *rt, const struct flowi4 *fl4,
1876                            const struct fib_result *res,
1877                            struct fib_info *fi, u16 type, u32 itag)
1878 {
1879         struct dst_entry *dst = &rt->dst;
1880
1881         if (fi) {
1882                 if (FIB_RES_GW(*res) &&
1883                     FIB_RES_NH(*res).nh_scope == RT_SCOPE_LINK)
1884                         rt->rt_gateway = FIB_RES_GW(*res);
1885                 rt_init_metrics(rt, fl4, fi);
1886 #ifdef CONFIG_IP_ROUTE_CLASSID
1887                 dst->tclassid = FIB_RES_NH(*res).nh_tclassid;
1888 #endif
1889         }
1890
1891         if (dst_mtu(dst) > IP_MAX_MTU)
1892                 dst_metric_set(dst, RTAX_MTU, IP_MAX_MTU);
1893
1894 #ifdef CONFIG_IP_ROUTE_CLASSID
1895 #ifdef CONFIG_IP_MULTIPLE_TABLES
1896         set_class_tag(rt, fib_rules_tclass(res));
1897 #endif
1898         set_class_tag(rt, itag);
1899 #endif
1900 }
1901
1902 static struct rtable *rt_dst_alloc(struct net_device *dev,
1903                                    bool nopolicy, bool noxfrm)
1904 {
1905         return dst_alloc(&ipv4_dst_ops, dev, 1, -1,
1906                          DST_HOST |
1907                          (nopolicy ? DST_NOPOLICY : 0) |
1908                          (noxfrm ? DST_NOXFRM : 0));
1909 }
1910
1911 /* called in rcu_read_lock() section */
1912 static int ip_route_input_mc(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1913                                 u8 tos, struct net_device *dev, int our)
1914 {
1915         unsigned int hash;
1916         struct rtable *rth;
1917         struct in_device *in_dev = __in_dev_get_rcu(dev);
1918         u32 itag = 0;
1919         int err;
1920
1921         /* Primary sanity checks. */
1922
1923         if (in_dev == NULL)
1924                 return -EINVAL;
1925
1926         if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) ||
1927             skb->protocol != htons(ETH_P_IP))
1928                 goto e_inval;
1929
1930         if (likely(!IN_DEV_ROUTE_LOCALNET(in_dev)))
1931                 if (ipv4_is_loopback(saddr))
1932                         goto e_inval;
1933
1934         if (ipv4_is_zeronet(saddr)) {
1935                 if (!ipv4_is_local_multicast(daddr))
1936                         goto e_inval;
1937         } else {
1938                 err = fib_validate_source(skb, saddr, 0, tos, 0, dev,
1939                                           in_dev, &itag);
1940                 if (err < 0)
1941                         goto e_err;
1942         }
1943         rth = rt_dst_alloc(dev_net(dev)->loopback_dev,
1944                            IN_DEV_CONF_GET(in_dev, NOPOLICY), false);
1945         if (!rth)
1946                 goto e_nobufs;
1947
1948 #ifdef CONFIG_IP_ROUTE_CLASSID
1949         rth->dst.tclassid = itag;
1950 #endif
1951         rth->dst.output = ip_rt_bug;
1952
1953         rth->rt_key_dst = daddr;
1954         rth->rt_key_src = saddr;
1955         rth->rt_genid   = rt_genid(dev_net(dev));
1956         rth->rt_flags   = RTCF_MULTICAST;
1957         rth->rt_type    = RTN_MULTICAST;
1958         rth->rt_key_tos = tos;
1959         rth->rt_dst     = daddr;
1960         rth->rt_src     = saddr;
1961         rth->rt_route_iif = dev->ifindex;
1962         rth->rt_iif     = dev->ifindex;
1963         rth->rt_oif     = 0;
1964         rth->rt_mark    = skb->mark;
1965         rth->rt_gateway = daddr;
1966         rth->rt_peer_genid = 0;
1967         rt_init_peer(rth, dev_net(dev)->ipv4.peers);
1968         rth->fi = NULL;
1969         if (our) {
1970                 rth->dst.input= ip_local_deliver;
1971                 rth->rt_flags |= RTCF_LOCAL;
1972         }
1973
1974 #ifdef CONFIG_IP_MROUTE
1975         if (!ipv4_is_local_multicast(daddr) && IN_DEV_MFORWARD(in_dev))
1976                 rth->dst.input = ip_mr_input;
1977 #endif
1978         RT_CACHE_STAT_INC(in_slow_mc);
1979
1980         hash = rt_hash(daddr, saddr, dev->ifindex, rt_genid(dev_net(dev)));
1981         rth = rt_intern_hash(hash, rth, skb, dev->ifindex);
1982         return IS_ERR(rth) ? PTR_ERR(rth) : 0;
1983
1984 e_nobufs:
1985         return -ENOBUFS;
1986 e_inval:
1987         return -EINVAL;
1988 e_err:
1989         return err;
1990 }
1991
1992
1993 static void ip_handle_martian_source(struct net_device *dev,
1994                                      struct in_device *in_dev,
1995                                      struct sk_buff *skb,
1996                                      __be32 daddr,
1997                                      __be32 saddr)
1998 {
1999         RT_CACHE_STAT_INC(in_martian_src);
2000 #ifdef CONFIG_IP_ROUTE_VERBOSE
2001         if (IN_DEV_LOG_MARTIANS(in_dev) && net_ratelimit()) {
2002                 /*
2003                  *      RFC1812 recommendation, if source is martian,
2004                  *      the only hint is MAC header.
2005                  */
2006                 pr_warn("martian source %pI4 from %pI4, on dev %s\n",
2007                         &daddr, &saddr, dev->name);
2008                 if (dev->hard_header_len && skb_mac_header_was_set(skb)) {
2009                         print_hex_dump(KERN_WARNING, "ll header: ",
2010                                        DUMP_PREFIX_OFFSET, 16, 1,
2011                                        skb_mac_header(skb),
2012                                        dev->hard_header_len, true);
2013                 }
2014         }
2015 #endif
2016 }
2017
2018 /* called in rcu_read_lock() section */
2019 static int __mkroute_input(struct sk_buff *skb,
2020                            const struct fib_result *res,
2021                            struct in_device *in_dev,
2022                            __be32 daddr, __be32 saddr, u32 tos,
2023                            struct rtable **result)
2024 {
2025         struct rtable *rth;
2026         int err;
2027         struct in_device *out_dev;
2028         unsigned int flags = 0;
2029         u32 itag;
2030
2031         /* get a working reference to the output device */
2032         out_dev = __in_dev_get_rcu(FIB_RES_DEV(*res));
2033         if (out_dev == NULL) {
2034                 net_crit_ratelimited("Bug in ip_route_input_slow(). Please report.\n");
2035                 return -EINVAL;
2036         }
2037
2038
2039         err = fib_validate_source(skb, saddr, daddr, tos, FIB_RES_OIF(*res),
2040                                   in_dev->dev, in_dev, &itag);
2041         if (err < 0) {
2042                 ip_handle_martian_source(in_dev->dev, in_dev, skb, daddr,
2043                                          saddr);
2044
2045                 goto cleanup;
2046         }
2047
2048         if (err)
2049                 flags |= RTCF_DIRECTSRC;
2050
2051         if (out_dev == in_dev && err &&
2052             (IN_DEV_SHARED_MEDIA(out_dev) ||
2053              inet_addr_onlink(out_dev, saddr, FIB_RES_GW(*res))))
2054                 flags |= RTCF_DOREDIRECT;
2055
2056         if (skb->protocol != htons(ETH_P_IP)) {
2057                 /* Not IP (i.e. ARP). Do not create route, if it is
2058                  * invalid for proxy arp. DNAT routes are always valid.
2059                  *
2060                  * Proxy arp feature have been extended to allow, ARP
2061                  * replies back to the same interface, to support
2062                  * Private VLAN switch technologies. See arp.c.
2063                  */
2064                 if (out_dev == in_dev &&
2065                     IN_DEV_PROXY_ARP_PVLAN(in_dev) == 0) {
2066                         err = -EINVAL;
2067                         goto cleanup;
2068                 }
2069         }
2070
2071         rth = rt_dst_alloc(out_dev->dev,
2072                            IN_DEV_CONF_GET(in_dev, NOPOLICY),
2073                            IN_DEV_CONF_GET(out_dev, NOXFRM));
2074         if (!rth) {
2075                 err = -ENOBUFS;
2076                 goto cleanup;
2077         }
2078
2079         rth->rt_key_dst = daddr;
2080         rth->rt_key_src = saddr;
2081         rth->rt_genid = rt_genid(dev_net(rth->dst.dev));
2082         rth->rt_flags = flags;
2083         rth->rt_type = res->type;
2084         rth->rt_key_tos = tos;
2085         rth->rt_dst     = daddr;
2086         rth->rt_src     = saddr;
2087         rth->rt_route_iif = in_dev->dev->ifindex;
2088         rth->rt_iif     = in_dev->dev->ifindex;
2089         rth->rt_oif     = 0;
2090         rth->rt_mark    = skb->mark;
2091         rth->rt_gateway = daddr;
2092         rth->rt_peer_genid = 0;
2093         rt_init_peer(rth, &res->table->tb_peers);
2094         rth->fi = NULL;
2095
2096         rth->dst.input = ip_forward;
2097         rth->dst.output = ip_output;
2098
2099         rt_set_nexthop(rth, NULL, res, res->fi, res->type, itag);
2100
2101         *result = rth;
2102         err = 0;
2103  cleanup:
2104         return err;
2105 }
2106
2107 static int ip_mkroute_input(struct sk_buff *skb,
2108                             struct fib_result *res,
2109                             const struct flowi4 *fl4,
2110                             struct in_device *in_dev,
2111                             __be32 daddr, __be32 saddr, u32 tos)
2112 {
2113         struct rtable *rth = NULL;
2114         int err;
2115         unsigned int hash;
2116
2117 #ifdef CONFIG_IP_ROUTE_MULTIPATH
2118         if (res->fi && res->fi->fib_nhs > 1)
2119                 fib_select_multipath(res);
2120 #endif
2121
2122         /* create a routing cache entry */
2123         err = __mkroute_input(skb, res, in_dev, daddr, saddr, tos, &rth);
2124         if (err)
2125                 return err;
2126
2127         /* put it into the cache */
2128         hash = rt_hash(daddr, saddr, fl4->flowi4_iif,
2129                        rt_genid(dev_net(rth->dst.dev)));
2130         rth = rt_intern_hash(hash, rth, skb, fl4->flowi4_iif);
2131         if (IS_ERR(rth))
2132                 return PTR_ERR(rth);
2133         return 0;
2134 }
2135
2136 /*
2137  *      NOTE. We drop all the packets that has local source
2138  *      addresses, because every properly looped back packet
2139  *      must have correct destination already attached by output routine.
2140  *
2141  *      Such approach solves two big problems:
2142  *      1. Not simplex devices are handled properly.
2143  *      2. IP spoofing attempts are filtered with 100% of guarantee.
2144  *      called with rcu_read_lock()
2145  */
2146
2147 static int ip_route_input_slow(struct sk_buff *skb, __be32 daddr, __be32 saddr,
2148                                u8 tos, struct net_device *dev)
2149 {
2150         struct fib_result res;
2151         struct in_device *in_dev = __in_dev_get_rcu(dev);
2152         struct flowi4   fl4;
2153         unsigned int    flags = 0;
2154         u32             itag = 0;
2155         struct rtable   *rth;
2156         unsigned int    hash;
2157         int             err = -EINVAL;
2158         struct net    *net = dev_net(dev);
2159
2160         /* IP on this device is disabled. */
2161
2162         if (!in_dev)
2163                 goto out;
2164
2165         /* Check for the most weird martians, which can be not detected
2166            by fib_lookup.
2167          */
2168
2169         if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr))
2170                 goto martian_source;
2171
2172         if (ipv4_is_lbcast(daddr) || (saddr == 0 && daddr == 0))
2173                 goto brd_input;
2174
2175         /* Accept zero addresses only to limited broadcast;
2176          * I even do not know to fix it or not. Waiting for complains :-)
2177          */
2178         if (ipv4_is_zeronet(saddr))
2179                 goto martian_source;
2180
2181         if (ipv4_is_zeronet(daddr))
2182                 goto martian_destination;
2183
2184         if (likely(!IN_DEV_ROUTE_LOCALNET(in_dev))) {
2185                 if (ipv4_is_loopback(daddr))
2186                         goto martian_destination;
2187
2188                 if (ipv4_is_loopback(saddr))
2189                         goto martian_source;
2190         }
2191
2192         /*
2193          *      Now we are ready to route packet.
2194          */
2195         fl4.flowi4_oif = 0;
2196         fl4.flowi4_iif = dev->ifindex;
2197         fl4.flowi4_mark = skb->mark;
2198         fl4.flowi4_tos = tos;
2199         fl4.flowi4_scope = RT_SCOPE_UNIVERSE;
2200         fl4.daddr = daddr;
2201         fl4.saddr = saddr;
2202         err = fib_lookup(net, &fl4, &res);
2203         if (err != 0)
2204                 goto no_route;
2205
2206         RT_CACHE_STAT_INC(in_slow_tot);
2207
2208         if (res.type == RTN_BROADCAST)
2209                 goto brd_input;
2210
2211         if (res.type == RTN_LOCAL) {
2212                 err = fib_validate_source(skb, saddr, daddr, tos,
2213                                           net->loopback_dev->ifindex,
2214                                           dev, in_dev, &itag);
2215                 if (err < 0)
2216                         goto martian_source_keep_err;
2217                 if (err)
2218                         flags |= RTCF_DIRECTSRC;
2219                 goto local_input;
2220         }
2221
2222         if (!IN_DEV_FORWARD(in_dev))
2223                 goto no_route;
2224         if (res.type != RTN_UNICAST)
2225                 goto martian_destination;
2226
2227         err = ip_mkroute_input(skb, &res, &fl4, in_dev, daddr, saddr, tos);
2228 out:    return err;
2229
2230 brd_input:
2231         if (skb->protocol != htons(ETH_P_IP))
2232                 goto e_inval;
2233
2234         if (!ipv4_is_zeronet(saddr)) {
2235                 err = fib_validate_source(skb, saddr, 0, tos, 0, dev,
2236                                           in_dev, &itag);
2237                 if (err < 0)
2238                         goto martian_source_keep_err;
2239                 if (err)
2240                         flags |= RTCF_DIRECTSRC;
2241         }
2242         flags |= RTCF_BROADCAST;
2243         res.type = RTN_BROADCAST;
2244         RT_CACHE_STAT_INC(in_brd);
2245
2246 local_input:
2247         rth = rt_dst_alloc(net->loopback_dev,
2248                            IN_DEV_CONF_GET(in_dev, NOPOLICY), false);
2249         if (!rth)
2250                 goto e_nobufs;
2251
2252         rth->dst.input= ip_local_deliver;
2253         rth->dst.output= ip_rt_bug;
2254 #ifdef CONFIG_IP_ROUTE_CLASSID
2255         rth->dst.tclassid = itag;
2256 #endif
2257
2258         rth->rt_key_dst = daddr;
2259         rth->rt_key_src = saddr;
2260         rth->rt_genid = rt_genid(net);
2261         rth->rt_flags   = flags|RTCF_LOCAL;
2262         rth->rt_type    = res.type;
2263         rth->rt_key_tos = tos;
2264         rth->rt_dst     = daddr;
2265         rth->rt_src     = saddr;
2266         rth->rt_route_iif = dev->ifindex;
2267         rth->rt_iif     = dev->ifindex;
2268         rth->rt_oif     = 0;
2269         rth->rt_mark    = skb->mark;
2270         rth->rt_gateway = daddr;
2271         rth->rt_peer_genid = 0;
2272         rt_init_peer(rth, net->ipv4.peers);
2273         rth->fi = NULL;
2274         if (res.type == RTN_UNREACHABLE) {
2275                 rth->dst.input= ip_error;
2276                 rth->dst.error= -err;
2277                 rth->rt_flags   &= ~RTCF_LOCAL;
2278         }
2279         hash = rt_hash(daddr, saddr, fl4.flowi4_iif, rt_genid(net));
2280         rth = rt_intern_hash(hash, rth, skb, fl4.flowi4_iif);
2281         err = 0;
2282         if (IS_ERR(rth))
2283                 err = PTR_ERR(rth);
2284         goto out;
2285
2286 no_route:
2287         RT_CACHE_STAT_INC(in_no_route);
2288         res.type = RTN_UNREACHABLE;
2289         if (err == -ESRCH)
2290                 err = -ENETUNREACH;
2291         goto local_input;
2292
2293         /*
2294          *      Do not cache martian addresses: they should be logged (RFC1812)
2295          */
2296 martian_destination:
2297         RT_CACHE_STAT_INC(in_martian_dst);
2298 #ifdef CONFIG_IP_ROUTE_VERBOSE
2299         if (IN_DEV_LOG_MARTIANS(in_dev))
2300                 net_warn_ratelimited("martian destination %pI4 from %pI4, dev %s\n",
2301                                      &daddr, &saddr, dev->name);
2302 #endif
2303
2304 e_inval:
2305         err = -EINVAL;
2306         goto out;
2307
2308 e_nobufs:
2309         err = -ENOBUFS;
2310         goto out;
2311
2312 martian_source:
2313         err = -EINVAL;
2314 martian_source_keep_err:
2315         ip_handle_martian_source(dev, in_dev, skb, daddr, saddr);
2316         goto out;
2317 }
2318
2319 int ip_route_input_common(struct sk_buff *skb, __be32 daddr, __be32 saddr,
2320                            u8 tos, struct net_device *dev, bool noref)
2321 {
2322         struct rtable   *rth;
2323         unsigned int    hash;
2324         int iif = dev->ifindex;
2325         struct net *net;
2326         int res;
2327
2328         net = dev_net(dev);
2329
2330         rcu_read_lock();
2331
2332         if (!rt_caching(net))
2333                 goto skip_cache;
2334
2335         tos &= IPTOS_RT_MASK;
2336         hash = rt_hash(daddr, saddr, iif, rt_genid(net));
2337
2338         for (rth = rcu_dereference(rt_hash_table[hash].chain); rth;
2339              rth = rcu_dereference(rth->dst.rt_next)) {
2340                 if ((((__force u32)rth->rt_key_dst ^ (__force u32)daddr) |
2341                      ((__force u32)rth->rt_key_src ^ (__force u32)saddr) |
2342                      (rth->rt_route_iif ^ iif) |
2343                      (rth->rt_key_tos ^ tos)) == 0 &&
2344                     rth->rt_mark == skb->mark &&
2345                     net_eq(dev_net(rth->dst.dev), net) &&
2346                     !rt_is_expired(rth)) {
2347                         ipv4_validate_peer(rth);
2348                         if (noref) {
2349                                 dst_use_noref(&rth->dst, jiffies);
2350                                 skb_dst_set_noref(skb, &rth->dst);
2351                         } else {
2352                                 dst_use(&rth->dst, jiffies);
2353                                 skb_dst_set(skb, &rth->dst);
2354                         }
2355                         RT_CACHE_STAT_INC(in_hit);
2356                         rcu_read_unlock();
2357                         return 0;
2358                 }
2359                 RT_CACHE_STAT_INC(in_hlist_search);
2360         }
2361
2362 skip_cache:
2363         /* Multicast recognition logic is moved from route cache to here.
2364            The problem was that too many Ethernet cards have broken/missing
2365            hardware multicast filters :-( As result the host on multicasting
2366            network acquires a lot of useless route cache entries, sort of
2367            SDR messages from all the world. Now we try to get rid of them.
2368            Really, provided software IP multicast filter is organized
2369            reasonably (at least, hashed), it does not result in a slowdown
2370            comparing with route cache reject entries.
2371            Note, that multicast routers are not affected, because
2372            route cache entry is created eventually.
2373          */
2374         if (ipv4_is_multicast(daddr)) {
2375                 struct in_device *in_dev = __in_dev_get_rcu(dev);
2376
2377                 if (in_dev) {
2378                         int our = ip_check_mc_rcu(in_dev, daddr, saddr,
2379                                                   ip_hdr(skb)->protocol);
2380                         if (our
2381 #ifdef CONFIG_IP_MROUTE
2382                                 ||
2383                             (!ipv4_is_local_multicast(daddr) &&
2384                              IN_DEV_MFORWARD(in_dev))
2385 #endif
2386                            ) {
2387                                 int res = ip_route_input_mc(skb, daddr, saddr,
2388                                                             tos, dev, our);
2389                                 rcu_read_unlock();
2390                                 return res;
2391                         }
2392                 }
2393                 rcu_read_unlock();
2394                 return -EINVAL;
2395         }
2396         res = ip_route_input_slow(skb, daddr, saddr, tos, dev);
2397         rcu_read_unlock();
2398         return res;
2399 }
2400 EXPORT_SYMBOL(ip_route_input_common);
2401
2402 /* called with rcu_read_lock() */
2403 static struct rtable *__mkroute_output(const struct fib_result *res,
2404                                        const struct flowi4 *fl4,
2405                                        __be32 orig_daddr, __be32 orig_saddr,
2406                                        int orig_oif, __u8 orig_rtos,
2407                                        struct net_device *dev_out,
2408                                        unsigned int flags)
2409 {
2410         struct fib_info *fi = res->fi;
2411         struct in_device *in_dev;
2412         u16 type = res->type;
2413         struct rtable *rth;
2414
2415         in_dev = __in_dev_get_rcu(dev_out);
2416         if (!in_dev)
2417                 return ERR_PTR(-EINVAL);
2418
2419         if (likely(!IN_DEV_ROUTE_LOCALNET(in_dev)))
2420                 if (ipv4_is_loopback(fl4->saddr) && !(dev_out->flags & IFF_LOOPBACK))
2421                         return ERR_PTR(-EINVAL);
2422
2423         if (ipv4_is_lbcast(fl4->daddr))
2424                 type = RTN_BROADCAST;
2425         else if (ipv4_is_multicast(fl4->daddr))
2426                 type = RTN_MULTICAST;
2427         else if (ipv4_is_zeronet(fl4->daddr))
2428                 return ERR_PTR(-EINVAL);
2429
2430         if (dev_out->flags & IFF_LOOPBACK)
2431                 flags |= RTCF_LOCAL;
2432
2433         if (type == RTN_BROADCAST) {
2434                 flags |= RTCF_BROADCAST | RTCF_LOCAL;
2435                 fi = NULL;
2436         } else if (type == RTN_MULTICAST) {
2437                 flags |= RTCF_MULTICAST | RTCF_LOCAL;
2438                 if (!ip_check_mc_rcu(in_dev, fl4->daddr, fl4->saddr,
2439                                      fl4->flowi4_proto))
2440                         flags &= ~RTCF_LOCAL;
2441                 /* If multicast route do not exist use
2442                  * default one, but do not gateway in this case.
2443                  * Yes, it is hack.
2444                  */
2445                 if (fi && res->prefixlen < 4)
2446                         fi = NULL;
2447         }
2448
2449         rth = rt_dst_alloc(dev_out,
2450                            IN_DEV_CONF_GET(in_dev, NOPOLICY),
2451                            IN_DEV_CONF_GET(in_dev, NOXFRM));
2452         if (!rth)
2453                 return ERR_PTR(-ENOBUFS);
2454
2455         rth->dst.output = ip_output;
2456
2457         rth->rt_key_dst = orig_daddr;
2458         rth->rt_key_src = orig_saddr;
2459         rth->rt_genid = rt_genid(dev_net(dev_out));
2460         rth->rt_flags   = flags;
2461         rth->rt_type    = type;
2462         rth->rt_key_tos = orig_rtos;
2463         rth->rt_dst     = fl4->daddr;
2464         rth->rt_src     = fl4->saddr;
2465         rth->rt_route_iif = 0;
2466         rth->rt_iif     = orig_oif ? : dev_out->ifindex;
2467         rth->rt_oif     = orig_oif;
2468         rth->rt_mark    = fl4->flowi4_mark;
2469         rth->rt_gateway = fl4->daddr;
2470         rth->rt_peer_genid = 0;
2471         rt_init_peer(rth, (res->table ?
2472                            &res->table->tb_peers :
2473                            dev_net(dev_out)->ipv4.peers));
2474         rth->fi = NULL;
2475
2476         RT_CACHE_STAT_INC(out_slow_tot);
2477
2478         if (flags & RTCF_LOCAL)
2479                 rth->dst.input = ip_local_deliver;
2480         if (flags & (RTCF_BROADCAST | RTCF_MULTICAST)) {
2481                 if (flags & RTCF_LOCAL &&
2482                     !(dev_out->flags & IFF_LOOPBACK)) {
2483                         rth->dst.output = ip_mc_output;
2484                         RT_CACHE_STAT_INC(out_slow_mc);
2485                 }
2486 #ifdef CONFIG_IP_MROUTE
2487                 if (type == RTN_MULTICAST) {
2488                         if (IN_DEV_MFORWARD(in_dev) &&
2489                             !ipv4_is_local_multicast(fl4->daddr)) {
2490                                 rth->dst.input = ip_mr_input;
2491                                 rth->dst.output = ip_mc_output;
2492                         }
2493                 }
2494 #endif
2495         }
2496
2497         rt_set_nexthop(rth, fl4, res, fi, type, 0);
2498
2499         if (fl4->flowi4_flags & FLOWI_FLAG_RT_NOCACHE)
2500                 rth->dst.flags |= DST_NOCACHE;
2501
2502         return rth;
2503 }
2504
2505 /*
2506  * Major route resolver routine.
2507  * called with rcu_read_lock();
2508  */
2509
2510 static struct rtable *ip_route_output_slow(struct net *net, struct flowi4 *fl4)
2511 {
2512         struct net_device *dev_out = NULL;
2513         __u8 tos = RT_FL_TOS(fl4);
2514         unsigned int flags = 0;
2515         struct fib_result res;
2516         struct rtable *rth;
2517         __be32 orig_daddr;
2518         __be32 orig_saddr;
2519         int orig_oif;
2520
2521         res.fi          = NULL;
2522         res.table       = NULL;
2523 #ifdef CONFIG_IP_MULTIPLE_TABLES
2524         res.r           = NULL;
2525 #endif
2526
2527         orig_daddr = fl4->daddr;
2528         orig_saddr = fl4->saddr;
2529         orig_oif = fl4->flowi4_oif;
2530
2531         fl4->flowi4_iif = net->loopback_dev->ifindex;
2532         fl4->flowi4_tos = tos & IPTOS_RT_MASK;
2533         fl4->flowi4_scope = ((tos & RTO_ONLINK) ?
2534                          RT_SCOPE_LINK : RT_SCOPE_UNIVERSE);
2535
2536         rcu_read_lock();
2537         if (fl4->saddr) {
2538                 rth = ERR_PTR(-EINVAL);
2539                 if (ipv4_is_multicast(fl4->saddr) ||
2540                     ipv4_is_lbcast(fl4->saddr) ||
2541                     ipv4_is_zeronet(fl4->saddr))
2542                         goto out;
2543
2544                 /* I removed check for oif == dev_out->oif here.
2545                    It was wrong for two reasons:
2546                    1. ip_dev_find(net, saddr) can return wrong iface, if saddr
2547                       is assigned to multiple interfaces.
2548                    2. Moreover, we are allowed to send packets with saddr
2549                       of another iface. --ANK
2550                  */
2551
2552                 if (fl4->flowi4_oif == 0 &&
2553                     (ipv4_is_multicast(fl4->daddr) ||
2554                      ipv4_is_lbcast(fl4->daddr))) {
2555                         /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
2556                         dev_out = __ip_dev_find(net, fl4->saddr, false);
2557                         if (dev_out == NULL)
2558                                 goto out;
2559
2560                         /* Special hack: user can direct multicasts
2561                            and limited broadcast via necessary interface
2562                            without fiddling with IP_MULTICAST_IF or IP_PKTINFO.
2563                            This hack is not just for fun, it allows
2564                            vic,vat and friends to work.
2565                            They bind socket to loopback, set ttl to zero
2566                            and expect that it will work.
2567                            From the viewpoint of routing cache they are broken,
2568                            because we are not allowed to build multicast path
2569                            with loopback source addr (look, routing cache
2570                            cannot know, that ttl is zero, so that packet
2571                            will not leave this host and route is valid).
2572                            Luckily, this hack is good workaround.
2573                          */
2574
2575                         fl4->flowi4_oif = dev_out->ifindex;
2576                         goto make_route;
2577                 }
2578
2579                 if (!(fl4->flowi4_flags & FLOWI_FLAG_ANYSRC)) {
2580                         /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
2581                         if (!__ip_dev_find(net, fl4->saddr, false))
2582                                 goto out;
2583                 }
2584         }
2585
2586
2587         if (fl4->flowi4_oif) {
2588                 dev_out = dev_get_by_index_rcu(net, fl4->flowi4_oif);
2589                 rth = ERR_PTR(-ENODEV);
2590                 if (dev_out == NULL)
2591                         goto out;
2592
2593                 /* RACE: Check return value of inet_select_addr instead. */
2594                 if (!(dev_out->flags & IFF_UP) || !__in_dev_get_rcu(dev_out)) {
2595                         rth = ERR_PTR(-ENETUNREACH);
2596                         goto out;
2597                 }
2598                 if (ipv4_is_local_multicast(fl4->daddr) ||
2599                     ipv4_is_lbcast(fl4->daddr)) {
2600                         if (!fl4->saddr)
2601                                 fl4->saddr = inet_select_addr(dev_out, 0,
2602                                                               RT_SCOPE_LINK);
2603                         goto make_route;
2604                 }
2605                 if (fl4->saddr) {
2606                         if (ipv4_is_multicast(fl4->daddr))
2607                                 fl4->saddr = inet_select_addr(dev_out, 0,
2608                                                               fl4->flowi4_scope);
2609                         else if (!fl4->daddr)
2610                                 fl4->saddr = inet_select_addr(dev_out, 0,
2611                                                               RT_SCOPE_HOST);
2612                 }
2613         }
2614
2615         if (!fl4->daddr) {
2616                 fl4->daddr = fl4->saddr;
2617                 if (!fl4->daddr)
2618                         fl4->daddr = fl4->saddr = htonl(INADDR_LOOPBACK);
2619                 dev_out = net->loopback_dev;
2620                 fl4->flowi4_oif = net->loopback_dev->ifindex;
2621                 res.type = RTN_LOCAL;
2622                 flags |= RTCF_LOCAL;
2623                 goto make_route;
2624         }
2625
2626         if (fib_lookup(net, fl4, &res)) {
2627                 res.fi = NULL;
2628                 res.table = NULL;
2629                 if (fl4->flowi4_oif) {
2630                         /* Apparently, routing tables are wrong. Assume,
2631                            that the destination is on link.
2632
2633                            WHY? DW.
2634                            Because we are allowed to send to iface
2635                            even if it has NO routes and NO assigned
2636                            addresses. When oif is specified, routing
2637                            tables are looked up with only one purpose:
2638                            to catch if destination is gatewayed, rather than
2639                            direct. Moreover, if MSG_DONTROUTE is set,
2640                            we send packet, ignoring both routing tables
2641                            and ifaddr state. --ANK
2642
2643
2644                            We could make it even if oif is unknown,
2645                            likely IPv6, but we do not.
2646                          */
2647
2648                         if (fl4->saddr == 0)
2649                                 fl4->saddr = inet_select_addr(dev_out, 0,
2650                                                               RT_SCOPE_LINK);
2651                         res.type = RTN_UNICAST;
2652                         goto make_route;
2653                 }
2654                 rth = ERR_PTR(-ENETUNREACH);
2655                 goto out;
2656         }
2657
2658         if (res.type == RTN_LOCAL) {
2659                 if (!fl4->saddr) {
2660                         if (res.fi->fib_prefsrc)
2661                                 fl4->saddr = res.fi->fib_prefsrc;
2662                         else
2663                                 fl4->saddr = fl4->daddr;
2664                 }
2665                 dev_out = net->loopback_dev;
2666                 fl4->flowi4_oif = dev_out->ifindex;
2667                 res.fi = NULL;
2668                 flags |= RTCF_LOCAL;
2669                 goto make_route;
2670         }
2671
2672 #ifdef CONFIG_IP_ROUTE_MULTIPATH
2673         if (res.fi->fib_nhs > 1 && fl4->flowi4_oif == 0)
2674                 fib_select_multipath(&res);
2675         else
2676 #endif
2677         if (!res.prefixlen &&
2678             res.table->tb_num_default > 1 &&
2679             res.type == RTN_UNICAST && !fl4->flowi4_oif)
2680                 fib_select_default(&res);
2681
2682         if (!fl4->saddr)
2683                 fl4->saddr = FIB_RES_PREFSRC(net, res);
2684
2685         dev_out = FIB_RES_DEV(res);
2686         fl4->flowi4_oif = dev_out->ifindex;
2687
2688
2689 make_route:
2690         rth = __mkroute_output(&res, fl4, orig_daddr, orig_saddr, orig_oif,
2691                                tos, dev_out, flags);
2692         if (!IS_ERR(rth)) {
2693                 unsigned int hash;
2694
2695                 hash = rt_hash(orig_daddr, orig_saddr, orig_oif,
2696                                rt_genid(dev_net(dev_out)));
2697                 rth = rt_intern_hash(hash, rth, NULL, orig_oif);
2698         }
2699
2700 out:
2701         rcu_read_unlock();
2702         return rth;
2703 }
2704
2705 struct rtable *__ip_route_output_key(struct net *net, struct flowi4 *flp4)
2706 {
2707         struct rtable *rth;
2708         unsigned int hash;
2709
2710         if (!rt_caching(net))
2711                 goto slow_output;
2712
2713         hash = rt_hash(flp4->daddr, flp4->saddr, flp4->flowi4_oif, rt_genid(net));
2714
2715         rcu_read_lock_bh();
2716         for (rth = rcu_dereference_bh(rt_hash_table[hash].chain); rth;
2717                 rth = rcu_dereference_bh(rth->dst.rt_next)) {
2718                 if (rth->rt_key_dst == flp4->daddr &&
2719                     rth->rt_key_src == flp4->saddr &&
2720                     rt_is_output_route(rth) &&
2721                     rth->rt_oif == flp4->flowi4_oif &&
2722                     rth->rt_mark == flp4->flowi4_mark &&
2723                     !((rth->rt_key_tos ^ flp4->flowi4_tos) &
2724                             (IPTOS_RT_MASK | RTO_ONLINK)) &&
2725                     net_eq(dev_net(rth->dst.dev), net) &&
2726                     !rt_is_expired(rth)) {
2727                         ipv4_validate_peer(rth);
2728                         dst_use(&rth->dst, jiffies);
2729                         RT_CACHE_STAT_INC(out_hit);
2730                         rcu_read_unlock_bh();
2731                         if (!flp4->saddr)
2732                                 flp4->saddr = rth->rt_src;
2733                         if (!flp4->daddr)
2734                                 flp4->daddr = rth->rt_dst;
2735                         return rth;
2736                 }
2737                 RT_CACHE_STAT_INC(out_hlist_search);
2738         }
2739         rcu_read_unlock_bh();
2740
2741 slow_output:
2742         return ip_route_output_slow(net, flp4);
2743 }
2744 EXPORT_SYMBOL_GPL(__ip_route_output_key);
2745
2746 static struct dst_entry *ipv4_blackhole_dst_check(struct dst_entry *dst, u32 cookie)
2747 {
2748         return NULL;
2749 }
2750
2751 static unsigned int ipv4_blackhole_mtu(const struct dst_entry *dst)
2752 {
2753         unsigned int mtu = dst_metric_raw(dst, RTAX_MTU);
2754
2755         return mtu ? : dst->dev->mtu;
2756 }
2757
2758 static void ipv4_rt_blackhole_update_pmtu(struct dst_entry *dst, u32 mtu)
2759 {
2760 }
2761
2762 static u32 *ipv4_rt_blackhole_cow_metrics(struct dst_entry *dst,
2763                                           unsigned long old)
2764 {
2765         return NULL;
2766 }
2767
2768 static struct dst_ops ipv4_dst_blackhole_ops = {
2769         .family                 =       AF_INET,
2770         .protocol               =       cpu_to_be16(ETH_P_IP),
2771         .destroy                =       ipv4_dst_destroy,
2772         .check                  =       ipv4_blackhole_dst_check,
2773         .mtu                    =       ipv4_blackhole_mtu,
2774         .default_advmss         =       ipv4_default_advmss,
2775         .update_pmtu            =       ipv4_rt_blackhole_update_pmtu,
2776         .cow_metrics            =       ipv4_rt_blackhole_cow_metrics,
2777         .neigh_lookup           =       ipv4_neigh_lookup,
2778 };
2779
2780 struct dst_entry *ipv4_blackhole_route(struct net *net, struct dst_entry *dst_orig)
2781 {
2782         struct rtable *rt = dst_alloc(&ipv4_dst_blackhole_ops, NULL, 1, 0, 0);
2783         struct rtable *ort = (struct rtable *) dst_orig;
2784
2785         if (rt) {
2786                 struct dst_entry *new = &rt->dst;
2787
2788                 new->__use = 1;
2789                 new->input = dst_discard;
2790                 new->output = dst_discard;
2791                 dst_copy_metrics(new, &ort->dst);
2792
2793                 new->dev = ort->dst.dev;
2794                 if (new->dev)
2795                         dev_hold(new->dev);
2796
2797                 rt->rt_key_dst = ort->rt_key_dst;
2798                 rt->rt_key_src = ort->rt_key_src;
2799                 rt->rt_key_tos = ort->rt_key_tos;
2800                 rt->rt_route_iif = ort->rt_route_iif;
2801                 rt->rt_iif = ort->rt_iif;
2802                 rt->rt_oif = ort->rt_oif;
2803                 rt->rt_mark = ort->rt_mark;
2804
2805                 rt->rt_genid = rt_genid(net);
2806                 rt->rt_flags = ort->rt_flags;
2807                 rt->rt_type = ort->rt_type;
2808                 rt->rt_dst = ort->rt_dst;
2809                 rt->rt_src = ort->rt_src;
2810                 rt->rt_gateway = ort->rt_gateway;
2811                 rt_transfer_peer(rt, ort);
2812                 rt->fi = ort->fi;
2813                 if (rt->fi)
2814                         atomic_inc(&rt->fi->fib_clntref);
2815
2816                 dst_free(new);
2817         }
2818
2819         dst_release(dst_orig);
2820
2821         return rt ? &rt->dst : ERR_PTR(-ENOMEM);
2822 }
2823
2824 struct rtable *ip_route_output_flow(struct net *net, struct flowi4 *flp4,
2825                                     struct sock *sk)
2826 {
2827         struct rtable *rt = __ip_route_output_key(net, flp4);
2828
2829         if (IS_ERR(rt))
2830                 return rt;
2831
2832         if (flp4->flowi4_proto)
2833                 rt = (struct rtable *) xfrm_lookup(net, &rt->dst,
2834                                                    flowi4_to_flowi(flp4),
2835                                                    sk, 0);
2836
2837         return rt;
2838 }
2839 EXPORT_SYMBOL_GPL(ip_route_output_flow);
2840
2841 static int rt_fill_info(struct net *net,
2842                         struct sk_buff *skb, u32 pid, u32 seq, int event,
2843                         int nowait, unsigned int flags)
2844 {
2845         struct rtable *rt = skb_rtable(skb);
2846         struct rtmsg *r;
2847         struct nlmsghdr *nlh;
2848         unsigned long expires = 0;
2849         u32 id = 0, ts = 0, tsage = 0, error;
2850
2851         nlh = nlmsg_put(skb, pid, seq, event, sizeof(*r), flags);
2852         if (nlh == NULL)
2853                 return -EMSGSIZE;
2854
2855         r = nlmsg_data(nlh);
2856         r->rtm_family    = AF_INET;
2857         r->rtm_dst_len  = 32;
2858         r->rtm_src_len  = 0;
2859         r->rtm_tos      = rt->rt_key_tos;
2860         r->rtm_table    = RT_TABLE_MAIN;
2861         if (nla_put_u32(skb, RTA_TABLE, RT_TABLE_MAIN))
2862                 goto nla_put_failure;
2863         r->rtm_type     = rt->rt_type;
2864         r->rtm_scope    = RT_SCOPE_UNIVERSE;
2865         r->rtm_protocol = RTPROT_UNSPEC;
2866         r->rtm_flags    = (rt->rt_flags & ~0xFFFF) | RTM_F_CLONED;
2867         if (rt->rt_flags & RTCF_NOTIFY)
2868                 r->rtm_flags |= RTM_F_NOTIFY;
2869
2870         if (nla_put_be32(skb, RTA_DST, rt->rt_dst))
2871                 goto nla_put_failure;
2872         if (rt->rt_key_src) {
2873                 r->rtm_src_len = 32;
2874                 if (nla_put_be32(skb, RTA_SRC, rt->rt_key_src))
2875                         goto nla_put_failure;
2876         }
2877         if (rt->dst.dev &&
2878             nla_put_u32(skb, RTA_OIF, rt->dst.dev->ifindex))
2879                 goto nla_put_failure;
2880 #ifdef CONFIG_IP_ROUTE_CLASSID
2881         if (rt->dst.tclassid &&
2882             nla_put_u32(skb, RTA_FLOW, rt->dst.tclassid))
2883                 goto nla_put_failure;
2884 #endif
2885         if (!rt_is_input_route(rt) &&
2886             rt->rt_src != rt->rt_key_src) {
2887                 if (nla_put_be32(skb, RTA_PREFSRC, rt->rt_src))
2888                         goto nla_put_failure;
2889         }
2890         if (rt->rt_dst != rt->rt_gateway &&
2891             nla_put_be32(skb, RTA_GATEWAY, rt->rt_gateway))
2892                 goto nla_put_failure;
2893
2894         if (rtnetlink_put_metrics(skb, dst_metrics_ptr(&rt->dst)) < 0)
2895                 goto nla_put_failure;
2896
2897         if (rt->rt_mark &&
2898             nla_put_be32(skb, RTA_MARK, rt->rt_mark))
2899                 goto nla_put_failure;
2900
2901         error = rt->dst.error;
2902         if (rt_has_peer(rt)) {
2903                 const struct inet_peer *peer = rt_peer_ptr(rt);
2904                 inet_peer_refcheck(peer);
2905                 id = atomic_read(&peer->ip_id_count) & 0xffff;
2906                 if (peer->tcp_ts_stamp) {
2907                         ts = peer->tcp_ts;
2908                         tsage = get_seconds() - peer->tcp_ts_stamp;
2909                 }
2910                 expires = ACCESS_ONCE(peer->pmtu_expires);
2911                 if (expires) {
2912                         if (time_before(jiffies, expires))
2913                                 expires -= jiffies;
2914                         else
2915                                 expires = 0;
2916                 }
2917         }
2918
2919         if (rt_is_input_route(rt)) {
2920 #ifdef CONFIG_IP_MROUTE
2921                 __be32 dst = rt->rt_dst;
2922
2923                 if (ipv4_is_multicast(dst) && !ipv4_is_local_multicast(dst) &&
2924                     IPV4_DEVCONF_ALL(net, MC_FORWARDING)) {
2925                         int err = ipmr_get_route(net, skb,
2926                                                  rt->rt_src, rt->rt_dst,
2927                                                  r, nowait);
2928                         if (err <= 0) {
2929                                 if (!nowait) {
2930                                         if (err == 0)
2931                                                 return 0;
2932                                         goto nla_put_failure;
2933                                 } else {
2934                                         if (err == -EMSGSIZE)
2935                                                 goto nla_put_failure;
2936                                         error = err;
2937                                 }
2938                         }
2939                 } else
2940 #endif
2941                         if (nla_put_u32(skb, RTA_IIF, rt->rt_iif))
2942                                 goto nla_put_failure;
2943         }
2944
2945         if (rtnl_put_cacheinfo(skb, &rt->dst, id, ts, tsage,
2946                                expires, error) < 0)
2947                 goto nla_put_failure;
2948
2949         return nlmsg_end(skb, nlh);
2950
2951 nla_put_failure:
2952         nlmsg_cancel(skb, nlh);
2953         return -EMSGSIZE;
2954 }
2955
2956 static int inet_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr *nlh, void *arg)
2957 {
2958         struct net *net = sock_net(in_skb->sk);
2959         struct rtmsg *rtm;
2960         struct nlattr *tb[RTA_MAX+1];
2961         struct rtable *rt = NULL;
2962         __be32 dst = 0;
2963         __be32 src = 0;
2964         u32 iif;
2965         int err;
2966         int mark;
2967         struct sk_buff *skb;
2968
2969         err = nlmsg_parse(nlh, sizeof(*rtm), tb, RTA_MAX, rtm_ipv4_policy);
2970         if (err < 0)
2971                 goto errout;
2972
2973         rtm = nlmsg_data(nlh);
2974
2975         skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
2976         if (skb == NULL) {
2977                 err = -ENOBUFS;
2978                 goto errout;
2979         }
2980
2981         /* Reserve room for dummy headers, this skb can pass
2982            through good chunk of routing engine.
2983          */
2984         skb_reset_mac_header(skb);
2985         skb_reset_network_header(skb);
2986
2987         /* Bugfix: need to give ip_route_input enough of an IP header to not gag. */
2988         ip_hdr(skb)->protocol = IPPROTO_ICMP;
2989         skb_reserve(skb, MAX_HEADER + sizeof(struct iphdr));
2990
2991         src = tb[RTA_SRC] ? nla_get_be32(tb[RTA_SRC]) : 0;
2992         dst = tb[RTA_DST] ? nla_get_be32(tb[RTA_DST]) : 0;
2993         iif = tb[RTA_IIF] ? nla_get_u32(tb[RTA_IIF]) : 0;
2994         mark = tb[RTA_MARK] ? nla_get_u32(tb[RTA_MARK]) : 0;
2995
2996         if (iif) {
2997                 struct net_device *dev;
2998
2999                 dev = __dev_get_by_index(net, iif);
3000                 if (dev == NULL) {
3001                         err = -ENODEV;
3002                         goto errout_free;
3003                 }
3004
3005                 skb->protocol   = htons(ETH_P_IP);
3006                 skb->dev        = dev;
3007                 skb->mark       = mark;
3008                 local_bh_disable();
3009                 err = ip_route_input(skb, dst, src, rtm->rtm_tos, dev);
3010                 local_bh_enable();
3011
3012                 rt = skb_rtable(skb);
3013                 if (err == 0 && rt->dst.error)
3014                         err = -rt->dst.error;
3015         } else {
3016                 struct flowi4 fl4 = {
3017                         .daddr = dst,
3018                         .saddr = src,
3019                         .flowi4_tos = rtm->rtm_tos,
3020                         .flowi4_oif = tb[RTA_OIF] ? nla_get_u32(tb[RTA_OIF]) : 0,
3021                         .flowi4_mark = mark,
3022                 };
3023                 rt = ip_route_output_key(net, &fl4);
3024
3025                 err = 0;
3026                 if (IS_ERR(rt))
3027                         err = PTR_ERR(rt);
3028         }
3029
3030         if (err)
3031                 goto errout_free;
3032
3033         skb_dst_set(skb, &rt->dst);
3034         if (rtm->rtm_flags & RTM_F_NOTIFY)
3035                 rt->rt_flags |= RTCF_NOTIFY;
3036
3037         err = rt_fill_info(net, skb, NETLINK_CB(in_skb).pid, nlh->nlmsg_seq,
3038                            RTM_NEWROUTE, 0, 0);
3039         if (err <= 0)
3040                 goto errout_free;
3041
3042         err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).pid);
3043 errout:
3044         return err;
3045
3046 errout_free:
3047         kfree_skb(skb);
3048         goto errout;
3049 }
3050
3051 int ip_rt_dump(struct sk_buff *skb,  struct netlink_callback *cb)
3052 {
3053         struct rtable *rt;
3054         int h, s_h;
3055         int idx, s_idx;
3056         struct net *net;
3057
3058         net = sock_net(skb->sk);
3059
3060         s_h = cb->args[0];
3061         if (s_h < 0)
3062                 s_h = 0;
3063         s_idx = idx = cb->args[1];
3064         for (h = s_h; h <= rt_hash_mask; h++, s_idx = 0) {
3065                 if (!rt_hash_table[h].chain)
3066                         continue;
3067                 rcu_read_lock_bh();
3068                 for (rt = rcu_dereference_bh(rt_hash_table[h].chain), idx = 0; rt;
3069                      rt = rcu_dereference_bh(rt->dst.rt_next), idx++) {
3070                         if (!net_eq(dev_net(rt->dst.dev), net) || idx < s_idx)
3071                                 continue;
3072                         if (rt_is_expired(rt))
3073                                 continue;
3074                         skb_dst_set_noref(skb, &rt->dst);
3075                         if (rt_fill_info(net, skb, NETLINK_CB(cb->skb).pid,
3076                                          cb->nlh->nlmsg_seq, RTM_NEWROUTE,
3077                                          1, NLM_F_MULTI) <= 0) {
3078                                 skb_dst_drop(skb);
3079                                 rcu_read_unlock_bh();
3080                                 goto done;
3081                         }
3082                         skb_dst_drop(skb);
3083                 }
3084                 rcu_read_unlock_bh();
3085         }
3086
3087 done:
3088         cb->args[0] = h;
3089         cb->args[1] = idx;
3090         return skb->len;
3091 }
3092
3093 void ip_rt_multicast_event(struct in_device *in_dev)
3094 {
3095         rt_cache_flush(dev_net(in_dev->dev), 0);
3096 }
3097
3098 #ifdef CONFIG_SYSCTL
3099 static int ipv4_sysctl_rtcache_flush(ctl_table *__ctl, int write,
3100                                         void __user *buffer,
3101                                         size_t *lenp, loff_t *ppos)
3102 {
3103         if (write) {
3104                 int flush_delay;
3105                 ctl_table ctl;
3106                 struct net *net;
3107
3108                 memcpy(&ctl, __ctl, sizeof(ctl));
3109                 ctl.data = &flush_delay;
3110                 proc_dointvec(&ctl, write, buffer, lenp, ppos);
3111
3112                 net = (struct net *)__ctl->extra1;
3113                 rt_cache_flush(net, flush_delay);
3114                 return 0;
3115         }
3116
3117         return -EINVAL;
3118 }
3119
3120 static ctl_table ipv4_route_table[] = {
3121         {
3122                 .procname       = "gc_thresh",
3123                 .data           = &ipv4_dst_ops.gc_thresh,
3124                 .maxlen         = sizeof(int),
3125                 .mode           = 0644,
3126                 .proc_handler   = proc_dointvec,
3127         },
3128         {
3129                 .procname       = "max_size",
3130                 .data           = &ip_rt_max_size,
3131                 .maxlen         = sizeof(int),
3132                 .mode           = 0644,
3133                 .proc_handler   = proc_dointvec,
3134         },
3135         {
3136                 /*  Deprecated. Use gc_min_interval_ms */
3137
3138                 .procname       = "gc_min_interval",
3139                 .data           = &ip_rt_gc_min_interval,
3140                 .maxlen         = sizeof(int),
3141                 .mode           = 0644,
3142                 .proc_handler   = proc_dointvec_jiffies,
3143         },
3144         {
3145                 .procname       = "gc_min_interval_ms",
3146                 .data           = &ip_rt_gc_min_interval,
3147                 .maxlen         = sizeof(int),
3148                 .mode           = 0644,
3149                 .proc_handler   = proc_dointvec_ms_jiffies,
3150         },
3151         {
3152                 .procname       = "gc_timeout",
3153                 .data           = &ip_rt_gc_timeout,
3154                 .maxlen         = sizeof(int),
3155                 .mode           = 0644,
3156                 .proc_handler   = proc_dointvec_jiffies,
3157         },
3158         {
3159                 .procname       = "gc_interval",
3160                 .data           = &ip_rt_gc_interval,
3161                 .maxlen         = sizeof(int),
3162                 .mode           = 0644,
3163                 .proc_handler   = proc_dointvec_jiffies,
3164         },
3165         {
3166                 .procname       = "redirect_load",
3167                 .data           = &ip_rt_redirect_load,
3168                 .maxlen         = sizeof(int),
3169                 .mode           = 0644,
3170                 .proc_handler   = proc_dointvec,
3171         },
3172         {
3173                 .procname       = "redirect_number",
3174                 .data           = &ip_rt_redirect_number,
3175                 .maxlen         = sizeof(int),
3176                 .mode           = 0644,
3177                 .proc_handler   = proc_dointvec,
3178         },
3179         {
3180                 .procname       = "redirect_silence",
3181                 .data           = &ip_rt_redirect_silence,
3182                 .maxlen         = sizeof(int),
3183                 .mode           = 0644,
3184                 .proc_handler   = proc_dointvec,
3185         },
3186         {
3187                 .procname       = "error_cost",
3188                 .data           = &ip_rt_error_cost,
3189                 .maxlen         = sizeof(int),
3190                 .mode           = 0644,
3191                 .proc_handler   = proc_dointvec,
3192         },
3193         {
3194                 .procname       = "error_burst",
3195                 .data           = &ip_rt_error_burst,
3196                 .maxlen         = sizeof(int),
3197                 .mode           = 0644,
3198                 .proc_handler   = proc_dointvec,
3199         },
3200         {
3201                 .procname       = "gc_elasticity",
3202                 .data           = &ip_rt_gc_elasticity,
3203                 .maxlen         = sizeof(int),
3204                 .mode           = 0644,
3205                 .proc_handler   = proc_dointvec,
3206         },
3207         {
3208                 .procname       = "mtu_expires",
3209                 .data           = &ip_rt_mtu_expires,
3210                 .maxlen         = sizeof(int),
3211                 .mode           = 0644,
3212                 .proc_handler   = proc_dointvec_jiffies,
3213         },
3214         {
3215                 .procname       = "min_pmtu",
3216                 .data           = &ip_rt_min_pmtu,
3217                 .maxlen         = sizeof(int),
3218                 .mode           = 0644,
3219                 .proc_handler   = proc_dointvec,
3220         },
3221         {
3222                 .procname       = "min_adv_mss",
3223                 .data           = &ip_rt_min_advmss,
3224                 .maxlen         = sizeof(int),
3225                 .mode           = 0644,
3226                 .proc_handler   = proc_dointvec,
3227         },
3228         { }
3229 };
3230
3231 static struct ctl_table ipv4_route_flush_table[] = {
3232         {
3233                 .procname       = "flush",
3234                 .maxlen         = sizeof(int),
3235                 .mode           = 0200,
3236                 .proc_handler   = ipv4_sysctl_rtcache_flush,
3237         },
3238         { },
3239 };
3240
3241 static __net_init int sysctl_route_net_init(struct net *net)
3242 {
3243         struct ctl_table *tbl;
3244
3245         tbl = ipv4_route_flush_table;
3246         if (!net_eq(net, &init_net)) {
3247                 tbl = kmemdup(tbl, sizeof(ipv4_route_flush_table), GFP_KERNEL);
3248                 if (tbl == NULL)
3249                         goto err_dup;
3250         }
3251         tbl[0].extra1 = net;
3252
3253         net->ipv4.route_hdr = register_net_sysctl(net, "net/ipv4/route", tbl);
3254         if (net->ipv4.route_hdr == NULL)
3255                 goto err_reg;
3256         return 0;
3257
3258 err_reg:
3259         if (tbl != ipv4_route_flush_table)
3260                 kfree(tbl);
3261 err_dup:
3262         return -ENOMEM;
3263 }
3264
3265 static __net_exit void sysctl_route_net_exit(struct net *net)
3266 {
3267         struct ctl_table *tbl;
3268
3269         tbl = net->ipv4.route_hdr->ctl_table_arg;
3270         unregister_net_sysctl_table(net->ipv4.route_hdr);
3271         BUG_ON(tbl == ipv4_route_flush_table);
3272         kfree(tbl);
3273 }
3274
3275 static __net_initdata struct pernet_operations sysctl_route_ops = {
3276         .init = sysctl_route_net_init,
3277         .exit = sysctl_route_net_exit,
3278 };
3279 #endif
3280
3281 static __net_init int rt_genid_init(struct net *net)
3282 {
3283         get_random_bytes(&net->ipv4.rt_genid,
3284                          sizeof(net->ipv4.rt_genid));
3285         get_random_bytes(&net->ipv4.dev_addr_genid,
3286                          sizeof(net->ipv4.dev_addr_genid));
3287         return 0;
3288 }
3289
3290 static __net_initdata struct pernet_operations rt_genid_ops = {
3291         .init = rt_genid_init,
3292 };
3293
3294 static int __net_init ipv4_inetpeer_init(struct net *net)
3295 {
3296         struct inet_peer_base *bp = kmalloc(sizeof(*bp), GFP_KERNEL);
3297
3298         if (!bp)
3299                 return -ENOMEM;
3300         inet_peer_base_init(bp);
3301         net->ipv4.peers = bp;
3302         return 0;
3303 }
3304
3305 static void __net_exit ipv4_inetpeer_exit(struct net *net)
3306 {
3307         struct inet_peer_base *bp = net->ipv4.peers;
3308
3309         net->ipv4.peers = NULL;
3310         inetpeer_invalidate_tree(bp);
3311         kfree(bp);
3312 }
3313
3314 static __net_initdata struct pernet_operations ipv4_inetpeer_ops = {
3315         .init   =       ipv4_inetpeer_init,
3316         .exit   =       ipv4_inetpeer_exit,
3317 };
3318
3319 #ifdef CONFIG_IP_ROUTE_CLASSID
3320 struct ip_rt_acct __percpu *ip_rt_acct __read_mostly;
3321 #endif /* CONFIG_IP_ROUTE_CLASSID */
3322
3323 static __initdata unsigned long rhash_entries;
3324 static int __init set_rhash_entries(char *str)
3325 {
3326         ssize_t ret;
3327
3328         if (!str)
3329                 return 0;
3330
3331         ret = kstrtoul(str, 0, &rhash_entries);
3332         if (ret)
3333                 return 0;
3334
3335         return 1;
3336 }
3337 __setup("rhash_entries=", set_rhash_entries);
3338
3339 int __init ip_rt_init(void)
3340 {
3341         int rc = 0;
3342
3343 #ifdef CONFIG_IP_ROUTE_CLASSID
3344         ip_rt_acct = __alloc_percpu(256 * sizeof(struct ip_rt_acct), __alignof__(struct ip_rt_acct));
3345         if (!ip_rt_acct)
3346                 panic("IP: failed to allocate ip_rt_acct\n");
3347 #endif
3348
3349         ipv4_dst_ops.kmem_cachep =
3350                 kmem_cache_create("ip_dst_cache", sizeof(struct rtable), 0,
3351                                   SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
3352
3353         ipv4_dst_blackhole_ops.kmem_cachep = ipv4_dst_ops.kmem_cachep;
3354
3355         if (dst_entries_init(&ipv4_dst_ops) < 0)
3356                 panic("IP: failed to allocate ipv4_dst_ops counter\n");
3357
3358         if (dst_entries_init(&ipv4_dst_blackhole_ops) < 0)
3359                 panic("IP: failed to allocate ipv4_dst_blackhole_ops counter\n");
3360
3361         rt_hash_table = (struct rt_hash_bucket *)
3362                 alloc_large_system_hash("IP route cache",
3363                                         sizeof(struct rt_hash_bucket),
3364                                         rhash_entries,
3365                                         (totalram_pages >= 128 * 1024) ?
3366                                         15 : 17,
3367                                         0,
3368                                         &rt_hash_log,
3369                                         &rt_hash_mask,
3370                                         0,
3371                                         rhash_entries ? 0 : 512 * 1024);
3372         memset(rt_hash_table, 0, (rt_hash_mask + 1) * sizeof(struct rt_hash_bucket));
3373         rt_hash_lock_init();
3374
3375         ipv4_dst_ops.gc_thresh = (rt_hash_mask + 1);
3376         ip_rt_max_size = (rt_hash_mask + 1) * 16;
3377
3378         devinet_init();
3379         ip_fib_init();
3380
3381         INIT_DELAYED_WORK_DEFERRABLE(&expires_work, rt_worker_func);
3382         expires_ljiffies = jiffies;
3383         schedule_delayed_work(&expires_work,
3384                 net_random() % ip_rt_gc_interval + ip_rt_gc_interval);
3385
3386         if (ip_rt_proc_init())
3387                 pr_err("Unable to create route proc files\n");
3388 #ifdef CONFIG_XFRM
3389         xfrm_init();
3390         xfrm4_init(ip_rt_max_size);
3391 #endif
3392         rtnl_register(PF_INET, RTM_GETROUTE, inet_rtm_getroute, NULL, NULL);
3393
3394 #ifdef CONFIG_SYSCTL
3395         register_pernet_subsys(&sysctl_route_ops);
3396 #endif
3397         register_pernet_subsys(&rt_genid_ops);
3398         register_pernet_subsys(&ipv4_inetpeer_ops);
3399         return rc;
3400 }
3401
3402 #ifdef CONFIG_SYSCTL
3403 /*
3404  * We really need to sanitize the damn ipv4 init order, then all
3405  * this nonsense will go away.
3406  */
3407 void __init ip_static_sysctl_init(void)
3408 {
3409         register_net_sysctl(&init_net, "net/ipv4/route", ipv4_route_table);
3410 }
3411 #endif