[SK_BUFF]: Introduce skb_copy_from_linear_data{_offset}
[linux-3.10.git] / net / ipv4 / ip_output.c
1 /*
2  * INET         An implementation of the TCP/IP protocol suite for the LINUX
3  *              operating system.  INET is implemented using the  BSD Socket
4  *              interface as the means of communication with the user level.
5  *
6  *              The Internet Protocol (IP) output module.
7  *
8  * Version:     $Id: ip_output.c,v 1.100 2002/02/01 22:01:03 davem Exp $
9  *
10  * Authors:     Ross Biro
11  *              Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12  *              Donald Becker, <becker@super.org>
13  *              Alan Cox, <Alan.Cox@linux.org>
14  *              Richard Underwood
15  *              Stefan Becker, <stefanb@yello.ping.de>
16  *              Jorge Cwik, <jorge@laser.satlink.net>
17  *              Arnt Gulbrandsen, <agulbra@nvg.unit.no>
18  *              Hirokazu Takahashi, <taka@valinux.co.jp>
19  *
20  *      See ip_input.c for original log
21  *
22  *      Fixes:
23  *              Alan Cox        :       Missing nonblock feature in ip_build_xmit.
24  *              Mike Kilburn    :       htons() missing in ip_build_xmit.
25  *              Bradford Johnson:       Fix faulty handling of some frames when
26  *                                      no route is found.
27  *              Alexander Demenshin:    Missing sk/skb free in ip_queue_xmit
28  *                                      (in case if packet not accepted by
29  *                                      output firewall rules)
30  *              Mike McLagan    :       Routing by source
31  *              Alexey Kuznetsov:       use new route cache
32  *              Andi Kleen:             Fix broken PMTU recovery and remove
33  *                                      some redundant tests.
34  *      Vitaly E. Lavrov        :       Transparent proxy revived after year coma.
35  *              Andi Kleen      :       Replace ip_reply with ip_send_reply.
36  *              Andi Kleen      :       Split fast and slow ip_build_xmit path
37  *                                      for decreased register pressure on x86
38  *                                      and more readibility.
39  *              Marc Boucher    :       When call_out_firewall returns FW_QUEUE,
40  *                                      silently drop skb instead of failing with -EPERM.
41  *              Detlev Wengorz  :       Copy protocol for fragments.
42  *              Hirokazu Takahashi:     HW checksumming for outgoing UDP
43  *                                      datagrams.
44  *              Hirokazu Takahashi:     sendfile() on UDP works now.
45  */
46
47 #include <asm/uaccess.h>
48 #include <asm/system.h>
49 #include <linux/module.h>
50 #include <linux/types.h>
51 #include <linux/kernel.h>
52 #include <linux/mm.h>
53 #include <linux/string.h>
54 #include <linux/errno.h>
55 #include <linux/highmem.h>
56
57 #include <linux/socket.h>
58 #include <linux/sockios.h>
59 #include <linux/in.h>
60 #include <linux/inet.h>
61 #include <linux/netdevice.h>
62 #include <linux/etherdevice.h>
63 #include <linux/proc_fs.h>
64 #include <linux/stat.h>
65 #include <linux/init.h>
66
67 #include <net/snmp.h>
68 #include <net/ip.h>
69 #include <net/protocol.h>
70 #include <net/route.h>
71 #include <net/xfrm.h>
72 #include <linux/skbuff.h>
73 #include <net/sock.h>
74 #include <net/arp.h>
75 #include <net/icmp.h>
76 #include <net/checksum.h>
77 #include <net/inetpeer.h>
78 #include <net/checksum.h>
79 #include <linux/igmp.h>
80 #include <linux/netfilter_ipv4.h>
81 #include <linux/netfilter_bridge.h>
82 #include <linux/mroute.h>
83 #include <linux/netlink.h>
84 #include <linux/tcp.h>
85
86 int sysctl_ip_default_ttl __read_mostly = IPDEFTTL;
87
88 /* Generate a checksum for an outgoing IP datagram. */
89 __inline__ void ip_send_check(struct iphdr *iph)
90 {
91         iph->check = 0;
92         iph->check = ip_fast_csum((unsigned char *)iph, iph->ihl);
93 }
94
95 /* dev_loopback_xmit for use with netfilter. */
96 static int ip_dev_loopback_xmit(struct sk_buff *newskb)
97 {
98         skb_reset_mac_header(newskb);
99         __skb_pull(newskb, skb_network_offset(newskb));
100         newskb->pkt_type = PACKET_LOOPBACK;
101         newskb->ip_summed = CHECKSUM_UNNECESSARY;
102         BUG_TRAP(newskb->dst);
103         netif_rx(newskb);
104         return 0;
105 }
106
107 static inline int ip_select_ttl(struct inet_sock *inet, struct dst_entry *dst)
108 {
109         int ttl = inet->uc_ttl;
110
111         if (ttl < 0)
112                 ttl = dst_metric(dst, RTAX_HOPLIMIT);
113         return ttl;
114 }
115
116 /*
117  *              Add an ip header to a skbuff and send it out.
118  *
119  */
120 int ip_build_and_send_pkt(struct sk_buff *skb, struct sock *sk,
121                           __be32 saddr, __be32 daddr, struct ip_options *opt)
122 {
123         struct inet_sock *inet = inet_sk(sk);
124         struct rtable *rt = (struct rtable *)skb->dst;
125         struct iphdr *iph;
126
127         /* Build the IP header. */
128         skb_push(skb, sizeof(struct iphdr) + (opt ? opt->optlen : 0));
129         skb_reset_network_header(skb);
130         iph = ip_hdr(skb);
131         iph->version  = 4;
132         iph->ihl      = 5;
133         iph->tos      = inet->tos;
134         if (ip_dont_fragment(sk, &rt->u.dst))
135                 iph->frag_off = htons(IP_DF);
136         else
137                 iph->frag_off = 0;
138         iph->ttl      = ip_select_ttl(inet, &rt->u.dst);
139         iph->daddr    = rt->rt_dst;
140         iph->saddr    = rt->rt_src;
141         iph->protocol = sk->sk_protocol;
142         iph->tot_len  = htons(skb->len);
143         ip_select_ident(iph, &rt->u.dst, sk);
144
145         if (opt && opt->optlen) {
146                 iph->ihl += opt->optlen>>2;
147                 ip_options_build(skb, opt, daddr, rt, 0);
148         }
149         ip_send_check(iph);
150
151         skb->priority = sk->sk_priority;
152
153         /* Send it out. */
154         return NF_HOOK(PF_INET, NF_IP_LOCAL_OUT, skb, NULL, rt->u.dst.dev,
155                        dst_output);
156 }
157
158 EXPORT_SYMBOL_GPL(ip_build_and_send_pkt);
159
160 static inline int ip_finish_output2(struct sk_buff *skb)
161 {
162         struct dst_entry *dst = skb->dst;
163         struct net_device *dev = dst->dev;
164         int hh_len = LL_RESERVED_SPACE(dev);
165
166         /* Be paranoid, rather than too clever. */
167         if (unlikely(skb_headroom(skb) < hh_len && dev->hard_header)) {
168                 struct sk_buff *skb2;
169
170                 skb2 = skb_realloc_headroom(skb, LL_RESERVED_SPACE(dev));
171                 if (skb2 == NULL) {
172                         kfree_skb(skb);
173                         return -ENOMEM;
174                 }
175                 if (skb->sk)
176                         skb_set_owner_w(skb2, skb->sk);
177                 kfree_skb(skb);
178                 skb = skb2;
179         }
180
181         if (dst->hh)
182                 return neigh_hh_output(dst->hh, skb);
183         else if (dst->neighbour)
184                 return dst->neighbour->output(skb);
185
186         if (net_ratelimit())
187                 printk(KERN_DEBUG "ip_finish_output2: No header cache and no neighbour!\n");
188         kfree_skb(skb);
189         return -EINVAL;
190 }
191
192 static inline int ip_finish_output(struct sk_buff *skb)
193 {
194 #if defined(CONFIG_NETFILTER) && defined(CONFIG_XFRM)
195         /* Policy lookup after SNAT yielded a new policy */
196         if (skb->dst->xfrm != NULL) {
197                 IPCB(skb)->flags |= IPSKB_REROUTED;
198                 return dst_output(skb);
199         }
200 #endif
201         if (skb->len > dst_mtu(skb->dst) && !skb_is_gso(skb))
202                 return ip_fragment(skb, ip_finish_output2);
203         else
204                 return ip_finish_output2(skb);
205 }
206
207 int ip_mc_output(struct sk_buff *skb)
208 {
209         struct sock *sk = skb->sk;
210         struct rtable *rt = (struct rtable*)skb->dst;
211         struct net_device *dev = rt->u.dst.dev;
212
213         /*
214          *      If the indicated interface is up and running, send the packet.
215          */
216         IP_INC_STATS(IPSTATS_MIB_OUTREQUESTS);
217
218         skb->dev = dev;
219         skb->protocol = htons(ETH_P_IP);
220
221         /*
222          *      Multicasts are looped back for other local users
223          */
224
225         if (rt->rt_flags&RTCF_MULTICAST) {
226                 if ((!sk || inet_sk(sk)->mc_loop)
227 #ifdef CONFIG_IP_MROUTE
228                 /* Small optimization: do not loopback not local frames,
229                    which returned after forwarding; they will be  dropped
230                    by ip_mr_input in any case.
231                    Note, that local frames are looped back to be delivered
232                    to local recipients.
233
234                    This check is duplicated in ip_mr_input at the moment.
235                  */
236                     && ((rt->rt_flags&RTCF_LOCAL) || !(IPCB(skb)->flags&IPSKB_FORWARDED))
237 #endif
238                 ) {
239                         struct sk_buff *newskb = skb_clone(skb, GFP_ATOMIC);
240                         if (newskb)
241                                 NF_HOOK(PF_INET, NF_IP_POST_ROUTING, newskb, NULL,
242                                         newskb->dev,
243                                         ip_dev_loopback_xmit);
244                 }
245
246                 /* Multicasts with ttl 0 must not go beyond the host */
247
248                 if (ip_hdr(skb)->ttl == 0) {
249                         kfree_skb(skb);
250                         return 0;
251                 }
252         }
253
254         if (rt->rt_flags&RTCF_BROADCAST) {
255                 struct sk_buff *newskb = skb_clone(skb, GFP_ATOMIC);
256                 if (newskb)
257                         NF_HOOK(PF_INET, NF_IP_POST_ROUTING, newskb, NULL,
258                                 newskb->dev, ip_dev_loopback_xmit);
259         }
260
261         return NF_HOOK_COND(PF_INET, NF_IP_POST_ROUTING, skb, NULL, skb->dev,
262                             ip_finish_output,
263                             !(IPCB(skb)->flags & IPSKB_REROUTED));
264 }
265
266 int ip_output(struct sk_buff *skb)
267 {
268         struct net_device *dev = skb->dst->dev;
269
270         IP_INC_STATS(IPSTATS_MIB_OUTREQUESTS);
271
272         skb->dev = dev;
273         skb->protocol = htons(ETH_P_IP);
274
275         return NF_HOOK_COND(PF_INET, NF_IP_POST_ROUTING, skb, NULL, dev,
276                             ip_finish_output,
277                             !(IPCB(skb)->flags & IPSKB_REROUTED));
278 }
279
280 int ip_queue_xmit(struct sk_buff *skb, int ipfragok)
281 {
282         struct sock *sk = skb->sk;
283         struct inet_sock *inet = inet_sk(sk);
284         struct ip_options *opt = inet->opt;
285         struct rtable *rt;
286         struct iphdr *iph;
287
288         /* Skip all of this if the packet is already routed,
289          * f.e. by something like SCTP.
290          */
291         rt = (struct rtable *) skb->dst;
292         if (rt != NULL)
293                 goto packet_routed;
294
295         /* Make sure we can route this packet. */
296         rt = (struct rtable *)__sk_dst_check(sk, 0);
297         if (rt == NULL) {
298                 __be32 daddr;
299
300                 /* Use correct destination address if we have options. */
301                 daddr = inet->daddr;
302                 if(opt && opt->srr)
303                         daddr = opt->faddr;
304
305                 {
306                         struct flowi fl = { .oif = sk->sk_bound_dev_if,
307                                             .nl_u = { .ip4_u =
308                                                       { .daddr = daddr,
309                                                         .saddr = inet->saddr,
310                                                         .tos = RT_CONN_FLAGS(sk) } },
311                                             .proto = sk->sk_protocol,
312                                             .uli_u = { .ports =
313                                                        { .sport = inet->sport,
314                                                          .dport = inet->dport } } };
315
316                         /* If this fails, retransmit mechanism of transport layer will
317                          * keep trying until route appears or the connection times
318                          * itself out.
319                          */
320                         security_sk_classify_flow(sk, &fl);
321                         if (ip_route_output_flow(&rt, &fl, sk, 0))
322                                 goto no_route;
323                 }
324                 sk_setup_caps(sk, &rt->u.dst);
325         }
326         skb->dst = dst_clone(&rt->u.dst);
327
328 packet_routed:
329         if (opt && opt->is_strictroute && rt->rt_dst != rt->rt_gateway)
330                 goto no_route;
331
332         /* OK, we know where to send it, allocate and build IP header. */
333         skb_push(skb, sizeof(struct iphdr) + (opt ? opt->optlen : 0));
334         skb_reset_network_header(skb);
335         iph = ip_hdr(skb);
336         *((__be16 *)iph) = htons((4 << 12) | (5 << 8) | (inet->tos & 0xff));
337         iph->tot_len = htons(skb->len);
338         if (ip_dont_fragment(sk, &rt->u.dst) && !ipfragok)
339                 iph->frag_off = htons(IP_DF);
340         else
341                 iph->frag_off = 0;
342         iph->ttl      = ip_select_ttl(inet, &rt->u.dst);
343         iph->protocol = sk->sk_protocol;
344         iph->saddr    = rt->rt_src;
345         iph->daddr    = rt->rt_dst;
346         /* Transport layer set skb->h.foo itself. */
347
348         if (opt && opt->optlen) {
349                 iph->ihl += opt->optlen >> 2;
350                 ip_options_build(skb, opt, inet->daddr, rt, 0);
351         }
352
353         ip_select_ident_more(iph, &rt->u.dst, sk,
354                              (skb_shinfo(skb)->gso_segs ?: 1) - 1);
355
356         /* Add an IP checksum. */
357         ip_send_check(iph);
358
359         skb->priority = sk->sk_priority;
360
361         return NF_HOOK(PF_INET, NF_IP_LOCAL_OUT, skb, NULL, rt->u.dst.dev,
362                        dst_output);
363
364 no_route:
365         IP_INC_STATS(IPSTATS_MIB_OUTNOROUTES);
366         kfree_skb(skb);
367         return -EHOSTUNREACH;
368 }
369
370
371 static void ip_copy_metadata(struct sk_buff *to, struct sk_buff *from)
372 {
373         to->pkt_type = from->pkt_type;
374         to->priority = from->priority;
375         to->protocol = from->protocol;
376         dst_release(to->dst);
377         to->dst = dst_clone(from->dst);
378         to->dev = from->dev;
379         to->mark = from->mark;
380
381         /* Copy the flags to each fragment. */
382         IPCB(to)->flags = IPCB(from)->flags;
383
384 #ifdef CONFIG_NET_SCHED
385         to->tc_index = from->tc_index;
386 #endif
387         nf_copy(to, from);
388 #if defined(CONFIG_IP_VS) || defined(CONFIG_IP_VS_MODULE)
389         to->ipvs_property = from->ipvs_property;
390 #endif
391         skb_copy_secmark(to, from);
392 }
393
394 /*
395  *      This IP datagram is too large to be sent in one piece.  Break it up into
396  *      smaller pieces (each of size equal to IP header plus
397  *      a block of the data of the original IP data part) that will yet fit in a
398  *      single device frame, and queue such a frame for sending.
399  */
400
401 int ip_fragment(struct sk_buff *skb, int (*output)(struct sk_buff*))
402 {
403         struct iphdr *iph;
404         int raw = 0;
405         int ptr;
406         struct net_device *dev;
407         struct sk_buff *skb2;
408         unsigned int mtu, hlen, left, len, ll_rs, pad;
409         int offset;
410         __be16 not_last_frag;
411         struct rtable *rt = (struct rtable*)skb->dst;
412         int err = 0;
413
414         dev = rt->u.dst.dev;
415
416         /*
417          *      Point into the IP datagram header.
418          */
419
420         iph = ip_hdr(skb);
421
422         if (unlikely((iph->frag_off & htons(IP_DF)) && !skb->local_df)) {
423                 IP_INC_STATS(IPSTATS_MIB_FRAGFAILS);
424                 icmp_send(skb, ICMP_DEST_UNREACH, ICMP_FRAG_NEEDED,
425                           htonl(dst_mtu(&rt->u.dst)));
426                 kfree_skb(skb);
427                 return -EMSGSIZE;
428         }
429
430         /*
431          *      Setup starting values.
432          */
433
434         hlen = iph->ihl * 4;
435         mtu = dst_mtu(&rt->u.dst) - hlen;       /* Size of data space */
436         IPCB(skb)->flags |= IPSKB_FRAG_COMPLETE;
437
438         /* When frag_list is given, use it. First, check its validity:
439          * some transformers could create wrong frag_list or break existing
440          * one, it is not prohibited. In this case fall back to copying.
441          *
442          * LATER: this step can be merged to real generation of fragments,
443          * we can switch to copy when see the first bad fragment.
444          */
445         if (skb_shinfo(skb)->frag_list) {
446                 struct sk_buff *frag;
447                 int first_len = skb_pagelen(skb);
448
449                 if (first_len - hlen > mtu ||
450                     ((first_len - hlen) & 7) ||
451                     (iph->frag_off & htons(IP_MF|IP_OFFSET)) ||
452                     skb_cloned(skb))
453                         goto slow_path;
454
455                 for (frag = skb_shinfo(skb)->frag_list; frag; frag = frag->next) {
456                         /* Correct geometry. */
457                         if (frag->len > mtu ||
458                             ((frag->len & 7) && frag->next) ||
459                             skb_headroom(frag) < hlen)
460                             goto slow_path;
461
462                         /* Partially cloned skb? */
463                         if (skb_shared(frag))
464                                 goto slow_path;
465
466                         BUG_ON(frag->sk);
467                         if (skb->sk) {
468                                 sock_hold(skb->sk);
469                                 frag->sk = skb->sk;
470                                 frag->destructor = sock_wfree;
471                                 skb->truesize -= frag->truesize;
472                         }
473                 }
474
475                 /* Everything is OK. Generate! */
476
477                 err = 0;
478                 offset = 0;
479                 frag = skb_shinfo(skb)->frag_list;
480                 skb_shinfo(skb)->frag_list = NULL;
481                 skb->data_len = first_len - skb_headlen(skb);
482                 skb->len = first_len;
483                 iph->tot_len = htons(first_len);
484                 iph->frag_off = htons(IP_MF);
485                 ip_send_check(iph);
486
487                 for (;;) {
488                         /* Prepare header of the next frame,
489                          * before previous one went down. */
490                         if (frag) {
491                                 frag->ip_summed = CHECKSUM_NONE;
492                                 skb_reset_transport_header(frag);
493                                 __skb_push(frag, hlen);
494                                 skb_reset_network_header(frag);
495                                 memcpy(skb_network_header(frag), iph, hlen);
496                                 iph = ip_hdr(frag);
497                                 iph->tot_len = htons(frag->len);
498                                 ip_copy_metadata(frag, skb);
499                                 if (offset == 0)
500                                         ip_options_fragment(frag);
501                                 offset += skb->len - hlen;
502                                 iph->frag_off = htons(offset>>3);
503                                 if (frag->next != NULL)
504                                         iph->frag_off |= htons(IP_MF);
505                                 /* Ready, complete checksum */
506                                 ip_send_check(iph);
507                         }
508
509                         err = output(skb);
510
511                         if (!err)
512                                 IP_INC_STATS(IPSTATS_MIB_FRAGCREATES);
513                         if (err || !frag)
514                                 break;
515
516                         skb = frag;
517                         frag = skb->next;
518                         skb->next = NULL;
519                 }
520
521                 if (err == 0) {
522                         IP_INC_STATS(IPSTATS_MIB_FRAGOKS);
523                         return 0;
524                 }
525
526                 while (frag) {
527                         skb = frag->next;
528                         kfree_skb(frag);
529                         frag = skb;
530                 }
531                 IP_INC_STATS(IPSTATS_MIB_FRAGFAILS);
532                 return err;
533         }
534
535 slow_path:
536         left = skb->len - hlen;         /* Space per frame */
537         ptr = raw + hlen;               /* Where to start from */
538
539         /* for bridged IP traffic encapsulated inside f.e. a vlan header,
540          * we need to make room for the encapsulating header
541          */
542         pad = nf_bridge_pad(skb);
543         ll_rs = LL_RESERVED_SPACE_EXTRA(rt->u.dst.dev, pad);
544         mtu -= pad;
545
546         /*
547          *      Fragment the datagram.
548          */
549
550         offset = (ntohs(iph->frag_off) & IP_OFFSET) << 3;
551         not_last_frag = iph->frag_off & htons(IP_MF);
552
553         /*
554          *      Keep copying data until we run out.
555          */
556
557         while (left > 0) {
558                 len = left;
559                 /* IF: it doesn't fit, use 'mtu' - the data space left */
560                 if (len > mtu)
561                         len = mtu;
562                 /* IF: we are not sending upto and including the packet end
563                    then align the next start on an eight byte boundary */
564                 if (len < left) {
565                         len &= ~7;
566                 }
567                 /*
568                  *      Allocate buffer.
569                  */
570
571                 if ((skb2 = alloc_skb(len+hlen+ll_rs, GFP_ATOMIC)) == NULL) {
572                         NETDEBUG(KERN_INFO "IP: frag: no memory for new fragment!\n");
573                         err = -ENOMEM;
574                         goto fail;
575                 }
576
577                 /*
578                  *      Set up data on packet
579                  */
580
581                 ip_copy_metadata(skb2, skb);
582                 skb_reserve(skb2, ll_rs);
583                 skb_put(skb2, len + hlen);
584                 skb_reset_network_header(skb2);
585                 skb2->transport_header = skb2->network_header + hlen;
586
587                 /*
588                  *      Charge the memory for the fragment to any owner
589                  *      it might possess
590                  */
591
592                 if (skb->sk)
593                         skb_set_owner_w(skb2, skb->sk);
594
595                 /*
596                  *      Copy the packet header into the new buffer.
597                  */
598
599                 skb_copy_from_linear_data(skb, skb_network_header(skb2), hlen);
600
601                 /*
602                  *      Copy a block of the IP datagram.
603                  */
604                 if (skb_copy_bits(skb, ptr, skb_transport_header(skb2), len))
605                         BUG();
606                 left -= len;
607
608                 /*
609                  *      Fill in the new header fields.
610                  */
611                 iph = ip_hdr(skb2);
612                 iph->frag_off = htons((offset >> 3));
613
614                 /* ANK: dirty, but effective trick. Upgrade options only if
615                  * the segment to be fragmented was THE FIRST (otherwise,
616                  * options are already fixed) and make it ONCE
617                  * on the initial skb, so that all the following fragments
618                  * will inherit fixed options.
619                  */
620                 if (offset == 0)
621                         ip_options_fragment(skb);
622
623                 /*
624                  *      Added AC : If we are fragmenting a fragment that's not the
625                  *                 last fragment then keep MF on each bit
626                  */
627                 if (left > 0 || not_last_frag)
628                         iph->frag_off |= htons(IP_MF);
629                 ptr += len;
630                 offset += len;
631
632                 /*
633                  *      Put this fragment into the sending queue.
634                  */
635                 iph->tot_len = htons(len + hlen);
636
637                 ip_send_check(iph);
638
639                 err = output(skb2);
640                 if (err)
641                         goto fail;
642
643                 IP_INC_STATS(IPSTATS_MIB_FRAGCREATES);
644         }
645         kfree_skb(skb);
646         IP_INC_STATS(IPSTATS_MIB_FRAGOKS);
647         return err;
648
649 fail:
650         kfree_skb(skb);
651         IP_INC_STATS(IPSTATS_MIB_FRAGFAILS);
652         return err;
653 }
654
655 EXPORT_SYMBOL(ip_fragment);
656
657 int
658 ip_generic_getfrag(void *from, char *to, int offset, int len, int odd, struct sk_buff *skb)
659 {
660         struct iovec *iov = from;
661
662         if (skb->ip_summed == CHECKSUM_PARTIAL) {
663                 if (memcpy_fromiovecend(to, iov, offset, len) < 0)
664                         return -EFAULT;
665         } else {
666                 __wsum csum = 0;
667                 if (csum_partial_copy_fromiovecend(to, iov, offset, len, &csum) < 0)
668                         return -EFAULT;
669                 skb->csum = csum_block_add(skb->csum, csum, odd);
670         }
671         return 0;
672 }
673
674 static inline __wsum
675 csum_page(struct page *page, int offset, int copy)
676 {
677         char *kaddr;
678         __wsum csum;
679         kaddr = kmap(page);
680         csum = csum_partial(kaddr + offset, copy, 0);
681         kunmap(page);
682         return csum;
683 }
684
685 static inline int ip_ufo_append_data(struct sock *sk,
686                         int getfrag(void *from, char *to, int offset, int len,
687                                int odd, struct sk_buff *skb),
688                         void *from, int length, int hh_len, int fragheaderlen,
689                         int transhdrlen, int mtu,unsigned int flags)
690 {
691         struct sk_buff *skb;
692         int err;
693
694         /* There is support for UDP fragmentation offload by network
695          * device, so create one single skb packet containing complete
696          * udp datagram
697          */
698         if ((skb = skb_peek_tail(&sk->sk_write_queue)) == NULL) {
699                 skb = sock_alloc_send_skb(sk,
700                         hh_len + fragheaderlen + transhdrlen + 20,
701                         (flags & MSG_DONTWAIT), &err);
702
703                 if (skb == NULL)
704                         return err;
705
706                 /* reserve space for Hardware header */
707                 skb_reserve(skb, hh_len);
708
709                 /* create space for UDP/IP header */
710                 skb_put(skb,fragheaderlen + transhdrlen);
711
712                 /* initialize network header pointer */
713                 skb_reset_network_header(skb);
714
715                 /* initialize protocol header pointer */
716                 skb->transport_header = skb->network_header + fragheaderlen;
717
718                 skb->ip_summed = CHECKSUM_PARTIAL;
719                 skb->csum = 0;
720                 sk->sk_sndmsg_off = 0;
721         }
722
723         err = skb_append_datato_frags(sk,skb, getfrag, from,
724                                (length - transhdrlen));
725         if (!err) {
726                 /* specify the length of each IP datagram fragment*/
727                 skb_shinfo(skb)->gso_size = mtu - fragheaderlen;
728                 skb_shinfo(skb)->gso_type = SKB_GSO_UDP;
729                 __skb_queue_tail(&sk->sk_write_queue, skb);
730
731                 return 0;
732         }
733         /* There is not enough support do UFO ,
734          * so follow normal path
735          */
736         kfree_skb(skb);
737         return err;
738 }
739
740 /*
741  *      ip_append_data() and ip_append_page() can make one large IP datagram
742  *      from many pieces of data. Each pieces will be holded on the socket
743  *      until ip_push_pending_frames() is called. Each piece can be a page
744  *      or non-page data.
745  *
746  *      Not only UDP, other transport protocols - e.g. raw sockets - can use
747  *      this interface potentially.
748  *
749  *      LATER: length must be adjusted by pad at tail, when it is required.
750  */
751 int ip_append_data(struct sock *sk,
752                    int getfrag(void *from, char *to, int offset, int len,
753                                int odd, struct sk_buff *skb),
754                    void *from, int length, int transhdrlen,
755                    struct ipcm_cookie *ipc, struct rtable *rt,
756                    unsigned int flags)
757 {
758         struct inet_sock *inet = inet_sk(sk);
759         struct sk_buff *skb;
760
761         struct ip_options *opt = NULL;
762         int hh_len;
763         int exthdrlen;
764         int mtu;
765         int copy;
766         int err;
767         int offset = 0;
768         unsigned int maxfraglen, fragheaderlen;
769         int csummode = CHECKSUM_NONE;
770
771         if (flags&MSG_PROBE)
772                 return 0;
773
774         if (skb_queue_empty(&sk->sk_write_queue)) {
775                 /*
776                  * setup for corking.
777                  */
778                 opt = ipc->opt;
779                 if (opt) {
780                         if (inet->cork.opt == NULL) {
781                                 inet->cork.opt = kmalloc(sizeof(struct ip_options) + 40, sk->sk_allocation);
782                                 if (unlikely(inet->cork.opt == NULL))
783                                         return -ENOBUFS;
784                         }
785                         memcpy(inet->cork.opt, opt, sizeof(struct ip_options)+opt->optlen);
786                         inet->cork.flags |= IPCORK_OPT;
787                         inet->cork.addr = ipc->addr;
788                 }
789                 dst_hold(&rt->u.dst);
790                 inet->cork.fragsize = mtu = dst_mtu(rt->u.dst.path);
791                 inet->cork.rt = rt;
792                 inet->cork.length = 0;
793                 sk->sk_sndmsg_page = NULL;
794                 sk->sk_sndmsg_off = 0;
795                 if ((exthdrlen = rt->u.dst.header_len) != 0) {
796                         length += exthdrlen;
797                         transhdrlen += exthdrlen;
798                 }
799         } else {
800                 rt = inet->cork.rt;
801                 if (inet->cork.flags & IPCORK_OPT)
802                         opt = inet->cork.opt;
803
804                 transhdrlen = 0;
805                 exthdrlen = 0;
806                 mtu = inet->cork.fragsize;
807         }
808         hh_len = LL_RESERVED_SPACE(rt->u.dst.dev);
809
810         fragheaderlen = sizeof(struct iphdr) + (opt ? opt->optlen : 0);
811         maxfraglen = ((mtu - fragheaderlen) & ~7) + fragheaderlen;
812
813         if (inet->cork.length + length > 0xFFFF - fragheaderlen) {
814                 ip_local_error(sk, EMSGSIZE, rt->rt_dst, inet->dport, mtu-exthdrlen);
815                 return -EMSGSIZE;
816         }
817
818         /*
819          * transhdrlen > 0 means that this is the first fragment and we wish
820          * it won't be fragmented in the future.
821          */
822         if (transhdrlen &&
823             length + fragheaderlen <= mtu &&
824             rt->u.dst.dev->features & NETIF_F_ALL_CSUM &&
825             !exthdrlen)
826                 csummode = CHECKSUM_PARTIAL;
827
828         inet->cork.length += length;
829         if (((length > mtu) && (sk->sk_protocol == IPPROTO_UDP)) &&
830                         (rt->u.dst.dev->features & NETIF_F_UFO)) {
831
832                 err = ip_ufo_append_data(sk, getfrag, from, length, hh_len,
833                                          fragheaderlen, transhdrlen, mtu,
834                                          flags);
835                 if (err)
836                         goto error;
837                 return 0;
838         }
839
840         /* So, what's going on in the loop below?
841          *
842          * We use calculated fragment length to generate chained skb,
843          * each of segments is IP fragment ready for sending to network after
844          * adding appropriate IP header.
845          */
846
847         if ((skb = skb_peek_tail(&sk->sk_write_queue)) == NULL)
848                 goto alloc_new_skb;
849
850         while (length > 0) {
851                 /* Check if the remaining data fits into current packet. */
852                 copy = mtu - skb->len;
853                 if (copy < length)
854                         copy = maxfraglen - skb->len;
855                 if (copy <= 0) {
856                         char *data;
857                         unsigned int datalen;
858                         unsigned int fraglen;
859                         unsigned int fraggap;
860                         unsigned int alloclen;
861                         struct sk_buff *skb_prev;
862 alloc_new_skb:
863                         skb_prev = skb;
864                         if (skb_prev)
865                                 fraggap = skb_prev->len - maxfraglen;
866                         else
867                                 fraggap = 0;
868
869                         /*
870                          * If remaining data exceeds the mtu,
871                          * we know we need more fragment(s).
872                          */
873                         datalen = length + fraggap;
874                         if (datalen > mtu - fragheaderlen)
875                                 datalen = maxfraglen - fragheaderlen;
876                         fraglen = datalen + fragheaderlen;
877
878                         if ((flags & MSG_MORE) &&
879                             !(rt->u.dst.dev->features&NETIF_F_SG))
880                                 alloclen = mtu;
881                         else
882                                 alloclen = datalen + fragheaderlen;
883
884                         /* The last fragment gets additional space at tail.
885                          * Note, with MSG_MORE we overallocate on fragments,
886                          * because we have no idea what fragment will be
887                          * the last.
888                          */
889                         if (datalen == length + fraggap)
890                                 alloclen += rt->u.dst.trailer_len;
891
892                         if (transhdrlen) {
893                                 skb = sock_alloc_send_skb(sk,
894                                                 alloclen + hh_len + 15,
895                                                 (flags & MSG_DONTWAIT), &err);
896                         } else {
897                                 skb = NULL;
898                                 if (atomic_read(&sk->sk_wmem_alloc) <=
899                                     2 * sk->sk_sndbuf)
900                                         skb = sock_wmalloc(sk,
901                                                            alloclen + hh_len + 15, 1,
902                                                            sk->sk_allocation);
903                                 if (unlikely(skb == NULL))
904                                         err = -ENOBUFS;
905                         }
906                         if (skb == NULL)
907                                 goto error;
908
909                         /*
910                          *      Fill in the control structures
911                          */
912                         skb->ip_summed = csummode;
913                         skb->csum = 0;
914                         skb_reserve(skb, hh_len);
915
916                         /*
917                          *      Find where to start putting bytes.
918                          */
919                         data = skb_put(skb, fraglen);
920                         skb_set_network_header(skb, exthdrlen);
921                         skb->transport_header = (skb->network_header +
922                                                  fragheaderlen);
923                         data += fragheaderlen;
924
925                         if (fraggap) {
926                                 skb->csum = skb_copy_and_csum_bits(
927                                         skb_prev, maxfraglen,
928                                         data + transhdrlen, fraggap, 0);
929                                 skb_prev->csum = csum_sub(skb_prev->csum,
930                                                           skb->csum);
931                                 data += fraggap;
932                                 pskb_trim_unique(skb_prev, maxfraglen);
933                         }
934
935                         copy = datalen - transhdrlen - fraggap;
936                         if (copy > 0 && getfrag(from, data + transhdrlen, offset, copy, fraggap, skb) < 0) {
937                                 err = -EFAULT;
938                                 kfree_skb(skb);
939                                 goto error;
940                         }
941
942                         offset += copy;
943                         length -= datalen - fraggap;
944                         transhdrlen = 0;
945                         exthdrlen = 0;
946                         csummode = CHECKSUM_NONE;
947
948                         /*
949                          * Put the packet on the pending queue.
950                          */
951                         __skb_queue_tail(&sk->sk_write_queue, skb);
952                         continue;
953                 }
954
955                 if (copy > length)
956                         copy = length;
957
958                 if (!(rt->u.dst.dev->features&NETIF_F_SG)) {
959                         unsigned int off;
960
961                         off = skb->len;
962                         if (getfrag(from, skb_put(skb, copy),
963                                         offset, copy, off, skb) < 0) {
964                                 __skb_trim(skb, off);
965                                 err = -EFAULT;
966                                 goto error;
967                         }
968                 } else {
969                         int i = skb_shinfo(skb)->nr_frags;
970                         skb_frag_t *frag = &skb_shinfo(skb)->frags[i-1];
971                         struct page *page = sk->sk_sndmsg_page;
972                         int off = sk->sk_sndmsg_off;
973                         unsigned int left;
974
975                         if (page && (left = PAGE_SIZE - off) > 0) {
976                                 if (copy >= left)
977                                         copy = left;
978                                 if (page != frag->page) {
979                                         if (i == MAX_SKB_FRAGS) {
980                                                 err = -EMSGSIZE;
981                                                 goto error;
982                                         }
983                                         get_page(page);
984                                         skb_fill_page_desc(skb, i, page, sk->sk_sndmsg_off, 0);
985                                         frag = &skb_shinfo(skb)->frags[i];
986                                 }
987                         } else if (i < MAX_SKB_FRAGS) {
988                                 if (copy > PAGE_SIZE)
989                                         copy = PAGE_SIZE;
990                                 page = alloc_pages(sk->sk_allocation, 0);
991                                 if (page == NULL)  {
992                                         err = -ENOMEM;
993                                         goto error;
994                                 }
995                                 sk->sk_sndmsg_page = page;
996                                 sk->sk_sndmsg_off = 0;
997
998                                 skb_fill_page_desc(skb, i, page, 0, 0);
999                                 frag = &skb_shinfo(skb)->frags[i];
1000                                 skb->truesize += PAGE_SIZE;
1001                                 atomic_add(PAGE_SIZE, &sk->sk_wmem_alloc);
1002                         } else {
1003                                 err = -EMSGSIZE;
1004                                 goto error;
1005                         }
1006                         if (getfrag(from, page_address(frag->page)+frag->page_offset+frag->size, offset, copy, skb->len, skb) < 0) {
1007                                 err = -EFAULT;
1008                                 goto error;
1009                         }
1010                         sk->sk_sndmsg_off += copy;
1011                         frag->size += copy;
1012                         skb->len += copy;
1013                         skb->data_len += copy;
1014                 }
1015                 offset += copy;
1016                 length -= copy;
1017         }
1018
1019         return 0;
1020
1021 error:
1022         inet->cork.length -= length;
1023         IP_INC_STATS(IPSTATS_MIB_OUTDISCARDS);
1024         return err;
1025 }
1026
1027 ssize_t ip_append_page(struct sock *sk, struct page *page,
1028                        int offset, size_t size, int flags)
1029 {
1030         struct inet_sock *inet = inet_sk(sk);
1031         struct sk_buff *skb;
1032         struct rtable *rt;
1033         struct ip_options *opt = NULL;
1034         int hh_len;
1035         int mtu;
1036         int len;
1037         int err;
1038         unsigned int maxfraglen, fragheaderlen, fraggap;
1039
1040         if (inet->hdrincl)
1041                 return -EPERM;
1042
1043         if (flags&MSG_PROBE)
1044                 return 0;
1045
1046         if (skb_queue_empty(&sk->sk_write_queue))
1047                 return -EINVAL;
1048
1049         rt = inet->cork.rt;
1050         if (inet->cork.flags & IPCORK_OPT)
1051                 opt = inet->cork.opt;
1052
1053         if (!(rt->u.dst.dev->features&NETIF_F_SG))
1054                 return -EOPNOTSUPP;
1055
1056         hh_len = LL_RESERVED_SPACE(rt->u.dst.dev);
1057         mtu = inet->cork.fragsize;
1058
1059         fragheaderlen = sizeof(struct iphdr) + (opt ? opt->optlen : 0);
1060         maxfraglen = ((mtu - fragheaderlen) & ~7) + fragheaderlen;
1061
1062         if (inet->cork.length + size > 0xFFFF - fragheaderlen) {
1063                 ip_local_error(sk, EMSGSIZE, rt->rt_dst, inet->dport, mtu);
1064                 return -EMSGSIZE;
1065         }
1066
1067         if ((skb = skb_peek_tail(&sk->sk_write_queue)) == NULL)
1068                 return -EINVAL;
1069
1070         inet->cork.length += size;
1071         if ((sk->sk_protocol == IPPROTO_UDP) &&
1072             (rt->u.dst.dev->features & NETIF_F_UFO)) {
1073                 skb_shinfo(skb)->gso_size = mtu - fragheaderlen;
1074                 skb_shinfo(skb)->gso_type = SKB_GSO_UDP;
1075         }
1076
1077
1078         while (size > 0) {
1079                 int i;
1080
1081                 if (skb_is_gso(skb))
1082                         len = size;
1083                 else {
1084
1085                         /* Check if the remaining data fits into current packet. */
1086                         len = mtu - skb->len;
1087                         if (len < size)
1088                                 len = maxfraglen - skb->len;
1089                 }
1090                 if (len <= 0) {
1091                         struct sk_buff *skb_prev;
1092                         int alloclen;
1093
1094                         skb_prev = skb;
1095                         fraggap = skb_prev->len - maxfraglen;
1096
1097                         alloclen = fragheaderlen + hh_len + fraggap + 15;
1098                         skb = sock_wmalloc(sk, alloclen, 1, sk->sk_allocation);
1099                         if (unlikely(!skb)) {
1100                                 err = -ENOBUFS;
1101                                 goto error;
1102                         }
1103
1104                         /*
1105                          *      Fill in the control structures
1106                          */
1107                         skb->ip_summed = CHECKSUM_NONE;
1108                         skb->csum = 0;
1109                         skb_reserve(skb, hh_len);
1110
1111                         /*
1112                          *      Find where to start putting bytes.
1113                          */
1114                         skb_put(skb, fragheaderlen + fraggap);
1115                         skb_reset_network_header(skb);
1116                         skb->transport_header = (skb->network_header +
1117                                                  fragheaderlen);
1118                         if (fraggap) {
1119                                 skb->csum = skb_copy_and_csum_bits(skb_prev,
1120                                                                    maxfraglen,
1121                                                     skb_transport_header(skb),
1122                                                                    fraggap, 0);
1123                                 skb_prev->csum = csum_sub(skb_prev->csum,
1124                                                           skb->csum);
1125                                 pskb_trim_unique(skb_prev, maxfraglen);
1126                         }
1127
1128                         /*
1129                          * Put the packet on the pending queue.
1130                          */
1131                         __skb_queue_tail(&sk->sk_write_queue, skb);
1132                         continue;
1133                 }
1134
1135                 i = skb_shinfo(skb)->nr_frags;
1136                 if (len > size)
1137                         len = size;
1138                 if (skb_can_coalesce(skb, i, page, offset)) {
1139                         skb_shinfo(skb)->frags[i-1].size += len;
1140                 } else if (i < MAX_SKB_FRAGS) {
1141                         get_page(page);
1142                         skb_fill_page_desc(skb, i, page, offset, len);
1143                 } else {
1144                         err = -EMSGSIZE;
1145                         goto error;
1146                 }
1147
1148                 if (skb->ip_summed == CHECKSUM_NONE) {
1149                         __wsum csum;
1150                         csum = csum_page(page, offset, len);
1151                         skb->csum = csum_block_add(skb->csum, csum, skb->len);
1152                 }
1153
1154                 skb->len += len;
1155                 skb->data_len += len;
1156                 offset += len;
1157                 size -= len;
1158         }
1159         return 0;
1160
1161 error:
1162         inet->cork.length -= size;
1163         IP_INC_STATS(IPSTATS_MIB_OUTDISCARDS);
1164         return err;
1165 }
1166
1167 /*
1168  *      Combined all pending IP fragments on the socket as one IP datagram
1169  *      and push them out.
1170  */
1171 int ip_push_pending_frames(struct sock *sk)
1172 {
1173         struct sk_buff *skb, *tmp_skb;
1174         struct sk_buff **tail_skb;
1175         struct inet_sock *inet = inet_sk(sk);
1176         struct ip_options *opt = NULL;
1177         struct rtable *rt = inet->cork.rt;
1178         struct iphdr *iph;
1179         __be16 df = 0;
1180         __u8 ttl;
1181         int err = 0;
1182
1183         if ((skb = __skb_dequeue(&sk->sk_write_queue)) == NULL)
1184                 goto out;
1185         tail_skb = &(skb_shinfo(skb)->frag_list);
1186
1187         /* move skb->data to ip header from ext header */
1188         if (skb->data < skb_network_header(skb))
1189                 __skb_pull(skb, skb_network_offset(skb));
1190         while ((tmp_skb = __skb_dequeue(&sk->sk_write_queue)) != NULL) {
1191                 __skb_pull(tmp_skb, skb_network_header_len(skb));
1192                 *tail_skb = tmp_skb;
1193                 tail_skb = &(tmp_skb->next);
1194                 skb->len += tmp_skb->len;
1195                 skb->data_len += tmp_skb->len;
1196                 skb->truesize += tmp_skb->truesize;
1197                 __sock_put(tmp_skb->sk);
1198                 tmp_skb->destructor = NULL;
1199                 tmp_skb->sk = NULL;
1200         }
1201
1202         /* Unless user demanded real pmtu discovery (IP_PMTUDISC_DO), we allow
1203          * to fragment the frame generated here. No matter, what transforms
1204          * how transforms change size of the packet, it will come out.
1205          */
1206         if (inet->pmtudisc != IP_PMTUDISC_DO)
1207                 skb->local_df = 1;
1208
1209         /* DF bit is set when we want to see DF on outgoing frames.
1210          * If local_df is set too, we still allow to fragment this frame
1211          * locally. */
1212         if (inet->pmtudisc == IP_PMTUDISC_DO ||
1213             (skb->len <= dst_mtu(&rt->u.dst) &&
1214              ip_dont_fragment(sk, &rt->u.dst)))
1215                 df = htons(IP_DF);
1216
1217         if (inet->cork.flags & IPCORK_OPT)
1218                 opt = inet->cork.opt;
1219
1220         if (rt->rt_type == RTN_MULTICAST)
1221                 ttl = inet->mc_ttl;
1222         else
1223                 ttl = ip_select_ttl(inet, &rt->u.dst);
1224
1225         iph = (struct iphdr *)skb->data;
1226         iph->version = 4;
1227         iph->ihl = 5;
1228         if (opt) {
1229                 iph->ihl += opt->optlen>>2;
1230                 ip_options_build(skb, opt, inet->cork.addr, rt, 0);
1231         }
1232         iph->tos = inet->tos;
1233         iph->tot_len = htons(skb->len);
1234         iph->frag_off = df;
1235         ip_select_ident(iph, &rt->u.dst, sk);
1236         iph->ttl = ttl;
1237         iph->protocol = sk->sk_protocol;
1238         iph->saddr = rt->rt_src;
1239         iph->daddr = rt->rt_dst;
1240         ip_send_check(iph);
1241
1242         skb->priority = sk->sk_priority;
1243         skb->dst = dst_clone(&rt->u.dst);
1244
1245         /* Netfilter gets whole the not fragmented skb. */
1246         err = NF_HOOK(PF_INET, NF_IP_LOCAL_OUT, skb, NULL,
1247                       skb->dst->dev, dst_output);
1248         if (err) {
1249                 if (err > 0)
1250                         err = inet->recverr ? net_xmit_errno(err) : 0;
1251                 if (err)
1252                         goto error;
1253         }
1254
1255 out:
1256         inet->cork.flags &= ~IPCORK_OPT;
1257         kfree(inet->cork.opt);
1258         inet->cork.opt = NULL;
1259         if (inet->cork.rt) {
1260                 ip_rt_put(inet->cork.rt);
1261                 inet->cork.rt = NULL;
1262         }
1263         return err;
1264
1265 error:
1266         IP_INC_STATS(IPSTATS_MIB_OUTDISCARDS);
1267         goto out;
1268 }
1269
1270 /*
1271  *      Throw away all pending data on the socket.
1272  */
1273 void ip_flush_pending_frames(struct sock *sk)
1274 {
1275         struct inet_sock *inet = inet_sk(sk);
1276         struct sk_buff *skb;
1277
1278         while ((skb = __skb_dequeue_tail(&sk->sk_write_queue)) != NULL)
1279                 kfree_skb(skb);
1280
1281         inet->cork.flags &= ~IPCORK_OPT;
1282         kfree(inet->cork.opt);
1283         inet->cork.opt = NULL;
1284         if (inet->cork.rt) {
1285                 ip_rt_put(inet->cork.rt);
1286                 inet->cork.rt = NULL;
1287         }
1288 }
1289
1290
1291 /*
1292  *      Fetch data from kernel space and fill in checksum if needed.
1293  */
1294 static int ip_reply_glue_bits(void *dptr, char *to, int offset,
1295                               int len, int odd, struct sk_buff *skb)
1296 {
1297         __wsum csum;
1298
1299         csum = csum_partial_copy_nocheck(dptr+offset, to, len, 0);
1300         skb->csum = csum_block_add(skb->csum, csum, odd);
1301         return 0;
1302 }
1303
1304 /*
1305  *      Generic function to send a packet as reply to another packet.
1306  *      Used to send TCP resets so far. ICMP should use this function too.
1307  *
1308  *      Should run single threaded per socket because it uses the sock
1309  *      structure to pass arguments.
1310  *
1311  *      LATER: switch from ip_build_xmit to ip_append_*
1312  */
1313 void ip_send_reply(struct sock *sk, struct sk_buff *skb, struct ip_reply_arg *arg,
1314                    unsigned int len)
1315 {
1316         struct inet_sock *inet = inet_sk(sk);
1317         struct {
1318                 struct ip_options       opt;
1319                 char                    data[40];
1320         } replyopts;
1321         struct ipcm_cookie ipc;
1322         __be32 daddr;
1323         struct rtable *rt = (struct rtable*)skb->dst;
1324
1325         if (ip_options_echo(&replyopts.opt, skb))
1326                 return;
1327
1328         daddr = ipc.addr = rt->rt_src;
1329         ipc.opt = NULL;
1330
1331         if (replyopts.opt.optlen) {
1332                 ipc.opt = &replyopts.opt;
1333
1334                 if (ipc.opt->srr)
1335                         daddr = replyopts.opt.faddr;
1336         }
1337
1338         {
1339                 struct flowi fl = { .nl_u = { .ip4_u =
1340                                               { .daddr = daddr,
1341                                                 .saddr = rt->rt_spec_dst,
1342                                                 .tos = RT_TOS(ip_hdr(skb)->tos) } },
1343                                     /* Not quite clean, but right. */
1344                                     .uli_u = { .ports =
1345                                                { .sport = tcp_hdr(skb)->dest,
1346                                                  .dport = tcp_hdr(skb)->source } },
1347                                     .proto = sk->sk_protocol };
1348                 security_skb_classify_flow(skb, &fl);
1349                 if (ip_route_output_key(&rt, &fl))
1350                         return;
1351         }
1352
1353         /* And let IP do all the hard work.
1354
1355            This chunk is not reenterable, hence spinlock.
1356            Note that it uses the fact, that this function is called
1357            with locally disabled BH and that sk cannot be already spinlocked.
1358          */
1359         bh_lock_sock(sk);
1360         inet->tos = ip_hdr(skb)->tos;
1361         sk->sk_priority = skb->priority;
1362         sk->sk_protocol = ip_hdr(skb)->protocol;
1363         ip_append_data(sk, ip_reply_glue_bits, arg->iov->iov_base, len, 0,
1364                        &ipc, rt, MSG_DONTWAIT);
1365         if ((skb = skb_peek(&sk->sk_write_queue)) != NULL) {
1366                 if (arg->csumoffset >= 0)
1367                         *((__sum16 *)skb_transport_header(skb) +
1368                           arg->csumoffset) = csum_fold(csum_add(skb->csum,
1369                                                                 arg->csum));
1370                 skb->ip_summed = CHECKSUM_NONE;
1371                 ip_push_pending_frames(sk);
1372         }
1373
1374         bh_unlock_sock(sk);
1375
1376         ip_rt_put(rt);
1377 }
1378
1379 void __init ip_init(void)
1380 {
1381         ip_rt_init();
1382         inet_initpeers();
1383
1384 #if defined(CONFIG_IP_MULTICAST) && defined(CONFIG_PROC_FS)
1385         igmp_mc_proc_init();
1386 #endif
1387 }
1388
1389 EXPORT_SYMBOL(ip_generic_getfrag);
1390 EXPORT_SYMBOL(ip_queue_xmit);
1391 EXPORT_SYMBOL(ip_send_check);