net: No more expensive sock_hold()/sock_put() on each tx
[linux-2.6.git] / net / ipv6 / ip6_output.c
index 7d92fd9..7c76e3d 100644 (file)
@@ -78,7 +78,7 @@ int __ip6_local_out(struct sk_buff *skb)
                len = 0;
        ipv6_hdr(skb)->payload_len = htons(len);
 
-       return nf_hook(PF_INET6, NF_INET_LOCAL_OUT, skb, NULL, skb->dst->dev,
+       return nf_hook(PF_INET6, NF_INET_LOCAL_OUT, skb, NULL, skb_dst(skb)->dev,
                       dst_output);
 }
 
@@ -96,7 +96,7 @@ EXPORT_SYMBOL_GPL(ip6_local_out);
 
 static int ip6_output_finish(struct sk_buff *skb)
 {
-       struct dst_entry *dst = skb->dst;
+       struct dst_entry *dst = skb_dst(skb);
 
        if (dst->hh)
                return neigh_hh_output(dst->hh, skb);
@@ -117,7 +117,7 @@ static int ip6_dev_loopback_xmit(struct sk_buff *newskb)
        __skb_pull(newskb, skb_network_offset(newskb));
        newskb->pkt_type = PACKET_LOOPBACK;
        newskb->ip_summed = CHECKSUM_UNNECESSARY;
-       WARN_ON(!newskb->dst);
+       WARN_ON(!skb_dst(newskb));
 
        netif_rx(newskb);
        return 0;
@@ -126,7 +126,7 @@ static int ip6_dev_loopback_xmit(struct sk_buff *newskb)
 
 static int ip6_output2(struct sk_buff *skb)
 {
-       struct dst_entry *dst = skb->dst;
+       struct dst_entry *dst = skb_dst(skb);
        struct net_device *dev = dst->dev;
 
        skb->protocol = htons(ETH_P_IPV6);
@@ -134,10 +134,11 @@ static int ip6_output2(struct sk_buff *skb)
 
        if (ipv6_addr_is_multicast(&ipv6_hdr(skb)->daddr)) {
                struct ipv6_pinfo* np = skb->sk ? inet6_sk(skb->sk) : NULL;
-               struct inet6_dev *idev = ip6_dst_idev(skb->dst);
+               struct inet6_dev *idev = ip6_dst_idev(skb_dst(skb));
 
                if (!(dev->flags & IFF_LOOPBACK) && (!np || np->mc_loop) &&
-                   ((mroute6_socket && !(IP6CB(skb)->flags & IP6SKB_FORWARDED)) ||
+                   ((mroute6_socket(dev_net(dev)) &&
+                    !(IP6CB(skb)->flags & IP6SKB_FORWARDED)) ||
                     ipv6_chk_mcast_addr(dev, &ipv6_hdr(skb)->daddr,
                                         &ipv6_hdr(skb)->saddr))) {
                        struct sk_buff *newskb = skb_clone(skb, GFP_ATOMIC);
@@ -158,7 +159,8 @@ static int ip6_output2(struct sk_buff *skb)
                        }
                }
 
-               IP6_INC_STATS(dev_net(dev), idev, IPSTATS_MIB_OUTMCASTPKTS);
+               IP6_UPD_PO_STATS(dev_net(dev), idev, IPSTATS_MIB_OUTMCAST,
+                               skb->len);
        }
 
        return NF_HOOK(PF_INET6, NF_INET_POST_ROUTING, skb, NULL, skb->dev,
@@ -170,21 +172,21 @@ static inline int ip6_skb_dst_mtu(struct sk_buff *skb)
        struct ipv6_pinfo *np = skb->sk ? inet6_sk(skb->sk) : NULL;
 
        return (np && np->pmtudisc == IPV6_PMTUDISC_PROBE) ?
-              skb->dst->dev->mtu : dst_mtu(skb->dst);
+              skb_dst(skb)->dev->mtu : dst_mtu(skb_dst(skb));
 }
 
 int ip6_output(struct sk_buff *skb)
 {
-       struct inet6_dev *idev = ip6_dst_idev(skb->dst);
+       struct inet6_dev *idev = ip6_dst_idev(skb_dst(skb));
        if (unlikely(idev->cnf.disable_ipv6)) {
-               IP6_INC_STATS(dev_net(skb->dst->dev), idev,
+               IP6_INC_STATS(dev_net(skb_dst(skb)->dev), idev,
                              IPSTATS_MIB_OUTDISCARDS);
                kfree_skb(skb);
                return 0;
        }
 
        if ((skb->len > ip6_skb_dst_mtu(skb) && !skb_is_gso(skb)) ||
-                               dst_allfrag(skb->dst))
+                               dst_allfrag(skb_dst(skb)))
                return ip6_fragment(skb, ip6_output2);
        else
                return ip6_output2(skb);
@@ -200,7 +202,7 @@ int ip6_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl,
        struct net *net = sock_net(sk);
        struct ipv6_pinfo *np = inet6_sk(sk);
        struct in6_addr *first_hop = &fl->fl6_dst;
-       struct dst_entry *dst = skb->dst;
+       struct dst_entry *dst = skb_dst(skb);
        struct ipv6hdr *hdr;
        u8  proto = fl->proto;
        int seg_len = skb->len;
@@ -220,7 +222,7 @@ int ip6_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl,
                if (skb_headroom(skb) < head_room) {
                        struct sk_buff *skb2 = skb_realloc_headroom(skb, head_room);
                        if (skb2 == NULL) {
-                               IP6_INC_STATS(net, ip6_dst_idev(skb->dst),
+                               IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)),
                                              IPSTATS_MIB_OUTDISCARDS);
                                kfree_skb(skb);
                                return -ENOBUFS;
@@ -274,8 +276,8 @@ int ip6_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl,
 
        mtu = dst_mtu(dst);
        if ((skb->len <= mtu) || skb->local_df || skb_is_gso(skb)) {
-               IP6_INC_STATS(net, ip6_dst_idev(skb->dst),
-                             IPSTATS_MIB_OUTREQUESTS);
+               IP6_UPD_PO_STATS(net, ip6_dst_idev(skb_dst(skb)),
+                             IPSTATS_MIB_OUT, skb->len);
                return NF_HOOK(PF_INET6, NF_INET_LOCAL_OUT, skb, NULL, dst->dev,
                                dst_output);
        }
@@ -284,7 +286,7 @@ int ip6_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl,
                printk(KERN_DEBUG "IPv6: sending pkt_too_big to self\n");
        skb->dev = dst->dev;
        icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu, skb->dev);
-       IP6_INC_STATS(net, ip6_dst_idev(skb->dst), IPSTATS_MIB_FRAGFAILS);
+       IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)), IPSTATS_MIB_FRAGFAILS);
        kfree_skb(skb);
        return -EMSGSIZE;
 }
@@ -414,7 +416,7 @@ static inline int ip6_forward_finish(struct sk_buff *skb)
 
 int ip6_forward(struct sk_buff *skb)
 {
-       struct dst_entry *dst = skb->dst;
+       struct dst_entry *dst = skb_dst(skb);
        struct ipv6hdr *hdr = ipv6_hdr(skb);
        struct inet6_skb_parm *opt = IP6CB(skb);
        struct net *net = dev_net(dst->dev);
@@ -483,7 +485,7 @@ int ip6_forward(struct sk_buff *skb)
                IP6_INC_STATS(net, ip6_dst_idev(dst), IPSTATS_MIB_INDISCARDS);
                goto drop;
        }
-       dst = skb->dst;
+       dst = skb_dst(skb);
 
        /* IPv6 specs say nothing about it, but it is clear that we cannot
           send redirects to source routed frames.
@@ -564,8 +566,8 @@ static void ip6_copy_metadata(struct sk_buff *to, struct sk_buff *from)
        to->pkt_type = from->pkt_type;
        to->priority = from->priority;
        to->protocol = from->protocol;
-       dst_release(to->dst);
-       to->dst = dst_clone(from->dst);
+       skb_dst_drop(to);
+       skb_dst_set(to, dst_clone(skb_dst(from)));
        to->dev = from->dev;
        to->mark = from->mark;
 
@@ -622,7 +624,7 @@ int ip6_find_1stfragopt(struct sk_buff *skb, u8 **nexthdr)
 static int ip6_fragment(struct sk_buff *skb, int (*output)(struct sk_buff *))
 {
        struct sk_buff *frag;
-       struct rt6_info *rt = (struct rt6_info*)skb->dst;
+       struct rt6_info *rt = (struct rt6_info*)skb_dst(skb);
        struct ipv6_pinfo *np = skb->sk ? inet6_sk(skb->sk) : NULL;
        struct ipv6hdr *tmp_hdr;
        struct frag_hdr *fh;
@@ -630,7 +632,7 @@ static int ip6_fragment(struct sk_buff *skb, int (*output)(struct sk_buff *))
        __be32 frag_id = 0;
        int ptr, offset = 0, err=0;
        u8 *prevhdr, nexthdr = 0;
-       struct net *net = dev_net(skb->dst->dev);
+       struct net *net = dev_net(skb_dst(skb)->dev);
 
        hlen = ip6_find_1stfragopt(skb, &prevhdr);
        nexthdr = *prevhdr;
@@ -642,9 +644,9 @@ static int ip6_fragment(struct sk_buff *skb, int (*output)(struct sk_buff *))
         * check should be redundant, but it's free.)
         */
        if (!skb->local_df) {
-               skb->dev = skb->dst->dev;
+               skb->dev = skb_dst(skb)->dev;
                icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu, skb->dev);
-               IP6_INC_STATS(net, ip6_dst_idev(skb->dst),
+               IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)),
                              IPSTATS_MIB_FRAGFAILS);
                kfree_skb(skb);
                return -EMSGSIZE;
@@ -656,7 +658,7 @@ static int ip6_fragment(struct sk_buff *skb, int (*output)(struct sk_buff *))
        }
        mtu -= hlen + sizeof(struct frag_hdr);
 
-       if (skb_shinfo(skb)->frag_list) {
+       if (skb_has_frags(skb)) {
                int first_len = skb_pagelen(skb);
                int truesizes = 0;
 
@@ -665,7 +667,7 @@ static int ip6_fragment(struct sk_buff *skb, int (*output)(struct sk_buff *))
                    skb_cloned(skb))
                        goto slow_path;
 
-               for (frag = skb_shinfo(skb)->frag_list; frag; frag = frag->next) {
+               skb_walk_frags(skb, frag) {
                        /* Correct geometry. */
                        if (frag->len > mtu ||
                            ((frag->len & 7) && frag->next) ||
@@ -678,7 +680,6 @@ static int ip6_fragment(struct sk_buff *skb, int (*output)(struct sk_buff *))
 
                        BUG_ON(frag->sk);
                        if (skb->sk) {
-                               sock_hold(skb->sk);
                                frag->sk = skb->sk;
                                frag->destructor = sock_wfree;
                                truesizes += frag->truesize;
@@ -688,13 +689,13 @@ static int ip6_fragment(struct sk_buff *skb, int (*output)(struct sk_buff *))
                err = 0;
                offset = 0;
                frag = skb_shinfo(skb)->frag_list;
-               skb_shinfo(skb)->frag_list = NULL;
+               skb_frag_list_init(skb);
                /* BUILD HEADER */
 
                *prevhdr = NEXTHDR_FRAGMENT;
                tmp_hdr = kmemdup(skb_network_header(skb), hlen, GFP_ATOMIC);
                if (!tmp_hdr) {
-                       IP6_INC_STATS(net, ip6_dst_idev(skb->dst),
+                       IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)),
                                      IPSTATS_MIB_FRAGFAILS);
                        return -ENOMEM;
                }
@@ -807,7 +808,7 @@ slow_path:
 
                if ((frag = alloc_skb(len+hlen+sizeof(struct frag_hdr)+LL_ALLOCATED_SPACE(rt->u.dst.dev), GFP_ATOMIC)) == NULL) {
                        NETDEBUG(KERN_INFO "IPv6: frag: no memory for new fragment!\n");
-                       IP6_INC_STATS(net, ip6_dst_idev(skb->dst),
+                       IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)),
                                      IPSTATS_MIB_FRAGFAILS);
                        err = -ENOMEM;
                        goto fail;
@@ -871,16 +872,16 @@ slow_path:
                if (err)
                        goto fail;
 
-               IP6_INC_STATS(net, ip6_dst_idev(skb->dst),
+               IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)),
                              IPSTATS_MIB_FRAGCREATES);
        }
-       IP6_INC_STATS(net, ip6_dst_idev(skb->dst),
+       IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)),
                      IPSTATS_MIB_FRAGOKS);
        kfree_skb(skb);
        return err;
 
 fail:
-       IP6_INC_STATS(net, ip6_dst_idev(skb->dst),
+       IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)),
                      IPSTATS_MIB_FRAGFAILS);
        kfree_skb(skb);
        return err;
@@ -1104,6 +1105,18 @@ static inline int ip6_ufo_append_data(struct sock *sk,
        return err;
 }
 
+static inline struct ipv6_opt_hdr *ip6_opt_dup(struct ipv6_opt_hdr *src,
+                                              gfp_t gfp)
+{
+       return src ? kmemdup(src, (src->hdrlen + 1) * 8, gfp) : NULL;
+}
+
+static inline struct ipv6_rt_hdr *ip6_rthdr_dup(struct ipv6_rt_hdr *src,
+                                               gfp_t gfp)
+{
+       return src ? kmemdup(src, (src->hdrlen + 1) * 8, gfp) : NULL;
+}
+
 int ip6_append_data(struct sock *sk, int getfrag(void *from, char *to,
        int offset, int len, int odd, struct sk_buff *skb),
        void *from, int length, int transhdrlen,
@@ -1129,17 +1142,37 @@ int ip6_append_data(struct sock *sk, int getfrag(void *from, char *to,
                 * setup for corking
                 */
                if (opt) {
-                       if (np->cork.opt == NULL) {
-                               np->cork.opt = kmalloc(opt->tot_len,
-                                                      sk->sk_allocation);
-                               if (unlikely(np->cork.opt == NULL))
-                                       return -ENOBUFS;
-                       } else if (np->cork.opt->tot_len < opt->tot_len) {
-                               printk(KERN_DEBUG "ip6_append_data: invalid option length\n");
+                       if (WARN_ON(np->cork.opt))
                                return -EINVAL;
-                       }
-                       memcpy(np->cork.opt, opt, opt->tot_len);
-                       inet->cork.flags |= IPCORK_OPT;
+
+                       np->cork.opt = kmalloc(opt->tot_len, sk->sk_allocation);
+                       if (unlikely(np->cork.opt == NULL))
+                               return -ENOBUFS;
+
+                       np->cork.opt->tot_len = opt->tot_len;
+                       np->cork.opt->opt_flen = opt->opt_flen;
+                       np->cork.opt->opt_nflen = opt->opt_nflen;
+
+                       np->cork.opt->dst0opt = ip6_opt_dup(opt->dst0opt,
+                                                           sk->sk_allocation);
+                       if (opt->dst0opt && !np->cork.opt->dst0opt)
+                               return -ENOBUFS;
+
+                       np->cork.opt->dst1opt = ip6_opt_dup(opt->dst1opt,
+                                                           sk->sk_allocation);
+                       if (opt->dst1opt && !np->cork.opt->dst1opt)
+                               return -ENOBUFS;
+
+                       np->cork.opt->hopopt = ip6_opt_dup(opt->hopopt,
+                                                          sk->sk_allocation);
+                       if (opt->hopopt && !np->cork.opt->hopopt)
+                               return -ENOBUFS;
+
+                       np->cork.opt->srcrt = ip6_rthdr_dup(opt->srcrt,
+                                                           sk->sk_allocation);
+                       if (opt->srcrt && !np->cork.opt->srcrt)
+                               return -ENOBUFS;
+
                        /* need source address above miyazawa*/
                }
                dst_hold(&rt->u.dst);
@@ -1166,8 +1199,7 @@ int ip6_append_data(struct sock *sk, int getfrag(void *from, char *to,
        } else {
                rt = (struct rt6_info *)inet->cork.dst;
                fl = &inet->cork.fl;
-               if (inet->cork.flags & IPCORK_OPT)
-                       opt = np->cork.opt;
+               opt = np->cork.opt;
                transhdrlen = 0;
                exthdrlen = 0;
                mtu = inet->cork.fragsize;
@@ -1406,9 +1438,15 @@ error:
 
 static void ip6_cork_release(struct inet_sock *inet, struct ipv6_pinfo *np)
 {
-       inet->cork.flags &= ~IPCORK_OPT;
-       kfree(np->cork.opt);
-       np->cork.opt = NULL;
+       if (np->cork.opt) {
+               kfree(np->cork.opt->dst0opt);
+               kfree(np->cork.opt->dst1opt);
+               kfree(np->cork.opt->hopopt);
+               kfree(np->cork.opt->srcrt);
+               kfree(np->cork.opt);
+               np->cork.opt = NULL;
+       }
+
        if (inet->cork.dst) {
                dst_release(inet->cork.dst);
                inet->cork.dst = NULL;
@@ -1477,10 +1515,10 @@ int ip6_push_pending_frames(struct sock *sk)
        skb->priority = sk->sk_priority;
        skb->mark = sk->sk_mark;
 
-       skb->dst = dst_clone(&rt->u.dst);
-       IP6_INC_STATS(net, rt->rt6i_idev, IPSTATS_MIB_OUTREQUESTS);
+       skb_dst_set(skb, dst_clone(&rt->u.dst));
+       IP6_UPD_PO_STATS(net, rt->rt6i_idev, IPSTATS_MIB_OUT, skb->len);
        if (proto == IPPROTO_ICMPV6) {
-               struct inet6_dev *idev = ip6_dst_idev(skb->dst);
+               struct inet6_dev *idev = ip6_dst_idev(skb_dst(skb));
 
                ICMP6MSGOUT_INC_STATS_BH(net, idev, icmp6_hdr(skb)->icmp6_type);
                ICMP6_INC_STATS_BH(net, idev, ICMP6_MIB_OUTMSGS);
@@ -1506,8 +1544,8 @@ void ip6_flush_pending_frames(struct sock *sk)
        struct sk_buff *skb;
 
        while ((skb = __skb_dequeue_tail(&sk->sk_write_queue)) != NULL) {
-               if (skb->dst)
-                       IP6_INC_STATS(sock_net(sk), ip6_dst_idev(skb->dst),
+               if (skb_dst(skb))
+                       IP6_INC_STATS(sock_net(sk), ip6_dst_idev(skb_dst(skb)),
                                      IPSTATS_MIB_OUTDISCARDS);
                kfree_skb(skb);
        }