Merge branch 'llseek' of git://git.kernel.org/pub/scm/linux/kernel/git/arnd/bkl
[linux-2.6.git] / net / ipv4 / inet_timewait_sock.c
index 75c2def..c5af909 100644 (file)
  */
 
 #include <linux/kernel.h>
+#include <linux/kmemcheck.h>
+#include <linux/slab.h>
 #include <net/inet_hashtables.h>
 #include <net/inet_timewait_sock.h>
 #include <net/ip.h>
 
+
+/**
+ *     inet_twsk_unhash - unhash a timewait socket from established hash
+ *     @tw: timewait socket
+ *
+ *     unhash a timewait socket from established hash, if hashed.
+ *     ehash lock must be held by caller.
+ *     Returns 1 if caller should call inet_twsk_put() after lock release.
+ */
+int inet_twsk_unhash(struct inet_timewait_sock *tw)
+{
+       if (hlist_nulls_unhashed(&tw->tw_node))
+               return 0;
+
+       hlist_nulls_del_rcu(&tw->tw_node);
+       sk_nulls_node_init(&tw->tw_node);
+       /*
+        * We cannot call inet_twsk_put() ourself under lock,
+        * caller must call it for us.
+        */
+       return 1;
+}
+
+/**
+ *     inet_twsk_bind_unhash - unhash a timewait socket from bind hash
+ *     @tw: timewait socket
+ *     @hashinfo: hashinfo pointer
+ *
+ *     unhash a timewait socket from bind hash, if hashed.
+ *     bind hash lock must be held by caller.
+ *     Returns 1 if caller should call inet_twsk_put() after lock release.
+ */
+int inet_twsk_bind_unhash(struct inet_timewait_sock *tw,
+                         struct inet_hashinfo *hashinfo)
+{
+       struct inet_bind_bucket *tb = tw->tw_tb;
+
+       if (!tb)
+               return 0;
+
+       __hlist_del(&tw->tw_bind_node);
+       tw->tw_tb = NULL;
+       inet_bind_bucket_destroy(hashinfo->bind_bucket_cachep, tb);
+       /*
+        * We cannot call inet_twsk_put() ourself under lock,
+        * caller must call it for us.
+        */
+       return 1;
+}
+
 /* Must be called with locally disabled BHs. */
 static void __inet_twsk_kill(struct inet_timewait_sock *tw,
                             struct inet_hashinfo *hashinfo)
 {
        struct inet_bind_hashbucket *bhead;
-       struct inet_bind_bucket *tb;
+       int refcnt;
        /* Unlink from established hashes. */
-       rwlock_t *lock = inet_ehash_lockp(hashinfo, tw->tw_hash);
+       spinlock_t *lock = inet_ehash_lockp(hashinfo, tw->tw_hash);
 
-       write_lock(lock);
-       if (hlist_unhashed(&tw->tw_node)) {
-               write_unlock(lock);
-               return;
-       }
-       __hlist_del(&tw->tw_node);
-       sk_node_init(&tw->tw_node);
-       write_unlock(lock);
+       spin_lock(lock);
+       refcnt = inet_twsk_unhash(tw);
+       spin_unlock(lock);
 
        /* Disassociate with bind bucket. */
        bhead = &hashinfo->bhash[inet_bhashfn(twsk_net(tw), tw->tw_num,
                        hashinfo->bhash_size)];
+
        spin_lock(&bhead->lock);
-       tb = tw->tw_tb;
-       __hlist_del(&tw->tw_bind_node);
-       tw->tw_tb = NULL;
-       inet_bind_bucket_destroy(hashinfo->bind_bucket_cachep, tb);
+       refcnt += inet_twsk_bind_unhash(tw, hashinfo);
        spin_unlock(&bhead->lock);
+
 #ifdef SOCK_REFCNT_DEBUG
        if (atomic_read(&tw->tw_refcnt) != 1) {
                printk(KERN_DEBUG "%s timewait_sock %p refcnt=%d\n",
                       tw->tw_prot->name, tw, atomic_read(&tw->tw_refcnt));
        }
 #endif
-       inet_twsk_put(tw);
+       while (refcnt) {
+               inet_twsk_put(tw);
+               refcnt--;
+       }
 }
 
-void inet_twsk_put(struct inet_timewait_sock *tw)
+static noinline void inet_twsk_free(struct inet_timewait_sock *tw)
 {
-       if (atomic_dec_and_test(&tw->tw_refcnt)) {
-               struct module *owner = tw->tw_prot->owner;
-               twsk_destructor((struct sock *)tw);
+       struct module *owner = tw->tw_prot->owner;
+       twsk_destructor((struct sock *)tw);
 #ifdef SOCK_REFCNT_DEBUG
-               printk(KERN_DEBUG "%s timewait_sock %p released\n",
-                      tw->tw_prot->name, tw);
+       pr_debug("%s timewait_sock %p released\n", tw->tw_prot->name, tw);
 #endif
-               release_net(twsk_net(tw));
-               kmem_cache_free(tw->tw_prot->twsk_prot->twsk_slab, tw);
-               module_put(owner);
-       }
+       release_net(twsk_net(tw));
+       kmem_cache_free(tw->tw_prot->twsk_prot->twsk_slab, tw);
+       module_put(owner);
+}
+
+void inet_twsk_put(struct inet_timewait_sock *tw)
+{
+       if (atomic_dec_and_test(&tw->tw_refcnt))
+               inet_twsk_free(tw);
 }
 EXPORT_SYMBOL_GPL(inet_twsk_put);
 
@@ -76,33 +128,45 @@ void __inet_twsk_hashdance(struct inet_timewait_sock *tw, struct sock *sk,
        const struct inet_sock *inet = inet_sk(sk);
        const struct inet_connection_sock *icsk = inet_csk(sk);
        struct inet_ehash_bucket *ehead = inet_ehash_bucket(hashinfo, sk->sk_hash);
-       rwlock_t *lock = inet_ehash_lockp(hashinfo, sk->sk_hash);
+       spinlock_t *lock = inet_ehash_lockp(hashinfo, sk->sk_hash);
        struct inet_bind_hashbucket *bhead;
        /* Step 1: Put TW into bind hash. Original socket stays there too.
           Note, that any socket with inet->num != 0 MUST be bound in
           binding cache, even if it is closed.
         */
-       bhead = &hashinfo->bhash[inet_bhashfn(twsk_net(tw), inet->num,
+       bhead = &hashinfo->bhash[inet_bhashfn(twsk_net(tw), inet->inet_num,
                        hashinfo->bhash_size)];
        spin_lock(&bhead->lock);
        tw->tw_tb = icsk->icsk_bind_hash;
-       BUG_TRAP(icsk->icsk_bind_hash);
+       WARN_ON(!icsk->icsk_bind_hash);
        inet_twsk_add_bind_node(tw, &tw->tw_tb->owners);
        spin_unlock(&bhead->lock);
 
-       write_lock(lock);
+       spin_lock(lock);
+
+       /*
+        * Step 2: Hash TW into TIMEWAIT chain.
+        * Should be done before removing sk from established chain
+        * because readers are lockless and search established first.
+        */
+       inet_twsk_add_node_rcu(tw, &ehead->twchain);
 
-       /* Step 2: Remove SK from established hash. */
-       if (__sk_del_node_init(sk))
+       /* Step 3: Remove SK from established hash. */
+       if (__sk_nulls_del_node_init_rcu(sk))
                sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
 
-       /* Step 3: Hash TW into TIMEWAIT chain. */
-       inet_twsk_add_node(tw, &ehead->twchain);
-       atomic_inc(&tw->tw_refcnt);
+       /*
+        * Notes :
+        * - We initially set tw_refcnt to 0 in inet_twsk_alloc()
+        * - We add one reference for the bhash link
+        * - We add one reference for the ehash link
+        * - We want this refcnt update done before allowing other
+        *   threads to find this tw in ehash chain.
+        */
+       atomic_add(1 + 1 + 1, &tw->tw_refcnt);
 
-       write_unlock(lock);
+       spin_unlock(lock);
 }
-
 EXPORT_SYMBOL_GPL(__inet_twsk_hashdance);
 
 struct inet_timewait_sock *inet_twsk_alloc(const struct sock *sk, const int state)
@@ -113,29 +177,36 @@ struct inet_timewait_sock *inet_twsk_alloc(const struct sock *sk, const int stat
        if (tw != NULL) {
                const struct inet_sock *inet = inet_sk(sk);
 
+               kmemcheck_annotate_bitfield(tw, flags);
+
                /* Give us an identity. */
-               tw->tw_daddr        = inet->daddr;
-               tw->tw_rcv_saddr    = inet->rcv_saddr;
+               tw->tw_daddr        = inet->inet_daddr;
+               tw->tw_rcv_saddr    = inet->inet_rcv_saddr;
                tw->tw_bound_dev_if = sk->sk_bound_dev_if;
-               tw->tw_num          = inet->num;
+               tw->tw_num          = inet->inet_num;
                tw->tw_state        = TCP_TIME_WAIT;
                tw->tw_substate     = state;
-               tw->tw_sport        = inet->sport;
-               tw->tw_dport        = inet->dport;
+               tw->tw_sport        = inet->inet_sport;
+               tw->tw_dport        = inet->inet_dport;
                tw->tw_family       = sk->sk_family;
                tw->tw_reuse        = sk->sk_reuse;
                tw->tw_hash         = sk->sk_hash;
                tw->tw_ipv6only     = 0;
+               tw->tw_transparent  = inet->transparent;
                tw->tw_prot         = sk->sk_prot_creator;
                twsk_net_set(tw, hold_net(sock_net(sk)));
-               atomic_set(&tw->tw_refcnt, 1);
+               /*
+                * Because we use RCU lookups, we should not set tw_refcnt
+                * to a non null value before everything is setup for this
+                * timewait socket.
+                */
+               atomic_set(&tw->tw_refcnt, 0);
                inet_twsk_dead_node_init(tw);
                __module_get(tw->tw_prot->owner);
        }
 
        return tw;
 }
-
 EXPORT_SYMBOL_GPL(inet_twsk_alloc);
 
 /* Returns non-zero if quota exceeded.  */
@@ -207,14 +278,13 @@ void inet_twdr_hangman(unsigned long data)
                /* We purged the entire slot, anything left?  */
                if (twdr->tw_count)
                        need_timer = 1;
+               twdr->slot = ((twdr->slot + 1) & (INET_TWDR_TWKILL_SLOTS - 1));
        }
-       twdr->slot = ((twdr->slot + 1) & (INET_TWDR_TWKILL_SLOTS - 1));
        if (need_timer)
                mod_timer(&twdr->tw_timer, jiffies + twdr->period);
 out:
        spin_unlock(&twdr->death_lock);
 }
-
 EXPORT_SYMBOL_GPL(inet_twdr_hangman);
 
 void inet_twdr_twkill_work(struct work_struct *work)
@@ -245,7 +315,6 @@ void inet_twdr_twkill_work(struct work_struct *work)
                spin_unlock_bh(&twdr->death_lock);
        }
 }
-
 EXPORT_SYMBOL_GPL(inet_twdr_twkill_work);
 
 /* These are always called from BH context.  See callers in
@@ -265,7 +334,6 @@ void inet_twsk_deschedule(struct inet_timewait_sock *tw,
        spin_unlock(&twdr->death_lock);
        __inet_twsk_kill(tw, twdr->hashinfo);
 }
-
 EXPORT_SYMBOL(inet_twsk_deschedule);
 
 void inet_twsk_schedule(struct inet_timewait_sock *tw,
@@ -346,7 +414,6 @@ void inet_twsk_schedule(struct inet_timewait_sock *tw,
                mod_timer(&twdr->tw_timer, jiffies + twdr->period);
        spin_unlock(&twdr->death_lock);
 }
-
 EXPORT_SYMBOL_GPL(inet_twsk_schedule);
 
 void inet_twdr_twcal_tick(unsigned long data)
@@ -407,5 +474,48 @@ out:
 #endif
        spin_unlock(&twdr->death_lock);
 }
-
 EXPORT_SYMBOL_GPL(inet_twdr_twcal_tick);
+
+void inet_twsk_purge(struct inet_hashinfo *hashinfo,
+                    struct inet_timewait_death_row *twdr, int family)
+{
+       struct inet_timewait_sock *tw;
+       struct sock *sk;
+       struct hlist_nulls_node *node;
+       unsigned int slot;
+
+       for (slot = 0; slot <= hashinfo->ehash_mask; slot++) {
+               struct inet_ehash_bucket *head = &hashinfo->ehash[slot];
+restart_rcu:
+               rcu_read_lock();
+restart:
+               sk_nulls_for_each_rcu(sk, node, &head->twchain) {
+                       tw = inet_twsk(sk);
+                       if ((tw->tw_family != family) ||
+                               atomic_read(&twsk_net(tw)->count))
+                               continue;
+
+                       if (unlikely(!atomic_inc_not_zero(&tw->tw_refcnt)))
+                               continue;
+
+                       if (unlikely((tw->tw_family != family) ||
+                                    atomic_read(&twsk_net(tw)->count))) {
+                               inet_twsk_put(tw);
+                               goto restart;
+                       }
+
+                       rcu_read_unlock();
+                       inet_twsk_deschedule(tw, twdr);
+                       inet_twsk_put(tw);
+                       goto restart_rcu;
+               }
+               /* If the nulls value we got at the end of this lookup is
+                * not the expected one, we must restart lookup.
+                * We probably met an item that was moved to another chain.
+                */
+               if (get_nulls_value(node) != slot)
+                       goto restart;
+               rcu_read_unlock();
+       }
+}
+EXPORT_SYMBOL_GPL(inet_twsk_purge);