cgroups: clean up cgroup_pidlist_find() a bit
[linux-2.6.git] / kernel / lockdep.c
index 28914a5..0c30d04 100644 (file)
@@ -49,7 +49,7 @@
 #include "lockdep_internals.h"
 
 #define CREATE_TRACE_POINTS
-#include <trace/events/lockdep.h>
+#include <trace/events/lock.h>
 
 #ifdef CONFIG_PROVE_LOCKING
 int prove_locking = 1;
@@ -73,11 +73,11 @@ module_param(lock_stat, int, 0644);
  * to use a raw spinlock - we really dont want the spinlock
  * code to recurse back into the lockdep code...
  */
-static raw_spinlock_t lockdep_lock = (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
+static arch_spinlock_t lockdep_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
 
 static int graph_lock(void)
 {
-       __raw_spin_lock(&lockdep_lock);
+       arch_spin_lock(&lockdep_lock);
        /*
         * Make sure that if another CPU detected a bug while
         * walking the graph we dont change it (while the other
@@ -85,7 +85,7 @@ static int graph_lock(void)
         * dropped already)
         */
        if (!debug_locks) {
-               __raw_spin_unlock(&lockdep_lock);
+               arch_spin_unlock(&lockdep_lock);
                return 0;
        }
        /* prevent any recursions within lockdep from causing deadlocks */
@@ -95,11 +95,11 @@ static int graph_lock(void)
 
 static inline int graph_unlock(void)
 {
-       if (debug_locks && !__raw_spin_is_locked(&lockdep_lock))
+       if (debug_locks && !arch_spin_is_locked(&lockdep_lock))
                return DEBUG_LOCKS_WARN_ON(1);
 
        current->lockdep_recursion--;
-       __raw_spin_unlock(&lockdep_lock);
+       arch_spin_unlock(&lockdep_lock);
        return 0;
 }
 
@@ -111,7 +111,7 @@ static inline int debug_locks_off_graph_unlock(void)
 {
        int ret = debug_locks_off();
 
-       __raw_spin_unlock(&lockdep_lock);
+       arch_spin_unlock(&lockdep_lock);
 
        return ret;
 }
@@ -140,7 +140,13 @@ static inline struct lock_class *hlock_class(struct held_lock *hlock)
 }
 
 #ifdef CONFIG_LOCK_STAT
-static DEFINE_PER_CPU(struct lock_class_stats[MAX_LOCKDEP_KEYS], lock_stats);
+static DEFINE_PER_CPU(struct lock_class_stats[MAX_LOCKDEP_KEYS],
+                     cpu_lock_stats);
+
+static inline u64 lockstat_clock(void)
+{
+       return cpu_clock(smp_processor_id());
+}
 
 static int lock_point(unsigned long points[], unsigned long ip)
 {
@@ -158,12 +164,12 @@ static int lock_point(unsigned long points[], unsigned long ip)
        return i;
 }
 
-static void lock_time_inc(struct lock_time *lt, s64 time)
+static void lock_time_inc(struct lock_time *lt, u64 time)
 {
        if (time > lt->max)
                lt->max = time;
 
-       if (time < lt->min || !lt->min)
+       if (time < lt->min || !lt->nr)
                lt->min = time;
 
        lt->total += time;
@@ -172,8 +178,15 @@ static void lock_time_inc(struct lock_time *lt, s64 time)
 
 static inline void lock_time_add(struct lock_time *src, struct lock_time *dst)
 {
-       dst->min += src->min;
-       dst->max += src->max;
+       if (!src->nr)
+               return;
+
+       if (src->max > dst->max)
+               dst->max = src->max;
+
+       if (src->min < dst->min || !dst->nr)
+               dst->min = src->min;
+
        dst->total += src->total;
        dst->nr += src->nr;
 }
@@ -186,7 +199,7 @@ struct lock_class_stats lock_stats(struct lock_class *class)
        memset(&stats, 0, sizeof(struct lock_class_stats));
        for_each_possible_cpu(cpu) {
                struct lock_class_stats *pcs =
-                       &per_cpu(lock_stats, cpu)[class - lock_classes];
+                       &per_cpu(cpu_lock_stats, cpu)[class - lock_classes];
 
                for (i = 0; i < ARRAY_SIZE(stats.contention_point); i++)
                        stats.contention_point[i] += pcs->contention_point[i];
@@ -213,7 +226,7 @@ void clear_lock_stats(struct lock_class *class)
 
        for_each_possible_cpu(cpu) {
                struct lock_class_stats *cpu_stats =
-                       &per_cpu(lock_stats, cpu)[class - lock_classes];
+                       &per_cpu(cpu_lock_stats, cpu)[class - lock_classes];
 
                memset(cpu_stats, 0, sizeof(struct lock_class_stats));
        }
@@ -223,23 +236,23 @@ void clear_lock_stats(struct lock_class *class)
 
 static struct lock_class_stats *get_lock_stats(struct lock_class *class)
 {
-       return &get_cpu_var(lock_stats)[class - lock_classes];
+       return &get_cpu_var(cpu_lock_stats)[class - lock_classes];
 }
 
 static void put_lock_stats(struct lock_class_stats *stats)
 {
-       put_cpu_var(lock_stats);
+       put_cpu_var(cpu_lock_stats);
 }
 
 static void lock_release_holdtime(struct held_lock *hlock)
 {
        struct lock_class_stats *stats;
-       s64 holdtime;
+       u64 holdtime;
 
        if (!lock_stat)
                return;
 
-       holdtime = sched_clock() - hlock->holdtime_stamp;
+       holdtime = lockstat_clock() - hlock->holdtime_stamp;
 
        stats = get_lock_stats(hlock_class(hlock));
        if (hlock->read)
@@ -374,7 +387,8 @@ static int save_trace(struct stack_trace *trace)
         * complete trace that maxes out the entries provided will be reported
         * as incomplete, friggin useless </rant>
         */
-       if (trace->entries[trace->nr_entries-1] == ULONG_MAX)
+       if (trace->nr_entries != 0 &&
+           trace->entries[trace->nr_entries-1] == ULONG_MAX)
                trace->nr_entries--;
 
        trace->max_entries = trace->nr_entries;
@@ -399,7 +413,6 @@ unsigned int nr_hardirq_chains;
 unsigned int nr_softirq_chains;
 unsigned int nr_process_chains;
 unsigned int max_lockdep_depth;
-unsigned int max_recursion_depth;
 
 #ifdef CONFIG_DEBUG_LOCKDEP
 /*
@@ -429,11 +442,8 @@ atomic_t redundant_softirqs_on;
 atomic_t redundant_softirqs_off;
 atomic_t nr_unused_locks;
 atomic_t nr_cyclic_checks;
-atomic_t nr_cyclic_check_recursions;
 atomic_t nr_find_usage_forwards_checks;
-atomic_t nr_find_usage_forwards_recursions;
 atomic_t nr_find_usage_backwards_checks;
-atomic_t nr_find_usage_backwards_recursions;
 #endif
 
 /*
@@ -582,6 +592,9 @@ static int static_obj(void *obj)
        if ((addr >= start) && (addr < end))
                return 1;
 
+       if (arch_is_kernel_data(addr))
+               return 1;
+
 #ifdef CONFIG_SMP
        /*
         * percpu var?
@@ -861,14 +874,15 @@ struct circular_queue {
 };
 
 static struct circular_queue lock_cq;
-static unsigned long bfs_accessed[BITS_TO_LONGS(MAX_LOCKDEP_ENTRIES)];
 
 unsigned int max_bfs_queue_depth;
 
+static unsigned int lockdep_dependency_gen_id;
+
 static inline void __cq_init(struct circular_queue *cq)
 {
        cq->front = cq->rear = 0;
-       bitmap_zero(bfs_accessed, MAX_LOCKDEP_ENTRIES);
+       lockdep_dependency_gen_id++;
 }
 
 static inline int __cq_empty(struct circular_queue *cq)
@@ -914,7 +928,7 @@ static inline void mark_lock_accessed(struct lock_list *lock,
        nr = lock - list_entries;
        WARN_ON(nr >= nr_list_entries);
        lock->parent = parent;
-       set_bit(nr, bfs_accessed);
+       lock->class->dep_gen_id = lockdep_dependency_gen_id;
 }
 
 static inline unsigned long lock_accessed(struct lock_list *lock)
@@ -923,7 +937,7 @@ static inline unsigned long lock_accessed(struct lock_list *lock)
 
        nr = lock - list_entries;
        WARN_ON(nr >= nr_list_entries);
-       return test_bit(nr, bfs_accessed);
+       return lock->class->dep_gen_id == lockdep_dependency_gen_id;
 }
 
 static inline struct lock_list *get_lock_parent(struct lock_list *child)
@@ -1156,9 +1170,9 @@ unsigned long lockdep_count_forward_deps(struct lock_class *class)
        this.class = class;
 
        local_irq_save(flags);
-       __raw_spin_lock(&lockdep_lock);
+       arch_spin_lock(&lockdep_lock);
        ret = __lockdep_count_forward_deps(&this);
-       __raw_spin_unlock(&lockdep_lock);
+       arch_spin_unlock(&lockdep_lock);
        local_irq_restore(flags);
 
        return ret;
@@ -1183,9 +1197,9 @@ unsigned long lockdep_count_backward_deps(struct lock_class *class)
        this.class = class;
 
        local_irq_save(flags);
-       __raw_spin_lock(&lockdep_lock);
+       arch_spin_lock(&lockdep_lock);
        ret = __lockdep_count_backward_deps(&this);
-       __raw_spin_unlock(&lockdep_lock);
+       arch_spin_unlock(&lockdep_lock);
        local_irq_restore(flags);
 
        return ret;
@@ -2133,7 +2147,7 @@ check_usage_backwards(struct task_struct *curr, struct held_lock *this,
                return ret;
 
        return print_irq_inversion_bug(curr, &root, target_entry,
-                                       this, 1, irqclass);
+                                       this, 0, irqclass);
 }
 
 void print_irqtrace_events(struct task_struct *curr)
@@ -2708,13 +2722,15 @@ EXPORT_SYMBOL_GPL(lockdep_init_map);
  */
 static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass,
                          int trylock, int read, int check, int hardirqs_off,
-                         struct lockdep_map *nest_lock, unsigned long ip)
+                         struct lockdep_map *nest_lock, unsigned long ip,
+                         int references)
 {
        struct task_struct *curr = current;
        struct lock_class *class = NULL;
        struct held_lock *hlock;
        unsigned int depth, id;
        int chain_head = 0;
+       int class_idx;
        u64 chain_key;
 
        if (!prove_locking)
@@ -2762,10 +2778,24 @@ static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass,
        if (DEBUG_LOCKS_WARN_ON(depth >= MAX_LOCK_DEPTH))
                return 0;
 
+       class_idx = class - lock_classes + 1;
+
+       if (depth) {
+               hlock = curr->held_locks + depth - 1;
+               if (hlock->class_idx == class_idx && nest_lock) {
+                       if (hlock->references)
+                               hlock->references++;
+                       else
+                               hlock->references = 2;
+
+                       return 1;
+               }
+       }
+
        hlock = curr->held_locks + depth;
        if (DEBUG_LOCKS_WARN_ON(!class))
                return 0;
-       hlock->class_idx = class - lock_classes + 1;
+       hlock->class_idx = class_idx;
        hlock->acquire_ip = ip;
        hlock->instance = lock;
        hlock->nest_lock = nest_lock;
@@ -2773,9 +2803,10 @@ static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass,
        hlock->read = read;
        hlock->check = check;
        hlock->hardirqs_off = !!hardirqs_off;
+       hlock->references = references;
 #ifdef CONFIG_LOCK_STAT
        hlock->waittime_stamp = 0;
-       hlock->holdtime_stamp = sched_clock();
+       hlock->holdtime_stamp = lockstat_clock();
 #endif
 
        if (check == 2 && !mark_irqflags(curr, hlock))
@@ -2881,6 +2912,30 @@ static int check_unlock(struct task_struct *curr, struct lockdep_map *lock,
        return 1;
 }
 
+static int match_held_lock(struct held_lock *hlock, struct lockdep_map *lock)
+{
+       if (hlock->instance == lock)
+               return 1;
+
+       if (hlock->references) {
+               struct lock_class *class = lock->class_cache;
+
+               if (!class)
+                       class = look_up_lock_class(lock, 0);
+
+               if (DEBUG_LOCKS_WARN_ON(!class))
+                       return 0;
+
+               if (DEBUG_LOCKS_WARN_ON(!hlock->nest_lock))
+                       return 0;
+
+               if (hlock->class_idx == class - lock_classes + 1)
+                       return 1;
+       }
+
+       return 0;
+}
+
 static int
 __lock_set_class(struct lockdep_map *lock, const char *name,
                 struct lock_class_key *key, unsigned int subclass,
@@ -2904,7 +2959,7 @@ __lock_set_class(struct lockdep_map *lock, const char *name,
                 */
                if (prev_hlock && prev_hlock->irq_context != hlock->irq_context)
                        break;
-               if (hlock->instance == lock)
+               if (match_held_lock(hlock, lock))
                        goto found_it;
                prev_hlock = hlock;
        }
@@ -2923,7 +2978,8 @@ found_it:
                if (!__lock_acquire(hlock->instance,
                        hlock_class(hlock)->subclass, hlock->trylock,
                                hlock->read, hlock->check, hlock->hardirqs_off,
-                               hlock->nest_lock, hlock->acquire_ip))
+                               hlock->nest_lock, hlock->acquire_ip,
+                               hlock->references))
                        return 0;
        }
 
@@ -2962,20 +3018,34 @@ lock_release_non_nested(struct task_struct *curr,
                 */
                if (prev_hlock && prev_hlock->irq_context != hlock->irq_context)
                        break;
-               if (hlock->instance == lock)
+               if (match_held_lock(hlock, lock))
                        goto found_it;
                prev_hlock = hlock;
        }
        return print_unlock_inbalance_bug(curr, lock, ip);
 
 found_it:
-       lock_release_holdtime(hlock);
+       if (hlock->instance == lock)
+               lock_release_holdtime(hlock);
+
+       if (hlock->references) {
+               hlock->references--;
+               if (hlock->references) {
+                       /*
+                        * We had, and after removing one, still have
+                        * references, the current lock stack is still
+                        * valid. We're done!
+                        */
+                       return 1;
+               }
+       }
 
        /*
         * We have the right lock to unlock, 'hlock' points to it.
         * Now we remove it from the stack, and add back the other
         * entries (if any), recalculating the hash along the way:
         */
+
        curr->lockdep_depth = i;
        curr->curr_chain_key = hlock->prev_chain_key;
 
@@ -2984,7 +3054,8 @@ found_it:
                if (!__lock_acquire(hlock->instance,
                        hlock_class(hlock)->subclass, hlock->trylock,
                                hlock->read, hlock->check, hlock->hardirqs_off,
-                               hlock->nest_lock, hlock->acquire_ip))
+                               hlock->nest_lock, hlock->acquire_ip,
+                               hlock->references))
                        return 0;
        }
 
@@ -3014,7 +3085,7 @@ static int lock_release_nested(struct task_struct *curr,
        /*
         * Is the unlock non-nested:
         */
-       if (hlock->instance != lock)
+       if (hlock->instance != lock || hlock->references)
                return lock_release_non_nested(curr, lock, ip);
        curr->lockdep_depth--;
 
@@ -3065,7 +3136,9 @@ static int __lock_is_held(struct lockdep_map *lock)
        int i;
 
        for (i = 0; i < curr->lockdep_depth; i++) {
-               if (curr->held_locks[i].instance == lock)
+               struct held_lock *hlock = curr->held_locks + i;
+
+               if (match_held_lock(hlock, lock))
                        return 1;
        }
 
@@ -3148,7 +3221,7 @@ void lock_acquire(struct lockdep_map *lock, unsigned int subclass,
 
        current->lockdep_recursion = 1;
        __lock_acquire(lock, subclass, trylock, read, check,
-                      irqs_disabled_flags(flags), nest_lock, ip);
+                      irqs_disabled_flags(flags), nest_lock, ip, 0);
        current->lockdep_recursion = 0;
        raw_local_irq_restore(flags);
 }
@@ -3252,7 +3325,7 @@ __lock_contended(struct lockdep_map *lock, unsigned long ip)
                 */
                if (prev_hlock && prev_hlock->irq_context != hlock->irq_context)
                        break;
-               if (hlock->instance == lock)
+               if (match_held_lock(hlock, lock))
                        goto found_it;
                prev_hlock = hlock;
        }
@@ -3260,7 +3333,10 @@ __lock_contended(struct lockdep_map *lock, unsigned long ip)
        return;
 
 found_it:
-       hlock->waittime_stamp = sched_clock();
+       if (hlock->instance != lock)
+               return;
+
+       hlock->waittime_stamp = lockstat_clock();
 
        contention_point = lock_point(hlock_class(hlock)->contention_point, ip);
        contending_point = lock_point(hlock_class(hlock)->contending_point,
@@ -3283,8 +3359,7 @@ __lock_acquired(struct lockdep_map *lock, unsigned long ip)
        struct held_lock *hlock, *prev_hlock;
        struct lock_class_stats *stats;
        unsigned int depth;
-       u64 now;
-       s64 waittime = 0;
+       u64 now, waittime = 0;
        int i, cpu;
 
        depth = curr->lockdep_depth;
@@ -3299,7 +3374,7 @@ __lock_acquired(struct lockdep_map *lock, unsigned long ip)
                 */
                if (prev_hlock && prev_hlock->irq_context != hlock->irq_context)
                        break;
-               if (hlock->instance == lock)
+               if (match_held_lock(hlock, lock))
                        goto found_it;
                prev_hlock = hlock;
        }
@@ -3307,9 +3382,12 @@ __lock_acquired(struct lockdep_map *lock, unsigned long ip)
        return;
 
 found_it:
+       if (hlock->instance != lock)
+               return;
+
        cpu = smp_processor_id();
        if (hlock->waittime_stamp) {
-               now = sched_clock();
+               now = lockstat_clock();
                waittime = now - hlock->waittime_stamp;
                hlock->holdtime_stamp = now;
        }
@@ -3537,10 +3615,11 @@ void __init lockdep_info(void)
                sizeof(struct list_head) * CLASSHASH_SIZE +
                sizeof(struct lock_list) * MAX_LOCKDEP_ENTRIES +
                sizeof(struct lock_chain) * MAX_LOCKDEP_CHAINS +
-               sizeof(struct list_head) * CHAINHASH_SIZE) / 1024
+               sizeof(struct list_head) * CHAINHASH_SIZE
 #ifdef CONFIG_PROVE_LOCKING
-               + sizeof(struct circular_queue) + sizeof(bfs_accessed)
+               + sizeof(struct circular_queue)
 #endif
+               ) / 1024
                );
 
        printk(" per task-struct memory footprint: %lu bytes\n",
@@ -3730,3 +3809,21 @@ void lockdep_sys_exit(void)
                lockdep_print_held_locks(curr);
        }
 }
+
+void lockdep_rcu_dereference(const char *file, const int line)
+{
+       struct task_struct *curr = current;
+
+       if (!debug_locks_off())
+               return;
+       printk("\n===================================================\n");
+       printk(  "[ INFO: suspicious rcu_dereference_check() usage. ]\n");
+       printk(  "---------------------------------------------------\n");
+       printk("%s:%d invoked rcu_dereference_check() without protection!\n",
+                       file, line);
+       printk("\nother info that might help us debug this:\n\n");
+       lockdep_print_held_locks(curr);
+       printk("\nstack backtrace:\n");
+       dump_stack();
+}
+EXPORT_SYMBOL_GPL(lockdep_rcu_dereference);