rcu: Permit call_rcu() from CPU_DYING notifiers
[linux-2.6.git] / kernel / lockdep_internals.h
index 1352409..4f560cf 100644 (file)
  * Lock-class usage-state bits:
  */
 enum lock_usage_bit {
-       LOCK_USED = 0,
-       LOCK_USED_IN_HARDIRQ,
-       LOCK_USED_IN_SOFTIRQ,
-       LOCK_USED_IN_RECLAIM_FS,
-       LOCK_ENABLED_SOFTIRQ,
-       LOCK_ENABLED_HARDIRQ,
-       LOCK_ENABLED_RECLAIM_FS,
-       LOCK_USED_IN_HARDIRQ_READ,
-       LOCK_USED_IN_SOFTIRQ_READ,
-       LOCK_USED_IN_RECLAIM_FS_READ,
-       LOCK_ENABLED_SOFTIRQ_READ,
-       LOCK_ENABLED_HARDIRQ_READ,
-       LOCK_ENABLED_RECLAIM_FS_READ,
+#define LOCKDEP_STATE(__STATE)         \
+       LOCK_USED_IN_##__STATE,         \
+       LOCK_USED_IN_##__STATE##_READ,  \
+       LOCK_ENABLED_##__STATE,         \
+       LOCK_ENABLED_##__STATE##_READ,
+#include "lockdep_states.h"
+#undef LOCKDEP_STATE
+       LOCK_USED,
        LOCK_USAGE_STATES
 };
 
 /*
  * Usage-state bitmasks:
  */
-#define LOCKF_USED                     (1 << LOCK_USED)
-#define LOCKF_USED_IN_HARDIRQ          (1 << LOCK_USED_IN_HARDIRQ)
-#define LOCKF_USED_IN_SOFTIRQ          (1 << LOCK_USED_IN_SOFTIRQ)
-#define LOCKF_USED_IN_RECLAIM_FS       (1 << LOCK_USED_IN_RECLAIM_FS)
-#define LOCKF_ENABLED_HARDIRQ          (1 << LOCK_ENABLED_HARDIRQ)
-#define LOCKF_ENABLED_SOFTIRQ          (1 << LOCK_ENABLED_SOFTIRQ)
-#define LOCKF_ENABLED_RECLAIM_FS       (1 << LOCK_ENABLED_RECLAIM_FS)
+#define __LOCKF(__STATE)       LOCKF_##__STATE = (1 << LOCK_##__STATE),
+
+enum {
+#define LOCKDEP_STATE(__STATE)                                         \
+       __LOCKF(USED_IN_##__STATE)                                      \
+       __LOCKF(USED_IN_##__STATE##_READ)                               \
+       __LOCKF(ENABLED_##__STATE)                                      \
+       __LOCKF(ENABLED_##__STATE##_READ)
+#include "lockdep_states.h"
+#undef LOCKDEP_STATE
+       __LOCKF(USED)
+};
 
 #define LOCKF_ENABLED_IRQ (LOCKF_ENABLED_HARDIRQ | LOCKF_ENABLED_SOFTIRQ)
 #define LOCKF_USED_IN_IRQ (LOCKF_USED_IN_HARDIRQ | LOCKF_USED_IN_SOFTIRQ)
 
-#define LOCKF_USED_IN_HARDIRQ_READ     (1 << LOCK_USED_IN_HARDIRQ_READ)
-#define LOCKF_USED_IN_SOFTIRQ_READ     (1 << LOCK_USED_IN_SOFTIRQ_READ)
-#define LOCKF_USED_IN_RECLAIM_FS_READ  (1 << LOCK_USED_IN_RECLAIM_FS_READ)
-#define LOCKF_ENABLED_HARDIRQ_READ     (1 << LOCK_ENABLED_HARDIRQ_READ)
-#define LOCKF_ENABLED_SOFTIRQ_READ     (1 << LOCK_ENABLED_SOFTIRQ_READ)
-#define LOCKF_ENABLED_RECLAIM_FS_READ  (1 << LOCK_ENABLED_RECLAIM_FS_READ)
-
 #define LOCKF_ENABLED_IRQ_READ \
                (LOCKF_ENABLED_HARDIRQ_READ | LOCKF_ENABLED_SOFTIRQ_READ)
 #define LOCKF_USED_IN_IRQ_READ \
@@ -61,9 +54,9 @@ enum lock_usage_bit {
  * table (if it's not there yet), and we check it for lock order
  * conflicts and deadlocks.
  */
-#define MAX_LOCKDEP_ENTRIES    8192UL
+#define MAX_LOCKDEP_ENTRIES    16384UL
 
-#define MAX_LOCKDEP_CHAINS_BITS        14
+#define MAX_LOCKDEP_CHAINS_BITS        15
 #define MAX_LOCKDEP_CHAINS     (1UL << MAX_LOCKDEP_CHAINS_BITS)
 
 #define MAX_LOCKDEP_CHAIN_HLOCKS (MAX_LOCKDEP_CHAINS*5)
@@ -77,9 +70,10 @@ enum lock_usage_bit {
 extern struct list_head all_lock_classes;
 extern struct lock_chain lock_chains[];
 
-extern void
-get_usage_chars(struct lock_class *class, char *c1, char *c2, char *c3,
-                                       char *c4, char *c5, char *c6);
+#define LOCK_USAGE_CHARS (1+LOCK_USAGE_STATES/2)
+
+extern void get_usage_chars(struct lock_class *class,
+                           char usage[LOCK_USAGE_CHARS]);
 
 extern const char * __get_key_name(struct lockdep_subclass_key *key, char *str);
 
@@ -97,6 +91,8 @@ extern unsigned int nr_process_chains;
 extern unsigned int max_lockdep_depth;
 extern unsigned int max_recursion_depth;
 
+extern unsigned int max_bfs_queue_depth;
+
 #ifdef CONFIG_PROVE_LOCKING
 extern unsigned long lockdep_count_forward_deps(struct lock_class *);
 extern unsigned long lockdep_count_backward_deps(struct lock_class *);
@@ -114,30 +110,60 @@ lockdep_count_backward_deps(struct lock_class *class)
 #endif
 
 #ifdef CONFIG_DEBUG_LOCKDEP
+
+#include <asm/local.h>
 /*
- * Various lockdep statistics:
+ * Various lockdep statistics.
+ * We want them per cpu as they are often accessed in fast path
+ * and we want to avoid too much cache bouncing.
  */
-extern atomic_t chain_lookup_hits;
-extern atomic_t chain_lookup_misses;
-extern atomic_t hardirqs_on_events;
-extern atomic_t hardirqs_off_events;
-extern atomic_t redundant_hardirqs_on;
-extern atomic_t redundant_hardirqs_off;
-extern atomic_t softirqs_on_events;
-extern atomic_t softirqs_off_events;
-extern atomic_t redundant_softirqs_on;
-extern atomic_t redundant_softirqs_off;
-extern atomic_t nr_unused_locks;
-extern atomic_t nr_cyclic_checks;
-extern atomic_t nr_cyclic_check_recursions;
-extern atomic_t nr_find_usage_forwards_checks;
-extern atomic_t nr_find_usage_forwards_recursions;
-extern atomic_t nr_find_usage_backwards_checks;
-extern atomic_t nr_find_usage_backwards_recursions;
-# define debug_atomic_inc(ptr)         atomic_inc(ptr)
-# define debug_atomic_dec(ptr)         atomic_dec(ptr)
-# define debug_atomic_read(ptr)                atomic_read(ptr)
+struct lockdep_stats {
+       int     chain_lookup_hits;
+       int     chain_lookup_misses;
+       int     hardirqs_on_events;
+       int     hardirqs_off_events;
+       int     redundant_hardirqs_on;
+       int     redundant_hardirqs_off;
+       int     softirqs_on_events;
+       int     softirqs_off_events;
+       int     redundant_softirqs_on;
+       int     redundant_softirqs_off;
+       int     nr_unused_locks;
+       int     nr_cyclic_checks;
+       int     nr_cyclic_check_recursions;
+       int     nr_find_usage_forwards_checks;
+       int     nr_find_usage_forwards_recursions;
+       int     nr_find_usage_backwards_checks;
+       int     nr_find_usage_backwards_recursions;
+};
+
+DECLARE_PER_CPU(struct lockdep_stats, lockdep_stats);
+
+#define __debug_atomic_inc(ptr)                                        \
+       this_cpu_inc(lockdep_stats.ptr);
+
+#define debug_atomic_inc(ptr)                  {               \
+       WARN_ON_ONCE(!irqs_disabled());                         \
+       __this_cpu_inc(lockdep_stats.ptr);                      \
+}
+
+#define debug_atomic_dec(ptr)                  {               \
+       WARN_ON_ONCE(!irqs_disabled());                         \
+       __this_cpu_dec(lockdep_stats.ptr);                      \
+}
+
+#define debug_atomic_read(ptr)         ({                              \
+       struct lockdep_stats *__cpu_lockdep_stats;                      \
+       unsigned long long __total = 0;                                 \
+       int __cpu;                                                      \
+       for_each_possible_cpu(__cpu) {                                  \
+               __cpu_lockdep_stats = &per_cpu(lockdep_stats, __cpu);   \
+               __total += __cpu_lockdep_stats->ptr;                    \
+       }                                                               \
+       __total;                                                        \
+})
 #else
+# define __debug_atomic_inc(ptr)       do { } while (0)
 # define debug_atomic_inc(ptr)         do { } while (0)
 # define debug_atomic_dec(ptr)         do { } while (0)
 # define debug_atomic_read(ptr)                0