lglock: introduce special lglock and brlock spin locks
Nick Piggin [Tue, 17 Aug 2010 18:37:37 +0000 (04:37 +1000)]
lglock: introduce special lglock and brlock spin locks

This patch introduces "local-global" locks (lglocks). These can be used to:

- Provide fast exclusive access to per-CPU data, with exclusive access to
  another CPU's data allowed but possibly subject to contention, and to provide
  very slow exclusive access to all per-CPU data.
- Or to provide very fast and scalable read serialisation, and to provide
  very slow exclusive serialisation of data (not necessarily per-CPU data).

Brlocks are also implemented as a short-hand notation for the latter use
case.

Thanks to Paul for local/global naming convention.

Cc: linux-kernel@vger.kernel.org
Cc: Al Viro <viro@ZenIV.linux.org.uk>
Cc: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
Signed-off-by: Nick Piggin <npiggin@kernel.dk>
Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>

include/linux/lglock.h [new file with mode: 0644]

diff --git a/include/linux/lglock.h b/include/linux/lglock.h
new file mode 100644 (file)
index 0000000..b288cb7
--- /dev/null
@@ -0,0 +1,172 @@
+/*
+ * Specialised local-global spinlock. Can only be declared as global variables
+ * to avoid overhead and keep things simple (and we don't want to start using
+ * these inside dynamically allocated structures).
+ *
+ * "local/global locks" (lglocks) can be used to:
+ *
+ * - Provide fast exclusive access to per-CPU data, with exclusive access to
+ *   another CPU's data allowed but possibly subject to contention, and to
+ *   provide very slow exclusive access to all per-CPU data.
+ * - Or to provide very fast and scalable read serialisation, and to provide
+ *   very slow exclusive serialisation of data (not necessarily per-CPU data).
+ *
+ * Brlocks are also implemented as a short-hand notation for the latter use
+ * case.
+ *
+ * Copyright 2009, 2010, Nick Piggin, Novell Inc.
+ */
+#ifndef __LINUX_LGLOCK_H
+#define __LINUX_LGLOCK_H
+
+#include <linux/spinlock.h>
+#include <linux/lockdep.h>
+#include <linux/percpu.h>
+
+/* can make br locks by using local lock for read side, global lock for write */
+#define br_lock_init(name)     name##_lock_init()
+#define br_read_lock(name)     name##_local_lock()
+#define br_read_unlock(name)   name##_local_unlock()
+#define br_write_lock(name)    name##_global_lock_online()
+#define br_write_unlock(name)  name##_global_unlock_online()
+
+#define DECLARE_BRLOCK(name)   DECLARE_LGLOCK(name)
+#define DEFINE_BRLOCK(name)    DEFINE_LGLOCK(name)
+
+
+#define lg_lock_init(name)     name##_lock_init()
+#define lg_local_lock(name)    name##_local_lock()
+#define lg_local_unlock(name)  name##_local_unlock()
+#define lg_local_lock_cpu(name, cpu)   name##_local_lock_cpu(cpu)
+#define lg_local_unlock_cpu(name, cpu) name##_local_unlock_cpu(cpu)
+#define lg_global_lock(name)   name##_global_lock()
+#define lg_global_unlock(name) name##_global_unlock()
+#define lg_global_lock_online(name) name##_global_lock_online()
+#define lg_global_unlock_online(name) name##_global_unlock_online()
+
+#ifdef CONFIG_DEBUG_LOCK_ALLOC
+#define LOCKDEP_INIT_MAP lockdep_init_map
+
+#define DEFINE_LGLOCK_LOCKDEP(name)                                    \
+ struct lock_class_key name##_lock_key;                                        \
+ struct lockdep_map name##_lock_dep_map;                               \
+ EXPORT_SYMBOL(name##_lock_dep_map)
+
+#else
+#define LOCKDEP_INIT_MAP(a, b, c, d)
+
+#define DEFINE_LGLOCK_LOCKDEP(name)
+#endif
+
+
+#define DECLARE_LGLOCK(name)                                           \
+ extern void name##_lock_init(void);                                   \
+ extern void name##_local_lock(void);                                  \
+ extern void name##_local_unlock(void);                                        \
+ extern void name##_local_lock_cpu(int cpu);                           \
+ extern void name##_local_unlock_cpu(int cpu);                         \
+ extern void name##_global_lock(void);                                 \
+ extern void name##_global_unlock(void);                               \
+ extern void name##_global_lock_online(void);                          \
+ extern void name##_global_unlock_online(void);                                \
+
+#define DEFINE_LGLOCK(name)                                            \
+                                                                       \
+ DEFINE_PER_CPU(arch_spinlock_t, name##_lock);                         \
+ DEFINE_LGLOCK_LOCKDEP(name);                                          \
+                                                                       \
+ void name##_lock_init(void) {                                         \
+       int i;                                                          \
+       LOCKDEP_INIT_MAP(&name##_lock_dep_map, #name, &name##_lock_key, 0); \
+       for_each_possible_cpu(i) {                                      \
+               arch_spinlock_t *lock;                                  \
+               lock = &per_cpu(name##_lock, i);                        \
+               *lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;     \
+       }                                                               \
+ }                                                                     \
+ EXPORT_SYMBOL(name##_lock_init);                                      \
+                                                                       \
+ void name##_local_lock(void) {                                                \
+       arch_spinlock_t *lock;                                          \
+       preempt_disable();                                              \
+       rwlock_acquire_read(&name##_lock_dep_map, 0, 0, _THIS_IP_);     \
+       lock = &__get_cpu_var(name##_lock);                             \
+       arch_spin_lock(lock);                                           \
+ }                                                                     \
+ EXPORT_SYMBOL(name##_local_lock);                                     \
+                                                                       \
+ void name##_local_unlock(void) {                                      \
+       arch_spinlock_t *lock;                                          \
+       rwlock_release(&name##_lock_dep_map, 1, _THIS_IP_);             \
+       lock = &__get_cpu_var(name##_lock);                             \
+       arch_spin_unlock(lock);                                         \
+       preempt_enable();                                               \
+ }                                                                     \
+ EXPORT_SYMBOL(name##_local_unlock);                                   \
+                                                                       \
+ void name##_local_lock_cpu(int cpu) {                                 \
+       arch_spinlock_t *lock;                                          \
+       preempt_disable();                                              \
+       rwlock_acquire_read(&name##_lock_dep_map, 0, 0, _THIS_IP_);     \
+       lock = &per_cpu(name##_lock, cpu);                              \
+       arch_spin_lock(lock);                                           \
+ }                                                                     \
+ EXPORT_SYMBOL(name##_local_lock_cpu);                                 \
+                                                                       \
+ void name##_local_unlock_cpu(int cpu) {                               \
+       arch_spinlock_t *lock;                                          \
+       rwlock_release(&name##_lock_dep_map, 1, _THIS_IP_);             \
+       lock = &per_cpu(name##_lock, cpu);                              \
+       arch_spin_unlock(lock);                                         \
+       preempt_enable();                                               \
+ }                                                                     \
+ EXPORT_SYMBOL(name##_local_unlock_cpu);                               \
+                                                                       \
+ void name##_global_lock_online(void) {                                        \
+       int i;                                                          \
+       preempt_disable();                                              \
+       rwlock_acquire(&name##_lock_dep_map, 0, 0, _RET_IP_);           \
+       for_each_online_cpu(i) {                                        \
+               arch_spinlock_t *lock;                                  \
+               lock = &per_cpu(name##_lock, i);                        \
+               arch_spin_lock(lock);                                   \
+       }                                                               \
+ }                                                                     \
+ EXPORT_SYMBOL(name##_global_lock_online);                             \
+                                                                       \
+ void name##_global_unlock_online(void) {                              \
+       int i;                                                          \
+       rwlock_release(&name##_lock_dep_map, 1, _RET_IP_);              \
+       for_each_online_cpu(i) {                                        \
+               arch_spinlock_t *lock;                                  \
+               lock = &per_cpu(name##_lock, i);                        \
+               arch_spin_unlock(lock);                                 \
+       }                                                               \
+       preempt_enable();                                               \
+ }                                                                     \
+ EXPORT_SYMBOL(name##_global_unlock_online);                           \
+                                                                       \
+ void name##_global_lock(void) {                                       \
+       int i;                                                          \
+       preempt_disable();                                              \
+       rwlock_acquire(&name##_lock_dep_map, 0, 0, _RET_IP_);           \
+       for_each_online_cpu(i) {                                        \
+               arch_spinlock_t *lock;                                  \
+               lock = &per_cpu(name##_lock, i);                        \
+               arch_spin_lock(lock);                                   \
+       }                                                               \
+ }                                                                     \
+ EXPORT_SYMBOL(name##_global_lock);                                    \
+                                                                       \
+ void name##_global_unlock(void) {                                     \
+       int i;                                                          \
+       rwlock_release(&name##_lock_dep_map, 1, _RET_IP_);              \
+       for_each_online_cpu(i) {                                        \
+               arch_spinlock_t *lock;                                  \
+               lock = &per_cpu(name##_lock, i);                        \
+               arch_spin_unlock(lock);                                 \
+       }                                                               \
+       preempt_enable();                                               \
+ }                                                                     \
+ EXPORT_SYMBOL(name##_global_unlock);
+#endif