fac3337547eb862e268587e082d885e0c01577d8
[linux-2.6.git] / include / linux / percpu.h
1 #ifndef __LINUX_PERCPU_H
2 #define __LINUX_PERCPU_H
3
4 #include <linux/preempt.h>
5 #include <linux/slab.h> /* For kmalloc() */
6 #include <linux/smp.h>
7 #include <linux/cpumask.h>
8
9 #include <asm/percpu.h>
10
11 #ifdef CONFIG_SMP
12 #define DEFINE_PER_CPU(type, name)                                      \
13         __attribute__((__section__(".data.percpu")))                    \
14         PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name
15
16 #ifdef MODULE
17 #define SHARED_ALIGNED_SECTION ".data.percpu"
18 #else
19 #define SHARED_ALIGNED_SECTION ".data.percpu.shared_aligned"
20 #endif
21
22 #define DEFINE_PER_CPU_SHARED_ALIGNED(type, name)                       \
23         __attribute__((__section__(SHARED_ALIGNED_SECTION)))            \
24         PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name             \
25         ____cacheline_aligned_in_smp
26 #else
27 #define DEFINE_PER_CPU(type, name)                                      \
28         PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name
29
30 #define DEFINE_PER_CPU_SHARED_ALIGNED(type, name)                     \
31         DEFINE_PER_CPU(type, name)
32 #endif
33
34 #define EXPORT_PER_CPU_SYMBOL(var) EXPORT_SYMBOL(per_cpu__##var)
35 #define EXPORT_PER_CPU_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(per_cpu__##var)
36
37 /* Enough to cover all DEFINE_PER_CPUs in kernel, including modules. */
38 #ifndef PERCPU_ENOUGH_ROOM
39 #ifdef CONFIG_MODULES
40 #define PERCPU_MODULE_RESERVE   8192
41 #else
42 #define PERCPU_MODULE_RESERVE   0
43 #endif
44
45 #define PERCPU_ENOUGH_ROOM                                              \
46         (__per_cpu_end - __per_cpu_start + PERCPU_MODULE_RESERVE)
47 #endif  /* PERCPU_ENOUGH_ROOM */
48
49 /*
50  * Must be an lvalue. Since @var must be a simple identifier,
51  * we force a syntax error here if it isn't.
52  */
53 #define get_cpu_var(var) (*({                           \
54         extern int simple_identifier_##var(void);       \
55         preempt_disable();                              \
56         &__get_cpu_var(var); }))
57 #define put_cpu_var(var) preempt_enable()
58
59 #ifdef CONFIG_SMP
60
61 struct percpu_data {
62         void *ptrs[1];
63 };
64
65 #define __percpu_disguise(pdata) (struct percpu_data *)~(unsigned long)(pdata)
66 /* 
67  * Use this to get to a cpu's version of the per-cpu object dynamically
68  * allocated. Non-atomic access to the current CPU's version should
69  * probably be combined with get_cpu()/put_cpu().
70  */ 
71 #define percpu_ptr(ptr, cpu)                              \
72 ({                                                        \
73         struct percpu_data *__p = __percpu_disguise(ptr); \
74         (__typeof__(ptr))__p->ptrs[(cpu)];                \
75 })
76
77 extern void *__percpu_alloc_mask(size_t size, gfp_t gfp, cpumask_t *mask);
78 extern void percpu_free(void *__pdata);
79
80 #else /* CONFIG_SMP */
81
82 #define percpu_ptr(ptr, cpu) ({ (void)(cpu); (ptr); })
83
84 static __always_inline void *__percpu_alloc_mask(size_t size, gfp_t gfp, cpumask_t *mask)
85 {
86         return kzalloc(size, gfp);
87 }
88
89 static inline void percpu_free(void *__pdata)
90 {
91         kfree(__pdata);
92 }
93
94 #endif /* CONFIG_SMP */
95
96 #define percpu_alloc_mask(size, gfp, mask) \
97         __percpu_alloc_mask((size), (gfp), &(mask))
98
99 #define percpu_alloc(size, gfp) percpu_alloc_mask((size), (gfp), cpu_online_map)
100
101 /* (legacy) interface for use without CPU hotplug handling */
102
103 #define __alloc_percpu(size)    percpu_alloc_mask((size), GFP_KERNEL, \
104                                                   cpu_possible_map)
105 #define alloc_percpu(type)      (type *)__alloc_percpu(sizeof(type))
106 #define free_percpu(ptr)        percpu_free((ptr))
107 #define per_cpu_ptr(ptr, cpu)   percpu_ptr((ptr), (cpu))
108
109 #endif /* __LINUX_PERCPU_H */