tracing: Fix sign fields in ftrace_define_fields_##call()
[linux-2.6.git] / kernel / spinlock.c
1 /*
2  * Copyright (2004) Linus Torvalds
3  *
4  * Author: Zwane Mwaikambo <zwane@fsmlabs.com>
5  *
6  * Copyright (2004, 2005) Ingo Molnar
7  *
8  * This file contains the spinlock/rwlock implementations for the
9  * SMP and the DEBUG_SPINLOCK cases. (UP-nondebug inlines them)
10  *
11  * Note that some architectures have special knowledge about the
12  * stack frames of these functions in their profile_pc. If you
13  * change anything significant here that could change the stack
14  * frame contact the architecture maintainers.
15  */
16
17 #include <linux/linkage.h>
18 #include <linux/preempt.h>
19 #include <linux/spinlock.h>
20 #include <linux/interrupt.h>
21 #include <linux/debug_locks.h>
22 #include <linux/module.h>
23
24 /*
25  * If lockdep is enabled then we use the non-preemption spin-ops
26  * even on CONFIG_PREEMPT, because lockdep assumes that interrupts are
27  * not re-enabled during lock-acquire (which the preempt-spin-ops do):
28  */
29 #if !defined(CONFIG_GENERIC_LOCKBREAK) || defined(CONFIG_DEBUG_LOCK_ALLOC)
30 /*
31  * The __lock_function inlines are taken from
32  * include/linux/spinlock_api_smp.h
33  */
34 #else
35 /*
36  * We build the __lock_function inlines here. They are too large for
37  * inlining all over the place, but here is only one user per function
38  * which embedds them into the calling _lock_function below.
39  *
40  * This could be a long-held lock. We both prepare to spin for a long
41  * time (making _this_ CPU preemptable if possible), and we also signal
42  * towards that other CPU that it should break the lock ASAP.
43  */
44 #define BUILD_LOCK_OPS(op, locktype)                                    \
45 void __lockfunc __##op##_lock(locktype##_t *lock)                       \
46 {                                                                       \
47         for (;;) {                                                      \
48                 preempt_disable();                                      \
49                 if (likely(_raw_##op##_trylock(lock)))                  \
50                         break;                                          \
51                 preempt_enable();                                       \
52                                                                         \
53                 if (!(lock)->break_lock)                                \
54                         (lock)->break_lock = 1;                         \
55                 while (!op##_can_lock(lock) && (lock)->break_lock)      \
56                         _raw_##op##_relax(&lock->raw_lock);             \
57         }                                                               \
58         (lock)->break_lock = 0;                                         \
59 }                                                                       \
60                                                                         \
61 unsigned long __lockfunc __##op##_lock_irqsave(locktype##_t *lock)      \
62 {                                                                       \
63         unsigned long flags;                                            \
64                                                                         \
65         for (;;) {                                                      \
66                 preempt_disable();                                      \
67                 local_irq_save(flags);                                  \
68                 if (likely(_raw_##op##_trylock(lock)))                  \
69                         break;                                          \
70                 local_irq_restore(flags);                               \
71                 preempt_enable();                                       \
72                                                                         \
73                 if (!(lock)->break_lock)                                \
74                         (lock)->break_lock = 1;                         \
75                 while (!op##_can_lock(lock) && (lock)->break_lock)      \
76                         _raw_##op##_relax(&lock->raw_lock);             \
77         }                                                               \
78         (lock)->break_lock = 0;                                         \
79         return flags;                                                   \
80 }                                                                       \
81                                                                         \
82 void __lockfunc __##op##_lock_irq(locktype##_t *lock)                   \
83 {                                                                       \
84         _##op##_lock_irqsave(lock);                                     \
85 }                                                                       \
86                                                                         \
87 void __lockfunc __##op##_lock_bh(locktype##_t *lock)                    \
88 {                                                                       \
89         unsigned long flags;                                            \
90                                                                         \
91         /*                                                      */      \
92         /* Careful: we must exclude softirqs too, hence the     */      \
93         /* irq-disabling. We use the generic preemption-aware   */      \
94         /* function:                                            */      \
95         /**/                                                            \
96         flags = _##op##_lock_irqsave(lock);                             \
97         local_bh_disable();                                             \
98         local_irq_restore(flags);                                       \
99 }                                                                       \
100
101 /*
102  * Build preemption-friendly versions of the following
103  * lock-spinning functions:
104  *
105  *         __[spin|read|write]_lock()
106  *         __[spin|read|write]_lock_irq()
107  *         __[spin|read|write]_lock_irqsave()
108  *         __[spin|read|write]_lock_bh()
109  */
110 BUILD_LOCK_OPS(spin, spinlock);
111 BUILD_LOCK_OPS(read, rwlock);
112 BUILD_LOCK_OPS(write, rwlock);
113
114 #endif
115
116 #ifdef CONFIG_DEBUG_LOCK_ALLOC
117
118 void __lockfunc _spin_lock_nested(spinlock_t *lock, int subclass)
119 {
120         preempt_disable();
121         spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
122         LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
123 }
124 EXPORT_SYMBOL(_spin_lock_nested);
125
126 unsigned long __lockfunc _spin_lock_irqsave_nested(spinlock_t *lock,
127                                                    int subclass)
128 {
129         unsigned long flags;
130
131         local_irq_save(flags);
132         preempt_disable();
133         spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
134         LOCK_CONTENDED_FLAGS(lock, _raw_spin_trylock, _raw_spin_lock,
135                                 _raw_spin_lock_flags, &flags);
136         return flags;
137 }
138 EXPORT_SYMBOL(_spin_lock_irqsave_nested);
139
140 void __lockfunc _spin_lock_nest_lock(spinlock_t *lock,
141                                      struct lockdep_map *nest_lock)
142 {
143         preempt_disable();
144         spin_acquire_nest(&lock->dep_map, 0, 0, nest_lock, _RET_IP_);
145         LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
146 }
147 EXPORT_SYMBOL(_spin_lock_nest_lock);
148
149 #endif
150
151 #ifndef CONFIG_INLINE_SPIN_TRYLOCK
152 int __lockfunc _spin_trylock(spinlock_t *lock)
153 {
154         return __spin_trylock(lock);
155 }
156 EXPORT_SYMBOL(_spin_trylock);
157 #endif
158
159 #ifndef CONFIG_INLINE_READ_TRYLOCK
160 int __lockfunc _read_trylock(rwlock_t *lock)
161 {
162         return __read_trylock(lock);
163 }
164 EXPORT_SYMBOL(_read_trylock);
165 #endif
166
167 #ifndef CONFIG_INLINE_WRITE_TRYLOCK
168 int __lockfunc _write_trylock(rwlock_t *lock)
169 {
170         return __write_trylock(lock);
171 }
172 EXPORT_SYMBOL(_write_trylock);
173 #endif
174
175 #ifndef CONFIG_INLINE_READ_LOCK
176 void __lockfunc _read_lock(rwlock_t *lock)
177 {
178         __read_lock(lock);
179 }
180 EXPORT_SYMBOL(_read_lock);
181 #endif
182
183 #ifndef CONFIG_INLINE_SPIN_LOCK_IRQSAVE
184 unsigned long __lockfunc _spin_lock_irqsave(spinlock_t *lock)
185 {
186         return __spin_lock_irqsave(lock);
187 }
188 EXPORT_SYMBOL(_spin_lock_irqsave);
189 #endif
190
191 #ifndef CONFIG_INLINE_SPIN_LOCK_IRQ
192 void __lockfunc _spin_lock_irq(spinlock_t *lock)
193 {
194         __spin_lock_irq(lock);
195 }
196 EXPORT_SYMBOL(_spin_lock_irq);
197 #endif
198
199 #ifndef CONFIG_INLINE_SPIN_LOCK_BH
200 void __lockfunc _spin_lock_bh(spinlock_t *lock)
201 {
202         __spin_lock_bh(lock);
203 }
204 EXPORT_SYMBOL(_spin_lock_bh);
205 #endif
206
207 #ifndef CONFIG_INLINE_READ_LOCK_IRQSAVE
208 unsigned long __lockfunc _read_lock_irqsave(rwlock_t *lock)
209 {
210         return __read_lock_irqsave(lock);
211 }
212 EXPORT_SYMBOL(_read_lock_irqsave);
213 #endif
214
215 #ifndef CONFIG_INLINE_READ_LOCK_IRQ
216 void __lockfunc _read_lock_irq(rwlock_t *lock)
217 {
218         __read_lock_irq(lock);
219 }
220 EXPORT_SYMBOL(_read_lock_irq);
221 #endif
222
223 #ifndef CONFIG_INLINE_READ_LOCK_BH
224 void __lockfunc _read_lock_bh(rwlock_t *lock)
225 {
226         __read_lock_bh(lock);
227 }
228 EXPORT_SYMBOL(_read_lock_bh);
229 #endif
230
231 #ifndef CONFIG_INLINE_WRITE_LOCK_IRQSAVE
232 unsigned long __lockfunc _write_lock_irqsave(rwlock_t *lock)
233 {
234         return __write_lock_irqsave(lock);
235 }
236 EXPORT_SYMBOL(_write_lock_irqsave);
237 #endif
238
239 #ifndef CONFIG_INLINE_WRITE_LOCK_IRQ
240 void __lockfunc _write_lock_irq(rwlock_t *lock)
241 {
242         __write_lock_irq(lock);
243 }
244 EXPORT_SYMBOL(_write_lock_irq);
245 #endif
246
247 #ifndef CONFIG_INLINE_WRITE_LOCK_BH
248 void __lockfunc _write_lock_bh(rwlock_t *lock)
249 {
250         __write_lock_bh(lock);
251 }
252 EXPORT_SYMBOL(_write_lock_bh);
253 #endif
254
255 #ifndef CONFIG_INLINE_SPIN_LOCK
256 void __lockfunc _spin_lock(spinlock_t *lock)
257 {
258         __spin_lock(lock);
259 }
260 EXPORT_SYMBOL(_spin_lock);
261 #endif
262
263 #ifndef CONFIG_INLINE_WRITE_LOCK
264 void __lockfunc _write_lock(rwlock_t *lock)
265 {
266         __write_lock(lock);
267 }
268 EXPORT_SYMBOL(_write_lock);
269 #endif
270
271 #ifndef CONFIG_INLINE_SPIN_UNLOCK
272 void __lockfunc _spin_unlock(spinlock_t *lock)
273 {
274         __spin_unlock(lock);
275 }
276 EXPORT_SYMBOL(_spin_unlock);
277 #endif
278
279 #ifndef CONFIG_INLINE_WRITE_UNLOCK
280 void __lockfunc _write_unlock(rwlock_t *lock)
281 {
282         __write_unlock(lock);
283 }
284 EXPORT_SYMBOL(_write_unlock);
285 #endif
286
287 #ifndef CONFIG_INLINE_READ_UNLOCK
288 void __lockfunc _read_unlock(rwlock_t *lock)
289 {
290         __read_unlock(lock);
291 }
292 EXPORT_SYMBOL(_read_unlock);
293 #endif
294
295 #ifndef CONFIG_INLINE_SPIN_UNLOCK_IRQRESTORE
296 void __lockfunc _spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags)
297 {
298         __spin_unlock_irqrestore(lock, flags);
299 }
300 EXPORT_SYMBOL(_spin_unlock_irqrestore);
301 #endif
302
303 #ifndef CONFIG_INLINE_SPIN_UNLOCK_IRQ
304 void __lockfunc _spin_unlock_irq(spinlock_t *lock)
305 {
306         __spin_unlock_irq(lock);
307 }
308 EXPORT_SYMBOL(_spin_unlock_irq);
309 #endif
310
311 #ifndef CONFIG_INLINE_SPIN_UNLOCK_BH
312 void __lockfunc _spin_unlock_bh(spinlock_t *lock)
313 {
314         __spin_unlock_bh(lock);
315 }
316 EXPORT_SYMBOL(_spin_unlock_bh);
317 #endif
318
319 #ifndef CONFIG_INLINE_READ_UNLOCK_IRQRESTORE
320 void __lockfunc _read_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
321 {
322         __read_unlock_irqrestore(lock, flags);
323 }
324 EXPORT_SYMBOL(_read_unlock_irqrestore);
325 #endif
326
327 #ifndef CONFIG_INLINE_READ_UNLOCK_IRQ
328 void __lockfunc _read_unlock_irq(rwlock_t *lock)
329 {
330         __read_unlock_irq(lock);
331 }
332 EXPORT_SYMBOL(_read_unlock_irq);
333 #endif
334
335 #ifndef CONFIG_INLINE_READ_UNLOCK_BH
336 void __lockfunc _read_unlock_bh(rwlock_t *lock)
337 {
338         __read_unlock_bh(lock);
339 }
340 EXPORT_SYMBOL(_read_unlock_bh);
341 #endif
342
343 #ifndef CONFIG_INLINE_WRITE_UNLOCK_IRQRESTORE
344 void __lockfunc _write_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
345 {
346         __write_unlock_irqrestore(lock, flags);
347 }
348 EXPORT_SYMBOL(_write_unlock_irqrestore);
349 #endif
350
351 #ifndef CONFIG_INLINE_WRITE_UNLOCK_IRQ
352 void __lockfunc _write_unlock_irq(rwlock_t *lock)
353 {
354         __write_unlock_irq(lock);
355 }
356 EXPORT_SYMBOL(_write_unlock_irq);
357 #endif
358
359 #ifndef CONFIG_INLINE_WRITE_UNLOCK_BH
360 void __lockfunc _write_unlock_bh(rwlock_t *lock)
361 {
362         __write_unlock_bh(lock);
363 }
364 EXPORT_SYMBOL(_write_unlock_bh);
365 #endif
366
367 #ifndef CONFIG_INLINE_SPIN_TRYLOCK_BH
368 int __lockfunc _spin_trylock_bh(spinlock_t *lock)
369 {
370         return __spin_trylock_bh(lock);
371 }
372 EXPORT_SYMBOL(_spin_trylock_bh);
373 #endif
374
375 notrace int in_lock_functions(unsigned long addr)
376 {
377         /* Linker adds these: start and end of __lockfunc functions */
378         extern char __lock_text_start[], __lock_text_end[];
379
380         return addr >= (unsigned long)__lock_text_start
381         && addr < (unsigned long)__lock_text_end;
382 }
383 EXPORT_SYMBOL(in_lock_functions);