Merge commit 'linus/master' into merge-linus
[linux-2.6.git] / kernel / spinlock.c
1 /*
2  * Copyright (2004) Linus Torvalds
3  *
4  * Author: Zwane Mwaikambo <zwane@fsmlabs.com>
5  *
6  * Copyright (2004, 2005) Ingo Molnar
7  *
8  * This file contains the spinlock/rwlock implementations for the
9  * SMP and the DEBUG_SPINLOCK cases. (UP-nondebug inlines them)
10  *
11  * Note that some architectures have special knowledge about the
12  * stack frames of these functions in their profile_pc. If you
13  * change anything significant here that could change the stack
14  * frame contact the architecture maintainers.
15  */
16
17 #include <linux/linkage.h>
18 #include <linux/preempt.h>
19 #include <linux/spinlock.h>
20 #include <linux/interrupt.h>
21 #include <linux/debug_locks.h>
22 #include <linux/module.h>
23
24 int __lockfunc _spin_trylock(spinlock_t *lock)
25 {
26         preempt_disable();
27         if (_raw_spin_trylock(lock)) {
28                 spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
29                 return 1;
30         }
31         
32         preempt_enable();
33         return 0;
34 }
35 EXPORT_SYMBOL(_spin_trylock);
36
37 int __lockfunc _read_trylock(rwlock_t *lock)
38 {
39         preempt_disable();
40         if (_raw_read_trylock(lock)) {
41                 rwlock_acquire_read(&lock->dep_map, 0, 1, _RET_IP_);
42                 return 1;
43         }
44
45         preempt_enable();
46         return 0;
47 }
48 EXPORT_SYMBOL(_read_trylock);
49
50 int __lockfunc _write_trylock(rwlock_t *lock)
51 {
52         preempt_disable();
53         if (_raw_write_trylock(lock)) {
54                 rwlock_acquire(&lock->dep_map, 0, 1, _RET_IP_);
55                 return 1;
56         }
57
58         preempt_enable();
59         return 0;
60 }
61 EXPORT_SYMBOL(_write_trylock);
62
63 /*
64  * If lockdep is enabled then we use the non-preemption spin-ops
65  * even on CONFIG_PREEMPT, because lockdep assumes that interrupts are
66  * not re-enabled during lock-acquire (which the preempt-spin-ops do):
67  */
68 #if !defined(CONFIG_GENERIC_LOCKBREAK) || defined(CONFIG_DEBUG_LOCK_ALLOC)
69
70 void __lockfunc _read_lock(rwlock_t *lock)
71 {
72         preempt_disable();
73         rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
74         LOCK_CONTENDED(lock, _raw_read_trylock, _raw_read_lock);
75 }
76 EXPORT_SYMBOL(_read_lock);
77
78 unsigned long __lockfunc _spin_lock_irqsave(spinlock_t *lock)
79 {
80         unsigned long flags;
81
82         local_irq_save(flags);
83         preempt_disable();
84         spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
85         /*
86          * On lockdep we dont want the hand-coded irq-enable of
87          * _raw_spin_lock_flags() code, because lockdep assumes
88          * that interrupts are not re-enabled during lock-acquire:
89          */
90 #ifdef CONFIG_LOCKDEP
91         LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
92 #else
93         _raw_spin_lock_flags(lock, &flags);
94 #endif
95         return flags;
96 }
97 EXPORT_SYMBOL(_spin_lock_irqsave);
98
99 void __lockfunc _spin_lock_irq(spinlock_t *lock)
100 {
101         local_irq_disable();
102         preempt_disable();
103         spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
104         LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
105 }
106 EXPORT_SYMBOL(_spin_lock_irq);
107
108 void __lockfunc _spin_lock_bh(spinlock_t *lock)
109 {
110         local_bh_disable();
111         preempt_disable();
112         spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
113         LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
114 }
115 EXPORT_SYMBOL(_spin_lock_bh);
116
117 unsigned long __lockfunc _read_lock_irqsave(rwlock_t *lock)
118 {
119         unsigned long flags;
120
121         local_irq_save(flags);
122         preempt_disable();
123         rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
124         LOCK_CONTENDED(lock, _raw_read_trylock, _raw_read_lock);
125         return flags;
126 }
127 EXPORT_SYMBOL(_read_lock_irqsave);
128
129 void __lockfunc _read_lock_irq(rwlock_t *lock)
130 {
131         local_irq_disable();
132         preempt_disable();
133         rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
134         LOCK_CONTENDED(lock, _raw_read_trylock, _raw_read_lock);
135 }
136 EXPORT_SYMBOL(_read_lock_irq);
137
138 void __lockfunc _read_lock_bh(rwlock_t *lock)
139 {
140         local_bh_disable();
141         preempt_disable();
142         rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
143         LOCK_CONTENDED(lock, _raw_read_trylock, _raw_read_lock);
144 }
145 EXPORT_SYMBOL(_read_lock_bh);
146
147 unsigned long __lockfunc _write_lock_irqsave(rwlock_t *lock)
148 {
149         unsigned long flags;
150
151         local_irq_save(flags);
152         preempt_disable();
153         rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
154         LOCK_CONTENDED(lock, _raw_write_trylock, _raw_write_lock);
155         return flags;
156 }
157 EXPORT_SYMBOL(_write_lock_irqsave);
158
159 void __lockfunc _write_lock_irq(rwlock_t *lock)
160 {
161         local_irq_disable();
162         preempt_disable();
163         rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
164         LOCK_CONTENDED(lock, _raw_write_trylock, _raw_write_lock);
165 }
166 EXPORT_SYMBOL(_write_lock_irq);
167
168 void __lockfunc _write_lock_bh(rwlock_t *lock)
169 {
170         local_bh_disable();
171         preempt_disable();
172         rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
173         LOCK_CONTENDED(lock, _raw_write_trylock, _raw_write_lock);
174 }
175 EXPORT_SYMBOL(_write_lock_bh);
176
177 void __lockfunc _spin_lock(spinlock_t *lock)
178 {
179         preempt_disable();
180         spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
181         LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
182 }
183
184 EXPORT_SYMBOL(_spin_lock);
185
186 void __lockfunc _write_lock(rwlock_t *lock)
187 {
188         preempt_disable();
189         rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
190         LOCK_CONTENDED(lock, _raw_write_trylock, _raw_write_lock);
191 }
192
193 EXPORT_SYMBOL(_write_lock);
194
195 #else /* CONFIG_PREEMPT: */
196
197 /*
198  * This could be a long-held lock. We both prepare to spin for a long
199  * time (making _this_ CPU preemptable if possible), and we also signal
200  * towards that other CPU that it should break the lock ASAP.
201  *
202  * (We do this in a function because inlining it would be excessive.)
203  */
204
205 #define BUILD_LOCK_OPS(op, locktype)                                    \
206 void __lockfunc _##op##_lock(locktype##_t *lock)                        \
207 {                                                                       \
208         for (;;) {                                                      \
209                 preempt_disable();                                      \
210                 if (likely(_raw_##op##_trylock(lock)))                  \
211                         break;                                          \
212                 preempt_enable();                                       \
213                                                                         \
214                 if (!(lock)->break_lock)                                \
215                         (lock)->break_lock = 1;                         \
216                 while (!op##_can_lock(lock) && (lock)->break_lock)      \
217                         _raw_##op##_relax(&lock->raw_lock);             \
218         }                                                               \
219         (lock)->break_lock = 0;                                         \
220 }                                                                       \
221                                                                         \
222 EXPORT_SYMBOL(_##op##_lock);                                            \
223                                                                         \
224 unsigned long __lockfunc _##op##_lock_irqsave(locktype##_t *lock)       \
225 {                                                                       \
226         unsigned long flags;                                            \
227                                                                         \
228         for (;;) {                                                      \
229                 preempt_disable();                                      \
230                 local_irq_save(flags);                                  \
231                 if (likely(_raw_##op##_trylock(lock)))                  \
232                         break;                                          \
233                 local_irq_restore(flags);                               \
234                 preempt_enable();                                       \
235                                                                         \
236                 if (!(lock)->break_lock)                                \
237                         (lock)->break_lock = 1;                         \
238                 while (!op##_can_lock(lock) && (lock)->break_lock)      \
239                         _raw_##op##_relax(&lock->raw_lock);             \
240         }                                                               \
241         (lock)->break_lock = 0;                                         \
242         return flags;                                                   \
243 }                                                                       \
244                                                                         \
245 EXPORT_SYMBOL(_##op##_lock_irqsave);                                    \
246                                                                         \
247 void __lockfunc _##op##_lock_irq(locktype##_t *lock)                    \
248 {                                                                       \
249         _##op##_lock_irqsave(lock);                                     \
250 }                                                                       \
251                                                                         \
252 EXPORT_SYMBOL(_##op##_lock_irq);                                        \
253                                                                         \
254 void __lockfunc _##op##_lock_bh(locktype##_t *lock)                     \
255 {                                                                       \
256         unsigned long flags;                                            \
257                                                                         \
258         /*                                                      */      \
259         /* Careful: we must exclude softirqs too, hence the     */      \
260         /* irq-disabling. We use the generic preemption-aware   */      \
261         /* function:                                            */      \
262         /**/                                                            \
263         flags = _##op##_lock_irqsave(lock);                             \
264         local_bh_disable();                                             \
265         local_irq_restore(flags);                                       \
266 }                                                                       \
267                                                                         \
268 EXPORT_SYMBOL(_##op##_lock_bh)
269
270 /*
271  * Build preemption-friendly versions of the following
272  * lock-spinning functions:
273  *
274  *         _[spin|read|write]_lock()
275  *         _[spin|read|write]_lock_irq()
276  *         _[spin|read|write]_lock_irqsave()
277  *         _[spin|read|write]_lock_bh()
278  */
279 BUILD_LOCK_OPS(spin, spinlock);
280 BUILD_LOCK_OPS(read, rwlock);
281 BUILD_LOCK_OPS(write, rwlock);
282
283 #endif /* CONFIG_PREEMPT */
284
285 #ifdef CONFIG_DEBUG_LOCK_ALLOC
286
287 void __lockfunc _spin_lock_nested(spinlock_t *lock, int subclass)
288 {
289         preempt_disable();
290         spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
291         LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
292 }
293 EXPORT_SYMBOL(_spin_lock_nested);
294
295 unsigned long __lockfunc _spin_lock_irqsave_nested(spinlock_t *lock, int subclass)
296 {
297         unsigned long flags;
298
299         local_irq_save(flags);
300         preempt_disable();
301         spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
302         /*
303          * On lockdep we dont want the hand-coded irq-enable of
304          * _raw_spin_lock_flags() code, because lockdep assumes
305          * that interrupts are not re-enabled during lock-acquire:
306          */
307 #ifdef CONFIG_LOCKDEP
308         LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
309 #else
310         _raw_spin_lock_flags(lock, &flags);
311 #endif
312         return flags;
313 }
314 EXPORT_SYMBOL(_spin_lock_irqsave_nested);
315
316 void __lockfunc _spin_lock_nest_lock(spinlock_t *lock,
317                                      struct lockdep_map *nest_lock)
318 {
319         preempt_disable();
320         spin_acquire_nest(&lock->dep_map, 0, 0, nest_lock, _RET_IP_);
321         LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
322 }
323 EXPORT_SYMBOL(_spin_lock_nest_lock);
324
325 #endif
326
327 void __lockfunc _spin_unlock(spinlock_t *lock)
328 {
329         spin_release(&lock->dep_map, 1, _RET_IP_);
330         _raw_spin_unlock(lock);
331         preempt_enable();
332 }
333 EXPORT_SYMBOL(_spin_unlock);
334
335 void __lockfunc _write_unlock(rwlock_t *lock)
336 {
337         rwlock_release(&lock->dep_map, 1, _RET_IP_);
338         _raw_write_unlock(lock);
339         preempt_enable();
340 }
341 EXPORT_SYMBOL(_write_unlock);
342
343 void __lockfunc _read_unlock(rwlock_t *lock)
344 {
345         rwlock_release(&lock->dep_map, 1, _RET_IP_);
346         _raw_read_unlock(lock);
347         preempt_enable();
348 }
349 EXPORT_SYMBOL(_read_unlock);
350
351 void __lockfunc _spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags)
352 {
353         spin_release(&lock->dep_map, 1, _RET_IP_);
354         _raw_spin_unlock(lock);
355         local_irq_restore(flags);
356         preempt_enable();
357 }
358 EXPORT_SYMBOL(_spin_unlock_irqrestore);
359
360 void __lockfunc _spin_unlock_irq(spinlock_t *lock)
361 {
362         spin_release(&lock->dep_map, 1, _RET_IP_);
363         _raw_spin_unlock(lock);
364         local_irq_enable();
365         preempt_enable();
366 }
367 EXPORT_SYMBOL(_spin_unlock_irq);
368
369 void __lockfunc _spin_unlock_bh(spinlock_t *lock)
370 {
371         spin_release(&lock->dep_map, 1, _RET_IP_);
372         _raw_spin_unlock(lock);
373         preempt_enable_no_resched();
374         local_bh_enable_ip((unsigned long)__builtin_return_address(0));
375 }
376 EXPORT_SYMBOL(_spin_unlock_bh);
377
378 void __lockfunc _read_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
379 {
380         rwlock_release(&lock->dep_map, 1, _RET_IP_);
381         _raw_read_unlock(lock);
382         local_irq_restore(flags);
383         preempt_enable();
384 }
385 EXPORT_SYMBOL(_read_unlock_irqrestore);
386
387 void __lockfunc _read_unlock_irq(rwlock_t *lock)
388 {
389         rwlock_release(&lock->dep_map, 1, _RET_IP_);
390         _raw_read_unlock(lock);
391         local_irq_enable();
392         preempt_enable();
393 }
394 EXPORT_SYMBOL(_read_unlock_irq);
395
396 void __lockfunc _read_unlock_bh(rwlock_t *lock)
397 {
398         rwlock_release(&lock->dep_map, 1, _RET_IP_);
399         _raw_read_unlock(lock);
400         preempt_enable_no_resched();
401         local_bh_enable_ip((unsigned long)__builtin_return_address(0));
402 }
403 EXPORT_SYMBOL(_read_unlock_bh);
404
405 void __lockfunc _write_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
406 {
407         rwlock_release(&lock->dep_map, 1, _RET_IP_);
408         _raw_write_unlock(lock);
409         local_irq_restore(flags);
410         preempt_enable();
411 }
412 EXPORT_SYMBOL(_write_unlock_irqrestore);
413
414 void __lockfunc _write_unlock_irq(rwlock_t *lock)
415 {
416         rwlock_release(&lock->dep_map, 1, _RET_IP_);
417         _raw_write_unlock(lock);
418         local_irq_enable();
419         preempt_enable();
420 }
421 EXPORT_SYMBOL(_write_unlock_irq);
422
423 void __lockfunc _write_unlock_bh(rwlock_t *lock)
424 {
425         rwlock_release(&lock->dep_map, 1, _RET_IP_);
426         _raw_write_unlock(lock);
427         preempt_enable_no_resched();
428         local_bh_enable_ip((unsigned long)__builtin_return_address(0));
429 }
430 EXPORT_SYMBOL(_write_unlock_bh);
431
432 int __lockfunc _spin_trylock_bh(spinlock_t *lock)
433 {
434         local_bh_disable();
435         preempt_disable();
436         if (_raw_spin_trylock(lock)) {
437                 spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
438                 return 1;
439         }
440
441         preempt_enable_no_resched();
442         local_bh_enable_ip((unsigned long)__builtin_return_address(0));
443         return 0;
444 }
445 EXPORT_SYMBOL(_spin_trylock_bh);
446
447 notrace int in_lock_functions(unsigned long addr)
448 {
449         /* Linker adds these: start and end of __lockfunc functions */
450         extern char __lock_text_start[], __lock_text_end[];
451
452         return addr >= (unsigned long)__lock_text_start
453         && addr < (unsigned long)__lock_text_end;
454 }
455 EXPORT_SYMBOL(in_lock_functions);