[PATCH] Directed yield: cpu_relax variants for spinlocks and rw-locks
[linux-2.6.git] / kernel / spinlock.c
1 /*
2  * Copyright (2004) Linus Torvalds
3  *
4  * Author: Zwane Mwaikambo <zwane@fsmlabs.com>
5  *
6  * Copyright (2004, 2005) Ingo Molnar
7  *
8  * This file contains the spinlock/rwlock implementations for the
9  * SMP and the DEBUG_SPINLOCK cases. (UP-nondebug inlines them)
10  *
11  * Note that some architectures have special knowledge about the
12  * stack frames of these functions in their profile_pc. If you
13  * change anything significant here that could change the stack
14  * frame contact the architecture maintainers.
15  */
16
17 #include <linux/linkage.h>
18 #include <linux/preempt.h>
19 #include <linux/spinlock.h>
20 #include <linux/interrupt.h>
21 #include <linux/debug_locks.h>
22 #include <linux/module.h>
23
24 int __lockfunc _spin_trylock(spinlock_t *lock)
25 {
26         preempt_disable();
27         if (_raw_spin_trylock(lock)) {
28                 spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
29                 return 1;
30         }
31         
32         preempt_enable();
33         return 0;
34 }
35 EXPORT_SYMBOL(_spin_trylock);
36
37 int __lockfunc _read_trylock(rwlock_t *lock)
38 {
39         preempt_disable();
40         if (_raw_read_trylock(lock)) {
41                 rwlock_acquire_read(&lock->dep_map, 0, 1, _RET_IP_);
42                 return 1;
43         }
44
45         preempt_enable();
46         return 0;
47 }
48 EXPORT_SYMBOL(_read_trylock);
49
50 int __lockfunc _write_trylock(rwlock_t *lock)
51 {
52         preempt_disable();
53         if (_raw_write_trylock(lock)) {
54                 rwlock_acquire(&lock->dep_map, 0, 1, _RET_IP_);
55                 return 1;
56         }
57
58         preempt_enable();
59         return 0;
60 }
61 EXPORT_SYMBOL(_write_trylock);
62
63 /*
64  * If lockdep is enabled then we use the non-preemption spin-ops
65  * even on CONFIG_PREEMPT, because lockdep assumes that interrupts are
66  * not re-enabled during lock-acquire (which the preempt-spin-ops do):
67  */
68 #if !defined(CONFIG_PREEMPT) || !defined(CONFIG_SMP) || \
69         defined(CONFIG_DEBUG_LOCK_ALLOC)
70
71 void __lockfunc _read_lock(rwlock_t *lock)
72 {
73         preempt_disable();
74         rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
75         _raw_read_lock(lock);
76 }
77 EXPORT_SYMBOL(_read_lock);
78
79 unsigned long __lockfunc _spin_lock_irqsave(spinlock_t *lock)
80 {
81         unsigned long flags;
82
83         local_irq_save(flags);
84         preempt_disable();
85         spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
86         /*
87          * On lockdep we dont want the hand-coded irq-enable of
88          * _raw_spin_lock_flags() code, because lockdep assumes
89          * that interrupts are not re-enabled during lock-acquire:
90          */
91 #ifdef CONFIG_PROVE_LOCKING
92         _raw_spin_lock(lock);
93 #else
94         _raw_spin_lock_flags(lock, &flags);
95 #endif
96         return flags;
97 }
98 EXPORT_SYMBOL(_spin_lock_irqsave);
99
100 void __lockfunc _spin_lock_irq(spinlock_t *lock)
101 {
102         local_irq_disable();
103         preempt_disable();
104         spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
105         _raw_spin_lock(lock);
106 }
107 EXPORT_SYMBOL(_spin_lock_irq);
108
109 void __lockfunc _spin_lock_bh(spinlock_t *lock)
110 {
111         local_bh_disable();
112         preempt_disable();
113         spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
114         _raw_spin_lock(lock);
115 }
116 EXPORT_SYMBOL(_spin_lock_bh);
117
118 unsigned long __lockfunc _read_lock_irqsave(rwlock_t *lock)
119 {
120         unsigned long flags;
121
122         local_irq_save(flags);
123         preempt_disable();
124         rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
125         _raw_read_lock(lock);
126         return flags;
127 }
128 EXPORT_SYMBOL(_read_lock_irqsave);
129
130 void __lockfunc _read_lock_irq(rwlock_t *lock)
131 {
132         local_irq_disable();
133         preempt_disable();
134         rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
135         _raw_read_lock(lock);
136 }
137 EXPORT_SYMBOL(_read_lock_irq);
138
139 void __lockfunc _read_lock_bh(rwlock_t *lock)
140 {
141         local_bh_disable();
142         preempt_disable();
143         rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
144         _raw_read_lock(lock);
145 }
146 EXPORT_SYMBOL(_read_lock_bh);
147
148 unsigned long __lockfunc _write_lock_irqsave(rwlock_t *lock)
149 {
150         unsigned long flags;
151
152         local_irq_save(flags);
153         preempt_disable();
154         rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
155         _raw_write_lock(lock);
156         return flags;
157 }
158 EXPORT_SYMBOL(_write_lock_irqsave);
159
160 void __lockfunc _write_lock_irq(rwlock_t *lock)
161 {
162         local_irq_disable();
163         preempt_disable();
164         rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
165         _raw_write_lock(lock);
166 }
167 EXPORT_SYMBOL(_write_lock_irq);
168
169 void __lockfunc _write_lock_bh(rwlock_t *lock)
170 {
171         local_bh_disable();
172         preempt_disable();
173         rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
174         _raw_write_lock(lock);
175 }
176 EXPORT_SYMBOL(_write_lock_bh);
177
178 void __lockfunc _spin_lock(spinlock_t *lock)
179 {
180         preempt_disable();
181         spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
182         _raw_spin_lock(lock);
183 }
184
185 EXPORT_SYMBOL(_spin_lock);
186
187 void __lockfunc _write_lock(rwlock_t *lock)
188 {
189         preempt_disable();
190         rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
191         _raw_write_lock(lock);
192 }
193
194 EXPORT_SYMBOL(_write_lock);
195
196 #else /* CONFIG_PREEMPT: */
197
198 /*
199  * This could be a long-held lock. We both prepare to spin for a long
200  * time (making _this_ CPU preemptable if possible), and we also signal
201  * towards that other CPU that it should break the lock ASAP.
202  *
203  * (We do this in a function because inlining it would be excessive.)
204  */
205
206 #define BUILD_LOCK_OPS(op, locktype)                                    \
207 void __lockfunc _##op##_lock(locktype##_t *lock)                        \
208 {                                                                       \
209         for (;;) {                                                      \
210                 preempt_disable();                                      \
211                 if (likely(_raw_##op##_trylock(lock)))                  \
212                         break;                                          \
213                 preempt_enable();                                       \
214                                                                         \
215                 if (!(lock)->break_lock)                                \
216                         (lock)->break_lock = 1;                         \
217                 while (!op##_can_lock(lock) && (lock)->break_lock)      \
218                         _raw_##op##_relax(&lock->raw_lock);             \
219         }                                                               \
220         (lock)->break_lock = 0;                                         \
221 }                                                                       \
222                                                                         \
223 EXPORT_SYMBOL(_##op##_lock);                                            \
224                                                                         \
225 unsigned long __lockfunc _##op##_lock_irqsave(locktype##_t *lock)       \
226 {                                                                       \
227         unsigned long flags;                                            \
228                                                                         \
229         for (;;) {                                                      \
230                 preempt_disable();                                      \
231                 local_irq_save(flags);                                  \
232                 if (likely(_raw_##op##_trylock(lock)))                  \
233                         break;                                          \
234                 local_irq_restore(flags);                               \
235                 preempt_enable();                                       \
236                                                                         \
237                 if (!(lock)->break_lock)                                \
238                         (lock)->break_lock = 1;                         \
239                 while (!op##_can_lock(lock) && (lock)->break_lock)      \
240                         _raw_##op##_relax(&lock->raw_lock);             \
241         }                                                               \
242         (lock)->break_lock = 0;                                         \
243         return flags;                                                   \
244 }                                                                       \
245                                                                         \
246 EXPORT_SYMBOL(_##op##_lock_irqsave);                                    \
247                                                                         \
248 void __lockfunc _##op##_lock_irq(locktype##_t *lock)                    \
249 {                                                                       \
250         _##op##_lock_irqsave(lock);                                     \
251 }                                                                       \
252                                                                         \
253 EXPORT_SYMBOL(_##op##_lock_irq);                                        \
254                                                                         \
255 void __lockfunc _##op##_lock_bh(locktype##_t *lock)                     \
256 {                                                                       \
257         unsigned long flags;                                            \
258                                                                         \
259         /*                                                      */      \
260         /* Careful: we must exclude softirqs too, hence the     */      \
261         /* irq-disabling. We use the generic preemption-aware   */      \
262         /* function:                                            */      \
263         /**/                                                            \
264         flags = _##op##_lock_irqsave(lock);                             \
265         local_bh_disable();                                             \
266         local_irq_restore(flags);                                       \
267 }                                                                       \
268                                                                         \
269 EXPORT_SYMBOL(_##op##_lock_bh)
270
271 /*
272  * Build preemption-friendly versions of the following
273  * lock-spinning functions:
274  *
275  *         _[spin|read|write]_lock()
276  *         _[spin|read|write]_lock_irq()
277  *         _[spin|read|write]_lock_irqsave()
278  *         _[spin|read|write]_lock_bh()
279  */
280 BUILD_LOCK_OPS(spin, spinlock);
281 BUILD_LOCK_OPS(read, rwlock);
282 BUILD_LOCK_OPS(write, rwlock);
283
284 #endif /* CONFIG_PREEMPT */
285
286 #ifdef CONFIG_DEBUG_LOCK_ALLOC
287
288 void __lockfunc _spin_lock_nested(spinlock_t *lock, int subclass)
289 {
290         preempt_disable();
291         spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
292         _raw_spin_lock(lock);
293 }
294
295 EXPORT_SYMBOL(_spin_lock_nested);
296
297 #endif
298
299 void __lockfunc _spin_unlock(spinlock_t *lock)
300 {
301         spin_release(&lock->dep_map, 1, _RET_IP_);
302         _raw_spin_unlock(lock);
303         preempt_enable();
304 }
305 EXPORT_SYMBOL(_spin_unlock);
306
307 void __lockfunc _write_unlock(rwlock_t *lock)
308 {
309         rwlock_release(&lock->dep_map, 1, _RET_IP_);
310         _raw_write_unlock(lock);
311         preempt_enable();
312 }
313 EXPORT_SYMBOL(_write_unlock);
314
315 void __lockfunc _read_unlock(rwlock_t *lock)
316 {
317         rwlock_release(&lock->dep_map, 1, _RET_IP_);
318         _raw_read_unlock(lock);
319         preempt_enable();
320 }
321 EXPORT_SYMBOL(_read_unlock);
322
323 void __lockfunc _spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags)
324 {
325         spin_release(&lock->dep_map, 1, _RET_IP_);
326         _raw_spin_unlock(lock);
327         local_irq_restore(flags);
328         preempt_enable();
329 }
330 EXPORT_SYMBOL(_spin_unlock_irqrestore);
331
332 void __lockfunc _spin_unlock_irq(spinlock_t *lock)
333 {
334         spin_release(&lock->dep_map, 1, _RET_IP_);
335         _raw_spin_unlock(lock);
336         local_irq_enable();
337         preempt_enable();
338 }
339 EXPORT_SYMBOL(_spin_unlock_irq);
340
341 void __lockfunc _spin_unlock_bh(spinlock_t *lock)
342 {
343         spin_release(&lock->dep_map, 1, _RET_IP_);
344         _raw_spin_unlock(lock);
345         preempt_enable_no_resched();
346         local_bh_enable_ip((unsigned long)__builtin_return_address(0));
347 }
348 EXPORT_SYMBOL(_spin_unlock_bh);
349
350 void __lockfunc _read_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
351 {
352         rwlock_release(&lock->dep_map, 1, _RET_IP_);
353         _raw_read_unlock(lock);
354         local_irq_restore(flags);
355         preempt_enable();
356 }
357 EXPORT_SYMBOL(_read_unlock_irqrestore);
358
359 void __lockfunc _read_unlock_irq(rwlock_t *lock)
360 {
361         rwlock_release(&lock->dep_map, 1, _RET_IP_);
362         _raw_read_unlock(lock);
363         local_irq_enable();
364         preempt_enable();
365 }
366 EXPORT_SYMBOL(_read_unlock_irq);
367
368 void __lockfunc _read_unlock_bh(rwlock_t *lock)
369 {
370         rwlock_release(&lock->dep_map, 1, _RET_IP_);
371         _raw_read_unlock(lock);
372         preempt_enable_no_resched();
373         local_bh_enable_ip((unsigned long)__builtin_return_address(0));
374 }
375 EXPORT_SYMBOL(_read_unlock_bh);
376
377 void __lockfunc _write_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
378 {
379         rwlock_release(&lock->dep_map, 1, _RET_IP_);
380         _raw_write_unlock(lock);
381         local_irq_restore(flags);
382         preempt_enable();
383 }
384 EXPORT_SYMBOL(_write_unlock_irqrestore);
385
386 void __lockfunc _write_unlock_irq(rwlock_t *lock)
387 {
388         rwlock_release(&lock->dep_map, 1, _RET_IP_);
389         _raw_write_unlock(lock);
390         local_irq_enable();
391         preempt_enable();
392 }
393 EXPORT_SYMBOL(_write_unlock_irq);
394
395 void __lockfunc _write_unlock_bh(rwlock_t *lock)
396 {
397         rwlock_release(&lock->dep_map, 1, _RET_IP_);
398         _raw_write_unlock(lock);
399         preempt_enable_no_resched();
400         local_bh_enable_ip((unsigned long)__builtin_return_address(0));
401 }
402 EXPORT_SYMBOL(_write_unlock_bh);
403
404 int __lockfunc _spin_trylock_bh(spinlock_t *lock)
405 {
406         local_bh_disable();
407         preempt_disable();
408         if (_raw_spin_trylock(lock)) {
409                 spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
410                 return 1;
411         }
412
413         preempt_enable_no_resched();
414         local_bh_enable_ip((unsigned long)__builtin_return_address(0));
415         return 0;
416 }
417 EXPORT_SYMBOL(_spin_trylock_bh);
418
419 int in_lock_functions(unsigned long addr)
420 {
421         /* Linker adds these: start and end of __lockfunc functions */
422         extern char __lock_text_start[], __lock_text_end[];
423
424         return addr >= (unsigned long)__lock_text_start
425         && addr < (unsigned long)__lock_text_end;
426 }
427 EXPORT_SYMBOL(in_lock_functions);