[PATCH] Directed yield: cpu_relax variants for spinlocks and rw-locks
[linux-2.6.git] / include / asm-mips / spinlock.h
1 /*
2  * This file is subject to the terms and conditions of the GNU General Public
3  * License.  See the file "COPYING" in the main directory of this archive
4  * for more details.
5  *
6  * Copyright (C) 1999, 2000 by Ralf Baechle
7  * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
8  */
9 #ifndef _ASM_SPINLOCK_H
10 #define _ASM_SPINLOCK_H
11
12 #include <asm/war.h>
13
14 /*
15  * Your basic SMP spinlocks, allowing only a single CPU anywhere
16  */
17
18 #define __raw_spin_is_locked(x)       ((x)->lock != 0)
19 #define __raw_spin_lock_flags(lock, flags) __raw_spin_lock(lock)
20 #define __raw_spin_unlock_wait(x) \
21         do { cpu_relax(); } while ((x)->lock)
22
23 /*
24  * Simple spin lock operations.  There are two variants, one clears IRQ's
25  * on the local processor, one does not.
26  *
27  * We make no fairness assumptions.  They have a cost.
28  */
29
30 static inline void __raw_spin_lock(raw_spinlock_t *lock)
31 {
32         unsigned int tmp;
33
34         if (R10000_LLSC_WAR) {
35                 __asm__ __volatile__(
36                 "       .set    noreorder       # __raw_spin_lock       \n"
37                 "1:     ll      %1, %2                                  \n"
38                 "       bnez    %1, 1b                                  \n"
39                 "        li     %1, 1                                   \n"
40                 "       sc      %1, %0                                  \n"
41                 "       beqzl   %1, 1b                                  \n"
42                 "        nop                                            \n"
43                 "       sync                                            \n"
44                 "       .set    reorder                                 \n"
45                 : "=m" (lock->lock), "=&r" (tmp)
46                 : "m" (lock->lock)
47                 : "memory");
48         } else {
49                 __asm__ __volatile__(
50                 "       .set    noreorder       # __raw_spin_lock       \n"
51                 "1:     ll      %1, %2                                  \n"
52                 "       bnez    %1, 1b                                  \n"
53                 "        li     %1, 1                                   \n"
54                 "       sc      %1, %0                                  \n"
55                 "       beqz    %1, 1b                                  \n"
56                 "        sync                                           \n"
57                 "       .set    reorder                                 \n"
58                 : "=m" (lock->lock), "=&r" (tmp)
59                 : "m" (lock->lock)
60                 : "memory");
61         }
62 }
63
64 static inline void __raw_spin_unlock(raw_spinlock_t *lock)
65 {
66         __asm__ __volatile__(
67         "       .set    noreorder       # __raw_spin_unlock     \n"
68         "       sync                                            \n"
69         "       sw      $0, %0                                  \n"
70         "       .set\treorder                                   \n"
71         : "=m" (lock->lock)
72         : "m" (lock->lock)
73         : "memory");
74 }
75
76 static inline unsigned int __raw_spin_trylock(raw_spinlock_t *lock)
77 {
78         unsigned int temp, res;
79
80         if (R10000_LLSC_WAR) {
81                 __asm__ __volatile__(
82                 "       .set    noreorder       # __raw_spin_trylock    \n"
83                 "1:     ll      %0, %3                                  \n"
84                 "       ori     %2, %0, 1                               \n"
85                 "       sc      %2, %1                                  \n"
86                 "       beqzl   %2, 1b                                  \n"
87                 "        nop                                            \n"
88                 "       andi    %2, %0, 1                               \n"
89                 "       sync                                            \n"
90                 "       .set    reorder"
91                 : "=&r" (temp), "=m" (lock->lock), "=&r" (res)
92                 : "m" (lock->lock)
93                 : "memory");
94         } else {
95                 __asm__ __volatile__(
96                 "       .set    noreorder       # __raw_spin_trylock    \n"
97                 "1:     ll      %0, %3                                  \n"
98                 "       ori     %2, %0, 1                               \n"
99                 "       sc      %2, %1                                  \n"
100                 "       beqz    %2, 1b                                  \n"
101                 "        andi   %2, %0, 1                               \n"
102                 "       sync                                            \n"
103                 "       .set    reorder"
104                 : "=&r" (temp), "=m" (lock->lock), "=&r" (res)
105                 : "m" (lock->lock)
106                 : "memory");
107         }
108
109         return res == 0;
110 }
111
112 /*
113  * Read-write spinlocks, allowing multiple readers but only one writer.
114  *
115  * NOTE! it is quite common to have readers in interrupts but no interrupt
116  * writers. For those circumstances we can "mix" irq-safe locks - any writer
117  * needs to get a irq-safe write-lock, but readers can get non-irqsafe
118  * read-locks.
119  */
120
121 /*
122  * read_can_lock - would read_trylock() succeed?
123  * @lock: the rwlock in question.
124  */
125 #define __raw_read_can_lock(rw) ((rw)->lock >= 0)
126
127 /*
128  * write_can_lock - would write_trylock() succeed?
129  * @lock: the rwlock in question.
130  */
131 #define __raw_write_can_lock(rw)        (!(rw)->lock)
132
133 static inline void __raw_read_lock(raw_rwlock_t *rw)
134 {
135         unsigned int tmp;
136
137         if (R10000_LLSC_WAR) {
138                 __asm__ __volatile__(
139                 "       .set    noreorder       # __raw_read_lock       \n"
140                 "1:     ll      %1, %2                                  \n"
141                 "       bltz    %1, 1b                                  \n"
142                 "        addu   %1, 1                                   \n"
143                 "       sc      %1, %0                                  \n"
144                 "       beqzl   %1, 1b                                  \n"
145                 "        nop                                            \n"
146                 "       sync                                            \n"
147                 "       .set    reorder                                 \n"
148                 : "=m" (rw->lock), "=&r" (tmp)
149                 : "m" (rw->lock)
150                 : "memory");
151         } else {
152                 __asm__ __volatile__(
153                 "       .set    noreorder       # __raw_read_lock       \n"
154                 "1:     ll      %1, %2                                  \n"
155                 "       bltz    %1, 1b                                  \n"
156                 "        addu   %1, 1                                   \n"
157                 "       sc      %1, %0                                  \n"
158                 "       beqz    %1, 1b                                  \n"
159                 "        sync                                           \n"
160                 "       .set    reorder                                 \n"
161                 : "=m" (rw->lock), "=&r" (tmp)
162                 : "m" (rw->lock)
163                 : "memory");
164         }
165 }
166
167 /* Note the use of sub, not subu which will make the kernel die with an
168    overflow exception if we ever try to unlock an rwlock that is already
169    unlocked or is being held by a writer.  */
170 static inline void __raw_read_unlock(raw_rwlock_t *rw)
171 {
172         unsigned int tmp;
173
174         if (R10000_LLSC_WAR) {
175                 __asm__ __volatile__(
176                 "1:     ll      %1, %2          # __raw_read_unlock     \n"
177                 "       sub     %1, 1                                   \n"
178                 "       sc      %1, %0                                  \n"
179                 "       beqzl   %1, 1b                                  \n"
180                 "       sync                                            \n"
181                 : "=m" (rw->lock), "=&r" (tmp)
182                 : "m" (rw->lock)
183                 : "memory");
184         } else {
185                 __asm__ __volatile__(
186                 "       .set    noreorder       # __raw_read_unlock     \n"
187                 "1:     ll      %1, %2                                  \n"
188                 "       sub     %1, 1                                   \n"
189                 "       sc      %1, %0                                  \n"
190                 "       beqz    %1, 1b                                  \n"
191                 "        sync                                           \n"
192                 "       .set    reorder                                 \n"
193                 : "=m" (rw->lock), "=&r" (tmp)
194                 : "m" (rw->lock)
195                 : "memory");
196         }
197 }
198
199 static inline void __raw_write_lock(raw_rwlock_t *rw)
200 {
201         unsigned int tmp;
202
203         if (R10000_LLSC_WAR) {
204                 __asm__ __volatile__(
205                 "       .set    noreorder       # __raw_write_lock      \n"
206                 "1:     ll      %1, %2                                  \n"
207                 "       bnez    %1, 1b                                  \n"
208                 "        lui    %1, 0x8000                              \n"
209                 "       sc      %1, %0                                  \n"
210                 "       beqzl   %1, 1b                                  \n"
211                 "        sync                                           \n"
212                 "       .set    reorder                                 \n"
213                 : "=m" (rw->lock), "=&r" (tmp)
214                 : "m" (rw->lock)
215                 : "memory");
216         } else {
217                 __asm__ __volatile__(
218                 "       .set    noreorder       # __raw_write_lock      \n"
219                 "1:     ll      %1, %2                                  \n"
220                 "       bnez    %1, 1b                                  \n"
221                 "        lui    %1, 0x8000                              \n"
222                 "       sc      %1, %0                                  \n"
223                 "       beqz    %1, 1b                                  \n"
224                 "        sync                                           \n"
225                 "       .set    reorder                                 \n"
226                 : "=m" (rw->lock), "=&r" (tmp)
227                 : "m" (rw->lock)
228                 : "memory");
229         }
230 }
231
232 static inline void __raw_write_unlock(raw_rwlock_t *rw)
233 {
234         __asm__ __volatile__(
235         "       sync                    # __raw_write_unlock    \n"
236         "       sw      $0, %0                                  \n"
237         : "=m" (rw->lock)
238         : "m" (rw->lock)
239         : "memory");
240 }
241
242 static inline int __raw_read_trylock(raw_rwlock_t *rw)
243 {
244         unsigned int tmp;
245         int ret;
246
247         if (R10000_LLSC_WAR) {
248                 __asm__ __volatile__(
249                 "       .set    noreorder       # __raw_read_trylock    \n"
250                 "       li      %2, 0                                   \n"
251                 "1:     ll      %1, %3                                  \n"
252                 "       bnez    %1, 2f                                  \n"
253                 "        addu   %1, 1                                   \n"
254                 "       sc      %1, %0                                  \n"
255                 "       beqzl   %1, 1b                                  \n"
256                 "       .set    reorder                                 \n"
257 #ifdef CONFIG_SMP
258                 "        sync                                           \n"
259 #endif
260                 "       li      %2, 1                                   \n"
261                 "2:                                                     \n"
262                 : "=m" (rw->lock), "=&r" (tmp), "=&r" (ret)
263                 : "m" (rw->lock)
264                 : "memory");
265         } else {
266                 __asm__ __volatile__(
267                 "       .set    noreorder       # __raw_read_trylock    \n"
268                 "       li      %2, 0                                   \n"
269                 "1:     ll      %1, %3                                  \n"
270                 "       bnez    %1, 2f                                  \n"
271                 "        addu   %1, 1                                   \n"
272                 "       sc      %1, %0                                  \n"
273                 "       beqz    %1, 1b                                  \n"
274                 "       .set    reorder                                 \n"
275 #ifdef CONFIG_SMP
276                 "        sync                                           \n"
277 #endif
278                 "       li      %2, 1                                   \n"
279                 "2:                                                     \n"
280                 : "=m" (rw->lock), "=&r" (tmp), "=&r" (ret)
281                 : "m" (rw->lock)
282                 : "memory");
283         }
284
285         return ret;
286 }
287
288 static inline int __raw_write_trylock(raw_rwlock_t *rw)
289 {
290         unsigned int tmp;
291         int ret;
292
293         if (R10000_LLSC_WAR) {
294                 __asm__ __volatile__(
295                 "       .set    noreorder       # __raw_write_trylock   \n"
296                 "       li      %2, 0                                   \n"
297                 "1:     ll      %1, %3                                  \n"
298                 "       bnez    %1, 2f                                  \n"
299                 "        lui    %1, 0x8000                              \n"
300                 "       sc      %1, %0                                  \n"
301                 "       beqzl   %1, 1b                                  \n"
302                 "        sync                                           \n"
303                 "       li      %2, 1                                   \n"
304                 "       .set    reorder                                 \n"
305                 "2:                                                     \n"
306                 : "=m" (rw->lock), "=&r" (tmp), "=&r" (ret)
307                 : "m" (rw->lock)
308                 : "memory");
309         } else {
310                 __asm__ __volatile__(
311                 "       .set    noreorder       # __raw_write_trylock   \n"
312                 "       li      %2, 0                                   \n"
313                 "1:     ll      %1, %3                                  \n"
314                 "       bnez    %1, 2f                                  \n"
315                 "       lui     %1, 0x8000                              \n"
316                 "       sc      %1, %0                                  \n"
317                 "       beqz    %1, 1b                                  \n"
318                 "        sync                                           \n"
319                 "       li      %2, 1                                   \n"
320                 "       .set    reorder                                 \n"
321                 "2:                                                     \n"
322                 : "=m" (rw->lock), "=&r" (tmp), "=&r" (ret)
323                 : "m" (rw->lock)
324                 : "memory");
325         }
326
327         return ret;
328 }
329
330
331 #define _raw_spin_relax(lock)   cpu_relax()
332 #define _raw_read_relax(lock)   cpu_relax()
333 #define _raw_write_relax(lock)  cpu_relax()
334
335 #endif /* _ASM_SPINLOCK_H */