Merge branch 'header-fixes-for-linus' of git://git.kernel.org/pub/scm/linux/kernel...
[linux-2.6.git] / arch / mips / include / asm / atomic.h
1 /*
2  * Atomic operations that C can't guarantee us.  Useful for
3  * resource counting etc..
4  *
5  * But use these as seldom as possible since they are much more slower
6  * than regular operations.
7  *
8  * This file is subject to the terms and conditions of the GNU General Public
9  * License.  See the file "COPYING" in the main directory of this archive
10  * for more details.
11  *
12  * Copyright (C) 1996, 97, 99, 2000, 03, 04, 06 by Ralf Baechle
13  */
14 #ifndef _ASM_ATOMIC_H
15 #define _ASM_ATOMIC_H
16
17 #include <linux/irqflags.h>
18 #include <linux/types.h>
19 #include <asm/barrier.h>
20 #include <asm/cpu-features.h>
21 #include <asm/war.h>
22 #include <asm/system.h>
23
24 #define ATOMIC_INIT(i)    { (i) }
25
26 /*
27  * atomic_read - read atomic variable
28  * @v: pointer of type atomic_t
29  *
30  * Atomically reads the value of @v.
31  */
32 #define atomic_read(v)          ((v)->counter)
33
34 /*
35  * atomic_set - set atomic variable
36  * @v: pointer of type atomic_t
37  * @i: required value
38  *
39  * Atomically sets the value of @v to @i.
40  */
41 #define atomic_set(v, i)                ((v)->counter = (i))
42
43 /*
44  * atomic_add - add integer to atomic variable
45  * @i: integer value to add
46  * @v: pointer of type atomic_t
47  *
48  * Atomically adds @i to @v.
49  */
50 static __inline__ void atomic_add(int i, atomic_t * v)
51 {
52         if (cpu_has_llsc && R10000_LLSC_WAR) {
53                 int temp;
54
55                 __asm__ __volatile__(
56                 "       .set    mips3                                   \n"
57                 "1:     ll      %0, %1          # atomic_add            \n"
58                 "       addu    %0, %2                                  \n"
59                 "       sc      %0, %1                                  \n"
60                 "       beqzl   %0, 1b                                  \n"
61                 "       .set    mips0                                   \n"
62                 : "=&r" (temp), "=m" (v->counter)
63                 : "Ir" (i), "m" (v->counter));
64         } else if (cpu_has_llsc) {
65                 int temp;
66
67                 __asm__ __volatile__(
68                 "       .set    mips3                                   \n"
69                 "1:     ll      %0, %1          # atomic_add            \n"
70                 "       addu    %0, %2                                  \n"
71                 "       sc      %0, %1                                  \n"
72                 "       beqz    %0, 2f                                  \n"
73                 "       .subsection 2                                   \n"
74                 "2:     b       1b                                      \n"
75                 "       .previous                                       \n"
76                 "       .set    mips0                                   \n"
77                 : "=&r" (temp), "=m" (v->counter)
78                 : "Ir" (i), "m" (v->counter));
79         } else {
80                 unsigned long flags;
81
82                 raw_local_irq_save(flags);
83                 v->counter += i;
84                 raw_local_irq_restore(flags);
85         }
86 }
87
88 /*
89  * atomic_sub - subtract the atomic variable
90  * @i: integer value to subtract
91  * @v: pointer of type atomic_t
92  *
93  * Atomically subtracts @i from @v.
94  */
95 static __inline__ void atomic_sub(int i, atomic_t * v)
96 {
97         if (cpu_has_llsc && R10000_LLSC_WAR) {
98                 int temp;
99
100                 __asm__ __volatile__(
101                 "       .set    mips3                                   \n"
102                 "1:     ll      %0, %1          # atomic_sub            \n"
103                 "       subu    %0, %2                                  \n"
104                 "       sc      %0, %1                                  \n"
105                 "       beqzl   %0, 1b                                  \n"
106                 "       .set    mips0                                   \n"
107                 : "=&r" (temp), "=m" (v->counter)
108                 : "Ir" (i), "m" (v->counter));
109         } else if (cpu_has_llsc) {
110                 int temp;
111
112                 __asm__ __volatile__(
113                 "       .set    mips3                                   \n"
114                 "1:     ll      %0, %1          # atomic_sub            \n"
115                 "       subu    %0, %2                                  \n"
116                 "       sc      %0, %1                                  \n"
117                 "       beqz    %0, 2f                                  \n"
118                 "       .subsection 2                                   \n"
119                 "2:     b       1b                                      \n"
120                 "       .previous                                       \n"
121                 "       .set    mips0                                   \n"
122                 : "=&r" (temp), "=m" (v->counter)
123                 : "Ir" (i), "m" (v->counter));
124         } else {
125                 unsigned long flags;
126
127                 raw_local_irq_save(flags);
128                 v->counter -= i;
129                 raw_local_irq_restore(flags);
130         }
131 }
132
133 /*
134  * Same as above, but return the result value
135  */
136 static __inline__ int atomic_add_return(int i, atomic_t * v)
137 {
138         int result;
139
140         smp_llsc_mb();
141
142         if (cpu_has_llsc && R10000_LLSC_WAR) {
143                 int temp;
144
145                 __asm__ __volatile__(
146                 "       .set    mips3                                   \n"
147                 "1:     ll      %1, %2          # atomic_add_return     \n"
148                 "       addu    %0, %1, %3                              \n"
149                 "       sc      %0, %2                                  \n"
150                 "       beqzl   %0, 1b                                  \n"
151                 "       addu    %0, %1, %3                              \n"
152                 "       .set    mips0                                   \n"
153                 : "=&r" (result), "=&r" (temp), "=m" (v->counter)
154                 : "Ir" (i), "m" (v->counter)
155                 : "memory");
156         } else if (cpu_has_llsc) {
157                 int temp;
158
159                 __asm__ __volatile__(
160                 "       .set    mips3                                   \n"
161                 "1:     ll      %1, %2          # atomic_add_return     \n"
162                 "       addu    %0, %1, %3                              \n"
163                 "       sc      %0, %2                                  \n"
164                 "       beqz    %0, 2f                                  \n"
165                 "       addu    %0, %1, %3                              \n"
166                 "       .subsection 2                                   \n"
167                 "2:     b       1b                                      \n"
168                 "       .previous                                       \n"
169                 "       .set    mips0                                   \n"
170                 : "=&r" (result), "=&r" (temp), "=m" (v->counter)
171                 : "Ir" (i), "m" (v->counter)
172                 : "memory");
173         } else {
174                 unsigned long flags;
175
176                 raw_local_irq_save(flags);
177                 result = v->counter;
178                 result += i;
179                 v->counter = result;
180                 raw_local_irq_restore(flags);
181         }
182
183         smp_llsc_mb();
184
185         return result;
186 }
187
188 static __inline__ int atomic_sub_return(int i, atomic_t * v)
189 {
190         int result;
191
192         smp_llsc_mb();
193
194         if (cpu_has_llsc && R10000_LLSC_WAR) {
195                 int temp;
196
197                 __asm__ __volatile__(
198                 "       .set    mips3                                   \n"
199                 "1:     ll      %1, %2          # atomic_sub_return     \n"
200                 "       subu    %0, %1, %3                              \n"
201                 "       sc      %0, %2                                  \n"
202                 "       beqzl   %0, 1b                                  \n"
203                 "       subu    %0, %1, %3                              \n"
204                 "       .set    mips0                                   \n"
205                 : "=&r" (result), "=&r" (temp), "=m" (v->counter)
206                 : "Ir" (i), "m" (v->counter)
207                 : "memory");
208         } else if (cpu_has_llsc) {
209                 int temp;
210
211                 __asm__ __volatile__(
212                 "       .set    mips3                                   \n"
213                 "1:     ll      %1, %2          # atomic_sub_return     \n"
214                 "       subu    %0, %1, %3                              \n"
215                 "       sc      %0, %2                                  \n"
216                 "       beqz    %0, 2f                                  \n"
217                 "       subu    %0, %1, %3                              \n"
218                 "       .subsection 2                                   \n"
219                 "2:     b       1b                                      \n"
220                 "       .previous                                       \n"
221                 "       .set    mips0                                   \n"
222                 : "=&r" (result), "=&r" (temp), "=m" (v->counter)
223                 : "Ir" (i), "m" (v->counter)
224                 : "memory");
225         } else {
226                 unsigned long flags;
227
228                 raw_local_irq_save(flags);
229                 result = v->counter;
230                 result -= i;
231                 v->counter = result;
232                 raw_local_irq_restore(flags);
233         }
234
235         smp_llsc_mb();
236
237         return result;
238 }
239
240 /*
241  * atomic_sub_if_positive - conditionally subtract integer from atomic variable
242  * @i: integer value to subtract
243  * @v: pointer of type atomic_t
244  *
245  * Atomically test @v and subtract @i if @v is greater or equal than @i.
246  * The function returns the old value of @v minus @i.
247  */
248 static __inline__ int atomic_sub_if_positive(int i, atomic_t * v)
249 {
250         int result;
251
252         smp_llsc_mb();
253
254         if (cpu_has_llsc && R10000_LLSC_WAR) {
255                 int temp;
256
257                 __asm__ __volatile__(
258                 "       .set    mips3                                   \n"
259                 "1:     ll      %1, %2          # atomic_sub_if_positive\n"
260                 "       subu    %0, %1, %3                              \n"
261                 "       bltz    %0, 1f                                  \n"
262                 "       sc      %0, %2                                  \n"
263                 "       .set    noreorder                               \n"
264                 "       beqzl   %0, 1b                                  \n"
265                 "        subu   %0, %1, %3                              \n"
266                 "       .set    reorder                                 \n"
267                 "1:                                                     \n"
268                 "       .set    mips0                                   \n"
269                 : "=&r" (result), "=&r" (temp), "=m" (v->counter)
270                 : "Ir" (i), "m" (v->counter)
271                 : "memory");
272         } else if (cpu_has_llsc) {
273                 int temp;
274
275                 __asm__ __volatile__(
276                 "       .set    mips3                                   \n"
277                 "1:     ll      %1, %2          # atomic_sub_if_positive\n"
278                 "       subu    %0, %1, %3                              \n"
279                 "       bltz    %0, 1f                                  \n"
280                 "       sc      %0, %2                                  \n"
281                 "       .set    noreorder                               \n"
282                 "       beqz    %0, 2f                                  \n"
283                 "        subu   %0, %1, %3                              \n"
284                 "       .set    reorder                                 \n"
285                 "       .subsection 2                                   \n"
286                 "2:     b       1b                                      \n"
287                 "       .previous                                       \n"
288                 "1:                                                     \n"
289                 "       .set    mips0                                   \n"
290                 : "=&r" (result), "=&r" (temp), "=m" (v->counter)
291                 : "Ir" (i), "m" (v->counter)
292                 : "memory");
293         } else {
294                 unsigned long flags;
295
296                 raw_local_irq_save(flags);
297                 result = v->counter;
298                 result -= i;
299                 if (result >= 0)
300                         v->counter = result;
301                 raw_local_irq_restore(flags);
302         }
303
304         smp_llsc_mb();
305
306         return result;
307 }
308
309 #define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n)))
310 #define atomic_xchg(v, new) (xchg(&((v)->counter), (new)))
311
312 /**
313  * atomic_add_unless - add unless the number is a given value
314  * @v: pointer of type atomic_t
315  * @a: the amount to add to v...
316  * @u: ...unless v is equal to u.
317  *
318  * Atomically adds @a to @v, so long as it was not @u.
319  * Returns non-zero if @v was not @u, and zero otherwise.
320  */
321 static __inline__ int atomic_add_unless(atomic_t *v, int a, int u)
322 {
323         int c, old;
324         c = atomic_read(v);
325         for (;;) {
326                 if (unlikely(c == (u)))
327                         break;
328                 old = atomic_cmpxchg((v), c, c + (a));
329                 if (likely(old == c))
330                         break;
331                 c = old;
332         }
333         return c != (u);
334 }
335 #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)
336
337 #define atomic_dec_return(v) atomic_sub_return(1, (v))
338 #define atomic_inc_return(v) atomic_add_return(1, (v))
339
340 /*
341  * atomic_sub_and_test - subtract value from variable and test result
342  * @i: integer value to subtract
343  * @v: pointer of type atomic_t
344  *
345  * Atomically subtracts @i from @v and returns
346  * true if the result is zero, or false for all
347  * other cases.
348  */
349 #define atomic_sub_and_test(i, v) (atomic_sub_return((i), (v)) == 0)
350
351 /*
352  * atomic_inc_and_test - increment and test
353  * @v: pointer of type atomic_t
354  *
355  * Atomically increments @v by 1
356  * and returns true if the result is zero, or false for all
357  * other cases.
358  */
359 #define atomic_inc_and_test(v) (atomic_inc_return(v) == 0)
360
361 /*
362  * atomic_dec_and_test - decrement by 1 and test
363  * @v: pointer of type atomic_t
364  *
365  * Atomically decrements @v by 1 and
366  * returns true if the result is 0, or false for all other
367  * cases.
368  */
369 #define atomic_dec_and_test(v) (atomic_sub_return(1, (v)) == 0)
370
371 /*
372  * atomic_dec_if_positive - decrement by 1 if old value positive
373  * @v: pointer of type atomic_t
374  */
375 #define atomic_dec_if_positive(v)       atomic_sub_if_positive(1, v)
376
377 /*
378  * atomic_inc - increment atomic variable
379  * @v: pointer of type atomic_t
380  *
381  * Atomically increments @v by 1.
382  */
383 #define atomic_inc(v) atomic_add(1, (v))
384
385 /*
386  * atomic_dec - decrement and test
387  * @v: pointer of type atomic_t
388  *
389  * Atomically decrements @v by 1.
390  */
391 #define atomic_dec(v) atomic_sub(1, (v))
392
393 /*
394  * atomic_add_negative - add and test if negative
395  * @v: pointer of type atomic_t
396  * @i: integer value to add
397  *
398  * Atomically adds @i to @v and returns true
399  * if the result is negative, or false when
400  * result is greater than or equal to zero.
401  */
402 #define atomic_add_negative(i, v) (atomic_add_return(i, (v)) < 0)
403
404 #ifdef CONFIG_64BIT
405
406 #define ATOMIC64_INIT(i)    { (i) }
407
408 /*
409  * atomic64_read - read atomic variable
410  * @v: pointer of type atomic64_t
411  *
412  */
413 #define atomic64_read(v)        ((v)->counter)
414
415 /*
416  * atomic64_set - set atomic variable
417  * @v: pointer of type atomic64_t
418  * @i: required value
419  */
420 #define atomic64_set(v, i)      ((v)->counter = (i))
421
422 /*
423  * atomic64_add - add integer to atomic variable
424  * @i: integer value to add
425  * @v: pointer of type atomic64_t
426  *
427  * Atomically adds @i to @v.
428  */
429 static __inline__ void atomic64_add(long i, atomic64_t * v)
430 {
431         if (cpu_has_llsc && R10000_LLSC_WAR) {
432                 long temp;
433
434                 __asm__ __volatile__(
435                 "       .set    mips3                                   \n"
436                 "1:     lld     %0, %1          # atomic64_add          \n"
437                 "       addu    %0, %2                                  \n"
438                 "       scd     %0, %1                                  \n"
439                 "       beqzl   %0, 1b                                  \n"
440                 "       .set    mips0                                   \n"
441                 : "=&r" (temp), "=m" (v->counter)
442                 : "Ir" (i), "m" (v->counter));
443         } else if (cpu_has_llsc) {
444                 long temp;
445
446                 __asm__ __volatile__(
447                 "       .set    mips3                                   \n"
448                 "1:     lld     %0, %1          # atomic64_add          \n"
449                 "       addu    %0, %2                                  \n"
450                 "       scd     %0, %1                                  \n"
451                 "       beqz    %0, 2f                                  \n"
452                 "       .subsection 2                                   \n"
453                 "2:     b       1b                                      \n"
454                 "       .previous                                       \n"
455                 "       .set    mips0                                   \n"
456                 : "=&r" (temp), "=m" (v->counter)
457                 : "Ir" (i), "m" (v->counter));
458         } else {
459                 unsigned long flags;
460
461                 raw_local_irq_save(flags);
462                 v->counter += i;
463                 raw_local_irq_restore(flags);
464         }
465 }
466
467 /*
468  * atomic64_sub - subtract the atomic variable
469  * @i: integer value to subtract
470  * @v: pointer of type atomic64_t
471  *
472  * Atomically subtracts @i from @v.
473  */
474 static __inline__ void atomic64_sub(long i, atomic64_t * v)
475 {
476         if (cpu_has_llsc && R10000_LLSC_WAR) {
477                 long temp;
478
479                 __asm__ __volatile__(
480                 "       .set    mips3                                   \n"
481                 "1:     lld     %0, %1          # atomic64_sub          \n"
482                 "       subu    %0, %2                                  \n"
483                 "       scd     %0, %1                                  \n"
484                 "       beqzl   %0, 1b                                  \n"
485                 "       .set    mips0                                   \n"
486                 : "=&r" (temp), "=m" (v->counter)
487                 : "Ir" (i), "m" (v->counter));
488         } else if (cpu_has_llsc) {
489                 long temp;
490
491                 __asm__ __volatile__(
492                 "       .set    mips3                                   \n"
493                 "1:     lld     %0, %1          # atomic64_sub          \n"
494                 "       subu    %0, %2                                  \n"
495                 "       scd     %0, %1                                  \n"
496                 "       beqz    %0, 2f                                  \n"
497                 "       .subsection 2                                   \n"
498                 "2:     b       1b                                      \n"
499                 "       .previous                                       \n"
500                 "       .set    mips0                                   \n"
501                 : "=&r" (temp), "=m" (v->counter)
502                 : "Ir" (i), "m" (v->counter));
503         } else {
504                 unsigned long flags;
505
506                 raw_local_irq_save(flags);
507                 v->counter -= i;
508                 raw_local_irq_restore(flags);
509         }
510 }
511
512 /*
513  * Same as above, but return the result value
514  */
515 static __inline__ long atomic64_add_return(long i, atomic64_t * v)
516 {
517         long result;
518
519         smp_llsc_mb();
520
521         if (cpu_has_llsc && R10000_LLSC_WAR) {
522                 long temp;
523
524                 __asm__ __volatile__(
525                 "       .set    mips3                                   \n"
526                 "1:     lld     %1, %2          # atomic64_add_return   \n"
527                 "       addu    %0, %1, %3                              \n"
528                 "       scd     %0, %2                                  \n"
529                 "       beqzl   %0, 1b                                  \n"
530                 "       addu    %0, %1, %3                              \n"
531                 "       .set    mips0                                   \n"
532                 : "=&r" (result), "=&r" (temp), "=m" (v->counter)
533                 : "Ir" (i), "m" (v->counter)
534                 : "memory");
535         } else if (cpu_has_llsc) {
536                 long temp;
537
538                 __asm__ __volatile__(
539                 "       .set    mips3                                   \n"
540                 "1:     lld     %1, %2          # atomic64_add_return   \n"
541                 "       addu    %0, %1, %3                              \n"
542                 "       scd     %0, %2                                  \n"
543                 "       beqz    %0, 2f                                  \n"
544                 "       addu    %0, %1, %3                              \n"
545                 "       .subsection 2                                   \n"
546                 "2:     b       1b                                      \n"
547                 "       .previous                                       \n"
548                 "       .set    mips0                                   \n"
549                 : "=&r" (result), "=&r" (temp), "=m" (v->counter)
550                 : "Ir" (i), "m" (v->counter)
551                 : "memory");
552         } else {
553                 unsigned long flags;
554
555                 raw_local_irq_save(flags);
556                 result = v->counter;
557                 result += i;
558                 v->counter = result;
559                 raw_local_irq_restore(flags);
560         }
561
562         smp_llsc_mb();
563
564         return result;
565 }
566
567 static __inline__ long atomic64_sub_return(long i, atomic64_t * v)
568 {
569         long result;
570
571         smp_llsc_mb();
572
573         if (cpu_has_llsc && R10000_LLSC_WAR) {
574                 long temp;
575
576                 __asm__ __volatile__(
577                 "       .set    mips3                                   \n"
578                 "1:     lld     %1, %2          # atomic64_sub_return   \n"
579                 "       subu    %0, %1, %3                              \n"
580                 "       scd     %0, %2                                  \n"
581                 "       beqzl   %0, 1b                                  \n"
582                 "       subu    %0, %1, %3                              \n"
583                 "       .set    mips0                                   \n"
584                 : "=&r" (result), "=&r" (temp), "=m" (v->counter)
585                 : "Ir" (i), "m" (v->counter)
586                 : "memory");
587         } else if (cpu_has_llsc) {
588                 long temp;
589
590                 __asm__ __volatile__(
591                 "       .set    mips3                                   \n"
592                 "1:     lld     %1, %2          # atomic64_sub_return   \n"
593                 "       subu    %0, %1, %3                              \n"
594                 "       scd     %0, %2                                  \n"
595                 "       beqz    %0, 2f                                  \n"
596                 "       subu    %0, %1, %3                              \n"
597                 "       .subsection 2                                   \n"
598                 "2:     b       1b                                      \n"
599                 "       .previous                                       \n"
600                 "       .set    mips0                                   \n"
601                 : "=&r" (result), "=&r" (temp), "=m" (v->counter)
602                 : "Ir" (i), "m" (v->counter)
603                 : "memory");
604         } else {
605                 unsigned long flags;
606
607                 raw_local_irq_save(flags);
608                 result = v->counter;
609                 result -= i;
610                 v->counter = result;
611                 raw_local_irq_restore(flags);
612         }
613
614         smp_llsc_mb();
615
616         return result;
617 }
618
619 /*
620  * atomic64_sub_if_positive - conditionally subtract integer from atomic variable
621  * @i: integer value to subtract
622  * @v: pointer of type atomic64_t
623  *
624  * Atomically test @v and subtract @i if @v is greater or equal than @i.
625  * The function returns the old value of @v minus @i.
626  */
627 static __inline__ long atomic64_sub_if_positive(long i, atomic64_t * v)
628 {
629         long result;
630
631         smp_llsc_mb();
632
633         if (cpu_has_llsc && R10000_LLSC_WAR) {
634                 long temp;
635
636                 __asm__ __volatile__(
637                 "       .set    mips3                                   \n"
638                 "1:     lld     %1, %2          # atomic64_sub_if_positive\n"
639                 "       dsubu   %0, %1, %3                              \n"
640                 "       bltz    %0, 1f                                  \n"
641                 "       scd     %0, %2                                  \n"
642                 "       .set    noreorder                               \n"
643                 "       beqzl   %0, 1b                                  \n"
644                 "        dsubu  %0, %1, %3                              \n"
645                 "       .set    reorder                                 \n"
646                 "1:                                                     \n"
647                 "       .set    mips0                                   \n"
648                 : "=&r" (result), "=&r" (temp), "=m" (v->counter)
649                 : "Ir" (i), "m" (v->counter)
650                 : "memory");
651         } else if (cpu_has_llsc) {
652                 long temp;
653
654                 __asm__ __volatile__(
655                 "       .set    mips3                                   \n"
656                 "1:     lld     %1, %2          # atomic64_sub_if_positive\n"
657                 "       dsubu   %0, %1, %3                              \n"
658                 "       bltz    %0, 1f                                  \n"
659                 "       scd     %0, %2                                  \n"
660                 "       .set    noreorder                               \n"
661                 "       beqz    %0, 2f                                  \n"
662                 "        dsubu  %0, %1, %3                              \n"
663                 "       .set    reorder                                 \n"
664                 "       .subsection 2                                   \n"
665                 "2:     b       1b                                      \n"
666                 "       .previous                                       \n"
667                 "1:                                                     \n"
668                 "       .set    mips0                                   \n"
669                 : "=&r" (result), "=&r" (temp), "=m" (v->counter)
670                 : "Ir" (i), "m" (v->counter)
671                 : "memory");
672         } else {
673                 unsigned long flags;
674
675                 raw_local_irq_save(flags);
676                 result = v->counter;
677                 result -= i;
678                 if (result >= 0)
679                         v->counter = result;
680                 raw_local_irq_restore(flags);
681         }
682
683         smp_llsc_mb();
684
685         return result;
686 }
687
688 #define atomic64_cmpxchg(v, o, n) \
689         ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n)))
690 #define atomic64_xchg(v, new) (xchg(&((v)->counter), (new)))
691
692 /**
693  * atomic64_add_unless - add unless the number is a given value
694  * @v: pointer of type atomic64_t
695  * @a: the amount to add to v...
696  * @u: ...unless v is equal to u.
697  *
698  * Atomically adds @a to @v, so long as it was not @u.
699  * Returns non-zero if @v was not @u, and zero otherwise.
700  */
701 static __inline__ int atomic64_add_unless(atomic64_t *v, long a, long u)
702 {
703         long c, old;
704         c = atomic64_read(v);
705         for (;;) {
706                 if (unlikely(c == (u)))
707                         break;
708                 old = atomic64_cmpxchg((v), c, c + (a));
709                 if (likely(old == c))
710                         break;
711                 c = old;
712         }
713         return c != (u);
714 }
715
716 #define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0)
717
718 #define atomic64_dec_return(v) atomic64_sub_return(1, (v))
719 #define atomic64_inc_return(v) atomic64_add_return(1, (v))
720
721 /*
722  * atomic64_sub_and_test - subtract value from variable and test result
723  * @i: integer value to subtract
724  * @v: pointer of type atomic64_t
725  *
726  * Atomically subtracts @i from @v and returns
727  * true if the result is zero, or false for all
728  * other cases.
729  */
730 #define atomic64_sub_and_test(i, v) (atomic64_sub_return((i), (v)) == 0)
731
732 /*
733  * atomic64_inc_and_test - increment and test
734  * @v: pointer of type atomic64_t
735  *
736  * Atomically increments @v by 1
737  * and returns true if the result is zero, or false for all
738  * other cases.
739  */
740 #define atomic64_inc_and_test(v) (atomic64_inc_return(v) == 0)
741
742 /*
743  * atomic64_dec_and_test - decrement by 1 and test
744  * @v: pointer of type atomic64_t
745  *
746  * Atomically decrements @v by 1 and
747  * returns true if the result is 0, or false for all other
748  * cases.
749  */
750 #define atomic64_dec_and_test(v) (atomic64_sub_return(1, (v)) == 0)
751
752 /*
753  * atomic64_dec_if_positive - decrement by 1 if old value positive
754  * @v: pointer of type atomic64_t
755  */
756 #define atomic64_dec_if_positive(v)     atomic64_sub_if_positive(1, v)
757
758 /*
759  * atomic64_inc - increment atomic variable
760  * @v: pointer of type atomic64_t
761  *
762  * Atomically increments @v by 1.
763  */
764 #define atomic64_inc(v) atomic64_add(1, (v))
765
766 /*
767  * atomic64_dec - decrement and test
768  * @v: pointer of type atomic64_t
769  *
770  * Atomically decrements @v by 1.
771  */
772 #define atomic64_dec(v) atomic64_sub(1, (v))
773
774 /*
775  * atomic64_add_negative - add and test if negative
776  * @v: pointer of type atomic64_t
777  * @i: integer value to add
778  *
779  * Atomically adds @i to @v and returns true
780  * if the result is negative, or false when
781  * result is greater than or equal to zero.
782  */
783 #define atomic64_add_negative(i, v) (atomic64_add_return(i, (v)) < 0)
784
785 #endif /* CONFIG_64BIT */
786
787 /*
788  * atomic*_return operations are serializing but not the non-*_return
789  * versions.
790  */
791 #define smp_mb__before_atomic_dec()     smp_llsc_mb()
792 #define smp_mb__after_atomic_dec()      smp_llsc_mb()
793 #define smp_mb__before_atomic_inc()     smp_llsc_mb()
794 #define smp_mb__after_atomic_inc()      smp_llsc_mb()
795
796 #include <asm-generic/atomic.h>
797
798 #endif /* _ASM_ATOMIC_H */