cpu: convert 'cpu' and 'machinecheck' sysdev_class to a regular subsystem
[linux-2.6.git] / arch / x86 / include / asm / cmpxchg_32.h
1 #ifndef _ASM_X86_CMPXCHG_32_H
2 #define _ASM_X86_CMPXCHG_32_H
3
4 /*
5  * Note: if you use set64_bit(), __cmpxchg64(), or their variants, you
6  *       you need to test for the feature in boot_cpu_data.
7  */
8
9 /*
10  * CMPXCHG8B only writes to the target if we had the previous
11  * value in registers, otherwise it acts as a read and gives us the
12  * "new previous" value.  That is why there is a loop.  Preloading
13  * EDX:EAX is a performance optimization: in the common case it means
14  * we need only one locked operation.
15  *
16  * A SIMD/3DNOW!/MMX/FPU 64-bit store here would require at the very
17  * least an FPU save and/or %cr0.ts manipulation.
18  *
19  * cmpxchg8b must be used with the lock prefix here to allow the
20  * instruction to be executed atomically.  We need to have the reader
21  * side to see the coherent 64bit value.
22  */
23 static inline void set_64bit(volatile u64 *ptr, u64 value)
24 {
25         u32 low  = value;
26         u32 high = value >> 32;
27         u64 prev = *ptr;
28
29         asm volatile("\n1:\t"
30                      LOCK_PREFIX "cmpxchg8b %0\n\t"
31                      "jnz 1b"
32                      : "=m" (*ptr), "+A" (prev)
33                      : "b" (low), "c" (high)
34                      : "memory");
35 }
36
37 #ifdef CONFIG_X86_CMPXCHG
38 #define __HAVE_ARCH_CMPXCHG 1
39 #endif
40
41 #ifdef CONFIG_X86_CMPXCHG64
42 #define cmpxchg64(ptr, o, n)                                            \
43         ((__typeof__(*(ptr)))__cmpxchg64((ptr), (unsigned long long)(o), \
44                                          (unsigned long long)(n)))
45 #define cmpxchg64_local(ptr, o, n)                                      \
46         ((__typeof__(*(ptr)))__cmpxchg64_local((ptr), (unsigned long long)(o), \
47                                                (unsigned long long)(n)))
48 #endif
49
50 static inline u64 __cmpxchg64(volatile u64 *ptr, u64 old, u64 new)
51 {
52         u64 prev;
53         asm volatile(LOCK_PREFIX "cmpxchg8b %1"
54                      : "=A" (prev),
55                        "+m" (*ptr)
56                      : "b" ((u32)new),
57                        "c" ((u32)(new >> 32)),
58                        "0" (old)
59                      : "memory");
60         return prev;
61 }
62
63 static inline u64 __cmpxchg64_local(volatile u64 *ptr, u64 old, u64 new)
64 {
65         u64 prev;
66         asm volatile("cmpxchg8b %1"
67                      : "=A" (prev),
68                        "+m" (*ptr)
69                      : "b" ((u32)new),
70                        "c" ((u32)(new >> 32)),
71                        "0" (old)
72                      : "memory");
73         return prev;
74 }
75
76 #ifndef CONFIG_X86_CMPXCHG
77 /*
78  * Building a kernel capable running on 80386. It may be necessary to
79  * simulate the cmpxchg on the 80386 CPU. For that purpose we define
80  * a function for each of the sizes we support.
81  */
82
83 extern unsigned long cmpxchg_386_u8(volatile void *, u8, u8);
84 extern unsigned long cmpxchg_386_u16(volatile void *, u16, u16);
85 extern unsigned long cmpxchg_386_u32(volatile void *, u32, u32);
86
87 static inline unsigned long cmpxchg_386(volatile void *ptr, unsigned long old,
88                                         unsigned long new, int size)
89 {
90         switch (size) {
91         case 1:
92                 return cmpxchg_386_u8(ptr, old, new);
93         case 2:
94                 return cmpxchg_386_u16(ptr, old, new);
95         case 4:
96                 return cmpxchg_386_u32(ptr, old, new);
97         }
98         return old;
99 }
100
101 #define cmpxchg(ptr, o, n)                                              \
102 ({                                                                      \
103         __typeof__(*(ptr)) __ret;                                       \
104         if (likely(boot_cpu_data.x86 > 3))                              \
105                 __ret = (__typeof__(*(ptr)))__cmpxchg((ptr),            \
106                                 (unsigned long)(o), (unsigned long)(n), \
107                                 sizeof(*(ptr)));                        \
108         else                                                            \
109                 __ret = (__typeof__(*(ptr)))cmpxchg_386((ptr),          \
110                                 (unsigned long)(o), (unsigned long)(n), \
111                                 sizeof(*(ptr)));                        \
112         __ret;                                                          \
113 })
114 #define cmpxchg_local(ptr, o, n)                                        \
115 ({                                                                      \
116         __typeof__(*(ptr)) __ret;                                       \
117         if (likely(boot_cpu_data.x86 > 3))                              \
118                 __ret = (__typeof__(*(ptr)))__cmpxchg_local((ptr),      \
119                                 (unsigned long)(o), (unsigned long)(n), \
120                                 sizeof(*(ptr)));                        \
121         else                                                            \
122                 __ret = (__typeof__(*(ptr)))cmpxchg_386((ptr),          \
123                                 (unsigned long)(o), (unsigned long)(n), \
124                                 sizeof(*(ptr)));                        \
125         __ret;                                                          \
126 })
127 #endif
128
129 #ifndef CONFIG_X86_CMPXCHG64
130 /*
131  * Building a kernel capable running on 80386 and 80486. It may be necessary
132  * to simulate the cmpxchg8b on the 80386 and 80486 CPU.
133  */
134
135 #define cmpxchg64(ptr, o, n)                                    \
136 ({                                                              \
137         __typeof__(*(ptr)) __ret;                               \
138         __typeof__(*(ptr)) __old = (o);                         \
139         __typeof__(*(ptr)) __new = (n);                         \
140         alternative_io(LOCK_PREFIX_HERE                         \
141                         "call cmpxchg8b_emu",                   \
142                         "lock; cmpxchg8b (%%esi)" ,             \
143                        X86_FEATURE_CX8,                         \
144                        "=A" (__ret),                            \
145                        "S" ((ptr)), "0" (__old),                \
146                        "b" ((unsigned int)__new),               \
147                        "c" ((unsigned int)(__new>>32))          \
148                        : "memory");                             \
149         __ret; })
150
151
152 #define cmpxchg64_local(ptr, o, n)                              \
153 ({                                                              \
154         __typeof__(*(ptr)) __ret;                               \
155         __typeof__(*(ptr)) __old = (o);                         \
156         __typeof__(*(ptr)) __new = (n);                         \
157         alternative_io("call cmpxchg8b_emu",                    \
158                        "cmpxchg8b (%%esi)" ,                    \
159                        X86_FEATURE_CX8,                         \
160                        "=A" (__ret),                            \
161                        "S" ((ptr)), "0" (__old),                \
162                        "b" ((unsigned int)__new),               \
163                        "c" ((unsigned int)(__new>>32))          \
164                        : "memory");                             \
165         __ret; })
166
167 #endif
168
169 #define cmpxchg8b(ptr, o1, o2, n1, n2)                          \
170 ({                                                              \
171         char __ret;                                             \
172         __typeof__(o2) __dummy;                                 \
173         __typeof__(*(ptr)) __old1 = (o1);                       \
174         __typeof__(o2) __old2 = (o2);                           \
175         __typeof__(*(ptr)) __new1 = (n1);                       \
176         __typeof__(o2) __new2 = (n2);                           \
177         asm volatile(LOCK_PREFIX "cmpxchg8b %2; setz %1"        \
178                        : "=d"(__dummy), "=a" (__ret), "+m" (*ptr)\
179                        : "a" (__old1), "d"(__old2),             \
180                          "b" (__new1), "c" (__new2)             \
181                        : "memory");                             \
182         __ret; })
183
184
185 #define cmpxchg8b_local(ptr, o1, o2, n1, n2)                    \
186 ({                                                              \
187         char __ret;                                             \
188         __typeof__(o2) __dummy;                                 \
189         __typeof__(*(ptr)) __old1 = (o1);                       \
190         __typeof__(o2) __old2 = (o2);                           \
191         __typeof__(*(ptr)) __new1 = (n1);                       \
192         __typeof__(o2) __new2 = (n2);                           \
193         asm volatile("cmpxchg8b %2; setz %1"                    \
194                        : "=d"(__dummy), "=a"(__ret), "+m" (*ptr)\
195                        : "a" (__old), "d"(__old2),              \
196                          "b" (__new1), "c" (__new2),            \
197                        : "memory");                             \
198         __ret; })
199
200
201 #define cmpxchg_double(ptr, o1, o2, n1, n2)                             \
202 ({                                                                      \
203         BUILD_BUG_ON(sizeof(*(ptr)) != 4);                              \
204         VM_BUG_ON((unsigned long)(ptr) % 8);                            \
205         cmpxchg8b((ptr), (o1), (o2), (n1), (n2));                       \
206 })
207
208 #define cmpxchg_double_local(ptr, o1, o2, n1, n2)                       \
209 ({                                                                      \
210        BUILD_BUG_ON(sizeof(*(ptr)) != 4);                               \
211        VM_BUG_ON((unsigned long)(ptr) % 8);                             \
212        cmpxchg16b_local((ptr), (o1), (o2), (n1), (n2));                 \
213 })
214
215 #define system_has_cmpxchg_double() cpu_has_cx8
216
217 #endif /* _ASM_X86_CMPXCHG_32_H */