Remove tas()
[linux-2.6.git] / include / asm-alpha / system.h
1 #ifndef __ALPHA_SYSTEM_H
2 #define __ALPHA_SYSTEM_H
3
4 #include <asm/pal.h>
5 #include <asm/page.h>
6 #include <asm/barrier.h>
7
8 /*
9  * System defines.. Note that this is included both from .c and .S
10  * files, so it does only defines, not any C code.
11  */
12
13 /*
14  * We leave one page for the initial stack page, and one page for
15  * the initial process structure. Also, the console eats 3 MB for
16  * the initial bootloader (one of which we can reclaim later).
17  */
18 #define BOOT_PCB        0x20000000
19 #define BOOT_ADDR       0x20000000
20 /* Remove when official MILO sources have ELF support: */
21 #define BOOT_SIZE       (16*1024)
22
23 #ifdef CONFIG_ALPHA_LEGACY_START_ADDRESS
24 #define KERNEL_START_PHYS       0x300000 /* Old bootloaders hardcoded this.  */
25 #else
26 #define KERNEL_START_PHYS       0x1000000 /* required: Wildfire/Titan/Marvel */
27 #endif
28
29 #define KERNEL_START    (PAGE_OFFSET+KERNEL_START_PHYS)
30 #define SWAPPER_PGD     KERNEL_START
31 #define INIT_STACK      (PAGE_OFFSET+KERNEL_START_PHYS+0x02000)
32 #define EMPTY_PGT       (PAGE_OFFSET+KERNEL_START_PHYS+0x04000)
33 #define EMPTY_PGE       (PAGE_OFFSET+KERNEL_START_PHYS+0x08000)
34 #define ZERO_PGE        (PAGE_OFFSET+KERNEL_START_PHYS+0x0A000)
35
36 #define START_ADDR      (PAGE_OFFSET+KERNEL_START_PHYS+0x10000)
37
38 /*
39  * This is setup by the secondary bootstrap loader.  Because
40  * the zero page is zeroed out as soon as the vm system is
41  * initialized, we need to copy things out into a more permanent
42  * place.
43  */
44 #define PARAM                   ZERO_PGE
45 #define COMMAND_LINE            ((char*)(PARAM + 0x0000))
46 #define INITRD_START            (*(unsigned long *) (PARAM+0x100))
47 #define INITRD_SIZE             (*(unsigned long *) (PARAM+0x108))
48
49 #ifndef __ASSEMBLY__
50 #include <linux/kernel.h>
51
52 /*
53  * This is the logout header that should be common to all platforms
54  * (assuming they are running OSF/1 PALcode, I guess).
55  */
56 struct el_common {
57         unsigned int    size;           /* size in bytes of logout area */
58         unsigned int    sbz1    : 30;   /* should be zero */
59         unsigned int    err2    :  1;   /* second error */
60         unsigned int    retry   :  1;   /* retry flag */
61         unsigned int    proc_offset;    /* processor-specific offset */
62         unsigned int    sys_offset;     /* system-specific offset */
63         unsigned int    code;           /* machine check code */
64         unsigned int    frame_rev;      /* frame revision */
65 };
66
67 /* Machine Check Frame for uncorrectable errors (Large format)
68  *      --- This is used to log uncorrectable errors such as
69  *          double bit ECC errors.
70  *      --- These errors are detected by both processor and systems.
71  */
72 struct el_common_EV5_uncorrectable_mcheck {
73         unsigned long   shadow[8];        /* Shadow reg. 8-14, 25           */
74         unsigned long   paltemp[24];      /* PAL TEMP REGS.                 */
75         unsigned long   exc_addr;         /* Address of excepting instruction*/
76         unsigned long   exc_sum;          /* Summary of arithmetic traps.   */
77         unsigned long   exc_mask;         /* Exception mask (from exc_sum). */
78         unsigned long   pal_base;         /* Base address for PALcode.      */
79         unsigned long   isr;              /* Interrupt Status Reg.          */
80         unsigned long   icsr;             /* CURRENT SETUP OF EV5 IBOX      */
81         unsigned long   ic_perr_stat;     /* I-CACHE Reg. <11> set Data parity
82                                                          <12> set TAG parity*/
83         unsigned long   dc_perr_stat;     /* D-CACHE error Reg. Bits set to 1:
84                                                      <2> Data error in bank 0
85                                                      <3> Data error in bank 1
86                                                      <4> Tag error in bank 0
87                                                      <5> Tag error in bank 1 */
88         unsigned long   va;               /* Effective VA of fault or miss. */
89         unsigned long   mm_stat;          /* Holds the reason for D-stream 
90                                              fault or D-cache parity errors */
91         unsigned long   sc_addr;          /* Address that was being accessed
92                                              when EV5 detected Secondary cache
93                                              failure.                 */
94         unsigned long   sc_stat;          /* Helps determine if the error was
95                                              TAG/Data parity(Secondary Cache)*/
96         unsigned long   bc_tag_addr;      /* Contents of EV5 BC_TAG_ADDR    */
97         unsigned long   ei_addr;          /* Physical address of any transfer
98                                              that is logged in EV5 EI_STAT */
99         unsigned long   fill_syndrome;    /* For correcting ECC errors.     */
100         unsigned long   ei_stat;          /* Helps identify reason of any 
101                                              processor uncorrectable error
102                                              at its external interface.     */
103         unsigned long   ld_lock;          /* Contents of EV5 LD_LOCK register*/
104 };
105
106 struct el_common_EV6_mcheck {
107         unsigned int FrameSize;         /* Bytes, including this field */
108         unsigned int FrameFlags;        /* <31> = Retry, <30> = Second Error */
109         unsigned int CpuOffset;         /* Offset to CPU-specific info */
110         unsigned int SystemOffset;      /* Offset to system-specific info */
111         unsigned int MCHK_Code;
112         unsigned int MCHK_Frame_Rev;
113         unsigned long I_STAT;           /* EV6 Internal Processor Registers */
114         unsigned long DC_STAT;          /* (See the 21264 Spec) */
115         unsigned long C_ADDR;
116         unsigned long DC1_SYNDROME;
117         unsigned long DC0_SYNDROME;
118         unsigned long C_STAT;
119         unsigned long C_STS;
120         unsigned long MM_STAT;
121         unsigned long EXC_ADDR;
122         unsigned long IER_CM;
123         unsigned long ISUM;
124         unsigned long RESERVED0;
125         unsigned long PAL_BASE;
126         unsigned long I_CTL;
127         unsigned long PCTX;
128 };
129
130 extern void halt(void) __attribute__((noreturn));
131 #define __halt() __asm__ __volatile__ ("call_pal %0 #halt" : : "i" (PAL_halt))
132
133 #define switch_to(P,N,L)                                                 \
134   do {                                                                   \
135     (L) = alpha_switch_to(virt_to_phys(&task_thread_info(N)->pcb), (P)); \
136     check_mmu_context();                                                 \
137   } while (0)
138
139 struct task_struct;
140 extern struct task_struct *alpha_switch_to(unsigned long, struct task_struct*);
141
142 /*
143  * On SMP systems, when the scheduler does migration-cost autodetection,
144  * it needs a way to flush as much of the CPU's caches as possible.
145  *
146  * TODO: fill this in!
147  */
148 static inline void sched_cacheflush(void)
149 {
150 }
151
152 #define imb() \
153 __asm__ __volatile__ ("call_pal %0 #imb" : : "i" (PAL_imb) : "memory")
154
155 #define draina() \
156 __asm__ __volatile__ ("call_pal %0 #draina" : : "i" (PAL_draina) : "memory")
157
158 enum implver_enum {
159         IMPLVER_EV4,
160         IMPLVER_EV5,
161         IMPLVER_EV6
162 };
163
164 #ifdef CONFIG_ALPHA_GENERIC
165 #define implver()                               \
166 ({ unsigned long __implver;                     \
167    __asm__ ("implver %0" : "=r"(__implver));    \
168    (enum implver_enum) __implver; })
169 #else
170 /* Try to eliminate some dead code.  */
171 #ifdef CONFIG_ALPHA_EV4
172 #define implver() IMPLVER_EV4
173 #endif
174 #ifdef CONFIG_ALPHA_EV5
175 #define implver() IMPLVER_EV5
176 #endif
177 #if defined(CONFIG_ALPHA_EV6)
178 #define implver() IMPLVER_EV6
179 #endif
180 #endif
181
182 enum amask_enum {
183         AMASK_BWX = (1UL << 0),
184         AMASK_FIX = (1UL << 1),
185         AMASK_CIX = (1UL << 2),
186         AMASK_MAX = (1UL << 8),
187         AMASK_PRECISE_TRAP = (1UL << 9),
188 };
189
190 #define amask(mask)                                             \
191 ({ unsigned long __amask, __input = (mask);                     \
192    __asm__ ("amask %1,%0" : "=r"(__amask) : "rI"(__input));     \
193    __amask; })
194
195 #define __CALL_PAL_R0(NAME, TYPE)                               \
196 static inline TYPE NAME(void)                                   \
197 {                                                               \
198         register TYPE __r0 __asm__("$0");                       \
199         __asm__ __volatile__(                                   \
200                 "call_pal %1 # " #NAME                          \
201                 :"=r" (__r0)                                    \
202                 :"i" (PAL_ ## NAME)                             \
203                 :"$1", "$16", "$22", "$23", "$24", "$25");      \
204         return __r0;                                            \
205 }
206
207 #define __CALL_PAL_W1(NAME, TYPE0)                              \
208 static inline void NAME(TYPE0 arg0)                             \
209 {                                                               \
210         register TYPE0 __r16 __asm__("$16") = arg0;             \
211         __asm__ __volatile__(                                   \
212                 "call_pal %1 # "#NAME                           \
213                 : "=r"(__r16)                                   \
214                 : "i"(PAL_ ## NAME), "0"(__r16)                 \
215                 : "$1", "$22", "$23", "$24", "$25");            \
216 }
217
218 #define __CALL_PAL_W2(NAME, TYPE0, TYPE1)                       \
219 static inline void NAME(TYPE0 arg0, TYPE1 arg1)                 \
220 {                                                               \
221         register TYPE0 __r16 __asm__("$16") = arg0;             \
222         register TYPE1 __r17 __asm__("$17") = arg1;             \
223         __asm__ __volatile__(                                   \
224                 "call_pal %2 # "#NAME                           \
225                 : "=r"(__r16), "=r"(__r17)                      \
226                 : "i"(PAL_ ## NAME), "0"(__r16), "1"(__r17)     \
227                 : "$1", "$22", "$23", "$24", "$25");            \
228 }
229
230 #define __CALL_PAL_RW1(NAME, RTYPE, TYPE0)                      \
231 static inline RTYPE NAME(TYPE0 arg0)                            \
232 {                                                               \
233         register RTYPE __r0 __asm__("$0");                      \
234         register TYPE0 __r16 __asm__("$16") = arg0;             \
235         __asm__ __volatile__(                                   \
236                 "call_pal %2 # "#NAME                           \
237                 : "=r"(__r16), "=r"(__r0)                       \
238                 : "i"(PAL_ ## NAME), "0"(__r16)                 \
239                 : "$1", "$22", "$23", "$24", "$25");            \
240         return __r0;                                            \
241 }
242
243 #define __CALL_PAL_RW2(NAME, RTYPE, TYPE0, TYPE1)               \
244 static inline RTYPE NAME(TYPE0 arg0, TYPE1 arg1)                \
245 {                                                               \
246         register RTYPE __r0 __asm__("$0");                      \
247         register TYPE0 __r16 __asm__("$16") = arg0;             \
248         register TYPE1 __r17 __asm__("$17") = arg1;             \
249         __asm__ __volatile__(                                   \
250                 "call_pal %3 # "#NAME                           \
251                 : "=r"(__r16), "=r"(__r17), "=r"(__r0)          \
252                 : "i"(PAL_ ## NAME), "0"(__r16), "1"(__r17)     \
253                 : "$1", "$22", "$23", "$24", "$25");            \
254         return __r0;                                            \
255 }
256
257 __CALL_PAL_W1(cflush, unsigned long);
258 __CALL_PAL_R0(rdmces, unsigned long);
259 __CALL_PAL_R0(rdps, unsigned long);
260 __CALL_PAL_R0(rdusp, unsigned long);
261 __CALL_PAL_RW1(swpipl, unsigned long, unsigned long);
262 __CALL_PAL_R0(whami, unsigned long);
263 __CALL_PAL_W2(wrent, void*, unsigned long);
264 __CALL_PAL_W1(wripir, unsigned long);
265 __CALL_PAL_W1(wrkgp, unsigned long);
266 __CALL_PAL_W1(wrmces, unsigned long);
267 __CALL_PAL_RW2(wrperfmon, unsigned long, unsigned long, unsigned long);
268 __CALL_PAL_W1(wrusp, unsigned long);
269 __CALL_PAL_W1(wrvptptr, unsigned long);
270
271 #define IPL_MIN         0
272 #define IPL_SW0         1
273 #define IPL_SW1         2
274 #define IPL_DEV0        3
275 #define IPL_DEV1        4
276 #define IPL_TIMER       5
277 #define IPL_PERF        6
278 #define IPL_POWERFAIL   6
279 #define IPL_MCHECK      7
280 #define IPL_MAX         7
281
282 #ifdef CONFIG_ALPHA_BROKEN_IRQ_MASK
283 #undef IPL_MIN
284 #define IPL_MIN         __min_ipl
285 extern int __min_ipl;
286 #endif
287
288 #define getipl()                (rdps() & 7)
289 #define setipl(ipl)             ((void) swpipl(ipl))
290
291 #define local_irq_disable()                     do { setipl(IPL_MAX); barrier(); } while(0)
292 #define local_irq_enable()                      do { barrier(); setipl(IPL_MIN); } while(0)
293 #define local_save_flags(flags) ((flags) = rdps())
294 #define local_irq_save(flags)   do { (flags) = swpipl(IPL_MAX); barrier(); } while(0)
295 #define local_irq_restore(flags)        do { barrier(); setipl(flags); barrier(); } while(0)
296
297 #define irqs_disabled() (getipl() == IPL_MAX)
298
299 /*
300  * TB routines..
301  */
302 #define __tbi(nr,arg,arg1...)                                   \
303 ({                                                              \
304         register unsigned long __r16 __asm__("$16") = (nr);     \
305         register unsigned long __r17 __asm__("$17"); arg;       \
306         __asm__ __volatile__(                                   \
307                 "call_pal %3 #__tbi"                            \
308                 :"=r" (__r16),"=r" (__r17)                      \
309                 :"0" (__r16),"i" (PAL_tbi) ,##arg1              \
310                 :"$0", "$1", "$22", "$23", "$24", "$25");       \
311 })
312
313 #define tbi(x,y)        __tbi(x,__r17=(y),"1" (__r17))
314 #define tbisi(x)        __tbi(1,__r17=(x),"1" (__r17))
315 #define tbisd(x)        __tbi(2,__r17=(x),"1" (__r17))
316 #define tbis(x)         __tbi(3,__r17=(x),"1" (__r17))
317 #define tbiap()         __tbi(-1, /* no second argument */)
318 #define tbia()          __tbi(-2, /* no second argument */)
319
320 /*
321  * Atomic exchange.
322  * Since it can be used to implement critical sections
323  * it must clobber "memory" (also for interrupts in UP).
324  */
325
326 static inline unsigned long
327 __xchg_u8(volatile char *m, unsigned long val)
328 {
329         unsigned long ret, tmp, addr64;
330
331         __asm__ __volatile__(
332         "       andnot  %4,7,%3\n"
333         "       insbl   %1,%4,%1\n"
334         "1:     ldq_l   %2,0(%3)\n"
335         "       extbl   %2,%4,%0\n"
336         "       mskbl   %2,%4,%2\n"
337         "       or      %1,%2,%2\n"
338         "       stq_c   %2,0(%3)\n"
339         "       beq     %2,2f\n"
340 #ifdef CONFIG_SMP
341         "       mb\n"
342 #endif
343         ".subsection 2\n"
344         "2:     br      1b\n"
345         ".previous"
346         : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64)
347         : "r" ((long)m), "1" (val) : "memory");
348
349         return ret;
350 }
351
352 static inline unsigned long
353 __xchg_u16(volatile short *m, unsigned long val)
354 {
355         unsigned long ret, tmp, addr64;
356
357         __asm__ __volatile__(
358         "       andnot  %4,7,%3\n"
359         "       inswl   %1,%4,%1\n"
360         "1:     ldq_l   %2,0(%3)\n"
361         "       extwl   %2,%4,%0\n"
362         "       mskwl   %2,%4,%2\n"
363         "       or      %1,%2,%2\n"
364         "       stq_c   %2,0(%3)\n"
365         "       beq     %2,2f\n"
366 #ifdef CONFIG_SMP
367         "       mb\n"
368 #endif
369         ".subsection 2\n"
370         "2:     br      1b\n"
371         ".previous"
372         : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64)
373         : "r" ((long)m), "1" (val) : "memory");
374
375         return ret;
376 }
377
378 static inline unsigned long
379 __xchg_u32(volatile int *m, unsigned long val)
380 {
381         unsigned long dummy;
382
383         __asm__ __volatile__(
384         "1:     ldl_l %0,%4\n"
385         "       bis $31,%3,%1\n"
386         "       stl_c %1,%2\n"
387         "       beq %1,2f\n"
388 #ifdef CONFIG_SMP
389         "       mb\n"
390 #endif
391         ".subsection 2\n"
392         "2:     br 1b\n"
393         ".previous"
394         : "=&r" (val), "=&r" (dummy), "=m" (*m)
395         : "rI" (val), "m" (*m) : "memory");
396
397         return val;
398 }
399
400 static inline unsigned long
401 __xchg_u64(volatile long *m, unsigned long val)
402 {
403         unsigned long dummy;
404
405         __asm__ __volatile__(
406         "1:     ldq_l %0,%4\n"
407         "       bis $31,%3,%1\n"
408         "       stq_c %1,%2\n"
409         "       beq %1,2f\n"
410 #ifdef CONFIG_SMP
411         "       mb\n"
412 #endif
413         ".subsection 2\n"
414         "2:     br 1b\n"
415         ".previous"
416         : "=&r" (val), "=&r" (dummy), "=m" (*m)
417         : "rI" (val), "m" (*m) : "memory");
418
419         return val;
420 }
421
422 /* This function doesn't exist, so you'll get a linker error
423    if something tries to do an invalid xchg().  */
424 extern void __xchg_called_with_bad_pointer(void);
425
426 #define __xchg(ptr, x, size) \
427 ({ \
428         unsigned long __xchg__res; \
429         volatile void *__xchg__ptr = (ptr); \
430         switch (size) { \
431                 case 1: __xchg__res = __xchg_u8(__xchg__ptr, x); break; \
432                 case 2: __xchg__res = __xchg_u16(__xchg__ptr, x); break; \
433                 case 4: __xchg__res = __xchg_u32(__xchg__ptr, x); break; \
434                 case 8: __xchg__res = __xchg_u64(__xchg__ptr, x); break; \
435                 default: __xchg_called_with_bad_pointer(); __xchg__res = x; \
436         } \
437         __xchg__res; \
438 })
439
440 #define xchg(ptr,x)                                                          \
441   ({                                                                         \
442      __typeof__(*(ptr)) _x_ = (x);                                           \
443      (__typeof__(*(ptr))) __xchg((ptr), (unsigned long)_x_, sizeof(*(ptr))); \
444   })
445
446 static inline unsigned long
447 __xchg_u8_local(volatile char *m, unsigned long val)
448 {
449         unsigned long ret, tmp, addr64;
450
451         __asm__ __volatile__(
452         "       andnot  %4,7,%3\n"
453         "       insbl   %1,%4,%1\n"
454         "1:     ldq_l   %2,0(%3)\n"
455         "       extbl   %2,%4,%0\n"
456         "       mskbl   %2,%4,%2\n"
457         "       or      %1,%2,%2\n"
458         "       stq_c   %2,0(%3)\n"
459         "       beq     %2,2f\n"
460         ".subsection 2\n"
461         "2:     br      1b\n"
462         ".previous"
463         : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64)
464         : "r" ((long)m), "1" (val) : "memory");
465
466         return ret;
467 }
468
469 static inline unsigned long
470 __xchg_u16_local(volatile short *m, unsigned long val)
471 {
472         unsigned long ret, tmp, addr64;
473
474         __asm__ __volatile__(
475         "       andnot  %4,7,%3\n"
476         "       inswl   %1,%4,%1\n"
477         "1:     ldq_l   %2,0(%3)\n"
478         "       extwl   %2,%4,%0\n"
479         "       mskwl   %2,%4,%2\n"
480         "       or      %1,%2,%2\n"
481         "       stq_c   %2,0(%3)\n"
482         "       beq     %2,2f\n"
483         ".subsection 2\n"
484         "2:     br      1b\n"
485         ".previous"
486         : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64)
487         : "r" ((long)m), "1" (val) : "memory");
488
489         return ret;
490 }
491
492 static inline unsigned long
493 __xchg_u32_local(volatile int *m, unsigned long val)
494 {
495         unsigned long dummy;
496
497         __asm__ __volatile__(
498         "1:     ldl_l %0,%4\n"
499         "       bis $31,%3,%1\n"
500         "       stl_c %1,%2\n"
501         "       beq %1,2f\n"
502         ".subsection 2\n"
503         "2:     br 1b\n"
504         ".previous"
505         : "=&r" (val), "=&r" (dummy), "=m" (*m)
506         : "rI" (val), "m" (*m) : "memory");
507
508         return val;
509 }
510
511 static inline unsigned long
512 __xchg_u64_local(volatile long *m, unsigned long val)
513 {
514         unsigned long dummy;
515
516         __asm__ __volatile__(
517         "1:     ldq_l %0,%4\n"
518         "       bis $31,%3,%1\n"
519         "       stq_c %1,%2\n"
520         "       beq %1,2f\n"
521         ".subsection 2\n"
522         "2:     br 1b\n"
523         ".previous"
524         : "=&r" (val), "=&r" (dummy), "=m" (*m)
525         : "rI" (val), "m" (*m) : "memory");
526
527         return val;
528 }
529
530 #define __xchg_local(ptr, x, size) \
531 ({ \
532         unsigned long __xchg__res; \
533         volatile void *__xchg__ptr = (ptr); \
534         switch (size) { \
535                 case 1: __xchg__res = __xchg_u8_local(__xchg__ptr, x); break; \
536                 case 2: __xchg__res = __xchg_u16_local(__xchg__ptr, x); break; \
537                 case 4: __xchg__res = __xchg_u32_local(__xchg__ptr, x); break; \
538                 case 8: __xchg__res = __xchg_u64_local(__xchg__ptr, x); break; \
539                 default: __xchg_called_with_bad_pointer(); __xchg__res = x; \
540         } \
541         __xchg__res; \
542 })
543
544 #define xchg_local(ptr,x)                                                    \
545   ({                                                                         \
546      __typeof__(*(ptr)) _x_ = (x);                                           \
547      (__typeof__(*(ptr))) __xchg_local((ptr), (unsigned long)_x_,            \
548                 sizeof(*(ptr))); \
549   })
550
551 /* 
552  * Atomic compare and exchange.  Compare OLD with MEM, if identical,
553  * store NEW in MEM.  Return the initial value in MEM.  Success is
554  * indicated by comparing RETURN with OLD.
555  *
556  * The memory barrier should be placed in SMP only when we actually
557  * make the change. If we don't change anything (so if the returned
558  * prev is equal to old) then we aren't acquiring anything new and
559  * we don't need any memory barrier as far I can tell.
560  */
561
562 #define __HAVE_ARCH_CMPXCHG 1
563
564 static inline unsigned long
565 __cmpxchg_u8(volatile char *m, long old, long new)
566 {
567         unsigned long prev, tmp, cmp, addr64;
568
569         __asm__ __volatile__(
570         "       andnot  %5,7,%4\n"
571         "       insbl   %1,%5,%1\n"
572         "1:     ldq_l   %2,0(%4)\n"
573         "       extbl   %2,%5,%0\n"
574         "       cmpeq   %0,%6,%3\n"
575         "       beq     %3,2f\n"
576         "       mskbl   %2,%5,%2\n"
577         "       or      %1,%2,%2\n"
578         "       stq_c   %2,0(%4)\n"
579         "       beq     %2,3f\n"
580 #ifdef CONFIG_SMP
581         "       mb\n"
582 #endif
583         "2:\n"
584         ".subsection 2\n"
585         "3:     br      1b\n"
586         ".previous"
587         : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64)
588         : "r" ((long)m), "Ir" (old), "1" (new) : "memory");
589
590         return prev;
591 }
592
593 static inline unsigned long
594 __cmpxchg_u16(volatile short *m, long old, long new)
595 {
596         unsigned long prev, tmp, cmp, addr64;
597
598         __asm__ __volatile__(
599         "       andnot  %5,7,%4\n"
600         "       inswl   %1,%5,%1\n"
601         "1:     ldq_l   %2,0(%4)\n"
602         "       extwl   %2,%5,%0\n"
603         "       cmpeq   %0,%6,%3\n"
604         "       beq     %3,2f\n"
605         "       mskwl   %2,%5,%2\n"
606         "       or      %1,%2,%2\n"
607         "       stq_c   %2,0(%4)\n"
608         "       beq     %2,3f\n"
609 #ifdef CONFIG_SMP
610         "       mb\n"
611 #endif
612         "2:\n"
613         ".subsection 2\n"
614         "3:     br      1b\n"
615         ".previous"
616         : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64)
617         : "r" ((long)m), "Ir" (old), "1" (new) : "memory");
618
619         return prev;
620 }
621
622 static inline unsigned long
623 __cmpxchg_u32(volatile int *m, int old, int new)
624 {
625         unsigned long prev, cmp;
626
627         __asm__ __volatile__(
628         "1:     ldl_l %0,%5\n"
629         "       cmpeq %0,%3,%1\n"
630         "       beq %1,2f\n"
631         "       mov %4,%1\n"
632         "       stl_c %1,%2\n"
633         "       beq %1,3f\n"
634 #ifdef CONFIG_SMP
635         "       mb\n"
636 #endif
637         "2:\n"
638         ".subsection 2\n"
639         "3:     br 1b\n"
640         ".previous"
641         : "=&r"(prev), "=&r"(cmp), "=m"(*m)
642         : "r"((long) old), "r"(new), "m"(*m) : "memory");
643
644         return prev;
645 }
646
647 static inline unsigned long
648 __cmpxchg_u64(volatile long *m, unsigned long old, unsigned long new)
649 {
650         unsigned long prev, cmp;
651
652         __asm__ __volatile__(
653         "1:     ldq_l %0,%5\n"
654         "       cmpeq %0,%3,%1\n"
655         "       beq %1,2f\n"
656         "       mov %4,%1\n"
657         "       stq_c %1,%2\n"
658         "       beq %1,3f\n"
659 #ifdef CONFIG_SMP
660         "       mb\n"
661 #endif
662         "2:\n"
663         ".subsection 2\n"
664         "3:     br 1b\n"
665         ".previous"
666         : "=&r"(prev), "=&r"(cmp), "=m"(*m)
667         : "r"((long) old), "r"(new), "m"(*m) : "memory");
668
669         return prev;
670 }
671
672 /* This function doesn't exist, so you'll get a linker error
673    if something tries to do an invalid cmpxchg().  */
674 extern void __cmpxchg_called_with_bad_pointer(void);
675
676 static __always_inline unsigned long
677 __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, int size)
678 {
679         switch (size) {
680                 case 1:
681                         return __cmpxchg_u8(ptr, old, new);
682                 case 2:
683                         return __cmpxchg_u16(ptr, old, new);
684                 case 4:
685                         return __cmpxchg_u32(ptr, old, new);
686                 case 8:
687                         return __cmpxchg_u64(ptr, old, new);
688         }
689         __cmpxchg_called_with_bad_pointer();
690         return old;
691 }
692
693 #define cmpxchg(ptr,o,n)                                                 \
694   ({                                                                     \
695      __typeof__(*(ptr)) _o_ = (o);                                       \
696      __typeof__(*(ptr)) _n_ = (n);                                       \
697      (__typeof__(*(ptr))) __cmpxchg((ptr), (unsigned long)_o_,           \
698                                     (unsigned long)_n_, sizeof(*(ptr))); \
699   })
700
701 static inline unsigned long
702 __cmpxchg_u8_local(volatile char *m, long old, long new)
703 {
704         unsigned long prev, tmp, cmp, addr64;
705
706         __asm__ __volatile__(
707         "       andnot  %5,7,%4\n"
708         "       insbl   %1,%5,%1\n"
709         "1:     ldq_l   %2,0(%4)\n"
710         "       extbl   %2,%5,%0\n"
711         "       cmpeq   %0,%6,%3\n"
712         "       beq     %3,2f\n"
713         "       mskbl   %2,%5,%2\n"
714         "       or      %1,%2,%2\n"
715         "       stq_c   %2,0(%4)\n"
716         "       beq     %2,3f\n"
717         "2:\n"
718         ".subsection 2\n"
719         "3:     br      1b\n"
720         ".previous"
721         : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64)
722         : "r" ((long)m), "Ir" (old), "1" (new) : "memory");
723
724         return prev;
725 }
726
727 static inline unsigned long
728 __cmpxchg_u16_local(volatile short *m, long old, long new)
729 {
730         unsigned long prev, tmp, cmp, addr64;
731
732         __asm__ __volatile__(
733         "       andnot  %5,7,%4\n"
734         "       inswl   %1,%5,%1\n"
735         "1:     ldq_l   %2,0(%4)\n"
736         "       extwl   %2,%5,%0\n"
737         "       cmpeq   %0,%6,%3\n"
738         "       beq     %3,2f\n"
739         "       mskwl   %2,%5,%2\n"
740         "       or      %1,%2,%2\n"
741         "       stq_c   %2,0(%4)\n"
742         "       beq     %2,3f\n"
743         "2:\n"
744         ".subsection 2\n"
745         "3:     br      1b\n"
746         ".previous"
747         : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64)
748         : "r" ((long)m), "Ir" (old), "1" (new) : "memory");
749
750         return prev;
751 }
752
753 static inline unsigned long
754 __cmpxchg_u32_local(volatile int *m, int old, int new)
755 {
756         unsigned long prev, cmp;
757
758         __asm__ __volatile__(
759         "1:     ldl_l %0,%5\n"
760         "       cmpeq %0,%3,%1\n"
761         "       beq %1,2f\n"
762         "       mov %4,%1\n"
763         "       stl_c %1,%2\n"
764         "       beq %1,3f\n"
765         "2:\n"
766         ".subsection 2\n"
767         "3:     br 1b\n"
768         ".previous"
769         : "=&r"(prev), "=&r"(cmp), "=m"(*m)
770         : "r"((long) old), "r"(new), "m"(*m) : "memory");
771
772         return prev;
773 }
774
775 static inline unsigned long
776 __cmpxchg_u64_local(volatile long *m, unsigned long old, unsigned long new)
777 {
778         unsigned long prev, cmp;
779
780         __asm__ __volatile__(
781         "1:     ldq_l %0,%5\n"
782         "       cmpeq %0,%3,%1\n"
783         "       beq %1,2f\n"
784         "       mov %4,%1\n"
785         "       stq_c %1,%2\n"
786         "       beq %1,3f\n"
787         "2:\n"
788         ".subsection 2\n"
789         "3:     br 1b\n"
790         ".previous"
791         : "=&r"(prev), "=&r"(cmp), "=m"(*m)
792         : "r"((long) old), "r"(new), "m"(*m) : "memory");
793
794         return prev;
795 }
796
797 static __always_inline unsigned long
798 __cmpxchg_local(volatile void *ptr, unsigned long old, unsigned long new,
799                 int size)
800 {
801         switch (size) {
802                 case 1:
803                         return __cmpxchg_u8_local(ptr, old, new);
804                 case 2:
805                         return __cmpxchg_u16_local(ptr, old, new);
806                 case 4:
807                         return __cmpxchg_u32_local(ptr, old, new);
808                 case 8:
809                         return __cmpxchg_u64_local(ptr, old, new);
810         }
811         __cmpxchg_called_with_bad_pointer();
812         return old;
813 }
814
815 #define cmpxchg_local(ptr,o,n)                                           \
816   ({                                                                     \
817      __typeof__(*(ptr)) _o_ = (o);                                       \
818      __typeof__(*(ptr)) _n_ = (n);                                       \
819      (__typeof__(*(ptr))) __cmpxchg_local((ptr), (unsigned long)_o_,     \
820                                     (unsigned long)_n_, sizeof(*(ptr))); \
821   })
822
823 #endif /* __ASSEMBLY__ */
824
825 #define arch_align_stack(x) (x)
826
827 #endif