Merge git://git.kernel.org/pub/scm/linux/kernel/git/davem/sparc-2.6
[linux-2.6.git] / arch / x86 / include / asm / system.h
1 #ifndef _ASM_X86_SYSTEM_H
2 #define _ASM_X86_SYSTEM_H
3
4 #include <asm/asm.h>
5 #include <asm/segment.h>
6 #include <asm/cpufeature.h>
7 #include <asm/cmpxchg.h>
8 #include <asm/nops.h>
9
10 #include <linux/kernel.h>
11 #include <linux/irqflags.h>
12
13 /* entries in ARCH_DLINFO: */
14 #if defined(CONFIG_IA32_EMULATION) || !defined(CONFIG_X86_64)
15 # define AT_VECTOR_SIZE_ARCH 2
16 #else /* else it's non-compat x86-64 */
17 # define AT_VECTOR_SIZE_ARCH 1
18 #endif
19
20 struct task_struct; /* one of the stranger aspects of C forward declarations */
21 struct task_struct *__switch_to(struct task_struct *prev,
22                                 struct task_struct *next);
23 struct tss_struct;
24 void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p,
25                       struct tss_struct *tss);
26 extern void show_regs_common(void);
27
28 #ifdef CONFIG_X86_32
29
30 #ifdef CONFIG_CC_STACKPROTECTOR
31 #define __switch_canary                                                 \
32         "movl %P[task_canary](%[next]), %%ebx\n\t"                      \
33         "movl %%ebx, "__percpu_arg([stack_canary])"\n\t"
34 #define __switch_canary_oparam                                          \
35         , [stack_canary] "=m" (stack_canary.canary)
36 #define __switch_canary_iparam                                          \
37         , [task_canary] "i" (offsetof(struct task_struct, stack_canary))
38 #else   /* CC_STACKPROTECTOR */
39 #define __switch_canary
40 #define __switch_canary_oparam
41 #define __switch_canary_iparam
42 #endif  /* CC_STACKPROTECTOR */
43
44 /*
45  * Saving eflags is important. It switches not only IOPL between tasks,
46  * it also protects other tasks from NT leaking through sysenter etc.
47  */
48 #define switch_to(prev, next, last)                                     \
49 do {                                                                    \
50         /*                                                              \
51          * Context-switching clobbers all registers, so we clobber      \
52          * them explicitly, via unused output variables.                \
53          * (EAX and EBP is not listed because EBP is saved/restored     \
54          * explicitly for wchan access and EAX is the return value of   \
55          * __switch_to())                                               \
56          */                                                             \
57         unsigned long ebx, ecx, edx, esi, edi;                          \
58                                                                         \
59         asm volatile("pushfl\n\t"               /* save    flags */     \
60                      "pushl %%ebp\n\t"          /* save    EBP   */     \
61                      "movl %%esp,%[prev_sp]\n\t"        /* save    ESP   */ \
62                      "movl %[next_sp],%%esp\n\t"        /* restore ESP   */ \
63                      "movl $1f,%[prev_ip]\n\t"  /* save    EIP   */     \
64                      "pushl %[next_ip]\n\t"     /* restore EIP   */     \
65                      __switch_canary                                    \
66                      "jmp __switch_to\n"        /* regparm call  */     \
67                      "1:\t"                                             \
68                      "popl %%ebp\n\t"           /* restore EBP   */     \
69                      "popfl\n"                  /* restore flags */     \
70                                                                         \
71                      /* output parameters */                            \
72                      : [prev_sp] "=m" (prev->thread.sp),                \
73                        [prev_ip] "=m" (prev->thread.ip),                \
74                        "=a" (last),                                     \
75                                                                         \
76                        /* clobbered output registers: */                \
77                        "=b" (ebx), "=c" (ecx), "=d" (edx),              \
78                        "=S" (esi), "=D" (edi)                           \
79                                                                         \
80                        __switch_canary_oparam                           \
81                                                                         \
82                        /* input parameters: */                          \
83                      : [next_sp]  "m" (next->thread.sp),                \
84                        [next_ip]  "m" (next->thread.ip),                \
85                                                                         \
86                        /* regparm parameters for __switch_to(): */      \
87                        [prev]     "a" (prev),                           \
88                        [next]     "d" (next)                            \
89                                                                         \
90                        __switch_canary_iparam                           \
91                                                                         \
92                      : /* reloaded segment registers */                 \
93                         "memory");                                      \
94 } while (0)
95
96 /*
97  * disable hlt during certain critical i/o operations
98  */
99 #define HAVE_DISABLE_HLT
100 #else
101 #define __SAVE(reg, offset) "movq %%" #reg ",(14-" #offset ")*8(%%rsp)\n\t"
102 #define __RESTORE(reg, offset) "movq (14-" #offset ")*8(%%rsp),%%" #reg "\n\t"
103
104 /* frame pointer must be last for get_wchan */
105 #define SAVE_CONTEXT    "pushf ; pushq %%rbp ; movq %%rsi,%%rbp\n\t"
106 #define RESTORE_CONTEXT "movq %%rbp,%%rsi ; popq %%rbp ; popf\t"
107
108 #define __EXTRA_CLOBBER  \
109         , "rcx", "rbx", "rdx", "r8", "r9", "r10", "r11", \
110           "r12", "r13", "r14", "r15"
111
112 #ifdef CONFIG_CC_STACKPROTECTOR
113 #define __switch_canary                                                   \
114         "movq %P[task_canary](%%rsi),%%r8\n\t"                            \
115         "movq %%r8,"__percpu_arg([gs_canary])"\n\t"
116 #define __switch_canary_oparam                                            \
117         , [gs_canary] "=m" (irq_stack_union.stack_canary)
118 #define __switch_canary_iparam                                            \
119         , [task_canary] "i" (offsetof(struct task_struct, stack_canary))
120 #else   /* CC_STACKPROTECTOR */
121 #define __switch_canary
122 #define __switch_canary_oparam
123 #define __switch_canary_iparam
124 #endif  /* CC_STACKPROTECTOR */
125
126 /* Save restore flags to clear handle leaking NT */
127 #define switch_to(prev, next, last) \
128         asm volatile(SAVE_CONTEXT                                         \
129              "movq %%rsp,%P[threadrsp](%[prev])\n\t" /* save RSP */       \
130              "movq %P[threadrsp](%[next]),%%rsp\n\t" /* restore RSP */    \
131              "call __switch_to\n\t"                                       \
132              "movq "__percpu_arg([current_task])",%%rsi\n\t"              \
133              __switch_canary                                              \
134              "movq %P[thread_info](%%rsi),%%r8\n\t"                       \
135              "movq %%rax,%%rdi\n\t"                                       \
136              "testl  %[_tif_fork],%P[ti_flags](%%r8)\n\t"                 \
137              "jnz   ret_from_fork\n\t"                                    \
138              RESTORE_CONTEXT                                              \
139              : "=a" (last)                                                \
140                __switch_canary_oparam                                     \
141              : [next] "S" (next), [prev] "D" (prev),                      \
142                [threadrsp] "i" (offsetof(struct task_struct, thread.sp)), \
143                [ti_flags] "i" (offsetof(struct thread_info, flags)),      \
144                [_tif_fork] "i" (_TIF_FORK),                               \
145                [thread_info] "i" (offsetof(struct task_struct, stack)),   \
146                [current_task] "m" (current_task)                          \
147                __switch_canary_iparam                                     \
148              : "memory", "cc" __EXTRA_CLOBBER)
149 #endif
150
151 #ifdef __KERNEL__
152
153 extern void native_load_gs_index(unsigned);
154
155 /*
156  * Load a segment. Fall back on loading the zero
157  * segment if something goes wrong..
158  */
159 #define loadsegment(seg, value)                                         \
160 do {                                                                    \
161         unsigned short __val = (value);                                 \
162                                                                         \
163         asm volatile("                                          \n"     \
164                      "1:        movl %k0,%%" #seg "             \n"     \
165                                                                         \
166                      ".section .fixup,\"ax\"                    \n"     \
167                      "2:        xorl %k0,%k0                    \n"     \
168                      "          jmp 1b                          \n"     \
169                      ".previous                                 \n"     \
170                                                                         \
171                      _ASM_EXTABLE(1b, 2b)                               \
172                                                                         \
173                      : "+r" (__val) : : "memory");                      \
174 } while (0)
175
176 /*
177  * Save a segment register away
178  */
179 #define savesegment(seg, value)                         \
180         asm("mov %%" #seg ",%0":"=r" (value) : : "memory")
181
182 /*
183  * x86_32 user gs accessors.
184  */
185 #ifdef CONFIG_X86_32
186 #ifdef CONFIG_X86_32_LAZY_GS
187 #define get_user_gs(regs)       (u16)({unsigned long v; savesegment(gs, v); v;})
188 #define set_user_gs(regs, v)    loadsegment(gs, (unsigned long)(v))
189 #define task_user_gs(tsk)       ((tsk)->thread.gs)
190 #define lazy_save_gs(v)         savesegment(gs, (v))
191 #define lazy_load_gs(v)         loadsegment(gs, (v))
192 #else   /* X86_32_LAZY_GS */
193 #define get_user_gs(regs)       (u16)((regs)->gs)
194 #define set_user_gs(regs, v)    do { (regs)->gs = (v); } while (0)
195 #define task_user_gs(tsk)       (task_pt_regs(tsk)->gs)
196 #define lazy_save_gs(v)         do { } while (0)
197 #define lazy_load_gs(v)         do { } while (0)
198 #endif  /* X86_32_LAZY_GS */
199 #endif  /* X86_32 */
200
201 static inline unsigned long get_limit(unsigned long segment)
202 {
203         unsigned long __limit;
204         asm("lsll %1,%0" : "=r" (__limit) : "r" (segment));
205         return __limit + 1;
206 }
207
208 static inline void native_clts(void)
209 {
210         asm volatile("clts");
211 }
212
213 /*
214  * Volatile isn't enough to prevent the compiler from reordering the
215  * read/write functions for the control registers and messing everything up.
216  * A memory clobber would solve the problem, but would prevent reordering of
217  * all loads stores around it, which can hurt performance. Solution is to
218  * use a variable and mimic reads and writes to it to enforce serialization
219  */
220 static unsigned long __force_order;
221
222 static inline unsigned long native_read_cr0(void)
223 {
224         unsigned long val;
225         asm volatile("mov %%cr0,%0\n\t" : "=r" (val), "=m" (__force_order));
226         return val;
227 }
228
229 static inline void native_write_cr0(unsigned long val)
230 {
231         asm volatile("mov %0,%%cr0": : "r" (val), "m" (__force_order));
232 }
233
234 static inline unsigned long native_read_cr2(void)
235 {
236         unsigned long val;
237         asm volatile("mov %%cr2,%0\n\t" : "=r" (val), "=m" (__force_order));
238         return val;
239 }
240
241 static inline void native_write_cr2(unsigned long val)
242 {
243         asm volatile("mov %0,%%cr2": : "r" (val), "m" (__force_order));
244 }
245
246 static inline unsigned long native_read_cr3(void)
247 {
248         unsigned long val;
249         asm volatile("mov %%cr3,%0\n\t" : "=r" (val), "=m" (__force_order));
250         return val;
251 }
252
253 static inline void native_write_cr3(unsigned long val)
254 {
255         asm volatile("mov %0,%%cr3": : "r" (val), "m" (__force_order));
256 }
257
258 static inline unsigned long native_read_cr4(void)
259 {
260         unsigned long val;
261         asm volatile("mov %%cr4,%0\n\t" : "=r" (val), "=m" (__force_order));
262         return val;
263 }
264
265 static inline unsigned long native_read_cr4_safe(void)
266 {
267         unsigned long val;
268         /* This could fault if %cr4 does not exist. In x86_64, a cr4 always
269          * exists, so it will never fail. */
270 #ifdef CONFIG_X86_32
271         asm volatile("1: mov %%cr4, %0\n"
272                      "2:\n"
273                      _ASM_EXTABLE(1b, 2b)
274                      : "=r" (val), "=m" (__force_order) : "0" (0));
275 #else
276         val = native_read_cr4();
277 #endif
278         return val;
279 }
280
281 static inline void native_write_cr4(unsigned long val)
282 {
283         asm volatile("mov %0,%%cr4": : "r" (val), "m" (__force_order));
284 }
285
286 #ifdef CONFIG_X86_64
287 static inline unsigned long native_read_cr8(void)
288 {
289         unsigned long cr8;
290         asm volatile("movq %%cr8,%0" : "=r" (cr8));
291         return cr8;
292 }
293
294 static inline void native_write_cr8(unsigned long val)
295 {
296         asm volatile("movq %0,%%cr8" :: "r" (val) : "memory");
297 }
298 #endif
299
300 static inline void native_wbinvd(void)
301 {
302         asm volatile("wbinvd": : :"memory");
303 }
304
305 #ifdef CONFIG_PARAVIRT
306 #include <asm/paravirt.h>
307 #else
308 #define read_cr0()      (native_read_cr0())
309 #define write_cr0(x)    (native_write_cr0(x))
310 #define read_cr2()      (native_read_cr2())
311 #define write_cr2(x)    (native_write_cr2(x))
312 #define read_cr3()      (native_read_cr3())
313 #define write_cr3(x)    (native_write_cr3(x))
314 #define read_cr4()      (native_read_cr4())
315 #define read_cr4_safe() (native_read_cr4_safe())
316 #define write_cr4(x)    (native_write_cr4(x))
317 #define wbinvd()        (native_wbinvd())
318 #ifdef CONFIG_X86_64
319 #define read_cr8()      (native_read_cr8())
320 #define write_cr8(x)    (native_write_cr8(x))
321 #define load_gs_index   native_load_gs_index
322 #endif
323
324 /* Clear the 'TS' bit */
325 #define clts()          (native_clts())
326
327 #endif/* CONFIG_PARAVIRT */
328
329 #define stts() write_cr0(read_cr0() | X86_CR0_TS)
330
331 #endif /* __KERNEL__ */
332
333 static inline void clflush(volatile void *__p)
334 {
335         asm volatile("clflush %0" : "+m" (*(volatile char __force *)__p));
336 }
337
338 #define nop() asm volatile ("nop")
339
340 void disable_hlt(void);
341 void enable_hlt(void);
342
343 void cpu_idle_wait(void);
344
345 extern unsigned long arch_align_stack(unsigned long sp);
346 extern void free_init_pages(char *what, unsigned long begin, unsigned long end);
347
348 void default_idle(void);
349
350 void stop_this_cpu(void *dummy);
351
352 /*
353  * Force strict CPU ordering.
354  * And yes, this is required on UP too when we're talking
355  * to devices.
356  */
357 #ifdef CONFIG_X86_32
358 /*
359  * Some non-Intel clones support out of order store. wmb() ceases to be a
360  * nop for these.
361  */
362 #define mb() alternative("lock; addl $0,0(%%esp)", "mfence", X86_FEATURE_XMM2)
363 #define rmb() alternative("lock; addl $0,0(%%esp)", "lfence", X86_FEATURE_XMM2)
364 #define wmb() alternative("lock; addl $0,0(%%esp)", "sfence", X86_FEATURE_XMM)
365 #else
366 #define mb()    asm volatile("mfence":::"memory")
367 #define rmb()   asm volatile("lfence":::"memory")
368 #define wmb()   asm volatile("sfence" ::: "memory")
369 #endif
370
371 /**
372  * read_barrier_depends - Flush all pending reads that subsequents reads
373  * depend on.
374  *
375  * No data-dependent reads from memory-like regions are ever reordered
376  * over this barrier.  All reads preceding this primitive are guaranteed
377  * to access memory (but not necessarily other CPUs' caches) before any
378  * reads following this primitive that depend on the data return by
379  * any of the preceding reads.  This primitive is much lighter weight than
380  * rmb() on most CPUs, and is never heavier weight than is
381  * rmb().
382  *
383  * These ordering constraints are respected by both the local CPU
384  * and the compiler.
385  *
386  * Ordering is not guaranteed by anything other than these primitives,
387  * not even by data dependencies.  See the documentation for
388  * memory_barrier() for examples and URLs to more information.
389  *
390  * For example, the following code would force ordering (the initial
391  * value of "a" is zero, "b" is one, and "p" is "&a"):
392  *
393  * <programlisting>
394  *      CPU 0                           CPU 1
395  *
396  *      b = 2;
397  *      memory_barrier();
398  *      p = &b;                         q = p;
399  *                                      read_barrier_depends();
400  *                                      d = *q;
401  * </programlisting>
402  *
403  * because the read of "*q" depends on the read of "p" and these
404  * two reads are separated by a read_barrier_depends().  However,
405  * the following code, with the same initial values for "a" and "b":
406  *
407  * <programlisting>
408  *      CPU 0                           CPU 1
409  *
410  *      a = 2;
411  *      memory_barrier();
412  *      b = 3;                          y = b;
413  *                                      read_barrier_depends();
414  *                                      x = a;
415  * </programlisting>
416  *
417  * does not enforce ordering, since there is no data dependency between
418  * the read of "a" and the read of "b".  Therefore, on some CPUs, such
419  * as Alpha, "y" could be set to 3 and "x" to 0.  Use rmb()
420  * in cases like this where there are no data dependencies.
421  **/
422
423 #define read_barrier_depends()  do { } while (0)
424
425 #ifdef CONFIG_SMP
426 #define smp_mb()        mb()
427 #ifdef CONFIG_X86_PPRO_FENCE
428 # define smp_rmb()      rmb()
429 #else
430 # define smp_rmb()      barrier()
431 #endif
432 #ifdef CONFIG_X86_OOSTORE
433 # define smp_wmb()      wmb()
434 #else
435 # define smp_wmb()      barrier()
436 #endif
437 #define smp_read_barrier_depends()      read_barrier_depends()
438 #define set_mb(var, value) do { (void)xchg(&var, value); } while (0)
439 #else
440 #define smp_mb()        barrier()
441 #define smp_rmb()       barrier()
442 #define smp_wmb()       barrier()
443 #define smp_read_barrier_depends()      do { } while (0)
444 #define set_mb(var, value) do { var = value; barrier(); } while (0)
445 #endif
446
447 /*
448  * Stop RDTSC speculation. This is needed when you need to use RDTSC
449  * (or get_cycles or vread that possibly accesses the TSC) in a defined
450  * code region.
451  *
452  * (Could use an alternative three way for this if there was one.)
453  */
454 static inline void rdtsc_barrier(void)
455 {
456         alternative(ASM_NOP3, "mfence", X86_FEATURE_MFENCE_RDTSC);
457         alternative(ASM_NOP3, "lfence", X86_FEATURE_LFENCE_RDTSC);
458 }
459
460 #endif /* _ASM_X86_SYSTEM_H */