Merge branch 'linus' into tracing/hw-breakpoints
[linux-2.6.git] / arch / x86 / kernel / i387.c
1 /*
2  *  Copyright (C) 1994 Linus Torvalds
3  *
4  *  Pentium III FXSR, SSE support
5  *  General FPU state handling cleanups
6  *      Gareth Hughes <gareth@valinux.com>, May 2000
7  */
8 #include <linux/module.h>
9 #include <linux/regset.h>
10 #include <linux/sched.h>
11
12 #include <asm/sigcontext.h>
13 #include <asm/processor.h>
14 #include <asm/math_emu.h>
15 #include <asm/uaccess.h>
16 #include <asm/ptrace.h>
17 #include <asm/i387.h>
18 #include <asm/user.h>
19
20 #ifdef CONFIG_X86_64
21 # include <asm/sigcontext32.h>
22 # include <asm/user32.h>
23 #else
24 # define save_i387_xstate_ia32          save_i387_xstate
25 # define restore_i387_xstate_ia32       restore_i387_xstate
26 # define _fpstate_ia32          _fpstate
27 # define _xstate_ia32           _xstate
28 # define sig_xstate_ia32_size   sig_xstate_size
29 # define fx_sw_reserved_ia32    fx_sw_reserved
30 # define user_i387_ia32_struct  user_i387_struct
31 # define user32_fxsr_struct     user_fxsr_struct
32 #endif
33
34 #ifdef CONFIG_MATH_EMULATION
35 # define HAVE_HWFP              (boot_cpu_data.hard_math)
36 #else
37 # define HAVE_HWFP              1
38 #endif
39
40 static unsigned int             mxcsr_feature_mask __read_mostly = 0xffffffffu;
41 unsigned int xstate_size;
42 unsigned int sig_xstate_ia32_size = sizeof(struct _fpstate_ia32);
43 static struct i387_fxsave_struct fx_scratch __cpuinitdata;
44
45 void __cpuinit mxcsr_feature_mask_init(void)
46 {
47         unsigned long mask = 0;
48
49         clts();
50         if (cpu_has_fxsr) {
51                 memset(&fx_scratch, 0, sizeof(struct i387_fxsave_struct));
52                 asm volatile("fxsave %0" : : "m" (fx_scratch));
53                 mask = fx_scratch.mxcsr_mask;
54                 if (mask == 0)
55                         mask = 0x0000ffbf;
56         }
57         mxcsr_feature_mask &= mask;
58         stts();
59 }
60
61 void __cpuinit init_thread_xstate(void)
62 {
63         if (!HAVE_HWFP) {
64                 xstate_size = sizeof(struct i387_soft_struct);
65                 return;
66         }
67
68         if (cpu_has_xsave) {
69                 xsave_cntxt_init();
70                 return;
71         }
72
73         if (cpu_has_fxsr)
74                 xstate_size = sizeof(struct i387_fxsave_struct);
75 #ifdef CONFIG_X86_32
76         else
77                 xstate_size = sizeof(struct i387_fsave_struct);
78 #endif
79 }
80
81 #ifdef CONFIG_X86_64
82 /*
83  * Called at bootup to set up the initial FPU state that is later cloned
84  * into all processes.
85  */
86 void __cpuinit fpu_init(void)
87 {
88         unsigned long oldcr0 = read_cr0();
89
90         set_in_cr4(X86_CR4_OSFXSR);
91         set_in_cr4(X86_CR4_OSXMMEXCPT);
92
93         write_cr0(oldcr0 & ~(X86_CR0_TS|X86_CR0_EM)); /* clear TS and EM */
94
95         /*
96          * Boot processor to setup the FP and extended state context info.
97          */
98         if (!smp_processor_id())
99                 init_thread_xstate();
100         xsave_init();
101
102         mxcsr_feature_mask_init();
103         /* clean state in init */
104         if (cpu_has_xsave)
105                 current_thread_info()->status = TS_XSAVE;
106         else
107                 current_thread_info()->status = 0;
108         clear_used_math();
109 }
110 #endif  /* CONFIG_X86_64 */
111
112 /*
113  * The _current_ task is using the FPU for the first time
114  * so initialize it and set the mxcsr to its default
115  * value at reset if we support XMM instructions and then
116  * remeber the current task has used the FPU.
117  */
118 int init_fpu(struct task_struct *tsk)
119 {
120         if (tsk_used_math(tsk)) {
121                 if (HAVE_HWFP && tsk == current)
122                         unlazy_fpu(tsk);
123                 return 0;
124         }
125
126         /*
127          * Memory allocation at the first usage of the FPU and other state.
128          */
129         if (!tsk->thread.xstate) {
130                 tsk->thread.xstate = kmem_cache_alloc(task_xstate_cachep,
131                                                       GFP_KERNEL);
132                 if (!tsk->thread.xstate)
133                         return -ENOMEM;
134         }
135
136 #ifdef CONFIG_X86_32
137         if (!HAVE_HWFP) {
138                 memset(tsk->thread.xstate, 0, xstate_size);
139                 finit_task(tsk);
140                 set_stopped_child_used_math(tsk);
141                 return 0;
142         }
143 #endif
144
145         if (cpu_has_fxsr) {
146                 struct i387_fxsave_struct *fx = &tsk->thread.xstate->fxsave;
147
148                 memset(fx, 0, xstate_size);
149                 fx->cwd = 0x37f;
150                 if (cpu_has_xmm)
151                         fx->mxcsr = MXCSR_DEFAULT;
152         } else {
153                 struct i387_fsave_struct *fp = &tsk->thread.xstate->fsave;
154                 memset(fp, 0, xstate_size);
155                 fp->cwd = 0xffff037fu;
156                 fp->swd = 0xffff0000u;
157                 fp->twd = 0xffffffffu;
158                 fp->fos = 0xffff0000u;
159         }
160         /*
161          * Only the device not available exception or ptrace can call init_fpu.
162          */
163         set_stopped_child_used_math(tsk);
164         return 0;
165 }
166
167 int fpregs_active(struct task_struct *target, const struct user_regset *regset)
168 {
169         return tsk_used_math(target) ? regset->n : 0;
170 }
171
172 int xfpregs_active(struct task_struct *target, const struct user_regset *regset)
173 {
174         return (cpu_has_fxsr && tsk_used_math(target)) ? regset->n : 0;
175 }
176
177 int xfpregs_get(struct task_struct *target, const struct user_regset *regset,
178                 unsigned int pos, unsigned int count,
179                 void *kbuf, void __user *ubuf)
180 {
181         int ret;
182
183         if (!cpu_has_fxsr)
184                 return -ENODEV;
185
186         ret = init_fpu(target);
187         if (ret)
188                 return ret;
189
190         return user_regset_copyout(&pos, &count, &kbuf, &ubuf,
191                                    &target->thread.xstate->fxsave, 0, -1);
192 }
193
194 int xfpregs_set(struct task_struct *target, const struct user_regset *regset,
195                 unsigned int pos, unsigned int count,
196                 const void *kbuf, const void __user *ubuf)
197 {
198         int ret;
199
200         if (!cpu_has_fxsr)
201                 return -ENODEV;
202
203         ret = init_fpu(target);
204         if (ret)
205                 return ret;
206
207         set_stopped_child_used_math(target);
208
209         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
210                                  &target->thread.xstate->fxsave, 0, -1);
211
212         /*
213          * mxcsr reserved bits must be masked to zero for security reasons.
214          */
215         target->thread.xstate->fxsave.mxcsr &= mxcsr_feature_mask;
216
217         /*
218          * update the header bits in the xsave header, indicating the
219          * presence of FP and SSE state.
220          */
221         if (cpu_has_xsave)
222                 target->thread.xstate->xsave.xsave_hdr.xstate_bv |= XSTATE_FPSSE;
223
224         return ret;
225 }
226
227 #if defined CONFIG_X86_32 || defined CONFIG_IA32_EMULATION
228
229 /*
230  * FPU tag word conversions.
231  */
232
233 static inline unsigned short twd_i387_to_fxsr(unsigned short twd)
234 {
235         unsigned int tmp; /* to avoid 16 bit prefixes in the code */
236
237         /* Transform each pair of bits into 01 (valid) or 00 (empty) */
238         tmp = ~twd;
239         tmp = (tmp | (tmp>>1)) & 0x5555; /* 0V0V0V0V0V0V0V0V */
240         /* and move the valid bits to the lower byte. */
241         tmp = (tmp | (tmp >> 1)) & 0x3333; /* 00VV00VV00VV00VV */
242         tmp = (tmp | (tmp >> 2)) & 0x0f0f; /* 0000VVVV0000VVVV */
243         tmp = (tmp | (tmp >> 4)) & 0x00ff; /* 00000000VVVVVVVV */
244
245         return tmp;
246 }
247
248 #define FPREG_ADDR(f, n)        ((void *)&(f)->st_space + (n) * 16);
249 #define FP_EXP_TAG_VALID        0
250 #define FP_EXP_TAG_ZERO         1
251 #define FP_EXP_TAG_SPECIAL      2
252 #define FP_EXP_TAG_EMPTY        3
253
254 static inline u32 twd_fxsr_to_i387(struct i387_fxsave_struct *fxsave)
255 {
256         struct _fpxreg *st;
257         u32 tos = (fxsave->swd >> 11) & 7;
258         u32 twd = (unsigned long) fxsave->twd;
259         u32 tag;
260         u32 ret = 0xffff0000u;
261         int i;
262
263         for (i = 0; i < 8; i++, twd >>= 1) {
264                 if (twd & 0x1) {
265                         st = FPREG_ADDR(fxsave, (i - tos) & 7);
266
267                         switch (st->exponent & 0x7fff) {
268                         case 0x7fff:
269                                 tag = FP_EXP_TAG_SPECIAL;
270                                 break;
271                         case 0x0000:
272                                 if (!st->significand[0] &&
273                                     !st->significand[1] &&
274                                     !st->significand[2] &&
275                                     !st->significand[3])
276                                         tag = FP_EXP_TAG_ZERO;
277                                 else
278                                         tag = FP_EXP_TAG_SPECIAL;
279                                 break;
280                         default:
281                                 if (st->significand[3] & 0x8000)
282                                         tag = FP_EXP_TAG_VALID;
283                                 else
284                                         tag = FP_EXP_TAG_SPECIAL;
285                                 break;
286                         }
287                 } else {
288                         tag = FP_EXP_TAG_EMPTY;
289                 }
290                 ret |= tag << (2 * i);
291         }
292         return ret;
293 }
294
295 /*
296  * FXSR floating point environment conversions.
297  */
298
299 static void
300 convert_from_fxsr(struct user_i387_ia32_struct *env, struct task_struct *tsk)
301 {
302         struct i387_fxsave_struct *fxsave = &tsk->thread.xstate->fxsave;
303         struct _fpreg *to = (struct _fpreg *) &env->st_space[0];
304         struct _fpxreg *from = (struct _fpxreg *) &fxsave->st_space[0];
305         int i;
306
307         env->cwd = fxsave->cwd | 0xffff0000u;
308         env->swd = fxsave->swd | 0xffff0000u;
309         env->twd = twd_fxsr_to_i387(fxsave);
310
311 #ifdef CONFIG_X86_64
312         env->fip = fxsave->rip;
313         env->foo = fxsave->rdp;
314         if (tsk == current) {
315                 /*
316                  * should be actually ds/cs at fpu exception time, but
317                  * that information is not available in 64bit mode.
318                  */
319                 asm("mov %%ds, %[fos]" : [fos] "=r" (env->fos));
320                 asm("mov %%cs, %[fcs]" : [fcs] "=r" (env->fcs));
321         } else {
322                 struct pt_regs *regs = task_pt_regs(tsk);
323
324                 env->fos = 0xffff0000 | tsk->thread.ds;
325                 env->fcs = regs->cs;
326         }
327 #else
328         env->fip = fxsave->fip;
329         env->fcs = (u16) fxsave->fcs | ((u32) fxsave->fop << 16);
330         env->foo = fxsave->foo;
331         env->fos = fxsave->fos;
332 #endif
333
334         for (i = 0; i < 8; ++i)
335                 memcpy(&to[i], &from[i], sizeof(to[0]));
336 }
337
338 static void convert_to_fxsr(struct task_struct *tsk,
339                             const struct user_i387_ia32_struct *env)
340
341 {
342         struct i387_fxsave_struct *fxsave = &tsk->thread.xstate->fxsave;
343         struct _fpreg *from = (struct _fpreg *) &env->st_space[0];
344         struct _fpxreg *to = (struct _fpxreg *) &fxsave->st_space[0];
345         int i;
346
347         fxsave->cwd = env->cwd;
348         fxsave->swd = env->swd;
349         fxsave->twd = twd_i387_to_fxsr(env->twd);
350         fxsave->fop = (u16) ((u32) env->fcs >> 16);
351 #ifdef CONFIG_X86_64
352         fxsave->rip = env->fip;
353         fxsave->rdp = env->foo;
354         /* cs and ds ignored */
355 #else
356         fxsave->fip = env->fip;
357         fxsave->fcs = (env->fcs & 0xffff);
358         fxsave->foo = env->foo;
359         fxsave->fos = env->fos;
360 #endif
361
362         for (i = 0; i < 8; ++i)
363                 memcpy(&to[i], &from[i], sizeof(from[0]));
364 }
365
366 int fpregs_get(struct task_struct *target, const struct user_regset *regset,
367                unsigned int pos, unsigned int count,
368                void *kbuf, void __user *ubuf)
369 {
370         struct user_i387_ia32_struct env;
371         int ret;
372
373         ret = init_fpu(target);
374         if (ret)
375                 return ret;
376
377         if (!HAVE_HWFP)
378                 return fpregs_soft_get(target, regset, pos, count, kbuf, ubuf);
379
380         if (!cpu_has_fxsr) {
381                 return user_regset_copyout(&pos, &count, &kbuf, &ubuf,
382                                            &target->thread.xstate->fsave, 0,
383                                            -1);
384         }
385
386         if (kbuf && pos == 0 && count == sizeof(env)) {
387                 convert_from_fxsr(kbuf, target);
388                 return 0;
389         }
390
391         convert_from_fxsr(&env, target);
392
393         return user_regset_copyout(&pos, &count, &kbuf, &ubuf, &env, 0, -1);
394 }
395
396 int fpregs_set(struct task_struct *target, const struct user_regset *regset,
397                unsigned int pos, unsigned int count,
398                const void *kbuf, const void __user *ubuf)
399 {
400         struct user_i387_ia32_struct env;
401         int ret;
402
403         ret = init_fpu(target);
404         if (ret)
405                 return ret;
406
407         set_stopped_child_used_math(target);
408
409         if (!HAVE_HWFP)
410                 return fpregs_soft_set(target, regset, pos, count, kbuf, ubuf);
411
412         if (!cpu_has_fxsr) {
413                 return user_regset_copyin(&pos, &count, &kbuf, &ubuf,
414                                           &target->thread.xstate->fsave, 0, -1);
415         }
416
417         if (pos > 0 || count < sizeof(env))
418                 convert_from_fxsr(&env, target);
419
420         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &env, 0, -1);
421         if (!ret)
422                 convert_to_fxsr(target, &env);
423
424         /*
425          * update the header bit in the xsave header, indicating the
426          * presence of FP.
427          */
428         if (cpu_has_xsave)
429                 target->thread.xstate->xsave.xsave_hdr.xstate_bv |= XSTATE_FP;
430         return ret;
431 }
432
433 /*
434  * Signal frame handlers.
435  */
436
437 static inline int save_i387_fsave(struct _fpstate_ia32 __user *buf)
438 {
439         struct task_struct *tsk = current;
440         struct i387_fsave_struct *fp = &tsk->thread.xstate->fsave;
441
442         fp->status = fp->swd;
443         if (__copy_to_user(buf, fp, sizeof(struct i387_fsave_struct)))
444                 return -1;
445         return 1;
446 }
447
448 static int save_i387_fxsave(struct _fpstate_ia32 __user *buf)
449 {
450         struct task_struct *tsk = current;
451         struct i387_fxsave_struct *fx = &tsk->thread.xstate->fxsave;
452         struct user_i387_ia32_struct env;
453         int err = 0;
454
455         convert_from_fxsr(&env, tsk);
456         if (__copy_to_user(buf, &env, sizeof(env)))
457                 return -1;
458
459         err |= __put_user(fx->swd, &buf->status);
460         err |= __put_user(X86_FXSR_MAGIC, &buf->magic);
461         if (err)
462                 return -1;
463
464         if (__copy_to_user(&buf->_fxsr_env[0], fx, xstate_size))
465                 return -1;
466         return 1;
467 }
468
469 static int save_i387_xsave(void __user *buf)
470 {
471         struct task_struct *tsk = current;
472         struct _fpstate_ia32 __user *fx = buf;
473         int err = 0;
474
475         /*
476          * For legacy compatible, we always set FP/SSE bits in the bit
477          * vector while saving the state to the user context.
478          * This will enable us capturing any changes(during sigreturn) to
479          * the FP/SSE bits by the legacy applications which don't touch
480          * xstate_bv in the xsave header.
481          *
482          * xsave aware applications can change the xstate_bv in the xsave
483          * header as well as change any contents in the memory layout.
484          * xrestore as part of sigreturn will capture all the changes.
485          */
486         tsk->thread.xstate->xsave.xsave_hdr.xstate_bv |= XSTATE_FPSSE;
487
488         if (save_i387_fxsave(fx) < 0)
489                 return -1;
490
491         err = __copy_to_user(&fx->sw_reserved, &fx_sw_reserved_ia32,
492                              sizeof(struct _fpx_sw_bytes));
493         err |= __put_user(FP_XSTATE_MAGIC2,
494                           (__u32 __user *) (buf + sig_xstate_ia32_size
495                                             - FP_XSTATE_MAGIC2_SIZE));
496         if (err)
497                 return -1;
498
499         return 1;
500 }
501
502 int save_i387_xstate_ia32(void __user *buf)
503 {
504         struct _fpstate_ia32 __user *fp = (struct _fpstate_ia32 __user *) buf;
505         struct task_struct *tsk = current;
506
507         if (!used_math())
508                 return 0;
509
510         if (!access_ok(VERIFY_WRITE, buf, sig_xstate_ia32_size))
511                 return -EACCES;
512         /*
513          * This will cause a "finit" to be triggered by the next
514          * attempted FPU operation by the 'current' process.
515          */
516         clear_used_math();
517
518         if (!HAVE_HWFP) {
519                 return fpregs_soft_get(current, NULL,
520                                        0, sizeof(struct user_i387_ia32_struct),
521                                        NULL, fp) ? -1 : 1;
522         }
523
524         unlazy_fpu(tsk);
525
526         if (cpu_has_xsave)
527                 return save_i387_xsave(fp);
528         if (cpu_has_fxsr)
529                 return save_i387_fxsave(fp);
530         else
531                 return save_i387_fsave(fp);
532 }
533
534 static inline int restore_i387_fsave(struct _fpstate_ia32 __user *buf)
535 {
536         struct task_struct *tsk = current;
537
538         return __copy_from_user(&tsk->thread.xstate->fsave, buf,
539                                 sizeof(struct i387_fsave_struct));
540 }
541
542 static int restore_i387_fxsave(struct _fpstate_ia32 __user *buf,
543                                unsigned int size)
544 {
545         struct task_struct *tsk = current;
546         struct user_i387_ia32_struct env;
547         int err;
548
549         err = __copy_from_user(&tsk->thread.xstate->fxsave, &buf->_fxsr_env[0],
550                                size);
551         /* mxcsr reserved bits must be masked to zero for security reasons */
552         tsk->thread.xstate->fxsave.mxcsr &= mxcsr_feature_mask;
553         if (err || __copy_from_user(&env, buf, sizeof(env)))
554                 return 1;
555         convert_to_fxsr(tsk, &env);
556
557         return 0;
558 }
559
560 static int restore_i387_xsave(void __user *buf)
561 {
562         struct _fpx_sw_bytes fx_sw_user;
563         struct _fpstate_ia32 __user *fx_user =
564                         ((struct _fpstate_ia32 __user *) buf);
565         struct i387_fxsave_struct __user *fx =
566                 (struct i387_fxsave_struct __user *) &fx_user->_fxsr_env[0];
567         struct xsave_hdr_struct *xsave_hdr =
568                                 &current->thread.xstate->xsave.xsave_hdr;
569         u64 mask;
570         int err;
571
572         if (check_for_xstate(fx, buf, &fx_sw_user))
573                 goto fx_only;
574
575         mask = fx_sw_user.xstate_bv;
576
577         err = restore_i387_fxsave(buf, fx_sw_user.xstate_size);
578
579         xsave_hdr->xstate_bv &= pcntxt_mask;
580         /*
581          * These bits must be zero.
582          */
583         xsave_hdr->reserved1[0] = xsave_hdr->reserved1[1] = 0;
584
585         /*
586          * Init the state that is not present in the memory layout
587          * and enabled by the OS.
588          */
589         mask = ~(pcntxt_mask & ~mask);
590         xsave_hdr->xstate_bv &= mask;
591
592         return err;
593 fx_only:
594         /*
595          * Couldn't find the extended state information in the memory
596          * layout. Restore the FP/SSE and init the other extended state
597          * enabled by the OS.
598          */
599         xsave_hdr->xstate_bv = XSTATE_FPSSE;
600         return restore_i387_fxsave(buf, sizeof(struct i387_fxsave_struct));
601 }
602
603 int restore_i387_xstate_ia32(void __user *buf)
604 {
605         int err;
606         struct task_struct *tsk = current;
607         struct _fpstate_ia32 __user *fp = (struct _fpstate_ia32 __user *) buf;
608
609         if (HAVE_HWFP)
610                 clear_fpu(tsk);
611
612         if (!buf) {
613                 if (used_math()) {
614                         clear_fpu(tsk);
615                         clear_used_math();
616                 }
617
618                 return 0;
619         } else
620                 if (!access_ok(VERIFY_READ, buf, sig_xstate_ia32_size))
621                         return -EACCES;
622
623         if (!used_math()) {
624                 err = init_fpu(tsk);
625                 if (err)
626                         return err;
627         }
628
629         if (HAVE_HWFP) {
630                 if (cpu_has_xsave)
631                         err = restore_i387_xsave(buf);
632                 else if (cpu_has_fxsr)
633                         err = restore_i387_fxsave(fp, sizeof(struct
634                                                            i387_fxsave_struct));
635                 else
636                         err = restore_i387_fsave(fp);
637         } else {
638                 err = fpregs_soft_set(current, NULL,
639                                       0, sizeof(struct user_i387_ia32_struct),
640                                       NULL, fp) != 0;
641         }
642         set_used_math();
643
644         return err;
645 }
646
647 /*
648  * FPU state for core dumps.
649  * This is only used for a.out dumps now.
650  * It is declared generically using elf_fpregset_t (which is
651  * struct user_i387_struct) but is in fact only used for 32-bit
652  * dumps, so on 64-bit it is really struct user_i387_ia32_struct.
653  */
654 int dump_fpu(struct pt_regs *regs, struct user_i387_struct *fpu)
655 {
656         struct task_struct *tsk = current;
657         int fpvalid;
658
659         fpvalid = !!used_math();
660         if (fpvalid)
661                 fpvalid = !fpregs_get(tsk, NULL,
662                                       0, sizeof(struct user_i387_ia32_struct),
663                                       fpu, NULL);
664
665         return fpvalid;
666 }
667 EXPORT_SYMBOL(dump_fpu);
668
669 #endif  /* CONFIG_X86_32 || CONFIG_IA32_EMULATION */