x86: make restore_fpu() use alternative assembler instructions
[linux-3.10.git] / arch / i386 / kernel / i387.c
1 /*
2  *  linux/arch/i386/kernel/i387.c
3  *
4  *  Copyright (C) 1994 Linus Torvalds
5  *
6  *  Pentium III FXSR, SSE support
7  *  General FPU state handling cleanups
8  *      Gareth Hughes <gareth@valinux.com>, May 2000
9  */
10
11 #include <linux/config.h>
12 #include <linux/sched.h>
13 #include <linux/module.h>
14 #include <asm/processor.h>
15 #include <asm/i387.h>
16 #include <asm/math_emu.h>
17 #include <asm/sigcontext.h>
18 #include <asm/user.h>
19 #include <asm/ptrace.h>
20 #include <asm/uaccess.h>
21
22 #ifdef CONFIG_MATH_EMULATION
23 #define HAVE_HWFP (boot_cpu_data.hard_math)
24 #else
25 #define HAVE_HWFP 1
26 #endif
27
28 static unsigned long mxcsr_feature_mask = 0xffffffff;
29
30 void mxcsr_feature_mask_init(void)
31 {
32         unsigned long mask = 0;
33         clts();
34         if (cpu_has_fxsr) {
35                 memset(&current->thread.i387.fxsave, 0, sizeof(struct i387_fxsave_struct));
36                 asm volatile("fxsave %0" : : "m" (current->thread.i387.fxsave)); 
37                 mask = current->thread.i387.fxsave.mxcsr_mask;
38                 if (mask == 0) mask = 0x0000ffbf;
39         } 
40         mxcsr_feature_mask &= mask;
41         stts();
42 }
43
44 /*
45  * The _current_ task is using the FPU for the first time
46  * so initialize it and set the mxcsr to its default
47  * value at reset if we support XMM instructions and then
48  * remeber the current task has used the FPU.
49  */
50 void init_fpu(struct task_struct *tsk)
51 {
52         if (cpu_has_fxsr) {
53                 memset(&tsk->thread.i387.fxsave, 0, sizeof(struct i387_fxsave_struct));
54                 tsk->thread.i387.fxsave.cwd = 0x37f;
55                 if (cpu_has_xmm)
56                         tsk->thread.i387.fxsave.mxcsr = 0x1f80;
57         } else {
58                 memset(&tsk->thread.i387.fsave, 0, sizeof(struct i387_fsave_struct));
59                 tsk->thread.i387.fsave.cwd = 0xffff037fu;
60                 tsk->thread.i387.fsave.swd = 0xffff0000u;
61                 tsk->thread.i387.fsave.twd = 0xffffffffu;
62                 tsk->thread.i387.fsave.fos = 0xffff0000u;
63         }
64         /* only the device not available exception or ptrace can call init_fpu */
65         set_stopped_child_used_math(tsk);
66 }
67
68 /*
69  * FPU lazy state save handling.
70  */
71
72 void kernel_fpu_begin(void)
73 {
74         struct thread_info *thread = current_thread_info();
75
76         preempt_disable();
77         if (thread->status & TS_USEDFPU) {
78                 __save_init_fpu(thread->task);
79                 return;
80         }
81         clts();
82 }
83 EXPORT_SYMBOL_GPL(kernel_fpu_begin);
84
85 /*
86  * FPU tag word conversions.
87  */
88
89 static inline unsigned short twd_i387_to_fxsr( unsigned short twd )
90 {
91         unsigned int tmp; /* to avoid 16 bit prefixes in the code */
92  
93         /* Transform each pair of bits into 01 (valid) or 00 (empty) */
94         tmp = ~twd;
95         tmp = (tmp | (tmp>>1)) & 0x5555; /* 0V0V0V0V0V0V0V0V */
96         /* and move the valid bits to the lower byte. */
97         tmp = (tmp | (tmp >> 1)) & 0x3333; /* 00VV00VV00VV00VV */
98         tmp = (tmp | (tmp >> 2)) & 0x0f0f; /* 0000VVVV0000VVVV */
99         tmp = (tmp | (tmp >> 4)) & 0x00ff; /* 00000000VVVVVVVV */
100         return tmp;
101 }
102
103 static inline unsigned long twd_fxsr_to_i387( struct i387_fxsave_struct *fxsave )
104 {
105         struct _fpxreg *st = NULL;
106         unsigned long tos = (fxsave->swd >> 11) & 7;
107         unsigned long twd = (unsigned long) fxsave->twd;
108         unsigned long tag;
109         unsigned long ret = 0xffff0000u;
110         int i;
111
112 #define FPREG_ADDR(f, n)        ((void *)&(f)->st_space + (n) * 16);
113
114         for ( i = 0 ; i < 8 ; i++ ) {
115                 if ( twd & 0x1 ) {
116                         st = FPREG_ADDR( fxsave, (i - tos) & 7 );
117
118                         switch ( st->exponent & 0x7fff ) {
119                         case 0x7fff:
120                                 tag = 2;                /* Special */
121                                 break;
122                         case 0x0000:
123                                 if ( !st->significand[0] &&
124                                      !st->significand[1] &&
125                                      !st->significand[2] &&
126                                      !st->significand[3] ) {
127                                         tag = 1;        /* Zero */
128                                 } else {
129                                         tag = 2;        /* Special */
130                                 }
131                                 break;
132                         default:
133                                 if ( st->significand[3] & 0x8000 ) {
134                                         tag = 0;        /* Valid */
135                                 } else {
136                                         tag = 2;        /* Special */
137                                 }
138                                 break;
139                         }
140                 } else {
141                         tag = 3;                        /* Empty */
142                 }
143                 ret |= (tag << (2 * i));
144                 twd = twd >> 1;
145         }
146         return ret;
147 }
148
149 /*
150  * FPU state interaction.
151  */
152
153 unsigned short get_fpu_cwd( struct task_struct *tsk )
154 {
155         if ( cpu_has_fxsr ) {
156                 return tsk->thread.i387.fxsave.cwd;
157         } else {
158                 return (unsigned short)tsk->thread.i387.fsave.cwd;
159         }
160 }
161
162 unsigned short get_fpu_swd( struct task_struct *tsk )
163 {
164         if ( cpu_has_fxsr ) {
165                 return tsk->thread.i387.fxsave.swd;
166         } else {
167                 return (unsigned short)tsk->thread.i387.fsave.swd;
168         }
169 }
170
171 #if 0
172 unsigned short get_fpu_twd( struct task_struct *tsk )
173 {
174         if ( cpu_has_fxsr ) {
175                 return tsk->thread.i387.fxsave.twd;
176         } else {
177                 return (unsigned short)tsk->thread.i387.fsave.twd;
178         }
179 }
180 #endif  /*  0  */
181
182 unsigned short get_fpu_mxcsr( struct task_struct *tsk )
183 {
184         if ( cpu_has_xmm ) {
185                 return tsk->thread.i387.fxsave.mxcsr;
186         } else {
187                 return 0x1f80;
188         }
189 }
190
191 #if 0
192
193 void set_fpu_cwd( struct task_struct *tsk, unsigned short cwd )
194 {
195         if ( cpu_has_fxsr ) {
196                 tsk->thread.i387.fxsave.cwd = cwd;
197         } else {
198                 tsk->thread.i387.fsave.cwd = ((long)cwd | 0xffff0000u);
199         }
200 }
201
202 void set_fpu_swd( struct task_struct *tsk, unsigned short swd )
203 {
204         if ( cpu_has_fxsr ) {
205                 tsk->thread.i387.fxsave.swd = swd;
206         } else {
207                 tsk->thread.i387.fsave.swd = ((long)swd | 0xffff0000u);
208         }
209 }
210
211 void set_fpu_twd( struct task_struct *tsk, unsigned short twd )
212 {
213         if ( cpu_has_fxsr ) {
214                 tsk->thread.i387.fxsave.twd = twd_i387_to_fxsr(twd);
215         } else {
216                 tsk->thread.i387.fsave.twd = ((long)twd | 0xffff0000u);
217         }
218 }
219
220 #endif  /*  0  */
221
222 /*
223  * FXSR floating point environment conversions.
224  */
225
226 static int convert_fxsr_to_user( struct _fpstate __user *buf,
227                                         struct i387_fxsave_struct *fxsave )
228 {
229         unsigned long env[7];
230         struct _fpreg __user *to;
231         struct _fpxreg *from;
232         int i;
233
234         env[0] = (unsigned long)fxsave->cwd | 0xffff0000ul;
235         env[1] = (unsigned long)fxsave->swd | 0xffff0000ul;
236         env[2] = twd_fxsr_to_i387(fxsave);
237         env[3] = fxsave->fip;
238         env[4] = fxsave->fcs | ((unsigned long)fxsave->fop << 16);
239         env[5] = fxsave->foo;
240         env[6] = fxsave->fos;
241
242         if ( __copy_to_user( buf, env, 7 * sizeof(unsigned long) ) )
243                 return 1;
244
245         to = &buf->_st[0];
246         from = (struct _fpxreg *) &fxsave->st_space[0];
247         for ( i = 0 ; i < 8 ; i++, to++, from++ ) {
248                 unsigned long __user *t = (unsigned long __user *)to;
249                 unsigned long *f = (unsigned long *)from;
250
251                 if (__put_user(*f, t) ||
252                                 __put_user(*(f + 1), t + 1) ||
253                                 __put_user(from->exponent, &to->exponent))
254                         return 1;
255         }
256         return 0;
257 }
258
259 static int convert_fxsr_from_user( struct i387_fxsave_struct *fxsave,
260                                           struct _fpstate __user *buf )
261 {
262         unsigned long env[7];
263         struct _fpxreg *to;
264         struct _fpreg __user *from;
265         int i;
266
267         if ( __copy_from_user( env, buf, 7 * sizeof(long) ) )
268                 return 1;
269
270         fxsave->cwd = (unsigned short)(env[0] & 0xffff);
271         fxsave->swd = (unsigned short)(env[1] & 0xffff);
272         fxsave->twd = twd_i387_to_fxsr((unsigned short)(env[2] & 0xffff));
273         fxsave->fip = env[3];
274         fxsave->fop = (unsigned short)((env[4] & 0xffff0000ul) >> 16);
275         fxsave->fcs = (env[4] & 0xffff);
276         fxsave->foo = env[5];
277         fxsave->fos = env[6];
278
279         to = (struct _fpxreg *) &fxsave->st_space[0];
280         from = &buf->_st[0];
281         for ( i = 0 ; i < 8 ; i++, to++, from++ ) {
282                 unsigned long *t = (unsigned long *)to;
283                 unsigned long __user *f = (unsigned long __user *)from;
284
285                 if (__get_user(*t, f) ||
286                                 __get_user(*(t + 1), f + 1) ||
287                                 __get_user(to->exponent, &from->exponent))
288                         return 1;
289         }
290         return 0;
291 }
292
293 /*
294  * Signal frame handlers.
295  */
296
297 static inline int save_i387_fsave( struct _fpstate __user *buf )
298 {
299         struct task_struct *tsk = current;
300
301         unlazy_fpu( tsk );
302         tsk->thread.i387.fsave.status = tsk->thread.i387.fsave.swd;
303         if ( __copy_to_user( buf, &tsk->thread.i387.fsave,
304                              sizeof(struct i387_fsave_struct) ) )
305                 return -1;
306         return 1;
307 }
308
309 static int save_i387_fxsave( struct _fpstate __user *buf )
310 {
311         struct task_struct *tsk = current;
312         int err = 0;
313
314         unlazy_fpu( tsk );
315
316         if ( convert_fxsr_to_user( buf, &tsk->thread.i387.fxsave ) )
317                 return -1;
318
319         err |= __put_user( tsk->thread.i387.fxsave.swd, &buf->status );
320         err |= __put_user( X86_FXSR_MAGIC, &buf->magic );
321         if ( err )
322                 return -1;
323
324         if ( __copy_to_user( &buf->_fxsr_env[0], &tsk->thread.i387.fxsave,
325                              sizeof(struct i387_fxsave_struct) ) )
326                 return -1;
327         return 1;
328 }
329
330 int save_i387( struct _fpstate __user *buf )
331 {
332         if ( !used_math() )
333                 return 0;
334
335         /* This will cause a "finit" to be triggered by the next
336          * attempted FPU operation by the 'current' process.
337          */
338         clear_used_math();
339
340         if ( HAVE_HWFP ) {
341                 if ( cpu_has_fxsr ) {
342                         return save_i387_fxsave( buf );
343                 } else {
344                         return save_i387_fsave( buf );
345                 }
346         } else {
347                 return save_i387_soft( &current->thread.i387.soft, buf );
348         }
349 }
350
351 static inline int restore_i387_fsave( struct _fpstate __user *buf )
352 {
353         struct task_struct *tsk = current;
354         clear_fpu( tsk );
355         return __copy_from_user( &tsk->thread.i387.fsave, buf,
356                                  sizeof(struct i387_fsave_struct) );
357 }
358
359 static int restore_i387_fxsave( struct _fpstate __user *buf )
360 {
361         int err;
362         struct task_struct *tsk = current;
363         clear_fpu( tsk );
364         err = __copy_from_user( &tsk->thread.i387.fxsave, &buf->_fxsr_env[0],
365                                 sizeof(struct i387_fxsave_struct) );
366         /* mxcsr reserved bits must be masked to zero for security reasons */
367         tsk->thread.i387.fxsave.mxcsr &= mxcsr_feature_mask;
368         return err ? 1 : convert_fxsr_from_user( &tsk->thread.i387.fxsave, buf );
369 }
370
371 int restore_i387( struct _fpstate __user *buf )
372 {
373         int err;
374
375         if ( HAVE_HWFP ) {
376                 if ( cpu_has_fxsr ) {
377                         err = restore_i387_fxsave( buf );
378                 } else {
379                         err = restore_i387_fsave( buf );
380                 }
381         } else {
382                 err = restore_i387_soft( &current->thread.i387.soft, buf );
383         }
384         set_used_math();
385         return err;
386 }
387
388 /*
389  * ptrace request handlers.
390  */
391
392 static inline int get_fpregs_fsave( struct user_i387_struct __user *buf,
393                                     struct task_struct *tsk )
394 {
395         return __copy_to_user( buf, &tsk->thread.i387.fsave,
396                                sizeof(struct user_i387_struct) );
397 }
398
399 static inline int get_fpregs_fxsave( struct user_i387_struct __user *buf,
400                                      struct task_struct *tsk )
401 {
402         return convert_fxsr_to_user( (struct _fpstate __user *)buf,
403                                      &tsk->thread.i387.fxsave );
404 }
405
406 int get_fpregs( struct user_i387_struct __user *buf, struct task_struct *tsk )
407 {
408         if ( HAVE_HWFP ) {
409                 if ( cpu_has_fxsr ) {
410                         return get_fpregs_fxsave( buf, tsk );
411                 } else {
412                         return get_fpregs_fsave( buf, tsk );
413                 }
414         } else {
415                 return save_i387_soft( &tsk->thread.i387.soft,
416                                        (struct _fpstate __user *)buf );
417         }
418 }
419
420 static inline int set_fpregs_fsave( struct task_struct *tsk,
421                                     struct user_i387_struct __user *buf )
422 {
423         return __copy_from_user( &tsk->thread.i387.fsave, buf,
424                                  sizeof(struct user_i387_struct) );
425 }
426
427 static inline int set_fpregs_fxsave( struct task_struct *tsk,
428                                      struct user_i387_struct __user *buf )
429 {
430         return convert_fxsr_from_user( &tsk->thread.i387.fxsave,
431                                        (struct _fpstate __user *)buf );
432 }
433
434 int set_fpregs( struct task_struct *tsk, struct user_i387_struct __user *buf )
435 {
436         if ( HAVE_HWFP ) {
437                 if ( cpu_has_fxsr ) {
438                         return set_fpregs_fxsave( tsk, buf );
439                 } else {
440                         return set_fpregs_fsave( tsk, buf );
441                 }
442         } else {
443                 return restore_i387_soft( &tsk->thread.i387.soft,
444                                           (struct _fpstate __user *)buf );
445         }
446 }
447
448 int get_fpxregs( struct user_fxsr_struct __user *buf, struct task_struct *tsk )
449 {
450         if ( cpu_has_fxsr ) {
451                 if (__copy_to_user( buf, &tsk->thread.i387.fxsave,
452                                     sizeof(struct user_fxsr_struct) ))
453                         return -EFAULT;
454                 return 0;
455         } else {
456                 return -EIO;
457         }
458 }
459
460 int set_fpxregs( struct task_struct *tsk, struct user_fxsr_struct __user *buf )
461 {
462         int ret = 0;
463
464         if ( cpu_has_fxsr ) {
465                 if (__copy_from_user( &tsk->thread.i387.fxsave, buf,
466                                   sizeof(struct user_fxsr_struct) ))
467                         ret = -EFAULT;
468                 /* mxcsr reserved bits must be masked to zero for security reasons */
469                 tsk->thread.i387.fxsave.mxcsr &= mxcsr_feature_mask;
470         } else {
471                 ret = -EIO;
472         }
473         return ret;
474 }
475
476 /*
477  * FPU state for core dumps.
478  */
479
480 static inline void copy_fpu_fsave( struct task_struct *tsk,
481                                    struct user_i387_struct *fpu )
482 {
483         memcpy( fpu, &tsk->thread.i387.fsave,
484                 sizeof(struct user_i387_struct) );
485 }
486
487 static inline void copy_fpu_fxsave( struct task_struct *tsk,
488                                    struct user_i387_struct *fpu )
489 {
490         unsigned short *to;
491         unsigned short *from;
492         int i;
493
494         memcpy( fpu, &tsk->thread.i387.fxsave, 7 * sizeof(long) );
495
496         to = (unsigned short *)&fpu->st_space[0];
497         from = (unsigned short *)&tsk->thread.i387.fxsave.st_space[0];
498         for ( i = 0 ; i < 8 ; i++, to += 5, from += 8 ) {
499                 memcpy( to, from, 5 * sizeof(unsigned short) );
500         }
501 }
502
503 int dump_fpu( struct pt_regs *regs, struct user_i387_struct *fpu )
504 {
505         int fpvalid;
506         struct task_struct *tsk = current;
507
508         fpvalid = !!used_math();
509         if ( fpvalid ) {
510                 unlazy_fpu( tsk );
511                 if ( cpu_has_fxsr ) {
512                         copy_fpu_fxsave( tsk, fpu );
513                 } else {
514                         copy_fpu_fsave( tsk, fpu );
515                 }
516         }
517
518         return fpvalid;
519 }
520 EXPORT_SYMBOL(dump_fpu);
521
522 int dump_task_fpu(struct task_struct *tsk, struct user_i387_struct *fpu)
523 {
524         int fpvalid = !!tsk_used_math(tsk);
525
526         if (fpvalid) {
527                 if (tsk == current)
528                         unlazy_fpu(tsk);
529                 if (cpu_has_fxsr)
530                         copy_fpu_fxsave(tsk, fpu);
531                 else
532                         copy_fpu_fsave(tsk, fpu);
533         }
534         return fpvalid;
535 }
536
537 int dump_task_extended_fpu(struct task_struct *tsk, struct user_fxsr_struct *fpu)
538 {
539         int fpvalid = tsk_used_math(tsk) && cpu_has_fxsr;
540
541         if (fpvalid) {
542                 if (tsk == current)
543                        unlazy_fpu(tsk);
544                 memcpy(fpu, &tsk->thread.i387.fxsave, sizeof(*fpu));
545         }
546         return fpvalid;
547 }