arch/x86/kernel/i387.c

   1 /*
   2  *  Copyright (C) 1994 Linus Torvalds
   3  *
   4  *  Pentium III FXSR, SSE support
   5  *  General FPU state handling cleanups
   6  *      Gareth Hughes <gareth@valinux.com>, May 2000
   7  */
   8 #include <linux/module.h>
   9 #include <linux/regset.h>
  10 #include <linux/sched.h>
  11 #include <linux/slab.h>
  12
  13 #include <asm/sigcontext.h>
  14 #include <asm/processor.h>
  15 #include <asm/math_emu.h>
  16 #include <asm/uaccess.h>
  17 #include <asm/ptrace.h>
  18 #include <asm/i387.h>
  19 #include <asm/fpu-internal.h>
  20 #include <asm/user.h>
  21
  22 #ifdef CONFIG_X86_64
  23 # include <asm/sigcontext32.h>
  24 # include <asm/user32.h>
  25 #else
  26 # define save_i387_xstate_ia32          save_i387_xstate
  27 # define restore_i387_xstate_ia32       restore_i387_xstate
  28 # define _fpstate_ia32          _fpstate
  29 # define _xstate_ia32           _xstate
  30 # define sig_xstate_ia32_size   sig_xstate_size
  31 # define fx_sw_reserved_ia32    fx_sw_reserved
  32 # define user_i387_ia32_struct  user_i387_struct
  33 # define user32_fxsr_struct     user_fxsr_struct
  34 #endif
  35
  36 /*
  37  * Were we in an interrupt that interrupted kernel mode?
  38  *
  39  * We can do a kernel_fpu_begin/end() pair *ONLY* if that
  40  * pair does nothing at all: the thread must not have fpu (so
  41  * that we don't try to save the FPU state), and TS must
  42  * be set (so that the clts/stts pair does nothing that is
  43  * visible in the interrupted kernel thread).
  44  */
  45 static inline bool interrupted_kernel_fpu_idle(void)
  46 {
  47         return !__thread_has_fpu(current) &&
  48                 (read_cr0() & X86_CR0_TS);
  49 }
  50
  51 /*
  52  * Were we in user mode (or vm86 mode) when we were
  53  * interrupted?
  54  *
  55  * Doing kernel_fpu_begin/end() is ok if we are running
  56  * in an interrupt context from user mode - we'll just
  57  * save the FPU state as required.
  58  */
  59 static inline bool interrupted_user_mode(void)
  60 {
  61         struct pt_regs *regs = get_irq_regs();
  62         return regs && user_mode_vm(regs);
  63 }
  64
  65 /*
  66  * Can we use the FPU in kernel mode with the
  67  * whole "kernel_fpu_begin/end()" sequence?
  68  *
  69  * It's always ok in process context (ie "not interrupt")
  70  * but it is sometimes ok even from an irq.
  71  */
  72 bool irq_fpu_usable(void)
  73 {
  74         return !in_interrupt() ||
  75                 interrupted_user_mode() ||
  76                 interrupted_kernel_fpu_idle();
  77 }
  78 EXPORT_SYMBOL(irq_fpu_usable);
  79
  80 void kernel_fpu_begin(void)
  81 {
  82         struct task_struct *me = current;
  83
  84         WARN_ON_ONCE(!irq_fpu_usable());
  85         preempt_disable();
  86         if (__thread_has_fpu(me)) {
  87                 __save_init_fpu(me);
  88                 __thread_clear_has_fpu(me);
  89                 /* We do 'stts()' in kernel_fpu_end() */
  90         } else {
  91                 percpu_write(fpu_owner_task, NULL);
  92                 clts();
  93         }
  94 }
  95 EXPORT_SYMBOL(kernel_fpu_begin);
  96
  97 void kernel_fpu_end(void)
  98 {
  99         stts();
 100         preempt_enable();
 101 }
 102 EXPORT_SYMBOL(kernel_fpu_end);
 103
 104 void unlazy_fpu(struct task_struct *tsk)
 105 {
 106         preempt_disable();
 107         if (__thread_has_fpu(tsk)) {
 108                 __save_init_fpu(tsk);
 109                 __thread_fpu_end(tsk);
 110         } else
 111                 tsk->fpu_counter = 0;
 112         preempt_enable();
 113 }
 114 EXPORT_SYMBOL(unlazy_fpu);
 115
 116 #ifdef CONFIG_MATH_EMULATION
 117 # define HAVE_HWFP              (boot_cpu_data.hard_math)
 118 #else
 119 # define HAVE_HWFP              1
 120 #endif
 121
 122 static unsigned int             mxcsr_feature_mask __read_mostly = 0xffffffffu;
 123 unsigned int xstate_size;
 124 EXPORT_SYMBOL_GPL(xstate_size);
 125 unsigned int sig_xstate_ia32_size = sizeof(struct _fpstate_ia32);
 126 static struct i387_fxsave_struct fx_scratch __cpuinitdata;
 127
 128 static void __cpuinit mxcsr_feature_mask_init(void)
 129 {
 130         unsigned long mask = 0;
 131
 132         clts();
 133         if (cpu_has_fxsr) {
 134                 memset(&fx_scratch, 0, sizeof(struct i387_fxsave_struct));
 135                 asm volatile("fxsave %0" : : "m" (fx_scratch));
 136                 mask = fx_scratch.mxcsr_mask;
 137                 if (mask == 0)
 138                         mask = 0x0000ffbf;
 139         }
 140         mxcsr_feature_mask &= mask;
 141         stts();
 142 }
 143
 144 static void __cpuinit init_thread_xstate(void)
 145 {
 146         /*
 147          * Note that xstate_size might be overwriten later during
 148          * xsave_init().
 149          */
 150
 151         if (!HAVE_HWFP) {
 152                 /*
 153                  * Disable xsave as we do not support it if i387
 154                  * emulation is enabled.
 155                  */
 156                 setup_clear_cpu_cap(X86_FEATURE_XSAVE);
 157                 setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT);
 158                 xstate_size = sizeof(struct i387_soft_struct);
 159                 return;
 160         }
 161
 162         if (cpu_has_fxsr)
 163                 xstate_size = sizeof(struct i387_fxsave_struct);
 164         else
 165                 xstate_size = sizeof(struct i387_fsave_struct);
 166 }
 167
 168 /*
 169  * Called at bootup to set up the initial FPU state that is later cloned
 170  * into all processes.
 171  */
 172
 173 void __cpuinit fpu_init(void)
 174 {
 175         unsigned long cr0;
 176         unsigned long cr4_mask = 0;
 177
 178         if (cpu_has_fxsr)
 179                 cr4_mask |= X86_CR4_OSFXSR;
 180         if (cpu_has_xmm)
 181                 cr4_mask |= X86_CR4_OSXMMEXCPT;
 182         if (cr4_mask)
 183                 set_in_cr4(cr4_mask);
 184
 185         cr0 = read_cr0();
 186         cr0 &= ~(X86_CR0_TS|X86_CR0_EM); /* clear TS and EM */
 187         if (!HAVE_HWFP)
 188                 cr0 |= X86_CR0_EM;
 189         write_cr0(cr0);
 190
 191         if (!smp_processor_id())
 192                 init_thread_xstate();
 193
 194         mxcsr_feature_mask_init();
 195         /* clean state in init */
 196         current_thread_info()->status = 0;
 197         clear_used_math();
 198 }
 199
 200 void fpu_finit(struct fpu *fpu)
 201 {
 202         if (!HAVE_HWFP) {
 203                 finit_soft_fpu(&fpu->state->soft);
 204                 return;
 205         }
 206
 207         if (cpu_has_fxsr) {
 208                 struct i387_fxsave_struct *fx = &fpu->state->fxsave;
 209
 210                 memset(fx, 0, xstate_size);
 211                 fx->cwd = 0x37f;
 212                 if (cpu_has_xmm)
 213                         fx->mxcsr = MXCSR_DEFAULT;
 214         } else {
 215                 struct i387_fsave_struct *fp = &fpu->state->fsave;
 216                 memset(fp, 0, xstate_size);
 217                 fp->cwd = 0xffff037fu;
 218                 fp->swd = 0xffff0000u;
 219                 fp->twd = 0xffffffffu;
 220                 fp->fos = 0xffff0000u;
 221         }
 222 }
 223 EXPORT_SYMBOL_GPL(fpu_finit);
 224
 225 /*
 226  * The _current_ task is using the FPU for the first time
 227  * so initialize it and set the mxcsr to its default
 228  * value at reset if we support XMM instructions and then
 229  * remember the current task has used the FPU.
 230  */
 231 int init_fpu(struct task_struct *tsk)
 232 {
 233         int ret;
 234
 235         if (tsk_used_math(tsk)) {
 236                 if (HAVE_HWFP && tsk == current)
 237                         unlazy_fpu(tsk);
 238                 return 0;
 239         }
 240
 241         /*
 242          * Memory allocation at the first usage of the FPU and other state.
 243          */
 244         ret = fpu_alloc(&tsk->thread.fpu);
 245         if (ret)
 246                 return ret;
 247
 248         fpu_finit(&tsk->thread.fpu);
 249
 250         set_stopped_child_used_math(tsk);
 251         return 0;
 252 }
 253 EXPORT_SYMBOL_GPL(init_fpu);
 254
 255 /*
 256  * The xstateregs_active() routine is the same as the fpregs_active() routine,
 257  * as the "regset->n" for the xstate regset will be updated based on the feature
 258  * capabilites supported by the xsave.
 259  */
 260 int fpregs_active(struct task_struct *target, const struct user_regset *regset)
 261 {
 262         return tsk_used_math(target) ? regset->n : 0;
 263 }
 264
 265 int xfpregs_active(struct task_struct *target, const struct user_regset *regset)
 266 {
 267         return (cpu_has_fxsr && tsk_used_math(target)) ? regset->n : 0;
 268 }
 269
 270 int xfpregs_get(struct task_struct *target, const struct user_regset *regset,
 271                 unsigned int pos, unsigned int count,
 272                 void *kbuf, void __user *ubuf)
 273 {
 274         int ret;
 275
 276         if (!cpu_has_fxsr)
 277                 return -ENODEV;
 278
 279         ret = init_fpu(target);
 280         if (ret)
 281                 return ret;
 282
 283         sanitize_i387_state(target);
 284
 285         return user_regset_copyout(&pos, &count, &kbuf, &ubuf,
 286                                    &target->thread.fpu.state->fxsave, 0, -1);
 287 }
 288
 289 int xfpregs_set(struct task_struct *target, const struct user_regset *regset,
 290                 unsigned int pos, unsigned int count,
 291                 const void *kbuf, const void __user *ubuf)
 292 {
 293         int ret;
 294
 295         if (!cpu_has_fxsr)
 296                 return -ENODEV;
 297
 298         ret = init_fpu(target);
 299         if (ret)
 300                 return ret;
 301
 302         sanitize_i387_state(target);
 303
 304         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
 305                                  &target->thread.fpu.state->fxsave, 0, -1);
 306
 307         /*
 308          * mxcsr reserved bits must be masked to zero for security reasons.
 309          */
 310         target->thread.fpu.state->fxsave.mxcsr &= mxcsr_feature_mask;
 311
 312         /*
 313          * update the header bits in the xsave header, indicating the
 314          * presence of FP and SSE state.
 315          */
 316         if (cpu_has_xsave)
 317                 target->thread.fpu.state->xsave.xsave_hdr.xstate_bv |= XSTATE_FPSSE;
 318
 319         return ret;
 320 }
 321
 322 int xstateregs_get(struct task_struct *target, const struct user_regset *regset,
 323                 unsigned int pos, unsigned int count,
 324                 void *kbuf, void __user *ubuf)
 325 {
 326         int ret;
 327
 328         if (!cpu_has_xsave)
 329                 return -ENODEV;
 330
 331         ret = init_fpu(target);
 332         if (ret)
 333                 return ret;
 334
 335         /*
 336          * Copy the 48bytes defined by the software first into the xstate
 337          * memory layout in the thread struct, so that we can copy the entire
 338          * xstateregs to the user using one user_regset_copyout().
 339          */
 340         memcpy(&target->thread.fpu.state->fxsave.sw_reserved,
 341                xstate_fx_sw_bytes, sizeof(xstate_fx_sw_bytes));
 342
 343         /*
 344          * Copy the xstate memory layout.
 345          */
 346         ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
 347                                   &target->thread.fpu.state->xsave, 0, -1);
 348         return ret;
 349 }
 350
 351 int xstateregs_set(struct task_struct *target, const struct user_regset *regset,
 352                   unsigned int pos, unsigned int count,
 353                   const void *kbuf, const void __user *ubuf)
 354 {
 355         int ret;
 356         struct xsave_hdr_struct *xsave_hdr;
 357
 358         if (!cpu_has_xsave)
 359                 return -ENODEV;
 360
 361         ret = init_fpu(target);
 362         if (ret)
 363                 return ret;
 364
 365         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
 366                                  &target->thread.fpu.state->xsave, 0, -1);
 367
 368         /*
 369          * mxcsr reserved bits must be masked to zero for security reasons.
 370          */
 371         target->thread.fpu.state->fxsave.mxcsr &= mxcsr_feature_mask;
 372
 373         xsave_hdr = &target->thread.fpu.state->xsave.xsave_hdr;
 374
 375         xsave_hdr->xstate_bv &= pcntxt_mask;
 376         /*
 377          * These bits must be zero.
 378          */
 379         xsave_hdr->reserved1[0] = xsave_hdr->reserved1[1] = 0;
 380
 381         return ret;
 382 }
 383
 384 #if defined CONFIG_X86_32 || defined CONFIG_IA32_EMULATION
 385
 386 /*
 387  * FPU tag word conversions.
 388  */
 389
 390 static inline unsigned short twd_i387_to_fxsr(unsigned short twd)
 391 {
 392         unsigned int tmp; /* to avoid 16 bit prefixes in the code */
 393
 394         /* Transform each pair of bits into 01 (valid) or 00 (empty) */
 395         tmp = ~twd;
 396         tmp = (tmp | (tmp>>1)) & 0x5555; /* 0V0V0V0V0V0V0V0V */
 397         /* and move the valid bits to the lower byte. */
 398         tmp = (tmp | (tmp >> 1)) & 0x3333; /* 00VV00VV00VV00VV */
 399         tmp = (tmp | (tmp >> 2)) & 0x0f0f; /* 0000VVVV0000VVVV */
 400         tmp = (tmp | (tmp >> 4)) & 0x00ff; /* 00000000VVVVVVVV */
 401
 402         return tmp;
 403 }
 404
 405 #define FPREG_ADDR(f, n)        ((void *)&(f)->st_space + (n) * 16)
 406 #define FP_EXP_TAG_VALID        0
 407 #define FP_EXP_TAG_ZERO         1
 408 #define FP_EXP_TAG_SPECIAL      2
 409 #define FP_EXP_TAG_EMPTY        3
 410
 411 static inline u32 twd_fxsr_to_i387(struct i387_fxsave_struct *fxsave)
 412 {
 413         struct _fpxreg *st;
 414         u32 tos = (fxsave->swd >> 11) & 7;
 415         u32 twd = (unsigned long) fxsave->twd;
 416         u32 tag;
 417         u32 ret = 0xffff0000u;
 418         int i;
 419
 420         for (i = 0; i < 8; i++, twd >>= 1) {
 421                 if (twd & 0x1) {
 422                         st = FPREG_ADDR(fxsave, (i - tos) & 7);
 423
 424                         switch (st->exponent & 0x7fff) {
 425                         case 0x7fff:
 426                                 tag = FP_EXP_TAG_SPECIAL;
 427                                 break;
 428                         case 0x0000:
 429                                 if (!st->significand[0] &&
 430                                     !st->significand[1] &&
 431                                     !st->significand[2] &&
 432                                     !st->significand[3])
 433                                         tag = FP_EXP_TAG_ZERO;
 434                                 else
 435                                         tag = FP_EXP_TAG_SPECIAL;
 436                                 break;
 437                         default:
 438                                 if (st->significand[3] & 0x8000)
 439                                         tag = FP_EXP_TAG_VALID;
 440                                 else
 441                                         tag = FP_EXP_TAG_SPECIAL;
 442                                 break;
 443                         }
 444                 } else {
 445                         tag = FP_EXP_TAG_EMPTY;
 446                 }
 447                 ret |= tag << (2 * i);
 448         }
 449         return ret;
 450 }
 451
 452 /*
 453  * FXSR floating point environment conversions.
 454  */
 455
 456 static void
 457 convert_from_fxsr(struct user_i387_ia32_struct *env, struct task_struct *tsk)
 458 {
 459         struct i387_fxsave_struct *fxsave = &tsk->thread.fpu.state->fxsave;
 460         struct _fpreg *to = (struct _fpreg *) &env->st_space[0];
 461         struct _fpxreg *from = (struct _fpxreg *) &fxsave->st_space[0];
 462         int i;
 463
 464         env->cwd = fxsave->cwd | 0xffff0000u;
 465         env->swd = fxsave->swd | 0xffff0000u;
 466         env->twd = twd_fxsr_to_i387(fxsave);
 467
 468 #ifdef CONFIG_X86_64
 469         env->fip = fxsave->rip;
 470         env->foo = fxsave->rdp;
 471         /*
 472          * should be actually ds/cs at fpu exception time, but
 473          * that information is not available in 64bit mode.
 474          */
 475         env->fcs = task_pt_regs(tsk)->cs;
 476         if (tsk == current) {
 477                 savesegment(ds, env->fos);
 478         } else {
 479                 env->fos = tsk->thread.ds;
 480         }
 481         env->fos |= 0xffff0000;
 482 #else
 483         env->fip = fxsave->fip;
 484         env->fcs = (u16) fxsave->fcs | ((u32) fxsave->fop << 16);
 485         env->foo = fxsave->foo;
 486         env->fos = fxsave->fos;
 487 #endif
 488
 489         for (i = 0; i < 8; ++i)
 490                 memcpy(&to[i], &from[i], sizeof(to[0]));
 491 }
 492
 493 static void convert_to_fxsr(struct task_struct *tsk,
 494                             const struct user_i387_ia32_struct *env)
 495
 496 {
 497         struct i387_fxsave_struct *fxsave = &tsk->thread.fpu.state->fxsave;
 498         struct _fpreg *from = (struct _fpreg *) &env->st_space[0];
 499         struct _fpxreg *to = (struct _fpxreg *) &fxsave->st_space[0];
 500         int i;
 501
 502         fxsave->cwd = env->cwd;
 503         fxsave->swd = env->swd;
 504         fxsave->twd = twd_i387_to_fxsr(env->twd);
 505         fxsave->fop = (u16) ((u32) env->fcs >> 16);
 506 #ifdef CONFIG_X86_64
 507         fxsave->rip = env->fip;
 508         fxsave->rdp = env->foo;
 509         /* cs and ds ignored */
 510 #else
 511         fxsave->fip = env->fip;
 512         fxsave->fcs = (env->fcs & 0xffff);
 513         fxsave->foo = env->foo;
 514         fxsave->fos = env->fos;
 515 #endif
 516
 517         for (i = 0; i < 8; ++i)
 518                 memcpy(&to[i], &from[i], sizeof(from[0]));
 519 }
 520
 521 int fpregs_get(struct task_struct *target, const struct user_regset *regset,
 522                unsigned int pos, unsigned int count,
 523                void *kbuf, void __user *ubuf)
 524 {
 525         struct user_i387_ia32_struct env;
 526         int ret;
 527
 528         ret = init_fpu(target);
 529         if (ret)
 530                 return ret;
 531
 532         if (!HAVE_HWFP)
 533                 return fpregs_soft_get(target, regset, pos, count, kbuf, ubuf);
 534
 535         if (!cpu_has_fxsr) {
 536                 return user_regset_copyout(&pos, &count, &kbuf, &ubuf,
 537                                            &target->thread.fpu.state->fsave, 0,
 538                                            -1);
 539         }
 540
 541         sanitize_i387_state(target);
 542
 543         if (kbuf && pos == 0 && count == sizeof(env)) {
 544                 convert_from_fxsr(kbuf, target);
 545                 return 0;
 546         }
 547
 548         convert_from_fxsr(&env, target);
 549
 550         return user_regset_copyout(&pos, &count, &kbuf, &ubuf, &env, 0, -1);
 551 }
 552
 553 int fpregs_set(struct task_struct *target, const struct user_regset *regset,
 554                unsigned int pos, unsigned int count,
 555                const void *kbuf, const void __user *ubuf)
 556 {
 557         struct user_i387_ia32_struct env;
 558         int ret;
 559
 560         ret = init_fpu(target);
 561         if (ret)
 562                 return ret;
 563
 564         sanitize_i387_state(target);
 565
 566         if (!HAVE_HWFP)
 567                 return fpregs_soft_set(target, regset, pos, count, kbuf, ubuf);
 568
 569         if (!cpu_has_fxsr) {
 570                 return user_regset_copyin(&pos, &count, &kbuf, &ubuf,
 571                                           &target->thread.fpu.state->fsave, 0, -1);
 572         }
 573
 574         if (pos > 0 || count < sizeof(env))
 575                 convert_from_fxsr(&env, target);
 576
 577         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &env, 0, -1);
 578         if (!ret)
 579                 convert_to_fxsr(target, &env);
 580
 581         /*
 582          * update the header bit in the xsave header, indicating the
 583          * presence of FP.
 584          */
 585         if (cpu_has_xsave)
 586                 target->thread.fpu.state->xsave.xsave_hdr.xstate_bv |= XSTATE_FP;
 587         return ret;
 588 }
 589
 590 /*
 591  * Signal frame handlers.
 592  */
 593
 594 static inline int save_i387_fsave(struct _fpstate_ia32 __user *buf)
 595 {
 596         struct task_struct *tsk = current;
 597         struct i387_fsave_struct *fp = &tsk->thread.fpu.state->fsave;
 598
 599         fp->status = fp->swd;
 600         if (__copy_to_user(buf, fp, sizeof(struct i387_fsave_struct)))
 601                 return -1;
 602         return 1;
 603 }
 604
 605 static int save_i387_fxsave(struct _fpstate_ia32 __user *buf)
 606 {
 607         struct task_struct *tsk = current;
 608         struct i387_fxsave_struct *fx = &tsk->thread.fpu.state->fxsave;
 609         struct user_i387_ia32_struct env;
 610         int err = 0;
 611
 612         convert_from_fxsr(&env, tsk);
 613         if (__copy_to_user(buf, &env, sizeof(env)))
 614                 return -1;
 615
 616         err |= __put_user(fx->swd, &buf->status);
 617         err |= __put_user(X86_FXSR_MAGIC, &buf->magic);
 618         if (err)
 619                 return -1;
 620
 621         if (__copy_to_user(&buf->_fxsr_env[0], fx, xstate_size))
 622                 return -1;
 623         return 1;
 624 }
 625
 626 static int save_i387_xsave(void __user *buf)
 627 {
 628         struct task_struct *tsk = current;
 629         struct _fpstate_ia32 __user *fx = buf;
 630         int err = 0;
 631
 632
 633         sanitize_i387_state(tsk);
 634
 635         /*
 636          * For legacy compatible, we always set FP/SSE bits in the bit
 637          * vector while saving the state to the user context.
 638          * This will enable us capturing any changes(during sigreturn) to
 639          * the FP/SSE bits by the legacy applications which don't touch
 640          * xstate_bv in the xsave header.
 641          *
 642          * xsave aware applications can change the xstate_bv in the xsave
 643          * header as well as change any contents in the memory layout.
 644          * xrestore as part of sigreturn will capture all the changes.
 645          */
 646         tsk->thread.fpu.state->xsave.xsave_hdr.xstate_bv |= XSTATE_FPSSE;
 647
 648         if (save_i387_fxsave(fx) < 0)
 649                 return -1;
 650
 651         err = __copy_to_user(&fx->sw_reserved, &fx_sw_reserved_ia32,
 652                              sizeof(struct _fpx_sw_bytes));
 653         err |= __put_user(FP_XSTATE_MAGIC2,
 654                           (__u32 __user *) (buf + sig_xstate_ia32_size
 655                                             - FP_XSTATE_MAGIC2_SIZE));
 656         if (err)
 657                 return -1;
 658
 659         return 1;
 660 }
 661
 662 int save_i387_xstate_ia32(void __user *buf)
 663 {
 664         struct _fpstate_ia32 __user *fp = (struct _fpstate_ia32 __user *) buf;
 665         struct task_struct *tsk = current;
 666
 667         if (!used_math())
 668                 return 0;
 669
 670         if (!access_ok(VERIFY_WRITE, buf, sig_xstate_ia32_size))
 671                 return -EACCES;
 672         /*
 673          * This will cause a "finit" to be triggered by the next
 674          * attempted FPU operation by the 'current' process.
 675          */
 676         clear_used_math();
 677
 678         if (!HAVE_HWFP) {
 679                 return fpregs_soft_get(current, NULL,
 680                                        0, sizeof(struct user_i387_ia32_struct),
 681                                        NULL, fp) ? -1 : 1;
 682         }
 683
 684         unlazy_fpu(tsk);
 685
 686         if (cpu_has_xsave)
 687                 return save_i387_xsave(fp);
 688         if (cpu_has_fxsr)
 689                 return save_i387_fxsave(fp);
 690         else
 691                 return save_i387_fsave(fp);
 692 }
 693
 694 static inline int restore_i387_fsave(struct _fpstate_ia32 __user *buf)
 695 {
 696         struct task_struct *tsk = current;
 697
 698         return __copy_from_user(&tsk->thread.fpu.state->fsave, buf,
 699                                 sizeof(struct i387_fsave_struct));
 700 }
 701
 702 static int restore_i387_fxsave(struct _fpstate_ia32 __user *buf,
 703                                unsigned int size)
 704 {
 705         struct task_struct *tsk = current;
 706         struct user_i387_ia32_struct env;
 707         int err;
 708
 709         err = __copy_from_user(&tsk->thread.fpu.state->fxsave, &buf->_fxsr_env[0],
 710                                size);
 711         /* mxcsr reserved bits must be masked to zero for security reasons */
 712         tsk->thread.fpu.state->fxsave.mxcsr &= mxcsr_feature_mask;
 713         if (err || __copy_from_user(&env, buf, sizeof(env)))
 714                 return 1;
 715         convert_to_fxsr(tsk, &env);
 716
 717         return 0;
 718 }
 719
 720 static int restore_i387_xsave(void __user *buf)
 721 {
 722         struct _fpx_sw_bytes fx_sw_user;
 723         struct _fpstate_ia32 __user *fx_user =
 724                         ((struct _fpstate_ia32 __user *) buf);
 725         struct i387_fxsave_struct __user *fx =
 726                 (struct i387_fxsave_struct __user *) &fx_user->_fxsr_env[0];
 727         struct xsave_hdr_struct *xsave_hdr =
 728                                 &current->thread.fpu.state->xsave.xsave_hdr;
 729         u64 mask;
 730         int err;
 731
 732         if (check_for_xstate(fx, buf, &fx_sw_user))
 733                 goto fx_only;
 734
 735         mask = fx_sw_user.xstate_bv;
 736
 737         err = restore_i387_fxsave(buf, fx_sw_user.xstate_size);
 738
 739         xsave_hdr->xstate_bv &= pcntxt_mask;
 740         /*
 741          * These bits must be zero.
 742          */
 743         xsave_hdr->reserved1[0] = xsave_hdr->reserved1[1] = 0;
 744
 745         /*
 746          * Init the state that is not present in the memory layout
 747          * and enabled by the OS.
 748          */
 749         mask = ~(pcntxt_mask & ~mask);
 750         xsave_hdr->xstate_bv &= mask;
 751
 752         return err;
 753 fx_only:
 754         /*
 755          * Couldn't find the extended state information in the memory
 756          * layout. Restore the FP/SSE and init the other extended state
 757          * enabled by the OS.
 758          */
 759         xsave_hdr->xstate_bv = XSTATE_FPSSE;
 760         return restore_i387_fxsave(buf, sizeof(struct i387_fxsave_struct));
 761 }
 762
 763 int restore_i387_xstate_ia32(void __user *buf)
 764 {
 765         int err;
 766         struct task_struct *tsk = current;
 767         struct _fpstate_ia32 __user *fp = (struct _fpstate_ia32 __user *) buf;
 768
 769         if (HAVE_HWFP)
 770                 clear_fpu(tsk);
 771
 772         if (!buf) {
 773                 if (used_math()) {
 774                         clear_fpu(tsk);
 775                         clear_used_math();
 776                 }
 777
 778                 return 0;
 779         } else
 780                 if (!access_ok(VERIFY_READ, buf, sig_xstate_ia32_size))
 781                         return -EACCES;
 782
 783         if (!used_math()) {
 784                 err = init_fpu(tsk);
 785                 if (err)
 786                         return err;
 787         }
 788
 789         if (HAVE_HWFP) {
 790                 if (cpu_has_xsave)
 791                         err = restore_i387_xsave(buf);
 792                 else if (cpu_has_fxsr)
 793                         err = restore_i387_fxsave(fp, sizeof(struct
 794                                                            i387_fxsave_struct));
 795                 else
 796                         err = restore_i387_fsave(fp);
 797         } else {
 798                 err = fpregs_soft_set(current, NULL,
 799                                       0, sizeof(struct user_i387_ia32_struct),
 800                                       NULL, fp) != 0;
 801         }
 802         set_used_math();
 803
 804         return err;
 805 }
 806
 807 /*
 808  * FPU state for core dumps.
 809  * This is only used for a.out dumps now.
 810  * It is declared generically using elf_fpregset_t (which is
 811  * struct user_i387_struct) but is in fact only used for 32-bit
 812  * dumps, so on 64-bit it is really struct user_i387_ia32_struct.
 813  */
 814 int dump_fpu(struct pt_regs *regs, struct user_i387_struct *fpu)
 815 {
 816         struct task_struct *tsk = current;
 817         int fpvalid;
 818
 819         fpvalid = !!used_math();
 820         if (fpvalid)
 821                 fpvalid = !fpregs_get(tsk, NULL,
 822                                       0, sizeof(struct user_i387_ia32_struct),
 823                                       fpu, NULL);
 824
 825         return fpvalid;
 826 }
 827 EXPORT_SYMBOL(dump_fpu);
 828
 829 #endif  /* CONFIG_X86_32 || CONFIG_IA32_EMULATION */