x86/fpu/signal: Split out the direct restore code

Prepare for smarter failure handling of the direct restore.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Borislav Petkov <bp@suse.de>
Link: https://lkml.kernel.org/r/20210623121457.493455414@linutronix.de

diff --git a/arch/x86/kernel/fpu/signal.c b/arch/x86/kernel/fpu/signal.c
index a1a7013..aa268d9 100644 (file)
--- a/arch/x86/kernel/fpu/signal.c
+++ b/arch/x86/kernel/fpu/signal.c
@@ -250,10 +250,8 @@ sanitize_restored_user_xstate(union fpregs_state *state,
        }
 }
 
-/*
- * Restore the FPU state directly from the userspace signal frame.
- */
-static int restore_fpregs_from_user(void __user *buf, u64 xrestore, bool fx_only)
+static int __restore_fpregs_from_user(void __user *buf, u64 xrestore,
+                                     bool fx_only)
 {
        if (use_xsave()) {
                u64 init_bv = xfeatures_mask_uabi() & ~xrestore;
@@ -274,6 +272,57 @@ static int restore_fpregs_from_user(void __user *buf, u64 xrestore, bool fx_only
        }
 }
 
+static int restore_fpregs_from_user(void __user *buf, u64 xrestore, bool fx_only)
+{
+       struct fpu *fpu = &current->thread.fpu;
+       int ret;
+
+       fpregs_lock();
+       pagefault_disable();
+       ret = __restore_fpregs_from_user(buf, xrestore, fx_only);
+       pagefault_enable();
+
+       if (unlikely(ret)) {
+               /*
+                * The above did an FPU restore operation, restricted to
+                * the user portion of the registers, and failed, but the
+                * microcode might have modified the FPU registers
+                * nevertheless.
+                *
+                * If the FPU registers do not belong to current, then
+                * invalidate the FPU register state otherwise the task
+                * might preempt current and return to user space with
+                * corrupted FPU registers.
+                *
+                * In case current owns the FPU registers then no further
+                * action is required. The fixup in the slow path will
+                * handle it correctly.
+                */
+               if (test_thread_flag(TIF_NEED_FPU_LOAD))
+                       __cpu_invalidate_fpregs_state();
+               fpregs_unlock();
+               return ret;
+       }
+
+       /*
+        * Restore supervisor states: previous context switch etc has done
+        * XSAVES and saved the supervisor states in the kernel buffer from
+        * which they can be restored now.
+        *
+        * It would be optimal to handle this with a single XRSTORS, but
+        * this does not work because the rest of the FPU registers have
+        * been restored from a user buffer directly. The single XRSTORS
+        * happens below, when the user buffer has been copied to the
+        * kernel one.
+        */
+       if (test_thread_flag(TIF_NEED_FPU_LOAD) && xfeatures_mask_supervisor())
+               os_xrstor(&fpu->state.xsave, xfeatures_mask_supervisor());
+
+       fpregs_mark_activate();
+       fpregs_unlock();
+       return 0;
+}
+
 static int __fpu_restore_sig(void __user *buf, void __user *buf_fx,
                             bool ia32_fxstate)
 {
@@ -298,61 +347,16 @@ static int __fpu_restore_sig(void __user *buf, void __user *buf_fx,
                user_xfeatures = fx_sw_user.xfeatures;
        }
 
-       if (!ia32_fxstate) {
+       if (likely(!ia32_fxstate)) {
                /*
                 * Attempt to restore the FPU registers directly from user
-                * memory. For that to succeed, the user access cannot cause
-                * page faults. If it does, fall back to the slow path below,
-                * going through the kernel buffer with the enabled pagefault
-                * handler.
+                * memory. For that to succeed, the user access cannot cause page
+                * faults. If it does, fall back to the slow path below, going
+                * through the kernel buffer with the enabled pagefault handler.
                 */
-               fpregs_lock();
-               pagefault_disable();
                ret = restore_fpregs_from_user(buf_fx, user_xfeatures, fx_only);
-               pagefault_enable();
-               if (!ret) {
-
-                       /*
-                        * Restore supervisor states: previous context switch
-                        * etc has done XSAVES and saved the supervisor states
-                        * in the kernel buffer from which they can be restored
-                        * now.
-                        *
-                        * We cannot do a single XRSTORS here - which would
-                        * be nice - because the rest of the FPU registers are
-                        * being restored from a user buffer directly. The
-                        * single XRSTORS happens below, when the user buffer
-                        * has been copied to the kernel one.
-                        */
-                       if (test_thread_flag(TIF_NEED_FPU_LOAD) &&
-                           xfeatures_mask_supervisor()) {
-                               os_xrstor(&fpu->state.xsave,
-                                         xfeatures_mask_supervisor());
-                       }
-                       fpregs_mark_activate();
-                       fpregs_unlock();
+               if (likely(!ret))
                        return 0;
-               }
-
-               /*
-                * The above did an FPU restore operation, restricted to
-                * the user portion of the registers, and failed, but the
-                * microcode might have modified the FPU registers
-                * nevertheless.
-                *
-                * If the FPU registers do not belong to current, then
-                * invalidate the FPU register state otherwise the task might
-                * preempt current and return to user space with corrupted
-                * FPU registers.
-                *
-                * In case current owns the FPU registers then no further
-                * action is required. The fixup below will handle it
-                * correctly.
-                */
-               if (test_thread_flag(TIF_NEED_FPU_LOAD))
-                       __cpu_invalidate_fpregs_state();
-
-               fpregs_unlock();
        } else {
                /*
                 * For 32-bit frames with fxstate, copy the fxstate so it can