x86_64, entry: Remove the syscall exit audit and schedule optimizations

We used to optimize rescheduling and audit on syscall exit.  Now
that the full slow path is reasonably fast, remove these
optimizations.  Syscall exit auditing is now handled exclusively by
syscall_trace_leave.

This adds something like 10ns to the previously optimized paths on
my computer, presumably due mostly to SAVE_REST / RESTORE_REST.

I think that we should eventually replace both the syscall and
non-paranoid interrupt exit slow paths with a pair of C functions
along the lines of the syscall entry hooks.

Link: http://lkml.kernel.org/r/22f2aa4a0361707a5cfb1de9d45260b39965dead.1421453410.git.luto@amacapital.net
Acked-by: Borislav Petkov <bp@suse.de>
Signed-off-by: Andy Lutomirski <luto@amacapital.net>

diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index eeab4cf..db13655 100644 (file)
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -361,15 +361,12 @@ system_call_fastpath:
  * Has incomplete stack frame and undefined top of stack.
  */
 ret_from_sys_call:
-       movl $_TIF_ALLWORK_MASK,%edi
-       /* edi: flagmask */
-sysret_check:
+       testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
+       jnz int_ret_from_sys_call_fixup /* Go the the slow path */
+
        LOCKDEP_SYS_EXIT
        DISABLE_INTERRUPTS(CLBR_NONE)
        TRACE_IRQS_OFF
-       movl TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET),%edx
-       andl %edi,%edx
-       jnz  sysret_careful
        CFI_REMEMBER_STATE
        /*
         * sysretq will re-enable interrupts:
@@ -383,49 +380,10 @@ sysret_check:
        USERGS_SYSRET64
 
        CFI_RESTORE_STATE
-       /* Handle reschedules */
-       /* edx: work, edi: workmask */
-sysret_careful:
-       bt $TIF_NEED_RESCHED,%edx
-       jnc sysret_signal
-       TRACE_IRQS_ON
-       ENABLE_INTERRUPTS(CLBR_NONE)
-       pushq_cfi %rdi
-       SCHEDULE_USER
-       popq_cfi %rdi
-       jmp sysret_check
 
-       /* Handle a signal */
-sysret_signal:
-       TRACE_IRQS_ON
-       ENABLE_INTERRUPTS(CLBR_NONE)
-#ifdef CONFIG_AUDITSYSCALL
-       bt $TIF_SYSCALL_AUDIT,%edx
-       jc sysret_audit
-#endif
-       /*
-        * We have a signal, or exit tracing or single-step.
-        * These all wind up with the iret return path anyway,
-        * so just join that path right now.
-        */
+int_ret_from_sys_call_fixup:
        FIXUP_TOP_OF_STACK %r11, -ARGOFFSET
-       jmp int_check_syscall_exit_work
-
-#ifdef CONFIG_AUDITSYSCALL
-       /*
-        * Return fast path for syscall audit.  Call __audit_syscall_exit()
-        * directly and then jump back to the fast path with TIF_SYSCALL_AUDIT
-        * masked off.
-        */
-sysret_audit:
-       movq RAX-ARGOFFSET(%rsp),%rsi   /* second arg, syscall return value */
-       cmpq $-MAX_ERRNO,%rsi   /* is it < -MAX_ERRNO? */
-       setbe %al               /* 1 if so, 0 if not */
-       movzbl %al,%edi         /* zero-extend that into %edi */
-       call __audit_syscall_exit
-       movl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),%edi
-       jmp sysret_check
-#endif /* CONFIG_AUDITSYSCALL */
+       jmp int_ret_from_sys_call
 
        /* Do syscall tracing */
 tracesys: