arm64: mte: avoid TFSRE0_EL1 related operations unless in async mode

There is no reason to touch TFSRE0_EL1 nor issue a DSB unless our task
is in asynchronous mode. Since these operations (especially the DSB) may
be expensive on certain microarchitectures, only perform them if
necessary.

Furthermore, stop clearing TFSRE0_EL1 on entry because it will be
cleared on exit and it is not necessary to have any particular value in
TFSRE0_EL1 between entry and exit.

Signed-off-by: Peter Collingbourne <pcc@google.com>
Link: https://linux-review.googlesource.com/id/Ib353a63e3d0abc2b0b008e96aa2d9692cfc1b815
Link: https://lore.kernel.org/r/20210709023532.2133673-1-pcc@google.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>

diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index 863d44f..5cf1601 100644 (file)
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -133,29 +133,37 @@ alternative_cb_end
        .endm
 
        /* Check for MTE asynchronous tag check faults */
-       .macro check_mte_async_tcf, tmp, ti_flags
+       .macro check_mte_async_tcf, tmp, ti_flags, thread_sctlr
 #ifdef CONFIG_ARM64_MTE
        .arch_extension lse
 alternative_if_not ARM64_MTE
        b       1f
 alternative_else_nop_endif
+       /*
+        * Asynchronous tag check faults are only possible in ASYNC (2) or
+        * ASYM (3) modes. In each of these modes bit 1 of SCTLR_EL1.TCF0 is
+        * set, so skip the check if it is unset.
+        */
+       tbz     \thread_sctlr, #(SCTLR_EL1_TCF0_SHIFT + 1), 1f
        mrs_s   \tmp, SYS_TFSRE0_EL1
        tbz     \tmp, #SYS_TFSR_EL1_TF0_SHIFT, 1f
        /* Asynchronous TCF occurred for TTBR0 access, set the TI flag */
        mov     \tmp, #_TIF_MTE_ASYNC_FAULT
        add     \ti_flags, tsk, #TSK_TI_FLAGS
        stset   \tmp, [\ti_flags]
-       msr_s   SYS_TFSRE0_EL1, xzr
 1:
 #endif
        .endm
 
        /* Clear the MTE asynchronous tag check faults */
-       .macro clear_mte_async_tcf
+       .macro clear_mte_async_tcf thread_sctlr
 #ifdef CONFIG_ARM64_MTE
 alternative_if ARM64_MTE
+       /* See comment in check_mte_async_tcf above. */
+       tbz     \thread_sctlr, #(SCTLR_EL1_TCF0_SHIFT + 1), 1f
        dsb     ish
        msr_s   SYS_TFSRE0_EL1, xzr
+1:
 alternative_else_nop_endif
 #endif
        .endm
@@ -231,8 +239,8 @@ alternative_else_nop_endif
        disable_step_tsk x19, x20
 
        /* Check for asynchronous tag check faults in user space */
-       check_mte_async_tcf x22, x23
-       apply_ssbd 1, x22, x23
+       ldr     x0, [tsk, THREAD_SCTLR_USER]
+       check_mte_async_tcf x22, x23, x0
 
 #ifdef CONFIG_ARM64_PTR_AUTH
 alternative_if ARM64_HAS_ADDRESS_AUTH
@@ -245,7 +253,6 @@ alternative_if ARM64_HAS_ADDRESS_AUTH
         * was disabled on kernel exit then we would have left the kernel IA
         * installed so there is no need to install it again.
         */
-       ldr     x0, [tsk, THREAD_SCTLR_USER]
        tbz     x0, SCTLR_ELx_ENIA_SHIFT, 1f
        __ptrauth_keys_install_kernel_nosync tsk, x20, x22, x23
        b       2f
@@ -258,6 +265,8 @@ alternative_if ARM64_HAS_ADDRESS_AUTH
 alternative_else_nop_endif
 #endif
 
+       apply_ssbd 1, x22, x23
+
        mte_set_kernel_gcr x22, x23
 
        scs_load tsk
@@ -362,6 +371,10 @@ alternative_else_nop_endif
 3:
        scs_save tsk
 
+       /* Ignore asynchronous tag check faults in the uaccess routines */
+       ldr     x0, [tsk, THREAD_SCTLR_USER]
+       clear_mte_async_tcf x0
+
 #ifdef CONFIG_ARM64_PTR_AUTH
 alternative_if ARM64_HAS_ADDRESS_AUTH
        /*
@@ -371,7 +384,6 @@ alternative_if ARM64_HAS_ADDRESS_AUTH
         *
         * No kernel C function calls after this.
         */
-       ldr     x0, [tsk, THREAD_SCTLR_USER]
        tbz     x0, SCTLR_ELx_ENIA_SHIFT, 1f
        __ptrauth_keys_install_user tsk, x0, x1, x2
        b       2f
@@ -599,8 +611,6 @@ SYM_CODE_START_LOCAL(ret_to_user)
        cbnz    x2, work_pending
 finish_ret_to_user:
        user_enter_irqoff
-       /* Ignore asynchronous tag check faults in the uaccess routines */
-       clear_mte_async_tcf
        enable_step_tsk x19, x2
 #ifdef CONFIG_GCC_PLUGIN_STACKLEAK
        bl      stackleak_erase