arm64: Move handling of erratum 1418040 into C code

Instead of dealing with erratum 1418040 on each entry and exit,
let's move the handling to __switch_to() instead, which has
several advantages:

- It can be applied when it matters (switching between 32 and 64
  bit tasks).
- It is written in C (yay!)
- It can rely on static keys rather than alternatives

Signed-off-by: Marc Zyngier <maz@kernel.org>
Tested-by: Sai Prakash Ranjan <saiprakash.ranjan@codeaurora.org>
Reviewed-by: Stephen Boyd <swboyd@chromium.org>
Acked-by: Will Deacon <will@kernel.org>
Link: https://lore.kernel.org/r/20200731173824.107480-2-maz@kernel.org
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>

diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index 2646178..55af8b5 100644 (file)
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -170,19 +170,6 @@ alternative_cb_end
        stp     x28, x29, [sp, #16 * 14]
 
        .if     \el == 0
-       .if     \regsize == 32
-       /*
-        * If we're returning from a 32-bit task on a system affected by
-        * 1418040 then re-enable userspace access to the virtual counter.
-        */
-#ifdef CONFIG_ARM64_ERRATUM_1418040
-alternative_if ARM64_WORKAROUND_1418040
-       mrs     x0, cntkctl_el1
-       orr     x0, x0, #2      // ARCH_TIMER_USR_VCT_ACCESS_EN
-       msr     cntkctl_el1, x0
-alternative_else_nop_endif
-#endif
-       .endif
        clear_gp_regs
        mrs     x21, sp_el0
        ldr_this_cpu    tsk, __entry_task, x20
@@ -294,14 +281,6 @@ alternative_else_nop_endif
        tst     x22, #PSR_MODE32_BIT            // native task?
        b.eq    3f
 
-#ifdef CONFIG_ARM64_ERRATUM_1418040
-alternative_if ARM64_WORKAROUND_1418040
-       mrs     x0, cntkctl_el1
-       bic     x0, x0, #2                      // ARCH_TIMER_USR_VCT_ACCESS_EN
-       msr     cntkctl_el1, x0
-alternative_else_nop_endif
-#endif
-
 #ifdef CONFIG_ARM64_ERRATUM_845719
 alternative_if ARM64_WORKAROUND_845719
 #ifdef CONFIG_PID_IN_CONTEXTIDR

diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c
index 84ec630..b63ce4c 100644 (file)
--- a/arch/arm64/kernel/process.c
+++ b/arch/arm64/kernel/process.c
@@ -515,6 +515,39 @@ static void entry_task_switch(struct task_struct *next)
        __this_cpu_write(__entry_task, next);
 }
 
+/*
+ * ARM erratum 1418040 handling, affecting the 32bit view of CNTVCT.
+ * Assuming the virtual counter is enabled at the beginning of times:
+ *
+ * - disable access when switching from a 64bit task to a 32bit task
+ * - enable access when switching from a 32bit task to a 64bit task
+ */
+static void erratum_1418040_thread_switch(struct task_struct *prev,
+                                         struct task_struct *next)
+{
+       bool prev32, next32;
+       u64 val;
+
+       if (!(IS_ENABLED(CONFIG_ARM64_ERRATUM_1418040) &&
+             cpus_have_const_cap(ARM64_WORKAROUND_1418040)))
+               return;
+
+       prev32 = is_compat_thread(task_thread_info(prev));
+       next32 = is_compat_thread(task_thread_info(next));
+
+       if (prev32 == next32)
+               return;
+
+       val = read_sysreg(cntkctl_el1);
+
+       if (!next32)
+               val |= ARCH_TIMER_USR_VCT_ACCESS_EN;
+       else
+               val &= ~ARCH_TIMER_USR_VCT_ACCESS_EN;
+
+       write_sysreg(val, cntkctl_el1);
+}
+
 /*
  * Thread switching.
  */
@@ -530,6 +563,7 @@ __notrace_funcgraph struct task_struct *__switch_to(struct task_struct *prev,
        entry_task_switch(next);
        uao_thread_switch(next);
        ssbs_thread_switch(next);
+       erratum_1418040_thread_switch(prev, next);
 
        /*
         * Complete any pending TLB or cache maintenance on this CPU in case