arm64: mte: Enable async tag check fault

[linux-2.6-microblaze.git] / arch / arm64 / kernel / mte.c

diff --git a/arch/arm64/kernel/mte.c b/arch/arm64/kernel/mte.c
index b3c70a6..b6336fb 100644 (file)
--- a/arch/arm64/kernel/mte.c
+++ b/arch/arm64/kernel/mte.c
@@ -26,6 +26,12 @@ u64 gcr_kernel_excl __ro_after_init;
 
 static bool report_fault_once = true;
 
+#ifdef CONFIG_KASAN_HW_TAGS
+/* Whether the MTE asynchronous mode is enabled. */
+DEFINE_STATIC_KEY_FALSE(mte_async_mode);
+EXPORT_SYMBOL_GPL(mte_async_mode);
+#endif
+
 static void mte_sync_page_tags(struct page *page, pte_t *ptep, bool check_swap)
 {
        pte_t old_pte = READ_ONCE(*ptep);
@@ -107,13 +113,45 @@ void mte_init_tags(u64 max_tag)
        write_sysreg_s(SYS_GCR_EL1_RRND | gcr_kernel_excl, SYS_GCR_EL1);
 }
 
-void mte_enable_kernel(void)
+static inline void __mte_enable_kernel(const char *mode, unsigned long tcf)
 {
        /* Enable MTE Sync Mode for EL1. */
-       sysreg_clear_set(sctlr_el1, SCTLR_ELx_TCF_MASK, SCTLR_ELx_TCF_SYNC);
+       sysreg_clear_set(sctlr_el1, SCTLR_ELx_TCF_MASK, tcf);
        isb();
+
+       pr_info_once("MTE: enabled in %s mode at EL1\n", mode);
+}
+
+#ifdef CONFIG_KASAN_HW_TAGS
+void mte_enable_kernel_sync(void)
+{
+       /*
+        * Make sure we enter this function when no PE has set
+        * async mode previously.
+        */
+       WARN_ONCE(system_uses_mte_async_mode(),
+                       "MTE async mode enabled system wide!");
+
+       __mte_enable_kernel("synchronous", SCTLR_ELx_TCF_SYNC);
 }
 
+void mte_enable_kernel_async(void)
+{
+       __mte_enable_kernel("asynchronous", SCTLR_ELx_TCF_ASYNC);
+
+       /*
+        * MTE async mode is set system wide by the first PE that
+        * executes this function.
+        *
+        * Note: If in future KASAN acquires a runtime switching
+        * mode in between sync and async, this strategy needs
+        * to be reviewed.
+        */
+       if (!system_uses_mte_async_mode())
+               static_branch_enable(&mte_async_mode);
+}
+#endif
+
 void mte_set_report_once(bool state)
 {
        WRITE_ONCE(report_fault_once, state);
@@ -124,6 +162,29 @@ bool mte_report_once(void)
        return READ_ONCE(report_fault_once);
 }
 
+#ifdef CONFIG_KASAN_HW_TAGS
+void mte_check_tfsr_el1(void)
+{
+       u64 tfsr_el1;
+
+       if (!system_supports_mte())
+               return;
+
+       tfsr_el1 = read_sysreg_s(SYS_TFSR_EL1);
+
+       if (unlikely(tfsr_el1 & SYS_TFSR_EL1_TF1)) {
+               /*
+                * Note: isb() is not required after this direct write
+                * because there is no indirect read subsequent to it
+                * (per ARM DDI 0487F.c table D13-1).
+                */
+               write_sysreg_s(0, SYS_TFSR_EL1);
+
+               kasan_report_async();
+       }
+}
+#endif
+
 static void update_sctlr_el1_tcf0(u64 tcf0)
 {
        /* ISB required for the kernel uaccess routines */
@@ -189,6 +250,19 @@ void mte_thread_switch(struct task_struct *next)
        /* avoid expensive SCTLR_EL1 accesses if no change */
        if (current->thread.sctlr_tcf0 != next->thread.sctlr_tcf0)
                update_sctlr_el1_tcf0(next->thread.sctlr_tcf0);
+       else
+               isb();
+
+       /*
+        * Check if an async tag exception occurred at EL1.
+        *
+        * Note: On the context switch path we rely on the dsb() present
+        * in __switch_to() to guarantee that the indirect writes to TFSR_EL1
+        * are synchronized before this point.
+        * isb() above is required for the same reason.
+        *
+        */
+       mte_check_tfsr_el1();
 }
 
 void mte_suspend_exit(void)