arm64: mte: Enable async tag check fault

[linux-2.6-microblaze.git] / arch / arm64 / kernel / mte.c

diff --git a/arch/arm64/kernel/mte.c b/arch/arm64/kernel/mte.c
index 80b62fe..b6336fb 100644 (file)
--- a/arch/arm64/kernel/mte.c
+++ b/arch/arm64/kernel/mte.c
@@ -19,12 +19,19 @@
 #include <asm/barrier.h>
 #include <asm/cpufeature.h>
 #include <asm/mte.h>
-#include <asm/mte-kasan.h>
 #include <asm/ptrace.h>
 #include <asm/sysreg.h>
 
 u64 gcr_kernel_excl __ro_after_init;
 
+static bool report_fault_once = true;
+
+#ifdef CONFIG_KASAN_HW_TAGS
+/* Whether the MTE asynchronous mode is enabled. */
+DEFINE_STATIC_KEY_FALSE(mte_async_mode);
+EXPORT_SYMBOL_GPL(mte_async_mode);
+#endif
+
 static void mte_sync_page_tags(struct page *page, pte_t *ptep, bool check_swap)
 {
        pte_t old_pte = READ_ONCE(*ptep);
@@ -86,51 +93,6 @@ int memcmp_pages(struct page *page1, struct page *page2)
        return ret;
 }
 
-u8 mte_get_mem_tag(void *addr)
-{
-       if (!system_supports_mte())
-               return 0xFF;
-
-       asm(__MTE_PREAMBLE "ldg %0, [%0]"
-           : "+r" (addr));
-
-       return mte_get_ptr_tag(addr);
-}
-
-u8 mte_get_random_tag(void)
-{
-       void *addr;
-
-       if (!system_supports_mte())
-               return 0xFF;
-
-       asm(__MTE_PREAMBLE "irg %0, %0"
-           : "+r" (addr));
-
-       return mte_get_ptr_tag(addr);
-}
-
-void *mte_set_mem_tag_range(void *addr, size_t size, u8 tag)
-{
-       void *ptr = addr;
-
-       if ((!system_supports_mte()) || (size == 0))
-               return addr;
-
-       /* Make sure that size is MTE granule aligned. */
-       WARN_ON(size & (MTE_GRANULE_SIZE - 1));
-
-       /* Make sure that the address is MTE granule aligned. */
-       WARN_ON((u64)addr & (MTE_GRANULE_SIZE - 1));
-
-       tag = 0xF0 | tag;
-       ptr = (void *)__tag_set(ptr, tag);
-
-       mte_assign_mem_tag_range(ptr, size);
-
-       return ptr;
-}
-
 void mte_init_tags(u64 max_tag)
 {
        static bool gcr_kernel_excl_initialized;
@@ -151,12 +113,77 @@ void mte_init_tags(u64 max_tag)
        write_sysreg_s(SYS_GCR_EL1_RRND | gcr_kernel_excl, SYS_GCR_EL1);
 }
 
-void mte_enable_kernel(void)
+static inline void __mte_enable_kernel(const char *mode, unsigned long tcf)
 {
        /* Enable MTE Sync Mode for EL1. */
-       sysreg_clear_set(sctlr_el1, SCTLR_ELx_TCF_MASK, SCTLR_ELx_TCF_SYNC);
+       sysreg_clear_set(sctlr_el1, SCTLR_ELx_TCF_MASK, tcf);
        isb();
+
+       pr_info_once("MTE: enabled in %s mode at EL1\n", mode);
+}
+
+#ifdef CONFIG_KASAN_HW_TAGS
+void mte_enable_kernel_sync(void)
+{
+       /*
+        * Make sure we enter this function when no PE has set
+        * async mode previously.
+        */
+       WARN_ONCE(system_uses_mte_async_mode(),
+                       "MTE async mode enabled system wide!");
+
+       __mte_enable_kernel("synchronous", SCTLR_ELx_TCF_SYNC);
+}
+
+void mte_enable_kernel_async(void)
+{
+       __mte_enable_kernel("asynchronous", SCTLR_ELx_TCF_ASYNC);
+
+       /*
+        * MTE async mode is set system wide by the first PE that
+        * executes this function.
+        *
+        * Note: If in future KASAN acquires a runtime switching
+        * mode in between sync and async, this strategy needs
+        * to be reviewed.
+        */
+       if (!system_uses_mte_async_mode())
+               static_branch_enable(&mte_async_mode);
 }
+#endif
+
+void mte_set_report_once(bool state)
+{
+       WRITE_ONCE(report_fault_once, state);
+}
+
+bool mte_report_once(void)
+{
+       return READ_ONCE(report_fault_once);
+}
+
+#ifdef CONFIG_KASAN_HW_TAGS
+void mte_check_tfsr_el1(void)
+{
+       u64 tfsr_el1;
+
+       if (!system_supports_mte())
+               return;
+
+       tfsr_el1 = read_sysreg_s(SYS_TFSR_EL1);
+
+       if (unlikely(tfsr_el1 & SYS_TFSR_EL1_TF1)) {
+               /*
+                * Note: isb() is not required after this direct write
+                * because there is no indirect read subsequent to it
+                * (per ARM DDI 0487F.c table D13-1).
+                */
+               write_sysreg_s(0, SYS_TFSR_EL1);
+
+               kasan_report_async();
+       }
+}
+#endif
 
 static void update_sctlr_el1_tcf0(u64 tcf0)
 {
@@ -223,6 +250,19 @@ void mte_thread_switch(struct task_struct *next)
        /* avoid expensive SCTLR_EL1 accesses if no change */
        if (current->thread.sctlr_tcf0 != next->thread.sctlr_tcf0)
                update_sctlr_el1_tcf0(next->thread.sctlr_tcf0);
+       else
+               isb();
+
+       /*
+        * Check if an async tag exception occurred at EL1.
+        *
+        * Note: On the context switch path we rely on the dsb() present
+        * in __switch_to() to guarantee that the indirect writes to TFSR_EL1
+        * are synchronized before this point.
+        * isb() above is required for the same reason.
+        *
+        */
+       mte_check_tfsr_el1();
 }
 
 void mte_suspend_exit(void)