ARM: 9358/2: Implement PAN for LPAE by TTBR0 page table walks disablement

With LPAE enabled, privileged no-access cannot be enforced using CPU
domains as such feature is not available. This patch implements PAN
by disabling TTBR0 page table walks while in kernel mode.

The ARM architecture allows page table walks to be split between TTBR0
and TTBR1. With LPAE enabled, the split is defined by a combination of
TTBCR T0SZ and T1SZ bits. Currently, an LPAE-enabled kernel uses TTBR0
for user addresses and TTBR1 for kernel addresses with the VMSPLIT_2G
and VMSPLIT_3G configurations. The main advantage for the 3:1 split is
that TTBR1 is reduced to 2 levels, so potentially faster TLB refill
(though usually the first level entries are already cached in the TLB).

The PAN support on LPAE-enabled kernels uses TTBR0 when running in user
space or in kernel space during user access routines (TTBCR T0SZ and
T1SZ are both 0). When running user accesses are disabled in kernel
mode, TTBR0 page table walks are disabled by setting TTBCR.EPD0. TTBR1
is used for kernel accesses (including loadable modules; anything
covered by swapper_pg_dir) by reducing the TTBCR.T0SZ to the minimum
(2^(32-7) = 32MB). To avoid user accesses potentially hitting stale TLB
entries, the ASID is switched to 0 (reserved) by setting TTBCR.A1 and
using the ASID value in TTBR1. The difference from a non-PAN kernel is
that with the 3:1 memory split, TTBR1 always uses 3 levels of page
tables.

As part of the change we are using preprocessor elif definied() clauses
so balance these clauses by converting relevant precedingt ifdef
clauses to if defined() clauses.

Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
Reviewed-by: Kees Cook <keescook@chromium.org>
Tested-by: Florian Fainelli <florian.fainelli@broadcom.com>
Signed-off-by: Linus Walleij <linus.walleij@linaro.org>
Signed-off-by: Russell King (Oracle) <rmk+kernel@armlinux.org.uk>

diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig
index b14aed3..1394a84 100644 (file)
--- a/arch/arm/Kconfig
+++ b/arch/arm/Kconfig
@@ -1233,9 +1233,9 @@ config HIGHPTE
          consumed by page tables.  Setting this option will allow
          user-space 2nd level page tables to reside in high memory.
 
-config CPU_SW_DOMAIN_PAN
-       bool "Enable use of CPU domains to implement privileged no-access"
-       depends on MMU && !ARM_LPAE
+config ARM_PAN
+       bool "Enable privileged no-access"
+       depends on MMU
        default y
        help
          Increase kernel security by ensuring that normal kernel accesses
@@ -1244,10 +1244,26 @@ config CPU_SW_DOMAIN_PAN
          by ensuring that magic values (such as LIST_POISON) will always
          fault when dereferenced.
 
+         The implementation uses CPU domains when !CONFIG_ARM_LPAE and
+         disabling of TTBR0 page table walks with CONFIG_ARM_LPAE.
+
+config CPU_SW_DOMAIN_PAN
+       def_bool y
+       depends on ARM_PAN && !ARM_LPAE
+       help
+         Enable use of CPU domains to implement privileged no-access.
+
          CPUs with low-vector mappings use a best-efforts implementation.
          Their lower 1MB needs to remain accessible for the vectors, but
          the remainder of userspace will become appropriately inaccessible.
 
+config CPU_TTBR0_PAN
+       def_bool y
+       depends on ARM_PAN && ARM_LPAE
+       help
+         Enable privileged no-access by disabling TTBR0 page table walks when
+         running in kernel mode.
+
 config HW_PERF_EVENTS
        def_bool y
        depends on ARM_PMU

diff --git a/arch/arm/include/asm/assembler.h b/arch/arm/include/asm/assembler.h
index aebe2c8..d33c1e2 100644 (file)
--- a/arch/arm/include/asm/assembler.h
+++ b/arch/arm/include/asm/assembler.h
@@ -21,6 +21,7 @@
 #include <asm/opcodes-virt.h>
 #include <asm/asm-offsets.h>
 #include <asm/page.h>
+#include <asm/pgtable.h>
 #include <asm/thread_info.h>
 #include <asm/uaccess-asm.h>
 

diff --git a/arch/arm/include/asm/pgtable-3level-hwdef.h b/arch/arm/include/asm/pgtable-3level-hwdef.h
index 19da775..323ad81 100644 (file)
--- a/arch/arm/include/asm/pgtable-3level-hwdef.h
+++ b/arch/arm/include/asm/pgtable-3level-hwdef.h
@@ -74,6 +74,7 @@
 #define PHYS_MASK_SHIFT                (40)
 #define PHYS_MASK              ((1ULL << PHYS_MASK_SHIFT) - 1)
 
+#ifndef CONFIG_CPU_TTBR0_PAN
 /*
  * TTBR0/TTBR1 split (PAGE_OFFSET):
  *   0x40000000: T0SZ = 2, T1SZ = 0 (not used)
@@ -93,6 +94,14 @@
 #endif
 
 #define TTBR1_SIZE     (((PAGE_OFFSET >> 30) - 1) << 16)
+#else
+/*
+ * With CONFIG_CPU_TTBR0_PAN enabled, TTBR1 is only used during uaccess
+ * disabled regions when TTBR0 is disabled.
+ */
+#define TTBR1_OFFSET   0                       /* pointing to swapper_pg_dir */
+#define TTBR1_SIZE     0                       /* TTBR1 size controlled via TTBCR.T0SZ */
+#endif
 
 /*
  * TTBCR register bits.

diff --git a/arch/arm/include/asm/ptrace.h b/arch/arm/include/asm/ptrace.h
index 14a38cc..6eb311f 100644 (file)
--- a/arch/arm/include/asm/ptrace.h
+++ b/arch/arm/include/asm/ptrace.h
@@ -20,6 +20,7 @@ struct pt_regs {
 struct svc_pt_regs {
        struct pt_regs regs;
        u32 dacr;
+       u32 ttbcr;
 };
 
 #define to_svc_pt_regs(r) container_of(r, struct svc_pt_regs, regs)

diff --git a/arch/arm/include/asm/uaccess-asm.h b/arch/arm/include/asm/uaccess-asm.h
index ea42ba2..4bccd89 100644 (file)
--- a/arch/arm/include/asm/uaccess-asm.h
+++ b/arch/arm/include/asm/uaccess-asm.h
@@ -39,7 +39,7 @@
 #endif
        .endm
 
-#ifdef CONFIG_CPU_SW_DOMAIN_PAN
+#if defined(CONFIG_CPU_SW_DOMAIN_PAN)
 
        .macro  uaccess_disable, tmp, isb=1
        /*
@@ -65,6 +65,37 @@
        .endif
        .endm
 
+#elif defined(CONFIG_CPU_TTBR0_PAN)
+
+       .macro  uaccess_disable, tmp, isb=1
+       /*
+        * Disable TTBR0 page table walks (EDP0 = 1), use the reserved ASID
+        * from TTBR1 (A1 = 1) and enable TTBR1 page table walks for kernel
+        * addresses by reducing TTBR0 range to 32MB (T0SZ = 7).
+        */
+       mrc     p15, 0, \tmp, c2, c0, 2         @ read TTBCR
+       orr     \tmp, \tmp, #TTBCR_EPD0 | TTBCR_T0SZ_MASK
+       orr     \tmp, \tmp, #TTBCR_A1
+       mcr     p15, 0, \tmp, c2, c0, 2         @ write TTBCR
+       .if     \isb
+       instr_sync
+       .endif
+       .endm
+
+       .macro  uaccess_enable, tmp, isb=1
+       /*
+        * Enable TTBR0 page table walks (T0SZ = 0, EDP0 = 0) and ASID from
+        * TTBR0 (A1 = 0).
+        */
+       mrc     p15, 0, \tmp, c2, c0, 2         @ read TTBCR
+       bic     \tmp, \tmp, #TTBCR_EPD0 | TTBCR_T0SZ_MASK
+       bic     \tmp, \tmp, #TTBCR_A1
+       mcr     p15, 0, \tmp, c2, c0, 2         @ write TTBCR
+       .if     \isb
+       instr_sync
+       .endif
+       .endm
+
 #else
 
        .macro  uaccess_disable, tmp, isb=1
@@ -79,6 +110,12 @@
 #define DACR(x...)     x
 #else
 #define DACR(x...)
+#endif
+
+#ifdef CONFIG_CPU_TTBR0_PAN
+#define PAN(x...)      x
+#else
+#define PAN(x...)
 #endif
 
        /*
@@ -94,6 +131,8 @@
        .macro  uaccess_entry, tsk, tmp0, tmp1, tmp2, disable
  DACR( mrc     p15, 0, \tmp0, c3, c0, 0)
  DACR( str     \tmp0, [sp, #SVC_DACR])
+ PAN(  mrc     p15, 0, \tmp0, c2, c0, 2)
+ PAN(  str     \tmp0, [sp, #SVC_TTBCR])
        .if \disable && IS_ENABLED(CONFIG_CPU_SW_DOMAIN_PAN)
        /* kernel=client, user=no access */
        mov     \tmp2, #DACR_UACCESS_DISABLE
@@ -112,8 +151,11 @@
        .macro  uaccess_exit, tsk, tmp0, tmp1
  DACR( ldr     \tmp0, [sp, #SVC_DACR])
  DACR( mcr     p15, 0, \tmp0, c3, c0, 0)
+ PAN(  ldr     \tmp0, [sp, #SVC_TTBCR])
+ PAN(  mcr     p15, 0, \tmp0, c2, c0, 2)
        .endm
 
 #undef DACR
+#undef PAN
 
 #endif /* __ASM_UACCESS_ASM_H__ */

diff --git a/arch/arm/include/asm/uaccess.h b/arch/arm/include/asm/uaccess.h
index 2278769..25d21d7 100644 (file)
--- a/arch/arm/include/asm/uaccess.h
+++ b/arch/arm/include/asm/uaccess.h
@@ -14,6 +14,8 @@
 #include <asm/domain.h>
 #include <asm/unaligned.h>
 #include <asm/unified.h>
+#include <asm/pgtable.h>
+#include <asm/proc-fns.h>
 #include <asm/compiler.h>
 
 #include <asm/extable.h>
@@ -24,7 +26,7 @@
  * perform such accesses (eg, via list poison values) which could then
  * be exploited for priviledge escalation.
  */
-#ifdef CONFIG_CPU_SW_DOMAIN_PAN
+#if defined(CONFIG_CPU_SW_DOMAIN_PAN)
 
 static __always_inline unsigned int uaccess_save_and_enable(void)
 {
@@ -43,6 +45,28 @@ static __always_inline void uaccess_restore(unsigned int flags)
        set_domain(flags);
 }
 
+#elif defined(CONFIG_CPU_TTBR0_PAN)
+
+static inline unsigned int uaccess_save_and_enable(void)
+{
+       unsigned int old_ttbcr = cpu_get_ttbcr();
+
+       /*
+        * Enable TTBR0 page table walks (T0SZ = 0, EDP0 = 0) and ASID from
+        * TTBR0 (A1 = 0).
+        */
+       cpu_set_ttbcr(old_ttbcr & ~(TTBCR_A1 | TTBCR_EPD0 | TTBCR_T0SZ_MASK));
+       isb();
+
+       return old_ttbcr;
+}
+
+static inline void uaccess_restore(unsigned int flags)
+{
+       cpu_set_ttbcr(flags);
+       isb();
+}
+
 #else
 
 static inline unsigned int uaccess_save_and_enable(void)

diff --git a/arch/arm/kernel/asm-offsets.c b/arch/arm/kernel/asm-offsets.c
index 4915662..4853875 100644 (file)
--- a/arch/arm/kernel/asm-offsets.c
+++ b/arch/arm/kernel/asm-offsets.c
@@ -85,6 +85,7 @@ int main(void)
   DEFINE(S_OLD_R0,             offsetof(struct pt_regs, ARM_ORIG_r0));
   DEFINE(PT_REGS_SIZE,         sizeof(struct pt_regs));
   DEFINE(SVC_DACR,             offsetof(struct svc_pt_regs, dacr));
+  DEFINE(SVC_TTBCR,            offsetof(struct svc_pt_regs, ttbcr));
   DEFINE(SVC_REGS_SIZE,                sizeof(struct svc_pt_regs));
   BLANK();
   DEFINE(SIGFRAME_RC3_OFFSET,  offsetof(struct sigframe, retcode[3]));

diff --git a/arch/arm/kernel/suspend.c b/arch/arm/kernel/suspend.c
index c3ec386..58a6441 100644 (file)
--- a/arch/arm/kernel/suspend.c
+++ b/arch/arm/kernel/suspend.c
@@ -12,6 +12,7 @@
 #include <asm/smp_plat.h>
 #include <asm/suspend.h>
 #include <asm/tlbflush.h>
+#include <asm/uaccess.h>
 
 extern int __cpu_suspend(unsigned long, int (*)(unsigned long), u32 cpuid);
 extern void cpu_resume_mmu(void);
@@ -26,6 +27,13 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long))
        if (!idmap_pgd)
                return -EINVAL;
 
+       /*
+        * Needed for the MMU disabling/enabing code to be able to run from
+        * TTBR0 addresses.
+        */
+       if (IS_ENABLED(CONFIG_CPU_TTBR0_PAN))
+               uaccess_save_and_enable();
+
        /*
         * Function graph tracer state gets incosistent when the kernel
         * calls functions that never return (aka suspend finishers) hence

diff --git a/arch/arm/lib/csumpartialcopyuser.S b/arch/arm/lib/csumpartialcopyuser.S
index 6928781..c289bde 100644 (file)
--- a/arch/arm/lib/csumpartialcopyuser.S
+++ b/arch/arm/lib/csumpartialcopyuser.S
@@ -13,7 +13,8 @@
 
                .text
 
-#ifdef CONFIG_CPU_SW_DOMAIN_PAN
+#if defined(CONFIG_CPU_SW_DOMAIN_PAN)
+
                .macro  save_regs
                mrc     p15, 0, ip, c3, c0, 0
                stmfd   sp!, {r1, r2, r4 - r8, ip, lr}
@@ -25,7 +26,23 @@
                mcr     p15, 0, ip, c3, c0, 0
                ret     lr
                .endm
+
+#elif defined(CONFIG_CPU_TTBR0_PAN)
+
+               .macro  save_regs
+               mrc     p15, 0, ip, c2, c0, 2           @ read TTBCR
+               stmfd   sp!, {r1, r2, r4 - r8, ip, lr}
+               uaccess_enable ip
+               .endm
+
+               .macro  load_regs
+               ldmfd   sp!, {r1, r2, r4 - r8, ip, lr}
+               mcr     p15, 0, ip, c2, c0, 2           @ restore TTBCR
+               ret     lr
+               .endm
+
 #else
+
                .macro  save_regs
                stmfd   sp!, {r1, r2, r4 - r8, lr}
                .endm
@@ -33,6 +50,7 @@
                .macro  load_regs
                ldmfd   sp!, {r1, r2, r4 - r8, pc}
                .endm
+
 #endif
 
                .macro  load1b, reg1

diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c
index 439dc6a..dfa9554 100644 (file)
--- a/arch/arm/mm/fault.c
+++ b/arch/arm/mm/fault.c
@@ -242,6 +242,27 @@ static inline bool is_permission_fault(unsigned int fsr)
        return false;
 }
 
+#ifdef CONFIG_CPU_TTBR0_PAN
+static inline bool ttbr0_usermode_access_allowed(struct pt_regs *regs)
+{
+       struct svc_pt_regs *svcregs;
+
+       /* If we are in user mode: permission granted */
+       if (user_mode(regs))
+               return true;
+
+       /* uaccess state saved above pt_regs on SVC exception entry */
+       svcregs = to_svc_pt_regs(regs);
+
+       return !(svcregs->ttbcr & TTBCR_EPD0);
+}
+#else
+static inline bool ttbr0_usermode_access_allowed(struct pt_regs *regs)
+{
+       return true;
+}
+#endif
+
 static int __kprobes
 do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
 {
@@ -285,6 +306,14 @@ do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
 
        perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr);
 
+       /*
+        * Privileged access aborts with CONFIG_CPU_TTBR0_PAN enabled are
+        * routed via the translation fault mechanism. Check whether uaccess
+        * is disabled while in kernel mode.
+        */
+       if (!ttbr0_usermode_access_allowed(regs))
+               goto no_context;
+
        if (!(flags & FAULT_FLAG_USER))
                goto lock_mmap;