arm64/sve: Probe SVE capabilities and usable vector lengths

[linux-2.6-microblaze.git] / arch / arm64 / kernel / cpufeature.c

diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index 0e192cb..036ad9d 100644 (file)
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -27,6 +27,7 @@
 #include <asm/cpu.h>
 #include <asm/cpufeature.h>
 #include <asm/cpu_ops.h>
+#include <asm/fpsimd.h>
 #include <asm/mmu_context.h>
 #include <asm/processor.h>
 #include <asm/sysreg.h>
@@ -287,6 +288,12 @@ static const struct arm64_ftr_bits ftr_id_dfr0[] = {
        ARM64_FTR_END,
 };
 
+static const struct arm64_ftr_bits ftr_zcr[] = {
+       ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE,
+               ZCR_ELx_LEN_SHIFT, ZCR_ELx_LEN_SIZE, 0),        /* LEN */
+       ARM64_FTR_END,
+};
+
 /*
  * Common ftr bits for a 32bit register with all hidden, strict
  * attributes, with 4bit feature fields and a default safe value of
@@ -353,6 +360,7 @@ static const struct __ftr_reg_entry {
        /* Op1 = 0, CRn = 0, CRm = 4 */
        ARM64_FTR_REG(SYS_ID_AA64PFR0_EL1, ftr_id_aa64pfr0),
        ARM64_FTR_REG(SYS_ID_AA64PFR1_EL1, ftr_raz),
+       ARM64_FTR_REG(SYS_ID_AA64ZFR0_EL1, ftr_raz),
 
        /* Op1 = 0, CRn = 0, CRm = 5 */
        ARM64_FTR_REG(SYS_ID_AA64DFR0_EL1, ftr_id_aa64dfr0),
@@ -367,6 +375,9 @@ static const struct __ftr_reg_entry {
        ARM64_FTR_REG(SYS_ID_AA64MMFR1_EL1, ftr_id_aa64mmfr1),
        ARM64_FTR_REG(SYS_ID_AA64MMFR2_EL1, ftr_id_aa64mmfr2),
 
+       /* Op1 = 0, CRn = 1, CRm = 2 */
+       ARM64_FTR_REG(SYS_ZCR_EL1, ftr_zcr),
+
        /* Op1 = 3, CRn = 0, CRm = 0 */
        { SYS_CTR_EL0, &arm64_ftr_reg_ctrel0 },
        ARM64_FTR_REG(SYS_DCZID_EL0, ftr_dczid),
@@ -504,6 +515,7 @@ void __init init_cpu_features(struct cpuinfo_arm64 *info)
        init_cpu_ftr_reg(SYS_ID_AA64MMFR2_EL1, info->reg_id_aa64mmfr2);
        init_cpu_ftr_reg(SYS_ID_AA64PFR0_EL1, info->reg_id_aa64pfr0);
        init_cpu_ftr_reg(SYS_ID_AA64PFR1_EL1, info->reg_id_aa64pfr1);
+       init_cpu_ftr_reg(SYS_ID_AA64ZFR0_EL1, info->reg_id_aa64zfr0);
 
        if (id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0)) {
                init_cpu_ftr_reg(SYS_ID_DFR0_EL1, info->reg_id_dfr0);
@@ -524,6 +536,10 @@ void __init init_cpu_features(struct cpuinfo_arm64 *info)
                init_cpu_ftr_reg(SYS_MVFR2_EL1, info->reg_mvfr2);
        }
 
+       if (id_aa64pfr0_sve(info->reg_id_aa64pfr0)) {
+               init_cpu_ftr_reg(SYS_ZCR_EL1, info->reg_zcr);
+               sve_init_vq_map();
+       }
 }
 
 static void update_cpu_ftr_reg(struct arm64_ftr_reg *reg, u64 new)
@@ -627,6 +643,9 @@ void update_cpu_features(int cpu,
        taint |= check_update_ftr_reg(SYS_ID_AA64PFR1_EL1, cpu,
                                      info->reg_id_aa64pfr1, boot->reg_id_aa64pfr1);
 
+       taint |= check_update_ftr_reg(SYS_ID_AA64ZFR0_EL1, cpu,
+                                     info->reg_id_aa64zfr0, boot->reg_id_aa64zfr0);
+
        /*
         * If we have AArch32, we care about 32-bit features for compat.
         * If the system doesn't support AArch32, don't update them.
@@ -674,6 +693,16 @@ void update_cpu_features(int cpu,
                                        info->reg_mvfr2, boot->reg_mvfr2);
        }
 
+       if (id_aa64pfr0_sve(info->reg_id_aa64pfr0)) {
+               taint |= check_update_ftr_reg(SYS_ZCR_EL1, cpu,
+                                       info->reg_zcr, boot->reg_zcr);
+
+               /* Probe vector lengths, unless we already gave up on SVE */
+               if (id_aa64pfr0_sve(read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1)) &&
+                   !sys_caps_initialised)
+                       sve_update_vq_map();
+       }
+
        /*
         * Mismatched CPU features are a recipe for disaster. Don't even
         * pretend to support them.
@@ -1106,6 +1135,23 @@ verify_local_cpu_features(const struct arm64_cpu_capabilities *caps)
        }
 }
 
+static void verify_sve_features(void)
+{
+       u64 safe_zcr = read_sanitised_ftr_reg(SYS_ZCR_EL1);
+       u64 zcr = read_zcr_features();
+
+       unsigned int safe_len = safe_zcr & ZCR_ELx_LEN_MASK;
+       unsigned int len = zcr & ZCR_ELx_LEN_MASK;
+
+       if (len < safe_len || sve_verify_vq_map()) {
+               pr_crit("CPU%d: SVE: required vector length(s) missing\n",
+                       smp_processor_id());
+               cpu_die_early();
+       }
+
+       /* Add checks on other ZCR bits here if necessary */
+}
+
 /*
  * Run through the enabled system capabilities and enable() it on this CPU.
  * The capabilities were decided based on the available CPUs at the boot time.
@@ -1119,8 +1165,12 @@ static void verify_local_cpu_capabilities(void)
        verify_local_cpu_errata_workarounds();
        verify_local_cpu_features(arm64_features);
        verify_local_elf_hwcaps(arm64_elf_hwcaps);
+
        if (system_supports_32bit_el0())
                verify_local_elf_hwcaps(compat_elf_hwcaps);
+
+       if (system_supports_sve())
+               verify_sve_features();
 }
 
 void check_local_cpu_capabilities(void)
@@ -1198,6 +1248,8 @@ void __init setup_cpu_features(void)
        if (system_supports_32bit_el0())
                setup_elf_hwcaps(compat_elf_hwcaps);
 
+       sve_setup();
+
        /* Advertise that we have computed the system capabilities */
        set_sys_caps_initialised();