x86/sev-es: Setup per-CPU GHCBs for the runtime handler

The runtime handler needs one GHCB per-CPU. Set them up and map them
unencrypted.

 [ bp: Touchups and simplification. ]

Signed-off-by: Tom Lendacky <thomas.lendacky@amd.com>
Signed-off-by: Joerg Roedel <jroedel@suse.de>
Signed-off-by: Borislav Petkov <bp@suse.de>
Link: https://lkml.kernel.org/r/20200907131613.12703-42-joro@8bytes.org

diff --git a/arch/x86/include/asm/mem_encrypt.h b/arch/x86/include/asm/mem_encrypt.h
index 4e72b73..c9f5df0 100644 (file)
--- a/arch/x86/include/asm/mem_encrypt.h
+++ b/arch/x86/include/asm/mem_encrypt.h
@@ -49,6 +49,7 @@ void __init mem_encrypt_free_decrypted_mem(void);
 /* Architecture __weak replacement functions */
 void __init mem_encrypt_init(void);
 
+void __init sev_es_init_vc_handling(void);
 bool sme_active(void);
 bool sev_active(void);
 bool sev_es_active(void);
@@ -72,6 +73,7 @@ static inline void __init sme_early_init(void) { }
 static inline void __init sme_encrypt_kernel(struct boot_params *bp) { }
 static inline void __init sme_enable(struct boot_params *bp) { }
 
+static inline void sev_es_init_vc_handling(void) { }
 static inline bool sme_active(void) { return false; }
 static inline bool sev_active(void) { return false; }
 static inline bool sev_es_active(void) { return false; }

diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c
index 213a427..720b1b6 100644 (file)
--- a/arch/x86/kernel/sev-es.c
+++ b/arch/x86/kernel/sev-es.c
@@ -8,8 +8,13 @@
  */
 
 #include <linux/sched/debug.h> /* For show_regs() */
-#include <linux/kernel.h>
+#include <linux/percpu-defs.h>
+#include <linux/mem_encrypt.h>
 #include <linux/printk.h>
+#include <linux/mm_types.h>
+#include <linux/set_memory.h>
+#include <linux/memblock.h>
+#include <linux/kernel.h>
 #include <linux/mm.h>
 
 #include <asm/sev-es.h>
@@ -29,6 +34,13 @@ static struct ghcb boot_ghcb_page __bss_decrypted __aligned(PAGE_SIZE);
  */
 static struct ghcb __initdata *boot_ghcb;
 
+/* #VC handler runtime per-CPU data */
+struct sev_es_runtime_data {
+       struct ghcb ghcb_page;
+};
+
+static DEFINE_PER_CPU(struct sev_es_runtime_data*, runtime_data);
+
 /* Needed in vc_early_forward_exception */
 void do_early_exception(struct pt_regs *regs, int trapnr);
 
@@ -198,6 +210,48 @@ static bool __init sev_es_setup_ghcb(void)
        return true;
 }
 
+static void __init alloc_runtime_data(int cpu)
+{
+       struct sev_es_runtime_data *data;
+
+       data = memblock_alloc(sizeof(*data), PAGE_SIZE);
+       if (!data)
+               panic("Can't allocate SEV-ES runtime data");
+
+       per_cpu(runtime_data, cpu) = data;
+}
+
+static void __init init_ghcb(int cpu)
+{
+       struct sev_es_runtime_data *data;
+       int err;
+
+       data = per_cpu(runtime_data, cpu);
+
+       err = early_set_memory_decrypted((unsigned long)&data->ghcb_page,
+                                        sizeof(data->ghcb_page));
+       if (err)
+               panic("Can't map GHCBs unencrypted");
+
+       memset(&data->ghcb_page, 0, sizeof(data->ghcb_page));
+}
+
+void __init sev_es_init_vc_handling(void)
+{
+       int cpu;
+
+       BUILD_BUG_ON(offsetof(struct sev_es_runtime_data, ghcb_page) % PAGE_SIZE);
+
+       if (!sev_es_active())
+               return;
+
+       /* Initialize per-cpu GHCB pages */
+       for_each_possible_cpu(cpu) {
+               alloc_runtime_data(cpu);
+               init_ghcb(cpu);
+       }
+}
+
 static void __init vc_early_forward_exception(struct es_em_ctxt *ctxt)
 {
        int trapnr = ctxt->fi.vector;

diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c
index 81a2fb7..2dc32b0 100644 (file)
--- a/arch/x86/kernel/traps.c
+++ b/arch/x86/kernel/traps.c
@@ -1074,6 +1074,9 @@ void __init trap_init(void)
        /* Init cpu_entry_area before IST entries are set up */
        setup_cpu_entry_areas();
 
+       /* Init GHCB memory pages when running as an SEV-ES guest */
+       sev_es_init_vc_handling();
+
        idt_setup_traps();
 
        /*