1 // SPDX-License-Identifier: GPL-2.0
3 * AMD Encrypted Register State Support
5 * Author: Joerg Roedel <jroedel@suse.de>
7 * This file is not compiled stand-alone. It contains code shared
8 * between the pre-decompression boot code and the running Linux kernel
9 * and is included directly into both code-bases.
12 #ifndef __BOOT_COMPRESSED
13 #define error(v) pr_err(v)
14 #define has_cpuflag(f) boot_cpu_has(f)
17 static bool __init sev_es_check_cpu_features(void)
19 if (!has_cpuflag(X86_FEATURE_RDRAND)) {
20 error("RDRAND instruction not supported - no trusted source of randomness available\n");
27 static void __noreturn sev_es_terminate(unsigned int reason)
29 u64 val = GHCB_MSR_TERM_REQ;
32 * Tell the hypervisor what went wrong - only reason-set 0 is
33 * currently supported.
35 val |= GHCB_SEV_TERM_REASON(0, reason);
37 /* Request Guest Termination from Hypvervisor */
38 sev_es_wr_ghcb_msr(val);
42 asm volatile("hlt\n" : : : "memory");
45 static bool sev_es_negotiate_protocol(void)
49 /* Do the GHCB protocol version negotiation */
50 sev_es_wr_ghcb_msr(GHCB_MSR_SEV_INFO_REQ);
52 val = sev_es_rd_ghcb_msr();
54 if (GHCB_MSR_INFO(val) != GHCB_MSR_SEV_INFO_RESP)
57 if (GHCB_MSR_PROTO_MAX(val) < GHCB_PROTO_OUR ||
58 GHCB_MSR_PROTO_MIN(val) > GHCB_PROTO_OUR)
64 static __always_inline void vc_ghcb_invalidate(struct ghcb *ghcb)
66 ghcb->save.sw_exit_code = 0;
67 memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap));
70 static bool vc_decoding_needed(unsigned long exit_code)
72 /* Exceptions don't require to decode the instruction */
73 return !(exit_code >= SVM_EXIT_EXCP_BASE &&
74 exit_code <= SVM_EXIT_LAST_EXCP);
77 static enum es_result vc_init_em_ctxt(struct es_em_ctxt *ctxt,
79 unsigned long exit_code)
81 enum es_result ret = ES_OK;
83 memset(ctxt, 0, sizeof(*ctxt));
86 if (vc_decoding_needed(exit_code))
87 ret = vc_decode_insn(ctxt);
92 static void vc_finish_insn(struct es_em_ctxt *ctxt)
94 ctxt->regs->ip += ctxt->insn.length;
97 static enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb,
98 struct es_em_ctxt *ctxt,
99 u64 exit_code, u64 exit_info_1,
104 /* Fill in protocol and format specifiers */
105 ghcb->protocol_version = GHCB_PROTOCOL_MAX;
106 ghcb->ghcb_usage = GHCB_DEFAULT_USAGE;
108 ghcb_set_sw_exit_code(ghcb, exit_code);
109 ghcb_set_sw_exit_info_1(ghcb, exit_info_1);
110 ghcb_set_sw_exit_info_2(ghcb, exit_info_2);
112 sev_es_wr_ghcb_msr(__pa(ghcb));
115 if ((ghcb->save.sw_exit_info_1 & 0xffffffff) == 1) {
116 u64 info = ghcb->save.sw_exit_info_2;
119 info = ghcb->save.sw_exit_info_2;
120 v = info & SVM_EVTINJ_VEC_MASK;
122 /* Check if exception information from hypervisor is sane. */
123 if ((info & SVM_EVTINJ_VALID) &&
124 ((v == X86_TRAP_GP) || (v == X86_TRAP_UD)) &&
125 ((info & SVM_EVTINJ_TYPE_MASK) == SVM_EVTINJ_TYPE_EXEPT)) {
127 if (info & SVM_EVTINJ_VALID_ERR)
128 ctxt->fi.error_code = info >> 32;
141 * Boot VC Handler - This is the first VC handler during boot, there is no GHCB
142 * page yet, so it only supports the MSR based communication with the
143 * hypervisor and only the CPUID exit-code.
145 void __init do_vc_no_ghcb(struct pt_regs *regs, unsigned long exit_code)
147 unsigned int fn = lower_bits(regs->ax, 32);
150 /* Only CPUID is supported via MSR protocol */
151 if (exit_code != SVM_EXIT_CPUID)
154 sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_EAX));
156 val = sev_es_rd_ghcb_msr();
157 if (GHCB_RESP_CODE(val) != GHCB_MSR_CPUID_RESP)
159 regs->ax = val >> 32;
161 sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_EBX));
163 val = sev_es_rd_ghcb_msr();
164 if (GHCB_RESP_CODE(val) != GHCB_MSR_CPUID_RESP)
166 regs->bx = val >> 32;
168 sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_ECX));
170 val = sev_es_rd_ghcb_msr();
171 if (GHCB_RESP_CODE(val) != GHCB_MSR_CPUID_RESP)
173 regs->cx = val >> 32;
175 sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_EDX));
177 val = sev_es_rd_ghcb_msr();
178 if (GHCB_RESP_CODE(val) != GHCB_MSR_CPUID_RESP)
180 regs->dx = val >> 32;
183 * This is a VC handler and the #VC is only raised when SEV-ES is
184 * active, which means SEV must be active too. Do sanity checks on the
185 * CPUID results to make sure the hypervisor does not trick the kernel
186 * into the no-sev path. This could map sensitive data unencrypted and
187 * make it accessible to the hypervisor.
189 * In particular, check for:
190 * - Availability of CPUID leaf 0x8000001f
193 * The hypervisor might still report the wrong C-bit position, but this
194 * can't be checked here.
197 if (fn == 0x80000000 && (regs->ax < 0x8000001f))
200 else if ((fn == 0x8000001f && !(regs->ax & BIT(1))))
204 /* Skip over the CPUID two-byte opcode */
210 /* Terminate the guest */
211 sev_es_terminate(GHCB_SEV_ES_REASON_GENERAL_REQUEST);
214 static enum es_result vc_insn_string_read(struct es_em_ctxt *ctxt,
215 void *src, char *buf,
216 unsigned int data_size,
220 int i, b = backwards ? -1 : 1;
221 enum es_result ret = ES_OK;
223 for (i = 0; i < count; i++) {
224 void *s = src + (i * data_size * b);
225 char *d = buf + (i * data_size);
227 ret = vc_read_mem(ctxt, s, d, data_size);
235 static enum es_result vc_insn_string_write(struct es_em_ctxt *ctxt,
236 void *dst, char *buf,
237 unsigned int data_size,
241 int i, s = backwards ? -1 : 1;
242 enum es_result ret = ES_OK;
244 for (i = 0; i < count; i++) {
245 void *d = dst + (i * data_size * s);
246 char *b = buf + (i * data_size);
248 ret = vc_write_mem(ctxt, d, b, data_size);
256 #define IOIO_TYPE_STR BIT(2)
257 #define IOIO_TYPE_IN 1
258 #define IOIO_TYPE_INS (IOIO_TYPE_IN | IOIO_TYPE_STR)
259 #define IOIO_TYPE_OUT 0
260 #define IOIO_TYPE_OUTS (IOIO_TYPE_OUT | IOIO_TYPE_STR)
262 #define IOIO_REP BIT(3)
264 #define IOIO_ADDR_64 BIT(9)
265 #define IOIO_ADDR_32 BIT(8)
266 #define IOIO_ADDR_16 BIT(7)
268 #define IOIO_DATA_32 BIT(6)
269 #define IOIO_DATA_16 BIT(5)
270 #define IOIO_DATA_8 BIT(4)
272 #define IOIO_SEG_ES (0 << 10)
273 #define IOIO_SEG_DS (3 << 10)
275 static enum es_result vc_ioio_exitinfo(struct es_em_ctxt *ctxt, u64 *exitinfo)
277 struct insn *insn = &ctxt->insn;
280 switch (insn->opcode.bytes[0]) {
284 *exitinfo |= IOIO_TYPE_INS;
285 *exitinfo |= IOIO_SEG_ES;
286 *exitinfo |= (ctxt->regs->dx & 0xffff) << 16;
292 *exitinfo |= IOIO_TYPE_OUTS;
293 *exitinfo |= IOIO_SEG_DS;
294 *exitinfo |= (ctxt->regs->dx & 0xffff) << 16;
297 /* IN immediate opcodes */
300 *exitinfo |= IOIO_TYPE_IN;
301 *exitinfo |= (u8)insn->immediate.value << 16;
304 /* OUT immediate opcodes */
307 *exitinfo |= IOIO_TYPE_OUT;
308 *exitinfo |= (u8)insn->immediate.value << 16;
311 /* IN register opcodes */
314 *exitinfo |= IOIO_TYPE_IN;
315 *exitinfo |= (ctxt->regs->dx & 0xffff) << 16;
318 /* OUT register opcodes */
321 *exitinfo |= IOIO_TYPE_OUT;
322 *exitinfo |= (ctxt->regs->dx & 0xffff) << 16;
326 return ES_DECODE_FAILED;
329 switch (insn->opcode.bytes[0]) {
336 /* Single byte opcodes */
337 *exitinfo |= IOIO_DATA_8;
340 /* Length determined by instruction parsing */
341 *exitinfo |= (insn->opnd_bytes == 2) ? IOIO_DATA_16
344 switch (insn->addr_bytes) {
346 *exitinfo |= IOIO_ADDR_16;
349 *exitinfo |= IOIO_ADDR_32;
352 *exitinfo |= IOIO_ADDR_64;
356 if (insn_has_rep_prefix(insn))
357 *exitinfo |= IOIO_REP;
362 static enum es_result vc_handle_ioio(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
364 struct pt_regs *regs = ctxt->regs;
365 u64 exit_info_1, exit_info_2;
368 ret = vc_ioio_exitinfo(ctxt, &exit_info_1);
372 if (exit_info_1 & IOIO_TYPE_STR) {
376 bool df = ((regs->flags & X86_EFLAGS_DF) == X86_EFLAGS_DF);
377 unsigned int io_bytes, exit_bytes;
378 unsigned int ghcb_count, op_count;
379 unsigned long es_base;
383 * For the string variants with rep prefix the amount of in/out
384 * operations per #VC exception is limited so that the kernel
385 * has a chance to take interrupts and re-schedule while the
386 * instruction is emulated.
388 io_bytes = (exit_info_1 >> 4) & 0x7;
389 ghcb_count = sizeof(ghcb->shared_buffer) / io_bytes;
391 op_count = (exit_info_1 & IOIO_REP) ? regs->cx : 1;
392 exit_info_2 = min(op_count, ghcb_count);
393 exit_bytes = exit_info_2 * io_bytes;
395 es_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_ES);
397 /* Read bytes of OUTS into the shared buffer */
398 if (!(exit_info_1 & IOIO_TYPE_IN)) {
399 ret = vc_insn_string_read(ctxt,
400 (void *)(es_base + regs->si),
401 ghcb->shared_buffer, io_bytes,
408 * Issue an VMGEXIT to the HV to consume the bytes from the
409 * shared buffer or to have it write them into the shared buffer
410 * depending on the instruction: OUTS or INS.
412 sw_scratch = __pa(ghcb) + offsetof(struct ghcb, shared_buffer);
413 ghcb_set_sw_scratch(ghcb, sw_scratch);
414 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO,
415 exit_info_1, exit_info_2);
419 /* Read bytes from shared buffer into the guest's destination. */
420 if (exit_info_1 & IOIO_TYPE_IN) {
421 ret = vc_insn_string_write(ctxt,
422 (void *)(es_base + regs->di),
423 ghcb->shared_buffer, io_bytes,
429 regs->di -= exit_bytes;
431 regs->di += exit_bytes;
434 regs->si -= exit_bytes;
436 regs->si += exit_bytes;
439 if (exit_info_1 & IOIO_REP)
440 regs->cx -= exit_info_2;
442 ret = regs->cx ? ES_RETRY : ES_OK;
446 /* IN/OUT into/from rAX */
448 int bits = (exit_info_1 & 0x70) >> 1;
451 if (!(exit_info_1 & IOIO_TYPE_IN))
452 rax = lower_bits(regs->ax, bits);
454 ghcb_set_rax(ghcb, rax);
456 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO, exit_info_1, 0);
460 if (exit_info_1 & IOIO_TYPE_IN) {
461 if (!ghcb_rax_is_valid(ghcb))
463 regs->ax = lower_bits(ghcb->save.rax, bits);
470 static enum es_result vc_handle_cpuid(struct ghcb *ghcb,
471 struct es_em_ctxt *ctxt)
473 struct pt_regs *regs = ctxt->regs;
474 u32 cr4 = native_read_cr4();
477 ghcb_set_rax(ghcb, regs->ax);
478 ghcb_set_rcx(ghcb, regs->cx);
480 if (cr4 & X86_CR4_OSXSAVE)
481 /* Safe to read xcr0 */
482 ghcb_set_xcr0(ghcb, xgetbv(XCR_XFEATURE_ENABLED_MASK));
484 /* xgetbv will cause #GP - use reset value for xcr0 */
485 ghcb_set_xcr0(ghcb, 1);
487 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_CPUID, 0, 0);
491 if (!(ghcb_rax_is_valid(ghcb) &&
492 ghcb_rbx_is_valid(ghcb) &&
493 ghcb_rcx_is_valid(ghcb) &&
494 ghcb_rdx_is_valid(ghcb)))
497 regs->ax = ghcb->save.rax;
498 regs->bx = ghcb->save.rbx;
499 regs->cx = ghcb->save.rcx;
500 regs->dx = ghcb->save.rdx;
505 static enum es_result vc_handle_rdtsc(struct ghcb *ghcb,
506 struct es_em_ctxt *ctxt,
507 unsigned long exit_code)
509 bool rdtscp = (exit_code == SVM_EXIT_RDTSCP);
512 ret = sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, 0, 0);
516 if (!(ghcb_rax_is_valid(ghcb) && ghcb_rdx_is_valid(ghcb) &&
517 (!rdtscp || ghcb_rcx_is_valid(ghcb))))
520 ctxt->regs->ax = ghcb->save.rax;
521 ctxt->regs->dx = ghcb->save.rdx;
523 ctxt->regs->cx = ghcb->save.rcx;