arch/x86/kernel/sev-es-shared.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * AMD Encrypted Register State Support
   4  *
   5  * Author: Joerg Roedel <jroedel@suse.de>
   6  *
   7  * This file is not compiled stand-alone. It contains code shared
   8  * between the pre-decompression boot code and the running Linux kernel
   9  * and is included directly into both code-bases.
  10  */
  11
  12 #ifndef __BOOT_COMPRESSED
  13 #define error(v)        pr_err(v)
  14 #define has_cpuflag(f)  boot_cpu_has(f)
  15 #endif
  16
  17 static bool __init sev_es_check_cpu_features(void)
  18 {
  19         if (!has_cpuflag(X86_FEATURE_RDRAND)) {
  20                 error("RDRAND instruction not supported - no trusted source of randomness available\n");
  21                 return false;
  22         }
  23
  24         return true;
  25 }
  26
  27 static void sev_es_terminate(unsigned int reason)
  28 {
  29         u64 val = GHCB_SEV_TERMINATE;
  30
  31         /*
  32          * Tell the hypervisor what went wrong - only reason-set 0 is
  33          * currently supported.
  34          */
  35         val |= GHCB_SEV_TERMINATE_REASON(0, reason);
  36
  37         /* Request Guest Termination from Hypvervisor */
  38         sev_es_wr_ghcb_msr(val);
  39         VMGEXIT();
  40
  41         while (true)
  42                 asm volatile("hlt\n" : : : "memory");
  43 }
  44
  45 static bool sev_es_negotiate_protocol(void)
  46 {
  47         u64 val;
  48
  49         /* Do the GHCB protocol version negotiation */
  50         sev_es_wr_ghcb_msr(GHCB_SEV_INFO_REQ);
  51         VMGEXIT();
  52         val = sev_es_rd_ghcb_msr();
  53
  54         if (GHCB_INFO(val) != GHCB_SEV_INFO)
  55                 return false;
  56
  57         if (GHCB_PROTO_MAX(val) < GHCB_PROTO_OUR ||
  58             GHCB_PROTO_MIN(val) > GHCB_PROTO_OUR)
  59                 return false;
  60
  61         return true;
  62 }
  63
  64 static __always_inline void vc_ghcb_invalidate(struct ghcb *ghcb)
  65 {
  66         memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap));
  67 }
  68
  69 static bool vc_decoding_needed(unsigned long exit_code)
  70 {
  71         /* Exceptions don't require to decode the instruction */
  72         return !(exit_code >= SVM_EXIT_EXCP_BASE &&
  73                  exit_code <= SVM_EXIT_LAST_EXCP);
  74 }
  75
  76 static enum es_result vc_init_em_ctxt(struct es_em_ctxt *ctxt,
  77                                       struct pt_regs *regs,
  78                                       unsigned long exit_code)
  79 {
  80         enum es_result ret = ES_OK;
  81
  82         memset(ctxt, 0, sizeof(*ctxt));
  83         ctxt->regs = regs;
  84
  85         if (vc_decoding_needed(exit_code))
  86                 ret = vc_decode_insn(ctxt);
  87
  88         return ret;
  89 }
  90
  91 static void vc_finish_insn(struct es_em_ctxt *ctxt)
  92 {
  93         ctxt->regs->ip += ctxt->insn.length;
  94 }
  95
  96 static enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb,
  97                                           struct es_em_ctxt *ctxt,
  98                                           u64 exit_code, u64 exit_info_1,
  99                                           u64 exit_info_2)
 100 {
 101         enum es_result ret;
 102
 103         /* Fill in protocol and format specifiers */
 104         ghcb->protocol_version = GHCB_PROTOCOL_MAX;
 105         ghcb->ghcb_usage       = GHCB_DEFAULT_USAGE;
 106
 107         ghcb_set_sw_exit_code(ghcb, exit_code);
 108         ghcb_set_sw_exit_info_1(ghcb, exit_info_1);
 109         ghcb_set_sw_exit_info_2(ghcb, exit_info_2);
 110
 111         sev_es_wr_ghcb_msr(__pa(ghcb));
 112         VMGEXIT();
 113
 114         if ((ghcb->save.sw_exit_info_1 & 0xffffffff) == 1) {
 115                 u64 info = ghcb->save.sw_exit_info_2;
 116                 unsigned long v;
 117
 118                 info = ghcb->save.sw_exit_info_2;
 119                 v = info & SVM_EVTINJ_VEC_MASK;
 120
 121                 /* Check if exception information from hypervisor is sane. */
 122                 if ((info & SVM_EVTINJ_VALID) &&
 123                     ((v == X86_TRAP_GP) || (v == X86_TRAP_UD)) &&
 124                     ((info & SVM_EVTINJ_TYPE_MASK) == SVM_EVTINJ_TYPE_EXEPT)) {
 125                         ctxt->fi.vector = v;
 126                         if (info & SVM_EVTINJ_VALID_ERR)
 127                                 ctxt->fi.error_code = info >> 32;
 128                         ret = ES_EXCEPTION;
 129                 } else {
 130                         ret = ES_VMM_ERROR;
 131                 }
 132         } else {
 133                 ret = ES_OK;
 134         }
 135
 136         return ret;
 137 }
 138
 139 /*
 140  * Boot VC Handler - This is the first VC handler during boot, there is no GHCB
 141  * page yet, so it only supports the MSR based communication with the
 142  * hypervisor and only the CPUID exit-code.
 143  */
 144 void __init do_vc_no_ghcb(struct pt_regs *regs, unsigned long exit_code)
 145 {
 146         unsigned int fn = lower_bits(regs->ax, 32);
 147         unsigned long val;
 148
 149         /* Only CPUID is supported via MSR protocol */
 150         if (exit_code != SVM_EXIT_CPUID)
 151                 goto fail;
 152
 153         sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_EAX));
 154         VMGEXIT();
 155         val = sev_es_rd_ghcb_msr();
 156         if (GHCB_SEV_GHCB_RESP_CODE(val) != GHCB_SEV_CPUID_RESP)
 157                 goto fail;
 158         regs->ax = val >> 32;
 159
 160         sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_EBX));
 161         VMGEXIT();
 162         val = sev_es_rd_ghcb_msr();
 163         if (GHCB_SEV_GHCB_RESP_CODE(val) != GHCB_SEV_CPUID_RESP)
 164                 goto fail;
 165         regs->bx = val >> 32;
 166
 167         sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_ECX));
 168         VMGEXIT();
 169         val = sev_es_rd_ghcb_msr();
 170         if (GHCB_SEV_GHCB_RESP_CODE(val) != GHCB_SEV_CPUID_RESP)
 171                 goto fail;
 172         regs->cx = val >> 32;
 173
 174         sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_EDX));
 175         VMGEXIT();
 176         val = sev_es_rd_ghcb_msr();
 177         if (GHCB_SEV_GHCB_RESP_CODE(val) != GHCB_SEV_CPUID_RESP)
 178                 goto fail;
 179         regs->dx = val >> 32;
 180
 181         /* Skip over the CPUID two-byte opcode */
 182         regs->ip += 2;
 183
 184         return;
 185
 186 fail:
 187         sev_es_wr_ghcb_msr(GHCB_SEV_TERMINATE);
 188         VMGEXIT();
 189
 190         /* Shouldn't get here - if we do halt the machine */
 191         while (true)
 192                 asm volatile("hlt\n");
 193 }
 194
 195 static enum es_result vc_insn_string_read(struct es_em_ctxt *ctxt,
 196                                           void *src, char *buf,
 197                                           unsigned int data_size,
 198                                           unsigned int count,
 199                                           bool backwards)
 200 {
 201         int i, b = backwards ? -1 : 1;
 202         enum es_result ret = ES_OK;
 203
 204         for (i = 0; i < count; i++) {
 205                 void *s = src + (i * data_size * b);
 206                 char *d = buf + (i * data_size);
 207
 208                 ret = vc_read_mem(ctxt, s, d, data_size);
 209                 if (ret != ES_OK)
 210                         break;
 211         }
 212
 213         return ret;
 214 }
 215
 216 static enum es_result vc_insn_string_write(struct es_em_ctxt *ctxt,
 217                                            void *dst, char *buf,
 218                                            unsigned int data_size,
 219                                            unsigned int count,
 220                                            bool backwards)
 221 {
 222         int i, s = backwards ? -1 : 1;
 223         enum es_result ret = ES_OK;
 224
 225         for (i = 0; i < count; i++) {
 226                 void *d = dst + (i * data_size * s);
 227                 char *b = buf + (i * data_size);
 228
 229                 ret = vc_write_mem(ctxt, d, b, data_size);
 230                 if (ret != ES_OK)
 231                         break;
 232         }
 233
 234         return ret;
 235 }
 236
 237 #define IOIO_TYPE_STR  BIT(2)
 238 #define IOIO_TYPE_IN   1
 239 #define IOIO_TYPE_INS  (IOIO_TYPE_IN | IOIO_TYPE_STR)
 240 #define IOIO_TYPE_OUT  0
 241 #define IOIO_TYPE_OUTS (IOIO_TYPE_OUT | IOIO_TYPE_STR)
 242
 243 #define IOIO_REP       BIT(3)
 244
 245 #define IOIO_ADDR_64   BIT(9)
 246 #define IOIO_ADDR_32   BIT(8)
 247 #define IOIO_ADDR_16   BIT(7)
 248
 249 #define IOIO_DATA_32   BIT(6)
 250 #define IOIO_DATA_16   BIT(5)
 251 #define IOIO_DATA_8    BIT(4)
 252
 253 #define IOIO_SEG_ES    (0 << 10)
 254 #define IOIO_SEG_DS    (3 << 10)
 255
 256 static enum es_result vc_ioio_exitinfo(struct es_em_ctxt *ctxt, u64 *exitinfo)
 257 {
 258         struct insn *insn = &ctxt->insn;
 259         *exitinfo = 0;
 260
 261         switch (insn->opcode.bytes[0]) {
 262         /* INS opcodes */
 263         case 0x6c:
 264         case 0x6d:
 265                 *exitinfo |= IOIO_TYPE_INS;
 266                 *exitinfo |= IOIO_SEG_ES;
 267                 *exitinfo |= (ctxt->regs->dx & 0xffff) << 16;
 268                 break;
 269
 270         /* OUTS opcodes */
 271         case 0x6e:
 272         case 0x6f:
 273                 *exitinfo |= IOIO_TYPE_OUTS;
 274                 *exitinfo |= IOIO_SEG_DS;
 275                 *exitinfo |= (ctxt->regs->dx & 0xffff) << 16;
 276                 break;
 277
 278         /* IN immediate opcodes */
 279         case 0xe4:
 280         case 0xe5:
 281                 *exitinfo |= IOIO_TYPE_IN;
 282                 *exitinfo |= (u64)insn->immediate.value << 16;
 283                 break;
 284
 285         /* OUT immediate opcodes */
 286         case 0xe6:
 287         case 0xe7:
 288                 *exitinfo |= IOIO_TYPE_OUT;
 289                 *exitinfo |= (u64)insn->immediate.value << 16;
 290                 break;
 291
 292         /* IN register opcodes */
 293         case 0xec:
 294         case 0xed:
 295                 *exitinfo |= IOIO_TYPE_IN;
 296                 *exitinfo |= (ctxt->regs->dx & 0xffff) << 16;
 297                 break;
 298
 299         /* OUT register opcodes */
 300         case 0xee:
 301         case 0xef:
 302                 *exitinfo |= IOIO_TYPE_OUT;
 303                 *exitinfo |= (ctxt->regs->dx & 0xffff) << 16;
 304                 break;
 305
 306         default:
 307                 return ES_DECODE_FAILED;
 308         }
 309
 310         switch (insn->opcode.bytes[0]) {
 311         case 0x6c:
 312         case 0x6e:
 313         case 0xe4:
 314         case 0xe6:
 315         case 0xec:
 316         case 0xee:
 317                 /* Single byte opcodes */
 318                 *exitinfo |= IOIO_DATA_8;
 319                 break;
 320         default:
 321                 /* Length determined by instruction parsing */
 322                 *exitinfo |= (insn->opnd_bytes == 2) ? IOIO_DATA_16
 323                                                      : IOIO_DATA_32;
 324         }
 325         switch (insn->addr_bytes) {
 326         case 2:
 327                 *exitinfo |= IOIO_ADDR_16;
 328                 break;
 329         case 4:
 330                 *exitinfo |= IOIO_ADDR_32;
 331                 break;
 332         case 8:
 333                 *exitinfo |= IOIO_ADDR_64;
 334                 break;
 335         }
 336
 337         if (insn_has_rep_prefix(insn))
 338                 *exitinfo |= IOIO_REP;
 339
 340         return ES_OK;
 341 }
 342
 343 static enum es_result vc_handle_ioio(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
 344 {
 345         struct pt_regs *regs = ctxt->regs;
 346         u64 exit_info_1, exit_info_2;
 347         enum es_result ret;
 348
 349         ret = vc_ioio_exitinfo(ctxt, &exit_info_1);
 350         if (ret != ES_OK)
 351                 return ret;
 352
 353         if (exit_info_1 & IOIO_TYPE_STR) {
 354
 355                 /* (REP) INS/OUTS */
 356
 357                 bool df = ((regs->flags & X86_EFLAGS_DF) == X86_EFLAGS_DF);
 358                 unsigned int io_bytes, exit_bytes;
 359                 unsigned int ghcb_count, op_count;
 360                 unsigned long es_base;
 361                 u64 sw_scratch;
 362
 363                 /*
 364                  * For the string variants with rep prefix the amount of in/out
 365                  * operations per #VC exception is limited so that the kernel
 366                  * has a chance to take interrupts and re-schedule while the
 367                  * instruction is emulated.
 368                  */
 369                 io_bytes   = (exit_info_1 >> 4) & 0x7;
 370                 ghcb_count = sizeof(ghcb->shared_buffer) / io_bytes;
 371
 372                 op_count    = (exit_info_1 & IOIO_REP) ? regs->cx : 1;
 373                 exit_info_2 = min(op_count, ghcb_count);
 374                 exit_bytes  = exit_info_2 * io_bytes;
 375
 376                 es_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_ES);
 377
 378                 /* Read bytes of OUTS into the shared buffer */
 379                 if (!(exit_info_1 & IOIO_TYPE_IN)) {
 380                         ret = vc_insn_string_read(ctxt,
 381                                                (void *)(es_base + regs->si),
 382                                                ghcb->shared_buffer, io_bytes,
 383                                                exit_info_2, df);
 384                         if (ret)
 385                                 return ret;
 386                 }
 387
 388                 /*
 389                  * Issue an VMGEXIT to the HV to consume the bytes from the
 390                  * shared buffer or to have it write them into the shared buffer
 391                  * depending on the instruction: OUTS or INS.
 392                  */
 393                 sw_scratch = __pa(ghcb) + offsetof(struct ghcb, shared_buffer);
 394                 ghcb_set_sw_scratch(ghcb, sw_scratch);
 395                 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO,
 396                                           exit_info_1, exit_info_2);
 397                 if (ret != ES_OK)
 398                         return ret;
 399
 400                 /* Read bytes from shared buffer into the guest's destination. */
 401                 if (exit_info_1 & IOIO_TYPE_IN) {
 402                         ret = vc_insn_string_write(ctxt,
 403                                                    (void *)(es_base + regs->di),
 404                                                    ghcb->shared_buffer, io_bytes,
 405                                                    exit_info_2, df);
 406                         if (ret)
 407                                 return ret;
 408
 409                         if (df)
 410                                 regs->di -= exit_bytes;
 411                         else
 412                                 regs->di += exit_bytes;
 413                 } else {
 414                         if (df)
 415                                 regs->si -= exit_bytes;
 416                         else
 417                                 regs->si += exit_bytes;
 418                 }
 419
 420                 if (exit_info_1 & IOIO_REP)
 421                         regs->cx -= exit_info_2;
 422
 423                 ret = regs->cx ? ES_RETRY : ES_OK;
 424
 425         } else {
 426
 427                 /* IN/OUT into/from rAX */
 428
 429                 int bits = (exit_info_1 & 0x70) >> 1;
 430                 u64 rax = 0;
 431
 432                 if (!(exit_info_1 & IOIO_TYPE_IN))
 433                         rax = lower_bits(regs->ax, bits);
 434
 435                 ghcb_set_rax(ghcb, rax);
 436
 437                 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO, exit_info_1, 0);
 438                 if (ret != ES_OK)
 439                         return ret;
 440
 441                 if (exit_info_1 & IOIO_TYPE_IN) {
 442                         if (!ghcb_rax_is_valid(ghcb))
 443                                 return ES_VMM_ERROR;
 444                         regs->ax = lower_bits(ghcb->save.rax, bits);
 445                 }
 446         }
 447
 448         return ret;
 449 }
 450
 451 static enum es_result vc_handle_cpuid(struct ghcb *ghcb,
 452                                       struct es_em_ctxt *ctxt)
 453 {
 454         struct pt_regs *regs = ctxt->regs;
 455         u32 cr4 = native_read_cr4();
 456         enum es_result ret;
 457
 458         ghcb_set_rax(ghcb, regs->ax);
 459         ghcb_set_rcx(ghcb, regs->cx);
 460
 461         if (cr4 & X86_CR4_OSXSAVE)
 462                 /* Safe to read xcr0 */
 463                 ghcb_set_xcr0(ghcb, xgetbv(XCR_XFEATURE_ENABLED_MASK));
 464         else
 465                 /* xgetbv will cause #GP - use reset value for xcr0 */
 466                 ghcb_set_xcr0(ghcb, 1);
 467
 468         ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_CPUID, 0, 0);
 469         if (ret != ES_OK)
 470                 return ret;
 471
 472         if (!(ghcb_rax_is_valid(ghcb) &&
 473               ghcb_rbx_is_valid(ghcb) &&
 474               ghcb_rcx_is_valid(ghcb) &&
 475               ghcb_rdx_is_valid(ghcb)))
 476                 return ES_VMM_ERROR;
 477
 478         regs->ax = ghcb->save.rax;
 479         regs->bx = ghcb->save.rbx;
 480         regs->cx = ghcb->save.rcx;
 481         regs->dx = ghcb->save.rdx;
 482
 483         return ES_OK;
 484 }
 485
 486 static enum es_result vc_handle_rdtsc(struct ghcb *ghcb,
 487                                       struct es_em_ctxt *ctxt,
 488                                       unsigned long exit_code)
 489 {
 490         bool rdtscp = (exit_code == SVM_EXIT_RDTSCP);
 491         enum es_result ret;
 492
 493         ret = sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, 0, 0);
 494         if (ret != ES_OK)
 495                 return ret;
 496
 497         if (!(ghcb_rax_is_valid(ghcb) && ghcb_rdx_is_valid(ghcb) &&
 498              (!rdtscp || ghcb_rcx_is_valid(ghcb))))
 499                 return ES_VMM_ERROR;
 500
 501         ctxt->regs->ax = ghcb->save.rax;
 502         ctxt->regs->dx = ghcb->save.rdx;
 503         if (rdtscp)
 504                 ctxt->regs->cx = ghcb->save.rcx;
 505
 506         return ES_OK;
 507 }