1 // SPDX-License-Identifier: GPL-2.0-only
2 /******************************************************************************
5 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
7 * Copyright (c) 2005 Keir Fraser
9 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
10 * privileged instructions:
12 * Copyright (C) 2006 Qumranet
13 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
15 * Avi Kivity <avi@qumranet.com>
16 * Yaniv Kamay <yaniv@qumranet.com>
18 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
21 #include <linux/kvm_host.h>
22 #include "kvm_cache_regs.h"
23 #include "kvm_emulate.h"
24 #include <linux/stringify.h>
25 #include <asm/debugreg.h>
26 #include <asm/nospec-branch.h>
38 #define OpImplicit 1ull /* No generic decode */
39 #define OpReg 2ull /* Register */
40 #define OpMem 3ull /* Memory */
41 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
42 #define OpDI 5ull /* ES:DI/EDI/RDI */
43 #define OpMem64 6ull /* Memory, 64-bit */
44 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
45 #define OpDX 8ull /* DX register */
46 #define OpCL 9ull /* CL register (for shifts) */
47 #define OpImmByte 10ull /* 8-bit sign extended immediate */
48 #define OpOne 11ull /* Implied 1 */
49 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
50 #define OpMem16 13ull /* Memory operand (16-bit). */
51 #define OpMem32 14ull /* Memory operand (32-bit). */
52 #define OpImmU 15ull /* Immediate operand, zero extended */
53 #define OpSI 16ull /* SI/ESI/RSI */
54 #define OpImmFAddr 17ull /* Immediate far address */
55 #define OpMemFAddr 18ull /* Far address in memory */
56 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
57 #define OpES 20ull /* ES */
58 #define OpCS 21ull /* CS */
59 #define OpSS 22ull /* SS */
60 #define OpDS 23ull /* DS */
61 #define OpFS 24ull /* FS */
62 #define OpGS 25ull /* GS */
63 #define OpMem8 26ull /* 8-bit zero extended memory operand */
64 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
65 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
66 #define OpAccLo 29ull /* Low part of extended acc (AX/AX/EAX/RAX) */
67 #define OpAccHi 30ull /* High part of extended acc (-/DX/EDX/RDX) */
69 #define OpBits 5 /* Width of operand field */
70 #define OpMask ((1ull << OpBits) - 1)
73 * Opcode effective-address decode tables.
74 * Note that we only emulate instructions that have at least one memory
75 * operand (excluding implicit stack references). We assume that stack
76 * references and instruction fetches will never occur in special memory
77 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
81 /* Operand sizes: 8-bit operands or specified/overridden size. */
82 #define ByteOp (1<<0) /* 8-bit operands. */
83 /* Destination operand type. */
85 #define ImplicitOps (OpImplicit << DstShift)
86 #define DstReg (OpReg << DstShift)
87 #define DstMem (OpMem << DstShift)
88 #define DstAcc (OpAcc << DstShift)
89 #define DstDI (OpDI << DstShift)
90 #define DstMem64 (OpMem64 << DstShift)
91 #define DstMem16 (OpMem16 << DstShift)
92 #define DstImmUByte (OpImmUByte << DstShift)
93 #define DstDX (OpDX << DstShift)
94 #define DstAccLo (OpAccLo << DstShift)
95 #define DstMask (OpMask << DstShift)
96 /* Source operand type. */
98 #define SrcNone (OpNone << SrcShift)
99 #define SrcReg (OpReg << SrcShift)
100 #define SrcMem (OpMem << SrcShift)
101 #define SrcMem16 (OpMem16 << SrcShift)
102 #define SrcMem32 (OpMem32 << SrcShift)
103 #define SrcImm (OpImm << SrcShift)
104 #define SrcImmByte (OpImmByte << SrcShift)
105 #define SrcOne (OpOne << SrcShift)
106 #define SrcImmUByte (OpImmUByte << SrcShift)
107 #define SrcImmU (OpImmU << SrcShift)
108 #define SrcSI (OpSI << SrcShift)
109 #define SrcXLat (OpXLat << SrcShift)
110 #define SrcImmFAddr (OpImmFAddr << SrcShift)
111 #define SrcMemFAddr (OpMemFAddr << SrcShift)
112 #define SrcAcc (OpAcc << SrcShift)
113 #define SrcImmU16 (OpImmU16 << SrcShift)
114 #define SrcImm64 (OpImm64 << SrcShift)
115 #define SrcDX (OpDX << SrcShift)
116 #define SrcMem8 (OpMem8 << SrcShift)
117 #define SrcAccHi (OpAccHi << SrcShift)
118 #define SrcMask (OpMask << SrcShift)
119 #define BitOp (1<<11)
120 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
121 #define String (1<<13) /* String instruction (rep capable) */
122 #define Stack (1<<14) /* Stack instruction (push/pop) */
123 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
124 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
125 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
126 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
127 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
128 #define Escape (5<<15) /* Escape to coprocessor instruction */
129 #define InstrDual (6<<15) /* Alternate instruction decoding of mod == 3 */
130 #define ModeDual (7<<15) /* Different instruction for 32/64 bit */
131 #define Sse (1<<18) /* SSE Vector instruction */
132 /* Generic ModRM decode. */
133 #define ModRM (1<<19)
134 /* Destination is only written; never read. */
137 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
138 #define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
139 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
140 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
141 #define Undefined (1<<25) /* No Such Instruction */
142 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
143 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
145 #define PageTable (1 << 29) /* instruction used to write page table */
146 #define NotImpl (1 << 30) /* instruction is not implemented */
147 /* Source 2 operand type */
148 #define Src2Shift (31)
149 #define Src2None (OpNone << Src2Shift)
150 #define Src2Mem (OpMem << Src2Shift)
151 #define Src2CL (OpCL << Src2Shift)
152 #define Src2ImmByte (OpImmByte << Src2Shift)
153 #define Src2One (OpOne << Src2Shift)
154 #define Src2Imm (OpImm << Src2Shift)
155 #define Src2ES (OpES << Src2Shift)
156 #define Src2CS (OpCS << Src2Shift)
157 #define Src2SS (OpSS << Src2Shift)
158 #define Src2DS (OpDS << Src2Shift)
159 #define Src2FS (OpFS << Src2Shift)
160 #define Src2GS (OpGS << Src2Shift)
161 #define Src2Mask (OpMask << Src2Shift)
162 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
163 #define AlignMask ((u64)7 << 41)
164 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
165 #define Unaligned ((u64)2 << 41) /* Explicitly unaligned (e.g. MOVDQU) */
166 #define Avx ((u64)3 << 41) /* Advanced Vector Extensions */
167 #define Aligned16 ((u64)4 << 41) /* Aligned to 16 byte boundary (e.g. FXSAVE) */
168 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
169 #define NoWrite ((u64)1 << 45) /* No writeback */
170 #define SrcWrite ((u64)1 << 46) /* Write back src operand */
171 #define NoMod ((u64)1 << 47) /* Mod field is ignored */
172 #define Intercept ((u64)1 << 48) /* Has valid intercept field */
173 #define CheckPerm ((u64)1 << 49) /* Has valid check_perm field */
174 #define PrivUD ((u64)1 << 51) /* #UD instead of #GP on CPL > 0 */
175 #define NearBranch ((u64)1 << 52) /* Near branches */
176 #define No16 ((u64)1 << 53) /* No 16 bit operand */
177 #define IncSP ((u64)1 << 54) /* SP is incremented before ModRM calc */
178 #define TwoMemOp ((u64)1 << 55) /* Instruction has two memory operand */
179 #define IsBranch ((u64)1 << 56) /* Instruction is considered a branch. */
181 #define DstXacc (DstAccLo | SrcAccHi | SrcWrite)
183 #define X2(x...) x, x
184 #define X3(x...) X2(x), x
185 #define X4(x...) X2(x), X2(x)
186 #define X5(x...) X4(x), x
187 #define X6(x...) X4(x), X2(x)
188 #define X7(x...) X4(x), X3(x)
189 #define X8(x...) X4(x), X4(x)
190 #define X16(x...) X8(x), X8(x)
192 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
193 #define FASTOP_SIZE (8 * (1 + HAS_KERNEL_IBT))
200 int (*execute)(struct x86_emulate_ctxt *ctxt);
201 const struct opcode *group;
202 const struct group_dual *gdual;
203 const struct gprefix *gprefix;
204 const struct escape *esc;
205 const struct instr_dual *idual;
206 const struct mode_dual *mdual;
207 void (*fastop)(struct fastop *fake);
209 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
213 struct opcode mod012[8];
214 struct opcode mod3[8];
218 struct opcode pfx_no;
219 struct opcode pfx_66;
220 struct opcode pfx_f2;
221 struct opcode pfx_f3;
226 struct opcode high[64];
230 struct opcode mod012;
235 struct opcode mode32;
236 struct opcode mode64;
239 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
241 enum x86_transfer_type {
243 X86_TRANSFER_CALL_JMP,
245 X86_TRANSFER_TASK_SWITCH,
248 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
250 if (!(ctxt->regs_valid & (1 << nr))) {
251 ctxt->regs_valid |= 1 << nr;
252 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
254 return ctxt->_regs[nr];
257 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
259 ctxt->regs_valid |= 1 << nr;
260 ctxt->regs_dirty |= 1 << nr;
261 return &ctxt->_regs[nr];
264 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
267 return reg_write(ctxt, nr);
270 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
274 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
275 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
278 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
280 ctxt->regs_dirty = 0;
281 ctxt->regs_valid = 0;
285 * These EFLAGS bits are restored from saved value during emulation, and
286 * any changes are written back to the saved value after emulation.
288 #define EFLAGS_MASK (X86_EFLAGS_OF|X86_EFLAGS_SF|X86_EFLAGS_ZF|X86_EFLAGS_AF|\
289 X86_EFLAGS_PF|X86_EFLAGS_CF)
298 * fastop functions have a special calling convention:
303 * flags: rflags (in/out)
304 * ex: rsi (in:fastop pointer, out:zero if exception)
306 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
307 * different operand sizes can be reached by calculation, rather than a jump
308 * table (which would be bigger than the code).
310 static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop);
312 #define __FOP_FUNC(name) \
313 ".align " __stringify(FASTOP_SIZE) " \n\t" \
314 ".type " name ", @function \n\t" \
318 #define FOP_FUNC(name) \
321 #define __FOP_RET(name) \
323 ".size " name ", .-" name "\n\t"
325 #define FOP_RET(name) \
328 #define FOP_START(op) \
329 extern void em_##op(struct fastop *fake); \
330 asm(".pushsection .text, \"ax\" \n\t" \
331 ".global em_" #op " \n\t" \
332 ".align " __stringify(FASTOP_SIZE) " \n\t" \
338 #define __FOPNOP(name) \
343 __FOPNOP(__stringify(__UNIQUE_ID(nop)))
345 #define FOP1E(op, dst) \
346 __FOP_FUNC(#op "_" #dst) \
347 "10: " #op " %" #dst " \n\t" \
348 __FOP_RET(#op "_" #dst)
350 #define FOP1EEX(op, dst) \
351 FOP1E(op, dst) _ASM_EXTABLE_TYPE_REG(10b, 11b, EX_TYPE_ZERO_REG, %%esi)
353 #define FASTOP1(op) \
358 ON64(FOP1E(op##q, rax)) \
361 /* 1-operand, using src2 (for MUL/DIV r/m) */
362 #define FASTOP1SRC2(op, name) \
367 ON64(FOP1E(op, rcx)) \
370 /* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
371 #define FASTOP1SRC2EX(op, name) \
376 ON64(FOP1EEX(op, rcx)) \
379 #define FOP2E(op, dst, src) \
380 __FOP_FUNC(#op "_" #dst "_" #src) \
381 #op " %" #src ", %" #dst " \n\t" \
382 __FOP_RET(#op "_" #dst "_" #src)
384 #define FASTOP2(op) \
386 FOP2E(op##b, al, dl) \
387 FOP2E(op##w, ax, dx) \
388 FOP2E(op##l, eax, edx) \
389 ON64(FOP2E(op##q, rax, rdx)) \
392 /* 2 operand, word only */
393 #define FASTOP2W(op) \
396 FOP2E(op##w, ax, dx) \
397 FOP2E(op##l, eax, edx) \
398 ON64(FOP2E(op##q, rax, rdx)) \
401 /* 2 operand, src is CL */
402 #define FASTOP2CL(op) \
404 FOP2E(op##b, al, cl) \
405 FOP2E(op##w, ax, cl) \
406 FOP2E(op##l, eax, cl) \
407 ON64(FOP2E(op##q, rax, cl)) \
410 /* 2 operand, src and dest are reversed */
411 #define FASTOP2R(op, name) \
413 FOP2E(op##b, dl, al) \
414 FOP2E(op##w, dx, ax) \
415 FOP2E(op##l, edx, eax) \
416 ON64(FOP2E(op##q, rdx, rax)) \
419 #define FOP3E(op, dst, src, src2) \
420 __FOP_FUNC(#op "_" #dst "_" #src "_" #src2) \
421 #op " %" #src2 ", %" #src ", %" #dst " \n\t"\
422 __FOP_RET(#op "_" #dst "_" #src "_" #src2)
424 /* 3-operand, word-only, src2=cl */
425 #define FASTOP3WCL(op) \
428 FOP3E(op##w, ax, dx, cl) \
429 FOP3E(op##l, eax, edx, cl) \
430 ON64(FOP3E(op##q, rax, rdx, cl)) \
433 /* Special case for SETcc - 1 instruction per cc */
436 * Depending on .config the SETcc functions look like:
438 * ENDBR [4 bytes; CONFIG_X86_KERNEL_IBT]
439 * SETcc %al [3 bytes]
441 * INT3 [1 byte; CONFIG_SLS]
443 * Which gives possible sizes 4, 5, 8 or 9. When rounded up to the
444 * next power-of-two alignment they become 4, 8 or 16 resp.
446 #define SETCC_LENGTH (ENDBR_INSN_SIZE + 4 + IS_ENABLED(CONFIG_SLS))
447 #define SETCC_ALIGN (4 << IS_ENABLED(CONFIG_SLS) << HAS_KERNEL_IBT)
448 static_assert(SETCC_LENGTH <= SETCC_ALIGN);
450 #define FOP_SETCC(op) \
451 ".align " __stringify(SETCC_ALIGN) " \n\t" \
452 ".type " #op ", @function \n\t" \
479 "pushf; sbb %al, %al; popf \n\t"
484 * XXX: inoutclob user must know where the argument is being expanded.
485 * Relying on CONFIG_CC_HAS_ASM_GOTO would allow us to remove _fault.
487 #define asm_safe(insn, inoutclob...) \
491 asm volatile("1:" insn "\n" \
493 _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_ONE_REG, %[_fault]) \
494 : [_fault] "+r"(_fault) inoutclob ); \
496 _fault ? X86EMUL_UNHANDLEABLE : X86EMUL_CONTINUE; \
499 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
500 enum x86_intercept intercept,
501 enum x86_intercept_stage stage)
503 struct x86_instruction_info info = {
504 .intercept = intercept,
505 .rep_prefix = ctxt->rep_prefix,
506 .modrm_mod = ctxt->modrm_mod,
507 .modrm_reg = ctxt->modrm_reg,
508 .modrm_rm = ctxt->modrm_rm,
509 .src_val = ctxt->src.val64,
510 .dst_val = ctxt->dst.val64,
511 .src_bytes = ctxt->src.bytes,
512 .dst_bytes = ctxt->dst.bytes,
513 .ad_bytes = ctxt->ad_bytes,
514 .next_rip = ctxt->eip,
517 return ctxt->ops->intercept(ctxt, &info, stage);
520 static void assign_masked(ulong *dest, ulong src, ulong mask)
522 *dest = (*dest & ~mask) | (src & mask);
525 static void assign_register(unsigned long *reg, u64 val, int bytes)
527 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
530 *(u8 *)reg = (u8)val;
533 *(u16 *)reg = (u16)val;
537 break; /* 64b: zero-extend */
544 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
546 return (1UL << (ctxt->ad_bytes << 3)) - 1;
549 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
552 struct desc_struct ss;
554 if (ctxt->mode == X86EMUL_MODE_PROT64)
556 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
557 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
560 static int stack_size(struct x86_emulate_ctxt *ctxt)
562 return (__fls(stack_mask(ctxt)) + 1) >> 3;
565 /* Access/update address held in a register, based on addressing mode. */
566 static inline unsigned long
567 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
569 if (ctxt->ad_bytes == sizeof(unsigned long))
572 return reg & ad_mask(ctxt);
575 static inline unsigned long
576 register_address(struct x86_emulate_ctxt *ctxt, int reg)
578 return address_mask(ctxt, reg_read(ctxt, reg));
581 static void masked_increment(ulong *reg, ulong mask, int inc)
583 assign_masked(reg, *reg + inc, mask);
587 register_address_increment(struct x86_emulate_ctxt *ctxt, int reg, int inc)
589 ulong *preg = reg_rmw(ctxt, reg);
591 assign_register(preg, *preg + inc, ctxt->ad_bytes);
594 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
596 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
599 static u32 desc_limit_scaled(struct desc_struct *desc)
601 u32 limit = get_desc_limit(desc);
603 return desc->g ? (limit << 12) | 0xfff : limit;
606 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
608 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
611 return ctxt->ops->get_cached_segment_base(ctxt, seg);
614 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
615 u32 error, bool valid)
618 ctxt->exception.vector = vec;
619 ctxt->exception.error_code = error;
620 ctxt->exception.error_code_valid = valid;
621 return X86EMUL_PROPAGATE_FAULT;
624 static int emulate_db(struct x86_emulate_ctxt *ctxt)
626 return emulate_exception(ctxt, DB_VECTOR, 0, false);
629 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
631 return emulate_exception(ctxt, GP_VECTOR, err, true);
634 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
636 return emulate_exception(ctxt, SS_VECTOR, err, true);
639 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
641 return emulate_exception(ctxt, UD_VECTOR, 0, false);
644 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
646 return emulate_exception(ctxt, TS_VECTOR, err, true);
649 static int emulate_de(struct x86_emulate_ctxt *ctxt)
651 return emulate_exception(ctxt, DE_VECTOR, 0, false);
654 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
656 return emulate_exception(ctxt, NM_VECTOR, 0, false);
659 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
662 struct desc_struct desc;
664 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
668 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
673 struct desc_struct desc;
675 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
676 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
679 static inline u8 ctxt_virt_addr_bits(struct x86_emulate_ctxt *ctxt)
681 return (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_LA57) ? 57 : 48;
684 static inline bool emul_is_noncanonical_address(u64 la,
685 struct x86_emulate_ctxt *ctxt)
687 return !__is_canonical_address(la, ctxt_virt_addr_bits(ctxt));
691 * x86 defines three classes of vector instructions: explicitly
692 * aligned, explicitly unaligned, and the rest, which change behaviour
693 * depending on whether they're AVX encoded or not.
695 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
696 * subject to the same check. FXSAVE and FXRSTOR are checked here too as their
697 * 512 bytes of data must be aligned to a 16 byte boundary.
699 static unsigned insn_alignment(struct x86_emulate_ctxt *ctxt, unsigned size)
701 u64 alignment = ctxt->d & AlignMask;
703 if (likely(size < 16))
718 static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt,
719 struct segmented_address addr,
720 unsigned *max_size, unsigned size,
721 bool write, bool fetch,
722 enum x86emul_mode mode, ulong *linear)
724 struct desc_struct desc;
731 la = seg_base(ctxt, addr.seg) + addr.ea;
734 case X86EMUL_MODE_PROT64:
736 va_bits = ctxt_virt_addr_bits(ctxt);
737 if (!__is_canonical_address(la, va_bits))
740 *max_size = min_t(u64, ~0u, (1ull << va_bits) - la);
741 if (size > *max_size)
745 *linear = la = (u32)la;
746 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
750 /* code segment in protected mode or read-only data segment */
751 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
752 || !(desc.type & 2)) && write)
754 /* unreadable code segment */
755 if (!fetch && (desc.type & 8) && !(desc.type & 2))
757 lim = desc_limit_scaled(&desc);
758 if (!(desc.type & 8) && (desc.type & 4)) {
759 /* expand-down segment */
762 lim = desc.d ? 0xffffffff : 0xffff;
766 if (lim == 0xffffffff)
769 *max_size = (u64)lim + 1 - addr.ea;
770 if (size > *max_size)
775 if (la & (insn_alignment(ctxt, size) - 1))
776 return emulate_gp(ctxt, 0);
777 return X86EMUL_CONTINUE;
779 if (addr.seg == VCPU_SREG_SS)
780 return emulate_ss(ctxt, 0);
782 return emulate_gp(ctxt, 0);
785 static int linearize(struct x86_emulate_ctxt *ctxt,
786 struct segmented_address addr,
787 unsigned size, bool write,
791 return __linearize(ctxt, addr, &max_size, size, write, false,
795 static inline int assign_eip(struct x86_emulate_ctxt *ctxt, ulong dst,
796 enum x86emul_mode mode)
801 struct segmented_address addr = { .seg = VCPU_SREG_CS,
804 if (ctxt->op_bytes != sizeof(unsigned long))
805 addr.ea = dst & ((1UL << (ctxt->op_bytes << 3)) - 1);
806 rc = __linearize(ctxt, addr, &max_size, 1, false, true, mode, &linear);
807 if (rc == X86EMUL_CONTINUE)
808 ctxt->_eip = addr.ea;
812 static inline int assign_eip_near(struct x86_emulate_ctxt *ctxt, ulong dst)
814 return assign_eip(ctxt, dst, ctxt->mode);
817 static int assign_eip_far(struct x86_emulate_ctxt *ctxt, ulong dst,
818 const struct desc_struct *cs_desc)
820 enum x86emul_mode mode = ctxt->mode;
824 if (ctxt->mode >= X86EMUL_MODE_PROT16) {
828 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
830 mode = X86EMUL_MODE_PROT64;
832 mode = X86EMUL_MODE_PROT32; /* temporary value */
835 if (mode == X86EMUL_MODE_PROT16 || mode == X86EMUL_MODE_PROT32)
836 mode = cs_desc->d ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
837 rc = assign_eip(ctxt, dst, mode);
838 if (rc == X86EMUL_CONTINUE)
843 static inline int jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
845 return assign_eip_near(ctxt, ctxt->_eip + rel);
848 static int linear_read_system(struct x86_emulate_ctxt *ctxt, ulong linear,
849 void *data, unsigned size)
851 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, true);
854 static int linear_write_system(struct x86_emulate_ctxt *ctxt,
855 ulong linear, void *data,
858 return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, true);
861 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
862 struct segmented_address addr,
869 rc = linearize(ctxt, addr, size, false, &linear);
870 if (rc != X86EMUL_CONTINUE)
872 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, false);
875 static int segmented_write_std(struct x86_emulate_ctxt *ctxt,
876 struct segmented_address addr,
883 rc = linearize(ctxt, addr, size, true, &linear);
884 if (rc != X86EMUL_CONTINUE)
886 return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, false);
890 * Prefetch the remaining bytes of the instruction without crossing page
891 * boundary if they are not in fetch_cache yet.
893 static int __do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt, int op_size)
896 unsigned size, max_size;
897 unsigned long linear;
898 int cur_size = ctxt->fetch.end - ctxt->fetch.data;
899 struct segmented_address addr = { .seg = VCPU_SREG_CS,
900 .ea = ctxt->eip + cur_size };
903 * We do not know exactly how many bytes will be needed, and
904 * __linearize is expensive, so fetch as much as possible. We
905 * just have to avoid going beyond the 15 byte limit, the end
906 * of the segment, or the end of the page.
908 * __linearize is called with size 0 so that it does not do any
909 * boundary check itself. Instead, we use max_size to check
912 rc = __linearize(ctxt, addr, &max_size, 0, false, true, ctxt->mode,
914 if (unlikely(rc != X86EMUL_CONTINUE))
917 size = min_t(unsigned, 15UL ^ cur_size, max_size);
918 size = min_t(unsigned, size, PAGE_SIZE - offset_in_page(linear));
921 * One instruction can only straddle two pages,
922 * and one has been loaded at the beginning of
923 * x86_decode_insn. So, if not enough bytes
924 * still, we must have hit the 15-byte boundary.
926 if (unlikely(size < op_size))
927 return emulate_gp(ctxt, 0);
929 rc = ctxt->ops->fetch(ctxt, linear, ctxt->fetch.end,
930 size, &ctxt->exception);
931 if (unlikely(rc != X86EMUL_CONTINUE))
933 ctxt->fetch.end += size;
934 return X86EMUL_CONTINUE;
937 static __always_inline int do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt,
940 unsigned done_size = ctxt->fetch.end - ctxt->fetch.ptr;
942 if (unlikely(done_size < size))
943 return __do_insn_fetch_bytes(ctxt, size - done_size);
945 return X86EMUL_CONTINUE;
948 /* Fetch next part of the instruction being emulated. */
949 #define insn_fetch(_type, _ctxt) \
952 rc = do_insn_fetch_bytes(_ctxt, sizeof(_type)); \
953 if (rc != X86EMUL_CONTINUE) \
955 ctxt->_eip += sizeof(_type); \
956 memcpy(&_x, ctxt->fetch.ptr, sizeof(_type)); \
957 ctxt->fetch.ptr += sizeof(_type); \
961 #define insn_fetch_arr(_arr, _size, _ctxt) \
963 rc = do_insn_fetch_bytes(_ctxt, _size); \
964 if (rc != X86EMUL_CONTINUE) \
966 ctxt->_eip += (_size); \
967 memcpy(_arr, ctxt->fetch.ptr, _size); \
968 ctxt->fetch.ptr += (_size); \
972 * Given the 'reg' portion of a ModRM byte, and a register block, return a
973 * pointer into the block that addresses the relevant register.
974 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
976 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
980 int highbyte_regs = (ctxt->rex_prefix == 0) && byteop;
982 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
983 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
985 p = reg_rmw(ctxt, modrm_reg);
989 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
990 struct segmented_address addr,
991 u16 *size, unsigned long *address, int op_bytes)
998 rc = segmented_read_std(ctxt, addr, size, 2);
999 if (rc != X86EMUL_CONTINUE)
1002 rc = segmented_read_std(ctxt, addr, address, op_bytes);
1016 FASTOP1SRC2(mul, mul_ex);
1017 FASTOP1SRC2(imul, imul_ex);
1018 FASTOP1SRC2EX(div, div_ex);
1019 FASTOP1SRC2EX(idiv, idiv_ex);
1048 FASTOP2R(cmp, cmp_r);
1050 static int em_bsf_c(struct x86_emulate_ctxt *ctxt)
1052 /* If src is zero, do not writeback, but update flags */
1053 if (ctxt->src.val == 0)
1054 ctxt->dst.type = OP_NONE;
1055 return fastop(ctxt, em_bsf);
1058 static int em_bsr_c(struct x86_emulate_ctxt *ctxt)
1060 /* If src is zero, do not writeback, but update flags */
1061 if (ctxt->src.val == 0)
1062 ctxt->dst.type = OP_NONE;
1063 return fastop(ctxt, em_bsr);
1066 static __always_inline u8 test_cc(unsigned int condition, unsigned long flags)
1069 void (*fop)(void) = (void *)em_setcc + SETCC_ALIGN * (condition & 0xf);
1071 flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
1072 asm("push %[flags]; popf; " CALL_NOSPEC
1073 : "=a"(rc) : [thunk_target]"r"(fop), [flags]"r"(flags));
1077 static void fetch_register_operand(struct operand *op)
1079 switch (op->bytes) {
1081 op->val = *(u8 *)op->addr.reg;
1084 op->val = *(u16 *)op->addr.reg;
1087 op->val = *(u32 *)op->addr.reg;
1090 op->val = *(u64 *)op->addr.reg;
1095 static int em_fninit(struct x86_emulate_ctxt *ctxt)
1097 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1098 return emulate_nm(ctxt);
1101 asm volatile("fninit");
1103 return X86EMUL_CONTINUE;
1106 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
1110 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1111 return emulate_nm(ctxt);
1114 asm volatile("fnstcw %0": "+m"(fcw));
1117 ctxt->dst.val = fcw;
1119 return X86EMUL_CONTINUE;
1122 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1126 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1127 return emulate_nm(ctxt);
1130 asm volatile("fnstsw %0": "+m"(fsw));
1133 ctxt->dst.val = fsw;
1135 return X86EMUL_CONTINUE;
1138 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1141 unsigned reg = ctxt->modrm_reg;
1143 if (!(ctxt->d & ModRM))
1144 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1146 if (ctxt->d & Sse) {
1150 kvm_read_sse_reg(reg, &op->vec_val);
1153 if (ctxt->d & Mmx) {
1162 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1163 op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);
1165 fetch_register_operand(op);
1166 op->orig_val = op->val;
1169 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1171 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1172 ctxt->modrm_seg = VCPU_SREG_SS;
1175 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1179 int index_reg, base_reg, scale;
1180 int rc = X86EMUL_CONTINUE;
1183 ctxt->modrm_reg = ((ctxt->rex_prefix << 1) & 8); /* REX.R */
1184 index_reg = (ctxt->rex_prefix << 2) & 8; /* REX.X */
1185 base_reg = (ctxt->rex_prefix << 3) & 8; /* REX.B */
1187 ctxt->modrm_mod = (ctxt->modrm & 0xc0) >> 6;
1188 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1189 ctxt->modrm_rm = base_reg | (ctxt->modrm & 0x07);
1190 ctxt->modrm_seg = VCPU_SREG_DS;
1192 if (ctxt->modrm_mod == 3 || (ctxt->d & NoMod)) {
1194 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1195 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
1197 if (ctxt->d & Sse) {
1200 op->addr.xmm = ctxt->modrm_rm;
1201 kvm_read_sse_reg(ctxt->modrm_rm, &op->vec_val);
1204 if (ctxt->d & Mmx) {
1207 op->addr.mm = ctxt->modrm_rm & 7;
1210 fetch_register_operand(op);
1216 if (ctxt->ad_bytes == 2) {
1217 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1218 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1219 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1220 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1222 /* 16-bit ModR/M decode. */
1223 switch (ctxt->modrm_mod) {
1225 if (ctxt->modrm_rm == 6)
1226 modrm_ea += insn_fetch(u16, ctxt);
1229 modrm_ea += insn_fetch(s8, ctxt);
1232 modrm_ea += insn_fetch(u16, ctxt);
1235 switch (ctxt->modrm_rm) {
1237 modrm_ea += bx + si;
1240 modrm_ea += bx + di;
1243 modrm_ea += bp + si;
1246 modrm_ea += bp + di;
1255 if (ctxt->modrm_mod != 0)
1262 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1263 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1264 ctxt->modrm_seg = VCPU_SREG_SS;
1265 modrm_ea = (u16)modrm_ea;
1267 /* 32/64-bit ModR/M decode. */
1268 if ((ctxt->modrm_rm & 7) == 4) {
1269 sib = insn_fetch(u8, ctxt);
1270 index_reg |= (sib >> 3) & 7;
1271 base_reg |= sib & 7;
1274 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1275 modrm_ea += insn_fetch(s32, ctxt);
1277 modrm_ea += reg_read(ctxt, base_reg);
1278 adjust_modrm_seg(ctxt, base_reg);
1279 /* Increment ESP on POP [ESP] */
1280 if ((ctxt->d & IncSP) &&
1281 base_reg == VCPU_REGS_RSP)
1282 modrm_ea += ctxt->op_bytes;
1285 modrm_ea += reg_read(ctxt, index_reg) << scale;
1286 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1287 modrm_ea += insn_fetch(s32, ctxt);
1288 if (ctxt->mode == X86EMUL_MODE_PROT64)
1289 ctxt->rip_relative = 1;
1291 base_reg = ctxt->modrm_rm;
1292 modrm_ea += reg_read(ctxt, base_reg);
1293 adjust_modrm_seg(ctxt, base_reg);
1295 switch (ctxt->modrm_mod) {
1297 modrm_ea += insn_fetch(s8, ctxt);
1300 modrm_ea += insn_fetch(s32, ctxt);
1304 op->addr.mem.ea = modrm_ea;
1305 if (ctxt->ad_bytes != 8)
1306 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
1312 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1315 int rc = X86EMUL_CONTINUE;
1318 switch (ctxt->ad_bytes) {
1320 op->addr.mem.ea = insn_fetch(u16, ctxt);
1323 op->addr.mem.ea = insn_fetch(u32, ctxt);
1326 op->addr.mem.ea = insn_fetch(u64, ctxt);
1333 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1337 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1338 mask = ~((long)ctxt->dst.bytes * 8 - 1);
1340 if (ctxt->src.bytes == 2)
1341 sv = (s16)ctxt->src.val & (s16)mask;
1342 else if (ctxt->src.bytes == 4)
1343 sv = (s32)ctxt->src.val & (s32)mask;
1345 sv = (s64)ctxt->src.val & (s64)mask;
1347 ctxt->dst.addr.mem.ea = address_mask(ctxt,
1348 ctxt->dst.addr.mem.ea + (sv >> 3));
1351 /* only subword offset */
1352 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1355 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1356 unsigned long addr, void *dest, unsigned size)
1359 struct read_cache *mc = &ctxt->mem_read;
1361 if (mc->pos < mc->end)
1364 WARN_ON((mc->end + size) >= sizeof(mc->data));
1366 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1368 if (rc != X86EMUL_CONTINUE)
1374 memcpy(dest, mc->data + mc->pos, size);
1376 return X86EMUL_CONTINUE;
1379 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1380 struct segmented_address addr,
1387 rc = linearize(ctxt, addr, size, false, &linear);
1388 if (rc != X86EMUL_CONTINUE)
1390 return read_emulated(ctxt, linear, data, size);
1393 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1394 struct segmented_address addr,
1401 rc = linearize(ctxt, addr, size, true, &linear);
1402 if (rc != X86EMUL_CONTINUE)
1404 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1408 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1409 struct segmented_address addr,
1410 const void *orig_data, const void *data,
1416 rc = linearize(ctxt, addr, size, true, &linear);
1417 if (rc != X86EMUL_CONTINUE)
1419 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1420 size, &ctxt->exception);
1423 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1424 unsigned int size, unsigned short port,
1427 struct read_cache *rc = &ctxt->io_read;
1429 if (rc->pos == rc->end) { /* refill pio read ahead */
1430 unsigned int in_page, n;
1431 unsigned int count = ctxt->rep_prefix ?
1432 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1433 in_page = (ctxt->eflags & X86_EFLAGS_DF) ?
1434 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1435 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1436 n = min3(in_page, (unsigned int)sizeof(rc->data) / size, count);
1439 rc->pos = rc->end = 0;
1440 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1445 if (ctxt->rep_prefix && (ctxt->d & String) &&
1446 !(ctxt->eflags & X86_EFLAGS_DF)) {
1447 ctxt->dst.data = rc->data + rc->pos;
1448 ctxt->dst.type = OP_MEM_STR;
1449 ctxt->dst.count = (rc->end - rc->pos) / size;
1452 memcpy(dest, rc->data + rc->pos, size);
1458 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1459 u16 index, struct desc_struct *desc)
1464 ctxt->ops->get_idt(ctxt, &dt);
1466 if (dt.size < index * 8 + 7)
1467 return emulate_gp(ctxt, index << 3 | 0x2);
1469 addr = dt.address + index * 8;
1470 return linear_read_system(ctxt, addr, desc, sizeof(*desc));
1473 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1474 u16 selector, struct desc_ptr *dt)
1476 const struct x86_emulate_ops *ops = ctxt->ops;
1479 if (selector & 1 << 2) {
1480 struct desc_struct desc;
1483 memset(dt, 0, sizeof(*dt));
1484 if (!ops->get_segment(ctxt, &sel, &desc, &base3,
1488 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1489 dt->address = get_desc_base(&desc) | ((u64)base3 << 32);
1491 ops->get_gdt(ctxt, dt);
1494 static int get_descriptor_ptr(struct x86_emulate_ctxt *ctxt,
1495 u16 selector, ulong *desc_addr_p)
1498 u16 index = selector >> 3;
1501 get_descriptor_table_ptr(ctxt, selector, &dt);
1503 if (dt.size < index * 8 + 7)
1504 return emulate_gp(ctxt, selector & 0xfffc);
1506 addr = dt.address + index * 8;
1508 #ifdef CONFIG_X86_64
1509 if (addr >> 32 != 0) {
1512 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1513 if (!(efer & EFER_LMA))
1518 *desc_addr_p = addr;
1519 return X86EMUL_CONTINUE;
1522 /* allowed just for 8 bytes segments */
1523 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1524 u16 selector, struct desc_struct *desc,
1529 rc = get_descriptor_ptr(ctxt, selector, desc_addr_p);
1530 if (rc != X86EMUL_CONTINUE)
1533 return linear_read_system(ctxt, *desc_addr_p, desc, sizeof(*desc));
1536 /* allowed just for 8 bytes segments */
1537 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1538 u16 selector, struct desc_struct *desc)
1543 rc = get_descriptor_ptr(ctxt, selector, &addr);
1544 if (rc != X86EMUL_CONTINUE)
1547 return linear_write_system(ctxt, addr, desc, sizeof(*desc));
1550 static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1551 u16 selector, int seg, u8 cpl,
1552 enum x86_transfer_type transfer,
1553 struct desc_struct *desc)
1555 struct desc_struct seg_desc, old_desc;
1557 unsigned err_vec = GP_VECTOR;
1559 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1565 memset(&seg_desc, 0, sizeof(seg_desc));
1567 if (ctxt->mode == X86EMUL_MODE_REAL) {
1568 /* set real mode segment descriptor (keep limit etc. for
1570 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1571 set_desc_base(&seg_desc, selector << 4);
1573 } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
1574 /* VM86 needs a clean new segment descriptor */
1575 set_desc_base(&seg_desc, selector << 4);
1576 set_desc_limit(&seg_desc, 0xffff);
1586 /* TR should be in GDT only */
1587 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1590 /* NULL selector is not valid for TR, CS and (except for long mode) SS */
1591 if (null_selector) {
1592 if (seg == VCPU_SREG_CS || seg == VCPU_SREG_TR)
1595 if (seg == VCPU_SREG_SS) {
1596 if (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl)
1600 * ctxt->ops->set_segment expects the CPL to be in
1601 * SS.DPL, so fake an expand-up 32-bit data segment.
1611 /* Skip all following checks */
1615 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1616 if (ret != X86EMUL_CONTINUE)
1619 err_code = selector & 0xfffc;
1620 err_vec = (transfer == X86_TRANSFER_TASK_SWITCH) ? TS_VECTOR :
1623 /* can't load system descriptor into segment selector */
1624 if (seg <= VCPU_SREG_GS && !seg_desc.s) {
1625 if (transfer == X86_TRANSFER_CALL_JMP)
1626 return X86EMUL_UNHANDLEABLE;
1635 * segment is not a writable data segment or segment
1636 * selector's RPL != CPL or segment selector's RPL != CPL
1638 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1642 if (!(seg_desc.type & 8))
1645 if (transfer == X86_TRANSFER_RET) {
1646 /* RET can never return to an inner privilege level. */
1649 /* Outer-privilege level return is not implemented */
1651 return X86EMUL_UNHANDLEABLE;
1653 if (transfer == X86_TRANSFER_RET || transfer == X86_TRANSFER_TASK_SWITCH) {
1654 if (seg_desc.type & 4) {
1663 } else { /* X86_TRANSFER_CALL_JMP */
1664 if (seg_desc.type & 4) {
1670 if (rpl > cpl || dpl != cpl)
1674 /* in long-mode d/b must be clear if l is set */
1675 if (seg_desc.d && seg_desc.l) {
1678 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1679 if (efer & EFER_LMA)
1683 /* CS(RPL) <- CPL */
1684 selector = (selector & 0xfffc) | cpl;
1687 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1690 err_vec = NP_VECTOR;
1693 old_desc = seg_desc;
1694 seg_desc.type |= 2; /* busy */
1695 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1696 sizeof(seg_desc), &ctxt->exception);
1697 if (ret != X86EMUL_CONTINUE)
1700 case VCPU_SREG_LDTR:
1701 if (seg_desc.s || seg_desc.type != 2)
1704 default: /* DS, ES, FS, or GS */
1706 * segment is not a data or readable code segment or
1707 * ((segment is a data or nonconforming code segment)
1708 * and (both RPL and CPL > DPL))
1710 if ((seg_desc.type & 0xa) == 0x8 ||
1711 (((seg_desc.type & 0xc) != 0xc) &&
1712 (rpl > dpl && cpl > dpl)))
1718 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1723 /* mark segment as accessed */
1724 if (!(seg_desc.type & 1)) {
1726 ret = write_segment_descriptor(ctxt, selector,
1728 if (ret != X86EMUL_CONTINUE)
1731 } else if (ctxt->mode == X86EMUL_MODE_PROT64) {
1732 ret = linear_read_system(ctxt, desc_addr+8, &base3, sizeof(base3));
1733 if (ret != X86EMUL_CONTINUE)
1735 if (emul_is_noncanonical_address(get_desc_base(&seg_desc) |
1736 ((u64)base3 << 32), ctxt))
1737 return emulate_gp(ctxt, 0);
1740 ctxt->ops->set_segment(ctxt, selector, &seg_desc, base3, seg);
1743 return X86EMUL_CONTINUE;
1745 return emulate_exception(ctxt, err_vec, err_code, true);
1748 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1749 u16 selector, int seg)
1751 u8 cpl = ctxt->ops->cpl(ctxt);
1754 * None of MOV, POP and LSS can load a NULL selector in CPL=3, but
1755 * they can load it at CPL<3 (Intel's manual says only LSS can,
1758 * However, the Intel manual says that putting IST=1/DPL=3 in
1759 * an interrupt gate will result in SS=3 (the AMD manual instead
1760 * says it doesn't), so allow SS=3 in __load_segment_descriptor
1761 * and only forbid it here.
1763 if (seg == VCPU_SREG_SS && selector == 3 &&
1764 ctxt->mode == X86EMUL_MODE_PROT64)
1765 return emulate_exception(ctxt, GP_VECTOR, 0, true);
1767 return __load_segment_descriptor(ctxt, selector, seg, cpl,
1768 X86_TRANSFER_NONE, NULL);
1771 static void write_register_operand(struct operand *op)
1773 return assign_register(op->addr.reg, op->val, op->bytes);
1776 static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
1780 write_register_operand(op);
1783 if (ctxt->lock_prefix)
1784 return segmented_cmpxchg(ctxt,
1790 return segmented_write(ctxt,
1796 return segmented_write(ctxt,
1799 op->bytes * op->count);
1802 kvm_write_sse_reg(op->addr.xmm, &op->vec_val);
1805 kvm_write_mmx_reg(op->addr.mm, &op->mm_val);
1813 return X86EMUL_CONTINUE;
1816 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1818 struct segmented_address addr;
1820 rsp_increment(ctxt, -bytes);
1821 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1822 addr.seg = VCPU_SREG_SS;
1824 return segmented_write(ctxt, addr, data, bytes);
1827 static int em_push(struct x86_emulate_ctxt *ctxt)
1829 /* Disable writeback. */
1830 ctxt->dst.type = OP_NONE;
1831 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1834 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1835 void *dest, int len)
1838 struct segmented_address addr;
1840 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1841 addr.seg = VCPU_SREG_SS;
1842 rc = segmented_read(ctxt, addr, dest, len);
1843 if (rc != X86EMUL_CONTINUE)
1846 rsp_increment(ctxt, len);
1850 static int em_pop(struct x86_emulate_ctxt *ctxt)
1852 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1855 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1856 void *dest, int len)
1859 unsigned long val, change_mask;
1860 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
1861 int cpl = ctxt->ops->cpl(ctxt);
1863 rc = emulate_pop(ctxt, &val, len);
1864 if (rc != X86EMUL_CONTINUE)
1867 change_mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
1868 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF |
1869 X86_EFLAGS_TF | X86_EFLAGS_DF | X86_EFLAGS_NT |
1870 X86_EFLAGS_AC | X86_EFLAGS_ID;
1872 switch(ctxt->mode) {
1873 case X86EMUL_MODE_PROT64:
1874 case X86EMUL_MODE_PROT32:
1875 case X86EMUL_MODE_PROT16:
1877 change_mask |= X86_EFLAGS_IOPL;
1879 change_mask |= X86_EFLAGS_IF;
1881 case X86EMUL_MODE_VM86:
1883 return emulate_gp(ctxt, 0);
1884 change_mask |= X86_EFLAGS_IF;
1886 default: /* real mode */
1887 change_mask |= (X86_EFLAGS_IOPL | X86_EFLAGS_IF);
1891 *(unsigned long *)dest =
1892 (ctxt->eflags & ~change_mask) | (val & change_mask);
1897 static int em_popf(struct x86_emulate_ctxt *ctxt)
1899 ctxt->dst.type = OP_REG;
1900 ctxt->dst.addr.reg = &ctxt->eflags;
1901 ctxt->dst.bytes = ctxt->op_bytes;
1902 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1905 static int em_enter(struct x86_emulate_ctxt *ctxt)
1908 unsigned frame_size = ctxt->src.val;
1909 unsigned nesting_level = ctxt->src2.val & 31;
1913 return X86EMUL_UNHANDLEABLE;
1915 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1916 rc = push(ctxt, &rbp, stack_size(ctxt));
1917 if (rc != X86EMUL_CONTINUE)
1919 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1921 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1922 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1924 return X86EMUL_CONTINUE;
1927 static int em_leave(struct x86_emulate_ctxt *ctxt)
1929 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1931 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1934 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1936 int seg = ctxt->src2.val;
1938 ctxt->src.val = get_segment_selector(ctxt, seg);
1939 if (ctxt->op_bytes == 4) {
1940 rsp_increment(ctxt, -2);
1944 return em_push(ctxt);
1947 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1949 int seg = ctxt->src2.val;
1950 unsigned long selector;
1953 rc = emulate_pop(ctxt, &selector, 2);
1954 if (rc != X86EMUL_CONTINUE)
1957 if (ctxt->modrm_reg == VCPU_SREG_SS)
1958 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
1959 if (ctxt->op_bytes > 2)
1960 rsp_increment(ctxt, ctxt->op_bytes - 2);
1962 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1966 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1968 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1969 int rc = X86EMUL_CONTINUE;
1970 int reg = VCPU_REGS_RAX;
1972 while (reg <= VCPU_REGS_RDI) {
1973 (reg == VCPU_REGS_RSP) ?
1974 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
1977 if (rc != X86EMUL_CONTINUE)
1986 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1988 ctxt->src.val = (unsigned long)ctxt->eflags & ~X86_EFLAGS_VM;
1989 return em_push(ctxt);
1992 static int em_popa(struct x86_emulate_ctxt *ctxt)
1994 int rc = X86EMUL_CONTINUE;
1995 int reg = VCPU_REGS_RDI;
1998 while (reg >= VCPU_REGS_RAX) {
1999 if (reg == VCPU_REGS_RSP) {
2000 rsp_increment(ctxt, ctxt->op_bytes);
2004 rc = emulate_pop(ctxt, &val, ctxt->op_bytes);
2005 if (rc != X86EMUL_CONTINUE)
2007 assign_register(reg_rmw(ctxt, reg), val, ctxt->op_bytes);
2013 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
2015 const struct x86_emulate_ops *ops = ctxt->ops;
2022 /* TODO: Add limit checks */
2023 ctxt->src.val = ctxt->eflags;
2025 if (rc != X86EMUL_CONTINUE)
2028 ctxt->eflags &= ~(X86_EFLAGS_IF | X86_EFLAGS_TF | X86_EFLAGS_AC);
2030 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
2032 if (rc != X86EMUL_CONTINUE)
2035 ctxt->src.val = ctxt->_eip;
2037 if (rc != X86EMUL_CONTINUE)
2040 ops->get_idt(ctxt, &dt);
2042 eip_addr = dt.address + (irq << 2);
2043 cs_addr = dt.address + (irq << 2) + 2;
2045 rc = linear_read_system(ctxt, cs_addr, &cs, 2);
2046 if (rc != X86EMUL_CONTINUE)
2049 rc = linear_read_system(ctxt, eip_addr, &eip, 2);
2050 if (rc != X86EMUL_CONTINUE)
2053 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
2054 if (rc != X86EMUL_CONTINUE)
2062 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
2066 invalidate_registers(ctxt);
2067 rc = __emulate_int_real(ctxt, irq);
2068 if (rc == X86EMUL_CONTINUE)
2069 writeback_registers(ctxt);
2073 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
2075 switch(ctxt->mode) {
2076 case X86EMUL_MODE_REAL:
2077 return __emulate_int_real(ctxt, irq);
2078 case X86EMUL_MODE_VM86:
2079 case X86EMUL_MODE_PROT16:
2080 case X86EMUL_MODE_PROT32:
2081 case X86EMUL_MODE_PROT64:
2083 /* Protected mode interrupts unimplemented yet */
2084 return X86EMUL_UNHANDLEABLE;
2088 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
2090 int rc = X86EMUL_CONTINUE;
2091 unsigned long temp_eip = 0;
2092 unsigned long temp_eflags = 0;
2093 unsigned long cs = 0;
2094 unsigned long mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
2095 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_TF |
2096 X86_EFLAGS_IF | X86_EFLAGS_DF | X86_EFLAGS_OF |
2097 X86_EFLAGS_IOPL | X86_EFLAGS_NT | X86_EFLAGS_RF |
2098 X86_EFLAGS_AC | X86_EFLAGS_ID |
2100 unsigned long vm86_mask = X86_EFLAGS_VM | X86_EFLAGS_VIF |
2103 /* TODO: Add stack limit check */
2105 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
2107 if (rc != X86EMUL_CONTINUE)
2110 if (temp_eip & ~0xffff)
2111 return emulate_gp(ctxt, 0);
2113 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2115 if (rc != X86EMUL_CONTINUE)
2118 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
2120 if (rc != X86EMUL_CONTINUE)
2123 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2125 if (rc != X86EMUL_CONTINUE)
2128 ctxt->_eip = temp_eip;
2130 if (ctxt->op_bytes == 4)
2131 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
2132 else if (ctxt->op_bytes == 2) {
2133 ctxt->eflags &= ~0xffff;
2134 ctxt->eflags |= temp_eflags;
2137 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
2138 ctxt->eflags |= X86_EFLAGS_FIXED;
2139 ctxt->ops->set_nmi_mask(ctxt, false);
2144 static int em_iret(struct x86_emulate_ctxt *ctxt)
2146 switch(ctxt->mode) {
2147 case X86EMUL_MODE_REAL:
2148 return emulate_iret_real(ctxt);
2149 case X86EMUL_MODE_VM86:
2150 case X86EMUL_MODE_PROT16:
2151 case X86EMUL_MODE_PROT32:
2152 case X86EMUL_MODE_PROT64:
2154 /* iret from protected mode unimplemented yet */
2155 return X86EMUL_UNHANDLEABLE;
2159 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
2163 struct desc_struct new_desc;
2164 u8 cpl = ctxt->ops->cpl(ctxt);
2166 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2168 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
2169 X86_TRANSFER_CALL_JMP,
2171 if (rc != X86EMUL_CONTINUE)
2174 rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
2175 /* Error handling is not implemented. */
2176 if (rc != X86EMUL_CONTINUE)
2177 return X86EMUL_UNHANDLEABLE;
2182 static int em_jmp_abs(struct x86_emulate_ctxt *ctxt)
2184 return assign_eip_near(ctxt, ctxt->src.val);
2187 static int em_call_near_abs(struct x86_emulate_ctxt *ctxt)
2192 old_eip = ctxt->_eip;
2193 rc = assign_eip_near(ctxt, ctxt->src.val);
2194 if (rc != X86EMUL_CONTINUE)
2196 ctxt->src.val = old_eip;
2201 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2203 u64 old = ctxt->dst.orig_val64;
2205 if (ctxt->dst.bytes == 16)
2206 return X86EMUL_UNHANDLEABLE;
2208 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
2209 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2210 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2211 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2212 ctxt->eflags &= ~X86_EFLAGS_ZF;
2214 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2215 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2217 ctxt->eflags |= X86_EFLAGS_ZF;
2219 return X86EMUL_CONTINUE;
2222 static int em_ret(struct x86_emulate_ctxt *ctxt)
2227 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2228 if (rc != X86EMUL_CONTINUE)
2231 return assign_eip_near(ctxt, eip);
2234 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2237 unsigned long eip, cs;
2238 int cpl = ctxt->ops->cpl(ctxt);
2239 struct desc_struct new_desc;
2241 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2242 if (rc != X86EMUL_CONTINUE)
2244 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2245 if (rc != X86EMUL_CONTINUE)
2247 rc = __load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS, cpl,
2250 if (rc != X86EMUL_CONTINUE)
2252 rc = assign_eip_far(ctxt, eip, &new_desc);
2253 /* Error handling is not implemented. */
2254 if (rc != X86EMUL_CONTINUE)
2255 return X86EMUL_UNHANDLEABLE;
2260 static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt)
2264 rc = em_ret_far(ctxt);
2265 if (rc != X86EMUL_CONTINUE)
2267 rsp_increment(ctxt, ctxt->src.val);
2268 return X86EMUL_CONTINUE;
2271 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2273 /* Save real source value, then compare EAX against destination. */
2274 ctxt->dst.orig_val = ctxt->dst.val;
2275 ctxt->dst.val = reg_read(ctxt, VCPU_REGS_RAX);
2276 ctxt->src.orig_val = ctxt->src.val;
2277 ctxt->src.val = ctxt->dst.orig_val;
2278 fastop(ctxt, em_cmp);
2280 if (ctxt->eflags & X86_EFLAGS_ZF) {
2281 /* Success: write back to memory; no update of EAX */
2282 ctxt->src.type = OP_NONE;
2283 ctxt->dst.val = ctxt->src.orig_val;
2285 /* Failure: write the value we saw to EAX. */
2286 ctxt->src.type = OP_REG;
2287 ctxt->src.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2288 ctxt->src.val = ctxt->dst.orig_val;
2289 /* Create write-cycle to dest by writing the same value */
2290 ctxt->dst.val = ctxt->dst.orig_val;
2292 return X86EMUL_CONTINUE;
2295 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2297 int seg = ctxt->src2.val;
2301 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2303 rc = load_segment_descriptor(ctxt, sel, seg);
2304 if (rc != X86EMUL_CONTINUE)
2307 ctxt->dst.val = ctxt->src.val;
2311 static int emulator_has_longmode(struct x86_emulate_ctxt *ctxt)
2313 #ifdef CONFIG_X86_64
2314 return ctxt->ops->guest_has_long_mode(ctxt);
2320 static void rsm_set_desc_flags(struct desc_struct *desc, u32 flags)
2322 desc->g = (flags >> 23) & 1;
2323 desc->d = (flags >> 22) & 1;
2324 desc->l = (flags >> 21) & 1;
2325 desc->avl = (flags >> 20) & 1;
2326 desc->p = (flags >> 15) & 1;
2327 desc->dpl = (flags >> 13) & 3;
2328 desc->s = (flags >> 12) & 1;
2329 desc->type = (flags >> 8) & 15;
2332 static int rsm_load_seg_32(struct x86_emulate_ctxt *ctxt, const char *smstate,
2335 struct desc_struct desc;
2339 selector = GET_SMSTATE(u32, smstate, 0x7fa8 + n * 4);
2342 offset = 0x7f84 + n * 12;
2344 offset = 0x7f2c + (n - 3) * 12;
2346 set_desc_base(&desc, GET_SMSTATE(u32, smstate, offset + 8));
2347 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, offset + 4));
2348 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, offset));
2349 ctxt->ops->set_segment(ctxt, selector, &desc, 0, n);
2350 return X86EMUL_CONTINUE;
2353 #ifdef CONFIG_X86_64
2354 static int rsm_load_seg_64(struct x86_emulate_ctxt *ctxt, const char *smstate,
2357 struct desc_struct desc;
2362 offset = 0x7e00 + n * 16;
2364 selector = GET_SMSTATE(u16, smstate, offset);
2365 rsm_set_desc_flags(&desc, GET_SMSTATE(u16, smstate, offset + 2) << 8);
2366 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, offset + 4));
2367 set_desc_base(&desc, GET_SMSTATE(u32, smstate, offset + 8));
2368 base3 = GET_SMSTATE(u32, smstate, offset + 12);
2370 ctxt->ops->set_segment(ctxt, selector, &desc, base3, n);
2371 return X86EMUL_CONTINUE;
2375 static int rsm_enter_protected_mode(struct x86_emulate_ctxt *ctxt,
2376 u64 cr0, u64 cr3, u64 cr4)
2381 /* In order to later set CR4.PCIDE, CR3[11:0] must be zero. */
2383 if (cr4 & X86_CR4_PCIDE) {
2388 bad = ctxt->ops->set_cr(ctxt, 3, cr3);
2390 return X86EMUL_UNHANDLEABLE;
2393 * First enable PAE, long mode needs it before CR0.PG = 1 is set.
2394 * Then enable protected mode. However, PCID cannot be enabled
2395 * if EFER.LMA=0, so set it separately.
2397 bad = ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE);
2399 return X86EMUL_UNHANDLEABLE;
2401 bad = ctxt->ops->set_cr(ctxt, 0, cr0);
2403 return X86EMUL_UNHANDLEABLE;
2405 if (cr4 & X86_CR4_PCIDE) {
2406 bad = ctxt->ops->set_cr(ctxt, 4, cr4);
2408 return X86EMUL_UNHANDLEABLE;
2410 bad = ctxt->ops->set_cr(ctxt, 3, cr3 | pcid);
2412 return X86EMUL_UNHANDLEABLE;
2417 return X86EMUL_CONTINUE;
2420 static int rsm_load_state_32(struct x86_emulate_ctxt *ctxt,
2421 const char *smstate)
2423 struct desc_struct desc;
2426 u32 val, cr0, cr3, cr4;
2429 cr0 = GET_SMSTATE(u32, smstate, 0x7ffc);
2430 cr3 = GET_SMSTATE(u32, smstate, 0x7ff8);
2431 ctxt->eflags = GET_SMSTATE(u32, smstate, 0x7ff4) | X86_EFLAGS_FIXED;
2432 ctxt->_eip = GET_SMSTATE(u32, smstate, 0x7ff0);
2434 for (i = 0; i < 8; i++)
2435 *reg_write(ctxt, i) = GET_SMSTATE(u32, smstate, 0x7fd0 + i * 4);
2437 val = GET_SMSTATE(u32, smstate, 0x7fcc);
2439 if (ctxt->ops->set_dr(ctxt, 6, val))
2440 return X86EMUL_UNHANDLEABLE;
2442 val = GET_SMSTATE(u32, smstate, 0x7fc8);
2444 if (ctxt->ops->set_dr(ctxt, 7, val))
2445 return X86EMUL_UNHANDLEABLE;
2447 selector = GET_SMSTATE(u32, smstate, 0x7fc4);
2448 set_desc_base(&desc, GET_SMSTATE(u32, smstate, 0x7f64));
2449 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, 0x7f60));
2450 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, 0x7f5c));
2451 ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_TR);
2453 selector = GET_SMSTATE(u32, smstate, 0x7fc0);
2454 set_desc_base(&desc, GET_SMSTATE(u32, smstate, 0x7f80));
2455 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, 0x7f7c));
2456 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, 0x7f78));
2457 ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_LDTR);
2459 dt.address = GET_SMSTATE(u32, smstate, 0x7f74);
2460 dt.size = GET_SMSTATE(u32, smstate, 0x7f70);
2461 ctxt->ops->set_gdt(ctxt, &dt);
2463 dt.address = GET_SMSTATE(u32, smstate, 0x7f58);
2464 dt.size = GET_SMSTATE(u32, smstate, 0x7f54);
2465 ctxt->ops->set_idt(ctxt, &dt);
2467 for (i = 0; i < 6; i++) {
2468 int r = rsm_load_seg_32(ctxt, smstate, i);
2469 if (r != X86EMUL_CONTINUE)
2473 cr4 = GET_SMSTATE(u32, smstate, 0x7f14);
2475 ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smstate, 0x7ef8));
2477 return rsm_enter_protected_mode(ctxt, cr0, cr3, cr4);
2480 #ifdef CONFIG_X86_64
2481 static int rsm_load_state_64(struct x86_emulate_ctxt *ctxt,
2482 const char *smstate)
2484 struct desc_struct desc;
2486 u64 val, cr0, cr3, cr4;
2491 for (i = 0; i < 16; i++)
2492 *reg_write(ctxt, i) = GET_SMSTATE(u64, smstate, 0x7ff8 - i * 8);
2494 ctxt->_eip = GET_SMSTATE(u64, smstate, 0x7f78);
2495 ctxt->eflags = GET_SMSTATE(u32, smstate, 0x7f70) | X86_EFLAGS_FIXED;
2497 val = GET_SMSTATE(u64, smstate, 0x7f68);
2499 if (ctxt->ops->set_dr(ctxt, 6, val))
2500 return X86EMUL_UNHANDLEABLE;
2502 val = GET_SMSTATE(u64, smstate, 0x7f60);
2504 if (ctxt->ops->set_dr(ctxt, 7, val))
2505 return X86EMUL_UNHANDLEABLE;
2507 cr0 = GET_SMSTATE(u64, smstate, 0x7f58);
2508 cr3 = GET_SMSTATE(u64, smstate, 0x7f50);
2509 cr4 = GET_SMSTATE(u64, smstate, 0x7f48);
2510 ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smstate, 0x7f00));
2511 val = GET_SMSTATE(u64, smstate, 0x7ed0);
2513 if (ctxt->ops->set_msr(ctxt, MSR_EFER, val & ~EFER_LMA))
2514 return X86EMUL_UNHANDLEABLE;
2516 selector = GET_SMSTATE(u32, smstate, 0x7e90);
2517 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, 0x7e92) << 8);
2518 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, 0x7e94));
2519 set_desc_base(&desc, GET_SMSTATE(u32, smstate, 0x7e98));
2520 base3 = GET_SMSTATE(u32, smstate, 0x7e9c);
2521 ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_TR);
2523 dt.size = GET_SMSTATE(u32, smstate, 0x7e84);
2524 dt.address = GET_SMSTATE(u64, smstate, 0x7e88);
2525 ctxt->ops->set_idt(ctxt, &dt);
2527 selector = GET_SMSTATE(u32, smstate, 0x7e70);
2528 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, 0x7e72) << 8);
2529 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, 0x7e74));
2530 set_desc_base(&desc, GET_SMSTATE(u32, smstate, 0x7e78));
2531 base3 = GET_SMSTATE(u32, smstate, 0x7e7c);
2532 ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_LDTR);
2534 dt.size = GET_SMSTATE(u32, smstate, 0x7e64);
2535 dt.address = GET_SMSTATE(u64, smstate, 0x7e68);
2536 ctxt->ops->set_gdt(ctxt, &dt);
2538 r = rsm_enter_protected_mode(ctxt, cr0, cr3, cr4);
2539 if (r != X86EMUL_CONTINUE)
2542 for (i = 0; i < 6; i++) {
2543 r = rsm_load_seg_64(ctxt, smstate, i);
2544 if (r != X86EMUL_CONTINUE)
2548 return X86EMUL_CONTINUE;
2552 static int em_rsm(struct x86_emulate_ctxt *ctxt)
2554 unsigned long cr0, cr4, efer;
2559 if ((ctxt->ops->get_hflags(ctxt) & X86EMUL_SMM_MASK) == 0)
2560 return emulate_ud(ctxt);
2562 smbase = ctxt->ops->get_smbase(ctxt);
2564 ret = ctxt->ops->read_phys(ctxt, smbase + 0xfe00, buf, sizeof(buf));
2565 if (ret != X86EMUL_CONTINUE)
2566 return X86EMUL_UNHANDLEABLE;
2568 if ((ctxt->ops->get_hflags(ctxt) & X86EMUL_SMM_INSIDE_NMI_MASK) == 0)
2569 ctxt->ops->set_nmi_mask(ctxt, false);
2571 ctxt->ops->exiting_smm(ctxt);
2574 * Get back to real mode, to prepare a safe state in which to load
2575 * CR0/CR3/CR4/EFER. It's all a bit more complicated if the vCPU
2576 * supports long mode.
2578 if (emulator_has_longmode(ctxt)) {
2579 struct desc_struct cs_desc;
2581 /* Zero CR4.PCIDE before CR0.PG. */
2582 cr4 = ctxt->ops->get_cr(ctxt, 4);
2583 if (cr4 & X86_CR4_PCIDE)
2584 ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE);
2586 /* A 32-bit code segment is required to clear EFER.LMA. */
2587 memset(&cs_desc, 0, sizeof(cs_desc));
2589 cs_desc.s = cs_desc.g = cs_desc.p = 1;
2590 ctxt->ops->set_segment(ctxt, 0, &cs_desc, 0, VCPU_SREG_CS);
2593 /* For the 64-bit case, this will clear EFER.LMA. */
2594 cr0 = ctxt->ops->get_cr(ctxt, 0);
2595 if (cr0 & X86_CR0_PE)
2596 ctxt->ops->set_cr(ctxt, 0, cr0 & ~(X86_CR0_PG | X86_CR0_PE));
2598 if (emulator_has_longmode(ctxt)) {
2599 /* Clear CR4.PAE before clearing EFER.LME. */
2600 cr4 = ctxt->ops->get_cr(ctxt, 4);
2601 if (cr4 & X86_CR4_PAE)
2602 ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PAE);
2604 /* And finally go back to 32-bit mode. */
2606 ctxt->ops->set_msr(ctxt, MSR_EFER, efer);
2610 * Give leave_smm() a chance to make ISA-specific changes to the vCPU
2611 * state (e.g. enter guest mode) before loading state from the SMM
2614 if (ctxt->ops->leave_smm(ctxt, buf))
2615 goto emulate_shutdown;
2617 #ifdef CONFIG_X86_64
2618 if (emulator_has_longmode(ctxt))
2619 ret = rsm_load_state_64(ctxt, buf);
2622 ret = rsm_load_state_32(ctxt, buf);
2624 if (ret != X86EMUL_CONTINUE)
2625 goto emulate_shutdown;
2628 * Note, the ctxt->ops callbacks are responsible for handling side
2629 * effects when writing MSRs and CRs, e.g. MMU context resets, CPUID
2630 * runtime updates, etc... If that changes, e.g. this flow is moved
2631 * out of the emulator to make it look more like enter_smm(), then
2632 * those side effects need to be explicitly handled for both success
2635 return X86EMUL_CONTINUE;
2638 ctxt->ops->triple_fault(ctxt);
2639 return X86EMUL_CONTINUE;
2643 setup_syscalls_segments(struct desc_struct *cs, struct desc_struct *ss)
2645 cs->l = 0; /* will be adjusted later */
2646 set_desc_base(cs, 0); /* flat segment */
2647 cs->g = 1; /* 4kb granularity */
2648 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2649 cs->type = 0x0b; /* Read, Execute, Accessed */
2651 cs->dpl = 0; /* will be adjusted later */
2656 set_desc_base(ss, 0); /* flat segment */
2657 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2658 ss->g = 1; /* 4kb granularity */
2660 ss->type = 0x03; /* Read/Write, Accessed */
2661 ss->d = 1; /* 32bit stack segment */
2668 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2670 u32 eax, ebx, ecx, edx;
2673 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, true);
2674 return is_guest_vendor_intel(ebx, ecx, edx);
2677 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2679 const struct x86_emulate_ops *ops = ctxt->ops;
2680 u32 eax, ebx, ecx, edx;
2683 * syscall should always be enabled in longmode - so only become
2684 * vendor specific (cpuid) if other modes are active...
2686 if (ctxt->mode == X86EMUL_MODE_PROT64)
2691 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, true);
2693 * remark: Intel CPUs only support "syscall" in 64bit longmode. Also a
2694 * 64bit guest with a 32bit compat-app running will #UD !! While this
2695 * behaviour can be fixed (by emulating) into AMD response - CPUs of
2696 * AMD can't behave like Intel.
2698 if (is_guest_vendor_intel(ebx, ecx, edx))
2701 if (is_guest_vendor_amd(ebx, ecx, edx) ||
2702 is_guest_vendor_hygon(ebx, ecx, edx))
2706 * default: (not Intel, not AMD, not Hygon), apply Intel's
2712 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2714 const struct x86_emulate_ops *ops = ctxt->ops;
2715 struct desc_struct cs, ss;
2720 /* syscall is not available in real mode */
2721 if (ctxt->mode == X86EMUL_MODE_REAL ||
2722 ctxt->mode == X86EMUL_MODE_VM86)
2723 return emulate_ud(ctxt);
2725 if (!(em_syscall_is_enabled(ctxt)))
2726 return emulate_ud(ctxt);
2728 ops->get_msr(ctxt, MSR_EFER, &efer);
2729 if (!(efer & EFER_SCE))
2730 return emulate_ud(ctxt);
2732 setup_syscalls_segments(&cs, &ss);
2733 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2735 cs_sel = (u16)(msr_data & 0xfffc);
2736 ss_sel = (u16)(msr_data + 8);
2738 if (efer & EFER_LMA) {
2742 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2743 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2745 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2746 if (efer & EFER_LMA) {
2747 #ifdef CONFIG_X86_64
2748 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags;
2751 ctxt->mode == X86EMUL_MODE_PROT64 ?
2752 MSR_LSTAR : MSR_CSTAR, &msr_data);
2753 ctxt->_eip = msr_data;
2755 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2756 ctxt->eflags &= ~msr_data;
2757 ctxt->eflags |= X86_EFLAGS_FIXED;
2761 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2762 ctxt->_eip = (u32)msr_data;
2764 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2767 ctxt->tf = (ctxt->eflags & X86_EFLAGS_TF) != 0;
2768 return X86EMUL_CONTINUE;
2771 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2773 const struct x86_emulate_ops *ops = ctxt->ops;
2774 struct desc_struct cs, ss;
2779 ops->get_msr(ctxt, MSR_EFER, &efer);
2780 /* inject #GP if in real mode */
2781 if (ctxt->mode == X86EMUL_MODE_REAL)
2782 return emulate_gp(ctxt, 0);
2785 * Not recognized on AMD in compat mode (but is recognized in legacy
2788 if ((ctxt->mode != X86EMUL_MODE_PROT64) && (efer & EFER_LMA)
2789 && !vendor_intel(ctxt))
2790 return emulate_ud(ctxt);
2792 /* sysenter/sysexit have not been tested in 64bit mode. */
2793 if (ctxt->mode == X86EMUL_MODE_PROT64)
2794 return X86EMUL_UNHANDLEABLE;
2796 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2797 if ((msr_data & 0xfffc) == 0x0)
2798 return emulate_gp(ctxt, 0);
2800 setup_syscalls_segments(&cs, &ss);
2801 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2802 cs_sel = (u16)msr_data & ~SEGMENT_RPL_MASK;
2803 ss_sel = cs_sel + 8;
2804 if (efer & EFER_LMA) {
2809 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2810 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2812 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2813 ctxt->_eip = (efer & EFER_LMA) ? msr_data : (u32)msr_data;
2815 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2816 *reg_write(ctxt, VCPU_REGS_RSP) = (efer & EFER_LMA) ? msr_data :
2818 if (efer & EFER_LMA)
2819 ctxt->mode = X86EMUL_MODE_PROT64;
2821 return X86EMUL_CONTINUE;
2824 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2826 const struct x86_emulate_ops *ops = ctxt->ops;
2827 struct desc_struct cs, ss;
2828 u64 msr_data, rcx, rdx;
2830 u16 cs_sel = 0, ss_sel = 0;
2832 /* inject #GP if in real mode or Virtual 8086 mode */
2833 if (ctxt->mode == X86EMUL_MODE_REAL ||
2834 ctxt->mode == X86EMUL_MODE_VM86)
2835 return emulate_gp(ctxt, 0);
2837 setup_syscalls_segments(&cs, &ss);
2839 if ((ctxt->rex_prefix & 0x8) != 0x0)
2840 usermode = X86EMUL_MODE_PROT64;
2842 usermode = X86EMUL_MODE_PROT32;
2844 rcx = reg_read(ctxt, VCPU_REGS_RCX);
2845 rdx = reg_read(ctxt, VCPU_REGS_RDX);
2849 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2851 case X86EMUL_MODE_PROT32:
2852 cs_sel = (u16)(msr_data + 16);
2853 if ((msr_data & 0xfffc) == 0x0)
2854 return emulate_gp(ctxt, 0);
2855 ss_sel = (u16)(msr_data + 24);
2859 case X86EMUL_MODE_PROT64:
2860 cs_sel = (u16)(msr_data + 32);
2861 if (msr_data == 0x0)
2862 return emulate_gp(ctxt, 0);
2863 ss_sel = cs_sel + 8;
2866 if (emul_is_noncanonical_address(rcx, ctxt) ||
2867 emul_is_noncanonical_address(rdx, ctxt))
2868 return emulate_gp(ctxt, 0);
2871 cs_sel |= SEGMENT_RPL_MASK;
2872 ss_sel |= SEGMENT_RPL_MASK;
2874 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2875 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2878 *reg_write(ctxt, VCPU_REGS_RSP) = rcx;
2880 return X86EMUL_CONTINUE;
2883 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2886 if (ctxt->mode == X86EMUL_MODE_REAL)
2888 if (ctxt->mode == X86EMUL_MODE_VM86)
2890 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
2891 return ctxt->ops->cpl(ctxt) > iopl;
2894 #define VMWARE_PORT_VMPORT (0x5658)
2895 #define VMWARE_PORT_VMRPC (0x5659)
2897 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2900 const struct x86_emulate_ops *ops = ctxt->ops;
2901 struct desc_struct tr_seg;
2904 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2905 unsigned mask = (1 << len) - 1;
2909 * VMware allows access to these ports even if denied
2910 * by TSS I/O permission bitmap. Mimic behavior.
2912 if (enable_vmware_backdoor &&
2913 ((port == VMWARE_PORT_VMPORT) || (port == VMWARE_PORT_VMRPC)))
2916 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2919 if (desc_limit_scaled(&tr_seg) < 103)
2921 base = get_desc_base(&tr_seg);
2922 #ifdef CONFIG_X86_64
2923 base |= ((u64)base3) << 32;
2925 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL, true);
2926 if (r != X86EMUL_CONTINUE)
2928 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2930 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL, true);
2931 if (r != X86EMUL_CONTINUE)
2933 if ((perm >> bit_idx) & mask)
2938 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2944 if (emulator_bad_iopl(ctxt))
2945 if (!emulator_io_port_access_allowed(ctxt, port, len))
2948 ctxt->perm_ok = true;
2953 static void string_registers_quirk(struct x86_emulate_ctxt *ctxt)
2956 * Intel CPUs mask the counter and pointers in quite strange
2957 * manner when ECX is zero due to REP-string optimizations.
2959 #ifdef CONFIG_X86_64
2960 if (ctxt->ad_bytes != 4 || !vendor_intel(ctxt))
2963 *reg_write(ctxt, VCPU_REGS_RCX) = 0;
2966 case 0xa4: /* movsb */
2967 case 0xa5: /* movsd/w */
2968 *reg_rmw(ctxt, VCPU_REGS_RSI) &= (u32)-1;
2970 case 0xaa: /* stosb */
2971 case 0xab: /* stosd/w */
2972 *reg_rmw(ctxt, VCPU_REGS_RDI) &= (u32)-1;
2977 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2978 struct tss_segment_16 *tss)
2980 tss->ip = ctxt->_eip;
2981 tss->flag = ctxt->eflags;
2982 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
2983 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
2984 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
2985 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
2986 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
2987 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
2988 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
2989 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2991 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2992 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2993 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2994 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2995 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2998 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2999 struct tss_segment_16 *tss)
3004 ctxt->_eip = tss->ip;
3005 ctxt->eflags = tss->flag | 2;
3006 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
3007 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
3008 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
3009 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
3010 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
3011 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
3012 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
3013 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
3016 * SDM says that segment selectors are loaded before segment
3019 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
3020 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
3021 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
3022 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
3023 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
3028 * Now load segment descriptors. If fault happens at this stage
3029 * it is handled in a context of new task
3031 ret = __load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR, cpl,
3032 X86_TRANSFER_TASK_SWITCH, NULL);
3033 if (ret != X86EMUL_CONTINUE)
3035 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
3036 X86_TRANSFER_TASK_SWITCH, NULL);
3037 if (ret != X86EMUL_CONTINUE)
3039 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
3040 X86_TRANSFER_TASK_SWITCH, NULL);
3041 if (ret != X86EMUL_CONTINUE)
3043 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
3044 X86_TRANSFER_TASK_SWITCH, NULL);
3045 if (ret != X86EMUL_CONTINUE)
3047 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
3048 X86_TRANSFER_TASK_SWITCH, NULL);
3049 if (ret != X86EMUL_CONTINUE)
3052 return X86EMUL_CONTINUE;
3055 static int task_switch_16(struct x86_emulate_ctxt *ctxt, u16 old_tss_sel,
3056 ulong old_tss_base, struct desc_struct *new_desc)
3058 struct tss_segment_16 tss_seg;
3060 u32 new_tss_base = get_desc_base(new_desc);
3062 ret = linear_read_system(ctxt, old_tss_base, &tss_seg, sizeof(tss_seg));
3063 if (ret != X86EMUL_CONTINUE)
3066 save_state_to_tss16(ctxt, &tss_seg);
3068 ret = linear_write_system(ctxt, old_tss_base, &tss_seg, sizeof(tss_seg));
3069 if (ret != X86EMUL_CONTINUE)
3072 ret = linear_read_system(ctxt, new_tss_base, &tss_seg, sizeof(tss_seg));
3073 if (ret != X86EMUL_CONTINUE)
3076 if (old_tss_sel != 0xffff) {
3077 tss_seg.prev_task_link = old_tss_sel;
3079 ret = linear_write_system(ctxt, new_tss_base,
3080 &tss_seg.prev_task_link,
3081 sizeof(tss_seg.prev_task_link));
3082 if (ret != X86EMUL_CONTINUE)
3086 return load_state_from_tss16(ctxt, &tss_seg);
3089 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
3090 struct tss_segment_32 *tss)
3092 /* CR3 and ldt selector are not saved intentionally */
3093 tss->eip = ctxt->_eip;
3094 tss->eflags = ctxt->eflags;
3095 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
3096 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
3097 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
3098 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
3099 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
3100 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
3101 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
3102 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
3104 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
3105 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
3106 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
3107 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
3108 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
3109 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
3112 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
3113 struct tss_segment_32 *tss)
3118 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
3119 return emulate_gp(ctxt, 0);
3120 ctxt->_eip = tss->eip;
3121 ctxt->eflags = tss->eflags | 2;
3123 /* General purpose registers */
3124 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
3125 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
3126 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
3127 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
3128 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
3129 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
3130 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
3131 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
3134 * SDM says that segment selectors are loaded before segment
3135 * descriptors. This is important because CPL checks will
3138 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
3139 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
3140 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
3141 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
3142 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
3143 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
3144 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
3147 * If we're switching between Protected Mode and VM86, we need to make
3148 * sure to update the mode before loading the segment descriptors so
3149 * that the selectors are interpreted correctly.
3151 if (ctxt->eflags & X86_EFLAGS_VM) {
3152 ctxt->mode = X86EMUL_MODE_VM86;
3155 ctxt->mode = X86EMUL_MODE_PROT32;
3160 * Now load segment descriptors. If fault happens at this stage
3161 * it is handled in a context of new task
3163 ret = __load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR,
3164 cpl, X86_TRANSFER_TASK_SWITCH, NULL);
3165 if (ret != X86EMUL_CONTINUE)
3167 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
3168 X86_TRANSFER_TASK_SWITCH, NULL);
3169 if (ret != X86EMUL_CONTINUE)
3171 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
3172 X86_TRANSFER_TASK_SWITCH, NULL);
3173 if (ret != X86EMUL_CONTINUE)
3175 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
3176 X86_TRANSFER_TASK_SWITCH, NULL);
3177 if (ret != X86EMUL_CONTINUE)
3179 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
3180 X86_TRANSFER_TASK_SWITCH, NULL);
3181 if (ret != X86EMUL_CONTINUE)
3183 ret = __load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS, cpl,
3184 X86_TRANSFER_TASK_SWITCH, NULL);
3185 if (ret != X86EMUL_CONTINUE)
3187 ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl,
3188 X86_TRANSFER_TASK_SWITCH, NULL);
3193 static int task_switch_32(struct x86_emulate_ctxt *ctxt, u16 old_tss_sel,
3194 ulong old_tss_base, struct desc_struct *new_desc)
3196 struct tss_segment_32 tss_seg;
3198 u32 new_tss_base = get_desc_base(new_desc);
3199 u32 eip_offset = offsetof(struct tss_segment_32, eip);
3200 u32 ldt_sel_offset = offsetof(struct tss_segment_32, ldt_selector);
3202 ret = linear_read_system(ctxt, old_tss_base, &tss_seg, sizeof(tss_seg));
3203 if (ret != X86EMUL_CONTINUE)
3206 save_state_to_tss32(ctxt, &tss_seg);
3208 /* Only GP registers and segment selectors are saved */
3209 ret = linear_write_system(ctxt, old_tss_base + eip_offset, &tss_seg.eip,
3210 ldt_sel_offset - eip_offset);
3211 if (ret != X86EMUL_CONTINUE)
3214 ret = linear_read_system(ctxt, new_tss_base, &tss_seg, sizeof(tss_seg));
3215 if (ret != X86EMUL_CONTINUE)
3218 if (old_tss_sel != 0xffff) {
3219 tss_seg.prev_task_link = old_tss_sel;
3221 ret = linear_write_system(ctxt, new_tss_base,
3222 &tss_seg.prev_task_link,
3223 sizeof(tss_seg.prev_task_link));
3224 if (ret != X86EMUL_CONTINUE)
3228 return load_state_from_tss32(ctxt, &tss_seg);
3231 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
3232 u16 tss_selector, int idt_index, int reason,
3233 bool has_error_code, u32 error_code)
3235 const struct x86_emulate_ops *ops = ctxt->ops;
3236 struct desc_struct curr_tss_desc, next_tss_desc;
3238 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
3239 ulong old_tss_base =
3240 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
3242 ulong desc_addr, dr7;
3244 /* FIXME: old_tss_base == ~0 ? */
3246 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
3247 if (ret != X86EMUL_CONTINUE)
3249 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
3250 if (ret != X86EMUL_CONTINUE)
3253 /* FIXME: check that next_tss_desc is tss */
3256 * Check privileges. The three cases are task switch caused by...
3258 * 1. jmp/call/int to task gate: Check against DPL of the task gate
3259 * 2. Exception/IRQ/iret: No check is performed
3260 * 3. jmp/call to TSS/task-gate: No check is performed since the
3261 * hardware checks it before exiting.
3263 if (reason == TASK_SWITCH_GATE) {
3264 if (idt_index != -1) {
3265 /* Software interrupts */
3266 struct desc_struct task_gate_desc;
3269 ret = read_interrupt_descriptor(ctxt, idt_index,
3271 if (ret != X86EMUL_CONTINUE)
3274 dpl = task_gate_desc.dpl;
3275 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
3276 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
3280 desc_limit = desc_limit_scaled(&next_tss_desc);
3281 if (!next_tss_desc.p ||
3282 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
3283 desc_limit < 0x2b)) {
3284 return emulate_ts(ctxt, tss_selector & 0xfffc);
3287 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
3288 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
3289 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
3292 if (reason == TASK_SWITCH_IRET)
3293 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
3295 /* set back link to prev task only if NT bit is set in eflags
3296 note that old_tss_sel is not used after this point */
3297 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
3298 old_tss_sel = 0xffff;
3300 if (next_tss_desc.type & 8)
3301 ret = task_switch_32(ctxt, old_tss_sel, old_tss_base, &next_tss_desc);
3303 ret = task_switch_16(ctxt, old_tss_sel,
3304 old_tss_base, &next_tss_desc);
3305 if (ret != X86EMUL_CONTINUE)
3308 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
3309 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
3311 if (reason != TASK_SWITCH_IRET) {
3312 next_tss_desc.type |= (1 << 1); /* set busy flag */
3313 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
3316 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
3317 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
3319 if (has_error_code) {
3320 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
3321 ctxt->lock_prefix = 0;
3322 ctxt->src.val = (unsigned long) error_code;
3323 ret = em_push(ctxt);
3326 ops->get_dr(ctxt, 7, &dr7);
3327 ops->set_dr(ctxt, 7, dr7 & ~(DR_LOCAL_ENABLE_MASK | DR_LOCAL_SLOWDOWN));
3332 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
3333 u16 tss_selector, int idt_index, int reason,
3334 bool has_error_code, u32 error_code)
3338 invalidate_registers(ctxt);
3339 ctxt->_eip = ctxt->eip;
3340 ctxt->dst.type = OP_NONE;
3342 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
3343 has_error_code, error_code);
3345 if (rc == X86EMUL_CONTINUE) {
3346 ctxt->eip = ctxt->_eip;
3347 writeback_registers(ctxt);
3350 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
3353 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
3356 int df = (ctxt->eflags & X86_EFLAGS_DF) ? -op->count : op->count;
3358 register_address_increment(ctxt, reg, df * op->bytes);
3359 op->addr.mem.ea = register_address(ctxt, reg);
3362 static int em_das(struct x86_emulate_ctxt *ctxt)
3365 bool af, cf, old_cf;
3367 cf = ctxt->eflags & X86_EFLAGS_CF;
3373 af = ctxt->eflags & X86_EFLAGS_AF;
3374 if ((al & 0x0f) > 9 || af) {
3376 cf = old_cf | (al >= 250);
3381 if (old_al > 0x99 || old_cf) {
3387 /* Set PF, ZF, SF */
3388 ctxt->src.type = OP_IMM;
3390 ctxt->src.bytes = 1;
3391 fastop(ctxt, em_or);
3392 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
3394 ctxt->eflags |= X86_EFLAGS_CF;
3396 ctxt->eflags |= X86_EFLAGS_AF;
3397 return X86EMUL_CONTINUE;
3400 static int em_aam(struct x86_emulate_ctxt *ctxt)
3404 if (ctxt->src.val == 0)
3405 return emulate_de(ctxt);
3407 al = ctxt->dst.val & 0xff;
3408 ah = al / ctxt->src.val;
3409 al %= ctxt->src.val;
3411 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);
3413 /* Set PF, ZF, SF */
3414 ctxt->src.type = OP_IMM;
3416 ctxt->src.bytes = 1;
3417 fastop(ctxt, em_or);
3419 return X86EMUL_CONTINUE;
3422 static int em_aad(struct x86_emulate_ctxt *ctxt)
3424 u8 al = ctxt->dst.val & 0xff;
3425 u8 ah = (ctxt->dst.val >> 8) & 0xff;
3427 al = (al + (ah * ctxt->src.val)) & 0xff;
3429 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
3431 /* Set PF, ZF, SF */
3432 ctxt->src.type = OP_IMM;
3434 ctxt->src.bytes = 1;
3435 fastop(ctxt, em_or);
3437 return X86EMUL_CONTINUE;
3440 static int em_call(struct x86_emulate_ctxt *ctxt)
3443 long rel = ctxt->src.val;
3445 ctxt->src.val = (unsigned long)ctxt->_eip;
3446 rc = jmp_rel(ctxt, rel);
3447 if (rc != X86EMUL_CONTINUE)
3449 return em_push(ctxt);
3452 static int em_call_far(struct x86_emulate_ctxt *ctxt)
3457 struct desc_struct old_desc, new_desc;
3458 const struct x86_emulate_ops *ops = ctxt->ops;
3459 int cpl = ctxt->ops->cpl(ctxt);
3460 enum x86emul_mode prev_mode = ctxt->mode;
3462 old_eip = ctxt->_eip;
3463 ops->get_segment(ctxt, &old_cs, &old_desc, NULL, VCPU_SREG_CS);
3465 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
3466 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
3467 X86_TRANSFER_CALL_JMP, &new_desc);
3468 if (rc != X86EMUL_CONTINUE)
3471 rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
3472 if (rc != X86EMUL_CONTINUE)
3475 ctxt->src.val = old_cs;
3477 if (rc != X86EMUL_CONTINUE)
3480 ctxt->src.val = old_eip;
3482 /* If we failed, we tainted the memory, but the very least we should
3484 if (rc != X86EMUL_CONTINUE) {
3485 pr_warn_once("faulting far call emulation tainted memory\n");
3490 ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS);
3491 ctxt->mode = prev_mode;
3496 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
3501 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
3502 if (rc != X86EMUL_CONTINUE)
3504 rc = assign_eip_near(ctxt, eip);
3505 if (rc != X86EMUL_CONTINUE)
3507 rsp_increment(ctxt, ctxt->src.val);
3508 return X86EMUL_CONTINUE;
3511 static int em_xchg(struct x86_emulate_ctxt *ctxt)
3513 /* Write back the register source. */
3514 ctxt->src.val = ctxt->dst.val;
3515 write_register_operand(&ctxt->src);
3517 /* Write back the memory destination with implicit LOCK prefix. */
3518 ctxt->dst.val = ctxt->src.orig_val;
3519 ctxt->lock_prefix = 1;
3520 return X86EMUL_CONTINUE;
3523 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
3525 ctxt->dst.val = ctxt->src2.val;
3526 return fastop(ctxt, em_imul);
3529 static int em_cwd(struct x86_emulate_ctxt *ctxt)
3531 ctxt->dst.type = OP_REG;
3532 ctxt->dst.bytes = ctxt->src.bytes;
3533 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
3534 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
3536 return X86EMUL_CONTINUE;
3539 static int em_rdpid(struct x86_emulate_ctxt *ctxt)
3543 if (!ctxt->ops->guest_has_rdpid(ctxt))
3544 return emulate_ud(ctxt);
3546 ctxt->ops->get_msr(ctxt, MSR_TSC_AUX, &tsc_aux);
3547 ctxt->dst.val = tsc_aux;
3548 return X86EMUL_CONTINUE;
3551 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
3555 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
3556 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
3557 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
3558 return X86EMUL_CONTINUE;
3561 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
3565 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
3566 return emulate_gp(ctxt, 0);
3567 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
3568 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
3569 return X86EMUL_CONTINUE;
3572 static int em_mov(struct x86_emulate_ctxt *ctxt)
3574 memcpy(ctxt->dst.valptr, ctxt->src.valptr, sizeof(ctxt->src.valptr));
3575 return X86EMUL_CONTINUE;
3578 static int em_movbe(struct x86_emulate_ctxt *ctxt)
3582 if (!ctxt->ops->guest_has_movbe(ctxt))
3583 return emulate_ud(ctxt);
3585 switch (ctxt->op_bytes) {
3588 * From MOVBE definition: "...When the operand size is 16 bits,
3589 * the upper word of the destination register remains unchanged
3592 * Both casting ->valptr and ->val to u16 breaks strict aliasing
3593 * rules so we have to do the operation almost per hand.
3595 tmp = (u16)ctxt->src.val;
3596 ctxt->dst.val &= ~0xffffUL;
3597 ctxt->dst.val |= (unsigned long)swab16(tmp);
3600 ctxt->dst.val = swab32((u32)ctxt->src.val);
3603 ctxt->dst.val = swab64(ctxt->src.val);
3608 return X86EMUL_CONTINUE;
3611 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3613 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3614 return emulate_gp(ctxt, 0);
3616 /* Disable writeback. */
3617 ctxt->dst.type = OP_NONE;
3618 return X86EMUL_CONTINUE;
3621 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3625 if (ctxt->mode == X86EMUL_MODE_PROT64)
3626 val = ctxt->src.val & ~0ULL;
3628 val = ctxt->src.val & ~0U;
3630 /* #UD condition is already handled. */
3631 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3632 return emulate_gp(ctxt, 0);
3634 /* Disable writeback. */
3635 ctxt->dst.type = OP_NONE;
3636 return X86EMUL_CONTINUE;
3639 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3641 u64 msr_index = reg_read(ctxt, VCPU_REGS_RCX);
3645 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3646 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3647 r = ctxt->ops->set_msr_with_filter(ctxt, msr_index, msr_data);
3649 if (r == X86EMUL_IO_NEEDED)
3653 return emulate_gp(ctxt, 0);
3655 return r < 0 ? X86EMUL_UNHANDLEABLE : X86EMUL_CONTINUE;
3658 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3660 u64 msr_index = reg_read(ctxt, VCPU_REGS_RCX);
3664 r = ctxt->ops->get_msr_with_filter(ctxt, msr_index, &msr_data);
3666 if (r == X86EMUL_IO_NEEDED)
3670 return emulate_gp(ctxt, 0);
3672 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3673 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3674 return X86EMUL_CONTINUE;
3677 static int em_store_sreg(struct x86_emulate_ctxt *ctxt, int segment)
3679 if (segment > VCPU_SREG_GS &&
3680 (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) &&
3681 ctxt->ops->cpl(ctxt) > 0)
3682 return emulate_gp(ctxt, 0);
3684 ctxt->dst.val = get_segment_selector(ctxt, segment);
3685 if (ctxt->dst.bytes == 4 && ctxt->dst.type == OP_MEM)
3686 ctxt->dst.bytes = 2;
3687 return X86EMUL_CONTINUE;
3690 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3692 if (ctxt->modrm_reg > VCPU_SREG_GS)
3693 return emulate_ud(ctxt);
3695 return em_store_sreg(ctxt, ctxt->modrm_reg);
3698 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3700 u16 sel = ctxt->src.val;
3702 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3703 return emulate_ud(ctxt);
3705 if (ctxt->modrm_reg == VCPU_SREG_SS)
3706 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3708 /* Disable writeback. */
3709 ctxt->dst.type = OP_NONE;
3710 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3713 static int em_sldt(struct x86_emulate_ctxt *ctxt)
3715 return em_store_sreg(ctxt, VCPU_SREG_LDTR);
3718 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3720 u16 sel = ctxt->src.val;
3722 /* Disable writeback. */
3723 ctxt->dst.type = OP_NONE;
3724 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3727 static int em_str(struct x86_emulate_ctxt *ctxt)
3729 return em_store_sreg(ctxt, VCPU_SREG_TR);
3732 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3734 u16 sel = ctxt->src.val;
3736 /* Disable writeback. */
3737 ctxt->dst.type = OP_NONE;
3738 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3741 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3746 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3747 if (rc == X86EMUL_CONTINUE)
3748 ctxt->ops->invlpg(ctxt, linear);
3749 /* Disable writeback. */
3750 ctxt->dst.type = OP_NONE;
3751 return X86EMUL_CONTINUE;
3754 static int em_clts(struct x86_emulate_ctxt *ctxt)
3758 cr0 = ctxt->ops->get_cr(ctxt, 0);
3760 ctxt->ops->set_cr(ctxt, 0, cr0);
3761 return X86EMUL_CONTINUE;
3764 static int em_hypercall(struct x86_emulate_ctxt *ctxt)
3766 int rc = ctxt->ops->fix_hypercall(ctxt);
3768 if (rc != X86EMUL_CONTINUE)
3771 /* Let the processor re-execute the fixed hypercall */
3772 ctxt->_eip = ctxt->eip;
3773 /* Disable writeback. */
3774 ctxt->dst.type = OP_NONE;
3775 return X86EMUL_CONTINUE;
3778 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3779 void (*get)(struct x86_emulate_ctxt *ctxt,
3780 struct desc_ptr *ptr))
3782 struct desc_ptr desc_ptr;
3784 if ((ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) &&
3785 ctxt->ops->cpl(ctxt) > 0)
3786 return emulate_gp(ctxt, 0);
3788 if (ctxt->mode == X86EMUL_MODE_PROT64)
3790 get(ctxt, &desc_ptr);
3791 if (ctxt->op_bytes == 2) {
3793 desc_ptr.address &= 0x00ffffff;
3795 /* Disable writeback. */
3796 ctxt->dst.type = OP_NONE;
3797 return segmented_write_std(ctxt, ctxt->dst.addr.mem,
3798 &desc_ptr, 2 + ctxt->op_bytes);
3801 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3803 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3806 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3808 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3811 static int em_lgdt_lidt(struct x86_emulate_ctxt *ctxt, bool lgdt)
3813 struct desc_ptr desc_ptr;
3816 if (ctxt->mode == X86EMUL_MODE_PROT64)
3818 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3819 &desc_ptr.size, &desc_ptr.address,
3821 if (rc != X86EMUL_CONTINUE)
3823 if (ctxt->mode == X86EMUL_MODE_PROT64 &&
3824 emul_is_noncanonical_address(desc_ptr.address, ctxt))
3825 return emulate_gp(ctxt, 0);
3827 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3829 ctxt->ops->set_idt(ctxt, &desc_ptr);
3830 /* Disable writeback. */
3831 ctxt->dst.type = OP_NONE;
3832 return X86EMUL_CONTINUE;
3835 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3837 return em_lgdt_lidt(ctxt, true);
3840 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3842 return em_lgdt_lidt(ctxt, false);
3845 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3847 if ((ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) &&
3848 ctxt->ops->cpl(ctxt) > 0)
3849 return emulate_gp(ctxt, 0);
3851 if (ctxt->dst.type == OP_MEM)
3852 ctxt->dst.bytes = 2;
3853 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3854 return X86EMUL_CONTINUE;
3857 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3859 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3860 | (ctxt->src.val & 0x0f));
3861 ctxt->dst.type = OP_NONE;
3862 return X86EMUL_CONTINUE;
3865 static int em_loop(struct x86_emulate_ctxt *ctxt)
3867 int rc = X86EMUL_CONTINUE;
3869 register_address_increment(ctxt, VCPU_REGS_RCX, -1);
3870 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3871 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3872 rc = jmp_rel(ctxt, ctxt->src.val);
3877 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3879 int rc = X86EMUL_CONTINUE;
3881 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3882 rc = jmp_rel(ctxt, ctxt->src.val);
3887 static int em_in(struct x86_emulate_ctxt *ctxt)
3889 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3891 return X86EMUL_IO_NEEDED;
3893 return X86EMUL_CONTINUE;
3896 static int em_out(struct x86_emulate_ctxt *ctxt)
3898 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3900 /* Disable writeback. */
3901 ctxt->dst.type = OP_NONE;
3902 return X86EMUL_CONTINUE;
3905 static int em_cli(struct x86_emulate_ctxt *ctxt)
3907 if (emulator_bad_iopl(ctxt))
3908 return emulate_gp(ctxt, 0);
3910 ctxt->eflags &= ~X86_EFLAGS_IF;
3911 return X86EMUL_CONTINUE;
3914 static int em_sti(struct x86_emulate_ctxt *ctxt)
3916 if (emulator_bad_iopl(ctxt))
3917 return emulate_gp(ctxt, 0);
3919 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3920 ctxt->eflags |= X86_EFLAGS_IF;
3921 return X86EMUL_CONTINUE;
3924 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3926 u32 eax, ebx, ecx, edx;
3929 ctxt->ops->get_msr(ctxt, MSR_MISC_FEATURES_ENABLES, &msr);
3930 if (msr & MSR_MISC_FEATURES_ENABLES_CPUID_FAULT &&
3931 ctxt->ops->cpl(ctxt)) {
3932 return emulate_gp(ctxt, 0);
3935 eax = reg_read(ctxt, VCPU_REGS_RAX);
3936 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3937 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
3938 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3939 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3940 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3941 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3942 return X86EMUL_CONTINUE;
3945 static int em_sahf(struct x86_emulate_ctxt *ctxt)
3949 flags = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF |
3951 flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8;
3953 ctxt->eflags &= ~0xffUL;
3954 ctxt->eflags |= flags | X86_EFLAGS_FIXED;
3955 return X86EMUL_CONTINUE;
3958 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3960 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3961 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3962 return X86EMUL_CONTINUE;
3965 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3967 switch (ctxt->op_bytes) {
3968 #ifdef CONFIG_X86_64
3970 asm("bswap %0" : "+r"(ctxt->dst.val));
3974 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3977 return X86EMUL_CONTINUE;
3980 static int em_clflush(struct x86_emulate_ctxt *ctxt)
3982 /* emulating clflush regardless of cpuid */
3983 return X86EMUL_CONTINUE;
3986 static int em_clflushopt(struct x86_emulate_ctxt *ctxt)
3988 /* emulating clflushopt regardless of cpuid */
3989 return X86EMUL_CONTINUE;
3992 static int em_movsxd(struct x86_emulate_ctxt *ctxt)
3994 ctxt->dst.val = (s32) ctxt->src.val;
3995 return X86EMUL_CONTINUE;
3998 static int check_fxsr(struct x86_emulate_ctxt *ctxt)
4000 if (!ctxt->ops->guest_has_fxsr(ctxt))
4001 return emulate_ud(ctxt);
4003 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
4004 return emulate_nm(ctxt);
4007 * Don't emulate a case that should never be hit, instead of working
4008 * around a lack of fxsave64/fxrstor64 on old compilers.
4010 if (ctxt->mode >= X86EMUL_MODE_PROT64)
4011 return X86EMUL_UNHANDLEABLE;
4013 return X86EMUL_CONTINUE;
4017 * Hardware doesn't save and restore XMM 0-7 without CR4.OSFXSR, but does save
4018 * and restore MXCSR.
4020 static size_t __fxstate_size(int nregs)
4022 return offsetof(struct fxregs_state, xmm_space[0]) + nregs * 16;
4025 static inline size_t fxstate_size(struct x86_emulate_ctxt *ctxt)
4028 if (ctxt->mode == X86EMUL_MODE_PROT64)
4029 return __fxstate_size(16);
4031 cr4_osfxsr = ctxt->ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR;
4032 return __fxstate_size(cr4_osfxsr ? 8 : 0);
4036 * FXSAVE and FXRSTOR have 4 different formats depending on execution mode,
4039 * - like (1), but FIP and FDP (foo) are only 16 bit. At least Intel CPUs
4040 * preserve whole 32 bit values, though, so (1) and (2) are the same wrt.
4042 * 3) 64-bit mode with REX.W prefix
4043 * - like (2), but XMM 8-15 are being saved and restored
4044 * 4) 64-bit mode without REX.W prefix
4045 * - like (3), but FIP and FDP are 64 bit
4047 * Emulation uses (3) for (1) and (2) and preserves XMM 8-15 to reach the
4048 * desired result. (4) is not emulated.
4050 * Note: Guest and host CPUID.(EAX=07H,ECX=0H):EBX[bit 13] (deprecate FPU CS
4051 * and FPU DS) should match.
4053 static int em_fxsave(struct x86_emulate_ctxt *ctxt)
4055 struct fxregs_state fx_state;
4058 rc = check_fxsr(ctxt);
4059 if (rc != X86EMUL_CONTINUE)
4064 rc = asm_safe("fxsave %[fx]", , [fx] "+m"(fx_state));
4068 if (rc != X86EMUL_CONTINUE)
4071 return segmented_write_std(ctxt, ctxt->memop.addr.mem, &fx_state,
4072 fxstate_size(ctxt));
4076 * FXRSTOR might restore XMM registers not provided by the guest. Fill
4077 * in the host registers (via FXSAVE) instead, so they won't be modified.
4078 * (preemption has to stay disabled until FXRSTOR).
4080 * Use noinline to keep the stack for other functions called by callers small.
4082 static noinline int fxregs_fixup(struct fxregs_state *fx_state,
4083 const size_t used_size)
4085 struct fxregs_state fx_tmp;
4088 rc = asm_safe("fxsave %[fx]", , [fx] "+m"(fx_tmp));
4089 memcpy((void *)fx_state + used_size, (void *)&fx_tmp + used_size,
4090 __fxstate_size(16) - used_size);
4095 static int em_fxrstor(struct x86_emulate_ctxt *ctxt)
4097 struct fxregs_state fx_state;
4101 rc = check_fxsr(ctxt);
4102 if (rc != X86EMUL_CONTINUE)
4105 size = fxstate_size(ctxt);
4106 rc = segmented_read_std(ctxt, ctxt->memop.addr.mem, &fx_state, size);
4107 if (rc != X86EMUL_CONTINUE)
4112 if (size < __fxstate_size(16)) {
4113 rc = fxregs_fixup(&fx_state, size);
4114 if (rc != X86EMUL_CONTINUE)
4118 if (fx_state.mxcsr >> 16) {
4119 rc = emulate_gp(ctxt, 0);
4123 if (rc == X86EMUL_CONTINUE)
4124 rc = asm_safe("fxrstor %[fx]", : [fx] "m"(fx_state));
4132 static int em_xsetbv(struct x86_emulate_ctxt *ctxt)
4136 eax = reg_read(ctxt, VCPU_REGS_RAX);
4137 edx = reg_read(ctxt, VCPU_REGS_RDX);
4138 ecx = reg_read(ctxt, VCPU_REGS_RCX);
4140 if (ctxt->ops->set_xcr(ctxt, ecx, ((u64)edx << 32) | eax))
4141 return emulate_gp(ctxt, 0);
4143 return X86EMUL_CONTINUE;
4146 static bool valid_cr(int nr)
4158 static int check_cr_access(struct x86_emulate_ctxt *ctxt)
4160 if (!valid_cr(ctxt->modrm_reg))
4161 return emulate_ud(ctxt);
4163 return X86EMUL_CONTINUE;
4166 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
4170 ctxt->ops->get_dr(ctxt, 7, &dr7);
4172 /* Check if DR7.Global_Enable is set */
4173 return dr7 & (1 << 13);
4176 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
4178 int dr = ctxt->modrm_reg;
4182 return emulate_ud(ctxt);
4184 cr4 = ctxt->ops->get_cr(ctxt, 4);
4185 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
4186 return emulate_ud(ctxt);
4188 if (check_dr7_gd(ctxt)) {
4191 ctxt->ops->get_dr(ctxt, 6, &dr6);
4192 dr6 &= ~DR_TRAP_BITS;
4193 dr6 |= DR6_BD | DR6_ACTIVE_LOW;
4194 ctxt->ops->set_dr(ctxt, 6, dr6);
4195 return emulate_db(ctxt);
4198 return X86EMUL_CONTINUE;
4201 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
4203 u64 new_val = ctxt->src.val64;
4204 int dr = ctxt->modrm_reg;
4206 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
4207 return emulate_gp(ctxt, 0);
4209 return check_dr_read(ctxt);
4212 static int check_svme(struct x86_emulate_ctxt *ctxt)
4216 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4218 if (!(efer & EFER_SVME))
4219 return emulate_ud(ctxt);
4221 return X86EMUL_CONTINUE;
4224 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
4226 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
4228 /* Valid physical address? */
4229 if (rax & 0xffff000000000000ULL)
4230 return emulate_gp(ctxt, 0);
4232 return check_svme(ctxt);
4235 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
4237 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
4239 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
4240 return emulate_gp(ctxt, 0);
4242 return X86EMUL_CONTINUE;
4245 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
4247 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
4248 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
4251 * VMware allows access to these Pseduo-PMCs even when read via RDPMC
4252 * in Ring3 when CR4.PCE=0.
4254 if (enable_vmware_backdoor && is_vmware_backdoor_pmc(rcx))
4255 return X86EMUL_CONTINUE;
4258 * If CR4.PCE is set, the SDM requires CPL=0 or CR0.PE=0. The CR0.PE
4259 * check however is unnecessary because CPL is always 0 outside
4262 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
4263 ctxt->ops->check_pmc(ctxt, rcx))
4264 return emulate_gp(ctxt, 0);
4266 return X86EMUL_CONTINUE;
4269 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
4271 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
4272 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
4273 return emulate_gp(ctxt, 0);
4275 return X86EMUL_CONTINUE;
4278 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
4280 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
4281 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
4282 return emulate_gp(ctxt, 0);
4284 return X86EMUL_CONTINUE;
4287 #define D(_y) { .flags = (_y) }
4288 #define DI(_y, _i) { .flags = (_y)|Intercept, .intercept = x86_intercept_##_i }
4289 #define DIP(_y, _i, _p) { .flags = (_y)|Intercept|CheckPerm, \
4290 .intercept = x86_intercept_##_i, .check_perm = (_p) }
4291 #define N D(NotImpl)
4292 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
4293 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
4294 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
4295 #define ID(_f, _i) { .flags = ((_f) | InstrDual | ModRM), .u.idual = (_i) }
4296 #define MD(_f, _m) { .flags = ((_f) | ModeDual), .u.mdual = (_m) }
4297 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
4298 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
4299 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
4300 #define II(_f, _e, _i) \
4301 { .flags = (_f)|Intercept, .u.execute = (_e), .intercept = x86_intercept_##_i }
4302 #define IIP(_f, _e, _i, _p) \
4303 { .flags = (_f)|Intercept|CheckPerm, .u.execute = (_e), \
4304 .intercept = x86_intercept_##_i, .check_perm = (_p) }
4305 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
4307 #define D2bv(_f) D((_f) | ByteOp), D(_f)
4308 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
4309 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
4310 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
4311 #define I2bvIP(_f, _e, _i, _p) \
4312 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
4314 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
4315 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
4316 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
4318 static const struct opcode group7_rm0[] = {
4320 I(SrcNone | Priv | EmulateOnUD, em_hypercall),
4324 static const struct opcode group7_rm1[] = {
4325 DI(SrcNone | Priv, monitor),
4326 DI(SrcNone | Priv, mwait),
4330 static const struct opcode group7_rm2[] = {
4332 II(ImplicitOps | Priv, em_xsetbv, xsetbv),
4336 static const struct opcode group7_rm3[] = {
4337 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
4338 II(SrcNone | Prot | EmulateOnUD, em_hypercall, vmmcall),
4339 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
4340 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
4341 DIP(SrcNone | Prot | Priv, stgi, check_svme),
4342 DIP(SrcNone | Prot | Priv, clgi, check_svme),
4343 DIP(SrcNone | Prot | Priv, skinit, check_svme),
4344 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
4347 static const struct opcode group7_rm7[] = {
4349 DIP(SrcNone, rdtscp, check_rdtsc),
4353 static const struct opcode group1[] = {
4355 F(Lock | PageTable, em_or),
4358 F(Lock | PageTable, em_and),
4364 static const struct opcode group1A[] = {
4365 I(DstMem | SrcNone | Mov | Stack | IncSP | TwoMemOp, em_pop), N, N, N, N, N, N, N,
4368 static const struct opcode group2[] = {
4369 F(DstMem | ModRM, em_rol),
4370 F(DstMem | ModRM, em_ror),
4371 F(DstMem | ModRM, em_rcl),
4372 F(DstMem | ModRM, em_rcr),
4373 F(DstMem | ModRM, em_shl),
4374 F(DstMem | ModRM, em_shr),
4375 F(DstMem | ModRM, em_shl),
4376 F(DstMem | ModRM, em_sar),
4379 static const struct opcode group3[] = {
4380 F(DstMem | SrcImm | NoWrite, em_test),
4381 F(DstMem | SrcImm | NoWrite, em_test),
4382 F(DstMem | SrcNone | Lock, em_not),
4383 F(DstMem | SrcNone | Lock, em_neg),
4384 F(DstXacc | Src2Mem, em_mul_ex),
4385 F(DstXacc | Src2Mem, em_imul_ex),
4386 F(DstXacc | Src2Mem, em_div_ex),
4387 F(DstXacc | Src2Mem, em_idiv_ex),
4390 static const struct opcode group4[] = {
4391 F(ByteOp | DstMem | SrcNone | Lock, em_inc),
4392 F(ByteOp | DstMem | SrcNone | Lock, em_dec),
4396 static const struct opcode group5[] = {
4397 F(DstMem | SrcNone | Lock, em_inc),
4398 F(DstMem | SrcNone | Lock, em_dec),
4399 I(SrcMem | NearBranch | IsBranch, em_call_near_abs),
4400 I(SrcMemFAddr | ImplicitOps | IsBranch, em_call_far),
4401 I(SrcMem | NearBranch | IsBranch, em_jmp_abs),
4402 I(SrcMemFAddr | ImplicitOps | IsBranch, em_jmp_far),
4403 I(SrcMem | Stack | TwoMemOp, em_push), D(Undefined),
4406 static const struct opcode group6[] = {
4407 II(Prot | DstMem, em_sldt, sldt),
4408 II(Prot | DstMem, em_str, str),
4409 II(Prot | Priv | SrcMem16, em_lldt, lldt),
4410 II(Prot | Priv | SrcMem16, em_ltr, ltr),
4414 static const struct group_dual group7 = { {
4415 II(Mov | DstMem, em_sgdt, sgdt),
4416 II(Mov | DstMem, em_sidt, sidt),
4417 II(SrcMem | Priv, em_lgdt, lgdt),
4418 II(SrcMem | Priv, em_lidt, lidt),
4419 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
4420 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
4421 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
4427 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
4428 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
4432 static const struct opcode group8[] = {
4434 F(DstMem | SrcImmByte | NoWrite, em_bt),
4435 F(DstMem | SrcImmByte | Lock | PageTable, em_bts),
4436 F(DstMem | SrcImmByte | Lock, em_btr),
4437 F(DstMem | SrcImmByte | Lock | PageTable, em_btc),
4441 * The "memory" destination is actually always a register, since we come
4442 * from the register case of group9.
4444 static const struct gprefix pfx_0f_c7_7 = {
4445 N, N, N, II(DstMem | ModRM | Op3264 | EmulateOnUD, em_rdpid, rdpid),
4449 static const struct group_dual group9 = { {
4450 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
4452 N, N, N, N, N, N, N,
4453 GP(0, &pfx_0f_c7_7),
4456 static const struct opcode group11[] = {
4457 I(DstMem | SrcImm | Mov | PageTable, em_mov),
4461 static const struct gprefix pfx_0f_ae_7 = {
4462 I(SrcMem | ByteOp, em_clflush), I(SrcMem | ByteOp, em_clflushopt), N, N,
4465 static const struct group_dual group15 = { {
4466 I(ModRM | Aligned16, em_fxsave),
4467 I(ModRM | Aligned16, em_fxrstor),
4468 N, N, N, N, N, GP(0, &pfx_0f_ae_7),
4470 N, N, N, N, N, N, N, N,
4473 static const struct gprefix pfx_0f_6f_0f_7f = {
4474 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
4477 static const struct instr_dual instr_dual_0f_2b = {
4481 static const struct gprefix pfx_0f_2b = {
4482 ID(0, &instr_dual_0f_2b), ID(0, &instr_dual_0f_2b), N, N,
4485 static const struct gprefix pfx_0f_10_0f_11 = {
4486 I(Unaligned, em_mov), I(Unaligned, em_mov), N, N,
4489 static const struct gprefix pfx_0f_28_0f_29 = {
4490 I(Aligned, em_mov), I(Aligned, em_mov), N, N,
4493 static const struct gprefix pfx_0f_e7 = {
4494 N, I(Sse, em_mov), N, N,
4497 static const struct escape escape_d9 = { {
4498 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstcw),
4501 N, N, N, N, N, N, N, N,
4503 N, N, N, N, N, N, N, N,
4505 N, N, N, N, N, N, N, N,
4507 N, N, N, N, N, N, N, N,
4509 N, N, N, N, N, N, N, N,
4511 N, N, N, N, N, N, N, N,
4513 N, N, N, N, N, N, N, N,
4515 N, N, N, N, N, N, N, N,
4518 static const struct escape escape_db = { {
4519 N, N, N, N, N, N, N, N,
4522 N, N, N, N, N, N, N, N,
4524 N, N, N, N, N, N, N, N,
4526 N, N, N, N, N, N, N, N,
4528 N, N, N, N, N, N, N, N,
4530 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
4532 N, N, N, N, N, N, N, N,
4534 N, N, N, N, N, N, N, N,
4536 N, N, N, N, N, N, N, N,
4539 static const struct escape escape_dd = { {
4540 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstsw),
4543 N, N, N, N, N, N, N, N,
4545 N, N, N, N, N, N, N, N,
4547 N, N, N, N, N, N, N, N,
4549 N, N, N, N, N, N, N, N,
4551 N, N, N, N, N, N, N, N,
4553 N, N, N, N, N, N, N, N,
4555 N, N, N, N, N, N, N, N,
4557 N, N, N, N, N, N, N, N,
4560 static const struct instr_dual instr_dual_0f_c3 = {
4561 I(DstMem | SrcReg | ModRM | No16 | Mov, em_mov), N
4564 static const struct mode_dual mode_dual_63 = {
4565 N, I(DstReg | SrcMem32 | ModRM | Mov, em_movsxd)
4568 static const struct opcode opcode_table[256] = {
4570 F6ALU(Lock, em_add),
4571 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
4572 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
4574 F6ALU(Lock | PageTable, em_or),
4575 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
4578 F6ALU(Lock, em_adc),
4579 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
4580 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
4582 F6ALU(Lock, em_sbb),
4583 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
4584 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
4586 F6ALU(Lock | PageTable, em_and), N, N,
4588 F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
4590 F6ALU(Lock, em_xor), N, N,
4592 F6ALU(NoWrite, em_cmp), N, N,
4594 X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
4596 X8(I(SrcReg | Stack, em_push)),
4598 X8(I(DstReg | Stack, em_pop)),
4600 I(ImplicitOps | Stack | No64, em_pusha),
4601 I(ImplicitOps | Stack | No64, em_popa),
4602 N, MD(ModRM, &mode_dual_63),
4605 I(SrcImm | Mov | Stack, em_push),
4606 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
4607 I(SrcImmByte | Mov | Stack, em_push),
4608 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
4609 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
4610 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
4612 X16(D(SrcImmByte | NearBranch | IsBranch)),
4614 G(ByteOp | DstMem | SrcImm, group1),
4615 G(DstMem | SrcImm, group1),
4616 G(ByteOp | DstMem | SrcImm | No64, group1),
4617 G(DstMem | SrcImmByte, group1),
4618 F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
4619 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
4621 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
4622 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
4623 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
4624 D(ModRM | SrcMem | NoAccess | DstReg),
4625 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
4628 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
4630 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
4631 I(SrcImmFAddr | No64 | IsBranch, em_call_far), N,
4632 II(ImplicitOps | Stack, em_pushf, pushf),
4633 II(ImplicitOps | Stack, em_popf, popf),
4634 I(ImplicitOps, em_sahf), I(ImplicitOps, em_lahf),
4636 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
4637 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
4638 I2bv(SrcSI | DstDI | Mov | String | TwoMemOp, em_mov),
4639 F2bv(SrcSI | DstDI | String | NoWrite | TwoMemOp, em_cmp_r),
4641 F2bv(DstAcc | SrcImm | NoWrite, em_test),
4642 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
4643 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
4644 F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp_r),
4646 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
4648 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
4650 G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
4651 I(ImplicitOps | NearBranch | SrcImmU16 | IsBranch, em_ret_near_imm),
4652 I(ImplicitOps | NearBranch | IsBranch, em_ret),
4653 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
4654 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
4655 G(ByteOp, group11), G(0, group11),
4657 I(Stack | SrcImmU16 | Src2ImmByte | IsBranch, em_enter),
4658 I(Stack | IsBranch, em_leave),
4659 I(ImplicitOps | SrcImmU16 | IsBranch, em_ret_far_imm),
4660 I(ImplicitOps | IsBranch, em_ret_far),
4661 D(ImplicitOps | IsBranch), DI(SrcImmByte | IsBranch, intn),
4662 D(ImplicitOps | No64 | IsBranch),
4663 II(ImplicitOps | IsBranch, em_iret, iret),
4665 G(Src2One | ByteOp, group2), G(Src2One, group2),
4666 G(Src2CL | ByteOp, group2), G(Src2CL, group2),
4667 I(DstAcc | SrcImmUByte | No64, em_aam),
4668 I(DstAcc | SrcImmUByte | No64, em_aad),
4669 F(DstAcc | ByteOp | No64, em_salc),
4670 I(DstAcc | SrcXLat | ByteOp, em_mov),
4672 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
4674 X3(I(SrcImmByte | NearBranch | IsBranch, em_loop)),
4675 I(SrcImmByte | NearBranch | IsBranch, em_jcxz),
4676 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
4677 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
4679 I(SrcImm | NearBranch | IsBranch, em_call),
4680 D(SrcImm | ImplicitOps | NearBranch | IsBranch),
4681 I(SrcImmFAddr | No64 | IsBranch, em_jmp_far),
4682 D(SrcImmByte | ImplicitOps | NearBranch | IsBranch),
4683 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
4684 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
4686 N, DI(ImplicitOps, icebp), N, N,
4687 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
4688 G(ByteOp, group3), G(0, group3),
4690 D(ImplicitOps), D(ImplicitOps),
4691 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
4692 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
4695 static const struct opcode twobyte_table[256] = {
4697 G(0, group6), GD(0, &group7), N, N,
4698 N, I(ImplicitOps | EmulateOnUD | IsBranch, em_syscall),
4699 II(ImplicitOps | Priv, em_clts, clts), N,
4700 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
4701 N, D(ImplicitOps | ModRM | SrcMem | NoAccess), N, N,
4703 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_10_0f_11),
4704 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_10_0f_11),
4706 D(ImplicitOps | ModRM | SrcMem | NoAccess), /* 4 * prefetch + 4 * reserved NOP */
4707 D(ImplicitOps | ModRM | SrcMem | NoAccess), N, N,
4708 D(ImplicitOps | ModRM | SrcMem | NoAccess), /* 8 * reserved NOP */
4709 D(ImplicitOps | ModRM | SrcMem | NoAccess), /* 8 * reserved NOP */
4710 D(ImplicitOps | ModRM | SrcMem | NoAccess), /* 8 * reserved NOP */
4711 D(ImplicitOps | ModRM | SrcMem | NoAccess), /* NOP + 7 * reserved NOP */
4713 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, cr_read, check_cr_access),
4714 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, dr_read, check_dr_read),
4715 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_cr_write, cr_write,
4717 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_dr_write, dr_write,
4720 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_28_0f_29),
4721 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_28_0f_29),
4722 N, GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_2b),
4725 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
4726 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
4727 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
4728 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
4729 I(ImplicitOps | EmulateOnUD | IsBranch, em_sysenter),
4730 I(ImplicitOps | Priv | EmulateOnUD | IsBranch, em_sysexit),
4732 N, N, N, N, N, N, N, N,
4734 X16(D(DstReg | SrcMem | ModRM)),
4736 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4741 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
4746 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
4748 X16(D(SrcImm | NearBranch | IsBranch)),
4750 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
4752 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
4753 II(ImplicitOps, em_cpuid, cpuid),
4754 F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
4755 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
4756 F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
4758 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
4759 II(EmulateOnUD | ImplicitOps, em_rsm, rsm),
4760 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
4761 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
4762 F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
4763 GD(0, &group15), F(DstReg | SrcMem | ModRM, em_imul),
4765 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable | SrcWrite, em_cmpxchg),
4766 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
4767 F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
4768 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
4769 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
4770 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4774 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
4775 I(DstReg | SrcMem | ModRM, em_bsf_c),
4776 I(DstReg | SrcMem | ModRM, em_bsr_c),
4777 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4779 F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
4780 N, ID(0, &instr_dual_0f_c3),
4781 N, N, N, GD(0, &group9),
4783 X8(I(DstReg, em_bswap)),
4785 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4787 N, N, N, N, N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_e7),
4788 N, N, N, N, N, N, N, N,
4790 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
4793 static const struct instr_dual instr_dual_0f_38_f0 = {
4794 I(DstReg | SrcMem | Mov, em_movbe), N
4797 static const struct instr_dual instr_dual_0f_38_f1 = {
4798 I(DstMem | SrcReg | Mov, em_movbe), N
4801 static const struct gprefix three_byte_0f_38_f0 = {
4802 ID(0, &instr_dual_0f_38_f0), N, N, N
4805 static const struct gprefix three_byte_0f_38_f1 = {
4806 ID(0, &instr_dual_0f_38_f1), N, N, N
4810 * Insns below are selected by the prefix which indexed by the third opcode
4813 static const struct opcode opcode_map_0f_38[256] = {
4815 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4817 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4819 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f0),
4820 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f1),
4841 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4845 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4851 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
4852 unsigned size, bool sign_extension)
4854 int rc = X86EMUL_CONTINUE;
4858 op->addr.mem.ea = ctxt->_eip;
4859 /* NB. Immediates are sign-extended as necessary. */
4860 switch (op->bytes) {
4862 op->val = insn_fetch(s8, ctxt);
4865 op->val = insn_fetch(s16, ctxt);
4868 op->val = insn_fetch(s32, ctxt);
4871 op->val = insn_fetch(s64, ctxt);
4874 if (!sign_extension) {
4875 switch (op->bytes) {
4883 op->val &= 0xffffffff;
4891 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
4894 int rc = X86EMUL_CONTINUE;
4898 decode_register_operand(ctxt, op);
4901 rc = decode_imm(ctxt, op, 1, false);
4904 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4908 if (ctxt->d & BitOp)
4909 fetch_bit_operand(ctxt);
4910 op->orig_val = op->val;
4913 ctxt->memop.bytes = (ctxt->op_bytes == 8) ? 16 : 8;
4917 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4918 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4919 fetch_register_operand(op);
4920 op->orig_val = op->val;
4924 op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;
4925 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4926 fetch_register_operand(op);
4927 op->orig_val = op->val;
4930 if (ctxt->d & ByteOp) {
4935 op->bytes = ctxt->op_bytes;
4936 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4937 fetch_register_operand(op);
4938 op->orig_val = op->val;
4942 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4944 register_address(ctxt, VCPU_REGS_RDI);
4945 op->addr.mem.seg = VCPU_SREG_ES;
4952 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4953 fetch_register_operand(op);
4958 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4961 rc = decode_imm(ctxt, op, 1, true);
4969 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
4972 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
4975 ctxt->memop.bytes = 1;
4976 if (ctxt->memop.type == OP_REG) {
4977 ctxt->memop.addr.reg = decode_register(ctxt,
4978 ctxt->modrm_rm, true);
4979 fetch_register_operand(&ctxt->memop);
4983 ctxt->memop.bytes = 2;
4986 ctxt->memop.bytes = 4;
4989 rc = decode_imm(ctxt, op, 2, false);
4992 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
4996 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4998 register_address(ctxt, VCPU_REGS_RSI);
4999 op->addr.mem.seg = ctxt->seg_override;
5005 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
5008 reg_read(ctxt, VCPU_REGS_RBX) +
5009 (reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
5010 op->addr.mem.seg = ctxt->seg_override;
5015 op->addr.mem.ea = ctxt->_eip;
5016 op->bytes = ctxt->op_bytes + 2;
5017 insn_fetch_arr(op->valptr, op->bytes, ctxt);
5020 ctxt->memop.bytes = ctxt->op_bytes + 2;
5024 op->val = VCPU_SREG_ES;
5028 op->val = VCPU_SREG_CS;
5032 op->val = VCPU_SREG_SS;
5036 op->val = VCPU_SREG_DS;
5040 op->val = VCPU_SREG_FS;
5044 op->val = VCPU_SREG_GS;
5047 /* Special instructions do their own operand decoding. */
5049 op->type = OP_NONE; /* Disable writeback. */
5057 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len, int emulation_type)
5059 int rc = X86EMUL_CONTINUE;
5060 int mode = ctxt->mode;
5061 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
5062 bool op_prefix = false;
5063 bool has_seg_override = false;
5064 struct opcode opcode;
5066 struct desc_struct desc;
5068 ctxt->memop.type = OP_NONE;
5069 ctxt->memopp = NULL;
5070 ctxt->_eip = ctxt->eip;
5071 ctxt->fetch.ptr = ctxt->fetch.data;
5072 ctxt->fetch.end = ctxt->fetch.data + insn_len;
5073 ctxt->opcode_len = 1;
5074 ctxt->intercept = x86_intercept_none;
5076 memcpy(ctxt->fetch.data, insn, insn_len);
5078 rc = __do_insn_fetch_bytes(ctxt, 1);
5079 if (rc != X86EMUL_CONTINUE)
5084 case X86EMUL_MODE_REAL:
5085 case X86EMUL_MODE_VM86:
5086 def_op_bytes = def_ad_bytes = 2;
5087 ctxt->ops->get_segment(ctxt, &dummy, &desc, NULL, VCPU_SREG_CS);
5089 def_op_bytes = def_ad_bytes = 4;
5091 case X86EMUL_MODE_PROT16:
5092 def_op_bytes = def_ad_bytes = 2;
5094 case X86EMUL_MODE_PROT32:
5095 def_op_bytes = def_ad_bytes = 4;
5097 #ifdef CONFIG_X86_64
5098 case X86EMUL_MODE_PROT64:
5104 return EMULATION_FAILED;
5107 ctxt->op_bytes = def_op_bytes;
5108 ctxt->ad_bytes = def_ad_bytes;
5110 /* Legacy prefixes. */
5112 switch (ctxt->b = insn_fetch(u8, ctxt)) {
5113 case 0x66: /* operand-size override */
5115 /* switch between 2/4 bytes */
5116 ctxt->op_bytes = def_op_bytes ^ 6;
5118 case 0x67: /* address-size override */
5119 if (mode == X86EMUL_MODE_PROT64)
5120 /* switch between 4/8 bytes */
5121 ctxt->ad_bytes = def_ad_bytes ^ 12;
5123 /* switch between 2/4 bytes */
5124 ctxt->ad_bytes = def_ad_bytes ^ 6;
5126 case 0x26: /* ES override */
5127 has_seg_override = true;
5128 ctxt->seg_override = VCPU_SREG_ES;
5130 case 0x2e: /* CS override */
5131 has_seg_override = true;
5132 ctxt->seg_override = VCPU_SREG_CS;
5134 case 0x36: /* SS override */
5135 has_seg_override = true;
5136 ctxt->seg_override = VCPU_SREG_SS;
5138 case 0x3e: /* DS override */
5139 has_seg_override = true;
5140 ctxt->seg_override = VCPU_SREG_DS;
5142 case 0x64: /* FS override */
5143 has_seg_override = true;
5144 ctxt->seg_override = VCPU_SREG_FS;
5146 case 0x65: /* GS override */
5147 has_seg_override = true;
5148 ctxt->seg_override = VCPU_SREG_GS;
5150 case 0x40 ... 0x4f: /* REX */
5151 if (mode != X86EMUL_MODE_PROT64)
5153 ctxt->rex_prefix = ctxt->b;
5155 case 0xf0: /* LOCK */
5156 ctxt->lock_prefix = 1;
5158 case 0xf2: /* REPNE/REPNZ */
5159 case 0xf3: /* REP/REPE/REPZ */
5160 ctxt->rep_prefix = ctxt->b;
5166 /* Any legacy prefix after a REX prefix nullifies its effect. */
5168 ctxt->rex_prefix = 0;
5174 if (ctxt->rex_prefix & 8)
5175 ctxt->op_bytes = 8; /* REX.W */
5177 /* Opcode byte(s). */
5178 opcode = opcode_table[ctxt->b];
5179 /* Two-byte opcode? */
5180 if (ctxt->b == 0x0f) {
5181 ctxt->opcode_len = 2;
5182 ctxt->b = insn_fetch(u8, ctxt);
5183 opcode = twobyte_table[ctxt->b];
5185 /* 0F_38 opcode map */
5186 if (ctxt->b == 0x38) {
5187 ctxt->opcode_len = 3;
5188 ctxt->b = insn_fetch(u8, ctxt);
5189 opcode = opcode_map_0f_38[ctxt->b];
5192 ctxt->d = opcode.flags;
5194 if (ctxt->d & ModRM)
5195 ctxt->modrm = insn_fetch(u8, ctxt);
5197 /* vex-prefix instructions are not implemented */
5198 if (ctxt->opcode_len == 1 && (ctxt->b == 0xc5 || ctxt->b == 0xc4) &&
5199 (mode == X86EMUL_MODE_PROT64 || (ctxt->modrm & 0xc0) == 0xc0)) {
5203 while (ctxt->d & GroupMask) {
5204 switch (ctxt->d & GroupMask) {
5206 goffset = (ctxt->modrm >> 3) & 7;
5207 opcode = opcode.u.group[goffset];
5210 goffset = (ctxt->modrm >> 3) & 7;
5211 if ((ctxt->modrm >> 6) == 3)
5212 opcode = opcode.u.gdual->mod3[goffset];
5214 opcode = opcode.u.gdual->mod012[goffset];
5217 goffset = ctxt->modrm & 7;
5218 opcode = opcode.u.group[goffset];
5221 if (ctxt->rep_prefix && op_prefix)
5222 return EMULATION_FAILED;
5223 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
5224 switch (simd_prefix) {
5225 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
5226 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
5227 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
5228 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
5232 if (ctxt->modrm > 0xbf) {
5233 size_t size = ARRAY_SIZE(opcode.u.esc->high);
5234 u32 index = array_index_nospec(
5235 ctxt->modrm - 0xc0, size);
5237 opcode = opcode.u.esc->high[index];
5239 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
5243 if ((ctxt->modrm >> 6) == 3)
5244 opcode = opcode.u.idual->mod3;
5246 opcode = opcode.u.idual->mod012;
5249 if (ctxt->mode == X86EMUL_MODE_PROT64)
5250 opcode = opcode.u.mdual->mode64;
5252 opcode = opcode.u.mdual->mode32;
5255 return EMULATION_FAILED;
5258 ctxt->d &= ~(u64)GroupMask;
5259 ctxt->d |= opcode.flags;
5262 ctxt->is_branch = opcode.flags & IsBranch;
5266 return EMULATION_FAILED;
5268 ctxt->execute = opcode.u.execute;
5270 if (unlikely(emulation_type & EMULTYPE_TRAP_UD) &&
5271 likely(!(ctxt->d & EmulateOnUD)))
5272 return EMULATION_FAILED;
5274 if (unlikely(ctxt->d &
5275 (NotImpl|Stack|Op3264|Sse|Mmx|Intercept|CheckPerm|NearBranch|
5278 * These are copied unconditionally here, and checked unconditionally
5279 * in x86_emulate_insn.
5281 ctxt->check_perm = opcode.check_perm;
5282 ctxt->intercept = opcode.intercept;
5284 if (ctxt->d & NotImpl)
5285 return EMULATION_FAILED;
5287 if (mode == X86EMUL_MODE_PROT64) {
5288 if (ctxt->op_bytes == 4 && (ctxt->d & Stack))
5290 else if (ctxt->d & NearBranch)
5294 if (ctxt->d & Op3264) {
5295 if (mode == X86EMUL_MODE_PROT64)
5301 if ((ctxt->d & No16) && ctxt->op_bytes == 2)
5305 ctxt->op_bytes = 16;
5306 else if (ctxt->d & Mmx)
5310 /* ModRM and SIB bytes. */
5311 if (ctxt->d & ModRM) {
5312 rc = decode_modrm(ctxt, &ctxt->memop);
5313 if (!has_seg_override) {
5314 has_seg_override = true;
5315 ctxt->seg_override = ctxt->modrm_seg;
5317 } else if (ctxt->d & MemAbs)
5318 rc = decode_abs(ctxt, &ctxt->memop);
5319 if (rc != X86EMUL_CONTINUE)
5322 if (!has_seg_override)
5323 ctxt->seg_override = VCPU_SREG_DS;
5325 ctxt->memop.addr.mem.seg = ctxt->seg_override;
5328 * Decode and fetch the source operand: register, memory
5331 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
5332 if (rc != X86EMUL_CONTINUE)
5336 * Decode and fetch the second source operand: register, memory
5339 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
5340 if (rc != X86EMUL_CONTINUE)
5343 /* Decode and fetch the destination operand: register or memory. */
5344 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
5346 if (ctxt->rip_relative && likely(ctxt->memopp))
5347 ctxt->memopp->addr.mem.ea = address_mask(ctxt,
5348 ctxt->memopp->addr.mem.ea + ctxt->_eip);
5351 if (rc == X86EMUL_PROPAGATE_FAULT)
5352 ctxt->have_exception = true;
5353 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
5356 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
5358 return ctxt->d & PageTable;
5361 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
5363 /* The second termination condition only applies for REPE
5364 * and REPNE. Test if the repeat string operation prefix is
5365 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
5366 * corresponding termination condition according to:
5367 * - if REPE/REPZ and ZF = 0 then done
5368 * - if REPNE/REPNZ and ZF = 1 then done
5370 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
5371 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
5372 && (((ctxt->rep_prefix == REPE_PREFIX) &&
5373 ((ctxt->eflags & X86_EFLAGS_ZF) == 0))
5374 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
5375 ((ctxt->eflags & X86_EFLAGS_ZF) == X86_EFLAGS_ZF))))
5381 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
5386 rc = asm_safe("fwait");
5389 if (unlikely(rc != X86EMUL_CONTINUE))
5390 return emulate_exception(ctxt, MF_VECTOR, 0, false);
5392 return X86EMUL_CONTINUE;
5395 static void fetch_possible_mmx_operand(struct operand *op)
5397 if (op->type == OP_MM)
5398 kvm_read_mmx_reg(op->addr.mm, &op->mm_val);
5401 static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop)
5403 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
5405 if (!(ctxt->d & ByteOp))
5406 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
5408 asm("push %[flags]; popf; " CALL_NOSPEC " ; pushf; pop %[flags]\n"
5409 : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
5410 [thunk_target]"+S"(fop), ASM_CALL_CONSTRAINT
5411 : "c"(ctxt->src2.val));
5413 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
5414 if (!fop) /* exception is returned in fop variable */
5415 return emulate_de(ctxt);
5416 return X86EMUL_CONTINUE;
5419 void init_decode_cache(struct x86_emulate_ctxt *ctxt)
5421 /* Clear fields that are set conditionally but read without a guard. */
5422 ctxt->rip_relative = false;
5423 ctxt->rex_prefix = 0;
5424 ctxt->lock_prefix = 0;
5425 ctxt->rep_prefix = 0;
5426 ctxt->regs_valid = 0;
5427 ctxt->regs_dirty = 0;
5429 ctxt->io_read.pos = 0;
5430 ctxt->io_read.end = 0;
5431 ctxt->mem_read.end = 0;
5434 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
5436 const struct x86_emulate_ops *ops = ctxt->ops;
5437 int rc = X86EMUL_CONTINUE;
5438 int saved_dst_type = ctxt->dst.type;
5439 unsigned emul_flags;
5441 ctxt->mem_read.pos = 0;
5443 /* LOCK prefix is allowed only with some instructions */
5444 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
5445 rc = emulate_ud(ctxt);
5449 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
5450 rc = emulate_ud(ctxt);
5454 emul_flags = ctxt->ops->get_hflags(ctxt);
5455 if (unlikely(ctxt->d &
5456 (No64|Undefined|Sse|Mmx|Intercept|CheckPerm|Priv|Prot|String))) {
5457 if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
5458 (ctxt->d & Undefined)) {
5459 rc = emulate_ud(ctxt);
5463 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
5464 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
5465 rc = emulate_ud(ctxt);
5469 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
5470 rc = emulate_nm(ctxt);
5474 if (ctxt->d & Mmx) {
5475 rc = flush_pending_x87_faults(ctxt);
5476 if (rc != X86EMUL_CONTINUE)
5479 * Now that we know the fpu is exception safe, we can fetch
5482 fetch_possible_mmx_operand(&ctxt->src);
5483 fetch_possible_mmx_operand(&ctxt->src2);
5484 if (!(ctxt->d & Mov))
5485 fetch_possible_mmx_operand(&ctxt->dst);
5488 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && ctxt->intercept) {
5489 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5490 X86_ICPT_PRE_EXCEPT);
5491 if (rc != X86EMUL_CONTINUE)
5495 /* Instruction can only be executed in protected mode */
5496 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
5497 rc = emulate_ud(ctxt);
5501 /* Privileged instruction can be executed only in CPL=0 */
5502 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
5503 if (ctxt->d & PrivUD)
5504 rc = emulate_ud(ctxt);
5506 rc = emulate_gp(ctxt, 0);
5510 /* Do instruction specific permission checks */
5511 if (ctxt->d & CheckPerm) {
5512 rc = ctxt->check_perm(ctxt);
5513 if (rc != X86EMUL_CONTINUE)
5517 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {
5518 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5519 X86_ICPT_POST_EXCEPT);
5520 if (rc != X86EMUL_CONTINUE)
5524 if (ctxt->rep_prefix && (ctxt->d & String)) {
5525 /* All REP prefixes have the same first termination condition */
5526 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
5527 string_registers_quirk(ctxt);
5528 ctxt->eip = ctxt->_eip;
5529 ctxt->eflags &= ~X86_EFLAGS_RF;
5535 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
5536 rc = segmented_read(ctxt, ctxt->src.addr.mem,
5537 ctxt->src.valptr, ctxt->src.bytes);
5538 if (rc != X86EMUL_CONTINUE)
5540 ctxt->src.orig_val64 = ctxt->src.val64;
5543 if (ctxt->src2.type == OP_MEM) {
5544 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
5545 &ctxt->src2.val, ctxt->src2.bytes);
5546 if (rc != X86EMUL_CONTINUE)
5550 if ((ctxt->d & DstMask) == ImplicitOps)
5554 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
5555 /* optimisation - avoid slow emulated read if Mov */
5556 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
5557 &ctxt->dst.val, ctxt->dst.bytes);
5558 if (rc != X86EMUL_CONTINUE) {
5559 if (!(ctxt->d & NoWrite) &&
5560 rc == X86EMUL_PROPAGATE_FAULT &&
5561 ctxt->exception.vector == PF_VECTOR)
5562 ctxt->exception.error_code |= PFERR_WRITE_MASK;
5566 /* Copy full 64-bit value for CMPXCHG8B. */
5567 ctxt->dst.orig_val64 = ctxt->dst.val64;
5571 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {
5572 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5573 X86_ICPT_POST_MEMACCESS);
5574 if (rc != X86EMUL_CONTINUE)
5578 if (ctxt->rep_prefix && (ctxt->d & String))
5579 ctxt->eflags |= X86_EFLAGS_RF;
5581 ctxt->eflags &= ~X86_EFLAGS_RF;
5583 if (ctxt->execute) {
5584 if (ctxt->d & Fastop)
5585 rc = fastop(ctxt, ctxt->fop);
5587 rc = ctxt->execute(ctxt);
5588 if (rc != X86EMUL_CONTINUE)
5593 if (ctxt->opcode_len == 2)
5595 else if (ctxt->opcode_len == 3)
5596 goto threebyte_insn;
5599 case 0x70 ... 0x7f: /* jcc (short) */
5600 if (test_cc(ctxt->b, ctxt->eflags))
5601 rc = jmp_rel(ctxt, ctxt->src.val);
5603 case 0x8d: /* lea r16/r32, m */
5604 ctxt->dst.val = ctxt->src.addr.mem.ea;
5606 case 0x90 ... 0x97: /* nop / xchg reg, rax */
5607 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
5608 ctxt->dst.type = OP_NONE;
5612 case 0x98: /* cbw/cwde/cdqe */
5613 switch (ctxt->op_bytes) {
5614 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
5615 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
5616 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
5619 case 0xcc: /* int3 */
5620 rc = emulate_int(ctxt, 3);
5622 case 0xcd: /* int n */
5623 rc = emulate_int(ctxt, ctxt->src.val);
5625 case 0xce: /* into */
5626 if (ctxt->eflags & X86_EFLAGS_OF)
5627 rc = emulate_int(ctxt, 4);
5629 case 0xe9: /* jmp rel */
5630 case 0xeb: /* jmp rel short */
5631 rc = jmp_rel(ctxt, ctxt->src.val);
5632 ctxt->dst.type = OP_NONE; /* Disable writeback. */
5634 case 0xf4: /* hlt */
5635 ctxt->ops->halt(ctxt);
5637 case 0xf5: /* cmc */
5638 /* complement carry flag from eflags reg */
5639 ctxt->eflags ^= X86_EFLAGS_CF;
5641 case 0xf8: /* clc */
5642 ctxt->eflags &= ~X86_EFLAGS_CF;
5644 case 0xf9: /* stc */
5645 ctxt->eflags |= X86_EFLAGS_CF;
5647 case 0xfc: /* cld */
5648 ctxt->eflags &= ~X86_EFLAGS_DF;
5650 case 0xfd: /* std */
5651 ctxt->eflags |= X86_EFLAGS_DF;
5654 goto cannot_emulate;
5657 if (rc != X86EMUL_CONTINUE)
5661 if (ctxt->d & SrcWrite) {
5662 BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);
5663 rc = writeback(ctxt, &ctxt->src);
5664 if (rc != X86EMUL_CONTINUE)
5667 if (!(ctxt->d & NoWrite)) {
5668 rc = writeback(ctxt, &ctxt->dst);
5669 if (rc != X86EMUL_CONTINUE)
5674 * restore dst type in case the decoding will be reused
5675 * (happens for string instruction )
5677 ctxt->dst.type = saved_dst_type;
5679 if ((ctxt->d & SrcMask) == SrcSI)
5680 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
5682 if ((ctxt->d & DstMask) == DstDI)
5683 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
5685 if (ctxt->rep_prefix && (ctxt->d & String)) {
5687 struct read_cache *r = &ctxt->io_read;
5688 if ((ctxt->d & SrcMask) == SrcSI)
5689 count = ctxt->src.count;
5691 count = ctxt->dst.count;
5692 register_address_increment(ctxt, VCPU_REGS_RCX, -count);
5694 if (!string_insn_completed(ctxt)) {
5696 * Re-enter guest when pio read ahead buffer is empty
5697 * or, if it is not used, after each 1024 iteration.
5699 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
5700 (r->end == 0 || r->end != r->pos)) {
5702 * Reset read cache. Usually happens before
5703 * decode, but since instruction is restarted
5704 * we have to do it here.
5706 ctxt->mem_read.end = 0;
5707 writeback_registers(ctxt);
5708 return EMULATION_RESTART;
5710 goto done; /* skip rip writeback */
5712 ctxt->eflags &= ~X86_EFLAGS_RF;
5715 ctxt->eip = ctxt->_eip;
5716 if (ctxt->mode != X86EMUL_MODE_PROT64)
5717 ctxt->eip = (u32)ctxt->_eip;
5720 if (rc == X86EMUL_PROPAGATE_FAULT) {
5721 WARN_ON(ctxt->exception.vector > 0x1f);
5722 ctxt->have_exception = true;
5724 if (rc == X86EMUL_INTERCEPTED)
5725 return EMULATION_INTERCEPTED;
5727 if (rc == X86EMUL_CONTINUE)
5728 writeback_registers(ctxt);
5730 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
5734 case 0x09: /* wbinvd */
5735 (ctxt->ops->wbinvd)(ctxt);
5737 case 0x08: /* invd */
5738 case 0x0d: /* GrpP (prefetch) */
5739 case 0x18: /* Grp16 (prefetch/nop) */
5740 case 0x1f: /* nop */
5742 case 0x20: /* mov cr, reg */
5743 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
5745 case 0x21: /* mov from dr to reg */
5746 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
5748 case 0x40 ... 0x4f: /* cmov */
5749 if (test_cc(ctxt->b, ctxt->eflags))
5750 ctxt->dst.val = ctxt->src.val;
5751 else if (ctxt->op_bytes != 4)
5752 ctxt->dst.type = OP_NONE; /* no writeback */
5754 case 0x80 ... 0x8f: /* jnz rel, etc*/
5755 if (test_cc(ctxt->b, ctxt->eflags))
5756 rc = jmp_rel(ctxt, ctxt->src.val);
5758 case 0x90 ... 0x9f: /* setcc r/m8 */
5759 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
5761 case 0xb6 ... 0xb7: /* movzx */
5762 ctxt->dst.bytes = ctxt->op_bytes;
5763 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
5764 : (u16) ctxt->src.val;
5766 case 0xbe ... 0xbf: /* movsx */
5767 ctxt->dst.bytes = ctxt->op_bytes;
5768 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
5769 (s16) ctxt->src.val;
5772 goto cannot_emulate;
5777 if (rc != X86EMUL_CONTINUE)
5783 return EMULATION_FAILED;
5786 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
5788 invalidate_registers(ctxt);
5791 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
5793 writeback_registers(ctxt);
5796 bool emulator_can_use_gpa(struct x86_emulate_ctxt *ctxt)
5798 if (ctxt->rep_prefix && (ctxt->d & String))
5801 if (ctxt->d & TwoMemOp)