1 // SPDX-License-Identifier: GPL-2.0-only
2 /******************************************************************************
5 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
7 * Copyright (c) 2005 Keir Fraser
9 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
10 * privileged instructions:
12 * Copyright (C) 2006 Qumranet
13 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
15 * Avi Kivity <avi@qumranet.com>
16 * Yaniv Kamay <yaniv@qumranet.com>
18 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
21 #include <linux/kvm_host.h>
22 #include "kvm_cache_regs.h"
23 #include "kvm_emulate.h"
24 #include <linux/stringify.h>
25 #include <asm/fpu/api.h>
26 #include <asm/debugreg.h>
27 #include <asm/nospec-branch.h>
38 #define OpImplicit 1ull /* No generic decode */
39 #define OpReg 2ull /* Register */
40 #define OpMem 3ull /* Memory */
41 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
42 #define OpDI 5ull /* ES:DI/EDI/RDI */
43 #define OpMem64 6ull /* Memory, 64-bit */
44 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
45 #define OpDX 8ull /* DX register */
46 #define OpCL 9ull /* CL register (for shifts) */
47 #define OpImmByte 10ull /* 8-bit sign extended immediate */
48 #define OpOne 11ull /* Implied 1 */
49 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
50 #define OpMem16 13ull /* Memory operand (16-bit). */
51 #define OpMem32 14ull /* Memory operand (32-bit). */
52 #define OpImmU 15ull /* Immediate operand, zero extended */
53 #define OpSI 16ull /* SI/ESI/RSI */
54 #define OpImmFAddr 17ull /* Immediate far address */
55 #define OpMemFAddr 18ull /* Far address in memory */
56 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
57 #define OpES 20ull /* ES */
58 #define OpCS 21ull /* CS */
59 #define OpSS 22ull /* SS */
60 #define OpDS 23ull /* DS */
61 #define OpFS 24ull /* FS */
62 #define OpGS 25ull /* GS */
63 #define OpMem8 26ull /* 8-bit zero extended memory operand */
64 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
65 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
66 #define OpAccLo 29ull /* Low part of extended acc (AX/AX/EAX/RAX) */
67 #define OpAccHi 30ull /* High part of extended acc (-/DX/EDX/RDX) */
69 #define OpBits 5 /* Width of operand field */
70 #define OpMask ((1ull << OpBits) - 1)
73 * Opcode effective-address decode tables.
74 * Note that we only emulate instructions that have at least one memory
75 * operand (excluding implicit stack references). We assume that stack
76 * references and instruction fetches will never occur in special memory
77 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
81 /* Operand sizes: 8-bit operands or specified/overridden size. */
82 #define ByteOp (1<<0) /* 8-bit operands. */
83 /* Destination operand type. */
85 #define ImplicitOps (OpImplicit << DstShift)
86 #define DstReg (OpReg << DstShift)
87 #define DstMem (OpMem << DstShift)
88 #define DstAcc (OpAcc << DstShift)
89 #define DstDI (OpDI << DstShift)
90 #define DstMem64 (OpMem64 << DstShift)
91 #define DstMem16 (OpMem16 << DstShift)
92 #define DstImmUByte (OpImmUByte << DstShift)
93 #define DstDX (OpDX << DstShift)
94 #define DstAccLo (OpAccLo << DstShift)
95 #define DstMask (OpMask << DstShift)
96 /* Source operand type. */
98 #define SrcNone (OpNone << SrcShift)
99 #define SrcReg (OpReg << SrcShift)
100 #define SrcMem (OpMem << SrcShift)
101 #define SrcMem16 (OpMem16 << SrcShift)
102 #define SrcMem32 (OpMem32 << SrcShift)
103 #define SrcImm (OpImm << SrcShift)
104 #define SrcImmByte (OpImmByte << SrcShift)
105 #define SrcOne (OpOne << SrcShift)
106 #define SrcImmUByte (OpImmUByte << SrcShift)
107 #define SrcImmU (OpImmU << SrcShift)
108 #define SrcSI (OpSI << SrcShift)
109 #define SrcXLat (OpXLat << SrcShift)
110 #define SrcImmFAddr (OpImmFAddr << SrcShift)
111 #define SrcMemFAddr (OpMemFAddr << SrcShift)
112 #define SrcAcc (OpAcc << SrcShift)
113 #define SrcImmU16 (OpImmU16 << SrcShift)
114 #define SrcImm64 (OpImm64 << SrcShift)
115 #define SrcDX (OpDX << SrcShift)
116 #define SrcMem8 (OpMem8 << SrcShift)
117 #define SrcAccHi (OpAccHi << SrcShift)
118 #define SrcMask (OpMask << SrcShift)
119 #define BitOp (1<<11)
120 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
121 #define String (1<<13) /* String instruction (rep capable) */
122 #define Stack (1<<14) /* Stack instruction (push/pop) */
123 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
124 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
125 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
126 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
127 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
128 #define Escape (5<<15) /* Escape to coprocessor instruction */
129 #define InstrDual (6<<15) /* Alternate instruction decoding of mod == 3 */
130 #define ModeDual (7<<15) /* Different instruction for 32/64 bit */
131 #define Sse (1<<18) /* SSE Vector instruction */
132 /* Generic ModRM decode. */
133 #define ModRM (1<<19)
134 /* Destination is only written; never read. */
137 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
138 #define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
139 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
140 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
141 #define Undefined (1<<25) /* No Such Instruction */
142 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
143 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
145 #define PageTable (1 << 29) /* instruction used to write page table */
146 #define NotImpl (1 << 30) /* instruction is not implemented */
147 /* Source 2 operand type */
148 #define Src2Shift (31)
149 #define Src2None (OpNone << Src2Shift)
150 #define Src2Mem (OpMem << Src2Shift)
151 #define Src2CL (OpCL << Src2Shift)
152 #define Src2ImmByte (OpImmByte << Src2Shift)
153 #define Src2One (OpOne << Src2Shift)
154 #define Src2Imm (OpImm << Src2Shift)
155 #define Src2ES (OpES << Src2Shift)
156 #define Src2CS (OpCS << Src2Shift)
157 #define Src2SS (OpSS << Src2Shift)
158 #define Src2DS (OpDS << Src2Shift)
159 #define Src2FS (OpFS << Src2Shift)
160 #define Src2GS (OpGS << Src2Shift)
161 #define Src2Mask (OpMask << Src2Shift)
162 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
163 #define AlignMask ((u64)7 << 41)
164 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
165 #define Unaligned ((u64)2 << 41) /* Explicitly unaligned (e.g. MOVDQU) */
166 #define Avx ((u64)3 << 41) /* Advanced Vector Extensions */
167 #define Aligned16 ((u64)4 << 41) /* Aligned to 16 byte boundary (e.g. FXSAVE) */
168 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
169 #define NoWrite ((u64)1 << 45) /* No writeback */
170 #define SrcWrite ((u64)1 << 46) /* Write back src operand */
171 #define NoMod ((u64)1 << 47) /* Mod field is ignored */
172 #define Intercept ((u64)1 << 48) /* Has valid intercept field */
173 #define CheckPerm ((u64)1 << 49) /* Has valid check_perm field */
174 #define PrivUD ((u64)1 << 51) /* #UD instead of #GP on CPL > 0 */
175 #define NearBranch ((u64)1 << 52) /* Near branches */
176 #define No16 ((u64)1 << 53) /* No 16 bit operand */
177 #define IncSP ((u64)1 << 54) /* SP is incremented before ModRM calc */
178 #define TwoMemOp ((u64)1 << 55) /* Instruction has two memory operand */
180 #define DstXacc (DstAccLo | SrcAccHi | SrcWrite)
182 #define X2(x...) x, x
183 #define X3(x...) X2(x), x
184 #define X4(x...) X2(x), X2(x)
185 #define X5(x...) X4(x), x
186 #define X6(x...) X4(x), X2(x)
187 #define X7(x...) X4(x), X3(x)
188 #define X8(x...) X4(x), X4(x)
189 #define X16(x...) X8(x), X8(x)
191 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
192 #define FASTOP_SIZE 8
198 int (*execute)(struct x86_emulate_ctxt *ctxt);
199 const struct opcode *group;
200 const struct group_dual *gdual;
201 const struct gprefix *gprefix;
202 const struct escape *esc;
203 const struct instr_dual *idual;
204 const struct mode_dual *mdual;
205 void (*fastop)(struct fastop *fake);
207 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
211 struct opcode mod012[8];
212 struct opcode mod3[8];
216 struct opcode pfx_no;
217 struct opcode pfx_66;
218 struct opcode pfx_f2;
219 struct opcode pfx_f3;
224 struct opcode high[64];
228 struct opcode mod012;
233 struct opcode mode32;
234 struct opcode mode64;
237 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
239 enum x86_transfer_type {
241 X86_TRANSFER_CALL_JMP,
243 X86_TRANSFER_TASK_SWITCH,
246 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
248 if (!(ctxt->regs_valid & (1 << nr))) {
249 ctxt->regs_valid |= 1 << nr;
250 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
252 return ctxt->_regs[nr];
255 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
257 ctxt->regs_valid |= 1 << nr;
258 ctxt->regs_dirty |= 1 << nr;
259 return &ctxt->_regs[nr];
262 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
265 return reg_write(ctxt, nr);
268 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
272 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
273 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
276 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
278 ctxt->regs_dirty = 0;
279 ctxt->regs_valid = 0;
283 * These EFLAGS bits are restored from saved value during emulation, and
284 * any changes are written back to the saved value after emulation.
286 #define EFLAGS_MASK (X86_EFLAGS_OF|X86_EFLAGS_SF|X86_EFLAGS_ZF|X86_EFLAGS_AF|\
287 X86_EFLAGS_PF|X86_EFLAGS_CF)
296 * fastop functions have a special calling convention:
301 * flags: rflags (in/out)
302 * ex: rsi (in:fastop pointer, out:zero if exception)
304 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
305 * different operand sizes can be reached by calculation, rather than a jump
306 * table (which would be bigger than the code).
308 static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop);
310 #define __FOP_FUNC(name) \
311 ".align " __stringify(FASTOP_SIZE) " \n\t" \
312 ".type " name ", @function \n\t" \
315 #define FOP_FUNC(name) \
318 #define __FOP_RET(name) \
320 ".size " name ", .-" name "\n\t"
322 #define FOP_RET(name) \
325 #define FOP_START(op) \
326 extern void em_##op(struct fastop *fake); \
327 asm(".pushsection .text, \"ax\" \n\t" \
328 ".global em_" #op " \n\t" \
329 ".align " __stringify(FASTOP_SIZE) " \n\t" \
335 #define __FOPNOP(name) \
340 __FOPNOP(__stringify(__UNIQUE_ID(nop)))
342 #define FOP1E(op, dst) \
343 __FOP_FUNC(#op "_" #dst) \
344 "10: " #op " %" #dst " \n\t" \
345 __FOP_RET(#op "_" #dst)
347 #define FOP1EEX(op, dst) \
348 FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
350 #define FASTOP1(op) \
355 ON64(FOP1E(op##q, rax)) \
358 /* 1-operand, using src2 (for MUL/DIV r/m) */
359 #define FASTOP1SRC2(op, name) \
364 ON64(FOP1E(op, rcx)) \
367 /* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
368 #define FASTOP1SRC2EX(op, name) \
373 ON64(FOP1EEX(op, rcx)) \
376 #define FOP2E(op, dst, src) \
377 __FOP_FUNC(#op "_" #dst "_" #src) \
378 #op " %" #src ", %" #dst " \n\t" \
379 __FOP_RET(#op "_" #dst "_" #src)
381 #define FASTOP2(op) \
383 FOP2E(op##b, al, dl) \
384 FOP2E(op##w, ax, dx) \
385 FOP2E(op##l, eax, edx) \
386 ON64(FOP2E(op##q, rax, rdx)) \
389 /* 2 operand, word only */
390 #define FASTOP2W(op) \
393 FOP2E(op##w, ax, dx) \
394 FOP2E(op##l, eax, edx) \
395 ON64(FOP2E(op##q, rax, rdx)) \
398 /* 2 operand, src is CL */
399 #define FASTOP2CL(op) \
401 FOP2E(op##b, al, cl) \
402 FOP2E(op##w, ax, cl) \
403 FOP2E(op##l, eax, cl) \
404 ON64(FOP2E(op##q, rax, cl)) \
407 /* 2 operand, src and dest are reversed */
408 #define FASTOP2R(op, name) \
410 FOP2E(op##b, dl, al) \
411 FOP2E(op##w, dx, ax) \
412 FOP2E(op##l, edx, eax) \
413 ON64(FOP2E(op##q, rdx, rax)) \
416 #define FOP3E(op, dst, src, src2) \
417 __FOP_FUNC(#op "_" #dst "_" #src "_" #src2) \
418 #op " %" #src2 ", %" #src ", %" #dst " \n\t"\
419 __FOP_RET(#op "_" #dst "_" #src "_" #src2)
421 /* 3-operand, word-only, src2=cl */
422 #define FASTOP3WCL(op) \
425 FOP3E(op##w, ax, dx, cl) \
426 FOP3E(op##l, eax, edx, cl) \
427 ON64(FOP3E(op##q, rax, rdx, cl)) \
430 /* Special case for SETcc - 1 instruction per cc */
431 #define FOP_SETCC(op) \
433 ".type " #op ", @function \n\t" \
438 asm(".pushsection .fixup, \"ax\"\n"
439 ".global kvm_fastop_exception \n"
440 "kvm_fastop_exception: xor %esi, %esi; ret\n"
464 "pushf; sbb %al, %al; popf \n\t"
469 * XXX: inoutclob user must know where the argument is being expanded.
470 * Relying on CONFIG_CC_HAS_ASM_GOTO would allow us to remove _fault.
472 #define asm_safe(insn, inoutclob...) \
476 asm volatile("1:" insn "\n" \
478 ".pushsection .fixup, \"ax\"\n" \
479 "3: movl $1, %[_fault]\n" \
482 _ASM_EXTABLE(1b, 3b) \
483 : [_fault] "+qm"(_fault) inoutclob ); \
485 _fault ? X86EMUL_UNHANDLEABLE : X86EMUL_CONTINUE; \
488 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
489 enum x86_intercept intercept,
490 enum x86_intercept_stage stage)
492 struct x86_instruction_info info = {
493 .intercept = intercept,
494 .rep_prefix = ctxt->rep_prefix,
495 .modrm_mod = ctxt->modrm_mod,
496 .modrm_reg = ctxt->modrm_reg,
497 .modrm_rm = ctxt->modrm_rm,
498 .src_val = ctxt->src.val64,
499 .dst_val = ctxt->dst.val64,
500 .src_bytes = ctxt->src.bytes,
501 .dst_bytes = ctxt->dst.bytes,
502 .ad_bytes = ctxt->ad_bytes,
503 .next_rip = ctxt->eip,
506 return ctxt->ops->intercept(ctxt, &info, stage);
509 static void assign_masked(ulong *dest, ulong src, ulong mask)
511 *dest = (*dest & ~mask) | (src & mask);
514 static void assign_register(unsigned long *reg, u64 val, int bytes)
516 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
519 *(u8 *)reg = (u8)val;
522 *(u16 *)reg = (u16)val;
526 break; /* 64b: zero-extend */
533 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
535 return (1UL << (ctxt->ad_bytes << 3)) - 1;
538 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
541 struct desc_struct ss;
543 if (ctxt->mode == X86EMUL_MODE_PROT64)
545 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
546 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
549 static int stack_size(struct x86_emulate_ctxt *ctxt)
551 return (__fls(stack_mask(ctxt)) + 1) >> 3;
554 /* Access/update address held in a register, based on addressing mode. */
555 static inline unsigned long
556 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
558 if (ctxt->ad_bytes == sizeof(unsigned long))
561 return reg & ad_mask(ctxt);
564 static inline unsigned long
565 register_address(struct x86_emulate_ctxt *ctxt, int reg)
567 return address_mask(ctxt, reg_read(ctxt, reg));
570 static void masked_increment(ulong *reg, ulong mask, int inc)
572 assign_masked(reg, *reg + inc, mask);
576 register_address_increment(struct x86_emulate_ctxt *ctxt, int reg, int inc)
578 ulong *preg = reg_rmw(ctxt, reg);
580 assign_register(preg, *preg + inc, ctxt->ad_bytes);
583 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
585 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
588 static u32 desc_limit_scaled(struct desc_struct *desc)
590 u32 limit = get_desc_limit(desc);
592 return desc->g ? (limit << 12) | 0xfff : limit;
595 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
597 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
600 return ctxt->ops->get_cached_segment_base(ctxt, seg);
603 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
604 u32 error, bool valid)
607 ctxt->exception.vector = vec;
608 ctxt->exception.error_code = error;
609 ctxt->exception.error_code_valid = valid;
610 return X86EMUL_PROPAGATE_FAULT;
613 static int emulate_db(struct x86_emulate_ctxt *ctxt)
615 return emulate_exception(ctxt, DB_VECTOR, 0, false);
618 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
620 return emulate_exception(ctxt, GP_VECTOR, err, true);
623 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
625 return emulate_exception(ctxt, SS_VECTOR, err, true);
628 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
630 return emulate_exception(ctxt, UD_VECTOR, 0, false);
633 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
635 return emulate_exception(ctxt, TS_VECTOR, err, true);
638 static int emulate_de(struct x86_emulate_ctxt *ctxt)
640 return emulate_exception(ctxt, DE_VECTOR, 0, false);
643 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
645 return emulate_exception(ctxt, NM_VECTOR, 0, false);
648 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
651 struct desc_struct desc;
653 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
657 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
662 struct desc_struct desc;
664 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
665 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
668 static inline u8 ctxt_virt_addr_bits(struct x86_emulate_ctxt *ctxt)
670 return (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_LA57) ? 57 : 48;
673 static inline bool emul_is_noncanonical_address(u64 la,
674 struct x86_emulate_ctxt *ctxt)
676 return get_canonical(la, ctxt_virt_addr_bits(ctxt)) != la;
680 * x86 defines three classes of vector instructions: explicitly
681 * aligned, explicitly unaligned, and the rest, which change behaviour
682 * depending on whether they're AVX encoded or not.
684 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
685 * subject to the same check. FXSAVE and FXRSTOR are checked here too as their
686 * 512 bytes of data must be aligned to a 16 byte boundary.
688 static unsigned insn_alignment(struct x86_emulate_ctxt *ctxt, unsigned size)
690 u64 alignment = ctxt->d & AlignMask;
692 if (likely(size < 16))
707 static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt,
708 struct segmented_address addr,
709 unsigned *max_size, unsigned size,
710 bool write, bool fetch,
711 enum x86emul_mode mode, ulong *linear)
713 struct desc_struct desc;
720 la = seg_base(ctxt, addr.seg) + addr.ea;
723 case X86EMUL_MODE_PROT64:
725 va_bits = ctxt_virt_addr_bits(ctxt);
726 if (get_canonical(la, va_bits) != la)
729 *max_size = min_t(u64, ~0u, (1ull << va_bits) - la);
730 if (size > *max_size)
734 *linear = la = (u32)la;
735 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
739 /* code segment in protected mode or read-only data segment */
740 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
741 || !(desc.type & 2)) && write)
743 /* unreadable code segment */
744 if (!fetch && (desc.type & 8) && !(desc.type & 2))
746 lim = desc_limit_scaled(&desc);
747 if (!(desc.type & 8) && (desc.type & 4)) {
748 /* expand-down segment */
751 lim = desc.d ? 0xffffffff : 0xffff;
755 if (lim == 0xffffffff)
758 *max_size = (u64)lim + 1 - addr.ea;
759 if (size > *max_size)
764 if (la & (insn_alignment(ctxt, size) - 1))
765 return emulate_gp(ctxt, 0);
766 return X86EMUL_CONTINUE;
768 if (addr.seg == VCPU_SREG_SS)
769 return emulate_ss(ctxt, 0);
771 return emulate_gp(ctxt, 0);
774 static int linearize(struct x86_emulate_ctxt *ctxt,
775 struct segmented_address addr,
776 unsigned size, bool write,
780 return __linearize(ctxt, addr, &max_size, size, write, false,
784 static inline int assign_eip(struct x86_emulate_ctxt *ctxt, ulong dst,
785 enum x86emul_mode mode)
790 struct segmented_address addr = { .seg = VCPU_SREG_CS,
793 if (ctxt->op_bytes != sizeof(unsigned long))
794 addr.ea = dst & ((1UL << (ctxt->op_bytes << 3)) - 1);
795 rc = __linearize(ctxt, addr, &max_size, 1, false, true, mode, &linear);
796 if (rc == X86EMUL_CONTINUE)
797 ctxt->_eip = addr.ea;
801 static inline int assign_eip_near(struct x86_emulate_ctxt *ctxt, ulong dst)
803 return assign_eip(ctxt, dst, ctxt->mode);
806 static int assign_eip_far(struct x86_emulate_ctxt *ctxt, ulong dst,
807 const struct desc_struct *cs_desc)
809 enum x86emul_mode mode = ctxt->mode;
813 if (ctxt->mode >= X86EMUL_MODE_PROT16) {
817 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
819 mode = X86EMUL_MODE_PROT64;
821 mode = X86EMUL_MODE_PROT32; /* temporary value */
824 if (mode == X86EMUL_MODE_PROT16 || mode == X86EMUL_MODE_PROT32)
825 mode = cs_desc->d ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
826 rc = assign_eip(ctxt, dst, mode);
827 if (rc == X86EMUL_CONTINUE)
832 static inline int jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
834 return assign_eip_near(ctxt, ctxt->_eip + rel);
837 static int linear_read_system(struct x86_emulate_ctxt *ctxt, ulong linear,
838 void *data, unsigned size)
840 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, true);
843 static int linear_write_system(struct x86_emulate_ctxt *ctxt,
844 ulong linear, void *data,
847 return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, true);
850 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
851 struct segmented_address addr,
858 rc = linearize(ctxt, addr, size, false, &linear);
859 if (rc != X86EMUL_CONTINUE)
861 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, false);
864 static int segmented_write_std(struct x86_emulate_ctxt *ctxt,
865 struct segmented_address addr,
872 rc = linearize(ctxt, addr, size, true, &linear);
873 if (rc != X86EMUL_CONTINUE)
875 return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, false);
879 * Prefetch the remaining bytes of the instruction without crossing page
880 * boundary if they are not in fetch_cache yet.
882 static int __do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt, int op_size)
885 unsigned size, max_size;
886 unsigned long linear;
887 int cur_size = ctxt->fetch.end - ctxt->fetch.data;
888 struct segmented_address addr = { .seg = VCPU_SREG_CS,
889 .ea = ctxt->eip + cur_size };
892 * We do not know exactly how many bytes will be needed, and
893 * __linearize is expensive, so fetch as much as possible. We
894 * just have to avoid going beyond the 15 byte limit, the end
895 * of the segment, or the end of the page.
897 * __linearize is called with size 0 so that it does not do any
898 * boundary check itself. Instead, we use max_size to check
901 rc = __linearize(ctxt, addr, &max_size, 0, false, true, ctxt->mode,
903 if (unlikely(rc != X86EMUL_CONTINUE))
906 size = min_t(unsigned, 15UL ^ cur_size, max_size);
907 size = min_t(unsigned, size, PAGE_SIZE - offset_in_page(linear));
910 * One instruction can only straddle two pages,
911 * and one has been loaded at the beginning of
912 * x86_decode_insn. So, if not enough bytes
913 * still, we must have hit the 15-byte boundary.
915 if (unlikely(size < op_size))
916 return emulate_gp(ctxt, 0);
918 rc = ctxt->ops->fetch(ctxt, linear, ctxt->fetch.end,
919 size, &ctxt->exception);
920 if (unlikely(rc != X86EMUL_CONTINUE))
922 ctxt->fetch.end += size;
923 return X86EMUL_CONTINUE;
926 static __always_inline int do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt,
929 unsigned done_size = ctxt->fetch.end - ctxt->fetch.ptr;
931 if (unlikely(done_size < size))
932 return __do_insn_fetch_bytes(ctxt, size - done_size);
934 return X86EMUL_CONTINUE;
937 /* Fetch next part of the instruction being emulated. */
938 #define insn_fetch(_type, _ctxt) \
941 rc = do_insn_fetch_bytes(_ctxt, sizeof(_type)); \
942 if (rc != X86EMUL_CONTINUE) \
944 ctxt->_eip += sizeof(_type); \
945 memcpy(&_x, ctxt->fetch.ptr, sizeof(_type)); \
946 ctxt->fetch.ptr += sizeof(_type); \
950 #define insn_fetch_arr(_arr, _size, _ctxt) \
952 rc = do_insn_fetch_bytes(_ctxt, _size); \
953 if (rc != X86EMUL_CONTINUE) \
955 ctxt->_eip += (_size); \
956 memcpy(_arr, ctxt->fetch.ptr, _size); \
957 ctxt->fetch.ptr += (_size); \
961 * Given the 'reg' portion of a ModRM byte, and a register block, return a
962 * pointer into the block that addresses the relevant register.
963 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
965 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
969 int highbyte_regs = (ctxt->rex_prefix == 0) && byteop;
971 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
972 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
974 p = reg_rmw(ctxt, modrm_reg);
978 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
979 struct segmented_address addr,
980 u16 *size, unsigned long *address, int op_bytes)
987 rc = segmented_read_std(ctxt, addr, size, 2);
988 if (rc != X86EMUL_CONTINUE)
991 rc = segmented_read_std(ctxt, addr, address, op_bytes);
1005 FASTOP1SRC2(mul, mul_ex);
1006 FASTOP1SRC2(imul, imul_ex);
1007 FASTOP1SRC2EX(div, div_ex);
1008 FASTOP1SRC2EX(idiv, idiv_ex);
1037 FASTOP2R(cmp, cmp_r);
1039 static int em_bsf_c(struct x86_emulate_ctxt *ctxt)
1041 /* If src is zero, do not writeback, but update flags */
1042 if (ctxt->src.val == 0)
1043 ctxt->dst.type = OP_NONE;
1044 return fastop(ctxt, em_bsf);
1047 static int em_bsr_c(struct x86_emulate_ctxt *ctxt)
1049 /* If src is zero, do not writeback, but update flags */
1050 if (ctxt->src.val == 0)
1051 ctxt->dst.type = OP_NONE;
1052 return fastop(ctxt, em_bsr);
1055 static __always_inline u8 test_cc(unsigned int condition, unsigned long flags)
1058 void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
1060 flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
1061 asm("push %[flags]; popf; " CALL_NOSPEC
1062 : "=a"(rc) : [thunk_target]"r"(fop), [flags]"r"(flags));
1066 static void fetch_register_operand(struct operand *op)
1068 switch (op->bytes) {
1070 op->val = *(u8 *)op->addr.reg;
1073 op->val = *(u16 *)op->addr.reg;
1076 op->val = *(u32 *)op->addr.reg;
1079 op->val = *(u64 *)op->addr.reg;
1084 static void emulator_get_fpu(void)
1088 fpregs_assert_state_consistent();
1089 if (test_thread_flag(TIF_NEED_FPU_LOAD))
1090 switch_fpu_return();
1093 static void emulator_put_fpu(void)
1098 static void read_sse_reg(sse128_t *data, int reg)
1102 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
1103 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
1104 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
1105 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
1106 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
1107 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
1108 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
1109 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
1110 #ifdef CONFIG_X86_64
1111 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
1112 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
1113 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
1114 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
1115 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
1116 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
1117 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
1118 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
1125 static void write_sse_reg(sse128_t *data, int reg)
1129 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
1130 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
1131 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
1132 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
1133 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
1134 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
1135 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
1136 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
1137 #ifdef CONFIG_X86_64
1138 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
1139 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
1140 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
1141 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
1142 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
1143 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
1144 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
1145 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
1152 static void read_mmx_reg(u64 *data, int reg)
1156 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
1157 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
1158 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
1159 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
1160 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
1161 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
1162 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
1163 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
1169 static void write_mmx_reg(u64 *data, int reg)
1173 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
1174 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
1175 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
1176 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
1177 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
1178 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
1179 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
1180 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
1186 static int em_fninit(struct x86_emulate_ctxt *ctxt)
1188 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1189 return emulate_nm(ctxt);
1192 asm volatile("fninit");
1194 return X86EMUL_CONTINUE;
1197 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
1201 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1202 return emulate_nm(ctxt);
1205 asm volatile("fnstcw %0": "+m"(fcw));
1208 ctxt->dst.val = fcw;
1210 return X86EMUL_CONTINUE;
1213 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1217 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1218 return emulate_nm(ctxt);
1221 asm volatile("fnstsw %0": "+m"(fsw));
1224 ctxt->dst.val = fsw;
1226 return X86EMUL_CONTINUE;
1229 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1232 unsigned reg = ctxt->modrm_reg;
1234 if (!(ctxt->d & ModRM))
1235 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1237 if (ctxt->d & Sse) {
1241 read_sse_reg(&op->vec_val, reg);
1244 if (ctxt->d & Mmx) {
1253 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1254 op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);
1256 fetch_register_operand(op);
1257 op->orig_val = op->val;
1260 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1262 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1263 ctxt->modrm_seg = VCPU_SREG_SS;
1266 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1270 int index_reg, base_reg, scale;
1271 int rc = X86EMUL_CONTINUE;
1274 ctxt->modrm_reg = ((ctxt->rex_prefix << 1) & 8); /* REX.R */
1275 index_reg = (ctxt->rex_prefix << 2) & 8; /* REX.X */
1276 base_reg = (ctxt->rex_prefix << 3) & 8; /* REX.B */
1278 ctxt->modrm_mod = (ctxt->modrm & 0xc0) >> 6;
1279 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1280 ctxt->modrm_rm = base_reg | (ctxt->modrm & 0x07);
1281 ctxt->modrm_seg = VCPU_SREG_DS;
1283 if (ctxt->modrm_mod == 3 || (ctxt->d & NoMod)) {
1285 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1286 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
1288 if (ctxt->d & Sse) {
1291 op->addr.xmm = ctxt->modrm_rm;
1292 read_sse_reg(&op->vec_val, ctxt->modrm_rm);
1295 if (ctxt->d & Mmx) {
1298 op->addr.mm = ctxt->modrm_rm & 7;
1301 fetch_register_operand(op);
1307 if (ctxt->ad_bytes == 2) {
1308 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1309 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1310 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1311 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1313 /* 16-bit ModR/M decode. */
1314 switch (ctxt->modrm_mod) {
1316 if (ctxt->modrm_rm == 6)
1317 modrm_ea += insn_fetch(u16, ctxt);
1320 modrm_ea += insn_fetch(s8, ctxt);
1323 modrm_ea += insn_fetch(u16, ctxt);
1326 switch (ctxt->modrm_rm) {
1328 modrm_ea += bx + si;
1331 modrm_ea += bx + di;
1334 modrm_ea += bp + si;
1337 modrm_ea += bp + di;
1346 if (ctxt->modrm_mod != 0)
1353 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1354 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1355 ctxt->modrm_seg = VCPU_SREG_SS;
1356 modrm_ea = (u16)modrm_ea;
1358 /* 32/64-bit ModR/M decode. */
1359 if ((ctxt->modrm_rm & 7) == 4) {
1360 sib = insn_fetch(u8, ctxt);
1361 index_reg |= (sib >> 3) & 7;
1362 base_reg |= sib & 7;
1365 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1366 modrm_ea += insn_fetch(s32, ctxt);
1368 modrm_ea += reg_read(ctxt, base_reg);
1369 adjust_modrm_seg(ctxt, base_reg);
1370 /* Increment ESP on POP [ESP] */
1371 if ((ctxt->d & IncSP) &&
1372 base_reg == VCPU_REGS_RSP)
1373 modrm_ea += ctxt->op_bytes;
1376 modrm_ea += reg_read(ctxt, index_reg) << scale;
1377 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1378 modrm_ea += insn_fetch(s32, ctxt);
1379 if (ctxt->mode == X86EMUL_MODE_PROT64)
1380 ctxt->rip_relative = 1;
1382 base_reg = ctxt->modrm_rm;
1383 modrm_ea += reg_read(ctxt, base_reg);
1384 adjust_modrm_seg(ctxt, base_reg);
1386 switch (ctxt->modrm_mod) {
1388 modrm_ea += insn_fetch(s8, ctxt);
1391 modrm_ea += insn_fetch(s32, ctxt);
1395 op->addr.mem.ea = modrm_ea;
1396 if (ctxt->ad_bytes != 8)
1397 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
1403 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1406 int rc = X86EMUL_CONTINUE;
1409 switch (ctxt->ad_bytes) {
1411 op->addr.mem.ea = insn_fetch(u16, ctxt);
1414 op->addr.mem.ea = insn_fetch(u32, ctxt);
1417 op->addr.mem.ea = insn_fetch(u64, ctxt);
1424 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1428 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1429 mask = ~((long)ctxt->dst.bytes * 8 - 1);
1431 if (ctxt->src.bytes == 2)
1432 sv = (s16)ctxt->src.val & (s16)mask;
1433 else if (ctxt->src.bytes == 4)
1434 sv = (s32)ctxt->src.val & (s32)mask;
1436 sv = (s64)ctxt->src.val & (s64)mask;
1438 ctxt->dst.addr.mem.ea = address_mask(ctxt,
1439 ctxt->dst.addr.mem.ea + (sv >> 3));
1442 /* only subword offset */
1443 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1446 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1447 unsigned long addr, void *dest, unsigned size)
1450 struct read_cache *mc = &ctxt->mem_read;
1452 if (mc->pos < mc->end)
1455 WARN_ON((mc->end + size) >= sizeof(mc->data));
1457 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1459 if (rc != X86EMUL_CONTINUE)
1465 memcpy(dest, mc->data + mc->pos, size);
1467 return X86EMUL_CONTINUE;
1470 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1471 struct segmented_address addr,
1478 rc = linearize(ctxt, addr, size, false, &linear);
1479 if (rc != X86EMUL_CONTINUE)
1481 return read_emulated(ctxt, linear, data, size);
1484 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1485 struct segmented_address addr,
1492 rc = linearize(ctxt, addr, size, true, &linear);
1493 if (rc != X86EMUL_CONTINUE)
1495 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1499 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1500 struct segmented_address addr,
1501 const void *orig_data, const void *data,
1507 rc = linearize(ctxt, addr, size, true, &linear);
1508 if (rc != X86EMUL_CONTINUE)
1510 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1511 size, &ctxt->exception);
1514 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1515 unsigned int size, unsigned short port,
1518 struct read_cache *rc = &ctxt->io_read;
1520 if (rc->pos == rc->end) { /* refill pio read ahead */
1521 unsigned int in_page, n;
1522 unsigned int count = ctxt->rep_prefix ?
1523 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1524 in_page = (ctxt->eflags & X86_EFLAGS_DF) ?
1525 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1526 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1527 n = min3(in_page, (unsigned int)sizeof(rc->data) / size, count);
1530 rc->pos = rc->end = 0;
1531 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1536 if (ctxt->rep_prefix && (ctxt->d & String) &&
1537 !(ctxt->eflags & X86_EFLAGS_DF)) {
1538 ctxt->dst.data = rc->data + rc->pos;
1539 ctxt->dst.type = OP_MEM_STR;
1540 ctxt->dst.count = (rc->end - rc->pos) / size;
1543 memcpy(dest, rc->data + rc->pos, size);
1549 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1550 u16 index, struct desc_struct *desc)
1555 ctxt->ops->get_idt(ctxt, &dt);
1557 if (dt.size < index * 8 + 7)
1558 return emulate_gp(ctxt, index << 3 | 0x2);
1560 addr = dt.address + index * 8;
1561 return linear_read_system(ctxt, addr, desc, sizeof(*desc));
1564 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1565 u16 selector, struct desc_ptr *dt)
1567 const struct x86_emulate_ops *ops = ctxt->ops;
1570 if (selector & 1 << 2) {
1571 struct desc_struct desc;
1574 memset(dt, 0, sizeof(*dt));
1575 if (!ops->get_segment(ctxt, &sel, &desc, &base3,
1579 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1580 dt->address = get_desc_base(&desc) | ((u64)base3 << 32);
1582 ops->get_gdt(ctxt, dt);
1585 static int get_descriptor_ptr(struct x86_emulate_ctxt *ctxt,
1586 u16 selector, ulong *desc_addr_p)
1589 u16 index = selector >> 3;
1592 get_descriptor_table_ptr(ctxt, selector, &dt);
1594 if (dt.size < index * 8 + 7)
1595 return emulate_gp(ctxt, selector & 0xfffc);
1597 addr = dt.address + index * 8;
1599 #ifdef CONFIG_X86_64
1600 if (addr >> 32 != 0) {
1603 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1604 if (!(efer & EFER_LMA))
1609 *desc_addr_p = addr;
1610 return X86EMUL_CONTINUE;
1613 /* allowed just for 8 bytes segments */
1614 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1615 u16 selector, struct desc_struct *desc,
1620 rc = get_descriptor_ptr(ctxt, selector, desc_addr_p);
1621 if (rc != X86EMUL_CONTINUE)
1624 return linear_read_system(ctxt, *desc_addr_p, desc, sizeof(*desc));
1627 /* allowed just for 8 bytes segments */
1628 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1629 u16 selector, struct desc_struct *desc)
1634 rc = get_descriptor_ptr(ctxt, selector, &addr);
1635 if (rc != X86EMUL_CONTINUE)
1638 return linear_write_system(ctxt, addr, desc, sizeof(*desc));
1641 static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1642 u16 selector, int seg, u8 cpl,
1643 enum x86_transfer_type transfer,
1644 struct desc_struct *desc)
1646 struct desc_struct seg_desc, old_desc;
1648 unsigned err_vec = GP_VECTOR;
1650 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1656 memset(&seg_desc, 0, sizeof(seg_desc));
1658 if (ctxt->mode == X86EMUL_MODE_REAL) {
1659 /* set real mode segment descriptor (keep limit etc. for
1661 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1662 set_desc_base(&seg_desc, selector << 4);
1664 } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
1665 /* VM86 needs a clean new segment descriptor */
1666 set_desc_base(&seg_desc, selector << 4);
1667 set_desc_limit(&seg_desc, 0xffff);
1677 /* TR should be in GDT only */
1678 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1681 /* NULL selector is not valid for TR, CS and (except for long mode) SS */
1682 if (null_selector) {
1683 if (seg == VCPU_SREG_CS || seg == VCPU_SREG_TR)
1686 if (seg == VCPU_SREG_SS) {
1687 if (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl)
1691 * ctxt->ops->set_segment expects the CPL to be in
1692 * SS.DPL, so fake an expand-up 32-bit data segment.
1702 /* Skip all following checks */
1706 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1707 if (ret != X86EMUL_CONTINUE)
1710 err_code = selector & 0xfffc;
1711 err_vec = (transfer == X86_TRANSFER_TASK_SWITCH) ? TS_VECTOR :
1714 /* can't load system descriptor into segment selector */
1715 if (seg <= VCPU_SREG_GS && !seg_desc.s) {
1716 if (transfer == X86_TRANSFER_CALL_JMP)
1717 return X86EMUL_UNHANDLEABLE;
1722 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1731 * segment is not a writable data segment or segment
1732 * selector's RPL != CPL or segment selector's RPL != CPL
1734 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1738 if (!(seg_desc.type & 8))
1741 if (seg_desc.type & 4) {
1747 if (rpl > cpl || dpl != cpl)
1750 /* in long-mode d/b must be clear if l is set */
1751 if (seg_desc.d && seg_desc.l) {
1754 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1755 if (efer & EFER_LMA)
1759 /* CS(RPL) <- CPL */
1760 selector = (selector & 0xfffc) | cpl;
1763 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1765 old_desc = seg_desc;
1766 seg_desc.type |= 2; /* busy */
1767 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1768 sizeof(seg_desc), &ctxt->exception);
1769 if (ret != X86EMUL_CONTINUE)
1772 case VCPU_SREG_LDTR:
1773 if (seg_desc.s || seg_desc.type != 2)
1776 default: /* DS, ES, FS, or GS */
1778 * segment is not a data or readable code segment or
1779 * ((segment is a data or nonconforming code segment)
1780 * and (both RPL and CPL > DPL))
1782 if ((seg_desc.type & 0xa) == 0x8 ||
1783 (((seg_desc.type & 0xc) != 0xc) &&
1784 (rpl > dpl && cpl > dpl)))
1790 /* mark segment as accessed */
1791 if (!(seg_desc.type & 1)) {
1793 ret = write_segment_descriptor(ctxt, selector,
1795 if (ret != X86EMUL_CONTINUE)
1798 } else if (ctxt->mode == X86EMUL_MODE_PROT64) {
1799 ret = linear_read_system(ctxt, desc_addr+8, &base3, sizeof(base3));
1800 if (ret != X86EMUL_CONTINUE)
1802 if (emul_is_noncanonical_address(get_desc_base(&seg_desc) |
1803 ((u64)base3 << 32), ctxt))
1804 return emulate_gp(ctxt, 0);
1807 ctxt->ops->set_segment(ctxt, selector, &seg_desc, base3, seg);
1810 return X86EMUL_CONTINUE;
1812 return emulate_exception(ctxt, err_vec, err_code, true);
1815 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1816 u16 selector, int seg)
1818 u8 cpl = ctxt->ops->cpl(ctxt);
1821 * None of MOV, POP and LSS can load a NULL selector in CPL=3, but
1822 * they can load it at CPL<3 (Intel's manual says only LSS can,
1825 * However, the Intel manual says that putting IST=1/DPL=3 in
1826 * an interrupt gate will result in SS=3 (the AMD manual instead
1827 * says it doesn't), so allow SS=3 in __load_segment_descriptor
1828 * and only forbid it here.
1830 if (seg == VCPU_SREG_SS && selector == 3 &&
1831 ctxt->mode == X86EMUL_MODE_PROT64)
1832 return emulate_exception(ctxt, GP_VECTOR, 0, true);
1834 return __load_segment_descriptor(ctxt, selector, seg, cpl,
1835 X86_TRANSFER_NONE, NULL);
1838 static void write_register_operand(struct operand *op)
1840 return assign_register(op->addr.reg, op->val, op->bytes);
1843 static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
1847 write_register_operand(op);
1850 if (ctxt->lock_prefix)
1851 return segmented_cmpxchg(ctxt,
1857 return segmented_write(ctxt,
1863 return segmented_write(ctxt,
1866 op->bytes * op->count);
1869 write_sse_reg(&op->vec_val, op->addr.xmm);
1872 write_mmx_reg(&op->mm_val, op->addr.mm);
1880 return X86EMUL_CONTINUE;
1883 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1885 struct segmented_address addr;
1887 rsp_increment(ctxt, -bytes);
1888 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1889 addr.seg = VCPU_SREG_SS;
1891 return segmented_write(ctxt, addr, data, bytes);
1894 static int em_push(struct x86_emulate_ctxt *ctxt)
1896 /* Disable writeback. */
1897 ctxt->dst.type = OP_NONE;
1898 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1901 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1902 void *dest, int len)
1905 struct segmented_address addr;
1907 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1908 addr.seg = VCPU_SREG_SS;
1909 rc = segmented_read(ctxt, addr, dest, len);
1910 if (rc != X86EMUL_CONTINUE)
1913 rsp_increment(ctxt, len);
1917 static int em_pop(struct x86_emulate_ctxt *ctxt)
1919 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1922 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1923 void *dest, int len)
1926 unsigned long val, change_mask;
1927 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
1928 int cpl = ctxt->ops->cpl(ctxt);
1930 rc = emulate_pop(ctxt, &val, len);
1931 if (rc != X86EMUL_CONTINUE)
1934 change_mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
1935 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF |
1936 X86_EFLAGS_TF | X86_EFLAGS_DF | X86_EFLAGS_NT |
1937 X86_EFLAGS_AC | X86_EFLAGS_ID;
1939 switch(ctxt->mode) {
1940 case X86EMUL_MODE_PROT64:
1941 case X86EMUL_MODE_PROT32:
1942 case X86EMUL_MODE_PROT16:
1944 change_mask |= X86_EFLAGS_IOPL;
1946 change_mask |= X86_EFLAGS_IF;
1948 case X86EMUL_MODE_VM86:
1950 return emulate_gp(ctxt, 0);
1951 change_mask |= X86_EFLAGS_IF;
1953 default: /* real mode */
1954 change_mask |= (X86_EFLAGS_IOPL | X86_EFLAGS_IF);
1958 *(unsigned long *)dest =
1959 (ctxt->eflags & ~change_mask) | (val & change_mask);
1964 static int em_popf(struct x86_emulate_ctxt *ctxt)
1966 ctxt->dst.type = OP_REG;
1967 ctxt->dst.addr.reg = &ctxt->eflags;
1968 ctxt->dst.bytes = ctxt->op_bytes;
1969 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1972 static int em_enter(struct x86_emulate_ctxt *ctxt)
1975 unsigned frame_size = ctxt->src.val;
1976 unsigned nesting_level = ctxt->src2.val & 31;
1980 return X86EMUL_UNHANDLEABLE;
1982 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1983 rc = push(ctxt, &rbp, stack_size(ctxt));
1984 if (rc != X86EMUL_CONTINUE)
1986 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1988 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1989 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1991 return X86EMUL_CONTINUE;
1994 static int em_leave(struct x86_emulate_ctxt *ctxt)
1996 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1998 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
2001 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
2003 int seg = ctxt->src2.val;
2005 ctxt->src.val = get_segment_selector(ctxt, seg);
2006 if (ctxt->op_bytes == 4) {
2007 rsp_increment(ctxt, -2);
2011 return em_push(ctxt);
2014 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
2016 int seg = ctxt->src2.val;
2017 unsigned long selector;
2020 rc = emulate_pop(ctxt, &selector, 2);
2021 if (rc != X86EMUL_CONTINUE)
2024 if (ctxt->modrm_reg == VCPU_SREG_SS)
2025 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
2026 if (ctxt->op_bytes > 2)
2027 rsp_increment(ctxt, ctxt->op_bytes - 2);
2029 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
2033 static int em_pusha(struct x86_emulate_ctxt *ctxt)
2035 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
2036 int rc = X86EMUL_CONTINUE;
2037 int reg = VCPU_REGS_RAX;
2039 while (reg <= VCPU_REGS_RDI) {
2040 (reg == VCPU_REGS_RSP) ?
2041 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
2044 if (rc != X86EMUL_CONTINUE)
2053 static int em_pushf(struct x86_emulate_ctxt *ctxt)
2055 ctxt->src.val = (unsigned long)ctxt->eflags & ~X86_EFLAGS_VM;
2056 return em_push(ctxt);
2059 static int em_popa(struct x86_emulate_ctxt *ctxt)
2061 int rc = X86EMUL_CONTINUE;
2062 int reg = VCPU_REGS_RDI;
2065 while (reg >= VCPU_REGS_RAX) {
2066 if (reg == VCPU_REGS_RSP) {
2067 rsp_increment(ctxt, ctxt->op_bytes);
2071 rc = emulate_pop(ctxt, &val, ctxt->op_bytes);
2072 if (rc != X86EMUL_CONTINUE)
2074 assign_register(reg_rmw(ctxt, reg), val, ctxt->op_bytes);
2080 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
2082 const struct x86_emulate_ops *ops = ctxt->ops;
2089 /* TODO: Add limit checks */
2090 ctxt->src.val = ctxt->eflags;
2092 if (rc != X86EMUL_CONTINUE)
2095 ctxt->eflags &= ~(X86_EFLAGS_IF | X86_EFLAGS_TF | X86_EFLAGS_AC);
2097 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
2099 if (rc != X86EMUL_CONTINUE)
2102 ctxt->src.val = ctxt->_eip;
2104 if (rc != X86EMUL_CONTINUE)
2107 ops->get_idt(ctxt, &dt);
2109 eip_addr = dt.address + (irq << 2);
2110 cs_addr = dt.address + (irq << 2) + 2;
2112 rc = linear_read_system(ctxt, cs_addr, &cs, 2);
2113 if (rc != X86EMUL_CONTINUE)
2116 rc = linear_read_system(ctxt, eip_addr, &eip, 2);
2117 if (rc != X86EMUL_CONTINUE)
2120 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
2121 if (rc != X86EMUL_CONTINUE)
2129 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
2133 invalidate_registers(ctxt);
2134 rc = __emulate_int_real(ctxt, irq);
2135 if (rc == X86EMUL_CONTINUE)
2136 writeback_registers(ctxt);
2140 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
2142 switch(ctxt->mode) {
2143 case X86EMUL_MODE_REAL:
2144 return __emulate_int_real(ctxt, irq);
2145 case X86EMUL_MODE_VM86:
2146 case X86EMUL_MODE_PROT16:
2147 case X86EMUL_MODE_PROT32:
2148 case X86EMUL_MODE_PROT64:
2150 /* Protected mode interrupts unimplemented yet */
2151 return X86EMUL_UNHANDLEABLE;
2155 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
2157 int rc = X86EMUL_CONTINUE;
2158 unsigned long temp_eip = 0;
2159 unsigned long temp_eflags = 0;
2160 unsigned long cs = 0;
2161 unsigned long mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
2162 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_TF |
2163 X86_EFLAGS_IF | X86_EFLAGS_DF | X86_EFLAGS_OF |
2164 X86_EFLAGS_IOPL | X86_EFLAGS_NT | X86_EFLAGS_RF |
2165 X86_EFLAGS_AC | X86_EFLAGS_ID |
2167 unsigned long vm86_mask = X86_EFLAGS_VM | X86_EFLAGS_VIF |
2170 /* TODO: Add stack limit check */
2172 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
2174 if (rc != X86EMUL_CONTINUE)
2177 if (temp_eip & ~0xffff)
2178 return emulate_gp(ctxt, 0);
2180 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2182 if (rc != X86EMUL_CONTINUE)
2185 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
2187 if (rc != X86EMUL_CONTINUE)
2190 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2192 if (rc != X86EMUL_CONTINUE)
2195 ctxt->_eip = temp_eip;
2197 if (ctxt->op_bytes == 4)
2198 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
2199 else if (ctxt->op_bytes == 2) {
2200 ctxt->eflags &= ~0xffff;
2201 ctxt->eflags |= temp_eflags;
2204 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
2205 ctxt->eflags |= X86_EFLAGS_FIXED;
2206 ctxt->ops->set_nmi_mask(ctxt, false);
2211 static int em_iret(struct x86_emulate_ctxt *ctxt)
2213 switch(ctxt->mode) {
2214 case X86EMUL_MODE_REAL:
2215 return emulate_iret_real(ctxt);
2216 case X86EMUL_MODE_VM86:
2217 case X86EMUL_MODE_PROT16:
2218 case X86EMUL_MODE_PROT32:
2219 case X86EMUL_MODE_PROT64:
2221 /* iret from protected mode unimplemented yet */
2222 return X86EMUL_UNHANDLEABLE;
2226 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
2230 struct desc_struct new_desc;
2231 u8 cpl = ctxt->ops->cpl(ctxt);
2233 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2235 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
2236 X86_TRANSFER_CALL_JMP,
2238 if (rc != X86EMUL_CONTINUE)
2241 rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
2242 /* Error handling is not implemented. */
2243 if (rc != X86EMUL_CONTINUE)
2244 return X86EMUL_UNHANDLEABLE;
2249 static int em_jmp_abs(struct x86_emulate_ctxt *ctxt)
2251 return assign_eip_near(ctxt, ctxt->src.val);
2254 static int em_call_near_abs(struct x86_emulate_ctxt *ctxt)
2259 old_eip = ctxt->_eip;
2260 rc = assign_eip_near(ctxt, ctxt->src.val);
2261 if (rc != X86EMUL_CONTINUE)
2263 ctxt->src.val = old_eip;
2268 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2270 u64 old = ctxt->dst.orig_val64;
2272 if (ctxt->dst.bytes == 16)
2273 return X86EMUL_UNHANDLEABLE;
2275 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
2276 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2277 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2278 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2279 ctxt->eflags &= ~X86_EFLAGS_ZF;
2281 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2282 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2284 ctxt->eflags |= X86_EFLAGS_ZF;
2286 return X86EMUL_CONTINUE;
2289 static int em_ret(struct x86_emulate_ctxt *ctxt)
2294 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2295 if (rc != X86EMUL_CONTINUE)
2298 return assign_eip_near(ctxt, eip);
2301 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2304 unsigned long eip, cs;
2305 int cpl = ctxt->ops->cpl(ctxt);
2306 struct desc_struct new_desc;
2308 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2309 if (rc != X86EMUL_CONTINUE)
2311 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2312 if (rc != X86EMUL_CONTINUE)
2314 /* Outer-privilege level return is not implemented */
2315 if (ctxt->mode >= X86EMUL_MODE_PROT16 && (cs & 3) > cpl)
2316 return X86EMUL_UNHANDLEABLE;
2317 rc = __load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS, cpl,
2320 if (rc != X86EMUL_CONTINUE)
2322 rc = assign_eip_far(ctxt, eip, &new_desc);
2323 /* Error handling is not implemented. */
2324 if (rc != X86EMUL_CONTINUE)
2325 return X86EMUL_UNHANDLEABLE;
2330 static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt)
2334 rc = em_ret_far(ctxt);
2335 if (rc != X86EMUL_CONTINUE)
2337 rsp_increment(ctxt, ctxt->src.val);
2338 return X86EMUL_CONTINUE;
2341 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2343 /* Save real source value, then compare EAX against destination. */
2344 ctxt->dst.orig_val = ctxt->dst.val;
2345 ctxt->dst.val = reg_read(ctxt, VCPU_REGS_RAX);
2346 ctxt->src.orig_val = ctxt->src.val;
2347 ctxt->src.val = ctxt->dst.orig_val;
2348 fastop(ctxt, em_cmp);
2350 if (ctxt->eflags & X86_EFLAGS_ZF) {
2351 /* Success: write back to memory; no update of EAX */
2352 ctxt->src.type = OP_NONE;
2353 ctxt->dst.val = ctxt->src.orig_val;
2355 /* Failure: write the value we saw to EAX. */
2356 ctxt->src.type = OP_REG;
2357 ctxt->src.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2358 ctxt->src.val = ctxt->dst.orig_val;
2359 /* Create write-cycle to dest by writing the same value */
2360 ctxt->dst.val = ctxt->dst.orig_val;
2362 return X86EMUL_CONTINUE;
2365 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2367 int seg = ctxt->src2.val;
2371 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2373 rc = load_segment_descriptor(ctxt, sel, seg);
2374 if (rc != X86EMUL_CONTINUE)
2377 ctxt->dst.val = ctxt->src.val;
2381 static int emulator_has_longmode(struct x86_emulate_ctxt *ctxt)
2383 #ifdef CONFIG_X86_64
2384 return ctxt->ops->guest_has_long_mode(ctxt);
2390 static void rsm_set_desc_flags(struct desc_struct *desc, u32 flags)
2392 desc->g = (flags >> 23) & 1;
2393 desc->d = (flags >> 22) & 1;
2394 desc->l = (flags >> 21) & 1;
2395 desc->avl = (flags >> 20) & 1;
2396 desc->p = (flags >> 15) & 1;
2397 desc->dpl = (flags >> 13) & 3;
2398 desc->s = (flags >> 12) & 1;
2399 desc->type = (flags >> 8) & 15;
2402 static int rsm_load_seg_32(struct x86_emulate_ctxt *ctxt, const char *smstate,
2405 struct desc_struct desc;
2409 selector = GET_SMSTATE(u32, smstate, 0x7fa8 + n * 4);
2412 offset = 0x7f84 + n * 12;
2414 offset = 0x7f2c + (n - 3) * 12;
2416 set_desc_base(&desc, GET_SMSTATE(u32, smstate, offset + 8));
2417 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, offset + 4));
2418 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, offset));
2419 ctxt->ops->set_segment(ctxt, selector, &desc, 0, n);
2420 return X86EMUL_CONTINUE;
2423 #ifdef CONFIG_X86_64
2424 static int rsm_load_seg_64(struct x86_emulate_ctxt *ctxt, const char *smstate,
2427 struct desc_struct desc;
2432 offset = 0x7e00 + n * 16;
2434 selector = GET_SMSTATE(u16, smstate, offset);
2435 rsm_set_desc_flags(&desc, GET_SMSTATE(u16, smstate, offset + 2) << 8);
2436 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, offset + 4));
2437 set_desc_base(&desc, GET_SMSTATE(u32, smstate, offset + 8));
2438 base3 = GET_SMSTATE(u32, smstate, offset + 12);
2440 ctxt->ops->set_segment(ctxt, selector, &desc, base3, n);
2441 return X86EMUL_CONTINUE;
2445 static int rsm_enter_protected_mode(struct x86_emulate_ctxt *ctxt,
2446 u64 cr0, u64 cr3, u64 cr4)
2451 /* In order to later set CR4.PCIDE, CR3[11:0] must be zero. */
2453 if (cr4 & X86_CR4_PCIDE) {
2458 bad = ctxt->ops->set_cr(ctxt, 3, cr3);
2460 return X86EMUL_UNHANDLEABLE;
2463 * First enable PAE, long mode needs it before CR0.PG = 1 is set.
2464 * Then enable protected mode. However, PCID cannot be enabled
2465 * if EFER.LMA=0, so set it separately.
2467 bad = ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE);
2469 return X86EMUL_UNHANDLEABLE;
2471 bad = ctxt->ops->set_cr(ctxt, 0, cr0);
2473 return X86EMUL_UNHANDLEABLE;
2475 if (cr4 & X86_CR4_PCIDE) {
2476 bad = ctxt->ops->set_cr(ctxt, 4, cr4);
2478 return X86EMUL_UNHANDLEABLE;
2480 bad = ctxt->ops->set_cr(ctxt, 3, cr3 | pcid);
2482 return X86EMUL_UNHANDLEABLE;
2487 return X86EMUL_CONTINUE;
2490 static int rsm_load_state_32(struct x86_emulate_ctxt *ctxt,
2491 const char *smstate)
2493 struct desc_struct desc;
2496 u32 val, cr0, cr3, cr4;
2499 cr0 = GET_SMSTATE(u32, smstate, 0x7ffc);
2500 cr3 = GET_SMSTATE(u32, smstate, 0x7ff8);
2501 ctxt->eflags = GET_SMSTATE(u32, smstate, 0x7ff4) | X86_EFLAGS_FIXED;
2502 ctxt->_eip = GET_SMSTATE(u32, smstate, 0x7ff0);
2504 for (i = 0; i < 8; i++)
2505 *reg_write(ctxt, i) = GET_SMSTATE(u32, smstate, 0x7fd0 + i * 4);
2507 val = GET_SMSTATE(u32, smstate, 0x7fcc);
2509 if (ctxt->ops->set_dr(ctxt, 6, val))
2510 return X86EMUL_UNHANDLEABLE;
2512 val = GET_SMSTATE(u32, smstate, 0x7fc8);
2514 if (ctxt->ops->set_dr(ctxt, 7, val))
2515 return X86EMUL_UNHANDLEABLE;
2517 selector = GET_SMSTATE(u32, smstate, 0x7fc4);
2518 set_desc_base(&desc, GET_SMSTATE(u32, smstate, 0x7f64));
2519 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, 0x7f60));
2520 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, 0x7f5c));
2521 ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_TR);
2523 selector = GET_SMSTATE(u32, smstate, 0x7fc0);
2524 set_desc_base(&desc, GET_SMSTATE(u32, smstate, 0x7f80));
2525 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, 0x7f7c));
2526 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, 0x7f78));
2527 ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_LDTR);
2529 dt.address = GET_SMSTATE(u32, smstate, 0x7f74);
2530 dt.size = GET_SMSTATE(u32, smstate, 0x7f70);
2531 ctxt->ops->set_gdt(ctxt, &dt);
2533 dt.address = GET_SMSTATE(u32, smstate, 0x7f58);
2534 dt.size = GET_SMSTATE(u32, smstate, 0x7f54);
2535 ctxt->ops->set_idt(ctxt, &dt);
2537 for (i = 0; i < 6; i++) {
2538 int r = rsm_load_seg_32(ctxt, smstate, i);
2539 if (r != X86EMUL_CONTINUE)
2543 cr4 = GET_SMSTATE(u32, smstate, 0x7f14);
2545 ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smstate, 0x7ef8));
2547 return rsm_enter_protected_mode(ctxt, cr0, cr3, cr4);
2550 #ifdef CONFIG_X86_64
2551 static int rsm_load_state_64(struct x86_emulate_ctxt *ctxt,
2552 const char *smstate)
2554 struct desc_struct desc;
2556 u64 val, cr0, cr3, cr4;
2561 for (i = 0; i < 16; i++)
2562 *reg_write(ctxt, i) = GET_SMSTATE(u64, smstate, 0x7ff8 - i * 8);
2564 ctxt->_eip = GET_SMSTATE(u64, smstate, 0x7f78);
2565 ctxt->eflags = GET_SMSTATE(u32, smstate, 0x7f70) | X86_EFLAGS_FIXED;
2567 val = GET_SMSTATE(u64, smstate, 0x7f68);
2569 if (ctxt->ops->set_dr(ctxt, 6, val))
2570 return X86EMUL_UNHANDLEABLE;
2572 val = GET_SMSTATE(u64, smstate, 0x7f60);
2574 if (ctxt->ops->set_dr(ctxt, 7, val))
2575 return X86EMUL_UNHANDLEABLE;
2577 cr0 = GET_SMSTATE(u64, smstate, 0x7f58);
2578 cr3 = GET_SMSTATE(u64, smstate, 0x7f50);
2579 cr4 = GET_SMSTATE(u64, smstate, 0x7f48);
2580 ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smstate, 0x7f00));
2581 val = GET_SMSTATE(u64, smstate, 0x7ed0);
2583 if (ctxt->ops->set_msr(ctxt, MSR_EFER, val & ~EFER_LMA))
2584 return X86EMUL_UNHANDLEABLE;
2586 selector = GET_SMSTATE(u32, smstate, 0x7e90);
2587 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, 0x7e92) << 8);
2588 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, 0x7e94));
2589 set_desc_base(&desc, GET_SMSTATE(u32, smstate, 0x7e98));
2590 base3 = GET_SMSTATE(u32, smstate, 0x7e9c);
2591 ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_TR);
2593 dt.size = GET_SMSTATE(u32, smstate, 0x7e84);
2594 dt.address = GET_SMSTATE(u64, smstate, 0x7e88);
2595 ctxt->ops->set_idt(ctxt, &dt);
2597 selector = GET_SMSTATE(u32, smstate, 0x7e70);
2598 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, 0x7e72) << 8);
2599 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, 0x7e74));
2600 set_desc_base(&desc, GET_SMSTATE(u32, smstate, 0x7e78));
2601 base3 = GET_SMSTATE(u32, smstate, 0x7e7c);
2602 ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_LDTR);
2604 dt.size = GET_SMSTATE(u32, smstate, 0x7e64);
2605 dt.address = GET_SMSTATE(u64, smstate, 0x7e68);
2606 ctxt->ops->set_gdt(ctxt, &dt);
2608 r = rsm_enter_protected_mode(ctxt, cr0, cr3, cr4);
2609 if (r != X86EMUL_CONTINUE)
2612 for (i = 0; i < 6; i++) {
2613 r = rsm_load_seg_64(ctxt, smstate, i);
2614 if (r != X86EMUL_CONTINUE)
2618 return X86EMUL_CONTINUE;
2622 static int em_rsm(struct x86_emulate_ctxt *ctxt)
2624 unsigned long cr0, cr4, efer;
2629 if ((ctxt->ops->get_hflags(ctxt) & X86EMUL_SMM_MASK) == 0)
2630 return emulate_ud(ctxt);
2632 smbase = ctxt->ops->get_smbase(ctxt);
2634 ret = ctxt->ops->read_phys(ctxt, smbase + 0xfe00, buf, sizeof(buf));
2635 if (ret != X86EMUL_CONTINUE)
2636 return X86EMUL_UNHANDLEABLE;
2638 if ((ctxt->ops->get_hflags(ctxt) & X86EMUL_SMM_INSIDE_NMI_MASK) == 0)
2639 ctxt->ops->set_nmi_mask(ctxt, false);
2641 ctxt->ops->set_hflags(ctxt, ctxt->ops->get_hflags(ctxt) &
2642 ~(X86EMUL_SMM_INSIDE_NMI_MASK | X86EMUL_SMM_MASK));
2645 * Get back to real mode, to prepare a safe state in which to load
2646 * CR0/CR3/CR4/EFER. It's all a bit more complicated if the vCPU
2647 * supports long mode.
2649 if (emulator_has_longmode(ctxt)) {
2650 struct desc_struct cs_desc;
2652 /* Zero CR4.PCIDE before CR0.PG. */
2653 cr4 = ctxt->ops->get_cr(ctxt, 4);
2654 if (cr4 & X86_CR4_PCIDE)
2655 ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE);
2657 /* A 32-bit code segment is required to clear EFER.LMA. */
2658 memset(&cs_desc, 0, sizeof(cs_desc));
2660 cs_desc.s = cs_desc.g = cs_desc.p = 1;
2661 ctxt->ops->set_segment(ctxt, 0, &cs_desc, 0, VCPU_SREG_CS);
2664 /* For the 64-bit case, this will clear EFER.LMA. */
2665 cr0 = ctxt->ops->get_cr(ctxt, 0);
2666 if (cr0 & X86_CR0_PE)
2667 ctxt->ops->set_cr(ctxt, 0, cr0 & ~(X86_CR0_PG | X86_CR0_PE));
2669 if (emulator_has_longmode(ctxt)) {
2670 /* Clear CR4.PAE before clearing EFER.LME. */
2671 cr4 = ctxt->ops->get_cr(ctxt, 4);
2672 if (cr4 & X86_CR4_PAE)
2673 ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PAE);
2675 /* And finally go back to 32-bit mode. */
2677 ctxt->ops->set_msr(ctxt, MSR_EFER, efer);
2681 * Give pre_leave_smm() a chance to make ISA-specific changes to the
2682 * vCPU state (e.g. enter guest mode) before loading state from the SMM
2685 if (ctxt->ops->pre_leave_smm(ctxt, buf))
2686 return X86EMUL_UNHANDLEABLE;
2688 #ifdef CONFIG_X86_64
2689 if (emulator_has_longmode(ctxt))
2690 ret = rsm_load_state_64(ctxt, buf);
2693 ret = rsm_load_state_32(ctxt, buf);
2695 if (ret != X86EMUL_CONTINUE) {
2696 /* FIXME: should triple fault */
2697 return X86EMUL_UNHANDLEABLE;
2700 ctxt->ops->post_leave_smm(ctxt);
2702 return X86EMUL_CONTINUE;
2706 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2707 struct desc_struct *cs, struct desc_struct *ss)
2709 cs->l = 0; /* will be adjusted later */
2710 set_desc_base(cs, 0); /* flat segment */
2711 cs->g = 1; /* 4kb granularity */
2712 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2713 cs->type = 0x0b; /* Read, Execute, Accessed */
2715 cs->dpl = 0; /* will be adjusted later */
2720 set_desc_base(ss, 0); /* flat segment */
2721 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2722 ss->g = 1; /* 4kb granularity */
2724 ss->type = 0x03; /* Read/Write, Accessed */
2725 ss->d = 1; /* 32bit stack segment */
2732 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2734 u32 eax, ebx, ecx, edx;
2737 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, true);
2738 return is_guest_vendor_intel(ebx, ecx, edx);
2741 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2743 const struct x86_emulate_ops *ops = ctxt->ops;
2744 u32 eax, ebx, ecx, edx;
2747 * syscall should always be enabled in longmode - so only become
2748 * vendor specific (cpuid) if other modes are active...
2750 if (ctxt->mode == X86EMUL_MODE_PROT64)
2755 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, true);
2757 * remark: Intel CPUs only support "syscall" in 64bit longmode. Also a
2758 * 64bit guest with a 32bit compat-app running will #UD !! While this
2759 * behaviour can be fixed (by emulating) into AMD response - CPUs of
2760 * AMD can't behave like Intel.
2762 if (is_guest_vendor_intel(ebx, ecx, edx))
2765 if (is_guest_vendor_amd(ebx, ecx, edx) ||
2766 is_guest_vendor_hygon(ebx, ecx, edx))
2770 * default: (not Intel, not AMD, not Hygon), apply Intel's
2776 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2778 const struct x86_emulate_ops *ops = ctxt->ops;
2779 struct desc_struct cs, ss;
2784 /* syscall is not available in real mode */
2785 if (ctxt->mode == X86EMUL_MODE_REAL ||
2786 ctxt->mode == X86EMUL_MODE_VM86)
2787 return emulate_ud(ctxt);
2789 if (!(em_syscall_is_enabled(ctxt)))
2790 return emulate_ud(ctxt);
2792 ops->get_msr(ctxt, MSR_EFER, &efer);
2793 if (!(efer & EFER_SCE))
2794 return emulate_ud(ctxt);
2796 setup_syscalls_segments(ctxt, &cs, &ss);
2797 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2799 cs_sel = (u16)(msr_data & 0xfffc);
2800 ss_sel = (u16)(msr_data + 8);
2802 if (efer & EFER_LMA) {
2806 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2807 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2809 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2810 if (efer & EFER_LMA) {
2811 #ifdef CONFIG_X86_64
2812 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags;
2815 ctxt->mode == X86EMUL_MODE_PROT64 ?
2816 MSR_LSTAR : MSR_CSTAR, &msr_data);
2817 ctxt->_eip = msr_data;
2819 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2820 ctxt->eflags &= ~msr_data;
2821 ctxt->eflags |= X86_EFLAGS_FIXED;
2825 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2826 ctxt->_eip = (u32)msr_data;
2828 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2831 ctxt->tf = (ctxt->eflags & X86_EFLAGS_TF) != 0;
2832 return X86EMUL_CONTINUE;
2835 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2837 const struct x86_emulate_ops *ops = ctxt->ops;
2838 struct desc_struct cs, ss;
2843 ops->get_msr(ctxt, MSR_EFER, &efer);
2844 /* inject #GP if in real mode */
2845 if (ctxt->mode == X86EMUL_MODE_REAL)
2846 return emulate_gp(ctxt, 0);
2849 * Not recognized on AMD in compat mode (but is recognized in legacy
2852 if ((ctxt->mode != X86EMUL_MODE_PROT64) && (efer & EFER_LMA)
2853 && !vendor_intel(ctxt))
2854 return emulate_ud(ctxt);
2856 /* sysenter/sysexit have not been tested in 64bit mode. */
2857 if (ctxt->mode == X86EMUL_MODE_PROT64)
2858 return X86EMUL_UNHANDLEABLE;
2860 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2861 if ((msr_data & 0xfffc) == 0x0)
2862 return emulate_gp(ctxt, 0);
2864 setup_syscalls_segments(ctxt, &cs, &ss);
2865 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2866 cs_sel = (u16)msr_data & ~SEGMENT_RPL_MASK;
2867 ss_sel = cs_sel + 8;
2868 if (efer & EFER_LMA) {
2873 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2874 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2876 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2877 ctxt->_eip = (efer & EFER_LMA) ? msr_data : (u32)msr_data;
2879 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2880 *reg_write(ctxt, VCPU_REGS_RSP) = (efer & EFER_LMA) ? msr_data :
2882 if (efer & EFER_LMA)
2883 ctxt->mode = X86EMUL_MODE_PROT64;
2885 return X86EMUL_CONTINUE;
2888 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2890 const struct x86_emulate_ops *ops = ctxt->ops;
2891 struct desc_struct cs, ss;
2892 u64 msr_data, rcx, rdx;
2894 u16 cs_sel = 0, ss_sel = 0;
2896 /* inject #GP if in real mode or Virtual 8086 mode */
2897 if (ctxt->mode == X86EMUL_MODE_REAL ||
2898 ctxt->mode == X86EMUL_MODE_VM86)
2899 return emulate_gp(ctxt, 0);
2901 setup_syscalls_segments(ctxt, &cs, &ss);
2903 if ((ctxt->rex_prefix & 0x8) != 0x0)
2904 usermode = X86EMUL_MODE_PROT64;
2906 usermode = X86EMUL_MODE_PROT32;
2908 rcx = reg_read(ctxt, VCPU_REGS_RCX);
2909 rdx = reg_read(ctxt, VCPU_REGS_RDX);
2913 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2915 case X86EMUL_MODE_PROT32:
2916 cs_sel = (u16)(msr_data + 16);
2917 if ((msr_data & 0xfffc) == 0x0)
2918 return emulate_gp(ctxt, 0);
2919 ss_sel = (u16)(msr_data + 24);
2923 case X86EMUL_MODE_PROT64:
2924 cs_sel = (u16)(msr_data + 32);
2925 if (msr_data == 0x0)
2926 return emulate_gp(ctxt, 0);
2927 ss_sel = cs_sel + 8;
2930 if (emul_is_noncanonical_address(rcx, ctxt) ||
2931 emul_is_noncanonical_address(rdx, ctxt))
2932 return emulate_gp(ctxt, 0);
2935 cs_sel |= SEGMENT_RPL_MASK;
2936 ss_sel |= SEGMENT_RPL_MASK;
2938 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2939 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2942 *reg_write(ctxt, VCPU_REGS_RSP) = rcx;
2944 return X86EMUL_CONTINUE;
2947 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2950 if (ctxt->mode == X86EMUL_MODE_REAL)
2952 if (ctxt->mode == X86EMUL_MODE_VM86)
2954 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
2955 return ctxt->ops->cpl(ctxt) > iopl;
2958 #define VMWARE_PORT_VMPORT (0x5658)
2959 #define VMWARE_PORT_VMRPC (0x5659)
2961 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2964 const struct x86_emulate_ops *ops = ctxt->ops;
2965 struct desc_struct tr_seg;
2968 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2969 unsigned mask = (1 << len) - 1;
2973 * VMware allows access to these ports even if denied
2974 * by TSS I/O permission bitmap. Mimic behavior.
2976 if (enable_vmware_backdoor &&
2977 ((port == VMWARE_PORT_VMPORT) || (port == VMWARE_PORT_VMRPC)))
2980 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2983 if (desc_limit_scaled(&tr_seg) < 103)
2985 base = get_desc_base(&tr_seg);
2986 #ifdef CONFIG_X86_64
2987 base |= ((u64)base3) << 32;
2989 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL, true);
2990 if (r != X86EMUL_CONTINUE)
2992 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2994 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL, true);
2995 if (r != X86EMUL_CONTINUE)
2997 if ((perm >> bit_idx) & mask)
3002 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
3008 if (emulator_bad_iopl(ctxt))
3009 if (!emulator_io_port_access_allowed(ctxt, port, len))
3012 ctxt->perm_ok = true;
3017 static void string_registers_quirk(struct x86_emulate_ctxt *ctxt)
3020 * Intel CPUs mask the counter and pointers in quite strange
3021 * manner when ECX is zero due to REP-string optimizations.
3023 #ifdef CONFIG_X86_64
3024 if (ctxt->ad_bytes != 4 || !vendor_intel(ctxt))
3027 *reg_write(ctxt, VCPU_REGS_RCX) = 0;
3030 case 0xa4: /* movsb */
3031 case 0xa5: /* movsd/w */
3032 *reg_rmw(ctxt, VCPU_REGS_RSI) &= (u32)-1;
3034 case 0xaa: /* stosb */
3035 case 0xab: /* stosd/w */
3036 *reg_rmw(ctxt, VCPU_REGS_RDI) &= (u32)-1;
3041 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
3042 struct tss_segment_16 *tss)
3044 tss->ip = ctxt->_eip;
3045 tss->flag = ctxt->eflags;
3046 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
3047 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
3048 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
3049 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
3050 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
3051 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
3052 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
3053 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
3055 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
3056 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
3057 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
3058 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
3059 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
3062 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
3063 struct tss_segment_16 *tss)
3068 ctxt->_eip = tss->ip;
3069 ctxt->eflags = tss->flag | 2;
3070 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
3071 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
3072 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
3073 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
3074 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
3075 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
3076 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
3077 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
3080 * SDM says that segment selectors are loaded before segment
3083 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
3084 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
3085 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
3086 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
3087 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
3092 * Now load segment descriptors. If fault happens at this stage
3093 * it is handled in a context of new task
3095 ret = __load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR, cpl,
3096 X86_TRANSFER_TASK_SWITCH, NULL);
3097 if (ret != X86EMUL_CONTINUE)
3099 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
3100 X86_TRANSFER_TASK_SWITCH, NULL);
3101 if (ret != X86EMUL_CONTINUE)
3103 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
3104 X86_TRANSFER_TASK_SWITCH, NULL);
3105 if (ret != X86EMUL_CONTINUE)
3107 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
3108 X86_TRANSFER_TASK_SWITCH, NULL);
3109 if (ret != X86EMUL_CONTINUE)
3111 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
3112 X86_TRANSFER_TASK_SWITCH, NULL);
3113 if (ret != X86EMUL_CONTINUE)
3116 return X86EMUL_CONTINUE;
3119 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
3120 u16 tss_selector, u16 old_tss_sel,
3121 ulong old_tss_base, struct desc_struct *new_desc)
3123 struct tss_segment_16 tss_seg;
3125 u32 new_tss_base = get_desc_base(new_desc);
3127 ret = linear_read_system(ctxt, old_tss_base, &tss_seg, sizeof(tss_seg));
3128 if (ret != X86EMUL_CONTINUE)
3131 save_state_to_tss16(ctxt, &tss_seg);
3133 ret = linear_write_system(ctxt, old_tss_base, &tss_seg, sizeof(tss_seg));
3134 if (ret != X86EMUL_CONTINUE)
3137 ret = linear_read_system(ctxt, new_tss_base, &tss_seg, sizeof(tss_seg));
3138 if (ret != X86EMUL_CONTINUE)
3141 if (old_tss_sel != 0xffff) {
3142 tss_seg.prev_task_link = old_tss_sel;
3144 ret = linear_write_system(ctxt, new_tss_base,
3145 &tss_seg.prev_task_link,
3146 sizeof(tss_seg.prev_task_link));
3147 if (ret != X86EMUL_CONTINUE)
3151 return load_state_from_tss16(ctxt, &tss_seg);
3154 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
3155 struct tss_segment_32 *tss)
3157 /* CR3 and ldt selector are not saved intentionally */
3158 tss->eip = ctxt->_eip;
3159 tss->eflags = ctxt->eflags;
3160 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
3161 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
3162 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
3163 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
3164 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
3165 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
3166 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
3167 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
3169 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
3170 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
3171 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
3172 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
3173 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
3174 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
3177 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
3178 struct tss_segment_32 *tss)
3183 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
3184 return emulate_gp(ctxt, 0);
3185 ctxt->_eip = tss->eip;
3186 ctxt->eflags = tss->eflags | 2;
3188 /* General purpose registers */
3189 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
3190 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
3191 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
3192 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
3193 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
3194 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
3195 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
3196 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
3199 * SDM says that segment selectors are loaded before segment
3200 * descriptors. This is important because CPL checks will
3203 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
3204 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
3205 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
3206 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
3207 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
3208 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
3209 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
3212 * If we're switching between Protected Mode and VM86, we need to make
3213 * sure to update the mode before loading the segment descriptors so
3214 * that the selectors are interpreted correctly.
3216 if (ctxt->eflags & X86_EFLAGS_VM) {
3217 ctxt->mode = X86EMUL_MODE_VM86;
3220 ctxt->mode = X86EMUL_MODE_PROT32;
3225 * Now load segment descriptors. If fault happens at this stage
3226 * it is handled in a context of new task
3228 ret = __load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR,
3229 cpl, X86_TRANSFER_TASK_SWITCH, NULL);
3230 if (ret != X86EMUL_CONTINUE)
3232 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
3233 X86_TRANSFER_TASK_SWITCH, NULL);
3234 if (ret != X86EMUL_CONTINUE)
3236 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
3237 X86_TRANSFER_TASK_SWITCH, NULL);
3238 if (ret != X86EMUL_CONTINUE)
3240 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
3241 X86_TRANSFER_TASK_SWITCH, NULL);
3242 if (ret != X86EMUL_CONTINUE)
3244 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
3245 X86_TRANSFER_TASK_SWITCH, NULL);
3246 if (ret != X86EMUL_CONTINUE)
3248 ret = __load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS, cpl,
3249 X86_TRANSFER_TASK_SWITCH, NULL);
3250 if (ret != X86EMUL_CONTINUE)
3252 ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl,
3253 X86_TRANSFER_TASK_SWITCH, NULL);
3258 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
3259 u16 tss_selector, u16 old_tss_sel,
3260 ulong old_tss_base, struct desc_struct *new_desc)
3262 struct tss_segment_32 tss_seg;
3264 u32 new_tss_base = get_desc_base(new_desc);
3265 u32 eip_offset = offsetof(struct tss_segment_32, eip);
3266 u32 ldt_sel_offset = offsetof(struct tss_segment_32, ldt_selector);
3268 ret = linear_read_system(ctxt, old_tss_base, &tss_seg, sizeof(tss_seg));
3269 if (ret != X86EMUL_CONTINUE)
3272 save_state_to_tss32(ctxt, &tss_seg);
3274 /* Only GP registers and segment selectors are saved */
3275 ret = linear_write_system(ctxt, old_tss_base + eip_offset, &tss_seg.eip,
3276 ldt_sel_offset - eip_offset);
3277 if (ret != X86EMUL_CONTINUE)
3280 ret = linear_read_system(ctxt, new_tss_base, &tss_seg, sizeof(tss_seg));
3281 if (ret != X86EMUL_CONTINUE)
3284 if (old_tss_sel != 0xffff) {
3285 tss_seg.prev_task_link = old_tss_sel;
3287 ret = linear_write_system(ctxt, new_tss_base,
3288 &tss_seg.prev_task_link,
3289 sizeof(tss_seg.prev_task_link));
3290 if (ret != X86EMUL_CONTINUE)
3294 return load_state_from_tss32(ctxt, &tss_seg);
3297 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
3298 u16 tss_selector, int idt_index, int reason,
3299 bool has_error_code, u32 error_code)
3301 const struct x86_emulate_ops *ops = ctxt->ops;
3302 struct desc_struct curr_tss_desc, next_tss_desc;
3304 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
3305 ulong old_tss_base =
3306 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
3308 ulong desc_addr, dr7;
3310 /* FIXME: old_tss_base == ~0 ? */
3312 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
3313 if (ret != X86EMUL_CONTINUE)
3315 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
3316 if (ret != X86EMUL_CONTINUE)
3319 /* FIXME: check that next_tss_desc is tss */
3322 * Check privileges. The three cases are task switch caused by...
3324 * 1. jmp/call/int to task gate: Check against DPL of the task gate
3325 * 2. Exception/IRQ/iret: No check is performed
3326 * 3. jmp/call to TSS/task-gate: No check is performed since the
3327 * hardware checks it before exiting.
3329 if (reason == TASK_SWITCH_GATE) {
3330 if (idt_index != -1) {
3331 /* Software interrupts */
3332 struct desc_struct task_gate_desc;
3335 ret = read_interrupt_descriptor(ctxt, idt_index,
3337 if (ret != X86EMUL_CONTINUE)
3340 dpl = task_gate_desc.dpl;
3341 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
3342 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
3346 desc_limit = desc_limit_scaled(&next_tss_desc);
3347 if (!next_tss_desc.p ||
3348 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
3349 desc_limit < 0x2b)) {
3350 return emulate_ts(ctxt, tss_selector & 0xfffc);
3353 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
3354 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
3355 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
3358 if (reason == TASK_SWITCH_IRET)
3359 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
3361 /* set back link to prev task only if NT bit is set in eflags
3362 note that old_tss_sel is not used after this point */
3363 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
3364 old_tss_sel = 0xffff;
3366 if (next_tss_desc.type & 8)
3367 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
3368 old_tss_base, &next_tss_desc);
3370 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
3371 old_tss_base, &next_tss_desc);
3372 if (ret != X86EMUL_CONTINUE)
3375 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
3376 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
3378 if (reason != TASK_SWITCH_IRET) {
3379 next_tss_desc.type |= (1 << 1); /* set busy flag */
3380 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
3383 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
3384 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
3386 if (has_error_code) {
3387 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
3388 ctxt->lock_prefix = 0;
3389 ctxt->src.val = (unsigned long) error_code;
3390 ret = em_push(ctxt);
3393 ops->get_dr(ctxt, 7, &dr7);
3394 ops->set_dr(ctxt, 7, dr7 & ~(DR_LOCAL_ENABLE_MASK | DR_LOCAL_SLOWDOWN));
3399 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
3400 u16 tss_selector, int idt_index, int reason,
3401 bool has_error_code, u32 error_code)
3405 invalidate_registers(ctxt);
3406 ctxt->_eip = ctxt->eip;
3407 ctxt->dst.type = OP_NONE;
3409 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
3410 has_error_code, error_code);
3412 if (rc == X86EMUL_CONTINUE) {
3413 ctxt->eip = ctxt->_eip;
3414 writeback_registers(ctxt);
3417 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
3420 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
3423 int df = (ctxt->eflags & X86_EFLAGS_DF) ? -op->count : op->count;
3425 register_address_increment(ctxt, reg, df * op->bytes);
3426 op->addr.mem.ea = register_address(ctxt, reg);
3429 static int em_das(struct x86_emulate_ctxt *ctxt)
3432 bool af, cf, old_cf;
3434 cf = ctxt->eflags & X86_EFLAGS_CF;
3440 af = ctxt->eflags & X86_EFLAGS_AF;
3441 if ((al & 0x0f) > 9 || af) {
3443 cf = old_cf | (al >= 250);
3448 if (old_al > 0x99 || old_cf) {
3454 /* Set PF, ZF, SF */
3455 ctxt->src.type = OP_IMM;
3457 ctxt->src.bytes = 1;
3458 fastop(ctxt, em_or);
3459 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
3461 ctxt->eflags |= X86_EFLAGS_CF;
3463 ctxt->eflags |= X86_EFLAGS_AF;
3464 return X86EMUL_CONTINUE;
3467 static int em_aam(struct x86_emulate_ctxt *ctxt)
3471 if (ctxt->src.val == 0)
3472 return emulate_de(ctxt);
3474 al = ctxt->dst.val & 0xff;
3475 ah = al / ctxt->src.val;
3476 al %= ctxt->src.val;
3478 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);
3480 /* Set PF, ZF, SF */
3481 ctxt->src.type = OP_IMM;
3483 ctxt->src.bytes = 1;
3484 fastop(ctxt, em_or);
3486 return X86EMUL_CONTINUE;
3489 static int em_aad(struct x86_emulate_ctxt *ctxt)
3491 u8 al = ctxt->dst.val & 0xff;
3492 u8 ah = (ctxt->dst.val >> 8) & 0xff;
3494 al = (al + (ah * ctxt->src.val)) & 0xff;
3496 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
3498 /* Set PF, ZF, SF */
3499 ctxt->src.type = OP_IMM;
3501 ctxt->src.bytes = 1;
3502 fastop(ctxt, em_or);
3504 return X86EMUL_CONTINUE;
3507 static int em_call(struct x86_emulate_ctxt *ctxt)
3510 long rel = ctxt->src.val;
3512 ctxt->src.val = (unsigned long)ctxt->_eip;
3513 rc = jmp_rel(ctxt, rel);
3514 if (rc != X86EMUL_CONTINUE)
3516 return em_push(ctxt);
3519 static int em_call_far(struct x86_emulate_ctxt *ctxt)
3524 struct desc_struct old_desc, new_desc;
3525 const struct x86_emulate_ops *ops = ctxt->ops;
3526 int cpl = ctxt->ops->cpl(ctxt);
3527 enum x86emul_mode prev_mode = ctxt->mode;
3529 old_eip = ctxt->_eip;
3530 ops->get_segment(ctxt, &old_cs, &old_desc, NULL, VCPU_SREG_CS);
3532 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
3533 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
3534 X86_TRANSFER_CALL_JMP, &new_desc);
3535 if (rc != X86EMUL_CONTINUE)
3538 rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
3539 if (rc != X86EMUL_CONTINUE)
3542 ctxt->src.val = old_cs;
3544 if (rc != X86EMUL_CONTINUE)
3547 ctxt->src.val = old_eip;
3549 /* If we failed, we tainted the memory, but the very least we should
3551 if (rc != X86EMUL_CONTINUE) {
3552 pr_warn_once("faulting far call emulation tainted memory\n");
3557 ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS);
3558 ctxt->mode = prev_mode;
3563 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
3568 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
3569 if (rc != X86EMUL_CONTINUE)
3571 rc = assign_eip_near(ctxt, eip);
3572 if (rc != X86EMUL_CONTINUE)
3574 rsp_increment(ctxt, ctxt->src.val);
3575 return X86EMUL_CONTINUE;
3578 static int em_xchg(struct x86_emulate_ctxt *ctxt)
3580 /* Write back the register source. */
3581 ctxt->src.val = ctxt->dst.val;
3582 write_register_operand(&ctxt->src);
3584 /* Write back the memory destination with implicit LOCK prefix. */
3585 ctxt->dst.val = ctxt->src.orig_val;
3586 ctxt->lock_prefix = 1;
3587 return X86EMUL_CONTINUE;
3590 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
3592 ctxt->dst.val = ctxt->src2.val;
3593 return fastop(ctxt, em_imul);
3596 static int em_cwd(struct x86_emulate_ctxt *ctxt)
3598 ctxt->dst.type = OP_REG;
3599 ctxt->dst.bytes = ctxt->src.bytes;
3600 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
3601 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
3603 return X86EMUL_CONTINUE;
3606 static int em_rdpid(struct x86_emulate_ctxt *ctxt)
3610 if (ctxt->ops->get_msr(ctxt, MSR_TSC_AUX, &tsc_aux))
3611 return emulate_ud(ctxt);
3612 ctxt->dst.val = tsc_aux;
3613 return X86EMUL_CONTINUE;
3616 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
3620 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
3621 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
3622 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
3623 return X86EMUL_CONTINUE;
3626 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
3630 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
3631 return emulate_gp(ctxt, 0);
3632 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
3633 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
3634 return X86EMUL_CONTINUE;
3637 static int em_mov(struct x86_emulate_ctxt *ctxt)
3639 memcpy(ctxt->dst.valptr, ctxt->src.valptr, sizeof(ctxt->src.valptr));
3640 return X86EMUL_CONTINUE;
3643 static int em_movbe(struct x86_emulate_ctxt *ctxt)
3647 if (!ctxt->ops->guest_has_movbe(ctxt))
3648 return emulate_ud(ctxt);
3650 switch (ctxt->op_bytes) {
3653 * From MOVBE definition: "...When the operand size is 16 bits,
3654 * the upper word of the destination register remains unchanged
3657 * Both casting ->valptr and ->val to u16 breaks strict aliasing
3658 * rules so we have to do the operation almost per hand.
3660 tmp = (u16)ctxt->src.val;
3661 ctxt->dst.val &= ~0xffffUL;
3662 ctxt->dst.val |= (unsigned long)swab16(tmp);
3665 ctxt->dst.val = swab32((u32)ctxt->src.val);
3668 ctxt->dst.val = swab64(ctxt->src.val);
3673 return X86EMUL_CONTINUE;
3676 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3678 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3679 return emulate_gp(ctxt, 0);
3681 /* Disable writeback. */
3682 ctxt->dst.type = OP_NONE;
3683 return X86EMUL_CONTINUE;
3686 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3690 if (ctxt->mode == X86EMUL_MODE_PROT64)
3691 val = ctxt->src.val & ~0ULL;
3693 val = ctxt->src.val & ~0U;
3695 /* #UD condition is already handled. */
3696 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3697 return emulate_gp(ctxt, 0);
3699 /* Disable writeback. */
3700 ctxt->dst.type = OP_NONE;
3701 return X86EMUL_CONTINUE;
3704 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3706 u64 msr_index = reg_read(ctxt, VCPU_REGS_RCX);
3710 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3711 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3712 r = ctxt->ops->set_msr(ctxt, msr_index, msr_data);
3714 if (r == X86EMUL_IO_NEEDED)
3718 return emulate_gp(ctxt, 0);
3720 return r < 0 ? X86EMUL_UNHANDLEABLE : X86EMUL_CONTINUE;
3723 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3725 u64 msr_index = reg_read(ctxt, VCPU_REGS_RCX);
3729 r = ctxt->ops->get_msr(ctxt, msr_index, &msr_data);
3731 if (r == X86EMUL_IO_NEEDED)
3735 return emulate_gp(ctxt, 0);
3737 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3738 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3739 return X86EMUL_CONTINUE;
3742 static int em_store_sreg(struct x86_emulate_ctxt *ctxt, int segment)
3744 if (segment > VCPU_SREG_GS &&
3745 (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) &&
3746 ctxt->ops->cpl(ctxt) > 0)
3747 return emulate_gp(ctxt, 0);
3749 ctxt->dst.val = get_segment_selector(ctxt, segment);
3750 if (ctxt->dst.bytes == 4 && ctxt->dst.type == OP_MEM)
3751 ctxt->dst.bytes = 2;
3752 return X86EMUL_CONTINUE;
3755 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3757 if (ctxt->modrm_reg > VCPU_SREG_GS)
3758 return emulate_ud(ctxt);
3760 return em_store_sreg(ctxt, ctxt->modrm_reg);
3763 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3765 u16 sel = ctxt->src.val;
3767 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3768 return emulate_ud(ctxt);
3770 if (ctxt->modrm_reg == VCPU_SREG_SS)
3771 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3773 /* Disable writeback. */
3774 ctxt->dst.type = OP_NONE;
3775 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3778 static int em_sldt(struct x86_emulate_ctxt *ctxt)
3780 return em_store_sreg(ctxt, VCPU_SREG_LDTR);
3783 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3785 u16 sel = ctxt->src.val;
3787 /* Disable writeback. */
3788 ctxt->dst.type = OP_NONE;
3789 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3792 static int em_str(struct x86_emulate_ctxt *ctxt)
3794 return em_store_sreg(ctxt, VCPU_SREG_TR);
3797 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3799 u16 sel = ctxt->src.val;
3801 /* Disable writeback. */
3802 ctxt->dst.type = OP_NONE;
3803 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3806 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3811 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3812 if (rc == X86EMUL_CONTINUE)
3813 ctxt->ops->invlpg(ctxt, linear);
3814 /* Disable writeback. */
3815 ctxt->dst.type = OP_NONE;
3816 return X86EMUL_CONTINUE;
3819 static int em_clts(struct x86_emulate_ctxt *ctxt)
3823 cr0 = ctxt->ops->get_cr(ctxt, 0);
3825 ctxt->ops->set_cr(ctxt, 0, cr0);
3826 return X86EMUL_CONTINUE;
3829 static int em_hypercall(struct x86_emulate_ctxt *ctxt)
3831 int rc = ctxt->ops->fix_hypercall(ctxt);
3833 if (rc != X86EMUL_CONTINUE)
3836 /* Let the processor re-execute the fixed hypercall */
3837 ctxt->_eip = ctxt->eip;
3838 /* Disable writeback. */
3839 ctxt->dst.type = OP_NONE;
3840 return X86EMUL_CONTINUE;
3843 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3844 void (*get)(struct x86_emulate_ctxt *ctxt,
3845 struct desc_ptr *ptr))
3847 struct desc_ptr desc_ptr;
3849 if ((ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) &&
3850 ctxt->ops->cpl(ctxt) > 0)
3851 return emulate_gp(ctxt, 0);
3853 if (ctxt->mode == X86EMUL_MODE_PROT64)
3855 get(ctxt, &desc_ptr);
3856 if (ctxt->op_bytes == 2) {
3858 desc_ptr.address &= 0x00ffffff;
3860 /* Disable writeback. */
3861 ctxt->dst.type = OP_NONE;
3862 return segmented_write_std(ctxt, ctxt->dst.addr.mem,
3863 &desc_ptr, 2 + ctxt->op_bytes);
3866 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3868 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3871 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3873 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3876 static int em_lgdt_lidt(struct x86_emulate_ctxt *ctxt, bool lgdt)
3878 struct desc_ptr desc_ptr;
3881 if (ctxt->mode == X86EMUL_MODE_PROT64)
3883 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3884 &desc_ptr.size, &desc_ptr.address,
3886 if (rc != X86EMUL_CONTINUE)
3888 if (ctxt->mode == X86EMUL_MODE_PROT64 &&
3889 emul_is_noncanonical_address(desc_ptr.address, ctxt))
3890 return emulate_gp(ctxt, 0);
3892 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3894 ctxt->ops->set_idt(ctxt, &desc_ptr);
3895 /* Disable writeback. */
3896 ctxt->dst.type = OP_NONE;
3897 return X86EMUL_CONTINUE;
3900 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3902 return em_lgdt_lidt(ctxt, true);
3905 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3907 return em_lgdt_lidt(ctxt, false);
3910 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3912 if ((ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) &&
3913 ctxt->ops->cpl(ctxt) > 0)
3914 return emulate_gp(ctxt, 0);
3916 if (ctxt->dst.type == OP_MEM)
3917 ctxt->dst.bytes = 2;
3918 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3919 return X86EMUL_CONTINUE;
3922 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3924 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3925 | (ctxt->src.val & 0x0f));
3926 ctxt->dst.type = OP_NONE;
3927 return X86EMUL_CONTINUE;
3930 static int em_loop(struct x86_emulate_ctxt *ctxt)
3932 int rc = X86EMUL_CONTINUE;
3934 register_address_increment(ctxt, VCPU_REGS_RCX, -1);
3935 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3936 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3937 rc = jmp_rel(ctxt, ctxt->src.val);
3942 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3944 int rc = X86EMUL_CONTINUE;
3946 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3947 rc = jmp_rel(ctxt, ctxt->src.val);
3952 static int em_in(struct x86_emulate_ctxt *ctxt)
3954 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3956 return X86EMUL_IO_NEEDED;
3958 return X86EMUL_CONTINUE;
3961 static int em_out(struct x86_emulate_ctxt *ctxt)
3963 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3965 /* Disable writeback. */
3966 ctxt->dst.type = OP_NONE;
3967 return X86EMUL_CONTINUE;
3970 static int em_cli(struct x86_emulate_ctxt *ctxt)
3972 if (emulator_bad_iopl(ctxt))
3973 return emulate_gp(ctxt, 0);
3975 ctxt->eflags &= ~X86_EFLAGS_IF;
3976 return X86EMUL_CONTINUE;
3979 static int em_sti(struct x86_emulate_ctxt *ctxt)
3981 if (emulator_bad_iopl(ctxt))
3982 return emulate_gp(ctxt, 0);
3984 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3985 ctxt->eflags |= X86_EFLAGS_IF;
3986 return X86EMUL_CONTINUE;
3989 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3991 u32 eax, ebx, ecx, edx;
3994 ctxt->ops->get_msr(ctxt, MSR_MISC_FEATURES_ENABLES, &msr);
3995 if (msr & MSR_MISC_FEATURES_ENABLES_CPUID_FAULT &&
3996 ctxt->ops->cpl(ctxt)) {
3997 return emulate_gp(ctxt, 0);
4000 eax = reg_read(ctxt, VCPU_REGS_RAX);
4001 ecx = reg_read(ctxt, VCPU_REGS_RCX);
4002 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
4003 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
4004 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
4005 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
4006 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
4007 return X86EMUL_CONTINUE;
4010 static int em_sahf(struct x86_emulate_ctxt *ctxt)
4014 flags = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF |
4016 flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8;
4018 ctxt->eflags &= ~0xffUL;
4019 ctxt->eflags |= flags | X86_EFLAGS_FIXED;
4020 return X86EMUL_CONTINUE;
4023 static int em_lahf(struct x86_emulate_ctxt *ctxt)
4025 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
4026 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
4027 return X86EMUL_CONTINUE;
4030 static int em_bswap(struct x86_emulate_ctxt *ctxt)
4032 switch (ctxt->op_bytes) {
4033 #ifdef CONFIG_X86_64
4035 asm("bswap %0" : "+r"(ctxt->dst.val));
4039 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
4042 return X86EMUL_CONTINUE;
4045 static int em_clflush(struct x86_emulate_ctxt *ctxt)
4047 /* emulating clflush regardless of cpuid */
4048 return X86EMUL_CONTINUE;
4051 static int em_clflushopt(struct x86_emulate_ctxt *ctxt)
4053 /* emulating clflushopt regardless of cpuid */
4054 return X86EMUL_CONTINUE;
4057 static int em_movsxd(struct x86_emulate_ctxt *ctxt)
4059 ctxt->dst.val = (s32) ctxt->src.val;
4060 return X86EMUL_CONTINUE;
4063 static int check_fxsr(struct x86_emulate_ctxt *ctxt)
4065 if (!ctxt->ops->guest_has_fxsr(ctxt))
4066 return emulate_ud(ctxt);
4068 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
4069 return emulate_nm(ctxt);
4072 * Don't emulate a case that should never be hit, instead of working
4073 * around a lack of fxsave64/fxrstor64 on old compilers.
4075 if (ctxt->mode >= X86EMUL_MODE_PROT64)
4076 return X86EMUL_UNHANDLEABLE;
4078 return X86EMUL_CONTINUE;
4082 * Hardware doesn't save and restore XMM 0-7 without CR4.OSFXSR, but does save
4083 * and restore MXCSR.
4085 static size_t __fxstate_size(int nregs)
4087 return offsetof(struct fxregs_state, xmm_space[0]) + nregs * 16;
4090 static inline size_t fxstate_size(struct x86_emulate_ctxt *ctxt)
4093 if (ctxt->mode == X86EMUL_MODE_PROT64)
4094 return __fxstate_size(16);
4096 cr4_osfxsr = ctxt->ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR;
4097 return __fxstate_size(cr4_osfxsr ? 8 : 0);
4101 * FXSAVE and FXRSTOR have 4 different formats depending on execution mode,
4104 * - like (1), but FIP and FDP (foo) are only 16 bit. At least Intel CPUs
4105 * preserve whole 32 bit values, though, so (1) and (2) are the same wrt.
4107 * 3) 64-bit mode with REX.W prefix
4108 * - like (2), but XMM 8-15 are being saved and restored
4109 * 4) 64-bit mode without REX.W prefix
4110 * - like (3), but FIP and FDP are 64 bit
4112 * Emulation uses (3) for (1) and (2) and preserves XMM 8-15 to reach the
4113 * desired result. (4) is not emulated.
4115 * Note: Guest and host CPUID.(EAX=07H,ECX=0H):EBX[bit 13] (deprecate FPU CS
4116 * and FPU DS) should match.
4118 static int em_fxsave(struct x86_emulate_ctxt *ctxt)
4120 struct fxregs_state fx_state;
4123 rc = check_fxsr(ctxt);
4124 if (rc != X86EMUL_CONTINUE)
4129 rc = asm_safe("fxsave %[fx]", , [fx] "+m"(fx_state));
4133 if (rc != X86EMUL_CONTINUE)
4136 return segmented_write_std(ctxt, ctxt->memop.addr.mem, &fx_state,
4137 fxstate_size(ctxt));
4141 * FXRSTOR might restore XMM registers not provided by the guest. Fill
4142 * in the host registers (via FXSAVE) instead, so they won't be modified.
4143 * (preemption has to stay disabled until FXRSTOR).
4145 * Use noinline to keep the stack for other functions called by callers small.
4147 static noinline int fxregs_fixup(struct fxregs_state *fx_state,
4148 const size_t used_size)
4150 struct fxregs_state fx_tmp;
4153 rc = asm_safe("fxsave %[fx]", , [fx] "+m"(fx_tmp));
4154 memcpy((void *)fx_state + used_size, (void *)&fx_tmp + used_size,
4155 __fxstate_size(16) - used_size);
4160 static int em_fxrstor(struct x86_emulate_ctxt *ctxt)
4162 struct fxregs_state fx_state;
4166 rc = check_fxsr(ctxt);
4167 if (rc != X86EMUL_CONTINUE)
4170 size = fxstate_size(ctxt);
4171 rc = segmented_read_std(ctxt, ctxt->memop.addr.mem, &fx_state, size);
4172 if (rc != X86EMUL_CONTINUE)
4177 if (size < __fxstate_size(16)) {
4178 rc = fxregs_fixup(&fx_state, size);
4179 if (rc != X86EMUL_CONTINUE)
4183 if (fx_state.mxcsr >> 16) {
4184 rc = emulate_gp(ctxt, 0);
4188 if (rc == X86EMUL_CONTINUE)
4189 rc = asm_safe("fxrstor %[fx]", : [fx] "m"(fx_state));
4197 static int em_xsetbv(struct x86_emulate_ctxt *ctxt)
4201 eax = reg_read(ctxt, VCPU_REGS_RAX);
4202 edx = reg_read(ctxt, VCPU_REGS_RDX);
4203 ecx = reg_read(ctxt, VCPU_REGS_RCX);
4205 if (ctxt->ops->set_xcr(ctxt, ecx, ((u64)edx << 32) | eax))
4206 return emulate_gp(ctxt, 0);
4208 return X86EMUL_CONTINUE;
4211 static bool valid_cr(int nr)
4223 static int check_cr_access(struct x86_emulate_ctxt *ctxt)
4225 if (!valid_cr(ctxt->modrm_reg))
4226 return emulate_ud(ctxt);
4228 return X86EMUL_CONTINUE;
4231 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
4235 ctxt->ops->get_dr(ctxt, 7, &dr7);
4237 /* Check if DR7.Global_Enable is set */
4238 return dr7 & (1 << 13);
4241 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
4243 int dr = ctxt->modrm_reg;
4247 return emulate_ud(ctxt);
4249 cr4 = ctxt->ops->get_cr(ctxt, 4);
4250 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
4251 return emulate_ud(ctxt);
4253 if (check_dr7_gd(ctxt)) {
4256 ctxt->ops->get_dr(ctxt, 6, &dr6);
4257 dr6 &= ~DR_TRAP_BITS;
4258 dr6 |= DR6_BD | DR6_ACTIVE_LOW;
4259 ctxt->ops->set_dr(ctxt, 6, dr6);
4260 return emulate_db(ctxt);
4263 return X86EMUL_CONTINUE;
4266 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
4268 u64 new_val = ctxt->src.val64;
4269 int dr = ctxt->modrm_reg;
4271 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
4272 return emulate_gp(ctxt, 0);
4274 return check_dr_read(ctxt);
4277 static int check_svme(struct x86_emulate_ctxt *ctxt)
4281 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4283 if (!(efer & EFER_SVME))
4284 return emulate_ud(ctxt);
4286 return X86EMUL_CONTINUE;
4289 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
4291 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
4293 /* Valid physical address? */
4294 if (rax & 0xffff000000000000ULL)
4295 return emulate_gp(ctxt, 0);
4297 return check_svme(ctxt);
4300 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
4302 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
4304 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
4305 return emulate_ud(ctxt);
4307 return X86EMUL_CONTINUE;
4310 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
4312 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
4313 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
4316 * VMware allows access to these Pseduo-PMCs even when read via RDPMC
4317 * in Ring3 when CR4.PCE=0.
4319 if (enable_vmware_backdoor && is_vmware_backdoor_pmc(rcx))
4320 return X86EMUL_CONTINUE;
4322 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
4323 ctxt->ops->check_pmc(ctxt, rcx))
4324 return emulate_gp(ctxt, 0);
4326 return X86EMUL_CONTINUE;
4329 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
4331 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
4332 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
4333 return emulate_gp(ctxt, 0);
4335 return X86EMUL_CONTINUE;
4338 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
4340 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
4341 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
4342 return emulate_gp(ctxt, 0);
4344 return X86EMUL_CONTINUE;
4347 #define D(_y) { .flags = (_y) }
4348 #define DI(_y, _i) { .flags = (_y)|Intercept, .intercept = x86_intercept_##_i }
4349 #define DIP(_y, _i, _p) { .flags = (_y)|Intercept|CheckPerm, \
4350 .intercept = x86_intercept_##_i, .check_perm = (_p) }
4351 #define N D(NotImpl)
4352 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
4353 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
4354 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
4355 #define ID(_f, _i) { .flags = ((_f) | InstrDual | ModRM), .u.idual = (_i) }
4356 #define MD(_f, _m) { .flags = ((_f) | ModeDual), .u.mdual = (_m) }
4357 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
4358 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
4359 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
4360 #define II(_f, _e, _i) \
4361 { .flags = (_f)|Intercept, .u.execute = (_e), .intercept = x86_intercept_##_i }
4362 #define IIP(_f, _e, _i, _p) \
4363 { .flags = (_f)|Intercept|CheckPerm, .u.execute = (_e), \
4364 .intercept = x86_intercept_##_i, .check_perm = (_p) }
4365 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
4367 #define D2bv(_f) D((_f) | ByteOp), D(_f)
4368 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
4369 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
4370 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
4371 #define I2bvIP(_f, _e, _i, _p) \
4372 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
4374 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
4375 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
4376 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
4378 static const struct opcode group7_rm0[] = {
4380 I(SrcNone | Priv | EmulateOnUD, em_hypercall),
4384 static const struct opcode group7_rm1[] = {
4385 DI(SrcNone | Priv, monitor),
4386 DI(SrcNone | Priv, mwait),
4390 static const struct opcode group7_rm2[] = {
4392 II(ImplicitOps | Priv, em_xsetbv, xsetbv),
4396 static const struct opcode group7_rm3[] = {
4397 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
4398 II(SrcNone | Prot | EmulateOnUD, em_hypercall, vmmcall),
4399 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
4400 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
4401 DIP(SrcNone | Prot | Priv, stgi, check_svme),
4402 DIP(SrcNone | Prot | Priv, clgi, check_svme),
4403 DIP(SrcNone | Prot | Priv, skinit, check_svme),
4404 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
4407 static const struct opcode group7_rm7[] = {
4409 DIP(SrcNone, rdtscp, check_rdtsc),
4413 static const struct opcode group1[] = {
4415 F(Lock | PageTable, em_or),
4418 F(Lock | PageTable, em_and),
4424 static const struct opcode group1A[] = {
4425 I(DstMem | SrcNone | Mov | Stack | IncSP | TwoMemOp, em_pop), N, N, N, N, N, N, N,
4428 static const struct opcode group2[] = {
4429 F(DstMem | ModRM, em_rol),
4430 F(DstMem | ModRM, em_ror),
4431 F(DstMem | ModRM, em_rcl),
4432 F(DstMem | ModRM, em_rcr),
4433 F(DstMem | ModRM, em_shl),
4434 F(DstMem | ModRM, em_shr),
4435 F(DstMem | ModRM, em_shl),
4436 F(DstMem | ModRM, em_sar),
4439 static const struct opcode group3[] = {
4440 F(DstMem | SrcImm | NoWrite, em_test),
4441 F(DstMem | SrcImm | NoWrite, em_test),
4442 F(DstMem | SrcNone | Lock, em_not),
4443 F(DstMem | SrcNone | Lock, em_neg),
4444 F(DstXacc | Src2Mem, em_mul_ex),
4445 F(DstXacc | Src2Mem, em_imul_ex),
4446 F(DstXacc | Src2Mem, em_div_ex),
4447 F(DstXacc | Src2Mem, em_idiv_ex),
4450 static const struct opcode group4[] = {
4451 F(ByteOp | DstMem | SrcNone | Lock, em_inc),
4452 F(ByteOp | DstMem | SrcNone | Lock, em_dec),
4456 static const struct opcode group5[] = {
4457 F(DstMem | SrcNone | Lock, em_inc),
4458 F(DstMem | SrcNone | Lock, em_dec),
4459 I(SrcMem | NearBranch, em_call_near_abs),
4460 I(SrcMemFAddr | ImplicitOps, em_call_far),
4461 I(SrcMem | NearBranch, em_jmp_abs),
4462 I(SrcMemFAddr | ImplicitOps, em_jmp_far),
4463 I(SrcMem | Stack | TwoMemOp, em_push), D(Undefined),
4466 static const struct opcode group6[] = {
4467 II(Prot | DstMem, em_sldt, sldt),
4468 II(Prot | DstMem, em_str, str),
4469 II(Prot | Priv | SrcMem16, em_lldt, lldt),
4470 II(Prot | Priv | SrcMem16, em_ltr, ltr),
4474 static const struct group_dual group7 = { {
4475 II(Mov | DstMem, em_sgdt, sgdt),
4476 II(Mov | DstMem, em_sidt, sidt),
4477 II(SrcMem | Priv, em_lgdt, lgdt),
4478 II(SrcMem | Priv, em_lidt, lidt),
4479 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
4480 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
4481 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
4487 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
4488 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
4492 static const struct opcode group8[] = {
4494 F(DstMem | SrcImmByte | NoWrite, em_bt),
4495 F(DstMem | SrcImmByte | Lock | PageTable, em_bts),
4496 F(DstMem | SrcImmByte | Lock, em_btr),
4497 F(DstMem | SrcImmByte | Lock | PageTable, em_btc),
4501 * The "memory" destination is actually always a register, since we come
4502 * from the register case of group9.
4504 static const struct gprefix pfx_0f_c7_7 = {
4505 N, N, N, II(DstMem | ModRM | Op3264 | EmulateOnUD, em_rdpid, rdtscp),
4509 static const struct group_dual group9 = { {
4510 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
4512 N, N, N, N, N, N, N,
4513 GP(0, &pfx_0f_c7_7),
4516 static const struct opcode group11[] = {
4517 I(DstMem | SrcImm | Mov | PageTable, em_mov),
4521 static const struct gprefix pfx_0f_ae_7 = {
4522 I(SrcMem | ByteOp, em_clflush), I(SrcMem | ByteOp, em_clflushopt), N, N,
4525 static const struct group_dual group15 = { {
4526 I(ModRM | Aligned16, em_fxsave),
4527 I(ModRM | Aligned16, em_fxrstor),
4528 N, N, N, N, N, GP(0, &pfx_0f_ae_7),
4530 N, N, N, N, N, N, N, N,
4533 static const struct gprefix pfx_0f_6f_0f_7f = {
4534 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
4537 static const struct instr_dual instr_dual_0f_2b = {
4541 static const struct gprefix pfx_0f_2b = {
4542 ID(0, &instr_dual_0f_2b), ID(0, &instr_dual_0f_2b), N, N,
4545 static const struct gprefix pfx_0f_10_0f_11 = {
4546 I(Unaligned, em_mov), I(Unaligned, em_mov), N, N,
4549 static const struct gprefix pfx_0f_28_0f_29 = {
4550 I(Aligned, em_mov), I(Aligned, em_mov), N, N,
4553 static const struct gprefix pfx_0f_e7 = {
4554 N, I(Sse, em_mov), N, N,
4557 static const struct escape escape_d9 = { {
4558 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstcw),
4561 N, N, N, N, N, N, N, N,
4563 N, N, N, N, N, N, N, N,
4565 N, N, N, N, N, N, N, N,
4567 N, N, N, N, N, N, N, N,
4569 N, N, N, N, N, N, N, N,
4571 N, N, N, N, N, N, N, N,
4573 N, N, N, N, N, N, N, N,
4575 N, N, N, N, N, N, N, N,
4578 static const struct escape escape_db = { {
4579 N, N, N, N, N, N, N, N,
4582 N, N, N, N, N, N, N, N,
4584 N, N, N, N, N, N, N, N,
4586 N, N, N, N, N, N, N, N,
4588 N, N, N, N, N, N, N, N,
4590 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
4592 N, N, N, N, N, N, N, N,
4594 N, N, N, N, N, N, N, N,
4596 N, N, N, N, N, N, N, N,
4599 static const struct escape escape_dd = { {
4600 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstsw),
4603 N, N, N, N, N, N, N, N,
4605 N, N, N, N, N, N, N, N,
4607 N, N, N, N, N, N, N, N,
4609 N, N, N, N, N, N, N, N,
4611 N, N, N, N, N, N, N, N,
4613 N, N, N, N, N, N, N, N,
4615 N, N, N, N, N, N, N, N,
4617 N, N, N, N, N, N, N, N,
4620 static const struct instr_dual instr_dual_0f_c3 = {
4621 I(DstMem | SrcReg | ModRM | No16 | Mov, em_mov), N
4624 static const struct mode_dual mode_dual_63 = {
4625 N, I(DstReg | SrcMem32 | ModRM | Mov, em_movsxd)
4628 static const struct opcode opcode_table[256] = {
4630 F6ALU(Lock, em_add),
4631 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
4632 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
4634 F6ALU(Lock | PageTable, em_or),
4635 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
4638 F6ALU(Lock, em_adc),
4639 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
4640 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
4642 F6ALU(Lock, em_sbb),
4643 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
4644 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
4646 F6ALU(Lock | PageTable, em_and), N, N,
4648 F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
4650 F6ALU(Lock, em_xor), N, N,
4652 F6ALU(NoWrite, em_cmp), N, N,
4654 X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
4656 X8(I(SrcReg | Stack, em_push)),
4658 X8(I(DstReg | Stack, em_pop)),
4660 I(ImplicitOps | Stack | No64, em_pusha),
4661 I(ImplicitOps | Stack | No64, em_popa),
4662 N, MD(ModRM, &mode_dual_63),
4665 I(SrcImm | Mov | Stack, em_push),
4666 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
4667 I(SrcImmByte | Mov | Stack, em_push),
4668 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
4669 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
4670 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
4672 X16(D(SrcImmByte | NearBranch)),
4674 G(ByteOp | DstMem | SrcImm, group1),
4675 G(DstMem | SrcImm, group1),
4676 G(ByteOp | DstMem | SrcImm | No64, group1),
4677 G(DstMem | SrcImmByte, group1),
4678 F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
4679 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
4681 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
4682 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
4683 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
4684 D(ModRM | SrcMem | NoAccess | DstReg),
4685 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
4688 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
4690 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
4691 I(SrcImmFAddr | No64, em_call_far), N,
4692 II(ImplicitOps | Stack, em_pushf, pushf),
4693 II(ImplicitOps | Stack, em_popf, popf),
4694 I(ImplicitOps, em_sahf), I(ImplicitOps, em_lahf),
4696 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
4697 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
4698 I2bv(SrcSI | DstDI | Mov | String | TwoMemOp, em_mov),
4699 F2bv(SrcSI | DstDI | String | NoWrite | TwoMemOp, em_cmp_r),
4701 F2bv(DstAcc | SrcImm | NoWrite, em_test),
4702 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
4703 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
4704 F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp_r),
4706 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
4708 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
4710 G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
4711 I(ImplicitOps | NearBranch | SrcImmU16, em_ret_near_imm),
4712 I(ImplicitOps | NearBranch, em_ret),
4713 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
4714 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
4715 G(ByteOp, group11), G(0, group11),
4717 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
4718 I(ImplicitOps | SrcImmU16, em_ret_far_imm),
4719 I(ImplicitOps, em_ret_far),
4720 D(ImplicitOps), DI(SrcImmByte, intn),
4721 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
4723 G(Src2One | ByteOp, group2), G(Src2One, group2),
4724 G(Src2CL | ByteOp, group2), G(Src2CL, group2),
4725 I(DstAcc | SrcImmUByte | No64, em_aam),
4726 I(DstAcc | SrcImmUByte | No64, em_aad),
4727 F(DstAcc | ByteOp | No64, em_salc),
4728 I(DstAcc | SrcXLat | ByteOp, em_mov),
4730 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
4732 X3(I(SrcImmByte | NearBranch, em_loop)),
4733 I(SrcImmByte | NearBranch, em_jcxz),
4734 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
4735 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
4737 I(SrcImm | NearBranch, em_call), D(SrcImm | ImplicitOps | NearBranch),
4738 I(SrcImmFAddr | No64, em_jmp_far),
4739 D(SrcImmByte | ImplicitOps | NearBranch),
4740 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
4741 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
4743 N, DI(ImplicitOps, icebp), N, N,
4744 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
4745 G(ByteOp, group3), G(0, group3),
4747 D(ImplicitOps), D(ImplicitOps),
4748 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
4749 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
4752 static const struct opcode twobyte_table[256] = {
4754 G(0, group6), GD(0, &group7), N, N,
4755 N, I(ImplicitOps | EmulateOnUD, em_syscall),
4756 II(ImplicitOps | Priv, em_clts, clts), N,
4757 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
4758 N, D(ImplicitOps | ModRM | SrcMem | NoAccess), N, N,
4760 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_10_0f_11),
4761 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_10_0f_11),
4763 D(ImplicitOps | ModRM | SrcMem | NoAccess), /* 4 * prefetch + 4 * reserved NOP */
4764 D(ImplicitOps | ModRM | SrcMem | NoAccess), N, N,
4765 D(ImplicitOps | ModRM | SrcMem | NoAccess), /* 8 * reserved NOP */
4766 D(ImplicitOps | ModRM | SrcMem | NoAccess), /* 8 * reserved NOP */
4767 D(ImplicitOps | ModRM | SrcMem | NoAccess), /* 8 * reserved NOP */
4768 D(ImplicitOps | ModRM | SrcMem | NoAccess), /* NOP + 7 * reserved NOP */
4770 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, cr_read, check_cr_access),
4771 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, dr_read, check_dr_read),
4772 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_cr_write, cr_write,
4774 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_dr_write, dr_write,
4777 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_28_0f_29),
4778 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_28_0f_29),
4779 N, GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_2b),
4782 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
4783 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
4784 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
4785 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
4786 I(ImplicitOps | EmulateOnUD, em_sysenter),
4787 I(ImplicitOps | Priv | EmulateOnUD, em_sysexit),
4789 N, N, N, N, N, N, N, N,
4791 X16(D(DstReg | SrcMem | ModRM)),
4793 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4798 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
4803 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
4805 X16(D(SrcImm | NearBranch)),
4807 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
4809 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
4810 II(ImplicitOps, em_cpuid, cpuid),
4811 F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
4812 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
4813 F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
4815 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
4816 II(EmulateOnUD | ImplicitOps, em_rsm, rsm),
4817 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
4818 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
4819 F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
4820 GD(0, &group15), F(DstReg | SrcMem | ModRM, em_imul),
4822 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable | SrcWrite, em_cmpxchg),
4823 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
4824 F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
4825 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
4826 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
4827 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4831 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
4832 I(DstReg | SrcMem | ModRM, em_bsf_c),
4833 I(DstReg | SrcMem | ModRM, em_bsr_c),
4834 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4836 F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
4837 N, ID(0, &instr_dual_0f_c3),
4838 N, N, N, GD(0, &group9),
4840 X8(I(DstReg, em_bswap)),
4842 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4844 N, N, N, N, N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_e7),
4845 N, N, N, N, N, N, N, N,
4847 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
4850 static const struct instr_dual instr_dual_0f_38_f0 = {
4851 I(DstReg | SrcMem | Mov, em_movbe), N
4854 static const struct instr_dual instr_dual_0f_38_f1 = {
4855 I(DstMem | SrcReg | Mov, em_movbe), N
4858 static const struct gprefix three_byte_0f_38_f0 = {
4859 ID(0, &instr_dual_0f_38_f0), N, N, N
4862 static const struct gprefix three_byte_0f_38_f1 = {
4863 ID(0, &instr_dual_0f_38_f1), N, N, N
4867 * Insns below are selected by the prefix which indexed by the third opcode
4870 static const struct opcode opcode_map_0f_38[256] = {
4872 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4874 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4876 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f0),
4877 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f1),
4898 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4902 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4908 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
4909 unsigned size, bool sign_extension)
4911 int rc = X86EMUL_CONTINUE;
4915 op->addr.mem.ea = ctxt->_eip;
4916 /* NB. Immediates are sign-extended as necessary. */
4917 switch (op->bytes) {
4919 op->val = insn_fetch(s8, ctxt);
4922 op->val = insn_fetch(s16, ctxt);
4925 op->val = insn_fetch(s32, ctxt);
4928 op->val = insn_fetch(s64, ctxt);
4931 if (!sign_extension) {
4932 switch (op->bytes) {
4940 op->val &= 0xffffffff;
4948 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
4951 int rc = X86EMUL_CONTINUE;
4955 decode_register_operand(ctxt, op);
4958 rc = decode_imm(ctxt, op, 1, false);
4961 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4965 if (ctxt->d & BitOp)
4966 fetch_bit_operand(ctxt);
4967 op->orig_val = op->val;
4970 ctxt->memop.bytes = (ctxt->op_bytes == 8) ? 16 : 8;
4974 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4975 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4976 fetch_register_operand(op);
4977 op->orig_val = op->val;
4981 op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;
4982 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4983 fetch_register_operand(op);
4984 op->orig_val = op->val;
4987 if (ctxt->d & ByteOp) {
4992 op->bytes = ctxt->op_bytes;
4993 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4994 fetch_register_operand(op);
4995 op->orig_val = op->val;
4999 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
5001 register_address(ctxt, VCPU_REGS_RDI);
5002 op->addr.mem.seg = VCPU_SREG_ES;
5009 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
5010 fetch_register_operand(op);
5015 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
5018 rc = decode_imm(ctxt, op, 1, true);
5026 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
5029 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
5032 ctxt->memop.bytes = 1;
5033 if (ctxt->memop.type == OP_REG) {
5034 ctxt->memop.addr.reg = decode_register(ctxt,
5035 ctxt->modrm_rm, true);
5036 fetch_register_operand(&ctxt->memop);
5040 ctxt->memop.bytes = 2;
5043 ctxt->memop.bytes = 4;
5046 rc = decode_imm(ctxt, op, 2, false);
5049 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
5053 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
5055 register_address(ctxt, VCPU_REGS_RSI);
5056 op->addr.mem.seg = ctxt->seg_override;
5062 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
5065 reg_read(ctxt, VCPU_REGS_RBX) +
5066 (reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
5067 op->addr.mem.seg = ctxt->seg_override;
5072 op->addr.mem.ea = ctxt->_eip;
5073 op->bytes = ctxt->op_bytes + 2;
5074 insn_fetch_arr(op->valptr, op->bytes, ctxt);
5077 ctxt->memop.bytes = ctxt->op_bytes + 2;
5081 op->val = VCPU_SREG_ES;
5085 op->val = VCPU_SREG_CS;
5089 op->val = VCPU_SREG_SS;
5093 op->val = VCPU_SREG_DS;
5097 op->val = VCPU_SREG_FS;
5101 op->val = VCPU_SREG_GS;
5104 /* Special instructions do their own operand decoding. */
5106 op->type = OP_NONE; /* Disable writeback. */
5114 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
5116 int rc = X86EMUL_CONTINUE;
5117 int mode = ctxt->mode;
5118 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
5119 bool op_prefix = false;
5120 bool has_seg_override = false;
5121 struct opcode opcode;
5123 struct desc_struct desc;
5125 ctxt->memop.type = OP_NONE;
5126 ctxt->memopp = NULL;
5127 ctxt->_eip = ctxt->eip;
5128 ctxt->fetch.ptr = ctxt->fetch.data;
5129 ctxt->fetch.end = ctxt->fetch.data + insn_len;
5130 ctxt->opcode_len = 1;
5131 ctxt->intercept = x86_intercept_none;
5133 memcpy(ctxt->fetch.data, insn, insn_len);
5135 rc = __do_insn_fetch_bytes(ctxt, 1);
5136 if (rc != X86EMUL_CONTINUE)
5141 case X86EMUL_MODE_REAL:
5142 case X86EMUL_MODE_VM86:
5143 def_op_bytes = def_ad_bytes = 2;
5144 ctxt->ops->get_segment(ctxt, &dummy, &desc, NULL, VCPU_SREG_CS);
5146 def_op_bytes = def_ad_bytes = 4;
5148 case X86EMUL_MODE_PROT16:
5149 def_op_bytes = def_ad_bytes = 2;
5151 case X86EMUL_MODE_PROT32:
5152 def_op_bytes = def_ad_bytes = 4;
5154 #ifdef CONFIG_X86_64
5155 case X86EMUL_MODE_PROT64:
5161 return EMULATION_FAILED;
5164 ctxt->op_bytes = def_op_bytes;
5165 ctxt->ad_bytes = def_ad_bytes;
5167 /* Legacy prefixes. */
5169 switch (ctxt->b = insn_fetch(u8, ctxt)) {
5170 case 0x66: /* operand-size override */
5172 /* switch between 2/4 bytes */
5173 ctxt->op_bytes = def_op_bytes ^ 6;
5175 case 0x67: /* address-size override */
5176 if (mode == X86EMUL_MODE_PROT64)
5177 /* switch between 4/8 bytes */
5178 ctxt->ad_bytes = def_ad_bytes ^ 12;
5180 /* switch between 2/4 bytes */
5181 ctxt->ad_bytes = def_ad_bytes ^ 6;
5183 case 0x26: /* ES override */
5184 has_seg_override = true;
5185 ctxt->seg_override = VCPU_SREG_ES;
5187 case 0x2e: /* CS override */
5188 has_seg_override = true;
5189 ctxt->seg_override = VCPU_SREG_CS;
5191 case 0x36: /* SS override */
5192 has_seg_override = true;
5193 ctxt->seg_override = VCPU_SREG_SS;
5195 case 0x3e: /* DS override */
5196 has_seg_override = true;
5197 ctxt->seg_override = VCPU_SREG_DS;
5199 case 0x64: /* FS override */
5200 has_seg_override = true;
5201 ctxt->seg_override = VCPU_SREG_FS;
5203 case 0x65: /* GS override */
5204 has_seg_override = true;
5205 ctxt->seg_override = VCPU_SREG_GS;
5207 case 0x40 ... 0x4f: /* REX */
5208 if (mode != X86EMUL_MODE_PROT64)
5210 ctxt->rex_prefix = ctxt->b;
5212 case 0xf0: /* LOCK */
5213 ctxt->lock_prefix = 1;
5215 case 0xf2: /* REPNE/REPNZ */
5216 case 0xf3: /* REP/REPE/REPZ */
5217 ctxt->rep_prefix = ctxt->b;
5223 /* Any legacy prefix after a REX prefix nullifies its effect. */
5225 ctxt->rex_prefix = 0;
5231 if (ctxt->rex_prefix & 8)
5232 ctxt->op_bytes = 8; /* REX.W */
5234 /* Opcode byte(s). */
5235 opcode = opcode_table[ctxt->b];
5236 /* Two-byte opcode? */
5237 if (ctxt->b == 0x0f) {
5238 ctxt->opcode_len = 2;
5239 ctxt->b = insn_fetch(u8, ctxt);
5240 opcode = twobyte_table[ctxt->b];
5242 /* 0F_38 opcode map */
5243 if (ctxt->b == 0x38) {
5244 ctxt->opcode_len = 3;
5245 ctxt->b = insn_fetch(u8, ctxt);
5246 opcode = opcode_map_0f_38[ctxt->b];
5249 ctxt->d = opcode.flags;
5251 if (ctxt->d & ModRM)
5252 ctxt->modrm = insn_fetch(u8, ctxt);
5254 /* vex-prefix instructions are not implemented */
5255 if (ctxt->opcode_len == 1 && (ctxt->b == 0xc5 || ctxt->b == 0xc4) &&
5256 (mode == X86EMUL_MODE_PROT64 || (ctxt->modrm & 0xc0) == 0xc0)) {
5260 while (ctxt->d & GroupMask) {
5261 switch (ctxt->d & GroupMask) {
5263 goffset = (ctxt->modrm >> 3) & 7;
5264 opcode = opcode.u.group[goffset];
5267 goffset = (ctxt->modrm >> 3) & 7;
5268 if ((ctxt->modrm >> 6) == 3)
5269 opcode = opcode.u.gdual->mod3[goffset];
5271 opcode = opcode.u.gdual->mod012[goffset];
5274 goffset = ctxt->modrm & 7;
5275 opcode = opcode.u.group[goffset];
5278 if (ctxt->rep_prefix && op_prefix)
5279 return EMULATION_FAILED;
5280 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
5281 switch (simd_prefix) {
5282 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
5283 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
5284 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
5285 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
5289 if (ctxt->modrm > 0xbf) {
5290 size_t size = ARRAY_SIZE(opcode.u.esc->high);
5291 u32 index = array_index_nospec(
5292 ctxt->modrm - 0xc0, size);
5294 opcode = opcode.u.esc->high[index];
5296 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
5300 if ((ctxt->modrm >> 6) == 3)
5301 opcode = opcode.u.idual->mod3;
5303 opcode = opcode.u.idual->mod012;
5306 if (ctxt->mode == X86EMUL_MODE_PROT64)
5307 opcode = opcode.u.mdual->mode64;
5309 opcode = opcode.u.mdual->mode32;
5312 return EMULATION_FAILED;
5315 ctxt->d &= ~(u64)GroupMask;
5316 ctxt->d |= opcode.flags;
5321 return EMULATION_FAILED;
5323 ctxt->execute = opcode.u.execute;
5325 if (unlikely(ctxt->ud) && likely(!(ctxt->d & EmulateOnUD)))
5326 return EMULATION_FAILED;
5328 if (unlikely(ctxt->d &
5329 (NotImpl|Stack|Op3264|Sse|Mmx|Intercept|CheckPerm|NearBranch|
5332 * These are copied unconditionally here, and checked unconditionally
5333 * in x86_emulate_insn.
5335 ctxt->check_perm = opcode.check_perm;
5336 ctxt->intercept = opcode.intercept;
5338 if (ctxt->d & NotImpl)
5339 return EMULATION_FAILED;
5341 if (mode == X86EMUL_MODE_PROT64) {
5342 if (ctxt->op_bytes == 4 && (ctxt->d & Stack))
5344 else if (ctxt->d & NearBranch)
5348 if (ctxt->d & Op3264) {
5349 if (mode == X86EMUL_MODE_PROT64)
5355 if ((ctxt->d & No16) && ctxt->op_bytes == 2)
5359 ctxt->op_bytes = 16;
5360 else if (ctxt->d & Mmx)
5364 /* ModRM and SIB bytes. */
5365 if (ctxt->d & ModRM) {
5366 rc = decode_modrm(ctxt, &ctxt->memop);
5367 if (!has_seg_override) {
5368 has_seg_override = true;
5369 ctxt->seg_override = ctxt->modrm_seg;
5371 } else if (ctxt->d & MemAbs)
5372 rc = decode_abs(ctxt, &ctxt->memop);
5373 if (rc != X86EMUL_CONTINUE)
5376 if (!has_seg_override)
5377 ctxt->seg_override = VCPU_SREG_DS;
5379 ctxt->memop.addr.mem.seg = ctxt->seg_override;
5382 * Decode and fetch the source operand: register, memory
5385 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
5386 if (rc != X86EMUL_CONTINUE)
5390 * Decode and fetch the second source operand: register, memory
5393 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
5394 if (rc != X86EMUL_CONTINUE)
5397 /* Decode and fetch the destination operand: register or memory. */
5398 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
5400 if (ctxt->rip_relative && likely(ctxt->memopp))
5401 ctxt->memopp->addr.mem.ea = address_mask(ctxt,
5402 ctxt->memopp->addr.mem.ea + ctxt->_eip);
5405 if (rc == X86EMUL_PROPAGATE_FAULT)
5406 ctxt->have_exception = true;
5407 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
5410 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
5412 return ctxt->d & PageTable;
5415 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
5417 /* The second termination condition only applies for REPE
5418 * and REPNE. Test if the repeat string operation prefix is
5419 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
5420 * corresponding termination condition according to:
5421 * - if REPE/REPZ and ZF = 0 then done
5422 * - if REPNE/REPNZ and ZF = 1 then done
5424 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
5425 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
5426 && (((ctxt->rep_prefix == REPE_PREFIX) &&
5427 ((ctxt->eflags & X86_EFLAGS_ZF) == 0))
5428 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
5429 ((ctxt->eflags & X86_EFLAGS_ZF) == X86_EFLAGS_ZF))))
5435 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
5440 rc = asm_safe("fwait");
5443 if (unlikely(rc != X86EMUL_CONTINUE))
5444 return emulate_exception(ctxt, MF_VECTOR, 0, false);
5446 return X86EMUL_CONTINUE;
5449 static void fetch_possible_mmx_operand(struct operand *op)
5451 if (op->type == OP_MM)
5452 read_mmx_reg(&op->mm_val, op->addr.mm);
5455 static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop)
5457 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
5459 if (!(ctxt->d & ByteOp))
5460 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
5462 asm("push %[flags]; popf; " CALL_NOSPEC " ; pushf; pop %[flags]\n"
5463 : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
5464 [thunk_target]"+S"(fop), ASM_CALL_CONSTRAINT
5465 : "c"(ctxt->src2.val));
5467 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
5468 if (!fop) /* exception is returned in fop variable */
5469 return emulate_de(ctxt);
5470 return X86EMUL_CONTINUE;
5473 void init_decode_cache(struct x86_emulate_ctxt *ctxt)
5475 memset(&ctxt->rip_relative, 0,
5476 (void *)&ctxt->modrm - (void *)&ctxt->rip_relative);
5478 ctxt->io_read.pos = 0;
5479 ctxt->io_read.end = 0;
5480 ctxt->mem_read.end = 0;
5483 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
5485 const struct x86_emulate_ops *ops = ctxt->ops;
5486 int rc = X86EMUL_CONTINUE;
5487 int saved_dst_type = ctxt->dst.type;
5488 unsigned emul_flags;
5490 ctxt->mem_read.pos = 0;
5492 /* LOCK prefix is allowed only with some instructions */
5493 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
5494 rc = emulate_ud(ctxt);
5498 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
5499 rc = emulate_ud(ctxt);
5503 emul_flags = ctxt->ops->get_hflags(ctxt);
5504 if (unlikely(ctxt->d &
5505 (No64|Undefined|Sse|Mmx|Intercept|CheckPerm|Priv|Prot|String))) {
5506 if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
5507 (ctxt->d & Undefined)) {
5508 rc = emulate_ud(ctxt);
5512 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
5513 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
5514 rc = emulate_ud(ctxt);
5518 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
5519 rc = emulate_nm(ctxt);
5523 if (ctxt->d & Mmx) {
5524 rc = flush_pending_x87_faults(ctxt);
5525 if (rc != X86EMUL_CONTINUE)
5528 * Now that we know the fpu is exception safe, we can fetch
5531 fetch_possible_mmx_operand(&ctxt->src);
5532 fetch_possible_mmx_operand(&ctxt->src2);
5533 if (!(ctxt->d & Mov))
5534 fetch_possible_mmx_operand(&ctxt->dst);
5537 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && ctxt->intercept) {
5538 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5539 X86_ICPT_PRE_EXCEPT);
5540 if (rc != X86EMUL_CONTINUE)
5544 /* Instruction can only be executed in protected mode */
5545 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
5546 rc = emulate_ud(ctxt);
5550 /* Privileged instruction can be executed only in CPL=0 */
5551 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
5552 if (ctxt->d & PrivUD)
5553 rc = emulate_ud(ctxt);
5555 rc = emulate_gp(ctxt, 0);
5559 /* Do instruction specific permission checks */
5560 if (ctxt->d & CheckPerm) {
5561 rc = ctxt->check_perm(ctxt);
5562 if (rc != X86EMUL_CONTINUE)
5566 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {
5567 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5568 X86_ICPT_POST_EXCEPT);
5569 if (rc != X86EMUL_CONTINUE)
5573 if (ctxt->rep_prefix && (ctxt->d & String)) {
5574 /* All REP prefixes have the same first termination condition */
5575 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
5576 string_registers_quirk(ctxt);
5577 ctxt->eip = ctxt->_eip;
5578 ctxt->eflags &= ~X86_EFLAGS_RF;
5584 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
5585 rc = segmented_read(ctxt, ctxt->src.addr.mem,
5586 ctxt->src.valptr, ctxt->src.bytes);
5587 if (rc != X86EMUL_CONTINUE)
5589 ctxt->src.orig_val64 = ctxt->src.val64;
5592 if (ctxt->src2.type == OP_MEM) {
5593 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
5594 &ctxt->src2.val, ctxt->src2.bytes);
5595 if (rc != X86EMUL_CONTINUE)
5599 if ((ctxt->d & DstMask) == ImplicitOps)
5603 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
5604 /* optimisation - avoid slow emulated read if Mov */
5605 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
5606 &ctxt->dst.val, ctxt->dst.bytes);
5607 if (rc != X86EMUL_CONTINUE) {
5608 if (!(ctxt->d & NoWrite) &&
5609 rc == X86EMUL_PROPAGATE_FAULT &&
5610 ctxt->exception.vector == PF_VECTOR)
5611 ctxt->exception.error_code |= PFERR_WRITE_MASK;
5615 /* Copy full 64-bit value for CMPXCHG8B. */
5616 ctxt->dst.orig_val64 = ctxt->dst.val64;
5620 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {
5621 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5622 X86_ICPT_POST_MEMACCESS);
5623 if (rc != X86EMUL_CONTINUE)
5627 if (ctxt->rep_prefix && (ctxt->d & String))
5628 ctxt->eflags |= X86_EFLAGS_RF;
5630 ctxt->eflags &= ~X86_EFLAGS_RF;
5632 if (ctxt->execute) {
5633 if (ctxt->d & Fastop)
5634 rc = fastop(ctxt, ctxt->fop);
5636 rc = ctxt->execute(ctxt);
5637 if (rc != X86EMUL_CONTINUE)
5642 if (ctxt->opcode_len == 2)
5644 else if (ctxt->opcode_len == 3)
5645 goto threebyte_insn;
5648 case 0x70 ... 0x7f: /* jcc (short) */
5649 if (test_cc(ctxt->b, ctxt->eflags))
5650 rc = jmp_rel(ctxt, ctxt->src.val);
5652 case 0x8d: /* lea r16/r32, m */
5653 ctxt->dst.val = ctxt->src.addr.mem.ea;
5655 case 0x90 ... 0x97: /* nop / xchg reg, rax */
5656 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
5657 ctxt->dst.type = OP_NONE;
5661 case 0x98: /* cbw/cwde/cdqe */
5662 switch (ctxt->op_bytes) {
5663 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
5664 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
5665 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
5668 case 0xcc: /* int3 */
5669 rc = emulate_int(ctxt, 3);
5671 case 0xcd: /* int n */
5672 rc = emulate_int(ctxt, ctxt->src.val);
5674 case 0xce: /* into */
5675 if (ctxt->eflags & X86_EFLAGS_OF)
5676 rc = emulate_int(ctxt, 4);
5678 case 0xe9: /* jmp rel */
5679 case 0xeb: /* jmp rel short */
5680 rc = jmp_rel(ctxt, ctxt->src.val);
5681 ctxt->dst.type = OP_NONE; /* Disable writeback. */
5683 case 0xf4: /* hlt */
5684 ctxt->ops->halt(ctxt);
5686 case 0xf5: /* cmc */
5687 /* complement carry flag from eflags reg */
5688 ctxt->eflags ^= X86_EFLAGS_CF;
5690 case 0xf8: /* clc */
5691 ctxt->eflags &= ~X86_EFLAGS_CF;
5693 case 0xf9: /* stc */
5694 ctxt->eflags |= X86_EFLAGS_CF;
5696 case 0xfc: /* cld */
5697 ctxt->eflags &= ~X86_EFLAGS_DF;
5699 case 0xfd: /* std */
5700 ctxt->eflags |= X86_EFLAGS_DF;
5703 goto cannot_emulate;
5706 if (rc != X86EMUL_CONTINUE)
5710 if (ctxt->d & SrcWrite) {
5711 BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);
5712 rc = writeback(ctxt, &ctxt->src);
5713 if (rc != X86EMUL_CONTINUE)
5716 if (!(ctxt->d & NoWrite)) {
5717 rc = writeback(ctxt, &ctxt->dst);
5718 if (rc != X86EMUL_CONTINUE)
5723 * restore dst type in case the decoding will be reused
5724 * (happens for string instruction )
5726 ctxt->dst.type = saved_dst_type;
5728 if ((ctxt->d & SrcMask) == SrcSI)
5729 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
5731 if ((ctxt->d & DstMask) == DstDI)
5732 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
5734 if (ctxt->rep_prefix && (ctxt->d & String)) {
5736 struct read_cache *r = &ctxt->io_read;
5737 if ((ctxt->d & SrcMask) == SrcSI)
5738 count = ctxt->src.count;
5740 count = ctxt->dst.count;
5741 register_address_increment(ctxt, VCPU_REGS_RCX, -count);
5743 if (!string_insn_completed(ctxt)) {
5745 * Re-enter guest when pio read ahead buffer is empty
5746 * or, if it is not used, after each 1024 iteration.
5748 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
5749 (r->end == 0 || r->end != r->pos)) {
5751 * Reset read cache. Usually happens before
5752 * decode, but since instruction is restarted
5753 * we have to do it here.
5755 ctxt->mem_read.end = 0;
5756 writeback_registers(ctxt);
5757 return EMULATION_RESTART;
5759 goto done; /* skip rip writeback */
5761 ctxt->eflags &= ~X86_EFLAGS_RF;
5764 ctxt->eip = ctxt->_eip;
5765 if (ctxt->mode != X86EMUL_MODE_PROT64)
5766 ctxt->eip = (u32)ctxt->_eip;
5769 if (rc == X86EMUL_PROPAGATE_FAULT) {
5770 WARN_ON(ctxt->exception.vector > 0x1f);
5771 ctxt->have_exception = true;
5773 if (rc == X86EMUL_INTERCEPTED)
5774 return EMULATION_INTERCEPTED;
5776 if (rc == X86EMUL_CONTINUE)
5777 writeback_registers(ctxt);
5779 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
5783 case 0x09: /* wbinvd */
5784 (ctxt->ops->wbinvd)(ctxt);
5786 case 0x08: /* invd */
5787 case 0x0d: /* GrpP (prefetch) */
5788 case 0x18: /* Grp16 (prefetch/nop) */
5789 case 0x1f: /* nop */
5791 case 0x20: /* mov cr, reg */
5792 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
5794 case 0x21: /* mov from dr to reg */
5795 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
5797 case 0x40 ... 0x4f: /* cmov */
5798 if (test_cc(ctxt->b, ctxt->eflags))
5799 ctxt->dst.val = ctxt->src.val;
5800 else if (ctxt->op_bytes != 4)
5801 ctxt->dst.type = OP_NONE; /* no writeback */
5803 case 0x80 ... 0x8f: /* jnz rel, etc*/
5804 if (test_cc(ctxt->b, ctxt->eflags))
5805 rc = jmp_rel(ctxt, ctxt->src.val);
5807 case 0x90 ... 0x9f: /* setcc r/m8 */
5808 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
5810 case 0xb6 ... 0xb7: /* movzx */
5811 ctxt->dst.bytes = ctxt->op_bytes;
5812 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
5813 : (u16) ctxt->src.val;
5815 case 0xbe ... 0xbf: /* movsx */
5816 ctxt->dst.bytes = ctxt->op_bytes;
5817 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
5818 (s16) ctxt->src.val;
5821 goto cannot_emulate;
5826 if (rc != X86EMUL_CONTINUE)
5832 return EMULATION_FAILED;
5835 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
5837 invalidate_registers(ctxt);
5840 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
5842 writeback_registers(ctxt);
5845 bool emulator_can_use_gpa(struct x86_emulate_ctxt *ctxt)
5847 if (ctxt->rep_prefix && (ctxt->d & String))
5850 if (ctxt->d & TwoMemOp)