1 // SPDX-License-Identifier: GPL-2.0-only
2 /******************************************************************************
5 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
7 * Copyright (c) 2005 Keir Fraser
9 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
10 * privileged instructions:
12 * Copyright (C) 2006 Qumranet
13 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
15 * Avi Kivity <avi@qumranet.com>
16 * Yaniv Kamay <yaniv@qumranet.com>
18 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
21 #include <linux/kvm_host.h>
22 #include "kvm_cache_regs.h"
23 #include <asm/kvm_emulate.h>
24 #include <linux/stringify.h>
25 #include <asm/fpu/api.h>
26 #include <asm/debugreg.h>
27 #include <asm/nospec-branch.h>
38 #define OpImplicit 1ull /* No generic decode */
39 #define OpReg 2ull /* Register */
40 #define OpMem 3ull /* Memory */
41 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
42 #define OpDI 5ull /* ES:DI/EDI/RDI */
43 #define OpMem64 6ull /* Memory, 64-bit */
44 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
45 #define OpDX 8ull /* DX register */
46 #define OpCL 9ull /* CL register (for shifts) */
47 #define OpImmByte 10ull /* 8-bit sign extended immediate */
48 #define OpOne 11ull /* Implied 1 */
49 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
50 #define OpMem16 13ull /* Memory operand (16-bit). */
51 #define OpMem32 14ull /* Memory operand (32-bit). */
52 #define OpImmU 15ull /* Immediate operand, zero extended */
53 #define OpSI 16ull /* SI/ESI/RSI */
54 #define OpImmFAddr 17ull /* Immediate far address */
55 #define OpMemFAddr 18ull /* Far address in memory */
56 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
57 #define OpES 20ull /* ES */
58 #define OpCS 21ull /* CS */
59 #define OpSS 22ull /* SS */
60 #define OpDS 23ull /* DS */
61 #define OpFS 24ull /* FS */
62 #define OpGS 25ull /* GS */
63 #define OpMem8 26ull /* 8-bit zero extended memory operand */
64 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
65 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
66 #define OpAccLo 29ull /* Low part of extended acc (AX/AX/EAX/RAX) */
67 #define OpAccHi 30ull /* High part of extended acc (-/DX/EDX/RDX) */
69 #define OpBits 5 /* Width of operand field */
70 #define OpMask ((1ull << OpBits) - 1)
73 * Opcode effective-address decode tables.
74 * Note that we only emulate instructions that have at least one memory
75 * operand (excluding implicit stack references). We assume that stack
76 * references and instruction fetches will never occur in special memory
77 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
81 /* Operand sizes: 8-bit operands or specified/overridden size. */
82 #define ByteOp (1<<0) /* 8-bit operands. */
83 /* Destination operand type. */
85 #define ImplicitOps (OpImplicit << DstShift)
86 #define DstReg (OpReg << DstShift)
87 #define DstMem (OpMem << DstShift)
88 #define DstAcc (OpAcc << DstShift)
89 #define DstDI (OpDI << DstShift)
90 #define DstMem64 (OpMem64 << DstShift)
91 #define DstMem16 (OpMem16 << DstShift)
92 #define DstImmUByte (OpImmUByte << DstShift)
93 #define DstDX (OpDX << DstShift)
94 #define DstAccLo (OpAccLo << DstShift)
95 #define DstMask (OpMask << DstShift)
96 /* Source operand type. */
98 #define SrcNone (OpNone << SrcShift)
99 #define SrcReg (OpReg << SrcShift)
100 #define SrcMem (OpMem << SrcShift)
101 #define SrcMem16 (OpMem16 << SrcShift)
102 #define SrcMem32 (OpMem32 << SrcShift)
103 #define SrcImm (OpImm << SrcShift)
104 #define SrcImmByte (OpImmByte << SrcShift)
105 #define SrcOne (OpOne << SrcShift)
106 #define SrcImmUByte (OpImmUByte << SrcShift)
107 #define SrcImmU (OpImmU << SrcShift)
108 #define SrcSI (OpSI << SrcShift)
109 #define SrcXLat (OpXLat << SrcShift)
110 #define SrcImmFAddr (OpImmFAddr << SrcShift)
111 #define SrcMemFAddr (OpMemFAddr << SrcShift)
112 #define SrcAcc (OpAcc << SrcShift)
113 #define SrcImmU16 (OpImmU16 << SrcShift)
114 #define SrcImm64 (OpImm64 << SrcShift)
115 #define SrcDX (OpDX << SrcShift)
116 #define SrcMem8 (OpMem8 << SrcShift)
117 #define SrcAccHi (OpAccHi << SrcShift)
118 #define SrcMask (OpMask << SrcShift)
119 #define BitOp (1<<11)
120 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
121 #define String (1<<13) /* String instruction (rep capable) */
122 #define Stack (1<<14) /* Stack instruction (push/pop) */
123 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
124 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
125 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
126 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
127 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
128 #define Escape (5<<15) /* Escape to coprocessor instruction */
129 #define InstrDual (6<<15) /* Alternate instruction decoding of mod == 3 */
130 #define ModeDual (7<<15) /* Different instruction for 32/64 bit */
131 #define Sse (1<<18) /* SSE Vector instruction */
132 /* Generic ModRM decode. */
133 #define ModRM (1<<19)
134 /* Destination is only written; never read. */
137 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
138 #define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
139 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
140 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
141 #define Undefined (1<<25) /* No Such Instruction */
142 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
143 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
145 #define PageTable (1 << 29) /* instruction used to write page table */
146 #define NotImpl (1 << 30) /* instruction is not implemented */
147 /* Source 2 operand type */
148 #define Src2Shift (31)
149 #define Src2None (OpNone << Src2Shift)
150 #define Src2Mem (OpMem << Src2Shift)
151 #define Src2CL (OpCL << Src2Shift)
152 #define Src2ImmByte (OpImmByte << Src2Shift)
153 #define Src2One (OpOne << Src2Shift)
154 #define Src2Imm (OpImm << Src2Shift)
155 #define Src2ES (OpES << Src2Shift)
156 #define Src2CS (OpCS << Src2Shift)
157 #define Src2SS (OpSS << Src2Shift)
158 #define Src2DS (OpDS << Src2Shift)
159 #define Src2FS (OpFS << Src2Shift)
160 #define Src2GS (OpGS << Src2Shift)
161 #define Src2Mask (OpMask << Src2Shift)
162 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
163 #define AlignMask ((u64)7 << 41)
164 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
165 #define Unaligned ((u64)2 << 41) /* Explicitly unaligned (e.g. MOVDQU) */
166 #define Avx ((u64)3 << 41) /* Advanced Vector Extensions */
167 #define Aligned16 ((u64)4 << 41) /* Aligned to 16 byte boundary (e.g. FXSAVE) */
168 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
169 #define NoWrite ((u64)1 << 45) /* No writeback */
170 #define SrcWrite ((u64)1 << 46) /* Write back src operand */
171 #define NoMod ((u64)1 << 47) /* Mod field is ignored */
172 #define Intercept ((u64)1 << 48) /* Has valid intercept field */
173 #define CheckPerm ((u64)1 << 49) /* Has valid check_perm field */
174 #define PrivUD ((u64)1 << 51) /* #UD instead of #GP on CPL > 0 */
175 #define NearBranch ((u64)1 << 52) /* Near branches */
176 #define No16 ((u64)1 << 53) /* No 16 bit operand */
177 #define IncSP ((u64)1 << 54) /* SP is incremented before ModRM calc */
178 #define TwoMemOp ((u64)1 << 55) /* Instruction has two memory operand */
180 #define DstXacc (DstAccLo | SrcAccHi | SrcWrite)
182 #define X2(x...) x, x
183 #define X3(x...) X2(x), x
184 #define X4(x...) X2(x), X2(x)
185 #define X5(x...) X4(x), x
186 #define X6(x...) X4(x), X2(x)
187 #define X7(x...) X4(x), X3(x)
188 #define X8(x...) X4(x), X4(x)
189 #define X16(x...) X8(x), X8(x)
191 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
192 #define FASTOP_SIZE 8
198 int (*execute)(struct x86_emulate_ctxt *ctxt);
199 const struct opcode *group;
200 const struct group_dual *gdual;
201 const struct gprefix *gprefix;
202 const struct escape *esc;
203 const struct instr_dual *idual;
204 const struct mode_dual *mdual;
205 void (*fastop)(struct fastop *fake);
207 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
211 struct opcode mod012[8];
212 struct opcode mod3[8];
216 struct opcode pfx_no;
217 struct opcode pfx_66;
218 struct opcode pfx_f2;
219 struct opcode pfx_f3;
224 struct opcode high[64];
228 struct opcode mod012;
233 struct opcode mode32;
234 struct opcode mode64;
237 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
239 enum x86_transfer_type {
241 X86_TRANSFER_CALL_JMP,
243 X86_TRANSFER_TASK_SWITCH,
246 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
248 if (!(ctxt->regs_valid & (1 << nr))) {
249 ctxt->regs_valid |= 1 << nr;
250 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
252 return ctxt->_regs[nr];
255 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
257 ctxt->regs_valid |= 1 << nr;
258 ctxt->regs_dirty |= 1 << nr;
259 return &ctxt->_regs[nr];
262 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
265 return reg_write(ctxt, nr);
268 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
272 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
273 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
276 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
278 ctxt->regs_dirty = 0;
279 ctxt->regs_valid = 0;
283 * These EFLAGS bits are restored from saved value during emulation, and
284 * any changes are written back to the saved value after emulation.
286 #define EFLAGS_MASK (X86_EFLAGS_OF|X86_EFLAGS_SF|X86_EFLAGS_ZF|X86_EFLAGS_AF|\
287 X86_EFLAGS_PF|X86_EFLAGS_CF)
296 * fastop functions have a special calling convention:
301 * flags: rflags (in/out)
302 * ex: rsi (in:fastop pointer, out:zero if exception)
304 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
305 * different operand sizes can be reached by calculation, rather than a jump
306 * table (which would be bigger than the code).
308 static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop);
310 #define __FOP_FUNC(name) \
311 ".align " __stringify(FASTOP_SIZE) " \n\t" \
312 ".type " name ", @function \n\t" \
315 #define FOP_FUNC(name) \
318 #define __FOP_RET(name) \
320 ".size " name ", .-" name "\n\t"
322 #define FOP_RET(name) \
325 #define FOP_START(op) \
326 extern void em_##op(struct fastop *fake); \
327 asm(".pushsection .text, \"ax\" \n\t" \
328 ".global em_" #op " \n\t" \
329 ".align " __stringify(FASTOP_SIZE) " \n\t" \
335 #define __FOPNOP(name) \
340 __FOPNOP(__stringify(__UNIQUE_ID(nop)))
342 #define FOP1E(op, dst) \
343 __FOP_FUNC(#op "_" #dst) \
344 "10: " #op " %" #dst " \n\t" \
345 __FOP_RET(#op "_" #dst)
347 #define FOP1EEX(op, dst) \
348 FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
350 #define FASTOP1(op) \
355 ON64(FOP1E(op##q, rax)) \
358 /* 1-operand, using src2 (for MUL/DIV r/m) */
359 #define FASTOP1SRC2(op, name) \
364 ON64(FOP1E(op, rcx)) \
367 /* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
368 #define FASTOP1SRC2EX(op, name) \
373 ON64(FOP1EEX(op, rcx)) \
376 #define FOP2E(op, dst, src) \
377 __FOP_FUNC(#op "_" #dst "_" #src) \
378 #op " %" #src ", %" #dst " \n\t" \
379 __FOP_RET(#op "_" #dst "_" #src)
381 #define FASTOP2(op) \
383 FOP2E(op##b, al, dl) \
384 FOP2E(op##w, ax, dx) \
385 FOP2E(op##l, eax, edx) \
386 ON64(FOP2E(op##q, rax, rdx)) \
389 /* 2 operand, word only */
390 #define FASTOP2W(op) \
393 FOP2E(op##w, ax, dx) \
394 FOP2E(op##l, eax, edx) \
395 ON64(FOP2E(op##q, rax, rdx)) \
398 /* 2 operand, src is CL */
399 #define FASTOP2CL(op) \
401 FOP2E(op##b, al, cl) \
402 FOP2E(op##w, ax, cl) \
403 FOP2E(op##l, eax, cl) \
404 ON64(FOP2E(op##q, rax, cl)) \
407 /* 2 operand, src and dest are reversed */
408 #define FASTOP2R(op, name) \
410 FOP2E(op##b, dl, al) \
411 FOP2E(op##w, dx, ax) \
412 FOP2E(op##l, edx, eax) \
413 ON64(FOP2E(op##q, rdx, rax)) \
416 #define FOP3E(op, dst, src, src2) \
417 __FOP_FUNC(#op "_" #dst "_" #src "_" #src2) \
418 #op " %" #src2 ", %" #src ", %" #dst " \n\t"\
419 __FOP_RET(#op "_" #dst "_" #src "_" #src2)
421 /* 3-operand, word-only, src2=cl */
422 #define FASTOP3WCL(op) \
425 FOP3E(op##w, ax, dx, cl) \
426 FOP3E(op##l, eax, edx, cl) \
427 ON64(FOP3E(op##q, rax, rdx, cl)) \
430 /* Special case for SETcc - 1 instruction per cc */
431 #define FOP_SETCC(op) \
433 ".type " #op ", @function \n\t" \
438 asm(".pushsection .fixup, \"ax\"\n"
439 ".global kvm_fastop_exception \n"
440 "kvm_fastop_exception: xor %esi, %esi; ret\n"
464 "pushf; sbb %al, %al; popf \n\t"
469 * XXX: inoutclob user must know where the argument is being expanded.
470 * Relying on CONFIG_CC_HAS_ASM_GOTO would allow us to remove _fault.
472 #define asm_safe(insn, inoutclob...) \
476 asm volatile("1:" insn "\n" \
478 ".pushsection .fixup, \"ax\"\n" \
479 "3: movl $1, %[_fault]\n" \
482 _ASM_EXTABLE(1b, 3b) \
483 : [_fault] "+qm"(_fault) inoutclob ); \
485 _fault ? X86EMUL_UNHANDLEABLE : X86EMUL_CONTINUE; \
488 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
489 enum x86_intercept intercept,
490 enum x86_intercept_stage stage)
492 struct x86_instruction_info info = {
493 .intercept = intercept,
494 .rep_prefix = ctxt->rep_prefix,
495 .modrm_mod = ctxt->modrm_mod,
496 .modrm_reg = ctxt->modrm_reg,
497 .modrm_rm = ctxt->modrm_rm,
498 .src_val = ctxt->src.val64,
499 .dst_val = ctxt->dst.val64,
500 .src_bytes = ctxt->src.bytes,
501 .dst_bytes = ctxt->dst.bytes,
502 .ad_bytes = ctxt->ad_bytes,
503 .next_rip = ctxt->eip,
506 return ctxt->ops->intercept(ctxt, &info, stage);
509 static void assign_masked(ulong *dest, ulong src, ulong mask)
511 *dest = (*dest & ~mask) | (src & mask);
514 static void assign_register(unsigned long *reg, u64 val, int bytes)
516 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
519 *(u8 *)reg = (u8)val;
522 *(u16 *)reg = (u16)val;
526 break; /* 64b: zero-extend */
533 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
535 return (1UL << (ctxt->ad_bytes << 3)) - 1;
538 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
541 struct desc_struct ss;
543 if (ctxt->mode == X86EMUL_MODE_PROT64)
545 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
546 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
549 static int stack_size(struct x86_emulate_ctxt *ctxt)
551 return (__fls(stack_mask(ctxt)) + 1) >> 3;
554 /* Access/update address held in a register, based on addressing mode. */
555 static inline unsigned long
556 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
558 if (ctxt->ad_bytes == sizeof(unsigned long))
561 return reg & ad_mask(ctxt);
564 static inline unsigned long
565 register_address(struct x86_emulate_ctxt *ctxt, int reg)
567 return address_mask(ctxt, reg_read(ctxt, reg));
570 static void masked_increment(ulong *reg, ulong mask, int inc)
572 assign_masked(reg, *reg + inc, mask);
576 register_address_increment(struct x86_emulate_ctxt *ctxt, int reg, int inc)
578 ulong *preg = reg_rmw(ctxt, reg);
580 assign_register(preg, *preg + inc, ctxt->ad_bytes);
583 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
585 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
588 static u32 desc_limit_scaled(struct desc_struct *desc)
590 u32 limit = get_desc_limit(desc);
592 return desc->g ? (limit << 12) | 0xfff : limit;
595 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
597 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
600 return ctxt->ops->get_cached_segment_base(ctxt, seg);
603 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
604 u32 error, bool valid)
607 ctxt->exception.vector = vec;
608 ctxt->exception.error_code = error;
609 ctxt->exception.error_code_valid = valid;
610 return X86EMUL_PROPAGATE_FAULT;
613 static int emulate_db(struct x86_emulate_ctxt *ctxt)
615 return emulate_exception(ctxt, DB_VECTOR, 0, false);
618 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
620 return emulate_exception(ctxt, GP_VECTOR, err, true);
623 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
625 return emulate_exception(ctxt, SS_VECTOR, err, true);
628 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
630 return emulate_exception(ctxt, UD_VECTOR, 0, false);
633 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
635 return emulate_exception(ctxt, TS_VECTOR, err, true);
638 static int emulate_de(struct x86_emulate_ctxt *ctxt)
640 return emulate_exception(ctxt, DE_VECTOR, 0, false);
643 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
645 return emulate_exception(ctxt, NM_VECTOR, 0, false);
648 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
651 struct desc_struct desc;
653 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
657 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
662 struct desc_struct desc;
664 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
665 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
669 * x86 defines three classes of vector instructions: explicitly
670 * aligned, explicitly unaligned, and the rest, which change behaviour
671 * depending on whether they're AVX encoded or not.
673 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
674 * subject to the same check. FXSAVE and FXRSTOR are checked here too as their
675 * 512 bytes of data must be aligned to a 16 byte boundary.
677 static unsigned insn_alignment(struct x86_emulate_ctxt *ctxt, unsigned size)
679 u64 alignment = ctxt->d & AlignMask;
681 if (likely(size < 16))
696 static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt,
697 struct segmented_address addr,
698 unsigned *max_size, unsigned size,
699 bool write, bool fetch,
700 enum x86emul_mode mode, ulong *linear)
702 struct desc_struct desc;
709 la = seg_base(ctxt, addr.seg) + addr.ea;
712 case X86EMUL_MODE_PROT64:
714 va_bits = ctxt_virt_addr_bits(ctxt);
715 if (get_canonical(la, va_bits) != la)
718 *max_size = min_t(u64, ~0u, (1ull << va_bits) - la);
719 if (size > *max_size)
723 *linear = la = (u32)la;
724 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
728 /* code segment in protected mode or read-only data segment */
729 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
730 || !(desc.type & 2)) && write)
732 /* unreadable code segment */
733 if (!fetch && (desc.type & 8) && !(desc.type & 2))
735 lim = desc_limit_scaled(&desc);
736 if (!(desc.type & 8) && (desc.type & 4)) {
737 /* expand-down segment */
740 lim = desc.d ? 0xffffffff : 0xffff;
744 if (lim == 0xffffffff)
747 *max_size = (u64)lim + 1 - addr.ea;
748 if (size > *max_size)
753 if (la & (insn_alignment(ctxt, size) - 1))
754 return emulate_gp(ctxt, 0);
755 return X86EMUL_CONTINUE;
757 if (addr.seg == VCPU_SREG_SS)
758 return emulate_ss(ctxt, 0);
760 return emulate_gp(ctxt, 0);
763 static int linearize(struct x86_emulate_ctxt *ctxt,
764 struct segmented_address addr,
765 unsigned size, bool write,
769 return __linearize(ctxt, addr, &max_size, size, write, false,
773 static inline int assign_eip(struct x86_emulate_ctxt *ctxt, ulong dst,
774 enum x86emul_mode mode)
779 struct segmented_address addr = { .seg = VCPU_SREG_CS,
782 if (ctxt->op_bytes != sizeof(unsigned long))
783 addr.ea = dst & ((1UL << (ctxt->op_bytes << 3)) - 1);
784 rc = __linearize(ctxt, addr, &max_size, 1, false, true, mode, &linear);
785 if (rc == X86EMUL_CONTINUE)
786 ctxt->_eip = addr.ea;
790 static inline int assign_eip_near(struct x86_emulate_ctxt *ctxt, ulong dst)
792 return assign_eip(ctxt, dst, ctxt->mode);
795 static int assign_eip_far(struct x86_emulate_ctxt *ctxt, ulong dst,
796 const struct desc_struct *cs_desc)
798 enum x86emul_mode mode = ctxt->mode;
802 if (ctxt->mode >= X86EMUL_MODE_PROT16) {
806 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
808 mode = X86EMUL_MODE_PROT64;
810 mode = X86EMUL_MODE_PROT32; /* temporary value */
813 if (mode == X86EMUL_MODE_PROT16 || mode == X86EMUL_MODE_PROT32)
814 mode = cs_desc->d ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
815 rc = assign_eip(ctxt, dst, mode);
816 if (rc == X86EMUL_CONTINUE)
821 static inline int jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
823 return assign_eip_near(ctxt, ctxt->_eip + rel);
826 static int linear_read_system(struct x86_emulate_ctxt *ctxt, ulong linear,
827 void *data, unsigned size)
829 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, true);
832 static int linear_write_system(struct x86_emulate_ctxt *ctxt,
833 ulong linear, void *data,
836 return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, true);
839 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
840 struct segmented_address addr,
847 rc = linearize(ctxt, addr, size, false, &linear);
848 if (rc != X86EMUL_CONTINUE)
850 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, false);
853 static int segmented_write_std(struct x86_emulate_ctxt *ctxt,
854 struct segmented_address addr,
861 rc = linearize(ctxt, addr, size, true, &linear);
862 if (rc != X86EMUL_CONTINUE)
864 return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, false);
868 * Prefetch the remaining bytes of the instruction without crossing page
869 * boundary if they are not in fetch_cache yet.
871 static int __do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt, int op_size)
874 unsigned size, max_size;
875 unsigned long linear;
876 int cur_size = ctxt->fetch.end - ctxt->fetch.data;
877 struct segmented_address addr = { .seg = VCPU_SREG_CS,
878 .ea = ctxt->eip + cur_size };
881 * We do not know exactly how many bytes will be needed, and
882 * __linearize is expensive, so fetch as much as possible. We
883 * just have to avoid going beyond the 15 byte limit, the end
884 * of the segment, or the end of the page.
886 * __linearize is called with size 0 so that it does not do any
887 * boundary check itself. Instead, we use max_size to check
890 rc = __linearize(ctxt, addr, &max_size, 0, false, true, ctxt->mode,
892 if (unlikely(rc != X86EMUL_CONTINUE))
895 size = min_t(unsigned, 15UL ^ cur_size, max_size);
896 size = min_t(unsigned, size, PAGE_SIZE - offset_in_page(linear));
899 * One instruction can only straddle two pages,
900 * and one has been loaded at the beginning of
901 * x86_decode_insn. So, if not enough bytes
902 * still, we must have hit the 15-byte boundary.
904 if (unlikely(size < op_size))
905 return emulate_gp(ctxt, 0);
907 rc = ctxt->ops->fetch(ctxt, linear, ctxt->fetch.end,
908 size, &ctxt->exception);
909 if (unlikely(rc != X86EMUL_CONTINUE))
911 ctxt->fetch.end += size;
912 return X86EMUL_CONTINUE;
915 static __always_inline int do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt,
918 unsigned done_size = ctxt->fetch.end - ctxt->fetch.ptr;
920 if (unlikely(done_size < size))
921 return __do_insn_fetch_bytes(ctxt, size - done_size);
923 return X86EMUL_CONTINUE;
926 /* Fetch next part of the instruction being emulated. */
927 #define insn_fetch(_type, _ctxt) \
930 rc = do_insn_fetch_bytes(_ctxt, sizeof(_type)); \
931 if (rc != X86EMUL_CONTINUE) \
933 ctxt->_eip += sizeof(_type); \
934 memcpy(&_x, ctxt->fetch.ptr, sizeof(_type)); \
935 ctxt->fetch.ptr += sizeof(_type); \
939 #define insn_fetch_arr(_arr, _size, _ctxt) \
941 rc = do_insn_fetch_bytes(_ctxt, _size); \
942 if (rc != X86EMUL_CONTINUE) \
944 ctxt->_eip += (_size); \
945 memcpy(_arr, ctxt->fetch.ptr, _size); \
946 ctxt->fetch.ptr += (_size); \
950 * Given the 'reg' portion of a ModRM byte, and a register block, return a
951 * pointer into the block that addresses the relevant register.
952 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
954 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
958 int highbyte_regs = (ctxt->rex_prefix == 0) && byteop;
960 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
961 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
963 p = reg_rmw(ctxt, modrm_reg);
967 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
968 struct segmented_address addr,
969 u16 *size, unsigned long *address, int op_bytes)
976 rc = segmented_read_std(ctxt, addr, size, 2);
977 if (rc != X86EMUL_CONTINUE)
980 rc = segmented_read_std(ctxt, addr, address, op_bytes);
994 FASTOP1SRC2(mul, mul_ex);
995 FASTOP1SRC2(imul, imul_ex);
996 FASTOP1SRC2EX(div, div_ex);
997 FASTOP1SRC2EX(idiv, idiv_ex);
1026 FASTOP2R(cmp, cmp_r);
1028 static int em_bsf_c(struct x86_emulate_ctxt *ctxt)
1030 /* If src is zero, do not writeback, but update flags */
1031 if (ctxt->src.val == 0)
1032 ctxt->dst.type = OP_NONE;
1033 return fastop(ctxt, em_bsf);
1036 static int em_bsr_c(struct x86_emulate_ctxt *ctxt)
1038 /* If src is zero, do not writeback, but update flags */
1039 if (ctxt->src.val == 0)
1040 ctxt->dst.type = OP_NONE;
1041 return fastop(ctxt, em_bsr);
1044 static __always_inline u8 test_cc(unsigned int condition, unsigned long flags)
1047 void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
1049 flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
1050 asm("push %[flags]; popf; " CALL_NOSPEC
1051 : "=a"(rc) : [thunk_target]"r"(fop), [flags]"r"(flags));
1055 static void fetch_register_operand(struct operand *op)
1057 switch (op->bytes) {
1059 op->val = *(u8 *)op->addr.reg;
1062 op->val = *(u16 *)op->addr.reg;
1065 op->val = *(u32 *)op->addr.reg;
1068 op->val = *(u64 *)op->addr.reg;
1073 static void emulator_get_fpu(void)
1077 fpregs_assert_state_consistent();
1078 if (test_thread_flag(TIF_NEED_FPU_LOAD))
1079 switch_fpu_return();
1082 static void emulator_put_fpu(void)
1087 static void read_sse_reg(sse128_t *data, int reg)
1091 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
1092 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
1093 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
1094 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
1095 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
1096 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
1097 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
1098 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
1099 #ifdef CONFIG_X86_64
1100 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
1101 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
1102 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
1103 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
1104 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
1105 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
1106 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
1107 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
1114 static void write_sse_reg(sse128_t *data, int reg)
1118 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
1119 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
1120 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
1121 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
1122 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
1123 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
1124 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
1125 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
1126 #ifdef CONFIG_X86_64
1127 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
1128 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
1129 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
1130 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
1131 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
1132 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
1133 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
1134 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
1141 static void read_mmx_reg(u64 *data, int reg)
1145 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
1146 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
1147 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
1148 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
1149 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
1150 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
1151 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
1152 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
1158 static void write_mmx_reg(u64 *data, int reg)
1162 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
1163 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
1164 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
1165 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
1166 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
1167 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
1168 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
1169 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
1175 static int em_fninit(struct x86_emulate_ctxt *ctxt)
1177 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1178 return emulate_nm(ctxt);
1181 asm volatile("fninit");
1183 return X86EMUL_CONTINUE;
1186 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
1190 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1191 return emulate_nm(ctxt);
1194 asm volatile("fnstcw %0": "+m"(fcw));
1197 ctxt->dst.val = fcw;
1199 return X86EMUL_CONTINUE;
1202 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1206 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1207 return emulate_nm(ctxt);
1210 asm volatile("fnstsw %0": "+m"(fsw));
1213 ctxt->dst.val = fsw;
1215 return X86EMUL_CONTINUE;
1218 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1221 unsigned reg = ctxt->modrm_reg;
1223 if (!(ctxt->d & ModRM))
1224 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1226 if (ctxt->d & Sse) {
1230 read_sse_reg(&op->vec_val, reg);
1233 if (ctxt->d & Mmx) {
1242 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1243 op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);
1245 fetch_register_operand(op);
1246 op->orig_val = op->val;
1249 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1251 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1252 ctxt->modrm_seg = VCPU_SREG_SS;
1255 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1259 int index_reg, base_reg, scale;
1260 int rc = X86EMUL_CONTINUE;
1263 ctxt->modrm_reg = ((ctxt->rex_prefix << 1) & 8); /* REX.R */
1264 index_reg = (ctxt->rex_prefix << 2) & 8; /* REX.X */
1265 base_reg = (ctxt->rex_prefix << 3) & 8; /* REX.B */
1267 ctxt->modrm_mod = (ctxt->modrm & 0xc0) >> 6;
1268 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1269 ctxt->modrm_rm = base_reg | (ctxt->modrm & 0x07);
1270 ctxt->modrm_seg = VCPU_SREG_DS;
1272 if (ctxt->modrm_mod == 3 || (ctxt->d & NoMod)) {
1274 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1275 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
1277 if (ctxt->d & Sse) {
1280 op->addr.xmm = ctxt->modrm_rm;
1281 read_sse_reg(&op->vec_val, ctxt->modrm_rm);
1284 if (ctxt->d & Mmx) {
1287 op->addr.mm = ctxt->modrm_rm & 7;
1290 fetch_register_operand(op);
1296 if (ctxt->ad_bytes == 2) {
1297 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1298 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1299 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1300 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1302 /* 16-bit ModR/M decode. */
1303 switch (ctxt->modrm_mod) {
1305 if (ctxt->modrm_rm == 6)
1306 modrm_ea += insn_fetch(u16, ctxt);
1309 modrm_ea += insn_fetch(s8, ctxt);
1312 modrm_ea += insn_fetch(u16, ctxt);
1315 switch (ctxt->modrm_rm) {
1317 modrm_ea += bx + si;
1320 modrm_ea += bx + di;
1323 modrm_ea += bp + si;
1326 modrm_ea += bp + di;
1335 if (ctxt->modrm_mod != 0)
1342 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1343 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1344 ctxt->modrm_seg = VCPU_SREG_SS;
1345 modrm_ea = (u16)modrm_ea;
1347 /* 32/64-bit ModR/M decode. */
1348 if ((ctxt->modrm_rm & 7) == 4) {
1349 sib = insn_fetch(u8, ctxt);
1350 index_reg |= (sib >> 3) & 7;
1351 base_reg |= sib & 7;
1354 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1355 modrm_ea += insn_fetch(s32, ctxt);
1357 modrm_ea += reg_read(ctxt, base_reg);
1358 adjust_modrm_seg(ctxt, base_reg);
1359 /* Increment ESP on POP [ESP] */
1360 if ((ctxt->d & IncSP) &&
1361 base_reg == VCPU_REGS_RSP)
1362 modrm_ea += ctxt->op_bytes;
1365 modrm_ea += reg_read(ctxt, index_reg) << scale;
1366 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1367 modrm_ea += insn_fetch(s32, ctxt);
1368 if (ctxt->mode == X86EMUL_MODE_PROT64)
1369 ctxt->rip_relative = 1;
1371 base_reg = ctxt->modrm_rm;
1372 modrm_ea += reg_read(ctxt, base_reg);
1373 adjust_modrm_seg(ctxt, base_reg);
1375 switch (ctxt->modrm_mod) {
1377 modrm_ea += insn_fetch(s8, ctxt);
1380 modrm_ea += insn_fetch(s32, ctxt);
1384 op->addr.mem.ea = modrm_ea;
1385 if (ctxt->ad_bytes != 8)
1386 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
1392 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1395 int rc = X86EMUL_CONTINUE;
1398 switch (ctxt->ad_bytes) {
1400 op->addr.mem.ea = insn_fetch(u16, ctxt);
1403 op->addr.mem.ea = insn_fetch(u32, ctxt);
1406 op->addr.mem.ea = insn_fetch(u64, ctxt);
1413 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1417 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1418 mask = ~((long)ctxt->dst.bytes * 8 - 1);
1420 if (ctxt->src.bytes == 2)
1421 sv = (s16)ctxt->src.val & (s16)mask;
1422 else if (ctxt->src.bytes == 4)
1423 sv = (s32)ctxt->src.val & (s32)mask;
1425 sv = (s64)ctxt->src.val & (s64)mask;
1427 ctxt->dst.addr.mem.ea = address_mask(ctxt,
1428 ctxt->dst.addr.mem.ea + (sv >> 3));
1431 /* only subword offset */
1432 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1435 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1436 unsigned long addr, void *dest, unsigned size)
1439 struct read_cache *mc = &ctxt->mem_read;
1441 if (mc->pos < mc->end)
1444 WARN_ON((mc->end + size) >= sizeof(mc->data));
1446 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1448 if (rc != X86EMUL_CONTINUE)
1454 memcpy(dest, mc->data + mc->pos, size);
1456 return X86EMUL_CONTINUE;
1459 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1460 struct segmented_address addr,
1467 rc = linearize(ctxt, addr, size, false, &linear);
1468 if (rc != X86EMUL_CONTINUE)
1470 return read_emulated(ctxt, linear, data, size);
1473 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1474 struct segmented_address addr,
1481 rc = linearize(ctxt, addr, size, true, &linear);
1482 if (rc != X86EMUL_CONTINUE)
1484 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1488 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1489 struct segmented_address addr,
1490 const void *orig_data, const void *data,
1496 rc = linearize(ctxt, addr, size, true, &linear);
1497 if (rc != X86EMUL_CONTINUE)
1499 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1500 size, &ctxt->exception);
1503 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1504 unsigned int size, unsigned short port,
1507 struct read_cache *rc = &ctxt->io_read;
1509 if (rc->pos == rc->end) { /* refill pio read ahead */
1510 unsigned int in_page, n;
1511 unsigned int count = ctxt->rep_prefix ?
1512 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1513 in_page = (ctxt->eflags & X86_EFLAGS_DF) ?
1514 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1515 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1516 n = min3(in_page, (unsigned int)sizeof(rc->data) / size, count);
1519 rc->pos = rc->end = 0;
1520 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1525 if (ctxt->rep_prefix && (ctxt->d & String) &&
1526 !(ctxt->eflags & X86_EFLAGS_DF)) {
1527 ctxt->dst.data = rc->data + rc->pos;
1528 ctxt->dst.type = OP_MEM_STR;
1529 ctxt->dst.count = (rc->end - rc->pos) / size;
1532 memcpy(dest, rc->data + rc->pos, size);
1538 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1539 u16 index, struct desc_struct *desc)
1544 ctxt->ops->get_idt(ctxt, &dt);
1546 if (dt.size < index * 8 + 7)
1547 return emulate_gp(ctxt, index << 3 | 0x2);
1549 addr = dt.address + index * 8;
1550 return linear_read_system(ctxt, addr, desc, sizeof(*desc));
1553 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1554 u16 selector, struct desc_ptr *dt)
1556 const struct x86_emulate_ops *ops = ctxt->ops;
1559 if (selector & 1 << 2) {
1560 struct desc_struct desc;
1563 memset(dt, 0, sizeof(*dt));
1564 if (!ops->get_segment(ctxt, &sel, &desc, &base3,
1568 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1569 dt->address = get_desc_base(&desc) | ((u64)base3 << 32);
1571 ops->get_gdt(ctxt, dt);
1574 static int get_descriptor_ptr(struct x86_emulate_ctxt *ctxt,
1575 u16 selector, ulong *desc_addr_p)
1578 u16 index = selector >> 3;
1581 get_descriptor_table_ptr(ctxt, selector, &dt);
1583 if (dt.size < index * 8 + 7)
1584 return emulate_gp(ctxt, selector & 0xfffc);
1586 addr = dt.address + index * 8;
1588 #ifdef CONFIG_X86_64
1589 if (addr >> 32 != 0) {
1592 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1593 if (!(efer & EFER_LMA))
1598 *desc_addr_p = addr;
1599 return X86EMUL_CONTINUE;
1602 /* allowed just for 8 bytes segments */
1603 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1604 u16 selector, struct desc_struct *desc,
1609 rc = get_descriptor_ptr(ctxt, selector, desc_addr_p);
1610 if (rc != X86EMUL_CONTINUE)
1613 return linear_read_system(ctxt, *desc_addr_p, desc, sizeof(*desc));
1616 /* allowed just for 8 bytes segments */
1617 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1618 u16 selector, struct desc_struct *desc)
1623 rc = get_descriptor_ptr(ctxt, selector, &addr);
1624 if (rc != X86EMUL_CONTINUE)
1627 return linear_write_system(ctxt, addr, desc, sizeof(*desc));
1630 static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1631 u16 selector, int seg, u8 cpl,
1632 enum x86_transfer_type transfer,
1633 struct desc_struct *desc)
1635 struct desc_struct seg_desc, old_desc;
1637 unsigned err_vec = GP_VECTOR;
1639 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1645 memset(&seg_desc, 0, sizeof(seg_desc));
1647 if (ctxt->mode == X86EMUL_MODE_REAL) {
1648 /* set real mode segment descriptor (keep limit etc. for
1650 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1651 set_desc_base(&seg_desc, selector << 4);
1653 } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
1654 /* VM86 needs a clean new segment descriptor */
1655 set_desc_base(&seg_desc, selector << 4);
1656 set_desc_limit(&seg_desc, 0xffff);
1666 /* TR should be in GDT only */
1667 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1670 /* NULL selector is not valid for TR, CS and (except for long mode) SS */
1671 if (null_selector) {
1672 if (seg == VCPU_SREG_CS || seg == VCPU_SREG_TR)
1675 if (seg == VCPU_SREG_SS) {
1676 if (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl)
1680 * ctxt->ops->set_segment expects the CPL to be in
1681 * SS.DPL, so fake an expand-up 32-bit data segment.
1691 /* Skip all following checks */
1695 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1696 if (ret != X86EMUL_CONTINUE)
1699 err_code = selector & 0xfffc;
1700 err_vec = (transfer == X86_TRANSFER_TASK_SWITCH) ? TS_VECTOR :
1703 /* can't load system descriptor into segment selector */
1704 if (seg <= VCPU_SREG_GS && !seg_desc.s) {
1705 if (transfer == X86_TRANSFER_CALL_JMP)
1706 return X86EMUL_UNHANDLEABLE;
1711 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1720 * segment is not a writable data segment or segment
1721 * selector's RPL != CPL or segment selector's RPL != CPL
1723 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1727 if (!(seg_desc.type & 8))
1730 if (seg_desc.type & 4) {
1736 if (rpl > cpl || dpl != cpl)
1739 /* in long-mode d/b must be clear if l is set */
1740 if (seg_desc.d && seg_desc.l) {
1743 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1744 if (efer & EFER_LMA)
1748 /* CS(RPL) <- CPL */
1749 selector = (selector & 0xfffc) | cpl;
1752 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1754 old_desc = seg_desc;
1755 seg_desc.type |= 2; /* busy */
1756 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1757 sizeof(seg_desc), &ctxt->exception);
1758 if (ret != X86EMUL_CONTINUE)
1761 case VCPU_SREG_LDTR:
1762 if (seg_desc.s || seg_desc.type != 2)
1765 default: /* DS, ES, FS, or GS */
1767 * segment is not a data or readable code segment or
1768 * ((segment is a data or nonconforming code segment)
1769 * and (both RPL and CPL > DPL))
1771 if ((seg_desc.type & 0xa) == 0x8 ||
1772 (((seg_desc.type & 0xc) != 0xc) &&
1773 (rpl > dpl && cpl > dpl)))
1779 /* mark segment as accessed */
1780 if (!(seg_desc.type & 1)) {
1782 ret = write_segment_descriptor(ctxt, selector,
1784 if (ret != X86EMUL_CONTINUE)
1787 } else if (ctxt->mode == X86EMUL_MODE_PROT64) {
1788 ret = linear_read_system(ctxt, desc_addr+8, &base3, sizeof(base3));
1789 if (ret != X86EMUL_CONTINUE)
1791 if (emul_is_noncanonical_address(get_desc_base(&seg_desc) |
1792 ((u64)base3 << 32), ctxt))
1793 return emulate_gp(ctxt, 0);
1796 ctxt->ops->set_segment(ctxt, selector, &seg_desc, base3, seg);
1799 return X86EMUL_CONTINUE;
1801 return emulate_exception(ctxt, err_vec, err_code, true);
1804 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1805 u16 selector, int seg)
1807 u8 cpl = ctxt->ops->cpl(ctxt);
1810 * None of MOV, POP and LSS can load a NULL selector in CPL=3, but
1811 * they can load it at CPL<3 (Intel's manual says only LSS can,
1814 * However, the Intel manual says that putting IST=1/DPL=3 in
1815 * an interrupt gate will result in SS=3 (the AMD manual instead
1816 * says it doesn't), so allow SS=3 in __load_segment_descriptor
1817 * and only forbid it here.
1819 if (seg == VCPU_SREG_SS && selector == 3 &&
1820 ctxt->mode == X86EMUL_MODE_PROT64)
1821 return emulate_exception(ctxt, GP_VECTOR, 0, true);
1823 return __load_segment_descriptor(ctxt, selector, seg, cpl,
1824 X86_TRANSFER_NONE, NULL);
1827 static void write_register_operand(struct operand *op)
1829 return assign_register(op->addr.reg, op->val, op->bytes);
1832 static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
1836 write_register_operand(op);
1839 if (ctxt->lock_prefix)
1840 return segmented_cmpxchg(ctxt,
1846 return segmented_write(ctxt,
1852 return segmented_write(ctxt,
1855 op->bytes * op->count);
1858 write_sse_reg(&op->vec_val, op->addr.xmm);
1861 write_mmx_reg(&op->mm_val, op->addr.mm);
1869 return X86EMUL_CONTINUE;
1872 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1874 struct segmented_address addr;
1876 rsp_increment(ctxt, -bytes);
1877 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1878 addr.seg = VCPU_SREG_SS;
1880 return segmented_write(ctxt, addr, data, bytes);
1883 static int em_push(struct x86_emulate_ctxt *ctxt)
1885 /* Disable writeback. */
1886 ctxt->dst.type = OP_NONE;
1887 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1890 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1891 void *dest, int len)
1894 struct segmented_address addr;
1896 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1897 addr.seg = VCPU_SREG_SS;
1898 rc = segmented_read(ctxt, addr, dest, len);
1899 if (rc != X86EMUL_CONTINUE)
1902 rsp_increment(ctxt, len);
1906 static int em_pop(struct x86_emulate_ctxt *ctxt)
1908 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1911 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1912 void *dest, int len)
1915 unsigned long val, change_mask;
1916 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
1917 int cpl = ctxt->ops->cpl(ctxt);
1919 rc = emulate_pop(ctxt, &val, len);
1920 if (rc != X86EMUL_CONTINUE)
1923 change_mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
1924 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF |
1925 X86_EFLAGS_TF | X86_EFLAGS_DF | X86_EFLAGS_NT |
1926 X86_EFLAGS_AC | X86_EFLAGS_ID;
1928 switch(ctxt->mode) {
1929 case X86EMUL_MODE_PROT64:
1930 case X86EMUL_MODE_PROT32:
1931 case X86EMUL_MODE_PROT16:
1933 change_mask |= X86_EFLAGS_IOPL;
1935 change_mask |= X86_EFLAGS_IF;
1937 case X86EMUL_MODE_VM86:
1939 return emulate_gp(ctxt, 0);
1940 change_mask |= X86_EFLAGS_IF;
1942 default: /* real mode */
1943 change_mask |= (X86_EFLAGS_IOPL | X86_EFLAGS_IF);
1947 *(unsigned long *)dest =
1948 (ctxt->eflags & ~change_mask) | (val & change_mask);
1953 static int em_popf(struct x86_emulate_ctxt *ctxt)
1955 ctxt->dst.type = OP_REG;
1956 ctxt->dst.addr.reg = &ctxt->eflags;
1957 ctxt->dst.bytes = ctxt->op_bytes;
1958 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1961 static int em_enter(struct x86_emulate_ctxt *ctxt)
1964 unsigned frame_size = ctxt->src.val;
1965 unsigned nesting_level = ctxt->src2.val & 31;
1969 return X86EMUL_UNHANDLEABLE;
1971 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1972 rc = push(ctxt, &rbp, stack_size(ctxt));
1973 if (rc != X86EMUL_CONTINUE)
1975 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1977 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1978 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1980 return X86EMUL_CONTINUE;
1983 static int em_leave(struct x86_emulate_ctxt *ctxt)
1985 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1987 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1990 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1992 int seg = ctxt->src2.val;
1994 ctxt->src.val = get_segment_selector(ctxt, seg);
1995 if (ctxt->op_bytes == 4) {
1996 rsp_increment(ctxt, -2);
2000 return em_push(ctxt);
2003 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
2005 int seg = ctxt->src2.val;
2006 unsigned long selector;
2009 rc = emulate_pop(ctxt, &selector, 2);
2010 if (rc != X86EMUL_CONTINUE)
2013 if (ctxt->modrm_reg == VCPU_SREG_SS)
2014 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
2015 if (ctxt->op_bytes > 2)
2016 rsp_increment(ctxt, ctxt->op_bytes - 2);
2018 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
2022 static int em_pusha(struct x86_emulate_ctxt *ctxt)
2024 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
2025 int rc = X86EMUL_CONTINUE;
2026 int reg = VCPU_REGS_RAX;
2028 while (reg <= VCPU_REGS_RDI) {
2029 (reg == VCPU_REGS_RSP) ?
2030 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
2033 if (rc != X86EMUL_CONTINUE)
2042 static int em_pushf(struct x86_emulate_ctxt *ctxt)
2044 ctxt->src.val = (unsigned long)ctxt->eflags & ~X86_EFLAGS_VM;
2045 return em_push(ctxt);
2048 static int em_popa(struct x86_emulate_ctxt *ctxt)
2050 int rc = X86EMUL_CONTINUE;
2051 int reg = VCPU_REGS_RDI;
2054 while (reg >= VCPU_REGS_RAX) {
2055 if (reg == VCPU_REGS_RSP) {
2056 rsp_increment(ctxt, ctxt->op_bytes);
2060 rc = emulate_pop(ctxt, &val, ctxt->op_bytes);
2061 if (rc != X86EMUL_CONTINUE)
2063 assign_register(reg_rmw(ctxt, reg), val, ctxt->op_bytes);
2069 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
2071 const struct x86_emulate_ops *ops = ctxt->ops;
2078 /* TODO: Add limit checks */
2079 ctxt->src.val = ctxt->eflags;
2081 if (rc != X86EMUL_CONTINUE)
2084 ctxt->eflags &= ~(X86_EFLAGS_IF | X86_EFLAGS_TF | X86_EFLAGS_AC);
2086 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
2088 if (rc != X86EMUL_CONTINUE)
2091 ctxt->src.val = ctxt->_eip;
2093 if (rc != X86EMUL_CONTINUE)
2096 ops->get_idt(ctxt, &dt);
2098 eip_addr = dt.address + (irq << 2);
2099 cs_addr = dt.address + (irq << 2) + 2;
2101 rc = linear_read_system(ctxt, cs_addr, &cs, 2);
2102 if (rc != X86EMUL_CONTINUE)
2105 rc = linear_read_system(ctxt, eip_addr, &eip, 2);
2106 if (rc != X86EMUL_CONTINUE)
2109 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
2110 if (rc != X86EMUL_CONTINUE)
2118 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
2122 invalidate_registers(ctxt);
2123 rc = __emulate_int_real(ctxt, irq);
2124 if (rc == X86EMUL_CONTINUE)
2125 writeback_registers(ctxt);
2129 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
2131 switch(ctxt->mode) {
2132 case X86EMUL_MODE_REAL:
2133 return __emulate_int_real(ctxt, irq);
2134 case X86EMUL_MODE_VM86:
2135 case X86EMUL_MODE_PROT16:
2136 case X86EMUL_MODE_PROT32:
2137 case X86EMUL_MODE_PROT64:
2139 /* Protected mode interrupts unimplemented yet */
2140 return X86EMUL_UNHANDLEABLE;
2144 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
2146 int rc = X86EMUL_CONTINUE;
2147 unsigned long temp_eip = 0;
2148 unsigned long temp_eflags = 0;
2149 unsigned long cs = 0;
2150 unsigned long mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
2151 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_TF |
2152 X86_EFLAGS_IF | X86_EFLAGS_DF | X86_EFLAGS_OF |
2153 X86_EFLAGS_IOPL | X86_EFLAGS_NT | X86_EFLAGS_RF |
2154 X86_EFLAGS_AC | X86_EFLAGS_ID |
2156 unsigned long vm86_mask = X86_EFLAGS_VM | X86_EFLAGS_VIF |
2159 /* TODO: Add stack limit check */
2161 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
2163 if (rc != X86EMUL_CONTINUE)
2166 if (temp_eip & ~0xffff)
2167 return emulate_gp(ctxt, 0);
2169 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2171 if (rc != X86EMUL_CONTINUE)
2174 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
2176 if (rc != X86EMUL_CONTINUE)
2179 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2181 if (rc != X86EMUL_CONTINUE)
2184 ctxt->_eip = temp_eip;
2186 if (ctxt->op_bytes == 4)
2187 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
2188 else if (ctxt->op_bytes == 2) {
2189 ctxt->eflags &= ~0xffff;
2190 ctxt->eflags |= temp_eflags;
2193 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
2194 ctxt->eflags |= X86_EFLAGS_FIXED;
2195 ctxt->ops->set_nmi_mask(ctxt, false);
2200 static int em_iret(struct x86_emulate_ctxt *ctxt)
2202 switch(ctxt->mode) {
2203 case X86EMUL_MODE_REAL:
2204 return emulate_iret_real(ctxt);
2205 case X86EMUL_MODE_VM86:
2206 case X86EMUL_MODE_PROT16:
2207 case X86EMUL_MODE_PROT32:
2208 case X86EMUL_MODE_PROT64:
2210 /* iret from protected mode unimplemented yet */
2211 return X86EMUL_UNHANDLEABLE;
2215 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
2219 struct desc_struct new_desc;
2220 u8 cpl = ctxt->ops->cpl(ctxt);
2222 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2224 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
2225 X86_TRANSFER_CALL_JMP,
2227 if (rc != X86EMUL_CONTINUE)
2230 rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
2231 /* Error handling is not implemented. */
2232 if (rc != X86EMUL_CONTINUE)
2233 return X86EMUL_UNHANDLEABLE;
2238 static int em_jmp_abs(struct x86_emulate_ctxt *ctxt)
2240 return assign_eip_near(ctxt, ctxt->src.val);
2243 static int em_call_near_abs(struct x86_emulate_ctxt *ctxt)
2248 old_eip = ctxt->_eip;
2249 rc = assign_eip_near(ctxt, ctxt->src.val);
2250 if (rc != X86EMUL_CONTINUE)
2252 ctxt->src.val = old_eip;
2257 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2259 u64 old = ctxt->dst.orig_val64;
2261 if (ctxt->dst.bytes == 16)
2262 return X86EMUL_UNHANDLEABLE;
2264 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
2265 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2266 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2267 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2268 ctxt->eflags &= ~X86_EFLAGS_ZF;
2270 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2271 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2273 ctxt->eflags |= X86_EFLAGS_ZF;
2275 return X86EMUL_CONTINUE;
2278 static int em_ret(struct x86_emulate_ctxt *ctxt)
2283 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2284 if (rc != X86EMUL_CONTINUE)
2287 return assign_eip_near(ctxt, eip);
2290 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2293 unsigned long eip, cs;
2294 int cpl = ctxt->ops->cpl(ctxt);
2295 struct desc_struct new_desc;
2297 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2298 if (rc != X86EMUL_CONTINUE)
2300 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2301 if (rc != X86EMUL_CONTINUE)
2303 /* Outer-privilege level return is not implemented */
2304 if (ctxt->mode >= X86EMUL_MODE_PROT16 && (cs & 3) > cpl)
2305 return X86EMUL_UNHANDLEABLE;
2306 rc = __load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS, cpl,
2309 if (rc != X86EMUL_CONTINUE)
2311 rc = assign_eip_far(ctxt, eip, &new_desc);
2312 /* Error handling is not implemented. */
2313 if (rc != X86EMUL_CONTINUE)
2314 return X86EMUL_UNHANDLEABLE;
2319 static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt)
2323 rc = em_ret_far(ctxt);
2324 if (rc != X86EMUL_CONTINUE)
2326 rsp_increment(ctxt, ctxt->src.val);
2327 return X86EMUL_CONTINUE;
2330 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2332 /* Save real source value, then compare EAX against destination. */
2333 ctxt->dst.orig_val = ctxt->dst.val;
2334 ctxt->dst.val = reg_read(ctxt, VCPU_REGS_RAX);
2335 ctxt->src.orig_val = ctxt->src.val;
2336 ctxt->src.val = ctxt->dst.orig_val;
2337 fastop(ctxt, em_cmp);
2339 if (ctxt->eflags & X86_EFLAGS_ZF) {
2340 /* Success: write back to memory; no update of EAX */
2341 ctxt->src.type = OP_NONE;
2342 ctxt->dst.val = ctxt->src.orig_val;
2344 /* Failure: write the value we saw to EAX. */
2345 ctxt->src.type = OP_REG;
2346 ctxt->src.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2347 ctxt->src.val = ctxt->dst.orig_val;
2348 /* Create write-cycle to dest by writing the same value */
2349 ctxt->dst.val = ctxt->dst.orig_val;
2351 return X86EMUL_CONTINUE;
2354 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2356 int seg = ctxt->src2.val;
2360 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2362 rc = load_segment_descriptor(ctxt, sel, seg);
2363 if (rc != X86EMUL_CONTINUE)
2366 ctxt->dst.val = ctxt->src.val;
2370 static int emulator_has_longmode(struct x86_emulate_ctxt *ctxt)
2372 #ifdef CONFIG_X86_64
2373 return ctxt->ops->guest_has_long_mode(ctxt);
2379 static void rsm_set_desc_flags(struct desc_struct *desc, u32 flags)
2381 desc->g = (flags >> 23) & 1;
2382 desc->d = (flags >> 22) & 1;
2383 desc->l = (flags >> 21) & 1;
2384 desc->avl = (flags >> 20) & 1;
2385 desc->p = (flags >> 15) & 1;
2386 desc->dpl = (flags >> 13) & 3;
2387 desc->s = (flags >> 12) & 1;
2388 desc->type = (flags >> 8) & 15;
2391 static int rsm_load_seg_32(struct x86_emulate_ctxt *ctxt, const char *smstate,
2394 struct desc_struct desc;
2398 selector = GET_SMSTATE(u32, smstate, 0x7fa8 + n * 4);
2401 offset = 0x7f84 + n * 12;
2403 offset = 0x7f2c + (n - 3) * 12;
2405 set_desc_base(&desc, GET_SMSTATE(u32, smstate, offset + 8));
2406 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, offset + 4));
2407 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, offset));
2408 ctxt->ops->set_segment(ctxt, selector, &desc, 0, n);
2409 return X86EMUL_CONTINUE;
2412 #ifdef CONFIG_X86_64
2413 static int rsm_load_seg_64(struct x86_emulate_ctxt *ctxt, const char *smstate,
2416 struct desc_struct desc;
2421 offset = 0x7e00 + n * 16;
2423 selector = GET_SMSTATE(u16, smstate, offset);
2424 rsm_set_desc_flags(&desc, GET_SMSTATE(u16, smstate, offset + 2) << 8);
2425 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, offset + 4));
2426 set_desc_base(&desc, GET_SMSTATE(u32, smstate, offset + 8));
2427 base3 = GET_SMSTATE(u32, smstate, offset + 12);
2429 ctxt->ops->set_segment(ctxt, selector, &desc, base3, n);
2430 return X86EMUL_CONTINUE;
2434 static int rsm_enter_protected_mode(struct x86_emulate_ctxt *ctxt,
2435 u64 cr0, u64 cr3, u64 cr4)
2440 /* In order to later set CR4.PCIDE, CR3[11:0] must be zero. */
2442 if (cr4 & X86_CR4_PCIDE) {
2447 bad = ctxt->ops->set_cr(ctxt, 3, cr3);
2449 return X86EMUL_UNHANDLEABLE;
2452 * First enable PAE, long mode needs it before CR0.PG = 1 is set.
2453 * Then enable protected mode. However, PCID cannot be enabled
2454 * if EFER.LMA=0, so set it separately.
2456 bad = ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE);
2458 return X86EMUL_UNHANDLEABLE;
2460 bad = ctxt->ops->set_cr(ctxt, 0, cr0);
2462 return X86EMUL_UNHANDLEABLE;
2464 if (cr4 & X86_CR4_PCIDE) {
2465 bad = ctxt->ops->set_cr(ctxt, 4, cr4);
2467 return X86EMUL_UNHANDLEABLE;
2469 bad = ctxt->ops->set_cr(ctxt, 3, cr3 | pcid);
2471 return X86EMUL_UNHANDLEABLE;
2476 return X86EMUL_CONTINUE;
2479 static int rsm_load_state_32(struct x86_emulate_ctxt *ctxt,
2480 const char *smstate)
2482 struct desc_struct desc;
2485 u32 val, cr0, cr3, cr4;
2488 cr0 = GET_SMSTATE(u32, smstate, 0x7ffc);
2489 cr3 = GET_SMSTATE(u32, smstate, 0x7ff8);
2490 ctxt->eflags = GET_SMSTATE(u32, smstate, 0x7ff4) | X86_EFLAGS_FIXED;
2491 ctxt->_eip = GET_SMSTATE(u32, smstate, 0x7ff0);
2493 for (i = 0; i < 8; i++)
2494 *reg_write(ctxt, i) = GET_SMSTATE(u32, smstate, 0x7fd0 + i * 4);
2496 val = GET_SMSTATE(u32, smstate, 0x7fcc);
2497 ctxt->ops->set_dr(ctxt, 6, (val & DR6_VOLATILE) | DR6_FIXED_1);
2498 val = GET_SMSTATE(u32, smstate, 0x7fc8);
2499 ctxt->ops->set_dr(ctxt, 7, (val & DR7_VOLATILE) | DR7_FIXED_1);
2501 selector = GET_SMSTATE(u32, smstate, 0x7fc4);
2502 set_desc_base(&desc, GET_SMSTATE(u32, smstate, 0x7f64));
2503 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, 0x7f60));
2504 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, 0x7f5c));
2505 ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_TR);
2507 selector = GET_SMSTATE(u32, smstate, 0x7fc0);
2508 set_desc_base(&desc, GET_SMSTATE(u32, smstate, 0x7f80));
2509 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, 0x7f7c));
2510 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, 0x7f78));
2511 ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_LDTR);
2513 dt.address = GET_SMSTATE(u32, smstate, 0x7f74);
2514 dt.size = GET_SMSTATE(u32, smstate, 0x7f70);
2515 ctxt->ops->set_gdt(ctxt, &dt);
2517 dt.address = GET_SMSTATE(u32, smstate, 0x7f58);
2518 dt.size = GET_SMSTATE(u32, smstate, 0x7f54);
2519 ctxt->ops->set_idt(ctxt, &dt);
2521 for (i = 0; i < 6; i++) {
2522 int r = rsm_load_seg_32(ctxt, smstate, i);
2523 if (r != X86EMUL_CONTINUE)
2527 cr4 = GET_SMSTATE(u32, smstate, 0x7f14);
2529 ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smstate, 0x7ef8));
2531 return rsm_enter_protected_mode(ctxt, cr0, cr3, cr4);
2534 #ifdef CONFIG_X86_64
2535 static int rsm_load_state_64(struct x86_emulate_ctxt *ctxt,
2536 const char *smstate)
2538 struct desc_struct desc;
2540 u64 val, cr0, cr3, cr4;
2545 for (i = 0; i < 16; i++)
2546 *reg_write(ctxt, i) = GET_SMSTATE(u64, smstate, 0x7ff8 - i * 8);
2548 ctxt->_eip = GET_SMSTATE(u64, smstate, 0x7f78);
2549 ctxt->eflags = GET_SMSTATE(u32, smstate, 0x7f70) | X86_EFLAGS_FIXED;
2551 val = GET_SMSTATE(u32, smstate, 0x7f68);
2552 ctxt->ops->set_dr(ctxt, 6, (val & DR6_VOLATILE) | DR6_FIXED_1);
2553 val = GET_SMSTATE(u32, smstate, 0x7f60);
2554 ctxt->ops->set_dr(ctxt, 7, (val & DR7_VOLATILE) | DR7_FIXED_1);
2556 cr0 = GET_SMSTATE(u64, smstate, 0x7f58);
2557 cr3 = GET_SMSTATE(u64, smstate, 0x7f50);
2558 cr4 = GET_SMSTATE(u64, smstate, 0x7f48);
2559 ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smstate, 0x7f00));
2560 val = GET_SMSTATE(u64, smstate, 0x7ed0);
2561 ctxt->ops->set_msr(ctxt, MSR_EFER, val & ~EFER_LMA);
2563 selector = GET_SMSTATE(u32, smstate, 0x7e90);
2564 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, 0x7e92) << 8);
2565 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, 0x7e94));
2566 set_desc_base(&desc, GET_SMSTATE(u32, smstate, 0x7e98));
2567 base3 = GET_SMSTATE(u32, smstate, 0x7e9c);
2568 ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_TR);
2570 dt.size = GET_SMSTATE(u32, smstate, 0x7e84);
2571 dt.address = GET_SMSTATE(u64, smstate, 0x7e88);
2572 ctxt->ops->set_idt(ctxt, &dt);
2574 selector = GET_SMSTATE(u32, smstate, 0x7e70);
2575 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, 0x7e72) << 8);
2576 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, 0x7e74));
2577 set_desc_base(&desc, GET_SMSTATE(u32, smstate, 0x7e78));
2578 base3 = GET_SMSTATE(u32, smstate, 0x7e7c);
2579 ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_LDTR);
2581 dt.size = GET_SMSTATE(u32, smstate, 0x7e64);
2582 dt.address = GET_SMSTATE(u64, smstate, 0x7e68);
2583 ctxt->ops->set_gdt(ctxt, &dt);
2585 r = rsm_enter_protected_mode(ctxt, cr0, cr3, cr4);
2586 if (r != X86EMUL_CONTINUE)
2589 for (i = 0; i < 6; i++) {
2590 r = rsm_load_seg_64(ctxt, smstate, i);
2591 if (r != X86EMUL_CONTINUE)
2595 return X86EMUL_CONTINUE;
2599 static int em_rsm(struct x86_emulate_ctxt *ctxt)
2601 unsigned long cr0, cr4, efer;
2606 if ((ctxt->ops->get_hflags(ctxt) & X86EMUL_SMM_MASK) == 0)
2607 return emulate_ud(ctxt);
2609 smbase = ctxt->ops->get_smbase(ctxt);
2611 ret = ctxt->ops->read_phys(ctxt, smbase + 0xfe00, buf, sizeof(buf));
2612 if (ret != X86EMUL_CONTINUE)
2613 return X86EMUL_UNHANDLEABLE;
2615 if ((ctxt->ops->get_hflags(ctxt) & X86EMUL_SMM_INSIDE_NMI_MASK) == 0)
2616 ctxt->ops->set_nmi_mask(ctxt, false);
2618 ctxt->ops->set_hflags(ctxt, ctxt->ops->get_hflags(ctxt) &
2619 ~(X86EMUL_SMM_INSIDE_NMI_MASK | X86EMUL_SMM_MASK));
2622 * Get back to real mode, to prepare a safe state in which to load
2623 * CR0/CR3/CR4/EFER. It's all a bit more complicated if the vCPU
2624 * supports long mode.
2626 if (emulator_has_longmode(ctxt)) {
2627 struct desc_struct cs_desc;
2629 /* Zero CR4.PCIDE before CR0.PG. */
2630 cr4 = ctxt->ops->get_cr(ctxt, 4);
2631 if (cr4 & X86_CR4_PCIDE)
2632 ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE);
2634 /* A 32-bit code segment is required to clear EFER.LMA. */
2635 memset(&cs_desc, 0, sizeof(cs_desc));
2637 cs_desc.s = cs_desc.g = cs_desc.p = 1;
2638 ctxt->ops->set_segment(ctxt, 0, &cs_desc, 0, VCPU_SREG_CS);
2641 /* For the 64-bit case, this will clear EFER.LMA. */
2642 cr0 = ctxt->ops->get_cr(ctxt, 0);
2643 if (cr0 & X86_CR0_PE)
2644 ctxt->ops->set_cr(ctxt, 0, cr0 & ~(X86_CR0_PG | X86_CR0_PE));
2646 if (emulator_has_longmode(ctxt)) {
2647 /* Clear CR4.PAE before clearing EFER.LME. */
2648 cr4 = ctxt->ops->get_cr(ctxt, 4);
2649 if (cr4 & X86_CR4_PAE)
2650 ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PAE);
2652 /* And finally go back to 32-bit mode. */
2654 ctxt->ops->set_msr(ctxt, MSR_EFER, efer);
2658 * Give pre_leave_smm() a chance to make ISA-specific changes to the
2659 * vCPU state (e.g. enter guest mode) before loading state from the SMM
2662 if (ctxt->ops->pre_leave_smm(ctxt, buf))
2663 return X86EMUL_UNHANDLEABLE;
2665 #ifdef CONFIG_X86_64
2666 if (emulator_has_longmode(ctxt))
2667 ret = rsm_load_state_64(ctxt, buf);
2670 ret = rsm_load_state_32(ctxt, buf);
2672 if (ret != X86EMUL_CONTINUE) {
2673 /* FIXME: should triple fault */
2674 return X86EMUL_UNHANDLEABLE;
2677 ctxt->ops->post_leave_smm(ctxt);
2679 return X86EMUL_CONTINUE;
2683 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2684 struct desc_struct *cs, struct desc_struct *ss)
2686 cs->l = 0; /* will be adjusted later */
2687 set_desc_base(cs, 0); /* flat segment */
2688 cs->g = 1; /* 4kb granularity */
2689 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2690 cs->type = 0x0b; /* Read, Execute, Accessed */
2692 cs->dpl = 0; /* will be adjusted later */
2697 set_desc_base(ss, 0); /* flat segment */
2698 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2699 ss->g = 1; /* 4kb granularity */
2701 ss->type = 0x03; /* Read/Write, Accessed */
2702 ss->d = 1; /* 32bit stack segment */
2709 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2711 u32 eax, ebx, ecx, edx;
2714 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
2715 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2716 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2717 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2720 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2722 const struct x86_emulate_ops *ops = ctxt->ops;
2723 u32 eax, ebx, ecx, edx;
2726 * syscall should always be enabled in longmode - so only become
2727 * vendor specific (cpuid) if other modes are active...
2729 if (ctxt->mode == X86EMUL_MODE_PROT64)
2734 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
2736 * Intel ("GenuineIntel")
2737 * remark: Intel CPUs only support "syscall" in 64bit
2738 * longmode. Also an 64bit guest with a
2739 * 32bit compat-app running will #UD !! While this
2740 * behaviour can be fixed (by emulating) into AMD
2741 * response - CPUs of AMD can't behave like Intel.
2743 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2744 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2745 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2748 /* AMD ("AuthenticAMD") */
2749 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2750 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2751 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2754 /* AMD ("AMDisbetter!") */
2755 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2756 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2757 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2760 /* Hygon ("HygonGenuine") */
2761 if (ebx == X86EMUL_CPUID_VENDOR_HygonGenuine_ebx &&
2762 ecx == X86EMUL_CPUID_VENDOR_HygonGenuine_ecx &&
2763 edx == X86EMUL_CPUID_VENDOR_HygonGenuine_edx)
2767 * default: (not Intel, not AMD, not Hygon), apply Intel's
2773 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2775 const struct x86_emulate_ops *ops = ctxt->ops;
2776 struct desc_struct cs, ss;
2781 /* syscall is not available in real mode */
2782 if (ctxt->mode == X86EMUL_MODE_REAL ||
2783 ctxt->mode == X86EMUL_MODE_VM86)
2784 return emulate_ud(ctxt);
2786 if (!(em_syscall_is_enabled(ctxt)))
2787 return emulate_ud(ctxt);
2789 ops->get_msr(ctxt, MSR_EFER, &efer);
2790 if (!(efer & EFER_SCE))
2791 return emulate_ud(ctxt);
2793 setup_syscalls_segments(ctxt, &cs, &ss);
2794 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2796 cs_sel = (u16)(msr_data & 0xfffc);
2797 ss_sel = (u16)(msr_data + 8);
2799 if (efer & EFER_LMA) {
2803 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2804 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2806 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2807 if (efer & EFER_LMA) {
2808 #ifdef CONFIG_X86_64
2809 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags;
2812 ctxt->mode == X86EMUL_MODE_PROT64 ?
2813 MSR_LSTAR : MSR_CSTAR, &msr_data);
2814 ctxt->_eip = msr_data;
2816 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2817 ctxt->eflags &= ~msr_data;
2818 ctxt->eflags |= X86_EFLAGS_FIXED;
2822 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2823 ctxt->_eip = (u32)msr_data;
2825 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2828 ctxt->tf = (ctxt->eflags & X86_EFLAGS_TF) != 0;
2829 return X86EMUL_CONTINUE;
2832 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2834 const struct x86_emulate_ops *ops = ctxt->ops;
2835 struct desc_struct cs, ss;
2840 ops->get_msr(ctxt, MSR_EFER, &efer);
2841 /* inject #GP if in real mode */
2842 if (ctxt->mode == X86EMUL_MODE_REAL)
2843 return emulate_gp(ctxt, 0);
2846 * Not recognized on AMD in compat mode (but is recognized in legacy
2849 if ((ctxt->mode != X86EMUL_MODE_PROT64) && (efer & EFER_LMA)
2850 && !vendor_intel(ctxt))
2851 return emulate_ud(ctxt);
2853 /* sysenter/sysexit have not been tested in 64bit mode. */
2854 if (ctxt->mode == X86EMUL_MODE_PROT64)
2855 return X86EMUL_UNHANDLEABLE;
2857 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2858 if ((msr_data & 0xfffc) == 0x0)
2859 return emulate_gp(ctxt, 0);
2861 setup_syscalls_segments(ctxt, &cs, &ss);
2862 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2863 cs_sel = (u16)msr_data & ~SEGMENT_RPL_MASK;
2864 ss_sel = cs_sel + 8;
2865 if (efer & EFER_LMA) {
2870 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2871 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2873 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2874 ctxt->_eip = (efer & EFER_LMA) ? msr_data : (u32)msr_data;
2876 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2877 *reg_write(ctxt, VCPU_REGS_RSP) = (efer & EFER_LMA) ? msr_data :
2880 return X86EMUL_CONTINUE;
2883 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2885 const struct x86_emulate_ops *ops = ctxt->ops;
2886 struct desc_struct cs, ss;
2887 u64 msr_data, rcx, rdx;
2889 u16 cs_sel = 0, ss_sel = 0;
2891 /* inject #GP if in real mode or Virtual 8086 mode */
2892 if (ctxt->mode == X86EMUL_MODE_REAL ||
2893 ctxt->mode == X86EMUL_MODE_VM86)
2894 return emulate_gp(ctxt, 0);
2896 setup_syscalls_segments(ctxt, &cs, &ss);
2898 if ((ctxt->rex_prefix & 0x8) != 0x0)
2899 usermode = X86EMUL_MODE_PROT64;
2901 usermode = X86EMUL_MODE_PROT32;
2903 rcx = reg_read(ctxt, VCPU_REGS_RCX);
2904 rdx = reg_read(ctxt, VCPU_REGS_RDX);
2908 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2910 case X86EMUL_MODE_PROT32:
2911 cs_sel = (u16)(msr_data + 16);
2912 if ((msr_data & 0xfffc) == 0x0)
2913 return emulate_gp(ctxt, 0);
2914 ss_sel = (u16)(msr_data + 24);
2918 case X86EMUL_MODE_PROT64:
2919 cs_sel = (u16)(msr_data + 32);
2920 if (msr_data == 0x0)
2921 return emulate_gp(ctxt, 0);
2922 ss_sel = cs_sel + 8;
2925 if (emul_is_noncanonical_address(rcx, ctxt) ||
2926 emul_is_noncanonical_address(rdx, ctxt))
2927 return emulate_gp(ctxt, 0);
2930 cs_sel |= SEGMENT_RPL_MASK;
2931 ss_sel |= SEGMENT_RPL_MASK;
2933 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2934 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2937 *reg_write(ctxt, VCPU_REGS_RSP) = rcx;
2939 return X86EMUL_CONTINUE;
2942 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2945 if (ctxt->mode == X86EMUL_MODE_REAL)
2947 if (ctxt->mode == X86EMUL_MODE_VM86)
2949 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
2950 return ctxt->ops->cpl(ctxt) > iopl;
2953 #define VMWARE_PORT_VMPORT (0x5658)
2954 #define VMWARE_PORT_VMRPC (0x5659)
2956 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2959 const struct x86_emulate_ops *ops = ctxt->ops;
2960 struct desc_struct tr_seg;
2963 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2964 unsigned mask = (1 << len) - 1;
2968 * VMware allows access to these ports even if denied
2969 * by TSS I/O permission bitmap. Mimic behavior.
2971 if (enable_vmware_backdoor &&
2972 ((port == VMWARE_PORT_VMPORT) || (port == VMWARE_PORT_VMRPC)))
2975 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2978 if (desc_limit_scaled(&tr_seg) < 103)
2980 base = get_desc_base(&tr_seg);
2981 #ifdef CONFIG_X86_64
2982 base |= ((u64)base3) << 32;
2984 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL, true);
2985 if (r != X86EMUL_CONTINUE)
2987 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2989 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL, true);
2990 if (r != X86EMUL_CONTINUE)
2992 if ((perm >> bit_idx) & mask)
2997 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
3003 if (emulator_bad_iopl(ctxt))
3004 if (!emulator_io_port_access_allowed(ctxt, port, len))
3007 ctxt->perm_ok = true;
3012 static void string_registers_quirk(struct x86_emulate_ctxt *ctxt)
3015 * Intel CPUs mask the counter and pointers in quite strange
3016 * manner when ECX is zero due to REP-string optimizations.
3018 #ifdef CONFIG_X86_64
3019 if (ctxt->ad_bytes != 4 || !vendor_intel(ctxt))
3022 *reg_write(ctxt, VCPU_REGS_RCX) = 0;
3025 case 0xa4: /* movsb */
3026 case 0xa5: /* movsd/w */
3027 *reg_rmw(ctxt, VCPU_REGS_RSI) &= (u32)-1;
3029 case 0xaa: /* stosb */
3030 case 0xab: /* stosd/w */
3031 *reg_rmw(ctxt, VCPU_REGS_RDI) &= (u32)-1;
3036 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
3037 struct tss_segment_16 *tss)
3039 tss->ip = ctxt->_eip;
3040 tss->flag = ctxt->eflags;
3041 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
3042 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
3043 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
3044 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
3045 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
3046 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
3047 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
3048 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
3050 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
3051 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
3052 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
3053 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
3054 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
3057 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
3058 struct tss_segment_16 *tss)
3063 ctxt->_eip = tss->ip;
3064 ctxt->eflags = tss->flag | 2;
3065 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
3066 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
3067 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
3068 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
3069 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
3070 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
3071 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
3072 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
3075 * SDM says that segment selectors are loaded before segment
3078 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
3079 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
3080 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
3081 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
3082 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
3087 * Now load segment descriptors. If fault happens at this stage
3088 * it is handled in a context of new task
3090 ret = __load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR, cpl,
3091 X86_TRANSFER_TASK_SWITCH, NULL);
3092 if (ret != X86EMUL_CONTINUE)
3094 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
3095 X86_TRANSFER_TASK_SWITCH, NULL);
3096 if (ret != X86EMUL_CONTINUE)
3098 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
3099 X86_TRANSFER_TASK_SWITCH, NULL);
3100 if (ret != X86EMUL_CONTINUE)
3102 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
3103 X86_TRANSFER_TASK_SWITCH, NULL);
3104 if (ret != X86EMUL_CONTINUE)
3106 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
3107 X86_TRANSFER_TASK_SWITCH, NULL);
3108 if (ret != X86EMUL_CONTINUE)
3111 return X86EMUL_CONTINUE;
3114 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
3115 u16 tss_selector, u16 old_tss_sel,
3116 ulong old_tss_base, struct desc_struct *new_desc)
3118 struct tss_segment_16 tss_seg;
3120 u32 new_tss_base = get_desc_base(new_desc);
3122 ret = linear_read_system(ctxt, old_tss_base, &tss_seg, sizeof(tss_seg));
3123 if (ret != X86EMUL_CONTINUE)
3126 save_state_to_tss16(ctxt, &tss_seg);
3128 ret = linear_write_system(ctxt, old_tss_base, &tss_seg, sizeof(tss_seg));
3129 if (ret != X86EMUL_CONTINUE)
3132 ret = linear_read_system(ctxt, new_tss_base, &tss_seg, sizeof(tss_seg));
3133 if (ret != X86EMUL_CONTINUE)
3136 if (old_tss_sel != 0xffff) {
3137 tss_seg.prev_task_link = old_tss_sel;
3139 ret = linear_write_system(ctxt, new_tss_base,
3140 &tss_seg.prev_task_link,
3141 sizeof(tss_seg.prev_task_link));
3142 if (ret != X86EMUL_CONTINUE)
3146 return load_state_from_tss16(ctxt, &tss_seg);
3149 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
3150 struct tss_segment_32 *tss)
3152 /* CR3 and ldt selector are not saved intentionally */
3153 tss->eip = ctxt->_eip;
3154 tss->eflags = ctxt->eflags;
3155 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
3156 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
3157 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
3158 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
3159 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
3160 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
3161 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
3162 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
3164 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
3165 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
3166 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
3167 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
3168 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
3169 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
3172 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
3173 struct tss_segment_32 *tss)
3178 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
3179 return emulate_gp(ctxt, 0);
3180 ctxt->_eip = tss->eip;
3181 ctxt->eflags = tss->eflags | 2;
3183 /* General purpose registers */
3184 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
3185 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
3186 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
3187 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
3188 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
3189 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
3190 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
3191 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
3194 * SDM says that segment selectors are loaded before segment
3195 * descriptors. This is important because CPL checks will
3198 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
3199 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
3200 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
3201 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
3202 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
3203 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
3204 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
3207 * If we're switching between Protected Mode and VM86, we need to make
3208 * sure to update the mode before loading the segment descriptors so
3209 * that the selectors are interpreted correctly.
3211 if (ctxt->eflags & X86_EFLAGS_VM) {
3212 ctxt->mode = X86EMUL_MODE_VM86;
3215 ctxt->mode = X86EMUL_MODE_PROT32;
3220 * Now load segment descriptors. If fault happenes at this stage
3221 * it is handled in a context of new task
3223 ret = __load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR,
3224 cpl, X86_TRANSFER_TASK_SWITCH, NULL);
3225 if (ret != X86EMUL_CONTINUE)
3227 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
3228 X86_TRANSFER_TASK_SWITCH, NULL);
3229 if (ret != X86EMUL_CONTINUE)
3231 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
3232 X86_TRANSFER_TASK_SWITCH, NULL);
3233 if (ret != X86EMUL_CONTINUE)
3235 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
3236 X86_TRANSFER_TASK_SWITCH, NULL);
3237 if (ret != X86EMUL_CONTINUE)
3239 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
3240 X86_TRANSFER_TASK_SWITCH, NULL);
3241 if (ret != X86EMUL_CONTINUE)
3243 ret = __load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS, cpl,
3244 X86_TRANSFER_TASK_SWITCH, NULL);
3245 if (ret != X86EMUL_CONTINUE)
3247 ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl,
3248 X86_TRANSFER_TASK_SWITCH, NULL);
3253 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
3254 u16 tss_selector, u16 old_tss_sel,
3255 ulong old_tss_base, struct desc_struct *new_desc)
3257 struct tss_segment_32 tss_seg;
3259 u32 new_tss_base = get_desc_base(new_desc);
3260 u32 eip_offset = offsetof(struct tss_segment_32, eip);
3261 u32 ldt_sel_offset = offsetof(struct tss_segment_32, ldt_selector);
3263 ret = linear_read_system(ctxt, old_tss_base, &tss_seg, sizeof(tss_seg));
3264 if (ret != X86EMUL_CONTINUE)
3267 save_state_to_tss32(ctxt, &tss_seg);
3269 /* Only GP registers and segment selectors are saved */
3270 ret = linear_write_system(ctxt, old_tss_base + eip_offset, &tss_seg.eip,
3271 ldt_sel_offset - eip_offset);
3272 if (ret != X86EMUL_CONTINUE)
3275 ret = linear_read_system(ctxt, new_tss_base, &tss_seg, sizeof(tss_seg));
3276 if (ret != X86EMUL_CONTINUE)
3279 if (old_tss_sel != 0xffff) {
3280 tss_seg.prev_task_link = old_tss_sel;
3282 ret = linear_write_system(ctxt, new_tss_base,
3283 &tss_seg.prev_task_link,
3284 sizeof(tss_seg.prev_task_link));
3285 if (ret != X86EMUL_CONTINUE)
3289 return load_state_from_tss32(ctxt, &tss_seg);
3292 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
3293 u16 tss_selector, int idt_index, int reason,
3294 bool has_error_code, u32 error_code)
3296 const struct x86_emulate_ops *ops = ctxt->ops;
3297 struct desc_struct curr_tss_desc, next_tss_desc;
3299 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
3300 ulong old_tss_base =
3301 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
3303 ulong desc_addr, dr7;
3305 /* FIXME: old_tss_base == ~0 ? */
3307 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
3308 if (ret != X86EMUL_CONTINUE)
3310 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
3311 if (ret != X86EMUL_CONTINUE)
3314 /* FIXME: check that next_tss_desc is tss */
3317 * Check privileges. The three cases are task switch caused by...
3319 * 1. jmp/call/int to task gate: Check against DPL of the task gate
3320 * 2. Exception/IRQ/iret: No check is performed
3321 * 3. jmp/call to TSS/task-gate: No check is performed since the
3322 * hardware checks it before exiting.
3324 if (reason == TASK_SWITCH_GATE) {
3325 if (idt_index != -1) {
3326 /* Software interrupts */
3327 struct desc_struct task_gate_desc;
3330 ret = read_interrupt_descriptor(ctxt, idt_index,
3332 if (ret != X86EMUL_CONTINUE)
3335 dpl = task_gate_desc.dpl;
3336 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
3337 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
3341 desc_limit = desc_limit_scaled(&next_tss_desc);
3342 if (!next_tss_desc.p ||
3343 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
3344 desc_limit < 0x2b)) {
3345 return emulate_ts(ctxt, tss_selector & 0xfffc);
3348 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
3349 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
3350 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
3353 if (reason == TASK_SWITCH_IRET)
3354 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
3356 /* set back link to prev task only if NT bit is set in eflags
3357 note that old_tss_sel is not used after this point */
3358 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
3359 old_tss_sel = 0xffff;
3361 if (next_tss_desc.type & 8)
3362 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
3363 old_tss_base, &next_tss_desc);
3365 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
3366 old_tss_base, &next_tss_desc);
3367 if (ret != X86EMUL_CONTINUE)
3370 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
3371 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
3373 if (reason != TASK_SWITCH_IRET) {
3374 next_tss_desc.type |= (1 << 1); /* set busy flag */
3375 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
3378 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
3379 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
3381 if (has_error_code) {
3382 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
3383 ctxt->lock_prefix = 0;
3384 ctxt->src.val = (unsigned long) error_code;
3385 ret = em_push(ctxt);
3388 ops->get_dr(ctxt, 7, &dr7);
3389 ops->set_dr(ctxt, 7, dr7 & ~(DR_LOCAL_ENABLE_MASK | DR_LOCAL_SLOWDOWN));
3394 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
3395 u16 tss_selector, int idt_index, int reason,
3396 bool has_error_code, u32 error_code)
3400 invalidate_registers(ctxt);
3401 ctxt->_eip = ctxt->eip;
3402 ctxt->dst.type = OP_NONE;
3404 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
3405 has_error_code, error_code);
3407 if (rc == X86EMUL_CONTINUE) {
3408 ctxt->eip = ctxt->_eip;
3409 writeback_registers(ctxt);
3412 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
3415 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
3418 int df = (ctxt->eflags & X86_EFLAGS_DF) ? -op->count : op->count;
3420 register_address_increment(ctxt, reg, df * op->bytes);
3421 op->addr.mem.ea = register_address(ctxt, reg);
3424 static int em_das(struct x86_emulate_ctxt *ctxt)
3427 bool af, cf, old_cf;
3429 cf = ctxt->eflags & X86_EFLAGS_CF;
3435 af = ctxt->eflags & X86_EFLAGS_AF;
3436 if ((al & 0x0f) > 9 || af) {
3438 cf = old_cf | (al >= 250);
3443 if (old_al > 0x99 || old_cf) {
3449 /* Set PF, ZF, SF */
3450 ctxt->src.type = OP_IMM;
3452 ctxt->src.bytes = 1;
3453 fastop(ctxt, em_or);
3454 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
3456 ctxt->eflags |= X86_EFLAGS_CF;
3458 ctxt->eflags |= X86_EFLAGS_AF;
3459 return X86EMUL_CONTINUE;
3462 static int em_aam(struct x86_emulate_ctxt *ctxt)
3466 if (ctxt->src.val == 0)
3467 return emulate_de(ctxt);
3469 al = ctxt->dst.val & 0xff;
3470 ah = al / ctxt->src.val;
3471 al %= ctxt->src.val;
3473 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);
3475 /* Set PF, ZF, SF */
3476 ctxt->src.type = OP_IMM;
3478 ctxt->src.bytes = 1;
3479 fastop(ctxt, em_or);
3481 return X86EMUL_CONTINUE;
3484 static int em_aad(struct x86_emulate_ctxt *ctxt)
3486 u8 al = ctxt->dst.val & 0xff;
3487 u8 ah = (ctxt->dst.val >> 8) & 0xff;
3489 al = (al + (ah * ctxt->src.val)) & 0xff;
3491 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
3493 /* Set PF, ZF, SF */
3494 ctxt->src.type = OP_IMM;
3496 ctxt->src.bytes = 1;
3497 fastop(ctxt, em_or);
3499 return X86EMUL_CONTINUE;
3502 static int em_call(struct x86_emulate_ctxt *ctxt)
3505 long rel = ctxt->src.val;
3507 ctxt->src.val = (unsigned long)ctxt->_eip;
3508 rc = jmp_rel(ctxt, rel);
3509 if (rc != X86EMUL_CONTINUE)
3511 return em_push(ctxt);
3514 static int em_call_far(struct x86_emulate_ctxt *ctxt)
3519 struct desc_struct old_desc, new_desc;
3520 const struct x86_emulate_ops *ops = ctxt->ops;
3521 int cpl = ctxt->ops->cpl(ctxt);
3522 enum x86emul_mode prev_mode = ctxt->mode;
3524 old_eip = ctxt->_eip;
3525 ops->get_segment(ctxt, &old_cs, &old_desc, NULL, VCPU_SREG_CS);
3527 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
3528 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
3529 X86_TRANSFER_CALL_JMP, &new_desc);
3530 if (rc != X86EMUL_CONTINUE)
3533 rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
3534 if (rc != X86EMUL_CONTINUE)
3537 ctxt->src.val = old_cs;
3539 if (rc != X86EMUL_CONTINUE)
3542 ctxt->src.val = old_eip;
3544 /* If we failed, we tainted the memory, but the very least we should
3546 if (rc != X86EMUL_CONTINUE) {
3547 pr_warn_once("faulting far call emulation tainted memory\n");
3552 ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS);
3553 ctxt->mode = prev_mode;
3558 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
3563 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
3564 if (rc != X86EMUL_CONTINUE)
3566 rc = assign_eip_near(ctxt, eip);
3567 if (rc != X86EMUL_CONTINUE)
3569 rsp_increment(ctxt, ctxt->src.val);
3570 return X86EMUL_CONTINUE;
3573 static int em_xchg(struct x86_emulate_ctxt *ctxt)
3575 /* Write back the register source. */
3576 ctxt->src.val = ctxt->dst.val;
3577 write_register_operand(&ctxt->src);
3579 /* Write back the memory destination with implicit LOCK prefix. */
3580 ctxt->dst.val = ctxt->src.orig_val;
3581 ctxt->lock_prefix = 1;
3582 return X86EMUL_CONTINUE;
3585 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
3587 ctxt->dst.val = ctxt->src2.val;
3588 return fastop(ctxt, em_imul);
3591 static int em_cwd(struct x86_emulate_ctxt *ctxt)
3593 ctxt->dst.type = OP_REG;
3594 ctxt->dst.bytes = ctxt->src.bytes;
3595 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
3596 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
3598 return X86EMUL_CONTINUE;
3601 static int em_rdpid(struct x86_emulate_ctxt *ctxt)
3605 if (ctxt->ops->get_msr(ctxt, MSR_TSC_AUX, &tsc_aux))
3606 return emulate_gp(ctxt, 0);
3607 ctxt->dst.val = tsc_aux;
3608 return X86EMUL_CONTINUE;
3611 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
3615 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
3616 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
3617 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
3618 return X86EMUL_CONTINUE;
3621 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
3625 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
3626 return emulate_gp(ctxt, 0);
3627 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
3628 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
3629 return X86EMUL_CONTINUE;
3632 static int em_mov(struct x86_emulate_ctxt *ctxt)
3634 memcpy(ctxt->dst.valptr, ctxt->src.valptr, sizeof(ctxt->src.valptr));
3635 return X86EMUL_CONTINUE;
3638 static int em_movbe(struct x86_emulate_ctxt *ctxt)
3642 if (!ctxt->ops->guest_has_movbe(ctxt))
3643 return emulate_ud(ctxt);
3645 switch (ctxt->op_bytes) {
3648 * From MOVBE definition: "...When the operand size is 16 bits,
3649 * the upper word of the destination register remains unchanged
3652 * Both casting ->valptr and ->val to u16 breaks strict aliasing
3653 * rules so we have to do the operation almost per hand.
3655 tmp = (u16)ctxt->src.val;
3656 ctxt->dst.val &= ~0xffffUL;
3657 ctxt->dst.val |= (unsigned long)swab16(tmp);
3660 ctxt->dst.val = swab32((u32)ctxt->src.val);
3663 ctxt->dst.val = swab64(ctxt->src.val);
3668 return X86EMUL_CONTINUE;
3671 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3673 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3674 return emulate_gp(ctxt, 0);
3676 /* Disable writeback. */
3677 ctxt->dst.type = OP_NONE;
3678 return X86EMUL_CONTINUE;
3681 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3685 if (ctxt->mode == X86EMUL_MODE_PROT64)
3686 val = ctxt->src.val & ~0ULL;
3688 val = ctxt->src.val & ~0U;
3690 /* #UD condition is already handled. */
3691 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3692 return emulate_gp(ctxt, 0);
3694 /* Disable writeback. */
3695 ctxt->dst.type = OP_NONE;
3696 return X86EMUL_CONTINUE;
3699 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3703 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3704 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3705 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3706 return emulate_gp(ctxt, 0);
3708 return X86EMUL_CONTINUE;
3711 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3715 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3716 return emulate_gp(ctxt, 0);
3718 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3719 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3720 return X86EMUL_CONTINUE;
3723 static int em_store_sreg(struct x86_emulate_ctxt *ctxt, int segment)
3725 if (segment > VCPU_SREG_GS &&
3726 (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) &&
3727 ctxt->ops->cpl(ctxt) > 0)
3728 return emulate_gp(ctxt, 0);
3730 ctxt->dst.val = get_segment_selector(ctxt, segment);
3731 if (ctxt->dst.bytes == 4 && ctxt->dst.type == OP_MEM)
3732 ctxt->dst.bytes = 2;
3733 return X86EMUL_CONTINUE;
3736 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3738 if (ctxt->modrm_reg > VCPU_SREG_GS)
3739 return emulate_ud(ctxt);
3741 return em_store_sreg(ctxt, ctxt->modrm_reg);
3744 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3746 u16 sel = ctxt->src.val;
3748 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3749 return emulate_ud(ctxt);
3751 if (ctxt->modrm_reg == VCPU_SREG_SS)
3752 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3754 /* Disable writeback. */
3755 ctxt->dst.type = OP_NONE;
3756 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3759 static int em_sldt(struct x86_emulate_ctxt *ctxt)
3761 return em_store_sreg(ctxt, VCPU_SREG_LDTR);
3764 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3766 u16 sel = ctxt->src.val;
3768 /* Disable writeback. */
3769 ctxt->dst.type = OP_NONE;
3770 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3773 static int em_str(struct x86_emulate_ctxt *ctxt)
3775 return em_store_sreg(ctxt, VCPU_SREG_TR);
3778 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3780 u16 sel = ctxt->src.val;
3782 /* Disable writeback. */
3783 ctxt->dst.type = OP_NONE;
3784 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3787 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3792 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3793 if (rc == X86EMUL_CONTINUE)
3794 ctxt->ops->invlpg(ctxt, linear);
3795 /* Disable writeback. */
3796 ctxt->dst.type = OP_NONE;
3797 return X86EMUL_CONTINUE;
3800 static int em_clts(struct x86_emulate_ctxt *ctxt)
3804 cr0 = ctxt->ops->get_cr(ctxt, 0);
3806 ctxt->ops->set_cr(ctxt, 0, cr0);
3807 return X86EMUL_CONTINUE;
3810 static int em_hypercall(struct x86_emulate_ctxt *ctxt)
3812 int rc = ctxt->ops->fix_hypercall(ctxt);
3814 if (rc != X86EMUL_CONTINUE)
3817 /* Let the processor re-execute the fixed hypercall */
3818 ctxt->_eip = ctxt->eip;
3819 /* Disable writeback. */
3820 ctxt->dst.type = OP_NONE;
3821 return X86EMUL_CONTINUE;
3824 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3825 void (*get)(struct x86_emulate_ctxt *ctxt,
3826 struct desc_ptr *ptr))
3828 struct desc_ptr desc_ptr;
3830 if ((ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) &&
3831 ctxt->ops->cpl(ctxt) > 0)
3832 return emulate_gp(ctxt, 0);
3834 if (ctxt->mode == X86EMUL_MODE_PROT64)
3836 get(ctxt, &desc_ptr);
3837 if (ctxt->op_bytes == 2) {
3839 desc_ptr.address &= 0x00ffffff;
3841 /* Disable writeback. */
3842 ctxt->dst.type = OP_NONE;
3843 return segmented_write_std(ctxt, ctxt->dst.addr.mem,
3844 &desc_ptr, 2 + ctxt->op_bytes);
3847 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3849 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3852 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3854 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3857 static int em_lgdt_lidt(struct x86_emulate_ctxt *ctxt, bool lgdt)
3859 struct desc_ptr desc_ptr;
3862 if (ctxt->mode == X86EMUL_MODE_PROT64)
3864 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3865 &desc_ptr.size, &desc_ptr.address,
3867 if (rc != X86EMUL_CONTINUE)
3869 if (ctxt->mode == X86EMUL_MODE_PROT64 &&
3870 emul_is_noncanonical_address(desc_ptr.address, ctxt))
3871 return emulate_gp(ctxt, 0);
3873 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3875 ctxt->ops->set_idt(ctxt, &desc_ptr);
3876 /* Disable writeback. */
3877 ctxt->dst.type = OP_NONE;
3878 return X86EMUL_CONTINUE;
3881 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3883 return em_lgdt_lidt(ctxt, true);
3886 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3888 return em_lgdt_lidt(ctxt, false);
3891 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3893 if ((ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) &&
3894 ctxt->ops->cpl(ctxt) > 0)
3895 return emulate_gp(ctxt, 0);
3897 if (ctxt->dst.type == OP_MEM)
3898 ctxt->dst.bytes = 2;
3899 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3900 return X86EMUL_CONTINUE;
3903 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3905 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3906 | (ctxt->src.val & 0x0f));
3907 ctxt->dst.type = OP_NONE;
3908 return X86EMUL_CONTINUE;
3911 static int em_loop(struct x86_emulate_ctxt *ctxt)
3913 int rc = X86EMUL_CONTINUE;
3915 register_address_increment(ctxt, VCPU_REGS_RCX, -1);
3916 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3917 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3918 rc = jmp_rel(ctxt, ctxt->src.val);
3923 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3925 int rc = X86EMUL_CONTINUE;
3927 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3928 rc = jmp_rel(ctxt, ctxt->src.val);
3933 static int em_in(struct x86_emulate_ctxt *ctxt)
3935 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3937 return X86EMUL_IO_NEEDED;
3939 return X86EMUL_CONTINUE;
3942 static int em_out(struct x86_emulate_ctxt *ctxt)
3944 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3946 /* Disable writeback. */
3947 ctxt->dst.type = OP_NONE;
3948 return X86EMUL_CONTINUE;
3951 static int em_cli(struct x86_emulate_ctxt *ctxt)
3953 if (emulator_bad_iopl(ctxt))
3954 return emulate_gp(ctxt, 0);
3956 ctxt->eflags &= ~X86_EFLAGS_IF;
3957 return X86EMUL_CONTINUE;
3960 static int em_sti(struct x86_emulate_ctxt *ctxt)
3962 if (emulator_bad_iopl(ctxt))
3963 return emulate_gp(ctxt, 0);
3965 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3966 ctxt->eflags |= X86_EFLAGS_IF;
3967 return X86EMUL_CONTINUE;
3970 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3972 u32 eax, ebx, ecx, edx;
3975 ctxt->ops->get_msr(ctxt, MSR_MISC_FEATURES_ENABLES, &msr);
3976 if (msr & MSR_MISC_FEATURES_ENABLES_CPUID_FAULT &&
3977 ctxt->ops->cpl(ctxt)) {
3978 return emulate_gp(ctxt, 0);
3981 eax = reg_read(ctxt, VCPU_REGS_RAX);
3982 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3983 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, true);
3984 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3985 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3986 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3987 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3988 return X86EMUL_CONTINUE;
3991 static int em_sahf(struct x86_emulate_ctxt *ctxt)
3995 flags = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF |
3997 flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8;
3999 ctxt->eflags &= ~0xffUL;
4000 ctxt->eflags |= flags | X86_EFLAGS_FIXED;
4001 return X86EMUL_CONTINUE;
4004 static int em_lahf(struct x86_emulate_ctxt *ctxt)
4006 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
4007 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
4008 return X86EMUL_CONTINUE;
4011 static int em_bswap(struct x86_emulate_ctxt *ctxt)
4013 switch (ctxt->op_bytes) {
4014 #ifdef CONFIG_X86_64
4016 asm("bswap %0" : "+r"(ctxt->dst.val));
4020 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
4023 return X86EMUL_CONTINUE;
4026 static int em_clflush(struct x86_emulate_ctxt *ctxt)
4028 /* emulating clflush regardless of cpuid */
4029 return X86EMUL_CONTINUE;
4032 static int em_movsxd(struct x86_emulate_ctxt *ctxt)
4034 ctxt->dst.val = (s32) ctxt->src.val;
4035 return X86EMUL_CONTINUE;
4038 static int check_fxsr(struct x86_emulate_ctxt *ctxt)
4040 if (!ctxt->ops->guest_has_fxsr(ctxt))
4041 return emulate_ud(ctxt);
4043 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
4044 return emulate_nm(ctxt);
4047 * Don't emulate a case that should never be hit, instead of working
4048 * around a lack of fxsave64/fxrstor64 on old compilers.
4050 if (ctxt->mode >= X86EMUL_MODE_PROT64)
4051 return X86EMUL_UNHANDLEABLE;
4053 return X86EMUL_CONTINUE;
4057 * Hardware doesn't save and restore XMM 0-7 without CR4.OSFXSR, but does save
4058 * and restore MXCSR.
4060 static size_t __fxstate_size(int nregs)
4062 return offsetof(struct fxregs_state, xmm_space[0]) + nregs * 16;
4065 static inline size_t fxstate_size(struct x86_emulate_ctxt *ctxt)
4068 if (ctxt->mode == X86EMUL_MODE_PROT64)
4069 return __fxstate_size(16);
4071 cr4_osfxsr = ctxt->ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR;
4072 return __fxstate_size(cr4_osfxsr ? 8 : 0);
4076 * FXSAVE and FXRSTOR have 4 different formats depending on execution mode,
4079 * - like (1), but FIP and FDP (foo) are only 16 bit. At least Intel CPUs
4080 * preserve whole 32 bit values, though, so (1) and (2) are the same wrt.
4082 * 3) 64-bit mode with REX.W prefix
4083 * - like (2), but XMM 8-15 are being saved and restored
4084 * 4) 64-bit mode without REX.W prefix
4085 * - like (3), but FIP and FDP are 64 bit
4087 * Emulation uses (3) for (1) and (2) and preserves XMM 8-15 to reach the
4088 * desired result. (4) is not emulated.
4090 * Note: Guest and host CPUID.(EAX=07H,ECX=0H):EBX[bit 13] (deprecate FPU CS
4091 * and FPU DS) should match.
4093 static int em_fxsave(struct x86_emulate_ctxt *ctxt)
4095 struct fxregs_state fx_state;
4098 rc = check_fxsr(ctxt);
4099 if (rc != X86EMUL_CONTINUE)
4104 rc = asm_safe("fxsave %[fx]", , [fx] "+m"(fx_state));
4108 if (rc != X86EMUL_CONTINUE)
4111 return segmented_write_std(ctxt, ctxt->memop.addr.mem, &fx_state,
4112 fxstate_size(ctxt));
4116 * FXRSTOR might restore XMM registers not provided by the guest. Fill
4117 * in the host registers (via FXSAVE) instead, so they won't be modified.
4118 * (preemption has to stay disabled until FXRSTOR).
4120 * Use noinline to keep the stack for other functions called by callers small.
4122 static noinline int fxregs_fixup(struct fxregs_state *fx_state,
4123 const size_t used_size)
4125 struct fxregs_state fx_tmp;
4128 rc = asm_safe("fxsave %[fx]", , [fx] "+m"(fx_tmp));
4129 memcpy((void *)fx_state + used_size, (void *)&fx_tmp + used_size,
4130 __fxstate_size(16) - used_size);
4135 static int em_fxrstor(struct x86_emulate_ctxt *ctxt)
4137 struct fxregs_state fx_state;
4141 rc = check_fxsr(ctxt);
4142 if (rc != X86EMUL_CONTINUE)
4145 size = fxstate_size(ctxt);
4146 rc = segmented_read_std(ctxt, ctxt->memop.addr.mem, &fx_state, size);
4147 if (rc != X86EMUL_CONTINUE)
4152 if (size < __fxstate_size(16)) {
4153 rc = fxregs_fixup(&fx_state, size);
4154 if (rc != X86EMUL_CONTINUE)
4158 if (fx_state.mxcsr >> 16) {
4159 rc = emulate_gp(ctxt, 0);
4163 if (rc == X86EMUL_CONTINUE)
4164 rc = asm_safe("fxrstor %[fx]", : [fx] "m"(fx_state));
4172 static int em_xsetbv(struct x86_emulate_ctxt *ctxt)
4176 eax = reg_read(ctxt, VCPU_REGS_RAX);
4177 edx = reg_read(ctxt, VCPU_REGS_RDX);
4178 ecx = reg_read(ctxt, VCPU_REGS_RCX);
4180 if (ctxt->ops->set_xcr(ctxt, ecx, ((u64)edx << 32) | eax))
4181 return emulate_gp(ctxt, 0);
4183 return X86EMUL_CONTINUE;
4186 static bool valid_cr(int nr)
4198 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
4200 if (!valid_cr(ctxt->modrm_reg))
4201 return emulate_ud(ctxt);
4203 return X86EMUL_CONTINUE;
4206 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
4208 u64 new_val = ctxt->src.val64;
4209 int cr = ctxt->modrm_reg;
4212 static u64 cr_reserved_bits[] = {
4213 0xffffffff00000000ULL,
4214 0, 0, 0, /* CR3 checked later */
4221 return emulate_ud(ctxt);
4223 if (new_val & cr_reserved_bits[cr])
4224 return emulate_gp(ctxt, 0);
4229 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
4230 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
4231 return emulate_gp(ctxt, 0);
4233 cr4 = ctxt->ops->get_cr(ctxt, 4);
4234 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4236 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
4237 !(cr4 & X86_CR4_PAE))
4238 return emulate_gp(ctxt, 0);
4245 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4246 if (efer & EFER_LMA) {
4248 u32 eax, ebx, ecx, edx;
4252 if (ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx,
4254 maxphyaddr = eax & 0xff;
4257 rsvd = rsvd_bits(maxphyaddr, 63);
4258 if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PCIDE)
4259 rsvd &= ~X86_CR3_PCID_NOFLUSH;
4263 return emulate_gp(ctxt, 0);
4268 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4270 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
4271 return emulate_gp(ctxt, 0);
4277 return X86EMUL_CONTINUE;
4280 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
4284 ctxt->ops->get_dr(ctxt, 7, &dr7);
4286 /* Check if DR7.Global_Enable is set */
4287 return dr7 & (1 << 13);
4290 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
4292 int dr = ctxt->modrm_reg;
4296 return emulate_ud(ctxt);
4298 cr4 = ctxt->ops->get_cr(ctxt, 4);
4299 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
4300 return emulate_ud(ctxt);
4302 if (check_dr7_gd(ctxt)) {
4305 ctxt->ops->get_dr(ctxt, 6, &dr6);
4306 dr6 &= ~DR_TRAP_BITS;
4307 dr6 |= DR6_BD | DR6_RTM;
4308 ctxt->ops->set_dr(ctxt, 6, dr6);
4309 return emulate_db(ctxt);
4312 return X86EMUL_CONTINUE;
4315 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
4317 u64 new_val = ctxt->src.val64;
4318 int dr = ctxt->modrm_reg;
4320 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
4321 return emulate_gp(ctxt, 0);
4323 return check_dr_read(ctxt);
4326 static int check_svme(struct x86_emulate_ctxt *ctxt)
4330 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4332 if (!(efer & EFER_SVME))
4333 return emulate_ud(ctxt);
4335 return X86EMUL_CONTINUE;
4338 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
4340 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
4342 /* Valid physical address? */
4343 if (rax & 0xffff000000000000ULL)
4344 return emulate_gp(ctxt, 0);
4346 return check_svme(ctxt);
4349 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
4351 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
4353 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
4354 return emulate_ud(ctxt);
4356 return X86EMUL_CONTINUE;
4359 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
4361 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
4362 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
4365 * VMware allows access to these Pseduo-PMCs even when read via RDPMC
4366 * in Ring3 when CR4.PCE=0.
4368 if (enable_vmware_backdoor && is_vmware_backdoor_pmc(rcx))
4369 return X86EMUL_CONTINUE;
4371 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
4372 ctxt->ops->check_pmc(ctxt, rcx))
4373 return emulate_gp(ctxt, 0);
4375 return X86EMUL_CONTINUE;
4378 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
4380 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
4381 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
4382 return emulate_gp(ctxt, 0);
4384 return X86EMUL_CONTINUE;
4387 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
4389 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
4390 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
4391 return emulate_gp(ctxt, 0);
4393 return X86EMUL_CONTINUE;
4396 #define D(_y) { .flags = (_y) }
4397 #define DI(_y, _i) { .flags = (_y)|Intercept, .intercept = x86_intercept_##_i }
4398 #define DIP(_y, _i, _p) { .flags = (_y)|Intercept|CheckPerm, \
4399 .intercept = x86_intercept_##_i, .check_perm = (_p) }
4400 #define N D(NotImpl)
4401 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
4402 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
4403 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
4404 #define ID(_f, _i) { .flags = ((_f) | InstrDual | ModRM), .u.idual = (_i) }
4405 #define MD(_f, _m) { .flags = ((_f) | ModeDual), .u.mdual = (_m) }
4406 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
4407 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
4408 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
4409 #define II(_f, _e, _i) \
4410 { .flags = (_f)|Intercept, .u.execute = (_e), .intercept = x86_intercept_##_i }
4411 #define IIP(_f, _e, _i, _p) \
4412 { .flags = (_f)|Intercept|CheckPerm, .u.execute = (_e), \
4413 .intercept = x86_intercept_##_i, .check_perm = (_p) }
4414 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
4416 #define D2bv(_f) D((_f) | ByteOp), D(_f)
4417 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
4418 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
4419 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
4420 #define I2bvIP(_f, _e, _i, _p) \
4421 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
4423 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
4424 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
4425 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
4427 static const struct opcode group7_rm0[] = {
4429 I(SrcNone | Priv | EmulateOnUD, em_hypercall),
4433 static const struct opcode group7_rm1[] = {
4434 DI(SrcNone | Priv, monitor),
4435 DI(SrcNone | Priv, mwait),
4439 static const struct opcode group7_rm2[] = {
4441 II(ImplicitOps | Priv, em_xsetbv, xsetbv),
4445 static const struct opcode group7_rm3[] = {
4446 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
4447 II(SrcNone | Prot | EmulateOnUD, em_hypercall, vmmcall),
4448 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
4449 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
4450 DIP(SrcNone | Prot | Priv, stgi, check_svme),
4451 DIP(SrcNone | Prot | Priv, clgi, check_svme),
4452 DIP(SrcNone | Prot | Priv, skinit, check_svme),
4453 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
4456 static const struct opcode group7_rm7[] = {
4458 DIP(SrcNone, rdtscp, check_rdtsc),
4462 static const struct opcode group1[] = {
4464 F(Lock | PageTable, em_or),
4467 F(Lock | PageTable, em_and),
4473 static const struct opcode group1A[] = {
4474 I(DstMem | SrcNone | Mov | Stack | IncSP | TwoMemOp, em_pop), N, N, N, N, N, N, N,
4477 static const struct opcode group2[] = {
4478 F(DstMem | ModRM, em_rol),
4479 F(DstMem | ModRM, em_ror),
4480 F(DstMem | ModRM, em_rcl),
4481 F(DstMem | ModRM, em_rcr),
4482 F(DstMem | ModRM, em_shl),
4483 F(DstMem | ModRM, em_shr),
4484 F(DstMem | ModRM, em_shl),
4485 F(DstMem | ModRM, em_sar),
4488 static const struct opcode group3[] = {
4489 F(DstMem | SrcImm | NoWrite, em_test),
4490 F(DstMem | SrcImm | NoWrite, em_test),
4491 F(DstMem | SrcNone | Lock, em_not),
4492 F(DstMem | SrcNone | Lock, em_neg),
4493 F(DstXacc | Src2Mem, em_mul_ex),
4494 F(DstXacc | Src2Mem, em_imul_ex),
4495 F(DstXacc | Src2Mem, em_div_ex),
4496 F(DstXacc | Src2Mem, em_idiv_ex),
4499 static const struct opcode group4[] = {
4500 F(ByteOp | DstMem | SrcNone | Lock, em_inc),
4501 F(ByteOp | DstMem | SrcNone | Lock, em_dec),
4505 static const struct opcode group5[] = {
4506 F(DstMem | SrcNone | Lock, em_inc),
4507 F(DstMem | SrcNone | Lock, em_dec),
4508 I(SrcMem | NearBranch, em_call_near_abs),
4509 I(SrcMemFAddr | ImplicitOps, em_call_far),
4510 I(SrcMem | NearBranch, em_jmp_abs),
4511 I(SrcMemFAddr | ImplicitOps, em_jmp_far),
4512 I(SrcMem | Stack | TwoMemOp, em_push), D(Undefined),
4515 static const struct opcode group6[] = {
4516 II(Prot | DstMem, em_sldt, sldt),
4517 II(Prot | DstMem, em_str, str),
4518 II(Prot | Priv | SrcMem16, em_lldt, lldt),
4519 II(Prot | Priv | SrcMem16, em_ltr, ltr),
4523 static const struct group_dual group7 = { {
4524 II(Mov | DstMem, em_sgdt, sgdt),
4525 II(Mov | DstMem, em_sidt, sidt),
4526 II(SrcMem | Priv, em_lgdt, lgdt),
4527 II(SrcMem | Priv, em_lidt, lidt),
4528 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
4529 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
4530 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
4536 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
4537 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
4541 static const struct opcode group8[] = {
4543 F(DstMem | SrcImmByte | NoWrite, em_bt),
4544 F(DstMem | SrcImmByte | Lock | PageTable, em_bts),
4545 F(DstMem | SrcImmByte | Lock, em_btr),
4546 F(DstMem | SrcImmByte | Lock | PageTable, em_btc),
4550 * The "memory" destination is actually always a register, since we come
4551 * from the register case of group9.
4553 static const struct gprefix pfx_0f_c7_7 = {
4554 N, N, N, II(DstMem | ModRM | Op3264 | EmulateOnUD, em_rdpid, rdtscp),
4558 static const struct group_dual group9 = { {
4559 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
4561 N, N, N, N, N, N, N,
4562 GP(0, &pfx_0f_c7_7),
4565 static const struct opcode group11[] = {
4566 I(DstMem | SrcImm | Mov | PageTable, em_mov),
4570 static const struct gprefix pfx_0f_ae_7 = {
4571 I(SrcMem | ByteOp, em_clflush), N, N, N,
4574 static const struct group_dual group15 = { {
4575 I(ModRM | Aligned16, em_fxsave),
4576 I(ModRM | Aligned16, em_fxrstor),
4577 N, N, N, N, N, GP(0, &pfx_0f_ae_7),
4579 N, N, N, N, N, N, N, N,
4582 static const struct gprefix pfx_0f_6f_0f_7f = {
4583 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
4586 static const struct instr_dual instr_dual_0f_2b = {
4590 static const struct gprefix pfx_0f_2b = {
4591 ID(0, &instr_dual_0f_2b), ID(0, &instr_dual_0f_2b), N, N,
4594 static const struct gprefix pfx_0f_10_0f_11 = {
4595 I(Unaligned, em_mov), I(Unaligned, em_mov), N, N,
4598 static const struct gprefix pfx_0f_28_0f_29 = {
4599 I(Aligned, em_mov), I(Aligned, em_mov), N, N,
4602 static const struct gprefix pfx_0f_e7 = {
4603 N, I(Sse, em_mov), N, N,
4606 static const struct escape escape_d9 = { {
4607 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstcw),
4610 N, N, N, N, N, N, N, N,
4612 N, N, N, N, N, N, N, N,
4614 N, N, N, N, N, N, N, N,
4616 N, N, N, N, N, N, N, N,
4618 N, N, N, N, N, N, N, N,
4620 N, N, N, N, N, N, N, N,
4622 N, N, N, N, N, N, N, N,
4624 N, N, N, N, N, N, N, N,
4627 static const struct escape escape_db = { {
4628 N, N, N, N, N, N, N, N,
4631 N, N, N, N, N, N, N, N,
4633 N, N, N, N, N, N, N, N,
4635 N, N, N, N, N, N, N, N,
4637 N, N, N, N, N, N, N, N,
4639 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
4641 N, N, N, N, N, N, N, N,
4643 N, N, N, N, N, N, N, N,
4645 N, N, N, N, N, N, N, N,
4648 static const struct escape escape_dd = { {
4649 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstsw),
4652 N, N, N, N, N, N, N, N,
4654 N, N, N, N, N, N, N, N,
4656 N, N, N, N, N, N, N, N,
4658 N, N, N, N, N, N, N, N,
4660 N, N, N, N, N, N, N, N,
4662 N, N, N, N, N, N, N, N,
4664 N, N, N, N, N, N, N, N,
4666 N, N, N, N, N, N, N, N,
4669 static const struct instr_dual instr_dual_0f_c3 = {
4670 I(DstMem | SrcReg | ModRM | No16 | Mov, em_mov), N
4673 static const struct mode_dual mode_dual_63 = {
4674 N, I(DstReg | SrcMem32 | ModRM | Mov, em_movsxd)
4677 static const struct opcode opcode_table[256] = {
4679 F6ALU(Lock, em_add),
4680 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
4681 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
4683 F6ALU(Lock | PageTable, em_or),
4684 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
4687 F6ALU(Lock, em_adc),
4688 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
4689 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
4691 F6ALU(Lock, em_sbb),
4692 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
4693 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
4695 F6ALU(Lock | PageTable, em_and), N, N,
4697 F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
4699 F6ALU(Lock, em_xor), N, N,
4701 F6ALU(NoWrite, em_cmp), N, N,
4703 X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
4705 X8(I(SrcReg | Stack, em_push)),
4707 X8(I(DstReg | Stack, em_pop)),
4709 I(ImplicitOps | Stack | No64, em_pusha),
4710 I(ImplicitOps | Stack | No64, em_popa),
4711 N, MD(ModRM, &mode_dual_63),
4714 I(SrcImm | Mov | Stack, em_push),
4715 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
4716 I(SrcImmByte | Mov | Stack, em_push),
4717 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
4718 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
4719 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
4721 X16(D(SrcImmByte | NearBranch)),
4723 G(ByteOp | DstMem | SrcImm, group1),
4724 G(DstMem | SrcImm, group1),
4725 G(ByteOp | DstMem | SrcImm | No64, group1),
4726 G(DstMem | SrcImmByte, group1),
4727 F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
4728 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
4730 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
4731 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
4732 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
4733 D(ModRM | SrcMem | NoAccess | DstReg),
4734 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
4737 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
4739 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
4740 I(SrcImmFAddr | No64, em_call_far), N,
4741 II(ImplicitOps | Stack, em_pushf, pushf),
4742 II(ImplicitOps | Stack, em_popf, popf),
4743 I(ImplicitOps, em_sahf), I(ImplicitOps, em_lahf),
4745 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
4746 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
4747 I2bv(SrcSI | DstDI | Mov | String | TwoMemOp, em_mov),
4748 F2bv(SrcSI | DstDI | String | NoWrite | TwoMemOp, em_cmp_r),
4750 F2bv(DstAcc | SrcImm | NoWrite, em_test),
4751 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
4752 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
4753 F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp_r),
4755 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
4757 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
4759 G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
4760 I(ImplicitOps | NearBranch | SrcImmU16, em_ret_near_imm),
4761 I(ImplicitOps | NearBranch, em_ret),
4762 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
4763 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
4764 G(ByteOp, group11), G(0, group11),
4766 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
4767 I(ImplicitOps | SrcImmU16, em_ret_far_imm),
4768 I(ImplicitOps, em_ret_far),
4769 D(ImplicitOps), DI(SrcImmByte, intn),
4770 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
4772 G(Src2One | ByteOp, group2), G(Src2One, group2),
4773 G(Src2CL | ByteOp, group2), G(Src2CL, group2),
4774 I(DstAcc | SrcImmUByte | No64, em_aam),
4775 I(DstAcc | SrcImmUByte | No64, em_aad),
4776 F(DstAcc | ByteOp | No64, em_salc),
4777 I(DstAcc | SrcXLat | ByteOp, em_mov),
4779 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
4781 X3(I(SrcImmByte | NearBranch, em_loop)),
4782 I(SrcImmByte | NearBranch, em_jcxz),
4783 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
4784 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
4786 I(SrcImm | NearBranch, em_call), D(SrcImm | ImplicitOps | NearBranch),
4787 I(SrcImmFAddr | No64, em_jmp_far),
4788 D(SrcImmByte | ImplicitOps | NearBranch),
4789 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
4790 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
4792 N, DI(ImplicitOps, icebp), N, N,
4793 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
4794 G(ByteOp, group3), G(0, group3),
4796 D(ImplicitOps), D(ImplicitOps),
4797 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
4798 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
4801 static const struct opcode twobyte_table[256] = {
4803 G(0, group6), GD(0, &group7), N, N,
4804 N, I(ImplicitOps | EmulateOnUD, em_syscall),
4805 II(ImplicitOps | Priv, em_clts, clts), N,
4806 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
4807 N, D(ImplicitOps | ModRM | SrcMem | NoAccess), N, N,
4809 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_10_0f_11),
4810 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_10_0f_11),
4812 D(ImplicitOps | ModRM | SrcMem | NoAccess),
4813 N, N, N, N, N, N, D(ImplicitOps | ModRM | SrcMem | NoAccess),
4815 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, cr_read, check_cr_read),
4816 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, dr_read, check_dr_read),
4817 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_cr_write, cr_write,
4819 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_dr_write, dr_write,
4822 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_28_0f_29),
4823 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_28_0f_29),
4824 N, GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_2b),
4827 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
4828 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
4829 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
4830 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
4831 I(ImplicitOps | EmulateOnUD, em_sysenter),
4832 I(ImplicitOps | Priv | EmulateOnUD, em_sysexit),
4834 N, N, N, N, N, N, N, N,
4836 X16(D(DstReg | SrcMem | ModRM)),
4838 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4843 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
4848 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
4850 X16(D(SrcImm | NearBranch)),
4852 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
4854 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
4855 II(ImplicitOps, em_cpuid, cpuid),
4856 F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
4857 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
4858 F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
4860 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
4861 II(EmulateOnUD | ImplicitOps, em_rsm, rsm),
4862 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
4863 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
4864 F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
4865 GD(0, &group15), F(DstReg | SrcMem | ModRM, em_imul),
4867 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable | SrcWrite, em_cmpxchg),
4868 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
4869 F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
4870 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
4871 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
4872 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4876 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
4877 I(DstReg | SrcMem | ModRM, em_bsf_c),
4878 I(DstReg | SrcMem | ModRM, em_bsr_c),
4879 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4881 F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
4882 N, ID(0, &instr_dual_0f_c3),
4883 N, N, N, GD(0, &group9),
4885 X8(I(DstReg, em_bswap)),
4887 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4889 N, N, N, N, N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_e7),
4890 N, N, N, N, N, N, N, N,
4892 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
4895 static const struct instr_dual instr_dual_0f_38_f0 = {
4896 I(DstReg | SrcMem | Mov, em_movbe), N
4899 static const struct instr_dual instr_dual_0f_38_f1 = {
4900 I(DstMem | SrcReg | Mov, em_movbe), N
4903 static const struct gprefix three_byte_0f_38_f0 = {
4904 ID(0, &instr_dual_0f_38_f0), N, N, N
4907 static const struct gprefix three_byte_0f_38_f1 = {
4908 ID(0, &instr_dual_0f_38_f1), N, N, N
4912 * Insns below are selected by the prefix which indexed by the third opcode
4915 static const struct opcode opcode_map_0f_38[256] = {
4917 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4919 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4921 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f0),
4922 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f1),
4943 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4947 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4953 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
4954 unsigned size, bool sign_extension)
4956 int rc = X86EMUL_CONTINUE;
4960 op->addr.mem.ea = ctxt->_eip;
4961 /* NB. Immediates are sign-extended as necessary. */
4962 switch (op->bytes) {
4964 op->val = insn_fetch(s8, ctxt);
4967 op->val = insn_fetch(s16, ctxt);
4970 op->val = insn_fetch(s32, ctxt);
4973 op->val = insn_fetch(s64, ctxt);
4976 if (!sign_extension) {
4977 switch (op->bytes) {
4985 op->val &= 0xffffffff;
4993 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
4996 int rc = X86EMUL_CONTINUE;
5000 decode_register_operand(ctxt, op);
5003 rc = decode_imm(ctxt, op, 1, false);
5006 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
5010 if (ctxt->d & BitOp)
5011 fetch_bit_operand(ctxt);
5012 op->orig_val = op->val;
5015 ctxt->memop.bytes = (ctxt->op_bytes == 8) ? 16 : 8;
5019 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
5020 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
5021 fetch_register_operand(op);
5022 op->orig_val = op->val;
5026 op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;
5027 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
5028 fetch_register_operand(op);
5029 op->orig_val = op->val;
5032 if (ctxt->d & ByteOp) {
5037 op->bytes = ctxt->op_bytes;
5038 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
5039 fetch_register_operand(op);
5040 op->orig_val = op->val;
5044 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
5046 register_address(ctxt, VCPU_REGS_RDI);
5047 op->addr.mem.seg = VCPU_SREG_ES;
5054 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
5055 fetch_register_operand(op);
5060 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
5063 rc = decode_imm(ctxt, op, 1, true);
5071 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
5074 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
5077 ctxt->memop.bytes = 1;
5078 if (ctxt->memop.type == OP_REG) {
5079 ctxt->memop.addr.reg = decode_register(ctxt,
5080 ctxt->modrm_rm, true);
5081 fetch_register_operand(&ctxt->memop);
5085 ctxt->memop.bytes = 2;
5088 ctxt->memop.bytes = 4;
5091 rc = decode_imm(ctxt, op, 2, false);
5094 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
5098 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
5100 register_address(ctxt, VCPU_REGS_RSI);
5101 op->addr.mem.seg = ctxt->seg_override;
5107 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
5110 reg_read(ctxt, VCPU_REGS_RBX) +
5111 (reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
5112 op->addr.mem.seg = ctxt->seg_override;
5117 op->addr.mem.ea = ctxt->_eip;
5118 op->bytes = ctxt->op_bytes + 2;
5119 insn_fetch_arr(op->valptr, op->bytes, ctxt);
5122 ctxt->memop.bytes = ctxt->op_bytes + 2;
5126 op->val = VCPU_SREG_ES;
5130 op->val = VCPU_SREG_CS;
5134 op->val = VCPU_SREG_SS;
5138 op->val = VCPU_SREG_DS;
5142 op->val = VCPU_SREG_FS;
5146 op->val = VCPU_SREG_GS;
5149 /* Special instructions do their own operand decoding. */
5151 op->type = OP_NONE; /* Disable writeback. */
5159 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
5161 int rc = X86EMUL_CONTINUE;
5162 int mode = ctxt->mode;
5163 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
5164 bool op_prefix = false;
5165 bool has_seg_override = false;
5166 struct opcode opcode;
5168 struct desc_struct desc;
5170 ctxt->memop.type = OP_NONE;
5171 ctxt->memopp = NULL;
5172 ctxt->_eip = ctxt->eip;
5173 ctxt->fetch.ptr = ctxt->fetch.data;
5174 ctxt->fetch.end = ctxt->fetch.data + insn_len;
5175 ctxt->opcode_len = 1;
5176 ctxt->intercept = x86_intercept_none;
5178 memcpy(ctxt->fetch.data, insn, insn_len);
5180 rc = __do_insn_fetch_bytes(ctxt, 1);
5181 if (rc != X86EMUL_CONTINUE)
5186 case X86EMUL_MODE_REAL:
5187 case X86EMUL_MODE_VM86:
5188 def_op_bytes = def_ad_bytes = 2;
5189 ctxt->ops->get_segment(ctxt, &dummy, &desc, NULL, VCPU_SREG_CS);
5191 def_op_bytes = def_ad_bytes = 4;
5193 case X86EMUL_MODE_PROT16:
5194 def_op_bytes = def_ad_bytes = 2;
5196 case X86EMUL_MODE_PROT32:
5197 def_op_bytes = def_ad_bytes = 4;
5199 #ifdef CONFIG_X86_64
5200 case X86EMUL_MODE_PROT64:
5206 return EMULATION_FAILED;
5209 ctxt->op_bytes = def_op_bytes;
5210 ctxt->ad_bytes = def_ad_bytes;
5212 /* Legacy prefixes. */
5214 switch (ctxt->b = insn_fetch(u8, ctxt)) {
5215 case 0x66: /* operand-size override */
5217 /* switch between 2/4 bytes */
5218 ctxt->op_bytes = def_op_bytes ^ 6;
5220 case 0x67: /* address-size override */
5221 if (mode == X86EMUL_MODE_PROT64)
5222 /* switch between 4/8 bytes */
5223 ctxt->ad_bytes = def_ad_bytes ^ 12;
5225 /* switch between 2/4 bytes */
5226 ctxt->ad_bytes = def_ad_bytes ^ 6;
5228 case 0x26: /* ES override */
5229 has_seg_override = true;
5230 ctxt->seg_override = VCPU_SREG_ES;
5232 case 0x2e: /* CS override */
5233 has_seg_override = true;
5234 ctxt->seg_override = VCPU_SREG_CS;
5236 case 0x36: /* SS override */
5237 has_seg_override = true;
5238 ctxt->seg_override = VCPU_SREG_SS;
5240 case 0x3e: /* DS override */
5241 has_seg_override = true;
5242 ctxt->seg_override = VCPU_SREG_DS;
5244 case 0x64: /* FS override */
5245 has_seg_override = true;
5246 ctxt->seg_override = VCPU_SREG_FS;
5248 case 0x65: /* GS override */
5249 has_seg_override = true;
5250 ctxt->seg_override = VCPU_SREG_GS;
5252 case 0x40 ... 0x4f: /* REX */
5253 if (mode != X86EMUL_MODE_PROT64)
5255 ctxt->rex_prefix = ctxt->b;
5257 case 0xf0: /* LOCK */
5258 ctxt->lock_prefix = 1;
5260 case 0xf2: /* REPNE/REPNZ */
5261 case 0xf3: /* REP/REPE/REPZ */
5262 ctxt->rep_prefix = ctxt->b;
5268 /* Any legacy prefix after a REX prefix nullifies its effect. */
5270 ctxt->rex_prefix = 0;
5276 if (ctxt->rex_prefix & 8)
5277 ctxt->op_bytes = 8; /* REX.W */
5279 /* Opcode byte(s). */
5280 opcode = opcode_table[ctxt->b];
5281 /* Two-byte opcode? */
5282 if (ctxt->b == 0x0f) {
5283 ctxt->opcode_len = 2;
5284 ctxt->b = insn_fetch(u8, ctxt);
5285 opcode = twobyte_table[ctxt->b];
5287 /* 0F_38 opcode map */
5288 if (ctxt->b == 0x38) {
5289 ctxt->opcode_len = 3;
5290 ctxt->b = insn_fetch(u8, ctxt);
5291 opcode = opcode_map_0f_38[ctxt->b];
5294 ctxt->d = opcode.flags;
5296 if (ctxt->d & ModRM)
5297 ctxt->modrm = insn_fetch(u8, ctxt);
5299 /* vex-prefix instructions are not implemented */
5300 if (ctxt->opcode_len == 1 && (ctxt->b == 0xc5 || ctxt->b == 0xc4) &&
5301 (mode == X86EMUL_MODE_PROT64 || (ctxt->modrm & 0xc0) == 0xc0)) {
5305 while (ctxt->d & GroupMask) {
5306 switch (ctxt->d & GroupMask) {
5308 goffset = (ctxt->modrm >> 3) & 7;
5309 opcode = opcode.u.group[goffset];
5312 goffset = (ctxt->modrm >> 3) & 7;
5313 if ((ctxt->modrm >> 6) == 3)
5314 opcode = opcode.u.gdual->mod3[goffset];
5316 opcode = opcode.u.gdual->mod012[goffset];
5319 goffset = ctxt->modrm & 7;
5320 opcode = opcode.u.group[goffset];
5323 if (ctxt->rep_prefix && op_prefix)
5324 return EMULATION_FAILED;
5325 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
5326 switch (simd_prefix) {
5327 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
5328 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
5329 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
5330 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
5334 if (ctxt->modrm > 0xbf) {
5335 size_t size = ARRAY_SIZE(opcode.u.esc->high);
5336 u32 index = array_index_nospec(
5337 ctxt->modrm - 0xc0, size);
5339 opcode = opcode.u.esc->high[index];
5341 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
5345 if ((ctxt->modrm >> 6) == 3)
5346 opcode = opcode.u.idual->mod3;
5348 opcode = opcode.u.idual->mod012;
5351 if (ctxt->mode == X86EMUL_MODE_PROT64)
5352 opcode = opcode.u.mdual->mode64;
5354 opcode = opcode.u.mdual->mode32;
5357 return EMULATION_FAILED;
5360 ctxt->d &= ~(u64)GroupMask;
5361 ctxt->d |= opcode.flags;
5366 return EMULATION_FAILED;
5368 ctxt->execute = opcode.u.execute;
5370 if (unlikely(ctxt->ud) && likely(!(ctxt->d & EmulateOnUD)))
5371 return EMULATION_FAILED;
5373 if (unlikely(ctxt->d &
5374 (NotImpl|Stack|Op3264|Sse|Mmx|Intercept|CheckPerm|NearBranch|
5377 * These are copied unconditionally here, and checked unconditionally
5378 * in x86_emulate_insn.
5380 ctxt->check_perm = opcode.check_perm;
5381 ctxt->intercept = opcode.intercept;
5383 if (ctxt->d & NotImpl)
5384 return EMULATION_FAILED;
5386 if (mode == X86EMUL_MODE_PROT64) {
5387 if (ctxt->op_bytes == 4 && (ctxt->d & Stack))
5389 else if (ctxt->d & NearBranch)
5393 if (ctxt->d & Op3264) {
5394 if (mode == X86EMUL_MODE_PROT64)
5400 if ((ctxt->d & No16) && ctxt->op_bytes == 2)
5404 ctxt->op_bytes = 16;
5405 else if (ctxt->d & Mmx)
5409 /* ModRM and SIB bytes. */
5410 if (ctxt->d & ModRM) {
5411 rc = decode_modrm(ctxt, &ctxt->memop);
5412 if (!has_seg_override) {
5413 has_seg_override = true;
5414 ctxt->seg_override = ctxt->modrm_seg;
5416 } else if (ctxt->d & MemAbs)
5417 rc = decode_abs(ctxt, &ctxt->memop);
5418 if (rc != X86EMUL_CONTINUE)
5421 if (!has_seg_override)
5422 ctxt->seg_override = VCPU_SREG_DS;
5424 ctxt->memop.addr.mem.seg = ctxt->seg_override;
5427 * Decode and fetch the source operand: register, memory
5430 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
5431 if (rc != X86EMUL_CONTINUE)
5435 * Decode and fetch the second source operand: register, memory
5438 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
5439 if (rc != X86EMUL_CONTINUE)
5442 /* Decode and fetch the destination operand: register or memory. */
5443 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
5445 if (ctxt->rip_relative && likely(ctxt->memopp))
5446 ctxt->memopp->addr.mem.ea = address_mask(ctxt,
5447 ctxt->memopp->addr.mem.ea + ctxt->_eip);
5450 if (rc == X86EMUL_PROPAGATE_FAULT)
5451 ctxt->have_exception = true;
5452 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
5455 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
5457 return ctxt->d & PageTable;
5460 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
5462 /* The second termination condition only applies for REPE
5463 * and REPNE. Test if the repeat string operation prefix is
5464 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
5465 * corresponding termination condition according to:
5466 * - if REPE/REPZ and ZF = 0 then done
5467 * - if REPNE/REPNZ and ZF = 1 then done
5469 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
5470 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
5471 && (((ctxt->rep_prefix == REPE_PREFIX) &&
5472 ((ctxt->eflags & X86_EFLAGS_ZF) == 0))
5473 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
5474 ((ctxt->eflags & X86_EFLAGS_ZF) == X86_EFLAGS_ZF))))
5480 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
5485 rc = asm_safe("fwait");
5488 if (unlikely(rc != X86EMUL_CONTINUE))
5489 return emulate_exception(ctxt, MF_VECTOR, 0, false);
5491 return X86EMUL_CONTINUE;
5494 static void fetch_possible_mmx_operand(struct operand *op)
5496 if (op->type == OP_MM)
5497 read_mmx_reg(&op->mm_val, op->addr.mm);
5500 static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop)
5502 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
5504 if (!(ctxt->d & ByteOp))
5505 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
5507 asm("push %[flags]; popf; " CALL_NOSPEC " ; pushf; pop %[flags]\n"
5508 : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
5509 [thunk_target]"+S"(fop), ASM_CALL_CONSTRAINT
5510 : "c"(ctxt->src2.val));
5512 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
5513 if (!fop) /* exception is returned in fop variable */
5514 return emulate_de(ctxt);
5515 return X86EMUL_CONTINUE;
5518 void init_decode_cache(struct x86_emulate_ctxt *ctxt)
5520 memset(&ctxt->rip_relative, 0,
5521 (void *)&ctxt->modrm - (void *)&ctxt->rip_relative);
5523 ctxt->io_read.pos = 0;
5524 ctxt->io_read.end = 0;
5525 ctxt->mem_read.end = 0;
5528 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
5530 const struct x86_emulate_ops *ops = ctxt->ops;
5531 int rc = X86EMUL_CONTINUE;
5532 int saved_dst_type = ctxt->dst.type;
5533 unsigned emul_flags;
5535 ctxt->mem_read.pos = 0;
5537 /* LOCK prefix is allowed only with some instructions */
5538 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
5539 rc = emulate_ud(ctxt);
5543 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
5544 rc = emulate_ud(ctxt);
5548 emul_flags = ctxt->ops->get_hflags(ctxt);
5549 if (unlikely(ctxt->d &
5550 (No64|Undefined|Sse|Mmx|Intercept|CheckPerm|Priv|Prot|String))) {
5551 if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
5552 (ctxt->d & Undefined)) {
5553 rc = emulate_ud(ctxt);
5557 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
5558 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
5559 rc = emulate_ud(ctxt);
5563 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
5564 rc = emulate_nm(ctxt);
5568 if (ctxt->d & Mmx) {
5569 rc = flush_pending_x87_faults(ctxt);
5570 if (rc != X86EMUL_CONTINUE)
5573 * Now that we know the fpu is exception safe, we can fetch
5576 fetch_possible_mmx_operand(&ctxt->src);
5577 fetch_possible_mmx_operand(&ctxt->src2);
5578 if (!(ctxt->d & Mov))
5579 fetch_possible_mmx_operand(&ctxt->dst);
5582 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && ctxt->intercept) {
5583 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5584 X86_ICPT_PRE_EXCEPT);
5585 if (rc != X86EMUL_CONTINUE)
5589 /* Instruction can only be executed in protected mode */
5590 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
5591 rc = emulate_ud(ctxt);
5595 /* Privileged instruction can be executed only in CPL=0 */
5596 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
5597 if (ctxt->d & PrivUD)
5598 rc = emulate_ud(ctxt);
5600 rc = emulate_gp(ctxt, 0);
5604 /* Do instruction specific permission checks */
5605 if (ctxt->d & CheckPerm) {
5606 rc = ctxt->check_perm(ctxt);
5607 if (rc != X86EMUL_CONTINUE)
5611 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {
5612 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5613 X86_ICPT_POST_EXCEPT);
5614 if (rc != X86EMUL_CONTINUE)
5618 if (ctxt->rep_prefix && (ctxt->d & String)) {
5619 /* All REP prefixes have the same first termination condition */
5620 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
5621 string_registers_quirk(ctxt);
5622 ctxt->eip = ctxt->_eip;
5623 ctxt->eflags &= ~X86_EFLAGS_RF;
5629 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
5630 rc = segmented_read(ctxt, ctxt->src.addr.mem,
5631 ctxt->src.valptr, ctxt->src.bytes);
5632 if (rc != X86EMUL_CONTINUE)
5634 ctxt->src.orig_val64 = ctxt->src.val64;
5637 if (ctxt->src2.type == OP_MEM) {
5638 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
5639 &ctxt->src2.val, ctxt->src2.bytes);
5640 if (rc != X86EMUL_CONTINUE)
5644 if ((ctxt->d & DstMask) == ImplicitOps)
5648 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
5649 /* optimisation - avoid slow emulated read if Mov */
5650 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
5651 &ctxt->dst.val, ctxt->dst.bytes);
5652 if (rc != X86EMUL_CONTINUE) {
5653 if (!(ctxt->d & NoWrite) &&
5654 rc == X86EMUL_PROPAGATE_FAULT &&
5655 ctxt->exception.vector == PF_VECTOR)
5656 ctxt->exception.error_code |= PFERR_WRITE_MASK;
5660 /* Copy full 64-bit value for CMPXCHG8B. */
5661 ctxt->dst.orig_val64 = ctxt->dst.val64;
5665 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {
5666 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5667 X86_ICPT_POST_MEMACCESS);
5668 if (rc != X86EMUL_CONTINUE)
5672 if (ctxt->rep_prefix && (ctxt->d & String))
5673 ctxt->eflags |= X86_EFLAGS_RF;
5675 ctxt->eflags &= ~X86_EFLAGS_RF;
5677 if (ctxt->execute) {
5678 if (ctxt->d & Fastop)
5679 rc = fastop(ctxt, ctxt->fop);
5681 rc = ctxt->execute(ctxt);
5682 if (rc != X86EMUL_CONTINUE)
5687 if (ctxt->opcode_len == 2)
5689 else if (ctxt->opcode_len == 3)
5690 goto threebyte_insn;
5693 case 0x70 ... 0x7f: /* jcc (short) */
5694 if (test_cc(ctxt->b, ctxt->eflags))
5695 rc = jmp_rel(ctxt, ctxt->src.val);
5697 case 0x8d: /* lea r16/r32, m */
5698 ctxt->dst.val = ctxt->src.addr.mem.ea;
5700 case 0x90 ... 0x97: /* nop / xchg reg, rax */
5701 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
5702 ctxt->dst.type = OP_NONE;
5706 case 0x98: /* cbw/cwde/cdqe */
5707 switch (ctxt->op_bytes) {
5708 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
5709 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
5710 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
5713 case 0xcc: /* int3 */
5714 rc = emulate_int(ctxt, 3);
5716 case 0xcd: /* int n */
5717 rc = emulate_int(ctxt, ctxt->src.val);
5719 case 0xce: /* into */
5720 if (ctxt->eflags & X86_EFLAGS_OF)
5721 rc = emulate_int(ctxt, 4);
5723 case 0xe9: /* jmp rel */
5724 case 0xeb: /* jmp rel short */
5725 rc = jmp_rel(ctxt, ctxt->src.val);
5726 ctxt->dst.type = OP_NONE; /* Disable writeback. */
5728 case 0xf4: /* hlt */
5729 ctxt->ops->halt(ctxt);
5731 case 0xf5: /* cmc */
5732 /* complement carry flag from eflags reg */
5733 ctxt->eflags ^= X86_EFLAGS_CF;
5735 case 0xf8: /* clc */
5736 ctxt->eflags &= ~X86_EFLAGS_CF;
5738 case 0xf9: /* stc */
5739 ctxt->eflags |= X86_EFLAGS_CF;
5741 case 0xfc: /* cld */
5742 ctxt->eflags &= ~X86_EFLAGS_DF;
5744 case 0xfd: /* std */
5745 ctxt->eflags |= X86_EFLAGS_DF;
5748 goto cannot_emulate;
5751 if (rc != X86EMUL_CONTINUE)
5755 if (ctxt->d & SrcWrite) {
5756 BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);
5757 rc = writeback(ctxt, &ctxt->src);
5758 if (rc != X86EMUL_CONTINUE)
5761 if (!(ctxt->d & NoWrite)) {
5762 rc = writeback(ctxt, &ctxt->dst);
5763 if (rc != X86EMUL_CONTINUE)
5768 * restore dst type in case the decoding will be reused
5769 * (happens for string instruction )
5771 ctxt->dst.type = saved_dst_type;
5773 if ((ctxt->d & SrcMask) == SrcSI)
5774 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
5776 if ((ctxt->d & DstMask) == DstDI)
5777 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
5779 if (ctxt->rep_prefix && (ctxt->d & String)) {
5781 struct read_cache *r = &ctxt->io_read;
5782 if ((ctxt->d & SrcMask) == SrcSI)
5783 count = ctxt->src.count;
5785 count = ctxt->dst.count;
5786 register_address_increment(ctxt, VCPU_REGS_RCX, -count);
5788 if (!string_insn_completed(ctxt)) {
5790 * Re-enter guest when pio read ahead buffer is empty
5791 * or, if it is not used, after each 1024 iteration.
5793 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
5794 (r->end == 0 || r->end != r->pos)) {
5796 * Reset read cache. Usually happens before
5797 * decode, but since instruction is restarted
5798 * we have to do it here.
5800 ctxt->mem_read.end = 0;
5801 writeback_registers(ctxt);
5802 return EMULATION_RESTART;
5804 goto done; /* skip rip writeback */
5806 ctxt->eflags &= ~X86_EFLAGS_RF;
5809 ctxt->eip = ctxt->_eip;
5812 if (rc == X86EMUL_PROPAGATE_FAULT) {
5813 WARN_ON(ctxt->exception.vector > 0x1f);
5814 ctxt->have_exception = true;
5816 if (rc == X86EMUL_INTERCEPTED)
5817 return EMULATION_INTERCEPTED;
5819 if (rc == X86EMUL_CONTINUE)
5820 writeback_registers(ctxt);
5822 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
5826 case 0x09: /* wbinvd */
5827 (ctxt->ops->wbinvd)(ctxt);
5829 case 0x08: /* invd */
5830 case 0x0d: /* GrpP (prefetch) */
5831 case 0x18: /* Grp16 (prefetch/nop) */
5832 case 0x1f: /* nop */
5834 case 0x20: /* mov cr, reg */
5835 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
5837 case 0x21: /* mov from dr to reg */
5838 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
5840 case 0x40 ... 0x4f: /* cmov */
5841 if (test_cc(ctxt->b, ctxt->eflags))
5842 ctxt->dst.val = ctxt->src.val;
5843 else if (ctxt->op_bytes != 4)
5844 ctxt->dst.type = OP_NONE; /* no writeback */
5846 case 0x80 ... 0x8f: /* jnz rel, etc*/
5847 if (test_cc(ctxt->b, ctxt->eflags))
5848 rc = jmp_rel(ctxt, ctxt->src.val);
5850 case 0x90 ... 0x9f: /* setcc r/m8 */
5851 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
5853 case 0xb6 ... 0xb7: /* movzx */
5854 ctxt->dst.bytes = ctxt->op_bytes;
5855 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
5856 : (u16) ctxt->src.val;
5858 case 0xbe ... 0xbf: /* movsx */
5859 ctxt->dst.bytes = ctxt->op_bytes;
5860 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
5861 (s16) ctxt->src.val;
5864 goto cannot_emulate;
5869 if (rc != X86EMUL_CONTINUE)
5875 return EMULATION_FAILED;
5878 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
5880 invalidate_registers(ctxt);
5883 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
5885 writeback_registers(ctxt);
5888 bool emulator_can_use_gpa(struct x86_emulate_ctxt *ctxt)
5890 if (ctxt->rep_prefix && (ctxt->d & String))
5893 if (ctxt->d & TwoMemOp)