1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
27 #include <linux/stringify.h>
36 #define OpImplicit 1ull /* No generic decode */
37 #define OpReg 2ull /* Register */
38 #define OpMem 3ull /* Memory */
39 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
40 #define OpDI 5ull /* ES:DI/EDI/RDI */
41 #define OpMem64 6ull /* Memory, 64-bit */
42 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
43 #define OpDX 8ull /* DX register */
44 #define OpCL 9ull /* CL register (for shifts) */
45 #define OpImmByte 10ull /* 8-bit sign extended immediate */
46 #define OpOne 11ull /* Implied 1 */
47 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
48 #define OpMem16 13ull /* Memory operand (16-bit). */
49 #define OpMem32 14ull /* Memory operand (32-bit). */
50 #define OpImmU 15ull /* Immediate operand, zero extended */
51 #define OpSI 16ull /* SI/ESI/RSI */
52 #define OpImmFAddr 17ull /* Immediate far address */
53 #define OpMemFAddr 18ull /* Far address in memory */
54 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
55 #define OpES 20ull /* ES */
56 #define OpCS 21ull /* CS */
57 #define OpSS 22ull /* SS */
58 #define OpDS 23ull /* DS */
59 #define OpFS 24ull /* FS */
60 #define OpGS 25ull /* GS */
61 #define OpMem8 26ull /* 8-bit zero extended memory operand */
62 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
63 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
64 #define OpAccLo 29ull /* Low part of extended acc (AX/AX/EAX/RAX) */
65 #define OpAccHi 30ull /* High part of extended acc (-/DX/EDX/RDX) */
67 #define OpBits 5 /* Width of operand field */
68 #define OpMask ((1ull << OpBits) - 1)
71 * Opcode effective-address decode tables.
72 * Note that we only emulate instructions that have at least one memory
73 * operand (excluding implicit stack references). We assume that stack
74 * references and instruction fetches will never occur in special memory
75 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
79 /* Operand sizes: 8-bit operands or specified/overridden size. */
80 #define ByteOp (1<<0) /* 8-bit operands. */
81 /* Destination operand type. */
83 #define ImplicitOps (OpImplicit << DstShift)
84 #define DstReg (OpReg << DstShift)
85 #define DstMem (OpMem << DstShift)
86 #define DstAcc (OpAcc << DstShift)
87 #define DstDI (OpDI << DstShift)
88 #define DstMem64 (OpMem64 << DstShift)
89 #define DstImmUByte (OpImmUByte << DstShift)
90 #define DstDX (OpDX << DstShift)
91 #define DstAccLo (OpAccLo << DstShift)
92 #define DstMask (OpMask << DstShift)
93 /* Source operand type. */
95 #define SrcNone (OpNone << SrcShift)
96 #define SrcReg (OpReg << SrcShift)
97 #define SrcMem (OpMem << SrcShift)
98 #define SrcMem16 (OpMem16 << SrcShift)
99 #define SrcMem32 (OpMem32 << SrcShift)
100 #define SrcImm (OpImm << SrcShift)
101 #define SrcImmByte (OpImmByte << SrcShift)
102 #define SrcOne (OpOne << SrcShift)
103 #define SrcImmUByte (OpImmUByte << SrcShift)
104 #define SrcImmU (OpImmU << SrcShift)
105 #define SrcSI (OpSI << SrcShift)
106 #define SrcXLat (OpXLat << SrcShift)
107 #define SrcImmFAddr (OpImmFAddr << SrcShift)
108 #define SrcMemFAddr (OpMemFAddr << SrcShift)
109 #define SrcAcc (OpAcc << SrcShift)
110 #define SrcImmU16 (OpImmU16 << SrcShift)
111 #define SrcImm64 (OpImm64 << SrcShift)
112 #define SrcDX (OpDX << SrcShift)
113 #define SrcMem8 (OpMem8 << SrcShift)
114 #define SrcAccHi (OpAccHi << SrcShift)
115 #define SrcMask (OpMask << SrcShift)
116 #define BitOp (1<<11)
117 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
118 #define String (1<<13) /* String instruction (rep capable) */
119 #define Stack (1<<14) /* Stack instruction (push/pop) */
120 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
121 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
122 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
123 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
124 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
125 #define Escape (5<<15) /* Escape to coprocessor instruction */
126 #define Sse (1<<18) /* SSE Vector instruction */
127 /* Generic ModRM decode. */
128 #define ModRM (1<<19)
129 /* Destination is only written; never read. */
132 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
133 #define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
134 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
135 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
136 #define Undefined (1<<25) /* No Such Instruction */
137 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
138 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
140 #define PageTable (1 << 29) /* instruction used to write page table */
141 #define NotImpl (1 << 30) /* instruction is not implemented */
142 /* Source 2 operand type */
143 #define Src2Shift (31)
144 #define Src2None (OpNone << Src2Shift)
145 #define Src2Mem (OpMem << Src2Shift)
146 #define Src2CL (OpCL << Src2Shift)
147 #define Src2ImmByte (OpImmByte << Src2Shift)
148 #define Src2One (OpOne << Src2Shift)
149 #define Src2Imm (OpImm << Src2Shift)
150 #define Src2ES (OpES << Src2Shift)
151 #define Src2CS (OpCS << Src2Shift)
152 #define Src2SS (OpSS << Src2Shift)
153 #define Src2DS (OpDS << Src2Shift)
154 #define Src2FS (OpFS << Src2Shift)
155 #define Src2GS (OpGS << Src2Shift)
156 #define Src2Mask (OpMask << Src2Shift)
157 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
158 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
159 #define Unaligned ((u64)1 << 42) /* Explicitly unaligned (e.g. MOVDQU) */
160 #define Avx ((u64)1 << 43) /* Advanced Vector Extensions */
161 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
162 #define NoWrite ((u64)1 << 45) /* No writeback */
163 #define SrcWrite ((u64)1 << 46) /* Write back src operand */
164 #define NoMod ((u64)1 << 47) /* Mod field is ignored */
165 #define Intercept ((u64)1 << 48) /* Has valid intercept field */
166 #define CheckPerm ((u64)1 << 49) /* Has valid check_perm field */
168 #define DstXacc (DstAccLo | SrcAccHi | SrcWrite)
170 #define X2(x...) x, x
171 #define X3(x...) X2(x), x
172 #define X4(x...) X2(x), X2(x)
173 #define X5(x...) X4(x), x
174 #define X6(x...) X4(x), X2(x)
175 #define X7(x...) X4(x), X3(x)
176 #define X8(x...) X4(x), X4(x)
177 #define X16(x...) X8(x), X8(x)
179 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
180 #define FASTOP_SIZE 8
183 * fastop functions have a special calling convention:
188 * flags: rflags (in/out)
189 * ex: rsi (in:fastop pointer, out:zero if exception)
191 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
192 * different operand sizes can be reached by calculation, rather than a jump
193 * table (which would be bigger than the code).
195 * fastop functions are declared as taking a never-defined fastop parameter,
196 * so they can't be called from C directly.
205 int (*execute)(struct x86_emulate_ctxt *ctxt);
206 const struct opcode *group;
207 const struct group_dual *gdual;
208 const struct gprefix *gprefix;
209 const struct escape *esc;
210 void (*fastop)(struct fastop *fake);
212 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
216 struct opcode mod012[8];
217 struct opcode mod3[8];
221 struct opcode pfx_no;
222 struct opcode pfx_66;
223 struct opcode pfx_f2;
224 struct opcode pfx_f3;
229 struct opcode high[64];
232 /* EFLAGS bit definitions. */
233 #define EFLG_ID (1<<21)
234 #define EFLG_VIP (1<<20)
235 #define EFLG_VIF (1<<19)
236 #define EFLG_AC (1<<18)
237 #define EFLG_VM (1<<17)
238 #define EFLG_RF (1<<16)
239 #define EFLG_IOPL (3<<12)
240 #define EFLG_NT (1<<14)
241 #define EFLG_OF (1<<11)
242 #define EFLG_DF (1<<10)
243 #define EFLG_IF (1<<9)
244 #define EFLG_TF (1<<8)
245 #define EFLG_SF (1<<7)
246 #define EFLG_ZF (1<<6)
247 #define EFLG_AF (1<<4)
248 #define EFLG_PF (1<<2)
249 #define EFLG_CF (1<<0)
251 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
252 #define EFLG_RESERVED_ONE_MASK 2
254 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
256 if (!(ctxt->regs_valid & (1 << nr))) {
257 ctxt->regs_valid |= 1 << nr;
258 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
260 return ctxt->_regs[nr];
263 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
265 ctxt->regs_valid |= 1 << nr;
266 ctxt->regs_dirty |= 1 << nr;
267 return &ctxt->_regs[nr];
270 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
273 return reg_write(ctxt, nr);
276 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
280 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
281 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
284 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
286 ctxt->regs_dirty = 0;
287 ctxt->regs_valid = 0;
291 * These EFLAGS bits are restored from saved value during emulation, and
292 * any changes are written back to the saved value after emulation.
294 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
302 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));
304 #define FOP_ALIGN ".align " __stringify(FASTOP_SIZE) " \n\t"
305 #define FOP_RET "ret \n\t"
307 #define FOP_START(op) \
308 extern void em_##op(struct fastop *fake); \
309 asm(".pushsection .text, \"ax\" \n\t" \
310 ".global em_" #op " \n\t" \
317 #define FOPNOP() FOP_ALIGN FOP_RET
319 #define FOP1E(op, dst) \
320 FOP_ALIGN "10: " #op " %" #dst " \n\t" FOP_RET
322 #define FOP1EEX(op, dst) \
323 FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
325 #define FASTOP1(op) \
330 ON64(FOP1E(op##q, rax)) \
333 /* 1-operand, using src2 (for MUL/DIV r/m) */
334 #define FASTOP1SRC2(op, name) \
339 ON64(FOP1E(op, rcx)) \
342 /* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
343 #define FASTOP1SRC2EX(op, name) \
348 ON64(FOP1EEX(op, rcx)) \
351 #define FOP2E(op, dst, src) \
352 FOP_ALIGN #op " %" #src ", %" #dst " \n\t" FOP_RET
354 #define FASTOP2(op) \
356 FOP2E(op##b, al, dl) \
357 FOP2E(op##w, ax, dx) \
358 FOP2E(op##l, eax, edx) \
359 ON64(FOP2E(op##q, rax, rdx)) \
362 /* 2 operand, word only */
363 #define FASTOP2W(op) \
366 FOP2E(op##w, ax, dx) \
367 FOP2E(op##l, eax, edx) \
368 ON64(FOP2E(op##q, rax, rdx)) \
371 /* 2 operand, src is CL */
372 #define FASTOP2CL(op) \
374 FOP2E(op##b, al, cl) \
375 FOP2E(op##w, ax, cl) \
376 FOP2E(op##l, eax, cl) \
377 ON64(FOP2E(op##q, rax, cl)) \
380 #define FOP3E(op, dst, src, src2) \
381 FOP_ALIGN #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
383 /* 3-operand, word-only, src2=cl */
384 #define FASTOP3WCL(op) \
387 FOP3E(op##w, ax, dx, cl) \
388 FOP3E(op##l, eax, edx, cl) \
389 ON64(FOP3E(op##q, rax, rdx, cl)) \
392 /* Special case for SETcc - 1 instruction per cc */
393 #define FOP_SETCC(op) ".align 4; " #op " %al; ret \n\t"
395 asm(".global kvm_fastop_exception \n"
396 "kvm_fastop_exception: xor %esi, %esi; ret");
417 FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET
420 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
421 enum x86_intercept intercept,
422 enum x86_intercept_stage stage)
424 struct x86_instruction_info info = {
425 .intercept = intercept,
426 .rep_prefix = ctxt->rep_prefix,
427 .modrm_mod = ctxt->modrm_mod,
428 .modrm_reg = ctxt->modrm_reg,
429 .modrm_rm = ctxt->modrm_rm,
430 .src_val = ctxt->src.val64,
431 .dst_val = ctxt->dst.val64,
432 .src_bytes = ctxt->src.bytes,
433 .dst_bytes = ctxt->dst.bytes,
434 .ad_bytes = ctxt->ad_bytes,
435 .next_rip = ctxt->eip,
438 return ctxt->ops->intercept(ctxt, &info, stage);
441 static void assign_masked(ulong *dest, ulong src, ulong mask)
443 *dest = (*dest & ~mask) | (src & mask);
446 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
448 return (1UL << (ctxt->ad_bytes << 3)) - 1;
451 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
454 struct desc_struct ss;
456 if (ctxt->mode == X86EMUL_MODE_PROT64)
458 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
459 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
462 static int stack_size(struct x86_emulate_ctxt *ctxt)
464 return (__fls(stack_mask(ctxt)) + 1) >> 3;
467 /* Access/update address held in a register, based on addressing mode. */
468 static inline unsigned long
469 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
471 if (ctxt->ad_bytes == sizeof(unsigned long))
474 return reg & ad_mask(ctxt);
477 static inline unsigned long
478 register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
480 return address_mask(ctxt, reg);
483 static void masked_increment(ulong *reg, ulong mask, int inc)
485 assign_masked(reg, *reg + inc, mask);
489 register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
493 if (ctxt->ad_bytes == sizeof(unsigned long))
496 mask = ad_mask(ctxt);
497 masked_increment(reg, mask, inc);
500 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
502 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
505 static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
507 register_address_increment(ctxt, &ctxt->_eip, rel);
510 static u32 desc_limit_scaled(struct desc_struct *desc)
512 u32 limit = get_desc_limit(desc);
514 return desc->g ? (limit << 12) | 0xfff : limit;
517 static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
519 ctxt->has_seg_override = true;
520 ctxt->seg_override = seg;
523 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
525 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
528 return ctxt->ops->get_cached_segment_base(ctxt, seg);
531 static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
533 if (!ctxt->has_seg_override)
536 return ctxt->seg_override;
539 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
540 u32 error, bool valid)
542 ctxt->exception.vector = vec;
543 ctxt->exception.error_code = error;
544 ctxt->exception.error_code_valid = valid;
545 return X86EMUL_PROPAGATE_FAULT;
548 static int emulate_db(struct x86_emulate_ctxt *ctxt)
550 return emulate_exception(ctxt, DB_VECTOR, 0, false);
553 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
555 return emulate_exception(ctxt, GP_VECTOR, err, true);
558 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
560 return emulate_exception(ctxt, SS_VECTOR, err, true);
563 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
565 return emulate_exception(ctxt, UD_VECTOR, 0, false);
568 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
570 return emulate_exception(ctxt, TS_VECTOR, err, true);
573 static int emulate_de(struct x86_emulate_ctxt *ctxt)
575 return emulate_exception(ctxt, DE_VECTOR, 0, false);
578 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
580 return emulate_exception(ctxt, NM_VECTOR, 0, false);
583 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
586 struct desc_struct desc;
588 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
592 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
597 struct desc_struct desc;
599 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
600 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
604 * x86 defines three classes of vector instructions: explicitly
605 * aligned, explicitly unaligned, and the rest, which change behaviour
606 * depending on whether they're AVX encoded or not.
608 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
609 * subject to the same check.
611 static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
613 if (likely(size < 16))
616 if (ctxt->d & Aligned)
618 else if (ctxt->d & Unaligned)
620 else if (ctxt->d & Avx)
626 static int __linearize(struct x86_emulate_ctxt *ctxt,
627 struct segmented_address addr,
628 unsigned size, bool write, bool fetch,
631 struct desc_struct desc;
638 la = seg_base(ctxt, addr.seg) + addr.ea;
639 switch (ctxt->mode) {
640 case X86EMUL_MODE_PROT64:
641 if (((signed long)la << 16) >> 16 != la)
642 return emulate_gp(ctxt, 0);
645 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
649 /* code segment in protected mode or read-only data segment */
650 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
651 || !(desc.type & 2)) && write)
653 /* unreadable code segment */
654 if (!fetch && (desc.type & 8) && !(desc.type & 2))
656 lim = desc_limit_scaled(&desc);
657 if ((desc.type & 8) || !(desc.type & 4)) {
658 /* expand-up segment */
659 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
662 /* expand-down segment */
663 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
665 lim = desc.d ? 0xffffffff : 0xffff;
666 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
669 cpl = ctxt->ops->cpl(ctxt);
670 if (!(desc.type & 8)) {
674 } else if ((desc.type & 8) && !(desc.type & 4)) {
675 /* nonconforming code segment */
678 } else if ((desc.type & 8) && (desc.type & 4)) {
679 /* conforming code segment */
685 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
687 if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
688 return emulate_gp(ctxt, 0);
690 return X86EMUL_CONTINUE;
692 if (addr.seg == VCPU_SREG_SS)
693 return emulate_ss(ctxt, sel);
695 return emulate_gp(ctxt, sel);
698 static int linearize(struct x86_emulate_ctxt *ctxt,
699 struct segmented_address addr,
700 unsigned size, bool write,
703 return __linearize(ctxt, addr, size, write, false, linear);
707 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
708 struct segmented_address addr,
715 rc = linearize(ctxt, addr, size, false, &linear);
716 if (rc != X86EMUL_CONTINUE)
718 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
722 * Fetch the next byte of the instruction being emulated which is pointed to
723 * by ctxt->_eip, then increment ctxt->_eip.
725 * Also prefetch the remaining bytes of the instruction without crossing page
726 * boundary if they are not in fetch_cache yet.
728 static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
730 struct fetch_cache *fc = &ctxt->fetch;
734 if (ctxt->_eip == fc->end) {
735 unsigned long linear;
736 struct segmented_address addr = { .seg = VCPU_SREG_CS,
738 cur_size = fc->end - fc->start;
739 size = min(15UL - cur_size,
740 PAGE_SIZE - offset_in_page(ctxt->_eip));
741 rc = __linearize(ctxt, addr, size, false, true, &linear);
742 if (unlikely(rc != X86EMUL_CONTINUE))
744 rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
745 size, &ctxt->exception);
746 if (unlikely(rc != X86EMUL_CONTINUE))
750 *dest = fc->data[ctxt->_eip - fc->start];
752 return X86EMUL_CONTINUE;
755 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
756 void *dest, unsigned size)
760 /* x86 instructions are limited to 15 bytes. */
761 if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
762 return X86EMUL_UNHANDLEABLE;
764 rc = do_insn_fetch_byte(ctxt, dest++);
765 if (rc != X86EMUL_CONTINUE)
768 return X86EMUL_CONTINUE;
771 /* Fetch next part of the instruction being emulated. */
772 #define insn_fetch(_type, _ctxt) \
773 ({ unsigned long _x; \
774 rc = do_insn_fetch(_ctxt, &_x, sizeof(_type)); \
775 if (rc != X86EMUL_CONTINUE) \
780 #define insn_fetch_arr(_arr, _size, _ctxt) \
781 ({ rc = do_insn_fetch(_ctxt, _arr, (_size)); \
782 if (rc != X86EMUL_CONTINUE) \
787 * Given the 'reg' portion of a ModRM byte, and a register block, return a
788 * pointer into the block that addresses the relevant register.
789 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
791 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
795 int highbyte_regs = (ctxt->rex_prefix == 0) && byteop;
797 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
798 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
800 p = reg_rmw(ctxt, modrm_reg);
804 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
805 struct segmented_address addr,
806 u16 *size, unsigned long *address, int op_bytes)
813 rc = segmented_read_std(ctxt, addr, size, 2);
814 if (rc != X86EMUL_CONTINUE)
817 rc = segmented_read_std(ctxt, addr, address, op_bytes);
831 FASTOP1SRC2(mul, mul_ex);
832 FASTOP1SRC2(imul, imul_ex);
833 FASTOP1SRC2EX(div, div_ex);
834 FASTOP1SRC2EX(idiv, idiv_ex);
863 static u8 test_cc(unsigned int condition, unsigned long flags)
866 void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
868 flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
869 asm("push %[flags]; popf; call *%[fastop]"
870 : "=a"(rc) : [fastop]"r"(fop), [flags]"r"(flags));
874 static void fetch_register_operand(struct operand *op)
878 op->val = *(u8 *)op->addr.reg;
881 op->val = *(u16 *)op->addr.reg;
884 op->val = *(u32 *)op->addr.reg;
887 op->val = *(u64 *)op->addr.reg;
892 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
894 ctxt->ops->get_fpu(ctxt);
896 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
897 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
898 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
899 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
900 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
901 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
902 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
903 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
905 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
906 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
907 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
908 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
909 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
910 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
911 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
912 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
916 ctxt->ops->put_fpu(ctxt);
919 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
922 ctxt->ops->get_fpu(ctxt);
924 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
925 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
926 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
927 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
928 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
929 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
930 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
931 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
933 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
934 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
935 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
936 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
937 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
938 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
939 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
940 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
944 ctxt->ops->put_fpu(ctxt);
947 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
949 ctxt->ops->get_fpu(ctxt);
951 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
952 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
953 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
954 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
955 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
956 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
957 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
958 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
961 ctxt->ops->put_fpu(ctxt);
964 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
966 ctxt->ops->get_fpu(ctxt);
968 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
969 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
970 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
971 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
972 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
973 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
974 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
975 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
978 ctxt->ops->put_fpu(ctxt);
981 static int em_fninit(struct x86_emulate_ctxt *ctxt)
983 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
984 return emulate_nm(ctxt);
986 ctxt->ops->get_fpu(ctxt);
987 asm volatile("fninit");
988 ctxt->ops->put_fpu(ctxt);
989 return X86EMUL_CONTINUE;
992 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
996 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
997 return emulate_nm(ctxt);
999 ctxt->ops->get_fpu(ctxt);
1000 asm volatile("fnstcw %0": "+m"(fcw));
1001 ctxt->ops->put_fpu(ctxt);
1003 /* force 2 byte destination */
1004 ctxt->dst.bytes = 2;
1005 ctxt->dst.val = fcw;
1007 return X86EMUL_CONTINUE;
1010 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1014 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1015 return emulate_nm(ctxt);
1017 ctxt->ops->get_fpu(ctxt);
1018 asm volatile("fnstsw %0": "+m"(fsw));
1019 ctxt->ops->put_fpu(ctxt);
1021 /* force 2 byte destination */
1022 ctxt->dst.bytes = 2;
1023 ctxt->dst.val = fsw;
1025 return X86EMUL_CONTINUE;
1028 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1031 unsigned reg = ctxt->modrm_reg;
1033 if (!(ctxt->d & ModRM))
1034 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1036 if (ctxt->d & Sse) {
1040 read_sse_reg(ctxt, &op->vec_val, reg);
1043 if (ctxt->d & Mmx) {
1052 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1053 op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);
1055 fetch_register_operand(op);
1056 op->orig_val = op->val;
1059 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1061 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1062 ctxt->modrm_seg = VCPU_SREG_SS;
1065 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1069 int index_reg, base_reg, scale;
1070 int rc = X86EMUL_CONTINUE;
1073 ctxt->modrm_reg = ((ctxt->rex_prefix << 1) & 8); /* REX.R */
1074 index_reg = (ctxt->rex_prefix << 2) & 8; /* REX.X */
1075 base_reg = (ctxt->rex_prefix << 3) & 8; /* REX.B */
1077 ctxt->modrm_mod = (ctxt->modrm & 0xc0) >> 6;
1078 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1079 ctxt->modrm_rm = base_reg | (ctxt->modrm & 0x07);
1080 ctxt->modrm_seg = VCPU_SREG_DS;
1082 if (ctxt->modrm_mod == 3 || (ctxt->d & NoMod)) {
1084 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1085 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
1087 if (ctxt->d & Sse) {
1090 op->addr.xmm = ctxt->modrm_rm;
1091 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1094 if (ctxt->d & Mmx) {
1097 op->addr.mm = ctxt->modrm_rm & 7;
1100 fetch_register_operand(op);
1106 if (ctxt->ad_bytes == 2) {
1107 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1108 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1109 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1110 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1112 /* 16-bit ModR/M decode. */
1113 switch (ctxt->modrm_mod) {
1115 if (ctxt->modrm_rm == 6)
1116 modrm_ea += insn_fetch(u16, ctxt);
1119 modrm_ea += insn_fetch(s8, ctxt);
1122 modrm_ea += insn_fetch(u16, ctxt);
1125 switch (ctxt->modrm_rm) {
1127 modrm_ea += bx + si;
1130 modrm_ea += bx + di;
1133 modrm_ea += bp + si;
1136 modrm_ea += bp + di;
1145 if (ctxt->modrm_mod != 0)
1152 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1153 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1154 ctxt->modrm_seg = VCPU_SREG_SS;
1155 modrm_ea = (u16)modrm_ea;
1157 /* 32/64-bit ModR/M decode. */
1158 if ((ctxt->modrm_rm & 7) == 4) {
1159 sib = insn_fetch(u8, ctxt);
1160 index_reg |= (sib >> 3) & 7;
1161 base_reg |= sib & 7;
1164 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1165 modrm_ea += insn_fetch(s32, ctxt);
1167 modrm_ea += reg_read(ctxt, base_reg);
1168 adjust_modrm_seg(ctxt, base_reg);
1171 modrm_ea += reg_read(ctxt, index_reg) << scale;
1172 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1173 if (ctxt->mode == X86EMUL_MODE_PROT64)
1174 ctxt->rip_relative = 1;
1176 base_reg = ctxt->modrm_rm;
1177 modrm_ea += reg_read(ctxt, base_reg);
1178 adjust_modrm_seg(ctxt, base_reg);
1180 switch (ctxt->modrm_mod) {
1182 if (ctxt->modrm_rm == 5)
1183 modrm_ea += insn_fetch(s32, ctxt);
1186 modrm_ea += insn_fetch(s8, ctxt);
1189 modrm_ea += insn_fetch(s32, ctxt);
1193 op->addr.mem.ea = modrm_ea;
1198 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1201 int rc = X86EMUL_CONTINUE;
1204 switch (ctxt->ad_bytes) {
1206 op->addr.mem.ea = insn_fetch(u16, ctxt);
1209 op->addr.mem.ea = insn_fetch(u32, ctxt);
1212 op->addr.mem.ea = insn_fetch(u64, ctxt);
1219 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1223 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1224 mask = ~((long)ctxt->dst.bytes * 8 - 1);
1226 if (ctxt->src.bytes == 2)
1227 sv = (s16)ctxt->src.val & (s16)mask;
1228 else if (ctxt->src.bytes == 4)
1229 sv = (s32)ctxt->src.val & (s32)mask;
1231 sv = (s64)ctxt->src.val & (s64)mask;
1233 ctxt->dst.addr.mem.ea += (sv >> 3);
1236 /* only subword offset */
1237 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1240 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1241 unsigned long addr, void *dest, unsigned size)
1244 struct read_cache *mc = &ctxt->mem_read;
1246 if (mc->pos < mc->end)
1249 WARN_ON((mc->end + size) >= sizeof(mc->data));
1251 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1253 if (rc != X86EMUL_CONTINUE)
1259 memcpy(dest, mc->data + mc->pos, size);
1261 return X86EMUL_CONTINUE;
1264 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1265 struct segmented_address addr,
1272 rc = linearize(ctxt, addr, size, false, &linear);
1273 if (rc != X86EMUL_CONTINUE)
1275 return read_emulated(ctxt, linear, data, size);
1278 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1279 struct segmented_address addr,
1286 rc = linearize(ctxt, addr, size, true, &linear);
1287 if (rc != X86EMUL_CONTINUE)
1289 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1293 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1294 struct segmented_address addr,
1295 const void *orig_data, const void *data,
1301 rc = linearize(ctxt, addr, size, true, &linear);
1302 if (rc != X86EMUL_CONTINUE)
1304 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1305 size, &ctxt->exception);
1308 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1309 unsigned int size, unsigned short port,
1312 struct read_cache *rc = &ctxt->io_read;
1314 if (rc->pos == rc->end) { /* refill pio read ahead */
1315 unsigned int in_page, n;
1316 unsigned int count = ctxt->rep_prefix ?
1317 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1318 in_page = (ctxt->eflags & EFLG_DF) ?
1319 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1320 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1321 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1325 rc->pos = rc->end = 0;
1326 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1331 if (ctxt->rep_prefix && (ctxt->d & String) &&
1332 !(ctxt->eflags & EFLG_DF)) {
1333 ctxt->dst.data = rc->data + rc->pos;
1334 ctxt->dst.type = OP_MEM_STR;
1335 ctxt->dst.count = (rc->end - rc->pos) / size;
1338 memcpy(dest, rc->data + rc->pos, size);
1344 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1345 u16 index, struct desc_struct *desc)
1350 ctxt->ops->get_idt(ctxt, &dt);
1352 if (dt.size < index * 8 + 7)
1353 return emulate_gp(ctxt, index << 3 | 0x2);
1355 addr = dt.address + index * 8;
1356 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1360 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1361 u16 selector, struct desc_ptr *dt)
1363 const struct x86_emulate_ops *ops = ctxt->ops;
1366 if (selector & 1 << 2) {
1367 struct desc_struct desc;
1370 memset (dt, 0, sizeof *dt);
1371 if (!ops->get_segment(ctxt, &sel, &desc, &base3,
1375 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1376 dt->address = get_desc_base(&desc) | ((u64)base3 << 32);
1378 ops->get_gdt(ctxt, dt);
1381 /* allowed just for 8 bytes segments */
1382 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1383 u16 selector, struct desc_struct *desc,
1387 u16 index = selector >> 3;
1390 get_descriptor_table_ptr(ctxt, selector, &dt);
1392 if (dt.size < index * 8 + 7)
1393 return emulate_gp(ctxt, selector & 0xfffc);
1395 *desc_addr_p = addr = dt.address + index * 8;
1396 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1400 /* allowed just for 8 bytes segments */
1401 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1402 u16 selector, struct desc_struct *desc)
1405 u16 index = selector >> 3;
1408 get_descriptor_table_ptr(ctxt, selector, &dt);
1410 if (dt.size < index * 8 + 7)
1411 return emulate_gp(ctxt, selector & 0xfffc);
1413 addr = dt.address + index * 8;
1414 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1418 /* Does not support long mode */
1419 static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1420 u16 selector, int seg, u8 cpl, bool in_task_switch)
1422 struct desc_struct seg_desc, old_desc;
1424 unsigned err_vec = GP_VECTOR;
1426 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1432 memset(&seg_desc, 0, sizeof seg_desc);
1434 if (ctxt->mode == X86EMUL_MODE_REAL) {
1435 /* set real mode segment descriptor (keep limit etc. for
1437 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1438 set_desc_base(&seg_desc, selector << 4);
1440 } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
1441 /* VM86 needs a clean new segment descriptor */
1442 set_desc_base(&seg_desc, selector << 4);
1443 set_desc_limit(&seg_desc, 0xffff);
1453 /* NULL selector is not valid for TR, CS and SS (except for long mode) */
1454 if ((seg == VCPU_SREG_CS
1455 || (seg == VCPU_SREG_SS
1456 && (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl))
1457 || seg == VCPU_SREG_TR)
1461 /* TR should be in GDT only */
1462 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1465 if (null_selector) /* for NULL selector skip all following checks */
1468 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1469 if (ret != X86EMUL_CONTINUE)
1472 err_code = selector & 0xfffc;
1473 err_vec = GP_VECTOR;
1475 /* can't load system descriptor into segment selector */
1476 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1480 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1489 * segment is not a writable data segment or segment
1490 * selector's RPL != CPL or segment selector's RPL != CPL
1492 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1496 if (in_task_switch && rpl != dpl)
1499 if (!(seg_desc.type & 8))
1502 if (seg_desc.type & 4) {
1508 if (rpl > cpl || dpl != cpl)
1511 /* CS(RPL) <- CPL */
1512 selector = (selector & 0xfffc) | cpl;
1515 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1517 old_desc = seg_desc;
1518 seg_desc.type |= 2; /* busy */
1519 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1520 sizeof(seg_desc), &ctxt->exception);
1521 if (ret != X86EMUL_CONTINUE)
1524 case VCPU_SREG_LDTR:
1525 if (seg_desc.s || seg_desc.type != 2)
1528 default: /* DS, ES, FS, or GS */
1530 * segment is not a data or readable code segment or
1531 * ((segment is a data or nonconforming code segment)
1532 * and (both RPL and CPL > DPL))
1534 if ((seg_desc.type & 0xa) == 0x8 ||
1535 (((seg_desc.type & 0xc) != 0xc) &&
1536 (rpl > dpl && cpl > dpl)))
1542 /* mark segment as accessed */
1544 ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1545 if (ret != X86EMUL_CONTINUE)
1547 } else if (ctxt->mode == X86EMUL_MODE_PROT64) {
1548 ret = ctxt->ops->read_std(ctxt, desc_addr+8, &base3,
1549 sizeof(base3), &ctxt->exception);
1550 if (ret != X86EMUL_CONTINUE)
1554 ctxt->ops->set_segment(ctxt, selector, &seg_desc, base3, seg);
1555 return X86EMUL_CONTINUE;
1557 emulate_exception(ctxt, err_vec, err_code, true);
1558 return X86EMUL_PROPAGATE_FAULT;
1561 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1562 u16 selector, int seg)
1564 u8 cpl = ctxt->ops->cpl(ctxt);
1565 return __load_segment_descriptor(ctxt, selector, seg, cpl, false);
1568 static void write_register_operand(struct operand *op)
1570 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1571 switch (op->bytes) {
1573 *(u8 *)op->addr.reg = (u8)op->val;
1576 *(u16 *)op->addr.reg = (u16)op->val;
1579 *op->addr.reg = (u32)op->val;
1580 break; /* 64b: zero-extend */
1582 *op->addr.reg = op->val;
1587 static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
1591 write_register_operand(op);
1594 if (ctxt->lock_prefix)
1595 return segmented_cmpxchg(ctxt,
1601 return segmented_write(ctxt,
1607 return segmented_write(ctxt,
1610 op->bytes * op->count);
1613 write_sse_reg(ctxt, &op->vec_val, op->addr.xmm);
1616 write_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
1624 return X86EMUL_CONTINUE;
1627 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1629 struct segmented_address addr;
1631 rsp_increment(ctxt, -bytes);
1632 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1633 addr.seg = VCPU_SREG_SS;
1635 return segmented_write(ctxt, addr, data, bytes);
1638 static int em_push(struct x86_emulate_ctxt *ctxt)
1640 /* Disable writeback. */
1641 ctxt->dst.type = OP_NONE;
1642 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1645 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1646 void *dest, int len)
1649 struct segmented_address addr;
1651 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1652 addr.seg = VCPU_SREG_SS;
1653 rc = segmented_read(ctxt, addr, dest, len);
1654 if (rc != X86EMUL_CONTINUE)
1657 rsp_increment(ctxt, len);
1661 static int em_pop(struct x86_emulate_ctxt *ctxt)
1663 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1666 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1667 void *dest, int len)
1670 unsigned long val, change_mask;
1671 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1672 int cpl = ctxt->ops->cpl(ctxt);
1674 rc = emulate_pop(ctxt, &val, len);
1675 if (rc != X86EMUL_CONTINUE)
1678 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1679 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1681 switch(ctxt->mode) {
1682 case X86EMUL_MODE_PROT64:
1683 case X86EMUL_MODE_PROT32:
1684 case X86EMUL_MODE_PROT16:
1686 change_mask |= EFLG_IOPL;
1688 change_mask |= EFLG_IF;
1690 case X86EMUL_MODE_VM86:
1692 return emulate_gp(ctxt, 0);
1693 change_mask |= EFLG_IF;
1695 default: /* real mode */
1696 change_mask |= (EFLG_IOPL | EFLG_IF);
1700 *(unsigned long *)dest =
1701 (ctxt->eflags & ~change_mask) | (val & change_mask);
1706 static int em_popf(struct x86_emulate_ctxt *ctxt)
1708 ctxt->dst.type = OP_REG;
1709 ctxt->dst.addr.reg = &ctxt->eflags;
1710 ctxt->dst.bytes = ctxt->op_bytes;
1711 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1714 static int em_enter(struct x86_emulate_ctxt *ctxt)
1717 unsigned frame_size = ctxt->src.val;
1718 unsigned nesting_level = ctxt->src2.val & 31;
1722 return X86EMUL_UNHANDLEABLE;
1724 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1725 rc = push(ctxt, &rbp, stack_size(ctxt));
1726 if (rc != X86EMUL_CONTINUE)
1728 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1730 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1731 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1733 return X86EMUL_CONTINUE;
1736 static int em_leave(struct x86_emulate_ctxt *ctxt)
1738 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1740 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1743 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1745 int seg = ctxt->src2.val;
1747 ctxt->src.val = get_segment_selector(ctxt, seg);
1749 return em_push(ctxt);
1752 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1754 int seg = ctxt->src2.val;
1755 unsigned long selector;
1758 rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1759 if (rc != X86EMUL_CONTINUE)
1762 if (ctxt->modrm_reg == VCPU_SREG_SS)
1763 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
1765 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1769 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1771 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1772 int rc = X86EMUL_CONTINUE;
1773 int reg = VCPU_REGS_RAX;
1775 while (reg <= VCPU_REGS_RDI) {
1776 (reg == VCPU_REGS_RSP) ?
1777 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
1780 if (rc != X86EMUL_CONTINUE)
1789 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1791 ctxt->src.val = (unsigned long)ctxt->eflags;
1792 return em_push(ctxt);
1795 static int em_popa(struct x86_emulate_ctxt *ctxt)
1797 int rc = X86EMUL_CONTINUE;
1798 int reg = VCPU_REGS_RDI;
1800 while (reg >= VCPU_REGS_RAX) {
1801 if (reg == VCPU_REGS_RSP) {
1802 rsp_increment(ctxt, ctxt->op_bytes);
1806 rc = emulate_pop(ctxt, reg_rmw(ctxt, reg), ctxt->op_bytes);
1807 if (rc != X86EMUL_CONTINUE)
1814 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1816 const struct x86_emulate_ops *ops = ctxt->ops;
1823 /* TODO: Add limit checks */
1824 ctxt->src.val = ctxt->eflags;
1826 if (rc != X86EMUL_CONTINUE)
1829 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1831 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1833 if (rc != X86EMUL_CONTINUE)
1836 ctxt->src.val = ctxt->_eip;
1838 if (rc != X86EMUL_CONTINUE)
1841 ops->get_idt(ctxt, &dt);
1843 eip_addr = dt.address + (irq << 2);
1844 cs_addr = dt.address + (irq << 2) + 2;
1846 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1847 if (rc != X86EMUL_CONTINUE)
1850 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1851 if (rc != X86EMUL_CONTINUE)
1854 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1855 if (rc != X86EMUL_CONTINUE)
1863 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1867 invalidate_registers(ctxt);
1868 rc = __emulate_int_real(ctxt, irq);
1869 if (rc == X86EMUL_CONTINUE)
1870 writeback_registers(ctxt);
1874 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1876 switch(ctxt->mode) {
1877 case X86EMUL_MODE_REAL:
1878 return __emulate_int_real(ctxt, irq);
1879 case X86EMUL_MODE_VM86:
1880 case X86EMUL_MODE_PROT16:
1881 case X86EMUL_MODE_PROT32:
1882 case X86EMUL_MODE_PROT64:
1884 /* Protected mode interrupts unimplemented yet */
1885 return X86EMUL_UNHANDLEABLE;
1889 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
1891 int rc = X86EMUL_CONTINUE;
1892 unsigned long temp_eip = 0;
1893 unsigned long temp_eflags = 0;
1894 unsigned long cs = 0;
1895 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1896 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1897 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1898 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1900 /* TODO: Add stack limit check */
1902 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
1904 if (rc != X86EMUL_CONTINUE)
1907 if (temp_eip & ~0xffff)
1908 return emulate_gp(ctxt, 0);
1910 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1912 if (rc != X86EMUL_CONTINUE)
1915 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
1917 if (rc != X86EMUL_CONTINUE)
1920 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1922 if (rc != X86EMUL_CONTINUE)
1925 ctxt->_eip = temp_eip;
1928 if (ctxt->op_bytes == 4)
1929 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1930 else if (ctxt->op_bytes == 2) {
1931 ctxt->eflags &= ~0xffff;
1932 ctxt->eflags |= temp_eflags;
1935 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1936 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1941 static int em_iret(struct x86_emulate_ctxt *ctxt)
1943 switch(ctxt->mode) {
1944 case X86EMUL_MODE_REAL:
1945 return emulate_iret_real(ctxt);
1946 case X86EMUL_MODE_VM86:
1947 case X86EMUL_MODE_PROT16:
1948 case X86EMUL_MODE_PROT32:
1949 case X86EMUL_MODE_PROT64:
1951 /* iret from protected mode unimplemented yet */
1952 return X86EMUL_UNHANDLEABLE;
1956 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
1961 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1963 rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
1964 if (rc != X86EMUL_CONTINUE)
1968 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
1969 return X86EMUL_CONTINUE;
1972 static int em_grp45(struct x86_emulate_ctxt *ctxt)
1974 int rc = X86EMUL_CONTINUE;
1976 switch (ctxt->modrm_reg) {
1977 case 2: /* call near abs */ {
1979 old_eip = ctxt->_eip;
1980 ctxt->_eip = ctxt->src.val;
1981 ctxt->src.val = old_eip;
1985 case 4: /* jmp abs */
1986 ctxt->_eip = ctxt->src.val;
1988 case 5: /* jmp far */
1989 rc = em_jmp_far(ctxt);
1998 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2000 u64 old = ctxt->dst.orig_val64;
2002 if (ctxt->dst.bytes == 16)
2003 return X86EMUL_UNHANDLEABLE;
2005 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
2006 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2007 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2008 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2009 ctxt->eflags &= ~EFLG_ZF;
2011 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2012 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2014 ctxt->eflags |= EFLG_ZF;
2016 return X86EMUL_CONTINUE;
2019 static int em_ret(struct x86_emulate_ctxt *ctxt)
2021 ctxt->dst.type = OP_REG;
2022 ctxt->dst.addr.reg = &ctxt->_eip;
2023 ctxt->dst.bytes = ctxt->op_bytes;
2024 return em_pop(ctxt);
2027 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2031 int cpl = ctxt->ops->cpl(ctxt);
2033 rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
2034 if (rc != X86EMUL_CONTINUE)
2036 if (ctxt->op_bytes == 4)
2037 ctxt->_eip = (u32)ctxt->_eip;
2038 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2039 if (rc != X86EMUL_CONTINUE)
2041 /* Outer-privilege level return is not implemented */
2042 if (ctxt->mode >= X86EMUL_MODE_PROT16 && (cs & 3) > cpl)
2043 return X86EMUL_UNHANDLEABLE;
2044 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2048 static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt)
2052 rc = em_ret_far(ctxt);
2053 if (rc != X86EMUL_CONTINUE)
2055 rsp_increment(ctxt, ctxt->src.val);
2056 return X86EMUL_CONTINUE;
2059 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2061 /* Save real source value, then compare EAX against destination. */
2062 ctxt->dst.orig_val = ctxt->dst.val;
2063 ctxt->dst.val = reg_read(ctxt, VCPU_REGS_RAX);
2064 ctxt->src.orig_val = ctxt->src.val;
2065 ctxt->src.val = ctxt->dst.orig_val;
2066 fastop(ctxt, em_cmp);
2068 if (ctxt->eflags & EFLG_ZF) {
2069 /* Success: write back to memory. */
2070 ctxt->dst.val = ctxt->src.orig_val;
2072 /* Failure: write the value we saw to EAX. */
2073 ctxt->dst.type = OP_REG;
2074 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2075 ctxt->dst.val = ctxt->dst.orig_val;
2077 return X86EMUL_CONTINUE;
2080 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2082 int seg = ctxt->src2.val;
2086 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2088 rc = load_segment_descriptor(ctxt, sel, seg);
2089 if (rc != X86EMUL_CONTINUE)
2092 ctxt->dst.val = ctxt->src.val;
2097 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2098 struct desc_struct *cs, struct desc_struct *ss)
2100 cs->l = 0; /* will be adjusted later */
2101 set_desc_base(cs, 0); /* flat segment */
2102 cs->g = 1; /* 4kb granularity */
2103 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2104 cs->type = 0x0b; /* Read, Execute, Accessed */
2106 cs->dpl = 0; /* will be adjusted later */
2111 set_desc_base(ss, 0); /* flat segment */
2112 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2113 ss->g = 1; /* 4kb granularity */
2115 ss->type = 0x03; /* Read/Write, Accessed */
2116 ss->d = 1; /* 32bit stack segment */
2123 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2125 u32 eax, ebx, ecx, edx;
2128 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2129 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2130 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2131 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2134 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2136 const struct x86_emulate_ops *ops = ctxt->ops;
2137 u32 eax, ebx, ecx, edx;
2140 * syscall should always be enabled in longmode - so only become
2141 * vendor specific (cpuid) if other modes are active...
2143 if (ctxt->mode == X86EMUL_MODE_PROT64)
2148 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2150 * Intel ("GenuineIntel")
2151 * remark: Intel CPUs only support "syscall" in 64bit
2152 * longmode. Also an 64bit guest with a
2153 * 32bit compat-app running will #UD !! While this
2154 * behaviour can be fixed (by emulating) into AMD
2155 * response - CPUs of AMD can't behave like Intel.
2157 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2158 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2159 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2162 /* AMD ("AuthenticAMD") */
2163 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2164 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2165 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2168 /* AMD ("AMDisbetter!") */
2169 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2170 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2171 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2174 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2178 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2180 const struct x86_emulate_ops *ops = ctxt->ops;
2181 struct desc_struct cs, ss;
2186 /* syscall is not available in real mode */
2187 if (ctxt->mode == X86EMUL_MODE_REAL ||
2188 ctxt->mode == X86EMUL_MODE_VM86)
2189 return emulate_ud(ctxt);
2191 if (!(em_syscall_is_enabled(ctxt)))
2192 return emulate_ud(ctxt);
2194 ops->get_msr(ctxt, MSR_EFER, &efer);
2195 setup_syscalls_segments(ctxt, &cs, &ss);
2197 if (!(efer & EFER_SCE))
2198 return emulate_ud(ctxt);
2200 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2202 cs_sel = (u16)(msr_data & 0xfffc);
2203 ss_sel = (u16)(msr_data + 8);
2205 if (efer & EFER_LMA) {
2209 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2210 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2212 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2213 if (efer & EFER_LMA) {
2214 #ifdef CONFIG_X86_64
2215 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags & ~EFLG_RF;
2218 ctxt->mode == X86EMUL_MODE_PROT64 ?
2219 MSR_LSTAR : MSR_CSTAR, &msr_data);
2220 ctxt->_eip = msr_data;
2222 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2223 ctxt->eflags &= ~(msr_data | EFLG_RF);
2227 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2228 ctxt->_eip = (u32)msr_data;
2230 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2233 return X86EMUL_CONTINUE;
2236 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2238 const struct x86_emulate_ops *ops = ctxt->ops;
2239 struct desc_struct cs, ss;
2244 ops->get_msr(ctxt, MSR_EFER, &efer);
2245 /* inject #GP if in real mode */
2246 if (ctxt->mode == X86EMUL_MODE_REAL)
2247 return emulate_gp(ctxt, 0);
2250 * Not recognized on AMD in compat mode (but is recognized in legacy
2253 if ((ctxt->mode == X86EMUL_MODE_PROT32) && (efer & EFER_LMA)
2254 && !vendor_intel(ctxt))
2255 return emulate_ud(ctxt);
2257 /* XXX sysenter/sysexit have not been tested in 64bit mode.
2258 * Therefore, we inject an #UD.
2260 if (ctxt->mode == X86EMUL_MODE_PROT64)
2261 return emulate_ud(ctxt);
2263 setup_syscalls_segments(ctxt, &cs, &ss);
2265 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2266 switch (ctxt->mode) {
2267 case X86EMUL_MODE_PROT32:
2268 if ((msr_data & 0xfffc) == 0x0)
2269 return emulate_gp(ctxt, 0);
2271 case X86EMUL_MODE_PROT64:
2272 if (msr_data == 0x0)
2273 return emulate_gp(ctxt, 0);
2279 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2280 cs_sel = (u16)msr_data;
2281 cs_sel &= ~SELECTOR_RPL_MASK;
2282 ss_sel = cs_sel + 8;
2283 ss_sel &= ~SELECTOR_RPL_MASK;
2284 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
2289 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2290 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2292 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2293 ctxt->_eip = msr_data;
2295 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2296 *reg_write(ctxt, VCPU_REGS_RSP) = msr_data;
2298 return X86EMUL_CONTINUE;
2301 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2303 const struct x86_emulate_ops *ops = ctxt->ops;
2304 struct desc_struct cs, ss;
2307 u16 cs_sel = 0, ss_sel = 0;
2309 /* inject #GP if in real mode or Virtual 8086 mode */
2310 if (ctxt->mode == X86EMUL_MODE_REAL ||
2311 ctxt->mode == X86EMUL_MODE_VM86)
2312 return emulate_gp(ctxt, 0);
2314 setup_syscalls_segments(ctxt, &cs, &ss);
2316 if ((ctxt->rex_prefix & 0x8) != 0x0)
2317 usermode = X86EMUL_MODE_PROT64;
2319 usermode = X86EMUL_MODE_PROT32;
2323 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2325 case X86EMUL_MODE_PROT32:
2326 cs_sel = (u16)(msr_data + 16);
2327 if ((msr_data & 0xfffc) == 0x0)
2328 return emulate_gp(ctxt, 0);
2329 ss_sel = (u16)(msr_data + 24);
2331 case X86EMUL_MODE_PROT64:
2332 cs_sel = (u16)(msr_data + 32);
2333 if (msr_data == 0x0)
2334 return emulate_gp(ctxt, 0);
2335 ss_sel = cs_sel + 8;
2340 cs_sel |= SELECTOR_RPL_MASK;
2341 ss_sel |= SELECTOR_RPL_MASK;
2343 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2344 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2346 ctxt->_eip = reg_read(ctxt, VCPU_REGS_RDX);
2347 *reg_write(ctxt, VCPU_REGS_RSP) = reg_read(ctxt, VCPU_REGS_RCX);
2349 return X86EMUL_CONTINUE;
2352 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2355 if (ctxt->mode == X86EMUL_MODE_REAL)
2357 if (ctxt->mode == X86EMUL_MODE_VM86)
2359 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2360 return ctxt->ops->cpl(ctxt) > iopl;
2363 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2366 const struct x86_emulate_ops *ops = ctxt->ops;
2367 struct desc_struct tr_seg;
2370 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2371 unsigned mask = (1 << len) - 1;
2374 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2377 if (desc_limit_scaled(&tr_seg) < 103)
2379 base = get_desc_base(&tr_seg);
2380 #ifdef CONFIG_X86_64
2381 base |= ((u64)base3) << 32;
2383 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2384 if (r != X86EMUL_CONTINUE)
2386 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2388 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2389 if (r != X86EMUL_CONTINUE)
2391 if ((perm >> bit_idx) & mask)
2396 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2402 if (emulator_bad_iopl(ctxt))
2403 if (!emulator_io_port_access_allowed(ctxt, port, len))
2406 ctxt->perm_ok = true;
2411 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2412 struct tss_segment_16 *tss)
2414 tss->ip = ctxt->_eip;
2415 tss->flag = ctxt->eflags;
2416 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
2417 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
2418 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
2419 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
2420 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
2421 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
2422 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
2423 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2425 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2426 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2427 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2428 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2429 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2432 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2433 struct tss_segment_16 *tss)
2438 ctxt->_eip = tss->ip;
2439 ctxt->eflags = tss->flag | 2;
2440 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
2441 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
2442 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
2443 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
2444 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
2445 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
2446 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
2447 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
2450 * SDM says that segment selectors are loaded before segment
2453 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2454 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2455 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2456 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2457 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2462 * Now load segment descriptors. If fault happens at this stage
2463 * it is handled in a context of new task
2465 ret = __load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR, cpl, true);
2466 if (ret != X86EMUL_CONTINUE)
2468 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl, true);
2469 if (ret != X86EMUL_CONTINUE)
2471 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl, true);
2472 if (ret != X86EMUL_CONTINUE)
2474 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl, true);
2475 if (ret != X86EMUL_CONTINUE)
2477 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl, true);
2478 if (ret != X86EMUL_CONTINUE)
2481 return X86EMUL_CONTINUE;
2484 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2485 u16 tss_selector, u16 old_tss_sel,
2486 ulong old_tss_base, struct desc_struct *new_desc)
2488 const struct x86_emulate_ops *ops = ctxt->ops;
2489 struct tss_segment_16 tss_seg;
2491 u32 new_tss_base = get_desc_base(new_desc);
2493 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2495 if (ret != X86EMUL_CONTINUE)
2496 /* FIXME: need to provide precise fault address */
2499 save_state_to_tss16(ctxt, &tss_seg);
2501 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2503 if (ret != X86EMUL_CONTINUE)
2504 /* FIXME: need to provide precise fault address */
2507 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2509 if (ret != X86EMUL_CONTINUE)
2510 /* FIXME: need to provide precise fault address */
2513 if (old_tss_sel != 0xffff) {
2514 tss_seg.prev_task_link = old_tss_sel;
2516 ret = ops->write_std(ctxt, new_tss_base,
2517 &tss_seg.prev_task_link,
2518 sizeof tss_seg.prev_task_link,
2520 if (ret != X86EMUL_CONTINUE)
2521 /* FIXME: need to provide precise fault address */
2525 return load_state_from_tss16(ctxt, &tss_seg);
2528 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2529 struct tss_segment_32 *tss)
2531 /* CR3 and ldt selector are not saved intentionally */
2532 tss->eip = ctxt->_eip;
2533 tss->eflags = ctxt->eflags;
2534 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
2535 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
2536 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
2537 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
2538 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
2539 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
2540 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
2541 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
2543 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2544 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2545 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2546 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2547 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2548 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2551 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2552 struct tss_segment_32 *tss)
2557 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2558 return emulate_gp(ctxt, 0);
2559 ctxt->_eip = tss->eip;
2560 ctxt->eflags = tss->eflags | 2;
2562 /* General purpose registers */
2563 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
2564 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
2565 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
2566 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
2567 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
2568 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
2569 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
2570 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
2573 * SDM says that segment selectors are loaded before segment
2574 * descriptors. This is important because CPL checks will
2577 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2578 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2579 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2580 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2581 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2582 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2583 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2586 * If we're switching between Protected Mode and VM86, we need to make
2587 * sure to update the mode before loading the segment descriptors so
2588 * that the selectors are interpreted correctly.
2590 if (ctxt->eflags & X86_EFLAGS_VM) {
2591 ctxt->mode = X86EMUL_MODE_VM86;
2594 ctxt->mode = X86EMUL_MODE_PROT32;
2599 * Now load segment descriptors. If fault happenes at this stage
2600 * it is handled in a context of new task
2602 ret = __load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR, cpl, true);
2603 if (ret != X86EMUL_CONTINUE)
2605 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl, true);
2606 if (ret != X86EMUL_CONTINUE)
2608 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl, true);
2609 if (ret != X86EMUL_CONTINUE)
2611 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl, true);
2612 if (ret != X86EMUL_CONTINUE)
2614 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl, true);
2615 if (ret != X86EMUL_CONTINUE)
2617 ret = __load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS, cpl, true);
2618 if (ret != X86EMUL_CONTINUE)
2620 ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl, true);
2621 if (ret != X86EMUL_CONTINUE)
2624 return X86EMUL_CONTINUE;
2627 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2628 u16 tss_selector, u16 old_tss_sel,
2629 ulong old_tss_base, struct desc_struct *new_desc)
2631 const struct x86_emulate_ops *ops = ctxt->ops;
2632 struct tss_segment_32 tss_seg;
2634 u32 new_tss_base = get_desc_base(new_desc);
2635 u32 eip_offset = offsetof(struct tss_segment_32, eip);
2636 u32 ldt_sel_offset = offsetof(struct tss_segment_32, ldt_selector);
2638 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2640 if (ret != X86EMUL_CONTINUE)
2641 /* FIXME: need to provide precise fault address */
2644 save_state_to_tss32(ctxt, &tss_seg);
2646 /* Only GP registers and segment selectors are saved */
2647 ret = ops->write_std(ctxt, old_tss_base + eip_offset, &tss_seg.eip,
2648 ldt_sel_offset - eip_offset, &ctxt->exception);
2649 if (ret != X86EMUL_CONTINUE)
2650 /* FIXME: need to provide precise fault address */
2653 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2655 if (ret != X86EMUL_CONTINUE)
2656 /* FIXME: need to provide precise fault address */
2659 if (old_tss_sel != 0xffff) {
2660 tss_seg.prev_task_link = old_tss_sel;
2662 ret = ops->write_std(ctxt, new_tss_base,
2663 &tss_seg.prev_task_link,
2664 sizeof tss_seg.prev_task_link,
2666 if (ret != X86EMUL_CONTINUE)
2667 /* FIXME: need to provide precise fault address */
2671 return load_state_from_tss32(ctxt, &tss_seg);
2674 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2675 u16 tss_selector, int idt_index, int reason,
2676 bool has_error_code, u32 error_code)
2678 const struct x86_emulate_ops *ops = ctxt->ops;
2679 struct desc_struct curr_tss_desc, next_tss_desc;
2681 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2682 ulong old_tss_base =
2683 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2687 /* FIXME: old_tss_base == ~0 ? */
2689 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
2690 if (ret != X86EMUL_CONTINUE)
2692 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
2693 if (ret != X86EMUL_CONTINUE)
2696 /* FIXME: check that next_tss_desc is tss */
2699 * Check privileges. The three cases are task switch caused by...
2701 * 1. jmp/call/int to task gate: Check against DPL of the task gate
2702 * 2. Exception/IRQ/iret: No check is performed
2703 * 3. jmp/call to TSS: Check against DPL of the TSS
2705 if (reason == TASK_SWITCH_GATE) {
2706 if (idt_index != -1) {
2707 /* Software interrupts */
2708 struct desc_struct task_gate_desc;
2711 ret = read_interrupt_descriptor(ctxt, idt_index,
2713 if (ret != X86EMUL_CONTINUE)
2716 dpl = task_gate_desc.dpl;
2717 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2718 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
2720 } else if (reason != TASK_SWITCH_IRET) {
2721 int dpl = next_tss_desc.dpl;
2722 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2723 return emulate_gp(ctxt, tss_selector);
2727 desc_limit = desc_limit_scaled(&next_tss_desc);
2728 if (!next_tss_desc.p ||
2729 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2730 desc_limit < 0x2b)) {
2731 emulate_ts(ctxt, tss_selector & 0xfffc);
2732 return X86EMUL_PROPAGATE_FAULT;
2735 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2736 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2737 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2740 if (reason == TASK_SWITCH_IRET)
2741 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2743 /* set back link to prev task only if NT bit is set in eflags
2744 note that old_tss_sel is not used after this point */
2745 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2746 old_tss_sel = 0xffff;
2748 if (next_tss_desc.type & 8)
2749 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2750 old_tss_base, &next_tss_desc);
2752 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2753 old_tss_base, &next_tss_desc);
2754 if (ret != X86EMUL_CONTINUE)
2757 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2758 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2760 if (reason != TASK_SWITCH_IRET) {
2761 next_tss_desc.type |= (1 << 1); /* set busy flag */
2762 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2765 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2766 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2768 if (has_error_code) {
2769 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2770 ctxt->lock_prefix = 0;
2771 ctxt->src.val = (unsigned long) error_code;
2772 ret = em_push(ctxt);
2778 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2779 u16 tss_selector, int idt_index, int reason,
2780 bool has_error_code, u32 error_code)
2784 invalidate_registers(ctxt);
2785 ctxt->_eip = ctxt->eip;
2786 ctxt->dst.type = OP_NONE;
2788 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
2789 has_error_code, error_code);
2791 if (rc == X86EMUL_CONTINUE) {
2792 ctxt->eip = ctxt->_eip;
2793 writeback_registers(ctxt);
2796 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2799 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
2802 int df = (ctxt->eflags & EFLG_DF) ? -op->count : op->count;
2804 register_address_increment(ctxt, reg_rmw(ctxt, reg), df * op->bytes);
2805 op->addr.mem.ea = register_address(ctxt, reg_read(ctxt, reg));
2808 static int em_das(struct x86_emulate_ctxt *ctxt)
2811 bool af, cf, old_cf;
2813 cf = ctxt->eflags & X86_EFLAGS_CF;
2819 af = ctxt->eflags & X86_EFLAGS_AF;
2820 if ((al & 0x0f) > 9 || af) {
2822 cf = old_cf | (al >= 250);
2827 if (old_al > 0x99 || old_cf) {
2833 /* Set PF, ZF, SF */
2834 ctxt->src.type = OP_IMM;
2836 ctxt->src.bytes = 1;
2837 fastop(ctxt, em_or);
2838 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2840 ctxt->eflags |= X86_EFLAGS_CF;
2842 ctxt->eflags |= X86_EFLAGS_AF;
2843 return X86EMUL_CONTINUE;
2846 static int em_aam(struct x86_emulate_ctxt *ctxt)
2850 if (ctxt->src.val == 0)
2851 return emulate_de(ctxt);
2853 al = ctxt->dst.val & 0xff;
2854 ah = al / ctxt->src.val;
2855 al %= ctxt->src.val;
2857 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);
2859 /* Set PF, ZF, SF */
2860 ctxt->src.type = OP_IMM;
2862 ctxt->src.bytes = 1;
2863 fastop(ctxt, em_or);
2865 return X86EMUL_CONTINUE;
2868 static int em_aad(struct x86_emulate_ctxt *ctxt)
2870 u8 al = ctxt->dst.val & 0xff;
2871 u8 ah = (ctxt->dst.val >> 8) & 0xff;
2873 al = (al + (ah * ctxt->src.val)) & 0xff;
2875 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
2877 /* Set PF, ZF, SF */
2878 ctxt->src.type = OP_IMM;
2880 ctxt->src.bytes = 1;
2881 fastop(ctxt, em_or);
2883 return X86EMUL_CONTINUE;
2886 static int em_call(struct x86_emulate_ctxt *ctxt)
2888 long rel = ctxt->src.val;
2890 ctxt->src.val = (unsigned long)ctxt->_eip;
2892 return em_push(ctxt);
2895 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2901 old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2902 old_eip = ctxt->_eip;
2904 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2905 if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
2906 return X86EMUL_CONTINUE;
2909 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2911 ctxt->src.val = old_cs;
2913 if (rc != X86EMUL_CONTINUE)
2916 ctxt->src.val = old_eip;
2917 return em_push(ctxt);
2920 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2924 ctxt->dst.type = OP_REG;
2925 ctxt->dst.addr.reg = &ctxt->_eip;
2926 ctxt->dst.bytes = ctxt->op_bytes;
2927 rc = emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
2928 if (rc != X86EMUL_CONTINUE)
2930 rsp_increment(ctxt, ctxt->src.val);
2931 return X86EMUL_CONTINUE;
2934 static int em_xchg(struct x86_emulate_ctxt *ctxt)
2936 /* Write back the register source. */
2937 ctxt->src.val = ctxt->dst.val;
2938 write_register_operand(&ctxt->src);
2940 /* Write back the memory destination with implicit LOCK prefix. */
2941 ctxt->dst.val = ctxt->src.orig_val;
2942 ctxt->lock_prefix = 1;
2943 return X86EMUL_CONTINUE;
2946 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2948 ctxt->dst.val = ctxt->src2.val;
2949 return fastop(ctxt, em_imul);
2952 static int em_cwd(struct x86_emulate_ctxt *ctxt)
2954 ctxt->dst.type = OP_REG;
2955 ctxt->dst.bytes = ctxt->src.bytes;
2956 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
2957 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
2959 return X86EMUL_CONTINUE;
2962 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2966 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
2967 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
2968 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
2969 return X86EMUL_CONTINUE;
2972 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
2976 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
2977 return emulate_gp(ctxt, 0);
2978 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
2979 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
2980 return X86EMUL_CONTINUE;
2983 static int em_mov(struct x86_emulate_ctxt *ctxt)
2985 memcpy(ctxt->dst.valptr, ctxt->src.valptr, sizeof(ctxt->src.valptr));
2986 return X86EMUL_CONTINUE;
2989 #define FFL(x) bit(X86_FEATURE_##x)
2991 static int em_movbe(struct x86_emulate_ctxt *ctxt)
2993 u32 ebx, ecx, edx, eax = 1;
2997 * Check MOVBE is set in the guest-visible CPUID leaf.
2999 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3000 if (!(ecx & FFL(MOVBE)))
3001 return emulate_ud(ctxt);
3003 switch (ctxt->op_bytes) {
3006 * From MOVBE definition: "...When the operand size is 16 bits,
3007 * the upper word of the destination register remains unchanged
3010 * Both casting ->valptr and ->val to u16 breaks strict aliasing
3011 * rules so we have to do the operation almost per hand.
3013 tmp = (u16)ctxt->src.val;
3014 ctxt->dst.val &= ~0xffffUL;
3015 ctxt->dst.val |= (unsigned long)swab16(tmp);
3018 ctxt->dst.val = swab32((u32)ctxt->src.val);
3021 ctxt->dst.val = swab64(ctxt->src.val);
3024 return X86EMUL_PROPAGATE_FAULT;
3026 return X86EMUL_CONTINUE;
3029 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3031 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3032 return emulate_gp(ctxt, 0);
3034 /* Disable writeback. */
3035 ctxt->dst.type = OP_NONE;
3036 return X86EMUL_CONTINUE;
3039 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3043 if (ctxt->mode == X86EMUL_MODE_PROT64)
3044 val = ctxt->src.val & ~0ULL;
3046 val = ctxt->src.val & ~0U;
3048 /* #UD condition is already handled. */
3049 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3050 return emulate_gp(ctxt, 0);
3052 /* Disable writeback. */
3053 ctxt->dst.type = OP_NONE;
3054 return X86EMUL_CONTINUE;
3057 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3061 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3062 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3063 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3064 return emulate_gp(ctxt, 0);
3066 return X86EMUL_CONTINUE;
3069 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3073 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3074 return emulate_gp(ctxt, 0);
3076 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3077 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3078 return X86EMUL_CONTINUE;
3081 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3083 if (ctxt->modrm_reg > VCPU_SREG_GS)
3084 return emulate_ud(ctxt);
3086 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3087 return X86EMUL_CONTINUE;
3090 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3092 u16 sel = ctxt->src.val;
3094 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3095 return emulate_ud(ctxt);
3097 if (ctxt->modrm_reg == VCPU_SREG_SS)
3098 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3100 /* Disable writeback. */
3101 ctxt->dst.type = OP_NONE;
3102 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3105 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3107 u16 sel = ctxt->src.val;
3109 /* Disable writeback. */
3110 ctxt->dst.type = OP_NONE;
3111 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3114 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3116 u16 sel = ctxt->src.val;
3118 /* Disable writeback. */
3119 ctxt->dst.type = OP_NONE;
3120 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3123 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3128 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3129 if (rc == X86EMUL_CONTINUE)
3130 ctxt->ops->invlpg(ctxt, linear);
3131 /* Disable writeback. */
3132 ctxt->dst.type = OP_NONE;
3133 return X86EMUL_CONTINUE;
3136 static int em_clts(struct x86_emulate_ctxt *ctxt)
3140 cr0 = ctxt->ops->get_cr(ctxt, 0);
3142 ctxt->ops->set_cr(ctxt, 0, cr0);
3143 return X86EMUL_CONTINUE;
3146 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
3150 if (ctxt->modrm_mod != 3 || ctxt->modrm_rm != 1)
3151 return X86EMUL_UNHANDLEABLE;
3153 rc = ctxt->ops->fix_hypercall(ctxt);
3154 if (rc != X86EMUL_CONTINUE)
3157 /* Let the processor re-execute the fixed hypercall */
3158 ctxt->_eip = ctxt->eip;
3159 /* Disable writeback. */
3160 ctxt->dst.type = OP_NONE;
3161 return X86EMUL_CONTINUE;
3164 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3165 void (*get)(struct x86_emulate_ctxt *ctxt,
3166 struct desc_ptr *ptr))
3168 struct desc_ptr desc_ptr;
3170 if (ctxt->mode == X86EMUL_MODE_PROT64)
3172 get(ctxt, &desc_ptr);
3173 if (ctxt->op_bytes == 2) {
3175 desc_ptr.address &= 0x00ffffff;
3177 /* Disable writeback. */
3178 ctxt->dst.type = OP_NONE;
3179 return segmented_write(ctxt, ctxt->dst.addr.mem,
3180 &desc_ptr, 2 + ctxt->op_bytes);
3183 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3185 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3188 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3190 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3193 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3195 struct desc_ptr desc_ptr;
3198 if (ctxt->mode == X86EMUL_MODE_PROT64)
3200 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3201 &desc_ptr.size, &desc_ptr.address,
3203 if (rc != X86EMUL_CONTINUE)
3205 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3206 /* Disable writeback. */
3207 ctxt->dst.type = OP_NONE;
3208 return X86EMUL_CONTINUE;
3211 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
3215 rc = ctxt->ops->fix_hypercall(ctxt);
3217 /* Disable writeback. */
3218 ctxt->dst.type = OP_NONE;
3222 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3224 struct desc_ptr desc_ptr;
3227 if (ctxt->mode == X86EMUL_MODE_PROT64)
3229 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3230 &desc_ptr.size, &desc_ptr.address,
3232 if (rc != X86EMUL_CONTINUE)
3234 ctxt->ops->set_idt(ctxt, &desc_ptr);
3235 /* Disable writeback. */
3236 ctxt->dst.type = OP_NONE;
3237 return X86EMUL_CONTINUE;
3240 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3242 if (ctxt->dst.type == OP_MEM)
3243 ctxt->dst.bytes = 2;
3244 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3245 return X86EMUL_CONTINUE;
3248 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3250 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3251 | (ctxt->src.val & 0x0f));
3252 ctxt->dst.type = OP_NONE;
3253 return X86EMUL_CONTINUE;
3256 static int em_loop(struct x86_emulate_ctxt *ctxt)
3258 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX), -1);
3259 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3260 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3261 jmp_rel(ctxt, ctxt->src.val);
3263 return X86EMUL_CONTINUE;
3266 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3268 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3269 jmp_rel(ctxt, ctxt->src.val);
3271 return X86EMUL_CONTINUE;
3274 static int em_in(struct x86_emulate_ctxt *ctxt)
3276 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3278 return X86EMUL_IO_NEEDED;
3280 return X86EMUL_CONTINUE;
3283 static int em_out(struct x86_emulate_ctxt *ctxt)
3285 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3287 /* Disable writeback. */
3288 ctxt->dst.type = OP_NONE;
3289 return X86EMUL_CONTINUE;
3292 static int em_cli(struct x86_emulate_ctxt *ctxt)
3294 if (emulator_bad_iopl(ctxt))
3295 return emulate_gp(ctxt, 0);
3297 ctxt->eflags &= ~X86_EFLAGS_IF;
3298 return X86EMUL_CONTINUE;
3301 static int em_sti(struct x86_emulate_ctxt *ctxt)
3303 if (emulator_bad_iopl(ctxt))
3304 return emulate_gp(ctxt, 0);
3306 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3307 ctxt->eflags |= X86_EFLAGS_IF;
3308 return X86EMUL_CONTINUE;
3311 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3313 u32 eax, ebx, ecx, edx;
3315 eax = reg_read(ctxt, VCPU_REGS_RAX);
3316 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3317 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3318 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3319 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3320 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3321 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3322 return X86EMUL_CONTINUE;
3325 static int em_sahf(struct x86_emulate_ctxt *ctxt)
3329 flags = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF;
3330 flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8;
3332 ctxt->eflags &= ~0xffUL;
3333 ctxt->eflags |= flags | X86_EFLAGS_FIXED;
3334 return X86EMUL_CONTINUE;
3337 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3339 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3340 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3341 return X86EMUL_CONTINUE;
3344 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3346 switch (ctxt->op_bytes) {
3347 #ifdef CONFIG_X86_64
3349 asm("bswap %0" : "+r"(ctxt->dst.val));
3353 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3356 return X86EMUL_CONTINUE;
3359 static bool valid_cr(int nr)
3371 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
3373 if (!valid_cr(ctxt->modrm_reg))
3374 return emulate_ud(ctxt);
3376 return X86EMUL_CONTINUE;
3379 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
3381 u64 new_val = ctxt->src.val64;
3382 int cr = ctxt->modrm_reg;
3385 static u64 cr_reserved_bits[] = {
3386 0xffffffff00000000ULL,
3387 0, 0, 0, /* CR3 checked later */
3394 return emulate_ud(ctxt);
3396 if (new_val & cr_reserved_bits[cr])
3397 return emulate_gp(ctxt, 0);
3402 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
3403 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
3404 return emulate_gp(ctxt, 0);
3406 cr4 = ctxt->ops->get_cr(ctxt, 4);
3407 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3409 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
3410 !(cr4 & X86_CR4_PAE))
3411 return emulate_gp(ctxt, 0);
3418 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3419 if (efer & EFER_LMA)
3420 rsvd = CR3_L_MODE_RESERVED_BITS;
3423 return emulate_gp(ctxt, 0);
3428 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3430 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
3431 return emulate_gp(ctxt, 0);
3437 return X86EMUL_CONTINUE;
3440 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
3444 ctxt->ops->get_dr(ctxt, 7, &dr7);
3446 /* Check if DR7.Global_Enable is set */
3447 return dr7 & (1 << 13);
3450 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
3452 int dr = ctxt->modrm_reg;
3456 return emulate_ud(ctxt);
3458 cr4 = ctxt->ops->get_cr(ctxt, 4);
3459 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
3460 return emulate_ud(ctxt);
3462 if (check_dr7_gd(ctxt))
3463 return emulate_db(ctxt);
3465 return X86EMUL_CONTINUE;
3468 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
3470 u64 new_val = ctxt->src.val64;
3471 int dr = ctxt->modrm_reg;
3473 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
3474 return emulate_gp(ctxt, 0);
3476 return check_dr_read(ctxt);
3479 static int check_svme(struct x86_emulate_ctxt *ctxt)
3483 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3485 if (!(efer & EFER_SVME))
3486 return emulate_ud(ctxt);
3488 return X86EMUL_CONTINUE;
3491 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
3493 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
3495 /* Valid physical address? */
3496 if (rax & 0xffff000000000000ULL)
3497 return emulate_gp(ctxt, 0);
3499 return check_svme(ctxt);
3502 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
3504 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3506 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3507 return emulate_ud(ctxt);
3509 return X86EMUL_CONTINUE;
3512 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
3514 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3515 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
3517 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3518 ctxt->ops->check_pmc(ctxt, rcx))
3519 return emulate_gp(ctxt, 0);
3521 return X86EMUL_CONTINUE;
3524 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
3526 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
3527 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3528 return emulate_gp(ctxt, 0);
3530 return X86EMUL_CONTINUE;
3533 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
3535 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
3536 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3537 return emulate_gp(ctxt, 0);
3539 return X86EMUL_CONTINUE;
3542 #define D(_y) { .flags = (_y) }
3543 #define DI(_y, _i) { .flags = (_y)|Intercept, .intercept = x86_intercept_##_i }
3544 #define DIP(_y, _i, _p) { .flags = (_y)|Intercept|CheckPerm, \
3545 .intercept = x86_intercept_##_i, .check_perm = (_p) }
3546 #define N D(NotImpl)
3547 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3548 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
3549 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
3550 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
3551 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3552 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
3553 #define II(_f, _e, _i) \
3554 { .flags = (_f)|Intercept, .u.execute = (_e), .intercept = x86_intercept_##_i }
3555 #define IIP(_f, _e, _i, _p) \
3556 { .flags = (_f)|Intercept|CheckPerm, .u.execute = (_e), \
3557 .intercept = x86_intercept_##_i, .check_perm = (_p) }
3558 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3560 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3561 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3562 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3563 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
3564 #define I2bvIP(_f, _e, _i, _p) \
3565 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
3567 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3568 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3569 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3571 static const struct opcode group7_rm1[] = {
3572 DI(SrcNone | Priv, monitor),
3573 DI(SrcNone | Priv, mwait),
3577 static const struct opcode group7_rm3[] = {
3578 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
3579 II(SrcNone | Prot | EmulateOnUD, em_vmmcall, vmmcall),
3580 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
3581 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
3582 DIP(SrcNone | Prot | Priv, stgi, check_svme),
3583 DIP(SrcNone | Prot | Priv, clgi, check_svme),
3584 DIP(SrcNone | Prot | Priv, skinit, check_svme),
3585 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
3588 static const struct opcode group7_rm7[] = {
3590 DIP(SrcNone, rdtscp, check_rdtsc),
3594 static const struct opcode group1[] = {
3596 F(Lock | PageTable, em_or),
3599 F(Lock | PageTable, em_and),
3605 static const struct opcode group1A[] = {
3606 I(DstMem | SrcNone | Mov | Stack, em_pop), N, N, N, N, N, N, N,
3609 static const struct opcode group2[] = {
3610 F(DstMem | ModRM, em_rol),
3611 F(DstMem | ModRM, em_ror),
3612 F(DstMem | ModRM, em_rcl),
3613 F(DstMem | ModRM, em_rcr),
3614 F(DstMem | ModRM, em_shl),
3615 F(DstMem | ModRM, em_shr),
3616 F(DstMem | ModRM, em_shl),
3617 F(DstMem | ModRM, em_sar),
3620 static const struct opcode group3[] = {
3621 F(DstMem | SrcImm | NoWrite, em_test),
3622 F(DstMem | SrcImm | NoWrite, em_test),
3623 F(DstMem | SrcNone | Lock, em_not),
3624 F(DstMem | SrcNone | Lock, em_neg),
3625 F(DstXacc | Src2Mem, em_mul_ex),
3626 F(DstXacc | Src2Mem, em_imul_ex),
3627 F(DstXacc | Src2Mem, em_div_ex),
3628 F(DstXacc | Src2Mem, em_idiv_ex),
3631 static const struct opcode group4[] = {
3632 F(ByteOp | DstMem | SrcNone | Lock, em_inc),
3633 F(ByteOp | DstMem | SrcNone | Lock, em_dec),
3637 static const struct opcode group5[] = {
3638 F(DstMem | SrcNone | Lock, em_inc),
3639 F(DstMem | SrcNone | Lock, em_dec),
3640 I(SrcMem | Stack, em_grp45),
3641 I(SrcMemFAddr | ImplicitOps | Stack, em_call_far),
3642 I(SrcMem | Stack, em_grp45),
3643 I(SrcMemFAddr | ImplicitOps, em_grp45),
3644 I(SrcMem | Stack, em_grp45), D(Undefined),
3647 static const struct opcode group6[] = {
3650 II(Prot | Priv | SrcMem16, em_lldt, lldt),
3651 II(Prot | Priv | SrcMem16, em_ltr, ltr),
3655 static const struct group_dual group7 = { {
3656 II(Mov | DstMem, em_sgdt, sgdt),
3657 II(Mov | DstMem, em_sidt, sidt),
3658 II(SrcMem | Priv, em_lgdt, lgdt),
3659 II(SrcMem | Priv, em_lidt, lidt),
3660 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3661 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3662 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3664 I(SrcNone | Priv | EmulateOnUD, em_vmcall),
3666 N, EXT(0, group7_rm3),
3667 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3668 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3672 static const struct opcode group8[] = {
3674 F(DstMem | SrcImmByte | NoWrite, em_bt),
3675 F(DstMem | SrcImmByte | Lock | PageTable, em_bts),
3676 F(DstMem | SrcImmByte | Lock, em_btr),
3677 F(DstMem | SrcImmByte | Lock | PageTable, em_btc),
3680 static const struct group_dual group9 = { {
3681 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
3683 N, N, N, N, N, N, N, N,
3686 static const struct opcode group11[] = {
3687 I(DstMem | SrcImm | Mov | PageTable, em_mov),
3691 static const struct gprefix pfx_0f_6f_0f_7f = {
3692 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
3695 static const struct gprefix pfx_vmovntpx = {
3696 I(0, em_mov), N, N, N,
3699 static const struct gprefix pfx_0f_28_0f_29 = {
3700 I(Aligned, em_mov), I(Aligned, em_mov), N, N,
3703 static const struct escape escape_d9 = { {
3704 N, N, N, N, N, N, N, I(DstMem, em_fnstcw),
3707 N, N, N, N, N, N, N, N,
3709 N, N, N, N, N, N, N, N,
3711 N, N, N, N, N, N, N, N,
3713 N, N, N, N, N, N, N, N,
3715 N, N, N, N, N, N, N, N,
3717 N, N, N, N, N, N, N, N,
3719 N, N, N, N, N, N, N, N,
3721 N, N, N, N, N, N, N, N,
3724 static const struct escape escape_db = { {
3725 N, N, N, N, N, N, N, N,
3728 N, N, N, N, N, N, N, N,
3730 N, N, N, N, N, N, N, N,
3732 N, N, N, N, N, N, N, N,
3734 N, N, N, N, N, N, N, N,
3736 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
3738 N, N, N, N, N, N, N, N,
3740 N, N, N, N, N, N, N, N,
3742 N, N, N, N, N, N, N, N,
3745 static const struct escape escape_dd = { {
3746 N, N, N, N, N, N, N, I(DstMem, em_fnstsw),
3749 N, N, N, N, N, N, N, N,
3751 N, N, N, N, N, N, N, N,
3753 N, N, N, N, N, N, N, N,
3755 N, N, N, N, N, N, N, N,
3757 N, N, N, N, N, N, N, N,
3759 N, N, N, N, N, N, N, N,
3761 N, N, N, N, N, N, N, N,
3763 N, N, N, N, N, N, N, N,
3766 static const struct opcode opcode_table[256] = {
3768 F6ALU(Lock, em_add),
3769 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
3770 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
3772 F6ALU(Lock | PageTable, em_or),
3773 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
3776 F6ALU(Lock, em_adc),
3777 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
3778 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
3780 F6ALU(Lock, em_sbb),
3781 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
3782 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
3784 F6ALU(Lock | PageTable, em_and), N, N,
3786 F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3788 F6ALU(Lock, em_xor), N, N,
3790 F6ALU(NoWrite, em_cmp), N, N,
3792 X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
3794 X8(I(SrcReg | Stack, em_push)),
3796 X8(I(DstReg | Stack, em_pop)),
3798 I(ImplicitOps | Stack | No64, em_pusha),
3799 I(ImplicitOps | Stack | No64, em_popa),
3800 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3803 I(SrcImm | Mov | Stack, em_push),
3804 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3805 I(SrcImmByte | Mov | Stack, em_push),
3806 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3807 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
3808 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
3812 G(ByteOp | DstMem | SrcImm, group1),
3813 G(DstMem | SrcImm, group1),
3814 G(ByteOp | DstMem | SrcImm | No64, group1),
3815 G(DstMem | SrcImmByte, group1),
3816 F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
3817 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
3819 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
3820 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3821 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
3822 D(ModRM | SrcMem | NoAccess | DstReg),
3823 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
3826 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3828 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3829 I(SrcImmFAddr | No64, em_call_far), N,
3830 II(ImplicitOps | Stack, em_pushf, pushf),
3831 II(ImplicitOps | Stack, em_popf, popf),
3832 I(ImplicitOps, em_sahf), I(ImplicitOps, em_lahf),
3834 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3835 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
3836 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3837 F2bv(SrcSI | DstDI | String | NoWrite, em_cmp),
3839 F2bv(DstAcc | SrcImm | NoWrite, em_test),
3840 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3841 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3842 F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp),
3844 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3846 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
3848 G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
3849 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3850 I(ImplicitOps | Stack, em_ret),
3851 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
3852 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
3853 G(ByteOp, group11), G(0, group11),
3855 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
3856 I(ImplicitOps | Stack | SrcImmU16, em_ret_far_imm),
3857 I(ImplicitOps | Stack, em_ret_far),
3858 D(ImplicitOps), DI(SrcImmByte, intn),
3859 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3861 G(Src2One | ByteOp, group2), G(Src2One, group2),
3862 G(Src2CL | ByteOp, group2), G(Src2CL, group2),
3863 I(DstAcc | SrcImmUByte | No64, em_aam),
3864 I(DstAcc | SrcImmUByte | No64, em_aad),
3865 F(DstAcc | ByteOp | No64, em_salc),
3866 I(DstAcc | SrcXLat | ByteOp, em_mov),
3868 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
3870 X3(I(SrcImmByte, em_loop)),
3871 I(SrcImmByte, em_jcxz),
3872 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
3873 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
3875 I(SrcImm | Stack, em_call), D(SrcImm | ImplicitOps),
3876 I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
3877 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
3878 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
3880 N, DI(ImplicitOps, icebp), N, N,
3881 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
3882 G(ByteOp, group3), G(0, group3),
3884 D(ImplicitOps), D(ImplicitOps),
3885 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
3886 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3889 static const struct opcode twobyte_table[256] = {
3891 G(0, group6), GD(0, &group7), N, N,
3892 N, I(ImplicitOps | EmulateOnUD, em_syscall),
3893 II(ImplicitOps | Priv, em_clts, clts), N,
3894 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3895 N, D(ImplicitOps | ModRM), N, N,
3897 N, N, N, N, N, N, N, N,
3898 D(ImplicitOps | ModRM), N, N, N, N, N, N, D(ImplicitOps | ModRM),
3900 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, cr_read, check_cr_read),
3901 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, dr_read, check_dr_read),
3902 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_cr_write, cr_write,
3904 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_dr_write, dr_write,
3907 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_28_0f_29),
3908 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_28_0f_29),
3909 N, GP(ModRM | DstMem | SrcReg | Sse | Mov | Aligned, &pfx_vmovntpx),
3912 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
3913 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
3914 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
3915 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
3916 I(ImplicitOps | EmulateOnUD, em_sysenter),
3917 I(ImplicitOps | Priv | EmulateOnUD, em_sysexit),
3919 N, N, N, N, N, N, N, N,
3921 X16(D(DstReg | SrcMem | ModRM)),
3923 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3928 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3933 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3937 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3939 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
3940 II(ImplicitOps, em_cpuid, cpuid),
3941 F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
3942 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
3943 F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
3945 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
3946 DI(ImplicitOps, rsm),
3947 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
3948 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
3949 F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
3950 D(ModRM), F(DstReg | SrcMem | ModRM, em_imul),
3952 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_cmpxchg),
3953 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
3954 F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
3955 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
3956 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
3957 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3961 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
3962 F(DstReg | SrcMem | ModRM, em_bsf), F(DstReg | SrcMem | ModRM, em_bsr),
3963 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3965 F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
3966 N, D(DstMem | SrcReg | ModRM | Mov),
3967 N, N, N, GD(0, &group9),
3969 X8(I(DstReg, em_bswap)),
3971 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3973 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3975 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
3978 static const struct gprefix three_byte_0f_38_f0 = {
3979 I(DstReg | SrcMem | Mov, em_movbe), N, N, N
3982 static const struct gprefix three_byte_0f_38_f1 = {
3983 I(DstMem | SrcReg | Mov, em_movbe), N, N, N
3987 * Insns below are selected by the prefix which indexed by the third opcode
3990 static const struct opcode opcode_map_0f_38[256] = {
3992 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
3994 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
3996 GP(EmulateOnUD | ModRM | Prefix, &three_byte_0f_38_f0),
3997 GP(EmulateOnUD | ModRM | Prefix, &three_byte_0f_38_f1),
4016 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4020 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4026 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
4027 unsigned size, bool sign_extension)
4029 int rc = X86EMUL_CONTINUE;
4033 op->addr.mem.ea = ctxt->_eip;
4034 /* NB. Immediates are sign-extended as necessary. */
4035 switch (op->bytes) {
4037 op->val = insn_fetch(s8, ctxt);
4040 op->val = insn_fetch(s16, ctxt);
4043 op->val = insn_fetch(s32, ctxt);
4046 op->val = insn_fetch(s64, ctxt);
4049 if (!sign_extension) {
4050 switch (op->bytes) {
4058 op->val &= 0xffffffff;
4066 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
4069 int rc = X86EMUL_CONTINUE;
4073 decode_register_operand(ctxt, op);
4076 rc = decode_imm(ctxt, op, 1, false);
4079 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4083 if (ctxt->d & BitOp)
4084 fetch_bit_operand(ctxt);
4085 op->orig_val = op->val;
4088 ctxt->memop.bytes = (ctxt->op_bytes == 8) ? 16 : 8;
4092 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4093 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4094 fetch_register_operand(op);
4095 op->orig_val = op->val;
4099 op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;
4100 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4101 fetch_register_operand(op);
4102 op->orig_val = op->val;
4105 if (ctxt->d & ByteOp) {
4110 op->bytes = ctxt->op_bytes;
4111 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4112 fetch_register_operand(op);
4113 op->orig_val = op->val;
4117 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4119 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RDI));
4120 op->addr.mem.seg = VCPU_SREG_ES;
4127 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4128 fetch_register_operand(op);
4132 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4135 rc = decode_imm(ctxt, op, 1, true);
4142 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
4145 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
4148 ctxt->memop.bytes = 1;
4149 if (ctxt->memop.type == OP_REG) {
4150 ctxt->memop.addr.reg = decode_register(ctxt,
4151 ctxt->modrm_rm, true);
4152 fetch_register_operand(&ctxt->memop);
4156 ctxt->memop.bytes = 2;
4159 ctxt->memop.bytes = 4;
4162 rc = decode_imm(ctxt, op, 2, false);
4165 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
4169 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4171 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RSI));
4172 op->addr.mem.seg = seg_override(ctxt);
4178 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4180 register_address(ctxt,
4181 reg_read(ctxt, VCPU_REGS_RBX) +
4182 (reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
4183 op->addr.mem.seg = seg_override(ctxt);
4188 op->addr.mem.ea = ctxt->_eip;
4189 op->bytes = ctxt->op_bytes + 2;
4190 insn_fetch_arr(op->valptr, op->bytes, ctxt);
4193 ctxt->memop.bytes = ctxt->op_bytes + 2;
4196 op->val = VCPU_SREG_ES;
4199 op->val = VCPU_SREG_CS;
4202 op->val = VCPU_SREG_SS;
4205 op->val = VCPU_SREG_DS;
4208 op->val = VCPU_SREG_FS;
4211 op->val = VCPU_SREG_GS;
4214 /* Special instructions do their own operand decoding. */
4216 op->type = OP_NONE; /* Disable writeback. */
4224 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
4226 int rc = X86EMUL_CONTINUE;
4227 int mode = ctxt->mode;
4228 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
4229 bool op_prefix = false;
4230 struct opcode opcode;
4232 ctxt->memop.type = OP_NONE;
4233 ctxt->memopp = NULL;
4234 ctxt->_eip = ctxt->eip;
4235 ctxt->fetch.start = ctxt->_eip;
4236 ctxt->fetch.end = ctxt->fetch.start + insn_len;
4237 ctxt->opcode_len = 1;
4239 memcpy(ctxt->fetch.data, insn, insn_len);
4242 case X86EMUL_MODE_REAL:
4243 case X86EMUL_MODE_VM86:
4244 case X86EMUL_MODE_PROT16:
4245 def_op_bytes = def_ad_bytes = 2;
4247 case X86EMUL_MODE_PROT32:
4248 def_op_bytes = def_ad_bytes = 4;
4250 #ifdef CONFIG_X86_64
4251 case X86EMUL_MODE_PROT64:
4257 return EMULATION_FAILED;
4260 ctxt->op_bytes = def_op_bytes;
4261 ctxt->ad_bytes = def_ad_bytes;
4263 /* Legacy prefixes. */
4265 switch (ctxt->b = insn_fetch(u8, ctxt)) {
4266 case 0x66: /* operand-size override */
4268 /* switch between 2/4 bytes */
4269 ctxt->op_bytes = def_op_bytes ^ 6;
4271 case 0x67: /* address-size override */
4272 if (mode == X86EMUL_MODE_PROT64)
4273 /* switch between 4/8 bytes */
4274 ctxt->ad_bytes = def_ad_bytes ^ 12;
4276 /* switch between 2/4 bytes */
4277 ctxt->ad_bytes = def_ad_bytes ^ 6;
4279 case 0x26: /* ES override */
4280 case 0x2e: /* CS override */
4281 case 0x36: /* SS override */
4282 case 0x3e: /* DS override */
4283 set_seg_override(ctxt, (ctxt->b >> 3) & 3);
4285 case 0x64: /* FS override */
4286 case 0x65: /* GS override */
4287 set_seg_override(ctxt, ctxt->b & 7);
4289 case 0x40 ... 0x4f: /* REX */
4290 if (mode != X86EMUL_MODE_PROT64)
4292 ctxt->rex_prefix = ctxt->b;
4294 case 0xf0: /* LOCK */
4295 ctxt->lock_prefix = 1;
4297 case 0xf2: /* REPNE/REPNZ */
4298 case 0xf3: /* REP/REPE/REPZ */
4299 ctxt->rep_prefix = ctxt->b;
4305 /* Any legacy prefix after a REX prefix nullifies its effect. */
4307 ctxt->rex_prefix = 0;
4313 if (ctxt->rex_prefix & 8)
4314 ctxt->op_bytes = 8; /* REX.W */
4316 /* Opcode byte(s). */
4317 opcode = opcode_table[ctxt->b];
4318 /* Two-byte opcode? */
4319 if (ctxt->b == 0x0f) {
4320 ctxt->opcode_len = 2;
4321 ctxt->b = insn_fetch(u8, ctxt);
4322 opcode = twobyte_table[ctxt->b];
4324 /* 0F_38 opcode map */
4325 if (ctxt->b == 0x38) {
4326 ctxt->opcode_len = 3;
4327 ctxt->b = insn_fetch(u8, ctxt);
4328 opcode = opcode_map_0f_38[ctxt->b];
4331 ctxt->d = opcode.flags;
4333 if (ctxt->d & ModRM)
4334 ctxt->modrm = insn_fetch(u8, ctxt);
4336 /* vex-prefix instructions are not implemented */
4337 if (ctxt->opcode_len == 1 && (ctxt->b == 0xc5 || ctxt->b == 0xc4) &&
4338 (mode == X86EMUL_MODE_PROT64 ||
4339 (mode >= X86EMUL_MODE_PROT16 && (ctxt->modrm & 0x80)))) {
4343 while (ctxt->d & GroupMask) {
4344 switch (ctxt->d & GroupMask) {
4346 goffset = (ctxt->modrm >> 3) & 7;
4347 opcode = opcode.u.group[goffset];
4350 goffset = (ctxt->modrm >> 3) & 7;
4351 if ((ctxt->modrm >> 6) == 3)
4352 opcode = opcode.u.gdual->mod3[goffset];
4354 opcode = opcode.u.gdual->mod012[goffset];
4357 goffset = ctxt->modrm & 7;
4358 opcode = opcode.u.group[goffset];
4361 if (ctxt->rep_prefix && op_prefix)
4362 return EMULATION_FAILED;
4363 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
4364 switch (simd_prefix) {
4365 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
4366 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
4367 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
4368 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
4372 if (ctxt->modrm > 0xbf)
4373 opcode = opcode.u.esc->high[ctxt->modrm - 0xc0];
4375 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
4378 return EMULATION_FAILED;
4381 ctxt->d &= ~(u64)GroupMask;
4382 ctxt->d |= opcode.flags;
4387 return EMULATION_FAILED;
4389 ctxt->execute = opcode.u.execute;
4391 if (unlikely(ctxt->d &
4392 (NotImpl|EmulateOnUD|Stack|Op3264|Sse|Mmx|Intercept|CheckPerm))) {
4394 * These are copied unconditionally here, and checked unconditionally
4395 * in x86_emulate_insn.
4397 ctxt->check_perm = opcode.check_perm;
4398 ctxt->intercept = opcode.intercept;
4400 if (ctxt->d & NotImpl)
4401 return EMULATION_FAILED;
4403 if (!(ctxt->d & EmulateOnUD) && ctxt->ud)
4404 return EMULATION_FAILED;
4406 if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
4409 if (ctxt->d & Op3264) {
4410 if (mode == X86EMUL_MODE_PROT64)
4417 ctxt->op_bytes = 16;
4418 else if (ctxt->d & Mmx)
4422 /* ModRM and SIB bytes. */
4423 if (ctxt->d & ModRM) {
4424 rc = decode_modrm(ctxt, &ctxt->memop);
4425 if (!ctxt->has_seg_override)
4426 set_seg_override(ctxt, ctxt->modrm_seg);
4427 } else if (ctxt->d & MemAbs)
4428 rc = decode_abs(ctxt, &ctxt->memop);
4429 if (rc != X86EMUL_CONTINUE)
4432 if (!ctxt->has_seg_override)
4433 set_seg_override(ctxt, VCPU_SREG_DS);
4435 ctxt->memop.addr.mem.seg = seg_override(ctxt);
4437 if (ctxt->memop.type == OP_MEM && ctxt->ad_bytes != 8)
4438 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
4441 * Decode and fetch the source operand: register, memory
4444 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
4445 if (rc != X86EMUL_CONTINUE)
4449 * Decode and fetch the second source operand: register, memory
4452 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
4453 if (rc != X86EMUL_CONTINUE)
4456 /* Decode and fetch the destination operand: register or memory. */
4457 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
4460 if (ctxt->memopp && ctxt->memopp->type == OP_MEM && ctxt->rip_relative)
4461 ctxt->memopp->addr.mem.ea += ctxt->_eip;
4463 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
4466 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
4468 return ctxt->d & PageTable;
4471 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
4473 /* The second termination condition only applies for REPE
4474 * and REPNE. Test if the repeat string operation prefix is
4475 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
4476 * corresponding termination condition according to:
4477 * - if REPE/REPZ and ZF = 0 then done
4478 * - if REPNE/REPNZ and ZF = 1 then done
4480 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
4481 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
4482 && (((ctxt->rep_prefix == REPE_PREFIX) &&
4483 ((ctxt->eflags & EFLG_ZF) == 0))
4484 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
4485 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
4491 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
4495 ctxt->ops->get_fpu(ctxt);
4496 asm volatile("1: fwait \n\t"
4498 ".pushsection .fixup,\"ax\" \n\t"
4500 "movb $1, %[fault] \n\t"
4503 _ASM_EXTABLE(1b, 3b)
4504 : [fault]"+qm"(fault));
4505 ctxt->ops->put_fpu(ctxt);
4507 if (unlikely(fault))
4508 return emulate_exception(ctxt, MF_VECTOR, 0, false);
4510 return X86EMUL_CONTINUE;
4513 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
4516 if (op->type == OP_MM)
4517 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
4520 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
4522 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
4523 if (!(ctxt->d & ByteOp))
4524 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
4525 asm("push %[flags]; popf; call *%[fastop]; pushf; pop %[flags]\n"
4526 : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
4528 : "c"(ctxt->src2.val));
4529 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
4530 if (!fop) /* exception is returned in fop variable */
4531 return emulate_de(ctxt);
4532 return X86EMUL_CONTINUE;
4535 void init_decode_cache(struct x86_emulate_ctxt *ctxt)
4537 memset(&ctxt->has_seg_override, 0,
4538 (void *)&ctxt->modrm - (void *)&ctxt->has_seg_override);
4540 ctxt->io_read.pos = 0;
4541 ctxt->io_read.end = 0;
4542 ctxt->mem_read.end = 0;
4545 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
4547 const struct x86_emulate_ops *ops = ctxt->ops;
4548 int rc = X86EMUL_CONTINUE;
4549 int saved_dst_type = ctxt->dst.type;
4551 ctxt->mem_read.pos = 0;
4553 /* LOCK prefix is allowed only with some instructions */
4554 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
4555 rc = emulate_ud(ctxt);
4559 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
4560 rc = emulate_ud(ctxt);
4564 if (unlikely(ctxt->d &
4565 (No64|Undefined|Sse|Mmx|Intercept|CheckPerm|Priv|Prot|String))) {
4566 if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
4567 (ctxt->d & Undefined)) {
4568 rc = emulate_ud(ctxt);
4572 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
4573 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
4574 rc = emulate_ud(ctxt);
4578 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
4579 rc = emulate_nm(ctxt);
4583 if (ctxt->d & Mmx) {
4584 rc = flush_pending_x87_faults(ctxt);
4585 if (rc != X86EMUL_CONTINUE)
4588 * Now that we know the fpu is exception safe, we can fetch
4591 fetch_possible_mmx_operand(ctxt, &ctxt->src);
4592 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
4593 if (!(ctxt->d & Mov))
4594 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
4597 if (unlikely(ctxt->guest_mode) && (ctxt->d & Intercept)) {
4598 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4599 X86_ICPT_PRE_EXCEPT);
4600 if (rc != X86EMUL_CONTINUE)
4604 /* Privileged instruction can be executed only in CPL=0 */
4605 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
4606 rc = emulate_gp(ctxt, 0);
4610 /* Instruction can only be executed in protected mode */
4611 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
4612 rc = emulate_ud(ctxt);
4616 /* Do instruction specific permission checks */
4617 if (ctxt->d & CheckPerm) {
4618 rc = ctxt->check_perm(ctxt);
4619 if (rc != X86EMUL_CONTINUE)
4623 if (unlikely(ctxt->guest_mode) && (ctxt->d & Intercept)) {
4624 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4625 X86_ICPT_POST_EXCEPT);
4626 if (rc != X86EMUL_CONTINUE)
4630 if (ctxt->rep_prefix && (ctxt->d & String)) {
4631 /* All REP prefixes have the same first termination condition */
4632 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
4633 ctxt->eip = ctxt->_eip;
4639 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
4640 rc = segmented_read(ctxt, ctxt->src.addr.mem,
4641 ctxt->src.valptr, ctxt->src.bytes);
4642 if (rc != X86EMUL_CONTINUE)
4644 ctxt->src.orig_val64 = ctxt->src.val64;
4647 if (ctxt->src2.type == OP_MEM) {
4648 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
4649 &ctxt->src2.val, ctxt->src2.bytes);
4650 if (rc != X86EMUL_CONTINUE)
4654 if ((ctxt->d & DstMask) == ImplicitOps)
4658 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
4659 /* optimisation - avoid slow emulated read if Mov */
4660 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
4661 &ctxt->dst.val, ctxt->dst.bytes);
4662 if (rc != X86EMUL_CONTINUE)
4665 ctxt->dst.orig_val = ctxt->dst.val;
4669 if (unlikely(ctxt->guest_mode) && (ctxt->d & Intercept)) {
4670 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4671 X86_ICPT_POST_MEMACCESS);
4672 if (rc != X86EMUL_CONTINUE)
4676 if (ctxt->execute) {
4677 if (ctxt->d & Fastop) {
4678 void (*fop)(struct fastop *) = (void *)ctxt->execute;
4679 rc = fastop(ctxt, fop);
4680 if (rc != X86EMUL_CONTINUE)
4684 rc = ctxt->execute(ctxt);
4685 if (rc != X86EMUL_CONTINUE)
4690 if (ctxt->opcode_len == 2)
4692 else if (ctxt->opcode_len == 3)
4693 goto threebyte_insn;
4696 case 0x63: /* movsxd */
4697 if (ctxt->mode != X86EMUL_MODE_PROT64)
4698 goto cannot_emulate;
4699 ctxt->dst.val = (s32) ctxt->src.val;
4701 case 0x70 ... 0x7f: /* jcc (short) */
4702 if (test_cc(ctxt->b, ctxt->eflags))
4703 jmp_rel(ctxt, ctxt->src.val);
4705 case 0x8d: /* lea r16/r32, m */
4706 ctxt->dst.val = ctxt->src.addr.mem.ea;
4708 case 0x90 ... 0x97: /* nop / xchg reg, rax */
4709 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
4710 ctxt->dst.type = OP_NONE;
4714 case 0x98: /* cbw/cwde/cdqe */
4715 switch (ctxt->op_bytes) {
4716 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
4717 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
4718 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
4721 case 0xcc: /* int3 */
4722 rc = emulate_int(ctxt, 3);
4724 case 0xcd: /* int n */
4725 rc = emulate_int(ctxt, ctxt->src.val);
4727 case 0xce: /* into */
4728 if (ctxt->eflags & EFLG_OF)
4729 rc = emulate_int(ctxt, 4);
4731 case 0xe9: /* jmp rel */
4732 case 0xeb: /* jmp rel short */
4733 jmp_rel(ctxt, ctxt->src.val);
4734 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4736 case 0xf4: /* hlt */
4737 ctxt->ops->halt(ctxt);
4739 case 0xf5: /* cmc */
4740 /* complement carry flag from eflags reg */
4741 ctxt->eflags ^= EFLG_CF;
4743 case 0xf8: /* clc */
4744 ctxt->eflags &= ~EFLG_CF;
4746 case 0xf9: /* stc */
4747 ctxt->eflags |= EFLG_CF;
4749 case 0xfc: /* cld */
4750 ctxt->eflags &= ~EFLG_DF;
4752 case 0xfd: /* std */
4753 ctxt->eflags |= EFLG_DF;
4756 goto cannot_emulate;
4759 if (rc != X86EMUL_CONTINUE)
4763 if (ctxt->d & SrcWrite) {
4764 BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);
4765 rc = writeback(ctxt, &ctxt->src);
4766 if (rc != X86EMUL_CONTINUE)
4769 if (!(ctxt->d & NoWrite)) {
4770 rc = writeback(ctxt, &ctxt->dst);
4771 if (rc != X86EMUL_CONTINUE)
4776 * restore dst type in case the decoding will be reused
4777 * (happens for string instruction )
4779 ctxt->dst.type = saved_dst_type;
4781 if ((ctxt->d & SrcMask) == SrcSI)
4782 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
4784 if ((ctxt->d & DstMask) == DstDI)
4785 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
4787 if (ctxt->rep_prefix && (ctxt->d & String)) {
4789 struct read_cache *r = &ctxt->io_read;
4790 if ((ctxt->d & SrcMask) == SrcSI)
4791 count = ctxt->src.count;
4793 count = ctxt->dst.count;
4794 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX),
4797 if (!string_insn_completed(ctxt)) {
4799 * Re-enter guest when pio read ahead buffer is empty
4800 * or, if it is not used, after each 1024 iteration.
4802 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
4803 (r->end == 0 || r->end != r->pos)) {
4805 * Reset read cache. Usually happens before
4806 * decode, but since instruction is restarted
4807 * we have to do it here.
4809 ctxt->mem_read.end = 0;
4810 writeback_registers(ctxt);
4811 return EMULATION_RESTART;
4813 goto done; /* skip rip writeback */
4817 ctxt->eip = ctxt->_eip;
4820 if (rc == X86EMUL_PROPAGATE_FAULT)
4821 ctxt->have_exception = true;
4822 if (rc == X86EMUL_INTERCEPTED)
4823 return EMULATION_INTERCEPTED;
4825 if (rc == X86EMUL_CONTINUE)
4826 writeback_registers(ctxt);
4828 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4832 case 0x09: /* wbinvd */
4833 (ctxt->ops->wbinvd)(ctxt);
4835 case 0x08: /* invd */
4836 case 0x0d: /* GrpP (prefetch) */
4837 case 0x18: /* Grp16 (prefetch/nop) */
4838 case 0x1f: /* nop */
4840 case 0x20: /* mov cr, reg */
4841 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
4843 case 0x21: /* mov from dr to reg */
4844 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
4846 case 0x40 ... 0x4f: /* cmov */
4847 if (test_cc(ctxt->b, ctxt->eflags))
4848 ctxt->dst.val = ctxt->src.val;
4849 else if (ctxt->mode != X86EMUL_MODE_PROT64 ||
4850 ctxt->op_bytes != 4)
4851 ctxt->dst.type = OP_NONE; /* no writeback */
4853 case 0x80 ... 0x8f: /* jnz rel, etc*/
4854 if (test_cc(ctxt->b, ctxt->eflags))
4855 jmp_rel(ctxt, ctxt->src.val);
4857 case 0x90 ... 0x9f: /* setcc r/m8 */
4858 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
4860 case 0xae: /* clflush */
4862 case 0xb6 ... 0xb7: /* movzx */
4863 ctxt->dst.bytes = ctxt->op_bytes;
4864 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
4865 : (u16) ctxt->src.val;
4867 case 0xbe ... 0xbf: /* movsx */
4868 ctxt->dst.bytes = ctxt->op_bytes;
4869 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
4870 (s16) ctxt->src.val;
4872 case 0xc3: /* movnti */
4873 ctxt->dst.bytes = ctxt->op_bytes;
4874 ctxt->dst.val = (ctxt->op_bytes == 8) ? (u64) ctxt->src.val :
4875 (u32) ctxt->src.val;
4878 goto cannot_emulate;
4883 if (rc != X86EMUL_CONTINUE)
4889 return EMULATION_FAILED;
4892 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
4894 invalidate_registers(ctxt);
4897 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
4899 writeback_registers(ctxt);