1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
27 #include <linux/stringify.h>
36 #define OpImplicit 1ull /* No generic decode */
37 #define OpReg 2ull /* Register */
38 #define OpMem 3ull /* Memory */
39 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
40 #define OpDI 5ull /* ES:DI/EDI/RDI */
41 #define OpMem64 6ull /* Memory, 64-bit */
42 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
43 #define OpDX 8ull /* DX register */
44 #define OpCL 9ull /* CL register (for shifts) */
45 #define OpImmByte 10ull /* 8-bit sign extended immediate */
46 #define OpOne 11ull /* Implied 1 */
47 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
48 #define OpMem16 13ull /* Memory operand (16-bit). */
49 #define OpMem32 14ull /* Memory operand (32-bit). */
50 #define OpImmU 15ull /* Immediate operand, zero extended */
51 #define OpSI 16ull /* SI/ESI/RSI */
52 #define OpImmFAddr 17ull /* Immediate far address */
53 #define OpMemFAddr 18ull /* Far address in memory */
54 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
55 #define OpES 20ull /* ES */
56 #define OpCS 21ull /* CS */
57 #define OpSS 22ull /* SS */
58 #define OpDS 23ull /* DS */
59 #define OpFS 24ull /* FS */
60 #define OpGS 25ull /* GS */
61 #define OpMem8 26ull /* 8-bit zero extended memory operand */
62 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
63 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
64 #define OpAccLo 29ull /* Low part of extended acc (AX/AX/EAX/RAX) */
65 #define OpAccHi 30ull /* High part of extended acc (-/DX/EDX/RDX) */
67 #define OpBits 5 /* Width of operand field */
68 #define OpMask ((1ull << OpBits) - 1)
71 * Opcode effective-address decode tables.
72 * Note that we only emulate instructions that have at least one memory
73 * operand (excluding implicit stack references). We assume that stack
74 * references and instruction fetches will never occur in special memory
75 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
79 /* Operand sizes: 8-bit operands or specified/overridden size. */
80 #define ByteOp (1<<0) /* 8-bit operands. */
81 /* Destination operand type. */
83 #define ImplicitOps (OpImplicit << DstShift)
84 #define DstReg (OpReg << DstShift)
85 #define DstMem (OpMem << DstShift)
86 #define DstAcc (OpAcc << DstShift)
87 #define DstDI (OpDI << DstShift)
88 #define DstMem64 (OpMem64 << DstShift)
89 #define DstImmUByte (OpImmUByte << DstShift)
90 #define DstDX (OpDX << DstShift)
91 #define DstAccLo (OpAccLo << DstShift)
92 #define DstMask (OpMask << DstShift)
93 /* Source operand type. */
95 #define SrcNone (OpNone << SrcShift)
96 #define SrcReg (OpReg << SrcShift)
97 #define SrcMem (OpMem << SrcShift)
98 #define SrcMem16 (OpMem16 << SrcShift)
99 #define SrcMem32 (OpMem32 << SrcShift)
100 #define SrcImm (OpImm << SrcShift)
101 #define SrcImmByte (OpImmByte << SrcShift)
102 #define SrcOne (OpOne << SrcShift)
103 #define SrcImmUByte (OpImmUByte << SrcShift)
104 #define SrcImmU (OpImmU << SrcShift)
105 #define SrcSI (OpSI << SrcShift)
106 #define SrcXLat (OpXLat << SrcShift)
107 #define SrcImmFAddr (OpImmFAddr << SrcShift)
108 #define SrcMemFAddr (OpMemFAddr << SrcShift)
109 #define SrcAcc (OpAcc << SrcShift)
110 #define SrcImmU16 (OpImmU16 << SrcShift)
111 #define SrcImm64 (OpImm64 << SrcShift)
112 #define SrcDX (OpDX << SrcShift)
113 #define SrcMem8 (OpMem8 << SrcShift)
114 #define SrcAccHi (OpAccHi << SrcShift)
115 #define SrcMask (OpMask << SrcShift)
116 #define BitOp (1<<11)
117 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
118 #define String (1<<13) /* String instruction (rep capable) */
119 #define Stack (1<<14) /* Stack instruction (push/pop) */
120 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
121 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
122 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
123 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
124 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
125 #define Escape (5<<15) /* Escape to coprocessor instruction */
126 #define Sse (1<<18) /* SSE Vector instruction */
127 /* Generic ModRM decode. */
128 #define ModRM (1<<19)
129 /* Destination is only written; never read. */
132 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
133 #define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
134 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
135 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
136 #define Undefined (1<<25) /* No Such Instruction */
137 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
138 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
140 #define PageTable (1 << 29) /* instruction used to write page table */
141 #define NotImpl (1 << 30) /* instruction is not implemented */
142 /* Source 2 operand type */
143 #define Src2Shift (31)
144 #define Src2None (OpNone << Src2Shift)
145 #define Src2Mem (OpMem << Src2Shift)
146 #define Src2CL (OpCL << Src2Shift)
147 #define Src2ImmByte (OpImmByte << Src2Shift)
148 #define Src2One (OpOne << Src2Shift)
149 #define Src2Imm (OpImm << Src2Shift)
150 #define Src2ES (OpES << Src2Shift)
151 #define Src2CS (OpCS << Src2Shift)
152 #define Src2SS (OpSS << Src2Shift)
153 #define Src2DS (OpDS << Src2Shift)
154 #define Src2FS (OpFS << Src2Shift)
155 #define Src2GS (OpGS << Src2Shift)
156 #define Src2Mask (OpMask << Src2Shift)
157 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
158 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
159 #define Unaligned ((u64)1 << 42) /* Explicitly unaligned (e.g. MOVDQU) */
160 #define Avx ((u64)1 << 43) /* Advanced Vector Extensions */
161 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
162 #define NoWrite ((u64)1 << 45) /* No writeback */
163 #define SrcWrite ((u64)1 << 46) /* Write back src operand */
164 #define NoMod ((u64)1 << 47) /* Mod field is ignored */
166 #define DstXacc (DstAccLo | SrcAccHi | SrcWrite)
168 #define X2(x...) x, x
169 #define X3(x...) X2(x), x
170 #define X4(x...) X2(x), X2(x)
171 #define X5(x...) X4(x), x
172 #define X6(x...) X4(x), X2(x)
173 #define X7(x...) X4(x), X3(x)
174 #define X8(x...) X4(x), X4(x)
175 #define X16(x...) X8(x), X8(x)
177 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
178 #define FASTOP_SIZE 8
181 * fastop functions have a special calling convention:
186 * flags: rflags (in/out)
187 * ex: rsi (in:fastop pointer, out:zero if exception)
189 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
190 * different operand sizes can be reached by calculation, rather than a jump
191 * table (which would be bigger than the code).
193 * fastop functions are declared as taking a never-defined fastop parameter,
194 * so they can't be called from C directly.
203 int (*execute)(struct x86_emulate_ctxt *ctxt);
204 const struct opcode *group;
205 const struct group_dual *gdual;
206 const struct gprefix *gprefix;
207 const struct escape *esc;
208 void (*fastop)(struct fastop *fake);
210 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
214 struct opcode mod012[8];
215 struct opcode mod3[8];
219 struct opcode pfx_no;
220 struct opcode pfx_66;
221 struct opcode pfx_f2;
222 struct opcode pfx_f3;
227 struct opcode high[64];
230 /* EFLAGS bit definitions. */
231 #define EFLG_ID (1<<21)
232 #define EFLG_VIP (1<<20)
233 #define EFLG_VIF (1<<19)
234 #define EFLG_AC (1<<18)
235 #define EFLG_VM (1<<17)
236 #define EFLG_RF (1<<16)
237 #define EFLG_IOPL (3<<12)
238 #define EFLG_NT (1<<14)
239 #define EFLG_OF (1<<11)
240 #define EFLG_DF (1<<10)
241 #define EFLG_IF (1<<9)
242 #define EFLG_TF (1<<8)
243 #define EFLG_SF (1<<7)
244 #define EFLG_ZF (1<<6)
245 #define EFLG_AF (1<<4)
246 #define EFLG_PF (1<<2)
247 #define EFLG_CF (1<<0)
249 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
250 #define EFLG_RESERVED_ONE_MASK 2
252 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
254 if (!(ctxt->regs_valid & (1 << nr))) {
255 ctxt->regs_valid |= 1 << nr;
256 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
258 return ctxt->_regs[nr];
261 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
263 ctxt->regs_valid |= 1 << nr;
264 ctxt->regs_dirty |= 1 << nr;
265 return &ctxt->_regs[nr];
268 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
271 return reg_write(ctxt, nr);
274 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
278 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
279 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
282 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
284 ctxt->regs_dirty = 0;
285 ctxt->regs_valid = 0;
289 * These EFLAGS bits are restored from saved value during emulation, and
290 * any changes are written back to the saved value after emulation.
292 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
300 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));
302 #define FOP_ALIGN ".align " __stringify(FASTOP_SIZE) " \n\t"
303 #define FOP_RET "ret \n\t"
305 #define FOP_START(op) \
306 extern void em_##op(struct fastop *fake); \
307 asm(".pushsection .text, \"ax\" \n\t" \
308 ".global em_" #op " \n\t" \
315 #define FOPNOP() FOP_ALIGN FOP_RET
317 #define FOP1E(op, dst) \
318 FOP_ALIGN "10: " #op " %" #dst " \n\t" FOP_RET
320 #define FOP1EEX(op, dst) \
321 FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
323 #define FASTOP1(op) \
328 ON64(FOP1E(op##q, rax)) \
331 /* 1-operand, using src2 (for MUL/DIV r/m) */
332 #define FASTOP1SRC2(op, name) \
337 ON64(FOP1E(op, rcx)) \
340 /* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
341 #define FASTOP1SRC2EX(op, name) \
346 ON64(FOP1EEX(op, rcx)) \
349 #define FOP2E(op, dst, src) \
350 FOP_ALIGN #op " %" #src ", %" #dst " \n\t" FOP_RET
352 #define FASTOP2(op) \
354 FOP2E(op##b, al, dl) \
355 FOP2E(op##w, ax, dx) \
356 FOP2E(op##l, eax, edx) \
357 ON64(FOP2E(op##q, rax, rdx)) \
360 /* 2 operand, word only */
361 #define FASTOP2W(op) \
364 FOP2E(op##w, ax, dx) \
365 FOP2E(op##l, eax, edx) \
366 ON64(FOP2E(op##q, rax, rdx)) \
369 /* 2 operand, src is CL */
370 #define FASTOP2CL(op) \
372 FOP2E(op##b, al, cl) \
373 FOP2E(op##w, ax, cl) \
374 FOP2E(op##l, eax, cl) \
375 ON64(FOP2E(op##q, rax, cl)) \
378 #define FOP3E(op, dst, src, src2) \
379 FOP_ALIGN #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
381 /* 3-operand, word-only, src2=cl */
382 #define FASTOP3WCL(op) \
385 FOP3E(op##w, ax, dx, cl) \
386 FOP3E(op##l, eax, edx, cl) \
387 ON64(FOP3E(op##q, rax, rdx, cl)) \
390 /* Special case for SETcc - 1 instruction per cc */
391 #define FOP_SETCC(op) ".align 4; " #op " %al; ret \n\t"
393 asm(".global kvm_fastop_exception \n"
394 "kvm_fastop_exception: xor %esi, %esi; ret");
415 FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET
418 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
419 enum x86_intercept intercept,
420 enum x86_intercept_stage stage)
422 struct x86_instruction_info info = {
423 .intercept = intercept,
424 .rep_prefix = ctxt->rep_prefix,
425 .modrm_mod = ctxt->modrm_mod,
426 .modrm_reg = ctxt->modrm_reg,
427 .modrm_rm = ctxt->modrm_rm,
428 .src_val = ctxt->src.val64,
429 .src_bytes = ctxt->src.bytes,
430 .dst_bytes = ctxt->dst.bytes,
431 .ad_bytes = ctxt->ad_bytes,
432 .next_rip = ctxt->eip,
435 return ctxt->ops->intercept(ctxt, &info, stage);
438 static void assign_masked(ulong *dest, ulong src, ulong mask)
440 *dest = (*dest & ~mask) | (src & mask);
443 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
445 return (1UL << (ctxt->ad_bytes << 3)) - 1;
448 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
451 struct desc_struct ss;
453 if (ctxt->mode == X86EMUL_MODE_PROT64)
455 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
456 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
459 static int stack_size(struct x86_emulate_ctxt *ctxt)
461 return (__fls(stack_mask(ctxt)) + 1) >> 3;
464 /* Access/update address held in a register, based on addressing mode. */
465 static inline unsigned long
466 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
468 if (ctxt->ad_bytes == sizeof(unsigned long))
471 return reg & ad_mask(ctxt);
474 static inline unsigned long
475 register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
477 return address_mask(ctxt, reg);
480 static void masked_increment(ulong *reg, ulong mask, int inc)
482 assign_masked(reg, *reg + inc, mask);
486 register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
490 if (ctxt->ad_bytes == sizeof(unsigned long))
493 mask = ad_mask(ctxt);
494 masked_increment(reg, mask, inc);
497 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
499 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
502 static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
504 register_address_increment(ctxt, &ctxt->_eip, rel);
507 static u32 desc_limit_scaled(struct desc_struct *desc)
509 u32 limit = get_desc_limit(desc);
511 return desc->g ? (limit << 12) | 0xfff : limit;
514 static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
516 ctxt->has_seg_override = true;
517 ctxt->seg_override = seg;
520 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
522 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
525 return ctxt->ops->get_cached_segment_base(ctxt, seg);
528 static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
530 if (!ctxt->has_seg_override)
533 return ctxt->seg_override;
536 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
537 u32 error, bool valid)
539 ctxt->exception.vector = vec;
540 ctxt->exception.error_code = error;
541 ctxt->exception.error_code_valid = valid;
542 return X86EMUL_PROPAGATE_FAULT;
545 static int emulate_db(struct x86_emulate_ctxt *ctxt)
547 return emulate_exception(ctxt, DB_VECTOR, 0, false);
550 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
552 return emulate_exception(ctxt, GP_VECTOR, err, true);
555 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
557 return emulate_exception(ctxt, SS_VECTOR, err, true);
560 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
562 return emulate_exception(ctxt, UD_VECTOR, 0, false);
565 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
567 return emulate_exception(ctxt, TS_VECTOR, err, true);
570 static int emulate_de(struct x86_emulate_ctxt *ctxt)
572 return emulate_exception(ctxt, DE_VECTOR, 0, false);
575 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
577 return emulate_exception(ctxt, NM_VECTOR, 0, false);
580 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
583 struct desc_struct desc;
585 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
589 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
594 struct desc_struct desc;
596 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
597 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
601 * x86 defines three classes of vector instructions: explicitly
602 * aligned, explicitly unaligned, and the rest, which change behaviour
603 * depending on whether they're AVX encoded or not.
605 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
606 * subject to the same check.
608 static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
610 if (likely(size < 16))
613 if (ctxt->d & Aligned)
615 else if (ctxt->d & Unaligned)
617 else if (ctxt->d & Avx)
623 static int __linearize(struct x86_emulate_ctxt *ctxt,
624 struct segmented_address addr,
625 unsigned size, bool write, bool fetch,
628 struct desc_struct desc;
635 la = seg_base(ctxt, addr.seg) + addr.ea;
636 switch (ctxt->mode) {
637 case X86EMUL_MODE_PROT64:
638 if (((signed long)la << 16) >> 16 != la)
639 return emulate_gp(ctxt, 0);
642 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
646 /* code segment in protected mode or read-only data segment */
647 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
648 || !(desc.type & 2)) && write)
650 /* unreadable code segment */
651 if (!fetch && (desc.type & 8) && !(desc.type & 2))
653 lim = desc_limit_scaled(&desc);
654 if ((desc.type & 8) || !(desc.type & 4)) {
655 /* expand-up segment */
656 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
659 /* expand-down segment */
660 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
662 lim = desc.d ? 0xffffffff : 0xffff;
663 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
666 cpl = ctxt->ops->cpl(ctxt);
667 if (!(desc.type & 8)) {
671 } else if ((desc.type & 8) && !(desc.type & 4)) {
672 /* nonconforming code segment */
675 } else if ((desc.type & 8) && (desc.type & 4)) {
676 /* conforming code segment */
682 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
684 if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
685 return emulate_gp(ctxt, 0);
687 return X86EMUL_CONTINUE;
689 if (addr.seg == VCPU_SREG_SS)
690 return emulate_ss(ctxt, sel);
692 return emulate_gp(ctxt, sel);
695 static int linearize(struct x86_emulate_ctxt *ctxt,
696 struct segmented_address addr,
697 unsigned size, bool write,
700 return __linearize(ctxt, addr, size, write, false, linear);
704 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
705 struct segmented_address addr,
712 rc = linearize(ctxt, addr, size, false, &linear);
713 if (rc != X86EMUL_CONTINUE)
715 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
719 * Fetch the next byte of the instruction being emulated which is pointed to
720 * by ctxt->_eip, then increment ctxt->_eip.
722 * Also prefetch the remaining bytes of the instruction without crossing page
723 * boundary if they are not in fetch_cache yet.
725 static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
727 struct fetch_cache *fc = &ctxt->fetch;
731 if (ctxt->_eip == fc->end) {
732 unsigned long linear;
733 struct segmented_address addr = { .seg = VCPU_SREG_CS,
735 cur_size = fc->end - fc->start;
736 size = min(15UL - cur_size,
737 PAGE_SIZE - offset_in_page(ctxt->_eip));
738 rc = __linearize(ctxt, addr, size, false, true, &linear);
739 if (unlikely(rc != X86EMUL_CONTINUE))
741 rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
742 size, &ctxt->exception);
743 if (unlikely(rc != X86EMUL_CONTINUE))
747 *dest = fc->data[ctxt->_eip - fc->start];
749 return X86EMUL_CONTINUE;
752 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
753 void *dest, unsigned size)
757 /* x86 instructions are limited to 15 bytes. */
758 if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
759 return X86EMUL_UNHANDLEABLE;
761 rc = do_insn_fetch_byte(ctxt, dest++);
762 if (rc != X86EMUL_CONTINUE)
765 return X86EMUL_CONTINUE;
768 /* Fetch next part of the instruction being emulated. */
769 #define insn_fetch(_type, _ctxt) \
770 ({ unsigned long _x; \
771 rc = do_insn_fetch(_ctxt, &_x, sizeof(_type)); \
772 if (rc != X86EMUL_CONTINUE) \
777 #define insn_fetch_arr(_arr, _size, _ctxt) \
778 ({ rc = do_insn_fetch(_ctxt, _arr, (_size)); \
779 if (rc != X86EMUL_CONTINUE) \
784 * Given the 'reg' portion of a ModRM byte, and a register block, return a
785 * pointer into the block that addresses the relevant register.
786 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
788 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
792 int highbyte_regs = (ctxt->rex_prefix == 0) && byteop;
794 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
795 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
797 p = reg_rmw(ctxt, modrm_reg);
801 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
802 struct segmented_address addr,
803 u16 *size, unsigned long *address, int op_bytes)
810 rc = segmented_read_std(ctxt, addr, size, 2);
811 if (rc != X86EMUL_CONTINUE)
814 rc = segmented_read_std(ctxt, addr, address, op_bytes);
828 FASTOP1SRC2(mul, mul_ex);
829 FASTOP1SRC2(imul, imul_ex);
830 FASTOP1SRC2EX(div, div_ex);
831 FASTOP1SRC2EX(idiv, idiv_ex);
860 static u8 test_cc(unsigned int condition, unsigned long flags)
863 void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
865 flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
866 asm("push %[flags]; popf; call *%[fastop]"
867 : "=a"(rc) : [fastop]"r"(fop), [flags]"r"(flags));
871 static void fetch_register_operand(struct operand *op)
875 op->val = *(u8 *)op->addr.reg;
878 op->val = *(u16 *)op->addr.reg;
881 op->val = *(u32 *)op->addr.reg;
884 op->val = *(u64 *)op->addr.reg;
889 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
891 ctxt->ops->get_fpu(ctxt);
893 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
894 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
895 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
896 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
897 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
898 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
899 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
900 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
902 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
903 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
904 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
905 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
906 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
907 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
908 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
909 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
913 ctxt->ops->put_fpu(ctxt);
916 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
919 ctxt->ops->get_fpu(ctxt);
921 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
922 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
923 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
924 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
925 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
926 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
927 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
928 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
930 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
931 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
932 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
933 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
934 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
935 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
936 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
937 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
941 ctxt->ops->put_fpu(ctxt);
944 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
946 ctxt->ops->get_fpu(ctxt);
948 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
949 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
950 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
951 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
952 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
953 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
954 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
955 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
958 ctxt->ops->put_fpu(ctxt);
961 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
963 ctxt->ops->get_fpu(ctxt);
965 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
966 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
967 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
968 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
969 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
970 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
971 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
972 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
975 ctxt->ops->put_fpu(ctxt);
978 static int em_fninit(struct x86_emulate_ctxt *ctxt)
980 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
981 return emulate_nm(ctxt);
983 ctxt->ops->get_fpu(ctxt);
984 asm volatile("fninit");
985 ctxt->ops->put_fpu(ctxt);
986 return X86EMUL_CONTINUE;
989 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
993 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
994 return emulate_nm(ctxt);
996 ctxt->ops->get_fpu(ctxt);
997 asm volatile("fnstcw %0": "+m"(fcw));
998 ctxt->ops->put_fpu(ctxt);
1000 /* force 2 byte destination */
1001 ctxt->dst.bytes = 2;
1002 ctxt->dst.val = fcw;
1004 return X86EMUL_CONTINUE;
1007 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1011 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1012 return emulate_nm(ctxt);
1014 ctxt->ops->get_fpu(ctxt);
1015 asm volatile("fnstsw %0": "+m"(fsw));
1016 ctxt->ops->put_fpu(ctxt);
1018 /* force 2 byte destination */
1019 ctxt->dst.bytes = 2;
1020 ctxt->dst.val = fsw;
1022 return X86EMUL_CONTINUE;
1025 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1028 unsigned reg = ctxt->modrm_reg;
1030 if (!(ctxt->d & ModRM))
1031 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1033 if (ctxt->d & Sse) {
1037 read_sse_reg(ctxt, &op->vec_val, reg);
1040 if (ctxt->d & Mmx) {
1049 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1050 op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);
1052 fetch_register_operand(op);
1053 op->orig_val = op->val;
1056 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1058 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1059 ctxt->modrm_seg = VCPU_SREG_SS;
1062 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1066 int index_reg = 0, base_reg = 0, scale;
1067 int rc = X86EMUL_CONTINUE;
1070 if (ctxt->rex_prefix) {
1071 ctxt->modrm_reg = (ctxt->rex_prefix & 4) << 1; /* REX.R */
1072 index_reg = (ctxt->rex_prefix & 2) << 2; /* REX.X */
1073 ctxt->modrm_rm = base_reg = (ctxt->rex_prefix & 1) << 3; /* REG.B */
1076 ctxt->modrm_mod |= (ctxt->modrm & 0xc0) >> 6;
1077 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1078 ctxt->modrm_rm |= (ctxt->modrm & 0x07);
1079 ctxt->modrm_seg = VCPU_SREG_DS;
1081 if (ctxt->modrm_mod == 3 || (ctxt->d & NoMod)) {
1083 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1084 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
1086 if (ctxt->d & Sse) {
1089 op->addr.xmm = ctxt->modrm_rm;
1090 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1093 if (ctxt->d & Mmx) {
1096 op->addr.xmm = ctxt->modrm_rm & 7;
1099 fetch_register_operand(op);
1105 if (ctxt->ad_bytes == 2) {
1106 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1107 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1108 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1109 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1111 /* 16-bit ModR/M decode. */
1112 switch (ctxt->modrm_mod) {
1114 if (ctxt->modrm_rm == 6)
1115 modrm_ea += insn_fetch(u16, ctxt);
1118 modrm_ea += insn_fetch(s8, ctxt);
1121 modrm_ea += insn_fetch(u16, ctxt);
1124 switch (ctxt->modrm_rm) {
1126 modrm_ea += bx + si;
1129 modrm_ea += bx + di;
1132 modrm_ea += bp + si;
1135 modrm_ea += bp + di;
1144 if (ctxt->modrm_mod != 0)
1151 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1152 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1153 ctxt->modrm_seg = VCPU_SREG_SS;
1154 modrm_ea = (u16)modrm_ea;
1156 /* 32/64-bit ModR/M decode. */
1157 if ((ctxt->modrm_rm & 7) == 4) {
1158 sib = insn_fetch(u8, ctxt);
1159 index_reg |= (sib >> 3) & 7;
1160 base_reg |= sib & 7;
1163 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1164 modrm_ea += insn_fetch(s32, ctxt);
1166 modrm_ea += reg_read(ctxt, base_reg);
1167 adjust_modrm_seg(ctxt, base_reg);
1170 modrm_ea += reg_read(ctxt, index_reg) << scale;
1171 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1172 if (ctxt->mode == X86EMUL_MODE_PROT64)
1173 ctxt->rip_relative = 1;
1175 base_reg = ctxt->modrm_rm;
1176 modrm_ea += reg_read(ctxt, base_reg);
1177 adjust_modrm_seg(ctxt, base_reg);
1179 switch (ctxt->modrm_mod) {
1181 if (ctxt->modrm_rm == 5)
1182 modrm_ea += insn_fetch(s32, ctxt);
1185 modrm_ea += insn_fetch(s8, ctxt);
1188 modrm_ea += insn_fetch(s32, ctxt);
1192 op->addr.mem.ea = modrm_ea;
1197 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1200 int rc = X86EMUL_CONTINUE;
1203 switch (ctxt->ad_bytes) {
1205 op->addr.mem.ea = insn_fetch(u16, ctxt);
1208 op->addr.mem.ea = insn_fetch(u32, ctxt);
1211 op->addr.mem.ea = insn_fetch(u64, ctxt);
1218 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1222 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1223 mask = ~(ctxt->dst.bytes * 8 - 1);
1225 if (ctxt->src.bytes == 2)
1226 sv = (s16)ctxt->src.val & (s16)mask;
1227 else if (ctxt->src.bytes == 4)
1228 sv = (s32)ctxt->src.val & (s32)mask;
1230 ctxt->dst.addr.mem.ea += (sv >> 3);
1233 /* only subword offset */
1234 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1237 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1238 unsigned long addr, void *dest, unsigned size)
1241 struct read_cache *mc = &ctxt->mem_read;
1243 if (mc->pos < mc->end)
1246 WARN_ON((mc->end + size) >= sizeof(mc->data));
1248 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1250 if (rc != X86EMUL_CONTINUE)
1256 memcpy(dest, mc->data + mc->pos, size);
1258 return X86EMUL_CONTINUE;
1261 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1262 struct segmented_address addr,
1269 rc = linearize(ctxt, addr, size, false, &linear);
1270 if (rc != X86EMUL_CONTINUE)
1272 return read_emulated(ctxt, linear, data, size);
1275 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1276 struct segmented_address addr,
1283 rc = linearize(ctxt, addr, size, true, &linear);
1284 if (rc != X86EMUL_CONTINUE)
1286 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1290 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1291 struct segmented_address addr,
1292 const void *orig_data, const void *data,
1298 rc = linearize(ctxt, addr, size, true, &linear);
1299 if (rc != X86EMUL_CONTINUE)
1301 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1302 size, &ctxt->exception);
1305 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1306 unsigned int size, unsigned short port,
1309 struct read_cache *rc = &ctxt->io_read;
1311 if (rc->pos == rc->end) { /* refill pio read ahead */
1312 unsigned int in_page, n;
1313 unsigned int count = ctxt->rep_prefix ?
1314 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1315 in_page = (ctxt->eflags & EFLG_DF) ?
1316 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1317 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1318 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1322 rc->pos = rc->end = 0;
1323 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1328 if (ctxt->rep_prefix && (ctxt->d & String) &&
1329 !(ctxt->eflags & EFLG_DF)) {
1330 ctxt->dst.data = rc->data + rc->pos;
1331 ctxt->dst.type = OP_MEM_STR;
1332 ctxt->dst.count = (rc->end - rc->pos) / size;
1335 memcpy(dest, rc->data + rc->pos, size);
1341 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1342 u16 index, struct desc_struct *desc)
1347 ctxt->ops->get_idt(ctxt, &dt);
1349 if (dt.size < index * 8 + 7)
1350 return emulate_gp(ctxt, index << 3 | 0x2);
1352 addr = dt.address + index * 8;
1353 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1357 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1358 u16 selector, struct desc_ptr *dt)
1360 const struct x86_emulate_ops *ops = ctxt->ops;
1362 if (selector & 1 << 2) {
1363 struct desc_struct desc;
1366 memset (dt, 0, sizeof *dt);
1367 if (!ops->get_segment(ctxt, &sel, &desc, NULL, VCPU_SREG_LDTR))
1370 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1371 dt->address = get_desc_base(&desc);
1373 ops->get_gdt(ctxt, dt);
1376 /* allowed just for 8 bytes segments */
1377 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1378 u16 selector, struct desc_struct *desc,
1382 u16 index = selector >> 3;
1385 get_descriptor_table_ptr(ctxt, selector, &dt);
1387 if (dt.size < index * 8 + 7)
1388 return emulate_gp(ctxt, selector & 0xfffc);
1390 *desc_addr_p = addr = dt.address + index * 8;
1391 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1395 /* allowed just for 8 bytes segments */
1396 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1397 u16 selector, struct desc_struct *desc)
1400 u16 index = selector >> 3;
1403 get_descriptor_table_ptr(ctxt, selector, &dt);
1405 if (dt.size < index * 8 + 7)
1406 return emulate_gp(ctxt, selector & 0xfffc);
1408 addr = dt.address + index * 8;
1409 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1413 /* Does not support long mode */
1414 static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1415 u16 selector, int seg, u8 cpl, bool in_task_switch)
1417 struct desc_struct seg_desc, old_desc;
1419 unsigned err_vec = GP_VECTOR;
1421 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1426 memset(&seg_desc, 0, sizeof seg_desc);
1428 if (ctxt->mode == X86EMUL_MODE_REAL) {
1429 /* set real mode segment descriptor (keep limit etc. for
1431 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1432 set_desc_base(&seg_desc, selector << 4);
1434 } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
1435 /* VM86 needs a clean new segment descriptor */
1436 set_desc_base(&seg_desc, selector << 4);
1437 set_desc_limit(&seg_desc, 0xffff);
1447 /* NULL selector is not valid for TR, CS and SS (except for long mode) */
1448 if ((seg == VCPU_SREG_CS
1449 || (seg == VCPU_SREG_SS
1450 && (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl))
1451 || seg == VCPU_SREG_TR)
1455 /* TR should be in GDT only */
1456 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1459 if (null_selector) /* for NULL selector skip all following checks */
1462 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1463 if (ret != X86EMUL_CONTINUE)
1466 err_code = selector & 0xfffc;
1467 err_vec = GP_VECTOR;
1469 /* can't load system descriptor into segment selector */
1470 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1474 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1483 * segment is not a writable data segment or segment
1484 * selector's RPL != CPL or segment selector's RPL != CPL
1486 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1490 if (in_task_switch && rpl != dpl)
1493 if (!(seg_desc.type & 8))
1496 if (seg_desc.type & 4) {
1502 if (rpl > cpl || dpl != cpl)
1505 /* CS(RPL) <- CPL */
1506 selector = (selector & 0xfffc) | cpl;
1509 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1511 old_desc = seg_desc;
1512 seg_desc.type |= 2; /* busy */
1513 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1514 sizeof(seg_desc), &ctxt->exception);
1515 if (ret != X86EMUL_CONTINUE)
1518 case VCPU_SREG_LDTR:
1519 if (seg_desc.s || seg_desc.type != 2)
1522 default: /* DS, ES, FS, or GS */
1524 * segment is not a data or readable code segment or
1525 * ((segment is a data or nonconforming code segment)
1526 * and (both RPL and CPL > DPL))
1528 if ((seg_desc.type & 0xa) == 0x8 ||
1529 (((seg_desc.type & 0xc) != 0xc) &&
1530 (rpl > dpl && cpl > dpl)))
1536 /* mark segment as accessed */
1538 ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1539 if (ret != X86EMUL_CONTINUE)
1543 ctxt->ops->set_segment(ctxt, selector, &seg_desc, 0, seg);
1544 return X86EMUL_CONTINUE;
1546 emulate_exception(ctxt, err_vec, err_code, true);
1547 return X86EMUL_PROPAGATE_FAULT;
1550 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1551 u16 selector, int seg)
1553 u8 cpl = ctxt->ops->cpl(ctxt);
1554 return __load_segment_descriptor(ctxt, selector, seg, cpl, false);
1557 static void write_register_operand(struct operand *op)
1559 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1560 switch (op->bytes) {
1562 *(u8 *)op->addr.reg = (u8)op->val;
1565 *(u16 *)op->addr.reg = (u16)op->val;
1568 *op->addr.reg = (u32)op->val;
1569 break; /* 64b: zero-extend */
1571 *op->addr.reg = op->val;
1576 static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
1582 write_register_operand(op);
1585 if (ctxt->lock_prefix)
1586 rc = segmented_cmpxchg(ctxt,
1592 rc = segmented_write(ctxt,
1596 if (rc != X86EMUL_CONTINUE)
1600 rc = segmented_write(ctxt,
1603 op->bytes * op->count);
1604 if (rc != X86EMUL_CONTINUE)
1608 write_sse_reg(ctxt, &op->vec_val, op->addr.xmm);
1611 write_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
1619 return X86EMUL_CONTINUE;
1622 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1624 struct segmented_address addr;
1626 rsp_increment(ctxt, -bytes);
1627 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1628 addr.seg = VCPU_SREG_SS;
1630 return segmented_write(ctxt, addr, data, bytes);
1633 static int em_push(struct x86_emulate_ctxt *ctxt)
1635 /* Disable writeback. */
1636 ctxt->dst.type = OP_NONE;
1637 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1640 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1641 void *dest, int len)
1644 struct segmented_address addr;
1646 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1647 addr.seg = VCPU_SREG_SS;
1648 rc = segmented_read(ctxt, addr, dest, len);
1649 if (rc != X86EMUL_CONTINUE)
1652 rsp_increment(ctxt, len);
1656 static int em_pop(struct x86_emulate_ctxt *ctxt)
1658 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1661 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1662 void *dest, int len)
1665 unsigned long val, change_mask;
1666 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1667 int cpl = ctxt->ops->cpl(ctxt);
1669 rc = emulate_pop(ctxt, &val, len);
1670 if (rc != X86EMUL_CONTINUE)
1673 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1674 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1676 switch(ctxt->mode) {
1677 case X86EMUL_MODE_PROT64:
1678 case X86EMUL_MODE_PROT32:
1679 case X86EMUL_MODE_PROT16:
1681 change_mask |= EFLG_IOPL;
1683 change_mask |= EFLG_IF;
1685 case X86EMUL_MODE_VM86:
1687 return emulate_gp(ctxt, 0);
1688 change_mask |= EFLG_IF;
1690 default: /* real mode */
1691 change_mask |= (EFLG_IOPL | EFLG_IF);
1695 *(unsigned long *)dest =
1696 (ctxt->eflags & ~change_mask) | (val & change_mask);
1701 static int em_popf(struct x86_emulate_ctxt *ctxt)
1703 ctxt->dst.type = OP_REG;
1704 ctxt->dst.addr.reg = &ctxt->eflags;
1705 ctxt->dst.bytes = ctxt->op_bytes;
1706 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1709 static int em_enter(struct x86_emulate_ctxt *ctxt)
1712 unsigned frame_size = ctxt->src.val;
1713 unsigned nesting_level = ctxt->src2.val & 31;
1717 return X86EMUL_UNHANDLEABLE;
1719 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1720 rc = push(ctxt, &rbp, stack_size(ctxt));
1721 if (rc != X86EMUL_CONTINUE)
1723 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1725 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1726 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1728 return X86EMUL_CONTINUE;
1731 static int em_leave(struct x86_emulate_ctxt *ctxt)
1733 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1735 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1738 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1740 int seg = ctxt->src2.val;
1742 ctxt->src.val = get_segment_selector(ctxt, seg);
1744 return em_push(ctxt);
1747 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1749 int seg = ctxt->src2.val;
1750 unsigned long selector;
1753 rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1754 if (rc != X86EMUL_CONTINUE)
1757 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1761 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1763 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1764 int rc = X86EMUL_CONTINUE;
1765 int reg = VCPU_REGS_RAX;
1767 while (reg <= VCPU_REGS_RDI) {
1768 (reg == VCPU_REGS_RSP) ?
1769 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
1772 if (rc != X86EMUL_CONTINUE)
1781 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1783 ctxt->src.val = (unsigned long)ctxt->eflags;
1784 return em_push(ctxt);
1787 static int em_popa(struct x86_emulate_ctxt *ctxt)
1789 int rc = X86EMUL_CONTINUE;
1790 int reg = VCPU_REGS_RDI;
1792 while (reg >= VCPU_REGS_RAX) {
1793 if (reg == VCPU_REGS_RSP) {
1794 rsp_increment(ctxt, ctxt->op_bytes);
1798 rc = emulate_pop(ctxt, reg_rmw(ctxt, reg), ctxt->op_bytes);
1799 if (rc != X86EMUL_CONTINUE)
1806 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1808 const struct x86_emulate_ops *ops = ctxt->ops;
1815 /* TODO: Add limit checks */
1816 ctxt->src.val = ctxt->eflags;
1818 if (rc != X86EMUL_CONTINUE)
1821 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1823 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1825 if (rc != X86EMUL_CONTINUE)
1828 ctxt->src.val = ctxt->_eip;
1830 if (rc != X86EMUL_CONTINUE)
1833 ops->get_idt(ctxt, &dt);
1835 eip_addr = dt.address + (irq << 2);
1836 cs_addr = dt.address + (irq << 2) + 2;
1838 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1839 if (rc != X86EMUL_CONTINUE)
1842 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1843 if (rc != X86EMUL_CONTINUE)
1846 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1847 if (rc != X86EMUL_CONTINUE)
1855 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1859 invalidate_registers(ctxt);
1860 rc = __emulate_int_real(ctxt, irq);
1861 if (rc == X86EMUL_CONTINUE)
1862 writeback_registers(ctxt);
1866 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1868 switch(ctxt->mode) {
1869 case X86EMUL_MODE_REAL:
1870 return __emulate_int_real(ctxt, irq);
1871 case X86EMUL_MODE_VM86:
1872 case X86EMUL_MODE_PROT16:
1873 case X86EMUL_MODE_PROT32:
1874 case X86EMUL_MODE_PROT64:
1876 /* Protected mode interrupts unimplemented yet */
1877 return X86EMUL_UNHANDLEABLE;
1881 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
1883 int rc = X86EMUL_CONTINUE;
1884 unsigned long temp_eip = 0;
1885 unsigned long temp_eflags = 0;
1886 unsigned long cs = 0;
1887 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1888 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1889 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1890 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1892 /* TODO: Add stack limit check */
1894 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
1896 if (rc != X86EMUL_CONTINUE)
1899 if (temp_eip & ~0xffff)
1900 return emulate_gp(ctxt, 0);
1902 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1904 if (rc != X86EMUL_CONTINUE)
1907 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
1909 if (rc != X86EMUL_CONTINUE)
1912 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1914 if (rc != X86EMUL_CONTINUE)
1917 ctxt->_eip = temp_eip;
1920 if (ctxt->op_bytes == 4)
1921 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1922 else if (ctxt->op_bytes == 2) {
1923 ctxt->eflags &= ~0xffff;
1924 ctxt->eflags |= temp_eflags;
1927 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1928 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1933 static int em_iret(struct x86_emulate_ctxt *ctxt)
1935 switch(ctxt->mode) {
1936 case X86EMUL_MODE_REAL:
1937 return emulate_iret_real(ctxt);
1938 case X86EMUL_MODE_VM86:
1939 case X86EMUL_MODE_PROT16:
1940 case X86EMUL_MODE_PROT32:
1941 case X86EMUL_MODE_PROT64:
1943 /* iret from protected mode unimplemented yet */
1944 return X86EMUL_UNHANDLEABLE;
1948 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
1953 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1955 rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
1956 if (rc != X86EMUL_CONTINUE)
1960 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
1961 return X86EMUL_CONTINUE;
1964 static int em_grp45(struct x86_emulate_ctxt *ctxt)
1966 int rc = X86EMUL_CONTINUE;
1968 switch (ctxt->modrm_reg) {
1969 case 2: /* call near abs */ {
1971 old_eip = ctxt->_eip;
1972 ctxt->_eip = ctxt->src.val;
1973 ctxt->src.val = old_eip;
1977 case 4: /* jmp abs */
1978 ctxt->_eip = ctxt->src.val;
1980 case 5: /* jmp far */
1981 rc = em_jmp_far(ctxt);
1990 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
1992 u64 old = ctxt->dst.orig_val64;
1994 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
1995 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
1996 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
1997 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
1998 ctxt->eflags &= ~EFLG_ZF;
2000 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2001 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2003 ctxt->eflags |= EFLG_ZF;
2005 return X86EMUL_CONTINUE;
2008 static int em_ret(struct x86_emulate_ctxt *ctxt)
2010 ctxt->dst.type = OP_REG;
2011 ctxt->dst.addr.reg = &ctxt->_eip;
2012 ctxt->dst.bytes = ctxt->op_bytes;
2013 return em_pop(ctxt);
2016 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2021 rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
2022 if (rc != X86EMUL_CONTINUE)
2024 if (ctxt->op_bytes == 4)
2025 ctxt->_eip = (u32)ctxt->_eip;
2026 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2027 if (rc != X86EMUL_CONTINUE)
2029 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2033 static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt)
2037 rc = em_ret_far(ctxt);
2038 if (rc != X86EMUL_CONTINUE)
2040 rsp_increment(ctxt, ctxt->src.val);
2041 return X86EMUL_CONTINUE;
2044 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2046 /* Save real source value, then compare EAX against destination. */
2047 ctxt->src.orig_val = ctxt->src.val;
2048 ctxt->src.val = reg_read(ctxt, VCPU_REGS_RAX);
2049 fastop(ctxt, em_cmp);
2051 if (ctxt->eflags & EFLG_ZF) {
2052 /* Success: write back to memory. */
2053 ctxt->dst.val = ctxt->src.orig_val;
2055 /* Failure: write the value we saw to EAX. */
2056 ctxt->dst.type = OP_REG;
2057 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2059 return X86EMUL_CONTINUE;
2062 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2064 int seg = ctxt->src2.val;
2068 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2070 rc = load_segment_descriptor(ctxt, sel, seg);
2071 if (rc != X86EMUL_CONTINUE)
2074 ctxt->dst.val = ctxt->src.val;
2079 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2080 struct desc_struct *cs, struct desc_struct *ss)
2082 cs->l = 0; /* will be adjusted later */
2083 set_desc_base(cs, 0); /* flat segment */
2084 cs->g = 1; /* 4kb granularity */
2085 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2086 cs->type = 0x0b; /* Read, Execute, Accessed */
2088 cs->dpl = 0; /* will be adjusted later */
2093 set_desc_base(ss, 0); /* flat segment */
2094 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2095 ss->g = 1; /* 4kb granularity */
2097 ss->type = 0x03; /* Read/Write, Accessed */
2098 ss->d = 1; /* 32bit stack segment */
2105 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2107 u32 eax, ebx, ecx, edx;
2110 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2111 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2112 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2113 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2116 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2118 const struct x86_emulate_ops *ops = ctxt->ops;
2119 u32 eax, ebx, ecx, edx;
2122 * syscall should always be enabled in longmode - so only become
2123 * vendor specific (cpuid) if other modes are active...
2125 if (ctxt->mode == X86EMUL_MODE_PROT64)
2130 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2132 * Intel ("GenuineIntel")
2133 * remark: Intel CPUs only support "syscall" in 64bit
2134 * longmode. Also an 64bit guest with a
2135 * 32bit compat-app running will #UD !! While this
2136 * behaviour can be fixed (by emulating) into AMD
2137 * response - CPUs of AMD can't behave like Intel.
2139 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2140 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2141 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2144 /* AMD ("AuthenticAMD") */
2145 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2146 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2147 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2150 /* AMD ("AMDisbetter!") */
2151 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2152 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2153 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2156 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2160 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2162 const struct x86_emulate_ops *ops = ctxt->ops;
2163 struct desc_struct cs, ss;
2168 /* syscall is not available in real mode */
2169 if (ctxt->mode == X86EMUL_MODE_REAL ||
2170 ctxt->mode == X86EMUL_MODE_VM86)
2171 return emulate_ud(ctxt);
2173 if (!(em_syscall_is_enabled(ctxt)))
2174 return emulate_ud(ctxt);
2176 ops->get_msr(ctxt, MSR_EFER, &efer);
2177 setup_syscalls_segments(ctxt, &cs, &ss);
2179 if (!(efer & EFER_SCE))
2180 return emulate_ud(ctxt);
2182 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2184 cs_sel = (u16)(msr_data & 0xfffc);
2185 ss_sel = (u16)(msr_data + 8);
2187 if (efer & EFER_LMA) {
2191 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2192 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2194 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2195 if (efer & EFER_LMA) {
2196 #ifdef CONFIG_X86_64
2197 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags & ~EFLG_RF;
2200 ctxt->mode == X86EMUL_MODE_PROT64 ?
2201 MSR_LSTAR : MSR_CSTAR, &msr_data);
2202 ctxt->_eip = msr_data;
2204 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2205 ctxt->eflags &= ~(msr_data | EFLG_RF);
2209 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2210 ctxt->_eip = (u32)msr_data;
2212 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2215 return X86EMUL_CONTINUE;
2218 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2220 const struct x86_emulate_ops *ops = ctxt->ops;
2221 struct desc_struct cs, ss;
2226 ops->get_msr(ctxt, MSR_EFER, &efer);
2227 /* inject #GP if in real mode */
2228 if (ctxt->mode == X86EMUL_MODE_REAL)
2229 return emulate_gp(ctxt, 0);
2232 * Not recognized on AMD in compat mode (but is recognized in legacy
2235 if ((ctxt->mode == X86EMUL_MODE_PROT32) && (efer & EFER_LMA)
2236 && !vendor_intel(ctxt))
2237 return emulate_ud(ctxt);
2239 /* XXX sysenter/sysexit have not been tested in 64bit mode.
2240 * Therefore, we inject an #UD.
2242 if (ctxt->mode == X86EMUL_MODE_PROT64)
2243 return emulate_ud(ctxt);
2245 setup_syscalls_segments(ctxt, &cs, &ss);
2247 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2248 switch (ctxt->mode) {
2249 case X86EMUL_MODE_PROT32:
2250 if ((msr_data & 0xfffc) == 0x0)
2251 return emulate_gp(ctxt, 0);
2253 case X86EMUL_MODE_PROT64:
2254 if (msr_data == 0x0)
2255 return emulate_gp(ctxt, 0);
2261 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2262 cs_sel = (u16)msr_data;
2263 cs_sel &= ~SELECTOR_RPL_MASK;
2264 ss_sel = cs_sel + 8;
2265 ss_sel &= ~SELECTOR_RPL_MASK;
2266 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
2271 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2272 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2274 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2275 ctxt->_eip = msr_data;
2277 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2278 *reg_write(ctxt, VCPU_REGS_RSP) = msr_data;
2280 return X86EMUL_CONTINUE;
2283 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2285 const struct x86_emulate_ops *ops = ctxt->ops;
2286 struct desc_struct cs, ss;
2289 u16 cs_sel = 0, ss_sel = 0;
2291 /* inject #GP if in real mode or Virtual 8086 mode */
2292 if (ctxt->mode == X86EMUL_MODE_REAL ||
2293 ctxt->mode == X86EMUL_MODE_VM86)
2294 return emulate_gp(ctxt, 0);
2296 setup_syscalls_segments(ctxt, &cs, &ss);
2298 if ((ctxt->rex_prefix & 0x8) != 0x0)
2299 usermode = X86EMUL_MODE_PROT64;
2301 usermode = X86EMUL_MODE_PROT32;
2305 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2307 case X86EMUL_MODE_PROT32:
2308 cs_sel = (u16)(msr_data + 16);
2309 if ((msr_data & 0xfffc) == 0x0)
2310 return emulate_gp(ctxt, 0);
2311 ss_sel = (u16)(msr_data + 24);
2313 case X86EMUL_MODE_PROT64:
2314 cs_sel = (u16)(msr_data + 32);
2315 if (msr_data == 0x0)
2316 return emulate_gp(ctxt, 0);
2317 ss_sel = cs_sel + 8;
2322 cs_sel |= SELECTOR_RPL_MASK;
2323 ss_sel |= SELECTOR_RPL_MASK;
2325 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2326 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2328 ctxt->_eip = reg_read(ctxt, VCPU_REGS_RDX);
2329 *reg_write(ctxt, VCPU_REGS_RSP) = reg_read(ctxt, VCPU_REGS_RCX);
2331 return X86EMUL_CONTINUE;
2334 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2337 if (ctxt->mode == X86EMUL_MODE_REAL)
2339 if (ctxt->mode == X86EMUL_MODE_VM86)
2341 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2342 return ctxt->ops->cpl(ctxt) > iopl;
2345 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2348 const struct x86_emulate_ops *ops = ctxt->ops;
2349 struct desc_struct tr_seg;
2352 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2353 unsigned mask = (1 << len) - 1;
2356 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2359 if (desc_limit_scaled(&tr_seg) < 103)
2361 base = get_desc_base(&tr_seg);
2362 #ifdef CONFIG_X86_64
2363 base |= ((u64)base3) << 32;
2365 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2366 if (r != X86EMUL_CONTINUE)
2368 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2370 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2371 if (r != X86EMUL_CONTINUE)
2373 if ((perm >> bit_idx) & mask)
2378 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2384 if (emulator_bad_iopl(ctxt))
2385 if (!emulator_io_port_access_allowed(ctxt, port, len))
2388 ctxt->perm_ok = true;
2393 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2394 struct tss_segment_16 *tss)
2396 tss->ip = ctxt->_eip;
2397 tss->flag = ctxt->eflags;
2398 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
2399 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
2400 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
2401 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
2402 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
2403 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
2404 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
2405 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2407 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2408 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2409 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2410 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2411 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2414 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2415 struct tss_segment_16 *tss)
2420 ctxt->_eip = tss->ip;
2421 ctxt->eflags = tss->flag | 2;
2422 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
2423 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
2424 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
2425 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
2426 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
2427 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
2428 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
2429 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
2432 * SDM says that segment selectors are loaded before segment
2435 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2436 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2437 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2438 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2439 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2444 * Now load segment descriptors. If fault happens at this stage
2445 * it is handled in a context of new task
2447 ret = __load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR, cpl, true);
2448 if (ret != X86EMUL_CONTINUE)
2450 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl, true);
2451 if (ret != X86EMUL_CONTINUE)
2453 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl, true);
2454 if (ret != X86EMUL_CONTINUE)
2456 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl, true);
2457 if (ret != X86EMUL_CONTINUE)
2459 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl, true);
2460 if (ret != X86EMUL_CONTINUE)
2463 return X86EMUL_CONTINUE;
2466 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2467 u16 tss_selector, u16 old_tss_sel,
2468 ulong old_tss_base, struct desc_struct *new_desc)
2470 const struct x86_emulate_ops *ops = ctxt->ops;
2471 struct tss_segment_16 tss_seg;
2473 u32 new_tss_base = get_desc_base(new_desc);
2475 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2477 if (ret != X86EMUL_CONTINUE)
2478 /* FIXME: need to provide precise fault address */
2481 save_state_to_tss16(ctxt, &tss_seg);
2483 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2485 if (ret != X86EMUL_CONTINUE)
2486 /* FIXME: need to provide precise fault address */
2489 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2491 if (ret != X86EMUL_CONTINUE)
2492 /* FIXME: need to provide precise fault address */
2495 if (old_tss_sel != 0xffff) {
2496 tss_seg.prev_task_link = old_tss_sel;
2498 ret = ops->write_std(ctxt, new_tss_base,
2499 &tss_seg.prev_task_link,
2500 sizeof tss_seg.prev_task_link,
2502 if (ret != X86EMUL_CONTINUE)
2503 /* FIXME: need to provide precise fault address */
2507 return load_state_from_tss16(ctxt, &tss_seg);
2510 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2511 struct tss_segment_32 *tss)
2513 /* CR3 and ldt selector are not saved intentionally */
2514 tss->eip = ctxt->_eip;
2515 tss->eflags = ctxt->eflags;
2516 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
2517 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
2518 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
2519 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
2520 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
2521 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
2522 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
2523 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
2525 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2526 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2527 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2528 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2529 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2530 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2533 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2534 struct tss_segment_32 *tss)
2539 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2540 return emulate_gp(ctxt, 0);
2541 ctxt->_eip = tss->eip;
2542 ctxt->eflags = tss->eflags | 2;
2544 /* General purpose registers */
2545 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
2546 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
2547 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
2548 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
2549 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
2550 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
2551 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
2552 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
2555 * SDM says that segment selectors are loaded before segment
2556 * descriptors. This is important because CPL checks will
2559 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2560 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2561 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2562 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2563 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2564 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2565 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2568 * If we're switching between Protected Mode and VM86, we need to make
2569 * sure to update the mode before loading the segment descriptors so
2570 * that the selectors are interpreted correctly.
2572 if (ctxt->eflags & X86_EFLAGS_VM) {
2573 ctxt->mode = X86EMUL_MODE_VM86;
2576 ctxt->mode = X86EMUL_MODE_PROT32;
2581 * Now load segment descriptors. If fault happenes at this stage
2582 * it is handled in a context of new task
2584 ret = __load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR, cpl, true);
2585 if (ret != X86EMUL_CONTINUE)
2587 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl, true);
2588 if (ret != X86EMUL_CONTINUE)
2590 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl, true);
2591 if (ret != X86EMUL_CONTINUE)
2593 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl, true);
2594 if (ret != X86EMUL_CONTINUE)
2596 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl, true);
2597 if (ret != X86EMUL_CONTINUE)
2599 ret = __load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS, cpl, true);
2600 if (ret != X86EMUL_CONTINUE)
2602 ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl, true);
2603 if (ret != X86EMUL_CONTINUE)
2606 return X86EMUL_CONTINUE;
2609 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2610 u16 tss_selector, u16 old_tss_sel,
2611 ulong old_tss_base, struct desc_struct *new_desc)
2613 const struct x86_emulate_ops *ops = ctxt->ops;
2614 struct tss_segment_32 tss_seg;
2616 u32 new_tss_base = get_desc_base(new_desc);
2617 u32 eip_offset = offsetof(struct tss_segment_32, eip);
2618 u32 ldt_sel_offset = offsetof(struct tss_segment_32, ldt_selector);
2620 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2622 if (ret != X86EMUL_CONTINUE)
2623 /* FIXME: need to provide precise fault address */
2626 save_state_to_tss32(ctxt, &tss_seg);
2628 /* Only GP registers and segment selectors are saved */
2629 ret = ops->write_std(ctxt, old_tss_base + eip_offset, &tss_seg.eip,
2630 ldt_sel_offset - eip_offset, &ctxt->exception);
2631 if (ret != X86EMUL_CONTINUE)
2632 /* FIXME: need to provide precise fault address */
2635 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2637 if (ret != X86EMUL_CONTINUE)
2638 /* FIXME: need to provide precise fault address */
2641 if (old_tss_sel != 0xffff) {
2642 tss_seg.prev_task_link = old_tss_sel;
2644 ret = ops->write_std(ctxt, new_tss_base,
2645 &tss_seg.prev_task_link,
2646 sizeof tss_seg.prev_task_link,
2648 if (ret != X86EMUL_CONTINUE)
2649 /* FIXME: need to provide precise fault address */
2653 return load_state_from_tss32(ctxt, &tss_seg);
2656 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2657 u16 tss_selector, int idt_index, int reason,
2658 bool has_error_code, u32 error_code)
2660 const struct x86_emulate_ops *ops = ctxt->ops;
2661 struct desc_struct curr_tss_desc, next_tss_desc;
2663 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2664 ulong old_tss_base =
2665 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2669 /* FIXME: old_tss_base == ~0 ? */
2671 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
2672 if (ret != X86EMUL_CONTINUE)
2674 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
2675 if (ret != X86EMUL_CONTINUE)
2678 /* FIXME: check that next_tss_desc is tss */
2681 * Check privileges. The three cases are task switch caused by...
2683 * 1. jmp/call/int to task gate: Check against DPL of the task gate
2684 * 2. Exception/IRQ/iret: No check is performed
2685 * 3. jmp/call to TSS: Check against DPL of the TSS
2687 if (reason == TASK_SWITCH_GATE) {
2688 if (idt_index != -1) {
2689 /* Software interrupts */
2690 struct desc_struct task_gate_desc;
2693 ret = read_interrupt_descriptor(ctxt, idt_index,
2695 if (ret != X86EMUL_CONTINUE)
2698 dpl = task_gate_desc.dpl;
2699 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2700 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
2702 } else if (reason != TASK_SWITCH_IRET) {
2703 int dpl = next_tss_desc.dpl;
2704 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2705 return emulate_gp(ctxt, tss_selector);
2709 desc_limit = desc_limit_scaled(&next_tss_desc);
2710 if (!next_tss_desc.p ||
2711 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2712 desc_limit < 0x2b)) {
2713 emulate_ts(ctxt, tss_selector & 0xfffc);
2714 return X86EMUL_PROPAGATE_FAULT;
2717 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2718 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2719 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2722 if (reason == TASK_SWITCH_IRET)
2723 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2725 /* set back link to prev task only if NT bit is set in eflags
2726 note that old_tss_sel is not used after this point */
2727 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2728 old_tss_sel = 0xffff;
2730 if (next_tss_desc.type & 8)
2731 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2732 old_tss_base, &next_tss_desc);
2734 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2735 old_tss_base, &next_tss_desc);
2736 if (ret != X86EMUL_CONTINUE)
2739 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2740 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2742 if (reason != TASK_SWITCH_IRET) {
2743 next_tss_desc.type |= (1 << 1); /* set busy flag */
2744 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2747 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2748 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2750 if (has_error_code) {
2751 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2752 ctxt->lock_prefix = 0;
2753 ctxt->src.val = (unsigned long) error_code;
2754 ret = em_push(ctxt);
2760 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2761 u16 tss_selector, int idt_index, int reason,
2762 bool has_error_code, u32 error_code)
2766 invalidate_registers(ctxt);
2767 ctxt->_eip = ctxt->eip;
2768 ctxt->dst.type = OP_NONE;
2770 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
2771 has_error_code, error_code);
2773 if (rc == X86EMUL_CONTINUE) {
2774 ctxt->eip = ctxt->_eip;
2775 writeback_registers(ctxt);
2778 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2781 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
2784 int df = (ctxt->eflags & EFLG_DF) ? -op->count : op->count;
2786 register_address_increment(ctxt, reg_rmw(ctxt, reg), df * op->bytes);
2787 op->addr.mem.ea = register_address(ctxt, reg_read(ctxt, reg));
2790 static int em_das(struct x86_emulate_ctxt *ctxt)
2793 bool af, cf, old_cf;
2795 cf = ctxt->eflags & X86_EFLAGS_CF;
2801 af = ctxt->eflags & X86_EFLAGS_AF;
2802 if ((al & 0x0f) > 9 || af) {
2804 cf = old_cf | (al >= 250);
2809 if (old_al > 0x99 || old_cf) {
2815 /* Set PF, ZF, SF */
2816 ctxt->src.type = OP_IMM;
2818 ctxt->src.bytes = 1;
2819 fastop(ctxt, em_or);
2820 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2822 ctxt->eflags |= X86_EFLAGS_CF;
2824 ctxt->eflags |= X86_EFLAGS_AF;
2825 return X86EMUL_CONTINUE;
2828 static int em_aam(struct x86_emulate_ctxt *ctxt)
2832 if (ctxt->src.val == 0)
2833 return emulate_de(ctxt);
2835 al = ctxt->dst.val & 0xff;
2836 ah = al / ctxt->src.val;
2837 al %= ctxt->src.val;
2839 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);
2841 /* Set PF, ZF, SF */
2842 ctxt->src.type = OP_IMM;
2844 ctxt->src.bytes = 1;
2845 fastop(ctxt, em_or);
2847 return X86EMUL_CONTINUE;
2850 static int em_aad(struct x86_emulate_ctxt *ctxt)
2852 u8 al = ctxt->dst.val & 0xff;
2853 u8 ah = (ctxt->dst.val >> 8) & 0xff;
2855 al = (al + (ah * ctxt->src.val)) & 0xff;
2857 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
2859 /* Set PF, ZF, SF */
2860 ctxt->src.type = OP_IMM;
2862 ctxt->src.bytes = 1;
2863 fastop(ctxt, em_or);
2865 return X86EMUL_CONTINUE;
2868 static int em_call(struct x86_emulate_ctxt *ctxt)
2870 long rel = ctxt->src.val;
2872 ctxt->src.val = (unsigned long)ctxt->_eip;
2874 return em_push(ctxt);
2877 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2883 old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2884 old_eip = ctxt->_eip;
2886 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2887 if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
2888 return X86EMUL_CONTINUE;
2891 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2893 ctxt->src.val = old_cs;
2895 if (rc != X86EMUL_CONTINUE)
2898 ctxt->src.val = old_eip;
2899 return em_push(ctxt);
2902 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2906 ctxt->dst.type = OP_REG;
2907 ctxt->dst.addr.reg = &ctxt->_eip;
2908 ctxt->dst.bytes = ctxt->op_bytes;
2909 rc = emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
2910 if (rc != X86EMUL_CONTINUE)
2912 rsp_increment(ctxt, ctxt->src.val);
2913 return X86EMUL_CONTINUE;
2916 static int em_xchg(struct x86_emulate_ctxt *ctxt)
2918 /* Write back the register source. */
2919 ctxt->src.val = ctxt->dst.val;
2920 write_register_operand(&ctxt->src);
2922 /* Write back the memory destination with implicit LOCK prefix. */
2923 ctxt->dst.val = ctxt->src.orig_val;
2924 ctxt->lock_prefix = 1;
2925 return X86EMUL_CONTINUE;
2928 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2930 ctxt->dst.val = ctxt->src2.val;
2931 return fastop(ctxt, em_imul);
2934 static int em_cwd(struct x86_emulate_ctxt *ctxt)
2936 ctxt->dst.type = OP_REG;
2937 ctxt->dst.bytes = ctxt->src.bytes;
2938 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
2939 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
2941 return X86EMUL_CONTINUE;
2944 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2948 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
2949 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
2950 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
2951 return X86EMUL_CONTINUE;
2954 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
2958 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
2959 return emulate_gp(ctxt, 0);
2960 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
2961 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
2962 return X86EMUL_CONTINUE;
2965 static int em_mov(struct x86_emulate_ctxt *ctxt)
2967 memcpy(ctxt->dst.valptr, ctxt->src.valptr, ctxt->op_bytes);
2968 return X86EMUL_CONTINUE;
2971 #define FFL(x) bit(X86_FEATURE_##x)
2973 static int em_movbe(struct x86_emulate_ctxt *ctxt)
2975 u32 ebx, ecx, edx, eax = 1;
2979 * Check MOVBE is set in the guest-visible CPUID leaf.
2981 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2982 if (!(ecx & FFL(MOVBE)))
2983 return emulate_ud(ctxt);
2985 switch (ctxt->op_bytes) {
2988 * From MOVBE definition: "...When the operand size is 16 bits,
2989 * the upper word of the destination register remains unchanged
2992 * Both casting ->valptr and ->val to u16 breaks strict aliasing
2993 * rules so we have to do the operation almost per hand.
2995 tmp = (u16)ctxt->src.val;
2996 ctxt->dst.val &= ~0xffffUL;
2997 ctxt->dst.val |= (unsigned long)swab16(tmp);
3000 ctxt->dst.val = swab32((u32)ctxt->src.val);
3003 ctxt->dst.val = swab64(ctxt->src.val);
3006 return X86EMUL_PROPAGATE_FAULT;
3008 return X86EMUL_CONTINUE;
3011 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3013 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3014 return emulate_gp(ctxt, 0);
3016 /* Disable writeback. */
3017 ctxt->dst.type = OP_NONE;
3018 return X86EMUL_CONTINUE;
3021 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3025 if (ctxt->mode == X86EMUL_MODE_PROT64)
3026 val = ctxt->src.val & ~0ULL;
3028 val = ctxt->src.val & ~0U;
3030 /* #UD condition is already handled. */
3031 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3032 return emulate_gp(ctxt, 0);
3034 /* Disable writeback. */
3035 ctxt->dst.type = OP_NONE;
3036 return X86EMUL_CONTINUE;
3039 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3043 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3044 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3045 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3046 return emulate_gp(ctxt, 0);
3048 return X86EMUL_CONTINUE;
3051 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3055 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3056 return emulate_gp(ctxt, 0);
3058 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3059 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3060 return X86EMUL_CONTINUE;
3063 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3065 if (ctxt->modrm_reg > VCPU_SREG_GS)
3066 return emulate_ud(ctxt);
3068 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3069 return X86EMUL_CONTINUE;
3072 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3074 u16 sel = ctxt->src.val;
3076 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3077 return emulate_ud(ctxt);
3079 if (ctxt->modrm_reg == VCPU_SREG_SS)
3080 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3082 /* Disable writeback. */
3083 ctxt->dst.type = OP_NONE;
3084 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3087 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3089 u16 sel = ctxt->src.val;
3091 /* Disable writeback. */
3092 ctxt->dst.type = OP_NONE;
3093 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3096 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3098 u16 sel = ctxt->src.val;
3100 /* Disable writeback. */
3101 ctxt->dst.type = OP_NONE;
3102 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3105 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3110 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3111 if (rc == X86EMUL_CONTINUE)
3112 ctxt->ops->invlpg(ctxt, linear);
3113 /* Disable writeback. */
3114 ctxt->dst.type = OP_NONE;
3115 return X86EMUL_CONTINUE;
3118 static int em_clts(struct x86_emulate_ctxt *ctxt)
3122 cr0 = ctxt->ops->get_cr(ctxt, 0);
3124 ctxt->ops->set_cr(ctxt, 0, cr0);
3125 return X86EMUL_CONTINUE;
3128 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
3132 if (ctxt->modrm_mod != 3 || ctxt->modrm_rm != 1)
3133 return X86EMUL_UNHANDLEABLE;
3135 rc = ctxt->ops->fix_hypercall(ctxt);
3136 if (rc != X86EMUL_CONTINUE)
3139 /* Let the processor re-execute the fixed hypercall */
3140 ctxt->_eip = ctxt->eip;
3141 /* Disable writeback. */
3142 ctxt->dst.type = OP_NONE;
3143 return X86EMUL_CONTINUE;
3146 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3147 void (*get)(struct x86_emulate_ctxt *ctxt,
3148 struct desc_ptr *ptr))
3150 struct desc_ptr desc_ptr;
3152 if (ctxt->mode == X86EMUL_MODE_PROT64)
3154 get(ctxt, &desc_ptr);
3155 if (ctxt->op_bytes == 2) {
3157 desc_ptr.address &= 0x00ffffff;
3159 /* Disable writeback. */
3160 ctxt->dst.type = OP_NONE;
3161 return segmented_write(ctxt, ctxt->dst.addr.mem,
3162 &desc_ptr, 2 + ctxt->op_bytes);
3165 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3167 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3170 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3172 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3175 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3177 struct desc_ptr desc_ptr;
3180 if (ctxt->mode == X86EMUL_MODE_PROT64)
3182 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3183 &desc_ptr.size, &desc_ptr.address,
3185 if (rc != X86EMUL_CONTINUE)
3187 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3188 /* Disable writeback. */
3189 ctxt->dst.type = OP_NONE;
3190 return X86EMUL_CONTINUE;
3193 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
3197 rc = ctxt->ops->fix_hypercall(ctxt);
3199 /* Disable writeback. */
3200 ctxt->dst.type = OP_NONE;
3204 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3206 struct desc_ptr desc_ptr;
3209 if (ctxt->mode == X86EMUL_MODE_PROT64)
3211 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3212 &desc_ptr.size, &desc_ptr.address,
3214 if (rc != X86EMUL_CONTINUE)
3216 ctxt->ops->set_idt(ctxt, &desc_ptr);
3217 /* Disable writeback. */
3218 ctxt->dst.type = OP_NONE;
3219 return X86EMUL_CONTINUE;
3222 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3224 ctxt->dst.bytes = 2;
3225 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3226 return X86EMUL_CONTINUE;
3229 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3231 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3232 | (ctxt->src.val & 0x0f));
3233 ctxt->dst.type = OP_NONE;
3234 return X86EMUL_CONTINUE;
3237 static int em_loop(struct x86_emulate_ctxt *ctxt)
3239 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX), -1);
3240 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3241 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3242 jmp_rel(ctxt, ctxt->src.val);
3244 return X86EMUL_CONTINUE;
3247 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3249 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3250 jmp_rel(ctxt, ctxt->src.val);
3252 return X86EMUL_CONTINUE;
3255 static int em_in(struct x86_emulate_ctxt *ctxt)
3257 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3259 return X86EMUL_IO_NEEDED;
3261 return X86EMUL_CONTINUE;
3264 static int em_out(struct x86_emulate_ctxt *ctxt)
3266 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3268 /* Disable writeback. */
3269 ctxt->dst.type = OP_NONE;
3270 return X86EMUL_CONTINUE;
3273 static int em_cli(struct x86_emulate_ctxt *ctxt)
3275 if (emulator_bad_iopl(ctxt))
3276 return emulate_gp(ctxt, 0);
3278 ctxt->eflags &= ~X86_EFLAGS_IF;
3279 return X86EMUL_CONTINUE;
3282 static int em_sti(struct x86_emulate_ctxt *ctxt)
3284 if (emulator_bad_iopl(ctxt))
3285 return emulate_gp(ctxt, 0);
3287 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3288 ctxt->eflags |= X86_EFLAGS_IF;
3289 return X86EMUL_CONTINUE;
3292 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3294 u32 eax, ebx, ecx, edx;
3296 eax = reg_read(ctxt, VCPU_REGS_RAX);
3297 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3298 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3299 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3300 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3301 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3302 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3303 return X86EMUL_CONTINUE;
3306 static int em_sahf(struct x86_emulate_ctxt *ctxt)
3310 flags = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF;
3311 flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8;
3313 ctxt->eflags &= ~0xffUL;
3314 ctxt->eflags |= flags | X86_EFLAGS_FIXED;
3315 return X86EMUL_CONTINUE;
3318 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3320 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3321 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3322 return X86EMUL_CONTINUE;
3325 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3327 switch (ctxt->op_bytes) {
3328 #ifdef CONFIG_X86_64
3330 asm("bswap %0" : "+r"(ctxt->dst.val));
3334 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3337 return X86EMUL_CONTINUE;
3340 static bool valid_cr(int nr)
3352 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
3354 if (!valid_cr(ctxt->modrm_reg))
3355 return emulate_ud(ctxt);
3357 return X86EMUL_CONTINUE;
3360 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
3362 u64 new_val = ctxt->src.val64;
3363 int cr = ctxt->modrm_reg;
3366 static u64 cr_reserved_bits[] = {
3367 0xffffffff00000000ULL,
3368 0, 0, 0, /* CR3 checked later */
3375 return emulate_ud(ctxt);
3377 if (new_val & cr_reserved_bits[cr])
3378 return emulate_gp(ctxt, 0);
3383 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
3384 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
3385 return emulate_gp(ctxt, 0);
3387 cr4 = ctxt->ops->get_cr(ctxt, 4);
3388 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3390 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
3391 !(cr4 & X86_CR4_PAE))
3392 return emulate_gp(ctxt, 0);
3399 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3400 if (efer & EFER_LMA)
3401 rsvd = CR3_L_MODE_RESERVED_BITS;
3404 return emulate_gp(ctxt, 0);
3409 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3411 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
3412 return emulate_gp(ctxt, 0);
3418 return X86EMUL_CONTINUE;
3421 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
3425 ctxt->ops->get_dr(ctxt, 7, &dr7);
3427 /* Check if DR7.Global_Enable is set */
3428 return dr7 & (1 << 13);
3431 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
3433 int dr = ctxt->modrm_reg;
3437 return emulate_ud(ctxt);
3439 cr4 = ctxt->ops->get_cr(ctxt, 4);
3440 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
3441 return emulate_ud(ctxt);
3443 if (check_dr7_gd(ctxt))
3444 return emulate_db(ctxt);
3446 return X86EMUL_CONTINUE;
3449 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
3451 u64 new_val = ctxt->src.val64;
3452 int dr = ctxt->modrm_reg;
3454 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
3455 return emulate_gp(ctxt, 0);
3457 return check_dr_read(ctxt);
3460 static int check_svme(struct x86_emulate_ctxt *ctxt)
3464 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3466 if (!(efer & EFER_SVME))
3467 return emulate_ud(ctxt);
3469 return X86EMUL_CONTINUE;
3472 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
3474 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
3476 /* Valid physical address? */
3477 if (rax & 0xffff000000000000ULL)
3478 return emulate_gp(ctxt, 0);
3480 return check_svme(ctxt);
3483 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
3485 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3487 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3488 return emulate_ud(ctxt);
3490 return X86EMUL_CONTINUE;
3493 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
3495 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3496 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
3498 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3500 return emulate_gp(ctxt, 0);
3502 return X86EMUL_CONTINUE;
3505 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
3507 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
3508 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3509 return emulate_gp(ctxt, 0);
3511 return X86EMUL_CONTINUE;
3514 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
3516 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
3517 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3518 return emulate_gp(ctxt, 0);
3520 return X86EMUL_CONTINUE;
3523 #define D(_y) { .flags = (_y) }
3524 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
3525 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
3526 .check_perm = (_p) }
3527 #define N D(NotImpl)
3528 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3529 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
3530 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
3531 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
3532 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3533 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
3534 #define II(_f, _e, _i) \
3535 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
3536 #define IIP(_f, _e, _i, _p) \
3537 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
3538 .check_perm = (_p) }
3539 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3541 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3542 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3543 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3544 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
3545 #define I2bvIP(_f, _e, _i, _p) \
3546 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
3548 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3549 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3550 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3552 static const struct opcode group7_rm1[] = {
3553 DI(SrcNone | Priv, monitor),
3554 DI(SrcNone | Priv, mwait),
3558 static const struct opcode group7_rm3[] = {
3559 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
3560 II(SrcNone | Prot | EmulateOnUD, em_vmmcall, vmmcall),
3561 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
3562 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
3563 DIP(SrcNone | Prot | Priv, stgi, check_svme),
3564 DIP(SrcNone | Prot | Priv, clgi, check_svme),
3565 DIP(SrcNone | Prot | Priv, skinit, check_svme),
3566 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
3569 static const struct opcode group7_rm7[] = {
3571 DIP(SrcNone, rdtscp, check_rdtsc),
3575 static const struct opcode group1[] = {
3577 F(Lock | PageTable, em_or),
3580 F(Lock | PageTable, em_and),
3586 static const struct opcode group1A[] = {
3587 I(DstMem | SrcNone | Mov | Stack, em_pop), N, N, N, N, N, N, N,
3590 static const struct opcode group2[] = {
3591 F(DstMem | ModRM, em_rol),
3592 F(DstMem | ModRM, em_ror),
3593 F(DstMem | ModRM, em_rcl),
3594 F(DstMem | ModRM, em_rcr),
3595 F(DstMem | ModRM, em_shl),
3596 F(DstMem | ModRM, em_shr),
3597 F(DstMem | ModRM, em_shl),
3598 F(DstMem | ModRM, em_sar),
3601 static const struct opcode group3[] = {
3602 F(DstMem | SrcImm | NoWrite, em_test),
3603 F(DstMem | SrcImm | NoWrite, em_test),
3604 F(DstMem | SrcNone | Lock, em_not),
3605 F(DstMem | SrcNone | Lock, em_neg),
3606 F(DstXacc | Src2Mem, em_mul_ex),
3607 F(DstXacc | Src2Mem, em_imul_ex),
3608 F(DstXacc | Src2Mem, em_div_ex),
3609 F(DstXacc | Src2Mem, em_idiv_ex),
3612 static const struct opcode group4[] = {
3613 F(ByteOp | DstMem | SrcNone | Lock, em_inc),
3614 F(ByteOp | DstMem | SrcNone | Lock, em_dec),
3618 static const struct opcode group5[] = {
3619 F(DstMem | SrcNone | Lock, em_inc),
3620 F(DstMem | SrcNone | Lock, em_dec),
3621 I(SrcMem | Stack, em_grp45),
3622 I(SrcMemFAddr | ImplicitOps | Stack, em_call_far),
3623 I(SrcMem | Stack, em_grp45),
3624 I(SrcMemFAddr | ImplicitOps, em_grp45),
3625 I(SrcMem | Stack, em_grp45), D(Undefined),
3628 static const struct opcode group6[] = {
3631 II(Prot | Priv | SrcMem16, em_lldt, lldt),
3632 II(Prot | Priv | SrcMem16, em_ltr, ltr),
3636 static const struct group_dual group7 = { {
3637 II(Mov | DstMem | Priv, em_sgdt, sgdt),
3638 II(Mov | DstMem | Priv, em_sidt, sidt),
3639 II(SrcMem | Priv, em_lgdt, lgdt),
3640 II(SrcMem | Priv, em_lidt, lidt),
3641 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3642 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3643 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3645 I(SrcNone | Priv | EmulateOnUD, em_vmcall),
3647 N, EXT(0, group7_rm3),
3648 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3649 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3653 static const struct opcode group8[] = {
3655 F(DstMem | SrcImmByte | NoWrite, em_bt),
3656 F(DstMem | SrcImmByte | Lock | PageTable, em_bts),
3657 F(DstMem | SrcImmByte | Lock, em_btr),
3658 F(DstMem | SrcImmByte | Lock | PageTable, em_btc),
3661 static const struct group_dual group9 = { {
3662 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
3664 N, N, N, N, N, N, N, N,
3667 static const struct opcode group11[] = {
3668 I(DstMem | SrcImm | Mov | PageTable, em_mov),
3672 static const struct gprefix pfx_0f_6f_0f_7f = {
3673 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
3676 static const struct gprefix pfx_vmovntpx = {
3677 I(0, em_mov), N, N, N,
3680 static const struct gprefix pfx_0f_28_0f_29 = {
3681 I(Aligned, em_mov), I(Aligned, em_mov), N, N,
3684 static const struct escape escape_d9 = { {
3685 N, N, N, N, N, N, N, I(DstMem, em_fnstcw),
3688 N, N, N, N, N, N, N, N,
3690 N, N, N, N, N, N, N, N,
3692 N, N, N, N, N, N, N, N,
3694 N, N, N, N, N, N, N, N,
3696 N, N, N, N, N, N, N, N,
3698 N, N, N, N, N, N, N, N,
3700 N, N, N, N, N, N, N, N,
3702 N, N, N, N, N, N, N, N,
3705 static const struct escape escape_db = { {
3706 N, N, N, N, N, N, N, N,
3709 N, N, N, N, N, N, N, N,
3711 N, N, N, N, N, N, N, N,
3713 N, N, N, N, N, N, N, N,
3715 N, N, N, N, N, N, N, N,
3717 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
3719 N, N, N, N, N, N, N, N,
3721 N, N, N, N, N, N, N, N,
3723 N, N, N, N, N, N, N, N,
3726 static const struct escape escape_dd = { {
3727 N, N, N, N, N, N, N, I(DstMem, em_fnstsw),
3730 N, N, N, N, N, N, N, N,
3732 N, N, N, N, N, N, N, N,
3734 N, N, N, N, N, N, N, N,
3736 N, N, N, N, N, N, N, N,
3738 N, N, N, N, N, N, N, N,
3740 N, N, N, N, N, N, N, N,
3742 N, N, N, N, N, N, N, N,
3744 N, N, N, N, N, N, N, N,
3747 static const struct opcode opcode_table[256] = {
3749 F6ALU(Lock, em_add),
3750 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
3751 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
3753 F6ALU(Lock | PageTable, em_or),
3754 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
3757 F6ALU(Lock, em_adc),
3758 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
3759 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
3761 F6ALU(Lock, em_sbb),
3762 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
3763 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
3765 F6ALU(Lock | PageTable, em_and), N, N,
3767 F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3769 F6ALU(Lock, em_xor), N, N,
3771 F6ALU(NoWrite, em_cmp), N, N,
3773 X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
3775 X8(I(SrcReg | Stack, em_push)),
3777 X8(I(DstReg | Stack, em_pop)),
3779 I(ImplicitOps | Stack | No64, em_pusha),
3780 I(ImplicitOps | Stack | No64, em_popa),
3781 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3784 I(SrcImm | Mov | Stack, em_push),
3785 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3786 I(SrcImmByte | Mov | Stack, em_push),
3787 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3788 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
3789 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
3793 G(ByteOp | DstMem | SrcImm, group1),
3794 G(DstMem | SrcImm, group1),
3795 G(ByteOp | DstMem | SrcImm | No64, group1),
3796 G(DstMem | SrcImmByte, group1),
3797 F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
3798 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
3800 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
3801 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3802 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
3803 D(ModRM | SrcMem | NoAccess | DstReg),
3804 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
3807 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3809 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3810 I(SrcImmFAddr | No64, em_call_far), N,
3811 II(ImplicitOps | Stack, em_pushf, pushf),
3812 II(ImplicitOps | Stack, em_popf, popf),
3813 I(ImplicitOps, em_sahf), I(ImplicitOps, em_lahf),
3815 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3816 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
3817 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3818 F2bv(SrcSI | DstDI | String | NoWrite, em_cmp),
3820 F2bv(DstAcc | SrcImm | NoWrite, em_test),
3821 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3822 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3823 F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp),
3825 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3827 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
3829 G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
3830 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3831 I(ImplicitOps | Stack, em_ret),
3832 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
3833 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
3834 G(ByteOp, group11), G(0, group11),
3836 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
3837 I(ImplicitOps | Stack | SrcImmU16, em_ret_far_imm),
3838 I(ImplicitOps | Stack, em_ret_far),
3839 D(ImplicitOps), DI(SrcImmByte, intn),
3840 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3842 G(Src2One | ByteOp, group2), G(Src2One, group2),
3843 G(Src2CL | ByteOp, group2), G(Src2CL, group2),
3844 I(DstAcc | SrcImmUByte | No64, em_aam),
3845 I(DstAcc | SrcImmUByte | No64, em_aad),
3846 F(DstAcc | ByteOp | No64, em_salc),
3847 I(DstAcc | SrcXLat | ByteOp, em_mov),
3849 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
3851 X3(I(SrcImmByte, em_loop)),
3852 I(SrcImmByte, em_jcxz),
3853 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
3854 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
3856 I(SrcImm | Stack, em_call), D(SrcImm | ImplicitOps),
3857 I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
3858 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
3859 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
3861 N, DI(ImplicitOps, icebp), N, N,
3862 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
3863 G(ByteOp, group3), G(0, group3),
3865 D(ImplicitOps), D(ImplicitOps),
3866 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
3867 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3870 static const struct opcode twobyte_table[256] = {
3872 G(0, group6), GD(0, &group7), N, N,
3873 N, I(ImplicitOps | EmulateOnUD, em_syscall),
3874 II(ImplicitOps | Priv, em_clts, clts), N,
3875 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3876 N, D(ImplicitOps | ModRM), N, N,
3878 N, N, N, N, N, N, N, N,
3879 D(ImplicitOps | ModRM), N, N, N, N, N, N, D(ImplicitOps | ModRM),
3881 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, cr_read, check_cr_read),
3882 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, dr_read, check_dr_read),
3883 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_cr_write, cr_write,
3885 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_dr_write, dr_write,
3888 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_28_0f_29),
3889 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_28_0f_29),
3890 N, GP(ModRM | DstMem | SrcReg | Sse | Mov | Aligned, &pfx_vmovntpx),
3893 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
3894 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
3895 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
3896 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
3897 I(ImplicitOps | EmulateOnUD, em_sysenter),
3898 I(ImplicitOps | Priv | EmulateOnUD, em_sysexit),
3900 N, N, N, N, N, N, N, N,
3902 X16(D(DstReg | SrcMem | ModRM | Mov)),
3904 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3909 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3914 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3918 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3920 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
3921 II(ImplicitOps, em_cpuid, cpuid),
3922 F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
3923 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
3924 F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
3926 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
3927 DI(ImplicitOps, rsm),
3928 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
3929 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
3930 F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
3931 D(ModRM), F(DstReg | SrcMem | ModRM, em_imul),
3933 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_cmpxchg),
3934 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
3935 F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
3936 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
3937 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
3938 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3942 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
3943 F(DstReg | SrcMem | ModRM, em_bsf), F(DstReg | SrcMem | ModRM, em_bsr),
3944 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3946 F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
3947 N, D(DstMem | SrcReg | ModRM | Mov),
3948 N, N, N, GD(0, &group9),
3950 X8(I(DstReg, em_bswap)),
3952 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3954 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3956 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
3959 static const struct gprefix three_byte_0f_38_f0 = {
3960 I(DstReg | SrcMem | Mov, em_movbe), N, N, N
3963 static const struct gprefix three_byte_0f_38_f1 = {
3964 I(DstMem | SrcReg | Mov, em_movbe), N, N, N
3968 * Insns below are selected by the prefix which indexed by the third opcode
3971 static const struct opcode opcode_map_0f_38[256] = {
3973 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
3975 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
3977 GP(EmulateOnUD | ModRM | Prefix, &three_byte_0f_38_f0),
3978 GP(EmulateOnUD | ModRM | Prefix, &three_byte_0f_38_f1),
3997 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4001 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4007 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
4008 unsigned size, bool sign_extension)
4010 int rc = X86EMUL_CONTINUE;
4014 op->addr.mem.ea = ctxt->_eip;
4015 /* NB. Immediates are sign-extended as necessary. */
4016 switch (op->bytes) {
4018 op->val = insn_fetch(s8, ctxt);
4021 op->val = insn_fetch(s16, ctxt);
4024 op->val = insn_fetch(s32, ctxt);
4027 op->val = insn_fetch(s64, ctxt);
4030 if (!sign_extension) {
4031 switch (op->bytes) {
4039 op->val &= 0xffffffff;
4047 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
4050 int rc = X86EMUL_CONTINUE;
4054 decode_register_operand(ctxt, op);
4057 rc = decode_imm(ctxt, op, 1, false);
4060 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4064 if ((ctxt->d & BitOp) && op == &ctxt->dst)
4065 fetch_bit_operand(ctxt);
4066 op->orig_val = op->val;
4069 ctxt->memop.bytes = 8;
4073 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4074 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4075 fetch_register_operand(op);
4076 op->orig_val = op->val;
4080 op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;
4081 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4082 fetch_register_operand(op);
4083 op->orig_val = op->val;
4086 if (ctxt->d & ByteOp) {
4091 op->bytes = ctxt->op_bytes;
4092 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4093 fetch_register_operand(op);
4094 op->orig_val = op->val;
4098 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4100 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RDI));
4101 op->addr.mem.seg = VCPU_SREG_ES;
4108 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4109 fetch_register_operand(op);
4113 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4116 rc = decode_imm(ctxt, op, 1, true);
4123 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
4126 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
4129 ctxt->memop.bytes = 1;
4130 if (ctxt->memop.type == OP_REG) {
4131 ctxt->memop.addr.reg = decode_register(ctxt,
4132 ctxt->modrm_rm, true);
4133 fetch_register_operand(&ctxt->memop);
4137 ctxt->memop.bytes = 2;
4140 ctxt->memop.bytes = 4;
4143 rc = decode_imm(ctxt, op, 2, false);
4146 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
4150 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4152 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RSI));
4153 op->addr.mem.seg = seg_override(ctxt);
4159 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4161 register_address(ctxt,
4162 reg_read(ctxt, VCPU_REGS_RBX) +
4163 (reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
4164 op->addr.mem.seg = seg_override(ctxt);
4169 op->addr.mem.ea = ctxt->_eip;
4170 op->bytes = ctxt->op_bytes + 2;
4171 insn_fetch_arr(op->valptr, op->bytes, ctxt);
4174 ctxt->memop.bytes = ctxt->op_bytes + 2;
4177 op->val = VCPU_SREG_ES;
4180 op->val = VCPU_SREG_CS;
4183 op->val = VCPU_SREG_SS;
4186 op->val = VCPU_SREG_DS;
4189 op->val = VCPU_SREG_FS;
4192 op->val = VCPU_SREG_GS;
4195 /* Special instructions do their own operand decoding. */
4197 op->type = OP_NONE; /* Disable writeback. */
4205 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
4207 int rc = X86EMUL_CONTINUE;
4208 int mode = ctxt->mode;
4209 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
4210 bool op_prefix = false;
4211 struct opcode opcode;
4213 ctxt->memop.type = OP_NONE;
4214 ctxt->memopp = NULL;
4215 ctxt->_eip = ctxt->eip;
4216 ctxt->fetch.start = ctxt->_eip;
4217 ctxt->fetch.end = ctxt->fetch.start + insn_len;
4218 ctxt->opcode_len = 1;
4220 memcpy(ctxt->fetch.data, insn, insn_len);
4223 case X86EMUL_MODE_REAL:
4224 case X86EMUL_MODE_VM86:
4225 case X86EMUL_MODE_PROT16:
4226 def_op_bytes = def_ad_bytes = 2;
4228 case X86EMUL_MODE_PROT32:
4229 def_op_bytes = def_ad_bytes = 4;
4231 #ifdef CONFIG_X86_64
4232 case X86EMUL_MODE_PROT64:
4238 return EMULATION_FAILED;
4241 ctxt->op_bytes = def_op_bytes;
4242 ctxt->ad_bytes = def_ad_bytes;
4244 /* Legacy prefixes. */
4246 switch (ctxt->b = insn_fetch(u8, ctxt)) {
4247 case 0x66: /* operand-size override */
4249 /* switch between 2/4 bytes */
4250 ctxt->op_bytes = def_op_bytes ^ 6;
4252 case 0x67: /* address-size override */
4253 if (mode == X86EMUL_MODE_PROT64)
4254 /* switch between 4/8 bytes */
4255 ctxt->ad_bytes = def_ad_bytes ^ 12;
4257 /* switch between 2/4 bytes */
4258 ctxt->ad_bytes = def_ad_bytes ^ 6;
4260 case 0x26: /* ES override */
4261 case 0x2e: /* CS override */
4262 case 0x36: /* SS override */
4263 case 0x3e: /* DS override */
4264 set_seg_override(ctxt, (ctxt->b >> 3) & 3);
4266 case 0x64: /* FS override */
4267 case 0x65: /* GS override */
4268 set_seg_override(ctxt, ctxt->b & 7);
4270 case 0x40 ... 0x4f: /* REX */
4271 if (mode != X86EMUL_MODE_PROT64)
4273 ctxt->rex_prefix = ctxt->b;
4275 case 0xf0: /* LOCK */
4276 ctxt->lock_prefix = 1;
4278 case 0xf2: /* REPNE/REPNZ */
4279 case 0xf3: /* REP/REPE/REPZ */
4280 ctxt->rep_prefix = ctxt->b;
4286 /* Any legacy prefix after a REX prefix nullifies its effect. */
4288 ctxt->rex_prefix = 0;
4294 if (ctxt->rex_prefix & 8)
4295 ctxt->op_bytes = 8; /* REX.W */
4297 /* Opcode byte(s). */
4298 opcode = opcode_table[ctxt->b];
4299 /* Two-byte opcode? */
4300 if (ctxt->b == 0x0f) {
4301 ctxt->opcode_len = 2;
4302 ctxt->b = insn_fetch(u8, ctxt);
4303 opcode = twobyte_table[ctxt->b];
4305 /* 0F_38 opcode map */
4306 if (ctxt->b == 0x38) {
4307 ctxt->opcode_len = 3;
4308 ctxt->b = insn_fetch(u8, ctxt);
4309 opcode = opcode_map_0f_38[ctxt->b];
4312 ctxt->d = opcode.flags;
4314 if (ctxt->d & ModRM)
4315 ctxt->modrm = insn_fetch(u8, ctxt);
4317 while (ctxt->d & GroupMask) {
4318 switch (ctxt->d & GroupMask) {
4320 goffset = (ctxt->modrm >> 3) & 7;
4321 opcode = opcode.u.group[goffset];
4324 goffset = (ctxt->modrm >> 3) & 7;
4325 if ((ctxt->modrm >> 6) == 3)
4326 opcode = opcode.u.gdual->mod3[goffset];
4328 opcode = opcode.u.gdual->mod012[goffset];
4331 goffset = ctxt->modrm & 7;
4332 opcode = opcode.u.group[goffset];
4335 if (ctxt->rep_prefix && op_prefix)
4336 return EMULATION_FAILED;
4337 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
4338 switch (simd_prefix) {
4339 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
4340 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
4341 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
4342 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
4346 if (ctxt->modrm > 0xbf)
4347 opcode = opcode.u.esc->high[ctxt->modrm - 0xc0];
4349 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
4352 return EMULATION_FAILED;
4355 ctxt->d &= ~(u64)GroupMask;
4356 ctxt->d |= opcode.flags;
4359 ctxt->execute = opcode.u.execute;
4360 ctxt->check_perm = opcode.check_perm;
4361 ctxt->intercept = opcode.intercept;
4364 if (ctxt->d == 0 || (ctxt->d & NotImpl))
4365 return EMULATION_FAILED;
4367 if (!(ctxt->d & EmulateOnUD) && ctxt->ud)
4368 return EMULATION_FAILED;
4370 if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
4373 if (ctxt->d & Op3264) {
4374 if (mode == X86EMUL_MODE_PROT64)
4381 ctxt->op_bytes = 16;
4382 else if (ctxt->d & Mmx)
4385 /* ModRM and SIB bytes. */
4386 if (ctxt->d & ModRM) {
4387 rc = decode_modrm(ctxt, &ctxt->memop);
4388 if (!ctxt->has_seg_override)
4389 set_seg_override(ctxt, ctxt->modrm_seg);
4390 } else if (ctxt->d & MemAbs)
4391 rc = decode_abs(ctxt, &ctxt->memop);
4392 if (rc != X86EMUL_CONTINUE)
4395 if (!ctxt->has_seg_override)
4396 set_seg_override(ctxt, VCPU_SREG_DS);
4398 ctxt->memop.addr.mem.seg = seg_override(ctxt);
4400 if (ctxt->memop.type == OP_MEM && ctxt->ad_bytes != 8)
4401 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
4404 * Decode and fetch the source operand: register, memory
4407 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
4408 if (rc != X86EMUL_CONTINUE)
4412 * Decode and fetch the second source operand: register, memory
4415 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
4416 if (rc != X86EMUL_CONTINUE)
4419 /* Decode and fetch the destination operand: register or memory. */
4420 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
4423 if (ctxt->memopp && ctxt->memopp->type == OP_MEM && ctxt->rip_relative)
4424 ctxt->memopp->addr.mem.ea += ctxt->_eip;
4426 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
4429 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
4431 return ctxt->d & PageTable;
4434 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
4436 /* The second termination condition only applies for REPE
4437 * and REPNE. Test if the repeat string operation prefix is
4438 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
4439 * corresponding termination condition according to:
4440 * - if REPE/REPZ and ZF = 0 then done
4441 * - if REPNE/REPNZ and ZF = 1 then done
4443 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
4444 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
4445 && (((ctxt->rep_prefix == REPE_PREFIX) &&
4446 ((ctxt->eflags & EFLG_ZF) == 0))
4447 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
4448 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
4454 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
4458 ctxt->ops->get_fpu(ctxt);
4459 asm volatile("1: fwait \n\t"
4461 ".pushsection .fixup,\"ax\" \n\t"
4463 "movb $1, %[fault] \n\t"
4466 _ASM_EXTABLE(1b, 3b)
4467 : [fault]"+qm"(fault));
4468 ctxt->ops->put_fpu(ctxt);
4470 if (unlikely(fault))
4471 return emulate_exception(ctxt, MF_VECTOR, 0, false);
4473 return X86EMUL_CONTINUE;
4476 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
4479 if (op->type == OP_MM)
4480 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
4483 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
4485 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
4486 if (!(ctxt->d & ByteOp))
4487 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
4488 asm("push %[flags]; popf; call *%[fastop]; pushf; pop %[flags]\n"
4489 : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
4491 : "c"(ctxt->src2.val));
4492 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
4493 if (!fop) /* exception is returned in fop variable */
4494 return emulate_de(ctxt);
4495 return X86EMUL_CONTINUE;
4498 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
4500 const struct x86_emulate_ops *ops = ctxt->ops;
4501 int rc = X86EMUL_CONTINUE;
4502 int saved_dst_type = ctxt->dst.type;
4504 ctxt->mem_read.pos = 0;
4506 if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
4507 (ctxt->d & Undefined)) {
4508 rc = emulate_ud(ctxt);
4512 /* LOCK prefix is allowed only with some instructions */
4513 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
4514 rc = emulate_ud(ctxt);
4518 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
4519 rc = emulate_ud(ctxt);
4523 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
4524 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
4525 rc = emulate_ud(ctxt);
4529 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
4530 rc = emulate_nm(ctxt);
4534 if (ctxt->d & Mmx) {
4535 rc = flush_pending_x87_faults(ctxt);
4536 if (rc != X86EMUL_CONTINUE)
4539 * Now that we know the fpu is exception safe, we can fetch
4542 fetch_possible_mmx_operand(ctxt, &ctxt->src);
4543 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
4544 if (!(ctxt->d & Mov))
4545 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
4548 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4549 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4550 X86_ICPT_PRE_EXCEPT);
4551 if (rc != X86EMUL_CONTINUE)
4555 /* Privileged instruction can be executed only in CPL=0 */
4556 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
4557 rc = emulate_gp(ctxt, 0);
4561 /* Instruction can only be executed in protected mode */
4562 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
4563 rc = emulate_ud(ctxt);
4567 /* Do instruction specific permission checks */
4568 if (ctxt->check_perm) {
4569 rc = ctxt->check_perm(ctxt);
4570 if (rc != X86EMUL_CONTINUE)
4574 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4575 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4576 X86_ICPT_POST_EXCEPT);
4577 if (rc != X86EMUL_CONTINUE)
4581 if (ctxt->rep_prefix && (ctxt->d & String)) {
4582 /* All REP prefixes have the same first termination condition */
4583 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
4584 ctxt->eip = ctxt->_eip;
4589 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
4590 rc = segmented_read(ctxt, ctxt->src.addr.mem,
4591 ctxt->src.valptr, ctxt->src.bytes);
4592 if (rc != X86EMUL_CONTINUE)
4594 ctxt->src.orig_val64 = ctxt->src.val64;
4597 if (ctxt->src2.type == OP_MEM) {
4598 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
4599 &ctxt->src2.val, ctxt->src2.bytes);
4600 if (rc != X86EMUL_CONTINUE)
4604 if ((ctxt->d & DstMask) == ImplicitOps)
4608 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
4609 /* optimisation - avoid slow emulated read if Mov */
4610 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
4611 &ctxt->dst.val, ctxt->dst.bytes);
4612 if (rc != X86EMUL_CONTINUE)
4615 ctxt->dst.orig_val = ctxt->dst.val;
4619 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4620 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4621 X86_ICPT_POST_MEMACCESS);
4622 if (rc != X86EMUL_CONTINUE)
4626 if (ctxt->execute) {
4627 if (ctxt->d & Fastop) {
4628 void (*fop)(struct fastop *) = (void *)ctxt->execute;
4629 rc = fastop(ctxt, fop);
4630 if (rc != X86EMUL_CONTINUE)
4634 rc = ctxt->execute(ctxt);
4635 if (rc != X86EMUL_CONTINUE)
4640 if (ctxt->opcode_len == 2)
4642 else if (ctxt->opcode_len == 3)
4643 goto threebyte_insn;
4646 case 0x63: /* movsxd */
4647 if (ctxt->mode != X86EMUL_MODE_PROT64)
4648 goto cannot_emulate;
4649 ctxt->dst.val = (s32) ctxt->src.val;
4651 case 0x70 ... 0x7f: /* jcc (short) */
4652 if (test_cc(ctxt->b, ctxt->eflags))
4653 jmp_rel(ctxt, ctxt->src.val);
4655 case 0x8d: /* lea r16/r32, m */
4656 ctxt->dst.val = ctxt->src.addr.mem.ea;
4658 case 0x90 ... 0x97: /* nop / xchg reg, rax */
4659 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
4663 case 0x98: /* cbw/cwde/cdqe */
4664 switch (ctxt->op_bytes) {
4665 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
4666 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
4667 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
4670 case 0xcc: /* int3 */
4671 rc = emulate_int(ctxt, 3);
4673 case 0xcd: /* int n */
4674 rc = emulate_int(ctxt, ctxt->src.val);
4676 case 0xce: /* into */
4677 if (ctxt->eflags & EFLG_OF)
4678 rc = emulate_int(ctxt, 4);
4680 case 0xe9: /* jmp rel */
4681 case 0xeb: /* jmp rel short */
4682 jmp_rel(ctxt, ctxt->src.val);
4683 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4685 case 0xf4: /* hlt */
4686 ctxt->ops->halt(ctxt);
4688 case 0xf5: /* cmc */
4689 /* complement carry flag from eflags reg */
4690 ctxt->eflags ^= EFLG_CF;
4692 case 0xf8: /* clc */
4693 ctxt->eflags &= ~EFLG_CF;
4695 case 0xf9: /* stc */
4696 ctxt->eflags |= EFLG_CF;
4698 case 0xfc: /* cld */
4699 ctxt->eflags &= ~EFLG_DF;
4701 case 0xfd: /* std */
4702 ctxt->eflags |= EFLG_DF;
4705 goto cannot_emulate;
4708 if (rc != X86EMUL_CONTINUE)
4712 if (!(ctxt->d & NoWrite)) {
4713 rc = writeback(ctxt, &ctxt->dst);
4714 if (rc != X86EMUL_CONTINUE)
4717 if (ctxt->d & SrcWrite) {
4718 BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);
4719 rc = writeback(ctxt, &ctxt->src);
4720 if (rc != X86EMUL_CONTINUE)
4725 * restore dst type in case the decoding will be reused
4726 * (happens for string instruction )
4728 ctxt->dst.type = saved_dst_type;
4730 if ((ctxt->d & SrcMask) == SrcSI)
4731 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
4733 if ((ctxt->d & DstMask) == DstDI)
4734 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
4736 if (ctxt->rep_prefix && (ctxt->d & String)) {
4738 struct read_cache *r = &ctxt->io_read;
4739 if ((ctxt->d & SrcMask) == SrcSI)
4740 count = ctxt->src.count;
4742 count = ctxt->dst.count;
4743 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX),
4746 if (!string_insn_completed(ctxt)) {
4748 * Re-enter guest when pio read ahead buffer is empty
4749 * or, if it is not used, after each 1024 iteration.
4751 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
4752 (r->end == 0 || r->end != r->pos)) {
4754 * Reset read cache. Usually happens before
4755 * decode, but since instruction is restarted
4756 * we have to do it here.
4758 ctxt->mem_read.end = 0;
4759 writeback_registers(ctxt);
4760 return EMULATION_RESTART;
4762 goto done; /* skip rip writeback */
4766 ctxt->eip = ctxt->_eip;
4769 if (rc == X86EMUL_PROPAGATE_FAULT)
4770 ctxt->have_exception = true;
4771 if (rc == X86EMUL_INTERCEPTED)
4772 return EMULATION_INTERCEPTED;
4774 if (rc == X86EMUL_CONTINUE)
4775 writeback_registers(ctxt);
4777 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4781 case 0x09: /* wbinvd */
4782 (ctxt->ops->wbinvd)(ctxt);
4784 case 0x08: /* invd */
4785 case 0x0d: /* GrpP (prefetch) */
4786 case 0x18: /* Grp16 (prefetch/nop) */
4787 case 0x1f: /* nop */
4789 case 0x20: /* mov cr, reg */
4790 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
4792 case 0x21: /* mov from dr to reg */
4793 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
4795 case 0x40 ... 0x4f: /* cmov */
4796 ctxt->dst.val = ctxt->dst.orig_val = ctxt->src.val;
4797 if (!test_cc(ctxt->b, ctxt->eflags))
4798 ctxt->dst.type = OP_NONE; /* no writeback */
4800 case 0x80 ... 0x8f: /* jnz rel, etc*/
4801 if (test_cc(ctxt->b, ctxt->eflags))
4802 jmp_rel(ctxt, ctxt->src.val);
4804 case 0x90 ... 0x9f: /* setcc r/m8 */
4805 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
4807 case 0xae: /* clflush */
4809 case 0xb6 ... 0xb7: /* movzx */
4810 ctxt->dst.bytes = ctxt->op_bytes;
4811 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
4812 : (u16) ctxt->src.val;
4814 case 0xbe ... 0xbf: /* movsx */
4815 ctxt->dst.bytes = ctxt->op_bytes;
4816 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
4817 (s16) ctxt->src.val;
4819 case 0xc3: /* movnti */
4820 ctxt->dst.bytes = ctxt->op_bytes;
4821 ctxt->dst.val = (ctxt->op_bytes == 4) ? (u32) ctxt->src.val :
4822 (u64) ctxt->src.val;
4825 goto cannot_emulate;
4830 if (rc != X86EMUL_CONTINUE)
4836 return EMULATION_FAILED;
4839 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
4841 invalidate_registers(ctxt);
4844 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
4846 writeback_registers(ctxt);