1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
27 #include <linux/stringify.h>
36 #define OpImplicit 1ull /* No generic decode */
37 #define OpReg 2ull /* Register */
38 #define OpMem 3ull /* Memory */
39 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
40 #define OpDI 5ull /* ES:DI/EDI/RDI */
41 #define OpMem64 6ull /* Memory, 64-bit */
42 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
43 #define OpDX 8ull /* DX register */
44 #define OpCL 9ull /* CL register (for shifts) */
45 #define OpImmByte 10ull /* 8-bit sign extended immediate */
46 #define OpOne 11ull /* Implied 1 */
47 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
48 #define OpMem16 13ull /* Memory operand (16-bit). */
49 #define OpMem32 14ull /* Memory operand (32-bit). */
50 #define OpImmU 15ull /* Immediate operand, zero extended */
51 #define OpSI 16ull /* SI/ESI/RSI */
52 #define OpImmFAddr 17ull /* Immediate far address */
53 #define OpMemFAddr 18ull /* Far address in memory */
54 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
55 #define OpES 20ull /* ES */
56 #define OpCS 21ull /* CS */
57 #define OpSS 22ull /* SS */
58 #define OpDS 23ull /* DS */
59 #define OpFS 24ull /* FS */
60 #define OpGS 25ull /* GS */
61 #define OpMem8 26ull /* 8-bit zero extended memory operand */
62 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
63 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
64 #define OpAccLo 29ull /* Low part of extended acc (AX/AX/EAX/RAX) */
65 #define OpAccHi 30ull /* High part of extended acc (-/DX/EDX/RDX) */
67 #define OpBits 5 /* Width of operand field */
68 #define OpMask ((1ull << OpBits) - 1)
71 * Opcode effective-address decode tables.
72 * Note that we only emulate instructions that have at least one memory
73 * operand (excluding implicit stack references). We assume that stack
74 * references and instruction fetches will never occur in special memory
75 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
79 /* Operand sizes: 8-bit operands or specified/overridden size. */
80 #define ByteOp (1<<0) /* 8-bit operands. */
81 /* Destination operand type. */
83 #define ImplicitOps (OpImplicit << DstShift)
84 #define DstReg (OpReg << DstShift)
85 #define DstMem (OpMem << DstShift)
86 #define DstAcc (OpAcc << DstShift)
87 #define DstDI (OpDI << DstShift)
88 #define DstMem64 (OpMem64 << DstShift)
89 #define DstImmUByte (OpImmUByte << DstShift)
90 #define DstDX (OpDX << DstShift)
91 #define DstAccLo (OpAccLo << DstShift)
92 #define DstMask (OpMask << DstShift)
93 /* Source operand type. */
95 #define SrcNone (OpNone << SrcShift)
96 #define SrcReg (OpReg << SrcShift)
97 #define SrcMem (OpMem << SrcShift)
98 #define SrcMem16 (OpMem16 << SrcShift)
99 #define SrcMem32 (OpMem32 << SrcShift)
100 #define SrcImm (OpImm << SrcShift)
101 #define SrcImmByte (OpImmByte << SrcShift)
102 #define SrcOne (OpOne << SrcShift)
103 #define SrcImmUByte (OpImmUByte << SrcShift)
104 #define SrcImmU (OpImmU << SrcShift)
105 #define SrcSI (OpSI << SrcShift)
106 #define SrcXLat (OpXLat << SrcShift)
107 #define SrcImmFAddr (OpImmFAddr << SrcShift)
108 #define SrcMemFAddr (OpMemFAddr << SrcShift)
109 #define SrcAcc (OpAcc << SrcShift)
110 #define SrcImmU16 (OpImmU16 << SrcShift)
111 #define SrcImm64 (OpImm64 << SrcShift)
112 #define SrcDX (OpDX << SrcShift)
113 #define SrcMem8 (OpMem8 << SrcShift)
114 #define SrcAccHi (OpAccHi << SrcShift)
115 #define SrcMask (OpMask << SrcShift)
116 #define BitOp (1<<11)
117 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
118 #define String (1<<13) /* String instruction (rep capable) */
119 #define Stack (1<<14) /* Stack instruction (push/pop) */
120 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
121 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
122 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
123 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
124 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
125 #define Escape (5<<15) /* Escape to coprocessor instruction */
126 #define Sse (1<<18) /* SSE Vector instruction */
127 /* Generic ModRM decode. */
128 #define ModRM (1<<19)
129 /* Destination is only written; never read. */
132 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
133 #define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
134 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
135 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
136 #define Undefined (1<<25) /* No Such Instruction */
137 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
138 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
140 #define PageTable (1 << 29) /* instruction used to write page table */
141 #define NotImpl (1 << 30) /* instruction is not implemented */
142 /* Source 2 operand type */
143 #define Src2Shift (31)
144 #define Src2None (OpNone << Src2Shift)
145 #define Src2Mem (OpMem << Src2Shift)
146 #define Src2CL (OpCL << Src2Shift)
147 #define Src2ImmByte (OpImmByte << Src2Shift)
148 #define Src2One (OpOne << Src2Shift)
149 #define Src2Imm (OpImm << Src2Shift)
150 #define Src2ES (OpES << Src2Shift)
151 #define Src2CS (OpCS << Src2Shift)
152 #define Src2SS (OpSS << Src2Shift)
153 #define Src2DS (OpDS << Src2Shift)
154 #define Src2FS (OpFS << Src2Shift)
155 #define Src2GS (OpGS << Src2Shift)
156 #define Src2Mask (OpMask << Src2Shift)
157 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
158 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
159 #define Unaligned ((u64)1 << 42) /* Explicitly unaligned (e.g. MOVDQU) */
160 #define Avx ((u64)1 << 43) /* Advanced Vector Extensions */
161 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
162 #define NoWrite ((u64)1 << 45) /* No writeback */
163 #define SrcWrite ((u64)1 << 46) /* Write back src operand */
164 #define NoMod ((u64)1 << 47) /* Mod field is ignored */
166 #define DstXacc (DstAccLo | SrcAccHi | SrcWrite)
168 #define X2(x...) x, x
169 #define X3(x...) X2(x), x
170 #define X4(x...) X2(x), X2(x)
171 #define X5(x...) X4(x), x
172 #define X6(x...) X4(x), X2(x)
173 #define X7(x...) X4(x), X3(x)
174 #define X8(x...) X4(x), X4(x)
175 #define X16(x...) X8(x), X8(x)
177 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
178 #define FASTOP_SIZE 8
181 * fastop functions have a special calling convention:
186 * flags: rflags (in/out)
187 * ex: rsi (in:fastop pointer, out:zero if exception)
189 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
190 * different operand sizes can be reached by calculation, rather than a jump
191 * table (which would be bigger than the code).
193 * fastop functions are declared as taking a never-defined fastop parameter,
194 * so they can't be called from C directly.
203 int (*execute)(struct x86_emulate_ctxt *ctxt);
204 const struct opcode *group;
205 const struct group_dual *gdual;
206 const struct gprefix *gprefix;
207 const struct escape *esc;
208 void (*fastop)(struct fastop *fake);
210 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
214 struct opcode mod012[8];
215 struct opcode mod3[8];
219 struct opcode pfx_no;
220 struct opcode pfx_66;
221 struct opcode pfx_f2;
222 struct opcode pfx_f3;
227 struct opcode high[64];
230 /* EFLAGS bit definitions. */
231 #define EFLG_ID (1<<21)
232 #define EFLG_VIP (1<<20)
233 #define EFLG_VIF (1<<19)
234 #define EFLG_AC (1<<18)
235 #define EFLG_VM (1<<17)
236 #define EFLG_RF (1<<16)
237 #define EFLG_IOPL (3<<12)
238 #define EFLG_NT (1<<14)
239 #define EFLG_OF (1<<11)
240 #define EFLG_DF (1<<10)
241 #define EFLG_IF (1<<9)
242 #define EFLG_TF (1<<8)
243 #define EFLG_SF (1<<7)
244 #define EFLG_ZF (1<<6)
245 #define EFLG_AF (1<<4)
246 #define EFLG_PF (1<<2)
247 #define EFLG_CF (1<<0)
249 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
250 #define EFLG_RESERVED_ONE_MASK 2
252 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
254 if (!(ctxt->regs_valid & (1 << nr))) {
255 ctxt->regs_valid |= 1 << nr;
256 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
258 return ctxt->_regs[nr];
261 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
263 ctxt->regs_valid |= 1 << nr;
264 ctxt->regs_dirty |= 1 << nr;
265 return &ctxt->_regs[nr];
268 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
271 return reg_write(ctxt, nr);
274 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
278 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
279 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
282 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
284 ctxt->regs_dirty = 0;
285 ctxt->regs_valid = 0;
289 * These EFLAGS bits are restored from saved value during emulation, and
290 * any changes are written back to the saved value after emulation.
292 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
300 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));
302 #define FOP_ALIGN ".align " __stringify(FASTOP_SIZE) " \n\t"
303 #define FOP_RET "ret \n\t"
305 #define FOP_START(op) \
306 extern void em_##op(struct fastop *fake); \
307 asm(".pushsection .text, \"ax\" \n\t" \
308 ".global em_" #op " \n\t" \
315 #define FOPNOP() FOP_ALIGN FOP_RET
317 #define FOP1E(op, dst) \
318 FOP_ALIGN "10: " #op " %" #dst " \n\t" FOP_RET
320 #define FOP1EEX(op, dst) \
321 FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
323 #define FASTOP1(op) \
328 ON64(FOP1E(op##q, rax)) \
331 /* 1-operand, using src2 (for MUL/DIV r/m) */
332 #define FASTOP1SRC2(op, name) \
337 ON64(FOP1E(op, rcx)) \
340 /* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
341 #define FASTOP1SRC2EX(op, name) \
346 ON64(FOP1EEX(op, rcx)) \
349 #define FOP2E(op, dst, src) \
350 FOP_ALIGN #op " %" #src ", %" #dst " \n\t" FOP_RET
352 #define FASTOP2(op) \
354 FOP2E(op##b, al, dl) \
355 FOP2E(op##w, ax, dx) \
356 FOP2E(op##l, eax, edx) \
357 ON64(FOP2E(op##q, rax, rdx)) \
360 /* 2 operand, word only */
361 #define FASTOP2W(op) \
364 FOP2E(op##w, ax, dx) \
365 FOP2E(op##l, eax, edx) \
366 ON64(FOP2E(op##q, rax, rdx)) \
369 /* 2 operand, src is CL */
370 #define FASTOP2CL(op) \
372 FOP2E(op##b, al, cl) \
373 FOP2E(op##w, ax, cl) \
374 FOP2E(op##l, eax, cl) \
375 ON64(FOP2E(op##q, rax, cl)) \
378 #define FOP3E(op, dst, src, src2) \
379 FOP_ALIGN #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
381 /* 3-operand, word-only, src2=cl */
382 #define FASTOP3WCL(op) \
385 FOP3E(op##w, ax, dx, cl) \
386 FOP3E(op##l, eax, edx, cl) \
387 ON64(FOP3E(op##q, rax, rdx, cl)) \
390 /* Special case for SETcc - 1 instruction per cc */
391 #define FOP_SETCC(op) ".align 4; " #op " %al; ret \n\t"
393 asm(".global kvm_fastop_exception \n"
394 "kvm_fastop_exception: xor %esi, %esi; ret");
415 FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET
418 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
419 enum x86_intercept intercept,
420 enum x86_intercept_stage stage)
422 struct x86_instruction_info info = {
423 .intercept = intercept,
424 .rep_prefix = ctxt->rep_prefix,
425 .modrm_mod = ctxt->modrm_mod,
426 .modrm_reg = ctxt->modrm_reg,
427 .modrm_rm = ctxt->modrm_rm,
428 .src_val = ctxt->src.val64,
429 .src_bytes = ctxt->src.bytes,
430 .dst_bytes = ctxt->dst.bytes,
431 .ad_bytes = ctxt->ad_bytes,
432 .next_rip = ctxt->eip,
435 return ctxt->ops->intercept(ctxt, &info, stage);
438 static void assign_masked(ulong *dest, ulong src, ulong mask)
440 *dest = (*dest & ~mask) | (src & mask);
443 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
445 return (1UL << (ctxt->ad_bytes << 3)) - 1;
448 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
451 struct desc_struct ss;
453 if (ctxt->mode == X86EMUL_MODE_PROT64)
455 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
456 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
459 static int stack_size(struct x86_emulate_ctxt *ctxt)
461 return (__fls(stack_mask(ctxt)) + 1) >> 3;
464 /* Access/update address held in a register, based on addressing mode. */
465 static inline unsigned long
466 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
468 if (ctxt->ad_bytes == sizeof(unsigned long))
471 return reg & ad_mask(ctxt);
474 static inline unsigned long
475 register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
477 return address_mask(ctxt, reg);
480 static void masked_increment(ulong *reg, ulong mask, int inc)
482 assign_masked(reg, *reg + inc, mask);
486 register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
490 if (ctxt->ad_bytes == sizeof(unsigned long))
493 mask = ad_mask(ctxt);
494 masked_increment(reg, mask, inc);
497 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
499 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
502 static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
504 register_address_increment(ctxt, &ctxt->_eip, rel);
507 static u32 desc_limit_scaled(struct desc_struct *desc)
509 u32 limit = get_desc_limit(desc);
511 return desc->g ? (limit << 12) | 0xfff : limit;
514 static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
516 ctxt->has_seg_override = true;
517 ctxt->seg_override = seg;
520 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
522 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
525 return ctxt->ops->get_cached_segment_base(ctxt, seg);
528 static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
530 if (!ctxt->has_seg_override)
533 return ctxt->seg_override;
536 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
537 u32 error, bool valid)
539 ctxt->exception.vector = vec;
540 ctxt->exception.error_code = error;
541 ctxt->exception.error_code_valid = valid;
542 return X86EMUL_PROPAGATE_FAULT;
545 static int emulate_db(struct x86_emulate_ctxt *ctxt)
547 return emulate_exception(ctxt, DB_VECTOR, 0, false);
550 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
552 return emulate_exception(ctxt, GP_VECTOR, err, true);
555 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
557 return emulate_exception(ctxt, SS_VECTOR, err, true);
560 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
562 return emulate_exception(ctxt, UD_VECTOR, 0, false);
565 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
567 return emulate_exception(ctxt, TS_VECTOR, err, true);
570 static int emulate_de(struct x86_emulate_ctxt *ctxt)
572 return emulate_exception(ctxt, DE_VECTOR, 0, false);
575 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
577 return emulate_exception(ctxt, NM_VECTOR, 0, false);
580 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
583 struct desc_struct desc;
585 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
589 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
594 struct desc_struct desc;
596 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
597 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
601 * x86 defines three classes of vector instructions: explicitly
602 * aligned, explicitly unaligned, and the rest, which change behaviour
603 * depending on whether they're AVX encoded or not.
605 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
606 * subject to the same check.
608 static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
610 if (likely(size < 16))
613 if (ctxt->d & Aligned)
615 else if (ctxt->d & Unaligned)
617 else if (ctxt->d & Avx)
623 static int __linearize(struct x86_emulate_ctxt *ctxt,
624 struct segmented_address addr,
625 unsigned size, bool write, bool fetch,
628 struct desc_struct desc;
635 la = seg_base(ctxt, addr.seg) + addr.ea;
636 switch (ctxt->mode) {
637 case X86EMUL_MODE_PROT64:
638 if (((signed long)la << 16) >> 16 != la)
639 return emulate_gp(ctxt, 0);
642 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
646 /* code segment in protected mode or read-only data segment */
647 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
648 || !(desc.type & 2)) && write)
650 /* unreadable code segment */
651 if (!fetch && (desc.type & 8) && !(desc.type & 2))
653 lim = desc_limit_scaled(&desc);
654 if ((desc.type & 8) || !(desc.type & 4)) {
655 /* expand-up segment */
656 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
659 /* expand-down segment */
660 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
662 lim = desc.d ? 0xffffffff : 0xffff;
663 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
666 cpl = ctxt->ops->cpl(ctxt);
667 if (!(desc.type & 8)) {
671 } else if ((desc.type & 8) && !(desc.type & 4)) {
672 /* nonconforming code segment */
675 } else if ((desc.type & 8) && (desc.type & 4)) {
676 /* conforming code segment */
682 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
684 if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
685 return emulate_gp(ctxt, 0);
687 return X86EMUL_CONTINUE;
689 if (addr.seg == VCPU_SREG_SS)
690 return emulate_ss(ctxt, sel);
692 return emulate_gp(ctxt, sel);
695 static int linearize(struct x86_emulate_ctxt *ctxt,
696 struct segmented_address addr,
697 unsigned size, bool write,
700 return __linearize(ctxt, addr, size, write, false, linear);
704 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
705 struct segmented_address addr,
712 rc = linearize(ctxt, addr, size, false, &linear);
713 if (rc != X86EMUL_CONTINUE)
715 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
719 * Fetch the next byte of the instruction being emulated which is pointed to
720 * by ctxt->_eip, then increment ctxt->_eip.
722 * Also prefetch the remaining bytes of the instruction without crossing page
723 * boundary if they are not in fetch_cache yet.
725 static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
727 struct fetch_cache *fc = &ctxt->fetch;
731 if (ctxt->_eip == fc->end) {
732 unsigned long linear;
733 struct segmented_address addr = { .seg = VCPU_SREG_CS,
735 cur_size = fc->end - fc->start;
736 size = min(15UL - cur_size,
737 PAGE_SIZE - offset_in_page(ctxt->_eip));
738 rc = __linearize(ctxt, addr, size, false, true, &linear);
739 if (unlikely(rc != X86EMUL_CONTINUE))
741 rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
742 size, &ctxt->exception);
743 if (unlikely(rc != X86EMUL_CONTINUE))
747 *dest = fc->data[ctxt->_eip - fc->start];
749 return X86EMUL_CONTINUE;
752 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
753 void *dest, unsigned size)
757 /* x86 instructions are limited to 15 bytes. */
758 if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
759 return X86EMUL_UNHANDLEABLE;
761 rc = do_insn_fetch_byte(ctxt, dest++);
762 if (rc != X86EMUL_CONTINUE)
765 return X86EMUL_CONTINUE;
768 /* Fetch next part of the instruction being emulated. */
769 #define insn_fetch(_type, _ctxt) \
770 ({ unsigned long _x; \
771 rc = do_insn_fetch(_ctxt, &_x, sizeof(_type)); \
772 if (rc != X86EMUL_CONTINUE) \
777 #define insn_fetch_arr(_arr, _size, _ctxt) \
778 ({ rc = do_insn_fetch(_ctxt, _arr, (_size)); \
779 if (rc != X86EMUL_CONTINUE) \
784 * Given the 'reg' portion of a ModRM byte, and a register block, return a
785 * pointer into the block that addresses the relevant register.
786 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
788 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
792 int highbyte_regs = (ctxt->rex_prefix == 0) && byteop;
794 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
795 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
797 p = reg_rmw(ctxt, modrm_reg);
801 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
802 struct segmented_address addr,
803 u16 *size, unsigned long *address, int op_bytes)
810 rc = segmented_read_std(ctxt, addr, size, 2);
811 if (rc != X86EMUL_CONTINUE)
814 rc = segmented_read_std(ctxt, addr, address, op_bytes);
828 FASTOP1SRC2(mul, mul_ex);
829 FASTOP1SRC2(imul, imul_ex);
830 FASTOP1SRC2EX(div, div_ex);
831 FASTOP1SRC2EX(idiv, idiv_ex);
860 static u8 test_cc(unsigned int condition, unsigned long flags)
863 void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
865 flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
866 asm("push %[flags]; popf; call *%[fastop]"
867 : "=a"(rc) : [fastop]"r"(fop), [flags]"r"(flags));
871 static void fetch_register_operand(struct operand *op)
875 op->val = *(u8 *)op->addr.reg;
878 op->val = *(u16 *)op->addr.reg;
881 op->val = *(u32 *)op->addr.reg;
884 op->val = *(u64 *)op->addr.reg;
889 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
891 ctxt->ops->get_fpu(ctxt);
893 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
894 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
895 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
896 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
897 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
898 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
899 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
900 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
902 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
903 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
904 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
905 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
906 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
907 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
908 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
909 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
913 ctxt->ops->put_fpu(ctxt);
916 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
919 ctxt->ops->get_fpu(ctxt);
921 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
922 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
923 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
924 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
925 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
926 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
927 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
928 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
930 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
931 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
932 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
933 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
934 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
935 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
936 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
937 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
941 ctxt->ops->put_fpu(ctxt);
944 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
946 ctxt->ops->get_fpu(ctxt);
948 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
949 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
950 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
951 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
952 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
953 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
954 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
955 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
958 ctxt->ops->put_fpu(ctxt);
961 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
963 ctxt->ops->get_fpu(ctxt);
965 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
966 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
967 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
968 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
969 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
970 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
971 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
972 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
975 ctxt->ops->put_fpu(ctxt);
978 static int em_fninit(struct x86_emulate_ctxt *ctxt)
980 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
981 return emulate_nm(ctxt);
983 ctxt->ops->get_fpu(ctxt);
984 asm volatile("fninit");
985 ctxt->ops->put_fpu(ctxt);
986 return X86EMUL_CONTINUE;
989 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
993 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
994 return emulate_nm(ctxt);
996 ctxt->ops->get_fpu(ctxt);
997 asm volatile("fnstcw %0": "+m"(fcw));
998 ctxt->ops->put_fpu(ctxt);
1000 /* force 2 byte destination */
1001 ctxt->dst.bytes = 2;
1002 ctxt->dst.val = fcw;
1004 return X86EMUL_CONTINUE;
1007 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1011 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1012 return emulate_nm(ctxt);
1014 ctxt->ops->get_fpu(ctxt);
1015 asm volatile("fnstsw %0": "+m"(fsw));
1016 ctxt->ops->put_fpu(ctxt);
1018 /* force 2 byte destination */
1019 ctxt->dst.bytes = 2;
1020 ctxt->dst.val = fsw;
1022 return X86EMUL_CONTINUE;
1025 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1028 unsigned reg = ctxt->modrm_reg;
1030 if (!(ctxt->d & ModRM))
1031 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1033 if (ctxt->d & Sse) {
1037 read_sse_reg(ctxt, &op->vec_val, reg);
1040 if (ctxt->d & Mmx) {
1049 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1050 op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);
1052 fetch_register_operand(op);
1053 op->orig_val = op->val;
1056 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1058 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1059 ctxt->modrm_seg = VCPU_SREG_SS;
1062 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1066 int index_reg = 0, base_reg = 0, scale;
1067 int rc = X86EMUL_CONTINUE;
1070 if (ctxt->rex_prefix) {
1071 ctxt->modrm_reg = (ctxt->rex_prefix & 4) << 1; /* REX.R */
1072 index_reg = (ctxt->rex_prefix & 2) << 2; /* REX.X */
1073 ctxt->modrm_rm = base_reg = (ctxt->rex_prefix & 1) << 3; /* REG.B */
1076 ctxt->modrm_mod |= (ctxt->modrm & 0xc0) >> 6;
1077 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1078 ctxt->modrm_rm |= (ctxt->modrm & 0x07);
1079 ctxt->modrm_seg = VCPU_SREG_DS;
1081 if (ctxt->modrm_mod == 3 || (ctxt->d & NoMod)) {
1083 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1084 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
1086 if (ctxt->d & Sse) {
1089 op->addr.xmm = ctxt->modrm_rm;
1090 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1093 if (ctxt->d & Mmx) {
1096 op->addr.xmm = ctxt->modrm_rm & 7;
1099 fetch_register_operand(op);
1105 if (ctxt->ad_bytes == 2) {
1106 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1107 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1108 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1109 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1111 /* 16-bit ModR/M decode. */
1112 switch (ctxt->modrm_mod) {
1114 if (ctxt->modrm_rm == 6)
1115 modrm_ea += insn_fetch(u16, ctxt);
1118 modrm_ea += insn_fetch(s8, ctxt);
1121 modrm_ea += insn_fetch(u16, ctxt);
1124 switch (ctxt->modrm_rm) {
1126 modrm_ea += bx + si;
1129 modrm_ea += bx + di;
1132 modrm_ea += bp + si;
1135 modrm_ea += bp + di;
1144 if (ctxt->modrm_mod != 0)
1151 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1152 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1153 ctxt->modrm_seg = VCPU_SREG_SS;
1154 modrm_ea = (u16)modrm_ea;
1156 /* 32/64-bit ModR/M decode. */
1157 if ((ctxt->modrm_rm & 7) == 4) {
1158 sib = insn_fetch(u8, ctxt);
1159 index_reg |= (sib >> 3) & 7;
1160 base_reg |= sib & 7;
1163 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1164 modrm_ea += insn_fetch(s32, ctxt);
1166 modrm_ea += reg_read(ctxt, base_reg);
1167 adjust_modrm_seg(ctxt, base_reg);
1170 modrm_ea += reg_read(ctxt, index_reg) << scale;
1171 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1172 if (ctxt->mode == X86EMUL_MODE_PROT64)
1173 ctxt->rip_relative = 1;
1175 base_reg = ctxt->modrm_rm;
1176 modrm_ea += reg_read(ctxt, base_reg);
1177 adjust_modrm_seg(ctxt, base_reg);
1179 switch (ctxt->modrm_mod) {
1181 if (ctxt->modrm_rm == 5)
1182 modrm_ea += insn_fetch(s32, ctxt);
1185 modrm_ea += insn_fetch(s8, ctxt);
1188 modrm_ea += insn_fetch(s32, ctxt);
1192 op->addr.mem.ea = modrm_ea;
1197 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1200 int rc = X86EMUL_CONTINUE;
1203 switch (ctxt->ad_bytes) {
1205 op->addr.mem.ea = insn_fetch(u16, ctxt);
1208 op->addr.mem.ea = insn_fetch(u32, ctxt);
1211 op->addr.mem.ea = insn_fetch(u64, ctxt);
1218 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1222 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1223 mask = ~(ctxt->dst.bytes * 8 - 1);
1225 if (ctxt->src.bytes == 2)
1226 sv = (s16)ctxt->src.val & (s16)mask;
1227 else if (ctxt->src.bytes == 4)
1228 sv = (s32)ctxt->src.val & (s32)mask;
1230 ctxt->dst.addr.mem.ea += (sv >> 3);
1233 /* only subword offset */
1234 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1237 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1238 unsigned long addr, void *dest, unsigned size)
1241 struct read_cache *mc = &ctxt->mem_read;
1243 if (mc->pos < mc->end)
1246 WARN_ON((mc->end + size) >= sizeof(mc->data));
1248 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1250 if (rc != X86EMUL_CONTINUE)
1256 memcpy(dest, mc->data + mc->pos, size);
1258 return X86EMUL_CONTINUE;
1261 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1262 struct segmented_address addr,
1269 rc = linearize(ctxt, addr, size, false, &linear);
1270 if (rc != X86EMUL_CONTINUE)
1272 return read_emulated(ctxt, linear, data, size);
1275 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1276 struct segmented_address addr,
1283 rc = linearize(ctxt, addr, size, true, &linear);
1284 if (rc != X86EMUL_CONTINUE)
1286 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1290 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1291 struct segmented_address addr,
1292 const void *orig_data, const void *data,
1298 rc = linearize(ctxt, addr, size, true, &linear);
1299 if (rc != X86EMUL_CONTINUE)
1301 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1302 size, &ctxt->exception);
1305 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1306 unsigned int size, unsigned short port,
1309 struct read_cache *rc = &ctxt->io_read;
1311 if (rc->pos == rc->end) { /* refill pio read ahead */
1312 unsigned int in_page, n;
1313 unsigned int count = ctxt->rep_prefix ?
1314 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1315 in_page = (ctxt->eflags & EFLG_DF) ?
1316 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1317 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1318 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1322 rc->pos = rc->end = 0;
1323 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1328 if (ctxt->rep_prefix && (ctxt->d & String) &&
1329 !(ctxt->eflags & EFLG_DF)) {
1330 ctxt->dst.data = rc->data + rc->pos;
1331 ctxt->dst.type = OP_MEM_STR;
1332 ctxt->dst.count = (rc->end - rc->pos) / size;
1335 memcpy(dest, rc->data + rc->pos, size);
1341 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1342 u16 index, struct desc_struct *desc)
1347 ctxt->ops->get_idt(ctxt, &dt);
1349 if (dt.size < index * 8 + 7)
1350 return emulate_gp(ctxt, index << 3 | 0x2);
1352 addr = dt.address + index * 8;
1353 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1357 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1358 u16 selector, struct desc_ptr *dt)
1360 const struct x86_emulate_ops *ops = ctxt->ops;
1362 if (selector & 1 << 2) {
1363 struct desc_struct desc;
1366 memset (dt, 0, sizeof *dt);
1367 if (!ops->get_segment(ctxt, &sel, &desc, NULL, VCPU_SREG_LDTR))
1370 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1371 dt->address = get_desc_base(&desc);
1373 ops->get_gdt(ctxt, dt);
1376 /* allowed just for 8 bytes segments */
1377 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1378 u16 selector, struct desc_struct *desc,
1382 u16 index = selector >> 3;
1385 get_descriptor_table_ptr(ctxt, selector, &dt);
1387 if (dt.size < index * 8 + 7)
1388 return emulate_gp(ctxt, selector & 0xfffc);
1390 *desc_addr_p = addr = dt.address + index * 8;
1391 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1395 /* allowed just for 8 bytes segments */
1396 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1397 u16 selector, struct desc_struct *desc)
1400 u16 index = selector >> 3;
1403 get_descriptor_table_ptr(ctxt, selector, &dt);
1405 if (dt.size < index * 8 + 7)
1406 return emulate_gp(ctxt, selector & 0xfffc);
1408 addr = dt.address + index * 8;
1409 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1413 /* Does not support long mode */
1414 static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1415 u16 selector, int seg, u8 cpl, bool in_task_switch)
1417 struct desc_struct seg_desc, old_desc;
1419 unsigned err_vec = GP_VECTOR;
1421 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1427 memset(&seg_desc, 0, sizeof seg_desc);
1429 if (ctxt->mode == X86EMUL_MODE_REAL) {
1430 /* set real mode segment descriptor (keep limit etc. for
1432 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1433 set_desc_base(&seg_desc, selector << 4);
1435 } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
1436 /* VM86 needs a clean new segment descriptor */
1437 set_desc_base(&seg_desc, selector << 4);
1438 set_desc_limit(&seg_desc, 0xffff);
1448 /* NULL selector is not valid for TR, CS and SS (except for long mode) */
1449 if ((seg == VCPU_SREG_CS
1450 || (seg == VCPU_SREG_SS
1451 && (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl))
1452 || seg == VCPU_SREG_TR)
1456 /* TR should be in GDT only */
1457 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1460 if (null_selector) /* for NULL selector skip all following checks */
1463 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1464 if (ret != X86EMUL_CONTINUE)
1467 err_code = selector & 0xfffc;
1468 err_vec = GP_VECTOR;
1470 /* can't load system descriptor into segment selector */
1471 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1475 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1484 * segment is not a writable data segment or segment
1485 * selector's RPL != CPL or segment selector's RPL != CPL
1487 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1491 if (in_task_switch && rpl != dpl)
1494 if (!(seg_desc.type & 8))
1497 if (seg_desc.type & 4) {
1503 if (rpl > cpl || dpl != cpl)
1506 /* CS(RPL) <- CPL */
1507 selector = (selector & 0xfffc) | cpl;
1510 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1512 old_desc = seg_desc;
1513 seg_desc.type |= 2; /* busy */
1514 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1515 sizeof(seg_desc), &ctxt->exception);
1516 if (ret != X86EMUL_CONTINUE)
1519 case VCPU_SREG_LDTR:
1520 if (seg_desc.s || seg_desc.type != 2)
1523 default: /* DS, ES, FS, or GS */
1525 * segment is not a data or readable code segment or
1526 * ((segment is a data or nonconforming code segment)
1527 * and (both RPL and CPL > DPL))
1529 if ((seg_desc.type & 0xa) == 0x8 ||
1530 (((seg_desc.type & 0xc) != 0xc) &&
1531 (rpl > dpl && cpl > dpl)))
1537 /* mark segment as accessed */
1539 ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1540 if (ret != X86EMUL_CONTINUE)
1542 } else if (ctxt->mode == X86EMUL_MODE_PROT64) {
1543 ret = ctxt->ops->read_std(ctxt, desc_addr+8, &base3,
1544 sizeof(base3), &ctxt->exception);
1545 if (ret != X86EMUL_CONTINUE)
1549 ctxt->ops->set_segment(ctxt, selector, &seg_desc, base3, seg);
1550 return X86EMUL_CONTINUE;
1552 emulate_exception(ctxt, err_vec, err_code, true);
1553 return X86EMUL_PROPAGATE_FAULT;
1556 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1557 u16 selector, int seg)
1559 u8 cpl = ctxt->ops->cpl(ctxt);
1560 return __load_segment_descriptor(ctxt, selector, seg, cpl, false);
1563 static void write_register_operand(struct operand *op)
1565 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1566 switch (op->bytes) {
1568 *(u8 *)op->addr.reg = (u8)op->val;
1571 *(u16 *)op->addr.reg = (u16)op->val;
1574 *op->addr.reg = (u32)op->val;
1575 break; /* 64b: zero-extend */
1577 *op->addr.reg = op->val;
1582 static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
1588 write_register_operand(op);
1591 if (ctxt->lock_prefix)
1592 rc = segmented_cmpxchg(ctxt,
1598 rc = segmented_write(ctxt,
1602 if (rc != X86EMUL_CONTINUE)
1606 rc = segmented_write(ctxt,
1609 op->bytes * op->count);
1610 if (rc != X86EMUL_CONTINUE)
1614 write_sse_reg(ctxt, &op->vec_val, op->addr.xmm);
1617 write_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
1625 return X86EMUL_CONTINUE;
1628 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1630 struct segmented_address addr;
1632 rsp_increment(ctxt, -bytes);
1633 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1634 addr.seg = VCPU_SREG_SS;
1636 return segmented_write(ctxt, addr, data, bytes);
1639 static int em_push(struct x86_emulate_ctxt *ctxt)
1641 /* Disable writeback. */
1642 ctxt->dst.type = OP_NONE;
1643 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1646 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1647 void *dest, int len)
1650 struct segmented_address addr;
1652 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1653 addr.seg = VCPU_SREG_SS;
1654 rc = segmented_read(ctxt, addr, dest, len);
1655 if (rc != X86EMUL_CONTINUE)
1658 rsp_increment(ctxt, len);
1662 static int em_pop(struct x86_emulate_ctxt *ctxt)
1664 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1667 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1668 void *dest, int len)
1671 unsigned long val, change_mask;
1672 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1673 int cpl = ctxt->ops->cpl(ctxt);
1675 rc = emulate_pop(ctxt, &val, len);
1676 if (rc != X86EMUL_CONTINUE)
1679 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1680 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1682 switch(ctxt->mode) {
1683 case X86EMUL_MODE_PROT64:
1684 case X86EMUL_MODE_PROT32:
1685 case X86EMUL_MODE_PROT16:
1687 change_mask |= EFLG_IOPL;
1689 change_mask |= EFLG_IF;
1691 case X86EMUL_MODE_VM86:
1693 return emulate_gp(ctxt, 0);
1694 change_mask |= EFLG_IF;
1696 default: /* real mode */
1697 change_mask |= (EFLG_IOPL | EFLG_IF);
1701 *(unsigned long *)dest =
1702 (ctxt->eflags & ~change_mask) | (val & change_mask);
1707 static int em_popf(struct x86_emulate_ctxt *ctxt)
1709 ctxt->dst.type = OP_REG;
1710 ctxt->dst.addr.reg = &ctxt->eflags;
1711 ctxt->dst.bytes = ctxt->op_bytes;
1712 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1715 static int em_enter(struct x86_emulate_ctxt *ctxt)
1718 unsigned frame_size = ctxt->src.val;
1719 unsigned nesting_level = ctxt->src2.val & 31;
1723 return X86EMUL_UNHANDLEABLE;
1725 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1726 rc = push(ctxt, &rbp, stack_size(ctxt));
1727 if (rc != X86EMUL_CONTINUE)
1729 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1731 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1732 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1734 return X86EMUL_CONTINUE;
1737 static int em_leave(struct x86_emulate_ctxt *ctxt)
1739 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1741 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1744 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1746 int seg = ctxt->src2.val;
1748 ctxt->src.val = get_segment_selector(ctxt, seg);
1750 return em_push(ctxt);
1753 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1755 int seg = ctxt->src2.val;
1756 unsigned long selector;
1759 rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1760 if (rc != X86EMUL_CONTINUE)
1763 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1767 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1769 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1770 int rc = X86EMUL_CONTINUE;
1771 int reg = VCPU_REGS_RAX;
1773 while (reg <= VCPU_REGS_RDI) {
1774 (reg == VCPU_REGS_RSP) ?
1775 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
1778 if (rc != X86EMUL_CONTINUE)
1787 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1789 ctxt->src.val = (unsigned long)ctxt->eflags;
1790 return em_push(ctxt);
1793 static int em_popa(struct x86_emulate_ctxt *ctxt)
1795 int rc = X86EMUL_CONTINUE;
1796 int reg = VCPU_REGS_RDI;
1798 while (reg >= VCPU_REGS_RAX) {
1799 if (reg == VCPU_REGS_RSP) {
1800 rsp_increment(ctxt, ctxt->op_bytes);
1804 rc = emulate_pop(ctxt, reg_rmw(ctxt, reg), ctxt->op_bytes);
1805 if (rc != X86EMUL_CONTINUE)
1812 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1814 const struct x86_emulate_ops *ops = ctxt->ops;
1821 /* TODO: Add limit checks */
1822 ctxt->src.val = ctxt->eflags;
1824 if (rc != X86EMUL_CONTINUE)
1827 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1829 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1831 if (rc != X86EMUL_CONTINUE)
1834 ctxt->src.val = ctxt->_eip;
1836 if (rc != X86EMUL_CONTINUE)
1839 ops->get_idt(ctxt, &dt);
1841 eip_addr = dt.address + (irq << 2);
1842 cs_addr = dt.address + (irq << 2) + 2;
1844 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1845 if (rc != X86EMUL_CONTINUE)
1848 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1849 if (rc != X86EMUL_CONTINUE)
1852 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1853 if (rc != X86EMUL_CONTINUE)
1861 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1865 invalidate_registers(ctxt);
1866 rc = __emulate_int_real(ctxt, irq);
1867 if (rc == X86EMUL_CONTINUE)
1868 writeback_registers(ctxt);
1872 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1874 switch(ctxt->mode) {
1875 case X86EMUL_MODE_REAL:
1876 return __emulate_int_real(ctxt, irq);
1877 case X86EMUL_MODE_VM86:
1878 case X86EMUL_MODE_PROT16:
1879 case X86EMUL_MODE_PROT32:
1880 case X86EMUL_MODE_PROT64:
1882 /* Protected mode interrupts unimplemented yet */
1883 return X86EMUL_UNHANDLEABLE;
1887 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
1889 int rc = X86EMUL_CONTINUE;
1890 unsigned long temp_eip = 0;
1891 unsigned long temp_eflags = 0;
1892 unsigned long cs = 0;
1893 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1894 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1895 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1896 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1898 /* TODO: Add stack limit check */
1900 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
1902 if (rc != X86EMUL_CONTINUE)
1905 if (temp_eip & ~0xffff)
1906 return emulate_gp(ctxt, 0);
1908 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1910 if (rc != X86EMUL_CONTINUE)
1913 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
1915 if (rc != X86EMUL_CONTINUE)
1918 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1920 if (rc != X86EMUL_CONTINUE)
1923 ctxt->_eip = temp_eip;
1926 if (ctxt->op_bytes == 4)
1927 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1928 else if (ctxt->op_bytes == 2) {
1929 ctxt->eflags &= ~0xffff;
1930 ctxt->eflags |= temp_eflags;
1933 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1934 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1939 static int em_iret(struct x86_emulate_ctxt *ctxt)
1941 switch(ctxt->mode) {
1942 case X86EMUL_MODE_REAL:
1943 return emulate_iret_real(ctxt);
1944 case X86EMUL_MODE_VM86:
1945 case X86EMUL_MODE_PROT16:
1946 case X86EMUL_MODE_PROT32:
1947 case X86EMUL_MODE_PROT64:
1949 /* iret from protected mode unimplemented yet */
1950 return X86EMUL_UNHANDLEABLE;
1954 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
1959 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1961 rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
1962 if (rc != X86EMUL_CONTINUE)
1966 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
1967 return X86EMUL_CONTINUE;
1970 static int em_grp45(struct x86_emulate_ctxt *ctxt)
1972 int rc = X86EMUL_CONTINUE;
1974 switch (ctxt->modrm_reg) {
1975 case 2: /* call near abs */ {
1977 old_eip = ctxt->_eip;
1978 ctxt->_eip = ctxt->src.val;
1979 ctxt->src.val = old_eip;
1983 case 4: /* jmp abs */
1984 ctxt->_eip = ctxt->src.val;
1986 case 5: /* jmp far */
1987 rc = em_jmp_far(ctxt);
1996 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
1998 u64 old = ctxt->dst.orig_val64;
2000 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
2001 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2002 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2003 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2004 ctxt->eflags &= ~EFLG_ZF;
2006 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2007 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2009 ctxt->eflags |= EFLG_ZF;
2011 return X86EMUL_CONTINUE;
2014 static int em_ret(struct x86_emulate_ctxt *ctxt)
2016 ctxt->dst.type = OP_REG;
2017 ctxt->dst.addr.reg = &ctxt->_eip;
2018 ctxt->dst.bytes = ctxt->op_bytes;
2019 return em_pop(ctxt);
2022 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2027 rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
2028 if (rc != X86EMUL_CONTINUE)
2030 if (ctxt->op_bytes == 4)
2031 ctxt->_eip = (u32)ctxt->_eip;
2032 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2033 if (rc != X86EMUL_CONTINUE)
2035 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2039 static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt)
2043 rc = em_ret_far(ctxt);
2044 if (rc != X86EMUL_CONTINUE)
2046 rsp_increment(ctxt, ctxt->src.val);
2047 return X86EMUL_CONTINUE;
2050 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2052 /* Save real source value, then compare EAX against destination. */
2053 ctxt->src.orig_val = ctxt->src.val;
2054 ctxt->src.val = reg_read(ctxt, VCPU_REGS_RAX);
2055 fastop(ctxt, em_cmp);
2057 if (ctxt->eflags & EFLG_ZF) {
2058 /* Success: write back to memory. */
2059 ctxt->dst.val = ctxt->src.orig_val;
2061 /* Failure: write the value we saw to EAX. */
2062 ctxt->dst.type = OP_REG;
2063 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2065 return X86EMUL_CONTINUE;
2068 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2070 int seg = ctxt->src2.val;
2074 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2076 rc = load_segment_descriptor(ctxt, sel, seg);
2077 if (rc != X86EMUL_CONTINUE)
2080 ctxt->dst.val = ctxt->src.val;
2085 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2086 struct desc_struct *cs, struct desc_struct *ss)
2088 cs->l = 0; /* will be adjusted later */
2089 set_desc_base(cs, 0); /* flat segment */
2090 cs->g = 1; /* 4kb granularity */
2091 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2092 cs->type = 0x0b; /* Read, Execute, Accessed */
2094 cs->dpl = 0; /* will be adjusted later */
2099 set_desc_base(ss, 0); /* flat segment */
2100 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2101 ss->g = 1; /* 4kb granularity */
2103 ss->type = 0x03; /* Read/Write, Accessed */
2104 ss->d = 1; /* 32bit stack segment */
2111 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2113 u32 eax, ebx, ecx, edx;
2116 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2117 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2118 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2119 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2122 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2124 const struct x86_emulate_ops *ops = ctxt->ops;
2125 u32 eax, ebx, ecx, edx;
2128 * syscall should always be enabled in longmode - so only become
2129 * vendor specific (cpuid) if other modes are active...
2131 if (ctxt->mode == X86EMUL_MODE_PROT64)
2136 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2138 * Intel ("GenuineIntel")
2139 * remark: Intel CPUs only support "syscall" in 64bit
2140 * longmode. Also an 64bit guest with a
2141 * 32bit compat-app running will #UD !! While this
2142 * behaviour can be fixed (by emulating) into AMD
2143 * response - CPUs of AMD can't behave like Intel.
2145 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2146 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2147 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2150 /* AMD ("AuthenticAMD") */
2151 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2152 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2153 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2156 /* AMD ("AMDisbetter!") */
2157 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2158 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2159 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2162 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2166 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2168 const struct x86_emulate_ops *ops = ctxt->ops;
2169 struct desc_struct cs, ss;
2174 /* syscall is not available in real mode */
2175 if (ctxt->mode == X86EMUL_MODE_REAL ||
2176 ctxt->mode == X86EMUL_MODE_VM86)
2177 return emulate_ud(ctxt);
2179 if (!(em_syscall_is_enabled(ctxt)))
2180 return emulate_ud(ctxt);
2182 ops->get_msr(ctxt, MSR_EFER, &efer);
2183 setup_syscalls_segments(ctxt, &cs, &ss);
2185 if (!(efer & EFER_SCE))
2186 return emulate_ud(ctxt);
2188 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2190 cs_sel = (u16)(msr_data & 0xfffc);
2191 ss_sel = (u16)(msr_data + 8);
2193 if (efer & EFER_LMA) {
2197 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2198 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2200 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2201 if (efer & EFER_LMA) {
2202 #ifdef CONFIG_X86_64
2203 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags & ~EFLG_RF;
2206 ctxt->mode == X86EMUL_MODE_PROT64 ?
2207 MSR_LSTAR : MSR_CSTAR, &msr_data);
2208 ctxt->_eip = msr_data;
2210 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2211 ctxt->eflags &= ~(msr_data | EFLG_RF);
2215 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2216 ctxt->_eip = (u32)msr_data;
2218 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2221 return X86EMUL_CONTINUE;
2224 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2226 const struct x86_emulate_ops *ops = ctxt->ops;
2227 struct desc_struct cs, ss;
2232 ops->get_msr(ctxt, MSR_EFER, &efer);
2233 /* inject #GP if in real mode */
2234 if (ctxt->mode == X86EMUL_MODE_REAL)
2235 return emulate_gp(ctxt, 0);
2238 * Not recognized on AMD in compat mode (but is recognized in legacy
2241 if ((ctxt->mode == X86EMUL_MODE_PROT32) && (efer & EFER_LMA)
2242 && !vendor_intel(ctxt))
2243 return emulate_ud(ctxt);
2245 /* XXX sysenter/sysexit have not been tested in 64bit mode.
2246 * Therefore, we inject an #UD.
2248 if (ctxt->mode == X86EMUL_MODE_PROT64)
2249 return emulate_ud(ctxt);
2251 setup_syscalls_segments(ctxt, &cs, &ss);
2253 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2254 switch (ctxt->mode) {
2255 case X86EMUL_MODE_PROT32:
2256 if ((msr_data & 0xfffc) == 0x0)
2257 return emulate_gp(ctxt, 0);
2259 case X86EMUL_MODE_PROT64:
2260 if (msr_data == 0x0)
2261 return emulate_gp(ctxt, 0);
2267 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2268 cs_sel = (u16)msr_data;
2269 cs_sel &= ~SELECTOR_RPL_MASK;
2270 ss_sel = cs_sel + 8;
2271 ss_sel &= ~SELECTOR_RPL_MASK;
2272 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
2277 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2278 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2280 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2281 ctxt->_eip = msr_data;
2283 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2284 *reg_write(ctxt, VCPU_REGS_RSP) = msr_data;
2286 return X86EMUL_CONTINUE;
2289 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2291 const struct x86_emulate_ops *ops = ctxt->ops;
2292 struct desc_struct cs, ss;
2295 u16 cs_sel = 0, ss_sel = 0;
2297 /* inject #GP if in real mode or Virtual 8086 mode */
2298 if (ctxt->mode == X86EMUL_MODE_REAL ||
2299 ctxt->mode == X86EMUL_MODE_VM86)
2300 return emulate_gp(ctxt, 0);
2302 setup_syscalls_segments(ctxt, &cs, &ss);
2304 if ((ctxt->rex_prefix & 0x8) != 0x0)
2305 usermode = X86EMUL_MODE_PROT64;
2307 usermode = X86EMUL_MODE_PROT32;
2311 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2313 case X86EMUL_MODE_PROT32:
2314 cs_sel = (u16)(msr_data + 16);
2315 if ((msr_data & 0xfffc) == 0x0)
2316 return emulate_gp(ctxt, 0);
2317 ss_sel = (u16)(msr_data + 24);
2319 case X86EMUL_MODE_PROT64:
2320 cs_sel = (u16)(msr_data + 32);
2321 if (msr_data == 0x0)
2322 return emulate_gp(ctxt, 0);
2323 ss_sel = cs_sel + 8;
2328 cs_sel |= SELECTOR_RPL_MASK;
2329 ss_sel |= SELECTOR_RPL_MASK;
2331 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2332 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2334 ctxt->_eip = reg_read(ctxt, VCPU_REGS_RDX);
2335 *reg_write(ctxt, VCPU_REGS_RSP) = reg_read(ctxt, VCPU_REGS_RCX);
2337 return X86EMUL_CONTINUE;
2340 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2343 if (ctxt->mode == X86EMUL_MODE_REAL)
2345 if (ctxt->mode == X86EMUL_MODE_VM86)
2347 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2348 return ctxt->ops->cpl(ctxt) > iopl;
2351 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2354 const struct x86_emulate_ops *ops = ctxt->ops;
2355 struct desc_struct tr_seg;
2358 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2359 unsigned mask = (1 << len) - 1;
2362 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2365 if (desc_limit_scaled(&tr_seg) < 103)
2367 base = get_desc_base(&tr_seg);
2368 #ifdef CONFIG_X86_64
2369 base |= ((u64)base3) << 32;
2371 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2372 if (r != X86EMUL_CONTINUE)
2374 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2376 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2377 if (r != X86EMUL_CONTINUE)
2379 if ((perm >> bit_idx) & mask)
2384 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2390 if (emulator_bad_iopl(ctxt))
2391 if (!emulator_io_port_access_allowed(ctxt, port, len))
2394 ctxt->perm_ok = true;
2399 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2400 struct tss_segment_16 *tss)
2402 tss->ip = ctxt->_eip;
2403 tss->flag = ctxt->eflags;
2404 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
2405 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
2406 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
2407 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
2408 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
2409 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
2410 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
2411 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2413 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2414 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2415 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2416 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2417 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2420 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2421 struct tss_segment_16 *tss)
2426 ctxt->_eip = tss->ip;
2427 ctxt->eflags = tss->flag | 2;
2428 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
2429 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
2430 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
2431 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
2432 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
2433 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
2434 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
2435 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
2438 * SDM says that segment selectors are loaded before segment
2441 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2442 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2443 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2444 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2445 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2450 * Now load segment descriptors. If fault happens at this stage
2451 * it is handled in a context of new task
2453 ret = __load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR, cpl, true);
2454 if (ret != X86EMUL_CONTINUE)
2456 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl, true);
2457 if (ret != X86EMUL_CONTINUE)
2459 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl, true);
2460 if (ret != X86EMUL_CONTINUE)
2462 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl, true);
2463 if (ret != X86EMUL_CONTINUE)
2465 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl, true);
2466 if (ret != X86EMUL_CONTINUE)
2469 return X86EMUL_CONTINUE;
2472 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2473 u16 tss_selector, u16 old_tss_sel,
2474 ulong old_tss_base, struct desc_struct *new_desc)
2476 const struct x86_emulate_ops *ops = ctxt->ops;
2477 struct tss_segment_16 tss_seg;
2479 u32 new_tss_base = get_desc_base(new_desc);
2481 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2483 if (ret != X86EMUL_CONTINUE)
2484 /* FIXME: need to provide precise fault address */
2487 save_state_to_tss16(ctxt, &tss_seg);
2489 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2491 if (ret != X86EMUL_CONTINUE)
2492 /* FIXME: need to provide precise fault address */
2495 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2497 if (ret != X86EMUL_CONTINUE)
2498 /* FIXME: need to provide precise fault address */
2501 if (old_tss_sel != 0xffff) {
2502 tss_seg.prev_task_link = old_tss_sel;
2504 ret = ops->write_std(ctxt, new_tss_base,
2505 &tss_seg.prev_task_link,
2506 sizeof tss_seg.prev_task_link,
2508 if (ret != X86EMUL_CONTINUE)
2509 /* FIXME: need to provide precise fault address */
2513 return load_state_from_tss16(ctxt, &tss_seg);
2516 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2517 struct tss_segment_32 *tss)
2519 /* CR3 and ldt selector are not saved intentionally */
2520 tss->eip = ctxt->_eip;
2521 tss->eflags = ctxt->eflags;
2522 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
2523 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
2524 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
2525 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
2526 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
2527 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
2528 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
2529 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
2531 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2532 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2533 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2534 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2535 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2536 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2539 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2540 struct tss_segment_32 *tss)
2545 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2546 return emulate_gp(ctxt, 0);
2547 ctxt->_eip = tss->eip;
2548 ctxt->eflags = tss->eflags | 2;
2550 /* General purpose registers */
2551 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
2552 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
2553 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
2554 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
2555 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
2556 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
2557 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
2558 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
2561 * SDM says that segment selectors are loaded before segment
2562 * descriptors. This is important because CPL checks will
2565 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2566 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2567 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2568 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2569 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2570 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2571 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2574 * If we're switching between Protected Mode and VM86, we need to make
2575 * sure to update the mode before loading the segment descriptors so
2576 * that the selectors are interpreted correctly.
2578 if (ctxt->eflags & X86_EFLAGS_VM) {
2579 ctxt->mode = X86EMUL_MODE_VM86;
2582 ctxt->mode = X86EMUL_MODE_PROT32;
2587 * Now load segment descriptors. If fault happenes at this stage
2588 * it is handled in a context of new task
2590 ret = __load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR, cpl, true);
2591 if (ret != X86EMUL_CONTINUE)
2593 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl, true);
2594 if (ret != X86EMUL_CONTINUE)
2596 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl, true);
2597 if (ret != X86EMUL_CONTINUE)
2599 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl, true);
2600 if (ret != X86EMUL_CONTINUE)
2602 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl, true);
2603 if (ret != X86EMUL_CONTINUE)
2605 ret = __load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS, cpl, true);
2606 if (ret != X86EMUL_CONTINUE)
2608 ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl, true);
2609 if (ret != X86EMUL_CONTINUE)
2612 return X86EMUL_CONTINUE;
2615 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2616 u16 tss_selector, u16 old_tss_sel,
2617 ulong old_tss_base, struct desc_struct *new_desc)
2619 const struct x86_emulate_ops *ops = ctxt->ops;
2620 struct tss_segment_32 tss_seg;
2622 u32 new_tss_base = get_desc_base(new_desc);
2623 u32 eip_offset = offsetof(struct tss_segment_32, eip);
2624 u32 ldt_sel_offset = offsetof(struct tss_segment_32, ldt_selector);
2626 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2628 if (ret != X86EMUL_CONTINUE)
2629 /* FIXME: need to provide precise fault address */
2632 save_state_to_tss32(ctxt, &tss_seg);
2634 /* Only GP registers and segment selectors are saved */
2635 ret = ops->write_std(ctxt, old_tss_base + eip_offset, &tss_seg.eip,
2636 ldt_sel_offset - eip_offset, &ctxt->exception);
2637 if (ret != X86EMUL_CONTINUE)
2638 /* FIXME: need to provide precise fault address */
2641 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2643 if (ret != X86EMUL_CONTINUE)
2644 /* FIXME: need to provide precise fault address */
2647 if (old_tss_sel != 0xffff) {
2648 tss_seg.prev_task_link = old_tss_sel;
2650 ret = ops->write_std(ctxt, new_tss_base,
2651 &tss_seg.prev_task_link,
2652 sizeof tss_seg.prev_task_link,
2654 if (ret != X86EMUL_CONTINUE)
2655 /* FIXME: need to provide precise fault address */
2659 return load_state_from_tss32(ctxt, &tss_seg);
2662 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2663 u16 tss_selector, int idt_index, int reason,
2664 bool has_error_code, u32 error_code)
2666 const struct x86_emulate_ops *ops = ctxt->ops;
2667 struct desc_struct curr_tss_desc, next_tss_desc;
2669 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2670 ulong old_tss_base =
2671 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2675 /* FIXME: old_tss_base == ~0 ? */
2677 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
2678 if (ret != X86EMUL_CONTINUE)
2680 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
2681 if (ret != X86EMUL_CONTINUE)
2684 /* FIXME: check that next_tss_desc is tss */
2687 * Check privileges. The three cases are task switch caused by...
2689 * 1. jmp/call/int to task gate: Check against DPL of the task gate
2690 * 2. Exception/IRQ/iret: No check is performed
2691 * 3. jmp/call to TSS: Check against DPL of the TSS
2693 if (reason == TASK_SWITCH_GATE) {
2694 if (idt_index != -1) {
2695 /* Software interrupts */
2696 struct desc_struct task_gate_desc;
2699 ret = read_interrupt_descriptor(ctxt, idt_index,
2701 if (ret != X86EMUL_CONTINUE)
2704 dpl = task_gate_desc.dpl;
2705 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2706 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
2708 } else if (reason != TASK_SWITCH_IRET) {
2709 int dpl = next_tss_desc.dpl;
2710 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2711 return emulate_gp(ctxt, tss_selector);
2715 desc_limit = desc_limit_scaled(&next_tss_desc);
2716 if (!next_tss_desc.p ||
2717 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2718 desc_limit < 0x2b)) {
2719 emulate_ts(ctxt, tss_selector & 0xfffc);
2720 return X86EMUL_PROPAGATE_FAULT;
2723 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2724 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2725 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2728 if (reason == TASK_SWITCH_IRET)
2729 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2731 /* set back link to prev task only if NT bit is set in eflags
2732 note that old_tss_sel is not used after this point */
2733 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2734 old_tss_sel = 0xffff;
2736 if (next_tss_desc.type & 8)
2737 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2738 old_tss_base, &next_tss_desc);
2740 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2741 old_tss_base, &next_tss_desc);
2742 if (ret != X86EMUL_CONTINUE)
2745 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2746 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2748 if (reason != TASK_SWITCH_IRET) {
2749 next_tss_desc.type |= (1 << 1); /* set busy flag */
2750 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2753 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2754 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2756 if (has_error_code) {
2757 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2758 ctxt->lock_prefix = 0;
2759 ctxt->src.val = (unsigned long) error_code;
2760 ret = em_push(ctxt);
2766 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2767 u16 tss_selector, int idt_index, int reason,
2768 bool has_error_code, u32 error_code)
2772 invalidate_registers(ctxt);
2773 ctxt->_eip = ctxt->eip;
2774 ctxt->dst.type = OP_NONE;
2776 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
2777 has_error_code, error_code);
2779 if (rc == X86EMUL_CONTINUE) {
2780 ctxt->eip = ctxt->_eip;
2781 writeback_registers(ctxt);
2784 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2787 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
2790 int df = (ctxt->eflags & EFLG_DF) ? -op->count : op->count;
2792 register_address_increment(ctxt, reg_rmw(ctxt, reg), df * op->bytes);
2793 op->addr.mem.ea = register_address(ctxt, reg_read(ctxt, reg));
2796 static int em_das(struct x86_emulate_ctxt *ctxt)
2799 bool af, cf, old_cf;
2801 cf = ctxt->eflags & X86_EFLAGS_CF;
2807 af = ctxt->eflags & X86_EFLAGS_AF;
2808 if ((al & 0x0f) > 9 || af) {
2810 cf = old_cf | (al >= 250);
2815 if (old_al > 0x99 || old_cf) {
2821 /* Set PF, ZF, SF */
2822 ctxt->src.type = OP_IMM;
2824 ctxt->src.bytes = 1;
2825 fastop(ctxt, em_or);
2826 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2828 ctxt->eflags |= X86_EFLAGS_CF;
2830 ctxt->eflags |= X86_EFLAGS_AF;
2831 return X86EMUL_CONTINUE;
2834 static int em_aam(struct x86_emulate_ctxt *ctxt)
2838 if (ctxt->src.val == 0)
2839 return emulate_de(ctxt);
2841 al = ctxt->dst.val & 0xff;
2842 ah = al / ctxt->src.val;
2843 al %= ctxt->src.val;
2845 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);
2847 /* Set PF, ZF, SF */
2848 ctxt->src.type = OP_IMM;
2850 ctxt->src.bytes = 1;
2851 fastop(ctxt, em_or);
2853 return X86EMUL_CONTINUE;
2856 static int em_aad(struct x86_emulate_ctxt *ctxt)
2858 u8 al = ctxt->dst.val & 0xff;
2859 u8 ah = (ctxt->dst.val >> 8) & 0xff;
2861 al = (al + (ah * ctxt->src.val)) & 0xff;
2863 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
2865 /* Set PF, ZF, SF */
2866 ctxt->src.type = OP_IMM;
2868 ctxt->src.bytes = 1;
2869 fastop(ctxt, em_or);
2871 return X86EMUL_CONTINUE;
2874 static int em_call(struct x86_emulate_ctxt *ctxt)
2876 long rel = ctxt->src.val;
2878 ctxt->src.val = (unsigned long)ctxt->_eip;
2880 return em_push(ctxt);
2883 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2889 old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2890 old_eip = ctxt->_eip;
2892 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2893 if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
2894 return X86EMUL_CONTINUE;
2897 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2899 ctxt->src.val = old_cs;
2901 if (rc != X86EMUL_CONTINUE)
2904 ctxt->src.val = old_eip;
2905 return em_push(ctxt);
2908 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2912 ctxt->dst.type = OP_REG;
2913 ctxt->dst.addr.reg = &ctxt->_eip;
2914 ctxt->dst.bytes = ctxt->op_bytes;
2915 rc = emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
2916 if (rc != X86EMUL_CONTINUE)
2918 rsp_increment(ctxt, ctxt->src.val);
2919 return X86EMUL_CONTINUE;
2922 static int em_xchg(struct x86_emulate_ctxt *ctxt)
2924 /* Write back the register source. */
2925 ctxt->src.val = ctxt->dst.val;
2926 write_register_operand(&ctxt->src);
2928 /* Write back the memory destination with implicit LOCK prefix. */
2929 ctxt->dst.val = ctxt->src.orig_val;
2930 ctxt->lock_prefix = 1;
2931 return X86EMUL_CONTINUE;
2934 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2936 ctxt->dst.val = ctxt->src2.val;
2937 return fastop(ctxt, em_imul);
2940 static int em_cwd(struct x86_emulate_ctxt *ctxt)
2942 ctxt->dst.type = OP_REG;
2943 ctxt->dst.bytes = ctxt->src.bytes;
2944 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
2945 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
2947 return X86EMUL_CONTINUE;
2950 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2954 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
2955 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
2956 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
2957 return X86EMUL_CONTINUE;
2960 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
2964 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
2965 return emulate_gp(ctxt, 0);
2966 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
2967 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
2968 return X86EMUL_CONTINUE;
2971 static int em_mov(struct x86_emulate_ctxt *ctxt)
2973 memcpy(ctxt->dst.valptr, ctxt->src.valptr, ctxt->op_bytes);
2974 return X86EMUL_CONTINUE;
2977 #define FFL(x) bit(X86_FEATURE_##x)
2979 static int em_movbe(struct x86_emulate_ctxt *ctxt)
2981 u32 ebx, ecx, edx, eax = 1;
2985 * Check MOVBE is set in the guest-visible CPUID leaf.
2987 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2988 if (!(ecx & FFL(MOVBE)))
2989 return emulate_ud(ctxt);
2991 switch (ctxt->op_bytes) {
2994 * From MOVBE definition: "...When the operand size is 16 bits,
2995 * the upper word of the destination register remains unchanged
2998 * Both casting ->valptr and ->val to u16 breaks strict aliasing
2999 * rules so we have to do the operation almost per hand.
3001 tmp = (u16)ctxt->src.val;
3002 ctxt->dst.val &= ~0xffffUL;
3003 ctxt->dst.val |= (unsigned long)swab16(tmp);
3006 ctxt->dst.val = swab32((u32)ctxt->src.val);
3009 ctxt->dst.val = swab64(ctxt->src.val);
3012 return X86EMUL_PROPAGATE_FAULT;
3014 return X86EMUL_CONTINUE;
3017 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3019 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3020 return emulate_gp(ctxt, 0);
3022 /* Disable writeback. */
3023 ctxt->dst.type = OP_NONE;
3024 return X86EMUL_CONTINUE;
3027 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3031 if (ctxt->mode == X86EMUL_MODE_PROT64)
3032 val = ctxt->src.val & ~0ULL;
3034 val = ctxt->src.val & ~0U;
3036 /* #UD condition is already handled. */
3037 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3038 return emulate_gp(ctxt, 0);
3040 /* Disable writeback. */
3041 ctxt->dst.type = OP_NONE;
3042 return X86EMUL_CONTINUE;
3045 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3049 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3050 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3051 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3052 return emulate_gp(ctxt, 0);
3054 return X86EMUL_CONTINUE;
3057 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3061 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3062 return emulate_gp(ctxt, 0);
3064 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3065 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3066 return X86EMUL_CONTINUE;
3069 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3071 if (ctxt->modrm_reg > VCPU_SREG_GS)
3072 return emulate_ud(ctxt);
3074 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3075 return X86EMUL_CONTINUE;
3078 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3080 u16 sel = ctxt->src.val;
3082 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3083 return emulate_ud(ctxt);
3085 if (ctxt->modrm_reg == VCPU_SREG_SS)
3086 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3088 /* Disable writeback. */
3089 ctxt->dst.type = OP_NONE;
3090 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3093 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3095 u16 sel = ctxt->src.val;
3097 /* Disable writeback. */
3098 ctxt->dst.type = OP_NONE;
3099 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3102 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3104 u16 sel = ctxt->src.val;
3106 /* Disable writeback. */
3107 ctxt->dst.type = OP_NONE;
3108 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3111 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3116 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3117 if (rc == X86EMUL_CONTINUE)
3118 ctxt->ops->invlpg(ctxt, linear);
3119 /* Disable writeback. */
3120 ctxt->dst.type = OP_NONE;
3121 return X86EMUL_CONTINUE;
3124 static int em_clts(struct x86_emulate_ctxt *ctxt)
3128 cr0 = ctxt->ops->get_cr(ctxt, 0);
3130 ctxt->ops->set_cr(ctxt, 0, cr0);
3131 return X86EMUL_CONTINUE;
3134 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
3138 if (ctxt->modrm_mod != 3 || ctxt->modrm_rm != 1)
3139 return X86EMUL_UNHANDLEABLE;
3141 rc = ctxt->ops->fix_hypercall(ctxt);
3142 if (rc != X86EMUL_CONTINUE)
3145 /* Let the processor re-execute the fixed hypercall */
3146 ctxt->_eip = ctxt->eip;
3147 /* Disable writeback. */
3148 ctxt->dst.type = OP_NONE;
3149 return X86EMUL_CONTINUE;
3152 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3153 void (*get)(struct x86_emulate_ctxt *ctxt,
3154 struct desc_ptr *ptr))
3156 struct desc_ptr desc_ptr;
3158 if (ctxt->mode == X86EMUL_MODE_PROT64)
3160 get(ctxt, &desc_ptr);
3161 if (ctxt->op_bytes == 2) {
3163 desc_ptr.address &= 0x00ffffff;
3165 /* Disable writeback. */
3166 ctxt->dst.type = OP_NONE;
3167 return segmented_write(ctxt, ctxt->dst.addr.mem,
3168 &desc_ptr, 2 + ctxt->op_bytes);
3171 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3173 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3176 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3178 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3181 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3183 struct desc_ptr desc_ptr;
3186 if (ctxt->mode == X86EMUL_MODE_PROT64)
3188 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3189 &desc_ptr.size, &desc_ptr.address,
3191 if (rc != X86EMUL_CONTINUE)
3193 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3194 /* Disable writeback. */
3195 ctxt->dst.type = OP_NONE;
3196 return X86EMUL_CONTINUE;
3199 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
3203 rc = ctxt->ops->fix_hypercall(ctxt);
3205 /* Disable writeback. */
3206 ctxt->dst.type = OP_NONE;
3210 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3212 struct desc_ptr desc_ptr;
3215 if (ctxt->mode == X86EMUL_MODE_PROT64)
3217 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3218 &desc_ptr.size, &desc_ptr.address,
3220 if (rc != X86EMUL_CONTINUE)
3222 ctxt->ops->set_idt(ctxt, &desc_ptr);
3223 /* Disable writeback. */
3224 ctxt->dst.type = OP_NONE;
3225 return X86EMUL_CONTINUE;
3228 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3230 ctxt->dst.bytes = 2;
3231 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3232 return X86EMUL_CONTINUE;
3235 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3237 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3238 | (ctxt->src.val & 0x0f));
3239 ctxt->dst.type = OP_NONE;
3240 return X86EMUL_CONTINUE;
3243 static int em_loop(struct x86_emulate_ctxt *ctxt)
3245 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX), -1);
3246 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3247 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3248 jmp_rel(ctxt, ctxt->src.val);
3250 return X86EMUL_CONTINUE;
3253 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3255 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3256 jmp_rel(ctxt, ctxt->src.val);
3258 return X86EMUL_CONTINUE;
3261 static int em_in(struct x86_emulate_ctxt *ctxt)
3263 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3265 return X86EMUL_IO_NEEDED;
3267 return X86EMUL_CONTINUE;
3270 static int em_out(struct x86_emulate_ctxt *ctxt)
3272 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3274 /* Disable writeback. */
3275 ctxt->dst.type = OP_NONE;
3276 return X86EMUL_CONTINUE;
3279 static int em_cli(struct x86_emulate_ctxt *ctxt)
3281 if (emulator_bad_iopl(ctxt))
3282 return emulate_gp(ctxt, 0);
3284 ctxt->eflags &= ~X86_EFLAGS_IF;
3285 return X86EMUL_CONTINUE;
3288 static int em_sti(struct x86_emulate_ctxt *ctxt)
3290 if (emulator_bad_iopl(ctxt))
3291 return emulate_gp(ctxt, 0);
3293 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3294 ctxt->eflags |= X86_EFLAGS_IF;
3295 return X86EMUL_CONTINUE;
3298 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3300 u32 eax, ebx, ecx, edx;
3302 eax = reg_read(ctxt, VCPU_REGS_RAX);
3303 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3304 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3305 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3306 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3307 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3308 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3309 return X86EMUL_CONTINUE;
3312 static int em_sahf(struct x86_emulate_ctxt *ctxt)
3316 flags = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF;
3317 flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8;
3319 ctxt->eflags &= ~0xffUL;
3320 ctxt->eflags |= flags | X86_EFLAGS_FIXED;
3321 return X86EMUL_CONTINUE;
3324 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3326 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3327 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3328 return X86EMUL_CONTINUE;
3331 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3333 switch (ctxt->op_bytes) {
3334 #ifdef CONFIG_X86_64
3336 asm("bswap %0" : "+r"(ctxt->dst.val));
3340 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3343 return X86EMUL_CONTINUE;
3346 static bool valid_cr(int nr)
3358 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
3360 if (!valid_cr(ctxt->modrm_reg))
3361 return emulate_ud(ctxt);
3363 return X86EMUL_CONTINUE;
3366 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
3368 u64 new_val = ctxt->src.val64;
3369 int cr = ctxt->modrm_reg;
3372 static u64 cr_reserved_bits[] = {
3373 0xffffffff00000000ULL,
3374 0, 0, 0, /* CR3 checked later */
3381 return emulate_ud(ctxt);
3383 if (new_val & cr_reserved_bits[cr])
3384 return emulate_gp(ctxt, 0);
3389 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
3390 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
3391 return emulate_gp(ctxt, 0);
3393 cr4 = ctxt->ops->get_cr(ctxt, 4);
3394 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3396 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
3397 !(cr4 & X86_CR4_PAE))
3398 return emulate_gp(ctxt, 0);
3405 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3406 if (efer & EFER_LMA)
3407 rsvd = CR3_L_MODE_RESERVED_BITS;
3410 return emulate_gp(ctxt, 0);
3415 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3417 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
3418 return emulate_gp(ctxt, 0);
3424 return X86EMUL_CONTINUE;
3427 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
3431 ctxt->ops->get_dr(ctxt, 7, &dr7);
3433 /* Check if DR7.Global_Enable is set */
3434 return dr7 & (1 << 13);
3437 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
3439 int dr = ctxt->modrm_reg;
3443 return emulate_ud(ctxt);
3445 cr4 = ctxt->ops->get_cr(ctxt, 4);
3446 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
3447 return emulate_ud(ctxt);
3449 if (check_dr7_gd(ctxt))
3450 return emulate_db(ctxt);
3452 return X86EMUL_CONTINUE;
3455 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
3457 u64 new_val = ctxt->src.val64;
3458 int dr = ctxt->modrm_reg;
3460 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
3461 return emulate_gp(ctxt, 0);
3463 return check_dr_read(ctxt);
3466 static int check_svme(struct x86_emulate_ctxt *ctxt)
3470 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3472 if (!(efer & EFER_SVME))
3473 return emulate_ud(ctxt);
3475 return X86EMUL_CONTINUE;
3478 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
3480 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
3482 /* Valid physical address? */
3483 if (rax & 0xffff000000000000ULL)
3484 return emulate_gp(ctxt, 0);
3486 return check_svme(ctxt);
3489 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
3491 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3493 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3494 return emulate_ud(ctxt);
3496 return X86EMUL_CONTINUE;
3499 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
3501 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3502 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
3504 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3506 return emulate_gp(ctxt, 0);
3508 return X86EMUL_CONTINUE;
3511 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
3513 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
3514 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3515 return emulate_gp(ctxt, 0);
3517 return X86EMUL_CONTINUE;
3520 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
3522 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
3523 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3524 return emulate_gp(ctxt, 0);
3526 return X86EMUL_CONTINUE;
3529 #define D(_y) { .flags = (_y) }
3530 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
3531 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
3532 .check_perm = (_p) }
3533 #define N D(NotImpl)
3534 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3535 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
3536 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
3537 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
3538 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3539 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
3540 #define II(_f, _e, _i) \
3541 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
3542 #define IIP(_f, _e, _i, _p) \
3543 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
3544 .check_perm = (_p) }
3545 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3547 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3548 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3549 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3550 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
3551 #define I2bvIP(_f, _e, _i, _p) \
3552 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
3554 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3555 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3556 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3558 static const struct opcode group7_rm1[] = {
3559 DI(SrcNone | Priv, monitor),
3560 DI(SrcNone | Priv, mwait),
3564 static const struct opcode group7_rm3[] = {
3565 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
3566 II(SrcNone | Prot | EmulateOnUD, em_vmmcall, vmmcall),
3567 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
3568 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
3569 DIP(SrcNone | Prot | Priv, stgi, check_svme),
3570 DIP(SrcNone | Prot | Priv, clgi, check_svme),
3571 DIP(SrcNone | Prot | Priv, skinit, check_svme),
3572 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
3575 static const struct opcode group7_rm7[] = {
3577 DIP(SrcNone, rdtscp, check_rdtsc),
3581 static const struct opcode group1[] = {
3583 F(Lock | PageTable, em_or),
3586 F(Lock | PageTable, em_and),
3592 static const struct opcode group1A[] = {
3593 I(DstMem | SrcNone | Mov | Stack, em_pop), N, N, N, N, N, N, N,
3596 static const struct opcode group2[] = {
3597 F(DstMem | ModRM, em_rol),
3598 F(DstMem | ModRM, em_ror),
3599 F(DstMem | ModRM, em_rcl),
3600 F(DstMem | ModRM, em_rcr),
3601 F(DstMem | ModRM, em_shl),
3602 F(DstMem | ModRM, em_shr),
3603 F(DstMem | ModRM, em_shl),
3604 F(DstMem | ModRM, em_sar),
3607 static const struct opcode group3[] = {
3608 F(DstMem | SrcImm | NoWrite, em_test),
3609 F(DstMem | SrcImm | NoWrite, em_test),
3610 F(DstMem | SrcNone | Lock, em_not),
3611 F(DstMem | SrcNone | Lock, em_neg),
3612 F(DstXacc | Src2Mem, em_mul_ex),
3613 F(DstXacc | Src2Mem, em_imul_ex),
3614 F(DstXacc | Src2Mem, em_div_ex),
3615 F(DstXacc | Src2Mem, em_idiv_ex),
3618 static const struct opcode group4[] = {
3619 F(ByteOp | DstMem | SrcNone | Lock, em_inc),
3620 F(ByteOp | DstMem | SrcNone | Lock, em_dec),
3624 static const struct opcode group5[] = {
3625 F(DstMem | SrcNone | Lock, em_inc),
3626 F(DstMem | SrcNone | Lock, em_dec),
3627 I(SrcMem | Stack, em_grp45),
3628 I(SrcMemFAddr | ImplicitOps | Stack, em_call_far),
3629 I(SrcMem | Stack, em_grp45),
3630 I(SrcMemFAddr | ImplicitOps, em_grp45),
3631 I(SrcMem | Stack, em_grp45), D(Undefined),
3634 static const struct opcode group6[] = {
3637 II(Prot | Priv | SrcMem16, em_lldt, lldt),
3638 II(Prot | Priv | SrcMem16, em_ltr, ltr),
3642 static const struct group_dual group7 = { {
3643 II(Mov | DstMem | Priv, em_sgdt, sgdt),
3644 II(Mov | DstMem | Priv, em_sidt, sidt),
3645 II(SrcMem | Priv, em_lgdt, lgdt),
3646 II(SrcMem | Priv, em_lidt, lidt),
3647 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3648 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3649 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3651 I(SrcNone | Priv | EmulateOnUD, em_vmcall),
3653 N, EXT(0, group7_rm3),
3654 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3655 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3659 static const struct opcode group8[] = {
3661 F(DstMem | SrcImmByte | NoWrite, em_bt),
3662 F(DstMem | SrcImmByte | Lock | PageTable, em_bts),
3663 F(DstMem | SrcImmByte | Lock, em_btr),
3664 F(DstMem | SrcImmByte | Lock | PageTable, em_btc),
3667 static const struct group_dual group9 = { {
3668 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
3670 N, N, N, N, N, N, N, N,
3673 static const struct opcode group11[] = {
3674 I(DstMem | SrcImm | Mov | PageTable, em_mov),
3678 static const struct gprefix pfx_0f_6f_0f_7f = {
3679 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
3682 static const struct gprefix pfx_vmovntpx = {
3683 I(0, em_mov), N, N, N,
3686 static const struct gprefix pfx_0f_28_0f_29 = {
3687 I(Aligned, em_mov), I(Aligned, em_mov), N, N,
3690 static const struct escape escape_d9 = { {
3691 N, N, N, N, N, N, N, I(DstMem, em_fnstcw),
3694 N, N, N, N, N, N, N, N,
3696 N, N, N, N, N, N, N, N,
3698 N, N, N, N, N, N, N, N,
3700 N, N, N, N, N, N, N, N,
3702 N, N, N, N, N, N, N, N,
3704 N, N, N, N, N, N, N, N,
3706 N, N, N, N, N, N, N, N,
3708 N, N, N, N, N, N, N, N,
3711 static const struct escape escape_db = { {
3712 N, N, N, N, N, N, N, N,
3715 N, N, N, N, N, N, N, N,
3717 N, N, N, N, N, N, N, N,
3719 N, N, N, N, N, N, N, N,
3721 N, N, N, N, N, N, N, N,
3723 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
3725 N, N, N, N, N, N, N, N,
3727 N, N, N, N, N, N, N, N,
3729 N, N, N, N, N, N, N, N,
3732 static const struct escape escape_dd = { {
3733 N, N, N, N, N, N, N, I(DstMem, em_fnstsw),
3736 N, N, N, N, N, N, N, N,
3738 N, N, N, N, N, N, N, N,
3740 N, N, N, N, N, N, N, N,
3742 N, N, N, N, N, N, N, N,
3744 N, N, N, N, N, N, N, N,
3746 N, N, N, N, N, N, N, N,
3748 N, N, N, N, N, N, N, N,
3750 N, N, N, N, N, N, N, N,
3753 static const struct opcode opcode_table[256] = {
3755 F6ALU(Lock, em_add),
3756 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
3757 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
3759 F6ALU(Lock | PageTable, em_or),
3760 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
3763 F6ALU(Lock, em_adc),
3764 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
3765 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
3767 F6ALU(Lock, em_sbb),
3768 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
3769 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
3771 F6ALU(Lock | PageTable, em_and), N, N,
3773 F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3775 F6ALU(Lock, em_xor), N, N,
3777 F6ALU(NoWrite, em_cmp), N, N,
3779 X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
3781 X8(I(SrcReg | Stack, em_push)),
3783 X8(I(DstReg | Stack, em_pop)),
3785 I(ImplicitOps | Stack | No64, em_pusha),
3786 I(ImplicitOps | Stack | No64, em_popa),
3787 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3790 I(SrcImm | Mov | Stack, em_push),
3791 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3792 I(SrcImmByte | Mov | Stack, em_push),
3793 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3794 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
3795 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
3799 G(ByteOp | DstMem | SrcImm, group1),
3800 G(DstMem | SrcImm, group1),
3801 G(ByteOp | DstMem | SrcImm | No64, group1),
3802 G(DstMem | SrcImmByte, group1),
3803 F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
3804 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
3806 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
3807 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3808 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
3809 D(ModRM | SrcMem | NoAccess | DstReg),
3810 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
3813 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3815 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3816 I(SrcImmFAddr | No64, em_call_far), N,
3817 II(ImplicitOps | Stack, em_pushf, pushf),
3818 II(ImplicitOps | Stack, em_popf, popf),
3819 I(ImplicitOps, em_sahf), I(ImplicitOps, em_lahf),
3821 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3822 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
3823 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3824 F2bv(SrcSI | DstDI | String | NoWrite, em_cmp),
3826 F2bv(DstAcc | SrcImm | NoWrite, em_test),
3827 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3828 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3829 F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp),
3831 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3833 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
3835 G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
3836 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3837 I(ImplicitOps | Stack, em_ret),
3838 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
3839 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
3840 G(ByteOp, group11), G(0, group11),
3842 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
3843 I(ImplicitOps | Stack | SrcImmU16, em_ret_far_imm),
3844 I(ImplicitOps | Stack, em_ret_far),
3845 D(ImplicitOps), DI(SrcImmByte, intn),
3846 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3848 G(Src2One | ByteOp, group2), G(Src2One, group2),
3849 G(Src2CL | ByteOp, group2), G(Src2CL, group2),
3850 I(DstAcc | SrcImmUByte | No64, em_aam),
3851 I(DstAcc | SrcImmUByte | No64, em_aad),
3852 F(DstAcc | ByteOp | No64, em_salc),
3853 I(DstAcc | SrcXLat | ByteOp, em_mov),
3855 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
3857 X3(I(SrcImmByte, em_loop)),
3858 I(SrcImmByte, em_jcxz),
3859 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
3860 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
3862 I(SrcImm | Stack, em_call), D(SrcImm | ImplicitOps),
3863 I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
3864 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
3865 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
3867 N, DI(ImplicitOps, icebp), N, N,
3868 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
3869 G(ByteOp, group3), G(0, group3),
3871 D(ImplicitOps), D(ImplicitOps),
3872 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
3873 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3876 static const struct opcode twobyte_table[256] = {
3878 G(0, group6), GD(0, &group7), N, N,
3879 N, I(ImplicitOps | EmulateOnUD, em_syscall),
3880 II(ImplicitOps | Priv, em_clts, clts), N,
3881 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3882 N, D(ImplicitOps | ModRM), N, N,
3884 N, N, N, N, N, N, N, N,
3885 D(ImplicitOps | ModRM), N, N, N, N, N, N, D(ImplicitOps | ModRM),
3887 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, cr_read, check_cr_read),
3888 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, dr_read, check_dr_read),
3889 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_cr_write, cr_write,
3891 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_dr_write, dr_write,
3894 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_28_0f_29),
3895 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_28_0f_29),
3896 N, GP(ModRM | DstMem | SrcReg | Sse | Mov | Aligned, &pfx_vmovntpx),
3899 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
3900 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
3901 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
3902 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
3903 I(ImplicitOps | EmulateOnUD, em_sysenter),
3904 I(ImplicitOps | Priv | EmulateOnUD, em_sysexit),
3906 N, N, N, N, N, N, N, N,
3908 X16(D(DstReg | SrcMem | ModRM | Mov)),
3910 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3915 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3920 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3924 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3926 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
3927 II(ImplicitOps, em_cpuid, cpuid),
3928 F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
3929 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
3930 F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
3932 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
3933 DI(ImplicitOps, rsm),
3934 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
3935 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
3936 F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
3937 D(ModRM), F(DstReg | SrcMem | ModRM, em_imul),
3939 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_cmpxchg),
3940 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
3941 F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
3942 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
3943 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
3944 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3948 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
3949 F(DstReg | SrcMem | ModRM, em_bsf), F(DstReg | SrcMem | ModRM, em_bsr),
3950 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3952 F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
3953 N, D(DstMem | SrcReg | ModRM | Mov),
3954 N, N, N, GD(0, &group9),
3956 X8(I(DstReg, em_bswap)),
3958 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3960 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3962 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
3965 static const struct gprefix three_byte_0f_38_f0 = {
3966 I(DstReg | SrcMem | Mov, em_movbe), N, N, N
3969 static const struct gprefix three_byte_0f_38_f1 = {
3970 I(DstMem | SrcReg | Mov, em_movbe), N, N, N
3974 * Insns below are selected by the prefix which indexed by the third opcode
3977 static const struct opcode opcode_map_0f_38[256] = {
3979 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
3981 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
3983 GP(EmulateOnUD | ModRM | Prefix, &three_byte_0f_38_f0),
3984 GP(EmulateOnUD | ModRM | Prefix, &three_byte_0f_38_f1),
4003 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4007 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4013 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
4014 unsigned size, bool sign_extension)
4016 int rc = X86EMUL_CONTINUE;
4020 op->addr.mem.ea = ctxt->_eip;
4021 /* NB. Immediates are sign-extended as necessary. */
4022 switch (op->bytes) {
4024 op->val = insn_fetch(s8, ctxt);
4027 op->val = insn_fetch(s16, ctxt);
4030 op->val = insn_fetch(s32, ctxt);
4033 op->val = insn_fetch(s64, ctxt);
4036 if (!sign_extension) {
4037 switch (op->bytes) {
4045 op->val &= 0xffffffff;
4053 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
4056 int rc = X86EMUL_CONTINUE;
4060 decode_register_operand(ctxt, op);
4063 rc = decode_imm(ctxt, op, 1, false);
4066 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4070 if ((ctxt->d & BitOp) && op == &ctxt->dst)
4071 fetch_bit_operand(ctxt);
4072 op->orig_val = op->val;
4075 ctxt->memop.bytes = 8;
4079 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4080 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4081 fetch_register_operand(op);
4082 op->orig_val = op->val;
4086 op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;
4087 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4088 fetch_register_operand(op);
4089 op->orig_val = op->val;
4092 if (ctxt->d & ByteOp) {
4097 op->bytes = ctxt->op_bytes;
4098 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4099 fetch_register_operand(op);
4100 op->orig_val = op->val;
4104 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4106 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RDI));
4107 op->addr.mem.seg = VCPU_SREG_ES;
4114 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4115 fetch_register_operand(op);
4119 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4122 rc = decode_imm(ctxt, op, 1, true);
4129 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
4132 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
4135 ctxt->memop.bytes = 1;
4136 if (ctxt->memop.type == OP_REG) {
4137 ctxt->memop.addr.reg = decode_register(ctxt,
4138 ctxt->modrm_rm, true);
4139 fetch_register_operand(&ctxt->memop);
4143 ctxt->memop.bytes = 2;
4146 ctxt->memop.bytes = 4;
4149 rc = decode_imm(ctxt, op, 2, false);
4152 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
4156 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4158 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RSI));
4159 op->addr.mem.seg = seg_override(ctxt);
4165 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4167 register_address(ctxt,
4168 reg_read(ctxt, VCPU_REGS_RBX) +
4169 (reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
4170 op->addr.mem.seg = seg_override(ctxt);
4175 op->addr.mem.ea = ctxt->_eip;
4176 op->bytes = ctxt->op_bytes + 2;
4177 insn_fetch_arr(op->valptr, op->bytes, ctxt);
4180 ctxt->memop.bytes = ctxt->op_bytes + 2;
4183 op->val = VCPU_SREG_ES;
4186 op->val = VCPU_SREG_CS;
4189 op->val = VCPU_SREG_SS;
4192 op->val = VCPU_SREG_DS;
4195 op->val = VCPU_SREG_FS;
4198 op->val = VCPU_SREG_GS;
4201 /* Special instructions do their own operand decoding. */
4203 op->type = OP_NONE; /* Disable writeback. */
4211 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
4213 int rc = X86EMUL_CONTINUE;
4214 int mode = ctxt->mode;
4215 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
4216 bool op_prefix = false;
4217 struct opcode opcode;
4219 ctxt->memop.type = OP_NONE;
4220 ctxt->memopp = NULL;
4221 ctxt->_eip = ctxt->eip;
4222 ctxt->fetch.start = ctxt->_eip;
4223 ctxt->fetch.end = ctxt->fetch.start + insn_len;
4224 ctxt->opcode_len = 1;
4226 memcpy(ctxt->fetch.data, insn, insn_len);
4229 case X86EMUL_MODE_REAL:
4230 case X86EMUL_MODE_VM86:
4231 case X86EMUL_MODE_PROT16:
4232 def_op_bytes = def_ad_bytes = 2;
4234 case X86EMUL_MODE_PROT32:
4235 def_op_bytes = def_ad_bytes = 4;
4237 #ifdef CONFIG_X86_64
4238 case X86EMUL_MODE_PROT64:
4244 return EMULATION_FAILED;
4247 ctxt->op_bytes = def_op_bytes;
4248 ctxt->ad_bytes = def_ad_bytes;
4250 /* Legacy prefixes. */
4252 switch (ctxt->b = insn_fetch(u8, ctxt)) {
4253 case 0x66: /* operand-size override */
4255 /* switch between 2/4 bytes */
4256 ctxt->op_bytes = def_op_bytes ^ 6;
4258 case 0x67: /* address-size override */
4259 if (mode == X86EMUL_MODE_PROT64)
4260 /* switch between 4/8 bytes */
4261 ctxt->ad_bytes = def_ad_bytes ^ 12;
4263 /* switch between 2/4 bytes */
4264 ctxt->ad_bytes = def_ad_bytes ^ 6;
4266 case 0x26: /* ES override */
4267 case 0x2e: /* CS override */
4268 case 0x36: /* SS override */
4269 case 0x3e: /* DS override */
4270 set_seg_override(ctxt, (ctxt->b >> 3) & 3);
4272 case 0x64: /* FS override */
4273 case 0x65: /* GS override */
4274 set_seg_override(ctxt, ctxt->b & 7);
4276 case 0x40 ... 0x4f: /* REX */
4277 if (mode != X86EMUL_MODE_PROT64)
4279 ctxt->rex_prefix = ctxt->b;
4281 case 0xf0: /* LOCK */
4282 ctxt->lock_prefix = 1;
4284 case 0xf2: /* REPNE/REPNZ */
4285 case 0xf3: /* REP/REPE/REPZ */
4286 ctxt->rep_prefix = ctxt->b;
4292 /* Any legacy prefix after a REX prefix nullifies its effect. */
4294 ctxt->rex_prefix = 0;
4300 if (ctxt->rex_prefix & 8)
4301 ctxt->op_bytes = 8; /* REX.W */
4303 /* Opcode byte(s). */
4304 opcode = opcode_table[ctxt->b];
4305 /* Two-byte opcode? */
4306 if (ctxt->b == 0x0f) {
4307 ctxt->opcode_len = 2;
4308 ctxt->b = insn_fetch(u8, ctxt);
4309 opcode = twobyte_table[ctxt->b];
4311 /* 0F_38 opcode map */
4312 if (ctxt->b == 0x38) {
4313 ctxt->opcode_len = 3;
4314 ctxt->b = insn_fetch(u8, ctxt);
4315 opcode = opcode_map_0f_38[ctxt->b];
4318 ctxt->d = opcode.flags;
4320 if (ctxt->d & ModRM)
4321 ctxt->modrm = insn_fetch(u8, ctxt);
4323 /* vex-prefix instructions are not implemented */
4324 if (ctxt->opcode_len == 1 && (ctxt->b == 0xc5 || ctxt->b == 0xc4) &&
4325 (mode == X86EMUL_MODE_PROT64 ||
4326 (mode >= X86EMUL_MODE_PROT16 && (ctxt->modrm & 0x80)))) {
4330 while (ctxt->d & GroupMask) {
4331 switch (ctxt->d & GroupMask) {
4333 goffset = (ctxt->modrm >> 3) & 7;
4334 opcode = opcode.u.group[goffset];
4337 goffset = (ctxt->modrm >> 3) & 7;
4338 if ((ctxt->modrm >> 6) == 3)
4339 opcode = opcode.u.gdual->mod3[goffset];
4341 opcode = opcode.u.gdual->mod012[goffset];
4344 goffset = ctxt->modrm & 7;
4345 opcode = opcode.u.group[goffset];
4348 if (ctxt->rep_prefix && op_prefix)
4349 return EMULATION_FAILED;
4350 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
4351 switch (simd_prefix) {
4352 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
4353 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
4354 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
4355 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
4359 if (ctxt->modrm > 0xbf)
4360 opcode = opcode.u.esc->high[ctxt->modrm - 0xc0];
4362 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
4365 return EMULATION_FAILED;
4368 ctxt->d &= ~(u64)GroupMask;
4369 ctxt->d |= opcode.flags;
4372 ctxt->execute = opcode.u.execute;
4373 ctxt->check_perm = opcode.check_perm;
4374 ctxt->intercept = opcode.intercept;
4377 if (ctxt->d == 0 || (ctxt->d & NotImpl))
4378 return EMULATION_FAILED;
4380 if (!(ctxt->d & EmulateOnUD) && ctxt->ud)
4381 return EMULATION_FAILED;
4383 if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
4386 if (ctxt->d & Op3264) {
4387 if (mode == X86EMUL_MODE_PROT64)
4394 ctxt->op_bytes = 16;
4395 else if (ctxt->d & Mmx)
4398 /* ModRM and SIB bytes. */
4399 if (ctxt->d & ModRM) {
4400 rc = decode_modrm(ctxt, &ctxt->memop);
4401 if (!ctxt->has_seg_override)
4402 set_seg_override(ctxt, ctxt->modrm_seg);
4403 } else if (ctxt->d & MemAbs)
4404 rc = decode_abs(ctxt, &ctxt->memop);
4405 if (rc != X86EMUL_CONTINUE)
4408 if (!ctxt->has_seg_override)
4409 set_seg_override(ctxt, VCPU_SREG_DS);
4411 ctxt->memop.addr.mem.seg = seg_override(ctxt);
4413 if (ctxt->memop.type == OP_MEM && ctxt->ad_bytes != 8)
4414 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
4417 * Decode and fetch the source operand: register, memory
4420 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
4421 if (rc != X86EMUL_CONTINUE)
4425 * Decode and fetch the second source operand: register, memory
4428 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
4429 if (rc != X86EMUL_CONTINUE)
4432 /* Decode and fetch the destination operand: register or memory. */
4433 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
4436 if (ctxt->memopp && ctxt->memopp->type == OP_MEM && ctxt->rip_relative)
4437 ctxt->memopp->addr.mem.ea += ctxt->_eip;
4439 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
4442 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
4444 return ctxt->d & PageTable;
4447 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
4449 /* The second termination condition only applies for REPE
4450 * and REPNE. Test if the repeat string operation prefix is
4451 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
4452 * corresponding termination condition according to:
4453 * - if REPE/REPZ and ZF = 0 then done
4454 * - if REPNE/REPNZ and ZF = 1 then done
4456 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
4457 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
4458 && (((ctxt->rep_prefix == REPE_PREFIX) &&
4459 ((ctxt->eflags & EFLG_ZF) == 0))
4460 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
4461 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
4467 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
4471 ctxt->ops->get_fpu(ctxt);
4472 asm volatile("1: fwait \n\t"
4474 ".pushsection .fixup,\"ax\" \n\t"
4476 "movb $1, %[fault] \n\t"
4479 _ASM_EXTABLE(1b, 3b)
4480 : [fault]"+qm"(fault));
4481 ctxt->ops->put_fpu(ctxt);
4483 if (unlikely(fault))
4484 return emulate_exception(ctxt, MF_VECTOR, 0, false);
4486 return X86EMUL_CONTINUE;
4489 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
4492 if (op->type == OP_MM)
4493 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
4496 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
4498 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
4499 if (!(ctxt->d & ByteOp))
4500 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
4501 asm("push %[flags]; popf; call *%[fastop]; pushf; pop %[flags]\n"
4502 : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
4504 : "c"(ctxt->src2.val));
4505 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
4506 if (!fop) /* exception is returned in fop variable */
4507 return emulate_de(ctxt);
4508 return X86EMUL_CONTINUE;
4511 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
4513 const struct x86_emulate_ops *ops = ctxt->ops;
4514 int rc = X86EMUL_CONTINUE;
4515 int saved_dst_type = ctxt->dst.type;
4517 ctxt->mem_read.pos = 0;
4519 if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
4520 (ctxt->d & Undefined)) {
4521 rc = emulate_ud(ctxt);
4525 /* LOCK prefix is allowed only with some instructions */
4526 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
4527 rc = emulate_ud(ctxt);
4531 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
4532 rc = emulate_ud(ctxt);
4536 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
4537 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
4538 rc = emulate_ud(ctxt);
4542 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
4543 rc = emulate_nm(ctxt);
4547 if (ctxt->d & Mmx) {
4548 rc = flush_pending_x87_faults(ctxt);
4549 if (rc != X86EMUL_CONTINUE)
4552 * Now that we know the fpu is exception safe, we can fetch
4555 fetch_possible_mmx_operand(ctxt, &ctxt->src);
4556 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
4557 if (!(ctxt->d & Mov))
4558 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
4561 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4562 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4563 X86_ICPT_PRE_EXCEPT);
4564 if (rc != X86EMUL_CONTINUE)
4568 /* Privileged instruction can be executed only in CPL=0 */
4569 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
4570 rc = emulate_gp(ctxt, 0);
4574 /* Instruction can only be executed in protected mode */
4575 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
4576 rc = emulate_ud(ctxt);
4580 /* Do instruction specific permission checks */
4581 if (ctxt->check_perm) {
4582 rc = ctxt->check_perm(ctxt);
4583 if (rc != X86EMUL_CONTINUE)
4587 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4588 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4589 X86_ICPT_POST_EXCEPT);
4590 if (rc != X86EMUL_CONTINUE)
4594 if (ctxt->rep_prefix && (ctxt->d & String)) {
4595 /* All REP prefixes have the same first termination condition */
4596 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
4597 ctxt->eip = ctxt->_eip;
4602 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
4603 rc = segmented_read(ctxt, ctxt->src.addr.mem,
4604 ctxt->src.valptr, ctxt->src.bytes);
4605 if (rc != X86EMUL_CONTINUE)
4607 ctxt->src.orig_val64 = ctxt->src.val64;
4610 if (ctxt->src2.type == OP_MEM) {
4611 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
4612 &ctxt->src2.val, ctxt->src2.bytes);
4613 if (rc != X86EMUL_CONTINUE)
4617 if ((ctxt->d & DstMask) == ImplicitOps)
4621 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
4622 /* optimisation - avoid slow emulated read if Mov */
4623 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
4624 &ctxt->dst.val, ctxt->dst.bytes);
4625 if (rc != X86EMUL_CONTINUE)
4628 ctxt->dst.orig_val = ctxt->dst.val;
4632 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4633 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4634 X86_ICPT_POST_MEMACCESS);
4635 if (rc != X86EMUL_CONTINUE)
4639 if (ctxt->execute) {
4640 if (ctxt->d & Fastop) {
4641 void (*fop)(struct fastop *) = (void *)ctxt->execute;
4642 rc = fastop(ctxt, fop);
4643 if (rc != X86EMUL_CONTINUE)
4647 rc = ctxt->execute(ctxt);
4648 if (rc != X86EMUL_CONTINUE)
4653 if (ctxt->opcode_len == 2)
4655 else if (ctxt->opcode_len == 3)
4656 goto threebyte_insn;
4659 case 0x63: /* movsxd */
4660 if (ctxt->mode != X86EMUL_MODE_PROT64)
4661 goto cannot_emulate;
4662 ctxt->dst.val = (s32) ctxt->src.val;
4664 case 0x70 ... 0x7f: /* jcc (short) */
4665 if (test_cc(ctxt->b, ctxt->eflags))
4666 jmp_rel(ctxt, ctxt->src.val);
4668 case 0x8d: /* lea r16/r32, m */
4669 ctxt->dst.val = ctxt->src.addr.mem.ea;
4671 case 0x90 ... 0x97: /* nop / xchg reg, rax */
4672 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
4676 case 0x98: /* cbw/cwde/cdqe */
4677 switch (ctxt->op_bytes) {
4678 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
4679 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
4680 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
4683 case 0xcc: /* int3 */
4684 rc = emulate_int(ctxt, 3);
4686 case 0xcd: /* int n */
4687 rc = emulate_int(ctxt, ctxt->src.val);
4689 case 0xce: /* into */
4690 if (ctxt->eflags & EFLG_OF)
4691 rc = emulate_int(ctxt, 4);
4693 case 0xe9: /* jmp rel */
4694 case 0xeb: /* jmp rel short */
4695 jmp_rel(ctxt, ctxt->src.val);
4696 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4698 case 0xf4: /* hlt */
4699 ctxt->ops->halt(ctxt);
4701 case 0xf5: /* cmc */
4702 /* complement carry flag from eflags reg */
4703 ctxt->eflags ^= EFLG_CF;
4705 case 0xf8: /* clc */
4706 ctxt->eflags &= ~EFLG_CF;
4708 case 0xf9: /* stc */
4709 ctxt->eflags |= EFLG_CF;
4711 case 0xfc: /* cld */
4712 ctxt->eflags &= ~EFLG_DF;
4714 case 0xfd: /* std */
4715 ctxt->eflags |= EFLG_DF;
4718 goto cannot_emulate;
4721 if (rc != X86EMUL_CONTINUE)
4725 if (!(ctxt->d & NoWrite)) {
4726 rc = writeback(ctxt, &ctxt->dst);
4727 if (rc != X86EMUL_CONTINUE)
4730 if (ctxt->d & SrcWrite) {
4731 BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);
4732 rc = writeback(ctxt, &ctxt->src);
4733 if (rc != X86EMUL_CONTINUE)
4738 * restore dst type in case the decoding will be reused
4739 * (happens for string instruction )
4741 ctxt->dst.type = saved_dst_type;
4743 if ((ctxt->d & SrcMask) == SrcSI)
4744 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
4746 if ((ctxt->d & DstMask) == DstDI)
4747 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
4749 if (ctxt->rep_prefix && (ctxt->d & String)) {
4751 struct read_cache *r = &ctxt->io_read;
4752 if ((ctxt->d & SrcMask) == SrcSI)
4753 count = ctxt->src.count;
4755 count = ctxt->dst.count;
4756 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX),
4759 if (!string_insn_completed(ctxt)) {
4761 * Re-enter guest when pio read ahead buffer is empty
4762 * or, if it is not used, after each 1024 iteration.
4764 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
4765 (r->end == 0 || r->end != r->pos)) {
4767 * Reset read cache. Usually happens before
4768 * decode, but since instruction is restarted
4769 * we have to do it here.
4771 ctxt->mem_read.end = 0;
4772 writeback_registers(ctxt);
4773 return EMULATION_RESTART;
4775 goto done; /* skip rip writeback */
4779 ctxt->eip = ctxt->_eip;
4782 if (rc == X86EMUL_PROPAGATE_FAULT)
4783 ctxt->have_exception = true;
4784 if (rc == X86EMUL_INTERCEPTED)
4785 return EMULATION_INTERCEPTED;
4787 if (rc == X86EMUL_CONTINUE)
4788 writeback_registers(ctxt);
4790 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4794 case 0x09: /* wbinvd */
4795 (ctxt->ops->wbinvd)(ctxt);
4797 case 0x08: /* invd */
4798 case 0x0d: /* GrpP (prefetch) */
4799 case 0x18: /* Grp16 (prefetch/nop) */
4800 case 0x1f: /* nop */
4802 case 0x20: /* mov cr, reg */
4803 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
4805 case 0x21: /* mov from dr to reg */
4806 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
4808 case 0x40 ... 0x4f: /* cmov */
4809 ctxt->dst.val = ctxt->dst.orig_val = ctxt->src.val;
4810 if (!test_cc(ctxt->b, ctxt->eflags))
4811 ctxt->dst.type = OP_NONE; /* no writeback */
4813 case 0x80 ... 0x8f: /* jnz rel, etc*/
4814 if (test_cc(ctxt->b, ctxt->eflags))
4815 jmp_rel(ctxt, ctxt->src.val);
4817 case 0x90 ... 0x9f: /* setcc r/m8 */
4818 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
4820 case 0xae: /* clflush */
4822 case 0xb6 ... 0xb7: /* movzx */
4823 ctxt->dst.bytes = ctxt->op_bytes;
4824 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
4825 : (u16) ctxt->src.val;
4827 case 0xbe ... 0xbf: /* movsx */
4828 ctxt->dst.bytes = ctxt->op_bytes;
4829 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
4830 (s16) ctxt->src.val;
4832 case 0xc3: /* movnti */
4833 ctxt->dst.bytes = ctxt->op_bytes;
4834 ctxt->dst.val = (ctxt->op_bytes == 4) ? (u32) ctxt->src.val :
4835 (u64) ctxt->src.val;
4838 goto cannot_emulate;
4843 if (rc != X86EMUL_CONTINUE)
4849 return EMULATION_FAILED;
4852 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
4854 invalidate_registers(ctxt);
4857 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
4859 writeback_registers(ctxt);