1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
32 * Opcode effective-address decode tables.
33 * Note that we only emulate instructions that have at least one memory
34 * operand (excluding implicit stack references). We assume that stack
35 * references and instruction fetches will never occur in special memory
36 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
40 /* Operand sizes: 8-bit operands or specified/overridden size. */
41 #define ByteOp (1<<0) /* 8-bit operands. */
42 /* Destination operand type. */
43 #define ImplicitOps (1<<1) /* Implicit in opcode. No generic decode. */
44 #define DstReg (2<<1) /* Register operand. */
45 #define DstMem (3<<1) /* Memory operand. */
46 #define DstAcc (4<<1) /* Destination Accumulator */
47 #define DstDI (5<<1) /* Destination is in ES:(E)DI */
48 #define DstMem64 (6<<1) /* 64bit memory operand */
49 #define DstImmUByte (7<<1) /* 8-bit unsigned immediate operand */
50 #define DstMask (7<<1)
51 /* Source operand type. */
52 #define SrcNone (0<<4) /* No source operand. */
53 #define SrcReg (1<<4) /* Register operand. */
54 #define SrcMem (2<<4) /* Memory operand. */
55 #define SrcMem16 (3<<4) /* Memory operand (16-bit). */
56 #define SrcMem32 (4<<4) /* Memory operand (32-bit). */
57 #define SrcImm (5<<4) /* Immediate operand. */
58 #define SrcImmByte (6<<4) /* 8-bit sign-extended immediate operand. */
59 #define SrcOne (7<<4) /* Implied '1' */
60 #define SrcImmUByte (8<<4) /* 8-bit unsigned immediate operand. */
61 #define SrcImmU (9<<4) /* Immediate operand, unsigned */
62 #define SrcSI (0xa<<4) /* Source is in the DS:RSI */
63 #define SrcImmFAddr (0xb<<4) /* Source is immediate far address */
64 #define SrcMemFAddr (0xc<<4) /* Source is far address in memory */
65 #define SrcAcc (0xd<<4) /* Source Accumulator */
66 #define SrcImmU16 (0xe<<4) /* Immediate operand, unsigned, 16 bits */
67 #define SrcMask (0xf<<4)
68 /* Generic ModRM decode. */
70 /* Destination is only written; never read. */
73 #define MemAbs (1<<11) /* Memory operand is absolute displacement */
74 #define String (1<<12) /* String instruction (rep capable) */
75 #define Stack (1<<13) /* Stack instruction (push/pop) */
76 #define Group (1<<14) /* Bits 3:5 of modrm byte extend opcode */
77 #define GroupDual (1<<15) /* Alternate decoding of mod == 3 */
78 #define Prefix (1<<16) /* Instruction varies with 66/f2/f3 prefix */
79 #define Sse (1<<17) /* SSE Vector instruction */
80 #define RMExt (1<<18) /* Opcode extension in ModRM r/m if mod == 3 */
82 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
83 #define VendorSpecific (1<<22) /* Vendor specific instruction */
84 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
85 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
86 #define Undefined (1<<25) /* No Such Instruction */
87 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
88 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
90 /* Source 2 operand type */
91 #define Src2None (0<<29)
92 #define Src2CL (1<<29)
93 #define Src2ImmByte (2<<29)
94 #define Src2One (3<<29)
95 #define Src2Imm (4<<29)
96 #define Src2Mask (7<<29)
99 #define X3(x...) X2(x), x
100 #define X4(x...) X2(x), X2(x)
101 #define X5(x...) X4(x), x
102 #define X6(x...) X4(x), X2(x)
103 #define X7(x...) X4(x), X3(x)
104 #define X8(x...) X4(x), X4(x)
105 #define X16(x...) X8(x), X8(x)
111 int (*execute)(struct x86_emulate_ctxt *ctxt);
112 struct opcode *group;
113 struct group_dual *gdual;
114 struct gprefix *gprefix;
116 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
120 struct opcode mod012[8];
121 struct opcode mod3[8];
125 struct opcode pfx_no;
126 struct opcode pfx_66;
127 struct opcode pfx_f2;
128 struct opcode pfx_f3;
131 /* EFLAGS bit definitions. */
132 #define EFLG_ID (1<<21)
133 #define EFLG_VIP (1<<20)
134 #define EFLG_VIF (1<<19)
135 #define EFLG_AC (1<<18)
136 #define EFLG_VM (1<<17)
137 #define EFLG_RF (1<<16)
138 #define EFLG_IOPL (3<<12)
139 #define EFLG_NT (1<<14)
140 #define EFLG_OF (1<<11)
141 #define EFLG_DF (1<<10)
142 #define EFLG_IF (1<<9)
143 #define EFLG_TF (1<<8)
144 #define EFLG_SF (1<<7)
145 #define EFLG_ZF (1<<6)
146 #define EFLG_AF (1<<4)
147 #define EFLG_PF (1<<2)
148 #define EFLG_CF (1<<0)
150 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
151 #define EFLG_RESERVED_ONE_MASK 2
154 * Instruction emulation:
155 * Most instructions are emulated directly via a fragment of inline assembly
156 * code. This allows us to save/restore EFLAGS and thus very easily pick up
157 * any modified flags.
160 #if defined(CONFIG_X86_64)
161 #define _LO32 "k" /* force 32-bit operand */
162 #define _STK "%%rsp" /* stack pointer */
163 #elif defined(__i386__)
164 #define _LO32 "" /* force 32-bit operand */
165 #define _STK "%%esp" /* stack pointer */
169 * These EFLAGS bits are restored from saved value during emulation, and
170 * any changes are written back to the saved value after emulation.
172 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
174 /* Before executing instruction: restore necessary bits in EFLAGS. */
175 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
176 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
177 "movl %"_sav",%"_LO32 _tmp"; " \
180 "movl %"_msk",%"_LO32 _tmp"; " \
181 "andl %"_LO32 _tmp",("_STK"); " \
183 "notl %"_LO32 _tmp"; " \
184 "andl %"_LO32 _tmp",("_STK"); " \
185 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
187 "orl %"_LO32 _tmp",("_STK"); " \
191 /* After executing instruction: write-back necessary bits in EFLAGS. */
192 #define _POST_EFLAGS(_sav, _msk, _tmp) \
193 /* _sav |= EFLAGS & _msk; */ \
196 "andl %"_msk",%"_LO32 _tmp"; " \
197 "orl %"_LO32 _tmp",%"_sav"; "
205 #define ____emulate_2op(_op, _src, _dst, _eflags, _x, _y, _suffix, _dsttype) \
207 __asm__ __volatile__ ( \
208 _PRE_EFLAGS("0", "4", "2") \
209 _op _suffix " %"_x"3,%1; " \
210 _POST_EFLAGS("0", "4", "2") \
211 : "=m" (_eflags), "+q" (*(_dsttype*)&(_dst).val),\
213 : _y ((_src).val), "i" (EFLAGS_MASK)); \
217 /* Raw emulation: instruction has two explicit operands. */
218 #define __emulate_2op_nobyte(_op,_src,_dst,_eflags,_wx,_wy,_lx,_ly,_qx,_qy) \
220 unsigned long _tmp; \
222 switch ((_dst).bytes) { \
224 ____emulate_2op(_op,_src,_dst,_eflags,_wx,_wy,"w",u16);\
227 ____emulate_2op(_op,_src,_dst,_eflags,_lx,_ly,"l",u32);\
230 ON64(____emulate_2op(_op,_src,_dst,_eflags,_qx,_qy,"q",u64)); \
235 #define __emulate_2op(_op,_src,_dst,_eflags,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
237 unsigned long _tmp; \
238 switch ((_dst).bytes) { \
240 ____emulate_2op(_op,_src,_dst,_eflags,_bx,_by,"b",u8); \
243 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
244 _wx, _wy, _lx, _ly, _qx, _qy); \
249 /* Source operand is byte-sized and may be restricted to just %cl. */
250 #define emulate_2op_SrcB(_op, _src, _dst, _eflags) \
251 __emulate_2op(_op, _src, _dst, _eflags, \
252 "b", "c", "b", "c", "b", "c", "b", "c")
254 /* Source operand is byte, word, long or quad sized. */
255 #define emulate_2op_SrcV(_op, _src, _dst, _eflags) \
256 __emulate_2op(_op, _src, _dst, _eflags, \
257 "b", "q", "w", "r", _LO32, "r", "", "r")
259 /* Source operand is word, long or quad sized. */
260 #define emulate_2op_SrcV_nobyte(_op, _src, _dst, _eflags) \
261 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
262 "w", "r", _LO32, "r", "", "r")
264 /* Instruction has three operands and one operand is stored in ECX register */
265 #define __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, _suffix, _type) \
267 unsigned long _tmp; \
268 _type _clv = (_cl).val; \
269 _type _srcv = (_src).val; \
270 _type _dstv = (_dst).val; \
272 __asm__ __volatile__ ( \
273 _PRE_EFLAGS("0", "5", "2") \
274 _op _suffix " %4,%1 \n" \
275 _POST_EFLAGS("0", "5", "2") \
276 : "=m" (_eflags), "+r" (_dstv), "=&r" (_tmp) \
277 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
280 (_cl).val = (unsigned long) _clv; \
281 (_src).val = (unsigned long) _srcv; \
282 (_dst).val = (unsigned long) _dstv; \
285 #define emulate_2op_cl(_op, _cl, _src, _dst, _eflags) \
287 switch ((_dst).bytes) { \
289 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
290 "w", unsigned short); \
293 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
294 "l", unsigned int); \
297 ON64(__emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
298 "q", unsigned long)); \
303 #define __emulate_1op(_op, _dst, _eflags, _suffix) \
305 unsigned long _tmp; \
307 __asm__ __volatile__ ( \
308 _PRE_EFLAGS("0", "3", "2") \
309 _op _suffix " %1; " \
310 _POST_EFLAGS("0", "3", "2") \
311 : "=m" (_eflags), "+m" ((_dst).val), \
313 : "i" (EFLAGS_MASK)); \
316 /* Instruction has only one explicit operand (no source operand). */
317 #define emulate_1op(_op, _dst, _eflags) \
319 switch ((_dst).bytes) { \
320 case 1: __emulate_1op(_op, _dst, _eflags, "b"); break; \
321 case 2: __emulate_1op(_op, _dst, _eflags, "w"); break; \
322 case 4: __emulate_1op(_op, _dst, _eflags, "l"); break; \
323 case 8: ON64(__emulate_1op(_op, _dst, _eflags, "q")); break; \
327 #define __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, _suffix) \
329 unsigned long _tmp; \
331 __asm__ __volatile__ ( \
332 _PRE_EFLAGS("0", "4", "1") \
333 _op _suffix " %5; " \
334 _POST_EFLAGS("0", "4", "1") \
335 : "=m" (_eflags), "=&r" (_tmp), \
336 "+a" (_rax), "+d" (_rdx) \
337 : "i" (EFLAGS_MASK), "m" ((_src).val), \
338 "a" (_rax), "d" (_rdx)); \
341 #define __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, _eflags, _suffix, _ex) \
343 unsigned long _tmp; \
345 __asm__ __volatile__ ( \
346 _PRE_EFLAGS("0", "5", "1") \
348 _op _suffix " %6; " \
350 _POST_EFLAGS("0", "5", "1") \
351 ".pushsection .fixup,\"ax\" \n\t" \
352 "3: movb $1, %4 \n\t" \
355 _ASM_EXTABLE(1b, 3b) \
356 : "=m" (_eflags), "=&r" (_tmp), \
357 "+a" (_rax), "+d" (_rdx), "+qm"(_ex) \
358 : "i" (EFLAGS_MASK), "m" ((_src).val), \
359 "a" (_rax), "d" (_rdx)); \
362 /* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
363 #define emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags) \
365 switch((_src).bytes) { \
366 case 1: __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, "b"); break; \
367 case 2: __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, "w"); break; \
368 case 4: __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, "l"); break; \
369 case 8: ON64(__emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, "q")); break; \
373 #define emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, _eflags, _ex) \
375 switch((_src).bytes) { \
377 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
378 _eflags, "b", _ex); \
381 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
382 _eflags, "w", _ex); \
385 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
386 _eflags, "l", _ex); \
389 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
390 _eflags, "q", _ex)); \
395 /* Fetch next part of the instruction being emulated. */
396 #define insn_fetch(_type, _size, _eip) \
397 ({ unsigned long _x; \
398 rc = do_insn_fetch(ctxt, ops, (_eip), &_x, (_size)); \
399 if (rc != X86EMUL_CONTINUE) \
405 #define insn_fetch_arr(_arr, _size, _eip) \
406 ({ rc = do_insn_fetch(ctxt, ops, (_eip), _arr, (_size)); \
407 if (rc != X86EMUL_CONTINUE) \
412 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
413 enum x86_intercept intercept,
414 enum x86_intercept_stage stage)
416 struct x86_instruction_info info = {
417 .intercept = intercept,
418 .rep_prefix = ctxt->decode.rep_prefix,
419 .modrm_mod = ctxt->decode.modrm_mod,
420 .modrm_reg = ctxt->decode.modrm_reg,
421 .modrm_rm = ctxt->decode.modrm_rm,
422 .src_val = ctxt->decode.src.val64,
423 .src_bytes = ctxt->decode.src.bytes,
424 .dst_bytes = ctxt->decode.dst.bytes,
425 .ad_bytes = ctxt->decode.ad_bytes,
426 .next_rip = ctxt->eip,
429 return ctxt->ops->intercept(ctxt->vcpu, &info, stage);
432 static inline unsigned long ad_mask(struct decode_cache *c)
434 return (1UL << (c->ad_bytes << 3)) - 1;
437 /* Access/update address held in a register, based on addressing mode. */
438 static inline unsigned long
439 address_mask(struct decode_cache *c, unsigned long reg)
441 if (c->ad_bytes == sizeof(unsigned long))
444 return reg & ad_mask(c);
447 static inline unsigned long
448 register_address(struct decode_cache *c, unsigned long reg)
450 return address_mask(c, reg);
454 register_address_increment(struct decode_cache *c, unsigned long *reg, int inc)
456 if (c->ad_bytes == sizeof(unsigned long))
459 *reg = (*reg & ~ad_mask(c)) | ((*reg + inc) & ad_mask(c));
462 static inline void jmp_rel(struct decode_cache *c, int rel)
464 register_address_increment(c, &c->eip, rel);
467 static u32 desc_limit_scaled(struct desc_struct *desc)
469 u32 limit = get_desc_limit(desc);
471 return desc->g ? (limit << 12) | 0xfff : limit;
474 static void set_seg_override(struct decode_cache *c, int seg)
476 c->has_seg_override = true;
477 c->seg_override = seg;
480 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt,
481 struct x86_emulate_ops *ops, int seg)
483 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
486 return ops->get_cached_segment_base(seg, ctxt->vcpu);
489 static unsigned seg_override(struct x86_emulate_ctxt *ctxt,
490 struct x86_emulate_ops *ops,
491 struct decode_cache *c)
493 if (!c->has_seg_override)
496 return c->seg_override;
499 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
500 u32 error, bool valid)
502 ctxt->exception.vector = vec;
503 ctxt->exception.error_code = error;
504 ctxt->exception.error_code_valid = valid;
505 return X86EMUL_PROPAGATE_FAULT;
508 static int emulate_db(struct x86_emulate_ctxt *ctxt)
510 return emulate_exception(ctxt, DB_VECTOR, 0, false);
513 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
515 return emulate_exception(ctxt, GP_VECTOR, err, true);
518 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
520 return emulate_exception(ctxt, SS_VECTOR, err, true);
523 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
525 return emulate_exception(ctxt, UD_VECTOR, 0, false);
528 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
530 return emulate_exception(ctxt, TS_VECTOR, err, true);
533 static int emulate_de(struct x86_emulate_ctxt *ctxt)
535 return emulate_exception(ctxt, DE_VECTOR, 0, false);
538 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
540 return emulate_exception(ctxt, NM_VECTOR, 0, false);
543 static int linearize(struct x86_emulate_ctxt *ctxt,
544 struct segmented_address addr,
545 unsigned size, bool write,
548 struct decode_cache *c = &ctxt->decode;
549 struct desc_struct desc;
555 la = seg_base(ctxt, ctxt->ops, addr.seg) + addr.ea;
556 switch (ctxt->mode) {
557 case X86EMUL_MODE_REAL:
559 case X86EMUL_MODE_PROT64:
560 if (((signed long)la << 16) >> 16 != la)
561 return emulate_gp(ctxt, 0);
564 usable = ctxt->ops->get_cached_descriptor(&desc, NULL, addr.seg,
568 /* code segment or read-only data segment */
569 if (((desc.type & 8) || !(desc.type & 2)) && write)
571 /* unreadable code segment */
572 if ((desc.type & 8) && !(desc.type & 2))
574 lim = desc_limit_scaled(&desc);
575 if ((desc.type & 8) || !(desc.type & 4)) {
576 /* expand-up segment */
577 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
580 /* exapand-down segment */
581 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
583 lim = desc.d ? 0xffffffff : 0xffff;
584 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
587 cpl = ctxt->ops->cpl(ctxt->vcpu);
588 rpl = ctxt->ops->get_segment_selector(addr.seg, ctxt->vcpu) & 3;
590 if (!(desc.type & 8)) {
594 } else if ((desc.type & 8) && !(desc.type & 4)) {
595 /* nonconforming code segment */
598 } else if ((desc.type & 8) && (desc.type & 4)) {
599 /* conforming code segment */
605 if (c->ad_bytes != 8)
608 return X86EMUL_CONTINUE;
610 if (addr.seg == VCPU_SREG_SS)
611 return emulate_ss(ctxt, addr.seg);
613 return emulate_gp(ctxt, addr.seg);
616 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
617 struct segmented_address addr,
624 rc = linearize(ctxt, addr, size, false, &linear);
625 if (rc != X86EMUL_CONTINUE)
627 return ctxt->ops->read_std(linear, data, size, ctxt->vcpu,
631 static int do_fetch_insn_byte(struct x86_emulate_ctxt *ctxt,
632 struct x86_emulate_ops *ops,
633 unsigned long eip, u8 *dest)
635 struct fetch_cache *fc = &ctxt->decode.fetch;
639 if (eip == fc->end) {
640 cur_size = fc->end - fc->start;
641 size = min(15UL - cur_size, PAGE_SIZE - offset_in_page(eip));
642 rc = ops->fetch(ctxt->cs_base + eip, fc->data + cur_size,
643 size, ctxt->vcpu, &ctxt->exception);
644 if (rc != X86EMUL_CONTINUE)
648 *dest = fc->data[eip - fc->start];
649 return X86EMUL_CONTINUE;
652 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
653 struct x86_emulate_ops *ops,
654 unsigned long eip, void *dest, unsigned size)
658 /* x86 instructions are limited to 15 bytes. */
659 if (eip + size - ctxt->eip > 15)
660 return X86EMUL_UNHANDLEABLE;
662 rc = do_fetch_insn_byte(ctxt, ops, eip++, dest++);
663 if (rc != X86EMUL_CONTINUE)
666 return X86EMUL_CONTINUE;
670 * Given the 'reg' portion of a ModRM byte, and a register block, return a
671 * pointer into the block that addresses the relevant register.
672 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
674 static void *decode_register(u8 modrm_reg, unsigned long *regs,
679 p = ®s[modrm_reg];
680 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
681 p = (unsigned char *)®s[modrm_reg & 3] + 1;
685 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
686 struct x86_emulate_ops *ops,
687 struct segmented_address addr,
688 u16 *size, unsigned long *address, int op_bytes)
695 rc = segmented_read_std(ctxt, addr, size, 2);
696 if (rc != X86EMUL_CONTINUE)
699 rc = segmented_read_std(ctxt, addr, address, op_bytes);
703 static int test_cc(unsigned int condition, unsigned int flags)
707 switch ((condition & 15) >> 1) {
709 rc |= (flags & EFLG_OF);
711 case 1: /* b/c/nae */
712 rc |= (flags & EFLG_CF);
715 rc |= (flags & EFLG_ZF);
718 rc |= (flags & (EFLG_CF|EFLG_ZF));
721 rc |= (flags & EFLG_SF);
724 rc |= (flags & EFLG_PF);
727 rc |= (flags & EFLG_ZF);
730 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
734 /* Odd condition identifiers (lsb == 1) have inverted sense. */
735 return (!!rc ^ (condition & 1));
738 static void fetch_register_operand(struct operand *op)
742 op->val = *(u8 *)op->addr.reg;
745 op->val = *(u16 *)op->addr.reg;
748 op->val = *(u32 *)op->addr.reg;
751 op->val = *(u64 *)op->addr.reg;
756 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
758 ctxt->ops->get_fpu(ctxt);
760 case 0: asm("movdqu %%xmm0, %0" : "=m"(*data)); break;
761 case 1: asm("movdqu %%xmm1, %0" : "=m"(*data)); break;
762 case 2: asm("movdqu %%xmm2, %0" : "=m"(*data)); break;
763 case 3: asm("movdqu %%xmm3, %0" : "=m"(*data)); break;
764 case 4: asm("movdqu %%xmm4, %0" : "=m"(*data)); break;
765 case 5: asm("movdqu %%xmm5, %0" : "=m"(*data)); break;
766 case 6: asm("movdqu %%xmm6, %0" : "=m"(*data)); break;
767 case 7: asm("movdqu %%xmm7, %0" : "=m"(*data)); break;
769 case 8: asm("movdqu %%xmm8, %0" : "=m"(*data)); break;
770 case 9: asm("movdqu %%xmm9, %0" : "=m"(*data)); break;
771 case 10: asm("movdqu %%xmm10, %0" : "=m"(*data)); break;
772 case 11: asm("movdqu %%xmm11, %0" : "=m"(*data)); break;
773 case 12: asm("movdqu %%xmm12, %0" : "=m"(*data)); break;
774 case 13: asm("movdqu %%xmm13, %0" : "=m"(*data)); break;
775 case 14: asm("movdqu %%xmm14, %0" : "=m"(*data)); break;
776 case 15: asm("movdqu %%xmm15, %0" : "=m"(*data)); break;
780 ctxt->ops->put_fpu(ctxt);
783 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
786 ctxt->ops->get_fpu(ctxt);
788 case 0: asm("movdqu %0, %%xmm0" : : "m"(*data)); break;
789 case 1: asm("movdqu %0, %%xmm1" : : "m"(*data)); break;
790 case 2: asm("movdqu %0, %%xmm2" : : "m"(*data)); break;
791 case 3: asm("movdqu %0, %%xmm3" : : "m"(*data)); break;
792 case 4: asm("movdqu %0, %%xmm4" : : "m"(*data)); break;
793 case 5: asm("movdqu %0, %%xmm5" : : "m"(*data)); break;
794 case 6: asm("movdqu %0, %%xmm6" : : "m"(*data)); break;
795 case 7: asm("movdqu %0, %%xmm7" : : "m"(*data)); break;
797 case 8: asm("movdqu %0, %%xmm8" : : "m"(*data)); break;
798 case 9: asm("movdqu %0, %%xmm9" : : "m"(*data)); break;
799 case 10: asm("movdqu %0, %%xmm10" : : "m"(*data)); break;
800 case 11: asm("movdqu %0, %%xmm11" : : "m"(*data)); break;
801 case 12: asm("movdqu %0, %%xmm12" : : "m"(*data)); break;
802 case 13: asm("movdqu %0, %%xmm13" : : "m"(*data)); break;
803 case 14: asm("movdqu %0, %%xmm14" : : "m"(*data)); break;
804 case 15: asm("movdqu %0, %%xmm15" : : "m"(*data)); break;
808 ctxt->ops->put_fpu(ctxt);
811 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
813 struct decode_cache *c,
816 unsigned reg = c->modrm_reg;
817 int highbyte_regs = c->rex_prefix == 0;
820 reg = (c->b & 7) | ((c->rex_prefix & 1) << 3);
826 read_sse_reg(ctxt, &op->vec_val, reg);
831 if ((c->d & ByteOp) && !inhibit_bytereg) {
832 op->addr.reg = decode_register(reg, c->regs, highbyte_regs);
835 op->addr.reg = decode_register(reg, c->regs, 0);
836 op->bytes = c->op_bytes;
838 fetch_register_operand(op);
839 op->orig_val = op->val;
842 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
843 struct x86_emulate_ops *ops,
846 struct decode_cache *c = &ctxt->decode;
848 int index_reg = 0, base_reg = 0, scale;
849 int rc = X86EMUL_CONTINUE;
853 c->modrm_reg = (c->rex_prefix & 4) << 1; /* REX.R */
854 index_reg = (c->rex_prefix & 2) << 2; /* REX.X */
855 c->modrm_rm = base_reg = (c->rex_prefix & 1) << 3; /* REG.B */
858 c->modrm = insn_fetch(u8, 1, c->eip);
859 c->modrm_mod |= (c->modrm & 0xc0) >> 6;
860 c->modrm_reg |= (c->modrm & 0x38) >> 3;
861 c->modrm_rm |= (c->modrm & 0x07);
862 c->modrm_seg = VCPU_SREG_DS;
864 if (c->modrm_mod == 3) {
866 op->bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
867 op->addr.reg = decode_register(c->modrm_rm,
868 c->regs, c->d & ByteOp);
872 op->addr.xmm = c->modrm_rm;
873 read_sse_reg(ctxt, &op->vec_val, c->modrm_rm);
876 fetch_register_operand(op);
882 if (c->ad_bytes == 2) {
883 unsigned bx = c->regs[VCPU_REGS_RBX];
884 unsigned bp = c->regs[VCPU_REGS_RBP];
885 unsigned si = c->regs[VCPU_REGS_RSI];
886 unsigned di = c->regs[VCPU_REGS_RDI];
888 /* 16-bit ModR/M decode. */
889 switch (c->modrm_mod) {
891 if (c->modrm_rm == 6)
892 modrm_ea += insn_fetch(u16, 2, c->eip);
895 modrm_ea += insn_fetch(s8, 1, c->eip);
898 modrm_ea += insn_fetch(u16, 2, c->eip);
901 switch (c->modrm_rm) {
921 if (c->modrm_mod != 0)
928 if (c->modrm_rm == 2 || c->modrm_rm == 3 ||
929 (c->modrm_rm == 6 && c->modrm_mod != 0))
930 c->modrm_seg = VCPU_SREG_SS;
931 modrm_ea = (u16)modrm_ea;
933 /* 32/64-bit ModR/M decode. */
934 if ((c->modrm_rm & 7) == 4) {
935 sib = insn_fetch(u8, 1, c->eip);
936 index_reg |= (sib >> 3) & 7;
940 if ((base_reg & 7) == 5 && c->modrm_mod == 0)
941 modrm_ea += insn_fetch(s32, 4, c->eip);
943 modrm_ea += c->regs[base_reg];
945 modrm_ea += c->regs[index_reg] << scale;
946 } else if ((c->modrm_rm & 7) == 5 && c->modrm_mod == 0) {
947 if (ctxt->mode == X86EMUL_MODE_PROT64)
950 modrm_ea += c->regs[c->modrm_rm];
951 switch (c->modrm_mod) {
953 if (c->modrm_rm == 5)
954 modrm_ea += insn_fetch(s32, 4, c->eip);
957 modrm_ea += insn_fetch(s8, 1, c->eip);
960 modrm_ea += insn_fetch(s32, 4, c->eip);
964 op->addr.mem.ea = modrm_ea;
969 static int decode_abs(struct x86_emulate_ctxt *ctxt,
970 struct x86_emulate_ops *ops,
973 struct decode_cache *c = &ctxt->decode;
974 int rc = X86EMUL_CONTINUE;
977 switch (c->ad_bytes) {
979 op->addr.mem.ea = insn_fetch(u16, 2, c->eip);
982 op->addr.mem.ea = insn_fetch(u32, 4, c->eip);
985 op->addr.mem.ea = insn_fetch(u64, 8, c->eip);
992 static void fetch_bit_operand(struct decode_cache *c)
996 if (c->dst.type == OP_MEM && c->src.type == OP_REG) {
997 mask = ~(c->dst.bytes * 8 - 1);
999 if (c->src.bytes == 2)
1000 sv = (s16)c->src.val & (s16)mask;
1001 else if (c->src.bytes == 4)
1002 sv = (s32)c->src.val & (s32)mask;
1004 c->dst.addr.mem.ea += (sv >> 3);
1007 /* only subword offset */
1008 c->src.val &= (c->dst.bytes << 3) - 1;
1011 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1012 struct x86_emulate_ops *ops,
1013 unsigned long addr, void *dest, unsigned size)
1016 struct read_cache *mc = &ctxt->decode.mem_read;
1019 int n = min(size, 8u);
1021 if (mc->pos < mc->end)
1024 rc = ops->read_emulated(addr, mc->data + mc->end, n,
1025 &ctxt->exception, ctxt->vcpu);
1026 if (rc != X86EMUL_CONTINUE)
1031 memcpy(dest, mc->data + mc->pos, n);
1036 return X86EMUL_CONTINUE;
1039 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1040 struct segmented_address addr,
1047 rc = linearize(ctxt, addr, size, false, &linear);
1048 if (rc != X86EMUL_CONTINUE)
1050 return read_emulated(ctxt, ctxt->ops, linear, data, size);
1053 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1054 struct segmented_address addr,
1061 rc = linearize(ctxt, addr, size, true, &linear);
1062 if (rc != X86EMUL_CONTINUE)
1064 return ctxt->ops->write_emulated(linear, data, size,
1065 &ctxt->exception, ctxt->vcpu);
1068 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1069 struct segmented_address addr,
1070 const void *orig_data, const void *data,
1076 rc = linearize(ctxt, addr, size, true, &linear);
1077 if (rc != X86EMUL_CONTINUE)
1079 return ctxt->ops->cmpxchg_emulated(linear, orig_data, data,
1080 size, &ctxt->exception, ctxt->vcpu);
1083 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1084 struct x86_emulate_ops *ops,
1085 unsigned int size, unsigned short port,
1088 struct read_cache *rc = &ctxt->decode.io_read;
1090 if (rc->pos == rc->end) { /* refill pio read ahead */
1091 struct decode_cache *c = &ctxt->decode;
1092 unsigned int in_page, n;
1093 unsigned int count = c->rep_prefix ?
1094 address_mask(c, c->regs[VCPU_REGS_RCX]) : 1;
1095 in_page = (ctxt->eflags & EFLG_DF) ?
1096 offset_in_page(c->regs[VCPU_REGS_RDI]) :
1097 PAGE_SIZE - offset_in_page(c->regs[VCPU_REGS_RDI]);
1098 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1102 rc->pos = rc->end = 0;
1103 if (!ops->pio_in_emulated(size, port, rc->data, n, ctxt->vcpu))
1108 memcpy(dest, rc->data + rc->pos, size);
1113 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1114 struct x86_emulate_ops *ops,
1115 u16 selector, struct desc_ptr *dt)
1117 if (selector & 1 << 2) {
1118 struct desc_struct desc;
1119 memset (dt, 0, sizeof *dt);
1120 if (!ops->get_cached_descriptor(&desc, NULL, VCPU_SREG_LDTR,
1124 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1125 dt->address = get_desc_base(&desc);
1127 ops->get_gdt(dt, ctxt->vcpu);
1130 /* allowed just for 8 bytes segments */
1131 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1132 struct x86_emulate_ops *ops,
1133 u16 selector, struct desc_struct *desc)
1136 u16 index = selector >> 3;
1140 get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1142 if (dt.size < index * 8 + 7)
1143 return emulate_gp(ctxt, selector & 0xfffc);
1144 addr = dt.address + index * 8;
1145 ret = ops->read_std(addr, desc, sizeof *desc, ctxt->vcpu,
1151 /* allowed just for 8 bytes segments */
1152 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1153 struct x86_emulate_ops *ops,
1154 u16 selector, struct desc_struct *desc)
1157 u16 index = selector >> 3;
1161 get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1163 if (dt.size < index * 8 + 7)
1164 return emulate_gp(ctxt, selector & 0xfffc);
1166 addr = dt.address + index * 8;
1167 ret = ops->write_std(addr, desc, sizeof *desc, ctxt->vcpu,
1173 /* Does not support long mode */
1174 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1175 struct x86_emulate_ops *ops,
1176 u16 selector, int seg)
1178 struct desc_struct seg_desc;
1180 unsigned err_vec = GP_VECTOR;
1182 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1185 memset(&seg_desc, 0, sizeof seg_desc);
1187 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1188 || ctxt->mode == X86EMUL_MODE_REAL) {
1189 /* set real mode segment descriptor */
1190 set_desc_base(&seg_desc, selector << 4);
1191 set_desc_limit(&seg_desc, 0xffff);
1198 /* NULL selector is not valid for TR, CS and SS */
1199 if ((seg == VCPU_SREG_CS || seg == VCPU_SREG_SS || seg == VCPU_SREG_TR)
1203 /* TR should be in GDT only */
1204 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1207 if (null_selector) /* for NULL selector skip all following checks */
1210 ret = read_segment_descriptor(ctxt, ops, selector, &seg_desc);
1211 if (ret != X86EMUL_CONTINUE)
1214 err_code = selector & 0xfffc;
1215 err_vec = GP_VECTOR;
1217 /* can't load system descriptor into segment selecor */
1218 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1222 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1228 cpl = ops->cpl(ctxt->vcpu);
1233 * segment is not a writable data segment or segment
1234 * selector's RPL != CPL or segment selector's RPL != CPL
1236 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1240 if (!(seg_desc.type & 8))
1243 if (seg_desc.type & 4) {
1249 if (rpl > cpl || dpl != cpl)
1252 /* CS(RPL) <- CPL */
1253 selector = (selector & 0xfffc) | cpl;
1256 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1259 case VCPU_SREG_LDTR:
1260 if (seg_desc.s || seg_desc.type != 2)
1263 default: /* DS, ES, FS, or GS */
1265 * segment is not a data or readable code segment or
1266 * ((segment is a data or nonconforming code segment)
1267 * and (both RPL and CPL > DPL))
1269 if ((seg_desc.type & 0xa) == 0x8 ||
1270 (((seg_desc.type & 0xc) != 0xc) &&
1271 (rpl > dpl && cpl > dpl)))
1277 /* mark segment as accessed */
1279 ret = write_segment_descriptor(ctxt, ops, selector, &seg_desc);
1280 if (ret != X86EMUL_CONTINUE)
1284 ops->set_segment_selector(selector, seg, ctxt->vcpu);
1285 ops->set_cached_descriptor(&seg_desc, 0, seg, ctxt->vcpu);
1286 return X86EMUL_CONTINUE;
1288 emulate_exception(ctxt, err_vec, err_code, true);
1289 return X86EMUL_PROPAGATE_FAULT;
1292 static void write_register_operand(struct operand *op)
1294 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1295 switch (op->bytes) {
1297 *(u8 *)op->addr.reg = (u8)op->val;
1300 *(u16 *)op->addr.reg = (u16)op->val;
1303 *op->addr.reg = (u32)op->val;
1304 break; /* 64b: zero-extend */
1306 *op->addr.reg = op->val;
1311 static inline int writeback(struct x86_emulate_ctxt *ctxt,
1312 struct x86_emulate_ops *ops)
1315 struct decode_cache *c = &ctxt->decode;
1317 switch (c->dst.type) {
1319 write_register_operand(&c->dst);
1323 rc = segmented_cmpxchg(ctxt,
1329 rc = segmented_write(ctxt,
1333 if (rc != X86EMUL_CONTINUE)
1337 write_sse_reg(ctxt, &c->dst.vec_val, c->dst.addr.xmm);
1345 return X86EMUL_CONTINUE;
1348 static inline void emulate_push(struct x86_emulate_ctxt *ctxt,
1349 struct x86_emulate_ops *ops)
1351 struct decode_cache *c = &ctxt->decode;
1353 c->dst.type = OP_MEM;
1354 c->dst.bytes = c->op_bytes;
1355 c->dst.val = c->src.val;
1356 register_address_increment(c, &c->regs[VCPU_REGS_RSP], -c->op_bytes);
1357 c->dst.addr.mem.ea = register_address(c, c->regs[VCPU_REGS_RSP]);
1358 c->dst.addr.mem.seg = VCPU_SREG_SS;
1361 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1362 struct x86_emulate_ops *ops,
1363 void *dest, int len)
1365 struct decode_cache *c = &ctxt->decode;
1367 struct segmented_address addr;
1369 addr.ea = register_address(c, c->regs[VCPU_REGS_RSP]);
1370 addr.seg = VCPU_SREG_SS;
1371 rc = segmented_read(ctxt, addr, dest, len);
1372 if (rc != X86EMUL_CONTINUE)
1375 register_address_increment(c, &c->regs[VCPU_REGS_RSP], len);
1379 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1380 struct x86_emulate_ops *ops,
1381 void *dest, int len)
1384 unsigned long val, change_mask;
1385 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1386 int cpl = ops->cpl(ctxt->vcpu);
1388 rc = emulate_pop(ctxt, ops, &val, len);
1389 if (rc != X86EMUL_CONTINUE)
1392 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1393 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1395 switch(ctxt->mode) {
1396 case X86EMUL_MODE_PROT64:
1397 case X86EMUL_MODE_PROT32:
1398 case X86EMUL_MODE_PROT16:
1400 change_mask |= EFLG_IOPL;
1402 change_mask |= EFLG_IF;
1404 case X86EMUL_MODE_VM86:
1406 return emulate_gp(ctxt, 0);
1407 change_mask |= EFLG_IF;
1409 default: /* real mode */
1410 change_mask |= (EFLG_IOPL | EFLG_IF);
1414 *(unsigned long *)dest =
1415 (ctxt->eflags & ~change_mask) | (val & change_mask);
1420 static void emulate_push_sreg(struct x86_emulate_ctxt *ctxt,
1421 struct x86_emulate_ops *ops, int seg)
1423 struct decode_cache *c = &ctxt->decode;
1425 c->src.val = ops->get_segment_selector(seg, ctxt->vcpu);
1427 emulate_push(ctxt, ops);
1430 static int emulate_pop_sreg(struct x86_emulate_ctxt *ctxt,
1431 struct x86_emulate_ops *ops, int seg)
1433 struct decode_cache *c = &ctxt->decode;
1434 unsigned long selector;
1437 rc = emulate_pop(ctxt, ops, &selector, c->op_bytes);
1438 if (rc != X86EMUL_CONTINUE)
1441 rc = load_segment_descriptor(ctxt, ops, (u16)selector, seg);
1445 static int emulate_pusha(struct x86_emulate_ctxt *ctxt,
1446 struct x86_emulate_ops *ops)
1448 struct decode_cache *c = &ctxt->decode;
1449 unsigned long old_esp = c->regs[VCPU_REGS_RSP];
1450 int rc = X86EMUL_CONTINUE;
1451 int reg = VCPU_REGS_RAX;
1453 while (reg <= VCPU_REGS_RDI) {
1454 (reg == VCPU_REGS_RSP) ?
1455 (c->src.val = old_esp) : (c->src.val = c->regs[reg]);
1457 emulate_push(ctxt, ops);
1459 rc = writeback(ctxt, ops);
1460 if (rc != X86EMUL_CONTINUE)
1466 /* Disable writeback. */
1467 c->dst.type = OP_NONE;
1472 static int emulate_popa(struct x86_emulate_ctxt *ctxt,
1473 struct x86_emulate_ops *ops)
1475 struct decode_cache *c = &ctxt->decode;
1476 int rc = X86EMUL_CONTINUE;
1477 int reg = VCPU_REGS_RDI;
1479 while (reg >= VCPU_REGS_RAX) {
1480 if (reg == VCPU_REGS_RSP) {
1481 register_address_increment(c, &c->regs[VCPU_REGS_RSP],
1486 rc = emulate_pop(ctxt, ops, &c->regs[reg], c->op_bytes);
1487 if (rc != X86EMUL_CONTINUE)
1494 int emulate_int_real(struct x86_emulate_ctxt *ctxt,
1495 struct x86_emulate_ops *ops, int irq)
1497 struct decode_cache *c = &ctxt->decode;
1504 /* TODO: Add limit checks */
1505 c->src.val = ctxt->eflags;
1506 emulate_push(ctxt, ops);
1507 rc = writeback(ctxt, ops);
1508 if (rc != X86EMUL_CONTINUE)
1511 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1513 c->src.val = ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
1514 emulate_push(ctxt, ops);
1515 rc = writeback(ctxt, ops);
1516 if (rc != X86EMUL_CONTINUE)
1519 c->src.val = c->eip;
1520 emulate_push(ctxt, ops);
1521 rc = writeback(ctxt, ops);
1522 if (rc != X86EMUL_CONTINUE)
1525 c->dst.type = OP_NONE;
1527 ops->get_idt(&dt, ctxt->vcpu);
1529 eip_addr = dt.address + (irq << 2);
1530 cs_addr = dt.address + (irq << 2) + 2;
1532 rc = ops->read_std(cs_addr, &cs, 2, ctxt->vcpu, &ctxt->exception);
1533 if (rc != X86EMUL_CONTINUE)
1536 rc = ops->read_std(eip_addr, &eip, 2, ctxt->vcpu, &ctxt->exception);
1537 if (rc != X86EMUL_CONTINUE)
1540 rc = load_segment_descriptor(ctxt, ops, cs, VCPU_SREG_CS);
1541 if (rc != X86EMUL_CONTINUE)
1549 static int emulate_int(struct x86_emulate_ctxt *ctxt,
1550 struct x86_emulate_ops *ops, int irq)
1552 switch(ctxt->mode) {
1553 case X86EMUL_MODE_REAL:
1554 return emulate_int_real(ctxt, ops, irq);
1555 case X86EMUL_MODE_VM86:
1556 case X86EMUL_MODE_PROT16:
1557 case X86EMUL_MODE_PROT32:
1558 case X86EMUL_MODE_PROT64:
1560 /* Protected mode interrupts unimplemented yet */
1561 return X86EMUL_UNHANDLEABLE;
1565 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt,
1566 struct x86_emulate_ops *ops)
1568 struct decode_cache *c = &ctxt->decode;
1569 int rc = X86EMUL_CONTINUE;
1570 unsigned long temp_eip = 0;
1571 unsigned long temp_eflags = 0;
1572 unsigned long cs = 0;
1573 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1574 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1575 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1576 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1578 /* TODO: Add stack limit check */
1580 rc = emulate_pop(ctxt, ops, &temp_eip, c->op_bytes);
1582 if (rc != X86EMUL_CONTINUE)
1585 if (temp_eip & ~0xffff)
1586 return emulate_gp(ctxt, 0);
1588 rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
1590 if (rc != X86EMUL_CONTINUE)
1593 rc = emulate_pop(ctxt, ops, &temp_eflags, c->op_bytes);
1595 if (rc != X86EMUL_CONTINUE)
1598 rc = load_segment_descriptor(ctxt, ops, (u16)cs, VCPU_SREG_CS);
1600 if (rc != X86EMUL_CONTINUE)
1606 if (c->op_bytes == 4)
1607 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1608 else if (c->op_bytes == 2) {
1609 ctxt->eflags &= ~0xffff;
1610 ctxt->eflags |= temp_eflags;
1613 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1614 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1619 static inline int emulate_iret(struct x86_emulate_ctxt *ctxt,
1620 struct x86_emulate_ops* ops)
1622 switch(ctxt->mode) {
1623 case X86EMUL_MODE_REAL:
1624 return emulate_iret_real(ctxt, ops);
1625 case X86EMUL_MODE_VM86:
1626 case X86EMUL_MODE_PROT16:
1627 case X86EMUL_MODE_PROT32:
1628 case X86EMUL_MODE_PROT64:
1630 /* iret from protected mode unimplemented yet */
1631 return X86EMUL_UNHANDLEABLE;
1635 static inline int emulate_grp1a(struct x86_emulate_ctxt *ctxt,
1636 struct x86_emulate_ops *ops)
1638 struct decode_cache *c = &ctxt->decode;
1640 return emulate_pop(ctxt, ops, &c->dst.val, c->dst.bytes);
1643 static inline void emulate_grp2(struct x86_emulate_ctxt *ctxt)
1645 struct decode_cache *c = &ctxt->decode;
1646 switch (c->modrm_reg) {
1648 emulate_2op_SrcB("rol", c->src, c->dst, ctxt->eflags);
1651 emulate_2op_SrcB("ror", c->src, c->dst, ctxt->eflags);
1654 emulate_2op_SrcB("rcl", c->src, c->dst, ctxt->eflags);
1657 emulate_2op_SrcB("rcr", c->src, c->dst, ctxt->eflags);
1659 case 4: /* sal/shl */
1660 case 6: /* sal/shl */
1661 emulate_2op_SrcB("sal", c->src, c->dst, ctxt->eflags);
1664 emulate_2op_SrcB("shr", c->src, c->dst, ctxt->eflags);
1667 emulate_2op_SrcB("sar", c->src, c->dst, ctxt->eflags);
1672 static inline int emulate_grp3(struct x86_emulate_ctxt *ctxt,
1673 struct x86_emulate_ops *ops)
1675 struct decode_cache *c = &ctxt->decode;
1676 unsigned long *rax = &c->regs[VCPU_REGS_RAX];
1677 unsigned long *rdx = &c->regs[VCPU_REGS_RDX];
1680 switch (c->modrm_reg) {
1681 case 0 ... 1: /* test */
1682 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
1685 c->dst.val = ~c->dst.val;
1688 emulate_1op("neg", c->dst, ctxt->eflags);
1691 emulate_1op_rax_rdx("mul", c->src, *rax, *rdx, ctxt->eflags);
1694 emulate_1op_rax_rdx("imul", c->src, *rax, *rdx, ctxt->eflags);
1697 emulate_1op_rax_rdx_ex("div", c->src, *rax, *rdx,
1701 emulate_1op_rax_rdx_ex("idiv", c->src, *rax, *rdx,
1705 return X86EMUL_UNHANDLEABLE;
1708 return emulate_de(ctxt);
1709 return X86EMUL_CONTINUE;
1712 static inline int emulate_grp45(struct x86_emulate_ctxt *ctxt,
1713 struct x86_emulate_ops *ops)
1715 struct decode_cache *c = &ctxt->decode;
1717 switch (c->modrm_reg) {
1719 emulate_1op("inc", c->dst, ctxt->eflags);
1722 emulate_1op("dec", c->dst, ctxt->eflags);
1724 case 2: /* call near abs */ {
1727 c->eip = c->src.val;
1728 c->src.val = old_eip;
1729 emulate_push(ctxt, ops);
1732 case 4: /* jmp abs */
1733 c->eip = c->src.val;
1736 emulate_push(ctxt, ops);
1739 return X86EMUL_CONTINUE;
1742 static inline int emulate_grp9(struct x86_emulate_ctxt *ctxt,
1743 struct x86_emulate_ops *ops)
1745 struct decode_cache *c = &ctxt->decode;
1746 u64 old = c->dst.orig_val64;
1748 if (((u32) (old >> 0) != (u32) c->regs[VCPU_REGS_RAX]) ||
1749 ((u32) (old >> 32) != (u32) c->regs[VCPU_REGS_RDX])) {
1750 c->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1751 c->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1752 ctxt->eflags &= ~EFLG_ZF;
1754 c->dst.val64 = ((u64)c->regs[VCPU_REGS_RCX] << 32) |
1755 (u32) c->regs[VCPU_REGS_RBX];
1757 ctxt->eflags |= EFLG_ZF;
1759 return X86EMUL_CONTINUE;
1762 static int emulate_ret_far(struct x86_emulate_ctxt *ctxt,
1763 struct x86_emulate_ops *ops)
1765 struct decode_cache *c = &ctxt->decode;
1769 rc = emulate_pop(ctxt, ops, &c->eip, c->op_bytes);
1770 if (rc != X86EMUL_CONTINUE)
1772 if (c->op_bytes == 4)
1773 c->eip = (u32)c->eip;
1774 rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
1775 if (rc != X86EMUL_CONTINUE)
1777 rc = load_segment_descriptor(ctxt, ops, (u16)cs, VCPU_SREG_CS);
1781 static int emulate_load_segment(struct x86_emulate_ctxt *ctxt,
1782 struct x86_emulate_ops *ops, int seg)
1784 struct decode_cache *c = &ctxt->decode;
1788 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
1790 rc = load_segment_descriptor(ctxt, ops, sel, seg);
1791 if (rc != X86EMUL_CONTINUE)
1794 c->dst.val = c->src.val;
1799 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1800 struct x86_emulate_ops *ops, struct desc_struct *cs,
1801 struct desc_struct *ss)
1803 memset(cs, 0, sizeof(struct desc_struct));
1804 ops->get_cached_descriptor(cs, NULL, VCPU_SREG_CS, ctxt->vcpu);
1805 memset(ss, 0, sizeof(struct desc_struct));
1807 cs->l = 0; /* will be adjusted later */
1808 set_desc_base(cs, 0); /* flat segment */
1809 cs->g = 1; /* 4kb granularity */
1810 set_desc_limit(cs, 0xfffff); /* 4GB limit */
1811 cs->type = 0x0b; /* Read, Execute, Accessed */
1813 cs->dpl = 0; /* will be adjusted later */
1817 set_desc_base(ss, 0); /* flat segment */
1818 set_desc_limit(ss, 0xfffff); /* 4GB limit */
1819 ss->g = 1; /* 4kb granularity */
1821 ss->type = 0x03; /* Read/Write, Accessed */
1822 ss->d = 1; /* 32bit stack segment */
1828 emulate_syscall(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1830 struct decode_cache *c = &ctxt->decode;
1831 struct desc_struct cs, ss;
1835 /* syscall is not available in real mode */
1836 if (ctxt->mode == X86EMUL_MODE_REAL ||
1837 ctxt->mode == X86EMUL_MODE_VM86)
1838 return emulate_ud(ctxt);
1840 setup_syscalls_segments(ctxt, ops, &cs, &ss);
1842 ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
1844 cs_sel = (u16)(msr_data & 0xfffc);
1845 ss_sel = (u16)(msr_data + 8);
1847 if (is_long_mode(ctxt->vcpu)) {
1851 ops->set_cached_descriptor(&cs, 0, VCPU_SREG_CS, ctxt->vcpu);
1852 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
1853 ops->set_cached_descriptor(&ss, 0, VCPU_SREG_SS, ctxt->vcpu);
1854 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
1856 c->regs[VCPU_REGS_RCX] = c->eip;
1857 if (is_long_mode(ctxt->vcpu)) {
1858 #ifdef CONFIG_X86_64
1859 c->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
1861 ops->get_msr(ctxt->vcpu,
1862 ctxt->mode == X86EMUL_MODE_PROT64 ?
1863 MSR_LSTAR : MSR_CSTAR, &msr_data);
1866 ops->get_msr(ctxt->vcpu, MSR_SYSCALL_MASK, &msr_data);
1867 ctxt->eflags &= ~(msr_data | EFLG_RF);
1871 ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
1872 c->eip = (u32)msr_data;
1874 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1877 return X86EMUL_CONTINUE;
1881 emulate_sysenter(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1883 struct decode_cache *c = &ctxt->decode;
1884 struct desc_struct cs, ss;
1888 /* inject #GP if in real mode */
1889 if (ctxt->mode == X86EMUL_MODE_REAL)
1890 return emulate_gp(ctxt, 0);
1892 /* XXX sysenter/sysexit have not been tested in 64bit mode.
1893 * Therefore, we inject an #UD.
1895 if (ctxt->mode == X86EMUL_MODE_PROT64)
1896 return emulate_ud(ctxt);
1898 setup_syscalls_segments(ctxt, ops, &cs, &ss);
1900 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
1901 switch (ctxt->mode) {
1902 case X86EMUL_MODE_PROT32:
1903 if ((msr_data & 0xfffc) == 0x0)
1904 return emulate_gp(ctxt, 0);
1906 case X86EMUL_MODE_PROT64:
1907 if (msr_data == 0x0)
1908 return emulate_gp(ctxt, 0);
1912 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1913 cs_sel = (u16)msr_data;
1914 cs_sel &= ~SELECTOR_RPL_MASK;
1915 ss_sel = cs_sel + 8;
1916 ss_sel &= ~SELECTOR_RPL_MASK;
1917 if (ctxt->mode == X86EMUL_MODE_PROT64
1918 || is_long_mode(ctxt->vcpu)) {
1923 ops->set_cached_descriptor(&cs, 0, VCPU_SREG_CS, ctxt->vcpu);
1924 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
1925 ops->set_cached_descriptor(&ss, 0, VCPU_SREG_SS, ctxt->vcpu);
1926 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
1928 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_EIP, &msr_data);
1931 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_ESP, &msr_data);
1932 c->regs[VCPU_REGS_RSP] = msr_data;
1934 return X86EMUL_CONTINUE;
1938 emulate_sysexit(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1940 struct decode_cache *c = &ctxt->decode;
1941 struct desc_struct cs, ss;
1946 /* inject #GP if in real mode or Virtual 8086 mode */
1947 if (ctxt->mode == X86EMUL_MODE_REAL ||
1948 ctxt->mode == X86EMUL_MODE_VM86)
1949 return emulate_gp(ctxt, 0);
1951 setup_syscalls_segments(ctxt, ops, &cs, &ss);
1953 if ((c->rex_prefix & 0x8) != 0x0)
1954 usermode = X86EMUL_MODE_PROT64;
1956 usermode = X86EMUL_MODE_PROT32;
1960 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
1962 case X86EMUL_MODE_PROT32:
1963 cs_sel = (u16)(msr_data + 16);
1964 if ((msr_data & 0xfffc) == 0x0)
1965 return emulate_gp(ctxt, 0);
1966 ss_sel = (u16)(msr_data + 24);
1968 case X86EMUL_MODE_PROT64:
1969 cs_sel = (u16)(msr_data + 32);
1970 if (msr_data == 0x0)
1971 return emulate_gp(ctxt, 0);
1972 ss_sel = cs_sel + 8;
1977 cs_sel |= SELECTOR_RPL_MASK;
1978 ss_sel |= SELECTOR_RPL_MASK;
1980 ops->set_cached_descriptor(&cs, 0, VCPU_SREG_CS, ctxt->vcpu);
1981 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
1982 ops->set_cached_descriptor(&ss, 0, VCPU_SREG_SS, ctxt->vcpu);
1983 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
1985 c->eip = c->regs[VCPU_REGS_RDX];
1986 c->regs[VCPU_REGS_RSP] = c->regs[VCPU_REGS_RCX];
1988 return X86EMUL_CONTINUE;
1991 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt,
1992 struct x86_emulate_ops *ops)
1995 if (ctxt->mode == X86EMUL_MODE_REAL)
1997 if (ctxt->mode == X86EMUL_MODE_VM86)
1999 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2000 return ops->cpl(ctxt->vcpu) > iopl;
2003 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2004 struct x86_emulate_ops *ops,
2007 struct desc_struct tr_seg;
2010 u16 io_bitmap_ptr, perm, bit_idx = port & 0x7;
2011 unsigned mask = (1 << len) - 1;
2014 ops->get_cached_descriptor(&tr_seg, &base3, VCPU_SREG_TR, ctxt->vcpu);
2017 if (desc_limit_scaled(&tr_seg) < 103)
2019 base = get_desc_base(&tr_seg);
2020 #ifdef CONFIG_X86_64
2021 base |= ((u64)base3) << 32;
2023 r = ops->read_std(base + 102, &io_bitmap_ptr, 2, ctxt->vcpu, NULL);
2024 if (r != X86EMUL_CONTINUE)
2026 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2028 r = ops->read_std(base + io_bitmap_ptr + port/8, &perm, 2, ctxt->vcpu,
2030 if (r != X86EMUL_CONTINUE)
2032 if ((perm >> bit_idx) & mask)
2037 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2038 struct x86_emulate_ops *ops,
2044 if (emulator_bad_iopl(ctxt, ops))
2045 if (!emulator_io_port_access_allowed(ctxt, ops, port, len))
2048 ctxt->perm_ok = true;
2053 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2054 struct x86_emulate_ops *ops,
2055 struct tss_segment_16 *tss)
2057 struct decode_cache *c = &ctxt->decode;
2060 tss->flag = ctxt->eflags;
2061 tss->ax = c->regs[VCPU_REGS_RAX];
2062 tss->cx = c->regs[VCPU_REGS_RCX];
2063 tss->dx = c->regs[VCPU_REGS_RDX];
2064 tss->bx = c->regs[VCPU_REGS_RBX];
2065 tss->sp = c->regs[VCPU_REGS_RSP];
2066 tss->bp = c->regs[VCPU_REGS_RBP];
2067 tss->si = c->regs[VCPU_REGS_RSI];
2068 tss->di = c->regs[VCPU_REGS_RDI];
2070 tss->es = ops->get_segment_selector(VCPU_SREG_ES, ctxt->vcpu);
2071 tss->cs = ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
2072 tss->ss = ops->get_segment_selector(VCPU_SREG_SS, ctxt->vcpu);
2073 tss->ds = ops->get_segment_selector(VCPU_SREG_DS, ctxt->vcpu);
2074 tss->ldt = ops->get_segment_selector(VCPU_SREG_LDTR, ctxt->vcpu);
2077 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2078 struct x86_emulate_ops *ops,
2079 struct tss_segment_16 *tss)
2081 struct decode_cache *c = &ctxt->decode;
2085 ctxt->eflags = tss->flag | 2;
2086 c->regs[VCPU_REGS_RAX] = tss->ax;
2087 c->regs[VCPU_REGS_RCX] = tss->cx;
2088 c->regs[VCPU_REGS_RDX] = tss->dx;
2089 c->regs[VCPU_REGS_RBX] = tss->bx;
2090 c->regs[VCPU_REGS_RSP] = tss->sp;
2091 c->regs[VCPU_REGS_RBP] = tss->bp;
2092 c->regs[VCPU_REGS_RSI] = tss->si;
2093 c->regs[VCPU_REGS_RDI] = tss->di;
2096 * SDM says that segment selectors are loaded before segment
2099 ops->set_segment_selector(tss->ldt, VCPU_SREG_LDTR, ctxt->vcpu);
2100 ops->set_segment_selector(tss->es, VCPU_SREG_ES, ctxt->vcpu);
2101 ops->set_segment_selector(tss->cs, VCPU_SREG_CS, ctxt->vcpu);
2102 ops->set_segment_selector(tss->ss, VCPU_SREG_SS, ctxt->vcpu);
2103 ops->set_segment_selector(tss->ds, VCPU_SREG_DS, ctxt->vcpu);
2106 * Now load segment descriptors. If fault happenes at this stage
2107 * it is handled in a context of new task
2109 ret = load_segment_descriptor(ctxt, ops, tss->ldt, VCPU_SREG_LDTR);
2110 if (ret != X86EMUL_CONTINUE)
2112 ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
2113 if (ret != X86EMUL_CONTINUE)
2115 ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
2116 if (ret != X86EMUL_CONTINUE)
2118 ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
2119 if (ret != X86EMUL_CONTINUE)
2121 ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
2122 if (ret != X86EMUL_CONTINUE)
2125 return X86EMUL_CONTINUE;
2128 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2129 struct x86_emulate_ops *ops,
2130 u16 tss_selector, u16 old_tss_sel,
2131 ulong old_tss_base, struct desc_struct *new_desc)
2133 struct tss_segment_16 tss_seg;
2135 u32 new_tss_base = get_desc_base(new_desc);
2137 ret = ops->read_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2139 if (ret != X86EMUL_CONTINUE)
2140 /* FIXME: need to provide precise fault address */
2143 save_state_to_tss16(ctxt, ops, &tss_seg);
2145 ret = ops->write_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2147 if (ret != X86EMUL_CONTINUE)
2148 /* FIXME: need to provide precise fault address */
2151 ret = ops->read_std(new_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2153 if (ret != X86EMUL_CONTINUE)
2154 /* FIXME: need to provide precise fault address */
2157 if (old_tss_sel != 0xffff) {
2158 tss_seg.prev_task_link = old_tss_sel;
2160 ret = ops->write_std(new_tss_base,
2161 &tss_seg.prev_task_link,
2162 sizeof tss_seg.prev_task_link,
2163 ctxt->vcpu, &ctxt->exception);
2164 if (ret != X86EMUL_CONTINUE)
2165 /* FIXME: need to provide precise fault address */
2169 return load_state_from_tss16(ctxt, ops, &tss_seg);
2172 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2173 struct x86_emulate_ops *ops,
2174 struct tss_segment_32 *tss)
2176 struct decode_cache *c = &ctxt->decode;
2178 tss->cr3 = ops->get_cr(3, ctxt->vcpu);
2180 tss->eflags = ctxt->eflags;
2181 tss->eax = c->regs[VCPU_REGS_RAX];
2182 tss->ecx = c->regs[VCPU_REGS_RCX];
2183 tss->edx = c->regs[VCPU_REGS_RDX];
2184 tss->ebx = c->regs[VCPU_REGS_RBX];
2185 tss->esp = c->regs[VCPU_REGS_RSP];
2186 tss->ebp = c->regs[VCPU_REGS_RBP];
2187 tss->esi = c->regs[VCPU_REGS_RSI];
2188 tss->edi = c->regs[VCPU_REGS_RDI];
2190 tss->es = ops->get_segment_selector(VCPU_SREG_ES, ctxt->vcpu);
2191 tss->cs = ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
2192 tss->ss = ops->get_segment_selector(VCPU_SREG_SS, ctxt->vcpu);
2193 tss->ds = ops->get_segment_selector(VCPU_SREG_DS, ctxt->vcpu);
2194 tss->fs = ops->get_segment_selector(VCPU_SREG_FS, ctxt->vcpu);
2195 tss->gs = ops->get_segment_selector(VCPU_SREG_GS, ctxt->vcpu);
2196 tss->ldt_selector = ops->get_segment_selector(VCPU_SREG_LDTR, ctxt->vcpu);
2199 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2200 struct x86_emulate_ops *ops,
2201 struct tss_segment_32 *tss)
2203 struct decode_cache *c = &ctxt->decode;
2206 if (ops->set_cr(3, tss->cr3, ctxt->vcpu))
2207 return emulate_gp(ctxt, 0);
2209 ctxt->eflags = tss->eflags | 2;
2210 c->regs[VCPU_REGS_RAX] = tss->eax;
2211 c->regs[VCPU_REGS_RCX] = tss->ecx;
2212 c->regs[VCPU_REGS_RDX] = tss->edx;
2213 c->regs[VCPU_REGS_RBX] = tss->ebx;
2214 c->regs[VCPU_REGS_RSP] = tss->esp;
2215 c->regs[VCPU_REGS_RBP] = tss->ebp;
2216 c->regs[VCPU_REGS_RSI] = tss->esi;
2217 c->regs[VCPU_REGS_RDI] = tss->edi;
2220 * SDM says that segment selectors are loaded before segment
2223 ops->set_segment_selector(tss->ldt_selector, VCPU_SREG_LDTR, ctxt->vcpu);
2224 ops->set_segment_selector(tss->es, VCPU_SREG_ES, ctxt->vcpu);
2225 ops->set_segment_selector(tss->cs, VCPU_SREG_CS, ctxt->vcpu);
2226 ops->set_segment_selector(tss->ss, VCPU_SREG_SS, ctxt->vcpu);
2227 ops->set_segment_selector(tss->ds, VCPU_SREG_DS, ctxt->vcpu);
2228 ops->set_segment_selector(tss->fs, VCPU_SREG_FS, ctxt->vcpu);
2229 ops->set_segment_selector(tss->gs, VCPU_SREG_GS, ctxt->vcpu);
2232 * Now load segment descriptors. If fault happenes at this stage
2233 * it is handled in a context of new task
2235 ret = load_segment_descriptor(ctxt, ops, tss->ldt_selector, VCPU_SREG_LDTR);
2236 if (ret != X86EMUL_CONTINUE)
2238 ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
2239 if (ret != X86EMUL_CONTINUE)
2241 ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
2242 if (ret != X86EMUL_CONTINUE)
2244 ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
2245 if (ret != X86EMUL_CONTINUE)
2247 ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
2248 if (ret != X86EMUL_CONTINUE)
2250 ret = load_segment_descriptor(ctxt, ops, tss->fs, VCPU_SREG_FS);
2251 if (ret != X86EMUL_CONTINUE)
2253 ret = load_segment_descriptor(ctxt, ops, tss->gs, VCPU_SREG_GS);
2254 if (ret != X86EMUL_CONTINUE)
2257 return X86EMUL_CONTINUE;
2260 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2261 struct x86_emulate_ops *ops,
2262 u16 tss_selector, u16 old_tss_sel,
2263 ulong old_tss_base, struct desc_struct *new_desc)
2265 struct tss_segment_32 tss_seg;
2267 u32 new_tss_base = get_desc_base(new_desc);
2269 ret = ops->read_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2271 if (ret != X86EMUL_CONTINUE)
2272 /* FIXME: need to provide precise fault address */
2275 save_state_to_tss32(ctxt, ops, &tss_seg);
2277 ret = ops->write_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2279 if (ret != X86EMUL_CONTINUE)
2280 /* FIXME: need to provide precise fault address */
2283 ret = ops->read_std(new_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2285 if (ret != X86EMUL_CONTINUE)
2286 /* FIXME: need to provide precise fault address */
2289 if (old_tss_sel != 0xffff) {
2290 tss_seg.prev_task_link = old_tss_sel;
2292 ret = ops->write_std(new_tss_base,
2293 &tss_seg.prev_task_link,
2294 sizeof tss_seg.prev_task_link,
2295 ctxt->vcpu, &ctxt->exception);
2296 if (ret != X86EMUL_CONTINUE)
2297 /* FIXME: need to provide precise fault address */
2301 return load_state_from_tss32(ctxt, ops, &tss_seg);
2304 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2305 struct x86_emulate_ops *ops,
2306 u16 tss_selector, int reason,
2307 bool has_error_code, u32 error_code)
2309 struct desc_struct curr_tss_desc, next_tss_desc;
2311 u16 old_tss_sel = ops->get_segment_selector(VCPU_SREG_TR, ctxt->vcpu);
2312 ulong old_tss_base =
2313 ops->get_cached_segment_base(VCPU_SREG_TR, ctxt->vcpu);
2316 /* FIXME: old_tss_base == ~0 ? */
2318 ret = read_segment_descriptor(ctxt, ops, tss_selector, &next_tss_desc);
2319 if (ret != X86EMUL_CONTINUE)
2321 ret = read_segment_descriptor(ctxt, ops, old_tss_sel, &curr_tss_desc);
2322 if (ret != X86EMUL_CONTINUE)
2325 /* FIXME: check that next_tss_desc is tss */
2327 if (reason != TASK_SWITCH_IRET) {
2328 if ((tss_selector & 3) > next_tss_desc.dpl ||
2329 ops->cpl(ctxt->vcpu) > next_tss_desc.dpl)
2330 return emulate_gp(ctxt, 0);
2333 desc_limit = desc_limit_scaled(&next_tss_desc);
2334 if (!next_tss_desc.p ||
2335 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2336 desc_limit < 0x2b)) {
2337 emulate_ts(ctxt, tss_selector & 0xfffc);
2338 return X86EMUL_PROPAGATE_FAULT;
2341 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2342 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2343 write_segment_descriptor(ctxt, ops, old_tss_sel,
2347 if (reason == TASK_SWITCH_IRET)
2348 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2350 /* set back link to prev task only if NT bit is set in eflags
2351 note that old_tss_sel is not used afetr this point */
2352 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2353 old_tss_sel = 0xffff;
2355 if (next_tss_desc.type & 8)
2356 ret = task_switch_32(ctxt, ops, tss_selector, old_tss_sel,
2357 old_tss_base, &next_tss_desc);
2359 ret = task_switch_16(ctxt, ops, tss_selector, old_tss_sel,
2360 old_tss_base, &next_tss_desc);
2361 if (ret != X86EMUL_CONTINUE)
2364 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2365 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2367 if (reason != TASK_SWITCH_IRET) {
2368 next_tss_desc.type |= (1 << 1); /* set busy flag */
2369 write_segment_descriptor(ctxt, ops, tss_selector,
2373 ops->set_cr(0, ops->get_cr(0, ctxt->vcpu) | X86_CR0_TS, ctxt->vcpu);
2374 ops->set_cached_descriptor(&next_tss_desc, 0, VCPU_SREG_TR, ctxt->vcpu);
2375 ops->set_segment_selector(tss_selector, VCPU_SREG_TR, ctxt->vcpu);
2377 if (has_error_code) {
2378 struct decode_cache *c = &ctxt->decode;
2380 c->op_bytes = c->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2382 c->src.val = (unsigned long) error_code;
2383 emulate_push(ctxt, ops);
2389 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2390 u16 tss_selector, int reason,
2391 bool has_error_code, u32 error_code)
2393 struct x86_emulate_ops *ops = ctxt->ops;
2394 struct decode_cache *c = &ctxt->decode;
2398 c->dst.type = OP_NONE;
2400 rc = emulator_do_task_switch(ctxt, ops, tss_selector, reason,
2401 has_error_code, error_code);
2403 if (rc == X86EMUL_CONTINUE) {
2404 rc = writeback(ctxt, ops);
2405 if (rc == X86EMUL_CONTINUE)
2409 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2412 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned seg,
2413 int reg, struct operand *op)
2415 struct decode_cache *c = &ctxt->decode;
2416 int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;
2418 register_address_increment(c, &c->regs[reg], df * op->bytes);
2419 op->addr.mem.ea = register_address(c, c->regs[reg]);
2420 op->addr.mem.seg = seg;
2423 static int em_push(struct x86_emulate_ctxt *ctxt)
2425 emulate_push(ctxt, ctxt->ops);
2426 return X86EMUL_CONTINUE;
2429 static int em_das(struct x86_emulate_ctxt *ctxt)
2431 struct decode_cache *c = &ctxt->decode;
2433 bool af, cf, old_cf;
2435 cf = ctxt->eflags & X86_EFLAGS_CF;
2441 af = ctxt->eflags & X86_EFLAGS_AF;
2442 if ((al & 0x0f) > 9 || af) {
2444 cf = old_cf | (al >= 250);
2449 if (old_al > 0x99 || old_cf) {
2455 /* Set PF, ZF, SF */
2456 c->src.type = OP_IMM;
2459 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
2460 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2462 ctxt->eflags |= X86_EFLAGS_CF;
2464 ctxt->eflags |= X86_EFLAGS_AF;
2465 return X86EMUL_CONTINUE;
2468 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2470 struct decode_cache *c = &ctxt->decode;
2475 old_cs = ctxt->ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
2478 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
2479 if (load_segment_descriptor(ctxt, ctxt->ops, sel, VCPU_SREG_CS))
2480 return X86EMUL_CONTINUE;
2483 memcpy(&c->eip, c->src.valptr, c->op_bytes);
2485 c->src.val = old_cs;
2486 emulate_push(ctxt, ctxt->ops);
2487 rc = writeback(ctxt, ctxt->ops);
2488 if (rc != X86EMUL_CONTINUE)
2491 c->src.val = old_eip;
2492 emulate_push(ctxt, ctxt->ops);
2493 rc = writeback(ctxt, ctxt->ops);
2494 if (rc != X86EMUL_CONTINUE)
2497 c->dst.type = OP_NONE;
2499 return X86EMUL_CONTINUE;
2502 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2504 struct decode_cache *c = &ctxt->decode;
2507 c->dst.type = OP_REG;
2508 c->dst.addr.reg = &c->eip;
2509 c->dst.bytes = c->op_bytes;
2510 rc = emulate_pop(ctxt, ctxt->ops, &c->dst.val, c->op_bytes);
2511 if (rc != X86EMUL_CONTINUE)
2513 register_address_increment(c, &c->regs[VCPU_REGS_RSP], c->src.val);
2514 return X86EMUL_CONTINUE;
2517 static int em_imul(struct x86_emulate_ctxt *ctxt)
2519 struct decode_cache *c = &ctxt->decode;
2521 emulate_2op_SrcV_nobyte("imul", c->src, c->dst, ctxt->eflags);
2522 return X86EMUL_CONTINUE;
2525 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2527 struct decode_cache *c = &ctxt->decode;
2529 c->dst.val = c->src2.val;
2530 return em_imul(ctxt);
2533 static int em_cwd(struct x86_emulate_ctxt *ctxt)
2535 struct decode_cache *c = &ctxt->decode;
2537 c->dst.type = OP_REG;
2538 c->dst.bytes = c->src.bytes;
2539 c->dst.addr.reg = &c->regs[VCPU_REGS_RDX];
2540 c->dst.val = ~((c->src.val >> (c->src.bytes * 8 - 1)) - 1);
2542 return X86EMUL_CONTINUE;
2545 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2547 struct decode_cache *c = &ctxt->decode;
2550 ctxt->ops->get_msr(ctxt->vcpu, MSR_IA32_TSC, &tsc);
2551 c->regs[VCPU_REGS_RAX] = (u32)tsc;
2552 c->regs[VCPU_REGS_RDX] = tsc >> 32;
2553 return X86EMUL_CONTINUE;
2556 static int em_mov(struct x86_emulate_ctxt *ctxt)
2558 struct decode_cache *c = &ctxt->decode;
2559 c->dst.val = c->src.val;
2560 return X86EMUL_CONTINUE;
2563 static int em_movdqu(struct x86_emulate_ctxt *ctxt)
2565 struct decode_cache *c = &ctxt->decode;
2566 memcpy(&c->dst.vec_val, &c->src.vec_val, c->op_bytes);
2567 return X86EMUL_CONTINUE;
2570 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
2572 struct decode_cache *c = &ctxt->decode;
2576 rc = linearize(ctxt, c->src.addr.mem, 1, false, &linear);
2577 if (rc == X86EMUL_CONTINUE)
2578 emulate_invlpg(ctxt->vcpu, linear);
2579 /* Disable writeback. */
2580 c->dst.type = OP_NONE;
2581 return X86EMUL_CONTINUE;
2584 static bool valid_cr(int nr)
2596 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
2598 struct decode_cache *c = &ctxt->decode;
2600 if (!valid_cr(c->modrm_reg))
2601 return emulate_ud(ctxt);
2603 return X86EMUL_CONTINUE;
2606 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
2608 struct decode_cache *c = &ctxt->decode;
2609 u64 new_val = c->src.val64;
2610 int cr = c->modrm_reg;
2612 static u64 cr_reserved_bits[] = {
2613 0xffffffff00000000ULL,
2614 0, 0, 0, /* CR3 checked later */
2621 return emulate_ud(ctxt);
2623 if (new_val & cr_reserved_bits[cr])
2624 return emulate_gp(ctxt, 0);
2629 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
2630 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
2631 return emulate_gp(ctxt, 0);
2633 cr4 = ctxt->ops->get_cr(4, ctxt->vcpu);
2634 ctxt->ops->get_msr(ctxt->vcpu, MSR_EFER, &efer);
2636 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
2637 !(cr4 & X86_CR4_PAE))
2638 return emulate_gp(ctxt, 0);
2645 if (is_long_mode(ctxt->vcpu))
2646 rsvd = CR3_L_MODE_RESERVED_BITS;
2647 else if (is_pae(ctxt->vcpu))
2648 rsvd = CR3_PAE_RESERVED_BITS;
2649 else if (is_paging(ctxt->vcpu))
2650 rsvd = CR3_NONPAE_RESERVED_BITS;
2653 return emulate_gp(ctxt, 0);
2660 cr4 = ctxt->ops->get_cr(4, ctxt->vcpu);
2661 ctxt->ops->get_msr(ctxt->vcpu, MSR_EFER, &efer);
2663 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
2664 return emulate_gp(ctxt, 0);
2670 return X86EMUL_CONTINUE;
2673 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
2677 ctxt->ops->get_dr(7, &dr7, ctxt->vcpu);
2679 /* Check if DR7.Global_Enable is set */
2680 return dr7 & (1 << 13);
2683 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
2685 struct decode_cache *c = &ctxt->decode;
2686 int dr = c->modrm_reg;
2690 return emulate_ud(ctxt);
2692 cr4 = ctxt->ops->get_cr(4, ctxt->vcpu);
2693 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
2694 return emulate_ud(ctxt);
2696 if (check_dr7_gd(ctxt))
2697 return emulate_db(ctxt);
2699 return X86EMUL_CONTINUE;
2702 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
2704 struct decode_cache *c = &ctxt->decode;
2705 u64 new_val = c->src.val64;
2706 int dr = c->modrm_reg;
2708 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
2709 return emulate_gp(ctxt, 0);
2711 return check_dr_read(ctxt);
2714 static int check_svme(struct x86_emulate_ctxt *ctxt)
2718 ctxt->ops->get_msr(ctxt->vcpu, MSR_EFER, &efer);
2720 if (!(efer & EFER_SVME))
2721 return emulate_ud(ctxt);
2723 return X86EMUL_CONTINUE;
2726 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
2728 u64 rax = kvm_register_read(ctxt->vcpu, VCPU_REGS_RAX);
2730 /* Valid physical address? */
2731 if (rax & 0xffff000000000000)
2732 return emulate_gp(ctxt, 0);
2734 return check_svme(ctxt);
2737 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
2739 u64 cr4 = ctxt->ops->get_cr(4, ctxt->vcpu);
2741 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt->vcpu))
2742 return emulate_ud(ctxt);
2744 return X86EMUL_CONTINUE;
2747 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
2749 u64 cr4 = ctxt->ops->get_cr(4, ctxt->vcpu);
2750 u64 rcx = kvm_register_read(ctxt->vcpu, VCPU_REGS_RCX);
2752 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt->vcpu)) ||
2754 return emulate_gp(ctxt, 0);
2756 return X86EMUL_CONTINUE;
2759 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
2761 struct decode_cache *c = &ctxt->decode;
2763 c->dst.bytes = min(c->dst.bytes, 4u);
2764 if (!emulator_io_permited(ctxt, ctxt->ops, c->src.val, c->dst.bytes))
2765 return emulate_gp(ctxt, 0);
2767 return X86EMUL_CONTINUE;
2770 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
2772 struct decode_cache *c = &ctxt->decode;
2774 c->src.bytes = min(c->src.bytes, 4u);
2775 if (!emulator_io_permited(ctxt, ctxt->ops, c->dst.val, c->src.bytes))
2776 return emulate_gp(ctxt, 0);
2778 return X86EMUL_CONTINUE;
2781 #define D(_y) { .flags = (_y) }
2782 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
2783 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
2784 .check_perm = (_p) }
2786 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
2787 #define G(_f, _g) { .flags = ((_f) | Group), .u.group = (_g) }
2788 #define GD(_f, _g) { .flags = ((_f) | Group | GroupDual), .u.gdual = (_g) }
2789 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
2790 #define II(_f, _e, _i) \
2791 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
2792 #define IIP(_f, _e, _i, _p) \
2793 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
2794 .check_perm = (_p) }
2795 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
2797 #define D2bv(_f) D((_f) | ByteOp), D(_f)
2798 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
2799 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
2801 #define D6ALU(_f) D2bv((_f) | DstMem | SrcReg | ModRM), \
2802 D2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock), \
2803 D2bv(((_f) & ~Lock) | DstAcc | SrcImm)
2805 static struct opcode group7_rm1[] = {
2806 DI(SrcNone | ModRM | Priv, monitor),
2807 DI(SrcNone | ModRM | Priv, mwait),
2811 static struct opcode group7_rm3[] = {
2812 DIP(SrcNone | ModRM | Prot | Priv, vmrun, check_svme_pa),
2813 DI(SrcNone | ModRM | Prot | VendorSpecific, vmmcall),
2814 DIP(SrcNone | ModRM | Prot | Priv, vmload, check_svme_pa),
2815 DIP(SrcNone | ModRM | Prot | Priv, vmsave, check_svme_pa),
2816 DIP(SrcNone | ModRM | Prot | Priv, stgi, check_svme),
2817 DIP(SrcNone | ModRM | Prot | Priv, clgi, check_svme),
2818 DIP(SrcNone | ModRM | Prot | Priv, skinit, check_svme),
2819 DIP(SrcNone | ModRM | Prot | Priv, invlpga, check_svme),
2822 static struct opcode group7_rm7[] = {
2824 DIP(SrcNone | ModRM, rdtscp, check_rdtsc),
2827 static struct opcode group1[] = {
2831 static struct opcode group1A[] = {
2832 D(DstMem | SrcNone | ModRM | Mov | Stack), N, N, N, N, N, N, N,
2835 static struct opcode group3[] = {
2836 D(DstMem | SrcImm | ModRM), D(DstMem | SrcImm | ModRM),
2837 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
2838 X4(D(SrcMem | ModRM)),
2841 static struct opcode group4[] = {
2842 D(ByteOp | DstMem | SrcNone | ModRM | Lock), D(ByteOp | DstMem | SrcNone | ModRM | Lock),
2846 static struct opcode group5[] = {
2847 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
2848 D(SrcMem | ModRM | Stack),
2849 I(SrcMemFAddr | ModRM | ImplicitOps | Stack, em_call_far),
2850 D(SrcMem | ModRM | Stack), D(SrcMemFAddr | ModRM | ImplicitOps),
2851 D(SrcMem | ModRM | Stack), N,
2854 static struct opcode group6[] = {
2855 DI(ModRM | Prot, sldt),
2856 DI(ModRM | Prot, str),
2857 DI(ModRM | Prot | Priv, lldt),
2858 DI(ModRM | Prot | Priv, ltr),
2862 static struct group_dual group7 = { {
2863 DI(ModRM | Mov | DstMem | Priv, sgdt),
2864 DI(ModRM | Mov | DstMem | Priv, sidt),
2865 DI(ModRM | SrcMem | Priv, lgdt), DI(ModRM | SrcMem | Priv, lidt),
2866 DI(SrcNone | ModRM | DstMem | Mov, smsw), N,
2867 DI(SrcMem16 | ModRM | Mov | Priv, lmsw),
2868 DI(SrcMem | ModRM | ByteOp | Priv | NoAccess, invlpg),
2870 D(SrcNone | ModRM | Priv | VendorSpecific), EXT(0, group7_rm1),
2871 N, EXT(0, group7_rm3),
2872 DI(SrcNone | ModRM | DstMem | Mov, smsw), N,
2873 DI(SrcMem16 | ModRM | Mov | Priv, lmsw), EXT(0, group7_rm7),
2876 static struct opcode group8[] = {
2878 D(DstMem | SrcImmByte | ModRM), D(DstMem | SrcImmByte | ModRM | Lock),
2879 D(DstMem | SrcImmByte | ModRM | Lock), D(DstMem | SrcImmByte | ModRM | Lock),
2882 static struct group_dual group9 = { {
2883 N, D(DstMem64 | ModRM | Lock), N, N, N, N, N, N,
2885 N, N, N, N, N, N, N, N,
2888 static struct opcode group11[] = {
2889 I(DstMem | SrcImm | ModRM | Mov, em_mov), X7(D(Undefined)),
2892 static struct gprefix pfx_0f_6f_0f_7f = {
2893 N, N, N, I(Sse, em_movdqu),
2896 static struct opcode opcode_table[256] = {
2899 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
2902 D(ImplicitOps | Stack | No64), N,
2905 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
2908 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
2912 D6ALU(Lock), N, I(ByteOp | DstAcc | No64, em_das),
2920 X8(I(SrcReg | Stack, em_push)),
2922 X8(D(DstReg | Stack)),
2924 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
2925 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
2928 I(SrcImm | Mov | Stack, em_push),
2929 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
2930 I(SrcImmByte | Mov | Stack, em_push),
2931 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
2932 D2bvIP(DstDI | Mov | String, ins, check_perm_in), /* insb, insw/insd */
2933 D2bvIP(SrcSI | ImplicitOps | String, outs, check_perm_out), /* outsb, outsw/outsd */
2937 G(ByteOp | DstMem | SrcImm | ModRM | Group, group1),
2938 G(DstMem | SrcImm | ModRM | Group, group1),
2939 G(ByteOp | DstMem | SrcImm | ModRM | No64 | Group, group1),
2940 G(DstMem | SrcImmByte | ModRM | Group, group1),
2941 D2bv(DstMem | SrcReg | ModRM), D2bv(DstMem | SrcReg | ModRM | Lock),
2943 I2bv(DstMem | SrcReg | ModRM | Mov, em_mov),
2944 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
2945 D(DstMem | SrcNone | ModRM | Mov), D(ModRM | SrcMem | NoAccess | DstReg),
2946 D(ImplicitOps | SrcMem16 | ModRM), G(0, group1A),
2948 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
2950 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
2951 I(SrcImmFAddr | No64, em_call_far), N,
2952 DI(ImplicitOps | Stack, pushf), DI(ImplicitOps | Stack, popf), N, N,
2954 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
2955 I2bv(DstMem | SrcAcc | Mov | MemAbs, em_mov),
2956 I2bv(SrcSI | DstDI | Mov | String, em_mov),
2957 D2bv(SrcSI | DstDI | String),
2959 D2bv(DstAcc | SrcImm),
2960 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
2961 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
2962 D2bv(SrcAcc | DstDI | String),
2964 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
2966 X8(I(DstReg | SrcImm | Mov, em_mov)),
2968 D2bv(DstMem | SrcImmByte | ModRM),
2969 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
2970 D(ImplicitOps | Stack),
2971 D(DstReg | SrcMemFAddr | ModRM | No64), D(DstReg | SrcMemFAddr | ModRM | No64),
2972 G(ByteOp, group11), G(0, group11),
2974 N, N, N, D(ImplicitOps | Stack),
2975 D(ImplicitOps), DI(SrcImmByte, intn),
2976 D(ImplicitOps | No64), DI(ImplicitOps, iret),
2978 D2bv(DstMem | SrcOne | ModRM), D2bv(DstMem | ModRM),
2981 N, N, N, N, N, N, N, N,
2984 D2bvIP(SrcImmUByte | DstAcc, in, check_perm_in),
2985 D2bvIP(SrcAcc | DstImmUByte, out, check_perm_out),
2987 D(SrcImm | Stack), D(SrcImm | ImplicitOps),
2988 D(SrcImmFAddr | No64), D(SrcImmByte | ImplicitOps),
2989 D2bvIP(SrcNone | DstAcc, in, check_perm_in),
2990 D2bvIP(SrcAcc | ImplicitOps, out, check_perm_out),
2992 N, DI(ImplicitOps, icebp), N, N,
2993 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
2994 G(ByteOp, group3), G(0, group3),
2996 D(ImplicitOps), D(ImplicitOps), D(ImplicitOps), D(ImplicitOps),
2997 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3000 static struct opcode twobyte_table[256] = {
3002 G(0, group6), GD(0, &group7), N, N,
3003 N, D(ImplicitOps | VendorSpecific), DI(ImplicitOps | Priv, clts), N,
3004 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3005 N, D(ImplicitOps | ModRM), N, N,
3007 N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
3009 DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
3010 DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
3011 DIP(ModRM | SrcMem | Priv | Op3264, cr_write, check_cr_write),
3012 DIP(ModRM | SrcMem | Priv | Op3264, dr_write, check_dr_write),
3014 N, N, N, N, N, N, N, N,
3016 DI(ImplicitOps | Priv, wrmsr),
3017 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
3018 DI(ImplicitOps | Priv, rdmsr),
3019 DIP(ImplicitOps | Priv, rdpmc, check_rdpmc),
3020 D(ImplicitOps | VendorSpecific), D(ImplicitOps | Priv | VendorSpecific),
3022 N, N, N, N, N, N, N, N,
3024 X16(D(DstReg | SrcMem | ModRM | Mov)),
3026 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3031 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3036 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3040 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3042 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
3043 DI(ImplicitOps, cpuid), D(DstMem | SrcReg | ModRM | BitOp),
3044 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3045 D(DstMem | SrcReg | Src2CL | ModRM), N, N,
3047 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
3048 DI(ImplicitOps, rsm), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3049 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3050 D(DstMem | SrcReg | Src2CL | ModRM),
3051 D(ModRM), I(DstReg | SrcMem | ModRM, em_imul),
3053 D2bv(DstMem | SrcReg | ModRM | Lock),
3054 D(DstReg | SrcMemFAddr | ModRM), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3055 D(DstReg | SrcMemFAddr | ModRM), D(DstReg | SrcMemFAddr | ModRM),
3056 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3059 G(BitOp, group8), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3060 D(DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
3061 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3063 D2bv(DstMem | SrcReg | ModRM | Lock),
3064 N, D(DstMem | SrcReg | ModRM | Mov),
3065 N, N, N, GD(0, &group9),
3066 N, N, N, N, N, N, N, N,
3068 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3070 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3072 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
3088 static unsigned imm_size(struct decode_cache *c)
3092 size = (c->d & ByteOp) ? 1 : c->op_bytes;
3098 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
3099 unsigned size, bool sign_extension)
3101 struct decode_cache *c = &ctxt->decode;
3102 struct x86_emulate_ops *ops = ctxt->ops;
3103 int rc = X86EMUL_CONTINUE;
3107 op->addr.mem.ea = c->eip;
3108 /* NB. Immediates are sign-extended as necessary. */
3109 switch (op->bytes) {
3111 op->val = insn_fetch(s8, 1, c->eip);
3114 op->val = insn_fetch(s16, 2, c->eip);
3117 op->val = insn_fetch(s32, 4, c->eip);
3120 if (!sign_extension) {
3121 switch (op->bytes) {
3129 op->val &= 0xffffffff;
3138 x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
3140 struct x86_emulate_ops *ops = ctxt->ops;
3141 struct decode_cache *c = &ctxt->decode;
3142 int rc = X86EMUL_CONTINUE;
3143 int mode = ctxt->mode;
3144 int def_op_bytes, def_ad_bytes, dual, goffset, simd_prefix;
3145 bool op_prefix = false;
3146 struct opcode opcode, *g_mod012, *g_mod3;
3147 struct operand memop = { .type = OP_NONE };
3150 c->fetch.start = c->eip;
3151 c->fetch.end = c->fetch.start + insn_len;
3153 memcpy(c->fetch.data, insn, insn_len);
3154 ctxt->cs_base = seg_base(ctxt, ops, VCPU_SREG_CS);
3157 case X86EMUL_MODE_REAL:
3158 case X86EMUL_MODE_VM86:
3159 case X86EMUL_MODE_PROT16:
3160 def_op_bytes = def_ad_bytes = 2;
3162 case X86EMUL_MODE_PROT32:
3163 def_op_bytes = def_ad_bytes = 4;
3165 #ifdef CONFIG_X86_64
3166 case X86EMUL_MODE_PROT64:
3175 c->op_bytes = def_op_bytes;
3176 c->ad_bytes = def_ad_bytes;
3178 /* Legacy prefixes. */
3180 switch (c->b = insn_fetch(u8, 1, c->eip)) {
3181 case 0x66: /* operand-size override */
3183 /* switch between 2/4 bytes */
3184 c->op_bytes = def_op_bytes ^ 6;
3186 case 0x67: /* address-size override */
3187 if (mode == X86EMUL_MODE_PROT64)
3188 /* switch between 4/8 bytes */
3189 c->ad_bytes = def_ad_bytes ^ 12;
3191 /* switch between 2/4 bytes */
3192 c->ad_bytes = def_ad_bytes ^ 6;
3194 case 0x26: /* ES override */
3195 case 0x2e: /* CS override */
3196 case 0x36: /* SS override */
3197 case 0x3e: /* DS override */
3198 set_seg_override(c, (c->b >> 3) & 3);
3200 case 0x64: /* FS override */
3201 case 0x65: /* GS override */
3202 set_seg_override(c, c->b & 7);
3204 case 0x40 ... 0x4f: /* REX */
3205 if (mode != X86EMUL_MODE_PROT64)
3207 c->rex_prefix = c->b;
3209 case 0xf0: /* LOCK */
3212 case 0xf2: /* REPNE/REPNZ */
3213 case 0xf3: /* REP/REPE/REPZ */
3214 c->rep_prefix = c->b;
3220 /* Any legacy prefix after a REX prefix nullifies its effect. */
3228 if (c->rex_prefix & 8)
3229 c->op_bytes = 8; /* REX.W */
3231 /* Opcode byte(s). */
3232 opcode = opcode_table[c->b];
3233 /* Two-byte opcode? */
3236 c->b = insn_fetch(u8, 1, c->eip);
3237 opcode = twobyte_table[c->b];
3239 c->d = opcode.flags;
3242 dual = c->d & GroupDual;
3243 c->modrm = insn_fetch(u8, 1, c->eip);
3246 if (c->d & GroupDual) {
3247 g_mod012 = opcode.u.gdual->mod012;
3248 g_mod3 = opcode.u.gdual->mod3;
3250 g_mod012 = g_mod3 = opcode.u.group;
3252 c->d &= ~(Group | GroupDual);
3254 goffset = (c->modrm >> 3) & 7;
3256 if ((c->modrm >> 6) == 3)
3257 opcode = g_mod3[goffset];
3259 opcode = g_mod012[goffset];
3261 if (opcode.flags & RMExt) {
3262 goffset = c->modrm & 7;
3263 opcode = opcode.u.group[goffset];
3266 c->d |= opcode.flags;
3269 if (c->d & Prefix) {
3270 if (c->rep_prefix && op_prefix)
3271 return X86EMUL_UNHANDLEABLE;
3272 simd_prefix = op_prefix ? 0x66 : c->rep_prefix;
3273 switch (simd_prefix) {
3274 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
3275 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
3276 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
3277 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
3279 c->d |= opcode.flags;
3282 c->execute = opcode.u.execute;
3283 c->check_perm = opcode.check_perm;
3284 c->intercept = opcode.intercept;
3287 if (c->d == 0 || (c->d & Undefined))
3290 if (!(c->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
3293 if (mode == X86EMUL_MODE_PROT64 && (c->d & Stack))
3296 if (c->d & Op3264) {
3297 if (mode == X86EMUL_MODE_PROT64)
3306 /* ModRM and SIB bytes. */
3308 rc = decode_modrm(ctxt, ops, &memop);
3309 if (!c->has_seg_override)
3310 set_seg_override(c, c->modrm_seg);
3311 } else if (c->d & MemAbs)
3312 rc = decode_abs(ctxt, ops, &memop);
3313 if (rc != X86EMUL_CONTINUE)
3316 if (!c->has_seg_override)
3317 set_seg_override(c, VCPU_SREG_DS);
3319 memop.addr.mem.seg = seg_override(ctxt, ops, c);
3321 if (memop.type == OP_MEM && c->ad_bytes != 8)
3322 memop.addr.mem.ea = (u32)memop.addr.mem.ea;
3324 if (memop.type == OP_MEM && c->rip_relative)
3325 memop.addr.mem.ea += c->eip;
3328 * Decode and fetch the source operand: register, memory
3331 switch (c->d & SrcMask) {
3335 decode_register_operand(ctxt, &c->src, c, 0);
3344 memop.bytes = (c->d & ByteOp) ? 1 :
3350 rc = decode_imm(ctxt, &c->src, 2, false);
3353 rc = decode_imm(ctxt, &c->src, imm_size(c), true);
3356 rc = decode_imm(ctxt, &c->src, imm_size(c), false);
3359 rc = decode_imm(ctxt, &c->src, 1, true);
3362 rc = decode_imm(ctxt, &c->src, 1, false);
3365 c->src.type = OP_REG;
3366 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3367 c->src.addr.reg = &c->regs[VCPU_REGS_RAX];
3368 fetch_register_operand(&c->src);
3375 c->src.type = OP_MEM;
3376 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3377 c->src.addr.mem.ea =
3378 register_address(c, c->regs[VCPU_REGS_RSI]);
3379 c->src.addr.mem.seg = seg_override(ctxt, ops, c),
3383 c->src.type = OP_IMM;
3384 c->src.addr.mem.ea = c->eip;
3385 c->src.bytes = c->op_bytes + 2;
3386 insn_fetch_arr(c->src.valptr, c->src.bytes, c->eip);
3389 memop.bytes = c->op_bytes + 2;
3394 if (rc != X86EMUL_CONTINUE)
3398 * Decode and fetch the second source operand: register, memory
3401 switch (c->d & Src2Mask) {
3406 c->src2.val = c->regs[VCPU_REGS_RCX] & 0x8;
3409 rc = decode_imm(ctxt, &c->src2, 1, true);
3416 rc = decode_imm(ctxt, &c->src2, imm_size(c), true);
3420 if (rc != X86EMUL_CONTINUE)
3423 /* Decode and fetch the destination operand: register or memory. */
3424 switch (c->d & DstMask) {
3426 decode_register_operand(ctxt, &c->dst, c,
3427 c->twobyte && (c->b == 0xb6 || c->b == 0xb7));
3430 c->dst.type = OP_IMM;
3431 c->dst.addr.mem.ea = c->eip;
3433 c->dst.val = insn_fetch(u8, 1, c->eip);
3438 if ((c->d & DstMask) == DstMem64)
3441 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3443 fetch_bit_operand(c);
3444 c->dst.orig_val = c->dst.val;
3447 c->dst.type = OP_REG;
3448 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3449 c->dst.addr.reg = &c->regs[VCPU_REGS_RAX];
3450 fetch_register_operand(&c->dst);
3451 c->dst.orig_val = c->dst.val;
3454 c->dst.type = OP_MEM;
3455 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3456 c->dst.addr.mem.ea =
3457 register_address(c, c->regs[VCPU_REGS_RDI]);
3458 c->dst.addr.mem.seg = VCPU_SREG_ES;
3462 /* Special instructions do their own operand decoding. */
3464 c->dst.type = OP_NONE; /* Disable writeback. */
3469 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
3472 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
3474 struct decode_cache *c = &ctxt->decode;
3476 /* The second termination condition only applies for REPE
3477 * and REPNE. Test if the repeat string operation prefix is
3478 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
3479 * corresponding termination condition according to:
3480 * - if REPE/REPZ and ZF = 0 then done
3481 * - if REPNE/REPNZ and ZF = 1 then done
3483 if (((c->b == 0xa6) || (c->b == 0xa7) ||
3484 (c->b == 0xae) || (c->b == 0xaf))
3485 && (((c->rep_prefix == REPE_PREFIX) &&
3486 ((ctxt->eflags & EFLG_ZF) == 0))
3487 || ((c->rep_prefix == REPNE_PREFIX) &&
3488 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
3495 x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
3497 struct x86_emulate_ops *ops = ctxt->ops;
3499 struct decode_cache *c = &ctxt->decode;
3500 int rc = X86EMUL_CONTINUE;
3501 int saved_dst_type = c->dst.type;
3502 int irq; /* Used for int 3, int, and into */
3504 ctxt->decode.mem_read.pos = 0;
3506 if (ctxt->mode == X86EMUL_MODE_PROT64 && (c->d & No64)) {
3507 rc = emulate_ud(ctxt);
3511 /* LOCK prefix is allowed only with some instructions */
3512 if (c->lock_prefix && (!(c->d & Lock) || c->dst.type != OP_MEM)) {
3513 rc = emulate_ud(ctxt);
3517 if ((c->d & SrcMask) == SrcMemFAddr && c->src.type != OP_MEM) {
3518 rc = emulate_ud(ctxt);
3523 && ((ops->get_cr(0, ctxt->vcpu) & X86_CR0_EM)
3524 || !(ops->get_cr(4, ctxt->vcpu) & X86_CR4_OSFXSR))) {
3525 rc = emulate_ud(ctxt);
3529 if ((c->d & Sse) && (ops->get_cr(0, ctxt->vcpu) & X86_CR0_TS)) {
3530 rc = emulate_nm(ctxt);
3534 if (unlikely(ctxt->guest_mode) && c->intercept) {
3535 rc = emulator_check_intercept(ctxt, c->intercept,
3536 X86_ICPT_PRE_EXCEPT);
3537 if (rc != X86EMUL_CONTINUE)
3541 /* Privileged instruction can be executed only in CPL=0 */
3542 if ((c->d & Priv) && ops->cpl(ctxt->vcpu)) {
3543 rc = emulate_gp(ctxt, 0);
3547 /* Instruction can only be executed in protected mode */
3548 if ((c->d & Prot) && !(ctxt->mode & X86EMUL_MODE_PROT)) {
3549 rc = emulate_ud(ctxt);
3553 /* Do instruction specific permission checks */
3554 if (c->check_perm) {
3555 rc = c->check_perm(ctxt);
3556 if (rc != X86EMUL_CONTINUE)
3560 if (unlikely(ctxt->guest_mode) && c->intercept) {
3561 rc = emulator_check_intercept(ctxt, c->intercept,
3562 X86_ICPT_POST_EXCEPT);
3563 if (rc != X86EMUL_CONTINUE)
3567 if (c->rep_prefix && (c->d & String)) {
3568 /* All REP prefixes have the same first termination condition */
3569 if (address_mask(c, c->regs[VCPU_REGS_RCX]) == 0) {
3575 if ((c->src.type == OP_MEM) && !(c->d & NoAccess)) {
3576 rc = segmented_read(ctxt, c->src.addr.mem,
3577 c->src.valptr, c->src.bytes);
3578 if (rc != X86EMUL_CONTINUE)
3580 c->src.orig_val64 = c->src.val64;
3583 if (c->src2.type == OP_MEM) {
3584 rc = segmented_read(ctxt, c->src2.addr.mem,
3585 &c->src2.val, c->src2.bytes);
3586 if (rc != X86EMUL_CONTINUE)
3590 if ((c->d & DstMask) == ImplicitOps)
3594 if ((c->dst.type == OP_MEM) && !(c->d & Mov)) {
3595 /* optimisation - avoid slow emulated read if Mov */
3596 rc = segmented_read(ctxt, c->dst.addr.mem,
3597 &c->dst.val, c->dst.bytes);
3598 if (rc != X86EMUL_CONTINUE)
3601 c->dst.orig_val = c->dst.val;
3605 if (unlikely(ctxt->guest_mode) && c->intercept) {
3606 rc = emulator_check_intercept(ctxt, c->intercept,
3607 X86_ICPT_POST_MEMACCESS);
3608 if (rc != X86EMUL_CONTINUE)
3613 rc = c->execute(ctxt);
3614 if (rc != X86EMUL_CONTINUE)
3625 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
3627 case 0x06: /* push es */
3628 emulate_push_sreg(ctxt, ops, VCPU_SREG_ES);
3630 case 0x07: /* pop es */
3631 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_ES);
3635 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
3637 case 0x0e: /* push cs */
3638 emulate_push_sreg(ctxt, ops, VCPU_SREG_CS);
3642 emulate_2op_SrcV("adc", c->src, c->dst, ctxt->eflags);
3644 case 0x16: /* push ss */
3645 emulate_push_sreg(ctxt, ops, VCPU_SREG_SS);
3647 case 0x17: /* pop ss */
3648 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_SS);
3652 emulate_2op_SrcV("sbb", c->src, c->dst, ctxt->eflags);
3654 case 0x1e: /* push ds */
3655 emulate_push_sreg(ctxt, ops, VCPU_SREG_DS);
3657 case 0x1f: /* pop ds */
3658 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_DS);
3662 emulate_2op_SrcV("and", c->src, c->dst, ctxt->eflags);
3666 emulate_2op_SrcV("sub", c->src, c->dst, ctxt->eflags);
3670 emulate_2op_SrcV("xor", c->src, c->dst, ctxt->eflags);
3674 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
3676 case 0x40 ... 0x47: /* inc r16/r32 */
3677 emulate_1op("inc", c->dst, ctxt->eflags);
3679 case 0x48 ... 0x4f: /* dec r16/r32 */
3680 emulate_1op("dec", c->dst, ctxt->eflags);
3682 case 0x58 ... 0x5f: /* pop reg */
3684 rc = emulate_pop(ctxt, ops, &c->dst.val, c->op_bytes);
3686 case 0x60: /* pusha */
3687 rc = emulate_pusha(ctxt, ops);
3689 case 0x61: /* popa */
3690 rc = emulate_popa(ctxt, ops);
3692 case 0x63: /* movsxd */
3693 if (ctxt->mode != X86EMUL_MODE_PROT64)
3694 goto cannot_emulate;
3695 c->dst.val = (s32) c->src.val;
3697 case 0x6c: /* insb */
3698 case 0x6d: /* insw/insd */
3699 c->src.val = c->regs[VCPU_REGS_RDX];
3701 case 0x6e: /* outsb */
3702 case 0x6f: /* outsw/outsd */
3703 c->dst.val = c->regs[VCPU_REGS_RDX];
3706 case 0x70 ... 0x7f: /* jcc (short) */
3707 if (test_cc(c->b, ctxt->eflags))
3708 jmp_rel(c, c->src.val);
3710 case 0x80 ... 0x83: /* Grp1 */
3711 switch (c->modrm_reg) {
3732 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
3734 case 0x86 ... 0x87: /* xchg */
3736 /* Write back the register source. */
3737 c->src.val = c->dst.val;
3738 write_register_operand(&c->src);
3740 * Write back the memory destination with implicit LOCK
3743 c->dst.val = c->src.orig_val;
3746 case 0x8c: /* mov r/m, sreg */
3747 if (c->modrm_reg > VCPU_SREG_GS) {
3748 rc = emulate_ud(ctxt);
3751 c->dst.val = ops->get_segment_selector(c->modrm_reg, ctxt->vcpu);
3753 case 0x8d: /* lea r16/r32, m */
3754 c->dst.val = c->src.addr.mem.ea;
3756 case 0x8e: { /* mov seg, r/m16 */
3761 if (c->modrm_reg == VCPU_SREG_CS ||
3762 c->modrm_reg > VCPU_SREG_GS) {
3763 rc = emulate_ud(ctxt);
3767 if (c->modrm_reg == VCPU_SREG_SS)
3768 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3770 rc = load_segment_descriptor(ctxt, ops, sel, c->modrm_reg);
3772 c->dst.type = OP_NONE; /* Disable writeback. */
3775 case 0x8f: /* pop (sole member of Grp1a) */
3776 rc = emulate_grp1a(ctxt, ops);
3778 case 0x90 ... 0x97: /* nop / xchg reg, rax */
3779 if (c->dst.addr.reg == &c->regs[VCPU_REGS_RAX])
3782 case 0x98: /* cbw/cwde/cdqe */
3783 switch (c->op_bytes) {
3784 case 2: c->dst.val = (s8)c->dst.val; break;
3785 case 4: c->dst.val = (s16)c->dst.val; break;
3786 case 8: c->dst.val = (s32)c->dst.val; break;
3789 case 0x9c: /* pushf */
3790 c->src.val = (unsigned long) ctxt->eflags;
3791 emulate_push(ctxt, ops);
3793 case 0x9d: /* popf */
3794 c->dst.type = OP_REG;
3795 c->dst.addr.reg = &ctxt->eflags;
3796 c->dst.bytes = c->op_bytes;
3797 rc = emulate_popf(ctxt, ops, &c->dst.val, c->op_bytes);
3799 case 0xa6 ... 0xa7: /* cmps */
3800 c->dst.type = OP_NONE; /* Disable writeback. */
3802 case 0xa8 ... 0xa9: /* test ax, imm */
3804 case 0xae ... 0xaf: /* scas */
3809 case 0xc3: /* ret */
3810 c->dst.type = OP_REG;
3811 c->dst.addr.reg = &c->eip;
3812 c->dst.bytes = c->op_bytes;
3813 goto pop_instruction;
3814 case 0xc4: /* les */
3815 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_ES);
3817 case 0xc5: /* lds */
3818 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_DS);
3820 case 0xcb: /* ret far */
3821 rc = emulate_ret_far(ctxt, ops);
3823 case 0xcc: /* int3 */
3826 case 0xcd: /* int n */
3829 rc = emulate_int(ctxt, ops, irq);
3831 case 0xce: /* into */
3832 if (ctxt->eflags & EFLG_OF) {
3837 case 0xcf: /* iret */
3838 rc = emulate_iret(ctxt, ops);
3840 case 0xd0 ... 0xd1: /* Grp2 */
3843 case 0xd2 ... 0xd3: /* Grp2 */
3844 c->src.val = c->regs[VCPU_REGS_RCX];
3847 case 0xe0 ... 0xe2: /* loop/loopz/loopnz */
3848 register_address_increment(c, &c->regs[VCPU_REGS_RCX], -1);
3849 if (address_mask(c, c->regs[VCPU_REGS_RCX]) != 0 &&
3850 (c->b == 0xe2 || test_cc(c->b ^ 0x5, ctxt->eflags)))
3851 jmp_rel(c, c->src.val);
3853 case 0xe3: /* jcxz/jecxz/jrcxz */
3854 if (address_mask(c, c->regs[VCPU_REGS_RCX]) == 0)
3855 jmp_rel(c, c->src.val);
3857 case 0xe4: /* inb */
3860 case 0xe6: /* outb */
3861 case 0xe7: /* out */
3863 case 0xe8: /* call (near) */ {
3864 long int rel = c->src.val;
3865 c->src.val = (unsigned long) c->eip;
3867 emulate_push(ctxt, ops);
3870 case 0xe9: /* jmp rel */
3872 case 0xea: { /* jmp far */
3875 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
3877 if (load_segment_descriptor(ctxt, ops, sel, VCPU_SREG_CS))
3881 memcpy(&c->eip, c->src.valptr, c->op_bytes);
3885 jmp: /* jmp rel short */
3886 jmp_rel(c, c->src.val);
3887 c->dst.type = OP_NONE; /* Disable writeback. */
3889 case 0xec: /* in al,dx */
3890 case 0xed: /* in (e/r)ax,dx */
3891 c->src.val = c->regs[VCPU_REGS_RDX];
3893 if (!pio_in_emulated(ctxt, ops, c->dst.bytes, c->src.val,
3895 goto done; /* IO is needed */
3897 case 0xee: /* out dx,al */
3898 case 0xef: /* out dx,(e/r)ax */
3899 c->dst.val = c->regs[VCPU_REGS_RDX];
3901 ops->pio_out_emulated(c->src.bytes, c->dst.val,
3902 &c->src.val, 1, ctxt->vcpu);
3903 c->dst.type = OP_NONE; /* Disable writeback. */
3905 case 0xf4: /* hlt */
3906 ctxt->vcpu->arch.halt_request = 1;
3908 case 0xf5: /* cmc */
3909 /* complement carry flag from eflags reg */
3910 ctxt->eflags ^= EFLG_CF;
3912 case 0xf6 ... 0xf7: /* Grp3 */
3913 rc = emulate_grp3(ctxt, ops);
3915 case 0xf8: /* clc */
3916 ctxt->eflags &= ~EFLG_CF;
3918 case 0xf9: /* stc */
3919 ctxt->eflags |= EFLG_CF;
3921 case 0xfa: /* cli */
3922 if (emulator_bad_iopl(ctxt, ops)) {
3923 rc = emulate_gp(ctxt, 0);
3926 ctxt->eflags &= ~X86_EFLAGS_IF;
3928 case 0xfb: /* sti */
3929 if (emulator_bad_iopl(ctxt, ops)) {
3930 rc = emulate_gp(ctxt, 0);
3933 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3934 ctxt->eflags |= X86_EFLAGS_IF;
3937 case 0xfc: /* cld */
3938 ctxt->eflags &= ~EFLG_DF;
3940 case 0xfd: /* std */
3941 ctxt->eflags |= EFLG_DF;
3943 case 0xfe: /* Grp4 */
3945 rc = emulate_grp45(ctxt, ops);
3947 case 0xff: /* Grp5 */
3948 if (c->modrm_reg == 5)
3952 goto cannot_emulate;
3955 if (rc != X86EMUL_CONTINUE)
3959 rc = writeback(ctxt, ops);
3960 if (rc != X86EMUL_CONTINUE)
3964 * restore dst type in case the decoding will be reused
3965 * (happens for string instruction )
3967 c->dst.type = saved_dst_type;
3969 if ((c->d & SrcMask) == SrcSI)
3970 string_addr_inc(ctxt, seg_override(ctxt, ops, c),
3971 VCPU_REGS_RSI, &c->src);
3973 if ((c->d & DstMask) == DstDI)
3974 string_addr_inc(ctxt, VCPU_SREG_ES, VCPU_REGS_RDI,
3977 if (c->rep_prefix && (c->d & String)) {
3978 struct read_cache *r = &ctxt->decode.io_read;
3979 register_address_increment(c, &c->regs[VCPU_REGS_RCX], -1);
3981 if (!string_insn_completed(ctxt)) {
3983 * Re-enter guest when pio read ahead buffer is empty
3984 * or, if it is not used, after each 1024 iteration.
3986 if ((r->end != 0 || c->regs[VCPU_REGS_RCX] & 0x3ff) &&
3987 (r->end == 0 || r->end != r->pos)) {
3989 * Reset read cache. Usually happens before
3990 * decode, but since instruction is restarted
3991 * we have to do it here.
3993 ctxt->decode.mem_read.end = 0;
3994 return EMULATION_RESTART;
3996 goto done; /* skip rip writeback */
4003 if (rc == X86EMUL_PROPAGATE_FAULT)
4004 ctxt->have_exception = true;
4005 if (rc == X86EMUL_INTERCEPTED)
4006 return EMULATION_INTERCEPTED;
4008 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4012 case 0x01: /* lgdt, lidt, lmsw */
4013 switch (c->modrm_reg) {
4015 unsigned long address;
4017 case 0: /* vmcall */
4018 if (c->modrm_mod != 3 || c->modrm_rm != 1)
4019 goto cannot_emulate;
4021 rc = kvm_fix_hypercall(ctxt->vcpu);
4022 if (rc != X86EMUL_CONTINUE)
4025 /* Let the processor re-execute the fixed hypercall */
4027 /* Disable writeback. */
4028 c->dst.type = OP_NONE;
4031 rc = read_descriptor(ctxt, ops, c->src.addr.mem,
4032 &size, &address, c->op_bytes);
4033 if (rc != X86EMUL_CONTINUE)
4035 realmode_lgdt(ctxt->vcpu, size, address);
4036 /* Disable writeback. */
4037 c->dst.type = OP_NONE;
4039 case 3: /* lidt/vmmcall */
4040 if (c->modrm_mod == 3) {
4041 switch (c->modrm_rm) {
4043 rc = kvm_fix_hypercall(ctxt->vcpu);
4046 goto cannot_emulate;
4049 rc = read_descriptor(ctxt, ops, c->src.addr.mem,
4052 if (rc != X86EMUL_CONTINUE)
4054 realmode_lidt(ctxt->vcpu, size, address);
4056 /* Disable writeback. */
4057 c->dst.type = OP_NONE;
4061 c->dst.val = ops->get_cr(0, ctxt->vcpu);
4064 ops->set_cr(0, (ops->get_cr(0, ctxt->vcpu) & ~0x0eul) |
4065 (c->src.val & 0x0f), ctxt->vcpu);
4066 c->dst.type = OP_NONE;
4068 case 5: /* not defined */
4070 rc = X86EMUL_PROPAGATE_FAULT;
4073 rc = em_invlpg(ctxt);
4076 goto cannot_emulate;
4079 case 0x05: /* syscall */
4080 rc = emulate_syscall(ctxt, ops);
4083 emulate_clts(ctxt->vcpu);
4085 case 0x09: /* wbinvd */
4086 kvm_emulate_wbinvd(ctxt->vcpu);
4088 case 0x08: /* invd */
4089 case 0x0d: /* GrpP (prefetch) */
4090 case 0x18: /* Grp16 (prefetch/nop) */
4092 case 0x20: /* mov cr, reg */
4093 c->dst.val = ops->get_cr(c->modrm_reg, ctxt->vcpu);
4095 case 0x21: /* mov from dr to reg */
4096 ops->get_dr(c->modrm_reg, &c->dst.val, ctxt->vcpu);
4098 case 0x22: /* mov reg, cr */
4099 if (ops->set_cr(c->modrm_reg, c->src.val, ctxt->vcpu)) {
4100 emulate_gp(ctxt, 0);
4101 rc = X86EMUL_PROPAGATE_FAULT;
4104 c->dst.type = OP_NONE;
4106 case 0x23: /* mov from reg to dr */
4107 if (ops->set_dr(c->modrm_reg, c->src.val &
4108 ((ctxt->mode == X86EMUL_MODE_PROT64) ?
4109 ~0ULL : ~0U), ctxt->vcpu) < 0) {
4110 /* #UD condition is already handled by the code above */
4111 emulate_gp(ctxt, 0);
4112 rc = X86EMUL_PROPAGATE_FAULT;
4116 c->dst.type = OP_NONE; /* no writeback */
4120 msr_data = (u32)c->regs[VCPU_REGS_RAX]
4121 | ((u64)c->regs[VCPU_REGS_RDX] << 32);
4122 if (ops->set_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], msr_data)) {
4123 emulate_gp(ctxt, 0);
4124 rc = X86EMUL_PROPAGATE_FAULT;
4127 rc = X86EMUL_CONTINUE;
4131 if (ops->get_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], &msr_data)) {
4132 emulate_gp(ctxt, 0);
4133 rc = X86EMUL_PROPAGATE_FAULT;
4136 c->regs[VCPU_REGS_RAX] = (u32)msr_data;
4137 c->regs[VCPU_REGS_RDX] = msr_data >> 32;
4139 rc = X86EMUL_CONTINUE;
4141 case 0x34: /* sysenter */
4142 rc = emulate_sysenter(ctxt, ops);
4144 case 0x35: /* sysexit */
4145 rc = emulate_sysexit(ctxt, ops);
4147 case 0x40 ... 0x4f: /* cmov */
4148 c->dst.val = c->dst.orig_val = c->src.val;
4149 if (!test_cc(c->b, ctxt->eflags))
4150 c->dst.type = OP_NONE; /* no writeback */
4152 case 0x80 ... 0x8f: /* jnz rel, etc*/
4153 if (test_cc(c->b, ctxt->eflags))
4154 jmp_rel(c, c->src.val);
4156 case 0x90 ... 0x9f: /* setcc r/m8 */
4157 c->dst.val = test_cc(c->b, ctxt->eflags);
4159 case 0xa0: /* push fs */
4160 emulate_push_sreg(ctxt, ops, VCPU_SREG_FS);
4162 case 0xa1: /* pop fs */
4163 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_FS);
4167 c->dst.type = OP_NONE;
4168 /* only subword offset */
4169 c->src.val &= (c->dst.bytes << 3) - 1;
4170 emulate_2op_SrcV_nobyte("bt", c->src, c->dst, ctxt->eflags);
4172 case 0xa4: /* shld imm8, r, r/m */
4173 case 0xa5: /* shld cl, r, r/m */
4174 emulate_2op_cl("shld", c->src2, c->src, c->dst, ctxt->eflags);
4176 case 0xa8: /* push gs */
4177 emulate_push_sreg(ctxt, ops, VCPU_SREG_GS);
4179 case 0xa9: /* pop gs */
4180 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_GS);
4184 emulate_2op_SrcV_nobyte("bts", c->src, c->dst, ctxt->eflags);
4186 case 0xac: /* shrd imm8, r, r/m */
4187 case 0xad: /* shrd cl, r, r/m */
4188 emulate_2op_cl("shrd", c->src2, c->src, c->dst, ctxt->eflags);
4190 case 0xae: /* clflush */
4192 case 0xb0 ... 0xb1: /* cmpxchg */
4194 * Save real source value, then compare EAX against
4197 c->src.orig_val = c->src.val;
4198 c->src.val = c->regs[VCPU_REGS_RAX];
4199 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
4200 if (ctxt->eflags & EFLG_ZF) {
4201 /* Success: write back to memory. */
4202 c->dst.val = c->src.orig_val;
4204 /* Failure: write the value we saw to EAX. */
4205 c->dst.type = OP_REG;
4206 c->dst.addr.reg = (unsigned long *)&c->regs[VCPU_REGS_RAX];
4209 case 0xb2: /* lss */
4210 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_SS);
4214 emulate_2op_SrcV_nobyte("btr", c->src, c->dst, ctxt->eflags);
4216 case 0xb4: /* lfs */
4217 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_FS);
4219 case 0xb5: /* lgs */
4220 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_GS);
4222 case 0xb6 ... 0xb7: /* movzx */
4223 c->dst.bytes = c->op_bytes;
4224 c->dst.val = (c->d & ByteOp) ? (u8) c->src.val
4227 case 0xba: /* Grp8 */
4228 switch (c->modrm_reg & 3) {
4241 emulate_2op_SrcV_nobyte("btc", c->src, c->dst, ctxt->eflags);
4243 case 0xbc: { /* bsf */
4245 __asm__ ("bsf %2, %0; setz %1"
4246 : "=r"(c->dst.val), "=q"(zf)
4248 ctxt->eflags &= ~X86_EFLAGS_ZF;
4250 ctxt->eflags |= X86_EFLAGS_ZF;
4251 c->dst.type = OP_NONE; /* Disable writeback. */
4255 case 0xbd: { /* bsr */
4257 __asm__ ("bsr %2, %0; setz %1"
4258 : "=r"(c->dst.val), "=q"(zf)
4260 ctxt->eflags &= ~X86_EFLAGS_ZF;
4262 ctxt->eflags |= X86_EFLAGS_ZF;
4263 c->dst.type = OP_NONE; /* Disable writeback. */
4267 case 0xbe ... 0xbf: /* movsx */
4268 c->dst.bytes = c->op_bytes;
4269 c->dst.val = (c->d & ByteOp) ? (s8) c->src.val :
4272 case 0xc0 ... 0xc1: /* xadd */
4273 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
4274 /* Write back the register source. */
4275 c->src.val = c->dst.orig_val;
4276 write_register_operand(&c->src);
4278 case 0xc3: /* movnti */
4279 c->dst.bytes = c->op_bytes;
4280 c->dst.val = (c->op_bytes == 4) ? (u32) c->src.val :
4283 case 0xc7: /* Grp9 (cmpxchg8b) */
4284 rc = emulate_grp9(ctxt, ops);
4287 goto cannot_emulate;
4290 if (rc != X86EMUL_CONTINUE)
4296 return EMULATION_FAILED;