1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
35 #define OpImplicit 1ull /* No generic decode */
36 #define OpReg 2ull /* Register */
37 #define OpMem 3ull /* Memory */
38 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
39 #define OpDI 5ull /* ES:DI/EDI/RDI */
40 #define OpMem64 6ull /* Memory, 64-bit */
41 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
42 #define OpDX 8ull /* DX register */
43 #define OpCL 9ull /* CL register (for shifts) */
44 #define OpImmByte 10ull /* 8-bit sign extended immediate */
45 #define OpOne 11ull /* Implied 1 */
46 #define OpImm 12ull /* Sign extended immediate */
48 #define OpBits 4 /* Width of operand field */
49 #define OpMask ((1ull << OpBits) - 1)
52 * Opcode effective-address decode tables.
53 * Note that we only emulate instructions that have at least one memory
54 * operand (excluding implicit stack references). We assume that stack
55 * references and instruction fetches will never occur in special memory
56 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
60 /* Operand sizes: 8-bit operands or specified/overridden size. */
61 #define ByteOp (1<<0) /* 8-bit operands. */
62 /* Destination operand type. */
64 #define ImplicitOps (OpImplicit << DstShift)
65 #define DstReg (OpReg << DstShift)
66 #define DstMem (OpMem << DstShift)
67 #define DstAcc (OpAcc << DstShift)
68 #define DstDI (OpDI << DstShift)
69 #define DstMem64 (OpMem64 << DstShift)
70 #define DstImmUByte (OpImmUByte << DstShift)
71 #define DstDX (OpDX << DstShift)
72 #define DstMask (OpMask << DstShift)
73 /* Source operand type. */
74 #define SrcNone (0<<5) /* No source operand. */
75 #define SrcReg (1<<5) /* Register operand. */
76 #define SrcMem (2<<5) /* Memory operand. */
77 #define SrcMem16 (3<<5) /* Memory operand (16-bit). */
78 #define SrcMem32 (4<<5) /* Memory operand (32-bit). */
79 #define SrcImm (5<<5) /* Immediate operand. */
80 #define SrcImmByte (6<<5) /* 8-bit sign-extended immediate operand. */
81 #define SrcOne (7<<5) /* Implied '1' */
82 #define SrcImmUByte (8<<5) /* 8-bit unsigned immediate operand. */
83 #define SrcImmU (9<<5) /* Immediate operand, unsigned */
84 #define SrcSI (0xa<<5) /* Source is in the DS:RSI */
85 #define SrcImmFAddr (0xb<<5) /* Source is immediate far address */
86 #define SrcMemFAddr (0xc<<5) /* Source is far address in memory */
87 #define SrcAcc (0xd<<5) /* Source Accumulator */
88 #define SrcImmU16 (0xe<<5) /* Immediate operand, unsigned, 16 bits */
89 #define SrcDX (0xf<<5) /* Source is in DX register */
90 #define SrcMask (0xf<<5)
91 /* Generic ModRM decode. */
93 /* Destination is only written; never read. */
96 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
97 #define String (1<<13) /* String instruction (rep capable) */
98 #define Stack (1<<14) /* Stack instruction (push/pop) */
99 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
100 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
101 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
102 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
103 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
104 #define Sse (1<<18) /* SSE Vector instruction */
106 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
107 #define VendorSpecific (1<<22) /* Vendor specific instruction */
108 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
109 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
110 #define Undefined (1<<25) /* No Such Instruction */
111 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
112 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
114 /* Source 2 operand type */
115 #define Src2Shift (29)
116 #define Src2None (OpNone << Src2Shift)
117 #define Src2CL (OpCL << Src2Shift)
118 #define Src2ImmByte (OpImmByte << Src2Shift)
119 #define Src2One (OpOne << Src2Shift)
120 #define Src2Imm (OpImm << Src2Shift)
121 #define Src2Mask (OpMask << Src2Shift)
123 #define X2(x...) x, x
124 #define X3(x...) X2(x), x
125 #define X4(x...) X2(x), X2(x)
126 #define X5(x...) X4(x), x
127 #define X6(x...) X4(x), X2(x)
128 #define X7(x...) X4(x), X3(x)
129 #define X8(x...) X4(x), X4(x)
130 #define X16(x...) X8(x), X8(x)
136 int (*execute)(struct x86_emulate_ctxt *ctxt);
137 struct opcode *group;
138 struct group_dual *gdual;
139 struct gprefix *gprefix;
141 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
145 struct opcode mod012[8];
146 struct opcode mod3[8];
150 struct opcode pfx_no;
151 struct opcode pfx_66;
152 struct opcode pfx_f2;
153 struct opcode pfx_f3;
156 /* EFLAGS bit definitions. */
157 #define EFLG_ID (1<<21)
158 #define EFLG_VIP (1<<20)
159 #define EFLG_VIF (1<<19)
160 #define EFLG_AC (1<<18)
161 #define EFLG_VM (1<<17)
162 #define EFLG_RF (1<<16)
163 #define EFLG_IOPL (3<<12)
164 #define EFLG_NT (1<<14)
165 #define EFLG_OF (1<<11)
166 #define EFLG_DF (1<<10)
167 #define EFLG_IF (1<<9)
168 #define EFLG_TF (1<<8)
169 #define EFLG_SF (1<<7)
170 #define EFLG_ZF (1<<6)
171 #define EFLG_AF (1<<4)
172 #define EFLG_PF (1<<2)
173 #define EFLG_CF (1<<0)
175 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
176 #define EFLG_RESERVED_ONE_MASK 2
179 * Instruction emulation:
180 * Most instructions are emulated directly via a fragment of inline assembly
181 * code. This allows us to save/restore EFLAGS and thus very easily pick up
182 * any modified flags.
185 #if defined(CONFIG_X86_64)
186 #define _LO32 "k" /* force 32-bit operand */
187 #define _STK "%%rsp" /* stack pointer */
188 #elif defined(__i386__)
189 #define _LO32 "" /* force 32-bit operand */
190 #define _STK "%%esp" /* stack pointer */
194 * These EFLAGS bits are restored from saved value during emulation, and
195 * any changes are written back to the saved value after emulation.
197 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
199 /* Before executing instruction: restore necessary bits in EFLAGS. */
200 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
201 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
202 "movl %"_sav",%"_LO32 _tmp"; " \
205 "movl %"_msk",%"_LO32 _tmp"; " \
206 "andl %"_LO32 _tmp",("_STK"); " \
208 "notl %"_LO32 _tmp"; " \
209 "andl %"_LO32 _tmp",("_STK"); " \
210 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
212 "orl %"_LO32 _tmp",("_STK"); " \
216 /* After executing instruction: write-back necessary bits in EFLAGS. */
217 #define _POST_EFLAGS(_sav, _msk, _tmp) \
218 /* _sav |= EFLAGS & _msk; */ \
221 "andl %"_msk",%"_LO32 _tmp"; " \
222 "orl %"_LO32 _tmp",%"_sav"; "
230 #define ____emulate_2op(ctxt, _op, _x, _y, _suffix, _dsttype) \
232 __asm__ __volatile__ ( \
233 _PRE_EFLAGS("0", "4", "2") \
234 _op _suffix " %"_x"3,%1; " \
235 _POST_EFLAGS("0", "4", "2") \
236 : "=m" ((ctxt)->eflags), \
237 "+q" (*(_dsttype*)&(ctxt)->dst.val), \
239 : _y ((ctxt)->src.val), "i" (EFLAGS_MASK)); \
243 /* Raw emulation: instruction has two explicit operands. */
244 #define __emulate_2op_nobyte(ctxt,_op,_wx,_wy,_lx,_ly,_qx,_qy) \
246 unsigned long _tmp; \
248 switch ((ctxt)->dst.bytes) { \
250 ____emulate_2op(ctxt,_op,_wx,_wy,"w",u16); \
253 ____emulate_2op(ctxt,_op,_lx,_ly,"l",u32); \
256 ON64(____emulate_2op(ctxt,_op,_qx,_qy,"q",u64)); \
261 #define __emulate_2op(ctxt,_op,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
263 unsigned long _tmp; \
264 switch ((ctxt)->dst.bytes) { \
266 ____emulate_2op(ctxt,_op,_bx,_by,"b",u8); \
269 __emulate_2op_nobyte(ctxt, _op, \
270 _wx, _wy, _lx, _ly, _qx, _qy); \
275 /* Source operand is byte-sized and may be restricted to just %cl. */
276 #define emulate_2op_SrcB(ctxt, _op) \
277 __emulate_2op(ctxt, _op, "b", "c", "b", "c", "b", "c", "b", "c")
279 /* Source operand is byte, word, long or quad sized. */
280 #define emulate_2op_SrcV(ctxt, _op) \
281 __emulate_2op(ctxt, _op, "b", "q", "w", "r", _LO32, "r", "", "r")
283 /* Source operand is word, long or quad sized. */
284 #define emulate_2op_SrcV_nobyte(ctxt, _op) \
285 __emulate_2op_nobyte(ctxt, _op, "w", "r", _LO32, "r", "", "r")
287 /* Instruction has three operands and one operand is stored in ECX register */
288 #define __emulate_2op_cl(ctxt, _op, _suffix, _type) \
290 unsigned long _tmp; \
291 _type _clv = (ctxt)->src2.val; \
292 _type _srcv = (ctxt)->src.val; \
293 _type _dstv = (ctxt)->dst.val; \
295 __asm__ __volatile__ ( \
296 _PRE_EFLAGS("0", "5", "2") \
297 _op _suffix " %4,%1 \n" \
298 _POST_EFLAGS("0", "5", "2") \
299 : "=m" ((ctxt)->eflags), "+r" (_dstv), "=&r" (_tmp) \
300 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
303 (ctxt)->src2.val = (unsigned long) _clv; \
304 (ctxt)->src2.val = (unsigned long) _srcv; \
305 (ctxt)->dst.val = (unsigned long) _dstv; \
308 #define emulate_2op_cl(ctxt, _op) \
310 switch ((ctxt)->dst.bytes) { \
312 __emulate_2op_cl(ctxt, _op, "w", u16); \
315 __emulate_2op_cl(ctxt, _op, "l", u32); \
318 ON64(__emulate_2op_cl(ctxt, _op, "q", ulong)); \
323 #define __emulate_1op(ctxt, _op, _suffix) \
325 unsigned long _tmp; \
327 __asm__ __volatile__ ( \
328 _PRE_EFLAGS("0", "3", "2") \
329 _op _suffix " %1; " \
330 _POST_EFLAGS("0", "3", "2") \
331 : "=m" ((ctxt)->eflags), "+m" ((ctxt)->dst.val), \
333 : "i" (EFLAGS_MASK)); \
336 /* Instruction has only one explicit operand (no source operand). */
337 #define emulate_1op(ctxt, _op) \
339 switch ((ctxt)->dst.bytes) { \
340 case 1: __emulate_1op(ctxt, _op, "b"); break; \
341 case 2: __emulate_1op(ctxt, _op, "w"); break; \
342 case 4: __emulate_1op(ctxt, _op, "l"); break; \
343 case 8: ON64(__emulate_1op(ctxt, _op, "q")); break; \
347 #define __emulate_1op_rax_rdx(ctxt, _op, _suffix, _ex) \
349 unsigned long _tmp; \
350 ulong *rax = &(ctxt)->regs[VCPU_REGS_RAX]; \
351 ulong *rdx = &(ctxt)->regs[VCPU_REGS_RDX]; \
353 __asm__ __volatile__ ( \
354 _PRE_EFLAGS("0", "5", "1") \
356 _op _suffix " %6; " \
358 _POST_EFLAGS("0", "5", "1") \
359 ".pushsection .fixup,\"ax\" \n\t" \
360 "3: movb $1, %4 \n\t" \
363 _ASM_EXTABLE(1b, 3b) \
364 : "=m" ((ctxt)->eflags), "=&r" (_tmp), \
365 "+a" (*rax), "+d" (*rdx), "+qm"(_ex) \
366 : "i" (EFLAGS_MASK), "m" ((ctxt)->src.val), \
367 "a" (*rax), "d" (*rdx)); \
370 /* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
371 #define emulate_1op_rax_rdx(ctxt, _op, _ex) \
373 switch((ctxt)->src.bytes) { \
375 __emulate_1op_rax_rdx(ctxt, _op, "b", _ex); \
378 __emulate_1op_rax_rdx(ctxt, _op, "w", _ex); \
381 __emulate_1op_rax_rdx(ctxt, _op, "l", _ex); \
384 __emulate_1op_rax_rdx(ctxt, _op, "q", _ex)); \
389 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
390 enum x86_intercept intercept,
391 enum x86_intercept_stage stage)
393 struct x86_instruction_info info = {
394 .intercept = intercept,
395 .rep_prefix = ctxt->rep_prefix,
396 .modrm_mod = ctxt->modrm_mod,
397 .modrm_reg = ctxt->modrm_reg,
398 .modrm_rm = ctxt->modrm_rm,
399 .src_val = ctxt->src.val64,
400 .src_bytes = ctxt->src.bytes,
401 .dst_bytes = ctxt->dst.bytes,
402 .ad_bytes = ctxt->ad_bytes,
403 .next_rip = ctxt->eip,
406 return ctxt->ops->intercept(ctxt, &info, stage);
409 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
411 return (1UL << (ctxt->ad_bytes << 3)) - 1;
414 /* Access/update address held in a register, based on addressing mode. */
415 static inline unsigned long
416 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
418 if (ctxt->ad_bytes == sizeof(unsigned long))
421 return reg & ad_mask(ctxt);
424 static inline unsigned long
425 register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
427 return address_mask(ctxt, reg);
431 register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
433 if (ctxt->ad_bytes == sizeof(unsigned long))
436 *reg = (*reg & ~ad_mask(ctxt)) | ((*reg + inc) & ad_mask(ctxt));
439 static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
441 register_address_increment(ctxt, &ctxt->_eip, rel);
444 static u32 desc_limit_scaled(struct desc_struct *desc)
446 u32 limit = get_desc_limit(desc);
448 return desc->g ? (limit << 12) | 0xfff : limit;
451 static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
453 ctxt->has_seg_override = true;
454 ctxt->seg_override = seg;
457 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
459 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
462 return ctxt->ops->get_cached_segment_base(ctxt, seg);
465 static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
467 if (!ctxt->has_seg_override)
470 return ctxt->seg_override;
473 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
474 u32 error, bool valid)
476 ctxt->exception.vector = vec;
477 ctxt->exception.error_code = error;
478 ctxt->exception.error_code_valid = valid;
479 return X86EMUL_PROPAGATE_FAULT;
482 static int emulate_db(struct x86_emulate_ctxt *ctxt)
484 return emulate_exception(ctxt, DB_VECTOR, 0, false);
487 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
489 return emulate_exception(ctxt, GP_VECTOR, err, true);
492 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
494 return emulate_exception(ctxt, SS_VECTOR, err, true);
497 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
499 return emulate_exception(ctxt, UD_VECTOR, 0, false);
502 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
504 return emulate_exception(ctxt, TS_VECTOR, err, true);
507 static int emulate_de(struct x86_emulate_ctxt *ctxt)
509 return emulate_exception(ctxt, DE_VECTOR, 0, false);
512 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
514 return emulate_exception(ctxt, NM_VECTOR, 0, false);
517 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
520 struct desc_struct desc;
522 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
526 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
531 struct desc_struct desc;
533 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
534 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
537 static int __linearize(struct x86_emulate_ctxt *ctxt,
538 struct segmented_address addr,
539 unsigned size, bool write, bool fetch,
542 struct desc_struct desc;
549 la = seg_base(ctxt, addr.seg) + addr.ea;
550 switch (ctxt->mode) {
551 case X86EMUL_MODE_REAL:
553 case X86EMUL_MODE_PROT64:
554 if (((signed long)la << 16) >> 16 != la)
555 return emulate_gp(ctxt, 0);
558 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
562 /* code segment or read-only data segment */
563 if (((desc.type & 8) || !(desc.type & 2)) && write)
565 /* unreadable code segment */
566 if (!fetch && (desc.type & 8) && !(desc.type & 2))
568 lim = desc_limit_scaled(&desc);
569 if ((desc.type & 8) || !(desc.type & 4)) {
570 /* expand-up segment */
571 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
574 /* exapand-down segment */
575 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
577 lim = desc.d ? 0xffffffff : 0xffff;
578 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
581 cpl = ctxt->ops->cpl(ctxt);
584 if (!(desc.type & 8)) {
588 } else if ((desc.type & 8) && !(desc.type & 4)) {
589 /* nonconforming code segment */
592 } else if ((desc.type & 8) && (desc.type & 4)) {
593 /* conforming code segment */
599 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
602 return X86EMUL_CONTINUE;
604 if (addr.seg == VCPU_SREG_SS)
605 return emulate_ss(ctxt, addr.seg);
607 return emulate_gp(ctxt, addr.seg);
610 static int linearize(struct x86_emulate_ctxt *ctxt,
611 struct segmented_address addr,
612 unsigned size, bool write,
615 return __linearize(ctxt, addr, size, write, false, linear);
619 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
620 struct segmented_address addr,
627 rc = linearize(ctxt, addr, size, false, &linear);
628 if (rc != X86EMUL_CONTINUE)
630 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
634 * Fetch the next byte of the instruction being emulated which is pointed to
635 * by ctxt->_eip, then increment ctxt->_eip.
637 * Also prefetch the remaining bytes of the instruction without crossing page
638 * boundary if they are not in fetch_cache yet.
640 static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
642 struct fetch_cache *fc = &ctxt->fetch;
646 if (ctxt->_eip == fc->end) {
647 unsigned long linear;
648 struct segmented_address addr = { .seg = VCPU_SREG_CS,
650 cur_size = fc->end - fc->start;
651 size = min(15UL - cur_size,
652 PAGE_SIZE - offset_in_page(ctxt->_eip));
653 rc = __linearize(ctxt, addr, size, false, true, &linear);
654 if (unlikely(rc != X86EMUL_CONTINUE))
656 rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
657 size, &ctxt->exception);
658 if (unlikely(rc != X86EMUL_CONTINUE))
662 *dest = fc->data[ctxt->_eip - fc->start];
664 return X86EMUL_CONTINUE;
667 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
668 void *dest, unsigned size)
672 /* x86 instructions are limited to 15 bytes. */
673 if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
674 return X86EMUL_UNHANDLEABLE;
676 rc = do_insn_fetch_byte(ctxt, dest++);
677 if (rc != X86EMUL_CONTINUE)
680 return X86EMUL_CONTINUE;
683 /* Fetch next part of the instruction being emulated. */
684 #define insn_fetch(_type, _ctxt) \
685 ({ unsigned long _x; \
686 rc = do_insn_fetch(_ctxt, &_x, sizeof(_type)); \
687 if (rc != X86EMUL_CONTINUE) \
692 #define insn_fetch_arr(_arr, _size, _ctxt) \
693 ({ rc = do_insn_fetch(_ctxt, _arr, (_size)); \
694 if (rc != X86EMUL_CONTINUE) \
699 * Given the 'reg' portion of a ModRM byte, and a register block, return a
700 * pointer into the block that addresses the relevant register.
701 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
703 static void *decode_register(u8 modrm_reg, unsigned long *regs,
708 p = ®s[modrm_reg];
709 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
710 p = (unsigned char *)®s[modrm_reg & 3] + 1;
714 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
715 struct segmented_address addr,
716 u16 *size, unsigned long *address, int op_bytes)
723 rc = segmented_read_std(ctxt, addr, size, 2);
724 if (rc != X86EMUL_CONTINUE)
727 rc = segmented_read_std(ctxt, addr, address, op_bytes);
731 static int test_cc(unsigned int condition, unsigned int flags)
735 switch ((condition & 15) >> 1) {
737 rc |= (flags & EFLG_OF);
739 case 1: /* b/c/nae */
740 rc |= (flags & EFLG_CF);
743 rc |= (flags & EFLG_ZF);
746 rc |= (flags & (EFLG_CF|EFLG_ZF));
749 rc |= (flags & EFLG_SF);
752 rc |= (flags & EFLG_PF);
755 rc |= (flags & EFLG_ZF);
758 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
762 /* Odd condition identifiers (lsb == 1) have inverted sense. */
763 return (!!rc ^ (condition & 1));
766 static void fetch_register_operand(struct operand *op)
770 op->val = *(u8 *)op->addr.reg;
773 op->val = *(u16 *)op->addr.reg;
776 op->val = *(u32 *)op->addr.reg;
779 op->val = *(u64 *)op->addr.reg;
784 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
786 ctxt->ops->get_fpu(ctxt);
788 case 0: asm("movdqu %%xmm0, %0" : "=m"(*data)); break;
789 case 1: asm("movdqu %%xmm1, %0" : "=m"(*data)); break;
790 case 2: asm("movdqu %%xmm2, %0" : "=m"(*data)); break;
791 case 3: asm("movdqu %%xmm3, %0" : "=m"(*data)); break;
792 case 4: asm("movdqu %%xmm4, %0" : "=m"(*data)); break;
793 case 5: asm("movdqu %%xmm5, %0" : "=m"(*data)); break;
794 case 6: asm("movdqu %%xmm6, %0" : "=m"(*data)); break;
795 case 7: asm("movdqu %%xmm7, %0" : "=m"(*data)); break;
797 case 8: asm("movdqu %%xmm8, %0" : "=m"(*data)); break;
798 case 9: asm("movdqu %%xmm9, %0" : "=m"(*data)); break;
799 case 10: asm("movdqu %%xmm10, %0" : "=m"(*data)); break;
800 case 11: asm("movdqu %%xmm11, %0" : "=m"(*data)); break;
801 case 12: asm("movdqu %%xmm12, %0" : "=m"(*data)); break;
802 case 13: asm("movdqu %%xmm13, %0" : "=m"(*data)); break;
803 case 14: asm("movdqu %%xmm14, %0" : "=m"(*data)); break;
804 case 15: asm("movdqu %%xmm15, %0" : "=m"(*data)); break;
808 ctxt->ops->put_fpu(ctxt);
811 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
814 ctxt->ops->get_fpu(ctxt);
816 case 0: asm("movdqu %0, %%xmm0" : : "m"(*data)); break;
817 case 1: asm("movdqu %0, %%xmm1" : : "m"(*data)); break;
818 case 2: asm("movdqu %0, %%xmm2" : : "m"(*data)); break;
819 case 3: asm("movdqu %0, %%xmm3" : : "m"(*data)); break;
820 case 4: asm("movdqu %0, %%xmm4" : : "m"(*data)); break;
821 case 5: asm("movdqu %0, %%xmm5" : : "m"(*data)); break;
822 case 6: asm("movdqu %0, %%xmm6" : : "m"(*data)); break;
823 case 7: asm("movdqu %0, %%xmm7" : : "m"(*data)); break;
825 case 8: asm("movdqu %0, %%xmm8" : : "m"(*data)); break;
826 case 9: asm("movdqu %0, %%xmm9" : : "m"(*data)); break;
827 case 10: asm("movdqu %0, %%xmm10" : : "m"(*data)); break;
828 case 11: asm("movdqu %0, %%xmm11" : : "m"(*data)); break;
829 case 12: asm("movdqu %0, %%xmm12" : : "m"(*data)); break;
830 case 13: asm("movdqu %0, %%xmm13" : : "m"(*data)); break;
831 case 14: asm("movdqu %0, %%xmm14" : : "m"(*data)); break;
832 case 15: asm("movdqu %0, %%xmm15" : : "m"(*data)); break;
836 ctxt->ops->put_fpu(ctxt);
839 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
843 unsigned reg = ctxt->modrm_reg;
844 int highbyte_regs = ctxt->rex_prefix == 0;
846 if (!(ctxt->d & ModRM))
847 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
853 read_sse_reg(ctxt, &op->vec_val, reg);
858 if ((ctxt->d & ByteOp) && !inhibit_bytereg) {
859 op->addr.reg = decode_register(reg, ctxt->regs, highbyte_regs);
862 op->addr.reg = decode_register(reg, ctxt->regs, 0);
863 op->bytes = ctxt->op_bytes;
865 fetch_register_operand(op);
866 op->orig_val = op->val;
869 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
873 int index_reg = 0, base_reg = 0, scale;
874 int rc = X86EMUL_CONTINUE;
877 if (ctxt->rex_prefix) {
878 ctxt->modrm_reg = (ctxt->rex_prefix & 4) << 1; /* REX.R */
879 index_reg = (ctxt->rex_prefix & 2) << 2; /* REX.X */
880 ctxt->modrm_rm = base_reg = (ctxt->rex_prefix & 1) << 3; /* REG.B */
883 ctxt->modrm = insn_fetch(u8, ctxt);
884 ctxt->modrm_mod |= (ctxt->modrm & 0xc0) >> 6;
885 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
886 ctxt->modrm_rm |= (ctxt->modrm & 0x07);
887 ctxt->modrm_seg = VCPU_SREG_DS;
889 if (ctxt->modrm_mod == 3) {
891 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
892 op->addr.reg = decode_register(ctxt->modrm_rm,
893 ctxt->regs, ctxt->d & ByteOp);
897 op->addr.xmm = ctxt->modrm_rm;
898 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
901 fetch_register_operand(op);
907 if (ctxt->ad_bytes == 2) {
908 unsigned bx = ctxt->regs[VCPU_REGS_RBX];
909 unsigned bp = ctxt->regs[VCPU_REGS_RBP];
910 unsigned si = ctxt->regs[VCPU_REGS_RSI];
911 unsigned di = ctxt->regs[VCPU_REGS_RDI];
913 /* 16-bit ModR/M decode. */
914 switch (ctxt->modrm_mod) {
916 if (ctxt->modrm_rm == 6)
917 modrm_ea += insn_fetch(u16, ctxt);
920 modrm_ea += insn_fetch(s8, ctxt);
923 modrm_ea += insn_fetch(u16, ctxt);
926 switch (ctxt->modrm_rm) {
946 if (ctxt->modrm_mod != 0)
953 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
954 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
955 ctxt->modrm_seg = VCPU_SREG_SS;
956 modrm_ea = (u16)modrm_ea;
958 /* 32/64-bit ModR/M decode. */
959 if ((ctxt->modrm_rm & 7) == 4) {
960 sib = insn_fetch(u8, ctxt);
961 index_reg |= (sib >> 3) & 7;
965 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
966 modrm_ea += insn_fetch(s32, ctxt);
968 modrm_ea += ctxt->regs[base_reg];
970 modrm_ea += ctxt->regs[index_reg] << scale;
971 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
972 if (ctxt->mode == X86EMUL_MODE_PROT64)
973 ctxt->rip_relative = 1;
975 modrm_ea += ctxt->regs[ctxt->modrm_rm];
976 switch (ctxt->modrm_mod) {
978 if (ctxt->modrm_rm == 5)
979 modrm_ea += insn_fetch(s32, ctxt);
982 modrm_ea += insn_fetch(s8, ctxt);
985 modrm_ea += insn_fetch(s32, ctxt);
989 op->addr.mem.ea = modrm_ea;
994 static int decode_abs(struct x86_emulate_ctxt *ctxt,
997 int rc = X86EMUL_CONTINUE;
1000 switch (ctxt->ad_bytes) {
1002 op->addr.mem.ea = insn_fetch(u16, ctxt);
1005 op->addr.mem.ea = insn_fetch(u32, ctxt);
1008 op->addr.mem.ea = insn_fetch(u64, ctxt);
1015 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1019 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1020 mask = ~(ctxt->dst.bytes * 8 - 1);
1022 if (ctxt->src.bytes == 2)
1023 sv = (s16)ctxt->src.val & (s16)mask;
1024 else if (ctxt->src.bytes == 4)
1025 sv = (s32)ctxt->src.val & (s32)mask;
1027 ctxt->dst.addr.mem.ea += (sv >> 3);
1030 /* only subword offset */
1031 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1034 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1035 unsigned long addr, void *dest, unsigned size)
1038 struct read_cache *mc = &ctxt->mem_read;
1041 int n = min(size, 8u);
1043 if (mc->pos < mc->end)
1046 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, n,
1048 if (rc != X86EMUL_CONTINUE)
1053 memcpy(dest, mc->data + mc->pos, n);
1058 return X86EMUL_CONTINUE;
1061 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1062 struct segmented_address addr,
1069 rc = linearize(ctxt, addr, size, false, &linear);
1070 if (rc != X86EMUL_CONTINUE)
1072 return read_emulated(ctxt, linear, data, size);
1075 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1076 struct segmented_address addr,
1083 rc = linearize(ctxt, addr, size, true, &linear);
1084 if (rc != X86EMUL_CONTINUE)
1086 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1090 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1091 struct segmented_address addr,
1092 const void *orig_data, const void *data,
1098 rc = linearize(ctxt, addr, size, true, &linear);
1099 if (rc != X86EMUL_CONTINUE)
1101 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1102 size, &ctxt->exception);
1105 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1106 unsigned int size, unsigned short port,
1109 struct read_cache *rc = &ctxt->io_read;
1111 if (rc->pos == rc->end) { /* refill pio read ahead */
1112 unsigned int in_page, n;
1113 unsigned int count = ctxt->rep_prefix ?
1114 address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) : 1;
1115 in_page = (ctxt->eflags & EFLG_DF) ?
1116 offset_in_page(ctxt->regs[VCPU_REGS_RDI]) :
1117 PAGE_SIZE - offset_in_page(ctxt->regs[VCPU_REGS_RDI]);
1118 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1122 rc->pos = rc->end = 0;
1123 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1128 memcpy(dest, rc->data + rc->pos, size);
1133 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1134 u16 selector, struct desc_ptr *dt)
1136 struct x86_emulate_ops *ops = ctxt->ops;
1138 if (selector & 1 << 2) {
1139 struct desc_struct desc;
1142 memset (dt, 0, sizeof *dt);
1143 if (!ops->get_segment(ctxt, &sel, &desc, NULL, VCPU_SREG_LDTR))
1146 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1147 dt->address = get_desc_base(&desc);
1149 ops->get_gdt(ctxt, dt);
1152 /* allowed just for 8 bytes segments */
1153 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1154 u16 selector, struct desc_struct *desc)
1157 u16 index = selector >> 3;
1160 get_descriptor_table_ptr(ctxt, selector, &dt);
1162 if (dt.size < index * 8 + 7)
1163 return emulate_gp(ctxt, selector & 0xfffc);
1165 addr = dt.address + index * 8;
1166 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1170 /* allowed just for 8 bytes segments */
1171 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1172 u16 selector, struct desc_struct *desc)
1175 u16 index = selector >> 3;
1178 get_descriptor_table_ptr(ctxt, selector, &dt);
1180 if (dt.size < index * 8 + 7)
1181 return emulate_gp(ctxt, selector & 0xfffc);
1183 addr = dt.address + index * 8;
1184 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1188 /* Does not support long mode */
1189 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1190 u16 selector, int seg)
1192 struct desc_struct seg_desc;
1194 unsigned err_vec = GP_VECTOR;
1196 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1199 memset(&seg_desc, 0, sizeof seg_desc);
1201 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1202 || ctxt->mode == X86EMUL_MODE_REAL) {
1203 /* set real mode segment descriptor */
1204 set_desc_base(&seg_desc, selector << 4);
1205 set_desc_limit(&seg_desc, 0xffff);
1212 /* NULL selector is not valid for TR, CS and SS */
1213 if ((seg == VCPU_SREG_CS || seg == VCPU_SREG_SS || seg == VCPU_SREG_TR)
1217 /* TR should be in GDT only */
1218 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1221 if (null_selector) /* for NULL selector skip all following checks */
1224 ret = read_segment_descriptor(ctxt, selector, &seg_desc);
1225 if (ret != X86EMUL_CONTINUE)
1228 err_code = selector & 0xfffc;
1229 err_vec = GP_VECTOR;
1231 /* can't load system descriptor into segment selecor */
1232 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1236 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1242 cpl = ctxt->ops->cpl(ctxt);
1247 * segment is not a writable data segment or segment
1248 * selector's RPL != CPL or segment selector's RPL != CPL
1250 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1254 if (!(seg_desc.type & 8))
1257 if (seg_desc.type & 4) {
1263 if (rpl > cpl || dpl != cpl)
1266 /* CS(RPL) <- CPL */
1267 selector = (selector & 0xfffc) | cpl;
1270 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1273 case VCPU_SREG_LDTR:
1274 if (seg_desc.s || seg_desc.type != 2)
1277 default: /* DS, ES, FS, or GS */
1279 * segment is not a data or readable code segment or
1280 * ((segment is a data or nonconforming code segment)
1281 * and (both RPL and CPL > DPL))
1283 if ((seg_desc.type & 0xa) == 0x8 ||
1284 (((seg_desc.type & 0xc) != 0xc) &&
1285 (rpl > dpl && cpl > dpl)))
1291 /* mark segment as accessed */
1293 ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1294 if (ret != X86EMUL_CONTINUE)
1298 ctxt->ops->set_segment(ctxt, selector, &seg_desc, 0, seg);
1299 return X86EMUL_CONTINUE;
1301 emulate_exception(ctxt, err_vec, err_code, true);
1302 return X86EMUL_PROPAGATE_FAULT;
1305 static void write_register_operand(struct operand *op)
1307 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1308 switch (op->bytes) {
1310 *(u8 *)op->addr.reg = (u8)op->val;
1313 *(u16 *)op->addr.reg = (u16)op->val;
1316 *op->addr.reg = (u32)op->val;
1317 break; /* 64b: zero-extend */
1319 *op->addr.reg = op->val;
1324 static int writeback(struct x86_emulate_ctxt *ctxt)
1328 switch (ctxt->dst.type) {
1330 write_register_operand(&ctxt->dst);
1333 if (ctxt->lock_prefix)
1334 rc = segmented_cmpxchg(ctxt,
1336 &ctxt->dst.orig_val,
1340 rc = segmented_write(ctxt,
1344 if (rc != X86EMUL_CONTINUE)
1348 write_sse_reg(ctxt, &ctxt->dst.vec_val, ctxt->dst.addr.xmm);
1356 return X86EMUL_CONTINUE;
1359 static int em_push(struct x86_emulate_ctxt *ctxt)
1361 struct segmented_address addr;
1363 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], -ctxt->op_bytes);
1364 addr.ea = register_address(ctxt, ctxt->regs[VCPU_REGS_RSP]);
1365 addr.seg = VCPU_SREG_SS;
1367 /* Disable writeback. */
1368 ctxt->dst.type = OP_NONE;
1369 return segmented_write(ctxt, addr, &ctxt->src.val, ctxt->op_bytes);
1372 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1373 void *dest, int len)
1376 struct segmented_address addr;
1378 addr.ea = register_address(ctxt, ctxt->regs[VCPU_REGS_RSP]);
1379 addr.seg = VCPU_SREG_SS;
1380 rc = segmented_read(ctxt, addr, dest, len);
1381 if (rc != X86EMUL_CONTINUE)
1384 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], len);
1388 static int em_pop(struct x86_emulate_ctxt *ctxt)
1390 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1393 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1394 void *dest, int len)
1397 unsigned long val, change_mask;
1398 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1399 int cpl = ctxt->ops->cpl(ctxt);
1401 rc = emulate_pop(ctxt, &val, len);
1402 if (rc != X86EMUL_CONTINUE)
1405 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1406 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1408 switch(ctxt->mode) {
1409 case X86EMUL_MODE_PROT64:
1410 case X86EMUL_MODE_PROT32:
1411 case X86EMUL_MODE_PROT16:
1413 change_mask |= EFLG_IOPL;
1415 change_mask |= EFLG_IF;
1417 case X86EMUL_MODE_VM86:
1419 return emulate_gp(ctxt, 0);
1420 change_mask |= EFLG_IF;
1422 default: /* real mode */
1423 change_mask |= (EFLG_IOPL | EFLG_IF);
1427 *(unsigned long *)dest =
1428 (ctxt->eflags & ~change_mask) | (val & change_mask);
1433 static int em_popf(struct x86_emulate_ctxt *ctxt)
1435 ctxt->dst.type = OP_REG;
1436 ctxt->dst.addr.reg = &ctxt->eflags;
1437 ctxt->dst.bytes = ctxt->op_bytes;
1438 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1441 static int emulate_push_sreg(struct x86_emulate_ctxt *ctxt, int seg)
1443 ctxt->src.val = get_segment_selector(ctxt, seg);
1445 return em_push(ctxt);
1448 static int emulate_pop_sreg(struct x86_emulate_ctxt *ctxt, int seg)
1450 unsigned long selector;
1453 rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1454 if (rc != X86EMUL_CONTINUE)
1457 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1461 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1463 unsigned long old_esp = ctxt->regs[VCPU_REGS_RSP];
1464 int rc = X86EMUL_CONTINUE;
1465 int reg = VCPU_REGS_RAX;
1467 while (reg <= VCPU_REGS_RDI) {
1468 (reg == VCPU_REGS_RSP) ?
1469 (ctxt->src.val = old_esp) : (ctxt->src.val = ctxt->regs[reg]);
1472 if (rc != X86EMUL_CONTINUE)
1481 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1483 ctxt->src.val = (unsigned long)ctxt->eflags;
1484 return em_push(ctxt);
1487 static int em_popa(struct x86_emulate_ctxt *ctxt)
1489 int rc = X86EMUL_CONTINUE;
1490 int reg = VCPU_REGS_RDI;
1492 while (reg >= VCPU_REGS_RAX) {
1493 if (reg == VCPU_REGS_RSP) {
1494 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP],
1499 rc = emulate_pop(ctxt, &ctxt->regs[reg], ctxt->op_bytes);
1500 if (rc != X86EMUL_CONTINUE)
1507 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1509 struct x86_emulate_ops *ops = ctxt->ops;
1516 /* TODO: Add limit checks */
1517 ctxt->src.val = ctxt->eflags;
1519 if (rc != X86EMUL_CONTINUE)
1522 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1524 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1526 if (rc != X86EMUL_CONTINUE)
1529 ctxt->src.val = ctxt->_eip;
1531 if (rc != X86EMUL_CONTINUE)
1534 ops->get_idt(ctxt, &dt);
1536 eip_addr = dt.address + (irq << 2);
1537 cs_addr = dt.address + (irq << 2) + 2;
1539 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1540 if (rc != X86EMUL_CONTINUE)
1543 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1544 if (rc != X86EMUL_CONTINUE)
1547 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1548 if (rc != X86EMUL_CONTINUE)
1556 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1558 switch(ctxt->mode) {
1559 case X86EMUL_MODE_REAL:
1560 return emulate_int_real(ctxt, irq);
1561 case X86EMUL_MODE_VM86:
1562 case X86EMUL_MODE_PROT16:
1563 case X86EMUL_MODE_PROT32:
1564 case X86EMUL_MODE_PROT64:
1566 /* Protected mode interrupts unimplemented yet */
1567 return X86EMUL_UNHANDLEABLE;
1571 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
1573 int rc = X86EMUL_CONTINUE;
1574 unsigned long temp_eip = 0;
1575 unsigned long temp_eflags = 0;
1576 unsigned long cs = 0;
1577 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1578 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1579 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1580 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1582 /* TODO: Add stack limit check */
1584 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
1586 if (rc != X86EMUL_CONTINUE)
1589 if (temp_eip & ~0xffff)
1590 return emulate_gp(ctxt, 0);
1592 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1594 if (rc != X86EMUL_CONTINUE)
1597 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
1599 if (rc != X86EMUL_CONTINUE)
1602 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1604 if (rc != X86EMUL_CONTINUE)
1607 ctxt->_eip = temp_eip;
1610 if (ctxt->op_bytes == 4)
1611 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1612 else if (ctxt->op_bytes == 2) {
1613 ctxt->eflags &= ~0xffff;
1614 ctxt->eflags |= temp_eflags;
1617 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1618 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1623 static int em_iret(struct x86_emulate_ctxt *ctxt)
1625 switch(ctxt->mode) {
1626 case X86EMUL_MODE_REAL:
1627 return emulate_iret_real(ctxt);
1628 case X86EMUL_MODE_VM86:
1629 case X86EMUL_MODE_PROT16:
1630 case X86EMUL_MODE_PROT32:
1631 case X86EMUL_MODE_PROT64:
1633 /* iret from protected mode unimplemented yet */
1634 return X86EMUL_UNHANDLEABLE;
1638 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
1643 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1645 rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
1646 if (rc != X86EMUL_CONTINUE)
1650 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
1651 return X86EMUL_CONTINUE;
1654 static int em_grp1a(struct x86_emulate_ctxt *ctxt)
1656 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->dst.bytes);
1659 static int em_grp2(struct x86_emulate_ctxt *ctxt)
1661 switch (ctxt->modrm_reg) {
1663 emulate_2op_SrcB(ctxt, "rol");
1666 emulate_2op_SrcB(ctxt, "ror");
1669 emulate_2op_SrcB(ctxt, "rcl");
1672 emulate_2op_SrcB(ctxt, "rcr");
1674 case 4: /* sal/shl */
1675 case 6: /* sal/shl */
1676 emulate_2op_SrcB(ctxt, "sal");
1679 emulate_2op_SrcB(ctxt, "shr");
1682 emulate_2op_SrcB(ctxt, "sar");
1685 return X86EMUL_CONTINUE;
1688 static int em_not(struct x86_emulate_ctxt *ctxt)
1690 ctxt->dst.val = ~ctxt->dst.val;
1691 return X86EMUL_CONTINUE;
1694 static int em_neg(struct x86_emulate_ctxt *ctxt)
1696 emulate_1op(ctxt, "neg");
1697 return X86EMUL_CONTINUE;
1700 static int em_mul_ex(struct x86_emulate_ctxt *ctxt)
1704 emulate_1op_rax_rdx(ctxt, "mul", ex);
1705 return X86EMUL_CONTINUE;
1708 static int em_imul_ex(struct x86_emulate_ctxt *ctxt)
1712 emulate_1op_rax_rdx(ctxt, "imul", ex);
1713 return X86EMUL_CONTINUE;
1716 static int em_div_ex(struct x86_emulate_ctxt *ctxt)
1720 emulate_1op_rax_rdx(ctxt, "div", de);
1722 return emulate_de(ctxt);
1723 return X86EMUL_CONTINUE;
1726 static int em_idiv_ex(struct x86_emulate_ctxt *ctxt)
1730 emulate_1op_rax_rdx(ctxt, "idiv", de);
1732 return emulate_de(ctxt);
1733 return X86EMUL_CONTINUE;
1736 static int em_grp45(struct x86_emulate_ctxt *ctxt)
1738 int rc = X86EMUL_CONTINUE;
1740 switch (ctxt->modrm_reg) {
1742 emulate_1op(ctxt, "inc");
1745 emulate_1op(ctxt, "dec");
1747 case 2: /* call near abs */ {
1749 old_eip = ctxt->_eip;
1750 ctxt->_eip = ctxt->src.val;
1751 ctxt->src.val = old_eip;
1755 case 4: /* jmp abs */
1756 ctxt->_eip = ctxt->src.val;
1758 case 5: /* jmp far */
1759 rc = em_jmp_far(ctxt);
1768 static int em_grp9(struct x86_emulate_ctxt *ctxt)
1770 u64 old = ctxt->dst.orig_val64;
1772 if (((u32) (old >> 0) != (u32) ctxt->regs[VCPU_REGS_RAX]) ||
1773 ((u32) (old >> 32) != (u32) ctxt->regs[VCPU_REGS_RDX])) {
1774 ctxt->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1775 ctxt->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1776 ctxt->eflags &= ~EFLG_ZF;
1778 ctxt->dst.val64 = ((u64)ctxt->regs[VCPU_REGS_RCX] << 32) |
1779 (u32) ctxt->regs[VCPU_REGS_RBX];
1781 ctxt->eflags |= EFLG_ZF;
1783 return X86EMUL_CONTINUE;
1786 static int em_ret(struct x86_emulate_ctxt *ctxt)
1788 ctxt->dst.type = OP_REG;
1789 ctxt->dst.addr.reg = &ctxt->_eip;
1790 ctxt->dst.bytes = ctxt->op_bytes;
1791 return em_pop(ctxt);
1794 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
1799 rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
1800 if (rc != X86EMUL_CONTINUE)
1802 if (ctxt->op_bytes == 4)
1803 ctxt->_eip = (u32)ctxt->_eip;
1804 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1805 if (rc != X86EMUL_CONTINUE)
1807 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1811 static int emulate_load_segment(struct x86_emulate_ctxt *ctxt, int seg)
1816 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1818 rc = load_segment_descriptor(ctxt, sel, seg);
1819 if (rc != X86EMUL_CONTINUE)
1822 ctxt->dst.val = ctxt->src.val;
1827 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1828 struct desc_struct *cs, struct desc_struct *ss)
1832 memset(cs, 0, sizeof(struct desc_struct));
1833 ctxt->ops->get_segment(ctxt, &selector, cs, NULL, VCPU_SREG_CS);
1834 memset(ss, 0, sizeof(struct desc_struct));
1836 cs->l = 0; /* will be adjusted later */
1837 set_desc_base(cs, 0); /* flat segment */
1838 cs->g = 1; /* 4kb granularity */
1839 set_desc_limit(cs, 0xfffff); /* 4GB limit */
1840 cs->type = 0x0b; /* Read, Execute, Accessed */
1842 cs->dpl = 0; /* will be adjusted later */
1846 set_desc_base(ss, 0); /* flat segment */
1847 set_desc_limit(ss, 0xfffff); /* 4GB limit */
1848 ss->g = 1; /* 4kb granularity */
1850 ss->type = 0x03; /* Read/Write, Accessed */
1851 ss->d = 1; /* 32bit stack segment */
1856 static int em_syscall(struct x86_emulate_ctxt *ctxt)
1858 struct x86_emulate_ops *ops = ctxt->ops;
1859 struct desc_struct cs, ss;
1864 /* syscall is not available in real mode */
1865 if (ctxt->mode == X86EMUL_MODE_REAL ||
1866 ctxt->mode == X86EMUL_MODE_VM86)
1867 return emulate_ud(ctxt);
1869 ops->get_msr(ctxt, MSR_EFER, &efer);
1870 setup_syscalls_segments(ctxt, &cs, &ss);
1872 ops->get_msr(ctxt, MSR_STAR, &msr_data);
1874 cs_sel = (u16)(msr_data & 0xfffc);
1875 ss_sel = (u16)(msr_data + 8);
1877 if (efer & EFER_LMA) {
1881 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
1882 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
1884 ctxt->regs[VCPU_REGS_RCX] = ctxt->_eip;
1885 if (efer & EFER_LMA) {
1886 #ifdef CONFIG_X86_64
1887 ctxt->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
1890 ctxt->mode == X86EMUL_MODE_PROT64 ?
1891 MSR_LSTAR : MSR_CSTAR, &msr_data);
1892 ctxt->_eip = msr_data;
1894 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
1895 ctxt->eflags &= ~(msr_data | EFLG_RF);
1899 ops->get_msr(ctxt, MSR_STAR, &msr_data);
1900 ctxt->_eip = (u32)msr_data;
1902 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1905 return X86EMUL_CONTINUE;
1908 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
1910 struct x86_emulate_ops *ops = ctxt->ops;
1911 struct desc_struct cs, ss;
1916 ops->get_msr(ctxt, MSR_EFER, &efer);
1917 /* inject #GP if in real mode */
1918 if (ctxt->mode == X86EMUL_MODE_REAL)
1919 return emulate_gp(ctxt, 0);
1921 /* XXX sysenter/sysexit have not been tested in 64bit mode.
1922 * Therefore, we inject an #UD.
1924 if (ctxt->mode == X86EMUL_MODE_PROT64)
1925 return emulate_ud(ctxt);
1927 setup_syscalls_segments(ctxt, &cs, &ss);
1929 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
1930 switch (ctxt->mode) {
1931 case X86EMUL_MODE_PROT32:
1932 if ((msr_data & 0xfffc) == 0x0)
1933 return emulate_gp(ctxt, 0);
1935 case X86EMUL_MODE_PROT64:
1936 if (msr_data == 0x0)
1937 return emulate_gp(ctxt, 0);
1941 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1942 cs_sel = (u16)msr_data;
1943 cs_sel &= ~SELECTOR_RPL_MASK;
1944 ss_sel = cs_sel + 8;
1945 ss_sel &= ~SELECTOR_RPL_MASK;
1946 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
1951 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
1952 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
1954 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
1955 ctxt->_eip = msr_data;
1957 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
1958 ctxt->regs[VCPU_REGS_RSP] = msr_data;
1960 return X86EMUL_CONTINUE;
1963 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
1965 struct x86_emulate_ops *ops = ctxt->ops;
1966 struct desc_struct cs, ss;
1969 u16 cs_sel = 0, ss_sel = 0;
1971 /* inject #GP if in real mode or Virtual 8086 mode */
1972 if (ctxt->mode == X86EMUL_MODE_REAL ||
1973 ctxt->mode == X86EMUL_MODE_VM86)
1974 return emulate_gp(ctxt, 0);
1976 setup_syscalls_segments(ctxt, &cs, &ss);
1978 if ((ctxt->rex_prefix & 0x8) != 0x0)
1979 usermode = X86EMUL_MODE_PROT64;
1981 usermode = X86EMUL_MODE_PROT32;
1985 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
1987 case X86EMUL_MODE_PROT32:
1988 cs_sel = (u16)(msr_data + 16);
1989 if ((msr_data & 0xfffc) == 0x0)
1990 return emulate_gp(ctxt, 0);
1991 ss_sel = (u16)(msr_data + 24);
1993 case X86EMUL_MODE_PROT64:
1994 cs_sel = (u16)(msr_data + 32);
1995 if (msr_data == 0x0)
1996 return emulate_gp(ctxt, 0);
1997 ss_sel = cs_sel + 8;
2002 cs_sel |= SELECTOR_RPL_MASK;
2003 ss_sel |= SELECTOR_RPL_MASK;
2005 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2006 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2008 ctxt->_eip = ctxt->regs[VCPU_REGS_RDX];
2009 ctxt->regs[VCPU_REGS_RSP] = ctxt->regs[VCPU_REGS_RCX];
2011 return X86EMUL_CONTINUE;
2014 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2017 if (ctxt->mode == X86EMUL_MODE_REAL)
2019 if (ctxt->mode == X86EMUL_MODE_VM86)
2021 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2022 return ctxt->ops->cpl(ctxt) > iopl;
2025 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2028 struct x86_emulate_ops *ops = ctxt->ops;
2029 struct desc_struct tr_seg;
2032 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2033 unsigned mask = (1 << len) - 1;
2036 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2039 if (desc_limit_scaled(&tr_seg) < 103)
2041 base = get_desc_base(&tr_seg);
2042 #ifdef CONFIG_X86_64
2043 base |= ((u64)base3) << 32;
2045 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2046 if (r != X86EMUL_CONTINUE)
2048 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2050 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2051 if (r != X86EMUL_CONTINUE)
2053 if ((perm >> bit_idx) & mask)
2058 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2064 if (emulator_bad_iopl(ctxt))
2065 if (!emulator_io_port_access_allowed(ctxt, port, len))
2068 ctxt->perm_ok = true;
2073 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2074 struct tss_segment_16 *tss)
2076 tss->ip = ctxt->_eip;
2077 tss->flag = ctxt->eflags;
2078 tss->ax = ctxt->regs[VCPU_REGS_RAX];
2079 tss->cx = ctxt->regs[VCPU_REGS_RCX];
2080 tss->dx = ctxt->regs[VCPU_REGS_RDX];
2081 tss->bx = ctxt->regs[VCPU_REGS_RBX];
2082 tss->sp = ctxt->regs[VCPU_REGS_RSP];
2083 tss->bp = ctxt->regs[VCPU_REGS_RBP];
2084 tss->si = ctxt->regs[VCPU_REGS_RSI];
2085 tss->di = ctxt->regs[VCPU_REGS_RDI];
2087 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2088 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2089 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2090 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2091 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2094 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2095 struct tss_segment_16 *tss)
2099 ctxt->_eip = tss->ip;
2100 ctxt->eflags = tss->flag | 2;
2101 ctxt->regs[VCPU_REGS_RAX] = tss->ax;
2102 ctxt->regs[VCPU_REGS_RCX] = tss->cx;
2103 ctxt->regs[VCPU_REGS_RDX] = tss->dx;
2104 ctxt->regs[VCPU_REGS_RBX] = tss->bx;
2105 ctxt->regs[VCPU_REGS_RSP] = tss->sp;
2106 ctxt->regs[VCPU_REGS_RBP] = tss->bp;
2107 ctxt->regs[VCPU_REGS_RSI] = tss->si;
2108 ctxt->regs[VCPU_REGS_RDI] = tss->di;
2111 * SDM says that segment selectors are loaded before segment
2114 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2115 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2116 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2117 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2118 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2121 * Now load segment descriptors. If fault happenes at this stage
2122 * it is handled in a context of new task
2124 ret = load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR);
2125 if (ret != X86EMUL_CONTINUE)
2127 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2128 if (ret != X86EMUL_CONTINUE)
2130 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2131 if (ret != X86EMUL_CONTINUE)
2133 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2134 if (ret != X86EMUL_CONTINUE)
2136 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2137 if (ret != X86EMUL_CONTINUE)
2140 return X86EMUL_CONTINUE;
2143 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2144 u16 tss_selector, u16 old_tss_sel,
2145 ulong old_tss_base, struct desc_struct *new_desc)
2147 struct x86_emulate_ops *ops = ctxt->ops;
2148 struct tss_segment_16 tss_seg;
2150 u32 new_tss_base = get_desc_base(new_desc);
2152 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2154 if (ret != X86EMUL_CONTINUE)
2155 /* FIXME: need to provide precise fault address */
2158 save_state_to_tss16(ctxt, &tss_seg);
2160 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2162 if (ret != X86EMUL_CONTINUE)
2163 /* FIXME: need to provide precise fault address */
2166 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2168 if (ret != X86EMUL_CONTINUE)
2169 /* FIXME: need to provide precise fault address */
2172 if (old_tss_sel != 0xffff) {
2173 tss_seg.prev_task_link = old_tss_sel;
2175 ret = ops->write_std(ctxt, new_tss_base,
2176 &tss_seg.prev_task_link,
2177 sizeof tss_seg.prev_task_link,
2179 if (ret != X86EMUL_CONTINUE)
2180 /* FIXME: need to provide precise fault address */
2184 return load_state_from_tss16(ctxt, &tss_seg);
2187 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2188 struct tss_segment_32 *tss)
2190 tss->cr3 = ctxt->ops->get_cr(ctxt, 3);
2191 tss->eip = ctxt->_eip;
2192 tss->eflags = ctxt->eflags;
2193 tss->eax = ctxt->regs[VCPU_REGS_RAX];
2194 tss->ecx = ctxt->regs[VCPU_REGS_RCX];
2195 tss->edx = ctxt->regs[VCPU_REGS_RDX];
2196 tss->ebx = ctxt->regs[VCPU_REGS_RBX];
2197 tss->esp = ctxt->regs[VCPU_REGS_RSP];
2198 tss->ebp = ctxt->regs[VCPU_REGS_RBP];
2199 tss->esi = ctxt->regs[VCPU_REGS_RSI];
2200 tss->edi = ctxt->regs[VCPU_REGS_RDI];
2202 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2203 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2204 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2205 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2206 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2207 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2208 tss->ldt_selector = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2211 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2212 struct tss_segment_32 *tss)
2216 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2217 return emulate_gp(ctxt, 0);
2218 ctxt->_eip = tss->eip;
2219 ctxt->eflags = tss->eflags | 2;
2220 ctxt->regs[VCPU_REGS_RAX] = tss->eax;
2221 ctxt->regs[VCPU_REGS_RCX] = tss->ecx;
2222 ctxt->regs[VCPU_REGS_RDX] = tss->edx;
2223 ctxt->regs[VCPU_REGS_RBX] = tss->ebx;
2224 ctxt->regs[VCPU_REGS_RSP] = tss->esp;
2225 ctxt->regs[VCPU_REGS_RBP] = tss->ebp;
2226 ctxt->regs[VCPU_REGS_RSI] = tss->esi;
2227 ctxt->regs[VCPU_REGS_RDI] = tss->edi;
2230 * SDM says that segment selectors are loaded before segment
2233 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2234 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2235 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2236 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2237 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2238 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2239 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2242 * Now load segment descriptors. If fault happenes at this stage
2243 * it is handled in a context of new task
2245 ret = load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2246 if (ret != X86EMUL_CONTINUE)
2248 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2249 if (ret != X86EMUL_CONTINUE)
2251 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2252 if (ret != X86EMUL_CONTINUE)
2254 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2255 if (ret != X86EMUL_CONTINUE)
2257 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2258 if (ret != X86EMUL_CONTINUE)
2260 ret = load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS);
2261 if (ret != X86EMUL_CONTINUE)
2263 ret = load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS);
2264 if (ret != X86EMUL_CONTINUE)
2267 return X86EMUL_CONTINUE;
2270 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2271 u16 tss_selector, u16 old_tss_sel,
2272 ulong old_tss_base, struct desc_struct *new_desc)
2274 struct x86_emulate_ops *ops = ctxt->ops;
2275 struct tss_segment_32 tss_seg;
2277 u32 new_tss_base = get_desc_base(new_desc);
2279 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2281 if (ret != X86EMUL_CONTINUE)
2282 /* FIXME: need to provide precise fault address */
2285 save_state_to_tss32(ctxt, &tss_seg);
2287 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2289 if (ret != X86EMUL_CONTINUE)
2290 /* FIXME: need to provide precise fault address */
2293 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2295 if (ret != X86EMUL_CONTINUE)
2296 /* FIXME: need to provide precise fault address */
2299 if (old_tss_sel != 0xffff) {
2300 tss_seg.prev_task_link = old_tss_sel;
2302 ret = ops->write_std(ctxt, new_tss_base,
2303 &tss_seg.prev_task_link,
2304 sizeof tss_seg.prev_task_link,
2306 if (ret != X86EMUL_CONTINUE)
2307 /* FIXME: need to provide precise fault address */
2311 return load_state_from_tss32(ctxt, &tss_seg);
2314 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2315 u16 tss_selector, int reason,
2316 bool has_error_code, u32 error_code)
2318 struct x86_emulate_ops *ops = ctxt->ops;
2319 struct desc_struct curr_tss_desc, next_tss_desc;
2321 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2322 ulong old_tss_base =
2323 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2326 /* FIXME: old_tss_base == ~0 ? */
2328 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2329 if (ret != X86EMUL_CONTINUE)
2331 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2332 if (ret != X86EMUL_CONTINUE)
2335 /* FIXME: check that next_tss_desc is tss */
2337 if (reason != TASK_SWITCH_IRET) {
2338 if ((tss_selector & 3) > next_tss_desc.dpl ||
2339 ops->cpl(ctxt) > next_tss_desc.dpl)
2340 return emulate_gp(ctxt, 0);
2343 desc_limit = desc_limit_scaled(&next_tss_desc);
2344 if (!next_tss_desc.p ||
2345 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2346 desc_limit < 0x2b)) {
2347 emulate_ts(ctxt, tss_selector & 0xfffc);
2348 return X86EMUL_PROPAGATE_FAULT;
2351 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2352 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2353 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2356 if (reason == TASK_SWITCH_IRET)
2357 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2359 /* set back link to prev task only if NT bit is set in eflags
2360 note that old_tss_sel is not used afetr this point */
2361 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2362 old_tss_sel = 0xffff;
2364 if (next_tss_desc.type & 8)
2365 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2366 old_tss_base, &next_tss_desc);
2368 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2369 old_tss_base, &next_tss_desc);
2370 if (ret != X86EMUL_CONTINUE)
2373 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2374 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2376 if (reason != TASK_SWITCH_IRET) {
2377 next_tss_desc.type |= (1 << 1); /* set busy flag */
2378 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2381 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2382 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2384 if (has_error_code) {
2385 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2386 ctxt->lock_prefix = 0;
2387 ctxt->src.val = (unsigned long) error_code;
2388 ret = em_push(ctxt);
2394 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2395 u16 tss_selector, int reason,
2396 bool has_error_code, u32 error_code)
2400 ctxt->_eip = ctxt->eip;
2401 ctxt->dst.type = OP_NONE;
2403 rc = emulator_do_task_switch(ctxt, tss_selector, reason,
2404 has_error_code, error_code);
2406 if (rc == X86EMUL_CONTINUE)
2407 ctxt->eip = ctxt->_eip;
2409 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2412 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned seg,
2413 int reg, struct operand *op)
2415 int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;
2417 register_address_increment(ctxt, &ctxt->regs[reg], df * op->bytes);
2418 op->addr.mem.ea = register_address(ctxt, ctxt->regs[reg]);
2419 op->addr.mem.seg = seg;
2422 static int em_das(struct x86_emulate_ctxt *ctxt)
2425 bool af, cf, old_cf;
2427 cf = ctxt->eflags & X86_EFLAGS_CF;
2433 af = ctxt->eflags & X86_EFLAGS_AF;
2434 if ((al & 0x0f) > 9 || af) {
2436 cf = old_cf | (al >= 250);
2441 if (old_al > 0x99 || old_cf) {
2447 /* Set PF, ZF, SF */
2448 ctxt->src.type = OP_IMM;
2450 ctxt->src.bytes = 1;
2451 emulate_2op_SrcV(ctxt, "or");
2452 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2454 ctxt->eflags |= X86_EFLAGS_CF;
2456 ctxt->eflags |= X86_EFLAGS_AF;
2457 return X86EMUL_CONTINUE;
2460 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2466 old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2467 old_eip = ctxt->_eip;
2469 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2470 if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
2471 return X86EMUL_CONTINUE;
2474 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2476 ctxt->src.val = old_cs;
2478 if (rc != X86EMUL_CONTINUE)
2481 ctxt->src.val = old_eip;
2482 return em_push(ctxt);
2485 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2489 ctxt->dst.type = OP_REG;
2490 ctxt->dst.addr.reg = &ctxt->_eip;
2491 ctxt->dst.bytes = ctxt->op_bytes;
2492 rc = emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
2493 if (rc != X86EMUL_CONTINUE)
2495 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], ctxt->src.val);
2496 return X86EMUL_CONTINUE;
2499 static int em_add(struct x86_emulate_ctxt *ctxt)
2501 emulate_2op_SrcV(ctxt, "add");
2502 return X86EMUL_CONTINUE;
2505 static int em_or(struct x86_emulate_ctxt *ctxt)
2507 emulate_2op_SrcV(ctxt, "or");
2508 return X86EMUL_CONTINUE;
2511 static int em_adc(struct x86_emulate_ctxt *ctxt)
2513 emulate_2op_SrcV(ctxt, "adc");
2514 return X86EMUL_CONTINUE;
2517 static int em_sbb(struct x86_emulate_ctxt *ctxt)
2519 emulate_2op_SrcV(ctxt, "sbb");
2520 return X86EMUL_CONTINUE;
2523 static int em_and(struct x86_emulate_ctxt *ctxt)
2525 emulate_2op_SrcV(ctxt, "and");
2526 return X86EMUL_CONTINUE;
2529 static int em_sub(struct x86_emulate_ctxt *ctxt)
2531 emulate_2op_SrcV(ctxt, "sub");
2532 return X86EMUL_CONTINUE;
2535 static int em_xor(struct x86_emulate_ctxt *ctxt)
2537 emulate_2op_SrcV(ctxt, "xor");
2538 return X86EMUL_CONTINUE;
2541 static int em_cmp(struct x86_emulate_ctxt *ctxt)
2543 emulate_2op_SrcV(ctxt, "cmp");
2544 /* Disable writeback. */
2545 ctxt->dst.type = OP_NONE;
2546 return X86EMUL_CONTINUE;
2549 static int em_test(struct x86_emulate_ctxt *ctxt)
2551 emulate_2op_SrcV(ctxt, "test");
2552 /* Disable writeback. */
2553 ctxt->dst.type = OP_NONE;
2554 return X86EMUL_CONTINUE;
2557 static int em_xchg(struct x86_emulate_ctxt *ctxt)
2559 /* Write back the register source. */
2560 ctxt->src.val = ctxt->dst.val;
2561 write_register_operand(&ctxt->src);
2563 /* Write back the memory destination with implicit LOCK prefix. */
2564 ctxt->dst.val = ctxt->src.orig_val;
2565 ctxt->lock_prefix = 1;
2566 return X86EMUL_CONTINUE;
2569 static int em_imul(struct x86_emulate_ctxt *ctxt)
2571 emulate_2op_SrcV_nobyte(ctxt, "imul");
2572 return X86EMUL_CONTINUE;
2575 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2577 ctxt->dst.val = ctxt->src2.val;
2578 return em_imul(ctxt);
2581 static int em_cwd(struct x86_emulate_ctxt *ctxt)
2583 ctxt->dst.type = OP_REG;
2584 ctxt->dst.bytes = ctxt->src.bytes;
2585 ctxt->dst.addr.reg = &ctxt->regs[VCPU_REGS_RDX];
2586 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
2588 return X86EMUL_CONTINUE;
2591 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2595 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
2596 ctxt->regs[VCPU_REGS_RAX] = (u32)tsc;
2597 ctxt->regs[VCPU_REGS_RDX] = tsc >> 32;
2598 return X86EMUL_CONTINUE;
2601 static int em_mov(struct x86_emulate_ctxt *ctxt)
2603 ctxt->dst.val = ctxt->src.val;
2604 return X86EMUL_CONTINUE;
2607 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
2609 if (ctxt->modrm_reg > VCPU_SREG_GS)
2610 return emulate_ud(ctxt);
2612 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
2613 return X86EMUL_CONTINUE;
2616 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
2618 u16 sel = ctxt->src.val;
2620 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
2621 return emulate_ud(ctxt);
2623 if (ctxt->modrm_reg == VCPU_SREG_SS)
2624 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
2626 /* Disable writeback. */
2627 ctxt->dst.type = OP_NONE;
2628 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
2631 static int em_movdqu(struct x86_emulate_ctxt *ctxt)
2633 memcpy(&ctxt->dst.vec_val, &ctxt->src.vec_val, ctxt->op_bytes);
2634 return X86EMUL_CONTINUE;
2637 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
2642 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
2643 if (rc == X86EMUL_CONTINUE)
2644 ctxt->ops->invlpg(ctxt, linear);
2645 /* Disable writeback. */
2646 ctxt->dst.type = OP_NONE;
2647 return X86EMUL_CONTINUE;
2650 static int em_clts(struct x86_emulate_ctxt *ctxt)
2654 cr0 = ctxt->ops->get_cr(ctxt, 0);
2656 ctxt->ops->set_cr(ctxt, 0, cr0);
2657 return X86EMUL_CONTINUE;
2660 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
2664 if (ctxt->modrm_mod != 3 || ctxt->modrm_rm != 1)
2665 return X86EMUL_UNHANDLEABLE;
2667 rc = ctxt->ops->fix_hypercall(ctxt);
2668 if (rc != X86EMUL_CONTINUE)
2671 /* Let the processor re-execute the fixed hypercall */
2672 ctxt->_eip = ctxt->eip;
2673 /* Disable writeback. */
2674 ctxt->dst.type = OP_NONE;
2675 return X86EMUL_CONTINUE;
2678 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
2680 struct desc_ptr desc_ptr;
2683 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
2684 &desc_ptr.size, &desc_ptr.address,
2686 if (rc != X86EMUL_CONTINUE)
2688 ctxt->ops->set_gdt(ctxt, &desc_ptr);
2689 /* Disable writeback. */
2690 ctxt->dst.type = OP_NONE;
2691 return X86EMUL_CONTINUE;
2694 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
2698 rc = ctxt->ops->fix_hypercall(ctxt);
2700 /* Disable writeback. */
2701 ctxt->dst.type = OP_NONE;
2705 static int em_lidt(struct x86_emulate_ctxt *ctxt)
2707 struct desc_ptr desc_ptr;
2710 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
2711 &desc_ptr.size, &desc_ptr.address,
2713 if (rc != X86EMUL_CONTINUE)
2715 ctxt->ops->set_idt(ctxt, &desc_ptr);
2716 /* Disable writeback. */
2717 ctxt->dst.type = OP_NONE;
2718 return X86EMUL_CONTINUE;
2721 static int em_smsw(struct x86_emulate_ctxt *ctxt)
2723 ctxt->dst.bytes = 2;
2724 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
2725 return X86EMUL_CONTINUE;
2728 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
2730 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
2731 | (ctxt->src.val & 0x0f));
2732 ctxt->dst.type = OP_NONE;
2733 return X86EMUL_CONTINUE;
2736 static int em_loop(struct x86_emulate_ctxt *ctxt)
2738 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1);
2739 if ((address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) != 0) &&
2740 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
2741 jmp_rel(ctxt, ctxt->src.val);
2743 return X86EMUL_CONTINUE;
2746 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
2748 if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0)
2749 jmp_rel(ctxt, ctxt->src.val);
2751 return X86EMUL_CONTINUE;
2754 static int em_cli(struct x86_emulate_ctxt *ctxt)
2756 if (emulator_bad_iopl(ctxt))
2757 return emulate_gp(ctxt, 0);
2759 ctxt->eflags &= ~X86_EFLAGS_IF;
2760 return X86EMUL_CONTINUE;
2763 static int em_sti(struct x86_emulate_ctxt *ctxt)
2765 if (emulator_bad_iopl(ctxt))
2766 return emulate_gp(ctxt, 0);
2768 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
2769 ctxt->eflags |= X86_EFLAGS_IF;
2770 return X86EMUL_CONTINUE;
2773 static bool valid_cr(int nr)
2785 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
2787 if (!valid_cr(ctxt->modrm_reg))
2788 return emulate_ud(ctxt);
2790 return X86EMUL_CONTINUE;
2793 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
2795 u64 new_val = ctxt->src.val64;
2796 int cr = ctxt->modrm_reg;
2799 static u64 cr_reserved_bits[] = {
2800 0xffffffff00000000ULL,
2801 0, 0, 0, /* CR3 checked later */
2808 return emulate_ud(ctxt);
2810 if (new_val & cr_reserved_bits[cr])
2811 return emulate_gp(ctxt, 0);
2816 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
2817 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
2818 return emulate_gp(ctxt, 0);
2820 cr4 = ctxt->ops->get_cr(ctxt, 4);
2821 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2823 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
2824 !(cr4 & X86_CR4_PAE))
2825 return emulate_gp(ctxt, 0);
2832 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2833 if (efer & EFER_LMA)
2834 rsvd = CR3_L_MODE_RESERVED_BITS;
2835 else if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PAE)
2836 rsvd = CR3_PAE_RESERVED_BITS;
2837 else if (ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PG)
2838 rsvd = CR3_NONPAE_RESERVED_BITS;
2841 return emulate_gp(ctxt, 0);
2848 cr4 = ctxt->ops->get_cr(ctxt, 4);
2849 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2851 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
2852 return emulate_gp(ctxt, 0);
2858 return X86EMUL_CONTINUE;
2861 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
2865 ctxt->ops->get_dr(ctxt, 7, &dr7);
2867 /* Check if DR7.Global_Enable is set */
2868 return dr7 & (1 << 13);
2871 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
2873 int dr = ctxt->modrm_reg;
2877 return emulate_ud(ctxt);
2879 cr4 = ctxt->ops->get_cr(ctxt, 4);
2880 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
2881 return emulate_ud(ctxt);
2883 if (check_dr7_gd(ctxt))
2884 return emulate_db(ctxt);
2886 return X86EMUL_CONTINUE;
2889 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
2891 u64 new_val = ctxt->src.val64;
2892 int dr = ctxt->modrm_reg;
2894 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
2895 return emulate_gp(ctxt, 0);
2897 return check_dr_read(ctxt);
2900 static int check_svme(struct x86_emulate_ctxt *ctxt)
2904 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2906 if (!(efer & EFER_SVME))
2907 return emulate_ud(ctxt);
2909 return X86EMUL_CONTINUE;
2912 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
2914 u64 rax = ctxt->regs[VCPU_REGS_RAX];
2916 /* Valid physical address? */
2917 if (rax & 0xffff000000000000ULL)
2918 return emulate_gp(ctxt, 0);
2920 return check_svme(ctxt);
2923 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
2925 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
2927 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
2928 return emulate_ud(ctxt);
2930 return X86EMUL_CONTINUE;
2933 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
2935 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
2936 u64 rcx = ctxt->regs[VCPU_REGS_RCX];
2938 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
2940 return emulate_gp(ctxt, 0);
2942 return X86EMUL_CONTINUE;
2945 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
2947 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
2948 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
2949 return emulate_gp(ctxt, 0);
2951 return X86EMUL_CONTINUE;
2954 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
2956 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
2957 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
2958 return emulate_gp(ctxt, 0);
2960 return X86EMUL_CONTINUE;
2963 #define D(_y) { .flags = (_y) }
2964 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
2965 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
2966 .check_perm = (_p) }
2968 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
2969 #define G(_f, _g) { .flags = ((_f) | Group), .u.group = (_g) }
2970 #define GD(_f, _g) { .flags = ((_f) | GroupDual), .u.gdual = (_g) }
2971 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
2972 #define II(_f, _e, _i) \
2973 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
2974 #define IIP(_f, _e, _i, _p) \
2975 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
2976 .check_perm = (_p) }
2977 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
2979 #define D2bv(_f) D((_f) | ByteOp), D(_f)
2980 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
2981 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
2983 #define I6ALU(_f, _e) I2bv((_f) | DstMem | SrcReg | ModRM, _e), \
2984 I2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
2985 I2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
2987 static struct opcode group7_rm1[] = {
2988 DI(SrcNone | ModRM | Priv, monitor),
2989 DI(SrcNone | ModRM | Priv, mwait),
2993 static struct opcode group7_rm3[] = {
2994 DIP(SrcNone | ModRM | Prot | Priv, vmrun, check_svme_pa),
2995 II(SrcNone | ModRM | Prot | VendorSpecific, em_vmmcall, vmmcall),
2996 DIP(SrcNone | ModRM | Prot | Priv, vmload, check_svme_pa),
2997 DIP(SrcNone | ModRM | Prot | Priv, vmsave, check_svme_pa),
2998 DIP(SrcNone | ModRM | Prot | Priv, stgi, check_svme),
2999 DIP(SrcNone | ModRM | Prot | Priv, clgi, check_svme),
3000 DIP(SrcNone | ModRM | Prot | Priv, skinit, check_svme),
3001 DIP(SrcNone | ModRM | Prot | Priv, invlpga, check_svme),
3004 static struct opcode group7_rm7[] = {
3006 DIP(SrcNone | ModRM, rdtscp, check_rdtsc),
3010 static struct opcode group1[] = {
3021 static struct opcode group1A[] = {
3022 D(DstMem | SrcNone | ModRM | Mov | Stack), N, N, N, N, N, N, N,
3025 static struct opcode group3[] = {
3026 I(DstMem | SrcImm | ModRM, em_test),
3027 I(DstMem | SrcImm | ModRM, em_test),
3028 I(DstMem | SrcNone | ModRM | Lock, em_not),
3029 I(DstMem | SrcNone | ModRM | Lock, em_neg),
3030 I(SrcMem | ModRM, em_mul_ex),
3031 I(SrcMem | ModRM, em_imul_ex),
3032 I(SrcMem | ModRM, em_div_ex),
3033 I(SrcMem | ModRM, em_idiv_ex),
3036 static struct opcode group4[] = {
3037 D(ByteOp | DstMem | SrcNone | ModRM | Lock), D(ByteOp | DstMem | SrcNone | ModRM | Lock),
3041 static struct opcode group5[] = {
3042 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
3043 D(SrcMem | ModRM | Stack),
3044 I(SrcMemFAddr | ModRM | ImplicitOps | Stack, em_call_far),
3045 D(SrcMem | ModRM | Stack), D(SrcMemFAddr | ModRM | ImplicitOps),
3046 D(SrcMem | ModRM | Stack), N,
3049 static struct opcode group6[] = {
3050 DI(ModRM | Prot, sldt),
3051 DI(ModRM | Prot, str),
3052 DI(ModRM | Prot | Priv, lldt),
3053 DI(ModRM | Prot | Priv, ltr),
3057 static struct group_dual group7 = { {
3058 DI(ModRM | Mov | DstMem | Priv, sgdt),
3059 DI(ModRM | Mov | DstMem | Priv, sidt),
3060 II(ModRM | SrcMem | Priv, em_lgdt, lgdt),
3061 II(ModRM | SrcMem | Priv, em_lidt, lidt),
3062 II(SrcNone | ModRM | DstMem | Mov, em_smsw, smsw), N,
3063 II(SrcMem16 | ModRM | Mov | Priv, em_lmsw, lmsw),
3064 II(SrcMem | ModRM | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3066 I(SrcNone | ModRM | Priv | VendorSpecific, em_vmcall),
3068 N, EXT(0, group7_rm3),
3069 II(SrcNone | ModRM | DstMem | Mov, em_smsw, smsw), N,
3070 II(SrcMem16 | ModRM | Mov | Priv, em_lmsw, lmsw), EXT(0, group7_rm7),
3073 static struct opcode group8[] = {
3075 D(DstMem | SrcImmByte | ModRM), D(DstMem | SrcImmByte | ModRM | Lock),
3076 D(DstMem | SrcImmByte | ModRM | Lock), D(DstMem | SrcImmByte | ModRM | Lock),
3079 static struct group_dual group9 = { {
3080 N, D(DstMem64 | ModRM | Lock), N, N, N, N, N, N,
3082 N, N, N, N, N, N, N, N,
3085 static struct opcode group11[] = {
3086 I(DstMem | SrcImm | ModRM | Mov, em_mov), X7(D(Undefined)),
3089 static struct gprefix pfx_0f_6f_0f_7f = {
3090 N, N, N, I(Sse, em_movdqu),
3093 static struct opcode opcode_table[256] = {
3095 I6ALU(Lock, em_add),
3096 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
3099 D(ImplicitOps | Stack | No64), N,
3101 I6ALU(Lock, em_adc),
3102 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
3104 I6ALU(Lock, em_sbb),
3105 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
3107 I6ALU(Lock, em_and), N, N,
3109 I6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3111 I6ALU(Lock, em_xor), N, N,
3113 I6ALU(0, em_cmp), N, N,
3117 X8(I(SrcReg | Stack, em_push)),
3119 X8(I(DstReg | Stack, em_pop)),
3121 I(ImplicitOps | Stack | No64, em_pusha),
3122 I(ImplicitOps | Stack | No64, em_popa),
3123 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3126 I(SrcImm | Mov | Stack, em_push),
3127 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3128 I(SrcImmByte | Mov | Stack, em_push),
3129 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3130 D2bvIP(DstDI | SrcDX | Mov | String, ins, check_perm_in), /* insb, insw/insd */
3131 D2bvIP(SrcSI | DstDX | String, outs, check_perm_out), /* outsb, outsw/outsd */
3135 G(ByteOp | DstMem | SrcImm | ModRM | Group, group1),
3136 G(DstMem | SrcImm | ModRM | Group, group1),
3137 G(ByteOp | DstMem | SrcImm | ModRM | No64 | Group, group1),
3138 G(DstMem | SrcImmByte | ModRM | Group, group1),
3139 I2bv(DstMem | SrcReg | ModRM, em_test),
3140 I2bv(DstMem | SrcReg | ModRM | Lock, em_xchg),
3142 I2bv(DstMem | SrcReg | ModRM | Mov, em_mov),
3143 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3144 I(DstMem | SrcNone | ModRM | Mov, em_mov_rm_sreg),
3145 D(ModRM | SrcMem | NoAccess | DstReg),
3146 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
3149 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3151 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3152 I(SrcImmFAddr | No64, em_call_far), N,
3153 II(ImplicitOps | Stack, em_pushf, pushf),
3154 II(ImplicitOps | Stack, em_popf, popf), N, N,
3156 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3157 I2bv(DstMem | SrcAcc | Mov | MemAbs, em_mov),
3158 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3159 I2bv(SrcSI | DstDI | String, em_cmp),
3161 I2bv(DstAcc | SrcImm, em_test),
3162 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3163 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3164 I2bv(SrcAcc | DstDI | String, em_cmp),
3166 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3168 X8(I(DstReg | SrcImm | Mov, em_mov)),
3170 D2bv(DstMem | SrcImmByte | ModRM),
3171 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3172 I(ImplicitOps | Stack, em_ret),
3173 D(DstReg | SrcMemFAddr | ModRM | No64), D(DstReg | SrcMemFAddr | ModRM | No64),
3174 G(ByteOp, group11), G(0, group11),
3176 N, N, N, I(ImplicitOps | Stack, em_ret_far),
3177 D(ImplicitOps), DI(SrcImmByte, intn),
3178 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3180 D2bv(DstMem | SrcOne | ModRM), D2bv(DstMem | ModRM),
3183 N, N, N, N, N, N, N, N,
3185 X3(I(SrcImmByte, em_loop)),
3186 I(SrcImmByte, em_jcxz),
3187 D2bvIP(SrcImmUByte | DstAcc, in, check_perm_in),
3188 D2bvIP(SrcAcc | DstImmUByte, out, check_perm_out),
3190 D(SrcImm | Stack), D(SrcImm | ImplicitOps),
3191 I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
3192 D2bvIP(SrcDX | DstAcc, in, check_perm_in),
3193 D2bvIP(SrcAcc | DstDX, out, check_perm_out),
3195 N, DI(ImplicitOps, icebp), N, N,
3196 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
3197 G(ByteOp, group3), G(0, group3),
3199 D(ImplicitOps), D(ImplicitOps),
3200 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
3201 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3204 static struct opcode twobyte_table[256] = {
3206 G(0, group6), GD(0, &group7), N, N,
3207 N, I(ImplicitOps | VendorSpecific, em_syscall),
3208 II(ImplicitOps | Priv, em_clts, clts), N,
3209 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3210 N, D(ImplicitOps | ModRM), N, N,
3212 N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
3214 DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
3215 DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
3216 DIP(ModRM | SrcMem | Priv | Op3264, cr_write, check_cr_write),
3217 DIP(ModRM | SrcMem | Priv | Op3264, dr_write, check_dr_write),
3219 N, N, N, N, N, N, N, N,
3221 DI(ImplicitOps | Priv, wrmsr),
3222 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
3223 DI(ImplicitOps | Priv, rdmsr),
3224 DIP(ImplicitOps | Priv, rdpmc, check_rdpmc),
3225 I(ImplicitOps | VendorSpecific, em_sysenter),
3226 I(ImplicitOps | Priv | VendorSpecific, em_sysexit),
3228 N, N, N, N, N, N, N, N,
3230 X16(D(DstReg | SrcMem | ModRM | Mov)),
3232 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3237 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3242 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3246 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3248 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
3249 DI(ImplicitOps, cpuid), D(DstMem | SrcReg | ModRM | BitOp),
3250 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3251 D(DstMem | SrcReg | Src2CL | ModRM), N, N,
3253 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
3254 DI(ImplicitOps, rsm), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3255 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3256 D(DstMem | SrcReg | Src2CL | ModRM),
3257 D(ModRM), I(DstReg | SrcMem | ModRM, em_imul),
3259 D2bv(DstMem | SrcReg | ModRM | Lock),
3260 D(DstReg | SrcMemFAddr | ModRM), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3261 D(DstReg | SrcMemFAddr | ModRM), D(DstReg | SrcMemFAddr | ModRM),
3262 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3265 G(BitOp, group8), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3266 D(DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
3267 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3269 D2bv(DstMem | SrcReg | ModRM | Lock),
3270 N, D(DstMem | SrcReg | ModRM | Mov),
3271 N, N, N, GD(0, &group9),
3272 N, N, N, N, N, N, N, N,
3274 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3276 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3278 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
3294 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
3298 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3304 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
3305 unsigned size, bool sign_extension)
3307 int rc = X86EMUL_CONTINUE;
3311 op->addr.mem.ea = ctxt->_eip;
3312 /* NB. Immediates are sign-extended as necessary. */
3313 switch (op->bytes) {
3315 op->val = insn_fetch(s8, ctxt);
3318 op->val = insn_fetch(s16, ctxt);
3321 op->val = insn_fetch(s32, ctxt);
3324 if (!sign_extension) {
3325 switch (op->bytes) {
3333 op->val &= 0xffffffff;
3341 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
3344 int rc = X86EMUL_CONTINUE;
3348 decode_register_operand(ctxt, op,
3349 ctxt->twobyte && (ctxt->b == 0xb6 || ctxt->b == 0xb7));
3353 op->addr.mem.ea = ctxt->_eip;
3355 op->val = insn_fetch(u8, ctxt);
3364 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3365 if (ctxt->d & BitOp)
3366 fetch_bit_operand(ctxt);
3367 op->orig_val = op->val;
3371 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3372 op->addr.reg = &ctxt->regs[VCPU_REGS_RAX];
3373 fetch_register_operand(op);
3374 op->orig_val = op->val;
3378 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3380 register_address(ctxt, ctxt->regs[VCPU_REGS_RDI]);
3381 op->addr.mem.seg = VCPU_SREG_ES;
3387 op->addr.reg = &ctxt->regs[VCPU_REGS_RDX];
3388 fetch_register_operand(op);
3392 op->val = ctxt->regs[VCPU_REGS_RCX] & 0xff;
3395 rc = decode_imm(ctxt, op, 1, true);
3402 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
3405 /* Special instructions do their own operand decoding. */
3407 op->type = OP_NONE; /* Disable writeback. */
3415 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
3417 int rc = X86EMUL_CONTINUE;
3418 int mode = ctxt->mode;
3419 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
3420 bool op_prefix = false;
3421 struct opcode opcode;
3423 ctxt->memop.type = OP_NONE;
3424 ctxt->memopp = NULL;
3425 ctxt->_eip = ctxt->eip;
3426 ctxt->fetch.start = ctxt->_eip;
3427 ctxt->fetch.end = ctxt->fetch.start + insn_len;
3429 memcpy(ctxt->fetch.data, insn, insn_len);
3432 case X86EMUL_MODE_REAL:
3433 case X86EMUL_MODE_VM86:
3434 case X86EMUL_MODE_PROT16:
3435 def_op_bytes = def_ad_bytes = 2;
3437 case X86EMUL_MODE_PROT32:
3438 def_op_bytes = def_ad_bytes = 4;
3440 #ifdef CONFIG_X86_64
3441 case X86EMUL_MODE_PROT64:
3447 return EMULATION_FAILED;
3450 ctxt->op_bytes = def_op_bytes;
3451 ctxt->ad_bytes = def_ad_bytes;
3453 /* Legacy prefixes. */
3455 switch (ctxt->b = insn_fetch(u8, ctxt)) {
3456 case 0x66: /* operand-size override */
3458 /* switch between 2/4 bytes */
3459 ctxt->op_bytes = def_op_bytes ^ 6;
3461 case 0x67: /* address-size override */
3462 if (mode == X86EMUL_MODE_PROT64)
3463 /* switch between 4/8 bytes */
3464 ctxt->ad_bytes = def_ad_bytes ^ 12;
3466 /* switch between 2/4 bytes */
3467 ctxt->ad_bytes = def_ad_bytes ^ 6;
3469 case 0x26: /* ES override */
3470 case 0x2e: /* CS override */
3471 case 0x36: /* SS override */
3472 case 0x3e: /* DS override */
3473 set_seg_override(ctxt, (ctxt->b >> 3) & 3);
3475 case 0x64: /* FS override */
3476 case 0x65: /* GS override */
3477 set_seg_override(ctxt, ctxt->b & 7);
3479 case 0x40 ... 0x4f: /* REX */
3480 if (mode != X86EMUL_MODE_PROT64)
3482 ctxt->rex_prefix = ctxt->b;
3484 case 0xf0: /* LOCK */
3485 ctxt->lock_prefix = 1;
3487 case 0xf2: /* REPNE/REPNZ */
3488 case 0xf3: /* REP/REPE/REPZ */
3489 ctxt->rep_prefix = ctxt->b;
3495 /* Any legacy prefix after a REX prefix nullifies its effect. */
3497 ctxt->rex_prefix = 0;
3503 if (ctxt->rex_prefix & 8)
3504 ctxt->op_bytes = 8; /* REX.W */
3506 /* Opcode byte(s). */
3507 opcode = opcode_table[ctxt->b];
3508 /* Two-byte opcode? */
3509 if (ctxt->b == 0x0f) {
3511 ctxt->b = insn_fetch(u8, ctxt);
3512 opcode = twobyte_table[ctxt->b];
3514 ctxt->d = opcode.flags;
3516 while (ctxt->d & GroupMask) {
3517 switch (ctxt->d & GroupMask) {
3519 ctxt->modrm = insn_fetch(u8, ctxt);
3521 goffset = (ctxt->modrm >> 3) & 7;
3522 opcode = opcode.u.group[goffset];
3525 ctxt->modrm = insn_fetch(u8, ctxt);
3527 goffset = (ctxt->modrm >> 3) & 7;
3528 if ((ctxt->modrm >> 6) == 3)
3529 opcode = opcode.u.gdual->mod3[goffset];
3531 opcode = opcode.u.gdual->mod012[goffset];
3534 goffset = ctxt->modrm & 7;
3535 opcode = opcode.u.group[goffset];
3538 if (ctxt->rep_prefix && op_prefix)
3539 return EMULATION_FAILED;
3540 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
3541 switch (simd_prefix) {
3542 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
3543 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
3544 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
3545 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
3549 return EMULATION_FAILED;
3552 ctxt->d &= ~(u64)GroupMask;
3553 ctxt->d |= opcode.flags;
3556 ctxt->execute = opcode.u.execute;
3557 ctxt->check_perm = opcode.check_perm;
3558 ctxt->intercept = opcode.intercept;
3561 if (ctxt->d == 0 || (ctxt->d & Undefined))
3562 return EMULATION_FAILED;
3564 if (!(ctxt->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
3565 return EMULATION_FAILED;
3567 if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
3570 if (ctxt->d & Op3264) {
3571 if (mode == X86EMUL_MODE_PROT64)
3578 ctxt->op_bytes = 16;
3580 /* ModRM and SIB bytes. */
3581 if (ctxt->d & ModRM) {
3582 rc = decode_modrm(ctxt, &ctxt->memop);
3583 if (!ctxt->has_seg_override)
3584 set_seg_override(ctxt, ctxt->modrm_seg);
3585 } else if (ctxt->d & MemAbs)
3586 rc = decode_abs(ctxt, &ctxt->memop);
3587 if (rc != X86EMUL_CONTINUE)
3590 if (!ctxt->has_seg_override)
3591 set_seg_override(ctxt, VCPU_SREG_DS);
3593 ctxt->memop.addr.mem.seg = seg_override(ctxt);
3595 if (ctxt->memop.type == OP_MEM && ctxt->ad_bytes != 8)
3596 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
3599 * Decode and fetch the source operand: register, memory
3602 switch (ctxt->d & SrcMask) {
3606 decode_register_operand(ctxt, &ctxt->src, 0);
3609 ctxt->memop.bytes = 2;
3612 ctxt->memop.bytes = 4;
3615 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3617 ctxt->src = ctxt->memop;
3618 ctxt->memopp = &ctxt->src;
3621 rc = decode_imm(ctxt, &ctxt->src, 2, false);
3624 rc = decode_imm(ctxt, &ctxt->src, imm_size(ctxt), true);
3627 rc = decode_imm(ctxt, &ctxt->src, imm_size(ctxt), false);
3630 rc = decode_imm(ctxt, &ctxt->src, 1, true);
3633 rc = decode_imm(ctxt, &ctxt->src, 1, false);
3636 ctxt->src.type = OP_REG;
3637 ctxt->src.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3638 ctxt->src.addr.reg = &ctxt->regs[VCPU_REGS_RAX];
3639 fetch_register_operand(&ctxt->src);
3642 ctxt->src.bytes = 1;
3646 ctxt->src.type = OP_MEM;
3647 ctxt->src.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3648 ctxt->src.addr.mem.ea =
3649 register_address(ctxt, ctxt->regs[VCPU_REGS_RSI]);
3650 ctxt->src.addr.mem.seg = seg_override(ctxt);
3654 ctxt->src.type = OP_IMM;
3655 ctxt->src.addr.mem.ea = ctxt->_eip;
3656 ctxt->src.bytes = ctxt->op_bytes + 2;
3657 insn_fetch_arr(ctxt->src.valptr, ctxt->src.bytes, ctxt);
3660 ctxt->memop.bytes = ctxt->op_bytes + 2;
3664 ctxt->src.type = OP_REG;
3665 ctxt->src.bytes = 2;
3666 ctxt->src.addr.reg = &ctxt->regs[VCPU_REGS_RDX];
3667 fetch_register_operand(&ctxt->src);
3671 if (rc != X86EMUL_CONTINUE)
3675 * Decode and fetch the second source operand: register, memory
3678 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
3679 if (rc != X86EMUL_CONTINUE)
3682 /* Decode and fetch the destination operand: register or memory. */
3683 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
3686 if (ctxt->memopp && ctxt->memopp->type == OP_MEM && ctxt->rip_relative)
3687 ctxt->memopp->addr.mem.ea += ctxt->_eip;
3689 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
3692 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
3694 /* The second termination condition only applies for REPE
3695 * and REPNE. Test if the repeat string operation prefix is
3696 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
3697 * corresponding termination condition according to:
3698 * - if REPE/REPZ and ZF = 0 then done
3699 * - if REPNE/REPNZ and ZF = 1 then done
3701 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
3702 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
3703 && (((ctxt->rep_prefix == REPE_PREFIX) &&
3704 ((ctxt->eflags & EFLG_ZF) == 0))
3705 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
3706 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
3712 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
3714 struct x86_emulate_ops *ops = ctxt->ops;
3716 int rc = X86EMUL_CONTINUE;
3717 int saved_dst_type = ctxt->dst.type;
3719 ctxt->mem_read.pos = 0;
3721 if (ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) {
3722 rc = emulate_ud(ctxt);
3726 /* LOCK prefix is allowed only with some instructions */
3727 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
3728 rc = emulate_ud(ctxt);
3732 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
3733 rc = emulate_ud(ctxt);
3738 && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)
3739 || !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
3740 rc = emulate_ud(ctxt);
3744 if ((ctxt->d & Sse) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
3745 rc = emulate_nm(ctxt);
3749 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
3750 rc = emulator_check_intercept(ctxt, ctxt->intercept,
3751 X86_ICPT_PRE_EXCEPT);
3752 if (rc != X86EMUL_CONTINUE)
3756 /* Privileged instruction can be executed only in CPL=0 */
3757 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
3758 rc = emulate_gp(ctxt, 0);
3762 /* Instruction can only be executed in protected mode */
3763 if ((ctxt->d & Prot) && !(ctxt->mode & X86EMUL_MODE_PROT)) {
3764 rc = emulate_ud(ctxt);
3768 /* Do instruction specific permission checks */
3769 if (ctxt->check_perm) {
3770 rc = ctxt->check_perm(ctxt);
3771 if (rc != X86EMUL_CONTINUE)
3775 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
3776 rc = emulator_check_intercept(ctxt, ctxt->intercept,
3777 X86_ICPT_POST_EXCEPT);
3778 if (rc != X86EMUL_CONTINUE)
3782 if (ctxt->rep_prefix && (ctxt->d & String)) {
3783 /* All REP prefixes have the same first termination condition */
3784 if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0) {
3785 ctxt->eip = ctxt->_eip;
3790 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
3791 rc = segmented_read(ctxt, ctxt->src.addr.mem,
3792 ctxt->src.valptr, ctxt->src.bytes);
3793 if (rc != X86EMUL_CONTINUE)
3795 ctxt->src.orig_val64 = ctxt->src.val64;
3798 if (ctxt->src2.type == OP_MEM) {
3799 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
3800 &ctxt->src2.val, ctxt->src2.bytes);
3801 if (rc != X86EMUL_CONTINUE)
3805 if ((ctxt->d & DstMask) == ImplicitOps)
3809 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
3810 /* optimisation - avoid slow emulated read if Mov */
3811 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
3812 &ctxt->dst.val, ctxt->dst.bytes);
3813 if (rc != X86EMUL_CONTINUE)
3816 ctxt->dst.orig_val = ctxt->dst.val;
3820 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
3821 rc = emulator_check_intercept(ctxt, ctxt->intercept,
3822 X86_ICPT_POST_MEMACCESS);
3823 if (rc != X86EMUL_CONTINUE)
3827 if (ctxt->execute) {
3828 rc = ctxt->execute(ctxt);
3829 if (rc != X86EMUL_CONTINUE)
3838 case 0x06: /* push es */
3839 rc = emulate_push_sreg(ctxt, VCPU_SREG_ES);
3841 case 0x07: /* pop es */
3842 rc = emulate_pop_sreg(ctxt, VCPU_SREG_ES);
3844 case 0x0e: /* push cs */
3845 rc = emulate_push_sreg(ctxt, VCPU_SREG_CS);
3847 case 0x16: /* push ss */
3848 rc = emulate_push_sreg(ctxt, VCPU_SREG_SS);
3850 case 0x17: /* pop ss */
3851 rc = emulate_pop_sreg(ctxt, VCPU_SREG_SS);
3853 case 0x1e: /* push ds */
3854 rc = emulate_push_sreg(ctxt, VCPU_SREG_DS);
3856 case 0x1f: /* pop ds */
3857 rc = emulate_pop_sreg(ctxt, VCPU_SREG_DS);
3859 case 0x40 ... 0x47: /* inc r16/r32 */
3860 emulate_1op(ctxt, "inc");
3862 case 0x48 ... 0x4f: /* dec r16/r32 */
3863 emulate_1op(ctxt, "dec");
3865 case 0x63: /* movsxd */
3866 if (ctxt->mode != X86EMUL_MODE_PROT64)
3867 goto cannot_emulate;
3868 ctxt->dst.val = (s32) ctxt->src.val;
3870 case 0x6c: /* insb */
3871 case 0x6d: /* insw/insd */
3872 ctxt->src.val = ctxt->regs[VCPU_REGS_RDX];
3874 case 0x6e: /* outsb */
3875 case 0x6f: /* outsw/outsd */
3876 ctxt->dst.val = ctxt->regs[VCPU_REGS_RDX];
3879 case 0x70 ... 0x7f: /* jcc (short) */
3880 if (test_cc(ctxt->b, ctxt->eflags))
3881 jmp_rel(ctxt, ctxt->src.val);
3883 case 0x8d: /* lea r16/r32, m */
3884 ctxt->dst.val = ctxt->src.addr.mem.ea;
3886 case 0x8f: /* pop (sole member of Grp1a) */
3887 rc = em_grp1a(ctxt);
3889 case 0x90 ... 0x97: /* nop / xchg reg, rax */
3890 if (ctxt->dst.addr.reg == &ctxt->regs[VCPU_REGS_RAX])
3894 case 0x98: /* cbw/cwde/cdqe */
3895 switch (ctxt->op_bytes) {
3896 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
3897 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
3898 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
3904 case 0xc4: /* les */
3905 rc = emulate_load_segment(ctxt, VCPU_SREG_ES);
3907 case 0xc5: /* lds */
3908 rc = emulate_load_segment(ctxt, VCPU_SREG_DS);
3910 case 0xcc: /* int3 */
3911 rc = emulate_int(ctxt, 3);
3913 case 0xcd: /* int n */
3914 rc = emulate_int(ctxt, ctxt->src.val);
3916 case 0xce: /* into */
3917 if (ctxt->eflags & EFLG_OF)
3918 rc = emulate_int(ctxt, 4);
3920 case 0xd0 ... 0xd1: /* Grp2 */
3923 case 0xd2 ... 0xd3: /* Grp2 */
3924 ctxt->src.val = ctxt->regs[VCPU_REGS_RCX];
3927 case 0xe4: /* inb */
3930 case 0xe6: /* outb */
3931 case 0xe7: /* out */
3933 case 0xe8: /* call (near) */ {
3934 long int rel = ctxt->src.val;
3935 ctxt->src.val = (unsigned long) ctxt->_eip;
3940 case 0xe9: /* jmp rel */
3941 case 0xeb: /* jmp rel short */
3942 jmp_rel(ctxt, ctxt->src.val);
3943 ctxt->dst.type = OP_NONE; /* Disable writeback. */
3945 case 0xec: /* in al,dx */
3946 case 0xed: /* in (e/r)ax,dx */
3948 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3950 goto done; /* IO is needed */
3952 case 0xee: /* out dx,al */
3953 case 0xef: /* out dx,(e/r)ax */
3955 ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3957 ctxt->dst.type = OP_NONE; /* Disable writeback. */
3959 case 0xf4: /* hlt */
3960 ctxt->ops->halt(ctxt);
3962 case 0xf5: /* cmc */
3963 /* complement carry flag from eflags reg */
3964 ctxt->eflags ^= EFLG_CF;
3966 case 0xf8: /* clc */
3967 ctxt->eflags &= ~EFLG_CF;
3969 case 0xf9: /* stc */
3970 ctxt->eflags |= EFLG_CF;
3972 case 0xfc: /* cld */
3973 ctxt->eflags &= ~EFLG_DF;
3975 case 0xfd: /* std */
3976 ctxt->eflags |= EFLG_DF;
3978 case 0xfe: /* Grp4 */
3979 rc = em_grp45(ctxt);
3981 case 0xff: /* Grp5 */
3982 rc = em_grp45(ctxt);
3985 goto cannot_emulate;
3988 if (rc != X86EMUL_CONTINUE)
3992 rc = writeback(ctxt);
3993 if (rc != X86EMUL_CONTINUE)
3997 * restore dst type in case the decoding will be reused
3998 * (happens for string instruction )
4000 ctxt->dst.type = saved_dst_type;
4002 if ((ctxt->d & SrcMask) == SrcSI)
4003 string_addr_inc(ctxt, seg_override(ctxt),
4004 VCPU_REGS_RSI, &ctxt->src);
4006 if ((ctxt->d & DstMask) == DstDI)
4007 string_addr_inc(ctxt, VCPU_SREG_ES, VCPU_REGS_RDI,
4010 if (ctxt->rep_prefix && (ctxt->d & String)) {
4011 struct read_cache *r = &ctxt->io_read;
4012 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1);
4014 if (!string_insn_completed(ctxt)) {
4016 * Re-enter guest when pio read ahead buffer is empty
4017 * or, if it is not used, after each 1024 iteration.
4019 if ((r->end != 0 || ctxt->regs[VCPU_REGS_RCX] & 0x3ff) &&
4020 (r->end == 0 || r->end != r->pos)) {
4022 * Reset read cache. Usually happens before
4023 * decode, but since instruction is restarted
4024 * we have to do it here.
4026 ctxt->mem_read.end = 0;
4027 return EMULATION_RESTART;
4029 goto done; /* skip rip writeback */
4033 ctxt->eip = ctxt->_eip;
4036 if (rc == X86EMUL_PROPAGATE_FAULT)
4037 ctxt->have_exception = true;
4038 if (rc == X86EMUL_INTERCEPTED)
4039 return EMULATION_INTERCEPTED;
4041 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4045 case 0x09: /* wbinvd */
4046 (ctxt->ops->wbinvd)(ctxt);
4048 case 0x08: /* invd */
4049 case 0x0d: /* GrpP (prefetch) */
4050 case 0x18: /* Grp16 (prefetch/nop) */
4052 case 0x20: /* mov cr, reg */
4053 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
4055 case 0x21: /* mov from dr to reg */
4056 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
4058 case 0x22: /* mov reg, cr */
4059 if (ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val)) {
4060 emulate_gp(ctxt, 0);
4061 rc = X86EMUL_PROPAGATE_FAULT;
4064 ctxt->dst.type = OP_NONE;
4066 case 0x23: /* mov from reg to dr */
4067 if (ops->set_dr(ctxt, ctxt->modrm_reg, ctxt->src.val &
4068 ((ctxt->mode == X86EMUL_MODE_PROT64) ?
4069 ~0ULL : ~0U)) < 0) {
4070 /* #UD condition is already handled by the code above */
4071 emulate_gp(ctxt, 0);
4072 rc = X86EMUL_PROPAGATE_FAULT;
4076 ctxt->dst.type = OP_NONE; /* no writeback */
4080 msr_data = (u32)ctxt->regs[VCPU_REGS_RAX]
4081 | ((u64)ctxt->regs[VCPU_REGS_RDX] << 32);
4082 if (ops->set_msr(ctxt, ctxt->regs[VCPU_REGS_RCX], msr_data)) {
4083 emulate_gp(ctxt, 0);
4084 rc = X86EMUL_PROPAGATE_FAULT;
4087 rc = X86EMUL_CONTINUE;
4091 if (ops->get_msr(ctxt, ctxt->regs[VCPU_REGS_RCX], &msr_data)) {
4092 emulate_gp(ctxt, 0);
4093 rc = X86EMUL_PROPAGATE_FAULT;
4096 ctxt->regs[VCPU_REGS_RAX] = (u32)msr_data;
4097 ctxt->regs[VCPU_REGS_RDX] = msr_data >> 32;
4099 rc = X86EMUL_CONTINUE;
4101 case 0x40 ... 0x4f: /* cmov */
4102 ctxt->dst.val = ctxt->dst.orig_val = ctxt->src.val;
4103 if (!test_cc(ctxt->b, ctxt->eflags))
4104 ctxt->dst.type = OP_NONE; /* no writeback */
4106 case 0x80 ... 0x8f: /* jnz rel, etc*/
4107 if (test_cc(ctxt->b, ctxt->eflags))
4108 jmp_rel(ctxt, ctxt->src.val);
4110 case 0x90 ... 0x9f: /* setcc r/m8 */
4111 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
4113 case 0xa0: /* push fs */
4114 rc = emulate_push_sreg(ctxt, VCPU_SREG_FS);
4116 case 0xa1: /* pop fs */
4117 rc = emulate_pop_sreg(ctxt, VCPU_SREG_FS);
4121 ctxt->dst.type = OP_NONE;
4122 /* only subword offset */
4123 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
4124 emulate_2op_SrcV_nobyte(ctxt, "bt");
4126 case 0xa4: /* shld imm8, r, r/m */
4127 case 0xa5: /* shld cl, r, r/m */
4128 emulate_2op_cl(ctxt, "shld");
4130 case 0xa8: /* push gs */
4131 rc = emulate_push_sreg(ctxt, VCPU_SREG_GS);
4133 case 0xa9: /* pop gs */
4134 rc = emulate_pop_sreg(ctxt, VCPU_SREG_GS);
4138 emulate_2op_SrcV_nobyte(ctxt, "bts");
4140 case 0xac: /* shrd imm8, r, r/m */
4141 case 0xad: /* shrd cl, r, r/m */
4142 emulate_2op_cl(ctxt, "shrd");
4144 case 0xae: /* clflush */
4146 case 0xb0 ... 0xb1: /* cmpxchg */
4148 * Save real source value, then compare EAX against
4151 ctxt->src.orig_val = ctxt->src.val;
4152 ctxt->src.val = ctxt->regs[VCPU_REGS_RAX];
4153 emulate_2op_SrcV(ctxt, "cmp");
4154 if (ctxt->eflags & EFLG_ZF) {
4155 /* Success: write back to memory. */
4156 ctxt->dst.val = ctxt->src.orig_val;
4158 /* Failure: write the value we saw to EAX. */
4159 ctxt->dst.type = OP_REG;
4160 ctxt->dst.addr.reg = (unsigned long *)&ctxt->regs[VCPU_REGS_RAX];
4163 case 0xb2: /* lss */
4164 rc = emulate_load_segment(ctxt, VCPU_SREG_SS);
4168 emulate_2op_SrcV_nobyte(ctxt, "btr");
4170 case 0xb4: /* lfs */
4171 rc = emulate_load_segment(ctxt, VCPU_SREG_FS);
4173 case 0xb5: /* lgs */
4174 rc = emulate_load_segment(ctxt, VCPU_SREG_GS);
4176 case 0xb6 ... 0xb7: /* movzx */
4177 ctxt->dst.bytes = ctxt->op_bytes;
4178 ctxt->dst.val = (ctxt->d & ByteOp) ? (u8) ctxt->src.val
4179 : (u16) ctxt->src.val;
4181 case 0xba: /* Grp8 */
4182 switch (ctxt->modrm_reg & 3) {
4195 emulate_2op_SrcV_nobyte(ctxt, "btc");
4197 case 0xbc: { /* bsf */
4199 __asm__ ("bsf %2, %0; setz %1"
4200 : "=r"(ctxt->dst.val), "=q"(zf)
4201 : "r"(ctxt->src.val));
4202 ctxt->eflags &= ~X86_EFLAGS_ZF;
4204 ctxt->eflags |= X86_EFLAGS_ZF;
4205 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4209 case 0xbd: { /* bsr */
4211 __asm__ ("bsr %2, %0; setz %1"
4212 : "=r"(ctxt->dst.val), "=q"(zf)
4213 : "r"(ctxt->src.val));
4214 ctxt->eflags &= ~X86_EFLAGS_ZF;
4216 ctxt->eflags |= X86_EFLAGS_ZF;
4217 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4221 case 0xbe ... 0xbf: /* movsx */
4222 ctxt->dst.bytes = ctxt->op_bytes;
4223 ctxt->dst.val = (ctxt->d & ByteOp) ? (s8) ctxt->src.val :
4224 (s16) ctxt->src.val;
4226 case 0xc0 ... 0xc1: /* xadd */
4227 emulate_2op_SrcV(ctxt, "add");
4228 /* Write back the register source. */
4229 ctxt->src.val = ctxt->dst.orig_val;
4230 write_register_operand(&ctxt->src);
4232 case 0xc3: /* movnti */
4233 ctxt->dst.bytes = ctxt->op_bytes;
4234 ctxt->dst.val = (ctxt->op_bytes == 4) ? (u32) ctxt->src.val :
4235 (u64) ctxt->src.val;
4237 case 0xc7: /* Grp9 (cmpxchg8b) */
4241 goto cannot_emulate;
4244 if (rc != X86EMUL_CONTINUE)
4250 return EMULATION_FAILED;