1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
35 #define OpImplicit 1 /* No generic decode */
36 #define OpReg 2 /* Register */
37 #define OpMem 3 /* Memory */
38 #define OpAcc 4 /* Accumulator: AL/AX/EAX/RAX */
39 #define OpDI 5 /* ES:DI/EDI/RDI */
40 #define OpMem64 6 /* Memory, 64-bit */
41 #define OpImmUByte 7 /* Zero-extended 8-bit immediate */
42 #define OpDX 8 /* DX register */
44 #define OpBits 4 /* Width of operand field */
45 #define OpMask ((1 << OpBits) - 1)
48 * Opcode effective-address decode tables.
49 * Note that we only emulate instructions that have at least one memory
50 * operand (excluding implicit stack references). We assume that stack
51 * references and instruction fetches will never occur in special memory
52 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
56 /* Operand sizes: 8-bit operands or specified/overridden size. */
57 #define ByteOp (1<<0) /* 8-bit operands. */
58 /* Destination operand type. */
60 #define ImplicitOps (OpImplicit << DstShift)
61 #define DstReg (OpReg << DstShift)
62 #define DstMem (OpMem << DstShift)
63 #define DstAcc (OpAcc << DstShift)
64 #define DstDI (OpDI << DstShift)
65 #define DstMem64 (OpMem64 << DstShift)
66 #define DstImmUByte (OpImmUByte << DstShift)
67 #define DstDX (OpDX << DstShift)
68 #define DstMask (OpMask << DstShift)
69 /* Source operand type. */
70 #define SrcNone (0<<5) /* No source operand. */
71 #define SrcReg (1<<5) /* Register operand. */
72 #define SrcMem (2<<5) /* Memory operand. */
73 #define SrcMem16 (3<<5) /* Memory operand (16-bit). */
74 #define SrcMem32 (4<<5) /* Memory operand (32-bit). */
75 #define SrcImm (5<<5) /* Immediate operand. */
76 #define SrcImmByte (6<<5) /* 8-bit sign-extended immediate operand. */
77 #define SrcOne (7<<5) /* Implied '1' */
78 #define SrcImmUByte (8<<5) /* 8-bit unsigned immediate operand. */
79 #define SrcImmU (9<<5) /* Immediate operand, unsigned */
80 #define SrcSI (0xa<<5) /* Source is in the DS:RSI */
81 #define SrcImmFAddr (0xb<<5) /* Source is immediate far address */
82 #define SrcMemFAddr (0xc<<5) /* Source is far address in memory */
83 #define SrcAcc (0xd<<5) /* Source Accumulator */
84 #define SrcImmU16 (0xe<<5) /* Immediate operand, unsigned, 16 bits */
85 #define SrcDX (0xf<<5) /* Source is in DX register */
86 #define SrcMask (0xf<<5)
87 /* Generic ModRM decode. */
89 /* Destination is only written; never read. */
92 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
93 #define String (1<<13) /* String instruction (rep capable) */
94 #define Stack (1<<14) /* Stack instruction (push/pop) */
95 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
96 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
97 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
98 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
99 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
100 #define Sse (1<<18) /* SSE Vector instruction */
102 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
103 #define VendorSpecific (1<<22) /* Vendor specific instruction */
104 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
105 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
106 #define Undefined (1<<25) /* No Such Instruction */
107 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
108 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
110 /* Source 2 operand type */
111 #define Src2None (0<<29)
112 #define Src2CL (1<<29)
113 #define Src2ImmByte (2<<29)
114 #define Src2One (3<<29)
115 #define Src2Imm (4<<29)
116 #define Src2Mask (7<<29)
118 #define X2(x...) x, x
119 #define X3(x...) X2(x), x
120 #define X4(x...) X2(x), X2(x)
121 #define X5(x...) X4(x), x
122 #define X6(x...) X4(x), X2(x)
123 #define X7(x...) X4(x), X3(x)
124 #define X8(x...) X4(x), X4(x)
125 #define X16(x...) X8(x), X8(x)
131 int (*execute)(struct x86_emulate_ctxt *ctxt);
132 struct opcode *group;
133 struct group_dual *gdual;
134 struct gprefix *gprefix;
136 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
140 struct opcode mod012[8];
141 struct opcode mod3[8];
145 struct opcode pfx_no;
146 struct opcode pfx_66;
147 struct opcode pfx_f2;
148 struct opcode pfx_f3;
151 /* EFLAGS bit definitions. */
152 #define EFLG_ID (1<<21)
153 #define EFLG_VIP (1<<20)
154 #define EFLG_VIF (1<<19)
155 #define EFLG_AC (1<<18)
156 #define EFLG_VM (1<<17)
157 #define EFLG_RF (1<<16)
158 #define EFLG_IOPL (3<<12)
159 #define EFLG_NT (1<<14)
160 #define EFLG_OF (1<<11)
161 #define EFLG_DF (1<<10)
162 #define EFLG_IF (1<<9)
163 #define EFLG_TF (1<<8)
164 #define EFLG_SF (1<<7)
165 #define EFLG_ZF (1<<6)
166 #define EFLG_AF (1<<4)
167 #define EFLG_PF (1<<2)
168 #define EFLG_CF (1<<0)
170 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
171 #define EFLG_RESERVED_ONE_MASK 2
174 * Instruction emulation:
175 * Most instructions are emulated directly via a fragment of inline assembly
176 * code. This allows us to save/restore EFLAGS and thus very easily pick up
177 * any modified flags.
180 #if defined(CONFIG_X86_64)
181 #define _LO32 "k" /* force 32-bit operand */
182 #define _STK "%%rsp" /* stack pointer */
183 #elif defined(__i386__)
184 #define _LO32 "" /* force 32-bit operand */
185 #define _STK "%%esp" /* stack pointer */
189 * These EFLAGS bits are restored from saved value during emulation, and
190 * any changes are written back to the saved value after emulation.
192 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
194 /* Before executing instruction: restore necessary bits in EFLAGS. */
195 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
196 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
197 "movl %"_sav",%"_LO32 _tmp"; " \
200 "movl %"_msk",%"_LO32 _tmp"; " \
201 "andl %"_LO32 _tmp",("_STK"); " \
203 "notl %"_LO32 _tmp"; " \
204 "andl %"_LO32 _tmp",("_STK"); " \
205 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
207 "orl %"_LO32 _tmp",("_STK"); " \
211 /* After executing instruction: write-back necessary bits in EFLAGS. */
212 #define _POST_EFLAGS(_sav, _msk, _tmp) \
213 /* _sav |= EFLAGS & _msk; */ \
216 "andl %"_msk",%"_LO32 _tmp"; " \
217 "orl %"_LO32 _tmp",%"_sav"; "
225 #define ____emulate_2op(ctxt, _op, _x, _y, _suffix, _dsttype) \
227 __asm__ __volatile__ ( \
228 _PRE_EFLAGS("0", "4", "2") \
229 _op _suffix " %"_x"3,%1; " \
230 _POST_EFLAGS("0", "4", "2") \
231 : "=m" ((ctxt)->eflags), \
232 "+q" (*(_dsttype*)&(ctxt)->dst.val), \
234 : _y ((ctxt)->src.val), "i" (EFLAGS_MASK)); \
238 /* Raw emulation: instruction has two explicit operands. */
239 #define __emulate_2op_nobyte(ctxt,_op,_wx,_wy,_lx,_ly,_qx,_qy) \
241 unsigned long _tmp; \
243 switch ((ctxt)->dst.bytes) { \
245 ____emulate_2op(ctxt,_op,_wx,_wy,"w",u16); \
248 ____emulate_2op(ctxt,_op,_lx,_ly,"l",u32); \
251 ON64(____emulate_2op(ctxt,_op,_qx,_qy,"q",u64)); \
256 #define __emulate_2op(ctxt,_op,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
258 unsigned long _tmp; \
259 switch ((ctxt)->dst.bytes) { \
261 ____emulate_2op(ctxt,_op,_bx,_by,"b",u8); \
264 __emulate_2op_nobyte(ctxt, _op, \
265 _wx, _wy, _lx, _ly, _qx, _qy); \
270 /* Source operand is byte-sized and may be restricted to just %cl. */
271 #define emulate_2op_SrcB(ctxt, _op) \
272 __emulate_2op(ctxt, _op, "b", "c", "b", "c", "b", "c", "b", "c")
274 /* Source operand is byte, word, long or quad sized. */
275 #define emulate_2op_SrcV(ctxt, _op) \
276 __emulate_2op(ctxt, _op, "b", "q", "w", "r", _LO32, "r", "", "r")
278 /* Source operand is word, long or quad sized. */
279 #define emulate_2op_SrcV_nobyte(ctxt, _op) \
280 __emulate_2op_nobyte(ctxt, _op, "w", "r", _LO32, "r", "", "r")
282 /* Instruction has three operands and one operand is stored in ECX register */
283 #define __emulate_2op_cl(ctxt, _op, _suffix, _type) \
285 unsigned long _tmp; \
286 _type _clv = (ctxt)->src2.val; \
287 _type _srcv = (ctxt)->src.val; \
288 _type _dstv = (ctxt)->dst.val; \
290 __asm__ __volatile__ ( \
291 _PRE_EFLAGS("0", "5", "2") \
292 _op _suffix " %4,%1 \n" \
293 _POST_EFLAGS("0", "5", "2") \
294 : "=m" ((ctxt)->eflags), "+r" (_dstv), "=&r" (_tmp) \
295 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
298 (ctxt)->src2.val = (unsigned long) _clv; \
299 (ctxt)->src2.val = (unsigned long) _srcv; \
300 (ctxt)->dst.val = (unsigned long) _dstv; \
303 #define emulate_2op_cl(ctxt, _op) \
305 switch ((ctxt)->dst.bytes) { \
307 __emulate_2op_cl(ctxt, _op, "w", u16); \
310 __emulate_2op_cl(ctxt, _op, "l", u32); \
313 ON64(__emulate_2op_cl(ctxt, _op, "q", ulong)); \
318 #define __emulate_1op(ctxt, _op, _suffix) \
320 unsigned long _tmp; \
322 __asm__ __volatile__ ( \
323 _PRE_EFLAGS("0", "3", "2") \
324 _op _suffix " %1; " \
325 _POST_EFLAGS("0", "3", "2") \
326 : "=m" ((ctxt)->eflags), "+m" ((ctxt)->dst.val), \
328 : "i" (EFLAGS_MASK)); \
331 /* Instruction has only one explicit operand (no source operand). */
332 #define emulate_1op(ctxt, _op) \
334 switch ((ctxt)->dst.bytes) { \
335 case 1: __emulate_1op(ctxt, _op, "b"); break; \
336 case 2: __emulate_1op(ctxt, _op, "w"); break; \
337 case 4: __emulate_1op(ctxt, _op, "l"); break; \
338 case 8: ON64(__emulate_1op(ctxt, _op, "q")); break; \
342 #define __emulate_1op_rax_rdx(ctxt, _op, _suffix, _ex) \
344 unsigned long _tmp; \
345 ulong *rax = &(ctxt)->regs[VCPU_REGS_RAX]; \
346 ulong *rdx = &(ctxt)->regs[VCPU_REGS_RDX]; \
348 __asm__ __volatile__ ( \
349 _PRE_EFLAGS("0", "5", "1") \
351 _op _suffix " %6; " \
353 _POST_EFLAGS("0", "5", "1") \
354 ".pushsection .fixup,\"ax\" \n\t" \
355 "3: movb $1, %4 \n\t" \
358 _ASM_EXTABLE(1b, 3b) \
359 : "=m" ((ctxt)->eflags), "=&r" (_tmp), \
360 "+a" (*rax), "+d" (*rdx), "+qm"(_ex) \
361 : "i" (EFLAGS_MASK), "m" ((ctxt)->src.val), \
362 "a" (*rax), "d" (*rdx)); \
365 /* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
366 #define emulate_1op_rax_rdx(ctxt, _op, _ex) \
368 switch((ctxt)->src.bytes) { \
370 __emulate_1op_rax_rdx(ctxt, _op, "b", _ex); \
373 __emulate_1op_rax_rdx(ctxt, _op, "w", _ex); \
376 __emulate_1op_rax_rdx(ctxt, _op, "l", _ex); \
379 __emulate_1op_rax_rdx(ctxt, _op, "q", _ex)); \
384 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
385 enum x86_intercept intercept,
386 enum x86_intercept_stage stage)
388 struct x86_instruction_info info = {
389 .intercept = intercept,
390 .rep_prefix = ctxt->rep_prefix,
391 .modrm_mod = ctxt->modrm_mod,
392 .modrm_reg = ctxt->modrm_reg,
393 .modrm_rm = ctxt->modrm_rm,
394 .src_val = ctxt->src.val64,
395 .src_bytes = ctxt->src.bytes,
396 .dst_bytes = ctxt->dst.bytes,
397 .ad_bytes = ctxt->ad_bytes,
398 .next_rip = ctxt->eip,
401 return ctxt->ops->intercept(ctxt, &info, stage);
404 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
406 return (1UL << (ctxt->ad_bytes << 3)) - 1;
409 /* Access/update address held in a register, based on addressing mode. */
410 static inline unsigned long
411 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
413 if (ctxt->ad_bytes == sizeof(unsigned long))
416 return reg & ad_mask(ctxt);
419 static inline unsigned long
420 register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
422 return address_mask(ctxt, reg);
426 register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
428 if (ctxt->ad_bytes == sizeof(unsigned long))
431 *reg = (*reg & ~ad_mask(ctxt)) | ((*reg + inc) & ad_mask(ctxt));
434 static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
436 register_address_increment(ctxt, &ctxt->_eip, rel);
439 static u32 desc_limit_scaled(struct desc_struct *desc)
441 u32 limit = get_desc_limit(desc);
443 return desc->g ? (limit << 12) | 0xfff : limit;
446 static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
448 ctxt->has_seg_override = true;
449 ctxt->seg_override = seg;
452 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
454 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
457 return ctxt->ops->get_cached_segment_base(ctxt, seg);
460 static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
462 if (!ctxt->has_seg_override)
465 return ctxt->seg_override;
468 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
469 u32 error, bool valid)
471 ctxt->exception.vector = vec;
472 ctxt->exception.error_code = error;
473 ctxt->exception.error_code_valid = valid;
474 return X86EMUL_PROPAGATE_FAULT;
477 static int emulate_db(struct x86_emulate_ctxt *ctxt)
479 return emulate_exception(ctxt, DB_VECTOR, 0, false);
482 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
484 return emulate_exception(ctxt, GP_VECTOR, err, true);
487 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
489 return emulate_exception(ctxt, SS_VECTOR, err, true);
492 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
494 return emulate_exception(ctxt, UD_VECTOR, 0, false);
497 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
499 return emulate_exception(ctxt, TS_VECTOR, err, true);
502 static int emulate_de(struct x86_emulate_ctxt *ctxt)
504 return emulate_exception(ctxt, DE_VECTOR, 0, false);
507 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
509 return emulate_exception(ctxt, NM_VECTOR, 0, false);
512 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
515 struct desc_struct desc;
517 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
521 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
526 struct desc_struct desc;
528 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
529 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
532 static int __linearize(struct x86_emulate_ctxt *ctxt,
533 struct segmented_address addr,
534 unsigned size, bool write, bool fetch,
537 struct desc_struct desc;
544 la = seg_base(ctxt, addr.seg) + addr.ea;
545 switch (ctxt->mode) {
546 case X86EMUL_MODE_REAL:
548 case X86EMUL_MODE_PROT64:
549 if (((signed long)la << 16) >> 16 != la)
550 return emulate_gp(ctxt, 0);
553 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
557 /* code segment or read-only data segment */
558 if (((desc.type & 8) || !(desc.type & 2)) && write)
560 /* unreadable code segment */
561 if (!fetch && (desc.type & 8) && !(desc.type & 2))
563 lim = desc_limit_scaled(&desc);
564 if ((desc.type & 8) || !(desc.type & 4)) {
565 /* expand-up segment */
566 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
569 /* exapand-down segment */
570 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
572 lim = desc.d ? 0xffffffff : 0xffff;
573 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
576 cpl = ctxt->ops->cpl(ctxt);
579 if (!(desc.type & 8)) {
583 } else if ((desc.type & 8) && !(desc.type & 4)) {
584 /* nonconforming code segment */
587 } else if ((desc.type & 8) && (desc.type & 4)) {
588 /* conforming code segment */
594 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
597 return X86EMUL_CONTINUE;
599 if (addr.seg == VCPU_SREG_SS)
600 return emulate_ss(ctxt, addr.seg);
602 return emulate_gp(ctxt, addr.seg);
605 static int linearize(struct x86_emulate_ctxt *ctxt,
606 struct segmented_address addr,
607 unsigned size, bool write,
610 return __linearize(ctxt, addr, size, write, false, linear);
614 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
615 struct segmented_address addr,
622 rc = linearize(ctxt, addr, size, false, &linear);
623 if (rc != X86EMUL_CONTINUE)
625 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
629 * Fetch the next byte of the instruction being emulated which is pointed to
630 * by ctxt->_eip, then increment ctxt->_eip.
632 * Also prefetch the remaining bytes of the instruction without crossing page
633 * boundary if they are not in fetch_cache yet.
635 static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
637 struct fetch_cache *fc = &ctxt->fetch;
641 if (ctxt->_eip == fc->end) {
642 unsigned long linear;
643 struct segmented_address addr = { .seg = VCPU_SREG_CS,
645 cur_size = fc->end - fc->start;
646 size = min(15UL - cur_size,
647 PAGE_SIZE - offset_in_page(ctxt->_eip));
648 rc = __linearize(ctxt, addr, size, false, true, &linear);
649 if (unlikely(rc != X86EMUL_CONTINUE))
651 rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
652 size, &ctxt->exception);
653 if (unlikely(rc != X86EMUL_CONTINUE))
657 *dest = fc->data[ctxt->_eip - fc->start];
659 return X86EMUL_CONTINUE;
662 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
663 void *dest, unsigned size)
667 /* x86 instructions are limited to 15 bytes. */
668 if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
669 return X86EMUL_UNHANDLEABLE;
671 rc = do_insn_fetch_byte(ctxt, dest++);
672 if (rc != X86EMUL_CONTINUE)
675 return X86EMUL_CONTINUE;
678 /* Fetch next part of the instruction being emulated. */
679 #define insn_fetch(_type, _ctxt) \
680 ({ unsigned long _x; \
681 rc = do_insn_fetch(_ctxt, &_x, sizeof(_type)); \
682 if (rc != X86EMUL_CONTINUE) \
687 #define insn_fetch_arr(_arr, _size, _ctxt) \
688 ({ rc = do_insn_fetch(_ctxt, _arr, (_size)); \
689 if (rc != X86EMUL_CONTINUE) \
694 * Given the 'reg' portion of a ModRM byte, and a register block, return a
695 * pointer into the block that addresses the relevant register.
696 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
698 static void *decode_register(u8 modrm_reg, unsigned long *regs,
703 p = ®s[modrm_reg];
704 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
705 p = (unsigned char *)®s[modrm_reg & 3] + 1;
709 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
710 struct segmented_address addr,
711 u16 *size, unsigned long *address, int op_bytes)
718 rc = segmented_read_std(ctxt, addr, size, 2);
719 if (rc != X86EMUL_CONTINUE)
722 rc = segmented_read_std(ctxt, addr, address, op_bytes);
726 static int test_cc(unsigned int condition, unsigned int flags)
730 switch ((condition & 15) >> 1) {
732 rc |= (flags & EFLG_OF);
734 case 1: /* b/c/nae */
735 rc |= (flags & EFLG_CF);
738 rc |= (flags & EFLG_ZF);
741 rc |= (flags & (EFLG_CF|EFLG_ZF));
744 rc |= (flags & EFLG_SF);
747 rc |= (flags & EFLG_PF);
750 rc |= (flags & EFLG_ZF);
753 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
757 /* Odd condition identifiers (lsb == 1) have inverted sense. */
758 return (!!rc ^ (condition & 1));
761 static void fetch_register_operand(struct operand *op)
765 op->val = *(u8 *)op->addr.reg;
768 op->val = *(u16 *)op->addr.reg;
771 op->val = *(u32 *)op->addr.reg;
774 op->val = *(u64 *)op->addr.reg;
779 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
781 ctxt->ops->get_fpu(ctxt);
783 case 0: asm("movdqu %%xmm0, %0" : "=m"(*data)); break;
784 case 1: asm("movdqu %%xmm1, %0" : "=m"(*data)); break;
785 case 2: asm("movdqu %%xmm2, %0" : "=m"(*data)); break;
786 case 3: asm("movdqu %%xmm3, %0" : "=m"(*data)); break;
787 case 4: asm("movdqu %%xmm4, %0" : "=m"(*data)); break;
788 case 5: asm("movdqu %%xmm5, %0" : "=m"(*data)); break;
789 case 6: asm("movdqu %%xmm6, %0" : "=m"(*data)); break;
790 case 7: asm("movdqu %%xmm7, %0" : "=m"(*data)); break;
792 case 8: asm("movdqu %%xmm8, %0" : "=m"(*data)); break;
793 case 9: asm("movdqu %%xmm9, %0" : "=m"(*data)); break;
794 case 10: asm("movdqu %%xmm10, %0" : "=m"(*data)); break;
795 case 11: asm("movdqu %%xmm11, %0" : "=m"(*data)); break;
796 case 12: asm("movdqu %%xmm12, %0" : "=m"(*data)); break;
797 case 13: asm("movdqu %%xmm13, %0" : "=m"(*data)); break;
798 case 14: asm("movdqu %%xmm14, %0" : "=m"(*data)); break;
799 case 15: asm("movdqu %%xmm15, %0" : "=m"(*data)); break;
803 ctxt->ops->put_fpu(ctxt);
806 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
809 ctxt->ops->get_fpu(ctxt);
811 case 0: asm("movdqu %0, %%xmm0" : : "m"(*data)); break;
812 case 1: asm("movdqu %0, %%xmm1" : : "m"(*data)); break;
813 case 2: asm("movdqu %0, %%xmm2" : : "m"(*data)); break;
814 case 3: asm("movdqu %0, %%xmm3" : : "m"(*data)); break;
815 case 4: asm("movdqu %0, %%xmm4" : : "m"(*data)); break;
816 case 5: asm("movdqu %0, %%xmm5" : : "m"(*data)); break;
817 case 6: asm("movdqu %0, %%xmm6" : : "m"(*data)); break;
818 case 7: asm("movdqu %0, %%xmm7" : : "m"(*data)); break;
820 case 8: asm("movdqu %0, %%xmm8" : : "m"(*data)); break;
821 case 9: asm("movdqu %0, %%xmm9" : : "m"(*data)); break;
822 case 10: asm("movdqu %0, %%xmm10" : : "m"(*data)); break;
823 case 11: asm("movdqu %0, %%xmm11" : : "m"(*data)); break;
824 case 12: asm("movdqu %0, %%xmm12" : : "m"(*data)); break;
825 case 13: asm("movdqu %0, %%xmm13" : : "m"(*data)); break;
826 case 14: asm("movdqu %0, %%xmm14" : : "m"(*data)); break;
827 case 15: asm("movdqu %0, %%xmm15" : : "m"(*data)); break;
831 ctxt->ops->put_fpu(ctxt);
834 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
838 unsigned reg = ctxt->modrm_reg;
839 int highbyte_regs = ctxt->rex_prefix == 0;
841 if (!(ctxt->d & ModRM))
842 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
848 read_sse_reg(ctxt, &op->vec_val, reg);
853 if ((ctxt->d & ByteOp) && !inhibit_bytereg) {
854 op->addr.reg = decode_register(reg, ctxt->regs, highbyte_regs);
857 op->addr.reg = decode_register(reg, ctxt->regs, 0);
858 op->bytes = ctxt->op_bytes;
860 fetch_register_operand(op);
861 op->orig_val = op->val;
864 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
868 int index_reg = 0, base_reg = 0, scale;
869 int rc = X86EMUL_CONTINUE;
872 if (ctxt->rex_prefix) {
873 ctxt->modrm_reg = (ctxt->rex_prefix & 4) << 1; /* REX.R */
874 index_reg = (ctxt->rex_prefix & 2) << 2; /* REX.X */
875 ctxt->modrm_rm = base_reg = (ctxt->rex_prefix & 1) << 3; /* REG.B */
878 ctxt->modrm = insn_fetch(u8, ctxt);
879 ctxt->modrm_mod |= (ctxt->modrm & 0xc0) >> 6;
880 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
881 ctxt->modrm_rm |= (ctxt->modrm & 0x07);
882 ctxt->modrm_seg = VCPU_SREG_DS;
884 if (ctxt->modrm_mod == 3) {
886 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
887 op->addr.reg = decode_register(ctxt->modrm_rm,
888 ctxt->regs, ctxt->d & ByteOp);
892 op->addr.xmm = ctxt->modrm_rm;
893 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
896 fetch_register_operand(op);
902 if (ctxt->ad_bytes == 2) {
903 unsigned bx = ctxt->regs[VCPU_REGS_RBX];
904 unsigned bp = ctxt->regs[VCPU_REGS_RBP];
905 unsigned si = ctxt->regs[VCPU_REGS_RSI];
906 unsigned di = ctxt->regs[VCPU_REGS_RDI];
908 /* 16-bit ModR/M decode. */
909 switch (ctxt->modrm_mod) {
911 if (ctxt->modrm_rm == 6)
912 modrm_ea += insn_fetch(u16, ctxt);
915 modrm_ea += insn_fetch(s8, ctxt);
918 modrm_ea += insn_fetch(u16, ctxt);
921 switch (ctxt->modrm_rm) {
941 if (ctxt->modrm_mod != 0)
948 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
949 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
950 ctxt->modrm_seg = VCPU_SREG_SS;
951 modrm_ea = (u16)modrm_ea;
953 /* 32/64-bit ModR/M decode. */
954 if ((ctxt->modrm_rm & 7) == 4) {
955 sib = insn_fetch(u8, ctxt);
956 index_reg |= (sib >> 3) & 7;
960 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
961 modrm_ea += insn_fetch(s32, ctxt);
963 modrm_ea += ctxt->regs[base_reg];
965 modrm_ea += ctxt->regs[index_reg] << scale;
966 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
967 if (ctxt->mode == X86EMUL_MODE_PROT64)
968 ctxt->rip_relative = 1;
970 modrm_ea += ctxt->regs[ctxt->modrm_rm];
971 switch (ctxt->modrm_mod) {
973 if (ctxt->modrm_rm == 5)
974 modrm_ea += insn_fetch(s32, ctxt);
977 modrm_ea += insn_fetch(s8, ctxt);
980 modrm_ea += insn_fetch(s32, ctxt);
984 op->addr.mem.ea = modrm_ea;
989 static int decode_abs(struct x86_emulate_ctxt *ctxt,
992 int rc = X86EMUL_CONTINUE;
995 switch (ctxt->ad_bytes) {
997 op->addr.mem.ea = insn_fetch(u16, ctxt);
1000 op->addr.mem.ea = insn_fetch(u32, ctxt);
1003 op->addr.mem.ea = insn_fetch(u64, ctxt);
1010 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1014 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1015 mask = ~(ctxt->dst.bytes * 8 - 1);
1017 if (ctxt->src.bytes == 2)
1018 sv = (s16)ctxt->src.val & (s16)mask;
1019 else if (ctxt->src.bytes == 4)
1020 sv = (s32)ctxt->src.val & (s32)mask;
1022 ctxt->dst.addr.mem.ea += (sv >> 3);
1025 /* only subword offset */
1026 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1029 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1030 unsigned long addr, void *dest, unsigned size)
1033 struct read_cache *mc = &ctxt->mem_read;
1036 int n = min(size, 8u);
1038 if (mc->pos < mc->end)
1041 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, n,
1043 if (rc != X86EMUL_CONTINUE)
1048 memcpy(dest, mc->data + mc->pos, n);
1053 return X86EMUL_CONTINUE;
1056 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1057 struct segmented_address addr,
1064 rc = linearize(ctxt, addr, size, false, &linear);
1065 if (rc != X86EMUL_CONTINUE)
1067 return read_emulated(ctxt, linear, data, size);
1070 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1071 struct segmented_address addr,
1078 rc = linearize(ctxt, addr, size, true, &linear);
1079 if (rc != X86EMUL_CONTINUE)
1081 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1085 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1086 struct segmented_address addr,
1087 const void *orig_data, const void *data,
1093 rc = linearize(ctxt, addr, size, true, &linear);
1094 if (rc != X86EMUL_CONTINUE)
1096 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1097 size, &ctxt->exception);
1100 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1101 unsigned int size, unsigned short port,
1104 struct read_cache *rc = &ctxt->io_read;
1106 if (rc->pos == rc->end) { /* refill pio read ahead */
1107 unsigned int in_page, n;
1108 unsigned int count = ctxt->rep_prefix ?
1109 address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) : 1;
1110 in_page = (ctxt->eflags & EFLG_DF) ?
1111 offset_in_page(ctxt->regs[VCPU_REGS_RDI]) :
1112 PAGE_SIZE - offset_in_page(ctxt->regs[VCPU_REGS_RDI]);
1113 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1117 rc->pos = rc->end = 0;
1118 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1123 memcpy(dest, rc->data + rc->pos, size);
1128 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1129 u16 selector, struct desc_ptr *dt)
1131 struct x86_emulate_ops *ops = ctxt->ops;
1133 if (selector & 1 << 2) {
1134 struct desc_struct desc;
1137 memset (dt, 0, sizeof *dt);
1138 if (!ops->get_segment(ctxt, &sel, &desc, NULL, VCPU_SREG_LDTR))
1141 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1142 dt->address = get_desc_base(&desc);
1144 ops->get_gdt(ctxt, dt);
1147 /* allowed just for 8 bytes segments */
1148 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1149 u16 selector, struct desc_struct *desc)
1152 u16 index = selector >> 3;
1155 get_descriptor_table_ptr(ctxt, selector, &dt);
1157 if (dt.size < index * 8 + 7)
1158 return emulate_gp(ctxt, selector & 0xfffc);
1160 addr = dt.address + index * 8;
1161 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1165 /* allowed just for 8 bytes segments */
1166 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1167 u16 selector, struct desc_struct *desc)
1170 u16 index = selector >> 3;
1173 get_descriptor_table_ptr(ctxt, selector, &dt);
1175 if (dt.size < index * 8 + 7)
1176 return emulate_gp(ctxt, selector & 0xfffc);
1178 addr = dt.address + index * 8;
1179 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1183 /* Does not support long mode */
1184 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1185 u16 selector, int seg)
1187 struct desc_struct seg_desc;
1189 unsigned err_vec = GP_VECTOR;
1191 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1194 memset(&seg_desc, 0, sizeof seg_desc);
1196 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1197 || ctxt->mode == X86EMUL_MODE_REAL) {
1198 /* set real mode segment descriptor */
1199 set_desc_base(&seg_desc, selector << 4);
1200 set_desc_limit(&seg_desc, 0xffff);
1207 /* NULL selector is not valid for TR, CS and SS */
1208 if ((seg == VCPU_SREG_CS || seg == VCPU_SREG_SS || seg == VCPU_SREG_TR)
1212 /* TR should be in GDT only */
1213 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1216 if (null_selector) /* for NULL selector skip all following checks */
1219 ret = read_segment_descriptor(ctxt, selector, &seg_desc);
1220 if (ret != X86EMUL_CONTINUE)
1223 err_code = selector & 0xfffc;
1224 err_vec = GP_VECTOR;
1226 /* can't load system descriptor into segment selecor */
1227 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1231 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1237 cpl = ctxt->ops->cpl(ctxt);
1242 * segment is not a writable data segment or segment
1243 * selector's RPL != CPL or segment selector's RPL != CPL
1245 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1249 if (!(seg_desc.type & 8))
1252 if (seg_desc.type & 4) {
1258 if (rpl > cpl || dpl != cpl)
1261 /* CS(RPL) <- CPL */
1262 selector = (selector & 0xfffc) | cpl;
1265 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1268 case VCPU_SREG_LDTR:
1269 if (seg_desc.s || seg_desc.type != 2)
1272 default: /* DS, ES, FS, or GS */
1274 * segment is not a data or readable code segment or
1275 * ((segment is a data or nonconforming code segment)
1276 * and (both RPL and CPL > DPL))
1278 if ((seg_desc.type & 0xa) == 0x8 ||
1279 (((seg_desc.type & 0xc) != 0xc) &&
1280 (rpl > dpl && cpl > dpl)))
1286 /* mark segment as accessed */
1288 ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1289 if (ret != X86EMUL_CONTINUE)
1293 ctxt->ops->set_segment(ctxt, selector, &seg_desc, 0, seg);
1294 return X86EMUL_CONTINUE;
1296 emulate_exception(ctxt, err_vec, err_code, true);
1297 return X86EMUL_PROPAGATE_FAULT;
1300 static void write_register_operand(struct operand *op)
1302 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1303 switch (op->bytes) {
1305 *(u8 *)op->addr.reg = (u8)op->val;
1308 *(u16 *)op->addr.reg = (u16)op->val;
1311 *op->addr.reg = (u32)op->val;
1312 break; /* 64b: zero-extend */
1314 *op->addr.reg = op->val;
1319 static int writeback(struct x86_emulate_ctxt *ctxt)
1323 switch (ctxt->dst.type) {
1325 write_register_operand(&ctxt->dst);
1328 if (ctxt->lock_prefix)
1329 rc = segmented_cmpxchg(ctxt,
1331 &ctxt->dst.orig_val,
1335 rc = segmented_write(ctxt,
1339 if (rc != X86EMUL_CONTINUE)
1343 write_sse_reg(ctxt, &ctxt->dst.vec_val, ctxt->dst.addr.xmm);
1351 return X86EMUL_CONTINUE;
1354 static int em_push(struct x86_emulate_ctxt *ctxt)
1356 struct segmented_address addr;
1358 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], -ctxt->op_bytes);
1359 addr.ea = register_address(ctxt, ctxt->regs[VCPU_REGS_RSP]);
1360 addr.seg = VCPU_SREG_SS;
1362 /* Disable writeback. */
1363 ctxt->dst.type = OP_NONE;
1364 return segmented_write(ctxt, addr, &ctxt->src.val, ctxt->op_bytes);
1367 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1368 void *dest, int len)
1371 struct segmented_address addr;
1373 addr.ea = register_address(ctxt, ctxt->regs[VCPU_REGS_RSP]);
1374 addr.seg = VCPU_SREG_SS;
1375 rc = segmented_read(ctxt, addr, dest, len);
1376 if (rc != X86EMUL_CONTINUE)
1379 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], len);
1383 static int em_pop(struct x86_emulate_ctxt *ctxt)
1385 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1388 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1389 void *dest, int len)
1392 unsigned long val, change_mask;
1393 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1394 int cpl = ctxt->ops->cpl(ctxt);
1396 rc = emulate_pop(ctxt, &val, len);
1397 if (rc != X86EMUL_CONTINUE)
1400 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1401 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1403 switch(ctxt->mode) {
1404 case X86EMUL_MODE_PROT64:
1405 case X86EMUL_MODE_PROT32:
1406 case X86EMUL_MODE_PROT16:
1408 change_mask |= EFLG_IOPL;
1410 change_mask |= EFLG_IF;
1412 case X86EMUL_MODE_VM86:
1414 return emulate_gp(ctxt, 0);
1415 change_mask |= EFLG_IF;
1417 default: /* real mode */
1418 change_mask |= (EFLG_IOPL | EFLG_IF);
1422 *(unsigned long *)dest =
1423 (ctxt->eflags & ~change_mask) | (val & change_mask);
1428 static int em_popf(struct x86_emulate_ctxt *ctxt)
1430 ctxt->dst.type = OP_REG;
1431 ctxt->dst.addr.reg = &ctxt->eflags;
1432 ctxt->dst.bytes = ctxt->op_bytes;
1433 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1436 static int emulate_push_sreg(struct x86_emulate_ctxt *ctxt, int seg)
1438 ctxt->src.val = get_segment_selector(ctxt, seg);
1440 return em_push(ctxt);
1443 static int emulate_pop_sreg(struct x86_emulate_ctxt *ctxt, int seg)
1445 unsigned long selector;
1448 rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1449 if (rc != X86EMUL_CONTINUE)
1452 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1456 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1458 unsigned long old_esp = ctxt->regs[VCPU_REGS_RSP];
1459 int rc = X86EMUL_CONTINUE;
1460 int reg = VCPU_REGS_RAX;
1462 while (reg <= VCPU_REGS_RDI) {
1463 (reg == VCPU_REGS_RSP) ?
1464 (ctxt->src.val = old_esp) : (ctxt->src.val = ctxt->regs[reg]);
1467 if (rc != X86EMUL_CONTINUE)
1476 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1478 ctxt->src.val = (unsigned long)ctxt->eflags;
1479 return em_push(ctxt);
1482 static int em_popa(struct x86_emulate_ctxt *ctxt)
1484 int rc = X86EMUL_CONTINUE;
1485 int reg = VCPU_REGS_RDI;
1487 while (reg >= VCPU_REGS_RAX) {
1488 if (reg == VCPU_REGS_RSP) {
1489 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP],
1494 rc = emulate_pop(ctxt, &ctxt->regs[reg], ctxt->op_bytes);
1495 if (rc != X86EMUL_CONTINUE)
1502 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1504 struct x86_emulate_ops *ops = ctxt->ops;
1511 /* TODO: Add limit checks */
1512 ctxt->src.val = ctxt->eflags;
1514 if (rc != X86EMUL_CONTINUE)
1517 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1519 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1521 if (rc != X86EMUL_CONTINUE)
1524 ctxt->src.val = ctxt->_eip;
1526 if (rc != X86EMUL_CONTINUE)
1529 ops->get_idt(ctxt, &dt);
1531 eip_addr = dt.address + (irq << 2);
1532 cs_addr = dt.address + (irq << 2) + 2;
1534 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1535 if (rc != X86EMUL_CONTINUE)
1538 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1539 if (rc != X86EMUL_CONTINUE)
1542 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1543 if (rc != X86EMUL_CONTINUE)
1551 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1553 switch(ctxt->mode) {
1554 case X86EMUL_MODE_REAL:
1555 return emulate_int_real(ctxt, irq);
1556 case X86EMUL_MODE_VM86:
1557 case X86EMUL_MODE_PROT16:
1558 case X86EMUL_MODE_PROT32:
1559 case X86EMUL_MODE_PROT64:
1561 /* Protected mode interrupts unimplemented yet */
1562 return X86EMUL_UNHANDLEABLE;
1566 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
1568 int rc = X86EMUL_CONTINUE;
1569 unsigned long temp_eip = 0;
1570 unsigned long temp_eflags = 0;
1571 unsigned long cs = 0;
1572 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1573 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1574 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1575 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1577 /* TODO: Add stack limit check */
1579 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
1581 if (rc != X86EMUL_CONTINUE)
1584 if (temp_eip & ~0xffff)
1585 return emulate_gp(ctxt, 0);
1587 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1589 if (rc != X86EMUL_CONTINUE)
1592 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
1594 if (rc != X86EMUL_CONTINUE)
1597 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1599 if (rc != X86EMUL_CONTINUE)
1602 ctxt->_eip = temp_eip;
1605 if (ctxt->op_bytes == 4)
1606 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1607 else if (ctxt->op_bytes == 2) {
1608 ctxt->eflags &= ~0xffff;
1609 ctxt->eflags |= temp_eflags;
1612 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1613 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1618 static int em_iret(struct x86_emulate_ctxt *ctxt)
1620 switch(ctxt->mode) {
1621 case X86EMUL_MODE_REAL:
1622 return emulate_iret_real(ctxt);
1623 case X86EMUL_MODE_VM86:
1624 case X86EMUL_MODE_PROT16:
1625 case X86EMUL_MODE_PROT32:
1626 case X86EMUL_MODE_PROT64:
1628 /* iret from protected mode unimplemented yet */
1629 return X86EMUL_UNHANDLEABLE;
1633 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
1638 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1640 rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
1641 if (rc != X86EMUL_CONTINUE)
1645 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
1646 return X86EMUL_CONTINUE;
1649 static int em_grp1a(struct x86_emulate_ctxt *ctxt)
1651 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->dst.bytes);
1654 static int em_grp2(struct x86_emulate_ctxt *ctxt)
1656 switch (ctxt->modrm_reg) {
1658 emulate_2op_SrcB(ctxt, "rol");
1661 emulate_2op_SrcB(ctxt, "ror");
1664 emulate_2op_SrcB(ctxt, "rcl");
1667 emulate_2op_SrcB(ctxt, "rcr");
1669 case 4: /* sal/shl */
1670 case 6: /* sal/shl */
1671 emulate_2op_SrcB(ctxt, "sal");
1674 emulate_2op_SrcB(ctxt, "shr");
1677 emulate_2op_SrcB(ctxt, "sar");
1680 return X86EMUL_CONTINUE;
1683 static int em_not(struct x86_emulate_ctxt *ctxt)
1685 ctxt->dst.val = ~ctxt->dst.val;
1686 return X86EMUL_CONTINUE;
1689 static int em_neg(struct x86_emulate_ctxt *ctxt)
1691 emulate_1op(ctxt, "neg");
1692 return X86EMUL_CONTINUE;
1695 static int em_mul_ex(struct x86_emulate_ctxt *ctxt)
1699 emulate_1op_rax_rdx(ctxt, "mul", ex);
1700 return X86EMUL_CONTINUE;
1703 static int em_imul_ex(struct x86_emulate_ctxt *ctxt)
1707 emulate_1op_rax_rdx(ctxt, "imul", ex);
1708 return X86EMUL_CONTINUE;
1711 static int em_div_ex(struct x86_emulate_ctxt *ctxt)
1715 emulate_1op_rax_rdx(ctxt, "div", de);
1717 return emulate_de(ctxt);
1718 return X86EMUL_CONTINUE;
1721 static int em_idiv_ex(struct x86_emulate_ctxt *ctxt)
1725 emulate_1op_rax_rdx(ctxt, "idiv", de);
1727 return emulate_de(ctxt);
1728 return X86EMUL_CONTINUE;
1731 static int em_grp45(struct x86_emulate_ctxt *ctxt)
1733 int rc = X86EMUL_CONTINUE;
1735 switch (ctxt->modrm_reg) {
1737 emulate_1op(ctxt, "inc");
1740 emulate_1op(ctxt, "dec");
1742 case 2: /* call near abs */ {
1744 old_eip = ctxt->_eip;
1745 ctxt->_eip = ctxt->src.val;
1746 ctxt->src.val = old_eip;
1750 case 4: /* jmp abs */
1751 ctxt->_eip = ctxt->src.val;
1753 case 5: /* jmp far */
1754 rc = em_jmp_far(ctxt);
1763 static int em_grp9(struct x86_emulate_ctxt *ctxt)
1765 u64 old = ctxt->dst.orig_val64;
1767 if (((u32) (old >> 0) != (u32) ctxt->regs[VCPU_REGS_RAX]) ||
1768 ((u32) (old >> 32) != (u32) ctxt->regs[VCPU_REGS_RDX])) {
1769 ctxt->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1770 ctxt->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1771 ctxt->eflags &= ~EFLG_ZF;
1773 ctxt->dst.val64 = ((u64)ctxt->regs[VCPU_REGS_RCX] << 32) |
1774 (u32) ctxt->regs[VCPU_REGS_RBX];
1776 ctxt->eflags |= EFLG_ZF;
1778 return X86EMUL_CONTINUE;
1781 static int em_ret(struct x86_emulate_ctxt *ctxt)
1783 ctxt->dst.type = OP_REG;
1784 ctxt->dst.addr.reg = &ctxt->_eip;
1785 ctxt->dst.bytes = ctxt->op_bytes;
1786 return em_pop(ctxt);
1789 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
1794 rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
1795 if (rc != X86EMUL_CONTINUE)
1797 if (ctxt->op_bytes == 4)
1798 ctxt->_eip = (u32)ctxt->_eip;
1799 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1800 if (rc != X86EMUL_CONTINUE)
1802 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1806 static int emulate_load_segment(struct x86_emulate_ctxt *ctxt, int seg)
1811 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1813 rc = load_segment_descriptor(ctxt, sel, seg);
1814 if (rc != X86EMUL_CONTINUE)
1817 ctxt->dst.val = ctxt->src.val;
1822 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1823 struct desc_struct *cs, struct desc_struct *ss)
1827 memset(cs, 0, sizeof(struct desc_struct));
1828 ctxt->ops->get_segment(ctxt, &selector, cs, NULL, VCPU_SREG_CS);
1829 memset(ss, 0, sizeof(struct desc_struct));
1831 cs->l = 0; /* will be adjusted later */
1832 set_desc_base(cs, 0); /* flat segment */
1833 cs->g = 1; /* 4kb granularity */
1834 set_desc_limit(cs, 0xfffff); /* 4GB limit */
1835 cs->type = 0x0b; /* Read, Execute, Accessed */
1837 cs->dpl = 0; /* will be adjusted later */
1841 set_desc_base(ss, 0); /* flat segment */
1842 set_desc_limit(ss, 0xfffff); /* 4GB limit */
1843 ss->g = 1; /* 4kb granularity */
1845 ss->type = 0x03; /* Read/Write, Accessed */
1846 ss->d = 1; /* 32bit stack segment */
1851 static int em_syscall(struct x86_emulate_ctxt *ctxt)
1853 struct x86_emulate_ops *ops = ctxt->ops;
1854 struct desc_struct cs, ss;
1859 /* syscall is not available in real mode */
1860 if (ctxt->mode == X86EMUL_MODE_REAL ||
1861 ctxt->mode == X86EMUL_MODE_VM86)
1862 return emulate_ud(ctxt);
1864 ops->get_msr(ctxt, MSR_EFER, &efer);
1865 setup_syscalls_segments(ctxt, &cs, &ss);
1867 ops->get_msr(ctxt, MSR_STAR, &msr_data);
1869 cs_sel = (u16)(msr_data & 0xfffc);
1870 ss_sel = (u16)(msr_data + 8);
1872 if (efer & EFER_LMA) {
1876 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
1877 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
1879 ctxt->regs[VCPU_REGS_RCX] = ctxt->_eip;
1880 if (efer & EFER_LMA) {
1881 #ifdef CONFIG_X86_64
1882 ctxt->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
1885 ctxt->mode == X86EMUL_MODE_PROT64 ?
1886 MSR_LSTAR : MSR_CSTAR, &msr_data);
1887 ctxt->_eip = msr_data;
1889 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
1890 ctxt->eflags &= ~(msr_data | EFLG_RF);
1894 ops->get_msr(ctxt, MSR_STAR, &msr_data);
1895 ctxt->_eip = (u32)msr_data;
1897 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1900 return X86EMUL_CONTINUE;
1903 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
1905 struct x86_emulate_ops *ops = ctxt->ops;
1906 struct desc_struct cs, ss;
1911 ops->get_msr(ctxt, MSR_EFER, &efer);
1912 /* inject #GP if in real mode */
1913 if (ctxt->mode == X86EMUL_MODE_REAL)
1914 return emulate_gp(ctxt, 0);
1916 /* XXX sysenter/sysexit have not been tested in 64bit mode.
1917 * Therefore, we inject an #UD.
1919 if (ctxt->mode == X86EMUL_MODE_PROT64)
1920 return emulate_ud(ctxt);
1922 setup_syscalls_segments(ctxt, &cs, &ss);
1924 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
1925 switch (ctxt->mode) {
1926 case X86EMUL_MODE_PROT32:
1927 if ((msr_data & 0xfffc) == 0x0)
1928 return emulate_gp(ctxt, 0);
1930 case X86EMUL_MODE_PROT64:
1931 if (msr_data == 0x0)
1932 return emulate_gp(ctxt, 0);
1936 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1937 cs_sel = (u16)msr_data;
1938 cs_sel &= ~SELECTOR_RPL_MASK;
1939 ss_sel = cs_sel + 8;
1940 ss_sel &= ~SELECTOR_RPL_MASK;
1941 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
1946 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
1947 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
1949 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
1950 ctxt->_eip = msr_data;
1952 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
1953 ctxt->regs[VCPU_REGS_RSP] = msr_data;
1955 return X86EMUL_CONTINUE;
1958 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
1960 struct x86_emulate_ops *ops = ctxt->ops;
1961 struct desc_struct cs, ss;
1964 u16 cs_sel = 0, ss_sel = 0;
1966 /* inject #GP if in real mode or Virtual 8086 mode */
1967 if (ctxt->mode == X86EMUL_MODE_REAL ||
1968 ctxt->mode == X86EMUL_MODE_VM86)
1969 return emulate_gp(ctxt, 0);
1971 setup_syscalls_segments(ctxt, &cs, &ss);
1973 if ((ctxt->rex_prefix & 0x8) != 0x0)
1974 usermode = X86EMUL_MODE_PROT64;
1976 usermode = X86EMUL_MODE_PROT32;
1980 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
1982 case X86EMUL_MODE_PROT32:
1983 cs_sel = (u16)(msr_data + 16);
1984 if ((msr_data & 0xfffc) == 0x0)
1985 return emulate_gp(ctxt, 0);
1986 ss_sel = (u16)(msr_data + 24);
1988 case X86EMUL_MODE_PROT64:
1989 cs_sel = (u16)(msr_data + 32);
1990 if (msr_data == 0x0)
1991 return emulate_gp(ctxt, 0);
1992 ss_sel = cs_sel + 8;
1997 cs_sel |= SELECTOR_RPL_MASK;
1998 ss_sel |= SELECTOR_RPL_MASK;
2000 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2001 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2003 ctxt->_eip = ctxt->regs[VCPU_REGS_RDX];
2004 ctxt->regs[VCPU_REGS_RSP] = ctxt->regs[VCPU_REGS_RCX];
2006 return X86EMUL_CONTINUE;
2009 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2012 if (ctxt->mode == X86EMUL_MODE_REAL)
2014 if (ctxt->mode == X86EMUL_MODE_VM86)
2016 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2017 return ctxt->ops->cpl(ctxt) > iopl;
2020 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2023 struct x86_emulate_ops *ops = ctxt->ops;
2024 struct desc_struct tr_seg;
2027 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2028 unsigned mask = (1 << len) - 1;
2031 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2034 if (desc_limit_scaled(&tr_seg) < 103)
2036 base = get_desc_base(&tr_seg);
2037 #ifdef CONFIG_X86_64
2038 base |= ((u64)base3) << 32;
2040 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2041 if (r != X86EMUL_CONTINUE)
2043 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2045 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2046 if (r != X86EMUL_CONTINUE)
2048 if ((perm >> bit_idx) & mask)
2053 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2059 if (emulator_bad_iopl(ctxt))
2060 if (!emulator_io_port_access_allowed(ctxt, port, len))
2063 ctxt->perm_ok = true;
2068 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2069 struct tss_segment_16 *tss)
2071 tss->ip = ctxt->_eip;
2072 tss->flag = ctxt->eflags;
2073 tss->ax = ctxt->regs[VCPU_REGS_RAX];
2074 tss->cx = ctxt->regs[VCPU_REGS_RCX];
2075 tss->dx = ctxt->regs[VCPU_REGS_RDX];
2076 tss->bx = ctxt->regs[VCPU_REGS_RBX];
2077 tss->sp = ctxt->regs[VCPU_REGS_RSP];
2078 tss->bp = ctxt->regs[VCPU_REGS_RBP];
2079 tss->si = ctxt->regs[VCPU_REGS_RSI];
2080 tss->di = ctxt->regs[VCPU_REGS_RDI];
2082 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2083 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2084 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2085 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2086 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2089 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2090 struct tss_segment_16 *tss)
2094 ctxt->_eip = tss->ip;
2095 ctxt->eflags = tss->flag | 2;
2096 ctxt->regs[VCPU_REGS_RAX] = tss->ax;
2097 ctxt->regs[VCPU_REGS_RCX] = tss->cx;
2098 ctxt->regs[VCPU_REGS_RDX] = tss->dx;
2099 ctxt->regs[VCPU_REGS_RBX] = tss->bx;
2100 ctxt->regs[VCPU_REGS_RSP] = tss->sp;
2101 ctxt->regs[VCPU_REGS_RBP] = tss->bp;
2102 ctxt->regs[VCPU_REGS_RSI] = tss->si;
2103 ctxt->regs[VCPU_REGS_RDI] = tss->di;
2106 * SDM says that segment selectors are loaded before segment
2109 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2110 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2111 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2112 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2113 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2116 * Now load segment descriptors. If fault happenes at this stage
2117 * it is handled in a context of new task
2119 ret = load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR);
2120 if (ret != X86EMUL_CONTINUE)
2122 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2123 if (ret != X86EMUL_CONTINUE)
2125 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2126 if (ret != X86EMUL_CONTINUE)
2128 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2129 if (ret != X86EMUL_CONTINUE)
2131 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2132 if (ret != X86EMUL_CONTINUE)
2135 return X86EMUL_CONTINUE;
2138 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2139 u16 tss_selector, u16 old_tss_sel,
2140 ulong old_tss_base, struct desc_struct *new_desc)
2142 struct x86_emulate_ops *ops = ctxt->ops;
2143 struct tss_segment_16 tss_seg;
2145 u32 new_tss_base = get_desc_base(new_desc);
2147 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2149 if (ret != X86EMUL_CONTINUE)
2150 /* FIXME: need to provide precise fault address */
2153 save_state_to_tss16(ctxt, &tss_seg);
2155 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2157 if (ret != X86EMUL_CONTINUE)
2158 /* FIXME: need to provide precise fault address */
2161 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2163 if (ret != X86EMUL_CONTINUE)
2164 /* FIXME: need to provide precise fault address */
2167 if (old_tss_sel != 0xffff) {
2168 tss_seg.prev_task_link = old_tss_sel;
2170 ret = ops->write_std(ctxt, new_tss_base,
2171 &tss_seg.prev_task_link,
2172 sizeof tss_seg.prev_task_link,
2174 if (ret != X86EMUL_CONTINUE)
2175 /* FIXME: need to provide precise fault address */
2179 return load_state_from_tss16(ctxt, &tss_seg);
2182 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2183 struct tss_segment_32 *tss)
2185 tss->cr3 = ctxt->ops->get_cr(ctxt, 3);
2186 tss->eip = ctxt->_eip;
2187 tss->eflags = ctxt->eflags;
2188 tss->eax = ctxt->regs[VCPU_REGS_RAX];
2189 tss->ecx = ctxt->regs[VCPU_REGS_RCX];
2190 tss->edx = ctxt->regs[VCPU_REGS_RDX];
2191 tss->ebx = ctxt->regs[VCPU_REGS_RBX];
2192 tss->esp = ctxt->regs[VCPU_REGS_RSP];
2193 tss->ebp = ctxt->regs[VCPU_REGS_RBP];
2194 tss->esi = ctxt->regs[VCPU_REGS_RSI];
2195 tss->edi = ctxt->regs[VCPU_REGS_RDI];
2197 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2198 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2199 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2200 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2201 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2202 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2203 tss->ldt_selector = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2206 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2207 struct tss_segment_32 *tss)
2211 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2212 return emulate_gp(ctxt, 0);
2213 ctxt->_eip = tss->eip;
2214 ctxt->eflags = tss->eflags | 2;
2215 ctxt->regs[VCPU_REGS_RAX] = tss->eax;
2216 ctxt->regs[VCPU_REGS_RCX] = tss->ecx;
2217 ctxt->regs[VCPU_REGS_RDX] = tss->edx;
2218 ctxt->regs[VCPU_REGS_RBX] = tss->ebx;
2219 ctxt->regs[VCPU_REGS_RSP] = tss->esp;
2220 ctxt->regs[VCPU_REGS_RBP] = tss->ebp;
2221 ctxt->regs[VCPU_REGS_RSI] = tss->esi;
2222 ctxt->regs[VCPU_REGS_RDI] = tss->edi;
2225 * SDM says that segment selectors are loaded before segment
2228 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2229 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2230 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2231 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2232 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2233 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2234 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2237 * Now load segment descriptors. If fault happenes at this stage
2238 * it is handled in a context of new task
2240 ret = load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2241 if (ret != X86EMUL_CONTINUE)
2243 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2244 if (ret != X86EMUL_CONTINUE)
2246 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2247 if (ret != X86EMUL_CONTINUE)
2249 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2250 if (ret != X86EMUL_CONTINUE)
2252 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2253 if (ret != X86EMUL_CONTINUE)
2255 ret = load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS);
2256 if (ret != X86EMUL_CONTINUE)
2258 ret = load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS);
2259 if (ret != X86EMUL_CONTINUE)
2262 return X86EMUL_CONTINUE;
2265 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2266 u16 tss_selector, u16 old_tss_sel,
2267 ulong old_tss_base, struct desc_struct *new_desc)
2269 struct x86_emulate_ops *ops = ctxt->ops;
2270 struct tss_segment_32 tss_seg;
2272 u32 new_tss_base = get_desc_base(new_desc);
2274 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2276 if (ret != X86EMUL_CONTINUE)
2277 /* FIXME: need to provide precise fault address */
2280 save_state_to_tss32(ctxt, &tss_seg);
2282 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2284 if (ret != X86EMUL_CONTINUE)
2285 /* FIXME: need to provide precise fault address */
2288 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2290 if (ret != X86EMUL_CONTINUE)
2291 /* FIXME: need to provide precise fault address */
2294 if (old_tss_sel != 0xffff) {
2295 tss_seg.prev_task_link = old_tss_sel;
2297 ret = ops->write_std(ctxt, new_tss_base,
2298 &tss_seg.prev_task_link,
2299 sizeof tss_seg.prev_task_link,
2301 if (ret != X86EMUL_CONTINUE)
2302 /* FIXME: need to provide precise fault address */
2306 return load_state_from_tss32(ctxt, &tss_seg);
2309 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2310 u16 tss_selector, int reason,
2311 bool has_error_code, u32 error_code)
2313 struct x86_emulate_ops *ops = ctxt->ops;
2314 struct desc_struct curr_tss_desc, next_tss_desc;
2316 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2317 ulong old_tss_base =
2318 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2321 /* FIXME: old_tss_base == ~0 ? */
2323 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2324 if (ret != X86EMUL_CONTINUE)
2326 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2327 if (ret != X86EMUL_CONTINUE)
2330 /* FIXME: check that next_tss_desc is tss */
2332 if (reason != TASK_SWITCH_IRET) {
2333 if ((tss_selector & 3) > next_tss_desc.dpl ||
2334 ops->cpl(ctxt) > next_tss_desc.dpl)
2335 return emulate_gp(ctxt, 0);
2338 desc_limit = desc_limit_scaled(&next_tss_desc);
2339 if (!next_tss_desc.p ||
2340 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2341 desc_limit < 0x2b)) {
2342 emulate_ts(ctxt, tss_selector & 0xfffc);
2343 return X86EMUL_PROPAGATE_FAULT;
2346 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2347 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2348 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2351 if (reason == TASK_SWITCH_IRET)
2352 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2354 /* set back link to prev task only if NT bit is set in eflags
2355 note that old_tss_sel is not used afetr this point */
2356 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2357 old_tss_sel = 0xffff;
2359 if (next_tss_desc.type & 8)
2360 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2361 old_tss_base, &next_tss_desc);
2363 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2364 old_tss_base, &next_tss_desc);
2365 if (ret != X86EMUL_CONTINUE)
2368 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2369 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2371 if (reason != TASK_SWITCH_IRET) {
2372 next_tss_desc.type |= (1 << 1); /* set busy flag */
2373 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2376 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2377 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2379 if (has_error_code) {
2380 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2381 ctxt->lock_prefix = 0;
2382 ctxt->src.val = (unsigned long) error_code;
2383 ret = em_push(ctxt);
2389 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2390 u16 tss_selector, int reason,
2391 bool has_error_code, u32 error_code)
2395 ctxt->_eip = ctxt->eip;
2396 ctxt->dst.type = OP_NONE;
2398 rc = emulator_do_task_switch(ctxt, tss_selector, reason,
2399 has_error_code, error_code);
2401 if (rc == X86EMUL_CONTINUE)
2402 ctxt->eip = ctxt->_eip;
2404 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2407 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned seg,
2408 int reg, struct operand *op)
2410 int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;
2412 register_address_increment(ctxt, &ctxt->regs[reg], df * op->bytes);
2413 op->addr.mem.ea = register_address(ctxt, ctxt->regs[reg]);
2414 op->addr.mem.seg = seg;
2417 static int em_das(struct x86_emulate_ctxt *ctxt)
2420 bool af, cf, old_cf;
2422 cf = ctxt->eflags & X86_EFLAGS_CF;
2428 af = ctxt->eflags & X86_EFLAGS_AF;
2429 if ((al & 0x0f) > 9 || af) {
2431 cf = old_cf | (al >= 250);
2436 if (old_al > 0x99 || old_cf) {
2442 /* Set PF, ZF, SF */
2443 ctxt->src.type = OP_IMM;
2445 ctxt->src.bytes = 1;
2446 emulate_2op_SrcV(ctxt, "or");
2447 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2449 ctxt->eflags |= X86_EFLAGS_CF;
2451 ctxt->eflags |= X86_EFLAGS_AF;
2452 return X86EMUL_CONTINUE;
2455 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2461 old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2462 old_eip = ctxt->_eip;
2464 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2465 if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
2466 return X86EMUL_CONTINUE;
2469 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2471 ctxt->src.val = old_cs;
2473 if (rc != X86EMUL_CONTINUE)
2476 ctxt->src.val = old_eip;
2477 return em_push(ctxt);
2480 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2484 ctxt->dst.type = OP_REG;
2485 ctxt->dst.addr.reg = &ctxt->_eip;
2486 ctxt->dst.bytes = ctxt->op_bytes;
2487 rc = emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
2488 if (rc != X86EMUL_CONTINUE)
2490 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], ctxt->src.val);
2491 return X86EMUL_CONTINUE;
2494 static int em_add(struct x86_emulate_ctxt *ctxt)
2496 emulate_2op_SrcV(ctxt, "add");
2497 return X86EMUL_CONTINUE;
2500 static int em_or(struct x86_emulate_ctxt *ctxt)
2502 emulate_2op_SrcV(ctxt, "or");
2503 return X86EMUL_CONTINUE;
2506 static int em_adc(struct x86_emulate_ctxt *ctxt)
2508 emulate_2op_SrcV(ctxt, "adc");
2509 return X86EMUL_CONTINUE;
2512 static int em_sbb(struct x86_emulate_ctxt *ctxt)
2514 emulate_2op_SrcV(ctxt, "sbb");
2515 return X86EMUL_CONTINUE;
2518 static int em_and(struct x86_emulate_ctxt *ctxt)
2520 emulate_2op_SrcV(ctxt, "and");
2521 return X86EMUL_CONTINUE;
2524 static int em_sub(struct x86_emulate_ctxt *ctxt)
2526 emulate_2op_SrcV(ctxt, "sub");
2527 return X86EMUL_CONTINUE;
2530 static int em_xor(struct x86_emulate_ctxt *ctxt)
2532 emulate_2op_SrcV(ctxt, "xor");
2533 return X86EMUL_CONTINUE;
2536 static int em_cmp(struct x86_emulate_ctxt *ctxt)
2538 emulate_2op_SrcV(ctxt, "cmp");
2539 /* Disable writeback. */
2540 ctxt->dst.type = OP_NONE;
2541 return X86EMUL_CONTINUE;
2544 static int em_test(struct x86_emulate_ctxt *ctxt)
2546 emulate_2op_SrcV(ctxt, "test");
2547 /* Disable writeback. */
2548 ctxt->dst.type = OP_NONE;
2549 return X86EMUL_CONTINUE;
2552 static int em_xchg(struct x86_emulate_ctxt *ctxt)
2554 /* Write back the register source. */
2555 ctxt->src.val = ctxt->dst.val;
2556 write_register_operand(&ctxt->src);
2558 /* Write back the memory destination with implicit LOCK prefix. */
2559 ctxt->dst.val = ctxt->src.orig_val;
2560 ctxt->lock_prefix = 1;
2561 return X86EMUL_CONTINUE;
2564 static int em_imul(struct x86_emulate_ctxt *ctxt)
2566 emulate_2op_SrcV_nobyte(ctxt, "imul");
2567 return X86EMUL_CONTINUE;
2570 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2572 ctxt->dst.val = ctxt->src2.val;
2573 return em_imul(ctxt);
2576 static int em_cwd(struct x86_emulate_ctxt *ctxt)
2578 ctxt->dst.type = OP_REG;
2579 ctxt->dst.bytes = ctxt->src.bytes;
2580 ctxt->dst.addr.reg = &ctxt->regs[VCPU_REGS_RDX];
2581 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
2583 return X86EMUL_CONTINUE;
2586 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2590 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
2591 ctxt->regs[VCPU_REGS_RAX] = (u32)tsc;
2592 ctxt->regs[VCPU_REGS_RDX] = tsc >> 32;
2593 return X86EMUL_CONTINUE;
2596 static int em_mov(struct x86_emulate_ctxt *ctxt)
2598 ctxt->dst.val = ctxt->src.val;
2599 return X86EMUL_CONTINUE;
2602 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
2604 if (ctxt->modrm_reg > VCPU_SREG_GS)
2605 return emulate_ud(ctxt);
2607 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
2608 return X86EMUL_CONTINUE;
2611 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
2613 u16 sel = ctxt->src.val;
2615 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
2616 return emulate_ud(ctxt);
2618 if (ctxt->modrm_reg == VCPU_SREG_SS)
2619 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
2621 /* Disable writeback. */
2622 ctxt->dst.type = OP_NONE;
2623 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
2626 static int em_movdqu(struct x86_emulate_ctxt *ctxt)
2628 memcpy(&ctxt->dst.vec_val, &ctxt->src.vec_val, ctxt->op_bytes);
2629 return X86EMUL_CONTINUE;
2632 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
2637 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
2638 if (rc == X86EMUL_CONTINUE)
2639 ctxt->ops->invlpg(ctxt, linear);
2640 /* Disable writeback. */
2641 ctxt->dst.type = OP_NONE;
2642 return X86EMUL_CONTINUE;
2645 static int em_clts(struct x86_emulate_ctxt *ctxt)
2649 cr0 = ctxt->ops->get_cr(ctxt, 0);
2651 ctxt->ops->set_cr(ctxt, 0, cr0);
2652 return X86EMUL_CONTINUE;
2655 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
2659 if (ctxt->modrm_mod != 3 || ctxt->modrm_rm != 1)
2660 return X86EMUL_UNHANDLEABLE;
2662 rc = ctxt->ops->fix_hypercall(ctxt);
2663 if (rc != X86EMUL_CONTINUE)
2666 /* Let the processor re-execute the fixed hypercall */
2667 ctxt->_eip = ctxt->eip;
2668 /* Disable writeback. */
2669 ctxt->dst.type = OP_NONE;
2670 return X86EMUL_CONTINUE;
2673 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
2675 struct desc_ptr desc_ptr;
2678 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
2679 &desc_ptr.size, &desc_ptr.address,
2681 if (rc != X86EMUL_CONTINUE)
2683 ctxt->ops->set_gdt(ctxt, &desc_ptr);
2684 /* Disable writeback. */
2685 ctxt->dst.type = OP_NONE;
2686 return X86EMUL_CONTINUE;
2689 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
2693 rc = ctxt->ops->fix_hypercall(ctxt);
2695 /* Disable writeback. */
2696 ctxt->dst.type = OP_NONE;
2700 static int em_lidt(struct x86_emulate_ctxt *ctxt)
2702 struct desc_ptr desc_ptr;
2705 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
2706 &desc_ptr.size, &desc_ptr.address,
2708 if (rc != X86EMUL_CONTINUE)
2710 ctxt->ops->set_idt(ctxt, &desc_ptr);
2711 /* Disable writeback. */
2712 ctxt->dst.type = OP_NONE;
2713 return X86EMUL_CONTINUE;
2716 static int em_smsw(struct x86_emulate_ctxt *ctxt)
2718 ctxt->dst.bytes = 2;
2719 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
2720 return X86EMUL_CONTINUE;
2723 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
2725 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
2726 | (ctxt->src.val & 0x0f));
2727 ctxt->dst.type = OP_NONE;
2728 return X86EMUL_CONTINUE;
2731 static int em_loop(struct x86_emulate_ctxt *ctxt)
2733 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1);
2734 if ((address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) != 0) &&
2735 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
2736 jmp_rel(ctxt, ctxt->src.val);
2738 return X86EMUL_CONTINUE;
2741 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
2743 if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0)
2744 jmp_rel(ctxt, ctxt->src.val);
2746 return X86EMUL_CONTINUE;
2749 static int em_cli(struct x86_emulate_ctxt *ctxt)
2751 if (emulator_bad_iopl(ctxt))
2752 return emulate_gp(ctxt, 0);
2754 ctxt->eflags &= ~X86_EFLAGS_IF;
2755 return X86EMUL_CONTINUE;
2758 static int em_sti(struct x86_emulate_ctxt *ctxt)
2760 if (emulator_bad_iopl(ctxt))
2761 return emulate_gp(ctxt, 0);
2763 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
2764 ctxt->eflags |= X86_EFLAGS_IF;
2765 return X86EMUL_CONTINUE;
2768 static bool valid_cr(int nr)
2780 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
2782 if (!valid_cr(ctxt->modrm_reg))
2783 return emulate_ud(ctxt);
2785 return X86EMUL_CONTINUE;
2788 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
2790 u64 new_val = ctxt->src.val64;
2791 int cr = ctxt->modrm_reg;
2794 static u64 cr_reserved_bits[] = {
2795 0xffffffff00000000ULL,
2796 0, 0, 0, /* CR3 checked later */
2803 return emulate_ud(ctxt);
2805 if (new_val & cr_reserved_bits[cr])
2806 return emulate_gp(ctxt, 0);
2811 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
2812 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
2813 return emulate_gp(ctxt, 0);
2815 cr4 = ctxt->ops->get_cr(ctxt, 4);
2816 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2818 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
2819 !(cr4 & X86_CR4_PAE))
2820 return emulate_gp(ctxt, 0);
2827 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2828 if (efer & EFER_LMA)
2829 rsvd = CR3_L_MODE_RESERVED_BITS;
2830 else if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PAE)
2831 rsvd = CR3_PAE_RESERVED_BITS;
2832 else if (ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PG)
2833 rsvd = CR3_NONPAE_RESERVED_BITS;
2836 return emulate_gp(ctxt, 0);
2843 cr4 = ctxt->ops->get_cr(ctxt, 4);
2844 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2846 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
2847 return emulate_gp(ctxt, 0);
2853 return X86EMUL_CONTINUE;
2856 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
2860 ctxt->ops->get_dr(ctxt, 7, &dr7);
2862 /* Check if DR7.Global_Enable is set */
2863 return dr7 & (1 << 13);
2866 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
2868 int dr = ctxt->modrm_reg;
2872 return emulate_ud(ctxt);
2874 cr4 = ctxt->ops->get_cr(ctxt, 4);
2875 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
2876 return emulate_ud(ctxt);
2878 if (check_dr7_gd(ctxt))
2879 return emulate_db(ctxt);
2881 return X86EMUL_CONTINUE;
2884 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
2886 u64 new_val = ctxt->src.val64;
2887 int dr = ctxt->modrm_reg;
2889 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
2890 return emulate_gp(ctxt, 0);
2892 return check_dr_read(ctxt);
2895 static int check_svme(struct x86_emulate_ctxt *ctxt)
2899 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2901 if (!(efer & EFER_SVME))
2902 return emulate_ud(ctxt);
2904 return X86EMUL_CONTINUE;
2907 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
2909 u64 rax = ctxt->regs[VCPU_REGS_RAX];
2911 /* Valid physical address? */
2912 if (rax & 0xffff000000000000ULL)
2913 return emulate_gp(ctxt, 0);
2915 return check_svme(ctxt);
2918 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
2920 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
2922 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
2923 return emulate_ud(ctxt);
2925 return X86EMUL_CONTINUE;
2928 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
2930 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
2931 u64 rcx = ctxt->regs[VCPU_REGS_RCX];
2933 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
2935 return emulate_gp(ctxt, 0);
2937 return X86EMUL_CONTINUE;
2940 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
2942 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
2943 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
2944 return emulate_gp(ctxt, 0);
2946 return X86EMUL_CONTINUE;
2949 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
2951 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
2952 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
2953 return emulate_gp(ctxt, 0);
2955 return X86EMUL_CONTINUE;
2958 #define D(_y) { .flags = (_y) }
2959 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
2960 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
2961 .check_perm = (_p) }
2963 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
2964 #define G(_f, _g) { .flags = ((_f) | Group), .u.group = (_g) }
2965 #define GD(_f, _g) { .flags = ((_f) | GroupDual), .u.gdual = (_g) }
2966 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
2967 #define II(_f, _e, _i) \
2968 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
2969 #define IIP(_f, _e, _i, _p) \
2970 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
2971 .check_perm = (_p) }
2972 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
2974 #define D2bv(_f) D((_f) | ByteOp), D(_f)
2975 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
2976 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
2978 #define I6ALU(_f, _e) I2bv((_f) | DstMem | SrcReg | ModRM, _e), \
2979 I2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
2980 I2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
2982 static struct opcode group7_rm1[] = {
2983 DI(SrcNone | ModRM | Priv, monitor),
2984 DI(SrcNone | ModRM | Priv, mwait),
2988 static struct opcode group7_rm3[] = {
2989 DIP(SrcNone | ModRM | Prot | Priv, vmrun, check_svme_pa),
2990 II(SrcNone | ModRM | Prot | VendorSpecific, em_vmmcall, vmmcall),
2991 DIP(SrcNone | ModRM | Prot | Priv, vmload, check_svme_pa),
2992 DIP(SrcNone | ModRM | Prot | Priv, vmsave, check_svme_pa),
2993 DIP(SrcNone | ModRM | Prot | Priv, stgi, check_svme),
2994 DIP(SrcNone | ModRM | Prot | Priv, clgi, check_svme),
2995 DIP(SrcNone | ModRM | Prot | Priv, skinit, check_svme),
2996 DIP(SrcNone | ModRM | Prot | Priv, invlpga, check_svme),
2999 static struct opcode group7_rm7[] = {
3001 DIP(SrcNone | ModRM, rdtscp, check_rdtsc),
3005 static struct opcode group1[] = {
3016 static struct opcode group1A[] = {
3017 D(DstMem | SrcNone | ModRM | Mov | Stack), N, N, N, N, N, N, N,
3020 static struct opcode group3[] = {
3021 I(DstMem | SrcImm | ModRM, em_test),
3022 I(DstMem | SrcImm | ModRM, em_test),
3023 I(DstMem | SrcNone | ModRM | Lock, em_not),
3024 I(DstMem | SrcNone | ModRM | Lock, em_neg),
3025 I(SrcMem | ModRM, em_mul_ex),
3026 I(SrcMem | ModRM, em_imul_ex),
3027 I(SrcMem | ModRM, em_div_ex),
3028 I(SrcMem | ModRM, em_idiv_ex),
3031 static struct opcode group4[] = {
3032 D(ByteOp | DstMem | SrcNone | ModRM | Lock), D(ByteOp | DstMem | SrcNone | ModRM | Lock),
3036 static struct opcode group5[] = {
3037 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
3038 D(SrcMem | ModRM | Stack),
3039 I(SrcMemFAddr | ModRM | ImplicitOps | Stack, em_call_far),
3040 D(SrcMem | ModRM | Stack), D(SrcMemFAddr | ModRM | ImplicitOps),
3041 D(SrcMem | ModRM | Stack), N,
3044 static struct opcode group6[] = {
3045 DI(ModRM | Prot, sldt),
3046 DI(ModRM | Prot, str),
3047 DI(ModRM | Prot | Priv, lldt),
3048 DI(ModRM | Prot | Priv, ltr),
3052 static struct group_dual group7 = { {
3053 DI(ModRM | Mov | DstMem | Priv, sgdt),
3054 DI(ModRM | Mov | DstMem | Priv, sidt),
3055 II(ModRM | SrcMem | Priv, em_lgdt, lgdt),
3056 II(ModRM | SrcMem | Priv, em_lidt, lidt),
3057 II(SrcNone | ModRM | DstMem | Mov, em_smsw, smsw), N,
3058 II(SrcMem16 | ModRM | Mov | Priv, em_lmsw, lmsw),
3059 II(SrcMem | ModRM | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3061 I(SrcNone | ModRM | Priv | VendorSpecific, em_vmcall),
3063 N, EXT(0, group7_rm3),
3064 II(SrcNone | ModRM | DstMem | Mov, em_smsw, smsw), N,
3065 II(SrcMem16 | ModRM | Mov | Priv, em_lmsw, lmsw), EXT(0, group7_rm7),
3068 static struct opcode group8[] = {
3070 D(DstMem | SrcImmByte | ModRM), D(DstMem | SrcImmByte | ModRM | Lock),
3071 D(DstMem | SrcImmByte | ModRM | Lock), D(DstMem | SrcImmByte | ModRM | Lock),
3074 static struct group_dual group9 = { {
3075 N, D(DstMem64 | ModRM | Lock), N, N, N, N, N, N,
3077 N, N, N, N, N, N, N, N,
3080 static struct opcode group11[] = {
3081 I(DstMem | SrcImm | ModRM | Mov, em_mov), X7(D(Undefined)),
3084 static struct gprefix pfx_0f_6f_0f_7f = {
3085 N, N, N, I(Sse, em_movdqu),
3088 static struct opcode opcode_table[256] = {
3090 I6ALU(Lock, em_add),
3091 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
3094 D(ImplicitOps | Stack | No64), N,
3096 I6ALU(Lock, em_adc),
3097 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
3099 I6ALU(Lock, em_sbb),
3100 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
3102 I6ALU(Lock, em_and), N, N,
3104 I6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3106 I6ALU(Lock, em_xor), N, N,
3108 I6ALU(0, em_cmp), N, N,
3112 X8(I(SrcReg | Stack, em_push)),
3114 X8(I(DstReg | Stack, em_pop)),
3116 I(ImplicitOps | Stack | No64, em_pusha),
3117 I(ImplicitOps | Stack | No64, em_popa),
3118 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3121 I(SrcImm | Mov | Stack, em_push),
3122 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3123 I(SrcImmByte | Mov | Stack, em_push),
3124 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3125 D2bvIP(DstDI | SrcDX | Mov | String, ins, check_perm_in), /* insb, insw/insd */
3126 D2bvIP(SrcSI | DstDX | String, outs, check_perm_out), /* outsb, outsw/outsd */
3130 G(ByteOp | DstMem | SrcImm | ModRM | Group, group1),
3131 G(DstMem | SrcImm | ModRM | Group, group1),
3132 G(ByteOp | DstMem | SrcImm | ModRM | No64 | Group, group1),
3133 G(DstMem | SrcImmByte | ModRM | Group, group1),
3134 I2bv(DstMem | SrcReg | ModRM, em_test),
3135 I2bv(DstMem | SrcReg | ModRM | Lock, em_xchg),
3137 I2bv(DstMem | SrcReg | ModRM | Mov, em_mov),
3138 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3139 I(DstMem | SrcNone | ModRM | Mov, em_mov_rm_sreg),
3140 D(ModRM | SrcMem | NoAccess | DstReg),
3141 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
3144 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3146 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3147 I(SrcImmFAddr | No64, em_call_far), N,
3148 II(ImplicitOps | Stack, em_pushf, pushf),
3149 II(ImplicitOps | Stack, em_popf, popf), N, N,
3151 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3152 I2bv(DstMem | SrcAcc | Mov | MemAbs, em_mov),
3153 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3154 I2bv(SrcSI | DstDI | String, em_cmp),
3156 I2bv(DstAcc | SrcImm, em_test),
3157 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3158 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3159 I2bv(SrcAcc | DstDI | String, em_cmp),
3161 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3163 X8(I(DstReg | SrcImm | Mov, em_mov)),
3165 D2bv(DstMem | SrcImmByte | ModRM),
3166 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3167 I(ImplicitOps | Stack, em_ret),
3168 D(DstReg | SrcMemFAddr | ModRM | No64), D(DstReg | SrcMemFAddr | ModRM | No64),
3169 G(ByteOp, group11), G(0, group11),
3171 N, N, N, I(ImplicitOps | Stack, em_ret_far),
3172 D(ImplicitOps), DI(SrcImmByte, intn),
3173 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3175 D2bv(DstMem | SrcOne | ModRM), D2bv(DstMem | ModRM),
3178 N, N, N, N, N, N, N, N,
3180 X3(I(SrcImmByte, em_loop)),
3181 I(SrcImmByte, em_jcxz),
3182 D2bvIP(SrcImmUByte | DstAcc, in, check_perm_in),
3183 D2bvIP(SrcAcc | DstImmUByte, out, check_perm_out),
3185 D(SrcImm | Stack), D(SrcImm | ImplicitOps),
3186 I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
3187 D2bvIP(SrcDX | DstAcc, in, check_perm_in),
3188 D2bvIP(SrcAcc | DstDX, out, check_perm_out),
3190 N, DI(ImplicitOps, icebp), N, N,
3191 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
3192 G(ByteOp, group3), G(0, group3),
3194 D(ImplicitOps), D(ImplicitOps),
3195 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
3196 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3199 static struct opcode twobyte_table[256] = {
3201 G(0, group6), GD(0, &group7), N, N,
3202 N, I(ImplicitOps | VendorSpecific, em_syscall),
3203 II(ImplicitOps | Priv, em_clts, clts), N,
3204 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3205 N, D(ImplicitOps | ModRM), N, N,
3207 N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
3209 DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
3210 DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
3211 DIP(ModRM | SrcMem | Priv | Op3264, cr_write, check_cr_write),
3212 DIP(ModRM | SrcMem | Priv | Op3264, dr_write, check_dr_write),
3214 N, N, N, N, N, N, N, N,
3216 DI(ImplicitOps | Priv, wrmsr),
3217 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
3218 DI(ImplicitOps | Priv, rdmsr),
3219 DIP(ImplicitOps | Priv, rdpmc, check_rdpmc),
3220 I(ImplicitOps | VendorSpecific, em_sysenter),
3221 I(ImplicitOps | Priv | VendorSpecific, em_sysexit),
3223 N, N, N, N, N, N, N, N,
3225 X16(D(DstReg | SrcMem | ModRM | Mov)),
3227 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3232 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3237 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3241 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3243 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
3244 DI(ImplicitOps, cpuid), D(DstMem | SrcReg | ModRM | BitOp),
3245 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3246 D(DstMem | SrcReg | Src2CL | ModRM), N, N,
3248 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
3249 DI(ImplicitOps, rsm), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3250 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3251 D(DstMem | SrcReg | Src2CL | ModRM),
3252 D(ModRM), I(DstReg | SrcMem | ModRM, em_imul),
3254 D2bv(DstMem | SrcReg | ModRM | Lock),
3255 D(DstReg | SrcMemFAddr | ModRM), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3256 D(DstReg | SrcMemFAddr | ModRM), D(DstReg | SrcMemFAddr | ModRM),
3257 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3260 G(BitOp, group8), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3261 D(DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
3262 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3264 D2bv(DstMem | SrcReg | ModRM | Lock),
3265 N, D(DstMem | SrcReg | ModRM | Mov),
3266 N, N, N, GD(0, &group9),
3267 N, N, N, N, N, N, N, N,
3269 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3271 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3273 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
3289 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
3293 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3299 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
3300 unsigned size, bool sign_extension)
3302 int rc = X86EMUL_CONTINUE;
3306 op->addr.mem.ea = ctxt->_eip;
3307 /* NB. Immediates are sign-extended as necessary. */
3308 switch (op->bytes) {
3310 op->val = insn_fetch(s8, ctxt);
3313 op->val = insn_fetch(s16, ctxt);
3316 op->val = insn_fetch(s32, ctxt);
3319 if (!sign_extension) {
3320 switch (op->bytes) {
3328 op->val &= 0xffffffff;
3336 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
3339 int rc = X86EMUL_CONTINUE;
3343 decode_register_operand(ctxt, op,
3344 ctxt->twobyte && (ctxt->b == 0xb6 || ctxt->b == 0xb7));
3348 op->addr.mem.ea = ctxt->_eip;
3350 op->val = insn_fetch(u8, ctxt);
3359 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3360 if (ctxt->d & BitOp)
3361 fetch_bit_operand(ctxt);
3362 op->orig_val = op->val;
3366 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3367 op->addr.reg = &ctxt->regs[VCPU_REGS_RAX];
3368 fetch_register_operand(op);
3369 op->orig_val = op->val;
3373 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3375 register_address(ctxt, ctxt->regs[VCPU_REGS_RDI]);
3376 op->addr.mem.seg = VCPU_SREG_ES;
3382 op->addr.reg = &ctxt->regs[VCPU_REGS_RDX];
3383 fetch_register_operand(op);
3386 /* Special instructions do their own operand decoding. */
3388 op->type = OP_NONE; /* Disable writeback. */
3396 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
3398 int rc = X86EMUL_CONTINUE;
3399 int mode = ctxt->mode;
3400 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
3401 bool op_prefix = false;
3402 struct opcode opcode;
3404 ctxt->memop.type = OP_NONE;
3405 ctxt->memopp = NULL;
3406 ctxt->_eip = ctxt->eip;
3407 ctxt->fetch.start = ctxt->_eip;
3408 ctxt->fetch.end = ctxt->fetch.start + insn_len;
3410 memcpy(ctxt->fetch.data, insn, insn_len);
3413 case X86EMUL_MODE_REAL:
3414 case X86EMUL_MODE_VM86:
3415 case X86EMUL_MODE_PROT16:
3416 def_op_bytes = def_ad_bytes = 2;
3418 case X86EMUL_MODE_PROT32:
3419 def_op_bytes = def_ad_bytes = 4;
3421 #ifdef CONFIG_X86_64
3422 case X86EMUL_MODE_PROT64:
3428 return EMULATION_FAILED;
3431 ctxt->op_bytes = def_op_bytes;
3432 ctxt->ad_bytes = def_ad_bytes;
3434 /* Legacy prefixes. */
3436 switch (ctxt->b = insn_fetch(u8, ctxt)) {
3437 case 0x66: /* operand-size override */
3439 /* switch between 2/4 bytes */
3440 ctxt->op_bytes = def_op_bytes ^ 6;
3442 case 0x67: /* address-size override */
3443 if (mode == X86EMUL_MODE_PROT64)
3444 /* switch between 4/8 bytes */
3445 ctxt->ad_bytes = def_ad_bytes ^ 12;
3447 /* switch between 2/4 bytes */
3448 ctxt->ad_bytes = def_ad_bytes ^ 6;
3450 case 0x26: /* ES override */
3451 case 0x2e: /* CS override */
3452 case 0x36: /* SS override */
3453 case 0x3e: /* DS override */
3454 set_seg_override(ctxt, (ctxt->b >> 3) & 3);
3456 case 0x64: /* FS override */
3457 case 0x65: /* GS override */
3458 set_seg_override(ctxt, ctxt->b & 7);
3460 case 0x40 ... 0x4f: /* REX */
3461 if (mode != X86EMUL_MODE_PROT64)
3463 ctxt->rex_prefix = ctxt->b;
3465 case 0xf0: /* LOCK */
3466 ctxt->lock_prefix = 1;
3468 case 0xf2: /* REPNE/REPNZ */
3469 case 0xf3: /* REP/REPE/REPZ */
3470 ctxt->rep_prefix = ctxt->b;
3476 /* Any legacy prefix after a REX prefix nullifies its effect. */
3478 ctxt->rex_prefix = 0;
3484 if (ctxt->rex_prefix & 8)
3485 ctxt->op_bytes = 8; /* REX.W */
3487 /* Opcode byte(s). */
3488 opcode = opcode_table[ctxt->b];
3489 /* Two-byte opcode? */
3490 if (ctxt->b == 0x0f) {
3492 ctxt->b = insn_fetch(u8, ctxt);
3493 opcode = twobyte_table[ctxt->b];
3495 ctxt->d = opcode.flags;
3497 while (ctxt->d & GroupMask) {
3498 switch (ctxt->d & GroupMask) {
3500 ctxt->modrm = insn_fetch(u8, ctxt);
3502 goffset = (ctxt->modrm >> 3) & 7;
3503 opcode = opcode.u.group[goffset];
3506 ctxt->modrm = insn_fetch(u8, ctxt);
3508 goffset = (ctxt->modrm >> 3) & 7;
3509 if ((ctxt->modrm >> 6) == 3)
3510 opcode = opcode.u.gdual->mod3[goffset];
3512 opcode = opcode.u.gdual->mod012[goffset];
3515 goffset = ctxt->modrm & 7;
3516 opcode = opcode.u.group[goffset];
3519 if (ctxt->rep_prefix && op_prefix)
3520 return EMULATION_FAILED;
3521 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
3522 switch (simd_prefix) {
3523 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
3524 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
3525 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
3526 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
3530 return EMULATION_FAILED;
3533 ctxt->d &= ~GroupMask;
3534 ctxt->d |= opcode.flags;
3537 ctxt->execute = opcode.u.execute;
3538 ctxt->check_perm = opcode.check_perm;
3539 ctxt->intercept = opcode.intercept;
3542 if (ctxt->d == 0 || (ctxt->d & Undefined))
3543 return EMULATION_FAILED;
3545 if (!(ctxt->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
3546 return EMULATION_FAILED;
3548 if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
3551 if (ctxt->d & Op3264) {
3552 if (mode == X86EMUL_MODE_PROT64)
3559 ctxt->op_bytes = 16;
3561 /* ModRM and SIB bytes. */
3562 if (ctxt->d & ModRM) {
3563 rc = decode_modrm(ctxt, &ctxt->memop);
3564 if (!ctxt->has_seg_override)
3565 set_seg_override(ctxt, ctxt->modrm_seg);
3566 } else if (ctxt->d & MemAbs)
3567 rc = decode_abs(ctxt, &ctxt->memop);
3568 if (rc != X86EMUL_CONTINUE)
3571 if (!ctxt->has_seg_override)
3572 set_seg_override(ctxt, VCPU_SREG_DS);
3574 ctxt->memop.addr.mem.seg = seg_override(ctxt);
3576 if (ctxt->memop.type == OP_MEM && ctxt->ad_bytes != 8)
3577 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
3580 * Decode and fetch the source operand: register, memory
3583 switch (ctxt->d & SrcMask) {
3587 decode_register_operand(ctxt, &ctxt->src, 0);
3590 ctxt->memop.bytes = 2;
3593 ctxt->memop.bytes = 4;
3596 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3598 ctxt->src = ctxt->memop;
3599 ctxt->memopp = &ctxt->src;
3602 rc = decode_imm(ctxt, &ctxt->src, 2, false);
3605 rc = decode_imm(ctxt, &ctxt->src, imm_size(ctxt), true);
3608 rc = decode_imm(ctxt, &ctxt->src, imm_size(ctxt), false);
3611 rc = decode_imm(ctxt, &ctxt->src, 1, true);
3614 rc = decode_imm(ctxt, &ctxt->src, 1, false);
3617 ctxt->src.type = OP_REG;
3618 ctxt->src.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3619 ctxt->src.addr.reg = &ctxt->regs[VCPU_REGS_RAX];
3620 fetch_register_operand(&ctxt->src);
3623 ctxt->src.bytes = 1;
3627 ctxt->src.type = OP_MEM;
3628 ctxt->src.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3629 ctxt->src.addr.mem.ea =
3630 register_address(ctxt, ctxt->regs[VCPU_REGS_RSI]);
3631 ctxt->src.addr.mem.seg = seg_override(ctxt);
3635 ctxt->src.type = OP_IMM;
3636 ctxt->src.addr.mem.ea = ctxt->_eip;
3637 ctxt->src.bytes = ctxt->op_bytes + 2;
3638 insn_fetch_arr(ctxt->src.valptr, ctxt->src.bytes, ctxt);
3641 ctxt->memop.bytes = ctxt->op_bytes + 2;
3645 ctxt->src.type = OP_REG;
3646 ctxt->src.bytes = 2;
3647 ctxt->src.addr.reg = &ctxt->regs[VCPU_REGS_RDX];
3648 fetch_register_operand(&ctxt->src);
3652 if (rc != X86EMUL_CONTINUE)
3656 * Decode and fetch the second source operand: register, memory
3659 switch (ctxt->d & Src2Mask) {
3663 ctxt->src2.bytes = 1;
3664 ctxt->src2.val = ctxt->regs[VCPU_REGS_RCX] & 0xff;
3667 rc = decode_imm(ctxt, &ctxt->src2, 1, true);
3670 ctxt->src2.bytes = 1;
3674 rc = decode_imm(ctxt, &ctxt->src2, imm_size(ctxt), true);
3678 if (rc != X86EMUL_CONTINUE)
3681 /* Decode and fetch the destination operand: register or memory. */
3682 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
3685 if (ctxt->memopp && ctxt->memopp->type == OP_MEM && ctxt->rip_relative)
3686 ctxt->memopp->addr.mem.ea += ctxt->_eip;
3688 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
3691 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
3693 /* The second termination condition only applies for REPE
3694 * and REPNE. Test if the repeat string operation prefix is
3695 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
3696 * corresponding termination condition according to:
3697 * - if REPE/REPZ and ZF = 0 then done
3698 * - if REPNE/REPNZ and ZF = 1 then done
3700 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
3701 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
3702 && (((ctxt->rep_prefix == REPE_PREFIX) &&
3703 ((ctxt->eflags & EFLG_ZF) == 0))
3704 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
3705 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
3711 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
3713 struct x86_emulate_ops *ops = ctxt->ops;
3715 int rc = X86EMUL_CONTINUE;
3716 int saved_dst_type = ctxt->dst.type;
3718 ctxt->mem_read.pos = 0;
3720 if (ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) {
3721 rc = emulate_ud(ctxt);
3725 /* LOCK prefix is allowed only with some instructions */
3726 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
3727 rc = emulate_ud(ctxt);
3731 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
3732 rc = emulate_ud(ctxt);
3737 && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)
3738 || !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
3739 rc = emulate_ud(ctxt);
3743 if ((ctxt->d & Sse) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
3744 rc = emulate_nm(ctxt);
3748 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
3749 rc = emulator_check_intercept(ctxt, ctxt->intercept,
3750 X86_ICPT_PRE_EXCEPT);
3751 if (rc != X86EMUL_CONTINUE)
3755 /* Privileged instruction can be executed only in CPL=0 */
3756 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
3757 rc = emulate_gp(ctxt, 0);
3761 /* Instruction can only be executed in protected mode */
3762 if ((ctxt->d & Prot) && !(ctxt->mode & X86EMUL_MODE_PROT)) {
3763 rc = emulate_ud(ctxt);
3767 /* Do instruction specific permission checks */
3768 if (ctxt->check_perm) {
3769 rc = ctxt->check_perm(ctxt);
3770 if (rc != X86EMUL_CONTINUE)
3774 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
3775 rc = emulator_check_intercept(ctxt, ctxt->intercept,
3776 X86_ICPT_POST_EXCEPT);
3777 if (rc != X86EMUL_CONTINUE)
3781 if (ctxt->rep_prefix && (ctxt->d & String)) {
3782 /* All REP prefixes have the same first termination condition */
3783 if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0) {
3784 ctxt->eip = ctxt->_eip;
3789 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
3790 rc = segmented_read(ctxt, ctxt->src.addr.mem,
3791 ctxt->src.valptr, ctxt->src.bytes);
3792 if (rc != X86EMUL_CONTINUE)
3794 ctxt->src.orig_val64 = ctxt->src.val64;
3797 if (ctxt->src2.type == OP_MEM) {
3798 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
3799 &ctxt->src2.val, ctxt->src2.bytes);
3800 if (rc != X86EMUL_CONTINUE)
3804 if ((ctxt->d & DstMask) == ImplicitOps)
3808 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
3809 /* optimisation - avoid slow emulated read if Mov */
3810 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
3811 &ctxt->dst.val, ctxt->dst.bytes);
3812 if (rc != X86EMUL_CONTINUE)
3815 ctxt->dst.orig_val = ctxt->dst.val;
3819 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
3820 rc = emulator_check_intercept(ctxt, ctxt->intercept,
3821 X86_ICPT_POST_MEMACCESS);
3822 if (rc != X86EMUL_CONTINUE)
3826 if (ctxt->execute) {
3827 rc = ctxt->execute(ctxt);
3828 if (rc != X86EMUL_CONTINUE)
3837 case 0x06: /* push es */
3838 rc = emulate_push_sreg(ctxt, VCPU_SREG_ES);
3840 case 0x07: /* pop es */
3841 rc = emulate_pop_sreg(ctxt, VCPU_SREG_ES);
3843 case 0x0e: /* push cs */
3844 rc = emulate_push_sreg(ctxt, VCPU_SREG_CS);
3846 case 0x16: /* push ss */
3847 rc = emulate_push_sreg(ctxt, VCPU_SREG_SS);
3849 case 0x17: /* pop ss */
3850 rc = emulate_pop_sreg(ctxt, VCPU_SREG_SS);
3852 case 0x1e: /* push ds */
3853 rc = emulate_push_sreg(ctxt, VCPU_SREG_DS);
3855 case 0x1f: /* pop ds */
3856 rc = emulate_pop_sreg(ctxt, VCPU_SREG_DS);
3858 case 0x40 ... 0x47: /* inc r16/r32 */
3859 emulate_1op(ctxt, "inc");
3861 case 0x48 ... 0x4f: /* dec r16/r32 */
3862 emulate_1op(ctxt, "dec");
3864 case 0x63: /* movsxd */
3865 if (ctxt->mode != X86EMUL_MODE_PROT64)
3866 goto cannot_emulate;
3867 ctxt->dst.val = (s32) ctxt->src.val;
3869 case 0x6c: /* insb */
3870 case 0x6d: /* insw/insd */
3871 ctxt->src.val = ctxt->regs[VCPU_REGS_RDX];
3873 case 0x6e: /* outsb */
3874 case 0x6f: /* outsw/outsd */
3875 ctxt->dst.val = ctxt->regs[VCPU_REGS_RDX];
3878 case 0x70 ... 0x7f: /* jcc (short) */
3879 if (test_cc(ctxt->b, ctxt->eflags))
3880 jmp_rel(ctxt, ctxt->src.val);
3882 case 0x8d: /* lea r16/r32, m */
3883 ctxt->dst.val = ctxt->src.addr.mem.ea;
3885 case 0x8f: /* pop (sole member of Grp1a) */
3886 rc = em_grp1a(ctxt);
3888 case 0x90 ... 0x97: /* nop / xchg reg, rax */
3889 if (ctxt->dst.addr.reg == &ctxt->regs[VCPU_REGS_RAX])
3893 case 0x98: /* cbw/cwde/cdqe */
3894 switch (ctxt->op_bytes) {
3895 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
3896 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
3897 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
3903 case 0xc4: /* les */
3904 rc = emulate_load_segment(ctxt, VCPU_SREG_ES);
3906 case 0xc5: /* lds */
3907 rc = emulate_load_segment(ctxt, VCPU_SREG_DS);
3909 case 0xcc: /* int3 */
3910 rc = emulate_int(ctxt, 3);
3912 case 0xcd: /* int n */
3913 rc = emulate_int(ctxt, ctxt->src.val);
3915 case 0xce: /* into */
3916 if (ctxt->eflags & EFLG_OF)
3917 rc = emulate_int(ctxt, 4);
3919 case 0xd0 ... 0xd1: /* Grp2 */
3922 case 0xd2 ... 0xd3: /* Grp2 */
3923 ctxt->src.val = ctxt->regs[VCPU_REGS_RCX];
3926 case 0xe4: /* inb */
3929 case 0xe6: /* outb */
3930 case 0xe7: /* out */
3932 case 0xe8: /* call (near) */ {
3933 long int rel = ctxt->src.val;
3934 ctxt->src.val = (unsigned long) ctxt->_eip;
3939 case 0xe9: /* jmp rel */
3940 case 0xeb: /* jmp rel short */
3941 jmp_rel(ctxt, ctxt->src.val);
3942 ctxt->dst.type = OP_NONE; /* Disable writeback. */
3944 case 0xec: /* in al,dx */
3945 case 0xed: /* in (e/r)ax,dx */
3947 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3949 goto done; /* IO is needed */
3951 case 0xee: /* out dx,al */
3952 case 0xef: /* out dx,(e/r)ax */
3954 ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3956 ctxt->dst.type = OP_NONE; /* Disable writeback. */
3958 case 0xf4: /* hlt */
3959 ctxt->ops->halt(ctxt);
3961 case 0xf5: /* cmc */
3962 /* complement carry flag from eflags reg */
3963 ctxt->eflags ^= EFLG_CF;
3965 case 0xf8: /* clc */
3966 ctxt->eflags &= ~EFLG_CF;
3968 case 0xf9: /* stc */
3969 ctxt->eflags |= EFLG_CF;
3971 case 0xfc: /* cld */
3972 ctxt->eflags &= ~EFLG_DF;
3974 case 0xfd: /* std */
3975 ctxt->eflags |= EFLG_DF;
3977 case 0xfe: /* Grp4 */
3978 rc = em_grp45(ctxt);
3980 case 0xff: /* Grp5 */
3981 rc = em_grp45(ctxt);
3984 goto cannot_emulate;
3987 if (rc != X86EMUL_CONTINUE)
3991 rc = writeback(ctxt);
3992 if (rc != X86EMUL_CONTINUE)
3996 * restore dst type in case the decoding will be reused
3997 * (happens for string instruction )
3999 ctxt->dst.type = saved_dst_type;
4001 if ((ctxt->d & SrcMask) == SrcSI)
4002 string_addr_inc(ctxt, seg_override(ctxt),
4003 VCPU_REGS_RSI, &ctxt->src);
4005 if ((ctxt->d & DstMask) == DstDI)
4006 string_addr_inc(ctxt, VCPU_SREG_ES, VCPU_REGS_RDI,
4009 if (ctxt->rep_prefix && (ctxt->d & String)) {
4010 struct read_cache *r = &ctxt->io_read;
4011 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1);
4013 if (!string_insn_completed(ctxt)) {
4015 * Re-enter guest when pio read ahead buffer is empty
4016 * or, if it is not used, after each 1024 iteration.
4018 if ((r->end != 0 || ctxt->regs[VCPU_REGS_RCX] & 0x3ff) &&
4019 (r->end == 0 || r->end != r->pos)) {
4021 * Reset read cache. Usually happens before
4022 * decode, but since instruction is restarted
4023 * we have to do it here.
4025 ctxt->mem_read.end = 0;
4026 return EMULATION_RESTART;
4028 goto done; /* skip rip writeback */
4032 ctxt->eip = ctxt->_eip;
4035 if (rc == X86EMUL_PROPAGATE_FAULT)
4036 ctxt->have_exception = true;
4037 if (rc == X86EMUL_INTERCEPTED)
4038 return EMULATION_INTERCEPTED;
4040 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4044 case 0x09: /* wbinvd */
4045 (ctxt->ops->wbinvd)(ctxt);
4047 case 0x08: /* invd */
4048 case 0x0d: /* GrpP (prefetch) */
4049 case 0x18: /* Grp16 (prefetch/nop) */
4051 case 0x20: /* mov cr, reg */
4052 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
4054 case 0x21: /* mov from dr to reg */
4055 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
4057 case 0x22: /* mov reg, cr */
4058 if (ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val)) {
4059 emulate_gp(ctxt, 0);
4060 rc = X86EMUL_PROPAGATE_FAULT;
4063 ctxt->dst.type = OP_NONE;
4065 case 0x23: /* mov from reg to dr */
4066 if (ops->set_dr(ctxt, ctxt->modrm_reg, ctxt->src.val &
4067 ((ctxt->mode == X86EMUL_MODE_PROT64) ?
4068 ~0ULL : ~0U)) < 0) {
4069 /* #UD condition is already handled by the code above */
4070 emulate_gp(ctxt, 0);
4071 rc = X86EMUL_PROPAGATE_FAULT;
4075 ctxt->dst.type = OP_NONE; /* no writeback */
4079 msr_data = (u32)ctxt->regs[VCPU_REGS_RAX]
4080 | ((u64)ctxt->regs[VCPU_REGS_RDX] << 32);
4081 if (ops->set_msr(ctxt, ctxt->regs[VCPU_REGS_RCX], msr_data)) {
4082 emulate_gp(ctxt, 0);
4083 rc = X86EMUL_PROPAGATE_FAULT;
4086 rc = X86EMUL_CONTINUE;
4090 if (ops->get_msr(ctxt, ctxt->regs[VCPU_REGS_RCX], &msr_data)) {
4091 emulate_gp(ctxt, 0);
4092 rc = X86EMUL_PROPAGATE_FAULT;
4095 ctxt->regs[VCPU_REGS_RAX] = (u32)msr_data;
4096 ctxt->regs[VCPU_REGS_RDX] = msr_data >> 32;
4098 rc = X86EMUL_CONTINUE;
4100 case 0x40 ... 0x4f: /* cmov */
4101 ctxt->dst.val = ctxt->dst.orig_val = ctxt->src.val;
4102 if (!test_cc(ctxt->b, ctxt->eflags))
4103 ctxt->dst.type = OP_NONE; /* no writeback */
4105 case 0x80 ... 0x8f: /* jnz rel, etc*/
4106 if (test_cc(ctxt->b, ctxt->eflags))
4107 jmp_rel(ctxt, ctxt->src.val);
4109 case 0x90 ... 0x9f: /* setcc r/m8 */
4110 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
4112 case 0xa0: /* push fs */
4113 rc = emulate_push_sreg(ctxt, VCPU_SREG_FS);
4115 case 0xa1: /* pop fs */
4116 rc = emulate_pop_sreg(ctxt, VCPU_SREG_FS);
4120 ctxt->dst.type = OP_NONE;
4121 /* only subword offset */
4122 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
4123 emulate_2op_SrcV_nobyte(ctxt, "bt");
4125 case 0xa4: /* shld imm8, r, r/m */
4126 case 0xa5: /* shld cl, r, r/m */
4127 emulate_2op_cl(ctxt, "shld");
4129 case 0xa8: /* push gs */
4130 rc = emulate_push_sreg(ctxt, VCPU_SREG_GS);
4132 case 0xa9: /* pop gs */
4133 rc = emulate_pop_sreg(ctxt, VCPU_SREG_GS);
4137 emulate_2op_SrcV_nobyte(ctxt, "bts");
4139 case 0xac: /* shrd imm8, r, r/m */
4140 case 0xad: /* shrd cl, r, r/m */
4141 emulate_2op_cl(ctxt, "shrd");
4143 case 0xae: /* clflush */
4145 case 0xb0 ... 0xb1: /* cmpxchg */
4147 * Save real source value, then compare EAX against
4150 ctxt->src.orig_val = ctxt->src.val;
4151 ctxt->src.val = ctxt->regs[VCPU_REGS_RAX];
4152 emulate_2op_SrcV(ctxt, "cmp");
4153 if (ctxt->eflags & EFLG_ZF) {
4154 /* Success: write back to memory. */
4155 ctxt->dst.val = ctxt->src.orig_val;
4157 /* Failure: write the value we saw to EAX. */
4158 ctxt->dst.type = OP_REG;
4159 ctxt->dst.addr.reg = (unsigned long *)&ctxt->regs[VCPU_REGS_RAX];
4162 case 0xb2: /* lss */
4163 rc = emulate_load_segment(ctxt, VCPU_SREG_SS);
4167 emulate_2op_SrcV_nobyte(ctxt, "btr");
4169 case 0xb4: /* lfs */
4170 rc = emulate_load_segment(ctxt, VCPU_SREG_FS);
4172 case 0xb5: /* lgs */
4173 rc = emulate_load_segment(ctxt, VCPU_SREG_GS);
4175 case 0xb6 ... 0xb7: /* movzx */
4176 ctxt->dst.bytes = ctxt->op_bytes;
4177 ctxt->dst.val = (ctxt->d & ByteOp) ? (u8) ctxt->src.val
4178 : (u16) ctxt->src.val;
4180 case 0xba: /* Grp8 */
4181 switch (ctxt->modrm_reg & 3) {
4194 emulate_2op_SrcV_nobyte(ctxt, "btc");
4196 case 0xbc: { /* bsf */
4198 __asm__ ("bsf %2, %0; setz %1"
4199 : "=r"(ctxt->dst.val), "=q"(zf)
4200 : "r"(ctxt->src.val));
4201 ctxt->eflags &= ~X86_EFLAGS_ZF;
4203 ctxt->eflags |= X86_EFLAGS_ZF;
4204 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4208 case 0xbd: { /* bsr */
4210 __asm__ ("bsr %2, %0; setz %1"
4211 : "=r"(ctxt->dst.val), "=q"(zf)
4212 : "r"(ctxt->src.val));
4213 ctxt->eflags &= ~X86_EFLAGS_ZF;
4215 ctxt->eflags |= X86_EFLAGS_ZF;
4216 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4220 case 0xbe ... 0xbf: /* movsx */
4221 ctxt->dst.bytes = ctxt->op_bytes;
4222 ctxt->dst.val = (ctxt->d & ByteOp) ? (s8) ctxt->src.val :
4223 (s16) ctxt->src.val;
4225 case 0xc0 ... 0xc1: /* xadd */
4226 emulate_2op_SrcV(ctxt, "add");
4227 /* Write back the register source. */
4228 ctxt->src.val = ctxt->dst.orig_val;
4229 write_register_operand(&ctxt->src);
4231 case 0xc3: /* movnti */
4232 ctxt->dst.bytes = ctxt->op_bytes;
4233 ctxt->dst.val = (ctxt->op_bytes == 4) ? (u32) ctxt->src.val :
4234 (u64) ctxt->src.val;
4236 case 0xc7: /* Grp9 (cmpxchg8b) */
4240 goto cannot_emulate;
4243 if (rc != X86EMUL_CONTINUE)
4249 return EMULATION_FAILED;