1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
35 #define OpImplicit 1ull /* No generic decode */
36 #define OpReg 2ull /* Register */
37 #define OpMem 3ull /* Memory */
38 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
39 #define OpDI 5ull /* ES:DI/EDI/RDI */
40 #define OpMem64 6ull /* Memory, 64-bit */
41 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
42 #define OpDX 8ull /* DX register */
43 #define OpCL 9ull /* CL register (for shifts) */
44 #define OpImmByte 10ull /* 8-bit sign extended immediate */
45 #define OpOne 11ull /* Implied 1 */
46 #define OpImm 12ull /* Sign extended immediate */
47 #define OpMem16 13ull /* Memory operand (16-bit). */
48 #define OpMem32 14ull /* Memory operand (32-bit). */
49 #define OpImmU 15ull /* Immediate operand, zero extended */
50 #define OpSI 16ull /* SI/ESI/RSI */
51 #define OpImmFAddr 17ull /* Immediate far address */
52 #define OpMemFAddr 18ull /* Far address in memory */
53 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
54 #define OpES 20ull /* ES */
55 #define OpCS 21ull /* CS */
56 #define OpSS 22ull /* SS */
57 #define OpDS 23ull /* DS */
58 #define OpFS 24ull /* FS */
59 #define OpGS 25ull /* GS */
60 #define OpMem8 26ull /* 8-bit zero extended memory operand */
62 #define OpBits 5 /* Width of operand field */
63 #define OpMask ((1ull << OpBits) - 1)
66 * Opcode effective-address decode tables.
67 * Note that we only emulate instructions that have at least one memory
68 * operand (excluding implicit stack references). We assume that stack
69 * references and instruction fetches will never occur in special memory
70 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
74 /* Operand sizes: 8-bit operands or specified/overridden size. */
75 #define ByteOp (1<<0) /* 8-bit operands. */
76 /* Destination operand type. */
78 #define ImplicitOps (OpImplicit << DstShift)
79 #define DstReg (OpReg << DstShift)
80 #define DstMem (OpMem << DstShift)
81 #define DstAcc (OpAcc << DstShift)
82 #define DstDI (OpDI << DstShift)
83 #define DstMem64 (OpMem64 << DstShift)
84 #define DstImmUByte (OpImmUByte << DstShift)
85 #define DstDX (OpDX << DstShift)
86 #define DstMask (OpMask << DstShift)
87 /* Source operand type. */
89 #define SrcNone (OpNone << SrcShift)
90 #define SrcReg (OpReg << SrcShift)
91 #define SrcMem (OpMem << SrcShift)
92 #define SrcMem16 (OpMem16 << SrcShift)
93 #define SrcMem32 (OpMem32 << SrcShift)
94 #define SrcImm (OpImm << SrcShift)
95 #define SrcImmByte (OpImmByte << SrcShift)
96 #define SrcOne (OpOne << SrcShift)
97 #define SrcImmUByte (OpImmUByte << SrcShift)
98 #define SrcImmU (OpImmU << SrcShift)
99 #define SrcSI (OpSI << SrcShift)
100 #define SrcImmFAddr (OpImmFAddr << SrcShift)
101 #define SrcMemFAddr (OpMemFAddr << SrcShift)
102 #define SrcAcc (OpAcc << SrcShift)
103 #define SrcImmU16 (OpImmU16 << SrcShift)
104 #define SrcDX (OpDX << SrcShift)
105 #define SrcMem8 (OpMem8 << SrcShift)
106 #define SrcMask (OpMask << SrcShift)
107 #define BitOp (1<<11)
108 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
109 #define String (1<<13) /* String instruction (rep capable) */
110 #define Stack (1<<14) /* Stack instruction (push/pop) */
111 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
112 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
113 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
114 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
115 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
116 #define Sse (1<<18) /* SSE Vector instruction */
117 /* Generic ModRM decode. */
118 #define ModRM (1<<19)
119 /* Destination is only written; never read. */
122 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
123 #define VendorSpecific (1<<22) /* Vendor specific instruction */
124 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
125 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
126 #define Undefined (1<<25) /* No Such Instruction */
127 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
128 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
130 #define PageTable (1 << 29) /* instruction used to write page table */
131 /* Source 2 operand type */
132 #define Src2Shift (30)
133 #define Src2None (OpNone << Src2Shift)
134 #define Src2CL (OpCL << Src2Shift)
135 #define Src2ImmByte (OpImmByte << Src2Shift)
136 #define Src2One (OpOne << Src2Shift)
137 #define Src2Imm (OpImm << Src2Shift)
138 #define Src2ES (OpES << Src2Shift)
139 #define Src2CS (OpCS << Src2Shift)
140 #define Src2SS (OpSS << Src2Shift)
141 #define Src2DS (OpDS << Src2Shift)
142 #define Src2FS (OpFS << Src2Shift)
143 #define Src2GS (OpGS << Src2Shift)
144 #define Src2Mask (OpMask << Src2Shift)
145 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
146 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
147 #define Unaligned ((u64)1 << 42) /* Explicitly unaligned (e.g. MOVDQU) */
148 #define Avx ((u64)1 << 43) /* Advanced Vector Extensions */
150 #define X2(x...) x, x
151 #define X3(x...) X2(x), x
152 #define X4(x...) X2(x), X2(x)
153 #define X5(x...) X4(x), x
154 #define X6(x...) X4(x), X2(x)
155 #define X7(x...) X4(x), X3(x)
156 #define X8(x...) X4(x), X4(x)
157 #define X16(x...) X8(x), X8(x)
163 int (*execute)(struct x86_emulate_ctxt *ctxt);
164 struct opcode *group;
165 struct group_dual *gdual;
166 struct gprefix *gprefix;
168 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
172 struct opcode mod012[8];
173 struct opcode mod3[8];
177 struct opcode pfx_no;
178 struct opcode pfx_66;
179 struct opcode pfx_f2;
180 struct opcode pfx_f3;
183 /* EFLAGS bit definitions. */
184 #define EFLG_ID (1<<21)
185 #define EFLG_VIP (1<<20)
186 #define EFLG_VIF (1<<19)
187 #define EFLG_AC (1<<18)
188 #define EFLG_VM (1<<17)
189 #define EFLG_RF (1<<16)
190 #define EFLG_IOPL (3<<12)
191 #define EFLG_NT (1<<14)
192 #define EFLG_OF (1<<11)
193 #define EFLG_DF (1<<10)
194 #define EFLG_IF (1<<9)
195 #define EFLG_TF (1<<8)
196 #define EFLG_SF (1<<7)
197 #define EFLG_ZF (1<<6)
198 #define EFLG_AF (1<<4)
199 #define EFLG_PF (1<<2)
200 #define EFLG_CF (1<<0)
202 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
203 #define EFLG_RESERVED_ONE_MASK 2
206 * Instruction emulation:
207 * Most instructions are emulated directly via a fragment of inline assembly
208 * code. This allows us to save/restore EFLAGS and thus very easily pick up
209 * any modified flags.
212 #if defined(CONFIG_X86_64)
213 #define _LO32 "k" /* force 32-bit operand */
214 #define _STK "%%rsp" /* stack pointer */
215 #elif defined(__i386__)
216 #define _LO32 "" /* force 32-bit operand */
217 #define _STK "%%esp" /* stack pointer */
221 * These EFLAGS bits are restored from saved value during emulation, and
222 * any changes are written back to the saved value after emulation.
224 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
226 /* Before executing instruction: restore necessary bits in EFLAGS. */
227 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
228 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
229 "movl %"_sav",%"_LO32 _tmp"; " \
232 "movl %"_msk",%"_LO32 _tmp"; " \
233 "andl %"_LO32 _tmp",("_STK"); " \
235 "notl %"_LO32 _tmp"; " \
236 "andl %"_LO32 _tmp",("_STK"); " \
237 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
239 "orl %"_LO32 _tmp",("_STK"); " \
243 /* After executing instruction: write-back necessary bits in EFLAGS. */
244 #define _POST_EFLAGS(_sav, _msk, _tmp) \
245 /* _sav |= EFLAGS & _msk; */ \
248 "andl %"_msk",%"_LO32 _tmp"; " \
249 "orl %"_LO32 _tmp",%"_sav"; "
257 #define ____emulate_2op(ctxt, _op, _x, _y, _suffix, _dsttype) \
259 __asm__ __volatile__ ( \
260 _PRE_EFLAGS("0", "4", "2") \
261 _op _suffix " %"_x"3,%1; " \
262 _POST_EFLAGS("0", "4", "2") \
263 : "=m" ((ctxt)->eflags), \
264 "+q" (*(_dsttype*)&(ctxt)->dst.val), \
266 : _y ((ctxt)->src.val), "i" (EFLAGS_MASK)); \
270 /* Raw emulation: instruction has two explicit operands. */
271 #define __emulate_2op_nobyte(ctxt,_op,_wx,_wy,_lx,_ly,_qx,_qy) \
273 unsigned long _tmp; \
275 switch ((ctxt)->dst.bytes) { \
277 ____emulate_2op(ctxt,_op,_wx,_wy,"w",u16); \
280 ____emulate_2op(ctxt,_op,_lx,_ly,"l",u32); \
283 ON64(____emulate_2op(ctxt,_op,_qx,_qy,"q",u64)); \
288 #define __emulate_2op(ctxt,_op,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
290 unsigned long _tmp; \
291 switch ((ctxt)->dst.bytes) { \
293 ____emulate_2op(ctxt,_op,_bx,_by,"b",u8); \
296 __emulate_2op_nobyte(ctxt, _op, \
297 _wx, _wy, _lx, _ly, _qx, _qy); \
302 /* Source operand is byte-sized and may be restricted to just %cl. */
303 #define emulate_2op_SrcB(ctxt, _op) \
304 __emulate_2op(ctxt, _op, "b", "c", "b", "c", "b", "c", "b", "c")
306 /* Source operand is byte, word, long or quad sized. */
307 #define emulate_2op_SrcV(ctxt, _op) \
308 __emulate_2op(ctxt, _op, "b", "q", "w", "r", _LO32, "r", "", "r")
310 /* Source operand is word, long or quad sized. */
311 #define emulate_2op_SrcV_nobyte(ctxt, _op) \
312 __emulate_2op_nobyte(ctxt, _op, "w", "r", _LO32, "r", "", "r")
314 /* Instruction has three operands and one operand is stored in ECX register */
315 #define __emulate_2op_cl(ctxt, _op, _suffix, _type) \
317 unsigned long _tmp; \
318 _type _clv = (ctxt)->src2.val; \
319 _type _srcv = (ctxt)->src.val; \
320 _type _dstv = (ctxt)->dst.val; \
322 __asm__ __volatile__ ( \
323 _PRE_EFLAGS("0", "5", "2") \
324 _op _suffix " %4,%1 \n" \
325 _POST_EFLAGS("0", "5", "2") \
326 : "=m" ((ctxt)->eflags), "+r" (_dstv), "=&r" (_tmp) \
327 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
330 (ctxt)->src2.val = (unsigned long) _clv; \
331 (ctxt)->src2.val = (unsigned long) _srcv; \
332 (ctxt)->dst.val = (unsigned long) _dstv; \
335 #define emulate_2op_cl(ctxt, _op) \
337 switch ((ctxt)->dst.bytes) { \
339 __emulate_2op_cl(ctxt, _op, "w", u16); \
342 __emulate_2op_cl(ctxt, _op, "l", u32); \
345 ON64(__emulate_2op_cl(ctxt, _op, "q", ulong)); \
350 #define __emulate_1op(ctxt, _op, _suffix) \
352 unsigned long _tmp; \
354 __asm__ __volatile__ ( \
355 _PRE_EFLAGS("0", "3", "2") \
356 _op _suffix " %1; " \
357 _POST_EFLAGS("0", "3", "2") \
358 : "=m" ((ctxt)->eflags), "+m" ((ctxt)->dst.val), \
360 : "i" (EFLAGS_MASK)); \
363 /* Instruction has only one explicit operand (no source operand). */
364 #define emulate_1op(ctxt, _op) \
366 switch ((ctxt)->dst.bytes) { \
367 case 1: __emulate_1op(ctxt, _op, "b"); break; \
368 case 2: __emulate_1op(ctxt, _op, "w"); break; \
369 case 4: __emulate_1op(ctxt, _op, "l"); break; \
370 case 8: ON64(__emulate_1op(ctxt, _op, "q")); break; \
374 #define __emulate_1op_rax_rdx(ctxt, _op, _suffix, _ex) \
376 unsigned long _tmp; \
377 ulong *rax = &(ctxt)->regs[VCPU_REGS_RAX]; \
378 ulong *rdx = &(ctxt)->regs[VCPU_REGS_RDX]; \
380 __asm__ __volatile__ ( \
381 _PRE_EFLAGS("0", "5", "1") \
383 _op _suffix " %6; " \
385 _POST_EFLAGS("0", "5", "1") \
386 ".pushsection .fixup,\"ax\" \n\t" \
387 "3: movb $1, %4 \n\t" \
390 _ASM_EXTABLE(1b, 3b) \
391 : "=m" ((ctxt)->eflags), "=&r" (_tmp), \
392 "+a" (*rax), "+d" (*rdx), "+qm"(_ex) \
393 : "i" (EFLAGS_MASK), "m" ((ctxt)->src.val), \
394 "a" (*rax), "d" (*rdx)); \
397 /* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
398 #define emulate_1op_rax_rdx(ctxt, _op, _ex) \
400 switch((ctxt)->src.bytes) { \
402 __emulate_1op_rax_rdx(ctxt, _op, "b", _ex); \
405 __emulate_1op_rax_rdx(ctxt, _op, "w", _ex); \
408 __emulate_1op_rax_rdx(ctxt, _op, "l", _ex); \
411 __emulate_1op_rax_rdx(ctxt, _op, "q", _ex)); \
416 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
417 enum x86_intercept intercept,
418 enum x86_intercept_stage stage)
420 struct x86_instruction_info info = {
421 .intercept = intercept,
422 .rep_prefix = ctxt->rep_prefix,
423 .modrm_mod = ctxt->modrm_mod,
424 .modrm_reg = ctxt->modrm_reg,
425 .modrm_rm = ctxt->modrm_rm,
426 .src_val = ctxt->src.val64,
427 .src_bytes = ctxt->src.bytes,
428 .dst_bytes = ctxt->dst.bytes,
429 .ad_bytes = ctxt->ad_bytes,
430 .next_rip = ctxt->eip,
433 return ctxt->ops->intercept(ctxt, &info, stage);
436 static void assign_masked(ulong *dest, ulong src, ulong mask)
438 *dest = (*dest & ~mask) | (src & mask);
441 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
443 return (1UL << (ctxt->ad_bytes << 3)) - 1;
446 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
449 struct desc_struct ss;
451 if (ctxt->mode == X86EMUL_MODE_PROT64)
453 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
454 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
457 static int stack_size(struct x86_emulate_ctxt *ctxt)
459 return (__fls(stack_mask(ctxt)) + 1) >> 3;
462 /* Access/update address held in a register, based on addressing mode. */
463 static inline unsigned long
464 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
466 if (ctxt->ad_bytes == sizeof(unsigned long))
469 return reg & ad_mask(ctxt);
472 static inline unsigned long
473 register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
475 return address_mask(ctxt, reg);
479 register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
481 if (ctxt->ad_bytes == sizeof(unsigned long))
484 *reg = (*reg & ~ad_mask(ctxt)) | ((*reg + inc) & ad_mask(ctxt));
487 static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
489 register_address_increment(ctxt, &ctxt->_eip, rel);
492 static u32 desc_limit_scaled(struct desc_struct *desc)
494 u32 limit = get_desc_limit(desc);
496 return desc->g ? (limit << 12) | 0xfff : limit;
499 static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
501 ctxt->has_seg_override = true;
502 ctxt->seg_override = seg;
505 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
507 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
510 return ctxt->ops->get_cached_segment_base(ctxt, seg);
513 static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
515 if (!ctxt->has_seg_override)
518 return ctxt->seg_override;
521 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
522 u32 error, bool valid)
524 ctxt->exception.vector = vec;
525 ctxt->exception.error_code = error;
526 ctxt->exception.error_code_valid = valid;
527 return X86EMUL_PROPAGATE_FAULT;
530 static int emulate_db(struct x86_emulate_ctxt *ctxt)
532 return emulate_exception(ctxt, DB_VECTOR, 0, false);
535 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
537 return emulate_exception(ctxt, GP_VECTOR, err, true);
540 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
542 return emulate_exception(ctxt, SS_VECTOR, err, true);
545 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
547 return emulate_exception(ctxt, UD_VECTOR, 0, false);
550 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
552 return emulate_exception(ctxt, TS_VECTOR, err, true);
555 static int emulate_de(struct x86_emulate_ctxt *ctxt)
557 return emulate_exception(ctxt, DE_VECTOR, 0, false);
560 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
562 return emulate_exception(ctxt, NM_VECTOR, 0, false);
565 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
568 struct desc_struct desc;
570 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
574 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
579 struct desc_struct desc;
581 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
582 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
586 * x86 defines three classes of vector instructions: explicitly
587 * aligned, explicitly unaligned, and the rest, which change behaviour
588 * depending on whether they're AVX encoded or not.
590 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
591 * subject to the same check.
593 static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
595 if (likely(size < 16))
598 if (ctxt->d & Aligned)
600 else if (ctxt->d & Unaligned)
602 else if (ctxt->d & Avx)
608 static int __linearize(struct x86_emulate_ctxt *ctxt,
609 struct segmented_address addr,
610 unsigned size, bool write, bool fetch,
613 struct desc_struct desc;
620 la = seg_base(ctxt, addr.seg) + addr.ea;
621 switch (ctxt->mode) {
622 case X86EMUL_MODE_REAL:
624 case X86EMUL_MODE_PROT64:
625 if (((signed long)la << 16) >> 16 != la)
626 return emulate_gp(ctxt, 0);
629 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
633 /* code segment or read-only data segment */
634 if (((desc.type & 8) || !(desc.type & 2)) && write)
636 /* unreadable code segment */
637 if (!fetch && (desc.type & 8) && !(desc.type & 2))
639 lim = desc_limit_scaled(&desc);
640 if ((desc.type & 8) || !(desc.type & 4)) {
641 /* expand-up segment */
642 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
645 /* exapand-down segment */
646 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
648 lim = desc.d ? 0xffffffff : 0xffff;
649 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
652 cpl = ctxt->ops->cpl(ctxt);
655 if (!(desc.type & 8)) {
659 } else if ((desc.type & 8) && !(desc.type & 4)) {
660 /* nonconforming code segment */
663 } else if ((desc.type & 8) && (desc.type & 4)) {
664 /* conforming code segment */
670 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
672 if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
673 return emulate_gp(ctxt, 0);
675 return X86EMUL_CONTINUE;
677 if (addr.seg == VCPU_SREG_SS)
678 return emulate_ss(ctxt, addr.seg);
680 return emulate_gp(ctxt, addr.seg);
683 static int linearize(struct x86_emulate_ctxt *ctxt,
684 struct segmented_address addr,
685 unsigned size, bool write,
688 return __linearize(ctxt, addr, size, write, false, linear);
692 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
693 struct segmented_address addr,
700 rc = linearize(ctxt, addr, size, false, &linear);
701 if (rc != X86EMUL_CONTINUE)
703 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
707 * Fetch the next byte of the instruction being emulated which is pointed to
708 * by ctxt->_eip, then increment ctxt->_eip.
710 * Also prefetch the remaining bytes of the instruction without crossing page
711 * boundary if they are not in fetch_cache yet.
713 static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
715 struct fetch_cache *fc = &ctxt->fetch;
719 if (ctxt->_eip == fc->end) {
720 unsigned long linear;
721 struct segmented_address addr = { .seg = VCPU_SREG_CS,
723 cur_size = fc->end - fc->start;
724 size = min(15UL - cur_size,
725 PAGE_SIZE - offset_in_page(ctxt->_eip));
726 rc = __linearize(ctxt, addr, size, false, true, &linear);
727 if (unlikely(rc != X86EMUL_CONTINUE))
729 rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
730 size, &ctxt->exception);
731 if (unlikely(rc != X86EMUL_CONTINUE))
735 *dest = fc->data[ctxt->_eip - fc->start];
737 return X86EMUL_CONTINUE;
740 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
741 void *dest, unsigned size)
745 /* x86 instructions are limited to 15 bytes. */
746 if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
747 return X86EMUL_UNHANDLEABLE;
749 rc = do_insn_fetch_byte(ctxt, dest++);
750 if (rc != X86EMUL_CONTINUE)
753 return X86EMUL_CONTINUE;
756 /* Fetch next part of the instruction being emulated. */
757 #define insn_fetch(_type, _ctxt) \
758 ({ unsigned long _x; \
759 rc = do_insn_fetch(_ctxt, &_x, sizeof(_type)); \
760 if (rc != X86EMUL_CONTINUE) \
765 #define insn_fetch_arr(_arr, _size, _ctxt) \
766 ({ rc = do_insn_fetch(_ctxt, _arr, (_size)); \
767 if (rc != X86EMUL_CONTINUE) \
772 * Given the 'reg' portion of a ModRM byte, and a register block, return a
773 * pointer into the block that addresses the relevant register.
774 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
776 static void *decode_register(u8 modrm_reg, unsigned long *regs,
781 p = ®s[modrm_reg];
782 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
783 p = (unsigned char *)®s[modrm_reg & 3] + 1;
787 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
788 struct segmented_address addr,
789 u16 *size, unsigned long *address, int op_bytes)
796 rc = segmented_read_std(ctxt, addr, size, 2);
797 if (rc != X86EMUL_CONTINUE)
800 rc = segmented_read_std(ctxt, addr, address, op_bytes);
804 static int test_cc(unsigned int condition, unsigned int flags)
808 switch ((condition & 15) >> 1) {
810 rc |= (flags & EFLG_OF);
812 case 1: /* b/c/nae */
813 rc |= (flags & EFLG_CF);
816 rc |= (flags & EFLG_ZF);
819 rc |= (flags & (EFLG_CF|EFLG_ZF));
822 rc |= (flags & EFLG_SF);
825 rc |= (flags & EFLG_PF);
828 rc |= (flags & EFLG_ZF);
831 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
835 /* Odd condition identifiers (lsb == 1) have inverted sense. */
836 return (!!rc ^ (condition & 1));
839 static void fetch_register_operand(struct operand *op)
843 op->val = *(u8 *)op->addr.reg;
846 op->val = *(u16 *)op->addr.reg;
849 op->val = *(u32 *)op->addr.reg;
852 op->val = *(u64 *)op->addr.reg;
857 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
859 ctxt->ops->get_fpu(ctxt);
861 case 0: asm("movdqu %%xmm0, %0" : "=m"(*data)); break;
862 case 1: asm("movdqu %%xmm1, %0" : "=m"(*data)); break;
863 case 2: asm("movdqu %%xmm2, %0" : "=m"(*data)); break;
864 case 3: asm("movdqu %%xmm3, %0" : "=m"(*data)); break;
865 case 4: asm("movdqu %%xmm4, %0" : "=m"(*data)); break;
866 case 5: asm("movdqu %%xmm5, %0" : "=m"(*data)); break;
867 case 6: asm("movdqu %%xmm6, %0" : "=m"(*data)); break;
868 case 7: asm("movdqu %%xmm7, %0" : "=m"(*data)); break;
870 case 8: asm("movdqu %%xmm8, %0" : "=m"(*data)); break;
871 case 9: asm("movdqu %%xmm9, %0" : "=m"(*data)); break;
872 case 10: asm("movdqu %%xmm10, %0" : "=m"(*data)); break;
873 case 11: asm("movdqu %%xmm11, %0" : "=m"(*data)); break;
874 case 12: asm("movdqu %%xmm12, %0" : "=m"(*data)); break;
875 case 13: asm("movdqu %%xmm13, %0" : "=m"(*data)); break;
876 case 14: asm("movdqu %%xmm14, %0" : "=m"(*data)); break;
877 case 15: asm("movdqu %%xmm15, %0" : "=m"(*data)); break;
881 ctxt->ops->put_fpu(ctxt);
884 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
887 ctxt->ops->get_fpu(ctxt);
889 case 0: asm("movdqu %0, %%xmm0" : : "m"(*data)); break;
890 case 1: asm("movdqu %0, %%xmm1" : : "m"(*data)); break;
891 case 2: asm("movdqu %0, %%xmm2" : : "m"(*data)); break;
892 case 3: asm("movdqu %0, %%xmm3" : : "m"(*data)); break;
893 case 4: asm("movdqu %0, %%xmm4" : : "m"(*data)); break;
894 case 5: asm("movdqu %0, %%xmm5" : : "m"(*data)); break;
895 case 6: asm("movdqu %0, %%xmm6" : : "m"(*data)); break;
896 case 7: asm("movdqu %0, %%xmm7" : : "m"(*data)); break;
898 case 8: asm("movdqu %0, %%xmm8" : : "m"(*data)); break;
899 case 9: asm("movdqu %0, %%xmm9" : : "m"(*data)); break;
900 case 10: asm("movdqu %0, %%xmm10" : : "m"(*data)); break;
901 case 11: asm("movdqu %0, %%xmm11" : : "m"(*data)); break;
902 case 12: asm("movdqu %0, %%xmm12" : : "m"(*data)); break;
903 case 13: asm("movdqu %0, %%xmm13" : : "m"(*data)); break;
904 case 14: asm("movdqu %0, %%xmm14" : : "m"(*data)); break;
905 case 15: asm("movdqu %0, %%xmm15" : : "m"(*data)); break;
909 ctxt->ops->put_fpu(ctxt);
912 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
914 ctxt->ops->get_fpu(ctxt);
916 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
917 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
918 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
919 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
920 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
921 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
922 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
923 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
926 ctxt->ops->put_fpu(ctxt);
929 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
931 ctxt->ops->get_fpu(ctxt);
933 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
934 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
935 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
936 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
937 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
938 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
939 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
940 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
943 ctxt->ops->put_fpu(ctxt);
946 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
949 unsigned reg = ctxt->modrm_reg;
950 int highbyte_regs = ctxt->rex_prefix == 0;
952 if (!(ctxt->d & ModRM))
953 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
959 read_sse_reg(ctxt, &op->vec_val, reg);
971 if (ctxt->d & ByteOp) {
972 op->addr.reg = decode_register(reg, ctxt->regs, highbyte_regs);
975 op->addr.reg = decode_register(reg, ctxt->regs, 0);
976 op->bytes = ctxt->op_bytes;
978 fetch_register_operand(op);
979 op->orig_val = op->val;
982 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
984 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
985 ctxt->modrm_seg = VCPU_SREG_SS;
988 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
992 int index_reg = 0, base_reg = 0, scale;
993 int rc = X86EMUL_CONTINUE;
996 if (ctxt->rex_prefix) {
997 ctxt->modrm_reg = (ctxt->rex_prefix & 4) << 1; /* REX.R */
998 index_reg = (ctxt->rex_prefix & 2) << 2; /* REX.X */
999 ctxt->modrm_rm = base_reg = (ctxt->rex_prefix & 1) << 3; /* REG.B */
1002 ctxt->modrm_mod |= (ctxt->modrm & 0xc0) >> 6;
1003 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1004 ctxt->modrm_rm |= (ctxt->modrm & 0x07);
1005 ctxt->modrm_seg = VCPU_SREG_DS;
1007 if (ctxt->modrm_mod == 3) {
1009 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1010 op->addr.reg = decode_register(ctxt->modrm_rm,
1011 ctxt->regs, ctxt->d & ByteOp);
1012 if (ctxt->d & Sse) {
1015 op->addr.xmm = ctxt->modrm_rm;
1016 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1019 if (ctxt->d & Mmx) {
1022 op->addr.xmm = ctxt->modrm_rm & 7;
1025 fetch_register_operand(op);
1031 if (ctxt->ad_bytes == 2) {
1032 unsigned bx = ctxt->regs[VCPU_REGS_RBX];
1033 unsigned bp = ctxt->regs[VCPU_REGS_RBP];
1034 unsigned si = ctxt->regs[VCPU_REGS_RSI];
1035 unsigned di = ctxt->regs[VCPU_REGS_RDI];
1037 /* 16-bit ModR/M decode. */
1038 switch (ctxt->modrm_mod) {
1040 if (ctxt->modrm_rm == 6)
1041 modrm_ea += insn_fetch(u16, ctxt);
1044 modrm_ea += insn_fetch(s8, ctxt);
1047 modrm_ea += insn_fetch(u16, ctxt);
1050 switch (ctxt->modrm_rm) {
1052 modrm_ea += bx + si;
1055 modrm_ea += bx + di;
1058 modrm_ea += bp + si;
1061 modrm_ea += bp + di;
1070 if (ctxt->modrm_mod != 0)
1077 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1078 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1079 ctxt->modrm_seg = VCPU_SREG_SS;
1080 modrm_ea = (u16)modrm_ea;
1082 /* 32/64-bit ModR/M decode. */
1083 if ((ctxt->modrm_rm & 7) == 4) {
1084 sib = insn_fetch(u8, ctxt);
1085 index_reg |= (sib >> 3) & 7;
1086 base_reg |= sib & 7;
1089 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1090 modrm_ea += insn_fetch(s32, ctxt);
1092 modrm_ea += ctxt->regs[base_reg];
1093 adjust_modrm_seg(ctxt, base_reg);
1096 modrm_ea += ctxt->regs[index_reg] << scale;
1097 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1098 if (ctxt->mode == X86EMUL_MODE_PROT64)
1099 ctxt->rip_relative = 1;
1101 base_reg = ctxt->modrm_rm;
1102 modrm_ea += ctxt->regs[base_reg];
1103 adjust_modrm_seg(ctxt, base_reg);
1105 switch (ctxt->modrm_mod) {
1107 if (ctxt->modrm_rm == 5)
1108 modrm_ea += insn_fetch(s32, ctxt);
1111 modrm_ea += insn_fetch(s8, ctxt);
1114 modrm_ea += insn_fetch(s32, ctxt);
1118 op->addr.mem.ea = modrm_ea;
1123 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1126 int rc = X86EMUL_CONTINUE;
1129 switch (ctxt->ad_bytes) {
1131 op->addr.mem.ea = insn_fetch(u16, ctxt);
1134 op->addr.mem.ea = insn_fetch(u32, ctxt);
1137 op->addr.mem.ea = insn_fetch(u64, ctxt);
1144 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1148 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1149 mask = ~(ctxt->dst.bytes * 8 - 1);
1151 if (ctxt->src.bytes == 2)
1152 sv = (s16)ctxt->src.val & (s16)mask;
1153 else if (ctxt->src.bytes == 4)
1154 sv = (s32)ctxt->src.val & (s32)mask;
1156 ctxt->dst.addr.mem.ea += (sv >> 3);
1159 /* only subword offset */
1160 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1163 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1164 unsigned long addr, void *dest, unsigned size)
1167 struct read_cache *mc = &ctxt->mem_read;
1170 int n = min(size, 8u);
1172 if (mc->pos < mc->end)
1175 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, n,
1177 if (rc != X86EMUL_CONTINUE)
1182 memcpy(dest, mc->data + mc->pos, n);
1187 return X86EMUL_CONTINUE;
1190 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1191 struct segmented_address addr,
1198 rc = linearize(ctxt, addr, size, false, &linear);
1199 if (rc != X86EMUL_CONTINUE)
1201 return read_emulated(ctxt, linear, data, size);
1204 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1205 struct segmented_address addr,
1212 rc = linearize(ctxt, addr, size, true, &linear);
1213 if (rc != X86EMUL_CONTINUE)
1215 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1219 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1220 struct segmented_address addr,
1221 const void *orig_data, const void *data,
1227 rc = linearize(ctxt, addr, size, true, &linear);
1228 if (rc != X86EMUL_CONTINUE)
1230 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1231 size, &ctxt->exception);
1234 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1235 unsigned int size, unsigned short port,
1238 struct read_cache *rc = &ctxt->io_read;
1240 if (rc->pos == rc->end) { /* refill pio read ahead */
1241 unsigned int in_page, n;
1242 unsigned int count = ctxt->rep_prefix ?
1243 address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) : 1;
1244 in_page = (ctxt->eflags & EFLG_DF) ?
1245 offset_in_page(ctxt->regs[VCPU_REGS_RDI]) :
1246 PAGE_SIZE - offset_in_page(ctxt->regs[VCPU_REGS_RDI]);
1247 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1251 rc->pos = rc->end = 0;
1252 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1257 memcpy(dest, rc->data + rc->pos, size);
1262 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1263 u16 index, struct desc_struct *desc)
1268 ctxt->ops->get_idt(ctxt, &dt);
1270 if (dt.size < index * 8 + 7)
1271 return emulate_gp(ctxt, index << 3 | 0x2);
1273 addr = dt.address + index * 8;
1274 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1278 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1279 u16 selector, struct desc_ptr *dt)
1281 struct x86_emulate_ops *ops = ctxt->ops;
1283 if (selector & 1 << 2) {
1284 struct desc_struct desc;
1287 memset (dt, 0, sizeof *dt);
1288 if (!ops->get_segment(ctxt, &sel, &desc, NULL, VCPU_SREG_LDTR))
1291 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1292 dt->address = get_desc_base(&desc);
1294 ops->get_gdt(ctxt, dt);
1297 /* allowed just for 8 bytes segments */
1298 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1299 u16 selector, struct desc_struct *desc,
1303 u16 index = selector >> 3;
1306 get_descriptor_table_ptr(ctxt, selector, &dt);
1308 if (dt.size < index * 8 + 7)
1309 return emulate_gp(ctxt, selector & 0xfffc);
1311 *desc_addr_p = addr = dt.address + index * 8;
1312 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1316 /* allowed just for 8 bytes segments */
1317 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1318 u16 selector, struct desc_struct *desc)
1321 u16 index = selector >> 3;
1324 get_descriptor_table_ptr(ctxt, selector, &dt);
1326 if (dt.size < index * 8 + 7)
1327 return emulate_gp(ctxt, selector & 0xfffc);
1329 addr = dt.address + index * 8;
1330 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1334 /* Does not support long mode */
1335 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1336 u16 selector, int seg)
1338 struct desc_struct seg_desc;
1340 unsigned err_vec = GP_VECTOR;
1342 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1346 memset(&seg_desc, 0, sizeof seg_desc);
1348 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1349 || ctxt->mode == X86EMUL_MODE_REAL) {
1350 /* set real mode segment descriptor */
1351 set_desc_base(&seg_desc, selector << 4);
1352 set_desc_limit(&seg_desc, 0xffff);
1356 if (ctxt->mode == X86EMUL_MODE_VM86)
1362 cpl = ctxt->ops->cpl(ctxt);
1364 /* NULL selector is not valid for TR, CS and SS (except for long mode) */
1365 if ((seg == VCPU_SREG_CS
1366 || (seg == VCPU_SREG_SS
1367 && (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl))
1368 || seg == VCPU_SREG_TR)
1372 /* TR should be in GDT only */
1373 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1376 if (null_selector) /* for NULL selector skip all following checks */
1379 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1380 if (ret != X86EMUL_CONTINUE)
1383 err_code = selector & 0xfffc;
1384 err_vec = GP_VECTOR;
1386 /* can't load system descriptor into segment selecor */
1387 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1391 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1400 * segment is not a writable data segment or segment
1401 * selector's RPL != CPL or segment selector's RPL != CPL
1403 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1407 if (!(seg_desc.type & 8))
1410 if (seg_desc.type & 4) {
1416 if (rpl > cpl || dpl != cpl)
1419 /* CS(RPL) <- CPL */
1420 selector = (selector & 0xfffc) | cpl;
1423 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1426 case VCPU_SREG_LDTR:
1427 if (seg_desc.s || seg_desc.type != 2)
1430 default: /* DS, ES, FS, or GS */
1432 * segment is not a data or readable code segment or
1433 * ((segment is a data or nonconforming code segment)
1434 * and (both RPL and CPL > DPL))
1436 if ((seg_desc.type & 0xa) == 0x8 ||
1437 (((seg_desc.type & 0xc) != 0xc) &&
1438 (rpl > dpl && cpl > dpl)))
1444 /* mark segment as accessed */
1446 ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1447 if (ret != X86EMUL_CONTINUE)
1451 ctxt->ops->set_segment(ctxt, selector, &seg_desc, 0, seg);
1452 return X86EMUL_CONTINUE;
1454 emulate_exception(ctxt, err_vec, err_code, true);
1455 return X86EMUL_PROPAGATE_FAULT;
1458 static void write_register_operand(struct operand *op)
1460 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1461 switch (op->bytes) {
1463 *(u8 *)op->addr.reg = (u8)op->val;
1466 *(u16 *)op->addr.reg = (u16)op->val;
1469 *op->addr.reg = (u32)op->val;
1470 break; /* 64b: zero-extend */
1472 *op->addr.reg = op->val;
1477 static int writeback(struct x86_emulate_ctxt *ctxt)
1481 switch (ctxt->dst.type) {
1483 write_register_operand(&ctxt->dst);
1486 if (ctxt->lock_prefix)
1487 rc = segmented_cmpxchg(ctxt,
1489 &ctxt->dst.orig_val,
1493 rc = segmented_write(ctxt,
1497 if (rc != X86EMUL_CONTINUE)
1501 write_sse_reg(ctxt, &ctxt->dst.vec_val, ctxt->dst.addr.xmm);
1504 write_mmx_reg(ctxt, &ctxt->dst.mm_val, ctxt->dst.addr.mm);
1512 return X86EMUL_CONTINUE;
1515 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1517 struct segmented_address addr;
1519 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], -bytes);
1520 addr.ea = register_address(ctxt, ctxt->regs[VCPU_REGS_RSP]);
1521 addr.seg = VCPU_SREG_SS;
1523 return segmented_write(ctxt, addr, data, bytes);
1526 static int em_push(struct x86_emulate_ctxt *ctxt)
1528 /* Disable writeback. */
1529 ctxt->dst.type = OP_NONE;
1530 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1533 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1534 void *dest, int len)
1537 struct segmented_address addr;
1539 addr.ea = register_address(ctxt, ctxt->regs[VCPU_REGS_RSP]);
1540 addr.seg = VCPU_SREG_SS;
1541 rc = segmented_read(ctxt, addr, dest, len);
1542 if (rc != X86EMUL_CONTINUE)
1545 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], len);
1549 static int em_pop(struct x86_emulate_ctxt *ctxt)
1551 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1554 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1555 void *dest, int len)
1558 unsigned long val, change_mask;
1559 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1560 int cpl = ctxt->ops->cpl(ctxt);
1562 rc = emulate_pop(ctxt, &val, len);
1563 if (rc != X86EMUL_CONTINUE)
1566 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1567 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1569 switch(ctxt->mode) {
1570 case X86EMUL_MODE_PROT64:
1571 case X86EMUL_MODE_PROT32:
1572 case X86EMUL_MODE_PROT16:
1574 change_mask |= EFLG_IOPL;
1576 change_mask |= EFLG_IF;
1578 case X86EMUL_MODE_VM86:
1580 return emulate_gp(ctxt, 0);
1581 change_mask |= EFLG_IF;
1583 default: /* real mode */
1584 change_mask |= (EFLG_IOPL | EFLG_IF);
1588 *(unsigned long *)dest =
1589 (ctxt->eflags & ~change_mask) | (val & change_mask);
1594 static int em_popf(struct x86_emulate_ctxt *ctxt)
1596 ctxt->dst.type = OP_REG;
1597 ctxt->dst.addr.reg = &ctxt->eflags;
1598 ctxt->dst.bytes = ctxt->op_bytes;
1599 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1602 static int em_enter(struct x86_emulate_ctxt *ctxt)
1605 unsigned frame_size = ctxt->src.val;
1606 unsigned nesting_level = ctxt->src2.val & 31;
1609 return X86EMUL_UNHANDLEABLE;
1611 rc = push(ctxt, &ctxt->regs[VCPU_REGS_RBP], stack_size(ctxt));
1612 if (rc != X86EMUL_CONTINUE)
1614 assign_masked(&ctxt->regs[VCPU_REGS_RBP], ctxt->regs[VCPU_REGS_RSP],
1616 assign_masked(&ctxt->regs[VCPU_REGS_RSP],
1617 ctxt->regs[VCPU_REGS_RSP] - frame_size,
1619 return X86EMUL_CONTINUE;
1622 static int em_leave(struct x86_emulate_ctxt *ctxt)
1624 assign_masked(&ctxt->regs[VCPU_REGS_RSP], ctxt->regs[VCPU_REGS_RBP],
1626 return emulate_pop(ctxt, &ctxt->regs[VCPU_REGS_RBP], ctxt->op_bytes);
1629 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1631 int seg = ctxt->src2.val;
1633 ctxt->src.val = get_segment_selector(ctxt, seg);
1635 return em_push(ctxt);
1638 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1640 int seg = ctxt->src2.val;
1641 unsigned long selector;
1644 rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1645 if (rc != X86EMUL_CONTINUE)
1648 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1652 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1654 unsigned long old_esp = ctxt->regs[VCPU_REGS_RSP];
1655 int rc = X86EMUL_CONTINUE;
1656 int reg = VCPU_REGS_RAX;
1658 while (reg <= VCPU_REGS_RDI) {
1659 (reg == VCPU_REGS_RSP) ?
1660 (ctxt->src.val = old_esp) : (ctxt->src.val = ctxt->regs[reg]);
1663 if (rc != X86EMUL_CONTINUE)
1672 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1674 ctxt->src.val = (unsigned long)ctxt->eflags;
1675 return em_push(ctxt);
1678 static int em_popa(struct x86_emulate_ctxt *ctxt)
1680 int rc = X86EMUL_CONTINUE;
1681 int reg = VCPU_REGS_RDI;
1683 while (reg >= VCPU_REGS_RAX) {
1684 if (reg == VCPU_REGS_RSP) {
1685 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP],
1690 rc = emulate_pop(ctxt, &ctxt->regs[reg], ctxt->op_bytes);
1691 if (rc != X86EMUL_CONTINUE)
1698 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1700 struct x86_emulate_ops *ops = ctxt->ops;
1707 /* TODO: Add limit checks */
1708 ctxt->src.val = ctxt->eflags;
1710 if (rc != X86EMUL_CONTINUE)
1713 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1715 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1717 if (rc != X86EMUL_CONTINUE)
1720 ctxt->src.val = ctxt->_eip;
1722 if (rc != X86EMUL_CONTINUE)
1725 ops->get_idt(ctxt, &dt);
1727 eip_addr = dt.address + (irq << 2);
1728 cs_addr = dt.address + (irq << 2) + 2;
1730 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1731 if (rc != X86EMUL_CONTINUE)
1734 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1735 if (rc != X86EMUL_CONTINUE)
1738 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1739 if (rc != X86EMUL_CONTINUE)
1747 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1749 switch(ctxt->mode) {
1750 case X86EMUL_MODE_REAL:
1751 return emulate_int_real(ctxt, irq);
1752 case X86EMUL_MODE_VM86:
1753 case X86EMUL_MODE_PROT16:
1754 case X86EMUL_MODE_PROT32:
1755 case X86EMUL_MODE_PROT64:
1757 /* Protected mode interrupts unimplemented yet */
1758 return X86EMUL_UNHANDLEABLE;
1762 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
1764 int rc = X86EMUL_CONTINUE;
1765 unsigned long temp_eip = 0;
1766 unsigned long temp_eflags = 0;
1767 unsigned long cs = 0;
1768 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1769 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1770 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1771 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1773 /* TODO: Add stack limit check */
1775 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
1777 if (rc != X86EMUL_CONTINUE)
1780 if (temp_eip & ~0xffff)
1781 return emulate_gp(ctxt, 0);
1783 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1785 if (rc != X86EMUL_CONTINUE)
1788 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
1790 if (rc != X86EMUL_CONTINUE)
1793 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1795 if (rc != X86EMUL_CONTINUE)
1798 ctxt->_eip = temp_eip;
1801 if (ctxt->op_bytes == 4)
1802 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1803 else if (ctxt->op_bytes == 2) {
1804 ctxt->eflags &= ~0xffff;
1805 ctxt->eflags |= temp_eflags;
1808 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1809 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1814 static int em_iret(struct x86_emulate_ctxt *ctxt)
1816 switch(ctxt->mode) {
1817 case X86EMUL_MODE_REAL:
1818 return emulate_iret_real(ctxt);
1819 case X86EMUL_MODE_VM86:
1820 case X86EMUL_MODE_PROT16:
1821 case X86EMUL_MODE_PROT32:
1822 case X86EMUL_MODE_PROT64:
1824 /* iret from protected mode unimplemented yet */
1825 return X86EMUL_UNHANDLEABLE;
1829 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
1834 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1836 rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
1837 if (rc != X86EMUL_CONTINUE)
1841 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
1842 return X86EMUL_CONTINUE;
1845 static int em_grp2(struct x86_emulate_ctxt *ctxt)
1847 switch (ctxt->modrm_reg) {
1849 emulate_2op_SrcB(ctxt, "rol");
1852 emulate_2op_SrcB(ctxt, "ror");
1855 emulate_2op_SrcB(ctxt, "rcl");
1858 emulate_2op_SrcB(ctxt, "rcr");
1860 case 4: /* sal/shl */
1861 case 6: /* sal/shl */
1862 emulate_2op_SrcB(ctxt, "sal");
1865 emulate_2op_SrcB(ctxt, "shr");
1868 emulate_2op_SrcB(ctxt, "sar");
1871 return X86EMUL_CONTINUE;
1874 static int em_not(struct x86_emulate_ctxt *ctxt)
1876 ctxt->dst.val = ~ctxt->dst.val;
1877 return X86EMUL_CONTINUE;
1880 static int em_neg(struct x86_emulate_ctxt *ctxt)
1882 emulate_1op(ctxt, "neg");
1883 return X86EMUL_CONTINUE;
1886 static int em_mul_ex(struct x86_emulate_ctxt *ctxt)
1890 emulate_1op_rax_rdx(ctxt, "mul", ex);
1891 return X86EMUL_CONTINUE;
1894 static int em_imul_ex(struct x86_emulate_ctxt *ctxt)
1898 emulate_1op_rax_rdx(ctxt, "imul", ex);
1899 return X86EMUL_CONTINUE;
1902 static int em_div_ex(struct x86_emulate_ctxt *ctxt)
1906 emulate_1op_rax_rdx(ctxt, "div", de);
1908 return emulate_de(ctxt);
1909 return X86EMUL_CONTINUE;
1912 static int em_idiv_ex(struct x86_emulate_ctxt *ctxt)
1916 emulate_1op_rax_rdx(ctxt, "idiv", de);
1918 return emulate_de(ctxt);
1919 return X86EMUL_CONTINUE;
1922 static int em_grp45(struct x86_emulate_ctxt *ctxt)
1924 int rc = X86EMUL_CONTINUE;
1926 switch (ctxt->modrm_reg) {
1928 emulate_1op(ctxt, "inc");
1931 emulate_1op(ctxt, "dec");
1933 case 2: /* call near abs */ {
1935 old_eip = ctxt->_eip;
1936 ctxt->_eip = ctxt->src.val;
1937 ctxt->src.val = old_eip;
1941 case 4: /* jmp abs */
1942 ctxt->_eip = ctxt->src.val;
1944 case 5: /* jmp far */
1945 rc = em_jmp_far(ctxt);
1954 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
1956 u64 old = ctxt->dst.orig_val64;
1958 if (((u32) (old >> 0) != (u32) ctxt->regs[VCPU_REGS_RAX]) ||
1959 ((u32) (old >> 32) != (u32) ctxt->regs[VCPU_REGS_RDX])) {
1960 ctxt->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1961 ctxt->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1962 ctxt->eflags &= ~EFLG_ZF;
1964 ctxt->dst.val64 = ((u64)ctxt->regs[VCPU_REGS_RCX] << 32) |
1965 (u32) ctxt->regs[VCPU_REGS_RBX];
1967 ctxt->eflags |= EFLG_ZF;
1969 return X86EMUL_CONTINUE;
1972 static int em_ret(struct x86_emulate_ctxt *ctxt)
1974 ctxt->dst.type = OP_REG;
1975 ctxt->dst.addr.reg = &ctxt->_eip;
1976 ctxt->dst.bytes = ctxt->op_bytes;
1977 return em_pop(ctxt);
1980 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
1985 rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
1986 if (rc != X86EMUL_CONTINUE)
1988 if (ctxt->op_bytes == 4)
1989 ctxt->_eip = (u32)ctxt->_eip;
1990 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1991 if (rc != X86EMUL_CONTINUE)
1993 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1997 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
1999 /* Save real source value, then compare EAX against destination. */
2000 ctxt->src.orig_val = ctxt->src.val;
2001 ctxt->src.val = ctxt->regs[VCPU_REGS_RAX];
2002 emulate_2op_SrcV(ctxt, "cmp");
2004 if (ctxt->eflags & EFLG_ZF) {
2005 /* Success: write back to memory. */
2006 ctxt->dst.val = ctxt->src.orig_val;
2008 /* Failure: write the value we saw to EAX. */
2009 ctxt->dst.type = OP_REG;
2010 ctxt->dst.addr.reg = (unsigned long *)&ctxt->regs[VCPU_REGS_RAX];
2012 return X86EMUL_CONTINUE;
2015 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2017 int seg = ctxt->src2.val;
2021 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2023 rc = load_segment_descriptor(ctxt, sel, seg);
2024 if (rc != X86EMUL_CONTINUE)
2027 ctxt->dst.val = ctxt->src.val;
2032 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2033 struct desc_struct *cs, struct desc_struct *ss)
2037 memset(cs, 0, sizeof(struct desc_struct));
2038 ctxt->ops->get_segment(ctxt, &selector, cs, NULL, VCPU_SREG_CS);
2039 memset(ss, 0, sizeof(struct desc_struct));
2041 cs->l = 0; /* will be adjusted later */
2042 set_desc_base(cs, 0); /* flat segment */
2043 cs->g = 1; /* 4kb granularity */
2044 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2045 cs->type = 0x0b; /* Read, Execute, Accessed */
2047 cs->dpl = 0; /* will be adjusted later */
2051 set_desc_base(ss, 0); /* flat segment */
2052 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2053 ss->g = 1; /* 4kb granularity */
2055 ss->type = 0x03; /* Read/Write, Accessed */
2056 ss->d = 1; /* 32bit stack segment */
2061 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2063 u32 eax, ebx, ecx, edx;
2066 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2067 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2068 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2069 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2072 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2074 struct x86_emulate_ops *ops = ctxt->ops;
2075 u32 eax, ebx, ecx, edx;
2078 * syscall should always be enabled in longmode - so only become
2079 * vendor specific (cpuid) if other modes are active...
2081 if (ctxt->mode == X86EMUL_MODE_PROT64)
2086 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2088 * Intel ("GenuineIntel")
2089 * remark: Intel CPUs only support "syscall" in 64bit
2090 * longmode. Also an 64bit guest with a
2091 * 32bit compat-app running will #UD !! While this
2092 * behaviour can be fixed (by emulating) into AMD
2093 * response - CPUs of AMD can't behave like Intel.
2095 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2096 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2097 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2100 /* AMD ("AuthenticAMD") */
2101 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2102 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2103 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2106 /* AMD ("AMDisbetter!") */
2107 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2108 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2109 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2112 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2116 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2118 struct x86_emulate_ops *ops = ctxt->ops;
2119 struct desc_struct cs, ss;
2124 /* syscall is not available in real mode */
2125 if (ctxt->mode == X86EMUL_MODE_REAL ||
2126 ctxt->mode == X86EMUL_MODE_VM86)
2127 return emulate_ud(ctxt);
2129 if (!(em_syscall_is_enabled(ctxt)))
2130 return emulate_ud(ctxt);
2132 ops->get_msr(ctxt, MSR_EFER, &efer);
2133 setup_syscalls_segments(ctxt, &cs, &ss);
2135 if (!(efer & EFER_SCE))
2136 return emulate_ud(ctxt);
2138 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2140 cs_sel = (u16)(msr_data & 0xfffc);
2141 ss_sel = (u16)(msr_data + 8);
2143 if (efer & EFER_LMA) {
2147 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2148 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2150 ctxt->regs[VCPU_REGS_RCX] = ctxt->_eip;
2151 if (efer & EFER_LMA) {
2152 #ifdef CONFIG_X86_64
2153 ctxt->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
2156 ctxt->mode == X86EMUL_MODE_PROT64 ?
2157 MSR_LSTAR : MSR_CSTAR, &msr_data);
2158 ctxt->_eip = msr_data;
2160 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2161 ctxt->eflags &= ~(msr_data | EFLG_RF);
2165 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2166 ctxt->_eip = (u32)msr_data;
2168 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2171 return X86EMUL_CONTINUE;
2174 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2176 struct x86_emulate_ops *ops = ctxt->ops;
2177 struct desc_struct cs, ss;
2182 ops->get_msr(ctxt, MSR_EFER, &efer);
2183 /* inject #GP if in real mode */
2184 if (ctxt->mode == X86EMUL_MODE_REAL)
2185 return emulate_gp(ctxt, 0);
2188 * Not recognized on AMD in compat mode (but is recognized in legacy
2191 if ((ctxt->mode == X86EMUL_MODE_PROT32) && (efer & EFER_LMA)
2192 && !vendor_intel(ctxt))
2193 return emulate_ud(ctxt);
2195 /* XXX sysenter/sysexit have not been tested in 64bit mode.
2196 * Therefore, we inject an #UD.
2198 if (ctxt->mode == X86EMUL_MODE_PROT64)
2199 return emulate_ud(ctxt);
2201 setup_syscalls_segments(ctxt, &cs, &ss);
2203 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2204 switch (ctxt->mode) {
2205 case X86EMUL_MODE_PROT32:
2206 if ((msr_data & 0xfffc) == 0x0)
2207 return emulate_gp(ctxt, 0);
2209 case X86EMUL_MODE_PROT64:
2210 if (msr_data == 0x0)
2211 return emulate_gp(ctxt, 0);
2215 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2216 cs_sel = (u16)msr_data;
2217 cs_sel &= ~SELECTOR_RPL_MASK;
2218 ss_sel = cs_sel + 8;
2219 ss_sel &= ~SELECTOR_RPL_MASK;
2220 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
2225 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2226 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2228 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2229 ctxt->_eip = msr_data;
2231 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2232 ctxt->regs[VCPU_REGS_RSP] = msr_data;
2234 return X86EMUL_CONTINUE;
2237 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2239 struct x86_emulate_ops *ops = ctxt->ops;
2240 struct desc_struct cs, ss;
2243 u16 cs_sel = 0, ss_sel = 0;
2245 /* inject #GP if in real mode or Virtual 8086 mode */
2246 if (ctxt->mode == X86EMUL_MODE_REAL ||
2247 ctxt->mode == X86EMUL_MODE_VM86)
2248 return emulate_gp(ctxt, 0);
2250 setup_syscalls_segments(ctxt, &cs, &ss);
2252 if ((ctxt->rex_prefix & 0x8) != 0x0)
2253 usermode = X86EMUL_MODE_PROT64;
2255 usermode = X86EMUL_MODE_PROT32;
2259 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2261 case X86EMUL_MODE_PROT32:
2262 cs_sel = (u16)(msr_data + 16);
2263 if ((msr_data & 0xfffc) == 0x0)
2264 return emulate_gp(ctxt, 0);
2265 ss_sel = (u16)(msr_data + 24);
2267 case X86EMUL_MODE_PROT64:
2268 cs_sel = (u16)(msr_data + 32);
2269 if (msr_data == 0x0)
2270 return emulate_gp(ctxt, 0);
2271 ss_sel = cs_sel + 8;
2276 cs_sel |= SELECTOR_RPL_MASK;
2277 ss_sel |= SELECTOR_RPL_MASK;
2279 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2280 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2282 ctxt->_eip = ctxt->regs[VCPU_REGS_RDX];
2283 ctxt->regs[VCPU_REGS_RSP] = ctxt->regs[VCPU_REGS_RCX];
2285 return X86EMUL_CONTINUE;
2288 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2291 if (ctxt->mode == X86EMUL_MODE_REAL)
2293 if (ctxt->mode == X86EMUL_MODE_VM86)
2295 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2296 return ctxt->ops->cpl(ctxt) > iopl;
2299 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2302 struct x86_emulate_ops *ops = ctxt->ops;
2303 struct desc_struct tr_seg;
2306 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2307 unsigned mask = (1 << len) - 1;
2310 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2313 if (desc_limit_scaled(&tr_seg) < 103)
2315 base = get_desc_base(&tr_seg);
2316 #ifdef CONFIG_X86_64
2317 base |= ((u64)base3) << 32;
2319 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2320 if (r != X86EMUL_CONTINUE)
2322 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2324 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2325 if (r != X86EMUL_CONTINUE)
2327 if ((perm >> bit_idx) & mask)
2332 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2338 if (emulator_bad_iopl(ctxt))
2339 if (!emulator_io_port_access_allowed(ctxt, port, len))
2342 ctxt->perm_ok = true;
2347 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2348 struct tss_segment_16 *tss)
2350 tss->ip = ctxt->_eip;
2351 tss->flag = ctxt->eflags;
2352 tss->ax = ctxt->regs[VCPU_REGS_RAX];
2353 tss->cx = ctxt->regs[VCPU_REGS_RCX];
2354 tss->dx = ctxt->regs[VCPU_REGS_RDX];
2355 tss->bx = ctxt->regs[VCPU_REGS_RBX];
2356 tss->sp = ctxt->regs[VCPU_REGS_RSP];
2357 tss->bp = ctxt->regs[VCPU_REGS_RBP];
2358 tss->si = ctxt->regs[VCPU_REGS_RSI];
2359 tss->di = ctxt->regs[VCPU_REGS_RDI];
2361 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2362 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2363 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2364 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2365 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2368 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2369 struct tss_segment_16 *tss)
2373 ctxt->_eip = tss->ip;
2374 ctxt->eflags = tss->flag | 2;
2375 ctxt->regs[VCPU_REGS_RAX] = tss->ax;
2376 ctxt->regs[VCPU_REGS_RCX] = tss->cx;
2377 ctxt->regs[VCPU_REGS_RDX] = tss->dx;
2378 ctxt->regs[VCPU_REGS_RBX] = tss->bx;
2379 ctxt->regs[VCPU_REGS_RSP] = tss->sp;
2380 ctxt->regs[VCPU_REGS_RBP] = tss->bp;
2381 ctxt->regs[VCPU_REGS_RSI] = tss->si;
2382 ctxt->regs[VCPU_REGS_RDI] = tss->di;
2385 * SDM says that segment selectors are loaded before segment
2388 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2389 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2390 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2391 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2392 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2395 * Now load segment descriptors. If fault happenes at this stage
2396 * it is handled in a context of new task
2398 ret = load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR);
2399 if (ret != X86EMUL_CONTINUE)
2401 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2402 if (ret != X86EMUL_CONTINUE)
2404 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2405 if (ret != X86EMUL_CONTINUE)
2407 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2408 if (ret != X86EMUL_CONTINUE)
2410 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2411 if (ret != X86EMUL_CONTINUE)
2414 return X86EMUL_CONTINUE;
2417 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2418 u16 tss_selector, u16 old_tss_sel,
2419 ulong old_tss_base, struct desc_struct *new_desc)
2421 struct x86_emulate_ops *ops = ctxt->ops;
2422 struct tss_segment_16 tss_seg;
2424 u32 new_tss_base = get_desc_base(new_desc);
2426 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2428 if (ret != X86EMUL_CONTINUE)
2429 /* FIXME: need to provide precise fault address */
2432 save_state_to_tss16(ctxt, &tss_seg);
2434 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2436 if (ret != X86EMUL_CONTINUE)
2437 /* FIXME: need to provide precise fault address */
2440 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2442 if (ret != X86EMUL_CONTINUE)
2443 /* FIXME: need to provide precise fault address */
2446 if (old_tss_sel != 0xffff) {
2447 tss_seg.prev_task_link = old_tss_sel;
2449 ret = ops->write_std(ctxt, new_tss_base,
2450 &tss_seg.prev_task_link,
2451 sizeof tss_seg.prev_task_link,
2453 if (ret != X86EMUL_CONTINUE)
2454 /* FIXME: need to provide precise fault address */
2458 return load_state_from_tss16(ctxt, &tss_seg);
2461 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2462 struct tss_segment_32 *tss)
2464 tss->cr3 = ctxt->ops->get_cr(ctxt, 3);
2465 tss->eip = ctxt->_eip;
2466 tss->eflags = ctxt->eflags;
2467 tss->eax = ctxt->regs[VCPU_REGS_RAX];
2468 tss->ecx = ctxt->regs[VCPU_REGS_RCX];
2469 tss->edx = ctxt->regs[VCPU_REGS_RDX];
2470 tss->ebx = ctxt->regs[VCPU_REGS_RBX];
2471 tss->esp = ctxt->regs[VCPU_REGS_RSP];
2472 tss->ebp = ctxt->regs[VCPU_REGS_RBP];
2473 tss->esi = ctxt->regs[VCPU_REGS_RSI];
2474 tss->edi = ctxt->regs[VCPU_REGS_RDI];
2476 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2477 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2478 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2479 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2480 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2481 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2482 tss->ldt_selector = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2485 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2486 struct tss_segment_32 *tss)
2490 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2491 return emulate_gp(ctxt, 0);
2492 ctxt->_eip = tss->eip;
2493 ctxt->eflags = tss->eflags | 2;
2495 /* General purpose registers */
2496 ctxt->regs[VCPU_REGS_RAX] = tss->eax;
2497 ctxt->regs[VCPU_REGS_RCX] = tss->ecx;
2498 ctxt->regs[VCPU_REGS_RDX] = tss->edx;
2499 ctxt->regs[VCPU_REGS_RBX] = tss->ebx;
2500 ctxt->regs[VCPU_REGS_RSP] = tss->esp;
2501 ctxt->regs[VCPU_REGS_RBP] = tss->ebp;
2502 ctxt->regs[VCPU_REGS_RSI] = tss->esi;
2503 ctxt->regs[VCPU_REGS_RDI] = tss->edi;
2506 * SDM says that segment selectors are loaded before segment
2509 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2510 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2511 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2512 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2513 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2514 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2515 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2518 * If we're switching between Protected Mode and VM86, we need to make
2519 * sure to update the mode before loading the segment descriptors so
2520 * that the selectors are interpreted correctly.
2522 * Need to get rflags to the vcpu struct immediately because it
2523 * influences the CPL which is checked at least when loading the segment
2524 * descriptors and when pushing an error code to the new kernel stack.
2526 * TODO Introduce a separate ctxt->ops->set_cpl callback
2528 if (ctxt->eflags & X86_EFLAGS_VM)
2529 ctxt->mode = X86EMUL_MODE_VM86;
2531 ctxt->mode = X86EMUL_MODE_PROT32;
2533 ctxt->ops->set_rflags(ctxt, ctxt->eflags);
2536 * Now load segment descriptors. If fault happenes at this stage
2537 * it is handled in a context of new task
2539 ret = load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2540 if (ret != X86EMUL_CONTINUE)
2542 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2543 if (ret != X86EMUL_CONTINUE)
2545 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2546 if (ret != X86EMUL_CONTINUE)
2548 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2549 if (ret != X86EMUL_CONTINUE)
2551 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2552 if (ret != X86EMUL_CONTINUE)
2554 ret = load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS);
2555 if (ret != X86EMUL_CONTINUE)
2557 ret = load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS);
2558 if (ret != X86EMUL_CONTINUE)
2561 return X86EMUL_CONTINUE;
2564 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2565 u16 tss_selector, u16 old_tss_sel,
2566 ulong old_tss_base, struct desc_struct *new_desc)
2568 struct x86_emulate_ops *ops = ctxt->ops;
2569 struct tss_segment_32 tss_seg;
2571 u32 new_tss_base = get_desc_base(new_desc);
2573 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2575 if (ret != X86EMUL_CONTINUE)
2576 /* FIXME: need to provide precise fault address */
2579 save_state_to_tss32(ctxt, &tss_seg);
2581 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2583 if (ret != X86EMUL_CONTINUE)
2584 /* FIXME: need to provide precise fault address */
2587 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2589 if (ret != X86EMUL_CONTINUE)
2590 /* FIXME: need to provide precise fault address */
2593 if (old_tss_sel != 0xffff) {
2594 tss_seg.prev_task_link = old_tss_sel;
2596 ret = ops->write_std(ctxt, new_tss_base,
2597 &tss_seg.prev_task_link,
2598 sizeof tss_seg.prev_task_link,
2600 if (ret != X86EMUL_CONTINUE)
2601 /* FIXME: need to provide precise fault address */
2605 return load_state_from_tss32(ctxt, &tss_seg);
2608 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2609 u16 tss_selector, int idt_index, int reason,
2610 bool has_error_code, u32 error_code)
2612 struct x86_emulate_ops *ops = ctxt->ops;
2613 struct desc_struct curr_tss_desc, next_tss_desc;
2615 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2616 ulong old_tss_base =
2617 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2621 /* FIXME: old_tss_base == ~0 ? */
2623 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
2624 if (ret != X86EMUL_CONTINUE)
2626 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
2627 if (ret != X86EMUL_CONTINUE)
2630 /* FIXME: check that next_tss_desc is tss */
2633 * Check privileges. The three cases are task switch caused by...
2635 * 1. jmp/call/int to task gate: Check against DPL of the task gate
2636 * 2. Exception/IRQ/iret: No check is performed
2637 * 3. jmp/call to TSS: Check agains DPL of the TSS
2639 if (reason == TASK_SWITCH_GATE) {
2640 if (idt_index != -1) {
2641 /* Software interrupts */
2642 struct desc_struct task_gate_desc;
2645 ret = read_interrupt_descriptor(ctxt, idt_index,
2647 if (ret != X86EMUL_CONTINUE)
2650 dpl = task_gate_desc.dpl;
2651 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2652 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
2654 } else if (reason != TASK_SWITCH_IRET) {
2655 int dpl = next_tss_desc.dpl;
2656 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2657 return emulate_gp(ctxt, tss_selector);
2661 desc_limit = desc_limit_scaled(&next_tss_desc);
2662 if (!next_tss_desc.p ||
2663 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2664 desc_limit < 0x2b)) {
2665 emulate_ts(ctxt, tss_selector & 0xfffc);
2666 return X86EMUL_PROPAGATE_FAULT;
2669 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2670 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2671 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2674 if (reason == TASK_SWITCH_IRET)
2675 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2677 /* set back link to prev task only if NT bit is set in eflags
2678 note that old_tss_sel is not used afetr this point */
2679 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2680 old_tss_sel = 0xffff;
2682 if (next_tss_desc.type & 8)
2683 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2684 old_tss_base, &next_tss_desc);
2686 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2687 old_tss_base, &next_tss_desc);
2688 if (ret != X86EMUL_CONTINUE)
2691 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2692 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2694 if (reason != TASK_SWITCH_IRET) {
2695 next_tss_desc.type |= (1 << 1); /* set busy flag */
2696 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2699 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2700 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2702 if (has_error_code) {
2703 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2704 ctxt->lock_prefix = 0;
2705 ctxt->src.val = (unsigned long) error_code;
2706 ret = em_push(ctxt);
2712 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2713 u16 tss_selector, int idt_index, int reason,
2714 bool has_error_code, u32 error_code)
2718 ctxt->_eip = ctxt->eip;
2719 ctxt->dst.type = OP_NONE;
2721 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
2722 has_error_code, error_code);
2724 if (rc == X86EMUL_CONTINUE)
2725 ctxt->eip = ctxt->_eip;
2727 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2730 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned seg,
2731 int reg, struct operand *op)
2733 int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;
2735 register_address_increment(ctxt, &ctxt->regs[reg], df * op->bytes);
2736 op->addr.mem.ea = register_address(ctxt, ctxt->regs[reg]);
2737 op->addr.mem.seg = seg;
2740 static int em_das(struct x86_emulate_ctxt *ctxt)
2743 bool af, cf, old_cf;
2745 cf = ctxt->eflags & X86_EFLAGS_CF;
2751 af = ctxt->eflags & X86_EFLAGS_AF;
2752 if ((al & 0x0f) > 9 || af) {
2754 cf = old_cf | (al >= 250);
2759 if (old_al > 0x99 || old_cf) {
2765 /* Set PF, ZF, SF */
2766 ctxt->src.type = OP_IMM;
2768 ctxt->src.bytes = 1;
2769 emulate_2op_SrcV(ctxt, "or");
2770 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2772 ctxt->eflags |= X86_EFLAGS_CF;
2774 ctxt->eflags |= X86_EFLAGS_AF;
2775 return X86EMUL_CONTINUE;
2778 static int em_call(struct x86_emulate_ctxt *ctxt)
2780 long rel = ctxt->src.val;
2782 ctxt->src.val = (unsigned long)ctxt->_eip;
2784 return em_push(ctxt);
2787 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2793 old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2794 old_eip = ctxt->_eip;
2796 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2797 if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
2798 return X86EMUL_CONTINUE;
2801 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2803 ctxt->src.val = old_cs;
2805 if (rc != X86EMUL_CONTINUE)
2808 ctxt->src.val = old_eip;
2809 return em_push(ctxt);
2812 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2816 ctxt->dst.type = OP_REG;
2817 ctxt->dst.addr.reg = &ctxt->_eip;
2818 ctxt->dst.bytes = ctxt->op_bytes;
2819 rc = emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
2820 if (rc != X86EMUL_CONTINUE)
2822 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], ctxt->src.val);
2823 return X86EMUL_CONTINUE;
2826 static int em_add(struct x86_emulate_ctxt *ctxt)
2828 emulate_2op_SrcV(ctxt, "add");
2829 return X86EMUL_CONTINUE;
2832 static int em_or(struct x86_emulate_ctxt *ctxt)
2834 emulate_2op_SrcV(ctxt, "or");
2835 return X86EMUL_CONTINUE;
2838 static int em_adc(struct x86_emulate_ctxt *ctxt)
2840 emulate_2op_SrcV(ctxt, "adc");
2841 return X86EMUL_CONTINUE;
2844 static int em_sbb(struct x86_emulate_ctxt *ctxt)
2846 emulate_2op_SrcV(ctxt, "sbb");
2847 return X86EMUL_CONTINUE;
2850 static int em_and(struct x86_emulate_ctxt *ctxt)
2852 emulate_2op_SrcV(ctxt, "and");
2853 return X86EMUL_CONTINUE;
2856 static int em_sub(struct x86_emulate_ctxt *ctxt)
2858 emulate_2op_SrcV(ctxt, "sub");
2859 return X86EMUL_CONTINUE;
2862 static int em_xor(struct x86_emulate_ctxt *ctxt)
2864 emulate_2op_SrcV(ctxt, "xor");
2865 return X86EMUL_CONTINUE;
2868 static int em_cmp(struct x86_emulate_ctxt *ctxt)
2870 emulate_2op_SrcV(ctxt, "cmp");
2871 /* Disable writeback. */
2872 ctxt->dst.type = OP_NONE;
2873 return X86EMUL_CONTINUE;
2876 static int em_test(struct x86_emulate_ctxt *ctxt)
2878 emulate_2op_SrcV(ctxt, "test");
2879 /* Disable writeback. */
2880 ctxt->dst.type = OP_NONE;
2881 return X86EMUL_CONTINUE;
2884 static int em_xchg(struct x86_emulate_ctxt *ctxt)
2886 /* Write back the register source. */
2887 ctxt->src.val = ctxt->dst.val;
2888 write_register_operand(&ctxt->src);
2890 /* Write back the memory destination with implicit LOCK prefix. */
2891 ctxt->dst.val = ctxt->src.orig_val;
2892 ctxt->lock_prefix = 1;
2893 return X86EMUL_CONTINUE;
2896 static int em_imul(struct x86_emulate_ctxt *ctxt)
2898 emulate_2op_SrcV_nobyte(ctxt, "imul");
2899 return X86EMUL_CONTINUE;
2902 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2904 ctxt->dst.val = ctxt->src2.val;
2905 return em_imul(ctxt);
2908 static int em_cwd(struct x86_emulate_ctxt *ctxt)
2910 ctxt->dst.type = OP_REG;
2911 ctxt->dst.bytes = ctxt->src.bytes;
2912 ctxt->dst.addr.reg = &ctxt->regs[VCPU_REGS_RDX];
2913 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
2915 return X86EMUL_CONTINUE;
2918 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2922 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
2923 ctxt->regs[VCPU_REGS_RAX] = (u32)tsc;
2924 ctxt->regs[VCPU_REGS_RDX] = tsc >> 32;
2925 return X86EMUL_CONTINUE;
2928 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
2932 if (ctxt->ops->read_pmc(ctxt, ctxt->regs[VCPU_REGS_RCX], &pmc))
2933 return emulate_gp(ctxt, 0);
2934 ctxt->regs[VCPU_REGS_RAX] = (u32)pmc;
2935 ctxt->regs[VCPU_REGS_RDX] = pmc >> 32;
2936 return X86EMUL_CONTINUE;
2939 static int em_mov(struct x86_emulate_ctxt *ctxt)
2941 memcpy(ctxt->dst.valptr, ctxt->src.valptr, ctxt->op_bytes);
2942 return X86EMUL_CONTINUE;
2945 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
2947 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
2948 return emulate_gp(ctxt, 0);
2950 /* Disable writeback. */
2951 ctxt->dst.type = OP_NONE;
2952 return X86EMUL_CONTINUE;
2955 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
2959 if (ctxt->mode == X86EMUL_MODE_PROT64)
2960 val = ctxt->src.val & ~0ULL;
2962 val = ctxt->src.val & ~0U;
2964 /* #UD condition is already handled. */
2965 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
2966 return emulate_gp(ctxt, 0);
2968 /* Disable writeback. */
2969 ctxt->dst.type = OP_NONE;
2970 return X86EMUL_CONTINUE;
2973 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
2977 msr_data = (u32)ctxt->regs[VCPU_REGS_RAX]
2978 | ((u64)ctxt->regs[VCPU_REGS_RDX] << 32);
2979 if (ctxt->ops->set_msr(ctxt, ctxt->regs[VCPU_REGS_RCX], msr_data))
2980 return emulate_gp(ctxt, 0);
2982 return X86EMUL_CONTINUE;
2985 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
2989 if (ctxt->ops->get_msr(ctxt, ctxt->regs[VCPU_REGS_RCX], &msr_data))
2990 return emulate_gp(ctxt, 0);
2992 ctxt->regs[VCPU_REGS_RAX] = (u32)msr_data;
2993 ctxt->regs[VCPU_REGS_RDX] = msr_data >> 32;
2994 return X86EMUL_CONTINUE;
2997 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
2999 if (ctxt->modrm_reg > VCPU_SREG_GS)
3000 return emulate_ud(ctxt);
3002 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3003 return X86EMUL_CONTINUE;
3006 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3008 u16 sel = ctxt->src.val;
3010 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3011 return emulate_ud(ctxt);
3013 if (ctxt->modrm_reg == VCPU_SREG_SS)
3014 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3016 /* Disable writeback. */
3017 ctxt->dst.type = OP_NONE;
3018 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3021 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3023 u16 sel = ctxt->src.val;
3025 /* Disable writeback. */
3026 ctxt->dst.type = OP_NONE;
3027 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3030 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3035 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3036 if (rc == X86EMUL_CONTINUE)
3037 ctxt->ops->invlpg(ctxt, linear);
3038 /* Disable writeback. */
3039 ctxt->dst.type = OP_NONE;
3040 return X86EMUL_CONTINUE;
3043 static int em_clts(struct x86_emulate_ctxt *ctxt)
3047 cr0 = ctxt->ops->get_cr(ctxt, 0);
3049 ctxt->ops->set_cr(ctxt, 0, cr0);
3050 return X86EMUL_CONTINUE;
3053 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
3057 if (ctxt->modrm_mod != 3 || ctxt->modrm_rm != 1)
3058 return X86EMUL_UNHANDLEABLE;
3060 rc = ctxt->ops->fix_hypercall(ctxt);
3061 if (rc != X86EMUL_CONTINUE)
3064 /* Let the processor re-execute the fixed hypercall */
3065 ctxt->_eip = ctxt->eip;
3066 /* Disable writeback. */
3067 ctxt->dst.type = OP_NONE;
3068 return X86EMUL_CONTINUE;
3071 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3072 void (*get)(struct x86_emulate_ctxt *ctxt,
3073 struct desc_ptr *ptr))
3075 struct desc_ptr desc_ptr;
3077 if (ctxt->mode == X86EMUL_MODE_PROT64)
3079 get(ctxt, &desc_ptr);
3080 if (ctxt->op_bytes == 2) {
3082 desc_ptr.address &= 0x00ffffff;
3084 /* Disable writeback. */
3085 ctxt->dst.type = OP_NONE;
3086 return segmented_write(ctxt, ctxt->dst.addr.mem,
3087 &desc_ptr, 2 + ctxt->op_bytes);
3090 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3092 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3095 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3097 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3100 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3102 struct desc_ptr desc_ptr;
3105 if (ctxt->mode == X86EMUL_MODE_PROT64)
3107 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3108 &desc_ptr.size, &desc_ptr.address,
3110 if (rc != X86EMUL_CONTINUE)
3112 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3113 /* Disable writeback. */
3114 ctxt->dst.type = OP_NONE;
3115 return X86EMUL_CONTINUE;
3118 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
3122 rc = ctxt->ops->fix_hypercall(ctxt);
3124 /* Disable writeback. */
3125 ctxt->dst.type = OP_NONE;
3129 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3131 struct desc_ptr desc_ptr;
3134 if (ctxt->mode == X86EMUL_MODE_PROT64)
3136 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3137 &desc_ptr.size, &desc_ptr.address,
3139 if (rc != X86EMUL_CONTINUE)
3141 ctxt->ops->set_idt(ctxt, &desc_ptr);
3142 /* Disable writeback. */
3143 ctxt->dst.type = OP_NONE;
3144 return X86EMUL_CONTINUE;
3147 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3149 ctxt->dst.bytes = 2;
3150 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3151 return X86EMUL_CONTINUE;
3154 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3156 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3157 | (ctxt->src.val & 0x0f));
3158 ctxt->dst.type = OP_NONE;
3159 return X86EMUL_CONTINUE;
3162 static int em_loop(struct x86_emulate_ctxt *ctxt)
3164 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1);
3165 if ((address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) != 0) &&
3166 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3167 jmp_rel(ctxt, ctxt->src.val);
3169 return X86EMUL_CONTINUE;
3172 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3174 if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0)
3175 jmp_rel(ctxt, ctxt->src.val);
3177 return X86EMUL_CONTINUE;
3180 static int em_in(struct x86_emulate_ctxt *ctxt)
3182 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3184 return X86EMUL_IO_NEEDED;
3186 return X86EMUL_CONTINUE;
3189 static int em_out(struct x86_emulate_ctxt *ctxt)
3191 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3193 /* Disable writeback. */
3194 ctxt->dst.type = OP_NONE;
3195 return X86EMUL_CONTINUE;
3198 static int em_cli(struct x86_emulate_ctxt *ctxt)
3200 if (emulator_bad_iopl(ctxt))
3201 return emulate_gp(ctxt, 0);
3203 ctxt->eflags &= ~X86_EFLAGS_IF;
3204 return X86EMUL_CONTINUE;
3207 static int em_sti(struct x86_emulate_ctxt *ctxt)
3209 if (emulator_bad_iopl(ctxt))
3210 return emulate_gp(ctxt, 0);
3212 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3213 ctxt->eflags |= X86_EFLAGS_IF;
3214 return X86EMUL_CONTINUE;
3217 static int em_bt(struct x86_emulate_ctxt *ctxt)
3219 /* Disable writeback. */
3220 ctxt->dst.type = OP_NONE;
3221 /* only subword offset */
3222 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
3224 emulate_2op_SrcV_nobyte(ctxt, "bt");
3225 return X86EMUL_CONTINUE;
3228 static int em_bts(struct x86_emulate_ctxt *ctxt)
3230 emulate_2op_SrcV_nobyte(ctxt, "bts");
3231 return X86EMUL_CONTINUE;
3234 static int em_btr(struct x86_emulate_ctxt *ctxt)
3236 emulate_2op_SrcV_nobyte(ctxt, "btr");
3237 return X86EMUL_CONTINUE;
3240 static int em_btc(struct x86_emulate_ctxt *ctxt)
3242 emulate_2op_SrcV_nobyte(ctxt, "btc");
3243 return X86EMUL_CONTINUE;
3246 static int em_bsf(struct x86_emulate_ctxt *ctxt)
3248 emulate_2op_SrcV_nobyte(ctxt, "bsf");
3249 return X86EMUL_CONTINUE;
3252 static int em_bsr(struct x86_emulate_ctxt *ctxt)
3254 emulate_2op_SrcV_nobyte(ctxt, "bsr");
3255 return X86EMUL_CONTINUE;
3258 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3260 u32 eax, ebx, ecx, edx;
3262 eax = ctxt->regs[VCPU_REGS_RAX];
3263 ecx = ctxt->regs[VCPU_REGS_RCX];
3264 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3265 ctxt->regs[VCPU_REGS_RAX] = eax;
3266 ctxt->regs[VCPU_REGS_RBX] = ebx;
3267 ctxt->regs[VCPU_REGS_RCX] = ecx;
3268 ctxt->regs[VCPU_REGS_RDX] = edx;
3269 return X86EMUL_CONTINUE;
3272 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3274 ctxt->regs[VCPU_REGS_RAX] &= ~0xff00UL;
3275 ctxt->regs[VCPU_REGS_RAX] |= (ctxt->eflags & 0xff) << 8;
3276 return X86EMUL_CONTINUE;
3279 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3281 switch (ctxt->op_bytes) {
3282 #ifdef CONFIG_X86_64
3284 asm("bswap %0" : "+r"(ctxt->dst.val));
3288 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3291 return X86EMUL_CONTINUE;
3294 static bool valid_cr(int nr)
3306 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
3308 if (!valid_cr(ctxt->modrm_reg))
3309 return emulate_ud(ctxt);
3311 return X86EMUL_CONTINUE;
3314 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
3316 u64 new_val = ctxt->src.val64;
3317 int cr = ctxt->modrm_reg;
3320 static u64 cr_reserved_bits[] = {
3321 0xffffffff00000000ULL,
3322 0, 0, 0, /* CR3 checked later */
3329 return emulate_ud(ctxt);
3331 if (new_val & cr_reserved_bits[cr])
3332 return emulate_gp(ctxt, 0);
3337 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
3338 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
3339 return emulate_gp(ctxt, 0);
3341 cr4 = ctxt->ops->get_cr(ctxt, 4);
3342 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3344 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
3345 !(cr4 & X86_CR4_PAE))
3346 return emulate_gp(ctxt, 0);
3353 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3354 if (efer & EFER_LMA)
3355 rsvd = CR3_L_MODE_RESERVED_BITS;
3356 else if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PAE)
3357 rsvd = CR3_PAE_RESERVED_BITS;
3358 else if (ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PG)
3359 rsvd = CR3_NONPAE_RESERVED_BITS;
3362 return emulate_gp(ctxt, 0);
3367 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3369 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
3370 return emulate_gp(ctxt, 0);
3376 return X86EMUL_CONTINUE;
3379 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
3383 ctxt->ops->get_dr(ctxt, 7, &dr7);
3385 /* Check if DR7.Global_Enable is set */
3386 return dr7 & (1 << 13);
3389 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
3391 int dr = ctxt->modrm_reg;
3395 return emulate_ud(ctxt);
3397 cr4 = ctxt->ops->get_cr(ctxt, 4);
3398 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
3399 return emulate_ud(ctxt);
3401 if (check_dr7_gd(ctxt))
3402 return emulate_db(ctxt);
3404 return X86EMUL_CONTINUE;
3407 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
3409 u64 new_val = ctxt->src.val64;
3410 int dr = ctxt->modrm_reg;
3412 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
3413 return emulate_gp(ctxt, 0);
3415 return check_dr_read(ctxt);
3418 static int check_svme(struct x86_emulate_ctxt *ctxt)
3422 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3424 if (!(efer & EFER_SVME))
3425 return emulate_ud(ctxt);
3427 return X86EMUL_CONTINUE;
3430 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
3432 u64 rax = ctxt->regs[VCPU_REGS_RAX];
3434 /* Valid physical address? */
3435 if (rax & 0xffff000000000000ULL)
3436 return emulate_gp(ctxt, 0);
3438 return check_svme(ctxt);
3441 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
3443 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3445 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3446 return emulate_ud(ctxt);
3448 return X86EMUL_CONTINUE;
3451 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
3453 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3454 u64 rcx = ctxt->regs[VCPU_REGS_RCX];
3456 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3458 return emulate_gp(ctxt, 0);
3460 return X86EMUL_CONTINUE;
3463 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
3465 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
3466 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3467 return emulate_gp(ctxt, 0);
3469 return X86EMUL_CONTINUE;
3472 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
3474 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
3475 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3476 return emulate_gp(ctxt, 0);
3478 return X86EMUL_CONTINUE;
3481 #define D(_y) { .flags = (_y) }
3482 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
3483 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
3484 .check_perm = (_p) }
3486 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3487 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
3488 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
3489 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3490 #define II(_f, _e, _i) \
3491 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
3492 #define IIP(_f, _e, _i, _p) \
3493 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
3494 .check_perm = (_p) }
3495 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3497 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3498 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3499 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3500 #define I2bvIP(_f, _e, _i, _p) \
3501 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
3503 #define I6ALU(_f, _e) I2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3504 I2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3505 I2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3507 static struct opcode group7_rm1[] = {
3508 DI(SrcNone | Priv, monitor),
3509 DI(SrcNone | Priv, mwait),
3513 static struct opcode group7_rm3[] = {
3514 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
3515 II(SrcNone | Prot | VendorSpecific, em_vmmcall, vmmcall),
3516 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
3517 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
3518 DIP(SrcNone | Prot | Priv, stgi, check_svme),
3519 DIP(SrcNone | Prot | Priv, clgi, check_svme),
3520 DIP(SrcNone | Prot | Priv, skinit, check_svme),
3521 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
3524 static struct opcode group7_rm7[] = {
3526 DIP(SrcNone, rdtscp, check_rdtsc),
3530 static struct opcode group1[] = {
3532 I(Lock | PageTable, em_or),
3535 I(Lock | PageTable, em_and),
3541 static struct opcode group1A[] = {
3542 I(DstMem | SrcNone | Mov | Stack, em_pop), N, N, N, N, N, N, N,
3545 static struct opcode group3[] = {
3546 I(DstMem | SrcImm, em_test),
3547 I(DstMem | SrcImm, em_test),
3548 I(DstMem | SrcNone | Lock, em_not),
3549 I(DstMem | SrcNone | Lock, em_neg),
3550 I(SrcMem, em_mul_ex),
3551 I(SrcMem, em_imul_ex),
3552 I(SrcMem, em_div_ex),
3553 I(SrcMem, em_idiv_ex),
3556 static struct opcode group4[] = {
3557 I(ByteOp | DstMem | SrcNone | Lock, em_grp45),
3558 I(ByteOp | DstMem | SrcNone | Lock, em_grp45),
3562 static struct opcode group5[] = {
3563 I(DstMem | SrcNone | Lock, em_grp45),
3564 I(DstMem | SrcNone | Lock, em_grp45),
3565 I(SrcMem | Stack, em_grp45),
3566 I(SrcMemFAddr | ImplicitOps | Stack, em_call_far),
3567 I(SrcMem | Stack, em_grp45),
3568 I(SrcMemFAddr | ImplicitOps, em_grp45),
3569 I(SrcMem | Stack, em_grp45), N,
3572 static struct opcode group6[] = {
3575 II(Prot | Priv | SrcMem16, em_lldt, lldt),
3576 DI(Prot | Priv, ltr),
3580 static struct group_dual group7 = { {
3581 II(Mov | DstMem | Priv, em_sgdt, sgdt),
3582 II(Mov | DstMem | Priv, em_sidt, sidt),
3583 II(SrcMem | Priv, em_lgdt, lgdt),
3584 II(SrcMem | Priv, em_lidt, lidt),
3585 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3586 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3587 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3589 I(SrcNone | Priv | VendorSpecific, em_vmcall),
3591 N, EXT(0, group7_rm3),
3592 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3593 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3597 static struct opcode group8[] = {
3599 I(DstMem | SrcImmByte, em_bt),
3600 I(DstMem | SrcImmByte | Lock | PageTable, em_bts),
3601 I(DstMem | SrcImmByte | Lock, em_btr),
3602 I(DstMem | SrcImmByte | Lock | PageTable, em_btc),
3605 static struct group_dual group9 = { {
3606 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
3608 N, N, N, N, N, N, N, N,
3611 static struct opcode group11[] = {
3612 I(DstMem | SrcImm | Mov | PageTable, em_mov),
3616 static struct gprefix pfx_0f_6f_0f_7f = {
3617 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
3620 static struct gprefix pfx_vmovntpx = {
3621 I(0, em_mov), N, N, N,
3624 static struct opcode opcode_table[256] = {
3626 I6ALU(Lock, em_add),
3627 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
3628 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
3630 I6ALU(Lock | PageTable, em_or),
3631 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
3634 I6ALU(Lock, em_adc),
3635 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
3636 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
3638 I6ALU(Lock, em_sbb),
3639 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
3640 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
3642 I6ALU(Lock | PageTable, em_and), N, N,
3644 I6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3646 I6ALU(Lock, em_xor), N, N,
3648 I6ALU(0, em_cmp), N, N,
3652 X8(I(SrcReg | Stack, em_push)),
3654 X8(I(DstReg | Stack, em_pop)),
3656 I(ImplicitOps | Stack | No64, em_pusha),
3657 I(ImplicitOps | Stack | No64, em_popa),
3658 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3661 I(SrcImm | Mov | Stack, em_push),
3662 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3663 I(SrcImmByte | Mov | Stack, em_push),
3664 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3665 I2bvIP(DstDI | SrcDX | Mov | String, em_in, ins, check_perm_in), /* insb, insw/insd */
3666 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
3670 G(ByteOp | DstMem | SrcImm, group1),
3671 G(DstMem | SrcImm, group1),
3672 G(ByteOp | DstMem | SrcImm | No64, group1),
3673 G(DstMem | SrcImmByte, group1),
3674 I2bv(DstMem | SrcReg | ModRM, em_test),
3675 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
3677 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
3678 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3679 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
3680 D(ModRM | SrcMem | NoAccess | DstReg),
3681 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
3684 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3686 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3687 I(SrcImmFAddr | No64, em_call_far), N,
3688 II(ImplicitOps | Stack, em_pushf, pushf),
3689 II(ImplicitOps | Stack, em_popf, popf), N, I(ImplicitOps, em_lahf),
3691 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3692 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
3693 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3694 I2bv(SrcSI | DstDI | String, em_cmp),
3696 I2bv(DstAcc | SrcImm, em_test),
3697 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3698 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3699 I2bv(SrcAcc | DstDI | String, em_cmp),
3701 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3703 X8(I(DstReg | SrcImm | Mov, em_mov)),
3705 D2bv(DstMem | SrcImmByte | ModRM),
3706 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3707 I(ImplicitOps | Stack, em_ret),
3708 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
3709 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
3710 G(ByteOp, group11), G(0, group11),
3712 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
3713 N, I(ImplicitOps | Stack, em_ret_far),
3714 D(ImplicitOps), DI(SrcImmByte, intn),
3715 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3717 D2bv(DstMem | SrcOne | ModRM), D2bv(DstMem | ModRM),
3720 N, N, N, N, N, N, N, N,
3722 X3(I(SrcImmByte, em_loop)),
3723 I(SrcImmByte, em_jcxz),
3724 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
3725 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
3727 I(SrcImm | Stack, em_call), D(SrcImm | ImplicitOps),
3728 I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
3729 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
3730 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
3732 N, DI(ImplicitOps, icebp), N, N,
3733 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
3734 G(ByteOp, group3), G(0, group3),
3736 D(ImplicitOps), D(ImplicitOps),
3737 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
3738 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3741 static struct opcode twobyte_table[256] = {
3743 G(0, group6), GD(0, &group7), N, N,
3744 N, I(ImplicitOps | VendorSpecific, em_syscall),
3745 II(ImplicitOps | Priv, em_clts, clts), N,
3746 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3747 N, D(ImplicitOps | ModRM), N, N,
3749 N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
3751 DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
3752 DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
3753 IIP(ModRM | SrcMem | Priv | Op3264, em_cr_write, cr_write, check_cr_write),
3754 IIP(ModRM | SrcMem | Priv | Op3264, em_dr_write, dr_write, check_dr_write),
3756 N, N, N, GP(ModRM | DstMem | SrcReg | Sse | Mov | Aligned, &pfx_vmovntpx),
3759 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
3760 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
3761 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
3762 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
3763 I(ImplicitOps | VendorSpecific, em_sysenter),
3764 I(ImplicitOps | Priv | VendorSpecific, em_sysexit),
3766 N, N, N, N, N, N, N, N,
3768 X16(D(DstReg | SrcMem | ModRM | Mov)),
3770 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3775 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3780 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3784 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3786 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
3787 II(ImplicitOps, em_cpuid, cpuid), I(DstMem | SrcReg | ModRM | BitOp, em_bt),
3788 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3789 D(DstMem | SrcReg | Src2CL | ModRM), N, N,
3791 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
3792 DI(ImplicitOps, rsm),
3793 I(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
3794 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3795 D(DstMem | SrcReg | Src2CL | ModRM),
3796 D(ModRM), I(DstReg | SrcMem | ModRM, em_imul),
3798 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_cmpxchg),
3799 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
3800 I(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
3801 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
3802 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
3803 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3807 I(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
3808 I(DstReg | SrcMem | ModRM, em_bsf), I(DstReg | SrcMem | ModRM, em_bsr),
3809 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3811 D2bv(DstMem | SrcReg | ModRM | Lock),
3812 N, D(DstMem | SrcReg | ModRM | Mov),
3813 N, N, N, GD(0, &group9),
3815 X8(I(DstReg, em_bswap)),
3817 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3819 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3821 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
3838 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
3842 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3848 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
3849 unsigned size, bool sign_extension)
3851 int rc = X86EMUL_CONTINUE;
3855 op->addr.mem.ea = ctxt->_eip;
3856 /* NB. Immediates are sign-extended as necessary. */
3857 switch (op->bytes) {
3859 op->val = insn_fetch(s8, ctxt);
3862 op->val = insn_fetch(s16, ctxt);
3865 op->val = insn_fetch(s32, ctxt);
3868 if (!sign_extension) {
3869 switch (op->bytes) {
3877 op->val &= 0xffffffff;
3885 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
3888 int rc = X86EMUL_CONTINUE;
3892 decode_register_operand(ctxt, op);
3895 rc = decode_imm(ctxt, op, 1, false);
3898 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3902 if ((ctxt->d & BitOp) && op == &ctxt->dst)
3903 fetch_bit_operand(ctxt);
3904 op->orig_val = op->val;
3907 ctxt->memop.bytes = 8;
3911 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3912 op->addr.reg = &ctxt->regs[VCPU_REGS_RAX];
3913 fetch_register_operand(op);
3914 op->orig_val = op->val;
3918 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3920 register_address(ctxt, ctxt->regs[VCPU_REGS_RDI]);
3921 op->addr.mem.seg = VCPU_SREG_ES;
3927 op->addr.reg = &ctxt->regs[VCPU_REGS_RDX];
3928 fetch_register_operand(op);
3932 op->val = ctxt->regs[VCPU_REGS_RCX] & 0xff;
3935 rc = decode_imm(ctxt, op, 1, true);
3942 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
3945 ctxt->memop.bytes = 1;
3948 ctxt->memop.bytes = 2;
3951 ctxt->memop.bytes = 4;
3954 rc = decode_imm(ctxt, op, 2, false);
3957 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
3961 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3963 register_address(ctxt, ctxt->regs[VCPU_REGS_RSI]);
3964 op->addr.mem.seg = seg_override(ctxt);
3969 op->addr.mem.ea = ctxt->_eip;
3970 op->bytes = ctxt->op_bytes + 2;
3971 insn_fetch_arr(op->valptr, op->bytes, ctxt);
3974 ctxt->memop.bytes = ctxt->op_bytes + 2;
3977 op->val = VCPU_SREG_ES;
3980 op->val = VCPU_SREG_CS;
3983 op->val = VCPU_SREG_SS;
3986 op->val = VCPU_SREG_DS;
3989 op->val = VCPU_SREG_FS;
3992 op->val = VCPU_SREG_GS;
3995 /* Special instructions do their own operand decoding. */
3997 op->type = OP_NONE; /* Disable writeback. */
4005 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
4007 int rc = X86EMUL_CONTINUE;
4008 int mode = ctxt->mode;
4009 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
4010 bool op_prefix = false;
4011 struct opcode opcode;
4013 ctxt->memop.type = OP_NONE;
4014 ctxt->memopp = NULL;
4015 ctxt->_eip = ctxt->eip;
4016 ctxt->fetch.start = ctxt->_eip;
4017 ctxt->fetch.end = ctxt->fetch.start + insn_len;
4019 memcpy(ctxt->fetch.data, insn, insn_len);
4022 case X86EMUL_MODE_REAL:
4023 case X86EMUL_MODE_VM86:
4024 case X86EMUL_MODE_PROT16:
4025 def_op_bytes = def_ad_bytes = 2;
4027 case X86EMUL_MODE_PROT32:
4028 def_op_bytes = def_ad_bytes = 4;
4030 #ifdef CONFIG_X86_64
4031 case X86EMUL_MODE_PROT64:
4037 return EMULATION_FAILED;
4040 ctxt->op_bytes = def_op_bytes;
4041 ctxt->ad_bytes = def_ad_bytes;
4043 /* Legacy prefixes. */
4045 switch (ctxt->b = insn_fetch(u8, ctxt)) {
4046 case 0x66: /* operand-size override */
4048 /* switch between 2/4 bytes */
4049 ctxt->op_bytes = def_op_bytes ^ 6;
4051 case 0x67: /* address-size override */
4052 if (mode == X86EMUL_MODE_PROT64)
4053 /* switch between 4/8 bytes */
4054 ctxt->ad_bytes = def_ad_bytes ^ 12;
4056 /* switch between 2/4 bytes */
4057 ctxt->ad_bytes = def_ad_bytes ^ 6;
4059 case 0x26: /* ES override */
4060 case 0x2e: /* CS override */
4061 case 0x36: /* SS override */
4062 case 0x3e: /* DS override */
4063 set_seg_override(ctxt, (ctxt->b >> 3) & 3);
4065 case 0x64: /* FS override */
4066 case 0x65: /* GS override */
4067 set_seg_override(ctxt, ctxt->b & 7);
4069 case 0x40 ... 0x4f: /* REX */
4070 if (mode != X86EMUL_MODE_PROT64)
4072 ctxt->rex_prefix = ctxt->b;
4074 case 0xf0: /* LOCK */
4075 ctxt->lock_prefix = 1;
4077 case 0xf2: /* REPNE/REPNZ */
4078 case 0xf3: /* REP/REPE/REPZ */
4079 ctxt->rep_prefix = ctxt->b;
4085 /* Any legacy prefix after a REX prefix nullifies its effect. */
4087 ctxt->rex_prefix = 0;
4093 if (ctxt->rex_prefix & 8)
4094 ctxt->op_bytes = 8; /* REX.W */
4096 /* Opcode byte(s). */
4097 opcode = opcode_table[ctxt->b];
4098 /* Two-byte opcode? */
4099 if (ctxt->b == 0x0f) {
4101 ctxt->b = insn_fetch(u8, ctxt);
4102 opcode = twobyte_table[ctxt->b];
4104 ctxt->d = opcode.flags;
4106 if (ctxt->d & ModRM)
4107 ctxt->modrm = insn_fetch(u8, ctxt);
4109 while (ctxt->d & GroupMask) {
4110 switch (ctxt->d & GroupMask) {
4112 goffset = (ctxt->modrm >> 3) & 7;
4113 opcode = opcode.u.group[goffset];
4116 goffset = (ctxt->modrm >> 3) & 7;
4117 if ((ctxt->modrm >> 6) == 3)
4118 opcode = opcode.u.gdual->mod3[goffset];
4120 opcode = opcode.u.gdual->mod012[goffset];
4123 goffset = ctxt->modrm & 7;
4124 opcode = opcode.u.group[goffset];
4127 if (ctxt->rep_prefix && op_prefix)
4128 return EMULATION_FAILED;
4129 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
4130 switch (simd_prefix) {
4131 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
4132 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
4133 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
4134 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
4138 return EMULATION_FAILED;
4141 ctxt->d &= ~(u64)GroupMask;
4142 ctxt->d |= opcode.flags;
4145 ctxt->execute = opcode.u.execute;
4146 ctxt->check_perm = opcode.check_perm;
4147 ctxt->intercept = opcode.intercept;
4150 if (ctxt->d == 0 || (ctxt->d & Undefined))
4151 return EMULATION_FAILED;
4153 if (!(ctxt->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
4154 return EMULATION_FAILED;
4156 if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
4159 if (ctxt->d & Op3264) {
4160 if (mode == X86EMUL_MODE_PROT64)
4167 ctxt->op_bytes = 16;
4168 else if (ctxt->d & Mmx)
4171 /* ModRM and SIB bytes. */
4172 if (ctxt->d & ModRM) {
4173 rc = decode_modrm(ctxt, &ctxt->memop);
4174 if (!ctxt->has_seg_override)
4175 set_seg_override(ctxt, ctxt->modrm_seg);
4176 } else if (ctxt->d & MemAbs)
4177 rc = decode_abs(ctxt, &ctxt->memop);
4178 if (rc != X86EMUL_CONTINUE)
4181 if (!ctxt->has_seg_override)
4182 set_seg_override(ctxt, VCPU_SREG_DS);
4184 ctxt->memop.addr.mem.seg = seg_override(ctxt);
4186 if (ctxt->memop.type == OP_MEM && ctxt->ad_bytes != 8)
4187 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
4190 * Decode and fetch the source operand: register, memory
4193 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
4194 if (rc != X86EMUL_CONTINUE)
4198 * Decode and fetch the second source operand: register, memory
4201 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
4202 if (rc != X86EMUL_CONTINUE)
4205 /* Decode and fetch the destination operand: register or memory. */
4206 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
4209 if (ctxt->memopp && ctxt->memopp->type == OP_MEM && ctxt->rip_relative)
4210 ctxt->memopp->addr.mem.ea += ctxt->_eip;
4212 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
4215 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
4217 return ctxt->d & PageTable;
4220 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
4222 /* The second termination condition only applies for REPE
4223 * and REPNE. Test if the repeat string operation prefix is
4224 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
4225 * corresponding termination condition according to:
4226 * - if REPE/REPZ and ZF = 0 then done
4227 * - if REPNE/REPNZ and ZF = 1 then done
4229 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
4230 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
4231 && (((ctxt->rep_prefix == REPE_PREFIX) &&
4232 ((ctxt->eflags & EFLG_ZF) == 0))
4233 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
4234 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
4240 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
4244 ctxt->ops->get_fpu(ctxt);
4245 asm volatile("1: fwait \n\t"
4247 ".pushsection .fixup,\"ax\" \n\t"
4249 "movb $1, %[fault] \n\t"
4252 _ASM_EXTABLE(1b, 3b)
4253 : [fault]"+qm"(fault));
4254 ctxt->ops->put_fpu(ctxt);
4256 if (unlikely(fault))
4257 return emulate_exception(ctxt, MF_VECTOR, 0, false);
4259 return X86EMUL_CONTINUE;
4262 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
4265 if (op->type == OP_MM)
4266 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
4269 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
4271 struct x86_emulate_ops *ops = ctxt->ops;
4272 int rc = X86EMUL_CONTINUE;
4273 int saved_dst_type = ctxt->dst.type;
4275 ctxt->mem_read.pos = 0;
4277 if (ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) {
4278 rc = emulate_ud(ctxt);
4282 /* LOCK prefix is allowed only with some instructions */
4283 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
4284 rc = emulate_ud(ctxt);
4288 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
4289 rc = emulate_ud(ctxt);
4293 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
4294 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
4295 rc = emulate_ud(ctxt);
4299 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
4300 rc = emulate_nm(ctxt);
4304 if (ctxt->d & Mmx) {
4305 rc = flush_pending_x87_faults(ctxt);
4306 if (rc != X86EMUL_CONTINUE)
4309 * Now that we know the fpu is exception safe, we can fetch
4312 fetch_possible_mmx_operand(ctxt, &ctxt->src);
4313 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
4314 if (!(ctxt->d & Mov))
4315 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
4318 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4319 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4320 X86_ICPT_PRE_EXCEPT);
4321 if (rc != X86EMUL_CONTINUE)
4325 /* Privileged instruction can be executed only in CPL=0 */
4326 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
4327 rc = emulate_gp(ctxt, 0);
4331 /* Instruction can only be executed in protected mode */
4332 if ((ctxt->d & Prot) && !(ctxt->mode & X86EMUL_MODE_PROT)) {
4333 rc = emulate_ud(ctxt);
4337 /* Do instruction specific permission checks */
4338 if (ctxt->check_perm) {
4339 rc = ctxt->check_perm(ctxt);
4340 if (rc != X86EMUL_CONTINUE)
4344 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4345 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4346 X86_ICPT_POST_EXCEPT);
4347 if (rc != X86EMUL_CONTINUE)
4351 if (ctxt->rep_prefix && (ctxt->d & String)) {
4352 /* All REP prefixes have the same first termination condition */
4353 if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0) {
4354 ctxt->eip = ctxt->_eip;
4359 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
4360 rc = segmented_read(ctxt, ctxt->src.addr.mem,
4361 ctxt->src.valptr, ctxt->src.bytes);
4362 if (rc != X86EMUL_CONTINUE)
4364 ctxt->src.orig_val64 = ctxt->src.val64;
4367 if (ctxt->src2.type == OP_MEM) {
4368 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
4369 &ctxt->src2.val, ctxt->src2.bytes);
4370 if (rc != X86EMUL_CONTINUE)
4374 if ((ctxt->d & DstMask) == ImplicitOps)
4378 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
4379 /* optimisation - avoid slow emulated read if Mov */
4380 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
4381 &ctxt->dst.val, ctxt->dst.bytes);
4382 if (rc != X86EMUL_CONTINUE)
4385 ctxt->dst.orig_val = ctxt->dst.val;
4389 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4390 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4391 X86_ICPT_POST_MEMACCESS);
4392 if (rc != X86EMUL_CONTINUE)
4396 if (ctxt->execute) {
4397 rc = ctxt->execute(ctxt);
4398 if (rc != X86EMUL_CONTINUE)
4407 case 0x40 ... 0x47: /* inc r16/r32 */
4408 emulate_1op(ctxt, "inc");
4410 case 0x48 ... 0x4f: /* dec r16/r32 */
4411 emulate_1op(ctxt, "dec");
4413 case 0x63: /* movsxd */
4414 if (ctxt->mode != X86EMUL_MODE_PROT64)
4415 goto cannot_emulate;
4416 ctxt->dst.val = (s32) ctxt->src.val;
4418 case 0x70 ... 0x7f: /* jcc (short) */
4419 if (test_cc(ctxt->b, ctxt->eflags))
4420 jmp_rel(ctxt, ctxt->src.val);
4422 case 0x8d: /* lea r16/r32, m */
4423 ctxt->dst.val = ctxt->src.addr.mem.ea;
4425 case 0x90 ... 0x97: /* nop / xchg reg, rax */
4426 if (ctxt->dst.addr.reg == &ctxt->regs[VCPU_REGS_RAX])
4430 case 0x98: /* cbw/cwde/cdqe */
4431 switch (ctxt->op_bytes) {
4432 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
4433 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
4434 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
4440 case 0xcc: /* int3 */
4441 rc = emulate_int(ctxt, 3);
4443 case 0xcd: /* int n */
4444 rc = emulate_int(ctxt, ctxt->src.val);
4446 case 0xce: /* into */
4447 if (ctxt->eflags & EFLG_OF)
4448 rc = emulate_int(ctxt, 4);
4450 case 0xd0 ... 0xd1: /* Grp2 */
4453 case 0xd2 ... 0xd3: /* Grp2 */
4454 ctxt->src.val = ctxt->regs[VCPU_REGS_RCX];
4457 case 0xe9: /* jmp rel */
4458 case 0xeb: /* jmp rel short */
4459 jmp_rel(ctxt, ctxt->src.val);
4460 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4462 case 0xf4: /* hlt */
4463 ctxt->ops->halt(ctxt);
4465 case 0xf5: /* cmc */
4466 /* complement carry flag from eflags reg */
4467 ctxt->eflags ^= EFLG_CF;
4469 case 0xf8: /* clc */
4470 ctxt->eflags &= ~EFLG_CF;
4472 case 0xf9: /* stc */
4473 ctxt->eflags |= EFLG_CF;
4475 case 0xfc: /* cld */
4476 ctxt->eflags &= ~EFLG_DF;
4478 case 0xfd: /* std */
4479 ctxt->eflags |= EFLG_DF;
4482 goto cannot_emulate;
4485 if (rc != X86EMUL_CONTINUE)
4489 rc = writeback(ctxt);
4490 if (rc != X86EMUL_CONTINUE)
4494 * restore dst type in case the decoding will be reused
4495 * (happens for string instruction )
4497 ctxt->dst.type = saved_dst_type;
4499 if ((ctxt->d & SrcMask) == SrcSI)
4500 string_addr_inc(ctxt, seg_override(ctxt),
4501 VCPU_REGS_RSI, &ctxt->src);
4503 if ((ctxt->d & DstMask) == DstDI)
4504 string_addr_inc(ctxt, VCPU_SREG_ES, VCPU_REGS_RDI,
4507 if (ctxt->rep_prefix && (ctxt->d & String)) {
4508 struct read_cache *r = &ctxt->io_read;
4509 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1);
4511 if (!string_insn_completed(ctxt)) {
4513 * Re-enter guest when pio read ahead buffer is empty
4514 * or, if it is not used, after each 1024 iteration.
4516 if ((r->end != 0 || ctxt->regs[VCPU_REGS_RCX] & 0x3ff) &&
4517 (r->end == 0 || r->end != r->pos)) {
4519 * Reset read cache. Usually happens before
4520 * decode, but since instruction is restarted
4521 * we have to do it here.
4523 ctxt->mem_read.end = 0;
4524 return EMULATION_RESTART;
4526 goto done; /* skip rip writeback */
4530 ctxt->eip = ctxt->_eip;
4533 if (rc == X86EMUL_PROPAGATE_FAULT)
4534 ctxt->have_exception = true;
4535 if (rc == X86EMUL_INTERCEPTED)
4536 return EMULATION_INTERCEPTED;
4538 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4542 case 0x09: /* wbinvd */
4543 (ctxt->ops->wbinvd)(ctxt);
4545 case 0x08: /* invd */
4546 case 0x0d: /* GrpP (prefetch) */
4547 case 0x18: /* Grp16 (prefetch/nop) */
4549 case 0x20: /* mov cr, reg */
4550 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
4552 case 0x21: /* mov from dr to reg */
4553 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
4555 case 0x40 ... 0x4f: /* cmov */
4556 ctxt->dst.val = ctxt->dst.orig_val = ctxt->src.val;
4557 if (!test_cc(ctxt->b, ctxt->eflags))
4558 ctxt->dst.type = OP_NONE; /* no writeback */
4560 case 0x80 ... 0x8f: /* jnz rel, etc*/
4561 if (test_cc(ctxt->b, ctxt->eflags))
4562 jmp_rel(ctxt, ctxt->src.val);
4564 case 0x90 ... 0x9f: /* setcc r/m8 */
4565 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
4567 case 0xa4: /* shld imm8, r, r/m */
4568 case 0xa5: /* shld cl, r, r/m */
4569 emulate_2op_cl(ctxt, "shld");
4571 case 0xac: /* shrd imm8, r, r/m */
4572 case 0xad: /* shrd cl, r, r/m */
4573 emulate_2op_cl(ctxt, "shrd");
4575 case 0xae: /* clflush */
4577 case 0xb6 ... 0xb7: /* movzx */
4578 ctxt->dst.bytes = ctxt->op_bytes;
4579 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
4580 : (u16) ctxt->src.val;
4582 case 0xbe ... 0xbf: /* movsx */
4583 ctxt->dst.bytes = ctxt->op_bytes;
4584 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
4585 (s16) ctxt->src.val;
4587 case 0xc0 ... 0xc1: /* xadd */
4588 emulate_2op_SrcV(ctxt, "add");
4589 /* Write back the register source. */
4590 ctxt->src.val = ctxt->dst.orig_val;
4591 write_register_operand(&ctxt->src);
4593 case 0xc3: /* movnti */
4594 ctxt->dst.bytes = ctxt->op_bytes;
4595 ctxt->dst.val = (ctxt->op_bytes == 4) ? (u32) ctxt->src.val :
4596 (u64) ctxt->src.val;
4599 goto cannot_emulate;
4602 if (rc != X86EMUL_CONTINUE)
4608 return EMULATION_FAILED;