1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
27 #include <linux/stringify.h>
36 #define OpImplicit 1ull /* No generic decode */
37 #define OpReg 2ull /* Register */
38 #define OpMem 3ull /* Memory */
39 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
40 #define OpDI 5ull /* ES:DI/EDI/RDI */
41 #define OpMem64 6ull /* Memory, 64-bit */
42 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
43 #define OpDX 8ull /* DX register */
44 #define OpCL 9ull /* CL register (for shifts) */
45 #define OpImmByte 10ull /* 8-bit sign extended immediate */
46 #define OpOne 11ull /* Implied 1 */
47 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
48 #define OpMem16 13ull /* Memory operand (16-bit). */
49 #define OpMem32 14ull /* Memory operand (32-bit). */
50 #define OpImmU 15ull /* Immediate operand, zero extended */
51 #define OpSI 16ull /* SI/ESI/RSI */
52 #define OpImmFAddr 17ull /* Immediate far address */
53 #define OpMemFAddr 18ull /* Far address in memory */
54 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
55 #define OpES 20ull /* ES */
56 #define OpCS 21ull /* CS */
57 #define OpSS 22ull /* SS */
58 #define OpDS 23ull /* DS */
59 #define OpFS 24ull /* FS */
60 #define OpGS 25ull /* GS */
61 #define OpMem8 26ull /* 8-bit zero extended memory operand */
62 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
64 #define OpBits 5 /* Width of operand field */
65 #define OpMask ((1ull << OpBits) - 1)
68 * Opcode effective-address decode tables.
69 * Note that we only emulate instructions that have at least one memory
70 * operand (excluding implicit stack references). We assume that stack
71 * references and instruction fetches will never occur in special memory
72 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
76 /* Operand sizes: 8-bit operands or specified/overridden size. */
77 #define ByteOp (1<<0) /* 8-bit operands. */
78 /* Destination operand type. */
80 #define ImplicitOps (OpImplicit << DstShift)
81 #define DstReg (OpReg << DstShift)
82 #define DstMem (OpMem << DstShift)
83 #define DstAcc (OpAcc << DstShift)
84 #define DstDI (OpDI << DstShift)
85 #define DstMem64 (OpMem64 << DstShift)
86 #define DstImmUByte (OpImmUByte << DstShift)
87 #define DstDX (OpDX << DstShift)
88 #define DstMask (OpMask << DstShift)
89 /* Source operand type. */
91 #define SrcNone (OpNone << SrcShift)
92 #define SrcReg (OpReg << SrcShift)
93 #define SrcMem (OpMem << SrcShift)
94 #define SrcMem16 (OpMem16 << SrcShift)
95 #define SrcMem32 (OpMem32 << SrcShift)
96 #define SrcImm (OpImm << SrcShift)
97 #define SrcImmByte (OpImmByte << SrcShift)
98 #define SrcOne (OpOne << SrcShift)
99 #define SrcImmUByte (OpImmUByte << SrcShift)
100 #define SrcImmU (OpImmU << SrcShift)
101 #define SrcSI (OpSI << SrcShift)
102 #define SrcImmFAddr (OpImmFAddr << SrcShift)
103 #define SrcMemFAddr (OpMemFAddr << SrcShift)
104 #define SrcAcc (OpAcc << SrcShift)
105 #define SrcImmU16 (OpImmU16 << SrcShift)
106 #define SrcImm64 (OpImm64 << SrcShift)
107 #define SrcDX (OpDX << SrcShift)
108 #define SrcMem8 (OpMem8 << SrcShift)
109 #define SrcMask (OpMask << SrcShift)
110 #define BitOp (1<<11)
111 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
112 #define String (1<<13) /* String instruction (rep capable) */
113 #define Stack (1<<14) /* Stack instruction (push/pop) */
114 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
115 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
116 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
117 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
118 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
119 #define Escape (5<<15) /* Escape to coprocessor instruction */
120 #define Sse (1<<18) /* SSE Vector instruction */
121 /* Generic ModRM decode. */
122 #define ModRM (1<<19)
123 /* Destination is only written; never read. */
126 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
127 #define VendorSpecific (1<<22) /* Vendor specific instruction */
128 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
129 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
130 #define Undefined (1<<25) /* No Such Instruction */
131 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
132 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
134 #define PageTable (1 << 29) /* instruction used to write page table */
135 /* Source 2 operand type */
136 #define Src2Shift (30)
137 #define Src2None (OpNone << Src2Shift)
138 #define Src2CL (OpCL << Src2Shift)
139 #define Src2ImmByte (OpImmByte << Src2Shift)
140 #define Src2One (OpOne << Src2Shift)
141 #define Src2Imm (OpImm << Src2Shift)
142 #define Src2ES (OpES << Src2Shift)
143 #define Src2CS (OpCS << Src2Shift)
144 #define Src2SS (OpSS << Src2Shift)
145 #define Src2DS (OpDS << Src2Shift)
146 #define Src2FS (OpFS << Src2Shift)
147 #define Src2GS (OpGS << Src2Shift)
148 #define Src2Mask (OpMask << Src2Shift)
149 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
150 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
151 #define Unaligned ((u64)1 << 42) /* Explicitly unaligned (e.g. MOVDQU) */
152 #define Avx ((u64)1 << 43) /* Advanced Vector Extensions */
153 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
154 #define NoWrite ((u64)1 << 45) /* No writeback */
156 #define X2(x...) x, x
157 #define X3(x...) X2(x), x
158 #define X4(x...) X2(x), X2(x)
159 #define X5(x...) X4(x), x
160 #define X6(x...) X4(x), X2(x)
161 #define X7(x...) X4(x), X3(x)
162 #define X8(x...) X4(x), X4(x)
163 #define X16(x...) X8(x), X8(x)
165 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
166 #define FASTOP_SIZE 8
169 * fastop functions have a special calling convention:
171 * dst: [rdx]:rax (in/out)
174 * flags: rflags (in/out)
176 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
177 * different operand sizes can be reached by calculation, rather than a jump
178 * table (which would be bigger than the code).
180 * fastop functions are declared as taking a never-defined fastop parameter,
181 * so they can't be called from C directly.
190 int (*execute)(struct x86_emulate_ctxt *ctxt);
191 const struct opcode *group;
192 const struct group_dual *gdual;
193 const struct gprefix *gprefix;
194 const struct escape *esc;
195 void (*fastop)(struct fastop *fake);
197 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
201 struct opcode mod012[8];
202 struct opcode mod3[8];
206 struct opcode pfx_no;
207 struct opcode pfx_66;
208 struct opcode pfx_f2;
209 struct opcode pfx_f3;
214 struct opcode high[64];
217 /* EFLAGS bit definitions. */
218 #define EFLG_ID (1<<21)
219 #define EFLG_VIP (1<<20)
220 #define EFLG_VIF (1<<19)
221 #define EFLG_AC (1<<18)
222 #define EFLG_VM (1<<17)
223 #define EFLG_RF (1<<16)
224 #define EFLG_IOPL (3<<12)
225 #define EFLG_NT (1<<14)
226 #define EFLG_OF (1<<11)
227 #define EFLG_DF (1<<10)
228 #define EFLG_IF (1<<9)
229 #define EFLG_TF (1<<8)
230 #define EFLG_SF (1<<7)
231 #define EFLG_ZF (1<<6)
232 #define EFLG_AF (1<<4)
233 #define EFLG_PF (1<<2)
234 #define EFLG_CF (1<<0)
236 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
237 #define EFLG_RESERVED_ONE_MASK 2
239 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
241 if (!(ctxt->regs_valid & (1 << nr))) {
242 ctxt->regs_valid |= 1 << nr;
243 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
245 return ctxt->_regs[nr];
248 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
250 ctxt->regs_valid |= 1 << nr;
251 ctxt->regs_dirty |= 1 << nr;
252 return &ctxt->_regs[nr];
255 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
258 return reg_write(ctxt, nr);
261 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
265 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
266 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
269 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
271 ctxt->regs_dirty = 0;
272 ctxt->regs_valid = 0;
276 * Instruction emulation:
277 * Most instructions are emulated directly via a fragment of inline assembly
278 * code. This allows us to save/restore EFLAGS and thus very easily pick up
279 * any modified flags.
282 #if defined(CONFIG_X86_64)
283 #define _LO32 "k" /* force 32-bit operand */
284 #define _STK "%%rsp" /* stack pointer */
285 #elif defined(__i386__)
286 #define _LO32 "" /* force 32-bit operand */
287 #define _STK "%%esp" /* stack pointer */
291 * These EFLAGS bits are restored from saved value during emulation, and
292 * any changes are written back to the saved value after emulation.
294 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
296 /* Before executing instruction: restore necessary bits in EFLAGS. */
297 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
298 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
299 "movl %"_sav",%"_LO32 _tmp"; " \
302 "movl %"_msk",%"_LO32 _tmp"; " \
303 "andl %"_LO32 _tmp",("_STK"); " \
305 "notl %"_LO32 _tmp"; " \
306 "andl %"_LO32 _tmp",("_STK"); " \
307 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
309 "orl %"_LO32 _tmp",("_STK"); " \
313 /* After executing instruction: write-back necessary bits in EFLAGS. */
314 #define _POST_EFLAGS(_sav, _msk, _tmp) \
315 /* _sav |= EFLAGS & _msk; */ \
318 "andl %"_msk",%"_LO32 _tmp"; " \
319 "orl %"_LO32 _tmp",%"_sav"; "
327 #define ____emulate_2op(ctxt, _op, _x, _y, _suffix, _dsttype) \
329 __asm__ __volatile__ ( \
330 _PRE_EFLAGS("0", "4", "2") \
331 _op _suffix " %"_x"3,%1; " \
332 _POST_EFLAGS("0", "4", "2") \
333 : "=m" ((ctxt)->eflags), \
334 "+q" (*(_dsttype*)&(ctxt)->dst.val), \
336 : _y ((ctxt)->src.val), "i" (EFLAGS_MASK)); \
340 /* Raw emulation: instruction has two explicit operands. */
341 #define __emulate_2op_nobyte(ctxt,_op,_wx,_wy,_lx,_ly,_qx,_qy) \
343 unsigned long _tmp; \
345 switch ((ctxt)->dst.bytes) { \
347 ____emulate_2op(ctxt,_op,_wx,_wy,"w",u16); \
350 ____emulate_2op(ctxt,_op,_lx,_ly,"l",u32); \
353 ON64(____emulate_2op(ctxt,_op,_qx,_qy,"q",u64)); \
358 #define __emulate_2op(ctxt,_op,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
360 unsigned long _tmp; \
361 switch ((ctxt)->dst.bytes) { \
363 ____emulate_2op(ctxt,_op,_bx,_by,"b",u8); \
366 __emulate_2op_nobyte(ctxt, _op, \
367 _wx, _wy, _lx, _ly, _qx, _qy); \
372 /* Source operand is byte-sized and may be restricted to just %cl. */
373 #define emulate_2op_SrcB(ctxt, _op) \
374 __emulate_2op(ctxt, _op, "b", "c", "b", "c", "b", "c", "b", "c")
376 /* Source operand is byte, word, long or quad sized. */
377 #define emulate_2op_SrcV(ctxt, _op) \
378 __emulate_2op(ctxt, _op, "b", "q", "w", "r", _LO32, "r", "", "r")
380 /* Source operand is word, long or quad sized. */
381 #define emulate_2op_SrcV_nobyte(ctxt, _op) \
382 __emulate_2op_nobyte(ctxt, _op, "w", "r", _LO32, "r", "", "r")
384 /* Instruction has three operands and one operand is stored in ECX register */
385 #define __emulate_2op_cl(ctxt, _op, _suffix, _type) \
387 unsigned long _tmp; \
388 _type _clv = (ctxt)->src2.val; \
389 _type _srcv = (ctxt)->src.val; \
390 _type _dstv = (ctxt)->dst.val; \
392 __asm__ __volatile__ ( \
393 _PRE_EFLAGS("0", "5", "2") \
394 _op _suffix " %4,%1 \n" \
395 _POST_EFLAGS("0", "5", "2") \
396 : "=m" ((ctxt)->eflags), "+r" (_dstv), "=&r" (_tmp) \
397 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
400 (ctxt)->src2.val = (unsigned long) _clv; \
401 (ctxt)->src2.val = (unsigned long) _srcv; \
402 (ctxt)->dst.val = (unsigned long) _dstv; \
405 #define emulate_2op_cl(ctxt, _op) \
407 switch ((ctxt)->dst.bytes) { \
409 __emulate_2op_cl(ctxt, _op, "w", u16); \
412 __emulate_2op_cl(ctxt, _op, "l", u32); \
415 ON64(__emulate_2op_cl(ctxt, _op, "q", ulong)); \
420 #define __emulate_1op(ctxt, _op, _suffix) \
422 unsigned long _tmp; \
424 __asm__ __volatile__ ( \
425 _PRE_EFLAGS("0", "3", "2") \
426 _op _suffix " %1; " \
427 _POST_EFLAGS("0", "3", "2") \
428 : "=m" ((ctxt)->eflags), "+m" ((ctxt)->dst.val), \
430 : "i" (EFLAGS_MASK)); \
433 /* Instruction has only one explicit operand (no source operand). */
434 #define emulate_1op(ctxt, _op) \
436 switch ((ctxt)->dst.bytes) { \
437 case 1: __emulate_1op(ctxt, _op, "b"); break; \
438 case 2: __emulate_1op(ctxt, _op, "w"); break; \
439 case 4: __emulate_1op(ctxt, _op, "l"); break; \
440 case 8: ON64(__emulate_1op(ctxt, _op, "q")); break; \
444 #define FOP_ALIGN ".align " __stringify(FASTOP_SIZE) " \n\t"
445 #define FOP_RET "ret \n\t"
447 #define FOP_START(op) \
448 extern void em_##op(struct fastop *fake); \
449 asm(".pushsection .text, \"ax\" \n\t" \
450 ".global em_" #op " \n\t" \
457 #define FOP1E(op, dst) \
458 FOP_ALIGN #op " %" #dst " \n\t" FOP_RET
460 #define FASTOP1(op) \
465 ON64(FOP1E(op##q, rax)) \
468 #define __emulate_1op_rax_rdx(ctxt, _op, _suffix, _ex) \
470 unsigned long _tmp; \
471 ulong *rax = reg_rmw((ctxt), VCPU_REGS_RAX); \
472 ulong *rdx = reg_rmw((ctxt), VCPU_REGS_RDX); \
474 __asm__ __volatile__ ( \
475 _PRE_EFLAGS("0", "5", "1") \
477 _op _suffix " %6; " \
479 _POST_EFLAGS("0", "5", "1") \
480 ".pushsection .fixup,\"ax\" \n\t" \
481 "3: movb $1, %4 \n\t" \
484 _ASM_EXTABLE(1b, 3b) \
485 : "=m" ((ctxt)->eflags), "=&r" (_tmp), \
486 "+a" (*rax), "+d" (*rdx), "+qm"(_ex) \
487 : "i" (EFLAGS_MASK), "m" ((ctxt)->src.val)); \
490 /* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
491 #define emulate_1op_rax_rdx(ctxt, _op, _ex) \
493 switch((ctxt)->src.bytes) { \
495 __emulate_1op_rax_rdx(ctxt, _op, "b", _ex); \
498 __emulate_1op_rax_rdx(ctxt, _op, "w", _ex); \
501 __emulate_1op_rax_rdx(ctxt, _op, "l", _ex); \
504 __emulate_1op_rax_rdx(ctxt, _op, "q", _ex)); \
509 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
510 enum x86_intercept intercept,
511 enum x86_intercept_stage stage)
513 struct x86_instruction_info info = {
514 .intercept = intercept,
515 .rep_prefix = ctxt->rep_prefix,
516 .modrm_mod = ctxt->modrm_mod,
517 .modrm_reg = ctxt->modrm_reg,
518 .modrm_rm = ctxt->modrm_rm,
519 .src_val = ctxt->src.val64,
520 .src_bytes = ctxt->src.bytes,
521 .dst_bytes = ctxt->dst.bytes,
522 .ad_bytes = ctxt->ad_bytes,
523 .next_rip = ctxt->eip,
526 return ctxt->ops->intercept(ctxt, &info, stage);
529 static void assign_masked(ulong *dest, ulong src, ulong mask)
531 *dest = (*dest & ~mask) | (src & mask);
534 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
536 return (1UL << (ctxt->ad_bytes << 3)) - 1;
539 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
542 struct desc_struct ss;
544 if (ctxt->mode == X86EMUL_MODE_PROT64)
546 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
547 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
550 static int stack_size(struct x86_emulate_ctxt *ctxt)
552 return (__fls(stack_mask(ctxt)) + 1) >> 3;
555 /* Access/update address held in a register, based on addressing mode. */
556 static inline unsigned long
557 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
559 if (ctxt->ad_bytes == sizeof(unsigned long))
562 return reg & ad_mask(ctxt);
565 static inline unsigned long
566 register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
568 return address_mask(ctxt, reg);
571 static void masked_increment(ulong *reg, ulong mask, int inc)
573 assign_masked(reg, *reg + inc, mask);
577 register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
581 if (ctxt->ad_bytes == sizeof(unsigned long))
584 mask = ad_mask(ctxt);
585 masked_increment(reg, mask, inc);
588 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
590 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
593 static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
595 register_address_increment(ctxt, &ctxt->_eip, rel);
598 static u32 desc_limit_scaled(struct desc_struct *desc)
600 u32 limit = get_desc_limit(desc);
602 return desc->g ? (limit << 12) | 0xfff : limit;
605 static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
607 ctxt->has_seg_override = true;
608 ctxt->seg_override = seg;
611 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
613 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
616 return ctxt->ops->get_cached_segment_base(ctxt, seg);
619 static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
621 if (!ctxt->has_seg_override)
624 return ctxt->seg_override;
627 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
628 u32 error, bool valid)
630 ctxt->exception.vector = vec;
631 ctxt->exception.error_code = error;
632 ctxt->exception.error_code_valid = valid;
633 return X86EMUL_PROPAGATE_FAULT;
636 static int emulate_db(struct x86_emulate_ctxt *ctxt)
638 return emulate_exception(ctxt, DB_VECTOR, 0, false);
641 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
643 return emulate_exception(ctxt, GP_VECTOR, err, true);
646 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
648 return emulate_exception(ctxt, SS_VECTOR, err, true);
651 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
653 return emulate_exception(ctxt, UD_VECTOR, 0, false);
656 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
658 return emulate_exception(ctxt, TS_VECTOR, err, true);
661 static int emulate_de(struct x86_emulate_ctxt *ctxt)
663 return emulate_exception(ctxt, DE_VECTOR, 0, false);
666 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
668 return emulate_exception(ctxt, NM_VECTOR, 0, false);
671 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
674 struct desc_struct desc;
676 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
680 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
685 struct desc_struct desc;
687 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
688 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
692 * x86 defines three classes of vector instructions: explicitly
693 * aligned, explicitly unaligned, and the rest, which change behaviour
694 * depending on whether they're AVX encoded or not.
696 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
697 * subject to the same check.
699 static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
701 if (likely(size < 16))
704 if (ctxt->d & Aligned)
706 else if (ctxt->d & Unaligned)
708 else if (ctxt->d & Avx)
714 static int __linearize(struct x86_emulate_ctxt *ctxt,
715 struct segmented_address addr,
716 unsigned size, bool write, bool fetch,
719 struct desc_struct desc;
726 la = seg_base(ctxt, addr.seg) + addr.ea;
727 switch (ctxt->mode) {
728 case X86EMUL_MODE_PROT64:
729 if (((signed long)la << 16) >> 16 != la)
730 return emulate_gp(ctxt, 0);
733 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
737 /* code segment in protected mode or read-only data segment */
738 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
739 || !(desc.type & 2)) && write)
741 /* unreadable code segment */
742 if (!fetch && (desc.type & 8) && !(desc.type & 2))
744 lim = desc_limit_scaled(&desc);
745 if ((desc.type & 8) || !(desc.type & 4)) {
746 /* expand-up segment */
747 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
750 /* expand-down segment */
751 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
753 lim = desc.d ? 0xffffffff : 0xffff;
754 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
757 cpl = ctxt->ops->cpl(ctxt);
758 if (!(desc.type & 8)) {
762 } else if ((desc.type & 8) && !(desc.type & 4)) {
763 /* nonconforming code segment */
766 } else if ((desc.type & 8) && (desc.type & 4)) {
767 /* conforming code segment */
773 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
775 if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
776 return emulate_gp(ctxt, 0);
778 return X86EMUL_CONTINUE;
780 if (addr.seg == VCPU_SREG_SS)
781 return emulate_ss(ctxt, sel);
783 return emulate_gp(ctxt, sel);
786 static int linearize(struct x86_emulate_ctxt *ctxt,
787 struct segmented_address addr,
788 unsigned size, bool write,
791 return __linearize(ctxt, addr, size, write, false, linear);
795 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
796 struct segmented_address addr,
803 rc = linearize(ctxt, addr, size, false, &linear);
804 if (rc != X86EMUL_CONTINUE)
806 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
810 * Fetch the next byte of the instruction being emulated which is pointed to
811 * by ctxt->_eip, then increment ctxt->_eip.
813 * Also prefetch the remaining bytes of the instruction without crossing page
814 * boundary if they are not in fetch_cache yet.
816 static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
818 struct fetch_cache *fc = &ctxt->fetch;
822 if (ctxt->_eip == fc->end) {
823 unsigned long linear;
824 struct segmented_address addr = { .seg = VCPU_SREG_CS,
826 cur_size = fc->end - fc->start;
827 size = min(15UL - cur_size,
828 PAGE_SIZE - offset_in_page(ctxt->_eip));
829 rc = __linearize(ctxt, addr, size, false, true, &linear);
830 if (unlikely(rc != X86EMUL_CONTINUE))
832 rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
833 size, &ctxt->exception);
834 if (unlikely(rc != X86EMUL_CONTINUE))
838 *dest = fc->data[ctxt->_eip - fc->start];
840 return X86EMUL_CONTINUE;
843 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
844 void *dest, unsigned size)
848 /* x86 instructions are limited to 15 bytes. */
849 if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
850 return X86EMUL_UNHANDLEABLE;
852 rc = do_insn_fetch_byte(ctxt, dest++);
853 if (rc != X86EMUL_CONTINUE)
856 return X86EMUL_CONTINUE;
859 /* Fetch next part of the instruction being emulated. */
860 #define insn_fetch(_type, _ctxt) \
861 ({ unsigned long _x; \
862 rc = do_insn_fetch(_ctxt, &_x, sizeof(_type)); \
863 if (rc != X86EMUL_CONTINUE) \
868 #define insn_fetch_arr(_arr, _size, _ctxt) \
869 ({ rc = do_insn_fetch(_ctxt, _arr, (_size)); \
870 if (rc != X86EMUL_CONTINUE) \
875 * Given the 'reg' portion of a ModRM byte, and a register block, return a
876 * pointer into the block that addresses the relevant register.
877 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
879 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
884 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
885 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
887 p = reg_rmw(ctxt, modrm_reg);
891 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
892 struct segmented_address addr,
893 u16 *size, unsigned long *address, int op_bytes)
900 rc = segmented_read_std(ctxt, addr, size, 2);
901 if (rc != X86EMUL_CONTINUE)
904 rc = segmented_read_std(ctxt, addr, address, op_bytes);
908 static int test_cc(unsigned int condition, unsigned int flags)
912 switch ((condition & 15) >> 1) {
914 rc |= (flags & EFLG_OF);
916 case 1: /* b/c/nae */
917 rc |= (flags & EFLG_CF);
920 rc |= (flags & EFLG_ZF);
923 rc |= (flags & (EFLG_CF|EFLG_ZF));
926 rc |= (flags & EFLG_SF);
929 rc |= (flags & EFLG_PF);
932 rc |= (flags & EFLG_ZF);
935 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
939 /* Odd condition identifiers (lsb == 1) have inverted sense. */
940 return (!!rc ^ (condition & 1));
943 static void fetch_register_operand(struct operand *op)
947 op->val = *(u8 *)op->addr.reg;
950 op->val = *(u16 *)op->addr.reg;
953 op->val = *(u32 *)op->addr.reg;
956 op->val = *(u64 *)op->addr.reg;
961 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
963 ctxt->ops->get_fpu(ctxt);
965 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
966 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
967 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
968 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
969 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
970 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
971 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
972 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
974 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
975 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
976 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
977 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
978 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
979 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
980 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
981 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
985 ctxt->ops->put_fpu(ctxt);
988 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
991 ctxt->ops->get_fpu(ctxt);
993 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
994 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
995 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
996 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
997 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
998 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
999 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
1000 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
1001 #ifdef CONFIG_X86_64
1002 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
1003 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
1004 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
1005 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
1006 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
1007 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
1008 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
1009 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
1013 ctxt->ops->put_fpu(ctxt);
1016 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1018 ctxt->ops->get_fpu(ctxt);
1020 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
1021 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
1022 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
1023 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
1024 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
1025 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
1026 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
1027 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
1030 ctxt->ops->put_fpu(ctxt);
1033 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1035 ctxt->ops->get_fpu(ctxt);
1037 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
1038 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
1039 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
1040 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
1041 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
1042 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
1043 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
1044 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
1047 ctxt->ops->put_fpu(ctxt);
1050 static int em_fninit(struct x86_emulate_ctxt *ctxt)
1052 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1053 return emulate_nm(ctxt);
1055 ctxt->ops->get_fpu(ctxt);
1056 asm volatile("fninit");
1057 ctxt->ops->put_fpu(ctxt);
1058 return X86EMUL_CONTINUE;
1061 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
1065 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1066 return emulate_nm(ctxt);
1068 ctxt->ops->get_fpu(ctxt);
1069 asm volatile("fnstcw %0": "+m"(fcw));
1070 ctxt->ops->put_fpu(ctxt);
1072 /* force 2 byte destination */
1073 ctxt->dst.bytes = 2;
1074 ctxt->dst.val = fcw;
1076 return X86EMUL_CONTINUE;
1079 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1083 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1084 return emulate_nm(ctxt);
1086 ctxt->ops->get_fpu(ctxt);
1087 asm volatile("fnstsw %0": "+m"(fsw));
1088 ctxt->ops->put_fpu(ctxt);
1090 /* force 2 byte destination */
1091 ctxt->dst.bytes = 2;
1092 ctxt->dst.val = fsw;
1094 return X86EMUL_CONTINUE;
1097 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1100 unsigned reg = ctxt->modrm_reg;
1101 int highbyte_regs = ctxt->rex_prefix == 0;
1103 if (!(ctxt->d & ModRM))
1104 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1106 if (ctxt->d & Sse) {
1110 read_sse_reg(ctxt, &op->vec_val, reg);
1113 if (ctxt->d & Mmx) {
1122 if (ctxt->d & ByteOp) {
1123 op->addr.reg = decode_register(ctxt, reg, highbyte_regs);
1126 op->addr.reg = decode_register(ctxt, reg, 0);
1127 op->bytes = ctxt->op_bytes;
1129 fetch_register_operand(op);
1130 op->orig_val = op->val;
1133 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1135 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1136 ctxt->modrm_seg = VCPU_SREG_SS;
1139 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1143 int index_reg = 0, base_reg = 0, scale;
1144 int rc = X86EMUL_CONTINUE;
1147 if (ctxt->rex_prefix) {
1148 ctxt->modrm_reg = (ctxt->rex_prefix & 4) << 1; /* REX.R */
1149 index_reg = (ctxt->rex_prefix & 2) << 2; /* REX.X */
1150 ctxt->modrm_rm = base_reg = (ctxt->rex_prefix & 1) << 3; /* REG.B */
1153 ctxt->modrm_mod |= (ctxt->modrm & 0xc0) >> 6;
1154 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1155 ctxt->modrm_rm |= (ctxt->modrm & 0x07);
1156 ctxt->modrm_seg = VCPU_SREG_DS;
1158 if (ctxt->modrm_mod == 3) {
1160 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1161 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm, ctxt->d & ByteOp);
1162 if (ctxt->d & Sse) {
1165 op->addr.xmm = ctxt->modrm_rm;
1166 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1169 if (ctxt->d & Mmx) {
1172 op->addr.xmm = ctxt->modrm_rm & 7;
1175 fetch_register_operand(op);
1181 if (ctxt->ad_bytes == 2) {
1182 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1183 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1184 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1185 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1187 /* 16-bit ModR/M decode. */
1188 switch (ctxt->modrm_mod) {
1190 if (ctxt->modrm_rm == 6)
1191 modrm_ea += insn_fetch(u16, ctxt);
1194 modrm_ea += insn_fetch(s8, ctxt);
1197 modrm_ea += insn_fetch(u16, ctxt);
1200 switch (ctxt->modrm_rm) {
1202 modrm_ea += bx + si;
1205 modrm_ea += bx + di;
1208 modrm_ea += bp + si;
1211 modrm_ea += bp + di;
1220 if (ctxt->modrm_mod != 0)
1227 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1228 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1229 ctxt->modrm_seg = VCPU_SREG_SS;
1230 modrm_ea = (u16)modrm_ea;
1232 /* 32/64-bit ModR/M decode. */
1233 if ((ctxt->modrm_rm & 7) == 4) {
1234 sib = insn_fetch(u8, ctxt);
1235 index_reg |= (sib >> 3) & 7;
1236 base_reg |= sib & 7;
1239 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1240 modrm_ea += insn_fetch(s32, ctxt);
1242 modrm_ea += reg_read(ctxt, base_reg);
1243 adjust_modrm_seg(ctxt, base_reg);
1246 modrm_ea += reg_read(ctxt, index_reg) << scale;
1247 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1248 if (ctxt->mode == X86EMUL_MODE_PROT64)
1249 ctxt->rip_relative = 1;
1251 base_reg = ctxt->modrm_rm;
1252 modrm_ea += reg_read(ctxt, base_reg);
1253 adjust_modrm_seg(ctxt, base_reg);
1255 switch (ctxt->modrm_mod) {
1257 if (ctxt->modrm_rm == 5)
1258 modrm_ea += insn_fetch(s32, ctxt);
1261 modrm_ea += insn_fetch(s8, ctxt);
1264 modrm_ea += insn_fetch(s32, ctxt);
1268 op->addr.mem.ea = modrm_ea;
1273 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1276 int rc = X86EMUL_CONTINUE;
1279 switch (ctxt->ad_bytes) {
1281 op->addr.mem.ea = insn_fetch(u16, ctxt);
1284 op->addr.mem.ea = insn_fetch(u32, ctxt);
1287 op->addr.mem.ea = insn_fetch(u64, ctxt);
1294 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1298 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1299 mask = ~(ctxt->dst.bytes * 8 - 1);
1301 if (ctxt->src.bytes == 2)
1302 sv = (s16)ctxt->src.val & (s16)mask;
1303 else if (ctxt->src.bytes == 4)
1304 sv = (s32)ctxt->src.val & (s32)mask;
1306 ctxt->dst.addr.mem.ea += (sv >> 3);
1309 /* only subword offset */
1310 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1313 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1314 unsigned long addr, void *dest, unsigned size)
1317 struct read_cache *mc = &ctxt->mem_read;
1319 if (mc->pos < mc->end)
1322 WARN_ON((mc->end + size) >= sizeof(mc->data));
1324 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1326 if (rc != X86EMUL_CONTINUE)
1332 memcpy(dest, mc->data + mc->pos, size);
1334 return X86EMUL_CONTINUE;
1337 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1338 struct segmented_address addr,
1345 rc = linearize(ctxt, addr, size, false, &linear);
1346 if (rc != X86EMUL_CONTINUE)
1348 return read_emulated(ctxt, linear, data, size);
1351 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1352 struct segmented_address addr,
1359 rc = linearize(ctxt, addr, size, true, &linear);
1360 if (rc != X86EMUL_CONTINUE)
1362 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1366 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1367 struct segmented_address addr,
1368 const void *orig_data, const void *data,
1374 rc = linearize(ctxt, addr, size, true, &linear);
1375 if (rc != X86EMUL_CONTINUE)
1377 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1378 size, &ctxt->exception);
1381 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1382 unsigned int size, unsigned short port,
1385 struct read_cache *rc = &ctxt->io_read;
1387 if (rc->pos == rc->end) { /* refill pio read ahead */
1388 unsigned int in_page, n;
1389 unsigned int count = ctxt->rep_prefix ?
1390 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1391 in_page = (ctxt->eflags & EFLG_DF) ?
1392 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1393 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1394 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1398 rc->pos = rc->end = 0;
1399 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1404 if (ctxt->rep_prefix && !(ctxt->eflags & EFLG_DF)) {
1405 ctxt->dst.data = rc->data + rc->pos;
1406 ctxt->dst.type = OP_MEM_STR;
1407 ctxt->dst.count = (rc->end - rc->pos) / size;
1410 memcpy(dest, rc->data + rc->pos, size);
1416 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1417 u16 index, struct desc_struct *desc)
1422 ctxt->ops->get_idt(ctxt, &dt);
1424 if (dt.size < index * 8 + 7)
1425 return emulate_gp(ctxt, index << 3 | 0x2);
1427 addr = dt.address + index * 8;
1428 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1432 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1433 u16 selector, struct desc_ptr *dt)
1435 const struct x86_emulate_ops *ops = ctxt->ops;
1437 if (selector & 1 << 2) {
1438 struct desc_struct desc;
1441 memset (dt, 0, sizeof *dt);
1442 if (!ops->get_segment(ctxt, &sel, &desc, NULL, VCPU_SREG_LDTR))
1445 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1446 dt->address = get_desc_base(&desc);
1448 ops->get_gdt(ctxt, dt);
1451 /* allowed just for 8 bytes segments */
1452 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1453 u16 selector, struct desc_struct *desc,
1457 u16 index = selector >> 3;
1460 get_descriptor_table_ptr(ctxt, selector, &dt);
1462 if (dt.size < index * 8 + 7)
1463 return emulate_gp(ctxt, selector & 0xfffc);
1465 *desc_addr_p = addr = dt.address + index * 8;
1466 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1470 /* allowed just for 8 bytes segments */
1471 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1472 u16 selector, struct desc_struct *desc)
1475 u16 index = selector >> 3;
1478 get_descriptor_table_ptr(ctxt, selector, &dt);
1480 if (dt.size < index * 8 + 7)
1481 return emulate_gp(ctxt, selector & 0xfffc);
1483 addr = dt.address + index * 8;
1484 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1488 /* Does not support long mode */
1489 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1490 u16 selector, int seg)
1492 struct desc_struct seg_desc, old_desc;
1494 unsigned err_vec = GP_VECTOR;
1496 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1501 memset(&seg_desc, 0, sizeof seg_desc);
1503 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1504 || ctxt->mode == X86EMUL_MODE_REAL) {
1505 /* set real mode segment descriptor */
1506 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1507 set_desc_base(&seg_desc, selector << 4);
1512 cpl = ctxt->ops->cpl(ctxt);
1514 /* NULL selector is not valid for TR, CS and SS (except for long mode) */
1515 if ((seg == VCPU_SREG_CS
1516 || (seg == VCPU_SREG_SS
1517 && (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl))
1518 || seg == VCPU_SREG_TR)
1522 /* TR should be in GDT only */
1523 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1526 if (null_selector) /* for NULL selector skip all following checks */
1529 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1530 if (ret != X86EMUL_CONTINUE)
1533 err_code = selector & 0xfffc;
1534 err_vec = GP_VECTOR;
1536 /* can't load system descriptor into segment selector */
1537 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1541 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1550 * segment is not a writable data segment or segment
1551 * selector's RPL != CPL or segment selector's RPL != CPL
1553 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1557 if (!(seg_desc.type & 8))
1560 if (seg_desc.type & 4) {
1566 if (rpl > cpl || dpl != cpl)
1569 /* CS(RPL) <- CPL */
1570 selector = (selector & 0xfffc) | cpl;
1573 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1575 old_desc = seg_desc;
1576 seg_desc.type |= 2; /* busy */
1577 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1578 sizeof(seg_desc), &ctxt->exception);
1579 if (ret != X86EMUL_CONTINUE)
1582 case VCPU_SREG_LDTR:
1583 if (seg_desc.s || seg_desc.type != 2)
1586 default: /* DS, ES, FS, or GS */
1588 * segment is not a data or readable code segment or
1589 * ((segment is a data or nonconforming code segment)
1590 * and (both RPL and CPL > DPL))
1592 if ((seg_desc.type & 0xa) == 0x8 ||
1593 (((seg_desc.type & 0xc) != 0xc) &&
1594 (rpl > dpl && cpl > dpl)))
1600 /* mark segment as accessed */
1602 ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1603 if (ret != X86EMUL_CONTINUE)
1607 ctxt->ops->set_segment(ctxt, selector, &seg_desc, 0, seg);
1608 return X86EMUL_CONTINUE;
1610 emulate_exception(ctxt, err_vec, err_code, true);
1611 return X86EMUL_PROPAGATE_FAULT;
1614 static void write_register_operand(struct operand *op)
1616 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1617 switch (op->bytes) {
1619 *(u8 *)op->addr.reg = (u8)op->val;
1622 *(u16 *)op->addr.reg = (u16)op->val;
1625 *op->addr.reg = (u32)op->val;
1626 break; /* 64b: zero-extend */
1628 *op->addr.reg = op->val;
1633 static int writeback(struct x86_emulate_ctxt *ctxt)
1637 if (ctxt->d & NoWrite)
1638 return X86EMUL_CONTINUE;
1640 switch (ctxt->dst.type) {
1642 write_register_operand(&ctxt->dst);
1645 if (ctxt->lock_prefix)
1646 rc = segmented_cmpxchg(ctxt,
1648 &ctxt->dst.orig_val,
1652 rc = segmented_write(ctxt,
1656 if (rc != X86EMUL_CONTINUE)
1660 rc = segmented_write(ctxt,
1663 ctxt->dst.bytes * ctxt->dst.count);
1664 if (rc != X86EMUL_CONTINUE)
1668 write_sse_reg(ctxt, &ctxt->dst.vec_val, ctxt->dst.addr.xmm);
1671 write_mmx_reg(ctxt, &ctxt->dst.mm_val, ctxt->dst.addr.mm);
1679 return X86EMUL_CONTINUE;
1682 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1684 struct segmented_address addr;
1686 rsp_increment(ctxt, -bytes);
1687 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1688 addr.seg = VCPU_SREG_SS;
1690 return segmented_write(ctxt, addr, data, bytes);
1693 static int em_push(struct x86_emulate_ctxt *ctxt)
1695 /* Disable writeback. */
1696 ctxt->dst.type = OP_NONE;
1697 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1700 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1701 void *dest, int len)
1704 struct segmented_address addr;
1706 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1707 addr.seg = VCPU_SREG_SS;
1708 rc = segmented_read(ctxt, addr, dest, len);
1709 if (rc != X86EMUL_CONTINUE)
1712 rsp_increment(ctxt, len);
1716 static int em_pop(struct x86_emulate_ctxt *ctxt)
1718 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1721 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1722 void *dest, int len)
1725 unsigned long val, change_mask;
1726 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1727 int cpl = ctxt->ops->cpl(ctxt);
1729 rc = emulate_pop(ctxt, &val, len);
1730 if (rc != X86EMUL_CONTINUE)
1733 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1734 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1736 switch(ctxt->mode) {
1737 case X86EMUL_MODE_PROT64:
1738 case X86EMUL_MODE_PROT32:
1739 case X86EMUL_MODE_PROT16:
1741 change_mask |= EFLG_IOPL;
1743 change_mask |= EFLG_IF;
1745 case X86EMUL_MODE_VM86:
1747 return emulate_gp(ctxt, 0);
1748 change_mask |= EFLG_IF;
1750 default: /* real mode */
1751 change_mask |= (EFLG_IOPL | EFLG_IF);
1755 *(unsigned long *)dest =
1756 (ctxt->eflags & ~change_mask) | (val & change_mask);
1761 static int em_popf(struct x86_emulate_ctxt *ctxt)
1763 ctxt->dst.type = OP_REG;
1764 ctxt->dst.addr.reg = &ctxt->eflags;
1765 ctxt->dst.bytes = ctxt->op_bytes;
1766 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1769 static int em_enter(struct x86_emulate_ctxt *ctxt)
1772 unsigned frame_size = ctxt->src.val;
1773 unsigned nesting_level = ctxt->src2.val & 31;
1777 return X86EMUL_UNHANDLEABLE;
1779 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1780 rc = push(ctxt, &rbp, stack_size(ctxt));
1781 if (rc != X86EMUL_CONTINUE)
1783 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1785 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1786 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1788 return X86EMUL_CONTINUE;
1791 static int em_leave(struct x86_emulate_ctxt *ctxt)
1793 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1795 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1798 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1800 int seg = ctxt->src2.val;
1802 ctxt->src.val = get_segment_selector(ctxt, seg);
1804 return em_push(ctxt);
1807 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1809 int seg = ctxt->src2.val;
1810 unsigned long selector;
1813 rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1814 if (rc != X86EMUL_CONTINUE)
1817 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1821 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1823 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1824 int rc = X86EMUL_CONTINUE;
1825 int reg = VCPU_REGS_RAX;
1827 while (reg <= VCPU_REGS_RDI) {
1828 (reg == VCPU_REGS_RSP) ?
1829 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
1832 if (rc != X86EMUL_CONTINUE)
1841 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1843 ctxt->src.val = (unsigned long)ctxt->eflags;
1844 return em_push(ctxt);
1847 static int em_popa(struct x86_emulate_ctxt *ctxt)
1849 int rc = X86EMUL_CONTINUE;
1850 int reg = VCPU_REGS_RDI;
1852 while (reg >= VCPU_REGS_RAX) {
1853 if (reg == VCPU_REGS_RSP) {
1854 rsp_increment(ctxt, ctxt->op_bytes);
1858 rc = emulate_pop(ctxt, reg_rmw(ctxt, reg), ctxt->op_bytes);
1859 if (rc != X86EMUL_CONTINUE)
1866 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1868 const struct x86_emulate_ops *ops = ctxt->ops;
1875 /* TODO: Add limit checks */
1876 ctxt->src.val = ctxt->eflags;
1878 if (rc != X86EMUL_CONTINUE)
1881 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1883 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1885 if (rc != X86EMUL_CONTINUE)
1888 ctxt->src.val = ctxt->_eip;
1890 if (rc != X86EMUL_CONTINUE)
1893 ops->get_idt(ctxt, &dt);
1895 eip_addr = dt.address + (irq << 2);
1896 cs_addr = dt.address + (irq << 2) + 2;
1898 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1899 if (rc != X86EMUL_CONTINUE)
1902 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1903 if (rc != X86EMUL_CONTINUE)
1906 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1907 if (rc != X86EMUL_CONTINUE)
1915 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1919 invalidate_registers(ctxt);
1920 rc = __emulate_int_real(ctxt, irq);
1921 if (rc == X86EMUL_CONTINUE)
1922 writeback_registers(ctxt);
1926 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1928 switch(ctxt->mode) {
1929 case X86EMUL_MODE_REAL:
1930 return __emulate_int_real(ctxt, irq);
1931 case X86EMUL_MODE_VM86:
1932 case X86EMUL_MODE_PROT16:
1933 case X86EMUL_MODE_PROT32:
1934 case X86EMUL_MODE_PROT64:
1936 /* Protected mode interrupts unimplemented yet */
1937 return X86EMUL_UNHANDLEABLE;
1941 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
1943 int rc = X86EMUL_CONTINUE;
1944 unsigned long temp_eip = 0;
1945 unsigned long temp_eflags = 0;
1946 unsigned long cs = 0;
1947 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1948 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1949 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1950 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1952 /* TODO: Add stack limit check */
1954 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
1956 if (rc != X86EMUL_CONTINUE)
1959 if (temp_eip & ~0xffff)
1960 return emulate_gp(ctxt, 0);
1962 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1964 if (rc != X86EMUL_CONTINUE)
1967 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
1969 if (rc != X86EMUL_CONTINUE)
1972 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1974 if (rc != X86EMUL_CONTINUE)
1977 ctxt->_eip = temp_eip;
1980 if (ctxt->op_bytes == 4)
1981 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1982 else if (ctxt->op_bytes == 2) {
1983 ctxt->eflags &= ~0xffff;
1984 ctxt->eflags |= temp_eflags;
1987 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1988 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1993 static int em_iret(struct x86_emulate_ctxt *ctxt)
1995 switch(ctxt->mode) {
1996 case X86EMUL_MODE_REAL:
1997 return emulate_iret_real(ctxt);
1998 case X86EMUL_MODE_VM86:
1999 case X86EMUL_MODE_PROT16:
2000 case X86EMUL_MODE_PROT32:
2001 case X86EMUL_MODE_PROT64:
2003 /* iret from protected mode unimplemented yet */
2004 return X86EMUL_UNHANDLEABLE;
2008 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
2013 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2015 rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
2016 if (rc != X86EMUL_CONTINUE)
2020 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2021 return X86EMUL_CONTINUE;
2024 static int em_grp2(struct x86_emulate_ctxt *ctxt)
2026 switch (ctxt->modrm_reg) {
2028 emulate_2op_SrcB(ctxt, "rol");
2031 emulate_2op_SrcB(ctxt, "ror");
2034 emulate_2op_SrcB(ctxt, "rcl");
2037 emulate_2op_SrcB(ctxt, "rcr");
2039 case 4: /* sal/shl */
2040 case 6: /* sal/shl */
2041 emulate_2op_SrcB(ctxt, "sal");
2044 emulate_2op_SrcB(ctxt, "shr");
2047 emulate_2op_SrcB(ctxt, "sar");
2050 return X86EMUL_CONTINUE;
2053 static int em_not(struct x86_emulate_ctxt *ctxt)
2055 ctxt->dst.val = ~ctxt->dst.val;
2056 return X86EMUL_CONTINUE;
2059 static int em_neg(struct x86_emulate_ctxt *ctxt)
2061 emulate_1op(ctxt, "neg");
2062 return X86EMUL_CONTINUE;
2065 static int em_mul_ex(struct x86_emulate_ctxt *ctxt)
2069 emulate_1op_rax_rdx(ctxt, "mul", ex);
2070 return X86EMUL_CONTINUE;
2073 static int em_imul_ex(struct x86_emulate_ctxt *ctxt)
2077 emulate_1op_rax_rdx(ctxt, "imul", ex);
2078 return X86EMUL_CONTINUE;
2081 static int em_div_ex(struct x86_emulate_ctxt *ctxt)
2085 emulate_1op_rax_rdx(ctxt, "div", de);
2087 return emulate_de(ctxt);
2088 return X86EMUL_CONTINUE;
2091 static int em_idiv_ex(struct x86_emulate_ctxt *ctxt)
2095 emulate_1op_rax_rdx(ctxt, "idiv", de);
2097 return emulate_de(ctxt);
2098 return X86EMUL_CONTINUE;
2101 static int em_grp45(struct x86_emulate_ctxt *ctxt)
2103 int rc = X86EMUL_CONTINUE;
2105 switch (ctxt->modrm_reg) {
2107 emulate_1op(ctxt, "inc");
2110 emulate_1op(ctxt, "dec");
2112 case 2: /* call near abs */ {
2114 old_eip = ctxt->_eip;
2115 ctxt->_eip = ctxt->src.val;
2116 ctxt->src.val = old_eip;
2120 case 4: /* jmp abs */
2121 ctxt->_eip = ctxt->src.val;
2123 case 5: /* jmp far */
2124 rc = em_jmp_far(ctxt);
2133 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2135 u64 old = ctxt->dst.orig_val64;
2137 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
2138 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2139 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2140 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2141 ctxt->eflags &= ~EFLG_ZF;
2143 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2144 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2146 ctxt->eflags |= EFLG_ZF;
2148 return X86EMUL_CONTINUE;
2151 static int em_ret(struct x86_emulate_ctxt *ctxt)
2153 ctxt->dst.type = OP_REG;
2154 ctxt->dst.addr.reg = &ctxt->_eip;
2155 ctxt->dst.bytes = ctxt->op_bytes;
2156 return em_pop(ctxt);
2159 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2164 rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
2165 if (rc != X86EMUL_CONTINUE)
2167 if (ctxt->op_bytes == 4)
2168 ctxt->_eip = (u32)ctxt->_eip;
2169 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2170 if (rc != X86EMUL_CONTINUE)
2172 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2176 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2178 /* Save real source value, then compare EAX against destination. */
2179 ctxt->src.orig_val = ctxt->src.val;
2180 ctxt->src.val = reg_read(ctxt, VCPU_REGS_RAX);
2181 emulate_2op_SrcV(ctxt, "cmp");
2183 if (ctxt->eflags & EFLG_ZF) {
2184 /* Success: write back to memory. */
2185 ctxt->dst.val = ctxt->src.orig_val;
2187 /* Failure: write the value we saw to EAX. */
2188 ctxt->dst.type = OP_REG;
2189 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2191 return X86EMUL_CONTINUE;
2194 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2196 int seg = ctxt->src2.val;
2200 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2202 rc = load_segment_descriptor(ctxt, sel, seg);
2203 if (rc != X86EMUL_CONTINUE)
2206 ctxt->dst.val = ctxt->src.val;
2211 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2212 struct desc_struct *cs, struct desc_struct *ss)
2214 cs->l = 0; /* will be adjusted later */
2215 set_desc_base(cs, 0); /* flat segment */
2216 cs->g = 1; /* 4kb granularity */
2217 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2218 cs->type = 0x0b; /* Read, Execute, Accessed */
2220 cs->dpl = 0; /* will be adjusted later */
2225 set_desc_base(ss, 0); /* flat segment */
2226 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2227 ss->g = 1; /* 4kb granularity */
2229 ss->type = 0x03; /* Read/Write, Accessed */
2230 ss->d = 1; /* 32bit stack segment */
2237 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2239 u32 eax, ebx, ecx, edx;
2242 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2243 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2244 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2245 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2248 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2250 const struct x86_emulate_ops *ops = ctxt->ops;
2251 u32 eax, ebx, ecx, edx;
2254 * syscall should always be enabled in longmode - so only become
2255 * vendor specific (cpuid) if other modes are active...
2257 if (ctxt->mode == X86EMUL_MODE_PROT64)
2262 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2264 * Intel ("GenuineIntel")
2265 * remark: Intel CPUs only support "syscall" in 64bit
2266 * longmode. Also an 64bit guest with a
2267 * 32bit compat-app running will #UD !! While this
2268 * behaviour can be fixed (by emulating) into AMD
2269 * response - CPUs of AMD can't behave like Intel.
2271 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2272 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2273 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2276 /* AMD ("AuthenticAMD") */
2277 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2278 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2279 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2282 /* AMD ("AMDisbetter!") */
2283 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2284 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2285 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2288 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2292 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2294 const struct x86_emulate_ops *ops = ctxt->ops;
2295 struct desc_struct cs, ss;
2300 /* syscall is not available in real mode */
2301 if (ctxt->mode == X86EMUL_MODE_REAL ||
2302 ctxt->mode == X86EMUL_MODE_VM86)
2303 return emulate_ud(ctxt);
2305 if (!(em_syscall_is_enabled(ctxt)))
2306 return emulate_ud(ctxt);
2308 ops->get_msr(ctxt, MSR_EFER, &efer);
2309 setup_syscalls_segments(ctxt, &cs, &ss);
2311 if (!(efer & EFER_SCE))
2312 return emulate_ud(ctxt);
2314 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2316 cs_sel = (u16)(msr_data & 0xfffc);
2317 ss_sel = (u16)(msr_data + 8);
2319 if (efer & EFER_LMA) {
2323 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2324 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2326 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2327 if (efer & EFER_LMA) {
2328 #ifdef CONFIG_X86_64
2329 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags & ~EFLG_RF;
2332 ctxt->mode == X86EMUL_MODE_PROT64 ?
2333 MSR_LSTAR : MSR_CSTAR, &msr_data);
2334 ctxt->_eip = msr_data;
2336 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2337 ctxt->eflags &= ~(msr_data | EFLG_RF);
2341 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2342 ctxt->_eip = (u32)msr_data;
2344 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2347 return X86EMUL_CONTINUE;
2350 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2352 const struct x86_emulate_ops *ops = ctxt->ops;
2353 struct desc_struct cs, ss;
2358 ops->get_msr(ctxt, MSR_EFER, &efer);
2359 /* inject #GP if in real mode */
2360 if (ctxt->mode == X86EMUL_MODE_REAL)
2361 return emulate_gp(ctxt, 0);
2364 * Not recognized on AMD in compat mode (but is recognized in legacy
2367 if ((ctxt->mode == X86EMUL_MODE_PROT32) && (efer & EFER_LMA)
2368 && !vendor_intel(ctxt))
2369 return emulate_ud(ctxt);
2371 /* XXX sysenter/sysexit have not been tested in 64bit mode.
2372 * Therefore, we inject an #UD.
2374 if (ctxt->mode == X86EMUL_MODE_PROT64)
2375 return emulate_ud(ctxt);
2377 setup_syscalls_segments(ctxt, &cs, &ss);
2379 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2380 switch (ctxt->mode) {
2381 case X86EMUL_MODE_PROT32:
2382 if ((msr_data & 0xfffc) == 0x0)
2383 return emulate_gp(ctxt, 0);
2385 case X86EMUL_MODE_PROT64:
2386 if (msr_data == 0x0)
2387 return emulate_gp(ctxt, 0);
2393 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2394 cs_sel = (u16)msr_data;
2395 cs_sel &= ~SELECTOR_RPL_MASK;
2396 ss_sel = cs_sel + 8;
2397 ss_sel &= ~SELECTOR_RPL_MASK;
2398 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
2403 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2404 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2406 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2407 ctxt->_eip = msr_data;
2409 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2410 *reg_write(ctxt, VCPU_REGS_RSP) = msr_data;
2412 return X86EMUL_CONTINUE;
2415 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2417 const struct x86_emulate_ops *ops = ctxt->ops;
2418 struct desc_struct cs, ss;
2421 u16 cs_sel = 0, ss_sel = 0;
2423 /* inject #GP if in real mode or Virtual 8086 mode */
2424 if (ctxt->mode == X86EMUL_MODE_REAL ||
2425 ctxt->mode == X86EMUL_MODE_VM86)
2426 return emulate_gp(ctxt, 0);
2428 setup_syscalls_segments(ctxt, &cs, &ss);
2430 if ((ctxt->rex_prefix & 0x8) != 0x0)
2431 usermode = X86EMUL_MODE_PROT64;
2433 usermode = X86EMUL_MODE_PROT32;
2437 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2439 case X86EMUL_MODE_PROT32:
2440 cs_sel = (u16)(msr_data + 16);
2441 if ((msr_data & 0xfffc) == 0x0)
2442 return emulate_gp(ctxt, 0);
2443 ss_sel = (u16)(msr_data + 24);
2445 case X86EMUL_MODE_PROT64:
2446 cs_sel = (u16)(msr_data + 32);
2447 if (msr_data == 0x0)
2448 return emulate_gp(ctxt, 0);
2449 ss_sel = cs_sel + 8;
2454 cs_sel |= SELECTOR_RPL_MASK;
2455 ss_sel |= SELECTOR_RPL_MASK;
2457 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2458 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2460 ctxt->_eip = reg_read(ctxt, VCPU_REGS_RDX);
2461 *reg_write(ctxt, VCPU_REGS_RSP) = reg_read(ctxt, VCPU_REGS_RCX);
2463 return X86EMUL_CONTINUE;
2466 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2469 if (ctxt->mode == X86EMUL_MODE_REAL)
2471 if (ctxt->mode == X86EMUL_MODE_VM86)
2473 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2474 return ctxt->ops->cpl(ctxt) > iopl;
2477 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2480 const struct x86_emulate_ops *ops = ctxt->ops;
2481 struct desc_struct tr_seg;
2484 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2485 unsigned mask = (1 << len) - 1;
2488 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2491 if (desc_limit_scaled(&tr_seg) < 103)
2493 base = get_desc_base(&tr_seg);
2494 #ifdef CONFIG_X86_64
2495 base |= ((u64)base3) << 32;
2497 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2498 if (r != X86EMUL_CONTINUE)
2500 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2502 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2503 if (r != X86EMUL_CONTINUE)
2505 if ((perm >> bit_idx) & mask)
2510 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2516 if (emulator_bad_iopl(ctxt))
2517 if (!emulator_io_port_access_allowed(ctxt, port, len))
2520 ctxt->perm_ok = true;
2525 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2526 struct tss_segment_16 *tss)
2528 tss->ip = ctxt->_eip;
2529 tss->flag = ctxt->eflags;
2530 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
2531 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
2532 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
2533 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
2534 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
2535 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
2536 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
2537 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2539 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2540 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2541 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2542 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2543 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2546 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2547 struct tss_segment_16 *tss)
2551 ctxt->_eip = tss->ip;
2552 ctxt->eflags = tss->flag | 2;
2553 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
2554 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
2555 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
2556 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
2557 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
2558 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
2559 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
2560 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
2563 * SDM says that segment selectors are loaded before segment
2566 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2567 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2568 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2569 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2570 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2573 * Now load segment descriptors. If fault happens at this stage
2574 * it is handled in a context of new task
2576 ret = load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR);
2577 if (ret != X86EMUL_CONTINUE)
2579 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2580 if (ret != X86EMUL_CONTINUE)
2582 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2583 if (ret != X86EMUL_CONTINUE)
2585 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2586 if (ret != X86EMUL_CONTINUE)
2588 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2589 if (ret != X86EMUL_CONTINUE)
2592 return X86EMUL_CONTINUE;
2595 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2596 u16 tss_selector, u16 old_tss_sel,
2597 ulong old_tss_base, struct desc_struct *new_desc)
2599 const struct x86_emulate_ops *ops = ctxt->ops;
2600 struct tss_segment_16 tss_seg;
2602 u32 new_tss_base = get_desc_base(new_desc);
2604 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2606 if (ret != X86EMUL_CONTINUE)
2607 /* FIXME: need to provide precise fault address */
2610 save_state_to_tss16(ctxt, &tss_seg);
2612 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2614 if (ret != X86EMUL_CONTINUE)
2615 /* FIXME: need to provide precise fault address */
2618 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2620 if (ret != X86EMUL_CONTINUE)
2621 /* FIXME: need to provide precise fault address */
2624 if (old_tss_sel != 0xffff) {
2625 tss_seg.prev_task_link = old_tss_sel;
2627 ret = ops->write_std(ctxt, new_tss_base,
2628 &tss_seg.prev_task_link,
2629 sizeof tss_seg.prev_task_link,
2631 if (ret != X86EMUL_CONTINUE)
2632 /* FIXME: need to provide precise fault address */
2636 return load_state_from_tss16(ctxt, &tss_seg);
2639 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2640 struct tss_segment_32 *tss)
2642 tss->cr3 = ctxt->ops->get_cr(ctxt, 3);
2643 tss->eip = ctxt->_eip;
2644 tss->eflags = ctxt->eflags;
2645 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
2646 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
2647 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
2648 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
2649 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
2650 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
2651 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
2652 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
2654 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2655 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2656 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2657 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2658 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2659 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2660 tss->ldt_selector = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2663 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2664 struct tss_segment_32 *tss)
2668 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2669 return emulate_gp(ctxt, 0);
2670 ctxt->_eip = tss->eip;
2671 ctxt->eflags = tss->eflags | 2;
2673 /* General purpose registers */
2674 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
2675 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
2676 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
2677 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
2678 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
2679 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
2680 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
2681 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
2684 * SDM says that segment selectors are loaded before segment
2687 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2688 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2689 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2690 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2691 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2692 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2693 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2696 * If we're switching between Protected Mode and VM86, we need to make
2697 * sure to update the mode before loading the segment descriptors so
2698 * that the selectors are interpreted correctly.
2700 * Need to get rflags to the vcpu struct immediately because it
2701 * influences the CPL which is checked at least when loading the segment
2702 * descriptors and when pushing an error code to the new kernel stack.
2704 * TODO Introduce a separate ctxt->ops->set_cpl callback
2706 if (ctxt->eflags & X86_EFLAGS_VM)
2707 ctxt->mode = X86EMUL_MODE_VM86;
2709 ctxt->mode = X86EMUL_MODE_PROT32;
2711 ctxt->ops->set_rflags(ctxt, ctxt->eflags);
2714 * Now load segment descriptors. If fault happenes at this stage
2715 * it is handled in a context of new task
2717 ret = load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2718 if (ret != X86EMUL_CONTINUE)
2720 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2721 if (ret != X86EMUL_CONTINUE)
2723 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2724 if (ret != X86EMUL_CONTINUE)
2726 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2727 if (ret != X86EMUL_CONTINUE)
2729 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2730 if (ret != X86EMUL_CONTINUE)
2732 ret = load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS);
2733 if (ret != X86EMUL_CONTINUE)
2735 ret = load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS);
2736 if (ret != X86EMUL_CONTINUE)
2739 return X86EMUL_CONTINUE;
2742 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2743 u16 tss_selector, u16 old_tss_sel,
2744 ulong old_tss_base, struct desc_struct *new_desc)
2746 const struct x86_emulate_ops *ops = ctxt->ops;
2747 struct tss_segment_32 tss_seg;
2749 u32 new_tss_base = get_desc_base(new_desc);
2751 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2753 if (ret != X86EMUL_CONTINUE)
2754 /* FIXME: need to provide precise fault address */
2757 save_state_to_tss32(ctxt, &tss_seg);
2759 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2761 if (ret != X86EMUL_CONTINUE)
2762 /* FIXME: need to provide precise fault address */
2765 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2767 if (ret != X86EMUL_CONTINUE)
2768 /* FIXME: need to provide precise fault address */
2771 if (old_tss_sel != 0xffff) {
2772 tss_seg.prev_task_link = old_tss_sel;
2774 ret = ops->write_std(ctxt, new_tss_base,
2775 &tss_seg.prev_task_link,
2776 sizeof tss_seg.prev_task_link,
2778 if (ret != X86EMUL_CONTINUE)
2779 /* FIXME: need to provide precise fault address */
2783 return load_state_from_tss32(ctxt, &tss_seg);
2786 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2787 u16 tss_selector, int idt_index, int reason,
2788 bool has_error_code, u32 error_code)
2790 const struct x86_emulate_ops *ops = ctxt->ops;
2791 struct desc_struct curr_tss_desc, next_tss_desc;
2793 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2794 ulong old_tss_base =
2795 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2799 /* FIXME: old_tss_base == ~0 ? */
2801 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
2802 if (ret != X86EMUL_CONTINUE)
2804 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
2805 if (ret != X86EMUL_CONTINUE)
2808 /* FIXME: check that next_tss_desc is tss */
2811 * Check privileges. The three cases are task switch caused by...
2813 * 1. jmp/call/int to task gate: Check against DPL of the task gate
2814 * 2. Exception/IRQ/iret: No check is performed
2815 * 3. jmp/call to TSS: Check against DPL of the TSS
2817 if (reason == TASK_SWITCH_GATE) {
2818 if (idt_index != -1) {
2819 /* Software interrupts */
2820 struct desc_struct task_gate_desc;
2823 ret = read_interrupt_descriptor(ctxt, idt_index,
2825 if (ret != X86EMUL_CONTINUE)
2828 dpl = task_gate_desc.dpl;
2829 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2830 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
2832 } else if (reason != TASK_SWITCH_IRET) {
2833 int dpl = next_tss_desc.dpl;
2834 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2835 return emulate_gp(ctxt, tss_selector);
2839 desc_limit = desc_limit_scaled(&next_tss_desc);
2840 if (!next_tss_desc.p ||
2841 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2842 desc_limit < 0x2b)) {
2843 emulate_ts(ctxt, tss_selector & 0xfffc);
2844 return X86EMUL_PROPAGATE_FAULT;
2847 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2848 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2849 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2852 if (reason == TASK_SWITCH_IRET)
2853 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2855 /* set back link to prev task only if NT bit is set in eflags
2856 note that old_tss_sel is not used after this point */
2857 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2858 old_tss_sel = 0xffff;
2860 if (next_tss_desc.type & 8)
2861 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2862 old_tss_base, &next_tss_desc);
2864 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2865 old_tss_base, &next_tss_desc);
2866 if (ret != X86EMUL_CONTINUE)
2869 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2870 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2872 if (reason != TASK_SWITCH_IRET) {
2873 next_tss_desc.type |= (1 << 1); /* set busy flag */
2874 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2877 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2878 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2880 if (has_error_code) {
2881 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2882 ctxt->lock_prefix = 0;
2883 ctxt->src.val = (unsigned long) error_code;
2884 ret = em_push(ctxt);
2890 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2891 u16 tss_selector, int idt_index, int reason,
2892 bool has_error_code, u32 error_code)
2896 invalidate_registers(ctxt);
2897 ctxt->_eip = ctxt->eip;
2898 ctxt->dst.type = OP_NONE;
2900 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
2901 has_error_code, error_code);
2903 if (rc == X86EMUL_CONTINUE) {
2904 ctxt->eip = ctxt->_eip;
2905 writeback_registers(ctxt);
2908 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2911 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
2914 int df = (ctxt->eflags & EFLG_DF) ? -op->count : op->count;
2916 register_address_increment(ctxt, reg_rmw(ctxt, reg), df * op->bytes);
2917 op->addr.mem.ea = register_address(ctxt, reg_read(ctxt, reg));
2920 static int em_das(struct x86_emulate_ctxt *ctxt)
2923 bool af, cf, old_cf;
2925 cf = ctxt->eflags & X86_EFLAGS_CF;
2931 af = ctxt->eflags & X86_EFLAGS_AF;
2932 if ((al & 0x0f) > 9 || af) {
2934 cf = old_cf | (al >= 250);
2939 if (old_al > 0x99 || old_cf) {
2945 /* Set PF, ZF, SF */
2946 ctxt->src.type = OP_IMM;
2948 ctxt->src.bytes = 1;
2949 emulate_2op_SrcV(ctxt, "or");
2950 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2952 ctxt->eflags |= X86_EFLAGS_CF;
2954 ctxt->eflags |= X86_EFLAGS_AF;
2955 return X86EMUL_CONTINUE;
2958 static int em_aad(struct x86_emulate_ctxt *ctxt)
2960 u8 al = ctxt->dst.val & 0xff;
2961 u8 ah = (ctxt->dst.val >> 8) & 0xff;
2963 al = (al + (ah * ctxt->src.val)) & 0xff;
2965 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
2967 ctxt->eflags &= ~(X86_EFLAGS_PF | X86_EFLAGS_SF | X86_EFLAGS_ZF);
2970 ctxt->eflags |= X86_EFLAGS_ZF;
2972 ctxt->eflags |= X86_EFLAGS_PF;
2974 ctxt->eflags |= X86_EFLAGS_SF;
2976 return X86EMUL_CONTINUE;
2979 static int em_call(struct x86_emulate_ctxt *ctxt)
2981 long rel = ctxt->src.val;
2983 ctxt->src.val = (unsigned long)ctxt->_eip;
2985 return em_push(ctxt);
2988 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2994 old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2995 old_eip = ctxt->_eip;
2997 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2998 if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
2999 return X86EMUL_CONTINUE;
3002 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
3004 ctxt->src.val = old_cs;
3006 if (rc != X86EMUL_CONTINUE)
3009 ctxt->src.val = old_eip;
3010 return em_push(ctxt);
3013 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
3017 ctxt->dst.type = OP_REG;
3018 ctxt->dst.addr.reg = &ctxt->_eip;
3019 ctxt->dst.bytes = ctxt->op_bytes;
3020 rc = emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
3021 if (rc != X86EMUL_CONTINUE)
3023 rsp_increment(ctxt, ctxt->src.val);
3024 return X86EMUL_CONTINUE;
3027 static int em_add(struct x86_emulate_ctxt *ctxt)
3029 emulate_2op_SrcV(ctxt, "add");
3030 return X86EMUL_CONTINUE;
3033 static int em_or(struct x86_emulate_ctxt *ctxt)
3035 emulate_2op_SrcV(ctxt, "or");
3036 return X86EMUL_CONTINUE;
3039 static int em_adc(struct x86_emulate_ctxt *ctxt)
3041 emulate_2op_SrcV(ctxt, "adc");
3042 return X86EMUL_CONTINUE;
3045 static int em_sbb(struct x86_emulate_ctxt *ctxt)
3047 emulate_2op_SrcV(ctxt, "sbb");
3048 return X86EMUL_CONTINUE;
3051 static int em_and(struct x86_emulate_ctxt *ctxt)
3053 emulate_2op_SrcV(ctxt, "and");
3054 return X86EMUL_CONTINUE;
3057 static int em_sub(struct x86_emulate_ctxt *ctxt)
3059 emulate_2op_SrcV(ctxt, "sub");
3060 return X86EMUL_CONTINUE;
3063 static int em_xor(struct x86_emulate_ctxt *ctxt)
3065 emulate_2op_SrcV(ctxt, "xor");
3066 return X86EMUL_CONTINUE;
3069 static int em_cmp(struct x86_emulate_ctxt *ctxt)
3071 emulate_2op_SrcV(ctxt, "cmp");
3072 return X86EMUL_CONTINUE;
3075 static int em_test(struct x86_emulate_ctxt *ctxt)
3077 emulate_2op_SrcV(ctxt, "test");
3078 return X86EMUL_CONTINUE;
3081 static int em_xchg(struct x86_emulate_ctxt *ctxt)
3083 /* Write back the register source. */
3084 ctxt->src.val = ctxt->dst.val;
3085 write_register_operand(&ctxt->src);
3087 /* Write back the memory destination with implicit LOCK prefix. */
3088 ctxt->dst.val = ctxt->src.orig_val;
3089 ctxt->lock_prefix = 1;
3090 return X86EMUL_CONTINUE;
3093 static int em_imul(struct x86_emulate_ctxt *ctxt)
3095 emulate_2op_SrcV_nobyte(ctxt, "imul");
3096 return X86EMUL_CONTINUE;
3099 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
3101 ctxt->dst.val = ctxt->src2.val;
3102 return em_imul(ctxt);
3105 static int em_cwd(struct x86_emulate_ctxt *ctxt)
3107 ctxt->dst.type = OP_REG;
3108 ctxt->dst.bytes = ctxt->src.bytes;
3109 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
3110 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
3112 return X86EMUL_CONTINUE;
3115 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
3119 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
3120 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
3121 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
3122 return X86EMUL_CONTINUE;
3125 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
3129 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
3130 return emulate_gp(ctxt, 0);
3131 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
3132 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
3133 return X86EMUL_CONTINUE;
3136 static int em_mov(struct x86_emulate_ctxt *ctxt)
3138 memcpy(ctxt->dst.valptr, ctxt->src.valptr, ctxt->op_bytes);
3139 return X86EMUL_CONTINUE;
3142 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3144 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3145 return emulate_gp(ctxt, 0);
3147 /* Disable writeback. */
3148 ctxt->dst.type = OP_NONE;
3149 return X86EMUL_CONTINUE;
3152 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3156 if (ctxt->mode == X86EMUL_MODE_PROT64)
3157 val = ctxt->src.val & ~0ULL;
3159 val = ctxt->src.val & ~0U;
3161 /* #UD condition is already handled. */
3162 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3163 return emulate_gp(ctxt, 0);
3165 /* Disable writeback. */
3166 ctxt->dst.type = OP_NONE;
3167 return X86EMUL_CONTINUE;
3170 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3174 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3175 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3176 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3177 return emulate_gp(ctxt, 0);
3179 return X86EMUL_CONTINUE;
3182 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3186 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3187 return emulate_gp(ctxt, 0);
3189 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3190 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3191 return X86EMUL_CONTINUE;
3194 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3196 if (ctxt->modrm_reg > VCPU_SREG_GS)
3197 return emulate_ud(ctxt);
3199 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3200 return X86EMUL_CONTINUE;
3203 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3205 u16 sel = ctxt->src.val;
3207 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3208 return emulate_ud(ctxt);
3210 if (ctxt->modrm_reg == VCPU_SREG_SS)
3211 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3213 /* Disable writeback. */
3214 ctxt->dst.type = OP_NONE;
3215 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3218 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3220 u16 sel = ctxt->src.val;
3222 /* Disable writeback. */
3223 ctxt->dst.type = OP_NONE;
3224 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3227 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3229 u16 sel = ctxt->src.val;
3231 /* Disable writeback. */
3232 ctxt->dst.type = OP_NONE;
3233 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3236 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3241 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3242 if (rc == X86EMUL_CONTINUE)
3243 ctxt->ops->invlpg(ctxt, linear);
3244 /* Disable writeback. */
3245 ctxt->dst.type = OP_NONE;
3246 return X86EMUL_CONTINUE;
3249 static int em_clts(struct x86_emulate_ctxt *ctxt)
3253 cr0 = ctxt->ops->get_cr(ctxt, 0);
3255 ctxt->ops->set_cr(ctxt, 0, cr0);
3256 return X86EMUL_CONTINUE;
3259 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
3263 if (ctxt->modrm_mod != 3 || ctxt->modrm_rm != 1)
3264 return X86EMUL_UNHANDLEABLE;
3266 rc = ctxt->ops->fix_hypercall(ctxt);
3267 if (rc != X86EMUL_CONTINUE)
3270 /* Let the processor re-execute the fixed hypercall */
3271 ctxt->_eip = ctxt->eip;
3272 /* Disable writeback. */
3273 ctxt->dst.type = OP_NONE;
3274 return X86EMUL_CONTINUE;
3277 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3278 void (*get)(struct x86_emulate_ctxt *ctxt,
3279 struct desc_ptr *ptr))
3281 struct desc_ptr desc_ptr;
3283 if (ctxt->mode == X86EMUL_MODE_PROT64)
3285 get(ctxt, &desc_ptr);
3286 if (ctxt->op_bytes == 2) {
3288 desc_ptr.address &= 0x00ffffff;
3290 /* Disable writeback. */
3291 ctxt->dst.type = OP_NONE;
3292 return segmented_write(ctxt, ctxt->dst.addr.mem,
3293 &desc_ptr, 2 + ctxt->op_bytes);
3296 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3298 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3301 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3303 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3306 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3308 struct desc_ptr desc_ptr;
3311 if (ctxt->mode == X86EMUL_MODE_PROT64)
3313 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3314 &desc_ptr.size, &desc_ptr.address,
3316 if (rc != X86EMUL_CONTINUE)
3318 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3319 /* Disable writeback. */
3320 ctxt->dst.type = OP_NONE;
3321 return X86EMUL_CONTINUE;
3324 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
3328 rc = ctxt->ops->fix_hypercall(ctxt);
3330 /* Disable writeback. */
3331 ctxt->dst.type = OP_NONE;
3335 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3337 struct desc_ptr desc_ptr;
3340 if (ctxt->mode == X86EMUL_MODE_PROT64)
3342 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3343 &desc_ptr.size, &desc_ptr.address,
3345 if (rc != X86EMUL_CONTINUE)
3347 ctxt->ops->set_idt(ctxt, &desc_ptr);
3348 /* Disable writeback. */
3349 ctxt->dst.type = OP_NONE;
3350 return X86EMUL_CONTINUE;
3353 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3355 ctxt->dst.bytes = 2;
3356 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3357 return X86EMUL_CONTINUE;
3360 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3362 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3363 | (ctxt->src.val & 0x0f));
3364 ctxt->dst.type = OP_NONE;
3365 return X86EMUL_CONTINUE;
3368 static int em_loop(struct x86_emulate_ctxt *ctxt)
3370 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX), -1);
3371 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3372 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3373 jmp_rel(ctxt, ctxt->src.val);
3375 return X86EMUL_CONTINUE;
3378 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3380 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3381 jmp_rel(ctxt, ctxt->src.val);
3383 return X86EMUL_CONTINUE;
3386 static int em_in(struct x86_emulate_ctxt *ctxt)
3388 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3390 return X86EMUL_IO_NEEDED;
3392 return X86EMUL_CONTINUE;
3395 static int em_out(struct x86_emulate_ctxt *ctxt)
3397 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3399 /* Disable writeback. */
3400 ctxt->dst.type = OP_NONE;
3401 return X86EMUL_CONTINUE;
3404 static int em_cli(struct x86_emulate_ctxt *ctxt)
3406 if (emulator_bad_iopl(ctxt))
3407 return emulate_gp(ctxt, 0);
3409 ctxt->eflags &= ~X86_EFLAGS_IF;
3410 return X86EMUL_CONTINUE;
3413 static int em_sti(struct x86_emulate_ctxt *ctxt)
3415 if (emulator_bad_iopl(ctxt))
3416 return emulate_gp(ctxt, 0);
3418 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3419 ctxt->eflags |= X86_EFLAGS_IF;
3420 return X86EMUL_CONTINUE;
3423 static int em_bt(struct x86_emulate_ctxt *ctxt)
3425 /* Disable writeback. */
3426 ctxt->dst.type = OP_NONE;
3427 /* only subword offset */
3428 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
3430 emulate_2op_SrcV_nobyte(ctxt, "bt");
3431 return X86EMUL_CONTINUE;
3434 static int em_bts(struct x86_emulate_ctxt *ctxt)
3436 emulate_2op_SrcV_nobyte(ctxt, "bts");
3437 return X86EMUL_CONTINUE;
3440 static int em_btr(struct x86_emulate_ctxt *ctxt)
3442 emulate_2op_SrcV_nobyte(ctxt, "btr");
3443 return X86EMUL_CONTINUE;
3446 static int em_btc(struct x86_emulate_ctxt *ctxt)
3448 emulate_2op_SrcV_nobyte(ctxt, "btc");
3449 return X86EMUL_CONTINUE;
3452 static int em_bsf(struct x86_emulate_ctxt *ctxt)
3454 emulate_2op_SrcV_nobyte(ctxt, "bsf");
3455 return X86EMUL_CONTINUE;
3458 static int em_bsr(struct x86_emulate_ctxt *ctxt)
3460 emulate_2op_SrcV_nobyte(ctxt, "bsr");
3461 return X86EMUL_CONTINUE;
3464 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3466 u32 eax, ebx, ecx, edx;
3468 eax = reg_read(ctxt, VCPU_REGS_RAX);
3469 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3470 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3471 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3472 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3473 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3474 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3475 return X86EMUL_CONTINUE;
3478 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3480 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3481 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3482 return X86EMUL_CONTINUE;
3485 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3487 switch (ctxt->op_bytes) {
3488 #ifdef CONFIG_X86_64
3490 asm("bswap %0" : "+r"(ctxt->dst.val));
3494 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3497 return X86EMUL_CONTINUE;
3500 static bool valid_cr(int nr)
3512 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
3514 if (!valid_cr(ctxt->modrm_reg))
3515 return emulate_ud(ctxt);
3517 return X86EMUL_CONTINUE;
3520 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
3522 u64 new_val = ctxt->src.val64;
3523 int cr = ctxt->modrm_reg;
3526 static u64 cr_reserved_bits[] = {
3527 0xffffffff00000000ULL,
3528 0, 0, 0, /* CR3 checked later */
3535 return emulate_ud(ctxt);
3537 if (new_val & cr_reserved_bits[cr])
3538 return emulate_gp(ctxt, 0);
3543 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
3544 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
3545 return emulate_gp(ctxt, 0);
3547 cr4 = ctxt->ops->get_cr(ctxt, 4);
3548 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3550 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
3551 !(cr4 & X86_CR4_PAE))
3552 return emulate_gp(ctxt, 0);
3559 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3560 if (efer & EFER_LMA)
3561 rsvd = CR3_L_MODE_RESERVED_BITS;
3562 else if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PAE)
3563 rsvd = CR3_PAE_RESERVED_BITS;
3564 else if (ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PG)
3565 rsvd = CR3_NONPAE_RESERVED_BITS;
3568 return emulate_gp(ctxt, 0);
3573 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3575 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
3576 return emulate_gp(ctxt, 0);
3582 return X86EMUL_CONTINUE;
3585 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
3589 ctxt->ops->get_dr(ctxt, 7, &dr7);
3591 /* Check if DR7.Global_Enable is set */
3592 return dr7 & (1 << 13);
3595 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
3597 int dr = ctxt->modrm_reg;
3601 return emulate_ud(ctxt);
3603 cr4 = ctxt->ops->get_cr(ctxt, 4);
3604 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
3605 return emulate_ud(ctxt);
3607 if (check_dr7_gd(ctxt))
3608 return emulate_db(ctxt);
3610 return X86EMUL_CONTINUE;
3613 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
3615 u64 new_val = ctxt->src.val64;
3616 int dr = ctxt->modrm_reg;
3618 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
3619 return emulate_gp(ctxt, 0);
3621 return check_dr_read(ctxt);
3624 static int check_svme(struct x86_emulate_ctxt *ctxt)
3628 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3630 if (!(efer & EFER_SVME))
3631 return emulate_ud(ctxt);
3633 return X86EMUL_CONTINUE;
3636 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
3638 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
3640 /* Valid physical address? */
3641 if (rax & 0xffff000000000000ULL)
3642 return emulate_gp(ctxt, 0);
3644 return check_svme(ctxt);
3647 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
3649 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3651 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3652 return emulate_ud(ctxt);
3654 return X86EMUL_CONTINUE;
3657 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
3659 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3660 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
3662 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3664 return emulate_gp(ctxt, 0);
3666 return X86EMUL_CONTINUE;
3669 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
3671 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
3672 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3673 return emulate_gp(ctxt, 0);
3675 return X86EMUL_CONTINUE;
3678 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
3680 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
3681 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3682 return emulate_gp(ctxt, 0);
3684 return X86EMUL_CONTINUE;
3687 #define D(_y) { .flags = (_y) }
3688 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
3689 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
3690 .check_perm = (_p) }
3692 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3693 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
3694 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
3695 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
3696 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3697 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
3698 #define II(_f, _e, _i) \
3699 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
3700 #define IIP(_f, _e, _i, _p) \
3701 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
3702 .check_perm = (_p) }
3703 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3705 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3706 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3707 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3708 #define I2bvIP(_f, _e, _i, _p) \
3709 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
3711 #define I6ALU(_f, _e) I2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3712 I2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3713 I2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3715 static const struct opcode group7_rm1[] = {
3716 DI(SrcNone | Priv, monitor),
3717 DI(SrcNone | Priv, mwait),
3721 static const struct opcode group7_rm3[] = {
3722 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
3723 II(SrcNone | Prot | VendorSpecific, em_vmmcall, vmmcall),
3724 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
3725 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
3726 DIP(SrcNone | Prot | Priv, stgi, check_svme),
3727 DIP(SrcNone | Prot | Priv, clgi, check_svme),
3728 DIP(SrcNone | Prot | Priv, skinit, check_svme),
3729 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
3732 static const struct opcode group7_rm7[] = {
3734 DIP(SrcNone, rdtscp, check_rdtsc),
3738 static const struct opcode group1[] = {
3740 I(Lock | PageTable, em_or),
3743 I(Lock | PageTable, em_and),
3749 static const struct opcode group1A[] = {
3750 I(DstMem | SrcNone | Mov | Stack, em_pop), N, N, N, N, N, N, N,
3753 static const struct opcode group3[] = {
3754 I(DstMem | SrcImm | NoWrite, em_test),
3755 I(DstMem | SrcImm | NoWrite, em_test),
3756 I(DstMem | SrcNone | Lock, em_not),
3757 I(DstMem | SrcNone | Lock, em_neg),
3758 I(SrcMem, em_mul_ex),
3759 I(SrcMem, em_imul_ex),
3760 I(SrcMem, em_div_ex),
3761 I(SrcMem, em_idiv_ex),
3764 static const struct opcode group4[] = {
3765 I(ByteOp | DstMem | SrcNone | Lock, em_grp45),
3766 I(ByteOp | DstMem | SrcNone | Lock, em_grp45),
3770 static const struct opcode group5[] = {
3771 I(DstMem | SrcNone | Lock, em_grp45),
3772 I(DstMem | SrcNone | Lock, em_grp45),
3773 I(SrcMem | Stack, em_grp45),
3774 I(SrcMemFAddr | ImplicitOps | Stack, em_call_far),
3775 I(SrcMem | Stack, em_grp45),
3776 I(SrcMemFAddr | ImplicitOps, em_grp45),
3777 I(SrcMem | Stack, em_grp45), N,
3780 static const struct opcode group6[] = {
3783 II(Prot | Priv | SrcMem16, em_lldt, lldt),
3784 II(Prot | Priv | SrcMem16, em_ltr, ltr),
3788 static const struct group_dual group7 = { {
3789 II(Mov | DstMem | Priv, em_sgdt, sgdt),
3790 II(Mov | DstMem | Priv, em_sidt, sidt),
3791 II(SrcMem | Priv, em_lgdt, lgdt),
3792 II(SrcMem | Priv, em_lidt, lidt),
3793 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3794 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3795 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3797 I(SrcNone | Priv | VendorSpecific, em_vmcall),
3799 N, EXT(0, group7_rm3),
3800 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3801 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3805 static const struct opcode group8[] = {
3807 I(DstMem | SrcImmByte, em_bt),
3808 I(DstMem | SrcImmByte | Lock | PageTable, em_bts),
3809 I(DstMem | SrcImmByte | Lock, em_btr),
3810 I(DstMem | SrcImmByte | Lock | PageTable, em_btc),
3813 static const struct group_dual group9 = { {
3814 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
3816 N, N, N, N, N, N, N, N,
3819 static const struct opcode group11[] = {
3820 I(DstMem | SrcImm | Mov | PageTable, em_mov),
3824 static const struct gprefix pfx_0f_6f_0f_7f = {
3825 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
3828 static const struct gprefix pfx_vmovntpx = {
3829 I(0, em_mov), N, N, N,
3832 static const struct escape escape_d9 = { {
3833 N, N, N, N, N, N, N, I(DstMem, em_fnstcw),
3836 N, N, N, N, N, N, N, N,
3838 N, N, N, N, N, N, N, N,
3840 N, N, N, N, N, N, N, N,
3842 N, N, N, N, N, N, N, N,
3844 N, N, N, N, N, N, N, N,
3846 N, N, N, N, N, N, N, N,
3848 N, N, N, N, N, N, N, N,
3850 N, N, N, N, N, N, N, N,
3853 static const struct escape escape_db = { {
3854 N, N, N, N, N, N, N, N,
3857 N, N, N, N, N, N, N, N,
3859 N, N, N, N, N, N, N, N,
3861 N, N, N, N, N, N, N, N,
3863 N, N, N, N, N, N, N, N,
3865 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
3867 N, N, N, N, N, N, N, N,
3869 N, N, N, N, N, N, N, N,
3871 N, N, N, N, N, N, N, N,
3874 static const struct escape escape_dd = { {
3875 N, N, N, N, N, N, N, I(DstMem, em_fnstsw),
3878 N, N, N, N, N, N, N, N,
3880 N, N, N, N, N, N, N, N,
3882 N, N, N, N, N, N, N, N,
3884 N, N, N, N, N, N, N, N,
3886 N, N, N, N, N, N, N, N,
3888 N, N, N, N, N, N, N, N,
3890 N, N, N, N, N, N, N, N,
3892 N, N, N, N, N, N, N, N,
3895 static const struct opcode opcode_table[256] = {
3897 I6ALU(Lock, em_add),
3898 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
3899 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
3901 I6ALU(Lock | PageTable, em_or),
3902 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
3905 I6ALU(Lock, em_adc),
3906 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
3907 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
3909 I6ALU(Lock, em_sbb),
3910 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
3911 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
3913 I6ALU(Lock | PageTable, em_and), N, N,
3915 I6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3917 I6ALU(Lock, em_xor), N, N,
3919 I6ALU(NoWrite, em_cmp), N, N,
3923 X8(I(SrcReg | Stack, em_push)),
3925 X8(I(DstReg | Stack, em_pop)),
3927 I(ImplicitOps | Stack | No64, em_pusha),
3928 I(ImplicitOps | Stack | No64, em_popa),
3929 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3932 I(SrcImm | Mov | Stack, em_push),
3933 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3934 I(SrcImmByte | Mov | Stack, em_push),
3935 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3936 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
3937 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
3941 G(ByteOp | DstMem | SrcImm, group1),
3942 G(DstMem | SrcImm, group1),
3943 G(ByteOp | DstMem | SrcImm | No64, group1),
3944 G(DstMem | SrcImmByte, group1),
3945 I2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
3946 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
3948 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
3949 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3950 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
3951 D(ModRM | SrcMem | NoAccess | DstReg),
3952 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
3955 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3957 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3958 I(SrcImmFAddr | No64, em_call_far), N,
3959 II(ImplicitOps | Stack, em_pushf, pushf),
3960 II(ImplicitOps | Stack, em_popf, popf), N, I(ImplicitOps, em_lahf),
3962 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3963 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
3964 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3965 I2bv(SrcSI | DstDI | String | NoWrite, em_cmp),
3967 I2bv(DstAcc | SrcImm | NoWrite, em_test),
3968 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3969 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3970 I2bv(SrcAcc | DstDI | String | NoWrite, em_cmp),
3972 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3974 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
3976 D2bv(DstMem | SrcImmByte | ModRM),
3977 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3978 I(ImplicitOps | Stack, em_ret),
3979 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
3980 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
3981 G(ByteOp, group11), G(0, group11),
3983 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
3984 N, I(ImplicitOps | Stack, em_ret_far),
3985 D(ImplicitOps), DI(SrcImmByte, intn),
3986 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3988 D2bv(DstMem | SrcOne | ModRM), D2bv(DstMem | ModRM),
3989 N, I(DstAcc | SrcImmByte | No64, em_aad), N, N,
3991 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
3993 X3(I(SrcImmByte, em_loop)),
3994 I(SrcImmByte, em_jcxz),
3995 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
3996 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
3998 I(SrcImm | Stack, em_call), D(SrcImm | ImplicitOps),
3999 I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
4000 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
4001 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
4003 N, DI(ImplicitOps, icebp), N, N,
4004 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
4005 G(ByteOp, group3), G(0, group3),
4007 D(ImplicitOps), D(ImplicitOps),
4008 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
4009 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
4012 static const struct opcode twobyte_table[256] = {
4014 G(0, group6), GD(0, &group7), N, N,
4015 N, I(ImplicitOps | VendorSpecific, em_syscall),
4016 II(ImplicitOps | Priv, em_clts, clts), N,
4017 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
4018 N, D(ImplicitOps | ModRM), N, N,
4020 N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
4022 DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
4023 DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
4024 IIP(ModRM | SrcMem | Priv | Op3264, em_cr_write, cr_write, check_cr_write),
4025 IIP(ModRM | SrcMem | Priv | Op3264, em_dr_write, dr_write, check_dr_write),
4027 N, N, N, GP(ModRM | DstMem | SrcReg | Sse | Mov | Aligned, &pfx_vmovntpx),
4030 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
4031 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
4032 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
4033 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
4034 I(ImplicitOps | VendorSpecific, em_sysenter),
4035 I(ImplicitOps | Priv | VendorSpecific, em_sysexit),
4037 N, N, N, N, N, N, N, N,
4039 X16(D(DstReg | SrcMem | ModRM | Mov)),
4041 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4046 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
4051 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
4055 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
4057 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
4058 II(ImplicitOps, em_cpuid, cpuid), I(DstMem | SrcReg | ModRM | BitOp, em_bt),
4059 D(DstMem | SrcReg | Src2ImmByte | ModRM),
4060 D(DstMem | SrcReg | Src2CL | ModRM), N, N,
4062 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
4063 DI(ImplicitOps, rsm),
4064 I(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
4065 D(DstMem | SrcReg | Src2ImmByte | ModRM),
4066 D(DstMem | SrcReg | Src2CL | ModRM),
4067 D(ModRM), I(DstReg | SrcMem | ModRM, em_imul),
4069 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_cmpxchg),
4070 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
4071 I(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
4072 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
4073 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
4074 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4078 I(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
4079 I(DstReg | SrcMem | ModRM, em_bsf), I(DstReg | SrcMem | ModRM, em_bsr),
4080 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4082 D2bv(DstMem | SrcReg | ModRM | Lock),
4083 N, D(DstMem | SrcReg | ModRM | Mov),
4084 N, N, N, GD(0, &group9),
4086 X8(I(DstReg, em_bswap)),
4088 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4090 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4092 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
4109 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4113 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4119 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
4120 unsigned size, bool sign_extension)
4122 int rc = X86EMUL_CONTINUE;
4126 op->addr.mem.ea = ctxt->_eip;
4127 /* NB. Immediates are sign-extended as necessary. */
4128 switch (op->bytes) {
4130 op->val = insn_fetch(s8, ctxt);
4133 op->val = insn_fetch(s16, ctxt);
4136 op->val = insn_fetch(s32, ctxt);
4139 op->val = insn_fetch(s64, ctxt);
4142 if (!sign_extension) {
4143 switch (op->bytes) {
4151 op->val &= 0xffffffff;
4159 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
4162 int rc = X86EMUL_CONTINUE;
4166 decode_register_operand(ctxt, op);
4169 rc = decode_imm(ctxt, op, 1, false);
4172 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4176 if ((ctxt->d & BitOp) && op == &ctxt->dst)
4177 fetch_bit_operand(ctxt);
4178 op->orig_val = op->val;
4181 ctxt->memop.bytes = 8;
4185 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4186 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4187 fetch_register_operand(op);
4188 op->orig_val = op->val;
4192 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4194 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RDI));
4195 op->addr.mem.seg = VCPU_SREG_ES;
4202 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4203 fetch_register_operand(op);
4207 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4210 rc = decode_imm(ctxt, op, 1, true);
4217 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
4220 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
4223 ctxt->memop.bytes = 1;
4226 ctxt->memop.bytes = 2;
4229 ctxt->memop.bytes = 4;
4232 rc = decode_imm(ctxt, op, 2, false);
4235 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
4239 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4241 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RSI));
4242 op->addr.mem.seg = seg_override(ctxt);
4248 op->addr.mem.ea = ctxt->_eip;
4249 op->bytes = ctxt->op_bytes + 2;
4250 insn_fetch_arr(op->valptr, op->bytes, ctxt);
4253 ctxt->memop.bytes = ctxt->op_bytes + 2;
4256 op->val = VCPU_SREG_ES;
4259 op->val = VCPU_SREG_CS;
4262 op->val = VCPU_SREG_SS;
4265 op->val = VCPU_SREG_DS;
4268 op->val = VCPU_SREG_FS;
4271 op->val = VCPU_SREG_GS;
4274 /* Special instructions do their own operand decoding. */
4276 op->type = OP_NONE; /* Disable writeback. */
4284 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
4286 int rc = X86EMUL_CONTINUE;
4287 int mode = ctxt->mode;
4288 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
4289 bool op_prefix = false;
4290 struct opcode opcode;
4292 ctxt->memop.type = OP_NONE;
4293 ctxt->memopp = NULL;
4294 ctxt->_eip = ctxt->eip;
4295 ctxt->fetch.start = ctxt->_eip;
4296 ctxt->fetch.end = ctxt->fetch.start + insn_len;
4298 memcpy(ctxt->fetch.data, insn, insn_len);
4301 case X86EMUL_MODE_REAL:
4302 case X86EMUL_MODE_VM86:
4303 case X86EMUL_MODE_PROT16:
4304 def_op_bytes = def_ad_bytes = 2;
4306 case X86EMUL_MODE_PROT32:
4307 def_op_bytes = def_ad_bytes = 4;
4309 #ifdef CONFIG_X86_64
4310 case X86EMUL_MODE_PROT64:
4316 return EMULATION_FAILED;
4319 ctxt->op_bytes = def_op_bytes;
4320 ctxt->ad_bytes = def_ad_bytes;
4322 /* Legacy prefixes. */
4324 switch (ctxt->b = insn_fetch(u8, ctxt)) {
4325 case 0x66: /* operand-size override */
4327 /* switch between 2/4 bytes */
4328 ctxt->op_bytes = def_op_bytes ^ 6;
4330 case 0x67: /* address-size override */
4331 if (mode == X86EMUL_MODE_PROT64)
4332 /* switch between 4/8 bytes */
4333 ctxt->ad_bytes = def_ad_bytes ^ 12;
4335 /* switch between 2/4 bytes */
4336 ctxt->ad_bytes = def_ad_bytes ^ 6;
4338 case 0x26: /* ES override */
4339 case 0x2e: /* CS override */
4340 case 0x36: /* SS override */
4341 case 0x3e: /* DS override */
4342 set_seg_override(ctxt, (ctxt->b >> 3) & 3);
4344 case 0x64: /* FS override */
4345 case 0x65: /* GS override */
4346 set_seg_override(ctxt, ctxt->b & 7);
4348 case 0x40 ... 0x4f: /* REX */
4349 if (mode != X86EMUL_MODE_PROT64)
4351 ctxt->rex_prefix = ctxt->b;
4353 case 0xf0: /* LOCK */
4354 ctxt->lock_prefix = 1;
4356 case 0xf2: /* REPNE/REPNZ */
4357 case 0xf3: /* REP/REPE/REPZ */
4358 ctxt->rep_prefix = ctxt->b;
4364 /* Any legacy prefix after a REX prefix nullifies its effect. */
4366 ctxt->rex_prefix = 0;
4372 if (ctxt->rex_prefix & 8)
4373 ctxt->op_bytes = 8; /* REX.W */
4375 /* Opcode byte(s). */
4376 opcode = opcode_table[ctxt->b];
4377 /* Two-byte opcode? */
4378 if (ctxt->b == 0x0f) {
4380 ctxt->b = insn_fetch(u8, ctxt);
4381 opcode = twobyte_table[ctxt->b];
4383 ctxt->d = opcode.flags;
4385 if (ctxt->d & ModRM)
4386 ctxt->modrm = insn_fetch(u8, ctxt);
4388 while (ctxt->d & GroupMask) {
4389 switch (ctxt->d & GroupMask) {
4391 goffset = (ctxt->modrm >> 3) & 7;
4392 opcode = opcode.u.group[goffset];
4395 goffset = (ctxt->modrm >> 3) & 7;
4396 if ((ctxt->modrm >> 6) == 3)
4397 opcode = opcode.u.gdual->mod3[goffset];
4399 opcode = opcode.u.gdual->mod012[goffset];
4402 goffset = ctxt->modrm & 7;
4403 opcode = opcode.u.group[goffset];
4406 if (ctxt->rep_prefix && op_prefix)
4407 return EMULATION_FAILED;
4408 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
4409 switch (simd_prefix) {
4410 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
4411 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
4412 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
4413 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
4417 if (ctxt->modrm > 0xbf)
4418 opcode = opcode.u.esc->high[ctxt->modrm - 0xc0];
4420 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
4423 return EMULATION_FAILED;
4426 ctxt->d &= ~(u64)GroupMask;
4427 ctxt->d |= opcode.flags;
4430 ctxt->execute = opcode.u.execute;
4431 ctxt->check_perm = opcode.check_perm;
4432 ctxt->intercept = opcode.intercept;
4435 if (ctxt->d == 0 || (ctxt->d & Undefined))
4436 return EMULATION_FAILED;
4438 if (!(ctxt->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
4439 return EMULATION_FAILED;
4441 if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
4444 if (ctxt->d & Op3264) {
4445 if (mode == X86EMUL_MODE_PROT64)
4452 ctxt->op_bytes = 16;
4453 else if (ctxt->d & Mmx)
4456 /* ModRM and SIB bytes. */
4457 if (ctxt->d & ModRM) {
4458 rc = decode_modrm(ctxt, &ctxt->memop);
4459 if (!ctxt->has_seg_override)
4460 set_seg_override(ctxt, ctxt->modrm_seg);
4461 } else if (ctxt->d & MemAbs)
4462 rc = decode_abs(ctxt, &ctxt->memop);
4463 if (rc != X86EMUL_CONTINUE)
4466 if (!ctxt->has_seg_override)
4467 set_seg_override(ctxt, VCPU_SREG_DS);
4469 ctxt->memop.addr.mem.seg = seg_override(ctxt);
4471 if (ctxt->memop.type == OP_MEM && ctxt->ad_bytes != 8)
4472 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
4475 * Decode and fetch the source operand: register, memory
4478 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
4479 if (rc != X86EMUL_CONTINUE)
4483 * Decode and fetch the second source operand: register, memory
4486 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
4487 if (rc != X86EMUL_CONTINUE)
4490 /* Decode and fetch the destination operand: register or memory. */
4491 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
4494 if (ctxt->memopp && ctxt->memopp->type == OP_MEM && ctxt->rip_relative)
4495 ctxt->memopp->addr.mem.ea += ctxt->_eip;
4497 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
4500 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
4502 return ctxt->d & PageTable;
4505 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
4507 /* The second termination condition only applies for REPE
4508 * and REPNE. Test if the repeat string operation prefix is
4509 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
4510 * corresponding termination condition according to:
4511 * - if REPE/REPZ and ZF = 0 then done
4512 * - if REPNE/REPNZ and ZF = 1 then done
4514 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
4515 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
4516 && (((ctxt->rep_prefix == REPE_PREFIX) &&
4517 ((ctxt->eflags & EFLG_ZF) == 0))
4518 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
4519 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
4525 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
4529 ctxt->ops->get_fpu(ctxt);
4530 asm volatile("1: fwait \n\t"
4532 ".pushsection .fixup,\"ax\" \n\t"
4534 "movb $1, %[fault] \n\t"
4537 _ASM_EXTABLE(1b, 3b)
4538 : [fault]"+qm"(fault));
4539 ctxt->ops->put_fpu(ctxt);
4541 if (unlikely(fault))
4542 return emulate_exception(ctxt, MF_VECTOR, 0, false);
4544 return X86EMUL_CONTINUE;
4547 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
4550 if (op->type == OP_MM)
4551 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
4554 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
4556 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
4557 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
4558 asm("push %[flags]; popf; call *%[fastop]; pushf; pop %[flags]\n"
4559 : "+a"(ctxt->dst.val), "+b"(ctxt->src.val), [flags]"+D"(flags)
4560 : "c"(ctxt->src2.val), [fastop]"S"(fop));
4561 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
4562 return X86EMUL_CONTINUE;
4565 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
4567 const struct x86_emulate_ops *ops = ctxt->ops;
4568 int rc = X86EMUL_CONTINUE;
4569 int saved_dst_type = ctxt->dst.type;
4571 ctxt->mem_read.pos = 0;
4573 if (ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) {
4574 rc = emulate_ud(ctxt);
4578 /* LOCK prefix is allowed only with some instructions */
4579 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
4580 rc = emulate_ud(ctxt);
4584 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
4585 rc = emulate_ud(ctxt);
4589 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
4590 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
4591 rc = emulate_ud(ctxt);
4595 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
4596 rc = emulate_nm(ctxt);
4600 if (ctxt->d & Mmx) {
4601 rc = flush_pending_x87_faults(ctxt);
4602 if (rc != X86EMUL_CONTINUE)
4605 * Now that we know the fpu is exception safe, we can fetch
4608 fetch_possible_mmx_operand(ctxt, &ctxt->src);
4609 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
4610 if (!(ctxt->d & Mov))
4611 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
4614 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4615 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4616 X86_ICPT_PRE_EXCEPT);
4617 if (rc != X86EMUL_CONTINUE)
4621 /* Privileged instruction can be executed only in CPL=0 */
4622 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
4623 rc = emulate_gp(ctxt, 0);
4627 /* Instruction can only be executed in protected mode */
4628 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
4629 rc = emulate_ud(ctxt);
4633 /* Do instruction specific permission checks */
4634 if (ctxt->check_perm) {
4635 rc = ctxt->check_perm(ctxt);
4636 if (rc != X86EMUL_CONTINUE)
4640 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4641 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4642 X86_ICPT_POST_EXCEPT);
4643 if (rc != X86EMUL_CONTINUE)
4647 if (ctxt->rep_prefix && (ctxt->d & String)) {
4648 /* All REP prefixes have the same first termination condition */
4649 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
4650 ctxt->eip = ctxt->_eip;
4655 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
4656 rc = segmented_read(ctxt, ctxt->src.addr.mem,
4657 ctxt->src.valptr, ctxt->src.bytes);
4658 if (rc != X86EMUL_CONTINUE)
4660 ctxt->src.orig_val64 = ctxt->src.val64;
4663 if (ctxt->src2.type == OP_MEM) {
4664 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
4665 &ctxt->src2.val, ctxt->src2.bytes);
4666 if (rc != X86EMUL_CONTINUE)
4670 if ((ctxt->d & DstMask) == ImplicitOps)
4674 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
4675 /* optimisation - avoid slow emulated read if Mov */
4676 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
4677 &ctxt->dst.val, ctxt->dst.bytes);
4678 if (rc != X86EMUL_CONTINUE)
4681 ctxt->dst.orig_val = ctxt->dst.val;
4685 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4686 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4687 X86_ICPT_POST_MEMACCESS);
4688 if (rc != X86EMUL_CONTINUE)
4692 if (ctxt->execute) {
4693 if (ctxt->d & Fastop) {
4694 void (*fop)(struct fastop *) = (void *)ctxt->execute;
4695 rc = fastop(ctxt, fop);
4696 if (rc != X86EMUL_CONTINUE)
4700 rc = ctxt->execute(ctxt);
4701 if (rc != X86EMUL_CONTINUE)
4710 case 0x40 ... 0x47: /* inc r16/r32 */
4711 emulate_1op(ctxt, "inc");
4713 case 0x48 ... 0x4f: /* dec r16/r32 */
4714 emulate_1op(ctxt, "dec");
4716 case 0x63: /* movsxd */
4717 if (ctxt->mode != X86EMUL_MODE_PROT64)
4718 goto cannot_emulate;
4719 ctxt->dst.val = (s32) ctxt->src.val;
4721 case 0x70 ... 0x7f: /* jcc (short) */
4722 if (test_cc(ctxt->b, ctxt->eflags))
4723 jmp_rel(ctxt, ctxt->src.val);
4725 case 0x8d: /* lea r16/r32, m */
4726 ctxt->dst.val = ctxt->src.addr.mem.ea;
4728 case 0x90 ... 0x97: /* nop / xchg reg, rax */
4729 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
4733 case 0x98: /* cbw/cwde/cdqe */
4734 switch (ctxt->op_bytes) {
4735 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
4736 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
4737 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
4743 case 0xcc: /* int3 */
4744 rc = emulate_int(ctxt, 3);
4746 case 0xcd: /* int n */
4747 rc = emulate_int(ctxt, ctxt->src.val);
4749 case 0xce: /* into */
4750 if (ctxt->eflags & EFLG_OF)
4751 rc = emulate_int(ctxt, 4);
4753 case 0xd0 ... 0xd1: /* Grp2 */
4756 case 0xd2 ... 0xd3: /* Grp2 */
4757 ctxt->src.val = reg_read(ctxt, VCPU_REGS_RCX);
4760 case 0xe9: /* jmp rel */
4761 case 0xeb: /* jmp rel short */
4762 jmp_rel(ctxt, ctxt->src.val);
4763 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4765 case 0xf4: /* hlt */
4766 ctxt->ops->halt(ctxt);
4768 case 0xf5: /* cmc */
4769 /* complement carry flag from eflags reg */
4770 ctxt->eflags ^= EFLG_CF;
4772 case 0xf8: /* clc */
4773 ctxt->eflags &= ~EFLG_CF;
4775 case 0xf9: /* stc */
4776 ctxt->eflags |= EFLG_CF;
4778 case 0xfc: /* cld */
4779 ctxt->eflags &= ~EFLG_DF;
4781 case 0xfd: /* std */
4782 ctxt->eflags |= EFLG_DF;
4785 goto cannot_emulate;
4788 if (rc != X86EMUL_CONTINUE)
4792 rc = writeback(ctxt);
4793 if (rc != X86EMUL_CONTINUE)
4797 * restore dst type in case the decoding will be reused
4798 * (happens for string instruction )
4800 ctxt->dst.type = saved_dst_type;
4802 if ((ctxt->d & SrcMask) == SrcSI)
4803 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
4805 if ((ctxt->d & DstMask) == DstDI)
4806 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
4808 if (ctxt->rep_prefix && (ctxt->d & String)) {
4810 struct read_cache *r = &ctxt->io_read;
4811 if ((ctxt->d & SrcMask) == SrcSI)
4812 count = ctxt->src.count;
4814 count = ctxt->dst.count;
4815 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX),
4818 if (!string_insn_completed(ctxt)) {
4820 * Re-enter guest when pio read ahead buffer is empty
4821 * or, if it is not used, after each 1024 iteration.
4823 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
4824 (r->end == 0 || r->end != r->pos)) {
4826 * Reset read cache. Usually happens before
4827 * decode, but since instruction is restarted
4828 * we have to do it here.
4830 ctxt->mem_read.end = 0;
4831 writeback_registers(ctxt);
4832 return EMULATION_RESTART;
4834 goto done; /* skip rip writeback */
4838 ctxt->eip = ctxt->_eip;
4841 if (rc == X86EMUL_PROPAGATE_FAULT)
4842 ctxt->have_exception = true;
4843 if (rc == X86EMUL_INTERCEPTED)
4844 return EMULATION_INTERCEPTED;
4846 if (rc == X86EMUL_CONTINUE)
4847 writeback_registers(ctxt);
4849 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4853 case 0x09: /* wbinvd */
4854 (ctxt->ops->wbinvd)(ctxt);
4856 case 0x08: /* invd */
4857 case 0x0d: /* GrpP (prefetch) */
4858 case 0x18: /* Grp16 (prefetch/nop) */
4860 case 0x20: /* mov cr, reg */
4861 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
4863 case 0x21: /* mov from dr to reg */
4864 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
4866 case 0x40 ... 0x4f: /* cmov */
4867 ctxt->dst.val = ctxt->dst.orig_val = ctxt->src.val;
4868 if (!test_cc(ctxt->b, ctxt->eflags))
4869 ctxt->dst.type = OP_NONE; /* no writeback */
4871 case 0x80 ... 0x8f: /* jnz rel, etc*/
4872 if (test_cc(ctxt->b, ctxt->eflags))
4873 jmp_rel(ctxt, ctxt->src.val);
4875 case 0x90 ... 0x9f: /* setcc r/m8 */
4876 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
4878 case 0xa4: /* shld imm8, r, r/m */
4879 case 0xa5: /* shld cl, r, r/m */
4880 emulate_2op_cl(ctxt, "shld");
4882 case 0xac: /* shrd imm8, r, r/m */
4883 case 0xad: /* shrd cl, r, r/m */
4884 emulate_2op_cl(ctxt, "shrd");
4886 case 0xae: /* clflush */
4888 case 0xb6 ... 0xb7: /* movzx */
4889 ctxt->dst.bytes = ctxt->op_bytes;
4890 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
4891 : (u16) ctxt->src.val;
4893 case 0xbe ... 0xbf: /* movsx */
4894 ctxt->dst.bytes = ctxt->op_bytes;
4895 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
4896 (s16) ctxt->src.val;
4898 case 0xc0 ... 0xc1: /* xadd */
4899 emulate_2op_SrcV(ctxt, "add");
4900 /* Write back the register source. */
4901 ctxt->src.val = ctxt->dst.orig_val;
4902 write_register_operand(&ctxt->src);
4904 case 0xc3: /* movnti */
4905 ctxt->dst.bytes = ctxt->op_bytes;
4906 ctxt->dst.val = (ctxt->op_bytes == 4) ? (u32) ctxt->src.val :
4907 (u64) ctxt->src.val;
4910 goto cannot_emulate;
4913 if (rc != X86EMUL_CONTINUE)
4919 return EMULATION_FAILED;
4922 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
4924 invalidate_registers(ctxt);
4927 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
4929 writeback_registers(ctxt);