1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
35 #define OpImplicit 1ull /* No generic decode */
36 #define OpReg 2ull /* Register */
37 #define OpMem 3ull /* Memory */
38 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
39 #define OpDI 5ull /* ES:DI/EDI/RDI */
40 #define OpMem64 6ull /* Memory, 64-bit */
41 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
42 #define OpDX 8ull /* DX register */
43 #define OpCL 9ull /* CL register (for shifts) */
44 #define OpImmByte 10ull /* 8-bit sign extended immediate */
45 #define OpOne 11ull /* Implied 1 */
46 #define OpImm 12ull /* Sign extended immediate */
47 #define OpMem16 13ull /* Memory operand (16-bit). */
48 #define OpMem32 14ull /* Memory operand (32-bit). */
49 #define OpImmU 15ull /* Immediate operand, zero extended */
50 #define OpSI 16ull /* SI/ESI/RSI */
51 #define OpImmFAddr 17ull /* Immediate far address */
52 #define OpMemFAddr 18ull /* Far address in memory */
53 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
54 #define OpES 20ull /* ES */
55 #define OpCS 21ull /* CS */
56 #define OpSS 22ull /* SS */
57 #define OpDS 23ull /* DS */
58 #define OpFS 24ull /* FS */
59 #define OpGS 25ull /* GS */
60 #define OpMem8 26ull /* 8-bit zero extended memory operand */
62 #define OpBits 5 /* Width of operand field */
63 #define OpMask ((1ull << OpBits) - 1)
66 * Opcode effective-address decode tables.
67 * Note that we only emulate instructions that have at least one memory
68 * operand (excluding implicit stack references). We assume that stack
69 * references and instruction fetches will never occur in special memory
70 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
74 /* Operand sizes: 8-bit operands or specified/overridden size. */
75 #define ByteOp (1<<0) /* 8-bit operands. */
76 /* Destination operand type. */
78 #define ImplicitOps (OpImplicit << DstShift)
79 #define DstReg (OpReg << DstShift)
80 #define DstMem (OpMem << DstShift)
81 #define DstAcc (OpAcc << DstShift)
82 #define DstDI (OpDI << DstShift)
83 #define DstMem64 (OpMem64 << DstShift)
84 #define DstImmUByte (OpImmUByte << DstShift)
85 #define DstDX (OpDX << DstShift)
86 #define DstMask (OpMask << DstShift)
87 /* Source operand type. */
89 #define SrcNone (OpNone << SrcShift)
90 #define SrcReg (OpReg << SrcShift)
91 #define SrcMem (OpMem << SrcShift)
92 #define SrcMem16 (OpMem16 << SrcShift)
93 #define SrcMem32 (OpMem32 << SrcShift)
94 #define SrcImm (OpImm << SrcShift)
95 #define SrcImmByte (OpImmByte << SrcShift)
96 #define SrcOne (OpOne << SrcShift)
97 #define SrcImmUByte (OpImmUByte << SrcShift)
98 #define SrcImmU (OpImmU << SrcShift)
99 #define SrcSI (OpSI << SrcShift)
100 #define SrcImmFAddr (OpImmFAddr << SrcShift)
101 #define SrcMemFAddr (OpMemFAddr << SrcShift)
102 #define SrcAcc (OpAcc << SrcShift)
103 #define SrcImmU16 (OpImmU16 << SrcShift)
104 #define SrcDX (OpDX << SrcShift)
105 #define SrcMem8 (OpMem8 << SrcShift)
106 #define SrcMask (OpMask << SrcShift)
107 #define BitOp (1<<11)
108 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
109 #define String (1<<13) /* String instruction (rep capable) */
110 #define Stack (1<<14) /* Stack instruction (push/pop) */
111 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
112 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
113 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
114 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
115 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
116 #define Sse (1<<18) /* SSE Vector instruction */
117 /* Generic ModRM decode. */
118 #define ModRM (1<<19)
119 /* Destination is only written; never read. */
122 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
123 #define VendorSpecific (1<<22) /* Vendor specific instruction */
124 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
125 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
126 #define Undefined (1<<25) /* No Such Instruction */
127 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
128 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
130 #define PageTable (1 << 29) /* instruction used to write page table */
131 /* Source 2 operand type */
132 #define Src2Shift (30)
133 #define Src2None (OpNone << Src2Shift)
134 #define Src2CL (OpCL << Src2Shift)
135 #define Src2ImmByte (OpImmByte << Src2Shift)
136 #define Src2One (OpOne << Src2Shift)
137 #define Src2Imm (OpImm << Src2Shift)
138 #define Src2ES (OpES << Src2Shift)
139 #define Src2CS (OpCS << Src2Shift)
140 #define Src2SS (OpSS << Src2Shift)
141 #define Src2DS (OpDS << Src2Shift)
142 #define Src2FS (OpFS << Src2Shift)
143 #define Src2GS (OpGS << Src2Shift)
144 #define Src2Mask (OpMask << Src2Shift)
145 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
146 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
147 #define Unaligned ((u64)1 << 42) /* Explicitly unaligned (e.g. MOVDQU) */
148 #define Avx ((u64)1 << 43) /* Advanced Vector Extensions */
150 #define X2(x...) x, x
151 #define X3(x...) X2(x), x
152 #define X4(x...) X2(x), X2(x)
153 #define X5(x...) X4(x), x
154 #define X6(x...) X4(x), X2(x)
155 #define X7(x...) X4(x), X3(x)
156 #define X8(x...) X4(x), X4(x)
157 #define X16(x...) X8(x), X8(x)
163 int (*execute)(struct x86_emulate_ctxt *ctxt);
164 const struct opcode *group;
165 const struct group_dual *gdual;
166 const struct gprefix *gprefix;
168 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
172 struct opcode mod012[8];
173 struct opcode mod3[8];
177 struct opcode pfx_no;
178 struct opcode pfx_66;
179 struct opcode pfx_f2;
180 struct opcode pfx_f3;
183 /* EFLAGS bit definitions. */
184 #define EFLG_ID (1<<21)
185 #define EFLG_VIP (1<<20)
186 #define EFLG_VIF (1<<19)
187 #define EFLG_AC (1<<18)
188 #define EFLG_VM (1<<17)
189 #define EFLG_RF (1<<16)
190 #define EFLG_IOPL (3<<12)
191 #define EFLG_NT (1<<14)
192 #define EFLG_OF (1<<11)
193 #define EFLG_DF (1<<10)
194 #define EFLG_IF (1<<9)
195 #define EFLG_TF (1<<8)
196 #define EFLG_SF (1<<7)
197 #define EFLG_ZF (1<<6)
198 #define EFLG_AF (1<<4)
199 #define EFLG_PF (1<<2)
200 #define EFLG_CF (1<<0)
202 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
203 #define EFLG_RESERVED_ONE_MASK 2
205 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
207 if (!(ctxt->regs_valid & (1 << nr))) {
208 ctxt->regs_valid |= 1 << nr;
209 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
211 return ctxt->_regs[nr];
214 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
216 ctxt->regs_valid |= 1 << nr;
217 ctxt->regs_dirty |= 1 << nr;
218 return &ctxt->_regs[nr];
221 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
224 return reg_write(ctxt, nr);
227 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
231 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
232 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
235 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
237 ctxt->regs_dirty = 0;
238 ctxt->regs_valid = 0;
242 * Instruction emulation:
243 * Most instructions are emulated directly via a fragment of inline assembly
244 * code. This allows us to save/restore EFLAGS and thus very easily pick up
245 * any modified flags.
248 #if defined(CONFIG_X86_64)
249 #define _LO32 "k" /* force 32-bit operand */
250 #define _STK "%%rsp" /* stack pointer */
251 #elif defined(__i386__)
252 #define _LO32 "" /* force 32-bit operand */
253 #define _STK "%%esp" /* stack pointer */
257 * These EFLAGS bits are restored from saved value during emulation, and
258 * any changes are written back to the saved value after emulation.
260 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
262 /* Before executing instruction: restore necessary bits in EFLAGS. */
263 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
264 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
265 "movl %"_sav",%"_LO32 _tmp"; " \
268 "movl %"_msk",%"_LO32 _tmp"; " \
269 "andl %"_LO32 _tmp",("_STK"); " \
271 "notl %"_LO32 _tmp"; " \
272 "andl %"_LO32 _tmp",("_STK"); " \
273 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
275 "orl %"_LO32 _tmp",("_STK"); " \
279 /* After executing instruction: write-back necessary bits in EFLAGS. */
280 #define _POST_EFLAGS(_sav, _msk, _tmp) \
281 /* _sav |= EFLAGS & _msk; */ \
284 "andl %"_msk",%"_LO32 _tmp"; " \
285 "orl %"_LO32 _tmp",%"_sav"; "
293 #define ____emulate_2op(ctxt, _op, _x, _y, _suffix, _dsttype) \
295 __asm__ __volatile__ ( \
296 _PRE_EFLAGS("0", "4", "2") \
297 _op _suffix " %"_x"3,%1; " \
298 _POST_EFLAGS("0", "4", "2") \
299 : "=m" ((ctxt)->eflags), \
300 "+q" (*(_dsttype*)&(ctxt)->dst.val), \
302 : _y ((ctxt)->src.val), "i" (EFLAGS_MASK)); \
306 /* Raw emulation: instruction has two explicit operands. */
307 #define __emulate_2op_nobyte(ctxt,_op,_wx,_wy,_lx,_ly,_qx,_qy) \
309 unsigned long _tmp; \
311 switch ((ctxt)->dst.bytes) { \
313 ____emulate_2op(ctxt,_op,_wx,_wy,"w",u16); \
316 ____emulate_2op(ctxt,_op,_lx,_ly,"l",u32); \
319 ON64(____emulate_2op(ctxt,_op,_qx,_qy,"q",u64)); \
324 #define __emulate_2op(ctxt,_op,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
326 unsigned long _tmp; \
327 switch ((ctxt)->dst.bytes) { \
329 ____emulate_2op(ctxt,_op,_bx,_by,"b",u8); \
332 __emulate_2op_nobyte(ctxt, _op, \
333 _wx, _wy, _lx, _ly, _qx, _qy); \
338 /* Source operand is byte-sized and may be restricted to just %cl. */
339 #define emulate_2op_SrcB(ctxt, _op) \
340 __emulate_2op(ctxt, _op, "b", "c", "b", "c", "b", "c", "b", "c")
342 /* Source operand is byte, word, long or quad sized. */
343 #define emulate_2op_SrcV(ctxt, _op) \
344 __emulate_2op(ctxt, _op, "b", "q", "w", "r", _LO32, "r", "", "r")
346 /* Source operand is word, long or quad sized. */
347 #define emulate_2op_SrcV_nobyte(ctxt, _op) \
348 __emulate_2op_nobyte(ctxt, _op, "w", "r", _LO32, "r", "", "r")
350 /* Instruction has three operands and one operand is stored in ECX register */
351 #define __emulate_2op_cl(ctxt, _op, _suffix, _type) \
353 unsigned long _tmp; \
354 _type _clv = (ctxt)->src2.val; \
355 _type _srcv = (ctxt)->src.val; \
356 _type _dstv = (ctxt)->dst.val; \
358 __asm__ __volatile__ ( \
359 _PRE_EFLAGS("0", "5", "2") \
360 _op _suffix " %4,%1 \n" \
361 _POST_EFLAGS("0", "5", "2") \
362 : "=m" ((ctxt)->eflags), "+r" (_dstv), "=&r" (_tmp) \
363 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
366 (ctxt)->src2.val = (unsigned long) _clv; \
367 (ctxt)->src2.val = (unsigned long) _srcv; \
368 (ctxt)->dst.val = (unsigned long) _dstv; \
371 #define emulate_2op_cl(ctxt, _op) \
373 switch ((ctxt)->dst.bytes) { \
375 __emulate_2op_cl(ctxt, _op, "w", u16); \
378 __emulate_2op_cl(ctxt, _op, "l", u32); \
381 ON64(__emulate_2op_cl(ctxt, _op, "q", ulong)); \
386 #define __emulate_1op(ctxt, _op, _suffix) \
388 unsigned long _tmp; \
390 __asm__ __volatile__ ( \
391 _PRE_EFLAGS("0", "3", "2") \
392 _op _suffix " %1; " \
393 _POST_EFLAGS("0", "3", "2") \
394 : "=m" ((ctxt)->eflags), "+m" ((ctxt)->dst.val), \
396 : "i" (EFLAGS_MASK)); \
399 /* Instruction has only one explicit operand (no source operand). */
400 #define emulate_1op(ctxt, _op) \
402 switch ((ctxt)->dst.bytes) { \
403 case 1: __emulate_1op(ctxt, _op, "b"); break; \
404 case 2: __emulate_1op(ctxt, _op, "w"); break; \
405 case 4: __emulate_1op(ctxt, _op, "l"); break; \
406 case 8: ON64(__emulate_1op(ctxt, _op, "q")); break; \
410 #define __emulate_1op_rax_rdx(ctxt, _op, _suffix, _ex) \
412 unsigned long _tmp; \
413 ulong *rax = reg_rmw((ctxt), VCPU_REGS_RAX); \
414 ulong *rdx = reg_rmw((ctxt), VCPU_REGS_RDX); \
416 __asm__ __volatile__ ( \
417 _PRE_EFLAGS("0", "5", "1") \
419 _op _suffix " %6; " \
421 _POST_EFLAGS("0", "5", "1") \
422 ".pushsection .fixup,\"ax\" \n\t" \
423 "3: movb $1, %4 \n\t" \
426 _ASM_EXTABLE(1b, 3b) \
427 : "=m" ((ctxt)->eflags), "=&r" (_tmp), \
428 "+a" (*rax), "+d" (*rdx), "+qm"(_ex) \
429 : "i" (EFLAGS_MASK), "m" ((ctxt)->src.val), \
430 "a" (*rax), "d" (*rdx)); \
433 /* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
434 #define emulate_1op_rax_rdx(ctxt, _op, _ex) \
436 switch((ctxt)->src.bytes) { \
438 __emulate_1op_rax_rdx(ctxt, _op, "b", _ex); \
441 __emulate_1op_rax_rdx(ctxt, _op, "w", _ex); \
444 __emulate_1op_rax_rdx(ctxt, _op, "l", _ex); \
447 __emulate_1op_rax_rdx(ctxt, _op, "q", _ex)); \
452 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
453 enum x86_intercept intercept,
454 enum x86_intercept_stage stage)
456 struct x86_instruction_info info = {
457 .intercept = intercept,
458 .rep_prefix = ctxt->rep_prefix,
459 .modrm_mod = ctxt->modrm_mod,
460 .modrm_reg = ctxt->modrm_reg,
461 .modrm_rm = ctxt->modrm_rm,
462 .src_val = ctxt->src.val64,
463 .src_bytes = ctxt->src.bytes,
464 .dst_bytes = ctxt->dst.bytes,
465 .ad_bytes = ctxt->ad_bytes,
466 .next_rip = ctxt->eip,
469 return ctxt->ops->intercept(ctxt, &info, stage);
472 static void assign_masked(ulong *dest, ulong src, ulong mask)
474 *dest = (*dest & ~mask) | (src & mask);
477 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
479 return (1UL << (ctxt->ad_bytes << 3)) - 1;
482 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
485 struct desc_struct ss;
487 if (ctxt->mode == X86EMUL_MODE_PROT64)
489 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
490 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
493 static int stack_size(struct x86_emulate_ctxt *ctxt)
495 return (__fls(stack_mask(ctxt)) + 1) >> 3;
498 /* Access/update address held in a register, based on addressing mode. */
499 static inline unsigned long
500 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
502 if (ctxt->ad_bytes == sizeof(unsigned long))
505 return reg & ad_mask(ctxt);
508 static inline unsigned long
509 register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
511 return address_mask(ctxt, reg);
514 static void masked_increment(ulong *reg, ulong mask, int inc)
516 assign_masked(reg, *reg + inc, mask);
520 register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
524 if (ctxt->ad_bytes == sizeof(unsigned long))
527 mask = ad_mask(ctxt);
528 masked_increment(reg, mask, inc);
531 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
533 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
536 static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
538 register_address_increment(ctxt, &ctxt->_eip, rel);
541 static u32 desc_limit_scaled(struct desc_struct *desc)
543 u32 limit = get_desc_limit(desc);
545 return desc->g ? (limit << 12) | 0xfff : limit;
548 static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
550 ctxt->has_seg_override = true;
551 ctxt->seg_override = seg;
554 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
556 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
559 return ctxt->ops->get_cached_segment_base(ctxt, seg);
562 static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
564 if (!ctxt->has_seg_override)
567 return ctxt->seg_override;
570 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
571 u32 error, bool valid)
573 ctxt->exception.vector = vec;
574 ctxt->exception.error_code = error;
575 ctxt->exception.error_code_valid = valid;
576 return X86EMUL_PROPAGATE_FAULT;
579 static int emulate_db(struct x86_emulate_ctxt *ctxt)
581 return emulate_exception(ctxt, DB_VECTOR, 0, false);
584 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
586 return emulate_exception(ctxt, GP_VECTOR, err, true);
589 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
591 return emulate_exception(ctxt, SS_VECTOR, err, true);
594 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
596 return emulate_exception(ctxt, UD_VECTOR, 0, false);
599 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
601 return emulate_exception(ctxt, TS_VECTOR, err, true);
604 static int emulate_de(struct x86_emulate_ctxt *ctxt)
606 return emulate_exception(ctxt, DE_VECTOR, 0, false);
609 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
611 return emulate_exception(ctxt, NM_VECTOR, 0, false);
614 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
617 struct desc_struct desc;
619 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
623 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
628 struct desc_struct desc;
630 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
631 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
635 * x86 defines three classes of vector instructions: explicitly
636 * aligned, explicitly unaligned, and the rest, which change behaviour
637 * depending on whether they're AVX encoded or not.
639 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
640 * subject to the same check.
642 static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
644 if (likely(size < 16))
647 if (ctxt->d & Aligned)
649 else if (ctxt->d & Unaligned)
651 else if (ctxt->d & Avx)
657 static int __linearize(struct x86_emulate_ctxt *ctxt,
658 struct segmented_address addr,
659 unsigned size, bool write, bool fetch,
662 struct desc_struct desc;
669 la = seg_base(ctxt, addr.seg) + addr.ea;
670 switch (ctxt->mode) {
671 case X86EMUL_MODE_PROT64:
672 if (((signed long)la << 16) >> 16 != la)
673 return emulate_gp(ctxt, 0);
676 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
680 /* code segment or read-only data segment */
681 if (((desc.type & 8) || !(desc.type & 2)) && write)
683 /* unreadable code segment */
684 if (!fetch && (desc.type & 8) && !(desc.type & 2))
686 lim = desc_limit_scaled(&desc);
687 if ((desc.type & 8) || !(desc.type & 4)) {
688 /* expand-up segment */
689 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
692 /* expand-down segment */
693 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
695 lim = desc.d ? 0xffffffff : 0xffff;
696 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
699 cpl = ctxt->ops->cpl(ctxt);
700 if (ctxt->mode == X86EMUL_MODE_REAL)
705 if (!(desc.type & 8)) {
709 } else if ((desc.type & 8) && !(desc.type & 4)) {
710 /* nonconforming code segment */
713 } else if ((desc.type & 8) && (desc.type & 4)) {
714 /* conforming code segment */
720 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
722 if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
723 return emulate_gp(ctxt, 0);
725 return X86EMUL_CONTINUE;
727 if (addr.seg == VCPU_SREG_SS)
728 return emulate_ss(ctxt, sel);
730 return emulate_gp(ctxt, sel);
733 static int linearize(struct x86_emulate_ctxt *ctxt,
734 struct segmented_address addr,
735 unsigned size, bool write,
738 return __linearize(ctxt, addr, size, write, false, linear);
742 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
743 struct segmented_address addr,
750 rc = linearize(ctxt, addr, size, false, &linear);
751 if (rc != X86EMUL_CONTINUE)
753 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
757 * Fetch the next byte of the instruction being emulated which is pointed to
758 * by ctxt->_eip, then increment ctxt->_eip.
760 * Also prefetch the remaining bytes of the instruction without crossing page
761 * boundary if they are not in fetch_cache yet.
763 static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
765 struct fetch_cache *fc = &ctxt->fetch;
769 if (ctxt->_eip == fc->end) {
770 unsigned long linear;
771 struct segmented_address addr = { .seg = VCPU_SREG_CS,
773 cur_size = fc->end - fc->start;
774 size = min(15UL - cur_size,
775 PAGE_SIZE - offset_in_page(ctxt->_eip));
776 rc = __linearize(ctxt, addr, size, false, true, &linear);
777 if (unlikely(rc != X86EMUL_CONTINUE))
779 rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
780 size, &ctxt->exception);
781 if (unlikely(rc != X86EMUL_CONTINUE))
785 *dest = fc->data[ctxt->_eip - fc->start];
787 return X86EMUL_CONTINUE;
790 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
791 void *dest, unsigned size)
795 /* x86 instructions are limited to 15 bytes. */
796 if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
797 return X86EMUL_UNHANDLEABLE;
799 rc = do_insn_fetch_byte(ctxt, dest++);
800 if (rc != X86EMUL_CONTINUE)
803 return X86EMUL_CONTINUE;
806 /* Fetch next part of the instruction being emulated. */
807 #define insn_fetch(_type, _ctxt) \
808 ({ unsigned long _x; \
809 rc = do_insn_fetch(_ctxt, &_x, sizeof(_type)); \
810 if (rc != X86EMUL_CONTINUE) \
815 #define insn_fetch_arr(_arr, _size, _ctxt) \
816 ({ rc = do_insn_fetch(_ctxt, _arr, (_size)); \
817 if (rc != X86EMUL_CONTINUE) \
822 * Given the 'reg' portion of a ModRM byte, and a register block, return a
823 * pointer into the block that addresses the relevant register.
824 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
826 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
831 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
832 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
834 p = reg_rmw(ctxt, modrm_reg);
838 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
839 struct segmented_address addr,
840 u16 *size, unsigned long *address, int op_bytes)
847 rc = segmented_read_std(ctxt, addr, size, 2);
848 if (rc != X86EMUL_CONTINUE)
851 rc = segmented_read_std(ctxt, addr, address, op_bytes);
855 static int test_cc(unsigned int condition, unsigned int flags)
859 switch ((condition & 15) >> 1) {
861 rc |= (flags & EFLG_OF);
863 case 1: /* b/c/nae */
864 rc |= (flags & EFLG_CF);
867 rc |= (flags & EFLG_ZF);
870 rc |= (flags & (EFLG_CF|EFLG_ZF));
873 rc |= (flags & EFLG_SF);
876 rc |= (flags & EFLG_PF);
879 rc |= (flags & EFLG_ZF);
882 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
886 /* Odd condition identifiers (lsb == 1) have inverted sense. */
887 return (!!rc ^ (condition & 1));
890 static void fetch_register_operand(struct operand *op)
894 op->val = *(u8 *)op->addr.reg;
897 op->val = *(u16 *)op->addr.reg;
900 op->val = *(u32 *)op->addr.reg;
903 op->val = *(u64 *)op->addr.reg;
908 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
910 ctxt->ops->get_fpu(ctxt);
912 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
913 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
914 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
915 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
916 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
917 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
918 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
919 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
921 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
922 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
923 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
924 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
925 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
926 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
927 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
928 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
932 ctxt->ops->put_fpu(ctxt);
935 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
938 ctxt->ops->get_fpu(ctxt);
940 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
941 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
942 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
943 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
944 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
945 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
946 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
947 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
949 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
950 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
951 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
952 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
953 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
954 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
955 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
956 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
960 ctxt->ops->put_fpu(ctxt);
963 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
965 ctxt->ops->get_fpu(ctxt);
967 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
968 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
969 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
970 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
971 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
972 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
973 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
974 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
977 ctxt->ops->put_fpu(ctxt);
980 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
982 ctxt->ops->get_fpu(ctxt);
984 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
985 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
986 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
987 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
988 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
989 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
990 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
991 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
994 ctxt->ops->put_fpu(ctxt);
997 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1000 unsigned reg = ctxt->modrm_reg;
1001 int highbyte_regs = ctxt->rex_prefix == 0;
1003 if (!(ctxt->d & ModRM))
1004 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1006 if (ctxt->d & Sse) {
1010 read_sse_reg(ctxt, &op->vec_val, reg);
1013 if (ctxt->d & Mmx) {
1022 if (ctxt->d & ByteOp) {
1023 op->addr.reg = decode_register(ctxt, reg, highbyte_regs);
1026 op->addr.reg = decode_register(ctxt, reg, 0);
1027 op->bytes = ctxt->op_bytes;
1029 fetch_register_operand(op);
1030 op->orig_val = op->val;
1033 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1035 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1036 ctxt->modrm_seg = VCPU_SREG_SS;
1039 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1043 int index_reg = 0, base_reg = 0, scale;
1044 int rc = X86EMUL_CONTINUE;
1047 if (ctxt->rex_prefix) {
1048 ctxt->modrm_reg = (ctxt->rex_prefix & 4) << 1; /* REX.R */
1049 index_reg = (ctxt->rex_prefix & 2) << 2; /* REX.X */
1050 ctxt->modrm_rm = base_reg = (ctxt->rex_prefix & 1) << 3; /* REG.B */
1053 ctxt->modrm_mod |= (ctxt->modrm & 0xc0) >> 6;
1054 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1055 ctxt->modrm_rm |= (ctxt->modrm & 0x07);
1056 ctxt->modrm_seg = VCPU_SREG_DS;
1058 if (ctxt->modrm_mod == 3) {
1060 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1061 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm, ctxt->d & ByteOp);
1062 if (ctxt->d & Sse) {
1065 op->addr.xmm = ctxt->modrm_rm;
1066 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1069 if (ctxt->d & Mmx) {
1072 op->addr.xmm = ctxt->modrm_rm & 7;
1075 fetch_register_operand(op);
1081 if (ctxt->ad_bytes == 2) {
1082 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1083 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1084 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1085 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1087 /* 16-bit ModR/M decode. */
1088 switch (ctxt->modrm_mod) {
1090 if (ctxt->modrm_rm == 6)
1091 modrm_ea += insn_fetch(u16, ctxt);
1094 modrm_ea += insn_fetch(s8, ctxt);
1097 modrm_ea += insn_fetch(u16, ctxt);
1100 switch (ctxt->modrm_rm) {
1102 modrm_ea += bx + si;
1105 modrm_ea += bx + di;
1108 modrm_ea += bp + si;
1111 modrm_ea += bp + di;
1120 if (ctxt->modrm_mod != 0)
1127 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1128 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1129 ctxt->modrm_seg = VCPU_SREG_SS;
1130 modrm_ea = (u16)modrm_ea;
1132 /* 32/64-bit ModR/M decode. */
1133 if ((ctxt->modrm_rm & 7) == 4) {
1134 sib = insn_fetch(u8, ctxt);
1135 index_reg |= (sib >> 3) & 7;
1136 base_reg |= sib & 7;
1139 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1140 modrm_ea += insn_fetch(s32, ctxt);
1142 modrm_ea += reg_read(ctxt, base_reg);
1143 adjust_modrm_seg(ctxt, base_reg);
1146 modrm_ea += reg_read(ctxt, index_reg) << scale;
1147 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1148 if (ctxt->mode == X86EMUL_MODE_PROT64)
1149 ctxt->rip_relative = 1;
1151 base_reg = ctxt->modrm_rm;
1152 modrm_ea += reg_read(ctxt, base_reg);
1153 adjust_modrm_seg(ctxt, base_reg);
1155 switch (ctxt->modrm_mod) {
1157 if (ctxt->modrm_rm == 5)
1158 modrm_ea += insn_fetch(s32, ctxt);
1161 modrm_ea += insn_fetch(s8, ctxt);
1164 modrm_ea += insn_fetch(s32, ctxt);
1168 op->addr.mem.ea = modrm_ea;
1173 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1176 int rc = X86EMUL_CONTINUE;
1179 switch (ctxt->ad_bytes) {
1181 op->addr.mem.ea = insn_fetch(u16, ctxt);
1184 op->addr.mem.ea = insn_fetch(u32, ctxt);
1187 op->addr.mem.ea = insn_fetch(u64, ctxt);
1194 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1198 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1199 mask = ~(ctxt->dst.bytes * 8 - 1);
1201 if (ctxt->src.bytes == 2)
1202 sv = (s16)ctxt->src.val & (s16)mask;
1203 else if (ctxt->src.bytes == 4)
1204 sv = (s32)ctxt->src.val & (s32)mask;
1206 ctxt->dst.addr.mem.ea += (sv >> 3);
1209 /* only subword offset */
1210 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1213 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1214 unsigned long addr, void *dest, unsigned size)
1217 struct read_cache *mc = &ctxt->mem_read;
1219 if (mc->pos < mc->end)
1222 WARN_ON((mc->end + size) >= sizeof(mc->data));
1224 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1226 if (rc != X86EMUL_CONTINUE)
1232 memcpy(dest, mc->data + mc->pos, size);
1234 return X86EMUL_CONTINUE;
1237 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1238 struct segmented_address addr,
1245 rc = linearize(ctxt, addr, size, false, &linear);
1246 if (rc != X86EMUL_CONTINUE)
1248 return read_emulated(ctxt, linear, data, size);
1251 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1252 struct segmented_address addr,
1259 rc = linearize(ctxt, addr, size, true, &linear);
1260 if (rc != X86EMUL_CONTINUE)
1262 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1266 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1267 struct segmented_address addr,
1268 const void *orig_data, const void *data,
1274 rc = linearize(ctxt, addr, size, true, &linear);
1275 if (rc != X86EMUL_CONTINUE)
1277 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1278 size, &ctxt->exception);
1281 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1282 unsigned int size, unsigned short port,
1285 struct read_cache *rc = &ctxt->io_read;
1287 if (rc->pos == rc->end) { /* refill pio read ahead */
1288 unsigned int in_page, n;
1289 unsigned int count = ctxt->rep_prefix ?
1290 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1291 in_page = (ctxt->eflags & EFLG_DF) ?
1292 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1293 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1294 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1298 rc->pos = rc->end = 0;
1299 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1304 memcpy(dest, rc->data + rc->pos, size);
1309 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1310 u16 index, struct desc_struct *desc)
1315 ctxt->ops->get_idt(ctxt, &dt);
1317 if (dt.size < index * 8 + 7)
1318 return emulate_gp(ctxt, index << 3 | 0x2);
1320 addr = dt.address + index * 8;
1321 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1325 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1326 u16 selector, struct desc_ptr *dt)
1328 const struct x86_emulate_ops *ops = ctxt->ops;
1330 if (selector & 1 << 2) {
1331 struct desc_struct desc;
1334 memset (dt, 0, sizeof *dt);
1335 if (!ops->get_segment(ctxt, &sel, &desc, NULL, VCPU_SREG_LDTR))
1338 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1339 dt->address = get_desc_base(&desc);
1341 ops->get_gdt(ctxt, dt);
1344 /* allowed just for 8 bytes segments */
1345 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1346 u16 selector, struct desc_struct *desc,
1350 u16 index = selector >> 3;
1353 get_descriptor_table_ptr(ctxt, selector, &dt);
1355 if (dt.size < index * 8 + 7)
1356 return emulate_gp(ctxt, selector & 0xfffc);
1358 *desc_addr_p = addr = dt.address + index * 8;
1359 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1363 /* allowed just for 8 bytes segments */
1364 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1365 u16 selector, struct desc_struct *desc)
1368 u16 index = selector >> 3;
1371 get_descriptor_table_ptr(ctxt, selector, &dt);
1373 if (dt.size < index * 8 + 7)
1374 return emulate_gp(ctxt, selector & 0xfffc);
1376 addr = dt.address + index * 8;
1377 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1381 /* Does not support long mode */
1382 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1383 u16 selector, int seg)
1385 struct desc_struct seg_desc, old_desc;
1387 unsigned err_vec = GP_VECTOR;
1389 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1394 memset(&seg_desc, 0, sizeof seg_desc);
1396 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1397 || ctxt->mode == X86EMUL_MODE_REAL) {
1398 /* set real mode segment descriptor */
1399 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1400 set_desc_base(&seg_desc, selector << 4);
1405 cpl = ctxt->ops->cpl(ctxt);
1407 /* NULL selector is not valid for TR, CS and SS (except for long mode) */
1408 if ((seg == VCPU_SREG_CS
1409 || (seg == VCPU_SREG_SS
1410 && (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl))
1411 || seg == VCPU_SREG_TR)
1415 /* TR should be in GDT only */
1416 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1419 if (null_selector) /* for NULL selector skip all following checks */
1422 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1423 if (ret != X86EMUL_CONTINUE)
1426 err_code = selector & 0xfffc;
1427 err_vec = GP_VECTOR;
1429 /* can't load system descriptor into segment selector */
1430 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1434 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1443 * segment is not a writable data segment or segment
1444 * selector's RPL != CPL or segment selector's RPL != CPL
1446 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1450 if (!(seg_desc.type & 8))
1453 if (seg_desc.type & 4) {
1459 if (rpl > cpl || dpl != cpl)
1462 /* CS(RPL) <- CPL */
1463 selector = (selector & 0xfffc) | cpl;
1466 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1468 old_desc = seg_desc;
1469 seg_desc.type |= 2; /* busy */
1470 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1471 sizeof(seg_desc), &ctxt->exception);
1472 if (ret != X86EMUL_CONTINUE)
1475 case VCPU_SREG_LDTR:
1476 if (seg_desc.s || seg_desc.type != 2)
1479 default: /* DS, ES, FS, or GS */
1481 * segment is not a data or readable code segment or
1482 * ((segment is a data or nonconforming code segment)
1483 * and (both RPL and CPL > DPL))
1485 if ((seg_desc.type & 0xa) == 0x8 ||
1486 (((seg_desc.type & 0xc) != 0xc) &&
1487 (rpl > dpl && cpl > dpl)))
1493 /* mark segment as accessed */
1495 ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1496 if (ret != X86EMUL_CONTINUE)
1500 ctxt->ops->set_segment(ctxt, selector, &seg_desc, 0, seg);
1501 return X86EMUL_CONTINUE;
1503 emulate_exception(ctxt, err_vec, err_code, true);
1504 return X86EMUL_PROPAGATE_FAULT;
1507 static void write_register_operand(struct operand *op)
1509 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1510 switch (op->bytes) {
1512 *(u8 *)op->addr.reg = (u8)op->val;
1515 *(u16 *)op->addr.reg = (u16)op->val;
1518 *op->addr.reg = (u32)op->val;
1519 break; /* 64b: zero-extend */
1521 *op->addr.reg = op->val;
1526 static int writeback(struct x86_emulate_ctxt *ctxt)
1530 switch (ctxt->dst.type) {
1532 write_register_operand(&ctxt->dst);
1535 if (ctxt->lock_prefix)
1536 rc = segmented_cmpxchg(ctxt,
1538 &ctxt->dst.orig_val,
1542 rc = segmented_write(ctxt,
1546 if (rc != X86EMUL_CONTINUE)
1550 write_sse_reg(ctxt, &ctxt->dst.vec_val, ctxt->dst.addr.xmm);
1553 write_mmx_reg(ctxt, &ctxt->dst.mm_val, ctxt->dst.addr.mm);
1561 return X86EMUL_CONTINUE;
1564 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1566 struct segmented_address addr;
1568 rsp_increment(ctxt, -bytes);
1569 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1570 addr.seg = VCPU_SREG_SS;
1572 return segmented_write(ctxt, addr, data, bytes);
1575 static int em_push(struct x86_emulate_ctxt *ctxt)
1577 /* Disable writeback. */
1578 ctxt->dst.type = OP_NONE;
1579 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1582 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1583 void *dest, int len)
1586 struct segmented_address addr;
1588 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1589 addr.seg = VCPU_SREG_SS;
1590 rc = segmented_read(ctxt, addr, dest, len);
1591 if (rc != X86EMUL_CONTINUE)
1594 rsp_increment(ctxt, len);
1598 static int em_pop(struct x86_emulate_ctxt *ctxt)
1600 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1603 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1604 void *dest, int len)
1607 unsigned long val, change_mask;
1608 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1609 int cpl = ctxt->ops->cpl(ctxt);
1611 rc = emulate_pop(ctxt, &val, len);
1612 if (rc != X86EMUL_CONTINUE)
1615 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1616 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1618 switch(ctxt->mode) {
1619 case X86EMUL_MODE_PROT64:
1620 case X86EMUL_MODE_PROT32:
1621 case X86EMUL_MODE_PROT16:
1623 change_mask |= EFLG_IOPL;
1625 change_mask |= EFLG_IF;
1627 case X86EMUL_MODE_VM86:
1629 return emulate_gp(ctxt, 0);
1630 change_mask |= EFLG_IF;
1632 default: /* real mode */
1633 change_mask |= (EFLG_IOPL | EFLG_IF);
1637 *(unsigned long *)dest =
1638 (ctxt->eflags & ~change_mask) | (val & change_mask);
1643 static int em_popf(struct x86_emulate_ctxt *ctxt)
1645 ctxt->dst.type = OP_REG;
1646 ctxt->dst.addr.reg = &ctxt->eflags;
1647 ctxt->dst.bytes = ctxt->op_bytes;
1648 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1651 static int em_enter(struct x86_emulate_ctxt *ctxt)
1654 unsigned frame_size = ctxt->src.val;
1655 unsigned nesting_level = ctxt->src2.val & 31;
1659 return X86EMUL_UNHANDLEABLE;
1661 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1662 rc = push(ctxt, &rbp, stack_size(ctxt));
1663 if (rc != X86EMUL_CONTINUE)
1665 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1667 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1668 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1670 return X86EMUL_CONTINUE;
1673 static int em_leave(struct x86_emulate_ctxt *ctxt)
1675 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1677 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1680 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1682 int seg = ctxt->src2.val;
1684 ctxt->src.val = get_segment_selector(ctxt, seg);
1686 return em_push(ctxt);
1689 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1691 int seg = ctxt->src2.val;
1692 unsigned long selector;
1695 rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1696 if (rc != X86EMUL_CONTINUE)
1699 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1703 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1705 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1706 int rc = X86EMUL_CONTINUE;
1707 int reg = VCPU_REGS_RAX;
1709 while (reg <= VCPU_REGS_RDI) {
1710 (reg == VCPU_REGS_RSP) ?
1711 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
1714 if (rc != X86EMUL_CONTINUE)
1723 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1725 ctxt->src.val = (unsigned long)ctxt->eflags;
1726 return em_push(ctxt);
1729 static int em_popa(struct x86_emulate_ctxt *ctxt)
1731 int rc = X86EMUL_CONTINUE;
1732 int reg = VCPU_REGS_RDI;
1734 while (reg >= VCPU_REGS_RAX) {
1735 if (reg == VCPU_REGS_RSP) {
1736 rsp_increment(ctxt, ctxt->op_bytes);
1740 rc = emulate_pop(ctxt, reg_rmw(ctxt, reg), ctxt->op_bytes);
1741 if (rc != X86EMUL_CONTINUE)
1748 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1750 const struct x86_emulate_ops *ops = ctxt->ops;
1757 /* TODO: Add limit checks */
1758 ctxt->src.val = ctxt->eflags;
1760 if (rc != X86EMUL_CONTINUE)
1763 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1765 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1767 if (rc != X86EMUL_CONTINUE)
1770 ctxt->src.val = ctxt->_eip;
1772 if (rc != X86EMUL_CONTINUE)
1775 ops->get_idt(ctxt, &dt);
1777 eip_addr = dt.address + (irq << 2);
1778 cs_addr = dt.address + (irq << 2) + 2;
1780 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1781 if (rc != X86EMUL_CONTINUE)
1784 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1785 if (rc != X86EMUL_CONTINUE)
1788 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1789 if (rc != X86EMUL_CONTINUE)
1797 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1801 invalidate_registers(ctxt);
1802 rc = __emulate_int_real(ctxt, irq);
1803 if (rc == X86EMUL_CONTINUE)
1804 writeback_registers(ctxt);
1808 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1810 switch(ctxt->mode) {
1811 case X86EMUL_MODE_REAL:
1812 return __emulate_int_real(ctxt, irq);
1813 case X86EMUL_MODE_VM86:
1814 case X86EMUL_MODE_PROT16:
1815 case X86EMUL_MODE_PROT32:
1816 case X86EMUL_MODE_PROT64:
1818 /* Protected mode interrupts unimplemented yet */
1819 return X86EMUL_UNHANDLEABLE;
1823 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
1825 int rc = X86EMUL_CONTINUE;
1826 unsigned long temp_eip = 0;
1827 unsigned long temp_eflags = 0;
1828 unsigned long cs = 0;
1829 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1830 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1831 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1832 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1834 /* TODO: Add stack limit check */
1836 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
1838 if (rc != X86EMUL_CONTINUE)
1841 if (temp_eip & ~0xffff)
1842 return emulate_gp(ctxt, 0);
1844 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1846 if (rc != X86EMUL_CONTINUE)
1849 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
1851 if (rc != X86EMUL_CONTINUE)
1854 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1856 if (rc != X86EMUL_CONTINUE)
1859 ctxt->_eip = temp_eip;
1862 if (ctxt->op_bytes == 4)
1863 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1864 else if (ctxt->op_bytes == 2) {
1865 ctxt->eflags &= ~0xffff;
1866 ctxt->eflags |= temp_eflags;
1869 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1870 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1875 static int em_iret(struct x86_emulate_ctxt *ctxt)
1877 switch(ctxt->mode) {
1878 case X86EMUL_MODE_REAL:
1879 return emulate_iret_real(ctxt);
1880 case X86EMUL_MODE_VM86:
1881 case X86EMUL_MODE_PROT16:
1882 case X86EMUL_MODE_PROT32:
1883 case X86EMUL_MODE_PROT64:
1885 /* iret from protected mode unimplemented yet */
1886 return X86EMUL_UNHANDLEABLE;
1890 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
1895 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1897 rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
1898 if (rc != X86EMUL_CONTINUE)
1902 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
1903 return X86EMUL_CONTINUE;
1906 static int em_grp2(struct x86_emulate_ctxt *ctxt)
1908 switch (ctxt->modrm_reg) {
1910 emulate_2op_SrcB(ctxt, "rol");
1913 emulate_2op_SrcB(ctxt, "ror");
1916 emulate_2op_SrcB(ctxt, "rcl");
1919 emulate_2op_SrcB(ctxt, "rcr");
1921 case 4: /* sal/shl */
1922 case 6: /* sal/shl */
1923 emulate_2op_SrcB(ctxt, "sal");
1926 emulate_2op_SrcB(ctxt, "shr");
1929 emulate_2op_SrcB(ctxt, "sar");
1932 return X86EMUL_CONTINUE;
1935 static int em_not(struct x86_emulate_ctxt *ctxt)
1937 ctxt->dst.val = ~ctxt->dst.val;
1938 return X86EMUL_CONTINUE;
1941 static int em_neg(struct x86_emulate_ctxt *ctxt)
1943 emulate_1op(ctxt, "neg");
1944 return X86EMUL_CONTINUE;
1947 static int em_mul_ex(struct x86_emulate_ctxt *ctxt)
1951 emulate_1op_rax_rdx(ctxt, "mul", ex);
1952 return X86EMUL_CONTINUE;
1955 static int em_imul_ex(struct x86_emulate_ctxt *ctxt)
1959 emulate_1op_rax_rdx(ctxt, "imul", ex);
1960 return X86EMUL_CONTINUE;
1963 static int em_div_ex(struct x86_emulate_ctxt *ctxt)
1967 emulate_1op_rax_rdx(ctxt, "div", de);
1969 return emulate_de(ctxt);
1970 return X86EMUL_CONTINUE;
1973 static int em_idiv_ex(struct x86_emulate_ctxt *ctxt)
1977 emulate_1op_rax_rdx(ctxt, "idiv", de);
1979 return emulate_de(ctxt);
1980 return X86EMUL_CONTINUE;
1983 static int em_grp45(struct x86_emulate_ctxt *ctxt)
1985 int rc = X86EMUL_CONTINUE;
1987 switch (ctxt->modrm_reg) {
1989 emulate_1op(ctxt, "inc");
1992 emulate_1op(ctxt, "dec");
1994 case 2: /* call near abs */ {
1996 old_eip = ctxt->_eip;
1997 ctxt->_eip = ctxt->src.val;
1998 ctxt->src.val = old_eip;
2002 case 4: /* jmp abs */
2003 ctxt->_eip = ctxt->src.val;
2005 case 5: /* jmp far */
2006 rc = em_jmp_far(ctxt);
2015 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2017 u64 old = ctxt->dst.orig_val64;
2019 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
2020 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2021 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2022 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2023 ctxt->eflags &= ~EFLG_ZF;
2025 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2026 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2028 ctxt->eflags |= EFLG_ZF;
2030 return X86EMUL_CONTINUE;
2033 static int em_ret(struct x86_emulate_ctxt *ctxt)
2035 ctxt->dst.type = OP_REG;
2036 ctxt->dst.addr.reg = &ctxt->_eip;
2037 ctxt->dst.bytes = ctxt->op_bytes;
2038 return em_pop(ctxt);
2041 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2046 rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
2047 if (rc != X86EMUL_CONTINUE)
2049 if (ctxt->op_bytes == 4)
2050 ctxt->_eip = (u32)ctxt->_eip;
2051 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2052 if (rc != X86EMUL_CONTINUE)
2054 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2058 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2060 /* Save real source value, then compare EAX against destination. */
2061 ctxt->src.orig_val = ctxt->src.val;
2062 ctxt->src.val = reg_read(ctxt, VCPU_REGS_RAX);
2063 emulate_2op_SrcV(ctxt, "cmp");
2065 if (ctxt->eflags & EFLG_ZF) {
2066 /* Success: write back to memory. */
2067 ctxt->dst.val = ctxt->src.orig_val;
2069 /* Failure: write the value we saw to EAX. */
2070 ctxt->dst.type = OP_REG;
2071 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2073 return X86EMUL_CONTINUE;
2076 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2078 int seg = ctxt->src2.val;
2082 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2084 rc = load_segment_descriptor(ctxt, sel, seg);
2085 if (rc != X86EMUL_CONTINUE)
2088 ctxt->dst.val = ctxt->src.val;
2093 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2094 struct desc_struct *cs, struct desc_struct *ss)
2096 cs->l = 0; /* will be adjusted later */
2097 set_desc_base(cs, 0); /* flat segment */
2098 cs->g = 1; /* 4kb granularity */
2099 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2100 cs->type = 0x0b; /* Read, Execute, Accessed */
2102 cs->dpl = 0; /* will be adjusted later */
2107 set_desc_base(ss, 0); /* flat segment */
2108 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2109 ss->g = 1; /* 4kb granularity */
2111 ss->type = 0x03; /* Read/Write, Accessed */
2112 ss->d = 1; /* 32bit stack segment */
2119 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2121 u32 eax, ebx, ecx, edx;
2124 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2125 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2126 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2127 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2130 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2132 const struct x86_emulate_ops *ops = ctxt->ops;
2133 u32 eax, ebx, ecx, edx;
2136 * syscall should always be enabled in longmode - so only become
2137 * vendor specific (cpuid) if other modes are active...
2139 if (ctxt->mode == X86EMUL_MODE_PROT64)
2144 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2146 * Intel ("GenuineIntel")
2147 * remark: Intel CPUs only support "syscall" in 64bit
2148 * longmode. Also an 64bit guest with a
2149 * 32bit compat-app running will #UD !! While this
2150 * behaviour can be fixed (by emulating) into AMD
2151 * response - CPUs of AMD can't behave like Intel.
2153 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2154 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2155 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2158 /* AMD ("AuthenticAMD") */
2159 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2160 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2161 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2164 /* AMD ("AMDisbetter!") */
2165 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2166 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2167 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2170 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2174 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2176 const struct x86_emulate_ops *ops = ctxt->ops;
2177 struct desc_struct cs, ss;
2182 /* syscall is not available in real mode */
2183 if (ctxt->mode == X86EMUL_MODE_REAL ||
2184 ctxt->mode == X86EMUL_MODE_VM86)
2185 return emulate_ud(ctxt);
2187 if (!(em_syscall_is_enabled(ctxt)))
2188 return emulate_ud(ctxt);
2190 ops->get_msr(ctxt, MSR_EFER, &efer);
2191 setup_syscalls_segments(ctxt, &cs, &ss);
2193 if (!(efer & EFER_SCE))
2194 return emulate_ud(ctxt);
2196 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2198 cs_sel = (u16)(msr_data & 0xfffc);
2199 ss_sel = (u16)(msr_data + 8);
2201 if (efer & EFER_LMA) {
2205 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2206 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2208 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2209 if (efer & EFER_LMA) {
2210 #ifdef CONFIG_X86_64
2211 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags & ~EFLG_RF;
2214 ctxt->mode == X86EMUL_MODE_PROT64 ?
2215 MSR_LSTAR : MSR_CSTAR, &msr_data);
2216 ctxt->_eip = msr_data;
2218 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2219 ctxt->eflags &= ~(msr_data | EFLG_RF);
2223 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2224 ctxt->_eip = (u32)msr_data;
2226 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2229 return X86EMUL_CONTINUE;
2232 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2234 const struct x86_emulate_ops *ops = ctxt->ops;
2235 struct desc_struct cs, ss;
2240 ops->get_msr(ctxt, MSR_EFER, &efer);
2241 /* inject #GP if in real mode */
2242 if (ctxt->mode == X86EMUL_MODE_REAL)
2243 return emulate_gp(ctxt, 0);
2246 * Not recognized on AMD in compat mode (but is recognized in legacy
2249 if ((ctxt->mode == X86EMUL_MODE_PROT32) && (efer & EFER_LMA)
2250 && !vendor_intel(ctxt))
2251 return emulate_ud(ctxt);
2253 /* XXX sysenter/sysexit have not been tested in 64bit mode.
2254 * Therefore, we inject an #UD.
2256 if (ctxt->mode == X86EMUL_MODE_PROT64)
2257 return emulate_ud(ctxt);
2259 setup_syscalls_segments(ctxt, &cs, &ss);
2261 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2262 switch (ctxt->mode) {
2263 case X86EMUL_MODE_PROT32:
2264 if ((msr_data & 0xfffc) == 0x0)
2265 return emulate_gp(ctxt, 0);
2267 case X86EMUL_MODE_PROT64:
2268 if (msr_data == 0x0)
2269 return emulate_gp(ctxt, 0);
2275 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2276 cs_sel = (u16)msr_data;
2277 cs_sel &= ~SELECTOR_RPL_MASK;
2278 ss_sel = cs_sel + 8;
2279 ss_sel &= ~SELECTOR_RPL_MASK;
2280 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
2285 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2286 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2288 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2289 ctxt->_eip = msr_data;
2291 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2292 *reg_write(ctxt, VCPU_REGS_RSP) = msr_data;
2294 return X86EMUL_CONTINUE;
2297 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2299 const struct x86_emulate_ops *ops = ctxt->ops;
2300 struct desc_struct cs, ss;
2303 u16 cs_sel = 0, ss_sel = 0;
2305 /* inject #GP if in real mode or Virtual 8086 mode */
2306 if (ctxt->mode == X86EMUL_MODE_REAL ||
2307 ctxt->mode == X86EMUL_MODE_VM86)
2308 return emulate_gp(ctxt, 0);
2310 setup_syscalls_segments(ctxt, &cs, &ss);
2312 if ((ctxt->rex_prefix & 0x8) != 0x0)
2313 usermode = X86EMUL_MODE_PROT64;
2315 usermode = X86EMUL_MODE_PROT32;
2319 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2321 case X86EMUL_MODE_PROT32:
2322 cs_sel = (u16)(msr_data + 16);
2323 if ((msr_data & 0xfffc) == 0x0)
2324 return emulate_gp(ctxt, 0);
2325 ss_sel = (u16)(msr_data + 24);
2327 case X86EMUL_MODE_PROT64:
2328 cs_sel = (u16)(msr_data + 32);
2329 if (msr_data == 0x0)
2330 return emulate_gp(ctxt, 0);
2331 ss_sel = cs_sel + 8;
2336 cs_sel |= SELECTOR_RPL_MASK;
2337 ss_sel |= SELECTOR_RPL_MASK;
2339 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2340 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2342 ctxt->_eip = reg_read(ctxt, VCPU_REGS_RDX);
2343 *reg_write(ctxt, VCPU_REGS_RSP) = reg_read(ctxt, VCPU_REGS_RCX);
2345 return X86EMUL_CONTINUE;
2348 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2351 if (ctxt->mode == X86EMUL_MODE_REAL)
2353 if (ctxt->mode == X86EMUL_MODE_VM86)
2355 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2356 return ctxt->ops->cpl(ctxt) > iopl;
2359 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2362 const struct x86_emulate_ops *ops = ctxt->ops;
2363 struct desc_struct tr_seg;
2366 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2367 unsigned mask = (1 << len) - 1;
2370 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2373 if (desc_limit_scaled(&tr_seg) < 103)
2375 base = get_desc_base(&tr_seg);
2376 #ifdef CONFIG_X86_64
2377 base |= ((u64)base3) << 32;
2379 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2380 if (r != X86EMUL_CONTINUE)
2382 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2384 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2385 if (r != X86EMUL_CONTINUE)
2387 if ((perm >> bit_idx) & mask)
2392 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2398 if (emulator_bad_iopl(ctxt))
2399 if (!emulator_io_port_access_allowed(ctxt, port, len))
2402 ctxt->perm_ok = true;
2407 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2408 struct tss_segment_16 *tss)
2410 tss->ip = ctxt->_eip;
2411 tss->flag = ctxt->eflags;
2412 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
2413 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
2414 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
2415 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
2416 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
2417 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
2418 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
2419 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2421 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2422 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2423 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2424 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2425 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2428 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2429 struct tss_segment_16 *tss)
2433 ctxt->_eip = tss->ip;
2434 ctxt->eflags = tss->flag | 2;
2435 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
2436 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
2437 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
2438 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
2439 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
2440 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
2441 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
2442 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
2445 * SDM says that segment selectors are loaded before segment
2448 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2449 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2450 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2451 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2452 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2455 * Now load segment descriptors. If fault happens at this stage
2456 * it is handled in a context of new task
2458 ret = load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR);
2459 if (ret != X86EMUL_CONTINUE)
2461 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2462 if (ret != X86EMUL_CONTINUE)
2464 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2465 if (ret != X86EMUL_CONTINUE)
2467 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2468 if (ret != X86EMUL_CONTINUE)
2470 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2471 if (ret != X86EMUL_CONTINUE)
2474 return X86EMUL_CONTINUE;
2477 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2478 u16 tss_selector, u16 old_tss_sel,
2479 ulong old_tss_base, struct desc_struct *new_desc)
2481 const struct x86_emulate_ops *ops = ctxt->ops;
2482 struct tss_segment_16 tss_seg;
2484 u32 new_tss_base = get_desc_base(new_desc);
2486 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2488 if (ret != X86EMUL_CONTINUE)
2489 /* FIXME: need to provide precise fault address */
2492 save_state_to_tss16(ctxt, &tss_seg);
2494 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2496 if (ret != X86EMUL_CONTINUE)
2497 /* FIXME: need to provide precise fault address */
2500 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2502 if (ret != X86EMUL_CONTINUE)
2503 /* FIXME: need to provide precise fault address */
2506 if (old_tss_sel != 0xffff) {
2507 tss_seg.prev_task_link = old_tss_sel;
2509 ret = ops->write_std(ctxt, new_tss_base,
2510 &tss_seg.prev_task_link,
2511 sizeof tss_seg.prev_task_link,
2513 if (ret != X86EMUL_CONTINUE)
2514 /* FIXME: need to provide precise fault address */
2518 return load_state_from_tss16(ctxt, &tss_seg);
2521 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2522 struct tss_segment_32 *tss)
2524 tss->cr3 = ctxt->ops->get_cr(ctxt, 3);
2525 tss->eip = ctxt->_eip;
2526 tss->eflags = ctxt->eflags;
2527 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
2528 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
2529 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
2530 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
2531 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
2532 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
2533 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
2534 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
2536 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2537 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2538 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2539 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2540 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2541 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2542 tss->ldt_selector = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2545 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2546 struct tss_segment_32 *tss)
2550 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2551 return emulate_gp(ctxt, 0);
2552 ctxt->_eip = tss->eip;
2553 ctxt->eflags = tss->eflags | 2;
2555 /* General purpose registers */
2556 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
2557 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
2558 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
2559 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
2560 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
2561 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
2562 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
2563 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
2566 * SDM says that segment selectors are loaded before segment
2569 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2570 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2571 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2572 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2573 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2574 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2575 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2578 * If we're switching between Protected Mode and VM86, we need to make
2579 * sure to update the mode before loading the segment descriptors so
2580 * that the selectors are interpreted correctly.
2582 * Need to get rflags to the vcpu struct immediately because it
2583 * influences the CPL which is checked at least when loading the segment
2584 * descriptors and when pushing an error code to the new kernel stack.
2586 * TODO Introduce a separate ctxt->ops->set_cpl callback
2588 if (ctxt->eflags & X86_EFLAGS_VM)
2589 ctxt->mode = X86EMUL_MODE_VM86;
2591 ctxt->mode = X86EMUL_MODE_PROT32;
2593 ctxt->ops->set_rflags(ctxt, ctxt->eflags);
2596 * Now load segment descriptors. If fault happenes at this stage
2597 * it is handled in a context of new task
2599 ret = load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2600 if (ret != X86EMUL_CONTINUE)
2602 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2603 if (ret != X86EMUL_CONTINUE)
2605 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2606 if (ret != X86EMUL_CONTINUE)
2608 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2609 if (ret != X86EMUL_CONTINUE)
2611 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2612 if (ret != X86EMUL_CONTINUE)
2614 ret = load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS);
2615 if (ret != X86EMUL_CONTINUE)
2617 ret = load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS);
2618 if (ret != X86EMUL_CONTINUE)
2621 return X86EMUL_CONTINUE;
2624 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2625 u16 tss_selector, u16 old_tss_sel,
2626 ulong old_tss_base, struct desc_struct *new_desc)
2628 const struct x86_emulate_ops *ops = ctxt->ops;
2629 struct tss_segment_32 tss_seg;
2631 u32 new_tss_base = get_desc_base(new_desc);
2633 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2635 if (ret != X86EMUL_CONTINUE)
2636 /* FIXME: need to provide precise fault address */
2639 save_state_to_tss32(ctxt, &tss_seg);
2641 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2643 if (ret != X86EMUL_CONTINUE)
2644 /* FIXME: need to provide precise fault address */
2647 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2649 if (ret != X86EMUL_CONTINUE)
2650 /* FIXME: need to provide precise fault address */
2653 if (old_tss_sel != 0xffff) {
2654 tss_seg.prev_task_link = old_tss_sel;
2656 ret = ops->write_std(ctxt, new_tss_base,
2657 &tss_seg.prev_task_link,
2658 sizeof tss_seg.prev_task_link,
2660 if (ret != X86EMUL_CONTINUE)
2661 /* FIXME: need to provide precise fault address */
2665 return load_state_from_tss32(ctxt, &tss_seg);
2668 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2669 u16 tss_selector, int idt_index, int reason,
2670 bool has_error_code, u32 error_code)
2672 const struct x86_emulate_ops *ops = ctxt->ops;
2673 struct desc_struct curr_tss_desc, next_tss_desc;
2675 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2676 ulong old_tss_base =
2677 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2681 /* FIXME: old_tss_base == ~0 ? */
2683 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
2684 if (ret != X86EMUL_CONTINUE)
2686 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
2687 if (ret != X86EMUL_CONTINUE)
2690 /* FIXME: check that next_tss_desc is tss */
2693 * Check privileges. The three cases are task switch caused by...
2695 * 1. jmp/call/int to task gate: Check against DPL of the task gate
2696 * 2. Exception/IRQ/iret: No check is performed
2697 * 3. jmp/call to TSS: Check against DPL of the TSS
2699 if (reason == TASK_SWITCH_GATE) {
2700 if (idt_index != -1) {
2701 /* Software interrupts */
2702 struct desc_struct task_gate_desc;
2705 ret = read_interrupt_descriptor(ctxt, idt_index,
2707 if (ret != X86EMUL_CONTINUE)
2710 dpl = task_gate_desc.dpl;
2711 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2712 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
2714 } else if (reason != TASK_SWITCH_IRET) {
2715 int dpl = next_tss_desc.dpl;
2716 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2717 return emulate_gp(ctxt, tss_selector);
2721 desc_limit = desc_limit_scaled(&next_tss_desc);
2722 if (!next_tss_desc.p ||
2723 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2724 desc_limit < 0x2b)) {
2725 emulate_ts(ctxt, tss_selector & 0xfffc);
2726 return X86EMUL_PROPAGATE_FAULT;
2729 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2730 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2731 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2734 if (reason == TASK_SWITCH_IRET)
2735 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2737 /* set back link to prev task only if NT bit is set in eflags
2738 note that old_tss_sel is not used after this point */
2739 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2740 old_tss_sel = 0xffff;
2742 if (next_tss_desc.type & 8)
2743 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2744 old_tss_base, &next_tss_desc);
2746 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2747 old_tss_base, &next_tss_desc);
2748 if (ret != X86EMUL_CONTINUE)
2751 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2752 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2754 if (reason != TASK_SWITCH_IRET) {
2755 next_tss_desc.type |= (1 << 1); /* set busy flag */
2756 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2759 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2760 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2762 if (has_error_code) {
2763 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2764 ctxt->lock_prefix = 0;
2765 ctxt->src.val = (unsigned long) error_code;
2766 ret = em_push(ctxt);
2772 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2773 u16 tss_selector, int idt_index, int reason,
2774 bool has_error_code, u32 error_code)
2778 invalidate_registers(ctxt);
2779 ctxt->_eip = ctxt->eip;
2780 ctxt->dst.type = OP_NONE;
2782 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
2783 has_error_code, error_code);
2785 if (rc == X86EMUL_CONTINUE) {
2786 ctxt->eip = ctxt->_eip;
2787 writeback_registers(ctxt);
2790 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2793 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
2796 int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;
2798 register_address_increment(ctxt, reg_rmw(ctxt, reg), df * op->bytes);
2799 op->addr.mem.ea = register_address(ctxt, reg_read(ctxt, reg));
2802 static int em_das(struct x86_emulate_ctxt *ctxt)
2805 bool af, cf, old_cf;
2807 cf = ctxt->eflags & X86_EFLAGS_CF;
2813 af = ctxt->eflags & X86_EFLAGS_AF;
2814 if ((al & 0x0f) > 9 || af) {
2816 cf = old_cf | (al >= 250);
2821 if (old_al > 0x99 || old_cf) {
2827 /* Set PF, ZF, SF */
2828 ctxt->src.type = OP_IMM;
2830 ctxt->src.bytes = 1;
2831 emulate_2op_SrcV(ctxt, "or");
2832 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2834 ctxt->eflags |= X86_EFLAGS_CF;
2836 ctxt->eflags |= X86_EFLAGS_AF;
2837 return X86EMUL_CONTINUE;
2840 static int em_call(struct x86_emulate_ctxt *ctxt)
2842 long rel = ctxt->src.val;
2844 ctxt->src.val = (unsigned long)ctxt->_eip;
2846 return em_push(ctxt);
2849 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2855 old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2856 old_eip = ctxt->_eip;
2858 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2859 if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
2860 return X86EMUL_CONTINUE;
2863 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2865 ctxt->src.val = old_cs;
2867 if (rc != X86EMUL_CONTINUE)
2870 ctxt->src.val = old_eip;
2871 return em_push(ctxt);
2874 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2878 ctxt->dst.type = OP_REG;
2879 ctxt->dst.addr.reg = &ctxt->_eip;
2880 ctxt->dst.bytes = ctxt->op_bytes;
2881 rc = emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
2882 if (rc != X86EMUL_CONTINUE)
2884 rsp_increment(ctxt, ctxt->src.val);
2885 return X86EMUL_CONTINUE;
2888 static int em_add(struct x86_emulate_ctxt *ctxt)
2890 emulate_2op_SrcV(ctxt, "add");
2891 return X86EMUL_CONTINUE;
2894 static int em_or(struct x86_emulate_ctxt *ctxt)
2896 emulate_2op_SrcV(ctxt, "or");
2897 return X86EMUL_CONTINUE;
2900 static int em_adc(struct x86_emulate_ctxt *ctxt)
2902 emulate_2op_SrcV(ctxt, "adc");
2903 return X86EMUL_CONTINUE;
2906 static int em_sbb(struct x86_emulate_ctxt *ctxt)
2908 emulate_2op_SrcV(ctxt, "sbb");
2909 return X86EMUL_CONTINUE;
2912 static int em_and(struct x86_emulate_ctxt *ctxt)
2914 emulate_2op_SrcV(ctxt, "and");
2915 return X86EMUL_CONTINUE;
2918 static int em_sub(struct x86_emulate_ctxt *ctxt)
2920 emulate_2op_SrcV(ctxt, "sub");
2921 return X86EMUL_CONTINUE;
2924 static int em_xor(struct x86_emulate_ctxt *ctxt)
2926 emulate_2op_SrcV(ctxt, "xor");
2927 return X86EMUL_CONTINUE;
2930 static int em_cmp(struct x86_emulate_ctxt *ctxt)
2932 emulate_2op_SrcV(ctxt, "cmp");
2933 /* Disable writeback. */
2934 ctxt->dst.type = OP_NONE;
2935 return X86EMUL_CONTINUE;
2938 static int em_test(struct x86_emulate_ctxt *ctxt)
2940 emulate_2op_SrcV(ctxt, "test");
2941 /* Disable writeback. */
2942 ctxt->dst.type = OP_NONE;
2943 return X86EMUL_CONTINUE;
2946 static int em_xchg(struct x86_emulate_ctxt *ctxt)
2948 /* Write back the register source. */
2949 ctxt->src.val = ctxt->dst.val;
2950 write_register_operand(&ctxt->src);
2952 /* Write back the memory destination with implicit LOCK prefix. */
2953 ctxt->dst.val = ctxt->src.orig_val;
2954 ctxt->lock_prefix = 1;
2955 return X86EMUL_CONTINUE;
2958 static int em_imul(struct x86_emulate_ctxt *ctxt)
2960 emulate_2op_SrcV_nobyte(ctxt, "imul");
2961 return X86EMUL_CONTINUE;
2964 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2966 ctxt->dst.val = ctxt->src2.val;
2967 return em_imul(ctxt);
2970 static int em_cwd(struct x86_emulate_ctxt *ctxt)
2972 ctxt->dst.type = OP_REG;
2973 ctxt->dst.bytes = ctxt->src.bytes;
2974 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
2975 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
2977 return X86EMUL_CONTINUE;
2980 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2984 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
2985 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
2986 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
2987 return X86EMUL_CONTINUE;
2990 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
2994 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
2995 return emulate_gp(ctxt, 0);
2996 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
2997 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
2998 return X86EMUL_CONTINUE;
3001 static int em_mov(struct x86_emulate_ctxt *ctxt)
3003 memcpy(ctxt->dst.valptr, ctxt->src.valptr, ctxt->op_bytes);
3004 return X86EMUL_CONTINUE;
3007 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3009 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3010 return emulate_gp(ctxt, 0);
3012 /* Disable writeback. */
3013 ctxt->dst.type = OP_NONE;
3014 return X86EMUL_CONTINUE;
3017 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3021 if (ctxt->mode == X86EMUL_MODE_PROT64)
3022 val = ctxt->src.val & ~0ULL;
3024 val = ctxt->src.val & ~0U;
3026 /* #UD condition is already handled. */
3027 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3028 return emulate_gp(ctxt, 0);
3030 /* Disable writeback. */
3031 ctxt->dst.type = OP_NONE;
3032 return X86EMUL_CONTINUE;
3035 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3039 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3040 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3041 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3042 return emulate_gp(ctxt, 0);
3044 return X86EMUL_CONTINUE;
3047 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3051 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3052 return emulate_gp(ctxt, 0);
3054 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3055 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3056 return X86EMUL_CONTINUE;
3059 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3061 if (ctxt->modrm_reg > VCPU_SREG_GS)
3062 return emulate_ud(ctxt);
3064 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3065 return X86EMUL_CONTINUE;
3068 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3070 u16 sel = ctxt->src.val;
3072 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3073 return emulate_ud(ctxt);
3075 if (ctxt->modrm_reg == VCPU_SREG_SS)
3076 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3078 /* Disable writeback. */
3079 ctxt->dst.type = OP_NONE;
3080 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3083 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3085 u16 sel = ctxt->src.val;
3087 /* Disable writeback. */
3088 ctxt->dst.type = OP_NONE;
3089 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3092 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3094 u16 sel = ctxt->src.val;
3096 /* Disable writeback. */
3097 ctxt->dst.type = OP_NONE;
3098 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3101 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3106 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3107 if (rc == X86EMUL_CONTINUE)
3108 ctxt->ops->invlpg(ctxt, linear);
3109 /* Disable writeback. */
3110 ctxt->dst.type = OP_NONE;
3111 return X86EMUL_CONTINUE;
3114 static int em_clts(struct x86_emulate_ctxt *ctxt)
3118 cr0 = ctxt->ops->get_cr(ctxt, 0);
3120 ctxt->ops->set_cr(ctxt, 0, cr0);
3121 return X86EMUL_CONTINUE;
3124 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
3128 if (ctxt->modrm_mod != 3 || ctxt->modrm_rm != 1)
3129 return X86EMUL_UNHANDLEABLE;
3131 rc = ctxt->ops->fix_hypercall(ctxt);
3132 if (rc != X86EMUL_CONTINUE)
3135 /* Let the processor re-execute the fixed hypercall */
3136 ctxt->_eip = ctxt->eip;
3137 /* Disable writeback. */
3138 ctxt->dst.type = OP_NONE;
3139 return X86EMUL_CONTINUE;
3142 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3143 void (*get)(struct x86_emulate_ctxt *ctxt,
3144 struct desc_ptr *ptr))
3146 struct desc_ptr desc_ptr;
3148 if (ctxt->mode == X86EMUL_MODE_PROT64)
3150 get(ctxt, &desc_ptr);
3151 if (ctxt->op_bytes == 2) {
3153 desc_ptr.address &= 0x00ffffff;
3155 /* Disable writeback. */
3156 ctxt->dst.type = OP_NONE;
3157 return segmented_write(ctxt, ctxt->dst.addr.mem,
3158 &desc_ptr, 2 + ctxt->op_bytes);
3161 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3163 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3166 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3168 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3171 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3173 struct desc_ptr desc_ptr;
3176 if (ctxt->mode == X86EMUL_MODE_PROT64)
3178 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3179 &desc_ptr.size, &desc_ptr.address,
3181 if (rc != X86EMUL_CONTINUE)
3183 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3184 /* Disable writeback. */
3185 ctxt->dst.type = OP_NONE;
3186 return X86EMUL_CONTINUE;
3189 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
3193 rc = ctxt->ops->fix_hypercall(ctxt);
3195 /* Disable writeback. */
3196 ctxt->dst.type = OP_NONE;
3200 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3202 struct desc_ptr desc_ptr;
3205 if (ctxt->mode == X86EMUL_MODE_PROT64)
3207 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3208 &desc_ptr.size, &desc_ptr.address,
3210 if (rc != X86EMUL_CONTINUE)
3212 ctxt->ops->set_idt(ctxt, &desc_ptr);
3213 /* Disable writeback. */
3214 ctxt->dst.type = OP_NONE;
3215 return X86EMUL_CONTINUE;
3218 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3220 ctxt->dst.bytes = 2;
3221 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3222 return X86EMUL_CONTINUE;
3225 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3227 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3228 | (ctxt->src.val & 0x0f));
3229 ctxt->dst.type = OP_NONE;
3230 return X86EMUL_CONTINUE;
3233 static int em_loop(struct x86_emulate_ctxt *ctxt)
3235 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX), -1);
3236 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3237 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3238 jmp_rel(ctxt, ctxt->src.val);
3240 return X86EMUL_CONTINUE;
3243 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3245 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3246 jmp_rel(ctxt, ctxt->src.val);
3248 return X86EMUL_CONTINUE;
3251 static int em_in(struct x86_emulate_ctxt *ctxt)
3253 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3255 return X86EMUL_IO_NEEDED;
3257 return X86EMUL_CONTINUE;
3260 static int em_out(struct x86_emulate_ctxt *ctxt)
3262 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3264 /* Disable writeback. */
3265 ctxt->dst.type = OP_NONE;
3266 return X86EMUL_CONTINUE;
3269 static int em_cli(struct x86_emulate_ctxt *ctxt)
3271 if (emulator_bad_iopl(ctxt))
3272 return emulate_gp(ctxt, 0);
3274 ctxt->eflags &= ~X86_EFLAGS_IF;
3275 return X86EMUL_CONTINUE;
3278 static int em_sti(struct x86_emulate_ctxt *ctxt)
3280 if (emulator_bad_iopl(ctxt))
3281 return emulate_gp(ctxt, 0);
3283 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3284 ctxt->eflags |= X86_EFLAGS_IF;
3285 return X86EMUL_CONTINUE;
3288 static int em_bt(struct x86_emulate_ctxt *ctxt)
3290 /* Disable writeback. */
3291 ctxt->dst.type = OP_NONE;
3292 /* only subword offset */
3293 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
3295 emulate_2op_SrcV_nobyte(ctxt, "bt");
3296 return X86EMUL_CONTINUE;
3299 static int em_bts(struct x86_emulate_ctxt *ctxt)
3301 emulate_2op_SrcV_nobyte(ctxt, "bts");
3302 return X86EMUL_CONTINUE;
3305 static int em_btr(struct x86_emulate_ctxt *ctxt)
3307 emulate_2op_SrcV_nobyte(ctxt, "btr");
3308 return X86EMUL_CONTINUE;
3311 static int em_btc(struct x86_emulate_ctxt *ctxt)
3313 emulate_2op_SrcV_nobyte(ctxt, "btc");
3314 return X86EMUL_CONTINUE;
3317 static int em_bsf(struct x86_emulate_ctxt *ctxt)
3319 emulate_2op_SrcV_nobyte(ctxt, "bsf");
3320 return X86EMUL_CONTINUE;
3323 static int em_bsr(struct x86_emulate_ctxt *ctxt)
3325 emulate_2op_SrcV_nobyte(ctxt, "bsr");
3326 return X86EMUL_CONTINUE;
3329 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3331 u32 eax, ebx, ecx, edx;
3333 eax = reg_read(ctxt, VCPU_REGS_RAX);
3334 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3335 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3336 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3337 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3338 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3339 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3340 return X86EMUL_CONTINUE;
3343 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3345 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3346 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3347 return X86EMUL_CONTINUE;
3350 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3352 switch (ctxt->op_bytes) {
3353 #ifdef CONFIG_X86_64
3355 asm("bswap %0" : "+r"(ctxt->dst.val));
3359 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3362 return X86EMUL_CONTINUE;
3365 static bool valid_cr(int nr)
3377 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
3379 if (!valid_cr(ctxt->modrm_reg))
3380 return emulate_ud(ctxt);
3382 return X86EMUL_CONTINUE;
3385 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
3387 u64 new_val = ctxt->src.val64;
3388 int cr = ctxt->modrm_reg;
3391 static u64 cr_reserved_bits[] = {
3392 0xffffffff00000000ULL,
3393 0, 0, 0, /* CR3 checked later */
3400 return emulate_ud(ctxt);
3402 if (new_val & cr_reserved_bits[cr])
3403 return emulate_gp(ctxt, 0);
3408 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
3409 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
3410 return emulate_gp(ctxt, 0);
3412 cr4 = ctxt->ops->get_cr(ctxt, 4);
3413 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3415 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
3416 !(cr4 & X86_CR4_PAE))
3417 return emulate_gp(ctxt, 0);
3424 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3425 if (efer & EFER_LMA)
3426 rsvd = CR3_L_MODE_RESERVED_BITS;
3427 else if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PAE)
3428 rsvd = CR3_PAE_RESERVED_BITS;
3429 else if (ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PG)
3430 rsvd = CR3_NONPAE_RESERVED_BITS;
3433 return emulate_gp(ctxt, 0);
3438 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3440 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
3441 return emulate_gp(ctxt, 0);
3447 return X86EMUL_CONTINUE;
3450 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
3454 ctxt->ops->get_dr(ctxt, 7, &dr7);
3456 /* Check if DR7.Global_Enable is set */
3457 return dr7 & (1 << 13);
3460 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
3462 int dr = ctxt->modrm_reg;
3466 return emulate_ud(ctxt);
3468 cr4 = ctxt->ops->get_cr(ctxt, 4);
3469 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
3470 return emulate_ud(ctxt);
3472 if (check_dr7_gd(ctxt))
3473 return emulate_db(ctxt);
3475 return X86EMUL_CONTINUE;
3478 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
3480 u64 new_val = ctxt->src.val64;
3481 int dr = ctxt->modrm_reg;
3483 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
3484 return emulate_gp(ctxt, 0);
3486 return check_dr_read(ctxt);
3489 static int check_svme(struct x86_emulate_ctxt *ctxt)
3493 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3495 if (!(efer & EFER_SVME))
3496 return emulate_ud(ctxt);
3498 return X86EMUL_CONTINUE;
3501 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
3503 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
3505 /* Valid physical address? */
3506 if (rax & 0xffff000000000000ULL)
3507 return emulate_gp(ctxt, 0);
3509 return check_svme(ctxt);
3512 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
3514 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3516 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3517 return emulate_ud(ctxt);
3519 return X86EMUL_CONTINUE;
3522 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
3524 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3525 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
3527 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3529 return emulate_gp(ctxt, 0);
3531 return X86EMUL_CONTINUE;
3534 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
3536 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
3537 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3538 return emulate_gp(ctxt, 0);
3540 return X86EMUL_CONTINUE;
3543 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
3545 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
3546 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3547 return emulate_gp(ctxt, 0);
3549 return X86EMUL_CONTINUE;
3552 #define D(_y) { .flags = (_y) }
3553 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
3554 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
3555 .check_perm = (_p) }
3557 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3558 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
3559 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
3560 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3561 #define II(_f, _e, _i) \
3562 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
3563 #define IIP(_f, _e, _i, _p) \
3564 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
3565 .check_perm = (_p) }
3566 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3568 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3569 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3570 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3571 #define I2bvIP(_f, _e, _i, _p) \
3572 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
3574 #define I6ALU(_f, _e) I2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3575 I2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3576 I2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3578 static const struct opcode group7_rm1[] = {
3579 DI(SrcNone | Priv, monitor),
3580 DI(SrcNone | Priv, mwait),
3584 static const struct opcode group7_rm3[] = {
3585 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
3586 II(SrcNone | Prot | VendorSpecific, em_vmmcall, vmmcall),
3587 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
3588 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
3589 DIP(SrcNone | Prot | Priv, stgi, check_svme),
3590 DIP(SrcNone | Prot | Priv, clgi, check_svme),
3591 DIP(SrcNone | Prot | Priv, skinit, check_svme),
3592 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
3595 static const struct opcode group7_rm7[] = {
3597 DIP(SrcNone, rdtscp, check_rdtsc),
3601 static const struct opcode group1[] = {
3603 I(Lock | PageTable, em_or),
3606 I(Lock | PageTable, em_and),
3612 static const struct opcode group1A[] = {
3613 I(DstMem | SrcNone | Mov | Stack, em_pop), N, N, N, N, N, N, N,
3616 static const struct opcode group3[] = {
3617 I(DstMem | SrcImm, em_test),
3618 I(DstMem | SrcImm, em_test),
3619 I(DstMem | SrcNone | Lock, em_not),
3620 I(DstMem | SrcNone | Lock, em_neg),
3621 I(SrcMem, em_mul_ex),
3622 I(SrcMem, em_imul_ex),
3623 I(SrcMem, em_div_ex),
3624 I(SrcMem, em_idiv_ex),
3627 static const struct opcode group4[] = {
3628 I(ByteOp | DstMem | SrcNone | Lock, em_grp45),
3629 I(ByteOp | DstMem | SrcNone | Lock, em_grp45),
3633 static const struct opcode group5[] = {
3634 I(DstMem | SrcNone | Lock, em_grp45),
3635 I(DstMem | SrcNone | Lock, em_grp45),
3636 I(SrcMem | Stack, em_grp45),
3637 I(SrcMemFAddr | ImplicitOps | Stack, em_call_far),
3638 I(SrcMem | Stack, em_grp45),
3639 I(SrcMemFAddr | ImplicitOps, em_grp45),
3640 I(SrcMem | Stack, em_grp45), N,
3643 static const struct opcode group6[] = {
3646 II(Prot | Priv | SrcMem16, em_lldt, lldt),
3647 II(Prot | Priv | SrcMem16, em_ltr, ltr),
3651 static const struct group_dual group7 = { {
3652 II(Mov | DstMem | Priv, em_sgdt, sgdt),
3653 II(Mov | DstMem | Priv, em_sidt, sidt),
3654 II(SrcMem | Priv, em_lgdt, lgdt),
3655 II(SrcMem | Priv, em_lidt, lidt),
3656 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3657 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3658 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3660 I(SrcNone | Priv | VendorSpecific, em_vmcall),
3662 N, EXT(0, group7_rm3),
3663 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3664 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3668 static const struct opcode group8[] = {
3670 I(DstMem | SrcImmByte, em_bt),
3671 I(DstMem | SrcImmByte | Lock | PageTable, em_bts),
3672 I(DstMem | SrcImmByte | Lock, em_btr),
3673 I(DstMem | SrcImmByte | Lock | PageTable, em_btc),
3676 static const struct group_dual group9 = { {
3677 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
3679 N, N, N, N, N, N, N, N,
3682 static const struct opcode group11[] = {
3683 I(DstMem | SrcImm | Mov | PageTable, em_mov),
3687 static const struct gprefix pfx_0f_6f_0f_7f = {
3688 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
3691 static const struct gprefix pfx_vmovntpx = {
3692 I(0, em_mov), N, N, N,
3695 static const struct opcode opcode_table[256] = {
3697 I6ALU(Lock, em_add),
3698 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
3699 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
3701 I6ALU(Lock | PageTable, em_or),
3702 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
3705 I6ALU(Lock, em_adc),
3706 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
3707 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
3709 I6ALU(Lock, em_sbb),
3710 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
3711 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
3713 I6ALU(Lock | PageTable, em_and), N, N,
3715 I6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3717 I6ALU(Lock, em_xor), N, N,
3719 I6ALU(0, em_cmp), N, N,
3723 X8(I(SrcReg | Stack, em_push)),
3725 X8(I(DstReg | Stack, em_pop)),
3727 I(ImplicitOps | Stack | No64, em_pusha),
3728 I(ImplicitOps | Stack | No64, em_popa),
3729 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3732 I(SrcImm | Mov | Stack, em_push),
3733 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3734 I(SrcImmByte | Mov | Stack, em_push),
3735 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3736 I2bvIP(DstDI | SrcDX | Mov | String, em_in, ins, check_perm_in), /* insb, insw/insd */
3737 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
3741 G(ByteOp | DstMem | SrcImm, group1),
3742 G(DstMem | SrcImm, group1),
3743 G(ByteOp | DstMem | SrcImm | No64, group1),
3744 G(DstMem | SrcImmByte, group1),
3745 I2bv(DstMem | SrcReg | ModRM, em_test),
3746 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
3748 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
3749 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3750 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
3751 D(ModRM | SrcMem | NoAccess | DstReg),
3752 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
3755 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3757 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3758 I(SrcImmFAddr | No64, em_call_far), N,
3759 II(ImplicitOps | Stack, em_pushf, pushf),
3760 II(ImplicitOps | Stack, em_popf, popf), N, I(ImplicitOps, em_lahf),
3762 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3763 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
3764 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3765 I2bv(SrcSI | DstDI | String, em_cmp),
3767 I2bv(DstAcc | SrcImm, em_test),
3768 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3769 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3770 I2bv(SrcAcc | DstDI | String, em_cmp),
3772 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3774 X8(I(DstReg | SrcImm | Mov, em_mov)),
3776 D2bv(DstMem | SrcImmByte | ModRM),
3777 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3778 I(ImplicitOps | Stack, em_ret),
3779 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
3780 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
3781 G(ByteOp, group11), G(0, group11),
3783 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
3784 N, I(ImplicitOps | Stack, em_ret_far),
3785 D(ImplicitOps), DI(SrcImmByte, intn),
3786 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3788 D2bv(DstMem | SrcOne | ModRM), D2bv(DstMem | ModRM),
3791 N, N, N, N, N, N, N, N,
3793 X3(I(SrcImmByte, em_loop)),
3794 I(SrcImmByte, em_jcxz),
3795 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
3796 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
3798 I(SrcImm | Stack, em_call), D(SrcImm | ImplicitOps),
3799 I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
3800 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
3801 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
3803 N, DI(ImplicitOps, icebp), N, N,
3804 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
3805 G(ByteOp, group3), G(0, group3),
3807 D(ImplicitOps), D(ImplicitOps),
3808 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
3809 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3812 static const struct opcode twobyte_table[256] = {
3814 G(0, group6), GD(0, &group7), N, N,
3815 N, I(ImplicitOps | VendorSpecific, em_syscall),
3816 II(ImplicitOps | Priv, em_clts, clts), N,
3817 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3818 N, D(ImplicitOps | ModRM), N, N,
3820 N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
3822 DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
3823 DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
3824 IIP(ModRM | SrcMem | Priv | Op3264, em_cr_write, cr_write, check_cr_write),
3825 IIP(ModRM | SrcMem | Priv | Op3264, em_dr_write, dr_write, check_dr_write),
3827 N, N, N, GP(ModRM | DstMem | SrcReg | Sse | Mov | Aligned, &pfx_vmovntpx),
3830 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
3831 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
3832 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
3833 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
3834 I(ImplicitOps | VendorSpecific, em_sysenter),
3835 I(ImplicitOps | Priv | VendorSpecific, em_sysexit),
3837 N, N, N, N, N, N, N, N,
3839 X16(D(DstReg | SrcMem | ModRM | Mov)),
3841 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3846 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3851 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3855 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3857 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
3858 II(ImplicitOps, em_cpuid, cpuid), I(DstMem | SrcReg | ModRM | BitOp, em_bt),
3859 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3860 D(DstMem | SrcReg | Src2CL | ModRM), N, N,
3862 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
3863 DI(ImplicitOps, rsm),
3864 I(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
3865 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3866 D(DstMem | SrcReg | Src2CL | ModRM),
3867 D(ModRM), I(DstReg | SrcMem | ModRM, em_imul),
3869 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_cmpxchg),
3870 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
3871 I(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
3872 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
3873 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
3874 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3878 I(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
3879 I(DstReg | SrcMem | ModRM, em_bsf), I(DstReg | SrcMem | ModRM, em_bsr),
3880 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3882 D2bv(DstMem | SrcReg | ModRM | Lock),
3883 N, D(DstMem | SrcReg | ModRM | Mov),
3884 N, N, N, GD(0, &group9),
3886 X8(I(DstReg, em_bswap)),
3888 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3890 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3892 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
3909 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
3913 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3919 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
3920 unsigned size, bool sign_extension)
3922 int rc = X86EMUL_CONTINUE;
3926 op->addr.mem.ea = ctxt->_eip;
3927 /* NB. Immediates are sign-extended as necessary. */
3928 switch (op->bytes) {
3930 op->val = insn_fetch(s8, ctxt);
3933 op->val = insn_fetch(s16, ctxt);
3936 op->val = insn_fetch(s32, ctxt);
3939 if (!sign_extension) {
3940 switch (op->bytes) {
3948 op->val &= 0xffffffff;
3956 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
3959 int rc = X86EMUL_CONTINUE;
3963 decode_register_operand(ctxt, op);
3966 rc = decode_imm(ctxt, op, 1, false);
3969 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3973 if ((ctxt->d & BitOp) && op == &ctxt->dst)
3974 fetch_bit_operand(ctxt);
3975 op->orig_val = op->val;
3978 ctxt->memop.bytes = 8;
3982 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3983 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
3984 fetch_register_operand(op);
3985 op->orig_val = op->val;
3989 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3991 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RDI));
3992 op->addr.mem.seg = VCPU_SREG_ES;
3998 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
3999 fetch_register_operand(op);
4003 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4006 rc = decode_imm(ctxt, op, 1, true);
4013 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
4016 ctxt->memop.bytes = 1;
4019 ctxt->memop.bytes = 2;
4022 ctxt->memop.bytes = 4;
4025 rc = decode_imm(ctxt, op, 2, false);
4028 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
4032 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4034 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RSI));
4035 op->addr.mem.seg = seg_override(ctxt);
4040 op->addr.mem.ea = ctxt->_eip;
4041 op->bytes = ctxt->op_bytes + 2;
4042 insn_fetch_arr(op->valptr, op->bytes, ctxt);
4045 ctxt->memop.bytes = ctxt->op_bytes + 2;
4048 op->val = VCPU_SREG_ES;
4051 op->val = VCPU_SREG_CS;
4054 op->val = VCPU_SREG_SS;
4057 op->val = VCPU_SREG_DS;
4060 op->val = VCPU_SREG_FS;
4063 op->val = VCPU_SREG_GS;
4066 /* Special instructions do their own operand decoding. */
4068 op->type = OP_NONE; /* Disable writeback. */
4076 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
4078 int rc = X86EMUL_CONTINUE;
4079 int mode = ctxt->mode;
4080 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
4081 bool op_prefix = false;
4082 struct opcode opcode;
4084 ctxt->memop.type = OP_NONE;
4085 ctxt->memopp = NULL;
4086 ctxt->_eip = ctxt->eip;
4087 ctxt->fetch.start = ctxt->_eip;
4088 ctxt->fetch.end = ctxt->fetch.start + insn_len;
4090 memcpy(ctxt->fetch.data, insn, insn_len);
4093 case X86EMUL_MODE_REAL:
4094 case X86EMUL_MODE_VM86:
4095 case X86EMUL_MODE_PROT16:
4096 def_op_bytes = def_ad_bytes = 2;
4098 case X86EMUL_MODE_PROT32:
4099 def_op_bytes = def_ad_bytes = 4;
4101 #ifdef CONFIG_X86_64
4102 case X86EMUL_MODE_PROT64:
4108 return EMULATION_FAILED;
4111 ctxt->op_bytes = def_op_bytes;
4112 ctxt->ad_bytes = def_ad_bytes;
4114 /* Legacy prefixes. */
4116 switch (ctxt->b = insn_fetch(u8, ctxt)) {
4117 case 0x66: /* operand-size override */
4119 /* switch between 2/4 bytes */
4120 ctxt->op_bytes = def_op_bytes ^ 6;
4122 case 0x67: /* address-size override */
4123 if (mode == X86EMUL_MODE_PROT64)
4124 /* switch between 4/8 bytes */
4125 ctxt->ad_bytes = def_ad_bytes ^ 12;
4127 /* switch between 2/4 bytes */
4128 ctxt->ad_bytes = def_ad_bytes ^ 6;
4130 case 0x26: /* ES override */
4131 case 0x2e: /* CS override */
4132 case 0x36: /* SS override */
4133 case 0x3e: /* DS override */
4134 set_seg_override(ctxt, (ctxt->b >> 3) & 3);
4136 case 0x64: /* FS override */
4137 case 0x65: /* GS override */
4138 set_seg_override(ctxt, ctxt->b & 7);
4140 case 0x40 ... 0x4f: /* REX */
4141 if (mode != X86EMUL_MODE_PROT64)
4143 ctxt->rex_prefix = ctxt->b;
4145 case 0xf0: /* LOCK */
4146 ctxt->lock_prefix = 1;
4148 case 0xf2: /* REPNE/REPNZ */
4149 case 0xf3: /* REP/REPE/REPZ */
4150 ctxt->rep_prefix = ctxt->b;
4156 /* Any legacy prefix after a REX prefix nullifies its effect. */
4158 ctxt->rex_prefix = 0;
4164 if (ctxt->rex_prefix & 8)
4165 ctxt->op_bytes = 8; /* REX.W */
4167 /* Opcode byte(s). */
4168 opcode = opcode_table[ctxt->b];
4169 /* Two-byte opcode? */
4170 if (ctxt->b == 0x0f) {
4172 ctxt->b = insn_fetch(u8, ctxt);
4173 opcode = twobyte_table[ctxt->b];
4175 ctxt->d = opcode.flags;
4177 if (ctxt->d & ModRM)
4178 ctxt->modrm = insn_fetch(u8, ctxt);
4180 while (ctxt->d & GroupMask) {
4181 switch (ctxt->d & GroupMask) {
4183 goffset = (ctxt->modrm >> 3) & 7;
4184 opcode = opcode.u.group[goffset];
4187 goffset = (ctxt->modrm >> 3) & 7;
4188 if ((ctxt->modrm >> 6) == 3)
4189 opcode = opcode.u.gdual->mod3[goffset];
4191 opcode = opcode.u.gdual->mod012[goffset];
4194 goffset = ctxt->modrm & 7;
4195 opcode = opcode.u.group[goffset];
4198 if (ctxt->rep_prefix && op_prefix)
4199 return EMULATION_FAILED;
4200 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
4201 switch (simd_prefix) {
4202 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
4203 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
4204 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
4205 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
4209 return EMULATION_FAILED;
4212 ctxt->d &= ~(u64)GroupMask;
4213 ctxt->d |= opcode.flags;
4216 ctxt->execute = opcode.u.execute;
4217 ctxt->check_perm = opcode.check_perm;
4218 ctxt->intercept = opcode.intercept;
4221 if (ctxt->d == 0 || (ctxt->d & Undefined))
4222 return EMULATION_FAILED;
4224 if (!(ctxt->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
4225 return EMULATION_FAILED;
4227 if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
4230 if (ctxt->d & Op3264) {
4231 if (mode == X86EMUL_MODE_PROT64)
4238 ctxt->op_bytes = 16;
4239 else if (ctxt->d & Mmx)
4242 /* ModRM and SIB bytes. */
4243 if (ctxt->d & ModRM) {
4244 rc = decode_modrm(ctxt, &ctxt->memop);
4245 if (!ctxt->has_seg_override)
4246 set_seg_override(ctxt, ctxt->modrm_seg);
4247 } else if (ctxt->d & MemAbs)
4248 rc = decode_abs(ctxt, &ctxt->memop);
4249 if (rc != X86EMUL_CONTINUE)
4252 if (!ctxt->has_seg_override)
4253 set_seg_override(ctxt, VCPU_SREG_DS);
4255 ctxt->memop.addr.mem.seg = seg_override(ctxt);
4257 if (ctxt->memop.type == OP_MEM && ctxt->ad_bytes != 8)
4258 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
4261 * Decode and fetch the source operand: register, memory
4264 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
4265 if (rc != X86EMUL_CONTINUE)
4269 * Decode and fetch the second source operand: register, memory
4272 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
4273 if (rc != X86EMUL_CONTINUE)
4276 /* Decode and fetch the destination operand: register or memory. */
4277 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
4280 if (ctxt->memopp && ctxt->memopp->type == OP_MEM && ctxt->rip_relative)
4281 ctxt->memopp->addr.mem.ea += ctxt->_eip;
4283 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
4286 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
4288 return ctxt->d & PageTable;
4291 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
4293 /* The second termination condition only applies for REPE
4294 * and REPNE. Test if the repeat string operation prefix is
4295 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
4296 * corresponding termination condition according to:
4297 * - if REPE/REPZ and ZF = 0 then done
4298 * - if REPNE/REPNZ and ZF = 1 then done
4300 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
4301 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
4302 && (((ctxt->rep_prefix == REPE_PREFIX) &&
4303 ((ctxt->eflags & EFLG_ZF) == 0))
4304 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
4305 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
4311 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
4315 ctxt->ops->get_fpu(ctxt);
4316 asm volatile("1: fwait \n\t"
4318 ".pushsection .fixup,\"ax\" \n\t"
4320 "movb $1, %[fault] \n\t"
4323 _ASM_EXTABLE(1b, 3b)
4324 : [fault]"+qm"(fault));
4325 ctxt->ops->put_fpu(ctxt);
4327 if (unlikely(fault))
4328 return emulate_exception(ctxt, MF_VECTOR, 0, false);
4330 return X86EMUL_CONTINUE;
4333 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
4336 if (op->type == OP_MM)
4337 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
4341 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
4343 const struct x86_emulate_ops *ops = ctxt->ops;
4344 int rc = X86EMUL_CONTINUE;
4345 int saved_dst_type = ctxt->dst.type;
4347 ctxt->mem_read.pos = 0;
4349 if (ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) {
4350 rc = emulate_ud(ctxt);
4354 /* LOCK prefix is allowed only with some instructions */
4355 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
4356 rc = emulate_ud(ctxt);
4360 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
4361 rc = emulate_ud(ctxt);
4365 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
4366 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
4367 rc = emulate_ud(ctxt);
4371 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
4372 rc = emulate_nm(ctxt);
4376 if (ctxt->d & Mmx) {
4377 rc = flush_pending_x87_faults(ctxt);
4378 if (rc != X86EMUL_CONTINUE)
4381 * Now that we know the fpu is exception safe, we can fetch
4384 fetch_possible_mmx_operand(ctxt, &ctxt->src);
4385 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
4386 if (!(ctxt->d & Mov))
4387 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
4390 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4391 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4392 X86_ICPT_PRE_EXCEPT);
4393 if (rc != X86EMUL_CONTINUE)
4397 /* Privileged instruction can be executed only in CPL=0 */
4398 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
4399 rc = emulate_gp(ctxt, 0);
4403 /* Instruction can only be executed in protected mode */
4404 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
4405 rc = emulate_ud(ctxt);
4409 /* Do instruction specific permission checks */
4410 if (ctxt->check_perm) {
4411 rc = ctxt->check_perm(ctxt);
4412 if (rc != X86EMUL_CONTINUE)
4416 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4417 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4418 X86_ICPT_POST_EXCEPT);
4419 if (rc != X86EMUL_CONTINUE)
4423 if (ctxt->rep_prefix && (ctxt->d & String)) {
4424 /* All REP prefixes have the same first termination condition */
4425 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
4426 ctxt->eip = ctxt->_eip;
4431 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
4432 rc = segmented_read(ctxt, ctxt->src.addr.mem,
4433 ctxt->src.valptr, ctxt->src.bytes);
4434 if (rc != X86EMUL_CONTINUE)
4436 ctxt->src.orig_val64 = ctxt->src.val64;
4439 if (ctxt->src2.type == OP_MEM) {
4440 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
4441 &ctxt->src2.val, ctxt->src2.bytes);
4442 if (rc != X86EMUL_CONTINUE)
4446 if ((ctxt->d & DstMask) == ImplicitOps)
4450 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
4451 /* optimisation - avoid slow emulated read if Mov */
4452 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
4453 &ctxt->dst.val, ctxt->dst.bytes);
4454 if (rc != X86EMUL_CONTINUE)
4457 ctxt->dst.orig_val = ctxt->dst.val;
4461 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4462 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4463 X86_ICPT_POST_MEMACCESS);
4464 if (rc != X86EMUL_CONTINUE)
4468 if (ctxt->execute) {
4469 rc = ctxt->execute(ctxt);
4470 if (rc != X86EMUL_CONTINUE)
4479 case 0x40 ... 0x47: /* inc r16/r32 */
4480 emulate_1op(ctxt, "inc");
4482 case 0x48 ... 0x4f: /* dec r16/r32 */
4483 emulate_1op(ctxt, "dec");
4485 case 0x63: /* movsxd */
4486 if (ctxt->mode != X86EMUL_MODE_PROT64)
4487 goto cannot_emulate;
4488 ctxt->dst.val = (s32) ctxt->src.val;
4490 case 0x70 ... 0x7f: /* jcc (short) */
4491 if (test_cc(ctxt->b, ctxt->eflags))
4492 jmp_rel(ctxt, ctxt->src.val);
4494 case 0x8d: /* lea r16/r32, m */
4495 ctxt->dst.val = ctxt->src.addr.mem.ea;
4497 case 0x90 ... 0x97: /* nop / xchg reg, rax */
4498 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
4502 case 0x98: /* cbw/cwde/cdqe */
4503 switch (ctxt->op_bytes) {
4504 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
4505 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
4506 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
4512 case 0xcc: /* int3 */
4513 rc = emulate_int(ctxt, 3);
4515 case 0xcd: /* int n */
4516 rc = emulate_int(ctxt, ctxt->src.val);
4518 case 0xce: /* into */
4519 if (ctxt->eflags & EFLG_OF)
4520 rc = emulate_int(ctxt, 4);
4522 case 0xd0 ... 0xd1: /* Grp2 */
4525 case 0xd2 ... 0xd3: /* Grp2 */
4526 ctxt->src.val = reg_read(ctxt, VCPU_REGS_RCX);
4529 case 0xe9: /* jmp rel */
4530 case 0xeb: /* jmp rel short */
4531 jmp_rel(ctxt, ctxt->src.val);
4532 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4534 case 0xf4: /* hlt */
4535 ctxt->ops->halt(ctxt);
4537 case 0xf5: /* cmc */
4538 /* complement carry flag from eflags reg */
4539 ctxt->eflags ^= EFLG_CF;
4541 case 0xf8: /* clc */
4542 ctxt->eflags &= ~EFLG_CF;
4544 case 0xf9: /* stc */
4545 ctxt->eflags |= EFLG_CF;
4547 case 0xfc: /* cld */
4548 ctxt->eflags &= ~EFLG_DF;
4550 case 0xfd: /* std */
4551 ctxt->eflags |= EFLG_DF;
4554 goto cannot_emulate;
4557 if (rc != X86EMUL_CONTINUE)
4561 rc = writeback(ctxt);
4562 if (rc != X86EMUL_CONTINUE)
4566 * restore dst type in case the decoding will be reused
4567 * (happens for string instruction )
4569 ctxt->dst.type = saved_dst_type;
4571 if ((ctxt->d & SrcMask) == SrcSI)
4572 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
4574 if ((ctxt->d & DstMask) == DstDI)
4575 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
4577 if (ctxt->rep_prefix && (ctxt->d & String)) {
4578 struct read_cache *r = &ctxt->io_read;
4579 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX), -1);
4581 if (!string_insn_completed(ctxt)) {
4583 * Re-enter guest when pio read ahead buffer is empty
4584 * or, if it is not used, after each 1024 iteration.
4586 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
4587 (r->end == 0 || r->end != r->pos)) {
4589 * Reset read cache. Usually happens before
4590 * decode, but since instruction is restarted
4591 * we have to do it here.
4593 ctxt->mem_read.end = 0;
4594 writeback_registers(ctxt);
4595 return EMULATION_RESTART;
4597 goto done; /* skip rip writeback */
4601 ctxt->eip = ctxt->_eip;
4604 if (rc == X86EMUL_PROPAGATE_FAULT)
4605 ctxt->have_exception = true;
4606 if (rc == X86EMUL_INTERCEPTED)
4607 return EMULATION_INTERCEPTED;
4609 if (rc == X86EMUL_CONTINUE)
4610 writeback_registers(ctxt);
4612 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4616 case 0x09: /* wbinvd */
4617 (ctxt->ops->wbinvd)(ctxt);
4619 case 0x08: /* invd */
4620 case 0x0d: /* GrpP (prefetch) */
4621 case 0x18: /* Grp16 (prefetch/nop) */
4623 case 0x20: /* mov cr, reg */
4624 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
4626 case 0x21: /* mov from dr to reg */
4627 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
4629 case 0x40 ... 0x4f: /* cmov */
4630 ctxt->dst.val = ctxt->dst.orig_val = ctxt->src.val;
4631 if (!test_cc(ctxt->b, ctxt->eflags))
4632 ctxt->dst.type = OP_NONE; /* no writeback */
4634 case 0x80 ... 0x8f: /* jnz rel, etc*/
4635 if (test_cc(ctxt->b, ctxt->eflags))
4636 jmp_rel(ctxt, ctxt->src.val);
4638 case 0x90 ... 0x9f: /* setcc r/m8 */
4639 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
4641 case 0xa4: /* shld imm8, r, r/m */
4642 case 0xa5: /* shld cl, r, r/m */
4643 emulate_2op_cl(ctxt, "shld");
4645 case 0xac: /* shrd imm8, r, r/m */
4646 case 0xad: /* shrd cl, r, r/m */
4647 emulate_2op_cl(ctxt, "shrd");
4649 case 0xae: /* clflush */
4651 case 0xb6 ... 0xb7: /* movzx */
4652 ctxt->dst.bytes = ctxt->op_bytes;
4653 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
4654 : (u16) ctxt->src.val;
4656 case 0xbe ... 0xbf: /* movsx */
4657 ctxt->dst.bytes = ctxt->op_bytes;
4658 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
4659 (s16) ctxt->src.val;
4661 case 0xc0 ... 0xc1: /* xadd */
4662 emulate_2op_SrcV(ctxt, "add");
4663 /* Write back the register source. */
4664 ctxt->src.val = ctxt->dst.orig_val;
4665 write_register_operand(&ctxt->src);
4667 case 0xc3: /* movnti */
4668 ctxt->dst.bytes = ctxt->op_bytes;
4669 ctxt->dst.val = (ctxt->op_bytes == 4) ? (u32) ctxt->src.val :
4670 (u64) ctxt->src.val;
4673 goto cannot_emulate;
4676 if (rc != X86EMUL_CONTINUE)
4682 return EMULATION_FAILED;
4685 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
4687 invalidate_registers(ctxt);
4690 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
4692 writeback_registers(ctxt);