1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
27 #include <linux/stringify.h>
36 #define OpImplicit 1ull /* No generic decode */
37 #define OpReg 2ull /* Register */
38 #define OpMem 3ull /* Memory */
39 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
40 #define OpDI 5ull /* ES:DI/EDI/RDI */
41 #define OpMem64 6ull /* Memory, 64-bit */
42 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
43 #define OpDX 8ull /* DX register */
44 #define OpCL 9ull /* CL register (for shifts) */
45 #define OpImmByte 10ull /* 8-bit sign extended immediate */
46 #define OpOne 11ull /* Implied 1 */
47 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
48 #define OpMem16 13ull /* Memory operand (16-bit). */
49 #define OpMem32 14ull /* Memory operand (32-bit). */
50 #define OpImmU 15ull /* Immediate operand, zero extended */
51 #define OpSI 16ull /* SI/ESI/RSI */
52 #define OpImmFAddr 17ull /* Immediate far address */
53 #define OpMemFAddr 18ull /* Far address in memory */
54 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
55 #define OpES 20ull /* ES */
56 #define OpCS 21ull /* CS */
57 #define OpSS 22ull /* SS */
58 #define OpDS 23ull /* DS */
59 #define OpFS 24ull /* FS */
60 #define OpGS 25ull /* GS */
61 #define OpMem8 26ull /* 8-bit zero extended memory operand */
62 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
64 #define OpBits 5 /* Width of operand field */
65 #define OpMask ((1ull << OpBits) - 1)
68 * Opcode effective-address decode tables.
69 * Note that we only emulate instructions that have at least one memory
70 * operand (excluding implicit stack references). We assume that stack
71 * references and instruction fetches will never occur in special memory
72 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
76 /* Operand sizes: 8-bit operands or specified/overridden size. */
77 #define ByteOp (1<<0) /* 8-bit operands. */
78 /* Destination operand type. */
80 #define ImplicitOps (OpImplicit << DstShift)
81 #define DstReg (OpReg << DstShift)
82 #define DstMem (OpMem << DstShift)
83 #define DstAcc (OpAcc << DstShift)
84 #define DstDI (OpDI << DstShift)
85 #define DstMem64 (OpMem64 << DstShift)
86 #define DstImmUByte (OpImmUByte << DstShift)
87 #define DstDX (OpDX << DstShift)
88 #define DstMask (OpMask << DstShift)
89 /* Source operand type. */
91 #define SrcNone (OpNone << SrcShift)
92 #define SrcReg (OpReg << SrcShift)
93 #define SrcMem (OpMem << SrcShift)
94 #define SrcMem16 (OpMem16 << SrcShift)
95 #define SrcMem32 (OpMem32 << SrcShift)
96 #define SrcImm (OpImm << SrcShift)
97 #define SrcImmByte (OpImmByte << SrcShift)
98 #define SrcOne (OpOne << SrcShift)
99 #define SrcImmUByte (OpImmUByte << SrcShift)
100 #define SrcImmU (OpImmU << SrcShift)
101 #define SrcSI (OpSI << SrcShift)
102 #define SrcImmFAddr (OpImmFAddr << SrcShift)
103 #define SrcMemFAddr (OpMemFAddr << SrcShift)
104 #define SrcAcc (OpAcc << SrcShift)
105 #define SrcImmU16 (OpImmU16 << SrcShift)
106 #define SrcImm64 (OpImm64 << SrcShift)
107 #define SrcDX (OpDX << SrcShift)
108 #define SrcMem8 (OpMem8 << SrcShift)
109 #define SrcMask (OpMask << SrcShift)
110 #define BitOp (1<<11)
111 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
112 #define String (1<<13) /* String instruction (rep capable) */
113 #define Stack (1<<14) /* Stack instruction (push/pop) */
114 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
115 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
116 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
117 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
118 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
119 #define Escape (5<<15) /* Escape to coprocessor instruction */
120 #define Sse (1<<18) /* SSE Vector instruction */
121 /* Generic ModRM decode. */
122 #define ModRM (1<<19)
123 /* Destination is only written; never read. */
126 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
127 #define VendorSpecific (1<<22) /* Vendor specific instruction */
128 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
129 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
130 #define Undefined (1<<25) /* No Such Instruction */
131 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
132 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
134 #define PageTable (1 << 29) /* instruction used to write page table */
135 /* Source 2 operand type */
136 #define Src2Shift (30)
137 #define Src2None (OpNone << Src2Shift)
138 #define Src2CL (OpCL << Src2Shift)
139 #define Src2ImmByte (OpImmByte << Src2Shift)
140 #define Src2One (OpOne << Src2Shift)
141 #define Src2Imm (OpImm << Src2Shift)
142 #define Src2ES (OpES << Src2Shift)
143 #define Src2CS (OpCS << Src2Shift)
144 #define Src2SS (OpSS << Src2Shift)
145 #define Src2DS (OpDS << Src2Shift)
146 #define Src2FS (OpFS << Src2Shift)
147 #define Src2GS (OpGS << Src2Shift)
148 #define Src2Mask (OpMask << Src2Shift)
149 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
150 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
151 #define Unaligned ((u64)1 << 42) /* Explicitly unaligned (e.g. MOVDQU) */
152 #define Avx ((u64)1 << 43) /* Advanced Vector Extensions */
153 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
154 #define NoWrite ((u64)1 << 45) /* No writeback */
156 #define X2(x...) x, x
157 #define X3(x...) X2(x), x
158 #define X4(x...) X2(x), X2(x)
159 #define X5(x...) X4(x), x
160 #define X6(x...) X4(x), X2(x)
161 #define X7(x...) X4(x), X3(x)
162 #define X8(x...) X4(x), X4(x)
163 #define X16(x...) X8(x), X8(x)
165 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
166 #define FASTOP_SIZE 8
169 * fastop functions have a special calling convention:
171 * dst: [rdx]:rax (in/out)
174 * flags: rflags (in/out)
176 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
177 * different operand sizes can be reached by calculation, rather than a jump
178 * table (which would be bigger than the code).
180 * fastop functions are declared as taking a never-defined fastop parameter,
181 * so they can't be called from C directly.
190 int (*execute)(struct x86_emulate_ctxt *ctxt);
191 const struct opcode *group;
192 const struct group_dual *gdual;
193 const struct gprefix *gprefix;
194 const struct escape *esc;
195 void (*fastop)(struct fastop *fake);
197 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
201 struct opcode mod012[8];
202 struct opcode mod3[8];
206 struct opcode pfx_no;
207 struct opcode pfx_66;
208 struct opcode pfx_f2;
209 struct opcode pfx_f3;
214 struct opcode high[64];
217 /* EFLAGS bit definitions. */
218 #define EFLG_ID (1<<21)
219 #define EFLG_VIP (1<<20)
220 #define EFLG_VIF (1<<19)
221 #define EFLG_AC (1<<18)
222 #define EFLG_VM (1<<17)
223 #define EFLG_RF (1<<16)
224 #define EFLG_IOPL (3<<12)
225 #define EFLG_NT (1<<14)
226 #define EFLG_OF (1<<11)
227 #define EFLG_DF (1<<10)
228 #define EFLG_IF (1<<9)
229 #define EFLG_TF (1<<8)
230 #define EFLG_SF (1<<7)
231 #define EFLG_ZF (1<<6)
232 #define EFLG_AF (1<<4)
233 #define EFLG_PF (1<<2)
234 #define EFLG_CF (1<<0)
236 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
237 #define EFLG_RESERVED_ONE_MASK 2
239 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
241 if (!(ctxt->regs_valid & (1 << nr))) {
242 ctxt->regs_valid |= 1 << nr;
243 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
245 return ctxt->_regs[nr];
248 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
250 ctxt->regs_valid |= 1 << nr;
251 ctxt->regs_dirty |= 1 << nr;
252 return &ctxt->_regs[nr];
255 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
258 return reg_write(ctxt, nr);
261 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
265 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
266 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
269 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
271 ctxt->regs_dirty = 0;
272 ctxt->regs_valid = 0;
276 * Instruction emulation:
277 * Most instructions are emulated directly via a fragment of inline assembly
278 * code. This allows us to save/restore EFLAGS and thus very easily pick up
279 * any modified flags.
282 #if defined(CONFIG_X86_64)
283 #define _LO32 "k" /* force 32-bit operand */
284 #define _STK "%%rsp" /* stack pointer */
285 #elif defined(__i386__)
286 #define _LO32 "" /* force 32-bit operand */
287 #define _STK "%%esp" /* stack pointer */
291 * These EFLAGS bits are restored from saved value during emulation, and
292 * any changes are written back to the saved value after emulation.
294 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
296 /* Before executing instruction: restore necessary bits in EFLAGS. */
297 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
298 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
299 "movl %"_sav",%"_LO32 _tmp"; " \
302 "movl %"_msk",%"_LO32 _tmp"; " \
303 "andl %"_LO32 _tmp",("_STK"); " \
305 "notl %"_LO32 _tmp"; " \
306 "andl %"_LO32 _tmp",("_STK"); " \
307 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
309 "orl %"_LO32 _tmp",("_STK"); " \
313 /* After executing instruction: write-back necessary bits in EFLAGS. */
314 #define _POST_EFLAGS(_sav, _msk, _tmp) \
315 /* _sav |= EFLAGS & _msk; */ \
318 "andl %"_msk",%"_LO32 _tmp"; " \
319 "orl %"_LO32 _tmp",%"_sav"; "
327 #define ____emulate_2op(ctxt, _op, _x, _y, _suffix, _dsttype) \
329 __asm__ __volatile__ ( \
330 _PRE_EFLAGS("0", "4", "2") \
331 _op _suffix " %"_x"3,%1; " \
332 _POST_EFLAGS("0", "4", "2") \
333 : "=m" ((ctxt)->eflags), \
334 "+q" (*(_dsttype*)&(ctxt)->dst.val), \
336 : _y ((ctxt)->src.val), "i" (EFLAGS_MASK)); \
340 /* Raw emulation: instruction has two explicit operands. */
341 #define __emulate_2op_nobyte(ctxt,_op,_wx,_wy,_lx,_ly,_qx,_qy) \
343 unsigned long _tmp; \
345 switch ((ctxt)->dst.bytes) { \
347 ____emulate_2op(ctxt,_op,_wx,_wy,"w",u16); \
350 ____emulate_2op(ctxt,_op,_lx,_ly,"l",u32); \
353 ON64(____emulate_2op(ctxt,_op,_qx,_qy,"q",u64)); \
358 #define __emulate_2op(ctxt,_op,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
360 unsigned long _tmp; \
361 switch ((ctxt)->dst.bytes) { \
363 ____emulate_2op(ctxt,_op,_bx,_by,"b",u8); \
366 __emulate_2op_nobyte(ctxt, _op, \
367 _wx, _wy, _lx, _ly, _qx, _qy); \
372 /* Source operand is byte-sized and may be restricted to just %cl. */
373 #define emulate_2op_SrcB(ctxt, _op) \
374 __emulate_2op(ctxt, _op, "b", "c", "b", "c", "b", "c", "b", "c")
376 /* Source operand is byte, word, long or quad sized. */
377 #define emulate_2op_SrcV(ctxt, _op) \
378 __emulate_2op(ctxt, _op, "b", "q", "w", "r", _LO32, "r", "", "r")
380 /* Source operand is word, long or quad sized. */
381 #define emulate_2op_SrcV_nobyte(ctxt, _op) \
382 __emulate_2op_nobyte(ctxt, _op, "w", "r", _LO32, "r", "", "r")
384 /* Instruction has three operands and one operand is stored in ECX register */
385 #define __emulate_2op_cl(ctxt, _op, _suffix, _type) \
387 unsigned long _tmp; \
388 _type _clv = (ctxt)->src2.val; \
389 _type _srcv = (ctxt)->src.val; \
390 _type _dstv = (ctxt)->dst.val; \
392 __asm__ __volatile__ ( \
393 _PRE_EFLAGS("0", "5", "2") \
394 _op _suffix " %4,%1 \n" \
395 _POST_EFLAGS("0", "5", "2") \
396 : "=m" ((ctxt)->eflags), "+r" (_dstv), "=&r" (_tmp) \
397 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
400 (ctxt)->src2.val = (unsigned long) _clv; \
401 (ctxt)->src2.val = (unsigned long) _srcv; \
402 (ctxt)->dst.val = (unsigned long) _dstv; \
405 #define emulate_2op_cl(ctxt, _op) \
407 switch ((ctxt)->dst.bytes) { \
409 __emulate_2op_cl(ctxt, _op, "w", u16); \
412 __emulate_2op_cl(ctxt, _op, "l", u32); \
415 ON64(__emulate_2op_cl(ctxt, _op, "q", ulong)); \
420 #define __emulate_1op(ctxt, _op, _suffix) \
422 unsigned long _tmp; \
424 __asm__ __volatile__ ( \
425 _PRE_EFLAGS("0", "3", "2") \
426 _op _suffix " %1; " \
427 _POST_EFLAGS("0", "3", "2") \
428 : "=m" ((ctxt)->eflags), "+m" ((ctxt)->dst.val), \
430 : "i" (EFLAGS_MASK)); \
433 /* Instruction has only one explicit operand (no source operand). */
434 #define emulate_1op(ctxt, _op) \
436 switch ((ctxt)->dst.bytes) { \
437 case 1: __emulate_1op(ctxt, _op, "b"); break; \
438 case 2: __emulate_1op(ctxt, _op, "w"); break; \
439 case 4: __emulate_1op(ctxt, _op, "l"); break; \
440 case 8: ON64(__emulate_1op(ctxt, _op, "q")); break; \
444 #define FOP_ALIGN ".align " __stringify(FASTOP_SIZE) " \n\t"
445 #define FOP_RET "ret \n\t"
447 #define FOP_START(op) \
448 extern void em_##op(struct fastop *fake); \
449 asm(".pushsection .text, \"ax\" \n\t" \
450 ".global em_" #op " \n\t" \
457 #define FOP1E(op, dst) \
458 FOP_ALIGN #op " %" #dst " \n\t" FOP_RET
460 #define FASTOP1(op) \
465 ON64(FOP1E(op##q, rax)) \
468 #define FOP2E(op, dst, src) \
469 FOP_ALIGN #op " %" #src ", %" #dst " \n\t" FOP_RET
471 #define FASTOP2(op) \
473 FOP2E(op##b, al, bl) \
474 FOP2E(op##w, ax, bx) \
475 FOP2E(op##l, eax, ebx) \
476 ON64(FOP2E(op##q, rax, rbx)) \
479 #define __emulate_1op_rax_rdx(ctxt, _op, _suffix, _ex) \
481 unsigned long _tmp; \
482 ulong *rax = reg_rmw((ctxt), VCPU_REGS_RAX); \
483 ulong *rdx = reg_rmw((ctxt), VCPU_REGS_RDX); \
485 __asm__ __volatile__ ( \
486 _PRE_EFLAGS("0", "5", "1") \
488 _op _suffix " %6; " \
490 _POST_EFLAGS("0", "5", "1") \
491 ".pushsection .fixup,\"ax\" \n\t" \
492 "3: movb $1, %4 \n\t" \
495 _ASM_EXTABLE(1b, 3b) \
496 : "=m" ((ctxt)->eflags), "=&r" (_tmp), \
497 "+a" (*rax), "+d" (*rdx), "+qm"(_ex) \
498 : "i" (EFLAGS_MASK), "m" ((ctxt)->src.val)); \
501 /* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
502 #define emulate_1op_rax_rdx(ctxt, _op, _ex) \
504 switch((ctxt)->src.bytes) { \
506 __emulate_1op_rax_rdx(ctxt, _op, "b", _ex); \
509 __emulate_1op_rax_rdx(ctxt, _op, "w", _ex); \
512 __emulate_1op_rax_rdx(ctxt, _op, "l", _ex); \
515 __emulate_1op_rax_rdx(ctxt, _op, "q", _ex)); \
520 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
521 enum x86_intercept intercept,
522 enum x86_intercept_stage stage)
524 struct x86_instruction_info info = {
525 .intercept = intercept,
526 .rep_prefix = ctxt->rep_prefix,
527 .modrm_mod = ctxt->modrm_mod,
528 .modrm_reg = ctxt->modrm_reg,
529 .modrm_rm = ctxt->modrm_rm,
530 .src_val = ctxt->src.val64,
531 .src_bytes = ctxt->src.bytes,
532 .dst_bytes = ctxt->dst.bytes,
533 .ad_bytes = ctxt->ad_bytes,
534 .next_rip = ctxt->eip,
537 return ctxt->ops->intercept(ctxt, &info, stage);
540 static void assign_masked(ulong *dest, ulong src, ulong mask)
542 *dest = (*dest & ~mask) | (src & mask);
545 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
547 return (1UL << (ctxt->ad_bytes << 3)) - 1;
550 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
553 struct desc_struct ss;
555 if (ctxt->mode == X86EMUL_MODE_PROT64)
557 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
558 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
561 static int stack_size(struct x86_emulate_ctxt *ctxt)
563 return (__fls(stack_mask(ctxt)) + 1) >> 3;
566 /* Access/update address held in a register, based on addressing mode. */
567 static inline unsigned long
568 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
570 if (ctxt->ad_bytes == sizeof(unsigned long))
573 return reg & ad_mask(ctxt);
576 static inline unsigned long
577 register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
579 return address_mask(ctxt, reg);
582 static void masked_increment(ulong *reg, ulong mask, int inc)
584 assign_masked(reg, *reg + inc, mask);
588 register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
592 if (ctxt->ad_bytes == sizeof(unsigned long))
595 mask = ad_mask(ctxt);
596 masked_increment(reg, mask, inc);
599 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
601 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
604 static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
606 register_address_increment(ctxt, &ctxt->_eip, rel);
609 static u32 desc_limit_scaled(struct desc_struct *desc)
611 u32 limit = get_desc_limit(desc);
613 return desc->g ? (limit << 12) | 0xfff : limit;
616 static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
618 ctxt->has_seg_override = true;
619 ctxt->seg_override = seg;
622 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
624 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
627 return ctxt->ops->get_cached_segment_base(ctxt, seg);
630 static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
632 if (!ctxt->has_seg_override)
635 return ctxt->seg_override;
638 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
639 u32 error, bool valid)
641 ctxt->exception.vector = vec;
642 ctxt->exception.error_code = error;
643 ctxt->exception.error_code_valid = valid;
644 return X86EMUL_PROPAGATE_FAULT;
647 static int emulate_db(struct x86_emulate_ctxt *ctxt)
649 return emulate_exception(ctxt, DB_VECTOR, 0, false);
652 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
654 return emulate_exception(ctxt, GP_VECTOR, err, true);
657 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
659 return emulate_exception(ctxt, SS_VECTOR, err, true);
662 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
664 return emulate_exception(ctxt, UD_VECTOR, 0, false);
667 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
669 return emulate_exception(ctxt, TS_VECTOR, err, true);
672 static int emulate_de(struct x86_emulate_ctxt *ctxt)
674 return emulate_exception(ctxt, DE_VECTOR, 0, false);
677 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
679 return emulate_exception(ctxt, NM_VECTOR, 0, false);
682 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
685 struct desc_struct desc;
687 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
691 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
696 struct desc_struct desc;
698 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
699 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
703 * x86 defines three classes of vector instructions: explicitly
704 * aligned, explicitly unaligned, and the rest, which change behaviour
705 * depending on whether they're AVX encoded or not.
707 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
708 * subject to the same check.
710 static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
712 if (likely(size < 16))
715 if (ctxt->d & Aligned)
717 else if (ctxt->d & Unaligned)
719 else if (ctxt->d & Avx)
725 static int __linearize(struct x86_emulate_ctxt *ctxt,
726 struct segmented_address addr,
727 unsigned size, bool write, bool fetch,
730 struct desc_struct desc;
737 la = seg_base(ctxt, addr.seg) + addr.ea;
738 switch (ctxt->mode) {
739 case X86EMUL_MODE_PROT64:
740 if (((signed long)la << 16) >> 16 != la)
741 return emulate_gp(ctxt, 0);
744 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
748 /* code segment in protected mode or read-only data segment */
749 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
750 || !(desc.type & 2)) && write)
752 /* unreadable code segment */
753 if (!fetch && (desc.type & 8) && !(desc.type & 2))
755 lim = desc_limit_scaled(&desc);
756 if ((desc.type & 8) || !(desc.type & 4)) {
757 /* expand-up segment */
758 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
761 /* expand-down segment */
762 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
764 lim = desc.d ? 0xffffffff : 0xffff;
765 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
768 cpl = ctxt->ops->cpl(ctxt);
769 if (!(desc.type & 8)) {
773 } else if ((desc.type & 8) && !(desc.type & 4)) {
774 /* nonconforming code segment */
777 } else if ((desc.type & 8) && (desc.type & 4)) {
778 /* conforming code segment */
784 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
786 if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
787 return emulate_gp(ctxt, 0);
789 return X86EMUL_CONTINUE;
791 if (addr.seg == VCPU_SREG_SS)
792 return emulate_ss(ctxt, sel);
794 return emulate_gp(ctxt, sel);
797 static int linearize(struct x86_emulate_ctxt *ctxt,
798 struct segmented_address addr,
799 unsigned size, bool write,
802 return __linearize(ctxt, addr, size, write, false, linear);
806 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
807 struct segmented_address addr,
814 rc = linearize(ctxt, addr, size, false, &linear);
815 if (rc != X86EMUL_CONTINUE)
817 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
821 * Fetch the next byte of the instruction being emulated which is pointed to
822 * by ctxt->_eip, then increment ctxt->_eip.
824 * Also prefetch the remaining bytes of the instruction without crossing page
825 * boundary if they are not in fetch_cache yet.
827 static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
829 struct fetch_cache *fc = &ctxt->fetch;
833 if (ctxt->_eip == fc->end) {
834 unsigned long linear;
835 struct segmented_address addr = { .seg = VCPU_SREG_CS,
837 cur_size = fc->end - fc->start;
838 size = min(15UL - cur_size,
839 PAGE_SIZE - offset_in_page(ctxt->_eip));
840 rc = __linearize(ctxt, addr, size, false, true, &linear);
841 if (unlikely(rc != X86EMUL_CONTINUE))
843 rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
844 size, &ctxt->exception);
845 if (unlikely(rc != X86EMUL_CONTINUE))
849 *dest = fc->data[ctxt->_eip - fc->start];
851 return X86EMUL_CONTINUE;
854 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
855 void *dest, unsigned size)
859 /* x86 instructions are limited to 15 bytes. */
860 if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
861 return X86EMUL_UNHANDLEABLE;
863 rc = do_insn_fetch_byte(ctxt, dest++);
864 if (rc != X86EMUL_CONTINUE)
867 return X86EMUL_CONTINUE;
870 /* Fetch next part of the instruction being emulated. */
871 #define insn_fetch(_type, _ctxt) \
872 ({ unsigned long _x; \
873 rc = do_insn_fetch(_ctxt, &_x, sizeof(_type)); \
874 if (rc != X86EMUL_CONTINUE) \
879 #define insn_fetch_arr(_arr, _size, _ctxt) \
880 ({ rc = do_insn_fetch(_ctxt, _arr, (_size)); \
881 if (rc != X86EMUL_CONTINUE) \
886 * Given the 'reg' portion of a ModRM byte, and a register block, return a
887 * pointer into the block that addresses the relevant register.
888 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
890 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
895 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
896 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
898 p = reg_rmw(ctxt, modrm_reg);
902 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
903 struct segmented_address addr,
904 u16 *size, unsigned long *address, int op_bytes)
911 rc = segmented_read_std(ctxt, addr, size, 2);
912 if (rc != X86EMUL_CONTINUE)
915 rc = segmented_read_std(ctxt, addr, address, op_bytes);
919 static int test_cc(unsigned int condition, unsigned int flags)
923 switch ((condition & 15) >> 1) {
925 rc |= (flags & EFLG_OF);
927 case 1: /* b/c/nae */
928 rc |= (flags & EFLG_CF);
931 rc |= (flags & EFLG_ZF);
934 rc |= (flags & (EFLG_CF|EFLG_ZF));
937 rc |= (flags & EFLG_SF);
940 rc |= (flags & EFLG_PF);
943 rc |= (flags & EFLG_ZF);
946 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
950 /* Odd condition identifiers (lsb == 1) have inverted sense. */
951 return (!!rc ^ (condition & 1));
954 static void fetch_register_operand(struct operand *op)
958 op->val = *(u8 *)op->addr.reg;
961 op->val = *(u16 *)op->addr.reg;
964 op->val = *(u32 *)op->addr.reg;
967 op->val = *(u64 *)op->addr.reg;
972 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
974 ctxt->ops->get_fpu(ctxt);
976 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
977 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
978 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
979 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
980 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
981 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
982 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
983 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
985 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
986 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
987 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
988 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
989 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
990 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
991 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
992 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
996 ctxt->ops->put_fpu(ctxt);
999 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
1002 ctxt->ops->get_fpu(ctxt);
1004 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
1005 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
1006 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
1007 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
1008 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
1009 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
1010 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
1011 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
1012 #ifdef CONFIG_X86_64
1013 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
1014 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
1015 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
1016 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
1017 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
1018 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
1019 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
1020 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
1024 ctxt->ops->put_fpu(ctxt);
1027 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1029 ctxt->ops->get_fpu(ctxt);
1031 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
1032 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
1033 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
1034 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
1035 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
1036 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
1037 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
1038 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
1041 ctxt->ops->put_fpu(ctxt);
1044 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1046 ctxt->ops->get_fpu(ctxt);
1048 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
1049 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
1050 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
1051 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
1052 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
1053 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
1054 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
1055 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
1058 ctxt->ops->put_fpu(ctxt);
1061 static int em_fninit(struct x86_emulate_ctxt *ctxt)
1063 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1064 return emulate_nm(ctxt);
1066 ctxt->ops->get_fpu(ctxt);
1067 asm volatile("fninit");
1068 ctxt->ops->put_fpu(ctxt);
1069 return X86EMUL_CONTINUE;
1072 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
1076 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1077 return emulate_nm(ctxt);
1079 ctxt->ops->get_fpu(ctxt);
1080 asm volatile("fnstcw %0": "+m"(fcw));
1081 ctxt->ops->put_fpu(ctxt);
1083 /* force 2 byte destination */
1084 ctxt->dst.bytes = 2;
1085 ctxt->dst.val = fcw;
1087 return X86EMUL_CONTINUE;
1090 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1094 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1095 return emulate_nm(ctxt);
1097 ctxt->ops->get_fpu(ctxt);
1098 asm volatile("fnstsw %0": "+m"(fsw));
1099 ctxt->ops->put_fpu(ctxt);
1101 /* force 2 byte destination */
1102 ctxt->dst.bytes = 2;
1103 ctxt->dst.val = fsw;
1105 return X86EMUL_CONTINUE;
1108 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1111 unsigned reg = ctxt->modrm_reg;
1112 int highbyte_regs = ctxt->rex_prefix == 0;
1114 if (!(ctxt->d & ModRM))
1115 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1117 if (ctxt->d & Sse) {
1121 read_sse_reg(ctxt, &op->vec_val, reg);
1124 if (ctxt->d & Mmx) {
1133 if (ctxt->d & ByteOp) {
1134 op->addr.reg = decode_register(ctxt, reg, highbyte_regs);
1137 op->addr.reg = decode_register(ctxt, reg, 0);
1138 op->bytes = ctxt->op_bytes;
1140 fetch_register_operand(op);
1141 op->orig_val = op->val;
1144 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1146 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1147 ctxt->modrm_seg = VCPU_SREG_SS;
1150 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1154 int index_reg = 0, base_reg = 0, scale;
1155 int rc = X86EMUL_CONTINUE;
1158 if (ctxt->rex_prefix) {
1159 ctxt->modrm_reg = (ctxt->rex_prefix & 4) << 1; /* REX.R */
1160 index_reg = (ctxt->rex_prefix & 2) << 2; /* REX.X */
1161 ctxt->modrm_rm = base_reg = (ctxt->rex_prefix & 1) << 3; /* REG.B */
1164 ctxt->modrm_mod |= (ctxt->modrm & 0xc0) >> 6;
1165 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1166 ctxt->modrm_rm |= (ctxt->modrm & 0x07);
1167 ctxt->modrm_seg = VCPU_SREG_DS;
1169 if (ctxt->modrm_mod == 3) {
1171 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1172 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm, ctxt->d & ByteOp);
1173 if (ctxt->d & Sse) {
1176 op->addr.xmm = ctxt->modrm_rm;
1177 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1180 if (ctxt->d & Mmx) {
1183 op->addr.xmm = ctxt->modrm_rm & 7;
1186 fetch_register_operand(op);
1192 if (ctxt->ad_bytes == 2) {
1193 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1194 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1195 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1196 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1198 /* 16-bit ModR/M decode. */
1199 switch (ctxt->modrm_mod) {
1201 if (ctxt->modrm_rm == 6)
1202 modrm_ea += insn_fetch(u16, ctxt);
1205 modrm_ea += insn_fetch(s8, ctxt);
1208 modrm_ea += insn_fetch(u16, ctxt);
1211 switch (ctxt->modrm_rm) {
1213 modrm_ea += bx + si;
1216 modrm_ea += bx + di;
1219 modrm_ea += bp + si;
1222 modrm_ea += bp + di;
1231 if (ctxt->modrm_mod != 0)
1238 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1239 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1240 ctxt->modrm_seg = VCPU_SREG_SS;
1241 modrm_ea = (u16)modrm_ea;
1243 /* 32/64-bit ModR/M decode. */
1244 if ((ctxt->modrm_rm & 7) == 4) {
1245 sib = insn_fetch(u8, ctxt);
1246 index_reg |= (sib >> 3) & 7;
1247 base_reg |= sib & 7;
1250 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1251 modrm_ea += insn_fetch(s32, ctxt);
1253 modrm_ea += reg_read(ctxt, base_reg);
1254 adjust_modrm_seg(ctxt, base_reg);
1257 modrm_ea += reg_read(ctxt, index_reg) << scale;
1258 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1259 if (ctxt->mode == X86EMUL_MODE_PROT64)
1260 ctxt->rip_relative = 1;
1262 base_reg = ctxt->modrm_rm;
1263 modrm_ea += reg_read(ctxt, base_reg);
1264 adjust_modrm_seg(ctxt, base_reg);
1266 switch (ctxt->modrm_mod) {
1268 if (ctxt->modrm_rm == 5)
1269 modrm_ea += insn_fetch(s32, ctxt);
1272 modrm_ea += insn_fetch(s8, ctxt);
1275 modrm_ea += insn_fetch(s32, ctxt);
1279 op->addr.mem.ea = modrm_ea;
1284 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1287 int rc = X86EMUL_CONTINUE;
1290 switch (ctxt->ad_bytes) {
1292 op->addr.mem.ea = insn_fetch(u16, ctxt);
1295 op->addr.mem.ea = insn_fetch(u32, ctxt);
1298 op->addr.mem.ea = insn_fetch(u64, ctxt);
1305 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1309 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1310 mask = ~(ctxt->dst.bytes * 8 - 1);
1312 if (ctxt->src.bytes == 2)
1313 sv = (s16)ctxt->src.val & (s16)mask;
1314 else if (ctxt->src.bytes == 4)
1315 sv = (s32)ctxt->src.val & (s32)mask;
1317 ctxt->dst.addr.mem.ea += (sv >> 3);
1320 /* only subword offset */
1321 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1324 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1325 unsigned long addr, void *dest, unsigned size)
1328 struct read_cache *mc = &ctxt->mem_read;
1330 if (mc->pos < mc->end)
1333 WARN_ON((mc->end + size) >= sizeof(mc->data));
1335 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1337 if (rc != X86EMUL_CONTINUE)
1343 memcpy(dest, mc->data + mc->pos, size);
1345 return X86EMUL_CONTINUE;
1348 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1349 struct segmented_address addr,
1356 rc = linearize(ctxt, addr, size, false, &linear);
1357 if (rc != X86EMUL_CONTINUE)
1359 return read_emulated(ctxt, linear, data, size);
1362 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1363 struct segmented_address addr,
1370 rc = linearize(ctxt, addr, size, true, &linear);
1371 if (rc != X86EMUL_CONTINUE)
1373 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1377 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1378 struct segmented_address addr,
1379 const void *orig_data, const void *data,
1385 rc = linearize(ctxt, addr, size, true, &linear);
1386 if (rc != X86EMUL_CONTINUE)
1388 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1389 size, &ctxt->exception);
1392 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1393 unsigned int size, unsigned short port,
1396 struct read_cache *rc = &ctxt->io_read;
1398 if (rc->pos == rc->end) { /* refill pio read ahead */
1399 unsigned int in_page, n;
1400 unsigned int count = ctxt->rep_prefix ?
1401 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1402 in_page = (ctxt->eflags & EFLG_DF) ?
1403 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1404 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1405 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1409 rc->pos = rc->end = 0;
1410 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1415 if (ctxt->rep_prefix && !(ctxt->eflags & EFLG_DF)) {
1416 ctxt->dst.data = rc->data + rc->pos;
1417 ctxt->dst.type = OP_MEM_STR;
1418 ctxt->dst.count = (rc->end - rc->pos) / size;
1421 memcpy(dest, rc->data + rc->pos, size);
1427 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1428 u16 index, struct desc_struct *desc)
1433 ctxt->ops->get_idt(ctxt, &dt);
1435 if (dt.size < index * 8 + 7)
1436 return emulate_gp(ctxt, index << 3 | 0x2);
1438 addr = dt.address + index * 8;
1439 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1443 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1444 u16 selector, struct desc_ptr *dt)
1446 const struct x86_emulate_ops *ops = ctxt->ops;
1448 if (selector & 1 << 2) {
1449 struct desc_struct desc;
1452 memset (dt, 0, sizeof *dt);
1453 if (!ops->get_segment(ctxt, &sel, &desc, NULL, VCPU_SREG_LDTR))
1456 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1457 dt->address = get_desc_base(&desc);
1459 ops->get_gdt(ctxt, dt);
1462 /* allowed just for 8 bytes segments */
1463 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1464 u16 selector, struct desc_struct *desc,
1468 u16 index = selector >> 3;
1471 get_descriptor_table_ptr(ctxt, selector, &dt);
1473 if (dt.size < index * 8 + 7)
1474 return emulate_gp(ctxt, selector & 0xfffc);
1476 *desc_addr_p = addr = dt.address + index * 8;
1477 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1481 /* allowed just for 8 bytes segments */
1482 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1483 u16 selector, struct desc_struct *desc)
1486 u16 index = selector >> 3;
1489 get_descriptor_table_ptr(ctxt, selector, &dt);
1491 if (dt.size < index * 8 + 7)
1492 return emulate_gp(ctxt, selector & 0xfffc);
1494 addr = dt.address + index * 8;
1495 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1499 /* Does not support long mode */
1500 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1501 u16 selector, int seg)
1503 struct desc_struct seg_desc, old_desc;
1505 unsigned err_vec = GP_VECTOR;
1507 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1512 memset(&seg_desc, 0, sizeof seg_desc);
1514 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1515 || ctxt->mode == X86EMUL_MODE_REAL) {
1516 /* set real mode segment descriptor */
1517 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1518 set_desc_base(&seg_desc, selector << 4);
1523 cpl = ctxt->ops->cpl(ctxt);
1525 /* NULL selector is not valid for TR, CS and SS (except for long mode) */
1526 if ((seg == VCPU_SREG_CS
1527 || (seg == VCPU_SREG_SS
1528 && (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl))
1529 || seg == VCPU_SREG_TR)
1533 /* TR should be in GDT only */
1534 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1537 if (null_selector) /* for NULL selector skip all following checks */
1540 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1541 if (ret != X86EMUL_CONTINUE)
1544 err_code = selector & 0xfffc;
1545 err_vec = GP_VECTOR;
1547 /* can't load system descriptor into segment selector */
1548 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1552 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1561 * segment is not a writable data segment or segment
1562 * selector's RPL != CPL or segment selector's RPL != CPL
1564 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1568 if (!(seg_desc.type & 8))
1571 if (seg_desc.type & 4) {
1577 if (rpl > cpl || dpl != cpl)
1580 /* CS(RPL) <- CPL */
1581 selector = (selector & 0xfffc) | cpl;
1584 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1586 old_desc = seg_desc;
1587 seg_desc.type |= 2; /* busy */
1588 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1589 sizeof(seg_desc), &ctxt->exception);
1590 if (ret != X86EMUL_CONTINUE)
1593 case VCPU_SREG_LDTR:
1594 if (seg_desc.s || seg_desc.type != 2)
1597 default: /* DS, ES, FS, or GS */
1599 * segment is not a data or readable code segment or
1600 * ((segment is a data or nonconforming code segment)
1601 * and (both RPL and CPL > DPL))
1603 if ((seg_desc.type & 0xa) == 0x8 ||
1604 (((seg_desc.type & 0xc) != 0xc) &&
1605 (rpl > dpl && cpl > dpl)))
1611 /* mark segment as accessed */
1613 ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1614 if (ret != X86EMUL_CONTINUE)
1618 ctxt->ops->set_segment(ctxt, selector, &seg_desc, 0, seg);
1619 return X86EMUL_CONTINUE;
1621 emulate_exception(ctxt, err_vec, err_code, true);
1622 return X86EMUL_PROPAGATE_FAULT;
1625 static void write_register_operand(struct operand *op)
1627 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1628 switch (op->bytes) {
1630 *(u8 *)op->addr.reg = (u8)op->val;
1633 *(u16 *)op->addr.reg = (u16)op->val;
1636 *op->addr.reg = (u32)op->val;
1637 break; /* 64b: zero-extend */
1639 *op->addr.reg = op->val;
1644 static int writeback(struct x86_emulate_ctxt *ctxt)
1648 if (ctxt->d & NoWrite)
1649 return X86EMUL_CONTINUE;
1651 switch (ctxt->dst.type) {
1653 write_register_operand(&ctxt->dst);
1656 if (ctxt->lock_prefix)
1657 rc = segmented_cmpxchg(ctxt,
1659 &ctxt->dst.orig_val,
1663 rc = segmented_write(ctxt,
1667 if (rc != X86EMUL_CONTINUE)
1671 rc = segmented_write(ctxt,
1674 ctxt->dst.bytes * ctxt->dst.count);
1675 if (rc != X86EMUL_CONTINUE)
1679 write_sse_reg(ctxt, &ctxt->dst.vec_val, ctxt->dst.addr.xmm);
1682 write_mmx_reg(ctxt, &ctxt->dst.mm_val, ctxt->dst.addr.mm);
1690 return X86EMUL_CONTINUE;
1693 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1695 struct segmented_address addr;
1697 rsp_increment(ctxt, -bytes);
1698 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1699 addr.seg = VCPU_SREG_SS;
1701 return segmented_write(ctxt, addr, data, bytes);
1704 static int em_push(struct x86_emulate_ctxt *ctxt)
1706 /* Disable writeback. */
1707 ctxt->dst.type = OP_NONE;
1708 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1711 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1712 void *dest, int len)
1715 struct segmented_address addr;
1717 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1718 addr.seg = VCPU_SREG_SS;
1719 rc = segmented_read(ctxt, addr, dest, len);
1720 if (rc != X86EMUL_CONTINUE)
1723 rsp_increment(ctxt, len);
1727 static int em_pop(struct x86_emulate_ctxt *ctxt)
1729 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1732 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1733 void *dest, int len)
1736 unsigned long val, change_mask;
1737 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1738 int cpl = ctxt->ops->cpl(ctxt);
1740 rc = emulate_pop(ctxt, &val, len);
1741 if (rc != X86EMUL_CONTINUE)
1744 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1745 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1747 switch(ctxt->mode) {
1748 case X86EMUL_MODE_PROT64:
1749 case X86EMUL_MODE_PROT32:
1750 case X86EMUL_MODE_PROT16:
1752 change_mask |= EFLG_IOPL;
1754 change_mask |= EFLG_IF;
1756 case X86EMUL_MODE_VM86:
1758 return emulate_gp(ctxt, 0);
1759 change_mask |= EFLG_IF;
1761 default: /* real mode */
1762 change_mask |= (EFLG_IOPL | EFLG_IF);
1766 *(unsigned long *)dest =
1767 (ctxt->eflags & ~change_mask) | (val & change_mask);
1772 static int em_popf(struct x86_emulate_ctxt *ctxt)
1774 ctxt->dst.type = OP_REG;
1775 ctxt->dst.addr.reg = &ctxt->eflags;
1776 ctxt->dst.bytes = ctxt->op_bytes;
1777 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1780 static int em_enter(struct x86_emulate_ctxt *ctxt)
1783 unsigned frame_size = ctxt->src.val;
1784 unsigned nesting_level = ctxt->src2.val & 31;
1788 return X86EMUL_UNHANDLEABLE;
1790 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1791 rc = push(ctxt, &rbp, stack_size(ctxt));
1792 if (rc != X86EMUL_CONTINUE)
1794 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1796 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1797 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1799 return X86EMUL_CONTINUE;
1802 static int em_leave(struct x86_emulate_ctxt *ctxt)
1804 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1806 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1809 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1811 int seg = ctxt->src2.val;
1813 ctxt->src.val = get_segment_selector(ctxt, seg);
1815 return em_push(ctxt);
1818 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1820 int seg = ctxt->src2.val;
1821 unsigned long selector;
1824 rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1825 if (rc != X86EMUL_CONTINUE)
1828 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1832 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1834 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1835 int rc = X86EMUL_CONTINUE;
1836 int reg = VCPU_REGS_RAX;
1838 while (reg <= VCPU_REGS_RDI) {
1839 (reg == VCPU_REGS_RSP) ?
1840 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
1843 if (rc != X86EMUL_CONTINUE)
1852 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1854 ctxt->src.val = (unsigned long)ctxt->eflags;
1855 return em_push(ctxt);
1858 static int em_popa(struct x86_emulate_ctxt *ctxt)
1860 int rc = X86EMUL_CONTINUE;
1861 int reg = VCPU_REGS_RDI;
1863 while (reg >= VCPU_REGS_RAX) {
1864 if (reg == VCPU_REGS_RSP) {
1865 rsp_increment(ctxt, ctxt->op_bytes);
1869 rc = emulate_pop(ctxt, reg_rmw(ctxt, reg), ctxt->op_bytes);
1870 if (rc != X86EMUL_CONTINUE)
1877 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1879 const struct x86_emulate_ops *ops = ctxt->ops;
1886 /* TODO: Add limit checks */
1887 ctxt->src.val = ctxt->eflags;
1889 if (rc != X86EMUL_CONTINUE)
1892 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1894 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1896 if (rc != X86EMUL_CONTINUE)
1899 ctxt->src.val = ctxt->_eip;
1901 if (rc != X86EMUL_CONTINUE)
1904 ops->get_idt(ctxt, &dt);
1906 eip_addr = dt.address + (irq << 2);
1907 cs_addr = dt.address + (irq << 2) + 2;
1909 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1910 if (rc != X86EMUL_CONTINUE)
1913 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1914 if (rc != X86EMUL_CONTINUE)
1917 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1918 if (rc != X86EMUL_CONTINUE)
1926 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1930 invalidate_registers(ctxt);
1931 rc = __emulate_int_real(ctxt, irq);
1932 if (rc == X86EMUL_CONTINUE)
1933 writeback_registers(ctxt);
1937 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1939 switch(ctxt->mode) {
1940 case X86EMUL_MODE_REAL:
1941 return __emulate_int_real(ctxt, irq);
1942 case X86EMUL_MODE_VM86:
1943 case X86EMUL_MODE_PROT16:
1944 case X86EMUL_MODE_PROT32:
1945 case X86EMUL_MODE_PROT64:
1947 /* Protected mode interrupts unimplemented yet */
1948 return X86EMUL_UNHANDLEABLE;
1952 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
1954 int rc = X86EMUL_CONTINUE;
1955 unsigned long temp_eip = 0;
1956 unsigned long temp_eflags = 0;
1957 unsigned long cs = 0;
1958 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1959 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1960 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1961 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1963 /* TODO: Add stack limit check */
1965 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
1967 if (rc != X86EMUL_CONTINUE)
1970 if (temp_eip & ~0xffff)
1971 return emulate_gp(ctxt, 0);
1973 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1975 if (rc != X86EMUL_CONTINUE)
1978 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
1980 if (rc != X86EMUL_CONTINUE)
1983 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1985 if (rc != X86EMUL_CONTINUE)
1988 ctxt->_eip = temp_eip;
1991 if (ctxt->op_bytes == 4)
1992 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1993 else if (ctxt->op_bytes == 2) {
1994 ctxt->eflags &= ~0xffff;
1995 ctxt->eflags |= temp_eflags;
1998 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1999 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
2004 static int em_iret(struct x86_emulate_ctxt *ctxt)
2006 switch(ctxt->mode) {
2007 case X86EMUL_MODE_REAL:
2008 return emulate_iret_real(ctxt);
2009 case X86EMUL_MODE_VM86:
2010 case X86EMUL_MODE_PROT16:
2011 case X86EMUL_MODE_PROT32:
2012 case X86EMUL_MODE_PROT64:
2014 /* iret from protected mode unimplemented yet */
2015 return X86EMUL_UNHANDLEABLE;
2019 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
2024 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2026 rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
2027 if (rc != X86EMUL_CONTINUE)
2031 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2032 return X86EMUL_CONTINUE;
2035 static int em_grp2(struct x86_emulate_ctxt *ctxt)
2037 switch (ctxt->modrm_reg) {
2039 emulate_2op_SrcB(ctxt, "rol");
2042 emulate_2op_SrcB(ctxt, "ror");
2045 emulate_2op_SrcB(ctxt, "rcl");
2048 emulate_2op_SrcB(ctxt, "rcr");
2050 case 4: /* sal/shl */
2051 case 6: /* sal/shl */
2052 emulate_2op_SrcB(ctxt, "sal");
2055 emulate_2op_SrcB(ctxt, "shr");
2058 emulate_2op_SrcB(ctxt, "sar");
2061 return X86EMUL_CONTINUE;
2067 static int em_mul_ex(struct x86_emulate_ctxt *ctxt)
2071 emulate_1op_rax_rdx(ctxt, "mul", ex);
2072 return X86EMUL_CONTINUE;
2075 static int em_imul_ex(struct x86_emulate_ctxt *ctxt)
2079 emulate_1op_rax_rdx(ctxt, "imul", ex);
2080 return X86EMUL_CONTINUE;
2083 static int em_div_ex(struct x86_emulate_ctxt *ctxt)
2087 emulate_1op_rax_rdx(ctxt, "div", de);
2089 return emulate_de(ctxt);
2090 return X86EMUL_CONTINUE;
2093 static int em_idiv_ex(struct x86_emulate_ctxt *ctxt)
2097 emulate_1op_rax_rdx(ctxt, "idiv", de);
2099 return emulate_de(ctxt);
2100 return X86EMUL_CONTINUE;
2103 static int em_grp45(struct x86_emulate_ctxt *ctxt)
2105 int rc = X86EMUL_CONTINUE;
2107 switch (ctxt->modrm_reg) {
2109 emulate_1op(ctxt, "inc");
2112 emulate_1op(ctxt, "dec");
2114 case 2: /* call near abs */ {
2116 old_eip = ctxt->_eip;
2117 ctxt->_eip = ctxt->src.val;
2118 ctxt->src.val = old_eip;
2122 case 4: /* jmp abs */
2123 ctxt->_eip = ctxt->src.val;
2125 case 5: /* jmp far */
2126 rc = em_jmp_far(ctxt);
2135 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2137 u64 old = ctxt->dst.orig_val64;
2139 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
2140 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2141 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2142 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2143 ctxt->eflags &= ~EFLG_ZF;
2145 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2146 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2148 ctxt->eflags |= EFLG_ZF;
2150 return X86EMUL_CONTINUE;
2153 static int em_ret(struct x86_emulate_ctxt *ctxt)
2155 ctxt->dst.type = OP_REG;
2156 ctxt->dst.addr.reg = &ctxt->_eip;
2157 ctxt->dst.bytes = ctxt->op_bytes;
2158 return em_pop(ctxt);
2161 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2166 rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
2167 if (rc != X86EMUL_CONTINUE)
2169 if (ctxt->op_bytes == 4)
2170 ctxt->_eip = (u32)ctxt->_eip;
2171 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2172 if (rc != X86EMUL_CONTINUE)
2174 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2178 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2180 /* Save real source value, then compare EAX against destination. */
2181 ctxt->src.orig_val = ctxt->src.val;
2182 ctxt->src.val = reg_read(ctxt, VCPU_REGS_RAX);
2183 emulate_2op_SrcV(ctxt, "cmp");
2185 if (ctxt->eflags & EFLG_ZF) {
2186 /* Success: write back to memory. */
2187 ctxt->dst.val = ctxt->src.orig_val;
2189 /* Failure: write the value we saw to EAX. */
2190 ctxt->dst.type = OP_REG;
2191 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2193 return X86EMUL_CONTINUE;
2196 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2198 int seg = ctxt->src2.val;
2202 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2204 rc = load_segment_descriptor(ctxt, sel, seg);
2205 if (rc != X86EMUL_CONTINUE)
2208 ctxt->dst.val = ctxt->src.val;
2213 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2214 struct desc_struct *cs, struct desc_struct *ss)
2216 cs->l = 0; /* will be adjusted later */
2217 set_desc_base(cs, 0); /* flat segment */
2218 cs->g = 1; /* 4kb granularity */
2219 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2220 cs->type = 0x0b; /* Read, Execute, Accessed */
2222 cs->dpl = 0; /* will be adjusted later */
2227 set_desc_base(ss, 0); /* flat segment */
2228 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2229 ss->g = 1; /* 4kb granularity */
2231 ss->type = 0x03; /* Read/Write, Accessed */
2232 ss->d = 1; /* 32bit stack segment */
2239 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2241 u32 eax, ebx, ecx, edx;
2244 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2245 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2246 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2247 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2250 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2252 const struct x86_emulate_ops *ops = ctxt->ops;
2253 u32 eax, ebx, ecx, edx;
2256 * syscall should always be enabled in longmode - so only become
2257 * vendor specific (cpuid) if other modes are active...
2259 if (ctxt->mode == X86EMUL_MODE_PROT64)
2264 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2266 * Intel ("GenuineIntel")
2267 * remark: Intel CPUs only support "syscall" in 64bit
2268 * longmode. Also an 64bit guest with a
2269 * 32bit compat-app running will #UD !! While this
2270 * behaviour can be fixed (by emulating) into AMD
2271 * response - CPUs of AMD can't behave like Intel.
2273 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2274 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2275 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2278 /* AMD ("AuthenticAMD") */
2279 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2280 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2281 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2284 /* AMD ("AMDisbetter!") */
2285 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2286 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2287 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2290 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2294 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2296 const struct x86_emulate_ops *ops = ctxt->ops;
2297 struct desc_struct cs, ss;
2302 /* syscall is not available in real mode */
2303 if (ctxt->mode == X86EMUL_MODE_REAL ||
2304 ctxt->mode == X86EMUL_MODE_VM86)
2305 return emulate_ud(ctxt);
2307 if (!(em_syscall_is_enabled(ctxt)))
2308 return emulate_ud(ctxt);
2310 ops->get_msr(ctxt, MSR_EFER, &efer);
2311 setup_syscalls_segments(ctxt, &cs, &ss);
2313 if (!(efer & EFER_SCE))
2314 return emulate_ud(ctxt);
2316 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2318 cs_sel = (u16)(msr_data & 0xfffc);
2319 ss_sel = (u16)(msr_data + 8);
2321 if (efer & EFER_LMA) {
2325 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2326 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2328 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2329 if (efer & EFER_LMA) {
2330 #ifdef CONFIG_X86_64
2331 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags & ~EFLG_RF;
2334 ctxt->mode == X86EMUL_MODE_PROT64 ?
2335 MSR_LSTAR : MSR_CSTAR, &msr_data);
2336 ctxt->_eip = msr_data;
2338 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2339 ctxt->eflags &= ~(msr_data | EFLG_RF);
2343 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2344 ctxt->_eip = (u32)msr_data;
2346 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2349 return X86EMUL_CONTINUE;
2352 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2354 const struct x86_emulate_ops *ops = ctxt->ops;
2355 struct desc_struct cs, ss;
2360 ops->get_msr(ctxt, MSR_EFER, &efer);
2361 /* inject #GP if in real mode */
2362 if (ctxt->mode == X86EMUL_MODE_REAL)
2363 return emulate_gp(ctxt, 0);
2366 * Not recognized on AMD in compat mode (but is recognized in legacy
2369 if ((ctxt->mode == X86EMUL_MODE_PROT32) && (efer & EFER_LMA)
2370 && !vendor_intel(ctxt))
2371 return emulate_ud(ctxt);
2373 /* XXX sysenter/sysexit have not been tested in 64bit mode.
2374 * Therefore, we inject an #UD.
2376 if (ctxt->mode == X86EMUL_MODE_PROT64)
2377 return emulate_ud(ctxt);
2379 setup_syscalls_segments(ctxt, &cs, &ss);
2381 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2382 switch (ctxt->mode) {
2383 case X86EMUL_MODE_PROT32:
2384 if ((msr_data & 0xfffc) == 0x0)
2385 return emulate_gp(ctxt, 0);
2387 case X86EMUL_MODE_PROT64:
2388 if (msr_data == 0x0)
2389 return emulate_gp(ctxt, 0);
2395 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2396 cs_sel = (u16)msr_data;
2397 cs_sel &= ~SELECTOR_RPL_MASK;
2398 ss_sel = cs_sel + 8;
2399 ss_sel &= ~SELECTOR_RPL_MASK;
2400 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
2405 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2406 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2408 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2409 ctxt->_eip = msr_data;
2411 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2412 *reg_write(ctxt, VCPU_REGS_RSP) = msr_data;
2414 return X86EMUL_CONTINUE;
2417 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2419 const struct x86_emulate_ops *ops = ctxt->ops;
2420 struct desc_struct cs, ss;
2423 u16 cs_sel = 0, ss_sel = 0;
2425 /* inject #GP if in real mode or Virtual 8086 mode */
2426 if (ctxt->mode == X86EMUL_MODE_REAL ||
2427 ctxt->mode == X86EMUL_MODE_VM86)
2428 return emulate_gp(ctxt, 0);
2430 setup_syscalls_segments(ctxt, &cs, &ss);
2432 if ((ctxt->rex_prefix & 0x8) != 0x0)
2433 usermode = X86EMUL_MODE_PROT64;
2435 usermode = X86EMUL_MODE_PROT32;
2439 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2441 case X86EMUL_MODE_PROT32:
2442 cs_sel = (u16)(msr_data + 16);
2443 if ((msr_data & 0xfffc) == 0x0)
2444 return emulate_gp(ctxt, 0);
2445 ss_sel = (u16)(msr_data + 24);
2447 case X86EMUL_MODE_PROT64:
2448 cs_sel = (u16)(msr_data + 32);
2449 if (msr_data == 0x0)
2450 return emulate_gp(ctxt, 0);
2451 ss_sel = cs_sel + 8;
2456 cs_sel |= SELECTOR_RPL_MASK;
2457 ss_sel |= SELECTOR_RPL_MASK;
2459 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2460 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2462 ctxt->_eip = reg_read(ctxt, VCPU_REGS_RDX);
2463 *reg_write(ctxt, VCPU_REGS_RSP) = reg_read(ctxt, VCPU_REGS_RCX);
2465 return X86EMUL_CONTINUE;
2468 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2471 if (ctxt->mode == X86EMUL_MODE_REAL)
2473 if (ctxt->mode == X86EMUL_MODE_VM86)
2475 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2476 return ctxt->ops->cpl(ctxt) > iopl;
2479 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2482 const struct x86_emulate_ops *ops = ctxt->ops;
2483 struct desc_struct tr_seg;
2486 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2487 unsigned mask = (1 << len) - 1;
2490 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2493 if (desc_limit_scaled(&tr_seg) < 103)
2495 base = get_desc_base(&tr_seg);
2496 #ifdef CONFIG_X86_64
2497 base |= ((u64)base3) << 32;
2499 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2500 if (r != X86EMUL_CONTINUE)
2502 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2504 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2505 if (r != X86EMUL_CONTINUE)
2507 if ((perm >> bit_idx) & mask)
2512 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2518 if (emulator_bad_iopl(ctxt))
2519 if (!emulator_io_port_access_allowed(ctxt, port, len))
2522 ctxt->perm_ok = true;
2527 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2528 struct tss_segment_16 *tss)
2530 tss->ip = ctxt->_eip;
2531 tss->flag = ctxt->eflags;
2532 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
2533 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
2534 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
2535 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
2536 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
2537 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
2538 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
2539 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2541 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2542 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2543 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2544 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2545 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2548 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2549 struct tss_segment_16 *tss)
2553 ctxt->_eip = tss->ip;
2554 ctxt->eflags = tss->flag | 2;
2555 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
2556 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
2557 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
2558 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
2559 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
2560 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
2561 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
2562 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
2565 * SDM says that segment selectors are loaded before segment
2568 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2569 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2570 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2571 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2572 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2575 * Now load segment descriptors. If fault happens at this stage
2576 * it is handled in a context of new task
2578 ret = load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR);
2579 if (ret != X86EMUL_CONTINUE)
2581 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2582 if (ret != X86EMUL_CONTINUE)
2584 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2585 if (ret != X86EMUL_CONTINUE)
2587 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2588 if (ret != X86EMUL_CONTINUE)
2590 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2591 if (ret != X86EMUL_CONTINUE)
2594 return X86EMUL_CONTINUE;
2597 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2598 u16 tss_selector, u16 old_tss_sel,
2599 ulong old_tss_base, struct desc_struct *new_desc)
2601 const struct x86_emulate_ops *ops = ctxt->ops;
2602 struct tss_segment_16 tss_seg;
2604 u32 new_tss_base = get_desc_base(new_desc);
2606 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2608 if (ret != X86EMUL_CONTINUE)
2609 /* FIXME: need to provide precise fault address */
2612 save_state_to_tss16(ctxt, &tss_seg);
2614 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2616 if (ret != X86EMUL_CONTINUE)
2617 /* FIXME: need to provide precise fault address */
2620 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2622 if (ret != X86EMUL_CONTINUE)
2623 /* FIXME: need to provide precise fault address */
2626 if (old_tss_sel != 0xffff) {
2627 tss_seg.prev_task_link = old_tss_sel;
2629 ret = ops->write_std(ctxt, new_tss_base,
2630 &tss_seg.prev_task_link,
2631 sizeof tss_seg.prev_task_link,
2633 if (ret != X86EMUL_CONTINUE)
2634 /* FIXME: need to provide precise fault address */
2638 return load_state_from_tss16(ctxt, &tss_seg);
2641 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2642 struct tss_segment_32 *tss)
2644 tss->cr3 = ctxt->ops->get_cr(ctxt, 3);
2645 tss->eip = ctxt->_eip;
2646 tss->eflags = ctxt->eflags;
2647 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
2648 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
2649 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
2650 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
2651 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
2652 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
2653 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
2654 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
2656 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2657 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2658 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2659 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2660 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2661 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2662 tss->ldt_selector = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2665 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2666 struct tss_segment_32 *tss)
2670 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2671 return emulate_gp(ctxt, 0);
2672 ctxt->_eip = tss->eip;
2673 ctxt->eflags = tss->eflags | 2;
2675 /* General purpose registers */
2676 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
2677 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
2678 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
2679 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
2680 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
2681 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
2682 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
2683 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
2686 * SDM says that segment selectors are loaded before segment
2689 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2690 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2691 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2692 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2693 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2694 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2695 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2698 * If we're switching between Protected Mode and VM86, we need to make
2699 * sure to update the mode before loading the segment descriptors so
2700 * that the selectors are interpreted correctly.
2702 * Need to get rflags to the vcpu struct immediately because it
2703 * influences the CPL which is checked at least when loading the segment
2704 * descriptors and when pushing an error code to the new kernel stack.
2706 * TODO Introduce a separate ctxt->ops->set_cpl callback
2708 if (ctxt->eflags & X86_EFLAGS_VM)
2709 ctxt->mode = X86EMUL_MODE_VM86;
2711 ctxt->mode = X86EMUL_MODE_PROT32;
2713 ctxt->ops->set_rflags(ctxt, ctxt->eflags);
2716 * Now load segment descriptors. If fault happenes at this stage
2717 * it is handled in a context of new task
2719 ret = load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2720 if (ret != X86EMUL_CONTINUE)
2722 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2723 if (ret != X86EMUL_CONTINUE)
2725 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2726 if (ret != X86EMUL_CONTINUE)
2728 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2729 if (ret != X86EMUL_CONTINUE)
2731 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2732 if (ret != X86EMUL_CONTINUE)
2734 ret = load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS);
2735 if (ret != X86EMUL_CONTINUE)
2737 ret = load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS);
2738 if (ret != X86EMUL_CONTINUE)
2741 return X86EMUL_CONTINUE;
2744 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2745 u16 tss_selector, u16 old_tss_sel,
2746 ulong old_tss_base, struct desc_struct *new_desc)
2748 const struct x86_emulate_ops *ops = ctxt->ops;
2749 struct tss_segment_32 tss_seg;
2751 u32 new_tss_base = get_desc_base(new_desc);
2753 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2755 if (ret != X86EMUL_CONTINUE)
2756 /* FIXME: need to provide precise fault address */
2759 save_state_to_tss32(ctxt, &tss_seg);
2761 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2763 if (ret != X86EMUL_CONTINUE)
2764 /* FIXME: need to provide precise fault address */
2767 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2769 if (ret != X86EMUL_CONTINUE)
2770 /* FIXME: need to provide precise fault address */
2773 if (old_tss_sel != 0xffff) {
2774 tss_seg.prev_task_link = old_tss_sel;
2776 ret = ops->write_std(ctxt, new_tss_base,
2777 &tss_seg.prev_task_link,
2778 sizeof tss_seg.prev_task_link,
2780 if (ret != X86EMUL_CONTINUE)
2781 /* FIXME: need to provide precise fault address */
2785 return load_state_from_tss32(ctxt, &tss_seg);
2788 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2789 u16 tss_selector, int idt_index, int reason,
2790 bool has_error_code, u32 error_code)
2792 const struct x86_emulate_ops *ops = ctxt->ops;
2793 struct desc_struct curr_tss_desc, next_tss_desc;
2795 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2796 ulong old_tss_base =
2797 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2801 /* FIXME: old_tss_base == ~0 ? */
2803 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
2804 if (ret != X86EMUL_CONTINUE)
2806 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
2807 if (ret != X86EMUL_CONTINUE)
2810 /* FIXME: check that next_tss_desc is tss */
2813 * Check privileges. The three cases are task switch caused by...
2815 * 1. jmp/call/int to task gate: Check against DPL of the task gate
2816 * 2. Exception/IRQ/iret: No check is performed
2817 * 3. jmp/call to TSS: Check against DPL of the TSS
2819 if (reason == TASK_SWITCH_GATE) {
2820 if (idt_index != -1) {
2821 /* Software interrupts */
2822 struct desc_struct task_gate_desc;
2825 ret = read_interrupt_descriptor(ctxt, idt_index,
2827 if (ret != X86EMUL_CONTINUE)
2830 dpl = task_gate_desc.dpl;
2831 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2832 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
2834 } else if (reason != TASK_SWITCH_IRET) {
2835 int dpl = next_tss_desc.dpl;
2836 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2837 return emulate_gp(ctxt, tss_selector);
2841 desc_limit = desc_limit_scaled(&next_tss_desc);
2842 if (!next_tss_desc.p ||
2843 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2844 desc_limit < 0x2b)) {
2845 emulate_ts(ctxt, tss_selector & 0xfffc);
2846 return X86EMUL_PROPAGATE_FAULT;
2849 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2850 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2851 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2854 if (reason == TASK_SWITCH_IRET)
2855 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2857 /* set back link to prev task only if NT bit is set in eflags
2858 note that old_tss_sel is not used after this point */
2859 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2860 old_tss_sel = 0xffff;
2862 if (next_tss_desc.type & 8)
2863 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2864 old_tss_base, &next_tss_desc);
2866 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2867 old_tss_base, &next_tss_desc);
2868 if (ret != X86EMUL_CONTINUE)
2871 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2872 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2874 if (reason != TASK_SWITCH_IRET) {
2875 next_tss_desc.type |= (1 << 1); /* set busy flag */
2876 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2879 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2880 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2882 if (has_error_code) {
2883 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2884 ctxt->lock_prefix = 0;
2885 ctxt->src.val = (unsigned long) error_code;
2886 ret = em_push(ctxt);
2892 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2893 u16 tss_selector, int idt_index, int reason,
2894 bool has_error_code, u32 error_code)
2898 invalidate_registers(ctxt);
2899 ctxt->_eip = ctxt->eip;
2900 ctxt->dst.type = OP_NONE;
2902 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
2903 has_error_code, error_code);
2905 if (rc == X86EMUL_CONTINUE) {
2906 ctxt->eip = ctxt->_eip;
2907 writeback_registers(ctxt);
2910 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2913 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
2916 int df = (ctxt->eflags & EFLG_DF) ? -op->count : op->count;
2918 register_address_increment(ctxt, reg_rmw(ctxt, reg), df * op->bytes);
2919 op->addr.mem.ea = register_address(ctxt, reg_read(ctxt, reg));
2922 static int em_das(struct x86_emulate_ctxt *ctxt)
2925 bool af, cf, old_cf;
2927 cf = ctxt->eflags & X86_EFLAGS_CF;
2933 af = ctxt->eflags & X86_EFLAGS_AF;
2934 if ((al & 0x0f) > 9 || af) {
2936 cf = old_cf | (al >= 250);
2941 if (old_al > 0x99 || old_cf) {
2947 /* Set PF, ZF, SF */
2948 ctxt->src.type = OP_IMM;
2950 ctxt->src.bytes = 1;
2951 emulate_2op_SrcV(ctxt, "or");
2952 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2954 ctxt->eflags |= X86_EFLAGS_CF;
2956 ctxt->eflags |= X86_EFLAGS_AF;
2957 return X86EMUL_CONTINUE;
2960 static int em_aad(struct x86_emulate_ctxt *ctxt)
2962 u8 al = ctxt->dst.val & 0xff;
2963 u8 ah = (ctxt->dst.val >> 8) & 0xff;
2965 al = (al + (ah * ctxt->src.val)) & 0xff;
2967 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
2969 ctxt->eflags &= ~(X86_EFLAGS_PF | X86_EFLAGS_SF | X86_EFLAGS_ZF);
2972 ctxt->eflags |= X86_EFLAGS_ZF;
2974 ctxt->eflags |= X86_EFLAGS_PF;
2976 ctxt->eflags |= X86_EFLAGS_SF;
2978 return X86EMUL_CONTINUE;
2981 static int em_call(struct x86_emulate_ctxt *ctxt)
2983 long rel = ctxt->src.val;
2985 ctxt->src.val = (unsigned long)ctxt->_eip;
2987 return em_push(ctxt);
2990 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2996 old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2997 old_eip = ctxt->_eip;
2999 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
3000 if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
3001 return X86EMUL_CONTINUE;
3004 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
3006 ctxt->src.val = old_cs;
3008 if (rc != X86EMUL_CONTINUE)
3011 ctxt->src.val = old_eip;
3012 return em_push(ctxt);
3015 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
3019 ctxt->dst.type = OP_REG;
3020 ctxt->dst.addr.reg = &ctxt->_eip;
3021 ctxt->dst.bytes = ctxt->op_bytes;
3022 rc = emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
3023 if (rc != X86EMUL_CONTINUE)
3025 rsp_increment(ctxt, ctxt->src.val);
3026 return X86EMUL_CONTINUE;
3029 static int em_add(struct x86_emulate_ctxt *ctxt)
3031 emulate_2op_SrcV(ctxt, "add");
3032 return X86EMUL_CONTINUE;
3035 static int em_or(struct x86_emulate_ctxt *ctxt)
3037 emulate_2op_SrcV(ctxt, "or");
3038 return X86EMUL_CONTINUE;
3041 static int em_adc(struct x86_emulate_ctxt *ctxt)
3043 emulate_2op_SrcV(ctxt, "adc");
3044 return X86EMUL_CONTINUE;
3047 static int em_sbb(struct x86_emulate_ctxt *ctxt)
3049 emulate_2op_SrcV(ctxt, "sbb");
3050 return X86EMUL_CONTINUE;
3053 static int em_and(struct x86_emulate_ctxt *ctxt)
3055 emulate_2op_SrcV(ctxt, "and");
3056 return X86EMUL_CONTINUE;
3059 static int em_sub(struct x86_emulate_ctxt *ctxt)
3061 emulate_2op_SrcV(ctxt, "sub");
3062 return X86EMUL_CONTINUE;
3065 static int em_xor(struct x86_emulate_ctxt *ctxt)
3067 emulate_2op_SrcV(ctxt, "xor");
3068 return X86EMUL_CONTINUE;
3071 static int em_cmp(struct x86_emulate_ctxt *ctxt)
3073 emulate_2op_SrcV(ctxt, "cmp");
3074 return X86EMUL_CONTINUE;
3077 static int em_test(struct x86_emulate_ctxt *ctxt)
3079 emulate_2op_SrcV(ctxt, "test");
3080 return X86EMUL_CONTINUE;
3083 static int em_xchg(struct x86_emulate_ctxt *ctxt)
3085 /* Write back the register source. */
3086 ctxt->src.val = ctxt->dst.val;
3087 write_register_operand(&ctxt->src);
3089 /* Write back the memory destination with implicit LOCK prefix. */
3090 ctxt->dst.val = ctxt->src.orig_val;
3091 ctxt->lock_prefix = 1;
3092 return X86EMUL_CONTINUE;
3095 static int em_imul(struct x86_emulate_ctxt *ctxt)
3097 emulate_2op_SrcV_nobyte(ctxt, "imul");
3098 return X86EMUL_CONTINUE;
3101 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
3103 ctxt->dst.val = ctxt->src2.val;
3104 return em_imul(ctxt);
3107 static int em_cwd(struct x86_emulate_ctxt *ctxt)
3109 ctxt->dst.type = OP_REG;
3110 ctxt->dst.bytes = ctxt->src.bytes;
3111 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
3112 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
3114 return X86EMUL_CONTINUE;
3117 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
3121 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
3122 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
3123 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
3124 return X86EMUL_CONTINUE;
3127 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
3131 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
3132 return emulate_gp(ctxt, 0);
3133 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
3134 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
3135 return X86EMUL_CONTINUE;
3138 static int em_mov(struct x86_emulate_ctxt *ctxt)
3140 memcpy(ctxt->dst.valptr, ctxt->src.valptr, ctxt->op_bytes);
3141 return X86EMUL_CONTINUE;
3144 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3146 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3147 return emulate_gp(ctxt, 0);
3149 /* Disable writeback. */
3150 ctxt->dst.type = OP_NONE;
3151 return X86EMUL_CONTINUE;
3154 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3158 if (ctxt->mode == X86EMUL_MODE_PROT64)
3159 val = ctxt->src.val & ~0ULL;
3161 val = ctxt->src.val & ~0U;
3163 /* #UD condition is already handled. */
3164 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3165 return emulate_gp(ctxt, 0);
3167 /* Disable writeback. */
3168 ctxt->dst.type = OP_NONE;
3169 return X86EMUL_CONTINUE;
3172 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3176 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3177 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3178 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3179 return emulate_gp(ctxt, 0);
3181 return X86EMUL_CONTINUE;
3184 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3188 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3189 return emulate_gp(ctxt, 0);
3191 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3192 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3193 return X86EMUL_CONTINUE;
3196 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3198 if (ctxt->modrm_reg > VCPU_SREG_GS)
3199 return emulate_ud(ctxt);
3201 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3202 return X86EMUL_CONTINUE;
3205 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3207 u16 sel = ctxt->src.val;
3209 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3210 return emulate_ud(ctxt);
3212 if (ctxt->modrm_reg == VCPU_SREG_SS)
3213 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3215 /* Disable writeback. */
3216 ctxt->dst.type = OP_NONE;
3217 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3220 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3222 u16 sel = ctxt->src.val;
3224 /* Disable writeback. */
3225 ctxt->dst.type = OP_NONE;
3226 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3229 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3231 u16 sel = ctxt->src.val;
3233 /* Disable writeback. */
3234 ctxt->dst.type = OP_NONE;
3235 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3238 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3243 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3244 if (rc == X86EMUL_CONTINUE)
3245 ctxt->ops->invlpg(ctxt, linear);
3246 /* Disable writeback. */
3247 ctxt->dst.type = OP_NONE;
3248 return X86EMUL_CONTINUE;
3251 static int em_clts(struct x86_emulate_ctxt *ctxt)
3255 cr0 = ctxt->ops->get_cr(ctxt, 0);
3257 ctxt->ops->set_cr(ctxt, 0, cr0);
3258 return X86EMUL_CONTINUE;
3261 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
3265 if (ctxt->modrm_mod != 3 || ctxt->modrm_rm != 1)
3266 return X86EMUL_UNHANDLEABLE;
3268 rc = ctxt->ops->fix_hypercall(ctxt);
3269 if (rc != X86EMUL_CONTINUE)
3272 /* Let the processor re-execute the fixed hypercall */
3273 ctxt->_eip = ctxt->eip;
3274 /* Disable writeback. */
3275 ctxt->dst.type = OP_NONE;
3276 return X86EMUL_CONTINUE;
3279 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3280 void (*get)(struct x86_emulate_ctxt *ctxt,
3281 struct desc_ptr *ptr))
3283 struct desc_ptr desc_ptr;
3285 if (ctxt->mode == X86EMUL_MODE_PROT64)
3287 get(ctxt, &desc_ptr);
3288 if (ctxt->op_bytes == 2) {
3290 desc_ptr.address &= 0x00ffffff;
3292 /* Disable writeback. */
3293 ctxt->dst.type = OP_NONE;
3294 return segmented_write(ctxt, ctxt->dst.addr.mem,
3295 &desc_ptr, 2 + ctxt->op_bytes);
3298 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3300 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3303 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3305 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3308 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3310 struct desc_ptr desc_ptr;
3313 if (ctxt->mode == X86EMUL_MODE_PROT64)
3315 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3316 &desc_ptr.size, &desc_ptr.address,
3318 if (rc != X86EMUL_CONTINUE)
3320 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3321 /* Disable writeback. */
3322 ctxt->dst.type = OP_NONE;
3323 return X86EMUL_CONTINUE;
3326 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
3330 rc = ctxt->ops->fix_hypercall(ctxt);
3332 /* Disable writeback. */
3333 ctxt->dst.type = OP_NONE;
3337 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3339 struct desc_ptr desc_ptr;
3342 if (ctxt->mode == X86EMUL_MODE_PROT64)
3344 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3345 &desc_ptr.size, &desc_ptr.address,
3347 if (rc != X86EMUL_CONTINUE)
3349 ctxt->ops->set_idt(ctxt, &desc_ptr);
3350 /* Disable writeback. */
3351 ctxt->dst.type = OP_NONE;
3352 return X86EMUL_CONTINUE;
3355 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3357 ctxt->dst.bytes = 2;
3358 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3359 return X86EMUL_CONTINUE;
3362 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3364 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3365 | (ctxt->src.val & 0x0f));
3366 ctxt->dst.type = OP_NONE;
3367 return X86EMUL_CONTINUE;
3370 static int em_loop(struct x86_emulate_ctxt *ctxt)
3372 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX), -1);
3373 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3374 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3375 jmp_rel(ctxt, ctxt->src.val);
3377 return X86EMUL_CONTINUE;
3380 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3382 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3383 jmp_rel(ctxt, ctxt->src.val);
3385 return X86EMUL_CONTINUE;
3388 static int em_in(struct x86_emulate_ctxt *ctxt)
3390 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3392 return X86EMUL_IO_NEEDED;
3394 return X86EMUL_CONTINUE;
3397 static int em_out(struct x86_emulate_ctxt *ctxt)
3399 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3401 /* Disable writeback. */
3402 ctxt->dst.type = OP_NONE;
3403 return X86EMUL_CONTINUE;
3406 static int em_cli(struct x86_emulate_ctxt *ctxt)
3408 if (emulator_bad_iopl(ctxt))
3409 return emulate_gp(ctxt, 0);
3411 ctxt->eflags &= ~X86_EFLAGS_IF;
3412 return X86EMUL_CONTINUE;
3415 static int em_sti(struct x86_emulate_ctxt *ctxt)
3417 if (emulator_bad_iopl(ctxt))
3418 return emulate_gp(ctxt, 0);
3420 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3421 ctxt->eflags |= X86_EFLAGS_IF;
3422 return X86EMUL_CONTINUE;
3425 static int em_bt(struct x86_emulate_ctxt *ctxt)
3427 /* Disable writeback. */
3428 ctxt->dst.type = OP_NONE;
3429 /* only subword offset */
3430 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
3432 emulate_2op_SrcV_nobyte(ctxt, "bt");
3433 return X86EMUL_CONTINUE;
3436 static int em_bts(struct x86_emulate_ctxt *ctxt)
3438 emulate_2op_SrcV_nobyte(ctxt, "bts");
3439 return X86EMUL_CONTINUE;
3442 static int em_btr(struct x86_emulate_ctxt *ctxt)
3444 emulate_2op_SrcV_nobyte(ctxt, "btr");
3445 return X86EMUL_CONTINUE;
3448 static int em_btc(struct x86_emulate_ctxt *ctxt)
3450 emulate_2op_SrcV_nobyte(ctxt, "btc");
3451 return X86EMUL_CONTINUE;
3454 static int em_bsf(struct x86_emulate_ctxt *ctxt)
3456 emulate_2op_SrcV_nobyte(ctxt, "bsf");
3457 return X86EMUL_CONTINUE;
3460 static int em_bsr(struct x86_emulate_ctxt *ctxt)
3462 emulate_2op_SrcV_nobyte(ctxt, "bsr");
3463 return X86EMUL_CONTINUE;
3466 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3468 u32 eax, ebx, ecx, edx;
3470 eax = reg_read(ctxt, VCPU_REGS_RAX);
3471 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3472 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3473 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3474 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3475 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3476 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3477 return X86EMUL_CONTINUE;
3480 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3482 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3483 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3484 return X86EMUL_CONTINUE;
3487 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3489 switch (ctxt->op_bytes) {
3490 #ifdef CONFIG_X86_64
3492 asm("bswap %0" : "+r"(ctxt->dst.val));
3496 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3499 return X86EMUL_CONTINUE;
3502 static bool valid_cr(int nr)
3514 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
3516 if (!valid_cr(ctxt->modrm_reg))
3517 return emulate_ud(ctxt);
3519 return X86EMUL_CONTINUE;
3522 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
3524 u64 new_val = ctxt->src.val64;
3525 int cr = ctxt->modrm_reg;
3528 static u64 cr_reserved_bits[] = {
3529 0xffffffff00000000ULL,
3530 0, 0, 0, /* CR3 checked later */
3537 return emulate_ud(ctxt);
3539 if (new_val & cr_reserved_bits[cr])
3540 return emulate_gp(ctxt, 0);
3545 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
3546 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
3547 return emulate_gp(ctxt, 0);
3549 cr4 = ctxt->ops->get_cr(ctxt, 4);
3550 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3552 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
3553 !(cr4 & X86_CR4_PAE))
3554 return emulate_gp(ctxt, 0);
3561 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3562 if (efer & EFER_LMA)
3563 rsvd = CR3_L_MODE_RESERVED_BITS;
3564 else if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PAE)
3565 rsvd = CR3_PAE_RESERVED_BITS;
3566 else if (ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PG)
3567 rsvd = CR3_NONPAE_RESERVED_BITS;
3570 return emulate_gp(ctxt, 0);
3575 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3577 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
3578 return emulate_gp(ctxt, 0);
3584 return X86EMUL_CONTINUE;
3587 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
3591 ctxt->ops->get_dr(ctxt, 7, &dr7);
3593 /* Check if DR7.Global_Enable is set */
3594 return dr7 & (1 << 13);
3597 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
3599 int dr = ctxt->modrm_reg;
3603 return emulate_ud(ctxt);
3605 cr4 = ctxt->ops->get_cr(ctxt, 4);
3606 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
3607 return emulate_ud(ctxt);
3609 if (check_dr7_gd(ctxt))
3610 return emulate_db(ctxt);
3612 return X86EMUL_CONTINUE;
3615 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
3617 u64 new_val = ctxt->src.val64;
3618 int dr = ctxt->modrm_reg;
3620 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
3621 return emulate_gp(ctxt, 0);
3623 return check_dr_read(ctxt);
3626 static int check_svme(struct x86_emulate_ctxt *ctxt)
3630 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3632 if (!(efer & EFER_SVME))
3633 return emulate_ud(ctxt);
3635 return X86EMUL_CONTINUE;
3638 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
3640 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
3642 /* Valid physical address? */
3643 if (rax & 0xffff000000000000ULL)
3644 return emulate_gp(ctxt, 0);
3646 return check_svme(ctxt);
3649 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
3651 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3653 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3654 return emulate_ud(ctxt);
3656 return X86EMUL_CONTINUE;
3659 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
3661 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3662 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
3664 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3666 return emulate_gp(ctxt, 0);
3668 return X86EMUL_CONTINUE;
3671 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
3673 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
3674 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3675 return emulate_gp(ctxt, 0);
3677 return X86EMUL_CONTINUE;
3680 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
3682 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
3683 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3684 return emulate_gp(ctxt, 0);
3686 return X86EMUL_CONTINUE;
3689 #define D(_y) { .flags = (_y) }
3690 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
3691 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
3692 .check_perm = (_p) }
3694 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3695 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
3696 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
3697 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
3698 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3699 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
3700 #define II(_f, _e, _i) \
3701 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
3702 #define IIP(_f, _e, _i, _p) \
3703 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
3704 .check_perm = (_p) }
3705 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3707 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3708 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3709 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3710 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
3711 #define I2bvIP(_f, _e, _i, _p) \
3712 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
3714 #define I6ALU(_f, _e) I2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3715 I2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3716 I2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3718 static const struct opcode group7_rm1[] = {
3719 DI(SrcNone | Priv, monitor),
3720 DI(SrcNone | Priv, mwait),
3724 static const struct opcode group7_rm3[] = {
3725 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
3726 II(SrcNone | Prot | VendorSpecific, em_vmmcall, vmmcall),
3727 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
3728 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
3729 DIP(SrcNone | Prot | Priv, stgi, check_svme),
3730 DIP(SrcNone | Prot | Priv, clgi, check_svme),
3731 DIP(SrcNone | Prot | Priv, skinit, check_svme),
3732 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
3735 static const struct opcode group7_rm7[] = {
3737 DIP(SrcNone, rdtscp, check_rdtsc),
3741 static const struct opcode group1[] = {
3743 I(Lock | PageTable, em_or),
3746 I(Lock | PageTable, em_and),
3752 static const struct opcode group1A[] = {
3753 I(DstMem | SrcNone | Mov | Stack, em_pop), N, N, N, N, N, N, N,
3756 static const struct opcode group3[] = {
3757 I(DstMem | SrcImm | NoWrite, em_test),
3758 I(DstMem | SrcImm | NoWrite, em_test),
3759 F(DstMem | SrcNone | Lock, em_not),
3760 F(DstMem | SrcNone | Lock, em_neg),
3761 I(SrcMem, em_mul_ex),
3762 I(SrcMem, em_imul_ex),
3763 I(SrcMem, em_div_ex),
3764 I(SrcMem, em_idiv_ex),
3767 static const struct opcode group4[] = {
3768 I(ByteOp | DstMem | SrcNone | Lock, em_grp45),
3769 I(ByteOp | DstMem | SrcNone | Lock, em_grp45),
3773 static const struct opcode group5[] = {
3774 I(DstMem | SrcNone | Lock, em_grp45),
3775 I(DstMem | SrcNone | Lock, em_grp45),
3776 I(SrcMem | Stack, em_grp45),
3777 I(SrcMemFAddr | ImplicitOps | Stack, em_call_far),
3778 I(SrcMem | Stack, em_grp45),
3779 I(SrcMemFAddr | ImplicitOps, em_grp45),
3780 I(SrcMem | Stack, em_grp45), N,
3783 static const struct opcode group6[] = {
3786 II(Prot | Priv | SrcMem16, em_lldt, lldt),
3787 II(Prot | Priv | SrcMem16, em_ltr, ltr),
3791 static const struct group_dual group7 = { {
3792 II(Mov | DstMem | Priv, em_sgdt, sgdt),
3793 II(Mov | DstMem | Priv, em_sidt, sidt),
3794 II(SrcMem | Priv, em_lgdt, lgdt),
3795 II(SrcMem | Priv, em_lidt, lidt),
3796 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3797 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3798 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3800 I(SrcNone | Priv | VendorSpecific, em_vmcall),
3802 N, EXT(0, group7_rm3),
3803 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3804 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3808 static const struct opcode group8[] = {
3810 I(DstMem | SrcImmByte, em_bt),
3811 I(DstMem | SrcImmByte | Lock | PageTable, em_bts),
3812 I(DstMem | SrcImmByte | Lock, em_btr),
3813 I(DstMem | SrcImmByte | Lock | PageTable, em_btc),
3816 static const struct group_dual group9 = { {
3817 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
3819 N, N, N, N, N, N, N, N,
3822 static const struct opcode group11[] = {
3823 I(DstMem | SrcImm | Mov | PageTable, em_mov),
3827 static const struct gprefix pfx_0f_6f_0f_7f = {
3828 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
3831 static const struct gprefix pfx_vmovntpx = {
3832 I(0, em_mov), N, N, N,
3835 static const struct escape escape_d9 = { {
3836 N, N, N, N, N, N, N, I(DstMem, em_fnstcw),
3839 N, N, N, N, N, N, N, N,
3841 N, N, N, N, N, N, N, N,
3843 N, N, N, N, N, N, N, N,
3845 N, N, N, N, N, N, N, N,
3847 N, N, N, N, N, N, N, N,
3849 N, N, N, N, N, N, N, N,
3851 N, N, N, N, N, N, N, N,
3853 N, N, N, N, N, N, N, N,
3856 static const struct escape escape_db = { {
3857 N, N, N, N, N, N, N, N,
3860 N, N, N, N, N, N, N, N,
3862 N, N, N, N, N, N, N, N,
3864 N, N, N, N, N, N, N, N,
3866 N, N, N, N, N, N, N, N,
3868 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
3870 N, N, N, N, N, N, N, N,
3872 N, N, N, N, N, N, N, N,
3874 N, N, N, N, N, N, N, N,
3877 static const struct escape escape_dd = { {
3878 N, N, N, N, N, N, N, I(DstMem, em_fnstsw),
3881 N, N, N, N, N, N, N, N,
3883 N, N, N, N, N, N, N, N,
3885 N, N, N, N, N, N, N, N,
3887 N, N, N, N, N, N, N, N,
3889 N, N, N, N, N, N, N, N,
3891 N, N, N, N, N, N, N, N,
3893 N, N, N, N, N, N, N, N,
3895 N, N, N, N, N, N, N, N,
3898 static const struct opcode opcode_table[256] = {
3900 I6ALU(Lock, em_add),
3901 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
3902 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
3904 I6ALU(Lock | PageTable, em_or),
3905 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
3908 I6ALU(Lock, em_adc),
3909 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
3910 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
3912 I6ALU(Lock, em_sbb),
3913 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
3914 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
3916 I6ALU(Lock | PageTable, em_and), N, N,
3918 I6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3920 I6ALU(Lock, em_xor), N, N,
3922 I6ALU(NoWrite, em_cmp), N, N,
3926 X8(I(SrcReg | Stack, em_push)),
3928 X8(I(DstReg | Stack, em_pop)),
3930 I(ImplicitOps | Stack | No64, em_pusha),
3931 I(ImplicitOps | Stack | No64, em_popa),
3932 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3935 I(SrcImm | Mov | Stack, em_push),
3936 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3937 I(SrcImmByte | Mov | Stack, em_push),
3938 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3939 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
3940 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
3944 G(ByteOp | DstMem | SrcImm, group1),
3945 G(DstMem | SrcImm, group1),
3946 G(ByteOp | DstMem | SrcImm | No64, group1),
3947 G(DstMem | SrcImmByte, group1),
3948 I2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
3949 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
3951 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
3952 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3953 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
3954 D(ModRM | SrcMem | NoAccess | DstReg),
3955 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
3958 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3960 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3961 I(SrcImmFAddr | No64, em_call_far), N,
3962 II(ImplicitOps | Stack, em_pushf, pushf),
3963 II(ImplicitOps | Stack, em_popf, popf), N, I(ImplicitOps, em_lahf),
3965 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3966 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
3967 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3968 I2bv(SrcSI | DstDI | String | NoWrite, em_cmp),
3970 I2bv(DstAcc | SrcImm | NoWrite, em_test),
3971 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3972 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3973 I2bv(SrcAcc | DstDI | String | NoWrite, em_cmp),
3975 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3977 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
3979 D2bv(DstMem | SrcImmByte | ModRM),
3980 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3981 I(ImplicitOps | Stack, em_ret),
3982 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
3983 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
3984 G(ByteOp, group11), G(0, group11),
3986 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
3987 N, I(ImplicitOps | Stack, em_ret_far),
3988 D(ImplicitOps), DI(SrcImmByte, intn),
3989 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3991 D2bv(DstMem | SrcOne | ModRM), D2bv(DstMem | ModRM),
3992 N, I(DstAcc | SrcImmByte | No64, em_aad), N, N,
3994 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
3996 X3(I(SrcImmByte, em_loop)),
3997 I(SrcImmByte, em_jcxz),
3998 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
3999 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
4001 I(SrcImm | Stack, em_call), D(SrcImm | ImplicitOps),
4002 I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
4003 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
4004 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
4006 N, DI(ImplicitOps, icebp), N, N,
4007 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
4008 G(ByteOp, group3), G(0, group3),
4010 D(ImplicitOps), D(ImplicitOps),
4011 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
4012 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
4015 static const struct opcode twobyte_table[256] = {
4017 G(0, group6), GD(0, &group7), N, N,
4018 N, I(ImplicitOps | VendorSpecific, em_syscall),
4019 II(ImplicitOps | Priv, em_clts, clts), N,
4020 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
4021 N, D(ImplicitOps | ModRM), N, N,
4023 N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
4025 DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
4026 DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
4027 IIP(ModRM | SrcMem | Priv | Op3264, em_cr_write, cr_write, check_cr_write),
4028 IIP(ModRM | SrcMem | Priv | Op3264, em_dr_write, dr_write, check_dr_write),
4030 N, N, N, GP(ModRM | DstMem | SrcReg | Sse | Mov | Aligned, &pfx_vmovntpx),
4033 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
4034 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
4035 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
4036 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
4037 I(ImplicitOps | VendorSpecific, em_sysenter),
4038 I(ImplicitOps | Priv | VendorSpecific, em_sysexit),
4040 N, N, N, N, N, N, N, N,
4042 X16(D(DstReg | SrcMem | ModRM | Mov)),
4044 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4049 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
4054 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
4058 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
4060 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
4061 II(ImplicitOps, em_cpuid, cpuid), I(DstMem | SrcReg | ModRM | BitOp, em_bt),
4062 D(DstMem | SrcReg | Src2ImmByte | ModRM),
4063 D(DstMem | SrcReg | Src2CL | ModRM), N, N,
4065 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
4066 DI(ImplicitOps, rsm),
4067 I(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
4068 D(DstMem | SrcReg | Src2ImmByte | ModRM),
4069 D(DstMem | SrcReg | Src2CL | ModRM),
4070 D(ModRM), I(DstReg | SrcMem | ModRM, em_imul),
4072 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_cmpxchg),
4073 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
4074 I(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
4075 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
4076 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
4077 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4081 I(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
4082 I(DstReg | SrcMem | ModRM, em_bsf), I(DstReg | SrcMem | ModRM, em_bsr),
4083 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4085 D2bv(DstMem | SrcReg | ModRM | Lock),
4086 N, D(DstMem | SrcReg | ModRM | Mov),
4087 N, N, N, GD(0, &group9),
4089 X8(I(DstReg, em_bswap)),
4091 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4093 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4095 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
4112 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4116 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4122 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
4123 unsigned size, bool sign_extension)
4125 int rc = X86EMUL_CONTINUE;
4129 op->addr.mem.ea = ctxt->_eip;
4130 /* NB. Immediates are sign-extended as necessary. */
4131 switch (op->bytes) {
4133 op->val = insn_fetch(s8, ctxt);
4136 op->val = insn_fetch(s16, ctxt);
4139 op->val = insn_fetch(s32, ctxt);
4142 op->val = insn_fetch(s64, ctxt);
4145 if (!sign_extension) {
4146 switch (op->bytes) {
4154 op->val &= 0xffffffff;
4162 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
4165 int rc = X86EMUL_CONTINUE;
4169 decode_register_operand(ctxt, op);
4172 rc = decode_imm(ctxt, op, 1, false);
4175 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4179 if ((ctxt->d & BitOp) && op == &ctxt->dst)
4180 fetch_bit_operand(ctxt);
4181 op->orig_val = op->val;
4184 ctxt->memop.bytes = 8;
4188 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4189 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4190 fetch_register_operand(op);
4191 op->orig_val = op->val;
4195 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4197 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RDI));
4198 op->addr.mem.seg = VCPU_SREG_ES;
4205 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4206 fetch_register_operand(op);
4210 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4213 rc = decode_imm(ctxt, op, 1, true);
4220 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
4223 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
4226 ctxt->memop.bytes = 1;
4229 ctxt->memop.bytes = 2;
4232 ctxt->memop.bytes = 4;
4235 rc = decode_imm(ctxt, op, 2, false);
4238 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
4242 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4244 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RSI));
4245 op->addr.mem.seg = seg_override(ctxt);
4251 op->addr.mem.ea = ctxt->_eip;
4252 op->bytes = ctxt->op_bytes + 2;
4253 insn_fetch_arr(op->valptr, op->bytes, ctxt);
4256 ctxt->memop.bytes = ctxt->op_bytes + 2;
4259 op->val = VCPU_SREG_ES;
4262 op->val = VCPU_SREG_CS;
4265 op->val = VCPU_SREG_SS;
4268 op->val = VCPU_SREG_DS;
4271 op->val = VCPU_SREG_FS;
4274 op->val = VCPU_SREG_GS;
4277 /* Special instructions do their own operand decoding. */
4279 op->type = OP_NONE; /* Disable writeback. */
4287 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
4289 int rc = X86EMUL_CONTINUE;
4290 int mode = ctxt->mode;
4291 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
4292 bool op_prefix = false;
4293 struct opcode opcode;
4295 ctxt->memop.type = OP_NONE;
4296 ctxt->memopp = NULL;
4297 ctxt->_eip = ctxt->eip;
4298 ctxt->fetch.start = ctxt->_eip;
4299 ctxt->fetch.end = ctxt->fetch.start + insn_len;
4301 memcpy(ctxt->fetch.data, insn, insn_len);
4304 case X86EMUL_MODE_REAL:
4305 case X86EMUL_MODE_VM86:
4306 case X86EMUL_MODE_PROT16:
4307 def_op_bytes = def_ad_bytes = 2;
4309 case X86EMUL_MODE_PROT32:
4310 def_op_bytes = def_ad_bytes = 4;
4312 #ifdef CONFIG_X86_64
4313 case X86EMUL_MODE_PROT64:
4319 return EMULATION_FAILED;
4322 ctxt->op_bytes = def_op_bytes;
4323 ctxt->ad_bytes = def_ad_bytes;
4325 /* Legacy prefixes. */
4327 switch (ctxt->b = insn_fetch(u8, ctxt)) {
4328 case 0x66: /* operand-size override */
4330 /* switch between 2/4 bytes */
4331 ctxt->op_bytes = def_op_bytes ^ 6;
4333 case 0x67: /* address-size override */
4334 if (mode == X86EMUL_MODE_PROT64)
4335 /* switch between 4/8 bytes */
4336 ctxt->ad_bytes = def_ad_bytes ^ 12;
4338 /* switch between 2/4 bytes */
4339 ctxt->ad_bytes = def_ad_bytes ^ 6;
4341 case 0x26: /* ES override */
4342 case 0x2e: /* CS override */
4343 case 0x36: /* SS override */
4344 case 0x3e: /* DS override */
4345 set_seg_override(ctxt, (ctxt->b >> 3) & 3);
4347 case 0x64: /* FS override */
4348 case 0x65: /* GS override */
4349 set_seg_override(ctxt, ctxt->b & 7);
4351 case 0x40 ... 0x4f: /* REX */
4352 if (mode != X86EMUL_MODE_PROT64)
4354 ctxt->rex_prefix = ctxt->b;
4356 case 0xf0: /* LOCK */
4357 ctxt->lock_prefix = 1;
4359 case 0xf2: /* REPNE/REPNZ */
4360 case 0xf3: /* REP/REPE/REPZ */
4361 ctxt->rep_prefix = ctxt->b;
4367 /* Any legacy prefix after a REX prefix nullifies its effect. */
4369 ctxt->rex_prefix = 0;
4375 if (ctxt->rex_prefix & 8)
4376 ctxt->op_bytes = 8; /* REX.W */
4378 /* Opcode byte(s). */
4379 opcode = opcode_table[ctxt->b];
4380 /* Two-byte opcode? */
4381 if (ctxt->b == 0x0f) {
4383 ctxt->b = insn_fetch(u8, ctxt);
4384 opcode = twobyte_table[ctxt->b];
4386 ctxt->d = opcode.flags;
4388 if (ctxt->d & ModRM)
4389 ctxt->modrm = insn_fetch(u8, ctxt);
4391 while (ctxt->d & GroupMask) {
4392 switch (ctxt->d & GroupMask) {
4394 goffset = (ctxt->modrm >> 3) & 7;
4395 opcode = opcode.u.group[goffset];
4398 goffset = (ctxt->modrm >> 3) & 7;
4399 if ((ctxt->modrm >> 6) == 3)
4400 opcode = opcode.u.gdual->mod3[goffset];
4402 opcode = opcode.u.gdual->mod012[goffset];
4405 goffset = ctxt->modrm & 7;
4406 opcode = opcode.u.group[goffset];
4409 if (ctxt->rep_prefix && op_prefix)
4410 return EMULATION_FAILED;
4411 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
4412 switch (simd_prefix) {
4413 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
4414 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
4415 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
4416 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
4420 if (ctxt->modrm > 0xbf)
4421 opcode = opcode.u.esc->high[ctxt->modrm - 0xc0];
4423 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
4426 return EMULATION_FAILED;
4429 ctxt->d &= ~(u64)GroupMask;
4430 ctxt->d |= opcode.flags;
4433 ctxt->execute = opcode.u.execute;
4434 ctxt->check_perm = opcode.check_perm;
4435 ctxt->intercept = opcode.intercept;
4438 if (ctxt->d == 0 || (ctxt->d & Undefined))
4439 return EMULATION_FAILED;
4441 if (!(ctxt->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
4442 return EMULATION_FAILED;
4444 if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
4447 if (ctxt->d & Op3264) {
4448 if (mode == X86EMUL_MODE_PROT64)
4455 ctxt->op_bytes = 16;
4456 else if (ctxt->d & Mmx)
4459 /* ModRM and SIB bytes. */
4460 if (ctxt->d & ModRM) {
4461 rc = decode_modrm(ctxt, &ctxt->memop);
4462 if (!ctxt->has_seg_override)
4463 set_seg_override(ctxt, ctxt->modrm_seg);
4464 } else if (ctxt->d & MemAbs)
4465 rc = decode_abs(ctxt, &ctxt->memop);
4466 if (rc != X86EMUL_CONTINUE)
4469 if (!ctxt->has_seg_override)
4470 set_seg_override(ctxt, VCPU_SREG_DS);
4472 ctxt->memop.addr.mem.seg = seg_override(ctxt);
4474 if (ctxt->memop.type == OP_MEM && ctxt->ad_bytes != 8)
4475 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
4478 * Decode and fetch the source operand: register, memory
4481 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
4482 if (rc != X86EMUL_CONTINUE)
4486 * Decode and fetch the second source operand: register, memory
4489 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
4490 if (rc != X86EMUL_CONTINUE)
4493 /* Decode and fetch the destination operand: register or memory. */
4494 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
4497 if (ctxt->memopp && ctxt->memopp->type == OP_MEM && ctxt->rip_relative)
4498 ctxt->memopp->addr.mem.ea += ctxt->_eip;
4500 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
4503 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
4505 return ctxt->d & PageTable;
4508 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
4510 /* The second termination condition only applies for REPE
4511 * and REPNE. Test if the repeat string operation prefix is
4512 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
4513 * corresponding termination condition according to:
4514 * - if REPE/REPZ and ZF = 0 then done
4515 * - if REPNE/REPNZ and ZF = 1 then done
4517 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
4518 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
4519 && (((ctxt->rep_prefix == REPE_PREFIX) &&
4520 ((ctxt->eflags & EFLG_ZF) == 0))
4521 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
4522 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
4528 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
4532 ctxt->ops->get_fpu(ctxt);
4533 asm volatile("1: fwait \n\t"
4535 ".pushsection .fixup,\"ax\" \n\t"
4537 "movb $1, %[fault] \n\t"
4540 _ASM_EXTABLE(1b, 3b)
4541 : [fault]"+qm"(fault));
4542 ctxt->ops->put_fpu(ctxt);
4544 if (unlikely(fault))
4545 return emulate_exception(ctxt, MF_VECTOR, 0, false);
4547 return X86EMUL_CONTINUE;
4550 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
4553 if (op->type == OP_MM)
4554 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
4557 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
4559 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
4560 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
4561 asm("push %[flags]; popf; call *%[fastop]; pushf; pop %[flags]\n"
4562 : "+a"(ctxt->dst.val), "+b"(ctxt->src.val), [flags]"+D"(flags)
4563 : "c"(ctxt->src2.val), [fastop]"S"(fop));
4564 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
4565 return X86EMUL_CONTINUE;
4568 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
4570 const struct x86_emulate_ops *ops = ctxt->ops;
4571 int rc = X86EMUL_CONTINUE;
4572 int saved_dst_type = ctxt->dst.type;
4574 ctxt->mem_read.pos = 0;
4576 if (ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) {
4577 rc = emulate_ud(ctxt);
4581 /* LOCK prefix is allowed only with some instructions */
4582 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
4583 rc = emulate_ud(ctxt);
4587 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
4588 rc = emulate_ud(ctxt);
4592 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
4593 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
4594 rc = emulate_ud(ctxt);
4598 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
4599 rc = emulate_nm(ctxt);
4603 if (ctxt->d & Mmx) {
4604 rc = flush_pending_x87_faults(ctxt);
4605 if (rc != X86EMUL_CONTINUE)
4608 * Now that we know the fpu is exception safe, we can fetch
4611 fetch_possible_mmx_operand(ctxt, &ctxt->src);
4612 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
4613 if (!(ctxt->d & Mov))
4614 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
4617 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4618 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4619 X86_ICPT_PRE_EXCEPT);
4620 if (rc != X86EMUL_CONTINUE)
4624 /* Privileged instruction can be executed only in CPL=0 */
4625 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
4626 rc = emulate_gp(ctxt, 0);
4630 /* Instruction can only be executed in protected mode */
4631 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
4632 rc = emulate_ud(ctxt);
4636 /* Do instruction specific permission checks */
4637 if (ctxt->check_perm) {
4638 rc = ctxt->check_perm(ctxt);
4639 if (rc != X86EMUL_CONTINUE)
4643 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4644 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4645 X86_ICPT_POST_EXCEPT);
4646 if (rc != X86EMUL_CONTINUE)
4650 if (ctxt->rep_prefix && (ctxt->d & String)) {
4651 /* All REP prefixes have the same first termination condition */
4652 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
4653 ctxt->eip = ctxt->_eip;
4658 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
4659 rc = segmented_read(ctxt, ctxt->src.addr.mem,
4660 ctxt->src.valptr, ctxt->src.bytes);
4661 if (rc != X86EMUL_CONTINUE)
4663 ctxt->src.orig_val64 = ctxt->src.val64;
4666 if (ctxt->src2.type == OP_MEM) {
4667 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
4668 &ctxt->src2.val, ctxt->src2.bytes);
4669 if (rc != X86EMUL_CONTINUE)
4673 if ((ctxt->d & DstMask) == ImplicitOps)
4677 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
4678 /* optimisation - avoid slow emulated read if Mov */
4679 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
4680 &ctxt->dst.val, ctxt->dst.bytes);
4681 if (rc != X86EMUL_CONTINUE)
4684 ctxt->dst.orig_val = ctxt->dst.val;
4688 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4689 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4690 X86_ICPT_POST_MEMACCESS);
4691 if (rc != X86EMUL_CONTINUE)
4695 if (ctxt->execute) {
4696 if (ctxt->d & Fastop) {
4697 void (*fop)(struct fastop *) = (void *)ctxt->execute;
4698 rc = fastop(ctxt, fop);
4699 if (rc != X86EMUL_CONTINUE)
4703 rc = ctxt->execute(ctxt);
4704 if (rc != X86EMUL_CONTINUE)
4713 case 0x40 ... 0x47: /* inc r16/r32 */
4714 emulate_1op(ctxt, "inc");
4716 case 0x48 ... 0x4f: /* dec r16/r32 */
4717 emulate_1op(ctxt, "dec");
4719 case 0x63: /* movsxd */
4720 if (ctxt->mode != X86EMUL_MODE_PROT64)
4721 goto cannot_emulate;
4722 ctxt->dst.val = (s32) ctxt->src.val;
4724 case 0x70 ... 0x7f: /* jcc (short) */
4725 if (test_cc(ctxt->b, ctxt->eflags))
4726 jmp_rel(ctxt, ctxt->src.val);
4728 case 0x8d: /* lea r16/r32, m */
4729 ctxt->dst.val = ctxt->src.addr.mem.ea;
4731 case 0x90 ... 0x97: /* nop / xchg reg, rax */
4732 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
4736 case 0x98: /* cbw/cwde/cdqe */
4737 switch (ctxt->op_bytes) {
4738 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
4739 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
4740 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
4746 case 0xcc: /* int3 */
4747 rc = emulate_int(ctxt, 3);
4749 case 0xcd: /* int n */
4750 rc = emulate_int(ctxt, ctxt->src.val);
4752 case 0xce: /* into */
4753 if (ctxt->eflags & EFLG_OF)
4754 rc = emulate_int(ctxt, 4);
4756 case 0xd0 ... 0xd1: /* Grp2 */
4759 case 0xd2 ... 0xd3: /* Grp2 */
4760 ctxt->src.val = reg_read(ctxt, VCPU_REGS_RCX);
4763 case 0xe9: /* jmp rel */
4764 case 0xeb: /* jmp rel short */
4765 jmp_rel(ctxt, ctxt->src.val);
4766 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4768 case 0xf4: /* hlt */
4769 ctxt->ops->halt(ctxt);
4771 case 0xf5: /* cmc */
4772 /* complement carry flag from eflags reg */
4773 ctxt->eflags ^= EFLG_CF;
4775 case 0xf8: /* clc */
4776 ctxt->eflags &= ~EFLG_CF;
4778 case 0xf9: /* stc */
4779 ctxt->eflags |= EFLG_CF;
4781 case 0xfc: /* cld */
4782 ctxt->eflags &= ~EFLG_DF;
4784 case 0xfd: /* std */
4785 ctxt->eflags |= EFLG_DF;
4788 goto cannot_emulate;
4791 if (rc != X86EMUL_CONTINUE)
4795 rc = writeback(ctxt);
4796 if (rc != X86EMUL_CONTINUE)
4800 * restore dst type in case the decoding will be reused
4801 * (happens for string instruction )
4803 ctxt->dst.type = saved_dst_type;
4805 if ((ctxt->d & SrcMask) == SrcSI)
4806 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
4808 if ((ctxt->d & DstMask) == DstDI)
4809 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
4811 if (ctxt->rep_prefix && (ctxt->d & String)) {
4813 struct read_cache *r = &ctxt->io_read;
4814 if ((ctxt->d & SrcMask) == SrcSI)
4815 count = ctxt->src.count;
4817 count = ctxt->dst.count;
4818 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX),
4821 if (!string_insn_completed(ctxt)) {
4823 * Re-enter guest when pio read ahead buffer is empty
4824 * or, if it is not used, after each 1024 iteration.
4826 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
4827 (r->end == 0 || r->end != r->pos)) {
4829 * Reset read cache. Usually happens before
4830 * decode, but since instruction is restarted
4831 * we have to do it here.
4833 ctxt->mem_read.end = 0;
4834 writeback_registers(ctxt);
4835 return EMULATION_RESTART;
4837 goto done; /* skip rip writeback */
4841 ctxt->eip = ctxt->_eip;
4844 if (rc == X86EMUL_PROPAGATE_FAULT)
4845 ctxt->have_exception = true;
4846 if (rc == X86EMUL_INTERCEPTED)
4847 return EMULATION_INTERCEPTED;
4849 if (rc == X86EMUL_CONTINUE)
4850 writeback_registers(ctxt);
4852 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4856 case 0x09: /* wbinvd */
4857 (ctxt->ops->wbinvd)(ctxt);
4859 case 0x08: /* invd */
4860 case 0x0d: /* GrpP (prefetch) */
4861 case 0x18: /* Grp16 (prefetch/nop) */
4863 case 0x20: /* mov cr, reg */
4864 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
4866 case 0x21: /* mov from dr to reg */
4867 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
4869 case 0x40 ... 0x4f: /* cmov */
4870 ctxt->dst.val = ctxt->dst.orig_val = ctxt->src.val;
4871 if (!test_cc(ctxt->b, ctxt->eflags))
4872 ctxt->dst.type = OP_NONE; /* no writeback */
4874 case 0x80 ... 0x8f: /* jnz rel, etc*/
4875 if (test_cc(ctxt->b, ctxt->eflags))
4876 jmp_rel(ctxt, ctxt->src.val);
4878 case 0x90 ... 0x9f: /* setcc r/m8 */
4879 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
4881 case 0xa4: /* shld imm8, r, r/m */
4882 case 0xa5: /* shld cl, r, r/m */
4883 emulate_2op_cl(ctxt, "shld");
4885 case 0xac: /* shrd imm8, r, r/m */
4886 case 0xad: /* shrd cl, r, r/m */
4887 emulate_2op_cl(ctxt, "shrd");
4889 case 0xae: /* clflush */
4891 case 0xb6 ... 0xb7: /* movzx */
4892 ctxt->dst.bytes = ctxt->op_bytes;
4893 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
4894 : (u16) ctxt->src.val;
4896 case 0xbe ... 0xbf: /* movsx */
4897 ctxt->dst.bytes = ctxt->op_bytes;
4898 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
4899 (s16) ctxt->src.val;
4901 case 0xc0 ... 0xc1: /* xadd */
4902 emulate_2op_SrcV(ctxt, "add");
4903 /* Write back the register source. */
4904 ctxt->src.val = ctxt->dst.orig_val;
4905 write_register_operand(&ctxt->src);
4907 case 0xc3: /* movnti */
4908 ctxt->dst.bytes = ctxt->op_bytes;
4909 ctxt->dst.val = (ctxt->op_bytes == 4) ? (u32) ctxt->src.val :
4910 (u64) ctxt->src.val;
4913 goto cannot_emulate;
4916 if (rc != X86EMUL_CONTINUE)
4922 return EMULATION_FAILED;
4925 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
4927 invalidate_registers(ctxt);
4930 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
4932 writeback_registers(ctxt);