1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
35 #define OpImplicit 1ull /* No generic decode */
36 #define OpReg 2ull /* Register */
37 #define OpMem 3ull /* Memory */
38 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
39 #define OpDI 5ull /* ES:DI/EDI/RDI */
40 #define OpMem64 6ull /* Memory, 64-bit */
41 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
42 #define OpDX 8ull /* DX register */
43 #define OpCL 9ull /* CL register (for shifts) */
44 #define OpImmByte 10ull /* 8-bit sign extended immediate */
45 #define OpOne 11ull /* Implied 1 */
46 #define OpImm 12ull /* Sign extended immediate */
47 #define OpMem16 13ull /* Memory operand (16-bit). */
48 #define OpMem32 14ull /* Memory operand (32-bit). */
49 #define OpImmU 15ull /* Immediate operand, zero extended */
50 #define OpSI 16ull /* SI/ESI/RSI */
51 #define OpImmFAddr 17ull /* Immediate far address */
52 #define OpMemFAddr 18ull /* Far address in memory */
53 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
54 #define OpES 20ull /* ES */
55 #define OpCS 21ull /* CS */
56 #define OpSS 22ull /* SS */
57 #define OpDS 23ull /* DS */
58 #define OpFS 24ull /* FS */
59 #define OpGS 25ull /* GS */
61 #define OpBits 5 /* Width of operand field */
62 #define OpMask ((1ull << OpBits) - 1)
65 * Opcode effective-address decode tables.
66 * Note that we only emulate instructions that have at least one memory
67 * operand (excluding implicit stack references). We assume that stack
68 * references and instruction fetches will never occur in special memory
69 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
73 /* Operand sizes: 8-bit operands or specified/overridden size. */
74 #define ByteOp (1<<0) /* 8-bit operands. */
75 /* Destination operand type. */
77 #define ImplicitOps (OpImplicit << DstShift)
78 #define DstReg (OpReg << DstShift)
79 #define DstMem (OpMem << DstShift)
80 #define DstAcc (OpAcc << DstShift)
81 #define DstDI (OpDI << DstShift)
82 #define DstMem64 (OpMem64 << DstShift)
83 #define DstImmUByte (OpImmUByte << DstShift)
84 #define DstDX (OpDX << DstShift)
85 #define DstMask (OpMask << DstShift)
86 /* Source operand type. */
88 #define SrcNone (OpNone << SrcShift)
89 #define SrcReg (OpReg << SrcShift)
90 #define SrcMem (OpMem << SrcShift)
91 #define SrcMem16 (OpMem16 << SrcShift)
92 #define SrcMem32 (OpMem32 << SrcShift)
93 #define SrcImm (OpImm << SrcShift)
94 #define SrcImmByte (OpImmByte << SrcShift)
95 #define SrcOne (OpOne << SrcShift)
96 #define SrcImmUByte (OpImmUByte << SrcShift)
97 #define SrcImmU (OpImmU << SrcShift)
98 #define SrcSI (OpSI << SrcShift)
99 #define SrcImmFAddr (OpImmFAddr << SrcShift)
100 #define SrcMemFAddr (OpMemFAddr << SrcShift)
101 #define SrcAcc (OpAcc << SrcShift)
102 #define SrcImmU16 (OpImmU16 << SrcShift)
103 #define SrcDX (OpDX << SrcShift)
104 #define SrcMask (OpMask << SrcShift)
105 #define BitOp (1<<11)
106 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
107 #define String (1<<13) /* String instruction (rep capable) */
108 #define Stack (1<<14) /* Stack instruction (push/pop) */
109 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
110 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
111 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
112 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
113 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
114 #define Sse (1<<18) /* SSE Vector instruction */
115 /* Generic ModRM decode. */
116 #define ModRM (1<<19)
117 /* Destination is only written; never read. */
120 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
121 #define VendorSpecific (1<<22) /* Vendor specific instruction */
122 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
123 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
124 #define Undefined (1<<25) /* No Such Instruction */
125 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
126 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
128 #define PageTable (1 << 29) /* instruction used to write page table */
129 /* Source 2 operand type */
130 #define Src2Shift (30)
131 #define Src2None (OpNone << Src2Shift)
132 #define Src2CL (OpCL << Src2Shift)
133 #define Src2ImmByte (OpImmByte << Src2Shift)
134 #define Src2One (OpOne << Src2Shift)
135 #define Src2Imm (OpImm << Src2Shift)
136 #define Src2ES (OpES << Src2Shift)
137 #define Src2CS (OpCS << Src2Shift)
138 #define Src2SS (OpSS << Src2Shift)
139 #define Src2DS (OpDS << Src2Shift)
140 #define Src2FS (OpFS << Src2Shift)
141 #define Src2GS (OpGS << Src2Shift)
142 #define Src2Mask (OpMask << Src2Shift)
144 #define X2(x...) x, x
145 #define X3(x...) X2(x), x
146 #define X4(x...) X2(x), X2(x)
147 #define X5(x...) X4(x), x
148 #define X6(x...) X4(x), X2(x)
149 #define X7(x...) X4(x), X3(x)
150 #define X8(x...) X4(x), X4(x)
151 #define X16(x...) X8(x), X8(x)
157 int (*execute)(struct x86_emulate_ctxt *ctxt);
158 struct opcode *group;
159 struct group_dual *gdual;
160 struct gprefix *gprefix;
162 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
166 struct opcode mod012[8];
167 struct opcode mod3[8];
171 struct opcode pfx_no;
172 struct opcode pfx_66;
173 struct opcode pfx_f2;
174 struct opcode pfx_f3;
177 /* EFLAGS bit definitions. */
178 #define EFLG_ID (1<<21)
179 #define EFLG_VIP (1<<20)
180 #define EFLG_VIF (1<<19)
181 #define EFLG_AC (1<<18)
182 #define EFLG_VM (1<<17)
183 #define EFLG_RF (1<<16)
184 #define EFLG_IOPL (3<<12)
185 #define EFLG_NT (1<<14)
186 #define EFLG_OF (1<<11)
187 #define EFLG_DF (1<<10)
188 #define EFLG_IF (1<<9)
189 #define EFLG_TF (1<<8)
190 #define EFLG_SF (1<<7)
191 #define EFLG_ZF (1<<6)
192 #define EFLG_AF (1<<4)
193 #define EFLG_PF (1<<2)
194 #define EFLG_CF (1<<0)
196 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
197 #define EFLG_RESERVED_ONE_MASK 2
200 * Instruction emulation:
201 * Most instructions are emulated directly via a fragment of inline assembly
202 * code. This allows us to save/restore EFLAGS and thus very easily pick up
203 * any modified flags.
206 #if defined(CONFIG_X86_64)
207 #define _LO32 "k" /* force 32-bit operand */
208 #define _STK "%%rsp" /* stack pointer */
209 #elif defined(__i386__)
210 #define _LO32 "" /* force 32-bit operand */
211 #define _STK "%%esp" /* stack pointer */
215 * These EFLAGS bits are restored from saved value during emulation, and
216 * any changes are written back to the saved value after emulation.
218 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
220 /* Before executing instruction: restore necessary bits in EFLAGS. */
221 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
222 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
223 "movl %"_sav",%"_LO32 _tmp"; " \
226 "movl %"_msk",%"_LO32 _tmp"; " \
227 "andl %"_LO32 _tmp",("_STK"); " \
229 "notl %"_LO32 _tmp"; " \
230 "andl %"_LO32 _tmp",("_STK"); " \
231 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
233 "orl %"_LO32 _tmp",("_STK"); " \
237 /* After executing instruction: write-back necessary bits in EFLAGS. */
238 #define _POST_EFLAGS(_sav, _msk, _tmp) \
239 /* _sav |= EFLAGS & _msk; */ \
242 "andl %"_msk",%"_LO32 _tmp"; " \
243 "orl %"_LO32 _tmp",%"_sav"; "
251 #define ____emulate_2op(ctxt, _op, _x, _y, _suffix, _dsttype) \
253 __asm__ __volatile__ ( \
254 _PRE_EFLAGS("0", "4", "2") \
255 _op _suffix " %"_x"3,%1; " \
256 _POST_EFLAGS("0", "4", "2") \
257 : "=m" ((ctxt)->eflags), \
258 "+q" (*(_dsttype*)&(ctxt)->dst.val), \
260 : _y ((ctxt)->src.val), "i" (EFLAGS_MASK)); \
264 /* Raw emulation: instruction has two explicit operands. */
265 #define __emulate_2op_nobyte(ctxt,_op,_wx,_wy,_lx,_ly,_qx,_qy) \
267 unsigned long _tmp; \
269 switch ((ctxt)->dst.bytes) { \
271 ____emulate_2op(ctxt,_op,_wx,_wy,"w",u16); \
274 ____emulate_2op(ctxt,_op,_lx,_ly,"l",u32); \
277 ON64(____emulate_2op(ctxt,_op,_qx,_qy,"q",u64)); \
282 #define __emulate_2op(ctxt,_op,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
284 unsigned long _tmp; \
285 switch ((ctxt)->dst.bytes) { \
287 ____emulate_2op(ctxt,_op,_bx,_by,"b",u8); \
290 __emulate_2op_nobyte(ctxt, _op, \
291 _wx, _wy, _lx, _ly, _qx, _qy); \
296 /* Source operand is byte-sized and may be restricted to just %cl. */
297 #define emulate_2op_SrcB(ctxt, _op) \
298 __emulate_2op(ctxt, _op, "b", "c", "b", "c", "b", "c", "b", "c")
300 /* Source operand is byte, word, long or quad sized. */
301 #define emulate_2op_SrcV(ctxt, _op) \
302 __emulate_2op(ctxt, _op, "b", "q", "w", "r", _LO32, "r", "", "r")
304 /* Source operand is word, long or quad sized. */
305 #define emulate_2op_SrcV_nobyte(ctxt, _op) \
306 __emulate_2op_nobyte(ctxt, _op, "w", "r", _LO32, "r", "", "r")
308 /* Instruction has three operands and one operand is stored in ECX register */
309 #define __emulate_2op_cl(ctxt, _op, _suffix, _type) \
311 unsigned long _tmp; \
312 _type _clv = (ctxt)->src2.val; \
313 _type _srcv = (ctxt)->src.val; \
314 _type _dstv = (ctxt)->dst.val; \
316 __asm__ __volatile__ ( \
317 _PRE_EFLAGS("0", "5", "2") \
318 _op _suffix " %4,%1 \n" \
319 _POST_EFLAGS("0", "5", "2") \
320 : "=m" ((ctxt)->eflags), "+r" (_dstv), "=&r" (_tmp) \
321 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
324 (ctxt)->src2.val = (unsigned long) _clv; \
325 (ctxt)->src2.val = (unsigned long) _srcv; \
326 (ctxt)->dst.val = (unsigned long) _dstv; \
329 #define emulate_2op_cl(ctxt, _op) \
331 switch ((ctxt)->dst.bytes) { \
333 __emulate_2op_cl(ctxt, _op, "w", u16); \
336 __emulate_2op_cl(ctxt, _op, "l", u32); \
339 ON64(__emulate_2op_cl(ctxt, _op, "q", ulong)); \
344 #define __emulate_1op(ctxt, _op, _suffix) \
346 unsigned long _tmp; \
348 __asm__ __volatile__ ( \
349 _PRE_EFLAGS("0", "3", "2") \
350 _op _suffix " %1; " \
351 _POST_EFLAGS("0", "3", "2") \
352 : "=m" ((ctxt)->eflags), "+m" ((ctxt)->dst.val), \
354 : "i" (EFLAGS_MASK)); \
357 /* Instruction has only one explicit operand (no source operand). */
358 #define emulate_1op(ctxt, _op) \
360 switch ((ctxt)->dst.bytes) { \
361 case 1: __emulate_1op(ctxt, _op, "b"); break; \
362 case 2: __emulate_1op(ctxt, _op, "w"); break; \
363 case 4: __emulate_1op(ctxt, _op, "l"); break; \
364 case 8: ON64(__emulate_1op(ctxt, _op, "q")); break; \
368 #define __emulate_1op_rax_rdx(ctxt, _op, _suffix, _ex) \
370 unsigned long _tmp; \
371 ulong *rax = &(ctxt)->regs[VCPU_REGS_RAX]; \
372 ulong *rdx = &(ctxt)->regs[VCPU_REGS_RDX]; \
374 __asm__ __volatile__ ( \
375 _PRE_EFLAGS("0", "5", "1") \
377 _op _suffix " %6; " \
379 _POST_EFLAGS("0", "5", "1") \
380 ".pushsection .fixup,\"ax\" \n\t" \
381 "3: movb $1, %4 \n\t" \
384 _ASM_EXTABLE(1b, 3b) \
385 : "=m" ((ctxt)->eflags), "=&r" (_tmp), \
386 "+a" (*rax), "+d" (*rdx), "+qm"(_ex) \
387 : "i" (EFLAGS_MASK), "m" ((ctxt)->src.val), \
388 "a" (*rax), "d" (*rdx)); \
391 /* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
392 #define emulate_1op_rax_rdx(ctxt, _op, _ex) \
394 switch((ctxt)->src.bytes) { \
396 __emulate_1op_rax_rdx(ctxt, _op, "b", _ex); \
399 __emulate_1op_rax_rdx(ctxt, _op, "w", _ex); \
402 __emulate_1op_rax_rdx(ctxt, _op, "l", _ex); \
405 __emulate_1op_rax_rdx(ctxt, _op, "q", _ex)); \
410 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
411 enum x86_intercept intercept,
412 enum x86_intercept_stage stage)
414 struct x86_instruction_info info = {
415 .intercept = intercept,
416 .rep_prefix = ctxt->rep_prefix,
417 .modrm_mod = ctxt->modrm_mod,
418 .modrm_reg = ctxt->modrm_reg,
419 .modrm_rm = ctxt->modrm_rm,
420 .src_val = ctxt->src.val64,
421 .src_bytes = ctxt->src.bytes,
422 .dst_bytes = ctxt->dst.bytes,
423 .ad_bytes = ctxt->ad_bytes,
424 .next_rip = ctxt->eip,
427 return ctxt->ops->intercept(ctxt, &info, stage);
430 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
432 return (1UL << (ctxt->ad_bytes << 3)) - 1;
435 /* Access/update address held in a register, based on addressing mode. */
436 static inline unsigned long
437 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
439 if (ctxt->ad_bytes == sizeof(unsigned long))
442 return reg & ad_mask(ctxt);
445 static inline unsigned long
446 register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
448 return address_mask(ctxt, reg);
452 register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
454 if (ctxt->ad_bytes == sizeof(unsigned long))
457 *reg = (*reg & ~ad_mask(ctxt)) | ((*reg + inc) & ad_mask(ctxt));
460 static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
462 register_address_increment(ctxt, &ctxt->_eip, rel);
465 static u32 desc_limit_scaled(struct desc_struct *desc)
467 u32 limit = get_desc_limit(desc);
469 return desc->g ? (limit << 12) | 0xfff : limit;
472 static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
474 ctxt->has_seg_override = true;
475 ctxt->seg_override = seg;
478 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
480 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
483 return ctxt->ops->get_cached_segment_base(ctxt, seg);
486 static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
488 if (!ctxt->has_seg_override)
491 return ctxt->seg_override;
494 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
495 u32 error, bool valid)
497 ctxt->exception.vector = vec;
498 ctxt->exception.error_code = error;
499 ctxt->exception.error_code_valid = valid;
500 return X86EMUL_PROPAGATE_FAULT;
503 static int emulate_db(struct x86_emulate_ctxt *ctxt)
505 return emulate_exception(ctxt, DB_VECTOR, 0, false);
508 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
510 return emulate_exception(ctxt, GP_VECTOR, err, true);
513 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
515 return emulate_exception(ctxt, SS_VECTOR, err, true);
518 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
520 return emulate_exception(ctxt, UD_VECTOR, 0, false);
523 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
525 return emulate_exception(ctxt, TS_VECTOR, err, true);
528 static int emulate_de(struct x86_emulate_ctxt *ctxt)
530 return emulate_exception(ctxt, DE_VECTOR, 0, false);
533 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
535 return emulate_exception(ctxt, NM_VECTOR, 0, false);
538 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
541 struct desc_struct desc;
543 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
547 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
552 struct desc_struct desc;
554 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
555 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
558 static int __linearize(struct x86_emulate_ctxt *ctxt,
559 struct segmented_address addr,
560 unsigned size, bool write, bool fetch,
563 struct desc_struct desc;
570 la = seg_base(ctxt, addr.seg) + addr.ea;
571 switch (ctxt->mode) {
572 case X86EMUL_MODE_REAL:
574 case X86EMUL_MODE_PROT64:
575 if (((signed long)la << 16) >> 16 != la)
576 return emulate_gp(ctxt, 0);
579 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
583 /* code segment or read-only data segment */
584 if (((desc.type & 8) || !(desc.type & 2)) && write)
586 /* unreadable code segment */
587 if (!fetch && (desc.type & 8) && !(desc.type & 2))
589 lim = desc_limit_scaled(&desc);
590 if ((desc.type & 8) || !(desc.type & 4)) {
591 /* expand-up segment */
592 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
595 /* exapand-down segment */
596 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
598 lim = desc.d ? 0xffffffff : 0xffff;
599 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
602 cpl = ctxt->ops->cpl(ctxt);
605 if (!(desc.type & 8)) {
609 } else if ((desc.type & 8) && !(desc.type & 4)) {
610 /* nonconforming code segment */
613 } else if ((desc.type & 8) && (desc.type & 4)) {
614 /* conforming code segment */
620 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
623 return X86EMUL_CONTINUE;
625 if (addr.seg == VCPU_SREG_SS)
626 return emulate_ss(ctxt, addr.seg);
628 return emulate_gp(ctxt, addr.seg);
631 static int linearize(struct x86_emulate_ctxt *ctxt,
632 struct segmented_address addr,
633 unsigned size, bool write,
636 return __linearize(ctxt, addr, size, write, false, linear);
640 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
641 struct segmented_address addr,
648 rc = linearize(ctxt, addr, size, false, &linear);
649 if (rc != X86EMUL_CONTINUE)
651 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
655 * Fetch the next byte of the instruction being emulated which is pointed to
656 * by ctxt->_eip, then increment ctxt->_eip.
658 * Also prefetch the remaining bytes of the instruction without crossing page
659 * boundary if they are not in fetch_cache yet.
661 static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
663 struct fetch_cache *fc = &ctxt->fetch;
667 if (ctxt->_eip == fc->end) {
668 unsigned long linear;
669 struct segmented_address addr = { .seg = VCPU_SREG_CS,
671 cur_size = fc->end - fc->start;
672 size = min(15UL - cur_size,
673 PAGE_SIZE - offset_in_page(ctxt->_eip));
674 rc = __linearize(ctxt, addr, size, false, true, &linear);
675 if (unlikely(rc != X86EMUL_CONTINUE))
677 rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
678 size, &ctxt->exception);
679 if (unlikely(rc != X86EMUL_CONTINUE))
683 *dest = fc->data[ctxt->_eip - fc->start];
685 return X86EMUL_CONTINUE;
688 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
689 void *dest, unsigned size)
693 /* x86 instructions are limited to 15 bytes. */
694 if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
695 return X86EMUL_UNHANDLEABLE;
697 rc = do_insn_fetch_byte(ctxt, dest++);
698 if (rc != X86EMUL_CONTINUE)
701 return X86EMUL_CONTINUE;
704 /* Fetch next part of the instruction being emulated. */
705 #define insn_fetch(_type, _ctxt) \
706 ({ unsigned long _x; \
707 rc = do_insn_fetch(_ctxt, &_x, sizeof(_type)); \
708 if (rc != X86EMUL_CONTINUE) \
713 #define insn_fetch_arr(_arr, _size, _ctxt) \
714 ({ rc = do_insn_fetch(_ctxt, _arr, (_size)); \
715 if (rc != X86EMUL_CONTINUE) \
720 * Given the 'reg' portion of a ModRM byte, and a register block, return a
721 * pointer into the block that addresses the relevant register.
722 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
724 static void *decode_register(u8 modrm_reg, unsigned long *regs,
729 p = ®s[modrm_reg];
730 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
731 p = (unsigned char *)®s[modrm_reg & 3] + 1;
735 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
736 struct segmented_address addr,
737 u16 *size, unsigned long *address, int op_bytes)
744 rc = segmented_read_std(ctxt, addr, size, 2);
745 if (rc != X86EMUL_CONTINUE)
748 rc = segmented_read_std(ctxt, addr, address, op_bytes);
752 static int test_cc(unsigned int condition, unsigned int flags)
756 switch ((condition & 15) >> 1) {
758 rc |= (flags & EFLG_OF);
760 case 1: /* b/c/nae */
761 rc |= (flags & EFLG_CF);
764 rc |= (flags & EFLG_ZF);
767 rc |= (flags & (EFLG_CF|EFLG_ZF));
770 rc |= (flags & EFLG_SF);
773 rc |= (flags & EFLG_PF);
776 rc |= (flags & EFLG_ZF);
779 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
783 /* Odd condition identifiers (lsb == 1) have inverted sense. */
784 return (!!rc ^ (condition & 1));
787 static void fetch_register_operand(struct operand *op)
791 op->val = *(u8 *)op->addr.reg;
794 op->val = *(u16 *)op->addr.reg;
797 op->val = *(u32 *)op->addr.reg;
800 op->val = *(u64 *)op->addr.reg;
805 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
807 ctxt->ops->get_fpu(ctxt);
809 case 0: asm("movdqu %%xmm0, %0" : "=m"(*data)); break;
810 case 1: asm("movdqu %%xmm1, %0" : "=m"(*data)); break;
811 case 2: asm("movdqu %%xmm2, %0" : "=m"(*data)); break;
812 case 3: asm("movdqu %%xmm3, %0" : "=m"(*data)); break;
813 case 4: asm("movdqu %%xmm4, %0" : "=m"(*data)); break;
814 case 5: asm("movdqu %%xmm5, %0" : "=m"(*data)); break;
815 case 6: asm("movdqu %%xmm6, %0" : "=m"(*data)); break;
816 case 7: asm("movdqu %%xmm7, %0" : "=m"(*data)); break;
818 case 8: asm("movdqu %%xmm8, %0" : "=m"(*data)); break;
819 case 9: asm("movdqu %%xmm9, %0" : "=m"(*data)); break;
820 case 10: asm("movdqu %%xmm10, %0" : "=m"(*data)); break;
821 case 11: asm("movdqu %%xmm11, %0" : "=m"(*data)); break;
822 case 12: asm("movdqu %%xmm12, %0" : "=m"(*data)); break;
823 case 13: asm("movdqu %%xmm13, %0" : "=m"(*data)); break;
824 case 14: asm("movdqu %%xmm14, %0" : "=m"(*data)); break;
825 case 15: asm("movdqu %%xmm15, %0" : "=m"(*data)); break;
829 ctxt->ops->put_fpu(ctxt);
832 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
835 ctxt->ops->get_fpu(ctxt);
837 case 0: asm("movdqu %0, %%xmm0" : : "m"(*data)); break;
838 case 1: asm("movdqu %0, %%xmm1" : : "m"(*data)); break;
839 case 2: asm("movdqu %0, %%xmm2" : : "m"(*data)); break;
840 case 3: asm("movdqu %0, %%xmm3" : : "m"(*data)); break;
841 case 4: asm("movdqu %0, %%xmm4" : : "m"(*data)); break;
842 case 5: asm("movdqu %0, %%xmm5" : : "m"(*data)); break;
843 case 6: asm("movdqu %0, %%xmm6" : : "m"(*data)); break;
844 case 7: asm("movdqu %0, %%xmm7" : : "m"(*data)); break;
846 case 8: asm("movdqu %0, %%xmm8" : : "m"(*data)); break;
847 case 9: asm("movdqu %0, %%xmm9" : : "m"(*data)); break;
848 case 10: asm("movdqu %0, %%xmm10" : : "m"(*data)); break;
849 case 11: asm("movdqu %0, %%xmm11" : : "m"(*data)); break;
850 case 12: asm("movdqu %0, %%xmm12" : : "m"(*data)); break;
851 case 13: asm("movdqu %0, %%xmm13" : : "m"(*data)); break;
852 case 14: asm("movdqu %0, %%xmm14" : : "m"(*data)); break;
853 case 15: asm("movdqu %0, %%xmm15" : : "m"(*data)); break;
857 ctxt->ops->put_fpu(ctxt);
860 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
864 unsigned reg = ctxt->modrm_reg;
865 int highbyte_regs = ctxt->rex_prefix == 0;
867 if (!(ctxt->d & ModRM))
868 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
874 read_sse_reg(ctxt, &op->vec_val, reg);
879 if ((ctxt->d & ByteOp) && !inhibit_bytereg) {
880 op->addr.reg = decode_register(reg, ctxt->regs, highbyte_regs);
883 op->addr.reg = decode_register(reg, ctxt->regs, 0);
884 op->bytes = ctxt->op_bytes;
886 fetch_register_operand(op);
887 op->orig_val = op->val;
890 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
894 int index_reg = 0, base_reg = 0, scale;
895 int rc = X86EMUL_CONTINUE;
898 if (ctxt->rex_prefix) {
899 ctxt->modrm_reg = (ctxt->rex_prefix & 4) << 1; /* REX.R */
900 index_reg = (ctxt->rex_prefix & 2) << 2; /* REX.X */
901 ctxt->modrm_rm = base_reg = (ctxt->rex_prefix & 1) << 3; /* REG.B */
904 ctxt->modrm = insn_fetch(u8, ctxt);
905 ctxt->modrm_mod |= (ctxt->modrm & 0xc0) >> 6;
906 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
907 ctxt->modrm_rm |= (ctxt->modrm & 0x07);
908 ctxt->modrm_seg = VCPU_SREG_DS;
910 if (ctxt->modrm_mod == 3) {
912 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
913 op->addr.reg = decode_register(ctxt->modrm_rm,
914 ctxt->regs, ctxt->d & ByteOp);
918 op->addr.xmm = ctxt->modrm_rm;
919 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
922 fetch_register_operand(op);
928 if (ctxt->ad_bytes == 2) {
929 unsigned bx = ctxt->regs[VCPU_REGS_RBX];
930 unsigned bp = ctxt->regs[VCPU_REGS_RBP];
931 unsigned si = ctxt->regs[VCPU_REGS_RSI];
932 unsigned di = ctxt->regs[VCPU_REGS_RDI];
934 /* 16-bit ModR/M decode. */
935 switch (ctxt->modrm_mod) {
937 if (ctxt->modrm_rm == 6)
938 modrm_ea += insn_fetch(u16, ctxt);
941 modrm_ea += insn_fetch(s8, ctxt);
944 modrm_ea += insn_fetch(u16, ctxt);
947 switch (ctxt->modrm_rm) {
967 if (ctxt->modrm_mod != 0)
974 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
975 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
976 ctxt->modrm_seg = VCPU_SREG_SS;
977 modrm_ea = (u16)modrm_ea;
979 /* 32/64-bit ModR/M decode. */
980 if ((ctxt->modrm_rm & 7) == 4) {
981 sib = insn_fetch(u8, ctxt);
982 index_reg |= (sib >> 3) & 7;
986 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
987 modrm_ea += insn_fetch(s32, ctxt);
989 modrm_ea += ctxt->regs[base_reg];
991 modrm_ea += ctxt->regs[index_reg] << scale;
992 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
993 if (ctxt->mode == X86EMUL_MODE_PROT64)
994 ctxt->rip_relative = 1;
996 modrm_ea += ctxt->regs[ctxt->modrm_rm];
997 switch (ctxt->modrm_mod) {
999 if (ctxt->modrm_rm == 5)
1000 modrm_ea += insn_fetch(s32, ctxt);
1003 modrm_ea += insn_fetch(s8, ctxt);
1006 modrm_ea += insn_fetch(s32, ctxt);
1010 op->addr.mem.ea = modrm_ea;
1015 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1018 int rc = X86EMUL_CONTINUE;
1021 switch (ctxt->ad_bytes) {
1023 op->addr.mem.ea = insn_fetch(u16, ctxt);
1026 op->addr.mem.ea = insn_fetch(u32, ctxt);
1029 op->addr.mem.ea = insn_fetch(u64, ctxt);
1036 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1040 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1041 mask = ~(ctxt->dst.bytes * 8 - 1);
1043 if (ctxt->src.bytes == 2)
1044 sv = (s16)ctxt->src.val & (s16)mask;
1045 else if (ctxt->src.bytes == 4)
1046 sv = (s32)ctxt->src.val & (s32)mask;
1048 ctxt->dst.addr.mem.ea += (sv >> 3);
1051 /* only subword offset */
1052 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1055 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1056 unsigned long addr, void *dest, unsigned size)
1059 struct read_cache *mc = &ctxt->mem_read;
1062 int n = min(size, 8u);
1064 if (mc->pos < mc->end)
1067 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, n,
1069 if (rc != X86EMUL_CONTINUE)
1074 memcpy(dest, mc->data + mc->pos, n);
1079 return X86EMUL_CONTINUE;
1082 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1083 struct segmented_address addr,
1090 rc = linearize(ctxt, addr, size, false, &linear);
1091 if (rc != X86EMUL_CONTINUE)
1093 return read_emulated(ctxt, linear, data, size);
1096 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1097 struct segmented_address addr,
1104 rc = linearize(ctxt, addr, size, true, &linear);
1105 if (rc != X86EMUL_CONTINUE)
1107 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1111 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1112 struct segmented_address addr,
1113 const void *orig_data, const void *data,
1119 rc = linearize(ctxt, addr, size, true, &linear);
1120 if (rc != X86EMUL_CONTINUE)
1122 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1123 size, &ctxt->exception);
1126 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1127 unsigned int size, unsigned short port,
1130 struct read_cache *rc = &ctxt->io_read;
1132 if (rc->pos == rc->end) { /* refill pio read ahead */
1133 unsigned int in_page, n;
1134 unsigned int count = ctxt->rep_prefix ?
1135 address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) : 1;
1136 in_page = (ctxt->eflags & EFLG_DF) ?
1137 offset_in_page(ctxt->regs[VCPU_REGS_RDI]) :
1138 PAGE_SIZE - offset_in_page(ctxt->regs[VCPU_REGS_RDI]);
1139 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1143 rc->pos = rc->end = 0;
1144 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1149 memcpy(dest, rc->data + rc->pos, size);
1154 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1155 u16 selector, struct desc_ptr *dt)
1157 struct x86_emulate_ops *ops = ctxt->ops;
1159 if (selector & 1 << 2) {
1160 struct desc_struct desc;
1163 memset (dt, 0, sizeof *dt);
1164 if (!ops->get_segment(ctxt, &sel, &desc, NULL, VCPU_SREG_LDTR))
1167 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1168 dt->address = get_desc_base(&desc);
1170 ops->get_gdt(ctxt, dt);
1173 /* allowed just for 8 bytes segments */
1174 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1175 u16 selector, struct desc_struct *desc)
1178 u16 index = selector >> 3;
1181 get_descriptor_table_ptr(ctxt, selector, &dt);
1183 if (dt.size < index * 8 + 7)
1184 return emulate_gp(ctxt, selector & 0xfffc);
1186 addr = dt.address + index * 8;
1187 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1191 /* allowed just for 8 bytes segments */
1192 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1193 u16 selector, struct desc_struct *desc)
1196 u16 index = selector >> 3;
1199 get_descriptor_table_ptr(ctxt, selector, &dt);
1201 if (dt.size < index * 8 + 7)
1202 return emulate_gp(ctxt, selector & 0xfffc);
1204 addr = dt.address + index * 8;
1205 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1209 /* Does not support long mode */
1210 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1211 u16 selector, int seg)
1213 struct desc_struct seg_desc;
1215 unsigned err_vec = GP_VECTOR;
1217 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1220 memset(&seg_desc, 0, sizeof seg_desc);
1222 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1223 || ctxt->mode == X86EMUL_MODE_REAL) {
1224 /* set real mode segment descriptor */
1225 set_desc_base(&seg_desc, selector << 4);
1226 set_desc_limit(&seg_desc, 0xffff);
1233 /* NULL selector is not valid for TR, CS and SS */
1234 if ((seg == VCPU_SREG_CS || seg == VCPU_SREG_SS || seg == VCPU_SREG_TR)
1238 /* TR should be in GDT only */
1239 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1242 if (null_selector) /* for NULL selector skip all following checks */
1245 ret = read_segment_descriptor(ctxt, selector, &seg_desc);
1246 if (ret != X86EMUL_CONTINUE)
1249 err_code = selector & 0xfffc;
1250 err_vec = GP_VECTOR;
1252 /* can't load system descriptor into segment selecor */
1253 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1257 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1263 cpl = ctxt->ops->cpl(ctxt);
1268 * segment is not a writable data segment or segment
1269 * selector's RPL != CPL or segment selector's RPL != CPL
1271 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1275 if (!(seg_desc.type & 8))
1278 if (seg_desc.type & 4) {
1284 if (rpl > cpl || dpl != cpl)
1287 /* CS(RPL) <- CPL */
1288 selector = (selector & 0xfffc) | cpl;
1291 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1294 case VCPU_SREG_LDTR:
1295 if (seg_desc.s || seg_desc.type != 2)
1298 default: /* DS, ES, FS, or GS */
1300 * segment is not a data or readable code segment or
1301 * ((segment is a data or nonconforming code segment)
1302 * and (both RPL and CPL > DPL))
1304 if ((seg_desc.type & 0xa) == 0x8 ||
1305 (((seg_desc.type & 0xc) != 0xc) &&
1306 (rpl > dpl && cpl > dpl)))
1312 /* mark segment as accessed */
1314 ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1315 if (ret != X86EMUL_CONTINUE)
1319 ctxt->ops->set_segment(ctxt, selector, &seg_desc, 0, seg);
1320 return X86EMUL_CONTINUE;
1322 emulate_exception(ctxt, err_vec, err_code, true);
1323 return X86EMUL_PROPAGATE_FAULT;
1326 static void write_register_operand(struct operand *op)
1328 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1329 switch (op->bytes) {
1331 *(u8 *)op->addr.reg = (u8)op->val;
1334 *(u16 *)op->addr.reg = (u16)op->val;
1337 *op->addr.reg = (u32)op->val;
1338 break; /* 64b: zero-extend */
1340 *op->addr.reg = op->val;
1345 static int writeback(struct x86_emulate_ctxt *ctxt)
1349 switch (ctxt->dst.type) {
1351 write_register_operand(&ctxt->dst);
1354 if (ctxt->lock_prefix)
1355 rc = segmented_cmpxchg(ctxt,
1357 &ctxt->dst.orig_val,
1361 rc = segmented_write(ctxt,
1365 if (rc != X86EMUL_CONTINUE)
1369 write_sse_reg(ctxt, &ctxt->dst.vec_val, ctxt->dst.addr.xmm);
1377 return X86EMUL_CONTINUE;
1380 static int em_push(struct x86_emulate_ctxt *ctxt)
1382 struct segmented_address addr;
1384 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], -ctxt->op_bytes);
1385 addr.ea = register_address(ctxt, ctxt->regs[VCPU_REGS_RSP]);
1386 addr.seg = VCPU_SREG_SS;
1388 /* Disable writeback. */
1389 ctxt->dst.type = OP_NONE;
1390 return segmented_write(ctxt, addr, &ctxt->src.val, ctxt->op_bytes);
1393 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1394 void *dest, int len)
1397 struct segmented_address addr;
1399 addr.ea = register_address(ctxt, ctxt->regs[VCPU_REGS_RSP]);
1400 addr.seg = VCPU_SREG_SS;
1401 rc = segmented_read(ctxt, addr, dest, len);
1402 if (rc != X86EMUL_CONTINUE)
1405 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], len);
1409 static int em_pop(struct x86_emulate_ctxt *ctxt)
1411 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1414 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1415 void *dest, int len)
1418 unsigned long val, change_mask;
1419 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1420 int cpl = ctxt->ops->cpl(ctxt);
1422 rc = emulate_pop(ctxt, &val, len);
1423 if (rc != X86EMUL_CONTINUE)
1426 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1427 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1429 switch(ctxt->mode) {
1430 case X86EMUL_MODE_PROT64:
1431 case X86EMUL_MODE_PROT32:
1432 case X86EMUL_MODE_PROT16:
1434 change_mask |= EFLG_IOPL;
1436 change_mask |= EFLG_IF;
1438 case X86EMUL_MODE_VM86:
1440 return emulate_gp(ctxt, 0);
1441 change_mask |= EFLG_IF;
1443 default: /* real mode */
1444 change_mask |= (EFLG_IOPL | EFLG_IF);
1448 *(unsigned long *)dest =
1449 (ctxt->eflags & ~change_mask) | (val & change_mask);
1454 static int em_popf(struct x86_emulate_ctxt *ctxt)
1456 ctxt->dst.type = OP_REG;
1457 ctxt->dst.addr.reg = &ctxt->eflags;
1458 ctxt->dst.bytes = ctxt->op_bytes;
1459 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1462 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1464 int seg = ctxt->src2.val;
1466 ctxt->src.val = get_segment_selector(ctxt, seg);
1468 return em_push(ctxt);
1471 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1473 int seg = ctxt->src2.val;
1474 unsigned long selector;
1477 rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1478 if (rc != X86EMUL_CONTINUE)
1481 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1485 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1487 unsigned long old_esp = ctxt->regs[VCPU_REGS_RSP];
1488 int rc = X86EMUL_CONTINUE;
1489 int reg = VCPU_REGS_RAX;
1491 while (reg <= VCPU_REGS_RDI) {
1492 (reg == VCPU_REGS_RSP) ?
1493 (ctxt->src.val = old_esp) : (ctxt->src.val = ctxt->regs[reg]);
1496 if (rc != X86EMUL_CONTINUE)
1505 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1507 ctxt->src.val = (unsigned long)ctxt->eflags;
1508 return em_push(ctxt);
1511 static int em_popa(struct x86_emulate_ctxt *ctxt)
1513 int rc = X86EMUL_CONTINUE;
1514 int reg = VCPU_REGS_RDI;
1516 while (reg >= VCPU_REGS_RAX) {
1517 if (reg == VCPU_REGS_RSP) {
1518 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP],
1523 rc = emulate_pop(ctxt, &ctxt->regs[reg], ctxt->op_bytes);
1524 if (rc != X86EMUL_CONTINUE)
1531 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1533 struct x86_emulate_ops *ops = ctxt->ops;
1540 /* TODO: Add limit checks */
1541 ctxt->src.val = ctxt->eflags;
1543 if (rc != X86EMUL_CONTINUE)
1546 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1548 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1550 if (rc != X86EMUL_CONTINUE)
1553 ctxt->src.val = ctxt->_eip;
1555 if (rc != X86EMUL_CONTINUE)
1558 ops->get_idt(ctxt, &dt);
1560 eip_addr = dt.address + (irq << 2);
1561 cs_addr = dt.address + (irq << 2) + 2;
1563 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1564 if (rc != X86EMUL_CONTINUE)
1567 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1568 if (rc != X86EMUL_CONTINUE)
1571 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1572 if (rc != X86EMUL_CONTINUE)
1580 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1582 switch(ctxt->mode) {
1583 case X86EMUL_MODE_REAL:
1584 return emulate_int_real(ctxt, irq);
1585 case X86EMUL_MODE_VM86:
1586 case X86EMUL_MODE_PROT16:
1587 case X86EMUL_MODE_PROT32:
1588 case X86EMUL_MODE_PROT64:
1590 /* Protected mode interrupts unimplemented yet */
1591 return X86EMUL_UNHANDLEABLE;
1595 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
1597 int rc = X86EMUL_CONTINUE;
1598 unsigned long temp_eip = 0;
1599 unsigned long temp_eflags = 0;
1600 unsigned long cs = 0;
1601 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1602 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1603 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1604 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1606 /* TODO: Add stack limit check */
1608 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
1610 if (rc != X86EMUL_CONTINUE)
1613 if (temp_eip & ~0xffff)
1614 return emulate_gp(ctxt, 0);
1616 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1618 if (rc != X86EMUL_CONTINUE)
1621 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
1623 if (rc != X86EMUL_CONTINUE)
1626 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1628 if (rc != X86EMUL_CONTINUE)
1631 ctxt->_eip = temp_eip;
1634 if (ctxt->op_bytes == 4)
1635 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1636 else if (ctxt->op_bytes == 2) {
1637 ctxt->eflags &= ~0xffff;
1638 ctxt->eflags |= temp_eflags;
1641 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1642 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1647 static int em_iret(struct x86_emulate_ctxt *ctxt)
1649 switch(ctxt->mode) {
1650 case X86EMUL_MODE_REAL:
1651 return emulate_iret_real(ctxt);
1652 case X86EMUL_MODE_VM86:
1653 case X86EMUL_MODE_PROT16:
1654 case X86EMUL_MODE_PROT32:
1655 case X86EMUL_MODE_PROT64:
1657 /* iret from protected mode unimplemented yet */
1658 return X86EMUL_UNHANDLEABLE;
1662 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
1667 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1669 rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
1670 if (rc != X86EMUL_CONTINUE)
1674 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
1675 return X86EMUL_CONTINUE;
1678 static int em_grp1a(struct x86_emulate_ctxt *ctxt)
1680 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->dst.bytes);
1683 static int em_grp2(struct x86_emulate_ctxt *ctxt)
1685 switch (ctxt->modrm_reg) {
1687 emulate_2op_SrcB(ctxt, "rol");
1690 emulate_2op_SrcB(ctxt, "ror");
1693 emulate_2op_SrcB(ctxt, "rcl");
1696 emulate_2op_SrcB(ctxt, "rcr");
1698 case 4: /* sal/shl */
1699 case 6: /* sal/shl */
1700 emulate_2op_SrcB(ctxt, "sal");
1703 emulate_2op_SrcB(ctxt, "shr");
1706 emulate_2op_SrcB(ctxt, "sar");
1709 return X86EMUL_CONTINUE;
1712 static int em_not(struct x86_emulate_ctxt *ctxt)
1714 ctxt->dst.val = ~ctxt->dst.val;
1715 return X86EMUL_CONTINUE;
1718 static int em_neg(struct x86_emulate_ctxt *ctxt)
1720 emulate_1op(ctxt, "neg");
1721 return X86EMUL_CONTINUE;
1724 static int em_mul_ex(struct x86_emulate_ctxt *ctxt)
1728 emulate_1op_rax_rdx(ctxt, "mul", ex);
1729 return X86EMUL_CONTINUE;
1732 static int em_imul_ex(struct x86_emulate_ctxt *ctxt)
1736 emulate_1op_rax_rdx(ctxt, "imul", ex);
1737 return X86EMUL_CONTINUE;
1740 static int em_div_ex(struct x86_emulate_ctxt *ctxt)
1744 emulate_1op_rax_rdx(ctxt, "div", de);
1746 return emulate_de(ctxt);
1747 return X86EMUL_CONTINUE;
1750 static int em_idiv_ex(struct x86_emulate_ctxt *ctxt)
1754 emulate_1op_rax_rdx(ctxt, "idiv", de);
1756 return emulate_de(ctxt);
1757 return X86EMUL_CONTINUE;
1760 static int em_grp45(struct x86_emulate_ctxt *ctxt)
1762 int rc = X86EMUL_CONTINUE;
1764 switch (ctxt->modrm_reg) {
1766 emulate_1op(ctxt, "inc");
1769 emulate_1op(ctxt, "dec");
1771 case 2: /* call near abs */ {
1773 old_eip = ctxt->_eip;
1774 ctxt->_eip = ctxt->src.val;
1775 ctxt->src.val = old_eip;
1779 case 4: /* jmp abs */
1780 ctxt->_eip = ctxt->src.val;
1782 case 5: /* jmp far */
1783 rc = em_jmp_far(ctxt);
1792 static int em_grp9(struct x86_emulate_ctxt *ctxt)
1794 u64 old = ctxt->dst.orig_val64;
1796 if (((u32) (old >> 0) != (u32) ctxt->regs[VCPU_REGS_RAX]) ||
1797 ((u32) (old >> 32) != (u32) ctxt->regs[VCPU_REGS_RDX])) {
1798 ctxt->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1799 ctxt->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1800 ctxt->eflags &= ~EFLG_ZF;
1802 ctxt->dst.val64 = ((u64)ctxt->regs[VCPU_REGS_RCX] << 32) |
1803 (u32) ctxt->regs[VCPU_REGS_RBX];
1805 ctxt->eflags |= EFLG_ZF;
1807 return X86EMUL_CONTINUE;
1810 static int em_ret(struct x86_emulate_ctxt *ctxt)
1812 ctxt->dst.type = OP_REG;
1813 ctxt->dst.addr.reg = &ctxt->_eip;
1814 ctxt->dst.bytes = ctxt->op_bytes;
1815 return em_pop(ctxt);
1818 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
1823 rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
1824 if (rc != X86EMUL_CONTINUE)
1826 if (ctxt->op_bytes == 4)
1827 ctxt->_eip = (u32)ctxt->_eip;
1828 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1829 if (rc != X86EMUL_CONTINUE)
1831 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1835 static int em_lseg(struct x86_emulate_ctxt *ctxt)
1837 int seg = ctxt->src2.val;
1841 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1843 rc = load_segment_descriptor(ctxt, sel, seg);
1844 if (rc != X86EMUL_CONTINUE)
1847 ctxt->dst.val = ctxt->src.val;
1852 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1853 struct desc_struct *cs, struct desc_struct *ss)
1857 memset(cs, 0, sizeof(struct desc_struct));
1858 ctxt->ops->get_segment(ctxt, &selector, cs, NULL, VCPU_SREG_CS);
1859 memset(ss, 0, sizeof(struct desc_struct));
1861 cs->l = 0; /* will be adjusted later */
1862 set_desc_base(cs, 0); /* flat segment */
1863 cs->g = 1; /* 4kb granularity */
1864 set_desc_limit(cs, 0xfffff); /* 4GB limit */
1865 cs->type = 0x0b; /* Read, Execute, Accessed */
1867 cs->dpl = 0; /* will be adjusted later */
1871 set_desc_base(ss, 0); /* flat segment */
1872 set_desc_limit(ss, 0xfffff); /* 4GB limit */
1873 ss->g = 1; /* 4kb granularity */
1875 ss->type = 0x03; /* Read/Write, Accessed */
1876 ss->d = 1; /* 32bit stack segment */
1881 static int em_syscall(struct x86_emulate_ctxt *ctxt)
1883 struct x86_emulate_ops *ops = ctxt->ops;
1884 struct desc_struct cs, ss;
1889 /* syscall is not available in real mode */
1890 if (ctxt->mode == X86EMUL_MODE_REAL ||
1891 ctxt->mode == X86EMUL_MODE_VM86)
1892 return emulate_ud(ctxt);
1894 ops->get_msr(ctxt, MSR_EFER, &efer);
1895 setup_syscalls_segments(ctxt, &cs, &ss);
1897 ops->get_msr(ctxt, MSR_STAR, &msr_data);
1899 cs_sel = (u16)(msr_data & 0xfffc);
1900 ss_sel = (u16)(msr_data + 8);
1902 if (efer & EFER_LMA) {
1906 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
1907 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
1909 ctxt->regs[VCPU_REGS_RCX] = ctxt->_eip;
1910 if (efer & EFER_LMA) {
1911 #ifdef CONFIG_X86_64
1912 ctxt->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
1915 ctxt->mode == X86EMUL_MODE_PROT64 ?
1916 MSR_LSTAR : MSR_CSTAR, &msr_data);
1917 ctxt->_eip = msr_data;
1919 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
1920 ctxt->eflags &= ~(msr_data | EFLG_RF);
1924 ops->get_msr(ctxt, MSR_STAR, &msr_data);
1925 ctxt->_eip = (u32)msr_data;
1927 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1930 return X86EMUL_CONTINUE;
1933 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
1935 struct x86_emulate_ops *ops = ctxt->ops;
1936 struct desc_struct cs, ss;
1941 ops->get_msr(ctxt, MSR_EFER, &efer);
1942 /* inject #GP if in real mode */
1943 if (ctxt->mode == X86EMUL_MODE_REAL)
1944 return emulate_gp(ctxt, 0);
1946 /* XXX sysenter/sysexit have not been tested in 64bit mode.
1947 * Therefore, we inject an #UD.
1949 if (ctxt->mode == X86EMUL_MODE_PROT64)
1950 return emulate_ud(ctxt);
1952 setup_syscalls_segments(ctxt, &cs, &ss);
1954 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
1955 switch (ctxt->mode) {
1956 case X86EMUL_MODE_PROT32:
1957 if ((msr_data & 0xfffc) == 0x0)
1958 return emulate_gp(ctxt, 0);
1960 case X86EMUL_MODE_PROT64:
1961 if (msr_data == 0x0)
1962 return emulate_gp(ctxt, 0);
1966 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1967 cs_sel = (u16)msr_data;
1968 cs_sel &= ~SELECTOR_RPL_MASK;
1969 ss_sel = cs_sel + 8;
1970 ss_sel &= ~SELECTOR_RPL_MASK;
1971 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
1976 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
1977 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
1979 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
1980 ctxt->_eip = msr_data;
1982 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
1983 ctxt->regs[VCPU_REGS_RSP] = msr_data;
1985 return X86EMUL_CONTINUE;
1988 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
1990 struct x86_emulate_ops *ops = ctxt->ops;
1991 struct desc_struct cs, ss;
1994 u16 cs_sel = 0, ss_sel = 0;
1996 /* inject #GP if in real mode or Virtual 8086 mode */
1997 if (ctxt->mode == X86EMUL_MODE_REAL ||
1998 ctxt->mode == X86EMUL_MODE_VM86)
1999 return emulate_gp(ctxt, 0);
2001 setup_syscalls_segments(ctxt, &cs, &ss);
2003 if ((ctxt->rex_prefix & 0x8) != 0x0)
2004 usermode = X86EMUL_MODE_PROT64;
2006 usermode = X86EMUL_MODE_PROT32;
2010 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2012 case X86EMUL_MODE_PROT32:
2013 cs_sel = (u16)(msr_data + 16);
2014 if ((msr_data & 0xfffc) == 0x0)
2015 return emulate_gp(ctxt, 0);
2016 ss_sel = (u16)(msr_data + 24);
2018 case X86EMUL_MODE_PROT64:
2019 cs_sel = (u16)(msr_data + 32);
2020 if (msr_data == 0x0)
2021 return emulate_gp(ctxt, 0);
2022 ss_sel = cs_sel + 8;
2027 cs_sel |= SELECTOR_RPL_MASK;
2028 ss_sel |= SELECTOR_RPL_MASK;
2030 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2031 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2033 ctxt->_eip = ctxt->regs[VCPU_REGS_RDX];
2034 ctxt->regs[VCPU_REGS_RSP] = ctxt->regs[VCPU_REGS_RCX];
2036 return X86EMUL_CONTINUE;
2039 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2042 if (ctxt->mode == X86EMUL_MODE_REAL)
2044 if (ctxt->mode == X86EMUL_MODE_VM86)
2046 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2047 return ctxt->ops->cpl(ctxt) > iopl;
2050 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2053 struct x86_emulate_ops *ops = ctxt->ops;
2054 struct desc_struct tr_seg;
2057 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2058 unsigned mask = (1 << len) - 1;
2061 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2064 if (desc_limit_scaled(&tr_seg) < 103)
2066 base = get_desc_base(&tr_seg);
2067 #ifdef CONFIG_X86_64
2068 base |= ((u64)base3) << 32;
2070 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2071 if (r != X86EMUL_CONTINUE)
2073 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2075 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2076 if (r != X86EMUL_CONTINUE)
2078 if ((perm >> bit_idx) & mask)
2083 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2089 if (emulator_bad_iopl(ctxt))
2090 if (!emulator_io_port_access_allowed(ctxt, port, len))
2093 ctxt->perm_ok = true;
2098 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2099 struct tss_segment_16 *tss)
2101 tss->ip = ctxt->_eip;
2102 tss->flag = ctxt->eflags;
2103 tss->ax = ctxt->regs[VCPU_REGS_RAX];
2104 tss->cx = ctxt->regs[VCPU_REGS_RCX];
2105 tss->dx = ctxt->regs[VCPU_REGS_RDX];
2106 tss->bx = ctxt->regs[VCPU_REGS_RBX];
2107 tss->sp = ctxt->regs[VCPU_REGS_RSP];
2108 tss->bp = ctxt->regs[VCPU_REGS_RBP];
2109 tss->si = ctxt->regs[VCPU_REGS_RSI];
2110 tss->di = ctxt->regs[VCPU_REGS_RDI];
2112 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2113 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2114 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2115 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2116 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2119 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2120 struct tss_segment_16 *tss)
2124 ctxt->_eip = tss->ip;
2125 ctxt->eflags = tss->flag | 2;
2126 ctxt->regs[VCPU_REGS_RAX] = tss->ax;
2127 ctxt->regs[VCPU_REGS_RCX] = tss->cx;
2128 ctxt->regs[VCPU_REGS_RDX] = tss->dx;
2129 ctxt->regs[VCPU_REGS_RBX] = tss->bx;
2130 ctxt->regs[VCPU_REGS_RSP] = tss->sp;
2131 ctxt->regs[VCPU_REGS_RBP] = tss->bp;
2132 ctxt->regs[VCPU_REGS_RSI] = tss->si;
2133 ctxt->regs[VCPU_REGS_RDI] = tss->di;
2136 * SDM says that segment selectors are loaded before segment
2139 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2140 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2141 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2142 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2143 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2146 * Now load segment descriptors. If fault happenes at this stage
2147 * it is handled in a context of new task
2149 ret = load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR);
2150 if (ret != X86EMUL_CONTINUE)
2152 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2153 if (ret != X86EMUL_CONTINUE)
2155 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2156 if (ret != X86EMUL_CONTINUE)
2158 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2159 if (ret != X86EMUL_CONTINUE)
2161 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2162 if (ret != X86EMUL_CONTINUE)
2165 return X86EMUL_CONTINUE;
2168 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2169 u16 tss_selector, u16 old_tss_sel,
2170 ulong old_tss_base, struct desc_struct *new_desc)
2172 struct x86_emulate_ops *ops = ctxt->ops;
2173 struct tss_segment_16 tss_seg;
2175 u32 new_tss_base = get_desc_base(new_desc);
2177 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2179 if (ret != X86EMUL_CONTINUE)
2180 /* FIXME: need to provide precise fault address */
2183 save_state_to_tss16(ctxt, &tss_seg);
2185 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2187 if (ret != X86EMUL_CONTINUE)
2188 /* FIXME: need to provide precise fault address */
2191 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2193 if (ret != X86EMUL_CONTINUE)
2194 /* FIXME: need to provide precise fault address */
2197 if (old_tss_sel != 0xffff) {
2198 tss_seg.prev_task_link = old_tss_sel;
2200 ret = ops->write_std(ctxt, new_tss_base,
2201 &tss_seg.prev_task_link,
2202 sizeof tss_seg.prev_task_link,
2204 if (ret != X86EMUL_CONTINUE)
2205 /* FIXME: need to provide precise fault address */
2209 return load_state_from_tss16(ctxt, &tss_seg);
2212 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2213 struct tss_segment_32 *tss)
2215 tss->cr3 = ctxt->ops->get_cr(ctxt, 3);
2216 tss->eip = ctxt->_eip;
2217 tss->eflags = ctxt->eflags;
2218 tss->eax = ctxt->regs[VCPU_REGS_RAX];
2219 tss->ecx = ctxt->regs[VCPU_REGS_RCX];
2220 tss->edx = ctxt->regs[VCPU_REGS_RDX];
2221 tss->ebx = ctxt->regs[VCPU_REGS_RBX];
2222 tss->esp = ctxt->regs[VCPU_REGS_RSP];
2223 tss->ebp = ctxt->regs[VCPU_REGS_RBP];
2224 tss->esi = ctxt->regs[VCPU_REGS_RSI];
2225 tss->edi = ctxt->regs[VCPU_REGS_RDI];
2227 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2228 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2229 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2230 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2231 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2232 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2233 tss->ldt_selector = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2236 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2237 struct tss_segment_32 *tss)
2241 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2242 return emulate_gp(ctxt, 0);
2243 ctxt->_eip = tss->eip;
2244 ctxt->eflags = tss->eflags | 2;
2245 ctxt->regs[VCPU_REGS_RAX] = tss->eax;
2246 ctxt->regs[VCPU_REGS_RCX] = tss->ecx;
2247 ctxt->regs[VCPU_REGS_RDX] = tss->edx;
2248 ctxt->regs[VCPU_REGS_RBX] = tss->ebx;
2249 ctxt->regs[VCPU_REGS_RSP] = tss->esp;
2250 ctxt->regs[VCPU_REGS_RBP] = tss->ebp;
2251 ctxt->regs[VCPU_REGS_RSI] = tss->esi;
2252 ctxt->regs[VCPU_REGS_RDI] = tss->edi;
2255 * SDM says that segment selectors are loaded before segment
2258 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2259 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2260 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2261 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2262 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2263 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2264 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2267 * Now load segment descriptors. If fault happenes at this stage
2268 * it is handled in a context of new task
2270 ret = load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2271 if (ret != X86EMUL_CONTINUE)
2273 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2274 if (ret != X86EMUL_CONTINUE)
2276 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2277 if (ret != X86EMUL_CONTINUE)
2279 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2280 if (ret != X86EMUL_CONTINUE)
2282 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2283 if (ret != X86EMUL_CONTINUE)
2285 ret = load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS);
2286 if (ret != X86EMUL_CONTINUE)
2288 ret = load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS);
2289 if (ret != X86EMUL_CONTINUE)
2292 return X86EMUL_CONTINUE;
2295 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2296 u16 tss_selector, u16 old_tss_sel,
2297 ulong old_tss_base, struct desc_struct *new_desc)
2299 struct x86_emulate_ops *ops = ctxt->ops;
2300 struct tss_segment_32 tss_seg;
2302 u32 new_tss_base = get_desc_base(new_desc);
2304 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2306 if (ret != X86EMUL_CONTINUE)
2307 /* FIXME: need to provide precise fault address */
2310 save_state_to_tss32(ctxt, &tss_seg);
2312 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2314 if (ret != X86EMUL_CONTINUE)
2315 /* FIXME: need to provide precise fault address */
2318 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2320 if (ret != X86EMUL_CONTINUE)
2321 /* FIXME: need to provide precise fault address */
2324 if (old_tss_sel != 0xffff) {
2325 tss_seg.prev_task_link = old_tss_sel;
2327 ret = ops->write_std(ctxt, new_tss_base,
2328 &tss_seg.prev_task_link,
2329 sizeof tss_seg.prev_task_link,
2331 if (ret != X86EMUL_CONTINUE)
2332 /* FIXME: need to provide precise fault address */
2336 return load_state_from_tss32(ctxt, &tss_seg);
2339 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2340 u16 tss_selector, int reason,
2341 bool has_error_code, u32 error_code)
2343 struct x86_emulate_ops *ops = ctxt->ops;
2344 struct desc_struct curr_tss_desc, next_tss_desc;
2346 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2347 ulong old_tss_base =
2348 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2351 /* FIXME: old_tss_base == ~0 ? */
2353 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2354 if (ret != X86EMUL_CONTINUE)
2356 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2357 if (ret != X86EMUL_CONTINUE)
2360 /* FIXME: check that next_tss_desc is tss */
2362 if (reason != TASK_SWITCH_IRET) {
2363 if ((tss_selector & 3) > next_tss_desc.dpl ||
2364 ops->cpl(ctxt) > next_tss_desc.dpl)
2365 return emulate_gp(ctxt, 0);
2368 desc_limit = desc_limit_scaled(&next_tss_desc);
2369 if (!next_tss_desc.p ||
2370 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2371 desc_limit < 0x2b)) {
2372 emulate_ts(ctxt, tss_selector & 0xfffc);
2373 return X86EMUL_PROPAGATE_FAULT;
2376 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2377 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2378 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2381 if (reason == TASK_SWITCH_IRET)
2382 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2384 /* set back link to prev task only if NT bit is set in eflags
2385 note that old_tss_sel is not used afetr this point */
2386 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2387 old_tss_sel = 0xffff;
2389 if (next_tss_desc.type & 8)
2390 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2391 old_tss_base, &next_tss_desc);
2393 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2394 old_tss_base, &next_tss_desc);
2395 if (ret != X86EMUL_CONTINUE)
2398 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2399 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2401 if (reason != TASK_SWITCH_IRET) {
2402 next_tss_desc.type |= (1 << 1); /* set busy flag */
2403 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2406 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2407 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2409 if (has_error_code) {
2410 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2411 ctxt->lock_prefix = 0;
2412 ctxt->src.val = (unsigned long) error_code;
2413 ret = em_push(ctxt);
2419 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2420 u16 tss_selector, int reason,
2421 bool has_error_code, u32 error_code)
2425 ctxt->_eip = ctxt->eip;
2426 ctxt->dst.type = OP_NONE;
2428 rc = emulator_do_task_switch(ctxt, tss_selector, reason,
2429 has_error_code, error_code);
2431 if (rc == X86EMUL_CONTINUE)
2432 ctxt->eip = ctxt->_eip;
2434 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2437 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned seg,
2438 int reg, struct operand *op)
2440 int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;
2442 register_address_increment(ctxt, &ctxt->regs[reg], df * op->bytes);
2443 op->addr.mem.ea = register_address(ctxt, ctxt->regs[reg]);
2444 op->addr.mem.seg = seg;
2447 static int em_das(struct x86_emulate_ctxt *ctxt)
2450 bool af, cf, old_cf;
2452 cf = ctxt->eflags & X86_EFLAGS_CF;
2458 af = ctxt->eflags & X86_EFLAGS_AF;
2459 if ((al & 0x0f) > 9 || af) {
2461 cf = old_cf | (al >= 250);
2466 if (old_al > 0x99 || old_cf) {
2472 /* Set PF, ZF, SF */
2473 ctxt->src.type = OP_IMM;
2475 ctxt->src.bytes = 1;
2476 emulate_2op_SrcV(ctxt, "or");
2477 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2479 ctxt->eflags |= X86_EFLAGS_CF;
2481 ctxt->eflags |= X86_EFLAGS_AF;
2482 return X86EMUL_CONTINUE;
2485 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2491 old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2492 old_eip = ctxt->_eip;
2494 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2495 if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
2496 return X86EMUL_CONTINUE;
2499 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2501 ctxt->src.val = old_cs;
2503 if (rc != X86EMUL_CONTINUE)
2506 ctxt->src.val = old_eip;
2507 return em_push(ctxt);
2510 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2514 ctxt->dst.type = OP_REG;
2515 ctxt->dst.addr.reg = &ctxt->_eip;
2516 ctxt->dst.bytes = ctxt->op_bytes;
2517 rc = emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
2518 if (rc != X86EMUL_CONTINUE)
2520 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], ctxt->src.val);
2521 return X86EMUL_CONTINUE;
2524 static int em_add(struct x86_emulate_ctxt *ctxt)
2526 emulate_2op_SrcV(ctxt, "add");
2527 return X86EMUL_CONTINUE;
2530 static int em_or(struct x86_emulate_ctxt *ctxt)
2532 emulate_2op_SrcV(ctxt, "or");
2533 return X86EMUL_CONTINUE;
2536 static int em_adc(struct x86_emulate_ctxt *ctxt)
2538 emulate_2op_SrcV(ctxt, "adc");
2539 return X86EMUL_CONTINUE;
2542 static int em_sbb(struct x86_emulate_ctxt *ctxt)
2544 emulate_2op_SrcV(ctxt, "sbb");
2545 return X86EMUL_CONTINUE;
2548 static int em_and(struct x86_emulate_ctxt *ctxt)
2550 emulate_2op_SrcV(ctxt, "and");
2551 return X86EMUL_CONTINUE;
2554 static int em_sub(struct x86_emulate_ctxt *ctxt)
2556 emulate_2op_SrcV(ctxt, "sub");
2557 return X86EMUL_CONTINUE;
2560 static int em_xor(struct x86_emulate_ctxt *ctxt)
2562 emulate_2op_SrcV(ctxt, "xor");
2563 return X86EMUL_CONTINUE;
2566 static int em_cmp(struct x86_emulate_ctxt *ctxt)
2568 emulate_2op_SrcV(ctxt, "cmp");
2569 /* Disable writeback. */
2570 ctxt->dst.type = OP_NONE;
2571 return X86EMUL_CONTINUE;
2574 static int em_test(struct x86_emulate_ctxt *ctxt)
2576 emulate_2op_SrcV(ctxt, "test");
2577 /* Disable writeback. */
2578 ctxt->dst.type = OP_NONE;
2579 return X86EMUL_CONTINUE;
2582 static int em_xchg(struct x86_emulate_ctxt *ctxt)
2584 /* Write back the register source. */
2585 ctxt->src.val = ctxt->dst.val;
2586 write_register_operand(&ctxt->src);
2588 /* Write back the memory destination with implicit LOCK prefix. */
2589 ctxt->dst.val = ctxt->src.orig_val;
2590 ctxt->lock_prefix = 1;
2591 return X86EMUL_CONTINUE;
2594 static int em_imul(struct x86_emulate_ctxt *ctxt)
2596 emulate_2op_SrcV_nobyte(ctxt, "imul");
2597 return X86EMUL_CONTINUE;
2600 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2602 ctxt->dst.val = ctxt->src2.val;
2603 return em_imul(ctxt);
2606 static int em_cwd(struct x86_emulate_ctxt *ctxt)
2608 ctxt->dst.type = OP_REG;
2609 ctxt->dst.bytes = ctxt->src.bytes;
2610 ctxt->dst.addr.reg = &ctxt->regs[VCPU_REGS_RDX];
2611 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
2613 return X86EMUL_CONTINUE;
2616 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2620 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
2621 ctxt->regs[VCPU_REGS_RAX] = (u32)tsc;
2622 ctxt->regs[VCPU_REGS_RDX] = tsc >> 32;
2623 return X86EMUL_CONTINUE;
2626 static int em_mov(struct x86_emulate_ctxt *ctxt)
2628 ctxt->dst.val = ctxt->src.val;
2629 return X86EMUL_CONTINUE;
2632 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
2634 if (ctxt->modrm_reg > VCPU_SREG_GS)
2635 return emulate_ud(ctxt);
2637 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
2638 return X86EMUL_CONTINUE;
2641 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
2643 u16 sel = ctxt->src.val;
2645 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
2646 return emulate_ud(ctxt);
2648 if (ctxt->modrm_reg == VCPU_SREG_SS)
2649 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
2651 /* Disable writeback. */
2652 ctxt->dst.type = OP_NONE;
2653 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
2656 static int em_movdqu(struct x86_emulate_ctxt *ctxt)
2658 memcpy(&ctxt->dst.vec_val, &ctxt->src.vec_val, ctxt->op_bytes);
2659 return X86EMUL_CONTINUE;
2662 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
2667 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
2668 if (rc == X86EMUL_CONTINUE)
2669 ctxt->ops->invlpg(ctxt, linear);
2670 /* Disable writeback. */
2671 ctxt->dst.type = OP_NONE;
2672 return X86EMUL_CONTINUE;
2675 static int em_clts(struct x86_emulate_ctxt *ctxt)
2679 cr0 = ctxt->ops->get_cr(ctxt, 0);
2681 ctxt->ops->set_cr(ctxt, 0, cr0);
2682 return X86EMUL_CONTINUE;
2685 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
2689 if (ctxt->modrm_mod != 3 || ctxt->modrm_rm != 1)
2690 return X86EMUL_UNHANDLEABLE;
2692 rc = ctxt->ops->fix_hypercall(ctxt);
2693 if (rc != X86EMUL_CONTINUE)
2696 /* Let the processor re-execute the fixed hypercall */
2697 ctxt->_eip = ctxt->eip;
2698 /* Disable writeback. */
2699 ctxt->dst.type = OP_NONE;
2700 return X86EMUL_CONTINUE;
2703 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
2705 struct desc_ptr desc_ptr;
2708 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
2709 &desc_ptr.size, &desc_ptr.address,
2711 if (rc != X86EMUL_CONTINUE)
2713 ctxt->ops->set_gdt(ctxt, &desc_ptr);
2714 /* Disable writeback. */
2715 ctxt->dst.type = OP_NONE;
2716 return X86EMUL_CONTINUE;
2719 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
2723 rc = ctxt->ops->fix_hypercall(ctxt);
2725 /* Disable writeback. */
2726 ctxt->dst.type = OP_NONE;
2730 static int em_lidt(struct x86_emulate_ctxt *ctxt)
2732 struct desc_ptr desc_ptr;
2735 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
2736 &desc_ptr.size, &desc_ptr.address,
2738 if (rc != X86EMUL_CONTINUE)
2740 ctxt->ops->set_idt(ctxt, &desc_ptr);
2741 /* Disable writeback. */
2742 ctxt->dst.type = OP_NONE;
2743 return X86EMUL_CONTINUE;
2746 static int em_smsw(struct x86_emulate_ctxt *ctxt)
2748 ctxt->dst.bytes = 2;
2749 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
2750 return X86EMUL_CONTINUE;
2753 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
2755 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
2756 | (ctxt->src.val & 0x0f));
2757 ctxt->dst.type = OP_NONE;
2758 return X86EMUL_CONTINUE;
2761 static int em_loop(struct x86_emulate_ctxt *ctxt)
2763 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1);
2764 if ((address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) != 0) &&
2765 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
2766 jmp_rel(ctxt, ctxt->src.val);
2768 return X86EMUL_CONTINUE;
2771 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
2773 if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0)
2774 jmp_rel(ctxt, ctxt->src.val);
2776 return X86EMUL_CONTINUE;
2779 static int em_cli(struct x86_emulate_ctxt *ctxt)
2781 if (emulator_bad_iopl(ctxt))
2782 return emulate_gp(ctxt, 0);
2784 ctxt->eflags &= ~X86_EFLAGS_IF;
2785 return X86EMUL_CONTINUE;
2788 static int em_sti(struct x86_emulate_ctxt *ctxt)
2790 if (emulator_bad_iopl(ctxt))
2791 return emulate_gp(ctxt, 0);
2793 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
2794 ctxt->eflags |= X86_EFLAGS_IF;
2795 return X86EMUL_CONTINUE;
2798 static bool valid_cr(int nr)
2810 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
2812 if (!valid_cr(ctxt->modrm_reg))
2813 return emulate_ud(ctxt);
2815 return X86EMUL_CONTINUE;
2818 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
2820 u64 new_val = ctxt->src.val64;
2821 int cr = ctxt->modrm_reg;
2824 static u64 cr_reserved_bits[] = {
2825 0xffffffff00000000ULL,
2826 0, 0, 0, /* CR3 checked later */
2833 return emulate_ud(ctxt);
2835 if (new_val & cr_reserved_bits[cr])
2836 return emulate_gp(ctxt, 0);
2841 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
2842 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
2843 return emulate_gp(ctxt, 0);
2845 cr4 = ctxt->ops->get_cr(ctxt, 4);
2846 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2848 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
2849 !(cr4 & X86_CR4_PAE))
2850 return emulate_gp(ctxt, 0);
2857 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2858 if (efer & EFER_LMA)
2859 rsvd = CR3_L_MODE_RESERVED_BITS;
2860 else if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PAE)
2861 rsvd = CR3_PAE_RESERVED_BITS;
2862 else if (ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PG)
2863 rsvd = CR3_NONPAE_RESERVED_BITS;
2866 return emulate_gp(ctxt, 0);
2873 cr4 = ctxt->ops->get_cr(ctxt, 4);
2874 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2876 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
2877 return emulate_gp(ctxt, 0);
2883 return X86EMUL_CONTINUE;
2886 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
2890 ctxt->ops->get_dr(ctxt, 7, &dr7);
2892 /* Check if DR7.Global_Enable is set */
2893 return dr7 & (1 << 13);
2896 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
2898 int dr = ctxt->modrm_reg;
2902 return emulate_ud(ctxt);
2904 cr4 = ctxt->ops->get_cr(ctxt, 4);
2905 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
2906 return emulate_ud(ctxt);
2908 if (check_dr7_gd(ctxt))
2909 return emulate_db(ctxt);
2911 return X86EMUL_CONTINUE;
2914 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
2916 u64 new_val = ctxt->src.val64;
2917 int dr = ctxt->modrm_reg;
2919 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
2920 return emulate_gp(ctxt, 0);
2922 return check_dr_read(ctxt);
2925 static int check_svme(struct x86_emulate_ctxt *ctxt)
2929 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2931 if (!(efer & EFER_SVME))
2932 return emulate_ud(ctxt);
2934 return X86EMUL_CONTINUE;
2937 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
2939 u64 rax = ctxt->regs[VCPU_REGS_RAX];
2941 /* Valid physical address? */
2942 if (rax & 0xffff000000000000ULL)
2943 return emulate_gp(ctxt, 0);
2945 return check_svme(ctxt);
2948 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
2950 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
2952 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
2953 return emulate_ud(ctxt);
2955 return X86EMUL_CONTINUE;
2958 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
2960 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
2961 u64 rcx = ctxt->regs[VCPU_REGS_RCX];
2963 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
2965 return emulate_gp(ctxt, 0);
2967 return X86EMUL_CONTINUE;
2970 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
2972 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
2973 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
2974 return emulate_gp(ctxt, 0);
2976 return X86EMUL_CONTINUE;
2979 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
2981 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
2982 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
2983 return emulate_gp(ctxt, 0);
2985 return X86EMUL_CONTINUE;
2988 #define D(_y) { .flags = (_y) }
2989 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
2990 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
2991 .check_perm = (_p) }
2993 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
2994 #define G(_f, _g) { .flags = ((_f) | Group), .u.group = (_g) }
2995 #define GD(_f, _g) { .flags = ((_f) | GroupDual), .u.gdual = (_g) }
2996 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
2997 #define II(_f, _e, _i) \
2998 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
2999 #define IIP(_f, _e, _i, _p) \
3000 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
3001 .check_perm = (_p) }
3002 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3004 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3005 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3006 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3008 #define I6ALU(_f, _e) I2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3009 I2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3010 I2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3012 static struct opcode group7_rm1[] = {
3013 DI(SrcNone | ModRM | Priv, monitor),
3014 DI(SrcNone | ModRM | Priv, mwait),
3018 static struct opcode group7_rm3[] = {
3019 DIP(SrcNone | ModRM | Prot | Priv, vmrun, check_svme_pa),
3020 II(SrcNone | ModRM | Prot | VendorSpecific, em_vmmcall, vmmcall),
3021 DIP(SrcNone | ModRM | Prot | Priv, vmload, check_svme_pa),
3022 DIP(SrcNone | ModRM | Prot | Priv, vmsave, check_svme_pa),
3023 DIP(SrcNone | ModRM | Prot | Priv, stgi, check_svme),
3024 DIP(SrcNone | ModRM | Prot | Priv, clgi, check_svme),
3025 DIP(SrcNone | ModRM | Prot | Priv, skinit, check_svme),
3026 DIP(SrcNone | ModRM | Prot | Priv, invlpga, check_svme),
3029 static struct opcode group7_rm7[] = {
3031 DIP(SrcNone | ModRM, rdtscp, check_rdtsc),
3035 static struct opcode group1[] = {
3037 I(Lock | PageTable, em_or),
3040 I(Lock | PageTable, em_and),
3046 static struct opcode group1A[] = {
3047 D(DstMem | SrcNone | ModRM | Mov | Stack), N, N, N, N, N, N, N,
3050 static struct opcode group3[] = {
3051 I(DstMem | SrcImm | ModRM, em_test),
3052 I(DstMem | SrcImm | ModRM, em_test),
3053 I(DstMem | SrcNone | ModRM | Lock, em_not),
3054 I(DstMem | SrcNone | ModRM | Lock, em_neg),
3055 I(SrcMem | ModRM, em_mul_ex),
3056 I(SrcMem | ModRM, em_imul_ex),
3057 I(SrcMem | ModRM, em_div_ex),
3058 I(SrcMem | ModRM, em_idiv_ex),
3061 static struct opcode group4[] = {
3062 D(ByteOp | DstMem | SrcNone | ModRM | Lock), D(ByteOp | DstMem | SrcNone | ModRM | Lock),
3066 static struct opcode group5[] = {
3067 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
3068 D(SrcMem | ModRM | Stack),
3069 I(SrcMemFAddr | ModRM | ImplicitOps | Stack, em_call_far),
3070 D(SrcMem | ModRM | Stack), D(SrcMemFAddr | ModRM | ImplicitOps),
3071 D(SrcMem | ModRM | Stack), N,
3074 static struct opcode group6[] = {
3075 DI(ModRM | Prot, sldt),
3076 DI(ModRM | Prot, str),
3077 DI(ModRM | Prot | Priv, lldt),
3078 DI(ModRM | Prot | Priv, ltr),
3082 static struct group_dual group7 = { {
3083 DI(ModRM | Mov | DstMem | Priv, sgdt),
3084 DI(ModRM | Mov | DstMem | Priv, sidt),
3085 II(ModRM | SrcMem | Priv, em_lgdt, lgdt),
3086 II(ModRM | SrcMem | Priv, em_lidt, lidt),
3087 II(SrcNone | ModRM | DstMem | Mov, em_smsw, smsw), N,
3088 II(SrcMem16 | ModRM | Mov | Priv, em_lmsw, lmsw),
3089 II(SrcMem | ModRM | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3091 I(SrcNone | ModRM | Priv | VendorSpecific, em_vmcall),
3093 N, EXT(0, group7_rm3),
3094 II(SrcNone | ModRM | DstMem | Mov, em_smsw, smsw), N,
3095 II(SrcMem16 | ModRM | Mov | Priv, em_lmsw, lmsw), EXT(0, group7_rm7),
3098 static struct opcode group8[] = {
3100 D(DstMem | SrcImmByte | ModRM),
3101 D(DstMem | SrcImmByte | ModRM | Lock | PageTable),
3102 D(DstMem | SrcImmByte | ModRM | Lock),
3103 D(DstMem | SrcImmByte | ModRM | Lock | PageTable),
3106 static struct group_dual group9 = { {
3107 N, D(DstMem64 | ModRM | Lock | PageTable), N, N, N, N, N, N,
3109 N, N, N, N, N, N, N, N,
3112 static struct opcode group11[] = {
3113 I(DstMem | SrcImm | ModRM | Mov | PageTable, em_mov),
3117 static struct gprefix pfx_0f_6f_0f_7f = {
3118 N, N, N, I(Sse, em_movdqu),
3121 static struct opcode opcode_table[256] = {
3123 I6ALU(Lock, em_add),
3124 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
3125 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
3127 I6ALU(Lock | PageTable, em_or),
3128 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
3131 I6ALU(Lock, em_adc),
3132 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
3133 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
3135 I6ALU(Lock, em_sbb),
3136 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
3137 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
3139 I6ALU(Lock | PageTable, em_and), N, N,
3141 I6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3143 I6ALU(Lock, em_xor), N, N,
3145 I6ALU(0, em_cmp), N, N,
3149 X8(I(SrcReg | Stack, em_push)),
3151 X8(I(DstReg | Stack, em_pop)),
3153 I(ImplicitOps | Stack | No64, em_pusha),
3154 I(ImplicitOps | Stack | No64, em_popa),
3155 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3158 I(SrcImm | Mov | Stack, em_push),
3159 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3160 I(SrcImmByte | Mov | Stack, em_push),
3161 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3162 D2bvIP(DstDI | SrcDX | Mov | String, ins, check_perm_in), /* insb, insw/insd */
3163 D2bvIP(SrcSI | DstDX | String, outs, check_perm_out), /* outsb, outsw/outsd */
3167 G(ByteOp | DstMem | SrcImm | ModRM | Group, group1),
3168 G(DstMem | SrcImm | ModRM | Group, group1),
3169 G(ByteOp | DstMem | SrcImm | ModRM | No64 | Group, group1),
3170 G(DstMem | SrcImmByte | ModRM | Group, group1),
3171 I2bv(DstMem | SrcReg | ModRM, em_test),
3172 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
3174 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
3175 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3176 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
3177 D(ModRM | SrcMem | NoAccess | DstReg),
3178 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
3181 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3183 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3184 I(SrcImmFAddr | No64, em_call_far), N,
3185 II(ImplicitOps | Stack, em_pushf, pushf),
3186 II(ImplicitOps | Stack, em_popf, popf), N, N,
3188 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3189 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
3190 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3191 I2bv(SrcSI | DstDI | String, em_cmp),
3193 I2bv(DstAcc | SrcImm, em_test),
3194 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3195 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3196 I2bv(SrcAcc | DstDI | String, em_cmp),
3198 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3200 X8(I(DstReg | SrcImm | Mov, em_mov)),
3202 D2bv(DstMem | SrcImmByte | ModRM),
3203 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3204 I(ImplicitOps | Stack, em_ret),
3205 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
3206 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
3207 G(ByteOp, group11), G(0, group11),
3209 N, N, N, I(ImplicitOps | Stack, em_ret_far),
3210 D(ImplicitOps), DI(SrcImmByte, intn),
3211 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3213 D2bv(DstMem | SrcOne | ModRM), D2bv(DstMem | ModRM),
3216 N, N, N, N, N, N, N, N,
3218 X3(I(SrcImmByte, em_loop)),
3219 I(SrcImmByte, em_jcxz),
3220 D2bvIP(SrcImmUByte | DstAcc, in, check_perm_in),
3221 D2bvIP(SrcAcc | DstImmUByte, out, check_perm_out),
3223 D(SrcImm | Stack), D(SrcImm | ImplicitOps),
3224 I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
3225 D2bvIP(SrcDX | DstAcc, in, check_perm_in),
3226 D2bvIP(SrcAcc | DstDX, out, check_perm_out),
3228 N, DI(ImplicitOps, icebp), N, N,
3229 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
3230 G(ByteOp, group3), G(0, group3),
3232 D(ImplicitOps), D(ImplicitOps),
3233 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
3234 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3237 static struct opcode twobyte_table[256] = {
3239 G(0, group6), GD(0, &group7), N, N,
3240 N, I(ImplicitOps | VendorSpecific, em_syscall),
3241 II(ImplicitOps | Priv, em_clts, clts), N,
3242 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3243 N, D(ImplicitOps | ModRM), N, N,
3245 N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
3247 DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
3248 DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
3249 DIP(ModRM | SrcMem | Priv | Op3264, cr_write, check_cr_write),
3250 DIP(ModRM | SrcMem | Priv | Op3264, dr_write, check_dr_write),
3252 N, N, N, N, N, N, N, N,
3254 DI(ImplicitOps | Priv, wrmsr),
3255 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
3256 DI(ImplicitOps | Priv, rdmsr),
3257 DIP(ImplicitOps | Priv, rdpmc, check_rdpmc),
3258 I(ImplicitOps | VendorSpecific, em_sysenter),
3259 I(ImplicitOps | Priv | VendorSpecific, em_sysexit),
3261 N, N, N, N, N, N, N, N,
3263 X16(D(DstReg | SrcMem | ModRM | Mov)),
3265 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3270 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3275 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3279 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3281 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
3282 DI(ImplicitOps, cpuid), D(DstMem | SrcReg | ModRM | BitOp),
3283 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3284 D(DstMem | SrcReg | Src2CL | ModRM), N, N,
3286 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
3287 DI(ImplicitOps, rsm),
3288 D(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable),
3289 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3290 D(DstMem | SrcReg | Src2CL | ModRM),
3291 D(ModRM), I(DstReg | SrcMem | ModRM, em_imul),
3293 D2bv(DstMem | SrcReg | ModRM | Lock | PageTable),
3294 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
3295 D(DstMem | SrcReg | ModRM | BitOp | Lock),
3296 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
3297 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
3298 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3301 G(BitOp, group8), D(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable),
3302 D(DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
3303 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3305 D2bv(DstMem | SrcReg | ModRM | Lock),
3306 N, D(DstMem | SrcReg | ModRM | Mov),
3307 N, N, N, GD(0, &group9),
3308 N, N, N, N, N, N, N, N,
3310 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3312 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3314 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
3330 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
3334 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3340 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
3341 unsigned size, bool sign_extension)
3343 int rc = X86EMUL_CONTINUE;
3347 op->addr.mem.ea = ctxt->_eip;
3348 /* NB. Immediates are sign-extended as necessary. */
3349 switch (op->bytes) {
3351 op->val = insn_fetch(s8, ctxt);
3354 op->val = insn_fetch(s16, ctxt);
3357 op->val = insn_fetch(s32, ctxt);
3360 if (!sign_extension) {
3361 switch (op->bytes) {
3369 op->val &= 0xffffffff;
3377 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
3380 int rc = X86EMUL_CONTINUE;
3384 decode_register_operand(ctxt, op,
3386 ctxt->twobyte && (ctxt->b == 0xb6 || ctxt->b == 0xb7));
3389 rc = decode_imm(ctxt, op, 1, false);
3392 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3396 if ((ctxt->d & BitOp) && op == &ctxt->dst)
3397 fetch_bit_operand(ctxt);
3398 op->orig_val = op->val;
3401 ctxt->memop.bytes = 8;
3405 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3406 op->addr.reg = &ctxt->regs[VCPU_REGS_RAX];
3407 fetch_register_operand(op);
3408 op->orig_val = op->val;
3412 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3414 register_address(ctxt, ctxt->regs[VCPU_REGS_RDI]);
3415 op->addr.mem.seg = VCPU_SREG_ES;
3421 op->addr.reg = &ctxt->regs[VCPU_REGS_RDX];
3422 fetch_register_operand(op);
3426 op->val = ctxt->regs[VCPU_REGS_RCX] & 0xff;
3429 rc = decode_imm(ctxt, op, 1, true);
3436 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
3439 ctxt->memop.bytes = 2;
3442 ctxt->memop.bytes = 4;
3445 rc = decode_imm(ctxt, op, 2, false);
3448 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
3452 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3454 register_address(ctxt, ctxt->regs[VCPU_REGS_RSI]);
3455 op->addr.mem.seg = seg_override(ctxt);
3460 op->addr.mem.ea = ctxt->_eip;
3461 op->bytes = ctxt->op_bytes + 2;
3462 insn_fetch_arr(op->valptr, op->bytes, ctxt);
3465 ctxt->memop.bytes = ctxt->op_bytes + 2;
3468 op->val = VCPU_SREG_ES;
3471 op->val = VCPU_SREG_CS;
3474 op->val = VCPU_SREG_SS;
3477 op->val = VCPU_SREG_DS;
3480 op->val = VCPU_SREG_FS;
3483 op->val = VCPU_SREG_GS;
3486 /* Special instructions do their own operand decoding. */
3488 op->type = OP_NONE; /* Disable writeback. */
3496 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
3498 int rc = X86EMUL_CONTINUE;
3499 int mode = ctxt->mode;
3500 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
3501 bool op_prefix = false;
3502 struct opcode opcode;
3504 ctxt->memop.type = OP_NONE;
3505 ctxt->memopp = NULL;
3506 ctxt->_eip = ctxt->eip;
3507 ctxt->fetch.start = ctxt->_eip;
3508 ctxt->fetch.end = ctxt->fetch.start + insn_len;
3510 memcpy(ctxt->fetch.data, insn, insn_len);
3513 case X86EMUL_MODE_REAL:
3514 case X86EMUL_MODE_VM86:
3515 case X86EMUL_MODE_PROT16:
3516 def_op_bytes = def_ad_bytes = 2;
3518 case X86EMUL_MODE_PROT32:
3519 def_op_bytes = def_ad_bytes = 4;
3521 #ifdef CONFIG_X86_64
3522 case X86EMUL_MODE_PROT64:
3528 return EMULATION_FAILED;
3531 ctxt->op_bytes = def_op_bytes;
3532 ctxt->ad_bytes = def_ad_bytes;
3534 /* Legacy prefixes. */
3536 switch (ctxt->b = insn_fetch(u8, ctxt)) {
3537 case 0x66: /* operand-size override */
3539 /* switch between 2/4 bytes */
3540 ctxt->op_bytes = def_op_bytes ^ 6;
3542 case 0x67: /* address-size override */
3543 if (mode == X86EMUL_MODE_PROT64)
3544 /* switch between 4/8 bytes */
3545 ctxt->ad_bytes = def_ad_bytes ^ 12;
3547 /* switch between 2/4 bytes */
3548 ctxt->ad_bytes = def_ad_bytes ^ 6;
3550 case 0x26: /* ES override */
3551 case 0x2e: /* CS override */
3552 case 0x36: /* SS override */
3553 case 0x3e: /* DS override */
3554 set_seg_override(ctxt, (ctxt->b >> 3) & 3);
3556 case 0x64: /* FS override */
3557 case 0x65: /* GS override */
3558 set_seg_override(ctxt, ctxt->b & 7);
3560 case 0x40 ... 0x4f: /* REX */
3561 if (mode != X86EMUL_MODE_PROT64)
3563 ctxt->rex_prefix = ctxt->b;
3565 case 0xf0: /* LOCK */
3566 ctxt->lock_prefix = 1;
3568 case 0xf2: /* REPNE/REPNZ */
3569 case 0xf3: /* REP/REPE/REPZ */
3570 ctxt->rep_prefix = ctxt->b;
3576 /* Any legacy prefix after a REX prefix nullifies its effect. */
3578 ctxt->rex_prefix = 0;
3584 if (ctxt->rex_prefix & 8)
3585 ctxt->op_bytes = 8; /* REX.W */
3587 /* Opcode byte(s). */
3588 opcode = opcode_table[ctxt->b];
3589 /* Two-byte opcode? */
3590 if (ctxt->b == 0x0f) {
3592 ctxt->b = insn_fetch(u8, ctxt);
3593 opcode = twobyte_table[ctxt->b];
3595 ctxt->d = opcode.flags;
3597 while (ctxt->d & GroupMask) {
3598 switch (ctxt->d & GroupMask) {
3600 ctxt->modrm = insn_fetch(u8, ctxt);
3602 goffset = (ctxt->modrm >> 3) & 7;
3603 opcode = opcode.u.group[goffset];
3606 ctxt->modrm = insn_fetch(u8, ctxt);
3608 goffset = (ctxt->modrm >> 3) & 7;
3609 if ((ctxt->modrm >> 6) == 3)
3610 opcode = opcode.u.gdual->mod3[goffset];
3612 opcode = opcode.u.gdual->mod012[goffset];
3615 goffset = ctxt->modrm & 7;
3616 opcode = opcode.u.group[goffset];
3619 if (ctxt->rep_prefix && op_prefix)
3620 return EMULATION_FAILED;
3621 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
3622 switch (simd_prefix) {
3623 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
3624 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
3625 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
3626 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
3630 return EMULATION_FAILED;
3633 ctxt->d &= ~(u64)GroupMask;
3634 ctxt->d |= opcode.flags;
3637 ctxt->execute = opcode.u.execute;
3638 ctxt->check_perm = opcode.check_perm;
3639 ctxt->intercept = opcode.intercept;
3642 if (ctxt->d == 0 || (ctxt->d & Undefined))
3643 return EMULATION_FAILED;
3645 if (!(ctxt->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
3646 return EMULATION_FAILED;
3648 if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
3651 if (ctxt->d & Op3264) {
3652 if (mode == X86EMUL_MODE_PROT64)
3659 ctxt->op_bytes = 16;
3661 /* ModRM and SIB bytes. */
3662 if (ctxt->d & ModRM) {
3663 rc = decode_modrm(ctxt, &ctxt->memop);
3664 if (!ctxt->has_seg_override)
3665 set_seg_override(ctxt, ctxt->modrm_seg);
3666 } else if (ctxt->d & MemAbs)
3667 rc = decode_abs(ctxt, &ctxt->memop);
3668 if (rc != X86EMUL_CONTINUE)
3671 if (!ctxt->has_seg_override)
3672 set_seg_override(ctxt, VCPU_SREG_DS);
3674 ctxt->memop.addr.mem.seg = seg_override(ctxt);
3676 if (ctxt->memop.type == OP_MEM && ctxt->ad_bytes != 8)
3677 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
3680 * Decode and fetch the source operand: register, memory
3683 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
3684 if (rc != X86EMUL_CONTINUE)
3688 * Decode and fetch the second source operand: register, memory
3691 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
3692 if (rc != X86EMUL_CONTINUE)
3695 /* Decode and fetch the destination operand: register or memory. */
3696 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
3699 if (ctxt->memopp && ctxt->memopp->type == OP_MEM && ctxt->rip_relative)
3700 ctxt->memopp->addr.mem.ea += ctxt->_eip;
3702 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
3705 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
3707 return ctxt->d & PageTable;
3710 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
3712 /* The second termination condition only applies for REPE
3713 * and REPNE. Test if the repeat string operation prefix is
3714 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
3715 * corresponding termination condition according to:
3716 * - if REPE/REPZ and ZF = 0 then done
3717 * - if REPNE/REPNZ and ZF = 1 then done
3719 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
3720 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
3721 && (((ctxt->rep_prefix == REPE_PREFIX) &&
3722 ((ctxt->eflags & EFLG_ZF) == 0))
3723 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
3724 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
3730 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
3732 struct x86_emulate_ops *ops = ctxt->ops;
3734 int rc = X86EMUL_CONTINUE;
3735 int saved_dst_type = ctxt->dst.type;
3737 ctxt->mem_read.pos = 0;
3739 if (ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) {
3740 rc = emulate_ud(ctxt);
3744 /* LOCK prefix is allowed only with some instructions */
3745 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
3746 rc = emulate_ud(ctxt);
3750 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
3751 rc = emulate_ud(ctxt);
3756 && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)
3757 || !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
3758 rc = emulate_ud(ctxt);
3762 if ((ctxt->d & Sse) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
3763 rc = emulate_nm(ctxt);
3767 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
3768 rc = emulator_check_intercept(ctxt, ctxt->intercept,
3769 X86_ICPT_PRE_EXCEPT);
3770 if (rc != X86EMUL_CONTINUE)
3774 /* Privileged instruction can be executed only in CPL=0 */
3775 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
3776 rc = emulate_gp(ctxt, 0);
3780 /* Instruction can only be executed in protected mode */
3781 if ((ctxt->d & Prot) && !(ctxt->mode & X86EMUL_MODE_PROT)) {
3782 rc = emulate_ud(ctxt);
3786 /* Do instruction specific permission checks */
3787 if (ctxt->check_perm) {
3788 rc = ctxt->check_perm(ctxt);
3789 if (rc != X86EMUL_CONTINUE)
3793 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
3794 rc = emulator_check_intercept(ctxt, ctxt->intercept,
3795 X86_ICPT_POST_EXCEPT);
3796 if (rc != X86EMUL_CONTINUE)
3800 if (ctxt->rep_prefix && (ctxt->d & String)) {
3801 /* All REP prefixes have the same first termination condition */
3802 if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0) {
3803 ctxt->eip = ctxt->_eip;
3808 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
3809 rc = segmented_read(ctxt, ctxt->src.addr.mem,
3810 ctxt->src.valptr, ctxt->src.bytes);
3811 if (rc != X86EMUL_CONTINUE)
3813 ctxt->src.orig_val64 = ctxt->src.val64;
3816 if (ctxt->src2.type == OP_MEM) {
3817 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
3818 &ctxt->src2.val, ctxt->src2.bytes);
3819 if (rc != X86EMUL_CONTINUE)
3823 if ((ctxt->d & DstMask) == ImplicitOps)
3827 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
3828 /* optimisation - avoid slow emulated read if Mov */
3829 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
3830 &ctxt->dst.val, ctxt->dst.bytes);
3831 if (rc != X86EMUL_CONTINUE)
3834 ctxt->dst.orig_val = ctxt->dst.val;
3838 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
3839 rc = emulator_check_intercept(ctxt, ctxt->intercept,
3840 X86_ICPT_POST_MEMACCESS);
3841 if (rc != X86EMUL_CONTINUE)
3845 if (ctxt->execute) {
3846 rc = ctxt->execute(ctxt);
3847 if (rc != X86EMUL_CONTINUE)
3856 case 0x40 ... 0x47: /* inc r16/r32 */
3857 emulate_1op(ctxt, "inc");
3859 case 0x48 ... 0x4f: /* dec r16/r32 */
3860 emulate_1op(ctxt, "dec");
3862 case 0x63: /* movsxd */
3863 if (ctxt->mode != X86EMUL_MODE_PROT64)
3864 goto cannot_emulate;
3865 ctxt->dst.val = (s32) ctxt->src.val;
3867 case 0x6c: /* insb */
3868 case 0x6d: /* insw/insd */
3869 ctxt->src.val = ctxt->regs[VCPU_REGS_RDX];
3871 case 0x6e: /* outsb */
3872 case 0x6f: /* outsw/outsd */
3873 ctxt->dst.val = ctxt->regs[VCPU_REGS_RDX];
3876 case 0x70 ... 0x7f: /* jcc (short) */
3877 if (test_cc(ctxt->b, ctxt->eflags))
3878 jmp_rel(ctxt, ctxt->src.val);
3880 case 0x8d: /* lea r16/r32, m */
3881 ctxt->dst.val = ctxt->src.addr.mem.ea;
3883 case 0x8f: /* pop (sole member of Grp1a) */
3884 rc = em_grp1a(ctxt);
3886 case 0x90 ... 0x97: /* nop / xchg reg, rax */
3887 if (ctxt->dst.addr.reg == &ctxt->regs[VCPU_REGS_RAX])
3891 case 0x98: /* cbw/cwde/cdqe */
3892 switch (ctxt->op_bytes) {
3893 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
3894 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
3895 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
3901 case 0xcc: /* int3 */
3902 rc = emulate_int(ctxt, 3);
3904 case 0xcd: /* int n */
3905 rc = emulate_int(ctxt, ctxt->src.val);
3907 case 0xce: /* into */
3908 if (ctxt->eflags & EFLG_OF)
3909 rc = emulate_int(ctxt, 4);
3911 case 0xd0 ... 0xd1: /* Grp2 */
3914 case 0xd2 ... 0xd3: /* Grp2 */
3915 ctxt->src.val = ctxt->regs[VCPU_REGS_RCX];
3918 case 0xe4: /* inb */
3921 case 0xe6: /* outb */
3922 case 0xe7: /* out */
3924 case 0xe8: /* call (near) */ {
3925 long int rel = ctxt->src.val;
3926 ctxt->src.val = (unsigned long) ctxt->_eip;
3931 case 0xe9: /* jmp rel */
3932 case 0xeb: /* jmp rel short */
3933 jmp_rel(ctxt, ctxt->src.val);
3934 ctxt->dst.type = OP_NONE; /* Disable writeback. */
3936 case 0xec: /* in al,dx */
3937 case 0xed: /* in (e/r)ax,dx */
3939 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3941 goto done; /* IO is needed */
3943 case 0xee: /* out dx,al */
3944 case 0xef: /* out dx,(e/r)ax */
3946 ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3948 ctxt->dst.type = OP_NONE; /* Disable writeback. */
3950 case 0xf4: /* hlt */
3951 ctxt->ops->halt(ctxt);
3953 case 0xf5: /* cmc */
3954 /* complement carry flag from eflags reg */
3955 ctxt->eflags ^= EFLG_CF;
3957 case 0xf8: /* clc */
3958 ctxt->eflags &= ~EFLG_CF;
3960 case 0xf9: /* stc */
3961 ctxt->eflags |= EFLG_CF;
3963 case 0xfc: /* cld */
3964 ctxt->eflags &= ~EFLG_DF;
3966 case 0xfd: /* std */
3967 ctxt->eflags |= EFLG_DF;
3969 case 0xfe: /* Grp4 */
3970 rc = em_grp45(ctxt);
3972 case 0xff: /* Grp5 */
3973 rc = em_grp45(ctxt);
3976 goto cannot_emulate;
3979 if (rc != X86EMUL_CONTINUE)
3983 rc = writeback(ctxt);
3984 if (rc != X86EMUL_CONTINUE)
3988 * restore dst type in case the decoding will be reused
3989 * (happens for string instruction )
3991 ctxt->dst.type = saved_dst_type;
3993 if ((ctxt->d & SrcMask) == SrcSI)
3994 string_addr_inc(ctxt, seg_override(ctxt),
3995 VCPU_REGS_RSI, &ctxt->src);
3997 if ((ctxt->d & DstMask) == DstDI)
3998 string_addr_inc(ctxt, VCPU_SREG_ES, VCPU_REGS_RDI,
4001 if (ctxt->rep_prefix && (ctxt->d & String)) {
4002 struct read_cache *r = &ctxt->io_read;
4003 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1);
4005 if (!string_insn_completed(ctxt)) {
4007 * Re-enter guest when pio read ahead buffer is empty
4008 * or, if it is not used, after each 1024 iteration.
4010 if ((r->end != 0 || ctxt->regs[VCPU_REGS_RCX] & 0x3ff) &&
4011 (r->end == 0 || r->end != r->pos)) {
4013 * Reset read cache. Usually happens before
4014 * decode, but since instruction is restarted
4015 * we have to do it here.
4017 ctxt->mem_read.end = 0;
4018 return EMULATION_RESTART;
4020 goto done; /* skip rip writeback */
4024 ctxt->eip = ctxt->_eip;
4027 if (rc == X86EMUL_PROPAGATE_FAULT)
4028 ctxt->have_exception = true;
4029 if (rc == X86EMUL_INTERCEPTED)
4030 return EMULATION_INTERCEPTED;
4032 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4036 case 0x09: /* wbinvd */
4037 (ctxt->ops->wbinvd)(ctxt);
4039 case 0x08: /* invd */
4040 case 0x0d: /* GrpP (prefetch) */
4041 case 0x18: /* Grp16 (prefetch/nop) */
4043 case 0x20: /* mov cr, reg */
4044 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
4046 case 0x21: /* mov from dr to reg */
4047 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
4049 case 0x22: /* mov reg, cr */
4050 if (ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val)) {
4051 emulate_gp(ctxt, 0);
4052 rc = X86EMUL_PROPAGATE_FAULT;
4055 ctxt->dst.type = OP_NONE;
4057 case 0x23: /* mov from reg to dr */
4058 if (ops->set_dr(ctxt, ctxt->modrm_reg, ctxt->src.val &
4059 ((ctxt->mode == X86EMUL_MODE_PROT64) ?
4060 ~0ULL : ~0U)) < 0) {
4061 /* #UD condition is already handled by the code above */
4062 emulate_gp(ctxt, 0);
4063 rc = X86EMUL_PROPAGATE_FAULT;
4067 ctxt->dst.type = OP_NONE; /* no writeback */
4071 msr_data = (u32)ctxt->regs[VCPU_REGS_RAX]
4072 | ((u64)ctxt->regs[VCPU_REGS_RDX] << 32);
4073 if (ops->set_msr(ctxt, ctxt->regs[VCPU_REGS_RCX], msr_data)) {
4074 emulate_gp(ctxt, 0);
4075 rc = X86EMUL_PROPAGATE_FAULT;
4078 rc = X86EMUL_CONTINUE;
4082 if (ops->get_msr(ctxt, ctxt->regs[VCPU_REGS_RCX], &msr_data)) {
4083 emulate_gp(ctxt, 0);
4084 rc = X86EMUL_PROPAGATE_FAULT;
4087 ctxt->regs[VCPU_REGS_RAX] = (u32)msr_data;
4088 ctxt->regs[VCPU_REGS_RDX] = msr_data >> 32;
4090 rc = X86EMUL_CONTINUE;
4092 case 0x40 ... 0x4f: /* cmov */
4093 ctxt->dst.val = ctxt->dst.orig_val = ctxt->src.val;
4094 if (!test_cc(ctxt->b, ctxt->eflags))
4095 ctxt->dst.type = OP_NONE; /* no writeback */
4097 case 0x80 ... 0x8f: /* jnz rel, etc*/
4098 if (test_cc(ctxt->b, ctxt->eflags))
4099 jmp_rel(ctxt, ctxt->src.val);
4101 case 0x90 ... 0x9f: /* setcc r/m8 */
4102 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
4106 ctxt->dst.type = OP_NONE;
4107 /* only subword offset */
4108 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
4109 emulate_2op_SrcV_nobyte(ctxt, "bt");
4111 case 0xa4: /* shld imm8, r, r/m */
4112 case 0xa5: /* shld cl, r, r/m */
4113 emulate_2op_cl(ctxt, "shld");
4117 emulate_2op_SrcV_nobyte(ctxt, "bts");
4119 case 0xac: /* shrd imm8, r, r/m */
4120 case 0xad: /* shrd cl, r, r/m */
4121 emulate_2op_cl(ctxt, "shrd");
4123 case 0xae: /* clflush */
4125 case 0xb0 ... 0xb1: /* cmpxchg */
4127 * Save real source value, then compare EAX against
4130 ctxt->src.orig_val = ctxt->src.val;
4131 ctxt->src.val = ctxt->regs[VCPU_REGS_RAX];
4132 emulate_2op_SrcV(ctxt, "cmp");
4133 if (ctxt->eflags & EFLG_ZF) {
4134 /* Success: write back to memory. */
4135 ctxt->dst.val = ctxt->src.orig_val;
4137 /* Failure: write the value we saw to EAX. */
4138 ctxt->dst.type = OP_REG;
4139 ctxt->dst.addr.reg = (unsigned long *)&ctxt->regs[VCPU_REGS_RAX];
4144 emulate_2op_SrcV_nobyte(ctxt, "btr");
4146 case 0xb6 ... 0xb7: /* movzx */
4147 ctxt->dst.bytes = ctxt->op_bytes;
4148 ctxt->dst.val = (ctxt->d & ByteOp) ? (u8) ctxt->src.val
4149 : (u16) ctxt->src.val;
4151 case 0xba: /* Grp8 */
4152 switch (ctxt->modrm_reg & 3) {
4165 emulate_2op_SrcV_nobyte(ctxt, "btc");
4167 case 0xbc: { /* bsf */
4169 __asm__ ("bsf %2, %0; setz %1"
4170 : "=r"(ctxt->dst.val), "=q"(zf)
4171 : "r"(ctxt->src.val));
4172 ctxt->eflags &= ~X86_EFLAGS_ZF;
4174 ctxt->eflags |= X86_EFLAGS_ZF;
4175 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4179 case 0xbd: { /* bsr */
4181 __asm__ ("bsr %2, %0; setz %1"
4182 : "=r"(ctxt->dst.val), "=q"(zf)
4183 : "r"(ctxt->src.val));
4184 ctxt->eflags &= ~X86_EFLAGS_ZF;
4186 ctxt->eflags |= X86_EFLAGS_ZF;
4187 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4191 case 0xbe ... 0xbf: /* movsx */
4192 ctxt->dst.bytes = ctxt->op_bytes;
4193 ctxt->dst.val = (ctxt->d & ByteOp) ? (s8) ctxt->src.val :
4194 (s16) ctxt->src.val;
4196 case 0xc0 ... 0xc1: /* xadd */
4197 emulate_2op_SrcV(ctxt, "add");
4198 /* Write back the register source. */
4199 ctxt->src.val = ctxt->dst.orig_val;
4200 write_register_operand(&ctxt->src);
4202 case 0xc3: /* movnti */
4203 ctxt->dst.bytes = ctxt->op_bytes;
4204 ctxt->dst.val = (ctxt->op_bytes == 4) ? (u32) ctxt->src.val :
4205 (u64) ctxt->src.val;
4207 case 0xc7: /* Grp9 (cmpxchg8b) */
4211 goto cannot_emulate;
4214 if (rc != X86EMUL_CONTINUE)
4220 return EMULATION_FAILED;