1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
35 #define OpImplicit 1ull /* No generic decode */
36 #define OpReg 2ull /* Register */
37 #define OpMem 3ull /* Memory */
38 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
39 #define OpDI 5ull /* ES:DI/EDI/RDI */
40 #define OpMem64 6ull /* Memory, 64-bit */
41 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
42 #define OpDX 8ull /* DX register */
43 #define OpCL 9ull /* CL register (for shifts) */
44 #define OpImmByte 10ull /* 8-bit sign extended immediate */
45 #define OpOne 11ull /* Implied 1 */
46 #define OpImm 12ull /* Sign extended immediate */
47 #define OpMem16 13ull /* Memory operand (16-bit). */
48 #define OpMem32 14ull /* Memory operand (32-bit). */
49 #define OpImmU 15ull /* Immediate operand, zero extended */
50 #define OpSI 16ull /* SI/ESI/RSI */
51 #define OpImmFAddr 17ull /* Immediate far address */
52 #define OpMemFAddr 18ull /* Far address in memory */
53 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
54 #define OpES 20ull /* ES */
55 #define OpCS 21ull /* CS */
56 #define OpSS 22ull /* SS */
57 #define OpDS 23ull /* DS */
58 #define OpFS 24ull /* FS */
59 #define OpGS 25ull /* GS */
61 #define OpBits 5 /* Width of operand field */
62 #define OpMask ((1ull << OpBits) - 1)
65 * Opcode effective-address decode tables.
66 * Note that we only emulate instructions that have at least one memory
67 * operand (excluding implicit stack references). We assume that stack
68 * references and instruction fetches will never occur in special memory
69 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
73 /* Operand sizes: 8-bit operands or specified/overridden size. */
74 #define ByteOp (1<<0) /* 8-bit operands. */
75 /* Destination operand type. */
77 #define ImplicitOps (OpImplicit << DstShift)
78 #define DstReg (OpReg << DstShift)
79 #define DstMem (OpMem << DstShift)
80 #define DstAcc (OpAcc << DstShift)
81 #define DstDI (OpDI << DstShift)
82 #define DstMem64 (OpMem64 << DstShift)
83 #define DstImmUByte (OpImmUByte << DstShift)
84 #define DstDX (OpDX << DstShift)
85 #define DstMask (OpMask << DstShift)
86 /* Source operand type. */
88 #define SrcNone (OpNone << SrcShift)
89 #define SrcReg (OpReg << SrcShift)
90 #define SrcMem (OpMem << SrcShift)
91 #define SrcMem16 (OpMem16 << SrcShift)
92 #define SrcMem32 (OpMem32 << SrcShift)
93 #define SrcImm (OpImm << SrcShift)
94 #define SrcImmByte (OpImmByte << SrcShift)
95 #define SrcOne (OpOne << SrcShift)
96 #define SrcImmUByte (OpImmUByte << SrcShift)
97 #define SrcImmU (OpImmU << SrcShift)
98 #define SrcSI (OpSI << SrcShift)
99 #define SrcImmFAddr (OpImmFAddr << SrcShift)
100 #define SrcMemFAddr (OpMemFAddr << SrcShift)
101 #define SrcAcc (OpAcc << SrcShift)
102 #define SrcImmU16 (OpImmU16 << SrcShift)
103 #define SrcDX (OpDX << SrcShift)
104 #define SrcMask (OpMask << SrcShift)
105 #define BitOp (1<<11)
106 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
107 #define String (1<<13) /* String instruction (rep capable) */
108 #define Stack (1<<14) /* Stack instruction (push/pop) */
109 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
110 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
111 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
112 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
113 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
114 #define Sse (1<<18) /* SSE Vector instruction */
115 /* Generic ModRM decode. */
116 #define ModRM (1<<19)
117 /* Destination is only written; never read. */
120 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
121 #define VendorSpecific (1<<22) /* Vendor specific instruction */
122 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
123 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
124 #define Undefined (1<<25) /* No Such Instruction */
125 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
126 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
128 #define PageTable (1 << 29) /* instruction used to write page table */
129 /* Source 2 operand type */
130 #define Src2Shift (30)
131 #define Src2None (OpNone << Src2Shift)
132 #define Src2CL (OpCL << Src2Shift)
133 #define Src2ImmByte (OpImmByte << Src2Shift)
134 #define Src2One (OpOne << Src2Shift)
135 #define Src2Imm (OpImm << Src2Shift)
136 #define Src2ES (OpES << Src2Shift)
137 #define Src2CS (OpCS << Src2Shift)
138 #define Src2SS (OpSS << Src2Shift)
139 #define Src2DS (OpDS << Src2Shift)
140 #define Src2FS (OpFS << Src2Shift)
141 #define Src2GS (OpGS << Src2Shift)
142 #define Src2Mask (OpMask << Src2Shift)
144 #define X2(x...) x, x
145 #define X3(x...) X2(x), x
146 #define X4(x...) X2(x), X2(x)
147 #define X5(x...) X4(x), x
148 #define X6(x...) X4(x), X2(x)
149 #define X7(x...) X4(x), X3(x)
150 #define X8(x...) X4(x), X4(x)
151 #define X16(x...) X8(x), X8(x)
157 int (*execute)(struct x86_emulate_ctxt *ctxt);
158 struct opcode *group;
159 struct group_dual *gdual;
160 struct gprefix *gprefix;
162 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
166 struct opcode mod012[8];
167 struct opcode mod3[8];
171 struct opcode pfx_no;
172 struct opcode pfx_66;
173 struct opcode pfx_f2;
174 struct opcode pfx_f3;
177 /* EFLAGS bit definitions. */
178 #define EFLG_ID (1<<21)
179 #define EFLG_VIP (1<<20)
180 #define EFLG_VIF (1<<19)
181 #define EFLG_AC (1<<18)
182 #define EFLG_VM (1<<17)
183 #define EFLG_RF (1<<16)
184 #define EFLG_IOPL (3<<12)
185 #define EFLG_NT (1<<14)
186 #define EFLG_OF (1<<11)
187 #define EFLG_DF (1<<10)
188 #define EFLG_IF (1<<9)
189 #define EFLG_TF (1<<8)
190 #define EFLG_SF (1<<7)
191 #define EFLG_ZF (1<<6)
192 #define EFLG_AF (1<<4)
193 #define EFLG_PF (1<<2)
194 #define EFLG_CF (1<<0)
196 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
197 #define EFLG_RESERVED_ONE_MASK 2
200 * Instruction emulation:
201 * Most instructions are emulated directly via a fragment of inline assembly
202 * code. This allows us to save/restore EFLAGS and thus very easily pick up
203 * any modified flags.
206 #if defined(CONFIG_X86_64)
207 #define _LO32 "k" /* force 32-bit operand */
208 #define _STK "%%rsp" /* stack pointer */
209 #elif defined(__i386__)
210 #define _LO32 "" /* force 32-bit operand */
211 #define _STK "%%esp" /* stack pointer */
215 * These EFLAGS bits are restored from saved value during emulation, and
216 * any changes are written back to the saved value after emulation.
218 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
220 /* Before executing instruction: restore necessary bits in EFLAGS. */
221 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
222 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
223 "movl %"_sav",%"_LO32 _tmp"; " \
226 "movl %"_msk",%"_LO32 _tmp"; " \
227 "andl %"_LO32 _tmp",("_STK"); " \
229 "notl %"_LO32 _tmp"; " \
230 "andl %"_LO32 _tmp",("_STK"); " \
231 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
233 "orl %"_LO32 _tmp",("_STK"); " \
237 /* After executing instruction: write-back necessary bits in EFLAGS. */
238 #define _POST_EFLAGS(_sav, _msk, _tmp) \
239 /* _sav |= EFLAGS & _msk; */ \
242 "andl %"_msk",%"_LO32 _tmp"; " \
243 "orl %"_LO32 _tmp",%"_sav"; "
251 #define ____emulate_2op(ctxt, _op, _x, _y, _suffix, _dsttype) \
253 __asm__ __volatile__ ( \
254 _PRE_EFLAGS("0", "4", "2") \
255 _op _suffix " %"_x"3,%1; " \
256 _POST_EFLAGS("0", "4", "2") \
257 : "=m" ((ctxt)->eflags), \
258 "+q" (*(_dsttype*)&(ctxt)->dst.val), \
260 : _y ((ctxt)->src.val), "i" (EFLAGS_MASK)); \
264 /* Raw emulation: instruction has two explicit operands. */
265 #define __emulate_2op_nobyte(ctxt,_op,_wx,_wy,_lx,_ly,_qx,_qy) \
267 unsigned long _tmp; \
269 switch ((ctxt)->dst.bytes) { \
271 ____emulate_2op(ctxt,_op,_wx,_wy,"w",u16); \
274 ____emulate_2op(ctxt,_op,_lx,_ly,"l",u32); \
277 ON64(____emulate_2op(ctxt,_op,_qx,_qy,"q",u64)); \
282 #define __emulate_2op(ctxt,_op,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
284 unsigned long _tmp; \
285 switch ((ctxt)->dst.bytes) { \
287 ____emulate_2op(ctxt,_op,_bx,_by,"b",u8); \
290 __emulate_2op_nobyte(ctxt, _op, \
291 _wx, _wy, _lx, _ly, _qx, _qy); \
296 /* Source operand is byte-sized and may be restricted to just %cl. */
297 #define emulate_2op_SrcB(ctxt, _op) \
298 __emulate_2op(ctxt, _op, "b", "c", "b", "c", "b", "c", "b", "c")
300 /* Source operand is byte, word, long or quad sized. */
301 #define emulate_2op_SrcV(ctxt, _op) \
302 __emulate_2op(ctxt, _op, "b", "q", "w", "r", _LO32, "r", "", "r")
304 /* Source operand is word, long or quad sized. */
305 #define emulate_2op_SrcV_nobyte(ctxt, _op) \
306 __emulate_2op_nobyte(ctxt, _op, "w", "r", _LO32, "r", "", "r")
308 /* Instruction has three operands and one operand is stored in ECX register */
309 #define __emulate_2op_cl(ctxt, _op, _suffix, _type) \
311 unsigned long _tmp; \
312 _type _clv = (ctxt)->src2.val; \
313 _type _srcv = (ctxt)->src.val; \
314 _type _dstv = (ctxt)->dst.val; \
316 __asm__ __volatile__ ( \
317 _PRE_EFLAGS("0", "5", "2") \
318 _op _suffix " %4,%1 \n" \
319 _POST_EFLAGS("0", "5", "2") \
320 : "=m" ((ctxt)->eflags), "+r" (_dstv), "=&r" (_tmp) \
321 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
324 (ctxt)->src2.val = (unsigned long) _clv; \
325 (ctxt)->src2.val = (unsigned long) _srcv; \
326 (ctxt)->dst.val = (unsigned long) _dstv; \
329 #define emulate_2op_cl(ctxt, _op) \
331 switch ((ctxt)->dst.bytes) { \
333 __emulate_2op_cl(ctxt, _op, "w", u16); \
336 __emulate_2op_cl(ctxt, _op, "l", u32); \
339 ON64(__emulate_2op_cl(ctxt, _op, "q", ulong)); \
344 #define __emulate_1op(ctxt, _op, _suffix) \
346 unsigned long _tmp; \
348 __asm__ __volatile__ ( \
349 _PRE_EFLAGS("0", "3", "2") \
350 _op _suffix " %1; " \
351 _POST_EFLAGS("0", "3", "2") \
352 : "=m" ((ctxt)->eflags), "+m" ((ctxt)->dst.val), \
354 : "i" (EFLAGS_MASK)); \
357 /* Instruction has only one explicit operand (no source operand). */
358 #define emulate_1op(ctxt, _op) \
360 switch ((ctxt)->dst.bytes) { \
361 case 1: __emulate_1op(ctxt, _op, "b"); break; \
362 case 2: __emulate_1op(ctxt, _op, "w"); break; \
363 case 4: __emulate_1op(ctxt, _op, "l"); break; \
364 case 8: ON64(__emulate_1op(ctxt, _op, "q")); break; \
368 #define __emulate_1op_rax_rdx(ctxt, _op, _suffix, _ex) \
370 unsigned long _tmp; \
371 ulong *rax = &(ctxt)->regs[VCPU_REGS_RAX]; \
372 ulong *rdx = &(ctxt)->regs[VCPU_REGS_RDX]; \
374 __asm__ __volatile__ ( \
375 _PRE_EFLAGS("0", "5", "1") \
377 _op _suffix " %6; " \
379 _POST_EFLAGS("0", "5", "1") \
380 ".pushsection .fixup,\"ax\" \n\t" \
381 "3: movb $1, %4 \n\t" \
384 _ASM_EXTABLE(1b, 3b) \
385 : "=m" ((ctxt)->eflags), "=&r" (_tmp), \
386 "+a" (*rax), "+d" (*rdx), "+qm"(_ex) \
387 : "i" (EFLAGS_MASK), "m" ((ctxt)->src.val), \
388 "a" (*rax), "d" (*rdx)); \
391 /* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
392 #define emulate_1op_rax_rdx(ctxt, _op, _ex) \
394 switch((ctxt)->src.bytes) { \
396 __emulate_1op_rax_rdx(ctxt, _op, "b", _ex); \
399 __emulate_1op_rax_rdx(ctxt, _op, "w", _ex); \
402 __emulate_1op_rax_rdx(ctxt, _op, "l", _ex); \
405 __emulate_1op_rax_rdx(ctxt, _op, "q", _ex)); \
410 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
411 enum x86_intercept intercept,
412 enum x86_intercept_stage stage)
414 struct x86_instruction_info info = {
415 .intercept = intercept,
416 .rep_prefix = ctxt->rep_prefix,
417 .modrm_mod = ctxt->modrm_mod,
418 .modrm_reg = ctxt->modrm_reg,
419 .modrm_rm = ctxt->modrm_rm,
420 .src_val = ctxt->src.val64,
421 .src_bytes = ctxt->src.bytes,
422 .dst_bytes = ctxt->dst.bytes,
423 .ad_bytes = ctxt->ad_bytes,
424 .next_rip = ctxt->eip,
427 return ctxt->ops->intercept(ctxt, &info, stage);
430 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
432 return (1UL << (ctxt->ad_bytes << 3)) - 1;
435 /* Access/update address held in a register, based on addressing mode. */
436 static inline unsigned long
437 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
439 if (ctxt->ad_bytes == sizeof(unsigned long))
442 return reg & ad_mask(ctxt);
445 static inline unsigned long
446 register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
448 return address_mask(ctxt, reg);
452 register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
454 if (ctxt->ad_bytes == sizeof(unsigned long))
457 *reg = (*reg & ~ad_mask(ctxt)) | ((*reg + inc) & ad_mask(ctxt));
460 static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
462 register_address_increment(ctxt, &ctxt->_eip, rel);
465 static u32 desc_limit_scaled(struct desc_struct *desc)
467 u32 limit = get_desc_limit(desc);
469 return desc->g ? (limit << 12) | 0xfff : limit;
472 static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
474 ctxt->has_seg_override = true;
475 ctxt->seg_override = seg;
478 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
480 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
483 return ctxt->ops->get_cached_segment_base(ctxt, seg);
486 static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
488 if (!ctxt->has_seg_override)
491 return ctxt->seg_override;
494 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
495 u32 error, bool valid)
497 ctxt->exception.vector = vec;
498 ctxt->exception.error_code = error;
499 ctxt->exception.error_code_valid = valid;
500 return X86EMUL_PROPAGATE_FAULT;
503 static int emulate_db(struct x86_emulate_ctxt *ctxt)
505 return emulate_exception(ctxt, DB_VECTOR, 0, false);
508 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
510 return emulate_exception(ctxt, GP_VECTOR, err, true);
513 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
515 return emulate_exception(ctxt, SS_VECTOR, err, true);
518 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
520 return emulate_exception(ctxt, UD_VECTOR, 0, false);
523 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
525 return emulate_exception(ctxt, TS_VECTOR, err, true);
528 static int emulate_de(struct x86_emulate_ctxt *ctxt)
530 return emulate_exception(ctxt, DE_VECTOR, 0, false);
533 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
535 return emulate_exception(ctxt, NM_VECTOR, 0, false);
538 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
541 struct desc_struct desc;
543 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
547 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
552 struct desc_struct desc;
554 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
555 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
558 static int __linearize(struct x86_emulate_ctxt *ctxt,
559 struct segmented_address addr,
560 unsigned size, bool write, bool fetch,
563 struct desc_struct desc;
570 la = seg_base(ctxt, addr.seg) + addr.ea;
571 switch (ctxt->mode) {
572 case X86EMUL_MODE_REAL:
574 case X86EMUL_MODE_PROT64:
575 if (((signed long)la << 16) >> 16 != la)
576 return emulate_gp(ctxt, 0);
579 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
583 /* code segment or read-only data segment */
584 if (((desc.type & 8) || !(desc.type & 2)) && write)
586 /* unreadable code segment */
587 if (!fetch && (desc.type & 8) && !(desc.type & 2))
589 lim = desc_limit_scaled(&desc);
590 if ((desc.type & 8) || !(desc.type & 4)) {
591 /* expand-up segment */
592 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
595 /* exapand-down segment */
596 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
598 lim = desc.d ? 0xffffffff : 0xffff;
599 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
602 cpl = ctxt->ops->cpl(ctxt);
605 if (!(desc.type & 8)) {
609 } else if ((desc.type & 8) && !(desc.type & 4)) {
610 /* nonconforming code segment */
613 } else if ((desc.type & 8) && (desc.type & 4)) {
614 /* conforming code segment */
620 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
623 return X86EMUL_CONTINUE;
625 if (addr.seg == VCPU_SREG_SS)
626 return emulate_ss(ctxt, addr.seg);
628 return emulate_gp(ctxt, addr.seg);
631 static int linearize(struct x86_emulate_ctxt *ctxt,
632 struct segmented_address addr,
633 unsigned size, bool write,
636 return __linearize(ctxt, addr, size, write, false, linear);
640 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
641 struct segmented_address addr,
648 rc = linearize(ctxt, addr, size, false, &linear);
649 if (rc != X86EMUL_CONTINUE)
651 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
655 * Fetch the next byte of the instruction being emulated which is pointed to
656 * by ctxt->_eip, then increment ctxt->_eip.
658 * Also prefetch the remaining bytes of the instruction without crossing page
659 * boundary if they are not in fetch_cache yet.
661 static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
663 struct fetch_cache *fc = &ctxt->fetch;
667 if (ctxt->_eip == fc->end) {
668 unsigned long linear;
669 struct segmented_address addr = { .seg = VCPU_SREG_CS,
671 cur_size = fc->end - fc->start;
672 size = min(15UL - cur_size,
673 PAGE_SIZE - offset_in_page(ctxt->_eip));
674 rc = __linearize(ctxt, addr, size, false, true, &linear);
675 if (unlikely(rc != X86EMUL_CONTINUE))
677 rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
678 size, &ctxt->exception);
679 if (unlikely(rc != X86EMUL_CONTINUE))
683 *dest = fc->data[ctxt->_eip - fc->start];
685 return X86EMUL_CONTINUE;
688 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
689 void *dest, unsigned size)
693 /* x86 instructions are limited to 15 bytes. */
694 if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
695 return X86EMUL_UNHANDLEABLE;
697 rc = do_insn_fetch_byte(ctxt, dest++);
698 if (rc != X86EMUL_CONTINUE)
701 return X86EMUL_CONTINUE;
704 /* Fetch next part of the instruction being emulated. */
705 #define insn_fetch(_type, _ctxt) \
706 ({ unsigned long _x; \
707 rc = do_insn_fetch(_ctxt, &_x, sizeof(_type)); \
708 if (rc != X86EMUL_CONTINUE) \
713 #define insn_fetch_arr(_arr, _size, _ctxt) \
714 ({ rc = do_insn_fetch(_ctxt, _arr, (_size)); \
715 if (rc != X86EMUL_CONTINUE) \
720 * Given the 'reg' portion of a ModRM byte, and a register block, return a
721 * pointer into the block that addresses the relevant register.
722 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
724 static void *decode_register(u8 modrm_reg, unsigned long *regs,
729 p = ®s[modrm_reg];
730 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
731 p = (unsigned char *)®s[modrm_reg & 3] + 1;
735 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
736 struct segmented_address addr,
737 u16 *size, unsigned long *address, int op_bytes)
744 rc = segmented_read_std(ctxt, addr, size, 2);
745 if (rc != X86EMUL_CONTINUE)
748 rc = segmented_read_std(ctxt, addr, address, op_bytes);
752 static int test_cc(unsigned int condition, unsigned int flags)
756 switch ((condition & 15) >> 1) {
758 rc |= (flags & EFLG_OF);
760 case 1: /* b/c/nae */
761 rc |= (flags & EFLG_CF);
764 rc |= (flags & EFLG_ZF);
767 rc |= (flags & (EFLG_CF|EFLG_ZF));
770 rc |= (flags & EFLG_SF);
773 rc |= (flags & EFLG_PF);
776 rc |= (flags & EFLG_ZF);
779 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
783 /* Odd condition identifiers (lsb == 1) have inverted sense. */
784 return (!!rc ^ (condition & 1));
787 static void fetch_register_operand(struct operand *op)
791 op->val = *(u8 *)op->addr.reg;
794 op->val = *(u16 *)op->addr.reg;
797 op->val = *(u32 *)op->addr.reg;
800 op->val = *(u64 *)op->addr.reg;
805 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
807 ctxt->ops->get_fpu(ctxt);
809 case 0: asm("movdqu %%xmm0, %0" : "=m"(*data)); break;
810 case 1: asm("movdqu %%xmm1, %0" : "=m"(*data)); break;
811 case 2: asm("movdqu %%xmm2, %0" : "=m"(*data)); break;
812 case 3: asm("movdqu %%xmm3, %0" : "=m"(*data)); break;
813 case 4: asm("movdqu %%xmm4, %0" : "=m"(*data)); break;
814 case 5: asm("movdqu %%xmm5, %0" : "=m"(*data)); break;
815 case 6: asm("movdqu %%xmm6, %0" : "=m"(*data)); break;
816 case 7: asm("movdqu %%xmm7, %0" : "=m"(*data)); break;
818 case 8: asm("movdqu %%xmm8, %0" : "=m"(*data)); break;
819 case 9: asm("movdqu %%xmm9, %0" : "=m"(*data)); break;
820 case 10: asm("movdqu %%xmm10, %0" : "=m"(*data)); break;
821 case 11: asm("movdqu %%xmm11, %0" : "=m"(*data)); break;
822 case 12: asm("movdqu %%xmm12, %0" : "=m"(*data)); break;
823 case 13: asm("movdqu %%xmm13, %0" : "=m"(*data)); break;
824 case 14: asm("movdqu %%xmm14, %0" : "=m"(*data)); break;
825 case 15: asm("movdqu %%xmm15, %0" : "=m"(*data)); break;
829 ctxt->ops->put_fpu(ctxt);
832 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
835 ctxt->ops->get_fpu(ctxt);
837 case 0: asm("movdqu %0, %%xmm0" : : "m"(*data)); break;
838 case 1: asm("movdqu %0, %%xmm1" : : "m"(*data)); break;
839 case 2: asm("movdqu %0, %%xmm2" : : "m"(*data)); break;
840 case 3: asm("movdqu %0, %%xmm3" : : "m"(*data)); break;
841 case 4: asm("movdqu %0, %%xmm4" : : "m"(*data)); break;
842 case 5: asm("movdqu %0, %%xmm5" : : "m"(*data)); break;
843 case 6: asm("movdqu %0, %%xmm6" : : "m"(*data)); break;
844 case 7: asm("movdqu %0, %%xmm7" : : "m"(*data)); break;
846 case 8: asm("movdqu %0, %%xmm8" : : "m"(*data)); break;
847 case 9: asm("movdqu %0, %%xmm9" : : "m"(*data)); break;
848 case 10: asm("movdqu %0, %%xmm10" : : "m"(*data)); break;
849 case 11: asm("movdqu %0, %%xmm11" : : "m"(*data)); break;
850 case 12: asm("movdqu %0, %%xmm12" : : "m"(*data)); break;
851 case 13: asm("movdqu %0, %%xmm13" : : "m"(*data)); break;
852 case 14: asm("movdqu %0, %%xmm14" : : "m"(*data)); break;
853 case 15: asm("movdqu %0, %%xmm15" : : "m"(*data)); break;
857 ctxt->ops->put_fpu(ctxt);
860 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
864 unsigned reg = ctxt->modrm_reg;
865 int highbyte_regs = ctxt->rex_prefix == 0;
867 if (!(ctxt->d & ModRM))
868 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
874 read_sse_reg(ctxt, &op->vec_val, reg);
879 if ((ctxt->d & ByteOp) && !inhibit_bytereg) {
880 op->addr.reg = decode_register(reg, ctxt->regs, highbyte_regs);
883 op->addr.reg = decode_register(reg, ctxt->regs, 0);
884 op->bytes = ctxt->op_bytes;
886 fetch_register_operand(op);
887 op->orig_val = op->val;
890 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
894 int index_reg = 0, base_reg = 0, scale;
895 int rc = X86EMUL_CONTINUE;
898 if (ctxt->rex_prefix) {
899 ctxt->modrm_reg = (ctxt->rex_prefix & 4) << 1; /* REX.R */
900 index_reg = (ctxt->rex_prefix & 2) << 2; /* REX.X */
901 ctxt->modrm_rm = base_reg = (ctxt->rex_prefix & 1) << 3; /* REG.B */
904 ctxt->modrm = insn_fetch(u8, ctxt);
905 ctxt->modrm_mod |= (ctxt->modrm & 0xc0) >> 6;
906 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
907 ctxt->modrm_rm |= (ctxt->modrm & 0x07);
908 ctxt->modrm_seg = VCPU_SREG_DS;
910 if (ctxt->modrm_mod == 3) {
912 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
913 op->addr.reg = decode_register(ctxt->modrm_rm,
914 ctxt->regs, ctxt->d & ByteOp);
918 op->addr.xmm = ctxt->modrm_rm;
919 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
922 fetch_register_operand(op);
928 if (ctxt->ad_bytes == 2) {
929 unsigned bx = ctxt->regs[VCPU_REGS_RBX];
930 unsigned bp = ctxt->regs[VCPU_REGS_RBP];
931 unsigned si = ctxt->regs[VCPU_REGS_RSI];
932 unsigned di = ctxt->regs[VCPU_REGS_RDI];
934 /* 16-bit ModR/M decode. */
935 switch (ctxt->modrm_mod) {
937 if (ctxt->modrm_rm == 6)
938 modrm_ea += insn_fetch(u16, ctxt);
941 modrm_ea += insn_fetch(s8, ctxt);
944 modrm_ea += insn_fetch(u16, ctxt);
947 switch (ctxt->modrm_rm) {
967 if (ctxt->modrm_mod != 0)
974 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
975 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
976 ctxt->modrm_seg = VCPU_SREG_SS;
977 modrm_ea = (u16)modrm_ea;
979 /* 32/64-bit ModR/M decode. */
980 if ((ctxt->modrm_rm & 7) == 4) {
981 sib = insn_fetch(u8, ctxt);
982 index_reg |= (sib >> 3) & 7;
986 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
987 modrm_ea += insn_fetch(s32, ctxt);
989 modrm_ea += ctxt->regs[base_reg];
991 modrm_ea += ctxt->regs[index_reg] << scale;
992 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
993 if (ctxt->mode == X86EMUL_MODE_PROT64)
994 ctxt->rip_relative = 1;
996 modrm_ea += ctxt->regs[ctxt->modrm_rm];
997 switch (ctxt->modrm_mod) {
999 if (ctxt->modrm_rm == 5)
1000 modrm_ea += insn_fetch(s32, ctxt);
1003 modrm_ea += insn_fetch(s8, ctxt);
1006 modrm_ea += insn_fetch(s32, ctxt);
1010 op->addr.mem.ea = modrm_ea;
1015 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1018 int rc = X86EMUL_CONTINUE;
1021 switch (ctxt->ad_bytes) {
1023 op->addr.mem.ea = insn_fetch(u16, ctxt);
1026 op->addr.mem.ea = insn_fetch(u32, ctxt);
1029 op->addr.mem.ea = insn_fetch(u64, ctxt);
1036 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1040 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1041 mask = ~(ctxt->dst.bytes * 8 - 1);
1043 if (ctxt->src.bytes == 2)
1044 sv = (s16)ctxt->src.val & (s16)mask;
1045 else if (ctxt->src.bytes == 4)
1046 sv = (s32)ctxt->src.val & (s32)mask;
1048 ctxt->dst.addr.mem.ea += (sv >> 3);
1051 /* only subword offset */
1052 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1055 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1056 unsigned long addr, void *dest, unsigned size)
1059 struct read_cache *mc = &ctxt->mem_read;
1062 int n = min(size, 8u);
1064 if (mc->pos < mc->end)
1067 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, n,
1069 if (rc != X86EMUL_CONTINUE)
1074 memcpy(dest, mc->data + mc->pos, n);
1079 return X86EMUL_CONTINUE;
1082 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1083 struct segmented_address addr,
1090 rc = linearize(ctxt, addr, size, false, &linear);
1091 if (rc != X86EMUL_CONTINUE)
1093 return read_emulated(ctxt, linear, data, size);
1096 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1097 struct segmented_address addr,
1104 rc = linearize(ctxt, addr, size, true, &linear);
1105 if (rc != X86EMUL_CONTINUE)
1107 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1111 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1112 struct segmented_address addr,
1113 const void *orig_data, const void *data,
1119 rc = linearize(ctxt, addr, size, true, &linear);
1120 if (rc != X86EMUL_CONTINUE)
1122 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1123 size, &ctxt->exception);
1126 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1127 unsigned int size, unsigned short port,
1130 struct read_cache *rc = &ctxt->io_read;
1132 if (rc->pos == rc->end) { /* refill pio read ahead */
1133 unsigned int in_page, n;
1134 unsigned int count = ctxt->rep_prefix ?
1135 address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) : 1;
1136 in_page = (ctxt->eflags & EFLG_DF) ?
1137 offset_in_page(ctxt->regs[VCPU_REGS_RDI]) :
1138 PAGE_SIZE - offset_in_page(ctxt->regs[VCPU_REGS_RDI]);
1139 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1143 rc->pos = rc->end = 0;
1144 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1149 memcpy(dest, rc->data + rc->pos, size);
1154 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1155 u16 selector, struct desc_ptr *dt)
1157 struct x86_emulate_ops *ops = ctxt->ops;
1159 if (selector & 1 << 2) {
1160 struct desc_struct desc;
1163 memset (dt, 0, sizeof *dt);
1164 if (!ops->get_segment(ctxt, &sel, &desc, NULL, VCPU_SREG_LDTR))
1167 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1168 dt->address = get_desc_base(&desc);
1170 ops->get_gdt(ctxt, dt);
1173 /* allowed just for 8 bytes segments */
1174 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1175 u16 selector, struct desc_struct *desc)
1178 u16 index = selector >> 3;
1181 get_descriptor_table_ptr(ctxt, selector, &dt);
1183 if (dt.size < index * 8 + 7)
1184 return emulate_gp(ctxt, selector & 0xfffc);
1186 addr = dt.address + index * 8;
1187 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1191 /* allowed just for 8 bytes segments */
1192 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1193 u16 selector, struct desc_struct *desc)
1196 u16 index = selector >> 3;
1199 get_descriptor_table_ptr(ctxt, selector, &dt);
1201 if (dt.size < index * 8 + 7)
1202 return emulate_gp(ctxt, selector & 0xfffc);
1204 addr = dt.address + index * 8;
1205 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1209 /* Does not support long mode */
1210 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1211 u16 selector, int seg)
1213 struct desc_struct seg_desc;
1215 unsigned err_vec = GP_VECTOR;
1217 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1220 memset(&seg_desc, 0, sizeof seg_desc);
1222 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1223 || ctxt->mode == X86EMUL_MODE_REAL) {
1224 /* set real mode segment descriptor */
1225 set_desc_base(&seg_desc, selector << 4);
1226 set_desc_limit(&seg_desc, 0xffff);
1233 /* NULL selector is not valid for TR, CS and SS */
1234 if ((seg == VCPU_SREG_CS || seg == VCPU_SREG_SS || seg == VCPU_SREG_TR)
1238 /* TR should be in GDT only */
1239 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1242 if (null_selector) /* for NULL selector skip all following checks */
1245 ret = read_segment_descriptor(ctxt, selector, &seg_desc);
1246 if (ret != X86EMUL_CONTINUE)
1249 err_code = selector & 0xfffc;
1250 err_vec = GP_VECTOR;
1252 /* can't load system descriptor into segment selecor */
1253 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1257 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1263 cpl = ctxt->ops->cpl(ctxt);
1268 * segment is not a writable data segment or segment
1269 * selector's RPL != CPL or segment selector's RPL != CPL
1271 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1275 if (!(seg_desc.type & 8))
1278 if (seg_desc.type & 4) {
1284 if (rpl > cpl || dpl != cpl)
1287 /* CS(RPL) <- CPL */
1288 selector = (selector & 0xfffc) | cpl;
1291 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1294 case VCPU_SREG_LDTR:
1295 if (seg_desc.s || seg_desc.type != 2)
1298 default: /* DS, ES, FS, or GS */
1300 * segment is not a data or readable code segment or
1301 * ((segment is a data or nonconforming code segment)
1302 * and (both RPL and CPL > DPL))
1304 if ((seg_desc.type & 0xa) == 0x8 ||
1305 (((seg_desc.type & 0xc) != 0xc) &&
1306 (rpl > dpl && cpl > dpl)))
1312 /* mark segment as accessed */
1314 ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1315 if (ret != X86EMUL_CONTINUE)
1319 ctxt->ops->set_segment(ctxt, selector, &seg_desc, 0, seg);
1320 return X86EMUL_CONTINUE;
1322 emulate_exception(ctxt, err_vec, err_code, true);
1323 return X86EMUL_PROPAGATE_FAULT;
1326 static void write_register_operand(struct operand *op)
1328 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1329 switch (op->bytes) {
1331 *(u8 *)op->addr.reg = (u8)op->val;
1334 *(u16 *)op->addr.reg = (u16)op->val;
1337 *op->addr.reg = (u32)op->val;
1338 break; /* 64b: zero-extend */
1340 *op->addr.reg = op->val;
1345 static int writeback(struct x86_emulate_ctxt *ctxt)
1349 switch (ctxt->dst.type) {
1351 write_register_operand(&ctxt->dst);
1354 if (ctxt->lock_prefix)
1355 rc = segmented_cmpxchg(ctxt,
1357 &ctxt->dst.orig_val,
1361 rc = segmented_write(ctxt,
1365 if (rc != X86EMUL_CONTINUE)
1369 write_sse_reg(ctxt, &ctxt->dst.vec_val, ctxt->dst.addr.xmm);
1377 return X86EMUL_CONTINUE;
1380 static int em_push(struct x86_emulate_ctxt *ctxt)
1382 struct segmented_address addr;
1384 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], -ctxt->op_bytes);
1385 addr.ea = register_address(ctxt, ctxt->regs[VCPU_REGS_RSP]);
1386 addr.seg = VCPU_SREG_SS;
1388 /* Disable writeback. */
1389 ctxt->dst.type = OP_NONE;
1390 return segmented_write(ctxt, addr, &ctxt->src.val, ctxt->op_bytes);
1393 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1394 void *dest, int len)
1397 struct segmented_address addr;
1399 addr.ea = register_address(ctxt, ctxt->regs[VCPU_REGS_RSP]);
1400 addr.seg = VCPU_SREG_SS;
1401 rc = segmented_read(ctxt, addr, dest, len);
1402 if (rc != X86EMUL_CONTINUE)
1405 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], len);
1409 static int em_pop(struct x86_emulate_ctxt *ctxt)
1411 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1414 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1415 void *dest, int len)
1418 unsigned long val, change_mask;
1419 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1420 int cpl = ctxt->ops->cpl(ctxt);
1422 rc = emulate_pop(ctxt, &val, len);
1423 if (rc != X86EMUL_CONTINUE)
1426 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1427 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1429 switch(ctxt->mode) {
1430 case X86EMUL_MODE_PROT64:
1431 case X86EMUL_MODE_PROT32:
1432 case X86EMUL_MODE_PROT16:
1434 change_mask |= EFLG_IOPL;
1436 change_mask |= EFLG_IF;
1438 case X86EMUL_MODE_VM86:
1440 return emulate_gp(ctxt, 0);
1441 change_mask |= EFLG_IF;
1443 default: /* real mode */
1444 change_mask |= (EFLG_IOPL | EFLG_IF);
1448 *(unsigned long *)dest =
1449 (ctxt->eflags & ~change_mask) | (val & change_mask);
1454 static int em_popf(struct x86_emulate_ctxt *ctxt)
1456 ctxt->dst.type = OP_REG;
1457 ctxt->dst.addr.reg = &ctxt->eflags;
1458 ctxt->dst.bytes = ctxt->op_bytes;
1459 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1462 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1464 int seg = ctxt->src2.val;
1466 ctxt->src.val = get_segment_selector(ctxt, seg);
1468 return em_push(ctxt);
1471 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1473 int seg = ctxt->src2.val;
1474 unsigned long selector;
1477 rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1478 if (rc != X86EMUL_CONTINUE)
1481 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1485 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1487 unsigned long old_esp = ctxt->regs[VCPU_REGS_RSP];
1488 int rc = X86EMUL_CONTINUE;
1489 int reg = VCPU_REGS_RAX;
1491 while (reg <= VCPU_REGS_RDI) {
1492 (reg == VCPU_REGS_RSP) ?
1493 (ctxt->src.val = old_esp) : (ctxt->src.val = ctxt->regs[reg]);
1496 if (rc != X86EMUL_CONTINUE)
1505 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1507 ctxt->src.val = (unsigned long)ctxt->eflags;
1508 return em_push(ctxt);
1511 static int em_popa(struct x86_emulate_ctxt *ctxt)
1513 int rc = X86EMUL_CONTINUE;
1514 int reg = VCPU_REGS_RDI;
1516 while (reg >= VCPU_REGS_RAX) {
1517 if (reg == VCPU_REGS_RSP) {
1518 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP],
1523 rc = emulate_pop(ctxt, &ctxt->regs[reg], ctxt->op_bytes);
1524 if (rc != X86EMUL_CONTINUE)
1531 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1533 struct x86_emulate_ops *ops = ctxt->ops;
1540 /* TODO: Add limit checks */
1541 ctxt->src.val = ctxt->eflags;
1543 if (rc != X86EMUL_CONTINUE)
1546 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1548 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1550 if (rc != X86EMUL_CONTINUE)
1553 ctxt->src.val = ctxt->_eip;
1555 if (rc != X86EMUL_CONTINUE)
1558 ops->get_idt(ctxt, &dt);
1560 eip_addr = dt.address + (irq << 2);
1561 cs_addr = dt.address + (irq << 2) + 2;
1563 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1564 if (rc != X86EMUL_CONTINUE)
1567 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1568 if (rc != X86EMUL_CONTINUE)
1571 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1572 if (rc != X86EMUL_CONTINUE)
1580 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1582 switch(ctxt->mode) {
1583 case X86EMUL_MODE_REAL:
1584 return emulate_int_real(ctxt, irq);
1585 case X86EMUL_MODE_VM86:
1586 case X86EMUL_MODE_PROT16:
1587 case X86EMUL_MODE_PROT32:
1588 case X86EMUL_MODE_PROT64:
1590 /* Protected mode interrupts unimplemented yet */
1591 return X86EMUL_UNHANDLEABLE;
1595 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
1597 int rc = X86EMUL_CONTINUE;
1598 unsigned long temp_eip = 0;
1599 unsigned long temp_eflags = 0;
1600 unsigned long cs = 0;
1601 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1602 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1603 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1604 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1606 /* TODO: Add stack limit check */
1608 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
1610 if (rc != X86EMUL_CONTINUE)
1613 if (temp_eip & ~0xffff)
1614 return emulate_gp(ctxt, 0);
1616 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1618 if (rc != X86EMUL_CONTINUE)
1621 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
1623 if (rc != X86EMUL_CONTINUE)
1626 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1628 if (rc != X86EMUL_CONTINUE)
1631 ctxt->_eip = temp_eip;
1634 if (ctxt->op_bytes == 4)
1635 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1636 else if (ctxt->op_bytes == 2) {
1637 ctxt->eflags &= ~0xffff;
1638 ctxt->eflags |= temp_eflags;
1641 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1642 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1647 static int em_iret(struct x86_emulate_ctxt *ctxt)
1649 switch(ctxt->mode) {
1650 case X86EMUL_MODE_REAL:
1651 return emulate_iret_real(ctxt);
1652 case X86EMUL_MODE_VM86:
1653 case X86EMUL_MODE_PROT16:
1654 case X86EMUL_MODE_PROT32:
1655 case X86EMUL_MODE_PROT64:
1657 /* iret from protected mode unimplemented yet */
1658 return X86EMUL_UNHANDLEABLE;
1662 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
1667 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1669 rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
1670 if (rc != X86EMUL_CONTINUE)
1674 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
1675 return X86EMUL_CONTINUE;
1678 static int em_grp1a(struct x86_emulate_ctxt *ctxt)
1680 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->dst.bytes);
1683 static int em_grp2(struct x86_emulate_ctxt *ctxt)
1685 switch (ctxt->modrm_reg) {
1687 emulate_2op_SrcB(ctxt, "rol");
1690 emulate_2op_SrcB(ctxt, "ror");
1693 emulate_2op_SrcB(ctxt, "rcl");
1696 emulate_2op_SrcB(ctxt, "rcr");
1698 case 4: /* sal/shl */
1699 case 6: /* sal/shl */
1700 emulate_2op_SrcB(ctxt, "sal");
1703 emulate_2op_SrcB(ctxt, "shr");
1706 emulate_2op_SrcB(ctxt, "sar");
1709 return X86EMUL_CONTINUE;
1712 static int em_not(struct x86_emulate_ctxt *ctxt)
1714 ctxt->dst.val = ~ctxt->dst.val;
1715 return X86EMUL_CONTINUE;
1718 static int em_neg(struct x86_emulate_ctxt *ctxt)
1720 emulate_1op(ctxt, "neg");
1721 return X86EMUL_CONTINUE;
1724 static int em_mul_ex(struct x86_emulate_ctxt *ctxt)
1728 emulate_1op_rax_rdx(ctxt, "mul", ex);
1729 return X86EMUL_CONTINUE;
1732 static int em_imul_ex(struct x86_emulate_ctxt *ctxt)
1736 emulate_1op_rax_rdx(ctxt, "imul", ex);
1737 return X86EMUL_CONTINUE;
1740 static int em_div_ex(struct x86_emulate_ctxt *ctxt)
1744 emulate_1op_rax_rdx(ctxt, "div", de);
1746 return emulate_de(ctxt);
1747 return X86EMUL_CONTINUE;
1750 static int em_idiv_ex(struct x86_emulate_ctxt *ctxt)
1754 emulate_1op_rax_rdx(ctxt, "idiv", de);
1756 return emulate_de(ctxt);
1757 return X86EMUL_CONTINUE;
1760 static int em_grp45(struct x86_emulate_ctxt *ctxt)
1762 int rc = X86EMUL_CONTINUE;
1764 switch (ctxt->modrm_reg) {
1766 emulate_1op(ctxt, "inc");
1769 emulate_1op(ctxt, "dec");
1771 case 2: /* call near abs */ {
1773 old_eip = ctxt->_eip;
1774 ctxt->_eip = ctxt->src.val;
1775 ctxt->src.val = old_eip;
1779 case 4: /* jmp abs */
1780 ctxt->_eip = ctxt->src.val;
1782 case 5: /* jmp far */
1783 rc = em_jmp_far(ctxt);
1792 static int em_grp9(struct x86_emulate_ctxt *ctxt)
1794 u64 old = ctxt->dst.orig_val64;
1796 if (((u32) (old >> 0) != (u32) ctxt->regs[VCPU_REGS_RAX]) ||
1797 ((u32) (old >> 32) != (u32) ctxt->regs[VCPU_REGS_RDX])) {
1798 ctxt->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1799 ctxt->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1800 ctxt->eflags &= ~EFLG_ZF;
1802 ctxt->dst.val64 = ((u64)ctxt->regs[VCPU_REGS_RCX] << 32) |
1803 (u32) ctxt->regs[VCPU_REGS_RBX];
1805 ctxt->eflags |= EFLG_ZF;
1807 return X86EMUL_CONTINUE;
1810 static int em_ret(struct x86_emulate_ctxt *ctxt)
1812 ctxt->dst.type = OP_REG;
1813 ctxt->dst.addr.reg = &ctxt->_eip;
1814 ctxt->dst.bytes = ctxt->op_bytes;
1815 return em_pop(ctxt);
1818 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
1823 rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
1824 if (rc != X86EMUL_CONTINUE)
1826 if (ctxt->op_bytes == 4)
1827 ctxt->_eip = (u32)ctxt->_eip;
1828 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1829 if (rc != X86EMUL_CONTINUE)
1831 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1835 static int em_lseg(struct x86_emulate_ctxt *ctxt)
1837 int seg = ctxt->src2.val;
1841 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1843 rc = load_segment_descriptor(ctxt, sel, seg);
1844 if (rc != X86EMUL_CONTINUE)
1847 ctxt->dst.val = ctxt->src.val;
1852 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1853 struct desc_struct *cs, struct desc_struct *ss)
1857 memset(cs, 0, sizeof(struct desc_struct));
1858 ctxt->ops->get_segment(ctxt, &selector, cs, NULL, VCPU_SREG_CS);
1859 memset(ss, 0, sizeof(struct desc_struct));
1861 cs->l = 0; /* will be adjusted later */
1862 set_desc_base(cs, 0); /* flat segment */
1863 cs->g = 1; /* 4kb granularity */
1864 set_desc_limit(cs, 0xfffff); /* 4GB limit */
1865 cs->type = 0x0b; /* Read, Execute, Accessed */
1867 cs->dpl = 0; /* will be adjusted later */
1871 set_desc_base(ss, 0); /* flat segment */
1872 set_desc_limit(ss, 0xfffff); /* 4GB limit */
1873 ss->g = 1; /* 4kb granularity */
1875 ss->type = 0x03; /* Read/Write, Accessed */
1876 ss->d = 1; /* 32bit stack segment */
1881 static int em_syscall(struct x86_emulate_ctxt *ctxt)
1883 struct x86_emulate_ops *ops = ctxt->ops;
1884 struct desc_struct cs, ss;
1889 /* syscall is not available in real mode */
1890 if (ctxt->mode == X86EMUL_MODE_REAL ||
1891 ctxt->mode == X86EMUL_MODE_VM86)
1892 return emulate_ud(ctxt);
1894 ops->get_msr(ctxt, MSR_EFER, &efer);
1895 setup_syscalls_segments(ctxt, &cs, &ss);
1897 ops->get_msr(ctxt, MSR_STAR, &msr_data);
1899 cs_sel = (u16)(msr_data & 0xfffc);
1900 ss_sel = (u16)(msr_data + 8);
1902 if (efer & EFER_LMA) {
1906 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
1907 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
1909 ctxt->regs[VCPU_REGS_RCX] = ctxt->_eip;
1910 if (efer & EFER_LMA) {
1911 #ifdef CONFIG_X86_64
1912 ctxt->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
1915 ctxt->mode == X86EMUL_MODE_PROT64 ?
1916 MSR_LSTAR : MSR_CSTAR, &msr_data);
1917 ctxt->_eip = msr_data;
1919 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
1920 ctxt->eflags &= ~(msr_data | EFLG_RF);
1924 ops->get_msr(ctxt, MSR_STAR, &msr_data);
1925 ctxt->_eip = (u32)msr_data;
1927 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1930 return X86EMUL_CONTINUE;
1933 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
1935 struct x86_emulate_ops *ops = ctxt->ops;
1936 struct desc_struct cs, ss;
1941 ops->get_msr(ctxt, MSR_EFER, &efer);
1942 /* inject #GP if in real mode */
1943 if (ctxt->mode == X86EMUL_MODE_REAL)
1944 return emulate_gp(ctxt, 0);
1946 /* XXX sysenter/sysexit have not been tested in 64bit mode.
1947 * Therefore, we inject an #UD.
1949 if (ctxt->mode == X86EMUL_MODE_PROT64)
1950 return emulate_ud(ctxt);
1952 setup_syscalls_segments(ctxt, &cs, &ss);
1954 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
1955 switch (ctxt->mode) {
1956 case X86EMUL_MODE_PROT32:
1957 if ((msr_data & 0xfffc) == 0x0)
1958 return emulate_gp(ctxt, 0);
1960 case X86EMUL_MODE_PROT64:
1961 if (msr_data == 0x0)
1962 return emulate_gp(ctxt, 0);
1966 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1967 cs_sel = (u16)msr_data;
1968 cs_sel &= ~SELECTOR_RPL_MASK;
1969 ss_sel = cs_sel + 8;
1970 ss_sel &= ~SELECTOR_RPL_MASK;
1971 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
1976 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
1977 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
1979 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
1980 ctxt->_eip = msr_data;
1982 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
1983 ctxt->regs[VCPU_REGS_RSP] = msr_data;
1985 return X86EMUL_CONTINUE;
1988 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
1990 struct x86_emulate_ops *ops = ctxt->ops;
1991 struct desc_struct cs, ss;
1994 u16 cs_sel = 0, ss_sel = 0;
1996 /* inject #GP if in real mode or Virtual 8086 mode */
1997 if (ctxt->mode == X86EMUL_MODE_REAL ||
1998 ctxt->mode == X86EMUL_MODE_VM86)
1999 return emulate_gp(ctxt, 0);
2001 setup_syscalls_segments(ctxt, &cs, &ss);
2003 if ((ctxt->rex_prefix & 0x8) != 0x0)
2004 usermode = X86EMUL_MODE_PROT64;
2006 usermode = X86EMUL_MODE_PROT32;
2010 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2012 case X86EMUL_MODE_PROT32:
2013 cs_sel = (u16)(msr_data + 16);
2014 if ((msr_data & 0xfffc) == 0x0)
2015 return emulate_gp(ctxt, 0);
2016 ss_sel = (u16)(msr_data + 24);
2018 case X86EMUL_MODE_PROT64:
2019 cs_sel = (u16)(msr_data + 32);
2020 if (msr_data == 0x0)
2021 return emulate_gp(ctxt, 0);
2022 ss_sel = cs_sel + 8;
2027 cs_sel |= SELECTOR_RPL_MASK;
2028 ss_sel |= SELECTOR_RPL_MASK;
2030 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2031 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2033 ctxt->_eip = ctxt->regs[VCPU_REGS_RDX];
2034 ctxt->regs[VCPU_REGS_RSP] = ctxt->regs[VCPU_REGS_RCX];
2036 return X86EMUL_CONTINUE;
2039 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2042 if (ctxt->mode == X86EMUL_MODE_REAL)
2044 if (ctxt->mode == X86EMUL_MODE_VM86)
2046 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2047 return ctxt->ops->cpl(ctxt) > iopl;
2050 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2053 struct x86_emulate_ops *ops = ctxt->ops;
2054 struct desc_struct tr_seg;
2057 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2058 unsigned mask = (1 << len) - 1;
2061 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2064 if (desc_limit_scaled(&tr_seg) < 103)
2066 base = get_desc_base(&tr_seg);
2067 #ifdef CONFIG_X86_64
2068 base |= ((u64)base3) << 32;
2070 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2071 if (r != X86EMUL_CONTINUE)
2073 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2075 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2076 if (r != X86EMUL_CONTINUE)
2078 if ((perm >> bit_idx) & mask)
2083 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2089 if (emulator_bad_iopl(ctxt))
2090 if (!emulator_io_port_access_allowed(ctxt, port, len))
2093 ctxt->perm_ok = true;
2098 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2099 struct tss_segment_16 *tss)
2101 tss->ip = ctxt->_eip;
2102 tss->flag = ctxt->eflags;
2103 tss->ax = ctxt->regs[VCPU_REGS_RAX];
2104 tss->cx = ctxt->regs[VCPU_REGS_RCX];
2105 tss->dx = ctxt->regs[VCPU_REGS_RDX];
2106 tss->bx = ctxt->regs[VCPU_REGS_RBX];
2107 tss->sp = ctxt->regs[VCPU_REGS_RSP];
2108 tss->bp = ctxt->regs[VCPU_REGS_RBP];
2109 tss->si = ctxt->regs[VCPU_REGS_RSI];
2110 tss->di = ctxt->regs[VCPU_REGS_RDI];
2112 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2113 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2114 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2115 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2116 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2119 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2120 struct tss_segment_16 *tss)
2124 ctxt->_eip = tss->ip;
2125 ctxt->eflags = tss->flag | 2;
2126 ctxt->regs[VCPU_REGS_RAX] = tss->ax;
2127 ctxt->regs[VCPU_REGS_RCX] = tss->cx;
2128 ctxt->regs[VCPU_REGS_RDX] = tss->dx;
2129 ctxt->regs[VCPU_REGS_RBX] = tss->bx;
2130 ctxt->regs[VCPU_REGS_RSP] = tss->sp;
2131 ctxt->regs[VCPU_REGS_RBP] = tss->bp;
2132 ctxt->regs[VCPU_REGS_RSI] = tss->si;
2133 ctxt->regs[VCPU_REGS_RDI] = tss->di;
2136 * SDM says that segment selectors are loaded before segment
2139 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2140 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2141 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2142 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2143 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2146 * Now load segment descriptors. If fault happenes at this stage
2147 * it is handled in a context of new task
2149 ret = load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR);
2150 if (ret != X86EMUL_CONTINUE)
2152 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2153 if (ret != X86EMUL_CONTINUE)
2155 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2156 if (ret != X86EMUL_CONTINUE)
2158 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2159 if (ret != X86EMUL_CONTINUE)
2161 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2162 if (ret != X86EMUL_CONTINUE)
2165 return X86EMUL_CONTINUE;
2168 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2169 u16 tss_selector, u16 old_tss_sel,
2170 ulong old_tss_base, struct desc_struct *new_desc)
2172 struct x86_emulate_ops *ops = ctxt->ops;
2173 struct tss_segment_16 tss_seg;
2175 u32 new_tss_base = get_desc_base(new_desc);
2177 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2179 if (ret != X86EMUL_CONTINUE)
2180 /* FIXME: need to provide precise fault address */
2183 save_state_to_tss16(ctxt, &tss_seg);
2185 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2187 if (ret != X86EMUL_CONTINUE)
2188 /* FIXME: need to provide precise fault address */
2191 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2193 if (ret != X86EMUL_CONTINUE)
2194 /* FIXME: need to provide precise fault address */
2197 if (old_tss_sel != 0xffff) {
2198 tss_seg.prev_task_link = old_tss_sel;
2200 ret = ops->write_std(ctxt, new_tss_base,
2201 &tss_seg.prev_task_link,
2202 sizeof tss_seg.prev_task_link,
2204 if (ret != X86EMUL_CONTINUE)
2205 /* FIXME: need to provide precise fault address */
2209 return load_state_from_tss16(ctxt, &tss_seg);
2212 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2213 struct tss_segment_32 *tss)
2215 tss->cr3 = ctxt->ops->get_cr(ctxt, 3);
2216 tss->eip = ctxt->_eip;
2217 tss->eflags = ctxt->eflags;
2218 tss->eax = ctxt->regs[VCPU_REGS_RAX];
2219 tss->ecx = ctxt->regs[VCPU_REGS_RCX];
2220 tss->edx = ctxt->regs[VCPU_REGS_RDX];
2221 tss->ebx = ctxt->regs[VCPU_REGS_RBX];
2222 tss->esp = ctxt->regs[VCPU_REGS_RSP];
2223 tss->ebp = ctxt->regs[VCPU_REGS_RBP];
2224 tss->esi = ctxt->regs[VCPU_REGS_RSI];
2225 tss->edi = ctxt->regs[VCPU_REGS_RDI];
2227 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2228 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2229 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2230 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2231 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2232 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2233 tss->ldt_selector = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2236 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2237 struct tss_segment_32 *tss)
2241 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2242 return emulate_gp(ctxt, 0);
2243 ctxt->_eip = tss->eip;
2244 ctxt->eflags = tss->eflags | 2;
2245 ctxt->regs[VCPU_REGS_RAX] = tss->eax;
2246 ctxt->regs[VCPU_REGS_RCX] = tss->ecx;
2247 ctxt->regs[VCPU_REGS_RDX] = tss->edx;
2248 ctxt->regs[VCPU_REGS_RBX] = tss->ebx;
2249 ctxt->regs[VCPU_REGS_RSP] = tss->esp;
2250 ctxt->regs[VCPU_REGS_RBP] = tss->ebp;
2251 ctxt->regs[VCPU_REGS_RSI] = tss->esi;
2252 ctxt->regs[VCPU_REGS_RDI] = tss->edi;
2255 * SDM says that segment selectors are loaded before segment
2258 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2259 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2260 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2261 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2262 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2263 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2264 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2267 * Now load segment descriptors. If fault happenes at this stage
2268 * it is handled in a context of new task
2270 ret = load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2271 if (ret != X86EMUL_CONTINUE)
2273 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2274 if (ret != X86EMUL_CONTINUE)
2276 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2277 if (ret != X86EMUL_CONTINUE)
2279 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2280 if (ret != X86EMUL_CONTINUE)
2282 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2283 if (ret != X86EMUL_CONTINUE)
2285 ret = load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS);
2286 if (ret != X86EMUL_CONTINUE)
2288 ret = load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS);
2289 if (ret != X86EMUL_CONTINUE)
2292 return X86EMUL_CONTINUE;
2295 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2296 u16 tss_selector, u16 old_tss_sel,
2297 ulong old_tss_base, struct desc_struct *new_desc)
2299 struct x86_emulate_ops *ops = ctxt->ops;
2300 struct tss_segment_32 tss_seg;
2302 u32 new_tss_base = get_desc_base(new_desc);
2304 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2306 if (ret != X86EMUL_CONTINUE)
2307 /* FIXME: need to provide precise fault address */
2310 save_state_to_tss32(ctxt, &tss_seg);
2312 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2314 if (ret != X86EMUL_CONTINUE)
2315 /* FIXME: need to provide precise fault address */
2318 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2320 if (ret != X86EMUL_CONTINUE)
2321 /* FIXME: need to provide precise fault address */
2324 if (old_tss_sel != 0xffff) {
2325 tss_seg.prev_task_link = old_tss_sel;
2327 ret = ops->write_std(ctxt, new_tss_base,
2328 &tss_seg.prev_task_link,
2329 sizeof tss_seg.prev_task_link,
2331 if (ret != X86EMUL_CONTINUE)
2332 /* FIXME: need to provide precise fault address */
2336 return load_state_from_tss32(ctxt, &tss_seg);
2339 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2340 u16 tss_selector, int reason,
2341 bool has_error_code, u32 error_code)
2343 struct x86_emulate_ops *ops = ctxt->ops;
2344 struct desc_struct curr_tss_desc, next_tss_desc;
2346 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2347 ulong old_tss_base =
2348 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2351 /* FIXME: old_tss_base == ~0 ? */
2353 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2354 if (ret != X86EMUL_CONTINUE)
2356 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2357 if (ret != X86EMUL_CONTINUE)
2360 /* FIXME: check that next_tss_desc is tss */
2362 if (reason != TASK_SWITCH_IRET) {
2363 if ((tss_selector & 3) > next_tss_desc.dpl ||
2364 ops->cpl(ctxt) > next_tss_desc.dpl)
2365 return emulate_gp(ctxt, 0);
2368 desc_limit = desc_limit_scaled(&next_tss_desc);
2369 if (!next_tss_desc.p ||
2370 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2371 desc_limit < 0x2b)) {
2372 emulate_ts(ctxt, tss_selector & 0xfffc);
2373 return X86EMUL_PROPAGATE_FAULT;
2376 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2377 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2378 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2381 if (reason == TASK_SWITCH_IRET)
2382 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2384 /* set back link to prev task only if NT bit is set in eflags
2385 note that old_tss_sel is not used afetr this point */
2386 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2387 old_tss_sel = 0xffff;
2389 if (next_tss_desc.type & 8)
2390 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2391 old_tss_base, &next_tss_desc);
2393 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2394 old_tss_base, &next_tss_desc);
2395 if (ret != X86EMUL_CONTINUE)
2398 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2399 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2401 if (reason != TASK_SWITCH_IRET) {
2402 next_tss_desc.type |= (1 << 1); /* set busy flag */
2403 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2406 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2407 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2409 if (has_error_code) {
2410 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2411 ctxt->lock_prefix = 0;
2412 ctxt->src.val = (unsigned long) error_code;
2413 ret = em_push(ctxt);
2419 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2420 u16 tss_selector, int reason,
2421 bool has_error_code, u32 error_code)
2425 ctxt->_eip = ctxt->eip;
2426 ctxt->dst.type = OP_NONE;
2428 rc = emulator_do_task_switch(ctxt, tss_selector, reason,
2429 has_error_code, error_code);
2431 if (rc == X86EMUL_CONTINUE)
2432 ctxt->eip = ctxt->_eip;
2434 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2437 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned seg,
2438 int reg, struct operand *op)
2440 int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;
2442 register_address_increment(ctxt, &ctxt->regs[reg], df * op->bytes);
2443 op->addr.mem.ea = register_address(ctxt, ctxt->regs[reg]);
2444 op->addr.mem.seg = seg;
2447 static int em_das(struct x86_emulate_ctxt *ctxt)
2450 bool af, cf, old_cf;
2452 cf = ctxt->eflags & X86_EFLAGS_CF;
2458 af = ctxt->eflags & X86_EFLAGS_AF;
2459 if ((al & 0x0f) > 9 || af) {
2461 cf = old_cf | (al >= 250);
2466 if (old_al > 0x99 || old_cf) {
2472 /* Set PF, ZF, SF */
2473 ctxt->src.type = OP_IMM;
2475 ctxt->src.bytes = 1;
2476 emulate_2op_SrcV(ctxt, "or");
2477 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2479 ctxt->eflags |= X86_EFLAGS_CF;
2481 ctxt->eflags |= X86_EFLAGS_AF;
2482 return X86EMUL_CONTINUE;
2485 static int em_call(struct x86_emulate_ctxt *ctxt)
2487 long rel = ctxt->src.val;
2489 ctxt->src.val = (unsigned long)ctxt->_eip;
2491 return em_push(ctxt);
2494 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2500 old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2501 old_eip = ctxt->_eip;
2503 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2504 if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
2505 return X86EMUL_CONTINUE;
2508 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2510 ctxt->src.val = old_cs;
2512 if (rc != X86EMUL_CONTINUE)
2515 ctxt->src.val = old_eip;
2516 return em_push(ctxt);
2519 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2523 ctxt->dst.type = OP_REG;
2524 ctxt->dst.addr.reg = &ctxt->_eip;
2525 ctxt->dst.bytes = ctxt->op_bytes;
2526 rc = emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
2527 if (rc != X86EMUL_CONTINUE)
2529 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], ctxt->src.val);
2530 return X86EMUL_CONTINUE;
2533 static int em_add(struct x86_emulate_ctxt *ctxt)
2535 emulate_2op_SrcV(ctxt, "add");
2536 return X86EMUL_CONTINUE;
2539 static int em_or(struct x86_emulate_ctxt *ctxt)
2541 emulate_2op_SrcV(ctxt, "or");
2542 return X86EMUL_CONTINUE;
2545 static int em_adc(struct x86_emulate_ctxt *ctxt)
2547 emulate_2op_SrcV(ctxt, "adc");
2548 return X86EMUL_CONTINUE;
2551 static int em_sbb(struct x86_emulate_ctxt *ctxt)
2553 emulate_2op_SrcV(ctxt, "sbb");
2554 return X86EMUL_CONTINUE;
2557 static int em_and(struct x86_emulate_ctxt *ctxt)
2559 emulate_2op_SrcV(ctxt, "and");
2560 return X86EMUL_CONTINUE;
2563 static int em_sub(struct x86_emulate_ctxt *ctxt)
2565 emulate_2op_SrcV(ctxt, "sub");
2566 return X86EMUL_CONTINUE;
2569 static int em_xor(struct x86_emulate_ctxt *ctxt)
2571 emulate_2op_SrcV(ctxt, "xor");
2572 return X86EMUL_CONTINUE;
2575 static int em_cmp(struct x86_emulate_ctxt *ctxt)
2577 emulate_2op_SrcV(ctxt, "cmp");
2578 /* Disable writeback. */
2579 ctxt->dst.type = OP_NONE;
2580 return X86EMUL_CONTINUE;
2583 static int em_test(struct x86_emulate_ctxt *ctxt)
2585 emulate_2op_SrcV(ctxt, "test");
2586 /* Disable writeback. */
2587 ctxt->dst.type = OP_NONE;
2588 return X86EMUL_CONTINUE;
2591 static int em_xchg(struct x86_emulate_ctxt *ctxt)
2593 /* Write back the register source. */
2594 ctxt->src.val = ctxt->dst.val;
2595 write_register_operand(&ctxt->src);
2597 /* Write back the memory destination with implicit LOCK prefix. */
2598 ctxt->dst.val = ctxt->src.orig_val;
2599 ctxt->lock_prefix = 1;
2600 return X86EMUL_CONTINUE;
2603 static int em_imul(struct x86_emulate_ctxt *ctxt)
2605 emulate_2op_SrcV_nobyte(ctxt, "imul");
2606 return X86EMUL_CONTINUE;
2609 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2611 ctxt->dst.val = ctxt->src2.val;
2612 return em_imul(ctxt);
2615 static int em_cwd(struct x86_emulate_ctxt *ctxt)
2617 ctxt->dst.type = OP_REG;
2618 ctxt->dst.bytes = ctxt->src.bytes;
2619 ctxt->dst.addr.reg = &ctxt->regs[VCPU_REGS_RDX];
2620 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
2622 return X86EMUL_CONTINUE;
2625 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2629 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
2630 ctxt->regs[VCPU_REGS_RAX] = (u32)tsc;
2631 ctxt->regs[VCPU_REGS_RDX] = tsc >> 32;
2632 return X86EMUL_CONTINUE;
2635 static int em_mov(struct x86_emulate_ctxt *ctxt)
2637 ctxt->dst.val = ctxt->src.val;
2638 return X86EMUL_CONTINUE;
2641 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
2643 if (ctxt->modrm_reg > VCPU_SREG_GS)
2644 return emulate_ud(ctxt);
2646 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
2647 return X86EMUL_CONTINUE;
2650 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
2652 u16 sel = ctxt->src.val;
2654 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
2655 return emulate_ud(ctxt);
2657 if (ctxt->modrm_reg == VCPU_SREG_SS)
2658 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
2660 /* Disable writeback. */
2661 ctxt->dst.type = OP_NONE;
2662 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
2665 static int em_movdqu(struct x86_emulate_ctxt *ctxt)
2667 memcpy(&ctxt->dst.vec_val, &ctxt->src.vec_val, ctxt->op_bytes);
2668 return X86EMUL_CONTINUE;
2671 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
2676 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
2677 if (rc == X86EMUL_CONTINUE)
2678 ctxt->ops->invlpg(ctxt, linear);
2679 /* Disable writeback. */
2680 ctxt->dst.type = OP_NONE;
2681 return X86EMUL_CONTINUE;
2684 static int em_clts(struct x86_emulate_ctxt *ctxt)
2688 cr0 = ctxt->ops->get_cr(ctxt, 0);
2690 ctxt->ops->set_cr(ctxt, 0, cr0);
2691 return X86EMUL_CONTINUE;
2694 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
2698 if (ctxt->modrm_mod != 3 || ctxt->modrm_rm != 1)
2699 return X86EMUL_UNHANDLEABLE;
2701 rc = ctxt->ops->fix_hypercall(ctxt);
2702 if (rc != X86EMUL_CONTINUE)
2705 /* Let the processor re-execute the fixed hypercall */
2706 ctxt->_eip = ctxt->eip;
2707 /* Disable writeback. */
2708 ctxt->dst.type = OP_NONE;
2709 return X86EMUL_CONTINUE;
2712 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
2714 struct desc_ptr desc_ptr;
2717 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
2718 &desc_ptr.size, &desc_ptr.address,
2720 if (rc != X86EMUL_CONTINUE)
2722 ctxt->ops->set_gdt(ctxt, &desc_ptr);
2723 /* Disable writeback. */
2724 ctxt->dst.type = OP_NONE;
2725 return X86EMUL_CONTINUE;
2728 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
2732 rc = ctxt->ops->fix_hypercall(ctxt);
2734 /* Disable writeback. */
2735 ctxt->dst.type = OP_NONE;
2739 static int em_lidt(struct x86_emulate_ctxt *ctxt)
2741 struct desc_ptr desc_ptr;
2744 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
2745 &desc_ptr.size, &desc_ptr.address,
2747 if (rc != X86EMUL_CONTINUE)
2749 ctxt->ops->set_idt(ctxt, &desc_ptr);
2750 /* Disable writeback. */
2751 ctxt->dst.type = OP_NONE;
2752 return X86EMUL_CONTINUE;
2755 static int em_smsw(struct x86_emulate_ctxt *ctxt)
2757 ctxt->dst.bytes = 2;
2758 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
2759 return X86EMUL_CONTINUE;
2762 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
2764 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
2765 | (ctxt->src.val & 0x0f));
2766 ctxt->dst.type = OP_NONE;
2767 return X86EMUL_CONTINUE;
2770 static int em_loop(struct x86_emulate_ctxt *ctxt)
2772 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1);
2773 if ((address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) != 0) &&
2774 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
2775 jmp_rel(ctxt, ctxt->src.val);
2777 return X86EMUL_CONTINUE;
2780 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
2782 if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0)
2783 jmp_rel(ctxt, ctxt->src.val);
2785 return X86EMUL_CONTINUE;
2788 static int em_in(struct x86_emulate_ctxt *ctxt)
2790 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
2792 return X86EMUL_IO_NEEDED;
2794 return X86EMUL_CONTINUE;
2797 static int em_out(struct x86_emulate_ctxt *ctxt)
2799 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
2801 /* Disable writeback. */
2802 ctxt->dst.type = OP_NONE;
2803 return X86EMUL_CONTINUE;
2806 static int em_cli(struct x86_emulate_ctxt *ctxt)
2808 if (emulator_bad_iopl(ctxt))
2809 return emulate_gp(ctxt, 0);
2811 ctxt->eflags &= ~X86_EFLAGS_IF;
2812 return X86EMUL_CONTINUE;
2815 static int em_sti(struct x86_emulate_ctxt *ctxt)
2817 if (emulator_bad_iopl(ctxt))
2818 return emulate_gp(ctxt, 0);
2820 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
2821 ctxt->eflags |= X86_EFLAGS_IF;
2822 return X86EMUL_CONTINUE;
2825 static int em_bt(struct x86_emulate_ctxt *ctxt)
2827 /* Disable writeback. */
2828 ctxt->dst.type = OP_NONE;
2829 /* only subword offset */
2830 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
2832 emulate_2op_SrcV_nobyte(ctxt, "bt");
2833 return X86EMUL_CONTINUE;
2836 static int em_bts(struct x86_emulate_ctxt *ctxt)
2838 emulate_2op_SrcV_nobyte(ctxt, "bts");
2839 return X86EMUL_CONTINUE;
2842 static int em_btr(struct x86_emulate_ctxt *ctxt)
2844 emulate_2op_SrcV_nobyte(ctxt, "btr");
2845 return X86EMUL_CONTINUE;
2848 static int em_btc(struct x86_emulate_ctxt *ctxt)
2850 emulate_2op_SrcV_nobyte(ctxt, "btc");
2851 return X86EMUL_CONTINUE;
2854 static bool valid_cr(int nr)
2866 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
2868 if (!valid_cr(ctxt->modrm_reg))
2869 return emulate_ud(ctxt);
2871 return X86EMUL_CONTINUE;
2874 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
2876 u64 new_val = ctxt->src.val64;
2877 int cr = ctxt->modrm_reg;
2880 static u64 cr_reserved_bits[] = {
2881 0xffffffff00000000ULL,
2882 0, 0, 0, /* CR3 checked later */
2889 return emulate_ud(ctxt);
2891 if (new_val & cr_reserved_bits[cr])
2892 return emulate_gp(ctxt, 0);
2897 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
2898 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
2899 return emulate_gp(ctxt, 0);
2901 cr4 = ctxt->ops->get_cr(ctxt, 4);
2902 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2904 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
2905 !(cr4 & X86_CR4_PAE))
2906 return emulate_gp(ctxt, 0);
2913 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2914 if (efer & EFER_LMA)
2915 rsvd = CR3_L_MODE_RESERVED_BITS;
2916 else if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PAE)
2917 rsvd = CR3_PAE_RESERVED_BITS;
2918 else if (ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PG)
2919 rsvd = CR3_NONPAE_RESERVED_BITS;
2922 return emulate_gp(ctxt, 0);
2929 cr4 = ctxt->ops->get_cr(ctxt, 4);
2930 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2932 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
2933 return emulate_gp(ctxt, 0);
2939 return X86EMUL_CONTINUE;
2942 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
2946 ctxt->ops->get_dr(ctxt, 7, &dr7);
2948 /* Check if DR7.Global_Enable is set */
2949 return dr7 & (1 << 13);
2952 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
2954 int dr = ctxt->modrm_reg;
2958 return emulate_ud(ctxt);
2960 cr4 = ctxt->ops->get_cr(ctxt, 4);
2961 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
2962 return emulate_ud(ctxt);
2964 if (check_dr7_gd(ctxt))
2965 return emulate_db(ctxt);
2967 return X86EMUL_CONTINUE;
2970 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
2972 u64 new_val = ctxt->src.val64;
2973 int dr = ctxt->modrm_reg;
2975 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
2976 return emulate_gp(ctxt, 0);
2978 return check_dr_read(ctxt);
2981 static int check_svme(struct x86_emulate_ctxt *ctxt)
2985 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2987 if (!(efer & EFER_SVME))
2988 return emulate_ud(ctxt);
2990 return X86EMUL_CONTINUE;
2993 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
2995 u64 rax = ctxt->regs[VCPU_REGS_RAX];
2997 /* Valid physical address? */
2998 if (rax & 0xffff000000000000ULL)
2999 return emulate_gp(ctxt, 0);
3001 return check_svme(ctxt);
3004 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
3006 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3008 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3009 return emulate_ud(ctxt);
3011 return X86EMUL_CONTINUE;
3014 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
3016 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3017 u64 rcx = ctxt->regs[VCPU_REGS_RCX];
3019 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3021 return emulate_gp(ctxt, 0);
3023 return X86EMUL_CONTINUE;
3026 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
3028 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
3029 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3030 return emulate_gp(ctxt, 0);
3032 return X86EMUL_CONTINUE;
3035 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
3037 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
3038 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3039 return emulate_gp(ctxt, 0);
3041 return X86EMUL_CONTINUE;
3044 #define D(_y) { .flags = (_y) }
3045 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
3046 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
3047 .check_perm = (_p) }
3049 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3050 #define G(_f, _g) { .flags = ((_f) | Group), .u.group = (_g) }
3051 #define GD(_f, _g) { .flags = ((_f) | GroupDual), .u.gdual = (_g) }
3052 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3053 #define II(_f, _e, _i) \
3054 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
3055 #define IIP(_f, _e, _i, _p) \
3056 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
3057 .check_perm = (_p) }
3058 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3060 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3061 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3062 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3063 #define I2bvIP(_f, _e, _i, _p) \
3064 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
3066 #define I6ALU(_f, _e) I2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3067 I2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3068 I2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3070 static struct opcode group7_rm1[] = {
3071 DI(SrcNone | ModRM | Priv, monitor),
3072 DI(SrcNone | ModRM | Priv, mwait),
3076 static struct opcode group7_rm3[] = {
3077 DIP(SrcNone | ModRM | Prot | Priv, vmrun, check_svme_pa),
3078 II(SrcNone | ModRM | Prot | VendorSpecific, em_vmmcall, vmmcall),
3079 DIP(SrcNone | ModRM | Prot | Priv, vmload, check_svme_pa),
3080 DIP(SrcNone | ModRM | Prot | Priv, vmsave, check_svme_pa),
3081 DIP(SrcNone | ModRM | Prot | Priv, stgi, check_svme),
3082 DIP(SrcNone | ModRM | Prot | Priv, clgi, check_svme),
3083 DIP(SrcNone | ModRM | Prot | Priv, skinit, check_svme),
3084 DIP(SrcNone | ModRM | Prot | Priv, invlpga, check_svme),
3087 static struct opcode group7_rm7[] = {
3089 DIP(SrcNone | ModRM, rdtscp, check_rdtsc),
3093 static struct opcode group1[] = {
3095 I(Lock | PageTable, em_or),
3098 I(Lock | PageTable, em_and),
3104 static struct opcode group1A[] = {
3105 D(DstMem | SrcNone | ModRM | Mov | Stack), N, N, N, N, N, N, N,
3108 static struct opcode group3[] = {
3109 I(DstMem | SrcImm | ModRM, em_test),
3110 I(DstMem | SrcImm | ModRM, em_test),
3111 I(DstMem | SrcNone | ModRM | Lock, em_not),
3112 I(DstMem | SrcNone | ModRM | Lock, em_neg),
3113 I(SrcMem | ModRM, em_mul_ex),
3114 I(SrcMem | ModRM, em_imul_ex),
3115 I(SrcMem | ModRM, em_div_ex),
3116 I(SrcMem | ModRM, em_idiv_ex),
3119 static struct opcode group4[] = {
3120 D(ByteOp | DstMem | SrcNone | ModRM | Lock), D(ByteOp | DstMem | SrcNone | ModRM | Lock),
3124 static struct opcode group5[] = {
3125 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
3126 D(SrcMem | ModRM | Stack),
3127 I(SrcMemFAddr | ModRM | ImplicitOps | Stack, em_call_far),
3128 D(SrcMem | ModRM | Stack), D(SrcMemFAddr | ModRM | ImplicitOps),
3129 D(SrcMem | ModRM | Stack), N,
3132 static struct opcode group6[] = {
3133 DI(ModRM | Prot, sldt),
3134 DI(ModRM | Prot, str),
3135 DI(ModRM | Prot | Priv, lldt),
3136 DI(ModRM | Prot | Priv, ltr),
3140 static struct group_dual group7 = { {
3141 DI(ModRM | Mov | DstMem | Priv, sgdt),
3142 DI(ModRM | Mov | DstMem | Priv, sidt),
3143 II(ModRM | SrcMem | Priv, em_lgdt, lgdt),
3144 II(ModRM | SrcMem | Priv, em_lidt, lidt),
3145 II(SrcNone | ModRM | DstMem | Mov, em_smsw, smsw), N,
3146 II(SrcMem16 | ModRM | Mov | Priv, em_lmsw, lmsw),
3147 II(SrcMem | ModRM | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3149 I(SrcNone | ModRM | Priv | VendorSpecific, em_vmcall),
3151 N, EXT(0, group7_rm3),
3152 II(SrcNone | ModRM | DstMem | Mov, em_smsw, smsw), N,
3153 II(SrcMem16 | ModRM | Mov | Priv, em_lmsw, lmsw), EXT(0, group7_rm7),
3156 static struct opcode group8[] = {
3158 I(DstMem | SrcImmByte | ModRM, em_bt),
3159 I(DstMem | SrcImmByte | ModRM | Lock | PageTable, em_bts),
3160 I(DstMem | SrcImmByte | ModRM | Lock, em_btr),
3161 I(DstMem | SrcImmByte | ModRM | Lock | PageTable, em_btc),
3164 static struct group_dual group9 = { {
3165 N, D(DstMem64 | ModRM | Lock | PageTable), N, N, N, N, N, N,
3167 N, N, N, N, N, N, N, N,
3170 static struct opcode group11[] = {
3171 I(DstMem | SrcImm | ModRM | Mov | PageTable, em_mov),
3175 static struct gprefix pfx_0f_6f_0f_7f = {
3176 N, N, N, I(Sse, em_movdqu),
3179 static struct opcode opcode_table[256] = {
3181 I6ALU(Lock, em_add),
3182 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
3183 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
3185 I6ALU(Lock | PageTable, em_or),
3186 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
3189 I6ALU(Lock, em_adc),
3190 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
3191 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
3193 I6ALU(Lock, em_sbb),
3194 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
3195 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
3197 I6ALU(Lock | PageTable, em_and), N, N,
3199 I6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3201 I6ALU(Lock, em_xor), N, N,
3203 I6ALU(0, em_cmp), N, N,
3207 X8(I(SrcReg | Stack, em_push)),
3209 X8(I(DstReg | Stack, em_pop)),
3211 I(ImplicitOps | Stack | No64, em_pusha),
3212 I(ImplicitOps | Stack | No64, em_popa),
3213 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3216 I(SrcImm | Mov | Stack, em_push),
3217 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3218 I(SrcImmByte | Mov | Stack, em_push),
3219 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3220 D2bvIP(DstDI | SrcDX | Mov | String, ins, check_perm_in), /* insb, insw/insd */
3221 D2bvIP(SrcSI | DstDX | String, outs, check_perm_out), /* outsb, outsw/outsd */
3225 G(ByteOp | DstMem | SrcImm | ModRM | Group, group1),
3226 G(DstMem | SrcImm | ModRM | Group, group1),
3227 G(ByteOp | DstMem | SrcImm | ModRM | No64 | Group, group1),
3228 G(DstMem | SrcImmByte | ModRM | Group, group1),
3229 I2bv(DstMem | SrcReg | ModRM, em_test),
3230 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
3232 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
3233 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3234 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
3235 D(ModRM | SrcMem | NoAccess | DstReg),
3236 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
3239 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3241 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3242 I(SrcImmFAddr | No64, em_call_far), N,
3243 II(ImplicitOps | Stack, em_pushf, pushf),
3244 II(ImplicitOps | Stack, em_popf, popf), N, N,
3246 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3247 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
3248 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3249 I2bv(SrcSI | DstDI | String, em_cmp),
3251 I2bv(DstAcc | SrcImm, em_test),
3252 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3253 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3254 I2bv(SrcAcc | DstDI | String, em_cmp),
3256 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3258 X8(I(DstReg | SrcImm | Mov, em_mov)),
3260 D2bv(DstMem | SrcImmByte | ModRM),
3261 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3262 I(ImplicitOps | Stack, em_ret),
3263 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
3264 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
3265 G(ByteOp, group11), G(0, group11),
3267 N, N, N, I(ImplicitOps | Stack, em_ret_far),
3268 D(ImplicitOps), DI(SrcImmByte, intn),
3269 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3271 D2bv(DstMem | SrcOne | ModRM), D2bv(DstMem | ModRM),
3274 N, N, N, N, N, N, N, N,
3276 X3(I(SrcImmByte, em_loop)),
3277 I(SrcImmByte, em_jcxz),
3278 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
3279 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
3281 I(SrcImm | Stack, em_call), D(SrcImm | ImplicitOps),
3282 I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
3283 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
3284 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
3286 N, DI(ImplicitOps, icebp), N, N,
3287 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
3288 G(ByteOp, group3), G(0, group3),
3290 D(ImplicitOps), D(ImplicitOps),
3291 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
3292 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3295 static struct opcode twobyte_table[256] = {
3297 G(0, group6), GD(0, &group7), N, N,
3298 N, I(ImplicitOps | VendorSpecific, em_syscall),
3299 II(ImplicitOps | Priv, em_clts, clts), N,
3300 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3301 N, D(ImplicitOps | ModRM), N, N,
3303 N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
3305 DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
3306 DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
3307 DIP(ModRM | SrcMem | Priv | Op3264, cr_write, check_cr_write),
3308 DIP(ModRM | SrcMem | Priv | Op3264, dr_write, check_dr_write),
3310 N, N, N, N, N, N, N, N,
3312 DI(ImplicitOps | Priv, wrmsr),
3313 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
3314 DI(ImplicitOps | Priv, rdmsr),
3315 DIP(ImplicitOps | Priv, rdpmc, check_rdpmc),
3316 I(ImplicitOps | VendorSpecific, em_sysenter),
3317 I(ImplicitOps | Priv | VendorSpecific, em_sysexit),
3319 N, N, N, N, N, N, N, N,
3321 X16(D(DstReg | SrcMem | ModRM | Mov)),
3323 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3328 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3333 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3337 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3339 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
3340 DI(ImplicitOps, cpuid), I(DstMem | SrcReg | ModRM | BitOp, em_bt),
3341 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3342 D(DstMem | SrcReg | Src2CL | ModRM), N, N,
3344 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
3345 DI(ImplicitOps, rsm),
3346 I(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
3347 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3348 D(DstMem | SrcReg | Src2CL | ModRM),
3349 D(ModRM), I(DstReg | SrcMem | ModRM, em_imul),
3351 D2bv(DstMem | SrcReg | ModRM | Lock | PageTable),
3352 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
3353 I(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
3354 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
3355 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
3356 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3360 I(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
3361 D(DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
3362 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3364 D2bv(DstMem | SrcReg | ModRM | Lock),
3365 N, D(DstMem | SrcReg | ModRM | Mov),
3366 N, N, N, GD(0, &group9),
3367 N, N, N, N, N, N, N, N,
3369 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3371 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3373 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
3390 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
3394 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3400 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
3401 unsigned size, bool sign_extension)
3403 int rc = X86EMUL_CONTINUE;
3407 op->addr.mem.ea = ctxt->_eip;
3408 /* NB. Immediates are sign-extended as necessary. */
3409 switch (op->bytes) {
3411 op->val = insn_fetch(s8, ctxt);
3414 op->val = insn_fetch(s16, ctxt);
3417 op->val = insn_fetch(s32, ctxt);
3420 if (!sign_extension) {
3421 switch (op->bytes) {
3429 op->val &= 0xffffffff;
3437 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
3440 int rc = X86EMUL_CONTINUE;
3444 decode_register_operand(ctxt, op,
3446 ctxt->twobyte && (ctxt->b == 0xb6 || ctxt->b == 0xb7));
3449 rc = decode_imm(ctxt, op, 1, false);
3452 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3456 if ((ctxt->d & BitOp) && op == &ctxt->dst)
3457 fetch_bit_operand(ctxt);
3458 op->orig_val = op->val;
3461 ctxt->memop.bytes = 8;
3465 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3466 op->addr.reg = &ctxt->regs[VCPU_REGS_RAX];
3467 fetch_register_operand(op);
3468 op->orig_val = op->val;
3472 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3474 register_address(ctxt, ctxt->regs[VCPU_REGS_RDI]);
3475 op->addr.mem.seg = VCPU_SREG_ES;
3481 op->addr.reg = &ctxt->regs[VCPU_REGS_RDX];
3482 fetch_register_operand(op);
3486 op->val = ctxt->regs[VCPU_REGS_RCX] & 0xff;
3489 rc = decode_imm(ctxt, op, 1, true);
3496 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
3499 ctxt->memop.bytes = 2;
3502 ctxt->memop.bytes = 4;
3505 rc = decode_imm(ctxt, op, 2, false);
3508 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
3512 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3514 register_address(ctxt, ctxt->regs[VCPU_REGS_RSI]);
3515 op->addr.mem.seg = seg_override(ctxt);
3520 op->addr.mem.ea = ctxt->_eip;
3521 op->bytes = ctxt->op_bytes + 2;
3522 insn_fetch_arr(op->valptr, op->bytes, ctxt);
3525 ctxt->memop.bytes = ctxt->op_bytes + 2;
3528 op->val = VCPU_SREG_ES;
3531 op->val = VCPU_SREG_CS;
3534 op->val = VCPU_SREG_SS;
3537 op->val = VCPU_SREG_DS;
3540 op->val = VCPU_SREG_FS;
3543 op->val = VCPU_SREG_GS;
3546 /* Special instructions do their own operand decoding. */
3548 op->type = OP_NONE; /* Disable writeback. */
3556 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
3558 int rc = X86EMUL_CONTINUE;
3559 int mode = ctxt->mode;
3560 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
3561 bool op_prefix = false;
3562 struct opcode opcode;
3564 ctxt->memop.type = OP_NONE;
3565 ctxt->memopp = NULL;
3566 ctxt->_eip = ctxt->eip;
3567 ctxt->fetch.start = ctxt->_eip;
3568 ctxt->fetch.end = ctxt->fetch.start + insn_len;
3570 memcpy(ctxt->fetch.data, insn, insn_len);
3573 case X86EMUL_MODE_REAL:
3574 case X86EMUL_MODE_VM86:
3575 case X86EMUL_MODE_PROT16:
3576 def_op_bytes = def_ad_bytes = 2;
3578 case X86EMUL_MODE_PROT32:
3579 def_op_bytes = def_ad_bytes = 4;
3581 #ifdef CONFIG_X86_64
3582 case X86EMUL_MODE_PROT64:
3588 return EMULATION_FAILED;
3591 ctxt->op_bytes = def_op_bytes;
3592 ctxt->ad_bytes = def_ad_bytes;
3594 /* Legacy prefixes. */
3596 switch (ctxt->b = insn_fetch(u8, ctxt)) {
3597 case 0x66: /* operand-size override */
3599 /* switch between 2/4 bytes */
3600 ctxt->op_bytes = def_op_bytes ^ 6;
3602 case 0x67: /* address-size override */
3603 if (mode == X86EMUL_MODE_PROT64)
3604 /* switch between 4/8 bytes */
3605 ctxt->ad_bytes = def_ad_bytes ^ 12;
3607 /* switch between 2/4 bytes */
3608 ctxt->ad_bytes = def_ad_bytes ^ 6;
3610 case 0x26: /* ES override */
3611 case 0x2e: /* CS override */
3612 case 0x36: /* SS override */
3613 case 0x3e: /* DS override */
3614 set_seg_override(ctxt, (ctxt->b >> 3) & 3);
3616 case 0x64: /* FS override */
3617 case 0x65: /* GS override */
3618 set_seg_override(ctxt, ctxt->b & 7);
3620 case 0x40 ... 0x4f: /* REX */
3621 if (mode != X86EMUL_MODE_PROT64)
3623 ctxt->rex_prefix = ctxt->b;
3625 case 0xf0: /* LOCK */
3626 ctxt->lock_prefix = 1;
3628 case 0xf2: /* REPNE/REPNZ */
3629 case 0xf3: /* REP/REPE/REPZ */
3630 ctxt->rep_prefix = ctxt->b;
3636 /* Any legacy prefix after a REX prefix nullifies its effect. */
3638 ctxt->rex_prefix = 0;
3644 if (ctxt->rex_prefix & 8)
3645 ctxt->op_bytes = 8; /* REX.W */
3647 /* Opcode byte(s). */
3648 opcode = opcode_table[ctxt->b];
3649 /* Two-byte opcode? */
3650 if (ctxt->b == 0x0f) {
3652 ctxt->b = insn_fetch(u8, ctxt);
3653 opcode = twobyte_table[ctxt->b];
3655 ctxt->d = opcode.flags;
3657 while (ctxt->d & GroupMask) {
3658 switch (ctxt->d & GroupMask) {
3660 ctxt->modrm = insn_fetch(u8, ctxt);
3662 goffset = (ctxt->modrm >> 3) & 7;
3663 opcode = opcode.u.group[goffset];
3666 ctxt->modrm = insn_fetch(u8, ctxt);
3668 goffset = (ctxt->modrm >> 3) & 7;
3669 if ((ctxt->modrm >> 6) == 3)
3670 opcode = opcode.u.gdual->mod3[goffset];
3672 opcode = opcode.u.gdual->mod012[goffset];
3675 goffset = ctxt->modrm & 7;
3676 opcode = opcode.u.group[goffset];
3679 if (ctxt->rep_prefix && op_prefix)
3680 return EMULATION_FAILED;
3681 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
3682 switch (simd_prefix) {
3683 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
3684 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
3685 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
3686 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
3690 return EMULATION_FAILED;
3693 ctxt->d &= ~(u64)GroupMask;
3694 ctxt->d |= opcode.flags;
3697 ctxt->execute = opcode.u.execute;
3698 ctxt->check_perm = opcode.check_perm;
3699 ctxt->intercept = opcode.intercept;
3702 if (ctxt->d == 0 || (ctxt->d & Undefined))
3703 return EMULATION_FAILED;
3705 if (!(ctxt->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
3706 return EMULATION_FAILED;
3708 if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
3711 if (ctxt->d & Op3264) {
3712 if (mode == X86EMUL_MODE_PROT64)
3719 ctxt->op_bytes = 16;
3721 /* ModRM and SIB bytes. */
3722 if (ctxt->d & ModRM) {
3723 rc = decode_modrm(ctxt, &ctxt->memop);
3724 if (!ctxt->has_seg_override)
3725 set_seg_override(ctxt, ctxt->modrm_seg);
3726 } else if (ctxt->d & MemAbs)
3727 rc = decode_abs(ctxt, &ctxt->memop);
3728 if (rc != X86EMUL_CONTINUE)
3731 if (!ctxt->has_seg_override)
3732 set_seg_override(ctxt, VCPU_SREG_DS);
3734 ctxt->memop.addr.mem.seg = seg_override(ctxt);
3736 if (ctxt->memop.type == OP_MEM && ctxt->ad_bytes != 8)
3737 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
3740 * Decode and fetch the source operand: register, memory
3743 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
3744 if (rc != X86EMUL_CONTINUE)
3748 * Decode and fetch the second source operand: register, memory
3751 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
3752 if (rc != X86EMUL_CONTINUE)
3755 /* Decode and fetch the destination operand: register or memory. */
3756 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
3759 if (ctxt->memopp && ctxt->memopp->type == OP_MEM && ctxt->rip_relative)
3760 ctxt->memopp->addr.mem.ea += ctxt->_eip;
3762 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
3765 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
3767 return ctxt->d & PageTable;
3770 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
3772 /* The second termination condition only applies for REPE
3773 * and REPNE. Test if the repeat string operation prefix is
3774 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
3775 * corresponding termination condition according to:
3776 * - if REPE/REPZ and ZF = 0 then done
3777 * - if REPNE/REPNZ and ZF = 1 then done
3779 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
3780 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
3781 && (((ctxt->rep_prefix == REPE_PREFIX) &&
3782 ((ctxt->eflags & EFLG_ZF) == 0))
3783 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
3784 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
3790 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
3792 struct x86_emulate_ops *ops = ctxt->ops;
3794 int rc = X86EMUL_CONTINUE;
3795 int saved_dst_type = ctxt->dst.type;
3797 ctxt->mem_read.pos = 0;
3799 if (ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) {
3800 rc = emulate_ud(ctxt);
3804 /* LOCK prefix is allowed only with some instructions */
3805 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
3806 rc = emulate_ud(ctxt);
3810 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
3811 rc = emulate_ud(ctxt);
3816 && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)
3817 || !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
3818 rc = emulate_ud(ctxt);
3822 if ((ctxt->d & Sse) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
3823 rc = emulate_nm(ctxt);
3827 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
3828 rc = emulator_check_intercept(ctxt, ctxt->intercept,
3829 X86_ICPT_PRE_EXCEPT);
3830 if (rc != X86EMUL_CONTINUE)
3834 /* Privileged instruction can be executed only in CPL=0 */
3835 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
3836 rc = emulate_gp(ctxt, 0);
3840 /* Instruction can only be executed in protected mode */
3841 if ((ctxt->d & Prot) && !(ctxt->mode & X86EMUL_MODE_PROT)) {
3842 rc = emulate_ud(ctxt);
3846 /* Do instruction specific permission checks */
3847 if (ctxt->check_perm) {
3848 rc = ctxt->check_perm(ctxt);
3849 if (rc != X86EMUL_CONTINUE)
3853 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
3854 rc = emulator_check_intercept(ctxt, ctxt->intercept,
3855 X86_ICPT_POST_EXCEPT);
3856 if (rc != X86EMUL_CONTINUE)
3860 if (ctxt->rep_prefix && (ctxt->d & String)) {
3861 /* All REP prefixes have the same first termination condition */
3862 if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0) {
3863 ctxt->eip = ctxt->_eip;
3868 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
3869 rc = segmented_read(ctxt, ctxt->src.addr.mem,
3870 ctxt->src.valptr, ctxt->src.bytes);
3871 if (rc != X86EMUL_CONTINUE)
3873 ctxt->src.orig_val64 = ctxt->src.val64;
3876 if (ctxt->src2.type == OP_MEM) {
3877 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
3878 &ctxt->src2.val, ctxt->src2.bytes);
3879 if (rc != X86EMUL_CONTINUE)
3883 if ((ctxt->d & DstMask) == ImplicitOps)
3887 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
3888 /* optimisation - avoid slow emulated read if Mov */
3889 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
3890 &ctxt->dst.val, ctxt->dst.bytes);
3891 if (rc != X86EMUL_CONTINUE)
3894 ctxt->dst.orig_val = ctxt->dst.val;
3898 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
3899 rc = emulator_check_intercept(ctxt, ctxt->intercept,
3900 X86_ICPT_POST_MEMACCESS);
3901 if (rc != X86EMUL_CONTINUE)
3905 if (ctxt->execute) {
3906 rc = ctxt->execute(ctxt);
3907 if (rc != X86EMUL_CONTINUE)
3916 case 0x40 ... 0x47: /* inc r16/r32 */
3917 emulate_1op(ctxt, "inc");
3919 case 0x48 ... 0x4f: /* dec r16/r32 */
3920 emulate_1op(ctxt, "dec");
3922 case 0x63: /* movsxd */
3923 if (ctxt->mode != X86EMUL_MODE_PROT64)
3924 goto cannot_emulate;
3925 ctxt->dst.val = (s32) ctxt->src.val;
3927 case 0x6c: /* insb */
3928 case 0x6d: /* insw/insd */
3929 ctxt->src.val = ctxt->regs[VCPU_REGS_RDX];
3932 case 0x6e: /* outsb */
3933 case 0x6f: /* outsw/outsd */
3934 ctxt->dst.val = ctxt->regs[VCPU_REGS_RDX];
3937 case 0x70 ... 0x7f: /* jcc (short) */
3938 if (test_cc(ctxt->b, ctxt->eflags))
3939 jmp_rel(ctxt, ctxt->src.val);
3941 case 0x8d: /* lea r16/r32, m */
3942 ctxt->dst.val = ctxt->src.addr.mem.ea;
3944 case 0x8f: /* pop (sole member of Grp1a) */
3945 rc = em_grp1a(ctxt);
3947 case 0x90 ... 0x97: /* nop / xchg reg, rax */
3948 if (ctxt->dst.addr.reg == &ctxt->regs[VCPU_REGS_RAX])
3952 case 0x98: /* cbw/cwde/cdqe */
3953 switch (ctxt->op_bytes) {
3954 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
3955 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
3956 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
3962 case 0xcc: /* int3 */
3963 rc = emulate_int(ctxt, 3);
3965 case 0xcd: /* int n */
3966 rc = emulate_int(ctxt, ctxt->src.val);
3968 case 0xce: /* into */
3969 if (ctxt->eflags & EFLG_OF)
3970 rc = emulate_int(ctxt, 4);
3972 case 0xd0 ... 0xd1: /* Grp2 */
3975 case 0xd2 ... 0xd3: /* Grp2 */
3976 ctxt->src.val = ctxt->regs[VCPU_REGS_RCX];
3979 case 0xe9: /* jmp rel */
3980 case 0xeb: /* jmp rel short */
3981 jmp_rel(ctxt, ctxt->src.val);
3982 ctxt->dst.type = OP_NONE; /* Disable writeback. */
3984 case 0xf4: /* hlt */
3985 ctxt->ops->halt(ctxt);
3987 case 0xf5: /* cmc */
3988 /* complement carry flag from eflags reg */
3989 ctxt->eflags ^= EFLG_CF;
3991 case 0xf8: /* clc */
3992 ctxt->eflags &= ~EFLG_CF;
3994 case 0xf9: /* stc */
3995 ctxt->eflags |= EFLG_CF;
3997 case 0xfc: /* cld */
3998 ctxt->eflags &= ~EFLG_DF;
4000 case 0xfd: /* std */
4001 ctxt->eflags |= EFLG_DF;
4003 case 0xfe: /* Grp4 */
4004 rc = em_grp45(ctxt);
4006 case 0xff: /* Grp5 */
4007 rc = em_grp45(ctxt);
4010 goto cannot_emulate;
4013 if (rc != X86EMUL_CONTINUE)
4017 rc = writeback(ctxt);
4018 if (rc != X86EMUL_CONTINUE)
4022 * restore dst type in case the decoding will be reused
4023 * (happens for string instruction )
4025 ctxt->dst.type = saved_dst_type;
4027 if ((ctxt->d & SrcMask) == SrcSI)
4028 string_addr_inc(ctxt, seg_override(ctxt),
4029 VCPU_REGS_RSI, &ctxt->src);
4031 if ((ctxt->d & DstMask) == DstDI)
4032 string_addr_inc(ctxt, VCPU_SREG_ES, VCPU_REGS_RDI,
4035 if (ctxt->rep_prefix && (ctxt->d & String)) {
4036 struct read_cache *r = &ctxt->io_read;
4037 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1);
4039 if (!string_insn_completed(ctxt)) {
4041 * Re-enter guest when pio read ahead buffer is empty
4042 * or, if it is not used, after each 1024 iteration.
4044 if ((r->end != 0 || ctxt->regs[VCPU_REGS_RCX] & 0x3ff) &&
4045 (r->end == 0 || r->end != r->pos)) {
4047 * Reset read cache. Usually happens before
4048 * decode, but since instruction is restarted
4049 * we have to do it here.
4051 ctxt->mem_read.end = 0;
4052 return EMULATION_RESTART;
4054 goto done; /* skip rip writeback */
4058 ctxt->eip = ctxt->_eip;
4061 if (rc == X86EMUL_PROPAGATE_FAULT)
4062 ctxt->have_exception = true;
4063 if (rc == X86EMUL_INTERCEPTED)
4064 return EMULATION_INTERCEPTED;
4066 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4070 case 0x09: /* wbinvd */
4071 (ctxt->ops->wbinvd)(ctxt);
4073 case 0x08: /* invd */
4074 case 0x0d: /* GrpP (prefetch) */
4075 case 0x18: /* Grp16 (prefetch/nop) */
4077 case 0x20: /* mov cr, reg */
4078 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
4080 case 0x21: /* mov from dr to reg */
4081 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
4083 case 0x22: /* mov reg, cr */
4084 if (ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val)) {
4085 emulate_gp(ctxt, 0);
4086 rc = X86EMUL_PROPAGATE_FAULT;
4089 ctxt->dst.type = OP_NONE;
4091 case 0x23: /* mov from reg to dr */
4092 if (ops->set_dr(ctxt, ctxt->modrm_reg, ctxt->src.val &
4093 ((ctxt->mode == X86EMUL_MODE_PROT64) ?
4094 ~0ULL : ~0U)) < 0) {
4095 /* #UD condition is already handled by the code above */
4096 emulate_gp(ctxt, 0);
4097 rc = X86EMUL_PROPAGATE_FAULT;
4101 ctxt->dst.type = OP_NONE; /* no writeback */
4105 msr_data = (u32)ctxt->regs[VCPU_REGS_RAX]
4106 | ((u64)ctxt->regs[VCPU_REGS_RDX] << 32);
4107 if (ops->set_msr(ctxt, ctxt->regs[VCPU_REGS_RCX], msr_data)) {
4108 emulate_gp(ctxt, 0);
4109 rc = X86EMUL_PROPAGATE_FAULT;
4112 rc = X86EMUL_CONTINUE;
4116 if (ops->get_msr(ctxt, ctxt->regs[VCPU_REGS_RCX], &msr_data)) {
4117 emulate_gp(ctxt, 0);
4118 rc = X86EMUL_PROPAGATE_FAULT;
4121 ctxt->regs[VCPU_REGS_RAX] = (u32)msr_data;
4122 ctxt->regs[VCPU_REGS_RDX] = msr_data >> 32;
4124 rc = X86EMUL_CONTINUE;
4126 case 0x40 ... 0x4f: /* cmov */
4127 ctxt->dst.val = ctxt->dst.orig_val = ctxt->src.val;
4128 if (!test_cc(ctxt->b, ctxt->eflags))
4129 ctxt->dst.type = OP_NONE; /* no writeback */
4131 case 0x80 ... 0x8f: /* jnz rel, etc*/
4132 if (test_cc(ctxt->b, ctxt->eflags))
4133 jmp_rel(ctxt, ctxt->src.val);
4135 case 0x90 ... 0x9f: /* setcc r/m8 */
4136 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
4138 case 0xa4: /* shld imm8, r, r/m */
4139 case 0xa5: /* shld cl, r, r/m */
4140 emulate_2op_cl(ctxt, "shld");
4142 case 0xac: /* shrd imm8, r, r/m */
4143 case 0xad: /* shrd cl, r, r/m */
4144 emulate_2op_cl(ctxt, "shrd");
4146 case 0xae: /* clflush */
4148 case 0xb0 ... 0xb1: /* cmpxchg */
4150 * Save real source value, then compare EAX against
4153 ctxt->src.orig_val = ctxt->src.val;
4154 ctxt->src.val = ctxt->regs[VCPU_REGS_RAX];
4155 emulate_2op_SrcV(ctxt, "cmp");
4156 if (ctxt->eflags & EFLG_ZF) {
4157 /* Success: write back to memory. */
4158 ctxt->dst.val = ctxt->src.orig_val;
4160 /* Failure: write the value we saw to EAX. */
4161 ctxt->dst.type = OP_REG;
4162 ctxt->dst.addr.reg = (unsigned long *)&ctxt->regs[VCPU_REGS_RAX];
4165 case 0xb6 ... 0xb7: /* movzx */
4166 ctxt->dst.bytes = ctxt->op_bytes;
4167 ctxt->dst.val = (ctxt->d & ByteOp) ? (u8) ctxt->src.val
4168 : (u16) ctxt->src.val;
4170 case 0xbc: { /* bsf */
4172 __asm__ ("bsf %2, %0; setz %1"
4173 : "=r"(ctxt->dst.val), "=q"(zf)
4174 : "r"(ctxt->src.val));
4175 ctxt->eflags &= ~X86_EFLAGS_ZF;
4177 ctxt->eflags |= X86_EFLAGS_ZF;
4178 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4182 case 0xbd: { /* bsr */
4184 __asm__ ("bsr %2, %0; setz %1"
4185 : "=r"(ctxt->dst.val), "=q"(zf)
4186 : "r"(ctxt->src.val));
4187 ctxt->eflags &= ~X86_EFLAGS_ZF;
4189 ctxt->eflags |= X86_EFLAGS_ZF;
4190 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4194 case 0xbe ... 0xbf: /* movsx */
4195 ctxt->dst.bytes = ctxt->op_bytes;
4196 ctxt->dst.val = (ctxt->d & ByteOp) ? (s8) ctxt->src.val :
4197 (s16) ctxt->src.val;
4199 case 0xc0 ... 0xc1: /* xadd */
4200 emulate_2op_SrcV(ctxt, "add");
4201 /* Write back the register source. */
4202 ctxt->src.val = ctxt->dst.orig_val;
4203 write_register_operand(&ctxt->src);
4205 case 0xc3: /* movnti */
4206 ctxt->dst.bytes = ctxt->op_bytes;
4207 ctxt->dst.val = (ctxt->op_bytes == 4) ? (u32) ctxt->src.val :
4208 (u64) ctxt->src.val;
4210 case 0xc7: /* Grp9 (cmpxchg8b) */
4214 goto cannot_emulate;
4217 if (rc != X86EMUL_CONTINUE)
4223 return EMULATION_FAILED;