1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
35 #define OpImplicit 1ull /* No generic decode */
36 #define OpReg 2ull /* Register */
37 #define OpMem 3ull /* Memory */
38 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
39 #define OpDI 5ull /* ES:DI/EDI/RDI */
40 #define OpMem64 6ull /* Memory, 64-bit */
41 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
42 #define OpDX 8ull /* DX register */
43 #define OpCL 9ull /* CL register (for shifts) */
44 #define OpImmByte 10ull /* 8-bit sign extended immediate */
45 #define OpOne 11ull /* Implied 1 */
46 #define OpImm 12ull /* Sign extended immediate */
47 #define OpMem16 13ull /* Memory operand (16-bit). */
48 #define OpMem32 14ull /* Memory operand (32-bit). */
49 #define OpImmU 15ull /* Immediate operand, zero extended */
50 #define OpSI 16ull /* SI/ESI/RSI */
51 #define OpImmFAddr 17ull /* Immediate far address */
52 #define OpMemFAddr 18ull /* Far address in memory */
53 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
54 #define OpES 20ull /* ES */
55 #define OpCS 21ull /* CS */
56 #define OpSS 22ull /* SS */
57 #define OpDS 23ull /* DS */
58 #define OpFS 24ull /* FS */
59 #define OpGS 25ull /* GS */
61 #define OpBits 5 /* Width of operand field */
62 #define OpMask ((1ull << OpBits) - 1)
65 * Opcode effective-address decode tables.
66 * Note that we only emulate instructions that have at least one memory
67 * operand (excluding implicit stack references). We assume that stack
68 * references and instruction fetches will never occur in special memory
69 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
73 /* Operand sizes: 8-bit operands or specified/overridden size. */
74 #define ByteOp (1<<0) /* 8-bit operands. */
75 /* Destination operand type. */
77 #define ImplicitOps (OpImplicit << DstShift)
78 #define DstReg (OpReg << DstShift)
79 #define DstMem (OpMem << DstShift)
80 #define DstAcc (OpAcc << DstShift)
81 #define DstDI (OpDI << DstShift)
82 #define DstMem64 (OpMem64 << DstShift)
83 #define DstImmUByte (OpImmUByte << DstShift)
84 #define DstDX (OpDX << DstShift)
85 #define DstMask (OpMask << DstShift)
86 /* Source operand type. */
88 #define SrcNone (OpNone << SrcShift)
89 #define SrcReg (OpReg << SrcShift)
90 #define SrcMem (OpMem << SrcShift)
91 #define SrcMem16 (OpMem16 << SrcShift)
92 #define SrcMem32 (OpMem32 << SrcShift)
93 #define SrcImm (OpImm << SrcShift)
94 #define SrcImmByte (OpImmByte << SrcShift)
95 #define SrcOne (OpOne << SrcShift)
96 #define SrcImmUByte (OpImmUByte << SrcShift)
97 #define SrcImmU (OpImmU << SrcShift)
98 #define SrcSI (OpSI << SrcShift)
99 #define SrcImmFAddr (OpImmFAddr << SrcShift)
100 #define SrcMemFAddr (OpMemFAddr << SrcShift)
101 #define SrcAcc (OpAcc << SrcShift)
102 #define SrcImmU16 (OpImmU16 << SrcShift)
103 #define SrcDX (OpDX << SrcShift)
104 #define SrcMask (OpMask << SrcShift)
105 #define BitOp (1<<11)
106 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
107 #define String (1<<13) /* String instruction (rep capable) */
108 #define Stack (1<<14) /* Stack instruction (push/pop) */
109 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
110 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
111 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
112 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
113 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
114 #define Sse (1<<18) /* SSE Vector instruction */
115 /* Generic ModRM decode. */
116 #define ModRM (1<<19)
117 /* Destination is only written; never read. */
120 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
121 #define VendorSpecific (1<<22) /* Vendor specific instruction */
122 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
123 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
124 #define Undefined (1<<25) /* No Such Instruction */
125 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
126 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
128 #define PageTable (1 << 29) /* instruction used to write page table */
129 /* Source 2 operand type */
130 #define Src2Shift (30)
131 #define Src2None (OpNone << Src2Shift)
132 #define Src2CL (OpCL << Src2Shift)
133 #define Src2ImmByte (OpImmByte << Src2Shift)
134 #define Src2One (OpOne << Src2Shift)
135 #define Src2Imm (OpImm << Src2Shift)
136 #define Src2ES (OpES << Src2Shift)
137 #define Src2CS (OpCS << Src2Shift)
138 #define Src2SS (OpSS << Src2Shift)
139 #define Src2DS (OpDS << Src2Shift)
140 #define Src2FS (OpFS << Src2Shift)
141 #define Src2GS (OpGS << Src2Shift)
142 #define Src2Mask (OpMask << Src2Shift)
144 #define X2(x...) x, x
145 #define X3(x...) X2(x), x
146 #define X4(x...) X2(x), X2(x)
147 #define X5(x...) X4(x), x
148 #define X6(x...) X4(x), X2(x)
149 #define X7(x...) X4(x), X3(x)
150 #define X8(x...) X4(x), X4(x)
151 #define X16(x...) X8(x), X8(x)
157 int (*execute)(struct x86_emulate_ctxt *ctxt);
158 struct opcode *group;
159 struct group_dual *gdual;
160 struct gprefix *gprefix;
162 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
166 struct opcode mod012[8];
167 struct opcode mod3[8];
171 struct opcode pfx_no;
172 struct opcode pfx_66;
173 struct opcode pfx_f2;
174 struct opcode pfx_f3;
177 /* EFLAGS bit definitions. */
178 #define EFLG_ID (1<<21)
179 #define EFLG_VIP (1<<20)
180 #define EFLG_VIF (1<<19)
181 #define EFLG_AC (1<<18)
182 #define EFLG_VM (1<<17)
183 #define EFLG_RF (1<<16)
184 #define EFLG_IOPL (3<<12)
185 #define EFLG_NT (1<<14)
186 #define EFLG_OF (1<<11)
187 #define EFLG_DF (1<<10)
188 #define EFLG_IF (1<<9)
189 #define EFLG_TF (1<<8)
190 #define EFLG_SF (1<<7)
191 #define EFLG_ZF (1<<6)
192 #define EFLG_AF (1<<4)
193 #define EFLG_PF (1<<2)
194 #define EFLG_CF (1<<0)
196 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
197 #define EFLG_RESERVED_ONE_MASK 2
200 * Instruction emulation:
201 * Most instructions are emulated directly via a fragment of inline assembly
202 * code. This allows us to save/restore EFLAGS and thus very easily pick up
203 * any modified flags.
206 #if defined(CONFIG_X86_64)
207 #define _LO32 "k" /* force 32-bit operand */
208 #define _STK "%%rsp" /* stack pointer */
209 #elif defined(__i386__)
210 #define _LO32 "" /* force 32-bit operand */
211 #define _STK "%%esp" /* stack pointer */
215 * These EFLAGS bits are restored from saved value during emulation, and
216 * any changes are written back to the saved value after emulation.
218 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
220 /* Before executing instruction: restore necessary bits in EFLAGS. */
221 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
222 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
223 "movl %"_sav",%"_LO32 _tmp"; " \
226 "movl %"_msk",%"_LO32 _tmp"; " \
227 "andl %"_LO32 _tmp",("_STK"); " \
229 "notl %"_LO32 _tmp"; " \
230 "andl %"_LO32 _tmp",("_STK"); " \
231 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
233 "orl %"_LO32 _tmp",("_STK"); " \
237 /* After executing instruction: write-back necessary bits in EFLAGS. */
238 #define _POST_EFLAGS(_sav, _msk, _tmp) \
239 /* _sav |= EFLAGS & _msk; */ \
242 "andl %"_msk",%"_LO32 _tmp"; " \
243 "orl %"_LO32 _tmp",%"_sav"; "
251 #define ____emulate_2op(ctxt, _op, _x, _y, _suffix, _dsttype) \
253 __asm__ __volatile__ ( \
254 _PRE_EFLAGS("0", "4", "2") \
255 _op _suffix " %"_x"3,%1; " \
256 _POST_EFLAGS("0", "4", "2") \
257 : "=m" ((ctxt)->eflags), \
258 "+q" (*(_dsttype*)&(ctxt)->dst.val), \
260 : _y ((ctxt)->src.val), "i" (EFLAGS_MASK)); \
264 /* Raw emulation: instruction has two explicit operands. */
265 #define __emulate_2op_nobyte(ctxt,_op,_wx,_wy,_lx,_ly,_qx,_qy) \
267 unsigned long _tmp; \
269 switch ((ctxt)->dst.bytes) { \
271 ____emulate_2op(ctxt,_op,_wx,_wy,"w",u16); \
274 ____emulate_2op(ctxt,_op,_lx,_ly,"l",u32); \
277 ON64(____emulate_2op(ctxt,_op,_qx,_qy,"q",u64)); \
282 #define __emulate_2op(ctxt,_op,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
284 unsigned long _tmp; \
285 switch ((ctxt)->dst.bytes) { \
287 ____emulate_2op(ctxt,_op,_bx,_by,"b",u8); \
290 __emulate_2op_nobyte(ctxt, _op, \
291 _wx, _wy, _lx, _ly, _qx, _qy); \
296 /* Source operand is byte-sized and may be restricted to just %cl. */
297 #define emulate_2op_SrcB(ctxt, _op) \
298 __emulate_2op(ctxt, _op, "b", "c", "b", "c", "b", "c", "b", "c")
300 /* Source operand is byte, word, long or quad sized. */
301 #define emulate_2op_SrcV(ctxt, _op) \
302 __emulate_2op(ctxt, _op, "b", "q", "w", "r", _LO32, "r", "", "r")
304 /* Source operand is word, long or quad sized. */
305 #define emulate_2op_SrcV_nobyte(ctxt, _op) \
306 __emulate_2op_nobyte(ctxt, _op, "w", "r", _LO32, "r", "", "r")
308 /* Instruction has three operands and one operand is stored in ECX register */
309 #define __emulate_2op_cl(ctxt, _op, _suffix, _type) \
311 unsigned long _tmp; \
312 _type _clv = (ctxt)->src2.val; \
313 _type _srcv = (ctxt)->src.val; \
314 _type _dstv = (ctxt)->dst.val; \
316 __asm__ __volatile__ ( \
317 _PRE_EFLAGS("0", "5", "2") \
318 _op _suffix " %4,%1 \n" \
319 _POST_EFLAGS("0", "5", "2") \
320 : "=m" ((ctxt)->eflags), "+r" (_dstv), "=&r" (_tmp) \
321 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
324 (ctxt)->src2.val = (unsigned long) _clv; \
325 (ctxt)->src2.val = (unsigned long) _srcv; \
326 (ctxt)->dst.val = (unsigned long) _dstv; \
329 #define emulate_2op_cl(ctxt, _op) \
331 switch ((ctxt)->dst.bytes) { \
333 __emulate_2op_cl(ctxt, _op, "w", u16); \
336 __emulate_2op_cl(ctxt, _op, "l", u32); \
339 ON64(__emulate_2op_cl(ctxt, _op, "q", ulong)); \
344 #define __emulate_1op(ctxt, _op, _suffix) \
346 unsigned long _tmp; \
348 __asm__ __volatile__ ( \
349 _PRE_EFLAGS("0", "3", "2") \
350 _op _suffix " %1; " \
351 _POST_EFLAGS("0", "3", "2") \
352 : "=m" ((ctxt)->eflags), "+m" ((ctxt)->dst.val), \
354 : "i" (EFLAGS_MASK)); \
357 /* Instruction has only one explicit operand (no source operand). */
358 #define emulate_1op(ctxt, _op) \
360 switch ((ctxt)->dst.bytes) { \
361 case 1: __emulate_1op(ctxt, _op, "b"); break; \
362 case 2: __emulate_1op(ctxt, _op, "w"); break; \
363 case 4: __emulate_1op(ctxt, _op, "l"); break; \
364 case 8: ON64(__emulate_1op(ctxt, _op, "q")); break; \
368 #define __emulate_1op_rax_rdx(ctxt, _op, _suffix, _ex) \
370 unsigned long _tmp; \
371 ulong *rax = &(ctxt)->regs[VCPU_REGS_RAX]; \
372 ulong *rdx = &(ctxt)->regs[VCPU_REGS_RDX]; \
374 __asm__ __volatile__ ( \
375 _PRE_EFLAGS("0", "5", "1") \
377 _op _suffix " %6; " \
379 _POST_EFLAGS("0", "5", "1") \
380 ".pushsection .fixup,\"ax\" \n\t" \
381 "3: movb $1, %4 \n\t" \
384 _ASM_EXTABLE(1b, 3b) \
385 : "=m" ((ctxt)->eflags), "=&r" (_tmp), \
386 "+a" (*rax), "+d" (*rdx), "+qm"(_ex) \
387 : "i" (EFLAGS_MASK), "m" ((ctxt)->src.val), \
388 "a" (*rax), "d" (*rdx)); \
391 /* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
392 #define emulate_1op_rax_rdx(ctxt, _op, _ex) \
394 switch((ctxt)->src.bytes) { \
396 __emulate_1op_rax_rdx(ctxt, _op, "b", _ex); \
399 __emulate_1op_rax_rdx(ctxt, _op, "w", _ex); \
402 __emulate_1op_rax_rdx(ctxt, _op, "l", _ex); \
405 __emulate_1op_rax_rdx(ctxt, _op, "q", _ex)); \
410 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
411 enum x86_intercept intercept,
412 enum x86_intercept_stage stage)
414 struct x86_instruction_info info = {
415 .intercept = intercept,
416 .rep_prefix = ctxt->rep_prefix,
417 .modrm_mod = ctxt->modrm_mod,
418 .modrm_reg = ctxt->modrm_reg,
419 .modrm_rm = ctxt->modrm_rm,
420 .src_val = ctxt->src.val64,
421 .src_bytes = ctxt->src.bytes,
422 .dst_bytes = ctxt->dst.bytes,
423 .ad_bytes = ctxt->ad_bytes,
424 .next_rip = ctxt->eip,
427 return ctxt->ops->intercept(ctxt, &info, stage);
430 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
432 return (1UL << (ctxt->ad_bytes << 3)) - 1;
435 /* Access/update address held in a register, based on addressing mode. */
436 static inline unsigned long
437 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
439 if (ctxt->ad_bytes == sizeof(unsigned long))
442 return reg & ad_mask(ctxt);
445 static inline unsigned long
446 register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
448 return address_mask(ctxt, reg);
452 register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
454 if (ctxt->ad_bytes == sizeof(unsigned long))
457 *reg = (*reg & ~ad_mask(ctxt)) | ((*reg + inc) & ad_mask(ctxt));
460 static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
462 register_address_increment(ctxt, &ctxt->_eip, rel);
465 static u32 desc_limit_scaled(struct desc_struct *desc)
467 u32 limit = get_desc_limit(desc);
469 return desc->g ? (limit << 12) | 0xfff : limit;
472 static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
474 ctxt->has_seg_override = true;
475 ctxt->seg_override = seg;
478 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
480 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
483 return ctxt->ops->get_cached_segment_base(ctxt, seg);
486 static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
488 if (!ctxt->has_seg_override)
491 return ctxt->seg_override;
494 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
495 u32 error, bool valid)
497 ctxt->exception.vector = vec;
498 ctxt->exception.error_code = error;
499 ctxt->exception.error_code_valid = valid;
500 return X86EMUL_PROPAGATE_FAULT;
503 static int emulate_db(struct x86_emulate_ctxt *ctxt)
505 return emulate_exception(ctxt, DB_VECTOR, 0, false);
508 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
510 return emulate_exception(ctxt, GP_VECTOR, err, true);
513 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
515 return emulate_exception(ctxt, SS_VECTOR, err, true);
518 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
520 return emulate_exception(ctxt, UD_VECTOR, 0, false);
523 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
525 return emulate_exception(ctxt, TS_VECTOR, err, true);
528 static int emulate_de(struct x86_emulate_ctxt *ctxt)
530 return emulate_exception(ctxt, DE_VECTOR, 0, false);
533 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
535 return emulate_exception(ctxt, NM_VECTOR, 0, false);
538 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
541 struct desc_struct desc;
543 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
547 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
552 struct desc_struct desc;
554 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
555 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
558 static int __linearize(struct x86_emulate_ctxt *ctxt,
559 struct segmented_address addr,
560 unsigned size, bool write, bool fetch,
563 struct desc_struct desc;
570 la = seg_base(ctxt, addr.seg) + addr.ea;
571 switch (ctxt->mode) {
572 case X86EMUL_MODE_REAL:
574 case X86EMUL_MODE_PROT64:
575 if (((signed long)la << 16) >> 16 != la)
576 return emulate_gp(ctxt, 0);
579 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
583 /* code segment or read-only data segment */
584 if (((desc.type & 8) || !(desc.type & 2)) && write)
586 /* unreadable code segment */
587 if (!fetch && (desc.type & 8) && !(desc.type & 2))
589 lim = desc_limit_scaled(&desc);
590 if ((desc.type & 8) || !(desc.type & 4)) {
591 /* expand-up segment */
592 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
595 /* exapand-down segment */
596 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
598 lim = desc.d ? 0xffffffff : 0xffff;
599 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
602 cpl = ctxt->ops->cpl(ctxt);
605 if (!(desc.type & 8)) {
609 } else if ((desc.type & 8) && !(desc.type & 4)) {
610 /* nonconforming code segment */
613 } else if ((desc.type & 8) && (desc.type & 4)) {
614 /* conforming code segment */
620 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
623 return X86EMUL_CONTINUE;
625 if (addr.seg == VCPU_SREG_SS)
626 return emulate_ss(ctxt, addr.seg);
628 return emulate_gp(ctxt, addr.seg);
631 static int linearize(struct x86_emulate_ctxt *ctxt,
632 struct segmented_address addr,
633 unsigned size, bool write,
636 return __linearize(ctxt, addr, size, write, false, linear);
640 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
641 struct segmented_address addr,
648 rc = linearize(ctxt, addr, size, false, &linear);
649 if (rc != X86EMUL_CONTINUE)
651 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
655 * Fetch the next byte of the instruction being emulated which is pointed to
656 * by ctxt->_eip, then increment ctxt->_eip.
658 * Also prefetch the remaining bytes of the instruction without crossing page
659 * boundary if they are not in fetch_cache yet.
661 static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
663 struct fetch_cache *fc = &ctxt->fetch;
667 if (ctxt->_eip == fc->end) {
668 unsigned long linear;
669 struct segmented_address addr = { .seg = VCPU_SREG_CS,
671 cur_size = fc->end - fc->start;
672 size = min(15UL - cur_size,
673 PAGE_SIZE - offset_in_page(ctxt->_eip));
674 rc = __linearize(ctxt, addr, size, false, true, &linear);
675 if (unlikely(rc != X86EMUL_CONTINUE))
677 rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
678 size, &ctxt->exception);
679 if (unlikely(rc != X86EMUL_CONTINUE))
683 *dest = fc->data[ctxt->_eip - fc->start];
685 return X86EMUL_CONTINUE;
688 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
689 void *dest, unsigned size)
693 /* x86 instructions are limited to 15 bytes. */
694 if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
695 return X86EMUL_UNHANDLEABLE;
697 rc = do_insn_fetch_byte(ctxt, dest++);
698 if (rc != X86EMUL_CONTINUE)
701 return X86EMUL_CONTINUE;
704 /* Fetch next part of the instruction being emulated. */
705 #define insn_fetch(_type, _ctxt) \
706 ({ unsigned long _x; \
707 rc = do_insn_fetch(_ctxt, &_x, sizeof(_type)); \
708 if (rc != X86EMUL_CONTINUE) \
713 #define insn_fetch_arr(_arr, _size, _ctxt) \
714 ({ rc = do_insn_fetch(_ctxt, _arr, (_size)); \
715 if (rc != X86EMUL_CONTINUE) \
720 * Given the 'reg' portion of a ModRM byte, and a register block, return a
721 * pointer into the block that addresses the relevant register.
722 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
724 static void *decode_register(u8 modrm_reg, unsigned long *regs,
729 p = ®s[modrm_reg];
730 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
731 p = (unsigned char *)®s[modrm_reg & 3] + 1;
735 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
736 struct segmented_address addr,
737 u16 *size, unsigned long *address, int op_bytes)
744 rc = segmented_read_std(ctxt, addr, size, 2);
745 if (rc != X86EMUL_CONTINUE)
748 rc = segmented_read_std(ctxt, addr, address, op_bytes);
752 static int test_cc(unsigned int condition, unsigned int flags)
756 switch ((condition & 15) >> 1) {
758 rc |= (flags & EFLG_OF);
760 case 1: /* b/c/nae */
761 rc |= (flags & EFLG_CF);
764 rc |= (flags & EFLG_ZF);
767 rc |= (flags & (EFLG_CF|EFLG_ZF));
770 rc |= (flags & EFLG_SF);
773 rc |= (flags & EFLG_PF);
776 rc |= (flags & EFLG_ZF);
779 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
783 /* Odd condition identifiers (lsb == 1) have inverted sense. */
784 return (!!rc ^ (condition & 1));
787 static void fetch_register_operand(struct operand *op)
791 op->val = *(u8 *)op->addr.reg;
794 op->val = *(u16 *)op->addr.reg;
797 op->val = *(u32 *)op->addr.reg;
800 op->val = *(u64 *)op->addr.reg;
805 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
807 ctxt->ops->get_fpu(ctxt);
809 case 0: asm("movdqu %%xmm0, %0" : "=m"(*data)); break;
810 case 1: asm("movdqu %%xmm1, %0" : "=m"(*data)); break;
811 case 2: asm("movdqu %%xmm2, %0" : "=m"(*data)); break;
812 case 3: asm("movdqu %%xmm3, %0" : "=m"(*data)); break;
813 case 4: asm("movdqu %%xmm4, %0" : "=m"(*data)); break;
814 case 5: asm("movdqu %%xmm5, %0" : "=m"(*data)); break;
815 case 6: asm("movdqu %%xmm6, %0" : "=m"(*data)); break;
816 case 7: asm("movdqu %%xmm7, %0" : "=m"(*data)); break;
818 case 8: asm("movdqu %%xmm8, %0" : "=m"(*data)); break;
819 case 9: asm("movdqu %%xmm9, %0" : "=m"(*data)); break;
820 case 10: asm("movdqu %%xmm10, %0" : "=m"(*data)); break;
821 case 11: asm("movdqu %%xmm11, %0" : "=m"(*data)); break;
822 case 12: asm("movdqu %%xmm12, %0" : "=m"(*data)); break;
823 case 13: asm("movdqu %%xmm13, %0" : "=m"(*data)); break;
824 case 14: asm("movdqu %%xmm14, %0" : "=m"(*data)); break;
825 case 15: asm("movdqu %%xmm15, %0" : "=m"(*data)); break;
829 ctxt->ops->put_fpu(ctxt);
832 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
835 ctxt->ops->get_fpu(ctxt);
837 case 0: asm("movdqu %0, %%xmm0" : : "m"(*data)); break;
838 case 1: asm("movdqu %0, %%xmm1" : : "m"(*data)); break;
839 case 2: asm("movdqu %0, %%xmm2" : : "m"(*data)); break;
840 case 3: asm("movdqu %0, %%xmm3" : : "m"(*data)); break;
841 case 4: asm("movdqu %0, %%xmm4" : : "m"(*data)); break;
842 case 5: asm("movdqu %0, %%xmm5" : : "m"(*data)); break;
843 case 6: asm("movdqu %0, %%xmm6" : : "m"(*data)); break;
844 case 7: asm("movdqu %0, %%xmm7" : : "m"(*data)); break;
846 case 8: asm("movdqu %0, %%xmm8" : : "m"(*data)); break;
847 case 9: asm("movdqu %0, %%xmm9" : : "m"(*data)); break;
848 case 10: asm("movdqu %0, %%xmm10" : : "m"(*data)); break;
849 case 11: asm("movdqu %0, %%xmm11" : : "m"(*data)); break;
850 case 12: asm("movdqu %0, %%xmm12" : : "m"(*data)); break;
851 case 13: asm("movdqu %0, %%xmm13" : : "m"(*data)); break;
852 case 14: asm("movdqu %0, %%xmm14" : : "m"(*data)); break;
853 case 15: asm("movdqu %0, %%xmm15" : : "m"(*data)); break;
857 ctxt->ops->put_fpu(ctxt);
860 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
864 unsigned reg = ctxt->modrm_reg;
865 int highbyte_regs = ctxt->rex_prefix == 0;
867 if (!(ctxt->d & ModRM))
868 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
874 read_sse_reg(ctxt, &op->vec_val, reg);
879 if ((ctxt->d & ByteOp) && !inhibit_bytereg) {
880 op->addr.reg = decode_register(reg, ctxt->regs, highbyte_regs);
883 op->addr.reg = decode_register(reg, ctxt->regs, 0);
884 op->bytes = ctxt->op_bytes;
886 fetch_register_operand(op);
887 op->orig_val = op->val;
890 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
894 int index_reg = 0, base_reg = 0, scale;
895 int rc = X86EMUL_CONTINUE;
898 if (ctxt->rex_prefix) {
899 ctxt->modrm_reg = (ctxt->rex_prefix & 4) << 1; /* REX.R */
900 index_reg = (ctxt->rex_prefix & 2) << 2; /* REX.X */
901 ctxt->modrm_rm = base_reg = (ctxt->rex_prefix & 1) << 3; /* REG.B */
904 ctxt->modrm = insn_fetch(u8, ctxt);
905 ctxt->modrm_mod |= (ctxt->modrm & 0xc0) >> 6;
906 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
907 ctxt->modrm_rm |= (ctxt->modrm & 0x07);
908 ctxt->modrm_seg = VCPU_SREG_DS;
910 if (ctxt->modrm_mod == 3) {
912 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
913 op->addr.reg = decode_register(ctxt->modrm_rm,
914 ctxt->regs, ctxt->d & ByteOp);
918 op->addr.xmm = ctxt->modrm_rm;
919 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
922 fetch_register_operand(op);
928 if (ctxt->ad_bytes == 2) {
929 unsigned bx = ctxt->regs[VCPU_REGS_RBX];
930 unsigned bp = ctxt->regs[VCPU_REGS_RBP];
931 unsigned si = ctxt->regs[VCPU_REGS_RSI];
932 unsigned di = ctxt->regs[VCPU_REGS_RDI];
934 /* 16-bit ModR/M decode. */
935 switch (ctxt->modrm_mod) {
937 if (ctxt->modrm_rm == 6)
938 modrm_ea += insn_fetch(u16, ctxt);
941 modrm_ea += insn_fetch(s8, ctxt);
944 modrm_ea += insn_fetch(u16, ctxt);
947 switch (ctxt->modrm_rm) {
967 if (ctxt->modrm_mod != 0)
974 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
975 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
976 ctxt->modrm_seg = VCPU_SREG_SS;
977 modrm_ea = (u16)modrm_ea;
979 /* 32/64-bit ModR/M decode. */
980 if ((ctxt->modrm_rm & 7) == 4) {
981 sib = insn_fetch(u8, ctxt);
982 index_reg |= (sib >> 3) & 7;
986 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
987 modrm_ea += insn_fetch(s32, ctxt);
989 modrm_ea += ctxt->regs[base_reg];
991 modrm_ea += ctxt->regs[index_reg] << scale;
992 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
993 if (ctxt->mode == X86EMUL_MODE_PROT64)
994 ctxt->rip_relative = 1;
996 modrm_ea += ctxt->regs[ctxt->modrm_rm];
997 switch (ctxt->modrm_mod) {
999 if (ctxt->modrm_rm == 5)
1000 modrm_ea += insn_fetch(s32, ctxt);
1003 modrm_ea += insn_fetch(s8, ctxt);
1006 modrm_ea += insn_fetch(s32, ctxt);
1010 op->addr.mem.ea = modrm_ea;
1015 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1018 int rc = X86EMUL_CONTINUE;
1021 switch (ctxt->ad_bytes) {
1023 op->addr.mem.ea = insn_fetch(u16, ctxt);
1026 op->addr.mem.ea = insn_fetch(u32, ctxt);
1029 op->addr.mem.ea = insn_fetch(u64, ctxt);
1036 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1040 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1041 mask = ~(ctxt->dst.bytes * 8 - 1);
1043 if (ctxt->src.bytes == 2)
1044 sv = (s16)ctxt->src.val & (s16)mask;
1045 else if (ctxt->src.bytes == 4)
1046 sv = (s32)ctxt->src.val & (s32)mask;
1048 ctxt->dst.addr.mem.ea += (sv >> 3);
1051 /* only subword offset */
1052 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1055 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1056 unsigned long addr, void *dest, unsigned size)
1059 struct read_cache *mc = &ctxt->mem_read;
1062 int n = min(size, 8u);
1064 if (mc->pos < mc->end)
1067 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, n,
1069 if (rc != X86EMUL_CONTINUE)
1074 memcpy(dest, mc->data + mc->pos, n);
1079 return X86EMUL_CONTINUE;
1082 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1083 struct segmented_address addr,
1090 rc = linearize(ctxt, addr, size, false, &linear);
1091 if (rc != X86EMUL_CONTINUE)
1093 return read_emulated(ctxt, linear, data, size);
1096 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1097 struct segmented_address addr,
1104 rc = linearize(ctxt, addr, size, true, &linear);
1105 if (rc != X86EMUL_CONTINUE)
1107 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1111 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1112 struct segmented_address addr,
1113 const void *orig_data, const void *data,
1119 rc = linearize(ctxt, addr, size, true, &linear);
1120 if (rc != X86EMUL_CONTINUE)
1122 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1123 size, &ctxt->exception);
1126 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1127 unsigned int size, unsigned short port,
1130 struct read_cache *rc = &ctxt->io_read;
1132 if (rc->pos == rc->end) { /* refill pio read ahead */
1133 unsigned int in_page, n;
1134 unsigned int count = ctxt->rep_prefix ?
1135 address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) : 1;
1136 in_page = (ctxt->eflags & EFLG_DF) ?
1137 offset_in_page(ctxt->regs[VCPU_REGS_RDI]) :
1138 PAGE_SIZE - offset_in_page(ctxt->regs[VCPU_REGS_RDI]);
1139 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1143 rc->pos = rc->end = 0;
1144 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1149 memcpy(dest, rc->data + rc->pos, size);
1154 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1155 u16 selector, struct desc_ptr *dt)
1157 struct x86_emulate_ops *ops = ctxt->ops;
1159 if (selector & 1 << 2) {
1160 struct desc_struct desc;
1163 memset (dt, 0, sizeof *dt);
1164 if (!ops->get_segment(ctxt, &sel, &desc, NULL, VCPU_SREG_LDTR))
1167 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1168 dt->address = get_desc_base(&desc);
1170 ops->get_gdt(ctxt, dt);
1173 /* allowed just for 8 bytes segments */
1174 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1175 u16 selector, struct desc_struct *desc)
1178 u16 index = selector >> 3;
1181 get_descriptor_table_ptr(ctxt, selector, &dt);
1183 if (dt.size < index * 8 + 7)
1184 return emulate_gp(ctxt, selector & 0xfffc);
1186 addr = dt.address + index * 8;
1187 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1191 /* allowed just for 8 bytes segments */
1192 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1193 u16 selector, struct desc_struct *desc)
1196 u16 index = selector >> 3;
1199 get_descriptor_table_ptr(ctxt, selector, &dt);
1201 if (dt.size < index * 8 + 7)
1202 return emulate_gp(ctxt, selector & 0xfffc);
1204 addr = dt.address + index * 8;
1205 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1209 /* Does not support long mode */
1210 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1211 u16 selector, int seg)
1213 struct desc_struct seg_desc;
1215 unsigned err_vec = GP_VECTOR;
1217 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1220 memset(&seg_desc, 0, sizeof seg_desc);
1222 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1223 || ctxt->mode == X86EMUL_MODE_REAL) {
1224 /* set real mode segment descriptor */
1225 set_desc_base(&seg_desc, selector << 4);
1226 set_desc_limit(&seg_desc, 0xffff);
1233 /* NULL selector is not valid for TR, CS and SS */
1234 if ((seg == VCPU_SREG_CS || seg == VCPU_SREG_SS || seg == VCPU_SREG_TR)
1238 /* TR should be in GDT only */
1239 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1242 if (null_selector) /* for NULL selector skip all following checks */
1245 ret = read_segment_descriptor(ctxt, selector, &seg_desc);
1246 if (ret != X86EMUL_CONTINUE)
1249 err_code = selector & 0xfffc;
1250 err_vec = GP_VECTOR;
1252 /* can't load system descriptor into segment selecor */
1253 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1257 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1263 cpl = ctxt->ops->cpl(ctxt);
1268 * segment is not a writable data segment or segment
1269 * selector's RPL != CPL or segment selector's RPL != CPL
1271 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1275 if (!(seg_desc.type & 8))
1278 if (seg_desc.type & 4) {
1284 if (rpl > cpl || dpl != cpl)
1287 /* CS(RPL) <- CPL */
1288 selector = (selector & 0xfffc) | cpl;
1291 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1294 case VCPU_SREG_LDTR:
1295 if (seg_desc.s || seg_desc.type != 2)
1298 default: /* DS, ES, FS, or GS */
1300 * segment is not a data or readable code segment or
1301 * ((segment is a data or nonconforming code segment)
1302 * and (both RPL and CPL > DPL))
1304 if ((seg_desc.type & 0xa) == 0x8 ||
1305 (((seg_desc.type & 0xc) != 0xc) &&
1306 (rpl > dpl && cpl > dpl)))
1312 /* mark segment as accessed */
1314 ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1315 if (ret != X86EMUL_CONTINUE)
1319 ctxt->ops->set_segment(ctxt, selector, &seg_desc, 0, seg);
1320 return X86EMUL_CONTINUE;
1322 emulate_exception(ctxt, err_vec, err_code, true);
1323 return X86EMUL_PROPAGATE_FAULT;
1326 static void write_register_operand(struct operand *op)
1328 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1329 switch (op->bytes) {
1331 *(u8 *)op->addr.reg = (u8)op->val;
1334 *(u16 *)op->addr.reg = (u16)op->val;
1337 *op->addr.reg = (u32)op->val;
1338 break; /* 64b: zero-extend */
1340 *op->addr.reg = op->val;
1345 static int writeback(struct x86_emulate_ctxt *ctxt)
1349 switch (ctxt->dst.type) {
1351 write_register_operand(&ctxt->dst);
1354 if (ctxt->lock_prefix)
1355 rc = segmented_cmpxchg(ctxt,
1357 &ctxt->dst.orig_val,
1361 rc = segmented_write(ctxt,
1365 if (rc != X86EMUL_CONTINUE)
1369 write_sse_reg(ctxt, &ctxt->dst.vec_val, ctxt->dst.addr.xmm);
1377 return X86EMUL_CONTINUE;
1380 static int em_push(struct x86_emulate_ctxt *ctxt)
1382 struct segmented_address addr;
1384 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], -ctxt->op_bytes);
1385 addr.ea = register_address(ctxt, ctxt->regs[VCPU_REGS_RSP]);
1386 addr.seg = VCPU_SREG_SS;
1388 /* Disable writeback. */
1389 ctxt->dst.type = OP_NONE;
1390 return segmented_write(ctxt, addr, &ctxt->src.val, ctxt->op_bytes);
1393 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1394 void *dest, int len)
1397 struct segmented_address addr;
1399 addr.ea = register_address(ctxt, ctxt->regs[VCPU_REGS_RSP]);
1400 addr.seg = VCPU_SREG_SS;
1401 rc = segmented_read(ctxt, addr, dest, len);
1402 if (rc != X86EMUL_CONTINUE)
1405 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], len);
1409 static int em_pop(struct x86_emulate_ctxt *ctxt)
1411 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1414 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1415 void *dest, int len)
1418 unsigned long val, change_mask;
1419 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1420 int cpl = ctxt->ops->cpl(ctxt);
1422 rc = emulate_pop(ctxt, &val, len);
1423 if (rc != X86EMUL_CONTINUE)
1426 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1427 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1429 switch(ctxt->mode) {
1430 case X86EMUL_MODE_PROT64:
1431 case X86EMUL_MODE_PROT32:
1432 case X86EMUL_MODE_PROT16:
1434 change_mask |= EFLG_IOPL;
1436 change_mask |= EFLG_IF;
1438 case X86EMUL_MODE_VM86:
1440 return emulate_gp(ctxt, 0);
1441 change_mask |= EFLG_IF;
1443 default: /* real mode */
1444 change_mask |= (EFLG_IOPL | EFLG_IF);
1448 *(unsigned long *)dest =
1449 (ctxt->eflags & ~change_mask) | (val & change_mask);
1454 static int em_popf(struct x86_emulate_ctxt *ctxt)
1456 ctxt->dst.type = OP_REG;
1457 ctxt->dst.addr.reg = &ctxt->eflags;
1458 ctxt->dst.bytes = ctxt->op_bytes;
1459 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1462 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1464 int seg = ctxt->src2.val;
1466 ctxt->src.val = get_segment_selector(ctxt, seg);
1468 return em_push(ctxt);
1471 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1473 int seg = ctxt->src2.val;
1474 unsigned long selector;
1477 rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1478 if (rc != X86EMUL_CONTINUE)
1481 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1485 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1487 unsigned long old_esp = ctxt->regs[VCPU_REGS_RSP];
1488 int rc = X86EMUL_CONTINUE;
1489 int reg = VCPU_REGS_RAX;
1491 while (reg <= VCPU_REGS_RDI) {
1492 (reg == VCPU_REGS_RSP) ?
1493 (ctxt->src.val = old_esp) : (ctxt->src.val = ctxt->regs[reg]);
1496 if (rc != X86EMUL_CONTINUE)
1505 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1507 ctxt->src.val = (unsigned long)ctxt->eflags;
1508 return em_push(ctxt);
1511 static int em_popa(struct x86_emulate_ctxt *ctxt)
1513 int rc = X86EMUL_CONTINUE;
1514 int reg = VCPU_REGS_RDI;
1516 while (reg >= VCPU_REGS_RAX) {
1517 if (reg == VCPU_REGS_RSP) {
1518 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP],
1523 rc = emulate_pop(ctxt, &ctxt->regs[reg], ctxt->op_bytes);
1524 if (rc != X86EMUL_CONTINUE)
1531 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1533 struct x86_emulate_ops *ops = ctxt->ops;
1540 /* TODO: Add limit checks */
1541 ctxt->src.val = ctxt->eflags;
1543 if (rc != X86EMUL_CONTINUE)
1546 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1548 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1550 if (rc != X86EMUL_CONTINUE)
1553 ctxt->src.val = ctxt->_eip;
1555 if (rc != X86EMUL_CONTINUE)
1558 ops->get_idt(ctxt, &dt);
1560 eip_addr = dt.address + (irq << 2);
1561 cs_addr = dt.address + (irq << 2) + 2;
1563 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1564 if (rc != X86EMUL_CONTINUE)
1567 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1568 if (rc != X86EMUL_CONTINUE)
1571 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1572 if (rc != X86EMUL_CONTINUE)
1580 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1582 switch(ctxt->mode) {
1583 case X86EMUL_MODE_REAL:
1584 return emulate_int_real(ctxt, irq);
1585 case X86EMUL_MODE_VM86:
1586 case X86EMUL_MODE_PROT16:
1587 case X86EMUL_MODE_PROT32:
1588 case X86EMUL_MODE_PROT64:
1590 /* Protected mode interrupts unimplemented yet */
1591 return X86EMUL_UNHANDLEABLE;
1595 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
1597 int rc = X86EMUL_CONTINUE;
1598 unsigned long temp_eip = 0;
1599 unsigned long temp_eflags = 0;
1600 unsigned long cs = 0;
1601 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1602 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1603 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1604 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1606 /* TODO: Add stack limit check */
1608 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
1610 if (rc != X86EMUL_CONTINUE)
1613 if (temp_eip & ~0xffff)
1614 return emulate_gp(ctxt, 0);
1616 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1618 if (rc != X86EMUL_CONTINUE)
1621 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
1623 if (rc != X86EMUL_CONTINUE)
1626 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1628 if (rc != X86EMUL_CONTINUE)
1631 ctxt->_eip = temp_eip;
1634 if (ctxt->op_bytes == 4)
1635 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1636 else if (ctxt->op_bytes == 2) {
1637 ctxt->eflags &= ~0xffff;
1638 ctxt->eflags |= temp_eflags;
1641 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1642 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1647 static int em_iret(struct x86_emulate_ctxt *ctxt)
1649 switch(ctxt->mode) {
1650 case X86EMUL_MODE_REAL:
1651 return emulate_iret_real(ctxt);
1652 case X86EMUL_MODE_VM86:
1653 case X86EMUL_MODE_PROT16:
1654 case X86EMUL_MODE_PROT32:
1655 case X86EMUL_MODE_PROT64:
1657 /* iret from protected mode unimplemented yet */
1658 return X86EMUL_UNHANDLEABLE;
1662 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
1667 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1669 rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
1670 if (rc != X86EMUL_CONTINUE)
1674 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
1675 return X86EMUL_CONTINUE;
1678 static int em_grp2(struct x86_emulate_ctxt *ctxt)
1680 switch (ctxt->modrm_reg) {
1682 emulate_2op_SrcB(ctxt, "rol");
1685 emulate_2op_SrcB(ctxt, "ror");
1688 emulate_2op_SrcB(ctxt, "rcl");
1691 emulate_2op_SrcB(ctxt, "rcr");
1693 case 4: /* sal/shl */
1694 case 6: /* sal/shl */
1695 emulate_2op_SrcB(ctxt, "sal");
1698 emulate_2op_SrcB(ctxt, "shr");
1701 emulate_2op_SrcB(ctxt, "sar");
1704 return X86EMUL_CONTINUE;
1707 static int em_not(struct x86_emulate_ctxt *ctxt)
1709 ctxt->dst.val = ~ctxt->dst.val;
1710 return X86EMUL_CONTINUE;
1713 static int em_neg(struct x86_emulate_ctxt *ctxt)
1715 emulate_1op(ctxt, "neg");
1716 return X86EMUL_CONTINUE;
1719 static int em_mul_ex(struct x86_emulate_ctxt *ctxt)
1723 emulate_1op_rax_rdx(ctxt, "mul", ex);
1724 return X86EMUL_CONTINUE;
1727 static int em_imul_ex(struct x86_emulate_ctxt *ctxt)
1731 emulate_1op_rax_rdx(ctxt, "imul", ex);
1732 return X86EMUL_CONTINUE;
1735 static int em_div_ex(struct x86_emulate_ctxt *ctxt)
1739 emulate_1op_rax_rdx(ctxt, "div", de);
1741 return emulate_de(ctxt);
1742 return X86EMUL_CONTINUE;
1745 static int em_idiv_ex(struct x86_emulate_ctxt *ctxt)
1749 emulate_1op_rax_rdx(ctxt, "idiv", de);
1751 return emulate_de(ctxt);
1752 return X86EMUL_CONTINUE;
1755 static int em_grp45(struct x86_emulate_ctxt *ctxt)
1757 int rc = X86EMUL_CONTINUE;
1759 switch (ctxt->modrm_reg) {
1761 emulate_1op(ctxt, "inc");
1764 emulate_1op(ctxt, "dec");
1766 case 2: /* call near abs */ {
1768 old_eip = ctxt->_eip;
1769 ctxt->_eip = ctxt->src.val;
1770 ctxt->src.val = old_eip;
1774 case 4: /* jmp abs */
1775 ctxt->_eip = ctxt->src.val;
1777 case 5: /* jmp far */
1778 rc = em_jmp_far(ctxt);
1787 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
1789 u64 old = ctxt->dst.orig_val64;
1791 if (((u32) (old >> 0) != (u32) ctxt->regs[VCPU_REGS_RAX]) ||
1792 ((u32) (old >> 32) != (u32) ctxt->regs[VCPU_REGS_RDX])) {
1793 ctxt->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1794 ctxt->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1795 ctxt->eflags &= ~EFLG_ZF;
1797 ctxt->dst.val64 = ((u64)ctxt->regs[VCPU_REGS_RCX] << 32) |
1798 (u32) ctxt->regs[VCPU_REGS_RBX];
1800 ctxt->eflags |= EFLG_ZF;
1802 return X86EMUL_CONTINUE;
1805 static int em_ret(struct x86_emulate_ctxt *ctxt)
1807 ctxt->dst.type = OP_REG;
1808 ctxt->dst.addr.reg = &ctxt->_eip;
1809 ctxt->dst.bytes = ctxt->op_bytes;
1810 return em_pop(ctxt);
1813 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
1818 rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
1819 if (rc != X86EMUL_CONTINUE)
1821 if (ctxt->op_bytes == 4)
1822 ctxt->_eip = (u32)ctxt->_eip;
1823 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1824 if (rc != X86EMUL_CONTINUE)
1826 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1830 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
1832 /* Save real source value, then compare EAX against destination. */
1833 ctxt->src.orig_val = ctxt->src.val;
1834 ctxt->src.val = ctxt->regs[VCPU_REGS_RAX];
1835 emulate_2op_SrcV(ctxt, "cmp");
1837 if (ctxt->eflags & EFLG_ZF) {
1838 /* Success: write back to memory. */
1839 ctxt->dst.val = ctxt->src.orig_val;
1841 /* Failure: write the value we saw to EAX. */
1842 ctxt->dst.type = OP_REG;
1843 ctxt->dst.addr.reg = (unsigned long *)&ctxt->regs[VCPU_REGS_RAX];
1845 return X86EMUL_CONTINUE;
1848 static int em_lseg(struct x86_emulate_ctxt *ctxt)
1850 int seg = ctxt->src2.val;
1854 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1856 rc = load_segment_descriptor(ctxt, sel, seg);
1857 if (rc != X86EMUL_CONTINUE)
1860 ctxt->dst.val = ctxt->src.val;
1865 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1866 struct desc_struct *cs, struct desc_struct *ss)
1870 memset(cs, 0, sizeof(struct desc_struct));
1871 ctxt->ops->get_segment(ctxt, &selector, cs, NULL, VCPU_SREG_CS);
1872 memset(ss, 0, sizeof(struct desc_struct));
1874 cs->l = 0; /* will be adjusted later */
1875 set_desc_base(cs, 0); /* flat segment */
1876 cs->g = 1; /* 4kb granularity */
1877 set_desc_limit(cs, 0xfffff); /* 4GB limit */
1878 cs->type = 0x0b; /* Read, Execute, Accessed */
1880 cs->dpl = 0; /* will be adjusted later */
1884 set_desc_base(ss, 0); /* flat segment */
1885 set_desc_limit(ss, 0xfffff); /* 4GB limit */
1886 ss->g = 1; /* 4kb granularity */
1888 ss->type = 0x03; /* Read/Write, Accessed */
1889 ss->d = 1; /* 32bit stack segment */
1894 static int em_syscall(struct x86_emulate_ctxt *ctxt)
1896 struct x86_emulate_ops *ops = ctxt->ops;
1897 struct desc_struct cs, ss;
1902 /* syscall is not available in real mode */
1903 if (ctxt->mode == X86EMUL_MODE_REAL ||
1904 ctxt->mode == X86EMUL_MODE_VM86)
1905 return emulate_ud(ctxt);
1907 ops->get_msr(ctxt, MSR_EFER, &efer);
1908 setup_syscalls_segments(ctxt, &cs, &ss);
1910 ops->get_msr(ctxt, MSR_STAR, &msr_data);
1912 cs_sel = (u16)(msr_data & 0xfffc);
1913 ss_sel = (u16)(msr_data + 8);
1915 if (efer & EFER_LMA) {
1919 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
1920 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
1922 ctxt->regs[VCPU_REGS_RCX] = ctxt->_eip;
1923 if (efer & EFER_LMA) {
1924 #ifdef CONFIG_X86_64
1925 ctxt->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
1928 ctxt->mode == X86EMUL_MODE_PROT64 ?
1929 MSR_LSTAR : MSR_CSTAR, &msr_data);
1930 ctxt->_eip = msr_data;
1932 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
1933 ctxt->eflags &= ~(msr_data | EFLG_RF);
1937 ops->get_msr(ctxt, MSR_STAR, &msr_data);
1938 ctxt->_eip = (u32)msr_data;
1940 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1943 return X86EMUL_CONTINUE;
1946 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
1948 struct x86_emulate_ops *ops = ctxt->ops;
1949 struct desc_struct cs, ss;
1954 ops->get_msr(ctxt, MSR_EFER, &efer);
1955 /* inject #GP if in real mode */
1956 if (ctxt->mode == X86EMUL_MODE_REAL)
1957 return emulate_gp(ctxt, 0);
1959 /* XXX sysenter/sysexit have not been tested in 64bit mode.
1960 * Therefore, we inject an #UD.
1962 if (ctxt->mode == X86EMUL_MODE_PROT64)
1963 return emulate_ud(ctxt);
1965 setup_syscalls_segments(ctxt, &cs, &ss);
1967 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
1968 switch (ctxt->mode) {
1969 case X86EMUL_MODE_PROT32:
1970 if ((msr_data & 0xfffc) == 0x0)
1971 return emulate_gp(ctxt, 0);
1973 case X86EMUL_MODE_PROT64:
1974 if (msr_data == 0x0)
1975 return emulate_gp(ctxt, 0);
1979 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1980 cs_sel = (u16)msr_data;
1981 cs_sel &= ~SELECTOR_RPL_MASK;
1982 ss_sel = cs_sel + 8;
1983 ss_sel &= ~SELECTOR_RPL_MASK;
1984 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
1989 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
1990 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
1992 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
1993 ctxt->_eip = msr_data;
1995 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
1996 ctxt->regs[VCPU_REGS_RSP] = msr_data;
1998 return X86EMUL_CONTINUE;
2001 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2003 struct x86_emulate_ops *ops = ctxt->ops;
2004 struct desc_struct cs, ss;
2007 u16 cs_sel = 0, ss_sel = 0;
2009 /* inject #GP if in real mode or Virtual 8086 mode */
2010 if (ctxt->mode == X86EMUL_MODE_REAL ||
2011 ctxt->mode == X86EMUL_MODE_VM86)
2012 return emulate_gp(ctxt, 0);
2014 setup_syscalls_segments(ctxt, &cs, &ss);
2016 if ((ctxt->rex_prefix & 0x8) != 0x0)
2017 usermode = X86EMUL_MODE_PROT64;
2019 usermode = X86EMUL_MODE_PROT32;
2023 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2025 case X86EMUL_MODE_PROT32:
2026 cs_sel = (u16)(msr_data + 16);
2027 if ((msr_data & 0xfffc) == 0x0)
2028 return emulate_gp(ctxt, 0);
2029 ss_sel = (u16)(msr_data + 24);
2031 case X86EMUL_MODE_PROT64:
2032 cs_sel = (u16)(msr_data + 32);
2033 if (msr_data == 0x0)
2034 return emulate_gp(ctxt, 0);
2035 ss_sel = cs_sel + 8;
2040 cs_sel |= SELECTOR_RPL_MASK;
2041 ss_sel |= SELECTOR_RPL_MASK;
2043 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2044 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2046 ctxt->_eip = ctxt->regs[VCPU_REGS_RDX];
2047 ctxt->regs[VCPU_REGS_RSP] = ctxt->regs[VCPU_REGS_RCX];
2049 return X86EMUL_CONTINUE;
2052 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2055 if (ctxt->mode == X86EMUL_MODE_REAL)
2057 if (ctxt->mode == X86EMUL_MODE_VM86)
2059 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2060 return ctxt->ops->cpl(ctxt) > iopl;
2063 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2066 struct x86_emulate_ops *ops = ctxt->ops;
2067 struct desc_struct tr_seg;
2070 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2071 unsigned mask = (1 << len) - 1;
2074 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2077 if (desc_limit_scaled(&tr_seg) < 103)
2079 base = get_desc_base(&tr_seg);
2080 #ifdef CONFIG_X86_64
2081 base |= ((u64)base3) << 32;
2083 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2084 if (r != X86EMUL_CONTINUE)
2086 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2088 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2089 if (r != X86EMUL_CONTINUE)
2091 if ((perm >> bit_idx) & mask)
2096 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2102 if (emulator_bad_iopl(ctxt))
2103 if (!emulator_io_port_access_allowed(ctxt, port, len))
2106 ctxt->perm_ok = true;
2111 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2112 struct tss_segment_16 *tss)
2114 tss->ip = ctxt->_eip;
2115 tss->flag = ctxt->eflags;
2116 tss->ax = ctxt->regs[VCPU_REGS_RAX];
2117 tss->cx = ctxt->regs[VCPU_REGS_RCX];
2118 tss->dx = ctxt->regs[VCPU_REGS_RDX];
2119 tss->bx = ctxt->regs[VCPU_REGS_RBX];
2120 tss->sp = ctxt->regs[VCPU_REGS_RSP];
2121 tss->bp = ctxt->regs[VCPU_REGS_RBP];
2122 tss->si = ctxt->regs[VCPU_REGS_RSI];
2123 tss->di = ctxt->regs[VCPU_REGS_RDI];
2125 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2126 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2127 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2128 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2129 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2132 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2133 struct tss_segment_16 *tss)
2137 ctxt->_eip = tss->ip;
2138 ctxt->eflags = tss->flag | 2;
2139 ctxt->regs[VCPU_REGS_RAX] = tss->ax;
2140 ctxt->regs[VCPU_REGS_RCX] = tss->cx;
2141 ctxt->regs[VCPU_REGS_RDX] = tss->dx;
2142 ctxt->regs[VCPU_REGS_RBX] = tss->bx;
2143 ctxt->regs[VCPU_REGS_RSP] = tss->sp;
2144 ctxt->regs[VCPU_REGS_RBP] = tss->bp;
2145 ctxt->regs[VCPU_REGS_RSI] = tss->si;
2146 ctxt->regs[VCPU_REGS_RDI] = tss->di;
2149 * SDM says that segment selectors are loaded before segment
2152 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2153 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2154 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2155 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2156 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2159 * Now load segment descriptors. If fault happenes at this stage
2160 * it is handled in a context of new task
2162 ret = load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR);
2163 if (ret != X86EMUL_CONTINUE)
2165 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2166 if (ret != X86EMUL_CONTINUE)
2168 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2169 if (ret != X86EMUL_CONTINUE)
2171 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2172 if (ret != X86EMUL_CONTINUE)
2174 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2175 if (ret != X86EMUL_CONTINUE)
2178 return X86EMUL_CONTINUE;
2181 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2182 u16 tss_selector, u16 old_tss_sel,
2183 ulong old_tss_base, struct desc_struct *new_desc)
2185 struct x86_emulate_ops *ops = ctxt->ops;
2186 struct tss_segment_16 tss_seg;
2188 u32 new_tss_base = get_desc_base(new_desc);
2190 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2192 if (ret != X86EMUL_CONTINUE)
2193 /* FIXME: need to provide precise fault address */
2196 save_state_to_tss16(ctxt, &tss_seg);
2198 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2200 if (ret != X86EMUL_CONTINUE)
2201 /* FIXME: need to provide precise fault address */
2204 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2206 if (ret != X86EMUL_CONTINUE)
2207 /* FIXME: need to provide precise fault address */
2210 if (old_tss_sel != 0xffff) {
2211 tss_seg.prev_task_link = old_tss_sel;
2213 ret = ops->write_std(ctxt, new_tss_base,
2214 &tss_seg.prev_task_link,
2215 sizeof tss_seg.prev_task_link,
2217 if (ret != X86EMUL_CONTINUE)
2218 /* FIXME: need to provide precise fault address */
2222 return load_state_from_tss16(ctxt, &tss_seg);
2225 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2226 struct tss_segment_32 *tss)
2228 tss->cr3 = ctxt->ops->get_cr(ctxt, 3);
2229 tss->eip = ctxt->_eip;
2230 tss->eflags = ctxt->eflags;
2231 tss->eax = ctxt->regs[VCPU_REGS_RAX];
2232 tss->ecx = ctxt->regs[VCPU_REGS_RCX];
2233 tss->edx = ctxt->regs[VCPU_REGS_RDX];
2234 tss->ebx = ctxt->regs[VCPU_REGS_RBX];
2235 tss->esp = ctxt->regs[VCPU_REGS_RSP];
2236 tss->ebp = ctxt->regs[VCPU_REGS_RBP];
2237 tss->esi = ctxt->regs[VCPU_REGS_RSI];
2238 tss->edi = ctxt->regs[VCPU_REGS_RDI];
2240 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2241 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2242 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2243 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2244 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2245 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2246 tss->ldt_selector = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2249 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2250 struct tss_segment_32 *tss)
2254 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2255 return emulate_gp(ctxt, 0);
2256 ctxt->_eip = tss->eip;
2257 ctxt->eflags = tss->eflags | 2;
2258 ctxt->regs[VCPU_REGS_RAX] = tss->eax;
2259 ctxt->regs[VCPU_REGS_RCX] = tss->ecx;
2260 ctxt->regs[VCPU_REGS_RDX] = tss->edx;
2261 ctxt->regs[VCPU_REGS_RBX] = tss->ebx;
2262 ctxt->regs[VCPU_REGS_RSP] = tss->esp;
2263 ctxt->regs[VCPU_REGS_RBP] = tss->ebp;
2264 ctxt->regs[VCPU_REGS_RSI] = tss->esi;
2265 ctxt->regs[VCPU_REGS_RDI] = tss->edi;
2268 * SDM says that segment selectors are loaded before segment
2271 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2272 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2273 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2274 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2275 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2276 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2277 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2280 * Now load segment descriptors. If fault happenes at this stage
2281 * it is handled in a context of new task
2283 ret = load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2284 if (ret != X86EMUL_CONTINUE)
2286 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2287 if (ret != X86EMUL_CONTINUE)
2289 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2290 if (ret != X86EMUL_CONTINUE)
2292 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2293 if (ret != X86EMUL_CONTINUE)
2295 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2296 if (ret != X86EMUL_CONTINUE)
2298 ret = load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS);
2299 if (ret != X86EMUL_CONTINUE)
2301 ret = load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS);
2302 if (ret != X86EMUL_CONTINUE)
2305 return X86EMUL_CONTINUE;
2308 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2309 u16 tss_selector, u16 old_tss_sel,
2310 ulong old_tss_base, struct desc_struct *new_desc)
2312 struct x86_emulate_ops *ops = ctxt->ops;
2313 struct tss_segment_32 tss_seg;
2315 u32 new_tss_base = get_desc_base(new_desc);
2317 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2319 if (ret != X86EMUL_CONTINUE)
2320 /* FIXME: need to provide precise fault address */
2323 save_state_to_tss32(ctxt, &tss_seg);
2325 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2327 if (ret != X86EMUL_CONTINUE)
2328 /* FIXME: need to provide precise fault address */
2331 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2333 if (ret != X86EMUL_CONTINUE)
2334 /* FIXME: need to provide precise fault address */
2337 if (old_tss_sel != 0xffff) {
2338 tss_seg.prev_task_link = old_tss_sel;
2340 ret = ops->write_std(ctxt, new_tss_base,
2341 &tss_seg.prev_task_link,
2342 sizeof tss_seg.prev_task_link,
2344 if (ret != X86EMUL_CONTINUE)
2345 /* FIXME: need to provide precise fault address */
2349 return load_state_from_tss32(ctxt, &tss_seg);
2352 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2353 u16 tss_selector, int reason,
2354 bool has_error_code, u32 error_code)
2356 struct x86_emulate_ops *ops = ctxt->ops;
2357 struct desc_struct curr_tss_desc, next_tss_desc;
2359 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2360 ulong old_tss_base =
2361 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2364 /* FIXME: old_tss_base == ~0 ? */
2366 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2367 if (ret != X86EMUL_CONTINUE)
2369 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2370 if (ret != X86EMUL_CONTINUE)
2373 /* FIXME: check that next_tss_desc is tss */
2375 if (reason != TASK_SWITCH_IRET) {
2376 if ((tss_selector & 3) > next_tss_desc.dpl ||
2377 ops->cpl(ctxt) > next_tss_desc.dpl)
2378 return emulate_gp(ctxt, 0);
2381 desc_limit = desc_limit_scaled(&next_tss_desc);
2382 if (!next_tss_desc.p ||
2383 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2384 desc_limit < 0x2b)) {
2385 emulate_ts(ctxt, tss_selector & 0xfffc);
2386 return X86EMUL_PROPAGATE_FAULT;
2389 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2390 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2391 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2394 if (reason == TASK_SWITCH_IRET)
2395 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2397 /* set back link to prev task only if NT bit is set in eflags
2398 note that old_tss_sel is not used afetr this point */
2399 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2400 old_tss_sel = 0xffff;
2402 if (next_tss_desc.type & 8)
2403 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2404 old_tss_base, &next_tss_desc);
2406 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2407 old_tss_base, &next_tss_desc);
2408 if (ret != X86EMUL_CONTINUE)
2411 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2412 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2414 if (reason != TASK_SWITCH_IRET) {
2415 next_tss_desc.type |= (1 << 1); /* set busy flag */
2416 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2419 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2420 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2422 if (has_error_code) {
2423 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2424 ctxt->lock_prefix = 0;
2425 ctxt->src.val = (unsigned long) error_code;
2426 ret = em_push(ctxt);
2432 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2433 u16 tss_selector, int reason,
2434 bool has_error_code, u32 error_code)
2438 ctxt->_eip = ctxt->eip;
2439 ctxt->dst.type = OP_NONE;
2441 rc = emulator_do_task_switch(ctxt, tss_selector, reason,
2442 has_error_code, error_code);
2444 if (rc == X86EMUL_CONTINUE)
2445 ctxt->eip = ctxt->_eip;
2447 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2450 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned seg,
2451 int reg, struct operand *op)
2453 int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;
2455 register_address_increment(ctxt, &ctxt->regs[reg], df * op->bytes);
2456 op->addr.mem.ea = register_address(ctxt, ctxt->regs[reg]);
2457 op->addr.mem.seg = seg;
2460 static int em_das(struct x86_emulate_ctxt *ctxt)
2463 bool af, cf, old_cf;
2465 cf = ctxt->eflags & X86_EFLAGS_CF;
2471 af = ctxt->eflags & X86_EFLAGS_AF;
2472 if ((al & 0x0f) > 9 || af) {
2474 cf = old_cf | (al >= 250);
2479 if (old_al > 0x99 || old_cf) {
2485 /* Set PF, ZF, SF */
2486 ctxt->src.type = OP_IMM;
2488 ctxt->src.bytes = 1;
2489 emulate_2op_SrcV(ctxt, "or");
2490 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2492 ctxt->eflags |= X86_EFLAGS_CF;
2494 ctxt->eflags |= X86_EFLAGS_AF;
2495 return X86EMUL_CONTINUE;
2498 static int em_call(struct x86_emulate_ctxt *ctxt)
2500 long rel = ctxt->src.val;
2502 ctxt->src.val = (unsigned long)ctxt->_eip;
2504 return em_push(ctxt);
2507 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2513 old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2514 old_eip = ctxt->_eip;
2516 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2517 if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
2518 return X86EMUL_CONTINUE;
2521 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2523 ctxt->src.val = old_cs;
2525 if (rc != X86EMUL_CONTINUE)
2528 ctxt->src.val = old_eip;
2529 return em_push(ctxt);
2532 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2536 ctxt->dst.type = OP_REG;
2537 ctxt->dst.addr.reg = &ctxt->_eip;
2538 ctxt->dst.bytes = ctxt->op_bytes;
2539 rc = emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
2540 if (rc != X86EMUL_CONTINUE)
2542 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], ctxt->src.val);
2543 return X86EMUL_CONTINUE;
2546 static int em_add(struct x86_emulate_ctxt *ctxt)
2548 emulate_2op_SrcV(ctxt, "add");
2549 return X86EMUL_CONTINUE;
2552 static int em_or(struct x86_emulate_ctxt *ctxt)
2554 emulate_2op_SrcV(ctxt, "or");
2555 return X86EMUL_CONTINUE;
2558 static int em_adc(struct x86_emulate_ctxt *ctxt)
2560 emulate_2op_SrcV(ctxt, "adc");
2561 return X86EMUL_CONTINUE;
2564 static int em_sbb(struct x86_emulate_ctxt *ctxt)
2566 emulate_2op_SrcV(ctxt, "sbb");
2567 return X86EMUL_CONTINUE;
2570 static int em_and(struct x86_emulate_ctxt *ctxt)
2572 emulate_2op_SrcV(ctxt, "and");
2573 return X86EMUL_CONTINUE;
2576 static int em_sub(struct x86_emulate_ctxt *ctxt)
2578 emulate_2op_SrcV(ctxt, "sub");
2579 return X86EMUL_CONTINUE;
2582 static int em_xor(struct x86_emulate_ctxt *ctxt)
2584 emulate_2op_SrcV(ctxt, "xor");
2585 return X86EMUL_CONTINUE;
2588 static int em_cmp(struct x86_emulate_ctxt *ctxt)
2590 emulate_2op_SrcV(ctxt, "cmp");
2591 /* Disable writeback. */
2592 ctxt->dst.type = OP_NONE;
2593 return X86EMUL_CONTINUE;
2596 static int em_test(struct x86_emulate_ctxt *ctxt)
2598 emulate_2op_SrcV(ctxt, "test");
2599 /* Disable writeback. */
2600 ctxt->dst.type = OP_NONE;
2601 return X86EMUL_CONTINUE;
2604 static int em_xchg(struct x86_emulate_ctxt *ctxt)
2606 /* Write back the register source. */
2607 ctxt->src.val = ctxt->dst.val;
2608 write_register_operand(&ctxt->src);
2610 /* Write back the memory destination with implicit LOCK prefix. */
2611 ctxt->dst.val = ctxt->src.orig_val;
2612 ctxt->lock_prefix = 1;
2613 return X86EMUL_CONTINUE;
2616 static int em_imul(struct x86_emulate_ctxt *ctxt)
2618 emulate_2op_SrcV_nobyte(ctxt, "imul");
2619 return X86EMUL_CONTINUE;
2622 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2624 ctxt->dst.val = ctxt->src2.val;
2625 return em_imul(ctxt);
2628 static int em_cwd(struct x86_emulate_ctxt *ctxt)
2630 ctxt->dst.type = OP_REG;
2631 ctxt->dst.bytes = ctxt->src.bytes;
2632 ctxt->dst.addr.reg = &ctxt->regs[VCPU_REGS_RDX];
2633 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
2635 return X86EMUL_CONTINUE;
2638 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2642 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
2643 ctxt->regs[VCPU_REGS_RAX] = (u32)tsc;
2644 ctxt->regs[VCPU_REGS_RDX] = tsc >> 32;
2645 return X86EMUL_CONTINUE;
2648 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
2652 if (ctxt->ops->read_pmc(ctxt, ctxt->regs[VCPU_REGS_RCX], &pmc))
2653 return emulate_gp(ctxt, 0);
2654 ctxt->regs[VCPU_REGS_RAX] = (u32)pmc;
2655 ctxt->regs[VCPU_REGS_RDX] = pmc >> 32;
2656 return X86EMUL_CONTINUE;
2659 static int em_mov(struct x86_emulate_ctxt *ctxt)
2661 ctxt->dst.val = ctxt->src.val;
2662 return X86EMUL_CONTINUE;
2665 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
2667 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
2668 return emulate_gp(ctxt, 0);
2670 /* Disable writeback. */
2671 ctxt->dst.type = OP_NONE;
2672 return X86EMUL_CONTINUE;
2675 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
2679 if (ctxt->mode == X86EMUL_MODE_PROT64)
2680 val = ctxt->src.val & ~0ULL;
2682 val = ctxt->src.val & ~0U;
2684 /* #UD condition is already handled. */
2685 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
2686 return emulate_gp(ctxt, 0);
2688 /* Disable writeback. */
2689 ctxt->dst.type = OP_NONE;
2690 return X86EMUL_CONTINUE;
2693 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
2697 msr_data = (u32)ctxt->regs[VCPU_REGS_RAX]
2698 | ((u64)ctxt->regs[VCPU_REGS_RDX] << 32);
2699 if (ctxt->ops->set_msr(ctxt, ctxt->regs[VCPU_REGS_RCX], msr_data))
2700 return emulate_gp(ctxt, 0);
2702 return X86EMUL_CONTINUE;
2705 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
2709 if (ctxt->ops->get_msr(ctxt, ctxt->regs[VCPU_REGS_RCX], &msr_data))
2710 return emulate_gp(ctxt, 0);
2712 ctxt->regs[VCPU_REGS_RAX] = (u32)msr_data;
2713 ctxt->regs[VCPU_REGS_RDX] = msr_data >> 32;
2714 return X86EMUL_CONTINUE;
2717 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
2719 if (ctxt->modrm_reg > VCPU_SREG_GS)
2720 return emulate_ud(ctxt);
2722 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
2723 return X86EMUL_CONTINUE;
2726 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
2728 u16 sel = ctxt->src.val;
2730 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
2731 return emulate_ud(ctxt);
2733 if (ctxt->modrm_reg == VCPU_SREG_SS)
2734 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
2736 /* Disable writeback. */
2737 ctxt->dst.type = OP_NONE;
2738 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
2741 static int em_movdqu(struct x86_emulate_ctxt *ctxt)
2743 memcpy(&ctxt->dst.vec_val, &ctxt->src.vec_val, ctxt->op_bytes);
2744 return X86EMUL_CONTINUE;
2747 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
2752 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
2753 if (rc == X86EMUL_CONTINUE)
2754 ctxt->ops->invlpg(ctxt, linear);
2755 /* Disable writeback. */
2756 ctxt->dst.type = OP_NONE;
2757 return X86EMUL_CONTINUE;
2760 static int em_clts(struct x86_emulate_ctxt *ctxt)
2764 cr0 = ctxt->ops->get_cr(ctxt, 0);
2766 ctxt->ops->set_cr(ctxt, 0, cr0);
2767 return X86EMUL_CONTINUE;
2770 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
2774 if (ctxt->modrm_mod != 3 || ctxt->modrm_rm != 1)
2775 return X86EMUL_UNHANDLEABLE;
2777 rc = ctxt->ops->fix_hypercall(ctxt);
2778 if (rc != X86EMUL_CONTINUE)
2781 /* Let the processor re-execute the fixed hypercall */
2782 ctxt->_eip = ctxt->eip;
2783 /* Disable writeback. */
2784 ctxt->dst.type = OP_NONE;
2785 return X86EMUL_CONTINUE;
2788 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
2790 struct desc_ptr desc_ptr;
2793 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
2794 &desc_ptr.size, &desc_ptr.address,
2796 if (rc != X86EMUL_CONTINUE)
2798 ctxt->ops->set_gdt(ctxt, &desc_ptr);
2799 /* Disable writeback. */
2800 ctxt->dst.type = OP_NONE;
2801 return X86EMUL_CONTINUE;
2804 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
2808 rc = ctxt->ops->fix_hypercall(ctxt);
2810 /* Disable writeback. */
2811 ctxt->dst.type = OP_NONE;
2815 static int em_lidt(struct x86_emulate_ctxt *ctxt)
2817 struct desc_ptr desc_ptr;
2820 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
2821 &desc_ptr.size, &desc_ptr.address,
2823 if (rc != X86EMUL_CONTINUE)
2825 ctxt->ops->set_idt(ctxt, &desc_ptr);
2826 /* Disable writeback. */
2827 ctxt->dst.type = OP_NONE;
2828 return X86EMUL_CONTINUE;
2831 static int em_smsw(struct x86_emulate_ctxt *ctxt)
2833 ctxt->dst.bytes = 2;
2834 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
2835 return X86EMUL_CONTINUE;
2838 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
2840 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
2841 | (ctxt->src.val & 0x0f));
2842 ctxt->dst.type = OP_NONE;
2843 return X86EMUL_CONTINUE;
2846 static int em_loop(struct x86_emulate_ctxt *ctxt)
2848 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1);
2849 if ((address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) != 0) &&
2850 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
2851 jmp_rel(ctxt, ctxt->src.val);
2853 return X86EMUL_CONTINUE;
2856 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
2858 if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0)
2859 jmp_rel(ctxt, ctxt->src.val);
2861 return X86EMUL_CONTINUE;
2864 static int em_in(struct x86_emulate_ctxt *ctxt)
2866 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
2868 return X86EMUL_IO_NEEDED;
2870 return X86EMUL_CONTINUE;
2873 static int em_out(struct x86_emulate_ctxt *ctxt)
2875 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
2877 /* Disable writeback. */
2878 ctxt->dst.type = OP_NONE;
2879 return X86EMUL_CONTINUE;
2882 static int em_cli(struct x86_emulate_ctxt *ctxt)
2884 if (emulator_bad_iopl(ctxt))
2885 return emulate_gp(ctxt, 0);
2887 ctxt->eflags &= ~X86_EFLAGS_IF;
2888 return X86EMUL_CONTINUE;
2891 static int em_sti(struct x86_emulate_ctxt *ctxt)
2893 if (emulator_bad_iopl(ctxt))
2894 return emulate_gp(ctxt, 0);
2896 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
2897 ctxt->eflags |= X86_EFLAGS_IF;
2898 return X86EMUL_CONTINUE;
2901 static int em_bt(struct x86_emulate_ctxt *ctxt)
2903 /* Disable writeback. */
2904 ctxt->dst.type = OP_NONE;
2905 /* only subword offset */
2906 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
2908 emulate_2op_SrcV_nobyte(ctxt, "bt");
2909 return X86EMUL_CONTINUE;
2912 static int em_bts(struct x86_emulate_ctxt *ctxt)
2914 emulate_2op_SrcV_nobyte(ctxt, "bts");
2915 return X86EMUL_CONTINUE;
2918 static int em_btr(struct x86_emulate_ctxt *ctxt)
2920 emulate_2op_SrcV_nobyte(ctxt, "btr");
2921 return X86EMUL_CONTINUE;
2924 static int em_btc(struct x86_emulate_ctxt *ctxt)
2926 emulate_2op_SrcV_nobyte(ctxt, "btc");
2927 return X86EMUL_CONTINUE;
2930 static int em_bsf(struct x86_emulate_ctxt *ctxt)
2934 __asm__ ("bsf %2, %0; setz %1"
2935 : "=r"(ctxt->dst.val), "=q"(zf)
2936 : "r"(ctxt->src.val));
2938 ctxt->eflags &= ~X86_EFLAGS_ZF;
2940 ctxt->eflags |= X86_EFLAGS_ZF;
2941 /* Disable writeback. */
2942 ctxt->dst.type = OP_NONE;
2944 return X86EMUL_CONTINUE;
2947 static int em_bsr(struct x86_emulate_ctxt *ctxt)
2951 __asm__ ("bsr %2, %0; setz %1"
2952 : "=r"(ctxt->dst.val), "=q"(zf)
2953 : "r"(ctxt->src.val));
2955 ctxt->eflags &= ~X86_EFLAGS_ZF;
2957 ctxt->eflags |= X86_EFLAGS_ZF;
2958 /* Disable writeback. */
2959 ctxt->dst.type = OP_NONE;
2961 return X86EMUL_CONTINUE;
2964 static bool valid_cr(int nr)
2976 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
2978 if (!valid_cr(ctxt->modrm_reg))
2979 return emulate_ud(ctxt);
2981 return X86EMUL_CONTINUE;
2984 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
2986 u64 new_val = ctxt->src.val64;
2987 int cr = ctxt->modrm_reg;
2990 static u64 cr_reserved_bits[] = {
2991 0xffffffff00000000ULL,
2992 0, 0, 0, /* CR3 checked later */
2999 return emulate_ud(ctxt);
3001 if (new_val & cr_reserved_bits[cr])
3002 return emulate_gp(ctxt, 0);
3007 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
3008 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
3009 return emulate_gp(ctxt, 0);
3011 cr4 = ctxt->ops->get_cr(ctxt, 4);
3012 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3014 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
3015 !(cr4 & X86_CR4_PAE))
3016 return emulate_gp(ctxt, 0);
3023 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3024 if (efer & EFER_LMA)
3025 rsvd = CR3_L_MODE_RESERVED_BITS;
3026 else if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PAE)
3027 rsvd = CR3_PAE_RESERVED_BITS;
3028 else if (ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PG)
3029 rsvd = CR3_NONPAE_RESERVED_BITS;
3032 return emulate_gp(ctxt, 0);
3037 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3039 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
3040 return emulate_gp(ctxt, 0);
3046 return X86EMUL_CONTINUE;
3049 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
3053 ctxt->ops->get_dr(ctxt, 7, &dr7);
3055 /* Check if DR7.Global_Enable is set */
3056 return dr7 & (1 << 13);
3059 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
3061 int dr = ctxt->modrm_reg;
3065 return emulate_ud(ctxt);
3067 cr4 = ctxt->ops->get_cr(ctxt, 4);
3068 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
3069 return emulate_ud(ctxt);
3071 if (check_dr7_gd(ctxt))
3072 return emulate_db(ctxt);
3074 return X86EMUL_CONTINUE;
3077 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
3079 u64 new_val = ctxt->src.val64;
3080 int dr = ctxt->modrm_reg;
3082 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
3083 return emulate_gp(ctxt, 0);
3085 return check_dr_read(ctxt);
3088 static int check_svme(struct x86_emulate_ctxt *ctxt)
3092 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3094 if (!(efer & EFER_SVME))
3095 return emulate_ud(ctxt);
3097 return X86EMUL_CONTINUE;
3100 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
3102 u64 rax = ctxt->regs[VCPU_REGS_RAX];
3104 /* Valid physical address? */
3105 if (rax & 0xffff000000000000ULL)
3106 return emulate_gp(ctxt, 0);
3108 return check_svme(ctxt);
3111 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
3113 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3115 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3116 return emulate_ud(ctxt);
3118 return X86EMUL_CONTINUE;
3121 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
3123 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3124 u64 rcx = ctxt->regs[VCPU_REGS_RCX];
3126 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3128 return emulate_gp(ctxt, 0);
3130 return X86EMUL_CONTINUE;
3133 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
3135 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
3136 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3137 return emulate_gp(ctxt, 0);
3139 return X86EMUL_CONTINUE;
3142 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
3144 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
3145 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3146 return emulate_gp(ctxt, 0);
3148 return X86EMUL_CONTINUE;
3151 #define D(_y) { .flags = (_y) }
3152 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
3153 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
3154 .check_perm = (_p) }
3156 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3157 #define G(_f, _g) { .flags = ((_f) | Group), .u.group = (_g) }
3158 #define GD(_f, _g) { .flags = ((_f) | GroupDual), .u.gdual = (_g) }
3159 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3160 #define II(_f, _e, _i) \
3161 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
3162 #define IIP(_f, _e, _i, _p) \
3163 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
3164 .check_perm = (_p) }
3165 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3167 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3168 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3169 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3170 #define I2bvIP(_f, _e, _i, _p) \
3171 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
3173 #define I6ALU(_f, _e) I2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3174 I2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3175 I2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3177 static struct opcode group7_rm1[] = {
3178 DI(SrcNone | ModRM | Priv, monitor),
3179 DI(SrcNone | ModRM | Priv, mwait),
3183 static struct opcode group7_rm3[] = {
3184 DIP(SrcNone | ModRM | Prot | Priv, vmrun, check_svme_pa),
3185 II(SrcNone | ModRM | Prot | VendorSpecific, em_vmmcall, vmmcall),
3186 DIP(SrcNone | ModRM | Prot | Priv, vmload, check_svme_pa),
3187 DIP(SrcNone | ModRM | Prot | Priv, vmsave, check_svme_pa),
3188 DIP(SrcNone | ModRM | Prot | Priv, stgi, check_svme),
3189 DIP(SrcNone | ModRM | Prot | Priv, clgi, check_svme),
3190 DIP(SrcNone | ModRM | Prot | Priv, skinit, check_svme),
3191 DIP(SrcNone | ModRM | Prot | Priv, invlpga, check_svme),
3194 static struct opcode group7_rm7[] = {
3196 DIP(SrcNone | ModRM, rdtscp, check_rdtsc),
3200 static struct opcode group1[] = {
3202 I(Lock | PageTable, em_or),
3205 I(Lock | PageTable, em_and),
3211 static struct opcode group1A[] = {
3212 I(DstMem | SrcNone | ModRM | Mov | Stack, em_pop), N, N, N, N, N, N, N,
3215 static struct opcode group3[] = {
3216 I(DstMem | SrcImm | ModRM, em_test),
3217 I(DstMem | SrcImm | ModRM, em_test),
3218 I(DstMem | SrcNone | ModRM | Lock, em_not),
3219 I(DstMem | SrcNone | ModRM | Lock, em_neg),
3220 I(SrcMem | ModRM, em_mul_ex),
3221 I(SrcMem | ModRM, em_imul_ex),
3222 I(SrcMem | ModRM, em_div_ex),
3223 I(SrcMem | ModRM, em_idiv_ex),
3226 static struct opcode group4[] = {
3227 I(ByteOp | DstMem | SrcNone | ModRM | Lock, em_grp45),
3228 I(ByteOp | DstMem | SrcNone | ModRM | Lock, em_grp45),
3232 static struct opcode group5[] = {
3233 I(DstMem | SrcNone | ModRM | Lock, em_grp45),
3234 I(DstMem | SrcNone | ModRM | Lock, em_grp45),
3235 I(SrcMem | ModRM | Stack, em_grp45),
3236 I(SrcMemFAddr | ModRM | ImplicitOps | Stack, em_call_far),
3237 I(SrcMem | ModRM | Stack, em_grp45),
3238 I(SrcMemFAddr | ModRM | ImplicitOps, em_grp45),
3239 I(SrcMem | ModRM | Stack, em_grp45), N,
3242 static struct opcode group6[] = {
3243 DI(ModRM | Prot, sldt),
3244 DI(ModRM | Prot, str),
3245 DI(ModRM | Prot | Priv, lldt),
3246 DI(ModRM | Prot | Priv, ltr),
3250 static struct group_dual group7 = { {
3251 DI(ModRM | Mov | DstMem | Priv, sgdt),
3252 DI(ModRM | Mov | DstMem | Priv, sidt),
3253 II(ModRM | SrcMem | Priv, em_lgdt, lgdt),
3254 II(ModRM | SrcMem | Priv, em_lidt, lidt),
3255 II(SrcNone | ModRM | DstMem | Mov, em_smsw, smsw), N,
3256 II(SrcMem16 | ModRM | Mov | Priv, em_lmsw, lmsw),
3257 II(SrcMem | ModRM | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3259 I(SrcNone | ModRM | Priv | VendorSpecific, em_vmcall),
3261 N, EXT(0, group7_rm3),
3262 II(SrcNone | ModRM | DstMem | Mov, em_smsw, smsw), N,
3263 II(SrcMem16 | ModRM | Mov | Priv, em_lmsw, lmsw), EXT(0, group7_rm7),
3266 static struct opcode group8[] = {
3268 I(DstMem | SrcImmByte | ModRM, em_bt),
3269 I(DstMem | SrcImmByte | ModRM | Lock | PageTable, em_bts),
3270 I(DstMem | SrcImmByte | ModRM | Lock, em_btr),
3271 I(DstMem | SrcImmByte | ModRM | Lock | PageTable, em_btc),
3274 static struct group_dual group9 = { {
3275 N, I(DstMem64 | ModRM | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
3277 N, N, N, N, N, N, N, N,
3280 static struct opcode group11[] = {
3281 I(DstMem | SrcImm | ModRM | Mov | PageTable, em_mov),
3285 static struct gprefix pfx_0f_6f_0f_7f = {
3286 N, N, N, I(Sse, em_movdqu),
3289 static struct opcode opcode_table[256] = {
3291 I6ALU(Lock, em_add),
3292 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
3293 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
3295 I6ALU(Lock | PageTable, em_or),
3296 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
3299 I6ALU(Lock, em_adc),
3300 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
3301 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
3303 I6ALU(Lock, em_sbb),
3304 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
3305 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
3307 I6ALU(Lock | PageTable, em_and), N, N,
3309 I6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3311 I6ALU(Lock, em_xor), N, N,
3313 I6ALU(0, em_cmp), N, N,
3317 X8(I(SrcReg | Stack, em_push)),
3319 X8(I(DstReg | Stack, em_pop)),
3321 I(ImplicitOps | Stack | No64, em_pusha),
3322 I(ImplicitOps | Stack | No64, em_popa),
3323 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3326 I(SrcImm | Mov | Stack, em_push),
3327 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3328 I(SrcImmByte | Mov | Stack, em_push),
3329 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3330 I2bvIP(DstDI | SrcDX | Mov | String, em_in, ins, check_perm_in), /* insb, insw/insd */
3331 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
3335 G(ByteOp | DstMem | SrcImm | ModRM | Group, group1),
3336 G(DstMem | SrcImm | ModRM | Group, group1),
3337 G(ByteOp | DstMem | SrcImm | ModRM | No64 | Group, group1),
3338 G(DstMem | SrcImmByte | ModRM | Group, group1),
3339 I2bv(DstMem | SrcReg | ModRM, em_test),
3340 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
3342 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
3343 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3344 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
3345 D(ModRM | SrcMem | NoAccess | DstReg),
3346 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
3349 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3351 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3352 I(SrcImmFAddr | No64, em_call_far), N,
3353 II(ImplicitOps | Stack, em_pushf, pushf),
3354 II(ImplicitOps | Stack, em_popf, popf), N, N,
3356 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3357 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
3358 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3359 I2bv(SrcSI | DstDI | String, em_cmp),
3361 I2bv(DstAcc | SrcImm, em_test),
3362 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3363 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3364 I2bv(SrcAcc | DstDI | String, em_cmp),
3366 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3368 X8(I(DstReg | SrcImm | Mov, em_mov)),
3370 D2bv(DstMem | SrcImmByte | ModRM),
3371 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3372 I(ImplicitOps | Stack, em_ret),
3373 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
3374 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
3375 G(ByteOp, group11), G(0, group11),
3377 N, N, N, I(ImplicitOps | Stack, em_ret_far),
3378 D(ImplicitOps), DI(SrcImmByte, intn),
3379 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3381 D2bv(DstMem | SrcOne | ModRM), D2bv(DstMem | ModRM),
3384 N, N, N, N, N, N, N, N,
3386 X3(I(SrcImmByte, em_loop)),
3387 I(SrcImmByte, em_jcxz),
3388 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
3389 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
3391 I(SrcImm | Stack, em_call), D(SrcImm | ImplicitOps),
3392 I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
3393 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
3394 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
3396 N, DI(ImplicitOps, icebp), N, N,
3397 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
3398 G(ByteOp, group3), G(0, group3),
3400 D(ImplicitOps), D(ImplicitOps),
3401 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
3402 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3405 static struct opcode twobyte_table[256] = {
3407 G(0, group6), GD(0, &group7), N, N,
3408 N, I(ImplicitOps | VendorSpecific, em_syscall),
3409 II(ImplicitOps | Priv, em_clts, clts), N,
3410 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3411 N, D(ImplicitOps | ModRM), N, N,
3413 N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
3415 DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
3416 DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
3417 IIP(ModRM | SrcMem | Priv | Op3264, em_cr_write, cr_write, check_cr_write),
3418 IIP(ModRM | SrcMem | Priv | Op3264, em_dr_write, dr_write, check_dr_write),
3420 N, N, N, N, N, N, N, N,
3422 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
3423 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
3424 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
3425 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
3426 I(ImplicitOps | VendorSpecific, em_sysenter),
3427 I(ImplicitOps | Priv | VendorSpecific, em_sysexit),
3429 N, N, N, N, N, N, N, N,
3431 X16(D(DstReg | SrcMem | ModRM | Mov)),
3433 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3438 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3443 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3447 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3449 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
3450 DI(ImplicitOps, cpuid), I(DstMem | SrcReg | ModRM | BitOp, em_bt),
3451 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3452 D(DstMem | SrcReg | Src2CL | ModRM), N, N,
3454 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
3455 DI(ImplicitOps, rsm),
3456 I(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
3457 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3458 D(DstMem | SrcReg | Src2CL | ModRM),
3459 D(ModRM), I(DstReg | SrcMem | ModRM, em_imul),
3461 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_cmpxchg),
3462 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
3463 I(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
3464 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
3465 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
3466 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3470 I(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
3471 I(DstReg | SrcMem | ModRM, em_bsf), I(DstReg | SrcMem | ModRM, em_bsr),
3472 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3474 D2bv(DstMem | SrcReg | ModRM | Lock),
3475 N, D(DstMem | SrcReg | ModRM | Mov),
3476 N, N, N, GD(0, &group9),
3477 N, N, N, N, N, N, N, N,
3479 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3481 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3483 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
3500 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
3504 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3510 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
3511 unsigned size, bool sign_extension)
3513 int rc = X86EMUL_CONTINUE;
3517 op->addr.mem.ea = ctxt->_eip;
3518 /* NB. Immediates are sign-extended as necessary. */
3519 switch (op->bytes) {
3521 op->val = insn_fetch(s8, ctxt);
3524 op->val = insn_fetch(s16, ctxt);
3527 op->val = insn_fetch(s32, ctxt);
3530 if (!sign_extension) {
3531 switch (op->bytes) {
3539 op->val &= 0xffffffff;
3547 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
3550 int rc = X86EMUL_CONTINUE;
3554 decode_register_operand(ctxt, op,
3556 ctxt->twobyte && (ctxt->b == 0xb6 || ctxt->b == 0xb7));
3559 rc = decode_imm(ctxt, op, 1, false);
3562 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3566 if ((ctxt->d & BitOp) && op == &ctxt->dst)
3567 fetch_bit_operand(ctxt);
3568 op->orig_val = op->val;
3571 ctxt->memop.bytes = 8;
3575 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3576 op->addr.reg = &ctxt->regs[VCPU_REGS_RAX];
3577 fetch_register_operand(op);
3578 op->orig_val = op->val;
3582 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3584 register_address(ctxt, ctxt->regs[VCPU_REGS_RDI]);
3585 op->addr.mem.seg = VCPU_SREG_ES;
3591 op->addr.reg = &ctxt->regs[VCPU_REGS_RDX];
3592 fetch_register_operand(op);
3596 op->val = ctxt->regs[VCPU_REGS_RCX] & 0xff;
3599 rc = decode_imm(ctxt, op, 1, true);
3606 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
3609 ctxt->memop.bytes = 2;
3612 ctxt->memop.bytes = 4;
3615 rc = decode_imm(ctxt, op, 2, false);
3618 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
3622 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3624 register_address(ctxt, ctxt->regs[VCPU_REGS_RSI]);
3625 op->addr.mem.seg = seg_override(ctxt);
3630 op->addr.mem.ea = ctxt->_eip;
3631 op->bytes = ctxt->op_bytes + 2;
3632 insn_fetch_arr(op->valptr, op->bytes, ctxt);
3635 ctxt->memop.bytes = ctxt->op_bytes + 2;
3638 op->val = VCPU_SREG_ES;
3641 op->val = VCPU_SREG_CS;
3644 op->val = VCPU_SREG_SS;
3647 op->val = VCPU_SREG_DS;
3650 op->val = VCPU_SREG_FS;
3653 op->val = VCPU_SREG_GS;
3656 /* Special instructions do their own operand decoding. */
3658 op->type = OP_NONE; /* Disable writeback. */
3666 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
3668 int rc = X86EMUL_CONTINUE;
3669 int mode = ctxt->mode;
3670 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
3671 bool op_prefix = false;
3672 struct opcode opcode;
3674 ctxt->memop.type = OP_NONE;
3675 ctxt->memopp = NULL;
3676 ctxt->_eip = ctxt->eip;
3677 ctxt->fetch.start = ctxt->_eip;
3678 ctxt->fetch.end = ctxt->fetch.start + insn_len;
3680 memcpy(ctxt->fetch.data, insn, insn_len);
3683 case X86EMUL_MODE_REAL:
3684 case X86EMUL_MODE_VM86:
3685 case X86EMUL_MODE_PROT16:
3686 def_op_bytes = def_ad_bytes = 2;
3688 case X86EMUL_MODE_PROT32:
3689 def_op_bytes = def_ad_bytes = 4;
3691 #ifdef CONFIG_X86_64
3692 case X86EMUL_MODE_PROT64:
3698 return EMULATION_FAILED;
3701 ctxt->op_bytes = def_op_bytes;
3702 ctxt->ad_bytes = def_ad_bytes;
3704 /* Legacy prefixes. */
3706 switch (ctxt->b = insn_fetch(u8, ctxt)) {
3707 case 0x66: /* operand-size override */
3709 /* switch between 2/4 bytes */
3710 ctxt->op_bytes = def_op_bytes ^ 6;
3712 case 0x67: /* address-size override */
3713 if (mode == X86EMUL_MODE_PROT64)
3714 /* switch between 4/8 bytes */
3715 ctxt->ad_bytes = def_ad_bytes ^ 12;
3717 /* switch between 2/4 bytes */
3718 ctxt->ad_bytes = def_ad_bytes ^ 6;
3720 case 0x26: /* ES override */
3721 case 0x2e: /* CS override */
3722 case 0x36: /* SS override */
3723 case 0x3e: /* DS override */
3724 set_seg_override(ctxt, (ctxt->b >> 3) & 3);
3726 case 0x64: /* FS override */
3727 case 0x65: /* GS override */
3728 set_seg_override(ctxt, ctxt->b & 7);
3730 case 0x40 ... 0x4f: /* REX */
3731 if (mode != X86EMUL_MODE_PROT64)
3733 ctxt->rex_prefix = ctxt->b;
3735 case 0xf0: /* LOCK */
3736 ctxt->lock_prefix = 1;
3738 case 0xf2: /* REPNE/REPNZ */
3739 case 0xf3: /* REP/REPE/REPZ */
3740 ctxt->rep_prefix = ctxt->b;
3746 /* Any legacy prefix after a REX prefix nullifies its effect. */
3748 ctxt->rex_prefix = 0;
3754 if (ctxt->rex_prefix & 8)
3755 ctxt->op_bytes = 8; /* REX.W */
3757 /* Opcode byte(s). */
3758 opcode = opcode_table[ctxt->b];
3759 /* Two-byte opcode? */
3760 if (ctxt->b == 0x0f) {
3762 ctxt->b = insn_fetch(u8, ctxt);
3763 opcode = twobyte_table[ctxt->b];
3765 ctxt->d = opcode.flags;
3767 while (ctxt->d & GroupMask) {
3768 switch (ctxt->d & GroupMask) {
3770 ctxt->modrm = insn_fetch(u8, ctxt);
3772 goffset = (ctxt->modrm >> 3) & 7;
3773 opcode = opcode.u.group[goffset];
3776 ctxt->modrm = insn_fetch(u8, ctxt);
3778 goffset = (ctxt->modrm >> 3) & 7;
3779 if ((ctxt->modrm >> 6) == 3)
3780 opcode = opcode.u.gdual->mod3[goffset];
3782 opcode = opcode.u.gdual->mod012[goffset];
3785 goffset = ctxt->modrm & 7;
3786 opcode = opcode.u.group[goffset];
3789 if (ctxt->rep_prefix && op_prefix)
3790 return EMULATION_FAILED;
3791 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
3792 switch (simd_prefix) {
3793 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
3794 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
3795 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
3796 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
3800 return EMULATION_FAILED;
3803 ctxt->d &= ~(u64)GroupMask;
3804 ctxt->d |= opcode.flags;
3807 ctxt->execute = opcode.u.execute;
3808 ctxt->check_perm = opcode.check_perm;
3809 ctxt->intercept = opcode.intercept;
3812 if (ctxt->d == 0 || (ctxt->d & Undefined))
3813 return EMULATION_FAILED;
3815 if (!(ctxt->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
3816 return EMULATION_FAILED;
3818 if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
3821 if (ctxt->d & Op3264) {
3822 if (mode == X86EMUL_MODE_PROT64)
3829 ctxt->op_bytes = 16;
3831 /* ModRM and SIB bytes. */
3832 if (ctxt->d & ModRM) {
3833 rc = decode_modrm(ctxt, &ctxt->memop);
3834 if (!ctxt->has_seg_override)
3835 set_seg_override(ctxt, ctxt->modrm_seg);
3836 } else if (ctxt->d & MemAbs)
3837 rc = decode_abs(ctxt, &ctxt->memop);
3838 if (rc != X86EMUL_CONTINUE)
3841 if (!ctxt->has_seg_override)
3842 set_seg_override(ctxt, VCPU_SREG_DS);
3844 ctxt->memop.addr.mem.seg = seg_override(ctxt);
3846 if (ctxt->memop.type == OP_MEM && ctxt->ad_bytes != 8)
3847 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
3850 * Decode and fetch the source operand: register, memory
3853 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
3854 if (rc != X86EMUL_CONTINUE)
3858 * Decode and fetch the second source operand: register, memory
3861 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
3862 if (rc != X86EMUL_CONTINUE)
3865 /* Decode and fetch the destination operand: register or memory. */
3866 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
3869 if (ctxt->memopp && ctxt->memopp->type == OP_MEM && ctxt->rip_relative)
3870 ctxt->memopp->addr.mem.ea += ctxt->_eip;
3872 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
3875 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
3877 return ctxt->d & PageTable;
3880 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
3882 /* The second termination condition only applies for REPE
3883 * and REPNE. Test if the repeat string operation prefix is
3884 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
3885 * corresponding termination condition according to:
3886 * - if REPE/REPZ and ZF = 0 then done
3887 * - if REPNE/REPNZ and ZF = 1 then done
3889 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
3890 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
3891 && (((ctxt->rep_prefix == REPE_PREFIX) &&
3892 ((ctxt->eflags & EFLG_ZF) == 0))
3893 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
3894 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
3900 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
3902 struct x86_emulate_ops *ops = ctxt->ops;
3903 int rc = X86EMUL_CONTINUE;
3904 int saved_dst_type = ctxt->dst.type;
3906 ctxt->mem_read.pos = 0;
3908 if (ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) {
3909 rc = emulate_ud(ctxt);
3913 /* LOCK prefix is allowed only with some instructions */
3914 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
3915 rc = emulate_ud(ctxt);
3919 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
3920 rc = emulate_ud(ctxt);
3925 && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)
3926 || !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
3927 rc = emulate_ud(ctxt);
3931 if ((ctxt->d & Sse) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
3932 rc = emulate_nm(ctxt);
3936 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
3937 rc = emulator_check_intercept(ctxt, ctxt->intercept,
3938 X86_ICPT_PRE_EXCEPT);
3939 if (rc != X86EMUL_CONTINUE)
3943 /* Privileged instruction can be executed only in CPL=0 */
3944 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
3945 rc = emulate_gp(ctxt, 0);
3949 /* Instruction can only be executed in protected mode */
3950 if ((ctxt->d & Prot) && !(ctxt->mode & X86EMUL_MODE_PROT)) {
3951 rc = emulate_ud(ctxt);
3955 /* Do instruction specific permission checks */
3956 if (ctxt->check_perm) {
3957 rc = ctxt->check_perm(ctxt);
3958 if (rc != X86EMUL_CONTINUE)
3962 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
3963 rc = emulator_check_intercept(ctxt, ctxt->intercept,
3964 X86_ICPT_POST_EXCEPT);
3965 if (rc != X86EMUL_CONTINUE)
3969 if (ctxt->rep_prefix && (ctxt->d & String)) {
3970 /* All REP prefixes have the same first termination condition */
3971 if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0) {
3972 ctxt->eip = ctxt->_eip;
3977 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
3978 rc = segmented_read(ctxt, ctxt->src.addr.mem,
3979 ctxt->src.valptr, ctxt->src.bytes);
3980 if (rc != X86EMUL_CONTINUE)
3982 ctxt->src.orig_val64 = ctxt->src.val64;
3985 if (ctxt->src2.type == OP_MEM) {
3986 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
3987 &ctxt->src2.val, ctxt->src2.bytes);
3988 if (rc != X86EMUL_CONTINUE)
3992 if ((ctxt->d & DstMask) == ImplicitOps)
3996 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
3997 /* optimisation - avoid slow emulated read if Mov */
3998 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
3999 &ctxt->dst.val, ctxt->dst.bytes);
4000 if (rc != X86EMUL_CONTINUE)
4003 ctxt->dst.orig_val = ctxt->dst.val;
4007 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4008 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4009 X86_ICPT_POST_MEMACCESS);
4010 if (rc != X86EMUL_CONTINUE)
4014 if (ctxt->execute) {
4015 rc = ctxt->execute(ctxt);
4016 if (rc != X86EMUL_CONTINUE)
4025 case 0x40 ... 0x47: /* inc r16/r32 */
4026 emulate_1op(ctxt, "inc");
4028 case 0x48 ... 0x4f: /* dec r16/r32 */
4029 emulate_1op(ctxt, "dec");
4031 case 0x63: /* movsxd */
4032 if (ctxt->mode != X86EMUL_MODE_PROT64)
4033 goto cannot_emulate;
4034 ctxt->dst.val = (s32) ctxt->src.val;
4036 case 0x70 ... 0x7f: /* jcc (short) */
4037 if (test_cc(ctxt->b, ctxt->eflags))
4038 jmp_rel(ctxt, ctxt->src.val);
4040 case 0x8d: /* lea r16/r32, m */
4041 ctxt->dst.val = ctxt->src.addr.mem.ea;
4043 case 0x90 ... 0x97: /* nop / xchg reg, rax */
4044 if (ctxt->dst.addr.reg == &ctxt->regs[VCPU_REGS_RAX])
4048 case 0x98: /* cbw/cwde/cdqe */
4049 switch (ctxt->op_bytes) {
4050 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
4051 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
4052 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
4058 case 0xcc: /* int3 */
4059 rc = emulate_int(ctxt, 3);
4061 case 0xcd: /* int n */
4062 rc = emulate_int(ctxt, ctxt->src.val);
4064 case 0xce: /* into */
4065 if (ctxt->eflags & EFLG_OF)
4066 rc = emulate_int(ctxt, 4);
4068 case 0xd0 ... 0xd1: /* Grp2 */
4071 case 0xd2 ... 0xd3: /* Grp2 */
4072 ctxt->src.val = ctxt->regs[VCPU_REGS_RCX];
4075 case 0xe9: /* jmp rel */
4076 case 0xeb: /* jmp rel short */
4077 jmp_rel(ctxt, ctxt->src.val);
4078 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4080 case 0xf4: /* hlt */
4081 ctxt->ops->halt(ctxt);
4083 case 0xf5: /* cmc */
4084 /* complement carry flag from eflags reg */
4085 ctxt->eflags ^= EFLG_CF;
4087 case 0xf8: /* clc */
4088 ctxt->eflags &= ~EFLG_CF;
4090 case 0xf9: /* stc */
4091 ctxt->eflags |= EFLG_CF;
4093 case 0xfc: /* cld */
4094 ctxt->eflags &= ~EFLG_DF;
4096 case 0xfd: /* std */
4097 ctxt->eflags |= EFLG_DF;
4100 goto cannot_emulate;
4103 if (rc != X86EMUL_CONTINUE)
4107 rc = writeback(ctxt);
4108 if (rc != X86EMUL_CONTINUE)
4112 * restore dst type in case the decoding will be reused
4113 * (happens for string instruction )
4115 ctxt->dst.type = saved_dst_type;
4117 if ((ctxt->d & SrcMask) == SrcSI)
4118 string_addr_inc(ctxt, seg_override(ctxt),
4119 VCPU_REGS_RSI, &ctxt->src);
4121 if ((ctxt->d & DstMask) == DstDI)
4122 string_addr_inc(ctxt, VCPU_SREG_ES, VCPU_REGS_RDI,
4125 if (ctxt->rep_prefix && (ctxt->d & String)) {
4126 struct read_cache *r = &ctxt->io_read;
4127 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1);
4129 if (!string_insn_completed(ctxt)) {
4131 * Re-enter guest when pio read ahead buffer is empty
4132 * or, if it is not used, after each 1024 iteration.
4134 if ((r->end != 0 || ctxt->regs[VCPU_REGS_RCX] & 0x3ff) &&
4135 (r->end == 0 || r->end != r->pos)) {
4137 * Reset read cache. Usually happens before
4138 * decode, but since instruction is restarted
4139 * we have to do it here.
4141 ctxt->mem_read.end = 0;
4142 return EMULATION_RESTART;
4144 goto done; /* skip rip writeback */
4148 ctxt->eip = ctxt->_eip;
4151 if (rc == X86EMUL_PROPAGATE_FAULT)
4152 ctxt->have_exception = true;
4153 if (rc == X86EMUL_INTERCEPTED)
4154 return EMULATION_INTERCEPTED;
4156 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4160 case 0x09: /* wbinvd */
4161 (ctxt->ops->wbinvd)(ctxt);
4163 case 0x08: /* invd */
4164 case 0x0d: /* GrpP (prefetch) */
4165 case 0x18: /* Grp16 (prefetch/nop) */
4167 case 0x20: /* mov cr, reg */
4168 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
4170 case 0x21: /* mov from dr to reg */
4171 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
4173 case 0x40 ... 0x4f: /* cmov */
4174 ctxt->dst.val = ctxt->dst.orig_val = ctxt->src.val;
4175 if (!test_cc(ctxt->b, ctxt->eflags))
4176 ctxt->dst.type = OP_NONE; /* no writeback */
4178 case 0x80 ... 0x8f: /* jnz rel, etc*/
4179 if (test_cc(ctxt->b, ctxt->eflags))
4180 jmp_rel(ctxt, ctxt->src.val);
4182 case 0x90 ... 0x9f: /* setcc r/m8 */
4183 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
4185 case 0xa4: /* shld imm8, r, r/m */
4186 case 0xa5: /* shld cl, r, r/m */
4187 emulate_2op_cl(ctxt, "shld");
4189 case 0xac: /* shrd imm8, r, r/m */
4190 case 0xad: /* shrd cl, r, r/m */
4191 emulate_2op_cl(ctxt, "shrd");
4193 case 0xae: /* clflush */
4195 case 0xb6 ... 0xb7: /* movzx */
4196 ctxt->dst.bytes = ctxt->op_bytes;
4197 ctxt->dst.val = (ctxt->d & ByteOp) ? (u8) ctxt->src.val
4198 : (u16) ctxt->src.val;
4200 case 0xbe ... 0xbf: /* movsx */
4201 ctxt->dst.bytes = ctxt->op_bytes;
4202 ctxt->dst.val = (ctxt->d & ByteOp) ? (s8) ctxt->src.val :
4203 (s16) ctxt->src.val;
4205 case 0xc0 ... 0xc1: /* xadd */
4206 emulate_2op_SrcV(ctxt, "add");
4207 /* Write back the register source. */
4208 ctxt->src.val = ctxt->dst.orig_val;
4209 write_register_operand(&ctxt->src);
4211 case 0xc3: /* movnti */
4212 ctxt->dst.bytes = ctxt->op_bytes;
4213 ctxt->dst.val = (ctxt->op_bytes == 4) ? (u32) ctxt->src.val :
4214 (u64) ctxt->src.val;
4217 goto cannot_emulate;
4220 if (rc != X86EMUL_CONTINUE)
4226 return EMULATION_FAILED;