1
/******************************************************************************
4
* Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6
* Copyright (c) 2005 Keir Fraser
8
* Linux coding style, mod r/m decoder, segment base fixes, real-mode
9
* privileged instructions:
11
* Copyright (C) 2006 Qumranet
12
* Copyright 2010 Red Hat, Inc. and/or its affiliates.
14
* Avi Kivity <avi@qumranet.com>
15
* Yaniv Kamay <yaniv@qumranet.com>
17
* This work is licensed under the terms of the GNU GPL, version 2. See
18
* the COPYING file in the top-level directory.
20
* From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23
#include <linux/kvm_host.h>
24
#include "kvm_cache_regs.h"
25
#include <linux/module.h>
26
#include <asm/kvm_emulate.h>
35
#define OpImplicit 1ull /* No generic decode */
36
#define OpReg 2ull /* Register */
37
#define OpMem 3ull /* Memory */
38
#define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
39
#define OpDI 5ull /* ES:DI/EDI/RDI */
40
#define OpMem64 6ull /* Memory, 64-bit */
41
#define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
42
#define OpDX 8ull /* DX register */
43
#define OpCL 9ull /* CL register (for shifts) */
44
#define OpImmByte 10ull /* 8-bit sign extended immediate */
45
#define OpOne 11ull /* Implied 1 */
46
#define OpImm 12ull /* Sign extended immediate */
47
#define OpMem16 13ull /* Memory operand (16-bit). */
48
#define OpMem32 14ull /* Memory operand (32-bit). */
49
#define OpImmU 15ull /* Immediate operand, zero extended */
50
#define OpSI 16ull /* SI/ESI/RSI */
51
#define OpImmFAddr 17ull /* Immediate far address */
52
#define OpMemFAddr 18ull /* Far address in memory */
53
#define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
54
#define OpES 20ull /* ES */
55
#define OpCS 21ull /* CS */
56
#define OpSS 22ull /* SS */
57
#define OpDS 23ull /* DS */
58
#define OpFS 24ull /* FS */
59
#define OpGS 25ull /* GS */
61
#define OpBits 5 /* Width of operand field */
62
#define OpMask ((1ull << OpBits) - 1)
65
* Opcode effective-address decode tables.
66
* Note that we only emulate instructions that have at least one memory
67
* operand (excluding implicit stack references). We assume that stack
68
* references and instruction fetches will never occur in special memory
69
* areas that require emulation. So, for example, 'mov <imm>,<reg>' need
73
/* Operand sizes: 8-bit operands or specified/overridden size. */
74
#define ByteOp (1<<0) /* 8-bit operands. */
75
/* Destination operand type. */
77
#define ImplicitOps (OpImplicit << DstShift)
78
#define DstReg (OpReg << DstShift)
79
#define DstMem (OpMem << DstShift)
80
#define DstAcc (OpAcc << DstShift)
81
#define DstDI (OpDI << DstShift)
82
#define DstMem64 (OpMem64 << DstShift)
83
#define DstImmUByte (OpImmUByte << DstShift)
84
#define DstDX (OpDX << DstShift)
85
#define DstMask (OpMask << DstShift)
86
/* Source operand type. */
88
#define SrcNone (OpNone << SrcShift)
89
#define SrcReg (OpReg << SrcShift)
90
#define SrcMem (OpMem << SrcShift)
91
#define SrcMem16 (OpMem16 << SrcShift)
92
#define SrcMem32 (OpMem32 << SrcShift)
93
#define SrcImm (OpImm << SrcShift)
94
#define SrcImmByte (OpImmByte << SrcShift)
95
#define SrcOne (OpOne << SrcShift)
96
#define SrcImmUByte (OpImmUByte << SrcShift)
97
#define SrcImmU (OpImmU << SrcShift)
98
#define SrcSI (OpSI << SrcShift)
99
#define SrcImmFAddr (OpImmFAddr << SrcShift)
100
#define SrcMemFAddr (OpMemFAddr << SrcShift)
101
#define SrcAcc (OpAcc << SrcShift)
102
#define SrcImmU16 (OpImmU16 << SrcShift)
103
#define SrcDX (OpDX << SrcShift)
104
#define SrcMask (OpMask << SrcShift)
105
#define BitOp (1<<11)
106
#define MemAbs (1<<12) /* Memory operand is absolute displacement */
107
#define String (1<<13) /* String instruction (rep capable) */
108
#define Stack (1<<14) /* Stack instruction (push/pop) */
109
#define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
110
#define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
111
#define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
112
#define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
113
#define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
114
#define Sse (1<<18) /* SSE Vector instruction */
115
/* Generic ModRM decode. */
116
#define ModRM (1<<19)
117
/* Destination is only written; never read. */
120
#define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
121
#define VendorSpecific (1<<22) /* Vendor specific instruction */
122
#define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
123
#define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
124
#define Undefined (1<<25) /* No Such Instruction */
125
#define Lock (1<<26) /* lock prefix is allowed for the instruction */
126
#define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
128
/* Source 2 operand type */
129
#define Src2Shift (29)
130
#define Src2None (OpNone << Src2Shift)
131
#define Src2CL (OpCL << Src2Shift)
132
#define Src2ImmByte (OpImmByte << Src2Shift)
133
#define Src2One (OpOne << Src2Shift)
134
#define Src2Imm (OpImm << Src2Shift)
135
#define Src2ES (OpES << Src2Shift)
136
#define Src2CS (OpCS << Src2Shift)
137
#define Src2SS (OpSS << Src2Shift)
138
#define Src2DS (OpDS << Src2Shift)
139
#define Src2FS (OpFS << Src2Shift)
140
#define Src2GS (OpGS << Src2Shift)
141
#define Src2Mask (OpMask << Src2Shift)
143
#define X2(x...) x, x
144
#define X3(x...) X2(x), x
145
#define X4(x...) X2(x), X2(x)
146
#define X5(x...) X4(x), x
147
#define X6(x...) X4(x), X2(x)
148
#define X7(x...) X4(x), X3(x)
149
#define X8(x...) X4(x), X4(x)
150
#define X16(x...) X8(x), X8(x)
156
int (*execute)(struct x86_emulate_ctxt *ctxt);
157
struct opcode *group;
158
struct group_dual *gdual;
159
struct gprefix *gprefix;
161
int (*check_perm)(struct x86_emulate_ctxt *ctxt);
165
struct opcode mod012[8];
166
struct opcode mod3[8];
170
struct opcode pfx_no;
171
struct opcode pfx_66;
172
struct opcode pfx_f2;
173
struct opcode pfx_f3;
176
/* EFLAGS bit definitions. */
177
#define EFLG_ID (1<<21)
178
#define EFLG_VIP (1<<20)
179
#define EFLG_VIF (1<<19)
180
#define EFLG_AC (1<<18)
181
#define EFLG_VM (1<<17)
182
#define EFLG_RF (1<<16)
183
#define EFLG_IOPL (3<<12)
184
#define EFLG_NT (1<<14)
185
#define EFLG_OF (1<<11)
186
#define EFLG_DF (1<<10)
187
#define EFLG_IF (1<<9)
188
#define EFLG_TF (1<<8)
189
#define EFLG_SF (1<<7)
190
#define EFLG_ZF (1<<6)
191
#define EFLG_AF (1<<4)
192
#define EFLG_PF (1<<2)
193
#define EFLG_CF (1<<0)
195
#define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
196
#define EFLG_RESERVED_ONE_MASK 2
199
* Instruction emulation:
200
* Most instructions are emulated directly via a fragment of inline assembly
201
* code. This allows us to save/restore EFLAGS and thus very easily pick up
202
* any modified flags.
205
#if defined(CONFIG_X86_64)
206
#define _LO32 "k" /* force 32-bit operand */
207
#define _STK "%%rsp" /* stack pointer */
208
#elif defined(__i386__)
209
#define _LO32 "" /* force 32-bit operand */
210
#define _STK "%%esp" /* stack pointer */
214
* These EFLAGS bits are restored from saved value during emulation, and
215
* any changes are written back to the saved value after emulation.
217
#define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
219
/* Before executing instruction: restore necessary bits in EFLAGS. */
220
#define _PRE_EFLAGS(_sav, _msk, _tmp) \
221
/* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
222
"movl %"_sav",%"_LO32 _tmp"; " \
225
"movl %"_msk",%"_LO32 _tmp"; " \
226
"andl %"_LO32 _tmp",("_STK"); " \
228
"notl %"_LO32 _tmp"; " \
229
"andl %"_LO32 _tmp",("_STK"); " \
230
"andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
232
"orl %"_LO32 _tmp",("_STK"); " \
236
/* After executing instruction: write-back necessary bits in EFLAGS. */
237
#define _POST_EFLAGS(_sav, _msk, _tmp) \
238
/* _sav |= EFLAGS & _msk; */ \
241
"andl %"_msk",%"_LO32 _tmp"; " \
242
"orl %"_LO32 _tmp",%"_sav"; "
250
#define ____emulate_2op(ctxt, _op, _x, _y, _suffix, _dsttype) \
252
__asm__ __volatile__ ( \
253
_PRE_EFLAGS("0", "4", "2") \
254
_op _suffix " %"_x"3,%1; " \
255
_POST_EFLAGS("0", "4", "2") \
256
: "=m" ((ctxt)->eflags), \
257
"+q" (*(_dsttype*)&(ctxt)->dst.val), \
259
: _y ((ctxt)->src.val), "i" (EFLAGS_MASK)); \
263
/* Raw emulation: instruction has two explicit operands. */
264
#define __emulate_2op_nobyte(ctxt,_op,_wx,_wy,_lx,_ly,_qx,_qy) \
266
unsigned long _tmp; \
268
switch ((ctxt)->dst.bytes) { \
270
____emulate_2op(ctxt,_op,_wx,_wy,"w",u16); \
273
____emulate_2op(ctxt,_op,_lx,_ly,"l",u32); \
276
ON64(____emulate_2op(ctxt,_op,_qx,_qy,"q",u64)); \
281
#define __emulate_2op(ctxt,_op,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
283
unsigned long _tmp; \
284
switch ((ctxt)->dst.bytes) { \
286
____emulate_2op(ctxt,_op,_bx,_by,"b",u8); \
289
__emulate_2op_nobyte(ctxt, _op, \
290
_wx, _wy, _lx, _ly, _qx, _qy); \
295
/* Source operand is byte-sized and may be restricted to just %cl. */
296
#define emulate_2op_SrcB(ctxt, _op) \
297
__emulate_2op(ctxt, _op, "b", "c", "b", "c", "b", "c", "b", "c")
299
/* Source operand is byte, word, long or quad sized. */
300
#define emulate_2op_SrcV(ctxt, _op) \
301
__emulate_2op(ctxt, _op, "b", "q", "w", "r", _LO32, "r", "", "r")
303
/* Source operand is word, long or quad sized. */
304
#define emulate_2op_SrcV_nobyte(ctxt, _op) \
305
__emulate_2op_nobyte(ctxt, _op, "w", "r", _LO32, "r", "", "r")
307
/* Instruction has three operands and one operand is stored in ECX register */
308
#define __emulate_2op_cl(ctxt, _op, _suffix, _type) \
310
unsigned long _tmp; \
311
_type _clv = (ctxt)->src2.val; \
312
_type _srcv = (ctxt)->src.val; \
313
_type _dstv = (ctxt)->dst.val; \
315
__asm__ __volatile__ ( \
316
_PRE_EFLAGS("0", "5", "2") \
317
_op _suffix " %4,%1 \n" \
318
_POST_EFLAGS("0", "5", "2") \
319
: "=m" ((ctxt)->eflags), "+r" (_dstv), "=&r" (_tmp) \
320
: "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
323
(ctxt)->src2.val = (unsigned long) _clv; \
324
(ctxt)->src2.val = (unsigned long) _srcv; \
325
(ctxt)->dst.val = (unsigned long) _dstv; \
328
#define emulate_2op_cl(ctxt, _op) \
330
switch ((ctxt)->dst.bytes) { \
332
__emulate_2op_cl(ctxt, _op, "w", u16); \
335
__emulate_2op_cl(ctxt, _op, "l", u32); \
338
ON64(__emulate_2op_cl(ctxt, _op, "q", ulong)); \
343
#define __emulate_1op(ctxt, _op, _suffix) \
345
unsigned long _tmp; \
347
__asm__ __volatile__ ( \
348
_PRE_EFLAGS("0", "3", "2") \
349
_op _suffix " %1; " \
350
_POST_EFLAGS("0", "3", "2") \
351
: "=m" ((ctxt)->eflags), "+m" ((ctxt)->dst.val), \
353
: "i" (EFLAGS_MASK)); \
356
/* Instruction has only one explicit operand (no source operand). */
357
#define emulate_1op(ctxt, _op) \
359
switch ((ctxt)->dst.bytes) { \
360
case 1: __emulate_1op(ctxt, _op, "b"); break; \
361
case 2: __emulate_1op(ctxt, _op, "w"); break; \
362
case 4: __emulate_1op(ctxt, _op, "l"); break; \
363
case 8: ON64(__emulate_1op(ctxt, _op, "q")); break; \
367
#define __emulate_1op_rax_rdx(ctxt, _op, _suffix, _ex) \
369
unsigned long _tmp; \
370
ulong *rax = &(ctxt)->regs[VCPU_REGS_RAX]; \
371
ulong *rdx = &(ctxt)->regs[VCPU_REGS_RDX]; \
373
__asm__ __volatile__ ( \
374
_PRE_EFLAGS("0", "5", "1") \
376
_op _suffix " %6; " \
378
_POST_EFLAGS("0", "5", "1") \
379
".pushsection .fixup,\"ax\" \n\t" \
380
"3: movb $1, %4 \n\t" \
383
_ASM_EXTABLE(1b, 3b) \
384
: "=m" ((ctxt)->eflags), "=&r" (_tmp), \
385
"+a" (*rax), "+d" (*rdx), "+qm"(_ex) \
386
: "i" (EFLAGS_MASK), "m" ((ctxt)->src.val), \
387
"a" (*rax), "d" (*rdx)); \
390
/* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
391
#define emulate_1op_rax_rdx(ctxt, _op, _ex) \
393
switch((ctxt)->src.bytes) { \
395
__emulate_1op_rax_rdx(ctxt, _op, "b", _ex); \
398
__emulate_1op_rax_rdx(ctxt, _op, "w", _ex); \
401
__emulate_1op_rax_rdx(ctxt, _op, "l", _ex); \
404
__emulate_1op_rax_rdx(ctxt, _op, "q", _ex)); \
409
static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
410
enum x86_intercept intercept,
411
enum x86_intercept_stage stage)
413
struct x86_instruction_info info = {
414
.intercept = intercept,
415
.rep_prefix = ctxt->rep_prefix,
416
.modrm_mod = ctxt->modrm_mod,
417
.modrm_reg = ctxt->modrm_reg,
418
.modrm_rm = ctxt->modrm_rm,
419
.src_val = ctxt->src.val64,
420
.src_bytes = ctxt->src.bytes,
421
.dst_bytes = ctxt->dst.bytes,
422
.ad_bytes = ctxt->ad_bytes,
423
.next_rip = ctxt->eip,
426
return ctxt->ops->intercept(ctxt, &info, stage);
429
static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
431
return (1UL << (ctxt->ad_bytes << 3)) - 1;
434
/* Access/update address held in a register, based on addressing mode. */
435
static inline unsigned long
436
address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
438
if (ctxt->ad_bytes == sizeof(unsigned long))
441
return reg & ad_mask(ctxt);
444
static inline unsigned long
445
register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
447
return address_mask(ctxt, reg);
451
register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
453
if (ctxt->ad_bytes == sizeof(unsigned long))
456
*reg = (*reg & ~ad_mask(ctxt)) | ((*reg + inc) & ad_mask(ctxt));
459
static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
461
register_address_increment(ctxt, &ctxt->_eip, rel);
464
static u32 desc_limit_scaled(struct desc_struct *desc)
466
u32 limit = get_desc_limit(desc);
468
return desc->g ? (limit << 12) | 0xfff : limit;
471
static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
473
ctxt->has_seg_override = true;
474
ctxt->seg_override = seg;
477
static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
479
if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
482
return ctxt->ops->get_cached_segment_base(ctxt, seg);
485
static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
487
if (!ctxt->has_seg_override)
490
return ctxt->seg_override;
493
static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
494
u32 error, bool valid)
496
ctxt->exception.vector = vec;
497
ctxt->exception.error_code = error;
498
ctxt->exception.error_code_valid = valid;
499
return X86EMUL_PROPAGATE_FAULT;
502
static int emulate_db(struct x86_emulate_ctxt *ctxt)
504
return emulate_exception(ctxt, DB_VECTOR, 0, false);
507
static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
509
return emulate_exception(ctxt, GP_VECTOR, err, true);
512
static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
514
return emulate_exception(ctxt, SS_VECTOR, err, true);
517
static int emulate_ud(struct x86_emulate_ctxt *ctxt)
519
return emulate_exception(ctxt, UD_VECTOR, 0, false);
522
static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
524
return emulate_exception(ctxt, TS_VECTOR, err, true);
527
static int emulate_de(struct x86_emulate_ctxt *ctxt)
529
return emulate_exception(ctxt, DE_VECTOR, 0, false);
532
static int emulate_nm(struct x86_emulate_ctxt *ctxt)
534
return emulate_exception(ctxt, NM_VECTOR, 0, false);
537
static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
540
struct desc_struct desc;
542
ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
546
static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
551
struct desc_struct desc;
553
ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
554
ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
557
static int __linearize(struct x86_emulate_ctxt *ctxt,
558
struct segmented_address addr,
559
unsigned size, bool write, bool fetch,
562
struct desc_struct desc;
569
la = seg_base(ctxt, addr.seg) + addr.ea;
570
switch (ctxt->mode) {
571
case X86EMUL_MODE_REAL:
573
case X86EMUL_MODE_PROT64:
574
if (((signed long)la << 16) >> 16 != la)
575
return emulate_gp(ctxt, 0);
578
usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
582
/* code segment or read-only data segment */
583
if (((desc.type & 8) || !(desc.type & 2)) && write)
585
/* unreadable code segment */
586
if (!fetch && (desc.type & 8) && !(desc.type & 2))
588
lim = desc_limit_scaled(&desc);
589
if ((desc.type & 8) || !(desc.type & 4)) {
590
/* expand-up segment */
591
if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
594
/* exapand-down segment */
595
if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
597
lim = desc.d ? 0xffffffff : 0xffff;
598
if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
601
cpl = ctxt->ops->cpl(ctxt);
604
if (!(desc.type & 8)) {
608
} else if ((desc.type & 8) && !(desc.type & 4)) {
609
/* nonconforming code segment */
612
} else if ((desc.type & 8) && (desc.type & 4)) {
613
/* conforming code segment */
619
if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
622
return X86EMUL_CONTINUE;
624
if (addr.seg == VCPU_SREG_SS)
625
return emulate_ss(ctxt, addr.seg);
627
return emulate_gp(ctxt, addr.seg);
630
static int linearize(struct x86_emulate_ctxt *ctxt,
631
struct segmented_address addr,
632
unsigned size, bool write,
635
return __linearize(ctxt, addr, size, write, false, linear);
639
static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
640
struct segmented_address addr,
647
rc = linearize(ctxt, addr, size, false, &linear);
648
if (rc != X86EMUL_CONTINUE)
650
return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
654
* Fetch the next byte of the instruction being emulated which is pointed to
655
* by ctxt->_eip, then increment ctxt->_eip.
657
* Also prefetch the remaining bytes of the instruction without crossing page
658
* boundary if they are not in fetch_cache yet.
660
static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
662
struct fetch_cache *fc = &ctxt->fetch;
666
if (ctxt->_eip == fc->end) {
667
unsigned long linear;
668
struct segmented_address addr = { .seg = VCPU_SREG_CS,
670
cur_size = fc->end - fc->start;
671
size = min(15UL - cur_size,
672
PAGE_SIZE - offset_in_page(ctxt->_eip));
673
rc = __linearize(ctxt, addr, size, false, true, &linear);
674
if (unlikely(rc != X86EMUL_CONTINUE))
676
rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
677
size, &ctxt->exception);
678
if (unlikely(rc != X86EMUL_CONTINUE))
682
*dest = fc->data[ctxt->_eip - fc->start];
684
return X86EMUL_CONTINUE;
687
static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
688
void *dest, unsigned size)
692
/* x86 instructions are limited to 15 bytes. */
693
if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
694
return X86EMUL_UNHANDLEABLE;
696
rc = do_insn_fetch_byte(ctxt, dest++);
697
if (rc != X86EMUL_CONTINUE)
700
return X86EMUL_CONTINUE;
703
/* Fetch next part of the instruction being emulated. */
704
#define insn_fetch(_type, _ctxt) \
705
({ unsigned long _x; \
706
rc = do_insn_fetch(_ctxt, &_x, sizeof(_type)); \
707
if (rc != X86EMUL_CONTINUE) \
712
#define insn_fetch_arr(_arr, _size, _ctxt) \
713
({ rc = do_insn_fetch(_ctxt, _arr, (_size)); \
714
if (rc != X86EMUL_CONTINUE) \
719
* Given the 'reg' portion of a ModRM byte, and a register block, return a
720
* pointer into the block that addresses the relevant register.
721
* @highbyte_regs specifies whether to decode AH,CH,DH,BH.
723
static void *decode_register(u8 modrm_reg, unsigned long *regs,
728
p = ®s[modrm_reg];
729
if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
730
p = (unsigned char *)®s[modrm_reg & 3] + 1;
734
static int read_descriptor(struct x86_emulate_ctxt *ctxt,
735
struct segmented_address addr,
736
u16 *size, unsigned long *address, int op_bytes)
743
rc = segmented_read_std(ctxt, addr, size, 2);
744
if (rc != X86EMUL_CONTINUE)
747
rc = segmented_read_std(ctxt, addr, address, op_bytes);
751
static int test_cc(unsigned int condition, unsigned int flags)
755
switch ((condition & 15) >> 1) {
757
rc |= (flags & EFLG_OF);
759
case 1: /* b/c/nae */
760
rc |= (flags & EFLG_CF);
763
rc |= (flags & EFLG_ZF);
766
rc |= (flags & (EFLG_CF|EFLG_ZF));
769
rc |= (flags & EFLG_SF);
772
rc |= (flags & EFLG_PF);
775
rc |= (flags & EFLG_ZF);
778
rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
782
/* Odd condition identifiers (lsb == 1) have inverted sense. */
783
return (!!rc ^ (condition & 1));
786
static void fetch_register_operand(struct operand *op)
790
op->val = *(u8 *)op->addr.reg;
793
op->val = *(u16 *)op->addr.reg;
796
op->val = *(u32 *)op->addr.reg;
799
op->val = *(u64 *)op->addr.reg;
804
static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
806
ctxt->ops->get_fpu(ctxt);
808
case 0: asm("movdqu %%xmm0, %0" : "=m"(*data)); break;
809
case 1: asm("movdqu %%xmm1, %0" : "=m"(*data)); break;
810
case 2: asm("movdqu %%xmm2, %0" : "=m"(*data)); break;
811
case 3: asm("movdqu %%xmm3, %0" : "=m"(*data)); break;
812
case 4: asm("movdqu %%xmm4, %0" : "=m"(*data)); break;
813
case 5: asm("movdqu %%xmm5, %0" : "=m"(*data)); break;
814
case 6: asm("movdqu %%xmm6, %0" : "=m"(*data)); break;
815
case 7: asm("movdqu %%xmm7, %0" : "=m"(*data)); break;
817
case 8: asm("movdqu %%xmm8, %0" : "=m"(*data)); break;
818
case 9: asm("movdqu %%xmm9, %0" : "=m"(*data)); break;
819
case 10: asm("movdqu %%xmm10, %0" : "=m"(*data)); break;
820
case 11: asm("movdqu %%xmm11, %0" : "=m"(*data)); break;
821
case 12: asm("movdqu %%xmm12, %0" : "=m"(*data)); break;
822
case 13: asm("movdqu %%xmm13, %0" : "=m"(*data)); break;
823
case 14: asm("movdqu %%xmm14, %0" : "=m"(*data)); break;
824
case 15: asm("movdqu %%xmm15, %0" : "=m"(*data)); break;
828
ctxt->ops->put_fpu(ctxt);
831
static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
834
ctxt->ops->get_fpu(ctxt);
836
case 0: asm("movdqu %0, %%xmm0" : : "m"(*data)); break;
837
case 1: asm("movdqu %0, %%xmm1" : : "m"(*data)); break;
838
case 2: asm("movdqu %0, %%xmm2" : : "m"(*data)); break;
839
case 3: asm("movdqu %0, %%xmm3" : : "m"(*data)); break;
840
case 4: asm("movdqu %0, %%xmm4" : : "m"(*data)); break;
841
case 5: asm("movdqu %0, %%xmm5" : : "m"(*data)); break;
842
case 6: asm("movdqu %0, %%xmm6" : : "m"(*data)); break;
843
case 7: asm("movdqu %0, %%xmm7" : : "m"(*data)); break;
845
case 8: asm("movdqu %0, %%xmm8" : : "m"(*data)); break;
846
case 9: asm("movdqu %0, %%xmm9" : : "m"(*data)); break;
847
case 10: asm("movdqu %0, %%xmm10" : : "m"(*data)); break;
848
case 11: asm("movdqu %0, %%xmm11" : : "m"(*data)); break;
849
case 12: asm("movdqu %0, %%xmm12" : : "m"(*data)); break;
850
case 13: asm("movdqu %0, %%xmm13" : : "m"(*data)); break;
851
case 14: asm("movdqu %0, %%xmm14" : : "m"(*data)); break;
852
case 15: asm("movdqu %0, %%xmm15" : : "m"(*data)); break;
856
ctxt->ops->put_fpu(ctxt);
859
static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
863
unsigned reg = ctxt->modrm_reg;
864
int highbyte_regs = ctxt->rex_prefix == 0;
866
if (!(ctxt->d & ModRM))
867
reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
873
read_sse_reg(ctxt, &op->vec_val, reg);
878
if ((ctxt->d & ByteOp) && !inhibit_bytereg) {
879
op->addr.reg = decode_register(reg, ctxt->regs, highbyte_regs);
882
op->addr.reg = decode_register(reg, ctxt->regs, 0);
883
op->bytes = ctxt->op_bytes;
885
fetch_register_operand(op);
886
op->orig_val = op->val;
889
static int decode_modrm(struct x86_emulate_ctxt *ctxt,
893
int index_reg = 0, base_reg = 0, scale;
894
int rc = X86EMUL_CONTINUE;
897
if (ctxt->rex_prefix) {
898
ctxt->modrm_reg = (ctxt->rex_prefix & 4) << 1; /* REX.R */
899
index_reg = (ctxt->rex_prefix & 2) << 2; /* REX.X */
900
ctxt->modrm_rm = base_reg = (ctxt->rex_prefix & 1) << 3; /* REG.B */
903
ctxt->modrm = insn_fetch(u8, ctxt);
904
ctxt->modrm_mod |= (ctxt->modrm & 0xc0) >> 6;
905
ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
906
ctxt->modrm_rm |= (ctxt->modrm & 0x07);
907
ctxt->modrm_seg = VCPU_SREG_DS;
909
if (ctxt->modrm_mod == 3) {
911
op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
912
op->addr.reg = decode_register(ctxt->modrm_rm,
913
ctxt->regs, ctxt->d & ByteOp);
917
op->addr.xmm = ctxt->modrm_rm;
918
read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
921
fetch_register_operand(op);
927
if (ctxt->ad_bytes == 2) {
928
unsigned bx = ctxt->regs[VCPU_REGS_RBX];
929
unsigned bp = ctxt->regs[VCPU_REGS_RBP];
930
unsigned si = ctxt->regs[VCPU_REGS_RSI];
931
unsigned di = ctxt->regs[VCPU_REGS_RDI];
933
/* 16-bit ModR/M decode. */
934
switch (ctxt->modrm_mod) {
936
if (ctxt->modrm_rm == 6)
937
modrm_ea += insn_fetch(u16, ctxt);
940
modrm_ea += insn_fetch(s8, ctxt);
943
modrm_ea += insn_fetch(u16, ctxt);
946
switch (ctxt->modrm_rm) {
966
if (ctxt->modrm_mod != 0)
973
if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
974
(ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
975
ctxt->modrm_seg = VCPU_SREG_SS;
976
modrm_ea = (u16)modrm_ea;
978
/* 32/64-bit ModR/M decode. */
979
if ((ctxt->modrm_rm & 7) == 4) {
980
sib = insn_fetch(u8, ctxt);
981
index_reg |= (sib >> 3) & 7;
985
if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
986
modrm_ea += insn_fetch(s32, ctxt);
988
modrm_ea += ctxt->regs[base_reg];
990
modrm_ea += ctxt->regs[index_reg] << scale;
991
} else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
992
if (ctxt->mode == X86EMUL_MODE_PROT64)
993
ctxt->rip_relative = 1;
995
modrm_ea += ctxt->regs[ctxt->modrm_rm];
996
switch (ctxt->modrm_mod) {
998
if (ctxt->modrm_rm == 5)
999
modrm_ea += insn_fetch(s32, ctxt);
1002
modrm_ea += insn_fetch(s8, ctxt);
1005
modrm_ea += insn_fetch(s32, ctxt);
1009
op->addr.mem.ea = modrm_ea;
1014
static int decode_abs(struct x86_emulate_ctxt *ctxt,
1017
int rc = X86EMUL_CONTINUE;
1020
switch (ctxt->ad_bytes) {
1022
op->addr.mem.ea = insn_fetch(u16, ctxt);
1025
op->addr.mem.ea = insn_fetch(u32, ctxt);
1028
op->addr.mem.ea = insn_fetch(u64, ctxt);
1035
static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1039
if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1040
mask = ~(ctxt->dst.bytes * 8 - 1);
1042
if (ctxt->src.bytes == 2)
1043
sv = (s16)ctxt->src.val & (s16)mask;
1044
else if (ctxt->src.bytes == 4)
1045
sv = (s32)ctxt->src.val & (s32)mask;
1047
ctxt->dst.addr.mem.ea += (sv >> 3);
1050
/* only subword offset */
1051
ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1054
static int read_emulated(struct x86_emulate_ctxt *ctxt,
1055
unsigned long addr, void *dest, unsigned size)
1058
struct read_cache *mc = &ctxt->mem_read;
1061
int n = min(size, 8u);
1063
if (mc->pos < mc->end)
1066
rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, n,
1068
if (rc != X86EMUL_CONTINUE)
1073
memcpy(dest, mc->data + mc->pos, n);
1078
return X86EMUL_CONTINUE;
1081
static int segmented_read(struct x86_emulate_ctxt *ctxt,
1082
struct segmented_address addr,
1089
rc = linearize(ctxt, addr, size, false, &linear);
1090
if (rc != X86EMUL_CONTINUE)
1092
return read_emulated(ctxt, linear, data, size);
1095
static int segmented_write(struct x86_emulate_ctxt *ctxt,
1096
struct segmented_address addr,
1103
rc = linearize(ctxt, addr, size, true, &linear);
1104
if (rc != X86EMUL_CONTINUE)
1106
return ctxt->ops->write_emulated(ctxt, linear, data, size,
1110
static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1111
struct segmented_address addr,
1112
const void *orig_data, const void *data,
1118
rc = linearize(ctxt, addr, size, true, &linear);
1119
if (rc != X86EMUL_CONTINUE)
1121
return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1122
size, &ctxt->exception);
1125
static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1126
unsigned int size, unsigned short port,
1129
struct read_cache *rc = &ctxt->io_read;
1131
if (rc->pos == rc->end) { /* refill pio read ahead */
1132
unsigned int in_page, n;
1133
unsigned int count = ctxt->rep_prefix ?
1134
address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) : 1;
1135
in_page = (ctxt->eflags & EFLG_DF) ?
1136
offset_in_page(ctxt->regs[VCPU_REGS_RDI]) :
1137
PAGE_SIZE - offset_in_page(ctxt->regs[VCPU_REGS_RDI]);
1138
n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1142
rc->pos = rc->end = 0;
1143
if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1148
memcpy(dest, rc->data + rc->pos, size);
1153
static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1154
u16 selector, struct desc_ptr *dt)
1156
struct x86_emulate_ops *ops = ctxt->ops;
1158
if (selector & 1 << 2) {
1159
struct desc_struct desc;
1162
memset (dt, 0, sizeof *dt);
1163
if (!ops->get_segment(ctxt, &sel, &desc, NULL, VCPU_SREG_LDTR))
1166
dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1167
dt->address = get_desc_base(&desc);
1169
ops->get_gdt(ctxt, dt);
1172
/* allowed just for 8 bytes segments */
1173
static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1174
u16 selector, struct desc_struct *desc)
1177
u16 index = selector >> 3;
1180
get_descriptor_table_ptr(ctxt, selector, &dt);
1182
if (dt.size < index * 8 + 7)
1183
return emulate_gp(ctxt, selector & 0xfffc);
1185
addr = dt.address + index * 8;
1186
return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1190
/* allowed just for 8 bytes segments */
1191
static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1192
u16 selector, struct desc_struct *desc)
1195
u16 index = selector >> 3;
1198
get_descriptor_table_ptr(ctxt, selector, &dt);
1200
if (dt.size < index * 8 + 7)
1201
return emulate_gp(ctxt, selector & 0xfffc);
1203
addr = dt.address + index * 8;
1204
return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1208
/* Does not support long mode */
1209
static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1210
u16 selector, int seg)
1212
struct desc_struct seg_desc;
1214
unsigned err_vec = GP_VECTOR;
1216
bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1219
memset(&seg_desc, 0, sizeof seg_desc);
1221
if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1222
|| ctxt->mode == X86EMUL_MODE_REAL) {
1223
/* set real mode segment descriptor */
1224
set_desc_base(&seg_desc, selector << 4);
1225
set_desc_limit(&seg_desc, 0xffff);
1232
/* NULL selector is not valid for TR, CS and SS */
1233
if ((seg == VCPU_SREG_CS || seg == VCPU_SREG_SS || seg == VCPU_SREG_TR)
1237
/* TR should be in GDT only */
1238
if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1241
if (null_selector) /* for NULL selector skip all following checks */
1244
ret = read_segment_descriptor(ctxt, selector, &seg_desc);
1245
if (ret != X86EMUL_CONTINUE)
1248
err_code = selector & 0xfffc;
1249
err_vec = GP_VECTOR;
1251
/* can't load system descriptor into segment selecor */
1252
if (seg <= VCPU_SREG_GS && !seg_desc.s)
1256
err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1262
cpl = ctxt->ops->cpl(ctxt);
1267
* segment is not a writable data segment or segment
1268
* selector's RPL != CPL or segment selector's RPL != CPL
1270
if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1274
if (!(seg_desc.type & 8))
1277
if (seg_desc.type & 4) {
1283
if (rpl > cpl || dpl != cpl)
1286
/* CS(RPL) <- CPL */
1287
selector = (selector & 0xfffc) | cpl;
1290
if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1293
case VCPU_SREG_LDTR:
1294
if (seg_desc.s || seg_desc.type != 2)
1297
default: /* DS, ES, FS, or GS */
1299
* segment is not a data or readable code segment or
1300
* ((segment is a data or nonconforming code segment)
1301
* and (both RPL and CPL > DPL))
1303
if ((seg_desc.type & 0xa) == 0x8 ||
1304
(((seg_desc.type & 0xc) != 0xc) &&
1305
(rpl > dpl && cpl > dpl)))
1311
/* mark segment as accessed */
1313
ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1314
if (ret != X86EMUL_CONTINUE)
1318
ctxt->ops->set_segment(ctxt, selector, &seg_desc, 0, seg);
1319
return X86EMUL_CONTINUE;
1321
emulate_exception(ctxt, err_vec, err_code, true);
1322
return X86EMUL_PROPAGATE_FAULT;
1325
static void write_register_operand(struct operand *op)
1327
/* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1328
switch (op->bytes) {
1330
*(u8 *)op->addr.reg = (u8)op->val;
1333
*(u16 *)op->addr.reg = (u16)op->val;
1336
*op->addr.reg = (u32)op->val;
1337
break; /* 64b: zero-extend */
1339
*op->addr.reg = op->val;
1344
static int writeback(struct x86_emulate_ctxt *ctxt)
1348
switch (ctxt->dst.type) {
1350
write_register_operand(&ctxt->dst);
1353
if (ctxt->lock_prefix)
1354
rc = segmented_cmpxchg(ctxt,
1356
&ctxt->dst.orig_val,
1360
rc = segmented_write(ctxt,
1364
if (rc != X86EMUL_CONTINUE)
1368
write_sse_reg(ctxt, &ctxt->dst.vec_val, ctxt->dst.addr.xmm);
1376
return X86EMUL_CONTINUE;
1379
static int em_push(struct x86_emulate_ctxt *ctxt)
1381
struct segmented_address addr;
1383
register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], -ctxt->op_bytes);
1384
addr.ea = register_address(ctxt, ctxt->regs[VCPU_REGS_RSP]);
1385
addr.seg = VCPU_SREG_SS;
1387
/* Disable writeback. */
1388
ctxt->dst.type = OP_NONE;
1389
return segmented_write(ctxt, addr, &ctxt->src.val, ctxt->op_bytes);
1392
static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1393
void *dest, int len)
1396
struct segmented_address addr;
1398
addr.ea = register_address(ctxt, ctxt->regs[VCPU_REGS_RSP]);
1399
addr.seg = VCPU_SREG_SS;
1400
rc = segmented_read(ctxt, addr, dest, len);
1401
if (rc != X86EMUL_CONTINUE)
1404
register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], len);
1408
static int em_pop(struct x86_emulate_ctxt *ctxt)
1410
return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1413
static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1414
void *dest, int len)
1417
unsigned long val, change_mask;
1418
int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1419
int cpl = ctxt->ops->cpl(ctxt);
1421
rc = emulate_pop(ctxt, &val, len);
1422
if (rc != X86EMUL_CONTINUE)
1425
change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1426
| EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1428
switch(ctxt->mode) {
1429
case X86EMUL_MODE_PROT64:
1430
case X86EMUL_MODE_PROT32:
1431
case X86EMUL_MODE_PROT16:
1433
change_mask |= EFLG_IOPL;
1435
change_mask |= EFLG_IF;
1437
case X86EMUL_MODE_VM86:
1439
return emulate_gp(ctxt, 0);
1440
change_mask |= EFLG_IF;
1442
default: /* real mode */
1443
change_mask |= (EFLG_IOPL | EFLG_IF);
1447
*(unsigned long *)dest =
1448
(ctxt->eflags & ~change_mask) | (val & change_mask);
1453
static int em_popf(struct x86_emulate_ctxt *ctxt)
1455
ctxt->dst.type = OP_REG;
1456
ctxt->dst.addr.reg = &ctxt->eflags;
1457
ctxt->dst.bytes = ctxt->op_bytes;
1458
return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1461
static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1463
int seg = ctxt->src2.val;
1465
ctxt->src.val = get_segment_selector(ctxt, seg);
1467
return em_push(ctxt);
1470
static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1472
int seg = ctxt->src2.val;
1473
unsigned long selector;
1476
rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1477
if (rc != X86EMUL_CONTINUE)
1480
rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1484
static int em_pusha(struct x86_emulate_ctxt *ctxt)
1486
unsigned long old_esp = ctxt->regs[VCPU_REGS_RSP];
1487
int rc = X86EMUL_CONTINUE;
1488
int reg = VCPU_REGS_RAX;
1490
while (reg <= VCPU_REGS_RDI) {
1491
(reg == VCPU_REGS_RSP) ?
1492
(ctxt->src.val = old_esp) : (ctxt->src.val = ctxt->regs[reg]);
1495
if (rc != X86EMUL_CONTINUE)
1504
static int em_pushf(struct x86_emulate_ctxt *ctxt)
1506
ctxt->src.val = (unsigned long)ctxt->eflags;
1507
return em_push(ctxt);
1510
static int em_popa(struct x86_emulate_ctxt *ctxt)
1512
int rc = X86EMUL_CONTINUE;
1513
int reg = VCPU_REGS_RDI;
1515
while (reg >= VCPU_REGS_RAX) {
1516
if (reg == VCPU_REGS_RSP) {
1517
register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP],
1522
rc = emulate_pop(ctxt, &ctxt->regs[reg], ctxt->op_bytes);
1523
if (rc != X86EMUL_CONTINUE)
1530
int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1532
struct x86_emulate_ops *ops = ctxt->ops;
1539
/* TODO: Add limit checks */
1540
ctxt->src.val = ctxt->eflags;
1542
if (rc != X86EMUL_CONTINUE)
1545
ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1547
ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1549
if (rc != X86EMUL_CONTINUE)
1552
ctxt->src.val = ctxt->_eip;
1554
if (rc != X86EMUL_CONTINUE)
1557
ops->get_idt(ctxt, &dt);
1559
eip_addr = dt.address + (irq << 2);
1560
cs_addr = dt.address + (irq << 2) + 2;
1562
rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1563
if (rc != X86EMUL_CONTINUE)
1566
rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1567
if (rc != X86EMUL_CONTINUE)
1570
rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1571
if (rc != X86EMUL_CONTINUE)
1579
static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1581
switch(ctxt->mode) {
1582
case X86EMUL_MODE_REAL:
1583
return emulate_int_real(ctxt, irq);
1584
case X86EMUL_MODE_VM86:
1585
case X86EMUL_MODE_PROT16:
1586
case X86EMUL_MODE_PROT32:
1587
case X86EMUL_MODE_PROT64:
1589
/* Protected mode interrupts unimplemented yet */
1590
return X86EMUL_UNHANDLEABLE;
1594
static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
1596
int rc = X86EMUL_CONTINUE;
1597
unsigned long temp_eip = 0;
1598
unsigned long temp_eflags = 0;
1599
unsigned long cs = 0;
1600
unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1601
EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1602
EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1603
unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1605
/* TODO: Add stack limit check */
1607
rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
1609
if (rc != X86EMUL_CONTINUE)
1612
if (temp_eip & ~0xffff)
1613
return emulate_gp(ctxt, 0);
1615
rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1617
if (rc != X86EMUL_CONTINUE)
1620
rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
1622
if (rc != X86EMUL_CONTINUE)
1625
rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1627
if (rc != X86EMUL_CONTINUE)
1630
ctxt->_eip = temp_eip;
1633
if (ctxt->op_bytes == 4)
1634
ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1635
else if (ctxt->op_bytes == 2) {
1636
ctxt->eflags &= ~0xffff;
1637
ctxt->eflags |= temp_eflags;
1640
ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1641
ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1646
static int em_iret(struct x86_emulate_ctxt *ctxt)
1648
switch(ctxt->mode) {
1649
case X86EMUL_MODE_REAL:
1650
return emulate_iret_real(ctxt);
1651
case X86EMUL_MODE_VM86:
1652
case X86EMUL_MODE_PROT16:
1653
case X86EMUL_MODE_PROT32:
1654
case X86EMUL_MODE_PROT64:
1656
/* iret from protected mode unimplemented yet */
1657
return X86EMUL_UNHANDLEABLE;
1661
static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
1666
memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1668
rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
1669
if (rc != X86EMUL_CONTINUE)
1673
memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
1674
return X86EMUL_CONTINUE;
1677
static int em_grp1a(struct x86_emulate_ctxt *ctxt)
1679
return emulate_pop(ctxt, &ctxt->dst.val, ctxt->dst.bytes);
1682
static int em_grp2(struct x86_emulate_ctxt *ctxt)
1684
switch (ctxt->modrm_reg) {
1686
emulate_2op_SrcB(ctxt, "rol");
1689
emulate_2op_SrcB(ctxt, "ror");
1692
emulate_2op_SrcB(ctxt, "rcl");
1695
emulate_2op_SrcB(ctxt, "rcr");
1697
case 4: /* sal/shl */
1698
case 6: /* sal/shl */
1699
emulate_2op_SrcB(ctxt, "sal");
1702
emulate_2op_SrcB(ctxt, "shr");
1705
emulate_2op_SrcB(ctxt, "sar");
1708
return X86EMUL_CONTINUE;
1711
static int em_not(struct x86_emulate_ctxt *ctxt)
1713
ctxt->dst.val = ~ctxt->dst.val;
1714
return X86EMUL_CONTINUE;
1717
static int em_neg(struct x86_emulate_ctxt *ctxt)
1719
emulate_1op(ctxt, "neg");
1720
return X86EMUL_CONTINUE;
1723
static int em_mul_ex(struct x86_emulate_ctxt *ctxt)
1727
emulate_1op_rax_rdx(ctxt, "mul", ex);
1728
return X86EMUL_CONTINUE;
1731
static int em_imul_ex(struct x86_emulate_ctxt *ctxt)
1735
emulate_1op_rax_rdx(ctxt, "imul", ex);
1736
return X86EMUL_CONTINUE;
1739
static int em_div_ex(struct x86_emulate_ctxt *ctxt)
1743
emulate_1op_rax_rdx(ctxt, "div", de);
1745
return emulate_de(ctxt);
1746
return X86EMUL_CONTINUE;
1749
static int em_idiv_ex(struct x86_emulate_ctxt *ctxt)
1753
emulate_1op_rax_rdx(ctxt, "idiv", de);
1755
return emulate_de(ctxt);
1756
return X86EMUL_CONTINUE;
1759
static int em_grp45(struct x86_emulate_ctxt *ctxt)
1761
int rc = X86EMUL_CONTINUE;
1763
switch (ctxt->modrm_reg) {
1765
emulate_1op(ctxt, "inc");
1768
emulate_1op(ctxt, "dec");
1770
case 2: /* call near abs */ {
1772
old_eip = ctxt->_eip;
1773
ctxt->_eip = ctxt->src.val;
1774
ctxt->src.val = old_eip;
1778
case 4: /* jmp abs */
1779
ctxt->_eip = ctxt->src.val;
1781
case 5: /* jmp far */
1782
rc = em_jmp_far(ctxt);
1791
static int em_grp9(struct x86_emulate_ctxt *ctxt)
1793
u64 old = ctxt->dst.orig_val64;
1795
if (((u32) (old >> 0) != (u32) ctxt->regs[VCPU_REGS_RAX]) ||
1796
((u32) (old >> 32) != (u32) ctxt->regs[VCPU_REGS_RDX])) {
1797
ctxt->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1798
ctxt->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1799
ctxt->eflags &= ~EFLG_ZF;
1801
ctxt->dst.val64 = ((u64)ctxt->regs[VCPU_REGS_RCX] << 32) |
1802
(u32) ctxt->regs[VCPU_REGS_RBX];
1804
ctxt->eflags |= EFLG_ZF;
1806
return X86EMUL_CONTINUE;
1809
static int em_ret(struct x86_emulate_ctxt *ctxt)
1811
ctxt->dst.type = OP_REG;
1812
ctxt->dst.addr.reg = &ctxt->_eip;
1813
ctxt->dst.bytes = ctxt->op_bytes;
1814
return em_pop(ctxt);
1817
static int em_ret_far(struct x86_emulate_ctxt *ctxt)
1822
rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
1823
if (rc != X86EMUL_CONTINUE)
1825
if (ctxt->op_bytes == 4)
1826
ctxt->_eip = (u32)ctxt->_eip;
1827
rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1828
if (rc != X86EMUL_CONTINUE)
1830
rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1834
static int em_lseg(struct x86_emulate_ctxt *ctxt)
1836
int seg = ctxt->src2.val;
1840
memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1842
rc = load_segment_descriptor(ctxt, sel, seg);
1843
if (rc != X86EMUL_CONTINUE)
1846
ctxt->dst.val = ctxt->src.val;
1851
setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1852
struct desc_struct *cs, struct desc_struct *ss)
1856
memset(cs, 0, sizeof(struct desc_struct));
1857
ctxt->ops->get_segment(ctxt, &selector, cs, NULL, VCPU_SREG_CS);
1858
memset(ss, 0, sizeof(struct desc_struct));
1860
cs->l = 0; /* will be adjusted later */
1861
set_desc_base(cs, 0); /* flat segment */
1862
cs->g = 1; /* 4kb granularity */
1863
set_desc_limit(cs, 0xfffff); /* 4GB limit */
1864
cs->type = 0x0b; /* Read, Execute, Accessed */
1866
cs->dpl = 0; /* will be adjusted later */
1870
set_desc_base(ss, 0); /* flat segment */
1871
set_desc_limit(ss, 0xfffff); /* 4GB limit */
1872
ss->g = 1; /* 4kb granularity */
1874
ss->type = 0x03; /* Read/Write, Accessed */
1875
ss->d = 1; /* 32bit stack segment */
1880
static int em_syscall(struct x86_emulate_ctxt *ctxt)
1882
struct x86_emulate_ops *ops = ctxt->ops;
1883
struct desc_struct cs, ss;
1888
/* syscall is not available in real mode */
1889
if (ctxt->mode == X86EMUL_MODE_REAL ||
1890
ctxt->mode == X86EMUL_MODE_VM86)
1891
return emulate_ud(ctxt);
1893
ops->get_msr(ctxt, MSR_EFER, &efer);
1894
setup_syscalls_segments(ctxt, &cs, &ss);
1896
ops->get_msr(ctxt, MSR_STAR, &msr_data);
1898
cs_sel = (u16)(msr_data & 0xfffc);
1899
ss_sel = (u16)(msr_data + 8);
1901
if (efer & EFER_LMA) {
1905
ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
1906
ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
1908
ctxt->regs[VCPU_REGS_RCX] = ctxt->_eip;
1909
if (efer & EFER_LMA) {
1910
#ifdef CONFIG_X86_64
1911
ctxt->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
1914
ctxt->mode == X86EMUL_MODE_PROT64 ?
1915
MSR_LSTAR : MSR_CSTAR, &msr_data);
1916
ctxt->_eip = msr_data;
1918
ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
1919
ctxt->eflags &= ~(msr_data | EFLG_RF);
1923
ops->get_msr(ctxt, MSR_STAR, &msr_data);
1924
ctxt->_eip = (u32)msr_data;
1926
ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1929
return X86EMUL_CONTINUE;
1932
static int em_sysenter(struct x86_emulate_ctxt *ctxt)
1934
struct x86_emulate_ops *ops = ctxt->ops;
1935
struct desc_struct cs, ss;
1940
ops->get_msr(ctxt, MSR_EFER, &efer);
1941
/* inject #GP if in real mode */
1942
if (ctxt->mode == X86EMUL_MODE_REAL)
1943
return emulate_gp(ctxt, 0);
1945
/* XXX sysenter/sysexit have not been tested in 64bit mode.
1946
* Therefore, we inject an #UD.
1948
if (ctxt->mode == X86EMUL_MODE_PROT64)
1949
return emulate_ud(ctxt);
1951
setup_syscalls_segments(ctxt, &cs, &ss);
1953
ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
1954
switch (ctxt->mode) {
1955
case X86EMUL_MODE_PROT32:
1956
if ((msr_data & 0xfffc) == 0x0)
1957
return emulate_gp(ctxt, 0);
1959
case X86EMUL_MODE_PROT64:
1960
if (msr_data == 0x0)
1961
return emulate_gp(ctxt, 0);
1965
ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1966
cs_sel = (u16)msr_data;
1967
cs_sel &= ~SELECTOR_RPL_MASK;
1968
ss_sel = cs_sel + 8;
1969
ss_sel &= ~SELECTOR_RPL_MASK;
1970
if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
1975
ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
1976
ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
1978
ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
1979
ctxt->_eip = msr_data;
1981
ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
1982
ctxt->regs[VCPU_REGS_RSP] = msr_data;
1984
return X86EMUL_CONTINUE;
1987
static int em_sysexit(struct x86_emulate_ctxt *ctxt)
1989
struct x86_emulate_ops *ops = ctxt->ops;
1990
struct desc_struct cs, ss;
1993
u16 cs_sel = 0, ss_sel = 0;
1995
/* inject #GP if in real mode or Virtual 8086 mode */
1996
if (ctxt->mode == X86EMUL_MODE_REAL ||
1997
ctxt->mode == X86EMUL_MODE_VM86)
1998
return emulate_gp(ctxt, 0);
2000
setup_syscalls_segments(ctxt, &cs, &ss);
2002
if ((ctxt->rex_prefix & 0x8) != 0x0)
2003
usermode = X86EMUL_MODE_PROT64;
2005
usermode = X86EMUL_MODE_PROT32;
2009
ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2011
case X86EMUL_MODE_PROT32:
2012
cs_sel = (u16)(msr_data + 16);
2013
if ((msr_data & 0xfffc) == 0x0)
2014
return emulate_gp(ctxt, 0);
2015
ss_sel = (u16)(msr_data + 24);
2017
case X86EMUL_MODE_PROT64:
2018
cs_sel = (u16)(msr_data + 32);
2019
if (msr_data == 0x0)
2020
return emulate_gp(ctxt, 0);
2021
ss_sel = cs_sel + 8;
2026
cs_sel |= SELECTOR_RPL_MASK;
2027
ss_sel |= SELECTOR_RPL_MASK;
2029
ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2030
ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2032
ctxt->_eip = ctxt->regs[VCPU_REGS_RDX];
2033
ctxt->regs[VCPU_REGS_RSP] = ctxt->regs[VCPU_REGS_RCX];
2035
return X86EMUL_CONTINUE;
2038
static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2041
if (ctxt->mode == X86EMUL_MODE_REAL)
2043
if (ctxt->mode == X86EMUL_MODE_VM86)
2045
iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2046
return ctxt->ops->cpl(ctxt) > iopl;
2049
static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2052
struct x86_emulate_ops *ops = ctxt->ops;
2053
struct desc_struct tr_seg;
2056
u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2057
unsigned mask = (1 << len) - 1;
2060
ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2063
if (desc_limit_scaled(&tr_seg) < 103)
2065
base = get_desc_base(&tr_seg);
2066
#ifdef CONFIG_X86_64
2067
base |= ((u64)base3) << 32;
2069
r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2070
if (r != X86EMUL_CONTINUE)
2072
if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2074
r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2075
if (r != X86EMUL_CONTINUE)
2077
if ((perm >> bit_idx) & mask)
2082
static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2088
if (emulator_bad_iopl(ctxt))
2089
if (!emulator_io_port_access_allowed(ctxt, port, len))
2092
ctxt->perm_ok = true;
2097
static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2098
struct tss_segment_16 *tss)
2100
tss->ip = ctxt->_eip;
2101
tss->flag = ctxt->eflags;
2102
tss->ax = ctxt->regs[VCPU_REGS_RAX];
2103
tss->cx = ctxt->regs[VCPU_REGS_RCX];
2104
tss->dx = ctxt->regs[VCPU_REGS_RDX];
2105
tss->bx = ctxt->regs[VCPU_REGS_RBX];
2106
tss->sp = ctxt->regs[VCPU_REGS_RSP];
2107
tss->bp = ctxt->regs[VCPU_REGS_RBP];
2108
tss->si = ctxt->regs[VCPU_REGS_RSI];
2109
tss->di = ctxt->regs[VCPU_REGS_RDI];
2111
tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2112
tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2113
tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2114
tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2115
tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2118
static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2119
struct tss_segment_16 *tss)
2123
ctxt->_eip = tss->ip;
2124
ctxt->eflags = tss->flag | 2;
2125
ctxt->regs[VCPU_REGS_RAX] = tss->ax;
2126
ctxt->regs[VCPU_REGS_RCX] = tss->cx;
2127
ctxt->regs[VCPU_REGS_RDX] = tss->dx;
2128
ctxt->regs[VCPU_REGS_RBX] = tss->bx;
2129
ctxt->regs[VCPU_REGS_RSP] = tss->sp;
2130
ctxt->regs[VCPU_REGS_RBP] = tss->bp;
2131
ctxt->regs[VCPU_REGS_RSI] = tss->si;
2132
ctxt->regs[VCPU_REGS_RDI] = tss->di;
2135
* SDM says that segment selectors are loaded before segment
2138
set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2139
set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2140
set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2141
set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2142
set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2145
* Now load segment descriptors. If fault happenes at this stage
2146
* it is handled in a context of new task
2148
ret = load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR);
2149
if (ret != X86EMUL_CONTINUE)
2151
ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2152
if (ret != X86EMUL_CONTINUE)
2154
ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2155
if (ret != X86EMUL_CONTINUE)
2157
ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2158
if (ret != X86EMUL_CONTINUE)
2160
ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2161
if (ret != X86EMUL_CONTINUE)
2164
return X86EMUL_CONTINUE;
2167
static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2168
u16 tss_selector, u16 old_tss_sel,
2169
ulong old_tss_base, struct desc_struct *new_desc)
2171
struct x86_emulate_ops *ops = ctxt->ops;
2172
struct tss_segment_16 tss_seg;
2174
u32 new_tss_base = get_desc_base(new_desc);
2176
ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2178
if (ret != X86EMUL_CONTINUE)
2179
/* FIXME: need to provide precise fault address */
2182
save_state_to_tss16(ctxt, &tss_seg);
2184
ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2186
if (ret != X86EMUL_CONTINUE)
2187
/* FIXME: need to provide precise fault address */
2190
ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2192
if (ret != X86EMUL_CONTINUE)
2193
/* FIXME: need to provide precise fault address */
2196
if (old_tss_sel != 0xffff) {
2197
tss_seg.prev_task_link = old_tss_sel;
2199
ret = ops->write_std(ctxt, new_tss_base,
2200
&tss_seg.prev_task_link,
2201
sizeof tss_seg.prev_task_link,
2203
if (ret != X86EMUL_CONTINUE)
2204
/* FIXME: need to provide precise fault address */
2208
return load_state_from_tss16(ctxt, &tss_seg);
2211
static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2212
struct tss_segment_32 *tss)
2214
tss->cr3 = ctxt->ops->get_cr(ctxt, 3);
2215
tss->eip = ctxt->_eip;
2216
tss->eflags = ctxt->eflags;
2217
tss->eax = ctxt->regs[VCPU_REGS_RAX];
2218
tss->ecx = ctxt->regs[VCPU_REGS_RCX];
2219
tss->edx = ctxt->regs[VCPU_REGS_RDX];
2220
tss->ebx = ctxt->regs[VCPU_REGS_RBX];
2221
tss->esp = ctxt->regs[VCPU_REGS_RSP];
2222
tss->ebp = ctxt->regs[VCPU_REGS_RBP];
2223
tss->esi = ctxt->regs[VCPU_REGS_RSI];
2224
tss->edi = ctxt->regs[VCPU_REGS_RDI];
2226
tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2227
tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2228
tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2229
tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2230
tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2231
tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2232
tss->ldt_selector = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2235
static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2236
struct tss_segment_32 *tss)
2240
if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2241
return emulate_gp(ctxt, 0);
2242
ctxt->_eip = tss->eip;
2243
ctxt->eflags = tss->eflags | 2;
2244
ctxt->regs[VCPU_REGS_RAX] = tss->eax;
2245
ctxt->regs[VCPU_REGS_RCX] = tss->ecx;
2246
ctxt->regs[VCPU_REGS_RDX] = tss->edx;
2247
ctxt->regs[VCPU_REGS_RBX] = tss->ebx;
2248
ctxt->regs[VCPU_REGS_RSP] = tss->esp;
2249
ctxt->regs[VCPU_REGS_RBP] = tss->ebp;
2250
ctxt->regs[VCPU_REGS_RSI] = tss->esi;
2251
ctxt->regs[VCPU_REGS_RDI] = tss->edi;
2254
* SDM says that segment selectors are loaded before segment
2257
set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2258
set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2259
set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2260
set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2261
set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2262
set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2263
set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2266
* Now load segment descriptors. If fault happenes at this stage
2267
* it is handled in a context of new task
2269
ret = load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2270
if (ret != X86EMUL_CONTINUE)
2272
ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2273
if (ret != X86EMUL_CONTINUE)
2275
ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2276
if (ret != X86EMUL_CONTINUE)
2278
ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2279
if (ret != X86EMUL_CONTINUE)
2281
ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2282
if (ret != X86EMUL_CONTINUE)
2284
ret = load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS);
2285
if (ret != X86EMUL_CONTINUE)
2287
ret = load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS);
2288
if (ret != X86EMUL_CONTINUE)
2291
return X86EMUL_CONTINUE;
2294
static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2295
u16 tss_selector, u16 old_tss_sel,
2296
ulong old_tss_base, struct desc_struct *new_desc)
2298
struct x86_emulate_ops *ops = ctxt->ops;
2299
struct tss_segment_32 tss_seg;
2301
u32 new_tss_base = get_desc_base(new_desc);
2303
ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2305
if (ret != X86EMUL_CONTINUE)
2306
/* FIXME: need to provide precise fault address */
2309
save_state_to_tss32(ctxt, &tss_seg);
2311
ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2313
if (ret != X86EMUL_CONTINUE)
2314
/* FIXME: need to provide precise fault address */
2317
ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2319
if (ret != X86EMUL_CONTINUE)
2320
/* FIXME: need to provide precise fault address */
2323
if (old_tss_sel != 0xffff) {
2324
tss_seg.prev_task_link = old_tss_sel;
2326
ret = ops->write_std(ctxt, new_tss_base,
2327
&tss_seg.prev_task_link,
2328
sizeof tss_seg.prev_task_link,
2330
if (ret != X86EMUL_CONTINUE)
2331
/* FIXME: need to provide precise fault address */
2335
return load_state_from_tss32(ctxt, &tss_seg);
2338
static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2339
u16 tss_selector, int reason,
2340
bool has_error_code, u32 error_code)
2342
struct x86_emulate_ops *ops = ctxt->ops;
2343
struct desc_struct curr_tss_desc, next_tss_desc;
2345
u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2346
ulong old_tss_base =
2347
ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2350
/* FIXME: old_tss_base == ~0 ? */
2352
ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2353
if (ret != X86EMUL_CONTINUE)
2355
ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2356
if (ret != X86EMUL_CONTINUE)
2359
/* FIXME: check that next_tss_desc is tss */
2361
if (reason != TASK_SWITCH_IRET) {
2362
if ((tss_selector & 3) > next_tss_desc.dpl ||
2363
ops->cpl(ctxt) > next_tss_desc.dpl)
2364
return emulate_gp(ctxt, 0);
2367
desc_limit = desc_limit_scaled(&next_tss_desc);
2368
if (!next_tss_desc.p ||
2369
((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2370
desc_limit < 0x2b)) {
2371
emulate_ts(ctxt, tss_selector & 0xfffc);
2372
return X86EMUL_PROPAGATE_FAULT;
2375
if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2376
curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2377
write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2380
if (reason == TASK_SWITCH_IRET)
2381
ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2383
/* set back link to prev task only if NT bit is set in eflags
2384
note that old_tss_sel is not used afetr this point */
2385
if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2386
old_tss_sel = 0xffff;
2388
if (next_tss_desc.type & 8)
2389
ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2390
old_tss_base, &next_tss_desc);
2392
ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2393
old_tss_base, &next_tss_desc);
2394
if (ret != X86EMUL_CONTINUE)
2397
if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2398
ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2400
if (reason != TASK_SWITCH_IRET) {
2401
next_tss_desc.type |= (1 << 1); /* set busy flag */
2402
write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2405
ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2406
ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2408
if (has_error_code) {
2409
ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2410
ctxt->lock_prefix = 0;
2411
ctxt->src.val = (unsigned long) error_code;
2412
ret = em_push(ctxt);
2418
int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2419
u16 tss_selector, int reason,
2420
bool has_error_code, u32 error_code)
2424
ctxt->_eip = ctxt->eip;
2425
ctxt->dst.type = OP_NONE;
2427
rc = emulator_do_task_switch(ctxt, tss_selector, reason,
2428
has_error_code, error_code);
2430
if (rc == X86EMUL_CONTINUE)
2431
ctxt->eip = ctxt->_eip;
2433
return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2436
static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned seg,
2437
int reg, struct operand *op)
2439
int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;
2441
register_address_increment(ctxt, &ctxt->regs[reg], df * op->bytes);
2442
op->addr.mem.ea = register_address(ctxt, ctxt->regs[reg]);
2443
op->addr.mem.seg = seg;
2446
static int em_das(struct x86_emulate_ctxt *ctxt)
2449
bool af, cf, old_cf;
2451
cf = ctxt->eflags & X86_EFLAGS_CF;
2457
af = ctxt->eflags & X86_EFLAGS_AF;
2458
if ((al & 0x0f) > 9 || af) {
2460
cf = old_cf | (al >= 250);
2465
if (old_al > 0x99 || old_cf) {
2471
/* Set PF, ZF, SF */
2472
ctxt->src.type = OP_IMM;
2474
ctxt->src.bytes = 1;
2475
emulate_2op_SrcV(ctxt, "or");
2476
ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2478
ctxt->eflags |= X86_EFLAGS_CF;
2480
ctxt->eflags |= X86_EFLAGS_AF;
2481
return X86EMUL_CONTINUE;
2484
static int em_call_far(struct x86_emulate_ctxt *ctxt)
2490
old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2491
old_eip = ctxt->_eip;
2493
memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2494
if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
2495
return X86EMUL_CONTINUE;
2498
memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2500
ctxt->src.val = old_cs;
2502
if (rc != X86EMUL_CONTINUE)
2505
ctxt->src.val = old_eip;
2506
return em_push(ctxt);
2509
static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2513
ctxt->dst.type = OP_REG;
2514
ctxt->dst.addr.reg = &ctxt->_eip;
2515
ctxt->dst.bytes = ctxt->op_bytes;
2516
rc = emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
2517
if (rc != X86EMUL_CONTINUE)
2519
register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], ctxt->src.val);
2520
return X86EMUL_CONTINUE;
2523
static int em_add(struct x86_emulate_ctxt *ctxt)
2525
emulate_2op_SrcV(ctxt, "add");
2526
return X86EMUL_CONTINUE;
2529
static int em_or(struct x86_emulate_ctxt *ctxt)
2531
emulate_2op_SrcV(ctxt, "or");
2532
return X86EMUL_CONTINUE;
2535
static int em_adc(struct x86_emulate_ctxt *ctxt)
2537
emulate_2op_SrcV(ctxt, "adc");
2538
return X86EMUL_CONTINUE;
2541
static int em_sbb(struct x86_emulate_ctxt *ctxt)
2543
emulate_2op_SrcV(ctxt, "sbb");
2544
return X86EMUL_CONTINUE;
2547
static int em_and(struct x86_emulate_ctxt *ctxt)
2549
emulate_2op_SrcV(ctxt, "and");
2550
return X86EMUL_CONTINUE;
2553
static int em_sub(struct x86_emulate_ctxt *ctxt)
2555
emulate_2op_SrcV(ctxt, "sub");
2556
return X86EMUL_CONTINUE;
2559
static int em_xor(struct x86_emulate_ctxt *ctxt)
2561
emulate_2op_SrcV(ctxt, "xor");
2562
return X86EMUL_CONTINUE;
2565
static int em_cmp(struct x86_emulate_ctxt *ctxt)
2567
emulate_2op_SrcV(ctxt, "cmp");
2568
/* Disable writeback. */
2569
ctxt->dst.type = OP_NONE;
2570
return X86EMUL_CONTINUE;
2573
static int em_test(struct x86_emulate_ctxt *ctxt)
2575
emulate_2op_SrcV(ctxt, "test");
2576
/* Disable writeback. */
2577
ctxt->dst.type = OP_NONE;
2578
return X86EMUL_CONTINUE;
2581
static int em_xchg(struct x86_emulate_ctxt *ctxt)
2583
/* Write back the register source. */
2584
ctxt->src.val = ctxt->dst.val;
2585
write_register_operand(&ctxt->src);
2587
/* Write back the memory destination with implicit LOCK prefix. */
2588
ctxt->dst.val = ctxt->src.orig_val;
2589
ctxt->lock_prefix = 1;
2590
return X86EMUL_CONTINUE;
2593
static int em_imul(struct x86_emulate_ctxt *ctxt)
2595
emulate_2op_SrcV_nobyte(ctxt, "imul");
2596
return X86EMUL_CONTINUE;
2599
static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2601
ctxt->dst.val = ctxt->src2.val;
2602
return em_imul(ctxt);
2605
static int em_cwd(struct x86_emulate_ctxt *ctxt)
2607
ctxt->dst.type = OP_REG;
2608
ctxt->dst.bytes = ctxt->src.bytes;
2609
ctxt->dst.addr.reg = &ctxt->regs[VCPU_REGS_RDX];
2610
ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
2612
return X86EMUL_CONTINUE;
2615
static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2619
ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
2620
ctxt->regs[VCPU_REGS_RAX] = (u32)tsc;
2621
ctxt->regs[VCPU_REGS_RDX] = tsc >> 32;
2622
return X86EMUL_CONTINUE;
2625
static int em_mov(struct x86_emulate_ctxt *ctxt)
2627
ctxt->dst.val = ctxt->src.val;
2628
return X86EMUL_CONTINUE;
2631
static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
2633
if (ctxt->modrm_reg > VCPU_SREG_GS)
2634
return emulate_ud(ctxt);
2636
ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
2637
return X86EMUL_CONTINUE;
2640
static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
2642
u16 sel = ctxt->src.val;
2644
if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
2645
return emulate_ud(ctxt);
2647
if (ctxt->modrm_reg == VCPU_SREG_SS)
2648
ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
2650
/* Disable writeback. */
2651
ctxt->dst.type = OP_NONE;
2652
return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
2655
static int em_movdqu(struct x86_emulate_ctxt *ctxt)
2657
memcpy(&ctxt->dst.vec_val, &ctxt->src.vec_val, ctxt->op_bytes);
2658
return X86EMUL_CONTINUE;
2661
static int em_invlpg(struct x86_emulate_ctxt *ctxt)
2666
rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
2667
if (rc == X86EMUL_CONTINUE)
2668
ctxt->ops->invlpg(ctxt, linear);
2669
/* Disable writeback. */
2670
ctxt->dst.type = OP_NONE;
2671
return X86EMUL_CONTINUE;
2674
static int em_clts(struct x86_emulate_ctxt *ctxt)
2678
cr0 = ctxt->ops->get_cr(ctxt, 0);
2680
ctxt->ops->set_cr(ctxt, 0, cr0);
2681
return X86EMUL_CONTINUE;
2684
static int em_vmcall(struct x86_emulate_ctxt *ctxt)
2688
if (ctxt->modrm_mod != 3 || ctxt->modrm_rm != 1)
2689
return X86EMUL_UNHANDLEABLE;
2691
rc = ctxt->ops->fix_hypercall(ctxt);
2692
if (rc != X86EMUL_CONTINUE)
2695
/* Let the processor re-execute the fixed hypercall */
2696
ctxt->_eip = ctxt->eip;
2697
/* Disable writeback. */
2698
ctxt->dst.type = OP_NONE;
2699
return X86EMUL_CONTINUE;
2702
static int em_lgdt(struct x86_emulate_ctxt *ctxt)
2704
struct desc_ptr desc_ptr;
2707
rc = read_descriptor(ctxt, ctxt->src.addr.mem,
2708
&desc_ptr.size, &desc_ptr.address,
2710
if (rc != X86EMUL_CONTINUE)
2712
ctxt->ops->set_gdt(ctxt, &desc_ptr);
2713
/* Disable writeback. */
2714
ctxt->dst.type = OP_NONE;
2715
return X86EMUL_CONTINUE;
2718
static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
2722
rc = ctxt->ops->fix_hypercall(ctxt);
2724
/* Disable writeback. */
2725
ctxt->dst.type = OP_NONE;
2729
static int em_lidt(struct x86_emulate_ctxt *ctxt)
2731
struct desc_ptr desc_ptr;
2734
rc = read_descriptor(ctxt, ctxt->src.addr.mem,
2735
&desc_ptr.size, &desc_ptr.address,
2737
if (rc != X86EMUL_CONTINUE)
2739
ctxt->ops->set_idt(ctxt, &desc_ptr);
2740
/* Disable writeback. */
2741
ctxt->dst.type = OP_NONE;
2742
return X86EMUL_CONTINUE;
2745
static int em_smsw(struct x86_emulate_ctxt *ctxt)
2747
ctxt->dst.bytes = 2;
2748
ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
2749
return X86EMUL_CONTINUE;
2752
static int em_lmsw(struct x86_emulate_ctxt *ctxt)
2754
ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
2755
| (ctxt->src.val & 0x0f));
2756
ctxt->dst.type = OP_NONE;
2757
return X86EMUL_CONTINUE;
2760
static int em_loop(struct x86_emulate_ctxt *ctxt)
2762
register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1);
2763
if ((address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) != 0) &&
2764
(ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
2765
jmp_rel(ctxt, ctxt->src.val);
2767
return X86EMUL_CONTINUE;
2770
static int em_jcxz(struct x86_emulate_ctxt *ctxt)
2772
if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0)
2773
jmp_rel(ctxt, ctxt->src.val);
2775
return X86EMUL_CONTINUE;
2778
static int em_cli(struct x86_emulate_ctxt *ctxt)
2780
if (emulator_bad_iopl(ctxt))
2781
return emulate_gp(ctxt, 0);
2783
ctxt->eflags &= ~X86_EFLAGS_IF;
2784
return X86EMUL_CONTINUE;
2787
static int em_sti(struct x86_emulate_ctxt *ctxt)
2789
if (emulator_bad_iopl(ctxt))
2790
return emulate_gp(ctxt, 0);
2792
ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
2793
ctxt->eflags |= X86_EFLAGS_IF;
2794
return X86EMUL_CONTINUE;
2797
static bool valid_cr(int nr)
2809
static int check_cr_read(struct x86_emulate_ctxt *ctxt)
2811
if (!valid_cr(ctxt->modrm_reg))
2812
return emulate_ud(ctxt);
2814
return X86EMUL_CONTINUE;
2817
static int check_cr_write(struct x86_emulate_ctxt *ctxt)
2819
u64 new_val = ctxt->src.val64;
2820
int cr = ctxt->modrm_reg;
2823
static u64 cr_reserved_bits[] = {
2824
0xffffffff00000000ULL,
2825
0, 0, 0, /* CR3 checked later */
2832
return emulate_ud(ctxt);
2834
if (new_val & cr_reserved_bits[cr])
2835
return emulate_gp(ctxt, 0);
2840
if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
2841
((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
2842
return emulate_gp(ctxt, 0);
2844
cr4 = ctxt->ops->get_cr(ctxt, 4);
2845
ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2847
if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
2848
!(cr4 & X86_CR4_PAE))
2849
return emulate_gp(ctxt, 0);
2856
ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2857
if (efer & EFER_LMA)
2858
rsvd = CR3_L_MODE_RESERVED_BITS;
2859
else if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PAE)
2860
rsvd = CR3_PAE_RESERVED_BITS;
2861
else if (ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PG)
2862
rsvd = CR3_NONPAE_RESERVED_BITS;
2865
return emulate_gp(ctxt, 0);
2872
cr4 = ctxt->ops->get_cr(ctxt, 4);
2873
ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2875
if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
2876
return emulate_gp(ctxt, 0);
2882
return X86EMUL_CONTINUE;
2885
static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
2889
ctxt->ops->get_dr(ctxt, 7, &dr7);
2891
/* Check if DR7.Global_Enable is set */
2892
return dr7 & (1 << 13);
2895
static int check_dr_read(struct x86_emulate_ctxt *ctxt)
2897
int dr = ctxt->modrm_reg;
2901
return emulate_ud(ctxt);
2903
cr4 = ctxt->ops->get_cr(ctxt, 4);
2904
if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
2905
return emulate_ud(ctxt);
2907
if (check_dr7_gd(ctxt))
2908
return emulate_db(ctxt);
2910
return X86EMUL_CONTINUE;
2913
static int check_dr_write(struct x86_emulate_ctxt *ctxt)
2915
u64 new_val = ctxt->src.val64;
2916
int dr = ctxt->modrm_reg;
2918
if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
2919
return emulate_gp(ctxt, 0);
2921
return check_dr_read(ctxt);
2924
static int check_svme(struct x86_emulate_ctxt *ctxt)
2928
ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2930
if (!(efer & EFER_SVME))
2931
return emulate_ud(ctxt);
2933
return X86EMUL_CONTINUE;
2936
static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
2938
u64 rax = ctxt->regs[VCPU_REGS_RAX];
2940
/* Valid physical address? */
2941
if (rax & 0xffff000000000000ULL)
2942
return emulate_gp(ctxt, 0);
2944
return check_svme(ctxt);
2947
static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
2949
u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
2951
if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
2952
return emulate_ud(ctxt);
2954
return X86EMUL_CONTINUE;
2957
static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
2959
u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
2960
u64 rcx = ctxt->regs[VCPU_REGS_RCX];
2962
if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
2964
return emulate_gp(ctxt, 0);
2966
return X86EMUL_CONTINUE;
2969
static int check_perm_in(struct x86_emulate_ctxt *ctxt)
2971
ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
2972
if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
2973
return emulate_gp(ctxt, 0);
2975
return X86EMUL_CONTINUE;
2978
static int check_perm_out(struct x86_emulate_ctxt *ctxt)
2980
ctxt->src.bytes = min(ctxt->src.bytes, 4u);
2981
if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
2982
return emulate_gp(ctxt, 0);
2984
return X86EMUL_CONTINUE;
2987
#define D(_y) { .flags = (_y) }
2988
#define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
2989
#define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
2990
.check_perm = (_p) }
2992
#define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
2993
#define G(_f, _g) { .flags = ((_f) | Group), .u.group = (_g) }
2994
#define GD(_f, _g) { .flags = ((_f) | GroupDual), .u.gdual = (_g) }
2995
#define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
2996
#define II(_f, _e, _i) \
2997
{ .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
2998
#define IIP(_f, _e, _i, _p) \
2999
{ .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
3000
.check_perm = (_p) }
3001
#define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3003
#define D2bv(_f) D((_f) | ByteOp), D(_f)
3004
#define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3005
#define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3007
#define I6ALU(_f, _e) I2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3008
I2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3009
I2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3011
static struct opcode group7_rm1[] = {
3012
DI(SrcNone | ModRM | Priv, monitor),
3013
DI(SrcNone | ModRM | Priv, mwait),
3017
static struct opcode group7_rm3[] = {
3018
DIP(SrcNone | ModRM | Prot | Priv, vmrun, check_svme_pa),
3019
II(SrcNone | ModRM | Prot | VendorSpecific, em_vmmcall, vmmcall),
3020
DIP(SrcNone | ModRM | Prot | Priv, vmload, check_svme_pa),
3021
DIP(SrcNone | ModRM | Prot | Priv, vmsave, check_svme_pa),
3022
DIP(SrcNone | ModRM | Prot | Priv, stgi, check_svme),
3023
DIP(SrcNone | ModRM | Prot | Priv, clgi, check_svme),
3024
DIP(SrcNone | ModRM | Prot | Priv, skinit, check_svme),
3025
DIP(SrcNone | ModRM | Prot | Priv, invlpga, check_svme),
3028
static struct opcode group7_rm7[] = {
3030
DIP(SrcNone | ModRM, rdtscp, check_rdtsc),
3034
static struct opcode group1[] = {
3045
static struct opcode group1A[] = {
3046
D(DstMem | SrcNone | ModRM | Mov | Stack), N, N, N, N, N, N, N,
3049
static struct opcode group3[] = {
3050
I(DstMem | SrcImm | ModRM, em_test),
3051
I(DstMem | SrcImm | ModRM, em_test),
3052
I(DstMem | SrcNone | ModRM | Lock, em_not),
3053
I(DstMem | SrcNone | ModRM | Lock, em_neg),
3054
I(SrcMem | ModRM, em_mul_ex),
3055
I(SrcMem | ModRM, em_imul_ex),
3056
I(SrcMem | ModRM, em_div_ex),
3057
I(SrcMem | ModRM, em_idiv_ex),
3060
static struct opcode group4[] = {
3061
D(ByteOp | DstMem | SrcNone | ModRM | Lock), D(ByteOp | DstMem | SrcNone | ModRM | Lock),
3065
static struct opcode group5[] = {
3066
D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
3067
D(SrcMem | ModRM | Stack),
3068
I(SrcMemFAddr | ModRM | ImplicitOps | Stack, em_call_far),
3069
D(SrcMem | ModRM | Stack), D(SrcMemFAddr | ModRM | ImplicitOps),
3070
D(SrcMem | ModRM | Stack), N,
3073
static struct opcode group6[] = {
3074
DI(ModRM | Prot, sldt),
3075
DI(ModRM | Prot, str),
3076
DI(ModRM | Prot | Priv, lldt),
3077
DI(ModRM | Prot | Priv, ltr),
3081
static struct group_dual group7 = { {
3082
DI(ModRM | Mov | DstMem | Priv, sgdt),
3083
DI(ModRM | Mov | DstMem | Priv, sidt),
3084
II(ModRM | SrcMem | Priv, em_lgdt, lgdt),
3085
II(ModRM | SrcMem | Priv, em_lidt, lidt),
3086
II(SrcNone | ModRM | DstMem | Mov, em_smsw, smsw), N,
3087
II(SrcMem16 | ModRM | Mov | Priv, em_lmsw, lmsw),
3088
II(SrcMem | ModRM | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3090
I(SrcNone | ModRM | Priv | VendorSpecific, em_vmcall),
3092
N, EXT(0, group7_rm3),
3093
II(SrcNone | ModRM | DstMem | Mov, em_smsw, smsw), N,
3094
II(SrcMem16 | ModRM | Mov | Priv, em_lmsw, lmsw), EXT(0, group7_rm7),
3097
static struct opcode group8[] = {
3099
D(DstMem | SrcImmByte | ModRM), D(DstMem | SrcImmByte | ModRM | Lock),
3100
D(DstMem | SrcImmByte | ModRM | Lock), D(DstMem | SrcImmByte | ModRM | Lock),
3103
static struct group_dual group9 = { {
3104
N, D(DstMem64 | ModRM | Lock), N, N, N, N, N, N,
3106
N, N, N, N, N, N, N, N,
3109
static struct opcode group11[] = {
3110
I(DstMem | SrcImm | ModRM | Mov, em_mov), X7(D(Undefined)),
3113
static struct gprefix pfx_0f_6f_0f_7f = {
3114
N, N, N, I(Sse, em_movdqu),
3117
static struct opcode opcode_table[256] = {
3119
I6ALU(Lock, em_add),
3120
I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
3121
I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
3124
I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
3127
I6ALU(Lock, em_adc),
3128
I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
3129
I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
3131
I6ALU(Lock, em_sbb),
3132
I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
3133
I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
3135
I6ALU(Lock, em_and), N, N,
3137
I6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3139
I6ALU(Lock, em_xor), N, N,
3141
I6ALU(0, em_cmp), N, N,
3145
X8(I(SrcReg | Stack, em_push)),
3147
X8(I(DstReg | Stack, em_pop)),
3149
I(ImplicitOps | Stack | No64, em_pusha),
3150
I(ImplicitOps | Stack | No64, em_popa),
3151
N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3154
I(SrcImm | Mov | Stack, em_push),
3155
I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3156
I(SrcImmByte | Mov | Stack, em_push),
3157
I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3158
D2bvIP(DstDI | SrcDX | Mov | String, ins, check_perm_in), /* insb, insw/insd */
3159
D2bvIP(SrcSI | DstDX | String, outs, check_perm_out), /* outsb, outsw/outsd */
3163
G(ByteOp | DstMem | SrcImm | ModRM | Group, group1),
3164
G(DstMem | SrcImm | ModRM | Group, group1),
3165
G(ByteOp | DstMem | SrcImm | ModRM | No64 | Group, group1),
3166
G(DstMem | SrcImmByte | ModRM | Group, group1),
3167
I2bv(DstMem | SrcReg | ModRM, em_test),
3168
I2bv(DstMem | SrcReg | ModRM | Lock, em_xchg),
3170
I2bv(DstMem | SrcReg | ModRM | Mov, em_mov),
3171
I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3172
I(DstMem | SrcNone | ModRM | Mov, em_mov_rm_sreg),
3173
D(ModRM | SrcMem | NoAccess | DstReg),
3174
I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
3177
DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3179
D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3180
I(SrcImmFAddr | No64, em_call_far), N,
3181
II(ImplicitOps | Stack, em_pushf, pushf),
3182
II(ImplicitOps | Stack, em_popf, popf), N, N,
3184
I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3185
I2bv(DstMem | SrcAcc | Mov | MemAbs, em_mov),
3186
I2bv(SrcSI | DstDI | Mov | String, em_mov),
3187
I2bv(SrcSI | DstDI | String, em_cmp),
3189
I2bv(DstAcc | SrcImm, em_test),
3190
I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3191
I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3192
I2bv(SrcAcc | DstDI | String, em_cmp),
3194
X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3196
X8(I(DstReg | SrcImm | Mov, em_mov)),
3198
D2bv(DstMem | SrcImmByte | ModRM),
3199
I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3200
I(ImplicitOps | Stack, em_ret),
3201
I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
3202
I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
3203
G(ByteOp, group11), G(0, group11),
3205
N, N, N, I(ImplicitOps | Stack, em_ret_far),
3206
D(ImplicitOps), DI(SrcImmByte, intn),
3207
D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3209
D2bv(DstMem | SrcOne | ModRM), D2bv(DstMem | ModRM),
3212
N, N, N, N, N, N, N, N,
3214
X3(I(SrcImmByte, em_loop)),
3215
I(SrcImmByte, em_jcxz),
3216
D2bvIP(SrcImmUByte | DstAcc, in, check_perm_in),
3217
D2bvIP(SrcAcc | DstImmUByte, out, check_perm_out),
3219
D(SrcImm | Stack), D(SrcImm | ImplicitOps),
3220
I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
3221
D2bvIP(SrcDX | DstAcc, in, check_perm_in),
3222
D2bvIP(SrcAcc | DstDX, out, check_perm_out),
3224
N, DI(ImplicitOps, icebp), N, N,
3225
DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
3226
G(ByteOp, group3), G(0, group3),
3228
D(ImplicitOps), D(ImplicitOps),
3229
I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
3230
D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3233
static struct opcode twobyte_table[256] = {
3235
G(0, group6), GD(0, &group7), N, N,
3236
N, I(ImplicitOps | VendorSpecific, em_syscall),
3237
II(ImplicitOps | Priv, em_clts, clts), N,
3238
DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3239
N, D(ImplicitOps | ModRM), N, N,
3241
N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
3243
DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
3244
DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
3245
DIP(ModRM | SrcMem | Priv | Op3264, cr_write, check_cr_write),
3246
DIP(ModRM | SrcMem | Priv | Op3264, dr_write, check_dr_write),
3248
N, N, N, N, N, N, N, N,
3250
DI(ImplicitOps | Priv, wrmsr),
3251
IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
3252
DI(ImplicitOps | Priv, rdmsr),
3253
DIP(ImplicitOps | Priv, rdpmc, check_rdpmc),
3254
I(ImplicitOps | VendorSpecific, em_sysenter),
3255
I(ImplicitOps | Priv | VendorSpecific, em_sysexit),
3257
N, N, N, N, N, N, N, N,
3259
X16(D(DstReg | SrcMem | ModRM | Mov)),
3261
N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3266
N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3271
N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3275
X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3277
I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
3278
DI(ImplicitOps, cpuid), D(DstMem | SrcReg | ModRM | BitOp),
3279
D(DstMem | SrcReg | Src2ImmByte | ModRM),
3280
D(DstMem | SrcReg | Src2CL | ModRM), N, N,
3282
I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
3283
DI(ImplicitOps, rsm), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3284
D(DstMem | SrcReg | Src2ImmByte | ModRM),
3285
D(DstMem | SrcReg | Src2CL | ModRM),
3286
D(ModRM), I(DstReg | SrcMem | ModRM, em_imul),
3288
D2bv(DstMem | SrcReg | ModRM | Lock),
3289
I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
3290
D(DstMem | SrcReg | ModRM | BitOp | Lock),
3291
I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
3292
I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
3293
D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3296
G(BitOp, group8), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3297
D(DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
3298
D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3300
D2bv(DstMem | SrcReg | ModRM | Lock),
3301
N, D(DstMem | SrcReg | ModRM | Mov),
3302
N, N, N, GD(0, &group9),
3303
N, N, N, N, N, N, N, N,
3305
N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3307
N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3309
N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
3325
static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
3329
size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3335
static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
3336
unsigned size, bool sign_extension)
3338
int rc = X86EMUL_CONTINUE;
3342
op->addr.mem.ea = ctxt->_eip;
3343
/* NB. Immediates are sign-extended as necessary. */
3344
switch (op->bytes) {
3346
op->val = insn_fetch(s8, ctxt);
3349
op->val = insn_fetch(s16, ctxt);
3352
op->val = insn_fetch(s32, ctxt);
3355
if (!sign_extension) {
3356
switch (op->bytes) {
3364
op->val &= 0xffffffff;
3372
static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
3375
int rc = X86EMUL_CONTINUE;
3379
decode_register_operand(ctxt, op,
3381
ctxt->twobyte && (ctxt->b == 0xb6 || ctxt->b == 0xb7));
3384
rc = decode_imm(ctxt, op, 1, false);
3387
ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3391
if ((ctxt->d & BitOp) && op == &ctxt->dst)
3392
fetch_bit_operand(ctxt);
3393
op->orig_val = op->val;
3396
ctxt->memop.bytes = 8;
3400
op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3401
op->addr.reg = &ctxt->regs[VCPU_REGS_RAX];
3402
fetch_register_operand(op);
3403
op->orig_val = op->val;
3407
op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3409
register_address(ctxt, ctxt->regs[VCPU_REGS_RDI]);
3410
op->addr.mem.seg = VCPU_SREG_ES;
3416
op->addr.reg = &ctxt->regs[VCPU_REGS_RDX];
3417
fetch_register_operand(op);
3421
op->val = ctxt->regs[VCPU_REGS_RCX] & 0xff;
3424
rc = decode_imm(ctxt, op, 1, true);
3431
rc = decode_imm(ctxt, op, imm_size(ctxt), true);
3434
ctxt->memop.bytes = 2;
3437
ctxt->memop.bytes = 4;
3440
rc = decode_imm(ctxt, op, 2, false);
3443
rc = decode_imm(ctxt, op, imm_size(ctxt), false);
3447
op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3449
register_address(ctxt, ctxt->regs[VCPU_REGS_RSI]);
3450
op->addr.mem.seg = seg_override(ctxt);
3455
op->addr.mem.ea = ctxt->_eip;
3456
op->bytes = ctxt->op_bytes + 2;
3457
insn_fetch_arr(op->valptr, op->bytes, ctxt);
3460
ctxt->memop.bytes = ctxt->op_bytes + 2;
3463
op->val = VCPU_SREG_ES;
3466
op->val = VCPU_SREG_CS;
3469
op->val = VCPU_SREG_SS;
3472
op->val = VCPU_SREG_DS;
3475
op->val = VCPU_SREG_FS;
3478
op->val = VCPU_SREG_GS;
3481
/* Special instructions do their own operand decoding. */
3483
op->type = OP_NONE; /* Disable writeback. */
3491
int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
3493
int rc = X86EMUL_CONTINUE;
3494
int mode = ctxt->mode;
3495
int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
3496
bool op_prefix = false;
3497
struct opcode opcode;
3499
ctxt->memop.type = OP_NONE;
3500
ctxt->memopp = NULL;
3501
ctxt->_eip = ctxt->eip;
3502
ctxt->fetch.start = ctxt->_eip;
3503
ctxt->fetch.end = ctxt->fetch.start + insn_len;
3505
memcpy(ctxt->fetch.data, insn, insn_len);
3508
case X86EMUL_MODE_REAL:
3509
case X86EMUL_MODE_VM86:
3510
case X86EMUL_MODE_PROT16:
3511
def_op_bytes = def_ad_bytes = 2;
3513
case X86EMUL_MODE_PROT32:
3514
def_op_bytes = def_ad_bytes = 4;
3516
#ifdef CONFIG_X86_64
3517
case X86EMUL_MODE_PROT64:
3523
return EMULATION_FAILED;
3526
ctxt->op_bytes = def_op_bytes;
3527
ctxt->ad_bytes = def_ad_bytes;
3529
/* Legacy prefixes. */
3531
switch (ctxt->b = insn_fetch(u8, ctxt)) {
3532
case 0x66: /* operand-size override */
3534
/* switch between 2/4 bytes */
3535
ctxt->op_bytes = def_op_bytes ^ 6;
3537
case 0x67: /* address-size override */
3538
if (mode == X86EMUL_MODE_PROT64)
3539
/* switch between 4/8 bytes */
3540
ctxt->ad_bytes = def_ad_bytes ^ 12;
3542
/* switch between 2/4 bytes */
3543
ctxt->ad_bytes = def_ad_bytes ^ 6;
3545
case 0x26: /* ES override */
3546
case 0x2e: /* CS override */
3547
case 0x36: /* SS override */
3548
case 0x3e: /* DS override */
3549
set_seg_override(ctxt, (ctxt->b >> 3) & 3);
3551
case 0x64: /* FS override */
3552
case 0x65: /* GS override */
3553
set_seg_override(ctxt, ctxt->b & 7);
3555
case 0x40 ... 0x4f: /* REX */
3556
if (mode != X86EMUL_MODE_PROT64)
3558
ctxt->rex_prefix = ctxt->b;
3560
case 0xf0: /* LOCK */
3561
ctxt->lock_prefix = 1;
3563
case 0xf2: /* REPNE/REPNZ */
3564
case 0xf3: /* REP/REPE/REPZ */
3565
ctxt->rep_prefix = ctxt->b;
3571
/* Any legacy prefix after a REX prefix nullifies its effect. */
3573
ctxt->rex_prefix = 0;
3579
if (ctxt->rex_prefix & 8)
3580
ctxt->op_bytes = 8; /* REX.W */
3582
/* Opcode byte(s). */
3583
opcode = opcode_table[ctxt->b];
3584
/* Two-byte opcode? */
3585
if (ctxt->b == 0x0f) {
3587
ctxt->b = insn_fetch(u8, ctxt);
3588
opcode = twobyte_table[ctxt->b];
3590
ctxt->d = opcode.flags;
3592
while (ctxt->d & GroupMask) {
3593
switch (ctxt->d & GroupMask) {
3595
ctxt->modrm = insn_fetch(u8, ctxt);
3597
goffset = (ctxt->modrm >> 3) & 7;
3598
opcode = opcode.u.group[goffset];
3601
ctxt->modrm = insn_fetch(u8, ctxt);
3603
goffset = (ctxt->modrm >> 3) & 7;
3604
if ((ctxt->modrm >> 6) == 3)
3605
opcode = opcode.u.gdual->mod3[goffset];
3607
opcode = opcode.u.gdual->mod012[goffset];
3610
goffset = ctxt->modrm & 7;
3611
opcode = opcode.u.group[goffset];
3614
if (ctxt->rep_prefix && op_prefix)
3615
return EMULATION_FAILED;
3616
simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
3617
switch (simd_prefix) {
3618
case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
3619
case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
3620
case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
3621
case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
3625
return EMULATION_FAILED;
3628
ctxt->d &= ~(u64)GroupMask;
3629
ctxt->d |= opcode.flags;
3632
ctxt->execute = opcode.u.execute;
3633
ctxt->check_perm = opcode.check_perm;
3634
ctxt->intercept = opcode.intercept;
3637
if (ctxt->d == 0 || (ctxt->d & Undefined))
3638
return EMULATION_FAILED;
3640
if (!(ctxt->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
3641
return EMULATION_FAILED;
3643
if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
3646
if (ctxt->d & Op3264) {
3647
if (mode == X86EMUL_MODE_PROT64)
3654
ctxt->op_bytes = 16;
3656
/* ModRM and SIB bytes. */
3657
if (ctxt->d & ModRM) {
3658
rc = decode_modrm(ctxt, &ctxt->memop);
3659
if (!ctxt->has_seg_override)
3660
set_seg_override(ctxt, ctxt->modrm_seg);
3661
} else if (ctxt->d & MemAbs)
3662
rc = decode_abs(ctxt, &ctxt->memop);
3663
if (rc != X86EMUL_CONTINUE)
3666
if (!ctxt->has_seg_override)
3667
set_seg_override(ctxt, VCPU_SREG_DS);
3669
ctxt->memop.addr.mem.seg = seg_override(ctxt);
3671
if (ctxt->memop.type == OP_MEM && ctxt->ad_bytes != 8)
3672
ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
3675
* Decode and fetch the source operand: register, memory
3678
rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
3679
if (rc != X86EMUL_CONTINUE)
3683
* Decode and fetch the second source operand: register, memory
3686
rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
3687
if (rc != X86EMUL_CONTINUE)
3690
/* Decode and fetch the destination operand: register or memory. */
3691
rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
3694
if (ctxt->memopp && ctxt->memopp->type == OP_MEM && ctxt->rip_relative)
3695
ctxt->memopp->addr.mem.ea += ctxt->_eip;
3697
return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
3700
static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
3702
/* The second termination condition only applies for REPE
3703
* and REPNE. Test if the repeat string operation prefix is
3704
* REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
3705
* corresponding termination condition according to:
3706
* - if REPE/REPZ and ZF = 0 then done
3707
* - if REPNE/REPNZ and ZF = 1 then done
3709
if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
3710
(ctxt->b == 0xae) || (ctxt->b == 0xaf))
3711
&& (((ctxt->rep_prefix == REPE_PREFIX) &&
3712
((ctxt->eflags & EFLG_ZF) == 0))
3713
|| ((ctxt->rep_prefix == REPNE_PREFIX) &&
3714
((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
3720
int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
3722
struct x86_emulate_ops *ops = ctxt->ops;
3724
int rc = X86EMUL_CONTINUE;
3725
int saved_dst_type = ctxt->dst.type;
3727
ctxt->mem_read.pos = 0;
3729
if (ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) {
3730
rc = emulate_ud(ctxt);
3734
/* LOCK prefix is allowed only with some instructions */
3735
if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
3736
rc = emulate_ud(ctxt);
3740
if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
3741
rc = emulate_ud(ctxt);
3746
&& ((ops->get_cr(ctxt, 0) & X86_CR0_EM)
3747
|| !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
3748
rc = emulate_ud(ctxt);
3752
if ((ctxt->d & Sse) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
3753
rc = emulate_nm(ctxt);
3757
if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
3758
rc = emulator_check_intercept(ctxt, ctxt->intercept,
3759
X86_ICPT_PRE_EXCEPT);
3760
if (rc != X86EMUL_CONTINUE)
3764
/* Privileged instruction can be executed only in CPL=0 */
3765
if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
3766
rc = emulate_gp(ctxt, 0);
3770
/* Instruction can only be executed in protected mode */
3771
if ((ctxt->d & Prot) && !(ctxt->mode & X86EMUL_MODE_PROT)) {
3772
rc = emulate_ud(ctxt);
3776
/* Do instruction specific permission checks */
3777
if (ctxt->check_perm) {
3778
rc = ctxt->check_perm(ctxt);
3779
if (rc != X86EMUL_CONTINUE)
3783
if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
3784
rc = emulator_check_intercept(ctxt, ctxt->intercept,
3785
X86_ICPT_POST_EXCEPT);
3786
if (rc != X86EMUL_CONTINUE)
3790
if (ctxt->rep_prefix && (ctxt->d & String)) {
3791
/* All REP prefixes have the same first termination condition */
3792
if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0) {
3793
ctxt->eip = ctxt->_eip;
3798
if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
3799
rc = segmented_read(ctxt, ctxt->src.addr.mem,
3800
ctxt->src.valptr, ctxt->src.bytes);
3801
if (rc != X86EMUL_CONTINUE)
3803
ctxt->src.orig_val64 = ctxt->src.val64;
3806
if (ctxt->src2.type == OP_MEM) {
3807
rc = segmented_read(ctxt, ctxt->src2.addr.mem,
3808
&ctxt->src2.val, ctxt->src2.bytes);
3809
if (rc != X86EMUL_CONTINUE)
3813
if ((ctxt->d & DstMask) == ImplicitOps)
3817
if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
3818
/* optimisation - avoid slow emulated read if Mov */
3819
rc = segmented_read(ctxt, ctxt->dst.addr.mem,
3820
&ctxt->dst.val, ctxt->dst.bytes);
3821
if (rc != X86EMUL_CONTINUE)
3824
ctxt->dst.orig_val = ctxt->dst.val;
3828
if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
3829
rc = emulator_check_intercept(ctxt, ctxt->intercept,
3830
X86_ICPT_POST_MEMACCESS);
3831
if (rc != X86EMUL_CONTINUE)
3835
if (ctxt->execute) {
3836
rc = ctxt->execute(ctxt);
3837
if (rc != X86EMUL_CONTINUE)
3846
case 0x40 ... 0x47: /* inc r16/r32 */
3847
emulate_1op(ctxt, "inc");
3849
case 0x48 ... 0x4f: /* dec r16/r32 */
3850
emulate_1op(ctxt, "dec");
3852
case 0x63: /* movsxd */
3853
if (ctxt->mode != X86EMUL_MODE_PROT64)
3854
goto cannot_emulate;
3855
ctxt->dst.val = (s32) ctxt->src.val;
3857
case 0x6c: /* insb */
3858
case 0x6d: /* insw/insd */
3859
ctxt->src.val = ctxt->regs[VCPU_REGS_RDX];
3861
case 0x6e: /* outsb */
3862
case 0x6f: /* outsw/outsd */
3863
ctxt->dst.val = ctxt->regs[VCPU_REGS_RDX];
3866
case 0x70 ... 0x7f: /* jcc (short) */
3867
if (test_cc(ctxt->b, ctxt->eflags))
3868
jmp_rel(ctxt, ctxt->src.val);
3870
case 0x8d: /* lea r16/r32, m */
3871
ctxt->dst.val = ctxt->src.addr.mem.ea;
3873
case 0x8f: /* pop (sole member of Grp1a) */
3874
rc = em_grp1a(ctxt);
3876
case 0x90 ... 0x97: /* nop / xchg reg, rax */
3877
if (ctxt->dst.addr.reg == &ctxt->regs[VCPU_REGS_RAX])
3881
case 0x98: /* cbw/cwde/cdqe */
3882
switch (ctxt->op_bytes) {
3883
case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
3884
case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
3885
case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
3891
case 0xcc: /* int3 */
3892
rc = emulate_int(ctxt, 3);
3894
case 0xcd: /* int n */
3895
rc = emulate_int(ctxt, ctxt->src.val);
3897
case 0xce: /* into */
3898
if (ctxt->eflags & EFLG_OF)
3899
rc = emulate_int(ctxt, 4);
3901
case 0xd0 ... 0xd1: /* Grp2 */
3904
case 0xd2 ... 0xd3: /* Grp2 */
3905
ctxt->src.val = ctxt->regs[VCPU_REGS_RCX];
3908
case 0xe4: /* inb */
3911
case 0xe6: /* outb */
3912
case 0xe7: /* out */
3914
case 0xe8: /* call (near) */ {
3915
long int rel = ctxt->src.val;
3916
ctxt->src.val = (unsigned long) ctxt->_eip;
3921
case 0xe9: /* jmp rel */
3922
case 0xeb: /* jmp rel short */
3923
jmp_rel(ctxt, ctxt->src.val);
3924
ctxt->dst.type = OP_NONE; /* Disable writeback. */
3926
case 0xec: /* in al,dx */
3927
case 0xed: /* in (e/r)ax,dx */
3929
if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3931
goto done; /* IO is needed */
3933
case 0xee: /* out dx,al */
3934
case 0xef: /* out dx,(e/r)ax */
3936
ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3938
ctxt->dst.type = OP_NONE; /* Disable writeback. */
3940
case 0xf4: /* hlt */
3941
ctxt->ops->halt(ctxt);
3943
case 0xf5: /* cmc */
3944
/* complement carry flag from eflags reg */
3945
ctxt->eflags ^= EFLG_CF;
3947
case 0xf8: /* clc */
3948
ctxt->eflags &= ~EFLG_CF;
3950
case 0xf9: /* stc */
3951
ctxt->eflags |= EFLG_CF;
3953
case 0xfc: /* cld */
3954
ctxt->eflags &= ~EFLG_DF;
3956
case 0xfd: /* std */
3957
ctxt->eflags |= EFLG_DF;
3959
case 0xfe: /* Grp4 */
3960
rc = em_grp45(ctxt);
3962
case 0xff: /* Grp5 */
3963
rc = em_grp45(ctxt);
3966
goto cannot_emulate;
3969
if (rc != X86EMUL_CONTINUE)
3973
rc = writeback(ctxt);
3974
if (rc != X86EMUL_CONTINUE)
3978
* restore dst type in case the decoding will be reused
3979
* (happens for string instruction )
3981
ctxt->dst.type = saved_dst_type;
3983
if ((ctxt->d & SrcMask) == SrcSI)
3984
string_addr_inc(ctxt, seg_override(ctxt),
3985
VCPU_REGS_RSI, &ctxt->src);
3987
if ((ctxt->d & DstMask) == DstDI)
3988
string_addr_inc(ctxt, VCPU_SREG_ES, VCPU_REGS_RDI,
3991
if (ctxt->rep_prefix && (ctxt->d & String)) {
3992
struct read_cache *r = &ctxt->io_read;
3993
register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1);
3995
if (!string_insn_completed(ctxt)) {
3997
* Re-enter guest when pio read ahead buffer is empty
3998
* or, if it is not used, after each 1024 iteration.
4000
if ((r->end != 0 || ctxt->regs[VCPU_REGS_RCX] & 0x3ff) &&
4001
(r->end == 0 || r->end != r->pos)) {
4003
* Reset read cache. Usually happens before
4004
* decode, but since instruction is restarted
4005
* we have to do it here.
4007
ctxt->mem_read.end = 0;
4008
return EMULATION_RESTART;
4010
goto done; /* skip rip writeback */
4014
ctxt->eip = ctxt->_eip;
4017
if (rc == X86EMUL_PROPAGATE_FAULT)
4018
ctxt->have_exception = true;
4019
if (rc == X86EMUL_INTERCEPTED)
4020
return EMULATION_INTERCEPTED;
4022
return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4026
case 0x09: /* wbinvd */
4027
(ctxt->ops->wbinvd)(ctxt);
4029
case 0x08: /* invd */
4030
case 0x0d: /* GrpP (prefetch) */
4031
case 0x18: /* Grp16 (prefetch/nop) */
4033
case 0x20: /* mov cr, reg */
4034
ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
4036
case 0x21: /* mov from dr to reg */
4037
ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
4039
case 0x22: /* mov reg, cr */
4040
if (ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val)) {
4041
emulate_gp(ctxt, 0);
4042
rc = X86EMUL_PROPAGATE_FAULT;
4045
ctxt->dst.type = OP_NONE;
4047
case 0x23: /* mov from reg to dr */
4048
if (ops->set_dr(ctxt, ctxt->modrm_reg, ctxt->src.val &
4049
((ctxt->mode == X86EMUL_MODE_PROT64) ?
4050
~0ULL : ~0U)) < 0) {
4051
/* #UD condition is already handled by the code above */
4052
emulate_gp(ctxt, 0);
4053
rc = X86EMUL_PROPAGATE_FAULT;
4057
ctxt->dst.type = OP_NONE; /* no writeback */
4061
msr_data = (u32)ctxt->regs[VCPU_REGS_RAX]
4062
| ((u64)ctxt->regs[VCPU_REGS_RDX] << 32);
4063
if (ops->set_msr(ctxt, ctxt->regs[VCPU_REGS_RCX], msr_data)) {
4064
emulate_gp(ctxt, 0);
4065
rc = X86EMUL_PROPAGATE_FAULT;
4068
rc = X86EMUL_CONTINUE;
4072
if (ops->get_msr(ctxt, ctxt->regs[VCPU_REGS_RCX], &msr_data)) {
4073
emulate_gp(ctxt, 0);
4074
rc = X86EMUL_PROPAGATE_FAULT;
4077
ctxt->regs[VCPU_REGS_RAX] = (u32)msr_data;
4078
ctxt->regs[VCPU_REGS_RDX] = msr_data >> 32;
4080
rc = X86EMUL_CONTINUE;
4082
case 0x40 ... 0x4f: /* cmov */
4083
ctxt->dst.val = ctxt->dst.orig_val = ctxt->src.val;
4084
if (!test_cc(ctxt->b, ctxt->eflags))
4085
ctxt->dst.type = OP_NONE; /* no writeback */
4087
case 0x80 ... 0x8f: /* jnz rel, etc*/
4088
if (test_cc(ctxt->b, ctxt->eflags))
4089
jmp_rel(ctxt, ctxt->src.val);
4091
case 0x90 ... 0x9f: /* setcc r/m8 */
4092
ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
4096
ctxt->dst.type = OP_NONE;
4097
/* only subword offset */
4098
ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
4099
emulate_2op_SrcV_nobyte(ctxt, "bt");
4101
case 0xa4: /* shld imm8, r, r/m */
4102
case 0xa5: /* shld cl, r, r/m */
4103
emulate_2op_cl(ctxt, "shld");
4107
emulate_2op_SrcV_nobyte(ctxt, "bts");
4109
case 0xac: /* shrd imm8, r, r/m */
4110
case 0xad: /* shrd cl, r, r/m */
4111
emulate_2op_cl(ctxt, "shrd");
4113
case 0xae: /* clflush */
4115
case 0xb0 ... 0xb1: /* cmpxchg */
4117
* Save real source value, then compare EAX against
4120
ctxt->src.orig_val = ctxt->src.val;
4121
ctxt->src.val = ctxt->regs[VCPU_REGS_RAX];
4122
emulate_2op_SrcV(ctxt, "cmp");
4123
if (ctxt->eflags & EFLG_ZF) {
4124
/* Success: write back to memory. */
4125
ctxt->dst.val = ctxt->src.orig_val;
4127
/* Failure: write the value we saw to EAX. */
4128
ctxt->dst.type = OP_REG;
4129
ctxt->dst.addr.reg = (unsigned long *)&ctxt->regs[VCPU_REGS_RAX];
4134
emulate_2op_SrcV_nobyte(ctxt, "btr");
4136
case 0xb6 ... 0xb7: /* movzx */
4137
ctxt->dst.bytes = ctxt->op_bytes;
4138
ctxt->dst.val = (ctxt->d & ByteOp) ? (u8) ctxt->src.val
4139
: (u16) ctxt->src.val;
4141
case 0xba: /* Grp8 */
4142
switch (ctxt->modrm_reg & 3) {
4155
emulate_2op_SrcV_nobyte(ctxt, "btc");
4157
case 0xbc: { /* bsf */
4159
__asm__ ("bsf %2, %0; setz %1"
4160
: "=r"(ctxt->dst.val), "=q"(zf)
4161
: "r"(ctxt->src.val));
4162
ctxt->eflags &= ~X86_EFLAGS_ZF;
4164
ctxt->eflags |= X86_EFLAGS_ZF;
4165
ctxt->dst.type = OP_NONE; /* Disable writeback. */
4169
case 0xbd: { /* bsr */
4171
__asm__ ("bsr %2, %0; setz %1"
4172
: "=r"(ctxt->dst.val), "=q"(zf)
4173
: "r"(ctxt->src.val));
4174
ctxt->eflags &= ~X86_EFLAGS_ZF;
4176
ctxt->eflags |= X86_EFLAGS_ZF;
4177
ctxt->dst.type = OP_NONE; /* Disable writeback. */
4181
case 0xbe ... 0xbf: /* movsx */
4182
ctxt->dst.bytes = ctxt->op_bytes;
4183
ctxt->dst.val = (ctxt->d & ByteOp) ? (s8) ctxt->src.val :
4184
(s16) ctxt->src.val;
4186
case 0xc0 ... 0xc1: /* xadd */
4187
emulate_2op_SrcV(ctxt, "add");
4188
/* Write back the register source. */
4189
ctxt->src.val = ctxt->dst.orig_val;
4190
write_register_operand(&ctxt->src);
4192
case 0xc3: /* movnti */
4193
ctxt->dst.bytes = ctxt->op_bytes;
4194
ctxt->dst.val = (ctxt->op_bytes == 4) ? (u32) ctxt->src.val :
4195
(u64) ctxt->src.val;
4197
case 0xc7: /* Grp9 (cmpxchg8b) */
4201
goto cannot_emulate;
4204
if (rc != X86EMUL_CONTINUE)
4210
return EMULATION_FAILED;