linux/arch/x86/kvm/x86_emulate.c
<<
>>
Prefs
   1/******************************************************************************
   2 * x86_emulate.c
   3 *
   4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
   5 *
   6 * Copyright (c) 2005 Keir Fraser
   7 *
   8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
   9 * privileged instructions:
  10 *
  11 * Copyright (C) 2006 Qumranet
  12 *
  13 *   Avi Kivity <avi@qumranet.com>
  14 *   Yaniv Kamay <yaniv@qumranet.com>
  15 *
  16 * This work is licensed under the terms of the GNU GPL, version 2.  See
  17 * the COPYING file in the top-level directory.
  18 *
  19 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
  20 */
  21
  22#ifndef __KERNEL__
  23#include <stdio.h>
  24#include <stdint.h>
  25#include <public/xen.h>
  26#define DPRINTF(_f, _a ...) printf(_f , ## _a)
  27#else
  28#include <linux/kvm_host.h>
  29#define DPRINTF(x...) do {} while (0)
  30#endif
  31#include <linux/module.h>
  32#include <asm/kvm_x86_emulate.h>
  33
  34/*
  35 * Opcode effective-address decode tables.
  36 * Note that we only emulate instructions that have at least one memory
  37 * operand (excluding implicit stack references). We assume that stack
  38 * references and instruction fetches will never occur in special memory
  39 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
  40 * not be handled.
  41 */
  42
  43/* Operand sizes: 8-bit operands or specified/overridden size. */
  44#define ByteOp      (1<<0)      /* 8-bit operands. */
  45/* Destination operand type. */
  46#define ImplicitOps (1<<1)      /* Implicit in opcode. No generic decode. */
  47#define DstReg      (2<<1)      /* Register operand. */
  48#define DstMem      (3<<1)      /* Memory operand. */
  49#define DstMask     (3<<1)
  50/* Source operand type. */
  51#define SrcNone     (0<<3)      /* No source operand. */
  52#define SrcImplicit (0<<3)      /* Source operand is implicit in the opcode. */
  53#define SrcReg      (1<<3)      /* Register operand. */
  54#define SrcMem      (2<<3)      /* Memory operand. */
  55#define SrcMem16    (3<<3)      /* Memory operand (16-bit). */
  56#define SrcMem32    (4<<3)      /* Memory operand (32-bit). */
  57#define SrcImm      (5<<3)      /* Immediate operand. */
  58#define SrcImmByte  (6<<3)      /* 8-bit sign-extended immediate operand. */
  59#define SrcMask     (7<<3)
  60/* Generic ModRM decode. */
  61#define ModRM       (1<<6)
  62/* Destination is only written; never read. */
  63#define Mov         (1<<7)
  64#define BitOp       (1<<8)
  65#define MemAbs      (1<<9)      /* Memory operand is absolute displacement */
  66#define String      (1<<10)     /* String instruction (rep capable) */
  67#define Stack       (1<<11)     /* Stack instruction (push/pop) */
  68#define Group       (1<<14)     /* Bits 3:5 of modrm byte extend opcode */
  69#define GroupDual   (1<<15)     /* Alternate decoding of mod == 3 */
  70#define GroupMask   0xff        /* Group number stored in bits 0:7 */
  71
  72enum {
  73        Group1_80, Group1_81, Group1_82, Group1_83,
  74        Group1A, Group3_Byte, Group3, Group4, Group5, Group7,
  75};
  76
  77static u16 opcode_table[256] = {
  78        /* 0x00 - 0x07 */
  79        ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
  80        ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
  81        0, 0, 0, 0,
  82        /* 0x08 - 0x0F */
  83        ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
  84        ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
  85        0, 0, 0, 0,
  86        /* 0x10 - 0x17 */
  87        ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
  88        ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
  89        0, 0, 0, 0,
  90        /* 0x18 - 0x1F */
  91        ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
  92        ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
  93        0, 0, 0, 0,
  94        /* 0x20 - 0x27 */
  95        ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
  96        ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
  97        SrcImmByte, SrcImm, 0, 0,
  98        /* 0x28 - 0x2F */
  99        ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
 100        ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
 101        0, 0, 0, 0,
 102        /* 0x30 - 0x37 */
 103        ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
 104        ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
 105        0, 0, 0, 0,
 106        /* 0x38 - 0x3F */
 107        ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
 108        ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
 109        0, 0, 0, 0,
 110        /* 0x40 - 0x47 */
 111        DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg,
 112        /* 0x48 - 0x4F */
 113        DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg,
 114        /* 0x50 - 0x57 */
 115        SrcReg | Stack, SrcReg | Stack, SrcReg | Stack, SrcReg | Stack,
 116        SrcReg | Stack, SrcReg | Stack, SrcReg | Stack, SrcReg | Stack,
 117        /* 0x58 - 0x5F */
 118        DstReg | Stack, DstReg | Stack, DstReg | Stack, DstReg | Stack,
 119        DstReg | Stack, DstReg | Stack, DstReg | Stack, DstReg | Stack,
 120        /* 0x60 - 0x67 */
 121        0, 0, 0, DstReg | SrcMem32 | ModRM | Mov /* movsxd (x86/64) */ ,
 122        0, 0, 0, 0,
 123        /* 0x68 - 0x6F */
 124        SrcImm | Mov | Stack, 0, SrcImmByte | Mov | Stack, 0,
 125        SrcNone  | ByteOp  | ImplicitOps, SrcNone  | ImplicitOps, /* insb, insw/insd */
 126        SrcNone  | ByteOp  | ImplicitOps, SrcNone  | ImplicitOps, /* outsb, outsw/outsd */
 127        /* 0x70 - 0x77 */
 128        ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps,
 129        ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps,
 130        /* 0x78 - 0x7F */
 131        ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps,
 132        ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps,
 133        /* 0x80 - 0x87 */
 134        Group | Group1_80, Group | Group1_81,
 135        Group | Group1_82, Group | Group1_83,
 136        ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
 137        ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
 138        /* 0x88 - 0x8F */
 139        ByteOp | DstMem | SrcReg | ModRM | Mov, DstMem | SrcReg | ModRM | Mov,
 140        ByteOp | DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
 141        DstMem | SrcReg | ModRM | Mov, ModRM | DstReg,
 142        DstReg | SrcMem | ModRM | Mov, Group | Group1A,
 143        /* 0x90 - 0x97 */
 144        DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg,
 145        /* 0x98 - 0x9F */
 146        0, 0, 0, 0, ImplicitOps | Stack, ImplicitOps | Stack, 0, 0,
 147        /* 0xA0 - 0xA7 */
 148        ByteOp | DstReg | SrcMem | Mov | MemAbs, DstReg | SrcMem | Mov | MemAbs,
 149        ByteOp | DstMem | SrcReg | Mov | MemAbs, DstMem | SrcReg | Mov | MemAbs,
 150        ByteOp | ImplicitOps | Mov | String, ImplicitOps | Mov | String,
 151        ByteOp | ImplicitOps | String, ImplicitOps | String,
 152        /* 0xA8 - 0xAF */
 153        0, 0, ByteOp | ImplicitOps | Mov | String, ImplicitOps | Mov | String,
 154        ByteOp | ImplicitOps | Mov | String, ImplicitOps | Mov | String,
 155        ByteOp | ImplicitOps | String, ImplicitOps | String,
 156        /* 0xB0 - 0xBF */
 157        0, 0, 0, 0, 0, 0, 0, 0,
 158        DstReg | SrcImm | Mov, 0, 0, 0, 0, 0, 0, 0,
 159        /* 0xC0 - 0xC7 */
 160        ByteOp | DstMem | SrcImm | ModRM, DstMem | SrcImmByte | ModRM,
 161        0, ImplicitOps | Stack, 0, 0,
 162        ByteOp | DstMem | SrcImm | ModRM | Mov, DstMem | SrcImm | ModRM | Mov,
 163        /* 0xC8 - 0xCF */
 164        0, 0, 0, 0, 0, 0, 0, 0,
 165        /* 0xD0 - 0xD7 */
 166        ByteOp | DstMem | SrcImplicit | ModRM, DstMem | SrcImplicit | ModRM,
 167        ByteOp | DstMem | SrcImplicit | ModRM, DstMem | SrcImplicit | ModRM,
 168        0, 0, 0, 0,
 169        /* 0xD8 - 0xDF */
 170        0, 0, 0, 0, 0, 0, 0, 0,
 171        /* 0xE0 - 0xE7 */
 172        0, 0, 0, 0, 0, 0, 0, 0,
 173        /* 0xE8 - 0xEF */
 174        ImplicitOps | Stack, SrcImm | ImplicitOps,
 175        ImplicitOps, SrcImmByte | ImplicitOps,
 176        0, 0, 0, 0,
 177        /* 0xF0 - 0xF7 */
 178        0, 0, 0, 0,
 179        ImplicitOps, ImplicitOps, Group | Group3_Byte, Group | Group3,
 180        /* 0xF8 - 0xFF */
 181        ImplicitOps, 0, ImplicitOps, ImplicitOps,
 182        0, 0, Group | Group4, Group | Group5,
 183};
 184
 185static u16 twobyte_table[256] = {
 186        /* 0x00 - 0x0F */
 187        0, Group | GroupDual | Group7, 0, 0, 0, 0, ImplicitOps, 0,
 188        ImplicitOps, ImplicitOps, 0, 0, 0, ImplicitOps | ModRM, 0, 0,
 189        /* 0x10 - 0x1F */
 190        0, 0, 0, 0, 0, 0, 0, 0, ImplicitOps | ModRM, 0, 0, 0, 0, 0, 0, 0,
 191        /* 0x20 - 0x2F */
 192        ModRM | ImplicitOps, ModRM, ModRM | ImplicitOps, ModRM, 0, 0, 0, 0,
 193        0, 0, 0, 0, 0, 0, 0, 0,
 194        /* 0x30 - 0x3F */
 195        ImplicitOps, 0, ImplicitOps, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 196        /* 0x40 - 0x47 */
 197        DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
 198        DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
 199        DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
 200        DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
 201        /* 0x48 - 0x4F */
 202        DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
 203        DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
 204        DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
 205        DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
 206        /* 0x50 - 0x5F */
 207        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 208        /* 0x60 - 0x6F */
 209        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 210        /* 0x70 - 0x7F */
 211        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 212        /* 0x80 - 0x8F */
 213        ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps,
 214        ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps,
 215        ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps,
 216        ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps,
 217        /* 0x90 - 0x9F */
 218        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 219        /* 0xA0 - 0xA7 */
 220        0, 0, 0, DstMem | SrcReg | ModRM | BitOp, 0, 0, 0, 0,
 221        /* 0xA8 - 0xAF */
 222        0, 0, 0, DstMem | SrcReg | ModRM | BitOp, 0, 0, ModRM, 0,
 223        /* 0xB0 - 0xB7 */
 224        ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM, 0,
 225            DstMem | SrcReg | ModRM | BitOp,
 226        0, 0, ByteOp | DstReg | SrcMem | ModRM | Mov,
 227            DstReg | SrcMem16 | ModRM | Mov,
 228        /* 0xB8 - 0xBF */
 229        0, 0, DstMem | SrcImmByte | ModRM, DstMem | SrcReg | ModRM | BitOp,
 230        0, 0, ByteOp | DstReg | SrcMem | ModRM | Mov,
 231            DstReg | SrcMem16 | ModRM | Mov,
 232        /* 0xC0 - 0xCF */
 233        0, 0, 0, DstMem | SrcReg | ModRM | Mov, 0, 0, 0, ImplicitOps | ModRM,
 234        0, 0, 0, 0, 0, 0, 0, 0,
 235        /* 0xD0 - 0xDF */
 236        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 237        /* 0xE0 - 0xEF */
 238        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 239        /* 0xF0 - 0xFF */
 240        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
 241};
 242
 243static u16 group_table[] = {
 244        [Group1_80*8] =
 245        ByteOp | DstMem | SrcImm | ModRM, ByteOp | DstMem | SrcImm | ModRM,
 246        ByteOp | DstMem | SrcImm | ModRM, ByteOp | DstMem | SrcImm | ModRM,
 247        ByteOp | DstMem | SrcImm | ModRM, ByteOp | DstMem | SrcImm | ModRM,
 248        ByteOp | DstMem | SrcImm | ModRM, ByteOp | DstMem | SrcImm | ModRM,
 249        [Group1_81*8] =
 250        DstMem | SrcImm | ModRM, DstMem | SrcImm | ModRM,
 251        DstMem | SrcImm | ModRM, DstMem | SrcImm | ModRM,
 252        DstMem | SrcImm | ModRM, DstMem | SrcImm | ModRM,
 253        DstMem | SrcImm | ModRM, DstMem | SrcImm | ModRM,
 254        [Group1_82*8] =
 255        ByteOp | DstMem | SrcImm | ModRM, ByteOp | DstMem | SrcImm | ModRM,
 256        ByteOp | DstMem | SrcImm | ModRM, ByteOp | DstMem | SrcImm | ModRM,
 257        ByteOp | DstMem | SrcImm | ModRM, ByteOp | DstMem | SrcImm | ModRM,
 258        ByteOp | DstMem | SrcImm | ModRM, ByteOp | DstMem | SrcImm | ModRM,
 259        [Group1_83*8] =
 260        DstMem | SrcImmByte | ModRM, DstMem | SrcImmByte | ModRM,
 261        DstMem | SrcImmByte | ModRM, DstMem | SrcImmByte | ModRM,
 262        DstMem | SrcImmByte | ModRM, DstMem | SrcImmByte | ModRM,
 263        DstMem | SrcImmByte | ModRM, DstMem | SrcImmByte | ModRM,
 264        [Group1A*8] =
 265        DstMem | SrcNone | ModRM | Mov | Stack, 0, 0, 0, 0, 0, 0, 0,
 266        [Group3_Byte*8] =
 267        ByteOp | SrcImm | DstMem | ModRM, 0,
 268        ByteOp | DstMem | SrcNone | ModRM, ByteOp | DstMem | SrcNone | ModRM,
 269        0, 0, 0, 0,
 270        [Group3*8] =
 271        DstMem | SrcImm | ModRM | SrcImm, 0,
 272        DstMem | SrcNone | ModRM, ByteOp | DstMem | SrcNone | ModRM,
 273        0, 0, 0, 0,
 274        [Group4*8] =
 275        ByteOp | DstMem | SrcNone | ModRM, ByteOp | DstMem | SrcNone | ModRM,
 276        0, 0, 0, 0, 0, 0,
 277        [Group5*8] =
 278        DstMem | SrcNone | ModRM, DstMem | SrcNone | ModRM, 0, 0,
 279        SrcMem | ModRM, 0, SrcMem | ModRM | Stack, 0,
 280        [Group7*8] =
 281        0, 0, ModRM | SrcMem, ModRM | SrcMem,
 282        SrcNone | ModRM | DstMem | Mov, 0,
 283        SrcMem16 | ModRM | Mov, SrcMem | ModRM | ByteOp,
 284};
 285
 286static u16 group2_table[] = {
 287        [Group7*8] =
 288        SrcNone | ModRM, 0, 0, 0,
 289        SrcNone | ModRM | DstMem | Mov, 0,
 290        SrcMem16 | ModRM | Mov, 0,
 291};
 292
 293/* EFLAGS bit definitions. */
 294#define EFLG_OF (1<<11)
 295#define EFLG_DF (1<<10)
 296#define EFLG_SF (1<<7)
 297#define EFLG_ZF (1<<6)
 298#define EFLG_AF (1<<4)
 299#define EFLG_PF (1<<2)
 300#define EFLG_CF (1<<0)
 301
 302/*
 303 * Instruction emulation:
 304 * Most instructions are emulated directly via a fragment of inline assembly
 305 * code. This allows us to save/restore EFLAGS and thus very easily pick up
 306 * any modified flags.
 307 */
 308
 309#if defined(CONFIG_X86_64)
 310#define _LO32 "k"               /* force 32-bit operand */
 311#define _STK  "%%rsp"           /* stack pointer */
 312#elif defined(__i386__)
 313#define _LO32 ""                /* force 32-bit operand */
 314#define _STK  "%%esp"           /* stack pointer */
 315#endif
 316
 317/*
 318 * These EFLAGS bits are restored from saved value during emulation, and
 319 * any changes are written back to the saved value after emulation.
 320 */
 321#define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
 322
 323/* Before executing instruction: restore necessary bits in EFLAGS. */
 324#define _PRE_EFLAGS(_sav, _msk, _tmp)                                   \
 325        /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
 326        "movl %"_sav",%"_LO32 _tmp"; "                                  \
 327        "push %"_tmp"; "                                                \
 328        "push %"_tmp"; "                                                \
 329        "movl %"_msk",%"_LO32 _tmp"; "                                  \
 330        "andl %"_LO32 _tmp",("_STK"); "                                 \
 331        "pushf; "                                                       \
 332        "notl %"_LO32 _tmp"; "                                          \
 333        "andl %"_LO32 _tmp",("_STK"); "                                 \
 334        "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); "   \
 335        "pop  %"_tmp"; "                                                \
 336        "orl  %"_LO32 _tmp",("_STK"); "                                 \
 337        "popf; "                                                        \
 338        "pop  %"_sav"; "
 339
 340/* After executing instruction: write-back necessary bits in EFLAGS. */
 341#define _POST_EFLAGS(_sav, _msk, _tmp) \
 342        /* _sav |= EFLAGS & _msk; */            \
 343        "pushf; "                               \
 344        "pop  %"_tmp"; "                        \
 345        "andl %"_msk",%"_LO32 _tmp"; "          \
 346        "orl  %"_LO32 _tmp",%"_sav"; "
 347
 348/* Raw emulation: instruction has two explicit operands. */
 349#define __emulate_2op_nobyte(_op,_src,_dst,_eflags,_wx,_wy,_lx,_ly,_qx,_qy) \
 350        do {                                                                \
 351                unsigned long _tmp;                                         \
 352                                                                            \
 353                switch ((_dst).bytes) {                                     \
 354                case 2:                                                     \
 355                        __asm__ __volatile__ (                              \
 356                                _PRE_EFLAGS("0", "4", "2")                  \
 357                                _op"w %"_wx"3,%1; "                         \
 358                                _POST_EFLAGS("0", "4", "2")                 \
 359                                : "=m" (_eflags), "=m" ((_dst).val),        \
 360                                  "=&r" (_tmp)                              \
 361                                : _wy ((_src).val), "i" (EFLAGS_MASK));     \
 362                        break;                                              \
 363                case 4:                                                     \
 364                        __asm__ __volatile__ (                              \
 365                                _PRE_EFLAGS("0", "4", "2")                  \
 366                                _op"l %"_lx"3,%1; "                         \
 367                                _POST_EFLAGS("0", "4", "2")                 \
 368                                : "=m" (_eflags), "=m" ((_dst).val),        \
 369                                  "=&r" (_tmp)                              \
 370                                : _ly ((_src).val), "i" (EFLAGS_MASK));     \
 371                        break;                                              \
 372                case 8:                                                     \
 373                        __emulate_2op_8byte(_op, _src, _dst,                \
 374                                            _eflags, _qx, _qy);             \
 375                        break;                                              \
 376                }                                                           \
 377        } while (0)
 378
 379#define __emulate_2op(_op,_src,_dst,_eflags,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
 380        do {                                                                 \
 381                unsigned long __tmp;                                         \
 382                switch ((_dst).bytes) {                                      \
 383                case 1:                                                      \
 384                        __asm__ __volatile__ (                               \
 385                                _PRE_EFLAGS("0", "4", "2")                   \
 386                                _op"b %"_bx"3,%1; "                          \
 387                                _POST_EFLAGS("0", "4", "2")                  \
 388                                : "=m" (_eflags), "=m" ((_dst).val),         \
 389                                  "=&r" (__tmp)                              \
 390                                : _by ((_src).val), "i" (EFLAGS_MASK));      \
 391                        break;                                               \
 392                default:                                                     \
 393                        __emulate_2op_nobyte(_op, _src, _dst, _eflags,       \
 394                                             _wx, _wy, _lx, _ly, _qx, _qy);  \
 395                        break;                                               \
 396                }                                                            \
 397        } while (0)
 398
 399/* Source operand is byte-sized and may be restricted to just %cl. */
 400#define emulate_2op_SrcB(_op, _src, _dst, _eflags)                      \
 401        __emulate_2op(_op, _src, _dst, _eflags,                         \
 402                      "b", "c", "b", "c", "b", "c", "b", "c")
 403
 404/* Source operand is byte, word, long or quad sized. */
 405#define emulate_2op_SrcV(_op, _src, _dst, _eflags)                      \
 406        __emulate_2op(_op, _src, _dst, _eflags,                         \
 407                      "b", "q", "w", "r", _LO32, "r", "", "r")
 408
 409/* Source operand is word, long or quad sized. */
 410#define emulate_2op_SrcV_nobyte(_op, _src, _dst, _eflags)               \
 411        __emulate_2op_nobyte(_op, _src, _dst, _eflags,                  \
 412                             "w", "r", _LO32, "r", "", "r")
 413
 414/* Instruction has only one explicit operand (no source operand). */
 415#define emulate_1op(_op, _dst, _eflags)                                    \
 416        do {                                                            \
 417                unsigned long _tmp;                                     \
 418                                                                        \
 419                switch ((_dst).bytes) {                                 \
 420                case 1:                                                 \
 421                        __asm__ __volatile__ (                          \
 422                                _PRE_EFLAGS("0", "3", "2")              \
 423                                _op"b %1; "                             \
 424                                _POST_EFLAGS("0", "3", "2")             \
 425                                : "=m" (_eflags), "=m" ((_dst).val),    \
 426                                  "=&r" (_tmp)                          \
 427                                : "i" (EFLAGS_MASK));                   \
 428                        break;                                          \
 429                case 2:                                                 \
 430                        __asm__ __volatile__ (                          \
 431                                _PRE_EFLAGS("0", "3", "2")              \
 432                                _op"w %1; "                             \
 433                                _POST_EFLAGS("0", "3", "2")             \
 434                                : "=m" (_eflags), "=m" ((_dst).val),    \
 435                                  "=&r" (_tmp)                          \
 436                                : "i" (EFLAGS_MASK));                   \
 437                        break;                                          \
 438                case 4:                                                 \
 439                        __asm__ __volatile__ (                          \
 440                                _PRE_EFLAGS("0", "3", "2")              \
 441                                _op"l %1; "                             \
 442                                _POST_EFLAGS("0", "3", "2")             \
 443                                : "=m" (_eflags), "=m" ((_dst).val),    \
 444                                  "=&r" (_tmp)                          \
 445                                : "i" (EFLAGS_MASK));                   \
 446                        break;                                          \
 447                case 8:                                                 \
 448                        __emulate_1op_8byte(_op, _dst, _eflags);        \
 449                        break;                                          \
 450                }                                                       \
 451        } while (0)
 452
 453/* Emulate an instruction with quadword operands (x86/64 only). */
 454#if defined(CONFIG_X86_64)
 455#define __emulate_2op_8byte(_op, _src, _dst, _eflags, _qx, _qy)           \
 456        do {                                                              \
 457                __asm__ __volatile__ (                                    \
 458                        _PRE_EFLAGS("0", "4", "2")                        \
 459                        _op"q %"_qx"3,%1; "                               \
 460                        _POST_EFLAGS("0", "4", "2")                       \
 461                        : "=m" (_eflags), "=m" ((_dst).val), "=&r" (_tmp) \
 462                        : _qy ((_src).val), "i" (EFLAGS_MASK));         \
 463        } while (0)
 464
 465#define __emulate_1op_8byte(_op, _dst, _eflags)                           \
 466        do {                                                              \
 467                __asm__ __volatile__ (                                    \
 468                        _PRE_EFLAGS("0", "3", "2")                        \
 469                        _op"q %1; "                                       \
 470                        _POST_EFLAGS("0", "3", "2")                       \
 471                        : "=m" (_eflags), "=m" ((_dst).val), "=&r" (_tmp) \
 472                        : "i" (EFLAGS_MASK));                             \
 473        } while (0)
 474
 475#elif defined(__i386__)
 476#define __emulate_2op_8byte(_op, _src, _dst, _eflags, _qx, _qy)
 477#define __emulate_1op_8byte(_op, _dst, _eflags)
 478#endif                          /* __i386__ */
 479
 480/* Fetch next part of the instruction being emulated. */
 481#define insn_fetch(_type, _size, _eip)                                  \
 482({      unsigned long _x;                                               \
 483        rc = do_insn_fetch(ctxt, ops, (_eip), &_x, (_size));            \
 484        if (rc != 0)                                                    \
 485                goto done;                                              \
 486        (_eip) += (_size);                                              \
 487        (_type)_x;                                                      \
 488})
 489
 490static inline unsigned long ad_mask(struct decode_cache *c)
 491{
 492        return (1UL << (c->ad_bytes << 3)) - 1;
 493}
 494
 495/* Access/update address held in a register, based on addressing mode. */
 496static inline unsigned long
 497address_mask(struct decode_cache *c, unsigned long reg)
 498{
 499        if (c->ad_bytes == sizeof(unsigned long))
 500                return reg;
 501        else
 502                return reg & ad_mask(c);
 503}
 504
 505static inline unsigned long
 506register_address(struct decode_cache *c, unsigned long base, unsigned long reg)
 507{
 508        return base + address_mask(c, reg);
 509}
 510
 511static inline void
 512register_address_increment(struct decode_cache *c, unsigned long *reg, int inc)
 513{
 514        if (c->ad_bytes == sizeof(unsigned long))
 515                *reg += inc;
 516        else
 517                *reg = (*reg & ~ad_mask(c)) | ((*reg + inc) & ad_mask(c));
 518}
 519
 520static inline void jmp_rel(struct decode_cache *c, int rel)
 521{
 522        register_address_increment(c, &c->eip, rel);
 523}
 524
 525static void set_seg_override(struct decode_cache *c, int seg)
 526{
 527        c->has_seg_override = true;
 528        c->seg_override = seg;
 529}
 530
 531static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
 532{
 533        if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
 534                return 0;
 535
 536        return kvm_x86_ops->get_segment_base(ctxt->vcpu, seg);
 537}
 538
 539static unsigned long seg_override_base(struct x86_emulate_ctxt *ctxt,
 540                                       struct decode_cache *c)
 541{
 542        if (!c->has_seg_override)
 543                return 0;
 544
 545        return seg_base(ctxt, c->seg_override);
 546}
 547
 548static unsigned long es_base(struct x86_emulate_ctxt *ctxt)
 549{
 550        return seg_base(ctxt, VCPU_SREG_ES);
 551}
 552
 553static unsigned long ss_base(struct x86_emulate_ctxt *ctxt)
 554{
 555        return seg_base(ctxt, VCPU_SREG_SS);
 556}
 557
 558static int do_fetch_insn_byte(struct x86_emulate_ctxt *ctxt,
 559                              struct x86_emulate_ops *ops,
 560                              unsigned long linear, u8 *dest)
 561{
 562        struct fetch_cache *fc = &ctxt->decode.fetch;
 563        int rc;
 564        int size;
 565
 566        if (linear < fc->start || linear >= fc->end) {
 567                size = min(15UL, PAGE_SIZE - offset_in_page(linear));
 568                rc = ops->read_std(linear, fc->data, size, ctxt->vcpu);
 569                if (rc)
 570                        return rc;
 571                fc->start = linear;
 572                fc->end = linear + size;
 573        }
 574        *dest = fc->data[linear - fc->start];
 575        return 0;
 576}
 577
 578static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
 579                         struct x86_emulate_ops *ops,
 580                         unsigned long eip, void *dest, unsigned size)
 581{
 582        int rc = 0;
 583
 584        /* x86 instructions are limited to 15 bytes. */
 585        if (eip + size - ctxt->decode.eip_orig > 15)
 586                return X86EMUL_UNHANDLEABLE;
 587        eip += ctxt->cs_base;
 588        while (size--) {
 589                rc = do_fetch_insn_byte(ctxt, ops, eip++, dest++);
 590                if (rc)
 591                        return rc;
 592        }
 593        return 0;
 594}
 595
 596/*
 597 * Given the 'reg' portion of a ModRM byte, and a register block, return a
 598 * pointer into the block that addresses the relevant register.
 599 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
 600 */
 601static void *decode_register(u8 modrm_reg, unsigned long *regs,
 602                             int highbyte_regs)
 603{
 604        void *p;
 605
 606        p = &regs[modrm_reg];
 607        if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
 608                p = (unsigned char *)&regs[modrm_reg & 3] + 1;
 609        return p;
 610}
 611
 612static int read_descriptor(struct x86_emulate_ctxt *ctxt,
 613                           struct x86_emulate_ops *ops,
 614                           void *ptr,
 615                           u16 *size, unsigned long *address, int op_bytes)
 616{
 617        int rc;
 618
 619        if (op_bytes == 2)
 620                op_bytes = 3;
 621        *address = 0;
 622        rc = ops->read_std((unsigned long)ptr, (unsigned long *)size, 2,
 623                           ctxt->vcpu);
 624        if (rc)
 625                return rc;
 626        rc = ops->read_std((unsigned long)ptr + 2, address, op_bytes,
 627                           ctxt->vcpu);
 628        return rc;
 629}
 630
 631static int test_cc(unsigned int condition, unsigned int flags)
 632{
 633        int rc = 0;
 634
 635        switch ((condition & 15) >> 1) {
 636        case 0: /* o */
 637                rc |= (flags & EFLG_OF);
 638                break;
 639        case 1: /* b/c/nae */
 640                rc |= (flags & EFLG_CF);
 641                break;
 642        case 2: /* z/e */
 643                rc |= (flags & EFLG_ZF);
 644                break;
 645        case 3: /* be/na */
 646                rc |= (flags & (EFLG_CF|EFLG_ZF));
 647                break;
 648        case 4: /* s */
 649                rc |= (flags & EFLG_SF);
 650                break;
 651        case 5: /* p/pe */
 652                rc |= (flags & EFLG_PF);
 653                break;
 654        case 7: /* le/ng */
 655                rc |= (flags & EFLG_ZF);
 656                /* fall through */
 657        case 6: /* l/nge */
 658                rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
 659                break;
 660        }
 661
 662        /* Odd condition identifiers (lsb == 1) have inverted sense. */
 663        return (!!rc ^ (condition & 1));
 664}
 665
 666static void decode_register_operand(struct operand *op,
 667                                    struct decode_cache *c,
 668                                    int inhibit_bytereg)
 669{
 670        unsigned reg = c->modrm_reg;
 671        int highbyte_regs = c->rex_prefix == 0;
 672
 673        if (!(c->d & ModRM))
 674                reg = (c->b & 7) | ((c->rex_prefix & 1) << 3);
 675        op->type = OP_REG;
 676        if ((c->d & ByteOp) && !inhibit_bytereg) {
 677                op->ptr = decode_register(reg, c->regs, highbyte_regs);
 678                op->val = *(u8 *)op->ptr;
 679                op->bytes = 1;
 680        } else {
 681                op->ptr = decode_register(reg, c->regs, 0);
 682                op->bytes = c->op_bytes;
 683                switch (op->bytes) {
 684                case 2:
 685                        op->val = *(u16 *)op->ptr;
 686                        break;
 687                case 4:
 688                        op->val = *(u32 *)op->ptr;
 689                        break;
 690                case 8:
 691                        op->val = *(u64 *) op->ptr;
 692                        break;
 693                }
 694        }
 695        op->orig_val = op->val;
 696}
 697
 698static int decode_modrm(struct x86_emulate_ctxt *ctxt,
 699                        struct x86_emulate_ops *ops)
 700{
 701        struct decode_cache *c = &ctxt->decode;
 702        u8 sib;
 703        int index_reg = 0, base_reg = 0, scale;
 704        int rc = 0;
 705
 706        if (c->rex_prefix) {
 707                c->modrm_reg = (c->rex_prefix & 4) << 1;        /* REX.R */
 708                index_reg = (c->rex_prefix & 2) << 2; /* REX.X */
 709                c->modrm_rm = base_reg = (c->rex_prefix & 1) << 3; /* REG.B */
 710        }
 711
 712        c->modrm = insn_fetch(u8, 1, c->eip);
 713        c->modrm_mod |= (c->modrm & 0xc0) >> 6;
 714        c->modrm_reg |= (c->modrm & 0x38) >> 3;
 715        c->modrm_rm |= (c->modrm & 0x07);
 716        c->modrm_ea = 0;
 717        c->use_modrm_ea = 1;
 718
 719        if (c->modrm_mod == 3) {
 720                c->modrm_ptr = decode_register(c->modrm_rm,
 721                                               c->regs, c->d & ByteOp);
 722                c->modrm_val = *(unsigned long *)c->modrm_ptr;
 723                return rc;
 724        }
 725
 726        if (c->ad_bytes == 2) {
 727                unsigned bx = c->regs[VCPU_REGS_RBX];
 728                unsigned bp = c->regs[VCPU_REGS_RBP];
 729                unsigned si = c->regs[VCPU_REGS_RSI];
 730                unsigned di = c->regs[VCPU_REGS_RDI];
 731
 732                /* 16-bit ModR/M decode. */
 733                switch (c->modrm_mod) {
 734                case 0:
 735                        if (c->modrm_rm == 6)
 736                                c->modrm_ea += insn_fetch(u16, 2, c->eip);
 737                        break;
 738                case 1:
 739                        c->modrm_ea += insn_fetch(s8, 1, c->eip);
 740                        break;
 741                case 2:
 742                        c->modrm_ea += insn_fetch(u16, 2, c->eip);
 743                        break;
 744                }
 745                switch (c->modrm_rm) {
 746                case 0:
 747                        c->modrm_ea += bx + si;
 748                        break;
 749                case 1:
 750                        c->modrm_ea += bx + di;
 751                        break;
 752                case 2:
 753                        c->modrm_ea += bp + si;
 754                        break;
 755                case 3:
 756                        c->modrm_ea += bp + di;
 757                        break;
 758                case 4:
 759                        c->modrm_ea += si;
 760                        break;
 761                case 5:
 762                        c->modrm_ea += di;
 763                        break;
 764                case 6:
 765                        if (c->modrm_mod != 0)
 766                                c->modrm_ea += bp;
 767                        break;
 768                case 7:
 769                        c->modrm_ea += bx;
 770                        break;
 771                }
 772                if (c->modrm_rm == 2 || c->modrm_rm == 3 ||
 773                    (c->modrm_rm == 6 && c->modrm_mod != 0))
 774                        if (!c->has_seg_override)
 775                                set_seg_override(c, VCPU_SREG_SS);
 776                c->modrm_ea = (u16)c->modrm_ea;
 777        } else {
 778                /* 32/64-bit ModR/M decode. */
 779                if ((c->modrm_rm & 7) == 4) {
 780                        sib = insn_fetch(u8, 1, c->eip);
 781                        index_reg |= (sib >> 3) & 7;
 782                        base_reg |= sib & 7;
 783                        scale = sib >> 6;
 784
 785                        if ((base_reg & 7) == 5 && c->modrm_mod == 0)
 786                                c->modrm_ea += insn_fetch(s32, 4, c->eip);
 787                        else
 788                                c->modrm_ea += c->regs[base_reg];
 789                        if (index_reg != 4)
 790                                c->modrm_ea += c->regs[index_reg] << scale;
 791                } else if ((c->modrm_rm & 7) == 5 && c->modrm_mod == 0) {
 792                        if (ctxt->mode == X86EMUL_MODE_PROT64)
 793                                c->rip_relative = 1;
 794                } else
 795                        c->modrm_ea += c->regs[c->modrm_rm];
 796                switch (c->modrm_mod) {
 797                case 0:
 798                        if (c->modrm_rm == 5)
 799                                c->modrm_ea += insn_fetch(s32, 4, c->eip);
 800                        break;
 801                case 1:
 802                        c->modrm_ea += insn_fetch(s8, 1, c->eip);
 803                        break;
 804                case 2:
 805                        c->modrm_ea += insn_fetch(s32, 4, c->eip);
 806                        break;
 807                }
 808        }
 809done:
 810        return rc;
 811}
 812
 813static int decode_abs(struct x86_emulate_ctxt *ctxt,
 814                      struct x86_emulate_ops *ops)
 815{
 816        struct decode_cache *c = &ctxt->decode;
 817        int rc = 0;
 818
 819        switch (c->ad_bytes) {
 820        case 2:
 821                c->modrm_ea = insn_fetch(u16, 2, c->eip);
 822                break;
 823        case 4:
 824                c->modrm_ea = insn_fetch(u32, 4, c->eip);
 825                break;
 826        case 8:
 827                c->modrm_ea = insn_fetch(u64, 8, c->eip);
 828                break;
 829        }
 830done:
 831        return rc;
 832}
 833
 834int
 835x86_decode_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
 836{
 837        struct decode_cache *c = &ctxt->decode;
 838        int rc = 0;
 839        int mode = ctxt->mode;
 840        int def_op_bytes, def_ad_bytes, group;
 841
 842        /* Shadow copy of register state. Committed on successful emulation. */
 843
 844        memset(c, 0, sizeof(struct decode_cache));
 845        c->eip = c->eip_orig = ctxt->vcpu->arch.rip;
 846        ctxt->cs_base = seg_base(ctxt, VCPU_SREG_CS);
 847        memcpy(c->regs, ctxt->vcpu->arch.regs, sizeof c->regs);
 848
 849        switch (mode) {
 850        case X86EMUL_MODE_REAL:
 851        case X86EMUL_MODE_PROT16:
 852                def_op_bytes = def_ad_bytes = 2;
 853                break;
 854        case X86EMUL_MODE_PROT32:
 855                def_op_bytes = def_ad_bytes = 4;
 856                break;
 857#ifdef CONFIG_X86_64
 858        case X86EMUL_MODE_PROT64:
 859                def_op_bytes = 4;
 860                def_ad_bytes = 8;
 861                break;
 862#endif
 863        default:
 864                return -1;
 865        }
 866
 867        c->op_bytes = def_op_bytes;
 868        c->ad_bytes = def_ad_bytes;
 869
 870        /* Legacy prefixes. */
 871        for (;;) {
 872                switch (c->b = insn_fetch(u8, 1, c->eip)) {
 873                case 0x66:      /* operand-size override */
 874                        /* switch between 2/4 bytes */
 875                        c->op_bytes = def_op_bytes ^ 6;
 876                        break;
 877                case 0x67:      /* address-size override */
 878                        if (mode == X86EMUL_MODE_PROT64)
 879                                /* switch between 4/8 bytes */
 880                                c->ad_bytes = def_ad_bytes ^ 12;
 881                        else
 882                                /* switch between 2/4 bytes */
 883                                c->ad_bytes = def_ad_bytes ^ 6;
 884                        break;
 885                case 0x26:      /* ES override */
 886                case 0x2e:      /* CS override */
 887                case 0x36:      /* SS override */
 888                case 0x3e:      /* DS override */
 889                        set_seg_override(c, (c->b >> 3) & 3);
 890                        break;
 891                case 0x64:      /* FS override */
 892                case 0x65:      /* GS override */
 893                        set_seg_override(c, c->b & 7);
 894                        break;
 895                case 0x40 ... 0x4f: /* REX */
 896                        if (mode != X86EMUL_MODE_PROT64)
 897                                goto done_prefixes;
 898                        c->rex_prefix = c->b;
 899                        continue;
 900                case 0xf0:      /* LOCK */
 901                        c->lock_prefix = 1;
 902                        break;
 903                case 0xf2:      /* REPNE/REPNZ */
 904                        c->rep_prefix = REPNE_PREFIX;
 905                        break;
 906                case 0xf3:      /* REP/REPE/REPZ */
 907                        c->rep_prefix = REPE_PREFIX;
 908                        break;
 909                default:
 910                        goto done_prefixes;
 911                }
 912
 913                /* Any legacy prefix after a REX prefix nullifies its effect. */
 914
 915                c->rex_prefix = 0;
 916        }
 917
 918done_prefixes:
 919
 920        /* REX prefix. */
 921        if (c->rex_prefix)
 922                if (c->rex_prefix & 8)
 923                        c->op_bytes = 8;        /* REX.W */
 924
 925        /* Opcode byte(s). */
 926        c->d = opcode_table[c->b];
 927        if (c->d == 0) {
 928                /* Two-byte opcode? */
 929                if (c->b == 0x0f) {
 930                        c->twobyte = 1;
 931                        c->b = insn_fetch(u8, 1, c->eip);
 932                        c->d = twobyte_table[c->b];
 933                }
 934        }
 935
 936        if (c->d & Group) {
 937                group = c->d & GroupMask;
 938                c->modrm = insn_fetch(u8, 1, c->eip);
 939                --c->eip;
 940
 941                group = (group << 3) + ((c->modrm >> 3) & 7);
 942                if ((c->d & GroupDual) && (c->modrm >> 6) == 3)
 943                        c->d = group2_table[group];
 944                else
 945                        c->d = group_table[group];
 946        }
 947
 948        /* Unrecognised? */
 949        if (c->d == 0) {
 950                DPRINTF("Cannot emulate %02x\n", c->b);
 951                return -1;
 952        }
 953
 954        if (mode == X86EMUL_MODE_PROT64 && (c->d & Stack))
 955                c->op_bytes = 8;
 956
 957        /* ModRM and SIB bytes. */
 958        if (c->d & ModRM)
 959                rc = decode_modrm(ctxt, ops);
 960        else if (c->d & MemAbs)
 961                rc = decode_abs(ctxt, ops);
 962        if (rc)
 963                goto done;
 964
 965        if (!c->has_seg_override)
 966                set_seg_override(c, VCPU_SREG_DS);
 967
 968        if (!(!c->twobyte && c->b == 0x8d))
 969                c->modrm_ea += seg_override_base(ctxt, c);
 970
 971        if (c->ad_bytes != 8)
 972                c->modrm_ea = (u32)c->modrm_ea;
 973        /*
 974         * Decode and fetch the source operand: register, memory
 975         * or immediate.
 976         */
 977        switch (c->d & SrcMask) {
 978        case SrcNone:
 979                break;
 980        case SrcReg:
 981                decode_register_operand(&c->src, c, 0);
 982                break;
 983        case SrcMem16:
 984                c->src.bytes = 2;
 985                goto srcmem_common;
 986        case SrcMem32:
 987                c->src.bytes = 4;
 988                goto srcmem_common;
 989        case SrcMem:
 990                c->src.bytes = (c->d & ByteOp) ? 1 :
 991                                                           c->op_bytes;
 992                /* Don't fetch the address for invlpg: it could be unmapped. */
 993                if (c->twobyte && c->b == 0x01 && c->modrm_reg == 7)
 994                        break;
 995        srcmem_common:
 996                /*
 997                 * For instructions with a ModR/M byte, switch to register
 998                 * access if Mod = 3.
 999                 */
1000                if ((c->d & ModRM) && c->modrm_mod == 3) {
1001                        c->src.type = OP_REG;
1002                        c->src.val = c->modrm_val;
1003                        c->src.ptr = c->modrm_ptr;
1004                        break;
1005                }
1006                c->src.type = OP_MEM;
1007                break;
1008        case SrcImm:
1009                c->src.type = OP_IMM;
1010                c->src.ptr = (unsigned long *)c->eip;
1011                c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1012                if (c->src.bytes == 8)
1013                        c->src.bytes = 4;
1014                /* NB. Immediates are sign-extended as necessary. */
1015                switch (c->src.bytes) {
1016                case 1:
1017                        c->src.val = insn_fetch(s8, 1, c->eip);
1018                        break;
1019                case 2:
1020                        c->src.val = insn_fetch(s16, 2, c->eip);
1021                        break;
1022                case 4:
1023                        c->src.val = insn_fetch(s32, 4, c->eip);
1024                        break;
1025                }
1026                break;
1027        case SrcImmByte:
1028                c->src.type = OP_IMM;
1029                c->src.ptr = (unsigned long *)c->eip;
1030                c->src.bytes = 1;
1031                c->src.val = insn_fetch(s8, 1, c->eip);
1032                break;
1033        }
1034
1035        /* Decode and fetch the destination operand: register or memory. */
1036        switch (c->d & DstMask) {
1037        case ImplicitOps:
1038                /* Special instructions do their own operand decoding. */
1039                return 0;
1040        case DstReg:
1041                decode_register_operand(&c->dst, c,
1042                         c->twobyte && (c->b == 0xb6 || c->b == 0xb7));
1043                break;
1044        case DstMem:
1045                if ((c->d & ModRM) && c->modrm_mod == 3) {
1046                        c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1047                        c->dst.type = OP_REG;
1048                        c->dst.val = c->dst.orig_val = c->modrm_val;
1049                        c->dst.ptr = c->modrm_ptr;
1050                        break;
1051                }
1052                c->dst.type = OP_MEM;
1053                break;
1054        }
1055
1056        if (c->rip_relative)
1057                c->modrm_ea += c->eip;
1058
1059done:
1060        return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0;
1061}
1062
1063static inline void emulate_push(struct x86_emulate_ctxt *ctxt)
1064{
1065        struct decode_cache *c = &ctxt->decode;
1066
1067        c->dst.type  = OP_MEM;
1068        c->dst.bytes = c->op_bytes;
1069        c->dst.val = c->src.val;
1070        register_address_increment(c, &c->regs[VCPU_REGS_RSP], -c->op_bytes);
1071        c->dst.ptr = (void *) register_address(c, ss_base(ctxt),
1072                                               c->regs[VCPU_REGS_RSP]);
1073}
1074
1075static inline int emulate_grp1a(struct x86_emulate_ctxt *ctxt,
1076                                struct x86_emulate_ops *ops)
1077{
1078        struct decode_cache *c = &ctxt->decode;
1079        int rc;
1080
1081        rc = ops->read_std(register_address(c, ss_base(ctxt),
1082                                            c->regs[VCPU_REGS_RSP]),
1083                           &c->dst.val, c->dst.bytes, ctxt->vcpu);
1084        if (rc != 0)
1085                return rc;
1086
1087        register_address_increment(c, &c->regs[VCPU_REGS_RSP], c->dst.bytes);
1088
1089        return 0;
1090}
1091
1092static inline void emulate_grp2(struct x86_emulate_ctxt *ctxt)
1093{
1094        struct decode_cache *c = &ctxt->decode;
1095        switch (c->modrm_reg) {
1096        case 0: /* rol */
1097                emulate_2op_SrcB("rol", c->src, c->dst, ctxt->eflags);
1098                break;
1099        case 1: /* ror */
1100                emulate_2op_SrcB("ror", c->src, c->dst, ctxt->eflags);
1101                break;
1102        case 2: /* rcl */
1103                emulate_2op_SrcB("rcl", c->src, c->dst, ctxt->eflags);
1104                break;
1105        case 3: /* rcr */
1106                emulate_2op_SrcB("rcr", c->src, c->dst, ctxt->eflags);
1107                break;
1108        case 4: /* sal/shl */
1109        case 6: /* sal/shl */
1110                emulate_2op_SrcB("sal", c->src, c->dst, ctxt->eflags);
1111                break;
1112        case 5: /* shr */
1113                emulate_2op_SrcB("shr", c->src, c->dst, ctxt->eflags);
1114                break;
1115        case 7: /* sar */
1116                emulate_2op_SrcB("sar", c->src, c->dst, ctxt->eflags);
1117                break;
1118        }
1119}
1120
1121static inline int emulate_grp3(struct x86_emulate_ctxt *ctxt,
1122                               struct x86_emulate_ops *ops)
1123{
1124        struct decode_cache *c = &ctxt->decode;
1125        int rc = 0;
1126
1127        switch (c->modrm_reg) {
1128        case 0 ... 1:   /* test */
1129                emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
1130                break;
1131        case 2: /* not */
1132                c->dst.val = ~c->dst.val;
1133                break;
1134        case 3: /* neg */
1135                emulate_1op("neg", c->dst, ctxt->eflags);
1136                break;
1137        default:
1138                DPRINTF("Cannot emulate %02x\n", c->b);
1139                rc = X86EMUL_UNHANDLEABLE;
1140                break;
1141        }
1142        return rc;
1143}
1144
1145static inline int emulate_grp45(struct x86_emulate_ctxt *ctxt,
1146                               struct x86_emulate_ops *ops)
1147{
1148        struct decode_cache *c = &ctxt->decode;
1149
1150        switch (c->modrm_reg) {
1151        case 0: /* inc */
1152                emulate_1op("inc", c->dst, ctxt->eflags);
1153                break;
1154        case 1: /* dec */
1155                emulate_1op("dec", c->dst, ctxt->eflags);
1156                break;
1157        case 4: /* jmp abs */
1158                c->eip = c->src.val;
1159                break;
1160        case 6: /* push */
1161                emulate_push(ctxt);
1162                break;
1163        }
1164        return 0;
1165}
1166
1167static inline int emulate_grp9(struct x86_emulate_ctxt *ctxt,
1168                               struct x86_emulate_ops *ops,
1169                               unsigned long memop)
1170{
1171        struct decode_cache *c = &ctxt->decode;
1172        u64 old, new;
1173        int rc;
1174
1175        rc = ops->read_emulated(memop, &old, 8, ctxt->vcpu);
1176        if (rc != 0)
1177                return rc;
1178
1179        if (((u32) (old >> 0) != (u32) c->regs[VCPU_REGS_RAX]) ||
1180            ((u32) (old >> 32) != (u32) c->regs[VCPU_REGS_RDX])) {
1181
1182                c->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1183                c->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1184                ctxt->eflags &= ~EFLG_ZF;
1185
1186        } else {
1187                new = ((u64)c->regs[VCPU_REGS_RCX] << 32) |
1188                       (u32) c->regs[VCPU_REGS_RBX];
1189
1190                rc = ops->cmpxchg_emulated(memop, &old, &new, 8, ctxt->vcpu);
1191                if (rc != 0)
1192                        return rc;
1193                ctxt->eflags |= EFLG_ZF;
1194        }
1195        return 0;
1196}
1197
1198static inline int writeback(struct x86_emulate_ctxt *ctxt,
1199                            struct x86_emulate_ops *ops)
1200{
1201        int rc;
1202        struct decode_cache *c = &ctxt->decode;
1203
1204        switch (c->dst.type) {
1205        case OP_REG:
1206                /* The 4-byte case *is* correct:
1207                 * in 64-bit mode we zero-extend.
1208                 */
1209                switch (c->dst.bytes) {
1210                case 1:
1211                        *(u8 *)c->dst.ptr = (u8)c->dst.val;
1212                        break;
1213                case 2:
1214                        *(u16 *)c->dst.ptr = (u16)c->dst.val;
1215                        break;
1216                case 4:
1217                        *c->dst.ptr = (u32)c->dst.val;
1218                        break;  /* 64b: zero-ext */
1219                case 8:
1220                        *c->dst.ptr = c->dst.val;
1221                        break;
1222                }
1223                break;
1224        case OP_MEM:
1225                if (c->lock_prefix)
1226                        rc = ops->cmpxchg_emulated(
1227                                        (unsigned long)c->dst.ptr,
1228                                        &c->dst.orig_val,
1229                                        &c->dst.val,
1230                                        c->dst.bytes,
1231                                        ctxt->vcpu);
1232                else
1233                        rc = ops->write_emulated(
1234                                        (unsigned long)c->dst.ptr,
1235                                        &c->dst.val,
1236                                        c->dst.bytes,
1237                                        ctxt->vcpu);
1238                if (rc != 0)
1239                        return rc;
1240                break;
1241        case OP_NONE:
1242                /* no writeback */
1243                break;
1244        default:
1245                break;
1246        }
1247        return 0;
1248}
1249
1250int
1251x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1252{
1253        unsigned long memop = 0;
1254        u64 msr_data;
1255        unsigned long saved_eip = 0;
1256        struct decode_cache *c = &ctxt->decode;
1257        int rc = 0;
1258
1259        /* Shadow copy of register state. Committed on successful emulation.
1260         * NOTE: we can copy them from vcpu as x86_decode_insn() doesn't
1261         * modify them.
1262         */
1263
1264        memcpy(c->regs, ctxt->vcpu->arch.regs, sizeof c->regs);
1265        saved_eip = c->eip;
1266
1267        if (((c->d & ModRM) && (c->modrm_mod != 3)) || (c->d & MemAbs))
1268                memop = c->modrm_ea;
1269
1270        if (c->rep_prefix && (c->d & String)) {
1271                /* All REP prefixes have the same first termination condition */
1272                if (c->regs[VCPU_REGS_RCX] == 0) {
1273                        ctxt->vcpu->arch.rip = c->eip;
1274                        goto done;
1275                }
1276                /* The second termination condition only applies for REPE
1277                 * and REPNE. Test if the repeat string operation prefix is
1278                 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
1279                 * corresponding termination condition according to:
1280                 *      - if REPE/REPZ and ZF = 0 then done
1281                 *      - if REPNE/REPNZ and ZF = 1 then done
1282                 */
1283                if ((c->b == 0xa6) || (c->b == 0xa7) ||
1284                                (c->b == 0xae) || (c->b == 0xaf)) {
1285                        if ((c->rep_prefix == REPE_PREFIX) &&
1286                                ((ctxt->eflags & EFLG_ZF) == 0)) {
1287                                        ctxt->vcpu->arch.rip = c->eip;
1288                                        goto done;
1289                        }
1290                        if ((c->rep_prefix == REPNE_PREFIX) &&
1291                                ((ctxt->eflags & EFLG_ZF) == EFLG_ZF)) {
1292                                ctxt->vcpu->arch.rip = c->eip;
1293                                goto done;
1294                        }
1295                }
1296                c->regs[VCPU_REGS_RCX]--;
1297                c->eip = ctxt->vcpu->arch.rip;
1298        }
1299
1300        if (c->src.type == OP_MEM) {
1301                c->src.ptr = (unsigned long *)memop;
1302                c->src.val = 0;
1303                rc = ops->read_emulated((unsigned long)c->src.ptr,
1304                                        &c->src.val,
1305                                        c->src.bytes,
1306                                        ctxt->vcpu);
1307                if (rc != 0)
1308                        goto done;
1309                c->src.orig_val = c->src.val;
1310        }
1311
1312        if ((c->d & DstMask) == ImplicitOps)
1313                goto special_insn;
1314
1315
1316        if (c->dst.type == OP_MEM) {
1317                c->dst.ptr = (unsigned long *)memop;
1318                c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1319                c->dst.val = 0;
1320                if (c->d & BitOp) {
1321                        unsigned long mask = ~(c->dst.bytes * 8 - 1);
1322
1323                        c->dst.ptr = (void *)c->dst.ptr +
1324                                                   (c->src.val & mask) / 8;
1325                }
1326                if (!(c->d & Mov) &&
1327                                   /* optimisation - avoid slow emulated read */
1328                    ((rc = ops->read_emulated((unsigned long)c->dst.ptr,
1329                                           &c->dst.val,
1330                                          c->dst.bytes, ctxt->vcpu)) != 0))
1331                        goto done;
1332        }
1333        c->dst.orig_val = c->dst.val;
1334
1335special_insn:
1336
1337        if (c->twobyte)
1338                goto twobyte_insn;
1339
1340        switch (c->b) {
1341        case 0x00 ... 0x05:
1342              add:              /* add */
1343                emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
1344                break;
1345        case 0x08 ... 0x0d:
1346              or:               /* or */
1347                emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
1348                break;
1349        case 0x10 ... 0x15:
1350              adc:              /* adc */
1351                emulate_2op_SrcV("adc", c->src, c->dst, ctxt->eflags);
1352                break;
1353        case 0x18 ... 0x1d:
1354              sbb:              /* sbb */
1355                emulate_2op_SrcV("sbb", c->src, c->dst, ctxt->eflags);
1356                break;
1357        case 0x20 ... 0x23:
1358              and:              /* and */
1359                emulate_2op_SrcV("and", c->src, c->dst, ctxt->eflags);
1360                break;
1361        case 0x24:              /* and al imm8 */
1362                c->dst.type = OP_REG;
1363                c->dst.ptr = &c->regs[VCPU_REGS_RAX];
1364                c->dst.val = *(u8 *)c->dst.ptr;
1365                c->dst.bytes = 1;
1366                c->dst.orig_val = c->dst.val;
1367                goto and;
1368        case 0x25:              /* and ax imm16, or eax imm32 */
1369                c->dst.type = OP_REG;
1370                c->dst.bytes = c->op_bytes;
1371                c->dst.ptr = &c->regs[VCPU_REGS_RAX];
1372                if (c->op_bytes == 2)
1373                        c->dst.val = *(u16 *)c->dst.ptr;
1374                else
1375                        c->dst.val = *(u32 *)c->dst.ptr;
1376                c->dst.orig_val = c->dst.val;
1377                goto and;
1378        case 0x28 ... 0x2d:
1379              sub:              /* sub */
1380                emulate_2op_SrcV("sub", c->src, c->dst, ctxt->eflags);
1381                break;
1382        case 0x30 ... 0x35:
1383              xor:              /* xor */
1384                emulate_2op_SrcV("xor", c->src, c->dst, ctxt->eflags);
1385                break;
1386        case 0x38 ... 0x3d:
1387              cmp:              /* cmp */
1388                emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
1389                break;
1390        case 0x40 ... 0x47: /* inc r16/r32 */
1391                emulate_1op("inc", c->dst, ctxt->eflags);
1392                break;
1393        case 0x48 ... 0x4f: /* dec r16/r32 */
1394                emulate_1op("dec", c->dst, ctxt->eflags);
1395                break;
1396        case 0x50 ... 0x57:  /* push reg */
1397                c->dst.type  = OP_MEM;
1398                c->dst.bytes = c->op_bytes;
1399                c->dst.val = c->src.val;
1400                register_address_increment(c, &c->regs[VCPU_REGS_RSP],
1401                                           -c->op_bytes);
1402                c->dst.ptr = (void *) register_address(
1403                        c, ss_base(ctxt), c->regs[VCPU_REGS_RSP]);
1404                break;
1405        case 0x58 ... 0x5f: /* pop reg */
1406        pop_instruction:
1407                if ((rc = ops->read_std(register_address(c, ss_base(ctxt),
1408                        c->regs[VCPU_REGS_RSP]), c->dst.ptr,
1409                        c->op_bytes, ctxt->vcpu)) != 0)
1410                        goto done;
1411
1412                register_address_increment(c, &c->regs[VCPU_REGS_RSP],
1413                                           c->op_bytes);
1414                c->dst.type = OP_NONE;  /* Disable writeback. */
1415                break;
1416        case 0x63:              /* movsxd */
1417                if (ctxt->mode != X86EMUL_MODE_PROT64)
1418                        goto cannot_emulate;
1419                c->dst.val = (s32) c->src.val;
1420                break;
1421        case 0x68: /* push imm */
1422        case 0x6a: /* push imm8 */
1423                emulate_push(ctxt);
1424                break;
1425        case 0x6c:              /* insb */
1426        case 0x6d:              /* insw/insd */
1427                 if (kvm_emulate_pio_string(ctxt->vcpu, NULL,
1428                                1,
1429                                (c->d & ByteOp) ? 1 : c->op_bytes,
1430                                c->rep_prefix ?
1431                                address_mask(c, c->regs[VCPU_REGS_RCX]) : 1,
1432                                (ctxt->eflags & EFLG_DF),
1433                                register_address(c, es_base(ctxt),
1434                                                 c->regs[VCPU_REGS_RDI]),
1435                                c->rep_prefix,
1436                                c->regs[VCPU_REGS_RDX]) == 0) {
1437                        c->eip = saved_eip;
1438                        return -1;
1439                }
1440                return 0;
1441        case 0x6e:              /* outsb */
1442        case 0x6f:              /* outsw/outsd */
1443                if (kvm_emulate_pio_string(ctxt->vcpu, NULL,
1444                                0,
1445                                (c->d & ByteOp) ? 1 : c->op_bytes,
1446                                c->rep_prefix ?
1447                                address_mask(c, c->regs[VCPU_REGS_RCX]) : 1,
1448                                (ctxt->eflags & EFLG_DF),
1449                                         register_address(c,
1450                                          seg_override_base(ctxt, c),
1451                                                 c->regs[VCPU_REGS_RSI]),
1452                                c->rep_prefix,
1453                                c->regs[VCPU_REGS_RDX]) == 0) {
1454                        c->eip = saved_eip;
1455                        return -1;
1456                }
1457                return 0;
1458        case 0x70 ... 0x7f: /* jcc (short) */ {
1459                int rel = insn_fetch(s8, 1, c->eip);
1460
1461                if (test_cc(c->b, ctxt->eflags))
1462                        jmp_rel(c, rel);
1463                break;
1464        }
1465        case 0x80 ... 0x83:     /* Grp1 */
1466                switch (c->modrm_reg) {
1467                case 0:
1468                        goto add;
1469                case 1:
1470                        goto or;
1471                case 2:
1472                        goto adc;
1473                case 3:
1474                        goto sbb;
1475                case 4:
1476                        goto and;
1477                case 5:
1478                        goto sub;
1479                case 6:
1480                        goto xor;
1481                case 7:
1482                        goto cmp;
1483                }
1484                break;
1485        case 0x84 ... 0x85:
1486                emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
1487                break;
1488        case 0x86 ... 0x87:     /* xchg */
1489        xchg:
1490                /* Write back the register source. */
1491                switch (c->dst.bytes) {
1492                case 1:
1493                        *(u8 *) c->src.ptr = (u8) c->dst.val;
1494                        break;
1495                case 2:
1496                        *(u16 *) c->src.ptr = (u16) c->dst.val;
1497                        break;
1498                case 4:
1499                        *c->src.ptr = (u32) c->dst.val;
1500                        break;  /* 64b reg: zero-extend */
1501                case 8:
1502                        *c->src.ptr = c->dst.val;
1503                        break;
1504                }
1505                /*
1506                 * Write back the memory destination with implicit LOCK
1507                 * prefix.
1508                 */
1509                c->dst.val = c->src.val;
1510                c->lock_prefix = 1;
1511                break;
1512        case 0x88 ... 0x8b:     /* mov */
1513                goto mov;
1514        case 0x8c: { /* mov r/m, sreg */
1515                struct kvm_segment segreg;
1516
1517                if (c->modrm_reg <= 5)
1518                        kvm_get_segment(ctxt->vcpu, &segreg, c->modrm_reg);
1519                else {
1520                        printk(KERN_INFO "0x8c: Invalid segreg in modrm byte 0x%02x\n",
1521                               c->modrm);
1522                        goto cannot_emulate;
1523                }
1524                c->dst.val = segreg.selector;
1525                break;
1526        }
1527        case 0x8d: /* lea r16/r32, m */
1528                c->dst.val = c->modrm_ea;
1529                break;
1530        case 0x8e: { /* mov seg, r/m16 */
1531                uint16_t sel;
1532                int type_bits;
1533                int err;
1534
1535                sel = c->src.val;
1536                if (c->modrm_reg <= 5) {
1537                        type_bits = (c->modrm_reg == 1) ? 9 : 1;
1538                        err = kvm_load_segment_descriptor(ctxt->vcpu, sel,
1539                                                          type_bits, c->modrm_reg);
1540                } else {
1541                        printk(KERN_INFO "Invalid segreg in modrm byte 0x%02x\n",
1542                                        c->modrm);
1543                        goto cannot_emulate;
1544                }
1545
1546                if (err < 0)
1547                        goto cannot_emulate;
1548
1549                c->dst.type = OP_NONE;  /* Disable writeback. */
1550                break;
1551        }
1552        case 0x8f:              /* pop (sole member of Grp1a) */
1553                rc = emulate_grp1a(ctxt, ops);
1554                if (rc != 0)
1555                        goto done;
1556                break;
1557        case 0x90: /* nop / xchg r8,rax */
1558                if (!(c->rex_prefix & 1)) { /* nop */
1559                        c->dst.type = OP_NONE;
1560                        break;
1561                }
1562        case 0x91 ... 0x97: /* xchg reg,rax */
1563                c->src.type = c->dst.type = OP_REG;
1564                c->src.bytes = c->dst.bytes = c->op_bytes;
1565                c->src.ptr = (unsigned long *) &c->regs[VCPU_REGS_RAX];
1566                c->src.val = *(c->src.ptr);
1567                goto xchg;
1568        case 0x9c: /* pushf */
1569                c->src.val =  (unsigned long) ctxt->eflags;
1570                emulate_push(ctxt);
1571                break;
1572        case 0x9d: /* popf */
1573                c->dst.ptr = (unsigned long *) &ctxt->eflags;
1574                goto pop_instruction;
1575        case 0xa0 ... 0xa1:     /* mov */
1576                c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX];
1577                c->dst.val = c->src.val;
1578                break;
1579        case 0xa2 ... 0xa3:     /* mov */
1580                c->dst.val = (unsigned long)c->regs[VCPU_REGS_RAX];
1581                break;
1582        case 0xa4 ... 0xa5:     /* movs */
1583                c->dst.type = OP_MEM;
1584                c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1585                c->dst.ptr = (unsigned long *)register_address(c,
1586                                                   es_base(ctxt),
1587                                                   c->regs[VCPU_REGS_RDI]);
1588                if ((rc = ops->read_emulated(register_address(c,
1589                                           seg_override_base(ctxt, c),
1590                                        c->regs[VCPU_REGS_RSI]),
1591                                        &c->dst.val,
1592                                        c->dst.bytes, ctxt->vcpu)) != 0)
1593                        goto done;
1594                register_address_increment(c, &c->regs[VCPU_REGS_RSI],
1595                                       (ctxt->eflags & EFLG_DF) ? -c->dst.bytes
1596                                                           : c->dst.bytes);
1597                register_address_increment(c, &c->regs[VCPU_REGS_RDI],
1598                                       (ctxt->eflags & EFLG_DF) ? -c->dst.bytes
1599                                                           : c->dst.bytes);
1600                break;
1601        case 0xa6 ... 0xa7:     /* cmps */
1602                c->src.type = OP_NONE; /* Disable writeback. */
1603                c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1604                c->src.ptr = (unsigned long *)register_address(c,
1605                                       seg_override_base(ctxt, c),
1606                                                   c->regs[VCPU_REGS_RSI]);
1607                if ((rc = ops->read_emulated((unsigned long)c->src.ptr,
1608                                                &c->src.val,
1609                                                c->src.bytes,
1610                                                ctxt->vcpu)) != 0)
1611                        goto done;
1612
1613                c->dst.type = OP_NONE; /* Disable writeback. */
1614                c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1615                c->dst.ptr = (unsigned long *)register_address(c,
1616                                                   es_base(ctxt),
1617                                                   c->regs[VCPU_REGS_RDI]);
1618                if ((rc = ops->read_emulated((unsigned long)c->dst.ptr,
1619                                                &c->dst.val,
1620                                                c->dst.bytes,
1621                                                ctxt->vcpu)) != 0)
1622                        goto done;
1623
1624                DPRINTF("cmps: mem1=0x%p mem2=0x%p\n", c->src.ptr, c->dst.ptr);
1625
1626                emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
1627
1628                register_address_increment(c, &c->regs[VCPU_REGS_RSI],
1629                                       (ctxt->eflags & EFLG_DF) ? -c->src.bytes
1630                                                                  : c->src.bytes);
1631                register_address_increment(c, &c->regs[VCPU_REGS_RDI],
1632                                       (ctxt->eflags & EFLG_DF) ? -c->dst.bytes
1633                                                                  : c->dst.bytes);
1634
1635                break;
1636        case 0xaa ... 0xab:     /* stos */
1637                c->dst.type = OP_MEM;
1638                c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1639                c->dst.ptr = (unsigned long *)register_address(c,
1640                                                   es_base(ctxt),
1641                                                   c->regs[VCPU_REGS_RDI]);
1642                c->dst.val = c->regs[VCPU_REGS_RAX];
1643                register_address_increment(c, &c->regs[VCPU_REGS_RDI],
1644                                       (ctxt->eflags & EFLG_DF) ? -c->dst.bytes
1645                                                           : c->dst.bytes);
1646                break;
1647        case 0xac ... 0xad:     /* lods */
1648                c->dst.type = OP_REG;
1649                c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1650                c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX];
1651                if ((rc = ops->read_emulated(register_address(c,
1652                                                 seg_override_base(ctxt, c),
1653                                                 c->regs[VCPU_REGS_RSI]),
1654                                                 &c->dst.val,
1655                                                 c->dst.bytes,
1656                                                 ctxt->vcpu)) != 0)
1657                        goto done;
1658                register_address_increment(c, &c->regs[VCPU_REGS_RSI],
1659                                       (ctxt->eflags & EFLG_DF) ? -c->dst.bytes
1660                                                           : c->dst.bytes);
1661                break;
1662        case 0xae ... 0xaf:     /* scas */
1663                DPRINTF("Urk! I don't handle SCAS.\n");
1664                goto cannot_emulate;
1665        case 0xb8: /* mov r, imm */
1666                goto mov;
1667        case 0xc0 ... 0xc1:
1668                emulate_grp2(ctxt);
1669                break;
1670        case 0xc3: /* ret */
1671                c->dst.ptr = &c->eip;
1672                goto pop_instruction;
1673        case 0xc6 ... 0xc7:     /* mov (sole member of Grp11) */
1674        mov:
1675                c->dst.val = c->src.val;
1676                break;
1677        case 0xd0 ... 0xd1:     /* Grp2 */
1678                c->src.val = 1;
1679                emulate_grp2(ctxt);
1680                break;
1681        case 0xd2 ... 0xd3:     /* Grp2 */
1682                c->src.val = c->regs[VCPU_REGS_RCX];
1683                emulate_grp2(ctxt);
1684                break;
1685        case 0xe8: /* call (near) */ {
1686                long int rel;
1687                switch (c->op_bytes) {
1688                case 2:
1689                        rel = insn_fetch(s16, 2, c->eip);
1690                        break;
1691                case 4:
1692                        rel = insn_fetch(s32, 4, c->eip);
1693                        break;
1694                default:
1695                        DPRINTF("Call: Invalid op_bytes\n");
1696                        goto cannot_emulate;
1697                }
1698                c->src.val = (unsigned long) c->eip;
1699                jmp_rel(c, rel);
1700                c->op_bytes = c->ad_bytes;
1701                emulate_push(ctxt);
1702                break;
1703        }
1704        case 0xe9: /* jmp rel */
1705                goto jmp;
1706        case 0xea: /* jmp far */ {
1707                uint32_t eip;
1708                uint16_t sel;
1709
1710                switch (c->op_bytes) {
1711                case 2:
1712                        eip = insn_fetch(u16, 2, c->eip);
1713                        break;
1714                case 4:
1715                        eip = insn_fetch(u32, 4, c->eip);
1716                        break;
1717                default:
1718                        DPRINTF("jmp far: Invalid op_bytes\n");
1719                        goto cannot_emulate;
1720                }
1721                sel = insn_fetch(u16, 2, c->eip);
1722                if (kvm_load_segment_descriptor(ctxt->vcpu, sel, 9, VCPU_SREG_CS) < 0) {
1723                        DPRINTF("jmp far: Failed to load CS descriptor\n");
1724                        goto cannot_emulate;
1725                }
1726
1727                c->eip = eip;
1728                break;
1729        }
1730        case 0xeb:
1731              jmp:              /* jmp rel short */
1732                jmp_rel(c, c->src.val);
1733                c->dst.type = OP_NONE; /* Disable writeback. */
1734                break;
1735        case 0xf4:              /* hlt */
1736                ctxt->vcpu->arch.halt_request = 1;
1737                break;
1738        case 0xf5:      /* cmc */
1739                /* complement carry flag from eflags reg */
1740                ctxt->eflags ^= EFLG_CF;
1741                c->dst.type = OP_NONE;  /* Disable writeback. */
1742                break;
1743        case 0xf6 ... 0xf7:     /* Grp3 */
1744                rc = emulate_grp3(ctxt, ops);
1745                if (rc != 0)
1746                        goto done;
1747                break;
1748        case 0xf8: /* clc */
1749                ctxt->eflags &= ~EFLG_CF;
1750                c->dst.type = OP_NONE;  /* Disable writeback. */
1751                break;
1752        case 0xfa: /* cli */
1753                ctxt->eflags &= ~X86_EFLAGS_IF;
1754                c->dst.type = OP_NONE;  /* Disable writeback. */
1755                break;
1756        case 0xfb: /* sti */
1757                ctxt->eflags |= X86_EFLAGS_IF;
1758                c->dst.type = OP_NONE;  /* Disable writeback. */
1759                break;
1760        case 0xfe ... 0xff:     /* Grp4/Grp5 */
1761                rc = emulate_grp45(ctxt, ops);
1762                if (rc != 0)
1763                        goto done;
1764                break;
1765        }
1766
1767writeback:
1768        rc = writeback(ctxt, ops);
1769        if (rc != 0)
1770                goto done;
1771
1772        /* Commit shadow register state. */
1773        memcpy(ctxt->vcpu->arch.regs, c->regs, sizeof c->regs);
1774        ctxt->vcpu->arch.rip = c->eip;
1775
1776done:
1777        if (rc == X86EMUL_UNHANDLEABLE) {
1778                c->eip = saved_eip;
1779                return -1;
1780        }
1781        return 0;
1782
1783twobyte_insn:
1784        switch (c->b) {
1785        case 0x01: /* lgdt, lidt, lmsw */
1786                switch (c->modrm_reg) {
1787                        u16 size;
1788                        unsigned long address;
1789
1790                case 0: /* vmcall */
1791                        if (c->modrm_mod != 3 || c->modrm_rm != 1)
1792                                goto cannot_emulate;
1793
1794                        rc = kvm_fix_hypercall(ctxt->vcpu);
1795                        if (rc)
1796                                goto done;
1797
1798                        /* Let the processor re-execute the fixed hypercall */
1799                        c->eip = ctxt->vcpu->arch.rip;
1800                        /* Disable writeback. */
1801                        c->dst.type = OP_NONE;
1802                        break;
1803                case 2: /* lgdt */
1804                        rc = read_descriptor(ctxt, ops, c->src.ptr,
1805                                             &size, &address, c->op_bytes);
1806                        if (rc)
1807                                goto done;
1808                        realmode_lgdt(ctxt->vcpu, size, address);
1809                        /* Disable writeback. */
1810                        c->dst.type = OP_NONE;
1811                        break;
1812                case 3: /* lidt/vmmcall */
1813                        if (c->modrm_mod == 3 && c->modrm_rm == 1) {
1814                                rc = kvm_fix_hypercall(ctxt->vcpu);
1815                                if (rc)
1816                                        goto done;
1817                                kvm_emulate_hypercall(ctxt->vcpu);
1818                        } else {
1819                                rc = read_descriptor(ctxt, ops, c->src.ptr,
1820                                                     &size, &address,
1821                                                     c->op_bytes);
1822                                if (rc)
1823                                        goto done;
1824                                realmode_lidt(ctxt->vcpu, size, address);
1825                        }
1826                        /* Disable writeback. */
1827                        c->dst.type = OP_NONE;
1828                        break;
1829                case 4: /* smsw */
1830                        c->dst.bytes = 2;
1831                        c->dst.val = realmode_get_cr(ctxt->vcpu, 0);
1832                        break;
1833                case 6: /* lmsw */
1834                        realmode_lmsw(ctxt->vcpu, (u16)c->src.val,
1835                                      &ctxt->eflags);
1836                        c->dst.type = OP_NONE;
1837                        break;
1838                case 7: /* invlpg*/
1839                        emulate_invlpg(ctxt->vcpu, memop);
1840                        /* Disable writeback. */
1841                        c->dst.type = OP_NONE;
1842                        break;
1843                default:
1844                        goto cannot_emulate;
1845                }
1846                break;
1847        case 0x06:
1848                emulate_clts(ctxt->vcpu);
1849                c->dst.type = OP_NONE;
1850                break;
1851        case 0x08:              /* invd */
1852        case 0x09:              /* wbinvd */
1853        case 0x0d:              /* GrpP (prefetch) */
1854        case 0x18:              /* Grp16 (prefetch/nop) */
1855                c->dst.type = OP_NONE;
1856                break;
1857        case 0x20: /* mov cr, reg */
1858                if (c->modrm_mod != 3)
1859                        goto cannot_emulate;
1860                c->regs[c->modrm_rm] =
1861                                realmode_get_cr(ctxt->vcpu, c->modrm_reg);
1862                c->dst.type = OP_NONE;  /* no writeback */
1863                break;
1864        case 0x21: /* mov from dr to reg */
1865                if (c->modrm_mod != 3)
1866                        goto cannot_emulate;
1867                rc = emulator_get_dr(ctxt, c->modrm_reg, &c->regs[c->modrm_rm]);
1868                if (rc)
1869                        goto cannot_emulate;
1870                c->dst.type = OP_NONE;  /* no writeback */
1871                break;
1872        case 0x22: /* mov reg, cr */
1873                if (c->modrm_mod != 3)
1874                        goto cannot_emulate;
1875                realmode_set_cr(ctxt->vcpu,
1876                                c->modrm_reg, c->modrm_val, &ctxt->eflags);
1877                c->dst.type = OP_NONE;
1878                break;
1879        case 0x23: /* mov from reg to dr */
1880                if (c->modrm_mod != 3)
1881                        goto cannot_emulate;
1882                rc = emulator_set_dr(ctxt, c->modrm_reg,
1883                                     c->regs[c->modrm_rm]);
1884                if (rc)
1885                        goto cannot_emulate;
1886                c->dst.type = OP_NONE;  /* no writeback */
1887                break;
1888        case 0x30:
1889                /* wrmsr */
1890                msr_data = (u32)c->regs[VCPU_REGS_RAX]
1891                        | ((u64)c->regs[VCPU_REGS_RDX] << 32);
1892                rc = kvm_set_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], msr_data);
1893                if (rc) {
1894                        kvm_inject_gp(ctxt->vcpu, 0);
1895                        c->eip = ctxt->vcpu->arch.rip;
1896                }
1897                rc = X86EMUL_CONTINUE;
1898                c->dst.type = OP_NONE;
1899                break;
1900        case 0x32:
1901                /* rdmsr */
1902                rc = kvm_get_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], &msr_data);
1903                if (rc) {
1904                        kvm_inject_gp(ctxt->vcpu, 0);
1905                        c->eip = ctxt->vcpu->arch.rip;
1906                } else {
1907                        c->regs[VCPU_REGS_RAX] = (u32)msr_data;
1908                        c->regs[VCPU_REGS_RDX] = msr_data >> 32;
1909                }
1910                rc = X86EMUL_CONTINUE;
1911                c->dst.type = OP_NONE;
1912                break;
1913        case 0x40 ... 0x4f:     /* cmov */
1914                c->dst.val = c->dst.orig_val = c->src.val;
1915                if (!test_cc(c->b, ctxt->eflags))
1916                        c->dst.type = OP_NONE; /* no writeback */
1917                break;
1918        case 0x80 ... 0x8f: /* jnz rel, etc*/ {
1919                long int rel;
1920
1921                switch (c->op_bytes) {
1922                case 2:
1923                        rel = insn_fetch(s16, 2, c->eip);
1924                        break;
1925                case 4:
1926                        rel = insn_fetch(s32, 4, c->eip);
1927                        break;
1928                case 8:
1929                        rel = insn_fetch(s64, 8, c->eip);
1930                        break;
1931                default:
1932                        DPRINTF("jnz: Invalid op_bytes\n");
1933                        goto cannot_emulate;
1934                }
1935                if (test_cc(c->b, ctxt->eflags))
1936                        jmp_rel(c, rel);
1937                c->dst.type = OP_NONE;
1938                break;
1939        }
1940        case 0xa3:
1941              bt:               /* bt */
1942                c->dst.type = OP_NONE;
1943                /* only subword offset */
1944                c->src.val &= (c->dst.bytes << 3) - 1;
1945                emulate_2op_SrcV_nobyte("bt", c->src, c->dst, ctxt->eflags);
1946                break;
1947        case 0xab:
1948              bts:              /* bts */
1949                /* only subword offset */
1950                c->src.val &= (c->dst.bytes << 3) - 1;
1951                emulate_2op_SrcV_nobyte("bts", c->src, c->dst, ctxt->eflags);
1952                break;
1953        case 0xae:              /* clflush */
1954                break;
1955        case 0xb0 ... 0xb1:     /* cmpxchg */
1956                /*
1957                 * Save real source value, then compare EAX against
1958                 * destination.
1959                 */
1960                c->src.orig_val = c->src.val;
1961                c->src.val = c->regs[VCPU_REGS_RAX];
1962                emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
1963                if (ctxt->eflags & EFLG_ZF) {
1964                        /* Success: write back to memory. */
1965                        c->dst.val = c->src.orig_val;
1966                } else {
1967                        /* Failure: write the value we saw to EAX. */
1968                        c->dst.type = OP_REG;
1969                        c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX];
1970                }
1971                break;
1972        case 0xb3:
1973              btr:              /* btr */
1974                /* only subword offset */
1975                c->src.val &= (c->dst.bytes << 3) - 1;
1976                emulate_2op_SrcV_nobyte("btr", c->src, c->dst, ctxt->eflags);
1977                break;
1978        case 0xb6 ... 0xb7:     /* movzx */
1979                c->dst.bytes = c->op_bytes;
1980                c->dst.val = (c->d & ByteOp) ? (u8) c->src.val
1981                                                       : (u16) c->src.val;
1982                break;
1983        case 0xba:              /* Grp8 */
1984                switch (c->modrm_reg & 3) {
1985                case 0:
1986                        goto bt;
1987                case 1:
1988                        goto bts;
1989                case 2:
1990                        goto btr;
1991                case 3:
1992                        goto btc;
1993                }
1994                break;
1995        case 0xbb:
1996              btc:              /* btc */
1997                /* only subword offset */
1998                c->src.val &= (c->dst.bytes << 3) - 1;
1999                emulate_2op_SrcV_nobyte("btc", c->src, c->dst, ctxt->eflags);
2000                break;
2001        case 0xbe ... 0xbf:     /* movsx */
2002                c->dst.bytes = c->op_bytes;
2003                c->dst.val = (c->d & ByteOp) ? (s8) c->src.val :
2004                                                        (s16) c->src.val;
2005                break;
2006        case 0xc3:              /* movnti */
2007                c->dst.bytes = c->op_bytes;
2008                c->dst.val = (c->op_bytes == 4) ? (u32) c->src.val :
2009                                                        (u64) c->src.val;
2010                break;
2011        case 0xc7:              /* Grp9 (cmpxchg8b) */
2012                rc = emulate_grp9(ctxt, ops, memop);
2013                if (rc != 0)
2014                        goto done;
2015                c->dst.type = OP_NONE;
2016                break;
2017        }
2018        goto writeback;
2019
2020cannot_emulate:
2021        DPRINTF("Cannot emulate %02x\n", c->b);
2022        c->eip = saved_eip;
2023        return -1;
2024}
2025