// SPDX-License-Identifier: GPL-2.0-only
/******************************************************************************
 * emulate.c
 *
 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
 *
 * Copyright (c) 2005 Keir Fraser
 *
 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
 * privileged instructions:
 *
 * Copyright (C) 2006 Qumranet
 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
 *
 *   Avi Kivity <avi@qumranet.com>
 *   Yaniv Kamay <yaniv@qumranet.com>
 *
 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
 */

#include <linux/kvm_host.h>
#include "kvm_cache_regs.h"
#include "kvm_emulate.h"
#include <linux/stringify.h>
#include <asm/debugreg.h>
#include <asm/nospec-branch.h>

#include "x86.h"
#include "tss.h"
#include "mmu.h"
#include "pmu.h"

/*
 * Operand types
 */
#define OpNone             0ull
#define OpImplicit         1ull  /* No generic decode */
#define OpReg              2ull  /* Register */
#define OpMem              3ull  /* Memory */
#define OpAcc              4ull  /* Accumulator: AL/AX/EAX/RAX */
#define OpDI               5ull  /* ES:DI/EDI/RDI */
#define OpMem64            6ull  /* Memory, 64-bit */
#define OpImmUByte         7ull  /* Zero-extended 8-bit immediate */
#define OpDX               8ull  /* DX register */
#define OpCL               9ull  /* CL register (for shifts) */
#define OpImmByte         10ull  /* 8-bit sign extended immediate */
#define OpOne             11ull  /* Implied 1 */
#define OpImm             12ull  /* Sign extended up to 32-bit immediate */
#define OpMem16           13ull  /* Memory operand (16-bit). */
#define OpMem32           14ull  /* Memory operand (32-bit). */
#define OpImmU            15ull  /* Immediate operand, zero extended */
#define OpSI              16ull  /* SI/ESI/RSI */
#define OpImmFAddr        17ull  /* Immediate far address */
#define OpMemFAddr        18ull  /* Far address in memory */
#define OpImmU16          19ull  /* Immediate operand, 16 bits, zero extended */
#define OpES              20ull  /* ES */
#define OpCS              21ull  /* CS */
#define OpSS              22ull  /* SS */
#define OpDS              23ull  /* DS */
#define OpFS              24ull  /* FS */
#define OpGS              25ull  /* GS */
#define OpMem8            26ull  /* 8-bit zero extended memory operand */
#define OpImm64           27ull  /* Sign extended 16/32/64-bit immediate */
#define OpXLat            28ull  /* memory at BX/EBX/RBX + zero-extended AL */
#define OpAccLo           29ull  /* Low part of extended acc (AX/AX/EAX/RAX) */
#define OpAccHi           30ull  /* High part of extended acc (-/DX/EDX/RDX) */

#define OpBits             5  /* Width of operand field */
#define OpMask             ((1ull << OpBits) - 1)

/*
 * Opcode effective-address decode tables.
 * Note that we only emulate instructions that have at least one memory
 * operand (excluding implicit stack references). We assume that stack
 * references and instruction fetches will never occur in special memory
 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
 * not be handled.
 */

/* Operand sizes: 8-bit operands or specified/overridden size. */
#define ByteOp      (1<<0)      /* 8-bit operands. */
/* Destination operand type. */
#define DstShift    1
#define ImplicitOps (OpImplicit << DstShift)
#define DstReg      (OpReg << DstShift)
#define DstMem      (OpMem << DstShift)
#define DstAcc      (OpAcc << DstShift)
#define DstDI       (OpDI << DstShift)
#define DstMem64    (OpMem64 << DstShift)
#define DstMem16    (OpMem16 << DstShift)
#define DstImmUByte (OpImmUByte << DstShift)
#define DstDX       (OpDX << DstShift)
#define DstAccLo    (OpAccLo << DstShift)
#define DstMask     (OpMask << DstShift)
/* Source operand type. */
#define SrcShift    6
#define SrcNone     (OpNone << SrcShift)
#define SrcReg      (OpReg << SrcShift)
#define SrcMem      (OpMem << SrcShift)
#define SrcMem16    (OpMem16 << SrcShift)
#define SrcMem32    (OpMem32 << SrcShift)
#define SrcImm      (OpImm << SrcShift)
#define SrcImmByte  (OpImmByte << SrcShift)
#define SrcOne      (OpOne << SrcShift)
#define SrcImmUByte (OpImmUByte << SrcShift)
#define SrcImmU     (OpImmU << SrcShift)
#define SrcSI       (OpSI << SrcShift)
#define SrcXLat     (OpXLat << SrcShift)
#define SrcImmFAddr (OpImmFAddr << SrcShift)
#define SrcMemFAddr (OpMemFAddr << SrcShift)
#define SrcAcc      (OpAcc << SrcShift)
#define SrcImmU16   (OpImmU16 << SrcShift)
#define SrcImm64    (OpImm64 << SrcShift)
#define SrcDX       (OpDX << SrcShift)
#define SrcMem8     (OpMem8 << SrcShift)
#define SrcAccHi    (OpAccHi << SrcShift)
#define SrcMask     (OpMask << SrcShift)
#define BitOp       (1<<11)
#define MemAbs      (1<<12)      /* Memory operand is absolute displacement */
#define String      (1<<13)     /* String instruction (rep capable) */
#define Stack       (1<<14)     /* Stack instruction (push/pop) */
#define GroupMask   (7<<15)     /* Opcode uses one of the group mechanisms */
#define Group       (1<<15)     /* Bits 3:5 of modrm byte extend opcode */
#define GroupDual   (2<<15)     /* Alternate decoding of mod == 3 */
#define Prefix      (3<<15)     /* Instruction varies with 66/f2/f3 prefix */
#define RMExt       (4<<15)     /* Opcode extension in ModRM r/m if mod == 3 */
#define Escape      (5<<15)     /* Escape to coprocessor instruction */
#define InstrDual   (6<<15)     /* Alternate instruction decoding of mod == 3 */
#define ModeDual    (7<<15)     /* Different instruction for 32/64 bit */
#define Sse         (1<<18)     /* SSE Vector instruction */
/* Generic ModRM decode. */
#define ModRM       (1<<19)
/* Destination is only written; never read. */
#define Mov         (1<<20)
/* Misc flags */
#define Prot        (1<<21) /* instruction generates #UD if not in prot-mode */
#define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
#define NoAccess    (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
#define Op3264      (1<<24) /* Operand is 64b in long mode, 32b otherwise */
#define Undefined   (1<<25) /* No Such Instruction */
#define Lock        (1<<26) /* lock prefix is allowed for the instruction */
#define Priv        (1<<27) /* instruction generates #GP if current CPL != 0 */
#define No64        (1<<28)
#define PageTable   (1 << 29)   /* instruction used to write page table */
#define NotImpl     (1 << 30)   /* instruction is not implemented */
/* Source 2 operand type */
#define Src2Shift   (31)
#define Src2None    (OpNone << Src2Shift)
#define Src2Mem     (OpMem << Src2Shift)
#define Src2CL      (OpCL << Src2Shift)
#define Src2ImmByte (OpImmByte << Src2Shift)
#define Src2One     (OpOne << Src2Shift)
#define Src2Imm     (OpImm << Src2Shift)
#define Src2ES      (OpES << Src2Shift)
#define Src2CS      (OpCS << Src2Shift)
#define Src2SS      (OpSS << Src2Shift)
#define Src2DS      (OpDS << Src2Shift)
#define Src2FS      (OpFS << Src2Shift)
#define Src2GS      (OpGS << Src2Shift)
#define Src2Mask    (OpMask << Src2Shift)
#define Mmx         ((u64)1 << 40)  /* MMX Vector instruction */
#define AlignMask   ((u64)7 << 41)
#define Aligned     ((u64)1 << 41)  /* Explicitly aligned (e.g. MOVDQA) */
#define Unaligned   ((u64)2 << 41)  /* Explicitly unaligned (e.g. MOVDQU) */
#define Avx         ((u64)3 << 41)  /* Advanced Vector Extensions */
#define Aligned16   ((u64)4 << 41)  /* Aligned to 16 byte boundary (e.g. FXSAVE) */
#define Fastop      ((u64)1 << 44)  /* Use opcode::u.fastop */
#define NoWrite     ((u64)1 << 45)  /* No writeback */
#define SrcWrite    ((u64)1 << 46)  /* Write back src operand */
#define NoMod       ((u64)1 << 47)  /* Mod field is ignored */
#define Intercept   ((u64)1 << 48)  /* Has valid intercept field */
#define CheckPerm   ((u64)1 << 49)  /* Has valid check_perm field */
#define PrivUD      ((u64)1 << 51)  /* #UD instead of #GP on CPL > 0 */
#define NearBranch  ((u64)1 << 52)  /* Near branches */
#define No16        ((u64)1 << 53)  /* No 16 bit operand */
#define IncSP       ((u64)1 << 54)  /* SP is incremented before ModRM calc */
#define TwoMemOp    ((u64)1 << 55)  /* Instruction has two memory operand */

#define DstXacc     (DstAccLo | SrcAccHi | SrcWrite)

#define X2(x...) x, x
#define X3(x...) X2(x), x
#define X4(x...) X2(x), X2(x)
#define X5(x...) X4(x), x
#define X6(x...) X4(x), X2(x)
#define X7(x...) X4(x), X3(x)
#define X8(x...) X4(x), X4(x)
#define X16(x...) X8(x), X8(x)

#define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
#define FASTOP_SIZE 8

struct opcode {
        u64 flags : 56;
        u64 intercept : 8;
        union {
                int (*execute)(struct x86_emulate_ctxt *ctxt);
                const struct opcode *group;
                const struct group_dual *gdual;
                const struct gprefix *gprefix;
                const struct escape *esc;
                const struct instr_dual *idual;
                const struct mode_dual *mdual;
                void (*fastop)(struct fastop *fake);
        } u;
        int (*check_perm)(struct x86_emulate_ctxt *ctxt);
};

struct group_dual {
        struct opcode mod012[8];
        struct opcode mod3[8];
};

struct gprefix {
        struct opcode pfx_no;
        struct opcode pfx_66;
        struct opcode pfx_f2;
        struct opcode pfx_f3;
};

struct escape {
        struct opcode op[8];
        struct opcode high[64];
};

struct instr_dual {
        struct opcode mod012;
        struct opcode mod3;
};

struct mode_dual {
        struct opcode mode32;
        struct opcode mode64;
};

#define EFLG_RESERVED_ZEROS_MASK 0xffc0802a

enum x86_transfer_type {
        X86_TRANSFER_NONE,
        X86_TRANSFER_CALL_JMP,
        X86_TRANSFER_RET,
        X86_TRANSFER_TASK_SWITCH,
};

static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
{
        if (!(ctxt->regs_valid & (1 << nr))) {
                ctxt->regs_valid |= 1 << nr;
                ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
        }
        return ctxt->_regs[nr];
}

static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
{
        ctxt->regs_valid |= 1 << nr;
        ctxt->regs_dirty |= 1 << nr;
        return &ctxt->_regs[nr];
}

static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
{
        reg_read(ctxt, nr);
        return reg_write(ctxt, nr);
}

static void writeback_registers(struct x86_emulate_ctxt *ctxt)
{
        unsigned reg;

        for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
                ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
}

static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
{
        ctxt->regs_dirty = 0;
        ctxt->regs_valid = 0;
}

/*
 * These EFLAGS bits are restored from saved value during emulation, and
 * any changes are written back to the saved value after emulation.
 */
#define EFLAGS_MASK (X86_EFLAGS_OF|X86_EFLAGS_SF|X86_EFLAGS_ZF|X86_EFLAGS_AF|\
                     X86_EFLAGS_PF|X86_EFLAGS_CF)

#ifdef CONFIG_X86_64
#define ON64(x) x
#else
#define ON64(x)
#endif

/*
 * fastop functions have a special calling convention:
 *
 * dst:    rax        (in/out)
 * src:    rdx        (in/out)
 * src2:   rcx        (in)
 * flags:  rflags     (in/out)
 * ex:     rsi        (in:fastop pointer, out:zero if exception)
 *
 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
 * different operand sizes can be reached by calculation, rather than a jump
 * table (which would be bigger than the code).
 */
static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop);

#define __FOP_FUNC(name) \
        ".align " __stringify(FASTOP_SIZE) " \n\t" \
        ".type " name ", @function \n\t" \
        name ":\n\t"

#define FOP_FUNC(name) \
        __FOP_FUNC(#name)

#define __FOP_RET(name) \
        "ret \n\t" \
        ".size " name ", .-" name "\n\t"

#define FOP_RET(name) \
        __FOP_RET(#name)

#define FOP_START(op) \
        extern void em_##op(struct fastop *fake); \
        asm(".pushsection .text, \"ax\" \n\t" \
            ".global em_" #op " \n\t" \
            ".align " __stringify(FASTOP_SIZE) " \n\t" \
            "em_" #op ":\n\t"

#define FOP_END \
            ".popsection")

#define __FOPNOP(name) \
        __FOP_FUNC(name) \
        __FOP_RET(name)

#define FOPNOP() \
        __FOPNOP(__stringify(__UNIQUE_ID(nop)))

#define FOP1E(op,  dst) \
        __FOP_FUNC(#op "_" #dst) \
        "10: " #op " %" #dst " \n\t" \
        __FOP_RET(#op "_" #dst)

#define FOP1EEX(op,  dst) \
        FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)

#define FASTOP1(op) \
        FOP_START(op) \
        FOP1E(op##b, al) \
        FOP1E(op##w, ax) \
        FOP1E(op##l, eax) \
        ON64(FOP1E(op##q, rax)) \
        FOP_END

/* 1-operand, using src2 (for MUL/DIV r/m) */
#define FASTOP1SRC2(op, name) \
        FOP_START(name) \
        FOP1E(op, cl) \
        FOP1E(op, cx) \
        FOP1E(op, ecx) \
        ON64(FOP1E(op, rcx)) \
        FOP_END

/* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
#define FASTOP1SRC2EX(op, name) \
        FOP_START(name) \
        FOP1EEX(op, cl) \
        FOP1EEX(op, cx) \
        FOP1EEX(op, ecx) \
        ON64(FOP1EEX(op, rcx)) \
        FOP_END

#define FOP2E(op,  dst, src)       \
        __FOP_FUNC(#op "_" #dst "_" #src) \
        #op " %" #src ", %" #dst " \n\t" \
        __FOP_RET(#op "_" #dst "_" #src)

#define FASTOP2(op) \
        FOP_START(op) \
        FOP2E(op##b, al, dl) \
        FOP2E(op##w, ax, dx) \
        FOP2E(op##l, eax, edx) \
        ON64(FOP2E(op##q, rax, rdx)) \
        FOP_END

/* 2 operand, word only */
#define FASTOP2W(op) \
        FOP_START(op) \
        FOPNOP() \
        FOP2E(op##w, ax, dx) \
        FOP2E(op##l, eax, edx) \
        ON64(FOP2E(op##q, rax, rdx)) \
        FOP_END

/* 2 operand, src is CL */
#define FASTOP2CL(op) \
        FOP_START(op) \
        FOP2E(op##b, al, cl) \
        FOP2E(op##w, ax, cl) \
        FOP2E(op##l, eax, cl) \
        ON64(FOP2E(op##q, rax, cl)) \
        FOP_END

/* 2 operand, src and dest are reversed */
#define FASTOP2R(op, name) \
        FOP_START(name) \
        FOP2E(op##b, dl, al) \
        FOP2E(op##w, dx, ax) \
        FOP2E(op##l, edx, eax) \
        ON64(FOP2E(op##q, rdx, rax)) \
        FOP_END

#define FOP3E(op,  dst, src, src2) \
        __FOP_FUNC(#op "_" #dst "_" #src "_" #src2) \
        #op " %" #src2 ", %" #src ", %" #dst " \n\t"\
        __FOP_RET(#op "_" #dst "_" #src "_" #src2)

/* 3-operand, word-only, src2=cl */
#define FASTOP3WCL(op) \
        FOP_START(op) \
        FOPNOP() \
        FOP3E(op##w, ax, dx, cl) \
        FOP3E(op##l, eax, edx, cl) \
        ON64(FOP3E(op##q, rax, rdx, cl)) \
        FOP_END

/* Special case for SETcc - 1 instruction per cc */
#define FOP_SETCC(op) \
        ".align 4 \n\t" \
        ".type " #op ", @function \n\t" \
        #op ": \n\t" \
        #op " %al \n\t" \
        __FOP_RET(#op)

asm(".pushsection .fixup, \"ax\"\n"
    ".global kvm_fastop_exception \n"
    "kvm_fastop_exception: xor %esi, %esi; ret\n"
    ".popsection");

FOP_START(setcc)
FOP_SETCC(seto)
FOP_SETCC(setno)
FOP_SETCC(setc)
FOP_SETCC(setnc)
FOP_SETCC(setz)
FOP_SETCC(setnz)
FOP_SETCC(setbe)
FOP_SETCC(setnbe)
FOP_SETCC(sets)
FOP_SETCC(setns)
FOP_SETCC(setp)
FOP_SETCC(setnp)
FOP_SETCC(setl)
FOP_SETCC(setnl)
FOP_SETCC(setle)
FOP_SETCC(setnle)
FOP_END;

FOP_START(salc)
FOP_FUNC(salc)
"pushf; sbb %al, %al; popf \n\t"
FOP_RET(salc)
FOP_END;

/*
 * XXX: inoutclob user must know where the argument is being expanded.
 *      Relying on CONFIG_CC_HAS_ASM_GOTO would allow us to remove _fault.
 */
#define asm_safe(insn, inoutclob...) \
({ \
        int _fault = 0; \
 \
        asm volatile("1:" insn "\n" \
                     "2:\n" \
                     ".pushsection .fixup, \"ax\"\n" \
                     "3: movl $1, %[_fault]\n" \
                     "   jmp  2b\n" \
                     ".popsection\n" \
                     _ASM_EXTABLE(1b, 3b) \
                     : [_fault] "+qm"(_fault) inoutclob ); \
 \
        _fault ? X86EMUL_UNHANDLEABLE : X86EMUL_CONTINUE; \
})

static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
                                    enum x86_intercept intercept,
                                    enum x86_intercept_stage stage)
{
        struct x86_instruction_info info = {
                .intercept  = intercept,
                .rep_prefix = ctxt->rep_prefix,
                .modrm_mod  = ctxt->modrm_mod,
                .modrm_reg  = ctxt->modrm_reg,
                .modrm_rm   = ctxt->modrm_rm,
                .src_val    = ctxt->src.val64,
                .dst_val    = ctxt->dst.val64,
                .src_bytes  = ctxt->src.bytes,
                .dst_bytes  = ctxt->dst.bytes,
                .ad_bytes   = ctxt->ad_bytes,
                .next_rip   = ctxt->eip,
        };

        return ctxt->ops->intercept(ctxt, &info, stage);
}

static void assign_masked(ulong *dest, ulong src, ulong mask)
{
        *dest = (*dest & ~mask) | (src & mask);
}

static void assign_register(unsigned long *reg, u64 val, int bytes)
{
        /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
        switch (bytes) {
        case 1:
                *(u8 *)reg = (u8)val;
                break;
        case 2:
                *(u16 *)reg = (u16)val;
                break;
        case 4:
                *reg = (u32)val;
                break;  /* 64b: zero-extend */
        case 8:
                *reg = val;
                break;
        }
}

static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
{
        return (1UL << (ctxt->ad_bytes << 3)) - 1;
}

static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
{
        u16 sel;
        struct desc_struct ss;

        if (ctxt->mode == X86EMUL_MODE_PROT64)
                return ~0UL;
        ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
        return ~0U >> ((ss.d ^ 1) * 16);  /* d=0: 0xffff; d=1: 0xffffffff */
}

static int stack_size(struct x86_emulate_ctxt *ctxt)
{
        return (__fls(stack_mask(ctxt)) + 1) >> 3;
}

/* Access/update address held in a register, based on addressing mode. */
static inline unsigned long
address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
{
        if (ctxt->ad_bytes == sizeof(unsigned long))
                return reg;
        else
                return reg & ad_mask(ctxt);
}

static inline unsigned long
register_address(struct x86_emulate_ctxt *ctxt, int reg)
{
        return address_mask(ctxt, reg_read(ctxt, reg));
}

static void masked_increment(ulong *reg, ulong mask, int inc)
{
        assign_masked(reg, *reg + inc, mask);
}

static inline void
register_address_increment(struct x86_emulate_ctxt *ctxt, int reg, int inc)
{
        ulong *preg = reg_rmw(ctxt, reg);

        assign_register(preg, *preg + inc, ctxt->ad_bytes);
}

static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
{
        masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
}

static u32 desc_limit_scaled(struct desc_struct *desc)
{
        u32 limit = get_desc_limit(desc);

        return desc->g ? (limit << 12) | 0xfff : limit;
}

static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
{
        if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
                return 0;

        return ctxt->ops->get_cached_segment_base(ctxt, seg);
}

static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
                             u32 error, bool valid)
{
        WARN_ON(vec > 0x1f);
        ctxt->exception.vector = vec;
        ctxt->exception.error_code = error;
        ctxt->exception.error_code_valid = valid;
        return X86EMUL_PROPAGATE_FAULT;
}

static int emulate_db(struct x86_emulate_ctxt *ctxt)
{
        return emulate_exception(ctxt, DB_VECTOR, 0, false);
}

static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
{
        return emulate_exception(ctxt, GP_VECTOR, err, true);
}

static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
{
        return emulate_exception(ctxt, SS_VECTOR, err, true);
}

static int emulate_ud(struct x86_emulate_ctxt *ctxt)
{
        return emulate_exception(ctxt, UD_VECTOR, 0, false);
}

static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
{
        return emulate_exception(ctxt, TS_VECTOR, err, true);
}

static int emulate_de(struct x86_emulate_ctxt *ctxt)
{
        return emulate_exception(ctxt, DE_VECTOR, 0, false);
}

static int emulate_nm(struct x86_emulate_ctxt *ctxt)
{
        return emulate_exception(ctxt, NM_VECTOR, 0, false);
}

static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
{
        u16 selector;
        struct desc_struct desc;

        ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
        return selector;
}

static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
                                 unsigned seg)
{
        u16 dummy;
        u32 base3;
        struct desc_struct desc;

        ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
        ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
}

static inline u8 ctxt_virt_addr_bits(struct x86_emulate_ctxt *ctxt)
{
        return (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_LA57) ? 57 : 48;
}

static inline bool emul_is_noncanonical_address(u64 la,
                                                struct x86_emulate_ctxt *ctxt)
{
        return get_canonical(la, ctxt_virt_addr_bits(ctxt)) != la;
}

/*
 * x86 defines three classes of vector instructions: explicitly
 * aligned, explicitly unaligned, and the rest, which change behaviour
 * depending on whether they're AVX encoded or not.
 *
 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
 * subject to the same check.  FXSAVE and FXRSTOR are checked here too as their
 * 512 bytes of data must be aligned to a 16 byte boundary.
 */
static unsigned insn_alignment(struct x86_emulate_ctxt *ctxt, unsigned size)
{
        u64 alignment = ctxt->d & AlignMask;

        if (likely(size < 16))
                return 1;

        switch (alignment) {
        case Unaligned:
        case Avx:
                return 1;
        case Aligned16:
                return 16;
        case Aligned:
        default:
                return size;
        }
}

static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt,
                                       struct segmented_address addr,
                                       unsigned *max_size, unsigned size,
                                       bool write, bool fetch,
                                       enum x86emul_mode mode, ulong *linear)
{
        struct desc_struct desc;
        bool usable;
        ulong la;
        u32 lim;
        u16 sel;
        u8  va_bits;

        la = seg_base(ctxt, addr.seg) + addr.ea;
        *max_size = 0;
        switch (mode) {
        case X86EMUL_MODE_PROT64:
                *linear = la;
                va_bits = ctxt_virt_addr_bits(ctxt);
                if (get_canonical(la, va_bits) != la)
                        goto bad;

                *max_size = min_t(u64, ~0u, (1ull << va_bits) - la);
                if (size > *max_size)
                        goto bad;
                break;
        default:
                *linear = la = (u32)la;
                usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
                                                addr.seg);
                if (!usable)
                        goto bad;
                /* code segment in protected mode or read-only data segment */
                if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
                                        || !(desc.type & 2)) && write)
                        goto bad;
                /* unreadable code segment */
                if (!fetch && (desc.type & 8) && !(desc.type & 2))
                        goto bad;
                lim = desc_limit_scaled(&desc);
                if (!(desc.type & 8) && (desc.type & 4)) {
                        /* expand-down segment */
                        if (addr.ea <= lim)
                                goto bad;
                        lim = desc.d ? 0xffffffff : 0xffff;
                }
                if (addr.ea > lim)
                        goto bad;
                if (lim == 0xffffffff)
                        *max_size = ~0u;
                else {
                        *max_size = (u64)lim + 1 - addr.ea;
                        if (size > *max_size)
                                goto bad;
                }
                break;
        }
        if (la & (insn_alignment(ctxt, size) - 1))
                return emulate_gp(ctxt, 0);
        return X86EMUL_CONTINUE;
bad:
        if (addr.seg == VCPU_SREG_SS)
                return emulate_ss(ctxt, 0);
        else
                return emulate_gp(ctxt, 0);
}

static int linearize(struct x86_emulate_ctxt *ctxt,
                     struct segmented_address addr,
                     unsigned size, bool write,
                     ulong *linear)
{
        unsigned max_size;
        return __linearize(ctxt, addr, &max_size, size, write, false,
                           ctxt->mode, linear);
}

static inline int assign_eip(struct x86_emulate_ctxt *ctxt, ulong dst,
                             enum x86emul_mode mode)
{
        ulong linear;
        int rc;
        unsigned max_size;
        struct segmented_address addr = { .seg = VCPU_SREG_CS,
                                           .ea = dst };

        if (ctxt->op_bytes != sizeof(unsigned long))
                addr.ea = dst & ((1UL << (ctxt->op_bytes << 3)) - 1);
        rc = __linearize(ctxt, addr, &max_size, 1, false, true, mode, &linear);
        if (rc == X86EMUL_CONTINUE)
                ctxt->_eip = addr.ea;
        return rc;
}

static inline int assign_eip_near(struct x86_emulate_ctxt *ctxt, ulong dst)
{
        return assign_eip(ctxt, dst, ctxt->mode);
}

static int assign_eip_far(struct x86_emulate_ctxt *ctxt, ulong dst,
                          const struct desc_struct *cs_desc)
{
        enum x86emul_mode mode = ctxt->mode;
        int rc;

#ifdef CONFIG_X86_64
        if (ctxt->mode >= X86EMUL_MODE_PROT16) {
                if (cs_desc->l) {
                        u64 efer = 0;

                        ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
                        if (efer & EFER_LMA)
                                mode = X86EMUL_MODE_PROT64;
                } else
                        mode = X86EMUL_MODE_PROT32; /* temporary value */
        }
#endif
        if (mode == X86EMUL_MODE_PROT16 || mode == X86EMUL_MODE_PROT32)
                mode = cs_desc->d ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
        rc = assign_eip(ctxt, dst, mode);
        if (rc == X86EMUL_CONTINUE)
                ctxt->mode = mode;
        return rc;
}

static inline int jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
{
        return assign_eip_near(ctxt, ctxt->_eip + rel);
}

static int linear_read_system(struct x86_emulate_ctxt *ctxt, ulong linear,
                              void *data, unsigned size)
{
        return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, true);
}

static int linear_write_system(struct x86_emulate_ctxt *ctxt,
                               ulong linear, void *data,
                               unsigned int size)
{
        return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, true);
}

static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
                              struct segmented_address addr,
                              void *data,
                              unsigned size)
{
        int rc;
        ulong linear;

        rc = linearize(ctxt, addr, size, false, &linear);
        if (rc != X86EMUL_CONTINUE)
                return rc;
        return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, false);
}

static int segmented_write_std(struct x86_emulate_ctxt *ctxt,
                               struct segmented_address addr,
                               void *data,
                               unsigned int size)
{
        int rc;
        ulong linear;

        rc = linearize(ctxt, addr, size, true, &linear);
        if (rc != X86EMUL_CONTINUE)
                return rc;
        return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, false);
}

/*
 * Prefetch the remaining bytes of the instruction without crossing page
 * boundary if they are not in fetch_cache yet.
 */
static int __do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt, int op_size)
{
        int rc;
        unsigned size, max_size;
        unsigned long linear;
        int cur_size = ctxt->fetch.end - ctxt->fetch.data;
        struct segmented_address addr = { .seg = VCPU_SREG_CS,
                                           .ea = ctxt->eip + cur_size };

        /*
         * We do not know exactly how many bytes will be needed, and
         * __linearize is expensive, so fetch as much as possible.  We
         * just have to avoid going beyond the 15 byte limit, the end
         * of the segment, or the end of the page.
         *
         * __linearize is called with size 0 so that it does not do any
         * boundary check itself.  Instead, we use max_size to check
         * against op_size.
         */
        rc = __linearize(ctxt, addr, &max_size, 0, false, true, ctxt->mode,
                         &linear);
        if (unlikely(rc != X86EMUL_CONTINUE))
                return rc;

        size = min_t(unsigned, 15UL ^ cur_size, max_size);
        size = min_t(unsigned, size, PAGE_SIZE - offset_in_page(linear));

        /*
         * One instruction can only straddle two pages,
         * and one has been loaded at the beginning of
         * x86_decode_insn.  So, if not enough bytes
         * still, we must have hit the 15-byte boundary.
         */
        if (unlikely(size < op_size))
                return emulate_gp(ctxt, 0);

        rc = ctxt->ops->fetch(ctxt, linear, ctxt->fetch.end,
                              size, &ctxt->exception);
        if (unlikely(rc != X86EMUL_CONTINUE))
                return rc;
        ctxt->fetch.end += size;
        return X86EMUL_CONTINUE;
}

static __always_inline int do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt,
                                               unsigned size)
{
        unsigned done_size = ctxt->fetch.end - ctxt->fetch.ptr;

        if (unlikely(done_size < size))
                return __do_insn_fetch_bytes(ctxt, size - done_size);
        else
                return X86EMUL_CONTINUE;
}

/* Fetch next part of the instruction being emulated. */
#define insn_fetch(_type, _ctxt)                                        \
({      _type _x;                                                       \
                                                                        \
        rc = do_insn_fetch_bytes(_ctxt, sizeof(_type));                 \
        if (rc != X86EMUL_CONTINUE)                                     \
                goto done;                                              \
        ctxt->_eip += sizeof(_type);                                    \
        memcpy(&_x, ctxt->fetch.ptr, sizeof(_type));                    \
        ctxt->fetch.ptr += sizeof(_type);                               \
        _x;                                                             \
})

#define insn_fetch_arr(_arr, _size, _ctxt)                              \
({                                                                      \
        rc = do_insn_fetch_bytes(_ctxt, _size);                         \
        if (rc != X86EMUL_CONTINUE)                                     \
                goto done;                                              \
        ctxt->_eip += (_size);                                          \
        memcpy(_arr, ctxt->fetch.ptr, _size);                           \
        ctxt->fetch.ptr += (_size);                                     \
})

/*
 * Given the 'reg' portion of a ModRM byte, and a register block, return a
 * pointer into the block that addresses the relevant register.
 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
 */
static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
                             int byteop)
{
        void *p;
        int highbyte_regs = (ctxt->rex_prefix == 0) && byteop;

        if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
                p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
        else
                p = reg_rmw(ctxt, modrm_reg);
        return p;
}

static int read_descriptor(struct x86_emulate_ctxt *ctxt,
                           struct segmented_address addr,
                           u16 *size, unsigned long *address, int op_bytes)
{
        int rc;

        if (op_bytes == 2)
                op_bytes = 3;
        *address = 0;
        rc = segmented_read_std(ctxt, addr, size, 2);
        if (rc != X86EMUL_CONTINUE)
                return rc;
        addr.ea += 2;
        rc = segmented_read_std(ctxt, addr, address, op_bytes);
        return rc;
}

FASTOP2(add);
FASTOP2(or);
FASTOP2(adc);
FASTOP2(sbb);
FASTOP2(and);
FASTOP2(sub);
FASTOP2(xor);

FASTOP2(cmp);
FASTOP2(test);

FASTOP1SRC2(mul, mul_ex);
FASTOP1SRC2(imul, imul_ex);
FASTOP1SRC2EX(div, div_ex);
FASTOP1SRC2EX(idiv, idiv_ex);

FASTOP3WCL(shld);
FASTOP3WCL(shrd);

FASTOP2W(imul);

FASTOP1(not);
FASTOP1(neg);
FASTOP1(inc);
FASTOP1(dec);

FASTOP2CL(rol);
FASTOP2CL(ror);
FASTOP2CL(rcl);
FASTOP2CL(rcr);
FASTOP2CL(shl);
FASTOP2CL(shr);
FASTOP2CL(sar);

FASTOP2W(bsf);
FASTOP2W(bsr);
FASTOP2W(bt);
FASTOP2W(bts);
FASTOP2W(btr);
FASTOP2W(btc);

FASTOP2(xadd);

FASTOP2R(cmp, cmp_r);

static int em_bsf_c(struct x86_emulate_ctxt *ctxt)
{
        /* If src is zero, do not writeback, but update flags */
        if (ctxt->src.val == 0)
                ctxt->dst.type = OP_NONE;
        return fastop(ctxt, em_bsf);
}

static int em_bsr_c(struct x86_emulate_ctxt *ctxt)
{
        /* If src is zero, do not writeback, but update flags */
        if (ctxt->src.val == 0)
                ctxt->dst.type = OP_NONE;
        return fastop(ctxt, em_bsr);
}

static __always_inline u8 test_cc(unsigned int condition, unsigned long flags)
{
        u8 rc;
        void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);

        flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
        asm("push %[flags]; popf; " CALL_NOSPEC
            : "=a"(rc) : [thunk_target]"r"(fop), [flags]"r"(flags));
        return rc;
}

static void fetch_register_operand(struct operand *op)
{
        switch (op->bytes) {
        case 1:
                op->val = *(u8 *)op->addr.reg;
                break;
        case 2:
                op->val = *(u16 *)op->addr.reg;
                break;
        case 4:
                op->val = *(u32 *)op->addr.reg;
                break;
        case 8:
                op->val = *(u64 *)op->addr.reg;
                break;
        }
}

static int em_fninit(struct x86_emulate_ctxt *ctxt)
{
        if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
                return emulate_nm(ctxt);

        kvm_fpu_get();
        asm volatile("fninit");
        kvm_fpu_put();
        return X86EMUL_CONTINUE;
}

static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
{
        u16 fcw;

        if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
                return emulate_nm(ctxt);

        kvm_fpu_get();
        asm volatile("fnstcw %0": "+m"(fcw));
        kvm_fpu_put();

        ctxt->dst.val = fcw;

        return X86EMUL_CONTINUE;
}

static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
{
        u16 fsw;

        if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
                return emulate_nm(ctxt);

        kvm_fpu_get();
        asm volatile("fnstsw %0": "+m"(fsw));
        kvm_fpu_put();

        ctxt->dst.val = fsw;

        return X86EMUL_CONTINUE;
}

static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
                                    struct operand *op)
{
        unsigned reg = ctxt->modrm_reg;

        if (!(ctxt->d & ModRM))
                reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);

        if (ctxt->d & Sse) {
                op->type = OP_XMM;
                op->bytes = 16;
                op->addr.xmm = reg;
                kvm_read_sse_reg(reg, &op->vec_val);
                return;
        }
        if (ctxt->d & Mmx) {
                reg &= 7;
                op->type = OP_MM;
                op->bytes = 8;
                op->addr.mm = reg;
                return;
        }

        op->type = OP_REG;
        op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
        op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);

        fetch_register_operand(op);
        op->orig_val = op->val;
}

static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
{
        if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
                ctxt->modrm_seg = VCPU_SREG_SS;
}

static int decode_modrm(struct x86_emulate_ctxt *ctxt,
                        struct operand *op)
{
        u8 sib;
        int index_reg, base_reg, scale;
        int rc = X86EMUL_CONTINUE;
        ulong modrm_ea = 0;

        ctxt->modrm_reg = ((ctxt->rex_prefix << 1) & 8); /* REX.R */
        index_reg = (ctxt->rex_prefix << 2) & 8; /* REX.X */
        base_reg = (ctxt->rex_prefix << 3) & 8; /* REX.B */

        ctxt->modrm_mod = (ctxt->modrm & 0xc0) >> 6;
        ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
        ctxt->modrm_rm = base_reg | (ctxt->modrm & 0x07);
        ctxt->modrm_seg = VCPU_SREG_DS;

        if (ctxt->modrm_mod == 3 || (ctxt->d & NoMod)) {
                op->type = OP_REG;
                op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
                op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
                                ctxt->d & ByteOp);
                if (ctxt->d & Sse) {
                        op->type = OP_XMM;
                        op->bytes = 16;
                        op->addr.xmm = ctxt->modrm_rm;
                        kvm_read_sse_reg(ctxt->modrm_rm, &op->vec_val);
                        return rc;
                }
                if (ctxt->d & Mmx) {
                        op->type = OP_MM;
                        op->bytes = 8;
                        op->addr.mm = ctxt->modrm_rm & 7;
                        return rc;
                }
                fetch_register_operand(op);
                return rc;
        }

        op->type = OP_MEM;

        if (ctxt->ad_bytes == 2) {
                unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
                unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
                unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
                unsigned di = reg_read(ctxt, VCPU_REGS_RDI);

                /* 16-bit ModR/M decode. */
                switch (ctxt->modrm_mod) {
                case 0:
                        if (ctxt->modrm_rm == 6)
                                modrm_ea += insn_fetch(u16, ctxt);
                        break;
                case 1:
                        modrm_ea += insn_fetch(s8, ctxt);
                        break;
                case 2:
                        modrm_ea += insn_fetch(u16, ctxt);
                        break;
                }
                switch (ctxt->modrm_rm) {
                case 0:
                        modrm_ea += bx + si;
                        break;
                case 1:
                        modrm_ea += bx + di;
                        break;
                case 2:
                        modrm_ea += bp + si;
                        break;
                case 3:
                        modrm_ea += bp + di;
                        break;
                case 4:
                        modrm_ea += si;
                        break;
                case 5:
                        modrm_ea += di;
                        break;
                case 6:
                        if (ctxt->modrm_mod != 0)
                                modrm_ea += bp;
                        break;
                case 7:
                        modrm_ea += bx;
                        break;
                }
                if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
                    (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
                        ctxt->modrm_seg = VCPU_SREG_SS;
                modrm_ea = (u16)modrm_ea;
        } else {
                /* 32/64-bit ModR/M decode. */
                if ((ctxt->modrm_rm & 7) == 4) {
                        sib = insn_fetch(u8, ctxt);
                        index_reg |= (sib >> 3) & 7;
                        base_reg |= sib & 7;
                        scale = sib >> 6;

                        if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
                                modrm_ea += insn_fetch(s32, ctxt);
                        else {
                                modrm_ea += reg_read(ctxt, base_reg);
                                adjust_modrm_seg(ctxt, base_reg);
                                /* Increment ESP on POP [ESP] */
                                if ((ctxt->d & IncSP) &&
                                    base_reg == VCPU_REGS_RSP)
                                        modrm_ea += ctxt->op_bytes;
                        }
                        if (index_reg != 4)
                                modrm_ea += reg_read(ctxt, index_reg) << scale;
                } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
                        modrm_ea += insn_fetch(s32, ctxt);
                        if (ctxt->mode == X86EMUL_MODE_PROT64)
                                ctxt->rip_relative = 1;
                } else {
                        base_reg = ctxt->modrm_rm;
                        modrm_ea += reg_read(ctxt, base_reg);
                        adjust_modrm_seg(ctxt, base_reg);
                }
                switch (ctxt->modrm_mod) {
                case 1:
                        modrm_ea += insn_fetch(s8, ctxt);
                        break;
                case 2:
                        modrm_ea += insn_fetch(s32, ctxt);
                        break;
                }
        }
        op->addr.mem.ea = modrm_ea;
        if (ctxt->ad_bytes != 8)
                ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;

done:
        return rc;
}

static int decode_abs(struct x86_emulate_ctxt *ctxt,
                      struct operand *op)
{
        int rc = X86EMUL_CONTINUE;

        op->type = OP_MEM;
        switch (ctxt->ad_bytes) {
        case 2:
                op->addr.mem.ea = insn_fetch(u16, ctxt);
                break;
        case 4:
                op->addr.mem.ea = insn_fetch(u32, ctxt);
                break;
        case 8:
                op->addr.mem.ea = insn_fetch(u64, ctxt);
                break;
        }
done:
        return rc;
}

static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
{
        long sv = 0, mask;

        if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
                mask = ~((long)ctxt->dst.bytes * 8 - 1);

                if (ctxt->src.bytes == 2)
                        sv = (s16)ctxt->src.val & (s16)mask;
                else if (ctxt->src.bytes == 4)
                        sv = (s32)ctxt->src.val & (s32)mask;
                else
                        sv = (s64)ctxt->src.val & (s64)mask;

                ctxt->dst.addr.mem.ea = address_mask(ctxt,
                                           ctxt->dst.addr.mem.ea + (sv >> 3));
        }

        /* only subword offset */
        ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
}

static int read_emulated(struct x86_emulate_ctxt *ctxt,
                         unsigned long addr, void *dest, unsigned size)
{
        int rc;
        struct read_cache *mc = &ctxt->mem_read;

        if (mc->pos < mc->end)
                goto read_cached;

        WARN_ON((mc->end + size) >= sizeof(mc->data));

        rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
                                      &ctxt->exception);
        if (rc != X86EMUL_CONTINUE)
                return rc;

        mc->end += size;

read_cached:
        memcpy(dest, mc->data + mc->pos, size);
        mc->pos += size;
        return X86EMUL_CONTINUE;
}

static int segmented_read(struct x86_emulate_ctxt *ctxt,
                          struct segmented_address addr,
                          void *data,
                          unsigned size)
{
        int rc;
        ulong linear;

        rc = linearize(ctxt, addr, size, false, &linear);
        if (rc != X86EMUL_CONTINUE)
                return rc;
        return read_emulated(ctxt, linear, data, size);
}

static int segmented_write(struct x86_emulate_ctxt *ctxt,
                           struct segmented_address addr,
                           const void *data,
                           unsigned size)
{
        int rc;
        ulong linear;

        rc = linearize(ctxt, addr, size, true, &linear);
        if (rc != X86EMUL_CONTINUE)
                return rc;
        return ctxt->ops->write_emulated(ctxt, linear, data, size,
                                         &ctxt->exception);
}

static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
                             struct segmented_address addr,
                             const void *orig_data, const void *data,
                             unsigned size)
{
        int rc;
        ulong linear;

        rc = linearize(ctxt, addr, size, true, &linear);
        if (rc != X86EMUL_CONTINUE)
                return rc;
        return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
                                           size, &ctxt->exception);
}

static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
                           unsigned int size, unsigned short port,
                           void *dest)
{
        struct read_cache *rc = &ctxt->io_read;

        if (rc->pos == rc->end) { /* refill pio read ahead */
                unsigned int in_page, n;
                unsigned int count = ctxt->rep_prefix ?
                        address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
                in_page = (ctxt->eflags & X86_EFLAGS_DF) ?
                        offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
                        PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
                n = min3(in_page, (unsigned int)sizeof(rc->data) / size, count);
                if (n == 0)
                        n = 1;
                rc->pos = rc->end = 0;
                if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
                        return 0;
                rc->end = n * size;
        }

        if (ctxt->rep_prefix && (ctxt->d & String) &&
            !(ctxt->eflags & X86_EFLAGS_DF)) {
                ctxt->dst.data = rc->data + rc->pos;
                ctxt->dst.type = OP_MEM_STR;
                ctxt->dst.count = (rc->end - rc->pos) / size;
                rc->pos = rc->end;
        } else {
                memcpy(dest, rc->data + rc->pos, size);
                rc->pos += size;
        }
        return 1;
}

static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
                                     u16 index, struct desc_struct *desc)
{
        struct desc_ptr dt;
        ulong addr;

        ctxt->ops->get_idt(ctxt, &dt);

        if (dt.size < index * 8 + 7)
                return emulate_gp(ctxt, index << 3 | 0x2);

        addr = dt.address + index * 8;
        return linear_read_system(ctxt, addr, desc, sizeof(*desc));
}

static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
                                     u16 selector, struct desc_ptr *dt)
{
        const struct x86_emulate_ops *ops = ctxt->ops;
        u32 base3 = 0;

        if (selector & 1 << 2) {
                struct desc_struct desc;
                u16 sel;

                memset(dt, 0, sizeof(*dt));
                if (!ops->get_segment(ctxt, &sel, &desc, &base3,
                                      VCPU_SREG_LDTR))
                        return;

                dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
                dt->address = get_desc_base(&desc) | ((u64)base3 << 32);
        } else
                ops->get_gdt(ctxt, dt);
}

static int get_descriptor_ptr(struct x86_emulate_ctxt *ctxt,
                              u16 selector, ulong *desc_addr_p)
{
        struct desc_ptr dt;
        u16 index = selector >> 3;
        ulong addr;

        get_descriptor_table_ptr(ctxt, selector, &dt);

        if (dt.size < index * 8 + 7)
                return emulate_gp(ctxt, selector & 0xfffc);

        addr = dt.address + index * 8;

#ifdef CONFIG_X86_64
        if (addr >> 32 != 0) {
                u64 efer = 0;

                ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
                if (!(efer & EFER_LMA))
                        addr &= (u32)-1;
        }
#endif

        *desc_addr_p = addr;
        return X86EMUL_CONTINUE;
}

/* allowed just for 8 bytes segments */
static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
                                   u16 selector, struct desc_struct *desc,
                                   ulong *desc_addr_p)
{
        int rc;

        rc = get_descriptor_ptr(ctxt, selector, desc_addr_p);
        if (rc != X86EMUL_CONTINUE)
                return rc;

        return linear_read_system(ctxt, *desc_addr_p, desc, sizeof(*desc));
}

/* allowed just for 8 bytes segments */
static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
                                    u16 selector, struct desc_struct *desc)
{
        int rc;
        ulong addr;

        rc = get_descriptor_ptr(ctxt, selector, &addr);
        if (rc != X86EMUL_CONTINUE)
                return rc;

        return linear_write_system(ctxt, addr, desc, sizeof(*desc));
}

static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
                                     u16 selector, int seg, u8 cpl,
                                     enum x86_transfer_type transfer,
                                     struct desc_struct *desc)
{
        struct desc_struct seg_desc, old_desc;
        u8 dpl, rpl;
        unsigned err_vec = GP_VECTOR;
        u32 err_code = 0;
        bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
        ulong desc_addr;
        int ret;
        u16 dummy;
        u32 base3 = 0;

        memset(&seg_desc, 0, sizeof(seg_desc));

        if (ctxt->mode == X86EMUL_MODE_REAL) {
                /* set real mode segment descriptor (keep limit etc. for
                 * unreal mode) */
                ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
                set_desc_base(&seg_desc, selector << 4);
                goto load;
        } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
                /* VM86 needs a clean new segment descriptor */
                set_desc_base(&seg_desc, selector << 4);
                set_desc_limit(&seg_desc, 0xffff);
                seg_desc.type = 3;
                seg_desc.p = 1;
                seg_desc.s = 1;
                seg_desc.dpl = 3;
                goto load;
        }

        rpl = selector & 3;

        /* TR should be in GDT only */
        if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
                goto exception;

        /* NULL selector is not valid for TR, CS and (except for long mode) SS */
        if (null_selector) {
                if (seg == VCPU_SREG_CS || seg == VCPU_SREG_TR)
                        goto exception;

                if (seg == VCPU_SREG_SS) {
                        if (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl)
                                goto exception;

                        /*
                         * ctxt->ops->set_segment expects the CPL to be in
                         * SS.DPL, so fake an expand-up 32-bit data segment.
                         */
                        seg_desc.type = 3;
                        seg_desc.p = 1;
                        seg_desc.s = 1;
                        seg_desc.dpl = cpl;
                        seg_desc.d = 1;
                        seg_desc.g = 1;
                }

                /* Skip all following checks */
                goto load;
        }

        ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
        if (ret != X86EMUL_CONTINUE)
                return ret;

        err_code = selector & 0xfffc;
        err_vec = (transfer == X86_TRANSFER_TASK_SWITCH) ? TS_VECTOR :
                                                           GP_VECTOR;

        /* can't load system descriptor into segment selector */
        if (seg <= VCPU_SREG_GS && !seg_desc.s) {
                if (transfer == X86_TRANSFER_CALL_JMP)
                        return X86EMUL_UNHANDLEABLE;
                goto exception;
        }

        if (!seg_desc.p) {
                err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
                goto exception;
        }

        dpl = seg_desc.dpl;

        switch (seg) {
        case VCPU_SREG_SS:
                /*
                 * segment is not a writable data segment or segment
                 * selector's RPL != CPL or segment selector's RPL != CPL
                 */
                if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
                        goto exception;
                break;
        case VCPU_SREG_CS:
                if (!(seg_desc.type & 8))
                        goto exception;

                if (seg_desc.type & 4) {
                        /* conforming */
                        if (dpl > cpl)
                                goto exception;
                } else {
                        /* nonconforming */
                        if (rpl > cpl || dpl != cpl)
                                goto exception;
                }
                /* in long-mode d/b must be clear if l is set */
                if (seg_desc.d && seg_desc.l) {
                        u64 efer = 0;

                        ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
                        if (efer & EFER_LMA)
                                goto exception;
                }

                /* CS(RPL) <- CPL */
                selector = (selector & 0xfffc) | cpl;
                break;
        case VCPU_SREG_TR:
                if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
                        goto exception;
                old_desc = seg_desc;
                seg_desc.type |= 2; /* busy */
                ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
                                                  sizeof(seg_desc), &ctxt->exception);
                if (ret != X86EMUL_CONTINUE)
                        return ret;
                break;
        case VCPU_SREG_LDTR:
                if (seg_desc.s || seg_desc.type != 2)
                        goto exception;
                break;
        default: /*  DS, ES, FS, or GS */
                /*
                 * segment is not a data or readable code segment or
                 * ((segment is a data or nonconforming code segment)
                 * and (both RPL and CPL > DPL))
                 */
                if ((seg_desc.type & 0xa) == 0x8 ||
                    (((seg_desc.type & 0xc) != 0xc) &&
                     (rpl > dpl && cpl > dpl)))
                        goto exception;
                break;
        }

        if (seg_desc.s) {
                /* mark segment as accessed */
                if (!(seg_desc.type & 1)) {
                        seg_desc.type |= 1;
                        ret = write_segment_descriptor(ctxt, selector,
                                                       &seg_desc);
                        if (ret != X86EMUL_CONTINUE)
                                return ret;
                }
        } else if (ctxt->mode == X86EMUL_MODE_PROT64) {
                ret = linear_read_system(ctxt, desc_addr+8, &base3, sizeof(base3));
                if (ret != X86EMUL_CONTINUE)
                        return ret;
                if (emul_is_noncanonical_address(get_desc_base(&seg_desc) |
                                ((u64)base3 << 32), ctxt))
                        return emulate_gp(ctxt, 0);
        }
load:
        ctxt->ops->set_segment(ctxt, selector, &seg_desc, base3, seg);
        if (desc)
                *desc = seg_desc;
        return X86EMUL_CONTINUE;
exception:
        return emulate_exception(ctxt, err_vec, err_code, true);
}

static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
                                   u16 selector, int seg)
{
        u8 cpl = ctxt->ops->cpl(ctxt);

        /*
         * None of MOV, POP and LSS can load a NULL selector in CPL=3, but
         * they can load it at CPL<3 (Intel's manual says only LSS can,
         * but it's wrong).
         *
         * However, the Intel manual says that putting IST=1/DPL=3 in
         * an interrupt gate will result in SS=3 (the AMD manual instead
         * says it doesn't), so allow SS=3 in __load_segment_descriptor
         * and only forbid it here.
         */
        if (seg == VCPU_SREG_SS && selector == 3 &&
            ctxt->mode == X86EMUL_MODE_PROT64)
                return emulate_exception(ctxt, GP_VECTOR, 0, true);

        return __load_segment_descriptor(ctxt, selector, seg, cpl,
                                         X86_TRANSFER_NONE, NULL);
}

static void write_register_operand(struct operand *op)
{
        return assign_register(op->addr.reg, op->val, op->bytes);
}

static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
{
        switch (op->type) {
        case OP_REG:
                write_register_operand(op);
                break;
        case OP_MEM:
                if (ctxt->lock_prefix)
                        return segmented_cmpxchg(ctxt,
                                                 op->addr.mem,
                                                 &op->orig_val,
                                                 &op->val,
                                                 op->bytes);
                else
                        return segmented_write(ctxt,
                                               op->addr.mem,
                                               &op->val,
                                               op->bytes);
                break;
        case OP_MEM_STR:
                return segmented_write(ctxt,
                                       op->addr.mem,
                                       op->data,
                                       op->bytes * op->count);
                break;
        case OP_XMM:
                kvm_write_sse_reg(op->addr.xmm, &op->vec_val);
                break;
        case OP_MM:
                kvm_write_mmx_reg(op->addr.mm, &op->mm_val);
                break;
        case OP_NONE:
                /* no writeback */
                break;
        default:
                break;
        }
        return X86EMUL_CONTINUE;
}

static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
{
        struct segmented_address addr;

        rsp_increment(ctxt, -bytes);
        addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
        addr.seg = VCPU_SREG_SS;

        return segmented_write(ctxt, addr, data, bytes);
}

static int em_push(struct x86_emulate_ctxt *ctxt)
{
        /* Disable writeback. */
        ctxt->dst.type = OP_NONE;
        return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
}

static int emulate_pop(struct x86_emulate_ctxt *ctxt,
                       void *dest, int len)
{
        int rc;
        struct segmented_address addr;

        addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
        addr.seg = VCPU_SREG_SS;
        rc = segmented_read(ctxt, addr, dest, len);
        if (rc != X86EMUL_CONTINUE)
                return rc;

        rsp_increment(ctxt, len);
        return rc;
}

static int em_pop(struct x86_emulate_ctxt *ctxt)
{
        return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
}

static int emulate_popf(struct x86_emulate_ctxt *ctxt,
                        void *dest, int len)
{
        int rc;
        unsigned long val, change_mask;
        int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
        int cpl = ctxt->ops->cpl(ctxt);

        rc = emulate_pop(ctxt, &val, len);
        if (rc != X86EMUL_CONTINUE)
                return rc;

        change_mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
                      X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF |
                      X86_EFLAGS_TF | X86_EFLAGS_DF | X86_EFLAGS_NT |
                      X86_EFLAGS_AC | X86_EFLAGS_ID;

        switch(ctxt->mode) {
        case X86EMUL_MODE_PROT64:
        case X86EMUL_MODE_PROT32:
        case X86EMUL_MODE_PROT16:
                if (cpl == 0)
                        change_mask |= X86_EFLAGS_IOPL;
                if (cpl <= iopl)
                        change_mask |= X86_EFLAGS_IF;
                break;
        case X86EMUL_MODE_VM86:
                if (iopl < 3)
                        return emulate_gp(ctxt, 0);
                change_mask |= X86_EFLAGS_IF;
                break;
        default: /* real mode */
                change_mask |= (X86_EFLAGS_IOPL | X86_EFLAGS_IF);
                break;
        }

        *(unsigned long *)dest =
                (ctxt->eflags & ~change_mask) | (val & change_mask);

        return rc;
}

static int em_popf(struct x86_emulate_ctxt *ctxt)
{
        ctxt->dst.type = OP_REG;
        ctxt->dst.addr.reg = &ctxt->eflags;
        ctxt->dst.bytes = ctxt->op_bytes;
        return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
}

static int em_enter(struct x86_emulate_ctxt *ctxt)
{
        int rc;
        unsigned frame_size = ctxt->src.val;
        unsigned nesting_level = ctxt->src2.val & 31;
        ulong rbp;

        if (nesting_level)
                return X86EMUL_UNHANDLEABLE;

        rbp = reg_read(ctxt, VCPU_REGS_RBP);
        rc = push(ctxt, &rbp, stack_size(ctxt));
        if (rc != X86EMUL_CONTINUE)
                return rc;
        assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
                      stack_mask(ctxt));
        assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
                      reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
                      stack_mask(ctxt));
        return X86EMUL_CONTINUE;
}

static int em_leave(struct x86_emulate_ctxt *ctxt)
{
        assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
                      stack_mask(ctxt));
        return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
}

static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
{
        int seg = ctxt->src2.val;

        ctxt->src.val = get_segment_selector(ctxt, seg);
        if (ctxt->op_bytes == 4) {
                rsp_increment(ctxt, -2);
                ctxt->op_bytes = 2;
        }

        return em_push(ctxt);
}

static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
{
        int seg = ctxt->src2.val;
        unsigned long selector;
        int rc;

        rc = emulate_pop(ctxt, &selector, 2);
        if (rc != X86EMUL_CONTINUE)
                return rc;

        if (ctxt->modrm_reg == VCPU_SREG_SS)
                ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
        if (ctxt->op_bytes > 2)
                rsp_increment(ctxt, ctxt->op_bytes - 2);

        rc = load_segment_descriptor(ctxt, (u16)selector, seg);
        return rc;
}

static int em_pusha(struct x86_emulate_ctxt *ctxt)
{
        unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
        int rc = X86EMUL_CONTINUE;
        int reg = VCPU_REGS_RAX;

        while (reg <= VCPU_REGS_RDI) {
                (reg == VCPU_REGS_RSP) ?
                (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));

                rc = em_push(ctxt);
                if (rc != X86EMUL_CONTINUE)
                        return rc;

                ++reg;
        }

        return rc;
}

static int em_pushf(struct x86_emulate_ctxt *ctxt)
{
        ctxt->src.val = (unsigned long)ctxt->eflags & ~X86_EFLAGS_VM;
        return em_push(ctxt);
}

static int em_popa(struct x86_emulate_ctxt *ctxt)
{
        int rc = X86EMUL_CONTINUE;
        int reg = VCPU_REGS_RDI;
        u32 val;

        while (reg >= VCPU_REGS_RAX) {
                if (reg == VCPU_REGS_RSP) {
                        rsp_increment(ctxt, ctxt->op_bytes);
                        --reg;
                }

                rc = emulate_pop(ctxt, &val, ctxt->op_bytes);
                if (rc != X86EMUL_CONTINUE)
                        break;
                assign_register(reg_rmw(ctxt, reg), val, ctxt->op_bytes);
                --reg;
        }
        return rc;
}

static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
{
        const struct x86_emulate_ops *ops = ctxt->ops;
        int rc;
        struct desc_ptr dt;
        gva_t cs_addr;
        gva_t eip_addr;
        u16 cs, eip;

        /* TODO: Add limit checks */
        ctxt->src.val = ctxt->eflags;
        rc = em_push(ctxt);
        if (rc != X86EMUL_CONTINUE)
                return rc;

        ctxt->eflags &= ~(X86_EFLAGS_IF | X86_EFLAGS_TF | X86_EFLAGS_AC);

        ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
        rc = em_push(ctxt);
        if (rc != X86EMUL_CONTINUE)
                return rc;

        ctxt->src.val = ctxt->_eip;
        rc = em_push(ctxt);