// SPDX-License-Identifier: GPL-2.0-only
/*
 * Kernel-based Virtual Machine driver for Linux
 *
 * derived from drivers/kvm/kvm_main.c
 *
 * Copyright (C) 2006 Qumranet, Inc.
 * Copyright (C) 2008 Qumranet, Inc.
 * Copyright IBM Corporation, 2008
 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
 *
 * Authors:
 *   Avi Kivity   <avi@qumranet.com>
 *   Yaniv Kamay  <yaniv@qumranet.com>
 *   Amit Shah    <amit.shah@qumranet.com>
 *   Ben-Ami Yassour <benami@il.ibm.com>
 */

#include <linux/kvm_host.h>
#include "irq.h"
#include "ioapic.h"
#include "mmu.h"
#include "i8254.h"
#include "tss.h"
#include "kvm_cache_regs.h"
#include "kvm_emulate.h"
#include "x86.h"
#include "cpuid.h"
#include "pmu.h"
#include "hyperv.h"
#include "lapic.h"
#include "xen.h"

#include <linux/clocksource.h>
#include <linux/interrupt.h>
#include <linux/kvm.h>
#include <linux/fs.h>
#include <linux/vmalloc.h>
#include <linux/export.h>
#include <linux/moduleparam.h>
#include <linux/mman.h>
#include <linux/highmem.h>
#include <linux/iommu.h>
#include <linux/intel-iommu.h>
#include <linux/cpufreq.h>
#include <linux/user-return-notifier.h>
#include <linux/srcu.h>
#include <linux/slab.h>
#include <linux/perf_event.h>
#include <linux/uaccess.h>
#include <linux/hash.h>
#include <linux/pci.h>
#include <linux/timekeeper_internal.h>
#include <linux/pvclock_gtod.h>
#include <linux/kvm_irqfd.h>
#include <linux/irqbypass.h>
#include <linux/sched/stat.h>
#include <linux/sched/isolation.h>
#include <linux/mem_encrypt.h>
#include <linux/entry-kvm.h>
#include <linux/suspend.h>

#include <trace/events/kvm.h>

#include <asm/debugreg.h>
#include <asm/msr.h>
#include <asm/desc.h>
#include <asm/mce.h>
#include <asm/pkru.h>
#include <linux/kernel_stat.h>
#include <asm/fpu/internal.h> /* Ugh! */
#include <asm/pvclock.h>
#include <asm/div64.h>
#include <asm/irq_remapping.h>
#include <asm/mshyperv.h>
#include <asm/hypervisor.h>
#include <asm/tlbflush.h>
#include <asm/intel_pt.h>
#include <asm/emulate_prefix.h>
#include <asm/sgx.h>
#include <clocksource/hyperv_timer.h>

#define CREATE_TRACE_POINTS
#include "trace.h"

#define MAX_IO_MSRS 256
#define KVM_MAX_MCE_BANKS 32
u64 __read_mostly kvm_mce_cap_supported = MCG_CTL_P | MCG_SER_P;
EXPORT_SYMBOL_GPL(kvm_mce_cap_supported);

#define emul_to_vcpu(ctxt) \
        ((struct kvm_vcpu *)(ctxt)->vcpu)

/* EFER defaults:
 * - enable syscall per default because its emulated by KVM
 * - enable LME and LMA per default on 64 bit KVM
 */
#ifdef CONFIG_X86_64
static
u64 __read_mostly efer_reserved_bits = ~((u64)(EFER_SCE | EFER_LME | EFER_LMA));
#else
static u64 __read_mostly efer_reserved_bits = ~((u64)EFER_SCE);
#endif

static u64 __read_mostly cr4_reserved_bits = CR4_RESERVED_BITS;

#define KVM_EXIT_HYPERCALL_VALID_MASK (1 << KVM_HC_MAP_GPA_RANGE)

#define KVM_X2APIC_API_VALID_FLAGS (KVM_X2APIC_API_USE_32BIT_IDS | \
                                    KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK)

static void update_cr8_intercept(struct kvm_vcpu *vcpu);
static void process_nmi(struct kvm_vcpu *vcpu);
static void process_smi(struct kvm_vcpu *vcpu);
static void enter_smm(struct kvm_vcpu *vcpu);
static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags);
static void store_regs(struct kvm_vcpu *vcpu);
static int sync_regs(struct kvm_vcpu *vcpu);

static int __set_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2);
static void __get_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2);

struct kvm_x86_ops kvm_x86_ops __read_mostly;
EXPORT_SYMBOL_GPL(kvm_x86_ops);

#define KVM_X86_OP(func)                                             \
        DEFINE_STATIC_CALL_NULL(kvm_x86_##func,                      \
                                *(((struct kvm_x86_ops *)0)->func));
#define KVM_X86_OP_NULL KVM_X86_OP
#include <asm/kvm-x86-ops.h>
EXPORT_STATIC_CALL_GPL(kvm_x86_get_cs_db_l_bits);
EXPORT_STATIC_CALL_GPL(kvm_x86_cache_reg);
EXPORT_STATIC_CALL_GPL(kvm_x86_tlb_flush_current);

static bool __read_mostly ignore_msrs = 0;
module_param(ignore_msrs, bool, S_IRUGO | S_IWUSR);

bool __read_mostly report_ignored_msrs = true;
module_param(report_ignored_msrs, bool, S_IRUGO | S_IWUSR);
EXPORT_SYMBOL_GPL(report_ignored_msrs);

unsigned int min_timer_period_us = 200;
module_param(min_timer_period_us, uint, S_IRUGO | S_IWUSR);

static bool __read_mostly kvmclock_periodic_sync = true;
module_param(kvmclock_periodic_sync, bool, S_IRUGO);

bool __read_mostly kvm_has_tsc_control;
EXPORT_SYMBOL_GPL(kvm_has_tsc_control);
u32  __read_mostly kvm_max_guest_tsc_khz;
EXPORT_SYMBOL_GPL(kvm_max_guest_tsc_khz);
u8   __read_mostly kvm_tsc_scaling_ratio_frac_bits;
EXPORT_SYMBOL_GPL(kvm_tsc_scaling_ratio_frac_bits);
u64  __read_mostly kvm_max_tsc_scaling_ratio;
EXPORT_SYMBOL_GPL(kvm_max_tsc_scaling_ratio);
u64 __read_mostly kvm_default_tsc_scaling_ratio;
EXPORT_SYMBOL_GPL(kvm_default_tsc_scaling_ratio);
bool __read_mostly kvm_has_bus_lock_exit;
EXPORT_SYMBOL_GPL(kvm_has_bus_lock_exit);

/* tsc tolerance in parts per million - default to 1/2 of the NTP threshold */
static u32 __read_mostly tsc_tolerance_ppm = 250;
module_param(tsc_tolerance_ppm, uint, S_IRUGO | S_IWUSR);

/*
 * lapic timer advance (tscdeadline mode only) in nanoseconds.  '-1' enables
 * adaptive tuning starting from default advancement of 1000ns.  '0' disables
 * advancement entirely.  Any other value is used as-is and disables adaptive
 * tuning, i.e. allows privileged userspace to set an exact advancement time.
 */
static int __read_mostly lapic_timer_advance_ns = -1;
module_param(lapic_timer_advance_ns, int, S_IRUGO | S_IWUSR);

static bool __read_mostly vector_hashing = true;
module_param(vector_hashing, bool, S_IRUGO);

bool __read_mostly enable_vmware_backdoor = false;
module_param(enable_vmware_backdoor, bool, S_IRUGO);
EXPORT_SYMBOL_GPL(enable_vmware_backdoor);

static bool __read_mostly force_emulation_prefix = false;
module_param(force_emulation_prefix, bool, S_IRUGO);

int __read_mostly pi_inject_timer = -1;
module_param(pi_inject_timer, bint, S_IRUGO | S_IWUSR);

/*
 * Restoring the host value for MSRs that are only consumed when running in
 * usermode, e.g. SYSCALL MSRs and TSC_AUX, can be deferred until the CPU
 * returns to userspace, i.e. the kernel can run with the guest's value.
 */
#define KVM_MAX_NR_USER_RETURN_MSRS 16

struct kvm_user_return_msrs {
        struct user_return_notifier urn;
        bool registered;
        struct kvm_user_return_msr_values {
                u64 host;
                u64 curr;
        } values[KVM_MAX_NR_USER_RETURN_MSRS];
};

u32 __read_mostly kvm_nr_uret_msrs;
EXPORT_SYMBOL_GPL(kvm_nr_uret_msrs);
static u32 __read_mostly kvm_uret_msrs_list[KVM_MAX_NR_USER_RETURN_MSRS];
static struct kvm_user_return_msrs __percpu *user_return_msrs;

#define KVM_SUPPORTED_XCR0     (XFEATURE_MASK_FP | XFEATURE_MASK_SSE \
                                | XFEATURE_MASK_YMM | XFEATURE_MASK_BNDREGS \
                                | XFEATURE_MASK_BNDCSR | XFEATURE_MASK_AVX512 \
                                | XFEATURE_MASK_PKRU)

u64 __read_mostly host_efer;
EXPORT_SYMBOL_GPL(host_efer);

bool __read_mostly allow_smaller_maxphyaddr = 0;
EXPORT_SYMBOL_GPL(allow_smaller_maxphyaddr);

bool __read_mostly enable_apicv = true;
EXPORT_SYMBOL_GPL(enable_apicv);

u64 __read_mostly host_xss;
EXPORT_SYMBOL_GPL(host_xss);
u64 __read_mostly supported_xss;
EXPORT_SYMBOL_GPL(supported_xss);

const struct _kvm_stats_desc kvm_vm_stats_desc[] = {
        KVM_GENERIC_VM_STATS(),
        STATS_DESC_COUNTER(VM, mmu_shadow_zapped),
        STATS_DESC_COUNTER(VM, mmu_pte_write),
        STATS_DESC_COUNTER(VM, mmu_pde_zapped),
        STATS_DESC_COUNTER(VM, mmu_flooded),
        STATS_DESC_COUNTER(VM, mmu_recycled),
        STATS_DESC_COUNTER(VM, mmu_cache_miss),
        STATS_DESC_ICOUNTER(VM, mmu_unsync),
        STATS_DESC_ICOUNTER(VM, lpages),
        STATS_DESC_ICOUNTER(VM, nx_lpage_splits),
        STATS_DESC_PCOUNTER(VM, max_mmu_page_hash_collisions)
};
static_assert(ARRAY_SIZE(kvm_vm_stats_desc) ==
                sizeof(struct kvm_vm_stat) / sizeof(u64));

const struct kvm_stats_header kvm_vm_stats_header = {
        .name_size = KVM_STATS_NAME_SIZE,
        .num_desc = ARRAY_SIZE(kvm_vm_stats_desc),
        .id_offset = sizeof(struct kvm_stats_header),
        .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE,
        .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE +
                       sizeof(kvm_vm_stats_desc),
};

const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = {
        KVM_GENERIC_VCPU_STATS(),
        STATS_DESC_COUNTER(VCPU, pf_fixed),
        STATS_DESC_COUNTER(VCPU, pf_guest),
        STATS_DESC_COUNTER(VCPU, tlb_flush),
        STATS_DESC_COUNTER(VCPU, invlpg),
        STATS_DESC_COUNTER(VCPU, exits),
        STATS_DESC_COUNTER(VCPU, io_exits),
        STATS_DESC_COUNTER(VCPU, mmio_exits),
        STATS_DESC_COUNTER(VCPU, signal_exits),
        STATS_DESC_COUNTER(VCPU, irq_window_exits),
        STATS_DESC_COUNTER(VCPU, nmi_window_exits),
        STATS_DESC_COUNTER(VCPU, l1d_flush),
        STATS_DESC_COUNTER(VCPU, halt_exits),
        STATS_DESC_COUNTER(VCPU, request_irq_exits),
        STATS_DESC_COUNTER(VCPU, irq_exits),
        STATS_DESC_COUNTER(VCPU, host_state_reload),
        STATS_DESC_COUNTER(VCPU, fpu_reload),
        STATS_DESC_COUNTER(VCPU, insn_emulation),
        STATS_DESC_COUNTER(VCPU, insn_emulation_fail),
        STATS_DESC_COUNTER(VCPU, hypercalls),
        STATS_DESC_COUNTER(VCPU, irq_injections),
        STATS_DESC_COUNTER(VCPU, nmi_injections),
        STATS_DESC_COUNTER(VCPU, req_event),
        STATS_DESC_COUNTER(VCPU, nested_run),
        STATS_DESC_COUNTER(VCPU, directed_yield_attempted),
        STATS_DESC_COUNTER(VCPU, directed_yield_successful),
        STATS_DESC_ICOUNTER(VCPU, guest_mode)
};
static_assert(ARRAY_SIZE(kvm_vcpu_stats_desc) ==
                sizeof(struct kvm_vcpu_stat) / sizeof(u64));

const struct kvm_stats_header kvm_vcpu_stats_header = {
        .name_size = KVM_STATS_NAME_SIZE,
        .num_desc = ARRAY_SIZE(kvm_vcpu_stats_desc),
        .id_offset = sizeof(struct kvm_stats_header),
        .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE,
        .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE +
                       sizeof(kvm_vcpu_stats_desc),
};

u64 __read_mostly host_xcr0;
u64 __read_mostly supported_xcr0;
EXPORT_SYMBOL_GPL(supported_xcr0);

static struct kmem_cache *x86_fpu_cache;

static struct kmem_cache *x86_emulator_cache;

/*
 * When called, it means the previous get/set msr reached an invalid msr.
 * Return true if we want to ignore/silent this failed msr access.
 */
static bool kvm_msr_ignored_check(u32 msr, u64 data, bool write)
{
        const char *op = write ? "wrmsr" : "rdmsr";

        if (ignore_msrs) {
                if (report_ignored_msrs)
                        kvm_pr_unimpl("ignored %s: 0x%x data 0x%llx\n",
                                      op, msr, data);
                /* Mask the error */
                return true;
        } else {
                kvm_debug_ratelimited("unhandled %s: 0x%x data 0x%llx\n",
                                      op, msr, data);
                return false;
        }
}

static struct kmem_cache *kvm_alloc_emulator_cache(void)
{
        unsigned int useroffset = offsetof(struct x86_emulate_ctxt, src);
        unsigned int size = sizeof(struct x86_emulate_ctxt);

        return kmem_cache_create_usercopy("x86_emulator", size,
                                          __alignof__(struct x86_emulate_ctxt),
                                          SLAB_ACCOUNT, useroffset,
                                          size - useroffset, NULL);
}

static int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt);

static inline void kvm_async_pf_hash_reset(struct kvm_vcpu *vcpu)
{
        int i;
        for (i = 0; i < ASYNC_PF_PER_VCPU; i++)
                vcpu->arch.apf.gfns[i] = ~0;
}

static void kvm_on_user_return(struct user_return_notifier *urn)
{
        unsigned slot;
        struct kvm_user_return_msrs *msrs
                = container_of(urn, struct kvm_user_return_msrs, urn);
        struct kvm_user_return_msr_values *values;
        unsigned long flags;

        /*
         * Disabling irqs at this point since the following code could be
         * interrupted and executed through kvm_arch_hardware_disable()
         */
        local_irq_save(flags);
        if (msrs->registered) {
                msrs->registered = false;
                user_return_notifier_unregister(urn);
        }
        local_irq_restore(flags);
        for (slot = 0; slot < kvm_nr_uret_msrs; ++slot) {
                values = &msrs->values[slot];
                if (values->host != values->curr) {
                        wrmsrl(kvm_uret_msrs_list[slot], values->host);
                        values->curr = values->host;
                }
        }
}

static int kvm_probe_user_return_msr(u32 msr)
{
        u64 val;
        int ret;

        preempt_disable();
        ret = rdmsrl_safe(msr, &val);
        if (ret)
                goto out;
        ret = wrmsrl_safe(msr, val);
out:
        preempt_enable();
        return ret;
}

int kvm_add_user_return_msr(u32 msr)
{
        BUG_ON(kvm_nr_uret_msrs >= KVM_MAX_NR_USER_RETURN_MSRS);

        if (kvm_probe_user_return_msr(msr))
                return -1;

        kvm_uret_msrs_list[kvm_nr_uret_msrs] = msr;
        return kvm_nr_uret_msrs++;
}
EXPORT_SYMBOL_GPL(kvm_add_user_return_msr);

int kvm_find_user_return_msr(u32 msr)
{
        int i;

        for (i = 0; i < kvm_nr_uret_msrs; ++i) {
                if (kvm_uret_msrs_list[i] == msr)
                        return i;
        }
        return -1;
}
EXPORT_SYMBOL_GPL(kvm_find_user_return_msr);

static void kvm_user_return_msr_cpu_online(void)
{
        unsigned int cpu = smp_processor_id();
        struct kvm_user_return_msrs *msrs = per_cpu_ptr(user_return_msrs, cpu);
        u64 value;
        int i;

        for (i = 0; i < kvm_nr_uret_msrs; ++i) {
                rdmsrl_safe(kvm_uret_msrs_list[i], &value);
                msrs->values[i].host = value;
                msrs->values[i].curr = value;
        }
}

int kvm_set_user_return_msr(unsigned slot, u64 value, u64 mask)
{
        unsigned int cpu = smp_processor_id();
        struct kvm_user_return_msrs *msrs = per_cpu_ptr(user_return_msrs, cpu);
        int err;

        value = (value & mask) | (msrs->values[slot].host & ~mask);
        if (value == msrs->values[slot].curr)
                return 0;
        err = wrmsrl_safe(kvm_uret_msrs_list[slot], value);
        if (err)
                return 1;

        msrs->values[slot].curr = value;
        if (!msrs->registered) {
                msrs->urn.on_user_return = kvm_on_user_return;
                user_return_notifier_register(&msrs->urn);
                msrs->registered = true;
        }
        return 0;
}
EXPORT_SYMBOL_GPL(kvm_set_user_return_msr);

static void drop_user_return_notifiers(void)
{
        unsigned int cpu = smp_processor_id();
        struct kvm_user_return_msrs *msrs = per_cpu_ptr(user_return_msrs, cpu);

        if (msrs->registered)
                kvm_on_user_return(&msrs->urn);
}

u64 kvm_get_apic_base(struct kvm_vcpu *vcpu)
{
        return vcpu->arch.apic_base;
}
EXPORT_SYMBOL_GPL(kvm_get_apic_base);

enum lapic_mode kvm_get_apic_mode(struct kvm_vcpu *vcpu)
{
        return kvm_apic_mode(kvm_get_apic_base(vcpu));
}
EXPORT_SYMBOL_GPL(kvm_get_apic_mode);

int kvm_set_apic_base(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
{
        enum lapic_mode old_mode = kvm_get_apic_mode(vcpu);
        enum lapic_mode new_mode = kvm_apic_mode(msr_info->data);
        u64 reserved_bits = kvm_vcpu_reserved_gpa_bits_raw(vcpu) | 0x2ff |
                (guest_cpuid_has(vcpu, X86_FEATURE_X2APIC) ? 0 : X2APIC_ENABLE);

        if ((msr_info->data & reserved_bits) != 0 || new_mode == LAPIC_MODE_INVALID)
                return 1;
        if (!msr_info->host_initiated) {
                if (old_mode == LAPIC_MODE_X2APIC && new_mode == LAPIC_MODE_XAPIC)
                        return 1;
                if (old_mode == LAPIC_MODE_DISABLED && new_mode == LAPIC_MODE_X2APIC)
                        return 1;
        }

        kvm_lapic_set_base(vcpu, msr_info->data);
        kvm_recalculate_apic_map(vcpu->kvm);
        return 0;
}
EXPORT_SYMBOL_GPL(kvm_set_apic_base);

asmlinkage __visible noinstr void kvm_spurious_fault(void)
{
        /* Fault while not rebooting.  We want the trace. */
        BUG_ON(!kvm_rebooting);
}
EXPORT_SYMBOL_GPL(kvm_spurious_fault);

#define EXCPT_BENIGN            0
#define EXCPT_CONTRIBUTORY      1
#define EXCPT_PF                2

static int exception_class(int vector)
{
        switch (vector) {
        case PF_VECTOR:
                return EXCPT_PF;
        case DE_VECTOR:
        case TS_VECTOR:
        case NP_VECTOR:
        case SS_VECTOR:
        case GP_VECTOR:
                return EXCPT_CONTRIBUTORY;
        default:
                break;
        }
        return EXCPT_BENIGN;
}

#define EXCPT_FAULT             0
#define EXCPT_TRAP              1
#define EXCPT_ABORT             2
#define EXCPT_INTERRUPT         3

static int exception_type(int vector)
{
        unsigned int mask;

        if (WARN_ON(vector > 31 || vector == NMI_VECTOR))
                return EXCPT_INTERRUPT;

        mask = 1 << vector;

        /* #DB is trap, as instruction watchpoints are handled elsewhere */
        if (mask & ((1 << DB_VECTOR) | (1 << BP_VECTOR) | (1 << OF_VECTOR)))
                return EXCPT_TRAP;

        if (mask & ((1 << DF_VECTOR) | (1 << MC_VECTOR)))
                return EXCPT_ABORT;

        /* Reserved exceptions will result in fault */
        return EXCPT_FAULT;
}

void kvm_deliver_exception_payload(struct kvm_vcpu *vcpu)
{
        unsigned nr = vcpu->arch.exception.nr;
        bool has_payload = vcpu->arch.exception.has_payload;
        unsigned long payload = vcpu->arch.exception.payload;

        if (!has_payload)
                return;

        switch (nr) {
        case DB_VECTOR:
                /*
                 * "Certain debug exceptions may clear bit 0-3.  The
                 * remaining contents of the DR6 register are never
                 * cleared by the processor".
                 */
                vcpu->arch.dr6 &= ~DR_TRAP_BITS;
                /*
                 * In order to reflect the #DB exception payload in guest
                 * dr6, three components need to be considered: active low
                 * bit, FIXED_1 bits and active high bits (e.g. DR6_BD,
                 * DR6_BS and DR6_BT)
                 * DR6_ACTIVE_LOW contains the FIXED_1 and active low bits.
                 * In the target guest dr6:
                 * FIXED_1 bits should always be set.
                 * Active low bits should be cleared if 1-setting in payload.
                 * Active high bits should be set if 1-setting in payload.
                 *
                 * Note, the payload is compatible with the pending debug
                 * exceptions/exit qualification under VMX, that active_low bits
                 * are active high in payload.
                 * So they need to be flipped for DR6.
                 */
                vcpu->arch.dr6 |= DR6_ACTIVE_LOW;
                vcpu->arch.dr6 |= payload;
                vcpu->arch.dr6 ^= payload & DR6_ACTIVE_LOW;

                /*
                 * The #DB payload is defined as compatible with the 'pending
                 * debug exceptions' field under VMX, not DR6. While bit 12 is
                 * defined in the 'pending debug exceptions' field (enabled
                 * breakpoint), it is reserved and must be zero in DR6.
                 */
                vcpu->arch.dr6 &= ~BIT(12);
                break;
        case PF_VECTOR:
                vcpu->arch.cr2 = payload;
                break;
        }

        vcpu->arch.exception.has_payload = false;
        vcpu->arch.exception.payload = 0;
}
EXPORT_SYMBOL_GPL(kvm_deliver_exception_payload);

static void kvm_multiple_exception(struct kvm_vcpu *vcpu,
                unsigned nr, bool has_error, u32 error_code,
                bool has_payload, unsigned long payload, bool reinject)
{
        u32 prev_nr;
        int class1, class2;

        kvm_make_request(KVM_REQ_EVENT, vcpu);

        if (!vcpu->arch.exception.pending && !vcpu->arch.exception.injected) {
        queue:
                if (reinject) {
                        /*
                         * On vmentry, vcpu->arch.exception.pending is only
                         * true if an event injection was blocked by
                         * nested_run_pending.  In that case, however,
                         * vcpu_enter_guest requests an immediate exit,
                         * and the guest shouldn't proceed far enough to
                         * need reinjection.
                         */
                        WARN_ON_ONCE(vcpu->arch.exception.pending);
                        vcpu->arch.exception.injected = true;
                        if (WARN_ON_ONCE(has_payload)) {
                                /*
                                 * A reinjected event has already
                                 * delivered its payload.
                                 */
                                has_payload = false;
                                payload = 0;
                        }
                } else {
                        vcpu->arch.exception.pending = true;
                        vcpu->arch.exception.injected = false;
                }
                vcpu->arch.exception.has_error_code = has_error;
                vcpu->arch.exception.nr = nr;
                vcpu->arch.exception.error_code = error_code;
                vcpu->arch.exception.has_payload = has_payload;
                vcpu->arch.exception.payload = payload;
                if (!is_guest_mode(vcpu))
                        kvm_deliver_exception_payload(vcpu);
                return;
        }

        /* to check exception */
        prev_nr = vcpu->arch.exception.nr;
        if (prev_nr == DF_VECTOR) {
                /* triple fault -> shutdown */
                kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
                return;
        }
        class1 = exception_class(prev_nr);
        class2 = exception_class(nr);
        if ((class1 == EXCPT_CONTRIBUTORY && class2 == EXCPT_CONTRIBUTORY)
                || (class1 == EXCPT_PF && class2 != EXCPT_BENIGN)) {
                /*
                 * Generate double fault per SDM Table 5-5.  Set
                 * exception.pending = true so that the double fault
                 * can trigger a nested vmexit.
                 */
                vcpu->arch.exception.pending = true;
                vcpu->arch.exception.injected = false;
                vcpu->arch.exception.has_error_code = true;
                vcpu->arch.exception.nr = DF_VECTOR;
                vcpu->arch.exception.error_code = 0;
                vcpu->arch.exception.has_payload = false;
                vcpu->arch.exception.payload = 0;
        } else
                /* replace previous exception with a new one in a hope
                   that instruction re-execution will regenerate lost
                   exception */
                goto queue;
}

void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr)
{
        kvm_multiple_exception(vcpu, nr, false, 0, false, 0, false);
}
EXPORT_SYMBOL_GPL(kvm_queue_exception);

void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr)
{
        kvm_multiple_exception(vcpu, nr, false, 0, false, 0, true);
}
EXPORT_SYMBOL_GPL(kvm_requeue_exception);

void kvm_queue_exception_p(struct kvm_vcpu *vcpu, unsigned nr,
                           unsigned long payload)
{
        kvm_multiple_exception(vcpu, nr, false, 0, true, payload, false);
}
EXPORT_SYMBOL_GPL(kvm_queue_exception_p);

static void kvm_queue_exception_e_p(struct kvm_vcpu *vcpu, unsigned nr,
                                    u32 error_code, unsigned long payload)
{
        kvm_multiple_exception(vcpu, nr, true, error_code,
                               true, payload, false);
}

int kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err)
{
        if (err)
                kvm_inject_gp(vcpu, 0);
        else
                return kvm_skip_emulated_instruction(vcpu);

        return 1;
}
EXPORT_SYMBOL_GPL(kvm_complete_insn_gp);

void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault)
{
        ++vcpu->stat.pf_guest;
        vcpu->arch.exception.nested_apf =
                is_guest_mode(vcpu) && fault->async_page_fault;
        if (vcpu->arch.exception.nested_apf) {
                vcpu->arch.apf.nested_apf_token = fault->address;
                kvm_queue_exception_e(vcpu, PF_VECTOR, fault->error_code);
        } else {
                kvm_queue_exception_e_p(vcpu, PF_VECTOR, fault->error_code,
                                        fault->address);
        }
}
EXPORT_SYMBOL_GPL(kvm_inject_page_fault);

bool kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu,
                                    struct x86_exception *fault)
{
        struct kvm_mmu *fault_mmu;
        WARN_ON_ONCE(fault->vector != PF_VECTOR);

        fault_mmu = fault->nested_page_fault ? vcpu->arch.mmu :
                                               vcpu->arch.walk_mmu;

        /*
         * Invalidate the TLB entry for the faulting address, if it exists,
         * else the access will fault indefinitely (and to emulate hardware).
         */
        if ((fault->error_code & PFERR_PRESENT_MASK) &&
            !(fault->error_code & PFERR_RSVD_MASK))
                kvm_mmu_invalidate_gva(vcpu, fault_mmu, fault->address,
                                       fault_mmu->root_hpa);

        fault_mmu->inject_page_fault(vcpu, fault);
        return fault->nested_page_fault;
}
EXPORT_SYMBOL_GPL(kvm_inject_emulated_page_fault);

void kvm_inject_nmi(struct kvm_vcpu *vcpu)
{
        atomic_inc(&vcpu->arch.nmi_queued);
        kvm_make_request(KVM_REQ_NMI, vcpu);
}
EXPORT_SYMBOL_GPL(kvm_inject_nmi);

void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)
{
        kvm_multiple_exception(vcpu, nr, true, error_code, false, 0, false);
}
EXPORT_SYMBOL_GPL(kvm_queue_exception_e);

void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)
{
        kvm_multiple_exception(vcpu, nr, true, error_code, false, 0, true);
}
EXPORT_SYMBOL_GPL(kvm_requeue_exception_e);

/*
 * Checks if cpl <= required_cpl; if true, return true.  Otherwise queue
 * a #GP and return false.
 */
bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl)
{
        if (static_call(kvm_x86_get_cpl)(vcpu) <= required_cpl)
                return true;
        kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
        return false;
}
EXPORT_SYMBOL_GPL(kvm_require_cpl);

bool kvm_require_dr(struct kvm_vcpu *vcpu, int dr)
{
        if ((dr != 4 && dr != 5) || !kvm_read_cr4_bits(vcpu, X86_CR4_DE))
                return true;

        kvm_queue_exception(vcpu, UD_VECTOR);
        return false;
}
EXPORT_SYMBOL_GPL(kvm_require_dr);

/*
 * This function will be used to read from the physical memory of the currently
 * running guest. The difference to kvm_vcpu_read_guest_page is that this function
 * can read from guest physical or from the guest's guest physical memory.
 */
int kvm_read_guest_page_mmu(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
                            gfn_t ngfn, void *data, int offset, int len,
                            u32 access)
{
        struct x86_exception exception;
        gfn_t real_gfn;
        gpa_t ngpa;

        ngpa     = gfn_to_gpa(ngfn);
        real_gfn = mmu->translate_gpa(vcpu, ngpa, access, &exception);
        if (real_gfn == UNMAPPED_GVA)
                return -EFAULT;

        real_gfn = gpa_to_gfn(real_gfn);

        return kvm_vcpu_read_guest_page(vcpu, real_gfn, data, offset, len);
}
EXPORT_SYMBOL_GPL(kvm_read_guest_page_mmu);

static inline u64 pdptr_rsvd_bits(struct kvm_vcpu *vcpu)
{
        return vcpu->arch.reserved_gpa_bits | rsvd_bits(5, 8) | rsvd_bits(1, 2);
}

/*
 * Load the pae pdptrs.  Return 1 if they are all valid, 0 otherwise.
 */
int load_pdptrs(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, unsigned long cr3)
{
        gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT;
        unsigned offset = ((cr3 & (PAGE_SIZE-1)) >> 5) << 2;
        int i;
        int ret;
        u64 pdpte[ARRAY_SIZE(mmu->pdptrs)];

        ret = kvm_read_guest_page_mmu(vcpu, mmu, pdpt_gfn, pdpte,
                                      offset * sizeof(u64), sizeof(pdpte),
                                      PFERR_USER_MASK|PFERR_WRITE_MASK);
        if (ret < 0) {
                ret = 0;
                goto out;
        }
        for (i = 0; i < ARRAY_SIZE(pdpte); ++i) {
                if ((pdpte[i] & PT_PRESENT_MASK) &&
                    (pdpte[i] & pdptr_rsvd_bits(vcpu))) {
                        ret = 0;
                        goto out;
                }
        }
        ret = 1;

        memcpy(mmu->pdptrs, pdpte, sizeof(mmu->pdptrs));
        kvm_register_mark_dirty(vcpu, VCPU_EXREG_PDPTR);
        vcpu->arch.pdptrs_from_userspace = false;

out:

        return ret;
}
EXPORT_SYMBOL_GPL(load_pdptrs);

void kvm_post_set_cr0(struct kvm_vcpu *vcpu, unsigned long old_cr0, unsigned long cr0)
{
        if ((cr0 ^ old_cr0) & X86_CR0_PG) {
                kvm_clear_async_pf_completion_queue(vcpu);
                kvm_async_pf_hash_reset(vcpu);
        }

        if ((cr0 ^ old_cr0) & KVM_MMU_CR0_ROLE_BITS)
                kvm_mmu_reset_context(vcpu);

        if (((cr0 ^ old_cr0) & X86_CR0_CD) &&
            kvm_arch_has_noncoherent_dma(vcpu->kvm) &&
            !kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))
                kvm_zap_gfn_range(vcpu->kvm, 0, ~0ULL);
}
EXPORT_SYMBOL_GPL(kvm_post_set_cr0);

int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
{
        unsigned long old_cr0 = kvm_read_cr0(vcpu);
        unsigned long pdptr_bits = X86_CR0_CD | X86_CR0_NW | X86_CR0_PG;

        cr0 |= X86_CR0_ET;

#ifdef CONFIG_X86_64
        if (cr0 & 0xffffffff00000000UL)
                return 1;
#endif

        cr0 &= ~CR0_RESERVED_BITS;

        if ((cr0 & X86_CR0_NW) && !(cr0 & X86_CR0_CD))
                return 1;

        if ((cr0 & X86_CR0_PG) && !(cr0 & X86_CR0_PE))
                return 1;

#ifdef CONFIG_X86_64
        if ((vcpu->arch.efer & EFER_LME) && !is_paging(vcpu) &&
            (cr0 & X86_CR0_PG)) {
                int cs_db, cs_l;

                if (!is_pae(vcpu))
                        return 1;
                static_call(kvm_x86_get_cs_db_l_bits)(vcpu, &cs_db, &cs_l);
                if (cs_l)
                        return 1;
        }
#endif
        if (!(vcpu->arch.efer & EFER_LME) && (cr0 & X86_CR0_PG) &&
            is_pae(vcpu) && ((cr0 ^ old_cr0) & pdptr_bits) &&
            !load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu)))
                return 1;

        if (!(cr0 & X86_CR0_PG) && kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE))
                return 1;

        static_call(kvm_x86_set_cr0)(vcpu, cr0);

        kvm_post_set_cr0(vcpu, old_cr0, cr0);

        return 0;
}
EXPORT_SYMBOL_GPL(kvm_set_cr0);

void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw)
{
        (void)kvm_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~0x0eul) | (msw & 0x0f));
}
EXPORT_SYMBOL_GPL(kvm_lmsw);

void kvm_load_guest_xsave_state(struct kvm_vcpu *vcpu)
{
        if (vcpu->arch.guest_state_protected)
                return;

        if (kvm_read_cr4_bits(vcpu, X86_CR4_OSXSAVE)) {

                if (vcpu->arch.xcr0 != host_xcr0)
                        xsetbv(XCR_XFEATURE_ENABLED_MASK, vcpu->arch.xcr0);

                if (vcpu->arch.xsaves_enabled &&
                    vcpu->arch.ia32_xss != host_xss)
                        wrmsrl(MSR_IA32_XSS, vcpu->arch.ia32_xss);
        }

        if (static_cpu_has(X86_FEATURE_PKU) &&
            (kvm_read_cr4_bits(vcpu, X86_CR4_PKE) ||
             (vcpu->arch.xcr0 & XFEATURE_MASK_PKRU)) &&
            vcpu->arch.pkru != vcpu->arch.host_pkru)
                write_pkru(vcpu->arch.pkru);
}
EXPORT_SYMBOL_GPL(kvm_load_guest_xsave_state);

void kvm_load_host_xsave_state(struct kvm_vcpu *vcpu)
{
        if (vcpu->arch.guest_state_protected)
                return;

        if (static_cpu_has(X86_FEATURE_PKU) &&
            (kvm_read_cr4_bits(vcpu, X86_CR4_PKE) ||
             (vcpu->arch.xcr0 & XFEATURE_MASK_PKRU))) {
                vcpu->arch.pkru = rdpkru();
                if (vcpu->arch.pkru != vcpu->arch.host_pkru)
                        write_pkru(vcpu->arch.host_pkru);
        }

        if (kvm_read_cr4_bits(vcpu, X86_CR4_OSXSAVE)) {

                if (vcpu->arch.xcr0 != host_xcr0)
                        xsetbv(XCR_XFEATURE_ENABLED_MASK, host_xcr0);

                if (vcpu->arch.xsaves_enabled &&
                    vcpu->arch.ia32_xss != host_xss)
                        wrmsrl(MSR_IA32_XSS, host_xss);
        }

}
EXPORT_SYMBOL_GPL(kvm_load_host_xsave_state);

static int __kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr)
{
        u64 xcr0 = xcr;
        u64 old_xcr0 = vcpu->arch.xcr0;
        u64 valid_bits;

        /* Only support XCR_XFEATURE_ENABLED_MASK(xcr0) now  */
        if (index != XCR_XFEATURE_ENABLED_MASK)
                return 1;
        if (!(xcr0 & XFEATURE_MASK_FP))
                return 1;
        if ((xcr0 & XFEATURE_MASK_YMM) && !(xcr0 & XFEATURE_MASK_SSE))
                return 1;

        /*
         * Do not allow the guest to set bits that we do not support
         * saving.  However, xcr0 bit 0 is always set, even if the
         * emulated CPU does not support XSAVE (see fx_init).
         */
        valid_bits = vcpu->arch.guest_supported_xcr0 | XFEATURE_MASK_FP;
        if (xcr0 & ~valid_bits)
                return 1;

        if ((!(xcr0 & XFEATURE_MASK_BNDREGS)) !=
            (!(xcr0 & XFEATURE_MASK_BNDCSR)))
                return 1;

        if (xcr0 & XFEATURE_MASK_AVX512) {

                if (!(xcr0 & XFEATURE_MASK_YMM))
                        return 1;
                if ((xcr0 & XFEATURE_MASK_AVX512) != XFEATURE_MASK_AVX512)
                        return 1;
        }
        vcpu->arch.xcr0 = xcr0;

        if ((xcr0 ^ old_xcr0) & XFEATURE_MASK_EXTEND)
                kvm_update_cpuid_runtime(vcpu);
        return 0;
}

int kvm_emulate_xsetbv(struct kvm_vcpu *vcpu)
{
        if (static_call(kvm_x86_get_cpl)(vcpu) != 0 ||
            __kvm_set_xcr(vcpu, kvm_rcx_read(vcpu), kvm_read_edx_eax(vcpu))) {
                kvm_inject_gp(vcpu, 0);
                return 1;
        }

        return kvm_skip_emulated_instruction(vcpu);
}
EXPORT_SYMBOL_GPL(kvm_emulate_xsetbv);

bool kvm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
{
        if (cr4 & cr4_reserved_bits)
                return false;

        if (cr4 & vcpu->arch.cr4_guest_rsvd_bits)
                return false;

        return static_call(kvm_x86_is_valid_cr4)(vcpu, cr4);
}
EXPORT_SYMBOL_GPL(kvm_is_valid_cr4);

void kvm_post_set_cr4(struct kvm_vcpu *vcpu, unsigned long old_cr4, unsigned long cr4)
{
        if (((cr4 ^ old_cr4) & KVM_MMU_CR4_ROLE_BITS) ||
            (!(cr4 & X86_CR4_PCIDE) && (old_cr4 & X86_CR4_PCIDE)))
                kvm_mmu_reset_context(vcpu);
}
EXPORT_SYMBOL_GPL(kvm_post_set_cr4);

int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
{
        unsigned long old_cr4 = kvm_read_cr4(vcpu);
        unsigned long pdptr_bits = X86_CR4_PGE | X86_CR4_PSE | X86_CR4_PAE |
                                   X86_CR4_SMEP;

        if (!kvm_is_valid_cr4(vcpu, cr4))
                return 1;

        if (is_long_mode(vcpu)) {
                if (!(cr4 & X86_CR4_PAE))
                        return 1;
                if ((cr4 ^ old_cr4) & X86_CR4_LA57)
                        return 1;
        } else if (is_paging(vcpu) && (cr4 & X86_CR4_PAE)
                   && ((cr4 ^ old_cr4) & pdptr_bits)
                   && !load_pdptrs(vcpu, vcpu->arch.walk_mmu,
                                   kvm_read_cr3(vcpu)))
                return 1;

        if ((cr4 & X86_CR4_PCIDE) && !(old_cr4 & X86_CR4_PCIDE)) {
                if (!guest_cpuid_has(vcpu, X86_FEATURE_PCID))
                        return 1;

                /* PCID can not be enabled when cr3[11:0]!=000H or EFER.LMA=0 */
                if ((kvm_read_cr3(vcpu) & X86_CR3_PCID_MASK) || !is_long_mode(vcpu))
                        return 1;
        }

        static_call(kvm_x86_set_cr4)(vcpu, cr4);

        kvm_post_set_cr4(vcpu, old_cr4, cr4);

        return 0;
}
EXPORT_SYMBOL_GPL(kvm_set_cr4);

static void kvm_invalidate_pcid(struct kvm_vcpu *vcpu, unsigned long pcid)
{
        struct kvm_mmu *mmu = vcpu->arch.mmu;
        unsigned long roots_to_free = 0;
        int i;

        /*
         * If neither the current CR3 nor any of the prev_roots use the given
         * PCID, then nothing needs to be done here because a resync will
         * happen anyway before switching to any other CR3.
         */
        if (kvm_get_active_pcid(vcpu) == pcid) {
                kvm_make_request(KVM_REQ_MMU_SYNC, vcpu);
                kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);
        }

        for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
                if (kvm_get_pcid(vcpu, mmu->prev_roots[i].pgd) == pcid)
                        roots_to_free |= KVM_MMU_ROOT_PREVIOUS(i);

        kvm_mmu_free_roots(vcpu, mmu, roots_to_free);
}

int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
{
        bool skip_tlb_flush = false;
        unsigned long pcid = 0;
#ifdef CONFIG_X86_64
        bool pcid_enabled = kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE);

        if (pcid_enabled) {
                skip_tlb_flush = cr3 & X86_CR3_PCID_NOFLUSH;
                cr3 &= ~X86_CR3_PCID_NOFLUSH;
                pcid = cr3 & X86_CR3_PCID_MASK;
        }
#endif

        /* PDPTRs are always reloaded for PAE paging. */
        if (cr3 == kvm_read_cr3(vcpu) && !is_pae_paging(vcpu))
                goto handle_tlb_flush;

        /*
         * Do not condition the GPA check on long mode, this helper is used to
         * stuff CR3, e.g. for RSM emulation, and there is no guarantee that
         * the current vCPU mode is accurate.
         */
        if (kvm_vcpu_is_illegal_gpa(vcpu, cr3))
                return 1;

        if (is_pae_paging(vcpu) && !load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3))
                return 1;

        if (cr3 != kvm_read_cr3(vcpu))
                kvm_mmu_new_pgd(vcpu, cr3);

        vcpu->arch.cr3 = cr3;
        kvm_register_mark_available(vcpu, VCPU_EXREG_CR3);

handle_tlb_flush:
        /*
         * A load of CR3 that flushes the TLB flushes only the current PCID,
         * even if PCID is disabled, in which case PCID=0 is flushed.  It's a
         * moot point in the end because _disabling_ PCID will flush all PCIDs,
         * and it's impossible to use a non-zero PCID when PCID is disabled,
         * i.e. only PCID=0 can be relevant.
         */
        if (!skip_tlb_flush)
                kvm_invalidate_pcid(vcpu, pcid);

        return 0;
}
EXPORT_SYMBOL_GPL(kvm_set_cr3);

int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
{
        if (cr8 & CR8_RESERVED_BITS)
                return 1;
        if (lapic_in_kernel(vcpu))
                kvm_lapic_set_tpr(vcpu, cr8);
        else
                vcpu->arch.cr8 = cr8;
        return 0;
}
EXPORT_SYMBOL_GPL(kvm_set_cr8);

unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu)
{
        if (lapic_in_kernel(vcpu))
                return kvm_lapic_get_cr8(vcpu);
        else
                return vcpu->arch.cr8;
}
EXPORT_SYMBOL_GPL(kvm_get_cr8);

static void kvm_update_dr0123(struct kvm_vcpu *vcpu)
{
        int i;

        if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) {
                for (i = 0; i < KVM_NR_DB_REGS; i++)
                        vcpu->arch.eff_db[i] = vcpu->arch.db[i];
                vcpu->arch.switch_db_regs |= KVM_DEBUGREG_RELOAD;
        }
}

void kvm_update_dr7(struct kvm_vcpu *vcpu)
{
        unsigned long dr7;

        if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
                dr7 = vcpu->arch.guest_debug_dr7;
        else
                dr7 = vcpu->arch.dr7;
        static_call(kvm_x86_set_dr7)(vcpu, dr7);
        vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_BP_ENABLED;
        if (dr7 & DR7_BP_EN_MASK)
                vcpu->arch.switch_db_regs |= KVM_DEBUGREG_BP_ENABLED;
}
EXPORT_SYMBOL_GPL(kvm_update_dr7);

static u64 kvm_dr6_fixed(struct kvm_vcpu *vcpu)
{
        u64 fixed = DR6_FIXED_1;

        if (!guest_cpuid_has(vcpu, X86_FEATURE_RTM))
                fixed |= DR6_RTM;

        if (!guest_cpuid_has(vcpu, X86_FEATURE_BUS_LOCK_DETECT))
                fixed |= DR6_BUS_LOCK;
        return fixed;
}

int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)
{
        size_t size = ARRAY_SIZE(vcpu->arch.db);

        switch (dr) {
        case 0 ... 3:
                vcpu->arch.db[array_index_nospec(dr, size)] = val;
                if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP))
                        vcpu->arch.eff_db[dr] = val;
                break;
        case 4:
        case 6:
                if (!kvm_dr6_valid(val))
                        return 1; /* #GP */
                vcpu->arch.dr6 = (val & DR6_VOLATILE) | kvm_dr6_fixed(vcpu);
                break;
        case 5:
        default: /* 7 */
                if (!kvm_dr7_valid(val))
                        return 1; /* #GP */
                vcpu->arch.dr7 = (val & DR7_VOLATILE) | DR7_FIXED_1;
                kvm_update_dr7(vcpu);
                break;
        }

        return 0;
}
EXPORT_SYMBOL_GPL(kvm_set_dr);

void kvm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *val)
{
        size_t size = ARRAY_SIZE(vcpu->arch.db);

        switch (dr) {
        case 0 ... 3:
                *val = vcpu->arch.db[array_index_nospec(dr, size)];
                break;
        case 4:
        case 6:
                *val = vcpu->arch.dr6;
                break;
        case 5:
        default: /* 7 */
                *val = vcpu->arch.dr7;
                break;
        }
}
EXPORT_SYMBOL_GPL(kvm_get_dr);

int kvm_emulate_rdpmc(struct kvm_vcpu *vcpu)
{
        u32 ecx = kvm_rcx_read(vcpu);
        u64 data;

        if (kvm_pmu_rdpmc(vcpu, ecx, &data)) {
                kvm_inject_gp(vcpu, 0);
                return 1;
        }

        kvm_rax_write(vcpu, (u32)data);
        kvm_rdx_write(vcpu, data >> 32);
        return kvm_skip_emulated_instruction(vcpu);
}
EXPORT_SYMBOL_GPL(kvm_emulate_rdpmc);

/*
 * List of msr numbers which we expose to userspace through KVM_GET_MSRS
 * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST.
 *
 * The three MSR lists(msrs_to_save, emulated_msrs, msr_based_features)
 * extract the supported MSRs from the related const lists.
 * msrs_to_save is selected from the msrs_to_save_all to reflect the
 * capabilities of the host cpu. This capabilities test skips MSRs that are
 * kvm-specific. Those are put in emulated_msrs_all; filtering of emulated_msrs
 * may depend on host virtualization features rather than host cpu features.
 */

static const u32 msrs_to_save_all[] = {
        MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
        MSR_STAR,
#ifdef CONFIG_X86_64
        MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR,
#endif
        MSR_IA32_TSC, MSR_IA32_CR_PAT, MSR_VM_HSAVE_PA,
        MSR_IA32_FEAT_CTL, MSR_IA32_BNDCFGS, MSR_TSC_AUX,
        MSR_IA32_SPEC_CTRL,
        MSR_IA32_RTIT_CTL, MSR_IA32_RTIT_STATUS, MSR_IA32_RTIT_CR3_MATCH,
        MSR_IA32_RTIT_OUTPUT_BASE, MSR_IA32_RTIT_OUTPUT_MASK,
        MSR_IA32_RTIT_ADDR0_A, MSR_IA32_RTIT_ADDR0_B,
        MSR_IA32_RTIT_ADDR1_A, MSR_IA32_RTIT_ADDR1_B,
        MSR_IA32_RTIT_ADDR2_A, MSR_IA32_RTIT_ADDR2_B,
        MSR_IA32_RTIT_ADDR3_A, MSR_IA32_RTIT_ADDR3_B,
        MSR_IA32_UMWAIT_CONTROL,

        MSR_ARCH_PERFMON_FIXED_CTR0, MSR_ARCH_PERFMON_FIXED_CTR1,
        MSR_ARCH_PERFMON_FIXED_CTR0 + 2, MSR_ARCH_PERFMON_FIXED_CTR0 + 3,
        MSR_CORE_PERF_FIXED_CTR_CTRL, MSR_CORE_PERF_GLOBAL_STATUS,
        MSR_CORE_PERF_GLOBAL_CTRL, MSR_CORE_PERF_GLOBAL_OVF_CTRL,
        MSR_ARCH_PERFMON_PERFCTR0, MSR_ARCH_PERFMON_PERFCTR1,
        MSR_ARCH_PERFMON_PERFCTR0 + 2, MSR_ARCH_PERFMON_PERFCTR0 + 3,
        MSR_ARCH_PERFMON_PERFCTR0 + 4, MSR_ARCH_PERFMON_PERFCTR0 + 5,
        MSR_ARCH_PERFMON_PERFCTR0 + 6, MSR_ARCH_PERFMON_PERFCTR0 + 7,
        MSR_ARCH_PERFMON_PERFCTR0 + 8, MSR_ARCH_PERFMON_PERFCTR0 + 9,
        MSR_ARCH_PERFMON_PERFCTR0 + 10, MSR_ARCH_PERFMON_PERFCTR0 + 11,
        MSR_ARCH_PERFMON_PERFCTR0 + 12, MSR_ARCH_PERFMON_PERFCTR0 + 13,
        MSR_ARCH_PERFMON_PERFCTR0 + 14, MSR_ARCH_PERFMON_PERFCTR0 + 15,
        MSR_ARCH_PERFMON_PERFCTR0 + 16, MSR_ARCH_PERFMON_PERFCTR0 + 17,
        MSR_ARCH_PERFMON_EVENTSEL0, MSR_ARCH_PERFMON_EVENTSEL1,
        MSR_ARCH_PERFMON_EVENTSEL0 + 2, MSR_ARCH_PERFMON_EVENTSEL0 + 3,
        MSR_ARCH_PERFMON_EVENTSEL0 + 4, MSR_ARCH_PERFMON_EVENTSEL0 + 5,
        MSR_ARCH_PERFMON_EVENTSEL0 + 6, MSR_ARCH_PERFMON_EVENTSEL0 + 7,
        MSR_ARCH_PERFMON_EVENTSEL0 + 8, MSR_ARCH_PERFMON_EVENTSEL0 + 9,
        MSR_ARCH_PERFMON_EVENTSEL0 + 10, MSR_ARCH_PERFMON_EVENTSEL0 + 11,
        MSR_ARCH_PERFMON_EVENTSEL0 + 12, MSR_ARCH_PERFMON_EVENTSEL0 + 13,
        MSR_ARCH_PERFMON_EVENTSEL0 + 14, MSR_ARCH_PERFMON_EVENTSEL0 + 15,
        MSR_ARCH_PERFMON_EVENTSEL0 + 16, MSR_ARCH_PERFMON_EVENTSEL0 + 17,
};

static u32 msrs_to_save[ARRAY_SIZE(msrs_to_save_all)];
static unsigned num_msrs_to_save;

static const u32 emulated_msrs_all[] = {
        MSR_KVM_SYSTEM_TIME, MSR_KVM_WALL_CLOCK,
        MSR_KVM_SYSTEM_TIME_NEW, MSR_KVM_WALL_CLOCK_NEW,
        HV_X64_MSR_GUEST_OS_ID, HV_X64_MSR_HYPERCALL,
        HV_X64_MSR_TIME_REF_COUNT, HV_X64_MSR_REFERENCE_TSC,
        HV_X64_MSR_TSC_FREQUENCY, HV_X64_MSR_APIC_FREQUENCY,
        HV_X64_MSR_CRASH_P0, HV_X64_MSR_CRASH_P1, HV_X64_MSR_CRASH_P2,
        HV_X64_MSR_CRASH_P3, HV_X64_MSR_CRASH_P4, HV_X64_MSR_CRASH_CTL,
        HV_X64_MSR_RESET,
        HV_X64_MSR_VP_INDEX,
        HV_X64_MSR_VP_RUNTIME,
        HV_X64_MSR_SCONTROL,
        HV_X64_MSR_STIMER0_CONFIG,
        HV_X64_MSR_VP_ASSIST_PAGE,
        HV_X64_MSR_REENLIGHTENMENT_CONTROL, HV_X64_MSR_TSC_EMULATION_CONTROL,
        HV_X64_MSR_TSC_EMULATION_STATUS,
        HV_X64_MSR_SYNDBG_OPTIONS,
        HV_X64_MSR_SYNDBG_CONTROL, HV_X64_MSR_SYNDBG_STATUS,
        HV_X64_MSR_SYNDBG_SEND_BUFFER, HV_X64_MSR_SYNDBG_RECV_BUFFER,
        HV_X64_MSR_SYNDBG_PENDING_BUFFER,

        MSR_KVM_ASYNC_PF_EN, MSR_KVM_STEAL_TIME,
        MSR_KVM_PV_EOI_EN, MSR_KVM_ASYNC_PF_INT, MSR_KVM_ASYNC_PF_ACK,

        MSR_IA32_TSC_ADJUST,
        MSR_IA32_TSC_DEADLINE,
        MSR_IA32_ARCH_CAPABILITIES,
        MSR_IA32_PERF_CAPABILITIES,
        MSR_IA32_MISC_ENABLE,
        MSR_IA32_MCG_STATUS,
        MSR_IA32_MCG_CTL,
        MSR_IA32_MCG_EXT_CTL,
        MSR_IA32_SMBASE,
        MSR_SMI_COUNT,
        MSR_PLATFORM_INFO,
        MSR_MISC_FEATURES_ENABLES,
        MSR_AMD64_VIRT_SPEC_CTRL,
        MSR_IA32_POWER_CTL,
        MSR_IA32_UCODE_REV,

        /*
         * The following list leaves out MSRs whose values are determined
         * by arch/x86/kvm/vmx/nested.c based on CPUID or other MSRs.
         * We always support the "true" VMX control MSRs, even if the host
         * processor does not, so I am putting these registers here rather
         * than in msrs_to_save_all.
         */
        MSR_IA32_VMX_BASIC,
        MSR_IA32_VMX_TRUE_PINBASED_CTLS,
        MSR_IA32_VMX_TRUE_PROCBASED_CTLS,
        MSR_IA32_VMX_TRUE_EXIT_CTLS,
        MSR_IA32_VMX_TRUE_ENTRY_CTLS,
        MSR_IA32_VMX_MISC,
        MSR_IA32_VMX_CR0_FIXED0,
        MSR_IA32_VMX_CR4_FIXED0,
        MSR_IA32_VMX_VMCS_ENUM,
        MSR_IA32_VMX_PROCBASED_CTLS2,
        MSR_IA32_VMX_EPT_VPID_CAP,
        MSR_IA32_VMX_VMFUNC,

        MSR_K7_HWCR,
        MSR_KVM_POLL_CONTROL,
};

static u32 emulated_msrs[ARRAY_SIZE(emulated_msrs_all)];
static unsigned num_emulated_msrs;

/*
 * List of msr numbers which are used to expose MSR-based features that
 * can be used by a hypervisor to validate requested CPU features.
 */
static const u32 msr_based_features_all[] = {
        MSR_IA32_VMX_BASIC,
        MSR_IA32_VMX_TRUE_PINBASED_CTLS,
        MSR_IA32_VMX_PINBASED_CTLS,
        MSR_IA32_VMX_TRUE_PROCBASED_CTLS,
        MSR_IA32_VMX_PROCBASED_CTLS,
        MSR_IA32_VMX_TRUE_EXIT_CTLS,
        MSR_IA32_VMX_EXIT_CTLS,
        MSR_IA32_VMX_TRUE_ENTRY_CTLS,
        MSR_IA32_VMX_ENTRY_CTLS,
        MSR_IA32_VMX_MISC,
        MSR_IA32_VMX_CR0_FIXED0,
        MSR_IA32_VMX_CR0_FIXED1,
        MSR_IA32_VMX_CR4_FIXED0,
        MSR_IA32_VMX_CR4_FIXED1,
        MSR_IA32_VMX_VMCS_ENUM,
        MSR_IA32_VMX_PROCBASED_CTLS2,
        MSR_IA32_VMX_EPT_VPID_CAP,
        MSR_IA32_VMX_VMFUNC,

        MSR_F10H_DECFG,
        MSR_IA32_UCODE_REV,
        MSR_IA32_ARCH_CAPABILITIES,
        MSR_IA32_PERF_CAPABILITIES,
};

static u32 msr_based_features[ARRAY_SIZE(msr_based_features_all)];
static unsigned int num_msr_based_features;

static u64 kvm_get_arch_capabilities(void)
{
        u64 data = 0;

        if (boot_cpu_has(X86_FEATURE_ARCH_CAPABILITIES))
                rdmsrl(MSR_IA32_ARCH_CAPABILITIES, data);

        /*
         * If nx_huge_pages is enabled, KVM's shadow paging will ensure that
         * the nested hypervisor runs with NX huge pages.  If it is not,
         * L1 is anyway vulnerable to ITLB_MULTIHIT exploits from other
         * L1 guests, so it need not worry about its own (L2) guests.
         */
        data |= ARCH_CAP_PSCHANGE_MC_NO;

        /*
         * If we're doing cache flushes (either "always" or "cond")
         * we will do one whenever the guest does a vmlaunch/vmresume.
         * If an outer hypervisor is doing the cache flush for us
         * (VMENTER_L1D_FLUSH_NESTED_VM), we can safely pass that
         * capability to the guest too, and if EPT is disabled we're not
         * vulnerable.  Overall, only VMENTER_L1D_FLUSH_NEVER will
         * require a nested hypervisor to do a flush of its own.
         */
        if (l1tf_vmx_mitigation != VMENTER_L1D_FLUSH_NEVER)
                data |= ARCH_CAP_SKIP_VMENTRY_L1DFLUSH;

        if (!boot_cpu_has_bug(X86_BUG_CPU_MELTDOWN))
                data |= ARCH_CAP_RDCL_NO;
        if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
                data |= ARCH_CAP_SSB_NO;
        if (!boot_cpu_has_bug(X86_BUG_MDS))
                data |= ARCH_CAP_MDS_NO;

        if (!boot_cpu_has(X86_FEATURE_RTM)) {
                /*
                 * If RTM=0 because the kernel has disabled TSX, the host might
                 * have TAA_NO or TSX_CTRL.  Clear TAA_NO (the guest sees RTM=0
                 * and therefore knows that there cannot be TAA) but keep
                 * TSX_CTRL: some buggy userspaces leave it set on tsx=on hosts,
                 * and we want to allow migrating those guests to tsx=off hosts.
                 */
                data &= ~ARCH_CAP_TAA_NO;
        } else if (!boot_cpu_has_bug(X86_BUG_TAA)) {
                data |= ARCH_CAP_TAA_NO;
        } else {
                /*
                 * Nothing to do here; we emulate TSX_CTRL if present on the
                 * host so the guest can choose between disabling TSX or
                 * using VERW to clear CPU buffers.
                 */
        }

        return data;
}

static int kvm_get_msr_feature(struct kvm_msr_entry *msr)
{
        switch (msr->index) {
        case MSR_IA32_ARCH_CAPABILITIES:
                msr->data = kvm_get_arch_capabilities();
                break;
        case MSR_IA32_UCODE_REV:
                rdmsrl_safe(msr->index, &msr->data);
                break;
        default:
                return static_call(kvm_x86_get_msr_feature)(msr);
        }
        return 0;
}

static int do_get_msr_feature(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
{
        struct kvm_msr_entry msr;
        int r;

        msr.index = index;
        r = kvm_get_msr_feature(&msr);

        if (r == KVM_MSR_RET_INVALID) {
                /* Unconditionally clear the output for simplicity */
                *data = 0;
                if (kvm_msr_ignored_check(index, 0, false))
                        r = 0;
        }

        if (r)
                return r;

        *data = msr.data;

        return 0;
}

static bool __kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer)
{
        if (efer & EFER_FFXSR && !guest_cpuid_has(vcpu, X86_FEATURE_FXSR_OPT))
                return false;

        if (efer & EFER_SVME && !guest_cpuid_has(vcpu, X86_FEATURE_SVM))
                return false;

        if (efer & (EFER_LME | EFER_LMA) &&
            !guest_cpuid_has(vcpu, X86_FEATURE_LM))
                return false;

        if (efer & EFER_NX && !guest_cpuid_has(vcpu, X86_FEATURE_NX))
                return false;

        return true;

}
bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer)
{
        if (efer & efer_reserved_bits)
                return false;

        return __kvm_valid_efer(vcpu, efer);
}
EXPORT_SYMBOL_GPL(kvm_valid_efer);

static int set_efer(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
{
        u64 old_efer = vcpu->arch.efer;
        u64 efer = msr_info->data;
        int r;

        if (efer & efer_reserved_bits)
                return 1;

        if (!msr_info->host_initiated) {
                if (!__kvm_valid_efer(vcpu, efer))
                        return 1;

                if (is_paging(vcpu) &&
                    (vcpu->arch.efer & EFER_LME) != (efer & EFER_LME))
                        return 1;
        }

        efer &= ~EFER_LMA;
        efer |= vcpu->arch.efer & EFER_LMA;

        r = static_call(kvm_x86_set_efer)(vcpu, efer);
        if (r) {
                WARN_ON(r > 0);
                return r;
        }

        /* Update reserved bits */
        if ((efer ^ old_efer) & EFER_NX)
                kvm_mmu_reset_context(vcpu);

        return 0;
}

void kvm_enable_efer_bits(u64 mask)
{
       efer_reserved_bits &= ~mask;
}
EXPORT_SYMBOL_GPL(kvm_enable_efer_bits);

bool kvm_msr_allowed(struct kvm_vcpu *vcpu, u32 index, u32 type)
{
        struct kvm_x86_msr_filter *msr_filter;
        struct msr_bitmap_range *ranges;
        struct kvm *kvm = vcpu->kvm;
        bool allowed;
        int idx;
        u32 i;

        /* x2APIC MSRs do not support filtering. */
        if (index >= 0x800 && index <= 0x8ff)
                return true;

        idx = srcu_read_lock(&kvm->srcu);

        msr_filter = srcu_dereference(kvm->arch.msr_filter, &kvm->srcu);
        if (!msr_filter) {
                allowed = true;
                goto out;
        }

        allowed = msr_filter->default_allow;
        ranges = msr_filter->ranges;

        for (i = 0; i < msr_filter->count; i++) {
                u32 start = ranges[i].base;
                u32 end = start + ranges[i].nmsrs;
                u32 flags = ranges[i].flags;
                unsigned long *bitmap = ranges[i].bitmap;

                if ((index >= start) && (index < end) && (flags & type)) {
                        allowed = !!test_bit(index - start, bitmap);
                        break;
                }
        }

out:
        srcu_read_unlock(&kvm->srcu, idx);

        return allowed;
}
EXPORT_SYMBOL_GPL(kvm_msr_allowed);

/*
 * Write @data into the MSR specified by @index.  Select MSR specific fault
 * checks are bypassed if @host_initiated is %true.
 * Returns 0 on success, non-0 otherwise.
 * Assumes vcpu_load() was already called.
 */
static int __kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data,
                         bool host_initiated)
{
        struct msr_data msr;

        if (!host_initiated && !kvm_msr_allowed(vcpu, index, KVM_MSR_FILTER_WRITE))
                return KVM_MSR_RET_FILTERED;

        switch (index) {
        case MSR_FS_BASE:
        case MSR_GS_BASE:
        case MSR_KERNEL_GS_BASE:
        case MSR_CSTAR:
        case MSR_LSTAR:
                if (is_noncanonical_address(data, vcpu))
                        return 1;
                break;
        case MSR_IA32_SYSENTER_EIP:
        case MSR_IA32_SYSENTER_ESP:
                /*
                 * IA32_SYSENTER_ESP and IA32_SYSENTER_EIP cause #GP if
                 * non-canonical address is written on Intel but not on
                 * AMD (which ignores the top 32-bits, because it does
                 * not implement 64-bit SYSENTER).
                 *
                 * 64-bit code should hence be able to write a non-canonical
                 * value on AMD.  Making the address canonical ensures that
                 * vmentry does not fail on Intel after writing a non-canonical
                 * value, and that something deterministic happens if the guest
                 * invokes 64-bit SYSENTER.
                 */
                data = get_canonical(data, vcpu_virt_addr_bits(vcpu));
                break;
        case MSR_TSC_AUX:
                if (!kvm_is_supported_user_return_msr(MSR_TSC_AUX))
                        return 1;

                if (!host_initiated &&
                    !guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP) &&
                    !guest_cpuid_has(vcpu, X86_FEATURE_RDPID))
                        return 1;

                /*
                 * Per Intel's SDM, bits 63:32 are reserved, but AMD's APM has
                 * incomplete and conflicting architectural behavior.  Current
                 * AMD CPUs completely ignore bits 63:32, i.e. they aren't
                 * reserved and always read as zeros.  Enforce Intel's reserved
                 * bits check if and only if the guest CPU is Intel, and clear
                 * the bits in all other cases.  This ensures cross-vendor
                 * migration will provide consistent behavior for the guest.
                 */
                if (guest_cpuid_is_intel(vcpu) && (data >> 32) != 0)
                        return 1;

                data = (u32)data;
                break;
        }

        msr.data = data;
        msr.index = index;
        msr.host_initiated = host_initiated;

        return static_call(kvm_x86_set_msr)(vcpu, &msr);
}

static int kvm_set_msr_ignored_check(struct kvm_vcpu *vcpu,
                                     u32 index, u64 data, bool host_initiated)
{
        int ret = __kvm_set_msr(vcpu, index, data, host_initiated);

        if (ret == KVM_MSR_RET_INVALID)
                if (kvm_msr_ignored_check(index, data, true))
                        ret = 0;

        return ret;
}

/*
 * Read the MSR specified by @index into @data.  Select MSR specific fault
 * checks are bypassed if @host_initiated is %true.
 * Returns 0 on success, non-0 otherwise.
 * Assumes vcpu_load() was already called.
 */
int __kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data,
                  bool host_initiated)
{
        struct msr_data msr;
        int ret;

        if (!host_initiated && !kvm_msr_allowed(vcpu, index, KVM_MSR_FILTER_READ))
                return KVM_MSR_RET_FILTERED;

        switch (index) {
        case MSR_TSC_AUX:
                if (!kvm_is_supported_user_return_msr(MSR_TSC_AUX))
                        return 1;

                if (!host_initiated &&
                    !guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP) &&
                    !guest_cpuid_has(vcpu, X86_FEATURE_RDPID))
                        return 1;
                break;
        }

        msr.index = index;
        msr.host_initiated = host_initiated;

        ret = static_call(kvm_x86_get_msr)(vcpu, &msr);
        if (!ret)
                *data = msr.data;
        return ret;
}

static int kvm_get_msr_ignored_check(struct kvm_vcpu *vcpu,
                                     u32 index, u64 *data, bool host_initiated)
{
        int ret = __kvm_get_msr(vcpu, index, data, host_initiated);

        if (ret == KVM_MSR_RET_INVALID) {
                /* Unconditionally clear *data for simplicity */
                *data = 0;
                if (kvm_msr_ignored_check(index, 0, false))
                        ret = 0;
        }

        return ret;
}

int kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data)
{
        return kvm_get_msr_ignored_check(vcpu, index, data, false);
}
EXPORT_SYMBOL_GPL(kvm_get_msr);

int kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data)
{
        return kvm_set_msr_ignored_check(vcpu, index, data, false);
}
EXPORT_SYMBOL_GPL(kvm_set_msr);

static int complete_emulated_rdmsr(struct kvm_vcpu *vcpu)
{
        int err = vcpu->run->msr.error;
        if (!err) {
                kvm_rax_write(vcpu, (u32)vcpu->run->msr.data);
                kvm_rdx_write(vcpu, vcpu->run->msr.data >> 32);
        }

        return static_call(kvm_x86_complete_emulated_msr)(vcpu, err);
}

static int complete_emulated_wrmsr(struct kvm_vcpu *vcpu)
{
        return static_call(kvm_x86_complete_emulated_msr)(vcpu, vcpu->run->msr.error);
}

static u64 kvm_msr_reason(int r)
{
        switch (r) {
        case KVM_MSR_RET_INVALID:
                return KVM_MSR_EXIT_REASON_UNKNOWN;
        case KVM_MSR_RET_FILTERED:
                return KVM_MSR_EXIT_REASON_FILTER;
        default:
                return KVM_MSR_EXIT_REASON_INVAL;
        }
}

static int kvm_msr_user_space(struct kvm_vcpu *vcpu, u32 index,
                              u32 exit_reason, u64 data,
                              int (*completion)(struct kvm_vcpu *vcpu),
                              int r)
{
        u64 msr_reason = kvm_msr_reason(r);

        /* Check if the user wanted to know about this MSR fault */
        if (!(vcpu->kvm->arch.user_space_msr_mask & msr_reason))
                return 0;

        vcpu->run->exit_reason = exit_reason;
        vcpu->run->msr.error = 0;
        memset(vcpu->run->msr.pad, 0, sizeof(vcpu->run->msr.pad));
        vcpu->run->msr.reason = msr_reason;
        vcpu->run->msr.index = index;
        vcpu->run->msr.data = data;
        vcpu->arch.complete_userspace_io = completion;

        return 1;
}

static int kvm_get_msr_user_space(struct kvm_vcpu *vcpu, u32 index, int r)
{
        return kvm_msr_user_space(vcpu, index, KVM_EXIT_X86_RDMSR, 0,
                                   complete_emulated_rdmsr, r);
}

static int kvm_set_msr_user_space(struct kvm_vcpu *vcpu, u32 index, u64 data, int r)
{
        return kvm_msr_user_space(vcpu, index, KVM_EXIT_X86_WRMSR, data,
                                   complete_emulated_wrmsr, r);
}

int kvm_emulate_rdmsr(struct kvm_vcpu *vcpu)
{
        u32 ecx = kvm_rcx_read(vcpu);
        u64 data;
        int r;

        r = kvm_get_msr(vcpu, ecx, &data);

        /* MSR read failed? See if we should ask user space */
        if (r && kvm_get_msr_user_space(vcpu, ecx, r)) {
                /* Bounce to user space */
                return 0;
        }

        if (!r) {
                trace_kvm_msr_read(ecx, data);

                kvm_rax_write(vcpu, data & -1u);
                kvm_rdx_write(vcpu, (data >> 32) & -1u);
        } else {
                trace_kvm_msr_read_ex(ecx);
        }

        return static_call(kvm_x86_complete_emulated_msr)(vcpu, r);
}
EXPORT_SYMBOL_GPL(kvm_emulate_rdmsr);

int kvm_emulate_wrmsr(struct kvm_vcpu *vcpu)
{
        u32 ecx = kvm_rcx_read(vcpu);
        u64 data = kvm_read_edx_eax(vcpu);
        int r;

        r = kvm_set_msr(vcpu, ecx, data);

        /* MSR write failed? See if we should ask user space */
        if (r && kvm_set_msr_user_space(vcpu, ecx, data, r))
                /* Bounce to user space */
                return 0;

        /* Signal all other negative errors to userspace */
        if (r < 0)
                return r;

        if (!r)
                trace_kvm_msr_write(ecx, data);
        else
                trace_kvm_msr_write_ex(ecx, data);

        return static_call(kvm_x86_complete_emulated_msr)(vcpu, r);
}
EXPORT_SYMBOL_GPL(kvm_emulate_wrmsr);

int kvm_emulate_as_nop(struct kvm_vcpu *vcpu)
{
        return kvm_skip_emulated_instruction(vcpu);
}
EXPORT_SYMBOL_GPL(kvm_emulate_as_nop);

int kvm_emulate_invd(struct kvm_vcpu *vcpu)
{
        /* Treat an INVD instruction as a NOP and just skip it. */
        return kvm_emulate_as_nop(vcpu);
}
EXPORT_SYMBOL_GPL(kvm_emulate_invd);

int kvm_emulate_mwait(struct kvm_vcpu *vcpu)
{
        pr_warn_once("kvm: MWAIT instruction emulated as NOP!\n");
        return kvm_emulate_as_nop(vcpu);
}
EXPORT_SYMBOL_GPL(kvm_emulate_mwait);

int kvm_handle_invalid_op(struct kvm_vcpu *vcpu)
{
        kvm_queue_exception(vcpu, UD_VECTOR);
        return 1;
}
EXPORT_SYMBOL_GPL(kvm_handle_invalid_op);

int kvm_emulate_monitor(struct kvm_vcpu *vcpu)
{
        pr_warn_once("kvm: MONITOR instruction emulated as NOP!\n");
        return kvm_emulate_as_nop(vcpu);
}
EXPORT_SYMBOL_GPL(kvm_emulate_monitor);

static inline bool kvm_vcpu_exit_request(struct kvm_vcpu *vcpu)
{
        xfer_to_guest_mode_prepare();
        return vcpu->mode == EXITING_GUEST_MODE || kvm_request_pending(vcpu) ||
                xfer_to_guest_mode_work_pending();
}

/*
 * The fast path for frequent and performance sensitive wrmsr emulation,
 * i.e. the sending of IPI, sending IPI early in the VM-Exit flow reduces
 * the latency of virtual IPI by avoiding the expensive bits of transitioning
 * from guest to host, e.g. reacquiring KVM's SRCU lock. In contrast to the
 * other cases which must be called after interrupts are enabled on the host.
 */
static int handle_fastpath_set_x2apic_icr_irqoff(struct kvm_vcpu *vcpu, u64 data)
{
        if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(vcpu->arch.apic))
                return 1;

        if (((data & APIC_SHORT_MASK) == APIC_DEST_NOSHORT) &&
                ((data & APIC_DEST_MASK) == APIC_DEST_PHYSICAL) &&
                ((data & APIC_MODE_MASK) == APIC_DM_FIXED) &&
                ((u32)(data >> 32) != X2APIC_BROADCAST)) {

                data &= ~(1 << 12);
                kvm_apic_send_ipi(vcpu->arch.apic, (u32)data, (u32)(data >> 32));
                kvm_lapic_set_reg(vcpu->arch.apic, APIC_ICR2, (u32)(data >> 32));
                kvm_lapic_set_reg(vcpu->arch.apic, APIC_ICR, (u32)data);
                trace_kvm_apic_write(APIC_ICR, (u32)data);
                return 0;
        }

        return 1;
}

static int handle_fastpath_set_tscdeadline(struct kvm_vcpu *vcpu, u64 data)
{
        if (!kvm_can_use_hv_timer(vcpu))
                return 1;

        kvm_set_lapic_tscdeadline_msr(vcpu, data);
        return 0;
}

fastpath_t handle_fastpath_set_msr_irqoff(struct kvm_vcpu *vcpu)
{
        u32 msr = kvm_rcx_read(vcpu);
        u64 data;
        fastpath_t ret = EXIT_FASTPATH_NONE;

        switch (msr) {
        case APIC_BASE_MSR + (APIC_ICR >> 4):
                data = kvm_read_edx_eax(vcpu);
                if (!handle_fastpath_set_x2apic_icr_irqoff(vcpu, data)) {
                        kvm_skip_emulated_instruction(vcpu);
                        ret = EXIT_FASTPATH_EXIT_HANDLED;
                }
                break;
        case MSR_IA32_TSC_DEADLINE:
                data = kvm_read_edx_eax(vcpu);
                if (!handle_fastpath_set_tscdeadline(vcpu, data)) {
                        kvm_skip_emulated_instruction(vcpu);
                        ret = EXIT_FASTPATH_REENTER_GUEST;
                }
                break;
        default:
                break;