linux/include/linux/kvm_host.h
<<
>>
Prefs
   1#ifndef __KVM_HOST_H
   2#define __KVM_HOST_H
   3
   4/*
   5 * This work is licensed under the terms of the GNU GPL, version 2.  See
   6 * the COPYING file in the top-level directory.
   7 */
   8
   9#include <linux/types.h>
  10#include <linux/hardirq.h>
  11#include <linux/list.h>
  12#include <linux/mutex.h>
  13#include <linux/spinlock.h>
  14#include <linux/signal.h>
  15#include <linux/sched.h>
  16#include <linux/bug.h>
  17#include <linux/mm.h>
  18#include <linux/mmu_notifier.h>
  19#include <linux/preempt.h>
  20#include <linux/msi.h>
  21#include <linux/slab.h>
  22#include <linux/rcupdate.h>
  23#include <linux/ratelimit.h>
  24#include <linux/err.h>
  25#include <asm/signal.h>
  26
  27#include <linux/kvm.h>
  28#include <linux/kvm_para.h>
  29
  30#include <linux/kvm_types.h>
  31
  32#include <asm/kvm_host.h>
  33
  34#ifndef KVM_MMIO_SIZE
  35#define KVM_MMIO_SIZE 8
  36#endif
  37
  38/*
  39 * The bit 16 ~ bit 31 of kvm_memory_region::flags are internally used
  40 * in kvm, other bits are visible for userspace which are defined in
  41 * include/linux/kvm_h.
  42 */
  43#define KVM_MEMSLOT_INVALID     (1UL << 16)
  44
  45/* Two fragments for cross MMIO pages. */
  46#define KVM_MAX_MMIO_FRAGMENTS  2
  47
  48/*
  49 * For the normal pfn, the highest 12 bits should be zero,
  50 * so we can mask these bits to indicate the error.
  51 */
  52#define KVM_PFN_ERR_MASK        (0xfffULL << 52)
  53
  54#define KVM_PFN_ERR_FAULT       (KVM_PFN_ERR_MASK)
  55#define KVM_PFN_ERR_HWPOISON    (KVM_PFN_ERR_MASK + 1)
  56#define KVM_PFN_ERR_BAD         (KVM_PFN_ERR_MASK + 2)
  57#define KVM_PFN_ERR_RO_FAULT    (KVM_PFN_ERR_MASK + 3)
  58
  59static inline bool is_error_pfn(pfn_t pfn)
  60{
  61        return !!(pfn & KVM_PFN_ERR_MASK);
  62}
  63
  64static inline bool is_noslot_pfn(pfn_t pfn)
  65{
  66        return pfn == KVM_PFN_ERR_BAD;
  67}
  68
  69static inline bool is_invalid_pfn(pfn_t pfn)
  70{
  71        return !is_noslot_pfn(pfn) && is_error_pfn(pfn);
  72}
  73
  74#define KVM_HVA_ERR_BAD         (PAGE_OFFSET)
  75#define KVM_HVA_ERR_RO_BAD      (PAGE_OFFSET + PAGE_SIZE)
  76
  77static inline bool kvm_is_error_hva(unsigned long addr)
  78{
  79        return addr >= PAGE_OFFSET;
  80}
  81
  82#define KVM_ERR_PTR_BAD_PAGE    (ERR_PTR(-ENOENT))
  83
  84static inline bool is_error_page(struct page *page)
  85{
  86        return IS_ERR(page);
  87}
  88
  89/*
  90 * vcpu->requests bit members
  91 */
  92#define KVM_REQ_TLB_FLUSH          0
  93#define KVM_REQ_MIGRATE_TIMER      1
  94#define KVM_REQ_REPORT_TPR_ACCESS  2
  95#define KVM_REQ_MMU_RELOAD         3
  96#define KVM_REQ_TRIPLE_FAULT       4
  97#define KVM_REQ_PENDING_TIMER      5
  98#define KVM_REQ_UNHALT             6
  99#define KVM_REQ_MMU_SYNC           7
 100#define KVM_REQ_CLOCK_UPDATE       8
 101#define KVM_REQ_KICK               9
 102#define KVM_REQ_DEACTIVATE_FPU    10
 103#define KVM_REQ_EVENT             11
 104#define KVM_REQ_APF_HALT          12
 105#define KVM_REQ_STEAL_UPDATE      13
 106#define KVM_REQ_NMI               14
 107#define KVM_REQ_IMMEDIATE_EXIT    15
 108#define KVM_REQ_PMU               16
 109#define KVM_REQ_PMI               17
 110
 111#define KVM_USERSPACE_IRQ_SOURCE_ID             0
 112#define KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID        1
 113
 114struct kvm;
 115struct kvm_vcpu;
 116extern struct kmem_cache *kvm_vcpu_cache;
 117
 118struct kvm_io_range {
 119        gpa_t addr;
 120        int len;
 121        struct kvm_io_device *dev;
 122};
 123
 124#define NR_IOBUS_DEVS 1000
 125
 126struct kvm_io_bus {
 127        int                   dev_count;
 128        struct kvm_io_range range[];
 129};
 130
 131enum kvm_bus {
 132        KVM_MMIO_BUS,
 133        KVM_PIO_BUS,
 134        KVM_NR_BUSES
 135};
 136
 137int kvm_io_bus_write(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
 138                     int len, const void *val);
 139int kvm_io_bus_read(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, int len,
 140                    void *val);
 141int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
 142                            int len, struct kvm_io_device *dev);
 143int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
 144                              struct kvm_io_device *dev);
 145
 146#ifdef CONFIG_KVM_ASYNC_PF
 147struct kvm_async_pf {
 148        struct work_struct work;
 149        struct list_head link;
 150        struct list_head queue;
 151        struct kvm_vcpu *vcpu;
 152        struct mm_struct *mm;
 153        gva_t gva;
 154        unsigned long addr;
 155        struct kvm_arch_async_pf arch;
 156        struct page *page;
 157        bool done;
 158};
 159
 160void kvm_clear_async_pf_completion_queue(struct kvm_vcpu *vcpu);
 161void kvm_check_async_pf_completion(struct kvm_vcpu *vcpu);
 162int kvm_setup_async_pf(struct kvm_vcpu *vcpu, gva_t gva, gfn_t gfn,
 163                       struct kvm_arch_async_pf *arch);
 164int kvm_async_pf_wakeup_all(struct kvm_vcpu *vcpu);
 165#endif
 166
 167enum {
 168        OUTSIDE_GUEST_MODE,
 169        IN_GUEST_MODE,
 170        EXITING_GUEST_MODE,
 171        READING_SHADOW_PAGE_TABLES,
 172};
 173
 174/*
 175 * Sometimes a large or cross-page mmio needs to be broken up into separate
 176 * exits for userspace servicing.
 177 */
 178struct kvm_mmio_fragment {
 179        gpa_t gpa;
 180        void *data;
 181        unsigned len;
 182};
 183
 184struct kvm_vcpu {
 185        struct kvm *kvm;
 186#ifdef CONFIG_PREEMPT_NOTIFIERS
 187        struct preempt_notifier preempt_notifier;
 188#endif
 189        int cpu;
 190        int vcpu_id;
 191        int srcu_idx;
 192        int mode;
 193        unsigned long requests;
 194        unsigned long guest_debug;
 195
 196        struct mutex mutex;
 197        struct kvm_run *run;
 198
 199        int fpu_active;
 200        int guest_fpu_loaded, guest_xcr0_loaded;
 201        wait_queue_head_t wq;
 202        struct pid *pid;
 203        int sigset_active;
 204        sigset_t sigset;
 205        struct kvm_vcpu_stat stat;
 206
 207#ifdef CONFIG_HAS_IOMEM
 208        int mmio_needed;
 209        int mmio_read_completed;
 210        int mmio_is_write;
 211        int mmio_cur_fragment;
 212        int mmio_nr_fragments;
 213        struct kvm_mmio_fragment mmio_fragments[KVM_MAX_MMIO_FRAGMENTS];
 214#endif
 215
 216#ifdef CONFIG_KVM_ASYNC_PF
 217        struct {
 218                u32 queued;
 219                struct list_head queue;
 220                struct list_head done;
 221                spinlock_t lock;
 222        } async_pf;
 223#endif
 224
 225#ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT
 226        /*
 227         * Cpu relax intercept or pause loop exit optimization
 228         * in_spin_loop: set when a vcpu does a pause loop exit
 229         *  or cpu relax intercepted.
 230         * dy_eligible: indicates whether vcpu is eligible for directed yield.
 231         */
 232        struct {
 233                bool in_spin_loop;
 234                bool dy_eligible;
 235        } spin_loop;
 236#endif
 237        struct kvm_vcpu_arch arch;
 238};
 239
 240static inline int kvm_vcpu_exiting_guest_mode(struct kvm_vcpu *vcpu)
 241{
 242        return cmpxchg(&vcpu->mode, IN_GUEST_MODE, EXITING_GUEST_MODE);
 243}
 244
 245/*
 246 * Some of the bitops functions do not support too long bitmaps.
 247 * This number must be determined not to exceed such limits.
 248 */
 249#define KVM_MEM_MAX_NR_PAGES ((1UL << 31) - 1)
 250
 251struct kvm_memory_slot {
 252        gfn_t base_gfn;
 253        unsigned long npages;
 254        unsigned long flags;
 255        unsigned long *dirty_bitmap;
 256        struct kvm_arch_memory_slot arch;
 257        unsigned long userspace_addr;
 258        int user_alloc;
 259        int id;
 260};
 261
 262static inline unsigned long kvm_dirty_bitmap_bytes(struct kvm_memory_slot *memslot)
 263{
 264        return ALIGN(memslot->npages, BITS_PER_LONG) / 8;
 265}
 266
 267struct kvm_kernel_irq_routing_entry {
 268        u32 gsi;
 269        u32 type;
 270        int (*set)(struct kvm_kernel_irq_routing_entry *e,
 271                   struct kvm *kvm, int irq_source_id, int level);
 272        union {
 273                struct {
 274                        unsigned irqchip;
 275                        unsigned pin;
 276                } irqchip;
 277                struct msi_msg msi;
 278        };
 279        struct hlist_node link;
 280};
 281
 282#ifdef __KVM_HAVE_IOAPIC
 283
 284struct kvm_irq_routing_table {
 285        int chip[KVM_NR_IRQCHIPS][KVM_IOAPIC_NUM_PINS];
 286        struct kvm_kernel_irq_routing_entry *rt_entries;
 287        u32 nr_rt_entries;
 288        /*
 289         * Array indexed by gsi. Each entry contains list of irq chips
 290         * the gsi is connected to.
 291         */
 292        struct hlist_head map[0];
 293};
 294
 295#else
 296
 297struct kvm_irq_routing_table {};
 298
 299#endif
 300
 301#ifndef KVM_MEM_SLOTS_NUM
 302#define KVM_MEM_SLOTS_NUM (KVM_MEMORY_SLOTS + KVM_PRIVATE_MEM_SLOTS)
 303#endif
 304
 305/*
 306 * Note:
 307 * memslots are not sorted by id anymore, please use id_to_memslot()
 308 * to get the memslot by its id.
 309 */
 310struct kvm_memslots {
 311        u64 generation;
 312        struct kvm_memory_slot memslots[KVM_MEM_SLOTS_NUM];
 313        /* The mapping table from slot id to the index in memslots[]. */
 314        int id_to_index[KVM_MEM_SLOTS_NUM];
 315};
 316
 317struct kvm {
 318        spinlock_t mmu_lock;
 319        struct mutex slots_lock;
 320        struct mm_struct *mm; /* userspace tied to this vm */
 321        struct kvm_memslots *memslots;
 322        struct srcu_struct srcu;
 323#ifdef CONFIG_KVM_APIC_ARCHITECTURE
 324        u32 bsp_vcpu_id;
 325#endif
 326        struct kvm_vcpu *vcpus[KVM_MAX_VCPUS];
 327        atomic_t online_vcpus;
 328        int last_boosted_vcpu;
 329        struct list_head vm_list;
 330        struct mutex lock;
 331        struct kvm_io_bus *buses[KVM_NR_BUSES];
 332#ifdef CONFIG_HAVE_KVM_EVENTFD
 333        struct {
 334                spinlock_t        lock;
 335                struct list_head  items;
 336                struct list_head  resampler_list;
 337                struct mutex      resampler_lock;
 338        } irqfds;
 339        struct list_head ioeventfds;
 340#endif
 341        struct kvm_vm_stat stat;
 342        struct kvm_arch arch;
 343        atomic_t users_count;
 344#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
 345        struct kvm_coalesced_mmio_ring *coalesced_mmio_ring;
 346        spinlock_t ring_lock;
 347        struct list_head coalesced_zones;
 348#endif
 349
 350        struct mutex irq_lock;
 351#ifdef CONFIG_HAVE_KVM_IRQCHIP
 352        /*
 353         * Update side is protected by irq_lock and,
 354         * if configured, irqfds.lock.
 355         */
 356        struct kvm_irq_routing_table __rcu *irq_routing;
 357        struct hlist_head mask_notifier_list;
 358        struct hlist_head irq_ack_notifier_list;
 359#endif
 360
 361#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
 362        struct mmu_notifier mmu_notifier;
 363        unsigned long mmu_notifier_seq;
 364        long mmu_notifier_count;
 365#endif
 366        long tlbs_dirty;
 367};
 368
 369#define kvm_err(fmt, ...) \
 370        pr_err("kvm [%i]: " fmt, task_pid_nr(current), ## __VA_ARGS__)
 371#define kvm_info(fmt, ...) \
 372        pr_info("kvm [%i]: " fmt, task_pid_nr(current), ## __VA_ARGS__)
 373#define kvm_debug(fmt, ...) \
 374        pr_debug("kvm [%i]: " fmt, task_pid_nr(current), ## __VA_ARGS__)
 375#define kvm_pr_unimpl(fmt, ...) \
 376        pr_err_ratelimited("kvm [%i]: " fmt, \
 377                           task_tgid_nr(current), ## __VA_ARGS__)
 378
 379/* The guest did something we don't support. */
 380#define vcpu_unimpl(vcpu, fmt, ...)                                     \
 381        kvm_pr_unimpl("vcpu%i " fmt, (vcpu)->vcpu_id, ## __VA_ARGS__)
 382
 383static inline struct kvm_vcpu *kvm_get_vcpu(struct kvm *kvm, int i)
 384{
 385        smp_rmb();
 386        return kvm->vcpus[i];
 387}
 388
 389#define kvm_for_each_vcpu(idx, vcpup, kvm) \
 390        for (idx = 0; \
 391             idx < atomic_read(&kvm->online_vcpus) && \
 392             (vcpup = kvm_get_vcpu(kvm, idx)) != NULL; \
 393             idx++)
 394
 395#define kvm_for_each_memslot(memslot, slots)    \
 396        for (memslot = &slots->memslots[0];     \
 397              memslot < slots->memslots + KVM_MEM_SLOTS_NUM && memslot->npages;\
 398                memslot++)
 399
 400int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id);
 401void kvm_vcpu_uninit(struct kvm_vcpu *vcpu);
 402
 403int __must_check vcpu_load(struct kvm_vcpu *vcpu);
 404void vcpu_put(struct kvm_vcpu *vcpu);
 405
 406int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align,
 407                  struct module *module);
 408void kvm_exit(void);
 409
 410void kvm_get_kvm(struct kvm *kvm);
 411void kvm_put_kvm(struct kvm *kvm);
 412void update_memslots(struct kvm_memslots *slots, struct kvm_memory_slot *new);
 413
 414static inline struct kvm_memslots *kvm_memslots(struct kvm *kvm)
 415{
 416        return rcu_dereference_check(kvm->memslots,
 417                        srcu_read_lock_held(&kvm->srcu)
 418                        || lockdep_is_held(&kvm->slots_lock));
 419}
 420
 421static inline struct kvm_memory_slot *
 422id_to_memslot(struct kvm_memslots *slots, int id)
 423{
 424        int index = slots->id_to_index[id];
 425        struct kvm_memory_slot *slot;
 426
 427        slot = &slots->memslots[index];
 428
 429        WARN_ON(slot->id != id);
 430        return slot;
 431}
 432
 433int kvm_set_memory_region(struct kvm *kvm,
 434                          struct kvm_userspace_memory_region *mem,
 435                          int user_alloc);
 436int __kvm_set_memory_region(struct kvm *kvm,
 437                            struct kvm_userspace_memory_region *mem,
 438                            int user_alloc);
 439void kvm_arch_free_memslot(struct kvm_memory_slot *free,
 440                           struct kvm_memory_slot *dont);
 441int kvm_arch_create_memslot(struct kvm_memory_slot *slot, unsigned long npages);
 442int kvm_arch_prepare_memory_region(struct kvm *kvm,
 443                                struct kvm_memory_slot *memslot,
 444                                struct kvm_memory_slot old,
 445                                struct kvm_userspace_memory_region *mem,
 446                                int user_alloc);
 447void kvm_arch_commit_memory_region(struct kvm *kvm,
 448                                struct kvm_userspace_memory_region *mem,
 449                                struct kvm_memory_slot old,
 450                                int user_alloc);
 451bool kvm_largepages_enabled(void);
 452void kvm_disable_largepages(void);
 453/* flush all memory translations */
 454void kvm_arch_flush_shadow_all(struct kvm *kvm);
 455/* flush memory translations pointing to 'slot' */
 456void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
 457                                   struct kvm_memory_slot *slot);
 458
 459int gfn_to_page_many_atomic(struct kvm *kvm, gfn_t gfn, struct page **pages,
 460                            int nr_pages);
 461
 462struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn);
 463unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn);
 464unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot, gfn_t gfn);
 465void kvm_release_page_clean(struct page *page);
 466void kvm_release_page_dirty(struct page *page);
 467void kvm_set_page_dirty(struct page *page);
 468void kvm_set_page_accessed(struct page *page);
 469
 470pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn);
 471pfn_t gfn_to_pfn_async(struct kvm *kvm, gfn_t gfn, bool *async,
 472                       bool write_fault, bool *writable);
 473pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn);
 474pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault,
 475                      bool *writable);
 476pfn_t gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn);
 477pfn_t gfn_to_pfn_memslot_atomic(struct kvm_memory_slot *slot, gfn_t gfn);
 478
 479void kvm_release_pfn_dirty(pfn_t pfn);
 480void kvm_release_pfn_clean(pfn_t pfn);
 481void kvm_set_pfn_dirty(pfn_t pfn);
 482void kvm_set_pfn_accessed(pfn_t pfn);
 483void kvm_get_pfn(pfn_t pfn);
 484
 485int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
 486                        int len);
 487int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data,
 488                          unsigned long len);
 489int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len);
 490int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
 491                           void *data, unsigned long len);
 492int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data,
 493                         int offset, int len);
 494int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
 495                    unsigned long len);
 496int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
 497                           void *data, unsigned long len);
 498int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
 499                              gpa_t gpa);
 500int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len);
 501int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len);
 502struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn);
 503int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn);
 504unsigned long kvm_host_page_size(struct kvm *kvm, gfn_t gfn);
 505void mark_page_dirty(struct kvm *kvm, gfn_t gfn);
 506void mark_page_dirty_in_slot(struct kvm *kvm, struct kvm_memory_slot *memslot,
 507                             gfn_t gfn);
 508
 509void kvm_vcpu_block(struct kvm_vcpu *vcpu);
 510void kvm_vcpu_kick(struct kvm_vcpu *vcpu);
 511bool kvm_vcpu_yield_to(struct kvm_vcpu *target);
 512void kvm_vcpu_on_spin(struct kvm_vcpu *vcpu);
 513void kvm_resched(struct kvm_vcpu *vcpu);
 514void kvm_load_guest_fpu(struct kvm_vcpu *vcpu);
 515void kvm_put_guest_fpu(struct kvm_vcpu *vcpu);
 516
 517void kvm_flush_remote_tlbs(struct kvm *kvm);
 518void kvm_reload_remote_mmus(struct kvm *kvm);
 519
 520long kvm_arch_dev_ioctl(struct file *filp,
 521                        unsigned int ioctl, unsigned long arg);
 522long kvm_arch_vcpu_ioctl(struct file *filp,
 523                         unsigned int ioctl, unsigned long arg);
 524int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf);
 525
 526int kvm_dev_ioctl_check_extension(long ext);
 527
 528int kvm_get_dirty_log(struct kvm *kvm,
 529                        struct kvm_dirty_log *log, int *is_dirty);
 530int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
 531                                struct kvm_dirty_log *log);
 532
 533int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
 534                                   struct
 535                                   kvm_userspace_memory_region *mem,
 536                                   int user_alloc);
 537int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level);
 538long kvm_arch_vm_ioctl(struct file *filp,
 539                       unsigned int ioctl, unsigned long arg);
 540
 541int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu);
 542int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu);
 543
 544int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
 545                                    struct kvm_translation *tr);
 546
 547int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs);
 548int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs);
 549int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
 550                                  struct kvm_sregs *sregs);
 551int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
 552                                  struct kvm_sregs *sregs);
 553int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
 554                                    struct kvm_mp_state *mp_state);
 555int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
 556                                    struct kvm_mp_state *mp_state);
 557int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
 558                                        struct kvm_guest_debug *dbg);
 559int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run);
 560
 561int kvm_arch_init(void *opaque);
 562void kvm_arch_exit(void);
 563
 564int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu);
 565void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu);
 566
 567void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu);
 568void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu);
 569void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu);
 570struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, unsigned int id);
 571int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu);
 572void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu);
 573
 574int kvm_arch_vcpu_reset(struct kvm_vcpu *vcpu);
 575int kvm_arch_hardware_enable(void *garbage);
 576void kvm_arch_hardware_disable(void *garbage);
 577int kvm_arch_hardware_setup(void);
 578void kvm_arch_hardware_unsetup(void);
 579void kvm_arch_check_processor_compat(void *rtn);
 580int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu);
 581int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu);
 582
 583void kvm_free_physmem(struct kvm *kvm);
 584
 585void *kvm_kvzalloc(unsigned long size);
 586void kvm_kvfree(const void *addr);
 587
 588#ifndef __KVM_HAVE_ARCH_VM_ALLOC
 589static inline struct kvm *kvm_arch_alloc_vm(void)
 590{
 591        return kzalloc(sizeof(struct kvm), GFP_KERNEL);
 592}
 593
 594static inline void kvm_arch_free_vm(struct kvm *kvm)
 595{
 596        kfree(kvm);
 597}
 598#endif
 599
 600static inline wait_queue_head_t *kvm_arch_vcpu_wq(struct kvm_vcpu *vcpu)
 601{
 602#ifdef __KVM_HAVE_ARCH_WQP
 603        return vcpu->arch.wqp;
 604#else
 605        return &vcpu->wq;
 606#endif
 607}
 608
 609int kvm_arch_init_vm(struct kvm *kvm, unsigned long type);
 610void kvm_arch_destroy_vm(struct kvm *kvm);
 611void kvm_free_all_assigned_devices(struct kvm *kvm);
 612void kvm_arch_sync_events(struct kvm *kvm);
 613
 614int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu);
 615void kvm_vcpu_kick(struct kvm_vcpu *vcpu);
 616
 617bool kvm_is_mmio_pfn(pfn_t pfn);
 618
 619struct kvm_irq_ack_notifier {
 620        struct hlist_node link;
 621        unsigned gsi;
 622        void (*irq_acked)(struct kvm_irq_ack_notifier *kian);
 623};
 624
 625struct kvm_assigned_dev_kernel {
 626        struct kvm_irq_ack_notifier ack_notifier;
 627        struct list_head list;
 628        int assigned_dev_id;
 629        int host_segnr;
 630        int host_busnr;
 631        int host_devfn;
 632        unsigned int entries_nr;
 633        int host_irq;
 634        bool host_irq_disabled;
 635        bool pci_2_3;
 636        struct msix_entry *host_msix_entries;
 637        int guest_irq;
 638        struct msix_entry *guest_msix_entries;
 639        unsigned long irq_requested_type;
 640        int irq_source_id;
 641        int flags;
 642        struct pci_dev *dev;
 643        struct kvm *kvm;
 644        spinlock_t intx_lock;
 645        spinlock_t intx_mask_lock;
 646        char irq_name[32];
 647        struct pci_saved_state *pci_saved_state;
 648};
 649
 650struct kvm_irq_mask_notifier {
 651        void (*func)(struct kvm_irq_mask_notifier *kimn, bool masked);
 652        int irq;
 653        struct hlist_node link;
 654};
 655
 656void kvm_register_irq_mask_notifier(struct kvm *kvm, int irq,
 657                                    struct kvm_irq_mask_notifier *kimn);
 658void kvm_unregister_irq_mask_notifier(struct kvm *kvm, int irq,
 659                                      struct kvm_irq_mask_notifier *kimn);
 660void kvm_fire_mask_notifiers(struct kvm *kvm, unsigned irqchip, unsigned pin,
 661                             bool mask);
 662
 663#ifdef __KVM_HAVE_IOAPIC
 664void kvm_get_intr_delivery_bitmask(struct kvm_ioapic *ioapic,
 665                                   union kvm_ioapic_redirect_entry *entry,
 666                                   unsigned long *deliver_bitmask);
 667#endif
 668int kvm_set_irq(struct kvm *kvm, int irq_source_id, u32 irq, int level);
 669int kvm_set_msi(struct kvm_kernel_irq_routing_entry *irq_entry, struct kvm *kvm,
 670                int irq_source_id, int level);
 671void kvm_notify_acked_irq(struct kvm *kvm, unsigned irqchip, unsigned pin);
 672void kvm_register_irq_ack_notifier(struct kvm *kvm,
 673                                   struct kvm_irq_ack_notifier *kian);
 674void kvm_unregister_irq_ack_notifier(struct kvm *kvm,
 675                                   struct kvm_irq_ack_notifier *kian);
 676int kvm_request_irq_source_id(struct kvm *kvm);
 677void kvm_free_irq_source_id(struct kvm *kvm, int irq_source_id);
 678
 679/* For vcpu->arch.iommu_flags */
 680#define KVM_IOMMU_CACHE_COHERENCY       0x1
 681
 682#ifdef CONFIG_IOMMU_API
 683int kvm_iommu_map_pages(struct kvm *kvm, struct kvm_memory_slot *slot);
 684void kvm_iommu_unmap_pages(struct kvm *kvm, struct kvm_memory_slot *slot);
 685int kvm_iommu_map_guest(struct kvm *kvm);
 686int kvm_iommu_unmap_guest(struct kvm *kvm);
 687int kvm_assign_device(struct kvm *kvm,
 688                      struct kvm_assigned_dev_kernel *assigned_dev);
 689int kvm_deassign_device(struct kvm *kvm,
 690                        struct kvm_assigned_dev_kernel *assigned_dev);
 691#else /* CONFIG_IOMMU_API */
 692static inline int kvm_iommu_map_pages(struct kvm *kvm,
 693                                      struct kvm_memory_slot *slot)
 694{
 695        return 0;
 696}
 697
 698static inline void kvm_iommu_unmap_pages(struct kvm *kvm,
 699                                         struct kvm_memory_slot *slot)
 700{
 701}
 702
 703static inline int kvm_iommu_map_guest(struct kvm *kvm)
 704{
 705        return -ENODEV;
 706}
 707
 708static inline int kvm_iommu_unmap_guest(struct kvm *kvm)
 709{
 710        return 0;
 711}
 712
 713static inline int kvm_assign_device(struct kvm *kvm,
 714                struct kvm_assigned_dev_kernel *assigned_dev)
 715{
 716        return 0;
 717}
 718
 719static inline int kvm_deassign_device(struct kvm *kvm,
 720                struct kvm_assigned_dev_kernel *assigned_dev)
 721{
 722        return 0;
 723}
 724#endif /* CONFIG_IOMMU_API */
 725
 726static inline void kvm_guest_enter(void)
 727{
 728        BUG_ON(preemptible());
 729        vtime_account(current);
 730        current->flags |= PF_VCPU;
 731        /* KVM does not hold any references to rcu protected data when it
 732         * switches CPU into a guest mode. In fact switching to a guest mode
 733         * is very similar to exiting to userspase from rcu point of view. In
 734         * addition CPU may stay in a guest mode for quite a long time (up to
 735         * one time slice). Lets treat guest mode as quiescent state, just like
 736         * we do with user-mode execution.
 737         */
 738        rcu_virt_note_context_switch(smp_processor_id());
 739}
 740
 741static inline void kvm_guest_exit(void)
 742{
 743        vtime_account(current);
 744        current->flags &= ~PF_VCPU;
 745}
 746
 747/*
 748 * search_memslots() and __gfn_to_memslot() are here because they are
 749 * used in non-modular code in arch/powerpc/kvm/book3s_hv_rm_mmu.c.
 750 * gfn_to_memslot() itself isn't here as an inline because that would
 751 * bloat other code too much.
 752 */
 753static inline struct kvm_memory_slot *
 754search_memslots(struct kvm_memslots *slots, gfn_t gfn)
 755{
 756        struct kvm_memory_slot *memslot;
 757
 758        kvm_for_each_memslot(memslot, slots)
 759                if (gfn >= memslot->base_gfn &&
 760                      gfn < memslot->base_gfn + memslot->npages)
 761                        return memslot;
 762
 763        return NULL;
 764}
 765
 766static inline struct kvm_memory_slot *
 767__gfn_to_memslot(struct kvm_memslots *slots, gfn_t gfn)
 768{
 769        return search_memslots(slots, gfn);
 770}
 771
 772static inline unsigned long
 773__gfn_to_hva_memslot(struct kvm_memory_slot *slot, gfn_t gfn)
 774{
 775        return slot->userspace_addr + (gfn - slot->base_gfn) * PAGE_SIZE;
 776}
 777
 778static inline int memslot_id(struct kvm *kvm, gfn_t gfn)
 779{
 780        return gfn_to_memslot(kvm, gfn)->id;
 781}
 782
 783static inline gfn_t gfn_to_index(gfn_t gfn, gfn_t base_gfn, int level)
 784{
 785        /* KVM_HPAGE_GFN_SHIFT(PT_PAGE_TABLE_LEVEL) must be 0. */
 786        return (gfn >> KVM_HPAGE_GFN_SHIFT(level)) -
 787                (base_gfn >> KVM_HPAGE_GFN_SHIFT(level));
 788}
 789
 790static inline gfn_t
 791hva_to_gfn_memslot(unsigned long hva, struct kvm_memory_slot *slot)
 792{
 793        gfn_t gfn_offset = (hva - slot->userspace_addr) >> PAGE_SHIFT;
 794
 795        return slot->base_gfn + gfn_offset;
 796}
 797
 798static inline gpa_t gfn_to_gpa(gfn_t gfn)
 799{
 800        return (gpa_t)gfn << PAGE_SHIFT;
 801}
 802
 803static inline gfn_t gpa_to_gfn(gpa_t gpa)
 804{
 805        return (gfn_t)(gpa >> PAGE_SHIFT);
 806}
 807
 808static inline hpa_t pfn_to_hpa(pfn_t pfn)
 809{
 810        return (hpa_t)pfn << PAGE_SHIFT;
 811}
 812
 813static inline void kvm_migrate_timers(struct kvm_vcpu *vcpu)
 814{
 815        set_bit(KVM_REQ_MIGRATE_TIMER, &vcpu->requests);
 816}
 817
 818enum kvm_stat_kind {
 819        KVM_STAT_VM,
 820        KVM_STAT_VCPU,
 821};
 822
 823struct kvm_stats_debugfs_item {
 824        const char *name;
 825        int offset;
 826        enum kvm_stat_kind kind;
 827        struct dentry *dentry;
 828};
 829extern struct kvm_stats_debugfs_item debugfs_entries[];
 830extern struct dentry *kvm_debugfs_dir;
 831
 832#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
 833static inline int mmu_notifier_retry(struct kvm_vcpu *vcpu, unsigned long mmu_seq)
 834{
 835        if (unlikely(vcpu->kvm->mmu_notifier_count))
 836                return 1;
 837        /*
 838         * Ensure the read of mmu_notifier_count happens before the read
 839         * of mmu_notifier_seq.  This interacts with the smp_wmb() in
 840         * mmu_notifier_invalidate_range_end to make sure that the caller
 841         * either sees the old (non-zero) value of mmu_notifier_count or
 842         * the new (incremented) value of mmu_notifier_seq.
 843         * PowerPC Book3s HV KVM calls this under a per-page lock
 844         * rather than under kvm->mmu_lock, for scalability, so
 845         * can't rely on kvm->mmu_lock to keep things ordered.
 846         */
 847        smp_rmb();
 848        if (vcpu->kvm->mmu_notifier_seq != mmu_seq)
 849                return 1;
 850        return 0;
 851}
 852#endif
 853
 854#ifdef KVM_CAP_IRQ_ROUTING
 855
 856#define KVM_MAX_IRQ_ROUTES 1024
 857
 858int kvm_setup_default_irq_routing(struct kvm *kvm);
 859int kvm_set_irq_routing(struct kvm *kvm,
 860                        const struct kvm_irq_routing_entry *entries,
 861                        unsigned nr,
 862                        unsigned flags);
 863void kvm_free_irq_routing(struct kvm *kvm);
 864
 865int kvm_send_userspace_msi(struct kvm *kvm, struct kvm_msi *msi);
 866
 867#else
 868
 869static inline void kvm_free_irq_routing(struct kvm *kvm) {}
 870
 871#endif
 872
 873#ifdef CONFIG_HAVE_KVM_EVENTFD
 874
 875void kvm_eventfd_init(struct kvm *kvm);
 876int kvm_irqfd(struct kvm *kvm, struct kvm_irqfd *args);
 877void kvm_irqfd_release(struct kvm *kvm);
 878void kvm_irq_routing_update(struct kvm *, struct kvm_irq_routing_table *);
 879int kvm_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args);
 880
 881#else
 882
 883static inline void kvm_eventfd_init(struct kvm *kvm) {}
 884
 885static inline int kvm_irqfd(struct kvm *kvm, struct kvm_irqfd *args)
 886{
 887        return -EINVAL;
 888}
 889
 890static inline void kvm_irqfd_release(struct kvm *kvm) {}
 891
 892#ifdef CONFIG_HAVE_KVM_IRQCHIP
 893static inline void kvm_irq_routing_update(struct kvm *kvm,
 894                                          struct kvm_irq_routing_table *irq_rt)
 895{
 896        rcu_assign_pointer(kvm->irq_routing, irq_rt);
 897}
 898#endif
 899
 900static inline int kvm_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args)
 901{
 902        return -ENOSYS;
 903}
 904
 905#endif /* CONFIG_HAVE_KVM_EVENTFD */
 906
 907#ifdef CONFIG_KVM_APIC_ARCHITECTURE
 908static inline bool kvm_vcpu_is_bsp(struct kvm_vcpu *vcpu)
 909{
 910        return vcpu->kvm->bsp_vcpu_id == vcpu->vcpu_id;
 911}
 912
 913bool kvm_vcpu_compatible(struct kvm_vcpu *vcpu);
 914
 915#else
 916
 917static inline bool kvm_vcpu_compatible(struct kvm_vcpu *vcpu) { return true; }
 918
 919#endif
 920
 921#ifdef __KVM_HAVE_DEVICE_ASSIGNMENT
 922
 923long kvm_vm_ioctl_assigned_device(struct kvm *kvm, unsigned ioctl,
 924                                  unsigned long arg);
 925
 926#else
 927
 928static inline long kvm_vm_ioctl_assigned_device(struct kvm *kvm, unsigned ioctl,
 929                                                unsigned long arg)
 930{
 931        return -ENOTTY;
 932}
 933
 934#endif
 935
 936static inline void kvm_make_request(int req, struct kvm_vcpu *vcpu)
 937{
 938        set_bit(req, &vcpu->requests);
 939}
 940
 941static inline bool kvm_check_request(int req, struct kvm_vcpu *vcpu)
 942{
 943        if (test_bit(req, &vcpu->requests)) {
 944                clear_bit(req, &vcpu->requests);
 945                return true;
 946        } else {
 947                return false;
 948        }
 949}
 950
 951#ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT
 952
 953static inline void kvm_vcpu_set_in_spin_loop(struct kvm_vcpu *vcpu, bool val)
 954{
 955        vcpu->spin_loop.in_spin_loop = val;
 956}
 957static inline void kvm_vcpu_set_dy_eligible(struct kvm_vcpu *vcpu, bool val)
 958{
 959        vcpu->spin_loop.dy_eligible = val;
 960}
 961
 962#else /* !CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT */
 963
 964static inline void kvm_vcpu_set_in_spin_loop(struct kvm_vcpu *vcpu, bool val)
 965{
 966}
 967
 968static inline void kvm_vcpu_set_dy_eligible(struct kvm_vcpu *vcpu, bool val)
 969{
 970}
 971
 972static inline bool kvm_vcpu_eligible_for_directed_yield(struct kvm_vcpu *vcpu)
 973{
 974        return true;
 975}
 976
 977#endif /* CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT */
 978#endif
 979
 980
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.