arch/x86/kvm/mmu/mmu.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Kernel-based Virtual Machine driver for Linux
   4  *
   5  * This module enables machines with Intel VT-x extensions to run virtual
   6  * machines without emulation or binary translation.
   7  *
   8  * MMU support
   9  *
  10  * Copyright (C) 2006 Qumranet, Inc.
  11  * Copyright 2010 Red Hat, Inc. and/or its affiliates.
  12  *
  13  * Authors:
  14  *   Yaniv Kamay  <yaniv@qumranet.com>
  15  *   Avi Kivity   <avi@qumranet.com>
  16  */
  17
  18 #include "irq.h"
  19 #include "ioapic.h"
  20 #include "mmu.h"
  21 #include "mmu_internal.h"
  22 #include "tdp_mmu.h"
  23 #include "x86.h"
  24 #include "kvm_cache_regs.h"
  25 #include "kvm_emulate.h"
  26 #include "cpuid.h"
  27 #include "spte.h"
  28
  29 #include <linux/kvm_host.h>
  30 #include <linux/types.h>
  31 #include <linux/string.h>
  32 #include <linux/mm.h>
  33 #include <linux/highmem.h>
  34 #include <linux/moduleparam.h>
  35 #include <linux/export.h>
  36 #include <linux/swap.h>
  37 #include <linux/hugetlb.h>
  38 #include <linux/compiler.h>
  39 #include <linux/srcu.h>
  40 #include <linux/slab.h>
  41 #include <linux/sched/signal.h>
  42 #include <linux/uaccess.h>
  43 #include <linux/hash.h>
  44 #include <linux/kern_levels.h>
  45 #include <linux/kthread.h>
  46
  47 #include <asm/page.h>
  48 #include <asm/memtype.h>
  49 #include <asm/cmpxchg.h>
  50 #include <asm/io.h>
  51 #include <asm/set_memory.h>
  52 #include <asm/vmx.h>
  53 #include <asm/kvm_page_track.h>
  54 #include "trace.h"
  55
  56 #include "paging.h"
  57
  58 extern bool itlb_multihit_kvm_mitigation;
  59
  60 int __read_mostly nx_huge_pages = -1;
  61 static uint __read_mostly nx_huge_pages_recovery_period_ms;
  62 #ifdef CONFIG_PREEMPT_RT
  63 /* Recovery can cause latency spikes, disable it for PREEMPT_RT.  */
  64 static uint __read_mostly nx_huge_pages_recovery_ratio = 0;
  65 #else
  66 static uint __read_mostly nx_huge_pages_recovery_ratio = 60;
  67 #endif
  68
  69 static int set_nx_huge_pages(const char *val, const struct kernel_param *kp);
  70 static int set_nx_huge_pages_recovery_param(const char *val, const struct kernel_param *kp);
  71
  72 static const struct kernel_param_ops nx_huge_pages_ops = {
  73         .set = set_nx_huge_pages,
  74         .get = param_get_bool,
  75 };
  76
  77 static const struct kernel_param_ops nx_huge_pages_recovery_param_ops = {
  78         .set = set_nx_huge_pages_recovery_param,
  79         .get = param_get_uint,
  80 };
  81
  82 module_param_cb(nx_huge_pages, &nx_huge_pages_ops, &nx_huge_pages, 0644);
  83 __MODULE_PARM_TYPE(nx_huge_pages, "bool");
  84 module_param_cb(nx_huge_pages_recovery_ratio, &nx_huge_pages_recovery_param_ops,
  85                 &nx_huge_pages_recovery_ratio, 0644);
  86 __MODULE_PARM_TYPE(nx_huge_pages_recovery_ratio, "uint");
  87 module_param_cb(nx_huge_pages_recovery_period_ms, &nx_huge_pages_recovery_param_ops,
  88                 &nx_huge_pages_recovery_period_ms, 0644);
  89 __MODULE_PARM_TYPE(nx_huge_pages_recovery_period_ms, "uint");
  90
  91 static bool __read_mostly force_flush_and_sync_on_reuse;
  92 module_param_named(flush_on_reuse, force_flush_and_sync_on_reuse, bool, 0644);
  93
  94 /*
  95  * When setting this variable to true it enables Two-Dimensional-Paging
  96  * where the hardware walks 2 page tables:
  97  * 1. the guest-virtual to guest-physical
  98  * 2. while doing 1. it walks guest-physical to host-physical
  99  * If the hardware supports that we don't need to do shadow paging.
 100  */
 101 bool tdp_enabled = false;
 102
 103 static int max_huge_page_level __read_mostly;
 104 static int tdp_root_level __read_mostly;
 105 static int max_tdp_level __read_mostly;
 106
 107 #ifdef MMU_DEBUG
 108 bool dbg = 0;
 109 module_param(dbg, bool, 0644);
 110 #endif
 111
 112 #define PTE_PREFETCH_NUM                8
 113
 114 #define PT32_LEVEL_BITS 10
 115
 116 #define PT32_LEVEL_SHIFT(level) \
 117                 (PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS)
 118
 119 #define PT32_LVL_OFFSET_MASK(level) \
 120         (PT32_BASE_ADDR_MASK & ((1ULL << (PAGE_SHIFT + (((level) - 1) \
 121                                                 * PT32_LEVEL_BITS))) - 1))
 122
 123 #define PT32_INDEX(address, level)\
 124         (((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1))
 125
 126
 127 #define PT32_BASE_ADDR_MASK PAGE_MASK
 128 #define PT32_DIR_BASE_ADDR_MASK \
 129         (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + PT32_LEVEL_BITS)) - 1))
 130 #define PT32_LVL_ADDR_MASK(level) \
 131         (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \
 132                                             * PT32_LEVEL_BITS))) - 1))
 133
 134 #include <trace/events/kvm.h>
 135
 136 /* make pte_list_desc fit well in cache lines */
 137 #define PTE_LIST_EXT 14
 138
 139 /*
 140  * Slight optimization of cacheline layout, by putting `more' and `spte_count'
 141  * at the start; then accessing it will only use one single cacheline for
 142  * either full (entries==PTE_LIST_EXT) case or entries<=6.
 143  */
 144 struct pte_list_desc {
 145         struct pte_list_desc *more;
 146         /*
 147          * Stores number of entries stored in the pte_list_desc.  No need to be
 148          * u64 but just for easier alignment.  When PTE_LIST_EXT, means full.
 149          */
 150         u64 spte_count;
 151         u64 *sptes[PTE_LIST_EXT];
 152 };
 153
 154 struct kvm_shadow_walk_iterator {
 155         u64 addr;
 156         hpa_t shadow_addr;
 157         u64 *sptep;
 158         int level;
 159         unsigned index;
 160 };
 161
 162 #define for_each_shadow_entry_using_root(_vcpu, _root, _addr, _walker)     \
 163         for (shadow_walk_init_using_root(&(_walker), (_vcpu),              \
 164                                          (_root), (_addr));                \
 165              shadow_walk_okay(&(_walker));                                 \
 166              shadow_walk_next(&(_walker)))
 167
 168 #define for_each_shadow_entry(_vcpu, _addr, _walker)            \
 169         for (shadow_walk_init(&(_walker), _vcpu, _addr);        \
 170              shadow_walk_okay(&(_walker));                      \
 171              shadow_walk_next(&(_walker)))
 172
 173 #define for_each_shadow_entry_lockless(_vcpu, _addr, _walker, spte)     \
 174         for (shadow_walk_init(&(_walker), _vcpu, _addr);                \
 175              shadow_walk_okay(&(_walker)) &&                            \
 176                 ({ spte = mmu_spte_get_lockless(_walker.sptep); 1; });  \
 177              __shadow_walk_next(&(_walker), spte))
 178
 179 static struct kmem_cache *pte_list_desc_cache;
 180 struct kmem_cache *mmu_page_header_cache;
 181 static struct percpu_counter kvm_total_used_mmu_pages;
 182
 183 static void mmu_spte_set(u64 *sptep, u64 spte);
 184
 185 struct kvm_mmu_role_regs {
 186         const unsigned long cr0;
 187         const unsigned long cr4;
 188         const u64 efer;
 189 };
 190
 191 #define CREATE_TRACE_POINTS
 192 #include "mmutrace.h"
 193
 194 /*
 195  * Yes, lot's of underscores.  They're a hint that you probably shouldn't be
 196  * reading from the role_regs.  Once the root_role is constructed, it becomes
 197  * the single source of truth for the MMU's state.
 198  */
 199 #define BUILD_MMU_ROLE_REGS_ACCESSOR(reg, name, flag)                   \
 200 static inline bool __maybe_unused                                       \
 201 ____is_##reg##_##name(const struct kvm_mmu_role_regs *regs)             \
 202 {                                                                       \
 203         return !!(regs->reg & flag);                                    \
 204 }
 205 BUILD_MMU_ROLE_REGS_ACCESSOR(cr0, pg, X86_CR0_PG);
 206 BUILD_MMU_ROLE_REGS_ACCESSOR(cr0, wp, X86_CR0_WP);
 207 BUILD_MMU_ROLE_REGS_ACCESSOR(cr4, pse, X86_CR4_PSE);
 208 BUILD_MMU_ROLE_REGS_ACCESSOR(cr4, pae, X86_CR4_PAE);
 209 BUILD_MMU_ROLE_REGS_ACCESSOR(cr4, smep, X86_CR4_SMEP);
 210 BUILD_MMU_ROLE_REGS_ACCESSOR(cr4, smap, X86_CR4_SMAP);
 211 BUILD_MMU_ROLE_REGS_ACCESSOR(cr4, pke, X86_CR4_PKE);
 212 BUILD_MMU_ROLE_REGS_ACCESSOR(cr4, la57, X86_CR4_LA57);
 213 BUILD_MMU_ROLE_REGS_ACCESSOR(efer, nx, EFER_NX);
 214 BUILD_MMU_ROLE_REGS_ACCESSOR(efer, lma, EFER_LMA);
 215
 216 /*
 217  * The MMU itself (with a valid role) is the single source of truth for the
 218  * MMU.  Do not use the regs used to build the MMU/role, nor the vCPU.  The
 219  * regs don't account for dependencies, e.g. clearing CR4 bits if CR0.PG=1,
 220  * and the vCPU may be incorrect/irrelevant.
 221  */
 222 #define BUILD_MMU_ROLE_ACCESSOR(base_or_ext, reg, name)         \
 223 static inline bool __maybe_unused is_##reg##_##name(struct kvm_mmu *mmu)        \
 224 {                                                               \
 225         return !!(mmu->cpu_role. base_or_ext . reg##_##name);   \
 226 }
 227 BUILD_MMU_ROLE_ACCESSOR(base, cr0, wp);
 228 BUILD_MMU_ROLE_ACCESSOR(ext,  cr4, pse);
 229 BUILD_MMU_ROLE_ACCESSOR(ext,  cr4, smep);
 230 BUILD_MMU_ROLE_ACCESSOR(ext,  cr4, smap);
 231 BUILD_MMU_ROLE_ACCESSOR(ext,  cr4, pke);
 232 BUILD_MMU_ROLE_ACCESSOR(ext,  cr4, la57);
 233 BUILD_MMU_ROLE_ACCESSOR(base, efer, nx);
 234 BUILD_MMU_ROLE_ACCESSOR(ext,  efer, lma);
 235
 236 static inline bool is_cr0_pg(struct kvm_mmu *mmu)
 237 {
 238         return mmu->cpu_role.base.level > 0;
 239 }
 240
 241 static inline bool is_cr4_pae(struct kvm_mmu *mmu)
 242 {
 243         return !mmu->cpu_role.base.has_4_byte_gpte;
 244 }
 245
 246 static struct kvm_mmu_role_regs vcpu_to_role_regs(struct kvm_vcpu *vcpu)
 247 {
 248         struct kvm_mmu_role_regs regs = {
 249                 .cr0 = kvm_read_cr0_bits(vcpu, KVM_MMU_CR0_ROLE_BITS),
 250                 .cr4 = kvm_read_cr4_bits(vcpu, KVM_MMU_CR4_ROLE_BITS),
 251                 .efer = vcpu->arch.efer,
 252         };
 253
 254         return regs;
 255 }
 256
 257 static inline bool kvm_available_flush_tlb_with_range(void)
 258 {
 259         return kvm_x86_ops.tlb_remote_flush_with_range;
 260 }
 261
 262 static void kvm_flush_remote_tlbs_with_range(struct kvm *kvm,
 263                 struct kvm_tlb_range *range)
 264 {
 265         int ret = -ENOTSUPP;
 266
 267         if (range && kvm_x86_ops.tlb_remote_flush_with_range)
 268                 ret = static_call(kvm_x86_tlb_remote_flush_with_range)(kvm, range);
 269
 270         if (ret)
 271                 kvm_flush_remote_tlbs(kvm);
 272 }
 273
 274 void kvm_flush_remote_tlbs_with_address(struct kvm *kvm,
 275                 u64 start_gfn, u64 pages)
 276 {
 277         struct kvm_tlb_range range;
 278
 279         range.start_gfn = start_gfn;
 280         range.pages = pages;
 281
 282         kvm_flush_remote_tlbs_with_range(kvm, &range);
 283 }
 284
 285 static void mark_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, u64 gfn,
 286                            unsigned int access)
 287 {
 288         u64 spte = make_mmio_spte(vcpu, gfn, access);
 289
 290         trace_mark_mmio_spte(sptep, gfn, spte);
 291         mmu_spte_set(sptep, spte);
 292 }
 293
 294 static gfn_t get_mmio_spte_gfn(u64 spte)
 295 {
 296         u64 gpa = spte & shadow_nonpresent_or_rsvd_lower_gfn_mask;
 297
 298         gpa |= (spte >> SHADOW_NONPRESENT_OR_RSVD_MASK_LEN)
 299                & shadow_nonpresent_or_rsvd_mask;
 300
 301         return gpa >> PAGE_SHIFT;
 302 }
 303
 304 static unsigned get_mmio_spte_access(u64 spte)
 305 {
 306         return spte & shadow_mmio_access_mask;
 307 }
 308
 309 static bool check_mmio_spte(struct kvm_vcpu *vcpu, u64 spte)
 310 {
 311         u64 kvm_gen, spte_gen, gen;
 312
 313         gen = kvm_vcpu_memslots(vcpu)->generation;
 314         if (unlikely(gen & KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS))
 315                 return false;
 316
 317         kvm_gen = gen & MMIO_SPTE_GEN_MASK;
 318         spte_gen = get_mmio_spte_generation(spte);
 319
 320         trace_check_mmio_spte(spte, kvm_gen, spte_gen);
 321         return likely(kvm_gen == spte_gen);
 322 }
 323
 324 static int is_cpuid_PSE36(void)
 325 {
 326         return 1;
 327 }
 328
 329 static gfn_t pse36_gfn_delta(u32 gpte)
 330 {
 331         int shift = 32 - PT32_DIR_PSE36_SHIFT - PAGE_SHIFT;
 332
 333         return (gpte & PT32_DIR_PSE36_MASK) << shift;
 334 }
 335
 336 #ifdef CONFIG_X86_64
 337 static void __set_spte(u64 *sptep, u64 spte)
 338 {
 339         WRITE_ONCE(*sptep, spte);
 340 }
 341
 342 static void __update_clear_spte_fast(u64 *sptep, u64 spte)
 343 {
 344         WRITE_ONCE(*sptep, spte);
 345 }
 346
 347 static u64 __update_clear_spte_slow(u64 *sptep, u64 spte)
 348 {
 349         return xchg(sptep, spte);
 350 }
 351
 352 static u64 __get_spte_lockless(u64 *sptep)
 353 {
 354         return READ_ONCE(*sptep);
 355 }
 356 #else
 357 union split_spte {
 358         struct {
 359                 u32 spte_low;
 360                 u32 spte_high;
 361         };
 362         u64 spte;
 363 };
 364
 365 static void count_spte_clear(u64 *sptep, u64 spte)
 366 {
 367         struct kvm_mmu_page *sp =  sptep_to_sp(sptep);
 368
 369         if (is_shadow_present_pte(spte))
 370                 return;
 371
 372         /* Ensure the spte is completely set before we increase the count */
 373         smp_wmb();
 374         sp->clear_spte_count++;
 375 }
 376
 377 static void __set_spte(u64 *sptep, u64 spte)
 378 {
 379         union split_spte *ssptep, sspte;
 380
 381         ssptep = (union split_spte *)sptep;
 382         sspte = (union split_spte)spte;
 383
 384         ssptep->spte_high = sspte.spte_high;
 385
 386         /*
 387          * If we map the spte from nonpresent to present, We should store
 388          * the high bits firstly, then set present bit, so cpu can not
 389          * fetch this spte while we are setting the spte.
 390          */
 391         smp_wmb();
 392
 393         WRITE_ONCE(ssptep->spte_low, sspte.spte_low);
 394 }
 395
 396 static void __update_clear_spte_fast(u64 *sptep, u64 spte)
 397 {
 398         union split_spte *ssptep, sspte;
 399
 400         ssptep = (union split_spte *)sptep;
 401         sspte = (union split_spte)spte;
 402
 403         WRITE_ONCE(ssptep->spte_low, sspte.spte_low);
 404
 405         /*
 406          * If we map the spte from present to nonpresent, we should clear
 407          * present bit firstly to avoid vcpu fetch the old high bits.
 408          */
 409         smp_wmb();
 410
 411         ssptep->spte_high = sspte.spte_high;
 412         count_spte_clear(sptep, spte);
 413 }
 414
 415 static u64 __update_clear_spte_slow(u64 *sptep, u64 spte)
 416 {
 417         union split_spte *ssptep, sspte, orig;
 418
 419         ssptep = (union split_spte *)sptep;
 420         sspte = (union split_spte)spte;
 421
 422         /* xchg acts as a barrier before the setting of the high bits */
 423         orig.spte_low = xchg(&ssptep->spte_low, sspte.spte_low);
 424         orig.spte_high = ssptep->spte_high;
 425         ssptep->spte_high = sspte.spte_high;
 426         count_spte_clear(sptep, spte);
 427
 428         return orig.spte;
 429 }
 430
 431 /*
 432  * The idea using the light way get the spte on x86_32 guest is from
 433  * gup_get_pte (mm/gup.c).
 434  *
 435  * An spte tlb flush may be pending, because kvm_set_pte_rmapp
 436  * coalesces them and we are running out of the MMU lock.  Therefore
 437  * we need to protect against in-progress updates of the spte.
 438  *
 439  * Reading the spte while an update is in progress may get the old value
 440  * for the high part of the spte.  The race is fine for a present->non-present
 441  * change (because the high part of the spte is ignored for non-present spte),
 442  * but for a present->present change we must reread the spte.
 443  *
 444  * All such changes are done in two steps (present->non-present and
 445  * non-present->present), hence it is enough to count the number of
 446  * present->non-present updates: if it changed while reading the spte,
 447  * we might have hit the race.  This is done using clear_spte_count.
 448  */
 449 static u64 __get_spte_lockless(u64 *sptep)
 450 {
 451         struct kvm_mmu_page *sp =  sptep_to_sp(sptep);
 452         union split_spte spte, *orig = (union split_spte *)sptep;
 453         int count;
 454
 455 retry:
 456         count = sp->clear_spte_count;
 457         smp_rmb();
 458
 459         spte.spte_low = orig->spte_low;
 460         smp_rmb();
 461
 462         spte.spte_high = orig->spte_high;
 463         smp_rmb();
 464
 465         if (unlikely(spte.spte_low != orig->spte_low ||
 466               count != sp->clear_spte_count))
 467                 goto retry;
 468
 469         return spte.spte;
 470 }
 471 #endif
 472
 473 /* Rules for using mmu_spte_set:
 474  * Set the sptep from nonpresent to present.
 475  * Note: the sptep being assigned *must* be either not present
 476  * or in a state where the hardware will not attempt to update
 477  * the spte.
 478  */
 479 static void mmu_spte_set(u64 *sptep, u64 new_spte)
 480 {
 481         WARN_ON(is_shadow_present_pte(*sptep));
 482         __set_spte(sptep, new_spte);
 483 }
 484
 485 /*
 486  * Update the SPTE (excluding the PFN), but do not track changes in its
 487  * accessed/dirty status.
 488  */
 489 static u64 mmu_spte_update_no_track(u64 *sptep, u64 new_spte)
 490 {
 491         u64 old_spte = *sptep;
 492
 493         WARN_ON(!is_shadow_present_pte(new_spte));
 494         check_spte_writable_invariants(new_spte);
 495
 496         if (!is_shadow_present_pte(old_spte)) {
 497                 mmu_spte_set(sptep, new_spte);
 498                 return old_spte;
 499         }
 500
 501         if (!spte_has_volatile_bits(old_spte))
 502                 __update_clear_spte_fast(sptep, new_spte);
 503         else
 504                 old_spte = __update_clear_spte_slow(sptep, new_spte);
 505
 506         WARN_ON(spte_to_pfn(old_spte) != spte_to_pfn(new_spte));
 507
 508         return old_spte;
 509 }
 510
 511 /* Rules for using mmu_spte_update:
 512  * Update the state bits, it means the mapped pfn is not changed.
 513  *
 514  * Whenever an MMU-writable SPTE is overwritten with a read-only SPTE, remote
 515  * TLBs must be flushed. Otherwise rmap_write_protect will find a read-only
 516  * spte, even though the writable spte might be cached on a CPU's TLB.
 517  *
 518  * Returns true if the TLB needs to be flushed
 519  */
 520 static bool mmu_spte_update(u64 *sptep, u64 new_spte)
 521 {
 522         bool flush = false;
 523         u64 old_spte = mmu_spte_update_no_track(sptep, new_spte);
 524
 525         if (!is_shadow_present_pte(old_spte))
 526                 return false;
 527
 528         /*
 529          * For the spte updated out of mmu-lock is safe, since
 530          * we always atomically update it, see the comments in
 531          * spte_has_volatile_bits().
 532          */
 533         if (is_mmu_writable_spte(old_spte) &&
 534               !is_writable_pte(new_spte))
 535                 flush = true;
 536
 537         /*
 538          * Flush TLB when accessed/dirty states are changed in the page tables,
 539          * to guarantee consistency between TLB and page tables.
 540          */
 541
 542         if (is_accessed_spte(old_spte) && !is_accessed_spte(new_spte)) {
 543                 flush = true;
 544                 kvm_set_pfn_accessed(spte_to_pfn(old_spte));
 545         }
 546
 547         if (is_dirty_spte(old_spte) && !is_dirty_spte(new_spte)) {
 548                 flush = true;
 549                 kvm_set_pfn_dirty(spte_to_pfn(old_spte));
 550         }
 551
 552         return flush;
 553 }
 554
 555 /*
 556  * Rules for using mmu_spte_clear_track_bits:
 557  * It sets the sptep from present to nonpresent, and track the
 558  * state bits, it is used to clear the last level sptep.
 559  * Returns the old PTE.
 560  */
 561 static int mmu_spte_clear_track_bits(struct kvm *kvm, u64 *sptep)
 562 {
 563         kvm_pfn_t pfn;
 564         u64 old_spte = *sptep;
 565         int level = sptep_to_sp(sptep)->role.level;
 566
 567         if (!is_shadow_present_pte(old_spte) ||
 568             !spte_has_volatile_bits(old_spte))
 569                 __update_clear_spte_fast(sptep, 0ull);
 570         else
 571                 old_spte = __update_clear_spte_slow(sptep, 0ull);
 572
 573         if (!is_shadow_present_pte(old_spte))
 574                 return old_spte;
 575
 576         kvm_update_page_stats(kvm, level, -1);
 577
 578         pfn = spte_to_pfn(old_spte);
 579
 580         /*
 581          * KVM does not hold the refcount of the page used by
 582          * kvm mmu, before reclaiming the page, we should
 583          * unmap it from mmu first.
 584          */
 585         WARN_ON(!kvm_is_reserved_pfn(pfn) && !page_count(pfn_to_page(pfn)));
 586
 587         if (is_accessed_spte(old_spte))
 588                 kvm_set_pfn_accessed(pfn);
 589
 590         if (is_dirty_spte(old_spte))
 591                 kvm_set_pfn_dirty(pfn);
 592
 593         return old_spte;
 594 }
 595
 596 /*
 597  * Rules for using mmu_spte_clear_no_track:
 598  * Directly clear spte without caring the state bits of sptep,
 599  * it is used to set the upper level spte.
 600  */
 601 static void mmu_spte_clear_no_track(u64 *sptep)
 602 {
 603         __update_clear_spte_fast(sptep, 0ull);
 604 }
 605
 606 static u64 mmu_spte_get_lockless(u64 *sptep)
 607 {
 608         return __get_spte_lockless(sptep);
 609 }
 610
 611 /* Returns the Accessed status of the PTE and resets it at the same time. */
 612 static bool mmu_spte_age(u64 *sptep)
 613 {
 614         u64 spte = mmu_spte_get_lockless(sptep);
 615
 616         if (!is_accessed_spte(spte))
 617                 return false;
 618
 619         if (spte_ad_enabled(spte)) {
 620                 clear_bit((ffs(shadow_accessed_mask) - 1),
 621                           (unsigned long *)sptep);
 622         } else {
 623                 /*
 624                  * Capture the dirty status of the page, so that it doesn't get
 625                  * lost when the SPTE is marked for access tracking.
 626                  */
 627                 if (is_writable_pte(spte))
 628                         kvm_set_pfn_dirty(spte_to_pfn(spte));
 629
 630                 spte = mark_spte_for_access_track(spte);
 631                 mmu_spte_update_no_track(sptep, spte);
 632         }
 633
 634         return true;
 635 }
 636
 637 static void walk_shadow_page_lockless_begin(struct kvm_vcpu *vcpu)
 638 {
 639         if (is_tdp_mmu(vcpu->arch.mmu)) {
 640                 kvm_tdp_mmu_walk_lockless_begin();
 641         } else {
 642                 /*
 643                  * Prevent page table teardown by making any free-er wait during
 644                  * kvm_flush_remote_tlbs() IPI to all active vcpus.
 645                  */
 646                 local_irq_disable();
 647
 648                 /*
 649                  * Make sure a following spte read is not reordered ahead of the write
 650                  * to vcpu->mode.
 651                  */
 652                 smp_store_mb(vcpu->mode, READING_SHADOW_PAGE_TABLES);
 653         }
 654 }
 655
 656 static void walk_shadow_page_lockless_end(struct kvm_vcpu *vcpu)
 657 {
 658         if (is_tdp_mmu(vcpu->arch.mmu)) {
 659                 kvm_tdp_mmu_walk_lockless_end();
 660         } else {
 661                 /*
 662                  * Make sure the write to vcpu->mode is not reordered in front of
 663                  * reads to sptes.  If it does, kvm_mmu_commit_zap_page() can see us
 664                  * OUTSIDE_GUEST_MODE and proceed to free the shadow page table.
 665                  */
 666                 smp_store_release(&vcpu->mode, OUTSIDE_GUEST_MODE);
 667                 local_irq_enable();
 668         }
 669 }
 670
 671 static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu, bool maybe_indirect)
 672 {
 673         int r;
 674
 675         /* 1 rmap, 1 parent PTE per level, and the prefetched rmaps. */
 676         r = kvm_mmu_topup_memory_cache(&vcpu->arch.mmu_pte_list_desc_cache,
 677                                        1 + PT64_ROOT_MAX_LEVEL + PTE_PREFETCH_NUM);
 678         if (r)
 679                 return r;
 680         r = kvm_mmu_topup_memory_cache(&vcpu->arch.mmu_shadow_page_cache,
 681                                        PT64_ROOT_MAX_LEVEL);
 682         if (r)
 683                 return r;
 684         if (maybe_indirect) {
 685                 r = kvm_mmu_topup_memory_cache(&vcpu->arch.mmu_gfn_array_cache,
 686                                                PT64_ROOT_MAX_LEVEL);
 687                 if (r)
 688                         return r;
 689         }
 690         return kvm_mmu_topup_memory_cache(&vcpu->arch.mmu_page_header_cache,
 691                                           PT64_ROOT_MAX_LEVEL);
 692 }
 693
 694 static void mmu_free_memory_caches(struct kvm_vcpu *vcpu)
 695 {
 696         kvm_mmu_free_memory_cache(&vcpu->arch.mmu_pte_list_desc_cache);
 697         kvm_mmu_free_memory_cache(&vcpu->arch.mmu_shadow_page_cache);
 698         kvm_mmu_free_memory_cache(&vcpu->arch.mmu_gfn_array_cache);
 699         kvm_mmu_free_memory_cache(&vcpu->arch.mmu_page_header_cache);
 700 }
 701
 702 static struct pte_list_desc *mmu_alloc_pte_list_desc(struct kvm_vcpu *vcpu)
 703 {
 704         return kvm_mmu_memory_cache_alloc(&vcpu->arch.mmu_pte_list_desc_cache);
 705 }
 706
 707 static void mmu_free_pte_list_desc(struct pte_list_desc *pte_list_desc)
 708 {
 709         kmem_cache_free(pte_list_desc_cache, pte_list_desc);
 710 }
 711
 712 static gfn_t kvm_mmu_page_get_gfn(struct kvm_mmu_page *sp, int index)
 713 {
 714         if (sp->role.passthrough)
 715                 return sp->gfn;
 716
 717         if (!sp->role.direct)
 718                 return sp->gfns[index];
 719
 720         return sp->gfn + (index << ((sp->role.level - 1) * PT64_LEVEL_BITS));
 721 }
 722
 723 static void kvm_mmu_page_set_gfn(struct kvm_mmu_page *sp, int index, gfn_t gfn)
 724 {
 725         if (sp->role.passthrough) {
 726                 WARN_ON_ONCE(gfn != sp->gfn);
 727                 return;
 728         }
 729
 730         if (!sp->role.direct) {
 731                 sp->gfns[index] = gfn;
 732                 return;
 733         }
 734
 735         if (WARN_ON(gfn != kvm_mmu_page_get_gfn(sp, index)))
 736                 pr_err_ratelimited("gfn mismatch under direct page %llx "
 737                                    "(expected %llx, got %llx)\n",
 738                                    sp->gfn,
 739                                    kvm_mmu_page_get_gfn(sp, index), gfn);
 740 }
 741
 742 /*
 743  * Return the pointer to the large page information for a given gfn,
 744  * handling slots that are not large page aligned.
 745  */
 746 static struct kvm_lpage_info *lpage_info_slot(gfn_t gfn,
 747                 const struct kvm_memory_slot *slot, int level)
 748 {
 749         unsigned long idx;
 750
 751         idx = gfn_to_index(gfn, slot->base_gfn, level);
 752         return &slot->arch.lpage_info[level - 2][idx];
 753 }
 754
 755 static void update_gfn_disallow_lpage_count(const struct kvm_memory_slot *slot,
 756                                             gfn_t gfn, int count)
 757 {
 758         struct kvm_lpage_info *linfo;
 759         int i;
 760
 761         for (i = PG_LEVEL_2M; i <= KVM_MAX_HUGEPAGE_LEVEL; ++i) {
 762                 linfo = lpage_info_slot(gfn, slot, i);
 763                 linfo->disallow_lpage += count;
 764                 WARN_ON(linfo->disallow_lpage < 0);
 765         }
 766 }
 767
 768 void kvm_mmu_gfn_disallow_lpage(const struct kvm_memory_slot *slot, gfn_t gfn)
 769 {
 770         update_gfn_disallow_lpage_count(slot, gfn, 1);
 771 }
 772
 773 void kvm_mmu_gfn_allow_lpage(const struct kvm_memory_slot *slot, gfn_t gfn)
 774 {
 775         update_gfn_disallow_lpage_count(slot, gfn, -1);
 776 }
 777
 778 static void account_shadowed(struct kvm *kvm, struct kvm_mmu_page *sp)
 779 {
 780         struct kvm_memslots *slots;
 781         struct kvm_memory_slot *slot;
 782         gfn_t gfn;
 783
 784         kvm->arch.indirect_shadow_pages++;
 785         gfn = sp->gfn;
 786         slots = kvm_memslots_for_spte_role(kvm, sp->role);
 787         slot = __gfn_to_memslot(slots, gfn);
 788
 789         /* the non-leaf shadow pages are keeping readonly. */
 790         if (sp->role.level > PG_LEVEL_4K)
 791                 return kvm_slot_page_track_add_page(kvm, slot, gfn,
 792                                                     KVM_PAGE_TRACK_WRITE);
 793
 794         kvm_mmu_gfn_disallow_lpage(slot, gfn);
 795 }
 796
 797 void account_huge_nx_page(struct kvm *kvm, struct kvm_mmu_page *sp)
 798 {
 799         if (sp->lpage_disallowed)
 800                 return;
 801
 802         ++kvm->stat.nx_lpage_splits;
 803         list_add_tail(&sp->lpage_disallowed_link,
 804                       &kvm->arch.lpage_disallowed_mmu_pages);
 805         sp->lpage_disallowed = true;
 806 }
 807
 808 static void unaccount_shadowed(struct kvm *kvm, struct kvm_mmu_page *sp)
 809 {
 810         struct kvm_memslots *slots;
 811         struct kvm_memory_slot *slot;
 812         gfn_t gfn;
 813
 814         kvm->arch.indirect_shadow_pages--;
 815         gfn = sp->gfn;
 816         slots = kvm_memslots_for_spte_role(kvm, sp->role);
 817         slot = __gfn_to_memslot(slots, gfn);
 818         if (sp->role.level > PG_LEVEL_4K)
 819                 return kvm_slot_page_track_remove_page(kvm, slot, gfn,
 820                                                        KVM_PAGE_TRACK_WRITE);
 821
 822         kvm_mmu_gfn_allow_lpage(slot, gfn);
 823 }
 824
 825 void unaccount_huge_nx_page(struct kvm *kvm, struct kvm_mmu_page *sp)
 826 {
 827         --kvm->stat.nx_lpage_splits;
 828         sp->lpage_disallowed = false;
 829         list_del(&sp->lpage_disallowed_link);
 830 }
 831
 832 static struct kvm_memory_slot *
 833 gfn_to_memslot_dirty_bitmap(struct kvm_vcpu *vcpu, gfn_t gfn,
 834                             bool no_dirty_log)
 835 {
 836         struct kvm_memory_slot *slot;
 837
 838         slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
 839         if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
 840                 return NULL;
 841         if (no_dirty_log && kvm_slot_dirty_track_enabled(slot))
 842                 return NULL;
 843
 844         return slot;
 845 }
 846
 847 /*
 848  * About rmap_head encoding:
 849  *
 850  * If the bit zero of rmap_head->val is clear, then it points to the only spte
 851  * in this rmap chain. Otherwise, (rmap_head->val & ~1) points to a struct
 852  * pte_list_desc containing more mappings.
 853  */
 854
 855 /*
 856  * Returns the number of pointers in the rmap chain, not counting the new one.
 857  */
 858 static int pte_list_add(struct kvm_vcpu *vcpu, u64 *spte,
 859                         struct kvm_rmap_head *rmap_head)
 860 {
 861         struct pte_list_desc *desc;
 862         int count = 0;
 863
 864         if (!rmap_head->val) {
 865                 rmap_printk("%p %llx 0->1\n", spte, *spte);
 866                 rmap_head->val = (unsigned long)spte;
 867         } else if (!(rmap_head->val & 1)) {
 868                 rmap_printk("%p %llx 1->many\n", spte, *spte);
 869                 desc = mmu_alloc_pte_list_desc(vcpu);
 870                 desc->sptes[0] = (u64 *)rmap_head->val;
 871                 desc->sptes[1] = spte;
 872                 desc->spte_count = 2;
 873                 rmap_head->val = (unsigned long)desc | 1;
 874                 ++count;
 875         } else {
 876                 rmap_printk("%p %llx many->many\n", spte, *spte);
 877                 desc = (struct pte_list_desc *)(rmap_head->val & ~1ul);
 878                 while (desc->spte_count == PTE_LIST_EXT) {
 879                         count += PTE_LIST_EXT;
 880                         if (!desc->more) {
 881                                 desc->more = mmu_alloc_pte_list_desc(vcpu);
 882                                 desc = desc->more;
 883                                 desc->spte_count = 0;
 884                                 break;
 885                         }
 886                         desc = desc->more;
 887                 }
 888                 count += desc->spte_count;
 889                 desc->sptes[desc->spte_count++] = spte;
 890         }
 891         return count;
 892 }
 893
 894 static void
 895 pte_list_desc_remove_entry(struct kvm_rmap_head *rmap_head,
 896                            struct pte_list_desc *desc, int i,
 897                            struct pte_list_desc *prev_desc)
 898 {
 899         int j = desc->spte_count - 1;
 900
 901         desc->sptes[i] = desc->sptes[j];
 902         desc->sptes[j] = NULL;
 903         desc->spte_count--;
 904         if (desc->spte_count)
 905                 return;
 906         if (!prev_desc && !desc->more)
 907                 rmap_head->val = 0;
 908         else
 909                 if (prev_desc)
 910                         prev_desc->more = desc->more;
 911                 else
 912                         rmap_head->val = (unsigned long)desc->more | 1;
 913         mmu_free_pte_list_desc(desc);
 914 }
 915
 916 static void __pte_list_remove(u64 *spte, struct kvm_rmap_head *rmap_head)
 917 {
 918         struct pte_list_desc *desc;
 919         struct pte_list_desc *prev_desc;
 920         int i;
 921
 922         if (!rmap_head->val) {
 923                 pr_err("%s: %p 0->BUG\n", __func__, spte);
 924                 BUG();
 925         } else if (!(rmap_head->val & 1)) {
 926                 rmap_printk("%p 1->0\n", spte);
 927                 if ((u64 *)rmap_head->val != spte) {
 928                         pr_err("%s:  %p 1->BUG\n", __func__, spte);
 929                         BUG();
 930                 }
 931                 rmap_head->val = 0;
 932         } else {
 933                 rmap_printk("%p many->many\n", spte);
 934                 desc = (struct pte_list_desc *)(rmap_head->val & ~1ul);
 935                 prev_desc = NULL;
 936                 while (desc) {
 937                         for (i = 0; i < desc->spte_count; ++i) {
 938                                 if (desc->sptes[i] == spte) {
 939                                         pte_list_desc_remove_entry(rmap_head,
 940                                                         desc, i, prev_desc);
 941                                         return;
 942                                 }
 943                         }
 944                         prev_desc = desc;
 945                         desc = desc->more;
 946                 }
 947                 pr_err("%s: %p many->many\n", __func__, spte);
 948                 BUG();
 949         }
 950 }
 951
 952 static void pte_list_remove(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
 953                             u64 *sptep)
 954 {
 955         mmu_spte_clear_track_bits(kvm, sptep);
 956         __pte_list_remove(sptep, rmap_head);
 957 }
 958
 959 /* Return true if rmap existed, false otherwise */
 960 static bool pte_list_destroy(struct kvm *kvm, struct kvm_rmap_head *rmap_head)
 961 {
 962         struct pte_list_desc *desc, *next;
 963         int i;
 964
 965         if (!rmap_head->val)
 966                 return false;
 967
 968         if (!(rmap_head->val & 1)) {
 969                 mmu_spte_clear_track_bits(kvm, (u64 *)rmap_head->val);
 970                 goto out;
 971         }
 972
 973         desc = (struct pte_list_desc *)(rmap_head->val & ~1ul);
 974
 975         for (; desc; desc = next) {
 976                 for (i = 0; i < desc->spte_count; i++)
 977                         mmu_spte_clear_track_bits(kvm, desc->sptes[i]);
 978                 next = desc->more;
 979                 mmu_free_pte_list_desc(desc);
 980         }
 981 out:
 982         /* rmap_head is meaningless now, remember to reset it */
 983         rmap_head->val = 0;
 984         return true;
 985 }
 986
 987 unsigned int pte_list_count(struct kvm_rmap_head *rmap_head)
 988 {
 989         struct pte_list_desc *desc;
 990         unsigned int count = 0;
 991
 992         if (!rmap_head->val)
 993                 return 0;
 994         else if (!(rmap_head->val & 1))
 995                 return 1;
 996
 997         desc = (struct pte_list_desc *)(rmap_head->val & ~1ul);
 998
 999         while (desc) {
1000                 count += desc->spte_count;
1001                 desc = desc->more;
1002         }
1003
1004         return count;
1005 }
1006
1007 static struct kvm_rmap_head *gfn_to_rmap(gfn_t gfn, int level,
1008                                          const struct kvm_memory_slot *slot)
1009 {
1010         unsigned long idx;
1011
1012         idx = gfn_to_index(gfn, slot->base_gfn, level);
1013         return &slot->arch.rmap[level - PG_LEVEL_4K][idx];
1014 }
1015
1016 static bool rmap_can_add(struct kvm_vcpu *vcpu)
1017 {
1018         struct kvm_mmu_memory_cache *mc;
1019
1020         mc = &vcpu->arch.mmu_pte_list_desc_cache;
1021         return kvm_mmu_memory_cache_nr_free_objects(mc);
1022 }
1023
1024 static void rmap_remove(struct kvm *kvm, u64 *spte)
1025 {
1026         struct kvm_memslots *slots;
1027         struct kvm_memory_slot *slot;
1028         struct kvm_mmu_page *sp;
1029         gfn_t gfn;
1030         struct kvm_rmap_head *rmap_head;
1031
1032         sp = sptep_to_sp(spte);
1033         gfn = kvm_mmu_page_get_gfn(sp, spte - sp->spt);
1034
1035         /*
1036          * Unlike rmap_add, rmap_remove does not run in the context of a vCPU
1037          * so we have to determine which memslots to use based on context
1038          * information in sp->role.
1039          */
1040         slots = kvm_memslots_for_spte_role(kvm, sp->role);
1041
1042         slot = __gfn_to_memslot(slots, gfn);
1043         rmap_head = gfn_to_rmap(gfn, sp->role.level, slot);
1044
1045         __pte_list_remove(spte, rmap_head);
1046 }
1047
1048 /*
1049  * Used by the following functions to iterate through the sptes linked by a
1050  * rmap.  All fields are private and not assumed to be used outside.
1051  */
1052 struct rmap_iterator {
1053         /* private fields */
1054         struct pte_list_desc *desc;     /* holds the sptep if not NULL */
1055         int pos;                        /* index of the sptep */
1056 };
1057
1058 /*
1059  * Iteration must be started by this function.  This should also be used after
1060  * removing/dropping sptes from the rmap link because in such cases the
1061  * information in the iterator may not be valid.
1062  *
1063  * Returns sptep if found, NULL otherwise.
1064  */
1065 static u64 *rmap_get_first(struct kvm_rmap_head *rmap_head,
1066                            struct rmap_iterator *iter)
1067 {
1068         u64 *sptep;
1069
1070         if (!rmap_head->val)
1071                 return NULL;
1072
1073         if (!(rmap_head->val & 1)) {
1074                 iter->desc = NULL;
1075                 sptep = (u64 *)rmap_head->val;
1076                 goto out;
1077         }
1078
1079         iter->desc = (struct pte_list_desc *)(rmap_head->val & ~1ul);
1080         iter->pos = 0;
1081         sptep = iter->desc->sptes[iter->pos];
1082 out:
1083         BUG_ON(!is_shadow_present_pte(*sptep));
1084         return sptep;
1085 }
1086
1087 /*
1088  * Must be used with a valid iterator: e.g. after rmap_get_first().
1089  *
1090  * Returns sptep if found, NULL otherwise.
1091  */
1092 static u64 *rmap_get_next(struct rmap_iterator *iter)
1093 {
1094         u64 *sptep;
1095
1096         if (iter->desc) {
1097                 if (iter->pos < PTE_LIST_EXT - 1) {
1098                         ++iter->pos;
1099                         sptep = iter->desc->sptes[iter->pos];
1100                         if (sptep)
1101                                 goto out;
1102                 }
1103
1104                 iter->desc = iter->desc->more;
1105
1106                 if (iter->desc) {
1107                         iter->pos = 0;
1108                         /* desc->sptes[0] cannot be NULL */
1109                         sptep = iter->desc->sptes[iter->pos];
1110                         goto out;
1111                 }
1112         }
1113
1114         return NULL;
1115 out:
1116         BUG_ON(!is_shadow_present_pte(*sptep));
1117         return sptep;
1118 }
1119
1120 #define for_each_rmap_spte(_rmap_head_, _iter_, _spte_)                 \
1121         for (_spte_ = rmap_get_first(_rmap_head_, _iter_);              \
1122              _spte_; _spte_ = rmap_get_next(_iter_))
1123
1124 static void drop_spte(struct kvm *kvm, u64 *sptep)
1125 {
1126         u64 old_spte = mmu_spte_clear_track_bits(kvm, sptep);
1127
1128         if (is_shadow_present_pte(old_spte))
1129                 rmap_remove(kvm, sptep);
1130 }
1131
1132
1133 static bool __drop_large_spte(struct kvm *kvm, u64 *sptep)
1134 {
1135         if (is_large_pte(*sptep)) {
1136                 WARN_ON(sptep_to_sp(sptep)->role.level == PG_LEVEL_4K);
1137                 drop_spte(kvm, sptep);
1138                 return true;
1139         }
1140
1141         return false;
1142 }
1143
1144 static void drop_large_spte(struct kvm_vcpu *vcpu, u64 *sptep)
1145 {
1146         if (__drop_large_spte(vcpu->kvm, sptep)) {
1147                 struct kvm_mmu_page *sp = sptep_to_sp(sptep);
1148
1149                 kvm_flush_remote_tlbs_with_address(vcpu->kvm, sp->gfn,
1150                         KVM_PAGES_PER_HPAGE(sp->role.level));
1151         }
1152 }
1153
1154 /*
1155  * Write-protect on the specified @sptep, @pt_protect indicates whether
1156  * spte write-protection is caused by protecting shadow page table.
1157  *
1158  * Note: write protection is difference between dirty logging and spte
1159  * protection:
1160  * - for dirty logging, the spte can be set to writable at anytime if
1161  *   its dirty bitmap is properly set.
1162  * - for spte protection, the spte can be writable only after unsync-ing
1163  *   shadow page.
1164  *
1165  * Return true if tlb need be flushed.
1166  */
1167 static bool spte_write_protect(u64 *sptep, bool pt_protect)
1168 {
1169         u64 spte = *sptep;
1170
1171         if (!is_writable_pte(spte) &&
1172             !(pt_protect && is_mmu_writable_spte(spte)))
1173                 return false;
1174
1175         rmap_printk("spte %p %llx\n", sptep, *sptep);
1176
1177         if (pt_protect)
1178                 spte &= ~shadow_mmu_writable_mask;
1179         spte = spte & ~PT_WRITABLE_MASK;
1180
1181         return mmu_spte_update(sptep, spte);
1182 }
1183
1184 static bool rmap_write_protect(struct kvm_rmap_head *rmap_head,
1185                                bool pt_protect)
1186 {
1187         u64 *sptep;
1188         struct rmap_iterator iter;
1189         bool flush = false;
1190
1191         for_each_rmap_spte(rmap_head, &iter, sptep)
1192                 flush |= spte_write_protect(sptep, pt_protect);
1193
1194         return flush;
1195 }
1196
1197 static bool spte_clear_dirty(u64 *sptep)
1198 {
1199         u64 spte = *sptep;
1200
1201         rmap_printk("spte %p %llx\n", sptep, *sptep);
1202
1203         MMU_WARN_ON(!spte_ad_enabled(spte));
1204         spte &= ~shadow_dirty_mask;
1205         return mmu_spte_update(sptep, spte);
1206 }
1207
1208 static bool spte_wrprot_for_clear_dirty(u64 *sptep)
1209 {
1210         bool was_writable = test_and_clear_bit(PT_WRITABLE_SHIFT,
1211                                                (unsigned long *)sptep);
1212         if (was_writable && !spte_ad_enabled(*sptep))
1213                 kvm_set_pfn_dirty(spte_to_pfn(*sptep));
1214
1215         return was_writable;
1216 }
1217
1218 /*
1219  * Gets the GFN ready for another round of dirty logging by clearing the
1220  *      - D bit on ad-enabled SPTEs, and
1221  *      - W bit on ad-disabled SPTEs.
1222  * Returns true iff any D or W bits were cleared.
1223  */
1224 static bool __rmap_clear_dirty(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1225                                const struct kvm_memory_slot *slot)
1226 {
1227         u64 *sptep;
1228         struct rmap_iterator iter;
1229         bool flush = false;
1230
1231         for_each_rmap_spte(rmap_head, &iter, sptep)
1232                 if (spte_ad_need_write_protect(*sptep))
1233                         flush |= spte_wrprot_for_clear_dirty(sptep);
1234                 else
1235                         flush |= spte_clear_dirty(sptep);
1236
1237         return flush;
1238 }
1239
1240 /**
1241  * kvm_mmu_write_protect_pt_masked - write protect selected PT level pages
1242  * @kvm: kvm instance
1243  * @slot: slot to protect
1244  * @gfn_offset: start of the BITS_PER_LONG pages we care about
1245  * @mask: indicates which pages we should protect
1246  *
1247  * Used when we do not need to care about huge page mappings.
1248  */
1249 static void kvm_mmu_write_protect_pt_masked(struct kvm *kvm,
1250                                      struct kvm_memory_slot *slot,
1251                                      gfn_t gfn_offset, unsigned long mask)
1252 {
1253         struct kvm_rmap_head *rmap_head;
1254
1255         if (is_tdp_mmu_enabled(kvm))
1256                 kvm_tdp_mmu_clear_dirty_pt_masked(kvm, slot,
1257                                 slot->base_gfn + gfn_offset, mask, true);
1258
1259         if (!kvm_memslots_have_rmaps(kvm))
1260                 return;
1261
1262         while (mask) {
1263                 rmap_head = gfn_to_rmap(slot->base_gfn + gfn_offset + __ffs(mask),
1264                                         PG_LEVEL_4K, slot);
1265                 rmap_write_protect(rmap_head, false);
1266
1267                 /* clear the first set bit */
1268                 mask &= mask - 1;
1269         }
1270 }
1271
1272 /**
1273  * kvm_mmu_clear_dirty_pt_masked - clear MMU D-bit for PT level pages, or write
1274  * protect the page if the D-bit isn't supported.
1275  * @kvm: kvm instance
1276  * @slot: slot to clear D-bit
1277  * @gfn_offset: start of the BITS_PER_LONG pages we care about
1278  * @mask: indicates which pages we should clear D-bit
1279  *
1280  * Used for PML to re-log the dirty GPAs after userspace querying dirty_bitmap.
1281  */
1282 static void kvm_mmu_clear_dirty_pt_masked(struct kvm *kvm,
1283                                          struct kvm_memory_slot *slot,
1284                                          gfn_t gfn_offset, unsigned long mask)
1285 {
1286         struct kvm_rmap_head *rmap_head;
1287
1288         if (is_tdp_mmu_enabled(kvm))
1289                 kvm_tdp_mmu_clear_dirty_pt_masked(kvm, slot,
1290                                 slot->base_gfn + gfn_offset, mask, false);
1291
1292         if (!kvm_memslots_have_rmaps(kvm))
1293                 return;
1294
1295         while (mask) {
1296                 rmap_head = gfn_to_rmap(slot->base_gfn + gfn_offset + __ffs(mask),
1297                                         PG_LEVEL_4K, slot);
1298                 __rmap_clear_dirty(kvm, rmap_head, slot);
1299
1300                 /* clear the first set bit */
1301                 mask &= mask - 1;
1302         }
1303 }
1304
1305 /**
1306  * kvm_arch_mmu_enable_log_dirty_pt_masked - enable dirty logging for selected
1307  * PT level pages.
1308  *
1309  * It calls kvm_mmu_write_protect_pt_masked to write protect selected pages to
1310  * enable dirty logging for them.
1311  *
1312  * We need to care about huge page mappings: e.g. during dirty logging we may
1313  * have such mappings.
1314  */
1315 void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm,
1316                                 struct kvm_memory_slot *slot,
1317                                 gfn_t gfn_offset, unsigned long mask)
1318 {
1319         /*
1320          * Huge pages are NOT write protected when we start dirty logging in
1321          * initially-all-set mode; must write protect them here so that they
1322          * are split to 4K on the first write.
1323          *
1324          * The gfn_offset is guaranteed to be aligned to 64, but the base_gfn
1325          * of memslot has no such restriction, so the range can cross two large
1326          * pages.
1327          */
1328         if (kvm_dirty_log_manual_protect_and_init_set(kvm)) {
1329                 gfn_t start = slot->base_gfn + gfn_offset + __ffs(mask);
1330                 gfn_t end = slot->base_gfn + gfn_offset + __fls(mask);
1331
1332                 if (READ_ONCE(eager_page_split))
1333                         kvm_mmu_try_split_huge_pages(kvm, slot, start, end, PG_LEVEL_4K);
1334
1335                 kvm_mmu_slot_gfn_write_protect(kvm, slot, start, PG_LEVEL_2M);
1336
1337                 /* Cross two large pages? */
1338                 if (ALIGN(start << PAGE_SHIFT, PMD_SIZE) !=
1339                     ALIGN(end << PAGE_SHIFT, PMD_SIZE))
1340                         kvm_mmu_slot_gfn_write_protect(kvm, slot, end,
1341                                                        PG_LEVEL_2M);
1342         }
1343
1344         /* Now handle 4K PTEs.  */
1345         if (kvm_x86_ops.cpu_dirty_log_size)
1346                 kvm_mmu_clear_dirty_pt_masked(kvm, slot, gfn_offset, mask);
1347         else
1348                 kvm_mmu_write_protect_pt_masked(kvm, slot, gfn_offset, mask);
1349 }
1350
1351 int kvm_cpu_dirty_log_size(void)
1352 {
1353         return kvm_x86_ops.cpu_dirty_log_size;
1354 }
1355
1356 bool kvm_mmu_slot_gfn_write_protect(struct kvm *kvm,
1357                                     struct kvm_memory_slot *slot, u64 gfn,
1358                                     int min_level)
1359 {
1360         struct kvm_rmap_head *rmap_head;
1361         int i;
1362         bool write_protected = false;
1363
1364         if (kvm_memslots_have_rmaps(kvm)) {
1365                 for (i = min_level; i <= KVM_MAX_HUGEPAGE_LEVEL; ++i) {
1366                         rmap_head = gfn_to_rmap(gfn, i, slot);
1367                         write_protected |= rmap_write_protect(rmap_head, true);
1368                 }
1369         }
1370
1371         if (is_tdp_mmu_enabled(kvm))
1372                 write_protected |=
1373                         kvm_tdp_mmu_write_protect_gfn(kvm, slot, gfn, min_level);
1374
1375         return write_protected;
1376 }
1377
1378 static bool kvm_vcpu_write_protect_gfn(struct kvm_vcpu *vcpu, u64 gfn)
1379 {
1380         struct kvm_memory_slot *slot;
1381
1382         slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
1383         return kvm_mmu_slot_gfn_write_protect(vcpu->kvm, slot, gfn, PG_LEVEL_4K);
1384 }
1385
1386 static bool kvm_zap_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1387                           const struct kvm_memory_slot *slot)
1388 {
1389         return pte_list_destroy(kvm, rmap_head);
1390 }
1391
1392 static bool kvm_unmap_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1393                             struct kvm_memory_slot *slot, gfn_t gfn, int level,
1394                             pte_t unused)
1395 {
1396         return kvm_zap_rmapp(kvm, rmap_head, slot);
1397 }
1398
1399 static bool kvm_set_pte_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1400                               struct kvm_memory_slot *slot, gfn_t gfn, int level,
1401                               pte_t pte)
1402 {
1403         u64 *sptep;
1404         struct rmap_iterator iter;
1405         bool need_flush = false;
1406         u64 new_spte;
1407         kvm_pfn_t new_pfn;
1408
1409         WARN_ON(pte_huge(pte));
1410         new_pfn = pte_pfn(pte);
1411
1412 restart:
1413         for_each_rmap_spte(rmap_head, &iter, sptep) {
1414                 rmap_printk("spte %p %llx gfn %llx (%d)\n",
1415                             sptep, *sptep, gfn, level);
1416
1417                 need_flush = true;
1418
1419                 if (pte_write(pte)) {
1420                         pte_list_remove(kvm, rmap_head, sptep);
1421                         goto restart;
1422                 } else {
1423                         new_spte = kvm_mmu_changed_pte_notifier_make_spte(
1424                                         *sptep, new_pfn);
1425
1426                         mmu_spte_clear_track_bits(kvm, sptep);
1427                         mmu_spte_set(sptep, new_spte);
1428                 }
1429         }
1430
1431         if (need_flush && kvm_available_flush_tlb_with_range()) {
1432                 kvm_flush_remote_tlbs_with_address(kvm, gfn, 1);
1433                 return false;
1434         }
1435
1436         return need_flush;
1437 }
1438
1439 struct slot_rmap_walk_iterator {
1440         /* input fields. */
1441         const struct kvm_memory_slot *slot;
1442         gfn_t start_gfn;
1443         gfn_t end_gfn;
1444         int start_level;
1445         int end_level;
1446
1447         /* output fields. */
1448         gfn_t gfn;
1449         struct kvm_rmap_head *rmap;
1450         int level;
1451
1452         /* private field. */
1453         struct kvm_rmap_head *end_rmap;
1454 };
1455
1456 static void
1457 rmap_walk_init_level(struct slot_rmap_walk_iterator *iterator, int level)
1458 {
1459         iterator->level = level;
1460         iterator->gfn = iterator->start_gfn;
1461         iterator->rmap = gfn_to_rmap(iterator->gfn, level, iterator->slot);
1462         iterator->end_rmap = gfn_to_rmap(iterator->end_gfn, level, iterator->slot);
1463 }
1464
1465 static void
1466 slot_rmap_walk_init(struct slot_rmap_walk_iterator *iterator,
1467                     const struct kvm_memory_slot *slot, int start_level,
1468                     int end_level, gfn_t start_gfn, gfn_t end_gfn)
1469 {
1470         iterator->slot = slot;
1471         iterator->start_level = start_level;
1472         iterator->end_level = end_level;
1473         iterator->start_gfn = start_gfn;
1474         iterator->end_gfn = end_gfn;
1475
1476         rmap_walk_init_level(iterator, iterator->start_level);
1477 }
1478
1479 static bool slot_rmap_walk_okay(struct slot_rmap_walk_iterator *iterator)
1480 {
1481         return !!iterator->rmap;
1482 }
1483
1484 static void slot_rmap_walk_next(struct slot_rmap_walk_iterator *iterator)
1485 {
1486         while (++iterator->rmap <= iterator->end_rmap) {
1487                 iterator->gfn += (1UL << KVM_HPAGE_GFN_SHIFT(iterator->level));
1488
1489                 if (iterator->rmap->val)
1490                         return;
1491         }
1492
1493         if (++iterator->level > iterator->end_level) {
1494                 iterator->rmap = NULL;
1495                 return;
1496         }
1497
1498         rmap_walk_init_level(iterator, iterator->level);
1499 }
1500
1501 #define for_each_slot_rmap_range(_slot_, _start_level_, _end_level_,    \
1502            _start_gfn, _end_gfn, _iter_)                                \
1503         for (slot_rmap_walk_init(_iter_, _slot_, _start_level_,         \
1504                                  _end_level_, _start_gfn, _end_gfn);    \
1505              slot_rmap_walk_okay(_iter_);                               \
1506              slot_rmap_walk_next(_iter_))
1507
1508 typedef bool (*rmap_handler_t)(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1509                                struct kvm_memory_slot *slot, gfn_t gfn,
1510                                int level, pte_t pte);
1511
1512 static __always_inline bool kvm_handle_gfn_range(struct kvm *kvm,
1513                                                  struct kvm_gfn_range *range,
1514                                                  rmap_handler_t handler)
1515 {
1516         struct slot_rmap_walk_iterator iterator;
1517         bool ret = false;
1518
1519         for_each_slot_rmap_range(range->slot, PG_LEVEL_4K, KVM_MAX_HUGEPAGE_LEVEL,
1520                                  range->start, range->end - 1, &iterator)
1521                 ret |= handler(kvm, iterator.rmap, range->slot, iterator.gfn,
1522                                iterator.level, range->pte);
1523
1524         return ret;
1525 }
1526
1527 bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range)
1528 {
1529         bool flush = false;
1530
1531         if (kvm_memslots_have_rmaps(kvm))
1532                 flush = kvm_handle_gfn_range(kvm, range, kvm_unmap_rmapp);
1533
1534         if (is_tdp_mmu_enabled(kvm))
1535                 flush = kvm_tdp_mmu_unmap_gfn_range(kvm, range, flush);
1536
1537         return flush;
1538 }
1539
1540 bool kvm_set_spte_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
1541 {
1542         bool flush = false;
1543
1544         if (kvm_memslots_have_rmaps(kvm))
1545                 flush = kvm_handle_gfn_range(kvm, range, kvm_set_pte_rmapp);
1546
1547         if (is_tdp_mmu_enabled(kvm))
1548                 flush |= kvm_tdp_mmu_set_spte_gfn(kvm, range);
1549
1550         return flush;
1551 }
1552
1553 static bool kvm_age_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1554                           struct kvm_memory_slot *slot, gfn_t gfn, int level,
1555                           pte_t unused)
1556 {
1557         u64 *sptep;
1558         struct rmap_iterator iter;
1559         int young = 0;
1560
1561         for_each_rmap_spte(rmap_head, &iter, sptep)
1562                 young |= mmu_spte_age(sptep);
1563
1564         return young;
1565 }
1566
1567 static bool kvm_test_age_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1568                                struct kvm_memory_slot *slot, gfn_t gfn,
1569                                int level, pte_t unused)
1570 {
1571         u64 *sptep;
1572         struct rmap_iterator iter;
1573
1574         for_each_rmap_spte(rmap_head, &iter, sptep)
1575                 if (is_accessed_spte(*sptep))
1576                         return true;
1577         return false;
1578 }
1579
1580 #define RMAP_RECYCLE_THRESHOLD 1000
1581
1582 static void rmap_add(struct kvm_vcpu *vcpu, struct kvm_memory_slot *slot,
1583                      u64 *spte, gfn_t gfn)
1584 {
1585         struct kvm_mmu_page *sp;
1586         struct kvm_rmap_head *rmap_head;
1587         int rmap_count;
1588
1589         sp = sptep_to_sp(spte);
1590         kvm_mmu_page_set_gfn(sp, spte - sp->spt, gfn);
1591         rmap_head = gfn_to_rmap(gfn, sp->role.level, slot);
1592         rmap_count = pte_list_add(vcpu, spte, rmap_head);
1593
1594         if (rmap_count > RMAP_RECYCLE_THRESHOLD) {
1595                 kvm_unmap_rmapp(vcpu->kvm, rmap_head, NULL, gfn, sp->role.level, __pte(0));
1596                 kvm_flush_remote_tlbs_with_address(
1597                                 vcpu->kvm, sp->gfn, KVM_PAGES_PER_HPAGE(sp->role.level));
1598         }
1599 }
1600
1601 bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
1602 {
1603         bool young = false;
1604
1605         if (kvm_memslots_have_rmaps(kvm))
1606                 young = kvm_handle_gfn_range(kvm, range, kvm_age_rmapp);
1607
1608         if (is_tdp_mmu_enabled(kvm))
1609                 young |= kvm_tdp_mmu_age_gfn_range(kvm, range);
1610
1611         return young;
1612 }
1613
1614 bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
1615 {
1616         bool young = false;
1617
1618         if (kvm_memslots_have_rmaps(kvm))
1619                 young = kvm_handle_gfn_range(kvm, range, kvm_test_age_rmapp);
1620
1621         if (is_tdp_mmu_enabled(kvm))
1622                 young |= kvm_tdp_mmu_test_age_gfn(kvm, range);
1623
1624         return young;
1625 }
1626
1627 #ifdef MMU_DEBUG
1628 static int is_empty_shadow_page(u64 *spt)
1629 {
1630         u64 *pos;
1631         u64 *end;
1632
1633         for (pos = spt, end = pos + PAGE_SIZE / sizeof(u64); pos != end; pos++)
1634                 if (is_shadow_present_pte(*pos)) {
1635                         printk(KERN_ERR "%s: %p %llx\n", __func__,
1636                                pos, *pos);
1637                         return 0;
1638                 }
1639         return 1;
1640 }
1641 #endif
1642
1643 /*
1644  * This value is the sum of all of the kvm instances's
1645  * kvm->arch.n_used_mmu_pages values.  We need a global,
1646  * aggregate version in order to make the slab shrinker
1647  * faster
1648  */
1649 static inline void kvm_mod_used_mmu_pages(struct kvm *kvm, long nr)
1650 {
1651         kvm->arch.n_used_mmu_pages += nr;
1652         percpu_counter_add(&kvm_total_used_mmu_pages, nr);
1653 }
1654
1655 static void kvm_mmu_free_page(struct kvm_mmu_page *sp)
1656 {
1657         MMU_WARN_ON(!is_empty_shadow_page(sp->spt));
1658         hlist_del(&sp->hash_link);
1659         list_del(&sp->link);
1660         free_page((unsigned long)sp->spt);
1661         if (!sp->role.direct)
1662                 free_page((unsigned long)sp->gfns);
1663         kmem_cache_free(mmu_page_header_cache, sp);
1664 }
1665
1666 static unsigned kvm_page_table_hashfn(gfn_t gfn)
1667 {
1668         return hash_64(gfn, KVM_MMU_HASH_SHIFT);
1669 }
1670
1671 static void mmu_page_add_parent_pte(struct kvm_vcpu *vcpu,
1672                                     struct kvm_mmu_page *sp, u64 *parent_pte)
1673 {
1674         if (!parent_pte)
1675                 return;
1676
1677         pte_list_add(vcpu, parent_pte, &sp->parent_ptes);
1678 }
1679
1680 static void mmu_page_remove_parent_pte(struct kvm_mmu_page *sp,
1681                                        u64 *parent_pte)
1682 {
1683         __pte_list_remove(parent_pte, &sp->parent_ptes);
1684 }
1685
1686 static void drop_parent_pte(struct kvm_mmu_page *sp,
1687                             u64 *parent_pte)
1688 {
1689         mmu_page_remove_parent_pte(sp, parent_pte);
1690         mmu_spte_clear_no_track(parent_pte);
1691 }
1692
1693 static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu, int direct)
1694 {
1695         struct kvm_mmu_page *sp;
1696
1697         sp = kvm_mmu_memory_cache_alloc(&vcpu->arch.mmu_page_header_cache);
1698         sp->spt = kvm_mmu_memory_cache_alloc(&vcpu->arch.mmu_shadow_page_cache);
1699         if (!direct)
1700                 sp->gfns = kvm_mmu_memory_cache_alloc(&vcpu->arch.mmu_gfn_array_cache);
1701         set_page_private(virt_to_page(sp->spt), (unsigned long)sp);
1702
1703         /*
1704          * active_mmu_pages must be a FIFO list, as kvm_zap_obsolete_pages()
1705          * depends on valid pages being added to the head of the list.  See
1706          * comments in kvm_zap_obsolete_pages().
1707          */
1708         sp->mmu_valid_gen = vcpu->kvm->arch.mmu_valid_gen;
1709         list_add(&sp->link, &vcpu->kvm->arch.active_mmu_pages);
1710         kvm_mod_used_mmu_pages(vcpu->kvm, +1);
1711         return sp;
1712 }
1713
1714 static void mark_unsync(u64 *spte);
1715 static void kvm_mmu_mark_parents_unsync(struct kvm_mmu_page *sp)
1716 {
1717         u64 *sptep;
1718         struct rmap_iterator iter;
1719
1720         for_each_rmap_spte(&sp->parent_ptes, &iter, sptep) {
1721                 mark_unsync(sptep);
1722         }
1723 }
1724
1725 static void mark_unsync(u64 *spte)
1726 {
1727         struct kvm_mmu_page *sp;
1728         unsigned int index;
1729
1730         sp = sptep_to_sp(spte);
1731         index = spte - sp->spt;
1732         if (__test_and_set_bit(index, sp->unsync_child_bitmap))
1733                 return;
1734         if (sp->unsync_children++)
1735                 return;
1736         kvm_mmu_mark_parents_unsync(sp);
1737 }
1738
1739 static int nonpaging_sync_page(struct kvm_vcpu *vcpu,
1740                                struct kvm_mmu_page *sp)
1741 {
1742         return -1;
1743 }
1744
1745 #define KVM_PAGE_ARRAY_NR 16
1746
1747 struct kvm_mmu_pages {
1748         struct mmu_page_and_offset {
1749                 struct kvm_mmu_page *sp;
1750                 unsigned int idx;
1751         } page[KVM_PAGE_ARRAY_NR];
1752         unsigned int nr;
1753 };
1754
1755 static int mmu_pages_add(struct kvm_mmu_pages *pvec, struct kvm_mmu_page *sp,
1756                          int idx)
1757 {
1758         int i;
1759
1760         if (sp->unsync)
1761                 for (i=0; i < pvec->nr; i++)
1762                         if (pvec->page[i].sp == sp)
1763                                 return 0;
1764
1765         pvec->page[pvec->nr].sp = sp;
1766         pvec->page[pvec->nr].idx = idx;
1767         pvec->nr++;
1768         return (pvec->nr == KVM_PAGE_ARRAY_NR);
1769 }
1770
1771 static inline void clear_unsync_child_bit(struct kvm_mmu_page *sp, int idx)
1772 {
1773         --sp->unsync_children;
1774         WARN_ON((int)sp->unsync_children < 0);
1775         __clear_bit(idx, sp->unsync_child_bitmap);
1776 }
1777
1778 static int __mmu_unsync_walk(struct kvm_mmu_page *sp,
1779                            struct kvm_mmu_pages *pvec)
1780 {
1781         int i, ret, nr_unsync_leaf = 0;
1782
1783         for_each_set_bit(i, sp->unsync_child_bitmap, 512) {
1784                 struct kvm_mmu_page *child;
1785                 u64 ent = sp->spt[i];
1786
1787                 if (!is_shadow_present_pte(ent) || is_large_pte(ent)) {
1788                         clear_unsync_child_bit(sp, i);
1789                         continue;
1790                 }
1791
1792                 child = to_shadow_page(ent & PT64_BASE_ADDR_MASK);
1793
1794                 if (child->unsync_children) {
1795                         if (mmu_pages_add(pvec, child, i))
1796                                 return -ENOSPC;
1797
1798                         ret = __mmu_unsync_walk(child, pvec);
1799                         if (!ret) {
1800                                 clear_unsync_child_bit(sp, i);
1801                                 continue;
1802                         } else if (ret > 0) {
1803                                 nr_unsync_leaf += ret;
1804                         } else
1805                                 return ret;
1806                 } else if (child->unsync) {
1807                         nr_unsync_leaf++;
1808                         if (mmu_pages_add(pvec, child, i))
1809                                 return -ENOSPC;
1810                 } else
1811                         clear_unsync_child_bit(sp, i);
1812         }
1813
1814         return nr_unsync_leaf;
1815 }
1816
1817 #define INVALID_INDEX (-1)
1818
1819 static int mmu_unsync_walk(struct kvm_mmu_page *sp,
1820                            struct kvm_mmu_pages *pvec)
1821 {
1822         pvec->nr = 0;
1823         if (!sp->unsync_children)
1824                 return 0;
1825
1826         mmu_pages_add(pvec, sp, INVALID_INDEX);
1827         return __mmu_unsync_walk(sp, pvec);
1828 }
1829
1830 static void kvm_unlink_unsync_page(struct kvm *kvm, struct kvm_mmu_page *sp)
1831 {
1832         WARN_ON(!sp->unsync);
1833         trace_kvm_mmu_sync_page(sp);
1834         sp->unsync = 0;
1835         --kvm->stat.mmu_unsync;
1836 }
1837
1838 static bool kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp,
1839                                      struct list_head *invalid_list);
1840 static void kvm_mmu_commit_zap_page(struct kvm *kvm,
1841                                     struct list_head *invalid_list);
1842
1843 static bool sp_has_gptes(struct kvm_mmu_page *sp)
1844 {
1845         if (sp->role.direct)
1846                 return false;
1847
1848         if (sp->role.passthrough)
1849                 return false;
1850
1851         return true;
1852 }
1853
1854 #define for_each_valid_sp(_kvm, _sp, _list)                             \
1855         hlist_for_each_entry(_sp, _list, hash_link)                     \
1856                 if (is_obsolete_sp((_kvm), (_sp))) {                    \
1857                 } else
1858
1859 #define for_each_gfn_valid_sp_with_gptes(_kvm, _sp, _gfn)               \
1860         for_each_valid_sp(_kvm, _sp,                                    \
1861           &(_kvm)->arch.mmu_page_hash[kvm_page_table_hashfn(_gfn)])     \
1862                 if ((_sp)->gfn != (_gfn) || !sp_has_gptes(_sp)) {} else
1863
1864 static int kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
1865                          struct list_head *invalid_list)
1866 {
1867         int ret = vcpu->arch.mmu->sync_page(vcpu, sp);
1868
1869         if (ret < 0)
1870                 kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list);
1871         return ret;
1872 }
1873
1874 static bool kvm_mmu_remote_flush_or_zap(struct kvm *kvm,
1875                                         struct list_head *invalid_list,
1876                                         bool remote_flush)
1877 {
1878         if (!remote_flush && list_empty(invalid_list))
1879                 return false;
1880
1881         if (!list_empty(invalid_list))
1882                 kvm_mmu_commit_zap_page(kvm, invalid_list);
1883         else
1884                 kvm_flush_remote_tlbs(kvm);
1885         return true;
1886 }
1887
1888 static bool is_obsolete_sp(struct kvm *kvm, struct kvm_mmu_page *sp)
1889 {
1890         if (sp->role.invalid)
1891                 return true;
1892
1893         /* TDP MMU pages due not use the MMU generation. */
1894         return !sp->tdp_mmu_page &&
1895                unlikely(sp->mmu_valid_gen != kvm->arch.mmu_valid_gen);
1896 }
1897
1898 struct mmu_page_path {
1899         struct kvm_mmu_page *parent[PT64_ROOT_MAX_LEVEL];
1900         unsigned int idx[PT64_ROOT_MAX_LEVEL];
1901 };
1902
1903 #define for_each_sp(pvec, sp, parents, i)                       \
1904                 for (i = mmu_pages_first(&pvec, &parents);      \
1905                         i < pvec.nr && ({ sp = pvec.page[i].sp; 1;});   \
1906                         i = mmu_pages_next(&pvec, &parents, i))
1907
1908 static int mmu_pages_next(struct kvm_mmu_pages *pvec,
1909                           struct mmu_page_path *parents,
1910                           int i)
1911 {
1912         int n;
1913
1914         for (n = i+1; n < pvec->nr; n++) {
1915                 struct kvm_mmu_page *sp = pvec->page[n].sp;
1916                 unsigned idx = pvec->page[n].idx;
1917                 int level = sp->role.level;
1918
1919                 parents->idx[level-1] = idx;
1920                 if (level == PG_LEVEL_4K)
1921                         break;
1922
1923                 parents->parent[level-2] = sp;
1924         }
1925
1926         return n;
1927 }
1928
1929 static int mmu_pages_first(struct kvm_mmu_pages *pvec,
1930                            struct mmu_page_path *parents)
1931 {
1932         struct kvm_mmu_page *sp;
1933         int level;
1934
1935         if (pvec->nr == 0)
1936                 return 0;
1937
1938         WARN_ON(pvec->page[0].idx != INVALID_INDEX);
1939
1940         sp = pvec->page[0].sp;
1941         level = sp->role.level;
1942         WARN_ON(level == PG_LEVEL_4K);
1943
1944         parents->parent[level-2] = sp;
1945
1946         /* Also set up a sentinel.  Further entries in pvec are all
1947          * children of sp, so this element is never overwritten.
1948          */
1949         parents->parent[level-1] = NULL;
1950         return mmu_pages_next(pvec, parents, 0);
1951 }
1952
1953 static void mmu_pages_clear_parents(struct mmu_page_path *parents)
1954 {
1955         struct kvm_mmu_page *sp;
1956         unsigned int level = 0;
1957
1958         do {
1959                 unsigned int idx = parents->idx[level];
1960                 sp = parents->parent[level];
1961                 if (!sp)
1962                         return;
1963
1964                 WARN_ON(idx == INVALID_INDEX);
1965                 clear_unsync_child_bit(sp, idx);
1966                 level++;
1967         } while (!sp->unsync_children);
1968 }
1969
1970 static int mmu_sync_children(struct kvm_vcpu *vcpu,
1971                              struct kvm_mmu_page *parent, bool can_yield)
1972 {
1973         int i;
1974         struct kvm_mmu_page *sp;
1975         struct mmu_page_path parents;
1976         struct kvm_mmu_pages pages;
1977         LIST_HEAD(invalid_list);
1978         bool flush = false;
1979
1980         while (mmu_unsync_walk(parent, &pages)) {
1981                 bool protected = false;
1982
1983                 for_each_sp(pages, sp, parents, i)
1984                         protected |= kvm_vcpu_write_protect_gfn(vcpu, sp->gfn);
1985
1986                 if (protected) {
1987                         kvm_mmu_remote_flush_or_zap(vcpu->kvm, &invalid_list, true);
1988                         flush = false;
1989                 }
1990
1991                 for_each_sp(pages, sp, parents, i) {
1992                         kvm_unlink_unsync_page(vcpu->kvm, sp);
1993                         flush |= kvm_sync_page(vcpu, sp, &invalid_list) > 0;
1994                         mmu_pages_clear_parents(&parents);
1995                 }
1996                 if (need_resched() || rwlock_needbreak(&vcpu->kvm->mmu_lock)) {
1997                         kvm_mmu_remote_flush_or_zap(vcpu->kvm, &invalid_list, flush);
1998                         if (!can_yield) {
1999                                 kvm_make_request(KVM_REQ_MMU_SYNC, vcpu);
2000                                 return -EINTR;
2001                         }
2002
2003                         cond_resched_rwlock_write(&vcpu->kvm->mmu_lock);
2004                         flush = false;
2005                 }
2006         }
2007
2008         kvm_mmu_remote_flush_or_zap(vcpu->kvm, &invalid_list, flush);
2009         return 0;
2010 }
2011
2012 static void __clear_sp_write_flooding_count(struct kvm_mmu_page *sp)
2013 {
2014         atomic_set(&sp->write_flooding_count,  0);
2015 }
2016
2017 static void clear_sp_write_flooding_count(u64 *spte)
2018 {
2019         __clear_sp_write_flooding_count(sptep_to_sp(spte));
2020 }
2021
2022 static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu,
2023                                              gfn_t gfn,
2024                                              gva_t gaddr,
2025                                              unsigned level,
2026                                              int direct,
2027                                              unsigned int access)
2028 {
2029         bool direct_mmu = vcpu->arch.mmu->root_role.direct;
2030         union kvm_mmu_page_role role;
2031         struct hlist_head *sp_list;
2032         unsigned quadrant;
2033         struct kvm_mmu_page *sp;
2034         int ret;
2035         int collisions = 0;
2036         LIST_HEAD(invalid_list);
2037
2038         role = vcpu->arch.mmu->root_role;
2039         role.level = level;
2040         role.direct = direct;
2041         role.access = access;
2042         if (role.has_4_byte_gpte) {
2043                 quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level));
2044                 quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1;
2045                 role.quadrant = quadrant;
2046         }
2047         if (level <= vcpu->arch.mmu->cpu_role.base.level)
2048                 role.passthrough = 0;
2049
2050         sp_list = &vcpu->kvm->arch.mmu_page_hash[kvm_page_table_hashfn(gfn)];
2051         for_each_valid_sp(vcpu->kvm, sp, sp_list) {
2052                 if (sp->gfn != gfn) {
2053                         collisions++;
2054                         continue;
2055                 }
2056
2057                 if (sp->role.word != role.word) {
2058                         /*
2059                          * If the guest is creating an upper-level page, zap
2060                          * unsync pages for the same gfn.  While it's possible
2061                          * the guest is using recursive page tables, in all
2062                          * likelihood the guest has stopped using the unsync
2063                          * page and is installing a completely unrelated page.
2064                          * Unsync pages must not be left as is, because the new
2065                          * upper-level page will be write-protected.
2066                          */
2067                         if (level > PG_LEVEL_4K && sp->unsync)
2068                                 kvm_mmu_prepare_zap_page(vcpu->kvm, sp,
2069                                                          &invalid_list);
2070                         continue;
2071                 }
2072
2073                 if (direct_mmu)
2074                         goto trace_get_page;
2075
2076                 if (sp->unsync) {
2077                         /*
2078                          * The page is good, but is stale.  kvm_sync_page does
2079                          * get the latest guest state, but (unlike mmu_unsync_children)
2080                          * it doesn't write-protect the page or mark it synchronized!
2081                          * This way the validity of the mapping is ensured, but the
2082                          * overhead of write protection is not incurred until the
2083                          * guest invalidates the TLB mapping.  This allows multiple
2084                          * SPs for a single gfn to be unsync.
2085                          *
2086                          * If the sync fails, the page is zapped.  If so, break
2087                          * in order to rebuild it.
2088                          */
2089                         ret = kvm_sync_page(vcpu, sp, &invalid_list);
2090                         if (ret < 0)
2091                                 break;
2092
2093                         WARN_ON(!list_empty(&invalid_list));
2094                         if (ret > 0)
2095                                 kvm_flush_remote_tlbs(vcpu->kvm);
2096                 }
2097
2098                 __clear_sp_write_flooding_count(sp);
2099
2100 trace_get_page:
2101                 trace_kvm_mmu_get_page(sp, false);
2102                 goto out;
2103         }
2104
2105         ++vcpu->kvm->stat.mmu_cache_miss;
2106
2107         sp = kvm_mmu_alloc_page(vcpu, direct);
2108
2109         sp->gfn = gfn;
2110         sp->role = role;
2111         hlist_add_head(&sp->hash_link, sp_list);
2112         if (sp_has_gptes(sp)) {
2113                 account_shadowed(vcpu->kvm, sp);
2114                 if (level == PG_LEVEL_4K && kvm_vcpu_write_protect_gfn(vcpu, gfn))
2115                         kvm_flush_remote_tlbs_with_address(vcpu->kvm, gfn, 1);
2116         }
2117         trace_kvm_mmu_get_page(sp, true);
2118 out:
2119         kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
2120
2121         if (collisions > vcpu->kvm->stat.max_mmu_page_hash_collisions)
2122                 vcpu->kvm->stat.max_mmu_page_hash_collisions = collisions;
2123         return sp;
2124 }
2125
2126 static void shadow_walk_init_using_root(struct kvm_shadow_walk_iterator *iterator,
2127                                         struct kvm_vcpu *vcpu, hpa_t root,
2128                                         u64 addr)
2129 {
2130         iterator->addr = addr;
2131         iterator->shadow_addr = root;
2132         iterator->level = vcpu->arch.mmu->root_role.level;
2133
2134         if (iterator->level >= PT64_ROOT_4LEVEL &&
2135             vcpu->arch.mmu->cpu_role.base.level < PT64_ROOT_4LEVEL &&
2136             !vcpu->arch.mmu->root_role.direct)
2137                 iterator->level = PT32E_ROOT_LEVEL;
2138
2139         if (iterator->level == PT32E_ROOT_LEVEL) {
2140                 /*
2141                  * prev_root is currently only used for 64-bit hosts. So only
2142                  * the active root_hpa is valid here.
2143                  */
2144                 BUG_ON(root != vcpu->arch.mmu->root.hpa);
2145
2146                 iterator->shadow_addr
2147                         = vcpu->arch.mmu->pae_root[(addr >> 30) & 3];
2148                 iterator->shadow_addr &= PT64_BASE_ADDR_MASK;
2149                 --iterator->level;
2150                 if (!iterator->shadow_addr)
2151                         iterator->level = 0;
2152         }
2153 }
2154
2155 static void shadow_walk_init(struct kvm_shadow_walk_iterator *iterator,
2156                              struct kvm_vcpu *vcpu, u64 addr)
2157 {
2158         shadow_walk_init_using_root(iterator, vcpu, vcpu->arch.mmu->root.hpa,
2159                                     addr);
2160 }
2161
2162 static bool shadow_walk_okay(struct kvm_shadow_walk_iterator *iterator)
2163 {
2164         if (iterator->level < PG_LEVEL_4K)
2165                 return false;
2166
2167         iterator->index = SHADOW_PT_INDEX(iterator->addr, iterator->level);
2168         iterator->sptep = ((u64 *)__va(iterator->shadow_addr)) + iterator->index;
2169         return true;
2170 }
2171
2172 static void __shadow_walk_next(struct kvm_shadow_walk_iterator *iterator,
2173                                u64 spte)
2174 {
2175         if (!is_shadow_present_pte(spte) || is_last_spte(spte, iterator->level)) {
2176                 iterator->level = 0;
2177                 return;
2178         }
2179
2180         iterator->shadow_addr = spte & PT64_BASE_ADDR_MASK;
2181         --iterator->level;
2182 }
2183
2184 static void shadow_walk_next(struct kvm_shadow_walk_iterator *iterator)
2185 {
2186         __shadow_walk_next(iterator, *iterator->sptep);
2187 }
2188
2189 static void link_shadow_page(struct kvm_vcpu *vcpu, u64 *sptep,
2190                              struct kvm_mmu_page *sp)
2191 {
2192         u64 spte;
2193
2194         BUILD_BUG_ON(VMX_EPT_WRITABLE_MASK != PT_WRITABLE_MASK);
2195
2196         spte = make_nonleaf_spte(sp->spt, sp_ad_disabled(sp));
2197
2198         mmu_spte_set(sptep, spte);
2199
2200         mmu_page_add_parent_pte(vcpu, sp, sptep);
2201
2202         if (sp->unsync_children || sp->unsync)
2203                 mark_unsync(sptep);
2204 }
2205
2206 static void validate_direct_spte(struct kvm_vcpu *vcpu, u64 *sptep,
2207                                    unsigned direct_access)
2208 {
2209         if (is_shadow_present_pte(*sptep) && !is_large_pte(*sptep)) {
2210                 struct kvm_mmu_page *child;
2211
2212                 /*
2213                  * For the direct sp, if the guest pte's dirty bit
2214                  * changed form clean to dirty, it will corrupt the
2215                  * sp's access: allow writable in the read-only sp,
2216                  * so we should update the spte at this point to get
2217                  * a new sp with the correct access.
2218                  */
2219                 child = to_shadow_page(*sptep & PT64_BASE_ADDR_MASK);
2220                 if (child->role.access == direct_access)
2221                         return;
2222
2223                 drop_parent_pte(child, sptep);
2224                 kvm_flush_remote_tlbs_with_address(vcpu->kvm, child->gfn, 1);
2225         }
2226 }
2227
2228 /* Returns the number of zapped non-leaf child shadow pages. */
2229 static int mmu_page_zap_pte(struct kvm *kvm, struct kvm_mmu_page *sp,
2230                             u64 *spte, struct list_head *invalid_list)
2231 {
2232         u64 pte;
2233         struct kvm_mmu_page *child;
2234
2235         pte = *spte;
2236         if (is_shadow_present_pte(pte)) {
2237                 if (is_last_spte(pte, sp->role.level)) {
2238                         drop_spte(kvm, spte);
2239                 } else {
2240                         child = to_shadow_page(pte & PT64_BASE_ADDR_MASK);
2241                         drop_parent_pte(child, spte);
2242
2243                         /*
2244                          * Recursively zap nested TDP SPs, parentless SPs are
2245                          * unlikely to be used again in the near future.  This
2246                          * avoids retaining a large number of stale nested SPs.
2247                          */
2248                         if (tdp_enabled && invalid_list &&
2249                             child->role.guest_mode && !child->parent_ptes.val)
2250                                 return kvm_mmu_prepare_zap_page(kvm, child,
2251                                                                 invalid_list);
2252                 }
2253         } else if (is_mmio_spte(pte)) {
2254                 mmu_spte_clear_no_track(spte);
2255         }
2256         return 0;
2257 }
2258
2259 static int kvm_mmu_page_unlink_children(struct kvm *kvm,
2260                                         struct kvm_mmu_page *sp,
2261                                         struct list_head *invalid_list)
2262 {
2263         int zapped = 0;
2264         unsigned i;
2265
2266         for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
2267                 zapped += mmu_page_zap_pte(kvm, sp, sp->spt + i, invalid_list);
2268
2269         return zapped;
2270 }
2271
2272 static void kvm_mmu_unlink_parents(struct kvm_mmu_page *sp)
2273 {
2274         u64 *sptep;
2275         struct rmap_iterator iter;
2276
2277         while ((sptep = rmap_get_first(&sp->parent_ptes, &iter)))
2278                 drop_parent_pte(sp, sptep);
2279 }
2280
2281 static int mmu_zap_unsync_children(struct kvm *kvm,
2282                                    struct kvm_mmu_page *parent,
2283                                    struct list_head *invalid_list)
2284 {
2285         int i, zapped = 0;
2286         struct mmu_page_path parents;
2287         struct kvm_mmu_pages pages;
2288
2289         if (parent->role.level == PG_LEVEL_4K)
2290                 return 0;
2291
2292         while (mmu_unsync_walk(parent, &pages)) {
2293                 struct kvm_mmu_page *sp;
2294
2295                 for_each_sp(pages, sp, parents, i) {
2296                         kvm_mmu_prepare_zap_page(kvm, sp, invalid_list);
2297                         mmu_pages_clear_parents(&parents);
2298                         zapped++;
2299                 }
2300         }
2301
2302         return zapped;
2303 }
2304
2305 static bool __kvm_mmu_prepare_zap_page(struct kvm *kvm,
2306                                        struct kvm_mmu_page *sp,
2307                                        struct list_head *invalid_list,
2308                                        int *nr_zapped)
2309 {
2310         bool list_unstable, zapped_root = false;
2311
2312         trace_kvm_mmu_prepare_zap_page(sp);
2313         ++kvm->stat.mmu_shadow_zapped;
2314         *nr_zapped = mmu_zap_unsync_children(kvm, sp, invalid_list);
2315         *nr_zapped += kvm_mmu_page_unlink_children(kvm, sp, invalid_list);
2316         kvm_mmu_unlink_parents(sp);
2317
2318         /* Zapping children means active_mmu_pages has become unstable. */
2319         list_unstable = *nr_zapped;
2320
2321         if (!sp->role.invalid && sp_has_gptes(sp))
2322                 unaccount_shadowed(kvm, sp);
2323
2324         if (sp->unsync)
2325                 kvm_unlink_unsync_page(kvm, sp);
2326         if (!sp->root_count) {
2327                 /* Count self */
2328                 (*nr_zapped)++;
2329
2330                 /*
2331                  * Already invalid pages (previously active roots) are not on
2332                  * the active page list.  See list_del() in the "else" case of
2333                  * !sp->root_count.
2334                  */
2335                 if (sp->role.invalid)
2336                         list_add(&sp->link, invalid_list);
2337                 else
2338                         list_move(&sp->link, invalid_list);
2339                 kvm_mod_used_mmu_pages(kvm, -1);
2340         } else {
2341                 /*
2342                  * Remove the active root from the active page list, the root
2343                  * will be explicitly freed when the root_count hits zero.
2344                  */
2345                 list_del(&sp->link);
2346
2347                 /*
2348                  * Obsolete pages cannot be used on any vCPUs, see the comment
2349                  * in kvm_mmu_zap_all_fast().  Note, is_obsolete_sp() also
2350                  * treats invalid shadow pages as being obsolete.
2351                  */
2352                 zapped_root = !is_obsolete_sp(kvm, sp);
2353         }
2354
2355         if (sp->lpage_disallowed)
2356                 unaccount_huge_nx_page(kvm, sp);
2357
2358         sp->role.invalid = 1;
2359
2360         /*
2361          * Make the request to free obsolete roots after marking the root
2362          * invalid, otherwise other vCPUs may not see it as invalid.
2363          */
2364         if (zapped_root)
2365                 kvm_make_all_cpus_request(kvm, KVM_REQ_MMU_FREE_OBSOLETE_ROOTS);
2366         return list_unstable;
2367 }
2368
2369 static bool kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp,
2370                                      struct list_head *invalid_list)
2371 {
2372         int nr_zapped;
2373
2374         __kvm_mmu_prepare_zap_page(kvm, sp, invalid_list, &nr_zapped);
2375         return nr_zapped;
2376 }
2377
2378 static void kvm_mmu_commit_zap_page(struct kvm *kvm,
2379                                     struct list_head *invalid_list)
2380 {
2381         struct kvm_mmu_page *sp, *nsp;
2382
2383         if (list_empty(invalid_list))
2384                 return;
2385
2386         /*
2387          * We need to make sure everyone sees our modifications to
2388          * the page tables and see changes to vcpu->mode here. The barrier
2389          * in the kvm_flush_remote_tlbs() achieves this. This pairs
2390          * with vcpu_enter_guest and walk_shadow_page_lockless_begin/end.
2391          *
2392          * In addition, kvm_flush_remote_tlbs waits for all vcpus to exit
2393          * guest mode and/or lockless shadow page table walks.
2394          */
2395         kvm_flush_remote_tlbs(kvm);
2396
2397         list_for_each_entry_safe(sp, nsp, invalid_list, link) {
2398                 WARN_ON(!sp->role.invalid || sp->root_count);
2399                 kvm_mmu_free_page(sp);
2400         }
2401 }
2402
2403 static unsigned long kvm_mmu_zap_oldest_mmu_pages(struct kvm *kvm,
2404                                                   unsigned long nr_to_zap)
2405 {
2406         unsigned long total_zapped = 0;
2407         struct kvm_mmu_page *sp, *tmp;
2408         LIST_HEAD(invalid_list);
2409         bool unstable;
2410         int nr_zapped;
2411
2412         if (list_empty(&kvm->arch.active_mmu_pages))
2413                 return 0;
2414
2415 restart:
2416         list_for_each_entry_safe_reverse(sp, tmp, &kvm->arch.active_mmu_pages, link) {
2417                 /*
2418                  * Don't zap active root pages, the page itself can't be freed
2419                  * and zapping it will just force vCPUs to realloc and reload.
2420                  */
2421                 if (sp->root_count)
2422                         continue;
2423
2424                 unstable = __kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list,
2425                                                       &nr_zapped);
2426                 total_zapped += nr_zapped;
2427                 if (total_zapped >= nr_to_zap)
2428                         break;
2429
2430                 if (unstable)
2431                         goto restart;
2432         }
2433
2434         kvm_mmu_commit_zap_page(kvm, &invalid_list);
2435
2436         kvm->stat.mmu_recycled += total_zapped;
2437         return total_zapped;
2438 }
2439
2440 static inline unsigned long kvm_mmu_available_pages(struct kvm *kvm)
2441 {
2442         if (kvm->arch.n_max_mmu_pages > kvm->arch.n_used_mmu_pages)
2443                 return kvm->arch.n_max_mmu_pages -
2444                         kvm->arch.n_used_mmu_pages;
2445
2446         return 0;
2447 }
2448
2449 static int make_mmu_pages_available(struct kvm_vcpu *vcpu)
2450 {
2451         unsigned long avail = kvm_mmu_available_pages(vcpu->kvm);
2452
2453         if (likely(avail >= KVM_MIN_FREE_MMU_PAGES))
2454                 return 0;
2455
2456         kvm_mmu_zap_oldest_mmu_pages(vcpu->kvm, KVM_REFILL_PAGES - avail);
2457
2458         /*
2459          * Note, this check is intentionally soft, it only guarantees that one
2460          * page is available, while the caller may end up allocating as many as
2461          * four pages, e.g. for PAE roots or for 5-level paging.  Temporarily
2462          * exceeding the (arbitrary by default) limit will not harm the host,
2463          * being too aggressive may unnecessarily kill the guest, and getting an
2464          * exact count is far more trouble than it's worth, especially in the
2465          * page fault paths.
2466          */
2467         if (!kvm_mmu_available_pages(vcpu->kvm))
2468                 return -ENOSPC;
2469         return 0;
2470 }
2471
2472 /*
2473  * Changing the number of mmu pages allocated to the vm
2474  * Note: if goal_nr_mmu_pages is too small, you will get dead lock
2475  */
2476 void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned long goal_nr_mmu_pages)
2477 {
2478         write_lock(&kvm->mmu_lock);
2479
2480         if (kvm->arch.n_used_mmu_pages > goal_nr_mmu_pages) {
2481                 kvm_mmu_zap_oldest_mmu_pages(kvm, kvm->arch.n_used_mmu_pages -
2482                                                   goal_nr_mmu_pages);
2483
2484                 goal_nr_mmu_pages = kvm->arch.n_used_mmu_pages;
2485         }
2486
2487         kvm->arch.n_max_mmu_pages = goal_nr_mmu_pages;
2488
2489         write_unlock(&kvm->mmu_lock);
2490 }
2491
2492 int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn)
2493 {
2494         struct kvm_mmu_page *sp;
2495         LIST_HEAD(invalid_list);
2496         int r;
2497
2498         pgprintk("%s: looking for gfn %llx\n", __func__, gfn);
2499         r = 0;
2500         write_lock(&kvm->mmu_lock);
2501         for_each_gfn_valid_sp_with_gptes(kvm, sp, gfn) {
2502                 pgprintk("%s: gfn %llx role %x\n", __func__, gfn,
2503                          sp->role.word);
2504                 r = 1;
2505                 kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list);
2506         }
2507         kvm_mmu_commit_zap_page(kvm, &invalid_list);
2508         write_unlock(&kvm->mmu_lock);
2509
2510         return r;
2511 }
2512
2513 static int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva)
2514 {
2515         gpa_t gpa;
2516         int r;
2517
2518         if (vcpu->arch.mmu->root_role.direct)
2519                 return 0;
2520
2521         gpa = kvm_mmu_gva_to_gpa_read(vcpu, gva, NULL);
2522
2523         r = kvm_mmu_unprotect_page(vcpu->kvm, gpa >> PAGE_SHIFT);
2524
2525         return r;
2526 }
2527
2528 static void kvm_unsync_page(struct kvm *kvm, struct kvm_mmu_page *sp)
2529 {
2530         trace_kvm_mmu_unsync_page(sp);
2531         ++kvm->stat.mmu_unsync;
2532         sp->unsync = 1;
2533
2534         kvm_mmu_mark_parents_unsync(sp);
2535 }
2536
2537 /*
2538  * Attempt to unsync any shadow pages that can be reached by the specified gfn,
2539  * KVM is creating a writable mapping for said gfn.  Returns 0 if all pages
2540  * were marked unsync (or if there is no shadow page), -EPERM if the SPTE must
2541  * be write-protected.
2542  */
2543 int mmu_try_to_unsync_pages(struct kvm *kvm, const struct kvm_memory_slot *slot,
2544                             gfn_t gfn, bool can_unsync, bool prefetch)
2545 {
2546         struct kvm_mmu_page *sp;
2547         bool locked = false;
2548
2549         /*
2550          * Force write-protection if the page is being tracked.  Note, the page
2551          * track machinery is used to write-protect upper-level shadow pages,
2552          * i.e. this guards the role.level == 4K assertion below!
2553          */
2554         if (kvm_slot_page_track_is_active(kvm, slot, gfn, KVM_PAGE_TRACK_WRITE))
2555                 return -EPERM;
2556
2557         /*
2558          * The page is not write-tracked, mark existing shadow pages unsync
2559          * unless KVM is synchronizing an unsync SP (can_unsync = false).  In
2560          * that case, KVM must complete emulation of the guest TLB flush before
2561          * allowing shadow pages to become unsync (writable by the guest).
2562          */
2563         for_each_gfn_valid_sp_with_gptes(kvm, sp, gfn) {
2564                 if (!can_unsync)
2565                         return -EPERM;
2566
2567                 if (sp->unsync)
2568                         continue;
2569
2570                 if (prefetch)
2571                         return -EEXIST;
2572
2573                 /*
2574                  * TDP MMU page faults require an additional spinlock as they
2575                  * run with mmu_lock held for read, not write, and the unsync
2576                  * logic is not thread safe.  Take the spinklock regardless of
2577                  * the MMU type to avoid extra conditionals/parameters, there's
2578                  * no meaningful penalty if mmu_lock is held for write.
2579                  */
2580                 if (!locked) {
2581                         locked = true;
2582                         spin_lock(&kvm->arch.mmu_unsync_pages_lock);
2583
2584                         /*
2585                          * Recheck after taking the spinlock, a different vCPU
2586                          * may have since marked the page unsync.  A false
2587                          * positive on the unprotected check above is not
2588                          * possible as clearing sp->unsync _must_ hold mmu_lock
2589                          * for write, i.e. unsync cannot transition from 0->1
2590                          * while this CPU holds mmu_lock for read (or write).
2591                          */
2592                         if (READ_ONCE(sp->unsync))
2593                                 continue;
2594                 }
2595
2596                 WARN_ON(sp->role.level != PG_LEVEL_4K);
2597                 kvm_unsync_page(kvm, sp);
2598         }
2599         if (locked)
2600                 spin_unlock(&kvm->arch.mmu_unsync_pages_lock);
2601
2602         /*
2603          * We need to ensure that the marking of unsync pages is visible
2604          * before the SPTE is updated to allow writes because
2605          * kvm_mmu_sync_roots() checks the unsync flags without holding
2606          * the MMU lock and so can race with this. If the SPTE was updated
2607          * before the page had been marked as unsync-ed, something like the
2608          * following could happen:
2609          *
2610          * CPU 1                    CPU 2
2611          * ---------------------------------------------------------------------
2612          * 1.2 Host updates SPTE
2613          *     to be writable
2614          *                      2.1 Guest writes a GPTE for GVA X.
2615          *                          (GPTE being in the guest page table shadowed
2616          *                           by the SP from CPU 1.)
2617          *                          This reads SPTE during the page table walk.
2618          *                          Since SPTE.W is read as 1, there is no
2619          *                          fault.
2620          *
2621          *                      2.2 Guest issues TLB flush.
2622          *                          That causes a VM Exit.
2623          *
2624          *                      2.3 Walking of unsync pages sees sp->unsync is
2625          *                          false and skips the page.
2626          *
2627          *                      2.4 Guest accesses GVA X.
2628          *                          Since the mapping in the SP was not updated,
2629          *                          so the old mapping for GVA X incorrectly
2630          *                          gets used.
2631          * 1.1 Host marks SP
2632          *     as unsync
2633          *     (sp->unsync = true)
2634          *
2635          * The write barrier below ensures that 1.1 happens before 1.2 and thus
2636          * the situation in 2.4 does not arise.  It pairs with the read barrier
2637          * in is_unsync_root(), placed between 2.1's load of SPTE.W and 2.3.
2638          */
2639         smp_wmb();
2640
2641         return 0;
2642 }
2643
2644 static int mmu_set_spte(struct kvm_vcpu *vcpu, struct kvm_memory_slot *slot,
2645                         u64 *sptep, unsigned int pte_access, gfn_t gfn,
2646                         kvm_pfn_t pfn, struct kvm_page_fault *fault)
2647 {
2648         struct kvm_mmu_page *sp = sptep_to_sp(sptep);
2649         int level = sp->role.level;
2650         int was_rmapped = 0;
2651         int ret = RET_PF_FIXED;
2652         bool flush = false;
2653         bool wrprot;
2654         u64 spte;
2655
2656         /* Prefetching always gets a writable pfn.  */
2657         bool host_writable = !fault || fault->map_writable;
2658         bool prefetch = !fault || fault->prefetch;
2659         bool write_fault = fault && fault->write;
2660
2661         pgprintk("%s: spte %llx write_fault %d gfn %llx\n", __func__,
2662                  *sptep, write_fault, gfn);
2663
2664         if (unlikely(is_noslot_pfn(pfn))) {
2665                 vcpu->stat.pf_mmio_spte_created++;
2666                 mark_mmio_spte(vcpu, sptep, gfn, pte_access);
2667                 return RET_PF_EMULATE;
2668         }
2669
2670         if (is_shadow_present_pte(*sptep)) {
2671                 /*
2672                  * If we overwrite a PTE page pointer with a 2MB PMD, unlink
2673                  * the parent of the now unreachable PTE.
2674                  */
2675                 if (level > PG_LEVEL_4K && !is_large_pte(*sptep)) {
2676                         struct kvm_mmu_page *child;
2677                         u64 pte = *sptep;
2678
2679                         child = to_shadow_page(pte & PT64_BASE_ADDR_MASK);
2680                         drop_parent_pte(child, sptep);
2681                         flush = true;
2682                 } else if (pfn != spte_to_pfn(*sptep)) {
2683                         pgprintk("hfn old %llx new %llx\n",
2684                                  spte_to_pfn(*sptep), pfn);
2685                         drop_spte(vcpu->kvm, sptep);
2686                         flush = true;
2687                 } else
2688                         was_rmapped = 1;
2689         }
2690
2691         wrprot = make_spte(vcpu, sp, slot, pte_access, gfn, pfn, *sptep, prefetch,
2692                            true, host_writable, &spte);
2693
2694         if (*sptep == spte) {
2695                 ret = RET_PF_SPURIOUS;
2696         } else {
2697                 flush |= mmu_spte_update(sptep, spte);
2698                 trace_kvm_mmu_set_spte(level, gfn, sptep);
2699         }
2700
2701         if (wrprot) {
2702                 if (write_fault)
2703                         ret = RET_PF_EMULATE;
2704         }
2705
2706         if (flush)
2707                 kvm_flush_remote_tlbs_with_address(vcpu->kvm, gfn,
2708                                 KVM_PAGES_PER_HPAGE(level));
2709
2710         pgprintk("%s: setting spte %llx\n", __func__, *sptep);
2711
2712         if (!was_rmapped) {
2713                 WARN_ON_ONCE(ret == RET_PF_SPURIOUS);
2714                 kvm_update_page_stats(vcpu->kvm, level, 1);
2715                 rmap_add(vcpu, slot, sptep, gfn);
2716         }
2717
2718         return ret;
2719 }
2720
2721 static int direct_pte_prefetch_many(struct kvm_vcpu *vcpu,
2722                                     struct kvm_mmu_page *sp,
2723                                     u64 *start, u64 *end)
2724 {
2725         struct page *pages[PTE_PREFETCH_NUM];
2726         struct kvm_memory_slot *slot;
2727         unsigned int access = sp->role.access;
2728         int i, ret;
2729         gfn_t gfn;
2730
2731         gfn = kvm_mmu_page_get_gfn(sp, start - sp->spt);
2732         slot = gfn_to_memslot_dirty_bitmap(vcpu, gfn, access & ACC_WRITE_MASK);
2733         if (!slot)
2734                 return -1;
2735
2736         ret = gfn_to_page_many_atomic(slot, gfn, pages, end - start);
2737         if (ret <= 0)
2738                 return -1;
2739
2740         for (i = 0; i < ret; i++, gfn++, start++) {
2741                 mmu_set_spte(vcpu, slot, start, access, gfn,
2742                              page_to_pfn(pages[i]), NULL);
2743                 put_page(pages[i]);
2744         }
2745
2746         return 0;
2747 }
2748
2749 static void __direct_pte_prefetch(struct kvm_vcpu *vcpu,
2750                                   struct kvm_mmu_page *sp, u64 *sptep)
2751 {
2752         u64 *spte, *start = NULL;
2753         int i;
2754
2755         WARN_ON(!sp->role.direct);
2756
2757         i = (sptep - sp->spt) & ~(PTE_PREFETCH_NUM - 1);
2758         spte = sp->spt + i;
2759
2760         for (i = 0; i < PTE_PREFETCH_NUM; i++, spte++) {
2761                 if (is_shadow_present_pte(*spte) || spte == sptep) {
2762                         if (!start)
2763                                 continue;
2764                         if (direct_pte_prefetch_many(vcpu, sp, start, spte) < 0)
2765                                 return;
2766                         start = NULL;
2767                 } else if (!start)
2768                         start = spte;
2769         }
2770         if (start)
2771                 direct_pte_prefetch_many(vcpu, sp, start, spte);
2772 }
2773
2774 static void direct_pte_prefetch(struct kvm_vcpu *vcpu, u64 *sptep)
2775 {
2776         struct kvm_mmu_page *sp;
2777
2778         sp = sptep_to_sp(sptep);
2779
2780         /*
2781          * Without accessed bits, there's no way to distinguish between
2782          * actually accessed translations and prefetched, so disable pte
2783          * prefetch if accessed bits aren't available.
2784          */
2785         if (sp_ad_disabled(sp))
2786                 return;
2787
2788         if (sp->role.level > PG_LEVEL_4K)
2789                 return;
2790
2791         /*
2792          * If addresses are being invalidated, skip prefetching to avoid
2793          * accidentally prefetching those addresses.
2794          */
2795         if (unlikely(vcpu->kvm->mmu_notifier_count))
2796                 return;
2797
2798         __direct_pte_prefetch(vcpu, sp, sptep);
2799 }
2800
2801 static int host_pfn_mapping_level(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn,
2802                                   const struct kvm_memory_slot *slot)
2803 {
2804         unsigned long hva;
2805         unsigned long flags;
2806         int level = PG_LEVEL_4K;
2807         pgd_t pgd;
2808         p4d_t p4d;
2809         pud_t pud;
2810         pmd_t pmd;
2811
2812         if (!PageCompound(pfn_to_page(pfn)) && !kvm_is_zone_device_pfn(pfn))
2813                 return PG_LEVEL_4K;
2814
2815         /*
2816          * Note, using the already-retrieved memslot and __gfn_to_hva_memslot()
2817          * is not solely for performance, it's also necessary to avoid the
2818          * "writable" check in __gfn_to_hva_many(), which will always fail on
2819          * read-only memslots due to gfn_to_hva() assuming writes.  Earlier
2820          * page fault steps have already verified the guest isn't writing a
2821          * read-only memslot.
2822          */
2823         hva = __gfn_to_hva_memslot(slot, gfn);
2824
2825         /*
2826          * Lookup the mapping level in the current mm.  The information
2827          * may become stale soon, but it is safe to use as long as
2828          * 1) mmu_notifier_retry was checked after taking mmu_lock, and
2829          * 2) mmu_lock is taken now.
2830          *
2831          * We still need to disable IRQs to prevent concurrent tear down
2832          * of page tables.
2833          */
2834         local_irq_save(flags);
2835
2836         pgd = READ_ONCE(*pgd_offset(kvm->mm, hva));
2837         if (pgd_none(pgd))
2838                 goto out;
2839
2840         p4d = READ_ONCE(*p4d_offset(&pgd, hva));
2841         if (p4d_none(p4d) || !p4d_present(p4d))
2842                 goto out;
2843
2844         pud = READ_ONCE(*pud_offset(&p4d, hva));
2845         if (pud_none(pud) || !pud_present(pud))
2846                 goto out;
2847
2848         if (pud_large(pud)) {
2849                 level = PG_LEVEL_1G;
2850                 goto out;
2851         }
2852
2853         pmd = READ_ONCE(*pmd_offset(&pud, hva));
2854         if (pmd_none(pmd) || !pmd_present(pmd))
2855                 goto out;
2856
2857         if (pmd_large(pmd))
2858                 level = PG_LEVEL_2M;
2859
2860 out:
2861         local_irq_restore(flags);
2862         return level;
2863 }
2864
2865 int kvm_mmu_max_mapping_level(struct kvm *kvm,
2866                               const struct kvm_memory_slot *slot, gfn_t gfn,
2867                               kvm_pfn_t pfn, int max_level)
2868 {
2869         struct kvm_lpage_info *linfo;
2870         int host_level;
2871
2872         max_level = min(max_level, max_huge_page_level);
2873         for ( ; max_level > PG_LEVEL_4K; max_level--) {
2874                 linfo = lpage_info_slot(gfn, slot, max_level);
2875                 if (!linfo->disallow_lpage)
2876                         break;
2877         }
2878
2879         if (max_level == PG_LEVEL_4K)
2880                 return PG_LEVEL_4K;
2881
2882         host_level = host_pfn_mapping_level(kvm, gfn, pfn, slot);
2883         return min(host_level, max_level);
2884 }
2885
2886 void kvm_mmu_hugepage_adjust(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
2887 {
2888         struct kvm_memory_slot *slot = fault->slot;
2889         kvm_pfn_t mask;
2890
2891         fault->huge_page_disallowed = fault->exec && fault->nx_huge_page_workaround_enabled;
2892
2893         if (unlikely(fault->max_level == PG_LEVEL_4K))
2894                 return;
2895
2896         if (is_error_noslot_pfn(fault->pfn) || kvm_is_reserved_pfn(fault->pfn))
2897                 return;
2898
2899         if (kvm_slot_dirty_track_enabled(slot))
2900                 return;
2901
2902         /*
2903          * Enforce the iTLB multihit workaround after capturing the requested
2904          * level, which will be used to do precise, accurate accounting.
2905          */
2906         fault->req_level = kvm_mmu_max_mapping_level(vcpu->kvm, slot,
2907                                                      fault->gfn, fault->pfn,
2908                                                      fault->max_level);
2909         if (fault->req_level == PG_LEVEL_4K || fault->huge_page_disallowed)
2910                 return;
2911
2912         /*
2913          * mmu_notifier_retry() was successful and mmu_lock is held, so
2914          * the pmd can't be split from under us.
2915          */
2916         fault->goal_level = fault->req_level;
2917         mask = KVM_PAGES_PER_HPAGE(fault->goal_level) - 1;
2918         VM_BUG_ON((fault->gfn & mask) != (fault->pfn & mask));
2919         fault->pfn &= ~mask;
2920 }
2921
2922 void disallowed_hugepage_adjust(struct kvm_page_fault *fault, u64 spte, int cur_level)
2923 {
2924         if (cur_level > PG_LEVEL_4K &&
2925             cur_level == fault->goal_level &&
2926             is_shadow_present_pte(spte) &&
2927             !is_large_pte(spte)) {
2928                 /*
2929                  * A small SPTE exists for this pfn, but FNAME(fetch)
2930                  * and __direct_map would like to create a large PTE
2931                  * instead: just force them to go down another level,
2932                  * patching back for them into pfn the next 9 bits of
2933                  * the address.
2934                  */
2935                 u64 page_mask = KVM_PAGES_PER_HPAGE(cur_level) -
2936                                 KVM_PAGES_PER_HPAGE(cur_level - 1);
2937                 fault->pfn |= fault->gfn & page_mask;
2938                 fault->goal_level--;
2939         }
2940 }
2941
2942 static int __direct_map(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
2943 {
2944         struct kvm_shadow_walk_iterator it;
2945         struct kvm_mmu_page *sp;
2946         int ret;
2947         gfn_t base_gfn = fault->gfn;
2948
2949         kvm_mmu_hugepage_adjust(vcpu, fault);
2950
2951         trace_kvm_mmu_spte_requested(fault);
2952         for_each_shadow_entry(vcpu, fault->addr, it) {
2953                 /*
2954                  * We cannot overwrite existing page tables with an NX
2955                  * large page, as the leaf could be executable.
2956                  */
2957                 if (fault->nx_huge_page_workaround_enabled)
2958                         disallowed_hugepage_adjust(fault, *it.sptep, it.level);
2959
2960                 base_gfn = fault->gfn & ~(KVM_PAGES_PER_HPAGE(it.level) - 1);
2961                 if (it.level == fault->goal_level)
2962                         break;
2963
2964                 drop_large_spte(vcpu, it.sptep);
2965                 if (is_shadow_present_pte(*it.sptep))
2966                         continue;
2967
2968                 sp = kvm_mmu_get_page(vcpu, base_gfn, it.addr,
2969                                       it.level - 1, true, ACC_ALL);
2970
2971                 link_shadow_page(vcpu, it.sptep, sp);
2972                 if (fault->is_tdp && fault->huge_page_disallowed &&
2973                     fault->req_level >= it.level)
2974                         account_huge_nx_page(vcpu->kvm, sp);
2975         }
2976
2977         if (WARN_ON_ONCE(it.level != fault->goal_level))
2978                 return -EFAULT;
2979
2980         ret = mmu_set_spte(vcpu, fault->slot, it.sptep, ACC_ALL,
2981                            base_gfn, fault->pfn, fault);
2982         if (ret == RET_PF_SPURIOUS)
2983                 return ret;
2984
2985         direct_pte_prefetch(vcpu, it.sptep);
2986         return ret;
2987 }
2988
2989 static void kvm_send_hwpoison_signal(unsigned long address, struct task_struct *tsk)
2990 {
2991         send_sig_mceerr(BUS_MCEERR_AR, (void __user *)address, PAGE_SHIFT, tsk);
2992 }
2993
2994 static int kvm_handle_bad_page(struct kvm_vcpu *vcpu, gfn_t gfn, kvm_pfn_t pfn)
2995 {
2996         /*
2997          * Do not cache the mmio info caused by writing the readonly gfn
2998          * into the spte otherwise read access on readonly gfn also can
2999          * caused mmio page fault and treat it as mmio access.
3000          */
3001         if (pfn == KVM_PFN_ERR_RO_FAULT)
3002                 return RET_PF_EMULATE;
3003
3004         if (pfn == KVM_PFN_ERR_HWPOISON) {
3005                 kvm_send_hwpoison_signal(kvm_vcpu_gfn_to_hva(vcpu, gfn), current);
3006                 return RET_PF_RETRY;
3007         }
3008
3009         return -EFAULT;
3010 }
3011
3012 static int handle_abnormal_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault,
3013                                unsigned int access)
3014 {
3015         /* The pfn is invalid, report the error! */
3016         if (unlikely(is_error_pfn(fault->pfn)))
3017                 return kvm_handle_bad_page(vcpu, fault->gfn, fault->pfn);
3018
3019         if (unlikely(!fault->slot)) {
3020                 gva_t gva = fault->is_tdp ? 0 : fault->addr;
3021
3022                 vcpu_cache_mmio_info(vcpu, gva, fault->gfn,
3023                                      access & shadow_mmio_access_mask);
3024                 /*
3025                  * If MMIO caching is disabled, emulate immediately without
3026                  * touching the shadow page tables as attempting to install an
3027                  * MMIO SPTE will just be an expensive nop.  Do not cache MMIO
3028                  * whose gfn is greater than host.MAXPHYADDR, any guest that
3029                  * generates such gfns is running nested and is being tricked
3030                  * by L0 userspace (you can observe gfn > L1.MAXPHYADDR if
3031                  * and only if L1's MAXPHYADDR is inaccurate with respect to
3032                  * the hardware's).
3033                  */
3034                 if (unlikely(!enable_mmio_caching) ||
3035                     unlikely(fault->gfn > kvm_mmu_max_gfn()))
3036                         return RET_PF_EMULATE;
3037         }
3038
3039         return RET_PF_CONTINUE;
3040 }
3041
3042 static bool page_fault_can_be_fast(struct kvm_page_fault *fault)
3043 {
3044         /*
3045          * Page faults with reserved bits set, i.e. faults on MMIO SPTEs, only
3046          * reach the common page fault handler if the SPTE has an invalid MMIO
3047          * generation number.  Refreshing the MMIO generation needs to go down
3048          * the slow path.  Note, EPT Misconfigs do NOT set the PRESENT flag!
3049          */
3050         if (fault->rsvd)
3051                 return false;
3052
3053         /*
3054          * #PF can be fast if:
3055          *
3056          * 1. The shadow page table entry is not present and A/D bits are
3057          *    disabled _by KVM_, which could mean that the fault is potentially
3058          *    caused by access tracking (if enabled).  If A/D bits are enabled
3059          *    by KVM, but disabled by L1 for L2, KVM is forced to disable A/D
3060          *    bits for L2 and employ access tracking, but the fast page fault
3061          *    mechanism only supports direct MMUs.
3062          * 2. The shadow page table entry is present, the access is a write,
3063          *    and no reserved bits are set (MMIO SPTEs cannot be "fixed"), i.e.
3064          *    the fault was caused by a write-protection violation.  If the
3065          *    SPTE is MMU-writable (determined later), the fault can be fixed
3066          *    by setting the Writable bit, which can be done out of mmu_lock.
3067          */
3068         if (!fault->present)
3069                 return !kvm_ad_enabled();
3070
3071         /*
3072          * Note, instruction fetches and writes are mutually exclusive, ignore
3073          * the "exec" flag.
3074          */
3075         return fault->write;
3076 }
3077
3078 /*
3079  * Returns true if the SPTE was fixed successfully. Otherwise,
3080  * someone else modified the SPTE from its original value.
3081  */
3082 static bool
3083 fast_pf_fix_direct_spte(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault,
3084                         u64 *sptep, u64 old_spte, u64 new_spte)
3085 {
3086         /*
3087          * Theoretically we could also set dirty bit (and flush TLB) here in
3088          * order to eliminate unnecessary PML logging. See comments in
3089          * set_spte. But fast_page_fault is very unlikely to happen with PML
3090          * enabled, so we do not do this. This might result in the same GPA
3091          * to be logged in PML buffer again when the write really happens, and
3092          * eventually to be called by mark_page_dirty twice. But it's also no
3093          * harm. This also avoids the TLB flush needed after setting dirty bit
3094          * so non-PML cases won't be impacted.
3095          *
3096          * Compare with set_spte where instead shadow_dirty_mask is set.
3097          */
3098         if (cmpxchg64(sptep, old_spte, new_spte) != old_spte)
3099                 return false;
3100
3101         if (is_writable_pte(new_spte) && !is_writable_pte(old_spte))
3102                 mark_page_dirty_in_slot(vcpu->kvm, fault->slot, fault->gfn);
3103
3104         return true;
3105 }
3106
3107 static bool is_access_allowed(struct kvm_page_fault *fault, u64 spte)
3108 {
3109         if (fault->exec)
3110                 return is_executable_pte(spte);
3111
3112         if (fault->write)
3113                 return is_writable_pte(spte);
3114
3115         /* Fault was on Read access */
3116         return spte & PT_PRESENT_MASK;
3117 }
3118
3119 /*
3120  * Returns the last level spte pointer of the shadow page walk for the given
3121  * gpa, and sets *spte to the spte value. This spte may be non-preset. If no
3122  * walk could be performed, returns NULL and *spte does not contain valid data.
3123  *
3124  * Contract:
3125  *  - Must be called between walk_shadow_page_lockless_{begin,end}.
3126  *  - The returned sptep must not be used after walk_shadow_page_lockless_end.
3127  */
3128 static u64 *fast_pf_get_last_sptep(struct kvm_vcpu *vcpu, gpa_t gpa, u64 *spte)
3129 {
3130         struct kvm_shadow_walk_iterator iterator;
3131         u64 old_spte;
3132         u64 *sptep = NULL;
3133
3134         for_each_shadow_entry_lockless(vcpu, gpa, iterator, old_spte) {
3135                 sptep = iterator.sptep;
3136                 *spte = old_spte;
3137         }
3138
3139         return sptep;
3140 }
3141
3142 /*
3143  * Returns one of RET_PF_INVALID, RET_PF_FIXED or RET_PF_SPURIOUS.
3144  */
3145 static int fast_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
3146 {
3147         struct kvm_mmu_page *sp;
3148         int ret = RET_PF_INVALID;
3149         u64 spte = 0ull;
3150         u64 *sptep = NULL;
3151         uint retry_count = 0;
3152
3153         if (!page_fault_can_be_fast(fault))
3154                 return ret;
3155
3156         walk_shadow_page_lockless_begin(vcpu);
3157
3158         do {
3159                 u64 new_spte;
3160
3161                 if (is_tdp_mmu(vcpu->arch.mmu))
3162                         sptep = kvm_tdp_mmu_fast_pf_get_last_sptep(vcpu, fault->addr, &spte);
3163                 else
3164                         sptep = fast_pf_get_last_sptep(vcpu, fault->addr, &spte);
3165
3166                 if (!is_shadow_present_pte(spte))
3167                         break;
3168
3169                 sp = sptep_to_sp(sptep);
3170                 if (!is_last_spte(spte, sp->role.level))
3171                         break;
3172
3173                 /*
3174                  * Check whether the memory access that caused the fault would
3175                  * still cause it if it were to be performed right now. If not,
3176                  * then this is a spurious fault caused by TLB lazily flushed,
3177                  * or some other CPU has already fixed the PTE after the
3178                  * current CPU took the fault.
3179                  *
3180                  * Need not check the access of upper level table entries since
3181                  * they are always ACC_ALL.
3182                  */
3183                 if (is_access_allowed(fault, spte)) {
3184                         ret = RET_PF_SPURIOUS;
3185                         break;
3186                 }
3187
3188                 new_spte = spte;
3189
3190                 /*
3191                  * KVM only supports fixing page faults outside of MMU lock for
3192                  * direct MMUs, nested MMUs are always indirect, and KVM always
3193                  * uses A/D bits for non-nested MMUs.  Thus, if A/D bits are
3194                  * enabled, the SPTE can't be an access-tracked SPTE.
3195                  */
3196                 if (unlikely(!kvm_ad_enabled()) && is_access_track_spte(spte))
3197                         new_spte = restore_acc_track_spte(new_spte);
3198
3199                 /*
3200                  * To keep things simple, only SPTEs that are MMU-writable can
3201                  * be made fully writable outside of mmu_lock, e.g. only SPTEs
3202                  * that were write-protected for dirty-logging or access
3203                  * tracking are handled here.  Don't bother checking if the
3204                  * SPTE is writable to prioritize running with A/D bits enabled.
3205                  * The is_access_allowed() check above handles the common case
3206                  * of the fault being spurious, and the SPTE is known to be
3207                  * shadow-present, i.e. except for access tracking restoration
3208                  * making the new SPTE writable, the check is wasteful.
3209                  */
3210                 if (fault->write && is_mmu_writable_spte(spte)) {
3211                         new_spte |= PT_WRITABLE_MASK;
3212
3213                         /*
3214                          * Do not fix write-permission on the large spte when
3215                          * dirty logging is enabled. Since we only dirty the
3216                          * first page into the dirty-bitmap in
3217                          * fast_pf_fix_direct_spte(), other pages are missed
3218                          * if its slot has dirty logging enabled.
3219                          *
3220                          * Instead, we let the slow page fault path create a
3221                          * normal spte to fix the access.
3222                          */
3223                         if (sp->role.level > PG_LEVEL_4K &&
3224                             kvm_slot_dirty_track_enabled(fault->slot))
3225                                 break;
3226                 }
3227
3228                 /* Verify that the fault can be handled in the fast path */
3229                 if (new_spte == spte ||
3230                     !is_access_allowed(fault, new_spte))
3231                         break;
3232
3233                 /*
3234                  * Currently, fast page fault only works for direct mapping
3235                  * since the gfn is not stable for indirect shadow page. See
3236                  * Documentation/virt/kvm/locking.rst to get more detail.
3237                  */
3238                 if (fast_pf_fix_direct_spte(vcpu, fault, sptep, spte, new_spte)) {
3239                         ret = RET_PF_FIXED;
3240                         break;
3241                 }
3242
3243                 if (++retry_count > 4) {
3244                         printk_once(KERN_WARNING
3245                                 "kvm: Fast #PF retrying more than 4 times.\n");
3246                         break;
3247                 }
3248
3249         } while (true);
3250
3251         trace_fast_page_fault(vcpu, fault, sptep, spte, ret);
3252         walk_shadow_page_lockless_end(vcpu);
3253
3254         if (ret != RET_PF_INVALID)
3255                 vcpu->stat.pf_fast++;
3256
3257         return ret;
3258 }
3259
3260 static void mmu_free_root_page(struct kvm *kvm, hpa_t *root_hpa,
3261                                struct list_head *invalid_list)
3262 {
3263         struct kvm_mmu_page *sp;
3264
3265         if (!VALID_PAGE(*root_hpa))
3266                 return;
3267
3268         sp = to_shadow_page(*root_hpa & PT64_BASE_ADDR_MASK);
3269         if (WARN_ON(!sp))
3270                 return;
3271
3272         if (is_tdp_mmu_page(sp))
3273                 kvm_tdp_mmu_put_root(kvm, sp, false);
3274         else if (!--sp->root_count && sp->role.invalid)
3275                 kvm_mmu_prepare_zap_page(kvm, sp, invalid_list);
3276
3277         *root_hpa = INVALID_PAGE;
3278 }
3279
3280 /* roots_to_free must be some combination of the KVM_MMU_ROOT_* flags */
3281 void kvm_mmu_free_roots(struct kvm *kvm, struct kvm_mmu *mmu,
3282                         ulong roots_to_free)
3283 {
3284         int i;
3285         LIST_HEAD(invalid_list);
3286         bool free_active_root;
3287
3288         BUILD_BUG_ON(KVM_MMU_NUM_PREV_ROOTS >= BITS_PER_LONG);
3289
3290         /* Before acquiring the MMU lock, see if we need to do any real work. */
3291         free_active_root = (roots_to_free & KVM_MMU_ROOT_CURRENT)
3292                 && VALID_PAGE(mmu->root.hpa);
3293
3294         if (!free_active_root) {
3295                 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
3296                         if ((roots_to_free & KVM_MMU_ROOT_PREVIOUS(i)) &&
3297                             VALID_PAGE(mmu->prev_roots[i].hpa))
3298                                 break;
3299
3300                 if (i == KVM_MMU_NUM_PREV_ROOTS)
3301                         return;
3302         }
3303
3304         write_lock(&kvm->mmu_lock);
3305
3306         for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
3307                 if (roots_to_free & KVM_MMU_ROOT_PREVIOUS(i))
3308                         mmu_free_root_page(kvm, &mmu->prev_roots[i].hpa,
3309                                            &invalid_list);
3310
3311         if (free_active_root) {
3312                 if (to_shadow_page(mmu->root.hpa)) {
3313                         mmu_free_root_page(kvm, &mmu->root.hpa, &invalid_list);
3314                 } else if (mmu->pae_root) {
3315                         for (i = 0; i < 4; ++i) {
3316                                 if (!IS_VALID_PAE_ROOT(mmu->pae_root[i]))
3317                                         continue;
3318
3319                                 mmu_free_root_page(kvm, &mmu->pae_root[i],
3320                                                    &invalid_list);
3321                                 mmu->pae_root[i] = INVALID_PAE_ROOT;
3322                         }
3323                 }
3324                 mmu->root.hpa = INVALID_PAGE;
3325                 mmu->root.pgd = 0;
3326         }
3327
3328         kvm_mmu_commit_zap_page(kvm, &invalid_list);
3329         write_unlock(&kvm->mmu_lock);
3330 }
3331 EXPORT_SYMBOL_GPL(kvm_mmu_free_roots);
3332
3333 void kvm_mmu_free_guest_mode_roots(struct kvm *kvm, struct kvm_mmu *mmu)
3334 {
3335         unsigned long roots_to_free = 0;
3336         hpa_t root_hpa;
3337         int i;
3338
3339         /*
3340          * This should not be called while L2 is active, L2 can't invalidate
3341          * _only_ its own roots, e.g. INVVPID unconditionally exits.
3342          */
3343         WARN_ON_ONCE(mmu->root_role.guest_mode);
3344
3345         for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) {
3346                 root_hpa = mmu->prev_roots[i].hpa;
3347                 if (!VALID_PAGE(root_hpa))
3348                         continue;
3349
3350                 if (!to_shadow_page(root_hpa) ||
3351                         to_shadow_page(root_hpa)->role.guest_mode)
3352                         roots_to_free |= KVM_MMU_ROOT_PREVIOUS(i);
3353         }
3354
3355         kvm_mmu_free_roots(kvm, mmu, roots_to_free);
3356 }
3357 EXPORT_SYMBOL_GPL(kvm_mmu_free_guest_mode_roots);
3358
3359
3360 static int mmu_check_root(struct kvm_vcpu *vcpu, gfn_t root_gfn)
3361 {
3362         int ret = 0;
3363
3364         if (!kvm_vcpu_is_visible_gfn(vcpu, root_gfn)) {
3365                 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
3366                 ret = 1;
3367         }
3368
3369         return ret;
3370 }
3371
3372 static hpa_t mmu_alloc_root(struct kvm_vcpu *vcpu, gfn_t gfn, gva_t gva,
3373                             u8 level, bool direct)
3374 {
3375         struct kvm_mmu_page *sp;
3376
3377         sp = kvm_mmu_get_page(vcpu, gfn, gva, level, direct, ACC_ALL);
3378         ++sp->root_count;
3379
3380         return __pa(sp->spt);
3381 }
3382
3383 static int mmu_alloc_direct_roots(struct kvm_vcpu *vcpu)
3384 {
3385         struct kvm_mmu *mmu = vcpu->arch.mmu;
3386         u8 shadow_root_level = mmu->root_role.level;
3387         hpa_t root;
3388         unsigned i;
3389         int r;
3390
3391         write_lock(&vcpu->kvm->mmu_lock);
3392         r = make_mmu_pages_available(vcpu);
3393         if (r < 0)
3394                 goto out_unlock;
3395
3396         if (is_tdp_mmu_enabled(vcpu->kvm)) {
3397                 root = kvm_tdp_mmu_get_vcpu_root_hpa(vcpu);
3398                 mmu->root.hpa = root;
3399         } else if (shadow_root_level >= PT64_ROOT_4LEVEL) {
3400                 root = mmu_alloc_root(vcpu, 0, 0, shadow_root_level, true);
3401                 mmu->root.hpa = root;
3402         } else if (shadow_root_level == PT32E_ROOT_LEVEL) {
3403                 if (WARN_ON_ONCE(!mmu->pae_root)) {
3404                         r = -EIO;
3405                         goto out_unlock;
3406                 }
3407
3408                 for (i = 0; i < 4; ++i) {
3409                         WARN_ON_ONCE(IS_VALID_PAE_ROOT(mmu->pae_root[i]));
3410
3411                         root = mmu_alloc_root(vcpu, i << (30 - PAGE_SHIFT),
3412                                               i << 30, PT32_ROOT_LEVEL, true);
3413                         mmu->pae_root[i] = root | PT_PRESENT_MASK |
3414                                            shadow_me_value;
3415                 }
3416                 mmu->root.hpa = __pa(mmu->pae_root);
3417         } else {
3418                 WARN_ONCE(1, "Bad TDP root level = %d\n", shadow_root_level);
3419                 r = -EIO;
3420                 goto out_unlock;
3421         }
3422
3423         /* root.pgd is ignored for direct MMUs. */
3424         mmu->root.pgd = 0;
3425 out_unlock:
3426         write_unlock(&vcpu->kvm->mmu_lock);
3427         return r;
3428 }
3429
3430 static int mmu_first_shadow_root_alloc(struct kvm *kvm)
3431 {
3432         struct kvm_memslots *slots;
3433         struct kvm_memory_slot *slot;
3434         int r = 0, i, bkt;
3435
3436         /*
3437          * Check if this is the first shadow root being allocated before
3438          * taking the lock.
3439          */
3440         if (kvm_shadow_root_allocated(kvm))
3441                 return 0;
3442
3443         mutex_lock(&kvm->slots_arch_lock);
3444
3445         /* Recheck, under the lock, whether this is the first shadow root. */
3446         if (kvm_shadow_root_allocated(kvm))
3447                 goto out_unlock;
3448
3449         /*
3450          * Check if anything actually needs to be allocated, e.g. all metadata
3451          * will be allocated upfront if TDP is disabled.
3452          */
3453         if (kvm_memslots_have_rmaps(kvm) &&
3454             kvm_page_track_write_tracking_enabled(kvm))
3455                 goto out_success;
3456
3457         for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
3458                 slots = __kvm_memslots(kvm, i);
3459                 kvm_for_each_memslot(slot, bkt, slots) {
3460                         /*
3461                          * Both of these functions are no-ops if the target is
3462                          * already allocated, so unconditionally calling both
3463                          * is safe.  Intentionally do NOT free allocations on
3464                          * failure to avoid having to track which allocations
3465                          * were made now versus when the memslot was created.
3466                          * The metadata is guaranteed to be freed when the slot
3467                          * is freed, and will be kept/used if userspace retries
3468                          * KVM_RUN instead of killing the VM.
3469                          */
3470                         r = memslot_rmap_alloc(slot, slot->npages);
3471                         if (r)
3472                                 goto out_unlock;
3473                         r = kvm_page_track_write_tracking_alloc(slot);
3474                         if (r)
3475                                 goto out_unlock;
3476                 }
3477         }
3478
3479         /*
3480          * Ensure that shadow_root_allocated becomes true strictly after
3481          * all the related pointers are set.
3482          */
3483 out_success:
3484         smp_store_release(&kvm->arch.shadow_root_allocated, true);
3485
3486 out_unlock:
3487         mutex_unlock(&kvm->slots_arch_lock);
3488         return r;
3489 }
3490
3491 static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu)
3492 {
3493         struct kvm_mmu *mmu = vcpu->arch.mmu;
3494         u64 pdptrs[4], pm_mask;
3495         gfn_t root_gfn, root_pgd;
3496         hpa_t root;
3497         unsigned i;
3498         int r;
3499
3500         root_pgd = mmu->get_guest_pgd(vcpu);
3501         root_gfn = root_pgd >> PAGE_SHIFT;
3502
3503         if (mmu_check_root(vcpu, root_gfn))
3504                 return 1;
3505
3506         /*
3507          * On SVM, reading PDPTRs might access guest memory, which might fault
3508          * and thus might sleep.  Grab the PDPTRs before acquiring mmu_lock.
3509          */
3510         if (mmu->cpu_role.base.level == PT32E_ROOT_LEVEL) {
3511                 for (i = 0; i < 4; ++i) {
3512                         pdptrs[i] = mmu->get_pdptr(vcpu, i);
3513                         if (!(pdptrs[i] & PT_PRESENT_MASK))
3514                                 continue;
3515
3516                         if (mmu_check_root(vcpu, pdptrs[i] >> PAGE_SHIFT))
3517                                 return 1;
3518                 }
3519         }
3520
3521         r = mmu_first_shadow_root_alloc(vcpu->kvm);
3522         if (r)
3523                 return r;
3524
3525         write_lock(&vcpu->kvm->mmu_lock);
3526         r = make_mmu_pages_available(vcpu);
3527         if (r < 0)
3528                 goto out_unlock;
3529
3530         /*
3531          * Do we shadow a long mode page table? If so we need to
3532          * write-protect the guests page table root.
3533          */
3534         if (mmu->cpu_role.base.level >= PT64_ROOT_4LEVEL) {
3535                 root = mmu_alloc_root(vcpu, root_gfn, 0,
3536                                       mmu->root_role.level, false);
3537                 mmu->root.hpa = root;
3538                 goto set_root_pgd;
3539         }
3540
3541         if (WARN_ON_ONCE(!mmu->pae_root)) {
3542                 r = -EIO;
3543                 goto out_unlock;
3544         }
3545
3546         /*
3547          * We shadow a 32 bit page table. This may be a legacy 2-level
3548          * or a PAE 3-level page table. In either case we need to be aware that
3549          * the shadow page table may be a PAE or a long mode page table.
3550          */
3551         pm_mask = PT_PRESENT_MASK | shadow_me_value;
3552         if (mmu->root_role.level >= PT64_ROOT_4LEVEL) {
3553                 pm_mask |= PT_ACCESSED_MASK | PT_WRITABLE_MASK | PT_USER_MASK;
3554
3555                 if (WARN_ON_ONCE(!mmu->pml4_root)) {
3556                         r = -EIO;
3557                         goto out_unlock;
3558                 }
3559                 mmu->pml4_root[0] = __pa(mmu->pae_root) | pm_mask;
3560
3561                 if (mmu->root_role.level == PT64_ROOT_5LEVEL) {
3562                         if (WARN_ON_ONCE(!mmu->pml5_root)) {
3563                                 r = -EIO;
3564                                 goto out_unlock;
3565                         }
3566                         mmu->pml5_root[0] = __pa(mmu->pml4_root) | pm_mask;
3567                 }
3568         }
3569
3570         for (i = 0; i < 4; ++i) {
3571                 WARN_ON_ONCE(IS_VALID_PAE_ROOT(mmu->pae_root[i]));
3572
3573                 if (mmu->cpu_role.base.level == PT32E_ROOT_LEVEL) {
3574                         if (!(pdptrs[i] & PT_PRESENT_MASK)) {
3575                                 mmu->pae_root[i] = INVALID_PAE_ROOT;
3576                                 continue;
3577                         }
3578                         root_gfn = pdptrs[i] >> PAGE_SHIFT;
3579                 }
3580
3581                 root = mmu_alloc_root(vcpu, root_gfn, i << 30,
3582                                       PT32_ROOT_LEVEL, false);
3583                 mmu->pae_root[i] = root | pm_mask;
3584         }
3585
3586         if (mmu->root_role.level == PT64_ROOT_5LEVEL)
3587                 mmu->root.hpa = __pa(mmu->pml5_root);
3588         else if (mmu->root_role.level == PT64_ROOT_4LEVEL)
3589                 mmu->root.hpa = __pa(mmu->pml4_root);
3590         else
3591                 mmu->root.hpa = __pa(mmu->pae_root);
3592
3593 set_root_pgd:
3594         mmu->root.pgd = root_pgd;
3595 out_unlock:
3596         write_unlock(&vcpu->kvm->mmu_lock);
3597
3598         return r;
3599 }
3600
3601 static int mmu_alloc_special_roots(struct kvm_vcpu *vcpu)
3602 {
3603         struct kvm_mmu *mmu = vcpu->arch.mmu;
3604         bool need_pml5 = mmu->root_role.level > PT64_ROOT_4LEVEL;
3605         u64 *pml5_root = NULL;
3606         u64 *pml4_root = NULL;
3607         u64 *pae_root;
3608
3609         /*
3610          * When shadowing 32-bit or PAE NPT with 64-bit NPT, the PML4 and PDP
3611          * tables are allocated and initialized at root creation as there is no
3612          * equivalent level in the guest's NPT to shadow.  Allocate the tables
3613          * on demand, as running a 32-bit L1 VMM on 64-bit KVM is very rare.
3614          */
3615         if (mmu->root_role.direct ||
3616             mmu->cpu_role.base.level >= PT64_ROOT_4LEVEL ||
3617             mmu->root_role.level < PT64_ROOT_4LEVEL)
3618                 return 0;
3619
3620         /*
3621          * NPT, the only paging mode that uses this horror, uses a fixed number
3622          * of levels for the shadow page tables, e.g. all MMUs are 4-level or
3623          * all MMus are 5-level.  Thus, this can safely require that pml5_root
3624          * is allocated if the other roots are valid and pml5 is needed, as any
3625          * prior MMU would also have required pml5.
3626          */
3627         if (mmu->pae_root && mmu->pml4_root && (!need_pml5 || mmu->pml5_root))
3628                 return 0;
3629
3630         /*
3631          * The special roots should always be allocated in concert.  Yell and
3632          * bail if KVM ends up in a state where only one of the roots is valid.
3633          */
3634         if (WARN_ON_ONCE(!tdp_enabled || mmu->pae_root || mmu->pml4_root ||
3635                          (need_pml5 && mmu->pml5_root)))
3636                 return -EIO;
3637
3638         /*
3639          * Unlike 32-bit NPT, the PDP table doesn't need to be in low mem, and
3640          * doesn't need to be decrypted.
3641          */
3642         pae_root = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT);
3643         if (!pae_root)
3644                 return -ENOMEM;
3645
3646 #ifdef CONFIG_X86_64
3647         pml4_root = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT);
3648         if (!pml4_root)
3649                 goto err_pml4;
3650
3651         if (need_pml5) {
3652                 pml5_root = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT);
3653                 if (!pml5_root)
3654                         goto err_pml5;
3655         }
3656 #endif
3657
3658         mmu->pae_root = pae_root;
3659         mmu->pml4_root = pml4_root;
3660         mmu->pml5_root = pml5_root;
3661
3662         return 0;
3663
3664 #ifdef CONFIG_X86_64
3665 err_pml5:
3666         free_page((unsigned long)pml4_root);
3667 err_pml4:
3668         free_page((unsigned long)pae_root);
3669         return -ENOMEM;
3670 #endif
3671 }
3672
3673 static bool is_unsync_root(hpa_t root)
3674 {
3675         struct kvm_mmu_page *sp;
3676
3677         if (!VALID_PAGE(root))
3678                 return false;
3679
3680         /*
3681          * The read barrier orders the CPU's read of SPTE.W during the page table
3682          * walk before the reads of sp->unsync/sp->unsync_children here.
3683          *
3684          * Even if another CPU was marking the SP as unsync-ed simultaneously,
3685          * any guest page table changes are not guaranteed to be visible anyway
3686          * until this VCPU issues a TLB flush strictly after those changes are
3687          * made.  We only need to ensure that the other CPU sets these flags
3688          * before any actual changes to the page tables are made.  The comments
3689          * in mmu_try_to_unsync_pages() describe what could go wrong if this
3690          * requirement isn't satisfied.
3691          */
3692         smp_rmb();
3693         sp = to_shadow_page(root);
3694
3695         /*
3696          * PAE roots (somewhat arbitrarily) aren't backed by shadow pages, the
3697          * PDPTEs for a given PAE root need to be synchronized individually.
3698          */
3699         if (WARN_ON_ONCE(!sp))
3700                 return false;
3701
3702         if (sp->unsync || sp->unsync_children)
3703                 return true;
3704
3705         return false;
3706 }
3707
3708 void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu)
3709 {
3710         int i;
3711         struct kvm_mmu_page *sp;
3712
3713         if (vcpu->arch.mmu->root_role.direct)
3714                 return;
3715
3716         if (!VALID_PAGE(vcpu->arch.mmu->root.hpa))
3717                 return;
3718
3719         vcpu_clear_mmio_info(vcpu, MMIO_GVA_ANY);
3720
3721         if (vcpu->arch.mmu->cpu_role.base.level >= PT64_ROOT_4LEVEL) {
3722                 hpa_t root = vcpu->arch.mmu->root.hpa;
3723                 sp = to_shadow_page(root);
3724
3725                 if (!is_unsync_root(root))
3726                         return;
3727
3728                 write_lock(&vcpu->kvm->mmu_lock);
3729                 mmu_sync_children(vcpu, sp, true);
3730                 write_unlock(&vcpu->kvm->mmu_lock);
3731                 return;
3732         }
3733
3734         write_lock(&vcpu->kvm->mmu_lock);
3735
3736         for (i = 0; i < 4; ++i) {
3737                 hpa_t root = vcpu->arch.mmu->pae_root[i];
3738
3739                 if (IS_VALID_PAE_ROOT(root)) {
3740                         root &= PT64_BASE_ADDR_MASK;
3741                         sp = to_shadow_page(root);
3742                         mmu_sync_children(vcpu, sp, true);
3743                 }
3744         }
3745
3746         write_unlock(&vcpu->kvm->mmu_lock);
3747 }
3748
3749 void kvm_mmu_sync_prev_roots(struct kvm_vcpu *vcpu)
3750 {
3751         unsigned long roots_to_free = 0;
3752         int i;
3753
3754         for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
3755                 if (is_unsync_root(vcpu->arch.mmu->prev_roots[i].hpa))
3756                         roots_to_free |= KVM_MMU_ROOT_PREVIOUS(i);
3757
3758         /* sync prev_roots by simply freeing them */
3759         kvm_mmu_free_roots(vcpu->kvm, vcpu->arch.mmu, roots_to_free);
3760 }
3761
3762 static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
3763                                   gpa_t vaddr, u64 access,
3764                                   struct x86_exception *exception)
3765 {
3766         if (exception)
3767                 exception->error_code = 0;
3768         return kvm_translate_gpa(vcpu, mmu, vaddr, access, exception);
3769 }
3770
3771 static bool mmio_info_in_cache(struct kvm_vcpu *vcpu, u64 addr, bool direct)
3772 {
3773         /*
3774          * A nested guest cannot use the MMIO cache if it is using nested
3775          * page tables, because cr2 is a nGPA while the cache stores GPAs.
3776          */
3777         if (mmu_is_nested(vcpu))
3778                 return false;
3779
3780         if (direct)
3781                 return vcpu_match_mmio_gpa(vcpu, addr);
3782
3783         return vcpu_match_mmio_gva(vcpu, addr);
3784 }
3785
3786 /*
3787  * Return the level of the lowest level SPTE added to sptes.
3788  * That SPTE may be non-present.
3789  *
3790  * Must be called between walk_shadow_page_lockless_{begin,end}.
3791  */
3792 static int get_walk(struct kvm_vcpu *vcpu, u64 addr, u64 *sptes, int *root_level)
3793 {
3794         struct kvm_shadow_walk_iterator iterator;
3795         int leaf = -1;
3796         u64 spte;
3797
3798         for (shadow_walk_init(&iterator, vcpu, addr),
3799              *root_level = iterator.level;
3800              shadow_walk_okay(&iterator);
3801              __shadow_walk_next(&iterator, spte)) {
3802                 leaf = iterator.level;
3803                 spte = mmu_spte_get_lockless(iterator.sptep);
3804
3805                 sptes[leaf] = spte;
3806         }
3807
3808         return leaf;
3809 }
3810
3811 /* return true if reserved bit(s) are detected on a valid, non-MMIO SPTE. */
3812 static bool get_mmio_spte(struct kvm_vcpu *vcpu, u64 addr, u64 *sptep)
3813 {
3814         u64 sptes[PT64_ROOT_MAX_LEVEL + 1];
3815         struct rsvd_bits_validate *rsvd_check;
3816         int root, leaf, level;
3817         bool reserved = false;
3818
3819         walk_shadow_page_lockless_begin(vcpu);
3820
3821         if (is_tdp_mmu(vcpu->arch.mmu))
3822                 leaf = kvm_tdp_mmu_get_walk(vcpu, addr, sptes, &root);
3823         else
3824                 leaf = get_walk(vcpu, addr, sptes, &root);
3825
3826         walk_shadow_page_lockless_end(vcpu);
3827
3828         if (unlikely(leaf < 0)) {
3829                 *sptep = 0ull;
3830                 return reserved;
3831         }
3832
3833         *sptep = sptes[leaf];
3834
3835         /*
3836          * Skip reserved bits checks on the terminal leaf if it's not a valid
3837          * SPTE.  Note, this also (intentionally) skips MMIO SPTEs, which, by
3838          * design, always have reserved bits set.  The purpose of the checks is
3839          * to detect reserved bits on non-MMIO SPTEs. i.e. buggy SPTEs.
3840          */
3841         if (!is_shadow_present_pte(sptes[leaf]))
3842                 leaf++;
3843
3844         rsvd_check = &vcpu->arch.mmu->shadow_zero_check;
3845
3846         for (level = root; level >= leaf; level--)
3847                 reserved |= is_rsvd_spte(rsvd_check, sptes[level], level);
3848
3849         if (reserved) {
3850                 pr_err("%s: reserved bits set on MMU-present spte, addr 0x%llx, hierarchy:\n",
3851                        __func__, addr);
3852                 for (level = root; level >= leaf; level--)
3853                         pr_err("------ spte = 0x%llx level = %d, rsvd bits = 0x%llx",
3854                                sptes[level], level,
3855                                get_rsvd_bits(rsvd_check, sptes[level], level));
3856         }
3857
3858         return reserved;
3859 }
3860
3861 static int handle_mmio_page_fault(struct kvm_vcpu *vcpu, u64 addr, bool direct)
3862 {
3863         u64 spte;
3864         bool reserved;
3865
3866         if (mmio_info_in_cache(vcpu, addr, direct))
3867                 return RET_PF_EMULATE;
3868
3869         reserved = get_mmio_spte(vcpu, addr, &spte);
3870         if (WARN_ON(reserved))
3871                 return -EINVAL;
3872
3873         if (is_mmio_spte(spte)) {
3874                 gfn_t gfn = get_mmio_spte_gfn(spte);
3875                 unsigned int access = get_mmio_spte_access(spte);
3876
3877                 if (!check_mmio_spte(vcpu, spte))
3878                         return RET_PF_INVALID;
3879
3880                 if (direct)
3881                         addr = 0;
3882
3883                 trace_handle_mmio_page_fault(addr, gfn, access);
3884                 vcpu_cache_mmio_info(vcpu, addr, gfn, access);
3885                 return RET_PF_EMULATE;
3886         }
3887
3888         /*
3889          * If the page table is zapped by other cpus, let CPU fault again on
3890          * the address.
3891          */
3892         return RET_PF_RETRY;
3893 }
3894
3895 static bool page_fault_handle_page_track(struct kvm_vcpu *vcpu,
3896                                          struct kvm_page_fault *fault)
3897 {
3898         if (unlikely(fault->rsvd))
3899                 return false;
3900
3901         if (!fault->present || !fault->write)
3902                 return false;
3903
3904         /*
3905          * guest is writing the page which is write tracked which can
3906          * not be fixed by page fault handler.
3907          */
3908         if (kvm_slot_page_track_is_active(vcpu->kvm, fault->slot, fault->gfn, KVM_PAGE_TRACK_WRITE))
3909                 return true;
3910
3911         return false;
3912 }
3913
3914 static void shadow_page_table_clear_flood(struct kvm_vcpu *vcpu, gva_t addr)
3915 {
3916         struct kvm_shadow_walk_iterator iterator;
3917         u64 spte;
3918
3919         walk_shadow_page_lockless_begin(vcpu);
3920         for_each_shadow_entry_lockless(vcpu, addr, iterator, spte)
3921                 clear_sp_write_flooding_count(iterator.sptep);
3922         walk_shadow_page_lockless_end(vcpu);
3923 }
3924
3925 static u32 alloc_apf_token(struct kvm_vcpu *vcpu)
3926 {
3927         /* make sure the token value is not 0 */
3928         u32 id = vcpu->arch.apf.id;
3929
3930         if (id << 12 == 0)
3931                 vcpu->arch.apf.id = 1;
3932
3933         return (vcpu->arch.apf.id++ << 12) | vcpu->vcpu_id;
3934 }
3935
3936 static bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
3937                                     gfn_t gfn)
3938 {
3939         struct kvm_arch_async_pf arch;
3940
3941         arch.token = alloc_apf_token(vcpu);
3942         arch.gfn = gfn;
3943         arch.direct_map = vcpu->arch.mmu->root_role.direct;
3944         arch.cr3 = vcpu->arch.mmu->get_guest_pgd(vcpu);
3945
3946         return kvm_setup_async_pf(vcpu, cr2_or_gpa,
3947                                   kvm_vcpu_gfn_to_hva(vcpu, gfn), &arch);
3948 }
3949
3950 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, struct kvm_async_pf *work)
3951 {
3952         int r;
3953
3954         if ((vcpu->arch.mmu->root_role.direct != work->arch.direct_map) ||
3955               work->wakeup_all)
3956                 return;
3957
3958         r = kvm_mmu_reload(vcpu);
3959         if (unlikely(r))
3960                 return;
3961
3962         if (!vcpu->arch.mmu->root_role.direct &&
3963               work->arch.cr3 != vcpu->arch.mmu->get_guest_pgd(vcpu))
3964                 return;
3965
3966         kvm_mmu_do_page_fault(vcpu, work->cr2_or_gpa, 0, true);
3967 }
3968
3969 static int kvm_faultin_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
3970 {
3971         struct kvm_memory_slot *slot = fault->slot;
3972         bool async;
3973
3974         /*
3975          * Retry the page fault if the gfn hit a memslot that is being deleted
3976          * or moved.  This ensures any existing SPTEs for the old memslot will
3977          * be zapped before KVM inserts a new MMIO SPTE for the gfn.
3978          */
3979         if (slot && (slot->flags & KVM_MEMSLOT_INVALID))
3980                 return RET_PF_RETRY;
3981
3982         if (!kvm_is_visible_memslot(slot)) {
3983                 /* Don't expose private memslots to L2. */
3984                 if (is_guest_mode(vcpu)) {
3985                         fault->slot = NULL;
3986                         fault->pfn = KVM_PFN_NOSLOT;
3987                         fault->map_writable = false;
3988                         return RET_PF_CONTINUE;
3989                 }
3990                 /*
3991                  * If the APIC access page exists but is disabled, go directly
3992                  * to emulation without caching the MMIO access or creating a
3993                  * MMIO SPTE.  That way the cache doesn't need to be purged
3994                  * when the AVIC is re-enabled.
3995                  */
3996                 if (slot && slot->id == APIC_ACCESS_PAGE_PRIVATE_MEMSLOT &&
3997                     !kvm_apicv_activated(vcpu->kvm))
3998                         return RET_PF_EMULATE;
3999         }
4000
4001         async = false;
4002         fault->pfn = __gfn_to_pfn_memslot(slot, fault->gfn, false, &async,
4003                                           fault->write, &fault->map_writable,
4004                                           &fault->hva);
4005         if (!async)
4006                 return RET_PF_CONTINUE; /* *pfn has correct page already */
4007
4008         if (!fault->prefetch && kvm_can_do_async_pf(vcpu)) {
4009                 trace_kvm_try_async_get_page(fault->addr, fault->gfn);
4010                 if (kvm_find_async_pf_gfn(vcpu, fault->gfn)) {
4011                         trace_kvm_async_pf_doublefault(fault->addr, fault->gfn);
4012                         kvm_make_request(KVM_REQ_APF_HALT, vcpu);
4013                         return RET_PF_RETRY;
4014                 } else if (kvm_arch_setup_async_pf(vcpu, fault->addr, fault->gfn)) {
4015                         return RET_PF_RETRY;
4016                 }
4017         }
4018
4019         fault->pfn = __gfn_to_pfn_memslot(slot, fault->gfn, false, NULL,
4020                                           fault->write, &fault->map_writable,
4021                                           &fault->hva);
4022         return RET_PF_CONTINUE;
4023 }
4024
4025 /*
4026  * Returns true if the page fault is stale and needs to be retried, i.e. if the
4027  * root was invalidated by a memslot update or a relevant mmu_notifier fired.
4028  */
4029 static bool is_page_fault_stale(struct kvm_vcpu *vcpu,
4030                                 struct kvm_page_fault *fault, int mmu_seq)
4031 {
4032         struct kvm_mmu_page *sp = to_shadow_page(vcpu->arch.mmu->root.hpa);
4033
4034         /* Special roots, e.g. pae_root, are not backed by shadow pages. */
4035         if (sp && is_obsolete_sp(vcpu->kvm, sp))
4036                 return true;
4037
4038         /*
4039          * Roots without an associated shadow page are considered invalid if
4040          * there is a pending request to free obsolete roots.  The request is
4041          * only a hint that the current root _may_ be obsolete and needs to be
4042          * reloaded, e.g. if the guest frees a PGD that KVM is tracking as a
4043          * previous root, then __kvm_mmu_prepare_zap_page() signals all vCPUs
4044          * to reload even if no vCPU is actively using the root.
4045          */
4046         if (!sp && kvm_test_request(KVM_REQ_MMU_FREE_OBSOLETE_ROOTS, vcpu))
4047                 return true;
4048
4049         return fault->slot &&
4050                mmu_notifier_retry_hva(vcpu->kvm, mmu_seq, fault->hva);
4051 }
4052
4053 static int direct_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
4054 {
4055         bool is_tdp_mmu_fault = is_tdp_mmu(vcpu->arch.mmu);
4056
4057         unsigned long mmu_seq;
4058         int r;
4059
4060         fault->gfn = fault->addr >> PAGE_SHIFT;
4061         fault->slot = kvm_vcpu_gfn_to_memslot(vcpu, fault->gfn);
4062
4063         if (page_fault_handle_page_track(vcpu, fault))
4064                 return RET_PF_EMULATE;
4065
4066         r = fast_page_fault(vcpu, fault);
4067         if (r != RET_PF_INVALID)
4068                 return r;
4069
4070         r = mmu_topup_memory_caches(vcpu, false);
4071         if (r)
4072                 return r;
4073
4074         mmu_seq = vcpu->kvm->mmu_notifier_seq;
4075         smp_rmb();
4076
4077         r = kvm_faultin_pfn(vcpu, fault);
4078         if (r != RET_PF_CONTINUE)
4079                 return r;
4080
4081         r = handle_abnormal_pfn(vcpu, fault, ACC_ALL);
4082         if (r != RET_PF_CONTINUE)
4083                 return r;
4084
4085         r = RET_PF_RETRY;
4086
4087         if (is_tdp_mmu_fault)
4088                 read_lock(&vcpu->kvm->mmu_lock);
4089         else
4090                 write_lock(&vcpu->kvm->mmu_lock);
4091
4092         if (is_page_fault_stale(vcpu, fault, mmu_seq))
4093                 goto out_unlock;
4094
4095         r = make_mmu_pages_available(vcpu);
4096         if (r)
4097                 goto out_unlock;
4098
4099         if (is_tdp_mmu_fault)
4100                 r = kvm_tdp_mmu_map(vcpu, fault);
4101         else
4102                 r = __direct_map(vcpu, fault);
4103
4104 out_unlock:
4105         if (is_tdp_mmu_fault)
4106                 read_unlock(&vcpu->kvm->mmu_lock);
4107         else
4108                 write_unlock(&vcpu->kvm->mmu_lock);
4109         kvm_release_pfn_clean(fault->pfn);
4110         return r;
4111 }
4112
4113 static int nonpaging_page_fault(struct kvm_vcpu *vcpu,
4114                                 struct kvm_page_fault *fault)
4115 {
4116         pgprintk("%s: gva %lx error %x\n", __func__, fault->addr, fault->error_code);
4117
4118         /* This path builds a PAE pagetable, we can map 2mb pages at maximum. */
4119         fault->max_level = PG_LEVEL_2M;
4120         return direct_page_fault(vcpu, fault);
4121 }
4122
4123 int kvm_handle_page_fault(struct kvm_vcpu *vcpu, u64 error_code,
4124                                 u64 fault_address, char *insn, int insn_len)
4125 {
4126         int r = 1;
4127         u32 flags = vcpu->arch.apf.host_apf_flags;
4128
4129 #ifndef CONFIG_X86_64
4130         /* A 64-bit CR2 should be impossible on 32-bit KVM. */
4131         if (WARN_ON_ONCE(fault_address >> 32))
4132                 return -EFAULT;
4133 #endif
4134
4135         vcpu->arch.l1tf_flush_l1d = true;
4136         if (!flags) {
4137                 trace_kvm_page_fault(fault_address, error_code);
4138
4139                 if (kvm_event_needs_reinjection(vcpu))
4140                         kvm_mmu_unprotect_page_virt(vcpu, fault_address);
4141                 r = kvm_mmu_page_fault(vcpu, fault_address, error_code, insn,
4142                                 insn_len);
4143         } else if (flags & KVM_PV_REASON_PAGE_NOT_PRESENT) {
4144                 vcpu->arch.apf.host_apf_flags = 0;
4145                 local_irq_disable();
4146                 kvm_async_pf_task_wait_schedule(fault_address);
4147                 local_irq_enable();
4148         } else {
4149                 WARN_ONCE(1, "Unexpected host async PF flags: %x\n", flags);
4150         }
4151
4152         return r;
4153 }
4154 EXPORT_SYMBOL_GPL(kvm_handle_page_fault);
4155
4156 int kvm_tdp_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
4157 {
4158         while (fault->max_level > PG_LEVEL_4K) {
4159                 int page_num = KVM_PAGES_PER_HPAGE(fault->max_level);
4160                 gfn_t base = (fault->addr >> PAGE_SHIFT) & ~(page_num - 1);
4161
4162                 if (kvm_mtrr_check_gfn_range_consistency(vcpu, base, page_num))
4163                         break;
4164
4165                 --fault->max_level;
4166         }
4167
4168         return direct_page_fault(vcpu, fault);
4169 }
4170
4171 static void nonpaging_init_context(struct kvm_mmu *context)
4172 {
4173         context->page_fault = nonpaging_page_fault;
4174         context->gva_to_gpa = nonpaging_gva_to_gpa;
4175         context->sync_page = nonpaging_sync_page;
4176         context->invlpg = NULL;
4177 }
4178
4179 static inline bool is_root_usable(struct kvm_mmu_root_info *root, gpa_t pgd,
4180                                   union kvm_mmu_page_role role)
4181 {
4182         return (role.direct || pgd == root->pgd) &&
4183                VALID_PAGE(root->hpa) &&
4184                role.word == to_shadow_page(root->hpa)->role.word;
4185 }
4186
4187 /*
4188  * Find out if a previously cached root matching the new pgd/role is available,
4189  * and insert the current root as the MRU in the cache.
4190  * If a matching root is found, it is assigned to kvm_mmu->root and
4191  * true is returned.
4192  * If no match is found, kvm_mmu->root is left invalid, the LRU root is
4193  * evicted to make room for the current root, and false is returned.
4194  */
4195 static bool cached_root_find_and_keep_current(struct kvm *kvm, struct kvm_mmu *mmu,
4196                                               gpa_t new_pgd,
4197                                               union kvm_mmu_page_role new_role)
4198 {
4199         uint i;
4200
4201         if (is_root_usable(&mmu->root, new_pgd, new_role))
4202                 return true;
4203
4204         for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) {
4205                 /*
4206                  * The swaps end up rotating the cache like this:
4207                  *   C   0 1 2 3   (on entry to the function)
4208                  *   0   C 1 2 3
4209                  *   1   C 0 2 3
4210                  *   2   C 0 1 3
4211                  *   3   C 0 1 2   (on exit from the loop)
4212                  */
4213                 swap(mmu->root, mmu->prev_roots[i]);
4214                 if (is_root_usable(&mmu->root, new_pgd, new_role))
4215                         return true;
4216         }
4217
4218         kvm_mmu_free_roots(kvm, mmu, KVM_MMU_ROOT_CURRENT);
4219         return false;
4220 }
4221
4222 /*
4223  * Find out if a previously cached root matching the new pgd/role is available.
4224  * On entry, mmu->root is invalid.
4225  * If a matching root is found, it is assigned to kvm_mmu->root, the LRU entry
4226  * of the cache becomes invalid, and true is returned.
4227  * If no match is found, kvm_mmu->root is left invalid and false is returned.
4228  */
4229 static bool cached_root_find_without_current(struct kvm *kvm, struct kvm_mmu *mmu,
4230                                              gpa_t new_pgd,
4231                                              union kvm_mmu_page_role new_role)
4232 {
4233         uint i;
4234
4235         for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
4236                 if (is_root_usable(&mmu->prev_roots[i], new_pgd, new_role))
4237                         goto hit;
4238
4239         return false;
4240
4241 hit:
4242         swap(mmu->root, mmu->prev_roots[i]);
4243         /* Bubble up the remaining roots.  */
4244         for (; i < KVM_MMU_NUM_PREV_ROOTS - 1; i++)
4245                 mmu->prev_roots[i] = mmu->prev_roots[i + 1];
4246         mmu->prev_roots[i].hpa = INVALID_PAGE;
4247         return true;
4248 }
4249
4250 static bool fast_pgd_switch(struct kvm *kvm, struct kvm_mmu *mmu,
4251                             gpa_t new_pgd, union kvm_mmu_page_role new_role)
4252 {
4253         /*
4254          * For now, limit the caching to 64-bit hosts+VMs in order to avoid
4255          * having to deal with PDPTEs. We may add support for 32-bit hosts/VMs
4256          * later if necessary.
4257          */
4258         if (VALID_PAGE(mmu->root.hpa) && !to_shadow_page(mmu->root.hpa))
4259                 kvm_mmu_free_roots(kvm, mmu, KVM_MMU_ROOT_CURRENT);
4260
4261         if (VALID_PAGE(mmu->root.hpa))
4262                 return cached_root_find_and_keep_current(kvm, mmu, new_pgd, new_role);
4263         else
4264                 return cached_root_find_without_current(kvm, mmu, new_pgd, new_role);
4265 }
4266
4267 void kvm_mmu_new_pgd(struct kvm_vcpu *vcpu, gpa_t new_pgd)
4268 {
4269         struct kvm_mmu *mmu = vcpu->arch.mmu;
4270         union kvm_mmu_page_role new_role = mmu->root_role;
4271
4272         if (!fast_pgd_switch(vcpu->kvm, mmu, new_pgd, new_role)) {
4273                 /* kvm_mmu_ensure_valid_pgd will set up a new root.  */
4274                 return;
4275         }
4276
4277         /*
4278          * It's possible that the cached previous root page is obsolete because
4279          * of a change in the MMU generation number. However, changing the
4280          * generation number is accompanied by KVM_REQ_MMU_FREE_OBSOLETE_ROOTS,
4281          * which will free the root set here and allocate a new one.
4282          */
4283         kvm_make_request(KVM_REQ_LOAD_MMU_PGD, vcpu);
4284
4285         if (force_flush_and_sync_on_reuse) {
4286                 kvm_make_request(KVM_REQ_MMU_SYNC, vcpu);
4287                 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);
4288         }
4289
4290         /*
4291          * The last MMIO access's GVA and GPA are cached in the VCPU. When
4292          * switching to a new CR3, that GVA->GPA mapping may no longer be
4293          * valid. So clear any cached MMIO info even when we don't need to sync
4294          * the shadow page tables.
4295          */
4296         vcpu_clear_mmio_info(vcpu, MMIO_GVA_ANY);
4297
4298         /*
4299          * If this is a direct root page, it doesn't have a write flooding
4300          * count. Otherwise, clear the write flooding count.
4301          */
4302         if (!new_role.direct)
4303                 __clear_sp_write_flooding_count(
4304                                 to_shadow_page(vcpu->arch.mmu->root.hpa));
4305 }
4306 EXPORT_SYMBOL_GPL(kvm_mmu_new_pgd);
4307
4308 static unsigned long get_cr3(struct kvm_vcpu *vcpu)
4309 {
4310         return kvm_read_cr3(vcpu);
4311 }
4312
4313 static bool sync_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, gfn_t gfn,
4314                            unsigned int access)
4315 {
4316         if (unlikely(is_mmio_spte(*sptep))) {
4317                 if (gfn != get_mmio_spte_gfn(*sptep)) {
4318                         mmu_spte_clear_no_track(sptep);
4319                         return true;
4320                 }
4321
4322                 mark_mmio_spte(vcpu, sptep, gfn, access);
4323                 return true;
4324         }
4325
4326         return false;
4327 }
4328
4329 #define PTTYPE_EPT 18 /* arbitrary */
4330 #define PTTYPE PTTYPE_EPT
4331 #include "paging_tmpl.h"
4332 #undef PTTYPE
4333
4334 #define PTTYPE 64
4335 #include "paging_tmpl.h"
4336 #undef PTTYPE
4337
4338 #define PTTYPE 32
4339 #include "paging_tmpl.h"
4340 #undef PTTYPE
4341
4342 static void
4343 __reset_rsvds_bits_mask(struct rsvd_bits_validate *rsvd_check,
4344                         u64 pa_bits_rsvd, int level, bool nx, bool gbpages,
4345                         bool pse, bool amd)
4346 {
4347         u64 gbpages_bit_rsvd = 0;
4348         u64 nonleaf_bit8_rsvd = 0;
4349         u64 high_bits_rsvd;
4350
4351         rsvd_check->bad_mt_xwr = 0;
4352
4353         if (!gbpages)
4354                 gbpages_bit_rsvd = rsvd_bits(7, 7);
4355
4356         if (level == PT32E_ROOT_LEVEL)
4357                 high_bits_rsvd = pa_bits_rsvd & rsvd_bits(0, 62);
4358         else
4359                 high_bits_rsvd = pa_bits_rsvd & rsvd_bits(0, 51);
4360
4361         /* Note, NX doesn't exist in PDPTEs, this is handled below. */
4362         if (!nx)
4363                 high_bits_rsvd |= rsvd_bits(63, 63);
4364
4365         /*
4366          * Non-leaf PML4Es and PDPEs reserve bit 8 (which would be the G bit for
4367          * leaf entries) on AMD CPUs only.
4368          */
4369         if (amd)
4370                 nonleaf_bit8_rsvd = rsvd_bits(8, 8);
4371
4372         switch (level) {
4373         case PT32_ROOT_LEVEL:
4374                 /* no rsvd bits for 2 level 4K page table entries */
4375                 rsvd_check->rsvd_bits_mask[0][1] = 0;
4376                 rsvd_check->rsvd_bits_mask[0][0] = 0;
4377                 rsvd_check->rsvd_bits_mask[1][0] =
4378                         rsvd_check->rsvd_bits_mask[0][0];
4379
4380                 if (!pse) {
4381                         rsvd_check->rsvd_bits_mask[1][1] = 0;
4382                         break;
4383                 }
4384
4385                 if (is_cpuid_PSE36())
4386                         /* 36bits PSE 4MB page */
4387                         rsvd_check->rsvd_bits_mask[1][1] = rsvd_bits(17, 21);
4388                 else
4389                         /* 32 bits PSE 4MB page */
4390                         rsvd_check->rsvd_bits_mask[1][1] = rsvd_bits(13, 21);
4391                 break;
4392         case PT32E_ROOT_LEVEL:
4393                 rsvd_check->rsvd_bits_mask[0][2] = rsvd_bits(63, 63) |
4394                                                    high_bits_rsvd |
4395                                                    rsvd_bits(5, 8) |
4396                                                    rsvd_bits(1, 2);     /* PDPTE */
4397                 rsvd_check->rsvd_bits_mask[0][1] = high_bits_rsvd;      /* PDE */
4398                 rsvd_check->rsvd_bits_mask[0][0] = high_bits_rsvd;      /* PTE */
4399                 rsvd_check->rsvd_bits_mask[1][1] = high_bits_rsvd |
4400                                                    rsvd_bits(13, 20);   /* large page */
4401                 rsvd_check->rsvd_bits_mask[1][0] =
4402                         rsvd_check->rsvd_bits_mask[0][0];
4403                 break;
4404         case PT64_ROOT_5LEVEL:
4405                 rsvd_check->rsvd_bits_mask[0][4] = high_bits_rsvd |
4406                                                    nonleaf_bit8_rsvd |
4407                                                    rsvd_bits(7, 7);
4408                 rsvd_check->rsvd_bits_mask[1][4] =
4409                         rsvd_check->rsvd_bits_mask[0][4];
4410                 fallthrough;
4411         case PT64_ROOT_4LEVEL:
4412                 rsvd_check->rsvd_bits_mask[0][3] = high_bits_rsvd |
4413                                                    nonleaf_bit8_rsvd |
4414                                                    rsvd_bits(7, 7);
4415                 rsvd_check->rsvd_bits_mask[0][2] = high_bits_rsvd |
4416                                                    gbpages_bit_rsvd;
4417                 rsvd_check->rsvd_bits_mask[0][1] = high_bits_rsvd;
4418                 rsvd_check->rsvd_bits_mask[0][0] = high_bits_rsvd;
4419                 rsvd_check->rsvd_bits_mask[1][3] =
4420                         rsvd_check->rsvd_bits_mask[0][3];
4421                 rsvd_check->rsvd_bits_mask[1][2] = high_bits_rsvd |
4422                                                    gbpages_bit_rsvd |
4423                                                    rsvd_bits(13, 29);
4424                 rsvd_check->rsvd_bits_mask[1][1] = high_bits_rsvd |
4425                                                    rsvd_bits(13, 20); /* large page */
4426                 rsvd_check->rsvd_bits_mask[1][0] =
4427                         rsvd_check->rsvd_bits_mask[0][0];
4428                 break;
4429         }
4430 }
4431
4432 static bool guest_can_use_gbpages(struct kvm_vcpu *vcpu)
4433 {
4434         /*
4435          * If TDP is enabled, let the guest use GBPAGES if they're supported in
4436          * hardware.  The hardware page walker doesn't let KVM disable GBPAGES,
4437          * i.e. won't treat them as reserved, and KVM doesn't redo the GVA->GPA
4438          * walk for performance and complexity reasons.  Not to mention KVM
4439          * _can't_ solve the problem because GVA->GPA walks aren't visible to
4440          * KVM once a TDP translation is installed.  Mimic hardware behavior so
4441          * that KVM's is at least consistent, i.e. doesn't randomly inject #PF.
4442          */
4443         return tdp_enabled ? boot_cpu_has(X86_FEATURE_GBPAGES) :
4444                              guest_cpuid_has(vcpu, X86_FEATURE_GBPAGES);
4445 }
4446
4447 static void reset_guest_rsvds_bits_mask(struct kvm_vcpu *vcpu,
4448                                         struct kvm_mmu *context)
4449 {
4450         __reset_rsvds_bits_mask(&context->guest_rsvd_check,
4451                                 vcpu->arch.reserved_gpa_bits,
4452                                 context->cpu_role.base.level, is_efer_nx(context),
4453                                 guest_can_use_gbpages(vcpu),
4454                                 is_cr4_pse(context),
4455                                 guest_cpuid_is_amd_or_hygon(vcpu));
4456 }
4457
4458 static void
4459 __reset_rsvds_bits_mask_ept(struct rsvd_bits_validate *rsvd_check,
4460                             u64 pa_bits_rsvd, bool execonly, int huge_page_level)
4461 {
4462         u64 high_bits_rsvd = pa_bits_rsvd & rsvd_bits(0, 51);
4463         u64 large_1g_rsvd = 0, large_2m_rsvd = 0;
4464         u64 bad_mt_xwr;
4465
4466         if (huge_page_level < PG_LEVEL_1G)
4467                 large_1g_rsvd = rsvd_bits(7, 7);
4468         if (huge_page_level < PG_LEVEL_2M)
4469                 large_2m_rsvd = rsvd_bits(7, 7);
4470
4471         rsvd_check->rsvd_bits_mask[0][4] = high_bits_rsvd | rsvd_bits(3, 7);
4472         rsvd_check->rsvd_bits_mask[0][3] = high_bits_rsvd | rsvd_bits(3, 7);
4473         rsvd_check->rsvd_bits_mask[0][2] = high_bits_rsvd | rsvd_bits(3, 6) | large_1g_rsvd;
4474         rsvd_check->rsvd_bits_mask[0][1] = high_bits_rsvd | rsvd_bits(3, 6) | large_2m_rsvd;
4475         rsvd_check->rsvd_bits_mask[0][0] = high_bits_rsvd;
4476
4477         /* large page */
4478         rsvd_check->rsvd_bits_mask[1][4] = rsvd_check->rsvd_bits_mask[0][4];
4479         rsvd_check->rsvd_bits_mask[1][3] = rsvd_check->rsvd_bits_mask[0][3];
4480         rsvd_check->rsvd_bits_mask[1][2] = high_bits_rsvd | rsvd_bits(12, 29) | large_1g_rsvd;
4481         rsvd_check->rsvd_bits_mask[1][1] = high_bits_rsvd | rsvd_bits(12, 20) | large_2m_rsvd;
4482         rsvd_check->rsvd_bits_mask[1][0] = rsvd_check->rsvd_bits_mask[0][0];
4483
4484         bad_mt_xwr = 0xFFull << (2 * 8);        /* bits 3..5 must not be 2 */
4485         bad_mt_xwr |= 0xFFull << (3 * 8);       /* bits 3..5 must not be 3 */
4486         bad_mt_xwr |= 0xFFull << (7 * 8);       /* bits 3..5 must not be 7 */
4487         bad_mt_xwr |= REPEAT_BYTE(1ull << 2);   /* bits 0..2 must not be 010 */
4488         bad_mt_xwr |= REPEAT_BYTE(1ull << 6);   /* bits 0..2 must not be 110 */
4489         if (!execonly) {
4490                 /* bits 0..2 must not be 100 unless VMX capabilities allow it */
4491                 bad_mt_xwr |= REPEAT_BYTE(1ull << 4);
4492         }
4493         rsvd_check->bad_mt_xwr = bad_mt_xwr;
4494 }
4495
4496 static void reset_rsvds_bits_mask_ept(struct kvm_vcpu *vcpu,
4497                 struct kvm_mmu *context, bool execonly, int huge_page_level)
4498 {
4499         __reset_rsvds_bits_mask_ept(&context->guest_rsvd_check,
4500                                     vcpu->arch.reserved_gpa_bits, execonly,
4501                                     huge_page_level);
4502 }
4503
4504 static inline u64 reserved_hpa_bits(void)
4505 {
4506         return rsvd_bits(shadow_phys_bits, 63);
4507 }
4508
4509 /*
4510  * the page table on host is the shadow page table for the page
4511  * table in guest or amd nested guest, its mmu features completely
4512  * follow the features in guest.
4513  */
4514 static void reset_shadow_zero_bits_mask(struct kvm_vcpu *vcpu,
4515                                         struct kvm_mmu *context)
4516 {
4517         /* @amd adds a check on bit of SPTEs, which KVM shouldn't use anyways. */
4518         bool is_amd = true;
4519         /* KVM doesn't use 2-level page tables for the shadow MMU. */
4520         bool is_pse = false;
4521         struct rsvd_bits_validate *shadow_zero_check;
4522         int i;
4523
4524         WARN_ON_ONCE(context->root_role.level < PT32E_ROOT_LEVEL);
4525
4526         shadow_zero_check = &context->shadow_zero_check;
4527         __reset_rsvds_bits_mask(shadow_zero_check, reserved_hpa_bits(),
4528                                 context->root_role.level,
4529                                 context->root_role.efer_nx,
4530                                 guest_can_use_gbpages(vcpu), is_pse, is_amd);
4531
4532         if (!shadow_me_mask)
4533                 return;
4534
4535         for (i = context->root_role.level; --i >= 0;) {
4536                 /*
4537                  * So far shadow_me_value is a constant during KVM's life
4538                  * time.  Bits in shadow_me_value are allowed to be set.
4539                  * Bits in shadow_me_mask but not in shadow_me_value are
4540                  * not allowed to be set.
4541                  */
4542                 shadow_zero_check->rsvd_bits_mask[0][i] |= shadow_me_mask;
4543                 shadow_zero_check->rsvd_bits_mask[1][i] |= shadow_me_mask;
4544                 shadow_zero_check->rsvd_bits_mask[0][i] &= ~shadow_me_value;
4545                 shadow_zero_check->rsvd_bits_mask[1][i] &= ~shadow_me_value;
4546         }
4547
4548 }
4549
4550 static inline bool boot_cpu_is_amd(void)
4551 {
4552         WARN_ON_ONCE(!tdp_enabled);
4553         return shadow_x_mask == 0;
4554 }
4555
4556 /*
4557  * the direct page table on host, use as much mmu features as
4558  * possible, however, kvm currently does not do execution-protection.
4559  */
4560 static void
4561 reset_tdp_shadow_zero_bits_mask(struct kvm_mmu *context)
4562 {
4563         struct rsvd_bits_validate *shadow_zero_check;
4564         int i;
4565
4566         shadow_zero_check = &context->shadow_zero_check;
4567
4568         if (boot_cpu_is_amd())
4569                 __reset_rsvds_bits_mask(shadow_zero_check, reserved_hpa_bits(),
4570                                         context->root_role.level, false,
4571                                         boot_cpu_has(X86_FEATURE_GBPAGES),
4572                                         false, true);
4573         else
4574                 __reset_rsvds_bits_mask_ept(shadow_zero_check,
4575                                             reserved_hpa_bits(), false,
4576                                             max_huge_page_level);
4577
4578         if (!shadow_me_mask)
4579                 return;
4580
4581         for (i = context->root_role.level; --i >= 0;) {
4582                 shadow_zero_check->rsvd_bits_mask[0][i] &= ~shadow_me_mask;
4583                 shadow_zero_check->rsvd_bits_mask[1][i] &= ~shadow_me_mask;
4584         }
4585 }
4586
4587 /*
4588  * as the comments in reset_shadow_zero_bits_mask() except it
4589  * is the shadow page table for intel nested guest.
4590  */
4591 static void
4592 reset_ept_shadow_zero_bits_mask(struct kvm_mmu *context, bool execonly)
4593 {
4594         __reset_rsvds_bits_mask_ept(&context->shadow_zero_check,
4595                                     reserved_hpa_bits(), execonly,
4596                                     max_huge_page_level);
4597 }
4598
4599 #define BYTE_MASK(access) \
4600         ((1 & (access) ? 2 : 0) | \
4601          (2 & (access) ? 4 : 0) | \
4602          (3 & (access) ? 8 : 0) | \
4603          (4 & (access) ? 16 : 0) | \
4604          (5 & (access) ? 32 : 0) | \
4605          (6 & (access) ? 64 : 0) | \
4606          (7 & (access) ? 128 : 0))
4607
4608
4609 static void update_permission_bitmask(struct kvm_mmu *mmu, bool ept)
4610 {
4611         unsigned byte;
4612
4613         const u8 x = BYTE_MASK(ACC_EXEC_MASK);
4614         const u8 w = BYTE_MASK(ACC_WRITE_MASK);
4615         const u8 u = BYTE_MASK(ACC_USER_MASK);
4616
4617         bool cr4_smep = is_cr4_smep(mmu);
4618         bool cr4_smap = is_cr4_smap(mmu);
4619         bool cr0_wp = is_cr0_wp(mmu);
4620         bool efer_nx = is_efer_nx(mmu);
4621
4622         for (byte = 0; byte < ARRAY_SIZE(mmu->permissions); ++byte) {
4623                 unsigned pfec = byte << 1;
4624
4625                 /*
4626                  * Each "*f" variable has a 1 bit for each UWX value
4627                  * that causes a fault with the given PFEC.
4628                  */
4629
4630                 /* Faults from writes to non-writable pages */
4631                 u8 wf = (pfec & PFERR_WRITE_MASK) ? (u8)~w : 0;
4632                 /* Faults from user mode accesses to supervisor pages */
4633                 u8 uf = (pfec & PFERR_USER_MASK) ? (u8)~u : 0;
4634                 /* Faults from fetches of non-executable pages*/
4635                 u8 ff = (pfec & PFERR_FETCH_MASK) ? (u8)~x : 0;
4636                 /* Faults from kernel mode fetches of user pages */
4637                 u8 smepf = 0;
4638                 /* Faults from kernel mode accesses of user pages */
4639                 u8 smapf = 0;
4640
4641                 if (!ept) {
4642                         /* Faults from kernel mode accesses to user pages */
4643                         u8 kf = (pfec & PFERR_USER_MASK) ? 0 : u;
4644
4645                         /* Not really needed: !nx will cause pte.nx to fault */
4646                         if (!efer_nx)
4647                                 ff = 0;
4648
4649                         /* Allow supervisor writes if !cr0.wp */
4650                         if (!cr0_wp)
4651                                 wf = (pfec & PFERR_USER_MASK) ? wf : 0;
4652
4653                         /* Disallow supervisor fetches of user code if cr4.smep */
4654                         if (cr4_smep)
4655                                 smepf = (pfec & PFERR_FETCH_MASK) ? kf : 0;
4656
4657                         /*
4658                          * SMAP:kernel-mode data accesses from user-mode
4659                          * mappings should fault. A fault is considered
4660                          * as a SMAP violation if all of the following
4661                          * conditions are true:
4662                          *   - X86_CR4_SMAP is set in CR4
4663                          *   - A user page is accessed
4664                          *   - The access is not a fetch
4665                          *   - The access is supervisor mode
4666                          *   - If implicit supervisor access or X86_EFLAGS_AC is clear
4667                          *
4668                          * Here, we cover the first four conditions.
4669                          * The fifth is computed dynamically in permission_fault();
4670                          * PFERR_RSVD_MASK bit will be set in PFEC if the access is
4671                          * *not* subject to SMAP restrictions.
4672                          */
4673                         if (cr4_smap)
4674                                 smapf = (pfec & (PFERR_RSVD_MASK|PFERR_FETCH_MASK)) ? 0 : kf;
4675                 }
4676
4677                 mmu->permissions[byte] = ff | uf | wf | smepf | smapf;
4678         }
4679 }
4680
4681 /*
4682 * PKU is an additional mechanism by which the paging controls access to
4683 * user-mode addresses based on the value in the PKRU register.  Protection
4684 * key violations are reported through a bit in the page fault error code.
4685 * Unlike other bits of the error code, the PK bit is not known at the
4686 * call site of e.g. gva_to_gpa; it must be computed directly in
4687 * permission_fault based on two bits of PKRU, on some machine state (CR4,
4688 * CR0, EFER, CPL), and on other bits of the error code and the page tables.
4689 *
4690 * In particular the following conditions come from the error code, the
4691 * page tables and the machine state:
4692 * - PK is always zero unless CR4.PKE=1 and EFER.LMA=1
4693 * - PK is always zero if RSVD=1 (reserved bit set) or F=1 (instruction fetch)
4694 * - PK is always zero if U=0 in the page tables
4695 * - PKRU.WD is ignored if CR0.WP=0 and the access is a supervisor access.
4696 *
4697 * The PKRU bitmask caches the result of these four conditions.  The error
4698 * code (minus the P bit) and the page table's U bit form an index into the
4699 * PKRU bitmask.  Two bits of the PKRU bitmask are then extracted and ANDed
4700 * with the two bits of the PKRU register corresponding to the protection key.
4701 * For the first three conditions above the bits will be 00, thus masking
4702 * away both AD and WD.  For all reads or if the last condition holds, WD
4703 * only will be masked away.
4704 */
4705 static void update_pkru_bitmask(struct kvm_mmu *mmu)
4706 {
4707         unsigned bit;
4708         bool wp;
4709
4710         mmu->pkru_mask = 0;
4711
4712         if (!is_cr4_pke(mmu))
4713                 return;
4714
4715         wp = is_cr0_wp(mmu);
4716
4717         for (bit = 0; bit < ARRAY_SIZE(mmu->permissions); ++bit) {
4718                 unsigned pfec, pkey_bits;
4719                 bool check_pkey, check_write, ff, uf, wf, pte_user;
4720
4721                 pfec = bit << 1;
4722                 ff = pfec & PFERR_FETCH_MASK;
4723                 uf = pfec & PFERR_USER_MASK;
4724                 wf = pfec & PFERR_WRITE_MASK;
4725
4726                 /* PFEC.RSVD is replaced by ACC_USER_MASK. */
4727                 pte_user = pfec & PFERR_RSVD_MASK;
4728
4729                 /*
4730                  * Only need to check the access which is not an
4731                  * instruction fetch and is to a user page.
4732                  */
4733                 check_pkey = (!ff && pte_user);
4734                 /*
4735                  * write access is controlled by PKRU if it is a
4736                  * user access or CR0.WP = 1.
4737                  */
4738                 check_write = check_pkey && wf && (uf || wp);
4739
4740                 /* PKRU.AD stops both read and write access. */
4741                 pkey_bits = !!check_pkey;
4742                 /* PKRU.WD stops write access. */
4743                 pkey_bits |= (!!check_write) << 1;
4744
4745                 mmu->pkru_mask |= (pkey_bits & 3) << pfec;
4746         }
4747 }
4748
4749 static void reset_guest_paging_metadata(struct kvm_vcpu *vcpu,
4750                                         struct kvm_mmu *mmu)
4751 {
4752         if (!is_cr0_pg(mmu))
4753                 return;
4754
4755         reset_guest_rsvds_bits_mask(vcpu, mmu);
4756         update_permission_bitmask(mmu, false);
4757         update_pkru_bitmask(mmu);
4758 }
4759
4760 static void paging64_init_context(struct kvm_mmu *context)
4761 {
4762         context->page_fault = paging64_page_fault;
4763         context->gva_to_gpa = paging64_gva_to_gpa;
4764         context->sync_page = paging64_sync_page;
4765         context->invlpg = paging64_invlpg;
4766 }
4767
4768 static void paging32_init_context(struct kvm_mmu *context)
4769 {
4770         context->page_fault = paging32_page_fault;
4771         context->gva_to_gpa = paging32_gva_to_gpa;
4772         context->sync_page = paging32_sync_page;
4773         context->invlpg = paging32_invlpg;
4774 }
4775
4776 static union kvm_cpu_role
4777 kvm_calc_cpu_role(struct kvm_vcpu *vcpu, const struct kvm_mmu_role_regs *regs)
4778 {
4779         union kvm_cpu_role role = {0};
4780
4781         role.base.access = ACC_ALL;
4782         role.base.smm = is_smm(vcpu);
4783         role.base.guest_mode = is_guest_mode(vcpu);
4784         role.ext.valid = 1;
4785
4786         if (!____is_cr0_pg(regs)) {
4787                 role.base.direct = 1;
4788                 return role;
4789         }
4790
4791         role.base.efer_nx = ____is_efer_nx(regs);
4792         role.base.cr0_wp = ____is_cr0_wp(regs);
4793         role.base.smep_andnot_wp = ____is_cr4_smep(regs) && !____is_cr0_wp(regs);
4794         role.base.smap_andnot_wp = ____is_cr4_smap(regs) && !____is_cr0_wp(regs);
4795         role.base.has_4_byte_gpte = !____is_cr4_pae(regs);
4796
4797         if (____is_efer_lma(regs))
4798                 role.base.level = ____is_cr4_la57(regs) ? PT64_ROOT_5LEVEL
4799                                                         : PT64_ROOT_4LEVEL;
4800         else if (____is_cr4_pae(regs))
4801                 role.base.level = PT32E_ROOT_LEVEL;
4802         else
4803                 role.base.level = PT32_ROOT_LEVEL;
4804
4805         role.ext.cr4_smep = ____is_cr4_smep(regs);
4806         role.ext.cr4_smap = ____is_cr4_smap(regs);
4807         role.ext.cr4_pse = ____is_cr4_pse(regs);
4808
4809         /* PKEY and LA57 are active iff long mode is active. */
4810         role.ext.cr4_pke = ____is_efer_lma(regs) && ____is_cr4_pke(regs);
4811         role.ext.cr4_la57 = ____is_efer_lma(regs) && ____is_cr4_la57(regs);
4812         role.ext.efer_lma = ____is_efer_lma(regs);
4813         return role;
4814 }
4815
4816 static inline int kvm_mmu_get_tdp_level(struct kvm_vcpu *vcpu)
4817 {
4818         /* tdp_root_level is architecture forced level, use it if nonzero */
4819         if (tdp_root_level)
4820                 return tdp_root_level;
4821
4822         /* Use 5-level TDP if and only if it's useful/necessary. */
4823         if (max_tdp_level == 5 && cpuid_maxphyaddr(vcpu) <= 48)
4824                 return 4;
4825
4826         return max_tdp_level;
4827 }
4828
4829 static union kvm_mmu_page_role
4830 kvm_calc_tdp_mmu_root_page_role(struct kvm_vcpu *vcpu,
4831                                 union kvm_cpu_role cpu_role)
4832 {
4833         union kvm_mmu_page_role role = {0};
4834
4835         role.access = ACC_ALL;
4836         role.cr0_wp = true;
4837         role.efer_nx = true;
4838         role.smm = cpu_role.base.smm;
4839         role.guest_mode = cpu_role.base.guest_mode;
4840         role.ad_disabled = !kvm_ad_enabled();
4841         role.level = kvm_mmu_get_tdp_level(vcpu);
4842         role.direct = true;
4843         role.has_4_byte_gpte = false;
4844
4845         return role;
4846 }
4847
4848 static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu,
4849                              union kvm_cpu_role cpu_role)
4850 {
4851         struct kvm_mmu *context = &vcpu->arch.root_mmu;
4852         union kvm_mmu_page_role root_role = kvm_calc_tdp_mmu_root_page_role(vcpu, cpu_role);
4853
4854         if (cpu_role.as_u64 == context->cpu_role.as_u64 &&
4855             root_role.word == context->root_role.word)
4856                 return;
4857
4858         context->cpu_role.as_u64 = cpu_role.as_u64;
4859         context->root_role.word = root_role.word;
4860         context->page_fault = kvm_tdp_page_fault;
4861         context->sync_page = nonpaging_sync_page;
4862         context->invlpg = NULL;
4863         context->get_guest_pgd = get_cr3;
4864         context->get_pdptr = kvm_pdptr_read;
4865         context->inject_page_fault = kvm_inject_page_fault;
4866
4867         if (!is_cr0_pg(context))
4868                 context->gva_to_gpa = nonpaging_gva_to_gpa;
4869         else if (is_cr4_pae(context))
4870                 context->gva_to_gpa = paging64_gva_to_gpa;
4871         else
4872                 context->gva_to_gpa = paging32_gva_to_gpa;
4873
4874         reset_guest_paging_metadata(vcpu, context);
4875         reset_tdp_shadow_zero_bits_mask(context);
4876 }
4877
4878 static void shadow_mmu_init_context(struct kvm_vcpu *vcpu, struct kvm_mmu *context,
4879                                     union kvm_cpu_role cpu_role,
4880                                     union kvm_mmu_page_role root_role)
4881 {
4882         if (cpu_role.as_u64 == context->cpu_role.as_u64 &&
4883             root_role.word == context->root_role.word)
4884                 return;
4885
4886         context->cpu_role.as_u64 = cpu_role.as_u64;
4887         context->root_role.word = root_role.word;
4888
4889         if (!is_cr0_pg(context))
4890                 nonpaging_init_context(context);
4891         else if (is_cr4_pae(context))
4892                 paging64_init_context(context);
4893         else
4894                 paging32_init_context(context);
4895
4896         reset_guest_paging_metadata(vcpu, context);
4897         reset_shadow_zero_bits_mask(vcpu, context);
4898 }
4899
4900 static void kvm_init_shadow_mmu(struct kvm_vcpu *vcpu,
4901                                 union kvm_cpu_role cpu_role)
4902 {
4903         struct kvm_mmu *context = &vcpu->arch.root_mmu;
4904         union kvm_mmu_page_role root_role;
4905
4906         root_role = cpu_role.base;
4907
4908         /* KVM uses PAE paging whenever the guest isn't using 64-bit paging. */
4909         root_role.level = max_t(u32, root_role.level, PT32E_ROOT_LEVEL);
4910
4911         /*
4912          * KVM forces EFER.NX=1 when TDP is disabled, reflect it in the MMU role.
4913          * KVM uses NX when TDP is disabled to handle a variety of scenarios,
4914          * notably for huge SPTEs if iTLB multi-hit mitigation is enabled and
4915          * to generate correct permissions for CR0.WP=0/CR4.SMEP=1/EFER.NX=0.
4916          * The iTLB multi-hit workaround can be toggled at any time, so assume
4917          * NX can be used by any non-nested shadow MMU to avoid having to reset
4918          * MMU contexts.
4919          */
4920         root_role.efer_nx = true;
4921
4922         shadow_mmu_init_context(vcpu, context, cpu_role, root_role);
4923 }
4924
4925 void kvm_init_shadow_npt_mmu(struct kvm_vcpu *vcpu, unsigned long cr0,
4926                              unsigned long cr4, u64 efer, gpa_t nested_cr3)
4927 {
4928         struct kvm_mmu *context = &vcpu->arch.guest_mmu;
4929         struct kvm_mmu_role_regs regs = {
4930                 .cr0 = cr0,
4931                 .cr4 = cr4 & ~X86_CR4_PKE,
4932                 .efer = efer,
4933         };
4934         union kvm_cpu_role cpu_role = kvm_calc_cpu_role(vcpu, &regs);
4935         union kvm_mmu_page_role root_role;
4936
4937         /* NPT requires CR0.PG=1. */
4938         WARN_ON_ONCE(cpu_role.base.direct);
4939
4940         root_role = cpu_role.base;
4941         root_role.level = kvm_mmu_get_tdp_level(vcpu);
4942         if (root_role.level == PT64_ROOT_5LEVEL &&
4943             cpu_role.base.level == PT64_ROOT_4LEVEL)
4944                 root_role.passthrough = 1;
4945
4946         shadow_mmu_init_context(vcpu, context, cpu_role, root_role);
4947         kvm_mmu_new_pgd(vcpu, nested_cr3);
4948 }
4949 EXPORT_SYMBOL_GPL(kvm_init_shadow_npt_mmu);
4950
4951 static union kvm_cpu_role
4952 kvm_calc_shadow_ept_root_page_role(struct kvm_vcpu *vcpu, bool accessed_dirty,
4953                                    bool execonly, u8 level)
4954 {
4955         union kvm_cpu_role role = {0};
4956
4957         /*
4958          * KVM does not support SMM transfer monitors, and consequently does not
4959          * support the "entry to SMM" control either.  role.base.smm is always 0.
4960          */
4961         WARN_ON_ONCE(is_smm(vcpu));
4962         role.base.level = level;
4963         role.base.has_4_byte_gpte = false;
4964         role.base.direct = false;
4965         role.base.ad_disabled = !accessed_dirty;
4966         role.base.guest_mode = true;
4967         role.base.access = ACC_ALL;
4968
4969         role.ext.word = 0;
4970         role.ext.execonly = execonly;
4971         role.ext.valid = 1;
4972
4973         return role;
4974 }
4975
4976 void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly,
4977                              int huge_page_level, bool accessed_dirty,
4978                              gpa_t new_eptp)
4979 {
4980         struct kvm_mmu *context = &vcpu->arch.guest_mmu;
4981         u8 level = vmx_eptp_page_walk_level(new_eptp);
4982         union kvm_cpu_role new_mode =
4983                 kvm_calc_shadow_ept_root_page_role(vcpu, accessed_dirty,
4984                                                    execonly, level);
4985
4986         if (new_mode.as_u64 != context->cpu_role.as_u64) {
4987                 /* EPT, and thus nested EPT, does not consume CR0, CR4, nor EFER. */
4988                 context->cpu_role.as_u64 = new_mode.as_u64;
4989                 context->root_role.word = new_mode.base.word;
4990
4991                 context->page_fault = ept_page_fault;
4992                 context->gva_to_gpa = ept_gva_to_gpa;
4993                 context->sync_page = ept_sync_page;
4994                 context->invlpg = ept_invlpg;
4995
4996                 update_permission_bitmask(context, true);
4997                 context->pkru_mask = 0;
4998                 reset_rsvds_bits_mask_ept(vcpu, context, execonly, huge_page_level);
4999                 reset_ept_shadow_zero_bits_mask(context, execonly);
5000         }
5001
5002         kvm_mmu_new_pgd(vcpu, new_eptp);
5003 }
5004 EXPORT_SYMBOL_GPL(kvm_init_shadow_ept_mmu);
5005
5006 static void init_kvm_softmmu(struct kvm_vcpu *vcpu,
5007                              union kvm_cpu_role cpu_role)
5008 {
5009         struct kvm_mmu *context = &vcpu->arch.root_mmu;
5010
5011         kvm_init_shadow_mmu(vcpu, cpu_role);
5012
5013         context->get_guest_pgd     = get_cr3;
5014         context->get_pdptr         = kvm_pdptr_read;
5015         context->inject_page_fault = kvm_inject_page_fault;
5016 }
5017
5018 static void init_kvm_nested_mmu(struct kvm_vcpu *vcpu,
5019                                 union kvm_cpu_role new_mode)
5020 {
5021         struct kvm_mmu *g_context = &vcpu->arch.nested_mmu;
5022
5023         if (new_mode.as_u64 == g_context->cpu_role.as_u64)
5024                 return;
5025
5026         g_context->cpu_role.as_u64   = new_mode.as_u64;
5027         g_context->get_guest_pgd     = get_cr3;
5028         g_context->get_pdptr         = kvm_pdptr_read;
5029         g_context->inject_page_fault = kvm_inject_page_fault;
5030
5031         /*
5032          * L2 page tables are never shadowed, so there is no need to sync
5033          * SPTEs.
5034          */
5035         g_context->invlpg            = NULL;
5036
5037         /*
5038          * Note that arch.mmu->gva_to_gpa translates l2_gpa to l1_gpa using
5039          * L1's nested page tables (e.g. EPT12). The nested translation
5040          * of l2_gva to l1_gpa is done by arch.nested_mmu.gva_to_gpa using
5041          * L2's page tables as the first level of translation and L1's
5042          * nested page tables as the second level of translation. Basically
5043          * the gva_to_gpa functions between mmu and nested_mmu are swapped.
5044          */
5045         if (!is_paging(vcpu))
5046                 g_context->gva_to_gpa = nonpaging_gva_to_gpa;
5047         else if (is_long_mode(vcpu))
5048                 g_context->gva_to_gpa = paging64_gva_to_gpa;
5049         else if (is_pae(vcpu))
5050                 g_context->gva_to_gpa = paging64_gva_to_gpa;
5051         else
5052                 g_context->gva_to_gpa = paging32_gva_to_gpa;
5053
5054         reset_guest_paging_metadata(vcpu, g_context);
5055 }
5056
5057 void kvm_init_mmu(struct kvm_vcpu *vcpu)
5058 {
5059         struct kvm_mmu_role_regs regs = vcpu_to_role_regs(vcpu);
5060         union kvm_cpu_role cpu_role = kvm_calc_cpu_role(vcpu, &regs);
5061
5062         if (mmu_is_nested(vcpu))
5063                 init_kvm_nested_mmu(vcpu, cpu_role);
5064         else if (tdp_enabled)
5065                 init_kvm_tdp_mmu(vcpu, cpu_role);
5066         else
5067                 init_kvm_softmmu(vcpu, cpu_role);
5068 }
5069 EXPORT_SYMBOL_GPL(kvm_init_mmu);
5070
5071 void kvm_mmu_after_set_cpuid(struct kvm_vcpu *vcpu)
5072 {
5073         /*
5074          * Invalidate all MMU roles to force them to reinitialize as CPUID
5075          * information is factored into reserved bit calculations.
5076          *
5077          * Correctly handling multiple vCPU models with respect to paging and
5078          * physical address properties) in a single VM would require tracking
5079          * all relevant CPUID information in kvm_mmu_page_role. That is very
5080          * undesirable as it would increase the memory requirements for
5081          * gfn_track (see struct kvm_mmu_page_role comments).  For now that
5082          * problem is swept under the rug; KVM's CPUID API is horrific and
5083          * it's all but impossible to solve it without introducing a new API.
5084          */
5085         vcpu->arch.root_mmu.root_role.word = 0;
5086         vcpu->arch.guest_mmu.root_role.word = 0;
5087         vcpu->arch.nested_mmu.root_role.word = 0;
5088         vcpu->arch.root_mmu.cpu_role.ext.valid = 0;
5089         vcpu->arch.guest_mmu.cpu_role.ext.valid = 0;
5090         vcpu->arch.nested_mmu.cpu_role.ext.valid = 0;
5091         kvm_mmu_reset_context(vcpu);
5092
5093         /*
5094          * Changing guest CPUID after KVM_RUN is forbidden, see the comment in
5095          * kvm_arch_vcpu_ioctl().
5096          */
5097         KVM_BUG_ON(vcpu->arch.last_vmentry_cpu != -1, vcpu->kvm);
5098 }
5099
5100 void kvm_mmu_reset_context(struct kvm_vcpu *vcpu)
5101 {
5102         kvm_mmu_unload(vcpu);
5103         kvm_init_mmu(vcpu);
5104 }
5105 EXPORT_SYMBOL_GPL(kvm_mmu_reset_context);
5106
5107 int kvm_mmu_load(struct kvm_vcpu *vcpu)
5108 {
5109         int r;
5110
5111         r = mmu_topup_memory_caches(vcpu, !vcpu->arch.mmu->root_role.direct);
5112         if (r)
5113                 goto out;
5114         r = mmu_alloc_special_roots(vcpu);
5115         if (r)
5116                 goto out;
5117         if (vcpu->arch.mmu->root_role.direct)
5118                 r = mmu_alloc_direct_roots(vcpu);
5119         else
5120                 r = mmu_alloc_shadow_roots(vcpu);
5121         if (r)
5122                 goto out;
5123
5124         kvm_mmu_sync_roots(vcpu);
5125
5126         kvm_mmu_load_pgd(vcpu);
5127
5128         /*
5129          * Flush any TLB entries for the new root, the provenance of the root
5130          * is unknown.  Even if KVM ensures there are no stale TLB entries
5131          * for a freed root, in theory another hypervisor could have left
5132          * stale entries.  Flushing on alloc also allows KVM to skip the TLB
5133          * flush when freeing a root (see kvm_tdp_mmu_put_root()).
5134          */
5135         static_call(kvm_x86_flush_tlb_current)(vcpu);
5136 out:
5137         return r;
5138 }
5139
5140 void kvm_mmu_unload(struct kvm_vcpu *vcpu)
5141 {
5142         struct kvm *kvm = vcpu->kvm;
5143
5144         kvm_mmu_free_roots(kvm, &vcpu->arch.root_mmu, KVM_MMU_ROOTS_ALL);
5145         WARN_ON(VALID_PAGE(vcpu->arch.root_mmu.root.hpa));
5146         kvm_mmu_free_roots(kvm, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL);
5147         WARN_ON(VALID_PAGE(vcpu->arch.guest_mmu.root.hpa));
5148         vcpu_clear_mmio_info(vcpu, MMIO_GVA_ANY);
5149 }
5150
5151 static bool is_obsolete_root(struct kvm *kvm, hpa_t root_hpa)
5152 {
5153         struct kvm_mmu_page *sp;
5154
5155         if (!VALID_PAGE(root_hpa))
5156                 return false;
5157
5158         /*
5159          * When freeing obsolete roots, treat roots as obsolete if they don't
5160          * have an associated shadow page.  This does mean KVM will get false
5161          * positives and free roots that don't strictly need to be freed, but
5162          * such false positives are relatively rare:
5163          *
5164          *  (a) only PAE paging and nested NPT has roots without shadow pages
5165          *  (b) remote reloads due to a memslot update obsoletes _all_ roots
5166          *  (c) KVM doesn't track previous roots for PAE paging, and the guest
5167          *      is unlikely to zap an in-use PGD.
5168          */
5169         sp = to_shadow_page(root_hpa);
5170         return !sp || is_obsolete_sp(kvm, sp);
5171 }
5172
5173 static void __kvm_mmu_free_obsolete_roots(struct kvm *kvm, struct kvm_mmu *mmu)
5174 {
5175         unsigned long roots_to_free = 0;
5176         int i;
5177
5178         if (is_obsolete_root(kvm, mmu->root.hpa))
5179                 roots_to_free |= KVM_MMU_ROOT_CURRENT;
5180
5181         for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) {
5182                 if (is_obsolete_root(kvm, mmu->prev_roots[i].hpa))
5183                         roots_to_free |= KVM_MMU_ROOT_PREVIOUS(i);
5184         }
5185
5186         if (roots_to_free)
5187                 kvm_mmu_free_roots(kvm, mmu, roots_to_free);
5188 }
5189
5190 void kvm_mmu_free_obsolete_roots(struct kvm_vcpu *vcpu)
5191 {
5192         __kvm_mmu_free_obsolete_roots(vcpu->kvm, &vcpu->arch.root_mmu);
5193         __kvm_mmu_free_obsolete_roots(vcpu->kvm, &vcpu->arch.guest_mmu);
5194 }
5195
5196 static bool need_remote_flush(u64 old, u64 new)
5197 {
5198         if (!is_shadow_present_pte(old))
5199                 return false;
5200         if (!is_shadow_present_pte(new))
5201                 return true;
5202         if ((old ^ new) & PT64_BASE_ADDR_MASK)
5203                 return true;
5204         old ^= shadow_nx_mask;
5205         new ^= shadow_nx_mask;
5206         return (old & ~new & PT64_PERM_MASK) != 0;
5207 }
5208
5209 static u64 mmu_pte_write_fetch_gpte(struct kvm_vcpu *vcpu, gpa_t *gpa,
5210                                     int *bytes)
5211 {
5212         u64 gentry = 0;
5213         int r;
5214
5215         /*
5216          * Assume that the pte write on a page table of the same type
5217          * as the current vcpu paging mode since we update the sptes only
5218          * when they have the same mode.
5219          */
5220         if (is_pae(vcpu) && *bytes == 4) {
5221                 /* Handle a 32-bit guest writing two halves of a 64-bit gpte */
5222                 *gpa &= ~(gpa_t)7;
5223                 *bytes = 8;
5224         }
5225
5226         if (*bytes == 4 || *bytes == 8) {
5227                 r = kvm_vcpu_read_guest_atomic(vcpu, *gpa, &gentry, *bytes);
5228                 if (r)
5229                         gentry = 0;
5230         }
5231
5232         return gentry;
5233 }
5234
5235 /*
5236  * If we're seeing too many writes to a page, it may no longer be a page table,
5237  * or we may be forking, in which case it is better to unmap the page.
5238  */
5239 static bool detect_write_flooding(struct kvm_mmu_page *sp)
5240 {
5241         /*
5242          * Skip write-flooding detected for the sp whose level is 1, because
5243          * it can become unsync, then the guest page is not write-protected.
5244          */
5245         if (sp->role.level == PG_LEVEL_4K)
5246                 return false;
5247
5248         atomic_inc(&sp->write_flooding_count);
5249         return atomic_read(&sp->write_flooding_count) >= 3;
5250 }
5251
5252 /*
5253  * Misaligned accesses are too much trouble to fix up; also, they usually
5254  * indicate a page is not used as a page table.
5255  */
5256 static bool detect_write_misaligned(struct kvm_mmu_page *sp, gpa_t gpa,
5257                                     int bytes)
5258 {
5259         unsigned offset, pte_size, misaligned;
5260
5261         pgprintk("misaligned: gpa %llx bytes %d role %x\n",
5262                  gpa, bytes, sp->role.word);
5263
5264         offset = offset_in_page(gpa);
5265         pte_size = sp->role.has_4_byte_gpte ? 4 : 8;
5266
5267         /*
5268          * Sometimes, the OS only writes the last one bytes to update status
5269          * bits, for example, in linux, andb instruction is used in clear_bit().
5270          */
5271         if (!(offset & (pte_size - 1)) && bytes == 1)
5272                 return false;
5273
5274         misaligned = (offset ^ (offset + bytes - 1)) & ~(pte_size - 1);
5275         misaligned |= bytes < 4;
5276
5277         return misaligned;
5278 }
5279
5280 static u64 *get_written_sptes(struct kvm_mmu_page *sp, gpa_t gpa, int *nspte)
5281 {
5282         unsigned page_offset, quadrant;
5283         u64 *spte;
5284         int level;
5285
5286         page_offset = offset_in_page(gpa);
5287         level = sp->role.level;
5288         *nspte = 1;
5289         if (sp->role.has_4_byte_gpte) {
5290                 page_offset <<= 1;      /* 32->64 */
5291                 /*
5292                  * A 32-bit pde maps 4MB while the shadow pdes map
5293                  * only 2MB.  So we need to double the offset again
5294                  * and zap two pdes instead of one.
5295                  */
5296                 if (level == PT32_ROOT_LEVEL) {
5297                         page_offset &= ~7; /* kill rounding error */
5298                         page_offset <<= 1;
5299                         *nspte = 2;
5300                 }
5301                 quadrant = page_offset >> PAGE_SHIFT;
5302                 page_offset &= ~PAGE_MASK;
5303                 if (quadrant != sp->role.quadrant)
5304                         return NULL;
5305         }
5306
5307         spte = &sp->spt[page_offset / sizeof(*spte)];
5308         return spte;
5309 }
5310
5311 static void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
5312                               const u8 *new, int bytes,
5313                               struct kvm_page_track_notifier_node *node)
5314 {
5315         gfn_t gfn = gpa >> PAGE_SHIFT;
5316         struct kvm_mmu_page *sp;
5317         LIST_HEAD(invalid_list);
5318         u64 entry, gentry, *spte;
5319         int npte;
5320         bool flush = false;
5321
5322         /*
5323          * If we don't have indirect shadow pages, it means no page is
5324          * write-protected, so we can exit simply.
5325          */
5326         if (!READ_ONCE(vcpu->kvm->arch.indirect_shadow_pages))
5327                 return;
5328
5329         pgprintk("%s: gpa %llx bytes %d\n", __func__, gpa, bytes);
5330
5331         /*
5332          * No need to care whether allocation memory is successful
5333          * or not since pte prefetch is skipped if it does not have
5334          * enough objects in the cache.
5335          */
5336         mmu_topup_memory_caches(vcpu, true);
5337
5338         write_lock(&vcpu->kvm->mmu_lock);
5339
5340         gentry = mmu_pte_write_fetch_gpte(vcpu, &gpa, &bytes);
5341
5342         ++vcpu->kvm->stat.mmu_pte_write;
5343
5344         for_each_gfn_valid_sp_with_gptes(vcpu->kvm, sp, gfn) {
5345                 if (detect_write_misaligned(sp, gpa, bytes) ||
5346                       detect_write_flooding(sp)) {
5347                         kvm_mmu_prepare_zap_page(vcpu->kvm, sp, &invalid_list);
5348                         ++vcpu->kvm->stat.mmu_flooded;
5349                         continue;
5350                 }
5351
5352                 spte = get_written_sptes(sp, gpa, &npte);
5353                 if (!spte)
5354                         continue;
5355
5356                 while (npte--) {
5357                         entry = *spte;
5358                         mmu_page_zap_pte(vcpu->kvm, sp, spte, NULL);
5359                         if (gentry && sp->role.level != PG_LEVEL_4K)
5360                                 ++vcpu->kvm->stat.mmu_pde_zapped;
5361                         if (need_remote_flush(entry, *spte))
5362                                 flush = true;
5363                         ++spte;
5364                 }
5365         }
5366         kvm_mmu_remote_flush_or_zap(vcpu->kvm, &invalid_list, flush);
5367         write_unlock(&vcpu->kvm->mmu_lock);
5368 }
5369
5370 int noinline kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 error_code,
5371                        void *insn, int insn_len)
5372 {
5373         int r, emulation_type = EMULTYPE_PF;
5374         bool direct = vcpu->arch.mmu->root_role.direct;
5375
5376         if (WARN_ON(!VALID_PAGE(vcpu->arch.mmu->root.hpa)))
5377                 return RET_PF_RETRY;
5378
5379         r = RET_PF_INVALID;
5380         if (unlikely(error_code & PFERR_RSVD_MASK)) {
5381                 r = handle_mmio_page_fault(vcpu, cr2_or_gpa, direct);
5382                 if (r == RET_PF_EMULATE)
5383                         goto emulate;
5384         }
5385
5386         if (r == RET_PF_INVALID) {
5387                 r = kvm_mmu_do_page_fault(vcpu, cr2_or_gpa,
5388                                           lower_32_bits(error_code), false);
5389                 if (KVM_BUG_ON(r == RET_PF_INVALID, vcpu->kvm))
5390                         return -EIO;
5391         }
5392
5393         if (r < 0)
5394                 return r;
5395         if (r != RET_PF_EMULATE)
5396                 return 1;
5397
5398         /*
5399          * Before emulating the instruction, check if the error code
5400          * was due to a RO violation while translating the guest page.
5401          * This can occur when using nested virtualization with nested
5402          * paging in both guests. If true, we simply unprotect the page
5403          * and resume the guest.
5404          */
5405         if (vcpu->arch.mmu->root_role.direct &&
5406             (error_code & PFERR_NESTED_GUEST_PAGE) == PFERR_NESTED_GUEST_PAGE) {
5407                 kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(cr2_or_gpa));
5408                 return 1;
5409         }
5410
5411         /*
5412          * vcpu->arch.mmu.page_fault returned RET_PF_EMULATE, but we can still
5413          * optimistically try to just unprotect the page and let the processor
5414          * re-execute the instruction that caused the page fault.  Do not allow
5415          * retrying MMIO emulation, as it's not only pointless but could also
5416          * cause us to enter an infinite loop because the processor will keep
5417          * faulting on the non-existent MMIO address.  Retrying an instruction
5418          * from a nested guest is also pointless and dangerous as we are only
5419          * explicitly shadowing L1's page tables, i.e. unprotecting something
5420          * for L1 isn't going to magically fix whatever issue cause L2 to fail.
5421          */
5422         if (!mmio_info_in_cache(vcpu, cr2_or_gpa, direct) && !is_guest_mode(vcpu))
5423                 emulation_type |= EMULTYPE_ALLOW_RETRY_PF;
5424 emulate:
5425         return x86_emulate_instruction(vcpu, cr2_or_gpa, emulation_type, insn,
5426                                        insn_len);
5427 }
5428 EXPORT_SYMBOL_GPL(kvm_mmu_page_fault);
5429
5430 void kvm_mmu_invalidate_gva(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
5431                             gva_t gva, hpa_t root_hpa)
5432 {
5433         int i;
5434
5435         /* It's actually a GPA for vcpu->arch.guest_mmu.  */
5436         if (mmu != &vcpu->arch.guest_mmu) {
5437                 /* INVLPG on a non-canonical address is a NOP according to the SDM.  */
5438                 if (is_noncanonical_address(gva, vcpu))
5439                         return;
5440
5441                 static_call(kvm_x86_flush_tlb_gva)(vcpu, gva);
5442         }
5443
5444         if (!mmu->invlpg)
5445                 return;
5446
5447         if (root_hpa == INVALID_PAGE) {
5448                 mmu->invlpg(vcpu, gva, mmu->root.hpa);
5449
5450                 /*
5451                  * INVLPG is required to invalidate any global mappings for the VA,
5452                  * irrespective of PCID. Since it would take us roughly similar amount
5453                  * of work to determine whether any of the prev_root mappings of the VA
5454                  * is marked global, or to just sync it blindly, so we might as well
5455                  * just always sync it.
5456                  *
5457                  * Mappings not reachable via the current cr3 or the prev_roots will be
5458                  * synced when switching to that cr3, so nothing needs to be done here
5459                  * for them.
5460                  */
5461                 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
5462                         if (VALID_PAGE(mmu->prev_roots[i].hpa))
5463                                 mmu->invlpg(vcpu, gva, mmu->prev_roots[i].hpa);
5464         } else {
5465                 mmu->invlpg(vcpu, gva, root_hpa);
5466         }
5467 }
5468
5469 void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
5470 {
5471         kvm_mmu_invalidate_gva(vcpu, vcpu->arch.walk_mmu, gva, INVALID_PAGE);
5472         ++vcpu->stat.invlpg;
5473 }
5474 EXPORT_SYMBOL_GPL(kvm_mmu_invlpg);
5475
5476
5477 void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid)
5478 {
5479         struct kvm_mmu *mmu = vcpu->arch.mmu;
5480         bool tlb_flush = false;
5481         uint i;
5482
5483         if (pcid == kvm_get_active_pcid(vcpu)) {
5484                 if (mmu->invlpg)
5485                         mmu->invlpg(vcpu, gva, mmu->root.hpa);
5486                 tlb_flush = true;
5487         }
5488
5489         for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) {
5490                 if (VALID_PAGE(mmu->prev_roots[i].hpa) &&
5491                     pcid == kvm_get_pcid(vcpu, mmu->prev_roots[i].pgd)) {
5492                         if (mmu->invlpg)
5493                                 mmu->invlpg(vcpu, gva, mmu->prev_roots[i].hpa);
5494                         tlb_flush = true;
5495                 }
5496         }
5497
5498         if (tlb_flush)
5499                 static_call(kvm_x86_flush_tlb_gva)(vcpu, gva);
5500
5501         ++vcpu->stat.invlpg;
5502
5503         /*
5504          * Mappings not reachable via the current cr3 or the prev_roots will be
5505          * synced when switching to that cr3, so nothing needs to be done here
5506          * for them.
5507          */
5508 }
5509
5510 void kvm_configure_mmu(bool enable_tdp, int tdp_forced_root_level,
5511                        int tdp_max_root_level, int tdp_huge_page_level)
5512 {
5513         tdp_enabled = enable_tdp;
5514         tdp_root_level = tdp_forced_root_level;
5515         max_tdp_level = tdp_max_root_level;
5516
5517         /*
5518          * max_huge_page_level reflects KVM's MMU capabilities irrespective
5519          * of kernel support, e.g. KVM may be capable of using 1GB pages when
5520          * the kernel is not.  But, KVM never creates a page size greater than
5521          * what is used by the kernel for any given HVA, i.e. the kernel's
5522          * capabilities are ultimately consulted by kvm_mmu_hugepage_adjust().
5523          */
5524         if (tdp_enabled)
5525                 max_huge_page_level = tdp_huge_page_level;
5526         else if (boot_cpu_has(X86_FEATURE_GBPAGES))
5527                 max_huge_page_level = PG_LEVEL_1G;
5528         else
5529                 max_huge_page_level = PG_LEVEL_2M;
5530 }
5531 EXPORT_SYMBOL_GPL(kvm_configure_mmu);
5532
5533 /* The return value indicates if tlb flush on all vcpus is needed. */
5534 typedef bool (*slot_level_handler) (struct kvm *kvm,
5535                                     struct kvm_rmap_head *rmap_head,
5536                                     const struct kvm_memory_slot *slot);
5537
5538 /* The caller should hold mmu-lock before calling this function. */
5539 static __always_inline bool
5540 slot_handle_level_range(struct kvm *kvm, const struct kvm_memory_slot *memslot,
5541                         slot_level_handler fn, int start_level, int end_level,
5542                         gfn_t start_gfn, gfn_t end_gfn, bool flush_on_yield,
5543                         bool flush)
5544 {
5545         struct slot_rmap_walk_iterator iterator;
5546
5547         for_each_slot_rmap_range(memslot, start_level, end_level, start_gfn,
5548                         end_gfn, &iterator) {
5549                 if (iterator.rmap)
5550                         flush |= fn(kvm, iterator.rmap, memslot);
5551
5552                 if (need_resched() || rwlock_needbreak(&kvm->mmu_lock)) {
5553                         if (flush && flush_on_yield) {
5554                                 kvm_flush_remote_tlbs_with_address(kvm,
5555                                                 start_gfn,
5556                                                 iterator.gfn - start_gfn + 1);
5557                                 flush = false;
5558                         }
5559                         cond_resched_rwlock_write(&kvm->mmu_lock);
5560                 }
5561         }
5562
5563         return flush;
5564 }
5565
5566 static __always_inline bool
5567 slot_handle_level(struct kvm *kvm, const struct kvm_memory_slot *memslot,
5568                   slot_level_handler fn, int start_level, int end_level,
5569                   bool flush_on_yield)
5570 {
5571         return slot_handle_level_range(kvm, memslot, fn, start_level,
5572                         end_level, memslot->base_gfn,
5573                         memslot->base_gfn + memslot->npages - 1,
5574                         flush_on_yield, false);
5575 }
5576
5577 static __always_inline bool
5578 slot_handle_level_4k(struct kvm *kvm, const struct kvm_memory_slot *memslot,
5579                      slot_level_handler fn, bool flush_on_yield)
5580 {
5581         return slot_handle_level(kvm, memslot, fn, PG_LEVEL_4K,
5582                                  PG_LEVEL_4K, flush_on_yield);
5583 }
5584
5585 static void free_mmu_pages(struct kvm_mmu *mmu)
5586 {
5587         if (!tdp_enabled && mmu->pae_root)
5588                 set_memory_encrypted((unsigned long)mmu->pae_root, 1);
5589         free_page((unsigned long)mmu->pae_root);
5590         free_page((unsigned long)mmu->pml4_root);
5591         free_page((unsigned long)mmu->pml5_root);
5592 }
5593
5594 static int __kvm_mmu_create(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu)
5595 {
5596         struct page *page;
5597         int i;
5598
5599         mmu->root.hpa = INVALID_PAGE;
5600         mmu->root.pgd = 0;
5601         for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
5602                 mmu->prev_roots[i] = KVM_MMU_ROOT_INFO_INVALID;
5603
5604         /* vcpu->arch.guest_mmu isn't used when !tdp_enabled. */
5605         if (!tdp_enabled && mmu == &vcpu->arch.guest_mmu)
5606                 return 0;
5607
5608         /*
5609          * When using PAE paging, the four PDPTEs are treated as 'root' pages,
5610          * while the PDP table is a per-vCPU construct that's allocated at MMU
5611          * creation.  When emulating 32-bit mode, cr3 is only 32 bits even on
5612          * x86_64.  Therefore we need to allocate the PDP table in the first
5613          * 4GB of memory, which happens to fit the DMA32 zone.  TDP paging
5614          * generally doesn't use PAE paging and can skip allocating the PDP
5615          * table.  The main exception, handled here, is SVM's 32-bit NPT.  The
5616          * other exception is for shadowing L1's 32-bit or PAE NPT on 64-bit
5617          * KVM; that horror is handled on-demand by mmu_alloc_special_roots().
5618          */
5619         if (tdp_enabled && kvm_mmu_get_tdp_level(vcpu) > PT32E_ROOT_LEVEL)
5620                 return 0;
5621
5622         page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_DMA32);
5623         if (!page)
5624                 return -ENOMEM;
5625
5626         mmu->pae_root = page_address(page);
5627
5628         /*
5629          * CR3 is only 32 bits when PAE paging is used, thus it's impossible to
5630          * get the CPU to treat the PDPTEs as encrypted.  Decrypt the page so
5631          * that KVM's writes and the CPU's reads get along.  Note, this is
5632          * only necessary when using shadow paging, as 64-bit NPT can get at
5633          * the C-bit even when shadowing 32-bit NPT, and SME isn't supported
5634          * by 32-bit kernels (when KVM itself uses 32-bit NPT).
5635          */
5636         if (!tdp_enabled)
5637                 set_memory_decrypted((unsigned long)mmu->pae_root, 1);
5638         else
5639                 WARN_ON_ONCE(shadow_me_value);
5640
5641         for (i = 0; i < 4; ++i)
5642                 mmu->pae_root[i] = INVALID_PAE_ROOT;
5643
5644         return 0;
5645 }
5646
5647 int kvm_mmu_create(struct kvm_vcpu *vcpu)
5648 {
5649         int ret;
5650
5651         vcpu->arch.mmu_pte_list_desc_cache.kmem_cache = pte_list_desc_cache;
5652         vcpu->arch.mmu_pte_list_desc_cache.gfp_zero = __GFP_ZERO;
5653
5654         vcpu->arch.mmu_page_header_cache.kmem_cache = mmu_page_header_cache;
5655         vcpu->arch.mmu_page_header_cache.gfp_zero = __GFP_ZERO;
5656
5657         vcpu->arch.mmu_shadow_page_cache.gfp_zero = __GFP_ZERO;
5658
5659         vcpu->arch.mmu = &vcpu->arch.root_mmu;
5660         vcpu->arch.walk_mmu = &vcpu->arch.root_mmu;
5661
5662         ret = __kvm_mmu_create(vcpu, &vcpu->arch.guest_mmu);
5663         if (ret)
5664                 return ret;
5665
5666         ret = __kvm_mmu_create(vcpu, &vcpu->arch.root_mmu);
5667         if (ret)
5668                 goto fail_allocate_root;
5669
5670         return ret;
5671  fail_allocate_root:
5672         free_mmu_pages(&vcpu->arch.guest_mmu);
5673         return ret;
5674 }
5675
5676 #define BATCH_ZAP_PAGES 10
5677 static void kvm_zap_obsolete_pages(struct kvm *kvm)
5678 {
5679         struct kvm_mmu_page *sp, *node;
5680         int nr_zapped, batch = 0;
5681         bool unstable;
5682
5683 restart:
5684         list_for_each_entry_safe_reverse(sp, node,
5685               &kvm->arch.active_mmu_pages, link) {
5686                 /*
5687                  * No obsolete valid page exists before a newly created page
5688                  * since active_mmu_pages is a FIFO list.
5689                  */
5690                 if (!is_obsolete_sp(kvm, sp))
5691                         break;
5692
5693                 /*
5694                  * Invalid pages should never land back on the list of active
5695                  * pages.  Skip the bogus page, otherwise we'll get stuck in an
5696                  * infinite loop if the page gets put back on the list (again).
5697                  */
5698                 if (WARN_ON(sp->role.invalid))
5699                         continue;
5700
5701                 /*
5702                  * No need to flush the TLB since we're only zapping shadow
5703                  * pages with an obsolete generation number and all vCPUS have
5704                  * loaded a new root, i.e. the shadow pages being zapped cannot
5705                  * be in active use by the guest.
5706                  */
5707                 if (batch >= BATCH_ZAP_PAGES &&
5708                     cond_resched_rwlock_write(&kvm->mmu_lock)) {
5709                         batch = 0;
5710                         goto restart;
5711                 }
5712
5713                 unstable = __kvm_mmu_prepare_zap_page(kvm, sp,
5714                                 &kvm->arch.zapped_obsolete_pages, &nr_zapped);
5715                 batch += nr_zapped;
5716
5717                 if (unstable)
5718                         goto restart;
5719         }
5720
5721         /*
5722          * Kick all vCPUs (via remote TLB flush) before freeing the page tables
5723          * to ensure KVM is not in the middle of a lockless shadow page table
5724          * walk, which may reference the pages.  The remote TLB flush itself is
5725          * not required and is simply a convenient way to kick vCPUs as needed.
5726          * KVM performs a local TLB flush when allocating a new root (see
5727          * kvm_mmu_load()), and the reload in the caller ensure no vCPUs are
5728          * running with an obsolete MMU.
5729          */
5730         kvm_mmu_commit_zap_page(kvm, &kvm->arch.zapped_obsolete_pages);
5731 }
5732
5733 /*
5734  * Fast invalidate all shadow pages and use lock-break technique
5735  * to zap obsolete pages.
5736  *
5737  * It's required when memslot is being deleted or VM is being
5738  * destroyed, in these cases, we should ensure that KVM MMU does
5739  * not use any resource of the being-deleted slot or all slots
5740  * after calling the function.
5741  */
5742 static void kvm_mmu_zap_all_fast(struct kvm *kvm)
5743 {
5744         lockdep_assert_held(&kvm->slots_lock);
5745
5746         write_lock(&kvm->mmu_lock);
5747         trace_kvm_mmu_zap_all_fast(kvm);
5748
5749         /*
5750          * Toggle mmu_valid_gen between '0' and '1'.  Because slots_lock is
5751          * held for the entire duration of zapping obsolete pages, it's
5752          * impossible for there to be multiple invalid generations associated
5753          * with *valid* shadow pages at any given time, i.e. there is exactly
5754          * one valid generation and (at most) one invalid generation.
5755          */
5756         kvm->arch.mmu_valid_gen = kvm->arch.mmu_valid_gen ? 0 : 1;
5757
5758         /*
5759          * In order to ensure all vCPUs drop their soon-to-be invalid roots,
5760          * invalidating TDP MMU roots must be done while holding mmu_lock for
5761          * write and in the same critical section as making the reload request,
5762          * e.g. before kvm_zap_obsolete_pages() could drop mmu_lock and yield.
5763          */
5764         if (is_tdp_mmu_enabled(kvm))
5765                 kvm_tdp_mmu_invalidate_all_roots(kvm);
5766
5767         /*
5768          * Notify all vcpus to reload its shadow page table and flush TLB.
5769          * Then all vcpus will switch to new shadow page table with the new
5770          * mmu_valid_gen.
5771          *
5772          * Note: we need to do this under the protection of mmu_lock,
5773          * otherwise, vcpu would purge shadow page but miss tlb flush.
5774          */
5775         kvm_make_all_cpus_request(kvm, KVM_REQ_MMU_FREE_OBSOLETE_ROOTS);
5776
5777         kvm_zap_obsolete_pages(kvm);
5778
5779         write_unlock(&kvm->mmu_lock);
5780
5781         /*
5782          * Zap the invalidated TDP MMU roots, all SPTEs must be dropped before
5783          * returning to the caller, e.g. if the zap is in response to a memslot
5784          * deletion, mmu_notifier callbacks will be unable to reach the SPTEs
5785          * associated with the deleted memslot once the update completes, and
5786          * Deferring the zap until the final reference to the root is put would
5787          * lead to use-after-free.
5788          */
5789         if (is_tdp_mmu_enabled(kvm))
5790                 kvm_tdp_mmu_zap_invalidated_roots(kvm);
5791 }
5792
5793 static bool kvm_has_zapped_obsolete_pages(struct kvm *kvm)
5794 {
5795         return unlikely(!list_empty_careful(&kvm->arch.zapped_obsolete_pages));
5796 }
5797
5798 static void kvm_mmu_invalidate_zap_pages_in_memslot(struct kvm *kvm,
5799                         struct kvm_memory_slot *slot,
5800                         struct kvm_page_track_notifier_node *node)
5801 {
5802         kvm_mmu_zap_all_fast(kvm);
5803 }
5804
5805 int kvm_mmu_init_vm(struct kvm *kvm)
5806 {
5807         struct kvm_page_track_notifier_node *node = &kvm->arch.mmu_sp_tracker;
5808         int r;
5809
5810         INIT_LIST_HEAD(&kvm->arch.active_mmu_pages);
5811         INIT_LIST_HEAD(&kvm->arch.zapped_obsolete_pages);
5812         INIT_LIST_HEAD(&kvm->arch.lpage_disallowed_mmu_pages);
5813         spin_lock_init(&kvm->arch.mmu_unsync_pages_lock);
5814
5815         r = kvm_mmu_init_tdp_mmu(kvm);
5816         if (r < 0)
5817                 return r;
5818
5819         node->track_write = kvm_mmu_pte_write;
5820         node->track_flush_slot = kvm_mmu_invalidate_zap_pages_in_memslot;
5821         kvm_page_track_register_notifier(kvm, node);
5822         return 0;
5823 }
5824
5825 void kvm_mmu_uninit_vm(struct kvm *kvm)
5826 {
5827         struct kvm_page_track_notifier_node *node = &kvm->arch.mmu_sp_tracker;
5828
5829         kvm_page_track_unregister_notifier(kvm, node);
5830
5831         kvm_mmu_uninit_tdp_mmu(kvm);
5832 }
5833
5834 static bool __kvm_zap_rmaps(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end)
5835 {
5836         const struct kvm_memory_slot *memslot;
5837         struct kvm_memslots *slots;
5838         struct kvm_memslot_iter iter;
5839         bool flush = false;
5840         gfn_t start, end;
5841         int i;
5842
5843         if (!kvm_memslots_have_rmaps(kvm))
5844                 return flush;
5845
5846         for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
5847                 slots = __kvm_memslots(kvm, i);
5848
5849                 kvm_for_each_memslot_in_gfn_range(&iter, slots, gfn_start, gfn_end) {
5850                         memslot = iter.slot;
5851                         start = max(gfn_start, memslot->base_gfn);
5852                         end = min(gfn_end, memslot->base_gfn + memslot->npages);
5853                         if (WARN_ON_ONCE(start >= end))
5854                                 continue;
5855
5856                         flush = slot_handle_level_range(kvm, memslot, kvm_zap_rmapp,
5857
5858                                                         PG_LEVEL_4K, KVM_MAX_HUGEPAGE_LEVEL,
5859                                                         start, end - 1, true, flush);
5860                 }
5861         }
5862
5863         return flush;
5864 }
5865
5866 /*
5867  * Invalidate (zap) SPTEs that cover GFNs from gfn_start and up to gfn_end
5868  * (not including it)
5869  */
5870 void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end)
5871 {
5872         bool flush;
5873         int i;
5874
5875         if (WARN_ON_ONCE(gfn_end <= gfn_start))
5876                 return;
5877
5878         write_lock(&kvm->mmu_lock);
5879
5880         kvm_inc_notifier_count(kvm, gfn_start, gfn_end);
5881
5882         flush = __kvm_zap_rmaps(kvm, gfn_start, gfn_end);
5883
5884         if (is_tdp_mmu_enabled(kvm)) {
5885                 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++)
5886                         flush = kvm_tdp_mmu_zap_leafs(kvm, i, gfn_start,
5887                                                       gfn_end, true, flush);
5888         }
5889
5890         if (flush)
5891                 kvm_flush_remote_tlbs_with_address(kvm, gfn_start,
5892                                                    gfn_end - gfn_start);
5893
5894         kvm_dec_notifier_count(kvm, gfn_start, gfn_end);
5895
5896         write_unlock(&kvm->mmu_lock);
5897 }
5898
5899 static bool slot_rmap_write_protect(struct kvm *kvm,
5900                                     struct kvm_rmap_head *rmap_head,
5901                                     const struct kvm_memory_slot *slot)
5902 {
5903         return rmap_write_protect(rmap_head, false);
5904 }
5905
5906 void kvm_mmu_slot_remove_write_access(struct kvm *kvm,
5907                                       const struct kvm_memory_slot *memslot,
5908                                       int start_level)
5909 {
5910         bool flush = false;
5911
5912         if (kvm_memslots_have_rmaps(kvm)) {
5913                 write_lock(&kvm->mmu_lock);
5914                 flush = slot_handle_level(kvm, memslot, slot_rmap_write_protect,
5915                                           start_level, KVM_MAX_HUGEPAGE_LEVEL,
5916                                           false);
5917                 write_unlock(&kvm->mmu_lock);
5918         }
5919
5920         if (is_tdp_mmu_enabled(kvm)) {
5921                 read_lock(&kvm->mmu_lock);
5922                 flush |= kvm_tdp_mmu_wrprot_slot(kvm, memslot, start_level);
5923                 read_unlock(&kvm->mmu_lock);
5924         }
5925
5926         /*
5927          * Flush TLBs if any SPTEs had to be write-protected to ensure that
5928          * guest writes are reflected in the dirty bitmap before the memslot
5929          * update completes, i.e. before enabling dirty logging is visible to
5930          * userspace.
5931          *
5932          * Perform the TLB flush outside the mmu_lock to reduce the amount of
5933          * time the lock is held. However, this does mean that another CPU can
5934          * now grab mmu_lock and encounter a write-protected SPTE while CPUs
5935          * still have a writable mapping for the associated GFN in their TLB.
5936          *
5937          * This is safe but requires KVM to be careful when making decisions
5938          * based on the write-protection status of an SPTE. Specifically, KVM
5939          * also write-protects SPTEs to monitor changes to guest page tables
5940          * during shadow paging, and must guarantee no CPUs can write to those
5941          * page before the lock is dropped. As mentioned in the previous
5942          * paragraph, a write-protected SPTE is no guarantee that CPU cannot
5943          * perform writes. So to determine if a TLB flush is truly required, KVM
5944          * will clear a separate software-only bit (MMU-writable) and skip the
5945          * flush if-and-only-if this bit was already clear.
5946          *
5947          * See is_writable_pte() for more details.
5948          */
5949         if (flush)
5950                 kvm_arch_flush_remote_tlbs_memslot(kvm, memslot);
5951 }
5952
5953 /* Must be called with the mmu_lock held in write-mode. */
5954 void kvm_mmu_try_split_huge_pages(struct kvm *kvm,
5955                                    const struct kvm_memory_slot *memslot,
5956                                    u64 start, u64 end,
5957                                    int target_level)
5958 {
5959         if (is_tdp_mmu_enabled(kvm))
5960                 kvm_tdp_mmu_try_split_huge_pages(kvm, memslot, start, end,
5961                                                  target_level, false);
5962
5963         /*
5964          * A TLB flush is unnecessary at this point for the same resons as in
5965          * kvm_mmu_slot_try_split_huge_pages().
5966          */
5967 }
5968
5969 void kvm_mmu_slot_try_split_huge_pages(struct kvm *kvm,
5970                                         const struct kvm_memory_slot *memslot,
5971                                         int target_level)
5972 {
5973         u64 start = memslot->base_gfn;
5974         u64 end = start + memslot->npages;
5975
5976         if (is_tdp_mmu_enabled(kvm)) {
5977                 read_lock(&kvm->mmu_lock);
5978                 kvm_tdp_mmu_try_split_huge_pages(kvm, memslot, start, end, target_level, true);
5979                 read_unlock(&kvm->mmu_lock);
5980         }
5981
5982         /*
5983          * No TLB flush is necessary here. KVM will flush TLBs after
5984          * write-protecting and/or clearing dirty on the newly split SPTEs to
5985          * ensure that guest writes are reflected in the dirty log before the
5986          * ioctl to enable dirty logging on this memslot completes. Since the
5987          * split SPTEs retain the write and dirty bits of the huge SPTE, it is
5988          * safe for KVM to decide if a TLB flush is necessary based on the split
5989          * SPTEs.
5990          */
5991 }
5992
5993 static bool kvm_mmu_zap_collapsible_spte(struct kvm *kvm,
5994                                          struct kvm_rmap_head *rmap_head,
5995                                          const struct kvm_memory_slot *slot)
5996 {
5997         u64 *sptep;
5998         struct rmap_iterator iter;
5999         int need_tlb_flush = 0;
6000         kvm_pfn_t pfn;
6001         struct kvm_mmu_page *sp;
6002
6003 restart:
6004         for_each_rmap_spte(rmap_head, &iter, sptep) {
6005                 sp = sptep_to_sp(sptep);
6006                 pfn = spte_to_pfn(*sptep);
6007
6008                 /*
6009                  * We cannot do huge page mapping for indirect shadow pages,
6010                  * which are found on the last rmap (level = 1) when not using
6011                  * tdp; such shadow pages are synced with the page table in
6012                  * the guest, and the guest page table is using 4K page size
6013                  * mapping if the indirect sp has level = 1.
6014                  */
6015                 if (sp->role.direct && !kvm_is_reserved_pfn(pfn) &&
6016                     sp->role.level < kvm_mmu_max_mapping_level(kvm, slot, sp->gfn,
6017                                                                pfn, PG_LEVEL_NUM)) {
6018                         pte_list_remove(kvm, rmap_head, sptep);
6019
6020                         if (kvm_available_flush_tlb_with_range())
6021                                 kvm_flush_remote_tlbs_with_address(kvm, sp->gfn,
6022                                         KVM_PAGES_PER_HPAGE(sp->role.level));
6023                         else
6024                                 need_tlb_flush = 1;
6025
6026                         goto restart;
6027                 }
6028         }
6029
6030         return need_tlb_flush;
6031 }
6032
6033 void kvm_mmu_zap_collapsible_sptes(struct kvm *kvm,
6034                                    const struct kvm_memory_slot *slot)
6035 {
6036         if (kvm_memslots_have_rmaps(kvm)) {
6037                 write_lock(&kvm->mmu_lock);
6038                 /*
6039                  * Zap only 4k SPTEs since the legacy MMU only supports dirty
6040                  * logging at a 4k granularity and never creates collapsible
6041                  * 2m SPTEs during dirty logging.
6042                  */
6043                 if (slot_handle_level_4k(kvm, slot, kvm_mmu_zap_collapsible_spte, true))
6044                         kvm_arch_flush_remote_tlbs_memslot(kvm, slot);
6045                 write_unlock(&kvm->mmu_lock);
6046         }
6047
6048         if (is_tdp_mmu_enabled(kvm)) {
6049                 read_lock(&kvm->mmu_lock);
6050                 kvm_tdp_mmu_zap_collapsible_sptes(kvm, slot);
6051                 read_unlock(&kvm->mmu_lock);
6052         }
6053 }
6054
6055 void kvm_arch_flush_remote_tlbs_memslot(struct kvm *kvm,
6056                                         const struct kvm_memory_slot *memslot)
6057 {
6058         /*
6059          * All current use cases for flushing the TLBs for a specific memslot
6060          * related to dirty logging, and many do the TLB flush out of mmu_lock.
6061          * The interaction between the various operations on memslot must be
6062          * serialized by slots_locks to ensure the TLB flush from one operation
6063          * is observed by any other operation on the same memslot.
6064          */
6065         lockdep_assert_held(&kvm->slots_lock);
6066         kvm_flush_remote_tlbs_with_address(kvm, memslot->base_gfn,
6067                                            memslot->npages);
6068 }
6069
6070 void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm,
6071                                    const struct kvm_memory_slot *memslot)
6072 {
6073         bool flush = false;
6074
6075         if (kvm_memslots_have_rmaps(kvm)) {
6076                 write_lock(&kvm->mmu_lock);
6077                 /*
6078                  * Clear dirty bits only on 4k SPTEs since the legacy MMU only
6079                  * support dirty logging at a 4k granularity.
6080                  */
6081                 flush = slot_handle_level_4k(kvm, memslot, __rmap_clear_dirty, false);
6082                 write_unlock(&kvm->mmu_lock);
6083         }
6084
6085         if (is_tdp_mmu_enabled(kvm)) {
6086                 read_lock(&kvm->mmu_lock);
6087                 flush |= kvm_tdp_mmu_clear_dirty_slot(kvm, memslot);
6088                 read_unlock(&kvm->mmu_lock);
6089         }
6090
6091         /*
6092          * It's also safe to flush TLBs out of mmu lock here as currently this
6093          * function is only used for dirty logging, in which case flushing TLB
6094          * out of mmu lock also guarantees no dirty pages will be lost in
6095          * dirty_bitmap.
6096          */
6097         if (flush)
6098                 kvm_arch_flush_remote_tlbs_memslot(kvm, memslot);
6099 }
6100
6101 void kvm_mmu_zap_all(struct kvm *kvm)
6102 {
6103         struct kvm_mmu_page *sp, *node;
6104         LIST_HEAD(invalid_list);
6105         int ign;
6106
6107         write_lock(&kvm->mmu_lock);
6108 restart:
6109         list_for_each_entry_safe(sp, node, &kvm->arch.active_mmu_pages, link) {
6110                 if (WARN_ON(sp->role.invalid))
6111                         continue;
6112                 if (__kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list, &ign))
6113                         goto restart;
6114                 if (cond_resched_rwlock_write(&kvm->mmu_lock))
6115                         goto restart;
6116         }
6117
6118         kvm_mmu_commit_zap_page(kvm, &invalid_list);
6119
6120         if (is_tdp_mmu_enabled(kvm))
6121                 kvm_tdp_mmu_zap_all(kvm);
6122
6123         write_unlock(&kvm->mmu_lock);
6124 }
6125
6126 void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, u64 gen)
6127 {
6128         WARN_ON(gen & KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS);
6129
6130         gen &= MMIO_SPTE_GEN_MASK;
6131
6132         /*
6133          * Generation numbers are incremented in multiples of the number of
6134          * address spaces in order to provide unique generations across all
6135          * address spaces.  Strip what is effectively the address space
6136          * modifier prior to checking for a wrap of the MMIO generation so
6137          * that a wrap in any address space is detected.
6138          */
6139         gen &= ~((u64)KVM_ADDRESS_SPACE_NUM - 1);
6140
6141         /*
6142          * The very rare case: if the MMIO generation number has wrapped,
6143          * zap all shadow pages.
6144          */
6145         if (unlikely(gen == 0)) {
6146                 kvm_debug_ratelimited("kvm: zapping shadow pages for mmio generation wraparound\n");
6147                 kvm_mmu_zap_all_fast(kvm);
6148         }
6149 }
6150
6151 static unsigned long
6152 mmu_shrink_scan(struct shrinker *shrink, struct shrink_control *sc)
6153 {
6154         struct kvm *kvm;
6155         int nr_to_scan = sc->nr_to_scan;
6156         unsigned long freed = 0;
6157
6158         mutex_lock(&kvm_lock);
6159
6160         list_for_each_entry(kvm, &vm_list, vm_list) {
6161                 int idx;
6162                 LIST_HEAD(invalid_list);
6163
6164                 /*
6165                  * Never scan more than sc->nr_to_scan VM instances.
6166                  * Will not hit this condition practically since we do not try
6167                  * to shrink more than one VM and it is very unlikely to see
6168                  * !n_used_mmu_pages so many times.
6169                  */
6170                 if (!nr_to_scan--)
6171                         break;
6172                 /*
6173                  * n_used_mmu_pages is accessed without holding kvm->mmu_lock
6174                  * here. We may skip a VM instance errorneosly, but we do not
6175                  * want to shrink a VM that only started to populate its MMU
6176                  * anyway.
6177                  */
6178                 if (!kvm->arch.n_used_mmu_pages &&
6179                     !kvm_has_zapped_obsolete_pages(kvm))
6180                         continue;
6181
6182                 idx = srcu_read_lock(&kvm->srcu);
6183                 write_lock(&kvm->mmu_lock);
6184
6185                 if (kvm_has_zapped_obsolete_pages(kvm)) {
6186                         kvm_mmu_commit_zap_page(kvm,
6187                               &kvm->arch.zapped_obsolete_pages);
6188                         goto unlock;
6189                 }
6190
6191                 freed = kvm_mmu_zap_oldest_mmu_pages(kvm, sc->nr_to_scan);
6192
6193 unlock:
6194                 write_unlock(&kvm->mmu_lock);
6195                 srcu_read_unlock(&kvm->srcu, idx);
6196
6197                 /*
6198                  * unfair on small ones
6199                  * per-vm shrinkers cry out
6200                  * sadness comes quickly
6201                  */
6202                 list_move_tail(&kvm->vm_list, &vm_list);
6203                 break;
6204         }
6205
6206         mutex_unlock(&kvm_lock);
6207         return freed;
6208 }
6209
6210 static unsigned long
6211 mmu_shrink_count(struct shrinker *shrink, struct shrink_control *sc)
6212 {
6213         return percpu_counter_read_positive(&kvm_total_used_mmu_pages);
6214 }
6215
6216 static struct shrinker mmu_shrinker = {
6217         .count_objects = mmu_shrink_count,
6218         .scan_objects = mmu_shrink_scan,
6219         .seeks = DEFAULT_SEEKS * 10,
6220 };
6221
6222 static void mmu_destroy_caches(void)
6223 {
6224         kmem_cache_destroy(pte_list_desc_cache);
6225         kmem_cache_destroy(mmu_page_header_cache);
6226 }
6227
6228 static bool get_nx_auto_mode(void)
6229 {
6230         /* Return true when CPU has the bug, and mitigations are ON */
6231         return boot_cpu_has_bug(X86_BUG_ITLB_MULTIHIT) && !cpu_mitigations_off();
6232 }
6233
6234 static void __set_nx_huge_pages(bool val)
6235 {
6236         nx_huge_pages = itlb_multihit_kvm_mitigation = val;
6237 }
6238
6239 static int set_nx_huge_pages(const char *val, const struct kernel_param *kp)
6240 {
6241         bool old_val = nx_huge_pages;
6242         bool new_val;
6243
6244         /* In "auto" mode deploy workaround only if CPU has the bug. */
6245         if (sysfs_streq(val, "off"))
6246                 new_val = 0;
6247         else if (sysfs_streq(val, "force"))
6248                 new_val = 1;
6249         else if (sysfs_streq(val, "auto"))
6250                 new_val = get_nx_auto_mode();
6251         else if (strtobool(val, &new_val) < 0)
6252                 return -EINVAL;
6253
6254         __set_nx_huge_pages(new_val);
6255
6256         if (new_val != old_val) {
6257                 struct kvm *kvm;
6258
6259                 mutex_lock(&kvm_lock);
6260
6261                 list_for_each_entry(kvm, &vm_list, vm_list) {
6262                         mutex_lock(&kvm->slots_lock);
6263                         kvm_mmu_zap_all_fast(kvm);
6264                         mutex_unlock(&kvm->slots_lock);
6265
6266                         wake_up_process(kvm->arch.nx_lpage_recovery_thread);
6267                 }
6268                 mutex_unlock(&kvm_lock);
6269         }
6270
6271         return 0;
6272 }
6273
6274 /*
6275  * nx_huge_pages needs to be resolved to true/false when kvm.ko is loaded, as
6276  * its default value of -1 is technically undefined behavior for a boolean.
6277  */
6278 void kvm_mmu_x86_module_init(void)
6279 {
6280         if (nx_huge_pages == -1)
6281                 __set_nx_huge_pages(get_nx_auto_mode());
6282 }
6283
6284 /*
6285  * The bulk of the MMU initialization is deferred until the vendor module is
6286  * loaded as many of the masks/values may be modified by VMX or SVM, i.e. need
6287  * to be reset when a potentially different vendor module is loaded.
6288  */
6289 int kvm_mmu_vendor_module_init(void)
6290 {
6291         int ret = -ENOMEM;
6292
6293         /*
6294          * MMU roles use union aliasing which is, generally speaking, an
6295          * undefined behavior. However, we supposedly know how compilers behave
6296          * and the current status quo is unlikely to change. Guardians below are
6297          * supposed to let us know if the assumption becomes false.
6298          */
6299         BUILD_BUG_ON(sizeof(union kvm_mmu_page_role) != sizeof(u32));
6300         BUILD_BUG_ON(sizeof(union kvm_mmu_extended_role) != sizeof(u32));
6301         BUILD_BUG_ON(sizeof(union kvm_cpu_role) != sizeof(u64));
6302
6303         kvm_mmu_reset_all_pte_masks();
6304
6305         pte_list_desc_cache = kmem_cache_create("pte_list_desc",
6306                                             sizeof(struct pte_list_desc),
6307                                             0, SLAB_ACCOUNT, NULL);
6308         if (!pte_list_desc_cache)
6309                 goto out;
6310
6311         mmu_page_header_cache = kmem_cache_create("kvm_mmu_page_header",
6312                                                   sizeof(struct kvm_mmu_page),
6313                                                   0, SLAB_ACCOUNT, NULL);
6314         if (!mmu_page_header_cache)
6315                 goto out;
6316
6317         if (percpu_counter_init(&kvm_total_used_mmu_pages, 0, GFP_KERNEL))
6318                 goto out;
6319
6320         ret = register_shrinker(&mmu_shrinker, "x86-mmu");
6321         if (ret)
6322                 goto out;
6323
6324         return 0;
6325
6326 out:
6327         mmu_destroy_caches();
6328         return ret;
6329 }
6330
6331 void kvm_mmu_destroy(struct kvm_vcpu *vcpu)
6332 {
6333         kvm_mmu_unload(vcpu);
6334         free_mmu_pages(&vcpu->arch.root_mmu);
6335         free_mmu_pages(&vcpu->arch.guest_mmu);
6336         mmu_free_memory_caches(vcpu);
6337 }
6338
6339 void kvm_mmu_vendor_module_exit(void)
6340 {
6341         mmu_destroy_caches();
6342         percpu_counter_destroy(&kvm_total_used_mmu_pages);
6343         unregister_shrinker(&mmu_shrinker);
6344 }
6345
6346 /*
6347  * Calculate the effective recovery period, accounting for '0' meaning "let KVM
6348  * select a halving time of 1 hour".  Returns true if recovery is enabled.
6349  */
6350 static bool calc_nx_huge_pages_recovery_period(uint *period)
6351 {
6352         /*
6353          * Use READ_ONCE to get the params, this may be called outside of the
6354          * param setters, e.g. by the kthread to compute its next timeout.
6355          */
6356         bool enabled = READ_ONCE(nx_huge_pages);
6357         uint ratio = READ_ONCE(nx_huge_pages_recovery_ratio);
6358
6359         if (!enabled || !ratio)
6360                 return false;
6361
6362         *period = READ_ONCE(nx_huge_pages_recovery_period_ms);
6363         if (!*period) {
6364                 /* Make sure the period is not less than one second.  */
6365                 ratio = min(ratio, 3600u);
6366                 *period = 60 * 60 * 1000 / ratio;
6367         }
6368         return true;
6369 }
6370
6371 static int set_nx_huge_pages_recovery_param(const char *val, const struct kernel_param *kp)
6372 {
6373         bool was_recovery_enabled, is_recovery_enabled;
6374         uint old_period, new_period;
6375         int err;
6376
6377         was_recovery_enabled = calc_nx_huge_pages_recovery_period(&old_period);
6378
6379         err = param_set_uint(val, kp);
6380         if (err)
6381                 return err;
6382
6383         is_recovery_enabled = calc_nx_huge_pages_recovery_period(&new_period);
6384
6385         if (is_recovery_enabled &&
6386             (!was_recovery_enabled || old_period > new_period)) {
6387                 struct kvm *kvm;
6388
6389                 mutex_lock(&kvm_lock);
6390
6391                 list_for_each_entry(kvm, &vm_list, vm_list)
6392                         wake_up_process(kvm->arch.nx_lpage_recovery_thread);
6393
6394                 mutex_unlock(&kvm_lock);
6395         }
6396
6397         return err;
6398 }
6399
6400 static void kvm_recover_nx_lpages(struct kvm *kvm)
6401 {
6402         unsigned long nx_lpage_splits = kvm->stat.nx_lpage_splits;
6403         int rcu_idx;
6404         struct kvm_mmu_page *sp;
6405         unsigned int ratio;
6406         LIST_HEAD(invalid_list);
6407         bool flush = false;
6408         ulong to_zap;
6409
6410         rcu_idx = srcu_read_lock(&kvm->srcu);
6411         write_lock(&kvm->mmu_lock);
6412
6413         /*
6414          * Zapping TDP MMU shadow pages, including the remote TLB flush, must
6415          * be done under RCU protection, because the pages are freed via RCU
6416          * callback.
6417          */
6418         rcu_read_lock();
6419
6420         ratio = READ_ONCE(nx_huge_pages_recovery_ratio);
6421         to_zap = ratio ? DIV_ROUND_UP(nx_lpage_splits, ratio) : 0;
6422         for ( ; to_zap; --to_zap) {
6423                 if (list_empty(&kvm->arch.lpage_disallowed_mmu_pages))
6424                         break;
6425
6426                 /*
6427                  * We use a separate list instead of just using active_mmu_pages
6428                  * because the number of lpage_disallowed pages is expected to
6429                  * be relatively small compared to the total.
6430                  */
6431                 sp = list_first_entry(&kvm->arch.lpage_disallowed_mmu_pages,
6432                                       struct kvm_mmu_page,
6433                                       lpage_disallowed_link);
6434                 WARN_ON_ONCE(!sp->lpage_disallowed);
6435                 if (is_tdp_mmu_page(sp)) {
6436                         flush |= kvm_tdp_mmu_zap_sp(kvm, sp);
6437                 } else {
6438                         kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list);
6439                         WARN_ON_ONCE(sp->lpage_disallowed);
6440                 }
6441
6442                 if (need_resched() || rwlock_needbreak(&kvm->mmu_lock)) {
6443                         kvm_mmu_remote_flush_or_zap(kvm, &invalid_list, flush);
6444                         rcu_read_unlock();
6445
6446                         cond_resched_rwlock_write(&kvm->mmu_lock);
6447                         flush = false;
6448
6449                         rcu_read_lock();
6450                 }
6451         }
6452         kvm_mmu_remote_flush_or_zap(kvm, &invalid_list, flush);
6453
6454         rcu_read_unlock();
6455
6456         write_unlock(&kvm->mmu_lock);
6457         srcu_read_unlock(&kvm->srcu, rcu_idx);
6458 }
6459
6460 static long get_nx_lpage_recovery_timeout(u64 start_time)
6461 {
6462         bool enabled;
6463         uint period;
6464
6465         enabled = calc_nx_huge_pages_recovery_period(&period);
6466
6467         return enabled ? start_time + msecs_to_jiffies(period) - get_jiffies_64()
6468                        : MAX_SCHEDULE_TIMEOUT;
6469 }
6470
6471 static int kvm_nx_lpage_recovery_worker(struct kvm *kvm, uintptr_t data)
6472 {
6473         u64 start_time;
6474         long remaining_time;
6475
6476         while (true) {
6477                 start_time = get_jiffies_64();
6478                 remaining_time = get_nx_lpage_recovery_timeout(start_time);
6479
6480                 set_current_state(TASK_INTERRUPTIBLE);
6481                 while (!kthread_should_stop() && remaining_time > 0) {
6482                         schedule_timeout(remaining_time);
6483                         remaining_time = get_nx_lpage_recovery_timeout(start_time);
6484                         set_current_state(TASK_INTERRUPTIBLE);
6485                 }
6486
6487                 set_current_state(TASK_RUNNING);
6488
6489                 if (kthread_should_stop())
6490                         return 0;
6491
6492                 kvm_recover_nx_lpages(kvm);
6493         }
6494 }
6495
6496 int kvm_mmu_post_init_vm(struct kvm *kvm)
6497 {
6498         int err;
6499
6500         err = kvm_vm_create_worker_thread(kvm, kvm_nx_lpage_recovery_worker, 0,
6501                                           "kvm-nx-lpage-recovery",
6502                                           &kvm->arch.nx_lpage_recovery_thread);
6503         if (!err)
6504                 kthread_unpark(kvm->arch.nx_lpage_recovery_thread);
6505
6506         return err;
6507 }
6508
6509 void kvm_mmu_pre_destroy_vm(struct kvm *kvm)
6510 {
6511         if (kvm->arch.nx_lpage_recovery_thread)
6512                 kthread_stop(kvm->arch.nx_lpage_recovery_thread);
6513 }