arch/x86/kvm/mmu/mmu.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Kernel-based Virtual Machine driver for Linux
   4  *
   5  * This module enables machines with Intel VT-x extensions to run virtual
   6  * machines without emulation or binary translation.
   7  *
   8  * MMU support
   9  *
  10  * Copyright (C) 2006 Qumranet, Inc.
  11  * Copyright 2010 Red Hat, Inc. and/or its affiliates.
  12  *
  13  * Authors:
  14  *   Yaniv Kamay  <yaniv@qumranet.com>
  15  *   Avi Kivity   <avi@qumranet.com>
  16  */
  17
  18 #include "irq.h"
  19 #include "ioapic.h"
  20 #include "mmu.h"
  21 #include "mmu_internal.h"
  22 #include "tdp_mmu.h"
  23 #include "x86.h"
  24 #include "kvm_cache_regs.h"
  25 #include "kvm_emulate.h"
  26 #include "cpuid.h"
  27 #include "spte.h"
  28
  29 #include <linux/kvm_host.h>
  30 #include <linux/types.h>
  31 #include <linux/string.h>
  32 #include <linux/mm.h>
  33 #include <linux/highmem.h>
  34 #include <linux/moduleparam.h>
  35 #include <linux/export.h>
  36 #include <linux/swap.h>
  37 #include <linux/hugetlb.h>
  38 #include <linux/compiler.h>
  39 #include <linux/srcu.h>
  40 #include <linux/slab.h>
  41 #include <linux/sched/signal.h>
  42 #include <linux/uaccess.h>
  43 #include <linux/hash.h>
  44 #include <linux/kern_levels.h>
  45 #include <linux/kthread.h>
  46
  47 #include <asm/page.h>
  48 #include <asm/memtype.h>
  49 #include <asm/cmpxchg.h>
  50 #include <asm/io.h>
  51 #include <asm/set_memory.h>
  52 #include <asm/vmx.h>
  53 #include <asm/kvm_page_track.h>
  54 #include "trace.h"
  55
  56 #include "paging.h"
  57
  58 extern bool itlb_multihit_kvm_mitigation;
  59
  60 int __read_mostly nx_huge_pages = -1;
  61 static uint __read_mostly nx_huge_pages_recovery_period_ms;
  62 #ifdef CONFIG_PREEMPT_RT
  63 /* Recovery can cause latency spikes, disable it for PREEMPT_RT.  */
  64 static uint __read_mostly nx_huge_pages_recovery_ratio = 0;
  65 #else
  66 static uint __read_mostly nx_huge_pages_recovery_ratio = 60;
  67 #endif
  68
  69 static int set_nx_huge_pages(const char *val, const struct kernel_param *kp);
  70 static int set_nx_huge_pages_recovery_param(const char *val, const struct kernel_param *kp);
  71
  72 static const struct kernel_param_ops nx_huge_pages_ops = {
  73         .set = set_nx_huge_pages,
  74         .get = param_get_bool,
  75 };
  76
  77 static const struct kernel_param_ops nx_huge_pages_recovery_param_ops = {
  78         .set = set_nx_huge_pages_recovery_param,
  79         .get = param_get_uint,
  80 };
  81
  82 module_param_cb(nx_huge_pages, &nx_huge_pages_ops, &nx_huge_pages, 0644);
  83 __MODULE_PARM_TYPE(nx_huge_pages, "bool");
  84 module_param_cb(nx_huge_pages_recovery_ratio, &nx_huge_pages_recovery_param_ops,
  85                 &nx_huge_pages_recovery_ratio, 0644);
  86 __MODULE_PARM_TYPE(nx_huge_pages_recovery_ratio, "uint");
  87 module_param_cb(nx_huge_pages_recovery_period_ms, &nx_huge_pages_recovery_param_ops,
  88                 &nx_huge_pages_recovery_period_ms, 0644);
  89 __MODULE_PARM_TYPE(nx_huge_pages_recovery_period_ms, "uint");
  90
  91 static bool __read_mostly force_flush_and_sync_on_reuse;
  92 module_param_named(flush_on_reuse, force_flush_and_sync_on_reuse, bool, 0644);
  93
  94 /*
  95  * When setting this variable to true it enables Two-Dimensional-Paging
  96  * where the hardware walks 2 page tables:
  97  * 1. the guest-virtual to guest-physical
  98  * 2. while doing 1. it walks guest-physical to host-physical
  99  * If the hardware supports that we don't need to do shadow paging.
 100  */
 101 bool tdp_enabled = false;
 102
 103 static int max_huge_page_level __read_mostly;
 104 static int tdp_root_level __read_mostly;
 105 static int max_tdp_level __read_mostly;
 106
 107 enum {
 108         AUDIT_PRE_PAGE_FAULT,
 109         AUDIT_POST_PAGE_FAULT,
 110         AUDIT_PRE_PTE_WRITE,
 111         AUDIT_POST_PTE_WRITE,
 112         AUDIT_PRE_SYNC,
 113         AUDIT_POST_SYNC
 114 };
 115
 116 #ifdef MMU_DEBUG
 117 bool dbg = 0;
 118 module_param(dbg, bool, 0644);
 119 #endif
 120
 121 #define PTE_PREFETCH_NUM                8
 122
 123 #define PT32_LEVEL_BITS 10
 124
 125 #define PT32_LEVEL_SHIFT(level) \
 126                 (PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS)
 127
 128 #define PT32_LVL_OFFSET_MASK(level) \
 129         (PT32_BASE_ADDR_MASK & ((1ULL << (PAGE_SHIFT + (((level) - 1) \
 130                                                 * PT32_LEVEL_BITS))) - 1))
 131
 132 #define PT32_INDEX(address, level)\
 133         (((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1))
 134
 135
 136 #define PT32_BASE_ADDR_MASK PAGE_MASK
 137 #define PT32_DIR_BASE_ADDR_MASK \
 138         (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + PT32_LEVEL_BITS)) - 1))
 139 #define PT32_LVL_ADDR_MASK(level) \
 140         (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \
 141                                             * PT32_LEVEL_BITS))) - 1))
 142
 143 #include <trace/events/kvm.h>
 144
 145 /* make pte_list_desc fit well in cache lines */
 146 #define PTE_LIST_EXT 14
 147
 148 /*
 149  * Slight optimization of cacheline layout, by putting `more' and `spte_count'
 150  * at the start; then accessing it will only use one single cacheline for
 151  * either full (entries==PTE_LIST_EXT) case or entries<=6.
 152  */
 153 struct pte_list_desc {
 154         struct pte_list_desc *more;
 155         /*
 156          * Stores number of entries stored in the pte_list_desc.  No need to be
 157          * u64 but just for easier alignment.  When PTE_LIST_EXT, means full.
 158          */
 159         u64 spte_count;
 160         u64 *sptes[PTE_LIST_EXT];
 161 };
 162
 163 struct kvm_shadow_walk_iterator {
 164         u64 addr;
 165         hpa_t shadow_addr;
 166         u64 *sptep;
 167         int level;
 168         unsigned index;
 169 };
 170
 171 #define for_each_shadow_entry_using_root(_vcpu, _root, _addr, _walker)     \
 172         for (shadow_walk_init_using_root(&(_walker), (_vcpu),              \
 173                                          (_root), (_addr));                \
 174              shadow_walk_okay(&(_walker));                                 \
 175              shadow_walk_next(&(_walker)))
 176
 177 #define for_each_shadow_entry(_vcpu, _addr, _walker)            \
 178         for (shadow_walk_init(&(_walker), _vcpu, _addr);        \
 179              shadow_walk_okay(&(_walker));                      \
 180              shadow_walk_next(&(_walker)))
 181
 182 #define for_each_shadow_entry_lockless(_vcpu, _addr, _walker, spte)     \
 183         for (shadow_walk_init(&(_walker), _vcpu, _addr);                \
 184              shadow_walk_okay(&(_walker)) &&                            \
 185                 ({ spte = mmu_spte_get_lockless(_walker.sptep); 1; });  \
 186              __shadow_walk_next(&(_walker), spte))
 187
 188 static struct kmem_cache *pte_list_desc_cache;
 189 struct kmem_cache *mmu_page_header_cache;
 190 static struct percpu_counter kvm_total_used_mmu_pages;
 191
 192 static void mmu_spte_set(u64 *sptep, u64 spte);
 193 static union kvm_mmu_page_role
 194 kvm_mmu_calc_root_page_role(struct kvm_vcpu *vcpu);
 195
 196 struct kvm_mmu_role_regs {
 197         const unsigned long cr0;
 198         const unsigned long cr4;
 199         const u64 efer;
 200 };
 201
 202 #define CREATE_TRACE_POINTS
 203 #include "mmutrace.h"
 204
 205 /*
 206  * Yes, lot's of underscores.  They're a hint that you probably shouldn't be
 207  * reading from the role_regs.  Once the mmu_role is constructed, it becomes
 208  * the single source of truth for the MMU's state.
 209  */
 210 #define BUILD_MMU_ROLE_REGS_ACCESSOR(reg, name, flag)                   \
 211 static inline bool __maybe_unused ____is_##reg##_##name(struct kvm_mmu_role_regs *regs)\
 212 {                                                                       \
 213         return !!(regs->reg & flag);                                    \
 214 }
 215 BUILD_MMU_ROLE_REGS_ACCESSOR(cr0, pg, X86_CR0_PG);
 216 BUILD_MMU_ROLE_REGS_ACCESSOR(cr0, wp, X86_CR0_WP);
 217 BUILD_MMU_ROLE_REGS_ACCESSOR(cr4, pse, X86_CR4_PSE);
 218 BUILD_MMU_ROLE_REGS_ACCESSOR(cr4, pae, X86_CR4_PAE);
 219 BUILD_MMU_ROLE_REGS_ACCESSOR(cr4, smep, X86_CR4_SMEP);
 220 BUILD_MMU_ROLE_REGS_ACCESSOR(cr4, smap, X86_CR4_SMAP);
 221 BUILD_MMU_ROLE_REGS_ACCESSOR(cr4, pke, X86_CR4_PKE);
 222 BUILD_MMU_ROLE_REGS_ACCESSOR(cr4, la57, X86_CR4_LA57);
 223 BUILD_MMU_ROLE_REGS_ACCESSOR(efer, nx, EFER_NX);
 224 BUILD_MMU_ROLE_REGS_ACCESSOR(efer, lma, EFER_LMA);
 225
 226 /*
 227  * The MMU itself (with a valid role) is the single source of truth for the
 228  * MMU.  Do not use the regs used to build the MMU/role, nor the vCPU.  The
 229  * regs don't account for dependencies, e.g. clearing CR4 bits if CR0.PG=1,
 230  * and the vCPU may be incorrect/irrelevant.
 231  */
 232 #define BUILD_MMU_ROLE_ACCESSOR(base_or_ext, reg, name)         \
 233 static inline bool __maybe_unused is_##reg##_##name(struct kvm_mmu *mmu)        \
 234 {                                                               \
 235         return !!(mmu->mmu_role. base_or_ext . reg##_##name);   \
 236 }
 237 BUILD_MMU_ROLE_ACCESSOR(ext,  cr0, pg);
 238 BUILD_MMU_ROLE_ACCESSOR(base, cr0, wp);
 239 BUILD_MMU_ROLE_ACCESSOR(ext,  cr4, pse);
 240 BUILD_MMU_ROLE_ACCESSOR(ext,  cr4, pae);
 241 BUILD_MMU_ROLE_ACCESSOR(ext,  cr4, smep);
 242 BUILD_MMU_ROLE_ACCESSOR(ext,  cr4, smap);
 243 BUILD_MMU_ROLE_ACCESSOR(ext,  cr4, pke);
 244 BUILD_MMU_ROLE_ACCESSOR(ext,  cr4, la57);
 245 BUILD_MMU_ROLE_ACCESSOR(base, efer, nx);
 246
 247 static struct kvm_mmu_role_regs vcpu_to_role_regs(struct kvm_vcpu *vcpu)
 248 {
 249         struct kvm_mmu_role_regs regs = {
 250                 .cr0 = kvm_read_cr0_bits(vcpu, KVM_MMU_CR0_ROLE_BITS),
 251                 .cr4 = kvm_read_cr4_bits(vcpu, KVM_MMU_CR4_ROLE_BITS),
 252                 .efer = vcpu->arch.efer,
 253         };
 254
 255         return regs;
 256 }
 257
 258 static int role_regs_to_root_level(struct kvm_mmu_role_regs *regs)
 259 {
 260         if (!____is_cr0_pg(regs))
 261                 return 0;
 262         else if (____is_efer_lma(regs))
 263                 return ____is_cr4_la57(regs) ? PT64_ROOT_5LEVEL :
 264                                                PT64_ROOT_4LEVEL;
 265         else if (____is_cr4_pae(regs))
 266                 return PT32E_ROOT_LEVEL;
 267         else
 268                 return PT32_ROOT_LEVEL;
 269 }
 270
 271 static inline bool kvm_available_flush_tlb_with_range(void)
 272 {
 273         return kvm_x86_ops.tlb_remote_flush_with_range;
 274 }
 275
 276 static void kvm_flush_remote_tlbs_with_range(struct kvm *kvm,
 277                 struct kvm_tlb_range *range)
 278 {
 279         int ret = -ENOTSUPP;
 280
 281         if (range && kvm_x86_ops.tlb_remote_flush_with_range)
 282                 ret = static_call(kvm_x86_tlb_remote_flush_with_range)(kvm, range);
 283
 284         if (ret)
 285                 kvm_flush_remote_tlbs(kvm);
 286 }
 287
 288 void kvm_flush_remote_tlbs_with_address(struct kvm *kvm,
 289                 u64 start_gfn, u64 pages)
 290 {
 291         struct kvm_tlb_range range;
 292
 293         range.start_gfn = start_gfn;
 294         range.pages = pages;
 295
 296         kvm_flush_remote_tlbs_with_range(kvm, &range);
 297 }
 298
 299 static void mark_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, u64 gfn,
 300                            unsigned int access)
 301 {
 302         u64 spte = make_mmio_spte(vcpu, gfn, access);
 303
 304         trace_mark_mmio_spte(sptep, gfn, spte);
 305         mmu_spte_set(sptep, spte);
 306 }
 307
 308 static gfn_t get_mmio_spte_gfn(u64 spte)
 309 {
 310         u64 gpa = spte & shadow_nonpresent_or_rsvd_lower_gfn_mask;
 311
 312         gpa |= (spte >> SHADOW_NONPRESENT_OR_RSVD_MASK_LEN)
 313                & shadow_nonpresent_or_rsvd_mask;
 314
 315         return gpa >> PAGE_SHIFT;
 316 }
 317
 318 static unsigned get_mmio_spte_access(u64 spte)
 319 {
 320         return spte & shadow_mmio_access_mask;
 321 }
 322
 323 static bool check_mmio_spte(struct kvm_vcpu *vcpu, u64 spte)
 324 {
 325         u64 kvm_gen, spte_gen, gen;
 326
 327         gen = kvm_vcpu_memslots(vcpu)->generation;
 328         if (unlikely(gen & KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS))
 329                 return false;
 330
 331         kvm_gen = gen & MMIO_SPTE_GEN_MASK;
 332         spte_gen = get_mmio_spte_generation(spte);
 333
 334         trace_check_mmio_spte(spte, kvm_gen, spte_gen);
 335         return likely(kvm_gen == spte_gen);
 336 }
 337
 338 static gpa_t translate_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access,
 339                                   struct x86_exception *exception)
 340 {
 341         return gpa;
 342 }
 343
 344 static int is_cpuid_PSE36(void)
 345 {
 346         return 1;
 347 }
 348
 349 static gfn_t pse36_gfn_delta(u32 gpte)
 350 {
 351         int shift = 32 - PT32_DIR_PSE36_SHIFT - PAGE_SHIFT;
 352
 353         return (gpte & PT32_DIR_PSE36_MASK) << shift;
 354 }
 355
 356 #ifdef CONFIG_X86_64
 357 static void __set_spte(u64 *sptep, u64 spte)
 358 {
 359         WRITE_ONCE(*sptep, spte);
 360 }
 361
 362 static void __update_clear_spte_fast(u64 *sptep, u64 spte)
 363 {
 364         WRITE_ONCE(*sptep, spte);
 365 }
 366
 367 static u64 __update_clear_spte_slow(u64 *sptep, u64 spte)
 368 {
 369         return xchg(sptep, spte);
 370 }
 371
 372 static u64 __get_spte_lockless(u64 *sptep)
 373 {
 374         return READ_ONCE(*sptep);
 375 }
 376 #else
 377 union split_spte {
 378         struct {
 379                 u32 spte_low;
 380                 u32 spte_high;
 381         };
 382         u64 spte;
 383 };
 384
 385 static void count_spte_clear(u64 *sptep, u64 spte)
 386 {
 387         struct kvm_mmu_page *sp =  sptep_to_sp(sptep);
 388
 389         if (is_shadow_present_pte(spte))
 390                 return;
 391
 392         /* Ensure the spte is completely set before we increase the count */
 393         smp_wmb();
 394         sp->clear_spte_count++;
 395 }
 396
 397 static void __set_spte(u64 *sptep, u64 spte)
 398 {
 399         union split_spte *ssptep, sspte;
 400
 401         ssptep = (union split_spte *)sptep;
 402         sspte = (union split_spte)spte;
 403
 404         ssptep->spte_high = sspte.spte_high;
 405
 406         /*
 407          * If we map the spte from nonpresent to present, We should store
 408          * the high bits firstly, then set present bit, so cpu can not
 409          * fetch this spte while we are setting the spte.
 410          */
 411         smp_wmb();
 412
 413         WRITE_ONCE(ssptep->spte_low, sspte.spte_low);
 414 }
 415
 416 static void __update_clear_spte_fast(u64 *sptep, u64 spte)
 417 {
 418         union split_spte *ssptep, sspte;
 419
 420         ssptep = (union split_spte *)sptep;
 421         sspte = (union split_spte)spte;
 422
 423         WRITE_ONCE(ssptep->spte_low, sspte.spte_low);
 424
 425         /*
 426          * If we map the spte from present to nonpresent, we should clear
 427          * present bit firstly to avoid vcpu fetch the old high bits.
 428          */
 429         smp_wmb();
 430
 431         ssptep->spte_high = sspte.spte_high;
 432         count_spte_clear(sptep, spte);
 433 }
 434
 435 static u64 __update_clear_spte_slow(u64 *sptep, u64 spte)
 436 {
 437         union split_spte *ssptep, sspte, orig;
 438
 439         ssptep = (union split_spte *)sptep;
 440         sspte = (union split_spte)spte;
 441
 442         /* xchg acts as a barrier before the setting of the high bits */
 443         orig.spte_low = xchg(&ssptep->spte_low, sspte.spte_low);
 444         orig.spte_high = ssptep->spte_high;
 445         ssptep->spte_high = sspte.spte_high;
 446         count_spte_clear(sptep, spte);
 447
 448         return orig.spte;
 449 }
 450
 451 /*
 452  * The idea using the light way get the spte on x86_32 guest is from
 453  * gup_get_pte (mm/gup.c).
 454  *
 455  * An spte tlb flush may be pending, because kvm_set_pte_rmapp
 456  * coalesces them and we are running out of the MMU lock.  Therefore
 457  * we need to protect against in-progress updates of the spte.
 458  *
 459  * Reading the spte while an update is in progress may get the old value
 460  * for the high part of the spte.  The race is fine for a present->non-present
 461  * change (because the high part of the spte is ignored for non-present spte),
 462  * but for a present->present change we must reread the spte.
 463  *
 464  * All such changes are done in two steps (present->non-present and
 465  * non-present->present), hence it is enough to count the number of
 466  * present->non-present updates: if it changed while reading the spte,
 467  * we might have hit the race.  This is done using clear_spte_count.
 468  */
 469 static u64 __get_spte_lockless(u64 *sptep)
 470 {
 471         struct kvm_mmu_page *sp =  sptep_to_sp(sptep);
 472         union split_spte spte, *orig = (union split_spte *)sptep;
 473         int count;
 474
 475 retry:
 476         count = sp->clear_spte_count;
 477         smp_rmb();
 478
 479         spte.spte_low = orig->spte_low;
 480         smp_rmb();
 481
 482         spte.spte_high = orig->spte_high;
 483         smp_rmb();
 484
 485         if (unlikely(spte.spte_low != orig->spte_low ||
 486               count != sp->clear_spte_count))
 487                 goto retry;
 488
 489         return spte.spte;
 490 }
 491 #endif
 492
 493 static bool spte_has_volatile_bits(u64 spte)
 494 {
 495         if (!is_shadow_present_pte(spte))
 496                 return false;
 497
 498         /*
 499          * Always atomically update spte if it can be updated
 500          * out of mmu-lock, it can ensure dirty bit is not lost,
 501          * also, it can help us to get a stable is_writable_pte()
 502          * to ensure tlb flush is not missed.
 503          */
 504         if (spte_can_locklessly_be_made_writable(spte) ||
 505             is_access_track_spte(spte))
 506                 return true;
 507
 508         if (spte_ad_enabled(spte)) {
 509                 if ((spte & shadow_accessed_mask) == 0 ||
 510                     (is_writable_pte(spte) && (spte & shadow_dirty_mask) == 0))
 511                         return true;
 512         }
 513
 514         return false;
 515 }
 516
 517 /* Rules for using mmu_spte_set:
 518  * Set the sptep from nonpresent to present.
 519  * Note: the sptep being assigned *must* be either not present
 520  * or in a state where the hardware will not attempt to update
 521  * the spte.
 522  */
 523 static void mmu_spte_set(u64 *sptep, u64 new_spte)
 524 {
 525         WARN_ON(is_shadow_present_pte(*sptep));
 526         __set_spte(sptep, new_spte);
 527 }
 528
 529 /*
 530  * Update the SPTE (excluding the PFN), but do not track changes in its
 531  * accessed/dirty status.
 532  */
 533 static u64 mmu_spte_update_no_track(u64 *sptep, u64 new_spte)
 534 {
 535         u64 old_spte = *sptep;
 536
 537         WARN_ON(!is_shadow_present_pte(new_spte));
 538
 539         if (!is_shadow_present_pte(old_spte)) {
 540                 mmu_spte_set(sptep, new_spte);
 541                 return old_spte;
 542         }
 543
 544         if (!spte_has_volatile_bits(old_spte))
 545                 __update_clear_spte_fast(sptep, new_spte);
 546         else
 547                 old_spte = __update_clear_spte_slow(sptep, new_spte);
 548
 549         WARN_ON(spte_to_pfn(old_spte) != spte_to_pfn(new_spte));
 550
 551         return old_spte;
 552 }
 553
 554 /* Rules for using mmu_spte_update:
 555  * Update the state bits, it means the mapped pfn is not changed.
 556  *
 557  * Whenever we overwrite a writable spte with a read-only one we
 558  * should flush remote TLBs. Otherwise rmap_write_protect
 559  * will find a read-only spte, even though the writable spte
 560  * might be cached on a CPU's TLB, the return value indicates this
 561  * case.
 562  *
 563  * Returns true if the TLB needs to be flushed
 564  */
 565 static bool mmu_spte_update(u64 *sptep, u64 new_spte)
 566 {
 567         bool flush = false;
 568         u64 old_spte = mmu_spte_update_no_track(sptep, new_spte);
 569
 570         if (!is_shadow_present_pte(old_spte))
 571                 return false;
 572
 573         /*
 574          * For the spte updated out of mmu-lock is safe, since
 575          * we always atomically update it, see the comments in
 576          * spte_has_volatile_bits().
 577          */
 578         if (spte_can_locklessly_be_made_writable(old_spte) &&
 579               !is_writable_pte(new_spte))
 580                 flush = true;
 581
 582         /*
 583          * Flush TLB when accessed/dirty states are changed in the page tables,
 584          * to guarantee consistency between TLB and page tables.
 585          */
 586
 587         if (is_accessed_spte(old_spte) && !is_accessed_spte(new_spte)) {
 588                 flush = true;
 589                 kvm_set_pfn_accessed(spte_to_pfn(old_spte));
 590         }
 591
 592         if (is_dirty_spte(old_spte) && !is_dirty_spte(new_spte)) {
 593                 flush = true;
 594                 kvm_set_pfn_dirty(spte_to_pfn(old_spte));
 595         }
 596
 597         return flush;
 598 }
 599
 600 /*
 601  * Rules for using mmu_spte_clear_track_bits:
 602  * It sets the sptep from present to nonpresent, and track the
 603  * state bits, it is used to clear the last level sptep.
 604  * Returns the old PTE.
 605  */
 606 static int mmu_spte_clear_track_bits(struct kvm *kvm, u64 *sptep)
 607 {
 608         kvm_pfn_t pfn;
 609         u64 old_spte = *sptep;
 610         int level = sptep_to_sp(sptep)->role.level;
 611
 612         if (!spte_has_volatile_bits(old_spte))
 613                 __update_clear_spte_fast(sptep, 0ull);
 614         else
 615                 old_spte = __update_clear_spte_slow(sptep, 0ull);
 616
 617         if (!is_shadow_present_pte(old_spte))
 618                 return old_spte;
 619
 620         kvm_update_page_stats(kvm, level, -1);
 621
 622         pfn = spte_to_pfn(old_spte);
 623
 624         /*
 625          * KVM does not hold the refcount of the page used by
 626          * kvm mmu, before reclaiming the page, we should
 627          * unmap it from mmu first.
 628          */
 629         WARN_ON(!kvm_is_reserved_pfn(pfn) && !page_count(pfn_to_page(pfn)));
 630
 631         if (is_accessed_spte(old_spte))
 632                 kvm_set_pfn_accessed(pfn);
 633
 634         if (is_dirty_spte(old_spte))
 635                 kvm_set_pfn_dirty(pfn);
 636
 637         return old_spte;
 638 }
 639
 640 /*
 641  * Rules for using mmu_spte_clear_no_track:
 642  * Directly clear spte without caring the state bits of sptep,
 643  * it is used to set the upper level spte.
 644  */
 645 static void mmu_spte_clear_no_track(u64 *sptep)
 646 {
 647         __update_clear_spte_fast(sptep, 0ull);
 648 }
 649
 650 static u64 mmu_spte_get_lockless(u64 *sptep)
 651 {
 652         return __get_spte_lockless(sptep);
 653 }
 654
 655 /* Restore an acc-track PTE back to a regular PTE */
 656 static u64 restore_acc_track_spte(u64 spte)
 657 {
 658         u64 new_spte = spte;
 659         u64 saved_bits = (spte >> SHADOW_ACC_TRACK_SAVED_BITS_SHIFT)
 660                          & SHADOW_ACC_TRACK_SAVED_BITS_MASK;
 661
 662         WARN_ON_ONCE(spte_ad_enabled(spte));
 663         WARN_ON_ONCE(!is_access_track_spte(spte));
 664
 665         new_spte &= ~shadow_acc_track_mask;
 666         new_spte &= ~(SHADOW_ACC_TRACK_SAVED_BITS_MASK <<
 667                       SHADOW_ACC_TRACK_SAVED_BITS_SHIFT);
 668         new_spte |= saved_bits;
 669
 670         return new_spte;
 671 }
 672
 673 /* Returns the Accessed status of the PTE and resets it at the same time. */
 674 static bool mmu_spte_age(u64 *sptep)
 675 {
 676         u64 spte = mmu_spte_get_lockless(sptep);
 677
 678         if (!is_accessed_spte(spte))
 679                 return false;
 680
 681         if (spte_ad_enabled(spte)) {
 682                 clear_bit((ffs(shadow_accessed_mask) - 1),
 683                           (unsigned long *)sptep);
 684         } else {
 685                 /*
 686                  * Capture the dirty status of the page, so that it doesn't get
 687                  * lost when the SPTE is marked for access tracking.
 688                  */
 689                 if (is_writable_pte(spte))
 690                         kvm_set_pfn_dirty(spte_to_pfn(spte));
 691
 692                 spte = mark_spte_for_access_track(spte);
 693                 mmu_spte_update_no_track(sptep, spte);
 694         }
 695
 696         return true;
 697 }
 698
 699 static void walk_shadow_page_lockless_begin(struct kvm_vcpu *vcpu)
 700 {
 701         if (is_tdp_mmu(vcpu->arch.mmu)) {
 702                 kvm_tdp_mmu_walk_lockless_begin();
 703         } else {
 704                 /*
 705                  * Prevent page table teardown by making any free-er wait during
 706                  * kvm_flush_remote_tlbs() IPI to all active vcpus.
 707                  */
 708                 local_irq_disable();
 709
 710                 /*
 711                  * Make sure a following spte read is not reordered ahead of the write
 712                  * to vcpu->mode.
 713                  */
 714                 smp_store_mb(vcpu->mode, READING_SHADOW_PAGE_TABLES);
 715         }
 716 }
 717
 718 static void walk_shadow_page_lockless_end(struct kvm_vcpu *vcpu)
 719 {
 720         if (is_tdp_mmu(vcpu->arch.mmu)) {
 721                 kvm_tdp_mmu_walk_lockless_end();
 722         } else {
 723                 /*
 724                  * Make sure the write to vcpu->mode is not reordered in front of
 725                  * reads to sptes.  If it does, kvm_mmu_commit_zap_page() can see us
 726                  * OUTSIDE_GUEST_MODE and proceed to free the shadow page table.
 727                  */
 728                 smp_store_release(&vcpu->mode, OUTSIDE_GUEST_MODE);
 729                 local_irq_enable();
 730         }
 731 }
 732
 733 static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu, bool maybe_indirect)
 734 {
 735         int r;
 736
 737         /* 1 rmap, 1 parent PTE per level, and the prefetched rmaps. */
 738         r = kvm_mmu_topup_memory_cache(&vcpu->arch.mmu_pte_list_desc_cache,
 739                                        1 + PT64_ROOT_MAX_LEVEL + PTE_PREFETCH_NUM);
 740         if (r)
 741                 return r;
 742         r = kvm_mmu_topup_memory_cache(&vcpu->arch.mmu_shadow_page_cache,
 743                                        PT64_ROOT_MAX_LEVEL);
 744         if (r)
 745                 return r;
 746         if (maybe_indirect) {
 747                 r = kvm_mmu_topup_memory_cache(&vcpu->arch.mmu_gfn_array_cache,
 748                                                PT64_ROOT_MAX_LEVEL);
 749                 if (r)
 750                         return r;
 751         }
 752         return kvm_mmu_topup_memory_cache(&vcpu->arch.mmu_page_header_cache,
 753                                           PT64_ROOT_MAX_LEVEL);
 754 }
 755
 756 static void mmu_free_memory_caches(struct kvm_vcpu *vcpu)
 757 {
 758         kvm_mmu_free_memory_cache(&vcpu->arch.mmu_pte_list_desc_cache);
 759         kvm_mmu_free_memory_cache(&vcpu->arch.mmu_shadow_page_cache);
 760         kvm_mmu_free_memory_cache(&vcpu->arch.mmu_gfn_array_cache);
 761         kvm_mmu_free_memory_cache(&vcpu->arch.mmu_page_header_cache);
 762 }
 763
 764 static struct pte_list_desc *mmu_alloc_pte_list_desc(struct kvm_vcpu *vcpu)
 765 {
 766         return kvm_mmu_memory_cache_alloc(&vcpu->arch.mmu_pte_list_desc_cache);
 767 }
 768
 769 static void mmu_free_pte_list_desc(struct pte_list_desc *pte_list_desc)
 770 {
 771         kmem_cache_free(pte_list_desc_cache, pte_list_desc);
 772 }
 773
 774 static gfn_t kvm_mmu_page_get_gfn(struct kvm_mmu_page *sp, int index)
 775 {
 776         if (!sp->role.direct)
 777                 return sp->gfns[index];
 778
 779         return sp->gfn + (index << ((sp->role.level - 1) * PT64_LEVEL_BITS));
 780 }
 781
 782 static void kvm_mmu_page_set_gfn(struct kvm_mmu_page *sp, int index, gfn_t gfn)
 783 {
 784         if (!sp->role.direct) {
 785                 sp->gfns[index] = gfn;
 786                 return;
 787         }
 788
 789         if (WARN_ON(gfn != kvm_mmu_page_get_gfn(sp, index)))
 790                 pr_err_ratelimited("gfn mismatch under direct page %llx "
 791                                    "(expected %llx, got %llx)\n",
 792                                    sp->gfn,
 793                                    kvm_mmu_page_get_gfn(sp, index), gfn);
 794 }
 795
 796 /*
 797  * Return the pointer to the large page information for a given gfn,
 798  * handling slots that are not large page aligned.
 799  */
 800 static struct kvm_lpage_info *lpage_info_slot(gfn_t gfn,
 801                 const struct kvm_memory_slot *slot, int level)
 802 {
 803         unsigned long idx;
 804
 805         idx = gfn_to_index(gfn, slot->base_gfn, level);
 806         return &slot->arch.lpage_info[level - 2][idx];
 807 }
 808
 809 static void update_gfn_disallow_lpage_count(const struct kvm_memory_slot *slot,
 810                                             gfn_t gfn, int count)
 811 {
 812         struct kvm_lpage_info *linfo;
 813         int i;
 814
 815         for (i = PG_LEVEL_2M; i <= KVM_MAX_HUGEPAGE_LEVEL; ++i) {
 816                 linfo = lpage_info_slot(gfn, slot, i);
 817                 linfo->disallow_lpage += count;
 818                 WARN_ON(linfo->disallow_lpage < 0);
 819         }
 820 }
 821
 822 void kvm_mmu_gfn_disallow_lpage(const struct kvm_memory_slot *slot, gfn_t gfn)
 823 {
 824         update_gfn_disallow_lpage_count(slot, gfn, 1);
 825 }
 826
 827 void kvm_mmu_gfn_allow_lpage(const struct kvm_memory_slot *slot, gfn_t gfn)
 828 {
 829         update_gfn_disallow_lpage_count(slot, gfn, -1);
 830 }
 831
 832 static void account_shadowed(struct kvm *kvm, struct kvm_mmu_page *sp)
 833 {
 834         struct kvm_memslots *slots;
 835         struct kvm_memory_slot *slot;
 836         gfn_t gfn;
 837
 838         kvm->arch.indirect_shadow_pages++;
 839         gfn = sp->gfn;
 840         slots = kvm_memslots_for_spte_role(kvm, sp->role);
 841         slot = __gfn_to_memslot(slots, gfn);
 842
 843         /* the non-leaf shadow pages are keeping readonly. */
 844         if (sp->role.level > PG_LEVEL_4K)
 845                 return kvm_slot_page_track_add_page(kvm, slot, gfn,
 846                                                     KVM_PAGE_TRACK_WRITE);
 847
 848         kvm_mmu_gfn_disallow_lpage(slot, gfn);
 849 }
 850
 851 void account_huge_nx_page(struct kvm *kvm, struct kvm_mmu_page *sp)
 852 {
 853         if (sp->lpage_disallowed)
 854                 return;
 855
 856         ++kvm->stat.nx_lpage_splits;
 857         list_add_tail(&sp->lpage_disallowed_link,
 858                       &kvm->arch.lpage_disallowed_mmu_pages);
 859         sp->lpage_disallowed = true;
 860 }
 861
 862 static void unaccount_shadowed(struct kvm *kvm, struct kvm_mmu_page *sp)
 863 {
 864         struct kvm_memslots *slots;
 865         struct kvm_memory_slot *slot;
 866         gfn_t gfn;
 867
 868         kvm->arch.indirect_shadow_pages--;
 869         gfn = sp->gfn;
 870         slots = kvm_memslots_for_spte_role(kvm, sp->role);
 871         slot = __gfn_to_memslot(slots, gfn);
 872         if (sp->role.level > PG_LEVEL_4K)
 873                 return kvm_slot_page_track_remove_page(kvm, slot, gfn,
 874                                                        KVM_PAGE_TRACK_WRITE);
 875
 876         kvm_mmu_gfn_allow_lpage(slot, gfn);
 877 }
 878
 879 void unaccount_huge_nx_page(struct kvm *kvm, struct kvm_mmu_page *sp)
 880 {
 881         --kvm->stat.nx_lpage_splits;
 882         sp->lpage_disallowed = false;
 883         list_del(&sp->lpage_disallowed_link);
 884 }
 885
 886 static struct kvm_memory_slot *
 887 gfn_to_memslot_dirty_bitmap(struct kvm_vcpu *vcpu, gfn_t gfn,
 888                             bool no_dirty_log)
 889 {
 890         struct kvm_memory_slot *slot;
 891
 892         slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
 893         if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
 894                 return NULL;
 895         if (no_dirty_log && kvm_slot_dirty_track_enabled(slot))
 896                 return NULL;
 897
 898         return slot;
 899 }
 900
 901 /*
 902  * About rmap_head encoding:
 903  *
 904  * If the bit zero of rmap_head->val is clear, then it points to the only spte
 905  * in this rmap chain. Otherwise, (rmap_head->val & ~1) points to a struct
 906  * pte_list_desc containing more mappings.
 907  */
 908
 909 /*
 910  * Returns the number of pointers in the rmap chain, not counting the new one.
 911  */
 912 static int pte_list_add(struct kvm_vcpu *vcpu, u64 *spte,
 913                         struct kvm_rmap_head *rmap_head)
 914 {
 915         struct pte_list_desc *desc;
 916         int count = 0;
 917
 918         if (!rmap_head->val) {
 919                 rmap_printk("%p %llx 0->1\n", spte, *spte);
 920                 rmap_head->val = (unsigned long)spte;
 921         } else if (!(rmap_head->val & 1)) {
 922                 rmap_printk("%p %llx 1->many\n", spte, *spte);
 923                 desc = mmu_alloc_pte_list_desc(vcpu);
 924                 desc->sptes[0] = (u64 *)rmap_head->val;
 925                 desc->sptes[1] = spte;
 926                 desc->spte_count = 2;
 927                 rmap_head->val = (unsigned long)desc | 1;
 928                 ++count;
 929         } else {
 930                 rmap_printk("%p %llx many->many\n", spte, *spte);
 931                 desc = (struct pte_list_desc *)(rmap_head->val & ~1ul);
 932                 while (desc->spte_count == PTE_LIST_EXT) {
 933                         count += PTE_LIST_EXT;
 934                         if (!desc->more) {
 935                                 desc->more = mmu_alloc_pte_list_desc(vcpu);
 936                                 desc = desc->more;
 937                                 desc->spte_count = 0;
 938                                 break;
 939                         }
 940                         desc = desc->more;
 941                 }
 942                 count += desc->spte_count;
 943                 desc->sptes[desc->spte_count++] = spte;
 944         }
 945         return count;
 946 }
 947
 948 static void
 949 pte_list_desc_remove_entry(struct kvm_rmap_head *rmap_head,
 950                            struct pte_list_desc *desc, int i,
 951                            struct pte_list_desc *prev_desc)
 952 {
 953         int j = desc->spte_count - 1;
 954
 955         desc->sptes[i] = desc->sptes[j];
 956         desc->sptes[j] = NULL;
 957         desc->spte_count--;
 958         if (desc->spte_count)
 959                 return;
 960         if (!prev_desc && !desc->more)
 961                 rmap_head->val = 0;
 962         else
 963                 if (prev_desc)
 964                         prev_desc->more = desc->more;
 965                 else
 966                         rmap_head->val = (unsigned long)desc->more | 1;
 967         mmu_free_pte_list_desc(desc);
 968 }
 969
 970 static void __pte_list_remove(u64 *spte, struct kvm_rmap_head *rmap_head)
 971 {
 972         struct pte_list_desc *desc;
 973         struct pte_list_desc *prev_desc;
 974         int i;
 975
 976         if (!rmap_head->val) {
 977                 pr_err("%s: %p 0->BUG\n", __func__, spte);
 978                 BUG();
 979         } else if (!(rmap_head->val & 1)) {
 980                 rmap_printk("%p 1->0\n", spte);
 981                 if ((u64 *)rmap_head->val != spte) {
 982                         pr_err("%s:  %p 1->BUG\n", __func__, spte);
 983                         BUG();
 984                 }
 985                 rmap_head->val = 0;
 986         } else {
 987                 rmap_printk("%p many->many\n", spte);
 988                 desc = (struct pte_list_desc *)(rmap_head->val & ~1ul);
 989                 prev_desc = NULL;
 990                 while (desc) {
 991                         for (i = 0; i < desc->spte_count; ++i) {
 992                                 if (desc->sptes[i] == spte) {
 993                                         pte_list_desc_remove_entry(rmap_head,
 994                                                         desc, i, prev_desc);
 995                                         return;
 996                                 }
 997                         }
 998                         prev_desc = desc;
 999                         desc = desc->more;
1000                 }
1001                 pr_err("%s: %p many->many\n", __func__, spte);
1002                 BUG();
1003         }
1004 }
1005
1006 static void pte_list_remove(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1007                             u64 *sptep)
1008 {
1009         mmu_spte_clear_track_bits(kvm, sptep);
1010         __pte_list_remove(sptep, rmap_head);
1011 }
1012
1013 /* Return true if rmap existed, false otherwise */
1014 static bool pte_list_destroy(struct kvm *kvm, struct kvm_rmap_head *rmap_head)
1015 {
1016         struct pte_list_desc *desc, *next;
1017         int i;
1018
1019         if (!rmap_head->val)
1020                 return false;
1021
1022         if (!(rmap_head->val & 1)) {
1023                 mmu_spte_clear_track_bits(kvm, (u64 *)rmap_head->val);
1024                 goto out;
1025         }
1026
1027         desc = (struct pte_list_desc *)(rmap_head->val & ~1ul);
1028
1029         for (; desc; desc = next) {
1030                 for (i = 0; i < desc->spte_count; i++)
1031                         mmu_spte_clear_track_bits(kvm, desc->sptes[i]);
1032                 next = desc->more;
1033                 mmu_free_pte_list_desc(desc);
1034         }
1035 out:
1036         /* rmap_head is meaningless now, remember to reset it */
1037         rmap_head->val = 0;
1038         return true;
1039 }
1040
1041 unsigned int pte_list_count(struct kvm_rmap_head *rmap_head)
1042 {
1043         struct pte_list_desc *desc;
1044         unsigned int count = 0;
1045
1046         if (!rmap_head->val)
1047                 return 0;
1048         else if (!(rmap_head->val & 1))
1049                 return 1;
1050
1051         desc = (struct pte_list_desc *)(rmap_head->val & ~1ul);
1052
1053         while (desc) {
1054                 count += desc->spte_count;
1055                 desc = desc->more;
1056         }
1057
1058         return count;
1059 }
1060
1061 static struct kvm_rmap_head *gfn_to_rmap(gfn_t gfn, int level,
1062                                          const struct kvm_memory_slot *slot)
1063 {
1064         unsigned long idx;
1065
1066         idx = gfn_to_index(gfn, slot->base_gfn, level);
1067         return &slot->arch.rmap[level - PG_LEVEL_4K][idx];
1068 }
1069
1070 static bool rmap_can_add(struct kvm_vcpu *vcpu)
1071 {
1072         struct kvm_mmu_memory_cache *mc;
1073
1074         mc = &vcpu->arch.mmu_pte_list_desc_cache;
1075         return kvm_mmu_memory_cache_nr_free_objects(mc);
1076 }
1077
1078 static void rmap_remove(struct kvm *kvm, u64 *spte)
1079 {
1080         struct kvm_memslots *slots;
1081         struct kvm_memory_slot *slot;
1082         struct kvm_mmu_page *sp;
1083         gfn_t gfn;
1084         struct kvm_rmap_head *rmap_head;
1085
1086         sp = sptep_to_sp(spte);
1087         gfn = kvm_mmu_page_get_gfn(sp, spte - sp->spt);
1088
1089         /*
1090          * Unlike rmap_add, rmap_remove does not run in the context of a vCPU
1091          * so we have to determine which memslots to use based on context
1092          * information in sp->role.
1093          */
1094         slots = kvm_memslots_for_spte_role(kvm, sp->role);
1095
1096         slot = __gfn_to_memslot(slots, gfn);
1097         rmap_head = gfn_to_rmap(gfn, sp->role.level, slot);
1098
1099         __pte_list_remove(spte, rmap_head);
1100 }
1101
1102 /*
1103  * Used by the following functions to iterate through the sptes linked by a
1104  * rmap.  All fields are private and not assumed to be used outside.
1105  */
1106 struct rmap_iterator {
1107         /* private fields */
1108         struct pte_list_desc *desc;     /* holds the sptep if not NULL */
1109         int pos;                        /* index of the sptep */
1110 };
1111
1112 /*
1113  * Iteration must be started by this function.  This should also be used after
1114  * removing/dropping sptes from the rmap link because in such cases the
1115  * information in the iterator may not be valid.
1116  *
1117  * Returns sptep if found, NULL otherwise.
1118  */
1119 static u64 *rmap_get_first(struct kvm_rmap_head *rmap_head,
1120                            struct rmap_iterator *iter)
1121 {
1122         u64 *sptep;
1123
1124         if (!rmap_head->val)
1125                 return NULL;
1126
1127         if (!(rmap_head->val & 1)) {
1128                 iter->desc = NULL;
1129                 sptep = (u64 *)rmap_head->val;
1130                 goto out;
1131         }
1132
1133         iter->desc = (struct pte_list_desc *)(rmap_head->val & ~1ul);
1134         iter->pos = 0;
1135         sptep = iter->desc->sptes[iter->pos];
1136 out:
1137         BUG_ON(!is_shadow_present_pte(*sptep));
1138         return sptep;
1139 }
1140
1141 /*
1142  * Must be used with a valid iterator: e.g. after rmap_get_first().
1143  *
1144  * Returns sptep if found, NULL otherwise.
1145  */
1146 static u64 *rmap_get_next(struct rmap_iterator *iter)
1147 {
1148         u64 *sptep;
1149
1150         if (iter->desc) {
1151                 if (iter->pos < PTE_LIST_EXT - 1) {
1152                         ++iter->pos;
1153                         sptep = iter->desc->sptes[iter->pos];
1154                         if (sptep)
1155                                 goto out;
1156                 }
1157
1158                 iter->desc = iter->desc->more;
1159
1160                 if (iter->desc) {
1161                         iter->pos = 0;
1162                         /* desc->sptes[0] cannot be NULL */
1163                         sptep = iter->desc->sptes[iter->pos];
1164                         goto out;
1165                 }
1166         }
1167
1168         return NULL;
1169 out:
1170         BUG_ON(!is_shadow_present_pte(*sptep));
1171         return sptep;
1172 }
1173
1174 #define for_each_rmap_spte(_rmap_head_, _iter_, _spte_)                 \
1175         for (_spte_ = rmap_get_first(_rmap_head_, _iter_);              \
1176              _spte_; _spte_ = rmap_get_next(_iter_))
1177
1178 static void drop_spte(struct kvm *kvm, u64 *sptep)
1179 {
1180         u64 old_spte = mmu_spte_clear_track_bits(kvm, sptep);
1181
1182         if (is_shadow_present_pte(old_spte))
1183                 rmap_remove(kvm, sptep);
1184 }
1185
1186
1187 static bool __drop_large_spte(struct kvm *kvm, u64 *sptep)
1188 {
1189         if (is_large_pte(*sptep)) {
1190                 WARN_ON(sptep_to_sp(sptep)->role.level == PG_LEVEL_4K);
1191                 drop_spte(kvm, sptep);
1192                 return true;
1193         }
1194
1195         return false;
1196 }
1197
1198 static void drop_large_spte(struct kvm_vcpu *vcpu, u64 *sptep)
1199 {
1200         if (__drop_large_spte(vcpu->kvm, sptep)) {
1201                 struct kvm_mmu_page *sp = sptep_to_sp(sptep);
1202
1203                 kvm_flush_remote_tlbs_with_address(vcpu->kvm, sp->gfn,
1204                         KVM_PAGES_PER_HPAGE(sp->role.level));
1205         }
1206 }
1207
1208 /*
1209  * Write-protect on the specified @sptep, @pt_protect indicates whether
1210  * spte write-protection is caused by protecting shadow page table.
1211  *
1212  * Note: write protection is difference between dirty logging and spte
1213  * protection:
1214  * - for dirty logging, the spte can be set to writable at anytime if
1215  *   its dirty bitmap is properly set.
1216  * - for spte protection, the spte can be writable only after unsync-ing
1217  *   shadow page.
1218  *
1219  * Return true if tlb need be flushed.
1220  */
1221 static bool spte_write_protect(u64 *sptep, bool pt_protect)
1222 {
1223         u64 spte = *sptep;
1224
1225         if (!is_writable_pte(spte) &&
1226               !(pt_protect && spte_can_locklessly_be_made_writable(spte)))
1227                 return false;
1228
1229         rmap_printk("spte %p %llx\n", sptep, *sptep);
1230
1231         if (pt_protect)
1232                 spte &= ~shadow_mmu_writable_mask;
1233         spte = spte & ~PT_WRITABLE_MASK;
1234
1235         return mmu_spte_update(sptep, spte);
1236 }
1237
1238 static bool __rmap_write_protect(struct kvm *kvm,
1239                                  struct kvm_rmap_head *rmap_head,
1240                                  bool pt_protect)
1241 {
1242         u64 *sptep;
1243         struct rmap_iterator iter;
1244         bool flush = false;
1245
1246         for_each_rmap_spte(rmap_head, &iter, sptep)
1247                 flush |= spte_write_protect(sptep, pt_protect);
1248
1249         return flush;
1250 }
1251
1252 static bool spte_clear_dirty(u64 *sptep)
1253 {
1254         u64 spte = *sptep;
1255
1256         rmap_printk("spte %p %llx\n", sptep, *sptep);
1257
1258         MMU_WARN_ON(!spte_ad_enabled(spte));
1259         spte &= ~shadow_dirty_mask;
1260         return mmu_spte_update(sptep, spte);
1261 }
1262
1263 static bool spte_wrprot_for_clear_dirty(u64 *sptep)
1264 {
1265         bool was_writable = test_and_clear_bit(PT_WRITABLE_SHIFT,
1266                                                (unsigned long *)sptep);
1267         if (was_writable && !spte_ad_enabled(*sptep))
1268                 kvm_set_pfn_dirty(spte_to_pfn(*sptep));
1269
1270         return was_writable;
1271 }
1272
1273 /*
1274  * Gets the GFN ready for another round of dirty logging by clearing the
1275  *      - D bit on ad-enabled SPTEs, and
1276  *      - W bit on ad-disabled SPTEs.
1277  * Returns true iff any D or W bits were cleared.
1278  */
1279 static bool __rmap_clear_dirty(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1280                                const struct kvm_memory_slot *slot)
1281 {
1282         u64 *sptep;
1283         struct rmap_iterator iter;
1284         bool flush = false;
1285
1286         for_each_rmap_spte(rmap_head, &iter, sptep)
1287                 if (spte_ad_need_write_protect(*sptep))
1288                         flush |= spte_wrprot_for_clear_dirty(sptep);
1289                 else
1290                         flush |= spte_clear_dirty(sptep);
1291
1292         return flush;
1293 }
1294
1295 /**
1296  * kvm_mmu_write_protect_pt_masked - write protect selected PT level pages
1297  * @kvm: kvm instance
1298  * @slot: slot to protect
1299  * @gfn_offset: start of the BITS_PER_LONG pages we care about
1300  * @mask: indicates which pages we should protect
1301  *
1302  * Used when we do not need to care about huge page mappings.
1303  */
1304 static void kvm_mmu_write_protect_pt_masked(struct kvm *kvm,
1305                                      struct kvm_memory_slot *slot,
1306                                      gfn_t gfn_offset, unsigned long mask)
1307 {
1308         struct kvm_rmap_head *rmap_head;
1309
1310         if (is_tdp_mmu_enabled(kvm))
1311                 kvm_tdp_mmu_clear_dirty_pt_masked(kvm, slot,
1312                                 slot->base_gfn + gfn_offset, mask, true);
1313
1314         if (!kvm_memslots_have_rmaps(kvm))
1315                 return;
1316
1317         while (mask) {
1318                 rmap_head = gfn_to_rmap(slot->base_gfn + gfn_offset + __ffs(mask),
1319                                         PG_LEVEL_4K, slot);
1320                 __rmap_write_protect(kvm, rmap_head, false);
1321
1322                 /* clear the first set bit */
1323                 mask &= mask - 1;
1324         }
1325 }
1326
1327 /**
1328  * kvm_mmu_clear_dirty_pt_masked - clear MMU D-bit for PT level pages, or write
1329  * protect the page if the D-bit isn't supported.
1330  * @kvm: kvm instance
1331  * @slot: slot to clear D-bit
1332  * @gfn_offset: start of the BITS_PER_LONG pages we care about
1333  * @mask: indicates which pages we should clear D-bit
1334  *
1335  * Used for PML to re-log the dirty GPAs after userspace querying dirty_bitmap.
1336  */
1337 static void kvm_mmu_clear_dirty_pt_masked(struct kvm *kvm,
1338                                          struct kvm_memory_slot *slot,
1339                                          gfn_t gfn_offset, unsigned long mask)
1340 {
1341         struct kvm_rmap_head *rmap_head;
1342
1343         if (is_tdp_mmu_enabled(kvm))
1344                 kvm_tdp_mmu_clear_dirty_pt_masked(kvm, slot,
1345                                 slot->base_gfn + gfn_offset, mask, false);
1346
1347         if (!kvm_memslots_have_rmaps(kvm))
1348                 return;
1349
1350         while (mask) {
1351                 rmap_head = gfn_to_rmap(slot->base_gfn + gfn_offset + __ffs(mask),
1352                                         PG_LEVEL_4K, slot);
1353                 __rmap_clear_dirty(kvm, rmap_head, slot);
1354
1355                 /* clear the first set bit */
1356                 mask &= mask - 1;
1357         }
1358 }
1359
1360 /**
1361  * kvm_arch_mmu_enable_log_dirty_pt_masked - enable dirty logging for selected
1362  * PT level pages.
1363  *
1364  * It calls kvm_mmu_write_protect_pt_masked to write protect selected pages to
1365  * enable dirty logging for them.
1366  *
1367  * We need to care about huge page mappings: e.g. during dirty logging we may
1368  * have such mappings.
1369  */
1370 void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm,
1371                                 struct kvm_memory_slot *slot,
1372                                 gfn_t gfn_offset, unsigned long mask)
1373 {
1374         /*
1375          * Huge pages are NOT write protected when we start dirty logging in
1376          * initially-all-set mode; must write protect them here so that they
1377          * are split to 4K on the first write.
1378          *
1379          * The gfn_offset is guaranteed to be aligned to 64, but the base_gfn
1380          * of memslot has no such restriction, so the range can cross two large
1381          * pages.
1382          */
1383         if (kvm_dirty_log_manual_protect_and_init_set(kvm)) {
1384                 gfn_t start = slot->base_gfn + gfn_offset + __ffs(mask);
1385                 gfn_t end = slot->base_gfn + gfn_offset + __fls(mask);
1386
1387                 kvm_mmu_slot_gfn_write_protect(kvm, slot, start, PG_LEVEL_2M);
1388
1389                 /* Cross two large pages? */
1390                 if (ALIGN(start << PAGE_SHIFT, PMD_SIZE) !=
1391                     ALIGN(end << PAGE_SHIFT, PMD_SIZE))
1392                         kvm_mmu_slot_gfn_write_protect(kvm, slot, end,
1393                                                        PG_LEVEL_2M);
1394         }
1395
1396         /* Now handle 4K PTEs.  */
1397         if (kvm_x86_ops.cpu_dirty_log_size)
1398                 kvm_mmu_clear_dirty_pt_masked(kvm, slot, gfn_offset, mask);
1399         else
1400                 kvm_mmu_write_protect_pt_masked(kvm, slot, gfn_offset, mask);
1401 }
1402
1403 int kvm_cpu_dirty_log_size(void)
1404 {
1405         return kvm_x86_ops.cpu_dirty_log_size;
1406 }
1407
1408 bool kvm_mmu_slot_gfn_write_protect(struct kvm *kvm,
1409                                     struct kvm_memory_slot *slot, u64 gfn,
1410                                     int min_level)
1411 {
1412         struct kvm_rmap_head *rmap_head;
1413         int i;
1414         bool write_protected = false;
1415
1416         if (kvm_memslots_have_rmaps(kvm)) {
1417                 for (i = min_level; i <= KVM_MAX_HUGEPAGE_LEVEL; ++i) {
1418                         rmap_head = gfn_to_rmap(gfn, i, slot);
1419                         write_protected |= __rmap_write_protect(kvm, rmap_head, true);
1420                 }
1421         }
1422
1423         if (is_tdp_mmu_enabled(kvm))
1424                 write_protected |=
1425                         kvm_tdp_mmu_write_protect_gfn(kvm, slot, gfn, min_level);
1426
1427         return write_protected;
1428 }
1429
1430 static bool rmap_write_protect(struct kvm_vcpu *vcpu, u64 gfn)
1431 {
1432         struct kvm_memory_slot *slot;
1433
1434         slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
1435         return kvm_mmu_slot_gfn_write_protect(vcpu->kvm, slot, gfn, PG_LEVEL_4K);
1436 }
1437
1438 static bool kvm_zap_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1439                           const struct kvm_memory_slot *slot)
1440 {
1441         return pte_list_destroy(kvm, rmap_head);
1442 }
1443
1444 static bool kvm_unmap_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1445                             struct kvm_memory_slot *slot, gfn_t gfn, int level,
1446                             pte_t unused)
1447 {
1448         return kvm_zap_rmapp(kvm, rmap_head, slot);
1449 }
1450
1451 static bool kvm_set_pte_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1452                               struct kvm_memory_slot *slot, gfn_t gfn, int level,
1453                               pte_t pte)
1454 {
1455         u64 *sptep;
1456         struct rmap_iterator iter;
1457         int need_flush = 0;
1458         u64 new_spte;
1459         kvm_pfn_t new_pfn;
1460
1461         WARN_ON(pte_huge(pte));
1462         new_pfn = pte_pfn(pte);
1463
1464 restart:
1465         for_each_rmap_spte(rmap_head, &iter, sptep) {
1466                 rmap_printk("spte %p %llx gfn %llx (%d)\n",
1467                             sptep, *sptep, gfn, level);
1468
1469                 need_flush = 1;
1470
1471                 if (pte_write(pte)) {
1472                         pte_list_remove(kvm, rmap_head, sptep);
1473                         goto restart;
1474                 } else {
1475                         new_spte = kvm_mmu_changed_pte_notifier_make_spte(
1476                                         *sptep, new_pfn);
1477
1478                         mmu_spte_clear_track_bits(kvm, sptep);
1479                         mmu_spte_set(sptep, new_spte);
1480                 }
1481         }
1482
1483         if (need_flush && kvm_available_flush_tlb_with_range()) {
1484                 kvm_flush_remote_tlbs_with_address(kvm, gfn, 1);
1485                 return 0;
1486         }
1487
1488         return need_flush;
1489 }
1490
1491 struct slot_rmap_walk_iterator {
1492         /* input fields. */
1493         const struct kvm_memory_slot *slot;
1494         gfn_t start_gfn;
1495         gfn_t end_gfn;
1496         int start_level;
1497         int end_level;
1498
1499         /* output fields. */
1500         gfn_t gfn;
1501         struct kvm_rmap_head *rmap;
1502         int level;
1503
1504         /* private field. */
1505         struct kvm_rmap_head *end_rmap;
1506 };
1507
1508 static void
1509 rmap_walk_init_level(struct slot_rmap_walk_iterator *iterator, int level)
1510 {
1511         iterator->level = level;
1512         iterator->gfn = iterator->start_gfn;
1513         iterator->rmap = gfn_to_rmap(iterator->gfn, level, iterator->slot);
1514         iterator->end_rmap = gfn_to_rmap(iterator->end_gfn, level, iterator->slot);
1515 }
1516
1517 static void
1518 slot_rmap_walk_init(struct slot_rmap_walk_iterator *iterator,
1519                     const struct kvm_memory_slot *slot, int start_level,
1520                     int end_level, gfn_t start_gfn, gfn_t end_gfn)
1521 {
1522         iterator->slot = slot;
1523         iterator->start_level = start_level;
1524         iterator->end_level = end_level;
1525         iterator->start_gfn = start_gfn;
1526         iterator->end_gfn = end_gfn;
1527
1528         rmap_walk_init_level(iterator, iterator->start_level);
1529 }
1530
1531 static bool slot_rmap_walk_okay(struct slot_rmap_walk_iterator *iterator)
1532 {
1533         return !!iterator->rmap;
1534 }
1535
1536 static void slot_rmap_walk_next(struct slot_rmap_walk_iterator *iterator)
1537 {
1538         if (++iterator->rmap <= iterator->end_rmap) {
1539                 iterator->gfn += (1UL << KVM_HPAGE_GFN_SHIFT(iterator->level));
1540                 return;
1541         }
1542
1543         if (++iterator->level > iterator->end_level) {
1544                 iterator->rmap = NULL;
1545                 return;
1546         }
1547
1548         rmap_walk_init_level(iterator, iterator->level);
1549 }
1550
1551 #define for_each_slot_rmap_range(_slot_, _start_level_, _end_level_,    \
1552            _start_gfn, _end_gfn, _iter_)                                \
1553         for (slot_rmap_walk_init(_iter_, _slot_, _start_level_,         \
1554                                  _end_level_, _start_gfn, _end_gfn);    \
1555              slot_rmap_walk_okay(_iter_);                               \
1556              slot_rmap_walk_next(_iter_))
1557
1558 typedef bool (*rmap_handler_t)(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1559                                struct kvm_memory_slot *slot, gfn_t gfn,
1560                                int level, pte_t pte);
1561
1562 static __always_inline bool kvm_handle_gfn_range(struct kvm *kvm,
1563                                                  struct kvm_gfn_range *range,
1564                                                  rmap_handler_t handler)
1565 {
1566         struct slot_rmap_walk_iterator iterator;
1567         bool ret = false;
1568
1569         for_each_slot_rmap_range(range->slot, PG_LEVEL_4K, KVM_MAX_HUGEPAGE_LEVEL,
1570                                  range->start, range->end - 1, &iterator)
1571                 ret |= handler(kvm, iterator.rmap, range->slot, iterator.gfn,
1572                                iterator.level, range->pte);
1573
1574         return ret;
1575 }
1576
1577 bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range)
1578 {
1579         bool flush = false;
1580
1581         if (kvm_memslots_have_rmaps(kvm))
1582                 flush = kvm_handle_gfn_range(kvm, range, kvm_unmap_rmapp);
1583
1584         if (is_tdp_mmu_enabled(kvm))
1585                 flush = kvm_tdp_mmu_unmap_gfn_range(kvm, range, flush);
1586
1587         return flush;
1588 }
1589
1590 bool kvm_set_spte_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
1591 {
1592         bool flush = false;
1593
1594         if (kvm_memslots_have_rmaps(kvm))
1595                 flush = kvm_handle_gfn_range(kvm, range, kvm_set_pte_rmapp);
1596
1597         if (is_tdp_mmu_enabled(kvm))
1598                 flush |= kvm_tdp_mmu_set_spte_gfn(kvm, range);
1599
1600         return flush;
1601 }
1602
1603 static bool kvm_age_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1604                           struct kvm_memory_slot *slot, gfn_t gfn, int level,
1605                           pte_t unused)
1606 {
1607         u64 *sptep;
1608         struct rmap_iterator iter;
1609         int young = 0;
1610
1611         for_each_rmap_spte(rmap_head, &iter, sptep)
1612                 young |= mmu_spte_age(sptep);
1613
1614         return young;
1615 }
1616
1617 static bool kvm_test_age_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1618                                struct kvm_memory_slot *slot, gfn_t gfn,
1619                                int level, pte_t unused)
1620 {
1621         u64 *sptep;
1622         struct rmap_iterator iter;
1623
1624         for_each_rmap_spte(rmap_head, &iter, sptep)
1625                 if (is_accessed_spte(*sptep))
1626                         return 1;
1627         return 0;
1628 }
1629
1630 #define RMAP_RECYCLE_THRESHOLD 1000
1631
1632 static void rmap_add(struct kvm_vcpu *vcpu, struct kvm_memory_slot *slot,
1633                      u64 *spte, gfn_t gfn)
1634 {
1635         struct kvm_mmu_page *sp;
1636         struct kvm_rmap_head *rmap_head;
1637         int rmap_count;
1638
1639         sp = sptep_to_sp(spte);
1640         kvm_mmu_page_set_gfn(sp, spte - sp->spt, gfn);
1641         rmap_head = gfn_to_rmap(gfn, sp->role.level, slot);
1642         rmap_count = pte_list_add(vcpu, spte, rmap_head);
1643
1644         if (rmap_count > RMAP_RECYCLE_THRESHOLD) {
1645                 kvm_unmap_rmapp(vcpu->kvm, rmap_head, NULL, gfn, sp->role.level, __pte(0));
1646                 kvm_flush_remote_tlbs_with_address(
1647                                 vcpu->kvm, sp->gfn, KVM_PAGES_PER_HPAGE(sp->role.level));
1648         }
1649 }
1650
1651 bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
1652 {
1653         bool young = false;
1654
1655         if (kvm_memslots_have_rmaps(kvm))
1656                 young = kvm_handle_gfn_range(kvm, range, kvm_age_rmapp);
1657
1658         if (is_tdp_mmu_enabled(kvm))
1659                 young |= kvm_tdp_mmu_age_gfn_range(kvm, range);
1660
1661         return young;
1662 }
1663
1664 bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
1665 {
1666         bool young = false;
1667
1668         if (kvm_memslots_have_rmaps(kvm))
1669                 young = kvm_handle_gfn_range(kvm, range, kvm_test_age_rmapp);
1670
1671         if (is_tdp_mmu_enabled(kvm))
1672                 young |= kvm_tdp_mmu_test_age_gfn(kvm, range);
1673
1674         return young;
1675 }
1676
1677 #ifdef MMU_DEBUG
1678 static int is_empty_shadow_page(u64 *spt)
1679 {
1680         u64 *pos;
1681         u64 *end;
1682
1683         for (pos = spt, end = pos + PAGE_SIZE / sizeof(u64); pos != end; pos++)
1684                 if (is_shadow_present_pte(*pos)) {
1685                         printk(KERN_ERR "%s: %p %llx\n", __func__,
1686                                pos, *pos);
1687                         return 0;
1688                 }
1689         return 1;
1690 }
1691 #endif
1692
1693 /*
1694  * This value is the sum of all of the kvm instances's
1695  * kvm->arch.n_used_mmu_pages values.  We need a global,
1696  * aggregate version in order to make the slab shrinker
1697  * faster
1698  */
1699 static inline void kvm_mod_used_mmu_pages(struct kvm *kvm, long nr)
1700 {
1701         kvm->arch.n_used_mmu_pages += nr;
1702         percpu_counter_add(&kvm_total_used_mmu_pages, nr);
1703 }
1704
1705 static void kvm_mmu_free_page(struct kvm_mmu_page *sp)
1706 {
1707         MMU_WARN_ON(!is_empty_shadow_page(sp->spt));
1708         hlist_del(&sp->hash_link);
1709         list_del(&sp->link);
1710         free_page((unsigned long)sp->spt);
1711         if (!sp->role.direct)
1712                 free_page((unsigned long)sp->gfns);
1713         kmem_cache_free(mmu_page_header_cache, sp);
1714 }
1715
1716 static unsigned kvm_page_table_hashfn(gfn_t gfn)
1717 {
1718         return hash_64(gfn, KVM_MMU_HASH_SHIFT);
1719 }
1720
1721 static void mmu_page_add_parent_pte(struct kvm_vcpu *vcpu,
1722                                     struct kvm_mmu_page *sp, u64 *parent_pte)
1723 {
1724         if (!parent_pte)
1725                 return;
1726
1727         pte_list_add(vcpu, parent_pte, &sp->parent_ptes);
1728 }
1729
1730 static void mmu_page_remove_parent_pte(struct kvm_mmu_page *sp,
1731                                        u64 *parent_pte)
1732 {
1733         __pte_list_remove(parent_pte, &sp->parent_ptes);
1734 }
1735
1736 static void drop_parent_pte(struct kvm_mmu_page *sp,
1737                             u64 *parent_pte)
1738 {
1739         mmu_page_remove_parent_pte(sp, parent_pte);
1740         mmu_spte_clear_no_track(parent_pte);
1741 }
1742
1743 static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu, int direct)
1744 {
1745         struct kvm_mmu_page *sp;
1746
1747         sp = kvm_mmu_memory_cache_alloc(&vcpu->arch.mmu_page_header_cache);
1748         sp->spt = kvm_mmu_memory_cache_alloc(&vcpu->arch.mmu_shadow_page_cache);
1749         if (!direct)
1750                 sp->gfns = kvm_mmu_memory_cache_alloc(&vcpu->arch.mmu_gfn_array_cache);
1751         set_page_private(virt_to_page(sp->spt), (unsigned long)sp);
1752
1753         /*
1754          * active_mmu_pages must be a FIFO list, as kvm_zap_obsolete_pages()
1755          * depends on valid pages being added to the head of the list.  See
1756          * comments in kvm_zap_obsolete_pages().
1757          */
1758         sp->mmu_valid_gen = vcpu->kvm->arch.mmu_valid_gen;
1759         list_add(&sp->link, &vcpu->kvm->arch.active_mmu_pages);
1760         kvm_mod_used_mmu_pages(vcpu->kvm, +1);
1761         return sp;
1762 }
1763
1764 static void mark_unsync(u64 *spte);
1765 static void kvm_mmu_mark_parents_unsync(struct kvm_mmu_page *sp)
1766 {
1767         u64 *sptep;
1768         struct rmap_iterator iter;
1769
1770         for_each_rmap_spte(&sp->parent_ptes, &iter, sptep) {
1771                 mark_unsync(sptep);
1772         }
1773 }
1774
1775 static void mark_unsync(u64 *spte)
1776 {
1777         struct kvm_mmu_page *sp;
1778         unsigned int index;
1779
1780         sp = sptep_to_sp(spte);
1781         index = spte - sp->spt;
1782         if (__test_and_set_bit(index, sp->unsync_child_bitmap))
1783                 return;
1784         if (sp->unsync_children++)
1785                 return;
1786         kvm_mmu_mark_parents_unsync(sp);
1787 }
1788
1789 static int nonpaging_sync_page(struct kvm_vcpu *vcpu,
1790                                struct kvm_mmu_page *sp)
1791 {
1792         return -1;
1793 }
1794
1795 #define KVM_PAGE_ARRAY_NR 16
1796
1797 struct kvm_mmu_pages {
1798         struct mmu_page_and_offset {
1799                 struct kvm_mmu_page *sp;
1800                 unsigned int idx;
1801         } page[KVM_PAGE_ARRAY_NR];
1802         unsigned int nr;
1803 };
1804
1805 static int mmu_pages_add(struct kvm_mmu_pages *pvec, struct kvm_mmu_page *sp,
1806                          int idx)
1807 {
1808         int i;
1809
1810         if (sp->unsync)
1811                 for (i=0; i < pvec->nr; i++)
1812                         if (pvec->page[i].sp == sp)
1813                                 return 0;
1814
1815         pvec->page[pvec->nr].sp = sp;
1816         pvec->page[pvec->nr].idx = idx;
1817         pvec->nr++;
1818         return (pvec->nr == KVM_PAGE_ARRAY_NR);
1819 }
1820
1821 static inline void clear_unsync_child_bit(struct kvm_mmu_page *sp, int idx)
1822 {
1823         --sp->unsync_children;
1824         WARN_ON((int)sp->unsync_children < 0);
1825         __clear_bit(idx, sp->unsync_child_bitmap);
1826 }
1827
1828 static int __mmu_unsync_walk(struct kvm_mmu_page *sp,
1829                            struct kvm_mmu_pages *pvec)
1830 {
1831         int i, ret, nr_unsync_leaf = 0;
1832
1833         for_each_set_bit(i, sp->unsync_child_bitmap, 512) {
1834                 struct kvm_mmu_page *child;
1835                 u64 ent = sp->spt[i];
1836
1837                 if (!is_shadow_present_pte(ent) || is_large_pte(ent)) {
1838                         clear_unsync_child_bit(sp, i);
1839                         continue;
1840                 }
1841
1842                 child = to_shadow_page(ent & PT64_BASE_ADDR_MASK);
1843
1844                 if (child->unsync_children) {
1845                         if (mmu_pages_add(pvec, child, i))
1846                                 return -ENOSPC;
1847
1848                         ret = __mmu_unsync_walk(child, pvec);
1849                         if (!ret) {
1850                                 clear_unsync_child_bit(sp, i);
1851                                 continue;
1852                         } else if (ret > 0) {
1853                                 nr_unsync_leaf += ret;
1854                         } else
1855                                 return ret;
1856                 } else if (child->unsync) {
1857                         nr_unsync_leaf++;
1858                         if (mmu_pages_add(pvec, child, i))
1859                                 return -ENOSPC;
1860                 } else
1861                         clear_unsync_child_bit(sp, i);
1862         }
1863
1864         return nr_unsync_leaf;
1865 }
1866
1867 #define INVALID_INDEX (-1)
1868
1869 static int mmu_unsync_walk(struct kvm_mmu_page *sp,
1870                            struct kvm_mmu_pages *pvec)
1871 {
1872         pvec->nr = 0;
1873         if (!sp->unsync_children)
1874                 return 0;
1875
1876         mmu_pages_add(pvec, sp, INVALID_INDEX);
1877         return __mmu_unsync_walk(sp, pvec);
1878 }
1879
1880 static void kvm_unlink_unsync_page(struct kvm *kvm, struct kvm_mmu_page *sp)
1881 {
1882         WARN_ON(!sp->unsync);
1883         trace_kvm_mmu_sync_page(sp);
1884         sp->unsync = 0;
1885         --kvm->stat.mmu_unsync;
1886 }
1887
1888 static bool kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp,
1889                                      struct list_head *invalid_list);
1890 static void kvm_mmu_commit_zap_page(struct kvm *kvm,
1891                                     struct list_head *invalid_list);
1892
1893 #define for_each_valid_sp(_kvm, _sp, _list)                             \
1894         hlist_for_each_entry(_sp, _list, hash_link)                     \
1895                 if (is_obsolete_sp((_kvm), (_sp))) {                    \
1896                 } else
1897
1898 #define for_each_gfn_indirect_valid_sp(_kvm, _sp, _gfn)                 \
1899         for_each_valid_sp(_kvm, _sp,                                    \
1900           &(_kvm)->arch.mmu_page_hash[kvm_page_table_hashfn(_gfn)])     \
1901                 if ((_sp)->gfn != (_gfn) || (_sp)->role.direct) {} else
1902
1903 static bool kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
1904                          struct list_head *invalid_list)
1905 {
1906         int ret = vcpu->arch.mmu->sync_page(vcpu, sp);
1907
1908         if (ret < 0) {
1909                 kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list);
1910                 return false;
1911         }
1912
1913         return !!ret;
1914 }
1915
1916 static bool kvm_mmu_remote_flush_or_zap(struct kvm *kvm,
1917                                         struct list_head *invalid_list,
1918                                         bool remote_flush)
1919 {
1920         if (!remote_flush && list_empty(invalid_list))
1921                 return false;
1922
1923         if (!list_empty(invalid_list))
1924                 kvm_mmu_commit_zap_page(kvm, invalid_list);
1925         else
1926                 kvm_flush_remote_tlbs(kvm);
1927         return true;
1928 }
1929
1930 #ifdef CONFIG_KVM_MMU_AUDIT
1931 #include "mmu_audit.c"
1932 #else
1933 static void kvm_mmu_audit(struct kvm_vcpu *vcpu, int point) { }
1934 static void mmu_audit_disable(void) { }
1935 #endif
1936
1937 static bool is_obsolete_sp(struct kvm *kvm, struct kvm_mmu_page *sp)
1938 {
1939         return sp->role.invalid ||
1940                unlikely(sp->mmu_valid_gen != kvm->arch.mmu_valid_gen);
1941 }
1942
1943 struct mmu_page_path {
1944         struct kvm_mmu_page *parent[PT64_ROOT_MAX_LEVEL];
1945         unsigned int idx[PT64_ROOT_MAX_LEVEL];
1946 };
1947
1948 #define for_each_sp(pvec, sp, parents, i)                       \
1949                 for (i = mmu_pages_first(&pvec, &parents);      \
1950                         i < pvec.nr && ({ sp = pvec.page[i].sp; 1;});   \
1951                         i = mmu_pages_next(&pvec, &parents, i))
1952
1953 static int mmu_pages_next(struct kvm_mmu_pages *pvec,
1954                           struct mmu_page_path *parents,
1955                           int i)
1956 {
1957         int n;
1958
1959         for (n = i+1; n < pvec->nr; n++) {
1960                 struct kvm_mmu_page *sp = pvec->page[n].sp;
1961                 unsigned idx = pvec->page[n].idx;
1962                 int level = sp->role.level;
1963
1964                 parents->idx[level-1] = idx;
1965                 if (level == PG_LEVEL_4K)
1966                         break;
1967
1968                 parents->parent[level-2] = sp;
1969         }
1970
1971         return n;
1972 }
1973
1974 static int mmu_pages_first(struct kvm_mmu_pages *pvec,
1975                            struct mmu_page_path *parents)
1976 {
1977         struct kvm_mmu_page *sp;
1978         int level;
1979
1980         if (pvec->nr == 0)
1981                 return 0;
1982
1983         WARN_ON(pvec->page[0].idx != INVALID_INDEX);
1984
1985         sp = pvec->page[0].sp;
1986         level = sp->role.level;
1987         WARN_ON(level == PG_LEVEL_4K);
1988
1989         parents->parent[level-2] = sp;
1990
1991         /* Also set up a sentinel.  Further entries in pvec are all
1992          * children of sp, so this element is never overwritten.
1993          */
1994         parents->parent[level-1] = NULL;
1995         return mmu_pages_next(pvec, parents, 0);
1996 }
1997
1998 static void mmu_pages_clear_parents(struct mmu_page_path *parents)
1999 {
2000         struct kvm_mmu_page *sp;
2001         unsigned int level = 0;
2002
2003         do {
2004                 unsigned int idx = parents->idx[level];
2005                 sp = parents->parent[level];
2006                 if (!sp)
2007                         return;
2008
2009                 WARN_ON(idx == INVALID_INDEX);
2010                 clear_unsync_child_bit(sp, idx);
2011                 level++;
2012         } while (!sp->unsync_children);
2013 }
2014
2015 static int mmu_sync_children(struct kvm_vcpu *vcpu,
2016                              struct kvm_mmu_page *parent, bool can_yield)
2017 {
2018         int i;
2019         struct kvm_mmu_page *sp;
2020         struct mmu_page_path parents;
2021         struct kvm_mmu_pages pages;
2022         LIST_HEAD(invalid_list);
2023         bool flush = false;
2024
2025         while (mmu_unsync_walk(parent, &pages)) {
2026                 bool protected = false;
2027
2028                 for_each_sp(pages, sp, parents, i)
2029                         protected |= rmap_write_protect(vcpu, sp->gfn);
2030
2031                 if (protected) {
2032                         kvm_mmu_remote_flush_or_zap(vcpu->kvm, &invalid_list, true);
2033                         flush = false;
2034                 }
2035
2036                 for_each_sp(pages, sp, parents, i) {
2037                         kvm_unlink_unsync_page(vcpu->kvm, sp);
2038                         flush |= kvm_sync_page(vcpu, sp, &invalid_list);
2039                         mmu_pages_clear_parents(&parents);
2040                 }
2041                 if (need_resched() || rwlock_needbreak(&vcpu->kvm->mmu_lock)) {
2042                         kvm_mmu_remote_flush_or_zap(vcpu->kvm, &invalid_list, flush);
2043                         if (!can_yield) {
2044                                 kvm_make_request(KVM_REQ_MMU_SYNC, vcpu);
2045                                 return -EINTR;
2046                         }
2047
2048                         cond_resched_rwlock_write(&vcpu->kvm->mmu_lock);
2049                         flush = false;
2050                 }
2051         }
2052
2053         kvm_mmu_remote_flush_or_zap(vcpu->kvm, &invalid_list, flush);
2054         return 0;
2055 }
2056
2057 static void __clear_sp_write_flooding_count(struct kvm_mmu_page *sp)
2058 {
2059         atomic_set(&sp->write_flooding_count,  0);
2060 }
2061
2062 static void clear_sp_write_flooding_count(u64 *spte)
2063 {
2064         __clear_sp_write_flooding_count(sptep_to_sp(spte));
2065 }
2066
2067 static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu,
2068                                              gfn_t gfn,
2069                                              gva_t gaddr,
2070                                              unsigned level,
2071                                              int direct,
2072                                              unsigned int access)
2073 {
2074         bool direct_mmu = vcpu->arch.mmu->direct_map;
2075         union kvm_mmu_page_role role;
2076         struct hlist_head *sp_list;
2077         unsigned quadrant;
2078         struct kvm_mmu_page *sp;
2079         int collisions = 0;
2080         LIST_HEAD(invalid_list);
2081
2082         role = vcpu->arch.mmu->mmu_role.base;
2083         role.level = level;
2084         role.direct = direct;
2085         if (role.direct)
2086                 role.gpte_is_8_bytes = true;
2087         role.access = access;
2088         if (!direct_mmu && vcpu->arch.mmu->root_level <= PT32_ROOT_LEVEL) {
2089                 quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level));
2090                 quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1;
2091                 role.quadrant = quadrant;
2092         }
2093
2094         sp_list = &vcpu->kvm->arch.mmu_page_hash[kvm_page_table_hashfn(gfn)];
2095         for_each_valid_sp(vcpu->kvm, sp, sp_list) {
2096                 if (sp->gfn != gfn) {
2097                         collisions++;
2098                         continue;
2099                 }
2100
2101                 if (sp->role.word != role.word) {
2102                         /*
2103                          * If the guest is creating an upper-level page, zap
2104                          * unsync pages for the same gfn.  While it's possible
2105                          * the guest is using recursive page tables, in all
2106                          * likelihood the guest has stopped using the unsync
2107                          * page and is installing a completely unrelated page.
2108                          * Unsync pages must not be left as is, because the new
2109                          * upper-level page will be write-protected.
2110                          */
2111                         if (level > PG_LEVEL_4K && sp->unsync)
2112                                 kvm_mmu_prepare_zap_page(vcpu->kvm, sp,
2113                                                          &invalid_list);
2114                         continue;
2115                 }
2116
2117                 if (direct_mmu)
2118                         goto trace_get_page;
2119
2120                 if (sp->unsync) {
2121                         /*
2122                          * The page is good, but is stale.  kvm_sync_page does
2123                          * get the latest guest state, but (unlike mmu_unsync_children)
2124                          * it doesn't write-protect the page or mark it synchronized!
2125                          * This way the validity of the mapping is ensured, but the
2126                          * overhead of write protection is not incurred until the
2127                          * guest invalidates the TLB mapping.  This allows multiple
2128                          * SPs for a single gfn to be unsync.
2129                          *
2130                          * If the sync fails, the page is zapped.  If so, break
2131                          * in order to rebuild it.
2132                          */
2133                         if (!kvm_sync_page(vcpu, sp, &invalid_list))
2134                                 break;
2135
2136                         WARN_ON(!list_empty(&invalid_list));
2137                         kvm_flush_remote_tlbs(vcpu->kvm);
2138                 }
2139
2140                 __clear_sp_write_flooding_count(sp);
2141
2142 trace_get_page:
2143                 trace_kvm_mmu_get_page(sp, false);
2144                 goto out;
2145         }
2146
2147         ++vcpu->kvm->stat.mmu_cache_miss;
2148
2149         sp = kvm_mmu_alloc_page(vcpu, direct);
2150
2151         sp->gfn = gfn;
2152         sp->role = role;
2153         hlist_add_head(&sp->hash_link, sp_list);
2154         if (!direct) {
2155                 account_shadowed(vcpu->kvm, sp);
2156                 if (level == PG_LEVEL_4K && rmap_write_protect(vcpu, gfn))
2157                         kvm_flush_remote_tlbs_with_address(vcpu->kvm, gfn, 1);
2158         }
2159         trace_kvm_mmu_get_page(sp, true);
2160 out:
2161         kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
2162
2163         if (collisions > vcpu->kvm->stat.max_mmu_page_hash_collisions)
2164                 vcpu->kvm->stat.max_mmu_page_hash_collisions = collisions;
2165         return sp;
2166 }
2167
2168 static void shadow_walk_init_using_root(struct kvm_shadow_walk_iterator *iterator,
2169                                         struct kvm_vcpu *vcpu, hpa_t root,
2170                                         u64 addr)
2171 {
2172         iterator->addr = addr;
2173         iterator->shadow_addr = root;
2174         iterator->level = vcpu->arch.mmu->shadow_root_level;
2175
2176         if (iterator->level >= PT64_ROOT_4LEVEL &&
2177             vcpu->arch.mmu->root_level < PT64_ROOT_4LEVEL &&
2178             !vcpu->arch.mmu->direct_map)
2179                 iterator->level = PT32E_ROOT_LEVEL;
2180
2181         if (iterator->level == PT32E_ROOT_LEVEL) {
2182                 /*
2183                  * prev_root is currently only used for 64-bit hosts. So only
2184                  * the active root_hpa is valid here.
2185                  */
2186                 BUG_ON(root != vcpu->arch.mmu->root_hpa);
2187
2188                 iterator->shadow_addr
2189                         = vcpu->arch.mmu->pae_root[(addr >> 30) & 3];
2190                 iterator->shadow_addr &= PT64_BASE_ADDR_MASK;
2191                 --iterator->level;
2192                 if (!iterator->shadow_addr)
2193                         iterator->level = 0;
2194         }
2195 }
2196
2197 static void shadow_walk_init(struct kvm_shadow_walk_iterator *iterator,
2198                              struct kvm_vcpu *vcpu, u64 addr)
2199 {
2200         shadow_walk_init_using_root(iterator, vcpu, vcpu->arch.mmu->root_hpa,
2201                                     addr);
2202 }
2203
2204 static bool shadow_walk_okay(struct kvm_shadow_walk_iterator *iterator)
2205 {
2206         if (iterator->level < PG_LEVEL_4K)
2207                 return false;
2208
2209         iterator->index = SHADOW_PT_INDEX(iterator->addr, iterator->level);
2210         iterator->sptep = ((u64 *)__va(iterator->shadow_addr)) + iterator->index;
2211         return true;
2212 }
2213
2214 static void __shadow_walk_next(struct kvm_shadow_walk_iterator *iterator,
2215                                u64 spte)
2216 {
2217         if (!is_shadow_present_pte(spte) || is_last_spte(spte, iterator->level)) {
2218                 iterator->level = 0;
2219                 return;
2220         }
2221
2222         iterator->shadow_addr = spte & PT64_BASE_ADDR_MASK;
2223         --iterator->level;
2224 }
2225
2226 static void shadow_walk_next(struct kvm_shadow_walk_iterator *iterator)
2227 {
2228         __shadow_walk_next(iterator, *iterator->sptep);
2229 }
2230
2231 static void link_shadow_page(struct kvm_vcpu *vcpu, u64 *sptep,
2232                              struct kvm_mmu_page *sp)
2233 {
2234         u64 spte;
2235
2236         BUILD_BUG_ON(VMX_EPT_WRITABLE_MASK != PT_WRITABLE_MASK);
2237
2238         spte = make_nonleaf_spte(sp->spt, sp_ad_disabled(sp));
2239
2240         mmu_spte_set(sptep, spte);
2241
2242         mmu_page_add_parent_pte(vcpu, sp, sptep);
2243
2244         if (sp->unsync_children || sp->unsync)
2245                 mark_unsync(sptep);
2246 }
2247
2248 static void validate_direct_spte(struct kvm_vcpu *vcpu, u64 *sptep,
2249                                    unsigned direct_access)
2250 {
2251         if (is_shadow_present_pte(*sptep) && !is_large_pte(*sptep)) {
2252                 struct kvm_mmu_page *child;
2253
2254                 /*
2255                  * For the direct sp, if the guest pte's dirty bit
2256                  * changed form clean to dirty, it will corrupt the
2257                  * sp's access: allow writable in the read-only sp,
2258                  * so we should update the spte at this point to get
2259                  * a new sp with the correct access.
2260                  */
2261                 child = to_shadow_page(*sptep & PT64_BASE_ADDR_MASK);
2262                 if (child->role.access == direct_access)
2263                         return;
2264
2265                 drop_parent_pte(child, sptep);
2266                 kvm_flush_remote_tlbs_with_address(vcpu->kvm, child->gfn, 1);
2267         }
2268 }
2269
2270 /* Returns the number of zapped non-leaf child shadow pages. */
2271 static int mmu_page_zap_pte(struct kvm *kvm, struct kvm_mmu_page *sp,
2272                             u64 *spte, struct list_head *invalid_list)
2273 {
2274         u64 pte;
2275         struct kvm_mmu_page *child;
2276
2277         pte = *spte;
2278         if (is_shadow_present_pte(pte)) {
2279                 if (is_last_spte(pte, sp->role.level)) {
2280                         drop_spte(kvm, spte);
2281                 } else {
2282                         child = to_shadow_page(pte & PT64_BASE_ADDR_MASK);
2283                         drop_parent_pte(child, spte);
2284
2285                         /*
2286                          * Recursively zap nested TDP SPs, parentless SPs are
2287                          * unlikely to be used again in the near future.  This
2288                          * avoids retaining a large number of stale nested SPs.
2289                          */
2290                         if (tdp_enabled && invalid_list &&
2291                             child->role.guest_mode && !child->parent_ptes.val)
2292                                 return kvm_mmu_prepare_zap_page(kvm, child,
2293                                                                 invalid_list);
2294                 }
2295         } else if (is_mmio_spte(pte)) {
2296                 mmu_spte_clear_no_track(spte);
2297         }
2298         return 0;
2299 }
2300
2301 static int kvm_mmu_page_unlink_children(struct kvm *kvm,
2302                                         struct kvm_mmu_page *sp,
2303                                         struct list_head *invalid_list)
2304 {
2305         int zapped = 0;
2306         unsigned i;
2307
2308         for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
2309                 zapped += mmu_page_zap_pte(kvm, sp, sp->spt + i, invalid_list);
2310
2311         return zapped;
2312 }
2313
2314 static void kvm_mmu_unlink_parents(struct kvm *kvm, struct kvm_mmu_page *sp)
2315 {
2316         u64 *sptep;
2317         struct rmap_iterator iter;
2318
2319         while ((sptep = rmap_get_first(&sp->parent_ptes, &iter)))
2320                 drop_parent_pte(sp, sptep);
2321 }
2322
2323 static int mmu_zap_unsync_children(struct kvm *kvm,
2324                                    struct kvm_mmu_page *parent,
2325                                    struct list_head *invalid_list)
2326 {
2327         int i, zapped = 0;
2328         struct mmu_page_path parents;
2329         struct kvm_mmu_pages pages;
2330
2331         if (parent->role.level == PG_LEVEL_4K)
2332                 return 0;
2333
2334         while (mmu_unsync_walk(parent, &pages)) {
2335                 struct kvm_mmu_page *sp;
2336
2337                 for_each_sp(pages, sp, parents, i) {
2338                         kvm_mmu_prepare_zap_page(kvm, sp, invalid_list);
2339                         mmu_pages_clear_parents(&parents);
2340                         zapped++;
2341                 }
2342         }
2343
2344         return zapped;
2345 }
2346
2347 static bool __kvm_mmu_prepare_zap_page(struct kvm *kvm,
2348                                        struct kvm_mmu_page *sp,
2349                                        struct list_head *invalid_list,
2350                                        int *nr_zapped)
2351 {
2352         bool list_unstable;
2353
2354         trace_kvm_mmu_prepare_zap_page(sp);
2355         ++kvm->stat.mmu_shadow_zapped;
2356         *nr_zapped = mmu_zap_unsync_children(kvm, sp, invalid_list);
2357         *nr_zapped += kvm_mmu_page_unlink_children(kvm, sp, invalid_list);
2358         kvm_mmu_unlink_parents(kvm, sp);
2359
2360         /* Zapping children means active_mmu_pages has become unstable. */
2361         list_unstable = *nr_zapped;
2362
2363         if (!sp->role.invalid && !sp->role.direct)
2364                 unaccount_shadowed(kvm, sp);
2365
2366         if (sp->unsync)
2367                 kvm_unlink_unsync_page(kvm, sp);
2368         if (!sp->root_count) {
2369                 /* Count self */
2370                 (*nr_zapped)++;
2371
2372                 /*
2373                  * Already invalid pages (previously active roots) are not on
2374                  * the active page list.  See list_del() in the "else" case of
2375                  * !sp->root_count.
2376                  */
2377                 if (sp->role.invalid)
2378                         list_add(&sp->link, invalid_list);
2379                 else
2380                         list_move(&sp->link, invalid_list);
2381                 kvm_mod_used_mmu_pages(kvm, -1);
2382         } else {
2383                 /*
2384                  * Remove the active root from the active page list, the root
2385                  * will be explicitly freed when the root_count hits zero.
2386                  */
2387                 list_del(&sp->link);
2388
2389                 /*
2390                  * Obsolete pages cannot be used on any vCPUs, see the comment
2391                  * in kvm_mmu_zap_all_fast().  Note, is_obsolete_sp() also
2392                  * treats invalid shadow pages as being obsolete.
2393                  */
2394                 if (!is_obsolete_sp(kvm, sp))
2395                         kvm_reload_remote_mmus(kvm);
2396         }
2397
2398         if (sp->lpage_disallowed)
2399                 unaccount_huge_nx_page(kvm, sp);
2400
2401         sp->role.invalid = 1;
2402         return list_unstable;
2403 }
2404
2405 static bool kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp,
2406                                      struct list_head *invalid_list)
2407 {
2408         int nr_zapped;
2409
2410         __kvm_mmu_prepare_zap_page(kvm, sp, invalid_list, &nr_zapped);
2411         return nr_zapped;
2412 }
2413
2414 static void kvm_mmu_commit_zap_page(struct kvm *kvm,
2415                                     struct list_head *invalid_list)
2416 {
2417         struct kvm_mmu_page *sp, *nsp;
2418
2419         if (list_empty(invalid_list))
2420                 return;
2421
2422         /*
2423          * We need to make sure everyone sees our modifications to
2424          * the page tables and see changes to vcpu->mode here. The barrier
2425          * in the kvm_flush_remote_tlbs() achieves this. This pairs
2426          * with vcpu_enter_guest and walk_shadow_page_lockless_begin/end.
2427          *
2428          * In addition, kvm_flush_remote_tlbs waits for all vcpus to exit
2429          * guest mode and/or lockless shadow page table walks.
2430          */
2431         kvm_flush_remote_tlbs(kvm);
2432
2433         list_for_each_entry_safe(sp, nsp, invalid_list, link) {
2434                 WARN_ON(!sp->role.invalid || sp->root_count);
2435                 kvm_mmu_free_page(sp);
2436         }
2437 }
2438
2439 static unsigned long kvm_mmu_zap_oldest_mmu_pages(struct kvm *kvm,
2440                                                   unsigned long nr_to_zap)
2441 {
2442         unsigned long total_zapped = 0;
2443         struct kvm_mmu_page *sp, *tmp;
2444         LIST_HEAD(invalid_list);
2445         bool unstable;
2446         int nr_zapped;
2447
2448         if (list_empty(&kvm->arch.active_mmu_pages))
2449                 return 0;
2450
2451 restart:
2452         list_for_each_entry_safe_reverse(sp, tmp, &kvm->arch.active_mmu_pages, link) {
2453                 /*
2454                  * Don't zap active root pages, the page itself can't be freed
2455                  * and zapping it will just force vCPUs to realloc and reload.
2456                  */
2457                 if (sp->root_count)
2458                         continue;
2459
2460                 unstable = __kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list,
2461                                                       &nr_zapped);
2462                 total_zapped += nr_zapped;
2463                 if (total_zapped >= nr_to_zap)
2464                         break;
2465
2466                 if (unstable)
2467                         goto restart;
2468         }
2469
2470         kvm_mmu_commit_zap_page(kvm, &invalid_list);
2471
2472         kvm->stat.mmu_recycled += total_zapped;
2473         return total_zapped;
2474 }
2475
2476 static inline unsigned long kvm_mmu_available_pages(struct kvm *kvm)
2477 {
2478         if (kvm->arch.n_max_mmu_pages > kvm->arch.n_used_mmu_pages)
2479                 return kvm->arch.n_max_mmu_pages -
2480                         kvm->arch.n_used_mmu_pages;
2481
2482         return 0;
2483 }
2484
2485 static int make_mmu_pages_available(struct kvm_vcpu *vcpu)
2486 {
2487         unsigned long avail = kvm_mmu_available_pages(vcpu->kvm);
2488
2489         if (likely(avail >= KVM_MIN_FREE_MMU_PAGES))
2490                 return 0;
2491
2492         kvm_mmu_zap_oldest_mmu_pages(vcpu->kvm, KVM_REFILL_PAGES - avail);
2493
2494         /*
2495          * Note, this check is intentionally soft, it only guarantees that one
2496          * page is available, while the caller may end up allocating as many as
2497          * four pages, e.g. for PAE roots or for 5-level paging.  Temporarily
2498          * exceeding the (arbitrary by default) limit will not harm the host,
2499          * being too aggressive may unnecessarily kill the guest, and getting an
2500          * exact count is far more trouble than it's worth, especially in the
2501          * page fault paths.
2502          */
2503         if (!kvm_mmu_available_pages(vcpu->kvm))
2504                 return -ENOSPC;
2505         return 0;
2506 }
2507
2508 /*
2509  * Changing the number of mmu pages allocated to the vm
2510  * Note: if goal_nr_mmu_pages is too small, you will get dead lock
2511  */
2512 void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned long goal_nr_mmu_pages)
2513 {
2514         write_lock(&kvm->mmu_lock);
2515
2516         if (kvm->arch.n_used_mmu_pages > goal_nr_mmu_pages) {
2517                 kvm_mmu_zap_oldest_mmu_pages(kvm, kvm->arch.n_used_mmu_pages -
2518                                                   goal_nr_mmu_pages);
2519
2520                 goal_nr_mmu_pages = kvm->arch.n_used_mmu_pages;
2521         }
2522
2523         kvm->arch.n_max_mmu_pages = goal_nr_mmu_pages;
2524
2525         write_unlock(&kvm->mmu_lock);
2526 }
2527
2528 int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn)
2529 {
2530         struct kvm_mmu_page *sp;
2531         LIST_HEAD(invalid_list);
2532         int r;
2533
2534         pgprintk("%s: looking for gfn %llx\n", __func__, gfn);
2535         r = 0;
2536         write_lock(&kvm->mmu_lock);
2537         for_each_gfn_indirect_valid_sp(kvm, sp, gfn) {
2538                 pgprintk("%s: gfn %llx role %x\n", __func__, gfn,
2539                          sp->role.word);
2540                 r = 1;
2541                 kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list);
2542         }
2543         kvm_mmu_commit_zap_page(kvm, &invalid_list);
2544         write_unlock(&kvm->mmu_lock);
2545
2546         return r;
2547 }
2548
2549 static int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva)
2550 {
2551         gpa_t gpa;
2552         int r;
2553
2554         if (vcpu->arch.mmu->direct_map)
2555                 return 0;
2556
2557         gpa = kvm_mmu_gva_to_gpa_read(vcpu, gva, NULL);
2558
2559         r = kvm_mmu_unprotect_page(vcpu->kvm, gpa >> PAGE_SHIFT);
2560
2561         return r;
2562 }
2563
2564 static void kvm_unsync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
2565 {
2566         trace_kvm_mmu_unsync_page(sp);
2567         ++vcpu->kvm->stat.mmu_unsync;
2568         sp->unsync = 1;
2569
2570         kvm_mmu_mark_parents_unsync(sp);
2571 }
2572
2573 /*
2574  * Attempt to unsync any shadow pages that can be reached by the specified gfn,
2575  * KVM is creating a writable mapping for said gfn.  Returns 0 if all pages
2576  * were marked unsync (or if there is no shadow page), -EPERM if the SPTE must
2577  * be write-protected.
2578  */
2579 int mmu_try_to_unsync_pages(struct kvm_vcpu *vcpu, struct kvm_memory_slot *slot,
2580                             gfn_t gfn, bool can_unsync, bool prefetch)
2581 {
2582         struct kvm_mmu_page *sp;
2583         bool locked = false;
2584
2585         /*
2586          * Force write-protection if the page is being tracked.  Note, the page
2587          * track machinery is used to write-protect upper-level shadow pages,
2588          * i.e. this guards the role.level == 4K assertion below!
2589          */
2590         if (kvm_slot_page_track_is_active(vcpu, slot, gfn, KVM_PAGE_TRACK_WRITE))
2591                 return -EPERM;
2592
2593         /*
2594          * The page is not write-tracked, mark existing shadow pages unsync
2595          * unless KVM is synchronizing an unsync SP (can_unsync = false).  In
2596          * that case, KVM must complete emulation of the guest TLB flush before
2597          * allowing shadow pages to become unsync (writable by the guest).
2598          */
2599         for_each_gfn_indirect_valid_sp(vcpu->kvm, sp, gfn) {
2600                 if (!can_unsync)
2601                         return -EPERM;
2602
2603                 if (sp->unsync)
2604                         continue;
2605
2606                 if (prefetch)
2607                         return -EEXIST;
2608
2609                 /*
2610                  * TDP MMU page faults require an additional spinlock as they
2611                  * run with mmu_lock held for read, not write, and the unsync
2612                  * logic is not thread safe.  Take the spinklock regardless of
2613                  * the MMU type to avoid extra conditionals/parameters, there's
2614                  * no meaningful penalty if mmu_lock is held for write.
2615                  */
2616                 if (!locked) {
2617                         locked = true;
2618                         spin_lock(&vcpu->kvm->arch.mmu_unsync_pages_lock);
2619
2620                         /*
2621                          * Recheck after taking the spinlock, a different vCPU
2622                          * may have since marked the page unsync.  A false
2623                          * positive on the unprotected check above is not
2624                          * possible as clearing sp->unsync _must_ hold mmu_lock
2625                          * for write, i.e. unsync cannot transition from 0->1
2626                          * while this CPU holds mmu_lock for read (or write).
2627                          */
2628                         if (READ_ONCE(sp->unsync))
2629                                 continue;
2630                 }
2631
2632                 WARN_ON(sp->role.level != PG_LEVEL_4K);
2633                 kvm_unsync_page(vcpu, sp);
2634         }
2635         if (locked)
2636                 spin_unlock(&vcpu->kvm->arch.mmu_unsync_pages_lock);
2637
2638         /*
2639          * We need to ensure that the marking of unsync pages is visible
2640          * before the SPTE is updated to allow writes because
2641          * kvm_mmu_sync_roots() checks the unsync flags without holding
2642          * the MMU lock and so can race with this. If the SPTE was updated
2643          * before the page had been marked as unsync-ed, something like the
2644          * following could happen:
2645          *
2646          * CPU 1                    CPU 2
2647          * ---------------------------------------------------------------------
2648          * 1.2 Host updates SPTE
2649          *     to be writable
2650          *                      2.1 Guest writes a GPTE for GVA X.
2651          *                          (GPTE being in the guest page table shadowed
2652          *                           by the SP from CPU 1.)
2653          *                          This reads SPTE during the page table walk.
2654          *                          Since SPTE.W is read as 1, there is no
2655          *                          fault.
2656          *
2657          *                      2.2 Guest issues TLB flush.
2658          *                          That causes a VM Exit.
2659          *
2660          *                      2.3 Walking of unsync pages sees sp->unsync is
2661          *                          false and skips the page.
2662          *
2663          *                      2.4 Guest accesses GVA X.
2664          *                          Since the mapping in the SP was not updated,
2665          *                          so the old mapping for GVA X incorrectly
2666          *                          gets used.
2667          * 1.1 Host marks SP
2668          *     as unsync
2669          *     (sp->unsync = true)
2670          *
2671          * The write barrier below ensures that 1.1 happens before 1.2 and thus
2672          * the situation in 2.4 does not arise.  It pairs with the read barrier
2673          * in is_unsync_root(), placed between 2.1's load of SPTE.W and 2.3.
2674          */
2675         smp_wmb();
2676
2677         return 0;
2678 }
2679
2680 static int mmu_set_spte(struct kvm_vcpu *vcpu, struct kvm_memory_slot *slot,
2681                         u64 *sptep, unsigned int pte_access, gfn_t gfn,
2682                         kvm_pfn_t pfn, struct kvm_page_fault *fault)
2683 {
2684         struct kvm_mmu_page *sp = sptep_to_sp(sptep);
2685         int level = sp->role.level;
2686         int was_rmapped = 0;
2687         int ret = RET_PF_FIXED;
2688         bool flush = false;
2689         bool wrprot;
2690         u64 spte;
2691
2692         /* Prefetching always gets a writable pfn.  */
2693         bool host_writable = !fault || fault->map_writable;
2694         bool prefetch = !fault || fault->prefetch;
2695         bool write_fault = fault && fault->write;
2696
2697         pgprintk("%s: spte %llx write_fault %d gfn %llx\n", __func__,
2698                  *sptep, write_fault, gfn);
2699
2700         if (unlikely(is_noslot_pfn(pfn))) {
2701                 mark_mmio_spte(vcpu, sptep, gfn, pte_access);
2702                 return RET_PF_EMULATE;
2703         }
2704
2705         if (is_shadow_present_pte(*sptep)) {
2706                 /*
2707                  * If we overwrite a PTE page pointer with a 2MB PMD, unlink
2708                  * the parent of the now unreachable PTE.
2709                  */
2710                 if (level > PG_LEVEL_4K && !is_large_pte(*sptep)) {
2711                         struct kvm_mmu_page *child;
2712                         u64 pte = *sptep;
2713
2714                         child = to_shadow_page(pte & PT64_BASE_ADDR_MASK);
2715                         drop_parent_pte(child, sptep);
2716                         flush = true;
2717                 } else if (pfn != spte_to_pfn(*sptep)) {
2718                         pgprintk("hfn old %llx new %llx\n",
2719                                  spte_to_pfn(*sptep), pfn);
2720                         drop_spte(vcpu->kvm, sptep);
2721                         flush = true;
2722                 } else
2723                         was_rmapped = 1;
2724         }
2725
2726         wrprot = make_spte(vcpu, sp, slot, pte_access, gfn, pfn, *sptep, prefetch,
2727                            true, host_writable, &spte);
2728
2729         if (*sptep == spte) {
2730                 ret = RET_PF_SPURIOUS;
2731         } else {
2732                 trace_kvm_mmu_set_spte(level, gfn, sptep);
2733                 flush |= mmu_spte_update(sptep, spte);
2734         }
2735
2736         if (wrprot) {
2737                 if (write_fault)
2738                         ret = RET_PF_EMULATE;
2739         }
2740
2741         if (flush)
2742                 kvm_flush_remote_tlbs_with_address(vcpu->kvm, gfn,
2743                                 KVM_PAGES_PER_HPAGE(level));
2744
2745         pgprintk("%s: setting spte %llx\n", __func__, *sptep);
2746
2747         if (!was_rmapped) {
2748                 WARN_ON_ONCE(ret == RET_PF_SPURIOUS);
2749                 kvm_update_page_stats(vcpu->kvm, level, 1);
2750                 rmap_add(vcpu, slot, sptep, gfn);
2751         }
2752
2753         return ret;
2754 }
2755
2756 static int direct_pte_prefetch_many(struct kvm_vcpu *vcpu,
2757                                     struct kvm_mmu_page *sp,
2758                                     u64 *start, u64 *end)
2759 {
2760         struct page *pages[PTE_PREFETCH_NUM];
2761         struct kvm_memory_slot *slot;
2762         unsigned int access = sp->role.access;
2763         int i, ret;
2764         gfn_t gfn;
2765
2766         gfn = kvm_mmu_page_get_gfn(sp, start - sp->spt);
2767         slot = gfn_to_memslot_dirty_bitmap(vcpu, gfn, access & ACC_WRITE_MASK);
2768         if (!slot)
2769                 return -1;
2770
2771         ret = gfn_to_page_many_atomic(slot, gfn, pages, end - start);
2772         if (ret <= 0)
2773                 return -1;
2774
2775         for (i = 0; i < ret; i++, gfn++, start++) {
2776                 mmu_set_spte(vcpu, slot, start, access, gfn,
2777                              page_to_pfn(pages[i]), NULL);
2778                 put_page(pages[i]);
2779         }
2780
2781         return 0;
2782 }
2783
2784 static void __direct_pte_prefetch(struct kvm_vcpu *vcpu,
2785                                   struct kvm_mmu_page *sp, u64 *sptep)
2786 {
2787         u64 *spte, *start = NULL;
2788         int i;
2789
2790         WARN_ON(!sp->role.direct);
2791
2792         i = (sptep - sp->spt) & ~(PTE_PREFETCH_NUM - 1);
2793         spte = sp->spt + i;
2794
2795         for (i = 0; i < PTE_PREFETCH_NUM; i++, spte++) {
2796                 if (is_shadow_present_pte(*spte) || spte == sptep) {
2797                         if (!start)
2798                                 continue;
2799                         if (direct_pte_prefetch_many(vcpu, sp, start, spte) < 0)
2800                                 return;
2801                         start = NULL;
2802                 } else if (!start)
2803                         start = spte;
2804         }
2805         if (start)
2806                 direct_pte_prefetch_many(vcpu, sp, start, spte);
2807 }
2808
2809 static void direct_pte_prefetch(struct kvm_vcpu *vcpu, u64 *sptep)
2810 {
2811         struct kvm_mmu_page *sp;
2812
2813         sp = sptep_to_sp(sptep);
2814
2815         /*
2816          * Without accessed bits, there's no way to distinguish between
2817          * actually accessed translations and prefetched, so disable pte
2818          * prefetch if accessed bits aren't available.
2819          */
2820         if (sp_ad_disabled(sp))
2821                 return;
2822
2823         if (sp->role.level > PG_LEVEL_4K)
2824                 return;
2825
2826         /*
2827          * If addresses are being invalidated, skip prefetching to avoid
2828          * accidentally prefetching those addresses.
2829          */
2830         if (unlikely(vcpu->kvm->mmu_notifier_count))
2831                 return;
2832
2833         __direct_pte_prefetch(vcpu, sp, sptep);
2834 }
2835
2836 static int host_pfn_mapping_level(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn,
2837                                   const struct kvm_memory_slot *slot)
2838 {
2839         unsigned long hva;
2840         pte_t *pte;
2841         int level;
2842
2843         if (!PageCompound(pfn_to_page(pfn)) && !kvm_is_zone_device_pfn(pfn))
2844                 return PG_LEVEL_4K;
2845
2846         /*
2847          * Note, using the already-retrieved memslot and __gfn_to_hva_memslot()
2848          * is not solely for performance, it's also necessary to avoid the
2849          * "writable" check in __gfn_to_hva_many(), which will always fail on
2850          * read-only memslots due to gfn_to_hva() assuming writes.  Earlier
2851          * page fault steps have already verified the guest isn't writing a
2852          * read-only memslot.
2853          */
2854         hva = __gfn_to_hva_memslot(slot, gfn);
2855
2856         pte = lookup_address_in_mm(kvm->mm, hva, &level);
2857         if (unlikely(!pte))
2858                 return PG_LEVEL_4K;
2859
2860         return level;
2861 }
2862
2863 int kvm_mmu_max_mapping_level(struct kvm *kvm,
2864                               const struct kvm_memory_slot *slot, gfn_t gfn,
2865                               kvm_pfn_t pfn, int max_level)
2866 {
2867         struct kvm_lpage_info *linfo;
2868         int host_level;
2869
2870         max_level = min(max_level, max_huge_page_level);
2871         for ( ; max_level > PG_LEVEL_4K; max_level--) {
2872                 linfo = lpage_info_slot(gfn, slot, max_level);
2873                 if (!linfo->disallow_lpage)
2874                         break;
2875         }
2876
2877         if (max_level == PG_LEVEL_4K)
2878                 return PG_LEVEL_4K;
2879
2880         host_level = host_pfn_mapping_level(kvm, gfn, pfn, slot);
2881         return min(host_level, max_level);
2882 }
2883
2884 void kvm_mmu_hugepage_adjust(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
2885 {
2886         struct kvm_memory_slot *slot = fault->slot;
2887         kvm_pfn_t mask;
2888
2889         fault->huge_page_disallowed = fault->exec && fault->nx_huge_page_workaround_enabled;
2890
2891         if (unlikely(fault->max_level == PG_LEVEL_4K))
2892                 return;
2893
2894         if (is_error_noslot_pfn(fault->pfn) || kvm_is_reserved_pfn(fault->pfn))
2895                 return;
2896
2897         if (kvm_slot_dirty_track_enabled(slot))
2898                 return;
2899
2900         /*
2901          * Enforce the iTLB multihit workaround after capturing the requested
2902          * level, which will be used to do precise, accurate accounting.
2903          */
2904         fault->req_level = kvm_mmu_max_mapping_level(vcpu->kvm, slot,
2905                                                      fault->gfn, fault->pfn,
2906                                                      fault->max_level);
2907         if (fault->req_level == PG_LEVEL_4K || fault->huge_page_disallowed)
2908                 return;
2909
2910         /*
2911          * mmu_notifier_retry() was successful and mmu_lock is held, so
2912          * the pmd can't be split from under us.
2913          */
2914         fault->goal_level = fault->req_level;
2915         mask = KVM_PAGES_PER_HPAGE(fault->goal_level) - 1;
2916         VM_BUG_ON((fault->gfn & mask) != (fault->pfn & mask));
2917         fault->pfn &= ~mask;
2918 }
2919
2920 void disallowed_hugepage_adjust(struct kvm_page_fault *fault, u64 spte, int cur_level)
2921 {
2922         if (cur_level > PG_LEVEL_4K &&
2923             cur_level == fault->goal_level &&
2924             is_shadow_present_pte(spte) &&
2925             !is_large_pte(spte)) {
2926                 /*
2927                  * A small SPTE exists for this pfn, but FNAME(fetch)
2928                  * and __direct_map would like to create a large PTE
2929                  * instead: just force them to go down another level,
2930                  * patching back for them into pfn the next 9 bits of
2931                  * the address.
2932                  */
2933                 u64 page_mask = KVM_PAGES_PER_HPAGE(cur_level) -
2934                                 KVM_PAGES_PER_HPAGE(cur_level - 1);
2935                 fault->pfn |= fault->gfn & page_mask;
2936                 fault->goal_level--;
2937         }
2938 }
2939
2940 static int __direct_map(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
2941 {
2942         struct kvm_shadow_walk_iterator it;
2943         struct kvm_mmu_page *sp;
2944         int ret;
2945         gfn_t base_gfn = fault->gfn;
2946
2947         kvm_mmu_hugepage_adjust(vcpu, fault);
2948
2949         trace_kvm_mmu_spte_requested(fault);
2950         for_each_shadow_entry(vcpu, fault->addr, it) {
2951                 /*
2952                  * We cannot overwrite existing page tables with an NX
2953                  * large page, as the leaf could be executable.
2954                  */
2955                 if (fault->nx_huge_page_workaround_enabled)
2956                         disallowed_hugepage_adjust(fault, *it.sptep, it.level);
2957
2958                 base_gfn = fault->gfn & ~(KVM_PAGES_PER_HPAGE(it.level) - 1);
2959                 if (it.level == fault->goal_level)
2960                         break;
2961
2962                 drop_large_spte(vcpu, it.sptep);
2963                 if (is_shadow_present_pte(*it.sptep))
2964                         continue;
2965
2966                 sp = kvm_mmu_get_page(vcpu, base_gfn, it.addr,
2967                                       it.level - 1, true, ACC_ALL);
2968
2969                 link_shadow_page(vcpu, it.sptep, sp);
2970                 if (fault->is_tdp && fault->huge_page_disallowed &&
2971                     fault->req_level >= it.level)
2972                         account_huge_nx_page(vcpu->kvm, sp);
2973         }
2974
2975         if (WARN_ON_ONCE(it.level != fault->goal_level))
2976                 return -EFAULT;
2977
2978         ret = mmu_set_spte(vcpu, fault->slot, it.sptep, ACC_ALL,
2979                            base_gfn, fault->pfn, fault);
2980         if (ret == RET_PF_SPURIOUS)
2981                 return ret;
2982
2983         direct_pte_prefetch(vcpu, it.sptep);
2984         ++vcpu->stat.pf_fixed;
2985         return ret;
2986 }
2987
2988 static void kvm_send_hwpoison_signal(unsigned long address, struct task_struct *tsk)
2989 {
2990         send_sig_mceerr(BUS_MCEERR_AR, (void __user *)address, PAGE_SHIFT, tsk);
2991 }
2992
2993 static int kvm_handle_bad_page(struct kvm_vcpu *vcpu, gfn_t gfn, kvm_pfn_t pfn)
2994 {
2995         /*
2996          * Do not cache the mmio info caused by writing the readonly gfn
2997          * into the spte otherwise read access on readonly gfn also can
2998          * caused mmio page fault and treat it as mmio access.
2999          */
3000         if (pfn == KVM_PFN_ERR_RO_FAULT)
3001                 return RET_PF_EMULATE;
3002
3003         if (pfn == KVM_PFN_ERR_HWPOISON) {
3004                 kvm_send_hwpoison_signal(kvm_vcpu_gfn_to_hva(vcpu, gfn), current);
3005                 return RET_PF_RETRY;
3006         }
3007
3008         return -EFAULT;
3009 }
3010
3011 static bool handle_abnormal_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault,
3012                                 unsigned int access, int *ret_val)
3013 {
3014         /* The pfn is invalid, report the error! */
3015         if (unlikely(is_error_pfn(fault->pfn))) {
3016                 *ret_val = kvm_handle_bad_page(vcpu, fault->gfn, fault->pfn);
3017                 return true;
3018         }
3019
3020         if (unlikely(!fault->slot)) {
3021                 gva_t gva = fault->is_tdp ? 0 : fault->addr;
3022
3023                 vcpu_cache_mmio_info(vcpu, gva, fault->gfn,
3024                                      access & shadow_mmio_access_mask);
3025                 /*
3026                  * If MMIO caching is disabled, emulate immediately without
3027                  * touching the shadow page tables as attempting to install an
3028                  * MMIO SPTE will just be an expensive nop.
3029                  */
3030                 if (unlikely(!shadow_mmio_value)) {
3031                         *ret_val = RET_PF_EMULATE;
3032                         return true;
3033                 }
3034         }
3035
3036         return false;
3037 }
3038
3039 static bool page_fault_can_be_fast(struct kvm_page_fault *fault)
3040 {
3041         /*
3042          * Do not fix the mmio spte with invalid generation number which
3043          * need to be updated by slow page fault path.
3044          */
3045         if (fault->rsvd)
3046                 return false;
3047
3048         /* See if the page fault is due to an NX violation */
3049         if (unlikely(fault->exec && fault->present))
3050                 return false;
3051
3052         /*
3053          * #PF can be fast if:
3054          * 1. The shadow page table entry is not present, which could mean that
3055          *    the fault is potentially caused by access tracking (if enabled).
3056          * 2. The shadow page table entry is present and the fault
3057          *    is caused by write-protect, that means we just need change the W
3058          *    bit of the spte which can be done out of mmu-lock.
3059          *
3060          * However, if access tracking is disabled we know that a non-present
3061          * page must be a genuine page fault where we have to create a new SPTE.
3062          * So, if access tracking is disabled, we return true only for write
3063          * accesses to a present page.
3064          */
3065
3066         return shadow_acc_track_mask != 0 || (fault->write && fault->present);
3067 }
3068
3069 /*
3070  * Returns true if the SPTE was fixed successfully. Otherwise,
3071  * someone else modified the SPTE from its original value.
3072  */
3073 static bool
3074 fast_pf_fix_direct_spte(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault,
3075                         u64 *sptep, u64 old_spte, u64 new_spte)
3076 {
3077         /*
3078          * Theoretically we could also set dirty bit (and flush TLB) here in
3079          * order to eliminate unnecessary PML logging. See comments in
3080          * set_spte. But fast_page_fault is very unlikely to happen with PML
3081          * enabled, so we do not do this. This might result in the same GPA
3082          * to be logged in PML buffer again when the write really happens, and
3083          * eventually to be called by mark_page_dirty twice. But it's also no
3084          * harm. This also avoids the TLB flush needed after setting dirty bit
3085          * so non-PML cases won't be impacted.
3086          *
3087          * Compare with set_spte where instead shadow_dirty_mask is set.
3088          */
3089         if (cmpxchg64(sptep, old_spte, new_spte) != old_spte)
3090                 return false;
3091
3092         if (is_writable_pte(new_spte) && !is_writable_pte(old_spte))
3093                 mark_page_dirty_in_slot(vcpu->kvm, fault->slot, fault->gfn);
3094
3095         return true;
3096 }
3097
3098 static bool is_access_allowed(struct kvm_page_fault *fault, u64 spte)
3099 {
3100         if (fault->exec)
3101                 return is_executable_pte(spte);
3102
3103         if (fault->write)
3104                 return is_writable_pte(spte);
3105
3106         /* Fault was on Read access */
3107         return spte & PT_PRESENT_MASK;
3108 }
3109
3110 /*
3111  * Returns the last level spte pointer of the shadow page walk for the given
3112  * gpa, and sets *spte to the spte value. This spte may be non-preset. If no
3113  * walk could be performed, returns NULL and *spte does not contain valid data.
3114  *
3115  * Contract:
3116  *  - Must be called between walk_shadow_page_lockless_{begin,end}.
3117  *  - The returned sptep must not be used after walk_shadow_page_lockless_end.
3118  */
3119 static u64 *fast_pf_get_last_sptep(struct kvm_vcpu *vcpu, gpa_t gpa, u64 *spte)
3120 {
3121         struct kvm_shadow_walk_iterator iterator;
3122         u64 old_spte;
3123         u64 *sptep = NULL;
3124
3125         for_each_shadow_entry_lockless(vcpu, gpa, iterator, old_spte) {
3126                 sptep = iterator.sptep;
3127                 *spte = old_spte;
3128         }
3129
3130         return sptep;
3131 }
3132
3133 /*
3134  * Returns one of RET_PF_INVALID, RET_PF_FIXED or RET_PF_SPURIOUS.
3135  */
3136 static int fast_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
3137 {
3138         struct kvm_mmu_page *sp;
3139         int ret = RET_PF_INVALID;
3140         u64 spte = 0ull;
3141         u64 *sptep = NULL;
3142         uint retry_count = 0;
3143
3144         if (!page_fault_can_be_fast(fault))
3145                 return ret;
3146
3147         walk_shadow_page_lockless_begin(vcpu);
3148
3149         do {
3150                 u64 new_spte;
3151
3152                 if (is_tdp_mmu(vcpu->arch.mmu))
3153                         sptep = kvm_tdp_mmu_fast_pf_get_last_sptep(vcpu, fault->addr, &spte);
3154                 else
3155                         sptep = fast_pf_get_last_sptep(vcpu, fault->addr, &spte);
3156
3157                 if (!is_shadow_present_pte(spte))
3158                         break;
3159
3160                 sp = sptep_to_sp(sptep);
3161                 if (!is_last_spte(spte, sp->role.level))
3162                         break;
3163
3164                 /*
3165                  * Check whether the memory access that caused the fault would
3166                  * still cause it if it were to be performed right now. If not,
3167                  * then this is a spurious fault caused by TLB lazily flushed,
3168                  * or some other CPU has already fixed the PTE after the
3169                  * current CPU took the fault.
3170                  *
3171                  * Need not check the access of upper level table entries since
3172                  * they are always ACC_ALL.
3173                  */
3174                 if (is_access_allowed(fault, spte)) {
3175                         ret = RET_PF_SPURIOUS;
3176                         break;
3177                 }
3178
3179                 new_spte = spte;
3180
3181                 if (is_access_track_spte(spte))
3182                         new_spte = restore_acc_track_spte(new_spte);
3183
3184                 /*
3185                  * Currently, to simplify the code, write-protection can
3186                  * be removed in the fast path only if the SPTE was
3187                  * write-protected for dirty-logging or access tracking.
3188                  */
3189                 if (fault->write &&
3190                     spte_can_locklessly_be_made_writable(spte)) {
3191                         new_spte |= PT_WRITABLE_MASK;
3192
3193                         /*
3194                          * Do not fix write-permission on the large spte when
3195                          * dirty logging is enabled. Since we only dirty the
3196                          * first page into the dirty-bitmap in
3197                          * fast_pf_fix_direct_spte(), other pages are missed
3198                          * if its slot has dirty logging enabled.
3199                          *
3200                          * Instead, we let the slow page fault path create a
3201                          * normal spte to fix the access.
3202                          */
3203                         if (sp->role.level > PG_LEVEL_4K &&
3204                             kvm_slot_dirty_track_enabled(fault->slot))
3205                                 break;
3206                 }
3207
3208                 /* Verify that the fault can be handled in the fast path */
3209                 if (new_spte == spte ||
3210                     !is_access_allowed(fault, new_spte))
3211                         break;
3212
3213                 /*
3214                  * Currently, fast page fault only works for direct mapping
3215                  * since the gfn is not stable for indirect shadow page. See
3216                  * Documentation/virt/kvm/locking.rst to get more detail.
3217                  */
3218                 if (fast_pf_fix_direct_spte(vcpu, fault, sptep, spte, new_spte)) {
3219                         ret = RET_PF_FIXED;
3220                         break;
3221                 }
3222
3223                 if (++retry_count > 4) {
3224                         printk_once(KERN_WARNING
3225                                 "kvm: Fast #PF retrying more than 4 times.\n");
3226                         break;
3227                 }
3228
3229         } while (true);
3230
3231         trace_fast_page_fault(vcpu, fault, sptep, spte, ret);
3232         walk_shadow_page_lockless_end(vcpu);
3233
3234         return ret;
3235 }
3236
3237 static void mmu_free_root_page(struct kvm *kvm, hpa_t *root_hpa,
3238                                struct list_head *invalid_list)
3239 {
3240         struct kvm_mmu_page *sp;
3241
3242         if (!VALID_PAGE(*root_hpa))
3243                 return;
3244
3245         sp = to_shadow_page(*root_hpa & PT64_BASE_ADDR_MASK);
3246
3247         if (is_tdp_mmu_page(sp))
3248                 kvm_tdp_mmu_put_root(kvm, sp, false);
3249         else if (!--sp->root_count && sp->role.invalid)
3250                 kvm_mmu_prepare_zap_page(kvm, sp, invalid_list);
3251
3252         *root_hpa = INVALID_PAGE;
3253 }
3254
3255 /* roots_to_free must be some combination of the KVM_MMU_ROOT_* flags */
3256 void kvm_mmu_free_roots(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
3257                         ulong roots_to_free)
3258 {
3259         struct kvm *kvm = vcpu->kvm;
3260         int i;
3261         LIST_HEAD(invalid_list);
3262         bool free_active_root = roots_to_free & KVM_MMU_ROOT_CURRENT;
3263
3264         BUILD_BUG_ON(KVM_MMU_NUM_PREV_ROOTS >= BITS_PER_LONG);
3265
3266         /* Before acquiring the MMU lock, see if we need to do any real work. */
3267         if (!(free_active_root && VALID_PAGE(mmu->root_hpa))) {
3268                 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
3269                         if ((roots_to_free & KVM_MMU_ROOT_PREVIOUS(i)) &&
3270                             VALID_PAGE(mmu->prev_roots[i].hpa))
3271                                 break;
3272
3273                 if (i == KVM_MMU_NUM_PREV_ROOTS)
3274                         return;
3275         }
3276
3277         write_lock(&kvm->mmu_lock);
3278
3279         for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
3280                 if (roots_to_free & KVM_MMU_ROOT_PREVIOUS(i))
3281                         mmu_free_root_page(kvm, &mmu->prev_roots[i].hpa,
3282                                            &invalid_list);
3283
3284         if (free_active_root) {
3285                 if (mmu->shadow_root_level >= PT64_ROOT_4LEVEL &&
3286                     (mmu->root_level >= PT64_ROOT_4LEVEL || mmu->direct_map)) {
3287                         mmu_free_root_page(kvm, &mmu->root_hpa, &invalid_list);
3288                 } else if (mmu->pae_root) {
3289                         for (i = 0; i < 4; ++i) {
3290                                 if (!IS_VALID_PAE_ROOT(mmu->pae_root[i]))
3291                                         continue;
3292
3293                                 mmu_free_root_page(kvm, &mmu->pae_root[i],
3294                                                    &invalid_list);
3295                                 mmu->pae_root[i] = INVALID_PAE_ROOT;
3296                         }
3297                 }
3298                 mmu->root_hpa = INVALID_PAGE;
3299                 mmu->root_pgd = 0;
3300         }
3301
3302         kvm_mmu_commit_zap_page(kvm, &invalid_list);
3303         write_unlock(&kvm->mmu_lock);
3304 }
3305 EXPORT_SYMBOL_GPL(kvm_mmu_free_roots);
3306
3307 void kvm_mmu_free_guest_mode_roots(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu)
3308 {
3309         unsigned long roots_to_free = 0;
3310         hpa_t root_hpa;
3311         int i;
3312
3313         /*
3314          * This should not be called while L2 is active, L2 can't invalidate
3315          * _only_ its own roots, e.g. INVVPID unconditionally exits.
3316          */
3317         WARN_ON_ONCE(mmu->mmu_role.base.guest_mode);
3318
3319         for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) {
3320                 root_hpa = mmu->prev_roots[i].hpa;
3321                 if (!VALID_PAGE(root_hpa))
3322                         continue;
3323
3324                 if (!to_shadow_page(root_hpa) ||
3325                         to_shadow_page(root_hpa)->role.guest_mode)
3326                         roots_to_free |= KVM_MMU_ROOT_PREVIOUS(i);
3327         }
3328
3329         kvm_mmu_free_roots(vcpu, mmu, roots_to_free);
3330 }
3331 EXPORT_SYMBOL_GPL(kvm_mmu_free_guest_mode_roots);
3332
3333
3334 static int mmu_check_root(struct kvm_vcpu *vcpu, gfn_t root_gfn)
3335 {
3336         int ret = 0;
3337
3338         if (!kvm_vcpu_is_visible_gfn(vcpu, root_gfn)) {
3339                 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
3340                 ret = 1;
3341         }
3342
3343         return ret;
3344 }
3345
3346 static hpa_t mmu_alloc_root(struct kvm_vcpu *vcpu, gfn_t gfn, gva_t gva,
3347                             u8 level, bool direct)
3348 {
3349         struct kvm_mmu_page *sp;
3350
3351         sp = kvm_mmu_get_page(vcpu, gfn, gva, level, direct, ACC_ALL);
3352         ++sp->root_count;
3353
3354         return __pa(sp->spt);
3355 }
3356
3357 static int mmu_alloc_direct_roots(struct kvm_vcpu *vcpu)
3358 {
3359         struct kvm_mmu *mmu = vcpu->arch.mmu;
3360         u8 shadow_root_level = mmu->shadow_root_level;
3361         hpa_t root;
3362         unsigned i;
3363         int r;
3364
3365         write_lock(&vcpu->kvm->mmu_lock);
3366         r = make_mmu_pages_available(vcpu);
3367         if (r < 0)
3368                 goto out_unlock;
3369
3370         if (is_tdp_mmu_enabled(vcpu->kvm)) {
3371                 root = kvm_tdp_mmu_get_vcpu_root_hpa(vcpu);
3372                 mmu->root_hpa = root;
3373         } else if (shadow_root_level >= PT64_ROOT_4LEVEL) {
3374                 root = mmu_alloc_root(vcpu, 0, 0, shadow_root_level, true);
3375                 mmu->root_hpa = root;
3376         } else if (shadow_root_level == PT32E_ROOT_LEVEL) {
3377                 if (WARN_ON_ONCE(!mmu->pae_root)) {
3378                         r = -EIO;
3379                         goto out_unlock;
3380                 }
3381
3382                 for (i = 0; i < 4; ++i) {
3383                         WARN_ON_ONCE(IS_VALID_PAE_ROOT(mmu->pae_root[i]));
3384
3385                         root = mmu_alloc_root(vcpu, i << (30 - PAGE_SHIFT),
3386                                               i << 30, PT32_ROOT_LEVEL, true);
3387                         mmu->pae_root[i] = root | PT_PRESENT_MASK |
3388                                            shadow_me_mask;
3389                 }
3390                 mmu->root_hpa = __pa(mmu->pae_root);
3391         } else {
3392                 WARN_ONCE(1, "Bad TDP root level = %d\n", shadow_root_level);
3393                 r = -EIO;
3394                 goto out_unlock;
3395         }
3396
3397         /* root_pgd is ignored for direct MMUs. */
3398         mmu->root_pgd = 0;
3399 out_unlock:
3400         write_unlock(&vcpu->kvm->mmu_lock);
3401         return r;
3402 }
3403
3404 static int mmu_first_shadow_root_alloc(struct kvm *kvm)
3405 {
3406         struct kvm_memslots *slots;
3407         struct kvm_memory_slot *slot;
3408         int r = 0, i;
3409
3410         /*
3411          * Check if this is the first shadow root being allocated before
3412          * taking the lock.
3413          */
3414         if (kvm_shadow_root_allocated(kvm))
3415                 return 0;
3416
3417         mutex_lock(&kvm->slots_arch_lock);
3418
3419         /* Recheck, under the lock, whether this is the first shadow root. */
3420         if (kvm_shadow_root_allocated(kvm))
3421                 goto out_unlock;
3422
3423         /*
3424          * Check if anything actually needs to be allocated, e.g. all metadata
3425          * will be allocated upfront if TDP is disabled.
3426          */
3427         if (kvm_memslots_have_rmaps(kvm) &&
3428             kvm_page_track_write_tracking_enabled(kvm))
3429                 goto out_success;
3430
3431         for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
3432                 slots = __kvm_memslots(kvm, i);
3433                 kvm_for_each_memslot(slot, slots) {
3434                         /*
3435                          * Both of these functions are no-ops if the target is
3436                          * already allocated, so unconditionally calling both
3437                          * is safe.  Intentionally do NOT free allocations on
3438                          * failure to avoid having to track which allocations
3439                          * were made now versus when the memslot was created.
3440                          * The metadata is guaranteed to be freed when the slot
3441                          * is freed, and will be kept/used if userspace retries
3442                          * KVM_RUN instead of killing the VM.
3443                          */
3444                         r = memslot_rmap_alloc(slot, slot->npages);
3445                         if (r)
3446                                 goto out_unlock;
3447                         r = kvm_page_track_write_tracking_alloc(slot);
3448                         if (r)
3449                                 goto out_unlock;
3450                 }
3451         }
3452
3453         /*
3454          * Ensure that shadow_root_allocated becomes true strictly after
3455          * all the related pointers are set.
3456          */
3457 out_success:
3458         smp_store_release(&kvm->arch.shadow_root_allocated, true);
3459
3460 out_unlock:
3461         mutex_unlock(&kvm->slots_arch_lock);
3462         return r;
3463 }
3464
3465 static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu)
3466 {
3467         struct kvm_mmu *mmu = vcpu->arch.mmu;
3468         u64 pdptrs[4], pm_mask;
3469         gfn_t root_gfn, root_pgd;
3470         hpa_t root;
3471         unsigned i;
3472         int r;
3473
3474         root_pgd = mmu->get_guest_pgd(vcpu);
3475         root_gfn = root_pgd >> PAGE_SHIFT;
3476
3477         if (mmu_check_root(vcpu, root_gfn))
3478                 return 1;
3479
3480         /*
3481          * On SVM, reading PDPTRs might access guest memory, which might fault
3482          * and thus might sleep.  Grab the PDPTRs before acquiring mmu_lock.
3483          */
3484         if (mmu->root_level == PT32E_ROOT_LEVEL) {
3485                 for (i = 0; i < 4; ++i) {
3486                         pdptrs[i] = mmu->get_pdptr(vcpu, i);
3487                         if (!(pdptrs[i] & PT_PRESENT_MASK))
3488                                 continue;
3489
3490                         if (mmu_check_root(vcpu, pdptrs[i] >> PAGE_SHIFT))
3491                                 return 1;
3492                 }
3493         }
3494
3495         r = mmu_first_shadow_root_alloc(vcpu->kvm);
3496         if (r)
3497                 return r;
3498
3499         write_lock(&vcpu->kvm->mmu_lock);
3500         r = make_mmu_pages_available(vcpu);
3501         if (r < 0)
3502                 goto out_unlock;
3503
3504         /*
3505          * Do we shadow a long mode page table? If so we need to
3506          * write-protect the guests page table root.
3507          */
3508         if (mmu->root_level >= PT64_ROOT_4LEVEL) {
3509                 root = mmu_alloc_root(vcpu, root_gfn, 0,
3510                                       mmu->shadow_root_level, false);
3511                 mmu->root_hpa = root;
3512                 goto set_root_pgd;
3513         }
3514
3515         if (WARN_ON_ONCE(!mmu->pae_root)) {
3516                 r = -EIO;
3517                 goto out_unlock;
3518         }
3519
3520         /*
3521          * We shadow a 32 bit page table. This may be a legacy 2-level
3522          * or a PAE 3-level page table. In either case we need to be aware that
3523          * the shadow page table may be a PAE or a long mode page table.
3524          */
3525         pm_mask = PT_PRESENT_MASK | shadow_me_mask;
3526         if (mmu->shadow_root_level >= PT64_ROOT_4LEVEL) {
3527                 pm_mask |= PT_ACCESSED_MASK | PT_WRITABLE_MASK | PT_USER_MASK;
3528
3529                 if (WARN_ON_ONCE(!mmu->pml4_root)) {
3530                         r = -EIO;
3531                         goto out_unlock;
3532                 }
3533                 mmu->pml4_root[0] = __pa(mmu->pae_root) | pm_mask;
3534
3535                 if (mmu->shadow_root_level == PT64_ROOT_5LEVEL) {
3536                         if (WARN_ON_ONCE(!mmu->pml5_root)) {
3537                                 r = -EIO;
3538                                 goto out_unlock;
3539                         }
3540                         mmu->pml5_root[0] = __pa(mmu->pml4_root) | pm_mask;
3541                 }
3542         }
3543
3544         for (i = 0; i < 4; ++i) {
3545                 WARN_ON_ONCE(IS_VALID_PAE_ROOT(mmu->pae_root[i]));
3546
3547                 if (mmu->root_level == PT32E_ROOT_LEVEL) {
3548                         if (!(pdptrs[i] & PT_PRESENT_MASK)) {
3549                                 mmu->pae_root[i] = INVALID_PAE_ROOT;
3550                                 continue;
3551                         }
3552                         root_gfn = pdptrs[i] >> PAGE_SHIFT;
3553                 }
3554
3555                 root = mmu_alloc_root(vcpu, root_gfn, i << 30,
3556                                       PT32_ROOT_LEVEL, false);
3557                 mmu->pae_root[i] = root | pm_mask;
3558         }
3559
3560         if (mmu->shadow_root_level == PT64_ROOT_5LEVEL)
3561                 mmu->root_hpa = __pa(mmu->pml5_root);
3562         else if (mmu->shadow_root_level == PT64_ROOT_4LEVEL)
3563                 mmu->root_hpa = __pa(mmu->pml4_root);
3564         else
3565                 mmu->root_hpa = __pa(mmu->pae_root);
3566
3567 set_root_pgd:
3568         mmu->root_pgd = root_pgd;
3569 out_unlock:
3570         write_unlock(&vcpu->kvm->mmu_lock);
3571
3572         return 0;
3573 }
3574
3575 static int mmu_alloc_special_roots(struct kvm_vcpu *vcpu)
3576 {
3577         struct kvm_mmu *mmu = vcpu->arch.mmu;
3578         bool need_pml5 = mmu->shadow_root_level > PT64_ROOT_4LEVEL;
3579         u64 *pml5_root = NULL;
3580         u64 *pml4_root = NULL;
3581         u64 *pae_root;
3582
3583         /*
3584          * When shadowing 32-bit or PAE NPT with 64-bit NPT, the PML4 and PDP
3585          * tables are allocated and initialized at root creation as there is no
3586          * equivalent level in the guest's NPT to shadow.  Allocate the tables
3587          * on demand, as running a 32-bit L1 VMM on 64-bit KVM is very rare.
3588          */
3589         if (mmu->direct_map || mmu->root_level >= PT64_ROOT_4LEVEL ||
3590             mmu->shadow_root_level < PT64_ROOT_4LEVEL)
3591                 return 0;
3592
3593         /*
3594          * NPT, the only paging mode that uses this horror, uses a fixed number
3595          * of levels for the shadow page tables, e.g. all MMUs are 4-level or
3596          * all MMus are 5-level.  Thus, this can safely require that pml5_root
3597          * is allocated if the other roots are valid and pml5 is needed, as any
3598          * prior MMU would also have required pml5.
3599          */
3600         if (mmu->pae_root && mmu->pml4_root && (!need_pml5 || mmu->pml5_root))
3601                 return 0;
3602
3603         /*
3604          * The special roots should always be allocated in concert.  Yell and
3605          * bail if KVM ends up in a state where only one of the roots is valid.
3606          */
3607         if (WARN_ON_ONCE(!tdp_enabled || mmu->pae_root || mmu->pml4_root ||
3608                          (need_pml5 && mmu->pml5_root)))
3609                 return -EIO;
3610
3611         /*
3612          * Unlike 32-bit NPT, the PDP table doesn't need to be in low mem, and
3613          * doesn't need to be decrypted.
3614          */
3615         pae_root = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT);
3616         if (!pae_root)
3617                 return -ENOMEM;
3618
3619 #ifdef CONFIG_X86_64
3620         pml4_root = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT);
3621         if (!pml4_root)
3622                 goto err_pml4;
3623
3624         if (need_pml5) {
3625                 pml5_root = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT);
3626                 if (!pml5_root)
3627                         goto err_pml5;
3628         }
3629 #endif
3630
3631         mmu->pae_root = pae_root;
3632         mmu->pml4_root = pml4_root;
3633         mmu->pml5_root = pml5_root;
3634
3635         return 0;
3636
3637 #ifdef CONFIG_X86_64
3638 err_pml5:
3639         free_page((unsigned long)pml4_root);
3640 err_pml4:
3641         free_page((unsigned long)pae_root);
3642         return -ENOMEM;
3643 #endif
3644 }
3645
3646 static bool is_unsync_root(hpa_t root)
3647 {
3648         struct kvm_mmu_page *sp;
3649
3650         if (!VALID_PAGE(root))
3651                 return false;
3652
3653         /*
3654          * The read barrier orders the CPU's read of SPTE.W during the page table
3655          * walk before the reads of sp->unsync/sp->unsync_children here.
3656          *
3657          * Even if another CPU was marking the SP as unsync-ed simultaneously,
3658          * any guest page table changes are not guaranteed to be visible anyway
3659          * until this VCPU issues a TLB flush strictly after those changes are
3660          * made.  We only need to ensure that the other CPU sets these flags
3661          * before any actual changes to the page tables are made.  The comments
3662          * in mmu_try_to_unsync_pages() describe what could go wrong if this
3663          * requirement isn't satisfied.
3664          */
3665         smp_rmb();
3666         sp = to_shadow_page(root);
3667         if (sp->unsync || sp->unsync_children)
3668                 return true;
3669
3670         return false;
3671 }
3672
3673 void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu)
3674 {
3675         int i;
3676         struct kvm_mmu_page *sp;
3677
3678         if (vcpu->arch.mmu->direct_map)
3679                 return;
3680
3681         if (!VALID_PAGE(vcpu->arch.mmu->root_hpa))
3682                 return;
3683
3684         vcpu_clear_mmio_info(vcpu, MMIO_GVA_ANY);
3685
3686         if (vcpu->arch.mmu->root_level >= PT64_ROOT_4LEVEL) {
3687                 hpa_t root = vcpu->arch.mmu->root_hpa;
3688                 sp = to_shadow_page(root);
3689
3690                 if (!is_unsync_root(root))
3691                         return;
3692
3693                 write_lock(&vcpu->kvm->mmu_lock);
3694                 kvm_mmu_audit(vcpu, AUDIT_PRE_SYNC);
3695
3696                 mmu_sync_children(vcpu, sp, true);
3697
3698                 kvm_mmu_audit(vcpu, AUDIT_POST_SYNC);
3699                 write_unlock(&vcpu->kvm->mmu_lock);
3700                 return;
3701         }
3702
3703         write_lock(&vcpu->kvm->mmu_lock);
3704         kvm_mmu_audit(vcpu, AUDIT_PRE_SYNC);
3705
3706         for (i = 0; i < 4; ++i) {
3707                 hpa_t root = vcpu->arch.mmu->pae_root[i];
3708
3709                 if (IS_VALID_PAE_ROOT(root)) {
3710                         root &= PT64_BASE_ADDR_MASK;
3711                         sp = to_shadow_page(root);
3712                         mmu_sync_children(vcpu, sp, true);
3713                 }
3714         }
3715
3716         kvm_mmu_audit(vcpu, AUDIT_POST_SYNC);
3717         write_unlock(&vcpu->kvm->mmu_lock);
3718 }
3719
3720 void kvm_mmu_sync_prev_roots(struct kvm_vcpu *vcpu)
3721 {
3722         unsigned long roots_to_free = 0;
3723         int i;
3724
3725         for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
3726                 if (is_unsync_root(vcpu->arch.mmu->prev_roots[i].hpa))
3727                         roots_to_free |= KVM_MMU_ROOT_PREVIOUS(i);
3728
3729         /* sync prev_roots by simply freeing them */
3730         kvm_mmu_free_roots(vcpu, vcpu->arch.mmu, roots_to_free);
3731 }
3732
3733 static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gpa_t vaddr,
3734                                   u32 access, struct x86_exception *exception)
3735 {
3736         if (exception)
3737                 exception->error_code = 0;
3738         return vaddr;
3739 }
3740
3741 static gpa_t nonpaging_gva_to_gpa_nested(struct kvm_vcpu *vcpu, gpa_t vaddr,
3742                                          u32 access,
3743                                          struct x86_exception *exception)
3744 {
3745         if (exception)
3746                 exception->error_code = 0;
3747         return vcpu->arch.nested_mmu.translate_gpa(vcpu, vaddr, access, exception);
3748 }
3749
3750 static bool mmio_info_in_cache(struct kvm_vcpu *vcpu, u64 addr, bool direct)
3751 {
3752         /*
3753          * A nested guest cannot use the MMIO cache if it is using nested
3754          * page tables, because cr2 is a nGPA while the cache stores GPAs.
3755          */
3756         if (mmu_is_nested(vcpu))
3757                 return false;
3758
3759         if (direct)
3760                 return vcpu_match_mmio_gpa(vcpu, addr);
3761
3762         return vcpu_match_mmio_gva(vcpu, addr);
3763 }
3764
3765 /*
3766  * Return the level of the lowest level SPTE added to sptes.
3767  * That SPTE may be non-present.
3768  *
3769  * Must be called between walk_shadow_page_lockless_{begin,end}.
3770  */
3771 static int get_walk(struct kvm_vcpu *vcpu, u64 addr, u64 *sptes, int *root_level)
3772 {
3773         struct kvm_shadow_walk_iterator iterator;
3774         int leaf = -1;
3775         u64 spte;
3776
3777         for (shadow_walk_init(&iterator, vcpu, addr),
3778              *root_level = iterator.level;
3779              shadow_walk_okay(&iterator);
3780              __shadow_walk_next(&iterator, spte)) {
3781                 leaf = iterator.level;
3782                 spte = mmu_spte_get_lockless(iterator.sptep);
3783
3784                 sptes[leaf] = spte;
3785         }
3786
3787         return leaf;
3788 }
3789
3790 /* return true if reserved bit(s) are detected on a valid, non-MMIO SPTE. */
3791 static bool get_mmio_spte(struct kvm_vcpu *vcpu, u64 addr, u64 *sptep)
3792 {
3793         u64 sptes[PT64_ROOT_MAX_LEVEL + 1];
3794         struct rsvd_bits_validate *rsvd_check;
3795         int root, leaf, level;
3796         bool reserved = false;
3797
3798         walk_shadow_page_lockless_begin(vcpu);
3799
3800         if (is_tdp_mmu(vcpu->arch.mmu))
3801                 leaf = kvm_tdp_mmu_get_walk(vcpu, addr, sptes, &root);
3802         else
3803                 leaf = get_walk(vcpu, addr, sptes, &root);
3804
3805         walk_shadow_page_lockless_end(vcpu);
3806
3807         if (unlikely(leaf < 0)) {
3808                 *sptep = 0ull;
3809                 return reserved;
3810         }
3811
3812         *sptep = sptes[leaf];
3813
3814         /*
3815          * Skip reserved bits checks on the terminal leaf if it's not a valid
3816          * SPTE.  Note, this also (intentionally) skips MMIO SPTEs, which, by
3817          * design, always have reserved bits set.  The purpose of the checks is
3818          * to detect reserved bits on non-MMIO SPTEs. i.e. buggy SPTEs.
3819          */
3820         if (!is_shadow_present_pte(sptes[leaf]))
3821                 leaf++;
3822
3823         rsvd_check = &vcpu->arch.mmu->shadow_zero_check;
3824
3825         for (level = root; level >= leaf; level--)
3826                 reserved |= is_rsvd_spte(rsvd_check, sptes[level], level);
3827
3828         if (reserved) {
3829                 pr_err("%s: reserved bits set on MMU-present spte, addr 0x%llx, hierarchy:\n",
3830                        __func__, addr);
3831                 for (level = root; level >= leaf; level--)
3832                         pr_err("------ spte = 0x%llx level = %d, rsvd bits = 0x%llx",
3833                                sptes[level], level,
3834                                get_rsvd_bits(rsvd_check, sptes[level], level));
3835         }
3836
3837         return reserved;
3838 }
3839
3840 static int handle_mmio_page_fault(struct kvm_vcpu *vcpu, u64 addr, bool direct)
3841 {
3842         u64 spte;
3843         bool reserved;
3844
3845         if (mmio_info_in_cache(vcpu, addr, direct))
3846                 return RET_PF_EMULATE;
3847
3848         reserved = get_mmio_spte(vcpu, addr, &spte);
3849         if (WARN_ON(reserved))
3850                 return -EINVAL;
3851
3852         if (is_mmio_spte(spte)) {
3853                 gfn_t gfn = get_mmio_spte_gfn(spte);
3854                 unsigned int access = get_mmio_spte_access(spte);
3855
3856                 if (!check_mmio_spte(vcpu, spte))
3857                         return RET_PF_INVALID;
3858
3859                 if (direct)
3860                         addr = 0;
3861
3862                 trace_handle_mmio_page_fault(addr, gfn, access);
3863                 vcpu_cache_mmio_info(vcpu, addr, gfn, access);
3864                 return RET_PF_EMULATE;
3865         }
3866
3867         /*
3868          * If the page table is zapped by other cpus, let CPU fault again on
3869          * the address.
3870          */
3871         return RET_PF_RETRY;
3872 }
3873
3874 static bool page_fault_handle_page_track(struct kvm_vcpu *vcpu,
3875                                          struct kvm_page_fault *fault)
3876 {
3877         if (unlikely(fault->rsvd))
3878                 return false;
3879
3880         if (!fault->present || !fault->write)
3881                 return false;
3882
3883         /*
3884          * guest is writing the page which is write tracked which can
3885          * not be fixed by page fault handler.
3886          */
3887         if (kvm_slot_page_track_is_active(vcpu, fault->slot, fault->gfn, KVM_PAGE_TRACK_WRITE))
3888                 return true;
3889
3890         return false;
3891 }
3892
3893 static void shadow_page_table_clear_flood(struct kvm_vcpu *vcpu, gva_t addr)
3894 {
3895         struct kvm_shadow_walk_iterator iterator;
3896         u64 spte;
3897
3898         walk_shadow_page_lockless_begin(vcpu);
3899         for_each_shadow_entry_lockless(vcpu, addr, iterator, spte)
3900                 clear_sp_write_flooding_count(iterator.sptep);
3901         walk_shadow_page_lockless_end(vcpu);
3902 }
3903
3904 static bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
3905                                     gfn_t gfn)
3906 {
3907         struct kvm_arch_async_pf arch;
3908
3909         arch.token = (vcpu->arch.apf.id++ << 12) | vcpu->vcpu_id;
3910         arch.gfn = gfn;
3911         arch.direct_map = vcpu->arch.mmu->direct_map;
3912         arch.cr3 = vcpu->arch.mmu->get_guest_pgd(vcpu);
3913
3914         return kvm_setup_async_pf(vcpu, cr2_or_gpa,
3915                                   kvm_vcpu_gfn_to_hva(vcpu, gfn), &arch);
3916 }
3917
3918 static bool kvm_faultin_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault, int *r)
3919 {
3920         struct kvm_memory_slot *slot = fault->slot;
3921         bool async;
3922
3923         /*
3924          * Retry the page fault if the gfn hit a memslot that is being deleted
3925          * or moved.  This ensures any existing SPTEs for the old memslot will
3926          * be zapped before KVM inserts a new MMIO SPTE for the gfn.
3927          */
3928         if (slot && (slot->flags & KVM_MEMSLOT_INVALID))
3929                 goto out_retry;
3930
3931         if (!kvm_is_visible_memslot(slot)) {
3932                 /* Don't expose private memslots to L2. */
3933                 if (is_guest_mode(vcpu)) {
3934                         fault->slot = NULL;
3935                         fault->pfn = KVM_PFN_NOSLOT;
3936                         fault->map_writable = false;
3937                         return false;
3938                 }
3939                 /*
3940                  * If the APIC access page exists but is disabled, go directly
3941                  * to emulation without caching the MMIO access or creating a
3942                  * MMIO SPTE.  That way the cache doesn't need to be purged
3943                  * when the AVIC is re-enabled.
3944                  */
3945                 if (slot && slot->id == APIC_ACCESS_PAGE_PRIVATE_MEMSLOT &&
3946                     !kvm_apicv_activated(vcpu->kvm)) {
3947                         *r = RET_PF_EMULATE;
3948                         return true;
3949                 }
3950         }
3951
3952         async = false;
3953         fault->pfn = __gfn_to_pfn_memslot(slot, fault->gfn, false, &async,
3954                                           fault->write, &fault->map_writable,
3955                                           &fault->hva);
3956         if (!async)
3957                 return false; /* *pfn has correct page already */
3958
3959         if (!fault->prefetch && kvm_can_do_async_pf(vcpu)) {
3960                 trace_kvm_try_async_get_page(fault->addr, fault->gfn);
3961                 if (kvm_find_async_pf_gfn(vcpu, fault->gfn)) {
3962                         trace_kvm_async_pf_doublefault(fault->addr, fault->gfn);
3963                         kvm_make_request(KVM_REQ_APF_HALT, vcpu);
3964                         goto out_retry;
3965                 } else if (kvm_arch_setup_async_pf(vcpu, fault->addr, fault->gfn))
3966                         goto out_retry;
3967         }
3968
3969         fault->pfn = __gfn_to_pfn_memslot(slot, fault->gfn, false, NULL,
3970                                           fault->write, &fault->map_writable,
3971                                           &fault->hva);
3972         return false;
3973
3974 out_retry:
3975         *r = RET_PF_RETRY;
3976         return true;
3977 }
3978
3979 static int direct_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
3980 {
3981         bool is_tdp_mmu_fault = is_tdp_mmu(vcpu->arch.mmu);
3982
3983         unsigned long mmu_seq;
3984         int r;
3985
3986         fault->gfn = fault->addr >> PAGE_SHIFT;
3987         fault->slot = kvm_vcpu_gfn_to_memslot(vcpu, fault->gfn);
3988
3989         if (page_fault_handle_page_track(vcpu, fault))
3990                 return RET_PF_EMULATE;
3991
3992         r = fast_page_fault(vcpu, fault);
3993         if (r != RET_PF_INVALID)
3994                 return r;
3995
3996         r = mmu_topup_memory_caches(vcpu, false);
3997         if (r)
3998                 return r;
3999
4000         mmu_seq = vcpu->kvm->mmu_notifier_seq;
4001         smp_rmb();
4002
4003         if (kvm_faultin_pfn(vcpu, fault, &r))
4004                 return r;
4005
4006         if (handle_abnormal_pfn(vcpu, fault, ACC_ALL, &r))
4007                 return r;
4008
4009         r = RET_PF_RETRY;
4010
4011         if (is_tdp_mmu_fault)
4012                 read_lock(&vcpu->kvm->mmu_lock);
4013         else
4014                 write_lock(&vcpu->kvm->mmu_lock);
4015
4016         if (fault->slot && mmu_notifier_retry_hva(vcpu->kvm, mmu_seq, fault->hva))
4017                 goto out_unlock;
4018         r = make_mmu_pages_available(vcpu);
4019         if (r)
4020                 goto out_unlock;
4021
4022         if (is_tdp_mmu_fault)
4023                 r = kvm_tdp_mmu_map(vcpu, fault);
4024         else
4025                 r = __direct_map(vcpu, fault);
4026
4027 out_unlock:
4028         if (is_tdp_mmu_fault)
4029                 read_unlock(&vcpu->kvm->mmu_lock);
4030         else
4031                 write_unlock(&vcpu->kvm->mmu_lock);
4032         kvm_release_pfn_clean(fault->pfn);
4033         return r;
4034 }
4035
4036 static int nonpaging_page_fault(struct kvm_vcpu *vcpu,
4037                                 struct kvm_page_fault *fault)
4038 {
4039         pgprintk("%s: gva %lx error %x\n", __func__, fault->addr, fault->error_code);
4040
4041         /* This path builds a PAE pagetable, we can map 2mb pages at maximum. */
4042         fault->max_level = PG_LEVEL_2M;
4043         return direct_page_fault(vcpu, fault);
4044 }
4045
4046 int kvm_handle_page_fault(struct kvm_vcpu *vcpu, u64 error_code,
4047                                 u64 fault_address, char *insn, int insn_len)
4048 {
4049         int r = 1;
4050         u32 flags = vcpu->arch.apf.host_apf_flags;
4051
4052 #ifndef CONFIG_X86_64
4053         /* A 64-bit CR2 should be impossible on 32-bit KVM. */
4054         if (WARN_ON_ONCE(fault_address >> 32))
4055                 return -EFAULT;
4056 #endif
4057
4058         vcpu->arch.l1tf_flush_l1d = true;
4059         if (!flags) {
4060                 trace_kvm_page_fault(fault_address, error_code);
4061
4062                 if (kvm_event_needs_reinjection(vcpu))
4063                         kvm_mmu_unprotect_page_virt(vcpu, fault_address);
4064                 r = kvm_mmu_page_fault(vcpu, fault_address, error_code, insn,
4065                                 insn_len);
4066         } else if (flags & KVM_PV_REASON_PAGE_NOT_PRESENT) {
4067                 vcpu->arch.apf.host_apf_flags = 0;
4068                 local_irq_disable();
4069                 kvm_async_pf_task_wait_schedule(fault_address);
4070                 local_irq_enable();
4071         } else {
4072                 WARN_ONCE(1, "Unexpected host async PF flags: %x\n", flags);
4073         }
4074
4075         return r;
4076 }
4077 EXPORT_SYMBOL_GPL(kvm_handle_page_fault);
4078
4079 int kvm_tdp_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
4080 {
4081         while (fault->max_level > PG_LEVEL_4K) {
4082                 int page_num = KVM_PAGES_PER_HPAGE(fault->max_level);
4083                 gfn_t base = (fault->addr >> PAGE_SHIFT) & ~(page_num - 1);
4084
4085                 if (kvm_mtrr_check_gfn_range_consistency(vcpu, base, page_num))
4086                         break;
4087
4088                 --fault->max_level;
4089         }
4090
4091         return direct_page_fault(vcpu, fault);
4092 }
4093
4094 static void nonpaging_init_context(struct kvm_mmu *context)
4095 {
4096         context->page_fault = nonpaging_page_fault;
4097         context->gva_to_gpa = nonpaging_gva_to_gpa;
4098         context->sync_page = nonpaging_sync_page;
4099         context->invlpg = NULL;
4100         context->direct_map = true;
4101 }
4102
4103 static inline bool is_root_usable(struct kvm_mmu_root_info *root, gpa_t pgd,
4104                                   union kvm_mmu_page_role role)
4105 {
4106         return (role.direct || pgd == root->pgd) &&
4107                VALID_PAGE(root->hpa) && to_shadow_page(root->hpa) &&
4108                role.word == to_shadow_page(root->hpa)->role.word;
4109 }
4110
4111 /*
4112  * Find out if a previously cached root matching the new pgd/role is available.
4113  * The current root is also inserted into the cache.
4114  * If a matching root was found, it is assigned to kvm_mmu->root_hpa and true is
4115  * returned.
4116  * Otherwise, the LRU root from the cache is assigned to kvm_mmu->root_hpa and
4117  * false is returned. This root should now be freed by the caller.
4118  */
4119 static bool cached_root_available(struct kvm_vcpu *vcpu, gpa_t new_pgd,
4120                                   union kvm_mmu_page_role new_role)
4121 {
4122         uint i;
4123         struct kvm_mmu_root_info root;
4124         struct kvm_mmu *mmu = vcpu->arch.mmu;
4125
4126         root.pgd = mmu->root_pgd;
4127         root.hpa = mmu->root_hpa;
4128
4129         if (is_root_usable(&root, new_pgd, new_role))
4130                 return true;
4131
4132         for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) {
4133                 swap(root, mmu->prev_roots[i]);
4134
4135                 if (is_root_usable(&root, new_pgd, new_role))
4136                         break;
4137         }
4138
4139         mmu->root_hpa = root.hpa;
4140         mmu->root_pgd = root.pgd;
4141
4142         return i < KVM_MMU_NUM_PREV_ROOTS;
4143 }
4144
4145 static bool fast_pgd_switch(struct kvm_vcpu *vcpu, gpa_t new_pgd,
4146                             union kvm_mmu_page_role new_role)
4147 {
4148         struct kvm_mmu *mmu = vcpu->arch.mmu;
4149
4150         /*
4151          * For now, limit the fast switch to 64-bit hosts+VMs in order to avoid
4152          * having to deal with PDPTEs. We may add support for 32-bit hosts/VMs
4153          * later if necessary.
4154          */
4155         if (mmu->shadow_root_level >= PT64_ROOT_4LEVEL &&
4156             mmu->root_level >= PT64_ROOT_4LEVEL)
4157                 return cached_root_available(vcpu, new_pgd, new_role);
4158
4159         return false;
4160 }
4161
4162 static void __kvm_mmu_new_pgd(struct kvm_vcpu *vcpu, gpa_t new_pgd,
4163                               union kvm_mmu_page_role new_role)
4164 {
4165         if (!fast_pgd_switch(vcpu, new_pgd, new_role)) {
4166                 kvm_mmu_free_roots(vcpu, vcpu->arch.mmu, KVM_MMU_ROOT_CURRENT);
4167                 return;
4168         }
4169
4170         /*
4171          * It's possible that the cached previous root page is obsolete because
4172          * of a change in the MMU generation number. However, changing the
4173          * generation number is accompanied by KVM_REQ_MMU_RELOAD, which will
4174          * free the root set here and allocate a new one.
4175          */
4176         kvm_make_request(KVM_REQ_LOAD_MMU_PGD, vcpu);
4177
4178         if (force_flush_and_sync_on_reuse) {
4179                 kvm_make_request(KVM_REQ_MMU_SYNC, vcpu);
4180                 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);
4181         }
4182
4183         /*
4184          * The last MMIO access's GVA and GPA are cached in the VCPU. When
4185          * switching to a new CR3, that GVA->GPA mapping may no longer be
4186          * valid. So clear any cached MMIO info even when we don't need to sync
4187          * the shadow page tables.
4188          */
4189         vcpu_clear_mmio_info(vcpu, MMIO_GVA_ANY);
4190
4191         /*
4192          * If this is a direct root page, it doesn't have a write flooding
4193          * count. Otherwise, clear the write flooding count.
4194          */
4195         if (!new_role.direct)
4196                 __clear_sp_write_flooding_count(
4197                                 to_shadow_page(vcpu->arch.mmu->root_hpa));
4198 }
4199
4200 void kvm_mmu_new_pgd(struct kvm_vcpu *vcpu, gpa_t new_pgd)
4201 {
4202         __kvm_mmu_new_pgd(vcpu, new_pgd, kvm_mmu_calc_root_page_role(vcpu));
4203 }
4204 EXPORT_SYMBOL_GPL(kvm_mmu_new_pgd);
4205
4206 static unsigned long get_cr3(struct kvm_vcpu *vcpu)
4207 {
4208         return kvm_read_cr3(vcpu);
4209 }
4210
4211 static bool sync_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, gfn_t gfn,
4212                            unsigned int access)
4213 {
4214         if (unlikely(is_mmio_spte(*sptep))) {
4215                 if (gfn != get_mmio_spte_gfn(*sptep)) {
4216                         mmu_spte_clear_no_track(sptep);
4217                         return true;
4218                 }
4219
4220                 mark_mmio_spte(vcpu, sptep, gfn, access);
4221                 return true;
4222         }
4223
4224         return false;
4225 }
4226
4227 #define PTTYPE_EPT 18 /* arbitrary */
4228 #define PTTYPE PTTYPE_EPT
4229 #include "paging_tmpl.h"
4230 #undef PTTYPE
4231
4232 #define PTTYPE 64
4233 #include "paging_tmpl.h"
4234 #undef PTTYPE
4235
4236 #define PTTYPE 32
4237 #include "paging_tmpl.h"
4238 #undef PTTYPE
4239
4240 static void
4241 __reset_rsvds_bits_mask(struct rsvd_bits_validate *rsvd_check,
4242                         u64 pa_bits_rsvd, int level, bool nx, bool gbpages,
4243                         bool pse, bool amd)
4244 {
4245         u64 gbpages_bit_rsvd = 0;
4246         u64 nonleaf_bit8_rsvd = 0;
4247         u64 high_bits_rsvd;
4248
4249         rsvd_check->bad_mt_xwr = 0;
4250
4251         if (!gbpages)
4252                 gbpages_bit_rsvd = rsvd_bits(7, 7);
4253
4254         if (level == PT32E_ROOT_LEVEL)
4255                 high_bits_rsvd = pa_bits_rsvd & rsvd_bits(0, 62);
4256         else
4257                 high_bits_rsvd = pa_bits_rsvd & rsvd_bits(0, 51);
4258
4259         /* Note, NX doesn't exist in PDPTEs, this is handled below. */
4260         if (!nx)
4261                 high_bits_rsvd |= rsvd_bits(63, 63);
4262
4263         /*
4264          * Non-leaf PML4Es and PDPEs reserve bit 8 (which would be the G bit for
4265          * leaf entries) on AMD CPUs only.
4266          */
4267         if (amd)
4268                 nonleaf_bit8_rsvd = rsvd_bits(8, 8);
4269
4270         switch (level) {
4271         case PT32_ROOT_LEVEL:
4272                 /* no rsvd bits for 2 level 4K page table entries */
4273                 rsvd_check->rsvd_bits_mask[0][1] = 0;
4274                 rsvd_check->rsvd_bits_mask[0][0] = 0;
4275                 rsvd_check->rsvd_bits_mask[1][0] =
4276                         rsvd_check->rsvd_bits_mask[0][0];
4277
4278                 if (!pse) {
4279                         rsvd_check->rsvd_bits_mask[1][1] = 0;
4280                         break;
4281                 }
4282
4283                 if (is_cpuid_PSE36())
4284                         /* 36bits PSE 4MB page */
4285                         rsvd_check->rsvd_bits_mask[1][1] = rsvd_bits(17, 21);
4286                 else
4287                         /* 32 bits PSE 4MB page */
4288                         rsvd_check->rsvd_bits_mask[1][1] = rsvd_bits(13, 21);
4289                 break;
4290         case PT32E_ROOT_LEVEL:
4291                 rsvd_check->rsvd_bits_mask[0][2] = rsvd_bits(63, 63) |
4292                                                    high_bits_rsvd |
4293                                                    rsvd_bits(5, 8) |
4294                                                    rsvd_bits(1, 2);     /* PDPTE */
4295                 rsvd_check->rsvd_bits_mask[0][1] = high_bits_rsvd;      /* PDE */
4296                 rsvd_check->rsvd_bits_mask[0][0] = high_bits_rsvd;      /* PTE */
4297                 rsvd_check->rsvd_bits_mask[1][1] = high_bits_rsvd |
4298                                                    rsvd_bits(13, 20);   /* large page */
4299                 rsvd_check->rsvd_bits_mask[1][0] =
4300                         rsvd_check->rsvd_bits_mask[0][0];
4301                 break;
4302         case PT64_ROOT_5LEVEL:
4303                 rsvd_check->rsvd_bits_mask[0][4] = high_bits_rsvd |
4304                                                    nonleaf_bit8_rsvd |
4305                                                    rsvd_bits(7, 7);
4306                 rsvd_check->rsvd_bits_mask[1][4] =
4307                         rsvd_check->rsvd_bits_mask[0][4];
4308                 fallthrough;
4309         case PT64_ROOT_4LEVEL:
4310                 rsvd_check->rsvd_bits_mask[0][3] = high_bits_rsvd |
4311                                                    nonleaf_bit8_rsvd |
4312                                                    rsvd_bits(7, 7);
4313                 rsvd_check->rsvd_bits_mask[0][2] = high_bits_rsvd |
4314                                                    gbpages_bit_rsvd;
4315                 rsvd_check->rsvd_bits_mask[0][1] = high_bits_rsvd;
4316                 rsvd_check->rsvd_bits_mask[0][0] = high_bits_rsvd;
4317                 rsvd_check->rsvd_bits_mask[1][3] =
4318                         rsvd_check->rsvd_bits_mask[0][3];
4319                 rsvd_check->rsvd_bits_mask[1][2] = high_bits_rsvd |
4320                                                    gbpages_bit_rsvd |
4321                                                    rsvd_bits(13, 29);
4322                 rsvd_check->rsvd_bits_mask[1][1] = high_bits_rsvd |
4323                                                    rsvd_bits(13, 20); /* large page */
4324                 rsvd_check->rsvd_bits_mask[1][0] =
4325                         rsvd_check->rsvd_bits_mask[0][0];
4326                 break;
4327         }
4328 }
4329
4330 static bool guest_can_use_gbpages(struct kvm_vcpu *vcpu)
4331 {
4332         /*
4333          * If TDP is enabled, let the guest use GBPAGES if they're supported in
4334          * hardware.  The hardware page walker doesn't let KVM disable GBPAGES,
4335          * i.e. won't treat them as reserved, and KVM doesn't redo the GVA->GPA
4336          * walk for performance and complexity reasons.  Not to mention KVM
4337          * _can't_ solve the problem because GVA->GPA walks aren't visible to
4338          * KVM once a TDP translation is installed.  Mimic hardware behavior so
4339          * that KVM's is at least consistent, i.e. doesn't randomly inject #PF.
4340          */
4341         return tdp_enabled ? boot_cpu_has(X86_FEATURE_GBPAGES) :
4342                              guest_cpuid_has(vcpu, X86_FEATURE_GBPAGES);
4343 }
4344
4345 static void reset_rsvds_bits_mask(struct kvm_vcpu *vcpu,
4346                                   struct kvm_mmu *context)
4347 {
4348         __reset_rsvds_bits_mask(&context->guest_rsvd_check,
4349                                 vcpu->arch.reserved_gpa_bits,
4350                                 context->root_level, is_efer_nx(context),
4351                                 guest_can_use_gbpages(vcpu),
4352                                 is_cr4_pse(context),
4353                                 guest_cpuid_is_amd_or_hygon(vcpu));
4354 }
4355
4356 static void
4357 __reset_rsvds_bits_mask_ept(struct rsvd_bits_validate *rsvd_check,
4358                             u64 pa_bits_rsvd, bool execonly)
4359 {
4360         u64 high_bits_rsvd = pa_bits_rsvd & rsvd_bits(0, 51);
4361         u64 bad_mt_xwr;
4362
4363         rsvd_check->rsvd_bits_mask[0][4] = high_bits_rsvd | rsvd_bits(3, 7);
4364         rsvd_check->rsvd_bits_mask[0][3] = high_bits_rsvd | rsvd_bits(3, 7);
4365         rsvd_check->rsvd_bits_mask[0][2] = high_bits_rsvd | rsvd_bits(3, 6);
4366         rsvd_check->rsvd_bits_mask[0][1] = high_bits_rsvd | rsvd_bits(3, 6);
4367         rsvd_check->rsvd_bits_mask[0][0] = high_bits_rsvd;
4368
4369         /* large page */
4370         rsvd_check->rsvd_bits_mask[1][4] = rsvd_check->rsvd_bits_mask[0][4];
4371         rsvd_check->rsvd_bits_mask[1][3] = rsvd_check->rsvd_bits_mask[0][3];
4372         rsvd_check->rsvd_bits_mask[1][2] = high_bits_rsvd | rsvd_bits(12, 29);
4373         rsvd_check->rsvd_bits_mask[1][1] = high_bits_rsvd | rsvd_bits(12, 20);
4374         rsvd_check->rsvd_bits_mask[1][0] = rsvd_check->rsvd_bits_mask[0][0];
4375
4376         bad_mt_xwr = 0xFFull << (2 * 8);        /* bits 3..5 must not be 2 */
4377         bad_mt_xwr |= 0xFFull << (3 * 8);       /* bits 3..5 must not be 3 */
4378         bad_mt_xwr |= 0xFFull << (7 * 8);       /* bits 3..5 must not be 7 */
4379         bad_mt_xwr |= REPEAT_BYTE(1ull << 2);   /* bits 0..2 must not be 010 */
4380         bad_mt_xwr |= REPEAT_BYTE(1ull << 6);   /* bits 0..2 must not be 110 */
4381         if (!execonly) {
4382                 /* bits 0..2 must not be 100 unless VMX capabilities allow it */
4383                 bad_mt_xwr |= REPEAT_BYTE(1ull << 4);
4384         }
4385         rsvd_check->bad_mt_xwr = bad_mt_xwr;
4386 }
4387
4388 static void reset_rsvds_bits_mask_ept(struct kvm_vcpu *vcpu,
4389                 struct kvm_mmu *context, bool execonly)
4390 {
4391         __reset_rsvds_bits_mask_ept(&context->guest_rsvd_check,
4392                                     vcpu->arch.reserved_gpa_bits, execonly);
4393 }
4394
4395 static inline u64 reserved_hpa_bits(void)
4396 {
4397         return rsvd_bits(shadow_phys_bits, 63);
4398 }
4399
4400 /*
4401  * the page table on host is the shadow page table for the page
4402  * table in guest or amd nested guest, its mmu features completely
4403  * follow the features in guest.
4404  */
4405 static void reset_shadow_zero_bits_mask(struct kvm_vcpu *vcpu,
4406                                         struct kvm_mmu *context)
4407 {
4408         /*
4409          * KVM uses NX when TDP is disabled to handle a variety of scenarios,
4410          * notably for huge SPTEs if iTLB multi-hit mitigation is enabled and
4411          * to generate correct permissions for CR0.WP=0/CR4.SMEP=1/EFER.NX=0.
4412          * The iTLB multi-hit workaround can be toggled at any time, so assume
4413          * NX can be used by any non-nested shadow MMU to avoid having to reset
4414          * MMU contexts.  Note, KVM forces EFER.NX=1 when TDP is disabled.
4415          */
4416         bool uses_nx = is_efer_nx(context) || !tdp_enabled;
4417
4418         /* @amd adds a check on bit of SPTEs, which KVM shouldn't use anyways. */
4419         bool is_amd = true;
4420         /* KVM doesn't use 2-level page tables for the shadow MMU. */
4421         bool is_pse = false;
4422         struct rsvd_bits_validate *shadow_zero_check;
4423         int i;
4424
4425         WARN_ON_ONCE(context->shadow_root_level < PT32E_ROOT_LEVEL);
4426
4427         shadow_zero_check = &context->shadow_zero_check;
4428         __reset_rsvds_bits_mask(shadow_zero_check, reserved_hpa_bits(),
4429                                 context->shadow_root_level, uses_nx,
4430                                 guest_can_use_gbpages(vcpu), is_pse, is_amd);
4431
4432         if (!shadow_me_mask)
4433                 return;
4434
4435         for (i = context->shadow_root_level; --i >= 0;) {
4436                 shadow_zero_check->rsvd_bits_mask[0][i] &= ~shadow_me_mask;
4437                 shadow_zero_check->rsvd_bits_mask[1][i] &= ~shadow_me_mask;
4438         }
4439
4440 }
4441
4442 static inline bool boot_cpu_is_amd(void)
4443 {
4444         WARN_ON_ONCE(!tdp_enabled);
4445         return shadow_x_mask == 0;
4446 }
4447
4448 /*
4449  * the direct page table on host, use as much mmu features as
4450  * possible, however, kvm currently does not do execution-protection.
4451  */
4452 static void
4453 reset_tdp_shadow_zero_bits_mask(struct kvm_vcpu *vcpu,
4454                                 struct kvm_mmu *context)
4455 {
4456         struct rsvd_bits_validate *shadow_zero_check;
4457         int i;
4458
4459         shadow_zero_check = &context->shadow_zero_check;
4460
4461         if (boot_cpu_is_amd())
4462                 __reset_rsvds_bits_mask(shadow_zero_check, reserved_hpa_bits(),
4463                                         context->shadow_root_level, false,
4464                                         boot_cpu_has(X86_FEATURE_GBPAGES),
4465                                         false, true);
4466         else
4467                 __reset_rsvds_bits_mask_ept(shadow_zero_check,
4468                                             reserved_hpa_bits(), false);
4469
4470         if (!shadow_me_mask)
4471                 return;
4472
4473         for (i = context->shadow_root_level; --i >= 0;) {
4474                 shadow_zero_check->rsvd_bits_mask[0][i] &= ~shadow_me_mask;
4475                 shadow_zero_check->rsvd_bits_mask[1][i] &= ~shadow_me_mask;
4476         }
4477 }
4478
4479 /*
4480  * as the comments in reset_shadow_zero_bits_mask() except it
4481  * is the shadow page table for intel nested guest.
4482  */
4483 static void
4484 reset_ept_shadow_zero_bits_mask(struct kvm_vcpu *vcpu,
4485                                 struct kvm_mmu *context, bool execonly)
4486 {
4487         __reset_rsvds_bits_mask_ept(&context->shadow_zero_check,
4488                                     reserved_hpa_bits(), execonly);
4489 }
4490
4491 #define BYTE_MASK(access) \
4492         ((1 & (access) ? 2 : 0) | \
4493          (2 & (access) ? 4 : 0) | \
4494          (3 & (access) ? 8 : 0) | \
4495          (4 & (access) ? 16 : 0) | \
4496          (5 & (access) ? 32 : 0) | \
4497          (6 & (access) ? 64 : 0) | \
4498          (7 & (access) ? 128 : 0))
4499
4500
4501 static void update_permission_bitmask(struct kvm_mmu *mmu, bool ept)
4502 {
4503         unsigned byte;
4504
4505         const u8 x = BYTE_MASK(ACC_EXEC_MASK);
4506         const u8 w = BYTE_MASK(ACC_WRITE_MASK);
4507         const u8 u = BYTE_MASK(ACC_USER_MASK);
4508
4509         bool cr4_smep = is_cr4_smep(mmu);
4510         bool cr4_smap = is_cr4_smap(mmu);
4511         bool cr0_wp = is_cr0_wp(mmu);
4512         bool efer_nx = is_efer_nx(mmu);
4513
4514         for (byte = 0; byte < ARRAY_SIZE(mmu->permissions); ++byte) {
4515                 unsigned pfec = byte << 1;
4516
4517                 /*
4518                  * Each "*f" variable has a 1 bit for each UWX value
4519                  * that causes a fault with the given PFEC.
4520                  */
4521
4522                 /* Faults from writes to non-writable pages */
4523                 u8 wf = (pfec & PFERR_WRITE_MASK) ? (u8)~w : 0;
4524                 /* Faults from user mode accesses to supervisor pages */
4525                 u8 uf = (pfec & PFERR_USER_MASK) ? (u8)~u : 0;
4526                 /* Faults from fetches of non-executable pages*/
4527                 u8 ff = (pfec & PFERR_FETCH_MASK) ? (u8)~x : 0;
4528                 /* Faults from kernel mode fetches of user pages */
4529                 u8 smepf = 0;
4530                 /* Faults from kernel mode accesses of user pages */
4531                 u8 smapf = 0;
4532
4533                 if (!ept) {
4534                         /* Faults from kernel mode accesses to user pages */
4535                         u8 kf = (pfec & PFERR_USER_MASK) ? 0 : u;
4536
4537                         /* Not really needed: !nx will cause pte.nx to fault */
4538                         if (!efer_nx)
4539                                 ff = 0;
4540
4541                         /* Allow supervisor writes if !cr0.wp */
4542                         if (!cr0_wp)
4543                                 wf = (pfec & PFERR_USER_MASK) ? wf : 0;
4544
4545                         /* Disallow supervisor fetches of user code if cr4.smep */
4546                         if (cr4_smep)
4547                                 smepf = (pfec & PFERR_FETCH_MASK) ? kf : 0;
4548
4549                         /*
4550                          * SMAP:kernel-mode data accesses from user-mode
4551                          * mappings should fault. A fault is considered
4552                          * as a SMAP violation if all of the following
4553                          * conditions are true:
4554                          *   - X86_CR4_SMAP is set in CR4
4555                          *   - A user page is accessed
4556                          *   - The access is not a fetch
4557                          *   - Page fault in kernel mode
4558                          *   - if CPL = 3 or X86_EFLAGS_AC is clear
4559                          *
4560                          * Here, we cover the first three conditions.
4561                          * The fourth is computed dynamically in permission_fault();
4562                          * PFERR_RSVD_MASK bit will be set in PFEC if the access is
4563                          * *not* subject to SMAP restrictions.
4564                          */
4565                         if (cr4_smap)
4566                                 smapf = (pfec & (PFERR_RSVD_MASK|PFERR_FETCH_MASK)) ? 0 : kf;
4567                 }
4568
4569                 mmu->permissions[byte] = ff | uf | wf | smepf | smapf;
4570         }
4571 }
4572
4573 /*
4574 * PKU is an additional mechanism by which the paging controls access to
4575 * user-mode addresses based on the value in the PKRU register.  Protection
4576 * key violations are reported through a bit in the page fault error code.
4577 * Unlike other bits of the error code, the PK bit is not known at the
4578 * call site of e.g. gva_to_gpa; it must be computed directly in
4579 * permission_fault based on two bits of PKRU, on some machine state (CR4,
4580 * CR0, EFER, CPL), and on other bits of the error code and the page tables.
4581 *
4582 * In particular the following conditions come from the error code, the
4583 * page tables and the machine state:
4584 * - PK is always zero unless CR4.PKE=1 and EFER.LMA=1
4585 * - PK is always zero if RSVD=1 (reserved bit set) or F=1 (instruction fetch)
4586 * - PK is always zero if U=0 in the page tables
4587 * - PKRU.WD is ignored if CR0.WP=0 and the access is a supervisor access.
4588 *
4589 * The PKRU bitmask caches the result of these four conditions.  The error
4590 * code (minus the P bit) and the page table's U bit form an index into the
4591 * PKRU bitmask.  Two bits of the PKRU bitmask are then extracted and ANDed
4592 * with the two bits of the PKRU register corresponding to the protection key.
4593 * For the first three conditions above the bits will be 00, thus masking
4594 * away both AD and WD.  For all reads or if the last condition holds, WD
4595 * only will be masked away.
4596 */
4597 static void update_pkru_bitmask(struct kvm_mmu *mmu)
4598 {
4599         unsigned bit;
4600         bool wp;
4601
4602         mmu->pkru_mask = 0;
4603
4604         if (!is_cr4_pke(mmu))
4605                 return;
4606
4607         wp = is_cr0_wp(mmu);
4608
4609         for (bit = 0; bit < ARRAY_SIZE(mmu->permissions); ++bit) {
4610                 unsigned pfec, pkey_bits;
4611                 bool check_pkey, check_write, ff, uf, wf, pte_user;
4612
4613                 pfec = bit << 1;
4614                 ff = pfec & PFERR_FETCH_MASK;
4615                 uf = pfec & PFERR_USER_MASK;
4616                 wf = pfec & PFERR_WRITE_MASK;
4617
4618                 /* PFEC.RSVD is replaced by ACC_USER_MASK. */
4619                 pte_user = pfec & PFERR_RSVD_MASK;
4620
4621                 /*
4622                  * Only need to check the access which is not an
4623                  * instruction fetch and is to a user page.
4624                  */
4625                 check_pkey = (!ff && pte_user);
4626                 /*
4627                  * write access is controlled by PKRU if it is a
4628                  * user access or CR0.WP = 1.
4629                  */
4630                 check_write = check_pkey && wf && (uf || wp);
4631
4632                 /* PKRU.AD stops both read and write access. */
4633                 pkey_bits = !!check_pkey;
4634                 /* PKRU.WD stops write access. */
4635                 pkey_bits |= (!!check_write) << 1;
4636
4637                 mmu->pkru_mask |= (pkey_bits & 3) << pfec;
4638         }
4639 }
4640
4641 static void reset_guest_paging_metadata(struct kvm_vcpu *vcpu,
4642                                         struct kvm_mmu *mmu)
4643 {
4644         if (!is_cr0_pg(mmu))
4645                 return;
4646
4647         reset_rsvds_bits_mask(vcpu, mmu);
4648         update_permission_bitmask(mmu, false);
4649         update_pkru_bitmask(mmu);
4650 }
4651
4652 static void paging64_init_context(struct kvm_mmu *context)
4653 {
4654         context->page_fault = paging64_page_fault;
4655         context->gva_to_gpa = paging64_gva_to_gpa;
4656         context->sync_page = paging64_sync_page;
4657         context->invlpg = paging64_invlpg;
4658         context->direct_map = false;
4659 }
4660
4661 static void paging32_init_context(struct kvm_mmu *context)
4662 {
4663         context->page_fault = paging32_page_fault;
4664         context->gva_to_gpa = paging32_gva_to_gpa;
4665         context->sync_page = paging32_sync_page;
4666         context->invlpg = paging32_invlpg;
4667         context->direct_map = false;
4668 }
4669
4670 static union kvm_mmu_extended_role kvm_calc_mmu_role_ext(struct kvm_vcpu *vcpu,
4671                                                          struct kvm_mmu_role_regs *regs)
4672 {
4673         union kvm_mmu_extended_role ext = {0};
4674
4675         if (____is_cr0_pg(regs)) {
4676                 ext.cr0_pg = 1;
4677                 ext.cr4_pae = ____is_cr4_pae(regs);
4678                 ext.cr4_smep = ____is_cr4_smep(regs);
4679                 ext.cr4_smap = ____is_cr4_smap(regs);
4680                 ext.cr4_pse = ____is_cr4_pse(regs);
4681
4682                 /* PKEY and LA57 are active iff long mode is active. */
4683                 ext.cr4_pke = ____is_efer_lma(regs) && ____is_cr4_pke(regs);
4684                 ext.cr4_la57 = ____is_efer_lma(regs) && ____is_cr4_la57(regs);
4685                 ext.efer_lma = ____is_efer_lma(regs);
4686         }
4687
4688         ext.valid = 1;
4689
4690         return ext;
4691 }
4692
4693 static union kvm_mmu_role kvm_calc_mmu_role_common(struct kvm_vcpu *vcpu,
4694                                                    struct kvm_mmu_role_regs *regs,
4695                                                    bool base_only)
4696 {
4697         union kvm_mmu_role role = {0};
4698
4699         role.base.access = ACC_ALL;
4700         if (____is_cr0_pg(regs)) {
4701                 role.base.efer_nx = ____is_efer_nx(regs);
4702                 role.base.cr0_wp = ____is_cr0_wp(regs);
4703         }
4704         role.base.smm = is_smm(vcpu);
4705         role.base.guest_mode = is_guest_mode(vcpu);
4706
4707         if (base_only)
4708                 return role;
4709
4710         role.ext = kvm_calc_mmu_role_ext(vcpu, regs);
4711
4712         return role;
4713 }
4714
4715 static inline int kvm_mmu_get_tdp_level(struct kvm_vcpu *vcpu)
4716 {
4717         /* tdp_root_level is architecture forced level, use it if nonzero */
4718         if (tdp_root_level)
4719                 return tdp_root_level;
4720
4721         /* Use 5-level TDP if and only if it's useful/necessary. */
4722         if (max_tdp_level == 5 && cpuid_maxphyaddr(vcpu) <= 48)
4723                 return 4;
4724
4725         return max_tdp_level;
4726 }
4727
4728 static union kvm_mmu_role
4729 kvm_calc_tdp_mmu_root_page_role(struct kvm_vcpu *vcpu,
4730                                 struct kvm_mmu_role_regs *regs, bool base_only)
4731 {
4732         union kvm_mmu_role role = kvm_calc_mmu_role_common(vcpu, regs, base_only);
4733
4734         role.base.ad_disabled = (shadow_accessed_mask == 0);
4735         role.base.level = kvm_mmu_get_tdp_level(vcpu);
4736         role.base.direct = true;
4737         role.base.gpte_is_8_bytes = true;
4738
4739         return role;
4740 }
4741
4742 static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu)
4743 {
4744         struct kvm_mmu *context = &vcpu->arch.root_mmu;
4745         struct kvm_mmu_role_regs regs = vcpu_to_role_regs(vcpu);
4746         union kvm_mmu_role new_role =
4747                 kvm_calc_tdp_mmu_root_page_role(vcpu, &regs, false);
4748
4749         if (new_role.as_u64 == context->mmu_role.as_u64)
4750                 return;
4751
4752         context->mmu_role.as_u64 = new_role.as_u64;
4753         context->page_fault = kvm_tdp_page_fault;
4754         context->sync_page = nonpaging_sync_page;
4755         context->invlpg = NULL;
4756         context->shadow_root_level = kvm_mmu_get_tdp_level(vcpu);
4757         context->direct_map = true;
4758         context->get_guest_pgd = get_cr3;
4759         context->get_pdptr = kvm_pdptr_read;
4760         context->inject_page_fault = kvm_inject_page_fault;
4761         context->root_level = role_regs_to_root_level(&regs);
4762
4763         if (!is_cr0_pg(context))
4764                 context->gva_to_gpa = nonpaging_gva_to_gpa;
4765         else if (is_cr4_pae(context))
4766                 context->gva_to_gpa = paging64_gva_to_gpa;
4767         else
4768                 context->gva_to_gpa = paging32_gva_to_gpa;
4769
4770         reset_guest_paging_metadata(vcpu, context);
4771         reset_tdp_shadow_zero_bits_mask(vcpu, context);
4772 }
4773
4774 static union kvm_mmu_role
4775 kvm_calc_shadow_root_page_role_common(struct kvm_vcpu *vcpu,
4776                                       struct kvm_mmu_role_regs *regs, bool base_only)
4777 {
4778         union kvm_mmu_role role = kvm_calc_mmu_role_common(vcpu, regs, base_only);
4779
4780         role.base.smep_andnot_wp = role.ext.cr4_smep && !____is_cr0_wp(regs);
4781         role.base.smap_andnot_wp = role.ext.cr4_smap && !____is_cr0_wp(regs);
4782         role.base.gpte_is_8_bytes = ____is_cr0_pg(regs) && ____is_cr4_pae(regs);
4783
4784         return role;
4785 }
4786
4787 static union kvm_mmu_role
4788 kvm_calc_shadow_mmu_root_page_role(struct kvm_vcpu *vcpu,
4789                                    struct kvm_mmu_role_regs *regs, bool base_only)
4790 {
4791         union kvm_mmu_role role =
4792                 kvm_calc_shadow_root_page_role_common(vcpu, regs, base_only);
4793
4794         role.base.direct = !____is_cr0_pg(regs);
4795
4796         if (!____is_efer_lma(regs))
4797                 role.base.level = PT32E_ROOT_LEVEL;
4798         else if (____is_cr4_la57(regs))
4799                 role.base.level = PT64_ROOT_5LEVEL;
4800         else
4801                 role.base.level = PT64_ROOT_4LEVEL;
4802
4803         return role;
4804 }
4805
4806 static void shadow_mmu_init_context(struct kvm_vcpu *vcpu, struct kvm_mmu *context,
4807                                     struct kvm_mmu_role_regs *regs,
4808                                     union kvm_mmu_role new_role)
4809 {
4810         if (new_role.as_u64 == context->mmu_role.as_u64)
4811                 return;
4812
4813         context->mmu_role.as_u64 = new_role.as_u64;
4814
4815         if (!is_cr0_pg(context))
4816                 nonpaging_init_context(context);
4817         else if (is_cr4_pae(context))
4818                 paging64_init_context(context);
4819         else
4820                 paging32_init_context(context);
4821         context->root_level = role_regs_to_root_level(regs);
4822
4823         reset_guest_paging_metadata(vcpu, context);
4824         context->shadow_root_level = new_role.base.level;
4825
4826         reset_shadow_zero_bits_mask(vcpu, context);
4827 }
4828
4829 static void kvm_init_shadow_mmu(struct kvm_vcpu *vcpu,
4830                                 struct kvm_mmu_role_regs *regs)
4831 {
4832         struct kvm_mmu *context = &vcpu->arch.root_mmu;
4833         union kvm_mmu_role new_role =
4834                 kvm_calc_shadow_mmu_root_page_role(vcpu, regs, false);
4835
4836         shadow_mmu_init_context(vcpu, context, regs, new_role);
4837 }
4838
4839 static union kvm_mmu_role
4840 kvm_calc_shadow_npt_root_page_role(struct kvm_vcpu *vcpu,
4841                                    struct kvm_mmu_role_regs *regs)
4842 {
4843         union kvm_mmu_role role =
4844                 kvm_calc_shadow_root_page_role_common(vcpu, regs, false);
4845
4846         role.base.direct = false;
4847         role.base.level = kvm_mmu_get_tdp_level(vcpu);
4848
4849         return role;
4850 }
4851
4852 void kvm_init_shadow_npt_mmu(struct kvm_vcpu *vcpu, unsigned long cr0,
4853                              unsigned long cr4, u64 efer, gpa_t nested_cr3)
4854 {
4855         struct kvm_mmu *context = &vcpu->arch.guest_mmu;
4856         struct kvm_mmu_role_regs regs = {
4857                 .cr0 = cr0,
4858                 .cr4 = cr4,
4859                 .efer = efer,
4860         };
4861         union kvm_mmu_role new_role;
4862
4863         new_role = kvm_calc_shadow_npt_root_page_role(vcpu, &regs);
4864
4865         __kvm_mmu_new_pgd(vcpu, nested_cr3, new_role.base);
4866
4867         shadow_mmu_init_context(vcpu, context, &regs, new_role);
4868 }
4869 EXPORT_SYMBOL_GPL(kvm_init_shadow_npt_mmu);
4870
4871 static union kvm_mmu_role
4872 kvm_calc_shadow_ept_root_page_role(struct kvm_vcpu *vcpu, bool accessed_dirty,
4873                                    bool execonly, u8 level)
4874 {
4875         union kvm_mmu_role role = {0};
4876
4877         /* SMM flag is inherited from root_mmu */
4878         role.base.smm = vcpu->arch.root_mmu.mmu_role.base.smm;
4879
4880         role.base.level = level;
4881         role.base.gpte_is_8_bytes = true;
4882         role.base.direct = false;
4883         role.base.ad_disabled = !accessed_dirty;
4884         role.base.guest_mode = true;
4885         role.base.access = ACC_ALL;
4886
4887         /* EPT, and thus nested EPT, does not consume CR0, CR4, nor EFER. */
4888         role.ext.word = 0;
4889         role.ext.execonly = execonly;
4890         role.ext.valid = 1;
4891
4892         return role;
4893 }
4894
4895 void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly,
4896                              bool accessed_dirty, gpa_t new_eptp)
4897 {
4898         struct kvm_mmu *context = &vcpu->arch.guest_mmu;
4899         u8 level = vmx_eptp_page_walk_level(new_eptp);
4900         union kvm_mmu_role new_role =
4901                 kvm_calc_shadow_ept_root_page_role(vcpu, accessed_dirty,
4902                                                    execonly, level);
4903
4904         __kvm_mmu_new_pgd(vcpu, new_eptp, new_role.base);
4905
4906         if (new_role.as_u64 == context->mmu_role.as_u64)
4907                 return;
4908
4909         context->mmu_role.as_u64 = new_role.as_u64;
4910
4911         context->shadow_root_level = level;
4912
4913         context->ept_ad = accessed_dirty;
4914         context->page_fault = ept_page_fault;
4915         context->gva_to_gpa = ept_gva_to_gpa;
4916         context->sync_page = ept_sync_page;
4917         context->invlpg = ept_invlpg;
4918         context->root_level = level;
4919         context->direct_map = false;
4920
4921         update_permission_bitmask(context, true);
4922         update_pkru_bitmask(context);
4923         reset_rsvds_bits_mask_ept(vcpu, context, execonly);
4924         reset_ept_shadow_zero_bits_mask(vcpu, context, execonly);
4925 }
4926 EXPORT_SYMBOL_GPL(kvm_init_shadow_ept_mmu);
4927
4928 static void init_kvm_softmmu(struct kvm_vcpu *vcpu)
4929 {
4930         struct kvm_mmu *context = &vcpu->arch.root_mmu;
4931         struct kvm_mmu_role_regs regs = vcpu_to_role_regs(vcpu);
4932
4933         kvm_init_shadow_mmu(vcpu, &regs);
4934
4935         context->get_guest_pgd     = get_cr3;
4936         context->get_pdptr         = kvm_pdptr_read;
4937         context->inject_page_fault = kvm_inject_page_fault;
4938 }
4939
4940 static union kvm_mmu_role
4941 kvm_calc_nested_mmu_role(struct kvm_vcpu *vcpu, struct kvm_mmu_role_regs *regs)
4942 {
4943         union kvm_mmu_role role;
4944
4945         role = kvm_calc_shadow_root_page_role_common(vcpu, regs, false);
4946
4947         /*
4948          * Nested MMUs are used only for walking L2's gva->gpa, they never have
4949          * shadow pages of their own and so "direct" has no meaning.   Set it
4950          * to "true" to try to detect bogus usage of the nested MMU.
4951          */
4952         role.base.direct = true;
4953         role.base.level = role_regs_to_root_level(regs);
4954         return role;
4955 }
4956
4957 static void init_kvm_nested_mmu(struct kvm_vcpu *vcpu)
4958 {
4959         struct kvm_mmu_role_regs regs = vcpu_to_role_regs(vcpu);
4960         union kvm_mmu_role new_role = kvm_calc_nested_mmu_role(vcpu, &regs);
4961         struct kvm_mmu *g_context = &vcpu->arch.nested_mmu;
4962
4963         if (new_role.as_u64 == g_context->mmu_role.as_u64)
4964                 return;
4965
4966         g_context->mmu_role.as_u64 = new_role.as_u64;
4967         g_context->get_guest_pgd     = get_cr3;
4968         g_context->get_pdptr         = kvm_pdptr_read;
4969         g_context->inject_page_fault = kvm_inject_page_fault;
4970         g_context->root_level        = new_role.base.level;
4971
4972         /*
4973          * L2 page tables are never shadowed, so there is no need to sync
4974          * SPTEs.
4975          */
4976         g_context->invlpg            = NULL;
4977
4978         /*
4979          * Note that arch.mmu->gva_to_gpa translates l2_gpa to l1_gpa using
4980          * L1's nested page tables (e.g. EPT12). The nested translation
4981          * of l2_gva to l1_gpa is done by arch.nested_mmu.gva_to_gpa using
4982          * L2's page tables as the first level of translation and L1's
4983          * nested page tables as the second level of translation. Basically
4984          * the gva_to_gpa functions between mmu and nested_mmu are swapped.
4985          */
4986         if (!is_paging(vcpu))
4987                 g_context->gva_to_gpa = nonpaging_gva_to_gpa_nested;
4988         else if (is_long_mode(vcpu))
4989                 g_context->gva_to_gpa = paging64_gva_to_gpa_nested;
4990         else if (is_pae(vcpu))
4991                 g_context->gva_to_gpa = paging64_gva_to_gpa_nested;
4992         else
4993                 g_context->gva_to_gpa = paging32_gva_to_gpa_nested;
4994
4995         reset_guest_paging_metadata(vcpu, g_context);
4996 }
4997
4998 void kvm_init_mmu(struct kvm_vcpu *vcpu)
4999 {
5000         if (mmu_is_nested(vcpu))
5001                 init_kvm_nested_mmu(vcpu);
5002         else if (tdp_enabled)
5003                 init_kvm_tdp_mmu(vcpu);
5004         else
5005                 init_kvm_softmmu(vcpu);
5006 }
5007 EXPORT_SYMBOL_GPL(kvm_init_mmu);
5008
5009 static union kvm_mmu_page_role
5010 kvm_mmu_calc_root_page_role(struct kvm_vcpu *vcpu)
5011 {
5012         struct kvm_mmu_role_regs regs = vcpu_to_role_regs(vcpu);
5013         union kvm_mmu_role role;
5014
5015         if (tdp_enabled)
5016                 role = kvm_calc_tdp_mmu_root_page_role(vcpu, &regs, true);
5017         else
5018                 role = kvm_calc_shadow_mmu_root_page_role(vcpu, &regs, true);
5019
5020         return role.base;
5021 }
5022
5023 void kvm_mmu_after_set_cpuid(struct kvm_vcpu *vcpu)
5024 {
5025         /*
5026          * Invalidate all MMU roles to force them to reinitialize as CPUID
5027          * information is factored into reserved bit calculations.
5028          *
5029          * Correctly handling multiple vCPU models with respect to paging and
5030          * physical address properties) in a single VM would require tracking
5031          * all relevant CPUID information in kvm_mmu_page_role. That is very
5032          * undesirable as it would increase the memory requirements for
5033          * gfn_track (see struct kvm_mmu_page_role comments).  For now that
5034          * problem is swept under the rug; KVM's CPUID API is horrific and
5035          * it's all but impossible to solve it without introducing a new API.
5036          */
5037         vcpu->arch.root_mmu.mmu_role.ext.valid = 0;
5038         vcpu->arch.guest_mmu.mmu_role.ext.valid = 0;
5039         vcpu->arch.nested_mmu.mmu_role.ext.valid = 0;
5040         kvm_mmu_reset_context(vcpu);
5041
5042         /*
5043          * Changing guest CPUID after KVM_RUN is forbidden, see the comment in
5044          * kvm_arch_vcpu_ioctl().
5045          */
5046         KVM_BUG_ON(vcpu->arch.last_vmentry_cpu != -1, vcpu->kvm);
5047 }
5048
5049 void kvm_mmu_reset_context(struct kvm_vcpu *vcpu)
5050 {
5051         kvm_mmu_unload(vcpu);
5052         kvm_init_mmu(vcpu);
5053 }
5054 EXPORT_SYMBOL_GPL(kvm_mmu_reset_context);
5055
5056 int kvm_mmu_load(struct kvm_vcpu *vcpu)
5057 {
5058         int r;
5059
5060         r = mmu_topup_memory_caches(vcpu, !vcpu->arch.mmu->direct_map);
5061         if (r)
5062                 goto out;
5063         r = mmu_alloc_special_roots(vcpu);
5064         if (r)
5065                 goto out;
5066         if (vcpu->arch.mmu->direct_map)
5067                 r = mmu_alloc_direct_roots(vcpu);
5068         else
5069                 r = mmu_alloc_shadow_roots(vcpu);
5070         if (r)
5071                 goto out;
5072
5073         kvm_mmu_sync_roots(vcpu);
5074
5075         kvm_mmu_load_pgd(vcpu);
5076         static_call(kvm_x86_tlb_flush_current)(vcpu);
5077 out:
5078         return r;
5079 }
5080
5081 void kvm_mmu_unload(struct kvm_vcpu *vcpu)
5082 {
5083         kvm_mmu_free_roots(vcpu, &vcpu->arch.root_mmu, KVM_MMU_ROOTS_ALL);
5084         WARN_ON(VALID_PAGE(vcpu->arch.root_mmu.root_hpa));
5085         kvm_mmu_free_roots(vcpu, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL);
5086         WARN_ON(VALID_PAGE(vcpu->arch.guest_mmu.root_hpa));
5087 }
5088
5089 static bool need_remote_flush(u64 old, u64 new)
5090 {
5091         if (!is_shadow_present_pte(old))
5092                 return false;
5093         if (!is_shadow_present_pte(new))
5094                 return true;
5095         if ((old ^ new) & PT64_BASE_ADDR_MASK)
5096                 return true;
5097         old ^= shadow_nx_mask;
5098         new ^= shadow_nx_mask;
5099         return (old & ~new & PT64_PERM_MASK) != 0;
5100 }
5101
5102 static u64 mmu_pte_write_fetch_gpte(struct kvm_vcpu *vcpu, gpa_t *gpa,
5103                                     int *bytes)
5104 {
5105         u64 gentry = 0;
5106         int r;
5107
5108         /*
5109          * Assume that the pte write on a page table of the same type
5110          * as the current vcpu paging mode since we update the sptes only
5111          * when they have the same mode.
5112          */
5113         if (is_pae(vcpu) && *bytes == 4) {
5114                 /* Handle a 32-bit guest writing two halves of a 64-bit gpte */
5115                 *gpa &= ~(gpa_t)7;
5116                 *bytes = 8;
5117         }
5118
5119         if (*bytes == 4 || *bytes == 8) {
5120                 r = kvm_vcpu_read_guest_atomic(vcpu, *gpa, &gentry, *bytes);
5121                 if (r)
5122                         gentry = 0;
5123         }
5124
5125         return gentry;
5126 }
5127
5128 /*
5129  * If we're seeing too many writes to a page, it may no longer be a page table,
5130  * or we may be forking, in which case it is better to unmap the page.
5131  */
5132 static bool detect_write_flooding(struct kvm_mmu_page *sp)
5133 {
5134         /*
5135          * Skip write-flooding detected for the sp whose level is 1, because
5136          * it can become unsync, then the guest page is not write-protected.
5137          */
5138         if (sp->role.level == PG_LEVEL_4K)
5139                 return false;
5140
5141         atomic_inc(&sp->write_flooding_count);
5142         return atomic_read(&sp->write_flooding_count) >= 3;
5143 }
5144
5145 /*
5146  * Misaligned accesses are too much trouble to fix up; also, they usually
5147  * indicate a page is not used as a page table.
5148  */
5149 static bool detect_write_misaligned(struct kvm_mmu_page *sp, gpa_t gpa,
5150                                     int bytes)
5151 {
5152         unsigned offset, pte_size, misaligned;
5153
5154         pgprintk("misaligned: gpa %llx bytes %d role %x\n",
5155                  gpa, bytes, sp->role.word);
5156
5157         offset = offset_in_page(gpa);
5158         pte_size = sp->role.gpte_is_8_bytes ? 8 : 4;
5159
5160         /*
5161          * Sometimes, the OS only writes the last one bytes to update status
5162          * bits, for example, in linux, andb instruction is used in clear_bit().
5163          */
5164         if (!(offset & (pte_size - 1)) && bytes == 1)
5165                 return false;
5166
5167         misaligned = (offset ^ (offset + bytes - 1)) & ~(pte_size - 1);
5168         misaligned |= bytes < 4;
5169
5170         return misaligned;
5171 }
5172
5173 static u64 *get_written_sptes(struct kvm_mmu_page *sp, gpa_t gpa, int *nspte)
5174 {
5175         unsigned page_offset, quadrant;
5176         u64 *spte;
5177         int level;
5178
5179         page_offset = offset_in_page(gpa);
5180         level = sp->role.level;
5181         *nspte = 1;
5182         if (!sp->role.gpte_is_8_bytes) {
5183                 page_offset <<= 1;      /* 32->64 */
5184                 /*
5185                  * A 32-bit pde maps 4MB while the shadow pdes map
5186                  * only 2MB.  So we need to double the offset again
5187                  * and zap two pdes instead of one.
5188                  */
5189                 if (level == PT32_ROOT_LEVEL) {
5190                         page_offset &= ~7; /* kill rounding error */
5191                         page_offset <<= 1;
5192                         *nspte = 2;
5193                 }
5194                 quadrant = page_offset >> PAGE_SHIFT;
5195                 page_offset &= ~PAGE_MASK;
5196                 if (quadrant != sp->role.quadrant)
5197                         return NULL;
5198         }
5199
5200         spte = &sp->spt[page_offset / sizeof(*spte)];
5201         return spte;
5202 }
5203
5204 static void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
5205                               const u8 *new, int bytes,
5206                               struct kvm_page_track_notifier_node *node)
5207 {
5208         gfn_t gfn = gpa >> PAGE_SHIFT;
5209         struct kvm_mmu_page *sp;
5210         LIST_HEAD(invalid_list);
5211         u64 entry, gentry, *spte;
5212         int npte;
5213         bool flush = false;
5214
5215         /*
5216          * If we don't have indirect shadow pages, it means no page is
5217          * write-protected, so we can exit simply.
5218          */
5219         if (!READ_ONCE(vcpu->kvm->arch.indirect_shadow_pages))
5220                 return;
5221
5222         pgprintk("%s: gpa %llx bytes %d\n", __func__, gpa, bytes);
5223
5224         /*
5225          * No need to care whether allocation memory is successful
5226          * or not since pte prefetch is skipped if it does not have
5227          * enough objects in the cache.
5228          */
5229         mmu_topup_memory_caches(vcpu, true);
5230
5231         write_lock(&vcpu->kvm->mmu_lock);
5232
5233         gentry = mmu_pte_write_fetch_gpte(vcpu, &gpa, &bytes);
5234
5235         ++vcpu->kvm->stat.mmu_pte_write;
5236         kvm_mmu_audit(vcpu, AUDIT_PRE_PTE_WRITE);
5237
5238         for_each_gfn_indirect_valid_sp(vcpu->kvm, sp, gfn) {
5239                 if (detect_write_misaligned(sp, gpa, bytes) ||
5240                       detect_write_flooding(sp)) {
5241                         kvm_mmu_prepare_zap_page(vcpu->kvm, sp, &invalid_list);
5242                         ++vcpu->kvm->stat.mmu_flooded;
5243                         continue;
5244                 }
5245
5246                 spte = get_written_sptes(sp, gpa, &npte);
5247                 if (!spte)
5248                         continue;
5249
5250                 while (npte--) {
5251                         entry = *spte;
5252                         mmu_page_zap_pte(vcpu->kvm, sp, spte, NULL);
5253                         if (gentry && sp->role.level != PG_LEVEL_4K)
5254                                 ++vcpu->kvm->stat.mmu_pde_zapped;
5255                         if (need_remote_flush(entry, *spte))
5256                                 flush = true;
5257                         ++spte;
5258                 }
5259         }
5260         kvm_mmu_remote_flush_or_zap(vcpu->kvm, &invalid_list, flush);
5261         kvm_mmu_audit(vcpu, AUDIT_POST_PTE_WRITE);
5262         write_unlock(&vcpu->kvm->mmu_lock);
5263 }
5264
5265 int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 error_code,
5266                        void *insn, int insn_len)
5267 {
5268         int r, emulation_type = EMULTYPE_PF;
5269         bool direct = vcpu->arch.mmu->direct_map;
5270
5271         if (WARN_ON(!VALID_PAGE(vcpu->arch.mmu->root_hpa)))
5272                 return RET_PF_RETRY;
5273
5274         r = RET_PF_INVALID;
5275         if (unlikely(error_code & PFERR_RSVD_MASK)) {
5276                 r = handle_mmio_page_fault(vcpu, cr2_or_gpa, direct);
5277                 if (r == RET_PF_EMULATE)
5278                         goto emulate;
5279         }
5280
5281         if (r == RET_PF_INVALID) {
5282                 r = kvm_mmu_do_page_fault(vcpu, cr2_or_gpa,
5283                                           lower_32_bits(error_code), false);
5284                 if (KVM_BUG_ON(r == RET_PF_INVALID, vcpu->kvm))
5285                         return -EIO;
5286         }
5287
5288         if (r < 0)
5289                 return r;
5290         if (r != RET_PF_EMULATE)
5291                 return 1;
5292
5293         /*
5294          * Before emulating the instruction, check if the error code
5295          * was due to a RO violation while translating the guest page.
5296          * This can occur when using nested virtualization with nested
5297          * paging in both guests. If true, we simply unprotect the page
5298          * and resume the guest.
5299          */
5300         if (vcpu->arch.mmu->direct_map &&
5301             (error_code & PFERR_NESTED_GUEST_PAGE) == PFERR_NESTED_GUEST_PAGE) {
5302                 kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(cr2_or_gpa));
5303                 return 1;
5304         }
5305
5306         /*
5307          * vcpu->arch.mmu.page_fault returned RET_PF_EMULATE, but we can still
5308          * optimistically try to just unprotect the page and let the processor
5309          * re-execute the instruction that caused the page fault.  Do not allow
5310          * retrying MMIO emulation, as it's not only pointless but could also
5311          * cause us to enter an infinite loop because the processor will keep
5312          * faulting on the non-existent MMIO address.  Retrying an instruction
5313          * from a nested guest is also pointless and dangerous as we are only
5314          * explicitly shadowing L1's page tables, i.e. unprotecting something
5315          * for L1 isn't going to magically fix whatever issue cause L2 to fail.
5316          */
5317         if (!mmio_info_in_cache(vcpu, cr2_or_gpa, direct) && !is_guest_mode(vcpu))
5318                 emulation_type |= EMULTYPE_ALLOW_RETRY_PF;
5319 emulate:
5320         return x86_emulate_instruction(vcpu, cr2_or_gpa, emulation_type, insn,
5321                                        insn_len);
5322 }
5323 EXPORT_SYMBOL_GPL(kvm_mmu_page_fault);
5324
5325 void kvm_mmu_invalidate_gva(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
5326                             gva_t gva, hpa_t root_hpa)
5327 {
5328         int i;
5329
5330         /* It's actually a GPA for vcpu->arch.guest_mmu.  */
5331         if (mmu != &vcpu->arch.guest_mmu) {
5332                 /* INVLPG on a non-canonical address is a NOP according to the SDM.  */
5333                 if (is_noncanonical_address(gva, vcpu))
5334                         return;
5335
5336                 static_call(kvm_x86_tlb_flush_gva)(vcpu, gva);
5337         }
5338
5339         if (!mmu->invlpg)
5340                 return;
5341
5342         if (root_hpa == INVALID_PAGE) {
5343                 mmu->invlpg(vcpu, gva, mmu->root_hpa);
5344
5345                 /*
5346                  * INVLPG is required to invalidate any global mappings for the VA,
5347                  * irrespective of PCID. Since it would take us roughly similar amount
5348                  * of work to determine whether any of the prev_root mappings of the VA
5349                  * is marked global, or to just sync it blindly, so we might as well
5350                  * just always sync it.
5351                  *
5352                  * Mappings not reachable via the current cr3 or the prev_roots will be
5353                  * synced when switching to that cr3, so nothing needs to be done here
5354                  * for them.
5355                  */
5356                 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
5357                         if (VALID_PAGE(mmu->prev_roots[i].hpa))
5358                                 mmu->invlpg(vcpu, gva, mmu->prev_roots[i].hpa);
5359         } else {
5360                 mmu->invlpg(vcpu, gva, root_hpa);
5361         }
5362 }
5363
5364 void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
5365 {
5366         kvm_mmu_invalidate_gva(vcpu, vcpu->arch.walk_mmu, gva, INVALID_PAGE);
5367         ++vcpu->stat.invlpg;
5368 }
5369 EXPORT_SYMBOL_GPL(kvm_mmu_invlpg);
5370
5371
5372 void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid)
5373 {
5374         struct kvm_mmu *mmu = vcpu->arch.mmu;
5375         bool tlb_flush = false;
5376         uint i;
5377
5378         if (pcid == kvm_get_active_pcid(vcpu)) {
5379                 mmu->invlpg(vcpu, gva, mmu->root_hpa);
5380                 tlb_flush = true;
5381         }
5382
5383         for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) {
5384                 if (VALID_PAGE(mmu->prev_roots[i].hpa) &&
5385                     pcid == kvm_get_pcid(vcpu, mmu->prev_roots[i].pgd)) {
5386                         mmu->invlpg(vcpu, gva, mmu->prev_roots[i].hpa);
5387                         tlb_flush = true;
5388                 }
5389         }
5390
5391         if (tlb_flush)
5392                 static_call(kvm_x86_tlb_flush_gva)(vcpu, gva);
5393
5394         ++vcpu->stat.invlpg;
5395
5396         /*
5397          * Mappings not reachable via the current cr3 or the prev_roots will be
5398          * synced when switching to that cr3, so nothing needs to be done here
5399          * for them.
5400          */
5401 }
5402
5403 void kvm_configure_mmu(bool enable_tdp, int tdp_forced_root_level,
5404                        int tdp_max_root_level, int tdp_huge_page_level)
5405 {
5406         tdp_enabled = enable_tdp;
5407         tdp_root_level = tdp_forced_root_level;
5408         max_tdp_level = tdp_max_root_level;
5409
5410         /*
5411          * max_huge_page_level reflects KVM's MMU capabilities irrespective
5412          * of kernel support, e.g. KVM may be capable of using 1GB pages when
5413          * the kernel is not.  But, KVM never creates a page size greater than
5414          * what is used by the kernel for any given HVA, i.e. the kernel's
5415          * capabilities are ultimately consulted by kvm_mmu_hugepage_adjust().
5416          */
5417         if (tdp_enabled)
5418                 max_huge_page_level = tdp_huge_page_level;
5419         else if (boot_cpu_has(X86_FEATURE_GBPAGES))
5420                 max_huge_page_level = PG_LEVEL_1G;
5421         else
5422                 max_huge_page_level = PG_LEVEL_2M;
5423 }
5424 EXPORT_SYMBOL_GPL(kvm_configure_mmu);
5425
5426 /* The return value indicates if tlb flush on all vcpus is needed. */
5427 typedef bool (*slot_level_handler) (struct kvm *kvm,
5428                                     struct kvm_rmap_head *rmap_head,
5429                                     const struct kvm_memory_slot *slot);
5430
5431 /* The caller should hold mmu-lock before calling this function. */
5432 static __always_inline bool
5433 slot_handle_level_range(struct kvm *kvm, const struct kvm_memory_slot *memslot,
5434                         slot_level_handler fn, int start_level, int end_level,
5435                         gfn_t start_gfn, gfn_t end_gfn, bool flush_on_yield,
5436                         bool flush)
5437 {
5438         struct slot_rmap_walk_iterator iterator;
5439
5440         for_each_slot_rmap_range(memslot, start_level, end_level, start_gfn,
5441                         end_gfn, &iterator) {
5442                 if (iterator.rmap)
5443                         flush |= fn(kvm, iterator.rmap, memslot);
5444
5445                 if (need_resched() || rwlock_needbreak(&kvm->mmu_lock)) {
5446                         if (flush && flush_on_yield) {
5447                                 kvm_flush_remote_tlbs_with_address(kvm,
5448                                                 start_gfn,
5449                                                 iterator.gfn - start_gfn + 1);
5450                                 flush = false;
5451                         }
5452                         cond_resched_rwlock_write(&kvm->mmu_lock);
5453                 }
5454         }
5455
5456         return flush;
5457 }
5458
5459 static __always_inline bool
5460 slot_handle_level(struct kvm *kvm, const struct kvm_memory_slot *memslot,
5461                   slot_level_handler fn, int start_level, int end_level,
5462                   bool flush_on_yield)
5463 {
5464         return slot_handle_level_range(kvm, memslot, fn, start_level,
5465                         end_level, memslot->base_gfn,
5466                         memslot->base_gfn + memslot->npages - 1,
5467                         flush_on_yield, false);
5468 }
5469
5470 static __always_inline bool
5471 slot_handle_level_4k(struct kvm *kvm, const struct kvm_memory_slot *memslot,
5472                      slot_level_handler fn, bool flush_on_yield)
5473 {
5474         return slot_handle_level(kvm, memslot, fn, PG_LEVEL_4K,
5475                                  PG_LEVEL_4K, flush_on_yield);
5476 }
5477
5478 static void free_mmu_pages(struct kvm_mmu *mmu)
5479 {
5480         if (!tdp_enabled && mmu->pae_root)
5481                 set_memory_encrypted((unsigned long)mmu->pae_root, 1);
5482         free_page((unsigned long)mmu->pae_root);
5483         free_page((unsigned long)mmu->pml4_root);
5484         free_page((unsigned long)mmu->pml5_root);
5485 }
5486
5487 static int __kvm_mmu_create(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu)
5488 {
5489         struct page *page;
5490         int i;
5491
5492         mmu->root_hpa = INVALID_PAGE;
5493         mmu->root_pgd = 0;
5494         mmu->translate_gpa = translate_gpa;
5495         for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
5496                 mmu->prev_roots[i] = KVM_MMU_ROOT_INFO_INVALID;
5497
5498         /*
5499          * When using PAE paging, the four PDPTEs are treated as 'root' pages,
5500          * while the PDP table is a per-vCPU construct that's allocated at MMU
5501          * creation.  When emulating 32-bit mode, cr3 is only 32 bits even on
5502          * x86_64.  Therefore we need to allocate the PDP table in the first
5503          * 4GB of memory, which happens to fit the DMA32 zone.  TDP paging
5504          * generally doesn't use PAE paging and can skip allocating the PDP
5505          * table.  The main exception, handled here, is SVM's 32-bit NPT.  The
5506          * other exception is for shadowing L1's 32-bit or PAE NPT on 64-bit
5507          * KVM; that horror is handled on-demand by mmu_alloc_shadow_roots().
5508          */
5509         if (tdp_enabled && kvm_mmu_get_tdp_level(vcpu) > PT32E_ROOT_LEVEL)
5510                 return 0;
5511
5512         page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_DMA32);
5513         if (!page)
5514                 return -ENOMEM;
5515
5516         mmu->pae_root = page_address(page);
5517
5518         /*
5519          * CR3 is only 32 bits when PAE paging is used, thus it's impossible to
5520          * get the CPU to treat the PDPTEs as encrypted.  Decrypt the page so
5521          * that KVM's writes and the CPU's reads get along.  Note, this is
5522          * only necessary when using shadow paging, as 64-bit NPT can get at
5523          * the C-bit even when shadowing 32-bit NPT, and SME isn't supported
5524          * by 32-bit kernels (when KVM itself uses 32-bit NPT).
5525          */
5526         if (!tdp_enabled)
5527                 set_memory_decrypted((unsigned long)mmu->pae_root, 1);
5528         else
5529                 WARN_ON_ONCE(shadow_me_mask);
5530
5531         for (i = 0; i < 4; ++i)
5532                 mmu->pae_root[i] = INVALID_PAE_ROOT;
5533
5534         return 0;
5535 }
5536
5537 int kvm_mmu_create(struct kvm_vcpu *vcpu)
5538 {
5539         int ret;
5540
5541         vcpu->arch.mmu_pte_list_desc_cache.kmem_cache = pte_list_desc_cache;
5542         vcpu->arch.mmu_pte_list_desc_cache.gfp_zero = __GFP_ZERO;
5543
5544         vcpu->arch.mmu_page_header_cache.kmem_cache = mmu_page_header_cache;
5545         vcpu->arch.mmu_page_header_cache.gfp_zero = __GFP_ZERO;
5546
5547         vcpu->arch.mmu_shadow_page_cache.gfp_zero = __GFP_ZERO;
5548
5549         vcpu->arch.mmu = &vcpu->arch.root_mmu;
5550         vcpu->arch.walk_mmu = &vcpu->arch.root_mmu;
5551
5552         vcpu->arch.nested_mmu.translate_gpa = translate_nested_gpa;
5553
5554         ret = __kvm_mmu_create(vcpu, &vcpu->arch.guest_mmu);
5555         if (ret)
5556                 return ret;
5557
5558         ret = __kvm_mmu_create(vcpu, &vcpu->arch.root_mmu);
5559         if (ret)
5560                 goto fail_allocate_root;
5561
5562         return ret;
5563  fail_allocate_root:
5564         free_mmu_pages(&vcpu->arch.guest_mmu);
5565         return ret;
5566 }
5567
5568 #define BATCH_ZAP_PAGES 10
5569 static void kvm_zap_obsolete_pages(struct kvm *kvm)
5570 {
5571         struct kvm_mmu_page *sp, *node;
5572         int nr_zapped, batch = 0;
5573
5574 restart:
5575         list_for_each_entry_safe_reverse(sp, node,
5576               &kvm->arch.active_mmu_pages, link) {
5577                 /*
5578                  * No obsolete valid page exists before a newly created page
5579                  * since active_mmu_pages is a FIFO list.
5580                  */
5581                 if (!is_obsolete_sp(kvm, sp))
5582                         break;
5583
5584                 /*
5585                  * Invalid pages should never land back on the list of active
5586                  * pages.  Skip the bogus page, otherwise we'll get stuck in an
5587                  * infinite loop if the page gets put back on the list (again).
5588                  */
5589                 if (WARN_ON(sp->role.invalid))
5590                         continue;
5591
5592                 /*
5593                  * No need to flush the TLB since we're only zapping shadow
5594                  * pages with an obsolete generation number and all vCPUS have
5595                  * loaded a new root, i.e. the shadow pages being zapped cannot
5596                  * be in active use by the guest.
5597                  */
5598                 if (batch >= BATCH_ZAP_PAGES &&
5599                     cond_resched_rwlock_write(&kvm->mmu_lock)) {
5600                         batch = 0;
5601                         goto restart;
5602                 }
5603
5604                 if (__kvm_mmu_prepare_zap_page(kvm, sp,
5605                                 &kvm->arch.zapped_obsolete_pages, &nr_zapped)) {
5606                         batch += nr_zapped;
5607                         goto restart;
5608                 }
5609         }
5610
5611         /*
5612          * Trigger a remote TLB flush before freeing the page tables to ensure
5613          * KVM is not in the middle of a lockless shadow page table walk, which
5614          * may reference the pages.
5615          */
5616         kvm_mmu_commit_zap_page(kvm, &kvm->arch.zapped_obsolete_pages);
5617 }
5618
5619 /*
5620  * Fast invalidate all shadow pages and use lock-break technique
5621  * to zap obsolete pages.
5622  *
5623  * It's required when memslot is being deleted or VM is being
5624  * destroyed, in these cases, we should ensure that KVM MMU does
5625  * not use any resource of the being-deleted slot or all slots
5626  * after calling the function.
5627  */
5628 static void kvm_mmu_zap_all_fast(struct kvm *kvm)
5629 {
5630         lockdep_assert_held(&kvm->slots_lock);
5631
5632         write_lock(&kvm->mmu_lock);
5633         trace_kvm_mmu_zap_all_fast(kvm);
5634
5635         /*
5636          * Toggle mmu_valid_gen between '0' and '1'.  Because slots_lock is
5637          * held for the entire duration of zapping obsolete pages, it's
5638          * impossible for there to be multiple invalid generations associated
5639          * with *valid* shadow pages at any given time, i.e. there is exactly
5640          * one valid generation and (at most) one invalid generation.
5641          */
5642         kvm->arch.mmu_valid_gen = kvm->arch.mmu_valid_gen ? 0 : 1;
5643
5644         /* In order to ensure all threads see this change when
5645          * handling the MMU reload signal, this must happen in the
5646          * same critical section as kvm_reload_remote_mmus, and
5647          * before kvm_zap_obsolete_pages as kvm_zap_obsolete_pages
5648          * could drop the MMU lock and yield.
5649          */
5650         if (is_tdp_mmu_enabled(kvm))
5651                 kvm_tdp_mmu_invalidate_all_roots(kvm);
5652
5653         /*
5654          * Notify all vcpus to reload its shadow page table and flush TLB.
5655          * Then all vcpus will switch to new shadow page table with the new
5656          * mmu_valid_gen.
5657          *
5658          * Note: we need to do this under the protection of mmu_lock,
5659          * otherwise, vcpu would purge shadow page but miss tlb flush.
5660          */
5661         kvm_reload_remote_mmus(kvm);
5662
5663         kvm_zap_obsolete_pages(kvm);
5664
5665         write_unlock(&kvm->mmu_lock);
5666
5667         if (is_tdp_mmu_enabled(kvm)) {
5668                 read_lock(&kvm->mmu_lock);
5669                 kvm_tdp_mmu_zap_invalidated_roots(kvm);
5670                 read_unlock(&kvm->mmu_lock);
5671         }
5672 }
5673
5674 static bool kvm_has_zapped_obsolete_pages(struct kvm *kvm)
5675 {
5676         return unlikely(!list_empty_careful(&kvm->arch.zapped_obsolete_pages));
5677 }
5678
5679 static void kvm_mmu_invalidate_zap_pages_in_memslot(struct kvm *kvm,
5680                         struct kvm_memory_slot *slot,
5681                         struct kvm_page_track_notifier_node *node)
5682 {
5683         kvm_mmu_zap_all_fast(kvm);
5684 }
5685
5686 void kvm_mmu_init_vm(struct kvm *kvm)
5687 {
5688         struct kvm_page_track_notifier_node *node = &kvm->arch.mmu_sp_tracker;
5689
5690         spin_lock_init(&kvm->arch.mmu_unsync_pages_lock);
5691
5692         kvm_mmu_init_tdp_mmu(kvm);
5693
5694         node->track_write = kvm_mmu_pte_write;
5695         node->track_flush_slot = kvm_mmu_invalidate_zap_pages_in_memslot;
5696         kvm_page_track_register_notifier(kvm, node);
5697 }
5698
5699 void kvm_mmu_uninit_vm(struct kvm *kvm)
5700 {
5701         struct kvm_page_track_notifier_node *node = &kvm->arch.mmu_sp_tracker;
5702
5703         kvm_page_track_unregister_notifier(kvm, node);
5704
5705         kvm_mmu_uninit_tdp_mmu(kvm);
5706 }
5707
5708 static bool __kvm_zap_rmaps(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end)
5709 {
5710         const struct kvm_memory_slot *memslot;
5711         struct kvm_memslots *slots;
5712         bool flush = false;
5713         gfn_t start, end;
5714         int i;
5715
5716         if (!kvm_memslots_have_rmaps(kvm))
5717                 return flush;
5718
5719         for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
5720                 slots = __kvm_memslots(kvm, i);
5721                 kvm_for_each_memslot(memslot, slots) {
5722                         start = max(gfn_start, memslot->base_gfn);
5723                         end = min(gfn_end, memslot->base_gfn + memslot->npages);
5724                         if (start >= end)
5725                                 continue;
5726
5727                         flush = slot_handle_level_range(kvm, memslot, kvm_zap_rmapp,
5728                                                         PG_LEVEL_4K, KVM_MAX_HUGEPAGE_LEVEL,
5729                                                         start, end - 1, true, flush);
5730                 }
5731         }
5732
5733         return flush;
5734 }
5735
5736 /*
5737  * Invalidate (zap) SPTEs that cover GFNs from gfn_start and up to gfn_end
5738  * (not including it)
5739  */
5740 void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end)
5741 {
5742         bool flush;
5743         int i;
5744
5745         write_lock(&kvm->mmu_lock);
5746
5747         kvm_inc_notifier_count(kvm, gfn_start, gfn_end);
5748
5749         flush = __kvm_zap_rmaps(kvm, gfn_start, gfn_end);
5750
5751         if (is_tdp_mmu_enabled(kvm)) {
5752                 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++)
5753                         flush = kvm_tdp_mmu_zap_gfn_range(kvm, i, gfn_start,
5754                                                           gfn_end, flush);
5755         }
5756
5757         if (flush)
5758                 kvm_flush_remote_tlbs_with_address(kvm, gfn_start,
5759                                                    gfn_end - gfn_start);
5760
5761         kvm_dec_notifier_count(kvm, gfn_start, gfn_end);
5762
5763         write_unlock(&kvm->mmu_lock);
5764 }
5765
5766 static bool slot_rmap_write_protect(struct kvm *kvm,
5767                                     struct kvm_rmap_head *rmap_head,
5768                                     const struct kvm_memory_slot *slot)
5769 {
5770         return __rmap_write_protect(kvm, rmap_head, false);
5771 }
5772
5773 void kvm_mmu_slot_remove_write_access(struct kvm *kvm,
5774                                       const struct kvm_memory_slot *memslot,
5775                                       int start_level)
5776 {
5777         bool flush = false;
5778
5779         if (kvm_memslots_have_rmaps(kvm)) {
5780                 write_lock(&kvm->mmu_lock);
5781                 flush = slot_handle_level(kvm, memslot, slot_rmap_write_protect,
5782                                           start_level, KVM_MAX_HUGEPAGE_LEVEL,
5783                                           false);
5784                 write_unlock(&kvm->mmu_lock);
5785         }
5786
5787         if (is_tdp_mmu_enabled(kvm)) {
5788                 read_lock(&kvm->mmu_lock);
5789                 flush |= kvm_tdp_mmu_wrprot_slot(kvm, memslot, start_level);
5790                 read_unlock(&kvm->mmu_lock);
5791         }
5792
5793         /*
5794          * We can flush all the TLBs out of the mmu lock without TLB
5795          * corruption since we just change the spte from writable to
5796          * readonly so that we only need to care the case of changing
5797          * spte from present to present (changing the spte from present
5798          * to nonpresent will flush all the TLBs immediately), in other
5799          * words, the only case we care is mmu_spte_update() where we
5800          * have checked Host-writable | MMU-writable instead of
5801          * PT_WRITABLE_MASK, that means it does not depend on PT_WRITABLE_MASK
5802          * anymore.
5803          */
5804         if (flush)
5805                 kvm_arch_flush_remote_tlbs_memslot(kvm, memslot);
5806 }
5807
5808 static bool kvm_mmu_zap_collapsible_spte(struct kvm *kvm,
5809                                          struct kvm_rmap_head *rmap_head,
5810                                          const struct kvm_memory_slot *slot)
5811 {
5812         u64 *sptep;
5813         struct rmap_iterator iter;
5814         int need_tlb_flush = 0;
5815         kvm_pfn_t pfn;
5816         struct kvm_mmu_page *sp;
5817
5818 restart:
5819         for_each_rmap_spte(rmap_head, &iter, sptep) {
5820                 sp = sptep_to_sp(sptep);
5821                 pfn = spte_to_pfn(*sptep);
5822
5823                 /*
5824                  * We cannot do huge page mapping for indirect shadow pages,
5825                  * which are found on the last rmap (level = 1) when not using
5826                  * tdp; such shadow pages are synced with the page table in
5827                  * the guest, and the guest page table is using 4K page size
5828                  * mapping if the indirect sp has level = 1.
5829                  */
5830                 if (sp->role.direct && !kvm_is_reserved_pfn(pfn) &&
5831                     sp->role.level < kvm_mmu_max_mapping_level(kvm, slot, sp->gfn,
5832                                                                pfn, PG_LEVEL_NUM)) {
5833                         pte_list_remove(kvm, rmap_head, sptep);
5834
5835                         if (kvm_available_flush_tlb_with_range())
5836                                 kvm_flush_remote_tlbs_with_address(kvm, sp->gfn,
5837                                         KVM_PAGES_PER_HPAGE(sp->role.level));
5838                         else
5839                                 need_tlb_flush = 1;
5840
5841                         goto restart;
5842                 }
5843         }
5844
5845         return need_tlb_flush;
5846 }
5847
5848 void kvm_mmu_zap_collapsible_sptes(struct kvm *kvm,
5849                                    const struct kvm_memory_slot *slot)
5850 {
5851         bool flush;
5852
5853         if (kvm_memslots_have_rmaps(kvm)) {
5854                 write_lock(&kvm->mmu_lock);
5855                 /*
5856                  * Zap only 4k SPTEs since the legacy MMU only supports dirty
5857                  * logging at a 4k granularity and never creates collapsible
5858                  * 2m SPTEs during dirty logging.
5859                  */
5860                 flush = slot_handle_level_4k(kvm, slot, kvm_mmu_zap_collapsible_spte, true);
5861                 if (flush)
5862                         kvm_arch_flush_remote_tlbs_memslot(kvm, slot);
5863                 write_unlock(&kvm->mmu_lock);
5864         }
5865
5866         if (is_tdp_mmu_enabled(kvm)) {
5867                 read_lock(&kvm->mmu_lock);
5868                 flush = kvm_tdp_mmu_zap_collapsible_sptes(kvm, slot, false);
5869                 if (flush)
5870                         kvm_arch_flush_remote_tlbs_memslot(kvm, slot);
5871                 read_unlock(&kvm->mmu_lock);
5872         }
5873 }
5874
5875 void kvm_arch_flush_remote_tlbs_memslot(struct kvm *kvm,
5876                                         const struct kvm_memory_slot *memslot)
5877 {
5878         /*
5879          * All current use cases for flushing the TLBs for a specific memslot
5880          * related to dirty logging, and many do the TLB flush out of mmu_lock.
5881          * The interaction between the various operations on memslot must be
5882          * serialized by slots_locks to ensure the TLB flush from one operation
5883          * is observed by any other operation on the same memslot.
5884          */
5885         lockdep_assert_held(&kvm->slots_lock);
5886         kvm_flush_remote_tlbs_with_address(kvm, memslot->base_gfn,
5887                                            memslot->npages);
5888 }
5889
5890 void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm,
5891                                    const struct kvm_memory_slot *memslot)
5892 {
5893         bool flush = false;
5894
5895         if (kvm_memslots_have_rmaps(kvm)) {
5896                 write_lock(&kvm->mmu_lock);
5897                 /*
5898                  * Clear dirty bits only on 4k SPTEs since the legacy MMU only
5899                  * support dirty logging at a 4k granularity.
5900                  */
5901                 flush = slot_handle_level_4k(kvm, memslot, __rmap_clear_dirty, false);
5902                 write_unlock(&kvm->mmu_lock);
5903         }
5904
5905         if (is_tdp_mmu_enabled(kvm)) {
5906                 read_lock(&kvm->mmu_lock);
5907                 flush |= kvm_tdp_mmu_clear_dirty_slot(kvm, memslot);
5908                 read_unlock(&kvm->mmu_lock);
5909         }
5910
5911         /*
5912          * It's also safe to flush TLBs out of mmu lock here as currently this
5913          * function is only used for dirty logging, in which case flushing TLB
5914          * out of mmu lock also guarantees no dirty pages will be lost in
5915          * dirty_bitmap.
5916          */
5917         if (flush)
5918                 kvm_arch_flush_remote_tlbs_memslot(kvm, memslot);
5919 }
5920
5921 void kvm_mmu_zap_all(struct kvm *kvm)
5922 {
5923         struct kvm_mmu_page *sp, *node;
5924         LIST_HEAD(invalid_list);
5925         int ign;
5926
5927         write_lock(&kvm->mmu_lock);
5928 restart:
5929         list_for_each_entry_safe(sp, node, &kvm->arch.active_mmu_pages, link) {
5930                 if (WARN_ON(sp->role.invalid))
5931                         continue;
5932                 if (__kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list, &ign))
5933                         goto restart;
5934                 if (cond_resched_rwlock_write(&kvm->mmu_lock))
5935                         goto restart;
5936         }
5937
5938         kvm_mmu_commit_zap_page(kvm, &invalid_list);
5939
5940         if (is_tdp_mmu_enabled(kvm))
5941                 kvm_tdp_mmu_zap_all(kvm);
5942
5943         write_unlock(&kvm->mmu_lock);
5944 }
5945
5946 void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, u64 gen)
5947 {
5948         WARN_ON(gen & KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS);
5949
5950         gen &= MMIO_SPTE_GEN_MASK;
5951
5952         /*
5953          * Generation numbers are incremented in multiples of the number of
5954          * address spaces in order to provide unique generations across all
5955          * address spaces.  Strip what is effectively the address space
5956          * modifier prior to checking for a wrap of the MMIO generation so
5957          * that a wrap in any address space is detected.
5958          */
5959         gen &= ~((u64)KVM_ADDRESS_SPACE_NUM - 1);
5960
5961         /*
5962          * The very rare case: if the MMIO generation number has wrapped,
5963          * zap all shadow pages.
5964          */
5965         if (unlikely(gen == 0)) {
5966                 kvm_debug_ratelimited("kvm: zapping shadow pages for mmio generation wraparound\n");
5967                 kvm_mmu_zap_all_fast(kvm);
5968         }
5969 }
5970
5971 static unsigned long
5972 mmu_shrink_scan(struct shrinker *shrink, struct shrink_control *sc)
5973 {
5974         struct kvm *kvm;
5975         int nr_to_scan = sc->nr_to_scan;
5976         unsigned long freed = 0;
5977
5978         mutex_lock(&kvm_lock);
5979
5980         list_for_each_entry(kvm, &vm_list, vm_list) {
5981                 int idx;
5982                 LIST_HEAD(invalid_list);
5983
5984                 /*
5985                  * Never scan more than sc->nr_to_scan VM instances.
5986                  * Will not hit this condition practically since we do not try
5987                  * to shrink more than one VM and it is very unlikely to see
5988                  * !n_used_mmu_pages so many times.
5989                  */
5990                 if (!nr_to_scan--)
5991                         break;
5992                 /*
5993                  * n_used_mmu_pages is accessed without holding kvm->mmu_lock
5994                  * here. We may skip a VM instance errorneosly, but we do not
5995                  * want to shrink a VM that only started to populate its MMU
5996                  * anyway.
5997                  */
5998                 if (!kvm->arch.n_used_mmu_pages &&
5999                     !kvm_has_zapped_obsolete_pages(kvm))
6000                         continue;
6001
6002                 idx = srcu_read_lock(&kvm->srcu);
6003                 write_lock(&kvm->mmu_lock);
6004
6005                 if (kvm_has_zapped_obsolete_pages(kvm)) {
6006                         kvm_mmu_commit_zap_page(kvm,
6007                               &kvm->arch.zapped_obsolete_pages);
6008                         goto unlock;
6009                 }
6010
6011                 freed = kvm_mmu_zap_oldest_mmu_pages(kvm, sc->nr_to_scan);
6012
6013 unlock:
6014                 write_unlock(&kvm->mmu_lock);
6015                 srcu_read_unlock(&kvm->srcu, idx);
6016
6017                 /*
6018                  * unfair on small ones
6019                  * per-vm shrinkers cry out
6020                  * sadness comes quickly
6021                  */
6022                 list_move_tail(&kvm->vm_list, &vm_list);
6023                 break;
6024         }
6025
6026         mutex_unlock(&kvm_lock);
6027         return freed;
6028 }
6029
6030 static unsigned long
6031 mmu_shrink_count(struct shrinker *shrink, struct shrink_control *sc)
6032 {
6033         return percpu_counter_read_positive(&kvm_total_used_mmu_pages);
6034 }
6035
6036 static struct shrinker mmu_shrinker = {
6037         .count_objects = mmu_shrink_count,
6038         .scan_objects = mmu_shrink_scan,
6039         .seeks = DEFAULT_SEEKS * 10,
6040 };
6041
6042 static void mmu_destroy_caches(void)
6043 {
6044         kmem_cache_destroy(pte_list_desc_cache);
6045         kmem_cache_destroy(mmu_page_header_cache);
6046 }
6047
6048 static bool get_nx_auto_mode(void)
6049 {
6050         /* Return true when CPU has the bug, and mitigations are ON */
6051         return boot_cpu_has_bug(X86_BUG_ITLB_MULTIHIT) && !cpu_mitigations_off();
6052 }
6053
6054 static void __set_nx_huge_pages(bool val)
6055 {
6056         nx_huge_pages = itlb_multihit_kvm_mitigation = val;
6057 }
6058
6059 static int set_nx_huge_pages(const char *val, const struct kernel_param *kp)
6060 {
6061         bool old_val = nx_huge_pages;
6062         bool new_val;
6063
6064         /* In "auto" mode deploy workaround only if CPU has the bug. */
6065         if (sysfs_streq(val, "off"))
6066                 new_val = 0;
6067         else if (sysfs_streq(val, "force"))
6068                 new_val = 1;
6069         else if (sysfs_streq(val, "auto"))
6070                 new_val = get_nx_auto_mode();
6071         else if (strtobool(val, &new_val) < 0)
6072                 return -EINVAL;
6073
6074         __set_nx_huge_pages(new_val);
6075
6076         if (new_val != old_val) {
6077                 struct kvm *kvm;
6078
6079                 mutex_lock(&kvm_lock);
6080
6081                 list_for_each_entry(kvm, &vm_list, vm_list) {
6082                         mutex_lock(&kvm->slots_lock);
6083                         kvm_mmu_zap_all_fast(kvm);
6084                         mutex_unlock(&kvm->slots_lock);
6085
6086                         wake_up_process(kvm->arch.nx_lpage_recovery_thread);
6087                 }
6088                 mutex_unlock(&kvm_lock);
6089         }
6090
6091         return 0;
6092 }
6093
6094 int kvm_mmu_module_init(void)
6095 {
6096         int ret = -ENOMEM;
6097
6098         if (nx_huge_pages == -1)
6099                 __set_nx_huge_pages(get_nx_auto_mode());
6100
6101         /*
6102          * MMU roles use union aliasing which is, generally speaking, an
6103          * undefined behavior. However, we supposedly know how compilers behave
6104          * and the current status quo is unlikely to change. Guardians below are
6105          * supposed to let us know if the assumption becomes false.
6106          */
6107         BUILD_BUG_ON(sizeof(union kvm_mmu_page_role) != sizeof(u32));
6108         BUILD_BUG_ON(sizeof(union kvm_mmu_extended_role) != sizeof(u32));
6109         BUILD_BUG_ON(sizeof(union kvm_mmu_role) != sizeof(u64));
6110
6111         kvm_mmu_reset_all_pte_masks();
6112
6113         pte_list_desc_cache = kmem_cache_create("pte_list_desc",
6114                                             sizeof(struct pte_list_desc),
6115                                             0, SLAB_ACCOUNT, NULL);
6116         if (!pte_list_desc_cache)
6117                 goto out;
6118
6119         mmu_page_header_cache = kmem_cache_create("kvm_mmu_page_header",
6120                                                   sizeof(struct kvm_mmu_page),
6121                                                   0, SLAB_ACCOUNT, NULL);
6122         if (!mmu_page_header_cache)
6123                 goto out;
6124
6125         if (percpu_counter_init(&kvm_total_used_mmu_pages, 0, GFP_KERNEL))
6126                 goto out;
6127
6128         ret = register_shrinker(&mmu_shrinker);
6129         if (ret)
6130                 goto out;
6131
6132         return 0;
6133
6134 out:
6135         mmu_destroy_caches();
6136         return ret;
6137 }
6138
6139 /*
6140  * Calculate mmu pages needed for kvm.
6141  */
6142 unsigned long kvm_mmu_calculate_default_mmu_pages(struct kvm *kvm)
6143 {
6144         unsigned long nr_mmu_pages;
6145         unsigned long nr_pages = 0;
6146         struct kvm_memslots *slots;
6147         struct kvm_memory_slot *memslot;
6148         int i;
6149
6150         for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
6151                 slots = __kvm_memslots(kvm, i);
6152
6153                 kvm_for_each_memslot(memslot, slots)
6154                         nr_pages += memslot->npages;
6155         }
6156
6157         nr_mmu_pages = nr_pages * KVM_PERMILLE_MMU_PAGES / 1000;
6158         nr_mmu_pages = max(nr_mmu_pages, KVM_MIN_ALLOC_MMU_PAGES);
6159
6160         return nr_mmu_pages;
6161 }
6162
6163 void kvm_mmu_destroy(struct kvm_vcpu *vcpu)
6164 {
6165         kvm_mmu_unload(vcpu);
6166         free_mmu_pages(&vcpu->arch.root_mmu);
6167         free_mmu_pages(&vcpu->arch.guest_mmu);
6168         mmu_free_memory_caches(vcpu);
6169 }
6170
6171 void kvm_mmu_module_exit(void)
6172 {
6173         mmu_destroy_caches();
6174         percpu_counter_destroy(&kvm_total_used_mmu_pages);
6175         unregister_shrinker(&mmu_shrinker);
6176         mmu_audit_disable();
6177 }
6178
6179 static int set_nx_huge_pages_recovery_param(const char *val, const struct kernel_param *kp)
6180 {
6181         bool was_recovery_enabled, is_recovery_enabled;
6182         uint old_period, new_period;
6183         int err;
6184
6185         was_recovery_enabled = nx_huge_pages_recovery_ratio;
6186         old_period = nx_huge_pages_recovery_period_ms;
6187
6188         err = param_set_uint(val, kp);
6189         if (err)
6190                 return err;
6191
6192         is_recovery_enabled = nx_huge_pages_recovery_ratio;
6193         new_period = nx_huge_pages_recovery_period_ms;
6194
6195         if (READ_ONCE(nx_huge_pages) && is_recovery_enabled &&
6196             (!was_recovery_enabled || old_period > new_period)) {
6197                 struct kvm *kvm;
6198
6199                 mutex_lock(&kvm_lock);
6200
6201                 list_for_each_entry(kvm, &vm_list, vm_list)
6202                         wake_up_process(kvm->arch.nx_lpage_recovery_thread);
6203
6204                 mutex_unlock(&kvm_lock);
6205         }
6206
6207         return err;
6208 }
6209
6210 static void kvm_recover_nx_lpages(struct kvm *kvm)
6211 {
6212         unsigned long nx_lpage_splits = kvm->stat.nx_lpage_splits;
6213         int rcu_idx;
6214         struct kvm_mmu_page *sp;
6215         unsigned int ratio;
6216         LIST_HEAD(invalid_list);
6217         bool flush = false;
6218         ulong to_zap;
6219
6220         rcu_idx = srcu_read_lock(&kvm->srcu);
6221         write_lock(&kvm->mmu_lock);
6222
6223         ratio = READ_ONCE(nx_huge_pages_recovery_ratio);
6224         to_zap = ratio ? DIV_ROUND_UP(nx_lpage_splits, ratio) : 0;
6225         for ( ; to_zap; --to_zap) {
6226                 if (list_empty(&kvm->arch.lpage_disallowed_mmu_pages))
6227                         break;
6228
6229                 /*
6230                  * We use a separate list instead of just using active_mmu_pages
6231                  * because the number of lpage_disallowed pages is expected to
6232                  * be relatively small compared to the total.
6233                  */
6234                 sp = list_first_entry(&kvm->arch.lpage_disallowed_mmu_pages,
6235                                       struct kvm_mmu_page,
6236                                       lpage_disallowed_link);
6237                 WARN_ON_ONCE(!sp->lpage_disallowed);
6238                 if (is_tdp_mmu_page(sp)) {
6239                         flush |= kvm_tdp_mmu_zap_sp(kvm, sp);
6240                 } else {
6241                         kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list);
6242                         WARN_ON_ONCE(sp->lpage_disallowed);
6243                 }
6244
6245                 if (need_resched() || rwlock_needbreak(&kvm->mmu_lock)) {
6246                         kvm_mmu_remote_flush_or_zap(kvm, &invalid_list, flush);
6247                         cond_resched_rwlock_write(&kvm->mmu_lock);
6248                         flush = false;
6249                 }
6250         }
6251         kvm_mmu_remote_flush_or_zap(kvm, &invalid_list, flush);
6252
6253         write_unlock(&kvm->mmu_lock);
6254         srcu_read_unlock(&kvm->srcu, rcu_idx);
6255 }
6256
6257 static long get_nx_lpage_recovery_timeout(u64 start_time)
6258 {
6259         uint ratio = READ_ONCE(nx_huge_pages_recovery_ratio);
6260         uint period = READ_ONCE(nx_huge_pages_recovery_period_ms);
6261
6262         if (!period && ratio) {
6263                 /* Make sure the period is not less than one second.  */
6264                 ratio = min(ratio, 3600u);
6265                 period = 60 * 60 * 1000 / ratio;
6266         }
6267
6268         return READ_ONCE(nx_huge_pages) && ratio
6269                 ? start_time + msecs_to_jiffies(period) - get_jiffies_64()
6270                 : MAX_SCHEDULE_TIMEOUT;
6271 }
6272
6273 static int kvm_nx_lpage_recovery_worker(struct kvm *kvm, uintptr_t data)
6274 {
6275         u64 start_time;
6276         long remaining_time;
6277
6278         while (true) {
6279                 start_time = get_jiffies_64();
6280                 remaining_time = get_nx_lpage_recovery_timeout(start_time);
6281
6282                 set_current_state(TASK_INTERRUPTIBLE);
6283                 while (!kthread_should_stop() && remaining_time > 0) {
6284                         schedule_timeout(remaining_time);
6285                         remaining_time = get_nx_lpage_recovery_timeout(start_time);
6286                         set_current_state(TASK_INTERRUPTIBLE);
6287                 }
6288
6289                 set_current_state(TASK_RUNNING);
6290
6291                 if (kthread_should_stop())
6292                         return 0;
6293
6294                 kvm_recover_nx_lpages(kvm);
6295         }
6296 }
6297
6298 int kvm_mmu_post_init_vm(struct kvm *kvm)
6299 {
6300         int err;
6301
6302         err = kvm_vm_create_worker_thread(kvm, kvm_nx_lpage_recovery_worker, 0,
6303                                           "kvm-nx-lpage-recovery",
6304                                           &kvm->arch.nx_lpage_recovery_thread);
6305         if (!err)
6306                 kthread_unpark(kvm->arch.nx_lpage_recovery_thread);
6307
6308         return err;
6309 }
6310
6311 void kvm_mmu_pre_destroy_vm(struct kvm *kvm)
6312 {
6313         if (kvm->arch.nx_lpage_recovery_thread)
6314                 kthread_stop(kvm->arch.nx_lpage_recovery_thread);
6315 }