KVM: arm64: Adjust partial code of hyp stage-1 map and guest stage-2 map

Procedures of hyp stage-1 map and guest stage-2 map are quite different,
but they are tied closely by function kvm_set_valid_leaf_pte().
So adjust the relative code for ease of code maintenance in the future.

Signed-off-by: Will Deacon <will@kernel.org>
Signed-off-by: Yanan Wang <wangyanan55@huawei.com>
Signed-off-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20210114121350.123684-2-wangyanan55@huawei.com

diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c
index bdf8e55..a11ac87 100644 (file)
--- a/arch/arm64/kvm/hyp/pgtable.c
+++ b/arch/arm64/kvm/hyp/pgtable.c
@@ -170,10 +170,9 @@ static void kvm_set_table_pte(kvm_pte_t *ptep, kvm_pte_t *childp)
        smp_store_release(ptep, pte);
 }
 
-static bool kvm_set_valid_leaf_pte(kvm_pte_t *ptep, u64 pa, kvm_pte_t attr,
-                                  u32 level)
+static kvm_pte_t kvm_init_valid_leaf_pte(u64 pa, kvm_pte_t attr, u32 level)
 {
-       kvm_pte_t old = *ptep, pte = kvm_phys_to_pte(pa);
+       kvm_pte_t pte = kvm_phys_to_pte(pa);
        u64 type = (level == KVM_PGTABLE_MAX_LEVELS - 1) ? KVM_PTE_TYPE_PAGE :
                                                           KVM_PTE_TYPE_BLOCK;
 
@@ -181,12 +180,7 @@ static bool kvm_set_valid_leaf_pte(kvm_pte_t *ptep, u64 pa, kvm_pte_t attr,
        pte |= FIELD_PREP(KVM_PTE_TYPE, type);
        pte |= KVM_PTE_VALID;
 
-       /* Tolerate KVM recreating the exact same mapping. */
-       if (kvm_pte_valid(old))
-               return old == pte;
-
-       smp_store_release(ptep, pte);
-       return true;
+       return pte;
 }
 
 static int kvm_pgtable_visitor_cb(struct kvm_pgtable_walk_data *data, u64 addr,
@@ -341,12 +335,17 @@ static int hyp_map_set_prot_attr(enum kvm_pgtable_prot prot,
 static bool hyp_map_walker_try_leaf(u64 addr, u64 end, u32 level,
                                    kvm_pte_t *ptep, struct hyp_map_data *data)
 {
+       kvm_pte_t new, old = *ptep;
        u64 granule = kvm_granule_size(level), phys = data->phys;
 
        if (!kvm_block_mapping_supported(addr, end, phys, level))
                return false;
 
-       WARN_ON(!kvm_set_valid_leaf_pte(ptep, phys, data->attr, level));
+       /* Tolerate KVM recreating the exact same mapping */
+       new = kvm_init_valid_leaf_pte(phys, data->attr, level);
+       if (old != new && !WARN_ON(kvm_pte_valid(old)))
+               smp_store_release(ptep, new);
+
        data->phys += granule;
        return true;
 }
@@ -465,27 +464,30 @@ static bool stage2_map_walker_try_leaf(u64 addr, u64 end, u32 level,
                                       kvm_pte_t *ptep,
                                       struct stage2_map_data *data)
 {
+       kvm_pte_t new, old = *ptep;
        u64 granule = kvm_granule_size(level), phys = data->phys;
+       struct page *page = virt_to_page(ptep);
 
        if (!kvm_block_mapping_supported(addr, end, phys, level))
                return false;
 
-       /*
-        * If the PTE was already valid, drop the refcount on the table
-        * early, as it will be bumped-up again in stage2_map_walk_leaf().
-        * This ensures that the refcount stays constant across a valid to
-        * valid PTE update.
-        */
-       if (kvm_pte_valid(*ptep))
-               put_page(virt_to_page(ptep));
+       new = kvm_init_valid_leaf_pte(phys, data->attr, level);
+       if (kvm_pte_valid(old)) {
+               /* Tolerate KVM recreating the exact same mapping */
+               if (old == new)
+                       goto out;
 
-       if (kvm_set_valid_leaf_pte(ptep, phys, data->attr, level))
-               goto out;
+               /*
+                * There's an existing different valid leaf entry, so perform
+                * break-before-make.
+                */
+               kvm_set_invalid_pte(ptep);
+               kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, data->mmu, addr, level);
+               put_page(page);
+       }
 
-       /* There's an existing valid leaf entry, so perform break-before-make */
-       kvm_set_invalid_pte(ptep);
-       kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, data->mmu, addr, level);
-       kvm_set_valid_leaf_pte(ptep, phys, data->attr, level);
+       smp_store_release(ptep, new);
+       get_page(page);
 out:
        data->phys += granule;
        return true;
@@ -527,7 +529,7 @@ static int stage2_map_walk_leaf(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,
        }
 
        if (stage2_map_walker_try_leaf(addr, end, level, ptep, data))
-               goto out_get_page;
+               return 0;
 
        if (WARN_ON(level == KVM_PGTABLE_MAX_LEVELS - 1))
                return -EINVAL;
@@ -551,9 +553,8 @@ static int stage2_map_walk_leaf(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,
        }
 
        kvm_set_table_pte(ptep, childp);
-
-out_get_page:
        get_page(page);
+
        return 0;
 }