riscv: mremap speedup - enable HAVE_MOVE_PUD and HAVE_MOVE_PMD

HAVE_MOVE_PUD enables remapping pages at the PUD level if both the source
and destination addresses are PUD-aligned.
HAVE_MOVE_PMD does similar speedup on the PMD level.

With HAVE_MOVE_PUD enabled, there is about a 143x improvement on qemu
With HAVE_MOVE_PMD enabled, there is about a 5x improvement on qemu

Signed-off-by: Jisheng Zhang <jszhang@kernel.org>
Signed-off-by: Palmer Dabbelt <palmerdabbelt@google.com>

diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index a160f60..b58596b 100644 (file)
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -82,6 +82,8 @@ config RISCV
        select HAVE_KPROBES
        select HAVE_KPROBES_ON_FTRACE
        select HAVE_KRETPROBES
+       select HAVE_MOVE_PMD
+       select HAVE_MOVE_PUD
        select HAVE_PCI
        select HAVE_PERF_EVENTS
        select HAVE_PERF_REGS

diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h
index 4b708ae..3b72862 100644 (file)
--- a/arch/riscv/include/asm/pgtable.h
+++ b/arch/riscv/include/asm/pgtable.h
@@ -241,6 +241,11 @@ static inline pte_t pmd_pte(pmd_t pmd)
        return __pte(pmd_val(pmd));
 }
 
+static inline pte_t pud_pte(pud_t pud)
+{
+       return __pte(pud_val(pud));
+}
+
 /* Yields the page frame number (PFN) of a page table entry */
 static inline unsigned long pte_pfn(pte_t pte)
 {
@@ -570,6 +575,12 @@ static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,
        return set_pte_at(mm, addr, (pte_t *)pmdp, pmd_pte(pmd));
 }
 
+static inline void set_pud_at(struct mm_struct *mm, unsigned long addr,
+                               pud_t *pudp, pud_t pud)
+{
+       return set_pte_at(mm, addr, (pte_t *)pudp, pud_pte(pud));
+}
+
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 static inline int pmd_trans_huge(pmd_t pmd)
 {