arch/riscv/mm/tlbflush.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 #include <linux/mm.h>
   4 #include <linux/smp.h>
   5 #include <linux/sched.h>
   6 #include <asm/sbi.h>
   7 #include <asm/mmu_context.h>
   8
   9 static inline void local_flush_tlb_all_asid(unsigned long asid)
  10 {
  11         __asm__ __volatile__ ("sfence.vma x0, %0"
  12                         :
  13                         : "r" (asid)
  14                         : "memory");
  15 }
  16
  17 static inline void local_flush_tlb_page_asid(unsigned long addr,
  18                 unsigned long asid)
  19 {
  20         __asm__ __volatile__ ("sfence.vma %0, %1"
  21                         :
  22                         : "r" (addr), "r" (asid)
  23                         : "memory");
  24 }
  25
  26 void flush_tlb_all(void)
  27 {
  28         sbi_remote_sfence_vma(NULL, 0, -1);
  29 }
  30
  31 static void __sbi_tlb_flush_range(struct mm_struct *mm, unsigned long start,
  32                                   unsigned long size, unsigned long stride)
  33 {
  34         struct cpumask *cmask = mm_cpumask(mm);
  35         unsigned int cpuid;
  36         bool broadcast;
  37
  38         if (cpumask_empty(cmask))
  39                 return;
  40
  41         cpuid = get_cpu();
  42         /* check if the tlbflush needs to be sent to other CPUs */
  43         broadcast = cpumask_any_but(cmask, cpuid) < nr_cpu_ids;
  44         if (static_branch_unlikely(&use_asid_allocator)) {
  45                 unsigned long asid = atomic_long_read(&mm->context.id);
  46
  47                 if (broadcast) {
  48                         sbi_remote_sfence_vma_asid(cmask, start, size, asid);
  49                 } else if (size <= stride) {
  50                         local_flush_tlb_page_asid(start, asid);
  51                 } else {
  52                         local_flush_tlb_all_asid(asid);
  53                 }
  54         } else {
  55                 if (broadcast) {
  56                         sbi_remote_sfence_vma(cmask, start, size);
  57                 } else if (size <= stride) {
  58                         local_flush_tlb_page(start);
  59                 } else {
  60                         local_flush_tlb_all();
  61                 }
  62         }
  63
  64         put_cpu();
  65 }
  66
  67 void flush_tlb_mm(struct mm_struct *mm)
  68 {
  69         __sbi_tlb_flush_range(mm, 0, -1, PAGE_SIZE);
  70 }
  71
  72 void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)
  73 {
  74         __sbi_tlb_flush_range(vma->vm_mm, addr, PAGE_SIZE, PAGE_SIZE);
  75 }
  76
  77 void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
  78                      unsigned long end)
  79 {
  80         __sbi_tlb_flush_range(vma->vm_mm, start, end - start, PAGE_SIZE);
  81 }
  82 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  83 void flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start,
  84                         unsigned long end)
  85 {
  86         __sbi_tlb_flush_range(vma->vm_mm, start, end - start, PMD_SIZE);
  87 }
  88 #endif