Merge tag 'mm-stable-2024-03-13-20-04' of git://git.kernel.org/pub/scm/linux/kernel...

[linux-2.6-microblaze.git] / mm / mmu_gather.c

diff --git a/mm/mmu_gather.c b/mm/mmu_gather.c
index 604ddf0..99b3e94 100644 (file)
--- a/mm/mmu_gather.c
+++ b/mm/mmu_gather.c
@@ -50,12 +50,21 @@ static bool tlb_next_batch(struct mmu_gather *tlb)
 #ifdef CONFIG_SMP
 static void tlb_flush_rmap_batch(struct mmu_gather_batch *batch, struct vm_area_struct *vma)
 {
+       struct encoded_page **pages = batch->encoded_pages;
+
        for (int i = 0; i < batch->nr; i++) {
-               struct encoded_page *enc = batch->encoded_pages[i];
+               struct encoded_page *enc = pages[i];
 
-               if (encoded_page_flags(enc)) {
+               if (encoded_page_flags(enc) & ENCODED_PAGE_BIT_DELAY_RMAP) {
                        struct page *page = encoded_page_ptr(enc);
-                       folio_remove_rmap_pte(page_folio(page), page, vma);
+                       unsigned int nr_pages = 1;
+
+                       if (unlikely(encoded_page_flags(enc) &
+                                    ENCODED_PAGE_BIT_NR_PAGES_NEXT))
+                               nr_pages = encoded_nr_pages(pages[++i]);
+
+                       folio_remove_rmap_ptes(page_folio(page), page, nr_pages,
+                                              vma);
                }
        }
 }
@@ -82,26 +91,62 @@ void tlb_flush_rmaps(struct mmu_gather *tlb, struct vm_area_struct *vma)
 }
 #endif
 
-static void tlb_batch_pages_flush(struct mmu_gather *tlb)
+/*
+ * We might end up freeing a lot of pages. Reschedule on a regular
+ * basis to avoid soft lockups in configurations without full
+ * preemption enabled. The magic number of 512 folios seems to work.
+ */
+#define MAX_NR_FOLIOS_PER_FREE         512
+
+static void __tlb_batch_free_encoded_pages(struct mmu_gather_batch *batch)
 {
-       struct mmu_gather_batch *batch;
+       struct encoded_page **pages = batch->encoded_pages;
+       unsigned int nr, nr_pages;
 
-       for (batch = &tlb->local; batch && batch->nr; batch = batch->next) {
-               struct encoded_page **pages = batch->encoded_pages;
+       while (batch->nr) {
+               if (!page_poisoning_enabled_static() && !want_init_on_free()) {
+                       nr = min(MAX_NR_FOLIOS_PER_FREE, batch->nr);
 
-               do {
                        /*
-                        * limit free batch count when PAGE_SIZE > 4K
+                        * Make sure we cover page + nr_pages, and don't leave
+                        * nr_pages behind when capping the number of entries.
+                        */
+                       if (unlikely(encoded_page_flags(pages[nr - 1]) &
+                                    ENCODED_PAGE_BIT_NR_PAGES_NEXT))
+                               nr++;
+               } else {
+                       /*
+                        * With page poisoning and init_on_free, the time it
+                        * takes to free memory grows proportionally with the
+                        * actual memory size. Therefore, limit based on the
+                        * actual memory size and not the number of involved
+                        * folios.
                         */
-                       unsigned int nr = min(512U, batch->nr);
+                       for (nr = 0, nr_pages = 0;
+                            nr < batch->nr && nr_pages < MAX_NR_FOLIOS_PER_FREE;
+                            nr++) {
+                               if (unlikely(encoded_page_flags(pages[nr]) &
+                                            ENCODED_PAGE_BIT_NR_PAGES_NEXT))
+                                       nr_pages += encoded_nr_pages(pages[++nr]);
+                               else
+                                       nr_pages++;
+                       }
+               }
 
-                       free_pages_and_swap_cache(pages, nr);
-                       pages += nr;
-                       batch->nr -= nr;
+               free_pages_and_swap_cache(pages, nr);
+               pages += nr;
+               batch->nr -= nr;
 
-                       cond_resched();
-               } while (batch->nr);
+               cond_resched();
        }
+}
+
+static void tlb_batch_pages_flush(struct mmu_gather *tlb)
+{
+       struct mmu_gather_batch *batch;
+
+       for (batch = &tlb->local; batch && batch->nr; batch = batch->next)
+               __tlb_batch_free_encoded_pages(batch);
        tlb->active = &tlb->local;
 }
 
@@ -116,14 +161,19 @@ static void tlb_batch_list_free(struct mmu_gather *tlb)
        tlb->local.next = NULL;
 }
 
-bool __tlb_remove_page_size(struct mmu_gather *tlb, struct encoded_page *page, int page_size)
+static bool __tlb_remove_folio_pages_size(struct mmu_gather *tlb,
+               struct page *page, unsigned int nr_pages, bool delay_rmap,
+               int page_size)
 {
+       int flags = delay_rmap ? ENCODED_PAGE_BIT_DELAY_RMAP : 0;
        struct mmu_gather_batch *batch;
 
        VM_BUG_ON(!tlb->end);
 
 #ifdef CONFIG_MMU_GATHER_PAGE_SIZE
        VM_WARN_ON(tlb->page_size != page_size);
+       VM_WARN_ON_ONCE(nr_pages != 1 && page_size != PAGE_SIZE);
+       VM_WARN_ON_ONCE(page_folio(page) != page_folio(page + nr_pages - 1));
 #endif
 
        batch = tlb->active;
@@ -131,17 +181,40 @@ bool __tlb_remove_page_size(struct mmu_gather *tlb, struct encoded_page *page, i
         * Add the page and check if we are full. If so
         * force a flush.
         */
-       batch->encoded_pages[batch->nr++] = page;
-       if (batch->nr == batch->max) {
+       if (likely(nr_pages == 1)) {
+               batch->encoded_pages[batch->nr++] = encode_page(page, flags);
+       } else {
+               flags |= ENCODED_PAGE_BIT_NR_PAGES_NEXT;
+               batch->encoded_pages[batch->nr++] = encode_page(page, flags);
+               batch->encoded_pages[batch->nr++] = encode_nr_pages(nr_pages);
+       }
+       /*
+        * Make sure that we can always add another "page" + "nr_pages",
+        * requiring two entries instead of only a single one.
+        */
+       if (batch->nr >= batch->max - 1) {
                if (!tlb_next_batch(tlb))
                        return true;
                batch = tlb->active;
        }
-       VM_BUG_ON_PAGE(batch->nr > batch->max, encoded_page_ptr(page));
+       VM_BUG_ON_PAGE(batch->nr > batch->max - 1, page);
 
        return false;
 }
 
+bool __tlb_remove_folio_pages(struct mmu_gather *tlb, struct page *page,
+               unsigned int nr_pages, bool delay_rmap)
+{
+       return __tlb_remove_folio_pages_size(tlb, page, nr_pages, delay_rmap,
+                                            PAGE_SIZE);
+}
+
+bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page,
+               bool delay_rmap, int page_size)
+{
+       return __tlb_remove_folio_pages_size(tlb, page, 1, delay_rmap, page_size);
+}
+
 #endif /* MMU_GATHER_NO_GATHER */
 
 #ifdef CONFIG_MMU_GATHER_TABLE_FREE