io_uring: reinstate the inflight tracking

[linux-2.6-microblaze.git] / mm / memremap.c

diff --git a/mm/memremap.c b/mm/memremap.c
index 6aa5f0c..af02236 100644 (file)
--- a/mm/memremap.c
+++ b/mm/memremap.c
@@ -4,7 +4,7 @@
 #include <linux/io.h>
 #include <linux/kasan.h>
 #include <linux/memory_hotplug.h>
-#include <linux/mm.h>
+#include <linux/memremap.h>
 #include <linux/pfn_t.h>
 #include <linux/swap.h>
 #include <linux/mmzone.h>
@@ -12,6 +12,7 @@
 #include <linux/types.h>
 #include <linux/wait_bit.h>
 #include <linux/xarray.h>
+#include "internal.h"
 
 static DEFINE_XARRAY(pgmap_array);
 
@@ -37,21 +38,19 @@ unsigned long memremap_compat_align(void)
 EXPORT_SYMBOL_GPL(memremap_compat_align);
 #endif
 
-#ifdef CONFIG_DEV_PAGEMAP_OPS
+#ifdef CONFIG_FS_DAX
 DEFINE_STATIC_KEY_FALSE(devmap_managed_key);
 EXPORT_SYMBOL(devmap_managed_key);
 
 static void devmap_managed_enable_put(struct dev_pagemap *pgmap)
 {
-       if (pgmap->type == MEMORY_DEVICE_PRIVATE ||
-           pgmap->type == MEMORY_DEVICE_FS_DAX)
+       if (pgmap->type == MEMORY_DEVICE_FS_DAX)
                static_branch_dec(&devmap_managed_key);
 }
 
 static void devmap_managed_enable_get(struct dev_pagemap *pgmap)
 {
-       if (pgmap->type == MEMORY_DEVICE_PRIVATE ||
-           pgmap->type == MEMORY_DEVICE_FS_DAX)
+       if (pgmap->type == MEMORY_DEVICE_FS_DAX)
                static_branch_inc(&devmap_managed_key);
 }
 #else
@@ -61,7 +60,7 @@ static void devmap_managed_enable_get(struct dev_pagemap *pgmap)
 static void devmap_managed_enable_put(struct dev_pagemap *pgmap)
 {
 }
-#endif /* CONFIG_DEV_PAGEMAP_OPS */
+#endif /* CONFIG_FS_DAX */
 
 static void pgmap_array_delete(struct range *range)
 {
@@ -102,23 +101,12 @@ static unsigned long pfn_end(struct dev_pagemap *pgmap, int range_id)
        return (range->start + range_len(range)) >> PAGE_SHIFT;
 }
 
-static unsigned long pfn_next(struct dev_pagemap *pgmap, unsigned long pfn)
-{
-       if (pfn % (1024 << pgmap->vmemmap_shift))
-               cond_resched();
-       return pfn + pgmap_vmemmap_nr(pgmap);
-}
-
 static unsigned long pfn_len(struct dev_pagemap *pgmap, unsigned long range_id)
 {
        return (pfn_end(pgmap, range_id) -
                pfn_first(pgmap, range_id)) >> pgmap->vmemmap_shift;
 }
 
-#define for_each_device_pfn(pfn, map, i) \
-       for (pfn = pfn_first(map, i); pfn < pfn_end(map, i); \
-            pfn = pfn_next(map, pfn))
-
 static void pageunmap_range(struct dev_pagemap *pgmap, int range_id)
 {
        struct range *range = &pgmap->ranges[range_id];
@@ -147,13 +135,11 @@ static void pageunmap_range(struct dev_pagemap *pgmap, int range_id)
 
 void memunmap_pages(struct dev_pagemap *pgmap)
 {
-       unsigned long pfn;
        int i;
 
        percpu_ref_kill(&pgmap->ref);
        for (i = 0; i < pgmap->nr_range; i++)
-               for_each_device_pfn(pfn, pgmap, i)
-                       put_page(pfn_to_page(pfn));
+               percpu_ref_put_many(&pgmap->ref, pfn_len(pgmap, i));
        wait_for_completion(&pgmap->done);
        percpu_ref_exit(&pgmap->ref);
 
@@ -282,7 +268,8 @@ static int pagemap_range(struct dev_pagemap *pgmap, struct mhp_params *params,
        return 0;
 
 err_add_memory:
-       kasan_remove_zero_shadow(__va(range->start), range_len(range));
+       if (!is_private)
+               kasan_remove_zero_shadow(__va(range->start), range_len(range));
 err_kasan:
        untrack_pfn(NULL, PHYS_PFN(range->start), range_len(range));
 err_pfn_remap:
@@ -328,8 +315,7 @@ void *memremap_pages(struct dev_pagemap *pgmap, int nid)
                }
                break;
        case MEMORY_DEVICE_FS_DAX:
-               if (!IS_ENABLED(CONFIG_ZONE_DEVICE) ||
-                   IS_ENABLED(CONFIG_FS_DAX_LIMITED)) {
+               if (IS_ENABLED(CONFIG_FS_DAX_LIMITED)) {
                        WARN(1, "File system DAX not supported\n");
                        return ERR_PTR(-EINVAL);
                }
@@ -465,21 +451,15 @@ struct dev_pagemap *get_dev_pagemap(unsigned long pfn,
 }
 EXPORT_SYMBOL_GPL(get_dev_pagemap);
 
-#ifdef CONFIG_DEV_PAGEMAP_OPS
-void free_devmap_managed_page(struct page *page)
+void free_zone_device_page(struct page *page)
 {
-       /* notify page idle for dax */
-       if (!is_device_private_page(page)) {
-               wake_up_var(&page->_refcount);
+       if (WARN_ON_ONCE(!page->pgmap->ops || !page->pgmap->ops->page_free))
                return;
-       }
-
-       __ClearPageWaiters(page);
 
        mem_cgroup_uncharge(page_folio(page));
 
        /*
-        * When a device_private page is freed, the page->mapping field
+        * When a device managed page is freed, the page->mapping field
         * may still contain a (stale) mapping value. For example, the
         * lower bits of page->mapping may still identify the page as an
         * anonymous page. Ultimately, this entire field is just stale
@@ -501,5 +481,27 @@ void free_devmap_managed_page(struct page *page)
         */
        page->mapping = NULL;
        page->pgmap->ops->page_free(page);
+
+       /*
+        * Reset the page count to 1 to prepare for handing out the page again.
+        */
+       set_page_count(page, 1);
+}
+
+#ifdef CONFIG_FS_DAX
+bool __put_devmap_managed_page(struct page *page)
+{
+       if (page->pgmap->type != MEMORY_DEVICE_FS_DAX)
+               return false;
+
+       /*
+        * fsdax page refcounts are 1-based, rather than 0-based: if
+        * refcount is 1, then the page is free and the refcount is
+        * stable because nobody holds a reference on the page.
+        */
+       if (page_ref_dec_return(page) == 1)
+               wake_up_var(&page->_refcount);
+       return true;
 }
-#endif /* CONFIG_DEV_PAGEMAP_OPS */
+EXPORT_SYMBOL(__put_devmap_managed_page);
+#endif /* CONFIG_FS_DAX */