mm: pagewalk: fix termination condition in walk_pte_range()

[linux-2.6-microblaze.git] / mm / huge_memory.c

diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 9b3ee79..b08b199 100644 (file)
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -177,16 +177,13 @@ static ssize_t enabled_store(struct kobject *kobj,
 {
        ssize_t ret = count;
 
-       if (!memcmp("always", buf,
-                   min(sizeof("always")-1, count))) {
+       if (sysfs_streq(buf, "always")) {
                clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags);
                set_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags);
-       } else if (!memcmp("madvise", buf,
-                          min(sizeof("madvise")-1, count))) {
+       } else if (sysfs_streq(buf, "madvise")) {
                clear_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags);
                set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags);
-       } else if (!memcmp("never", buf,
-                          min(sizeof("never")-1, count))) {
+       } else if (sysfs_streq(buf, "never")) {
                clear_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags);
                clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags);
        } else
@@ -250,32 +247,27 @@ static ssize_t defrag_store(struct kobject *kobj,
                            struct kobj_attribute *attr,
                            const char *buf, size_t count)
 {
-       if (!memcmp("always", buf,
-                   min(sizeof("always")-1, count))) {
+       if (sysfs_streq(buf, "always")) {
                clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags);
                clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags);
                clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags);
                set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags);
-       } else if (!memcmp("defer+madvise", buf,
-                   min(sizeof("defer+madvise")-1, count))) {
+       } else if (sysfs_streq(buf, "defer+madvise")) {
                clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags);
                clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags);
                clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags);
                set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags);
-       } else if (!memcmp("defer", buf,
-                   min(sizeof("defer")-1, count))) {
+       } else if (sysfs_streq(buf, "defer")) {
                clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags);
                clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags);
                clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags);
                set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags);
-       } else if (!memcmp("madvise", buf,
-                          min(sizeof("madvise")-1, count))) {
+       } else if (sysfs_streq(buf, "madvise")) {
                clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags);
                clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags);
                clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags);
                set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags);
-       } else if (!memcmp("never", buf,
-                          min(sizeof("never")-1, count))) {
+       } else if (sysfs_streq(buf, "never")) {
                clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags);
                clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags);
                clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags);
@@ -538,13 +530,13 @@ bool is_transparent_hugepage(struct page *page)
 }
 EXPORT_SYMBOL_GPL(is_transparent_hugepage);
 
-static unsigned long __thp_get_unmapped_area(struct file *filp, unsigned long len,
+static unsigned long __thp_get_unmapped_area(struct file *filp,
+               unsigned long addr, unsigned long len,
                loff_t off, unsigned long flags, unsigned long size)
 {
-       unsigned long addr;
        loff_t off_end = off + len;
        loff_t off_align = round_up(off, size);
-       unsigned long len_pad;
+       unsigned long len_pad, ret;
 
        if (off_end <= off_align || (off_end - off_align) < size)
                return 0;
@@ -553,30 +545,40 @@ static unsigned long __thp_get_unmapped_area(struct file *filp, unsigned long le
        if (len_pad < len || (off + len_pad) < off)
                return 0;
 
-       addr = current->mm->get_unmapped_area(filp, 0, len_pad,
+       ret = current->mm->get_unmapped_area(filp, addr, len_pad,
                                              off >> PAGE_SHIFT, flags);
-       if (IS_ERR_VALUE(addr))
+
+       /*
+        * The failure might be due to length padding. The caller will retry
+        * without the padding.
+        */
+       if (IS_ERR_VALUE(ret))
                return 0;
 
-       addr += (off - addr) & (size - 1);
-       return addr;
+       /*
+        * Do not try to align to THP boundary if allocation at the address
+        * hint succeeds.
+        */
+       if (ret == addr)
+               return addr;
+
+       ret += (off - ret) & (size - 1);
+       return ret;
 }
 
 unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr,
                unsigned long len, unsigned long pgoff, unsigned long flags)
 {
+       unsigned long ret;
        loff_t off = (loff_t)pgoff << PAGE_SHIFT;
 
-       if (addr)
-               goto out;
        if (!IS_DAX(filp->f_mapping->host) || !IS_ENABLED(CONFIG_FS_DAX_PMD))
                goto out;
 
-       addr = __thp_get_unmapped_area(filp, len, off, flags, PMD_SIZE);
-       if (addr)
-               return addr;
-
- out:
+       ret = __thp_get_unmapped_area(filp, addr, len, off, flags, PMD_SIZE);
+       if (ret)
+               return ret;
+out:
        return current->mm->get_unmapped_area(filp, addr, len, pgoff, flags);
 }
 EXPORT_SYMBOL_GPL(thp_get_unmapped_area);
@@ -2705,7 +2707,7 @@ int split_huge_page_to_list(struct page *page, struct list_head *list)
 {
        struct page *head = compound_head(page);
        struct pglist_data *pgdata = NODE_DATA(page_to_nid(head));
-       struct deferred_split *ds_queue = get_deferred_split_queue(page);
+       struct deferred_split *ds_queue = get_deferred_split_queue(head);
        struct anon_vma *anon_vma = NULL;
        struct address_space *mapping = NULL;
        int count, mapcount, extra_pins, ret;
@@ -2713,11 +2715,11 @@ int split_huge_page_to_list(struct page *page, struct list_head *list)
        unsigned long flags;
        pgoff_t end;
 
-       VM_BUG_ON_PAGE(is_huge_zero_page(page), page);
-       VM_BUG_ON_PAGE(!PageLocked(page), page);
-       VM_BUG_ON_PAGE(!PageCompound(page), page);
+       VM_BUG_ON_PAGE(is_huge_zero_page(head), head);
+       VM_BUG_ON_PAGE(!PageLocked(head), head);
+       VM_BUG_ON_PAGE(!PageCompound(head), head);
 
-       if (PageWriteback(page))
+       if (PageWriteback(head))
                return -EBUSY;
 
        if (PageAnon(head)) {
@@ -2768,7 +2770,7 @@ int split_huge_page_to_list(struct page *page, struct list_head *list)
                goto out_unlock;
        }
 
-       mlocked = PageMlocked(page);
+       mlocked = PageMlocked(head);
        unmap_page(head);
        VM_BUG_ON_PAGE(compound_mapcount(head), head);
 
@@ -2800,14 +2802,14 @@ int split_huge_page_to_list(struct page *page, struct list_head *list)
                        ds_queue->split_queue_len--;
                        list_del(page_deferred_list(head));
                }
+               spin_unlock(&ds_queue->split_queue_lock);
                if (mapping) {
-                       if (PageSwapBacked(page))
-                               __dec_node_page_state(page, NR_SHMEM_THPS);
+                       if (PageSwapBacked(head))
+                               __dec_node_page_state(head, NR_SHMEM_THPS);
                        else
-                               __dec_node_page_state(page, NR_FILE_THPS);
+                               __dec_node_page_state(head, NR_FILE_THPS);
                }
 
-               spin_unlock(&ds_queue->split_queue_lock);
                __split_huge_page(page, list, end, flags);
                if (PageSwapCache(head)) {
                        swp_entry_t entry = { .val = page_private(head) };