drm/ttm: introduce pool_shrink_rwsem

[linux-2.6-microblaze.git] / mm / mremap.c

diff --git a/mm/mremap.c b/mm/mremap.c
index 382e81c..ce8a23e 100644 (file)
--- a/mm/mremap.c
+++ b/mm/mremap.c
@@ -489,10 +489,62 @@ static bool move_pgt_entry(enum pgt_entry entry, struct vm_area_struct *vma,
        return moved;
 }
 
+/*
+ * A helper to check if aligning down is OK. The aligned address should fall
+ * on *no mapping*. For the stack moving down, that's a special move within
+ * the VMA that is created to span the source and destination of the move,
+ * so we make an exception for it.
+ */
+static bool can_align_down(struct vm_area_struct *vma, unsigned long addr_to_align,
+                           unsigned long mask, bool for_stack)
+{
+       unsigned long addr_masked = addr_to_align & mask;
+
+       /*
+        * If @addr_to_align of either source or destination is not the beginning
+        * of the corresponding VMA, we can't align down or we will destroy part
+        * of the current mapping.
+        */
+       if (!for_stack && vma->vm_start != addr_to_align)
+               return false;
+
+       /* In the stack case we explicitly permit in-VMA alignment. */
+       if (for_stack && addr_masked >= vma->vm_start)
+               return true;
+
+       /*
+        * Make sure the realignment doesn't cause the address to fall on an
+        * existing mapping.
+        */
+       return find_vma_intersection(vma->vm_mm, addr_masked, vma->vm_start) == NULL;
+}
+
+/* Opportunistically realign to specified boundary for faster copy. */
+static void try_realign_addr(unsigned long *old_addr, struct vm_area_struct *old_vma,
+                            unsigned long *new_addr, struct vm_area_struct *new_vma,
+                            unsigned long mask, bool for_stack)
+{
+       /* Skip if the addresses are already aligned. */
+       if ((*old_addr & ~mask) == 0)
+               return;
+
+       /* Only realign if the new and old addresses are mutually aligned. */
+       if ((*old_addr & ~mask) != (*new_addr & ~mask))
+               return;
+
+       /* Ensure realignment doesn't cause overlap with existing mappings. */
+       if (!can_align_down(old_vma, *old_addr, mask, for_stack) ||
+           !can_align_down(new_vma, *new_addr, mask, for_stack))
+               return;
+
+       *old_addr = *old_addr & mask;
+       *new_addr = *new_addr & mask;
+}
+
 unsigned long move_page_tables(struct vm_area_struct *vma,
                unsigned long old_addr, struct vm_area_struct *new_vma,
                unsigned long new_addr, unsigned long len,
-               bool need_rmap_locks)
+               bool need_rmap_locks, bool for_stack)
 {
        unsigned long extent, old_end;
        struct mmu_notifier_range range;
@@ -508,6 +560,14 @@ unsigned long move_page_tables(struct vm_area_struct *vma,
                return move_hugetlb_page_tables(vma, new_vma, old_addr,
                                                new_addr, len);
 
+       /*
+        * If possible, realign addresses to PMD boundary for faster copy.
+        * Only realign if the mremap copying hits a PMD boundary.
+        */
+       if (len >= PMD_SIZE - (old_addr & ~PMD_MASK))
+               try_realign_addr(&old_addr, vma, &new_addr, new_vma, PMD_MASK,
+                                for_stack);
+
        flush_cache_range(vma, old_addr, old_end);
        mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma->vm_mm,
                                old_addr, old_end);
@@ -577,6 +637,13 @@ again:
 
        mmu_notifier_invalidate_range_end(&range);
 
+       /*
+        * Prevent negative return values when {old,new}_addr was realigned
+        * but we broke out of the above loop for the first PMD itself.
+        */
+       if (len + old_addr < old_end)
+               return 0;
+
        return len + old_addr - old_end;        /* how much done */
 }
 
@@ -646,7 +713,7 @@ static unsigned long move_vma(struct vm_area_struct *vma,
        }
 
        moved_len = move_page_tables(vma, old_addr, new_vma, new_addr, old_len,
-                                    need_rmap_locks);
+                                    need_rmap_locks, false);
        if (moved_len < old_len) {
                err = -ENOMEM;
        } else if (vma->vm_ops && vma->vm_ops->mremap) {
@@ -660,7 +727,7 @@ static unsigned long move_vma(struct vm_area_struct *vma,
                 * and then proceed to unmap new area instead of old.
                 */
                move_page_tables(new_vma, new_addr, vma, old_addr, moved_len,
-                                true);
+                                true, false);
                vma = new_vma;
                old_len = new_len;
                old_addr = new_addr;
@@ -1037,12 +1104,14 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,
                        pgoff_t extension_pgoff = vma->vm_pgoff +
                                ((extension_start - vma->vm_start) >> PAGE_SHIFT);
                        VMA_ITERATOR(vmi, mm, extension_start);
+                       long charged = 0;
 
                        if (vma->vm_flags & VM_ACCOUNT) {
                                if (security_vm_enough_memory_mm(mm, pages)) {
                                        ret = -ENOMEM;
                                        goto out;
                                }
+                               charged = pages;
                        }
 
                        /*
@@ -1058,7 +1127,7 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,
                                vma->vm_file, extension_pgoff, vma_policy(vma),
                                vma->vm_userfaultfd_ctx, anon_vma_name(vma));
                        if (!vma) {
-                               vm_unacct_memory(pages);
+                               vm_unacct_memory(charged);
                                ret = -ENOMEM;
                                goto out;
                        }