projects
/
linux-2.6-microblaze.git
/ blobdiff
commit
grep
author
committer
pickaxe
?
search:
re
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
Merge tag 'drm-misc-fixes-2021-05-20' of git://anongit.freedesktop.org/drm/drm-misc...
[linux-2.6-microblaze.git]
/
mm
/
khugepaged.c
diff --git
a/mm/khugepaged.c
b/mm/khugepaged.c
index
a7d6cb9
..
6c0185f
100644
(file)
--- a/
mm/khugepaged.c
+++ b/
mm/khugepaged.c
@@
-481,7
+481,7
@@
int __khugepaged_enter(struct mm_struct *mm)
return -ENOMEM;
/* __khugepaged_exit() must not run from under us */
return -ENOMEM;
/* __khugepaged_exit() must not run from under us */
- VM_BUG_ON_MM(
atomic_read(&mm->mm_users) == 0
, mm);
+ VM_BUG_ON_MM(
khugepaged_test_exit(mm)
, mm);
if (unlikely(test_and_set_bit(MMF_VM_HUGEPAGE, &mm->flags))) {
free_mm_slot(mm_slot);
return 0;
if (unlikely(test_and_set_bit(MMF_VM_HUGEPAGE, &mm->flags))) {
free_mm_slot(mm_slot);
return 0;
@@
-667,7
+667,7
@@
static int __collapse_huge_page_isolate(struct vm_area_struct *vma,
*
* The page table that maps the page has been already unlinked
* from the page table tree and this process cannot get
*
* The page table that maps the page has been already unlinked
* from the page table tree and this process cannot get
- * an additinal pin on the page.
+ * an additi
o
nal pin on the page.
*
* New pins can come later if the page is shared across fork,
* but not from this process. The other process cannot write to
*
* New pins can come later if the page is shared across fork,
* but not from this process. The other process cannot write to
@@
-716,17
+716,17
@@
next:
if (pte_write(pteval))
writable = true;
}
if (pte_write(pteval))
writable = true;
}
- if (likely(writable)) {
- if (likely(referenced)) {
- result = SCAN_SUCCEED;
- trace_mm_collapse_huge_page_isolate(page, none_or_zero,
- referenced, writable, result);
- return 1;
- }
- } else {
+
+ if (unlikely(!writable)) {
result = SCAN_PAGE_RO;
result = SCAN_PAGE_RO;
+ } else if (unlikely(!referenced)) {
+ result = SCAN_LACK_REFERENCED_PAGE;
+ } else {
+ result = SCAN_SUCCEED;
+ trace_mm_collapse_huge_page_isolate(page, none_or_zero,
+ referenced, writable, result);
+ return 1;
}
}
-
out:
release_pte_pages(pte, _pte, compound_pagelist);
trace_mm_collapse_huge_page_isolate(page, none_or_zero,
out:
release_pte_pages(pte, _pte, compound_pagelist);
trace_mm_collapse_huge_page_isolate(page, none_or_zero,
@@
-809,7
+809,7
@@
static bool khugepaged_scan_abort(int nid)
* If node_reclaim_mode is disabled, then no extra effort is made to
* allocate memory locally.
*/
* If node_reclaim_mode is disabled, then no extra effort is made to
* allocate memory locally.
*/
- if (!node_reclaim_
mode
)
+ if (!node_reclaim_
enabled()
)
return false;
/* If there is a count for this node already, it must be acceptable */
return false;
/* If there is a count for this node already, it must be acceptable */
@@
-1128,10
+1128,10
@@
static void collapse_huge_page(struct mm_struct *mm,
mmap_write_lock(mm);
result = hugepage_vma_revalidate(mm, address, &vma);
if (result)
mmap_write_lock(mm);
result = hugepage_vma_revalidate(mm, address, &vma);
if (result)
- goto out;
+ goto out
_up_write
;
/* check if the pmd is still valid */
if (mm_find_pmd(mm, address) != pmd)
/* check if the pmd is still valid */
if (mm_find_pmd(mm, address) != pmd)
- goto out;
+ goto out
_up_write
;
anon_vma_lock_write(vma->anon_vma);
anon_vma_lock_write(vma->anon_vma);
@@
-1171,7
+1171,7
@@
static void collapse_huge_page(struct mm_struct *mm,
spin_unlock(pmd_ptl);
anon_vma_unlock_write(vma->anon_vma);
result = SCAN_FAIL;
spin_unlock(pmd_ptl);
anon_vma_unlock_write(vma->anon_vma);
result = SCAN_FAIL;
- goto out;
+ goto out
_up_write
;
}
/*
}
/*
@@
-1183,19
+1183,18
@@
static void collapse_huge_page(struct mm_struct *mm,
__collapse_huge_page_copy(pte, new_page, vma, address, pte_ptl,
&compound_pagelist);
pte_unmap(pte);
__collapse_huge_page_copy(pte, new_page, vma, address, pte_ptl,
&compound_pagelist);
pte_unmap(pte);
+ /*
+ * spin_lock() below is not the equivalent of smp_wmb(), but
+ * the smp_wmb() inside __SetPageUptodate() can be reused to
+ * avoid the copy_huge_page writes to become visible after
+ * the set_pmd_at() write.
+ */
__SetPageUptodate(new_page);
pgtable = pmd_pgtable(_pmd);
_pmd = mk_huge_pmd(new_page, vma->vm_page_prot);
_pmd = maybe_pmd_mkwrite(pmd_mkdirty(_pmd), vma);
__SetPageUptodate(new_page);
pgtable = pmd_pgtable(_pmd);
_pmd = mk_huge_pmd(new_page, vma->vm_page_prot);
_pmd = maybe_pmd_mkwrite(pmd_mkdirty(_pmd), vma);
- /*
- * spin_lock() below is not the equivalent of smp_wmb(), so
- * this is needed to avoid the copy_huge_page writes to become
- * visible after the set_pmd_at() write.
- */
- smp_wmb();
-
spin_lock(pmd_ptl);
BUG_ON(!pmd_none(*pmd));
page_add_new_anon_rmap(new_page, vma, address, true);
spin_lock(pmd_ptl);
BUG_ON(!pmd_none(*pmd));
page_add_new_anon_rmap(new_page, vma, address, true);
@@
-1216,8
+1215,6
@@
out_nolock:
mem_cgroup_uncharge(*hpage);
trace_mm_collapse_huge_page(mm, isolated, result);
return;
mem_cgroup_uncharge(*hpage);
trace_mm_collapse_huge_page(mm, isolated, result);
return;
-out:
- goto out_up_write;
}
static int khugepaged_scan_pmd(struct mm_struct *mm,
}
static int khugepaged_scan_pmd(struct mm_struct *mm,
@@
-1274,10
+1271,6
@@
static int khugepaged_scan_pmd(struct mm_struct *mm,
goto out_unmap;
}
}
goto out_unmap;
}
}
- if (!pte_present(pteval)) {
- result = SCAN_PTE_NON_PRESENT;
- goto out_unmap;
- }
if (pte_uffd_wp(pteval)) {
/*
* Don't collapse the page if any of the small
if (pte_uffd_wp(pteval)) {
/*
* Don't collapse the page if any of the small
@@
-1447,7
+1440,7
@@
void collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr)
int i;
if (!vma || !vma->vm_file ||
int i;
if (!vma || !vma->vm_file ||
-
vma->vm_start > haddr || vma->vm_end < haddr + HPAGE_PMD_SIZE
)
+
!range_in_vma(vma, haddr, haddr + HPAGE_PMD_SIZE)
)
return;
/*
return;
/*
@@
-1533,16
+1526,16
@@
abort:
goto drop_hpage;
}
goto drop_hpage;
}
-static
int
khugepaged_collapse_pte_mapped_thps(struct mm_slot *mm_slot)
+static
void
khugepaged_collapse_pte_mapped_thps(struct mm_slot *mm_slot)
{
struct mm_struct *mm = mm_slot->mm;
int i;
if (likely(mm_slot->nr_pte_mapped_thp == 0))
{
struct mm_struct *mm = mm_slot->mm;
int i;
if (likely(mm_slot->nr_pte_mapped_thp == 0))
- return
0
;
+ return;
if (!mmap_write_trylock(mm))
if (!mmap_write_trylock(mm))
- return
-EBUSY
;
+ return;
if (unlikely(khugepaged_test_exit(mm)))
goto out;
if (unlikely(khugepaged_test_exit(mm)))
goto out;
@@
-1553,7
+1546,6
@@
static int khugepaged_collapse_pte_mapped_thps(struct mm_slot *mm_slot)
out:
mm_slot->nr_pte_mapped_thp = 0;
mmap_write_unlock(mm);
out:
mm_slot->nr_pte_mapped_thp = 0;
mmap_write_unlock(mm);
- return 0;
}
static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff)
}
static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff)
@@
-2057,9
+2049,8
@@
static void khugepaged_scan_file(struct mm_struct *mm,
BUILD_BUG();
}
BUILD_BUG();
}
-static
int
khugepaged_collapse_pte_mapped_thps(struct mm_slot *mm_slot)
+static
void
khugepaged_collapse_pte_mapped_thps(struct mm_slot *mm_slot)
{
{
- return 0;
}
#endif
}
#endif
@@
-2205,11
+2196,9
@@
static void khugepaged_do_scan(void)
{
struct page *hpage = NULL;
unsigned int progress = 0, pass_through_head = 0;
{
struct page *hpage = NULL;
unsigned int progress = 0, pass_through_head = 0;
- unsigned int pages =
khugepaged_pages_to_scan
;
+ unsigned int pages =
READ_ONCE(khugepaged_pages_to_scan)
;
bool wait = true;
bool wait = true;
- barrier(); /* write khugepaged_pages_to_scan to local stack */
-
lru_add_drain_all();
while (progress < pages) {
lru_add_drain_all();
while (progress < pages) {