#include <linux/cma.h>
#include <linux/migrate.h>
#include <linux/nospec.h>
+#include <linux/delayacct.h>
#include <asm/page.h>
#include <asm/pgalloc.h>
* sharing with another vma.
*/
;
- } else if (unlikely(is_hugetlb_entry_migration(entry) ||
- is_hugetlb_entry_hwpoisoned(entry))) {
+ } else if (unlikely(is_hugetlb_entry_hwpoisoned(entry))) {
+ bool uffd_wp = huge_pte_uffd_wp(entry);
+
+ if (!userfaultfd_wp(dst_vma) && uffd_wp)
+ entry = huge_pte_clear_uffd_wp(entry);
+ set_huge_pte_at(dst, addr, dst_pte, entry);
+ } else if (unlikely(is_hugetlb_entry_migration(entry))) {
swp_entry_t swp_entry = pte_to_swp_entry(entry);
bool uffd_wp = huge_pte_uffd_wp(entry);
pte = huge_ptep_get(ptep);
old_page = pte_page(pte);
+ delayacct_wpcopy_start();
+
retry_avoidcopy:
/*
* If no-one else is actually using this page, we're the exclusive
page_move_anon_rmap(old_page, vma);
if (likely(!unshare))
set_huge_ptep_writable(vma, haddr, ptep);
+
+ delayacct_wpcopy_end();
return 0;
}
VM_BUG_ON_PAGE(PageAnon(old_page) && PageAnonExclusive(old_page),
* race occurs while re-acquiring page table
* lock, and our job is done.
*/
+ delayacct_wpcopy_end();
return 0;
}
put_page(old_page);
spin_lock(ptl); /* Caller expects lock to be held */
+
+ delayacct_wpcopy_end();
return ret;
}
page = alloc_huge_page(dst_vma, dst_addr, 0);
if (IS_ERR(page)) {
+ put_page(*pagep);
ret = -ENOMEM;
*pagep = NULL;
goto out;