1 // SPDX-License-Identifier: GPL-2.0-only
5 * Copyright (C) 1993 Linus Torvalds
6 * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
7 * SMP-safe vmalloc/vfree/ioremap, Tigran Aivazian <tigran@veritas.com>, May 2000
8 * Major rework to support vmap/vunmap, Christoph Hellwig, SGI, August 2002
9 * Numa awareness, Christoph Lameter, SGI, June 2005
12 #include <linux/vmalloc.h>
14 #include <linux/module.h>
15 #include <linux/highmem.h>
16 #include <linux/sched/signal.h>
17 #include <linux/slab.h>
18 #include <linux/spinlock.h>
19 #include <linux/interrupt.h>
20 #include <linux/proc_fs.h>
21 #include <linux/seq_file.h>
22 #include <linux/set_memory.h>
23 #include <linux/debugobjects.h>
24 #include <linux/kallsyms.h>
25 #include <linux/list.h>
26 #include <linux/notifier.h>
27 #include <linux/rbtree.h>
28 #include <linux/radix-tree.h>
29 #include <linux/rcupdate.h>
30 #include <linux/pfn.h>
31 #include <linux/kmemleak.h>
32 #include <linux/atomic.h>
33 #include <linux/compiler.h>
34 #include <linux/llist.h>
35 #include <linux/bitops.h>
36 #include <linux/rbtree_augmented.h>
37 #include <linux/overflow.h>
39 #include <linux/uaccess.h>
40 #include <asm/tlbflush.h>
41 #include <asm/shmparam.h>
45 bool is_vmalloc_addr(const void *x)
47 unsigned long addr = (unsigned long)x;
49 return addr >= VMALLOC_START && addr < VMALLOC_END;
51 EXPORT_SYMBOL(is_vmalloc_addr);
53 struct vfree_deferred {
54 struct llist_head list;
55 struct work_struct wq;
57 static DEFINE_PER_CPU(struct vfree_deferred, vfree_deferred);
59 static void __vunmap(const void *, int);
61 static void free_work(struct work_struct *w)
63 struct vfree_deferred *p = container_of(w, struct vfree_deferred, wq);
64 struct llist_node *t, *llnode;
66 llist_for_each_safe(llnode, t, llist_del_all(&p->list))
67 __vunmap((void *)llnode, 1);
70 /*** Page table manipulation functions ***/
72 static void vunmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
77 pte = pte_offset_kernel(pmd, addr);
79 pte_t ptent = ptep_get_and_clear(&init_mm, addr, pte);
80 WARN_ON(!pte_none(ptent) && !pte_present(ptent));
81 } while (pte++, addr += PAGE_SIZE, addr != end);
82 *mask |= PGTBL_PTE_MODIFIED;
85 static void vunmap_pmd_range(pud_t *pud, unsigned long addr, unsigned long end,
92 pmd = pmd_offset(pud, addr);
94 next = pmd_addr_end(addr, end);
96 cleared = pmd_clear_huge(pmd);
97 if (cleared || pmd_bad(*pmd))
98 *mask |= PGTBL_PMD_MODIFIED;
102 if (pmd_none_or_clear_bad(pmd))
104 vunmap_pte_range(pmd, addr, next, mask);
105 } while (pmd++, addr = next, addr != end);
108 static void vunmap_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end,
109 pgtbl_mod_mask *mask)
115 pud = pud_offset(p4d, addr);
117 next = pud_addr_end(addr, end);
119 cleared = pud_clear_huge(pud);
120 if (cleared || pud_bad(*pud))
121 *mask |= PGTBL_PUD_MODIFIED;
125 if (pud_none_or_clear_bad(pud))
127 vunmap_pmd_range(pud, addr, next, mask);
128 } while (pud++, addr = next, addr != end);
131 static void vunmap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end,
132 pgtbl_mod_mask *mask)
138 p4d = p4d_offset(pgd, addr);
140 next = p4d_addr_end(addr, end);
142 cleared = p4d_clear_huge(p4d);
143 if (cleared || p4d_bad(*p4d))
144 *mask |= PGTBL_P4D_MODIFIED;
148 if (p4d_none_or_clear_bad(p4d))
150 vunmap_pud_range(p4d, addr, next, mask);
151 } while (p4d++, addr = next, addr != end);
155 * unmap_kernel_range_noflush - unmap kernel VM area
156 * @start: start of the VM area to unmap
157 * @size: size of the VM area to unmap
159 * Unmap PFN_UP(@size) pages at @addr. The VM area @addr and @size specify
160 * should have been allocated using get_vm_area() and its friends.
163 * This function does NOT do any cache flushing. The caller is responsible
164 * for calling flush_cache_vunmap() on to-be-mapped areas before calling this
165 * function and flush_tlb_kernel_range() after.
167 void unmap_kernel_range_noflush(unsigned long start, unsigned long size)
169 unsigned long end = start + size;
172 unsigned long addr = start;
173 pgtbl_mod_mask mask = 0;
177 pgd = pgd_offset_k(addr);
179 next = pgd_addr_end(addr, end);
181 mask |= PGTBL_PGD_MODIFIED;
182 if (pgd_none_or_clear_bad(pgd))
184 vunmap_p4d_range(pgd, addr, next, &mask);
185 } while (pgd++, addr = next, addr != end);
187 if (mask & ARCH_PAGE_TABLE_SYNC_MASK)
188 arch_sync_kernel_mappings(start, end);
191 static int vmap_pte_range(pmd_t *pmd, unsigned long addr,
192 unsigned long end, pgprot_t prot, struct page **pages, int *nr,
193 pgtbl_mod_mask *mask)
198 * nr is a running index into the array which helps higher level
199 * callers keep track of where we're up to.
202 pte = pte_alloc_kernel_track(pmd, addr, mask);
206 struct page *page = pages[*nr];
208 if (WARN_ON(!pte_none(*pte)))
212 set_pte_at(&init_mm, addr, pte, mk_pte(page, prot));
214 } while (pte++, addr += PAGE_SIZE, addr != end);
215 *mask |= PGTBL_PTE_MODIFIED;
219 static int vmap_pmd_range(pud_t *pud, unsigned long addr,
220 unsigned long end, pgprot_t prot, struct page **pages, int *nr,
221 pgtbl_mod_mask *mask)
226 pmd = pmd_alloc_track(&init_mm, pud, addr, mask);
230 next = pmd_addr_end(addr, end);
231 if (vmap_pte_range(pmd, addr, next, prot, pages, nr, mask))
233 } while (pmd++, addr = next, addr != end);
237 static int vmap_pud_range(p4d_t *p4d, unsigned long addr,
238 unsigned long end, pgprot_t prot, struct page **pages, int *nr,
239 pgtbl_mod_mask *mask)
244 pud = pud_alloc_track(&init_mm, p4d, addr, mask);
248 next = pud_addr_end(addr, end);
249 if (vmap_pmd_range(pud, addr, next, prot, pages, nr, mask))
251 } while (pud++, addr = next, addr != end);
255 static int vmap_p4d_range(pgd_t *pgd, unsigned long addr,
256 unsigned long end, pgprot_t prot, struct page **pages, int *nr,
257 pgtbl_mod_mask *mask)
262 p4d = p4d_alloc_track(&init_mm, pgd, addr, mask);
266 next = p4d_addr_end(addr, end);
267 if (vmap_pud_range(p4d, addr, next, prot, pages, nr, mask))
269 } while (p4d++, addr = next, addr != end);
274 * map_kernel_range_noflush - map kernel VM area with the specified pages
275 * @addr: start of the VM area to map
276 * @size: size of the VM area to map
277 * @prot: page protection flags to use
278 * @pages: pages to map
280 * Map PFN_UP(@size) pages at @addr. The VM area @addr and @size specify should
281 * have been allocated using get_vm_area() and its friends.
284 * This function does NOT do any cache flushing. The caller is responsible for
285 * calling flush_cache_vmap() on to-be-mapped areas before calling this
289 * 0 on success, -errno on failure.
291 int map_kernel_range_noflush(unsigned long addr, unsigned long size,
292 pgprot_t prot, struct page **pages)
294 unsigned long start = addr;
295 unsigned long end = addr + size;
300 pgtbl_mod_mask mask = 0;
303 pgd = pgd_offset_k(addr);
305 next = pgd_addr_end(addr, end);
307 mask |= PGTBL_PGD_MODIFIED;
308 err = vmap_p4d_range(pgd, addr, next, prot, pages, &nr, &mask);
311 } while (pgd++, addr = next, addr != end);
313 if (mask & ARCH_PAGE_TABLE_SYNC_MASK)
314 arch_sync_kernel_mappings(start, end);
319 int map_kernel_range(unsigned long start, unsigned long size, pgprot_t prot,
324 ret = map_kernel_range_noflush(start, size, prot, pages);
325 flush_cache_vmap(start, start + size);
329 int is_vmalloc_or_module_addr(const void *x)
332 * ARM, x86-64 and sparc64 put modules in a special place,
333 * and fall back on vmalloc() if that fails. Others
334 * just put it in the vmalloc space.
336 #if defined(CONFIG_MODULES) && defined(MODULES_VADDR)
337 unsigned long addr = (unsigned long)x;
338 if (addr >= MODULES_VADDR && addr < MODULES_END)
341 return is_vmalloc_addr(x);
345 * Walk a vmap address to the struct page it maps.
347 struct page *vmalloc_to_page(const void *vmalloc_addr)
349 unsigned long addr = (unsigned long) vmalloc_addr;
350 struct page *page = NULL;
351 pgd_t *pgd = pgd_offset_k(addr);
358 * XXX we might need to change this if we add VIRTUAL_BUG_ON for
359 * architectures that do not vmalloc module space
361 VIRTUAL_BUG_ON(!is_vmalloc_or_module_addr(vmalloc_addr));
365 p4d = p4d_offset(pgd, addr);
368 pud = pud_offset(p4d, addr);
371 * Don't dereference bad PUD or PMD (below) entries. This will also
372 * identify huge mappings, which we may encounter on architectures
373 * that define CONFIG_HAVE_ARCH_HUGE_VMAP=y. Such regions will be
374 * identified as vmalloc addresses by is_vmalloc_addr(), but are
375 * not [unambiguously] associated with a struct page, so there is
376 * no correct value to return for them.
378 WARN_ON_ONCE(pud_bad(*pud));
379 if (pud_none(*pud) || pud_bad(*pud))
381 pmd = pmd_offset(pud, addr);
382 WARN_ON_ONCE(pmd_bad(*pmd));
383 if (pmd_none(*pmd) || pmd_bad(*pmd))
386 ptep = pte_offset_map(pmd, addr);
388 if (pte_present(pte))
389 page = pte_page(pte);
393 EXPORT_SYMBOL(vmalloc_to_page);
396 * Map a vmalloc()-space virtual address to the physical page frame number.
398 unsigned long vmalloc_to_pfn(const void *vmalloc_addr)
400 return page_to_pfn(vmalloc_to_page(vmalloc_addr));
402 EXPORT_SYMBOL(vmalloc_to_pfn);
405 /*** Global kva allocator ***/
407 #define DEBUG_AUGMENT_PROPAGATE_CHECK 0
408 #define DEBUG_AUGMENT_LOWEST_MATCH_CHECK 0
411 static DEFINE_SPINLOCK(vmap_area_lock);
412 static DEFINE_SPINLOCK(free_vmap_area_lock);
413 /* Export for kexec only */
414 LIST_HEAD(vmap_area_list);
415 static LLIST_HEAD(vmap_purge_list);
416 static struct rb_root vmap_area_root = RB_ROOT;
417 static bool vmap_initialized __read_mostly;
420 * This kmem_cache is used for vmap_area objects. Instead of
421 * allocating from slab we reuse an object from this cache to
422 * make things faster. Especially in "no edge" splitting of
425 static struct kmem_cache *vmap_area_cachep;
428 * This linked list is used in pair with free_vmap_area_root.
429 * It gives O(1) access to prev/next to perform fast coalescing.
431 static LIST_HEAD(free_vmap_area_list);
434 * This augment red-black tree represents the free vmap space.
435 * All vmap_area objects in this tree are sorted by va->va_start
436 * address. It is used for allocation and merging when a vmap
437 * object is released.
439 * Each vmap_area node contains a maximum available free block
440 * of its sub-tree, right or left. Therefore it is possible to
441 * find a lowest match of free area.
443 static struct rb_root free_vmap_area_root = RB_ROOT;
446 * Preload a CPU with one object for "no edge" split case. The
447 * aim is to get rid of allocations from the atomic context, thus
448 * to use more permissive allocation masks.
450 static DEFINE_PER_CPU(struct vmap_area *, ne_fit_preload_node);
452 static __always_inline unsigned long
453 va_size(struct vmap_area *va)
455 return (va->va_end - va->va_start);
458 static __always_inline unsigned long
459 get_subtree_max_size(struct rb_node *node)
461 struct vmap_area *va;
463 va = rb_entry_safe(node, struct vmap_area, rb_node);
464 return va ? va->subtree_max_size : 0;
468 * Gets called when remove the node and rotate.
470 static __always_inline unsigned long
471 compute_subtree_max_size(struct vmap_area *va)
473 return max3(va_size(va),
474 get_subtree_max_size(va->rb_node.rb_left),
475 get_subtree_max_size(va->rb_node.rb_right));
478 RB_DECLARE_CALLBACKS_MAX(static, free_vmap_area_rb_augment_cb,
479 struct vmap_area, rb_node, unsigned long, subtree_max_size, va_size)
481 static void purge_vmap_area_lazy(void);
482 static BLOCKING_NOTIFIER_HEAD(vmap_notify_list);
483 static unsigned long lazy_max_pages(void);
485 static atomic_long_t nr_vmalloc_pages;
487 unsigned long vmalloc_nr_pages(void)
489 return atomic_long_read(&nr_vmalloc_pages);
492 static struct vmap_area *__find_vmap_area(unsigned long addr)
494 struct rb_node *n = vmap_area_root.rb_node;
497 struct vmap_area *va;
499 va = rb_entry(n, struct vmap_area, rb_node);
500 if (addr < va->va_start)
502 else if (addr >= va->va_end)
512 * This function returns back addresses of parent node
513 * and its left or right link for further processing.
515 static __always_inline struct rb_node **
516 find_va_links(struct vmap_area *va,
517 struct rb_root *root, struct rb_node *from,
518 struct rb_node **parent)
520 struct vmap_area *tmp_va;
521 struct rb_node **link;
524 link = &root->rb_node;
525 if (unlikely(!*link)) {
534 * Go to the bottom of the tree. When we hit the last point
535 * we end up with parent rb_node and correct direction, i name
536 * it link, where the new va->rb_node will be attached to.
539 tmp_va = rb_entry(*link, struct vmap_area, rb_node);
542 * During the traversal we also do some sanity check.
543 * Trigger the BUG() if there are sides(left/right)
546 if (va->va_start < tmp_va->va_end &&
547 va->va_end <= tmp_va->va_start)
548 link = &(*link)->rb_left;
549 else if (va->va_end > tmp_va->va_start &&
550 va->va_start >= tmp_va->va_end)
551 link = &(*link)->rb_right;
556 *parent = &tmp_va->rb_node;
560 static __always_inline struct list_head *
561 get_va_next_sibling(struct rb_node *parent, struct rb_node **link)
563 struct list_head *list;
565 if (unlikely(!parent))
567 * The red-black tree where we try to find VA neighbors
568 * before merging or inserting is empty, i.e. it means
569 * there is no free vmap space. Normally it does not
570 * happen but we handle this case anyway.
574 list = &rb_entry(parent, struct vmap_area, rb_node)->list;
575 return (&parent->rb_right == link ? list->next : list);
578 static __always_inline void
579 link_va(struct vmap_area *va, struct rb_root *root,
580 struct rb_node *parent, struct rb_node **link, struct list_head *head)
583 * VA is still not in the list, but we can
584 * identify its future previous list_head node.
586 if (likely(parent)) {
587 head = &rb_entry(parent, struct vmap_area, rb_node)->list;
588 if (&parent->rb_right != link)
592 /* Insert to the rb-tree */
593 rb_link_node(&va->rb_node, parent, link);
594 if (root == &free_vmap_area_root) {
596 * Some explanation here. Just perform simple insertion
597 * to the tree. We do not set va->subtree_max_size to
598 * its current size before calling rb_insert_augmented().
599 * It is because of we populate the tree from the bottom
600 * to parent levels when the node _is_ in the tree.
602 * Therefore we set subtree_max_size to zero after insertion,
603 * to let __augment_tree_propagate_from() puts everything to
604 * the correct order later on.
606 rb_insert_augmented(&va->rb_node,
607 root, &free_vmap_area_rb_augment_cb);
608 va->subtree_max_size = 0;
610 rb_insert_color(&va->rb_node, root);
613 /* Address-sort this list */
614 list_add(&va->list, head);
617 static __always_inline void
618 unlink_va(struct vmap_area *va, struct rb_root *root)
620 if (WARN_ON(RB_EMPTY_NODE(&va->rb_node)))
623 if (root == &free_vmap_area_root)
624 rb_erase_augmented(&va->rb_node,
625 root, &free_vmap_area_rb_augment_cb);
627 rb_erase(&va->rb_node, root);
630 RB_CLEAR_NODE(&va->rb_node);
633 #if DEBUG_AUGMENT_PROPAGATE_CHECK
635 augment_tree_propagate_check(struct rb_node *n)
637 struct vmap_area *va;
638 struct rb_node *node;
645 va = rb_entry(n, struct vmap_area, rb_node);
646 size = va->subtree_max_size;
650 va = rb_entry(node, struct vmap_area, rb_node);
652 if (get_subtree_max_size(node->rb_left) == size) {
653 node = node->rb_left;
655 if (va_size(va) == size) {
660 node = node->rb_right;
665 va = rb_entry(n, struct vmap_area, rb_node);
666 pr_emerg("tree is corrupted: %lu, %lu\n",
667 va_size(va), va->subtree_max_size);
670 augment_tree_propagate_check(n->rb_left);
671 augment_tree_propagate_check(n->rb_right);
676 * This function populates subtree_max_size from bottom to upper
677 * levels starting from VA point. The propagation must be done
678 * when VA size is modified by changing its va_start/va_end. Or
679 * in case of newly inserting of VA to the tree.
681 * It means that __augment_tree_propagate_from() must be called:
682 * - After VA has been inserted to the tree(free path);
683 * - After VA has been shrunk(allocation path);
684 * - After VA has been increased(merging path).
686 * Please note that, it does not mean that upper parent nodes
687 * and their subtree_max_size are recalculated all the time up
696 * For example if we modify the node 4, shrinking it to 2, then
697 * no any modification is required. If we shrink the node 2 to 1
698 * its subtree_max_size is updated only, and set to 1. If we shrink
699 * the node 8 to 6, then its subtree_max_size is set to 6 and parent
702 static __always_inline void
703 augment_tree_propagate_from(struct vmap_area *va)
705 struct rb_node *node = &va->rb_node;
706 unsigned long new_va_sub_max_size;
709 va = rb_entry(node, struct vmap_area, rb_node);
710 new_va_sub_max_size = compute_subtree_max_size(va);
713 * If the newly calculated maximum available size of the
714 * subtree is equal to the current one, then it means that
715 * the tree is propagated correctly. So we have to stop at
716 * this point to save cycles.
718 if (va->subtree_max_size == new_va_sub_max_size)
721 va->subtree_max_size = new_va_sub_max_size;
722 node = rb_parent(&va->rb_node);
725 #if DEBUG_AUGMENT_PROPAGATE_CHECK
726 augment_tree_propagate_check(free_vmap_area_root.rb_node);
731 insert_vmap_area(struct vmap_area *va,
732 struct rb_root *root, struct list_head *head)
734 struct rb_node **link;
735 struct rb_node *parent;
737 link = find_va_links(va, root, NULL, &parent);
738 link_va(va, root, parent, link, head);
742 insert_vmap_area_augment(struct vmap_area *va,
743 struct rb_node *from, struct rb_root *root,
744 struct list_head *head)
746 struct rb_node **link;
747 struct rb_node *parent;
750 link = find_va_links(va, NULL, from, &parent);
752 link = find_va_links(va, root, NULL, &parent);
754 link_va(va, root, parent, link, head);
755 augment_tree_propagate_from(va);
759 * Merge de-allocated chunk of VA memory with previous
760 * and next free blocks. If coalesce is not done a new
761 * free area is inserted. If VA has been merged, it is
764 static __always_inline struct vmap_area *
765 merge_or_add_vmap_area(struct vmap_area *va,
766 struct rb_root *root, struct list_head *head)
768 struct vmap_area *sibling;
769 struct list_head *next;
770 struct rb_node **link;
771 struct rb_node *parent;
775 * Find a place in the tree where VA potentially will be
776 * inserted, unless it is merged with its sibling/siblings.
778 link = find_va_links(va, root, NULL, &parent);
781 * Get next node of VA to check if merging can be done.
783 next = get_va_next_sibling(parent, link);
784 if (unlikely(next == NULL))
790 * |<------VA------>|<-----Next----->|
795 sibling = list_entry(next, struct vmap_area, list);
796 if (sibling->va_start == va->va_end) {
797 sibling->va_start = va->va_start;
799 /* Check and update the tree if needed. */
800 augment_tree_propagate_from(sibling);
802 /* Free vmap_area object. */
803 kmem_cache_free(vmap_area_cachep, va);
805 /* Point to the new merged area. */
814 * |<-----Prev----->|<------VA------>|
818 if (next->prev != head) {
819 sibling = list_entry(next->prev, struct vmap_area, list);
820 if (sibling->va_end == va->va_start) {
821 sibling->va_end = va->va_end;
823 /* Check and update the tree if needed. */
824 augment_tree_propagate_from(sibling);
829 /* Free vmap_area object. */
830 kmem_cache_free(vmap_area_cachep, va);
832 /* Point to the new merged area. */
840 link_va(va, root, parent, link, head);
841 augment_tree_propagate_from(va);
847 static __always_inline bool
848 is_within_this_va(struct vmap_area *va, unsigned long size,
849 unsigned long align, unsigned long vstart)
851 unsigned long nva_start_addr;
853 if (va->va_start > vstart)
854 nva_start_addr = ALIGN(va->va_start, align);
856 nva_start_addr = ALIGN(vstart, align);
858 /* Can be overflowed due to big size or alignment. */
859 if (nva_start_addr + size < nva_start_addr ||
860 nva_start_addr < vstart)
863 return (nva_start_addr + size <= va->va_end);
867 * Find the first free block(lowest start address) in the tree,
868 * that will accomplish the request corresponding to passing
871 static __always_inline struct vmap_area *
872 find_vmap_lowest_match(unsigned long size,
873 unsigned long align, unsigned long vstart)
875 struct vmap_area *va;
876 struct rb_node *node;
877 unsigned long length;
879 /* Start from the root. */
880 node = free_vmap_area_root.rb_node;
882 /* Adjust the search size for alignment overhead. */
883 length = size + align - 1;
886 va = rb_entry(node, struct vmap_area, rb_node);
888 if (get_subtree_max_size(node->rb_left) >= length &&
889 vstart < va->va_start) {
890 node = node->rb_left;
892 if (is_within_this_va(va, size, align, vstart))
896 * Does not make sense to go deeper towards the right
897 * sub-tree if it does not have a free block that is
898 * equal or bigger to the requested search length.
900 if (get_subtree_max_size(node->rb_right) >= length) {
901 node = node->rb_right;
906 * OK. We roll back and find the first right sub-tree,
907 * that will satisfy the search criteria. It can happen
908 * only once due to "vstart" restriction.
910 while ((node = rb_parent(node))) {
911 va = rb_entry(node, struct vmap_area, rb_node);
912 if (is_within_this_va(va, size, align, vstart))
915 if (get_subtree_max_size(node->rb_right) >= length &&
916 vstart <= va->va_start) {
917 node = node->rb_right;
927 #if DEBUG_AUGMENT_LOWEST_MATCH_CHECK
928 #include <linux/random.h>
930 static struct vmap_area *
931 find_vmap_lowest_linear_match(unsigned long size,
932 unsigned long align, unsigned long vstart)
934 struct vmap_area *va;
936 list_for_each_entry(va, &free_vmap_area_list, list) {
937 if (!is_within_this_va(va, size, align, vstart))
947 find_vmap_lowest_match_check(unsigned long size)
949 struct vmap_area *va_1, *va_2;
950 unsigned long vstart;
953 get_random_bytes(&rnd, sizeof(rnd));
954 vstart = VMALLOC_START + rnd;
956 va_1 = find_vmap_lowest_match(size, 1, vstart);
957 va_2 = find_vmap_lowest_linear_match(size, 1, vstart);
960 pr_emerg("not lowest: t: 0x%p, l: 0x%p, v: 0x%lx\n",
967 FL_FIT_TYPE = 1, /* full fit */
968 LE_FIT_TYPE = 2, /* left edge fit */
969 RE_FIT_TYPE = 3, /* right edge fit */
970 NE_FIT_TYPE = 4 /* no edge fit */
973 static __always_inline enum fit_type
974 classify_va_fit_type(struct vmap_area *va,
975 unsigned long nva_start_addr, unsigned long size)
979 /* Check if it is within VA. */
980 if (nva_start_addr < va->va_start ||
981 nva_start_addr + size > va->va_end)
985 if (va->va_start == nva_start_addr) {
986 if (va->va_end == nva_start_addr + size)
990 } else if (va->va_end == nva_start_addr + size) {
999 static __always_inline int
1000 adjust_va_to_fit_type(struct vmap_area *va,
1001 unsigned long nva_start_addr, unsigned long size,
1004 struct vmap_area *lva = NULL;
1006 if (type == FL_FIT_TYPE) {
1008 * No need to split VA, it fully fits.
1014 unlink_va(va, &free_vmap_area_root);
1015 kmem_cache_free(vmap_area_cachep, va);
1016 } else if (type == LE_FIT_TYPE) {
1018 * Split left edge of fit VA.
1024 va->va_start += size;
1025 } else if (type == RE_FIT_TYPE) {
1027 * Split right edge of fit VA.
1033 va->va_end = nva_start_addr;
1034 } else if (type == NE_FIT_TYPE) {
1036 * Split no edge of fit VA.
1042 lva = __this_cpu_xchg(ne_fit_preload_node, NULL);
1043 if (unlikely(!lva)) {
1045 * For percpu allocator we do not do any pre-allocation
1046 * and leave it as it is. The reason is it most likely
1047 * never ends up with NE_FIT_TYPE splitting. In case of
1048 * percpu allocations offsets and sizes are aligned to
1049 * fixed align request, i.e. RE_FIT_TYPE and FL_FIT_TYPE
1050 * are its main fitting cases.
1052 * There are a few exceptions though, as an example it is
1053 * a first allocation (early boot up) when we have "one"
1054 * big free space that has to be split.
1056 * Also we can hit this path in case of regular "vmap"
1057 * allocations, if "this" current CPU was not preloaded.
1058 * See the comment in alloc_vmap_area() why. If so, then
1059 * GFP_NOWAIT is used instead to get an extra object for
1060 * split purpose. That is rare and most time does not
1063 * What happens if an allocation gets failed. Basically,
1064 * an "overflow" path is triggered to purge lazily freed
1065 * areas to free some memory, then, the "retry" path is
1066 * triggered to repeat one more time. See more details
1067 * in alloc_vmap_area() function.
1069 lva = kmem_cache_alloc(vmap_area_cachep, GFP_NOWAIT);
1075 * Build the remainder.
1077 lva->va_start = va->va_start;
1078 lva->va_end = nva_start_addr;
1081 * Shrink this VA to remaining size.
1083 va->va_start = nva_start_addr + size;
1088 if (type != FL_FIT_TYPE) {
1089 augment_tree_propagate_from(va);
1091 if (lva) /* type == NE_FIT_TYPE */
1092 insert_vmap_area_augment(lva, &va->rb_node,
1093 &free_vmap_area_root, &free_vmap_area_list);
1100 * Returns a start address of the newly allocated area, if success.
1101 * Otherwise a vend is returned that indicates failure.
1103 static __always_inline unsigned long
1104 __alloc_vmap_area(unsigned long size, unsigned long align,
1105 unsigned long vstart, unsigned long vend)
1107 unsigned long nva_start_addr;
1108 struct vmap_area *va;
1112 va = find_vmap_lowest_match(size, align, vstart);
1116 if (va->va_start > vstart)
1117 nva_start_addr = ALIGN(va->va_start, align);
1119 nva_start_addr = ALIGN(vstart, align);
1121 /* Check the "vend" restriction. */
1122 if (nva_start_addr + size > vend)
1125 /* Classify what we have found. */
1126 type = classify_va_fit_type(va, nva_start_addr, size);
1127 if (WARN_ON_ONCE(type == NOTHING_FIT))
1130 /* Update the free vmap_area. */
1131 ret = adjust_va_to_fit_type(va, nva_start_addr, size, type);
1135 #if DEBUG_AUGMENT_LOWEST_MATCH_CHECK
1136 find_vmap_lowest_match_check(size);
1139 return nva_start_addr;
1143 * Free a region of KVA allocated by alloc_vmap_area
1145 static void free_vmap_area(struct vmap_area *va)
1148 * Remove from the busy tree/list.
1150 spin_lock(&vmap_area_lock);
1151 unlink_va(va, &vmap_area_root);
1152 spin_unlock(&vmap_area_lock);
1155 * Insert/Merge it back to the free tree/list.
1157 spin_lock(&free_vmap_area_lock);
1158 merge_or_add_vmap_area(va, &free_vmap_area_root, &free_vmap_area_list);
1159 spin_unlock(&free_vmap_area_lock);
1163 * Allocate a region of KVA of the specified size and alignment, within the
1166 static struct vmap_area *alloc_vmap_area(unsigned long size,
1167 unsigned long align,
1168 unsigned long vstart, unsigned long vend,
1169 int node, gfp_t gfp_mask)
1171 struct vmap_area *va, *pva;
1177 BUG_ON(offset_in_page(size));
1178 BUG_ON(!is_power_of_2(align));
1180 if (unlikely(!vmap_initialized))
1181 return ERR_PTR(-EBUSY);
1184 gfp_mask = gfp_mask & GFP_RECLAIM_MASK;
1186 va = kmem_cache_alloc_node(vmap_area_cachep, gfp_mask, node);
1188 return ERR_PTR(-ENOMEM);
1191 * Only scan the relevant parts containing pointers to other objects
1192 * to avoid false negatives.
1194 kmemleak_scan_area(&va->rb_node, SIZE_MAX, gfp_mask);
1198 * Preload this CPU with one extra vmap_area object. It is used
1199 * when fit type of free area is NE_FIT_TYPE. Please note, it
1200 * does not guarantee that an allocation occurs on a CPU that
1201 * is preloaded, instead we minimize the case when it is not.
1202 * It can happen because of cpu migration, because there is a
1203 * race until the below spinlock is taken.
1205 * The preload is done in non-atomic context, thus it allows us
1206 * to use more permissive allocation masks to be more stable under
1207 * low memory condition and high memory pressure. In rare case,
1208 * if not preloaded, GFP_NOWAIT is used.
1210 * Set "pva" to NULL here, because of "retry" path.
1214 if (!this_cpu_read(ne_fit_preload_node))
1216 * Even if it fails we do not really care about that.
1217 * Just proceed as it is. If needed "overflow" path
1218 * will refill the cache we allocate from.
1220 pva = kmem_cache_alloc_node(vmap_area_cachep, gfp_mask, node);
1222 spin_lock(&free_vmap_area_lock);
1224 if (pva && __this_cpu_cmpxchg(ne_fit_preload_node, NULL, pva))
1225 kmem_cache_free(vmap_area_cachep, pva);
1228 * If an allocation fails, the "vend" address is
1229 * returned. Therefore trigger the overflow path.
1231 addr = __alloc_vmap_area(size, align, vstart, vend);
1232 spin_unlock(&free_vmap_area_lock);
1234 if (unlikely(addr == vend))
1237 va->va_start = addr;
1238 va->va_end = addr + size;
1242 spin_lock(&vmap_area_lock);
1243 insert_vmap_area(va, &vmap_area_root, &vmap_area_list);
1244 spin_unlock(&vmap_area_lock);
1246 BUG_ON(!IS_ALIGNED(va->va_start, align));
1247 BUG_ON(va->va_start < vstart);
1248 BUG_ON(va->va_end > vend);
1250 ret = kasan_populate_vmalloc(addr, size);
1253 return ERR_PTR(ret);
1260 purge_vmap_area_lazy();
1265 if (gfpflags_allow_blocking(gfp_mask)) {
1266 unsigned long freed = 0;
1267 blocking_notifier_call_chain(&vmap_notify_list, 0, &freed);
1274 if (!(gfp_mask & __GFP_NOWARN) && printk_ratelimit())
1275 pr_warn("vmap allocation for size %lu failed: use vmalloc=<size> to increase size\n",
1278 kmem_cache_free(vmap_area_cachep, va);
1279 return ERR_PTR(-EBUSY);
1282 int register_vmap_purge_notifier(struct notifier_block *nb)
1284 return blocking_notifier_chain_register(&vmap_notify_list, nb);
1286 EXPORT_SYMBOL_GPL(register_vmap_purge_notifier);
1288 int unregister_vmap_purge_notifier(struct notifier_block *nb)
1290 return blocking_notifier_chain_unregister(&vmap_notify_list, nb);
1292 EXPORT_SYMBOL_GPL(unregister_vmap_purge_notifier);
1295 * lazy_max_pages is the maximum amount of virtual address space we gather up
1296 * before attempting to purge with a TLB flush.
1298 * There is a tradeoff here: a larger number will cover more kernel page tables
1299 * and take slightly longer to purge, but it will linearly reduce the number of
1300 * global TLB flushes that must be performed. It would seem natural to scale
1301 * this number up linearly with the number of CPUs (because vmapping activity
1302 * could also scale linearly with the number of CPUs), however it is likely
1303 * that in practice, workloads might be constrained in other ways that mean
1304 * vmap activity will not scale linearly with CPUs. Also, I want to be
1305 * conservative and not introduce a big latency on huge systems, so go with
1306 * a less aggressive log scale. It will still be an improvement over the old
1307 * code, and it will be simple to change the scale factor if we find that it
1308 * becomes a problem on bigger systems.
1310 static unsigned long lazy_max_pages(void)
1314 log = fls(num_online_cpus());
1316 return log * (32UL * 1024 * 1024 / PAGE_SIZE);
1319 static atomic_long_t vmap_lazy_nr = ATOMIC_LONG_INIT(0);
1322 * Serialize vmap purging. There is no actual criticial section protected
1323 * by this look, but we want to avoid concurrent calls for performance
1324 * reasons and to make the pcpu_get_vm_areas more deterministic.
1326 static DEFINE_MUTEX(vmap_purge_lock);
1328 /* for per-CPU blocks */
1329 static void purge_fragmented_blocks_allcpus(void);
1332 * called before a call to iounmap() if the caller wants vm_area_struct's
1333 * immediately freed.
1335 void set_iounmap_nonlazy(void)
1337 atomic_long_set(&vmap_lazy_nr, lazy_max_pages()+1);
1341 * Purges all lazily-freed vmap areas.
1343 static bool __purge_vmap_area_lazy(unsigned long start, unsigned long end)
1345 unsigned long resched_threshold;
1346 struct llist_node *valist;
1347 struct vmap_area *va;
1348 struct vmap_area *n_va;
1350 lockdep_assert_held(&vmap_purge_lock);
1352 valist = llist_del_all(&vmap_purge_list);
1353 if (unlikely(valist == NULL))
1357 * First make sure the mappings are removed from all page-tables
1358 * before they are freed.
1360 vmalloc_sync_unmappings();
1363 * TODO: to calculate a flush range without looping.
1364 * The list can be up to lazy_max_pages() elements.
1366 llist_for_each_entry(va, valist, purge_list) {
1367 if (va->va_start < start)
1368 start = va->va_start;
1369 if (va->va_end > end)
1373 flush_tlb_kernel_range(start, end);
1374 resched_threshold = lazy_max_pages() << 1;
1376 spin_lock(&free_vmap_area_lock);
1377 llist_for_each_entry_safe(va, n_va, valist, purge_list) {
1378 unsigned long nr = (va->va_end - va->va_start) >> PAGE_SHIFT;
1379 unsigned long orig_start = va->va_start;
1380 unsigned long orig_end = va->va_end;
1383 * Finally insert or merge lazily-freed area. It is
1384 * detached and there is no need to "unlink" it from
1387 va = merge_or_add_vmap_area(va, &free_vmap_area_root,
1388 &free_vmap_area_list);
1390 if (is_vmalloc_or_module_addr((void *)orig_start))
1391 kasan_release_vmalloc(orig_start, orig_end,
1392 va->va_start, va->va_end);
1394 atomic_long_sub(nr, &vmap_lazy_nr);
1396 if (atomic_long_read(&vmap_lazy_nr) < resched_threshold)
1397 cond_resched_lock(&free_vmap_area_lock);
1399 spin_unlock(&free_vmap_area_lock);
1404 * Kick off a purge of the outstanding lazy areas. Don't bother if somebody
1405 * is already purging.
1407 static void try_purge_vmap_area_lazy(void)
1409 if (mutex_trylock(&vmap_purge_lock)) {
1410 __purge_vmap_area_lazy(ULONG_MAX, 0);
1411 mutex_unlock(&vmap_purge_lock);
1416 * Kick off a purge of the outstanding lazy areas.
1418 static void purge_vmap_area_lazy(void)
1420 mutex_lock(&vmap_purge_lock);
1421 purge_fragmented_blocks_allcpus();
1422 __purge_vmap_area_lazy(ULONG_MAX, 0);
1423 mutex_unlock(&vmap_purge_lock);
1427 * Free a vmap area, caller ensuring that the area has been unmapped
1428 * and flush_cache_vunmap had been called for the correct range
1431 static void free_vmap_area_noflush(struct vmap_area *va)
1433 unsigned long nr_lazy;
1435 spin_lock(&vmap_area_lock);
1436 unlink_va(va, &vmap_area_root);
1437 spin_unlock(&vmap_area_lock);
1439 nr_lazy = atomic_long_add_return((va->va_end - va->va_start) >>
1440 PAGE_SHIFT, &vmap_lazy_nr);
1442 /* After this point, we may free va at any time */
1443 llist_add(&va->purge_list, &vmap_purge_list);
1445 if (unlikely(nr_lazy > lazy_max_pages()))
1446 try_purge_vmap_area_lazy();
1450 * Free and unmap a vmap area
1452 static void free_unmap_vmap_area(struct vmap_area *va)
1454 flush_cache_vunmap(va->va_start, va->va_end);
1455 unmap_kernel_range_noflush(va->va_start, va->va_end - va->va_start);
1456 if (debug_pagealloc_enabled_static())
1457 flush_tlb_kernel_range(va->va_start, va->va_end);
1459 free_vmap_area_noflush(va);
1462 static struct vmap_area *find_vmap_area(unsigned long addr)
1464 struct vmap_area *va;
1466 spin_lock(&vmap_area_lock);
1467 va = __find_vmap_area(addr);
1468 spin_unlock(&vmap_area_lock);
1473 /*** Per cpu kva allocator ***/
1476 * vmap space is limited especially on 32 bit architectures. Ensure there is
1477 * room for at least 16 percpu vmap blocks per CPU.
1480 * If we had a constant VMALLOC_START and VMALLOC_END, we'd like to be able
1481 * to #define VMALLOC_SPACE (VMALLOC_END-VMALLOC_START). Guess
1482 * instead (we just need a rough idea)
1484 #if BITS_PER_LONG == 32
1485 #define VMALLOC_SPACE (128UL*1024*1024)
1487 #define VMALLOC_SPACE (128UL*1024*1024*1024)
1490 #define VMALLOC_PAGES (VMALLOC_SPACE / PAGE_SIZE)
1491 #define VMAP_MAX_ALLOC BITS_PER_LONG /* 256K with 4K pages */
1492 #define VMAP_BBMAP_BITS_MAX 1024 /* 4MB with 4K pages */
1493 #define VMAP_BBMAP_BITS_MIN (VMAP_MAX_ALLOC*2)
1494 #define VMAP_MIN(x, y) ((x) < (y) ? (x) : (y)) /* can't use min() */
1495 #define VMAP_MAX(x, y) ((x) > (y) ? (x) : (y)) /* can't use max() */
1496 #define VMAP_BBMAP_BITS \
1497 VMAP_MIN(VMAP_BBMAP_BITS_MAX, \
1498 VMAP_MAX(VMAP_BBMAP_BITS_MIN, \
1499 VMALLOC_PAGES / roundup_pow_of_two(NR_CPUS) / 16))
1501 #define VMAP_BLOCK_SIZE (VMAP_BBMAP_BITS * PAGE_SIZE)
1503 struct vmap_block_queue {
1505 struct list_head free;
1510 struct vmap_area *va;
1511 unsigned long free, dirty;
1512 unsigned long dirty_min, dirty_max; /*< dirty range */
1513 struct list_head free_list;
1514 struct rcu_head rcu_head;
1515 struct list_head purge;
1518 /* Queue of free and dirty vmap blocks, for allocation and flushing purposes */
1519 static DEFINE_PER_CPU(struct vmap_block_queue, vmap_block_queue);
1522 * Radix tree of vmap blocks, indexed by address, to quickly find a vmap block
1523 * in the free path. Could get rid of this if we change the API to return a
1524 * "cookie" from alloc, to be passed to free. But no big deal yet.
1526 static DEFINE_SPINLOCK(vmap_block_tree_lock);
1527 static RADIX_TREE(vmap_block_tree, GFP_ATOMIC);
1530 * We should probably have a fallback mechanism to allocate virtual memory
1531 * out of partially filled vmap blocks. However vmap block sizing should be
1532 * fairly reasonable according to the vmalloc size, so it shouldn't be a
1536 static unsigned long addr_to_vb_idx(unsigned long addr)
1538 addr -= VMALLOC_START & ~(VMAP_BLOCK_SIZE-1);
1539 addr /= VMAP_BLOCK_SIZE;
1543 static void *vmap_block_vaddr(unsigned long va_start, unsigned long pages_off)
1547 addr = va_start + (pages_off << PAGE_SHIFT);
1548 BUG_ON(addr_to_vb_idx(addr) != addr_to_vb_idx(va_start));
1549 return (void *)addr;
1553 * new_vmap_block - allocates new vmap_block and occupies 2^order pages in this
1554 * block. Of course pages number can't exceed VMAP_BBMAP_BITS
1555 * @order: how many 2^order pages should be occupied in newly allocated block
1556 * @gfp_mask: flags for the page level allocator
1558 * Return: virtual address in a newly allocated block or ERR_PTR(-errno)
1560 static void *new_vmap_block(unsigned int order, gfp_t gfp_mask)
1562 struct vmap_block_queue *vbq;
1563 struct vmap_block *vb;
1564 struct vmap_area *va;
1565 unsigned long vb_idx;
1569 node = numa_node_id();
1571 vb = kmalloc_node(sizeof(struct vmap_block),
1572 gfp_mask & GFP_RECLAIM_MASK, node);
1574 return ERR_PTR(-ENOMEM);
1576 va = alloc_vmap_area(VMAP_BLOCK_SIZE, VMAP_BLOCK_SIZE,
1577 VMALLOC_START, VMALLOC_END,
1581 return ERR_CAST(va);
1584 err = radix_tree_preload(gfp_mask);
1585 if (unlikely(err)) {
1588 return ERR_PTR(err);
1591 vaddr = vmap_block_vaddr(va->va_start, 0);
1592 spin_lock_init(&vb->lock);
1594 /* At least something should be left free */
1595 BUG_ON(VMAP_BBMAP_BITS <= (1UL << order));
1596 vb->free = VMAP_BBMAP_BITS - (1UL << order);
1598 vb->dirty_min = VMAP_BBMAP_BITS;
1600 INIT_LIST_HEAD(&vb->free_list);
1602 vb_idx = addr_to_vb_idx(va->va_start);
1603 spin_lock(&vmap_block_tree_lock);
1604 err = radix_tree_insert(&vmap_block_tree, vb_idx, vb);
1605 spin_unlock(&vmap_block_tree_lock);
1607 radix_tree_preload_end();
1609 vbq = &get_cpu_var(vmap_block_queue);
1610 spin_lock(&vbq->lock);
1611 list_add_tail_rcu(&vb->free_list, &vbq->free);
1612 spin_unlock(&vbq->lock);
1613 put_cpu_var(vmap_block_queue);
1618 static void free_vmap_block(struct vmap_block *vb)
1620 struct vmap_block *tmp;
1621 unsigned long vb_idx;
1623 vb_idx = addr_to_vb_idx(vb->va->va_start);
1624 spin_lock(&vmap_block_tree_lock);
1625 tmp = radix_tree_delete(&vmap_block_tree, vb_idx);
1626 spin_unlock(&vmap_block_tree_lock);
1629 free_vmap_area_noflush(vb->va);
1630 kfree_rcu(vb, rcu_head);
1633 static void purge_fragmented_blocks(int cpu)
1636 struct vmap_block *vb;
1637 struct vmap_block *n_vb;
1638 struct vmap_block_queue *vbq = &per_cpu(vmap_block_queue, cpu);
1641 list_for_each_entry_rcu(vb, &vbq->free, free_list) {
1643 if (!(vb->free + vb->dirty == VMAP_BBMAP_BITS && vb->dirty != VMAP_BBMAP_BITS))
1646 spin_lock(&vb->lock);
1647 if (vb->free + vb->dirty == VMAP_BBMAP_BITS && vb->dirty != VMAP_BBMAP_BITS) {
1648 vb->free = 0; /* prevent further allocs after releasing lock */
1649 vb->dirty = VMAP_BBMAP_BITS; /* prevent purging it again */
1651 vb->dirty_max = VMAP_BBMAP_BITS;
1652 spin_lock(&vbq->lock);
1653 list_del_rcu(&vb->free_list);
1654 spin_unlock(&vbq->lock);
1655 spin_unlock(&vb->lock);
1656 list_add_tail(&vb->purge, &purge);
1658 spin_unlock(&vb->lock);
1662 list_for_each_entry_safe(vb, n_vb, &purge, purge) {
1663 list_del(&vb->purge);
1664 free_vmap_block(vb);
1668 static void purge_fragmented_blocks_allcpus(void)
1672 for_each_possible_cpu(cpu)
1673 purge_fragmented_blocks(cpu);
1676 static void *vb_alloc(unsigned long size, gfp_t gfp_mask)
1678 struct vmap_block_queue *vbq;
1679 struct vmap_block *vb;
1683 BUG_ON(offset_in_page(size));
1684 BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC);
1685 if (WARN_ON(size == 0)) {
1687 * Allocating 0 bytes isn't what caller wants since
1688 * get_order(0) returns funny result. Just warn and terminate
1693 order = get_order(size);
1696 vbq = &get_cpu_var(vmap_block_queue);
1697 list_for_each_entry_rcu(vb, &vbq->free, free_list) {
1698 unsigned long pages_off;
1700 spin_lock(&vb->lock);
1701 if (vb->free < (1UL << order)) {
1702 spin_unlock(&vb->lock);
1706 pages_off = VMAP_BBMAP_BITS - vb->free;
1707 vaddr = vmap_block_vaddr(vb->va->va_start, pages_off);
1708 vb->free -= 1UL << order;
1709 if (vb->free == 0) {
1710 spin_lock(&vbq->lock);
1711 list_del_rcu(&vb->free_list);
1712 spin_unlock(&vbq->lock);
1715 spin_unlock(&vb->lock);
1719 put_cpu_var(vmap_block_queue);
1722 /* Allocate new block if nothing was found */
1724 vaddr = new_vmap_block(order, gfp_mask);
1729 static void vb_free(unsigned long addr, unsigned long size)
1731 unsigned long offset;
1732 unsigned long vb_idx;
1734 struct vmap_block *vb;
1736 BUG_ON(offset_in_page(size));
1737 BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC);
1739 flush_cache_vunmap(addr, addr + size);
1741 order = get_order(size);
1743 offset = (addr & (VMAP_BLOCK_SIZE - 1)) >> PAGE_SHIFT;
1745 vb_idx = addr_to_vb_idx(addr);
1747 vb = radix_tree_lookup(&vmap_block_tree, vb_idx);
1751 unmap_kernel_range_noflush(addr, size);
1753 if (debug_pagealloc_enabled_static())
1754 flush_tlb_kernel_range(addr, addr + size);
1756 spin_lock(&vb->lock);
1758 /* Expand dirty range */
1759 vb->dirty_min = min(vb->dirty_min, offset);
1760 vb->dirty_max = max(vb->dirty_max, offset + (1UL << order));
1762 vb->dirty += 1UL << order;
1763 if (vb->dirty == VMAP_BBMAP_BITS) {
1765 spin_unlock(&vb->lock);
1766 free_vmap_block(vb);
1768 spin_unlock(&vb->lock);
1771 static void _vm_unmap_aliases(unsigned long start, unsigned long end, int flush)
1775 if (unlikely(!vmap_initialized))
1780 for_each_possible_cpu(cpu) {
1781 struct vmap_block_queue *vbq = &per_cpu(vmap_block_queue, cpu);
1782 struct vmap_block *vb;
1785 list_for_each_entry_rcu(vb, &vbq->free, free_list) {
1786 spin_lock(&vb->lock);
1788 unsigned long va_start = vb->va->va_start;
1791 s = va_start + (vb->dirty_min << PAGE_SHIFT);
1792 e = va_start + (vb->dirty_max << PAGE_SHIFT);
1794 start = min(s, start);
1799 spin_unlock(&vb->lock);
1804 mutex_lock(&vmap_purge_lock);
1805 purge_fragmented_blocks_allcpus();
1806 if (!__purge_vmap_area_lazy(start, end) && flush)
1807 flush_tlb_kernel_range(start, end);
1808 mutex_unlock(&vmap_purge_lock);
1812 * vm_unmap_aliases - unmap outstanding lazy aliases in the vmap layer
1814 * The vmap/vmalloc layer lazily flushes kernel virtual mappings primarily
1815 * to amortize TLB flushing overheads. What this means is that any page you
1816 * have now, may, in a former life, have been mapped into kernel virtual
1817 * address by the vmap layer and so there might be some CPUs with TLB entries
1818 * still referencing that page (additional to the regular 1:1 kernel mapping).
1820 * vm_unmap_aliases flushes all such lazy mappings. After it returns, we can
1821 * be sure that none of the pages we have control over will have any aliases
1822 * from the vmap layer.
1824 void vm_unmap_aliases(void)
1826 unsigned long start = ULONG_MAX, end = 0;
1829 _vm_unmap_aliases(start, end, flush);
1831 EXPORT_SYMBOL_GPL(vm_unmap_aliases);
1834 * vm_unmap_ram - unmap linear kernel address space set up by vm_map_ram
1835 * @mem: the pointer returned by vm_map_ram
1836 * @count: the count passed to that vm_map_ram call (cannot unmap partial)
1838 void vm_unmap_ram(const void *mem, unsigned int count)
1840 unsigned long size = (unsigned long)count << PAGE_SHIFT;
1841 unsigned long addr = (unsigned long)mem;
1842 struct vmap_area *va;
1846 BUG_ON(addr < VMALLOC_START);
1847 BUG_ON(addr > VMALLOC_END);
1848 BUG_ON(!PAGE_ALIGNED(addr));
1850 kasan_poison_vmalloc(mem, size);
1852 if (likely(count <= VMAP_MAX_ALLOC)) {
1853 debug_check_no_locks_freed(mem, size);
1854 vb_free(addr, size);
1858 va = find_vmap_area(addr);
1860 debug_check_no_locks_freed((void *)va->va_start,
1861 (va->va_end - va->va_start));
1862 free_unmap_vmap_area(va);
1864 EXPORT_SYMBOL(vm_unmap_ram);
1867 * vm_map_ram - map pages linearly into kernel virtual address (vmalloc space)
1868 * @pages: an array of pointers to the pages to be mapped
1869 * @count: number of pages
1870 * @node: prefer to allocate data structures on this node
1871 * @prot: memory protection to use. PAGE_KERNEL for regular RAM
1873 * If you use this function for less than VMAP_MAX_ALLOC pages, it could be
1874 * faster than vmap so it's good. But if you mix long-life and short-life
1875 * objects with vm_map_ram(), it could consume lots of address space through
1876 * fragmentation (especially on a 32bit machine). You could see failures in
1877 * the end. Please use this function for short-lived objects.
1879 * Returns: a pointer to the address that has been mapped, or %NULL on failure
1881 void *vm_map_ram(struct page **pages, unsigned int count, int node)
1883 unsigned long size = (unsigned long)count << PAGE_SHIFT;
1887 if (likely(count <= VMAP_MAX_ALLOC)) {
1888 mem = vb_alloc(size, GFP_KERNEL);
1891 addr = (unsigned long)mem;
1893 struct vmap_area *va;
1894 va = alloc_vmap_area(size, PAGE_SIZE,
1895 VMALLOC_START, VMALLOC_END, node, GFP_KERNEL);
1899 addr = va->va_start;
1903 kasan_unpoison_vmalloc(mem, size);
1905 if (map_kernel_range(addr, size, PAGE_KERNEL, pages) < 0) {
1906 vm_unmap_ram(mem, count);
1911 EXPORT_SYMBOL(vm_map_ram);
1913 static struct vm_struct *vmlist __initdata;
1916 * vm_area_add_early - add vmap area early during boot
1917 * @vm: vm_struct to add
1919 * This function is used to add fixed kernel vm area to vmlist before
1920 * vmalloc_init() is called. @vm->addr, @vm->size, and @vm->flags
1921 * should contain proper values and the other fields should be zero.
1923 * DO NOT USE THIS FUNCTION UNLESS YOU KNOW WHAT YOU'RE DOING.
1925 void __init vm_area_add_early(struct vm_struct *vm)
1927 struct vm_struct *tmp, **p;
1929 BUG_ON(vmap_initialized);
1930 for (p = &vmlist; (tmp = *p) != NULL; p = &tmp->next) {
1931 if (tmp->addr >= vm->addr) {
1932 BUG_ON(tmp->addr < vm->addr + vm->size);
1935 BUG_ON(tmp->addr + tmp->size > vm->addr);
1942 * vm_area_register_early - register vmap area early during boot
1943 * @vm: vm_struct to register
1944 * @align: requested alignment
1946 * This function is used to register kernel vm area before
1947 * vmalloc_init() is called. @vm->size and @vm->flags should contain
1948 * proper values on entry and other fields should be zero. On return,
1949 * vm->addr contains the allocated address.
1951 * DO NOT USE THIS FUNCTION UNLESS YOU KNOW WHAT YOU'RE DOING.
1953 void __init vm_area_register_early(struct vm_struct *vm, size_t align)
1955 static size_t vm_init_off __initdata;
1958 addr = ALIGN(VMALLOC_START + vm_init_off, align);
1959 vm_init_off = PFN_ALIGN(addr + vm->size) - VMALLOC_START;
1961 vm->addr = (void *)addr;
1963 vm_area_add_early(vm);
1966 static void vmap_init_free_space(void)
1968 unsigned long vmap_start = 1;
1969 const unsigned long vmap_end = ULONG_MAX;
1970 struct vmap_area *busy, *free;
1974 * -|-----|.....|-----|-----|-----|.....|-
1976 * |<--------------------------------->|
1978 list_for_each_entry(busy, &vmap_area_list, list) {
1979 if (busy->va_start - vmap_start > 0) {
1980 free = kmem_cache_zalloc(vmap_area_cachep, GFP_NOWAIT);
1981 if (!WARN_ON_ONCE(!free)) {
1982 free->va_start = vmap_start;
1983 free->va_end = busy->va_start;
1985 insert_vmap_area_augment(free, NULL,
1986 &free_vmap_area_root,
1987 &free_vmap_area_list);
1991 vmap_start = busy->va_end;
1994 if (vmap_end - vmap_start > 0) {
1995 free = kmem_cache_zalloc(vmap_area_cachep, GFP_NOWAIT);
1996 if (!WARN_ON_ONCE(!free)) {
1997 free->va_start = vmap_start;
1998 free->va_end = vmap_end;
2000 insert_vmap_area_augment(free, NULL,
2001 &free_vmap_area_root,
2002 &free_vmap_area_list);
2007 void __init vmalloc_init(void)
2009 struct vmap_area *va;
2010 struct vm_struct *tmp;
2014 * Create the cache for vmap_area objects.
2016 vmap_area_cachep = KMEM_CACHE(vmap_area, SLAB_PANIC);
2018 for_each_possible_cpu(i) {
2019 struct vmap_block_queue *vbq;
2020 struct vfree_deferred *p;
2022 vbq = &per_cpu(vmap_block_queue, i);
2023 spin_lock_init(&vbq->lock);
2024 INIT_LIST_HEAD(&vbq->free);
2025 p = &per_cpu(vfree_deferred, i);
2026 init_llist_head(&p->list);
2027 INIT_WORK(&p->wq, free_work);
2030 /* Import existing vmlist entries. */
2031 for (tmp = vmlist; tmp; tmp = tmp->next) {
2032 va = kmem_cache_zalloc(vmap_area_cachep, GFP_NOWAIT);
2033 if (WARN_ON_ONCE(!va))
2036 va->va_start = (unsigned long)tmp->addr;
2037 va->va_end = va->va_start + tmp->size;
2039 insert_vmap_area(va, &vmap_area_root, &vmap_area_list);
2043 * Now we can initialize a free vmap space.
2045 vmap_init_free_space();
2046 vmap_initialized = true;
2050 * unmap_kernel_range - unmap kernel VM area and flush cache and TLB
2051 * @addr: start of the VM area to unmap
2052 * @size: size of the VM area to unmap
2054 * Similar to unmap_kernel_range_noflush() but flushes vcache before
2055 * the unmapping and tlb after.
2057 void unmap_kernel_range(unsigned long addr, unsigned long size)
2059 unsigned long end = addr + size;
2061 flush_cache_vunmap(addr, end);
2062 unmap_kernel_range_noflush(addr, size);
2063 flush_tlb_kernel_range(addr, end);
2066 static inline void setup_vmalloc_vm_locked(struct vm_struct *vm,
2067 struct vmap_area *va, unsigned long flags, const void *caller)
2070 vm->addr = (void *)va->va_start;
2071 vm->size = va->va_end - va->va_start;
2072 vm->caller = caller;
2076 static void setup_vmalloc_vm(struct vm_struct *vm, struct vmap_area *va,
2077 unsigned long flags, const void *caller)
2079 spin_lock(&vmap_area_lock);
2080 setup_vmalloc_vm_locked(vm, va, flags, caller);
2081 spin_unlock(&vmap_area_lock);
2084 static void clear_vm_uninitialized_flag(struct vm_struct *vm)
2087 * Before removing VM_UNINITIALIZED,
2088 * we should make sure that vm has proper values.
2089 * Pair with smp_rmb() in show_numa_info().
2092 vm->flags &= ~VM_UNINITIALIZED;
2095 static struct vm_struct *__get_vm_area_node(unsigned long size,
2096 unsigned long align, unsigned long flags, unsigned long start,
2097 unsigned long end, int node, gfp_t gfp_mask, const void *caller)
2099 struct vmap_area *va;
2100 struct vm_struct *area;
2101 unsigned long requested_size = size;
2103 BUG_ON(in_interrupt());
2104 size = PAGE_ALIGN(size);
2105 if (unlikely(!size))
2108 if (flags & VM_IOREMAP)
2109 align = 1ul << clamp_t(int, get_count_order_long(size),
2110 PAGE_SHIFT, IOREMAP_MAX_ORDER);
2112 area = kzalloc_node(sizeof(*area), gfp_mask & GFP_RECLAIM_MASK, node);
2113 if (unlikely(!area))
2116 if (!(flags & VM_NO_GUARD))
2119 va = alloc_vmap_area(size, align, start, end, node, gfp_mask);
2125 kasan_unpoison_vmalloc((void *)va->va_start, requested_size);
2127 setup_vmalloc_vm(area, va, flags, caller);
2132 struct vm_struct *__get_vm_area_caller(unsigned long size, unsigned long flags,
2133 unsigned long start, unsigned long end,
2136 return __get_vm_area_node(size, 1, flags, start, end, NUMA_NO_NODE,
2137 GFP_KERNEL, caller);
2141 * get_vm_area - reserve a contiguous kernel virtual area
2142 * @size: size of the area
2143 * @flags: %VM_IOREMAP for I/O mappings or VM_ALLOC
2145 * Search an area of @size in the kernel virtual mapping area,
2146 * and reserved it for out purposes. Returns the area descriptor
2147 * on success or %NULL on failure.
2149 * Return: the area descriptor on success or %NULL on failure.
2151 struct vm_struct *get_vm_area(unsigned long size, unsigned long flags)
2153 return __get_vm_area_node(size, 1, flags, VMALLOC_START, VMALLOC_END,
2154 NUMA_NO_NODE, GFP_KERNEL,
2155 __builtin_return_address(0));
2158 struct vm_struct *get_vm_area_caller(unsigned long size, unsigned long flags,
2161 return __get_vm_area_node(size, 1, flags, VMALLOC_START, VMALLOC_END,
2162 NUMA_NO_NODE, GFP_KERNEL, caller);
2166 * find_vm_area - find a continuous kernel virtual area
2167 * @addr: base address
2169 * Search for the kernel VM area starting at @addr, and return it.
2170 * It is up to the caller to do all required locking to keep the returned
2173 * Return: pointer to the found area or %NULL on faulure
2175 struct vm_struct *find_vm_area(const void *addr)
2177 struct vmap_area *va;
2179 va = find_vmap_area((unsigned long)addr);
2187 * remove_vm_area - find and remove a continuous kernel virtual area
2188 * @addr: base address
2190 * Search for the kernel VM area starting at @addr, and remove it.
2191 * This function returns the found VM area, but using it is NOT safe
2192 * on SMP machines, except for its size or flags.
2194 * Return: pointer to the found area or %NULL on faulure
2196 struct vm_struct *remove_vm_area(const void *addr)
2198 struct vmap_area *va;
2202 spin_lock(&vmap_area_lock);
2203 va = __find_vmap_area((unsigned long)addr);
2205 struct vm_struct *vm = va->vm;
2208 spin_unlock(&vmap_area_lock);
2210 kasan_free_shadow(vm);
2211 free_unmap_vmap_area(va);
2216 spin_unlock(&vmap_area_lock);
2220 static inline void set_area_direct_map(const struct vm_struct *area,
2221 int (*set_direct_map)(struct page *page))
2225 for (i = 0; i < area->nr_pages; i++)
2226 if (page_address(area->pages[i]))
2227 set_direct_map(area->pages[i]);
2230 /* Handle removing and resetting vm mappings related to the vm_struct. */
2231 static void vm_remove_mappings(struct vm_struct *area, int deallocate_pages)
2233 unsigned long start = ULONG_MAX, end = 0;
2234 int flush_reset = area->flags & VM_FLUSH_RESET_PERMS;
2238 remove_vm_area(area->addr);
2240 /* If this is not VM_FLUSH_RESET_PERMS memory, no need for the below. */
2245 * If not deallocating pages, just do the flush of the VM area and
2248 if (!deallocate_pages) {
2254 * If execution gets here, flush the vm mapping and reset the direct
2255 * map. Find the start and end range of the direct mappings to make sure
2256 * the vm_unmap_aliases() flush includes the direct map.
2258 for (i = 0; i < area->nr_pages; i++) {
2259 unsigned long addr = (unsigned long)page_address(area->pages[i]);
2261 start = min(addr, start);
2262 end = max(addr + PAGE_SIZE, end);
2268 * Set direct map to something invalid so that it won't be cached if
2269 * there are any accesses after the TLB flush, then flush the TLB and
2270 * reset the direct map permissions to the default.
2272 set_area_direct_map(area, set_direct_map_invalid_noflush);
2273 _vm_unmap_aliases(start, end, flush_dmap);
2274 set_area_direct_map(area, set_direct_map_default_noflush);
2277 static void __vunmap(const void *addr, int deallocate_pages)
2279 struct vm_struct *area;
2284 if (WARN(!PAGE_ALIGNED(addr), "Trying to vfree() bad address (%p)\n",
2288 area = find_vm_area(addr);
2289 if (unlikely(!area)) {
2290 WARN(1, KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n",
2295 debug_check_no_locks_freed(area->addr, get_vm_area_size(area));
2296 debug_check_no_obj_freed(area->addr, get_vm_area_size(area));
2298 kasan_poison_vmalloc(area->addr, area->size);
2300 vm_remove_mappings(area, deallocate_pages);
2302 if (deallocate_pages) {
2305 for (i = 0; i < area->nr_pages; i++) {
2306 struct page *page = area->pages[i];
2309 __free_pages(page, 0);
2311 atomic_long_sub(area->nr_pages, &nr_vmalloc_pages);
2313 kvfree(area->pages);
2320 static inline void __vfree_deferred(const void *addr)
2323 * Use raw_cpu_ptr() because this can be called from preemptible
2324 * context. Preemption is absolutely fine here, because the llist_add()
2325 * implementation is lockless, so it works even if we are adding to
2326 * nother cpu's list. schedule_work() should be fine with this too.
2328 struct vfree_deferred *p = raw_cpu_ptr(&vfree_deferred);
2330 if (llist_add((struct llist_node *)addr, &p->list))
2331 schedule_work(&p->wq);
2335 * vfree_atomic - release memory allocated by vmalloc()
2336 * @addr: memory base address
2338 * This one is just like vfree() but can be called in any atomic context
2341 void vfree_atomic(const void *addr)
2345 kmemleak_free(addr);
2349 __vfree_deferred(addr);
2352 static void __vfree(const void *addr)
2354 if (unlikely(in_interrupt()))
2355 __vfree_deferred(addr);
2361 * vfree - release memory allocated by vmalloc()
2362 * @addr: memory base address
2364 * Free the virtually continuous memory area starting at @addr, as
2365 * obtained from vmalloc(), vmalloc_32() or __vmalloc(). If @addr is
2366 * NULL, no operation is performed.
2368 * Must not be called in NMI context (strictly speaking, only if we don't
2369 * have CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG, but making the calling
2370 * conventions for vfree() arch-depenedent would be a really bad idea)
2372 * May sleep if called *not* from interrupt context.
2374 * NOTE: assumes that the object at @addr has a size >= sizeof(llist_node)
2376 void vfree(const void *addr)
2380 kmemleak_free(addr);
2382 might_sleep_if(!in_interrupt());
2389 EXPORT_SYMBOL(vfree);
2392 * vunmap - release virtual mapping obtained by vmap()
2393 * @addr: memory base address
2395 * Free the virtually contiguous memory area starting at @addr,
2396 * which was created from the page array passed to vmap().
2398 * Must not be called in interrupt context.
2400 void vunmap(const void *addr)
2402 BUG_ON(in_interrupt());
2407 EXPORT_SYMBOL(vunmap);
2410 * vmap - map an array of pages into virtually contiguous space
2411 * @pages: array of page pointers
2412 * @count: number of pages to map
2413 * @flags: vm_area->flags
2414 * @prot: page protection for the mapping
2416 * Maps @count pages from @pages into contiguous kernel virtual
2419 * Return: the address of the area or %NULL on failure
2421 void *vmap(struct page **pages, unsigned int count,
2422 unsigned long flags, pgprot_t prot)
2424 struct vm_struct *area;
2425 unsigned long size; /* In bytes */
2429 if (count > totalram_pages())
2432 size = (unsigned long)count << PAGE_SHIFT;
2433 area = get_vm_area_caller(size, flags, __builtin_return_address(0));
2437 if (map_kernel_range((unsigned long)area->addr, size, pgprot_nx(prot),
2445 EXPORT_SYMBOL(vmap);
2447 static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
2448 pgprot_t prot, int node)
2450 struct page **pages;
2451 unsigned int nr_pages, array_size, i;
2452 const gfp_t nested_gfp = (gfp_mask & GFP_RECLAIM_MASK) | __GFP_ZERO;
2453 const gfp_t alloc_mask = gfp_mask | __GFP_NOWARN;
2454 const gfp_t highmem_mask = (gfp_mask & (GFP_DMA | GFP_DMA32)) ?
2458 nr_pages = get_vm_area_size(area) >> PAGE_SHIFT;
2459 array_size = (nr_pages * sizeof(struct page *));
2461 /* Please note that the recursion is strictly bounded. */
2462 if (array_size > PAGE_SIZE) {
2463 pages = __vmalloc_node(array_size, 1, nested_gfp|highmem_mask,
2464 node, area->caller);
2466 pages = kmalloc_node(array_size, nested_gfp, node);
2470 remove_vm_area(area->addr);
2475 area->pages = pages;
2476 area->nr_pages = nr_pages;
2478 for (i = 0; i < area->nr_pages; i++) {
2481 if (node == NUMA_NO_NODE)
2482 page = alloc_page(alloc_mask|highmem_mask);
2484 page = alloc_pages_node(node, alloc_mask|highmem_mask, 0);
2486 if (unlikely(!page)) {
2487 /* Successfully allocated i pages, free them in __vunmap() */
2489 atomic_long_add(area->nr_pages, &nr_vmalloc_pages);
2492 area->pages[i] = page;
2493 if (gfpflags_allow_blocking(gfp_mask))
2496 atomic_long_add(area->nr_pages, &nr_vmalloc_pages);
2498 if (map_kernel_range((unsigned long)area->addr, get_vm_area_size(area),
2505 warn_alloc(gfp_mask, NULL,
2506 "vmalloc: allocation failure, allocated %ld of %ld bytes",
2507 (area->nr_pages*PAGE_SIZE), area->size);
2508 __vfree(area->addr);
2513 * __vmalloc_node_range - allocate virtually contiguous memory
2514 * @size: allocation size
2515 * @align: desired alignment
2516 * @start: vm area range start
2517 * @end: vm area range end
2518 * @gfp_mask: flags for the page level allocator
2519 * @prot: protection mask for the allocated pages
2520 * @vm_flags: additional vm area flags (e.g. %VM_NO_GUARD)
2521 * @node: node to use for allocation or NUMA_NO_NODE
2522 * @caller: caller's return address
2524 * Allocate enough pages to cover @size from the page level
2525 * allocator with @gfp_mask flags. Map them into contiguous
2526 * kernel virtual space, using a pagetable protection of @prot.
2528 * Return: the address of the area or %NULL on failure
2530 void *__vmalloc_node_range(unsigned long size, unsigned long align,
2531 unsigned long start, unsigned long end, gfp_t gfp_mask,
2532 pgprot_t prot, unsigned long vm_flags, int node,
2535 struct vm_struct *area;
2537 unsigned long real_size = size;
2539 size = PAGE_ALIGN(size);
2540 if (!size || (size >> PAGE_SHIFT) > totalram_pages())
2543 area = __get_vm_area_node(real_size, align, VM_ALLOC | VM_UNINITIALIZED |
2544 vm_flags, start, end, node, gfp_mask, caller);
2548 addr = __vmalloc_area_node(area, gfp_mask, prot, node);
2553 * In this function, newly allocated vm_struct has VM_UNINITIALIZED
2554 * flag. It means that vm_struct is not fully initialized.
2555 * Now, it is fully initialized, so remove this flag here.
2557 clear_vm_uninitialized_flag(area);
2559 kmemleak_vmalloc(area, size, gfp_mask);
2564 warn_alloc(gfp_mask, NULL,
2565 "vmalloc: allocation failure: %lu bytes", real_size);
2570 * __vmalloc_node - allocate virtually contiguous memory
2571 * @size: allocation size
2572 * @align: desired alignment
2573 * @gfp_mask: flags for the page level allocator
2574 * @node: node to use for allocation or NUMA_NO_NODE
2575 * @caller: caller's return address
2577 * Allocate enough pages to cover @size from the page level allocator with
2578 * @gfp_mask flags. Map them into contiguous kernel virtual space.
2580 * Reclaim modifiers in @gfp_mask - __GFP_NORETRY, __GFP_RETRY_MAYFAIL
2581 * and __GFP_NOFAIL are not supported
2583 * Any use of gfp flags outside of GFP_KERNEL should be consulted
2586 * Return: pointer to the allocated memory or %NULL on error
2588 void *__vmalloc_node(unsigned long size, unsigned long align,
2589 gfp_t gfp_mask, int node, const void *caller)
2591 return __vmalloc_node_range(size, align, VMALLOC_START, VMALLOC_END,
2592 gfp_mask, PAGE_KERNEL, 0, node, caller);
2595 * This is only for performance analysis of vmalloc and stress purpose.
2596 * It is required by vmalloc test module, therefore do not use it other
2599 #ifdef CONFIG_TEST_VMALLOC_MODULE
2600 EXPORT_SYMBOL_GPL(__vmalloc_node);
2603 void *__vmalloc(unsigned long size, gfp_t gfp_mask)
2605 return __vmalloc_node(size, 1, gfp_mask, NUMA_NO_NODE,
2606 __builtin_return_address(0));
2608 EXPORT_SYMBOL(__vmalloc);
2611 * vmalloc - allocate virtually contiguous memory
2612 * @size: allocation size
2614 * Allocate enough pages to cover @size from the page level
2615 * allocator and map them into contiguous kernel virtual space.
2617 * For tight control over page level allocator and protection flags
2618 * use __vmalloc() instead.
2620 * Return: pointer to the allocated memory or %NULL on error
2622 void *vmalloc(unsigned long size)
2624 return __vmalloc_node(size, 1, GFP_KERNEL, NUMA_NO_NODE,
2625 __builtin_return_address(0));
2627 EXPORT_SYMBOL(vmalloc);
2630 * vzalloc - allocate virtually contiguous memory with zero fill
2631 * @size: allocation size
2633 * Allocate enough pages to cover @size from the page level
2634 * allocator and map them into contiguous kernel virtual space.
2635 * The memory allocated is set to zero.
2637 * For tight control over page level allocator and protection flags
2638 * use __vmalloc() instead.
2640 * Return: pointer to the allocated memory or %NULL on error
2642 void *vzalloc(unsigned long size)
2644 return __vmalloc_node(size, 1, GFP_KERNEL | __GFP_ZERO, NUMA_NO_NODE,
2645 __builtin_return_address(0));
2647 EXPORT_SYMBOL(vzalloc);
2650 * vmalloc_user - allocate zeroed virtually contiguous memory for userspace
2651 * @size: allocation size
2653 * The resulting memory area is zeroed so it can be mapped to userspace
2654 * without leaking data.
2656 * Return: pointer to the allocated memory or %NULL on error
2658 void *vmalloc_user(unsigned long size)
2660 return __vmalloc_node_range(size, SHMLBA, VMALLOC_START, VMALLOC_END,
2661 GFP_KERNEL | __GFP_ZERO, PAGE_KERNEL,
2662 VM_USERMAP, NUMA_NO_NODE,
2663 __builtin_return_address(0));
2665 EXPORT_SYMBOL(vmalloc_user);
2668 * vmalloc_node - allocate memory on a specific node
2669 * @size: allocation size
2672 * Allocate enough pages to cover @size from the page level
2673 * allocator and map them into contiguous kernel virtual space.
2675 * For tight control over page level allocator and protection flags
2676 * use __vmalloc() instead.
2678 * Return: pointer to the allocated memory or %NULL on error
2680 void *vmalloc_node(unsigned long size, int node)
2682 return __vmalloc_node(size, 1, GFP_KERNEL, node,
2683 __builtin_return_address(0));
2685 EXPORT_SYMBOL(vmalloc_node);
2688 * vzalloc_node - allocate memory on a specific node with zero fill
2689 * @size: allocation size
2692 * Allocate enough pages to cover @size from the page level
2693 * allocator and map them into contiguous kernel virtual space.
2694 * The memory allocated is set to zero.
2696 * Return: pointer to the allocated memory or %NULL on error
2698 void *vzalloc_node(unsigned long size, int node)
2700 return __vmalloc_node(size, 1, GFP_KERNEL | __GFP_ZERO, node,
2701 __builtin_return_address(0));
2703 EXPORT_SYMBOL(vzalloc_node);
2706 * vmalloc_exec - allocate virtually contiguous, executable memory
2707 * @size: allocation size
2709 * Kernel-internal function to allocate enough pages to cover @size
2710 * the page level allocator and map them into contiguous and
2711 * executable kernel virtual space.
2713 * For tight control over page level allocator and protection flags
2714 * use __vmalloc() instead.
2716 * Return: pointer to the allocated memory or %NULL on error
2718 void *vmalloc_exec(unsigned long size)
2720 return __vmalloc_node_range(size, 1, VMALLOC_START, VMALLOC_END,
2721 GFP_KERNEL, PAGE_KERNEL_EXEC, VM_FLUSH_RESET_PERMS,
2722 NUMA_NO_NODE, __builtin_return_address(0));
2725 #if defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA32)
2726 #define GFP_VMALLOC32 (GFP_DMA32 | GFP_KERNEL)
2727 #elif defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA)
2728 #define GFP_VMALLOC32 (GFP_DMA | GFP_KERNEL)
2731 * 64b systems should always have either DMA or DMA32 zones. For others
2732 * GFP_DMA32 should do the right thing and use the normal zone.
2734 #define GFP_VMALLOC32 GFP_DMA32 | GFP_KERNEL
2738 * vmalloc_32 - allocate virtually contiguous memory (32bit addressable)
2739 * @size: allocation size
2741 * Allocate enough 32bit PA addressable pages to cover @size from the
2742 * page level allocator and map them into contiguous kernel virtual space.
2744 * Return: pointer to the allocated memory or %NULL on error
2746 void *vmalloc_32(unsigned long size)
2748 return __vmalloc_node(size, 1, GFP_VMALLOC32, NUMA_NO_NODE,
2749 __builtin_return_address(0));
2751 EXPORT_SYMBOL(vmalloc_32);
2754 * vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory
2755 * @size: allocation size
2757 * The resulting memory area is 32bit addressable and zeroed so it can be
2758 * mapped to userspace without leaking data.
2760 * Return: pointer to the allocated memory or %NULL on error
2762 void *vmalloc_32_user(unsigned long size)
2764 return __vmalloc_node_range(size, SHMLBA, VMALLOC_START, VMALLOC_END,
2765 GFP_VMALLOC32 | __GFP_ZERO, PAGE_KERNEL,
2766 VM_USERMAP, NUMA_NO_NODE,
2767 __builtin_return_address(0));
2769 EXPORT_SYMBOL(vmalloc_32_user);
2772 * small helper routine , copy contents to buf from addr.
2773 * If the page is not present, fill zero.
2776 static int aligned_vread(char *buf, char *addr, unsigned long count)
2782 unsigned long offset, length;
2784 offset = offset_in_page(addr);
2785 length = PAGE_SIZE - offset;
2788 p = vmalloc_to_page(addr);
2790 * To do safe access to this _mapped_ area, we need
2791 * lock. But adding lock here means that we need to add
2792 * overhead of vmalloc()/vfree() calles for this _debug_
2793 * interface, rarely used. Instead of that, we'll use
2794 * kmap() and get small overhead in this access function.
2798 * we can expect USER0 is not used (see vread/vwrite's
2799 * function description)
2801 void *map = kmap_atomic(p);
2802 memcpy(buf, map + offset, length);
2805 memset(buf, 0, length);
2815 static int aligned_vwrite(char *buf, char *addr, unsigned long count)
2821 unsigned long offset, length;
2823 offset = offset_in_page(addr);
2824 length = PAGE_SIZE - offset;
2827 p = vmalloc_to_page(addr);
2829 * To do safe access to this _mapped_ area, we need
2830 * lock. But adding lock here means that we need to add
2831 * overhead of vmalloc()/vfree() calles for this _debug_
2832 * interface, rarely used. Instead of that, we'll use
2833 * kmap() and get small overhead in this access function.
2837 * we can expect USER0 is not used (see vread/vwrite's
2838 * function description)
2840 void *map = kmap_atomic(p);
2841 memcpy(map + offset, buf, length);
2853 * vread() - read vmalloc area in a safe way.
2854 * @buf: buffer for reading data
2855 * @addr: vm address.
2856 * @count: number of bytes to be read.
2858 * This function checks that addr is a valid vmalloc'ed area, and
2859 * copy data from that area to a given buffer. If the given memory range
2860 * of [addr...addr+count) includes some valid address, data is copied to
2861 * proper area of @buf. If there are memory holes, they'll be zero-filled.
2862 * IOREMAP area is treated as memory hole and no copy is done.
2864 * If [addr...addr+count) doesn't includes any intersects with alive
2865 * vm_struct area, returns 0. @buf should be kernel's buffer.
2867 * Note: In usual ops, vread() is never necessary because the caller
2868 * should know vmalloc() area is valid and can use memcpy().
2869 * This is for routines which have to access vmalloc area without
2870 * any information, as /dev/kmem.
2872 * Return: number of bytes for which addr and buf should be increased
2873 * (same number as @count) or %0 if [addr...addr+count) doesn't
2874 * include any intersection with valid vmalloc area
2876 long vread(char *buf, char *addr, unsigned long count)
2878 struct vmap_area *va;
2879 struct vm_struct *vm;
2880 char *vaddr, *buf_start = buf;
2881 unsigned long buflen = count;
2884 /* Don't allow overflow */
2885 if ((unsigned long) addr + count < count)
2886 count = -(unsigned long) addr;
2888 spin_lock(&vmap_area_lock);
2889 list_for_each_entry(va, &vmap_area_list, list) {
2897 vaddr = (char *) vm->addr;
2898 if (addr >= vaddr + get_vm_area_size(vm))
2900 while (addr < vaddr) {
2908 n = vaddr + get_vm_area_size(vm) - addr;
2911 if (!(vm->flags & VM_IOREMAP))
2912 aligned_vread(buf, addr, n);
2913 else /* IOREMAP area is treated as memory hole */
2920 spin_unlock(&vmap_area_lock);
2922 if (buf == buf_start)
2924 /* zero-fill memory holes */
2925 if (buf != buf_start + buflen)
2926 memset(buf, 0, buflen - (buf - buf_start));
2932 * vwrite() - write vmalloc area in a safe way.
2933 * @buf: buffer for source data
2934 * @addr: vm address.
2935 * @count: number of bytes to be read.
2937 * This function checks that addr is a valid vmalloc'ed area, and
2938 * copy data from a buffer to the given addr. If specified range of
2939 * [addr...addr+count) includes some valid address, data is copied from
2940 * proper area of @buf. If there are memory holes, no copy to hole.
2941 * IOREMAP area is treated as memory hole and no copy is done.
2943 * If [addr...addr+count) doesn't includes any intersects with alive
2944 * vm_struct area, returns 0. @buf should be kernel's buffer.
2946 * Note: In usual ops, vwrite() is never necessary because the caller
2947 * should know vmalloc() area is valid and can use memcpy().
2948 * This is for routines which have to access vmalloc area without
2949 * any information, as /dev/kmem.
2951 * Return: number of bytes for which addr and buf should be
2952 * increased (same number as @count) or %0 if [addr...addr+count)
2953 * doesn't include any intersection with valid vmalloc area
2955 long vwrite(char *buf, char *addr, unsigned long count)
2957 struct vmap_area *va;
2958 struct vm_struct *vm;
2960 unsigned long n, buflen;
2963 /* Don't allow overflow */
2964 if ((unsigned long) addr + count < count)
2965 count = -(unsigned long) addr;
2968 spin_lock(&vmap_area_lock);
2969 list_for_each_entry(va, &vmap_area_list, list) {
2977 vaddr = (char *) vm->addr;
2978 if (addr >= vaddr + get_vm_area_size(vm))
2980 while (addr < vaddr) {
2987 n = vaddr + get_vm_area_size(vm) - addr;
2990 if (!(vm->flags & VM_IOREMAP)) {
2991 aligned_vwrite(buf, addr, n);
2999 spin_unlock(&vmap_area_lock);
3006 * remap_vmalloc_range_partial - map vmalloc pages to userspace
3007 * @vma: vma to cover
3008 * @uaddr: target user address to start at
3009 * @kaddr: virtual address of vmalloc kernel memory
3010 * @pgoff: offset from @kaddr to start at
3011 * @size: size of map area
3013 * Returns: 0 for success, -Exxx on failure
3015 * This function checks that @kaddr is a valid vmalloc'ed area,
3016 * and that it is big enough to cover the range starting at
3017 * @uaddr in @vma. Will return failure if that criteria isn't
3020 * Similar to remap_pfn_range() (see mm/memory.c)
3022 int remap_vmalloc_range_partial(struct vm_area_struct *vma, unsigned long uaddr,
3023 void *kaddr, unsigned long pgoff,
3026 struct vm_struct *area;
3028 unsigned long end_index;
3030 if (check_shl_overflow(pgoff, PAGE_SHIFT, &off))
3033 size = PAGE_ALIGN(size);
3035 if (!PAGE_ALIGNED(uaddr) || !PAGE_ALIGNED(kaddr))
3038 area = find_vm_area(kaddr);
3042 if (!(area->flags & (VM_USERMAP | VM_DMA_COHERENT)))
3045 if (check_add_overflow(size, off, &end_index) ||
3046 end_index > get_vm_area_size(area))
3051 struct page *page = vmalloc_to_page(kaddr);
3054 ret = vm_insert_page(vma, uaddr, page);
3063 vma->vm_flags |= VM_DONTEXPAND | VM_DONTDUMP;
3067 EXPORT_SYMBOL(remap_vmalloc_range_partial);
3070 * remap_vmalloc_range - map vmalloc pages to userspace
3071 * @vma: vma to cover (map full range of vma)
3072 * @addr: vmalloc memory
3073 * @pgoff: number of pages into addr before first page to map
3075 * Returns: 0 for success, -Exxx on failure
3077 * This function checks that addr is a valid vmalloc'ed area, and
3078 * that it is big enough to cover the vma. Will return failure if
3079 * that criteria isn't met.
3081 * Similar to remap_pfn_range() (see mm/memory.c)
3083 int remap_vmalloc_range(struct vm_area_struct *vma, void *addr,
3084 unsigned long pgoff)
3086 return remap_vmalloc_range_partial(vma, vma->vm_start,
3088 vma->vm_end - vma->vm_start);
3090 EXPORT_SYMBOL(remap_vmalloc_range);
3093 * Implement stubs for vmalloc_sync_[un]mappings () if the architecture chose
3096 * The purpose of this function is to make sure the vmalloc area
3097 * mappings are identical in all page-tables in the system.
3099 void __weak vmalloc_sync_mappings(void)
3103 void __weak vmalloc_sync_unmappings(void)
3107 static int f(pte_t *pte, unsigned long addr, void *data)
3119 * alloc_vm_area - allocate a range of kernel address space
3120 * @size: size of the area
3121 * @ptes: returns the PTEs for the address space
3123 * Returns: NULL on failure, vm_struct on success
3125 * This function reserves a range of kernel address space, and
3126 * allocates pagetables to map that range. No actual mappings
3129 * If @ptes is non-NULL, pointers to the PTEs (in init_mm)
3130 * allocated for the VM area are returned.
3132 struct vm_struct *alloc_vm_area(size_t size, pte_t **ptes)
3134 struct vm_struct *area;
3136 area = get_vm_area_caller(size, VM_IOREMAP,
3137 __builtin_return_address(0));
3142 * This ensures that page tables are constructed for this region
3143 * of kernel virtual address space and mapped into init_mm.
3145 if (apply_to_page_range(&init_mm, (unsigned long)area->addr,
3146 size, f, ptes ? &ptes : NULL)) {
3153 EXPORT_SYMBOL_GPL(alloc_vm_area);
3155 void free_vm_area(struct vm_struct *area)
3157 struct vm_struct *ret;
3158 ret = remove_vm_area(area->addr);
3159 BUG_ON(ret != area);
3162 EXPORT_SYMBOL_GPL(free_vm_area);
3165 static struct vmap_area *node_to_va(struct rb_node *n)
3167 return rb_entry_safe(n, struct vmap_area, rb_node);
3171 * pvm_find_va_enclose_addr - find the vmap_area @addr belongs to
3172 * @addr: target address
3174 * Returns: vmap_area if it is found. If there is no such area
3175 * the first highest(reverse order) vmap_area is returned
3176 * i.e. va->va_start < addr && va->va_end < addr or NULL
3177 * if there are no any areas before @addr.
3179 static struct vmap_area *
3180 pvm_find_va_enclose_addr(unsigned long addr)
3182 struct vmap_area *va, *tmp;
3185 n = free_vmap_area_root.rb_node;
3189 tmp = rb_entry(n, struct vmap_area, rb_node);
3190 if (tmp->va_start <= addr) {
3192 if (tmp->va_end >= addr)
3205 * pvm_determine_end_from_reverse - find the highest aligned address
3206 * of free block below VMALLOC_END
3208 * in - the VA we start the search(reverse order);
3209 * out - the VA with the highest aligned end address.
3211 * Returns: determined end address within vmap_area
3213 static unsigned long
3214 pvm_determine_end_from_reverse(struct vmap_area **va, unsigned long align)
3216 unsigned long vmalloc_end = VMALLOC_END & ~(align - 1);
3220 list_for_each_entry_from_reverse((*va),
3221 &free_vmap_area_list, list) {
3222 addr = min((*va)->va_end & ~(align - 1), vmalloc_end);
3223 if ((*va)->va_start < addr)
3232 * pcpu_get_vm_areas - allocate vmalloc areas for percpu allocator
3233 * @offsets: array containing offset of each area
3234 * @sizes: array containing size of each area
3235 * @nr_vms: the number of areas to allocate
3236 * @align: alignment, all entries in @offsets and @sizes must be aligned to this
3238 * Returns: kmalloc'd vm_struct pointer array pointing to allocated
3239 * vm_structs on success, %NULL on failure
3241 * Percpu allocator wants to use congruent vm areas so that it can
3242 * maintain the offsets among percpu areas. This function allocates
3243 * congruent vmalloc areas for it with GFP_KERNEL. These areas tend to
3244 * be scattered pretty far, distance between two areas easily going up
3245 * to gigabytes. To avoid interacting with regular vmallocs, these
3246 * areas are allocated from top.
3248 * Despite its complicated look, this allocator is rather simple. It
3249 * does everything top-down and scans free blocks from the end looking
3250 * for matching base. While scanning, if any of the areas do not fit the
3251 * base address is pulled down to fit the area. Scanning is repeated till
3252 * all the areas fit and then all necessary data structures are inserted
3253 * and the result is returned.
3255 struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets,
3256 const size_t *sizes, int nr_vms,
3259 const unsigned long vmalloc_start = ALIGN(VMALLOC_START, align);
3260 const unsigned long vmalloc_end = VMALLOC_END & ~(align - 1);
3261 struct vmap_area **vas, *va;
3262 struct vm_struct **vms;
3263 int area, area2, last_area, term_area;
3264 unsigned long base, start, size, end, last_end, orig_start, orig_end;
3265 bool purged = false;
3268 /* verify parameters and allocate data structures */
3269 BUG_ON(offset_in_page(align) || !is_power_of_2(align));
3270 for (last_area = 0, area = 0; area < nr_vms; area++) {
3271 start = offsets[area];
3272 end = start + sizes[area];
3274 /* is everything aligned properly? */
3275 BUG_ON(!IS_ALIGNED(offsets[area], align));
3276 BUG_ON(!IS_ALIGNED(sizes[area], align));
3278 /* detect the area with the highest address */
3279 if (start > offsets[last_area])
3282 for (area2 = area + 1; area2 < nr_vms; area2++) {
3283 unsigned long start2 = offsets[area2];
3284 unsigned long end2 = start2 + sizes[area2];
3286 BUG_ON(start2 < end && start < end2);
3289 last_end = offsets[last_area] + sizes[last_area];
3291 if (vmalloc_end - vmalloc_start < last_end) {
3296 vms = kcalloc(nr_vms, sizeof(vms[0]), GFP_KERNEL);
3297 vas = kcalloc(nr_vms, sizeof(vas[0]), GFP_KERNEL);
3301 for (area = 0; area < nr_vms; area++) {
3302 vas[area] = kmem_cache_zalloc(vmap_area_cachep, GFP_KERNEL);
3303 vms[area] = kzalloc(sizeof(struct vm_struct), GFP_KERNEL);
3304 if (!vas[area] || !vms[area])
3308 spin_lock(&free_vmap_area_lock);
3310 /* start scanning - we scan from the top, begin with the last area */
3311 area = term_area = last_area;
3312 start = offsets[area];
3313 end = start + sizes[area];
3315 va = pvm_find_va_enclose_addr(vmalloc_end);
3316 base = pvm_determine_end_from_reverse(&va, align) - end;
3320 * base might have underflowed, add last_end before
3323 if (base + last_end < vmalloc_start + last_end)
3327 * Fitting base has not been found.
3333 * If required width exceeds current VA block, move
3334 * base downwards and then recheck.
3336 if (base + end > va->va_end) {
3337 base = pvm_determine_end_from_reverse(&va, align) - end;
3343 * If this VA does not fit, move base downwards and recheck.
3345 if (base + start < va->va_start) {
3346 va = node_to_va(rb_prev(&va->rb_node));
3347 base = pvm_determine_end_from_reverse(&va, align) - end;
3353 * This area fits, move on to the previous one. If
3354 * the previous one is the terminal one, we're done.
3356 area = (area + nr_vms - 1) % nr_vms;
3357 if (area == term_area)
3360 start = offsets[area];
3361 end = start + sizes[area];
3362 va = pvm_find_va_enclose_addr(base + end);
3365 /* we've found a fitting base, insert all va's */
3366 for (area = 0; area < nr_vms; area++) {
3369 start = base + offsets[area];
3372 va = pvm_find_va_enclose_addr(start);
3373 if (WARN_ON_ONCE(va == NULL))
3374 /* It is a BUG(), but trigger recovery instead. */
3377 type = classify_va_fit_type(va, start, size);
3378 if (WARN_ON_ONCE(type == NOTHING_FIT))
3379 /* It is a BUG(), but trigger recovery instead. */
3382 ret = adjust_va_to_fit_type(va, start, size, type);
3386 /* Allocated area. */
3388 va->va_start = start;
3389 va->va_end = start + size;
3392 spin_unlock(&free_vmap_area_lock);
3394 /* populate the kasan shadow space */
3395 for (area = 0; area < nr_vms; area++) {
3396 if (kasan_populate_vmalloc(vas[area]->va_start, sizes[area]))
3397 goto err_free_shadow;
3399 kasan_unpoison_vmalloc((void *)vas[area]->va_start,
3403 /* insert all vm's */
3404 spin_lock(&vmap_area_lock);
3405 for (area = 0; area < nr_vms; area++) {
3406 insert_vmap_area(vas[area], &vmap_area_root, &vmap_area_list);
3408 setup_vmalloc_vm_locked(vms[area], vas[area], VM_ALLOC,
3411 spin_unlock(&vmap_area_lock);
3418 * Remove previously allocated areas. There is no
3419 * need in removing these areas from the busy tree,
3420 * because they are inserted only on the final step
3421 * and when pcpu_get_vm_areas() is success.
3424 orig_start = vas[area]->va_start;
3425 orig_end = vas[area]->va_end;
3426 va = merge_or_add_vmap_area(vas[area], &free_vmap_area_root,
3427 &free_vmap_area_list);
3428 kasan_release_vmalloc(orig_start, orig_end,
3429 va->va_start, va->va_end);
3434 spin_unlock(&free_vmap_area_lock);
3436 purge_vmap_area_lazy();
3439 /* Before "retry", check if we recover. */
3440 for (area = 0; area < nr_vms; area++) {
3444 vas[area] = kmem_cache_zalloc(
3445 vmap_area_cachep, GFP_KERNEL);
3454 for (area = 0; area < nr_vms; area++) {
3456 kmem_cache_free(vmap_area_cachep, vas[area]);
3466 spin_lock(&free_vmap_area_lock);
3468 * We release all the vmalloc shadows, even the ones for regions that
3469 * hadn't been successfully added. This relies on kasan_release_vmalloc
3470 * being able to tolerate this case.
3472 for (area = 0; area < nr_vms; area++) {
3473 orig_start = vas[area]->va_start;
3474 orig_end = vas[area]->va_end;
3475 va = merge_or_add_vmap_area(vas[area], &free_vmap_area_root,
3476 &free_vmap_area_list);
3477 kasan_release_vmalloc(orig_start, orig_end,
3478 va->va_start, va->va_end);
3482 spin_unlock(&free_vmap_area_lock);
3489 * pcpu_free_vm_areas - free vmalloc areas for percpu allocator
3490 * @vms: vm_struct pointer array returned by pcpu_get_vm_areas()
3491 * @nr_vms: the number of allocated areas
3493 * Free vm_structs and the array allocated by pcpu_get_vm_areas().
3495 void pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms)
3499 for (i = 0; i < nr_vms; i++)
3500 free_vm_area(vms[i]);
3503 #endif /* CONFIG_SMP */
3505 #ifdef CONFIG_PROC_FS
3506 static void *s_start(struct seq_file *m, loff_t *pos)
3507 __acquires(&vmap_purge_lock)
3508 __acquires(&vmap_area_lock)
3510 mutex_lock(&vmap_purge_lock);
3511 spin_lock(&vmap_area_lock);
3513 return seq_list_start(&vmap_area_list, *pos);
3516 static void *s_next(struct seq_file *m, void *p, loff_t *pos)
3518 return seq_list_next(p, &vmap_area_list, pos);
3521 static void s_stop(struct seq_file *m, void *p)
3522 __releases(&vmap_purge_lock)
3523 __releases(&vmap_area_lock)
3525 mutex_unlock(&vmap_purge_lock);
3526 spin_unlock(&vmap_area_lock);
3529 static void show_numa_info(struct seq_file *m, struct vm_struct *v)
3531 if (IS_ENABLED(CONFIG_NUMA)) {
3532 unsigned int nr, *counters = m->private;
3537 if (v->flags & VM_UNINITIALIZED)
3539 /* Pair with smp_wmb() in clear_vm_uninitialized_flag() */
3542 memset(counters, 0, nr_node_ids * sizeof(unsigned int));
3544 for (nr = 0; nr < v->nr_pages; nr++)
3545 counters[page_to_nid(v->pages[nr])]++;
3547 for_each_node_state(nr, N_HIGH_MEMORY)
3549 seq_printf(m, " N%u=%u", nr, counters[nr]);
3553 static void show_purge_info(struct seq_file *m)
3555 struct llist_node *head;
3556 struct vmap_area *va;
3558 head = READ_ONCE(vmap_purge_list.first);
3562 llist_for_each_entry(va, head, purge_list) {
3563 seq_printf(m, "0x%pK-0x%pK %7ld unpurged vm_area\n",
3564 (void *)va->va_start, (void *)va->va_end,
3565 va->va_end - va->va_start);
3569 static int s_show(struct seq_file *m, void *p)
3571 struct vmap_area *va;
3572 struct vm_struct *v;
3574 va = list_entry(p, struct vmap_area, list);
3577 * s_show can encounter race with remove_vm_area, !vm on behalf
3578 * of vmap area is being tear down or vm_map_ram allocation.
3581 seq_printf(m, "0x%pK-0x%pK %7ld vm_map_ram\n",
3582 (void *)va->va_start, (void *)va->va_end,
3583 va->va_end - va->va_start);
3590 seq_printf(m, "0x%pK-0x%pK %7ld",
3591 v->addr, v->addr + v->size, v->size);
3594 seq_printf(m, " %pS", v->caller);
3597 seq_printf(m, " pages=%d", v->nr_pages);
3600 seq_printf(m, " phys=%pa", &v->phys_addr);
3602 if (v->flags & VM_IOREMAP)
3603 seq_puts(m, " ioremap");
3605 if (v->flags & VM_ALLOC)
3606 seq_puts(m, " vmalloc");
3608 if (v->flags & VM_MAP)
3609 seq_puts(m, " vmap");
3611 if (v->flags & VM_USERMAP)
3612 seq_puts(m, " user");
3614 if (v->flags & VM_DMA_COHERENT)
3615 seq_puts(m, " dma-coherent");
3617 if (is_vmalloc_addr(v->pages))
3618 seq_puts(m, " vpages");
3620 show_numa_info(m, v);
3624 * As a final step, dump "unpurged" areas. Note,
3625 * that entire "/proc/vmallocinfo" output will not
3626 * be address sorted, because the purge list is not
3629 if (list_is_last(&va->list, &vmap_area_list))
3635 static const struct seq_operations vmalloc_op = {
3642 static int __init proc_vmalloc_init(void)
3644 if (IS_ENABLED(CONFIG_NUMA))
3645 proc_create_seq_private("vmallocinfo", 0400, NULL,
3647 nr_node_ids * sizeof(unsigned int), NULL);
3649 proc_create_seq("vmallocinfo", 0400, NULL, &vmalloc_op);
3652 module_init(proc_vmalloc_init);