1 // SPDX-License-Identifier: GPL-2.0-only
5 * Copyright (C) 1993 Linus Torvalds
6 * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
7 * SMP-safe vmalloc/vfree/ioremap, Tigran Aivazian <tigran@veritas.com>, May 2000
8 * Major rework to support vmap/vunmap, Christoph Hellwig, SGI, August 2002
9 * Numa awareness, Christoph Lameter, SGI, June 2005
12 #include <linux/vmalloc.h>
14 #include <linux/module.h>
15 #include <linux/highmem.h>
16 #include <linux/sched/signal.h>
17 #include <linux/slab.h>
18 #include <linux/spinlock.h>
19 #include <linux/interrupt.h>
20 #include <linux/proc_fs.h>
21 #include <linux/seq_file.h>
22 #include <linux/set_memory.h>
23 #include <linux/debugobjects.h>
24 #include <linux/kallsyms.h>
25 #include <linux/list.h>
26 #include <linux/notifier.h>
27 #include <linux/rbtree.h>
28 #include <linux/radix-tree.h>
29 #include <linux/rcupdate.h>
30 #include <linux/pfn.h>
31 #include <linux/kmemleak.h>
32 #include <linux/atomic.h>
33 #include <linux/compiler.h>
34 #include <linux/llist.h>
35 #include <linux/bitops.h>
36 #include <linux/rbtree_augmented.h>
37 #include <linux/overflow.h>
39 #include <linux/uaccess.h>
40 #include <asm/tlbflush.h>
41 #include <asm/shmparam.h>
45 bool is_vmalloc_addr(const void *x)
47 unsigned long addr = (unsigned long)x;
49 return addr >= VMALLOC_START && addr < VMALLOC_END;
51 EXPORT_SYMBOL(is_vmalloc_addr);
53 struct vfree_deferred {
54 struct llist_head list;
55 struct work_struct wq;
57 static DEFINE_PER_CPU(struct vfree_deferred, vfree_deferred);
59 static void __vunmap(const void *, int);
61 static void free_work(struct work_struct *w)
63 struct vfree_deferred *p = container_of(w, struct vfree_deferred, wq);
64 struct llist_node *t, *llnode;
66 llist_for_each_safe(llnode, t, llist_del_all(&p->list))
67 __vunmap((void *)llnode, 1);
70 /*** Page table manipulation functions ***/
72 static void vunmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end)
76 pte = pte_offset_kernel(pmd, addr);
78 pte_t ptent = ptep_get_and_clear(&init_mm, addr, pte);
79 WARN_ON(!pte_none(ptent) && !pte_present(ptent));
80 } while (pte++, addr += PAGE_SIZE, addr != end);
83 static void vunmap_pmd_range(pud_t *pud, unsigned long addr, unsigned long end)
88 pmd = pmd_offset(pud, addr);
90 next = pmd_addr_end(addr, end);
91 if (pmd_clear_huge(pmd))
93 if (pmd_none_or_clear_bad(pmd))
95 vunmap_pte_range(pmd, addr, next);
96 } while (pmd++, addr = next, addr != end);
99 static void vunmap_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end)
104 pud = pud_offset(p4d, addr);
106 next = pud_addr_end(addr, end);
107 if (pud_clear_huge(pud))
109 if (pud_none_or_clear_bad(pud))
111 vunmap_pmd_range(pud, addr, next);
112 } while (pud++, addr = next, addr != end);
115 static void vunmap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end)
120 p4d = p4d_offset(pgd, addr);
122 next = p4d_addr_end(addr, end);
123 if (p4d_clear_huge(p4d))
125 if (p4d_none_or_clear_bad(p4d))
127 vunmap_pud_range(p4d, addr, next);
128 } while (p4d++, addr = next, addr != end);
132 * unmap_kernel_range_noflush - unmap kernel VM area
133 * @addr: start of the VM area to unmap
134 * @size: size of the VM area to unmap
136 * Unmap PFN_UP(@size) pages at @addr. The VM area @addr and @size specify
137 * should have been allocated using get_vm_area() and its friends.
140 * This function does NOT do any cache flushing. The caller is responsible
141 * for calling flush_cache_vunmap() on to-be-mapped areas before calling this
142 * function and flush_tlb_kernel_range() after.
144 void unmap_kernel_range_noflush(unsigned long addr, unsigned long size)
146 unsigned long end = addr + size;
151 pgd = pgd_offset_k(addr);
153 next = pgd_addr_end(addr, end);
154 if (pgd_none_or_clear_bad(pgd))
156 vunmap_p4d_range(pgd, addr, next);
157 } while (pgd++, addr = next, addr != end);
160 static int vmap_pte_range(pmd_t *pmd, unsigned long addr,
161 unsigned long end, pgprot_t prot, struct page **pages, int *nr)
166 * nr is a running index into the array which helps higher level
167 * callers keep track of where we're up to.
170 pte = pte_alloc_kernel(pmd, addr);
174 struct page *page = pages[*nr];
176 if (WARN_ON(!pte_none(*pte)))
180 set_pte_at(&init_mm, addr, pte, mk_pte(page, prot));
182 } while (pte++, addr += PAGE_SIZE, addr != end);
186 static int vmap_pmd_range(pud_t *pud, unsigned long addr,
187 unsigned long end, pgprot_t prot, struct page **pages, int *nr)
192 pmd = pmd_alloc(&init_mm, pud, addr);
196 next = pmd_addr_end(addr, end);
197 if (vmap_pte_range(pmd, addr, next, prot, pages, nr))
199 } while (pmd++, addr = next, addr != end);
203 static int vmap_pud_range(p4d_t *p4d, unsigned long addr,
204 unsigned long end, pgprot_t prot, struct page **pages, int *nr)
209 pud = pud_alloc(&init_mm, p4d, addr);
213 next = pud_addr_end(addr, end);
214 if (vmap_pmd_range(pud, addr, next, prot, pages, nr))
216 } while (pud++, addr = next, addr != end);
220 static int vmap_p4d_range(pgd_t *pgd, unsigned long addr,
221 unsigned long end, pgprot_t prot, struct page **pages, int *nr)
226 p4d = p4d_alloc(&init_mm, pgd, addr);
230 next = p4d_addr_end(addr, end);
231 if (vmap_pud_range(p4d, addr, next, prot, pages, nr))
233 } while (p4d++, addr = next, addr != end);
238 * map_kernel_range_noflush - map kernel VM area with the specified pages
239 * @addr: start of the VM area to map
240 * @size: size of the VM area to map
241 * @prot: page protection flags to use
242 * @pages: pages to map
244 * Map PFN_UP(@size) pages at @addr. The VM area @addr and @size specify should
245 * have been allocated using get_vm_area() and its friends.
248 * This function does NOT do any cache flushing. The caller is responsible for
249 * calling flush_cache_vmap() on to-be-mapped areas before calling this
253 * 0 on success, -errno on failure.
255 int map_kernel_range_noflush(unsigned long addr, unsigned long size,
256 pgprot_t prot, struct page **pages)
258 unsigned long end = addr + size;
265 pgd = pgd_offset_k(addr);
267 next = pgd_addr_end(addr, end);
268 err = vmap_p4d_range(pgd, addr, next, prot, pages, &nr);
271 } while (pgd++, addr = next, addr != end);
276 int map_kernel_range(unsigned long start, unsigned long size, pgprot_t prot,
281 ret = map_kernel_range_noflush(start, size, prot, pages);
282 flush_cache_vmap(start, start + size);
286 int is_vmalloc_or_module_addr(const void *x)
289 * ARM, x86-64 and sparc64 put modules in a special place,
290 * and fall back on vmalloc() if that fails. Others
291 * just put it in the vmalloc space.
293 #if defined(CONFIG_MODULES) && defined(MODULES_VADDR)
294 unsigned long addr = (unsigned long)x;
295 if (addr >= MODULES_VADDR && addr < MODULES_END)
298 return is_vmalloc_addr(x);
302 * Walk a vmap address to the struct page it maps.
304 struct page *vmalloc_to_page(const void *vmalloc_addr)
306 unsigned long addr = (unsigned long) vmalloc_addr;
307 struct page *page = NULL;
308 pgd_t *pgd = pgd_offset_k(addr);
315 * XXX we might need to change this if we add VIRTUAL_BUG_ON for
316 * architectures that do not vmalloc module space
318 VIRTUAL_BUG_ON(!is_vmalloc_or_module_addr(vmalloc_addr));
322 p4d = p4d_offset(pgd, addr);
325 pud = pud_offset(p4d, addr);
328 * Don't dereference bad PUD or PMD (below) entries. This will also
329 * identify huge mappings, which we may encounter on architectures
330 * that define CONFIG_HAVE_ARCH_HUGE_VMAP=y. Such regions will be
331 * identified as vmalloc addresses by is_vmalloc_addr(), but are
332 * not [unambiguously] associated with a struct page, so there is
333 * no correct value to return for them.
335 WARN_ON_ONCE(pud_bad(*pud));
336 if (pud_none(*pud) || pud_bad(*pud))
338 pmd = pmd_offset(pud, addr);
339 WARN_ON_ONCE(pmd_bad(*pmd));
340 if (pmd_none(*pmd) || pmd_bad(*pmd))
343 ptep = pte_offset_map(pmd, addr);
345 if (pte_present(pte))
346 page = pte_page(pte);
350 EXPORT_SYMBOL(vmalloc_to_page);
353 * Map a vmalloc()-space virtual address to the physical page frame number.
355 unsigned long vmalloc_to_pfn(const void *vmalloc_addr)
357 return page_to_pfn(vmalloc_to_page(vmalloc_addr));
359 EXPORT_SYMBOL(vmalloc_to_pfn);
362 /*** Global kva allocator ***/
364 #define DEBUG_AUGMENT_PROPAGATE_CHECK 0
365 #define DEBUG_AUGMENT_LOWEST_MATCH_CHECK 0
368 static DEFINE_SPINLOCK(vmap_area_lock);
369 static DEFINE_SPINLOCK(free_vmap_area_lock);
370 /* Export for kexec only */
371 LIST_HEAD(vmap_area_list);
372 static LLIST_HEAD(vmap_purge_list);
373 static struct rb_root vmap_area_root = RB_ROOT;
374 static bool vmap_initialized __read_mostly;
377 * This kmem_cache is used for vmap_area objects. Instead of
378 * allocating from slab we reuse an object from this cache to
379 * make things faster. Especially in "no edge" splitting of
382 static struct kmem_cache *vmap_area_cachep;
385 * This linked list is used in pair with free_vmap_area_root.
386 * It gives O(1) access to prev/next to perform fast coalescing.
388 static LIST_HEAD(free_vmap_area_list);
391 * This augment red-black tree represents the free vmap space.
392 * All vmap_area objects in this tree are sorted by va->va_start
393 * address. It is used for allocation and merging when a vmap
394 * object is released.
396 * Each vmap_area node contains a maximum available free block
397 * of its sub-tree, right or left. Therefore it is possible to
398 * find a lowest match of free area.
400 static struct rb_root free_vmap_area_root = RB_ROOT;
403 * Preload a CPU with one object for "no edge" split case. The
404 * aim is to get rid of allocations from the atomic context, thus
405 * to use more permissive allocation masks.
407 static DEFINE_PER_CPU(struct vmap_area *, ne_fit_preload_node);
409 static __always_inline unsigned long
410 va_size(struct vmap_area *va)
412 return (va->va_end - va->va_start);
415 static __always_inline unsigned long
416 get_subtree_max_size(struct rb_node *node)
418 struct vmap_area *va;
420 va = rb_entry_safe(node, struct vmap_area, rb_node);
421 return va ? va->subtree_max_size : 0;
425 * Gets called when remove the node and rotate.
427 static __always_inline unsigned long
428 compute_subtree_max_size(struct vmap_area *va)
430 return max3(va_size(va),
431 get_subtree_max_size(va->rb_node.rb_left),
432 get_subtree_max_size(va->rb_node.rb_right));
435 RB_DECLARE_CALLBACKS_MAX(static, free_vmap_area_rb_augment_cb,
436 struct vmap_area, rb_node, unsigned long, subtree_max_size, va_size)
438 static void purge_vmap_area_lazy(void);
439 static BLOCKING_NOTIFIER_HEAD(vmap_notify_list);
440 static unsigned long lazy_max_pages(void);
442 static atomic_long_t nr_vmalloc_pages;
444 unsigned long vmalloc_nr_pages(void)
446 return atomic_long_read(&nr_vmalloc_pages);
449 static struct vmap_area *__find_vmap_area(unsigned long addr)
451 struct rb_node *n = vmap_area_root.rb_node;
454 struct vmap_area *va;
456 va = rb_entry(n, struct vmap_area, rb_node);
457 if (addr < va->va_start)
459 else if (addr >= va->va_end)
469 * This function returns back addresses of parent node
470 * and its left or right link for further processing.
472 static __always_inline struct rb_node **
473 find_va_links(struct vmap_area *va,
474 struct rb_root *root, struct rb_node *from,
475 struct rb_node **parent)
477 struct vmap_area *tmp_va;
478 struct rb_node **link;
481 link = &root->rb_node;
482 if (unlikely(!*link)) {
491 * Go to the bottom of the tree. When we hit the last point
492 * we end up with parent rb_node and correct direction, i name
493 * it link, where the new va->rb_node will be attached to.
496 tmp_va = rb_entry(*link, struct vmap_area, rb_node);
499 * During the traversal we also do some sanity check.
500 * Trigger the BUG() if there are sides(left/right)
503 if (va->va_start < tmp_va->va_end &&
504 va->va_end <= tmp_va->va_start)
505 link = &(*link)->rb_left;
506 else if (va->va_end > tmp_va->va_start &&
507 va->va_start >= tmp_va->va_end)
508 link = &(*link)->rb_right;
513 *parent = &tmp_va->rb_node;
517 static __always_inline struct list_head *
518 get_va_next_sibling(struct rb_node *parent, struct rb_node **link)
520 struct list_head *list;
522 if (unlikely(!parent))
524 * The red-black tree where we try to find VA neighbors
525 * before merging or inserting is empty, i.e. it means
526 * there is no free vmap space. Normally it does not
527 * happen but we handle this case anyway.
531 list = &rb_entry(parent, struct vmap_area, rb_node)->list;
532 return (&parent->rb_right == link ? list->next : list);
535 static __always_inline void
536 link_va(struct vmap_area *va, struct rb_root *root,
537 struct rb_node *parent, struct rb_node **link, struct list_head *head)
540 * VA is still not in the list, but we can
541 * identify its future previous list_head node.
543 if (likely(parent)) {
544 head = &rb_entry(parent, struct vmap_area, rb_node)->list;
545 if (&parent->rb_right != link)
549 /* Insert to the rb-tree */
550 rb_link_node(&va->rb_node, parent, link);
551 if (root == &free_vmap_area_root) {
553 * Some explanation here. Just perform simple insertion
554 * to the tree. We do not set va->subtree_max_size to
555 * its current size before calling rb_insert_augmented().
556 * It is because of we populate the tree from the bottom
557 * to parent levels when the node _is_ in the tree.
559 * Therefore we set subtree_max_size to zero after insertion,
560 * to let __augment_tree_propagate_from() puts everything to
561 * the correct order later on.
563 rb_insert_augmented(&va->rb_node,
564 root, &free_vmap_area_rb_augment_cb);
565 va->subtree_max_size = 0;
567 rb_insert_color(&va->rb_node, root);
570 /* Address-sort this list */
571 list_add(&va->list, head);
574 static __always_inline void
575 unlink_va(struct vmap_area *va, struct rb_root *root)
577 if (WARN_ON(RB_EMPTY_NODE(&va->rb_node)))
580 if (root == &free_vmap_area_root)
581 rb_erase_augmented(&va->rb_node,
582 root, &free_vmap_area_rb_augment_cb);
584 rb_erase(&va->rb_node, root);
587 RB_CLEAR_NODE(&va->rb_node);
590 #if DEBUG_AUGMENT_PROPAGATE_CHECK
592 augment_tree_propagate_check(struct rb_node *n)
594 struct vmap_area *va;
595 struct rb_node *node;
602 va = rb_entry(n, struct vmap_area, rb_node);
603 size = va->subtree_max_size;
607 va = rb_entry(node, struct vmap_area, rb_node);
609 if (get_subtree_max_size(node->rb_left) == size) {
610 node = node->rb_left;
612 if (va_size(va) == size) {
617 node = node->rb_right;
622 va = rb_entry(n, struct vmap_area, rb_node);
623 pr_emerg("tree is corrupted: %lu, %lu\n",
624 va_size(va), va->subtree_max_size);
627 augment_tree_propagate_check(n->rb_left);
628 augment_tree_propagate_check(n->rb_right);
633 * This function populates subtree_max_size from bottom to upper
634 * levels starting from VA point. The propagation must be done
635 * when VA size is modified by changing its va_start/va_end. Or
636 * in case of newly inserting of VA to the tree.
638 * It means that __augment_tree_propagate_from() must be called:
639 * - After VA has been inserted to the tree(free path);
640 * - After VA has been shrunk(allocation path);
641 * - After VA has been increased(merging path).
643 * Please note that, it does not mean that upper parent nodes
644 * and their subtree_max_size are recalculated all the time up
653 * For example if we modify the node 4, shrinking it to 2, then
654 * no any modification is required. If we shrink the node 2 to 1
655 * its subtree_max_size is updated only, and set to 1. If we shrink
656 * the node 8 to 6, then its subtree_max_size is set to 6 and parent
659 static __always_inline void
660 augment_tree_propagate_from(struct vmap_area *va)
662 struct rb_node *node = &va->rb_node;
663 unsigned long new_va_sub_max_size;
666 va = rb_entry(node, struct vmap_area, rb_node);
667 new_va_sub_max_size = compute_subtree_max_size(va);
670 * If the newly calculated maximum available size of the
671 * subtree is equal to the current one, then it means that
672 * the tree is propagated correctly. So we have to stop at
673 * this point to save cycles.
675 if (va->subtree_max_size == new_va_sub_max_size)
678 va->subtree_max_size = new_va_sub_max_size;
679 node = rb_parent(&va->rb_node);
682 #if DEBUG_AUGMENT_PROPAGATE_CHECK
683 augment_tree_propagate_check(free_vmap_area_root.rb_node);
688 insert_vmap_area(struct vmap_area *va,
689 struct rb_root *root, struct list_head *head)
691 struct rb_node **link;
692 struct rb_node *parent;
694 link = find_va_links(va, root, NULL, &parent);
695 link_va(va, root, parent, link, head);
699 insert_vmap_area_augment(struct vmap_area *va,
700 struct rb_node *from, struct rb_root *root,
701 struct list_head *head)
703 struct rb_node **link;
704 struct rb_node *parent;
707 link = find_va_links(va, NULL, from, &parent);
709 link = find_va_links(va, root, NULL, &parent);
711 link_va(va, root, parent, link, head);
712 augment_tree_propagate_from(va);
716 * Merge de-allocated chunk of VA memory with previous
717 * and next free blocks. If coalesce is not done a new
718 * free area is inserted. If VA has been merged, it is
721 static __always_inline struct vmap_area *
722 merge_or_add_vmap_area(struct vmap_area *va,
723 struct rb_root *root, struct list_head *head)
725 struct vmap_area *sibling;
726 struct list_head *next;
727 struct rb_node **link;
728 struct rb_node *parent;
732 * Find a place in the tree where VA potentially will be
733 * inserted, unless it is merged with its sibling/siblings.
735 link = find_va_links(va, root, NULL, &parent);
738 * Get next node of VA to check if merging can be done.
740 next = get_va_next_sibling(parent, link);
741 if (unlikely(next == NULL))
747 * |<------VA------>|<-----Next----->|
752 sibling = list_entry(next, struct vmap_area, list);
753 if (sibling->va_start == va->va_end) {
754 sibling->va_start = va->va_start;
756 /* Check and update the tree if needed. */
757 augment_tree_propagate_from(sibling);
759 /* Free vmap_area object. */
760 kmem_cache_free(vmap_area_cachep, va);
762 /* Point to the new merged area. */
771 * |<-----Prev----->|<------VA------>|
775 if (next->prev != head) {
776 sibling = list_entry(next->prev, struct vmap_area, list);
777 if (sibling->va_end == va->va_start) {
778 sibling->va_end = va->va_end;
780 /* Check and update the tree if needed. */
781 augment_tree_propagate_from(sibling);
786 /* Free vmap_area object. */
787 kmem_cache_free(vmap_area_cachep, va);
789 /* Point to the new merged area. */
797 link_va(va, root, parent, link, head);
798 augment_tree_propagate_from(va);
804 static __always_inline bool
805 is_within_this_va(struct vmap_area *va, unsigned long size,
806 unsigned long align, unsigned long vstart)
808 unsigned long nva_start_addr;
810 if (va->va_start > vstart)
811 nva_start_addr = ALIGN(va->va_start, align);
813 nva_start_addr = ALIGN(vstart, align);
815 /* Can be overflowed due to big size or alignment. */
816 if (nva_start_addr + size < nva_start_addr ||
817 nva_start_addr < vstart)
820 return (nva_start_addr + size <= va->va_end);
824 * Find the first free block(lowest start address) in the tree,
825 * that will accomplish the request corresponding to passing
828 static __always_inline struct vmap_area *
829 find_vmap_lowest_match(unsigned long size,
830 unsigned long align, unsigned long vstart)
832 struct vmap_area *va;
833 struct rb_node *node;
834 unsigned long length;
836 /* Start from the root. */
837 node = free_vmap_area_root.rb_node;
839 /* Adjust the search size for alignment overhead. */
840 length = size + align - 1;
843 va = rb_entry(node, struct vmap_area, rb_node);
845 if (get_subtree_max_size(node->rb_left) >= length &&
846 vstart < va->va_start) {
847 node = node->rb_left;
849 if (is_within_this_va(va, size, align, vstart))
853 * Does not make sense to go deeper towards the right
854 * sub-tree if it does not have a free block that is
855 * equal or bigger to the requested search length.
857 if (get_subtree_max_size(node->rb_right) >= length) {
858 node = node->rb_right;
863 * OK. We roll back and find the first right sub-tree,
864 * that will satisfy the search criteria. It can happen
865 * only once due to "vstart" restriction.
867 while ((node = rb_parent(node))) {
868 va = rb_entry(node, struct vmap_area, rb_node);
869 if (is_within_this_va(va, size, align, vstart))
872 if (get_subtree_max_size(node->rb_right) >= length &&
873 vstart <= va->va_start) {
874 node = node->rb_right;
884 #if DEBUG_AUGMENT_LOWEST_MATCH_CHECK
885 #include <linux/random.h>
887 static struct vmap_area *
888 find_vmap_lowest_linear_match(unsigned long size,
889 unsigned long align, unsigned long vstart)
891 struct vmap_area *va;
893 list_for_each_entry(va, &free_vmap_area_list, list) {
894 if (!is_within_this_va(va, size, align, vstart))
904 find_vmap_lowest_match_check(unsigned long size)
906 struct vmap_area *va_1, *va_2;
907 unsigned long vstart;
910 get_random_bytes(&rnd, sizeof(rnd));
911 vstart = VMALLOC_START + rnd;
913 va_1 = find_vmap_lowest_match(size, 1, vstart);
914 va_2 = find_vmap_lowest_linear_match(size, 1, vstart);
917 pr_emerg("not lowest: t: 0x%p, l: 0x%p, v: 0x%lx\n",
924 FL_FIT_TYPE = 1, /* full fit */
925 LE_FIT_TYPE = 2, /* left edge fit */
926 RE_FIT_TYPE = 3, /* right edge fit */
927 NE_FIT_TYPE = 4 /* no edge fit */
930 static __always_inline enum fit_type
931 classify_va_fit_type(struct vmap_area *va,
932 unsigned long nva_start_addr, unsigned long size)
936 /* Check if it is within VA. */
937 if (nva_start_addr < va->va_start ||
938 nva_start_addr + size > va->va_end)
942 if (va->va_start == nva_start_addr) {
943 if (va->va_end == nva_start_addr + size)
947 } else if (va->va_end == nva_start_addr + size) {
956 static __always_inline int
957 adjust_va_to_fit_type(struct vmap_area *va,
958 unsigned long nva_start_addr, unsigned long size,
961 struct vmap_area *lva = NULL;
963 if (type == FL_FIT_TYPE) {
965 * No need to split VA, it fully fits.
971 unlink_va(va, &free_vmap_area_root);
972 kmem_cache_free(vmap_area_cachep, va);
973 } else if (type == LE_FIT_TYPE) {
975 * Split left edge of fit VA.
981 va->va_start += size;
982 } else if (type == RE_FIT_TYPE) {
984 * Split right edge of fit VA.
990 va->va_end = nva_start_addr;
991 } else if (type == NE_FIT_TYPE) {
993 * Split no edge of fit VA.
999 lva = __this_cpu_xchg(ne_fit_preload_node, NULL);
1000 if (unlikely(!lva)) {
1002 * For percpu allocator we do not do any pre-allocation
1003 * and leave it as it is. The reason is it most likely
1004 * never ends up with NE_FIT_TYPE splitting. In case of
1005 * percpu allocations offsets and sizes are aligned to
1006 * fixed align request, i.e. RE_FIT_TYPE and FL_FIT_TYPE
1007 * are its main fitting cases.
1009 * There are a few exceptions though, as an example it is
1010 * a first allocation (early boot up) when we have "one"
1011 * big free space that has to be split.
1013 * Also we can hit this path in case of regular "vmap"
1014 * allocations, if "this" current CPU was not preloaded.
1015 * See the comment in alloc_vmap_area() why. If so, then
1016 * GFP_NOWAIT is used instead to get an extra object for
1017 * split purpose. That is rare and most time does not
1020 * What happens if an allocation gets failed. Basically,
1021 * an "overflow" path is triggered to purge lazily freed
1022 * areas to free some memory, then, the "retry" path is
1023 * triggered to repeat one more time. See more details
1024 * in alloc_vmap_area() function.
1026 lva = kmem_cache_alloc(vmap_area_cachep, GFP_NOWAIT);
1032 * Build the remainder.
1034 lva->va_start = va->va_start;
1035 lva->va_end = nva_start_addr;
1038 * Shrink this VA to remaining size.
1040 va->va_start = nva_start_addr + size;
1045 if (type != FL_FIT_TYPE) {
1046 augment_tree_propagate_from(va);
1048 if (lva) /* type == NE_FIT_TYPE */
1049 insert_vmap_area_augment(lva, &va->rb_node,
1050 &free_vmap_area_root, &free_vmap_area_list);
1057 * Returns a start address of the newly allocated area, if success.
1058 * Otherwise a vend is returned that indicates failure.
1060 static __always_inline unsigned long
1061 __alloc_vmap_area(unsigned long size, unsigned long align,
1062 unsigned long vstart, unsigned long vend)
1064 unsigned long nva_start_addr;
1065 struct vmap_area *va;
1069 va = find_vmap_lowest_match(size, align, vstart);
1073 if (va->va_start > vstart)
1074 nva_start_addr = ALIGN(va->va_start, align);
1076 nva_start_addr = ALIGN(vstart, align);
1078 /* Check the "vend" restriction. */
1079 if (nva_start_addr + size > vend)
1082 /* Classify what we have found. */
1083 type = classify_va_fit_type(va, nva_start_addr, size);
1084 if (WARN_ON_ONCE(type == NOTHING_FIT))
1087 /* Update the free vmap_area. */
1088 ret = adjust_va_to_fit_type(va, nva_start_addr, size, type);
1092 #if DEBUG_AUGMENT_LOWEST_MATCH_CHECK
1093 find_vmap_lowest_match_check(size);
1096 return nva_start_addr;
1100 * Free a region of KVA allocated by alloc_vmap_area
1102 static void free_vmap_area(struct vmap_area *va)
1105 * Remove from the busy tree/list.
1107 spin_lock(&vmap_area_lock);
1108 unlink_va(va, &vmap_area_root);
1109 spin_unlock(&vmap_area_lock);
1112 * Insert/Merge it back to the free tree/list.
1114 spin_lock(&free_vmap_area_lock);
1115 merge_or_add_vmap_area(va, &free_vmap_area_root, &free_vmap_area_list);
1116 spin_unlock(&free_vmap_area_lock);
1120 * Allocate a region of KVA of the specified size and alignment, within the
1123 static struct vmap_area *alloc_vmap_area(unsigned long size,
1124 unsigned long align,
1125 unsigned long vstart, unsigned long vend,
1126 int node, gfp_t gfp_mask)
1128 struct vmap_area *va, *pva;
1134 BUG_ON(offset_in_page(size));
1135 BUG_ON(!is_power_of_2(align));
1137 if (unlikely(!vmap_initialized))
1138 return ERR_PTR(-EBUSY);
1141 gfp_mask = gfp_mask & GFP_RECLAIM_MASK;
1143 va = kmem_cache_alloc_node(vmap_area_cachep, gfp_mask, node);
1145 return ERR_PTR(-ENOMEM);
1148 * Only scan the relevant parts containing pointers to other objects
1149 * to avoid false negatives.
1151 kmemleak_scan_area(&va->rb_node, SIZE_MAX, gfp_mask);
1155 * Preload this CPU with one extra vmap_area object. It is used
1156 * when fit type of free area is NE_FIT_TYPE. Please note, it
1157 * does not guarantee that an allocation occurs on a CPU that
1158 * is preloaded, instead we minimize the case when it is not.
1159 * It can happen because of cpu migration, because there is a
1160 * race until the below spinlock is taken.
1162 * The preload is done in non-atomic context, thus it allows us
1163 * to use more permissive allocation masks to be more stable under
1164 * low memory condition and high memory pressure. In rare case,
1165 * if not preloaded, GFP_NOWAIT is used.
1167 * Set "pva" to NULL here, because of "retry" path.
1171 if (!this_cpu_read(ne_fit_preload_node))
1173 * Even if it fails we do not really care about that.
1174 * Just proceed as it is. If needed "overflow" path
1175 * will refill the cache we allocate from.
1177 pva = kmem_cache_alloc_node(vmap_area_cachep, gfp_mask, node);
1179 spin_lock(&free_vmap_area_lock);
1181 if (pva && __this_cpu_cmpxchg(ne_fit_preload_node, NULL, pva))
1182 kmem_cache_free(vmap_area_cachep, pva);
1185 * If an allocation fails, the "vend" address is
1186 * returned. Therefore trigger the overflow path.
1188 addr = __alloc_vmap_area(size, align, vstart, vend);
1189 spin_unlock(&free_vmap_area_lock);
1191 if (unlikely(addr == vend))
1194 va->va_start = addr;
1195 va->va_end = addr + size;
1199 spin_lock(&vmap_area_lock);
1200 insert_vmap_area(va, &vmap_area_root, &vmap_area_list);
1201 spin_unlock(&vmap_area_lock);
1203 BUG_ON(!IS_ALIGNED(va->va_start, align));
1204 BUG_ON(va->va_start < vstart);
1205 BUG_ON(va->va_end > vend);
1207 ret = kasan_populate_vmalloc(addr, size);
1210 return ERR_PTR(ret);
1217 purge_vmap_area_lazy();
1222 if (gfpflags_allow_blocking(gfp_mask)) {
1223 unsigned long freed = 0;
1224 blocking_notifier_call_chain(&vmap_notify_list, 0, &freed);
1231 if (!(gfp_mask & __GFP_NOWARN) && printk_ratelimit())
1232 pr_warn("vmap allocation for size %lu failed: use vmalloc=<size> to increase size\n",
1235 kmem_cache_free(vmap_area_cachep, va);
1236 return ERR_PTR(-EBUSY);
1239 int register_vmap_purge_notifier(struct notifier_block *nb)
1241 return blocking_notifier_chain_register(&vmap_notify_list, nb);
1243 EXPORT_SYMBOL_GPL(register_vmap_purge_notifier);
1245 int unregister_vmap_purge_notifier(struct notifier_block *nb)
1247 return blocking_notifier_chain_unregister(&vmap_notify_list, nb);
1249 EXPORT_SYMBOL_GPL(unregister_vmap_purge_notifier);
1252 * lazy_max_pages is the maximum amount of virtual address space we gather up
1253 * before attempting to purge with a TLB flush.
1255 * There is a tradeoff here: a larger number will cover more kernel page tables
1256 * and take slightly longer to purge, but it will linearly reduce the number of
1257 * global TLB flushes that must be performed. It would seem natural to scale
1258 * this number up linearly with the number of CPUs (because vmapping activity
1259 * could also scale linearly with the number of CPUs), however it is likely
1260 * that in practice, workloads might be constrained in other ways that mean
1261 * vmap activity will not scale linearly with CPUs. Also, I want to be
1262 * conservative and not introduce a big latency on huge systems, so go with
1263 * a less aggressive log scale. It will still be an improvement over the old
1264 * code, and it will be simple to change the scale factor if we find that it
1265 * becomes a problem on bigger systems.
1267 static unsigned long lazy_max_pages(void)
1271 log = fls(num_online_cpus());
1273 return log * (32UL * 1024 * 1024 / PAGE_SIZE);
1276 static atomic_long_t vmap_lazy_nr = ATOMIC_LONG_INIT(0);
1279 * Serialize vmap purging. There is no actual criticial section protected
1280 * by this look, but we want to avoid concurrent calls for performance
1281 * reasons and to make the pcpu_get_vm_areas more deterministic.
1283 static DEFINE_MUTEX(vmap_purge_lock);
1285 /* for per-CPU blocks */
1286 static void purge_fragmented_blocks_allcpus(void);
1289 * called before a call to iounmap() if the caller wants vm_area_struct's
1290 * immediately freed.
1292 void set_iounmap_nonlazy(void)
1294 atomic_long_set(&vmap_lazy_nr, lazy_max_pages()+1);
1298 * Purges all lazily-freed vmap areas.
1300 static bool __purge_vmap_area_lazy(unsigned long start, unsigned long end)
1302 unsigned long resched_threshold;
1303 struct llist_node *valist;
1304 struct vmap_area *va;
1305 struct vmap_area *n_va;
1307 lockdep_assert_held(&vmap_purge_lock);
1309 valist = llist_del_all(&vmap_purge_list);
1310 if (unlikely(valist == NULL))
1314 * First make sure the mappings are removed from all page-tables
1315 * before they are freed.
1317 vmalloc_sync_unmappings();
1320 * TODO: to calculate a flush range without looping.
1321 * The list can be up to lazy_max_pages() elements.
1323 llist_for_each_entry(va, valist, purge_list) {
1324 if (va->va_start < start)
1325 start = va->va_start;
1326 if (va->va_end > end)
1330 flush_tlb_kernel_range(start, end);
1331 resched_threshold = lazy_max_pages() << 1;
1333 spin_lock(&free_vmap_area_lock);
1334 llist_for_each_entry_safe(va, n_va, valist, purge_list) {
1335 unsigned long nr = (va->va_end - va->va_start) >> PAGE_SHIFT;
1336 unsigned long orig_start = va->va_start;
1337 unsigned long orig_end = va->va_end;
1340 * Finally insert or merge lazily-freed area. It is
1341 * detached and there is no need to "unlink" it from
1344 va = merge_or_add_vmap_area(va, &free_vmap_area_root,
1345 &free_vmap_area_list);
1347 if (is_vmalloc_or_module_addr((void *)orig_start))
1348 kasan_release_vmalloc(orig_start, orig_end,
1349 va->va_start, va->va_end);
1351 atomic_long_sub(nr, &vmap_lazy_nr);
1353 if (atomic_long_read(&vmap_lazy_nr) < resched_threshold)
1354 cond_resched_lock(&free_vmap_area_lock);
1356 spin_unlock(&free_vmap_area_lock);
1361 * Kick off a purge of the outstanding lazy areas. Don't bother if somebody
1362 * is already purging.
1364 static void try_purge_vmap_area_lazy(void)
1366 if (mutex_trylock(&vmap_purge_lock)) {
1367 __purge_vmap_area_lazy(ULONG_MAX, 0);
1368 mutex_unlock(&vmap_purge_lock);
1373 * Kick off a purge of the outstanding lazy areas.
1375 static void purge_vmap_area_lazy(void)
1377 mutex_lock(&vmap_purge_lock);
1378 purge_fragmented_blocks_allcpus();
1379 __purge_vmap_area_lazy(ULONG_MAX, 0);
1380 mutex_unlock(&vmap_purge_lock);
1384 * Free a vmap area, caller ensuring that the area has been unmapped
1385 * and flush_cache_vunmap had been called for the correct range
1388 static void free_vmap_area_noflush(struct vmap_area *va)
1390 unsigned long nr_lazy;
1392 spin_lock(&vmap_area_lock);
1393 unlink_va(va, &vmap_area_root);
1394 spin_unlock(&vmap_area_lock);
1396 nr_lazy = atomic_long_add_return((va->va_end - va->va_start) >>
1397 PAGE_SHIFT, &vmap_lazy_nr);
1399 /* After this point, we may free va at any time */
1400 llist_add(&va->purge_list, &vmap_purge_list);
1402 if (unlikely(nr_lazy > lazy_max_pages()))
1403 try_purge_vmap_area_lazy();
1407 * Free and unmap a vmap area
1409 static void free_unmap_vmap_area(struct vmap_area *va)
1411 flush_cache_vunmap(va->va_start, va->va_end);
1412 unmap_kernel_range_noflush(va->va_start, va->va_end - va->va_start);
1413 if (debug_pagealloc_enabled_static())
1414 flush_tlb_kernel_range(va->va_start, va->va_end);
1416 free_vmap_area_noflush(va);
1419 static struct vmap_area *find_vmap_area(unsigned long addr)
1421 struct vmap_area *va;
1423 spin_lock(&vmap_area_lock);
1424 va = __find_vmap_area(addr);
1425 spin_unlock(&vmap_area_lock);
1430 /*** Per cpu kva allocator ***/
1433 * vmap space is limited especially on 32 bit architectures. Ensure there is
1434 * room for at least 16 percpu vmap blocks per CPU.
1437 * If we had a constant VMALLOC_START and VMALLOC_END, we'd like to be able
1438 * to #define VMALLOC_SPACE (VMALLOC_END-VMALLOC_START). Guess
1439 * instead (we just need a rough idea)
1441 #if BITS_PER_LONG == 32
1442 #define VMALLOC_SPACE (128UL*1024*1024)
1444 #define VMALLOC_SPACE (128UL*1024*1024*1024)
1447 #define VMALLOC_PAGES (VMALLOC_SPACE / PAGE_SIZE)
1448 #define VMAP_MAX_ALLOC BITS_PER_LONG /* 256K with 4K pages */
1449 #define VMAP_BBMAP_BITS_MAX 1024 /* 4MB with 4K pages */
1450 #define VMAP_BBMAP_BITS_MIN (VMAP_MAX_ALLOC*2)
1451 #define VMAP_MIN(x, y) ((x) < (y) ? (x) : (y)) /* can't use min() */
1452 #define VMAP_MAX(x, y) ((x) > (y) ? (x) : (y)) /* can't use max() */
1453 #define VMAP_BBMAP_BITS \
1454 VMAP_MIN(VMAP_BBMAP_BITS_MAX, \
1455 VMAP_MAX(VMAP_BBMAP_BITS_MIN, \
1456 VMALLOC_PAGES / roundup_pow_of_two(NR_CPUS) / 16))
1458 #define VMAP_BLOCK_SIZE (VMAP_BBMAP_BITS * PAGE_SIZE)
1460 struct vmap_block_queue {
1462 struct list_head free;
1467 struct vmap_area *va;
1468 unsigned long free, dirty;
1469 unsigned long dirty_min, dirty_max; /*< dirty range */
1470 struct list_head free_list;
1471 struct rcu_head rcu_head;
1472 struct list_head purge;
1475 /* Queue of free and dirty vmap blocks, for allocation and flushing purposes */
1476 static DEFINE_PER_CPU(struct vmap_block_queue, vmap_block_queue);
1479 * Radix tree of vmap blocks, indexed by address, to quickly find a vmap block
1480 * in the free path. Could get rid of this if we change the API to return a
1481 * "cookie" from alloc, to be passed to free. But no big deal yet.
1483 static DEFINE_SPINLOCK(vmap_block_tree_lock);
1484 static RADIX_TREE(vmap_block_tree, GFP_ATOMIC);
1487 * We should probably have a fallback mechanism to allocate virtual memory
1488 * out of partially filled vmap blocks. However vmap block sizing should be
1489 * fairly reasonable according to the vmalloc size, so it shouldn't be a
1493 static unsigned long addr_to_vb_idx(unsigned long addr)
1495 addr -= VMALLOC_START & ~(VMAP_BLOCK_SIZE-1);
1496 addr /= VMAP_BLOCK_SIZE;
1500 static void *vmap_block_vaddr(unsigned long va_start, unsigned long pages_off)
1504 addr = va_start + (pages_off << PAGE_SHIFT);
1505 BUG_ON(addr_to_vb_idx(addr) != addr_to_vb_idx(va_start));
1506 return (void *)addr;
1510 * new_vmap_block - allocates new vmap_block and occupies 2^order pages in this
1511 * block. Of course pages number can't exceed VMAP_BBMAP_BITS
1512 * @order: how many 2^order pages should be occupied in newly allocated block
1513 * @gfp_mask: flags for the page level allocator
1515 * Return: virtual address in a newly allocated block or ERR_PTR(-errno)
1517 static void *new_vmap_block(unsigned int order, gfp_t gfp_mask)
1519 struct vmap_block_queue *vbq;
1520 struct vmap_block *vb;
1521 struct vmap_area *va;
1522 unsigned long vb_idx;
1526 node = numa_node_id();
1528 vb = kmalloc_node(sizeof(struct vmap_block),
1529 gfp_mask & GFP_RECLAIM_MASK, node);
1531 return ERR_PTR(-ENOMEM);
1533 va = alloc_vmap_area(VMAP_BLOCK_SIZE, VMAP_BLOCK_SIZE,
1534 VMALLOC_START, VMALLOC_END,
1538 return ERR_CAST(va);
1541 err = radix_tree_preload(gfp_mask);
1542 if (unlikely(err)) {
1545 return ERR_PTR(err);
1548 vaddr = vmap_block_vaddr(va->va_start, 0);
1549 spin_lock_init(&vb->lock);
1551 /* At least something should be left free */
1552 BUG_ON(VMAP_BBMAP_BITS <= (1UL << order));
1553 vb->free = VMAP_BBMAP_BITS - (1UL << order);
1555 vb->dirty_min = VMAP_BBMAP_BITS;
1557 INIT_LIST_HEAD(&vb->free_list);
1559 vb_idx = addr_to_vb_idx(va->va_start);
1560 spin_lock(&vmap_block_tree_lock);
1561 err = radix_tree_insert(&vmap_block_tree, vb_idx, vb);
1562 spin_unlock(&vmap_block_tree_lock);
1564 radix_tree_preload_end();
1566 vbq = &get_cpu_var(vmap_block_queue);
1567 spin_lock(&vbq->lock);
1568 list_add_tail_rcu(&vb->free_list, &vbq->free);
1569 spin_unlock(&vbq->lock);
1570 put_cpu_var(vmap_block_queue);
1575 static void free_vmap_block(struct vmap_block *vb)
1577 struct vmap_block *tmp;
1578 unsigned long vb_idx;
1580 vb_idx = addr_to_vb_idx(vb->va->va_start);
1581 spin_lock(&vmap_block_tree_lock);
1582 tmp = radix_tree_delete(&vmap_block_tree, vb_idx);
1583 spin_unlock(&vmap_block_tree_lock);
1586 free_vmap_area_noflush(vb->va);
1587 kfree_rcu(vb, rcu_head);
1590 static void purge_fragmented_blocks(int cpu)
1593 struct vmap_block *vb;
1594 struct vmap_block *n_vb;
1595 struct vmap_block_queue *vbq = &per_cpu(vmap_block_queue, cpu);
1598 list_for_each_entry_rcu(vb, &vbq->free, free_list) {
1600 if (!(vb->free + vb->dirty == VMAP_BBMAP_BITS && vb->dirty != VMAP_BBMAP_BITS))
1603 spin_lock(&vb->lock);
1604 if (vb->free + vb->dirty == VMAP_BBMAP_BITS && vb->dirty != VMAP_BBMAP_BITS) {
1605 vb->free = 0; /* prevent further allocs after releasing lock */
1606 vb->dirty = VMAP_BBMAP_BITS; /* prevent purging it again */
1608 vb->dirty_max = VMAP_BBMAP_BITS;
1609 spin_lock(&vbq->lock);
1610 list_del_rcu(&vb->free_list);
1611 spin_unlock(&vbq->lock);
1612 spin_unlock(&vb->lock);
1613 list_add_tail(&vb->purge, &purge);
1615 spin_unlock(&vb->lock);
1619 list_for_each_entry_safe(vb, n_vb, &purge, purge) {
1620 list_del(&vb->purge);
1621 free_vmap_block(vb);
1625 static void purge_fragmented_blocks_allcpus(void)
1629 for_each_possible_cpu(cpu)
1630 purge_fragmented_blocks(cpu);
1633 static void *vb_alloc(unsigned long size, gfp_t gfp_mask)
1635 struct vmap_block_queue *vbq;
1636 struct vmap_block *vb;
1640 BUG_ON(offset_in_page(size));
1641 BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC);
1642 if (WARN_ON(size == 0)) {
1644 * Allocating 0 bytes isn't what caller wants since
1645 * get_order(0) returns funny result. Just warn and terminate
1650 order = get_order(size);
1653 vbq = &get_cpu_var(vmap_block_queue);
1654 list_for_each_entry_rcu(vb, &vbq->free, free_list) {
1655 unsigned long pages_off;
1657 spin_lock(&vb->lock);
1658 if (vb->free < (1UL << order)) {
1659 spin_unlock(&vb->lock);
1663 pages_off = VMAP_BBMAP_BITS - vb->free;
1664 vaddr = vmap_block_vaddr(vb->va->va_start, pages_off);
1665 vb->free -= 1UL << order;
1666 if (vb->free == 0) {
1667 spin_lock(&vbq->lock);
1668 list_del_rcu(&vb->free_list);
1669 spin_unlock(&vbq->lock);
1672 spin_unlock(&vb->lock);
1676 put_cpu_var(vmap_block_queue);
1679 /* Allocate new block if nothing was found */
1681 vaddr = new_vmap_block(order, gfp_mask);
1686 static void vb_free(unsigned long addr, unsigned long size)
1688 unsigned long offset;
1689 unsigned long vb_idx;
1691 struct vmap_block *vb;
1693 BUG_ON(offset_in_page(size));
1694 BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC);
1696 flush_cache_vunmap(addr, addr + size);
1698 order = get_order(size);
1700 offset = (addr & (VMAP_BLOCK_SIZE - 1)) >> PAGE_SHIFT;
1702 vb_idx = addr_to_vb_idx(addr);
1704 vb = radix_tree_lookup(&vmap_block_tree, vb_idx);
1708 unmap_kernel_range_noflush(addr, size);
1710 if (debug_pagealloc_enabled_static())
1711 flush_tlb_kernel_range(addr, addr + size);
1713 spin_lock(&vb->lock);
1715 /* Expand dirty range */
1716 vb->dirty_min = min(vb->dirty_min, offset);
1717 vb->dirty_max = max(vb->dirty_max, offset + (1UL << order));
1719 vb->dirty += 1UL << order;
1720 if (vb->dirty == VMAP_BBMAP_BITS) {
1722 spin_unlock(&vb->lock);
1723 free_vmap_block(vb);
1725 spin_unlock(&vb->lock);
1728 static void _vm_unmap_aliases(unsigned long start, unsigned long end, int flush)
1732 if (unlikely(!vmap_initialized))
1737 for_each_possible_cpu(cpu) {
1738 struct vmap_block_queue *vbq = &per_cpu(vmap_block_queue, cpu);
1739 struct vmap_block *vb;
1742 list_for_each_entry_rcu(vb, &vbq->free, free_list) {
1743 spin_lock(&vb->lock);
1745 unsigned long va_start = vb->va->va_start;
1748 s = va_start + (vb->dirty_min << PAGE_SHIFT);
1749 e = va_start + (vb->dirty_max << PAGE_SHIFT);
1751 start = min(s, start);
1756 spin_unlock(&vb->lock);
1761 mutex_lock(&vmap_purge_lock);
1762 purge_fragmented_blocks_allcpus();
1763 if (!__purge_vmap_area_lazy(start, end) && flush)
1764 flush_tlb_kernel_range(start, end);
1765 mutex_unlock(&vmap_purge_lock);
1769 * vm_unmap_aliases - unmap outstanding lazy aliases in the vmap layer
1771 * The vmap/vmalloc layer lazily flushes kernel virtual mappings primarily
1772 * to amortize TLB flushing overheads. What this means is that any page you
1773 * have now, may, in a former life, have been mapped into kernel virtual
1774 * address by the vmap layer and so there might be some CPUs with TLB entries
1775 * still referencing that page (additional to the regular 1:1 kernel mapping).
1777 * vm_unmap_aliases flushes all such lazy mappings. After it returns, we can
1778 * be sure that none of the pages we have control over will have any aliases
1779 * from the vmap layer.
1781 void vm_unmap_aliases(void)
1783 unsigned long start = ULONG_MAX, end = 0;
1786 _vm_unmap_aliases(start, end, flush);
1788 EXPORT_SYMBOL_GPL(vm_unmap_aliases);
1791 * vm_unmap_ram - unmap linear kernel address space set up by vm_map_ram
1792 * @mem: the pointer returned by vm_map_ram
1793 * @count: the count passed to that vm_map_ram call (cannot unmap partial)
1795 void vm_unmap_ram(const void *mem, unsigned int count)
1797 unsigned long size = (unsigned long)count << PAGE_SHIFT;
1798 unsigned long addr = (unsigned long)mem;
1799 struct vmap_area *va;
1803 BUG_ON(addr < VMALLOC_START);
1804 BUG_ON(addr > VMALLOC_END);
1805 BUG_ON(!PAGE_ALIGNED(addr));
1807 kasan_poison_vmalloc(mem, size);
1809 if (likely(count <= VMAP_MAX_ALLOC)) {
1810 debug_check_no_locks_freed(mem, size);
1811 vb_free(addr, size);
1815 va = find_vmap_area(addr);
1817 debug_check_no_locks_freed((void *)va->va_start,
1818 (va->va_end - va->va_start));
1819 free_unmap_vmap_area(va);
1821 EXPORT_SYMBOL(vm_unmap_ram);
1824 * vm_map_ram - map pages linearly into kernel virtual address (vmalloc space)
1825 * @pages: an array of pointers to the pages to be mapped
1826 * @count: number of pages
1827 * @node: prefer to allocate data structures on this node
1828 * @prot: memory protection to use. PAGE_KERNEL for regular RAM
1830 * If you use this function for less than VMAP_MAX_ALLOC pages, it could be
1831 * faster than vmap so it's good. But if you mix long-life and short-life
1832 * objects with vm_map_ram(), it could consume lots of address space through
1833 * fragmentation (especially on a 32bit machine). You could see failures in
1834 * the end. Please use this function for short-lived objects.
1836 * Returns: a pointer to the address that has been mapped, or %NULL on failure
1838 void *vm_map_ram(struct page **pages, unsigned int count, int node)
1840 unsigned long size = (unsigned long)count << PAGE_SHIFT;
1844 if (likely(count <= VMAP_MAX_ALLOC)) {
1845 mem = vb_alloc(size, GFP_KERNEL);
1848 addr = (unsigned long)mem;
1850 struct vmap_area *va;
1851 va = alloc_vmap_area(size, PAGE_SIZE,
1852 VMALLOC_START, VMALLOC_END, node, GFP_KERNEL);
1856 addr = va->va_start;
1860 kasan_unpoison_vmalloc(mem, size);
1862 if (map_kernel_range(addr, size, PAGE_KERNEL, pages) < 0) {
1863 vm_unmap_ram(mem, count);
1868 EXPORT_SYMBOL(vm_map_ram);
1870 static struct vm_struct *vmlist __initdata;
1873 * vm_area_add_early - add vmap area early during boot
1874 * @vm: vm_struct to add
1876 * This function is used to add fixed kernel vm area to vmlist before
1877 * vmalloc_init() is called. @vm->addr, @vm->size, and @vm->flags
1878 * should contain proper values and the other fields should be zero.
1880 * DO NOT USE THIS FUNCTION UNLESS YOU KNOW WHAT YOU'RE DOING.
1882 void __init vm_area_add_early(struct vm_struct *vm)
1884 struct vm_struct *tmp, **p;
1886 BUG_ON(vmap_initialized);
1887 for (p = &vmlist; (tmp = *p) != NULL; p = &tmp->next) {
1888 if (tmp->addr >= vm->addr) {
1889 BUG_ON(tmp->addr < vm->addr + vm->size);
1892 BUG_ON(tmp->addr + tmp->size > vm->addr);
1899 * vm_area_register_early - register vmap area early during boot
1900 * @vm: vm_struct to register
1901 * @align: requested alignment
1903 * This function is used to register kernel vm area before
1904 * vmalloc_init() is called. @vm->size and @vm->flags should contain
1905 * proper values on entry and other fields should be zero. On return,
1906 * vm->addr contains the allocated address.
1908 * DO NOT USE THIS FUNCTION UNLESS YOU KNOW WHAT YOU'RE DOING.
1910 void __init vm_area_register_early(struct vm_struct *vm, size_t align)
1912 static size_t vm_init_off __initdata;
1915 addr = ALIGN(VMALLOC_START + vm_init_off, align);
1916 vm_init_off = PFN_ALIGN(addr + vm->size) - VMALLOC_START;
1918 vm->addr = (void *)addr;
1920 vm_area_add_early(vm);
1923 static void vmap_init_free_space(void)
1925 unsigned long vmap_start = 1;
1926 const unsigned long vmap_end = ULONG_MAX;
1927 struct vmap_area *busy, *free;
1931 * -|-----|.....|-----|-----|-----|.....|-
1933 * |<--------------------------------->|
1935 list_for_each_entry(busy, &vmap_area_list, list) {
1936 if (busy->va_start - vmap_start > 0) {
1937 free = kmem_cache_zalloc(vmap_area_cachep, GFP_NOWAIT);
1938 if (!WARN_ON_ONCE(!free)) {
1939 free->va_start = vmap_start;
1940 free->va_end = busy->va_start;
1942 insert_vmap_area_augment(free, NULL,
1943 &free_vmap_area_root,
1944 &free_vmap_area_list);
1948 vmap_start = busy->va_end;
1951 if (vmap_end - vmap_start > 0) {
1952 free = kmem_cache_zalloc(vmap_area_cachep, GFP_NOWAIT);
1953 if (!WARN_ON_ONCE(!free)) {
1954 free->va_start = vmap_start;
1955 free->va_end = vmap_end;
1957 insert_vmap_area_augment(free, NULL,
1958 &free_vmap_area_root,
1959 &free_vmap_area_list);
1964 void __init vmalloc_init(void)
1966 struct vmap_area *va;
1967 struct vm_struct *tmp;
1971 * Create the cache for vmap_area objects.
1973 vmap_area_cachep = KMEM_CACHE(vmap_area, SLAB_PANIC);
1975 for_each_possible_cpu(i) {
1976 struct vmap_block_queue *vbq;
1977 struct vfree_deferred *p;
1979 vbq = &per_cpu(vmap_block_queue, i);
1980 spin_lock_init(&vbq->lock);
1981 INIT_LIST_HEAD(&vbq->free);
1982 p = &per_cpu(vfree_deferred, i);
1983 init_llist_head(&p->list);
1984 INIT_WORK(&p->wq, free_work);
1987 /* Import existing vmlist entries. */
1988 for (tmp = vmlist; tmp; tmp = tmp->next) {
1989 va = kmem_cache_zalloc(vmap_area_cachep, GFP_NOWAIT);
1990 if (WARN_ON_ONCE(!va))
1993 va->va_start = (unsigned long)tmp->addr;
1994 va->va_end = va->va_start + tmp->size;
1996 insert_vmap_area(va, &vmap_area_root, &vmap_area_list);
2000 * Now we can initialize a free vmap space.
2002 vmap_init_free_space();
2003 vmap_initialized = true;
2007 * unmap_kernel_range - unmap kernel VM area and flush cache and TLB
2008 * @addr: start of the VM area to unmap
2009 * @size: size of the VM area to unmap
2011 * Similar to unmap_kernel_range_noflush() but flushes vcache before
2012 * the unmapping and tlb after.
2014 void unmap_kernel_range(unsigned long addr, unsigned long size)
2016 unsigned long end = addr + size;
2018 flush_cache_vunmap(addr, end);
2019 unmap_kernel_range_noflush(addr, size);
2020 flush_tlb_kernel_range(addr, end);
2023 static inline void setup_vmalloc_vm_locked(struct vm_struct *vm,
2024 struct vmap_area *va, unsigned long flags, const void *caller)
2027 vm->addr = (void *)va->va_start;
2028 vm->size = va->va_end - va->va_start;
2029 vm->caller = caller;
2033 static void setup_vmalloc_vm(struct vm_struct *vm, struct vmap_area *va,
2034 unsigned long flags, const void *caller)
2036 spin_lock(&vmap_area_lock);
2037 setup_vmalloc_vm_locked(vm, va, flags, caller);
2038 spin_unlock(&vmap_area_lock);
2041 static void clear_vm_uninitialized_flag(struct vm_struct *vm)
2044 * Before removing VM_UNINITIALIZED,
2045 * we should make sure that vm has proper values.
2046 * Pair with smp_rmb() in show_numa_info().
2049 vm->flags &= ~VM_UNINITIALIZED;
2052 static struct vm_struct *__get_vm_area_node(unsigned long size,
2053 unsigned long align, unsigned long flags, unsigned long start,
2054 unsigned long end, int node, gfp_t gfp_mask, const void *caller)
2056 struct vmap_area *va;
2057 struct vm_struct *area;
2058 unsigned long requested_size = size;
2060 BUG_ON(in_interrupt());
2061 size = PAGE_ALIGN(size);
2062 if (unlikely(!size))
2065 if (flags & VM_IOREMAP)
2066 align = 1ul << clamp_t(int, get_count_order_long(size),
2067 PAGE_SHIFT, IOREMAP_MAX_ORDER);
2069 area = kzalloc_node(sizeof(*area), gfp_mask & GFP_RECLAIM_MASK, node);
2070 if (unlikely(!area))
2073 if (!(flags & VM_NO_GUARD))
2076 va = alloc_vmap_area(size, align, start, end, node, gfp_mask);
2082 kasan_unpoison_vmalloc((void *)va->va_start, requested_size);
2084 setup_vmalloc_vm(area, va, flags, caller);
2089 struct vm_struct *__get_vm_area_caller(unsigned long size, unsigned long flags,
2090 unsigned long start, unsigned long end,
2093 return __get_vm_area_node(size, 1, flags, start, end, NUMA_NO_NODE,
2094 GFP_KERNEL, caller);
2098 * get_vm_area - reserve a contiguous kernel virtual area
2099 * @size: size of the area
2100 * @flags: %VM_IOREMAP for I/O mappings or VM_ALLOC
2102 * Search an area of @size in the kernel virtual mapping area,
2103 * and reserved it for out purposes. Returns the area descriptor
2104 * on success or %NULL on failure.
2106 * Return: the area descriptor on success or %NULL on failure.
2108 struct vm_struct *get_vm_area(unsigned long size, unsigned long flags)
2110 return __get_vm_area_node(size, 1, flags, VMALLOC_START, VMALLOC_END,
2111 NUMA_NO_NODE, GFP_KERNEL,
2112 __builtin_return_address(0));
2115 struct vm_struct *get_vm_area_caller(unsigned long size, unsigned long flags,
2118 return __get_vm_area_node(size, 1, flags, VMALLOC_START, VMALLOC_END,
2119 NUMA_NO_NODE, GFP_KERNEL, caller);
2123 * find_vm_area - find a continuous kernel virtual area
2124 * @addr: base address
2126 * Search for the kernel VM area starting at @addr, and return it.
2127 * It is up to the caller to do all required locking to keep the returned
2130 * Return: pointer to the found area or %NULL on faulure
2132 struct vm_struct *find_vm_area(const void *addr)
2134 struct vmap_area *va;
2136 va = find_vmap_area((unsigned long)addr);
2144 * remove_vm_area - find and remove a continuous kernel virtual area
2145 * @addr: base address
2147 * Search for the kernel VM area starting at @addr, and remove it.
2148 * This function returns the found VM area, but using it is NOT safe
2149 * on SMP machines, except for its size or flags.
2151 * Return: pointer to the found area or %NULL on faulure
2153 struct vm_struct *remove_vm_area(const void *addr)
2155 struct vmap_area *va;
2159 spin_lock(&vmap_area_lock);
2160 va = __find_vmap_area((unsigned long)addr);
2162 struct vm_struct *vm = va->vm;
2165 spin_unlock(&vmap_area_lock);
2167 kasan_free_shadow(vm);
2168 free_unmap_vmap_area(va);
2173 spin_unlock(&vmap_area_lock);
2177 static inline void set_area_direct_map(const struct vm_struct *area,
2178 int (*set_direct_map)(struct page *page))
2182 for (i = 0; i < area->nr_pages; i++)
2183 if (page_address(area->pages[i]))
2184 set_direct_map(area->pages[i]);
2187 /* Handle removing and resetting vm mappings related to the vm_struct. */
2188 static void vm_remove_mappings(struct vm_struct *area, int deallocate_pages)
2190 unsigned long start = ULONG_MAX, end = 0;
2191 int flush_reset = area->flags & VM_FLUSH_RESET_PERMS;
2195 remove_vm_area(area->addr);
2197 /* If this is not VM_FLUSH_RESET_PERMS memory, no need for the below. */
2202 * If not deallocating pages, just do the flush of the VM area and
2205 if (!deallocate_pages) {
2211 * If execution gets here, flush the vm mapping and reset the direct
2212 * map. Find the start and end range of the direct mappings to make sure
2213 * the vm_unmap_aliases() flush includes the direct map.
2215 for (i = 0; i < area->nr_pages; i++) {
2216 unsigned long addr = (unsigned long)page_address(area->pages[i]);
2218 start = min(addr, start);
2219 end = max(addr + PAGE_SIZE, end);
2225 * Set direct map to something invalid so that it won't be cached if
2226 * there are any accesses after the TLB flush, then flush the TLB and
2227 * reset the direct map permissions to the default.
2229 set_area_direct_map(area, set_direct_map_invalid_noflush);
2230 _vm_unmap_aliases(start, end, flush_dmap);
2231 set_area_direct_map(area, set_direct_map_default_noflush);
2234 static void __vunmap(const void *addr, int deallocate_pages)
2236 struct vm_struct *area;
2241 if (WARN(!PAGE_ALIGNED(addr), "Trying to vfree() bad address (%p)\n",
2245 area = find_vm_area(addr);
2246 if (unlikely(!area)) {
2247 WARN(1, KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n",
2252 debug_check_no_locks_freed(area->addr, get_vm_area_size(area));
2253 debug_check_no_obj_freed(area->addr, get_vm_area_size(area));
2255 kasan_poison_vmalloc(area->addr, area->size);
2257 vm_remove_mappings(area, deallocate_pages);
2259 if (deallocate_pages) {
2262 for (i = 0; i < area->nr_pages; i++) {
2263 struct page *page = area->pages[i];
2266 __free_pages(page, 0);
2268 atomic_long_sub(area->nr_pages, &nr_vmalloc_pages);
2270 kvfree(area->pages);
2277 static inline void __vfree_deferred(const void *addr)
2280 * Use raw_cpu_ptr() because this can be called from preemptible
2281 * context. Preemption is absolutely fine here, because the llist_add()
2282 * implementation is lockless, so it works even if we are adding to
2283 * nother cpu's list. schedule_work() should be fine with this too.
2285 struct vfree_deferred *p = raw_cpu_ptr(&vfree_deferred);
2287 if (llist_add((struct llist_node *)addr, &p->list))
2288 schedule_work(&p->wq);
2292 * vfree_atomic - release memory allocated by vmalloc()
2293 * @addr: memory base address
2295 * This one is just like vfree() but can be called in any atomic context
2298 void vfree_atomic(const void *addr)
2302 kmemleak_free(addr);
2306 __vfree_deferred(addr);
2309 static void __vfree(const void *addr)
2311 if (unlikely(in_interrupt()))
2312 __vfree_deferred(addr);
2318 * vfree - release memory allocated by vmalloc()
2319 * @addr: memory base address
2321 * Free the virtually continuous memory area starting at @addr, as
2322 * obtained from vmalloc(), vmalloc_32() or __vmalloc(). If @addr is
2323 * NULL, no operation is performed.
2325 * Must not be called in NMI context (strictly speaking, only if we don't
2326 * have CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG, but making the calling
2327 * conventions for vfree() arch-depenedent would be a really bad idea)
2329 * May sleep if called *not* from interrupt context.
2331 * NOTE: assumes that the object at @addr has a size >= sizeof(llist_node)
2333 void vfree(const void *addr)
2337 kmemleak_free(addr);
2339 might_sleep_if(!in_interrupt());
2346 EXPORT_SYMBOL(vfree);
2349 * vunmap - release virtual mapping obtained by vmap()
2350 * @addr: memory base address
2352 * Free the virtually contiguous memory area starting at @addr,
2353 * which was created from the page array passed to vmap().
2355 * Must not be called in interrupt context.
2357 void vunmap(const void *addr)
2359 BUG_ON(in_interrupt());
2364 EXPORT_SYMBOL(vunmap);
2367 * vmap - map an array of pages into virtually contiguous space
2368 * @pages: array of page pointers
2369 * @count: number of pages to map
2370 * @flags: vm_area->flags
2371 * @prot: page protection for the mapping
2373 * Maps @count pages from @pages into contiguous kernel virtual
2376 * Return: the address of the area or %NULL on failure
2378 void *vmap(struct page **pages, unsigned int count,
2379 unsigned long flags, pgprot_t prot)
2381 struct vm_struct *area;
2382 unsigned long size; /* In bytes */
2386 if (count > totalram_pages())
2389 size = (unsigned long)count << PAGE_SHIFT;
2390 area = get_vm_area_caller(size, flags, __builtin_return_address(0));
2394 if (map_kernel_range((unsigned long)area->addr, size, prot,
2402 EXPORT_SYMBOL(vmap);
2404 static void *__vmalloc_node(unsigned long size, unsigned long align,
2405 gfp_t gfp_mask, pgprot_t prot,
2406 int node, const void *caller);
2407 static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
2408 pgprot_t prot, int node)
2410 struct page **pages;
2411 unsigned int nr_pages, array_size, i;
2412 const gfp_t nested_gfp = (gfp_mask & GFP_RECLAIM_MASK) | __GFP_ZERO;
2413 const gfp_t alloc_mask = gfp_mask | __GFP_NOWARN;
2414 const gfp_t highmem_mask = (gfp_mask & (GFP_DMA | GFP_DMA32)) ?
2418 nr_pages = get_vm_area_size(area) >> PAGE_SHIFT;
2419 array_size = (nr_pages * sizeof(struct page *));
2421 /* Please note that the recursion is strictly bounded. */
2422 if (array_size > PAGE_SIZE) {
2423 pages = __vmalloc_node(array_size, 1, nested_gfp|highmem_mask,
2424 PAGE_KERNEL, node, area->caller);
2426 pages = kmalloc_node(array_size, nested_gfp, node);
2430 remove_vm_area(area->addr);
2435 area->pages = pages;
2436 area->nr_pages = nr_pages;
2438 for (i = 0; i < area->nr_pages; i++) {
2441 if (node == NUMA_NO_NODE)
2442 page = alloc_page(alloc_mask|highmem_mask);
2444 page = alloc_pages_node(node, alloc_mask|highmem_mask, 0);
2446 if (unlikely(!page)) {
2447 /* Successfully allocated i pages, free them in __vunmap() */
2449 atomic_long_add(area->nr_pages, &nr_vmalloc_pages);
2452 area->pages[i] = page;
2453 if (gfpflags_allow_blocking(gfp_mask))
2456 atomic_long_add(area->nr_pages, &nr_vmalloc_pages);
2458 if (map_kernel_range((unsigned long)area->addr, get_vm_area_size(area),
2465 warn_alloc(gfp_mask, NULL,
2466 "vmalloc: allocation failure, allocated %ld of %ld bytes",
2467 (area->nr_pages*PAGE_SIZE), area->size);
2468 __vfree(area->addr);
2473 * __vmalloc_node_range - allocate virtually contiguous memory
2474 * @size: allocation size
2475 * @align: desired alignment
2476 * @start: vm area range start
2477 * @end: vm area range end
2478 * @gfp_mask: flags for the page level allocator
2479 * @prot: protection mask for the allocated pages
2480 * @vm_flags: additional vm area flags (e.g. %VM_NO_GUARD)
2481 * @node: node to use for allocation or NUMA_NO_NODE
2482 * @caller: caller's return address
2484 * Allocate enough pages to cover @size from the page level
2485 * allocator with @gfp_mask flags. Map them into contiguous
2486 * kernel virtual space, using a pagetable protection of @prot.
2488 * Return: the address of the area or %NULL on failure
2490 void *__vmalloc_node_range(unsigned long size, unsigned long align,
2491 unsigned long start, unsigned long end, gfp_t gfp_mask,
2492 pgprot_t prot, unsigned long vm_flags, int node,
2495 struct vm_struct *area;
2497 unsigned long real_size = size;
2499 size = PAGE_ALIGN(size);
2500 if (!size || (size >> PAGE_SHIFT) > totalram_pages())
2503 area = __get_vm_area_node(real_size, align, VM_ALLOC | VM_UNINITIALIZED |
2504 vm_flags, start, end, node, gfp_mask, caller);
2508 addr = __vmalloc_area_node(area, gfp_mask, prot, node);
2513 * In this function, newly allocated vm_struct has VM_UNINITIALIZED
2514 * flag. It means that vm_struct is not fully initialized.
2515 * Now, it is fully initialized, so remove this flag here.
2517 clear_vm_uninitialized_flag(area);
2519 kmemleak_vmalloc(area, size, gfp_mask);
2524 warn_alloc(gfp_mask, NULL,
2525 "vmalloc: allocation failure: %lu bytes", real_size);
2530 * This is only for performance analysis of vmalloc and stress purpose.
2531 * It is required by vmalloc test module, therefore do not use it other
2534 #ifdef CONFIG_TEST_VMALLOC_MODULE
2535 EXPORT_SYMBOL_GPL(__vmalloc_node_range);
2539 * __vmalloc_node - allocate virtually contiguous memory
2540 * @size: allocation size
2541 * @align: desired alignment
2542 * @gfp_mask: flags for the page level allocator
2543 * @prot: protection mask for the allocated pages
2544 * @node: node to use for allocation or NUMA_NO_NODE
2545 * @caller: caller's return address
2547 * Allocate enough pages to cover @size from the page level
2548 * allocator with @gfp_mask flags. Map them into contiguous
2549 * kernel virtual space, using a pagetable protection of @prot.
2551 * Reclaim modifiers in @gfp_mask - __GFP_NORETRY, __GFP_RETRY_MAYFAIL
2552 * and __GFP_NOFAIL are not supported
2554 * Any use of gfp flags outside of GFP_KERNEL should be consulted
2557 * Return: pointer to the allocated memory or %NULL on error
2559 static void *__vmalloc_node(unsigned long size, unsigned long align,
2560 gfp_t gfp_mask, pgprot_t prot,
2561 int node, const void *caller)
2563 return __vmalloc_node_range(size, align, VMALLOC_START, VMALLOC_END,
2564 gfp_mask, prot, 0, node, caller);
2567 void *__vmalloc(unsigned long size, gfp_t gfp_mask, pgprot_t prot)
2569 return __vmalloc_node(size, 1, gfp_mask, prot, NUMA_NO_NODE,
2570 __builtin_return_address(0));
2572 EXPORT_SYMBOL(__vmalloc);
2574 static inline void *__vmalloc_node_flags(unsigned long size,
2575 int node, gfp_t flags)
2577 return __vmalloc_node(size, 1, flags, PAGE_KERNEL,
2578 node, __builtin_return_address(0));
2582 void *__vmalloc_node_flags_caller(unsigned long size, int node, gfp_t flags,
2585 return __vmalloc_node(size, 1, flags, PAGE_KERNEL, node, caller);
2589 * vmalloc - allocate virtually contiguous memory
2590 * @size: allocation size
2592 * Allocate enough pages to cover @size from the page level
2593 * allocator and map them into contiguous kernel virtual space.
2595 * For tight control over page level allocator and protection flags
2596 * use __vmalloc() instead.
2598 * Return: pointer to the allocated memory or %NULL on error
2600 void *vmalloc(unsigned long size)
2602 return __vmalloc_node_flags(size, NUMA_NO_NODE,
2605 EXPORT_SYMBOL(vmalloc);
2608 * vzalloc - allocate virtually contiguous memory with zero fill
2609 * @size: allocation size
2611 * Allocate enough pages to cover @size from the page level
2612 * allocator and map them into contiguous kernel virtual space.
2613 * The memory allocated is set to zero.
2615 * For tight control over page level allocator and protection flags
2616 * use __vmalloc() instead.
2618 * Return: pointer to the allocated memory or %NULL on error
2620 void *vzalloc(unsigned long size)
2622 return __vmalloc_node_flags(size, NUMA_NO_NODE,
2623 GFP_KERNEL | __GFP_ZERO);
2625 EXPORT_SYMBOL(vzalloc);
2628 * vmalloc_user - allocate zeroed virtually contiguous memory for userspace
2629 * @size: allocation size
2631 * The resulting memory area is zeroed so it can be mapped to userspace
2632 * without leaking data.
2634 * Return: pointer to the allocated memory or %NULL on error
2636 void *vmalloc_user(unsigned long size)
2638 return __vmalloc_node_range(size, SHMLBA, VMALLOC_START, VMALLOC_END,
2639 GFP_KERNEL | __GFP_ZERO, PAGE_KERNEL,
2640 VM_USERMAP, NUMA_NO_NODE,
2641 __builtin_return_address(0));
2643 EXPORT_SYMBOL(vmalloc_user);
2646 * vmalloc_node - allocate memory on a specific node
2647 * @size: allocation size
2650 * Allocate enough pages to cover @size from the page level
2651 * allocator and map them into contiguous kernel virtual space.
2653 * For tight control over page level allocator and protection flags
2654 * use __vmalloc() instead.
2656 * Return: pointer to the allocated memory or %NULL on error
2658 void *vmalloc_node(unsigned long size, int node)
2660 return __vmalloc_node(size, 1, GFP_KERNEL, PAGE_KERNEL,
2661 node, __builtin_return_address(0));
2663 EXPORT_SYMBOL(vmalloc_node);
2666 * vzalloc_node - allocate memory on a specific node with zero fill
2667 * @size: allocation size
2670 * Allocate enough pages to cover @size from the page level
2671 * allocator and map them into contiguous kernel virtual space.
2672 * The memory allocated is set to zero.
2674 * For tight control over page level allocator and protection flags
2675 * use __vmalloc_node() instead.
2677 * Return: pointer to the allocated memory or %NULL on error
2679 void *vzalloc_node(unsigned long size, int node)
2681 return __vmalloc_node_flags(size, node,
2682 GFP_KERNEL | __GFP_ZERO);
2684 EXPORT_SYMBOL(vzalloc_node);
2687 * vmalloc_user_node_flags - allocate memory for userspace on a specific node
2688 * @size: allocation size
2690 * @flags: flags for the page level allocator
2692 * The resulting memory area is zeroed so it can be mapped to userspace
2693 * without leaking data.
2695 * Return: pointer to the allocated memory or %NULL on error
2697 void *vmalloc_user_node_flags(unsigned long size, int node, gfp_t flags)
2699 return __vmalloc_node_range(size, SHMLBA, VMALLOC_START, VMALLOC_END,
2700 flags | __GFP_ZERO, PAGE_KERNEL,
2702 __builtin_return_address(0));
2704 EXPORT_SYMBOL(vmalloc_user_node_flags);
2707 * vmalloc_exec - allocate virtually contiguous, executable memory
2708 * @size: allocation size
2710 * Kernel-internal function to allocate enough pages to cover @size
2711 * the page level allocator and map them into contiguous and
2712 * executable kernel virtual space.
2714 * For tight control over page level allocator and protection flags
2715 * use __vmalloc() instead.
2717 * Return: pointer to the allocated memory or %NULL on error
2719 void *vmalloc_exec(unsigned long size)
2721 return __vmalloc_node_range(size, 1, VMALLOC_START, VMALLOC_END,
2722 GFP_KERNEL, PAGE_KERNEL_EXEC, VM_FLUSH_RESET_PERMS,
2723 NUMA_NO_NODE, __builtin_return_address(0));
2726 #if defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA32)
2727 #define GFP_VMALLOC32 (GFP_DMA32 | GFP_KERNEL)
2728 #elif defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA)
2729 #define GFP_VMALLOC32 (GFP_DMA | GFP_KERNEL)
2732 * 64b systems should always have either DMA or DMA32 zones. For others
2733 * GFP_DMA32 should do the right thing and use the normal zone.
2735 #define GFP_VMALLOC32 GFP_DMA32 | GFP_KERNEL
2739 * vmalloc_32 - allocate virtually contiguous memory (32bit addressable)
2740 * @size: allocation size
2742 * Allocate enough 32bit PA addressable pages to cover @size from the
2743 * page level allocator and map them into contiguous kernel virtual space.
2745 * Return: pointer to the allocated memory or %NULL on error
2747 void *vmalloc_32(unsigned long size)
2749 return __vmalloc_node(size, 1, GFP_VMALLOC32, PAGE_KERNEL,
2750 NUMA_NO_NODE, __builtin_return_address(0));
2752 EXPORT_SYMBOL(vmalloc_32);
2755 * vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory
2756 * @size: allocation size
2758 * The resulting memory area is 32bit addressable and zeroed so it can be
2759 * mapped to userspace without leaking data.
2761 * Return: pointer to the allocated memory or %NULL on error
2763 void *vmalloc_32_user(unsigned long size)
2765 return __vmalloc_node_range(size, SHMLBA, VMALLOC_START, VMALLOC_END,
2766 GFP_VMALLOC32 | __GFP_ZERO, PAGE_KERNEL,
2767 VM_USERMAP, NUMA_NO_NODE,
2768 __builtin_return_address(0));
2770 EXPORT_SYMBOL(vmalloc_32_user);
2773 * small helper routine , copy contents to buf from addr.
2774 * If the page is not present, fill zero.
2777 static int aligned_vread(char *buf, char *addr, unsigned long count)
2783 unsigned long offset, length;
2785 offset = offset_in_page(addr);
2786 length = PAGE_SIZE - offset;
2789 p = vmalloc_to_page(addr);
2791 * To do safe access to this _mapped_ area, we need
2792 * lock. But adding lock here means that we need to add
2793 * overhead of vmalloc()/vfree() calles for this _debug_
2794 * interface, rarely used. Instead of that, we'll use
2795 * kmap() and get small overhead in this access function.
2799 * we can expect USER0 is not used (see vread/vwrite's
2800 * function description)
2802 void *map = kmap_atomic(p);
2803 memcpy(buf, map + offset, length);
2806 memset(buf, 0, length);
2816 static int aligned_vwrite(char *buf, char *addr, unsigned long count)
2822 unsigned long offset, length;
2824 offset = offset_in_page(addr);
2825 length = PAGE_SIZE - offset;
2828 p = vmalloc_to_page(addr);
2830 * To do safe access to this _mapped_ area, we need
2831 * lock. But adding lock here means that we need to add
2832 * overhead of vmalloc()/vfree() calles for this _debug_
2833 * interface, rarely used. Instead of that, we'll use
2834 * kmap() and get small overhead in this access function.
2838 * we can expect USER0 is not used (see vread/vwrite's
2839 * function description)
2841 void *map = kmap_atomic(p);
2842 memcpy(map + offset, buf, length);
2854 * vread() - read vmalloc area in a safe way.
2855 * @buf: buffer for reading data
2856 * @addr: vm address.
2857 * @count: number of bytes to be read.
2859 * This function checks that addr is a valid vmalloc'ed area, and
2860 * copy data from that area to a given buffer. If the given memory range
2861 * of [addr...addr+count) includes some valid address, data is copied to
2862 * proper area of @buf. If there are memory holes, they'll be zero-filled.
2863 * IOREMAP area is treated as memory hole and no copy is done.
2865 * If [addr...addr+count) doesn't includes any intersects with alive
2866 * vm_struct area, returns 0. @buf should be kernel's buffer.
2868 * Note: In usual ops, vread() is never necessary because the caller
2869 * should know vmalloc() area is valid and can use memcpy().
2870 * This is for routines which have to access vmalloc area without
2871 * any information, as /dev/kmem.
2873 * Return: number of bytes for which addr and buf should be increased
2874 * (same number as @count) or %0 if [addr...addr+count) doesn't
2875 * include any intersection with valid vmalloc area
2877 long vread(char *buf, char *addr, unsigned long count)
2879 struct vmap_area *va;
2880 struct vm_struct *vm;
2881 char *vaddr, *buf_start = buf;
2882 unsigned long buflen = count;
2885 /* Don't allow overflow */
2886 if ((unsigned long) addr + count < count)
2887 count = -(unsigned long) addr;
2889 spin_lock(&vmap_area_lock);
2890 list_for_each_entry(va, &vmap_area_list, list) {
2898 vaddr = (char *) vm->addr;
2899 if (addr >= vaddr + get_vm_area_size(vm))
2901 while (addr < vaddr) {
2909 n = vaddr + get_vm_area_size(vm) - addr;
2912 if (!(vm->flags & VM_IOREMAP))
2913 aligned_vread(buf, addr, n);
2914 else /* IOREMAP area is treated as memory hole */
2921 spin_unlock(&vmap_area_lock);
2923 if (buf == buf_start)
2925 /* zero-fill memory holes */
2926 if (buf != buf_start + buflen)
2927 memset(buf, 0, buflen - (buf - buf_start));
2933 * vwrite() - write vmalloc area in a safe way.
2934 * @buf: buffer for source data
2935 * @addr: vm address.
2936 * @count: number of bytes to be read.
2938 * This function checks that addr is a valid vmalloc'ed area, and
2939 * copy data from a buffer to the given addr. If specified range of
2940 * [addr...addr+count) includes some valid address, data is copied from
2941 * proper area of @buf. If there are memory holes, no copy to hole.
2942 * IOREMAP area is treated as memory hole and no copy is done.
2944 * If [addr...addr+count) doesn't includes any intersects with alive
2945 * vm_struct area, returns 0. @buf should be kernel's buffer.
2947 * Note: In usual ops, vwrite() is never necessary because the caller
2948 * should know vmalloc() area is valid and can use memcpy().
2949 * This is for routines which have to access vmalloc area without
2950 * any information, as /dev/kmem.
2952 * Return: number of bytes for which addr and buf should be
2953 * increased (same number as @count) or %0 if [addr...addr+count)
2954 * doesn't include any intersection with valid vmalloc area
2956 long vwrite(char *buf, char *addr, unsigned long count)
2958 struct vmap_area *va;
2959 struct vm_struct *vm;
2961 unsigned long n, buflen;
2964 /* Don't allow overflow */
2965 if ((unsigned long) addr + count < count)
2966 count = -(unsigned long) addr;
2969 spin_lock(&vmap_area_lock);
2970 list_for_each_entry(va, &vmap_area_list, list) {
2978 vaddr = (char *) vm->addr;
2979 if (addr >= vaddr + get_vm_area_size(vm))
2981 while (addr < vaddr) {
2988 n = vaddr + get_vm_area_size(vm) - addr;
2991 if (!(vm->flags & VM_IOREMAP)) {
2992 aligned_vwrite(buf, addr, n);
3000 spin_unlock(&vmap_area_lock);
3007 * remap_vmalloc_range_partial - map vmalloc pages to userspace
3008 * @vma: vma to cover
3009 * @uaddr: target user address to start at
3010 * @kaddr: virtual address of vmalloc kernel memory
3011 * @pgoff: offset from @kaddr to start at
3012 * @size: size of map area
3014 * Returns: 0 for success, -Exxx on failure
3016 * This function checks that @kaddr is a valid vmalloc'ed area,
3017 * and that it is big enough to cover the range starting at
3018 * @uaddr in @vma. Will return failure if that criteria isn't
3021 * Similar to remap_pfn_range() (see mm/memory.c)
3023 int remap_vmalloc_range_partial(struct vm_area_struct *vma, unsigned long uaddr,
3024 void *kaddr, unsigned long pgoff,
3027 struct vm_struct *area;
3029 unsigned long end_index;
3031 if (check_shl_overflow(pgoff, PAGE_SHIFT, &off))
3034 size = PAGE_ALIGN(size);
3036 if (!PAGE_ALIGNED(uaddr) || !PAGE_ALIGNED(kaddr))
3039 area = find_vm_area(kaddr);
3043 if (!(area->flags & (VM_USERMAP | VM_DMA_COHERENT)))
3046 if (check_add_overflow(size, off, &end_index) ||
3047 end_index > get_vm_area_size(area))
3052 struct page *page = vmalloc_to_page(kaddr);
3055 ret = vm_insert_page(vma, uaddr, page);
3064 vma->vm_flags |= VM_DONTEXPAND | VM_DONTDUMP;
3068 EXPORT_SYMBOL(remap_vmalloc_range_partial);
3071 * remap_vmalloc_range - map vmalloc pages to userspace
3072 * @vma: vma to cover (map full range of vma)
3073 * @addr: vmalloc memory
3074 * @pgoff: number of pages into addr before first page to map
3076 * Returns: 0 for success, -Exxx on failure
3078 * This function checks that addr is a valid vmalloc'ed area, and
3079 * that it is big enough to cover the vma. Will return failure if
3080 * that criteria isn't met.
3082 * Similar to remap_pfn_range() (see mm/memory.c)
3084 int remap_vmalloc_range(struct vm_area_struct *vma, void *addr,
3085 unsigned long pgoff)
3087 return remap_vmalloc_range_partial(vma, vma->vm_start,
3089 vma->vm_end - vma->vm_start);
3091 EXPORT_SYMBOL(remap_vmalloc_range);
3094 * Implement stubs for vmalloc_sync_[un]mappings () if the architecture chose
3097 * The purpose of this function is to make sure the vmalloc area
3098 * mappings are identical in all page-tables in the system.
3100 void __weak vmalloc_sync_mappings(void)
3104 void __weak vmalloc_sync_unmappings(void)
3108 static int f(pte_t *pte, unsigned long addr, void *data)
3120 * alloc_vm_area - allocate a range of kernel address space
3121 * @size: size of the area
3122 * @ptes: returns the PTEs for the address space
3124 * Returns: NULL on failure, vm_struct on success
3126 * This function reserves a range of kernel address space, and
3127 * allocates pagetables to map that range. No actual mappings
3130 * If @ptes is non-NULL, pointers to the PTEs (in init_mm)
3131 * allocated for the VM area are returned.
3133 struct vm_struct *alloc_vm_area(size_t size, pte_t **ptes)
3135 struct vm_struct *area;
3137 area = get_vm_area_caller(size, VM_IOREMAP,
3138 __builtin_return_address(0));
3143 * This ensures that page tables are constructed for this region
3144 * of kernel virtual address space and mapped into init_mm.
3146 if (apply_to_page_range(&init_mm, (unsigned long)area->addr,
3147 size, f, ptes ? &ptes : NULL)) {
3154 EXPORT_SYMBOL_GPL(alloc_vm_area);
3156 void free_vm_area(struct vm_struct *area)
3158 struct vm_struct *ret;
3159 ret = remove_vm_area(area->addr);
3160 BUG_ON(ret != area);
3163 EXPORT_SYMBOL_GPL(free_vm_area);
3166 static struct vmap_area *node_to_va(struct rb_node *n)
3168 return rb_entry_safe(n, struct vmap_area, rb_node);
3172 * pvm_find_va_enclose_addr - find the vmap_area @addr belongs to
3173 * @addr: target address
3175 * Returns: vmap_area if it is found. If there is no such area
3176 * the first highest(reverse order) vmap_area is returned
3177 * i.e. va->va_start < addr && va->va_end < addr or NULL
3178 * if there are no any areas before @addr.
3180 static struct vmap_area *
3181 pvm_find_va_enclose_addr(unsigned long addr)
3183 struct vmap_area *va, *tmp;
3186 n = free_vmap_area_root.rb_node;
3190 tmp = rb_entry(n, struct vmap_area, rb_node);
3191 if (tmp->va_start <= addr) {
3193 if (tmp->va_end >= addr)
3206 * pvm_determine_end_from_reverse - find the highest aligned address
3207 * of free block below VMALLOC_END
3209 * in - the VA we start the search(reverse order);
3210 * out - the VA with the highest aligned end address.
3212 * Returns: determined end address within vmap_area
3214 static unsigned long
3215 pvm_determine_end_from_reverse(struct vmap_area **va, unsigned long align)
3217 unsigned long vmalloc_end = VMALLOC_END & ~(align - 1);
3221 list_for_each_entry_from_reverse((*va),
3222 &free_vmap_area_list, list) {
3223 addr = min((*va)->va_end & ~(align - 1), vmalloc_end);
3224 if ((*va)->va_start < addr)
3233 * pcpu_get_vm_areas - allocate vmalloc areas for percpu allocator
3234 * @offsets: array containing offset of each area
3235 * @sizes: array containing size of each area
3236 * @nr_vms: the number of areas to allocate
3237 * @align: alignment, all entries in @offsets and @sizes must be aligned to this
3239 * Returns: kmalloc'd vm_struct pointer array pointing to allocated
3240 * vm_structs on success, %NULL on failure
3242 * Percpu allocator wants to use congruent vm areas so that it can
3243 * maintain the offsets among percpu areas. This function allocates
3244 * congruent vmalloc areas for it with GFP_KERNEL. These areas tend to
3245 * be scattered pretty far, distance between two areas easily going up
3246 * to gigabytes. To avoid interacting with regular vmallocs, these
3247 * areas are allocated from top.
3249 * Despite its complicated look, this allocator is rather simple. It
3250 * does everything top-down and scans free blocks from the end looking
3251 * for matching base. While scanning, if any of the areas do not fit the
3252 * base address is pulled down to fit the area. Scanning is repeated till
3253 * all the areas fit and then all necessary data structures are inserted
3254 * and the result is returned.
3256 struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets,
3257 const size_t *sizes, int nr_vms,
3260 const unsigned long vmalloc_start = ALIGN(VMALLOC_START, align);
3261 const unsigned long vmalloc_end = VMALLOC_END & ~(align - 1);
3262 struct vmap_area **vas, *va;
3263 struct vm_struct **vms;
3264 int area, area2, last_area, term_area;
3265 unsigned long base, start, size, end, last_end, orig_start, orig_end;
3266 bool purged = false;
3269 /* verify parameters and allocate data structures */
3270 BUG_ON(offset_in_page(align) || !is_power_of_2(align));
3271 for (last_area = 0, area = 0; area < nr_vms; area++) {
3272 start = offsets[area];
3273 end = start + sizes[area];
3275 /* is everything aligned properly? */
3276 BUG_ON(!IS_ALIGNED(offsets[area], align));
3277 BUG_ON(!IS_ALIGNED(sizes[area], align));
3279 /* detect the area with the highest address */
3280 if (start > offsets[last_area])
3283 for (area2 = area + 1; area2 < nr_vms; area2++) {
3284 unsigned long start2 = offsets[area2];
3285 unsigned long end2 = start2 + sizes[area2];
3287 BUG_ON(start2 < end && start < end2);
3290 last_end = offsets[last_area] + sizes[last_area];
3292 if (vmalloc_end - vmalloc_start < last_end) {
3297 vms = kcalloc(nr_vms, sizeof(vms[0]), GFP_KERNEL);
3298 vas = kcalloc(nr_vms, sizeof(vas[0]), GFP_KERNEL);
3302 for (area = 0; area < nr_vms; area++) {
3303 vas[area] = kmem_cache_zalloc(vmap_area_cachep, GFP_KERNEL);
3304 vms[area] = kzalloc(sizeof(struct vm_struct), GFP_KERNEL);
3305 if (!vas[area] || !vms[area])
3309 spin_lock(&free_vmap_area_lock);
3311 /* start scanning - we scan from the top, begin with the last area */
3312 area = term_area = last_area;
3313 start = offsets[area];
3314 end = start + sizes[area];
3316 va = pvm_find_va_enclose_addr(vmalloc_end);
3317 base = pvm_determine_end_from_reverse(&va, align) - end;
3321 * base might have underflowed, add last_end before
3324 if (base + last_end < vmalloc_start + last_end)
3328 * Fitting base has not been found.
3334 * If required width exceeds current VA block, move
3335 * base downwards and then recheck.
3337 if (base + end > va->va_end) {
3338 base = pvm_determine_end_from_reverse(&va, align) - end;
3344 * If this VA does not fit, move base downwards and recheck.
3346 if (base + start < va->va_start) {
3347 va = node_to_va(rb_prev(&va->rb_node));
3348 base = pvm_determine_end_from_reverse(&va, align) - end;
3354 * This area fits, move on to the previous one. If
3355 * the previous one is the terminal one, we're done.
3357 area = (area + nr_vms - 1) % nr_vms;
3358 if (area == term_area)
3361 start = offsets[area];
3362 end = start + sizes[area];
3363 va = pvm_find_va_enclose_addr(base + end);
3366 /* we've found a fitting base, insert all va's */
3367 for (area = 0; area < nr_vms; area++) {
3370 start = base + offsets[area];
3373 va = pvm_find_va_enclose_addr(start);
3374 if (WARN_ON_ONCE(va == NULL))
3375 /* It is a BUG(), but trigger recovery instead. */
3378 type = classify_va_fit_type(va, start, size);
3379 if (WARN_ON_ONCE(type == NOTHING_FIT))
3380 /* It is a BUG(), but trigger recovery instead. */
3383 ret = adjust_va_to_fit_type(va, start, size, type);
3387 /* Allocated area. */
3389 va->va_start = start;
3390 va->va_end = start + size;
3393 spin_unlock(&free_vmap_area_lock);
3395 /* populate the kasan shadow space */
3396 for (area = 0; area < nr_vms; area++) {
3397 if (kasan_populate_vmalloc(vas[area]->va_start, sizes[area]))
3398 goto err_free_shadow;
3400 kasan_unpoison_vmalloc((void *)vas[area]->va_start,
3404 /* insert all vm's */
3405 spin_lock(&vmap_area_lock);
3406 for (area = 0; area < nr_vms; area++) {
3407 insert_vmap_area(vas[area], &vmap_area_root, &vmap_area_list);
3409 setup_vmalloc_vm_locked(vms[area], vas[area], VM_ALLOC,
3412 spin_unlock(&vmap_area_lock);
3419 * Remove previously allocated areas. There is no
3420 * need in removing these areas from the busy tree,
3421 * because they are inserted only on the final step
3422 * and when pcpu_get_vm_areas() is success.
3425 orig_start = vas[area]->va_start;
3426 orig_end = vas[area]->va_end;
3427 va = merge_or_add_vmap_area(vas[area], &free_vmap_area_root,
3428 &free_vmap_area_list);
3429 kasan_release_vmalloc(orig_start, orig_end,
3430 va->va_start, va->va_end);
3435 spin_unlock(&free_vmap_area_lock);
3437 purge_vmap_area_lazy();
3440 /* Before "retry", check if we recover. */
3441 for (area = 0; area < nr_vms; area++) {
3445 vas[area] = kmem_cache_zalloc(
3446 vmap_area_cachep, GFP_KERNEL);
3455 for (area = 0; area < nr_vms; area++) {
3457 kmem_cache_free(vmap_area_cachep, vas[area]);
3467 spin_lock(&free_vmap_area_lock);
3469 * We release all the vmalloc shadows, even the ones for regions that
3470 * hadn't been successfully added. This relies on kasan_release_vmalloc
3471 * being able to tolerate this case.
3473 for (area = 0; area < nr_vms; area++) {
3474 orig_start = vas[area]->va_start;
3475 orig_end = vas[area]->va_end;
3476 va = merge_or_add_vmap_area(vas[area], &free_vmap_area_root,
3477 &free_vmap_area_list);
3478 kasan_release_vmalloc(orig_start, orig_end,
3479 va->va_start, va->va_end);
3483 spin_unlock(&free_vmap_area_lock);
3490 * pcpu_free_vm_areas - free vmalloc areas for percpu allocator
3491 * @vms: vm_struct pointer array returned by pcpu_get_vm_areas()
3492 * @nr_vms: the number of allocated areas
3494 * Free vm_structs and the array allocated by pcpu_get_vm_areas().
3496 void pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms)
3500 for (i = 0; i < nr_vms; i++)
3501 free_vm_area(vms[i]);
3504 #endif /* CONFIG_SMP */
3506 #ifdef CONFIG_PROC_FS
3507 static void *s_start(struct seq_file *m, loff_t *pos)
3508 __acquires(&vmap_purge_lock)
3509 __acquires(&vmap_area_lock)
3511 mutex_lock(&vmap_purge_lock);
3512 spin_lock(&vmap_area_lock);
3514 return seq_list_start(&vmap_area_list, *pos);
3517 static void *s_next(struct seq_file *m, void *p, loff_t *pos)
3519 return seq_list_next(p, &vmap_area_list, pos);
3522 static void s_stop(struct seq_file *m, void *p)
3523 __releases(&vmap_purge_lock)
3524 __releases(&vmap_area_lock)
3526 mutex_unlock(&vmap_purge_lock);
3527 spin_unlock(&vmap_area_lock);
3530 static void show_numa_info(struct seq_file *m, struct vm_struct *v)
3532 if (IS_ENABLED(CONFIG_NUMA)) {
3533 unsigned int nr, *counters = m->private;
3538 if (v->flags & VM_UNINITIALIZED)
3540 /* Pair with smp_wmb() in clear_vm_uninitialized_flag() */
3543 memset(counters, 0, nr_node_ids * sizeof(unsigned int));
3545 for (nr = 0; nr < v->nr_pages; nr++)
3546 counters[page_to_nid(v->pages[nr])]++;
3548 for_each_node_state(nr, N_HIGH_MEMORY)
3550 seq_printf(m, " N%u=%u", nr, counters[nr]);
3554 static void show_purge_info(struct seq_file *m)
3556 struct llist_node *head;
3557 struct vmap_area *va;
3559 head = READ_ONCE(vmap_purge_list.first);
3563 llist_for_each_entry(va, head, purge_list) {
3564 seq_printf(m, "0x%pK-0x%pK %7ld unpurged vm_area\n",
3565 (void *)va->va_start, (void *)va->va_end,
3566 va->va_end - va->va_start);
3570 static int s_show(struct seq_file *m, void *p)
3572 struct vmap_area *va;
3573 struct vm_struct *v;
3575 va = list_entry(p, struct vmap_area, list);
3578 * s_show can encounter race with remove_vm_area, !vm on behalf
3579 * of vmap area is being tear down or vm_map_ram allocation.
3582 seq_printf(m, "0x%pK-0x%pK %7ld vm_map_ram\n",
3583 (void *)va->va_start, (void *)va->va_end,
3584 va->va_end - va->va_start);
3591 seq_printf(m, "0x%pK-0x%pK %7ld",
3592 v->addr, v->addr + v->size, v->size);
3595 seq_printf(m, " %pS", v->caller);
3598 seq_printf(m, " pages=%d", v->nr_pages);
3601 seq_printf(m, " phys=%pa", &v->phys_addr);
3603 if (v->flags & VM_IOREMAP)
3604 seq_puts(m, " ioremap");
3606 if (v->flags & VM_ALLOC)
3607 seq_puts(m, " vmalloc");
3609 if (v->flags & VM_MAP)
3610 seq_puts(m, " vmap");
3612 if (v->flags & VM_USERMAP)
3613 seq_puts(m, " user");
3615 if (v->flags & VM_DMA_COHERENT)
3616 seq_puts(m, " dma-coherent");
3618 if (is_vmalloc_addr(v->pages))
3619 seq_puts(m, " vpages");
3621 show_numa_info(m, v);
3625 * As a final step, dump "unpurged" areas. Note,
3626 * that entire "/proc/vmallocinfo" output will not
3627 * be address sorted, because the purge list is not
3630 if (list_is_last(&va->list, &vmap_area_list))
3636 static const struct seq_operations vmalloc_op = {
3643 static int __init proc_vmalloc_init(void)
3645 if (IS_ENABLED(CONFIG_NUMA))
3646 proc_create_seq_private("vmallocinfo", 0400, NULL,
3648 nr_node_ids * sizeof(unsigned int), NULL);
3650 proc_create_seq("vmallocinfo", 0400, NULL, &vmalloc_op);
3653 module_init(proc_vmalloc_init);