mm/sparse-vmemmap.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Virtual Memory Map support
   4  *
   5  * (C) 2007 sgi. Christoph Lameter.
   6  *
   7  * Virtual memory maps allow VM primitives pfn_to_page, page_to_pfn,
   8  * virt_to_page, page_address() to be implemented as a base offset
   9  * calculation without memory access.
  10  *
  11  * However, virtual mappings need a page table and TLBs. Many Linux
  12  * architectures already map their physical space using 1-1 mappings
  13  * via TLBs. For those arches the virtual memory map is essentially
  14  * for free if we use the same page size as the 1-1 mappings. In that
  15  * case the overhead consists of a few additional pages that are
  16  * allocated to create a view of memory for vmemmap.
  17  *
  18  * The architecture is expected to provide a vmemmap_populate() function
  19  * to instantiate the mapping.
  20  */
  21 #include <linux/mm.h>
  22 #include <linux/mmzone.h>
  23 #include <linux/memblock.h>
  24 #include <linux/memremap.h>
  25 #include <linux/highmem.h>
  26 #include <linux/slab.h>
  27 #include <linux/spinlock.h>
  28 #include <linux/vmalloc.h>
  29 #include <linux/sched.h>
  30 #include <linux/pgtable.h>
  31 #include <linux/bootmem_info.h>
  32
  33 #include <asm/dma.h>
  34 #include <asm/pgalloc.h>
  35 #include <asm/tlbflush.h>
  36
  37 /**
  38  * struct vmemmap_remap_walk - walk vmemmap page table
  39  *
  40  * @remap_pte:          called for each lowest-level entry (PTE).
  41  * @reuse_page:         the page which is reused for the tail vmemmap pages.
  42  * @reuse_addr:         the virtual address of the @reuse_page page.
  43  * @vmemmap_pages:      the list head of the vmemmap pages that can be freed
  44  *                      or is mapped from.
  45  */
  46 struct vmemmap_remap_walk {
  47         void (*remap_pte)(pte_t *pte, unsigned long addr,
  48                           struct vmemmap_remap_walk *walk);
  49         struct page *reuse_page;
  50         unsigned long reuse_addr;
  51         struct list_head *vmemmap_pages;
  52 };
  53
  54 static void vmemmap_pte_range(pmd_t *pmd, unsigned long addr,
  55                               unsigned long end,
  56                               struct vmemmap_remap_walk *walk)
  57 {
  58         pte_t *pte = pte_offset_kernel(pmd, addr);
  59
  60         /*
  61          * The reuse_page is found 'first' in table walk before we start
  62          * remapping (which is calling @walk->remap_pte).
  63          */
  64         if (!walk->reuse_page) {
  65                 walk->reuse_page = pte_page(*pte);
  66                 /*
  67                  * Because the reuse address is part of the range that we are
  68                  * walking, skip the reuse address range.
  69                  */
  70                 addr += PAGE_SIZE;
  71                 pte++;
  72         }
  73
  74         for (; addr != end; addr += PAGE_SIZE, pte++)
  75                 walk->remap_pte(pte, addr, walk);
  76 }
  77
  78 static void vmemmap_pmd_range(pud_t *pud, unsigned long addr,
  79                               unsigned long end,
  80                               struct vmemmap_remap_walk *walk)
  81 {
  82         pmd_t *pmd;
  83         unsigned long next;
  84
  85         pmd = pmd_offset(pud, addr);
  86         do {
  87                 BUG_ON(pmd_leaf(*pmd));
  88
  89                 next = pmd_addr_end(addr, end);
  90                 vmemmap_pte_range(pmd, addr, next, walk);
  91         } while (pmd++, addr = next, addr != end);
  92 }
  93
  94 static void vmemmap_pud_range(p4d_t *p4d, unsigned long addr,
  95                               unsigned long end,
  96                               struct vmemmap_remap_walk *walk)
  97 {
  98         pud_t *pud;
  99         unsigned long next;
 100
 101         pud = pud_offset(p4d, addr);
 102         do {
 103                 next = pud_addr_end(addr, end);
 104                 vmemmap_pmd_range(pud, addr, next, walk);
 105         } while (pud++, addr = next, addr != end);
 106 }
 107
 108 static void vmemmap_p4d_range(pgd_t *pgd, unsigned long addr,
 109                               unsigned long end,
 110                               struct vmemmap_remap_walk *walk)
 111 {
 112         p4d_t *p4d;
 113         unsigned long next;
 114
 115         p4d = p4d_offset(pgd, addr);
 116         do {
 117                 next = p4d_addr_end(addr, end);
 118                 vmemmap_pud_range(p4d, addr, next, walk);
 119         } while (p4d++, addr = next, addr != end);
 120 }
 121
 122 static void vmemmap_remap_range(unsigned long start, unsigned long end,
 123                                 struct vmemmap_remap_walk *walk)
 124 {
 125         unsigned long addr = start;
 126         unsigned long next;
 127         pgd_t *pgd;
 128
 129         VM_BUG_ON(!IS_ALIGNED(start, PAGE_SIZE));
 130         VM_BUG_ON(!IS_ALIGNED(end, PAGE_SIZE));
 131
 132         pgd = pgd_offset_k(addr);
 133         do {
 134                 next = pgd_addr_end(addr, end);
 135                 vmemmap_p4d_range(pgd, addr, next, walk);
 136         } while (pgd++, addr = next, addr != end);
 137
 138         /*
 139          * We only change the mapping of the vmemmap virtual address range
 140          * [@start + PAGE_SIZE, end), so we only need to flush the TLB which
 141          * belongs to the range.
 142          */
 143         flush_tlb_kernel_range(start + PAGE_SIZE, end);
 144 }
 145
 146 /*
 147  * Free a vmemmap page. A vmemmap page can be allocated from the memblock
 148  * allocator or buddy allocator. If the PG_reserved flag is set, it means
 149  * that it allocated from the memblock allocator, just free it via the
 150  * free_bootmem_page(). Otherwise, use __free_page().
 151  */
 152 static inline void free_vmemmap_page(struct page *page)
 153 {
 154         if (PageReserved(page))
 155                 free_bootmem_page(page);
 156         else
 157                 __free_page(page);
 158 }
 159
 160 /* Free a list of the vmemmap pages */
 161 static void free_vmemmap_page_list(struct list_head *list)
 162 {
 163         struct page *page, *next;
 164
 165         list_for_each_entry_safe(page, next, list, lru) {
 166                 list_del(&page->lru);
 167                 free_vmemmap_page(page);
 168         }
 169 }
 170
 171 static void vmemmap_remap_pte(pte_t *pte, unsigned long addr,
 172                               struct vmemmap_remap_walk *walk)
 173 {
 174         /*
 175          * Remap the tail pages as read-only to catch illegal write operation
 176          * to the tail pages.
 177          */
 178         pgprot_t pgprot = PAGE_KERNEL_RO;
 179         pte_t entry = mk_pte(walk->reuse_page, pgprot);
 180         struct page *page = pte_page(*pte);
 181
 182         list_add(&page->lru, walk->vmemmap_pages);
 183         set_pte_at(&init_mm, addr, pte, entry);
 184 }
 185
 186 /**
 187  * vmemmap_remap_free - remap the vmemmap virtual address range [@start, @end)
 188  *                      to the page which @reuse is mapped to, then free vmemmap
 189  *                      which the range are mapped to.
 190  * @start:      start address of the vmemmap virtual address range that we want
 191  *              to remap.
 192  * @end:        end address of the vmemmap virtual address range that we want to
 193  *              remap.
 194  * @reuse:      reuse address.
 195  *
 196  * Note: This function depends on vmemmap being base page mapped. Please make
 197  * sure that we disable PMD mapping of vmemmap pages when calling this function.
 198  */
 199 void vmemmap_remap_free(unsigned long start, unsigned long end,
 200                         unsigned long reuse)
 201 {
 202         LIST_HEAD(vmemmap_pages);
 203         struct vmemmap_remap_walk walk = {
 204                 .remap_pte      = vmemmap_remap_pte,
 205                 .reuse_addr     = reuse,
 206                 .vmemmap_pages  = &vmemmap_pages,
 207         };
 208
 209         /*
 210          * In order to make remapping routine most efficient for the huge pages,
 211          * the routine of vmemmap page table walking has the following rules
 212          * (see more details from the vmemmap_pte_range()):
 213          *
 214          * - The range [@start, @end) and the range [@reuse, @reuse + PAGE_SIZE)
 215          *   should be continuous.
 216          * - The @reuse address is part of the range [@reuse, @end) that we are
 217          *   walking which is passed to vmemmap_remap_range().
 218          * - The @reuse address is the first in the complete range.
 219          *
 220          * So we need to make sure that @start and @reuse meet the above rules.
 221          */
 222         BUG_ON(start - reuse != PAGE_SIZE);
 223
 224         vmemmap_remap_range(reuse, end, &walk);
 225         free_vmemmap_page_list(&vmemmap_pages);
 226 }
 227
 228 static void vmemmap_restore_pte(pte_t *pte, unsigned long addr,
 229                                 struct vmemmap_remap_walk *walk)
 230 {
 231         pgprot_t pgprot = PAGE_KERNEL;
 232         struct page *page;
 233         void *to;
 234
 235         BUG_ON(pte_page(*pte) != walk->reuse_page);
 236
 237         page = list_first_entry(walk->vmemmap_pages, struct page, lru);
 238         list_del(&page->lru);
 239         to = page_to_virt(page);
 240         copy_page(to, (void *)walk->reuse_addr);
 241
 242         set_pte_at(&init_mm, addr, pte, mk_pte(page, pgprot));
 243 }
 244
 245 static int alloc_vmemmap_page_list(unsigned long start, unsigned long end,
 246                                    gfp_t gfp_mask, struct list_head *list)
 247 {
 248         unsigned long nr_pages = (end - start) >> PAGE_SHIFT;
 249         int nid = page_to_nid((struct page *)start);
 250         struct page *page, *next;
 251
 252         while (nr_pages--) {
 253                 page = alloc_pages_node(nid, gfp_mask, 0);
 254                 if (!page)
 255                         goto out;
 256                 list_add_tail(&page->lru, list);
 257         }
 258
 259         return 0;
 260 out:
 261         list_for_each_entry_safe(page, next, list, lru)
 262                 __free_pages(page, 0);
 263         return -ENOMEM;
 264 }
 265
 266 /**
 267  * vmemmap_remap_alloc - remap the vmemmap virtual address range [@start, end)
 268  *                       to the page which is from the @vmemmap_pages
 269  *                       respectively.
 270  * @start:      start address of the vmemmap virtual address range that we want
 271  *              to remap.
 272  * @end:        end address of the vmemmap virtual address range that we want to
 273  *              remap.
 274  * @reuse:      reuse address.
 275  * @gfp_mask:   GFP flag for allocating vmemmap pages.
 276  */
 277 int vmemmap_remap_alloc(unsigned long start, unsigned long end,
 278                         unsigned long reuse, gfp_t gfp_mask)
 279 {
 280         LIST_HEAD(vmemmap_pages);
 281         struct vmemmap_remap_walk walk = {
 282                 .remap_pte      = vmemmap_restore_pte,
 283                 .reuse_addr     = reuse,
 284                 .vmemmap_pages  = &vmemmap_pages,
 285         };
 286
 287         /* See the comment in the vmemmap_remap_free(). */
 288         BUG_ON(start - reuse != PAGE_SIZE);
 289
 290         might_sleep_if(gfpflags_allow_blocking(gfp_mask));
 291
 292         if (alloc_vmemmap_page_list(start, end, gfp_mask, &vmemmap_pages))
 293                 return -ENOMEM;
 294
 295         vmemmap_remap_range(reuse, end, &walk);
 296
 297         return 0;
 298 }
 299
 300 /*
 301  * Allocate a block of memory to be used to back the virtual memory map
 302  * or to back the page tables that are used to create the mapping.
 303  * Uses the main allocators if they are available, else bootmem.
 304  */
 305
 306 static void * __ref __earlyonly_bootmem_alloc(int node,
 307                                 unsigned long size,
 308                                 unsigned long align,
 309                                 unsigned long goal)
 310 {
 311         return memblock_alloc_try_nid_raw(size, align, goal,
 312                                                MEMBLOCK_ALLOC_ACCESSIBLE, node);
 313 }
 314
 315 void * __meminit vmemmap_alloc_block(unsigned long size, int node)
 316 {
 317         /* If the main allocator is up use that, fallback to bootmem. */
 318         if (slab_is_available()) {
 319                 gfp_t gfp_mask = GFP_KERNEL|__GFP_RETRY_MAYFAIL|__GFP_NOWARN;
 320                 int order = get_order(size);
 321                 static bool warned;
 322                 struct page *page;
 323
 324                 page = alloc_pages_node(node, gfp_mask, order);
 325                 if (page)
 326                         return page_address(page);
 327
 328                 if (!warned) {
 329                         warn_alloc(gfp_mask & ~__GFP_NOWARN, NULL,
 330                                    "vmemmap alloc failure: order:%u", order);
 331                         warned = true;
 332                 }
 333                 return NULL;
 334         } else
 335                 return __earlyonly_bootmem_alloc(node, size, size,
 336                                 __pa(MAX_DMA_ADDRESS));
 337 }
 338
 339 static void * __meminit altmap_alloc_block_buf(unsigned long size,
 340                                                struct vmem_altmap *altmap);
 341
 342 /* need to make sure size is all the same during early stage */
 343 void * __meminit vmemmap_alloc_block_buf(unsigned long size, int node,
 344                                          struct vmem_altmap *altmap)
 345 {
 346         void *ptr;
 347
 348         if (altmap)
 349                 return altmap_alloc_block_buf(size, altmap);
 350
 351         ptr = sparse_buffer_alloc(size);
 352         if (!ptr)
 353                 ptr = vmemmap_alloc_block(size, node);
 354         return ptr;
 355 }
 356
 357 static unsigned long __meminit vmem_altmap_next_pfn(struct vmem_altmap *altmap)
 358 {
 359         return altmap->base_pfn + altmap->reserve + altmap->alloc
 360                 + altmap->align;
 361 }
 362
 363 static unsigned long __meminit vmem_altmap_nr_free(struct vmem_altmap *altmap)
 364 {
 365         unsigned long allocated = altmap->alloc + altmap->align;
 366
 367         if (altmap->free > allocated)
 368                 return altmap->free - allocated;
 369         return 0;
 370 }
 371
 372 static void * __meminit altmap_alloc_block_buf(unsigned long size,
 373                                                struct vmem_altmap *altmap)
 374 {
 375         unsigned long pfn, nr_pfns, nr_align;
 376
 377         if (size & ~PAGE_MASK) {
 378                 pr_warn_once("%s: allocations must be multiple of PAGE_SIZE (%ld)\n",
 379                                 __func__, size);
 380                 return NULL;
 381         }
 382
 383         pfn = vmem_altmap_next_pfn(altmap);
 384         nr_pfns = size >> PAGE_SHIFT;
 385         nr_align = 1UL << find_first_bit(&nr_pfns, BITS_PER_LONG);
 386         nr_align = ALIGN(pfn, nr_align) - pfn;
 387         if (nr_pfns + nr_align > vmem_altmap_nr_free(altmap))
 388                 return NULL;
 389
 390         altmap->alloc += nr_pfns;
 391         altmap->align += nr_align;
 392         pfn += nr_align;
 393
 394         pr_debug("%s: pfn: %#lx alloc: %ld align: %ld nr: %#lx\n",
 395                         __func__, pfn, altmap->alloc, altmap->align, nr_pfns);
 396         return __va(__pfn_to_phys(pfn));
 397 }
 398
 399 void __meminit vmemmap_verify(pte_t *pte, int node,
 400                                 unsigned long start, unsigned long end)
 401 {
 402         unsigned long pfn = pte_pfn(*pte);
 403         int actual_node = early_pfn_to_nid(pfn);
 404
 405         if (node_distance(actual_node, node) > LOCAL_DISTANCE)
 406                 pr_warn("[%lx-%lx] potential offnode page_structs\n",
 407                         start, end - 1);
 408 }
 409
 410 pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node,
 411                                        struct vmem_altmap *altmap)
 412 {
 413         pte_t *pte = pte_offset_kernel(pmd, addr);
 414         if (pte_none(*pte)) {
 415                 pte_t entry;
 416                 void *p;
 417
 418                 p = vmemmap_alloc_block_buf(PAGE_SIZE, node, altmap);
 419                 if (!p)
 420                         return NULL;
 421                 entry = pfn_pte(__pa(p) >> PAGE_SHIFT, PAGE_KERNEL);
 422                 set_pte_at(&init_mm, addr, pte, entry);
 423         }
 424         return pte;
 425 }
 426
 427 static void * __meminit vmemmap_alloc_block_zero(unsigned long size, int node)
 428 {
 429         void *p = vmemmap_alloc_block(size, node);
 430
 431         if (!p)
 432                 return NULL;
 433         memset(p, 0, size);
 434
 435         return p;
 436 }
 437
 438 pmd_t * __meminit vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node)
 439 {
 440         pmd_t *pmd = pmd_offset(pud, addr);
 441         if (pmd_none(*pmd)) {
 442                 void *p = vmemmap_alloc_block_zero(PAGE_SIZE, node);
 443                 if (!p)
 444                         return NULL;
 445                 pmd_populate_kernel(&init_mm, pmd, p);
 446         }
 447         return pmd;
 448 }
 449
 450 pud_t * __meminit vmemmap_pud_populate(p4d_t *p4d, unsigned long addr, int node)
 451 {
 452         pud_t *pud = pud_offset(p4d, addr);
 453         if (pud_none(*pud)) {
 454                 void *p = vmemmap_alloc_block_zero(PAGE_SIZE, node);
 455                 if (!p)
 456                         return NULL;
 457                 pud_populate(&init_mm, pud, p);
 458         }
 459         return pud;
 460 }
 461
 462 p4d_t * __meminit vmemmap_p4d_populate(pgd_t *pgd, unsigned long addr, int node)
 463 {
 464         p4d_t *p4d = p4d_offset(pgd, addr);
 465         if (p4d_none(*p4d)) {
 466                 void *p = vmemmap_alloc_block_zero(PAGE_SIZE, node);
 467                 if (!p)
 468                         return NULL;
 469                 p4d_populate(&init_mm, p4d, p);
 470         }
 471         return p4d;
 472 }
 473
 474 pgd_t * __meminit vmemmap_pgd_populate(unsigned long addr, int node)
 475 {
 476         pgd_t *pgd = pgd_offset_k(addr);
 477         if (pgd_none(*pgd)) {
 478                 void *p = vmemmap_alloc_block_zero(PAGE_SIZE, node);
 479                 if (!p)
 480                         return NULL;
 481                 pgd_populate(&init_mm, pgd, p);
 482         }
 483         return pgd;
 484 }
 485
 486 int __meminit vmemmap_populate_basepages(unsigned long start, unsigned long end,
 487                                          int node, struct vmem_altmap *altmap)
 488 {
 489         unsigned long addr = start;
 490         pgd_t *pgd;
 491         p4d_t *p4d;
 492         pud_t *pud;
 493         pmd_t *pmd;
 494         pte_t *pte;
 495
 496         for (; addr < end; addr += PAGE_SIZE) {
 497                 pgd = vmemmap_pgd_populate(addr, node);
 498                 if (!pgd)
 499                         return -ENOMEM;
 500                 p4d = vmemmap_p4d_populate(pgd, addr, node);
 501                 if (!p4d)
 502                         return -ENOMEM;
 503                 pud = vmemmap_pud_populate(p4d, addr, node);
 504                 if (!pud)
 505                         return -ENOMEM;
 506                 pmd = vmemmap_pmd_populate(pud, addr, node);
 507                 if (!pmd)
 508                         return -ENOMEM;
 509                 pte = vmemmap_pte_populate(pmd, addr, node, altmap);
 510                 if (!pte)
 511                         return -ENOMEM;
 512                 vmemmap_verify(pte, node, addr, addr + PAGE_SIZE);
 513         }
 514
 515         return 0;
 516 }
 517
 518 struct page * __meminit __populate_section_memmap(unsigned long pfn,
 519                 unsigned long nr_pages, int nid, struct vmem_altmap *altmap)
 520 {
 521         unsigned long start = (unsigned long) pfn_to_page(pfn);
 522         unsigned long end = start + nr_pages * sizeof(struct page);
 523
 524         if (WARN_ON_ONCE(!IS_ALIGNED(pfn, PAGES_PER_SUBSECTION) ||
 525                 !IS_ALIGNED(nr_pages, PAGES_PER_SUBSECTION)))
 526                 return NULL;
 527
 528         if (vmemmap_populate(start, end, nid, altmap))
 529                 return NULL;
 530
 531         return pfn_to_page(pfn);
 532 }