arch/x86/platform/efi/efi.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Common EFI (Extensible Firmware Interface) support functions
   4  * Based on Extensible Firmware Interface Specification version 1.0
   5  *
   6  * Copyright (C) 1999 VA Linux Systems
   7  * Copyright (C) 1999 Walt Drummond <drummond@valinux.com>
   8  * Copyright (C) 1999-2002 Hewlett-Packard Co.
   9  *      David Mosberger-Tang <davidm@hpl.hp.com>
  10  *      Stephane Eranian <eranian@hpl.hp.com>
  11  * Copyright (C) 2005-2008 Intel Co.
  12  *      Fenghua Yu <fenghua.yu@intel.com>
  13  *      Bibo Mao <bibo.mao@intel.com>
  14  *      Chandramouli Narayanan <mouli@linux.intel.com>
  15  *      Huang Ying <ying.huang@intel.com>
  16  * Copyright (C) 2013 SuSE Labs
  17  *      Borislav Petkov <bp@suse.de> - runtime services VA mapping
  18  *
  19  * Copied from efi_32.c to eliminate the duplicated code between EFI
  20  * 32/64 support code. --ying 2007-10-26
  21  *
  22  * All EFI Runtime Services are not implemented yet as EFI only
  23  * supports physical mode addressing on SoftSDV. This is to be fixed
  24  * in a future version.  --drummond 1999-07-20
  25  *
  26  * Implemented EFI runtime services and virtual mode calls.  --davidm
  27  *
  28  * Goutham Rao: <goutham.rao@intel.com>
  29  *      Skip non-WB memory and ignore empty memory ranges.
  30  */
  31
  32 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  33
  34 #include <linux/kernel.h>
  35 #include <linux/init.h>
  36 #include <linux/efi.h>
  37 #include <linux/efi-bgrt.h>
  38 #include <linux/export.h>
  39 #include <linux/memblock.h>
  40 #include <linux/slab.h>
  41 #include <linux/spinlock.h>
  42 #include <linux/uaccess.h>
  43 #include <linux/time.h>
  44 #include <linux/io.h>
  45 #include <linux/reboot.h>
  46 #include <linux/bcd.h>
  47
  48 #include <asm/setup.h>
  49 #include <asm/efi.h>
  50 #include <asm/e820/api.h>
  51 #include <asm/time.h>
  52 #include <asm/set_memory.h>
  53 #include <asm/tlbflush.h>
  54 #include <asm/x86_init.h>
  55 #include <asm/uv/uv.h>
  56
  57 static efi_system_table_t efi_systab __initdata;
  58 static u64 efi_systab_phys __initdata;
  59
  60 static unsigned long prop_phys = EFI_INVALID_TABLE_ADDR;
  61 static unsigned long uga_phys = EFI_INVALID_TABLE_ADDR;
  62
  63 static efi_config_table_type_t arch_tables[] __initdata = {
  64         {EFI_PROPERTIES_TABLE_GUID, "PROP", &prop_phys},
  65         {UGA_IO_PROTOCOL_GUID, "UGA", &uga_phys},
  66 #ifdef CONFIG_X86_UV
  67         {UV_SYSTEM_TABLE_GUID, "UVsystab", &uv_systab_phys},
  68 #endif
  69         {NULL_GUID, NULL, NULL},
  70 };
  71
  72 static const unsigned long * const efi_tables[] = {
  73         &efi.acpi,
  74         &efi.acpi20,
  75         &efi.smbios,
  76         &efi.smbios3,
  77         &uga_phys,
  78 #ifdef CONFIG_X86_UV
  79         &uv_systab_phys,
  80 #endif
  81         &efi.fw_vendor,
  82         &efi.runtime,
  83         &efi.config_table,
  84         &efi.esrt,
  85         &prop_phys,
  86         &efi_mem_attr_table,
  87 #ifdef CONFIG_EFI_RCI2_TABLE
  88         &rci2_table_phys,
  89 #endif
  90 };
  91
  92 u64 efi_setup;          /* efi setup_data physical address */
  93
  94 static int add_efi_memmap __initdata;
  95 static int __init setup_add_efi_memmap(char *arg)
  96 {
  97         add_efi_memmap = 1;
  98         return 0;
  99 }
 100 early_param("add_efi_memmap", setup_add_efi_memmap);
 101
 102 void __init efi_find_mirror(void)
 103 {
 104         efi_memory_desc_t *md;
 105         u64 mirror_size = 0, total_size = 0;
 106
 107         if (!efi_enabled(EFI_MEMMAP))
 108                 return;
 109
 110         for_each_efi_memory_desc(md) {
 111                 unsigned long long start = md->phys_addr;
 112                 unsigned long long size = md->num_pages << EFI_PAGE_SHIFT;
 113
 114                 total_size += size;
 115                 if (md->attribute & EFI_MEMORY_MORE_RELIABLE) {
 116                         memblock_mark_mirror(start, size);
 117                         mirror_size += size;
 118                 }
 119         }
 120         if (mirror_size)
 121                 pr_info("Memory: %lldM/%lldM mirrored memory\n",
 122                         mirror_size>>20, total_size>>20);
 123 }
 124
 125 /*
 126  * Tell the kernel about the EFI memory map.  This might include
 127  * more than the max 128 entries that can fit in the passed in e820
 128  * legacy (zeropage) memory map, but the kernel's e820 table can hold
 129  * E820_MAX_ENTRIES.
 130  */
 131
 132 static void __init do_add_efi_memmap(void)
 133 {
 134         efi_memory_desc_t *md;
 135
 136         if (!efi_enabled(EFI_MEMMAP))
 137                 return;
 138
 139         for_each_efi_memory_desc(md) {
 140                 unsigned long long start = md->phys_addr;
 141                 unsigned long long size = md->num_pages << EFI_PAGE_SHIFT;
 142                 int e820_type;
 143
 144                 switch (md->type) {
 145                 case EFI_LOADER_CODE:
 146                 case EFI_LOADER_DATA:
 147                 case EFI_BOOT_SERVICES_CODE:
 148                 case EFI_BOOT_SERVICES_DATA:
 149                 case EFI_CONVENTIONAL_MEMORY:
 150                         if (efi_soft_reserve_enabled()
 151                             && (md->attribute & EFI_MEMORY_SP))
 152                                 e820_type = E820_TYPE_SOFT_RESERVED;
 153                         else if (md->attribute & EFI_MEMORY_WB)
 154                                 e820_type = E820_TYPE_RAM;
 155                         else
 156                                 e820_type = E820_TYPE_RESERVED;
 157                         break;
 158                 case EFI_ACPI_RECLAIM_MEMORY:
 159                         e820_type = E820_TYPE_ACPI;
 160                         break;
 161                 case EFI_ACPI_MEMORY_NVS:
 162                         e820_type = E820_TYPE_NVS;
 163                         break;
 164                 case EFI_UNUSABLE_MEMORY:
 165                         e820_type = E820_TYPE_UNUSABLE;
 166                         break;
 167                 case EFI_PERSISTENT_MEMORY:
 168                         e820_type = E820_TYPE_PMEM;
 169                         break;
 170                 default:
 171                         /*
 172                          * EFI_RESERVED_TYPE EFI_RUNTIME_SERVICES_CODE
 173                          * EFI_RUNTIME_SERVICES_DATA EFI_MEMORY_MAPPED_IO
 174                          * EFI_MEMORY_MAPPED_IO_PORT_SPACE EFI_PAL_CODE
 175                          */
 176                         e820_type = E820_TYPE_RESERVED;
 177                         break;
 178                 }
 179
 180                 e820__range_add(start, size, e820_type);
 181         }
 182         e820__update_table(e820_table);
 183 }
 184
 185 /*
 186  * Given add_efi_memmap defaults to 0 and there there is no alternative
 187  * e820 mechanism for soft-reserved memory, import the full EFI memory
 188  * map if soft reservations are present and enabled. Otherwise, the
 189  * mechanism to disable the kernel's consideration of EFI_MEMORY_SP is
 190  * the efi=nosoftreserve option.
 191  */
 192 static bool do_efi_soft_reserve(void)
 193 {
 194         efi_memory_desc_t *md;
 195
 196         if (!efi_enabled(EFI_MEMMAP))
 197                 return false;
 198
 199         if (!efi_soft_reserve_enabled())
 200                 return false;
 201
 202         for_each_efi_memory_desc(md)
 203                 if (md->type == EFI_CONVENTIONAL_MEMORY &&
 204                     (md->attribute & EFI_MEMORY_SP))
 205                         return true;
 206         return false;
 207 }
 208
 209 int __init efi_memblock_x86_reserve_range(void)
 210 {
 211         struct efi_info *e = &boot_params.efi_info;
 212         struct efi_memory_map_data data;
 213         phys_addr_t pmap;
 214         int rv;
 215
 216         if (efi_enabled(EFI_PARAVIRT))
 217                 return 0;
 218
 219         /* Can't handle firmware tables above 4GB on i386 */
 220         if (IS_ENABLED(CONFIG_X86_32) && e->efi_memmap_hi > 0) {
 221                 pr_err("Memory map is above 4GB, disabling EFI.\n");
 222                 return -EINVAL;
 223         }
 224         pmap = (phys_addr_t)(e->efi_memmap | ((u64)e->efi_memmap_hi << 32));
 225
 226         data.phys_map           = pmap;
 227         data.size               = e->efi_memmap_size;
 228         data.desc_size          = e->efi_memdesc_size;
 229         data.desc_version       = e->efi_memdesc_version;
 230
 231         rv = efi_memmap_init_early(&data);
 232         if (rv)
 233                 return rv;
 234
 235         if (add_efi_memmap || do_efi_soft_reserve())
 236                 do_add_efi_memmap();
 237
 238         efi_fake_memmap_early();
 239
 240         WARN(efi.memmap.desc_version != 1,
 241              "Unexpected EFI_MEMORY_DESCRIPTOR version %ld",
 242              efi.memmap.desc_version);
 243
 244         memblock_reserve(pmap, efi.memmap.nr_map * efi.memmap.desc_size);
 245
 246         return 0;
 247 }
 248
 249 #define OVERFLOW_ADDR_SHIFT     (64 - EFI_PAGE_SHIFT)
 250 #define OVERFLOW_ADDR_MASK      (U64_MAX << OVERFLOW_ADDR_SHIFT)
 251 #define U64_HIGH_BIT            (~(U64_MAX >> 1))
 252
 253 static bool __init efi_memmap_entry_valid(const efi_memory_desc_t *md, int i)
 254 {
 255         u64 end = (md->num_pages << EFI_PAGE_SHIFT) + md->phys_addr - 1;
 256         u64 end_hi = 0;
 257         char buf[64];
 258
 259         if (md->num_pages == 0) {
 260                 end = 0;
 261         } else if (md->num_pages > EFI_PAGES_MAX ||
 262                    EFI_PAGES_MAX - md->num_pages <
 263                    (md->phys_addr >> EFI_PAGE_SHIFT)) {
 264                 end_hi = (md->num_pages & OVERFLOW_ADDR_MASK)
 265                         >> OVERFLOW_ADDR_SHIFT;
 266
 267                 if ((md->phys_addr & U64_HIGH_BIT) && !(end & U64_HIGH_BIT))
 268                         end_hi += 1;
 269         } else {
 270                 return true;
 271         }
 272
 273         pr_warn_once(FW_BUG "Invalid EFI memory map entries:\n");
 274
 275         if (end_hi) {
 276                 pr_warn("mem%02u: %s range=[0x%016llx-0x%llx%016llx] (invalid)\n",
 277                         i, efi_md_typeattr_format(buf, sizeof(buf), md),
 278                         md->phys_addr, end_hi, end);
 279         } else {
 280                 pr_warn("mem%02u: %s range=[0x%016llx-0x%016llx] (invalid)\n",
 281                         i, efi_md_typeattr_format(buf, sizeof(buf), md),
 282                         md->phys_addr, end);
 283         }
 284         return false;
 285 }
 286
 287 static void __init efi_clean_memmap(void)
 288 {
 289         efi_memory_desc_t *out = efi.memmap.map;
 290         const efi_memory_desc_t *in = out;
 291         const efi_memory_desc_t *end = efi.memmap.map_end;
 292         int i, n_removal;
 293
 294         for (i = n_removal = 0; in < end; i++) {
 295                 if (efi_memmap_entry_valid(in, i)) {
 296                         if (out != in)
 297                                 memcpy(out, in, efi.memmap.desc_size);
 298                         out = (void *)out + efi.memmap.desc_size;
 299                 } else {
 300                         n_removal++;
 301                 }
 302                 in = (void *)in + efi.memmap.desc_size;
 303         }
 304
 305         if (n_removal > 0) {
 306                 struct efi_memory_map_data data = {
 307                         .phys_map       = efi.memmap.phys_map,
 308                         .desc_version   = efi.memmap.desc_version,
 309                         .desc_size      = efi.memmap.desc_size,
 310                         .size           = efi.memmap.desc_size * (efi.memmap.nr_map - n_removal),
 311                         .flags          = 0,
 312                 };
 313
 314                 pr_warn("Removing %d invalid memory map entries.\n", n_removal);
 315                 efi_memmap_install(&data);
 316         }
 317 }
 318
 319 void __init efi_print_memmap(void)
 320 {
 321         efi_memory_desc_t *md;
 322         int i = 0;
 323
 324         for_each_efi_memory_desc(md) {
 325                 char buf[64];
 326
 327                 pr_info("mem%02u: %s range=[0x%016llx-0x%016llx] (%lluMB)\n",
 328                         i++, efi_md_typeattr_format(buf, sizeof(buf), md),
 329                         md->phys_addr,
 330                         md->phys_addr + (md->num_pages << EFI_PAGE_SHIFT) - 1,
 331                         (md->num_pages >> (20 - EFI_PAGE_SHIFT)));
 332         }
 333 }
 334
 335 static int __init efi_systab_init(u64 phys)
 336 {
 337         int size = efi_enabled(EFI_64BIT) ? sizeof(efi_system_table_64_t)
 338                                           : sizeof(efi_system_table_32_t);
 339         const efi_table_hdr_t *hdr;
 340         bool over4g = false;
 341         void *p;
 342         int ret;
 343
 344         hdr = p = early_memremap_ro(phys, size);
 345         if (p == NULL) {
 346                 pr_err("Couldn't map the system table!\n");
 347                 return -ENOMEM;
 348         }
 349
 350         ret = efi_systab_check_header(hdr, 1);
 351         if (ret) {
 352                 early_memunmap(p, size);
 353                 return ret;
 354         }
 355
 356         if (efi_enabled(EFI_64BIT)) {
 357                 const efi_system_table_64_t *systab64 = p;
 358
 359                 efi_systab.hdr                  = systab64->hdr;
 360                 efi_systab.fw_vendor            = systab64->fw_vendor;
 361                 efi_systab.fw_revision          = systab64->fw_revision;
 362                 efi_systab.con_in_handle        = systab64->con_in_handle;
 363                 efi_systab.con_in               = systab64->con_in;
 364                 efi_systab.con_out_handle       = systab64->con_out_handle;
 365                 efi_systab.con_out              = (void *)(unsigned long)systab64->con_out;
 366                 efi_systab.stderr_handle        = systab64->stderr_handle;
 367                 efi_systab.stderr               = systab64->stderr;
 368                 efi_systab.runtime              = (void *)(unsigned long)systab64->runtime;
 369                 efi_systab.boottime             = (void *)(unsigned long)systab64->boottime;
 370                 efi_systab.nr_tables            = systab64->nr_tables;
 371                 efi_systab.tables               = systab64->tables;
 372
 373                 over4g = systab64->con_in_handle        > U32_MAX ||
 374                          systab64->con_in               > U32_MAX ||
 375                          systab64->con_out_handle       > U32_MAX ||
 376                          systab64->con_out              > U32_MAX ||
 377                          systab64->stderr_handle        > U32_MAX ||
 378                          systab64->stderr               > U32_MAX ||
 379                          systab64->boottime             > U32_MAX;
 380
 381                 if (efi_setup) {
 382                         struct efi_setup_data *data;
 383
 384                         data = early_memremap_ro(efi_setup, sizeof(*data));
 385                         if (!data) {
 386                                 early_memunmap(p, size);
 387                                 return -ENOMEM;
 388                         }
 389
 390                         efi_systab.fw_vendor    = (unsigned long)data->fw_vendor;
 391                         efi_systab.runtime      = (void *)(unsigned long)data->runtime;
 392                         efi_systab.tables       = (unsigned long)data->tables;
 393
 394                         over4g |= data->fw_vendor       > U32_MAX ||
 395                                   data->runtime         > U32_MAX ||
 396                                   data->tables          > U32_MAX;
 397
 398                         early_memunmap(data, sizeof(*data));
 399                 } else {
 400                         over4g |= systab64->fw_vendor   > U32_MAX ||
 401                                   systab64->runtime     > U32_MAX ||
 402                                   systab64->tables      > U32_MAX;
 403                 }
 404         } else {
 405                 const efi_system_table_32_t *systab32 = p;
 406
 407                 efi_systab.hdr                  = systab32->hdr;
 408                 efi_systab.fw_vendor            = systab32->fw_vendor;
 409                 efi_systab.fw_revision          = systab32->fw_revision;
 410                 efi_systab.con_in_handle        = systab32->con_in_handle;
 411                 efi_systab.con_in               = systab32->con_in;
 412                 efi_systab.con_out_handle       = systab32->con_out_handle;
 413                 efi_systab.con_out              = (void *)(unsigned long)systab32->con_out;
 414                 efi_systab.stderr_handle        = systab32->stderr_handle;
 415                 efi_systab.stderr               = systab32->stderr;
 416                 efi_systab.runtime              = (void *)(unsigned long)systab32->runtime;
 417                 efi_systab.boottime             = (void *)(unsigned long)systab32->boottime;
 418                 efi_systab.nr_tables            = systab32->nr_tables;
 419                 efi_systab.tables               = systab32->tables;
 420         }
 421
 422         efi_systab_report_header(hdr, efi_systab.fw_vendor);
 423         early_memunmap(p, size);
 424
 425         if (IS_ENABLED(CONFIG_X86_32) && over4g) {
 426                 pr_err("EFI data located above 4GB, disabling EFI.\n");
 427                 return -EINVAL;
 428         }
 429
 430         efi.systab = &efi_systab;
 431         return 0;
 432 }
 433
 434 void __init efi_init(void)
 435 {
 436         if (IS_ENABLED(CONFIG_X86_32) &&
 437             (boot_params.efi_info.efi_systab_hi ||
 438              boot_params.efi_info.efi_memmap_hi)) {
 439                 pr_info("Table located above 4GB, disabling EFI.\n");
 440                 return;
 441         }
 442
 443         efi_systab_phys = boot_params.efi_info.efi_systab |
 444                           ((__u64)boot_params.efi_info.efi_systab_hi << 32);
 445
 446         if (efi_systab_init(efi_systab_phys))
 447                 return;
 448
 449         efi.config_table = (unsigned long)efi.systab->tables;
 450         efi.fw_vendor    = (unsigned long)efi.systab->fw_vendor;
 451         efi.runtime      = (unsigned long)efi.systab->runtime;
 452
 453         if (efi_reuse_config(efi.systab->tables, efi.systab->nr_tables))
 454                 return;
 455
 456         if (efi_config_init(arch_tables))
 457                 return;
 458
 459         /*
 460          * Note: We currently don't support runtime services on an EFI
 461          * that doesn't match the kernel 32/64-bit mode.
 462          */
 463
 464         if (!efi_runtime_supported())
 465                 pr_info("No EFI runtime due to 32/64-bit mismatch with kernel\n");
 466
 467         if (!efi_runtime_supported() || efi_runtime_disabled()) {
 468                 efi_memmap_unmap();
 469                 return;
 470         }
 471
 472         /* Parse the EFI Properties table if it exists */
 473         if (prop_phys != EFI_INVALID_TABLE_ADDR) {
 474                 efi_properties_table_t *tbl;
 475
 476                 tbl = early_memremap_ro(prop_phys, sizeof(*tbl));
 477                 if (tbl == NULL) {
 478                         pr_err("Could not map Properties table!\n");
 479                 } else {
 480                         if (tbl->memory_protection_attribute &
 481                             EFI_PROPERTIES_RUNTIME_MEMORY_PROTECTION_NON_EXECUTABLE_PE_DATA)
 482                                 set_bit(EFI_NX_PE_DATA, &efi.flags);
 483
 484                         early_memunmap(tbl, sizeof(*tbl));
 485                 }
 486         }
 487
 488         set_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 489         efi_clean_memmap();
 490
 491         if (efi_enabled(EFI_DBG))
 492                 efi_print_memmap();
 493 }
 494
 495 #if defined(CONFIG_X86_32) || defined(CONFIG_X86_UV)
 496
 497 void __init efi_set_executable(efi_memory_desc_t *md, bool executable)
 498 {
 499         u64 addr, npages;
 500
 501         addr = md->virt_addr;
 502         npages = md->num_pages;
 503
 504         memrange_efi_to_native(&addr, &npages);
 505
 506         if (executable)
 507                 set_memory_x(addr, npages);
 508         else
 509                 set_memory_nx(addr, npages);
 510 }
 511
 512 void __init runtime_code_page_mkexec(void)
 513 {
 514         efi_memory_desc_t *md;
 515
 516         /* Make EFI runtime service code area executable */
 517         for_each_efi_memory_desc(md) {
 518                 if (md->type != EFI_RUNTIME_SERVICES_CODE)
 519                         continue;
 520
 521                 efi_set_executable(md, true);
 522         }
 523 }
 524
 525 void __init efi_memory_uc(u64 addr, unsigned long size)
 526 {
 527         unsigned long page_shift = 1UL << EFI_PAGE_SHIFT;
 528         u64 npages;
 529
 530         npages = round_up(size, page_shift) / page_shift;
 531         memrange_efi_to_native(&addr, &npages);
 532         set_memory_uc(addr, npages);
 533 }
 534
 535 void __init old_map_region(efi_memory_desc_t *md)
 536 {
 537         u64 start_pfn, end_pfn, end;
 538         unsigned long size;
 539         void *va;
 540
 541         start_pfn = PFN_DOWN(md->phys_addr);
 542         size      = md->num_pages << PAGE_SHIFT;
 543         end       = md->phys_addr + size;
 544         end_pfn   = PFN_UP(end);
 545
 546         if (pfn_range_is_mapped(start_pfn, end_pfn)) {
 547                 va = __va(md->phys_addr);
 548
 549                 if (!(md->attribute & EFI_MEMORY_WB))
 550                         efi_memory_uc((u64)(unsigned long)va, size);
 551         } else
 552                 va = efi_ioremap(md->phys_addr, size,
 553                                  md->type, md->attribute);
 554
 555         md->virt_addr = (u64) (unsigned long) va;
 556         if (!va)
 557                 pr_err("ioremap of 0x%llX failed!\n",
 558                        (unsigned long long)md->phys_addr);
 559 }
 560
 561 #endif
 562
 563 /* Merge contiguous regions of the same type and attribute */
 564 static void __init efi_merge_regions(void)
 565 {
 566         efi_memory_desc_t *md, *prev_md = NULL;
 567
 568         for_each_efi_memory_desc(md) {
 569                 u64 prev_size;
 570
 571                 if (!prev_md) {
 572                         prev_md = md;
 573                         continue;
 574                 }
 575
 576                 if (prev_md->type != md->type ||
 577                     prev_md->attribute != md->attribute) {
 578                         prev_md = md;
 579                         continue;
 580                 }
 581
 582                 prev_size = prev_md->num_pages << EFI_PAGE_SHIFT;
 583
 584                 if (md->phys_addr == (prev_md->phys_addr + prev_size)) {
 585                         prev_md->num_pages += md->num_pages;
 586                         md->type = EFI_RESERVED_TYPE;
 587                         md->attribute = 0;
 588                         continue;
 589                 }
 590                 prev_md = md;
 591         }
 592 }
 593
 594 static void __init get_systab_virt_addr(efi_memory_desc_t *md)
 595 {
 596         unsigned long size;
 597         u64 end, systab;
 598
 599         size = md->num_pages << EFI_PAGE_SHIFT;
 600         end = md->phys_addr + size;
 601         systab = efi_systab_phys;
 602         if (md->phys_addr <= systab && systab < end) {
 603                 systab += md->virt_addr - md->phys_addr;
 604                 efi.systab = (efi_system_table_t *)(unsigned long)systab;
 605         }
 606 }
 607
 608 static void *realloc_pages(void *old_memmap, int old_shift)
 609 {
 610         void *ret;
 611
 612         ret = (void *)__get_free_pages(GFP_KERNEL, old_shift + 1);
 613         if (!ret)
 614                 goto out;
 615
 616         /*
 617          * A first-time allocation doesn't have anything to copy.
 618          */
 619         if (!old_memmap)
 620                 return ret;
 621
 622         memcpy(ret, old_memmap, PAGE_SIZE << old_shift);
 623
 624 out:
 625         free_pages((unsigned long)old_memmap, old_shift);
 626         return ret;
 627 }
 628
 629 /*
 630  * Iterate the EFI memory map in reverse order because the regions
 631  * will be mapped top-down. The end result is the same as if we had
 632  * mapped things forward, but doesn't require us to change the
 633  * existing implementation of efi_map_region().
 634  */
 635 static inline void *efi_map_next_entry_reverse(void *entry)
 636 {
 637         /* Initial call */
 638         if (!entry)
 639                 return efi.memmap.map_end - efi.memmap.desc_size;
 640
 641         entry -= efi.memmap.desc_size;
 642         if (entry < efi.memmap.map)
 643                 return NULL;
 644
 645         return entry;
 646 }
 647
 648 /*
 649  * efi_map_next_entry - Return the next EFI memory map descriptor
 650  * @entry: Previous EFI memory map descriptor
 651  *
 652  * This is a helper function to iterate over the EFI memory map, which
 653  * we do in different orders depending on the current configuration.
 654  *
 655  * To begin traversing the memory map @entry must be %NULL.
 656  *
 657  * Returns %NULL when we reach the end of the memory map.
 658  */
 659 static void *efi_map_next_entry(void *entry)
 660 {
 661         if (!efi_have_uv1_memmap() && efi_enabled(EFI_64BIT)) {
 662                 /*
 663                  * Starting in UEFI v2.5 the EFI_PROPERTIES_TABLE
 664                  * config table feature requires us to map all entries
 665                  * in the same order as they appear in the EFI memory
 666                  * map. That is to say, entry N must have a lower
 667                  * virtual address than entry N+1. This is because the
 668                  * firmware toolchain leaves relative references in
 669                  * the code/data sections, which are split and become
 670                  * separate EFI memory regions. Mapping things
 671                  * out-of-order leads to the firmware accessing
 672                  * unmapped addresses.
 673                  *
 674                  * Since we need to map things this way whether or not
 675                  * the kernel actually makes use of
 676                  * EFI_PROPERTIES_TABLE, let's just switch to this
 677                  * scheme by default for 64-bit.
 678                  */
 679                 return efi_map_next_entry_reverse(entry);
 680         }
 681
 682         /* Initial call */
 683         if (!entry)
 684                 return efi.memmap.map;
 685
 686         entry += efi.memmap.desc_size;
 687         if (entry >= efi.memmap.map_end)
 688                 return NULL;
 689
 690         return entry;
 691 }
 692
 693 static bool should_map_region(efi_memory_desc_t *md)
 694 {
 695         /*
 696          * Runtime regions always require runtime mappings (obviously).
 697          */
 698         if (md->attribute & EFI_MEMORY_RUNTIME)
 699                 return true;
 700
 701         /*
 702          * 32-bit EFI doesn't suffer from the bug that requires us to
 703          * reserve boot services regions, and mixed mode support
 704          * doesn't exist for 32-bit kernels.
 705          */
 706         if (IS_ENABLED(CONFIG_X86_32))
 707                 return false;
 708
 709         /*
 710          * EFI specific purpose memory may be reserved by default
 711          * depending on kernel config and boot options.
 712          */
 713         if (md->type == EFI_CONVENTIONAL_MEMORY &&
 714             efi_soft_reserve_enabled() &&
 715             (md->attribute & EFI_MEMORY_SP))
 716                 return false;
 717
 718         /*
 719          * Map all of RAM so that we can access arguments in the 1:1
 720          * mapping when making EFI runtime calls.
 721          */
 722         if (efi_is_mixed()) {
 723                 if (md->type == EFI_CONVENTIONAL_MEMORY ||
 724                     md->type == EFI_LOADER_DATA ||
 725                     md->type == EFI_LOADER_CODE)
 726                         return true;
 727         }
 728
 729         /*
 730          * Map boot services regions as a workaround for buggy
 731          * firmware that accesses them even when they shouldn't.
 732          *
 733          * See efi_{reserve,free}_boot_services().
 734          */
 735         if (md->type == EFI_BOOT_SERVICES_CODE ||
 736             md->type == EFI_BOOT_SERVICES_DATA)
 737                 return true;
 738
 739         return false;
 740 }
 741
 742 /*
 743  * Map the efi memory ranges of the runtime services and update new_mmap with
 744  * virtual addresses.
 745  */
 746 static void * __init efi_map_regions(int *count, int *pg_shift)
 747 {
 748         void *p, *new_memmap = NULL;
 749         unsigned long left = 0;
 750         unsigned long desc_size;
 751         efi_memory_desc_t *md;
 752
 753         desc_size = efi.memmap.desc_size;
 754
 755         p = NULL;
 756         while ((p = efi_map_next_entry(p))) {
 757                 md = p;
 758
 759                 if (!should_map_region(md))
 760                         continue;
 761
 762                 efi_map_region(md);
 763                 get_systab_virt_addr(md);
 764
 765                 if (left < desc_size) {
 766                         new_memmap = realloc_pages(new_memmap, *pg_shift);
 767                         if (!new_memmap)
 768                                 return NULL;
 769
 770                         left += PAGE_SIZE << *pg_shift;
 771                         (*pg_shift)++;
 772                 }
 773
 774                 memcpy(new_memmap + (*count * desc_size), md, desc_size);
 775
 776                 left -= desc_size;
 777                 (*count)++;
 778         }
 779
 780         return new_memmap;
 781 }
 782
 783 static void __init kexec_enter_virtual_mode(void)
 784 {
 785 #ifdef CONFIG_KEXEC_CORE
 786         efi_memory_desc_t *md;
 787         unsigned int num_pages;
 788
 789         efi.systab = NULL;
 790
 791         /*
 792          * We don't do virtual mode, since we don't do runtime services, on
 793          * non-native EFI. With the UV1 memmap, we don't do runtime services in
 794          * kexec kernel because in the initial boot something else might
 795          * have been mapped at these virtual addresses.
 796          */
 797         if (efi_is_mixed() || efi_have_uv1_memmap()) {
 798                 efi_memmap_unmap();
 799                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 800                 return;
 801         }
 802
 803         if (efi_alloc_page_tables()) {
 804                 pr_err("Failed to allocate EFI page tables\n");
 805                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 806                 return;
 807         }
 808
 809         /*
 810         * Map efi regions which were passed via setup_data. The virt_addr is a
 811         * fixed addr which was used in first kernel of a kexec boot.
 812         */
 813         for_each_efi_memory_desc(md) {
 814                 efi_map_region_fixed(md); /* FIXME: add error handling */
 815                 get_systab_virt_addr(md);
 816         }
 817
 818         /*
 819          * Unregister the early EFI memmap from efi_init() and install
 820          * the new EFI memory map.
 821          */
 822         efi_memmap_unmap();
 823
 824         if (efi_memmap_init_late(efi.memmap.phys_map,
 825                                  efi.memmap.desc_size * efi.memmap.nr_map)) {
 826                 pr_err("Failed to remap late EFI memory map\n");
 827                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 828                 return;
 829         }
 830
 831         BUG_ON(!efi.systab);
 832
 833         num_pages = ALIGN(efi.memmap.nr_map * efi.memmap.desc_size, PAGE_SIZE);
 834         num_pages >>= PAGE_SHIFT;
 835
 836         if (efi_setup_page_tables(efi.memmap.phys_map, num_pages)) {
 837                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 838                 return;
 839         }
 840
 841         efi_sync_low_kernel_mappings();
 842
 843         /*
 844          * Now that EFI is in virtual mode, update the function
 845          * pointers in the runtime service table to the new virtual addresses.
 846          *
 847          * Call EFI services through wrapper functions.
 848          */
 849         efi.runtime_version = efi_systab.hdr.revision;
 850
 851         efi_native_runtime_setup();
 852 #endif
 853 }
 854
 855 /*
 856  * This function will switch the EFI runtime services to virtual mode.
 857  * Essentially, we look through the EFI memmap and map every region that
 858  * has the runtime attribute bit set in its memory descriptor into the
 859  * efi_pgd page table.
 860  *
 861  * The old method which used to update that memory descriptor with the
 862  * virtual address obtained from ioremap() is still supported when the
 863  * kernel is booted on SG1 UV1 hardware. Same old method enabled the
 864  * runtime services to be called without having to thunk back into
 865  * physical mode for every invocation.
 866  *
 867  * The new method does a pagetable switch in a preemption-safe manner
 868  * so that we're in a different address space when calling a runtime
 869  * function. For function arguments passing we do copy the PUDs of the
 870  * kernel page table into efi_pgd prior to each call.
 871  *
 872  * Specially for kexec boot, efi runtime maps in previous kernel should
 873  * be passed in via setup_data. In that case runtime ranges will be mapped
 874  * to the same virtual addresses as the first kernel, see
 875  * kexec_enter_virtual_mode().
 876  */
 877 static void __init __efi_enter_virtual_mode(void)
 878 {
 879         int count = 0, pg_shift = 0;
 880         void *new_memmap = NULL;
 881         efi_status_t status;
 882         unsigned long pa;
 883
 884         efi.systab = NULL;
 885
 886         if (efi_alloc_page_tables()) {
 887                 pr_err("Failed to allocate EFI page tables\n");
 888                 goto err;
 889         }
 890
 891         efi_merge_regions();
 892         new_memmap = efi_map_regions(&count, &pg_shift);
 893         if (!new_memmap) {
 894                 pr_err("Error reallocating memory, EFI runtime non-functional!\n");
 895                 goto err;
 896         }
 897
 898         pa = __pa(new_memmap);
 899
 900         /*
 901          * Unregister the early EFI memmap from efi_init() and install
 902          * the new EFI memory map that we are about to pass to the
 903          * firmware via SetVirtualAddressMap().
 904          */
 905         efi_memmap_unmap();
 906
 907         if (efi_memmap_init_late(pa, efi.memmap.desc_size * count)) {
 908                 pr_err("Failed to remap late EFI memory map\n");
 909                 goto err;
 910         }
 911
 912         if (efi_enabled(EFI_DBG)) {
 913                 pr_info("EFI runtime memory map:\n");
 914                 efi_print_memmap();
 915         }
 916
 917         if (WARN_ON(!efi.systab))
 918                 goto err;
 919
 920         if (efi_setup_page_tables(pa, 1 << pg_shift))
 921                 goto err;
 922
 923         efi_sync_low_kernel_mappings();
 924
 925         status = efi_set_virtual_address_map(efi.memmap.desc_size * count,
 926                                              efi.memmap.desc_size,
 927                                              efi.memmap.desc_version,
 928                                              (efi_memory_desc_t *)pa);
 929         if (status != EFI_SUCCESS) {
 930                 pr_err("Unable to switch EFI into virtual mode (status=%lx)!\n",
 931                        status);
 932                 goto err;
 933         }
 934
 935         efi_free_boot_services();
 936
 937         /*
 938          * Now that EFI is in virtual mode, update the function
 939          * pointers in the runtime service table to the new virtual addresses.
 940          *
 941          * Call EFI services through wrapper functions.
 942          */
 943         efi.runtime_version = efi_systab.hdr.revision;
 944
 945         if (!efi_is_mixed())
 946                 efi_native_runtime_setup();
 947         else
 948                 efi_thunk_runtime_setup();
 949
 950         /*
 951          * Apply more restrictive page table mapping attributes now that
 952          * SVAM() has been called and the firmware has performed all
 953          * necessary relocation fixups for the new virtual addresses.
 954          */
 955         efi_runtime_update_mappings();
 956
 957         /* clean DUMMY object */
 958         efi_delete_dummy_variable();
 959         return;
 960
 961 err:
 962         clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 963 }
 964
 965 void __init efi_enter_virtual_mode(void)
 966 {
 967         if (efi_enabled(EFI_PARAVIRT))
 968                 return;
 969
 970         if (efi_setup)
 971                 kexec_enter_virtual_mode();
 972         else
 973                 __efi_enter_virtual_mode();
 974
 975         efi_dump_pagetable();
 976 }
 977
 978 bool efi_is_table_address(unsigned long phys_addr)
 979 {
 980         unsigned int i;
 981
 982         if (phys_addr == EFI_INVALID_TABLE_ADDR)
 983                 return false;
 984
 985         for (i = 0; i < ARRAY_SIZE(efi_tables); i++)
 986                 if (*(efi_tables[i]) == phys_addr)
 987                         return true;
 988
 989         return false;
 990 }
 991
 992 char *efi_systab_show_arch(char *str)
 993 {
 994         if (uga_phys != EFI_INVALID_TABLE_ADDR)
 995                 str += sprintf(str, "UGA=0x%lx\n", uga_phys);
 996         return str;
 997 }