arch/x86/platform/efi/efi.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Common EFI (Extensible Firmware Interface) support functions
   4  * Based on Extensible Firmware Interface Specification version 1.0
   5  *
   6  * Copyright (C) 1999 VA Linux Systems
   7  * Copyright (C) 1999 Walt Drummond <drummond@valinux.com>
   8  * Copyright (C) 1999-2002 Hewlett-Packard Co.
   9  *      David Mosberger-Tang <davidm@hpl.hp.com>
  10  *      Stephane Eranian <eranian@hpl.hp.com>
  11  * Copyright (C) 2005-2008 Intel Co.
  12  *      Fenghua Yu <fenghua.yu@intel.com>
  13  *      Bibo Mao <bibo.mao@intel.com>
  14  *      Chandramouli Narayanan <mouli@linux.intel.com>
  15  *      Huang Ying <ying.huang@intel.com>
  16  * Copyright (C) 2013 SuSE Labs
  17  *      Borislav Petkov <bp@suse.de> - runtime services VA mapping
  18  *
  19  * Copied from efi_32.c to eliminate the duplicated code between EFI
  20  * 32/64 support code. --ying 2007-10-26
  21  *
  22  * All EFI Runtime Services are not implemented yet as EFI only
  23  * supports physical mode addressing on SoftSDV. This is to be fixed
  24  * in a future version.  --drummond 1999-07-20
  25  *
  26  * Implemented EFI runtime services and virtual mode calls.  --davidm
  27  *
  28  * Goutham Rao: <goutham.rao@intel.com>
  29  *      Skip non-WB memory and ignore empty memory ranges.
  30  */
  31
  32 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  33
  34 #include <linux/kernel.h>
  35 #include <linux/init.h>
  36 #include <linux/efi.h>
  37 #include <linux/efi-bgrt.h>
  38 #include <linux/export.h>
  39 #include <linux/memblock.h>
  40 #include <linux/slab.h>
  41 #include <linux/spinlock.h>
  42 #include <linux/uaccess.h>
  43 #include <linux/time.h>
  44 #include <linux/io.h>
  45 #include <linux/reboot.h>
  46 #include <linux/bcd.h>
  47
  48 #include <asm/setup.h>
  49 #include <asm/efi.h>
  50 #include <asm/e820/api.h>
  51 #include <asm/time.h>
  52 #include <asm/set_memory.h>
  53 #include <asm/tlbflush.h>
  54 #include <asm/x86_init.h>
  55 #include <asm/uv/uv.h>
  56
  57 static struct efi efi_phys __initdata;
  58 static efi_system_table_t efi_systab __initdata;
  59
  60 static efi_config_table_type_t arch_tables[] __initdata = {
  61 #ifdef CONFIG_X86_UV
  62         {UV_SYSTEM_TABLE_GUID, "UVsystab", &uv_systab_phys},
  63 #endif
  64         {NULL_GUID, NULL, NULL},
  65 };
  66
  67 static const unsigned long * const efi_tables[] = {
  68         &efi.mps,
  69         &efi.acpi,
  70         &efi.acpi20,
  71         &efi.smbios,
  72         &efi.smbios3,
  73         &efi.boot_info,
  74         &efi.hcdp,
  75         &efi.uga,
  76 #ifdef CONFIG_X86_UV
  77         &uv_systab_phys,
  78 #endif
  79         &efi.fw_vendor,
  80         &efi.runtime,
  81         &efi.config_table,
  82         &efi.esrt,
  83         &efi.properties_table,
  84         &efi.mem_attr_table,
  85 #ifdef CONFIG_EFI_RCI2_TABLE
  86         &rci2_table_phys,
  87 #endif
  88 };
  89
  90 u64 efi_setup;          /* efi setup_data physical address */
  91
  92 static int add_efi_memmap __initdata;
  93 static int __init setup_add_efi_memmap(char *arg)
  94 {
  95         add_efi_memmap = 1;
  96         return 0;
  97 }
  98 early_param("add_efi_memmap", setup_add_efi_memmap);
  99
 100 static efi_status_t __init phys_efi_set_virtual_address_map(
 101         unsigned long memory_map_size,
 102         unsigned long descriptor_size,
 103         u32 descriptor_version,
 104         efi_memory_desc_t *virtual_map)
 105 {
 106         efi_status_t status;
 107         unsigned long flags;
 108         pgd_t *save_pgd;
 109
 110         save_pgd = efi_call_phys_prolog();
 111         if (!save_pgd)
 112                 return EFI_ABORTED;
 113
 114         /* Disable interrupts around EFI calls: */
 115         local_irq_save(flags);
 116         status = efi_call_phys(efi_phys.set_virtual_address_map,
 117                                memory_map_size, descriptor_size,
 118                                descriptor_version, virtual_map);
 119         local_irq_restore(flags);
 120
 121         efi_call_phys_epilog(save_pgd);
 122
 123         return status;
 124 }
 125
 126 void __init efi_find_mirror(void)
 127 {
 128         efi_memory_desc_t *md;
 129         u64 mirror_size = 0, total_size = 0;
 130
 131         if (!efi_enabled(EFI_MEMMAP))
 132                 return;
 133
 134         for_each_efi_memory_desc(md) {
 135                 unsigned long long start = md->phys_addr;
 136                 unsigned long long size = md->num_pages << EFI_PAGE_SHIFT;
 137
 138                 total_size += size;
 139                 if (md->attribute & EFI_MEMORY_MORE_RELIABLE) {
 140                         memblock_mark_mirror(start, size);
 141                         mirror_size += size;
 142                 }
 143         }
 144         if (mirror_size)
 145                 pr_info("Memory: %lldM/%lldM mirrored memory\n",
 146                         mirror_size>>20, total_size>>20);
 147 }
 148
 149 /*
 150  * Tell the kernel about the EFI memory map.  This might include
 151  * more than the max 128 entries that can fit in the e820 legacy
 152  * (zeropage) memory map.
 153  */
 154
 155 static void __init do_add_efi_memmap(void)
 156 {
 157         efi_memory_desc_t *md;
 158
 159         for_each_efi_memory_desc(md) {
 160                 unsigned long long start = md->phys_addr;
 161                 unsigned long long size = md->num_pages << EFI_PAGE_SHIFT;
 162                 int e820_type;
 163
 164                 switch (md->type) {
 165                 case EFI_LOADER_CODE:
 166                 case EFI_LOADER_DATA:
 167                 case EFI_BOOT_SERVICES_CODE:
 168                 case EFI_BOOT_SERVICES_DATA:
 169                 case EFI_CONVENTIONAL_MEMORY:
 170                         if (md->attribute & EFI_MEMORY_WB)
 171                                 e820_type = E820_TYPE_RAM;
 172                         else
 173                                 e820_type = E820_TYPE_RESERVED;
 174                         break;
 175                 case EFI_ACPI_RECLAIM_MEMORY:
 176                         e820_type = E820_TYPE_ACPI;
 177                         break;
 178                 case EFI_ACPI_MEMORY_NVS:
 179                         e820_type = E820_TYPE_NVS;
 180                         break;
 181                 case EFI_UNUSABLE_MEMORY:
 182                         e820_type = E820_TYPE_UNUSABLE;
 183                         break;
 184                 case EFI_PERSISTENT_MEMORY:
 185                         e820_type = E820_TYPE_PMEM;
 186                         break;
 187                 default:
 188                         /*
 189                          * EFI_RESERVED_TYPE EFI_RUNTIME_SERVICES_CODE
 190                          * EFI_RUNTIME_SERVICES_DATA EFI_MEMORY_MAPPED_IO
 191                          * EFI_MEMORY_MAPPED_IO_PORT_SPACE EFI_PAL_CODE
 192                          */
 193                         e820_type = E820_TYPE_RESERVED;
 194                         break;
 195                 }
 196                 e820__range_add(start, size, e820_type);
 197         }
 198         e820__update_table(e820_table);
 199 }
 200
 201 int __init efi_memblock_x86_reserve_range(void)
 202 {
 203         struct efi_info *e = &boot_params.efi_info;
 204         struct efi_memory_map_data data;
 205         phys_addr_t pmap;
 206         int rv;
 207
 208         if (efi_enabled(EFI_PARAVIRT))
 209                 return 0;
 210
 211 #ifdef CONFIG_X86_32
 212         /* Can't handle data above 4GB at this time */
 213         if (e->efi_memmap_hi) {
 214                 pr_err("Memory map is above 4GB, disabling EFI.\n");
 215                 return -EINVAL;
 216         }
 217         pmap =  e->efi_memmap;
 218 #else
 219         pmap = (e->efi_memmap | ((__u64)e->efi_memmap_hi << 32));
 220 #endif
 221         data.phys_map           = pmap;
 222         data.size               = e->efi_memmap_size;
 223         data.desc_size          = e->efi_memdesc_size;
 224         data.desc_version       = e->efi_memdesc_version;
 225
 226         rv = efi_memmap_init_early(&data);
 227         if (rv)
 228                 return rv;
 229
 230         if (add_efi_memmap)
 231                 do_add_efi_memmap();
 232
 233         WARN(efi.memmap.desc_version != 1,
 234              "Unexpected EFI_MEMORY_DESCRIPTOR version %ld",
 235              efi.memmap.desc_version);
 236
 237         memblock_reserve(pmap, efi.memmap.nr_map * efi.memmap.desc_size);
 238
 239         return 0;
 240 }
 241
 242 #define OVERFLOW_ADDR_SHIFT     (64 - EFI_PAGE_SHIFT)
 243 #define OVERFLOW_ADDR_MASK      (U64_MAX << OVERFLOW_ADDR_SHIFT)
 244 #define U64_HIGH_BIT            (~(U64_MAX >> 1))
 245
 246 static bool __init efi_memmap_entry_valid(const efi_memory_desc_t *md, int i)
 247 {
 248         u64 end = (md->num_pages << EFI_PAGE_SHIFT) + md->phys_addr - 1;
 249         u64 end_hi = 0;
 250         char buf[64];
 251
 252         if (md->num_pages == 0) {
 253                 end = 0;
 254         } else if (md->num_pages > EFI_PAGES_MAX ||
 255                    EFI_PAGES_MAX - md->num_pages <
 256                    (md->phys_addr >> EFI_PAGE_SHIFT)) {
 257                 end_hi = (md->num_pages & OVERFLOW_ADDR_MASK)
 258                         >> OVERFLOW_ADDR_SHIFT;
 259
 260                 if ((md->phys_addr & U64_HIGH_BIT) && !(end & U64_HIGH_BIT))
 261                         end_hi += 1;
 262         } else {
 263                 return true;
 264         }
 265
 266         pr_warn_once(FW_BUG "Invalid EFI memory map entries:\n");
 267
 268         if (end_hi) {
 269                 pr_warn("mem%02u: %s range=[0x%016llx-0x%llx%016llx] (invalid)\n",
 270                         i, efi_md_typeattr_format(buf, sizeof(buf), md),
 271                         md->phys_addr, end_hi, end);
 272         } else {
 273                 pr_warn("mem%02u: %s range=[0x%016llx-0x%016llx] (invalid)\n",
 274                         i, efi_md_typeattr_format(buf, sizeof(buf), md),
 275                         md->phys_addr, end);
 276         }
 277         return false;
 278 }
 279
 280 static void __init efi_clean_memmap(void)
 281 {
 282         efi_memory_desc_t *out = efi.memmap.map;
 283         const efi_memory_desc_t *in = out;
 284         const efi_memory_desc_t *end = efi.memmap.map_end;
 285         int i, n_removal;
 286
 287         for (i = n_removal = 0; in < end; i++) {
 288                 if (efi_memmap_entry_valid(in, i)) {
 289                         if (out != in)
 290                                 memcpy(out, in, efi.memmap.desc_size);
 291                         out = (void *)out + efi.memmap.desc_size;
 292                 } else {
 293                         n_removal++;
 294                 }
 295                 in = (void *)in + efi.memmap.desc_size;
 296         }
 297
 298         if (n_removal > 0) {
 299                 u64 size = efi.memmap.nr_map - n_removal;
 300
 301                 pr_warn("Removing %d invalid memory map entries.\n", n_removal);
 302                 efi_memmap_install(efi.memmap.phys_map, size);
 303         }
 304 }
 305
 306 void __init efi_print_memmap(void)
 307 {
 308         efi_memory_desc_t *md;
 309         int i = 0;
 310
 311         for_each_efi_memory_desc(md) {
 312                 char buf[64];
 313
 314                 pr_info("mem%02u: %s range=[0x%016llx-0x%016llx] (%lluMB)\n",
 315                         i++, efi_md_typeattr_format(buf, sizeof(buf), md),
 316                         md->phys_addr,
 317                         md->phys_addr + (md->num_pages << EFI_PAGE_SHIFT) - 1,
 318                         (md->num_pages >> (20 - EFI_PAGE_SHIFT)));
 319         }
 320 }
 321
 322 static int __init efi_systab_init(void *phys)
 323 {
 324         if (efi_enabled(EFI_64BIT)) {
 325                 efi_system_table_64_t *systab64;
 326                 struct efi_setup_data *data = NULL;
 327                 u64 tmp = 0;
 328
 329                 if (efi_setup) {
 330                         data = early_memremap(efi_setup, sizeof(*data));
 331                         if (!data)
 332                                 return -ENOMEM;
 333                 }
 334                 systab64 = early_memremap((unsigned long)phys,
 335                                          sizeof(*systab64));
 336                 if (systab64 == NULL) {
 337                         pr_err("Couldn't map the system table!\n");
 338                         if (data)
 339                                 early_memunmap(data, sizeof(*data));
 340                         return -ENOMEM;
 341                 }
 342
 343                 efi_systab.hdr = systab64->hdr;
 344                 efi_systab.fw_vendor = data ? (unsigned long)data->fw_vendor :
 345                                               systab64->fw_vendor;
 346                 tmp |= data ? data->fw_vendor : systab64->fw_vendor;
 347                 efi_systab.fw_revision = systab64->fw_revision;
 348                 efi_systab.con_in_handle = systab64->con_in_handle;
 349                 tmp |= systab64->con_in_handle;
 350                 efi_systab.con_in = systab64->con_in;
 351                 tmp |= systab64->con_in;
 352                 efi_systab.con_out_handle = systab64->con_out_handle;
 353                 tmp |= systab64->con_out_handle;
 354                 efi_systab.con_out = systab64->con_out;
 355                 tmp |= systab64->con_out;
 356                 efi_systab.stderr_handle = systab64->stderr_handle;
 357                 tmp |= systab64->stderr_handle;
 358                 efi_systab.stderr = systab64->stderr;
 359                 tmp |= systab64->stderr;
 360                 efi_systab.runtime = data ?
 361                                      (void *)(unsigned long)data->runtime :
 362                                      (void *)(unsigned long)systab64->runtime;
 363                 tmp |= data ? data->runtime : systab64->runtime;
 364                 efi_systab.boottime = (void *)(unsigned long)systab64->boottime;
 365                 tmp |= systab64->boottime;
 366                 efi_systab.nr_tables = systab64->nr_tables;
 367                 efi_systab.tables = data ? (unsigned long)data->tables :
 368                                            systab64->tables;
 369                 tmp |= data ? data->tables : systab64->tables;
 370
 371                 early_memunmap(systab64, sizeof(*systab64));
 372                 if (data)
 373                         early_memunmap(data, sizeof(*data));
 374 #ifdef CONFIG_X86_32
 375                 if (tmp >> 32) {
 376                         pr_err("EFI data located above 4GB, disabling EFI.\n");
 377                         return -EINVAL;
 378                 }
 379 #endif
 380         } else {
 381                 efi_system_table_32_t *systab32;
 382
 383                 systab32 = early_memremap((unsigned long)phys,
 384                                          sizeof(*systab32));
 385                 if (systab32 == NULL) {
 386                         pr_err("Couldn't map the system table!\n");
 387                         return -ENOMEM;
 388                 }
 389
 390                 efi_systab.hdr = systab32->hdr;
 391                 efi_systab.fw_vendor = systab32->fw_vendor;
 392                 efi_systab.fw_revision = systab32->fw_revision;
 393                 efi_systab.con_in_handle = systab32->con_in_handle;
 394                 efi_systab.con_in = systab32->con_in;
 395                 efi_systab.con_out_handle = systab32->con_out_handle;
 396                 efi_systab.con_out = systab32->con_out;
 397                 efi_systab.stderr_handle = systab32->stderr_handle;
 398                 efi_systab.stderr = systab32->stderr;
 399                 efi_systab.runtime = (void *)(unsigned long)systab32->runtime;
 400                 efi_systab.boottime = (void *)(unsigned long)systab32->boottime;
 401                 efi_systab.nr_tables = systab32->nr_tables;
 402                 efi_systab.tables = systab32->tables;
 403
 404                 early_memunmap(systab32, sizeof(*systab32));
 405         }
 406
 407         efi.systab = &efi_systab;
 408
 409         /*
 410          * Verify the EFI Table
 411          */
 412         if (efi.systab->hdr.signature != EFI_SYSTEM_TABLE_SIGNATURE) {
 413                 pr_err("System table signature incorrect!\n");
 414                 return -EINVAL;
 415         }
 416         if ((efi.systab->hdr.revision >> 16) == 0)
 417                 pr_err("Warning: System table version %d.%02d, expected 1.00 or greater!\n",
 418                        efi.systab->hdr.revision >> 16,
 419                        efi.systab->hdr.revision & 0xffff);
 420
 421         return 0;
 422 }
 423
 424 static int __init efi_runtime_init32(void)
 425 {
 426         efi_runtime_services_32_t *runtime;
 427
 428         runtime = early_memremap((unsigned long)efi.systab->runtime,
 429                         sizeof(efi_runtime_services_32_t));
 430         if (!runtime) {
 431                 pr_err("Could not map the runtime service table!\n");
 432                 return -ENOMEM;
 433         }
 434
 435         /*
 436          * We will only need *early* access to the SetVirtualAddressMap
 437          * EFI runtime service. All other runtime services will be called
 438          * via the virtual mapping.
 439          */
 440         efi_phys.set_virtual_address_map =
 441                         (efi_set_virtual_address_map_t *)
 442                         (unsigned long)runtime->set_virtual_address_map;
 443         early_memunmap(runtime, sizeof(efi_runtime_services_32_t));
 444
 445         return 0;
 446 }
 447
 448 static int __init efi_runtime_init64(void)
 449 {
 450         efi_runtime_services_64_t *runtime;
 451
 452         runtime = early_memremap((unsigned long)efi.systab->runtime,
 453                         sizeof(efi_runtime_services_64_t));
 454         if (!runtime) {
 455                 pr_err("Could not map the runtime service table!\n");
 456                 return -ENOMEM;
 457         }
 458
 459         /*
 460          * We will only need *early* access to the SetVirtualAddressMap
 461          * EFI runtime service. All other runtime services will be called
 462          * via the virtual mapping.
 463          */
 464         efi_phys.set_virtual_address_map =
 465                         (efi_set_virtual_address_map_t *)
 466                         (unsigned long)runtime->set_virtual_address_map;
 467         early_memunmap(runtime, sizeof(efi_runtime_services_64_t));
 468
 469         return 0;
 470 }
 471
 472 static int __init efi_runtime_init(void)
 473 {
 474         int rv;
 475
 476         /*
 477          * Check out the runtime services table. We need to map
 478          * the runtime services table so that we can grab the physical
 479          * address of several of the EFI runtime functions, needed to
 480          * set the firmware into virtual mode.
 481          *
 482          * When EFI_PARAVIRT is in force then we could not map runtime
 483          * service memory region because we do not have direct access to it.
 484          * However, runtime services are available through proxy functions
 485          * (e.g. in case of Xen dom0 EFI implementation they call special
 486          * hypercall which executes relevant EFI functions) and that is why
 487          * they are always enabled.
 488          */
 489
 490         if (!efi_enabled(EFI_PARAVIRT)) {
 491                 if (efi_enabled(EFI_64BIT))
 492                         rv = efi_runtime_init64();
 493                 else
 494                         rv = efi_runtime_init32();
 495
 496                 if (rv)
 497                         return rv;
 498         }
 499
 500         set_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 501
 502         return 0;
 503 }
 504
 505 void __init efi_init(void)
 506 {
 507         efi_char16_t *c16;
 508         char vendor[100] = "unknown";
 509         int i = 0;
 510         void *tmp;
 511
 512 #ifdef CONFIG_X86_32
 513         if (boot_params.efi_info.efi_systab_hi ||
 514             boot_params.efi_info.efi_memmap_hi) {
 515                 pr_info("Table located above 4GB, disabling EFI.\n");
 516                 return;
 517         }
 518         efi_phys.systab = (efi_system_table_t *)boot_params.efi_info.efi_systab;
 519 #else
 520         efi_phys.systab = (efi_system_table_t *)
 521                           (boot_params.efi_info.efi_systab |
 522                           ((__u64)boot_params.efi_info.efi_systab_hi<<32));
 523 #endif
 524
 525         if (efi_systab_init(efi_phys.systab))
 526                 return;
 527
 528         efi.config_table = (unsigned long)efi.systab->tables;
 529         efi.fw_vendor    = (unsigned long)efi.systab->fw_vendor;
 530         efi.runtime      = (unsigned long)efi.systab->runtime;
 531
 532         /*
 533          * Show what we know for posterity
 534          */
 535         c16 = tmp = early_memremap(efi.systab->fw_vendor, 2);
 536         if (c16) {
 537                 for (i = 0; i < sizeof(vendor) - 1 && *c16; ++i)
 538                         vendor[i] = *c16++;
 539                 vendor[i] = '\0';
 540         } else
 541                 pr_err("Could not map the firmware vendor!\n");
 542         early_memunmap(tmp, 2);
 543
 544         pr_info("EFI v%u.%.02u by %s\n",
 545                 efi.systab->hdr.revision >> 16,
 546                 efi.systab->hdr.revision & 0xffff, vendor);
 547
 548         if (efi_reuse_config(efi.systab->tables, efi.systab->nr_tables))
 549                 return;
 550
 551         if (efi_config_init(arch_tables))
 552                 return;
 553
 554         /*
 555          * Note: We currently don't support runtime services on an EFI
 556          * that doesn't match the kernel 32/64-bit mode.
 557          */
 558
 559         if (!efi_runtime_supported())
 560                 pr_info("No EFI runtime due to 32/64-bit mismatch with kernel\n");
 561         else {
 562                 if (efi_runtime_disabled() || efi_runtime_init()) {
 563                         efi_memmap_unmap();
 564                         return;
 565                 }
 566         }
 567
 568         efi_clean_memmap();
 569
 570         if (efi_enabled(EFI_DBG))
 571                 efi_print_memmap();
 572 }
 573
 574 void __init efi_set_executable(efi_memory_desc_t *md, bool executable)
 575 {
 576         u64 addr, npages;
 577
 578         addr = md->virt_addr;
 579         npages = md->num_pages;
 580
 581         memrange_efi_to_native(&addr, &npages);
 582
 583         if (executable)
 584                 set_memory_x(addr, npages);
 585         else
 586                 set_memory_nx(addr, npages);
 587 }
 588
 589 void __init runtime_code_page_mkexec(void)
 590 {
 591         efi_memory_desc_t *md;
 592
 593         /* Make EFI runtime service code area executable */
 594         for_each_efi_memory_desc(md) {
 595                 if (md->type != EFI_RUNTIME_SERVICES_CODE)
 596                         continue;
 597
 598                 efi_set_executable(md, true);
 599         }
 600 }
 601
 602 void __init efi_memory_uc(u64 addr, unsigned long size)
 603 {
 604         unsigned long page_shift = 1UL << EFI_PAGE_SHIFT;
 605         u64 npages;
 606
 607         npages = round_up(size, page_shift) / page_shift;
 608         memrange_efi_to_native(&addr, &npages);
 609         set_memory_uc(addr, npages);
 610 }
 611
 612 void __init old_map_region(efi_memory_desc_t *md)
 613 {
 614         u64 start_pfn, end_pfn, end;
 615         unsigned long size;
 616         void *va;
 617
 618         start_pfn = PFN_DOWN(md->phys_addr);
 619         size      = md->num_pages << PAGE_SHIFT;
 620         end       = md->phys_addr + size;
 621         end_pfn   = PFN_UP(end);
 622
 623         if (pfn_range_is_mapped(start_pfn, end_pfn)) {
 624                 va = __va(md->phys_addr);
 625
 626                 if (!(md->attribute & EFI_MEMORY_WB))
 627                         efi_memory_uc((u64)(unsigned long)va, size);
 628         } else
 629                 va = efi_ioremap(md->phys_addr, size,
 630                                  md->type, md->attribute);
 631
 632         md->virt_addr = (u64) (unsigned long) va;
 633         if (!va)
 634                 pr_err("ioremap of 0x%llX failed!\n",
 635                        (unsigned long long)md->phys_addr);
 636 }
 637
 638 /* Merge contiguous regions of the same type and attribute */
 639 static void __init efi_merge_regions(void)
 640 {
 641         efi_memory_desc_t *md, *prev_md = NULL;
 642
 643         for_each_efi_memory_desc(md) {
 644                 u64 prev_size;
 645
 646                 if (!prev_md) {
 647                         prev_md = md;
 648                         continue;
 649                 }
 650
 651                 if (prev_md->type != md->type ||
 652                     prev_md->attribute != md->attribute) {
 653                         prev_md = md;
 654                         continue;
 655                 }
 656
 657                 prev_size = prev_md->num_pages << EFI_PAGE_SHIFT;
 658
 659                 if (md->phys_addr == (prev_md->phys_addr + prev_size)) {
 660                         prev_md->num_pages += md->num_pages;
 661                         md->type = EFI_RESERVED_TYPE;
 662                         md->attribute = 0;
 663                         continue;
 664                 }
 665                 prev_md = md;
 666         }
 667 }
 668
 669 static void __init get_systab_virt_addr(efi_memory_desc_t *md)
 670 {
 671         unsigned long size;
 672         u64 end, systab;
 673
 674         size = md->num_pages << EFI_PAGE_SHIFT;
 675         end = md->phys_addr + size;
 676         systab = (u64)(unsigned long)efi_phys.systab;
 677         if (md->phys_addr <= systab && systab < end) {
 678                 systab += md->virt_addr - md->phys_addr;
 679                 efi.systab = (efi_system_table_t *)(unsigned long)systab;
 680         }
 681 }
 682
 683 static void *realloc_pages(void *old_memmap, int old_shift)
 684 {
 685         void *ret;
 686
 687         ret = (void *)__get_free_pages(GFP_KERNEL, old_shift + 1);
 688         if (!ret)
 689                 goto out;
 690
 691         /*
 692          * A first-time allocation doesn't have anything to copy.
 693          */
 694         if (!old_memmap)
 695                 return ret;
 696
 697         memcpy(ret, old_memmap, PAGE_SIZE << old_shift);
 698
 699 out:
 700         free_pages((unsigned long)old_memmap, old_shift);
 701         return ret;
 702 }
 703
 704 /*
 705  * Iterate the EFI memory map in reverse order because the regions
 706  * will be mapped top-down. The end result is the same as if we had
 707  * mapped things forward, but doesn't require us to change the
 708  * existing implementation of efi_map_region().
 709  */
 710 static inline void *efi_map_next_entry_reverse(void *entry)
 711 {
 712         /* Initial call */
 713         if (!entry)
 714                 return efi.memmap.map_end - efi.memmap.desc_size;
 715
 716         entry -= efi.memmap.desc_size;
 717         if (entry < efi.memmap.map)
 718                 return NULL;
 719
 720         return entry;
 721 }
 722
 723 /*
 724  * efi_map_next_entry - Return the next EFI memory map descriptor
 725  * @entry: Previous EFI memory map descriptor
 726  *
 727  * This is a helper function to iterate over the EFI memory map, which
 728  * we do in different orders depending on the current configuration.
 729  *
 730  * To begin traversing the memory map @entry must be %NULL.
 731  *
 732  * Returns %NULL when we reach the end of the memory map.
 733  */
 734 static void *efi_map_next_entry(void *entry)
 735 {
 736         if (!efi_enabled(EFI_OLD_MEMMAP) && efi_enabled(EFI_64BIT)) {
 737                 /*
 738                  * Starting in UEFI v2.5 the EFI_PROPERTIES_TABLE
 739                  * config table feature requires us to map all entries
 740                  * in the same order as they appear in the EFI memory
 741                  * map. That is to say, entry N must have a lower
 742                  * virtual address than entry N+1. This is because the
 743                  * firmware toolchain leaves relative references in
 744                  * the code/data sections, which are split and become
 745                  * separate EFI memory regions. Mapping things
 746                  * out-of-order leads to the firmware accessing
 747                  * unmapped addresses.
 748                  *
 749                  * Since we need to map things this way whether or not
 750                  * the kernel actually makes use of
 751                  * EFI_PROPERTIES_TABLE, let's just switch to this
 752                  * scheme by default for 64-bit.
 753                  */
 754                 return efi_map_next_entry_reverse(entry);
 755         }
 756
 757         /* Initial call */
 758         if (!entry)
 759                 return efi.memmap.map;
 760
 761         entry += efi.memmap.desc_size;
 762         if (entry >= efi.memmap.map_end)
 763                 return NULL;
 764
 765         return entry;
 766 }
 767
 768 static bool should_map_region(efi_memory_desc_t *md)
 769 {
 770         /*
 771          * Runtime regions always require runtime mappings (obviously).
 772          */
 773         if (md->attribute & EFI_MEMORY_RUNTIME)
 774                 return true;
 775
 776         /*
 777          * 32-bit EFI doesn't suffer from the bug that requires us to
 778          * reserve boot services regions, and mixed mode support
 779          * doesn't exist for 32-bit kernels.
 780          */
 781         if (IS_ENABLED(CONFIG_X86_32))
 782                 return false;
 783
 784         /*
 785          * Map all of RAM so that we can access arguments in the 1:1
 786          * mapping when making EFI runtime calls.
 787          */
 788         if (IS_ENABLED(CONFIG_EFI_MIXED) && !efi_is_native()) {
 789                 if (md->type == EFI_CONVENTIONAL_MEMORY ||
 790                     md->type == EFI_LOADER_DATA ||
 791                     md->type == EFI_LOADER_CODE)
 792                         return true;
 793         }
 794
 795         /*
 796          * Map boot services regions as a workaround for buggy
 797          * firmware that accesses them even when they shouldn't.
 798          *
 799          * See efi_{reserve,free}_boot_services().
 800          */
 801         if (md->type == EFI_BOOT_SERVICES_CODE ||
 802             md->type == EFI_BOOT_SERVICES_DATA)
 803                 return true;
 804
 805         return false;
 806 }
 807
 808 /*
 809  * Map the efi memory ranges of the runtime services and update new_mmap with
 810  * virtual addresses.
 811  */
 812 static void * __init efi_map_regions(int *count, int *pg_shift)
 813 {
 814         void *p, *new_memmap = NULL;
 815         unsigned long left = 0;
 816         unsigned long desc_size;
 817         efi_memory_desc_t *md;
 818
 819         desc_size = efi.memmap.desc_size;
 820
 821         p = NULL;
 822         while ((p = efi_map_next_entry(p))) {
 823                 md = p;
 824
 825                 if (!should_map_region(md))
 826                         continue;
 827
 828                 efi_map_region(md);
 829                 get_systab_virt_addr(md);
 830
 831                 if (left < desc_size) {
 832                         new_memmap = realloc_pages(new_memmap, *pg_shift);
 833                         if (!new_memmap)
 834                                 return NULL;
 835
 836                         left += PAGE_SIZE << *pg_shift;
 837                         (*pg_shift)++;
 838                 }
 839
 840                 memcpy(new_memmap + (*count * desc_size), md, desc_size);
 841
 842                 left -= desc_size;
 843                 (*count)++;
 844         }
 845
 846         return new_memmap;
 847 }
 848
 849 static void __init kexec_enter_virtual_mode(void)
 850 {
 851 #ifdef CONFIG_KEXEC_CORE
 852         efi_memory_desc_t *md;
 853         unsigned int num_pages;
 854
 855         efi.systab = NULL;
 856
 857         /*
 858          * We don't do virtual mode, since we don't do runtime services, on
 859          * non-native EFI. With efi=old_map, we don't do runtime services in
 860          * kexec kernel because in the initial boot something else might
 861          * have been mapped at these virtual addresses.
 862          */
 863         if (!efi_is_native() || efi_enabled(EFI_OLD_MEMMAP)) {
 864                 efi_memmap_unmap();
 865                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 866                 return;
 867         }
 868
 869         if (efi_alloc_page_tables()) {
 870                 pr_err("Failed to allocate EFI page tables\n");
 871                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 872                 return;
 873         }
 874
 875         /*
 876         * Map efi regions which were passed via setup_data. The virt_addr is a
 877         * fixed addr which was used in first kernel of a kexec boot.
 878         */
 879         for_each_efi_memory_desc(md) {
 880                 efi_map_region_fixed(md); /* FIXME: add error handling */
 881                 get_systab_virt_addr(md);
 882         }
 883
 884         /*
 885          * Unregister the early EFI memmap from efi_init() and install
 886          * the new EFI memory map.
 887          */
 888         efi_memmap_unmap();
 889
 890         if (efi_memmap_init_late(efi.memmap.phys_map,
 891                                  efi.memmap.desc_size * efi.memmap.nr_map)) {
 892                 pr_err("Failed to remap late EFI memory map\n");
 893                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 894                 return;
 895         }
 896
 897         BUG_ON(!efi.systab);
 898
 899         num_pages = ALIGN(efi.memmap.nr_map * efi.memmap.desc_size, PAGE_SIZE);
 900         num_pages >>= PAGE_SHIFT;
 901
 902         if (efi_setup_page_tables(efi.memmap.phys_map, num_pages)) {
 903                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 904                 return;
 905         }
 906
 907         efi_sync_low_kernel_mappings();
 908
 909         /*
 910          * Now that EFI is in virtual mode, update the function
 911          * pointers in the runtime service table to the new virtual addresses.
 912          *
 913          * Call EFI services through wrapper functions.
 914          */
 915         efi.runtime_version = efi_systab.hdr.revision;
 916
 917         efi_native_runtime_setup();
 918
 919         efi.set_virtual_address_map = NULL;
 920
 921         if (efi_enabled(EFI_OLD_MEMMAP) && (__supported_pte_mask & _PAGE_NX))
 922                 runtime_code_page_mkexec();
 923 #endif
 924 }
 925
 926 /*
 927  * This function will switch the EFI runtime services to virtual mode.
 928  * Essentially, we look through the EFI memmap and map every region that
 929  * has the runtime attribute bit set in its memory descriptor into the
 930  * efi_pgd page table.
 931  *
 932  * The old method which used to update that memory descriptor with the
 933  * virtual address obtained from ioremap() is still supported when the
 934  * kernel is booted with efi=old_map on its command line. Same old
 935  * method enabled the runtime services to be called without having to
 936  * thunk back into physical mode for every invocation.
 937  *
 938  * The new method does a pagetable switch in a preemption-safe manner
 939  * so that we're in a different address space when calling a runtime
 940  * function. For function arguments passing we do copy the PUDs of the
 941  * kernel page table into efi_pgd prior to each call.
 942  *
 943  * Specially for kexec boot, efi runtime maps in previous kernel should
 944  * be passed in via setup_data. In that case runtime ranges will be mapped
 945  * to the same virtual addresses as the first kernel, see
 946  * kexec_enter_virtual_mode().
 947  */
 948 static void __init __efi_enter_virtual_mode(void)
 949 {
 950         int count = 0, pg_shift = 0;
 951         void *new_memmap = NULL;
 952         efi_status_t status;
 953         unsigned long pa;
 954
 955         efi.systab = NULL;
 956
 957         if (efi_alloc_page_tables()) {
 958                 pr_err("Failed to allocate EFI page tables\n");
 959                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 960                 return;
 961         }
 962
 963         efi_merge_regions();
 964         new_memmap = efi_map_regions(&count, &pg_shift);
 965         if (!new_memmap) {
 966                 pr_err("Error reallocating memory, EFI runtime non-functional!\n");
 967                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 968                 return;
 969         }
 970
 971         pa = __pa(new_memmap);
 972
 973         /*
 974          * Unregister the early EFI memmap from efi_init() and install
 975          * the new EFI memory map that we are about to pass to the
 976          * firmware via SetVirtualAddressMap().
 977          */
 978         efi_memmap_unmap();
 979
 980         if (efi_memmap_init_late(pa, efi.memmap.desc_size * count)) {
 981                 pr_err("Failed to remap late EFI memory map\n");
 982                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 983                 return;
 984         }
 985
 986         if (efi_enabled(EFI_DBG)) {
 987                 pr_info("EFI runtime memory map:\n");
 988                 efi_print_memmap();
 989         }
 990
 991         BUG_ON(!efi.systab);
 992
 993         if (efi_setup_page_tables(pa, 1 << pg_shift)) {
 994                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 995                 return;
 996         }
 997
 998         efi_sync_low_kernel_mappings();
 999
1000         if (efi_is_native()) {
1001                 status = phys_efi_set_virtual_address_map(
1002                                 efi.memmap.desc_size * count,
1003                                 efi.memmap.desc_size,
1004                                 efi.memmap.desc_version,
1005                                 (efi_memory_desc_t *)pa);
1006         } else {
1007                 status = efi_thunk_set_virtual_address_map(
1008                                 efi_phys.set_virtual_address_map,
1009                                 efi.memmap.desc_size * count,
1010                                 efi.memmap.desc_size,
1011                                 efi.memmap.desc_version,
1012                                 (efi_memory_desc_t *)pa);
1013         }
1014
1015         if (status != EFI_SUCCESS) {
1016                 pr_alert("Unable to switch EFI into virtual mode (status=%lx)!\n",
1017                          status);
1018                 panic("EFI call to SetVirtualAddressMap() failed!");
1019         }
1020
1021         efi_free_boot_services();
1022
1023         /*
1024          * Now that EFI is in virtual mode, update the function
1025          * pointers in the runtime service table to the new virtual addresses.
1026          *
1027          * Call EFI services through wrapper functions.
1028          */
1029         efi.runtime_version = efi_systab.hdr.revision;
1030
1031         if (efi_is_native())
1032                 efi_native_runtime_setup();
1033         else
1034                 efi_thunk_runtime_setup();
1035
1036         efi.set_virtual_address_map = NULL;
1037
1038         /*
1039          * Apply more restrictive page table mapping attributes now that
1040          * SVAM() has been called and the firmware has performed all
1041          * necessary relocation fixups for the new virtual addresses.
1042          */
1043         efi_runtime_update_mappings();
1044
1045         /* clean DUMMY object */
1046         efi_delete_dummy_variable();
1047 }
1048
1049 void __init efi_enter_virtual_mode(void)
1050 {
1051         if (efi_enabled(EFI_PARAVIRT))
1052                 return;
1053
1054         if (efi_setup)
1055                 kexec_enter_virtual_mode();
1056         else
1057                 __efi_enter_virtual_mode();
1058
1059         efi_dump_pagetable();
1060 }
1061
1062 static int __init arch_parse_efi_cmdline(char *str)
1063 {
1064         if (!str) {
1065                 pr_warn("need at least one option\n");
1066                 return -EINVAL;
1067         }
1068
1069         if (parse_option_str(str, "old_map"))
1070                 set_bit(EFI_OLD_MEMMAP, &efi.flags);
1071
1072         return 0;
1073 }
1074 early_param("efi", arch_parse_efi_cmdline);
1075
1076 bool efi_is_table_address(unsigned long phys_addr)
1077 {
1078         unsigned int i;
1079
1080         if (phys_addr == EFI_INVALID_TABLE_ADDR)
1081                 return false;
1082
1083         for (i = 0; i < ARRAY_SIZE(efi_tables); i++)
1084                 if (*(efi_tables[i]) == phys_addr)
1085                         return true;
1086
1087         return false;
1088 }