arch/powerpc/kernel/kexec_elf_64.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Load ELF vmlinux file for the kexec_file_load syscall.
   4  *
   5  * Copyright (C) 2004  Adam Litke (agl@us.ibm.com)
   6  * Copyright (C) 2004  IBM Corp.
   7  * Copyright (C) 2005  R Sharada (sharada@in.ibm.com)
   8  * Copyright (C) 2006  Mohan Kumar M (mohan@in.ibm.com)
   9  * Copyright (C) 2016  IBM Corporation
  10  *
  11  * Based on kexec-tools' kexec-elf-exec.c and kexec-elf-ppc64.c.
  12  * Heavily modified for the kernel by
  13  * Thiago Jung Bauermann <bauerman@linux.vnet.ibm.com>.
  14  */
  15
  16 #define pr_fmt(fmt)     "kexec_elf: " fmt
  17
  18 #include <linux/elf.h>
  19 #include <linux/kexec.h>
  20 #include <linux/libfdt.h>
  21 #include <linux/module.h>
  22 #include <linux/of_fdt.h>
  23 #include <linux/slab.h>
  24 #include <linux/types.h>
  25
  26 #define PURGATORY_STACK_SIZE    (16 * 1024)
  27
  28 #define elf_addr_to_cpu elf64_to_cpu
  29
  30 #ifndef Elf_Rel
  31 #define Elf_Rel         Elf64_Rel
  32 #endif /* Elf_Rel */
  33
  34 struct elf_info {
  35         /*
  36          * Where the ELF binary contents are kept.
  37          * Memory managed by the user of the struct.
  38          */
  39         const char *buffer;
  40
  41         const struct elfhdr *ehdr;
  42         const struct elf_phdr *proghdrs;
  43         struct elf_shdr *sechdrs;
  44 };
  45
  46 static inline bool elf_is_elf_file(const struct elfhdr *ehdr)
  47 {
  48        return memcmp(ehdr->e_ident, ELFMAG, SELFMAG) == 0;
  49 }
  50
  51 static uint64_t elf64_to_cpu(const struct elfhdr *ehdr, uint64_t value)
  52 {
  53         if (ehdr->e_ident[EI_DATA] == ELFDATA2LSB)
  54                 value = le64_to_cpu(value);
  55         else if (ehdr->e_ident[EI_DATA] == ELFDATA2MSB)
  56                 value = be64_to_cpu(value);
  57
  58         return value;
  59 }
  60
  61 static uint16_t elf16_to_cpu(const struct elfhdr *ehdr, uint16_t value)
  62 {
  63         if (ehdr->e_ident[EI_DATA] == ELFDATA2LSB)
  64                 value = le16_to_cpu(value);
  65         else if (ehdr->e_ident[EI_DATA] == ELFDATA2MSB)
  66                 value = be16_to_cpu(value);
  67
  68         return value;
  69 }
  70
  71 static uint32_t elf32_to_cpu(const struct elfhdr *ehdr, uint32_t value)
  72 {
  73         if (ehdr->e_ident[EI_DATA] == ELFDATA2LSB)
  74                 value = le32_to_cpu(value);
  75         else if (ehdr->e_ident[EI_DATA] == ELFDATA2MSB)
  76                 value = be32_to_cpu(value);
  77
  78         return value;
  79 }
  80
  81 /**
  82  * elf_is_ehdr_sane - check that it is safe to use the ELF header
  83  * @buf_len:    size of the buffer in which the ELF file is loaded.
  84  */
  85 static bool elf_is_ehdr_sane(const struct elfhdr *ehdr, size_t buf_len)
  86 {
  87         if (ehdr->e_phnum > 0 && ehdr->e_phentsize != sizeof(struct elf_phdr)) {
  88                 pr_debug("Bad program header size.\n");
  89                 return false;
  90         } else if (ehdr->e_shnum > 0 &&
  91                    ehdr->e_shentsize != sizeof(struct elf_shdr)) {
  92                 pr_debug("Bad section header size.\n");
  93                 return false;
  94         } else if (ehdr->e_ident[EI_VERSION] != EV_CURRENT ||
  95                    ehdr->e_version != EV_CURRENT) {
  96                 pr_debug("Unknown ELF version.\n");
  97                 return false;
  98         }
  99
 100         if (ehdr->e_phoff > 0 && ehdr->e_phnum > 0) {
 101                 size_t phdr_size;
 102
 103                 /*
 104                  * e_phnum is at most 65535 so calculating the size of the
 105                  * program header cannot overflow.
 106                  */
 107                 phdr_size = sizeof(struct elf_phdr) * ehdr->e_phnum;
 108
 109                 /* Sanity check the program header table location. */
 110                 if (ehdr->e_phoff + phdr_size < ehdr->e_phoff) {
 111                         pr_debug("Program headers at invalid location.\n");
 112                         return false;
 113                 } else if (ehdr->e_phoff + phdr_size > buf_len) {
 114                         pr_debug("Program headers truncated.\n");
 115                         return false;
 116                 }
 117         }
 118
 119         if (ehdr->e_shoff > 0 && ehdr->e_shnum > 0) {
 120                 size_t shdr_size;
 121
 122                 /*
 123                  * e_shnum is at most 65536 so calculating
 124                  * the size of the section header cannot overflow.
 125                  */
 126                 shdr_size = sizeof(struct elf_shdr) * ehdr->e_shnum;
 127
 128                 /* Sanity check the section header table location. */
 129                 if (ehdr->e_shoff + shdr_size < ehdr->e_shoff) {
 130                         pr_debug("Section headers at invalid location.\n");
 131                         return false;
 132                 } else if (ehdr->e_shoff + shdr_size > buf_len) {
 133                         pr_debug("Section headers truncated.\n");
 134                         return false;
 135                 }
 136         }
 137
 138         return true;
 139 }
 140
 141 static int elf_read_ehdr(const char *buf, size_t len, struct elfhdr *ehdr)
 142 {
 143         struct elfhdr *buf_ehdr;
 144
 145         if (len < sizeof(*buf_ehdr)) {
 146                 pr_debug("Buffer is too small to hold ELF header.\n");
 147                 return -ENOEXEC;
 148         }
 149
 150         memset(ehdr, 0, sizeof(*ehdr));
 151         memcpy(ehdr->e_ident, buf, sizeof(ehdr->e_ident));
 152         if (!elf_is_elf_file(ehdr)) {
 153                 pr_debug("No ELF header magic.\n");
 154                 return -ENOEXEC;
 155         }
 156
 157         if (ehdr->e_ident[EI_CLASS] != ELF_CLASS) {
 158                 pr_debug("Not a supported ELF class.\n");
 159                 return -ENOEXEC;
 160         } else  if (ehdr->e_ident[EI_DATA] != ELFDATA2LSB &&
 161                 ehdr->e_ident[EI_DATA] != ELFDATA2MSB) {
 162                 pr_debug("Not a supported ELF data format.\n");
 163                 return -ENOEXEC;
 164         }
 165
 166         buf_ehdr = (struct elfhdr *) buf;
 167         if (elf16_to_cpu(ehdr, buf_ehdr->e_ehsize) != sizeof(*buf_ehdr)) {
 168                 pr_debug("Bad ELF header size.\n");
 169                 return -ENOEXEC;
 170         }
 171
 172         ehdr->e_type      = elf16_to_cpu(ehdr, buf_ehdr->e_type);
 173         ehdr->e_machine   = elf16_to_cpu(ehdr, buf_ehdr->e_machine);
 174         ehdr->e_version   = elf32_to_cpu(ehdr, buf_ehdr->e_version);
 175         ehdr->e_entry     = elf_addr_to_cpu(ehdr, buf_ehdr->e_entry);
 176         ehdr->e_phoff     = elf_addr_to_cpu(ehdr, buf_ehdr->e_phoff);
 177         ehdr->e_shoff     = elf_addr_to_cpu(ehdr, buf_ehdr->e_shoff);
 178         ehdr->e_flags     = elf32_to_cpu(ehdr, buf_ehdr->e_flags);
 179         ehdr->e_phentsize = elf16_to_cpu(ehdr, buf_ehdr->e_phentsize);
 180         ehdr->e_phnum     = elf16_to_cpu(ehdr, buf_ehdr->e_phnum);
 181         ehdr->e_shentsize = elf16_to_cpu(ehdr, buf_ehdr->e_shentsize);
 182         ehdr->e_shnum     = elf16_to_cpu(ehdr, buf_ehdr->e_shnum);
 183         ehdr->e_shstrndx  = elf16_to_cpu(ehdr, buf_ehdr->e_shstrndx);
 184
 185         return elf_is_ehdr_sane(ehdr, len) ? 0 : -ENOEXEC;
 186 }
 187
 188 /**
 189  * elf_is_phdr_sane - check that it is safe to use the program header
 190  * @buf_len:    size of the buffer in which the ELF file is loaded.
 191  */
 192 static bool elf_is_phdr_sane(const struct elf_phdr *phdr, size_t buf_len)
 193 {
 194
 195         if (phdr->p_offset + phdr->p_filesz < phdr->p_offset) {
 196                 pr_debug("ELF segment location wraps around.\n");
 197                 return false;
 198         } else if (phdr->p_offset + phdr->p_filesz > buf_len) {
 199                 pr_debug("ELF segment not in file.\n");
 200                 return false;
 201         } else if (phdr->p_paddr + phdr->p_memsz < phdr->p_paddr) {
 202                 pr_debug("ELF segment address wraps around.\n");
 203                 return false;
 204         }
 205
 206         return true;
 207 }
 208
 209 static int elf_read_phdr(const char *buf, size_t len, struct elf_info *elf_info,
 210                          int idx)
 211 {
 212         /* Override the const in proghdrs, we are the ones doing the loading. */
 213         struct elf_phdr *phdr = (struct elf_phdr *) &elf_info->proghdrs[idx];
 214         const char *pbuf;
 215         struct elf_phdr *buf_phdr;
 216
 217         pbuf = buf + elf_info->ehdr->e_phoff + (idx * sizeof(*buf_phdr));
 218         buf_phdr = (struct elf_phdr *) pbuf;
 219
 220         phdr->p_type   = elf32_to_cpu(elf_info->ehdr, buf_phdr->p_type);
 221         phdr->p_offset = elf_addr_to_cpu(elf_info->ehdr, buf_phdr->p_offset);
 222         phdr->p_paddr  = elf_addr_to_cpu(elf_info->ehdr, buf_phdr->p_paddr);
 223         phdr->p_vaddr  = elf_addr_to_cpu(elf_info->ehdr, buf_phdr->p_vaddr);
 224         phdr->p_flags  = elf32_to_cpu(elf_info->ehdr, buf_phdr->p_flags);
 225
 226         /*
 227          * The following fields have a type equivalent to Elf_Addr
 228          * both in 32 bit and 64 bit ELF.
 229          */
 230         phdr->p_filesz = elf_addr_to_cpu(elf_info->ehdr, buf_phdr->p_filesz);
 231         phdr->p_memsz  = elf_addr_to_cpu(elf_info->ehdr, buf_phdr->p_memsz);
 232         phdr->p_align  = elf_addr_to_cpu(elf_info->ehdr, buf_phdr->p_align);
 233
 234         return elf_is_phdr_sane(phdr, len) ? 0 : -ENOEXEC;
 235 }
 236
 237 /**
 238  * elf_read_phdrs - read the program headers from the buffer
 239  *
 240  * This function assumes that the program header table was checked for sanity.
 241  * Use elf_is_ehdr_sane() if it wasn't.
 242  */
 243 static int elf_read_phdrs(const char *buf, size_t len,
 244                           struct elf_info *elf_info)
 245 {
 246         size_t phdr_size, i;
 247         const struct elfhdr *ehdr = elf_info->ehdr;
 248
 249         /*
 250          * e_phnum is at most 65535 so calculating the size of the
 251          * program header cannot overflow.
 252          */
 253         phdr_size = sizeof(struct elf_phdr) * ehdr->e_phnum;
 254
 255         elf_info->proghdrs = kzalloc(phdr_size, GFP_KERNEL);
 256         if (!elf_info->proghdrs)
 257                 return -ENOMEM;
 258
 259         for (i = 0; i < ehdr->e_phnum; i++) {
 260                 int ret;
 261
 262                 ret = elf_read_phdr(buf, len, elf_info, i);
 263                 if (ret) {
 264                         kfree(elf_info->proghdrs);
 265                         elf_info->proghdrs = NULL;
 266                         return ret;
 267                 }
 268         }
 269
 270         return 0;
 271 }
 272
 273 /**
 274  * elf_is_shdr_sane - check that it is safe to use the section header
 275  * @buf_len:    size of the buffer in which the ELF file is loaded.
 276  */
 277 static bool elf_is_shdr_sane(const struct elf_shdr *shdr, size_t buf_len)
 278 {
 279         bool size_ok;
 280
 281         /* SHT_NULL headers have undefined values, so we can't check them. */
 282         if (shdr->sh_type == SHT_NULL)
 283                 return true;
 284
 285         /* Now verify sh_entsize */
 286         switch (shdr->sh_type) {
 287         case SHT_SYMTAB:
 288                 size_ok = shdr->sh_entsize == sizeof(Elf_Sym);
 289                 break;
 290         case SHT_RELA:
 291                 size_ok = shdr->sh_entsize == sizeof(Elf_Rela);
 292                 break;
 293         case SHT_DYNAMIC:
 294                 size_ok = shdr->sh_entsize == sizeof(Elf_Dyn);
 295                 break;
 296         case SHT_REL:
 297                 size_ok = shdr->sh_entsize == sizeof(Elf_Rel);
 298                 break;
 299         case SHT_NOTE:
 300         case SHT_PROGBITS:
 301         case SHT_HASH:
 302         case SHT_NOBITS:
 303         default:
 304                 /*
 305                  * This is a section whose entsize requirements
 306                  * I don't care about.  If I don't know about
 307                  * the section I can't care about it's entsize
 308                  * requirements.
 309                  */
 310                 size_ok = true;
 311                 break;
 312         }
 313
 314         if (!size_ok) {
 315                 pr_debug("ELF section with wrong entry size.\n");
 316                 return false;
 317         } else if (shdr->sh_addr + shdr->sh_size < shdr->sh_addr) {
 318                 pr_debug("ELF section address wraps around.\n");
 319                 return false;
 320         }
 321
 322         if (shdr->sh_type != SHT_NOBITS) {
 323                 if (shdr->sh_offset + shdr->sh_size < shdr->sh_offset) {
 324                         pr_debug("ELF section location wraps around.\n");
 325                         return false;
 326                 } else if (shdr->sh_offset + shdr->sh_size > buf_len) {
 327                         pr_debug("ELF section not in file.\n");
 328                         return false;
 329                 }
 330         }
 331
 332         return true;
 333 }
 334
 335 static int elf_read_shdr(const char *buf, size_t len, struct elf_info *elf_info,
 336                          int idx)
 337 {
 338         struct elf_shdr *shdr = &elf_info->sechdrs[idx];
 339         const struct elfhdr *ehdr = elf_info->ehdr;
 340         const char *sbuf;
 341         struct elf_shdr *buf_shdr;
 342
 343         sbuf = buf + ehdr->e_shoff + idx * sizeof(*buf_shdr);
 344         buf_shdr = (struct elf_shdr *) sbuf;
 345
 346         shdr->sh_name      = elf32_to_cpu(ehdr, buf_shdr->sh_name);
 347         shdr->sh_type      = elf32_to_cpu(ehdr, buf_shdr->sh_type);
 348         shdr->sh_addr      = elf_addr_to_cpu(ehdr, buf_shdr->sh_addr);
 349         shdr->sh_offset    = elf_addr_to_cpu(ehdr, buf_shdr->sh_offset);
 350         shdr->sh_link      = elf32_to_cpu(ehdr, buf_shdr->sh_link);
 351         shdr->sh_info      = elf32_to_cpu(ehdr, buf_shdr->sh_info);
 352
 353         /*
 354          * The following fields have a type equivalent to Elf_Addr
 355          * both in 32 bit and 64 bit ELF.
 356          */
 357         shdr->sh_flags     = elf_addr_to_cpu(ehdr, buf_shdr->sh_flags);
 358         shdr->sh_size      = elf_addr_to_cpu(ehdr, buf_shdr->sh_size);
 359         shdr->sh_addralign = elf_addr_to_cpu(ehdr, buf_shdr->sh_addralign);
 360         shdr->sh_entsize   = elf_addr_to_cpu(ehdr, buf_shdr->sh_entsize);
 361
 362         return elf_is_shdr_sane(shdr, len) ? 0 : -ENOEXEC;
 363 }
 364
 365 /**
 366  * elf_read_shdrs - read the section headers from the buffer
 367  *
 368  * This function assumes that the section header table was checked for sanity.
 369  * Use elf_is_ehdr_sane() if it wasn't.
 370  */
 371 static int elf_read_shdrs(const char *buf, size_t len,
 372                           struct elf_info *elf_info)
 373 {
 374         size_t shdr_size, i;
 375
 376         /*
 377          * e_shnum is at most 65536 so calculating
 378          * the size of the section header cannot overflow.
 379          */
 380         shdr_size = sizeof(struct elf_shdr) * elf_info->ehdr->e_shnum;
 381
 382         elf_info->sechdrs = kzalloc(shdr_size, GFP_KERNEL);
 383         if (!elf_info->sechdrs)
 384                 return -ENOMEM;
 385
 386         for (i = 0; i < elf_info->ehdr->e_shnum; i++) {
 387                 int ret;
 388
 389                 ret = elf_read_shdr(buf, len, elf_info, i);
 390                 if (ret) {
 391                         kfree(elf_info->sechdrs);
 392                         elf_info->sechdrs = NULL;
 393                         return ret;
 394                 }
 395         }
 396
 397         return 0;
 398 }
 399
 400 /**
 401  * elf_read_from_buffer - read ELF file and sets up ELF header and ELF info
 402  * @buf:        Buffer to read ELF file from.
 403  * @len:        Size of @buf.
 404  * @ehdr:       Pointer to existing struct which will be populated.
 405  * @elf_info:   Pointer to existing struct which will be populated.
 406  *
 407  * This function allows reading ELF files with different byte order than
 408  * the kernel, byte-swapping the fields as needed.
 409  *
 410  * Return:
 411  * On success returns 0, and the caller should call elf_free_info(elf_info) to
 412  * free the memory allocated for the section and program headers.
 413  */
 414 int elf_read_from_buffer(const char *buf, size_t len, struct elfhdr *ehdr,
 415                          struct elf_info *elf_info)
 416 {
 417         int ret;
 418
 419         ret = elf_read_ehdr(buf, len, ehdr);
 420         if (ret)
 421                 return ret;
 422
 423         elf_info->buffer = buf;
 424         elf_info->ehdr = ehdr;
 425         if (ehdr->e_phoff > 0 && ehdr->e_phnum > 0) {
 426                 ret = elf_read_phdrs(buf, len, elf_info);
 427                 if (ret)
 428                         return ret;
 429         }
 430         if (ehdr->e_shoff > 0 && ehdr->e_shnum > 0) {
 431                 ret = elf_read_shdrs(buf, len, elf_info);
 432                 if (ret) {
 433                         kfree(elf_info->proghdrs);
 434                         return ret;
 435                 }
 436         }
 437
 438         return 0;
 439 }
 440
 441 /**
 442  * elf_free_info - free memory allocated by elf_read_from_buffer
 443  */
 444 void elf_free_info(struct elf_info *elf_info)
 445 {
 446         kfree(elf_info->proghdrs);
 447         kfree(elf_info->sechdrs);
 448         memset(elf_info, 0, sizeof(*elf_info));
 449 }
 450 /**
 451  * build_elf_exec_info - read ELF executable and check that we can use it
 452  */
 453 static int build_elf_exec_info(const char *buf, size_t len, struct elfhdr *ehdr,
 454                                struct elf_info *elf_info)
 455 {
 456         int i;
 457         int ret;
 458
 459         ret = elf_read_from_buffer(buf, len, ehdr, elf_info);
 460         if (ret)
 461                 return ret;
 462
 463         /* Big endian vmlinux has type ET_DYN. */
 464         if (ehdr->e_type != ET_EXEC && ehdr->e_type != ET_DYN) {
 465                 pr_err("Not an ELF executable.\n");
 466                 goto error;
 467         } else if (!elf_info->proghdrs) {
 468                 pr_err("No ELF program header.\n");
 469                 goto error;
 470         }
 471
 472         for (i = 0; i < ehdr->e_phnum; i++) {
 473                 /*
 474                  * Kexec does not support loading interpreters.
 475                  * In addition this check keeps us from attempting
 476                  * to kexec ordinay executables.
 477                  */
 478                 if (elf_info->proghdrs[i].p_type == PT_INTERP) {
 479                         pr_err("Requires an ELF interpreter.\n");
 480                         goto error;
 481                 }
 482         }
 483
 484         return 0;
 485 error:
 486         elf_free_info(elf_info);
 487         return -ENOEXEC;
 488 }
 489
 490 static int elf64_probe(const char *buf, unsigned long len)
 491 {
 492         struct elfhdr ehdr;
 493         struct elf_info elf_info;
 494         int ret;
 495
 496         ret = build_elf_exec_info(buf, len, &ehdr, &elf_info);
 497         if (ret)
 498                 return ret;
 499
 500         elf_free_info(&elf_info);
 501
 502         return elf_check_arch(&ehdr) ? 0 : -ENOEXEC;
 503 }
 504
 505 /**
 506  * elf_exec_load - load ELF executable image
 507  * @lowest_load_addr:   On return, will be the address where the first PT_LOAD
 508  *                      section will be loaded in memory.
 509  *
 510  * Return:
 511  * 0 on success, negative value on failure.
 512  */
 513 static int elf_exec_load(struct kimage *image, struct elfhdr *ehdr,
 514                          struct elf_info *elf_info,
 515                          unsigned long *lowest_load_addr)
 516 {
 517         unsigned long base = 0, lowest_addr = UINT_MAX;
 518         int ret;
 519         size_t i;
 520         struct kexec_buf kbuf = { .image = image, .buf_max = ppc64_rma_size,
 521                                   .top_down = false };
 522
 523         /* Read in the PT_LOAD segments. */
 524         for (i = 0; i < ehdr->e_phnum; i++) {
 525                 unsigned long load_addr;
 526                 size_t size;
 527                 const struct elf_phdr *phdr;
 528
 529                 phdr = &elf_info->proghdrs[i];
 530                 if (phdr->p_type != PT_LOAD)
 531                         continue;
 532
 533                 size = phdr->p_filesz;
 534                 if (size > phdr->p_memsz)
 535                         size = phdr->p_memsz;
 536
 537                 kbuf.buffer = (void *) elf_info->buffer + phdr->p_offset;
 538                 kbuf.bufsz = size;
 539                 kbuf.memsz = phdr->p_memsz;
 540                 kbuf.buf_align = phdr->p_align;
 541                 kbuf.buf_min = phdr->p_paddr + base;
 542                 kbuf.mem = KEXEC_BUF_MEM_UNKNOWN;
 543                 ret = kexec_add_buffer(&kbuf);
 544                 if (ret)
 545                         goto out;
 546                 load_addr = kbuf.mem;
 547
 548                 if (load_addr < lowest_addr)
 549                         lowest_addr = load_addr;
 550         }
 551
 552         /* Update entry point to reflect new load address. */
 553         ehdr->e_entry += base;
 554
 555         *lowest_load_addr = lowest_addr;
 556         ret = 0;
 557  out:
 558         return ret;
 559 }
 560
 561 static void *elf64_load(struct kimage *image, char *kernel_buf,
 562                         unsigned long kernel_len, char *initrd,
 563                         unsigned long initrd_len, char *cmdline,
 564                         unsigned long cmdline_len)
 565 {
 566         int ret;
 567         unsigned int fdt_size;
 568         unsigned long kernel_load_addr;
 569         unsigned long initrd_load_addr = 0, fdt_load_addr;
 570         void *fdt;
 571         const void *slave_code;
 572         struct elfhdr ehdr;
 573         struct elf_info elf_info;
 574         struct kexec_buf kbuf = { .image = image, .buf_min = 0,
 575                                   .buf_max = ppc64_rma_size };
 576         struct kexec_buf pbuf = { .image = image, .buf_min = 0,
 577                                   .buf_max = ppc64_rma_size, .top_down = true,
 578                                   .mem = KEXEC_BUF_MEM_UNKNOWN };
 579
 580         ret = build_elf_exec_info(kernel_buf, kernel_len, &ehdr, &elf_info);
 581         if (ret)
 582                 goto out;
 583
 584         ret = elf_exec_load(image, &ehdr, &elf_info, &kernel_load_addr);
 585         if (ret)
 586                 goto out;
 587
 588         pr_debug("Loaded the kernel at 0x%lx\n", kernel_load_addr);
 589
 590         ret = kexec_load_purgatory(image, &pbuf);
 591         if (ret) {
 592                 pr_err("Loading purgatory failed.\n");
 593                 goto out;
 594         }
 595
 596         pr_debug("Loaded purgatory at 0x%lx\n", pbuf.mem);
 597
 598         if (initrd != NULL) {
 599                 kbuf.buffer = initrd;
 600                 kbuf.bufsz = kbuf.memsz = initrd_len;
 601                 kbuf.buf_align = PAGE_SIZE;
 602                 kbuf.top_down = false;
 603                 kbuf.mem = KEXEC_BUF_MEM_UNKNOWN;
 604                 ret = kexec_add_buffer(&kbuf);
 605                 if (ret)
 606                         goto out;
 607                 initrd_load_addr = kbuf.mem;
 608
 609                 pr_debug("Loaded initrd at 0x%lx\n", initrd_load_addr);
 610         }
 611
 612         fdt_size = fdt_totalsize(initial_boot_params) * 2;
 613         fdt = kmalloc(fdt_size, GFP_KERNEL);
 614         if (!fdt) {
 615                 pr_err("Not enough memory for the device tree.\n");
 616                 ret = -ENOMEM;
 617                 goto out;
 618         }
 619         ret = fdt_open_into(initial_boot_params, fdt, fdt_size);
 620         if (ret < 0) {
 621                 pr_err("Error setting up the new device tree.\n");
 622                 ret = -EINVAL;
 623                 goto out;
 624         }
 625
 626         ret = setup_new_fdt(image, fdt, initrd_load_addr, initrd_len, cmdline);
 627         if (ret)
 628                 goto out;
 629
 630         fdt_pack(fdt);
 631
 632         kbuf.buffer = fdt;
 633         kbuf.bufsz = kbuf.memsz = fdt_size;
 634         kbuf.buf_align = PAGE_SIZE;
 635         kbuf.top_down = true;
 636         kbuf.mem = KEXEC_BUF_MEM_UNKNOWN;
 637         ret = kexec_add_buffer(&kbuf);
 638         if (ret)
 639                 goto out;
 640         fdt_load_addr = kbuf.mem;
 641
 642         pr_debug("Loaded device tree at 0x%lx\n", fdt_load_addr);
 643
 644         slave_code = elf_info.buffer + elf_info.proghdrs[0].p_offset;
 645         ret = setup_purgatory(image, slave_code, fdt, kernel_load_addr,
 646                               fdt_load_addr);
 647         if (ret)
 648                 pr_err("Error setting up the purgatory.\n");
 649
 650 out:
 651         elf_free_info(&elf_info);
 652
 653         /* Make kimage_file_post_load_cleanup free the fdt buffer for us. */
 654         return ret ? ERR_PTR(ret) : fdt;
 655 }
 656
 657 const struct kexec_file_ops kexec_elf64_ops = {
 658         .probe = elf64_probe,
 659         .load = elf64_load,
 660 };