1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /* Kernel module help for x86.
3 Copyright (C) 2001 Rusty Russell.
7 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
9 #include <linux/moduleloader.h>
10 #include <linux/elf.h>
11 #include <linux/vmalloc.h>
13 #include <linux/string.h>
14 #include <linux/kernel.h>
15 #include <linux/kasan.h>
16 #include <linux/bug.h>
18 #include <linux/gfp.h>
19 #include <linux/jump_label.h>
20 #include <linux/random.h>
21 #include <linux/memory.h>
23 #include <asm/text-patching.h>
25 #include <asm/pgtable.h>
26 #include <asm/setup.h>
27 #include <asm/unwind.h>
30 #define DEBUGP(fmt, ...) \
31 printk(KERN_DEBUG fmt, ##__VA_ARGS__)
33 #define DEBUGP(fmt, ...) \
36 printk(KERN_DEBUG fmt, ##__VA_ARGS__); \
40 #ifdef CONFIG_RANDOMIZE_BASE
41 static unsigned long module_load_offset;
43 /* Mutex protects the module_load_offset. */
44 static DEFINE_MUTEX(module_kaslr_mutex);
46 static unsigned long int get_module_load_offset(void)
48 if (kaslr_enabled()) {
49 mutex_lock(&module_kaslr_mutex);
51 * Calculate the module_load_offset the first time this
52 * code is called. Once calculated it stays the same until
55 if (module_load_offset == 0)
57 (get_random_int() % 1024 + 1) * PAGE_SIZE;
58 mutex_unlock(&module_kaslr_mutex);
60 return module_load_offset;
63 static unsigned long int get_module_load_offset(void)
69 void *module_alloc(unsigned long size)
73 if (PAGE_ALIGN(size) > MODULES_LEN)
76 p = __vmalloc_node_range(size, MODULE_ALIGN,
77 MODULES_VADDR + get_module_load_offset(),
78 MODULES_END, GFP_KERNEL,
79 PAGE_KERNEL, 0, NUMA_NO_NODE,
80 __builtin_return_address(0));
81 if (p && (kasan_module_alloc(p, size) < 0)) {
90 int apply_relocate(Elf32_Shdr *sechdrs,
92 unsigned int symindex,
97 Elf32_Rel *rel = (void *)sechdrs[relsec].sh_addr;
101 DEBUGP("Applying relocate section %u to %u\n",
102 relsec, sechdrs[relsec].sh_info);
103 for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rel); i++) {
104 /* This is where to make the change */
105 location = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr
107 /* This is the symbol it is referring to. Note that all
108 undefined symbols have been resolved. */
109 sym = (Elf32_Sym *)sechdrs[symindex].sh_addr
110 + ELF32_R_SYM(rel[i].r_info);
112 switch (ELF32_R_TYPE(rel[i].r_info)) {
114 /* We add the value into the location given */
115 *location += sym->st_value;
118 /* Add the value, subtract its position */
119 *location += sym->st_value - (uint32_t)location;
122 pr_err("%s: Unknown relocation: %u\n",
123 me->name, ELF32_R_TYPE(rel[i].r_info));
130 static int __apply_relocate_add(Elf64_Shdr *sechdrs,
132 unsigned int symindex,
135 void *(*write)(void *dest, const void *src, size_t len))
138 Elf64_Rela *rel = (void *)sechdrs[relsec].sh_addr;
143 DEBUGP("Applying relocate section %u to %u\n",
144 relsec, sechdrs[relsec].sh_info);
145 for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rel); i++) {
146 /* This is where to make the change */
147 loc = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr
150 /* This is the symbol it is referring to. Note that all
151 undefined symbols have been resolved. */
152 sym = (Elf64_Sym *)sechdrs[symindex].sh_addr
153 + ELF64_R_SYM(rel[i].r_info);
155 DEBUGP("type %d st_value %Lx r_addend %Lx loc %Lx\n",
156 (int)ELF64_R_TYPE(rel[i].r_info),
157 sym->st_value, rel[i].r_addend, (u64)loc);
159 val = sym->st_value + rel[i].r_addend;
161 switch (ELF64_R_TYPE(rel[i].r_info)) {
165 if (*(u64 *)loc != 0)
166 goto invalid_relocation;
170 if (*(u32 *)loc != 0)
171 goto invalid_relocation;
173 if (val != *(u32 *)loc)
177 if (*(s32 *)loc != 0)
178 goto invalid_relocation;
180 if ((s64)val != *(s32 *)loc)
185 if (*(u32 *)loc != 0)
186 goto invalid_relocation;
190 if ((s64)val != *(s32 *)loc)
195 if (*(u64 *)loc != 0)
196 goto invalid_relocation;
201 pr_err("%s: Unknown rela relocation: %llu\n",
202 me->name, ELF64_R_TYPE(rel[i].r_info));
209 pr_err("x86/modules: Skipping invalid relocation target, existing value is nonzero for type %d, loc %p, val %Lx\n",
210 (int)ELF64_R_TYPE(rel[i].r_info), loc, val);
214 pr_err("overflow in relocation type %d val %Lx\n",
215 (int)ELF64_R_TYPE(rel[i].r_info), val);
216 pr_err("`%s' likely not compiled with -mcmodel=kernel\n",
221 int apply_relocate_add(Elf64_Shdr *sechdrs,
223 unsigned int symindex,
228 bool early = me->state == MODULE_STATE_UNFORMED;
229 void *(*write)(void *, const void *, size_t) = memcpy;
233 mutex_lock(&text_mutex);
236 ret = __apply_relocate_add(sechdrs, strtab, symindex, relsec, me,
241 mutex_unlock(&text_mutex);
249 int module_finalize(const Elf_Ehdr *hdr,
250 const Elf_Shdr *sechdrs,
253 const Elf_Shdr *s, *text = NULL, *alt = NULL, *locks = NULL,
254 *para = NULL, *orc = NULL, *orc_ip = NULL;
255 char *secstrings = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset;
257 for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) {
258 if (!strcmp(".text", secstrings + s->sh_name))
260 if (!strcmp(".altinstructions", secstrings + s->sh_name))
262 if (!strcmp(".smp_locks", secstrings + s->sh_name))
264 if (!strcmp(".parainstructions", secstrings + s->sh_name))
266 if (!strcmp(".orc_unwind", secstrings + s->sh_name))
268 if (!strcmp(".orc_unwind_ip", secstrings + s->sh_name))
273 /* patch .altinstructions */
274 void *aseg = (void *)alt->sh_addr;
275 apply_alternatives(aseg, aseg + alt->sh_size);
278 void *lseg = (void *)locks->sh_addr;
279 void *tseg = (void *)text->sh_addr;
280 alternatives_smp_module_add(me, me->name,
281 lseg, lseg + locks->sh_size,
282 tseg, tseg + text->sh_size);
286 void *pseg = (void *)para->sh_addr;
287 apply_paravirt(pseg, pseg + para->sh_size);
290 /* make jump label nops */
291 jump_label_apply_nops(me);
294 unwind_module_init(me, (void *)orc_ip->sh_addr, orc_ip->sh_size,
295 (void *)orc->sh_addr, orc->sh_size);
300 void module_arch_cleanup(struct module *mod)
302 alternatives_smp_module_del(mod);