1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | /* Kernel module help for x86. |
3 | Copyright (C) 2001 Rusty Russell. |
4 | |
5 | */ |
6 | |
7 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt |
8 | |
9 | #include <linux/moduleloader.h> |
10 | #include <linux/elf.h> |
11 | #include <linux/vmalloc.h> |
12 | #include <linux/fs.h> |
13 | #include <linux/string.h> |
14 | #include <linux/kernel.h> |
15 | #include <linux/kasan.h> |
16 | #include <linux/bug.h> |
17 | #include <linux/mm.h> |
18 | #include <linux/gfp.h> |
19 | #include <linux/jump_label.h> |
20 | #include <linux/random.h> |
21 | #include <linux/memory.h> |
22 | |
23 | #include <asm/text-patching.h> |
24 | #include <asm/page.h> |
25 | #include <asm/setup.h> |
26 | #include <asm/unwind.h> |
27 | |
28 | #if 0 |
29 | #define DEBUGP(fmt, ...) \ |
30 | printk(KERN_DEBUG fmt, ##__VA_ARGS__) |
31 | #else |
32 | #define DEBUGP(fmt, ...) \ |
33 | do { \ |
34 | if (0) \ |
35 | printk(KERN_DEBUG fmt, ##__VA_ARGS__); \ |
36 | } while (0) |
37 | #endif |
38 | |
39 | #ifdef CONFIG_RANDOMIZE_BASE |
40 | static unsigned long module_load_offset; |
41 | |
42 | /* Mutex protects the module_load_offset. */ |
43 | static DEFINE_MUTEX(module_kaslr_mutex); |
44 | |
45 | static unsigned long int get_module_load_offset(void) |
46 | { |
47 | if (kaslr_enabled()) { |
48 | mutex_lock(&module_kaslr_mutex); |
49 | /* |
50 | * Calculate the module_load_offset the first time this |
51 | * code is called. Once calculated it stays the same until |
52 | * reboot. |
53 | */ |
54 | if (module_load_offset == 0) |
55 | module_load_offset = |
56 | get_random_u32_inclusive(floor: 1, ceil: 1024) * PAGE_SIZE; |
57 | mutex_unlock(lock: &module_kaslr_mutex); |
58 | } |
59 | return module_load_offset; |
60 | } |
61 | #else |
62 | static unsigned long int get_module_load_offset(void) |
63 | { |
64 | return 0; |
65 | } |
66 | #endif |
67 | |
68 | void *module_alloc(unsigned long size) |
69 | { |
70 | gfp_t gfp_mask = GFP_KERNEL; |
71 | void *p; |
72 | |
73 | if (PAGE_ALIGN(size) > MODULES_LEN) |
74 | return NULL; |
75 | |
76 | p = __vmalloc_node_range(size, MODULE_ALIGN, |
77 | MODULES_VADDR + get_module_load_offset(), |
78 | MODULES_END, gfp_mask, PAGE_KERNEL, |
79 | VM_FLUSH_RESET_PERMS | VM_DEFER_KMEMLEAK, |
80 | NUMA_NO_NODE, caller: __builtin_return_address(0)); |
81 | |
82 | if (p && (kasan_alloc_module_shadow(addr: p, size, gfp_mask) < 0)) { |
83 | vfree(addr: p); |
84 | return NULL; |
85 | } |
86 | |
87 | return p; |
88 | } |
89 | |
90 | #ifdef CONFIG_X86_32 |
91 | int apply_relocate(Elf32_Shdr *sechdrs, |
92 | const char *strtab, |
93 | unsigned int symindex, |
94 | unsigned int relsec, |
95 | struct module *me) |
96 | { |
97 | unsigned int i; |
98 | Elf32_Rel *rel = (void *)sechdrs[relsec].sh_addr; |
99 | Elf32_Sym *sym; |
100 | uint32_t *location; |
101 | |
102 | DEBUGP("Applying relocate section %u to %u\n" , |
103 | relsec, sechdrs[relsec].sh_info); |
104 | for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rel); i++) { |
105 | /* This is where to make the change */ |
106 | location = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr |
107 | + rel[i].r_offset; |
108 | /* This is the symbol it is referring to. Note that all |
109 | undefined symbols have been resolved. */ |
110 | sym = (Elf32_Sym *)sechdrs[symindex].sh_addr |
111 | + ELF32_R_SYM(rel[i].r_info); |
112 | |
113 | switch (ELF32_R_TYPE(rel[i].r_info)) { |
114 | case R_386_32: |
115 | /* We add the value into the location given */ |
116 | *location += sym->st_value; |
117 | break; |
118 | case R_386_PC32: |
119 | case R_386_PLT32: |
120 | /* Add the value, subtract its position */ |
121 | *location += sym->st_value - (uint32_t)location; |
122 | break; |
123 | default: |
124 | pr_err("%s: Unknown relocation: %u\n" , |
125 | me->name, ELF32_R_TYPE(rel[i].r_info)); |
126 | return -ENOEXEC; |
127 | } |
128 | } |
129 | return 0; |
130 | } |
131 | #else /*X86_64*/ |
132 | static int __write_relocate_add(Elf64_Shdr *sechdrs, |
133 | const char *strtab, |
134 | unsigned int symindex, |
135 | unsigned int relsec, |
136 | struct module *me, |
137 | void *(*write)(void *dest, const void *src, size_t len), |
138 | bool apply) |
139 | { |
140 | unsigned int i; |
141 | Elf64_Rela *rel = (void *)sechdrs[relsec].sh_addr; |
142 | Elf64_Sym *sym; |
143 | void *loc; |
144 | u64 val; |
145 | u64 zero = 0ULL; |
146 | |
147 | DEBUGP("%s relocate section %u to %u\n" , |
148 | apply ? "Applying" : "Clearing" , |
149 | relsec, sechdrs[relsec].sh_info); |
150 | for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rel); i++) { |
151 | size_t size; |
152 | |
153 | /* This is where to make the change */ |
154 | loc = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr |
155 | + rel[i].r_offset; |
156 | |
157 | /* This is the symbol it is referring to. Note that all |
158 | undefined symbols have been resolved. */ |
159 | sym = (Elf64_Sym *)sechdrs[symindex].sh_addr |
160 | + ELF64_R_SYM(rel[i].r_info); |
161 | |
162 | DEBUGP("type %d st_value %Lx r_addend %Lx loc %Lx\n" , |
163 | (int)ELF64_R_TYPE(rel[i].r_info), |
164 | sym->st_value, rel[i].r_addend, (u64)loc); |
165 | |
166 | val = sym->st_value + rel[i].r_addend; |
167 | |
168 | switch (ELF64_R_TYPE(rel[i].r_info)) { |
169 | case R_X86_64_NONE: |
170 | continue; /* nothing to write */ |
171 | case R_X86_64_64: |
172 | size = 8; |
173 | break; |
174 | case R_X86_64_32: |
175 | if (val != *(u32 *)&val) |
176 | goto overflow; |
177 | size = 4; |
178 | break; |
179 | case R_X86_64_32S: |
180 | if ((s64)val != *(s32 *)&val) |
181 | goto overflow; |
182 | size = 4; |
183 | break; |
184 | case R_X86_64_PC32: |
185 | case R_X86_64_PLT32: |
186 | val -= (u64)loc; |
187 | size = 4; |
188 | break; |
189 | case R_X86_64_PC64: |
190 | val -= (u64)loc; |
191 | size = 8; |
192 | break; |
193 | default: |
194 | pr_err("%s: Unknown rela relocation: %llu\n" , |
195 | me->name, ELF64_R_TYPE(rel[i].r_info)); |
196 | return -ENOEXEC; |
197 | } |
198 | |
199 | if (apply) { |
200 | if (memcmp(p: loc, q: &zero, size)) { |
201 | pr_err("x86/modules: Invalid relocation target, existing value is nonzero for type %d, loc %p, val %Lx\n" , |
202 | (int)ELF64_R_TYPE(rel[i].r_info), loc, val); |
203 | return -ENOEXEC; |
204 | } |
205 | write(loc, &val, size); |
206 | } else { |
207 | if (memcmp(p: loc, q: &val, size)) { |
208 | pr_warn("x86/modules: Invalid relocation target, existing value does not match expected value for type %d, loc %p, val %Lx\n" , |
209 | (int)ELF64_R_TYPE(rel[i].r_info), loc, val); |
210 | return -ENOEXEC; |
211 | } |
212 | write(loc, &zero, size); |
213 | } |
214 | } |
215 | return 0; |
216 | |
217 | overflow: |
218 | pr_err("overflow in relocation type %d val %Lx\n" , |
219 | (int)ELF64_R_TYPE(rel[i].r_info), val); |
220 | pr_err("`%s' likely not compiled with -mcmodel=kernel\n" , |
221 | me->name); |
222 | return -ENOEXEC; |
223 | } |
224 | |
225 | static int write_relocate_add(Elf64_Shdr *sechdrs, |
226 | const char *strtab, |
227 | unsigned int symindex, |
228 | unsigned int relsec, |
229 | struct module *me, |
230 | bool apply) |
231 | { |
232 | int ret; |
233 | bool early = me->state == MODULE_STATE_UNFORMED; |
234 | void *(*write)(void *, const void *, size_t) = memcpy; |
235 | |
236 | if (!early) { |
237 | write = text_poke; |
238 | mutex_lock(&text_mutex); |
239 | } |
240 | |
241 | ret = __write_relocate_add(sechdrs, strtab, symindex, relsec, me, |
242 | write, apply); |
243 | |
244 | if (!early) { |
245 | text_poke_sync(); |
246 | mutex_unlock(lock: &text_mutex); |
247 | } |
248 | |
249 | return ret; |
250 | } |
251 | |
252 | int apply_relocate_add(Elf64_Shdr *sechdrs, |
253 | const char *strtab, |
254 | unsigned int symindex, |
255 | unsigned int relsec, |
256 | struct module *me) |
257 | { |
258 | return write_relocate_add(sechdrs, strtab, symindex, relsec, me, apply: true); |
259 | } |
260 | |
261 | #ifdef CONFIG_LIVEPATCH |
262 | void clear_relocate_add(Elf64_Shdr *sechdrs, |
263 | const char *strtab, |
264 | unsigned int symindex, |
265 | unsigned int relsec, |
266 | struct module *me) |
267 | { |
268 | write_relocate_add(sechdrs, strtab, symindex, relsec, me, apply: false); |
269 | } |
270 | #endif |
271 | |
272 | #endif |
273 | |
274 | int module_finalize(const Elf_Ehdr *hdr, |
275 | const Elf_Shdr *sechdrs, |
276 | struct module *me) |
277 | { |
278 | const Elf_Shdr *s, *alt = NULL, *locks = NULL, |
279 | *para = NULL, *orc = NULL, *orc_ip = NULL, |
280 | *retpolines = NULL, *returns = NULL, *ibt_endbr = NULL, |
281 | *calls = NULL, *cfi = NULL; |
282 | char *secstrings = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset; |
283 | |
284 | for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) { |
285 | if (!strcmp(".altinstructions" , secstrings + s->sh_name)) |
286 | alt = s; |
287 | if (!strcmp(".smp_locks" , secstrings + s->sh_name)) |
288 | locks = s; |
289 | if (!strcmp(".parainstructions" , secstrings + s->sh_name)) |
290 | para = s; |
291 | if (!strcmp(".orc_unwind" , secstrings + s->sh_name)) |
292 | orc = s; |
293 | if (!strcmp(".orc_unwind_ip" , secstrings + s->sh_name)) |
294 | orc_ip = s; |
295 | if (!strcmp(".retpoline_sites" , secstrings + s->sh_name)) |
296 | retpolines = s; |
297 | if (!strcmp(".return_sites" , secstrings + s->sh_name)) |
298 | returns = s; |
299 | if (!strcmp(".call_sites" , secstrings + s->sh_name)) |
300 | calls = s; |
301 | if (!strcmp(".cfi_sites" , secstrings + s->sh_name)) |
302 | cfi = s; |
303 | if (!strcmp(".ibt_endbr_seal" , secstrings + s->sh_name)) |
304 | ibt_endbr = s; |
305 | } |
306 | |
307 | /* |
308 | * See alternative_instructions() for the ordering rules between the |
309 | * various patching types. |
310 | */ |
311 | if (para) { |
312 | void *pseg = (void *)para->sh_addr; |
313 | apply_paravirt(start: pseg, end: pseg + para->sh_size); |
314 | } |
315 | if (retpolines || cfi) { |
316 | void *rseg = NULL, *cseg = NULL; |
317 | unsigned int rsize = 0, csize = 0; |
318 | |
319 | if (retpolines) { |
320 | rseg = (void *)retpolines->sh_addr; |
321 | rsize = retpolines->sh_size; |
322 | } |
323 | |
324 | if (cfi) { |
325 | cseg = (void *)cfi->sh_addr; |
326 | csize = cfi->sh_size; |
327 | } |
328 | |
329 | apply_fineibt(start_retpoline: rseg, end_retpoine: rseg + rsize, start_cfi: cseg, end_cfi: cseg + csize); |
330 | } |
331 | if (retpolines) { |
332 | void *rseg = (void *)retpolines->sh_addr; |
333 | apply_retpolines(start: rseg, end: rseg + retpolines->sh_size); |
334 | } |
335 | if (returns) { |
336 | void *rseg = (void *)returns->sh_addr; |
337 | apply_returns(start: rseg, end: rseg + returns->sh_size); |
338 | } |
339 | if (alt) { |
340 | /* patch .altinstructions */ |
341 | void *aseg = (void *)alt->sh_addr; |
342 | apply_alternatives(start: aseg, end: aseg + alt->sh_size); |
343 | } |
344 | if (calls || para) { |
345 | struct callthunk_sites cs = {}; |
346 | |
347 | if (calls) { |
348 | cs.call_start = (void *)calls->sh_addr; |
349 | cs.call_end = (void *)calls->sh_addr + calls->sh_size; |
350 | } |
351 | |
352 | if (para) { |
353 | cs.pv_start = (void *)para->sh_addr; |
354 | cs.pv_end = (void *)para->sh_addr + para->sh_size; |
355 | } |
356 | |
357 | callthunks_patch_module_calls(sites: &cs, mod: me); |
358 | } |
359 | if (ibt_endbr) { |
360 | void *iseg = (void *)ibt_endbr->sh_addr; |
361 | apply_seal_endbr(start: iseg, end: iseg + ibt_endbr->sh_size); |
362 | } |
363 | if (locks) { |
364 | void *lseg = (void *)locks->sh_addr; |
365 | void *text = me->mem[MOD_TEXT].base; |
366 | void *text_end = text + me->mem[MOD_TEXT].size; |
367 | alternatives_smp_module_add(mod: me, name: me->name, |
368 | locks: lseg, locks_end: lseg + locks->sh_size, |
369 | text, text_end); |
370 | } |
371 | |
372 | if (orc && orc_ip) |
373 | unwind_module_init(mod: me, orc_ip: (void *)orc_ip->sh_addr, orc_ip_size: orc_ip->sh_size, |
374 | orc: (void *)orc->sh_addr, orc_size: orc->sh_size); |
375 | |
376 | return 0; |
377 | } |
378 | |
379 | void module_arch_cleanup(struct module *mod) |
380 | { |
381 | alternatives_smp_module_del(mod); |
382 | } |
383 | |