1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * linux/arch/arm/kernel/module.c |
4 | * |
5 | * Copyright (C) 2002 Russell King. |
6 | * Modified for nommu by Hyok S. Choi |
7 | * |
8 | * Module allocation method suggested by Andi Kleen. |
9 | */ |
10 | #include <linux/module.h> |
11 | #include <linux/moduleloader.h> |
12 | #include <linux/kernel.h> |
13 | #include <linux/mm.h> |
14 | #include <linux/elf.h> |
15 | #include <linux/vmalloc.h> |
16 | #include <linux/fs.h> |
17 | #include <linux/string.h> |
18 | #include <linux/gfp.h> |
19 | |
20 | #include <asm/sections.h> |
21 | #include <asm/smp_plat.h> |
22 | #include <asm/unwind.h> |
23 | #include <asm/opcodes.h> |
24 | |
25 | #ifdef CONFIG_XIP_KERNEL |
26 | /* |
27 | * The XIP kernel text is mapped in the module area for modules and |
28 | * some other stuff to work without any indirect relocations. |
29 | * MODULES_VADDR is redefined here and not in asm/memory.h to avoid |
30 | * recompiling the whole kernel when CONFIG_XIP_KERNEL is turned on/off. |
31 | */ |
32 | #undef MODULES_VADDR |
33 | #define MODULES_VADDR (((unsigned long)_exiprom + ~PMD_MASK) & PMD_MASK) |
34 | #endif |
35 | |
36 | #ifdef CONFIG_MMU |
37 | void *module_alloc(unsigned long size) |
38 | { |
39 | gfp_t gfp_mask = GFP_KERNEL; |
40 | void *p; |
41 | |
42 | /* Silence the initial allocation */ |
43 | if (IS_ENABLED(CONFIG_ARM_MODULE_PLTS)) |
44 | gfp_mask |= __GFP_NOWARN; |
45 | |
46 | p = __vmalloc_node_range(size, align: 1, MODULES_VADDR, MODULES_END, |
47 | gfp_mask, PAGE_KERNEL_EXEC, vm_flags: 0, NUMA_NO_NODE, |
48 | caller: __builtin_return_address(0)); |
49 | if (!IS_ENABLED(CONFIG_ARM_MODULE_PLTS) || p) |
50 | return p; |
51 | return __vmalloc_node_range(size, align: 1, VMALLOC_START, VMALLOC_END, |
52 | GFP_KERNEL, PAGE_KERNEL_EXEC, vm_flags: 0, NUMA_NO_NODE, |
53 | caller: __builtin_return_address(0)); |
54 | } |
55 | #endif |
56 | |
57 | bool module_init_section(const char *name) |
58 | { |
59 | return strstarts(str: name, prefix: ".init" ) || |
60 | strstarts(str: name, prefix: ".ARM.extab.init" ) || |
61 | strstarts(str: name, prefix: ".ARM.exidx.init" ); |
62 | } |
63 | |
64 | bool module_exit_section(const char *name) |
65 | { |
66 | return strstarts(str: name, prefix: ".exit" ) || |
67 | strstarts(str: name, prefix: ".ARM.extab.exit" ) || |
68 | strstarts(str: name, prefix: ".ARM.exidx.exit" ); |
69 | } |
70 | |
71 | #ifdef CONFIG_ARM_HAS_GROUP_RELOCS |
72 | /* |
73 | * This implements the partitioning algorithm for group relocations as |
74 | * documented in the ARM AArch32 ELF psABI (IHI 0044). |
75 | * |
76 | * A single PC-relative symbol reference is divided in up to 3 add or subtract |
77 | * operations, where the final one could be incorporated into a load/store |
78 | * instruction with immediate offset. E.g., |
79 | * |
80 | * ADD Rd, PC, #... or ADD Rd, PC, #... |
81 | * ADD Rd, Rd, #... ADD Rd, Rd, #... |
82 | * LDR Rd, [Rd, #...] ADD Rd, Rd, #... |
83 | * |
84 | * The latter has a guaranteed range of only 16 MiB (3x8 == 24 bits), so it is |
85 | * of limited use in the kernel. However, the ADD/ADD/LDR combo has a range of |
86 | * -/+ 256 MiB, (2x8 + 12 == 28 bits), which means it has sufficient range for |
87 | * any in-kernel symbol reference (unless module PLTs are being used). |
88 | * |
89 | * The main advantage of this approach over the typical pattern using a literal |
90 | * load is that literal loads may miss in the D-cache, and generally lead to |
91 | * lower cache efficiency for variables that are referenced often from many |
92 | * different places in the code. |
93 | */ |
94 | static u32 get_group_rem(u32 group, u32 *offset) |
95 | { |
96 | u32 val = *offset; |
97 | u32 shift; |
98 | do { |
99 | shift = val ? (31 - __fls(val)) & ~1 : 32; |
100 | *offset = val; |
101 | if (!val) |
102 | break; |
103 | val &= 0xffffff >> shift; |
104 | } while (group--); |
105 | return shift; |
106 | } |
107 | #endif |
108 | |
109 | int |
110 | apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex, |
111 | unsigned int relindex, struct module *module) |
112 | { |
113 | Elf32_Shdr *symsec = sechdrs + symindex; |
114 | Elf32_Shdr *relsec = sechdrs + relindex; |
115 | Elf32_Shdr *dstsec = sechdrs + relsec->sh_info; |
116 | Elf32_Rel *rel = (void *)relsec->sh_addr; |
117 | unsigned int i; |
118 | |
119 | for (i = 0; i < relsec->sh_size / sizeof(Elf32_Rel); i++, rel++) { |
120 | unsigned long loc; |
121 | Elf32_Sym *sym; |
122 | const char *symname; |
123 | #ifdef CONFIG_ARM_HAS_GROUP_RELOCS |
124 | u32 shift, group = 1; |
125 | #endif |
126 | s32 offset; |
127 | u32 tmp; |
128 | #ifdef CONFIG_THUMB2_KERNEL |
129 | u32 upper, lower, sign, j1, j2; |
130 | #endif |
131 | |
132 | offset = ELF32_R_SYM(rel->r_info); |
133 | if (offset < 0 || offset > (symsec->sh_size / sizeof(Elf32_Sym))) { |
134 | pr_err("%s: section %u reloc %u: bad relocation sym offset\n" , |
135 | module->name, relindex, i); |
136 | return -ENOEXEC; |
137 | } |
138 | |
139 | sym = ((Elf32_Sym *)symsec->sh_addr) + offset; |
140 | symname = strtab + sym->st_name; |
141 | |
142 | if (rel->r_offset < 0 || rel->r_offset > dstsec->sh_size - sizeof(u32)) { |
143 | pr_err("%s: section %u reloc %u sym '%s': out of bounds relocation, offset %d size %u\n" , |
144 | module->name, relindex, i, symname, |
145 | rel->r_offset, dstsec->sh_size); |
146 | return -ENOEXEC; |
147 | } |
148 | |
149 | loc = dstsec->sh_addr + rel->r_offset; |
150 | |
151 | switch (ELF32_R_TYPE(rel->r_info)) { |
152 | case R_ARM_NONE: |
153 | /* ignore */ |
154 | break; |
155 | |
156 | case R_ARM_ABS32: |
157 | case R_ARM_TARGET1: |
158 | *(u32 *)loc += sym->st_value; |
159 | break; |
160 | |
161 | case R_ARM_PC24: |
162 | case R_ARM_CALL: |
163 | case R_ARM_JUMP24: |
164 | if (sym->st_value & 3) { |
165 | pr_err("%s: section %u reloc %u sym '%s': unsupported interworking call (ARM -> Thumb)\n" , |
166 | module->name, relindex, i, symname); |
167 | return -ENOEXEC; |
168 | } |
169 | |
170 | offset = __mem_to_opcode_arm(*(u32 *)loc); |
171 | offset = (offset & 0x00ffffff) << 2; |
172 | offset = sign_extend32(value: offset, index: 25); |
173 | |
174 | offset += sym->st_value - loc; |
175 | |
176 | /* |
177 | * Route through a PLT entry if 'offset' exceeds the |
178 | * supported range. Note that 'offset + loc + 8' |
179 | * contains the absolute jump target, i.e., |
180 | * @sym + addend, corrected for the +8 PC bias. |
181 | */ |
182 | if (IS_ENABLED(CONFIG_ARM_MODULE_PLTS) && |
183 | (offset <= (s32)0xfe000000 || |
184 | offset >= (s32)0x02000000)) |
185 | offset = get_module_plt(module, loc, |
186 | offset + loc + 8) |
187 | - loc - 8; |
188 | |
189 | if (offset <= (s32)0xfe000000 || |
190 | offset >= (s32)0x02000000) { |
191 | pr_err("%s: section %u reloc %u sym '%s': relocation %u out of range (%#lx -> %#x)\n" , |
192 | module->name, relindex, i, symname, |
193 | ELF32_R_TYPE(rel->r_info), loc, |
194 | sym->st_value); |
195 | return -ENOEXEC; |
196 | } |
197 | |
198 | offset >>= 2; |
199 | offset &= 0x00ffffff; |
200 | |
201 | *(u32 *)loc &= __opcode_to_mem_arm(0xff000000); |
202 | *(u32 *)loc |= __opcode_to_mem_arm(offset); |
203 | break; |
204 | |
205 | case R_ARM_V4BX: |
206 | /* Preserve Rm and the condition code. Alter |
207 | * other bits to re-code instruction as |
208 | * MOV PC,Rm. |
209 | */ |
210 | *(u32 *)loc &= __opcode_to_mem_arm(0xf000000f); |
211 | *(u32 *)loc |= __opcode_to_mem_arm(0x01a0f000); |
212 | break; |
213 | |
214 | case R_ARM_PREL31: |
215 | offset = (*(s32 *)loc << 1) >> 1; /* sign extend */ |
216 | offset += sym->st_value - loc; |
217 | if (offset >= 0x40000000 || offset < -0x40000000) { |
218 | pr_err("%s: section %u reloc %u sym '%s': relocation %u out of range (%#lx -> %#x)\n" , |
219 | module->name, relindex, i, symname, |
220 | ELF32_R_TYPE(rel->r_info), loc, |
221 | sym->st_value); |
222 | return -ENOEXEC; |
223 | } |
224 | *(u32 *)loc &= 0x80000000; |
225 | *(u32 *)loc |= offset & 0x7fffffff; |
226 | break; |
227 | |
228 | case R_ARM_REL32: |
229 | *(u32 *)loc += sym->st_value - loc; |
230 | break; |
231 | |
232 | case R_ARM_MOVW_ABS_NC: |
233 | case R_ARM_MOVT_ABS: |
234 | case R_ARM_MOVW_PREL_NC: |
235 | case R_ARM_MOVT_PREL: |
236 | offset = tmp = __mem_to_opcode_arm(*(u32 *)loc); |
237 | offset = ((offset & 0xf0000) >> 4) | (offset & 0xfff); |
238 | offset = sign_extend32(value: offset, index: 15); |
239 | |
240 | offset += sym->st_value; |
241 | if (ELF32_R_TYPE(rel->r_info) == R_ARM_MOVT_PREL || |
242 | ELF32_R_TYPE(rel->r_info) == R_ARM_MOVW_PREL_NC) |
243 | offset -= loc; |
244 | if (ELF32_R_TYPE(rel->r_info) == R_ARM_MOVT_ABS || |
245 | ELF32_R_TYPE(rel->r_info) == R_ARM_MOVT_PREL) |
246 | offset >>= 16; |
247 | |
248 | tmp &= 0xfff0f000; |
249 | tmp |= ((offset & 0xf000) << 4) | |
250 | (offset & 0x0fff); |
251 | |
252 | *(u32 *)loc = __opcode_to_mem_arm(tmp); |
253 | break; |
254 | |
255 | #ifdef CONFIG_ARM_HAS_GROUP_RELOCS |
256 | case R_ARM_ALU_PC_G0_NC: |
257 | group = 0; |
258 | fallthrough; |
259 | case R_ARM_ALU_PC_G1_NC: |
260 | tmp = __mem_to_opcode_arm(*(u32 *)loc); |
261 | offset = ror32(tmp & 0xff, (tmp & 0xf00) >> 7); |
262 | if (tmp & BIT(22)) |
263 | offset = -offset; |
264 | offset += sym->st_value - loc; |
265 | if (offset < 0) { |
266 | offset = -offset; |
267 | tmp = (tmp & ~BIT(23)) | BIT(22); // SUB opcode |
268 | } else { |
269 | tmp = (tmp & ~BIT(22)) | BIT(23); // ADD opcode |
270 | } |
271 | |
272 | shift = get_group_rem(group, &offset); |
273 | if (shift < 24) { |
274 | offset >>= 24 - shift; |
275 | offset |= (shift + 8) << 7; |
276 | } |
277 | *(u32 *)loc = __opcode_to_mem_arm((tmp & ~0xfff) | offset); |
278 | break; |
279 | |
280 | case R_ARM_LDR_PC_G2: |
281 | tmp = __mem_to_opcode_arm(*(u32 *)loc); |
282 | offset = tmp & 0xfff; |
283 | if (~tmp & BIT(23)) // U bit cleared? |
284 | offset = -offset; |
285 | offset += sym->st_value - loc; |
286 | if (offset < 0) { |
287 | offset = -offset; |
288 | tmp &= ~BIT(23); // clear U bit |
289 | } else { |
290 | tmp |= BIT(23); // set U bit |
291 | } |
292 | get_group_rem(2, &offset); |
293 | |
294 | if (offset > 0xfff) { |
295 | pr_err("%s: section %u reloc %u sym '%s': relocation %u out of range (%#lx -> %#x)\n" , |
296 | module->name, relindex, i, symname, |
297 | ELF32_R_TYPE(rel->r_info), loc, |
298 | sym->st_value); |
299 | return -ENOEXEC; |
300 | } |
301 | *(u32 *)loc = __opcode_to_mem_arm((tmp & ~0xfff) | offset); |
302 | break; |
303 | #endif |
304 | #ifdef CONFIG_THUMB2_KERNEL |
305 | case R_ARM_THM_CALL: |
306 | case R_ARM_THM_JUMP24: |
307 | /* |
308 | * For function symbols, only Thumb addresses are |
309 | * allowed (no interworking). |
310 | * |
311 | * For non-function symbols, the destination |
312 | * has no specific ARM/Thumb disposition, so |
313 | * the branch is resolved under the assumption |
314 | * that interworking is not required. |
315 | */ |
316 | if (ELF32_ST_TYPE(sym->st_info) == STT_FUNC && |
317 | !(sym->st_value & 1)) { |
318 | pr_err("%s: section %u reloc %u sym '%s': unsupported interworking call (Thumb -> ARM)\n" , |
319 | module->name, relindex, i, symname); |
320 | return -ENOEXEC; |
321 | } |
322 | |
323 | upper = __mem_to_opcode_thumb16(*(u16 *)loc); |
324 | lower = __mem_to_opcode_thumb16(*(u16 *)(loc + 2)); |
325 | |
326 | /* |
327 | * 25 bit signed address range (Thumb-2 BL and B.W |
328 | * instructions): |
329 | * S:I1:I2:imm10:imm11:0 |
330 | * where: |
331 | * S = upper[10] = offset[24] |
332 | * I1 = ~(J1 ^ S) = offset[23] |
333 | * I2 = ~(J2 ^ S) = offset[22] |
334 | * imm10 = upper[9:0] = offset[21:12] |
335 | * imm11 = lower[10:0] = offset[11:1] |
336 | * J1 = lower[13] |
337 | * J2 = lower[11] |
338 | */ |
339 | sign = (upper >> 10) & 1; |
340 | j1 = (lower >> 13) & 1; |
341 | j2 = (lower >> 11) & 1; |
342 | offset = (sign << 24) | ((~(j1 ^ sign) & 1) << 23) | |
343 | ((~(j2 ^ sign) & 1) << 22) | |
344 | ((upper & 0x03ff) << 12) | |
345 | ((lower & 0x07ff) << 1); |
346 | offset = sign_extend32(offset, 24); |
347 | offset += sym->st_value - loc; |
348 | |
349 | /* |
350 | * Route through a PLT entry if 'offset' exceeds the |
351 | * supported range. |
352 | */ |
353 | if (IS_ENABLED(CONFIG_ARM_MODULE_PLTS) && |
354 | (offset <= (s32)0xff000000 || |
355 | offset >= (s32)0x01000000)) |
356 | offset = get_module_plt(module, loc, |
357 | offset + loc + 4) |
358 | - loc - 4; |
359 | |
360 | if (offset <= (s32)0xff000000 || |
361 | offset >= (s32)0x01000000) { |
362 | pr_err("%s: section %u reloc %u sym '%s': relocation %u out of range (%#lx -> %#x)\n" , |
363 | module->name, relindex, i, symname, |
364 | ELF32_R_TYPE(rel->r_info), loc, |
365 | sym->st_value); |
366 | return -ENOEXEC; |
367 | } |
368 | |
369 | sign = (offset >> 24) & 1; |
370 | j1 = sign ^ (~(offset >> 23) & 1); |
371 | j2 = sign ^ (~(offset >> 22) & 1); |
372 | upper = (u16)((upper & 0xf800) | (sign << 10) | |
373 | ((offset >> 12) & 0x03ff)); |
374 | lower = (u16)((lower & 0xd000) | |
375 | (j1 << 13) | (j2 << 11) | |
376 | ((offset >> 1) & 0x07ff)); |
377 | |
378 | *(u16 *)loc = __opcode_to_mem_thumb16(upper); |
379 | *(u16 *)(loc + 2) = __opcode_to_mem_thumb16(lower); |
380 | break; |
381 | |
382 | case R_ARM_THM_MOVW_ABS_NC: |
383 | case R_ARM_THM_MOVT_ABS: |
384 | case R_ARM_THM_MOVW_PREL_NC: |
385 | case R_ARM_THM_MOVT_PREL: |
386 | upper = __mem_to_opcode_thumb16(*(u16 *)loc); |
387 | lower = __mem_to_opcode_thumb16(*(u16 *)(loc + 2)); |
388 | |
389 | /* |
390 | * MOVT/MOVW instructions encoding in Thumb-2: |
391 | * |
392 | * i = upper[10] |
393 | * imm4 = upper[3:0] |
394 | * imm3 = lower[14:12] |
395 | * imm8 = lower[7:0] |
396 | * |
397 | * imm16 = imm4:i:imm3:imm8 |
398 | */ |
399 | offset = ((upper & 0x000f) << 12) | |
400 | ((upper & 0x0400) << 1) | |
401 | ((lower & 0x7000) >> 4) | (lower & 0x00ff); |
402 | offset = sign_extend32(offset, 15); |
403 | offset += sym->st_value; |
404 | |
405 | if (ELF32_R_TYPE(rel->r_info) == R_ARM_THM_MOVT_PREL || |
406 | ELF32_R_TYPE(rel->r_info) == R_ARM_THM_MOVW_PREL_NC) |
407 | offset -= loc; |
408 | if (ELF32_R_TYPE(rel->r_info) == R_ARM_THM_MOVT_ABS || |
409 | ELF32_R_TYPE(rel->r_info) == R_ARM_THM_MOVT_PREL) |
410 | offset >>= 16; |
411 | |
412 | upper = (u16)((upper & 0xfbf0) | |
413 | ((offset & 0xf000) >> 12) | |
414 | ((offset & 0x0800) >> 1)); |
415 | lower = (u16)((lower & 0x8f00) | |
416 | ((offset & 0x0700) << 4) | |
417 | (offset & 0x00ff)); |
418 | *(u16 *)loc = __opcode_to_mem_thumb16(upper); |
419 | *(u16 *)(loc + 2) = __opcode_to_mem_thumb16(lower); |
420 | break; |
421 | #endif |
422 | |
423 | default: |
424 | pr_err("%s: unknown relocation: %u\n" , |
425 | module->name, ELF32_R_TYPE(rel->r_info)); |
426 | return -ENOEXEC; |
427 | } |
428 | } |
429 | return 0; |
430 | } |
431 | |
432 | struct mod_unwind_map { |
433 | const Elf_Shdr *unw_sec; |
434 | const Elf_Shdr *txt_sec; |
435 | }; |
436 | |
437 | static const Elf_Shdr *find_mod_section(const Elf32_Ehdr *hdr, |
438 | const Elf_Shdr *sechdrs, const char *name) |
439 | { |
440 | const Elf_Shdr *s, *se; |
441 | const char *secstrs = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset; |
442 | |
443 | for (s = sechdrs, se = sechdrs + hdr->e_shnum; s < se; s++) |
444 | if (strcmp(name, secstrs + s->sh_name) == 0) |
445 | return s; |
446 | |
447 | return NULL; |
448 | } |
449 | |
450 | extern void fixup_pv_table(const void *, unsigned long); |
451 | extern void fixup_smp(const void *, unsigned long); |
452 | |
453 | int module_finalize(const Elf32_Ehdr *hdr, const Elf_Shdr *sechdrs, |
454 | struct module *mod) |
455 | { |
456 | const Elf_Shdr *s = NULL; |
457 | #ifdef CONFIG_ARM_UNWIND |
458 | const char *secstrs = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset; |
459 | const Elf_Shdr *sechdrs_end = sechdrs + hdr->e_shnum; |
460 | struct list_head *unwind_list = &mod->arch.unwind_list; |
461 | |
462 | INIT_LIST_HEAD(unwind_list); |
463 | mod->arch.init_table = NULL; |
464 | |
465 | for (s = sechdrs; s < sechdrs_end; s++) { |
466 | const char *secname = secstrs + s->sh_name; |
467 | const char *txtname; |
468 | const Elf_Shdr *txt_sec; |
469 | |
470 | if (!(s->sh_flags & SHF_ALLOC) || |
471 | s->sh_type != ELF_SECTION_UNWIND) |
472 | continue; |
473 | |
474 | if (!strcmp(".ARM.exidx" , secname)) |
475 | txtname = ".text" ; |
476 | else |
477 | txtname = secname + strlen(".ARM.exidx" ); |
478 | txt_sec = find_mod_section(hdr, sechdrs, txtname); |
479 | |
480 | if (txt_sec) { |
481 | struct unwind_table *table = |
482 | unwind_table_add(s->sh_addr, |
483 | s->sh_size, |
484 | txt_sec->sh_addr, |
485 | txt_sec->sh_size); |
486 | |
487 | list_add(&table->mod_list, unwind_list); |
488 | |
489 | /* save init table for module_arch_freeing_init */ |
490 | if (strcmp(".ARM.exidx.init.text" , secname) == 0) |
491 | mod->arch.init_table = table; |
492 | } |
493 | } |
494 | #endif |
495 | #ifdef CONFIG_ARM_PATCH_PHYS_VIRT |
496 | s = find_mod_section(hdr, sechdrs, ".pv_table" ); |
497 | if (s) |
498 | fixup_pv_table((void *)s->sh_addr, s->sh_size); |
499 | #endif |
500 | s = find_mod_section(hdr, sechdrs, name: ".alt.smp.init" ); |
501 | if (s && !is_smp()) |
502 | #ifdef CONFIG_SMP_ON_UP |
503 | fixup_smp((void *)s->sh_addr, s->sh_size); |
504 | #else |
505 | return -EINVAL; |
506 | #endif |
507 | return 0; |
508 | } |
509 | |
510 | void |
511 | module_arch_cleanup(struct module *mod) |
512 | { |
513 | #ifdef CONFIG_ARM_UNWIND |
514 | struct unwind_table *tmp; |
515 | struct unwind_table *n; |
516 | |
517 | list_for_each_entry_safe(tmp, n, |
518 | &mod->arch.unwind_list, mod_list) { |
519 | list_del(&tmp->mod_list); |
520 | unwind_table_del(tmp); |
521 | } |
522 | mod->arch.init_table = NULL; |
523 | #endif |
524 | } |
525 | |
526 | void __weak module_arch_freeing_init(struct module *mod) |
527 | { |
528 | #ifdef CONFIG_ARM_UNWIND |
529 | struct unwind_table *init = mod->arch.init_table; |
530 | |
531 | if (init) { |
532 | mod->arch.init_table = NULL; |
533 | list_del(&init->mod_list); |
534 | unwind_table_del(init); |
535 | } |
536 | #endif |
537 | } |
538 | |