1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | /* Kernel module help for PPC64. |
3 | Copyright (C) 2001, 2003 Rusty Russell IBM Corporation. |
4 | |
5 | */ |
6 | |
7 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt |
8 | |
9 | #include <linux/module.h> |
10 | #include <linux/elf.h> |
11 | #include <linux/moduleloader.h> |
12 | #include <linux/err.h> |
13 | #include <linux/vmalloc.h> |
14 | #include <linux/ftrace.h> |
15 | #include <linux/bug.h> |
16 | #include <linux/uaccess.h> |
17 | #include <linux/kernel.h> |
18 | #include <asm/module.h> |
19 | #include <asm/firmware.h> |
20 | #include <asm/code-patching.h> |
21 | #include <linux/sort.h> |
22 | #include <asm/setup.h> |
23 | #include <asm/sections.h> |
24 | #include <asm/inst.h> |
25 | |
26 | /* FIXME: We don't do .init separately. To do this, we'd need to have |
27 | a separate r2 value in the init and core section, and stub between |
28 | them, too. |
29 | |
30 | Using a magic allocator which places modules within 32MB solves |
31 | this, and makes other things simpler. Anton? |
32 | --RR. */ |
33 | |
34 | bool module_elf_check_arch(Elf_Ehdr *hdr) |
35 | { |
36 | unsigned long abi_level = hdr->e_flags & 0x3; |
37 | |
38 | if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V2)) |
39 | return abi_level == 2; |
40 | else |
41 | return abi_level < 2; |
42 | } |
43 | |
44 | #ifdef CONFIG_PPC64_ELF_ABI_V2 |
45 | |
46 | static func_desc_t func_desc(unsigned long addr) |
47 | { |
48 | func_desc_t desc = { |
49 | .addr = addr, |
50 | }; |
51 | |
52 | return desc; |
53 | } |
54 | |
55 | /* PowerPC64 specific values for the Elf64_Sym st_other field. */ |
56 | #define STO_PPC64_LOCAL_BIT 5 |
57 | #define STO_PPC64_LOCAL_MASK (7 << STO_PPC64_LOCAL_BIT) |
58 | #define PPC64_LOCAL_ENTRY_OFFSET(other) \ |
59 | (((1 << (((other) & STO_PPC64_LOCAL_MASK) >> STO_PPC64_LOCAL_BIT)) >> 2) << 2) |
60 | |
61 | static unsigned int local_entry_offset(const Elf64_Sym *sym) |
62 | { |
63 | /* sym->st_other indicates offset to local entry point |
64 | * (otherwise it will assume r12 is the address of the start |
65 | * of function and try to derive r2 from it). */ |
66 | return PPC64_LOCAL_ENTRY_OFFSET(sym->st_other); |
67 | } |
68 | #else |
69 | |
70 | static func_desc_t func_desc(unsigned long addr) |
71 | { |
72 | return *(struct func_desc *)addr; |
73 | } |
74 | static unsigned int local_entry_offset(const Elf64_Sym *sym) |
75 | { |
76 | return 0; |
77 | } |
78 | |
79 | void *dereference_module_function_descriptor(struct module *mod, void *ptr) |
80 | { |
81 | if (ptr < (void *)mod->arch.start_opd || |
82 | ptr >= (void *)mod->arch.end_opd) |
83 | return ptr; |
84 | |
85 | return dereference_function_descriptor(ptr); |
86 | } |
87 | #endif |
88 | |
89 | static unsigned long func_addr(unsigned long addr) |
90 | { |
91 | return func_desc(addr).addr; |
92 | } |
93 | |
94 | static unsigned long stub_func_addr(func_desc_t func) |
95 | { |
96 | return func.addr; |
97 | } |
98 | |
99 | #define STUB_MAGIC 0x73747562 /* stub */ |
100 | |
101 | /* Like PPC32, we need little trampolines to do > 24-bit jumps (into |
102 | the kernel itself). But on PPC64, these need to be used for every |
103 | jump, actually, to reset r2 (TOC+0x8000). */ |
104 | struct ppc64_stub_entry { |
105 | /* |
106 | * 28 byte jump instruction sequence (7 instructions) that can |
107 | * hold ppc64_stub_insns or stub_insns. Must be 8-byte aligned |
108 | * with PCREL kernels that use prefix instructions in the stub. |
109 | */ |
110 | u32 jump[7]; |
111 | /* Used by ftrace to identify stubs */ |
112 | u32 magic; |
113 | /* Data for the above code */ |
114 | func_desc_t funcdata; |
115 | } __aligned(8); |
116 | |
117 | struct ppc64_got_entry { |
118 | u64 addr; |
119 | }; |
120 | |
121 | /* |
122 | * PPC64 uses 24 bit jumps, but we need to jump into other modules or |
123 | * the kernel which may be further. So we jump to a stub. |
124 | * |
125 | * Target address and TOC are loaded from function descriptor in the |
126 | * ppc64_stub_entry. |
127 | * |
128 | * r12 is used to generate the target address, which is required for the |
129 | * ELFv2 global entry point calling convention. |
130 | * |
131 | * TOC handling: |
132 | * - PCREL does not have a TOC. |
133 | * - ELFv2 non-PCREL just has to save r2, the callee is responsible for |
134 | * setting its own TOC pointer at the global entry address. |
135 | * - ELFv1 must load the new TOC pointer from the function descriptor. |
136 | */ |
137 | static u32 ppc64_stub_insns[] = { |
138 | #ifdef CONFIG_PPC_KERNEL_PCREL |
139 | /* pld r12,addr */ |
140 | PPC_PREFIX_8LS | __PPC_PRFX_R(1), |
141 | PPC_INST_PLD | ___PPC_RT(_R12), |
142 | #else |
143 | PPC_RAW_ADDIS(_R11, _R2, 0), |
144 | PPC_RAW_ADDI(_R11, _R11, 0), |
145 | /* Save current r2 value in magic place on the stack. */ |
146 | PPC_RAW_STD(_R2, _R1, R2_STACK_OFFSET), |
147 | PPC_RAW_LD(_R12, _R11, 32), |
148 | #ifdef CONFIG_PPC64_ELF_ABI_V1 |
149 | /* Set up new r2 from function descriptor */ |
150 | PPC_RAW_LD(_R2, _R11, 40), |
151 | #endif |
152 | #endif |
153 | PPC_RAW_MTCTR(_R12), |
154 | PPC_RAW_BCTR(), |
155 | }; |
156 | |
157 | /* |
158 | * Count how many different r_type relocations (different symbol, |
159 | * different addend). |
160 | */ |
161 | static unsigned int count_relocs(const Elf64_Rela *rela, unsigned int num, |
162 | unsigned long r_type) |
163 | { |
164 | unsigned int i, r_info, r_addend, _count_relocs; |
165 | |
166 | /* FIXME: Only count external ones --RR */ |
167 | _count_relocs = 0; |
168 | r_info = 0; |
169 | r_addend = 0; |
170 | for (i = 0; i < num; i++) |
171 | /* Only count r_type relocs, others don't need stubs */ |
172 | if (ELF64_R_TYPE(rela[i].r_info) == r_type && |
173 | (r_info != ELF64_R_SYM(rela[i].r_info) || |
174 | r_addend != rela[i].r_addend)) { |
175 | _count_relocs++; |
176 | r_info = ELF64_R_SYM(rela[i].r_info); |
177 | r_addend = rela[i].r_addend; |
178 | } |
179 | |
180 | return _count_relocs; |
181 | } |
182 | |
183 | static int relacmp(const void *_x, const void *_y) |
184 | { |
185 | const Elf64_Rela *x, *y; |
186 | |
187 | y = (Elf64_Rela *)_x; |
188 | x = (Elf64_Rela *)_y; |
189 | |
190 | /* Compare the entire r_info (as opposed to ELF64_R_SYM(r_info) only) to |
191 | * make the comparison cheaper/faster. It won't affect the sorting or |
192 | * the counting algorithms' performance |
193 | */ |
194 | if (x->r_info < y->r_info) |
195 | return -1; |
196 | else if (x->r_info > y->r_info) |
197 | return 1; |
198 | else if (x->r_addend < y->r_addend) |
199 | return -1; |
200 | else if (x->r_addend > y->r_addend) |
201 | return 1; |
202 | else |
203 | return 0; |
204 | } |
205 | |
206 | /* Get size of potential trampolines required. */ |
207 | static unsigned long get_stubs_size(const Elf64_Ehdr *hdr, |
208 | const Elf64_Shdr *sechdrs) |
209 | { |
210 | /* One extra reloc so it's always 0-addr terminated */ |
211 | unsigned long relocs = 1; |
212 | unsigned i; |
213 | |
214 | /* Every relocated section... */ |
215 | for (i = 1; i < hdr->e_shnum; i++) { |
216 | if (sechdrs[i].sh_type == SHT_RELA) { |
217 | pr_debug("Found relocations in section %u\n" , i); |
218 | pr_debug("Ptr: %p. Number: %Lu\n" , |
219 | (void *)sechdrs[i].sh_addr, |
220 | sechdrs[i].sh_size / sizeof(Elf64_Rela)); |
221 | |
222 | /* Sort the relocation information based on a symbol and |
223 | * addend key. This is a stable O(n*log n) complexity |
224 | * algorithm but it will reduce the complexity of |
225 | * count_relocs() to linear complexity O(n) |
226 | */ |
227 | sort(base: (void *)sechdrs[i].sh_addr, |
228 | num: sechdrs[i].sh_size / sizeof(Elf64_Rela), |
229 | size: sizeof(Elf64_Rela), cmp_func: relacmp, NULL); |
230 | |
231 | relocs += count_relocs(rela: (void *)sechdrs[i].sh_addr, |
232 | num: sechdrs[i].sh_size |
233 | / sizeof(Elf64_Rela), |
234 | r_type: R_PPC_REL24); |
235 | #ifdef CONFIG_PPC_KERNEL_PCREL |
236 | relocs += count_relocs((void *)sechdrs[i].sh_addr, |
237 | sechdrs[i].sh_size |
238 | / sizeof(Elf64_Rela), |
239 | R_PPC64_REL24_NOTOC); |
240 | #endif |
241 | } |
242 | } |
243 | |
244 | #ifdef CONFIG_DYNAMIC_FTRACE |
245 | /* make the trampoline to the ftrace_caller */ |
246 | relocs++; |
247 | #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS |
248 | /* an additional one for ftrace_regs_caller */ |
249 | relocs++; |
250 | #endif |
251 | #endif |
252 | |
253 | pr_debug("Looks like a total of %lu stubs, max\n" , relocs); |
254 | return relocs * sizeof(struct ppc64_stub_entry); |
255 | } |
256 | |
257 | #ifdef CONFIG_PPC_KERNEL_PCREL |
258 | static int count_pcpu_relocs(const Elf64_Shdr *sechdrs, |
259 | const Elf64_Rela *rela, unsigned int num, |
260 | unsigned int symindex, unsigned int pcpu) |
261 | { |
262 | unsigned int i, r_info, r_addend, _count_relocs; |
263 | |
264 | _count_relocs = 0; |
265 | r_info = 0; |
266 | r_addend = 0; |
267 | |
268 | for (i = 0; i < num; i++) { |
269 | Elf64_Sym *sym; |
270 | |
271 | /* This is the symbol it is referring to */ |
272 | sym = (Elf64_Sym *)sechdrs[symindex].sh_addr |
273 | + ELF64_R_SYM(rela[i].r_info); |
274 | |
275 | if (sym->st_shndx == pcpu && |
276 | (r_info != ELF64_R_SYM(rela[i].r_info) || |
277 | r_addend != rela[i].r_addend)) { |
278 | _count_relocs++; |
279 | r_info = ELF64_R_SYM(rela[i].r_info); |
280 | r_addend = rela[i].r_addend; |
281 | } |
282 | } |
283 | |
284 | return _count_relocs; |
285 | } |
286 | |
287 | /* Get size of potential GOT required. */ |
288 | static unsigned long get_got_size(const Elf64_Ehdr *hdr, |
289 | const Elf64_Shdr *sechdrs, |
290 | struct module *me) |
291 | { |
292 | /* One extra reloc so it's always 0-addr terminated */ |
293 | unsigned long relocs = 1; |
294 | unsigned int i, symindex = 0; |
295 | |
296 | for (i = 1; i < hdr->e_shnum; i++) { |
297 | if (sechdrs[i].sh_type == SHT_SYMTAB) { |
298 | symindex = i; |
299 | break; |
300 | } |
301 | } |
302 | WARN_ON_ONCE(!symindex); |
303 | |
304 | /* Every relocated section... */ |
305 | for (i = 1; i < hdr->e_shnum; i++) { |
306 | if (sechdrs[i].sh_type == SHT_RELA) { |
307 | pr_debug("Found relocations in section %u\n" , i); |
308 | pr_debug("Ptr: %p. Number: %llu\n" , (void *)sechdrs[i].sh_addr, |
309 | sechdrs[i].sh_size / sizeof(Elf64_Rela)); |
310 | |
311 | /* |
312 | * Sort the relocation information based on a symbol and |
313 | * addend key. This is a stable O(n*log n) complexity |
314 | * algorithm but it will reduce the complexity of |
315 | * count_relocs() to linear complexity O(n) |
316 | */ |
317 | sort((void *)sechdrs[i].sh_addr, |
318 | sechdrs[i].sh_size / sizeof(Elf64_Rela), |
319 | sizeof(Elf64_Rela), relacmp, NULL); |
320 | |
321 | relocs += count_relocs((void *)sechdrs[i].sh_addr, |
322 | sechdrs[i].sh_size |
323 | / sizeof(Elf64_Rela), |
324 | R_PPC64_GOT_PCREL34); |
325 | |
326 | /* |
327 | * Percpu data access typically gets linked with |
328 | * REL34 relocations, but the percpu section gets |
329 | * moved at load time and requires that to be |
330 | * converted to GOT linkage. |
331 | */ |
332 | if (IS_ENABLED(CONFIG_SMP) && symindex) |
333 | relocs += count_pcpu_relocs(sechdrs, |
334 | (void *)sechdrs[i].sh_addr, |
335 | sechdrs[i].sh_size |
336 | / sizeof(Elf64_Rela), |
337 | symindex, me->arch.pcpu_section); |
338 | } |
339 | } |
340 | |
341 | pr_debug("Looks like a total of %lu GOT entries, max\n" , relocs); |
342 | return relocs * sizeof(struct ppc64_got_entry); |
343 | } |
344 | #else /* CONFIG_PPC_KERNEL_PCREL */ |
345 | |
346 | /* Still needed for ELFv2, for .TOC. */ |
347 | static void dedotify_versions(struct modversion_info *vers, |
348 | unsigned long size) |
349 | { |
350 | struct modversion_info *end; |
351 | |
352 | for (end = (void *)vers + size; vers < end; vers++) |
353 | if (vers->name[0] == '.') { |
354 | memmove(vers->name, vers->name+1, strlen(vers->name)); |
355 | } |
356 | } |
357 | |
358 | /* |
359 | * Undefined symbols which refer to .funcname, hack to funcname. Make .TOC. |
360 | * seem to be defined (value set later). |
361 | */ |
362 | static void dedotify(Elf64_Sym *syms, unsigned int numsyms, char *strtab) |
363 | { |
364 | unsigned int i; |
365 | |
366 | for (i = 1; i < numsyms; i++) { |
367 | if (syms[i].st_shndx == SHN_UNDEF) { |
368 | char *name = strtab + syms[i].st_name; |
369 | if (name[0] == '.') { |
370 | if (strcmp(name+1, "TOC." ) == 0) |
371 | syms[i].st_shndx = SHN_ABS; |
372 | syms[i].st_name++; |
373 | } |
374 | } |
375 | } |
376 | } |
377 | |
378 | static Elf64_Sym *find_dot_toc(Elf64_Shdr *sechdrs, |
379 | const char *strtab, |
380 | unsigned int symindex) |
381 | { |
382 | unsigned int i, numsyms; |
383 | Elf64_Sym *syms; |
384 | |
385 | syms = (Elf64_Sym *)sechdrs[symindex].sh_addr; |
386 | numsyms = sechdrs[symindex].sh_size / sizeof(Elf64_Sym); |
387 | |
388 | for (i = 1; i < numsyms; i++) { |
389 | if (syms[i].st_shndx == SHN_ABS |
390 | && strcmp(strtab + syms[i].st_name, "TOC." ) == 0) |
391 | return &syms[i]; |
392 | } |
393 | return NULL; |
394 | } |
395 | #endif /* CONFIG_PPC_KERNEL_PCREL */ |
396 | |
397 | bool module_init_section(const char *name) |
398 | { |
399 | /* We don't handle .init for the moment: always return false. */ |
400 | return false; |
401 | } |
402 | |
403 | int module_frob_arch_sections(Elf64_Ehdr *hdr, |
404 | Elf64_Shdr *sechdrs, |
405 | char *secstrings, |
406 | struct module *me) |
407 | { |
408 | unsigned int i; |
409 | |
410 | /* Find .toc and .stubs sections, symtab and strtab */ |
411 | for (i = 1; i < hdr->e_shnum; i++) { |
412 | if (strcmp(secstrings + sechdrs[i].sh_name, ".stubs" ) == 0) |
413 | me->arch.stubs_section = i; |
414 | #ifdef CONFIG_PPC_KERNEL_PCREL |
415 | else if (strcmp(secstrings + sechdrs[i].sh_name, ".data..percpu" ) == 0) |
416 | me->arch.pcpu_section = i; |
417 | else if (strcmp(secstrings + sechdrs[i].sh_name, ".mygot" ) == 0) { |
418 | me->arch.got_section = i; |
419 | if (sechdrs[i].sh_addralign < 8) |
420 | sechdrs[i].sh_addralign = 8; |
421 | } |
422 | #else |
423 | else if (strcmp(secstrings + sechdrs[i].sh_name, ".toc" ) == 0) { |
424 | me->arch.toc_section = i; |
425 | if (sechdrs[i].sh_addralign < 8) |
426 | sechdrs[i].sh_addralign = 8; |
427 | } |
428 | else if (strcmp(secstrings+sechdrs[i].sh_name,"__versions" )==0) |
429 | dedotify_versions(vers: (void *)hdr + sechdrs[i].sh_offset, |
430 | size: sechdrs[i].sh_size); |
431 | |
432 | if (sechdrs[i].sh_type == SHT_SYMTAB) |
433 | dedotify(syms: (void *)hdr + sechdrs[i].sh_offset, |
434 | numsyms: sechdrs[i].sh_size / sizeof(Elf64_Sym), |
435 | strtab: (void *)hdr |
436 | + sechdrs[sechdrs[i].sh_link].sh_offset); |
437 | #endif |
438 | } |
439 | |
440 | if (!me->arch.stubs_section) { |
441 | pr_err("%s: doesn't contain .stubs.\n" , me->name); |
442 | return -ENOEXEC; |
443 | } |
444 | |
445 | #ifdef CONFIG_PPC_KERNEL_PCREL |
446 | if (!me->arch.got_section) { |
447 | pr_err("%s: doesn't contain .mygot.\n" , me->name); |
448 | return -ENOEXEC; |
449 | } |
450 | |
451 | /* Override the got size */ |
452 | sechdrs[me->arch.got_section].sh_size = get_got_size(hdr, sechdrs, me); |
453 | #else |
454 | /* If we don't have a .toc, just use .stubs. We need to set r2 |
455 | to some reasonable value in case the module calls out to |
456 | other functions via a stub, or if a function pointer escapes |
457 | the module by some means. */ |
458 | if (!me->arch.toc_section) |
459 | me->arch.toc_section = me->arch.stubs_section; |
460 | #endif |
461 | |
462 | /* Override the stubs size */ |
463 | sechdrs[me->arch.stubs_section].sh_size = get_stubs_size(hdr, sechdrs); |
464 | |
465 | return 0; |
466 | } |
467 | |
468 | #if defined(CONFIG_MPROFILE_KERNEL) || defined(CONFIG_ARCH_USING_PATCHABLE_FUNCTION_ENTRY) |
469 | |
470 | static u32 stub_insns[] = { |
471 | #ifdef CONFIG_PPC_KERNEL_PCREL |
472 | PPC_RAW_LD(_R12, _R13, offsetof(struct paca_struct, kernelbase)), |
473 | PPC_RAW_NOP(), /* align the prefix insn */ |
474 | /* paddi r12,r12,addr */ |
475 | PPC_PREFIX_MLS | __PPC_PRFX_R(0), |
476 | PPC_INST_PADDI | ___PPC_RT(_R12) | ___PPC_RA(_R12), |
477 | PPC_RAW_MTCTR(_R12), |
478 | PPC_RAW_BCTR(), |
479 | #else |
480 | PPC_RAW_LD(_R12, _R13, offsetof(struct paca_struct, kernel_toc)), |
481 | PPC_RAW_ADDIS(_R12, _R12, 0), |
482 | PPC_RAW_ADDI(_R12, _R12, 0), |
483 | PPC_RAW_MTCTR(_R12), |
484 | PPC_RAW_BCTR(), |
485 | #endif |
486 | }; |
487 | |
488 | /* |
489 | * For mprofile-kernel we use a special stub for ftrace_caller() because we |
490 | * can't rely on r2 containing this module's TOC when we enter the stub. |
491 | * |
492 | * That can happen if the function calling us didn't need to use the toc. In |
493 | * that case it won't have setup r2, and the r2 value will be either the |
494 | * kernel's toc, or possibly another modules toc. |
495 | * |
496 | * To deal with that this stub uses the kernel toc, which is always accessible |
497 | * via the paca (in r13). The target (ftrace_caller()) is responsible for |
498 | * saving and restoring the toc before returning. |
499 | */ |
500 | static inline int create_ftrace_stub(struct ppc64_stub_entry *entry, |
501 | unsigned long addr, |
502 | struct module *me) |
503 | { |
504 | long reladdr; |
505 | |
506 | if ((unsigned long)entry->jump % 8 != 0) { |
507 | pr_err("%s: Address of stub entry is not 8-byte aligned\n" , me->name); |
508 | return 0; |
509 | } |
510 | |
511 | BUILD_BUG_ON(sizeof(stub_insns) > sizeof(entry->jump)); |
512 | memcpy(entry->jump, stub_insns, sizeof(stub_insns)); |
513 | |
514 | if (IS_ENABLED(CONFIG_PPC_KERNEL_PCREL)) { |
515 | /* Stub uses address relative to kernel base (from the paca) */ |
516 | reladdr = addr - local_paca->kernelbase; |
517 | if (reladdr > 0x1FFFFFFFFL || reladdr < -0x200000000L) { |
518 | pr_err("%s: Address of %ps out of range of 34-bit relative address.\n" , |
519 | me->name, (void *)addr); |
520 | return 0; |
521 | } |
522 | |
523 | entry->jump[2] |= IMM_H18(reladdr); |
524 | entry->jump[3] |= IMM_L(reladdr); |
525 | } else { |
526 | /* Stub uses address relative to kernel toc (from the paca) */ |
527 | reladdr = addr - kernel_toc_addr(); |
528 | if (reladdr > 0x7FFFFFFF || reladdr < -(0x80000000L)) { |
529 | pr_err("%s: Address of %ps out of range of kernel_toc.\n" , |
530 | me->name, (void *)addr); |
531 | return 0; |
532 | } |
533 | |
534 | entry->jump[1] |= PPC_HA(reladdr); |
535 | entry->jump[2] |= PPC_LO(reladdr); |
536 | } |
537 | |
538 | /* Even though we don't use funcdata in the stub, it's needed elsewhere. */ |
539 | entry->funcdata = func_desc(addr); |
540 | entry->magic = STUB_MAGIC; |
541 | |
542 | return 1; |
543 | } |
544 | |
545 | static bool is_mprofile_ftrace_call(const char *name) |
546 | { |
547 | if (!strcmp("_mcount" , name)) |
548 | return true; |
549 | #ifdef CONFIG_DYNAMIC_FTRACE |
550 | if (!strcmp("ftrace_caller" , name)) |
551 | return true; |
552 | #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS |
553 | if (!strcmp("ftrace_regs_caller" , name)) |
554 | return true; |
555 | #endif |
556 | #endif |
557 | |
558 | return false; |
559 | } |
560 | #else |
561 | static inline int create_ftrace_stub(struct ppc64_stub_entry *entry, |
562 | unsigned long addr, |
563 | struct module *me) |
564 | { |
565 | return 0; |
566 | } |
567 | |
568 | static bool is_mprofile_ftrace_call(const char *name) |
569 | { |
570 | return false; |
571 | } |
572 | #endif |
573 | |
574 | /* |
575 | * r2 is the TOC pointer: it actually points 0x8000 into the TOC (this gives the |
576 | * value maximum span in an instruction which uses a signed offset). Round down |
577 | * to a 256 byte boundary for the odd case where we are setting up r2 without a |
578 | * .toc section. |
579 | */ |
580 | static inline unsigned long my_r2(const Elf64_Shdr *sechdrs, struct module *me) |
581 | { |
582 | #ifndef CONFIG_PPC_KERNEL_PCREL |
583 | return (sechdrs[me->arch.toc_section].sh_addr & ~0xfful) + 0x8000; |
584 | #else |
585 | return -1; |
586 | #endif |
587 | } |
588 | |
589 | /* Patch stub to reference function and correct r2 value. */ |
590 | static inline int create_stub(const Elf64_Shdr *sechdrs, |
591 | struct ppc64_stub_entry *entry, |
592 | unsigned long addr, |
593 | struct module *me, |
594 | const char *name) |
595 | { |
596 | long reladdr; |
597 | func_desc_t desc; |
598 | int i; |
599 | |
600 | if (is_mprofile_ftrace_call(name)) |
601 | return create_ftrace_stub(entry, addr, me); |
602 | |
603 | if ((unsigned long)entry->jump % 8 != 0) { |
604 | pr_err("%s: Address of stub entry is not 8-byte aligned\n" , me->name); |
605 | return 0; |
606 | } |
607 | |
608 | BUILD_BUG_ON(sizeof(ppc64_stub_insns) > sizeof(entry->jump)); |
609 | for (i = 0; i < ARRAY_SIZE(ppc64_stub_insns); i++) { |
610 | if (patch_instruction(&entry->jump[i], |
611 | ppc_inst(ppc64_stub_insns[i]))) |
612 | return 0; |
613 | } |
614 | |
615 | if (IS_ENABLED(CONFIG_PPC_KERNEL_PCREL)) { |
616 | /* Stub uses address relative to itself! */ |
617 | reladdr = 0 + offsetof(struct ppc64_stub_entry, funcdata); |
618 | BUILD_BUG_ON(reladdr != 32); |
619 | if (reladdr > 0x1FFFFFFFFL || reladdr < -0x200000000L) { |
620 | pr_err("%s: Address of %p out of range of 34-bit relative address.\n" , |
621 | me->name, (void *)reladdr); |
622 | return 0; |
623 | } |
624 | pr_debug("Stub %p get data from reladdr %li\n" , entry, reladdr); |
625 | |
626 | /* May not even need this if we're relative to 0 */ |
627 | if (patch_instruction(&entry->jump[0], |
628 | ppc_inst_prefix(entry->jump[0] | IMM_H18(reladdr), |
629 | entry->jump[1] | IMM_L(reladdr)))) |
630 | return 0; |
631 | |
632 | } else { |
633 | /* Stub uses address relative to r2. */ |
634 | reladdr = (unsigned long)entry - my_r2(sechdrs, me); |
635 | if (reladdr > 0x7FFFFFFF || reladdr < -(0x80000000L)) { |
636 | pr_err("%s: Address %p of stub out of range of %p.\n" , |
637 | me->name, (void *)reladdr, (void *)my_r2); |
638 | return 0; |
639 | } |
640 | pr_debug("Stub %p get data from reladdr %li\n" , entry, reladdr); |
641 | |
642 | if (patch_instruction(&entry->jump[0], |
643 | ppc_inst(entry->jump[0] | PPC_HA(reladdr)))) |
644 | return 0; |
645 | |
646 | if (patch_instruction(&entry->jump[1], |
647 | ppc_inst(entry->jump[1] | PPC_LO(reladdr)))) |
648 | return 0; |
649 | } |
650 | |
651 | // func_desc_t is 8 bytes if ABIv2, else 16 bytes |
652 | desc = func_desc(addr); |
653 | for (i = 0; i < sizeof(func_desc_t) / sizeof(u32); i++) { |
654 | if (patch_instruction(((u32 *)&entry->funcdata) + i, |
655 | ppc_inst(((u32 *)(&desc))[i]))) |
656 | return 0; |
657 | } |
658 | |
659 | if (patch_instruction(&entry->magic, ppc_inst(STUB_MAGIC))) |
660 | return 0; |
661 | |
662 | return 1; |
663 | } |
664 | |
665 | /* Create stub to jump to function described in this OPD/ptr: we need the |
666 | stub to set up the TOC ptr (r2) for the function. */ |
667 | static unsigned long stub_for_addr(const Elf64_Shdr *sechdrs, |
668 | unsigned long addr, |
669 | struct module *me, |
670 | const char *name) |
671 | { |
672 | struct ppc64_stub_entry *stubs; |
673 | unsigned int i, num_stubs; |
674 | |
675 | num_stubs = sechdrs[me->arch.stubs_section].sh_size / sizeof(*stubs); |
676 | |
677 | /* Find this stub, or if that fails, the next avail. entry */ |
678 | stubs = (void *)sechdrs[me->arch.stubs_section].sh_addr; |
679 | for (i = 0; stub_func_addr(func: stubs[i].funcdata); i++) { |
680 | if (WARN_ON(i >= num_stubs)) |
681 | return 0; |
682 | |
683 | if (stub_func_addr(func: stubs[i].funcdata) == func_addr(addr)) |
684 | return (unsigned long)&stubs[i]; |
685 | } |
686 | |
687 | if (!create_stub(sechdrs, entry: &stubs[i], addr, me, name)) |
688 | return 0; |
689 | |
690 | return (unsigned long)&stubs[i]; |
691 | } |
692 | |
693 | #ifdef CONFIG_PPC_KERNEL_PCREL |
694 | /* Create GOT to load the location described in this ptr */ |
695 | static unsigned long got_for_addr(const Elf64_Shdr *sechdrs, |
696 | unsigned long addr, |
697 | struct module *me, |
698 | const char *name) |
699 | { |
700 | struct ppc64_got_entry *got; |
701 | unsigned int i, num_got; |
702 | |
703 | if (!IS_ENABLED(CONFIG_PPC_KERNEL_PCREL)) |
704 | return addr; |
705 | |
706 | num_got = sechdrs[me->arch.got_section].sh_size / sizeof(*got); |
707 | |
708 | /* Find this stub, or if that fails, the next avail. entry */ |
709 | got = (void *)sechdrs[me->arch.got_section].sh_addr; |
710 | for (i = 0; got[i].addr; i++) { |
711 | if (WARN_ON(i >= num_got)) |
712 | return 0; |
713 | |
714 | if (got[i].addr == addr) |
715 | return (unsigned long)&got[i]; |
716 | } |
717 | |
718 | got[i].addr = addr; |
719 | |
720 | return (unsigned long)&got[i]; |
721 | } |
722 | #endif |
723 | |
724 | /* We expect a noop next: if it is, replace it with instruction to |
725 | restore r2. */ |
726 | static int restore_r2(const char *name, u32 *instruction, struct module *me) |
727 | { |
728 | u32 *prev_insn = instruction - 1; |
729 | u32 insn_val = *instruction; |
730 | |
731 | if (IS_ENABLED(CONFIG_PPC_KERNEL_PCREL)) |
732 | return 0; |
733 | |
734 | if (is_mprofile_ftrace_call(name)) |
735 | return 0; |
736 | |
737 | /* |
738 | * Make sure the branch isn't a sibling call. Sibling calls aren't |
739 | * "link" branches and they don't return, so they don't need the r2 |
740 | * restore afterwards. |
741 | */ |
742 | if (!instr_is_relative_link_branch(ppc_inst(*prev_insn))) |
743 | return 0; |
744 | |
745 | /* |
746 | * For livepatch, the restore r2 instruction might have already been |
747 | * written previously, if the referenced symbol is in a previously |
748 | * unloaded module which is now being loaded again. In that case, skip |
749 | * the warning and the instruction write. |
750 | */ |
751 | if (insn_val == PPC_INST_LD_TOC) |
752 | return 0; |
753 | |
754 | if (insn_val != PPC_RAW_NOP()) { |
755 | pr_err("%s: Expected nop after call, got %08x at %pS\n" , |
756 | me->name, insn_val, instruction); |
757 | return -ENOEXEC; |
758 | } |
759 | |
760 | /* ld r2,R2_STACK_OFFSET(r1) */ |
761 | return patch_instruction(instruction, ppc_inst(PPC_INST_LD_TOC)); |
762 | } |
763 | |
764 | int apply_relocate_add(Elf64_Shdr *sechdrs, |
765 | const char *strtab, |
766 | unsigned int symindex, |
767 | unsigned int relsec, |
768 | struct module *me) |
769 | { |
770 | unsigned int i; |
771 | Elf64_Rela *rela = (void *)sechdrs[relsec].sh_addr; |
772 | Elf64_Sym *sym; |
773 | unsigned long *location; |
774 | unsigned long value; |
775 | |
776 | pr_debug("Applying ADD relocate section %u to %u\n" , relsec, |
777 | sechdrs[relsec].sh_info); |
778 | |
779 | #ifndef CONFIG_PPC_KERNEL_PCREL |
780 | /* First time we're called, we can fix up .TOC. */ |
781 | if (!me->arch.toc_fixed) { |
782 | sym = find_dot_toc(sechdrs, strtab, symindex); |
783 | /* It's theoretically possible that a module doesn't want a |
784 | * .TOC. so don't fail it just for that. */ |
785 | if (sym) |
786 | sym->st_value = my_r2(sechdrs, me); |
787 | me->arch.toc_fixed = true; |
788 | } |
789 | #endif |
790 | for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rela); i++) { |
791 | /* This is where to make the change */ |
792 | location = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr |
793 | + rela[i].r_offset; |
794 | /* This is the symbol it is referring to */ |
795 | sym = (Elf64_Sym *)sechdrs[symindex].sh_addr |
796 | + ELF64_R_SYM(rela[i].r_info); |
797 | |
798 | pr_debug("RELOC at %p: %li-type as %s (0x%lx) + %li\n" , |
799 | location, (long)ELF64_R_TYPE(rela[i].r_info), |
800 | strtab + sym->st_name, (unsigned long)sym->st_value, |
801 | (long)rela[i].r_addend); |
802 | |
803 | /* `Everything is relative'. */ |
804 | value = sym->st_value + rela[i].r_addend; |
805 | |
806 | switch (ELF64_R_TYPE(rela[i].r_info)) { |
807 | case R_PPC64_ADDR32: |
808 | /* Simply set it */ |
809 | *(u32 *)location = value; |
810 | break; |
811 | |
812 | case R_PPC64_ADDR64: |
813 | /* Simply set it */ |
814 | *(unsigned long *)location = value; |
815 | break; |
816 | |
817 | #ifndef CONFIG_PPC_KERNEL_PCREL |
818 | case R_PPC64_TOC: |
819 | *(unsigned long *)location = my_r2(sechdrs, me); |
820 | break; |
821 | |
822 | case R_PPC64_TOC16: |
823 | /* Subtract TOC pointer */ |
824 | value -= my_r2(sechdrs, me); |
825 | if (value + 0x8000 > 0xffff) { |
826 | pr_err("%s: bad TOC16 relocation (0x%lx)\n" , |
827 | me->name, value); |
828 | return -ENOEXEC; |
829 | } |
830 | *((uint16_t *) location) |
831 | = (*((uint16_t *) location) & ~0xffff) |
832 | | (value & 0xffff); |
833 | break; |
834 | |
835 | case R_PPC64_TOC16_LO: |
836 | /* Subtract TOC pointer */ |
837 | value -= my_r2(sechdrs, me); |
838 | *((uint16_t *) location) |
839 | = (*((uint16_t *) location) & ~0xffff) |
840 | | (value & 0xffff); |
841 | break; |
842 | |
843 | case R_PPC64_TOC16_DS: |
844 | /* Subtract TOC pointer */ |
845 | value -= my_r2(sechdrs, me); |
846 | if ((value & 3) != 0 || value + 0x8000 > 0xffff) { |
847 | pr_err("%s: bad TOC16_DS relocation (0x%lx)\n" , |
848 | me->name, value); |
849 | return -ENOEXEC; |
850 | } |
851 | *((uint16_t *) location) |
852 | = (*((uint16_t *) location) & ~0xfffc) |
853 | | (value & 0xfffc); |
854 | break; |
855 | |
856 | case R_PPC64_TOC16_LO_DS: |
857 | /* Subtract TOC pointer */ |
858 | value -= my_r2(sechdrs, me); |
859 | if ((value & 3) != 0) { |
860 | pr_err("%s: bad TOC16_LO_DS relocation (0x%lx)\n" , |
861 | me->name, value); |
862 | return -ENOEXEC; |
863 | } |
864 | *((uint16_t *) location) |
865 | = (*((uint16_t *) location) & ~0xfffc) |
866 | | (value & 0xfffc); |
867 | break; |
868 | |
869 | case R_PPC64_TOC16_HA: |
870 | /* Subtract TOC pointer */ |
871 | value -= my_r2(sechdrs, me); |
872 | value = ((value + 0x8000) >> 16); |
873 | *((uint16_t *) location) |
874 | = (*((uint16_t *) location) & ~0xffff) |
875 | | (value & 0xffff); |
876 | break; |
877 | #endif |
878 | |
879 | case R_PPC_REL24: |
880 | #ifdef CONFIG_PPC_KERNEL_PCREL |
881 | /* PCREL still generates REL24 for mcount */ |
882 | case R_PPC64_REL24_NOTOC: |
883 | #endif |
884 | /* FIXME: Handle weak symbols here --RR */ |
885 | if (sym->st_shndx == SHN_UNDEF || |
886 | sym->st_shndx == SHN_LIVEPATCH) { |
887 | /* External: go via stub */ |
888 | value = stub_for_addr(sechdrs, addr: value, me, |
889 | name: strtab + sym->st_name); |
890 | if (!value) |
891 | return -ENOENT; |
892 | if (restore_r2(name: strtab + sym->st_name, |
893 | instruction: (u32 *)location + 1, me)) |
894 | return -ENOEXEC; |
895 | } else |
896 | value += local_entry_offset(sym); |
897 | |
898 | /* Convert value to relative */ |
899 | value -= (unsigned long)location; |
900 | if (value + 0x2000000 > 0x3ffffff || (value & 3) != 0){ |
901 | pr_err("%s: REL24 %li out of range!\n" , |
902 | me->name, (long int)value); |
903 | return -ENOEXEC; |
904 | } |
905 | |
906 | /* Only replace bits 2 through 26 */ |
907 | value = (*(uint32_t *)location & ~PPC_LI_MASK) | PPC_LI(value); |
908 | |
909 | if (patch_instruction((u32 *)location, ppc_inst(value))) |
910 | return -EFAULT; |
911 | |
912 | break; |
913 | |
914 | case R_PPC64_REL64: |
915 | /* 64 bits relative (used by features fixups) */ |
916 | *location = value - (unsigned long)location; |
917 | break; |
918 | |
919 | case R_PPC64_REL32: |
920 | /* 32 bits relative (used by relative exception tables) */ |
921 | /* Convert value to relative */ |
922 | value -= (unsigned long)location; |
923 | if (value + 0x80000000 > 0xffffffff) { |
924 | pr_err("%s: REL32 %li out of range!\n" , |
925 | me->name, (long int)value); |
926 | return -ENOEXEC; |
927 | } |
928 | *(u32 *)location = value; |
929 | break; |
930 | |
931 | #ifdef CONFIG_PPC_KERNEL_PCREL |
932 | case R_PPC64_PCREL34: { |
933 | unsigned long absvalue = value; |
934 | |
935 | /* Convert value to relative */ |
936 | value -= (unsigned long)location; |
937 | |
938 | if (value + 0x200000000 > 0x3ffffffff) { |
939 | if (sym->st_shndx != me->arch.pcpu_section) { |
940 | pr_err("%s: REL34 %li out of range!\n" , |
941 | me->name, (long)value); |
942 | return -ENOEXEC; |
943 | } |
944 | |
945 | /* |
946 | * per-cpu section is special cased because |
947 | * it is moved during loading, so has to be |
948 | * converted to use GOT. |
949 | */ |
950 | value = got_for_addr(sechdrs, absvalue, me, |
951 | strtab + sym->st_name); |
952 | if (!value) |
953 | return -ENOENT; |
954 | value -= (unsigned long)location; |
955 | |
956 | /* Turn pla into pld */ |
957 | if (patch_instruction((u32 *)location, |
958 | ppc_inst_prefix((*(u32 *)location & ~0x02000000), |
959 | (*((u32 *)location + 1) & ~0xf8000000) | 0xe4000000))) |
960 | return -EFAULT; |
961 | } |
962 | |
963 | if (patch_instruction((u32 *)location, |
964 | ppc_inst_prefix((*(u32 *)location & ~0x3ffff) | IMM_H18(value), |
965 | (*((u32 *)location + 1) & ~0xffff) | IMM_L(value)))) |
966 | return -EFAULT; |
967 | |
968 | break; |
969 | } |
970 | |
971 | #else |
972 | case R_PPC64_TOCSAVE: |
973 | /* |
974 | * Marker reloc indicates we don't have to save r2. |
975 | * That would only save us one instruction, so ignore |
976 | * it. |
977 | */ |
978 | break; |
979 | #endif |
980 | |
981 | case R_PPC64_ENTRY: |
982 | if (IS_ENABLED(CONFIG_PPC_KERNEL_PCREL)) |
983 | break; |
984 | |
985 | /* |
986 | * Optimize ELFv2 large code model entry point if |
987 | * the TOC is within 2GB range of current location. |
988 | */ |
989 | value = my_r2(sechdrs, me) - (unsigned long)location; |
990 | if (value + 0x80008000 > 0xffffffff) |
991 | break; |
992 | /* |
993 | * Check for the large code model prolog sequence: |
994 | * ld r2, ...(r12) |
995 | * add r2, r2, r12 |
996 | */ |
997 | if ((((uint32_t *)location)[0] & ~0xfffc) != PPC_RAW_LD(_R2, _R12, 0)) |
998 | break; |
999 | if (((uint32_t *)location)[1] != PPC_RAW_ADD(_R2, _R2, _R12)) |
1000 | break; |
1001 | /* |
1002 | * If found, replace it with: |
1003 | * addis r2, r12, (.TOC.-func)@ha |
1004 | * addi r2, r2, (.TOC.-func)@l |
1005 | */ |
1006 | ((uint32_t *)location)[0] = PPC_RAW_ADDIS(_R2, _R12, PPC_HA(value)); |
1007 | ((uint32_t *)location)[1] = PPC_RAW_ADDI(_R2, _R2, PPC_LO(value)); |
1008 | break; |
1009 | |
1010 | case R_PPC64_REL16_HA: |
1011 | /* Subtract location pointer */ |
1012 | value -= (unsigned long)location; |
1013 | value = ((value + 0x8000) >> 16); |
1014 | *((uint16_t *) location) |
1015 | = (*((uint16_t *) location) & ~0xffff) |
1016 | | (value & 0xffff); |
1017 | break; |
1018 | |
1019 | case R_PPC64_REL16_LO: |
1020 | /* Subtract location pointer */ |
1021 | value -= (unsigned long)location; |
1022 | *((uint16_t *) location) |
1023 | = (*((uint16_t *) location) & ~0xffff) |
1024 | | (value & 0xffff); |
1025 | break; |
1026 | |
1027 | #ifdef CONFIG_PPC_KERNEL_PCREL |
1028 | case R_PPC64_GOT_PCREL34: |
1029 | value = got_for_addr(sechdrs, value, me, |
1030 | strtab + sym->st_name); |
1031 | if (!value) |
1032 | return -ENOENT; |
1033 | value -= (unsigned long)location; |
1034 | ((uint32_t *)location)[0] = (((uint32_t *)location)[0] & ~0x3ffff) | |
1035 | ((value >> 16) & 0x3ffff); |
1036 | ((uint32_t *)location)[1] = (((uint32_t *)location)[1] & ~0xffff) | |
1037 | (value & 0xffff); |
1038 | break; |
1039 | #endif |
1040 | |
1041 | default: |
1042 | pr_err("%s: Unknown ADD relocation: %lu\n" , |
1043 | me->name, |
1044 | (unsigned long)ELF64_R_TYPE(rela[i].r_info)); |
1045 | return -ENOEXEC; |
1046 | } |
1047 | } |
1048 | |
1049 | return 0; |
1050 | } |
1051 | |
1052 | #ifdef CONFIG_DYNAMIC_FTRACE |
1053 | int module_trampoline_target(struct module *mod, unsigned long addr, |
1054 | unsigned long *target) |
1055 | { |
1056 | struct ppc64_stub_entry *stub; |
1057 | func_desc_t funcdata; |
1058 | u32 magic; |
1059 | |
1060 | if (!within_module_core(addr, mod)) { |
1061 | pr_err("%s: stub %lx not in module %s\n" , __func__, addr, mod->name); |
1062 | return -EFAULT; |
1063 | } |
1064 | |
1065 | stub = (struct ppc64_stub_entry *)addr; |
1066 | |
1067 | if (copy_from_kernel_nofault(dst: &magic, src: &stub->magic, |
1068 | size: sizeof(magic))) { |
1069 | pr_err("%s: fault reading magic for stub %lx for %s\n" , __func__, addr, mod->name); |
1070 | return -EFAULT; |
1071 | } |
1072 | |
1073 | if (magic != STUB_MAGIC) { |
1074 | pr_err("%s: bad magic for stub %lx for %s\n" , __func__, addr, mod->name); |
1075 | return -EFAULT; |
1076 | } |
1077 | |
1078 | if (copy_from_kernel_nofault(dst: &funcdata, src: &stub->funcdata, |
1079 | size: sizeof(funcdata))) { |
1080 | pr_err("%s: fault reading funcdata for stub %lx for %s\n" , __func__, addr, mod->name); |
1081 | return -EFAULT; |
1082 | } |
1083 | |
1084 | *target = stub_func_addr(func: funcdata); |
1085 | |
1086 | return 0; |
1087 | } |
1088 | |
1089 | int module_finalize_ftrace(struct module *mod, const Elf_Shdr *sechdrs) |
1090 | { |
1091 | mod->arch.tramp = stub_for_addr(sechdrs, |
1092 | addr: (unsigned long)ftrace_caller, |
1093 | me: mod, |
1094 | name: "ftrace_caller" ); |
1095 | #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS |
1096 | mod->arch.tramp_regs = stub_for_addr(sechdrs, |
1097 | addr: (unsigned long)ftrace_regs_caller, |
1098 | me: mod, |
1099 | name: "ftrace_regs_caller" ); |
1100 | if (!mod->arch.tramp_regs) |
1101 | return -ENOENT; |
1102 | #endif |
1103 | |
1104 | if (!mod->arch.tramp) |
1105 | return -ENOENT; |
1106 | |
1107 | return 0; |
1108 | } |
1109 | #endif |
1110 | |