1/* Machine-dependent ELF dynamic relocation inline functions. x86-64 version.
2 Copyright (C) 2001-2024 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
4
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
9
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
14
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <https://www.gnu.org/licenses/>. */
18
19#ifndef dl_machine_h
20#define dl_machine_h
21
22#define ELF_MACHINE_NAME "x86_64"
23
24#include <assert.h>
25#include <stdint.h>
26#include <sys/param.h>
27#include <sysdep.h>
28#include <tls.h>
29#include <dl-tlsdesc.h>
30#include <dl-static-tls.h>
31#include <dl-machine-rel.h>
32#include <isa-level.h>
33#ifdef __CET__
34# include <dl-cet.h>
35#else
36# define RTLD_START_ENABLE_X86_FEATURES
37#endif
38
39/* Translate a processor specific dynamic tag to the index in l_info array. */
40#define DT_X86_64(x) (DT_X86_64_##x - DT_LOPROC + DT_NUM)
41
42/* Return nonzero iff ELF header is compatible with the running host. */
43static inline int __attribute__ ((unused))
44elf_machine_matches_host (const ElfW(Ehdr) *ehdr)
45{
46 return ehdr->e_machine == EM_X86_64;
47}
48
49
50/* Return the run-time load address of the shared object. */
51static inline ElfW(Addr) __attribute__ ((unused))
52elf_machine_load_address (void)
53{
54 extern const ElfW(Ehdr) __ehdr_start attribute_hidden;
55 return (ElfW(Addr)) &__ehdr_start;
56}
57
58/* Return the link-time address of _DYNAMIC. */
59static inline ElfW(Addr) __attribute__ ((unused))
60elf_machine_dynamic (void)
61{
62 extern ElfW(Dyn) _DYNAMIC[] attribute_hidden;
63 return (ElfW(Addr)) _DYNAMIC - elf_machine_load_address ();
64}
65
66/* Set up the loaded object described by L so its unrelocated PLT
67 entries will jump to the on-demand fixup code in dl-runtime.c. */
68
69static inline int __attribute__ ((unused, always_inline))
70elf_machine_runtime_setup (struct link_map *l, struct r_scope_elem *scope[],
71 int lazy, int profile)
72{
73 Elf64_Addr *got;
74 extern void _dl_runtime_profile_sse (ElfW(Word)) attribute_hidden;
75 extern void _dl_runtime_profile_avx (ElfW(Word)) attribute_hidden;
76 extern void _dl_runtime_profile_avx512 (ElfW(Word)) attribute_hidden;
77
78 if (l->l_info[DT_JMPREL] && lazy)
79 {
80 /* The GOT entries for functions in the PLT have not yet been filled
81 in. Their initial contents will arrange when called to push an
82 offset into the .rel.plt section, push _GLOBAL_OFFSET_TABLE_[1],
83 and then jump to _GLOBAL_OFFSET_TABLE_[2]. */
84 got = (Elf64_Addr *) D_PTR (l, l_info[DT_PLTGOT]);
85 /* If a library is prelinked but we have to relocate anyway,
86 we have to be able to undo the prelinking of .got.plt.
87 The prelinker saved us here address of .plt + 0x16. */
88 if (got[1])
89 {
90 l->l_mach.plt = got[1] + l->l_addr;
91 l->l_mach.gotplt = (ElfW(Addr)) &got[3];
92 }
93 /* Identify this shared object. */
94 *(ElfW(Addr) *) (got + 1) = (ElfW(Addr)) l;
95
96#ifdef SHARED
97 /* The got[2] entry contains the address of a function which gets
98 called to get the address of a so far unresolved function and
99 jump to it. The profiling extension of the dynamic linker allows
100 to intercept the calls to collect information. In this case we
101 don't store the address in the GOT so that all future calls also
102 end in this function. */
103 if (__glibc_unlikely (profile))
104 {
105 const struct cpu_features* cpu_features = __get_cpu_features ();
106 if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512F))
107 *(ElfW(Addr) *) (got + 2) = (ElfW(Addr)) &_dl_runtime_profile_avx512;
108 else if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX))
109 *(ElfW(Addr) *) (got + 2) = (ElfW(Addr)) &_dl_runtime_profile_avx;
110 else
111 *(ElfW(Addr) *) (got + 2) = (ElfW(Addr)) &_dl_runtime_profile_sse;
112
113 if (GLRO(dl_profile) != NULL
114 && _dl_name_match_p (GLRO(dl_profile), map: l))
115 /* This is the object we are looking for. Say that we really
116 want profiling and the timers are started. */
117 GL(dl_profile_map) = l;
118 }
119 else
120#endif
121 {
122 /* This function will get called to fix up the GOT entry
123 indicated by the offset on the stack, and then jump to
124 the resolved address. */
125 *(ElfW(Addr) *) (got + 2)
126 = (ElfW(Addr)) GLRO(dl_x86_64_runtime_resolve);
127 }
128 }
129
130 return lazy;
131}
132
133/* Initial entry point code for the dynamic linker.
134 The C function `_dl_start' is the real entry point;
135 its return value is the user program's entry point. */
136#define RTLD_START asm ("\n\
137.text\n\
138 .align 16\n\
139.globl _start\n\
140.globl _dl_start_user\n\
141_start:\n\
142 mov %" RSP_LP ", %" RDI_LP "\n\
143 call _dl_start\n\
144_dl_start_user:\n\
145 # Save the user entry point address in %r12.\n\
146 mov %" RAX_LP ", %" R12_LP "\n\
147 # Save %rsp value in %r13.\n\
148 mov %" RSP_LP ", % " R13_LP "\n\
149"\
150 RTLD_START_ENABLE_X86_FEATURES \
151"\
152 # Read the original argument count.\n\
153 mov (%rsp), %" RDX_LP "\n\
154 # Call _dl_init (struct link_map *main_map, int argc, char **argv, char **env)\n\
155 # argc -> rsi\n\
156 mov %" RDX_LP ", %" RSI_LP "\n\
157 # And align stack for the _dl_init call. \n\
158 and $-16, %" RSP_LP "\n\
159 # _dl_loaded -> rdi\n\
160 mov _rtld_local(%rip), %" RDI_LP "\n\
161 # env -> rcx\n\
162 lea 2*" LP_SIZE "(%r13,%rdx," LP_SIZE "), %" RCX_LP "\n\
163 # argv -> rdx\n\
164 lea " LP_SIZE "(%r13), %" RDX_LP "\n\
165 # Clear %rbp to mark outermost frame obviously even for constructors.\n\
166 xorl %ebp, %ebp\n\
167 # Call the function to run the initializers.\n\
168 call _dl_init\n\
169 # Pass our finalizer function to the user in %rdx, as per ELF ABI.\n\
170 lea _dl_fini(%rip), %" RDX_LP "\n\
171 # And make sure %rsp points to argc stored on the stack.\n\
172 mov %" R13_LP ", %" RSP_LP "\n\
173 # Jump to the user's entry point.\n\
174 jmp *%r12\n\
175.previous\n\
176");
177
178/* ELF_RTYPE_CLASS_PLT iff TYPE describes relocation of a PLT entry or
179 TLS variable, so undefined references should not be allowed to
180 define the value.
181 ELF_RTYPE_CLASS_COPY iff TYPE should not be allowed to resolve to one
182 of the main executable's symbols, as for a COPY reloc. */
183#define elf_machine_type_class(type) \
184 ((((type) == R_X86_64_JUMP_SLOT \
185 || (type) == R_X86_64_DTPMOD64 \
186 || (type) == R_X86_64_DTPOFF64 \
187 || (type) == R_X86_64_TPOFF64 \
188 || (type) == R_X86_64_TLSDESC) \
189 * ELF_RTYPE_CLASS_PLT) \
190 | (((type) == R_X86_64_COPY) * ELF_RTYPE_CLASS_COPY))
191
192/* A reloc type used for ld.so cmdline arg lookups to reject PLT entries. */
193#define ELF_MACHINE_JMP_SLOT R_X86_64_JUMP_SLOT
194
195/* The relative ifunc relocation. */
196// XXX This is a work-around for a broken linker. Remove!
197#define ELF_MACHINE_IRELATIVE R_X86_64_IRELATIVE
198
199/* We define an initialization function. This is called very early in
200 _dl_sysdep_start. */
201#define DL_PLATFORM_INIT dl_platform_init ()
202
203static inline void __attribute__ ((unused))
204dl_platform_init (void)
205{
206#if IS_IN (rtld)
207 /* _dl_x86_init_cpu_features is a wrapper for init_cpu_features which
208 has been called early from __libc_start_main in static executable. */
209 _dl_x86_init_cpu_features ();
210#else
211 if (GLRO(dl_platform) != NULL && *GLRO(dl_platform) == '\0')
212 /* Avoid an empty string which would disturb us. */
213 GLRO(dl_platform) = NULL;
214#endif
215}
216
217static inline ElfW(Addr)
218elf_machine_fixup_plt (struct link_map *map, lookup_t t,
219 const ElfW(Sym) *refsym, const ElfW(Sym) *sym,
220 const ElfW(Rela) *reloc,
221 ElfW(Addr) *reloc_addr, ElfW(Addr) value)
222{
223 return *reloc_addr = value;
224}
225
226/* Return the final value of a PLT relocation. On x86-64 the
227 JUMP_SLOT relocation ignores the addend. */
228static inline ElfW(Addr)
229elf_machine_plt_value (struct link_map *map, const ElfW(Rela) *reloc,
230 ElfW(Addr) value)
231{
232 return value;
233}
234
235
236/* Names of the architecture-specific auditing callback functions. */
237#ifdef __LP64__
238#define ARCH_LA_PLTENTER x86_64_gnu_pltenter
239#define ARCH_LA_PLTEXIT x86_64_gnu_pltexit
240#else
241#define ARCH_LA_PLTENTER x32_gnu_pltenter
242#define ARCH_LA_PLTEXIT x32_gnu_pltexit
243#endif
244
245#endif /* !dl_machine_h */
246
247#ifdef RESOLVE_MAP
248
249/* Perform the relocation specified by RELOC and SYM (which is fully resolved).
250 MAP is the object containing the reloc. */
251
252static inline void __attribute__((always_inline))
253elf_machine_rela(struct link_map *map, struct r_scope_elem *scope[],
254 const ElfW(Rela) *reloc, const ElfW(Sym) *sym,
255 const struct r_found_version *version,
256 void *const reloc_addr_arg, int skip_ifunc) {
257 ElfW(Addr) *const reloc_addr = reloc_addr_arg;
258 const unsigned long int r_type = ELFW(R_TYPE) (reloc->r_info);
259
260# if !defined RTLD_BOOTSTRAP
261 if (__glibc_unlikely (r_type == R_X86_64_RELATIVE))
262 *reloc_addr = map->l_addr + reloc->r_addend;
263 else
264# endif
265# if !defined RTLD_BOOTSTRAP
266 /* l_addr + r_addend may be > 0xffffffff and R_X86_64_RELATIVE64
267 relocation updates the whole 64-bit entry. */
268 if (__glibc_unlikely (r_type == R_X86_64_RELATIVE64))
269 *(Elf64_Addr *) reloc_addr = (Elf64_Addr) map->l_addr + reloc->r_addend;
270 else
271# endif
272 if (__glibc_unlikely (r_type == R_X86_64_NONE))
273 return;
274 else
275 {
276# ifndef RTLD_BOOTSTRAP
277 const ElfW(Sym) *const refsym = sym;
278# endif
279 struct link_map *sym_map = RESOLVE_MAP (map, scope, &sym, version,
280 r_type);
281 ElfW(Addr) value = SYMBOL_ADDRESS (sym_map, sym, true);
282
283 if (sym != NULL
284 && __glibc_unlikely (ELFW(ST_TYPE) (sym->st_info) == STT_GNU_IFUNC)
285 && __glibc_likely (sym->st_shndx != SHN_UNDEF)
286 && __glibc_likely (!skip_ifunc))
287 {
288# ifndef RTLD_BOOTSTRAP
289 if (sym_map != map
290 && !sym_map->l_relocated)
291 {
292 const char *strtab
293 = (const char *) D_PTR (map, l_info[DT_STRTAB]);
294 if (sym_map->l_type == lt_executable)
295 _dl_fatal_printf ("\
296%s: IFUNC symbol '%s' referenced in '%s' is defined in the executable \
297and creates an unsatisfiable circular dependency.\n",
298 RTLD_PROGNAME, strtab + refsym->st_name,
299 map->l_name);
300 else
301 _dl_error_printf ("\
302%s: Relink `%s' with `%s' for IFUNC symbol `%s'\n",
303 RTLD_PROGNAME, map->l_name,
304 sym_map->l_name,
305 strtab + refsym->st_name);
306 }
307# endif
308 value = ((ElfW(Addr) (*) (void)) value) ();
309 }
310
311 switch (r_type)
312 {
313 case R_X86_64_JUMP_SLOT:
314 map->l_has_jump_slot_reloc = true;
315 /* fallthrough */
316 case R_X86_64_GLOB_DAT:
317 *reloc_addr = value;
318 break;
319
320# ifndef RTLD_BOOTSTRAP
321# ifdef __ILP32__
322 case R_X86_64_SIZE64:
323 /* Set to symbol size plus addend. */
324 *(Elf64_Addr *) (uintptr_t) reloc_addr
325 = (Elf64_Addr) sym->st_size + reloc->r_addend;
326 break;
327
328 case R_X86_64_SIZE32:
329# else
330 case R_X86_64_SIZE64:
331# endif
332 /* Set to symbol size plus addend. */
333 value = sym->st_size;
334 *reloc_addr = value + reloc->r_addend;
335 break;
336
337 case R_X86_64_DTPMOD64:
338 /* Get the information from the link map returned by the
339 resolve function. */
340 if (sym_map != NULL)
341 *reloc_addr = sym_map->l_tls_modid;
342 break;
343 case R_X86_64_DTPOFF64:
344 /* During relocation all TLS symbols are defined and used.
345 Therefore the offset is already correct. */
346 if (sym != NULL)
347 {
348 value = sym->st_value + reloc->r_addend;
349# ifdef __ILP32__
350 /* This relocation type computes a signed offset that is
351 usually negative. The symbol and addend values are 32
352 bits but the GOT entry is 64 bits wide and the whole
353 64-bit entry is used as a signed quantity, so we need
354 to sign-extend the computed value to 64 bits. */
355 *(Elf64_Sxword *) reloc_addr = (Elf64_Sxword) (Elf32_Sword) value;
356# else
357 *reloc_addr = value;
358# endif
359 }
360 break;
361 case R_X86_64_TLSDESC:
362 {
363 struct tlsdesc volatile *td =
364 (struct tlsdesc volatile *)reloc_addr;
365
366 if (! sym)
367 {
368 td->arg = (void*)reloc->r_addend;
369 td->entry = _dl_tlsdesc_undefweak;
370 }
371 else
372 {
373# ifndef SHARED
374 CHECK_STATIC_TLS (map, sym_map);
375# else
376 if (!TRY_STATIC_TLS (map, sym_map))
377 {
378 td->arg = _dl_make_tlsdesc_dynamic
379 (sym_map, sym->st_value + reloc->r_addend);
380 td->entry = GLRO(dl_x86_tlsdesc_dynamic);
381 }
382 else
383# endif
384 {
385 td->arg = (void*)(sym->st_value - sym_map->l_tls_offset
386 + reloc->r_addend);
387 td->entry = _dl_tlsdesc_return;
388 }
389 }
390 break;
391 }
392 case R_X86_64_TPOFF64:
393 /* The offset is negative, forward from the thread pointer. */
394 if (sym != NULL)
395 {
396 CHECK_STATIC_TLS (map, sym_map);
397 /* We know the offset of the object the symbol is contained in.
398 It is a negative value which will be added to the
399 thread pointer. */
400 value = (sym->st_value + reloc->r_addend
401 - sym_map->l_tls_offset);
402# ifdef __ILP32__
403 /* The symbol and addend values are 32 bits but the GOT
404 entry is 64 bits wide and the whole 64-bit entry is used
405 as a signed quantity, so we need to sign-extend the
406 computed value to 64 bits. */
407 *(Elf64_Sxword *) reloc_addr = (Elf64_Sxword) (Elf32_Sword) value;
408# else
409 *reloc_addr = value;
410# endif
411 }
412 break;
413
414 case R_X86_64_64:
415 /* value + r_addend may be > 0xffffffff and R_X86_64_64
416 relocation updates the whole 64-bit entry. */
417 *(Elf64_Addr *) reloc_addr = (Elf64_Addr) value + reloc->r_addend;
418 break;
419# ifndef __ILP32__
420 case R_X86_64_SIZE32:
421 /* Set to symbol size plus addend. */
422 value = sym->st_size;
423# endif
424 /* Fall through. */
425 case R_X86_64_32:
426 value += reloc->r_addend;
427 *(unsigned int *) reloc_addr = value;
428
429 const char *fmt;
430 if (__glibc_unlikely (value > UINT_MAX))
431 {
432 const char *strtab;
433
434 fmt = "\
435%s: Symbol `%s' causes overflow in R_X86_64_32 relocation\n";
436 print_err:
437 strtab = (const char *) D_PTR (map, l_info[DT_STRTAB]);
438
439 _dl_error_printf (fmt, RTLD_PROGNAME, strtab + refsym->st_name);
440 }
441 break;
442 /* Not needed for dl-conflict.c. */
443 case R_X86_64_PC32:
444 value += reloc->r_addend - (ElfW(Addr)) reloc_addr;
445 *(unsigned int *) reloc_addr = value;
446 if (__glibc_unlikely (value != (int) value))
447 {
448 fmt = "\
449%s: Symbol `%s' causes overflow in R_X86_64_PC32 relocation\n";
450 goto print_err;
451 }
452 break;
453 case R_X86_64_COPY:
454 if (sym == NULL)
455 /* This can happen in trace mode if an object could not be
456 found. */
457 break;
458 memcpy (reloc_addr_arg, (void *) value,
459 MIN (sym->st_size, refsym->st_size));
460 if (__glibc_unlikely (sym->st_size > refsym->st_size)
461 || (__glibc_unlikely (sym->st_size < refsym->st_size)
462 && GLRO(dl_verbose)))
463 {
464 fmt = "\
465%s: Symbol `%s' has different size in shared object, consider re-linking\n";
466 goto print_err;
467 }
468 break;
469 case R_X86_64_IRELATIVE:
470 value = map->l_addr + reloc->r_addend;
471 if (__glibc_likely (!skip_ifunc))
472 value = ((ElfW(Addr) (*) (void)) value) ();
473 *reloc_addr = value;
474 break;
475 default:
476 _dl_reloc_bad_type (map, r_type, 0);
477 break;
478# endif /* !RTLD_BOOTSTRAP */
479 }
480 }
481}
482
483static inline void
484__attribute ((always_inline))
485elf_machine_rela_relative (ElfW(Addr) l_addr, const ElfW(Rela) *reloc,
486 void *const reloc_addr_arg)
487{
488 ElfW(Addr) *const reloc_addr = reloc_addr_arg;
489#if !defined RTLD_BOOTSTRAP
490 /* l_addr + r_addend may be > 0xffffffff and R_X86_64_RELATIVE64
491 relocation updates the whole 64-bit entry. */
492 if (__glibc_unlikely (ELFW(R_TYPE) (reloc->r_info) == R_X86_64_RELATIVE64))
493 *(Elf64_Addr *) reloc_addr = (Elf64_Addr) l_addr + reloc->r_addend;
494 else
495#endif
496 {
497 assert (ELFW(R_TYPE) (reloc->r_info) == R_X86_64_RELATIVE);
498 *reloc_addr = l_addr + reloc->r_addend;
499 }
500}
501
502static inline void
503__attribute ((always_inline))
504elf_machine_lazy_rel (struct link_map *map, struct r_scope_elem *scope[],
505 ElfW(Addr) l_addr, const ElfW(Rela) *reloc,
506 int skip_ifunc)
507{
508 ElfW(Addr) *const reloc_addr = (void *) (l_addr + reloc->r_offset);
509 const unsigned long int r_type = ELFW(R_TYPE) (reloc->r_info);
510
511 /* Check for unexpected PLT reloc type. */
512 if (__glibc_likely (r_type == R_X86_64_JUMP_SLOT))
513 {
514 /* Prelink has been deprecated. */
515 if (__glibc_likely (map->l_mach.plt == 0))
516 *reloc_addr += l_addr;
517 else
518 *reloc_addr =
519 map->l_mach.plt
520 + (((ElfW(Addr)) reloc_addr) - map->l_mach.gotplt) * 2;
521 }
522 else if (__glibc_likely (r_type == R_X86_64_TLSDESC))
523 {
524 const Elf_Symndx symndx = ELFW (R_SYM) (reloc->r_info);
525 const ElfW (Sym) *symtab = (const void *)D_PTR (map, l_info[DT_SYMTAB]);
526 const ElfW (Sym) *sym = &symtab[symndx];
527 const struct r_found_version *version = NULL;
528
529 if (map->l_info[VERSYMIDX (DT_VERSYM)] != NULL)
530 {
531 const ElfW (Half) *vernum =
532 (const void *)D_PTR (map, l_info[VERSYMIDX (DT_VERSYM)]);
533 version = &map->l_versions[vernum[symndx] & 0x7fff];
534 }
535
536 /* Always initialize TLS descriptors completely at load time, in
537 case static TLS is allocated for it that requires locking. */
538 elf_machine_rela (map, scope, reloc, sym, version, reloc_addr, skip_ifunc);
539 }
540 else if (__glibc_unlikely (r_type == R_X86_64_IRELATIVE))
541 {
542 ElfW(Addr) value = map->l_addr + reloc->r_addend;
543 if (__glibc_likely (!skip_ifunc))
544 value = ((ElfW(Addr) (*) (void)) value) ();
545 *reloc_addr = value;
546 }
547 else
548 _dl_reloc_bad_type (map, r_type, 1);
549}
550
551#endif /* RESOLVE_MAP */
552
553#if !defined ELF_DYNAMIC_AFTER_RELOC && !defined RTLD_BOOTSTRAP \
554 && defined SHARED
555# define ELF_DYNAMIC_AFTER_RELOC(map, lazy) \
556 x86_64_dynamic_after_reloc (map, (lazy))
557
558# define JMP32_INSN_OPCODE 0xe9
559# define JMP32_INSN_SIZE 5
560# define JMPABS_INSN_OPCODE 0xa100d5
561# define JMPABS_INSN_SIZE 11
562# define INT3_INSN_OPCODE 0xcc
563
564static const char *
565x86_64_reloc_symbol_name (struct link_map *map, const ElfW(Rela) *reloc)
566{
567 const ElfW(Sym) *const symtab
568 = (const void *) map->l_info[DT_SYMTAB]->d_un.d_ptr;
569 const ElfW(Sym) *const refsym = &symtab[ELFW (R_SYM) (reloc->r_info)];
570 const char *strtab = (const char *) map->l_info[DT_STRTAB]->d_un.d_ptr;
571 return strtab + refsym->st_name;
572}
573
574static void
575x86_64_rewrite_plt (struct link_map *map, ElfW(Addr) plt_rewrite)
576{
577 ElfW(Addr) l_addr = map->l_addr;
578 ElfW(Addr) pltent = map->l_info[DT_X86_64 (PLTENT)]->d_un.d_val;
579 ElfW(Addr) start = map->l_info[DT_JMPREL]->d_un.d_ptr;
580 ElfW(Addr) size = map->l_info[DT_PLTRELSZ]->d_un.d_val;
581 const ElfW(Rela) *reloc = (const void *) start;
582 const ElfW(Rela) *reloc_end = (const void *) (start + size);
583
584# ifdef __CET__
585 bool ibt_enabled_p = dl_cet_ibt_enabled ();
586# else
587 bool ibt_enabled_p = false;
588# endif
589
590 if (__glibc_unlikely (GLRO(dl_debug_mask) & DL_DEBUG_FILES))
591 _dl_debug_printf (fmt: "\nchanging PLT in '%s' to direct branch\n",
592 DSO_FILENAME (map->l_name));
593
594 for (; reloc < reloc_end; reloc++)
595 if (ELFW(R_TYPE) (reloc->r_info) == R_X86_64_JUMP_SLOT)
596 {
597 /* Get the value from the GOT entry. */
598 ElfW(Addr) value = *(ElfW(Addr) *) (l_addr + reloc->r_offset);
599
600 /* Get the corresponding PLT entry from r_addend. */
601 ElfW(Addr) branch_start = l_addr + reloc->r_addend;
602 /* Skip ENDBR64 if IBT isn't enabled. */
603 if (!ibt_enabled_p)
604 branch_start = ALIGN_DOWN (branch_start, pltent);
605 /* Get the displacement from the branch target. NB: We must use
606 64-bit integer on x32 to avoid overflow. */
607 uint64_t disp = (uint64_t) value - branch_start - JMP32_INSN_SIZE;
608 ElfW(Addr) plt_end;
609 ElfW(Addr) pad;
610
611 plt_end = (branch_start | (pltent - 1)) + 1;
612
613 /* Update the PLT entry. */
614 if (((uint64_t) disp + (uint64_t) ((uint32_t) INT32_MIN))
615 <= (uint64_t) UINT32_MAX)
616 {
617 pad = branch_start + JMP32_INSN_SIZE;
618
619 if (__glibc_unlikely (pad > plt_end))
620 continue;
621
622 /* If the target branch can be reached with a direct branch,
623 rewrite the PLT entry with a direct branch. */
624 if (__glibc_unlikely (GLRO(dl_debug_mask) & DL_DEBUG_BINDINGS))
625 {
626 const char *sym_name = x86_64_reloc_symbol_name (map,
627 reloc);
628 _dl_debug_printf (fmt: "changing '%s' PLT entry in '%s' to "
629 "direct branch\n", sym_name,
630 DSO_FILENAME (map->l_name));
631 }
632
633 /* Write out direct branch. */
634 *(uint8_t *) branch_start = JMP32_INSN_OPCODE;
635 *(uint32_t *) (branch_start + 1) = disp;
636 }
637 else
638 {
639 if (GL(dl_x86_feature_control).plt_rewrite
640 != plt_rewrite_jmpabs)
641 {
642 if (__glibc_unlikely (GLRO(dl_debug_mask)
643 & DL_DEBUG_BINDINGS))
644 {
645 const char *sym_name
646 = x86_64_reloc_symbol_name (map, reloc);
647 _dl_debug_printf (fmt: "skipping '%s' PLT entry in '%s'\n",
648 sym_name,
649 DSO_FILENAME (map->l_name));
650 }
651 continue;
652 }
653
654 pad = branch_start + JMPABS_INSN_SIZE;
655
656 if (__glibc_unlikely (pad > plt_end))
657 continue;
658
659 /* Rewrite the PLT entry with JMPABS. */
660 if (__glibc_unlikely (GLRO(dl_debug_mask) & DL_DEBUG_BINDINGS))
661 {
662 const char *sym_name = x86_64_reloc_symbol_name (map,
663 reloc);
664 _dl_debug_printf (fmt: "changing '%s' PLT entry in '%s' to "
665 "JMPABS\n", sym_name,
666 DSO_FILENAME (map->l_name));
667 }
668
669 /* "jmpabs $target" for 64-bit displacement. NB: JMPABS has
670 a 3-byte opcode + 64bit address. There is a 1-byte overlap
671 between 4-byte write and 8-byte write. */
672 *(uint32_t *) (branch_start) = JMPABS_INSN_OPCODE;
673 *(uint64_t *) (branch_start + 3) = value;
674 }
675
676 /* Fill the unused part of the PLT entry with INT3. */
677 for (; pad < plt_end; pad++)
678 *(uint8_t *) pad = INT3_INSN_OPCODE;
679 }
680}
681
682static inline void
683x86_64_rewrite_plt_in_place (struct link_map *map)
684{
685 /* Adjust DT_X86_64_PLT address and DT_X86_64_PLTSZ values. */
686 ElfW(Addr) plt = (map->l_info[DT_X86_64 (PLT)]->d_un.d_ptr
687 + map->l_addr);
688 size_t pagesize = GLRO(dl_pagesize);
689 ElfW(Addr) plt_aligned = ALIGN_DOWN (plt, pagesize);
690 size_t pltsz = (map->l_info[DT_X86_64 (PLTSZ)]->d_un.d_val
691 + plt - plt_aligned);
692
693 if (__glibc_unlikely (GLRO(dl_debug_mask) & DL_DEBUG_FILES))
694 _dl_debug_printf (fmt: "\nchanging PLT in '%s' to writable\n",
695 DSO_FILENAME (map->l_name));
696
697 if (__glibc_unlikely (__mprotect ((void *) plt_aligned, pltsz,
698 PROT_WRITE | PROT_READ) < 0))
699 {
700 if (__glibc_unlikely (GLRO(dl_debug_mask) & DL_DEBUG_FILES))
701 _dl_debug_printf (fmt: "\nfailed to change PLT in '%s' to writable\n",
702 DSO_FILENAME (map->l_name));
703 return;
704 }
705
706 x86_64_rewrite_plt (map, plt_rewrite: plt_aligned);
707
708 if (__glibc_unlikely (GLRO(dl_debug_mask) & DL_DEBUG_FILES))
709 _dl_debug_printf (fmt: "\nchanging PLT in '%s' back to read-only\n",
710 DSO_FILENAME (map->l_name));
711
712 if (__glibc_unlikely (__mprotect ((void *) plt_aligned, pltsz,
713 PROT_EXEC | PROT_READ) < 0))
714 _dl_signal_error (0, DSO_FILENAME (map->l_name), NULL,
715 "failed to change PLT back to read-only");
716}
717
718/* Rewrite PLT entries to direct branch if possible. */
719
720static inline void
721x86_64_dynamic_after_reloc (struct link_map *map, int lazy)
722{
723 /* Ignore DT_X86_64_PLT if the lazy binding is enabled. */
724 if (lazy != 0)
725 return;
726
727 /* Ignore DT_X86_64_PLT if PLT rewrite isn't enabled. */
728 if (__glibc_likely (GL(dl_x86_feature_control).plt_rewrite
729 == plt_rewrite_none))
730 return;
731
732 if (__glibc_likely (map->l_info[DT_X86_64 (PLT)] == NULL))
733 return;
734
735 /* Ignore DT_X86_64_PLT if there is no R_X86_64_JUMP_SLOT. */
736 if (map->l_has_jump_slot_reloc == 0)
737 return;
738
739 /* Ignore DT_X86_64_PLT if
740 1. DT_JMPREL isn't available or its value is 0.
741 2. DT_PLTRELSZ is 0.
742 3. DT_X86_64_PLTENT isn't available or its value is smaller than
743 16 bytes.
744 4. DT_X86_64_PLTSZ isn't available or its value is smaller than
745 DT_X86_64_PLTENT's value or isn't a multiple of DT_X86_64_PLTENT's
746 value. */
747 if (map->l_info[DT_JMPREL] == NULL
748 || map->l_info[DT_JMPREL]->d_un.d_ptr == 0
749 || map->l_info[DT_PLTRELSZ]->d_un.d_val == 0
750 || map->l_info[DT_X86_64 (PLTSZ)] == NULL
751 || map->l_info[DT_X86_64 (PLTENT)] == NULL
752 || map->l_info[DT_X86_64 (PLTENT)]->d_un.d_val < 16
753 || (map->l_info[DT_X86_64 (PLTSZ)]->d_un.d_val
754 < map->l_info[DT_X86_64 (PLTENT)]->d_un.d_val)
755 || (map->l_info[DT_X86_64 (PLTSZ)]->d_un.d_val
756 % map->l_info[DT_X86_64 (PLTENT)]->d_un.d_val) != 0)
757 return;
758
759 x86_64_rewrite_plt_in_place (map);
760}
761#endif
762

source code of glibc/sysdeps/x86_64/dl-machine.h