1//===- AArch64.cpp --------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "InputFiles.h"
10#include "OutputSections.h"
11#include "Symbols.h"
12#include "SyntheticSections.h"
13#include "Target.h"
14#include "llvm/BinaryFormat/ELF.h"
15#include "llvm/Support/Endian.h"
16
17using namespace llvm;
18using namespace llvm::support::endian;
19using namespace llvm::ELF;
20using namespace lld;
21using namespace lld::elf;
22
23// Page(Expr) is the page address of the expression Expr, defined
24// as (Expr & ~0xFFF). (This applies even if the machine page size
25// supported by the platform has a different value.)
26uint64_t elf::getAArch64Page(uint64_t expr) {
27 return expr & ~static_cast<uint64_t>(0xFFF);
28}
29
30// A BTI landing pad is a valid target for an indirect branch when the Branch
31// Target Identification has been enabled. As linker generated branches are
32// via x16 the BTI landing pads are defined as: BTI C, BTI J, BTI JC, PACIASP,
33// PACIBSP.
34bool elf::isAArch64BTILandingPad(Ctx &ctx, Symbol &s, int64_t a) {
35 // PLT entries accessed indirectly have a BTI c.
36 if (s.isInPlt(ctx))
37 return true;
38 Defined *d = dyn_cast<Defined>(Val: &s);
39 if (!isa_and_nonnull<InputSection>(Val: d->section))
40 // All places that we cannot disassemble are responsible for making
41 // the target a BTI landing pad.
42 return true;
43 InputSection *isec = cast<InputSection>(Val: d->section);
44 uint64_t off = d->value + a;
45 // Likely user error, but protect ourselves against out of bounds
46 // access.
47 if (off >= isec->getSize())
48 return true;
49 const uint8_t *buf = isec->content().begin();
50 const uint32_t instr = read32le(P: buf + off);
51 // All BTI instructions are HINT instructions which all have same encoding
52 // apart from bits [11:5]
53 if ((instr & 0xd503201f) == 0xd503201f &&
54 is_contained(Set: {/*PACIASP*/ 0xd503233f, /*PACIBSP*/ 0xd503237f,
55 /*BTI C*/ 0xd503245f, /*BTI J*/ 0xd503249f,
56 /*BTI JC*/ 0xd50324df},
57 Element: instr))
58 return true;
59 return false;
60}
61
62namespace {
63class AArch64 : public TargetInfo {
64public:
65 AArch64(Ctx &);
66 RelExpr getRelExpr(RelType type, const Symbol &s,
67 const uint8_t *loc) const override;
68 RelType getDynRel(RelType type) const override;
69 int64_t getImplicitAddend(const uint8_t *buf, RelType type) const override;
70 void writeGotPlt(uint8_t *buf, const Symbol &s) const override;
71 void writeIgotPlt(uint8_t *buf, const Symbol &s) const override;
72 void writePltHeader(uint8_t *buf) const override;
73 void writePlt(uint8_t *buf, const Symbol &sym,
74 uint64_t pltEntryAddr) const override;
75 bool needsThunk(RelExpr expr, RelType type, const InputFile *file,
76 uint64_t branchAddr, const Symbol &s,
77 int64_t a) const override;
78 uint32_t getThunkSectionSpacing() const override;
79 bool inBranchRange(RelType type, uint64_t src, uint64_t dst) const override;
80 bool usesOnlyLowPageBits(RelType type) const override;
81 void relocate(uint8_t *loc, const Relocation &rel,
82 uint64_t val) const override;
83 RelExpr adjustTlsExpr(RelType type, RelExpr expr) const override;
84 void relocateAlloc(InputSectionBase &sec, uint8_t *buf) const override;
85
86private:
87 void relaxTlsGdToLe(uint8_t *loc, const Relocation &rel, uint64_t val) const;
88 void relaxTlsGdToIe(uint8_t *loc, const Relocation &rel, uint64_t val) const;
89 void relaxTlsIeToLe(uint8_t *loc, const Relocation &rel, uint64_t val) const;
90};
91
92struct AArch64Relaxer {
93 Ctx &ctx;
94 bool safeToRelaxAdrpLdr = false;
95
96 AArch64Relaxer(Ctx &ctx, ArrayRef<Relocation> relocs);
97 bool tryRelaxAdrpAdd(const Relocation &adrpRel, const Relocation &addRel,
98 uint64_t secAddr, uint8_t *buf) const;
99 bool tryRelaxAdrpLdr(const Relocation &adrpRel, const Relocation &ldrRel,
100 uint64_t secAddr, uint8_t *buf) const;
101};
102} // namespace
103
104// Return the bits [Start, End] from Val shifted Start bits.
105// For instance, getBits(0xF0, 4, 8) returns 0xF.
106static uint64_t getBits(uint64_t val, int start, int end) {
107 uint64_t mask = ((uint64_t)1 << (end + 1 - start)) - 1;
108 return (val >> start) & mask;
109}
110
111AArch64::AArch64(Ctx &ctx) : TargetInfo(ctx) {
112 copyRel = R_AARCH64_COPY;
113 relativeRel = R_AARCH64_RELATIVE;
114 iRelativeRel = R_AARCH64_IRELATIVE;
115 gotRel = R_AARCH64_GLOB_DAT;
116 pltRel = R_AARCH64_JUMP_SLOT;
117 symbolicRel = R_AARCH64_ABS64;
118 tlsDescRel = R_AARCH64_TLSDESC;
119 tlsGotRel = R_AARCH64_TLS_TPREL64;
120 pltHeaderSize = 32;
121 pltEntrySize = 16;
122 ipltEntrySize = 16;
123 defaultMaxPageSize = 65536;
124
125 // Align to the 2 MiB page size (known as a superpage or huge page).
126 // FreeBSD automatically promotes 2 MiB-aligned allocations.
127 defaultImageBase = 0x200000;
128
129 needsThunks = true;
130}
131
132RelExpr AArch64::getRelExpr(RelType type, const Symbol &s,
133 const uint8_t *loc) const {
134 switch (type) {
135 case R_AARCH64_ABS16:
136 case R_AARCH64_ABS32:
137 case R_AARCH64_ABS64:
138 case R_AARCH64_ADD_ABS_LO12_NC:
139 case R_AARCH64_LDST128_ABS_LO12_NC:
140 case R_AARCH64_LDST16_ABS_LO12_NC:
141 case R_AARCH64_LDST32_ABS_LO12_NC:
142 case R_AARCH64_LDST64_ABS_LO12_NC:
143 case R_AARCH64_LDST8_ABS_LO12_NC:
144 case R_AARCH64_MOVW_SABS_G0:
145 case R_AARCH64_MOVW_SABS_G1:
146 case R_AARCH64_MOVW_SABS_G2:
147 case R_AARCH64_MOVW_UABS_G0:
148 case R_AARCH64_MOVW_UABS_G0_NC:
149 case R_AARCH64_MOVW_UABS_G1:
150 case R_AARCH64_MOVW_UABS_G1_NC:
151 case R_AARCH64_MOVW_UABS_G2:
152 case R_AARCH64_MOVW_UABS_G2_NC:
153 case R_AARCH64_MOVW_UABS_G3:
154 return R_ABS;
155 case R_AARCH64_AUTH_ABS64:
156 return RE_AARCH64_AUTH;
157 case R_AARCH64_TLSDESC_ADR_PAGE21:
158 return RE_AARCH64_TLSDESC_PAGE;
159 case R_AARCH64_AUTH_TLSDESC_ADR_PAGE21:
160 return RE_AARCH64_AUTH_TLSDESC_PAGE;
161 case R_AARCH64_TLSDESC_LD64_LO12:
162 case R_AARCH64_TLSDESC_ADD_LO12:
163 return R_TLSDESC;
164 case R_AARCH64_AUTH_TLSDESC_LD64_LO12:
165 case R_AARCH64_AUTH_TLSDESC_ADD_LO12:
166 return RE_AARCH64_AUTH_TLSDESC;
167 case R_AARCH64_TLSDESC_CALL:
168 return R_TLSDESC_CALL;
169 case R_AARCH64_TLSLE_ADD_TPREL_HI12:
170 case R_AARCH64_TLSLE_ADD_TPREL_LO12_NC:
171 case R_AARCH64_TLSLE_LDST8_TPREL_LO12_NC:
172 case R_AARCH64_TLSLE_LDST16_TPREL_LO12_NC:
173 case R_AARCH64_TLSLE_LDST32_TPREL_LO12_NC:
174 case R_AARCH64_TLSLE_LDST64_TPREL_LO12_NC:
175 case R_AARCH64_TLSLE_LDST128_TPREL_LO12_NC:
176 case R_AARCH64_TLSLE_MOVW_TPREL_G0:
177 case R_AARCH64_TLSLE_MOVW_TPREL_G0_NC:
178 case R_AARCH64_TLSLE_MOVW_TPREL_G1:
179 case R_AARCH64_TLSLE_MOVW_TPREL_G1_NC:
180 case R_AARCH64_TLSLE_MOVW_TPREL_G2:
181 return R_TPREL;
182 case R_AARCH64_CALL26:
183 case R_AARCH64_CONDBR19:
184 case R_AARCH64_JUMP26:
185 case R_AARCH64_TSTBR14:
186 return R_PLT_PC;
187 case R_AARCH64_PLT32:
188 const_cast<Symbol &>(s).thunkAccessed = true;
189 return R_PLT_PC;
190 case R_AARCH64_PREL16:
191 case R_AARCH64_PREL32:
192 case R_AARCH64_PREL64:
193 case R_AARCH64_ADR_PREL_LO21:
194 case R_AARCH64_LD_PREL_LO19:
195 case R_AARCH64_MOVW_PREL_G0:
196 case R_AARCH64_MOVW_PREL_G0_NC:
197 case R_AARCH64_MOVW_PREL_G1:
198 case R_AARCH64_MOVW_PREL_G1_NC:
199 case R_AARCH64_MOVW_PREL_G2:
200 case R_AARCH64_MOVW_PREL_G2_NC:
201 case R_AARCH64_MOVW_PREL_G3:
202 return R_PC;
203 case R_AARCH64_ADR_PREL_PG_HI21:
204 case R_AARCH64_ADR_PREL_PG_HI21_NC:
205 return RE_AARCH64_PAGE_PC;
206 case R_AARCH64_LD64_GOT_LO12_NC:
207 case R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC:
208 return R_GOT;
209 case R_AARCH64_AUTH_LD64_GOT_LO12_NC:
210 case R_AARCH64_AUTH_GOT_ADD_LO12_NC:
211 return RE_AARCH64_AUTH_GOT;
212 case R_AARCH64_AUTH_GOT_LD_PREL19:
213 case R_AARCH64_AUTH_GOT_ADR_PREL_LO21:
214 return RE_AARCH64_AUTH_GOT_PC;
215 case R_AARCH64_LD64_GOTPAGE_LO15:
216 return RE_AARCH64_GOT_PAGE;
217 case R_AARCH64_ADR_GOT_PAGE:
218 case R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21:
219 return RE_AARCH64_GOT_PAGE_PC;
220 case R_AARCH64_AUTH_ADR_GOT_PAGE:
221 return RE_AARCH64_AUTH_GOT_PAGE_PC;
222 case R_AARCH64_GOTPCREL32:
223 case R_AARCH64_GOT_LD_PREL19:
224 return R_GOT_PC;
225 case R_AARCH64_NONE:
226 return R_NONE;
227 default:
228 Err(ctx) << getErrorLoc(ctx, loc) << "unknown relocation (" << type.v
229 << ") against symbol " << &s;
230 return R_NONE;
231 }
232}
233
234RelExpr AArch64::adjustTlsExpr(RelType type, RelExpr expr) const {
235 if (expr == R_RELAX_TLS_GD_TO_IE) {
236 if (type == R_AARCH64_TLSDESC_ADR_PAGE21)
237 return RE_AARCH64_RELAX_TLS_GD_TO_IE_PAGE_PC;
238 return R_RELAX_TLS_GD_TO_IE_ABS;
239 }
240 return expr;
241}
242
243bool AArch64::usesOnlyLowPageBits(RelType type) const {
244 switch (type) {
245 default:
246 return false;
247 case R_AARCH64_ADD_ABS_LO12_NC:
248 case R_AARCH64_LD64_GOT_LO12_NC:
249 case R_AARCH64_LDST128_ABS_LO12_NC:
250 case R_AARCH64_LDST16_ABS_LO12_NC:
251 case R_AARCH64_LDST32_ABS_LO12_NC:
252 case R_AARCH64_LDST64_ABS_LO12_NC:
253 case R_AARCH64_LDST8_ABS_LO12_NC:
254 case R_AARCH64_TLSDESC_ADD_LO12:
255 case R_AARCH64_TLSDESC_LD64_LO12:
256 case R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC:
257 return true;
258 }
259}
260
261RelType AArch64::getDynRel(RelType type) const {
262 if (type == R_AARCH64_ABS64 || type == R_AARCH64_AUTH_ABS64)
263 return type;
264 return R_AARCH64_NONE;
265}
266
267int64_t AArch64::getImplicitAddend(const uint8_t *buf, RelType type) const {
268 switch (type) {
269 case R_AARCH64_TLSDESC:
270 return read64(ctx, p: buf + 8);
271 case R_AARCH64_NONE:
272 case R_AARCH64_GLOB_DAT:
273 case R_AARCH64_AUTH_GLOB_DAT:
274 case R_AARCH64_JUMP_SLOT:
275 return 0;
276 case R_AARCH64_ABS16:
277 case R_AARCH64_PREL16:
278 return SignExtend64<16>(x: read16(ctx, p: buf));
279 case R_AARCH64_ABS32:
280 case R_AARCH64_PREL32:
281 return SignExtend64<32>(x: read32(ctx, p: buf));
282 case R_AARCH64_ABS64:
283 case R_AARCH64_PREL64:
284 case R_AARCH64_RELATIVE:
285 case R_AARCH64_IRELATIVE:
286 case R_AARCH64_TLS_TPREL64:
287 return read64(ctx, p: buf);
288
289 // The following relocation types all point at instructions, and
290 // relocate an immediate field in the instruction.
291 //
292 // The general rule, from AAELF64 §5.7.2 "Addends and PC-bias",
293 // says: "If the relocation relocates an instruction the immediate
294 // field of the instruction is extracted, scaled as required by
295 // the instruction field encoding, and sign-extended to 64 bits".
296
297 // The R_AARCH64_MOVW family operates on wide MOV/MOVK/MOVZ
298 // instructions, which have a 16-bit immediate field with its low
299 // bit in bit 5 of the instruction encoding. When the immediate
300 // field is used as an implicit addend for REL-type relocations,
301 // it is treated as added to the low bits of the output value, not
302 // shifted depending on the relocation type.
303 //
304 // This allows REL relocations to express the requirement 'please
305 // add 12345 to this symbol value and give me the four 16-bit
306 // chunks of the result', by putting the same addend 12345 in all
307 // four instructions. Carries between the 16-bit chunks are
308 // handled correctly, because the whole 64-bit addition is done
309 // once per relocation.
310 case R_AARCH64_MOVW_UABS_G0:
311 case R_AARCH64_MOVW_UABS_G0_NC:
312 case R_AARCH64_MOVW_UABS_G1:
313 case R_AARCH64_MOVW_UABS_G1_NC:
314 case R_AARCH64_MOVW_UABS_G2:
315 case R_AARCH64_MOVW_UABS_G2_NC:
316 case R_AARCH64_MOVW_UABS_G3:
317 return SignExtend64<16>(x: getBits(val: read32le(P: buf), start: 5, end: 20));
318
319 // R_AARCH64_TSTBR14 points at a TBZ or TBNZ instruction, which
320 // has a 14-bit offset measured in instructions, i.e. shifted left
321 // by 2.
322 case R_AARCH64_TSTBR14:
323 return SignExtend64<16>(x: getBits(val: read32le(P: buf), start: 5, end: 18) << 2);
324
325 // R_AARCH64_CONDBR19 operates on the ordinary B.cond instruction,
326 // which has a 19-bit offset measured in instructions.
327 //
328 // R_AARCH64_LD_PREL_LO19 operates on the LDR (literal)
329 // instruction, which also has a 19-bit offset, measured in 4-byte
330 // chunks. So the calculation is the same as for
331 // R_AARCH64_CONDBR19.
332 case R_AARCH64_CONDBR19:
333 case R_AARCH64_LD_PREL_LO19:
334 return SignExtend64<21>(x: getBits(val: read32le(P: buf), start: 5, end: 23) << 2);
335
336 // R_AARCH64_ADD_ABS_LO12_NC operates on ADD (immediate). The
337 // immediate can optionally be shifted left by 12 bits, but this
338 // relocation is intended for the case where it is not.
339 case R_AARCH64_ADD_ABS_LO12_NC:
340 return SignExtend64<12>(x: getBits(val: read32le(P: buf), start: 10, end: 21));
341
342 // R_AARCH64_ADR_PREL_LO21 operates on an ADR instruction, whose
343 // 21-bit immediate is split between two bits high up in the word
344 // (in fact the two _lowest_ order bits of the value) and 19 bits
345 // lower down.
346 //
347 // R_AARCH64_ADR_PREL_PG_HI21[_NC] operate on an ADRP instruction,
348 // which encodes the immediate in the same way, but will shift it
349 // left by 12 bits when the instruction executes. For the same
350 // reason as the MOVW family, we don't apply that left shift here.
351 case R_AARCH64_ADR_PREL_LO21:
352 case R_AARCH64_ADR_PREL_PG_HI21:
353 case R_AARCH64_ADR_PREL_PG_HI21_NC:
354 return SignExtend64<21>(x: (getBits(val: read32le(P: buf), start: 5, end: 23) << 2) |
355 getBits(val: read32le(P: buf), start: 29, end: 30));
356
357 // R_AARCH64_{JUMP,CALL}26 operate on B and BL, which have a
358 // 26-bit offset measured in instructions.
359 case R_AARCH64_JUMP26:
360 case R_AARCH64_CALL26:
361 return SignExtend64<28>(x: getBits(val: read32le(P: buf), start: 0, end: 25) << 2);
362
363 default:
364 InternalErr(ctx, buf) << "cannot read addend for relocation " << type;
365 return 0;
366 }
367}
368
369void AArch64::writeGotPlt(uint8_t *buf, const Symbol &) const {
370 write64(ctx, p: buf, v: ctx.in.plt->getVA());
371}
372
373void AArch64::writeIgotPlt(uint8_t *buf, const Symbol &s) const {
374 if (ctx.arg.writeAddends)
375 write64(ctx, p: buf, v: s.getVA(ctx));
376}
377
378void AArch64::writePltHeader(uint8_t *buf) const {
379 const uint8_t pltData[] = {
380 0xf0, 0x7b, 0xbf, 0xa9, // stp x16, x30, [sp,#-16]!
381 0x10, 0x00, 0x00, 0x90, // adrp x16, Page(&(.got.plt[2]))
382 0x11, 0x02, 0x40, 0xf9, // ldr x17, [x16, Offset(&(.got.plt[2]))]
383 0x10, 0x02, 0x00, 0x91, // add x16, x16, Offset(&(.got.plt[2]))
384 0x20, 0x02, 0x1f, 0xd6, // br x17
385 0x1f, 0x20, 0x03, 0xd5, // nop
386 0x1f, 0x20, 0x03, 0xd5, // nop
387 0x1f, 0x20, 0x03, 0xd5 // nop
388 };
389 memcpy(dest: buf, src: pltData, n: sizeof(pltData));
390
391 uint64_t got = ctx.in.gotPlt->getVA();
392 uint64_t plt = ctx.in.plt->getVA();
393 relocateNoSym(loc: buf + 4, type: R_AARCH64_ADR_PREL_PG_HI21,
394 val: getAArch64Page(expr: got + 16) - getAArch64Page(expr: plt + 4));
395 relocateNoSym(loc: buf + 8, type: R_AARCH64_LDST64_ABS_LO12_NC, val: got + 16);
396 relocateNoSym(loc: buf + 12, type: R_AARCH64_ADD_ABS_LO12_NC, val: got + 16);
397}
398
399void AArch64::writePlt(uint8_t *buf, const Symbol &sym,
400 uint64_t pltEntryAddr) const {
401 const uint8_t inst[] = {
402 0x10, 0x00, 0x00, 0x90, // adrp x16, Page(&(.got.plt[n]))
403 0x11, 0x02, 0x40, 0xf9, // ldr x17, [x16, Offset(&(.got.plt[n]))]
404 0x10, 0x02, 0x00, 0x91, // add x16, x16, Offset(&(.got.plt[n]))
405 0x20, 0x02, 0x1f, 0xd6 // br x17
406 };
407 memcpy(dest: buf, src: inst, n: sizeof(inst));
408
409 uint64_t gotPltEntryAddr = sym.getGotPltVA(ctx);
410 relocateNoSym(loc: buf, type: R_AARCH64_ADR_PREL_PG_HI21,
411 val: getAArch64Page(expr: gotPltEntryAddr) - getAArch64Page(expr: pltEntryAddr));
412 relocateNoSym(loc: buf + 4, type: R_AARCH64_LDST64_ABS_LO12_NC, val: gotPltEntryAddr);
413 relocateNoSym(loc: buf + 8, type: R_AARCH64_ADD_ABS_LO12_NC, val: gotPltEntryAddr);
414}
415
416bool AArch64::needsThunk(RelExpr expr, RelType type, const InputFile *file,
417 uint64_t branchAddr, const Symbol &s,
418 int64_t a) const {
419 // If s is an undefined weak symbol and does not have a PLT entry then it will
420 // be resolved as a branch to the next instruction. If it is hidden, its
421 // binding has been converted to local, so we just check isUndefined() here. A
422 // undefined non-weak symbol will have been errored.
423 if (s.isUndefined() && !s.isInPlt(ctx))
424 return false;
425 // ELF for the ARM 64-bit architecture, section Call and Jump relocations
426 // only permits range extension thunks for R_AARCH64_CALL26 and
427 // R_AARCH64_JUMP26 relocation types.
428 if (type != R_AARCH64_CALL26 && type != R_AARCH64_JUMP26 &&
429 type != R_AARCH64_PLT32)
430 return false;
431 uint64_t dst = expr == R_PLT_PC ? s.getPltVA(ctx) : s.getVA(ctx, addend: a);
432 return !inBranchRange(type, src: branchAddr, dst);
433}
434
435uint32_t AArch64::getThunkSectionSpacing() const {
436 // See comment in Arch/ARM.cpp for a more detailed explanation of
437 // getThunkSectionSpacing(). For AArch64 the only branches we are permitted to
438 // Thunk have a range of +/- 128 MiB
439 return (128 * 1024 * 1024) - 0x30000;
440}
441
442bool AArch64::inBranchRange(RelType type, uint64_t src, uint64_t dst) const {
443 if (type != R_AARCH64_CALL26 && type != R_AARCH64_JUMP26 &&
444 type != R_AARCH64_PLT32)
445 return true;
446 // The AArch64 call and unconditional branch instructions have a range of
447 // +/- 128 MiB. The PLT32 relocation supports a range up to +/- 2 GiB.
448 uint64_t range =
449 type == R_AARCH64_PLT32 ? (UINT64_C(1) << 31) : (128 * 1024 * 1024);
450 if (dst > src) {
451 // Immediate of branch is signed.
452 range -= 4;
453 return dst - src <= range;
454 }
455 return src - dst <= range;
456}
457
458static void write32AArch64Addr(uint8_t *l, uint64_t imm) {
459 uint32_t immLo = (imm & 0x3) << 29;
460 uint32_t immHi = (imm & 0x1FFFFC) << 3;
461 uint64_t mask = (0x3 << 29) | (0x1FFFFC << 3);
462 write32le(P: l, V: (read32le(P: l) & ~mask) | immLo | immHi);
463}
464
465static void writeMaskedBits32le(uint8_t *p, int32_t v, uint32_t mask) {
466 write32le(P: p, V: (read32le(P: p) & ~mask) | v);
467}
468
469// Update the immediate field in a AARCH64 ldr, str, and add instruction.
470static void write32Imm12(uint8_t *l, uint64_t imm) {
471 writeMaskedBits32le(p: l, v: (imm & 0xFFF) << 10, mask: 0xFFF << 10);
472}
473
474// Update the immediate field in an AArch64 movk, movn or movz instruction
475// for a signed relocation, and update the opcode of a movn or movz instruction
476// to match the sign of the operand.
477static void writeSMovWImm(uint8_t *loc, uint32_t imm) {
478 uint32_t inst = read32le(P: loc);
479 // Opcode field is bits 30, 29, with 10 = movz, 00 = movn and 11 = movk.
480 if (!(inst & (1 << 29))) {
481 // movn or movz.
482 if (imm & 0x10000) {
483 // Change opcode to movn, which takes an inverted operand.
484 imm ^= 0xFFFF;
485 inst &= ~(1 << 30);
486 } else {
487 // Change opcode to movz.
488 inst |= 1 << 30;
489 }
490 }
491 write32le(P: loc, V: inst | ((imm & 0xFFFF) << 5));
492}
493
494void AArch64::relocate(uint8_t *loc, const Relocation &rel,
495 uint64_t val) const {
496 switch (rel.type) {
497 case R_AARCH64_ABS16:
498 case R_AARCH64_PREL16:
499 checkIntUInt(ctx, loc, v: val, n: 16, rel);
500 write16(ctx, p: loc, v: val);
501 break;
502 case R_AARCH64_ABS32:
503 case R_AARCH64_PREL32:
504 checkIntUInt(ctx, loc, v: val, n: 32, rel);
505 write32(ctx, p: loc, v: val);
506 break;
507 case R_AARCH64_PLT32:
508 case R_AARCH64_GOTPCREL32:
509 checkInt(ctx, loc, v: val, n: 32, rel);
510 write32(ctx, p: loc, v: val);
511 break;
512 case R_AARCH64_ABS64:
513 // AArch64 relocations to tagged symbols have extended semantics, as
514 // described here:
515 // https://github.com/ARM-software/abi-aa/blob/main/memtagabielf64/memtagabielf64.rst#841extended-semantics-of-r_aarch64_relative.
516 // tl;dr: encode the symbol's special addend in the place, which is an
517 // offset to the point where the logical tag is derived from. Quick hack, if
518 // the addend is within the symbol's bounds, no need to encode the tag
519 // derivation offset.
520 if (rel.sym && rel.sym->isTagged() &&
521 (rel.addend < 0 ||
522 rel.addend >= static_cast<int64_t>(rel.sym->getSize())))
523 write64(ctx, p: loc, v: -rel.addend);
524 else
525 write64(ctx, p: loc, v: val);
526 break;
527 case R_AARCH64_PREL64:
528 write64(ctx, p: loc, v: val);
529 break;
530 case R_AARCH64_AUTH_ABS64:
531 // If val is wider than 32 bits, the relocation must have been moved from
532 // .relr.auth.dyn to .rela.dyn, and the addend write is not needed.
533 //
534 // If val fits in 32 bits, we have two potential scenarios:
535 // * True RELR: Write the 32-bit `val`.
536 // * RELA: Even if the value now fits in 32 bits, it might have been
537 // converted from RELR during an iteration in
538 // finalizeAddressDependentContent(). Writing the value is harmless
539 // because dynamic linking ignores it.
540 if (isInt<32>(x: val))
541 write32(ctx, p: loc, v: val);
542 break;
543 case R_AARCH64_ADD_ABS_LO12_NC:
544 case R_AARCH64_AUTH_GOT_ADD_LO12_NC:
545 write32Imm12(l: loc, imm: val);
546 break;
547 case R_AARCH64_ADR_GOT_PAGE:
548 case R_AARCH64_AUTH_ADR_GOT_PAGE:
549 case R_AARCH64_ADR_PREL_PG_HI21:
550 case R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21:
551 case R_AARCH64_TLSDESC_ADR_PAGE21:
552 case R_AARCH64_AUTH_TLSDESC_ADR_PAGE21:
553 checkInt(ctx, loc, v: val, n: 33, rel);
554 [[fallthrough]];
555 case R_AARCH64_ADR_PREL_PG_HI21_NC:
556 write32AArch64Addr(l: loc, imm: val >> 12);
557 break;
558 case R_AARCH64_ADR_PREL_LO21:
559 case R_AARCH64_AUTH_GOT_ADR_PREL_LO21:
560 checkInt(ctx, loc, v: val, n: 21, rel);
561 write32AArch64Addr(l: loc, imm: val);
562 break;
563 case R_AARCH64_JUMP26:
564 // Normally we would just write the bits of the immediate field, however
565 // when patching instructions for the cpu errata fix -fix-cortex-a53-843419
566 // we want to replace a non-branch instruction with a branch immediate
567 // instruction. By writing all the bits of the instruction including the
568 // opcode and the immediate (0 001 | 01 imm26) we can do this
569 // transformation by placing a R_AARCH64_JUMP26 relocation at the offset of
570 // the instruction we want to patch.
571 write32le(P: loc, V: 0x14000000);
572 [[fallthrough]];
573 case R_AARCH64_CALL26:
574 checkInt(ctx, loc, v: val, n: 28, rel);
575 writeMaskedBits32le(p: loc, v: (val & 0x0FFFFFFC) >> 2, mask: 0x0FFFFFFC >> 2);
576 break;
577 case R_AARCH64_CONDBR19:
578 case R_AARCH64_LD_PREL_LO19:
579 case R_AARCH64_GOT_LD_PREL19:
580 case R_AARCH64_AUTH_GOT_LD_PREL19:
581 checkAlignment(ctx, loc, v: val, n: 4, rel);
582 checkInt(ctx, loc, v: val, n: 21, rel);
583 writeMaskedBits32le(p: loc, v: (val & 0x1FFFFC) << 3, mask: 0x1FFFFC << 3);
584 break;
585 case R_AARCH64_LDST8_ABS_LO12_NC:
586 case R_AARCH64_TLSLE_LDST8_TPREL_LO12_NC:
587 write32Imm12(l: loc, imm: getBits(val, start: 0, end: 11));
588 break;
589 case R_AARCH64_LDST16_ABS_LO12_NC:
590 case R_AARCH64_TLSLE_LDST16_TPREL_LO12_NC:
591 checkAlignment(ctx, loc, v: val, n: 2, rel);
592 write32Imm12(l: loc, imm: getBits(val, start: 1, end: 11));
593 break;
594 case R_AARCH64_LDST32_ABS_LO12_NC:
595 case R_AARCH64_TLSLE_LDST32_TPREL_LO12_NC:
596 checkAlignment(ctx, loc, v: val, n: 4, rel);
597 write32Imm12(l: loc, imm: getBits(val, start: 2, end: 11));
598 break;
599 case R_AARCH64_LDST64_ABS_LO12_NC:
600 case R_AARCH64_LD64_GOT_LO12_NC:
601 case R_AARCH64_AUTH_LD64_GOT_LO12_NC:
602 case R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC:
603 case R_AARCH64_TLSLE_LDST64_TPREL_LO12_NC:
604 case R_AARCH64_TLSDESC_LD64_LO12:
605 case R_AARCH64_AUTH_TLSDESC_LD64_LO12:
606 checkAlignment(ctx, loc, v: val, n: 8, rel);
607 write32Imm12(l: loc, imm: getBits(val, start: 3, end: 11));
608 break;
609 case R_AARCH64_LDST128_ABS_LO12_NC:
610 case R_AARCH64_TLSLE_LDST128_TPREL_LO12_NC:
611 checkAlignment(ctx, loc, v: val, n: 16, rel);
612 write32Imm12(l: loc, imm: getBits(val, start: 4, end: 11));
613 break;
614 case R_AARCH64_LD64_GOTPAGE_LO15:
615 checkAlignment(ctx, loc, v: val, n: 8, rel);
616 write32Imm12(l: loc, imm: getBits(val, start: 3, end: 14));
617 break;
618 case R_AARCH64_MOVW_UABS_G0:
619 checkUInt(ctx, loc, v: val, n: 16, rel);
620 [[fallthrough]];
621 case R_AARCH64_MOVW_UABS_G0_NC:
622 writeMaskedBits32le(p: loc, v: (val & 0xFFFF) << 5, mask: 0xFFFF << 5);
623 break;
624 case R_AARCH64_MOVW_UABS_G1:
625 checkUInt(ctx, loc, v: val, n: 32, rel);
626 [[fallthrough]];
627 case R_AARCH64_MOVW_UABS_G1_NC:
628 writeMaskedBits32le(p: loc, v: (val & 0xFFFF0000) >> 11, mask: 0xFFFF0000 >> 11);
629 break;
630 case R_AARCH64_MOVW_UABS_G2:
631 checkUInt(ctx, loc, v: val, n: 48, rel);
632 [[fallthrough]];
633 case R_AARCH64_MOVW_UABS_G2_NC:
634 writeMaskedBits32le(p: loc, v: (val & 0xFFFF00000000) >> 27,
635 mask: 0xFFFF00000000 >> 27);
636 break;
637 case R_AARCH64_MOVW_UABS_G3:
638 writeMaskedBits32le(p: loc, v: (val & 0xFFFF000000000000) >> 43,
639 mask: 0xFFFF000000000000 >> 43);
640 break;
641 case R_AARCH64_MOVW_PREL_G0:
642 case R_AARCH64_MOVW_SABS_G0:
643 case R_AARCH64_TLSLE_MOVW_TPREL_G0:
644 checkInt(ctx, loc, v: val, n: 17, rel);
645 [[fallthrough]];
646 case R_AARCH64_MOVW_PREL_G0_NC:
647 case R_AARCH64_TLSLE_MOVW_TPREL_G0_NC:
648 writeSMovWImm(loc, imm: val);
649 break;
650 case R_AARCH64_MOVW_PREL_G1:
651 case R_AARCH64_MOVW_SABS_G1:
652 case R_AARCH64_TLSLE_MOVW_TPREL_G1:
653 checkInt(ctx, loc, v: val, n: 33, rel);
654 [[fallthrough]];
655 case R_AARCH64_MOVW_PREL_G1_NC:
656 case R_AARCH64_TLSLE_MOVW_TPREL_G1_NC:
657 writeSMovWImm(loc, imm: val >> 16);
658 break;
659 case R_AARCH64_MOVW_PREL_G2:
660 case R_AARCH64_MOVW_SABS_G2:
661 case R_AARCH64_TLSLE_MOVW_TPREL_G2:
662 checkInt(ctx, loc, v: val, n: 49, rel);
663 [[fallthrough]];
664 case R_AARCH64_MOVW_PREL_G2_NC:
665 writeSMovWImm(loc, imm: val >> 32);
666 break;
667 case R_AARCH64_MOVW_PREL_G3:
668 writeSMovWImm(loc, imm: val >> 48);
669 break;
670 case R_AARCH64_TSTBR14:
671 checkInt(ctx, loc, v: val, n: 16, rel);
672 writeMaskedBits32le(p: loc, v: (val & 0xFFFC) << 3, mask: 0xFFFC << 3);
673 break;
674 case R_AARCH64_TLSLE_ADD_TPREL_HI12:
675 checkUInt(ctx, loc, v: val, n: 24, rel);
676 write32Imm12(l: loc, imm: val >> 12);
677 break;
678 case R_AARCH64_TLSLE_ADD_TPREL_LO12_NC:
679 case R_AARCH64_TLSDESC_ADD_LO12:
680 case R_AARCH64_AUTH_TLSDESC_ADD_LO12:
681 write32Imm12(l: loc, imm: val);
682 break;
683 case R_AARCH64_TLSDESC:
684 // For R_AARCH64_TLSDESC the addend is stored in the second 64-bit word.
685 write64(ctx, p: loc + 8, v: val);
686 break;
687 default:
688 llvm_unreachable("unknown relocation");
689 }
690}
691
692void AArch64::relaxTlsGdToLe(uint8_t *loc, const Relocation &rel,
693 uint64_t val) const {
694 // TLSDESC Global-Dynamic relocation are in the form:
695 // adrp x0, :tlsdesc:v [R_AARCH64_TLSDESC_ADR_PAGE21]
696 // ldr x1, [x0, #:tlsdesc_lo12:v [R_AARCH64_TLSDESC_LD64_LO12]
697 // add x0, x0, :tlsdesc_los:v [R_AARCH64_TLSDESC_ADD_LO12]
698 // .tlsdesccall [R_AARCH64_TLSDESC_CALL]
699 // blr x1
700 // And it can optimized to:
701 // movz x0, #0x0, lsl #16
702 // movk x0, #0x10
703 // nop
704 // nop
705 checkUInt(ctx, loc, v: val, n: 32, rel);
706
707 switch (rel.type) {
708 case R_AARCH64_TLSDESC_ADD_LO12:
709 case R_AARCH64_TLSDESC_CALL:
710 write32le(P: loc, V: 0xd503201f); // nop
711 return;
712 case R_AARCH64_TLSDESC_ADR_PAGE21:
713 write32le(P: loc, V: 0xd2a00000 | (((val >> 16) & 0xffff) << 5)); // movz
714 return;
715 case R_AARCH64_TLSDESC_LD64_LO12:
716 write32le(P: loc, V: 0xf2800000 | ((val & 0xffff) << 5)); // movk
717 return;
718 default:
719 llvm_unreachable("unsupported relocation for TLS GD to LE relaxation");
720 }
721}
722
723void AArch64::relaxTlsGdToIe(uint8_t *loc, const Relocation &rel,
724 uint64_t val) const {
725 // TLSDESC Global-Dynamic relocation are in the form:
726 // adrp x0, :tlsdesc:v [R_AARCH64_TLSDESC_ADR_PAGE21]
727 // ldr x1, [x0, #:tlsdesc_lo12:v [R_AARCH64_TLSDESC_LD64_LO12]
728 // add x0, x0, :tlsdesc_los:v [R_AARCH64_TLSDESC_ADD_LO12]
729 // .tlsdesccall [R_AARCH64_TLSDESC_CALL]
730 // blr x1
731 // And it can optimized to:
732 // adrp x0, :gottprel:v
733 // ldr x0, [x0, :gottprel_lo12:v]
734 // nop
735 // nop
736
737 switch (rel.type) {
738 case R_AARCH64_TLSDESC_ADD_LO12:
739 case R_AARCH64_TLSDESC_CALL:
740 write32le(P: loc, V: 0xd503201f); // nop
741 break;
742 case R_AARCH64_TLSDESC_ADR_PAGE21:
743 write32le(P: loc, V: 0x90000000); // adrp
744 relocateNoSym(loc, type: R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21, val);
745 break;
746 case R_AARCH64_TLSDESC_LD64_LO12:
747 write32le(P: loc, V: 0xf9400000); // ldr
748 relocateNoSym(loc, type: R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC, val);
749 break;
750 default:
751 llvm_unreachable("unsupported relocation for TLS GD to LE relaxation");
752 }
753}
754
755void AArch64::relaxTlsIeToLe(uint8_t *loc, const Relocation &rel,
756 uint64_t val) const {
757 checkUInt(ctx, loc, v: val, n: 32, rel);
758
759 if (rel.type == R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21) {
760 // Generate MOVZ.
761 uint32_t regNo = read32le(P: loc) & 0x1f;
762 write32le(P: loc, V: (0xd2a00000 | regNo) | (((val >> 16) & 0xffff) << 5));
763 return;
764 }
765 if (rel.type == R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC) {
766 // Generate MOVK.
767 uint32_t regNo = read32le(P: loc) & 0x1f;
768 write32le(P: loc, V: (0xf2800000 | regNo) | ((val & 0xffff) << 5));
769 return;
770 }
771 llvm_unreachable("invalid relocation for TLS IE to LE relaxation");
772}
773
774AArch64Relaxer::AArch64Relaxer(Ctx &ctx, ArrayRef<Relocation> relocs)
775 : ctx(ctx) {
776 if (!ctx.arg.relax)
777 return;
778 // Check if R_AARCH64_ADR_GOT_PAGE and R_AARCH64_LD64_GOT_LO12_NC
779 // always appear in pairs.
780 size_t i = 0;
781 const size_t size = relocs.size();
782 for (; i != size; ++i) {
783 if (relocs[i].type == R_AARCH64_ADR_GOT_PAGE) {
784 if (i + 1 < size && relocs[i + 1].type == R_AARCH64_LD64_GOT_LO12_NC) {
785 ++i;
786 continue;
787 }
788 break;
789 } else if (relocs[i].type == R_AARCH64_LD64_GOT_LO12_NC) {
790 break;
791 }
792 }
793 safeToRelaxAdrpLdr = i == size;
794}
795
796bool AArch64Relaxer::tryRelaxAdrpAdd(const Relocation &adrpRel,
797 const Relocation &addRel, uint64_t secAddr,
798 uint8_t *buf) const {
799 // When the address of sym is within the range of ADR then
800 // we may relax
801 // ADRP xn, sym
802 // ADD xn, xn, :lo12: sym
803 // to
804 // NOP
805 // ADR xn, sym
806 if (!ctx.arg.relax || adrpRel.type != R_AARCH64_ADR_PREL_PG_HI21 ||
807 addRel.type != R_AARCH64_ADD_ABS_LO12_NC)
808 return false;
809 // Check if the relocations apply to consecutive instructions.
810 if (adrpRel.offset + 4 != addRel.offset)
811 return false;
812 if (adrpRel.sym != addRel.sym)
813 return false;
814 if (adrpRel.addend != 0 || addRel.addend != 0)
815 return false;
816
817 uint32_t adrpInstr = read32le(P: buf + adrpRel.offset);
818 uint32_t addInstr = read32le(P: buf + addRel.offset);
819 // Check if the first instruction is ADRP and the second instruction is ADD.
820 if ((adrpInstr & 0x9f000000) != 0x90000000 ||
821 (addInstr & 0xffc00000) != 0x91000000)
822 return false;
823 uint32_t adrpDestReg = adrpInstr & 0x1f;
824 uint32_t addDestReg = addInstr & 0x1f;
825 uint32_t addSrcReg = (addInstr >> 5) & 0x1f;
826 if (adrpDestReg != addDestReg || adrpDestReg != addSrcReg)
827 return false;
828
829 Symbol &sym = *adrpRel.sym;
830 // Check if the address difference is within 1MiB range.
831 int64_t val = sym.getVA(ctx) - (secAddr + addRel.offset);
832 if (val < -1024 * 1024 || val >= 1024 * 1024)
833 return false;
834
835 Relocation adrRel = {.expr: R_ABS, .type: R_AARCH64_ADR_PREL_LO21, .offset: addRel.offset,
836 /*addend=*/0, .sym: &sym};
837 // nop
838 write32le(P: buf + adrpRel.offset, V: 0xd503201f);
839 // adr x_<dest_reg>
840 write32le(P: buf + adrRel.offset, V: 0x10000000 | adrpDestReg);
841 ctx.target->relocate(loc: buf + adrRel.offset, rel: adrRel, val);
842 return true;
843}
844
845bool AArch64Relaxer::tryRelaxAdrpLdr(const Relocation &adrpRel,
846 const Relocation &ldrRel, uint64_t secAddr,
847 uint8_t *buf) const {
848 if (!safeToRelaxAdrpLdr)
849 return false;
850
851 // When the definition of sym is not preemptible then we may
852 // be able to relax
853 // ADRP xn, :got: sym
854 // LDR xn, [ xn :got_lo12: sym]
855 // to
856 // ADRP xn, sym
857 // ADD xn, xn, :lo_12: sym
858
859 if (adrpRel.type != R_AARCH64_ADR_GOT_PAGE ||
860 ldrRel.type != R_AARCH64_LD64_GOT_LO12_NC)
861 return false;
862 // Check if the relocations apply to consecutive instructions.
863 if (adrpRel.offset + 4 != ldrRel.offset)
864 return false;
865 // Check if the relocations reference the same symbol and
866 // skip undefined, preemptible and STT_GNU_IFUNC symbols.
867 if (!adrpRel.sym || adrpRel.sym != ldrRel.sym || !adrpRel.sym->isDefined() ||
868 adrpRel.sym->isPreemptible || adrpRel.sym->isGnuIFunc())
869 return false;
870 // Check if the addends of the both relocations are zero.
871 if (adrpRel.addend != 0 || ldrRel.addend != 0)
872 return false;
873 uint32_t adrpInstr = read32le(P: buf + adrpRel.offset);
874 uint32_t ldrInstr = read32le(P: buf + ldrRel.offset);
875 // Check if the first instruction is ADRP and the second instruction is LDR.
876 if ((adrpInstr & 0x9f000000) != 0x90000000 ||
877 (ldrInstr & 0x3b000000) != 0x39000000)
878 return false;
879 // Check the value of the sf bit.
880 if (!(ldrInstr >> 31))
881 return false;
882 uint32_t adrpDestReg = adrpInstr & 0x1f;
883 uint32_t ldrDestReg = ldrInstr & 0x1f;
884 uint32_t ldrSrcReg = (ldrInstr >> 5) & 0x1f;
885 // Check if ADPR and LDR use the same register.
886 if (adrpDestReg != ldrDestReg || adrpDestReg != ldrSrcReg)
887 return false;
888
889 Symbol &sym = *adrpRel.sym;
890 // GOT references to absolute symbols can't be relaxed to use ADRP/ADD in
891 // position-independent code because these instructions produce a relative
892 // address.
893 if (ctx.arg.isPic && !cast<Defined>(Val&: sym).section)
894 return false;
895 // Check if the address difference is within 4GB range.
896 int64_t val =
897 getAArch64Page(expr: sym.getVA(ctx)) - getAArch64Page(expr: secAddr + adrpRel.offset);
898 if (val != llvm::SignExtend64(X: val, B: 33))
899 return false;
900
901 Relocation adrpSymRel = {.expr: RE_AARCH64_PAGE_PC, .type: R_AARCH64_ADR_PREL_PG_HI21,
902 .offset: adrpRel.offset, /*addend=*/0, .sym: &sym};
903 Relocation addRel = {.expr: R_ABS, .type: R_AARCH64_ADD_ABS_LO12_NC, .offset: ldrRel.offset,
904 /*addend=*/0, .sym: &sym};
905
906 // adrp x_<dest_reg>
907 write32le(P: buf + adrpSymRel.offset, V: 0x90000000 | adrpDestReg);
908 // add x_<dest reg>, x_<dest reg>
909 write32le(P: buf + addRel.offset, V: 0x91000000 | adrpDestReg | (adrpDestReg << 5));
910
911 ctx.target->relocate(
912 loc: buf + adrpSymRel.offset, rel: adrpSymRel,
913 val: SignExtend64(X: getAArch64Page(expr: sym.getVA(ctx)) -
914 getAArch64Page(expr: secAddr + adrpSymRel.offset),
915 B: 64));
916 ctx.target->relocate(loc: buf + addRel.offset, rel: addRel,
917 val: SignExtend64(X: sym.getVA(ctx), B: 64));
918 tryRelaxAdrpAdd(adrpRel: adrpSymRel, addRel, secAddr, buf);
919 return true;
920}
921
922// Tagged symbols have upper address bits that are added by the dynamic loader,
923// and thus need the full 64-bit GOT entry. Do not relax such symbols.
924static bool needsGotForMemtag(const Relocation &rel) {
925 return rel.sym->isTagged() && needsGot(expr: rel.expr);
926}
927
928void AArch64::relocateAlloc(InputSectionBase &sec, uint8_t *buf) const {
929 uint64_t secAddr = sec.getOutputSection()->addr;
930 if (auto *s = dyn_cast<InputSection>(Val: &sec))
931 secAddr += s->outSecOff;
932 else if (auto *ehIn = dyn_cast<EhInputSection>(Val: &sec))
933 secAddr += ehIn->getParent()->outSecOff;
934 AArch64Relaxer relaxer(ctx, sec.relocs());
935 for (size_t i = 0, size = sec.relocs().size(); i != size; ++i) {
936 const Relocation &rel = sec.relocs()[i];
937 uint8_t *loc = buf + rel.offset;
938 const uint64_t val = sec.getRelocTargetVA(ctx, r: rel, p: secAddr + rel.offset);
939
940 if (needsGotForMemtag(rel)) {
941 relocate(loc, rel, val);
942 continue;
943 }
944
945 switch (rel.expr) {
946 case RE_AARCH64_GOT_PAGE_PC:
947 if (i + 1 < size &&
948 relaxer.tryRelaxAdrpLdr(adrpRel: rel, ldrRel: sec.relocs()[i + 1], secAddr, buf)) {
949 ++i;
950 continue;
951 }
952 break;
953 case RE_AARCH64_PAGE_PC:
954 if (i + 1 < size &&
955 relaxer.tryRelaxAdrpAdd(adrpRel: rel, addRel: sec.relocs()[i + 1], secAddr, buf)) {
956 ++i;
957 continue;
958 }
959 break;
960 case RE_AARCH64_RELAX_TLS_GD_TO_IE_PAGE_PC:
961 case R_RELAX_TLS_GD_TO_IE_ABS:
962 relaxTlsGdToIe(loc, rel, val);
963 continue;
964 case R_RELAX_TLS_GD_TO_LE:
965 relaxTlsGdToLe(loc, rel, val);
966 continue;
967 case R_RELAX_TLS_IE_TO_LE:
968 relaxTlsIeToLe(loc, rel, val);
969 continue;
970 default:
971 break;
972 }
973 relocate(loc, rel, val);
974 }
975}
976
977// AArch64 may use security features in variant PLT sequences. These are:
978// Pointer Authentication (PAC), introduced in armv8.3-a and Branch Target
979// Indicator (BTI) introduced in armv8.5-a. The additional instructions used
980// in the variant Plt sequences are encoded in the Hint space so they can be
981// deployed on older architectures, which treat the instructions as a nop.
982// PAC and BTI can be combined leading to the following combinations:
983// writePltHeader
984// writePltHeaderBti (no PAC Header needed)
985// writePlt
986// writePltBti (BTI only)
987// writePltPac (PAC only)
988// writePltBtiPac (BTI and PAC)
989//
990// When PAC is enabled the dynamic loader encrypts the address that it places
991// in the .got.plt using the pacia1716 instruction which encrypts the value in
992// x17 using the modifier in x16. The static linker places autia1716 before the
993// indirect branch to x17 to authenticate the address in x17 with the modifier
994// in x16. This makes it more difficult for an attacker to modify the value in
995// the .got.plt.
996//
997// When BTI is enabled all indirect branches must land on a bti instruction.
998// The static linker must place a bti instruction at the start of any PLT entry
999// that may be the target of an indirect branch. As the PLT entries call the
1000// lazy resolver indirectly this must have a bti instruction at start. In
1001// general a bti instruction is not needed for a PLT entry as indirect calls
1002// are resolved to the function address and not the PLT entry for the function.
1003// There are a small number of cases where the PLT address can escape, such as
1004// taking the address of a function or ifunc via a non got-generating
1005// relocation, and a shared library refers to that symbol.
1006//
1007// We use the bti c variant of the instruction which permits indirect branches
1008// (br) via x16/x17 and indirect function calls (blr) via any register. The ABI
1009// guarantees that all indirect branches from code requiring BTI protection
1010// will go via x16/x17
1011
1012namespace {
1013class AArch64BtiPac final : public AArch64 {
1014public:
1015 AArch64BtiPac(Ctx &);
1016 void writePltHeader(uint8_t *buf) const override;
1017 void writePlt(uint8_t *buf, const Symbol &sym,
1018 uint64_t pltEntryAddr) const override;
1019
1020private:
1021 bool btiHeader; // bti instruction needed in PLT Header and Entry
1022 enum {
1023 PEK_NoAuth,
1024 PEK_AuthHint, // use autia1716 instr for authenticated branch in PLT entry
1025 PEK_Auth, // use braa instr for authenticated branch in PLT entry
1026 } pacEntryKind;
1027};
1028} // namespace
1029
1030AArch64BtiPac::AArch64BtiPac(Ctx &ctx) : AArch64(ctx) {
1031 btiHeader = (ctx.arg.andFeatures & GNU_PROPERTY_AARCH64_FEATURE_1_BTI);
1032 // A BTI (Branch Target Indicator) Plt Entry is only required if the
1033 // address of the PLT entry can be taken by the program, which permits an
1034 // indirect jump to the PLT entry. This can happen when the address
1035 // of the PLT entry for a function is canonicalised due to the address of
1036 // the function in an executable being taken by a shared library, or
1037 // non-preemptible ifunc referenced by non-GOT-generating, non-PLT-generating
1038 // relocations.
1039 // The PAC PLT entries require dynamic loader support and this isn't known
1040 // from properties in the objects, so we use the command line flag.
1041 // By default we only use hint-space instructions, but if we detect the
1042 // PAuthABI, which requires v8.3-A, we can use the non-hint space
1043 // instructions.
1044
1045 if (ctx.arg.zPacPlt) {
1046 if (llvm::any_of(Range&: ctx.aarch64PauthAbiCoreInfo,
1047 P: [](uint8_t c) { return c != 0; }))
1048 pacEntryKind = PEK_Auth;
1049 else
1050 pacEntryKind = PEK_AuthHint;
1051 } else {
1052 pacEntryKind = PEK_NoAuth;
1053 }
1054
1055 if (btiHeader || (pacEntryKind != PEK_NoAuth)) {
1056 pltEntrySize = 24;
1057 ipltEntrySize = 24;
1058 }
1059}
1060
1061void AArch64BtiPac::writePltHeader(uint8_t *buf) const {
1062 const uint8_t btiData[] = { 0x5f, 0x24, 0x03, 0xd5 }; // bti c
1063 const uint8_t pltData[] = {
1064 0xf0, 0x7b, 0xbf, 0xa9, // stp x16, x30, [sp,#-16]!
1065 0x10, 0x00, 0x00, 0x90, // adrp x16, Page(&(.got.plt[2]))
1066 0x11, 0x02, 0x40, 0xf9, // ldr x17, [x16, Offset(&(.got.plt[2]))]
1067 0x10, 0x02, 0x00, 0x91, // add x16, x16, Offset(&(.got.plt[2]))
1068 0x20, 0x02, 0x1f, 0xd6, // br x17
1069 0x1f, 0x20, 0x03, 0xd5, // nop
1070 0x1f, 0x20, 0x03, 0xd5 // nop
1071 };
1072 const uint8_t nopData[] = { 0x1f, 0x20, 0x03, 0xd5 }; // nop
1073
1074 uint64_t got = ctx.in.gotPlt->getVA();
1075 uint64_t plt = ctx.in.plt->getVA();
1076
1077 if (btiHeader) {
1078 // PltHeader is called indirectly by plt[N]. Prefix pltData with a BTI C
1079 // instruction.
1080 memcpy(dest: buf, src: btiData, n: sizeof(btiData));
1081 buf += sizeof(btiData);
1082 plt += sizeof(btiData);
1083 }
1084 memcpy(dest: buf, src: pltData, n: sizeof(pltData));
1085
1086 relocateNoSym(loc: buf + 4, type: R_AARCH64_ADR_PREL_PG_HI21,
1087 val: getAArch64Page(expr: got + 16) - getAArch64Page(expr: plt + 4));
1088 relocateNoSym(loc: buf + 8, type: R_AARCH64_LDST64_ABS_LO12_NC, val: got + 16);
1089 relocateNoSym(loc: buf + 12, type: R_AARCH64_ADD_ABS_LO12_NC, val: got + 16);
1090 if (!btiHeader)
1091 // We didn't add the BTI c instruction so round out size with NOP.
1092 memcpy(dest: buf + sizeof(pltData), src: nopData, n: sizeof(nopData));
1093}
1094
1095void AArch64BtiPac::writePlt(uint8_t *buf, const Symbol &sym,
1096 uint64_t pltEntryAddr) const {
1097 // The PLT entry is of the form:
1098 // [btiData] addrInst (pacBr | stdBr) [nopData]
1099 const uint8_t btiData[] = { 0x5f, 0x24, 0x03, 0xd5 }; // bti c
1100 const uint8_t addrInst[] = {
1101 0x10, 0x00, 0x00, 0x90, // adrp x16, Page(&(.got.plt[n]))
1102 0x11, 0x02, 0x40, 0xf9, // ldr x17, [x16, Offset(&(.got.plt[n]))]
1103 0x10, 0x02, 0x00, 0x91 // add x16, x16, Offset(&(.got.plt[n]))
1104 };
1105 const uint8_t pacHintBr[] = {
1106 0x9f, 0x21, 0x03, 0xd5, // autia1716
1107 0x20, 0x02, 0x1f, 0xd6 // br x17
1108 };
1109 const uint8_t pacBr[] = {
1110 0x30, 0x0a, 0x1f, 0xd7, // braa x17, x16
1111 0x1f, 0x20, 0x03, 0xd5 // nop
1112 };
1113 const uint8_t stdBr[] = {
1114 0x20, 0x02, 0x1f, 0xd6, // br x17
1115 0x1f, 0x20, 0x03, 0xd5 // nop
1116 };
1117 const uint8_t nopData[] = { 0x1f, 0x20, 0x03, 0xd5 }; // nop
1118
1119 // NEEDS_COPY indicates a non-ifunc canonical PLT entry whose address may
1120 // escape to shared objects. isInIplt indicates a non-preemptible ifunc. Its
1121 // address may escape if referenced by a direct relocation. If relative
1122 // vtables are used then if the vtable is in a shared object the offsets will
1123 // be to the PLT entry. The condition is conservative.
1124 bool hasBti = btiHeader &&
1125 (sym.hasFlag(bit: NEEDS_COPY) || sym.isInIplt || sym.thunkAccessed);
1126 if (hasBti) {
1127 memcpy(dest: buf, src: btiData, n: sizeof(btiData));
1128 buf += sizeof(btiData);
1129 pltEntryAddr += sizeof(btiData);
1130 }
1131
1132 uint64_t gotPltEntryAddr = sym.getGotPltVA(ctx);
1133 memcpy(dest: buf, src: addrInst, n: sizeof(addrInst));
1134 relocateNoSym(loc: buf, type: R_AARCH64_ADR_PREL_PG_HI21,
1135 val: getAArch64Page(expr: gotPltEntryAddr) - getAArch64Page(expr: pltEntryAddr));
1136 relocateNoSym(loc: buf + 4, type: R_AARCH64_LDST64_ABS_LO12_NC, val: gotPltEntryAddr);
1137 relocateNoSym(loc: buf + 8, type: R_AARCH64_ADD_ABS_LO12_NC, val: gotPltEntryAddr);
1138
1139 if (pacEntryKind != PEK_NoAuth)
1140 memcpy(dest: buf + sizeof(addrInst),
1141 src: pacEntryKind == PEK_AuthHint ? pacHintBr : pacBr,
1142 n: sizeof(pacEntryKind == PEK_AuthHint ? pacHintBr : pacBr));
1143 else
1144 memcpy(dest: buf + sizeof(addrInst), src: stdBr, n: sizeof(stdBr));
1145 if (!hasBti)
1146 // We didn't add the BTI c instruction so round out size with NOP.
1147 memcpy(dest: buf + sizeof(addrInst) + sizeof(stdBr), src: nopData, n: sizeof(nopData));
1148}
1149
1150template <class ELFT>
1151static void
1152addTaggedSymbolReferences(Ctx &ctx, InputSectionBase &sec,
1153 DenseMap<Symbol *, unsigned> &referenceCount) {
1154 assert(sec.type == SHT_AARCH64_MEMTAG_GLOBALS_STATIC);
1155
1156 const RelsOrRelas<ELFT> rels = sec.relsOrRelas<ELFT>();
1157 if (rels.areRelocsRel())
1158 ErrAlways(ctx)
1159 << "non-RELA relocations are not allowed with memtag globals";
1160
1161 for (const typename ELFT::Rela &rel : rels.relas) {
1162 Symbol &sym = sec.file->getRelocTargetSym(rel);
1163 // Linker-synthesized symbols such as __executable_start may be referenced
1164 // as tagged in input objfiles, and we don't want them to be tagged. A
1165 // cheap way to exclude them is the type check, but their type is
1166 // STT_NOTYPE. In addition, this save us from checking untaggable symbols,
1167 // like functions or TLS symbols.
1168 if (sym.type != STT_OBJECT)
1169 continue;
1170 // STB_LOCAL symbols can't be referenced from outside the object file, and
1171 // thus don't need to be checked for references from other object files.
1172 if (sym.binding == STB_LOCAL) {
1173 sym.setIsTagged(true);
1174 continue;
1175 }
1176 ++referenceCount[&sym];
1177 }
1178 sec.markDead();
1179}
1180
1181// A tagged symbol must be denoted as being tagged by all references and the
1182// chosen definition. For simplicity, here, it must also be denoted as tagged
1183// for all definitions. Otherwise:
1184//
1185// 1. A tagged definition can be used by an untagged declaration, in which case
1186// the untagged access may be PC-relative, causing a tag mismatch at
1187// runtime.
1188// 2. An untagged definition can be used by a tagged declaration, where the
1189// compiler has taken advantage of the increased alignment of the tagged
1190// declaration, but the alignment at runtime is wrong, causing a fault.
1191//
1192// Ideally, this isn't a problem, as any TU that imports or exports tagged
1193// symbols should also be built with tagging. But, to handle these cases, we
1194// demote the symbol to be untagged.
1195void elf::createTaggedSymbols(Ctx &ctx) {
1196 assert(hasMemtag(ctx));
1197
1198 // First, collect all symbols that are marked as tagged, and count how many
1199 // times they're marked as tagged.
1200 DenseMap<Symbol *, unsigned> taggedSymbolReferenceCount;
1201 for (InputFile *file : ctx.objectFiles) {
1202 if (file->kind() != InputFile::ObjKind)
1203 continue;
1204 for (InputSectionBase *section : file->getSections()) {
1205 if (!section || section->type != SHT_AARCH64_MEMTAG_GLOBALS_STATIC ||
1206 section == &InputSection::discarded)
1207 continue;
1208 invokeELFT(addTaggedSymbolReferences, ctx, *section,
1209 taggedSymbolReferenceCount);
1210 }
1211 }
1212
1213 // Now, go through all the symbols. If the number of declarations +
1214 // definitions to a symbol exceeds the amount of times they're marked as
1215 // tagged, it means we have an objfile that uses the untagged variant of the
1216 // symbol.
1217 for (InputFile *file : ctx.objectFiles) {
1218 if (file->kind() != InputFile::BinaryKind &&
1219 file->kind() != InputFile::ObjKind)
1220 continue;
1221
1222 for (Symbol *symbol : file->getSymbols()) {
1223 // See `addTaggedSymbolReferences` for more details.
1224 if (symbol->type != STT_OBJECT ||
1225 symbol->binding == STB_LOCAL)
1226 continue;
1227 auto it = taggedSymbolReferenceCount.find(Val: symbol);
1228 if (it == taggedSymbolReferenceCount.end()) continue;
1229 unsigned &remainingAllowedTaggedRefs = it->second;
1230 if (remainingAllowedTaggedRefs == 0) {
1231 taggedSymbolReferenceCount.erase(I: it);
1232 continue;
1233 }
1234 --remainingAllowedTaggedRefs;
1235 }
1236 }
1237
1238 // `addTaggedSymbolReferences` has already checked that we have RELA
1239 // relocations, the only other way to get written addends is with
1240 // --apply-dynamic-relocs.
1241 if (!taggedSymbolReferenceCount.empty() && ctx.arg.writeAddends)
1242 ErrAlways(ctx) << "--apply-dynamic-relocs cannot be used with MTE globals";
1243
1244 // Now, `taggedSymbolReferenceCount` should only contain symbols that are
1245 // defined as tagged exactly the same amount as it's referenced, meaning all
1246 // uses are tagged.
1247 for (auto &[symbol, remainingTaggedRefs] : taggedSymbolReferenceCount) {
1248 assert(remainingTaggedRefs == 0 &&
1249 "Symbol is defined as tagged more times than it's used");
1250 symbol->setIsTagged(true);
1251 }
1252}
1253
1254void elf::setAArch64TargetInfo(Ctx &ctx) {
1255 if ((ctx.arg.andFeatures & GNU_PROPERTY_AARCH64_FEATURE_1_BTI) ||
1256 ctx.arg.zPacPlt)
1257 ctx.target.reset(p: new AArch64BtiPac(ctx));
1258 else
1259 ctx.target.reset(p: new AArch64(ctx));
1260}
1261

source code of lld/ELF/Arch/AArch64.cpp