1 | //===- AArch64.cpp --------------------------------------------------------===// |
---|---|
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | |
9 | #include "InputFiles.h" |
10 | #include "OutputSections.h" |
11 | #include "Symbols.h" |
12 | #include "SyntheticSections.h" |
13 | #include "Target.h" |
14 | #include "llvm/BinaryFormat/ELF.h" |
15 | #include "llvm/Support/Endian.h" |
16 | |
17 | using namespace llvm; |
18 | using namespace llvm::support::endian; |
19 | using namespace llvm::ELF; |
20 | using namespace lld; |
21 | using namespace lld::elf; |
22 | |
23 | // Page(Expr) is the page address of the expression Expr, defined |
24 | // as (Expr & ~0xFFF). (This applies even if the machine page size |
25 | // supported by the platform has a different value.) |
26 | uint64_t elf::getAArch64Page(uint64_t expr) { |
27 | return expr & ~static_cast<uint64_t>(0xFFF); |
28 | } |
29 | |
30 | // A BTI landing pad is a valid target for an indirect branch when the Branch |
31 | // Target Identification has been enabled. As linker generated branches are |
32 | // via x16 the BTI landing pads are defined as: BTI C, BTI J, BTI JC, PACIASP, |
33 | // PACIBSP. |
34 | bool elf::isAArch64BTILandingPad(Ctx &ctx, Symbol &s, int64_t a) { |
35 | // PLT entries accessed indirectly have a BTI c. |
36 | if (s.isInPlt(ctx)) |
37 | return true; |
38 | Defined *d = dyn_cast<Defined>(Val: &s); |
39 | if (!isa_and_nonnull<InputSection>(Val: d->section)) |
40 | // All places that we cannot disassemble are responsible for making |
41 | // the target a BTI landing pad. |
42 | return true; |
43 | InputSection *isec = cast<InputSection>(Val: d->section); |
44 | uint64_t off = d->value + a; |
45 | // Likely user error, but protect ourselves against out of bounds |
46 | // access. |
47 | if (off >= isec->getSize()) |
48 | return true; |
49 | const uint8_t *buf = isec->content().begin(); |
50 | const uint32_t instr = read32le(P: buf + off); |
51 | // All BTI instructions are HINT instructions which all have same encoding |
52 | // apart from bits [11:5] |
53 | if ((instr & 0xd503201f) == 0xd503201f && |
54 | is_contained(Set: {/*PACIASP*/ 0xd503233f, /*PACIBSP*/ 0xd503237f, |
55 | /*BTI C*/ 0xd503245f, /*BTI J*/ 0xd503249f, |
56 | /*BTI JC*/ 0xd50324df}, |
57 | Element: instr)) |
58 | return true; |
59 | return false; |
60 | } |
61 | |
62 | namespace { |
63 | class AArch64 : public TargetInfo { |
64 | public: |
65 | AArch64(Ctx &); |
66 | RelExpr getRelExpr(RelType type, const Symbol &s, |
67 | const uint8_t *loc) const override; |
68 | RelType getDynRel(RelType type) const override; |
69 | int64_t getImplicitAddend(const uint8_t *buf, RelType type) const override; |
70 | void writeGotPlt(uint8_t *buf, const Symbol &s) const override; |
71 | void writeIgotPlt(uint8_t *buf, const Symbol &s) const override; |
72 | void writePltHeader(uint8_t *buf) const override; |
73 | void writePlt(uint8_t *buf, const Symbol &sym, |
74 | uint64_t pltEntryAddr) const override; |
75 | bool needsThunk(RelExpr expr, RelType type, const InputFile *file, |
76 | uint64_t branchAddr, const Symbol &s, |
77 | int64_t a) const override; |
78 | uint32_t getThunkSectionSpacing() const override; |
79 | bool inBranchRange(RelType type, uint64_t src, uint64_t dst) const override; |
80 | bool usesOnlyLowPageBits(RelType type) const override; |
81 | void relocate(uint8_t *loc, const Relocation &rel, |
82 | uint64_t val) const override; |
83 | RelExpr adjustTlsExpr(RelType type, RelExpr expr) const override; |
84 | void relocateAlloc(InputSectionBase &sec, uint8_t *buf) const override; |
85 | |
86 | private: |
87 | void relaxTlsGdToLe(uint8_t *loc, const Relocation &rel, uint64_t val) const; |
88 | void relaxTlsGdToIe(uint8_t *loc, const Relocation &rel, uint64_t val) const; |
89 | void relaxTlsIeToLe(uint8_t *loc, const Relocation &rel, uint64_t val) const; |
90 | }; |
91 | |
92 | struct AArch64Relaxer { |
93 | Ctx &ctx; |
94 | bool safeToRelaxAdrpLdr = false; |
95 | |
96 | AArch64Relaxer(Ctx &ctx, ArrayRef<Relocation> relocs); |
97 | bool tryRelaxAdrpAdd(const Relocation &adrpRel, const Relocation &addRel, |
98 | uint64_t secAddr, uint8_t *buf) const; |
99 | bool tryRelaxAdrpLdr(const Relocation &adrpRel, const Relocation &ldrRel, |
100 | uint64_t secAddr, uint8_t *buf) const; |
101 | }; |
102 | } // namespace |
103 | |
104 | // Return the bits [Start, End] from Val shifted Start bits. |
105 | // For instance, getBits(0xF0, 4, 8) returns 0xF. |
106 | static uint64_t getBits(uint64_t val, int start, int end) { |
107 | uint64_t mask = ((uint64_t)1 << (end + 1 - start)) - 1; |
108 | return (val >> start) & mask; |
109 | } |
110 | |
111 | AArch64::AArch64(Ctx &ctx) : TargetInfo(ctx) { |
112 | copyRel = R_AARCH64_COPY; |
113 | relativeRel = R_AARCH64_RELATIVE; |
114 | iRelativeRel = R_AARCH64_IRELATIVE; |
115 | gotRel = R_AARCH64_GLOB_DAT; |
116 | pltRel = R_AARCH64_JUMP_SLOT; |
117 | symbolicRel = R_AARCH64_ABS64; |
118 | tlsDescRel = R_AARCH64_TLSDESC; |
119 | tlsGotRel = R_AARCH64_TLS_TPREL64; |
120 | pltHeaderSize = 32; |
121 | pltEntrySize = 16; |
122 | ipltEntrySize = 16; |
123 | defaultMaxPageSize = 65536; |
124 | |
125 | // Align to the 2 MiB page size (known as a superpage or huge page). |
126 | // FreeBSD automatically promotes 2 MiB-aligned allocations. |
127 | defaultImageBase = 0x200000; |
128 | |
129 | needsThunks = true; |
130 | } |
131 | |
132 | RelExpr AArch64::getRelExpr(RelType type, const Symbol &s, |
133 | const uint8_t *loc) const { |
134 | switch (type) { |
135 | case R_AARCH64_ABS16: |
136 | case R_AARCH64_ABS32: |
137 | case R_AARCH64_ABS64: |
138 | case R_AARCH64_ADD_ABS_LO12_NC: |
139 | case R_AARCH64_LDST128_ABS_LO12_NC: |
140 | case R_AARCH64_LDST16_ABS_LO12_NC: |
141 | case R_AARCH64_LDST32_ABS_LO12_NC: |
142 | case R_AARCH64_LDST64_ABS_LO12_NC: |
143 | case R_AARCH64_LDST8_ABS_LO12_NC: |
144 | case R_AARCH64_MOVW_SABS_G0: |
145 | case R_AARCH64_MOVW_SABS_G1: |
146 | case R_AARCH64_MOVW_SABS_G2: |
147 | case R_AARCH64_MOVW_UABS_G0: |
148 | case R_AARCH64_MOVW_UABS_G0_NC: |
149 | case R_AARCH64_MOVW_UABS_G1: |
150 | case R_AARCH64_MOVW_UABS_G1_NC: |
151 | case R_AARCH64_MOVW_UABS_G2: |
152 | case R_AARCH64_MOVW_UABS_G2_NC: |
153 | case R_AARCH64_MOVW_UABS_G3: |
154 | return R_ABS; |
155 | case R_AARCH64_AUTH_ABS64: |
156 | return RE_AARCH64_AUTH; |
157 | case R_AARCH64_TLSDESC_ADR_PAGE21: |
158 | return RE_AARCH64_TLSDESC_PAGE; |
159 | case R_AARCH64_AUTH_TLSDESC_ADR_PAGE21: |
160 | return RE_AARCH64_AUTH_TLSDESC_PAGE; |
161 | case R_AARCH64_TLSDESC_LD64_LO12: |
162 | case R_AARCH64_TLSDESC_ADD_LO12: |
163 | return R_TLSDESC; |
164 | case R_AARCH64_AUTH_TLSDESC_LD64_LO12: |
165 | case R_AARCH64_AUTH_TLSDESC_ADD_LO12: |
166 | return RE_AARCH64_AUTH_TLSDESC; |
167 | case R_AARCH64_TLSDESC_CALL: |
168 | return R_TLSDESC_CALL; |
169 | case R_AARCH64_TLSLE_ADD_TPREL_HI12: |
170 | case R_AARCH64_TLSLE_ADD_TPREL_LO12_NC: |
171 | case R_AARCH64_TLSLE_LDST8_TPREL_LO12_NC: |
172 | case R_AARCH64_TLSLE_LDST16_TPREL_LO12_NC: |
173 | case R_AARCH64_TLSLE_LDST32_TPREL_LO12_NC: |
174 | case R_AARCH64_TLSLE_LDST64_TPREL_LO12_NC: |
175 | case R_AARCH64_TLSLE_LDST128_TPREL_LO12_NC: |
176 | case R_AARCH64_TLSLE_MOVW_TPREL_G0: |
177 | case R_AARCH64_TLSLE_MOVW_TPREL_G0_NC: |
178 | case R_AARCH64_TLSLE_MOVW_TPREL_G1: |
179 | case R_AARCH64_TLSLE_MOVW_TPREL_G1_NC: |
180 | case R_AARCH64_TLSLE_MOVW_TPREL_G2: |
181 | return R_TPREL; |
182 | case R_AARCH64_CALL26: |
183 | case R_AARCH64_CONDBR19: |
184 | case R_AARCH64_JUMP26: |
185 | case R_AARCH64_TSTBR14: |
186 | return R_PLT_PC; |
187 | case R_AARCH64_PLT32: |
188 | const_cast<Symbol &>(s).thunkAccessed = true; |
189 | return R_PLT_PC; |
190 | case R_AARCH64_PREL16: |
191 | case R_AARCH64_PREL32: |
192 | case R_AARCH64_PREL64: |
193 | case R_AARCH64_ADR_PREL_LO21: |
194 | case R_AARCH64_LD_PREL_LO19: |
195 | case R_AARCH64_MOVW_PREL_G0: |
196 | case R_AARCH64_MOVW_PREL_G0_NC: |
197 | case R_AARCH64_MOVW_PREL_G1: |
198 | case R_AARCH64_MOVW_PREL_G1_NC: |
199 | case R_AARCH64_MOVW_PREL_G2: |
200 | case R_AARCH64_MOVW_PREL_G2_NC: |
201 | case R_AARCH64_MOVW_PREL_G3: |
202 | return R_PC; |
203 | case R_AARCH64_ADR_PREL_PG_HI21: |
204 | case R_AARCH64_ADR_PREL_PG_HI21_NC: |
205 | return RE_AARCH64_PAGE_PC; |
206 | case R_AARCH64_LD64_GOT_LO12_NC: |
207 | case R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC: |
208 | return R_GOT; |
209 | case R_AARCH64_AUTH_LD64_GOT_LO12_NC: |
210 | case R_AARCH64_AUTH_GOT_ADD_LO12_NC: |
211 | return RE_AARCH64_AUTH_GOT; |
212 | case R_AARCH64_AUTH_GOT_LD_PREL19: |
213 | case R_AARCH64_AUTH_GOT_ADR_PREL_LO21: |
214 | return RE_AARCH64_AUTH_GOT_PC; |
215 | case R_AARCH64_LD64_GOTPAGE_LO15: |
216 | return RE_AARCH64_GOT_PAGE; |
217 | case R_AARCH64_ADR_GOT_PAGE: |
218 | case R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21: |
219 | return RE_AARCH64_GOT_PAGE_PC; |
220 | case R_AARCH64_AUTH_ADR_GOT_PAGE: |
221 | return RE_AARCH64_AUTH_GOT_PAGE_PC; |
222 | case R_AARCH64_GOTPCREL32: |
223 | case R_AARCH64_GOT_LD_PREL19: |
224 | return R_GOT_PC; |
225 | case R_AARCH64_NONE: |
226 | return R_NONE; |
227 | default: |
228 | Err(ctx) << getErrorLoc(ctx, loc) << "unknown relocation ("<< type.v |
229 | << ") against symbol "<< &s; |
230 | return R_NONE; |
231 | } |
232 | } |
233 | |
234 | RelExpr AArch64::adjustTlsExpr(RelType type, RelExpr expr) const { |
235 | if (expr == R_RELAX_TLS_GD_TO_IE) { |
236 | if (type == R_AARCH64_TLSDESC_ADR_PAGE21) |
237 | return RE_AARCH64_RELAX_TLS_GD_TO_IE_PAGE_PC; |
238 | return R_RELAX_TLS_GD_TO_IE_ABS; |
239 | } |
240 | return expr; |
241 | } |
242 | |
243 | bool AArch64::usesOnlyLowPageBits(RelType type) const { |
244 | switch (type) { |
245 | default: |
246 | return false; |
247 | case R_AARCH64_ADD_ABS_LO12_NC: |
248 | case R_AARCH64_LD64_GOT_LO12_NC: |
249 | case R_AARCH64_LDST128_ABS_LO12_NC: |
250 | case R_AARCH64_LDST16_ABS_LO12_NC: |
251 | case R_AARCH64_LDST32_ABS_LO12_NC: |
252 | case R_AARCH64_LDST64_ABS_LO12_NC: |
253 | case R_AARCH64_LDST8_ABS_LO12_NC: |
254 | case R_AARCH64_TLSDESC_ADD_LO12: |
255 | case R_AARCH64_TLSDESC_LD64_LO12: |
256 | case R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC: |
257 | return true; |
258 | } |
259 | } |
260 | |
261 | RelType AArch64::getDynRel(RelType type) const { |
262 | if (type == R_AARCH64_ABS64 || type == R_AARCH64_AUTH_ABS64) |
263 | return type; |
264 | return R_AARCH64_NONE; |
265 | } |
266 | |
267 | int64_t AArch64::getImplicitAddend(const uint8_t *buf, RelType type) const { |
268 | switch (type) { |
269 | case R_AARCH64_TLSDESC: |
270 | return read64(ctx, p: buf + 8); |
271 | case R_AARCH64_NONE: |
272 | case R_AARCH64_GLOB_DAT: |
273 | case R_AARCH64_AUTH_GLOB_DAT: |
274 | case R_AARCH64_JUMP_SLOT: |
275 | return 0; |
276 | case R_AARCH64_ABS16: |
277 | case R_AARCH64_PREL16: |
278 | return SignExtend64<16>(x: read16(ctx, p: buf)); |
279 | case R_AARCH64_ABS32: |
280 | case R_AARCH64_PREL32: |
281 | return SignExtend64<32>(x: read32(ctx, p: buf)); |
282 | case R_AARCH64_ABS64: |
283 | case R_AARCH64_PREL64: |
284 | case R_AARCH64_RELATIVE: |
285 | case R_AARCH64_IRELATIVE: |
286 | case R_AARCH64_TLS_TPREL64: |
287 | return read64(ctx, p: buf); |
288 | |
289 | // The following relocation types all point at instructions, and |
290 | // relocate an immediate field in the instruction. |
291 | // |
292 | // The general rule, from AAELF64 §5.7.2 "Addends and PC-bias", |
293 | // says: "If the relocation relocates an instruction the immediate |
294 | // field of the instruction is extracted, scaled as required by |
295 | // the instruction field encoding, and sign-extended to 64 bits". |
296 | |
297 | // The R_AARCH64_MOVW family operates on wide MOV/MOVK/MOVZ |
298 | // instructions, which have a 16-bit immediate field with its low |
299 | // bit in bit 5 of the instruction encoding. When the immediate |
300 | // field is used as an implicit addend for REL-type relocations, |
301 | // it is treated as added to the low bits of the output value, not |
302 | // shifted depending on the relocation type. |
303 | // |
304 | // This allows REL relocations to express the requirement 'please |
305 | // add 12345 to this symbol value and give me the four 16-bit |
306 | // chunks of the result', by putting the same addend 12345 in all |
307 | // four instructions. Carries between the 16-bit chunks are |
308 | // handled correctly, because the whole 64-bit addition is done |
309 | // once per relocation. |
310 | case R_AARCH64_MOVW_UABS_G0: |
311 | case R_AARCH64_MOVW_UABS_G0_NC: |
312 | case R_AARCH64_MOVW_UABS_G1: |
313 | case R_AARCH64_MOVW_UABS_G1_NC: |
314 | case R_AARCH64_MOVW_UABS_G2: |
315 | case R_AARCH64_MOVW_UABS_G2_NC: |
316 | case R_AARCH64_MOVW_UABS_G3: |
317 | return SignExtend64<16>(x: getBits(val: read32le(P: buf), start: 5, end: 20)); |
318 | |
319 | // R_AARCH64_TSTBR14 points at a TBZ or TBNZ instruction, which |
320 | // has a 14-bit offset measured in instructions, i.e. shifted left |
321 | // by 2. |
322 | case R_AARCH64_TSTBR14: |
323 | return SignExtend64<16>(x: getBits(val: read32le(P: buf), start: 5, end: 18) << 2); |
324 | |
325 | // R_AARCH64_CONDBR19 operates on the ordinary B.cond instruction, |
326 | // which has a 19-bit offset measured in instructions. |
327 | // |
328 | // R_AARCH64_LD_PREL_LO19 operates on the LDR (literal) |
329 | // instruction, which also has a 19-bit offset, measured in 4-byte |
330 | // chunks. So the calculation is the same as for |
331 | // R_AARCH64_CONDBR19. |
332 | case R_AARCH64_CONDBR19: |
333 | case R_AARCH64_LD_PREL_LO19: |
334 | return SignExtend64<21>(x: getBits(val: read32le(P: buf), start: 5, end: 23) << 2); |
335 | |
336 | // R_AARCH64_ADD_ABS_LO12_NC operates on ADD (immediate). The |
337 | // immediate can optionally be shifted left by 12 bits, but this |
338 | // relocation is intended for the case where it is not. |
339 | case R_AARCH64_ADD_ABS_LO12_NC: |
340 | return SignExtend64<12>(x: getBits(val: read32le(P: buf), start: 10, end: 21)); |
341 | |
342 | // R_AARCH64_ADR_PREL_LO21 operates on an ADR instruction, whose |
343 | // 21-bit immediate is split between two bits high up in the word |
344 | // (in fact the two _lowest_ order bits of the value) and 19 bits |
345 | // lower down. |
346 | // |
347 | // R_AARCH64_ADR_PREL_PG_HI21[_NC] operate on an ADRP instruction, |
348 | // which encodes the immediate in the same way, but will shift it |
349 | // left by 12 bits when the instruction executes. For the same |
350 | // reason as the MOVW family, we don't apply that left shift here. |
351 | case R_AARCH64_ADR_PREL_LO21: |
352 | case R_AARCH64_ADR_PREL_PG_HI21: |
353 | case R_AARCH64_ADR_PREL_PG_HI21_NC: |
354 | return SignExtend64<21>(x: (getBits(val: read32le(P: buf), start: 5, end: 23) << 2) | |
355 | getBits(val: read32le(P: buf), start: 29, end: 30)); |
356 | |
357 | // R_AARCH64_{JUMP,CALL}26 operate on B and BL, which have a |
358 | // 26-bit offset measured in instructions. |
359 | case R_AARCH64_JUMP26: |
360 | case R_AARCH64_CALL26: |
361 | return SignExtend64<28>(x: getBits(val: read32le(P: buf), start: 0, end: 25) << 2); |
362 | |
363 | default: |
364 | InternalErr(ctx, buf) << "cannot read addend for relocation "<< type; |
365 | return 0; |
366 | } |
367 | } |
368 | |
369 | void AArch64::writeGotPlt(uint8_t *buf, const Symbol &) const { |
370 | write64(ctx, p: buf, v: ctx.in.plt->getVA()); |
371 | } |
372 | |
373 | void AArch64::writeIgotPlt(uint8_t *buf, const Symbol &s) const { |
374 | if (ctx.arg.writeAddends) |
375 | write64(ctx, p: buf, v: s.getVA(ctx)); |
376 | } |
377 | |
378 | void AArch64::writePltHeader(uint8_t *buf) const { |
379 | const uint8_t pltData[] = { |
380 | 0xf0, 0x7b, 0xbf, 0xa9, // stp x16, x30, [sp,#-16]! |
381 | 0x10, 0x00, 0x00, 0x90, // adrp x16, Page(&(.got.plt[2])) |
382 | 0x11, 0x02, 0x40, 0xf9, // ldr x17, [x16, Offset(&(.got.plt[2]))] |
383 | 0x10, 0x02, 0x00, 0x91, // add x16, x16, Offset(&(.got.plt[2])) |
384 | 0x20, 0x02, 0x1f, 0xd6, // br x17 |
385 | 0x1f, 0x20, 0x03, 0xd5, // nop |
386 | 0x1f, 0x20, 0x03, 0xd5, // nop |
387 | 0x1f, 0x20, 0x03, 0xd5 // nop |
388 | }; |
389 | memcpy(dest: buf, src: pltData, n: sizeof(pltData)); |
390 | |
391 | uint64_t got = ctx.in.gotPlt->getVA(); |
392 | uint64_t plt = ctx.in.plt->getVA(); |
393 | relocateNoSym(loc: buf + 4, type: R_AARCH64_ADR_PREL_PG_HI21, |
394 | val: getAArch64Page(expr: got + 16) - getAArch64Page(expr: plt + 4)); |
395 | relocateNoSym(loc: buf + 8, type: R_AARCH64_LDST64_ABS_LO12_NC, val: got + 16); |
396 | relocateNoSym(loc: buf + 12, type: R_AARCH64_ADD_ABS_LO12_NC, val: got + 16); |
397 | } |
398 | |
399 | void AArch64::writePlt(uint8_t *buf, const Symbol &sym, |
400 | uint64_t pltEntryAddr) const { |
401 | const uint8_t inst[] = { |
402 | 0x10, 0x00, 0x00, 0x90, // adrp x16, Page(&(.got.plt[n])) |
403 | 0x11, 0x02, 0x40, 0xf9, // ldr x17, [x16, Offset(&(.got.plt[n]))] |
404 | 0x10, 0x02, 0x00, 0x91, // add x16, x16, Offset(&(.got.plt[n])) |
405 | 0x20, 0x02, 0x1f, 0xd6 // br x17 |
406 | }; |
407 | memcpy(dest: buf, src: inst, n: sizeof(inst)); |
408 | |
409 | uint64_t gotPltEntryAddr = sym.getGotPltVA(ctx); |
410 | relocateNoSym(loc: buf, type: R_AARCH64_ADR_PREL_PG_HI21, |
411 | val: getAArch64Page(expr: gotPltEntryAddr) - getAArch64Page(expr: pltEntryAddr)); |
412 | relocateNoSym(loc: buf + 4, type: R_AARCH64_LDST64_ABS_LO12_NC, val: gotPltEntryAddr); |
413 | relocateNoSym(loc: buf + 8, type: R_AARCH64_ADD_ABS_LO12_NC, val: gotPltEntryAddr); |
414 | } |
415 | |
416 | bool AArch64::needsThunk(RelExpr expr, RelType type, const InputFile *file, |
417 | uint64_t branchAddr, const Symbol &s, |
418 | int64_t a) const { |
419 | // If s is an undefined weak symbol and does not have a PLT entry then it will |
420 | // be resolved as a branch to the next instruction. If it is hidden, its |
421 | // binding has been converted to local, so we just check isUndefined() here. A |
422 | // undefined non-weak symbol will have been errored. |
423 | if (s.isUndefined() && !s.isInPlt(ctx)) |
424 | return false; |
425 | // ELF for the ARM 64-bit architecture, section Call and Jump relocations |
426 | // only permits range extension thunks for R_AARCH64_CALL26 and |
427 | // R_AARCH64_JUMP26 relocation types. |
428 | if (type != R_AARCH64_CALL26 && type != R_AARCH64_JUMP26 && |
429 | type != R_AARCH64_PLT32) |
430 | return false; |
431 | uint64_t dst = expr == R_PLT_PC ? s.getPltVA(ctx) : s.getVA(ctx, addend: a); |
432 | return !inBranchRange(type, src: branchAddr, dst); |
433 | } |
434 | |
435 | uint32_t AArch64::getThunkSectionSpacing() const { |
436 | // See comment in Arch/ARM.cpp for a more detailed explanation of |
437 | // getThunkSectionSpacing(). For AArch64 the only branches we are permitted to |
438 | // Thunk have a range of +/- 128 MiB |
439 | return (128 * 1024 * 1024) - 0x30000; |
440 | } |
441 | |
442 | bool AArch64::inBranchRange(RelType type, uint64_t src, uint64_t dst) const { |
443 | if (type != R_AARCH64_CALL26 && type != R_AARCH64_JUMP26 && |
444 | type != R_AARCH64_PLT32) |
445 | return true; |
446 | // The AArch64 call and unconditional branch instructions have a range of |
447 | // +/- 128 MiB. The PLT32 relocation supports a range up to +/- 2 GiB. |
448 | uint64_t range = |
449 | type == R_AARCH64_PLT32 ? (UINT64_C(1) << 31) : (128 * 1024 * 1024); |
450 | if (dst > src) { |
451 | // Immediate of branch is signed. |
452 | range -= 4; |
453 | return dst - src <= range; |
454 | } |
455 | return src - dst <= range; |
456 | } |
457 | |
458 | static void write32AArch64Addr(uint8_t *l, uint64_t imm) { |
459 | uint32_t immLo = (imm & 0x3) << 29; |
460 | uint32_t immHi = (imm & 0x1FFFFC) << 3; |
461 | uint64_t mask = (0x3 << 29) | (0x1FFFFC << 3); |
462 | write32le(P: l, V: (read32le(P: l) & ~mask) | immLo | immHi); |
463 | } |
464 | |
465 | static void writeMaskedBits32le(uint8_t *p, int32_t v, uint32_t mask) { |
466 | write32le(P: p, V: (read32le(P: p) & ~mask) | v); |
467 | } |
468 | |
469 | // Update the immediate field in a AARCH64 ldr, str, and add instruction. |
470 | static void write32Imm12(uint8_t *l, uint64_t imm) { |
471 | writeMaskedBits32le(p: l, v: (imm & 0xFFF) << 10, mask: 0xFFF << 10); |
472 | } |
473 | |
474 | // Update the immediate field in an AArch64 movk, movn or movz instruction |
475 | // for a signed relocation, and update the opcode of a movn or movz instruction |
476 | // to match the sign of the operand. |
477 | static void writeSMovWImm(uint8_t *loc, uint32_t imm) { |
478 | uint32_t inst = read32le(P: loc); |
479 | // Opcode field is bits 30, 29, with 10 = movz, 00 = movn and 11 = movk. |
480 | if (!(inst & (1 << 29))) { |
481 | // movn or movz. |
482 | if (imm & 0x10000) { |
483 | // Change opcode to movn, which takes an inverted operand. |
484 | imm ^= 0xFFFF; |
485 | inst &= ~(1 << 30); |
486 | } else { |
487 | // Change opcode to movz. |
488 | inst |= 1 << 30; |
489 | } |
490 | } |
491 | write32le(P: loc, V: inst | ((imm & 0xFFFF) << 5)); |
492 | } |
493 | |
494 | void AArch64::relocate(uint8_t *loc, const Relocation &rel, |
495 | uint64_t val) const { |
496 | switch (rel.type) { |
497 | case R_AARCH64_ABS16: |
498 | case R_AARCH64_PREL16: |
499 | checkIntUInt(ctx, loc, v: val, n: 16, rel); |
500 | write16(ctx, p: loc, v: val); |
501 | break; |
502 | case R_AARCH64_ABS32: |
503 | case R_AARCH64_PREL32: |
504 | checkIntUInt(ctx, loc, v: val, n: 32, rel); |
505 | write32(ctx, p: loc, v: val); |
506 | break; |
507 | case R_AARCH64_PLT32: |
508 | case R_AARCH64_GOTPCREL32: |
509 | checkInt(ctx, loc, v: val, n: 32, rel); |
510 | write32(ctx, p: loc, v: val); |
511 | break; |
512 | case R_AARCH64_ABS64: |
513 | // AArch64 relocations to tagged symbols have extended semantics, as |
514 | // described here: |
515 | // https://github.com/ARM-software/abi-aa/blob/main/memtagabielf64/memtagabielf64.rst#841extended-semantics-of-r_aarch64_relative. |
516 | // tl;dr: encode the symbol's special addend in the place, which is an |
517 | // offset to the point where the logical tag is derived from. Quick hack, if |
518 | // the addend is within the symbol's bounds, no need to encode the tag |
519 | // derivation offset. |
520 | if (rel.sym && rel.sym->isTagged() && |
521 | (rel.addend < 0 || |
522 | rel.addend >= static_cast<int64_t>(rel.sym->getSize()))) |
523 | write64(ctx, p: loc, v: -rel.addend); |
524 | else |
525 | write64(ctx, p: loc, v: val); |
526 | break; |
527 | case R_AARCH64_PREL64: |
528 | write64(ctx, p: loc, v: val); |
529 | break; |
530 | case R_AARCH64_AUTH_ABS64: |
531 | // If val is wider than 32 bits, the relocation must have been moved from |
532 | // .relr.auth.dyn to .rela.dyn, and the addend write is not needed. |
533 | // |
534 | // If val fits in 32 bits, we have two potential scenarios: |
535 | // * True RELR: Write the 32-bit `val`. |
536 | // * RELA: Even if the value now fits in 32 bits, it might have been |
537 | // converted from RELR during an iteration in |
538 | // finalizeAddressDependentContent(). Writing the value is harmless |
539 | // because dynamic linking ignores it. |
540 | if (isInt<32>(x: val)) |
541 | write32(ctx, p: loc, v: val); |
542 | break; |
543 | case R_AARCH64_ADD_ABS_LO12_NC: |
544 | case R_AARCH64_AUTH_GOT_ADD_LO12_NC: |
545 | write32Imm12(l: loc, imm: val); |
546 | break; |
547 | case R_AARCH64_ADR_GOT_PAGE: |
548 | case R_AARCH64_AUTH_ADR_GOT_PAGE: |
549 | case R_AARCH64_ADR_PREL_PG_HI21: |
550 | case R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21: |
551 | case R_AARCH64_TLSDESC_ADR_PAGE21: |
552 | case R_AARCH64_AUTH_TLSDESC_ADR_PAGE21: |
553 | checkInt(ctx, loc, v: val, n: 33, rel); |
554 | [[fallthrough]]; |
555 | case R_AARCH64_ADR_PREL_PG_HI21_NC: |
556 | write32AArch64Addr(l: loc, imm: val >> 12); |
557 | break; |
558 | case R_AARCH64_ADR_PREL_LO21: |
559 | case R_AARCH64_AUTH_GOT_ADR_PREL_LO21: |
560 | checkInt(ctx, loc, v: val, n: 21, rel); |
561 | write32AArch64Addr(l: loc, imm: val); |
562 | break; |
563 | case R_AARCH64_JUMP26: |
564 | // Normally we would just write the bits of the immediate field, however |
565 | // when patching instructions for the cpu errata fix -fix-cortex-a53-843419 |
566 | // we want to replace a non-branch instruction with a branch immediate |
567 | // instruction. By writing all the bits of the instruction including the |
568 | // opcode and the immediate (0 001 | 01 imm26) we can do this |
569 | // transformation by placing a R_AARCH64_JUMP26 relocation at the offset of |
570 | // the instruction we want to patch. |
571 | write32le(P: loc, V: 0x14000000); |
572 | [[fallthrough]]; |
573 | case R_AARCH64_CALL26: |
574 | checkInt(ctx, loc, v: val, n: 28, rel); |
575 | writeMaskedBits32le(p: loc, v: (val & 0x0FFFFFFC) >> 2, mask: 0x0FFFFFFC >> 2); |
576 | break; |
577 | case R_AARCH64_CONDBR19: |
578 | case R_AARCH64_LD_PREL_LO19: |
579 | case R_AARCH64_GOT_LD_PREL19: |
580 | case R_AARCH64_AUTH_GOT_LD_PREL19: |
581 | checkAlignment(ctx, loc, v: val, n: 4, rel); |
582 | checkInt(ctx, loc, v: val, n: 21, rel); |
583 | writeMaskedBits32le(p: loc, v: (val & 0x1FFFFC) << 3, mask: 0x1FFFFC << 3); |
584 | break; |
585 | case R_AARCH64_LDST8_ABS_LO12_NC: |
586 | case R_AARCH64_TLSLE_LDST8_TPREL_LO12_NC: |
587 | write32Imm12(l: loc, imm: getBits(val, start: 0, end: 11)); |
588 | break; |
589 | case R_AARCH64_LDST16_ABS_LO12_NC: |
590 | case R_AARCH64_TLSLE_LDST16_TPREL_LO12_NC: |
591 | checkAlignment(ctx, loc, v: val, n: 2, rel); |
592 | write32Imm12(l: loc, imm: getBits(val, start: 1, end: 11)); |
593 | break; |
594 | case R_AARCH64_LDST32_ABS_LO12_NC: |
595 | case R_AARCH64_TLSLE_LDST32_TPREL_LO12_NC: |
596 | checkAlignment(ctx, loc, v: val, n: 4, rel); |
597 | write32Imm12(l: loc, imm: getBits(val, start: 2, end: 11)); |
598 | break; |
599 | case R_AARCH64_LDST64_ABS_LO12_NC: |
600 | case R_AARCH64_LD64_GOT_LO12_NC: |
601 | case R_AARCH64_AUTH_LD64_GOT_LO12_NC: |
602 | case R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC: |
603 | case R_AARCH64_TLSLE_LDST64_TPREL_LO12_NC: |
604 | case R_AARCH64_TLSDESC_LD64_LO12: |
605 | case R_AARCH64_AUTH_TLSDESC_LD64_LO12: |
606 | checkAlignment(ctx, loc, v: val, n: 8, rel); |
607 | write32Imm12(l: loc, imm: getBits(val, start: 3, end: 11)); |
608 | break; |
609 | case R_AARCH64_LDST128_ABS_LO12_NC: |
610 | case R_AARCH64_TLSLE_LDST128_TPREL_LO12_NC: |
611 | checkAlignment(ctx, loc, v: val, n: 16, rel); |
612 | write32Imm12(l: loc, imm: getBits(val, start: 4, end: 11)); |
613 | break; |
614 | case R_AARCH64_LD64_GOTPAGE_LO15: |
615 | checkAlignment(ctx, loc, v: val, n: 8, rel); |
616 | write32Imm12(l: loc, imm: getBits(val, start: 3, end: 14)); |
617 | break; |
618 | case R_AARCH64_MOVW_UABS_G0: |
619 | checkUInt(ctx, loc, v: val, n: 16, rel); |
620 | [[fallthrough]]; |
621 | case R_AARCH64_MOVW_UABS_G0_NC: |
622 | writeMaskedBits32le(p: loc, v: (val & 0xFFFF) << 5, mask: 0xFFFF << 5); |
623 | break; |
624 | case R_AARCH64_MOVW_UABS_G1: |
625 | checkUInt(ctx, loc, v: val, n: 32, rel); |
626 | [[fallthrough]]; |
627 | case R_AARCH64_MOVW_UABS_G1_NC: |
628 | writeMaskedBits32le(p: loc, v: (val & 0xFFFF0000) >> 11, mask: 0xFFFF0000 >> 11); |
629 | break; |
630 | case R_AARCH64_MOVW_UABS_G2: |
631 | checkUInt(ctx, loc, v: val, n: 48, rel); |
632 | [[fallthrough]]; |
633 | case R_AARCH64_MOVW_UABS_G2_NC: |
634 | writeMaskedBits32le(p: loc, v: (val & 0xFFFF00000000) >> 27, |
635 | mask: 0xFFFF00000000 >> 27); |
636 | break; |
637 | case R_AARCH64_MOVW_UABS_G3: |
638 | writeMaskedBits32le(p: loc, v: (val & 0xFFFF000000000000) >> 43, |
639 | mask: 0xFFFF000000000000 >> 43); |
640 | break; |
641 | case R_AARCH64_MOVW_PREL_G0: |
642 | case R_AARCH64_MOVW_SABS_G0: |
643 | case R_AARCH64_TLSLE_MOVW_TPREL_G0: |
644 | checkInt(ctx, loc, v: val, n: 17, rel); |
645 | [[fallthrough]]; |
646 | case R_AARCH64_MOVW_PREL_G0_NC: |
647 | case R_AARCH64_TLSLE_MOVW_TPREL_G0_NC: |
648 | writeSMovWImm(loc, imm: val); |
649 | break; |
650 | case R_AARCH64_MOVW_PREL_G1: |
651 | case R_AARCH64_MOVW_SABS_G1: |
652 | case R_AARCH64_TLSLE_MOVW_TPREL_G1: |
653 | checkInt(ctx, loc, v: val, n: 33, rel); |
654 | [[fallthrough]]; |
655 | case R_AARCH64_MOVW_PREL_G1_NC: |
656 | case R_AARCH64_TLSLE_MOVW_TPREL_G1_NC: |
657 | writeSMovWImm(loc, imm: val >> 16); |
658 | break; |
659 | case R_AARCH64_MOVW_PREL_G2: |
660 | case R_AARCH64_MOVW_SABS_G2: |
661 | case R_AARCH64_TLSLE_MOVW_TPREL_G2: |
662 | checkInt(ctx, loc, v: val, n: 49, rel); |
663 | [[fallthrough]]; |
664 | case R_AARCH64_MOVW_PREL_G2_NC: |
665 | writeSMovWImm(loc, imm: val >> 32); |
666 | break; |
667 | case R_AARCH64_MOVW_PREL_G3: |
668 | writeSMovWImm(loc, imm: val >> 48); |
669 | break; |
670 | case R_AARCH64_TSTBR14: |
671 | checkInt(ctx, loc, v: val, n: 16, rel); |
672 | writeMaskedBits32le(p: loc, v: (val & 0xFFFC) << 3, mask: 0xFFFC << 3); |
673 | break; |
674 | case R_AARCH64_TLSLE_ADD_TPREL_HI12: |
675 | checkUInt(ctx, loc, v: val, n: 24, rel); |
676 | write32Imm12(l: loc, imm: val >> 12); |
677 | break; |
678 | case R_AARCH64_TLSLE_ADD_TPREL_LO12_NC: |
679 | case R_AARCH64_TLSDESC_ADD_LO12: |
680 | case R_AARCH64_AUTH_TLSDESC_ADD_LO12: |
681 | write32Imm12(l: loc, imm: val); |
682 | break; |
683 | case R_AARCH64_TLSDESC: |
684 | // For R_AARCH64_TLSDESC the addend is stored in the second 64-bit word. |
685 | write64(ctx, p: loc + 8, v: val); |
686 | break; |
687 | default: |
688 | llvm_unreachable("unknown relocation"); |
689 | } |
690 | } |
691 | |
692 | void AArch64::relaxTlsGdToLe(uint8_t *loc, const Relocation &rel, |
693 | uint64_t val) const { |
694 | // TLSDESC Global-Dynamic relocation are in the form: |
695 | // adrp x0, :tlsdesc:v [R_AARCH64_TLSDESC_ADR_PAGE21] |
696 | // ldr x1, [x0, #:tlsdesc_lo12:v [R_AARCH64_TLSDESC_LD64_LO12] |
697 | // add x0, x0, :tlsdesc_los:v [R_AARCH64_TLSDESC_ADD_LO12] |
698 | // .tlsdesccall [R_AARCH64_TLSDESC_CALL] |
699 | // blr x1 |
700 | // And it can optimized to: |
701 | // movz x0, #0x0, lsl #16 |
702 | // movk x0, #0x10 |
703 | // nop |
704 | // nop |
705 | checkUInt(ctx, loc, v: val, n: 32, rel); |
706 | |
707 | switch (rel.type) { |
708 | case R_AARCH64_TLSDESC_ADD_LO12: |
709 | case R_AARCH64_TLSDESC_CALL: |
710 | write32le(P: loc, V: 0xd503201f); // nop |
711 | return; |
712 | case R_AARCH64_TLSDESC_ADR_PAGE21: |
713 | write32le(P: loc, V: 0xd2a00000 | (((val >> 16) & 0xffff) << 5)); // movz |
714 | return; |
715 | case R_AARCH64_TLSDESC_LD64_LO12: |
716 | write32le(P: loc, V: 0xf2800000 | ((val & 0xffff) << 5)); // movk |
717 | return; |
718 | default: |
719 | llvm_unreachable("unsupported relocation for TLS GD to LE relaxation"); |
720 | } |
721 | } |
722 | |
723 | void AArch64::relaxTlsGdToIe(uint8_t *loc, const Relocation &rel, |
724 | uint64_t val) const { |
725 | // TLSDESC Global-Dynamic relocation are in the form: |
726 | // adrp x0, :tlsdesc:v [R_AARCH64_TLSDESC_ADR_PAGE21] |
727 | // ldr x1, [x0, #:tlsdesc_lo12:v [R_AARCH64_TLSDESC_LD64_LO12] |
728 | // add x0, x0, :tlsdesc_los:v [R_AARCH64_TLSDESC_ADD_LO12] |
729 | // .tlsdesccall [R_AARCH64_TLSDESC_CALL] |
730 | // blr x1 |
731 | // And it can optimized to: |
732 | // adrp x0, :gottprel:v |
733 | // ldr x0, [x0, :gottprel_lo12:v] |
734 | // nop |
735 | // nop |
736 | |
737 | switch (rel.type) { |
738 | case R_AARCH64_TLSDESC_ADD_LO12: |
739 | case R_AARCH64_TLSDESC_CALL: |
740 | write32le(P: loc, V: 0xd503201f); // nop |
741 | break; |
742 | case R_AARCH64_TLSDESC_ADR_PAGE21: |
743 | write32le(P: loc, V: 0x90000000); // adrp |
744 | relocateNoSym(loc, type: R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21, val); |
745 | break; |
746 | case R_AARCH64_TLSDESC_LD64_LO12: |
747 | write32le(P: loc, V: 0xf9400000); // ldr |
748 | relocateNoSym(loc, type: R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC, val); |
749 | break; |
750 | default: |
751 | llvm_unreachable("unsupported relocation for TLS GD to LE relaxation"); |
752 | } |
753 | } |
754 | |
755 | void AArch64::relaxTlsIeToLe(uint8_t *loc, const Relocation &rel, |
756 | uint64_t val) const { |
757 | checkUInt(ctx, loc, v: val, n: 32, rel); |
758 | |
759 | if (rel.type == R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21) { |
760 | // Generate MOVZ. |
761 | uint32_t regNo = read32le(P: loc) & 0x1f; |
762 | write32le(P: loc, V: (0xd2a00000 | regNo) | (((val >> 16) & 0xffff) << 5)); |
763 | return; |
764 | } |
765 | if (rel.type == R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC) { |
766 | // Generate MOVK. |
767 | uint32_t regNo = read32le(P: loc) & 0x1f; |
768 | write32le(P: loc, V: (0xf2800000 | regNo) | ((val & 0xffff) << 5)); |
769 | return; |
770 | } |
771 | llvm_unreachable("invalid relocation for TLS IE to LE relaxation"); |
772 | } |
773 | |
774 | AArch64Relaxer::AArch64Relaxer(Ctx &ctx, ArrayRef<Relocation> relocs) |
775 | : ctx(ctx) { |
776 | if (!ctx.arg.relax) |
777 | return; |
778 | // Check if R_AARCH64_ADR_GOT_PAGE and R_AARCH64_LD64_GOT_LO12_NC |
779 | // always appear in pairs. |
780 | size_t i = 0; |
781 | const size_t size = relocs.size(); |
782 | for (; i != size; ++i) { |
783 | if (relocs[i].type == R_AARCH64_ADR_GOT_PAGE) { |
784 | if (i + 1 < size && relocs[i + 1].type == R_AARCH64_LD64_GOT_LO12_NC) { |
785 | ++i; |
786 | continue; |
787 | } |
788 | break; |
789 | } else if (relocs[i].type == R_AARCH64_LD64_GOT_LO12_NC) { |
790 | break; |
791 | } |
792 | } |
793 | safeToRelaxAdrpLdr = i == size; |
794 | } |
795 | |
796 | bool AArch64Relaxer::tryRelaxAdrpAdd(const Relocation &adrpRel, |
797 | const Relocation &addRel, uint64_t secAddr, |
798 | uint8_t *buf) const { |
799 | // When the address of sym is within the range of ADR then |
800 | // we may relax |
801 | // ADRP xn, sym |
802 | // ADD xn, xn, :lo12: sym |
803 | // to |
804 | // NOP |
805 | // ADR xn, sym |
806 | if (!ctx.arg.relax || adrpRel.type != R_AARCH64_ADR_PREL_PG_HI21 || |
807 | addRel.type != R_AARCH64_ADD_ABS_LO12_NC) |
808 | return false; |
809 | // Check if the relocations apply to consecutive instructions. |
810 | if (adrpRel.offset + 4 != addRel.offset) |
811 | return false; |
812 | if (adrpRel.sym != addRel.sym) |
813 | return false; |
814 | if (adrpRel.addend != 0 || addRel.addend != 0) |
815 | return false; |
816 | |
817 | uint32_t adrpInstr = read32le(P: buf + adrpRel.offset); |
818 | uint32_t addInstr = read32le(P: buf + addRel.offset); |
819 | // Check if the first instruction is ADRP and the second instruction is ADD. |
820 | if ((adrpInstr & 0x9f000000) != 0x90000000 || |
821 | (addInstr & 0xffc00000) != 0x91000000) |
822 | return false; |
823 | uint32_t adrpDestReg = adrpInstr & 0x1f; |
824 | uint32_t addDestReg = addInstr & 0x1f; |
825 | uint32_t addSrcReg = (addInstr >> 5) & 0x1f; |
826 | if (adrpDestReg != addDestReg || adrpDestReg != addSrcReg) |
827 | return false; |
828 | |
829 | Symbol &sym = *adrpRel.sym; |
830 | // Check if the address difference is within 1MiB range. |
831 | int64_t val = sym.getVA(ctx) - (secAddr + addRel.offset); |
832 | if (val < -1024 * 1024 || val >= 1024 * 1024) |
833 | return false; |
834 | |
835 | Relocation adrRel = {.expr: R_ABS, .type: R_AARCH64_ADR_PREL_LO21, .offset: addRel.offset, |
836 | /*addend=*/0, .sym: &sym}; |
837 | // nop |
838 | write32le(P: buf + adrpRel.offset, V: 0xd503201f); |
839 | // adr x_<dest_reg> |
840 | write32le(P: buf + adrRel.offset, V: 0x10000000 | adrpDestReg); |
841 | ctx.target->relocate(loc: buf + adrRel.offset, rel: adrRel, val); |
842 | return true; |
843 | } |
844 | |
845 | bool AArch64Relaxer::tryRelaxAdrpLdr(const Relocation &adrpRel, |
846 | const Relocation &ldrRel, uint64_t secAddr, |
847 | uint8_t *buf) const { |
848 | if (!safeToRelaxAdrpLdr) |
849 | return false; |
850 | |
851 | // When the definition of sym is not preemptible then we may |
852 | // be able to relax |
853 | // ADRP xn, :got: sym |
854 | // LDR xn, [ xn :got_lo12: sym] |
855 | // to |
856 | // ADRP xn, sym |
857 | // ADD xn, xn, :lo_12: sym |
858 | |
859 | if (adrpRel.type != R_AARCH64_ADR_GOT_PAGE || |
860 | ldrRel.type != R_AARCH64_LD64_GOT_LO12_NC) |
861 | return false; |
862 | // Check if the relocations apply to consecutive instructions. |
863 | if (adrpRel.offset + 4 != ldrRel.offset) |
864 | return false; |
865 | // Check if the relocations reference the same symbol and |
866 | // skip undefined, preemptible and STT_GNU_IFUNC symbols. |
867 | if (!adrpRel.sym || adrpRel.sym != ldrRel.sym || !adrpRel.sym->isDefined() || |
868 | adrpRel.sym->isPreemptible || adrpRel.sym->isGnuIFunc()) |
869 | return false; |
870 | // Check if the addends of the both relocations are zero. |
871 | if (adrpRel.addend != 0 || ldrRel.addend != 0) |
872 | return false; |
873 | uint32_t adrpInstr = read32le(P: buf + adrpRel.offset); |
874 | uint32_t ldrInstr = read32le(P: buf + ldrRel.offset); |
875 | // Check if the first instruction is ADRP and the second instruction is LDR. |
876 | if ((adrpInstr & 0x9f000000) != 0x90000000 || |
877 | (ldrInstr & 0x3b000000) != 0x39000000) |
878 | return false; |
879 | // Check the value of the sf bit. |
880 | if (!(ldrInstr >> 31)) |
881 | return false; |
882 | uint32_t adrpDestReg = adrpInstr & 0x1f; |
883 | uint32_t ldrDestReg = ldrInstr & 0x1f; |
884 | uint32_t ldrSrcReg = (ldrInstr >> 5) & 0x1f; |
885 | // Check if ADPR and LDR use the same register. |
886 | if (adrpDestReg != ldrDestReg || adrpDestReg != ldrSrcReg) |
887 | return false; |
888 | |
889 | Symbol &sym = *adrpRel.sym; |
890 | // GOT references to absolute symbols can't be relaxed to use ADRP/ADD in |
891 | // position-independent code because these instructions produce a relative |
892 | // address. |
893 | if (ctx.arg.isPic && !cast<Defined>(Val&: sym).section) |
894 | return false; |
895 | // Check if the address difference is within 4GB range. |
896 | int64_t val = |
897 | getAArch64Page(expr: sym.getVA(ctx)) - getAArch64Page(expr: secAddr + adrpRel.offset); |
898 | if (val != llvm::SignExtend64(X: val, B: 33)) |
899 | return false; |
900 | |
901 | Relocation adrpSymRel = {.expr: RE_AARCH64_PAGE_PC, .type: R_AARCH64_ADR_PREL_PG_HI21, |
902 | .offset: adrpRel.offset, /*addend=*/0, .sym: &sym}; |
903 | Relocation addRel = {.expr: R_ABS, .type: R_AARCH64_ADD_ABS_LO12_NC, .offset: ldrRel.offset, |
904 | /*addend=*/0, .sym: &sym}; |
905 | |
906 | // adrp x_<dest_reg> |
907 | write32le(P: buf + adrpSymRel.offset, V: 0x90000000 | adrpDestReg); |
908 | // add x_<dest reg>, x_<dest reg> |
909 | write32le(P: buf + addRel.offset, V: 0x91000000 | adrpDestReg | (adrpDestReg << 5)); |
910 | |
911 | ctx.target->relocate( |
912 | loc: buf + adrpSymRel.offset, rel: adrpSymRel, |
913 | val: SignExtend64(X: getAArch64Page(expr: sym.getVA(ctx)) - |
914 | getAArch64Page(expr: secAddr + adrpSymRel.offset), |
915 | B: 64)); |
916 | ctx.target->relocate(loc: buf + addRel.offset, rel: addRel, |
917 | val: SignExtend64(X: sym.getVA(ctx), B: 64)); |
918 | tryRelaxAdrpAdd(adrpRel: adrpSymRel, addRel, secAddr, buf); |
919 | return true; |
920 | } |
921 | |
922 | // Tagged symbols have upper address bits that are added by the dynamic loader, |
923 | // and thus need the full 64-bit GOT entry. Do not relax such symbols. |
924 | static bool needsGotForMemtag(const Relocation &rel) { |
925 | return rel.sym->isTagged() && needsGot(expr: rel.expr); |
926 | } |
927 | |
928 | void AArch64::relocateAlloc(InputSectionBase &sec, uint8_t *buf) const { |
929 | uint64_t secAddr = sec.getOutputSection()->addr; |
930 | if (auto *s = dyn_cast<InputSection>(Val: &sec)) |
931 | secAddr += s->outSecOff; |
932 | else if (auto *ehIn = dyn_cast<EhInputSection>(Val: &sec)) |
933 | secAddr += ehIn->getParent()->outSecOff; |
934 | AArch64Relaxer relaxer(ctx, sec.relocs()); |
935 | for (size_t i = 0, size = sec.relocs().size(); i != size; ++i) { |
936 | const Relocation &rel = sec.relocs()[i]; |
937 | uint8_t *loc = buf + rel.offset; |
938 | const uint64_t val = sec.getRelocTargetVA(ctx, r: rel, p: secAddr + rel.offset); |
939 | |
940 | if (needsGotForMemtag(rel)) { |
941 | relocate(loc, rel, val); |
942 | continue; |
943 | } |
944 | |
945 | switch (rel.expr) { |
946 | case RE_AARCH64_GOT_PAGE_PC: |
947 | if (i + 1 < size && |
948 | relaxer.tryRelaxAdrpLdr(adrpRel: rel, ldrRel: sec.relocs()[i + 1], secAddr, buf)) { |
949 | ++i; |
950 | continue; |
951 | } |
952 | break; |
953 | case RE_AARCH64_PAGE_PC: |
954 | if (i + 1 < size && |
955 | relaxer.tryRelaxAdrpAdd(adrpRel: rel, addRel: sec.relocs()[i + 1], secAddr, buf)) { |
956 | ++i; |
957 | continue; |
958 | } |
959 | break; |
960 | case RE_AARCH64_RELAX_TLS_GD_TO_IE_PAGE_PC: |
961 | case R_RELAX_TLS_GD_TO_IE_ABS: |
962 | relaxTlsGdToIe(loc, rel, val); |
963 | continue; |
964 | case R_RELAX_TLS_GD_TO_LE: |
965 | relaxTlsGdToLe(loc, rel, val); |
966 | continue; |
967 | case R_RELAX_TLS_IE_TO_LE: |
968 | relaxTlsIeToLe(loc, rel, val); |
969 | continue; |
970 | default: |
971 | break; |
972 | } |
973 | relocate(loc, rel, val); |
974 | } |
975 | } |
976 | |
977 | // AArch64 may use security features in variant PLT sequences. These are: |
978 | // Pointer Authentication (PAC), introduced in armv8.3-a and Branch Target |
979 | // Indicator (BTI) introduced in armv8.5-a. The additional instructions used |
980 | // in the variant Plt sequences are encoded in the Hint space so they can be |
981 | // deployed on older architectures, which treat the instructions as a nop. |
982 | // PAC and BTI can be combined leading to the following combinations: |
983 | // writePltHeader |
984 | // writePltHeaderBti (no PAC Header needed) |
985 | // writePlt |
986 | // writePltBti (BTI only) |
987 | // writePltPac (PAC only) |
988 | // writePltBtiPac (BTI and PAC) |
989 | // |
990 | // When PAC is enabled the dynamic loader encrypts the address that it places |
991 | // in the .got.plt using the pacia1716 instruction which encrypts the value in |
992 | // x17 using the modifier in x16. The static linker places autia1716 before the |
993 | // indirect branch to x17 to authenticate the address in x17 with the modifier |
994 | // in x16. This makes it more difficult for an attacker to modify the value in |
995 | // the .got.plt. |
996 | // |
997 | // When BTI is enabled all indirect branches must land on a bti instruction. |
998 | // The static linker must place a bti instruction at the start of any PLT entry |
999 | // that may be the target of an indirect branch. As the PLT entries call the |
1000 | // lazy resolver indirectly this must have a bti instruction at start. In |
1001 | // general a bti instruction is not needed for a PLT entry as indirect calls |
1002 | // are resolved to the function address and not the PLT entry for the function. |
1003 | // There are a small number of cases where the PLT address can escape, such as |
1004 | // taking the address of a function or ifunc via a non got-generating |
1005 | // relocation, and a shared library refers to that symbol. |
1006 | // |
1007 | // We use the bti c variant of the instruction which permits indirect branches |
1008 | // (br) via x16/x17 and indirect function calls (blr) via any register. The ABI |
1009 | // guarantees that all indirect branches from code requiring BTI protection |
1010 | // will go via x16/x17 |
1011 | |
1012 | namespace { |
1013 | class AArch64BtiPac final : public AArch64 { |
1014 | public: |
1015 | AArch64BtiPac(Ctx &); |
1016 | void writePltHeader(uint8_t *buf) const override; |
1017 | void writePlt(uint8_t *buf, const Symbol &sym, |
1018 | uint64_t pltEntryAddr) const override; |
1019 | |
1020 | private: |
1021 | bool btiHeader; // bti instruction needed in PLT Header and Entry |
1022 | enum { |
1023 | PEK_NoAuth, |
1024 | PEK_AuthHint, // use autia1716 instr for authenticated branch in PLT entry |
1025 | PEK_Auth, // use braa instr for authenticated branch in PLT entry |
1026 | } pacEntryKind; |
1027 | }; |
1028 | } // namespace |
1029 | |
1030 | AArch64BtiPac::AArch64BtiPac(Ctx &ctx) : AArch64(ctx) { |
1031 | btiHeader = (ctx.arg.andFeatures & GNU_PROPERTY_AARCH64_FEATURE_1_BTI); |
1032 | // A BTI (Branch Target Indicator) Plt Entry is only required if the |
1033 | // address of the PLT entry can be taken by the program, which permits an |
1034 | // indirect jump to the PLT entry. This can happen when the address |
1035 | // of the PLT entry for a function is canonicalised due to the address of |
1036 | // the function in an executable being taken by a shared library, or |
1037 | // non-preemptible ifunc referenced by non-GOT-generating, non-PLT-generating |
1038 | // relocations. |
1039 | // The PAC PLT entries require dynamic loader support and this isn't known |
1040 | // from properties in the objects, so we use the command line flag. |
1041 | // By default we only use hint-space instructions, but if we detect the |
1042 | // PAuthABI, which requires v8.3-A, we can use the non-hint space |
1043 | // instructions. |
1044 | |
1045 | if (ctx.arg.zPacPlt) { |
1046 | if (llvm::any_of(Range&: ctx.aarch64PauthAbiCoreInfo, |
1047 | P: [](uint8_t c) { return c != 0; })) |
1048 | pacEntryKind = PEK_Auth; |
1049 | else |
1050 | pacEntryKind = PEK_AuthHint; |
1051 | } else { |
1052 | pacEntryKind = PEK_NoAuth; |
1053 | } |
1054 | |
1055 | if (btiHeader || (pacEntryKind != PEK_NoAuth)) { |
1056 | pltEntrySize = 24; |
1057 | ipltEntrySize = 24; |
1058 | } |
1059 | } |
1060 | |
1061 | void AArch64BtiPac::writePltHeader(uint8_t *buf) const { |
1062 | const uint8_t btiData[] = { 0x5f, 0x24, 0x03, 0xd5 }; // bti c |
1063 | const uint8_t pltData[] = { |
1064 | 0xf0, 0x7b, 0xbf, 0xa9, // stp x16, x30, [sp,#-16]! |
1065 | 0x10, 0x00, 0x00, 0x90, // adrp x16, Page(&(.got.plt[2])) |
1066 | 0x11, 0x02, 0x40, 0xf9, // ldr x17, [x16, Offset(&(.got.plt[2]))] |
1067 | 0x10, 0x02, 0x00, 0x91, // add x16, x16, Offset(&(.got.plt[2])) |
1068 | 0x20, 0x02, 0x1f, 0xd6, // br x17 |
1069 | 0x1f, 0x20, 0x03, 0xd5, // nop |
1070 | 0x1f, 0x20, 0x03, 0xd5 // nop |
1071 | }; |
1072 | const uint8_t nopData[] = { 0x1f, 0x20, 0x03, 0xd5 }; // nop |
1073 | |
1074 | uint64_t got = ctx.in.gotPlt->getVA(); |
1075 | uint64_t plt = ctx.in.plt->getVA(); |
1076 | |
1077 | if (btiHeader) { |
1078 | // PltHeader is called indirectly by plt[N]. Prefix pltData with a BTI C |
1079 | // instruction. |
1080 | memcpy(dest: buf, src: btiData, n: sizeof(btiData)); |
1081 | buf += sizeof(btiData); |
1082 | plt += sizeof(btiData); |
1083 | } |
1084 | memcpy(dest: buf, src: pltData, n: sizeof(pltData)); |
1085 | |
1086 | relocateNoSym(loc: buf + 4, type: R_AARCH64_ADR_PREL_PG_HI21, |
1087 | val: getAArch64Page(expr: got + 16) - getAArch64Page(expr: plt + 4)); |
1088 | relocateNoSym(loc: buf + 8, type: R_AARCH64_LDST64_ABS_LO12_NC, val: got + 16); |
1089 | relocateNoSym(loc: buf + 12, type: R_AARCH64_ADD_ABS_LO12_NC, val: got + 16); |
1090 | if (!btiHeader) |
1091 | // We didn't add the BTI c instruction so round out size with NOP. |
1092 | memcpy(dest: buf + sizeof(pltData), src: nopData, n: sizeof(nopData)); |
1093 | } |
1094 | |
1095 | void AArch64BtiPac::writePlt(uint8_t *buf, const Symbol &sym, |
1096 | uint64_t pltEntryAddr) const { |
1097 | // The PLT entry is of the form: |
1098 | // [btiData] addrInst (pacBr | stdBr) [nopData] |
1099 | const uint8_t btiData[] = { 0x5f, 0x24, 0x03, 0xd5 }; // bti c |
1100 | const uint8_t addrInst[] = { |
1101 | 0x10, 0x00, 0x00, 0x90, // adrp x16, Page(&(.got.plt[n])) |
1102 | 0x11, 0x02, 0x40, 0xf9, // ldr x17, [x16, Offset(&(.got.plt[n]))] |
1103 | 0x10, 0x02, 0x00, 0x91 // add x16, x16, Offset(&(.got.plt[n])) |
1104 | }; |
1105 | const uint8_t pacHintBr[] = { |
1106 | 0x9f, 0x21, 0x03, 0xd5, // autia1716 |
1107 | 0x20, 0x02, 0x1f, 0xd6 // br x17 |
1108 | }; |
1109 | const uint8_t pacBr[] = { |
1110 | 0x30, 0x0a, 0x1f, 0xd7, // braa x17, x16 |
1111 | 0x1f, 0x20, 0x03, 0xd5 // nop |
1112 | }; |
1113 | const uint8_t stdBr[] = { |
1114 | 0x20, 0x02, 0x1f, 0xd6, // br x17 |
1115 | 0x1f, 0x20, 0x03, 0xd5 // nop |
1116 | }; |
1117 | const uint8_t nopData[] = { 0x1f, 0x20, 0x03, 0xd5 }; // nop |
1118 | |
1119 | // NEEDS_COPY indicates a non-ifunc canonical PLT entry whose address may |
1120 | // escape to shared objects. isInIplt indicates a non-preemptible ifunc. Its |
1121 | // address may escape if referenced by a direct relocation. If relative |
1122 | // vtables are used then if the vtable is in a shared object the offsets will |
1123 | // be to the PLT entry. The condition is conservative. |
1124 | bool hasBti = btiHeader && |
1125 | (sym.hasFlag(bit: NEEDS_COPY) || sym.isInIplt || sym.thunkAccessed); |
1126 | if (hasBti) { |
1127 | memcpy(dest: buf, src: btiData, n: sizeof(btiData)); |
1128 | buf += sizeof(btiData); |
1129 | pltEntryAddr += sizeof(btiData); |
1130 | } |
1131 | |
1132 | uint64_t gotPltEntryAddr = sym.getGotPltVA(ctx); |
1133 | memcpy(dest: buf, src: addrInst, n: sizeof(addrInst)); |
1134 | relocateNoSym(loc: buf, type: R_AARCH64_ADR_PREL_PG_HI21, |
1135 | val: getAArch64Page(expr: gotPltEntryAddr) - getAArch64Page(expr: pltEntryAddr)); |
1136 | relocateNoSym(loc: buf + 4, type: R_AARCH64_LDST64_ABS_LO12_NC, val: gotPltEntryAddr); |
1137 | relocateNoSym(loc: buf + 8, type: R_AARCH64_ADD_ABS_LO12_NC, val: gotPltEntryAddr); |
1138 | |
1139 | if (pacEntryKind != PEK_NoAuth) |
1140 | memcpy(dest: buf + sizeof(addrInst), |
1141 | src: pacEntryKind == PEK_AuthHint ? pacHintBr : pacBr, |
1142 | n: sizeof(pacEntryKind == PEK_AuthHint ? pacHintBr : pacBr)); |
1143 | else |
1144 | memcpy(dest: buf + sizeof(addrInst), src: stdBr, n: sizeof(stdBr)); |
1145 | if (!hasBti) |
1146 | // We didn't add the BTI c instruction so round out size with NOP. |
1147 | memcpy(dest: buf + sizeof(addrInst) + sizeof(stdBr), src: nopData, n: sizeof(nopData)); |
1148 | } |
1149 | |
1150 | template <class ELFT> |
1151 | static void |
1152 | addTaggedSymbolReferences(Ctx &ctx, InputSectionBase &sec, |
1153 | DenseMap<Symbol *, unsigned> &referenceCount) { |
1154 | assert(sec.type == SHT_AARCH64_MEMTAG_GLOBALS_STATIC); |
1155 | |
1156 | const RelsOrRelas<ELFT> rels = sec.relsOrRelas<ELFT>(); |
1157 | if (rels.areRelocsRel()) |
1158 | ErrAlways(ctx) |
1159 | << "non-RELA relocations are not allowed with memtag globals"; |
1160 | |
1161 | for (const typename ELFT::Rela &rel : rels.relas) { |
1162 | Symbol &sym = sec.file->getRelocTargetSym(rel); |
1163 | // Linker-synthesized symbols such as __executable_start may be referenced |
1164 | // as tagged in input objfiles, and we don't want them to be tagged. A |
1165 | // cheap way to exclude them is the type check, but their type is |
1166 | // STT_NOTYPE. In addition, this save us from checking untaggable symbols, |
1167 | // like functions or TLS symbols. |
1168 | if (sym.type != STT_OBJECT) |
1169 | continue; |
1170 | // STB_LOCAL symbols can't be referenced from outside the object file, and |
1171 | // thus don't need to be checked for references from other object files. |
1172 | if (sym.binding == STB_LOCAL) { |
1173 | sym.setIsTagged(true); |
1174 | continue; |
1175 | } |
1176 | ++referenceCount[&sym]; |
1177 | } |
1178 | sec.markDead(); |
1179 | } |
1180 | |
1181 | // A tagged symbol must be denoted as being tagged by all references and the |
1182 | // chosen definition. For simplicity, here, it must also be denoted as tagged |
1183 | // for all definitions. Otherwise: |
1184 | // |
1185 | // 1. A tagged definition can be used by an untagged declaration, in which case |
1186 | // the untagged access may be PC-relative, causing a tag mismatch at |
1187 | // runtime. |
1188 | // 2. An untagged definition can be used by a tagged declaration, where the |
1189 | // compiler has taken advantage of the increased alignment of the tagged |
1190 | // declaration, but the alignment at runtime is wrong, causing a fault. |
1191 | // |
1192 | // Ideally, this isn't a problem, as any TU that imports or exports tagged |
1193 | // symbols should also be built with tagging. But, to handle these cases, we |
1194 | // demote the symbol to be untagged. |
1195 | void elf::createTaggedSymbols(Ctx &ctx) { |
1196 | assert(hasMemtag(ctx)); |
1197 | |
1198 | // First, collect all symbols that are marked as tagged, and count how many |
1199 | // times they're marked as tagged. |
1200 | DenseMap<Symbol *, unsigned> taggedSymbolReferenceCount; |
1201 | for (InputFile *file : ctx.objectFiles) { |
1202 | if (file->kind() != InputFile::ObjKind) |
1203 | continue; |
1204 | for (InputSectionBase *section : file->getSections()) { |
1205 | if (!section || section->type != SHT_AARCH64_MEMTAG_GLOBALS_STATIC || |
1206 | section == &InputSection::discarded) |
1207 | continue; |
1208 | invokeELFT(addTaggedSymbolReferences, ctx, *section, |
1209 | taggedSymbolReferenceCount); |
1210 | } |
1211 | } |
1212 | |
1213 | // Now, go through all the symbols. If the number of declarations + |
1214 | // definitions to a symbol exceeds the amount of times they're marked as |
1215 | // tagged, it means we have an objfile that uses the untagged variant of the |
1216 | // symbol. |
1217 | for (InputFile *file : ctx.objectFiles) { |
1218 | if (file->kind() != InputFile::BinaryKind && |
1219 | file->kind() != InputFile::ObjKind) |
1220 | continue; |
1221 | |
1222 | for (Symbol *symbol : file->getSymbols()) { |
1223 | // See `addTaggedSymbolReferences` for more details. |
1224 | if (symbol->type != STT_OBJECT || |
1225 | symbol->binding == STB_LOCAL) |
1226 | continue; |
1227 | auto it = taggedSymbolReferenceCount.find(Val: symbol); |
1228 | if (it == taggedSymbolReferenceCount.end()) continue; |
1229 | unsigned &remainingAllowedTaggedRefs = it->second; |
1230 | if (remainingAllowedTaggedRefs == 0) { |
1231 | taggedSymbolReferenceCount.erase(I: it); |
1232 | continue; |
1233 | } |
1234 | --remainingAllowedTaggedRefs; |
1235 | } |
1236 | } |
1237 | |
1238 | // `addTaggedSymbolReferences` has already checked that we have RELA |
1239 | // relocations, the only other way to get written addends is with |
1240 | // --apply-dynamic-relocs. |
1241 | if (!taggedSymbolReferenceCount.empty() && ctx.arg.writeAddends) |
1242 | ErrAlways(ctx) << "--apply-dynamic-relocs cannot be used with MTE globals"; |
1243 | |
1244 | // Now, `taggedSymbolReferenceCount` should only contain symbols that are |
1245 | // defined as tagged exactly the same amount as it's referenced, meaning all |
1246 | // uses are tagged. |
1247 | for (auto &[symbol, remainingTaggedRefs] : taggedSymbolReferenceCount) { |
1248 | assert(remainingTaggedRefs == 0 && |
1249 | "Symbol is defined as tagged more times than it's used"); |
1250 | symbol->setIsTagged(true); |
1251 | } |
1252 | } |
1253 | |
1254 | void elf::setAArch64TargetInfo(Ctx &ctx) { |
1255 | if ((ctx.arg.andFeatures & GNU_PROPERTY_AARCH64_FEATURE_1_BTI) || |
1256 | ctx.arg.zPacPlt) |
1257 | ctx.target.reset(p: new AArch64BtiPac(ctx)); |
1258 | else |
1259 | ctx.target.reset(p: new AArch64(ctx)); |
1260 | } |
1261 |
Definitions
- getAArch64Page
- isAArch64BTILandingPad
- AArch64
- AArch64Relaxer
- getBits
- AArch64
- getRelExpr
- adjustTlsExpr
- usesOnlyLowPageBits
- getDynRel
- getImplicitAddend
- writeGotPlt
- writeIgotPlt
- writePltHeader
- writePlt
- needsThunk
- getThunkSectionSpacing
- inBranchRange
- write32AArch64Addr
- writeMaskedBits32le
- write32Imm12
- writeSMovWImm
- relocate
- relaxTlsGdToLe
- relaxTlsGdToIe
- relaxTlsIeToLe
- AArch64Relaxer
- tryRelaxAdrpAdd
- tryRelaxAdrpLdr
- needsGotForMemtag
- relocateAlloc
- AArch64BtiPac
- AArch64BtiPac
- writePltHeader
- writePlt
- addTaggedSymbolReferences
- createTaggedSymbols
Improve your Profiling and Debugging skills
Find out more