1 | //===- RISCV.cpp ----------------------------------------------------------===// |
---|---|
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | |
9 | #include "InputFiles.h" |
10 | #include "OutputSections.h" |
11 | #include "Symbols.h" |
12 | #include "SyntheticSections.h" |
13 | #include "Target.h" |
14 | #include "llvm/Support/ELFAttributes.h" |
15 | #include "llvm/Support/LEB128.h" |
16 | #include "llvm/Support/RISCVAttributeParser.h" |
17 | #include "llvm/Support/RISCVAttributes.h" |
18 | #include "llvm/Support/TimeProfiler.h" |
19 | #include "llvm/TargetParser/RISCVISAInfo.h" |
20 | |
21 | using namespace llvm; |
22 | using namespace llvm::object; |
23 | using namespace llvm::support::endian; |
24 | using namespace llvm::ELF; |
25 | using namespace lld; |
26 | using namespace lld::elf; |
27 | |
28 | namespace { |
29 | |
30 | class RISCV final : public TargetInfo { |
31 | public: |
32 | RISCV(Ctx &); |
33 | uint32_t calcEFlags() const override; |
34 | int64_t getImplicitAddend(const uint8_t *buf, RelType type) const override; |
35 | void writeGotHeader(uint8_t *buf) const override; |
36 | void writeGotPlt(uint8_t *buf, const Symbol &s) const override; |
37 | void writeIgotPlt(uint8_t *buf, const Symbol &s) const override; |
38 | void writePltHeader(uint8_t *buf) const override; |
39 | void writePlt(uint8_t *buf, const Symbol &sym, |
40 | uint64_t pltEntryAddr) const override; |
41 | RelType getDynRel(RelType type) const override; |
42 | RelExpr getRelExpr(RelType type, const Symbol &s, |
43 | const uint8_t *loc) const override; |
44 | void relocate(uint8_t *loc, const Relocation &rel, |
45 | uint64_t val) const override; |
46 | void relocateAlloc(InputSectionBase &sec, uint8_t *buf) const override; |
47 | bool relaxOnce(int pass) const override; |
48 | void finalizeRelax(int passes) const override; |
49 | }; |
50 | |
51 | } // end anonymous namespace |
52 | |
53 | // These are internal relocation numbers for GP/X0 relaxation. They aren't part |
54 | // of the psABI spec. |
55 | #define INTERNAL_R_RISCV_GPREL_I 256 |
56 | #define INTERNAL_R_RISCV_GPREL_S 257 |
57 | #define INTERNAL_R_RISCV_X0REL_I 258 |
58 | #define INTERNAL_R_RISCV_X0REL_S 259 |
59 | |
60 | const uint64_t dtpOffset = 0x800; |
61 | |
62 | namespace { |
63 | enum Op { |
64 | ADDI = 0x13, |
65 | AUIPC = 0x17, |
66 | JALR = 0x67, |
67 | LD = 0x3003, |
68 | LUI = 0x37, |
69 | LW = 0x2003, |
70 | SRLI = 0x5013, |
71 | SUB = 0x40000033, |
72 | }; |
73 | |
74 | enum Reg { |
75 | X_X0 = 0, |
76 | X_RA = 1, |
77 | X_GP = 3, |
78 | X_TP = 4, |
79 | X_T0 = 5, |
80 | X_T1 = 6, |
81 | X_T2 = 7, |
82 | X_A0 = 10, |
83 | X_T3 = 28, |
84 | }; |
85 | } // namespace |
86 | |
87 | static uint32_t hi20(uint32_t val) { return (val + 0x800) >> 12; } |
88 | static uint32_t lo12(uint32_t val) { return val & 4095; } |
89 | |
90 | static uint32_t itype(uint32_t op, uint32_t rd, uint32_t rs1, uint32_t imm) { |
91 | return op | (rd << 7) | (rs1 << 15) | (imm << 20); |
92 | } |
93 | static uint32_t rtype(uint32_t op, uint32_t rd, uint32_t rs1, uint32_t rs2) { |
94 | return op | (rd << 7) | (rs1 << 15) | (rs2 << 20); |
95 | } |
96 | static uint32_t utype(uint32_t op, uint32_t rd, uint32_t imm) { |
97 | return op | (rd << 7) | (imm << 12); |
98 | } |
99 | |
100 | // Extract bits v[begin:end], where range is inclusive, and begin must be < 63. |
101 | static uint32_t extractBits(uint64_t v, uint32_t begin, uint32_t end) { |
102 | return (v & ((1ULL << (begin + 1)) - 1)) >> end; |
103 | } |
104 | |
105 | static uint32_t setLO12_I(uint32_t insn, uint32_t imm) { |
106 | return (insn & 0xfffff) | (imm << 20); |
107 | } |
108 | static uint32_t setLO12_S(uint32_t insn, uint32_t imm) { |
109 | return (insn & 0x1fff07f) | (extractBits(v: imm, begin: 11, end: 5) << 25) | |
110 | (extractBits(v: imm, begin: 4, end: 0) << 7); |
111 | } |
112 | |
113 | RISCV::RISCV(Ctx &ctx) : TargetInfo(ctx) { |
114 | copyRel = R_RISCV_COPY; |
115 | pltRel = R_RISCV_JUMP_SLOT; |
116 | relativeRel = R_RISCV_RELATIVE; |
117 | iRelativeRel = R_RISCV_IRELATIVE; |
118 | if (ctx.arg.is64) { |
119 | symbolicRel = R_RISCV_64; |
120 | tlsModuleIndexRel = R_RISCV_TLS_DTPMOD64; |
121 | tlsOffsetRel = R_RISCV_TLS_DTPREL64; |
122 | tlsGotRel = R_RISCV_TLS_TPREL64; |
123 | } else { |
124 | symbolicRel = R_RISCV_32; |
125 | tlsModuleIndexRel = R_RISCV_TLS_DTPMOD32; |
126 | tlsOffsetRel = R_RISCV_TLS_DTPREL32; |
127 | tlsGotRel = R_RISCV_TLS_TPREL32; |
128 | } |
129 | gotRel = symbolicRel; |
130 | tlsDescRel = R_RISCV_TLSDESC; |
131 | |
132 | // .got[0] = _DYNAMIC |
133 | gotHeaderEntriesNum = 1; |
134 | |
135 | // .got.plt[0] = _dl_runtime_resolve, .got.plt[1] = link_map |
136 | gotPltHeaderEntriesNum = 2; |
137 | |
138 | pltHeaderSize = 32; |
139 | pltEntrySize = 16; |
140 | ipltEntrySize = 16; |
141 | } |
142 | |
143 | static uint32_t getEFlags(Ctx &ctx, InputFile *f) { |
144 | if (ctx.arg.is64) |
145 | return cast<ObjFile<ELF64LE>>(Val: f)->getObj().getHeader().e_flags; |
146 | return cast<ObjFile<ELF32LE>>(Val: f)->getObj().getHeader().e_flags; |
147 | } |
148 | |
149 | uint32_t RISCV::calcEFlags() const { |
150 | // If there are only binary input files (from -b binary), use a |
151 | // value of 0 for the ELF header flags. |
152 | if (ctx.objectFiles.empty()) |
153 | return 0; |
154 | |
155 | uint32_t target = getEFlags(ctx, f: ctx.objectFiles.front()); |
156 | for (InputFile *f : ctx.objectFiles) { |
157 | uint32_t eflags = getEFlags(ctx, f); |
158 | if (eflags & EF_RISCV_RVC) |
159 | target |= EF_RISCV_RVC; |
160 | |
161 | if ((eflags & EF_RISCV_FLOAT_ABI) != (target & EF_RISCV_FLOAT_ABI)) |
162 | Err(ctx) << f |
163 | << ": cannot link object files with different " |
164 | "floating-point ABI from " |
165 | << ctx.objectFiles[0]; |
166 | |
167 | if ((eflags & EF_RISCV_RVE) != (target & EF_RISCV_RVE)) |
168 | Err(ctx) << f << ": cannot link object files with different EF_RISCV_RVE"; |
169 | } |
170 | |
171 | return target; |
172 | } |
173 | |
174 | int64_t RISCV::getImplicitAddend(const uint8_t *buf, RelType type) const { |
175 | switch (type) { |
176 | default: |
177 | InternalErr(ctx, buf) << "cannot read addend for relocation "<< type; |
178 | return 0; |
179 | case R_RISCV_32: |
180 | case R_RISCV_TLS_DTPMOD32: |
181 | case R_RISCV_TLS_DTPREL32: |
182 | case R_RISCV_TLS_TPREL32: |
183 | return SignExtend64<32>(x: read32le(P: buf)); |
184 | case R_RISCV_64: |
185 | case R_RISCV_TLS_DTPMOD64: |
186 | case R_RISCV_TLS_DTPREL64: |
187 | case R_RISCV_TLS_TPREL64: |
188 | return read64le(P: buf); |
189 | case R_RISCV_RELATIVE: |
190 | case R_RISCV_IRELATIVE: |
191 | return ctx.arg.is64 ? read64le(P: buf) : read32le(P: buf); |
192 | case R_RISCV_NONE: |
193 | case R_RISCV_JUMP_SLOT: |
194 | // These relocations are defined as not having an implicit addend. |
195 | return 0; |
196 | case R_RISCV_TLSDESC: |
197 | return ctx.arg.is64 ? read64le(P: buf + 8) : read32le(P: buf + 4); |
198 | } |
199 | } |
200 | |
201 | void RISCV::writeGotHeader(uint8_t *buf) const { |
202 | if (ctx.arg.is64) |
203 | write64le(P: buf, V: ctx.mainPart->dynamic->getVA()); |
204 | else |
205 | write32le(P: buf, V: ctx.mainPart->dynamic->getVA()); |
206 | } |
207 | |
208 | void RISCV::writeGotPlt(uint8_t *buf, const Symbol &s) const { |
209 | if (ctx.arg.is64) |
210 | write64le(P: buf, V: ctx.in.plt->getVA()); |
211 | else |
212 | write32le(P: buf, V: ctx.in.plt->getVA()); |
213 | } |
214 | |
215 | void RISCV::writeIgotPlt(uint8_t *buf, const Symbol &s) const { |
216 | if (ctx.arg.writeAddends) { |
217 | if (ctx.arg.is64) |
218 | write64le(P: buf, V: s.getVA(ctx)); |
219 | else |
220 | write32le(P: buf, V: s.getVA(ctx)); |
221 | } |
222 | } |
223 | |
224 | void RISCV::writePltHeader(uint8_t *buf) const { |
225 | // 1: auipc t2, %pcrel_hi(.got.plt) |
226 | // sub t1, t1, t3 |
227 | // l[wd] t3, %pcrel_lo(1b)(t2); t3 = _dl_runtime_resolve |
228 | // addi t1, t1, -pltHeaderSize-12; t1 = &.plt[i] - &.plt[0] |
229 | // addi t0, t2, %pcrel_lo(1b) |
230 | // srli t1, t1, (rv64?1:2); t1 = &.got.plt[i] - &.got.plt[0] |
231 | // l[wd] t0, Wordsize(t0); t0 = link_map |
232 | // jr t3 |
233 | uint32_t offset = ctx.in.gotPlt->getVA() - ctx.in.plt->getVA(); |
234 | uint32_t load = ctx.arg.is64 ? LD : LW; |
235 | write32le(P: buf + 0, V: utype(op: AUIPC, rd: X_T2, imm: hi20(val: offset))); |
236 | write32le(P: buf + 4, V: rtype(op: SUB, rd: X_T1, rs1: X_T1, rs2: X_T3)); |
237 | write32le(P: buf + 8, V: itype(op: load, rd: X_T3, rs1: X_T2, imm: lo12(val: offset))); |
238 | write32le(P: buf + 12, V: itype(op: ADDI, rd: X_T1, rs1: X_T1, imm: -ctx.target->pltHeaderSize - 12)); |
239 | write32le(P: buf + 16, V: itype(op: ADDI, rd: X_T0, rs1: X_T2, imm: lo12(val: offset))); |
240 | write32le(P: buf + 20, V: itype(op: SRLI, rd: X_T1, rs1: X_T1, imm: ctx.arg.is64 ? 1 : 2)); |
241 | write32le(P: buf + 24, V: itype(op: load, rd: X_T0, rs1: X_T0, imm: ctx.arg.wordsize)); |
242 | write32le(P: buf + 28, V: itype(op: JALR, rd: 0, rs1: X_T3, imm: 0)); |
243 | } |
244 | |
245 | void RISCV::writePlt(uint8_t *buf, const Symbol &sym, |
246 | uint64_t pltEntryAddr) const { |
247 | // 1: auipc t3, %pcrel_hi(f@.got.plt) |
248 | // l[wd] t3, %pcrel_lo(1b)(t3) |
249 | // jalr t1, t3 |
250 | // nop |
251 | uint32_t offset = sym.getGotPltVA(ctx) - pltEntryAddr; |
252 | write32le(P: buf + 0, V: utype(op: AUIPC, rd: X_T3, imm: hi20(val: offset))); |
253 | write32le(P: buf + 4, V: itype(op: ctx.arg.is64 ? LD : LW, rd: X_T3, rs1: X_T3, imm: lo12(val: offset))); |
254 | write32le(P: buf + 8, V: itype(op: JALR, rd: X_T1, rs1: X_T3, imm: 0)); |
255 | write32le(P: buf + 12, V: itype(op: ADDI, rd: 0, rs1: 0, imm: 0)); |
256 | } |
257 | |
258 | RelType RISCV::getDynRel(RelType type) const { |
259 | return type == ctx.target->symbolicRel ? type |
260 | : static_cast<RelType>(R_RISCV_NONE); |
261 | } |
262 | |
263 | RelExpr RISCV::getRelExpr(const RelType type, const Symbol &s, |
264 | const uint8_t *loc) const { |
265 | switch (type) { |
266 | case R_RISCV_NONE: |
267 | return R_NONE; |
268 | case R_RISCV_32: |
269 | case R_RISCV_64: |
270 | case R_RISCV_HI20: |
271 | case R_RISCV_LO12_I: |
272 | case R_RISCV_LO12_S: |
273 | return R_ABS; |
274 | case R_RISCV_ADD8: |
275 | case R_RISCV_ADD16: |
276 | case R_RISCV_ADD32: |
277 | case R_RISCV_ADD64: |
278 | case R_RISCV_SET6: |
279 | case R_RISCV_SET8: |
280 | case R_RISCV_SET16: |
281 | case R_RISCV_SET32: |
282 | case R_RISCV_SUB6: |
283 | case R_RISCV_SUB8: |
284 | case R_RISCV_SUB16: |
285 | case R_RISCV_SUB32: |
286 | case R_RISCV_SUB64: |
287 | return RE_RISCV_ADD; |
288 | case R_RISCV_JAL: |
289 | case R_RISCV_BRANCH: |
290 | case R_RISCV_PCREL_HI20: |
291 | case R_RISCV_RVC_BRANCH: |
292 | case R_RISCV_RVC_JUMP: |
293 | case R_RISCV_32_PCREL: |
294 | return R_PC; |
295 | case R_RISCV_CALL: |
296 | case R_RISCV_CALL_PLT: |
297 | case R_RISCV_PLT32: |
298 | return R_PLT_PC; |
299 | case R_RISCV_GOT_HI20: |
300 | case R_RISCV_GOT32_PCREL: |
301 | return R_GOT_PC; |
302 | case R_RISCV_PCREL_LO12_I: |
303 | case R_RISCV_PCREL_LO12_S: |
304 | return RE_RISCV_PC_INDIRECT; |
305 | case R_RISCV_TLSDESC_HI20: |
306 | case R_RISCV_TLSDESC_LOAD_LO12: |
307 | case R_RISCV_TLSDESC_ADD_LO12: |
308 | return R_TLSDESC_PC; |
309 | case R_RISCV_TLSDESC_CALL: |
310 | return R_TLSDESC_CALL; |
311 | case R_RISCV_TLS_GD_HI20: |
312 | return R_TLSGD_PC; |
313 | case R_RISCV_TLS_GOT_HI20: |
314 | return R_GOT_PC; |
315 | case R_RISCV_TPREL_HI20: |
316 | case R_RISCV_TPREL_LO12_I: |
317 | case R_RISCV_TPREL_LO12_S: |
318 | return R_TPREL; |
319 | case R_RISCV_ALIGN: |
320 | return R_RELAX_HINT; |
321 | case R_RISCV_TPREL_ADD: |
322 | case R_RISCV_RELAX: |
323 | return ctx.arg.relax ? R_RELAX_HINT : R_NONE; |
324 | case R_RISCV_SET_ULEB128: |
325 | case R_RISCV_SUB_ULEB128: |
326 | return RE_RISCV_LEB128; |
327 | default: |
328 | Err(ctx) << getErrorLoc(ctx, loc) << "unknown relocation ("<< type.v |
329 | << ") against symbol "<< &s; |
330 | return R_NONE; |
331 | } |
332 | } |
333 | |
334 | void RISCV::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const { |
335 | const unsigned bits = ctx.arg.wordsize * 8; |
336 | |
337 | switch (rel.type) { |
338 | case R_RISCV_32: |
339 | write32le(P: loc, V: val); |
340 | return; |
341 | case R_RISCV_64: |
342 | write64le(P: loc, V: val); |
343 | return; |
344 | |
345 | case R_RISCV_RVC_BRANCH: { |
346 | checkInt(ctx, loc, v: val, n: 9, rel); |
347 | checkAlignment(ctx, loc, v: val, n: 2, rel); |
348 | uint16_t insn = read16le(P: loc) & 0xE383; |
349 | uint16_t imm8 = extractBits(v: val, begin: 8, end: 8) << 12; |
350 | uint16_t imm4_3 = extractBits(v: val, begin: 4, end: 3) << 10; |
351 | uint16_t imm7_6 = extractBits(v: val, begin: 7, end: 6) << 5; |
352 | uint16_t imm2_1 = extractBits(v: val, begin: 2, end: 1) << 3; |
353 | uint16_t imm5 = extractBits(v: val, begin: 5, end: 5) << 2; |
354 | insn |= imm8 | imm4_3 | imm7_6 | imm2_1 | imm5; |
355 | |
356 | write16le(P: loc, V: insn); |
357 | return; |
358 | } |
359 | |
360 | case R_RISCV_RVC_JUMP: { |
361 | checkInt(ctx, loc, v: val, n: 12, rel); |
362 | checkAlignment(ctx, loc, v: val, n: 2, rel); |
363 | uint16_t insn = read16le(P: loc) & 0xE003; |
364 | uint16_t imm11 = extractBits(v: val, begin: 11, end: 11) << 12; |
365 | uint16_t imm4 = extractBits(v: val, begin: 4, end: 4) << 11; |
366 | uint16_t imm9_8 = extractBits(v: val, begin: 9, end: 8) << 9; |
367 | uint16_t imm10 = extractBits(v: val, begin: 10, end: 10) << 8; |
368 | uint16_t imm6 = extractBits(v: val, begin: 6, end: 6) << 7; |
369 | uint16_t imm7 = extractBits(v: val, begin: 7, end: 7) << 6; |
370 | uint16_t imm3_1 = extractBits(v: val, begin: 3, end: 1) << 3; |
371 | uint16_t imm5 = extractBits(v: val, begin: 5, end: 5) << 2; |
372 | insn |= imm11 | imm4 | imm9_8 | imm10 | imm6 | imm7 | imm3_1 | imm5; |
373 | |
374 | write16le(P: loc, V: insn); |
375 | return; |
376 | } |
377 | |
378 | case R_RISCV_JAL: { |
379 | checkInt(ctx, loc, v: val, n: 21, rel); |
380 | checkAlignment(ctx, loc, v: val, n: 2, rel); |
381 | |
382 | uint32_t insn = read32le(P: loc) & 0xFFF; |
383 | uint32_t imm20 = extractBits(v: val, begin: 20, end: 20) << 31; |
384 | uint32_t imm10_1 = extractBits(v: val, begin: 10, end: 1) << 21; |
385 | uint32_t imm11 = extractBits(v: val, begin: 11, end: 11) << 20; |
386 | uint32_t imm19_12 = extractBits(v: val, begin: 19, end: 12) << 12; |
387 | insn |= imm20 | imm10_1 | imm11 | imm19_12; |
388 | |
389 | write32le(P: loc, V: insn); |
390 | return; |
391 | } |
392 | |
393 | case R_RISCV_BRANCH: { |
394 | checkInt(ctx, loc, v: val, n: 13, rel); |
395 | checkAlignment(ctx, loc, v: val, n: 2, rel); |
396 | |
397 | uint32_t insn = read32le(P: loc) & 0x1FFF07F; |
398 | uint32_t imm12 = extractBits(v: val, begin: 12, end: 12) << 31; |
399 | uint32_t imm10_5 = extractBits(v: val, begin: 10, end: 5) << 25; |
400 | uint32_t imm4_1 = extractBits(v: val, begin: 4, end: 1) << 8; |
401 | uint32_t imm11 = extractBits(v: val, begin: 11, end: 11) << 7; |
402 | insn |= imm12 | imm10_5 | imm4_1 | imm11; |
403 | |
404 | write32le(P: loc, V: insn); |
405 | return; |
406 | } |
407 | |
408 | // auipc + jalr pair |
409 | case R_RISCV_CALL: |
410 | case R_RISCV_CALL_PLT: { |
411 | int64_t hi = SignExtend64(X: val + 0x800, B: bits) >> 12; |
412 | checkInt(ctx, loc, v: hi, n: 20, rel); |
413 | if (isInt<20>(x: hi)) { |
414 | relocateNoSym(loc, type: R_RISCV_PCREL_HI20, val); |
415 | relocateNoSym(loc: loc + 4, type: R_RISCV_PCREL_LO12_I, val); |
416 | } |
417 | return; |
418 | } |
419 | |
420 | case R_RISCV_GOT_HI20: |
421 | case R_RISCV_PCREL_HI20: |
422 | case R_RISCV_TLSDESC_HI20: |
423 | case R_RISCV_TLS_GD_HI20: |
424 | case R_RISCV_TLS_GOT_HI20: |
425 | case R_RISCV_TPREL_HI20: |
426 | case R_RISCV_HI20: { |
427 | uint64_t hi = val + 0x800; |
428 | checkInt(ctx, loc, v: SignExtend64(X: hi, B: bits) >> 12, n: 20, rel); |
429 | write32le(P: loc, V: (read32le(P: loc) & 0xFFF) | (hi & 0xFFFFF000)); |
430 | return; |
431 | } |
432 | |
433 | case R_RISCV_PCREL_LO12_I: |
434 | case R_RISCV_TLSDESC_LOAD_LO12: |
435 | case R_RISCV_TLSDESC_ADD_LO12: |
436 | case R_RISCV_TPREL_LO12_I: |
437 | case R_RISCV_LO12_I: { |
438 | uint64_t hi = (val + 0x800) >> 12; |
439 | uint64_t lo = val - (hi << 12); |
440 | write32le(P: loc, V: setLO12_I(insn: read32le(P: loc), imm: lo & 0xfff)); |
441 | return; |
442 | } |
443 | |
444 | case R_RISCV_PCREL_LO12_S: |
445 | case R_RISCV_TPREL_LO12_S: |
446 | case R_RISCV_LO12_S: { |
447 | uint64_t hi = (val + 0x800) >> 12; |
448 | uint64_t lo = val - (hi << 12); |
449 | write32le(P: loc, V: setLO12_S(insn: read32le(P: loc), imm: lo)); |
450 | return; |
451 | } |
452 | |
453 | case INTERNAL_R_RISCV_X0REL_I: |
454 | case INTERNAL_R_RISCV_X0REL_S: { |
455 | checkInt(ctx, loc, v: val, n: 12, rel); |
456 | uint32_t insn = (read32le(P: loc) & ~(31 << 15)) | (X_X0 << 15); |
457 | if (rel.type == INTERNAL_R_RISCV_X0REL_I) |
458 | insn = setLO12_I(insn, imm: val); |
459 | else |
460 | insn = setLO12_S(insn, imm: val); |
461 | write32le(P: loc, V: insn); |
462 | return; |
463 | } |
464 | |
465 | case INTERNAL_R_RISCV_GPREL_I: |
466 | case INTERNAL_R_RISCV_GPREL_S: { |
467 | Defined *gp = ctx.sym.riscvGlobalPointer; |
468 | int64_t displace = SignExtend64(X: val - gp->getVA(ctx), B: bits); |
469 | checkInt(ctx, loc, v: displace, n: 12, rel); |
470 | uint32_t insn = (read32le(P: loc) & ~(31 << 15)) | (X_GP << 15); |
471 | if (rel.type == INTERNAL_R_RISCV_GPREL_I) |
472 | insn = setLO12_I(insn, imm: displace); |
473 | else |
474 | insn = setLO12_S(insn, imm: displace); |
475 | write32le(P: loc, V: insn); |
476 | return; |
477 | } |
478 | |
479 | case R_RISCV_ADD8: |
480 | *loc += val; |
481 | return; |
482 | case R_RISCV_ADD16: |
483 | write16le(P: loc, V: read16le(P: loc) + val); |
484 | return; |
485 | case R_RISCV_ADD32: |
486 | write32le(P: loc, V: read32le(P: loc) + val); |
487 | return; |
488 | case R_RISCV_ADD64: |
489 | write64le(P: loc, V: read64le(P: loc) + val); |
490 | return; |
491 | case R_RISCV_SUB6: |
492 | *loc = (*loc & 0xc0) | (((*loc & 0x3f) - val) & 0x3f); |
493 | return; |
494 | case R_RISCV_SUB8: |
495 | *loc -= val; |
496 | return; |
497 | case R_RISCV_SUB16: |
498 | write16le(P: loc, V: read16le(P: loc) - val); |
499 | return; |
500 | case R_RISCV_SUB32: |
501 | write32le(P: loc, V: read32le(P: loc) - val); |
502 | return; |
503 | case R_RISCV_SUB64: |
504 | write64le(P: loc, V: read64le(P: loc) - val); |
505 | return; |
506 | case R_RISCV_SET6: |
507 | *loc = (*loc & 0xc0) | (val & 0x3f); |
508 | return; |
509 | case R_RISCV_SET8: |
510 | *loc = val; |
511 | return; |
512 | case R_RISCV_SET16: |
513 | write16le(P: loc, V: val); |
514 | return; |
515 | case R_RISCV_SET32: |
516 | case R_RISCV_32_PCREL: |
517 | case R_RISCV_PLT32: |
518 | case R_RISCV_GOT32_PCREL: |
519 | checkInt(ctx, loc, v: val, n: 32, rel); |
520 | write32le(P: loc, V: val); |
521 | return; |
522 | |
523 | case R_RISCV_TLS_DTPREL32: |
524 | write32le(P: loc, V: val - dtpOffset); |
525 | break; |
526 | case R_RISCV_TLS_DTPREL64: |
527 | write64le(P: loc, V: val - dtpOffset); |
528 | break; |
529 | |
530 | case R_RISCV_RELAX: |
531 | return; |
532 | case R_RISCV_TLSDESC: |
533 | // The addend is stored in the second word. |
534 | if (ctx.arg.is64) |
535 | write64le(P: loc + 8, V: val); |
536 | else |
537 | write32le(P: loc + 4, V: val); |
538 | break; |
539 | default: |
540 | llvm_unreachable("unknown relocation"); |
541 | } |
542 | } |
543 | |
544 | static bool relaxable(ArrayRef<Relocation> relocs, size_t i) { |
545 | return i + 1 != relocs.size() && relocs[i + 1].type == R_RISCV_RELAX; |
546 | } |
547 | |
548 | static void tlsdescToIe(Ctx &ctx, uint8_t *loc, const Relocation &rel, |
549 | uint64_t val) { |
550 | switch (rel.type) { |
551 | case R_RISCV_TLSDESC_HI20: |
552 | case R_RISCV_TLSDESC_LOAD_LO12: |
553 | write32le(P: loc, V: 0x00000013); // nop |
554 | break; |
555 | case R_RISCV_TLSDESC_ADD_LO12: |
556 | write32le(P: loc, V: utype(op: AUIPC, rd: X_A0, imm: hi20(val))); // auipc a0,<hi20> |
557 | break; |
558 | case R_RISCV_TLSDESC_CALL: |
559 | if (ctx.arg.is64) |
560 | write32le(P: loc, V: itype(op: LD, rd: X_A0, rs1: X_A0, imm: lo12(val))); // ld a0,<lo12>(a0) |
561 | else |
562 | write32le(P: loc, V: itype(op: LW, rd: X_A0, rs1: X_A0, imm: lo12(val))); // lw a0,<lo12>(a0) |
563 | break; |
564 | default: |
565 | llvm_unreachable("unsupported relocation for TLSDESC to IE"); |
566 | } |
567 | } |
568 | |
569 | static void tlsdescToLe(uint8_t *loc, const Relocation &rel, uint64_t val) { |
570 | switch (rel.type) { |
571 | case R_RISCV_TLSDESC_HI20: |
572 | case R_RISCV_TLSDESC_LOAD_LO12: |
573 | write32le(P: loc, V: 0x00000013); // nop |
574 | return; |
575 | case R_RISCV_TLSDESC_ADD_LO12: |
576 | if (isInt<12>(x: val)) |
577 | write32le(P: loc, V: 0x00000013); // nop |
578 | else |
579 | write32le(P: loc, V: utype(op: LUI, rd: X_A0, imm: hi20(val))); // lui a0,<hi20> |
580 | return; |
581 | case R_RISCV_TLSDESC_CALL: |
582 | if (isInt<12>(x: val)) |
583 | write32le(P: loc, V: itype(op: ADDI, rd: X_A0, rs1: 0, imm: val)); // addi a0,zero,<lo12> |
584 | else |
585 | write32le(P: loc, V: itype(op: ADDI, rd: X_A0, rs1: X_A0, imm: lo12(val))); // addi a0,a0,<lo12> |
586 | return; |
587 | default: |
588 | llvm_unreachable("unsupported relocation for TLSDESC to LE"); |
589 | } |
590 | } |
591 | |
592 | void RISCV::relocateAlloc(InputSectionBase &sec, uint8_t *buf) const { |
593 | uint64_t secAddr = sec.getOutputSection()->addr; |
594 | if (auto *s = dyn_cast<InputSection>(Val: &sec)) |
595 | secAddr += s->outSecOff; |
596 | else if (auto *ehIn = dyn_cast<EhInputSection>(Val: &sec)) |
597 | secAddr += ehIn->getParent()->outSecOff; |
598 | uint64_t tlsdescVal = 0; |
599 | bool tlsdescRelax = false, isToLe = false; |
600 | const ArrayRef<Relocation> relocs = sec.relocs(); |
601 | for (size_t i = 0, size = relocs.size(); i != size; ++i) { |
602 | const Relocation &rel = relocs[i]; |
603 | uint8_t *loc = buf + rel.offset; |
604 | uint64_t val = sec.getRelocTargetVA(ctx, r: rel, p: secAddr + rel.offset); |
605 | |
606 | switch (rel.expr) { |
607 | case R_RELAX_HINT: |
608 | continue; |
609 | case R_TLSDESC_PC: |
610 | // For R_RISCV_TLSDESC_HI20, store &got(sym)-PC to be used by the |
611 | // following two instructions L[DW] and ADDI. |
612 | if (rel.type == R_RISCV_TLSDESC_HI20) |
613 | tlsdescVal = val; |
614 | else |
615 | val = tlsdescVal; |
616 | break; |
617 | case R_RELAX_TLS_GD_TO_IE: |
618 | // Only R_RISCV_TLSDESC_HI20 reaches here. tlsdescVal will be finalized |
619 | // after we see R_RISCV_TLSDESC_ADD_LO12 in the R_RELAX_TLS_GD_TO_LE case. |
620 | // The net effect is that tlsdescVal will be smaller than `val` to take |
621 | // into account of NOP instructions (in the absence of R_RISCV_RELAX) |
622 | // before AUIPC. |
623 | tlsdescVal = val + rel.offset; |
624 | isToLe = false; |
625 | tlsdescRelax = relaxable(relocs, i); |
626 | if (!tlsdescRelax) |
627 | tlsdescToIe(ctx, loc, rel, val); |
628 | continue; |
629 | case R_RELAX_TLS_GD_TO_LE: |
630 | // See the comment in handleTlsRelocation. For TLSDESC=>IE, |
631 | // R_RISCV_TLSDESC_{LOAD_LO12,ADD_LO12,CALL} also reach here. If isToLe is |
632 | // false, this is actually TLSDESC=>IE optimization. |
633 | if (rel.type == R_RISCV_TLSDESC_HI20) { |
634 | tlsdescVal = val; |
635 | isToLe = true; |
636 | tlsdescRelax = relaxable(relocs, i); |
637 | } else { |
638 | if (!isToLe && rel.type == R_RISCV_TLSDESC_ADD_LO12) |
639 | tlsdescVal -= rel.offset; |
640 | val = tlsdescVal; |
641 | } |
642 | // When NOP conversion is eligible and relaxation applies, don't write a |
643 | // NOP in case an unrelated instruction follows the current instruction. |
644 | if (tlsdescRelax && |
645 | (rel.type == R_RISCV_TLSDESC_HI20 || |
646 | rel.type == R_RISCV_TLSDESC_LOAD_LO12 || |
647 | (rel.type == R_RISCV_TLSDESC_ADD_LO12 && isToLe && !hi20(val)))) |
648 | continue; |
649 | if (isToLe) |
650 | tlsdescToLe(loc, rel, val); |
651 | else |
652 | tlsdescToIe(ctx, loc, rel, val); |
653 | continue; |
654 | case RE_RISCV_LEB128: |
655 | if (i + 1 < size) { |
656 | const Relocation &rel1 = relocs[i + 1]; |
657 | if (rel.type == R_RISCV_SET_ULEB128 && |
658 | rel1.type == R_RISCV_SUB_ULEB128 && rel.offset == rel1.offset) { |
659 | auto val = rel.sym->getVA(ctx, addend: rel.addend) - |
660 | rel1.sym->getVA(ctx, addend: rel1.addend); |
661 | if (overwriteULEB128(bufLoc: loc, val) >= 0x80) |
662 | Err(ctx) << sec.getLocation(offset: rel.offset) << ": ULEB128 value "<< val |
663 | << " exceeds available space; references '"<< rel.sym |
664 | << "'"; |
665 | ++i; |
666 | continue; |
667 | } |
668 | } |
669 | Err(ctx) << sec.getLocation(offset: rel.offset) |
670 | << ": R_RISCV_SET_ULEB128 not paired with R_RISCV_SUB_SET128"; |
671 | return; |
672 | default: |
673 | break; |
674 | } |
675 | relocate(loc, rel, val); |
676 | } |
677 | } |
678 | |
679 | void elf::initSymbolAnchors(Ctx &ctx) { |
680 | SmallVector<InputSection *, 0> storage; |
681 | for (OutputSection *osec : ctx.outputSections) { |
682 | if (!(osec->flags & SHF_EXECINSTR)) |
683 | continue; |
684 | for (InputSection *sec : getInputSections(os: *osec, storage)) { |
685 | sec->relaxAux = make<RelaxAux>(); |
686 | if (sec->relocs().size()) { |
687 | sec->relaxAux->relocDeltas = |
688 | std::make_unique<uint32_t[]>(num: sec->relocs().size()); |
689 | sec->relaxAux->relocTypes = |
690 | std::make_unique<RelType[]>(num: sec->relocs().size()); |
691 | } |
692 | } |
693 | } |
694 | // Store anchors (st_value and st_value+st_size) for symbols relative to text |
695 | // sections. |
696 | // |
697 | // For a defined symbol foo, we may have `d->file != file` with --wrap=foo. |
698 | // We should process foo, as the defining object file's symbol table may not |
699 | // contain foo after redirectSymbols changed the foo entry to __wrap_foo. To |
700 | // avoid adding a Defined that is undefined in one object file, use |
701 | // `!d->scriptDefined` to exclude symbols that are definitely not wrapped. |
702 | // |
703 | // `relaxAux->anchors` may contain duplicate symbols, but that is fine. |
704 | for (InputFile *file : ctx.objectFiles) |
705 | for (Symbol *sym : file->getSymbols()) { |
706 | auto *d = dyn_cast<Defined>(Val: sym); |
707 | if (!d || (d->file != file && !d->scriptDefined)) |
708 | continue; |
709 | if (auto *sec = dyn_cast_or_null<InputSection>(Val: d->section)) |
710 | if (sec->flags & SHF_EXECINSTR && sec->relaxAux) { |
711 | // If sec is discarded, relaxAux will be nullptr. |
712 | sec->relaxAux->anchors.push_back(Elt: {.offset: d->value, .d: d, .end: false}); |
713 | sec->relaxAux->anchors.push_back(Elt: {.offset: d->value + d->size, .d: d, .end: true}); |
714 | } |
715 | } |
716 | // Sort anchors by offset so that we can find the closest relocation |
717 | // efficiently. For a zero size symbol, ensure that its start anchor precedes |
718 | // its end anchor. For two symbols with anchors at the same offset, their |
719 | // order does not matter. |
720 | for (OutputSection *osec : ctx.outputSections) { |
721 | if (!(osec->flags & SHF_EXECINSTR)) |
722 | continue; |
723 | for (InputSection *sec : getInputSections(os: *osec, storage)) { |
724 | llvm::sort(C&: sec->relaxAux->anchors, Comp: [](auto &a, auto &b) { |
725 | return std::make_pair(a.offset, a.end) < |
726 | std::make_pair(b.offset, b.end); |
727 | }); |
728 | } |
729 | } |
730 | } |
731 | |
732 | // Relax R_RISCV_CALL/R_RISCV_CALL_PLT auipc+jalr to c.j, c.jal, or jal. |
733 | static void relaxCall(Ctx &ctx, const InputSection &sec, size_t i, uint64_t loc, |
734 | Relocation &r, uint32_t &remove) { |
735 | const bool rvc = getEFlags(ctx, f: sec.file) & EF_RISCV_RVC; |
736 | const Symbol &sym = *r.sym; |
737 | const uint64_t insnPair = read64le(P: sec.content().data() + r.offset); |
738 | const uint32_t rd = extractBits(v: insnPair, begin: 32 + 11, end: 32 + 7); |
739 | const uint64_t dest = |
740 | (r.expr == R_PLT_PC ? sym.getPltVA(ctx) : sym.getVA(ctx)) + r.addend; |
741 | const int64_t displace = dest - loc; |
742 | |
743 | // When the caller specifies the old value of `remove`, disallow its |
744 | // increment. |
745 | if (remove >= 6 && rvc && isInt<12>(x: displace) && rd == 0) { |
746 | sec.relaxAux->relocTypes[i] = R_RISCV_RVC_JUMP; |
747 | sec.relaxAux->writes.push_back(Elt: 0xa001); // c.j |
748 | remove = 6; |
749 | } else if (remove >= 6 && rvc && isInt<12>(x: displace) && rd == X_RA && |
750 | !ctx.arg.is64) { // RV32C only |
751 | sec.relaxAux->relocTypes[i] = R_RISCV_RVC_JUMP; |
752 | sec.relaxAux->writes.push_back(Elt: 0x2001); // c.jal |
753 | remove = 6; |
754 | } else if (remove >= 4 && isInt<21>(x: displace)) { |
755 | sec.relaxAux->relocTypes[i] = R_RISCV_JAL; |
756 | sec.relaxAux->writes.push_back(Elt: 0x6f | rd << 7); // jal |
757 | remove = 4; |
758 | } else { |
759 | remove = 0; |
760 | } |
761 | } |
762 | |
763 | // Relax local-exec TLS when hi20 is zero. |
764 | static void relaxTlsLe(Ctx &ctx, const InputSection &sec, size_t i, |
765 | uint64_t loc, Relocation &r, uint32_t &remove) { |
766 | uint64_t val = r.sym->getVA(ctx, addend: r.addend); |
767 | if (hi20(val) != 0) |
768 | return; |
769 | uint32_t insn = read32le(P: sec.content().data() + r.offset); |
770 | switch (r.type) { |
771 | case R_RISCV_TPREL_HI20: |
772 | case R_RISCV_TPREL_ADD: |
773 | // Remove lui rd, %tprel_hi(x) and add rd, rd, tp, %tprel_add(x). |
774 | sec.relaxAux->relocTypes[i] = R_RISCV_RELAX; |
775 | remove = 4; |
776 | break; |
777 | case R_RISCV_TPREL_LO12_I: |
778 | // addi rd, rd, %tprel_lo(x) => addi rd, tp, st_value(x) |
779 | sec.relaxAux->relocTypes[i] = R_RISCV_32; |
780 | insn = (insn & ~(31 << 15)) | (X_TP << 15); |
781 | sec.relaxAux->writes.push_back(Elt: setLO12_I(insn, imm: val)); |
782 | break; |
783 | case R_RISCV_TPREL_LO12_S: |
784 | // sw rs, %tprel_lo(x)(rd) => sw rs, st_value(x)(rd) |
785 | sec.relaxAux->relocTypes[i] = R_RISCV_32; |
786 | insn = (insn & ~(31 << 15)) | (X_TP << 15); |
787 | sec.relaxAux->writes.push_back(Elt: setLO12_S(insn, imm: val)); |
788 | break; |
789 | } |
790 | } |
791 | |
792 | static void relaxHi20Lo12(Ctx &ctx, const InputSection &sec, size_t i, |
793 | uint64_t loc, Relocation &r, uint32_t &remove) { |
794 | |
795 | // Fold into use of x0+offset |
796 | if (isInt<12>(x: r.sym->getVA(ctx, addend: r.addend))) { |
797 | switch (r.type) { |
798 | case R_RISCV_HI20: |
799 | // Remove lui rd, %hi20(x). |
800 | sec.relaxAux->relocTypes[i] = R_RISCV_RELAX; |
801 | remove = 4; |
802 | break; |
803 | case R_RISCV_LO12_I: |
804 | sec.relaxAux->relocTypes[i] = INTERNAL_R_RISCV_X0REL_I; |
805 | break; |
806 | case R_RISCV_LO12_S: |
807 | sec.relaxAux->relocTypes[i] = INTERNAL_R_RISCV_X0REL_S; |
808 | break; |
809 | } |
810 | return; |
811 | } |
812 | |
813 | const Defined *gp = ctx.sym.riscvGlobalPointer; |
814 | if (!gp) |
815 | return; |
816 | |
817 | if (!isInt<12>(x: r.sym->getVA(ctx, addend: r.addend) - gp->getVA(ctx))) |
818 | return; |
819 | |
820 | switch (r.type) { |
821 | case R_RISCV_HI20: |
822 | // Remove lui rd, %hi20(x). |
823 | sec.relaxAux->relocTypes[i] = R_RISCV_RELAX; |
824 | remove = 4; |
825 | break; |
826 | case R_RISCV_LO12_I: |
827 | sec.relaxAux->relocTypes[i] = INTERNAL_R_RISCV_GPREL_I; |
828 | break; |
829 | case R_RISCV_LO12_S: |
830 | sec.relaxAux->relocTypes[i] = INTERNAL_R_RISCV_GPREL_S; |
831 | break; |
832 | } |
833 | } |
834 | |
835 | static bool relax(Ctx &ctx, int pass, InputSection &sec) { |
836 | const uint64_t secAddr = sec.getVA(); |
837 | const MutableArrayRef<Relocation> relocs = sec.relocs(); |
838 | auto &aux = *sec.relaxAux; |
839 | bool changed = false; |
840 | ArrayRef<SymbolAnchor> sa = ArrayRef(aux.anchors); |
841 | uint64_t delta = 0; |
842 | bool tlsdescRelax = false, toLeShortForm = false; |
843 | |
844 | std::fill_n(first: aux.relocTypes.get(), n: relocs.size(), value: R_RISCV_NONE); |
845 | aux.writes.clear(); |
846 | for (auto [i, r] : llvm::enumerate(First: relocs)) { |
847 | const uint64_t loc = secAddr + r.offset - delta; |
848 | uint32_t &cur = aux.relocDeltas[i], remove = 0; |
849 | switch (r.type) { |
850 | case R_RISCV_ALIGN: { |
851 | const uint64_t nextLoc = loc + r.addend; |
852 | const uint64_t align = PowerOf2Ceil(A: r.addend + 2); |
853 | // All bytes beyond the alignment boundary should be removed. |
854 | remove = nextLoc - ((loc + align - 1) & -align); |
855 | // If we can't satisfy this alignment, we've found a bad input. |
856 | if (LLVM_UNLIKELY(static_cast<int32_t>(remove) < 0)) { |
857 | Err(ctx) << getErrorLoc(ctx, loc: (const uint8_t *)loc) |
858 | << "insufficient padding bytes for "<< r.type << ": " |
859 | << r.addend |
860 | << " bytes available " |
861 | "for requested alignment of " |
862 | << align << " bytes"; |
863 | remove = 0; |
864 | } |
865 | break; |
866 | } |
867 | case R_RISCV_CALL: |
868 | case R_RISCV_CALL_PLT: |
869 | // Prevent oscillation between states by disallowing the increment of |
870 | // `remove` after a few passes. The previous `remove` value is |
871 | // `cur-delta`. |
872 | if (relaxable(relocs, i)) { |
873 | remove = pass < 4 ? 6 : cur - delta; |
874 | relaxCall(ctx, sec, i, loc, r, remove); |
875 | } |
876 | break; |
877 | case R_RISCV_TPREL_HI20: |
878 | case R_RISCV_TPREL_ADD: |
879 | case R_RISCV_TPREL_LO12_I: |
880 | case R_RISCV_TPREL_LO12_S: |
881 | if (relaxable(relocs, i)) |
882 | relaxTlsLe(ctx, sec, i, loc, r, remove); |
883 | break; |
884 | case R_RISCV_HI20: |
885 | case R_RISCV_LO12_I: |
886 | case R_RISCV_LO12_S: |
887 | if (relaxable(relocs, i)) |
888 | relaxHi20Lo12(ctx, sec, i, loc, r, remove); |
889 | break; |
890 | case R_RISCV_TLSDESC_HI20: |
891 | // For TLSDESC=>LE, we can use the short form if hi20 is zero. |
892 | tlsdescRelax = relaxable(relocs, i); |
893 | toLeShortForm = tlsdescRelax && r.expr == R_RELAX_TLS_GD_TO_LE && |
894 | !hi20(val: r.sym->getVA(ctx, addend: r.addend)); |
895 | [[fallthrough]]; |
896 | case R_RISCV_TLSDESC_LOAD_LO12: |
897 | // For TLSDESC=>LE/IE, AUIPC and L[DW] are removed if relaxable. |
898 | if (tlsdescRelax && r.expr != R_TLSDESC_PC) |
899 | remove = 4; |
900 | break; |
901 | case R_RISCV_TLSDESC_ADD_LO12: |
902 | if (toLeShortForm) |
903 | remove = 4; |
904 | break; |
905 | } |
906 | |
907 | // For all anchors whose offsets are <= r.offset, they are preceded by |
908 | // the previous relocation whose `relocDeltas` value equals `delta`. |
909 | // Decrease their st_value and update their st_size. |
910 | for (; sa.size() && sa[0].offset <= r.offset; sa = sa.slice(N: 1)) { |
911 | if (sa[0].end) |
912 | sa[0].d->size = sa[0].offset - delta - sa[0].d->value; |
913 | else |
914 | sa[0].d->value = sa[0].offset - delta; |
915 | } |
916 | delta += remove; |
917 | if (delta != cur) { |
918 | cur = delta; |
919 | changed = true; |
920 | } |
921 | } |
922 | |
923 | for (const SymbolAnchor &a : sa) { |
924 | if (a.end) |
925 | a.d->size = a.offset - delta - a.d->value; |
926 | else |
927 | a.d->value = a.offset - delta; |
928 | } |
929 | // Inform assignAddresses that the size has changed. |
930 | if (!isUInt<32>(x: delta)) |
931 | Err(ctx) << "section size decrease is too large: "<< delta; |
932 | sec.bytesDropped = delta; |
933 | return changed; |
934 | } |
935 | |
936 | // When relaxing just R_RISCV_ALIGN, relocDeltas is usually changed only once in |
937 | // the absence of a linker script. For call and load/store R_RISCV_RELAX, code |
938 | // shrinkage may reduce displacement and make more relocations eligible for |
939 | // relaxation. Code shrinkage may increase displacement to a call/load/store |
940 | // target at a higher fixed address, invalidating an earlier relaxation. Any |
941 | // change in section sizes can have cascading effect and require another |
942 | // relaxation pass. |
943 | bool RISCV::relaxOnce(int pass) const { |
944 | llvm::TimeTraceScope timeScope("RISC-V relaxOnce"); |
945 | if (ctx.arg.relocatable) |
946 | return false; |
947 | |
948 | if (pass == 0) |
949 | initSymbolAnchors(ctx); |
950 | |
951 | SmallVector<InputSection *, 0> storage; |
952 | bool changed = false; |
953 | for (OutputSection *osec : ctx.outputSections) { |
954 | if (!(osec->flags & SHF_EXECINSTR)) |
955 | continue; |
956 | for (InputSection *sec : getInputSections(os: *osec, storage)) |
957 | changed |= relax(ctx, pass, sec&: *sec); |
958 | } |
959 | return changed; |
960 | } |
961 | |
962 | void RISCV::finalizeRelax(int passes) const { |
963 | llvm::TimeTraceScope timeScope("Finalize RISC-V relaxation"); |
964 | Log(ctx) << "relaxation passes: "<< passes; |
965 | SmallVector<InputSection *, 0> storage; |
966 | for (OutputSection *osec : ctx.outputSections) { |
967 | if (!(osec->flags & SHF_EXECINSTR)) |
968 | continue; |
969 | for (InputSection *sec : getInputSections(os: *osec, storage)) { |
970 | RelaxAux &aux = *sec->relaxAux; |
971 | if (!aux.relocDeltas) |
972 | continue; |
973 | |
974 | MutableArrayRef<Relocation> rels = sec->relocs(); |
975 | ArrayRef<uint8_t> old = sec->content(); |
976 | size_t newSize = old.size() - aux.relocDeltas[rels.size() - 1]; |
977 | size_t writesIdx = 0; |
978 | uint8_t *p = ctx.bAlloc.Allocate<uint8_t>(Num: newSize); |
979 | uint64_t offset = 0; |
980 | int64_t delta = 0; |
981 | sec->content_ = p; |
982 | sec->size = newSize; |
983 | sec->bytesDropped = 0; |
984 | |
985 | // Update section content: remove NOPs for R_RISCV_ALIGN and rewrite |
986 | // instructions for relaxed relocations. |
987 | for (size_t i = 0, e = rels.size(); i != e; ++i) { |
988 | uint32_t remove = aux.relocDeltas[i] - delta; |
989 | delta = aux.relocDeltas[i]; |
990 | if (remove == 0 && aux.relocTypes[i] == R_RISCV_NONE) |
991 | continue; |
992 | |
993 | // Copy from last location to the current relocated location. |
994 | const Relocation &r = rels[i]; |
995 | uint64_t size = r.offset - offset; |
996 | memcpy(dest: p, src: old.data() + offset, n: size); |
997 | p += size; |
998 | |
999 | // For R_RISCV_ALIGN, we will place `offset` in a location (among NOPs) |
1000 | // to satisfy the alignment requirement. If both `remove` and r.addend |
1001 | // are multiples of 4, it is as if we have skipped some NOPs. Otherwise |
1002 | // we are in the middle of a 4-byte NOP, and we need to rewrite the NOP |
1003 | // sequence. |
1004 | int64_t skip = 0; |
1005 | if (r.type == R_RISCV_ALIGN) { |
1006 | if (remove % 4 || r.addend % 4) { |
1007 | skip = r.addend - remove; |
1008 | int64_t j = 0; |
1009 | for (; j + 4 <= skip; j += 4) |
1010 | write32le(P: p + j, V: 0x00000013); // nop |
1011 | if (j != skip) { |
1012 | assert(j + 2 == skip); |
1013 | write16le(P: p + j, V: 0x0001); // c.nop |
1014 | } |
1015 | } |
1016 | } else if (RelType newType = aux.relocTypes[i]) { |
1017 | switch (newType) { |
1018 | case INTERNAL_R_RISCV_GPREL_I: |
1019 | case INTERNAL_R_RISCV_GPREL_S: |
1020 | case INTERNAL_R_RISCV_X0REL_I: |
1021 | case INTERNAL_R_RISCV_X0REL_S: |
1022 | break; |
1023 | case R_RISCV_RELAX: |
1024 | // Used by relaxTlsLe to indicate the relocation is ignored. |
1025 | break; |
1026 | case R_RISCV_RVC_JUMP: |
1027 | skip = 2; |
1028 | write16le(P: p, V: aux.writes[writesIdx++]); |
1029 | break; |
1030 | case R_RISCV_JAL: |
1031 | skip = 4; |
1032 | write32le(P: p, V: aux.writes[writesIdx++]); |
1033 | break; |
1034 | case R_RISCV_32: |
1035 | // Used by relaxTlsLe to write a uint32_t then suppress the handling |
1036 | // in relocateAlloc. |
1037 | skip = 4; |
1038 | write32le(P: p, V: aux.writes[writesIdx++]); |
1039 | aux.relocTypes[i] = R_RISCV_NONE; |
1040 | break; |
1041 | default: |
1042 | llvm_unreachable("unsupported type"); |
1043 | } |
1044 | } |
1045 | |
1046 | p += skip; |
1047 | offset = r.offset + skip + remove; |
1048 | } |
1049 | memcpy(dest: p, src: old.data() + offset, n: old.size() - offset); |
1050 | |
1051 | // Subtract the previous relocDeltas value from the relocation offset. |
1052 | // For a pair of R_RISCV_CALL/R_RISCV_RELAX with the same offset, decrease |
1053 | // their r_offset by the same delta. |
1054 | delta = 0; |
1055 | for (size_t i = 0, e = rels.size(); i != e;) { |
1056 | uint64_t cur = rels[i].offset; |
1057 | do { |
1058 | rels[i].offset -= delta; |
1059 | if (aux.relocTypes[i] != R_RISCV_NONE) |
1060 | rels[i].type = aux.relocTypes[i]; |
1061 | } while (++i != e && rels[i].offset == cur); |
1062 | delta = aux.relocDeltas[i - 1]; |
1063 | } |
1064 | } |
1065 | } |
1066 | } |
1067 | |
1068 | namespace { |
1069 | // Representation of the merged .riscv.attributes input sections. The psABI |
1070 | // specifies merge policy for attributes. E.g. if we link an object without an |
1071 | // extension with an object with the extension, the output Tag_RISCV_arch shall |
1072 | // contain the extension. Some tools like objdump parse .riscv.attributes and |
1073 | // disabling some instructions if the first Tag_RISCV_arch does not contain an |
1074 | // extension. |
1075 | class RISCVAttributesSection final : public SyntheticSection { |
1076 | public: |
1077 | RISCVAttributesSection(Ctx &ctx) |
1078 | : SyntheticSection(ctx, ".riscv.attributes", SHT_RISCV_ATTRIBUTES, 0, 1) { |
1079 | } |
1080 | |
1081 | size_t getSize() const override { return size; } |
1082 | void writeTo(uint8_t *buf) override; |
1083 | |
1084 | static constexpr StringRef vendor = "riscv"; |
1085 | DenseMap<unsigned, unsigned> intAttr; |
1086 | DenseMap<unsigned, StringRef> strAttr; |
1087 | size_t size = 0; |
1088 | }; |
1089 | } // namespace |
1090 | |
1091 | static void mergeArch(Ctx &ctx, RISCVISAUtils::OrderedExtensionMap &mergedExts, |
1092 | unsigned &mergedXlen, const InputSectionBase *sec, |
1093 | StringRef s) { |
1094 | auto maybeInfo = RISCVISAInfo::parseNormalizedArchString(Arch: s); |
1095 | if (!maybeInfo) { |
1096 | Err(ctx) << sec << ": "<< s << ": "<< maybeInfo.takeError(); |
1097 | return; |
1098 | } |
1099 | |
1100 | // Merge extensions. |
1101 | RISCVISAInfo &info = **maybeInfo; |
1102 | if (mergedExts.empty()) { |
1103 | mergedExts = info.getExtensions(); |
1104 | mergedXlen = info.getXLen(); |
1105 | } else { |
1106 | for (const auto &ext : info.getExtensions()) { |
1107 | auto p = mergedExts.insert(x: ext); |
1108 | if (!p.second) { |
1109 | if (std::tie(args&: p.first->second.Major, args&: p.first->second.Minor) < |
1110 | std::tie(args: ext.second.Major, args: ext.second.Minor)) |
1111 | p.first->second = ext.second; |
1112 | } |
1113 | } |
1114 | } |
1115 | } |
1116 | |
1117 | static void mergeAtomic(Ctx &ctx, DenseMap<unsigned, unsigned>::iterator it, |
1118 | const InputSectionBase *oldSection, |
1119 | const InputSectionBase *newSection, |
1120 | RISCVAttrs::RISCVAtomicAbiTag oldTag, |
1121 | RISCVAttrs::RISCVAtomicAbiTag newTag) { |
1122 | using RISCVAttrs::RISCVAtomicAbiTag; |
1123 | // Same tags stay the same, and UNKNOWN is compatible with anything |
1124 | if (oldTag == newTag || newTag == RISCVAtomicAbiTag::UNKNOWN) |
1125 | return; |
1126 | |
1127 | auto reportAbiError = [&]() { |
1128 | Err(ctx) << "atomic abi mismatch for "<< oldSection->name << "\n>>> " |
1129 | << oldSection << ": atomic_abi="<< static_cast<unsigned>(oldTag) |
1130 | << "\n>>> "<< newSection |
1131 | << ": atomic_abi="<< static_cast<unsigned>(newTag); |
1132 | }; |
1133 | |
1134 | auto reportUnknownAbiError = [&](const InputSectionBase *section, |
1135 | RISCVAtomicAbiTag tag) { |
1136 | switch (tag) { |
1137 | case RISCVAtomicAbiTag::UNKNOWN: |
1138 | case RISCVAtomicAbiTag::A6C: |
1139 | case RISCVAtomicAbiTag::A6S: |
1140 | case RISCVAtomicAbiTag::A7: |
1141 | return; |
1142 | }; |
1143 | Err(ctx) << "unknown atomic abi for "<< section->name << "\n>>> " |
1144 | << section << ": atomic_abi="<< static_cast<unsigned>(tag); |
1145 | }; |
1146 | switch (oldTag) { |
1147 | case RISCVAtomicAbiTag::UNKNOWN: |
1148 | it->getSecond() = static_cast<unsigned>(newTag); |
1149 | return; |
1150 | case RISCVAtomicAbiTag::A6C: |
1151 | switch (newTag) { |
1152 | case RISCVAtomicAbiTag::A6S: |
1153 | it->getSecond() = static_cast<unsigned>(RISCVAtomicAbiTag::A6C); |
1154 | return; |
1155 | case RISCVAtomicAbiTag::A7: |
1156 | reportAbiError(); |
1157 | return; |
1158 | case RISCVAttrs::RISCVAtomicAbiTag::UNKNOWN: |
1159 | case RISCVAttrs::RISCVAtomicAbiTag::A6C: |
1160 | return; |
1161 | }; |
1162 | break; |
1163 | |
1164 | case RISCVAtomicAbiTag::A6S: |
1165 | switch (newTag) { |
1166 | case RISCVAtomicAbiTag::A6C: |
1167 | it->getSecond() = static_cast<unsigned>(RISCVAtomicAbiTag::A6C); |
1168 | return; |
1169 | case RISCVAtomicAbiTag::A7: |
1170 | it->getSecond() = static_cast<unsigned>(RISCVAtomicAbiTag::A7); |
1171 | return; |
1172 | case RISCVAttrs::RISCVAtomicAbiTag::UNKNOWN: |
1173 | case RISCVAttrs::RISCVAtomicAbiTag::A6S: |
1174 | return; |
1175 | }; |
1176 | break; |
1177 | |
1178 | case RISCVAtomicAbiTag::A7: |
1179 | switch (newTag) { |
1180 | case RISCVAtomicAbiTag::A6S: |
1181 | it->getSecond() = static_cast<unsigned>(RISCVAtomicAbiTag::A7); |
1182 | return; |
1183 | case RISCVAtomicAbiTag::A6C: |
1184 | reportAbiError(); |
1185 | return; |
1186 | case RISCVAttrs::RISCVAtomicAbiTag::UNKNOWN: |
1187 | case RISCVAttrs::RISCVAtomicAbiTag::A7: |
1188 | return; |
1189 | }; |
1190 | break; |
1191 | }; |
1192 | |
1193 | // If we get here, then we have an invalid tag, so report it. |
1194 | // Putting these checks at the end allows us to only do these checks when we |
1195 | // need to, since this is expected to be a rare occurrence. |
1196 | reportUnknownAbiError(oldSection, oldTag); |
1197 | reportUnknownAbiError(newSection, newTag); |
1198 | } |
1199 | |
1200 | static RISCVAttributesSection * |
1201 | mergeAttributesSection(Ctx &ctx, |
1202 | const SmallVector<InputSectionBase *, 0> §ions) { |
1203 | using RISCVAttrs::RISCVAtomicAbiTag; |
1204 | RISCVISAUtils::OrderedExtensionMap exts; |
1205 | const InputSectionBase *firstStackAlign = nullptr; |
1206 | const InputSectionBase *firstAtomicAbi = nullptr; |
1207 | unsigned firstStackAlignValue = 0, xlen = 0; |
1208 | bool hasArch = false; |
1209 | |
1210 | ctx.in.riscvAttributes = std::make_unique<RISCVAttributesSection>(args&: ctx); |
1211 | auto &merged = static_cast<RISCVAttributesSection &>(*ctx.in.riscvAttributes); |
1212 | |
1213 | // Collect all tags values from attributes section. |
1214 | const auto &attributesTags = RISCVAttrs::getRISCVAttributeTags(); |
1215 | for (const InputSectionBase *sec : sections) { |
1216 | RISCVAttributeParser parser; |
1217 | if (Error e = parser.parse(section: sec->content(), endian: llvm::endianness::little)) |
1218 | Warn(ctx) << sec << ": "<< std::move(e); |
1219 | for (const auto &tag : attributesTags) { |
1220 | switch (RISCVAttrs::AttrType(tag.attr)) { |
1221 | // Integer attributes. |
1222 | case RISCVAttrs::STACK_ALIGN: |
1223 | if (auto i = parser.getAttributeValue(tag: tag.attr)) { |
1224 | auto r = merged.intAttr.try_emplace(Key: tag.attr, Args&: *i); |
1225 | if (r.second) { |
1226 | firstStackAlign = sec; |
1227 | firstStackAlignValue = *i; |
1228 | } else if (r.first->second != *i) { |
1229 | Err(ctx) << sec << " has stack_align="<< *i << " but " |
1230 | << firstStackAlign |
1231 | << " has stack_align="<< firstStackAlignValue; |
1232 | } |
1233 | } |
1234 | continue; |
1235 | case RISCVAttrs::UNALIGNED_ACCESS: |
1236 | if (auto i = parser.getAttributeValue(tag: tag.attr)) |
1237 | merged.intAttr[tag.attr] |= *i; |
1238 | continue; |
1239 | |
1240 | // String attributes. |
1241 | case RISCVAttrs::ARCH: |
1242 | if (auto s = parser.getAttributeString(tag: tag.attr)) { |
1243 | hasArch = true; |
1244 | mergeArch(ctx, mergedExts&: exts, mergedXlen&: xlen, sec, s: *s); |
1245 | } |
1246 | continue; |
1247 | |
1248 | // Attributes which use the default handling. |
1249 | case RISCVAttrs::PRIV_SPEC: |
1250 | case RISCVAttrs::PRIV_SPEC_MINOR: |
1251 | case RISCVAttrs::PRIV_SPEC_REVISION: |
1252 | break; |
1253 | |
1254 | case RISCVAttrs::AttrType::ATOMIC_ABI: |
1255 | if (auto i = parser.getAttributeValue(tag: tag.attr)) { |
1256 | auto r = merged.intAttr.try_emplace(Key: tag.attr, Args&: *i); |
1257 | if (r.second) |
1258 | firstAtomicAbi = sec; |
1259 | else |
1260 | mergeAtomic(ctx, it: r.first, oldSection: firstAtomicAbi, newSection: sec, |
1261 | oldTag: static_cast<RISCVAtomicAbiTag>(r.first->getSecond()), |
1262 | newTag: static_cast<RISCVAtomicAbiTag>(*i)); |
1263 | } |
1264 | continue; |
1265 | } |
1266 | |
1267 | // Fallback for deprecated priv_spec* and other unknown attributes: retain |
1268 | // the attribute if all input sections agree on the value. GNU ld uses 0 |
1269 | // and empty strings as default values which are not dumped to the output. |
1270 | // TODO Adjust after resolution to |
1271 | // https://github.com/riscv-non-isa/riscv-elf-psabi-doc/issues/352 |
1272 | if (tag.attr % 2 == 0) { |
1273 | if (auto i = parser.getAttributeValue(tag: tag.attr)) { |
1274 | auto r = merged.intAttr.try_emplace(Key: tag.attr, Args&: *i); |
1275 | if (!r.second && r.first->second != *i) |
1276 | r.first->second = 0; |
1277 | } |
1278 | } else if (auto s = parser.getAttributeString(tag: tag.attr)) { |
1279 | auto r = merged.strAttr.try_emplace(Key: tag.attr, Args&: *s); |
1280 | if (!r.second && r.first->second != *s) |
1281 | r.first->second = {}; |
1282 | } |
1283 | } |
1284 | } |
1285 | |
1286 | if (hasArch && xlen != 0) { |
1287 | if (auto result = RISCVISAInfo::createFromExtMap(XLen: xlen, Exts: exts)) { |
1288 | merged.strAttr.try_emplace(Key: RISCVAttrs::ARCH, |
1289 | Args: ctx.saver.save(S: (*result)->toString())); |
1290 | } else { |
1291 | Err(ctx) << result.takeError(); |
1292 | } |
1293 | } |
1294 | |
1295 | // The total size of headers: format-version [ <section-length> "vendor-name" |
1296 | // [ <file-tag> <size>. |
1297 | size_t size = 5 + merged.vendor.size() + 1 + 5; |
1298 | for (auto &attr : merged.intAttr) |
1299 | if (attr.second != 0) |
1300 | size += getULEB128Size(Value: attr.first) + getULEB128Size(Value: attr.second); |
1301 | for (auto &attr : merged.strAttr) |
1302 | if (!attr.second.empty()) |
1303 | size += getULEB128Size(Value: attr.first) + attr.second.size() + 1; |
1304 | merged.size = size; |
1305 | return &merged; |
1306 | } |
1307 | |
1308 | void RISCVAttributesSection::writeTo(uint8_t *buf) { |
1309 | const size_t size = getSize(); |
1310 | uint8_t *const end = buf + size; |
1311 | *buf = ELFAttrs::Format_Version; |
1312 | write32(ctx, p: buf + 1, v: size - 1); |
1313 | buf += 5; |
1314 | |
1315 | memcpy(dest: buf, src: vendor.data(), n: vendor.size()); |
1316 | buf += vendor.size() + 1; |
1317 | |
1318 | *buf = ELFAttrs::File; |
1319 | write32(ctx, p: buf + 1, v: end - buf); |
1320 | buf += 5; |
1321 | |
1322 | for (auto &attr : intAttr) { |
1323 | if (attr.second == 0) |
1324 | continue; |
1325 | buf += encodeULEB128(Value: attr.first, p: buf); |
1326 | buf += encodeULEB128(Value: attr.second, p: buf); |
1327 | } |
1328 | for (auto &attr : strAttr) { |
1329 | if (attr.second.empty()) |
1330 | continue; |
1331 | buf += encodeULEB128(Value: attr.first, p: buf); |
1332 | memcpy(dest: buf, src: attr.second.data(), n: attr.second.size()); |
1333 | buf += attr.second.size() + 1; |
1334 | } |
1335 | } |
1336 | |
1337 | void elf::mergeRISCVAttributesSections(Ctx &ctx) { |
1338 | // Find the first input SHT_RISCV_ATTRIBUTES; return if not found. |
1339 | size_t place = |
1340 | llvm::find_if(Range&: ctx.inputSections, |
1341 | P: [](auto *s) { return s->type == SHT_RISCV_ATTRIBUTES; }) - |
1342 | ctx.inputSections.begin(); |
1343 | if (place == ctx.inputSections.size()) |
1344 | return; |
1345 | |
1346 | // Extract all SHT_RISCV_ATTRIBUTES sections into `sections`. |
1347 | SmallVector<InputSectionBase *, 0> sections; |
1348 | llvm::erase_if(C&: ctx.inputSections, P: [&](InputSectionBase *s) { |
1349 | if (s->type != SHT_RISCV_ATTRIBUTES) |
1350 | return false; |
1351 | sections.push_back(Elt: s); |
1352 | return true; |
1353 | }); |
1354 | |
1355 | // Add the merged section. |
1356 | ctx.inputSections.insert(I: ctx.inputSections.begin() + place, |
1357 | Elt: mergeAttributesSection(ctx, sections)); |
1358 | } |
1359 | |
1360 | void elf::setRISCVTargetInfo(Ctx &ctx) { ctx.target.reset(p: new RISCV(ctx)); } |
1361 |
Definitions
- RISCV
- dtpOffset
- Op
- Reg
- hi20
- lo12
- itype
- rtype
- utype
- extractBits
- setLO12_I
- setLO12_S
- RISCV
- getEFlags
- calcEFlags
- getImplicitAddend
- writeGotHeader
- writeGotPlt
- writeIgotPlt
- writePltHeader
- writePlt
- getDynRel
- getRelExpr
- relocate
- relaxable
- tlsdescToIe
- tlsdescToLe
- relocateAlloc
- initSymbolAnchors
- relaxCall
- relaxTlsLe
- relaxHi20Lo12
- relax
- relaxOnce
- finalizeRelax
- RISCVAttributesSection
- RISCVAttributesSection
- getSize
- vendor
- mergeArch
- mergeAtomic
- mergeAttributesSection
- writeTo
- mergeRISCVAttributesSections
Learn to use CMake with our Intro Training
Find out more