1//===- bolt/Core/BinaryEmitter.cpp - Emit code and data -------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the collection of functions and classes used for
10// emission of code and data into object/binary file.
11//
12//===----------------------------------------------------------------------===//
13
14#include "bolt/Core/BinaryEmitter.h"
15#include "bolt/Core/BinaryContext.h"
16#include "bolt/Core/BinaryFunction.h"
17#include "bolt/Core/DebugData.h"
18#include "bolt/Core/FunctionLayout.h"
19#include "bolt/Utils/CommandLineOpts.h"
20#include "bolt/Utils/Utils.h"
21#include "llvm/DebugInfo/DWARF/DWARFCompileUnit.h"
22#include "llvm/MC/MCSection.h"
23#include "llvm/MC/MCStreamer.h"
24#include "llvm/Support/CommandLine.h"
25#include "llvm/Support/LEB128.h"
26#include "llvm/Support/SMLoc.h"
27
28#define DEBUG_TYPE "bolt"
29
30using namespace llvm;
31using namespace bolt;
32
33namespace opts {
34
35extern cl::opt<JumpTableSupportLevel> JumpTables;
36extern cl::opt<bool> PreserveBlocksAlignment;
37
38cl::opt<bool> AlignBlocks("align-blocks", cl::desc("align basic blocks"),
39 cl::cat(BoltOptCategory));
40
41static cl::list<std::string>
42BreakFunctionNames("break-funcs",
43 cl::CommaSeparated,
44 cl::desc("list of functions to core dump on (debugging)"),
45 cl::value_desc("func1,func2,func3,..."),
46 cl::Hidden,
47 cl::cat(BoltCategory));
48
49static cl::list<std::string>
50 FunctionPadSpec("pad-funcs", cl::CommaSeparated,
51 cl::desc("list of functions to pad with amount of bytes"),
52 cl::value_desc("func1:pad1,func2:pad2,func3:pad3,..."),
53 cl::Hidden, cl::cat(BoltCategory));
54
55static cl::list<std::string> FunctionPadBeforeSpec(
56 "pad-funcs-before", cl::CommaSeparated,
57 cl::desc("list of functions to pad with amount of bytes"),
58 cl::value_desc("func1:pad1,func2:pad2,func3:pad3,..."), cl::Hidden,
59 cl::cat(BoltCategory));
60
61static cl::opt<bool> MarkFuncs(
62 "mark-funcs",
63 cl::desc("mark function boundaries with break instruction to make "
64 "sure we accidentally don't cross them"),
65 cl::ReallyHidden, cl::cat(BoltCategory));
66
67static cl::opt<bool> PrintJumpTables("print-jump-tables",
68 cl::desc("print jump tables"), cl::Hidden,
69 cl::cat(BoltCategory));
70
71static cl::opt<bool>
72X86AlignBranchBoundaryHotOnly("x86-align-branch-boundary-hot-only",
73 cl::desc("only apply branch boundary alignment in hot code"),
74 cl::init(Val: true),
75 cl::cat(BoltOptCategory));
76
77size_t padFunction(std::map<std::string, size_t> &FunctionPadding,
78 const cl::list<std::string> &Spec,
79 const BinaryFunction &Function) {
80 if (FunctionPadding.empty() && !Spec.empty()) {
81 for (const std::string &Spec : Spec) {
82 size_t N = Spec.find(c: ':');
83 if (N == std::string::npos)
84 continue;
85 std::string Name = Spec.substr(pos: 0, n: N);
86 size_t Padding = std::stoull(str: Spec.substr(pos: N + 1));
87 FunctionPadding[Name] = Padding;
88 }
89 }
90
91 for (auto &FPI : FunctionPadding) {
92 std::string Name = FPI.first;
93 size_t Padding = FPI.second;
94 if (Function.hasNameRegex(NameRegex: Name))
95 return Padding;
96 }
97
98 return 0;
99}
100
101size_t padFunctionBefore(const BinaryFunction &Function) {
102 static std::map<std::string, size_t> CacheFunctionPadding;
103 return padFunction(FunctionPadding&: CacheFunctionPadding, Spec: FunctionPadBeforeSpec, Function);
104}
105size_t padFunctionAfter(const BinaryFunction &Function) {
106 static std::map<std::string, size_t> CacheFunctionPadding;
107 return padFunction(FunctionPadding&: CacheFunctionPadding, Spec: FunctionPadSpec, Function);
108}
109
110} // namespace opts
111
112namespace {
113using JumpTable = bolt::JumpTable;
114
115class BinaryEmitter {
116private:
117 BinaryEmitter(const BinaryEmitter &) = delete;
118 BinaryEmitter &operator=(const BinaryEmitter &) = delete;
119
120 MCStreamer &Streamer;
121 BinaryContext &BC;
122
123public:
124 BinaryEmitter(MCStreamer &Streamer, BinaryContext &BC)
125 : Streamer(Streamer), BC(BC) {}
126
127 /// Emit all code and data.
128 void emitAll(StringRef OrgSecPrefix);
129
130 /// Emit function code. The caller is responsible for emitting function
131 /// symbol(s) and setting the section to emit the code to.
132 void emitFunctionBody(BinaryFunction &BF, FunctionFragment &FF,
133 bool EmitCodeOnly = false);
134
135private:
136 /// Emit function code.
137 void emitFunctions();
138
139 /// Emit a single function.
140 bool emitFunction(BinaryFunction &BF, FunctionFragment &FF);
141
142 /// Helper for emitFunctionBody to write data inside a function
143 /// (used for AArch64)
144 void emitConstantIslands(BinaryFunction &BF, bool EmitColdPart,
145 BinaryFunction *OnBehalfOf = nullptr);
146
147 /// Emit jump tables for the function.
148 void emitJumpTables(const BinaryFunction &BF);
149
150 /// Emit jump table data. Callee supplies sections for the data.
151 void emitJumpTable(const JumpTable &JT, MCSection *HotSection,
152 MCSection *ColdSection);
153
154 void emitCFIInstruction(const MCCFIInstruction &Inst) const;
155
156 /// Emit exception handling ranges for the function fragment.
157 void emitLSDA(BinaryFunction &BF, const FunctionFragment &FF);
158
159 /// Emit line number information corresponding to \p NewLoc. \p PrevLoc
160 /// provides a context for de-duplication of line number info.
161 /// \p FirstInstr indicates if \p NewLoc represents the first instruction
162 /// in a sequence, such as a function fragment.
163 ///
164 /// If \p NewLoc location matches \p PrevLoc, no new line number entry will be
165 /// created and the function will return \p PrevLoc while \p InstrLabel will
166 /// be ignored. Otherwise, the caller should use \p InstrLabel to mark the
167 /// corresponding instruction by emitting \p InstrLabel before it.
168 /// If \p InstrLabel is set by the caller, its value will be used with \p
169 /// \p NewLoc. If it was nullptr on entry, it will be populated with a pointer
170 /// to a new temp symbol used with \p NewLoc.
171 ///
172 /// Return new current location which is either \p NewLoc or \p PrevLoc.
173 SMLoc emitLineInfo(const BinaryFunction &BF, SMLoc NewLoc, SMLoc PrevLoc,
174 bool FirstInstr, MCSymbol *&InstrLabel);
175
176 /// Use \p FunctionEndSymbol to mark the end of the line info sequence.
177 /// Note that it does not automatically result in the insertion of the EOS
178 /// marker in the line table program, but provides one to the DWARF generator
179 /// when it needs it.
180 void emitLineInfoEnd(const BinaryFunction &BF, MCSymbol *FunctionEndSymbol);
181
182 /// Emit debug line info for unprocessed functions from CUs that include
183 /// emitted functions.
184 void emitDebugLineInfoForOriginalFunctions();
185
186 /// Emit debug line for CUs that were not modified.
187 void emitDebugLineInfoForUnprocessedCUs();
188
189 /// Emit data sections that have code references in them.
190 void emitDataSections(StringRef OrgSecPrefix);
191};
192
193} // anonymous namespace
194
195void BinaryEmitter::emitAll(StringRef OrgSecPrefix) {
196 Streamer.initSections(NoExecStack: false, STI: *BC.STI);
197 Streamer.setUseAssemblerInfoForParsing(false);
198
199 if (opts::UpdateDebugSections && BC.isELF()) {
200 // Force the emission of debug line info into allocatable section to ensure
201 // JITLink will process it.
202 //
203 // NB: on MachO all sections are required for execution, hence no need
204 // to change flags/attributes.
205 MCSectionELF *ELFDwarfLineSection =
206 static_cast<MCSectionELF *>(BC.MOFI->getDwarfLineSection());
207 ELFDwarfLineSection->setFlags(ELF::SHF_ALLOC);
208 MCSectionELF *ELFDwarfLineStrSection =
209 static_cast<MCSectionELF *>(BC.MOFI->getDwarfLineStrSection());
210 ELFDwarfLineStrSection->setFlags(ELF::SHF_ALLOC);
211 }
212
213 if (RuntimeLibrary *RtLibrary = BC.getRuntimeLibrary())
214 RtLibrary->emitBinary(BC, Streamer);
215
216 BC.getTextSection()->setAlignment(Align(opts::AlignText));
217
218 emitFunctions();
219
220 if (opts::UpdateDebugSections) {
221 emitDebugLineInfoForOriginalFunctions();
222 DwarfLineTable::emit(BC, Streamer);
223 }
224
225 emitDataSections(OrgSecPrefix);
226
227 // TODO Enable for Mach-O once BinaryContext::getDataSection supports it.
228 if (BC.isELF())
229 AddressMap::emit(Streamer, BC);
230 Streamer.setUseAssemblerInfoForParsing(true);
231}
232
233void BinaryEmitter::emitFunctions() {
234 auto emit = [&](const std::vector<BinaryFunction *> &Functions) {
235 const bool HasProfile = BC.NumProfiledFuncs > 0;
236 const bool OriginalAllowAutoPadding = Streamer.getAllowAutoPadding();
237 for (BinaryFunction *Function : Functions) {
238 if (!BC.shouldEmit(Function: *Function))
239 continue;
240
241 LLVM_DEBUG(dbgs() << "BOLT: generating code for function \"" << *Function
242 << "\" : " << Function->getFunctionNumber() << '\n');
243
244 // Was any part of the function emitted.
245 bool Emitted = false;
246
247 // Turn off Intel JCC Erratum mitigation for cold code if requested
248 if (HasProfile && opts::X86AlignBranchBoundaryHotOnly &&
249 !Function->hasValidProfile())
250 Streamer.setAllowAutoPadding(false);
251
252 FunctionLayout &Layout = Function->getLayout();
253 Emitted |= emitFunction(BF&: *Function, FF&: Layout.getMainFragment());
254
255 if (Function->isSplit()) {
256 if (opts::X86AlignBranchBoundaryHotOnly)
257 Streamer.setAllowAutoPadding(false);
258
259 assert((Layout.fragment_size() == 1 || Function->isSimple()) &&
260 "Only simple functions can have fragments");
261 for (FunctionFragment &FF : Layout.getSplitFragments()) {
262 // Skip empty fragments so no symbols and sections for empty fragments
263 // are generated
264 if (FF.empty() && !Function->hasConstantIsland())
265 continue;
266 Emitted |= emitFunction(BF&: *Function, FF);
267 }
268 }
269
270 Streamer.setAllowAutoPadding(OriginalAllowAutoPadding);
271
272 if (Emitted)
273 Function->setEmitted(/*KeepCFG=*/opts::PrintCacheMetrics);
274 }
275 };
276
277 // Mark the start of hot text.
278 if (opts::HotText) {
279 Streamer.switchSection(Section: BC.getTextSection());
280 Streamer.emitLabel(Symbol: BC.getHotTextStartSymbol());
281 }
282
283 // Emit functions in sorted order.
284 std::vector<BinaryFunction *> SortedFunctions = BC.getSortedFunctions();
285 emit(SortedFunctions);
286
287 // Emit functions added by BOLT.
288 emit(BC.getInjectedBinaryFunctions());
289
290 // Mark the end of hot text.
291 if (opts::HotText) {
292 if (BC.HasWarmSection)
293 Streamer.switchSection(Section: BC.getCodeSection(SectionName: BC.getWarmCodeSectionName()));
294 else
295 Streamer.switchSection(Section: BC.getTextSection());
296 Streamer.emitLabel(Symbol: BC.getHotTextEndSymbol());
297 }
298}
299
300bool BinaryEmitter::emitFunction(BinaryFunction &Function,
301 FunctionFragment &FF) {
302 if (Function.size() == 0 && !Function.hasIslandsInfo())
303 return false;
304
305 if (Function.getState() == BinaryFunction::State::Empty)
306 return false;
307
308 // Avoid emitting function without instructions when overwriting the original
309 // function in-place. Otherwise, emit the empty function to define the symbol.
310 if (!BC.HasRelocations && !Function.hasNonPseudoInstructions())
311 return false;
312
313 MCSection *Section =
314 BC.getCodeSection(SectionName: Function.getCodeSectionName(Fragment: FF.getFragmentNum()));
315 Streamer.switchSection(Section);
316 Section->setHasInstructions(true);
317 BC.Ctx->addGenDwarfSection(Sec: Section);
318
319 if (BC.HasRelocations) {
320 // Set section alignment to at least maximum possible object alignment.
321 // We need this to support LongJmp and other passes that calculates
322 // tentative layout.
323 Section->ensureMinAlignment(MinAlignment: Align(opts::AlignFunctions));
324
325 Streamer.emitCodeAlignment(Alignment: Function.getMinAlign(), STI: &*BC.STI);
326 uint16_t MaxAlignBytes = FF.isSplitFragment()
327 ? Function.getMaxColdAlignmentBytes()
328 : Function.getMaxAlignmentBytes();
329 if (MaxAlignBytes > 0)
330 Streamer.emitCodeAlignment(Alignment: Function.getAlign(), STI: &*BC.STI, MaxBytesToEmit: MaxAlignBytes);
331 } else {
332 Streamer.emitCodeAlignment(Alignment: Function.getAlign(), STI: &*BC.STI);
333 }
334
335 if (size_t Padding = opts::padFunctionBefore(Function)) {
336 // Handle padFuncsBefore after the above alignment logic but before
337 // symbol addresses are decided.
338 if (!BC.HasRelocations) {
339 BC.errs() << "BOLT-ERROR: -pad-before-funcs is not supported in "
340 << "non-relocation mode\n";
341 exit(status: 1);
342 }
343
344 // Preserve Function.getMinAlign().
345 if (!isAligned(Lhs: Function.getMinAlign(), SizeInBytes: Padding)) {
346 BC.errs() << "BOLT-ERROR: user-requested " << Padding
347 << " padding bytes before function " << Function
348 << " is not a multiple of the minimum function alignment ("
349 << Function.getMinAlign().value() << ").\n";
350 exit(status: 1);
351 }
352
353 LLVM_DEBUG(dbgs() << "BOLT-DEBUG: padding before function " << Function
354 << " with " << Padding << " bytes\n");
355
356 // Since the padding is not executed, it can be null bytes.
357 Streamer.emitFill(NumBytes: Padding, FillValue: 0);
358 }
359
360 MCContext &Context = Streamer.getContext();
361 const MCAsmInfo *MAI = Context.getAsmInfo();
362
363 MCSymbol *const StartSymbol = Function.getSymbol(Fragment: FF.getFragmentNum());
364
365 // Emit all symbols associated with the main function entry.
366 if (FF.isMainFragment()) {
367 for (MCSymbol *Symbol : Function.getSymbols()) {
368 Streamer.emitSymbolAttribute(Symbol, Attribute: MCSA_ELF_TypeFunction);
369 Streamer.emitLabel(Symbol);
370 }
371 } else {
372 Streamer.emitSymbolAttribute(Symbol: StartSymbol, Attribute: MCSA_ELF_TypeFunction);
373 Streamer.emitLabel(Symbol: StartSymbol);
374 }
375
376 const bool NeedsFDE =
377 Function.hasCFI() && !(Function.isPatch() && Function.isAnonymous());
378 // Emit CFI start
379 if (NeedsFDE) {
380 Streamer.emitCFIStartProc(/*IsSimple=*/false);
381 if (Function.getPersonalityFunction() != nullptr)
382 Streamer.emitCFIPersonality(Sym: Function.getPersonalityFunction(),
383 Encoding: Function.getPersonalityEncoding());
384 MCSymbol *LSDASymbol = Function.getLSDASymbol(F: FF.getFragmentNum());
385 if (LSDASymbol)
386 Streamer.emitCFILsda(Sym: LSDASymbol, Encoding: BC.LSDAEncoding);
387 else
388 Streamer.emitCFILsda(Sym: 0, Encoding: dwarf::DW_EH_PE_omit);
389 // Emit CFI instructions relative to the CIE
390 for (const MCCFIInstruction &CFIInstr : Function.cie()) {
391 // Only write CIE CFI insns that LLVM will not already emit
392 const std::vector<MCCFIInstruction> &FrameInstrs =
393 MAI->getInitialFrameState();
394 if (!llvm::is_contained(Range: FrameInstrs, Element: CFIInstr))
395 emitCFIInstruction(Inst: CFIInstr);
396 }
397 }
398
399 assert((Function.empty() || !(*Function.begin()).isCold()) &&
400 "first basic block should never be cold");
401
402 // Emit UD2 at the beginning if requested by user.
403 if (!opts::BreakFunctionNames.empty()) {
404 for (std::string &Name : opts::BreakFunctionNames) {
405 if (Function.hasNameRegex(NameRegex: Name)) {
406 Streamer.emitIntValue(Value: 0x0B0F, Size: 2); // UD2: 0F 0B
407 break;
408 }
409 }
410 }
411
412 // Emit code.
413 emitFunctionBody(BF&: Function, FF, /*EmitCodeOnly=*/false);
414
415 // Emit padding if requested.
416 if (size_t Padding = opts::padFunctionAfter(Function)) {
417 LLVM_DEBUG(dbgs() << "BOLT-DEBUG: padding function " << Function << " with "
418 << Padding << " bytes\n");
419 Streamer.emitFill(NumBytes: Padding, FillValue: MAI->getTextAlignFillValue());
420 }
421
422 if (opts::MarkFuncs)
423 Streamer.emitBytes(Data: BC.MIB->getTrapFillValue());
424
425 // Emit CFI end
426 if (NeedsFDE)
427 Streamer.emitCFIEndProc();
428
429 MCSymbol *EndSymbol = Function.getFunctionEndLabel(Fragment: FF.getFragmentNum());
430 Streamer.emitLabel(Symbol: EndSymbol);
431
432 if (MAI->hasDotTypeDotSizeDirective()) {
433 const MCExpr *SizeExpr = MCBinaryExpr::createSub(
434 LHS: MCSymbolRefExpr::create(Symbol: EndSymbol, Ctx&: Context),
435 RHS: MCSymbolRefExpr::create(Symbol: StartSymbol, Ctx&: Context), Ctx&: Context);
436 Streamer.emitELFSize(Symbol: StartSymbol, Value: SizeExpr);
437 }
438
439 if (opts::UpdateDebugSections && Function.getDWARFUnit())
440 emitLineInfoEnd(BF: Function, FunctionEndSymbol: EndSymbol);
441
442 // Exception handling info for the function.
443 emitLSDA(BF&: Function, FF);
444
445 if (FF.isMainFragment() && opts::JumpTables > JTS_NONE)
446 emitJumpTables(BF: Function);
447
448 return true;
449}
450
451void BinaryEmitter::emitFunctionBody(BinaryFunction &BF, FunctionFragment &FF,
452 bool EmitCodeOnly) {
453 if (!EmitCodeOnly && FF.isSplitFragment() && BF.hasConstantIsland()) {
454 assert(BF.getLayout().isHotColdSplit() &&
455 "Constant island support only with hot/cold split");
456 BF.duplicateConstantIslands();
457 }
458
459 // Track the first emitted instruction with debug info.
460 bool FirstInstr = true;
461 for (BinaryBasicBlock *const BB : FF) {
462 if ((opts::AlignBlocks || opts::PreserveBlocksAlignment) &&
463 BB->getAlignment() > 1)
464 Streamer.emitCodeAlignment(Alignment: BB->getAlign(), STI: &*BC.STI,
465 MaxBytesToEmit: BB->getAlignmentMaxBytes());
466 Streamer.emitLabel(Symbol: BB->getLabel());
467 if (!EmitCodeOnly) {
468 if (MCSymbol *EntrySymbol = BF.getSecondaryEntryPointSymbol(BB: *BB))
469 Streamer.emitLabel(Symbol: EntrySymbol);
470 }
471
472 SMLoc LastLocSeen;
473 for (auto I = BB->begin(), E = BB->end(); I != E; ++I) {
474 MCInst &Instr = *I;
475
476 if (EmitCodeOnly && BC.MIB->isPseudo(Inst: Instr))
477 continue;
478
479 // Handle pseudo instructions.
480 if (BC.MIB->isCFI(Inst: Instr)) {
481 emitCFIInstruction(Inst: *BF.getCFIFor(Instr));
482 continue;
483 }
484
485 if (!EmitCodeOnly) {
486 // A symbol to be emitted before the instruction to mark its location.
487 MCSymbol *InstrLabel = BC.MIB->getInstLabel(Inst: Instr);
488
489 if (opts::UpdateDebugSections && BF.getDWARFUnit()) {
490 LastLocSeen = emitLineInfo(BF, NewLoc: Instr.getLoc(), PrevLoc: LastLocSeen,
491 FirstInstr, InstrLabel);
492 FirstInstr = false;
493 }
494
495 // Prepare to tag this location with a label if we need to keep track of
496 // the location of calls/returns for BOLT address translation maps
497 if (BF.requiresAddressTranslation() && BC.MIB->getOffset(Inst: Instr)) {
498 const uint32_t Offset = *BC.MIB->getOffset(Inst: Instr);
499 if (!InstrLabel)
500 InstrLabel = BC.Ctx->createTempSymbol();
501 BB->getLocSyms().emplace_back(args: Offset, args&: InstrLabel);
502 }
503
504 if (InstrLabel)
505 Streamer.emitLabel(Symbol: InstrLabel);
506 }
507
508 // Emit sized NOPs via MCAsmBackend::writeNopData() interface on x86.
509 // This is a workaround for invalid NOPs handling by asm/disasm layer.
510 if (BC.isX86() && BC.MIB->isNoop(Inst: Instr)) {
511 if (std::optional<uint32_t> Size = BC.MIB->getSize(Inst: Instr)) {
512 SmallString<15> Code;
513 raw_svector_ostream VecOS(Code);
514 BC.MAB->writeNopData(OS&: VecOS, Count: *Size, STI: BC.STI.get());
515 Streamer.emitBytes(Data: Code);
516 continue;
517 }
518 }
519
520 Streamer.emitInstruction(Inst: Instr, STI: *BC.STI);
521 }
522 }
523
524 if (!EmitCodeOnly)
525 emitConstantIslands(BF, EmitColdPart: FF.isSplitFragment());
526}
527
528void BinaryEmitter::emitConstantIslands(BinaryFunction &BF, bool EmitColdPart,
529 BinaryFunction *OnBehalfOf) {
530 if (!BF.hasIslandsInfo())
531 return;
532
533 BinaryFunction::IslandInfo &Islands = BF.getIslandInfo();
534 if (Islands.DataOffsets.empty() && Islands.Dependency.empty())
535 return;
536
537 // AArch64 requires CI to be aligned to 8 bytes due to access instructions
538 // restrictions. E.g. the ldr with imm, where imm must be aligned to 8 bytes.
539 const uint16_t Alignment = OnBehalfOf
540 ? OnBehalfOf->getConstantIslandAlignment()
541 : BF.getConstantIslandAlignment();
542 Streamer.emitCodeAlignment(Alignment: Align(Alignment), STI: &*BC.STI);
543
544 if (!OnBehalfOf) {
545 if (!EmitColdPart)
546 Streamer.emitLabel(Symbol: BF.getFunctionConstantIslandLabel());
547 else
548 Streamer.emitLabel(Symbol: BF.getFunctionColdConstantIslandLabel());
549 }
550
551 assert((!OnBehalfOf || Islands.Proxies[OnBehalfOf].size() > 0) &&
552 "spurious OnBehalfOf constant island emission");
553
554 assert(!BF.isInjected() &&
555 "injected functions should not have constant islands");
556 // Raw contents of the function.
557 StringRef SectionContents = BF.getOriginSection()->getContents();
558
559 // Raw contents of the function.
560 StringRef FunctionContents = SectionContents.substr(
561 Start: BF.getAddress() - BF.getOriginSection()->getAddress(), N: BF.getMaxSize());
562
563 if (opts::Verbosity && !OnBehalfOf)
564 BC.outs() << "BOLT-INFO: emitting constant island for function " << BF
565 << "\n";
566
567 // We split the island into smaller blocks and output labels between them.
568 auto IS = Islands.Offsets.begin();
569 for (auto DataIter = Islands.DataOffsets.begin();
570 DataIter != Islands.DataOffsets.end(); ++DataIter) {
571 uint64_t FunctionOffset = *DataIter;
572 uint64_t EndOffset = 0ULL;
573
574 // Determine size of this data chunk
575 auto NextData = std::next(x: DataIter);
576 auto CodeIter = Islands.CodeOffsets.lower_bound(x: *DataIter);
577 if (CodeIter == Islands.CodeOffsets.end() &&
578 NextData == Islands.DataOffsets.end())
579 EndOffset = BF.getMaxSize();
580 else if (CodeIter == Islands.CodeOffsets.end())
581 EndOffset = *NextData;
582 else if (NextData == Islands.DataOffsets.end())
583 EndOffset = *CodeIter;
584 else
585 EndOffset = (*CodeIter > *NextData) ? *NextData : *CodeIter;
586
587 if (FunctionOffset == EndOffset)
588 continue; // Size is zero, nothing to emit
589
590 auto emitCI = [&](uint64_t &FunctionOffset, uint64_t EndOffset) {
591 if (FunctionOffset >= EndOffset)
592 return;
593
594 for (auto It = Islands.Relocations.lower_bound(x: FunctionOffset);
595 It != Islands.Relocations.end(); ++It) {
596 if (It->first >= EndOffset)
597 break;
598
599 const Relocation &Relocation = It->second;
600 if (FunctionOffset < Relocation.Offset) {
601 Streamer.emitBytes(
602 Data: FunctionContents.slice(Start: FunctionOffset, End: Relocation.Offset));
603 FunctionOffset = Relocation.Offset;
604 }
605
606 LLVM_DEBUG(
607 dbgs() << "BOLT-DEBUG: emitting constant island relocation"
608 << " for " << BF << " at offset 0x"
609 << Twine::utohexstr(Relocation.Offset) << " with size "
610 << Relocation::getSizeForType(Relocation.Type) << '\n');
611
612 FunctionOffset += Relocation.emit(Streamer: &Streamer);
613 }
614
615 assert(FunctionOffset <= EndOffset && "overflow error");
616 if (FunctionOffset < EndOffset) {
617 Streamer.emitBytes(Data: FunctionContents.slice(Start: FunctionOffset, End: EndOffset));
618 FunctionOffset = EndOffset;
619 }
620 };
621
622 // Emit labels, relocs and data
623 while (IS != Islands.Offsets.end() && IS->first < EndOffset) {
624 auto NextLabelOffset =
625 IS == Islands.Offsets.end() ? EndOffset : IS->first;
626 auto NextStop = std::min(a: NextLabelOffset, b: EndOffset);
627 assert(NextStop <= EndOffset && "internal overflow error");
628 emitCI(FunctionOffset, NextStop);
629 if (IS != Islands.Offsets.end() && FunctionOffset == IS->first) {
630 // This is a slightly complex code to decide which label to emit. We
631 // have 4 cases to handle: regular symbol, cold symbol, regular or cold
632 // symbol being emitted on behalf of an external function.
633 if (!OnBehalfOf) {
634 if (!EmitColdPart) {
635 LLVM_DEBUG(dbgs() << "BOLT-DEBUG: emitted label "
636 << IS->second->getName() << " at offset 0x"
637 << Twine::utohexstr(IS->first) << '\n');
638 if (IS->second->isUndefined())
639 Streamer.emitLabel(Symbol: IS->second);
640 else
641 assert(BF.hasName(std::string(IS->second->getName())));
642 } else if (Islands.ColdSymbols.count(Val: IS->second) != 0) {
643 LLVM_DEBUG(dbgs()
644 << "BOLT-DEBUG: emitted label "
645 << Islands.ColdSymbols[IS->second]->getName() << '\n');
646 if (Islands.ColdSymbols[IS->second]->isUndefined())
647 Streamer.emitLabel(Symbol: Islands.ColdSymbols[IS->second]);
648 }
649 } else {
650 if (!EmitColdPart) {
651 if (MCSymbol *Sym = Islands.Proxies[OnBehalfOf][IS->second]) {
652 LLVM_DEBUG(dbgs() << "BOLT-DEBUG: emitted label "
653 << Sym->getName() << '\n');
654 Streamer.emitLabel(Symbol: Sym);
655 }
656 } else if (MCSymbol *Sym =
657 Islands.ColdProxies[OnBehalfOf][IS->second]) {
658 LLVM_DEBUG(dbgs() << "BOLT-DEBUG: emitted label " << Sym->getName()
659 << '\n');
660 Streamer.emitLabel(Symbol: Sym);
661 }
662 }
663 ++IS;
664 }
665 }
666 assert(FunctionOffset <= EndOffset && "overflow error");
667 emitCI(FunctionOffset, EndOffset);
668 }
669 assert(IS == Islands.Offsets.end() && "some symbols were not emitted!");
670
671 if (OnBehalfOf)
672 return;
673 // Now emit constant islands from other functions that we may have used in
674 // this function.
675 for (BinaryFunction *ExternalFunc : Islands.Dependency)
676 emitConstantIslands(BF&: *ExternalFunc, EmitColdPart, OnBehalfOf: &BF);
677}
678
679SMLoc BinaryEmitter::emitLineInfo(const BinaryFunction &BF, SMLoc NewLoc,
680 SMLoc PrevLoc, bool FirstInstr,
681 MCSymbol *&InstrLabel) {
682 DWARFUnit *FunctionCU = BF.getDWARFUnit();
683 const DWARFDebugLine::LineTable *FunctionLineTable = BF.getDWARFLineTable();
684 assert(FunctionCU && "cannot emit line info for function without CU");
685
686 DebugLineTableRowRef RowReference = DebugLineTableRowRef::fromSMLoc(Loc: NewLoc);
687
688 // Check if no new line info needs to be emitted.
689 if (RowReference == DebugLineTableRowRef::NULL_ROW ||
690 NewLoc.getPointer() == PrevLoc.getPointer())
691 return PrevLoc;
692
693 unsigned CurrentFilenum = 0;
694 const DWARFDebugLine::LineTable *CurrentLineTable = FunctionLineTable;
695
696 // If the CU id from the current instruction location does not
697 // match the CU id from the current function, it means that we
698 // have come across some inlined code. We must look up the CU
699 // for the instruction's original function and get the line table
700 // from that.
701 const uint64_t FunctionUnitIndex = FunctionCU->getOffset();
702 const uint32_t CurrentUnitIndex = RowReference.DwCompileUnitIndex;
703 if (CurrentUnitIndex != FunctionUnitIndex) {
704 CurrentLineTable = BC.DwCtx->getLineTableForUnit(
705 U: BC.DwCtx->getCompileUnitForOffset(Offset: CurrentUnitIndex));
706 // Add filename from the inlined function to the current CU.
707 CurrentFilenum = BC.addDebugFilenameToUnit(
708 DestCUID: FunctionUnitIndex, SrcCUID: CurrentUnitIndex,
709 FileIndex: CurrentLineTable->Rows[RowReference.RowIndex - 1].File);
710 }
711
712 const DWARFDebugLine::Row &CurrentRow =
713 CurrentLineTable->Rows[RowReference.RowIndex - 1];
714 if (!CurrentFilenum)
715 CurrentFilenum = CurrentRow.File;
716
717 unsigned Flags = (DWARF2_FLAG_IS_STMT * CurrentRow.IsStmt) |
718 (DWARF2_FLAG_BASIC_BLOCK * CurrentRow.BasicBlock) |
719 (DWARF2_FLAG_PROLOGUE_END * CurrentRow.PrologueEnd) |
720 (DWARF2_FLAG_EPILOGUE_BEGIN * CurrentRow.EpilogueBegin);
721
722 // Always emit is_stmt at the beginning of function fragment.
723 if (FirstInstr)
724 Flags |= DWARF2_FLAG_IS_STMT;
725
726 BC.Ctx->setCurrentDwarfLoc(FileNum: CurrentFilenum, Line: CurrentRow.Line, Column: CurrentRow.Column,
727 Flags, Isa: CurrentRow.Isa, Discriminator: CurrentRow.Discriminator);
728 const MCDwarfLoc &DwarfLoc = BC.Ctx->getCurrentDwarfLoc();
729 BC.Ctx->clearDwarfLocSeen();
730
731 if (!InstrLabel)
732 InstrLabel = BC.Ctx->createTempSymbol();
733
734 BC.getDwarfLineTable(CUID: FunctionUnitIndex)
735 .getMCLineSections()
736 .addLineEntry(LineEntry: MCDwarfLineEntry(InstrLabel, DwarfLoc),
737 Sec: Streamer.getCurrentSectionOnly());
738
739 return NewLoc;
740}
741
742void BinaryEmitter::emitLineInfoEnd(const BinaryFunction &BF,
743 MCSymbol *FunctionEndLabel) {
744 DWARFUnit *FunctionCU = BF.getDWARFUnit();
745 assert(FunctionCU && "DWARF unit expected");
746 BC.Ctx->setCurrentDwarfLoc(FileNum: 0, Line: 0, Column: 0, DWARF2_FLAG_END_SEQUENCE, Isa: 0, Discriminator: 0);
747 const MCDwarfLoc &DwarfLoc = BC.Ctx->getCurrentDwarfLoc();
748 BC.Ctx->clearDwarfLocSeen();
749 BC.getDwarfLineTable(CUID: FunctionCU->getOffset())
750 .getMCLineSections()
751 .addLineEntry(LineEntry: MCDwarfLineEntry(FunctionEndLabel, DwarfLoc),
752 Sec: Streamer.getCurrentSectionOnly());
753}
754
755void BinaryEmitter::emitJumpTables(const BinaryFunction &BF) {
756 MCSection *ReadOnlySection = BC.MOFI->getReadOnlySection();
757 MCSection *ReadOnlyColdSection = BC.MOFI->getContext().getELFSection(
758 Section: ".rodata.cold", Type: ELF::SHT_PROGBITS, Flags: ELF::SHF_ALLOC);
759
760 if (!BF.hasJumpTables())
761 return;
762
763 if (opts::PrintJumpTables)
764 BC.outs() << "BOLT-INFO: jump tables for function " << BF << ":\n";
765
766 for (auto &JTI : BF.jumpTables()) {
767 JumpTable &JT = *JTI.second;
768 // Only emit shared jump tables once, when processing the first parent
769 if (JT.Parents.size() > 1 && JT.Parents[0] != &BF)
770 continue;
771 if (opts::PrintJumpTables)
772 JT.print(OS&: BC.outs());
773 if (opts::JumpTables == JTS_BASIC) {
774 JT.updateOriginal();
775 } else {
776 MCSection *HotSection, *ColdSection;
777 if (BF.isSimple()) {
778 HotSection = ReadOnlySection;
779 ColdSection = ReadOnlyColdSection;
780 } else {
781 HotSection = BF.hasProfile() ? ReadOnlySection : ReadOnlyColdSection;
782 ColdSection = HotSection;
783 }
784 emitJumpTable(JT, HotSection, ColdSection);
785 }
786 }
787}
788
789void BinaryEmitter::emitJumpTable(const JumpTable &JT, MCSection *HotSection,
790 MCSection *ColdSection) {
791 // Pre-process entries for aggressive splitting.
792 // Each label represents a separate switch table and gets its own count
793 // determining its destination.
794 std::map<MCSymbol *, uint64_t> LabelCounts;
795 if (opts::JumpTables > JTS_SPLIT && !JT.Counts.empty()) {
796 auto It = JT.Labels.find(x: 0);
797 assert(It != JT.Labels.end());
798 MCSymbol *CurrentLabel = It->second;
799 uint64_t CurrentLabelCount = 0;
800 for (unsigned Index = 0; Index < JT.Entries.size(); ++Index) {
801 auto LI = JT.Labels.find(x: Index * JT.EntrySize);
802 if (LI != JT.Labels.end()) {
803 LabelCounts[CurrentLabel] = CurrentLabelCount;
804 CurrentLabel = LI->second;
805 CurrentLabelCount = 0;
806 }
807 CurrentLabelCount += JT.Counts[Index].Count;
808 }
809 LabelCounts[CurrentLabel] = CurrentLabelCount;
810 } else {
811 Streamer.switchSection(Section: JT.Count > 0 ? HotSection : ColdSection);
812 Streamer.emitValueToAlignment(Alignment: Align(JT.EntrySize));
813 }
814 MCSymbol *JTLabel = nullptr;
815 uint64_t Offset = 0;
816 for (MCSymbol *Entry : JT.Entries) {
817 auto LI = JT.Labels.find(x: Offset);
818 if (LI == JT.Labels.end())
819 goto emitEntry;
820 JTLabel = LI->second;
821 LLVM_DEBUG({
822 dbgs() << "BOLT-DEBUG: emitting jump table " << JTLabel->getName()
823 << " (originally was at address 0x"
824 << Twine::utohexstr(JT.getAddress() + Offset)
825 << (Offset ? ") as part of larger jump table\n" : ")\n");
826 });
827 if (!LabelCounts.empty()) {
828 const uint64_t JTCount = LabelCounts[JTLabel];
829 LLVM_DEBUG(dbgs() << "BOLT-DEBUG: jump table count: " << JTCount << '\n');
830 Streamer.switchSection(Section: JTCount ? HotSection : ColdSection);
831 Streamer.emitValueToAlignment(Alignment: Align(JT.EntrySize));
832 }
833 // Emit all labels registered at the address of this jump table
834 // to sync with our global symbol table. We may have two labels
835 // registered at this address if one label was created via
836 // getOrCreateGlobalSymbol() (e.g. LEA instructions referencing
837 // this location) and another via getOrCreateJumpTable(). This
838 // creates a race where the symbols created by these two
839 // functions may or may not be the same, but they are both
840 // registered in our symbol table at the same address. By
841 // emitting them all here we make sure there is no ambiguity
842 // that depends on the order that these symbols were created, so
843 // whenever this address is referenced in the binary, it is
844 // certain to point to the jump table identified at this
845 // address.
846 if (BinaryData *BD = BC.getBinaryDataByName(Name: JTLabel->getName())) {
847 for (MCSymbol *S : BD->getSymbols())
848 Streamer.emitLabel(Symbol: S);
849 } else {
850 Streamer.emitLabel(Symbol: JTLabel);
851 }
852 emitEntry:
853 if (JT.Type == JumpTable::JTT_NORMAL) {
854 Streamer.emitSymbolValue(Sym: Entry, Size: JT.OutputEntrySize);
855 } else { // JTT_PIC
856 const MCSymbolRefExpr *JTExpr =
857 MCSymbolRefExpr::create(Symbol: JTLabel, Ctx&: Streamer.getContext());
858 const MCSymbolRefExpr *E =
859 MCSymbolRefExpr::create(Symbol: Entry, Ctx&: Streamer.getContext());
860 const MCBinaryExpr *Value =
861 MCBinaryExpr::createSub(LHS: E, RHS: JTExpr, Ctx&: Streamer.getContext());
862 Streamer.emitValue(Value, Size: JT.EntrySize);
863 }
864 Offset += JT.EntrySize;
865 }
866}
867
868void BinaryEmitter::emitCFIInstruction(const MCCFIInstruction &Inst) const {
869 switch (Inst.getOperation()) {
870 default:
871 llvm_unreachable("Unexpected instruction");
872 case MCCFIInstruction::OpDefCfaOffset:
873 Streamer.emitCFIDefCfaOffset(Offset: Inst.getOffset());
874 break;
875 case MCCFIInstruction::OpAdjustCfaOffset:
876 Streamer.emitCFIAdjustCfaOffset(Adjustment: Inst.getOffset());
877 break;
878 case MCCFIInstruction::OpDefCfa:
879 Streamer.emitCFIDefCfa(Register: Inst.getRegister(), Offset: Inst.getOffset());
880 break;
881 case MCCFIInstruction::OpDefCfaRegister:
882 Streamer.emitCFIDefCfaRegister(Register: Inst.getRegister());
883 break;
884 case MCCFIInstruction::OpOffset:
885 Streamer.emitCFIOffset(Register: Inst.getRegister(), Offset: Inst.getOffset());
886 break;
887 case MCCFIInstruction::OpRegister:
888 Streamer.emitCFIRegister(Register1: Inst.getRegister(), Register2: Inst.getRegister2());
889 break;
890 case MCCFIInstruction::OpWindowSave:
891 Streamer.emitCFIWindowSave();
892 break;
893 case MCCFIInstruction::OpNegateRAState:
894 Streamer.emitCFINegateRAState();
895 break;
896 case MCCFIInstruction::OpSameValue:
897 Streamer.emitCFISameValue(Register: Inst.getRegister());
898 break;
899 case MCCFIInstruction::OpGnuArgsSize:
900 Streamer.emitCFIGnuArgsSize(Size: Inst.getOffset());
901 break;
902 case MCCFIInstruction::OpEscape:
903 Streamer.AddComment(T: Inst.getComment());
904 Streamer.emitCFIEscape(Values: Inst.getValues());
905 break;
906 case MCCFIInstruction::OpRestore:
907 Streamer.emitCFIRestore(Register: Inst.getRegister());
908 break;
909 case MCCFIInstruction::OpUndefined:
910 Streamer.emitCFIUndefined(Register: Inst.getRegister());
911 break;
912 }
913}
914
915// The code is based on EHStreamer::emitExceptionTable().
916void BinaryEmitter::emitLSDA(BinaryFunction &BF, const FunctionFragment &FF) {
917 const BinaryFunction::CallSitesRange Sites =
918 BF.getCallSites(F: FF.getFragmentNum());
919 if (Sites.empty())
920 return;
921
922 Streamer.switchSection(Section: BC.MOFI->getLSDASection());
923
924 const unsigned TTypeEncoding = BF.getLSDATypeEncoding();
925 const unsigned TTypeEncodingSize = BC.getDWARFEncodingSize(Encoding: TTypeEncoding);
926 const uint16_t TTypeAlignment = 4;
927
928 // Type tables have to be aligned at 4 bytes.
929 Streamer.emitValueToAlignment(Alignment: Align(TTypeAlignment));
930
931 // Emit the LSDA label.
932 MCSymbol *LSDASymbol = BF.getLSDASymbol(F: FF.getFragmentNum());
933 assert(LSDASymbol && "no LSDA symbol set");
934 Streamer.emitLabel(Symbol: LSDASymbol);
935
936 // Corresponding FDE start.
937 const MCSymbol *StartSymbol = BF.getSymbol(Fragment: FF.getFragmentNum());
938
939 // Emit the LSDA header.
940
941 // If LPStart is omitted, then the start of the FDE is used as a base for
942 // landing pad displacements. Then, if a cold fragment starts with
943 // a landing pad, this means that the first landing pad offset will be 0.
944 // However, C++ runtime will treat 0 as if there is no landing pad, thus we
945 // cannot emit LP offset as 0.
946 //
947 // As a solution, for fixed-address binaries we set LPStart to 0, and for
948 // position-independent binaries we offset LP start by one byte.
949 bool NeedsLPAdjustment = false;
950 std::function<void(const MCSymbol *)> emitLandingPad;
951
952 // Check if there's a symbol associated with a landing pad fragment.
953 const MCSymbol *LPStartSymbol = BF.getLPStartSymbol(F: FF.getFragmentNum());
954 if (!LPStartSymbol) {
955 // Since landing pads are not in the same fragment, we fall back to emitting
956 // absolute addresses for this FDE.
957 if (opts::Verbosity >= 2) {
958 BC.outs() << "BOLT-INFO: falling back to generating absolute-address "
959 << "exception ranges for " << BF << '\n';
960 }
961
962 assert(BC.HasFixedLoadAddress &&
963 "Cannot emit absolute-address landing pads for PIE/DSO");
964
965 Streamer.emitIntValue(Value: dwarf::DW_EH_PE_udata4, Size: 1); // LPStart format
966 Streamer.emitIntValue(Value: 0, Size: 4); // LPStart
967 emitLandingPad = [&](const MCSymbol *LPSymbol) {
968 if (LPSymbol)
969 Streamer.emitSymbolValue(Sym: LPSymbol, Size: 4);
970 else
971 Streamer.emitIntValue(Value: 0, Size: 4);
972 };
973 } else {
974 std::optional<FragmentNum> LPFN = BF.getLPFragment(F: FF.getFragmentNum());
975 const FunctionFragment &LPFragment = BF.getLayout().getFragment(Num: *LPFN);
976 NeedsLPAdjustment =
977 (!LPFragment.empty() && LPFragment.front()->isLandingPad());
978
979 // Emit LPStart encoding and optionally LPStart.
980 if (NeedsLPAdjustment || LPStartSymbol != StartSymbol) {
981 Streamer.emitIntValue(Value: dwarf::DW_EH_PE_pcrel | dwarf::DW_EH_PE_sdata4, Size: 1);
982 MCSymbol *DotSymbol = BC.Ctx->createTempSymbol(Name: "LPBase");
983 Streamer.emitLabel(Symbol: DotSymbol);
984
985 const MCExpr *LPStartExpr = MCBinaryExpr::createSub(
986 LHS: MCSymbolRefExpr::create(Symbol: LPStartSymbol, Ctx&: *BC.Ctx),
987 RHS: MCSymbolRefExpr::create(Symbol: DotSymbol, Ctx&: *BC.Ctx), Ctx&: *BC.Ctx);
988 if (NeedsLPAdjustment)
989 LPStartExpr = MCBinaryExpr::createSub(
990 LHS: LPStartExpr, RHS: MCConstantExpr::create(Value: 1, Ctx&: *BC.Ctx), Ctx&: *BC.Ctx);
991 Streamer.emitValue(Value: LPStartExpr, Size: 4);
992 } else {
993 // DW_EH_PE_omit means FDE start (StartSymbol) will be used as LPStart.
994 Streamer.emitIntValue(Value: dwarf::DW_EH_PE_omit, Size: 1);
995 }
996 emitLandingPad = [&](const MCSymbol *LPSymbol) {
997 if (LPSymbol) {
998 const MCExpr *LPOffsetExpr = MCBinaryExpr::createSub(
999 LHS: MCSymbolRefExpr::create(Symbol: LPSymbol, Ctx&: *BC.Ctx),
1000 RHS: MCSymbolRefExpr::create(Symbol: LPStartSymbol, Ctx&: *BC.Ctx), Ctx&: *BC.Ctx);
1001 if (NeedsLPAdjustment)
1002 LPOffsetExpr = MCBinaryExpr::createAdd(
1003 LHS: LPOffsetExpr, RHS: MCConstantExpr::create(Value: 1, Ctx&: *BC.Ctx), Ctx&: *BC.Ctx);
1004 Streamer.emitULEB128Value(Value: LPOffsetExpr);
1005 } else {
1006 Streamer.emitULEB128IntValue(Value: 0);
1007 }
1008 };
1009 }
1010
1011 Streamer.emitIntValue(Value: TTypeEncoding, Size: 1); // TType format
1012
1013 MCSymbol *TTBaseLabel = nullptr;
1014 if (TTypeEncoding != dwarf::DW_EH_PE_omit) {
1015 TTBaseLabel = BC.Ctx->createTempSymbol(Name: "TTBase");
1016 MCSymbol *TTBaseRefLabel = BC.Ctx->createTempSymbol(Name: "TTBaseRef");
1017 Streamer.emitAbsoluteSymbolDiffAsULEB128(Hi: TTBaseLabel, Lo: TTBaseRefLabel);
1018 Streamer.emitLabel(Symbol: TTBaseRefLabel);
1019 }
1020
1021 // Emit encoding of entries in the call site table. The format is used for the
1022 // call site start, length, and corresponding landing pad.
1023 if (!LPStartSymbol)
1024 Streamer.emitIntValue(Value: dwarf::DW_EH_PE_sdata4, Size: 1);
1025 else
1026 Streamer.emitIntValue(Value: dwarf::DW_EH_PE_uleb128, Size: 1);
1027
1028 MCSymbol *CSTStartLabel = BC.Ctx->createTempSymbol(Name: "CSTStart");
1029 MCSymbol *CSTEndLabel = BC.Ctx->createTempSymbol(Name: "CSTEnd");
1030 Streamer.emitAbsoluteSymbolDiffAsULEB128(Hi: CSTEndLabel, Lo: CSTStartLabel);
1031
1032 Streamer.emitLabel(Symbol: CSTStartLabel);
1033 for (const auto &FragmentCallSite : Sites) {
1034 const BinaryFunction::CallSite &CallSite = FragmentCallSite.second;
1035 const MCSymbol *BeginLabel = CallSite.Start;
1036 const MCSymbol *EndLabel = CallSite.End;
1037
1038 assert(BeginLabel && "start EH label expected");
1039 assert(EndLabel && "end EH label expected");
1040
1041 // Start of the range is emitted relative to the start of current
1042 // function split part.
1043 if (!LPStartSymbol) {
1044 Streamer.emitAbsoluteSymbolDiff(Hi: BeginLabel, Lo: StartSymbol, Size: 4);
1045 Streamer.emitAbsoluteSymbolDiff(Hi: EndLabel, Lo: BeginLabel, Size: 4);
1046 } else {
1047 Streamer.emitAbsoluteSymbolDiffAsULEB128(Hi: BeginLabel, Lo: StartSymbol);
1048 Streamer.emitAbsoluteSymbolDiffAsULEB128(Hi: EndLabel, Lo: BeginLabel);
1049 }
1050 emitLandingPad(CallSite.LP);
1051 Streamer.emitULEB128IntValue(Value: CallSite.Action);
1052 }
1053 Streamer.emitLabel(Symbol: CSTEndLabel);
1054
1055 // Write out action, type, and type index tables at the end.
1056 //
1057 // For action and type index tables there's no need to change the original
1058 // table format unless we are doing function splitting, in which case we can
1059 // split and optimize the tables.
1060 //
1061 // For type table we (re-)encode the table using TTypeEncoding matching
1062 // the current assembler mode.
1063 for (uint8_t const &Byte : BF.getLSDAActionTable())
1064 Streamer.emitIntValue(Value: Byte, Size: 1);
1065
1066 const BinaryFunction::LSDATypeTableTy &TypeTable =
1067 (TTypeEncoding & dwarf::DW_EH_PE_indirect) ? BF.getLSDATypeAddressTable()
1068 : BF.getLSDATypeTable();
1069 assert(TypeTable.size() == BF.getLSDATypeTable().size() &&
1070 "indirect type table size mismatch");
1071
1072 Streamer.emitValueToAlignment(Alignment: Align(TTypeAlignment));
1073
1074 for (int Index = TypeTable.size() - 1; Index >= 0; --Index) {
1075 const uint64_t TypeAddress = TypeTable[Index];
1076 switch (TTypeEncoding & 0x70) {
1077 default:
1078 llvm_unreachable("unsupported TTypeEncoding");
1079 case dwarf::DW_EH_PE_absptr:
1080 Streamer.emitIntValue(Value: TypeAddress, Size: TTypeEncodingSize);
1081 break;
1082 case dwarf::DW_EH_PE_pcrel: {
1083 if (TypeAddress) {
1084 const MCSymbol *TypeSymbol =
1085 BC.getOrCreateGlobalSymbol(Address: TypeAddress, Prefix: "TI", Size: 0, Alignment: TTypeAlignment);
1086 MCSymbol *DotSymbol = BC.Ctx->createNamedTempSymbol();
1087 Streamer.emitLabel(Symbol: DotSymbol);
1088 const MCBinaryExpr *SubDotExpr = MCBinaryExpr::createSub(
1089 LHS: MCSymbolRefExpr::create(Symbol: TypeSymbol, Ctx&: *BC.Ctx),
1090 RHS: MCSymbolRefExpr::create(Symbol: DotSymbol, Ctx&: *BC.Ctx), Ctx&: *BC.Ctx);
1091 Streamer.emitValue(Value: SubDotExpr, Size: TTypeEncodingSize);
1092 } else {
1093 Streamer.emitIntValue(Value: 0, Size: TTypeEncodingSize);
1094 }
1095 break;
1096 }
1097 }
1098 }
1099
1100 if (TTypeEncoding != dwarf::DW_EH_PE_omit)
1101 Streamer.emitLabel(Symbol: TTBaseLabel);
1102
1103 for (uint8_t const &Byte : BF.getLSDATypeIndexTable())
1104 Streamer.emitIntValue(Value: Byte, Size: 1);
1105}
1106
1107void BinaryEmitter::emitDebugLineInfoForOriginalFunctions() {
1108 // If a function is in a CU containing at least one processed function, we
1109 // have to rewrite the whole line table for that CU. For unprocessed functions
1110 // we use data from the input line table.
1111 for (auto &It : BC.getBinaryFunctions()) {
1112 const BinaryFunction &Function = It.second;
1113
1114 // If the function was emitted, its line info was emitted with it.
1115 if (Function.isEmitted())
1116 continue;
1117
1118 const DWARFDebugLine::LineTable *LineTable = Function.getDWARFLineTable();
1119 if (!LineTable)
1120 continue; // nothing to update for this function
1121
1122 const uint64_t Address = Function.getAddress();
1123 std::vector<uint32_t> Results;
1124 if (!LineTable->lookupAddressRange(
1125 Address: {.Address: Address, .SectionIndex: object::SectionedAddress::UndefSection},
1126 Size: Function.getSize(), Result&: Results))
1127 continue;
1128
1129 if (Results.empty())
1130 continue;
1131
1132 // The first row returned could be the last row matching the start address.
1133 // Find the first row with the same address that is not the end of the
1134 // sequence.
1135 uint64_t FirstRow = Results.front();
1136 while (FirstRow > 0) {
1137 const DWARFDebugLine::Row &PrevRow = LineTable->Rows[FirstRow - 1];
1138 if (PrevRow.Address.Address != Address || PrevRow.EndSequence)
1139 break;
1140 --FirstRow;
1141 }
1142
1143 const uint64_t EndOfSequenceAddress =
1144 Function.getAddress() + Function.getMaxSize();
1145 BC.getDwarfLineTable(CUID: Function.getDWARFUnit()->getOffset())
1146 .addLineTableSequence(Table: LineTable, FirstRow, LastRow: Results.back(),
1147 EndOfSequenceAddress);
1148 }
1149
1150 // For units that are completely unprocessed, use original debug line contents
1151 // eliminating the need to regenerate line info program.
1152 emitDebugLineInfoForUnprocessedCUs();
1153}
1154
1155void BinaryEmitter::emitDebugLineInfoForUnprocessedCUs() {
1156 // Sorted list of section offsets provides boundaries for section fragments,
1157 // where each fragment is the unit's contribution to debug line section.
1158 std::vector<uint64_t> StmtListOffsets;
1159 StmtListOffsets.reserve(n: BC.DwCtx->getNumCompileUnits());
1160 for (const std::unique_ptr<DWARFUnit> &CU : BC.DwCtx->compile_units()) {
1161 DWARFDie CUDie = CU->getUnitDIE();
1162 auto StmtList = dwarf::toSectionOffset(V: CUDie.find(Attr: dwarf::DW_AT_stmt_list));
1163 if (!StmtList)
1164 continue;
1165
1166 StmtListOffsets.push_back(x: *StmtList);
1167 }
1168 llvm::sort(C&: StmtListOffsets);
1169
1170 // For each CU that was not processed, emit its line info as a binary blob.
1171 for (const std::unique_ptr<DWARFUnit> &CU : BC.DwCtx->compile_units()) {
1172 if (BC.ProcessedCUs.count(x: CU.get()))
1173 continue;
1174
1175 DWARFDie CUDie = CU->getUnitDIE();
1176 auto StmtList = dwarf::toSectionOffset(V: CUDie.find(Attr: dwarf::DW_AT_stmt_list));
1177 if (!StmtList)
1178 continue;
1179
1180 StringRef DebugLineContents = CU->getLineSection().Data;
1181
1182 const uint64_t Begin = *StmtList;
1183
1184 // Statement list ends where the next unit contribution begins, or at the
1185 // end of the section.
1186 auto It = llvm::upper_bound(Range&: StmtListOffsets, Value: Begin);
1187 const uint64_t End =
1188 It == StmtListOffsets.end() ? DebugLineContents.size() : *It;
1189
1190 BC.getDwarfLineTable(CUID: CU->getOffset())
1191 .addRawContents(DebugLineContents: DebugLineContents.slice(Start: Begin, End));
1192 }
1193}
1194
1195void BinaryEmitter::emitDataSections(StringRef OrgSecPrefix) {
1196 for (BinarySection &Section : BC.sections()) {
1197 if (!Section.hasRelocations())
1198 continue;
1199
1200 StringRef Prefix = Section.hasSectionRef() ? OrgSecPrefix : "";
1201 Section.emitAsData(Streamer, SectionName: Prefix + Section.getName());
1202 Section.clearRelocations();
1203 }
1204}
1205
1206namespace llvm {
1207namespace bolt {
1208
1209void emitBinaryContext(MCStreamer &Streamer, BinaryContext &BC,
1210 StringRef OrgSecPrefix) {
1211 BinaryEmitter(Streamer, BC).emitAll(OrgSecPrefix);
1212}
1213
1214void emitFunctionBody(MCStreamer &Streamer, BinaryFunction &BF,
1215 FunctionFragment &FF, bool EmitCodeOnly) {
1216 BinaryEmitter(Streamer, BF.getBinaryContext())
1217 .emitFunctionBody(BF, FF, EmitCodeOnly);
1218}
1219
1220} // namespace bolt
1221} // namespace llvm
1222

source code of bolt/lib/Core/BinaryEmitter.cpp