1//===- bolt/Core/BinaryEmitter.cpp - Emit code and data -------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the collection of functions and classes used for
10// emission of code and data into object/binary file.
11//
12//===----------------------------------------------------------------------===//
13
14#include "bolt/Core/BinaryEmitter.h"
15#include "bolt/Core/BinaryContext.h"
16#include "bolt/Core/BinaryFunction.h"
17#include "bolt/Core/DebugData.h"
18#include "bolt/Core/FunctionLayout.h"
19#include "bolt/Utils/CommandLineOpts.h"
20#include "bolt/Utils/Utils.h"
21#include "llvm/DebugInfo/DWARF/DWARFCompileUnit.h"
22#include "llvm/MC/MCSection.h"
23#include "llvm/MC/MCStreamer.h"
24#include "llvm/Support/CommandLine.h"
25#include "llvm/Support/LEB128.h"
26#include "llvm/Support/SMLoc.h"
27
28#define DEBUG_TYPE "bolt"
29
30using namespace llvm;
31using namespace bolt;
32
33namespace opts {
34
35extern cl::opt<JumpTableSupportLevel> JumpTables;
36extern cl::opt<bool> PreserveBlocksAlignment;
37
38cl::opt<bool> AlignBlocks("align-blocks", cl::desc("align basic blocks"),
39 cl::cat(BoltOptCategory));
40
41cl::opt<MacroFusionType>
42AlignMacroOpFusion("align-macro-fusion",
43 cl::desc("fix instruction alignment for macro-fusion (x86 relocation mode)"),
44 cl::init(Val: MFT_HOT),
45 cl::values(clEnumValN(MFT_NONE, "none",
46 "do not insert alignment no-ops for macro-fusion"),
47 clEnumValN(MFT_HOT, "hot",
48 "only insert alignment no-ops on hot execution paths (default)"),
49 clEnumValN(MFT_ALL, "all",
50 "always align instructions to allow macro-fusion")),
51 cl::ZeroOrMore,
52 cl::cat(BoltRelocCategory));
53
54static cl::list<std::string>
55BreakFunctionNames("break-funcs",
56 cl::CommaSeparated,
57 cl::desc("list of functions to core dump on (debugging)"),
58 cl::value_desc("func1,func2,func3,..."),
59 cl::Hidden,
60 cl::cat(BoltCategory));
61
62static cl::list<std::string>
63FunctionPadSpec("pad-funcs",
64 cl::CommaSeparated,
65 cl::desc("list of functions to pad with amount of bytes"),
66 cl::value_desc("func1:pad1,func2:pad2,func3:pad3,..."),
67 cl::Hidden,
68 cl::cat(BoltCategory));
69
70static cl::opt<bool> MarkFuncs(
71 "mark-funcs",
72 cl::desc("mark function boundaries with break instruction to make "
73 "sure we accidentally don't cross them"),
74 cl::ReallyHidden, cl::cat(BoltCategory));
75
76static cl::opt<bool> PrintJumpTables("print-jump-tables",
77 cl::desc("print jump tables"), cl::Hidden,
78 cl::cat(BoltCategory));
79
80static cl::opt<bool>
81X86AlignBranchBoundaryHotOnly("x86-align-branch-boundary-hot-only",
82 cl::desc("only apply branch boundary alignment in hot code"),
83 cl::init(Val: true),
84 cl::cat(BoltOptCategory));
85
86size_t padFunction(const BinaryFunction &Function) {
87 static std::map<std::string, size_t> FunctionPadding;
88
89 if (FunctionPadding.empty() && !FunctionPadSpec.empty()) {
90 for (std::string &Spec : FunctionPadSpec) {
91 size_t N = Spec.find(c: ':');
92 if (N == std::string::npos)
93 continue;
94 std::string Name = Spec.substr(pos: 0, n: N);
95 size_t Padding = std::stoull(str: Spec.substr(pos: N + 1));
96 FunctionPadding[Name] = Padding;
97 }
98 }
99
100 for (auto &FPI : FunctionPadding) {
101 std::string Name = FPI.first;
102 size_t Padding = FPI.second;
103 if (Function.hasNameRegex(NameRegex: Name))
104 return Padding;
105 }
106
107 return 0;
108}
109
110} // namespace opts
111
112namespace {
113using JumpTable = bolt::JumpTable;
114
115class BinaryEmitter {
116private:
117 BinaryEmitter(const BinaryEmitter &) = delete;
118 BinaryEmitter &operator=(const BinaryEmitter &) = delete;
119
120 MCStreamer &Streamer;
121 BinaryContext &BC;
122
123public:
124 BinaryEmitter(MCStreamer &Streamer, BinaryContext &BC)
125 : Streamer(Streamer), BC(BC) {}
126
127 /// Emit all code and data.
128 void emitAll(StringRef OrgSecPrefix);
129
130 /// Emit function code. The caller is responsible for emitting function
131 /// symbol(s) and setting the section to emit the code to.
132 void emitFunctionBody(BinaryFunction &BF, FunctionFragment &FF,
133 bool EmitCodeOnly = false);
134
135private:
136 /// Emit function code.
137 void emitFunctions();
138
139 /// Emit a single function.
140 bool emitFunction(BinaryFunction &BF, FunctionFragment &FF);
141
142 /// Helper for emitFunctionBody to write data inside a function
143 /// (used for AArch64)
144 void emitConstantIslands(BinaryFunction &BF, bool EmitColdPart,
145 BinaryFunction *OnBehalfOf = nullptr);
146
147 /// Emit jump tables for the function.
148 void emitJumpTables(const BinaryFunction &BF);
149
150 /// Emit jump table data. Callee supplies sections for the data.
151 void emitJumpTable(const JumpTable &JT, MCSection *HotSection,
152 MCSection *ColdSection);
153
154 void emitCFIInstruction(const MCCFIInstruction &Inst) const;
155
156 /// Emit exception handling ranges for the function.
157 void emitLSDA(BinaryFunction &BF, const FunctionFragment &FF);
158
159 /// Emit line number information corresponding to \p NewLoc. \p PrevLoc
160 /// provides a context for de-duplication of line number info.
161 /// \p FirstInstr indicates if \p NewLoc represents the first instruction
162 /// in a sequence, such as a function fragment.
163 ///
164 /// If \p NewLoc location matches \p PrevLoc, no new line number entry will be
165 /// created and the function will return \p PrevLoc while \p InstrLabel will
166 /// be ignored. Otherwise, the caller should use \p InstrLabel to mark the
167 /// corresponding instruction by emitting \p InstrLabel before it.
168 /// If \p InstrLabel is set by the caller, its value will be used with \p
169 /// \p NewLoc. If it was nullptr on entry, it will be populated with a pointer
170 /// to a new temp symbol used with \p NewLoc.
171 ///
172 /// Return new current location which is either \p NewLoc or \p PrevLoc.
173 SMLoc emitLineInfo(const BinaryFunction &BF, SMLoc NewLoc, SMLoc PrevLoc,
174 bool FirstInstr, MCSymbol *&InstrLabel);
175
176 /// Use \p FunctionEndSymbol to mark the end of the line info sequence.
177 /// Note that it does not automatically result in the insertion of the EOS
178 /// marker in the line table program, but provides one to the DWARF generator
179 /// when it needs it.
180 void emitLineInfoEnd(const BinaryFunction &BF, MCSymbol *FunctionEndSymbol);
181
182 /// Emit debug line info for unprocessed functions from CUs that include
183 /// emitted functions.
184 void emitDebugLineInfoForOriginalFunctions();
185
186 /// Emit debug line for CUs that were not modified.
187 void emitDebugLineInfoForUnprocessedCUs();
188
189 /// Emit data sections that have code references in them.
190 void emitDataSections(StringRef OrgSecPrefix);
191};
192
193} // anonymous namespace
194
195void BinaryEmitter::emitAll(StringRef OrgSecPrefix) {
196 Streamer.initSections(NoExecStack: false, STI: *BC.STI);
197
198 if (opts::UpdateDebugSections && BC.isELF()) {
199 // Force the emission of debug line info into allocatable section to ensure
200 // JITLink will process it.
201 //
202 // NB: on MachO all sections are required for execution, hence no need
203 // to change flags/attributes.
204 MCSectionELF *ELFDwarfLineSection =
205 static_cast<MCSectionELF *>(BC.MOFI->getDwarfLineSection());
206 ELFDwarfLineSection->setFlags(ELF::SHF_ALLOC);
207 MCSectionELF *ELFDwarfLineStrSection =
208 static_cast<MCSectionELF *>(BC.MOFI->getDwarfLineStrSection());
209 ELFDwarfLineStrSection->setFlags(ELF::SHF_ALLOC);
210 }
211
212 if (RuntimeLibrary *RtLibrary = BC.getRuntimeLibrary())
213 RtLibrary->emitBinary(BC, Streamer);
214
215 BC.getTextSection()->setAlignment(Align(opts::AlignText));
216
217 emitFunctions();
218
219 if (opts::UpdateDebugSections) {
220 emitDebugLineInfoForOriginalFunctions();
221 DwarfLineTable::emit(BC, Streamer);
222 }
223
224 emitDataSections(OrgSecPrefix);
225
226 // TODO Enable for Mach-O once BinaryContext::getDataSection supports it.
227 if (BC.isELF())
228 AddressMap::emit(Streamer, BC);
229}
230
231void BinaryEmitter::emitFunctions() {
232 auto emit = [&](const std::vector<BinaryFunction *> &Functions) {
233 const bool HasProfile = BC.NumProfiledFuncs > 0;
234 const bool OriginalAllowAutoPadding = Streamer.getAllowAutoPadding();
235 for (BinaryFunction *Function : Functions) {
236 if (!BC.shouldEmit(Function: *Function))
237 continue;
238
239 LLVM_DEBUG(dbgs() << "BOLT: generating code for function \"" << *Function
240 << "\" : " << Function->getFunctionNumber() << '\n');
241
242 // Was any part of the function emitted.
243 bool Emitted = false;
244
245 // Turn off Intel JCC Erratum mitigation for cold code if requested
246 if (HasProfile && opts::X86AlignBranchBoundaryHotOnly &&
247 !Function->hasValidProfile())
248 Streamer.setAllowAutoPadding(false);
249
250 FunctionLayout &Layout = Function->getLayout();
251 Emitted |= emitFunction(BF&: *Function, FF&: Layout.getMainFragment());
252
253 if (Function->isSplit()) {
254 if (opts::X86AlignBranchBoundaryHotOnly)
255 Streamer.setAllowAutoPadding(false);
256
257 assert((Layout.fragment_size() == 1 || Function->isSimple()) &&
258 "Only simple functions can have fragments");
259 for (FunctionFragment &FF : Layout.getSplitFragments()) {
260 // Skip empty fragments so no symbols and sections for empty fragments
261 // are generated
262 if (FF.empty() && !Function->hasConstantIsland())
263 continue;
264 Emitted |= emitFunction(BF&: *Function, FF);
265 }
266 }
267
268 Streamer.setAllowAutoPadding(OriginalAllowAutoPadding);
269
270 if (Emitted)
271 Function->setEmitted(/*KeepCFG=*/opts::PrintCacheMetrics);
272 }
273 };
274
275 // Mark the start of hot text.
276 if (opts::HotText) {
277 Streamer.switchSection(Section: BC.getTextSection());
278 Streamer.emitLabel(Symbol: BC.getHotTextStartSymbol());
279 }
280
281 // Emit functions in sorted order.
282 std::vector<BinaryFunction *> SortedFunctions = BC.getSortedFunctions();
283 emit(SortedFunctions);
284
285 // Emit functions added by BOLT.
286 emit(BC.getInjectedBinaryFunctions());
287
288 // Mark the end of hot text.
289 if (opts::HotText) {
290 if (BC.HasWarmSection)
291 Streamer.switchSection(Section: BC.getCodeSection(SectionName: BC.getWarmCodeSectionName()));
292 else
293 Streamer.switchSection(Section: BC.getTextSection());
294 Streamer.emitLabel(Symbol: BC.getHotTextEndSymbol());
295 }
296}
297
298bool BinaryEmitter::emitFunction(BinaryFunction &Function,
299 FunctionFragment &FF) {
300 if (Function.size() == 0 && !Function.hasIslandsInfo())
301 return false;
302
303 if (Function.getState() == BinaryFunction::State::Empty)
304 return false;
305
306 // Avoid emitting function without instructions when overwriting the original
307 // function in-place. Otherwise, emit the empty function to define the symbol.
308 if (!BC.HasRelocations && !Function.hasNonPseudoInstructions())
309 return false;
310
311 MCSection *Section =
312 BC.getCodeSection(SectionName: Function.getCodeSectionName(Fragment: FF.getFragmentNum()));
313 Streamer.switchSection(Section);
314 Section->setHasInstructions(true);
315 BC.Ctx->addGenDwarfSection(Sec: Section);
316
317 if (BC.HasRelocations) {
318 // Set section alignment to at least maximum possible object alignment.
319 // We need this to support LongJmp and other passes that calculates
320 // tentative layout.
321 Section->ensureMinAlignment(MinAlignment: Align(opts::AlignFunctions));
322
323 Streamer.emitCodeAlignment(Alignment: Function.getMinAlign(), STI: &*BC.STI);
324 uint16_t MaxAlignBytes = FF.isSplitFragment()
325 ? Function.getMaxColdAlignmentBytes()
326 : Function.getMaxAlignmentBytes();
327 if (MaxAlignBytes > 0)
328 Streamer.emitCodeAlignment(Alignment: Function.getAlign(), STI: &*BC.STI, MaxBytesToEmit: MaxAlignBytes);
329 } else {
330 Streamer.emitCodeAlignment(Alignment: Function.getAlign(), STI: &*BC.STI);
331 }
332
333 MCContext &Context = Streamer.getContext();
334 const MCAsmInfo *MAI = Context.getAsmInfo();
335
336 MCSymbol *const StartSymbol = Function.getSymbol(Fragment: FF.getFragmentNum());
337
338 // Emit all symbols associated with the main function entry.
339 if (FF.isMainFragment()) {
340 for (MCSymbol *Symbol : Function.getSymbols()) {
341 Streamer.emitSymbolAttribute(Symbol, Attribute: MCSA_ELF_TypeFunction);
342 Streamer.emitLabel(Symbol);
343 }
344 } else {
345 Streamer.emitSymbolAttribute(Symbol: StartSymbol, Attribute: MCSA_ELF_TypeFunction);
346 Streamer.emitLabel(Symbol: StartSymbol);
347 }
348
349 // Emit CFI start
350 if (Function.hasCFI()) {
351 Streamer.emitCFIStartProc(/*IsSimple=*/false);
352 if (Function.getPersonalityFunction() != nullptr)
353 Streamer.emitCFIPersonality(Sym: Function.getPersonalityFunction(),
354 Encoding: Function.getPersonalityEncoding());
355 MCSymbol *LSDASymbol = Function.getLSDASymbol(F: FF.getFragmentNum());
356 if (LSDASymbol)
357 Streamer.emitCFILsda(Sym: LSDASymbol, Encoding: BC.LSDAEncoding);
358 else
359 Streamer.emitCFILsda(Sym: 0, Encoding: dwarf::DW_EH_PE_omit);
360 // Emit CFI instructions relative to the CIE
361 for (const MCCFIInstruction &CFIInstr : Function.cie()) {
362 // Only write CIE CFI insns that LLVM will not already emit
363 const std::vector<MCCFIInstruction> &FrameInstrs =
364 MAI->getInitialFrameState();
365 if (!llvm::is_contained(Range: FrameInstrs, Element: CFIInstr))
366 emitCFIInstruction(Inst: CFIInstr);
367 }
368 }
369
370 assert((Function.empty() || !(*Function.begin()).isCold()) &&
371 "first basic block should never be cold");
372
373 // Emit UD2 at the beginning if requested by user.
374 if (!opts::BreakFunctionNames.empty()) {
375 for (std::string &Name : opts::BreakFunctionNames) {
376 if (Function.hasNameRegex(NameRegex: Name)) {
377 Streamer.emitIntValue(Value: 0x0B0F, Size: 2); // UD2: 0F 0B
378 break;
379 }
380 }
381 }
382
383 // Emit code.
384 emitFunctionBody(BF&: Function, FF, /*EmitCodeOnly=*/false);
385
386 // Emit padding if requested.
387 if (size_t Padding = opts::padFunction(Function)) {
388 LLVM_DEBUG(dbgs() << "BOLT-DEBUG: padding function " << Function << " with "
389 << Padding << " bytes\n");
390 Streamer.emitFill(NumBytes: Padding, FillValue: MAI->getTextAlignFillValue());
391 }
392
393 if (opts::MarkFuncs)
394 Streamer.emitBytes(Data: BC.MIB->getTrapFillValue());
395
396 // Emit CFI end
397 if (Function.hasCFI())
398 Streamer.emitCFIEndProc();
399
400 MCSymbol *EndSymbol = Function.getFunctionEndLabel(Fragment: FF.getFragmentNum());
401 Streamer.emitLabel(Symbol: EndSymbol);
402
403 if (MAI->hasDotTypeDotSizeDirective()) {
404 const MCExpr *SizeExpr = MCBinaryExpr::createSub(
405 LHS: MCSymbolRefExpr::create(Symbol: EndSymbol, Ctx&: Context),
406 RHS: MCSymbolRefExpr::create(Symbol: StartSymbol, Ctx&: Context), Ctx&: Context);
407 Streamer.emitELFSize(Symbol: StartSymbol, Value: SizeExpr);
408 }
409
410 if (opts::UpdateDebugSections && Function.getDWARFUnit())
411 emitLineInfoEnd(BF: Function, FunctionEndSymbol: EndSymbol);
412
413 // Exception handling info for the function.
414 emitLSDA(BF&: Function, FF);
415
416 if (FF.isMainFragment() && opts::JumpTables > JTS_NONE)
417 emitJumpTables(BF: Function);
418
419 return true;
420}
421
422void BinaryEmitter::emitFunctionBody(BinaryFunction &BF, FunctionFragment &FF,
423 bool EmitCodeOnly) {
424 if (!EmitCodeOnly && FF.isSplitFragment() && BF.hasConstantIsland()) {
425 assert(BF.getLayout().isHotColdSplit() &&
426 "Constant island support only with hot/cold split");
427 BF.duplicateConstantIslands();
428 }
429
430 if (!FF.empty() && FF.front()->isLandingPad()) {
431 assert(!FF.front()->isEntryPoint() &&
432 "Landing pad cannot be entry point of function");
433 // If the first block of the fragment is a landing pad, it's offset from the
434 // start of the area that the corresponding LSDA describes is zero. In this
435 // case, the call site entries in that LSDA have 0 as offset to the landing
436 // pad, which the runtime interprets as "no handler". To prevent this,
437 // insert some padding.
438 Streamer.emitBytes(Data: BC.MIB->getTrapFillValue());
439 }
440
441 // Track the first emitted instruction with debug info.
442 bool FirstInstr = true;
443 for (BinaryBasicBlock *const BB : FF) {
444 if ((opts::AlignBlocks || opts::PreserveBlocksAlignment) &&
445 BB->getAlignment() > 1)
446 Streamer.emitCodeAlignment(Alignment: BB->getAlign(), STI: &*BC.STI,
447 MaxBytesToEmit: BB->getAlignmentMaxBytes());
448 Streamer.emitLabel(Symbol: BB->getLabel());
449 if (!EmitCodeOnly) {
450 if (MCSymbol *EntrySymbol = BF.getSecondaryEntryPointSymbol(BB: *BB))
451 Streamer.emitLabel(Symbol: EntrySymbol);
452 }
453
454 // Check if special alignment for macro-fusion is needed.
455 bool MayNeedMacroFusionAlignment =
456 (opts::AlignMacroOpFusion == MFT_ALL) ||
457 (opts::AlignMacroOpFusion == MFT_HOT && BB->getKnownExecutionCount());
458 BinaryBasicBlock::const_iterator MacroFusionPair;
459 if (MayNeedMacroFusionAlignment) {
460 MacroFusionPair = BB->getMacroOpFusionPair();
461 if (MacroFusionPair == BB->end())
462 MayNeedMacroFusionAlignment = false;
463 }
464
465 SMLoc LastLocSeen;
466 // Remember if the last instruction emitted was a prefix.
467 bool LastIsPrefix = false;
468 for (auto I = BB->begin(), E = BB->end(); I != E; ++I) {
469 MCInst &Instr = *I;
470
471 if (EmitCodeOnly && BC.MIB->isPseudo(Inst: Instr))
472 continue;
473
474 // Handle pseudo instructions.
475 if (BC.MIB->isCFI(Inst: Instr)) {
476 emitCFIInstruction(Inst: *BF.getCFIFor(Instr));
477 continue;
478 }
479
480 // Handle macro-fusion alignment. If we emitted a prefix as
481 // the last instruction, we should've already emitted the associated
482 // alignment hint, so don't emit it twice.
483 if (MayNeedMacroFusionAlignment && !LastIsPrefix &&
484 I == MacroFusionPair) {
485 // This assumes the second instruction in the macro-op pair will get
486 // assigned to its own MCRelaxableFragment. Since all JCC instructions
487 // are relaxable, we should be safe.
488 }
489
490 if (!EmitCodeOnly) {
491 // A symbol to be emitted before the instruction to mark its location.
492 MCSymbol *InstrLabel = BC.MIB->getInstLabel(Inst: Instr);
493
494 if (opts::UpdateDebugSections && BF.getDWARFUnit()) {
495 LastLocSeen = emitLineInfo(BF, NewLoc: Instr.getLoc(), PrevLoc: LastLocSeen,
496 FirstInstr, InstrLabel);
497 FirstInstr = false;
498 }
499
500 // Prepare to tag this location with a label if we need to keep track of
501 // the location of calls/returns for BOLT address translation maps
502 if (BF.requiresAddressTranslation() && BC.MIB->getOffset(Inst: Instr)) {
503 const uint32_t Offset = *BC.MIB->getOffset(Inst: Instr);
504 if (!InstrLabel)
505 InstrLabel = BC.Ctx->createTempSymbol();
506 BB->getLocSyms().emplace_back(args: Offset, args&: InstrLabel);
507 }
508
509 if (InstrLabel)
510 Streamer.emitLabel(Symbol: InstrLabel);
511 }
512
513 // Emit sized NOPs via MCAsmBackend::writeNopData() interface on x86.
514 // This is a workaround for invalid NOPs handling by asm/disasm layer.
515 if (BC.isX86() && BC.MIB->isNoop(Inst: Instr)) {
516 if (std::optional<uint32_t> Size = BC.MIB->getSize(Inst: Instr)) {
517 SmallString<15> Code;
518 raw_svector_ostream VecOS(Code);
519 BC.MAB->writeNopData(OS&: VecOS, Count: *Size, STI: BC.STI.get());
520 Streamer.emitBytes(Data: Code);
521 continue;
522 }
523 }
524
525 Streamer.emitInstruction(Inst: Instr, STI: *BC.STI);
526 LastIsPrefix = BC.MIB->isPrefix(Inst: Instr);
527 }
528 }
529
530 if (!EmitCodeOnly)
531 emitConstantIslands(BF, EmitColdPart: FF.isSplitFragment());
532}
533
534void BinaryEmitter::emitConstantIslands(BinaryFunction &BF, bool EmitColdPart,
535 BinaryFunction *OnBehalfOf) {
536 if (!BF.hasIslandsInfo())
537 return;
538
539 BinaryFunction::IslandInfo &Islands = BF.getIslandInfo();
540 if (Islands.DataOffsets.empty() && Islands.Dependency.empty())
541 return;
542
543 // AArch64 requires CI to be aligned to 8 bytes due to access instructions
544 // restrictions. E.g. the ldr with imm, where imm must be aligned to 8 bytes.
545 const uint16_t Alignment = OnBehalfOf
546 ? OnBehalfOf->getConstantIslandAlignment()
547 : BF.getConstantIslandAlignment();
548 Streamer.emitCodeAlignment(Alignment: Align(Alignment), STI: &*BC.STI);
549
550 if (!OnBehalfOf) {
551 if (!EmitColdPart)
552 Streamer.emitLabel(Symbol: BF.getFunctionConstantIslandLabel());
553 else
554 Streamer.emitLabel(Symbol: BF.getFunctionColdConstantIslandLabel());
555 }
556
557 assert((!OnBehalfOf || Islands.Proxies[OnBehalfOf].size() > 0) &&
558 "spurious OnBehalfOf constant island emission");
559
560 assert(!BF.isInjected() &&
561 "injected functions should not have constant islands");
562 // Raw contents of the function.
563 StringRef SectionContents = BF.getOriginSection()->getContents();
564
565 // Raw contents of the function.
566 StringRef FunctionContents = SectionContents.substr(
567 Start: BF.getAddress() - BF.getOriginSection()->getAddress(), N: BF.getMaxSize());
568
569 if (opts::Verbosity && !OnBehalfOf)
570 BC.outs() << "BOLT-INFO: emitting constant island for function " << BF
571 << "\n";
572
573 // We split the island into smaller blocks and output labels between them.
574 auto IS = Islands.Offsets.begin();
575 for (auto DataIter = Islands.DataOffsets.begin();
576 DataIter != Islands.DataOffsets.end(); ++DataIter) {
577 uint64_t FunctionOffset = *DataIter;
578 uint64_t EndOffset = 0ULL;
579
580 // Determine size of this data chunk
581 auto NextData = std::next(x: DataIter);
582 auto CodeIter = Islands.CodeOffsets.lower_bound(x: *DataIter);
583 if (CodeIter == Islands.CodeOffsets.end() &&
584 NextData == Islands.DataOffsets.end())
585 EndOffset = BF.getMaxSize();
586 else if (CodeIter == Islands.CodeOffsets.end())
587 EndOffset = *NextData;
588 else if (NextData == Islands.DataOffsets.end())
589 EndOffset = *CodeIter;
590 else
591 EndOffset = (*CodeIter > *NextData) ? *NextData : *CodeIter;
592
593 if (FunctionOffset == EndOffset)
594 continue; // Size is zero, nothing to emit
595
596 auto emitCI = [&](uint64_t &FunctionOffset, uint64_t EndOffset) {
597 if (FunctionOffset >= EndOffset)
598 return;
599
600 for (auto It = Islands.Relocations.lower_bound(x: FunctionOffset);
601 It != Islands.Relocations.end(); ++It) {
602 if (It->first >= EndOffset)
603 break;
604
605 const Relocation &Relocation = It->second;
606 if (FunctionOffset < Relocation.Offset) {
607 Streamer.emitBytes(
608 Data: FunctionContents.slice(Start: FunctionOffset, End: Relocation.Offset));
609 FunctionOffset = Relocation.Offset;
610 }
611
612 LLVM_DEBUG(
613 dbgs() << "BOLT-DEBUG: emitting constant island relocation"
614 << " for " << BF << " at offset 0x"
615 << Twine::utohexstr(Relocation.Offset) << " with size "
616 << Relocation::getSizeForType(Relocation.Type) << '\n');
617
618 FunctionOffset += Relocation.emit(Streamer: &Streamer);
619 }
620
621 assert(FunctionOffset <= EndOffset && "overflow error");
622 if (FunctionOffset < EndOffset) {
623 Streamer.emitBytes(Data: FunctionContents.slice(Start: FunctionOffset, End: EndOffset));
624 FunctionOffset = EndOffset;
625 }
626 };
627
628 // Emit labels, relocs and data
629 while (IS != Islands.Offsets.end() && IS->first < EndOffset) {
630 auto NextLabelOffset =
631 IS == Islands.Offsets.end() ? EndOffset : IS->first;
632 auto NextStop = std::min(a: NextLabelOffset, b: EndOffset);
633 assert(NextStop <= EndOffset && "internal overflow error");
634 emitCI(FunctionOffset, NextStop);
635 if (IS != Islands.Offsets.end() && FunctionOffset == IS->first) {
636 // This is a slightly complex code to decide which label to emit. We
637 // have 4 cases to handle: regular symbol, cold symbol, regular or cold
638 // symbol being emitted on behalf of an external function.
639 if (!OnBehalfOf) {
640 if (!EmitColdPart) {
641 LLVM_DEBUG(dbgs() << "BOLT-DEBUG: emitted label "
642 << IS->second->getName() << " at offset 0x"
643 << Twine::utohexstr(IS->first) << '\n');
644 if (IS->second->isUndefined())
645 Streamer.emitLabel(Symbol: IS->second);
646 else
647 assert(BF.hasName(std::string(IS->second->getName())));
648 } else if (Islands.ColdSymbols.count(Val: IS->second) != 0) {
649 LLVM_DEBUG(dbgs()
650 << "BOLT-DEBUG: emitted label "
651 << Islands.ColdSymbols[IS->second]->getName() << '\n');
652 if (Islands.ColdSymbols[IS->second]->isUndefined())
653 Streamer.emitLabel(Symbol: Islands.ColdSymbols[IS->second]);
654 }
655 } else {
656 if (!EmitColdPart) {
657 if (MCSymbol *Sym = Islands.Proxies[OnBehalfOf][IS->second]) {
658 LLVM_DEBUG(dbgs() << "BOLT-DEBUG: emitted label "
659 << Sym->getName() << '\n');
660 Streamer.emitLabel(Symbol: Sym);
661 }
662 } else if (MCSymbol *Sym =
663 Islands.ColdProxies[OnBehalfOf][IS->second]) {
664 LLVM_DEBUG(dbgs() << "BOLT-DEBUG: emitted label " << Sym->getName()
665 << '\n');
666 Streamer.emitLabel(Symbol: Sym);
667 }
668 }
669 ++IS;
670 }
671 }
672 assert(FunctionOffset <= EndOffset && "overflow error");
673 emitCI(FunctionOffset, EndOffset);
674 }
675 assert(IS == Islands.Offsets.end() && "some symbols were not emitted!");
676
677 if (OnBehalfOf)
678 return;
679 // Now emit constant islands from other functions that we may have used in
680 // this function.
681 for (BinaryFunction *ExternalFunc : Islands.Dependency)
682 emitConstantIslands(BF&: *ExternalFunc, EmitColdPart, OnBehalfOf: &BF);
683}
684
685SMLoc BinaryEmitter::emitLineInfo(const BinaryFunction &BF, SMLoc NewLoc,
686 SMLoc PrevLoc, bool FirstInstr,
687 MCSymbol *&InstrLabel) {
688 DWARFUnit *FunctionCU = BF.getDWARFUnit();
689 const DWARFDebugLine::LineTable *FunctionLineTable = BF.getDWARFLineTable();
690 assert(FunctionCU && "cannot emit line info for function without CU");
691
692 DebugLineTableRowRef RowReference = DebugLineTableRowRef::fromSMLoc(Loc: NewLoc);
693
694 // Check if no new line info needs to be emitted.
695 if (RowReference == DebugLineTableRowRef::NULL_ROW ||
696 NewLoc.getPointer() == PrevLoc.getPointer())
697 return PrevLoc;
698
699 unsigned CurrentFilenum = 0;
700 const DWARFDebugLine::LineTable *CurrentLineTable = FunctionLineTable;
701
702 // If the CU id from the current instruction location does not
703 // match the CU id from the current function, it means that we
704 // have come across some inlined code. We must look up the CU
705 // for the instruction's original function and get the line table
706 // from that.
707 const uint64_t FunctionUnitIndex = FunctionCU->getOffset();
708 const uint32_t CurrentUnitIndex = RowReference.DwCompileUnitIndex;
709 if (CurrentUnitIndex != FunctionUnitIndex) {
710 CurrentLineTable = BC.DwCtx->getLineTableForUnit(
711 U: BC.DwCtx->getCompileUnitForOffset(Offset: CurrentUnitIndex));
712 // Add filename from the inlined function to the current CU.
713 CurrentFilenum = BC.addDebugFilenameToUnit(
714 DestCUID: FunctionUnitIndex, SrcCUID: CurrentUnitIndex,
715 FileIndex: CurrentLineTable->Rows[RowReference.RowIndex - 1].File);
716 }
717
718 const DWARFDebugLine::Row &CurrentRow =
719 CurrentLineTable->Rows[RowReference.RowIndex - 1];
720 if (!CurrentFilenum)
721 CurrentFilenum = CurrentRow.File;
722
723 unsigned Flags = (DWARF2_FLAG_IS_STMT * CurrentRow.IsStmt) |
724 (DWARF2_FLAG_BASIC_BLOCK * CurrentRow.BasicBlock) |
725 (DWARF2_FLAG_PROLOGUE_END * CurrentRow.PrologueEnd) |
726 (DWARF2_FLAG_EPILOGUE_BEGIN * CurrentRow.EpilogueBegin);
727
728 // Always emit is_stmt at the beginning of function fragment.
729 if (FirstInstr)
730 Flags |= DWARF2_FLAG_IS_STMT;
731
732 BC.Ctx->setCurrentDwarfLoc(FileNum: CurrentFilenum, Line: CurrentRow.Line, Column: CurrentRow.Column,
733 Flags, Isa: CurrentRow.Isa, Discriminator: CurrentRow.Discriminator);
734 const MCDwarfLoc &DwarfLoc = BC.Ctx->getCurrentDwarfLoc();
735 BC.Ctx->clearDwarfLocSeen();
736
737 if (!InstrLabel)
738 InstrLabel = BC.Ctx->createTempSymbol();
739
740 BC.getDwarfLineTable(CUID: FunctionUnitIndex)
741 .getMCLineSections()
742 .addLineEntry(LineEntry: MCDwarfLineEntry(InstrLabel, DwarfLoc),
743 Sec: Streamer.getCurrentSectionOnly());
744
745 return NewLoc;
746}
747
748void BinaryEmitter::emitLineInfoEnd(const BinaryFunction &BF,
749 MCSymbol *FunctionEndLabel) {
750 DWARFUnit *FunctionCU = BF.getDWARFUnit();
751 assert(FunctionCU && "DWARF unit expected");
752 BC.Ctx->setCurrentDwarfLoc(FileNum: 0, Line: 0, Column: 0, DWARF2_FLAG_END_SEQUENCE, Isa: 0, Discriminator: 0);
753 const MCDwarfLoc &DwarfLoc = BC.Ctx->getCurrentDwarfLoc();
754 BC.Ctx->clearDwarfLocSeen();
755 BC.getDwarfLineTable(CUID: FunctionCU->getOffset())
756 .getMCLineSections()
757 .addLineEntry(LineEntry: MCDwarfLineEntry(FunctionEndLabel, DwarfLoc),
758 Sec: Streamer.getCurrentSectionOnly());
759}
760
761void BinaryEmitter::emitJumpTables(const BinaryFunction &BF) {
762 MCSection *ReadOnlySection = BC.MOFI->getReadOnlySection();
763 MCSection *ReadOnlyColdSection = BC.MOFI->getContext().getELFSection(
764 Section: ".rodata.cold", Type: ELF::SHT_PROGBITS, Flags: ELF::SHF_ALLOC);
765
766 if (!BF.hasJumpTables())
767 return;
768
769 if (opts::PrintJumpTables)
770 BC.outs() << "BOLT-INFO: jump tables for function " << BF << ":\n";
771
772 for (auto &JTI : BF.jumpTables()) {
773 JumpTable &JT = *JTI.second;
774 // Only emit shared jump tables once, when processing the first parent
775 if (JT.Parents.size() > 1 && JT.Parents[0] != &BF)
776 continue;
777 if (opts::PrintJumpTables)
778 JT.print(OS&: BC.outs());
779 if (opts::JumpTables == JTS_BASIC && BC.HasRelocations) {
780 JT.updateOriginal();
781 } else {
782 MCSection *HotSection, *ColdSection;
783 if (opts::JumpTables == JTS_BASIC) {
784 // In non-relocation mode we have to emit jump tables in local sections.
785 // This way we only overwrite them when the corresponding function is
786 // overwritten.
787 std::string Name = ".local." + JT.Labels[0]->getName().str();
788 std::replace(first: Name.begin(), last: Name.end(), old_value: '/', new_value: '.');
789 BinarySection &Section =
790 BC.registerOrUpdateSection(Name, ELFType: ELF::SHT_PROGBITS, ELFFlags: ELF::SHF_ALLOC);
791 Section.setAnonymous(true);
792 JT.setOutputSection(Section);
793 HotSection = BC.getDataSection(SectionName: Name);
794 ColdSection = HotSection;
795 } else {
796 if (BF.isSimple()) {
797 HotSection = ReadOnlySection;
798 ColdSection = ReadOnlyColdSection;
799 } else {
800 HotSection = BF.hasProfile() ? ReadOnlySection : ReadOnlyColdSection;
801 ColdSection = HotSection;
802 }
803 }
804 emitJumpTable(JT, HotSection, ColdSection);
805 }
806 }
807}
808
809void BinaryEmitter::emitJumpTable(const JumpTable &JT, MCSection *HotSection,
810 MCSection *ColdSection) {
811 // Pre-process entries for aggressive splitting.
812 // Each label represents a separate switch table and gets its own count
813 // determining its destination.
814 std::map<MCSymbol *, uint64_t> LabelCounts;
815 if (opts::JumpTables > JTS_SPLIT && !JT.Counts.empty()) {
816 MCSymbol *CurrentLabel = JT.Labels.at(k: 0);
817 uint64_t CurrentLabelCount = 0;
818 for (unsigned Index = 0; Index < JT.Entries.size(); ++Index) {
819 auto LI = JT.Labels.find(x: Index * JT.EntrySize);
820 if (LI != JT.Labels.end()) {
821 LabelCounts[CurrentLabel] = CurrentLabelCount;
822 CurrentLabel = LI->second;
823 CurrentLabelCount = 0;
824 }
825 CurrentLabelCount += JT.Counts[Index].Count;
826 }
827 LabelCounts[CurrentLabel] = CurrentLabelCount;
828 } else {
829 Streamer.switchSection(Section: JT.Count > 0 ? HotSection : ColdSection);
830 Streamer.emitValueToAlignment(Alignment: Align(JT.EntrySize));
831 }
832 MCSymbol *LastLabel = nullptr;
833 uint64_t Offset = 0;
834 for (MCSymbol *Entry : JT.Entries) {
835 auto LI = JT.Labels.find(x: Offset);
836 if (LI != JT.Labels.end()) {
837 LLVM_DEBUG({
838 dbgs() << "BOLT-DEBUG: emitting jump table " << LI->second->getName()
839 << " (originally was at address 0x"
840 << Twine::utohexstr(JT.getAddress() + Offset)
841 << (Offset ? ") as part of larger jump table\n" : ")\n");
842 });
843 if (!LabelCounts.empty()) {
844 LLVM_DEBUG(dbgs() << "BOLT-DEBUG: jump table count: "
845 << LabelCounts[LI->second] << '\n');
846 if (LabelCounts[LI->second] > 0)
847 Streamer.switchSection(Section: HotSection);
848 else
849 Streamer.switchSection(Section: ColdSection);
850 Streamer.emitValueToAlignment(Alignment: Align(JT.EntrySize));
851 }
852 // Emit all labels registered at the address of this jump table
853 // to sync with our global symbol table. We may have two labels
854 // registered at this address if one label was created via
855 // getOrCreateGlobalSymbol() (e.g. LEA instructions referencing
856 // this location) and another via getOrCreateJumpTable(). This
857 // creates a race where the symbols created by these two
858 // functions may or may not be the same, but they are both
859 // registered in our symbol table at the same address. By
860 // emitting them all here we make sure there is no ambiguity
861 // that depends on the order that these symbols were created, so
862 // whenever this address is referenced in the binary, it is
863 // certain to point to the jump table identified at this
864 // address.
865 if (BinaryData *BD = BC.getBinaryDataByName(Name: LI->second->getName())) {
866 for (MCSymbol *S : BD->getSymbols())
867 Streamer.emitLabel(Symbol: S);
868 } else {
869 Streamer.emitLabel(Symbol: LI->second);
870 }
871 LastLabel = LI->second;
872 }
873 if (JT.Type == JumpTable::JTT_NORMAL) {
874 Streamer.emitSymbolValue(Sym: Entry, Size: JT.OutputEntrySize);
875 } else { // JTT_PIC
876 const MCSymbolRefExpr *JTExpr =
877 MCSymbolRefExpr::create(Symbol: LastLabel, Ctx&: Streamer.getContext());
878 const MCSymbolRefExpr *E =
879 MCSymbolRefExpr::create(Symbol: Entry, Ctx&: Streamer.getContext());
880 const MCBinaryExpr *Value =
881 MCBinaryExpr::createSub(LHS: E, RHS: JTExpr, Ctx&: Streamer.getContext());
882 Streamer.emitValue(Value, Size: JT.EntrySize);
883 }
884 Offset += JT.EntrySize;
885 }
886}
887
888void BinaryEmitter::emitCFIInstruction(const MCCFIInstruction &Inst) const {
889 switch (Inst.getOperation()) {
890 default:
891 llvm_unreachable("Unexpected instruction");
892 case MCCFIInstruction::OpDefCfaOffset:
893 Streamer.emitCFIDefCfaOffset(Offset: Inst.getOffset());
894 break;
895 case MCCFIInstruction::OpAdjustCfaOffset:
896 Streamer.emitCFIAdjustCfaOffset(Adjustment: Inst.getOffset());
897 break;
898 case MCCFIInstruction::OpDefCfa:
899 Streamer.emitCFIDefCfa(Register: Inst.getRegister(), Offset: Inst.getOffset());
900 break;
901 case MCCFIInstruction::OpDefCfaRegister:
902 Streamer.emitCFIDefCfaRegister(Register: Inst.getRegister());
903 break;
904 case MCCFIInstruction::OpOffset:
905 Streamer.emitCFIOffset(Register: Inst.getRegister(), Offset: Inst.getOffset());
906 break;
907 case MCCFIInstruction::OpRegister:
908 Streamer.emitCFIRegister(Register1: Inst.getRegister(), Register2: Inst.getRegister2());
909 break;
910 case MCCFIInstruction::OpWindowSave:
911 Streamer.emitCFIWindowSave();
912 break;
913 case MCCFIInstruction::OpNegateRAState:
914 Streamer.emitCFINegateRAState();
915 break;
916 case MCCFIInstruction::OpSameValue:
917 Streamer.emitCFISameValue(Register: Inst.getRegister());
918 break;
919 case MCCFIInstruction::OpGnuArgsSize:
920 Streamer.emitCFIGnuArgsSize(Size: Inst.getOffset());
921 break;
922 case MCCFIInstruction::OpEscape:
923 Streamer.AddComment(T: Inst.getComment());
924 Streamer.emitCFIEscape(Values: Inst.getValues());
925 break;
926 case MCCFIInstruction::OpRestore:
927 Streamer.emitCFIRestore(Register: Inst.getRegister());
928 break;
929 case MCCFIInstruction::OpUndefined:
930 Streamer.emitCFIUndefined(Register: Inst.getRegister());
931 break;
932 }
933}
934
935// The code is based on EHStreamer::emitExceptionTable().
936void BinaryEmitter::emitLSDA(BinaryFunction &BF, const FunctionFragment &FF) {
937 const BinaryFunction::CallSitesRange Sites =
938 BF.getCallSites(F: FF.getFragmentNum());
939 if (Sites.empty())
940 return;
941
942 // Calculate callsite table size. Size of each callsite entry is:
943 //
944 // sizeof(start) + sizeof(length) + sizeof(LP) + sizeof(uleb128(action))
945 //
946 // or
947 //
948 // sizeof(dwarf::DW_EH_PE_data4) * 3 + sizeof(uleb128(action))
949 uint64_t CallSiteTableLength = llvm::size(Range: Sites) * 4 * 3;
950 for (const auto &FragmentCallSite : Sites)
951 CallSiteTableLength += getULEB128Size(Value: FragmentCallSite.second.Action);
952
953 Streamer.switchSection(Section: BC.MOFI->getLSDASection());
954
955 const unsigned TTypeEncoding = BF.getLSDATypeEncoding();
956 const unsigned TTypeEncodingSize = BC.getDWARFEncodingSize(Encoding: TTypeEncoding);
957 const uint16_t TTypeAlignment = 4;
958
959 // Type tables have to be aligned at 4 bytes.
960 Streamer.emitValueToAlignment(Alignment: Align(TTypeAlignment));
961
962 // Emit the LSDA label.
963 MCSymbol *LSDASymbol = BF.getLSDASymbol(F: FF.getFragmentNum());
964 assert(LSDASymbol && "no LSDA symbol set");
965 Streamer.emitLabel(Symbol: LSDASymbol);
966
967 // Corresponding FDE start.
968 const MCSymbol *StartSymbol = BF.getSymbol(Fragment: FF.getFragmentNum());
969
970 // Emit the LSDA header.
971
972 // If LPStart is omitted, then the start of the FDE is used as a base for
973 // landing pad displacements. Then if a cold fragment starts with
974 // a landing pad, this means that the first landing pad offset will be 0.
975 // As a result, the exception handling runtime will ignore this landing pad
976 // because zero offset denotes the absence of a landing pad.
977 // For this reason, when the binary has fixed starting address we emit LPStart
978 // as 0 and output the absolute value of the landing pad in the table.
979 //
980 // If the base address can change, we cannot use absolute addresses for
981 // landing pads (at least not without runtime relocations). Hence, we fall
982 // back to emitting landing pads relative to the FDE start.
983 // As we are emitting label differences, we have to guarantee both labels are
984 // defined in the same section and hence cannot place the landing pad into a
985 // cold fragment when the corresponding call site is in the hot fragment.
986 // Because of this issue and the previously described issue of possible
987 // zero-offset landing pad we have to place landing pads in the same section
988 // as the corresponding invokes for shared objects.
989 std::function<void(const MCSymbol *)> emitLandingPad;
990 if (BC.HasFixedLoadAddress) {
991 Streamer.emitIntValue(Value: dwarf::DW_EH_PE_udata4, Size: 1); // LPStart format
992 Streamer.emitIntValue(Value: 0, Size: 4); // LPStart
993 emitLandingPad = [&](const MCSymbol *LPSymbol) {
994 if (!LPSymbol)
995 Streamer.emitIntValue(Value: 0, Size: 4);
996 else
997 Streamer.emitSymbolValue(Sym: LPSymbol, Size: 4);
998 };
999 } else {
1000 Streamer.emitIntValue(Value: dwarf::DW_EH_PE_omit, Size: 1); // LPStart format
1001 emitLandingPad = [&](const MCSymbol *LPSymbol) {
1002 if (!LPSymbol)
1003 Streamer.emitIntValue(Value: 0, Size: 4);
1004 else
1005 Streamer.emitAbsoluteSymbolDiff(Hi: LPSymbol, Lo: StartSymbol, Size: 4);
1006 };
1007 }
1008
1009 Streamer.emitIntValue(Value: TTypeEncoding, Size: 1); // TType format
1010
1011 // See the comment in EHStreamer::emitExceptionTable() on to use
1012 // uleb128 encoding (which can use variable number of bytes to encode the same
1013 // value) to ensure type info table is properly aligned at 4 bytes without
1014 // iteratively fixing sizes of the tables.
1015 unsigned CallSiteTableLengthSize = getULEB128Size(Value: CallSiteTableLength);
1016 unsigned TTypeBaseOffset =
1017 sizeof(int8_t) + // Call site format
1018 CallSiteTableLengthSize + // Call site table length size
1019 CallSiteTableLength + // Call site table length
1020 BF.getLSDAActionTable().size() + // Actions table size
1021 BF.getLSDATypeTable().size() * TTypeEncodingSize; // Types table size
1022 unsigned TTypeBaseOffsetSize = getULEB128Size(Value: TTypeBaseOffset);
1023 unsigned TotalSize = sizeof(int8_t) + // LPStart format
1024 sizeof(int8_t) + // TType format
1025 TTypeBaseOffsetSize + // TType base offset size
1026 TTypeBaseOffset; // TType base offset
1027 unsigned SizeAlign = (4 - TotalSize) & 3;
1028
1029 if (TTypeEncoding != dwarf::DW_EH_PE_omit)
1030 // Account for any extra padding that will be added to the call site table
1031 // length.
1032 Streamer.emitULEB128IntValue(Value: TTypeBaseOffset,
1033 /*PadTo=*/TTypeBaseOffsetSize + SizeAlign);
1034
1035 // Emit the landing pad call site table. We use signed data4 since we can emit
1036 // a landing pad in a different part of the split function that could appear
1037 // earlier in the address space than LPStart.
1038 Streamer.emitIntValue(Value: dwarf::DW_EH_PE_sdata4, Size: 1);
1039 Streamer.emitULEB128IntValue(Value: CallSiteTableLength);
1040
1041 for (const auto &FragmentCallSite : Sites) {
1042 const BinaryFunction::CallSite &CallSite = FragmentCallSite.second;
1043 const MCSymbol *BeginLabel = CallSite.Start;
1044 const MCSymbol *EndLabel = CallSite.End;
1045
1046 assert(BeginLabel && "start EH label expected");
1047 assert(EndLabel && "end EH label expected");
1048
1049 // Start of the range is emitted relative to the start of current
1050 // function split part.
1051 Streamer.emitAbsoluteSymbolDiff(Hi: BeginLabel, Lo: StartSymbol, Size: 4);
1052 Streamer.emitAbsoluteSymbolDiff(Hi: EndLabel, Lo: BeginLabel, Size: 4);
1053 emitLandingPad(CallSite.LP);
1054 Streamer.emitULEB128IntValue(Value: CallSite.Action);
1055 }
1056
1057 // Write out action, type, and type index tables at the end.
1058 //
1059 // For action and type index tables there's no need to change the original
1060 // table format unless we are doing function splitting, in which case we can
1061 // split and optimize the tables.
1062 //
1063 // For type table we (re-)encode the table using TTypeEncoding matching
1064 // the current assembler mode.
1065 for (uint8_t const &Byte : BF.getLSDAActionTable())
1066 Streamer.emitIntValue(Value: Byte, Size: 1);
1067
1068 const BinaryFunction::LSDATypeTableTy &TypeTable =
1069 (TTypeEncoding & dwarf::DW_EH_PE_indirect) ? BF.getLSDATypeAddressTable()
1070 : BF.getLSDATypeTable();
1071 assert(TypeTable.size() == BF.getLSDATypeTable().size() &&
1072 "indirect type table size mismatch");
1073
1074 for (int Index = TypeTable.size() - 1; Index >= 0; --Index) {
1075 const uint64_t TypeAddress = TypeTable[Index];
1076 switch (TTypeEncoding & 0x70) {
1077 default:
1078 llvm_unreachable("unsupported TTypeEncoding");
1079 case dwarf::DW_EH_PE_absptr:
1080 Streamer.emitIntValue(Value: TypeAddress, Size: TTypeEncodingSize);
1081 break;
1082 case dwarf::DW_EH_PE_pcrel: {
1083 if (TypeAddress) {
1084 const MCSymbol *TypeSymbol =
1085 BC.getOrCreateGlobalSymbol(Address: TypeAddress, Prefix: "TI", Size: 0, Alignment: TTypeAlignment);
1086 MCSymbol *DotSymbol = BC.Ctx->createNamedTempSymbol();
1087 Streamer.emitLabel(Symbol: DotSymbol);
1088 const MCBinaryExpr *SubDotExpr = MCBinaryExpr::createSub(
1089 LHS: MCSymbolRefExpr::create(Symbol: TypeSymbol, Ctx&: *BC.Ctx),
1090 RHS: MCSymbolRefExpr::create(Symbol: DotSymbol, Ctx&: *BC.Ctx), Ctx&: *BC.Ctx);
1091 Streamer.emitValue(Value: SubDotExpr, Size: TTypeEncodingSize);
1092 } else {
1093 Streamer.emitIntValue(Value: 0, Size: TTypeEncodingSize);
1094 }
1095 break;
1096 }
1097 }
1098 }
1099 for (uint8_t const &Byte : BF.getLSDATypeIndexTable())
1100 Streamer.emitIntValue(Value: Byte, Size: 1);
1101}
1102
1103void BinaryEmitter::emitDebugLineInfoForOriginalFunctions() {
1104 // If a function is in a CU containing at least one processed function, we
1105 // have to rewrite the whole line table for that CU. For unprocessed functions
1106 // we use data from the input line table.
1107 for (auto &It : BC.getBinaryFunctions()) {
1108 const BinaryFunction &Function = It.second;
1109
1110 // If the function was emitted, its line info was emitted with it.
1111 if (Function.isEmitted())
1112 continue;
1113
1114 const DWARFDebugLine::LineTable *LineTable = Function.getDWARFLineTable();
1115 if (!LineTable)
1116 continue; // nothing to update for this function
1117
1118 const uint64_t Address = Function.getAddress();
1119 std::vector<uint32_t> Results;
1120 if (!LineTable->lookupAddressRange(
1121 Address: {.Address: Address, .SectionIndex: object::SectionedAddress::UndefSection},
1122 Size: Function.getSize(), Result&: Results))
1123 continue;
1124
1125 if (Results.empty())
1126 continue;
1127
1128 // The first row returned could be the last row matching the start address.
1129 // Find the first row with the same address that is not the end of the
1130 // sequence.
1131 uint64_t FirstRow = Results.front();
1132 while (FirstRow > 0) {
1133 const DWARFDebugLine::Row &PrevRow = LineTable->Rows[FirstRow - 1];
1134 if (PrevRow.Address.Address != Address || PrevRow.EndSequence)
1135 break;
1136 --FirstRow;
1137 }
1138
1139 const uint64_t EndOfSequenceAddress =
1140 Function.getAddress() + Function.getMaxSize();
1141 BC.getDwarfLineTable(CUID: Function.getDWARFUnit()->getOffset())
1142 .addLineTableSequence(Table: LineTable, FirstRow, LastRow: Results.back(),
1143 EndOfSequenceAddress);
1144 }
1145
1146 // For units that are completely unprocessed, use original debug line contents
1147 // eliminating the need to regenerate line info program.
1148 emitDebugLineInfoForUnprocessedCUs();
1149}
1150
1151void BinaryEmitter::emitDebugLineInfoForUnprocessedCUs() {
1152 // Sorted list of section offsets provides boundaries for section fragments,
1153 // where each fragment is the unit's contribution to debug line section.
1154 std::vector<uint64_t> StmtListOffsets;
1155 StmtListOffsets.reserve(n: BC.DwCtx->getNumCompileUnits());
1156 for (const std::unique_ptr<DWARFUnit> &CU : BC.DwCtx->compile_units()) {
1157 DWARFDie CUDie = CU->getUnitDIE();
1158 auto StmtList = dwarf::toSectionOffset(V: CUDie.find(Attr: dwarf::DW_AT_stmt_list));
1159 if (!StmtList)
1160 continue;
1161
1162 StmtListOffsets.push_back(x: *StmtList);
1163 }
1164 llvm::sort(C&: StmtListOffsets);
1165
1166 // For each CU that was not processed, emit its line info as a binary blob.
1167 for (const std::unique_ptr<DWARFUnit> &CU : BC.DwCtx->compile_units()) {
1168 if (BC.ProcessedCUs.count(x: CU.get()))
1169 continue;
1170
1171 DWARFDie CUDie = CU->getUnitDIE();
1172 auto StmtList = dwarf::toSectionOffset(V: CUDie.find(Attr: dwarf::DW_AT_stmt_list));
1173 if (!StmtList)
1174 continue;
1175
1176 StringRef DebugLineContents = CU->getLineSection().Data;
1177
1178 const uint64_t Begin = *StmtList;
1179
1180 // Statement list ends where the next unit contribution begins, or at the
1181 // end of the section.
1182 auto It = llvm::upper_bound(Range&: StmtListOffsets, Value: Begin);
1183 const uint64_t End =
1184 It == StmtListOffsets.end() ? DebugLineContents.size() : *It;
1185
1186 BC.getDwarfLineTable(CUID: CU->getOffset())
1187 .addRawContents(DebugLineContents: DebugLineContents.slice(Start: Begin, End));
1188 }
1189}
1190
1191void BinaryEmitter::emitDataSections(StringRef OrgSecPrefix) {
1192 for (BinarySection &Section : BC.sections()) {
1193 if (!Section.hasRelocations())
1194 continue;
1195
1196 StringRef Prefix = Section.hasSectionRef() ? OrgSecPrefix : "";
1197 Section.emitAsData(Streamer, SectionName: Prefix + Section.getName());
1198 Section.clearRelocations();
1199 }
1200}
1201
1202namespace llvm {
1203namespace bolt {
1204
1205void emitBinaryContext(MCStreamer &Streamer, BinaryContext &BC,
1206 StringRef OrgSecPrefix) {
1207 BinaryEmitter(Streamer, BC).emitAll(OrgSecPrefix);
1208}
1209
1210void emitFunctionBody(MCStreamer &Streamer, BinaryFunction &BF,
1211 FunctionFragment &FF, bool EmitCodeOnly) {
1212 BinaryEmitter(Streamer, BF.getBinaryContext())
1213 .emitFunctionBody(BF, FF, EmitCodeOnly);
1214}
1215
1216} // namespace bolt
1217} // namespace llvm
1218

source code of bolt/lib/Core/BinaryEmitter.cpp