1//===-- ProfiledBinary.cpp - Binary decoder ---------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "ProfiledBinary.h"
10#include "ErrorHandling.h"
11#include "MissingFrameInferrer.h"
12#include "ProfileGenerator.h"
13#include "llvm/DebugInfo/Symbolize/SymbolizableModule.h"
14#include "llvm/Demangle/Demangle.h"
15#include "llvm/IR/DebugInfoMetadata.h"
16#include "llvm/MC/TargetRegistry.h"
17#include "llvm/Object/COFF.h"
18#include "llvm/Support/CommandLine.h"
19#include "llvm/Support/Debug.h"
20#include "llvm/Support/Format.h"
21#include "llvm/Support/TargetSelect.h"
22#include "llvm/TargetParser/Triple.h"
23#include <optional>
24
25#define DEBUG_TYPE "load-binary"
26
27using namespace llvm;
28using namespace sampleprof;
29
30cl::opt<bool> ShowDisassemblyOnly("show-disassembly-only",
31 cl::desc("Print disassembled code."));
32
33cl::opt<bool> ShowSourceLocations("show-source-locations",
34 cl::desc("Print source locations."));
35
36static cl::opt<bool>
37 ShowCanonicalFnName("show-canonical-fname",
38 cl::desc("Print canonical function name."));
39
40static cl::opt<bool> ShowPseudoProbe(
41 "show-pseudo-probe",
42 cl::desc("Print pseudo probe section and disassembled info."));
43
44static cl::opt<bool> UseDwarfCorrelation(
45 "use-dwarf-correlation",
46 cl::desc("Use dwarf for profile correlation even when binary contains "
47 "pseudo probe."));
48
49static cl::opt<std::string>
50 DWPPath("dwp", cl::init(Val: ""),
51 cl::desc("Path of .dwp file. When not specified, it will be "
52 "<binary>.dwp in the same directory as the main binary."));
53
54static cl::list<std::string> DisassembleFunctions(
55 "disassemble-functions", cl::CommaSeparated,
56 cl::desc("List of functions to print disassembly for. Accept demangled "
57 "names only. Only work with show-disassembly-only"));
58
59extern cl::opt<bool> ShowDetailedWarning;
60extern cl::opt<bool> InferMissingFrames;
61
62namespace llvm {
63namespace sampleprof {
64
65static const Target *getTarget(const ObjectFile *Obj) {
66 Triple TheTriple = Obj->makeTriple();
67 std::string Error;
68 std::string ArchName;
69 const Target *TheTarget =
70 TargetRegistry::lookupTarget(ArchName, TheTriple, Error);
71 if (!TheTarget)
72 exitWithError(Message: Error, Whence: Obj->getFileName());
73 return TheTarget;
74}
75
76void BinarySizeContextTracker::addInstructionForContext(
77 const SampleContextFrameVector &Context, uint32_t InstrSize) {
78 ContextTrieNode *CurNode = &RootContext;
79 bool IsLeaf = true;
80 for (const auto &Callsite : reverse(C: Context)) {
81 FunctionId CallerName = Callsite.Func;
82 LineLocation CallsiteLoc = IsLeaf ? LineLocation(0, 0) : Callsite.Location;
83 CurNode = CurNode->getOrCreateChildContext(CallSite: CallsiteLoc, ChildName: CallerName);
84 IsLeaf = false;
85 }
86
87 CurNode->addFunctionSize(FSize: InstrSize);
88}
89
90uint32_t
91BinarySizeContextTracker::getFuncSizeForContext(const ContextTrieNode *Node) {
92 ContextTrieNode *CurrNode = &RootContext;
93 ContextTrieNode *PrevNode = nullptr;
94
95 std::optional<uint32_t> Size;
96
97 // Start from top-level context-less function, traverse down the reverse
98 // context trie to find the best/longest match for given context, then
99 // retrieve the size.
100 LineLocation CallSiteLoc(0, 0);
101 while (CurrNode && Node->getParentContext() != nullptr) {
102 PrevNode = CurrNode;
103 CurrNode = CurrNode->getChildContext(CallSite: CallSiteLoc, ChildName: Node->getFuncName());
104 if (CurrNode && CurrNode->getFunctionSize())
105 Size = *CurrNode->getFunctionSize();
106 CallSiteLoc = Node->getCallSiteLoc();
107 Node = Node->getParentContext();
108 }
109
110 // If we traversed all nodes along the path of the context and haven't
111 // found a size yet, pivot to look for size from sibling nodes, i.e size
112 // of inlinee under different context.
113 if (!Size) {
114 if (!CurrNode)
115 CurrNode = PrevNode;
116 while (!Size && CurrNode && !CurrNode->getAllChildContext().empty()) {
117 CurrNode = &CurrNode->getAllChildContext().begin()->second;
118 if (CurrNode->getFunctionSize())
119 Size = *CurrNode->getFunctionSize();
120 }
121 }
122
123 assert(Size && "We should at least find one context size.");
124 return *Size;
125}
126
127void BinarySizeContextTracker::trackInlineesOptimizedAway(
128 MCPseudoProbeDecoder &ProbeDecoder) {
129 ProbeFrameStack ProbeContext;
130 for (const auto &Child : ProbeDecoder.getDummyInlineRoot().getChildren())
131 trackInlineesOptimizedAway(ProbeDecoder, ProbeNode&: *Child.second.get(), Context&: ProbeContext);
132}
133
134void BinarySizeContextTracker::trackInlineesOptimizedAway(
135 MCPseudoProbeDecoder &ProbeDecoder,
136 MCDecodedPseudoProbeInlineTree &ProbeNode, ProbeFrameStack &ProbeContext) {
137 StringRef FuncName =
138 ProbeDecoder.getFuncDescForGUID(GUID: ProbeNode.Guid)->FuncName;
139 ProbeContext.emplace_back(Args&: FuncName, Args: 0);
140
141 // This ProbeContext has a probe, so it has code before inlining and
142 // optimization. Make sure we mark its size as known.
143 if (!ProbeNode.getProbes().empty()) {
144 ContextTrieNode *SizeContext = &RootContext;
145 for (auto &ProbeFrame : reverse(C&: ProbeContext)) {
146 StringRef CallerName = ProbeFrame.first;
147 LineLocation CallsiteLoc(ProbeFrame.second, 0);
148 SizeContext =
149 SizeContext->getOrCreateChildContext(CallSite: CallsiteLoc,
150 ChildName: FunctionId(CallerName));
151 }
152 // Add 0 size to make known.
153 SizeContext->addFunctionSize(FSize: 0);
154 }
155
156 // DFS down the probe inline tree
157 for (const auto &ChildNode : ProbeNode.getChildren()) {
158 InlineSite Location = ChildNode.first;
159 ProbeContext.back().second = std::get<1>(t&: Location);
160 trackInlineesOptimizedAway(ProbeDecoder, ProbeNode&: *ChildNode.second.get(),
161 ProbeContext);
162 }
163
164 ProbeContext.pop_back();
165}
166
167ProfiledBinary::ProfiledBinary(const StringRef ExeBinPath,
168 const StringRef DebugBinPath)
169 : Path(ExeBinPath), DebugBinaryPath(DebugBinPath),
170 SymbolizerOpts(getSymbolizerOpts()), ProEpilogTracker(this),
171 Symbolizer(std::make_unique<symbolize::LLVMSymbolizer>(args&: SymbolizerOpts)),
172 TrackFuncContextSize(EnableCSPreInliner && UseContextCostForPreInliner) {
173 // Point to executable binary if debug info binary is not specified.
174 SymbolizerPath = DebugBinPath.empty() ? ExeBinPath : DebugBinPath;
175 if (InferMissingFrames)
176 MissingContextInferrer = std::make_unique<MissingFrameInferrer>(args: this);
177 load();
178}
179
180ProfiledBinary::~ProfiledBinary() {}
181
182void ProfiledBinary::warnNoFuncEntry() {
183 uint64_t NoFuncEntryNum = 0;
184 for (auto &F : BinaryFunctions) {
185 if (F.second.Ranges.empty())
186 continue;
187 bool hasFuncEntry = false;
188 for (auto &R : F.second.Ranges) {
189 if (FuncRange *FR = findFuncRangeForStartAddr(Address: R.first)) {
190 if (FR->IsFuncEntry) {
191 hasFuncEntry = true;
192 break;
193 }
194 }
195 }
196
197 if (!hasFuncEntry) {
198 NoFuncEntryNum++;
199 if (ShowDetailedWarning)
200 WithColor::warning()
201 << "Failed to determine function entry for " << F.first
202 << " due to inconsistent name from symbol table and dwarf info.\n";
203 }
204 }
205 emitWarningSummary(Num: NoFuncEntryNum, Total: BinaryFunctions.size(),
206 Msg: "of functions failed to determine function entry due to "
207 "inconsistent name from symbol table and dwarf info.");
208}
209
210void ProfiledBinary::load() {
211 // Attempt to open the binary.
212 OwningBinary<Binary> OBinary = unwrapOrError(EO: createBinary(Path), Args&: Path);
213 Binary &ExeBinary = *OBinary.getBinary();
214
215 IsCOFF = isa<COFFObjectFile>(Val: &ExeBinary);
216 if (!isa<ELFObjectFileBase>(Val: &ExeBinary) && !IsCOFF)
217 exitWithError(Message: "not a valid ELF/COFF image", Whence: Path);
218
219 auto *Obj = cast<ObjectFile>(Val: &ExeBinary);
220 TheTriple = Obj->makeTriple();
221
222 LLVM_DEBUG(dbgs() << "Loading " << Path << "\n");
223
224 // Find the preferred load address for text sections.
225 setPreferredTextSegmentAddresses(Obj);
226
227 // Load debug info of subprograms from DWARF section.
228 // If path of debug info binary is specified, use the debug info from it,
229 // otherwise use the debug info from the executable binary.
230 if (!DebugBinaryPath.empty()) {
231 OwningBinary<Binary> DebugPath =
232 unwrapOrError(EO: createBinary(Path: DebugBinaryPath), Args&: DebugBinaryPath);
233 loadSymbolsFromDWARF(Obj&: *cast<ObjectFile>(Val: DebugPath.getBinary()));
234 } else {
235 loadSymbolsFromDWARF(Obj&: *cast<ObjectFile>(Val: &ExeBinary));
236 }
237
238 DisassembleFunctionSet.insert(begin: DisassembleFunctions.begin(),
239 end: DisassembleFunctions.end());
240
241 if (auto *ELFObj = dyn_cast<ELFObjectFileBase>(Val: Obj)) {
242 checkPseudoProbe(Obj: ELFObj);
243 if (UsePseudoProbes)
244 populateElfSymbolAddressList(O: ELFObj);
245
246 if (ShowDisassemblyOnly)
247 decodePseudoProbe(Obj: ELFObj);
248 }
249
250 // Disassemble the text sections.
251 disassemble(O: Obj);
252
253 // Use function start and return address to infer prolog and epilog
254 ProEpilogTracker.inferPrologAddresses(FuncStartAddressMap&: StartAddrToFuncRangeMap);
255 ProEpilogTracker.inferEpilogAddresses(RetAddrs&: RetAddressSet);
256
257 warnNoFuncEntry();
258
259 // TODO: decode other sections.
260}
261
262bool ProfiledBinary::inlineContextEqual(uint64_t Address1, uint64_t Address2) {
263 const SampleContextFrameVector &Context1 =
264 getCachedFrameLocationStack(Address: Address1);
265 const SampleContextFrameVector &Context2 =
266 getCachedFrameLocationStack(Address: Address2);
267 if (Context1.size() != Context2.size())
268 return false;
269 if (Context1.empty())
270 return false;
271 // The leaf frame contains location within the leaf, and it
272 // needs to be remove that as it's not part of the calling context
273 return std::equal(first1: Context1.begin(), last1: Context1.begin() + Context1.size() - 1,
274 first2: Context2.begin(), last2: Context2.begin() + Context2.size() - 1);
275}
276
277SampleContextFrameVector
278ProfiledBinary::getExpandedContext(const SmallVectorImpl<uint64_t> &Stack,
279 bool &WasLeafInlined) {
280 SampleContextFrameVector ContextVec;
281 if (Stack.empty())
282 return ContextVec;
283 // Process from frame root to leaf
284 for (auto Address : Stack) {
285 const SampleContextFrameVector &ExpandedContext =
286 getCachedFrameLocationStack(Address);
287 // An instruction without a valid debug line will be ignored by sample
288 // processing
289 if (ExpandedContext.empty())
290 return SampleContextFrameVector();
291 // Set WasLeafInlined to the size of inlined frame count for the last
292 // address which is leaf
293 WasLeafInlined = (ExpandedContext.size() > 1);
294 ContextVec.append(RHS: ExpandedContext);
295 }
296
297 // Replace with decoded base discriminator
298 for (auto &Frame : ContextVec) {
299 Frame.Location.Discriminator = ProfileGeneratorBase::getBaseDiscriminator(
300 Discriminator: Frame.Location.Discriminator, UseFSD: UseFSDiscriminator);
301 }
302
303 assert(ContextVec.size() && "Context length should be at least 1");
304
305 // Compress the context string except for the leaf frame
306 auto LeafFrame = ContextVec.back();
307 LeafFrame.Location = LineLocation(0, 0);
308 ContextVec.pop_back();
309 CSProfileGenerator::compressRecursionContext(Context&: ContextVec);
310 CSProfileGenerator::trimContext(S&: ContextVec);
311 ContextVec.push_back(Elt: LeafFrame);
312 return ContextVec;
313}
314
315template <class ELFT>
316void ProfiledBinary::setPreferredTextSegmentAddresses(const ELFFile<ELFT> &Obj,
317 StringRef FileName) {
318 const auto &PhdrRange = unwrapOrError(Obj.program_headers(), FileName);
319 // FIXME: This should be the page size of the system running profiling.
320 // However such info isn't available at post-processing time, assuming
321 // 4K page now. Note that we don't use EXEC_PAGESIZE from <linux/param.h>
322 // because we may build the tools on non-linux.
323 uint64_t PageSize = 0x1000;
324 for (const typename ELFT::Phdr &Phdr : PhdrRange) {
325 if (Phdr.p_type == ELF::PT_LOAD) {
326 if (!FirstLoadableAddress)
327 FirstLoadableAddress = Phdr.p_vaddr & ~(PageSize - 1U);
328 if (Phdr.p_flags & ELF::PF_X) {
329 // Segments will always be loaded at a page boundary.
330 PreferredTextSegmentAddresses.push_back(Phdr.p_vaddr &
331 ~(PageSize - 1U));
332 TextSegmentOffsets.push_back(Phdr.p_offset & ~(PageSize - 1U));
333 }
334 }
335 }
336
337 if (PreferredTextSegmentAddresses.empty())
338 exitWithError(Message: "no executable segment found", Whence: FileName);
339}
340
341void ProfiledBinary::setPreferredTextSegmentAddresses(const COFFObjectFile *Obj,
342 StringRef FileName) {
343 uint64_t ImageBase = Obj->getImageBase();
344 if (!ImageBase)
345 exitWithError(Message: "Not a COFF image", Whence: FileName);
346
347 PreferredTextSegmentAddresses.push_back(x: ImageBase);
348 FirstLoadableAddress = ImageBase;
349
350 for (SectionRef Section : Obj->sections()) {
351 const coff_section *Sec = Obj->getCOFFSection(Section);
352 if (Sec->Characteristics & COFF::IMAGE_SCN_CNT_CODE)
353 TextSegmentOffsets.push_back(x: Sec->VirtualAddress);
354 }
355}
356
357void ProfiledBinary::setPreferredTextSegmentAddresses(const ObjectFile *Obj) {
358 if (const auto *ELFObj = dyn_cast<ELF32LEObjectFile>(Val: Obj))
359 setPreferredTextSegmentAddresses(Obj: ELFObj->getELFFile(), FileName: Obj->getFileName());
360 else if (const auto *ELFObj = dyn_cast<ELF32BEObjectFile>(Val: Obj))
361 setPreferredTextSegmentAddresses(Obj: ELFObj->getELFFile(), FileName: Obj->getFileName());
362 else if (const auto *ELFObj = dyn_cast<ELF64LEObjectFile>(Val: Obj))
363 setPreferredTextSegmentAddresses(Obj: ELFObj->getELFFile(), FileName: Obj->getFileName());
364 else if (const auto *ELFObj = dyn_cast<ELF64BEObjectFile>(Val: Obj))
365 setPreferredTextSegmentAddresses(Obj: ELFObj->getELFFile(), FileName: Obj->getFileName());
366 else if (const auto *COFFObj = dyn_cast<COFFObjectFile>(Val: Obj))
367 setPreferredTextSegmentAddresses(Obj: COFFObj, FileName: Obj->getFileName());
368 else
369 llvm_unreachable("invalid object format");
370}
371
372void ProfiledBinary::checkPseudoProbe(const ELFObjectFileBase *Obj) {
373 if (UseDwarfCorrelation)
374 return;
375
376 bool HasProbeDescSection = false;
377 bool HasPseudoProbeSection = false;
378
379 StringRef FileName = Obj->getFileName();
380 for (section_iterator SI = Obj->section_begin(), SE = Obj->section_end();
381 SI != SE; ++SI) {
382 const SectionRef &Section = *SI;
383 StringRef SectionName = unwrapOrError(EO: Section.getName(), Args&: FileName);
384 if (SectionName == ".pseudo_probe_desc") {
385 HasProbeDescSection = true;
386 } else if (SectionName == ".pseudo_probe") {
387 HasPseudoProbeSection = true;
388 }
389 }
390
391 // set UsePseudoProbes flag, used for PerfReader
392 UsePseudoProbes = HasProbeDescSection && HasPseudoProbeSection;
393}
394
395void ProfiledBinary::decodePseudoProbe(const ELFObjectFileBase *Obj) {
396 if (!UsePseudoProbes)
397 return;
398
399 MCPseudoProbeDecoder::Uint64Set GuidFilter;
400 MCPseudoProbeDecoder::Uint64Map FuncStartAddresses;
401 if (ShowDisassemblyOnly) {
402 if (DisassembleFunctionSet.empty()) {
403 FuncStartAddresses = SymbolStartAddrs;
404 } else {
405 for (auto &F : DisassembleFunctionSet) {
406 auto GUID = Function::getGUID(GlobalName: F.first());
407 if (auto StartAddr = SymbolStartAddrs.lookup(Val: GUID)) {
408 FuncStartAddresses[GUID] = StartAddr;
409 FuncRange &Range = StartAddrToFuncRangeMap[StartAddr];
410 GuidFilter.insert(V: Function::getGUID(GlobalName: Range.getFuncName()));
411 }
412 }
413 }
414 } else {
415 for (auto *F : ProfiledFunctions) {
416 GuidFilter.insert(V: Function::getGUID(GlobalName: F->FuncName));
417 for (auto &Range : F->Ranges) {
418 auto GUIDs = StartAddrToSymMap.equal_range(x: Range.first);
419 for (auto I = GUIDs.first; I != GUIDs.second; ++I)
420 FuncStartAddresses[I->second] = I->first;
421 }
422 }
423 }
424
425 StringRef FileName = Obj->getFileName();
426 for (section_iterator SI = Obj->section_begin(), SE = Obj->section_end();
427 SI != SE; ++SI) {
428 const SectionRef &Section = *SI;
429 StringRef SectionName = unwrapOrError(EO: Section.getName(), Args&: FileName);
430
431 if (SectionName == ".pseudo_probe_desc") {
432 StringRef Contents = unwrapOrError(EO: Section.getContents(), Args&: FileName);
433 if (!ProbeDecoder.buildGUID2FuncDescMap(
434 Start: reinterpret_cast<const uint8_t *>(Contents.data()),
435 Size: Contents.size()))
436 exitWithError(
437 Message: "Pseudo Probe decoder fail in .pseudo_probe_desc section");
438 } else if (SectionName == ".pseudo_probe") {
439 StringRef Contents = unwrapOrError(EO: Section.getContents(), Args&: FileName);
440 if (!ProbeDecoder.buildAddress2ProbeMap(
441 Start: reinterpret_cast<const uint8_t *>(Contents.data()),
442 Size: Contents.size(), GuildFilter: GuidFilter, FuncStartAddrs: FuncStartAddresses))
443 exitWithError(Message: "Pseudo Probe decoder fail in .pseudo_probe section");
444 }
445 }
446
447 // Build TopLevelProbeFrameMap to track size for optimized inlinees when probe
448 // is available
449 if (TrackFuncContextSize) {
450 for (const auto &Child : ProbeDecoder.getDummyInlineRoot().getChildren()) {
451 auto *Frame = Child.second.get();
452 StringRef FuncName =
453 ProbeDecoder.getFuncDescForGUID(GUID: Frame->Guid)->FuncName;
454 TopLevelProbeFrameMap[FuncName] = Frame;
455 }
456 }
457
458 if (ShowPseudoProbe)
459 ProbeDecoder.printGUID2FuncDescMap(OS&: outs());
460}
461
462void ProfiledBinary::decodePseudoProbe() {
463 OwningBinary<Binary> OBinary = unwrapOrError(EO: createBinary(Path), Args&: Path);
464 Binary &ExeBinary = *OBinary.getBinary();
465 auto *Obj = cast<ELFObjectFileBase>(Val: &ExeBinary);
466 decodePseudoProbe(Obj);
467}
468
469void ProfiledBinary::setIsFuncEntry(FuncRange *FuncRange,
470 StringRef RangeSymName) {
471 // Skip external function symbol.
472 if (!FuncRange)
473 return;
474
475 // Set IsFuncEntry to ture if there is only one range in the function or the
476 // RangeSymName from ELF is equal to its DWARF-based function name.
477 if (FuncRange->Func->Ranges.size() == 1 ||
478 (!FuncRange->IsFuncEntry && FuncRange->getFuncName() == RangeSymName))
479 FuncRange->IsFuncEntry = true;
480}
481
482bool ProfiledBinary::dissassembleSymbol(std::size_t SI, ArrayRef<uint8_t> Bytes,
483 SectionSymbolsTy &Symbols,
484 const SectionRef &Section) {
485 std::size_t SE = Symbols.size();
486 uint64_t SectionAddress = Section.getAddress();
487 uint64_t SectSize = Section.getSize();
488 uint64_t StartAddress = Symbols[SI].Addr;
489 uint64_t NextStartAddress =
490 (SI + 1 < SE) ? Symbols[SI + 1].Addr : SectionAddress + SectSize;
491 FuncRange *FRange = findFuncRange(Address: StartAddress);
492 setIsFuncEntry(FuncRange: FRange, RangeSymName: FunctionSamples::getCanonicalFnName(FnName: Symbols[SI].Name));
493 StringRef SymbolName =
494 ShowCanonicalFnName
495 ? FunctionSamples::getCanonicalFnName(FnName: Symbols[SI].Name)
496 : Symbols[SI].Name;
497 bool ShowDisassembly =
498 ShowDisassemblyOnly && (DisassembleFunctionSet.empty() ||
499 DisassembleFunctionSet.count(Key: SymbolName));
500 if (ShowDisassembly)
501 outs() << '<' << SymbolName << ">:\n";
502
503 uint64_t Address = StartAddress;
504 // Size of a consecutive invalid instruction range starting from Address -1
505 // backwards.
506 uint64_t InvalidInstLength = 0;
507 while (Address < NextStartAddress) {
508 MCInst Inst;
509 uint64_t Size;
510 // Disassemble an instruction.
511 bool Disassembled = DisAsm->getInstruction(
512 Instr&: Inst, Size, Bytes: Bytes.slice(N: Address - SectionAddress), Address, CStream&: nulls());
513 if (Size == 0)
514 Size = 1;
515
516 if (ShowDisassembly) {
517 if (ShowPseudoProbe) {
518 ProbeDecoder.printProbeForAddress(OS&: outs(), Address);
519 }
520 outs() << format(Fmt: "%8" PRIx64 ":", Vals: Address);
521 size_t Start = outs().tell();
522 if (Disassembled)
523 IPrinter->printInst(MI: &Inst, Address: Address + Size, Annot: "", STI: *STI.get(), OS&: outs());
524 else
525 outs() << "\t<unknown>";
526 if (ShowSourceLocations) {
527 unsigned Cur = outs().tell() - Start;
528 if (Cur < 40)
529 outs().indent(NumSpaces: 40 - Cur);
530 InstructionPointer IP(this, Address);
531 outs() << getReversedLocWithContext(
532 Context: symbolize(IP, UseCanonicalFnName: ShowCanonicalFnName, UseProbeDiscriminator: ShowPseudoProbe));
533 }
534 outs() << "\n";
535 }
536
537 if (Disassembled) {
538 const MCInstrDesc &MCDesc = MII->get(Opcode: Inst.getOpcode());
539
540 // Record instruction size.
541 AddressToInstSizeMap[Address] = Size;
542
543 // Populate address maps.
544 CodeAddressVec.push_back(x: Address);
545 if (MCDesc.isCall()) {
546 CallAddressSet.insert(x: Address);
547 UncondBranchAddrSet.insert(x: Address);
548 } else if (MCDesc.isReturn()) {
549 RetAddressSet.insert(x: Address);
550 UncondBranchAddrSet.insert(x: Address);
551 } else if (MCDesc.isBranch()) {
552 if (MCDesc.isUnconditionalBranch())
553 UncondBranchAddrSet.insert(x: Address);
554 BranchAddressSet.insert(x: Address);
555 }
556
557 // Record potential call targets for tail frame inference later-on.
558 if (InferMissingFrames && FRange) {
559 uint64_t Target = 0;
560 MIA->evaluateBranch(Inst, Addr: Address, Size, Target);
561 if (MCDesc.isCall()) {
562 // Indirect call targets are unknown at this point. Recording the
563 // unknown target (zero) for further LBR-based refinement.
564 MissingContextInferrer->CallEdges[Address].insert(x: Target);
565 } else if (MCDesc.isUnconditionalBranch()) {
566 assert(Target &&
567 "target should be known for unconditional direct branch");
568 // Any inter-function unconditional jump is considered tail call at
569 // this point. This is not 100% accurate and could further be
570 // optimized based on some source annotation.
571 FuncRange *ToFRange = findFuncRange(Address: Target);
572 if (ToFRange && ToFRange->Func != FRange->Func)
573 MissingContextInferrer->TailCallEdges[Address].insert(x: Target);
574 LLVM_DEBUG({
575 dbgs() << "Direct Tail call: " << format("%8" PRIx64 ":", Address);
576 IPrinter->printInst(&Inst, Address + Size, "", *STI.get(), dbgs());
577 dbgs() << "\n";
578 });
579 } else if (MCDesc.isIndirectBranch() && MCDesc.isBarrier()) {
580 // This is an indirect branch but not necessarily an indirect tail
581 // call. The isBarrier check is to filter out conditional branch.
582 // Similar with indirect call targets, recording the unknown target
583 // (zero) for further LBR-based refinement.
584 MissingContextInferrer->TailCallEdges[Address].insert(x: Target);
585 LLVM_DEBUG({
586 dbgs() << "Indirect Tail call: "
587 << format("%8" PRIx64 ":", Address);
588 IPrinter->printInst(&Inst, Address + Size, "", *STI.get(), dbgs());
589 dbgs() << "\n";
590 });
591 }
592 }
593
594 if (InvalidInstLength) {
595 AddrsWithInvalidInstruction.insert(
596 V: {Address - InvalidInstLength, Address - 1});
597 InvalidInstLength = 0;
598 }
599 } else {
600 InvalidInstLength += Size;
601 }
602
603 Address += Size;
604 }
605
606 if (InvalidInstLength)
607 AddrsWithInvalidInstruction.insert(
608 V: {Address - InvalidInstLength, Address - 1});
609
610 if (ShowDisassembly)
611 outs() << "\n";
612
613 return true;
614}
615
616void ProfiledBinary::setUpDisassembler(const ObjectFile *Obj) {
617 const Target *TheTarget = getTarget(Obj);
618 std::string TripleName = TheTriple.getTriple();
619 StringRef FileName = Obj->getFileName();
620
621 MRI.reset(p: TheTarget->createMCRegInfo(TT: TripleName));
622 if (!MRI)
623 exitWithError(Message: "no register info for target " + TripleName, Whence: FileName);
624
625 MCTargetOptions MCOptions;
626 AsmInfo.reset(p: TheTarget->createMCAsmInfo(MRI: *MRI, TheTriple: TripleName, Options: MCOptions));
627 if (!AsmInfo)
628 exitWithError(Message: "no assembly info for target " + TripleName, Whence: FileName);
629
630 Expected<SubtargetFeatures> Features = Obj->getFeatures();
631 if (!Features)
632 exitWithError(E: Features.takeError(), Whence: FileName);
633 STI.reset(
634 p: TheTarget->createMCSubtargetInfo(TheTriple: TripleName, CPU: "", Features: Features->getString()));
635 if (!STI)
636 exitWithError(Message: "no subtarget info for target " + TripleName, Whence: FileName);
637
638 MII.reset(p: TheTarget->createMCInstrInfo());
639 if (!MII)
640 exitWithError(Message: "no instruction info for target " + TripleName, Whence: FileName);
641
642 MCContext Ctx(Triple(TripleName), AsmInfo.get(), MRI.get(), STI.get());
643 std::unique_ptr<MCObjectFileInfo> MOFI(
644 TheTarget->createMCObjectFileInfo(Ctx, /*PIC=*/false));
645 Ctx.setObjectFileInfo(MOFI.get());
646 DisAsm.reset(p: TheTarget->createMCDisassembler(STI: *STI, Ctx));
647 if (!DisAsm)
648 exitWithError(Message: "no disassembler for target " + TripleName, Whence: FileName);
649
650 MIA.reset(p: TheTarget->createMCInstrAnalysis(Info: MII.get()));
651
652 int AsmPrinterVariant = AsmInfo->getAssemblerDialect();
653 IPrinter.reset(p: TheTarget->createMCInstPrinter(
654 T: Triple(TripleName), SyntaxVariant: AsmPrinterVariant, MAI: *AsmInfo, MII: *MII, MRI: *MRI));
655 IPrinter->setPrintBranchImmAsAddress(true);
656}
657
658void ProfiledBinary::disassemble(const ObjectFile *Obj) {
659 // Set up disassembler and related components.
660 setUpDisassembler(Obj);
661
662 // Create a mapping from virtual address to symbol name. The symbols in text
663 // sections are the candidates to dissassemble.
664 std::map<SectionRef, SectionSymbolsTy> AllSymbols;
665 StringRef FileName = Obj->getFileName();
666 for (const SymbolRef &Symbol : Obj->symbols()) {
667 const uint64_t Addr = unwrapOrError(EO: Symbol.getAddress(), Args&: FileName);
668 const StringRef Name = unwrapOrError(EO: Symbol.getName(), Args&: FileName);
669 section_iterator SecI = unwrapOrError(EO: Symbol.getSection(), Args&: FileName);
670 if (SecI != Obj->section_end())
671 AllSymbols[*SecI].push_back(x: SymbolInfoTy(Addr, Name, ELF::STT_NOTYPE));
672 }
673
674 // Sort all the symbols. Use a stable sort to stabilize the output.
675 for (std::pair<const SectionRef, SectionSymbolsTy> &SecSyms : AllSymbols)
676 stable_sort(Range&: SecSyms.second);
677
678 assert((DisassembleFunctionSet.empty() || ShowDisassemblyOnly) &&
679 "Functions to disassemble should be only specified together with "
680 "--show-disassembly-only");
681
682 if (ShowDisassemblyOnly)
683 outs() << "\nDisassembly of " << FileName << ":\n";
684
685 // Dissassemble a text section.
686 for (section_iterator SI = Obj->section_begin(), SE = Obj->section_end();
687 SI != SE; ++SI) {
688 const SectionRef &Section = *SI;
689 if (!Section.isText())
690 continue;
691
692 uint64_t ImageLoadAddr = getPreferredBaseAddress();
693 uint64_t SectionAddress = Section.getAddress() - ImageLoadAddr;
694 uint64_t SectSize = Section.getSize();
695 if (!SectSize)
696 continue;
697
698 // Register the text section.
699 TextSections.insert(x: {SectionAddress, SectSize});
700
701 StringRef SectionName = unwrapOrError(EO: Section.getName(), Args&: FileName);
702
703 if (ShowDisassemblyOnly) {
704 outs() << "\nDisassembly of section " << SectionName;
705 outs() << " [" << format(Fmt: "0x%" PRIx64, Vals: Section.getAddress()) << ", "
706 << format(Fmt: "0x%" PRIx64, Vals: Section.getAddress() + SectSize)
707 << "]:\n\n";
708 }
709
710 if (isa<ELFObjectFileBase>(Val: Obj) && SectionName == ".plt")
711 continue;
712
713 // Get the section data.
714 ArrayRef<uint8_t> Bytes =
715 arrayRefFromStringRef(Input: unwrapOrError(EO: Section.getContents(), Args&: FileName));
716
717 // Get the list of all the symbols in this section.
718 SectionSymbolsTy &Symbols = AllSymbols[Section];
719
720 // Disassemble symbol by symbol.
721 for (std::size_t SI = 0, SE = Symbols.size(); SI != SE; ++SI) {
722 if (!dissassembleSymbol(SI, Bytes, Symbols, Section))
723 exitWithError(Message: "disassembling error", Whence: FileName);
724 }
725 }
726
727 if (!AddrsWithInvalidInstruction.empty()) {
728 if (ShowDetailedWarning) {
729 for (auto &Addr : AddrsWithInvalidInstruction) {
730 WithColor::warning()
731 << "Invalid instructions at " << format(Fmt: "%8" PRIx64, Vals: Addr.first)
732 << " - " << format(Fmt: "%8" PRIx64, Vals: Addr.second) << "\n";
733 }
734 }
735 WithColor::warning() << "Found " << AddrsWithInvalidInstruction.size()
736 << " invalid instructions\n";
737 AddrsWithInvalidInstruction.clear();
738 }
739
740 // Dissassemble rodata section to check if FS discriminator symbol exists.
741 checkUseFSDiscriminator(Obj, AllSymbols);
742}
743
744void ProfiledBinary::checkUseFSDiscriminator(
745 const ObjectFile *Obj, std::map<SectionRef, SectionSymbolsTy> &AllSymbols) {
746 const char *FSDiscriminatorVar = "__llvm_fs_discriminator__";
747 for (section_iterator SI = Obj->section_begin(), SE = Obj->section_end();
748 SI != SE; ++SI) {
749 const SectionRef &Section = *SI;
750 if (!Section.isData() || Section.getSize() == 0)
751 continue;
752 SectionSymbolsTy &Symbols = AllSymbols[Section];
753
754 for (std::size_t SI = 0, SE = Symbols.size(); SI != SE; ++SI) {
755 if (Symbols[SI].Name == FSDiscriminatorVar) {
756 UseFSDiscriminator = true;
757 return;
758 }
759 }
760 }
761}
762
763void ProfiledBinary::populateElfSymbolAddressList(
764 const ELFObjectFileBase *Obj) {
765 // Create a mapping from virtual address to symbol GUID and the other way
766 // around.
767 StringRef FileName = Obj->getFileName();
768 for (const SymbolRef &Symbol : Obj->symbols()) {
769 const uint64_t Addr = unwrapOrError(EO: Symbol.getAddress(), Args&: FileName);
770 const StringRef Name = unwrapOrError(EO: Symbol.getName(), Args&: FileName);
771 uint64_t GUID = Function::getGUID(GlobalName: Name);
772 SymbolStartAddrs[GUID] = Addr;
773 StartAddrToSymMap.emplace(args: Addr, args&: GUID);
774 }
775}
776
777void ProfiledBinary::loadSymbolsFromDWARFUnit(DWARFUnit &CompilationUnit) {
778 for (const auto &DieInfo : CompilationUnit.dies()) {
779 llvm::DWARFDie Die(&CompilationUnit, &DieInfo);
780
781 if (!Die.isSubprogramDIE())
782 continue;
783 auto Name = Die.getName(Kind: llvm::DINameKind::LinkageName);
784 if (!Name)
785 Name = Die.getName(Kind: llvm::DINameKind::ShortName);
786 if (!Name)
787 continue;
788
789 auto RangesOrError = Die.getAddressRanges();
790 if (!RangesOrError)
791 continue;
792 const DWARFAddressRangesVector &Ranges = RangesOrError.get();
793
794 if (Ranges.empty())
795 continue;
796
797 // Different DWARF symbols can have same function name, search or create
798 // BinaryFunction indexed by the name.
799 auto Ret = BinaryFunctions.emplace(args&: Name, args: BinaryFunction());
800 auto &Func = Ret.first->second;
801 if (Ret.second)
802 Func.FuncName = Ret.first->first;
803
804 for (const auto &Range : Ranges) {
805 uint64_t StartAddress = Range.LowPC;
806 uint64_t EndAddress = Range.HighPC;
807
808 if (EndAddress <= StartAddress ||
809 StartAddress < getPreferredBaseAddress())
810 continue;
811
812 // We may want to know all ranges for one function. Here group the
813 // ranges and store them into BinaryFunction.
814 Func.Ranges.emplace_back(args&: StartAddress, args&: EndAddress);
815
816 auto R = StartAddrToFuncRangeMap.emplace(args&: StartAddress, args: FuncRange());
817 if (R.second) {
818 FuncRange &FRange = R.first->second;
819 FRange.Func = &Func;
820 FRange.StartAddress = StartAddress;
821 FRange.EndAddress = EndAddress;
822 } else {
823 AddrsWithMultipleSymbols.insert(V: StartAddress);
824 if (ShowDetailedWarning)
825 WithColor::warning()
826 << "Duplicated symbol start address at "
827 << format(Fmt: "%8" PRIx64, Vals: StartAddress) << " "
828 << R.first->second.getFuncName() << " and " << Name << "\n";
829 }
830 }
831 }
832}
833
834void ProfiledBinary::loadSymbolsFromDWARF(ObjectFile &Obj) {
835 auto DebugContext = llvm::DWARFContext::create(
836 Obj, RelocAction: DWARFContext::ProcessDebugRelocations::Process, L: nullptr, DWPName: DWPPath);
837 if (!DebugContext)
838 exitWithError(Message: "Error creating the debug info context", Whence: Path);
839
840 for (const auto &CompilationUnit : DebugContext->compile_units())
841 loadSymbolsFromDWARFUnit(CompilationUnit&: *CompilationUnit.get());
842
843 // Handles DWO sections that can either be in .o, .dwo or .dwp files.
844 uint32_t NumOfDWOMissing = 0;
845 for (const auto &CompilationUnit : DebugContext->compile_units()) {
846 DWARFUnit *const DwarfUnit = CompilationUnit.get();
847 if (DwarfUnit->getDWOId()) {
848 DWARFUnit *DWOCU = DwarfUnit->getNonSkeletonUnitDIE(ExtractUnitDIEOnly: false).getDwarfUnit();
849 if (!DWOCU->isDWOUnit()) {
850 NumOfDWOMissing++;
851 if (ShowDetailedWarning) {
852 std::string DWOName = dwarf::toString(
853 V: DwarfUnit->getUnitDIE().find(
854 Attrs: {dwarf::DW_AT_dwo_name, dwarf::DW_AT_GNU_dwo_name}),
855 Default: "");
856 WithColor::warning() << "DWO debug information for " << DWOName
857 << " was not loaded.\n";
858 }
859 continue;
860 }
861 loadSymbolsFromDWARFUnit(CompilationUnit&: *DWOCU);
862 }
863 }
864
865 if (NumOfDWOMissing)
866 WithColor::warning()
867 << " DWO debug information was not loaded for " << NumOfDWOMissing
868 << " modules. Please check the .o, .dwo or .dwp path.\n";
869 if (BinaryFunctions.empty())
870 WithColor::warning() << "Loading of DWARF info completed, but no binary "
871 "functions have been retrieved.\n";
872 // Populate the hash binary function map for MD5 function name lookup. This
873 // is done after BinaryFunctions are finalized.
874 for (auto &BinaryFunction : BinaryFunctions) {
875 HashBinaryFunctions[MD5Hash(Str: StringRef(BinaryFunction.first))] =
876 &BinaryFunction.second;
877 }
878
879 if (!AddrsWithMultipleSymbols.empty()) {
880 WithColor::warning() << "Found " << AddrsWithMultipleSymbols.size()
881 << " start addresses with multiple symbols\n";
882 AddrsWithMultipleSymbols.clear();
883 }
884}
885
886void ProfiledBinary::populateSymbolListFromDWARF(
887 ProfileSymbolList &SymbolList) {
888 for (auto &I : StartAddrToFuncRangeMap)
889 SymbolList.add(Name: I.second.getFuncName());
890}
891
892symbolize::LLVMSymbolizer::Options ProfiledBinary::getSymbolizerOpts() const {
893 symbolize::LLVMSymbolizer::Options SymbolizerOpts;
894 SymbolizerOpts.PrintFunctions =
895 DILineInfoSpecifier::FunctionNameKind::LinkageName;
896 SymbolizerOpts.Demangle = false;
897 SymbolizerOpts.DefaultArch = TheTriple.getArchName().str();
898 SymbolizerOpts.UseSymbolTable = false;
899 SymbolizerOpts.RelativeAddresses = false;
900 SymbolizerOpts.DWPName = DWPPath;
901 return SymbolizerOpts;
902}
903
904SampleContextFrameVector ProfiledBinary::symbolize(const InstructionPointer &IP,
905 bool UseCanonicalFnName,
906 bool UseProbeDiscriminator) {
907 assert(this == IP.Binary &&
908 "Binary should only symbolize its own instruction");
909 auto Addr = object::SectionedAddress{.Address: IP.Address,
910 .SectionIndex: object::SectionedAddress::UndefSection};
911 DIInliningInfo InlineStack = unwrapOrError(
912 EO: Symbolizer->symbolizeInlinedCode(ModuleName: SymbolizerPath.str(), ModuleOffset: Addr),
913 Args&: SymbolizerPath);
914
915 SampleContextFrameVector CallStack;
916 for (int32_t I = InlineStack.getNumberOfFrames() - 1; I >= 0; I--) {
917 const auto &CallerFrame = InlineStack.getFrame(Index: I);
918 if (CallerFrame.FunctionName.empty() ||
919 (CallerFrame.FunctionName == "<invalid>"))
920 break;
921
922 StringRef FunctionName(CallerFrame.FunctionName);
923 if (UseCanonicalFnName)
924 FunctionName = FunctionSamples::getCanonicalFnName(FnName: FunctionName);
925
926 uint32_t Discriminator = CallerFrame.Discriminator;
927 uint32_t LineOffset = (CallerFrame.Line - CallerFrame.StartLine) & 0xffff;
928 if (UseProbeDiscriminator) {
929 LineOffset =
930 PseudoProbeDwarfDiscriminator::extractProbeIndex(Value: Discriminator);
931 Discriminator = 0;
932 }
933
934 LineLocation Line(LineOffset, Discriminator);
935 auto It = NameStrings.insert(x: FunctionName.str());
936 CallStack.emplace_back(Args: FunctionId(StringRef(*It.first)), Args&: Line);
937 }
938
939 return CallStack;
940}
941
942void ProfiledBinary::computeInlinedContextSizeForRange(uint64_t RangeBegin,
943 uint64_t RangeEnd) {
944 InstructionPointer IP(this, RangeBegin, true);
945
946 if (IP.Address != RangeBegin)
947 WithColor::warning() << "Invalid start instruction at "
948 << format(Fmt: "%8" PRIx64, Vals: RangeBegin) << "\n";
949
950 if (IP.Address >= RangeEnd)
951 return;
952
953 do {
954 const SampleContextFrameVector SymbolizedCallStack =
955 getFrameLocationStack(Address: IP.Address, UseProbeDiscriminator: UsePseudoProbes);
956 uint64_t Size = AddressToInstSizeMap[IP.Address];
957 // Record instruction size for the corresponding context
958 FuncSizeTracker.addInstructionForContext(Context: SymbolizedCallStack, InstrSize: Size);
959
960 } while (IP.advance() && IP.Address < RangeEnd);
961}
962
963void ProfiledBinary::computeInlinedContextSizeForFunc(
964 const BinaryFunction *Func) {
965 // Note that a function can be spilt into multiple ranges, so compute for all
966 // ranges of the function.
967 for (const auto &Range : Func->Ranges)
968 computeInlinedContextSizeForRange(RangeBegin: Range.first, RangeEnd: Range.second);
969
970 // Track optimized-away inlinee for probed binary. A function inlined and then
971 // optimized away should still have their probes left over in places.
972 if (usePseudoProbes()) {
973 auto I = TopLevelProbeFrameMap.find(Key: Func->FuncName);
974 if (I != TopLevelProbeFrameMap.end()) {
975 BinarySizeContextTracker::ProbeFrameStack ProbeContext;
976 FuncSizeTracker.trackInlineesOptimizedAway(ProbeDecoder, ProbeNode&: *I->second,
977 ProbeContext);
978 }
979 }
980}
981
982void ProfiledBinary::inferMissingFrames(
983 const SmallVectorImpl<uint64_t> &Context,
984 SmallVectorImpl<uint64_t> &NewContext) {
985 MissingContextInferrer->inferMissingFrames(Context, NewContext);
986}
987
988InstructionPointer::InstructionPointer(const ProfiledBinary *Binary,
989 uint64_t Address, bool RoundToNext)
990 : Binary(Binary), Address(Address) {
991 Index = Binary->getIndexForAddr(Address);
992 if (RoundToNext) {
993 // we might get address which is not the code
994 // it should round to the next valid address
995 if (Index >= Binary->getCodeAddrVecSize())
996 this->Address = UINT64_MAX;
997 else
998 this->Address = Binary->getAddressforIndex(Index);
999 }
1000}
1001
1002bool InstructionPointer::advance() {
1003 Index++;
1004 if (Index >= Binary->getCodeAddrVecSize()) {
1005 Address = UINT64_MAX;
1006 return false;
1007 }
1008 Address = Binary->getAddressforIndex(Index);
1009 return true;
1010}
1011
1012bool InstructionPointer::backward() {
1013 if (Index == 0) {
1014 Address = 0;
1015 return false;
1016 }
1017 Index--;
1018 Address = Binary->getAddressforIndex(Index);
1019 return true;
1020}
1021
1022void InstructionPointer::update(uint64_t Addr) {
1023 Address = Addr;
1024 Index = Binary->getIndexForAddr(Address);
1025}
1026
1027} // end namespace sampleprof
1028} // end namespace llvm
1029

source code of llvm/tools/llvm-profgen/ProfiledBinary.cpp