1//===- bolt/Profile/DataReader.cpp - Perf data reader ---------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This family of functions reads profile data written by the perf2bolt
10// utility and stores it in memory for llvm-bolt consumption.
11//
12//===----------------------------------------------------------------------===//
13
14#include "bolt/Profile/DataReader.h"
15#include "bolt/Core/BinaryFunction.h"
16#include "bolt/Passes/MCF.h"
17#include "bolt/Utils/Utils.h"
18#include "llvm/Support/CommandLine.h"
19#include "llvm/Support/Debug.h"
20#include "llvm/Support/Errc.h"
21
22#undef DEBUG_TYPE
23#define DEBUG_TYPE "bolt-prof"
24
25using namespace llvm;
26
27namespace opts {
28
29extern cl::OptionCategory BoltCategory;
30extern llvm::cl::opt<unsigned> Verbosity;
31
32static cl::opt<bool>
33DumpData("dump-data",
34 cl::desc("dump parsed bolt data for debugging"),
35 cl::Hidden,
36 cl::cat(BoltCategory));
37
38} // namespace opts
39
40namespace llvm {
41namespace bolt {
42
43namespace {
44
45/// Return true if the function name can change across compilations.
46bool hasVolatileName(const BinaryFunction &BF) {
47 for (const StringRef &Name : BF.getNames())
48 if (getLTOCommonName(Name))
49 return true;
50
51 return false;
52}
53
54/// Return standard escaped name of the function possibly renamed by BOLT.
55std::string normalizeName(StringRef NameRef) {
56 // Strip "PG." prefix used for globalized locals.
57 NameRef = NameRef.starts_with(Prefix: "PG.") ? NameRef.substr(Start: 2) : NameRef;
58 return getEscapedName(Name: NameRef);
59}
60
61} // anonymous namespace
62
63raw_ostream &operator<<(raw_ostream &OS, const Location &Loc) {
64 if (Loc.IsSymbol) {
65 OS << Loc.Name;
66 if (Loc.Offset)
67 OS << "+" << Twine::utohexstr(Val: Loc.Offset);
68 } else {
69 OS << Twine::utohexstr(Val: Loc.Offset);
70 }
71 return OS;
72}
73
74void FuncBranchData::appendFrom(const FuncBranchData &FBD, uint64_t Offset) {
75 Data.insert(position: Data.end(), first: FBD.Data.begin(), last: FBD.Data.end());
76 for (auto I = Data.begin(), E = Data.end(); I != E; ++I) {
77 if (I->From.Name == FBD.Name) {
78 I->From.Name = this->Name;
79 I->From.Offset += Offset;
80 }
81 if (I->To.Name == FBD.Name) {
82 I->To.Name = this->Name;
83 I->To.Offset += Offset;
84 }
85 }
86 llvm::stable_sort(Range&: Data);
87 ExecutionCount += FBD.ExecutionCount;
88 ExternEntryCount += FBD.ExternEntryCount;
89 for (auto I = FBD.EntryData.begin(), E = FBD.EntryData.end(); I != E; ++I) {
90 assert(I->To.Name == FBD.Name);
91 auto NewElmt = EntryData.insert(position: EntryData.end(), x: *I);
92 NewElmt->To.Name = this->Name;
93 NewElmt->To.Offset += Offset;
94 }
95}
96
97uint64_t FuncBranchData::getNumExecutedBranches() const {
98 uint64_t ExecutedBranches = 0;
99 for (const BranchInfo &BI : Data) {
100 int64_t BranchCount = BI.Branches;
101 assert(BranchCount >= 0 && "branch execution count should not be negative");
102 ExecutedBranches += BranchCount;
103 }
104 return ExecutedBranches;
105}
106
107void BasicSampleInfo::mergeWith(const BasicSampleInfo &SI) { Hits += SI.Hits; }
108
109void BasicSampleInfo::print(raw_ostream &OS) const {
110 OS << Loc.IsSymbol << " " << Loc.Name << " " << Twine::utohexstr(Val: Loc.Offset)
111 << " " << Hits << "\n";
112}
113
114uint64_t FuncBasicSampleData::getSamples(uint64_t Start, uint64_t End) const {
115 assert(llvm::is_sorted(Data));
116 struct Compare {
117 bool operator()(const BasicSampleInfo &SI, const uint64_t Val) const {
118 return SI.Loc.Offset < Val;
119 }
120 bool operator()(const uint64_t Val, const BasicSampleInfo &SI) const {
121 return Val < SI.Loc.Offset;
122 }
123 };
124 uint64_t Result = 0;
125 for (auto I = llvm::lower_bound(Range: Data, Value&: Start, C: Compare()),
126 E = llvm::lower_bound(Range: Data, Value&: End, C: Compare());
127 I != E; ++I)
128 Result += I->Hits;
129 return Result;
130}
131
132uint64_t FuncBasicSampleData::getSamples() const {
133 uint64_t Result = 0;
134 for (const BasicSampleInfo &I : Data)
135 Result += I.Hits;
136 return Result;
137}
138
139void FuncBasicSampleData::bumpCount(uint64_t Offset, uint64_t Count) {
140 auto Iter = Index.find(Val: Offset);
141 if (Iter == Index.end()) {
142 Data.emplace_back(args: Location(true, Name, Offset), args&: Count);
143 Index[Offset] = Data.size() - 1;
144 return;
145 }
146 BasicSampleInfo &SI = Data[Iter->second];
147 SI.Hits += Count;
148}
149
150void FuncBranchData::bumpBranchCount(uint64_t OffsetFrom, uint64_t OffsetTo,
151 uint64_t Count, uint64_t Mispreds) {
152 auto Iter = IntraIndex[OffsetFrom].find(Val: OffsetTo);
153 if (Iter == IntraIndex[OffsetFrom].end()) {
154 Data.emplace_back(args: Location(true, Name, OffsetFrom),
155 args: Location(true, Name, OffsetTo), args&: Mispreds, args&: Count);
156 IntraIndex[OffsetFrom][OffsetTo] = Data.size() - 1;
157 return;
158 }
159 BranchInfo &BI = Data[Iter->second];
160 BI.Branches += Count;
161 BI.Mispreds += Mispreds;
162}
163
164void FuncBranchData::bumpCallCount(uint64_t OffsetFrom, const Location &To,
165 uint64_t Count, uint64_t Mispreds) {
166 auto Iter = InterIndex[OffsetFrom].find(Val: To);
167 if (Iter == InterIndex[OffsetFrom].end()) {
168 Data.emplace_back(args: Location(true, Name, OffsetFrom), args: To, args&: Mispreds, args&: Count);
169 InterIndex[OffsetFrom][To] = Data.size() - 1;
170 return;
171 }
172 BranchInfo &BI = Data[Iter->second];
173 BI.Branches += Count;
174 BI.Mispreds += Mispreds;
175}
176
177void FuncBranchData::bumpEntryCount(const Location &From, uint64_t OffsetTo,
178 uint64_t Count, uint64_t Mispreds) {
179 auto Iter = EntryIndex[OffsetTo].find(Val: From);
180 if (Iter == EntryIndex[OffsetTo].end()) {
181 EntryData.emplace_back(args: From, args: Location(true, Name, OffsetTo), args&: Mispreds,
182 args&: Count);
183 EntryIndex[OffsetTo][From] = EntryData.size() - 1;
184 return;
185 }
186 BranchInfo &BI = EntryData[Iter->second];
187 BI.Branches += Count;
188 BI.Mispreds += Mispreds;
189}
190
191void BranchInfo::mergeWith(const BranchInfo &BI) {
192 Branches += BI.Branches;
193 Mispreds += BI.Mispreds;
194}
195
196void BranchInfo::print(raw_ostream &OS) const {
197 OS << From.IsSymbol << " " << From.Name << " "
198 << Twine::utohexstr(Val: From.Offset) << " " << To.IsSymbol << " " << To.Name
199 << " " << Twine::utohexstr(Val: To.Offset) << " " << Mispreds << " " << Branches
200 << '\n';
201}
202
203ErrorOr<const BranchInfo &> FuncBranchData::getBranch(uint64_t From,
204 uint64_t To) const {
205 for (const BranchInfo &I : Data)
206 if (I.From.Offset == From && I.To.Offset == To && I.From.Name == I.To.Name)
207 return I;
208
209 return make_error_code(E: llvm::errc::invalid_argument);
210}
211
212ErrorOr<const BranchInfo &>
213FuncBranchData::getDirectCallBranch(uint64_t From) const {
214 // Commented out because it can be expensive.
215 // assert(std::is_sorted(Data.begin(), Data.end()));
216 struct Compare {
217 bool operator()(const BranchInfo &BI, const uint64_t Val) const {
218 return BI.From.Offset < Val;
219 }
220 bool operator()(const uint64_t Val, const BranchInfo &BI) const {
221 return Val < BI.From.Offset;
222 }
223 };
224 auto Range = std::equal_range(first: Data.begin(), last: Data.end(), val: From, comp: Compare());
225 for (const auto &RI : llvm::make_range(p: Range))
226 if (RI.From.Name != RI.To.Name)
227 return RI;
228
229 return make_error_code(E: llvm::errc::invalid_argument);
230}
231
232void MemInfo::print(raw_ostream &OS) const {
233 OS << (Offset.IsSymbol + 3) << " " << Offset.Name << " "
234 << Twine::utohexstr(Val: Offset.Offset) << " " << (Addr.IsSymbol + 3) << " "
235 << Addr.Name << " " << Twine::utohexstr(Val: Addr.Offset) << " " << Count
236 << "\n";
237}
238
239void MemInfo::prettyPrint(raw_ostream &OS) const {
240 OS << "(PC: " << Offset << ", M: " << Addr << ", C: " << Count << ")";
241}
242
243void FuncMemData::update(const Location &Offset, const Location &Addr) {
244 auto Iter = EventIndex[Offset.Offset].find(Val: Addr);
245 if (Iter == EventIndex[Offset.Offset].end()) {
246 Data.emplace_back(args: MemInfo(Offset, Addr, 1));
247 EventIndex[Offset.Offset][Addr] = Data.size() - 1;
248 return;
249 }
250 ++Data[Iter->second].Count;
251}
252
253Error DataReader::preprocessProfile(BinaryContext &BC) {
254 if (std::error_code EC = parseInput())
255 return errorCodeToError(EC);
256
257 if (opts::DumpData)
258 dump();
259
260 if (collectedInBoltedBinary())
261 outs() << "BOLT-INFO: profile collection done on a binary already "
262 "processed by BOLT\n";
263
264 for (auto &BFI : BC.getBinaryFunctions()) {
265 BinaryFunction &Function = BFI.second;
266 if (FuncMemData *MemData = getMemDataForNames(FuncNames: Function.getNames())) {
267 setMemData(BF: Function, FMD: MemData);
268 MemData->Used = true;
269 }
270 if (FuncBranchData *FuncData = getBranchDataForNames(FuncNames: Function.getNames())) {
271 setBranchData(BF: Function, FBD: FuncData);
272 Function.ExecutionCount = FuncData->ExecutionCount;
273 Function.ExternEntryCount = FuncData->ExternEntryCount;
274 FuncData->Used = true;
275 }
276 }
277
278 for (auto &BFI : BC.getBinaryFunctions()) {
279 BinaryFunction &Function = BFI.second;
280 matchProfileMemData(BF&: Function);
281 }
282
283 return Error::success();
284}
285
286Error DataReader::readProfilePreCFG(BinaryContext &BC) {
287 for (auto &BFI : BC.getBinaryFunctions()) {
288 BinaryFunction &Function = BFI.second;
289 FuncMemData *MemoryData = getMemData(BF: Function);
290 if (!MemoryData)
291 continue;
292
293 for (MemInfo &MI : MemoryData->Data) {
294 const uint64_t Offset = MI.Offset.Offset;
295 auto II = Function.Instructions.find(x: Offset);
296 if (II == Function.Instructions.end()) {
297 // Ignore bad instruction address.
298 continue;
299 }
300
301 auto &MemAccessProfile =
302 BC.MIB->getOrCreateAnnotationAs<MemoryAccessProfile>(
303 Inst&: II->second, Name: "MemoryAccessProfile");
304 BinaryData *BD = nullptr;
305 if (MI.Addr.IsSymbol)
306 BD = BC.getBinaryDataByName(Name: MI.Addr.Name);
307 MemAccessProfile.AddressAccessInfo.push_back(
308 Elt: {.MemoryObject: BD, .Offset: MI.Addr.Offset, .Count: MI.Count});
309 auto NextII = std::next(x: II);
310 if (NextII == Function.Instructions.end())
311 MemAccessProfile.NextInstrOffset = Function.getSize();
312 else
313 MemAccessProfile.NextInstrOffset = II->first;
314 }
315 Function.HasMemoryProfile = true;
316 }
317
318 return Error::success();
319}
320
321Error DataReader::readProfile(BinaryContext &BC) {
322 for (auto &BFI : BC.getBinaryFunctions()) {
323 BinaryFunction &Function = BFI.second;
324 readProfile(BF&: Function);
325 }
326
327 uint64_t NumUnused = 0;
328 for (const auto &KV : NamesToBranches) {
329 const FuncBranchData &FBD = KV.second;
330 if (!FBD.Used)
331 ++NumUnused;
332 }
333 BC.setNumUnusedProfiledObjects(NumUnused);
334
335 return Error::success();
336}
337
338std::error_code DataReader::parseInput() {
339 ErrorOr<std::unique_ptr<MemoryBuffer>> MB =
340 MemoryBuffer::getFileOrSTDIN(Filename);
341 if (std::error_code EC = MB.getError()) {
342 Diag << "cannot open " << Filename << ": " << EC.message() << "\n";
343 return EC;
344 }
345 FileBuf = std::move(MB.get());
346 ParsingBuf = FileBuf->getBuffer();
347 if (std::error_code EC = parse())
348 return EC;
349 if (!ParsingBuf.empty())
350 Diag << "WARNING: invalid profile data detected at line " << Line
351 << ". Possibly corrupted profile.\n";
352
353 buildLTONameMaps();
354
355 return std::error_code();
356}
357
358void DataReader::readProfile(BinaryFunction &BF) {
359 if (BF.empty())
360 return;
361
362 if (!hasLBR()) {
363 BF.ProfileFlags = BinaryFunction::PF_BASIC;
364 readBasicSampleData(BF);
365 return;
366 }
367
368 BF.ProfileFlags = BinaryFunction::PF_BRANCH;
369
370 // Possibly assign/re-assign branch profile data.
371 matchProfileData(BF);
372
373 FuncBranchData *FBD = getBranchData(BF);
374 if (!FBD)
375 return;
376
377 // Assign basic block counts to function entry points. These only include
378 // counts for outside entries.
379 //
380 // There is a slight skew introduced here as branches originated from RETs
381 // may be accounted for in the execution count of an entry block if the last
382 // instruction in a predecessor fall-through block is a call. This situation
383 // should rarely happen because there are few multiple-entry functions.
384 for (const BranchInfo &BI : FBD->EntryData) {
385 BinaryBasicBlock *BB = BF.getBasicBlockAtOffset(Offset: BI.To.Offset);
386 if (BB && (BB->isEntryPoint() || BB->isLandingPad())) {
387 uint64_t Count = BB->getExecutionCount();
388 if (Count == BinaryBasicBlock::COUNT_NO_PROFILE)
389 Count = 0;
390 BB->setExecutionCount(Count + BI.Branches);
391 }
392 }
393
394 for (const BranchInfo &BI : FBD->Data) {
395 if (BI.From.Name != BI.To.Name)
396 continue;
397
398 if (!recordBranch(BF, From: BI.From.Offset, To: BI.To.Offset, Count: BI.Branches,
399 Mispreds: BI.Mispreds)) {
400 LLVM_DEBUG(dbgs() << "bad branch : " << BI.From.Offset << " -> "
401 << BI.To.Offset << '\n');
402 }
403 }
404
405 // Convert branch data into annotations.
406 convertBranchData(BF);
407}
408
409void DataReader::matchProfileData(BinaryFunction &BF) {
410 // This functionality is available for LBR-mode only
411 // TODO: Implement evaluateProfileData() for samples, checking whether
412 // sample addresses match instruction addresses in the function
413 if (!hasLBR())
414 return;
415
416 FuncBranchData *FBD = getBranchData(BF);
417 if (FBD) {
418 BF.ProfileMatchRatio = evaluateProfileData(BF, BranchData: *FBD);
419 BF.RawSampleCount = FBD->getNumExecutedBranches();
420 if (BF.ProfileMatchRatio == 1.0f) {
421 if (fetchProfileForOtherEntryPoints(BF)) {
422 BF.ProfileMatchRatio = evaluateProfileData(BF, BranchData: *FBD);
423 BF.ExecutionCount = FBD->ExecutionCount;
424 BF.ExternEntryCount = FBD->ExternEntryCount;
425 BF.RawSampleCount = FBD->getNumExecutedBranches();
426 }
427 return;
428 }
429 }
430
431 // Check if the function name can fluctuate between several compilations
432 // possibly triggered by minor unrelated code changes in the source code
433 // of the input binary.
434 if (!hasVolatileName(BF))
435 return;
436
437 // Check for a profile that matches with 100% confidence.
438 const std::vector<FuncBranchData *> AllBranchData =
439 getBranchDataForNamesRegex(FuncNames: BF.getNames());
440 for (FuncBranchData *NewBranchData : AllBranchData) {
441 // Prevent functions from sharing the same profile.
442 if (NewBranchData->Used)
443 continue;
444
445 if (evaluateProfileData(BF, BranchData: *NewBranchData) != 1.0f)
446 continue;
447
448 if (FBD)
449 FBD->Used = false;
450
451 // Update function profile data with the new set.
452 setBranchData(BF, FBD: NewBranchData);
453 NewBranchData->Used = true;
454 BF.ExecutionCount = NewBranchData->ExecutionCount;
455 BF.ExternEntryCount = NewBranchData->ExternEntryCount;
456 BF.ProfileMatchRatio = 1.0f;
457 break;
458 }
459}
460
461void DataReader::matchProfileMemData(BinaryFunction &BF) {
462 const std::vector<FuncMemData *> AllMemData =
463 getMemDataForNamesRegex(FuncNames: BF.getNames());
464 for (FuncMemData *NewMemData : AllMemData) {
465 // Prevent functions from sharing the same profile.
466 if (NewMemData->Used)
467 continue;
468
469 if (FuncMemData *MD = getMemData(BF))
470 MD->Used = false;
471
472 // Update function profile data with the new set.
473 setMemData(BF, FMD: NewMemData);
474 NewMemData->Used = true;
475 break;
476 }
477}
478
479bool DataReader::fetchProfileForOtherEntryPoints(BinaryFunction &BF) {
480 BinaryContext &BC = BF.getBinaryContext();
481
482 FuncBranchData *FBD = getBranchData(BF);
483 if (!FBD)
484 return false;
485
486 // Check if we are missing profiling data for secondary entry points
487 bool First = true;
488 bool Updated = false;
489 for (BinaryBasicBlock *BB : BF.BasicBlocks) {
490 if (First) {
491 First = false;
492 continue;
493 }
494 if (BB->isEntryPoint()) {
495 uint64_t EntryAddress = BB->getOffset() + BF.getAddress();
496 // Look for branch data associated with this entry point
497 if (BinaryData *BD = BC.getBinaryDataAtAddress(Address: EntryAddress)) {
498 if (FuncBranchData *Data = getBranchDataForSymbols(Symbols: BD->getSymbols())) {
499 FBD->appendFrom(FBD: *Data, Offset: BB->getOffset());
500 Data->Used = true;
501 Updated = true;
502 }
503 }
504 }
505 }
506
507 return Updated;
508}
509
510float DataReader::evaluateProfileData(BinaryFunction &BF,
511 const FuncBranchData &BranchData) const {
512 BinaryContext &BC = BF.getBinaryContext();
513
514 // Until we define a minimal profile, we consider an empty branch data to be
515 // a valid profile. It could happen to a function without branches when we
516 // still have an EntryData for the execution count.
517 if (BranchData.Data.empty())
518 return 1.0f;
519
520 uint64_t NumMatchedBranches = 0;
521 for (const BranchInfo &BI : BranchData.Data) {
522 bool IsValid = false;
523 if (BI.From.Name == BI.To.Name) {
524 // Try to record information with 0 count.
525 IsValid = recordBranch(BF, From: BI.From.Offset, To: BI.To.Offset, Count: 0);
526 } else if (collectedInBoltedBinary()) {
527 // We can't check branch source for collections in bolted binaries because
528 // the source of the branch may be mapped to the first instruction in a BB
529 // instead of the original branch (which may not exist in the source bin).
530 IsValid = true;
531 } else {
532 // The branch has to originate from this function.
533 // Check for calls, tail calls, rets and indirect branches.
534 // When matching profiling info, we did not reach the stage
535 // when we identify tail calls, so they are still represented
536 // by regular branch instructions and we need isBranch() here.
537 MCInst *Instr = BF.getInstructionAtOffset(Offset: BI.From.Offset);
538 // If it's a prefix - skip it.
539 if (Instr && BC.MIB->isPrefix(Inst: *Instr))
540 Instr = BF.getInstructionAtOffset(Offset: BI.From.Offset + 1);
541 if (Instr && (BC.MIB->isCall(Inst: *Instr) || BC.MIB->isBranch(Inst: *Instr) ||
542 BC.MIB->isReturn(Inst: *Instr)))
543 IsValid = true;
544 }
545
546 if (IsValid) {
547 ++NumMatchedBranches;
548 continue;
549 }
550
551 LLVM_DEBUG(dbgs() << "\tinvalid branch in " << BF << " : 0x"
552 << Twine::utohexstr(BI.From.Offset) << " -> ";
553 if (BI.From.Name == BI.To.Name) dbgs()
554 << "0x" << Twine::utohexstr(BI.To.Offset) << '\n';
555 else dbgs() << "<outbounds>\n";);
556 }
557
558 const float MatchRatio = (float)NumMatchedBranches / BranchData.Data.size();
559 if (opts::Verbosity >= 2 && NumMatchedBranches < BranchData.Data.size())
560 errs() << "BOLT-WARNING: profile branches match only "
561 << format(Fmt: "%.1f%%", Vals: MatchRatio * 100.0f) << " ("
562 << NumMatchedBranches << '/' << BranchData.Data.size()
563 << ") for function " << BF << '\n';
564
565 return MatchRatio;
566}
567
568void DataReader::readBasicSampleData(BinaryFunction &BF) {
569 FuncBasicSampleData *SampleDataOrErr = getFuncBasicSampleData(FuncNames: BF.getNames());
570 if (!SampleDataOrErr)
571 return;
572
573 // Basic samples mode territory (without LBR info)
574 // First step is to assign BB execution count based on samples from perf
575 BF.ProfileMatchRatio = 1.0f;
576 BF.removeTagsFromProfile();
577 bool NormalizeByInsnCount = usesEvent(Name: "cycles") || usesEvent(Name: "instructions");
578 bool NormalizeByCalls = usesEvent(Name: "branches");
579 static bool NagUser = true;
580 if (NagUser) {
581 outs()
582 << "BOLT-INFO: operating with basic samples profiling data (no LBR).\n";
583 if (NormalizeByInsnCount)
584 outs() << "BOLT-INFO: normalizing samples by instruction count.\n";
585 else if (NormalizeByCalls)
586 outs() << "BOLT-INFO: normalizing samples by branches.\n";
587
588 NagUser = false;
589 }
590 uint64_t LastOffset = BF.getSize();
591 uint64_t TotalEntryCount = 0;
592 for (BinaryFunction::BasicBlockOffset &BBOffset :
593 llvm::reverse(C&: BF.BasicBlockOffsets)) {
594 uint64_t CurOffset = BBOffset.first;
595 // Always work with samples multiplied by 1000 to avoid losing them if we
596 // later need to normalize numbers
597 uint64_t NumSamples =
598 SampleDataOrErr->getSamples(Start: CurOffset, End: LastOffset) * 1000;
599 if (NormalizeByInsnCount && BBOffset.second->getNumNonPseudos()) {
600 NumSamples /= BBOffset.second->getNumNonPseudos();
601 } else if (NormalizeByCalls) {
602 uint32_t NumCalls = BBOffset.second->getNumCalls();
603 NumSamples /= NumCalls + 1;
604 }
605 BBOffset.second->setExecutionCount(NumSamples);
606 if (BBOffset.second->isEntryPoint())
607 TotalEntryCount += NumSamples;
608 LastOffset = CurOffset;
609 }
610
611 BF.ExecutionCount = TotalEntryCount;
612}
613
614void DataReader::convertBranchData(BinaryFunction &BF) const {
615 BinaryContext &BC = BF.getBinaryContext();
616
617 if (BF.empty())
618 return;
619
620 FuncBranchData *FBD = getBranchData(BF);
621 if (!FBD)
622 return;
623
624 // Profile information for calls.
625 //
626 // There are 3 cases that we annotate differently:
627 // 1) Conditional tail calls that could be mispredicted.
628 // 2) Indirect calls to multiple destinations with mispredictions.
629 // Before we validate CFG we have to handle indirect branches here too.
630 // 3) Regular direct calls. The count could be different from containing
631 // basic block count. Keep this data in case we find it useful.
632 //
633 for (BranchInfo &BI : FBD->Data) {
634 // Ignore internal branches.
635 if (BI.To.IsSymbol && BI.To.Name == BI.From.Name && BI.To.Offset != 0)
636 continue;
637
638 MCInst *Instr = BF.getInstructionAtOffset(Offset: BI.From.Offset);
639 if (!Instr ||
640 (!BC.MIB->isCall(Inst: *Instr) && !BC.MIB->isIndirectBranch(Inst: *Instr)))
641 continue;
642
643 auto setOrUpdateAnnotation = [&](StringRef Name, uint64_t Count) {
644 if (opts::Verbosity >= 1 && BC.MIB->hasAnnotation(Inst: *Instr, Name))
645 errs() << "BOLT-WARNING: duplicate " << Name << " info for offset 0x"
646 << Twine::utohexstr(Val: BI.From.Offset) << " in function " << BF
647 << '\n';
648 auto &Value = BC.MIB->getOrCreateAnnotationAs<uint64_t>(Inst&: *Instr, Name);
649 Value += Count;
650 };
651
652 if (BC.MIB->isIndirectCall(Inst: *Instr) || BC.MIB->isIndirectBranch(Inst: *Instr)) {
653 IndirectCallSiteProfile &CSP =
654 BC.MIB->getOrCreateAnnotationAs<IndirectCallSiteProfile>(
655 Inst&: *Instr, Name: "CallProfile");
656 MCSymbol *CalleeSymbol = nullptr;
657 if (BI.To.IsSymbol) {
658 if (BinaryData *BD = BC.getBinaryDataByName(Name: BI.To.Name))
659 CalleeSymbol = BD->getSymbol();
660 }
661 CSP.emplace_back(Args&: CalleeSymbol, Args&: BI.Branches, Args&: BI.Mispreds);
662 } else if (BC.MIB->getConditionalTailCall(Inst: *Instr)) {
663 setOrUpdateAnnotation("CTCTakenCount", BI.Branches);
664 setOrUpdateAnnotation("CTCMispredCount", BI.Mispreds);
665 } else {
666 setOrUpdateAnnotation("Count", BI.Branches);
667 }
668 }
669}
670
671bool DataReader::recordBranch(BinaryFunction &BF, uint64_t From, uint64_t To,
672 uint64_t Count, uint64_t Mispreds) const {
673 BinaryContext &BC = BF.getBinaryContext();
674
675 BinaryBasicBlock *FromBB = BF.getBasicBlockContainingOffset(Offset: From);
676 const BinaryBasicBlock *ToBB = BF.getBasicBlockContainingOffset(Offset: To);
677
678 if (!FromBB || !ToBB) {
679 LLVM_DEBUG(dbgs() << "failed to get block for recorded branch\n");
680 return false;
681 }
682
683 // Could be bad LBR data; ignore the branch. In the case of data collected
684 // in binaries optimized by BOLT, a source BB may be mapped to two output
685 // BBs as a result of optimizations. In that case, a branch between these
686 // two will be recorded as a branch from A going to A in the source address
687 // space. Keep processing.
688 if (From == To)
689 return true;
690
691 // Return from a tail call.
692 if (FromBB->succ_size() == 0)
693 return true;
694
695 // Very rarely we will see ignored branches. Do a linear check.
696 for (std::pair<uint32_t, uint32_t> &Branch : BF.IgnoredBranches)
697 if (Branch ==
698 std::make_pair(x: static_cast<uint32_t>(From), y: static_cast<uint32_t>(To)))
699 return true;
700
701 bool OffsetMatches = !!(To == ToBB->getOffset());
702 if (!OffsetMatches) {
703 // Skip the nops to support old .fdata
704 uint64_t Offset = ToBB->getOffset();
705 for (const MCInst &Instr : *ToBB) {
706 if (!BC.MIB->isNoop(Inst: Instr))
707 break;
708
709 if (std::optional<uint32_t> Size = BC.MIB->getSize(Inst: Instr))
710 Offset += *Size;
711 }
712
713 if (To == Offset)
714 OffsetMatches = true;
715 }
716
717 if (!OffsetMatches) {
718 // "To" could be referring to nop instructions in between 2 basic blocks.
719 // While building the CFG we make sure these nops are attributed to the
720 // previous basic block, thus we check if the destination belongs to the
721 // gap past the last instruction.
722 const MCInst *LastInstr = ToBB->getLastNonPseudoInstr();
723 if (LastInstr) {
724 const uint32_t LastInstrOffset =
725 BC.MIB->getOffsetWithDefault(Inst: *LastInstr, Default: 0);
726
727 // With old .fdata we are getting FT branches for "jcc,jmp" sequences.
728 if (To == LastInstrOffset && BC.MIB->isUnconditionalBranch(Inst: *LastInstr))
729 return true;
730
731 if (To <= LastInstrOffset) {
732 LLVM_DEBUG(dbgs() << "branch recorded into the middle of the block"
733 << " in " << BF << " : " << From << " -> " << To
734 << '\n');
735 return false;
736 }
737 }
738
739 // The real destination is the layout successor of the detected ToBB.
740 if (ToBB == BF.getLayout().block_back())
741 return false;
742 const BinaryBasicBlock *NextBB =
743 BF.getLayout().getBlock(Index: ToBB->getIndex() + 1);
744 assert((NextBB && NextBB->getOffset() > ToBB->getOffset()) && "bad layout");
745 ToBB = NextBB;
746 }
747
748 // If there's no corresponding instruction for 'From', we have probably
749 // discarded it as a FT from __builtin_unreachable.
750 MCInst *FromInstruction = BF.getInstructionAtOffset(Offset: From);
751 if (!FromInstruction) {
752 // If the data was collected in a bolted binary, the From addresses may be
753 // translated to the first instruction of the source BB if BOLT inserted
754 // a new branch that did not exist in the source (we can't map it to the
755 // source instruction, so we map it to the first instr of source BB).
756 // We do not keep offsets for random instructions. So the check above will
757 // evaluate to true if the first instr is not a branch (call/jmp/ret/etc)
758 if (collectedInBoltedBinary()) {
759 if (FromBB->getInputOffset() != From) {
760 LLVM_DEBUG(dbgs() << "offset " << From << " does not match a BB in "
761 << BF << '\n');
762 return false;
763 }
764 FromInstruction = nullptr;
765 } else {
766 LLVM_DEBUG(dbgs() << "no instruction for offset " << From << " in " << BF
767 << '\n');
768 return false;
769 }
770 }
771
772 if (!FromBB->getSuccessor(Label: ToBB->getLabel())) {
773 // Check if this is a recursive call or a return from a recursive call.
774 if (FromInstruction && ToBB->isEntryPoint() &&
775 (BC.MIB->isCall(Inst: *FromInstruction) ||
776 BC.MIB->isIndirectBranch(Inst: *FromInstruction))) {
777 // Execution count is already accounted for.
778 return true;
779 }
780 // For data collected in a bolted binary, we may have created two output BBs
781 // that map to one original block. Branches between these two blocks will
782 // appear here as one BB jumping to itself, even though it has no loop
783 // edges. Ignore these.
784 if (collectedInBoltedBinary() && FromBB == ToBB)
785 return true;
786
787 // Allow passthrough blocks.
788 BinaryBasicBlock *FTSuccessor = FromBB->getConditionalSuccessor(Condition: false);
789 if (FTSuccessor && FTSuccessor->succ_size() == 1 &&
790 FTSuccessor->getSuccessor(Label: ToBB->getLabel())) {
791 BinaryBasicBlock::BinaryBranchInfo &FTBI =
792 FTSuccessor->getBranchInfo(Succ: *ToBB);
793 FTBI.Count += Count;
794 if (Count)
795 FTBI.MispredictedCount += Mispreds;
796 ToBB = FTSuccessor;
797 } else {
798 LLVM_DEBUG(dbgs() << "invalid branch in " << BF
799 << formatv(": {0:x} -> {1:x}\n", From, To));
800 return false;
801 }
802 }
803
804 BinaryBasicBlock::BinaryBranchInfo &BI = FromBB->getBranchInfo(Succ: *ToBB);
805 BI.Count += Count;
806 // Only update mispredicted count if it the count was real.
807 if (Count) {
808 BI.MispredictedCount += Mispreds;
809 }
810
811 return true;
812}
813
814void DataReader::reportError(StringRef ErrorMsg) {
815 Diag << "Error reading BOLT data input file: line " << Line << ", column "
816 << Col << ": " << ErrorMsg << '\n';
817}
818
819bool DataReader::expectAndConsumeFS() {
820 if (ParsingBuf[0] != FieldSeparator) {
821 reportError(ErrorMsg: "expected field separator");
822 return false;
823 }
824 ParsingBuf = ParsingBuf.drop_front(N: 1);
825 Col += 1;
826 return true;
827}
828
829void DataReader::consumeAllRemainingFS() {
830 while (ParsingBuf[0] == FieldSeparator) {
831 ParsingBuf = ParsingBuf.drop_front(N: 1);
832 Col += 1;
833 }
834}
835
836bool DataReader::checkAndConsumeNewLine() {
837 if (ParsingBuf[0] != '\n')
838 return false;
839
840 ParsingBuf = ParsingBuf.drop_front(N: 1);
841 Col = 0;
842 Line += 1;
843 return true;
844}
845
846ErrorOr<StringRef> DataReader::parseString(char EndChar, bool EndNl) {
847 if (EndChar == '\\') {
848 reportError(ErrorMsg: "EndChar could not be backslash");
849 return make_error_code(E: llvm::errc::io_error);
850 }
851
852 std::string EndChars(1, EndChar);
853 EndChars.push_back(c: '\\');
854 if (EndNl)
855 EndChars.push_back(c: '\n');
856
857 size_t StringEnd = 0;
858 do {
859 StringEnd = ParsingBuf.find_first_of(Chars: EndChars, From: StringEnd);
860 if (StringEnd == StringRef::npos ||
861 (StringEnd == 0 && ParsingBuf[StringEnd] != '\\')) {
862 reportError(ErrorMsg: "malformed field");
863 return make_error_code(E: llvm::errc::io_error);
864 }
865
866 if (ParsingBuf[StringEnd] != '\\')
867 break;
868
869 StringEnd += 2;
870 } while (true);
871
872 StringRef Str = ParsingBuf.substr(Start: 0, N: StringEnd);
873
874 // If EndNl was set and nl was found instead of EndChar, do not consume the
875 // new line.
876 bool EndNlInsteadOfEndChar = ParsingBuf[StringEnd] == '\n' && EndChar != '\n';
877 unsigned End = EndNlInsteadOfEndChar ? StringEnd : StringEnd + 1;
878
879 ParsingBuf = ParsingBuf.drop_front(N: End);
880 if (EndChar == '\n') {
881 Col = 0;
882 Line += 1;
883 } else {
884 Col += End;
885 }
886 return Str;
887}
888
889ErrorOr<int64_t> DataReader::parseNumberField(char EndChar, bool EndNl) {
890 ErrorOr<StringRef> NumStrRes = parseString(EndChar, EndNl);
891 if (std::error_code EC = NumStrRes.getError())
892 return EC;
893 StringRef NumStr = NumStrRes.get();
894 int64_t Num;
895 if (NumStr.getAsInteger(Radix: 10, Result&: Num)) {
896 reportError(ErrorMsg: "expected decimal number");
897 Diag << "Found: " << NumStr << "\n";
898 return make_error_code(E: llvm::errc::io_error);
899 }
900 return Num;
901}
902
903ErrorOr<uint64_t> DataReader::parseHexField(char EndChar, bool EndNl) {
904 ErrorOr<StringRef> NumStrRes = parseString(EndChar, EndNl);
905 if (std::error_code EC = NumStrRes.getError())
906 return EC;
907 StringRef NumStr = NumStrRes.get();
908 uint64_t Num;
909 if (NumStr.getAsInteger(Radix: 16, Result&: Num)) {
910 reportError(ErrorMsg: "expected hexidecimal number");
911 Diag << "Found: " << NumStr << "\n";
912 return make_error_code(E: llvm::errc::io_error);
913 }
914 return Num;
915}
916
917ErrorOr<Location> DataReader::parseLocation(char EndChar, bool EndNl,
918 bool ExpectMemLoc) {
919 // Read whether the location of the branch should be DSO or a symbol
920 // 0 means it is a DSO. 1 means it is a global symbol. 2 means it is a local
921 // symbol.
922 // The symbol flag is also used to tag memory load events by adding 3 to the
923 // base values, i.e. 3 not a symbol, 4 global symbol and 5 local symbol.
924 if (!ExpectMemLoc && ParsingBuf[0] != '0' && ParsingBuf[0] != '1' &&
925 ParsingBuf[0] != '2') {
926 reportError(ErrorMsg: "expected 0, 1 or 2");
927 return make_error_code(E: llvm::errc::io_error);
928 }
929
930 if (ExpectMemLoc && ParsingBuf[0] != '3' && ParsingBuf[0] != '4' &&
931 ParsingBuf[0] != '5') {
932 reportError(ErrorMsg: "expected 3, 4 or 5");
933 return make_error_code(E: llvm::errc::io_error);
934 }
935
936 bool IsSymbol =
937 (!ExpectMemLoc && (ParsingBuf[0] == '1' || ParsingBuf[0] == '2')) ||
938 (ExpectMemLoc && (ParsingBuf[0] == '4' || ParsingBuf[0] == '5'));
939 ParsingBuf = ParsingBuf.drop_front(N: 1);
940 Col += 1;
941
942 if (!expectAndConsumeFS())
943 return make_error_code(E: llvm::errc::io_error);
944 consumeAllRemainingFS();
945
946 // Read the string containing the symbol or the DSO name
947 ErrorOr<StringRef> NameRes = parseString(EndChar: FieldSeparator);
948 if (std::error_code EC = NameRes.getError())
949 return EC;
950 StringRef Name = NameRes.get();
951 consumeAllRemainingFS();
952
953 // Read the offset
954 ErrorOr<uint64_t> Offset = parseHexField(EndChar, EndNl);
955 if (std::error_code EC = Offset.getError())
956 return EC;
957
958 return Location(IsSymbol, Name, Offset.get());
959}
960
961ErrorOr<BranchInfo> DataReader::parseBranchInfo() {
962 ErrorOr<Location> Res = parseLocation(EndChar: FieldSeparator);
963 if (std::error_code EC = Res.getError())
964 return EC;
965 Location From = Res.get();
966
967 consumeAllRemainingFS();
968 Res = parseLocation(EndChar: FieldSeparator);
969 if (std::error_code EC = Res.getError())
970 return EC;
971 Location To = Res.get();
972
973 consumeAllRemainingFS();
974 ErrorOr<int64_t> MRes = parseNumberField(EndChar: FieldSeparator);
975 if (std::error_code EC = MRes.getError())
976 return EC;
977 int64_t NumMispreds = MRes.get();
978
979 consumeAllRemainingFS();
980 ErrorOr<int64_t> BRes = parseNumberField(EndChar: FieldSeparator, /* EndNl = */ true);
981 if (std::error_code EC = BRes.getError())
982 return EC;
983 int64_t NumBranches = BRes.get();
984
985 consumeAllRemainingFS();
986 if (!checkAndConsumeNewLine()) {
987 reportError(ErrorMsg: "expected end of line");
988 return make_error_code(E: llvm::errc::io_error);
989 }
990
991 return BranchInfo(std::move(From), std::move(To), NumMispreds, NumBranches);
992}
993
994ErrorOr<MemInfo> DataReader::parseMemInfo() {
995 ErrorOr<Location> Res = parseMemLocation(EndChar: FieldSeparator);
996 if (std::error_code EC = Res.getError())
997 return EC;
998 Location Offset = Res.get();
999
1000 consumeAllRemainingFS();
1001 Res = parseMemLocation(EndChar: FieldSeparator);
1002 if (std::error_code EC = Res.getError())
1003 return EC;
1004 Location Addr = Res.get();
1005
1006 consumeAllRemainingFS();
1007 ErrorOr<int64_t> CountRes = parseNumberField(EndChar: FieldSeparator, EndNl: true);
1008 if (std::error_code EC = CountRes.getError())
1009 return EC;
1010
1011 consumeAllRemainingFS();
1012 if (!checkAndConsumeNewLine()) {
1013 reportError(ErrorMsg: "expected end of line");
1014 return make_error_code(E: llvm::errc::io_error);
1015 }
1016
1017 return MemInfo(Offset, Addr, CountRes.get());
1018}
1019
1020ErrorOr<BasicSampleInfo> DataReader::parseSampleInfo() {
1021 ErrorOr<Location> Res = parseLocation(EndChar: FieldSeparator);
1022 if (std::error_code EC = Res.getError())
1023 return EC;
1024 Location Address = Res.get();
1025
1026 consumeAllRemainingFS();
1027 ErrorOr<int64_t> BRes = parseNumberField(EndChar: FieldSeparator, /* EndNl = */ true);
1028 if (std::error_code EC = BRes.getError())
1029 return EC;
1030 int64_t Occurrences = BRes.get();
1031
1032 consumeAllRemainingFS();
1033 if (!checkAndConsumeNewLine()) {
1034 reportError(ErrorMsg: "expected end of line");
1035 return make_error_code(E: llvm::errc::io_error);
1036 }
1037
1038 return BasicSampleInfo(std::move(Address), Occurrences);
1039}
1040
1041ErrorOr<bool> DataReader::maybeParseNoLBRFlag() {
1042 if (!ParsingBuf.consume_front(Prefix: "no_lbr"))
1043 return false;
1044 Col += 6;
1045
1046 if (ParsingBuf.size() > 0 && ParsingBuf[0] == ' ')
1047 ParsingBuf = ParsingBuf.drop_front(N: 1);
1048
1049 while (ParsingBuf.size() > 0 && ParsingBuf[0] != '\n') {
1050 ErrorOr<StringRef> EventName = parseString(EndChar: ' ', EndNl: true);
1051 if (!EventName)
1052 return make_error_code(E: llvm::errc::io_error);
1053 EventNames.insert(key: EventName.get());
1054 }
1055
1056 if (!checkAndConsumeNewLine()) {
1057 reportError(ErrorMsg: "malformed no_lbr line");
1058 return make_error_code(E: llvm::errc::io_error);
1059 }
1060 return true;
1061}
1062
1063ErrorOr<bool> DataReader::maybeParseBATFlag() {
1064 if (!ParsingBuf.consume_front(Prefix: "boltedcollection"))
1065 return false;
1066 Col += 16;
1067
1068 if (!checkAndConsumeNewLine()) {
1069 reportError(ErrorMsg: "malformed boltedcollection line");
1070 return make_error_code(E: llvm::errc::io_error);
1071 }
1072 return true;
1073}
1074
1075bool DataReader::hasBranchData() {
1076 if (ParsingBuf.size() == 0)
1077 return false;
1078
1079 if (ParsingBuf[0] == '0' || ParsingBuf[0] == '1' || ParsingBuf[0] == '2')
1080 return true;
1081 return false;
1082}
1083
1084bool DataReader::hasMemData() {
1085 if (ParsingBuf.size() == 0)
1086 return false;
1087
1088 if (ParsingBuf[0] == '3' || ParsingBuf[0] == '4' || ParsingBuf[0] == '5')
1089 return true;
1090 return false;
1091}
1092
1093std::error_code DataReader::parseInNoLBRMode() {
1094 auto GetOrCreateFuncEntry = [&](StringRef Name) {
1095 return NamesToBasicSamples.try_emplace(k: Name, args&: Name).first;
1096 };
1097
1098 auto GetOrCreateFuncMemEntry = [&](StringRef Name) {
1099 return NamesToMemEvents.try_emplace(k: Name, args&: Name).first;
1100 };
1101
1102 while (hasBranchData()) {
1103 ErrorOr<BasicSampleInfo> Res = parseSampleInfo();
1104 if (std::error_code EC = Res.getError())
1105 return EC;
1106
1107 BasicSampleInfo SI = Res.get();
1108
1109 // Ignore samples not involving known locations
1110 if (!SI.Loc.IsSymbol)
1111 continue;
1112
1113 auto I = GetOrCreateFuncEntry(SI.Loc.Name);
1114 I->second.Data.emplace_back(args: std::move(SI));
1115 }
1116
1117 while (hasMemData()) {
1118 ErrorOr<MemInfo> Res = parseMemInfo();
1119 if (std::error_code EC = Res.getError())
1120 return EC;
1121
1122 MemInfo MI = Res.get();
1123
1124 // Ignore memory events not involving known pc.
1125 if (!MI.Offset.IsSymbol)
1126 continue;
1127
1128 auto I = GetOrCreateFuncMemEntry(MI.Offset.Name);
1129 I->second.Data.emplace_back(args: std::move(MI));
1130 }
1131
1132 for (auto &FuncBasicSamples : NamesToBasicSamples)
1133 llvm::stable_sort(Range&: FuncBasicSamples.second.Data);
1134
1135 for (auto &MemEvents : NamesToMemEvents)
1136 llvm::stable_sort(Range&: MemEvents.second.Data);
1137
1138 return std::error_code();
1139}
1140
1141std::error_code DataReader::parse() {
1142 auto GetOrCreateFuncEntry = [&](StringRef Name) {
1143 return NamesToBranches.try_emplace(k: Name, args&: Name).first;
1144 };
1145
1146 auto GetOrCreateFuncMemEntry = [&](StringRef Name) {
1147 return NamesToMemEvents.try_emplace(k: Name, args&: Name).first;
1148 };
1149
1150 Col = 0;
1151 Line = 1;
1152 ErrorOr<bool> FlagOrErr = maybeParseNoLBRFlag();
1153 if (!FlagOrErr)
1154 return FlagOrErr.getError();
1155 NoLBRMode = *FlagOrErr;
1156
1157 ErrorOr<bool> BATFlagOrErr = maybeParseBATFlag();
1158 if (!BATFlagOrErr)
1159 return BATFlagOrErr.getError();
1160 BATMode = *BATFlagOrErr;
1161
1162 if (!hasBranchData() && !hasMemData()) {
1163 Diag << "ERROR: no valid profile data found\n";
1164 return make_error_code(E: llvm::errc::io_error);
1165 }
1166
1167 if (NoLBRMode)
1168 return parseInNoLBRMode();
1169
1170 while (hasBranchData()) {
1171 ErrorOr<BranchInfo> Res = parseBranchInfo();
1172 if (std::error_code EC = Res.getError())
1173 return EC;
1174
1175 BranchInfo BI = Res.get();
1176
1177 // Ignore branches not involving known location.
1178 if (!BI.From.IsSymbol && !BI.To.IsSymbol)
1179 continue;
1180
1181 auto I = GetOrCreateFuncEntry(BI.From.Name);
1182 I->second.Data.emplace_back(args: std::move(BI));
1183
1184 // Add entry data for branches to another function or branches
1185 // to entry points (including recursive calls)
1186 if (BI.To.IsSymbol && (BI.From.Name != BI.To.Name || BI.To.Offset == 0)) {
1187 I = GetOrCreateFuncEntry(BI.To.Name);
1188 I->second.EntryData.emplace_back(args: std::move(BI));
1189 }
1190
1191 // If destination is the function start - update execution count.
1192 // NB: the data is skewed since we cannot tell tail recursion from
1193 // branches to the function start.
1194 if (BI.To.IsSymbol && BI.To.Offset == 0) {
1195 I = GetOrCreateFuncEntry(BI.To.Name);
1196 I->second.ExecutionCount += BI.Branches;
1197 if (!BI.From.IsSymbol)
1198 I->second.ExternEntryCount += BI.Branches;
1199 }
1200 }
1201
1202 while (hasMemData()) {
1203 ErrorOr<MemInfo> Res = parseMemInfo();
1204 if (std::error_code EC = Res.getError())
1205 return EC;
1206
1207 MemInfo MI = Res.get();
1208
1209 // Ignore memory events not involving known pc.
1210 if (!MI.Offset.IsSymbol)
1211 continue;
1212
1213 auto I = GetOrCreateFuncMemEntry(MI.Offset.Name);
1214 I->second.Data.emplace_back(args: std::move(MI));
1215 }
1216
1217 for (auto &FuncBranches : NamesToBranches)
1218 llvm::stable_sort(Range&: FuncBranches.second.Data);
1219
1220 for (auto &MemEvents : NamesToMemEvents)
1221 llvm::stable_sort(Range&: MemEvents.second.Data);
1222
1223 return std::error_code();
1224}
1225
1226void DataReader::buildLTONameMaps() {
1227 for (auto &FuncData : NamesToBranches) {
1228 const StringRef FuncName = FuncData.first;
1229 const std::optional<StringRef> CommonName = getLTOCommonName(Name: FuncName);
1230 if (CommonName)
1231 LTOCommonNameMap[*CommonName].push_back(x: &FuncData.second);
1232 }
1233
1234 for (auto &FuncData : NamesToMemEvents) {
1235 const StringRef FuncName = FuncData.first;
1236 const std::optional<StringRef> CommonName = getLTOCommonName(Name: FuncName);
1237 if (CommonName)
1238 LTOCommonNameMemMap[*CommonName].push_back(x: &FuncData.second);
1239 }
1240}
1241
1242template <typename MapTy>
1243static typename MapTy::mapped_type *
1244fetchMapEntry(MapTy &Map, const std::vector<MCSymbol *> &Symbols) {
1245 // Do a reverse order iteration since the name in profile has a higher chance
1246 // of matching a name at the end of the list.
1247 for (const MCSymbol *Symbol : llvm::reverse(C: Symbols)) {
1248 auto I = Map.find(normalizeName(NameRef: Symbol->getName()));
1249 if (I != Map.end())
1250 return &I->second;
1251 }
1252 return nullptr;
1253}
1254
1255template <typename MapTy>
1256static typename MapTy::mapped_type *
1257fetchMapEntry(MapTy &Map, const std::vector<StringRef> &FuncNames) {
1258 // Do a reverse order iteration since the name in profile has a higher chance
1259 // of matching a name at the end of the list.
1260 for (StringRef Name : llvm::reverse(C: FuncNames)) {
1261 auto I = Map.find(normalizeName(NameRef: Name));
1262 if (I != Map.end())
1263 return &I->second;
1264 }
1265 return nullptr;
1266}
1267
1268template <typename MapTy>
1269static std::vector<typename MapTy::mapped_type *>
1270fetchMapEntriesRegex(MapTy &Map,
1271 const StringMap<std::vector<typename MapTy::mapped_type *>>
1272 &LTOCommonNameMap,
1273 const std::vector<StringRef> &FuncNames) {
1274 std::vector<typename MapTy::mapped_type *> AllData;
1275 // Do a reverse order iteration since the name in profile has a higher chance
1276 // of matching a name at the end of the list.
1277 for (StringRef FuncName : llvm::reverse(C: FuncNames)) {
1278 std::string Name = normalizeName(NameRef: FuncName);
1279 const std::optional<StringRef> LTOCommonName = getLTOCommonName(Name);
1280 if (LTOCommonName) {
1281 auto I = LTOCommonNameMap.find(*LTOCommonName);
1282 if (I != LTOCommonNameMap.end()) {
1283 const std::vector<typename MapTy::mapped_type *> &CommonData =
1284 I->second;
1285 AllData.insert(AllData.end(), CommonData.begin(), CommonData.end());
1286 }
1287 } else {
1288 auto I = Map.find(Name);
1289 if (I != Map.end())
1290 return {&I->second};
1291 }
1292 }
1293 return AllData;
1294}
1295
1296bool DataReader::mayHaveProfileData(const BinaryFunction &Function) {
1297 if (getBranchData(BF: Function) || getMemData(BF: Function))
1298 return true;
1299
1300 if (getFuncBasicSampleData(FuncNames: Function.getNames()) ||
1301 getBranchDataForNames(FuncNames: Function.getNames()) ||
1302 getMemDataForNames(FuncNames: Function.getNames()))
1303 return true;
1304
1305 if (!hasVolatileName(BF: Function))
1306 return false;
1307
1308 const std::vector<FuncBranchData *> AllBranchData =
1309 getBranchDataForNamesRegex(FuncNames: Function.getNames());
1310 if (!AllBranchData.empty())
1311 return true;
1312
1313 const std::vector<FuncMemData *> AllMemData =
1314 getMemDataForNamesRegex(FuncNames: Function.getNames());
1315 if (!AllMemData.empty())
1316 return true;
1317
1318 return false;
1319}
1320
1321FuncBranchData *
1322DataReader::getBranchDataForNames(const std::vector<StringRef> &FuncNames) {
1323 return fetchMapEntry<NamesToBranchesMapTy>(Map&: NamesToBranches, FuncNames);
1324}
1325
1326FuncBranchData *
1327DataReader::getBranchDataForSymbols(const std::vector<MCSymbol *> &Symbols) {
1328 return fetchMapEntry<NamesToBranchesMapTy>(Map&: NamesToBranches, Symbols);
1329}
1330
1331FuncMemData *
1332DataReader::getMemDataForNames(const std::vector<StringRef> &FuncNames) {
1333 return fetchMapEntry<NamesToMemEventsMapTy>(Map&: NamesToMemEvents, FuncNames);
1334}
1335
1336FuncBasicSampleData *
1337DataReader::getFuncBasicSampleData(const std::vector<StringRef> &FuncNames) {
1338 return fetchMapEntry<NamesToBasicSamplesMapTy>(Map&: NamesToBasicSamples,
1339 FuncNames);
1340}
1341
1342std::vector<FuncBranchData *> DataReader::getBranchDataForNamesRegex(
1343 const std::vector<StringRef> &FuncNames) {
1344 return fetchMapEntriesRegex(Map&: NamesToBranches, LTOCommonNameMap, FuncNames);
1345}
1346
1347std::vector<FuncMemData *>
1348DataReader::getMemDataForNamesRegex(const std::vector<StringRef> &FuncNames) {
1349 return fetchMapEntriesRegex(Map&: NamesToMemEvents, LTOCommonNameMap: LTOCommonNameMemMap, FuncNames);
1350}
1351
1352bool DataReader::hasLocalsWithFileName() const {
1353 for (const auto &Func : NamesToBranches) {
1354 const StringRef &FuncName = Func.first;
1355 if (FuncName.count(C: '/') == 2 && FuncName[0] != '/')
1356 return true;
1357 }
1358 return false;
1359}
1360
1361void DataReader::dump() const {
1362 for (const auto &KV : NamesToBranches) {
1363 const StringRef Name = KV.first;
1364 const FuncBranchData &FBD = KV.second;
1365 Diag << Name << " branches:\n";
1366 for (const BranchInfo &BI : FBD.Data)
1367 Diag << BI.From.Name << " " << BI.From.Offset << " " << BI.To.Name << " "
1368 << BI.To.Offset << " " << BI.Mispreds << " " << BI.Branches << "\n";
1369 Diag << Name << " entry points:\n";
1370 for (const BranchInfo &BI : FBD.EntryData)
1371 Diag << BI.From.Name << " " << BI.From.Offset << " " << BI.To.Name << " "
1372 << BI.To.Offset << " " << BI.Mispreds << " " << BI.Branches << "\n";
1373 }
1374
1375 for (auto I = EventNames.begin(), E = EventNames.end(); I != E; ++I) {
1376 StringRef Event = I->getKey();
1377 Diag << "Data was collected with event: " << Event << "\n";
1378 }
1379 for (const auto &KV : NamesToBasicSamples) {
1380 const StringRef Name = KV.first;
1381 const FuncBasicSampleData &FSD = KV.second;
1382 Diag << Name << " samples:\n";
1383 for (const BasicSampleInfo &SI : FSD.Data)
1384 Diag << SI.Loc.Name << " " << SI.Loc.Offset << " " << SI.Hits << "\n";
1385 }
1386
1387 for (const auto &KV : NamesToMemEvents) {
1388 const StringRef Name = KV.first;
1389 const FuncMemData &FMD = KV.second;
1390 Diag << "Memory events for " << Name;
1391 Location LastOffset(0);
1392 for (const MemInfo &MI : FMD.Data) {
1393 if (MI.Offset == LastOffset)
1394 Diag << ", " << MI.Addr << "/" << MI.Count;
1395 else
1396 Diag << "\n" << MI.Offset << ": " << MI.Addr << "/" << MI.Count;
1397 LastOffset = MI.Offset;
1398 }
1399 Diag << "\n";
1400 }
1401}
1402
1403} // namespace bolt
1404} // namespace llvm
1405

source code of bolt/lib/Profile/DataReader.cpp