1 | //===- bolt/Profile/DataReader.cpp - Perf data reader ---------------------===// |
---|---|
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This family of functions reads profile data written by the perf2bolt |
10 | // utility and stores it in memory for llvm-bolt consumption. |
11 | // |
12 | //===----------------------------------------------------------------------===// |
13 | |
14 | #include "bolt/Profile/DataReader.h" |
15 | #include "bolt/Core/BinaryFunction.h" |
16 | #include "bolt/Passes/MCF.h" |
17 | #include "bolt/Utils/Utils.h" |
18 | #include "llvm/Support/CommandLine.h" |
19 | #include "llvm/Support/Debug.h" |
20 | #include "llvm/Support/Errc.h" |
21 | |
22 | #undef DEBUG_TYPE |
23 | #define DEBUG_TYPE "bolt-prof" |
24 | |
25 | using namespace llvm; |
26 | |
27 | namespace opts { |
28 | |
29 | extern cl::OptionCategory BoltCategory; |
30 | extern llvm::cl::opt<unsigned> Verbosity; |
31 | |
32 | static cl::opt<bool> |
33 | DumpData("dump-data", |
34 | cl::desc("dump parsed bolt data for debugging"), |
35 | cl::Hidden, |
36 | cl::cat(BoltCategory)); |
37 | |
38 | } // namespace opts |
39 | |
40 | namespace llvm { |
41 | namespace bolt { |
42 | |
43 | namespace { |
44 | |
45 | /// Return true if the function name can change across compilations. |
46 | bool hasVolatileName(const BinaryFunction &BF) { |
47 | for (const StringRef &Name : BF.getNames()) |
48 | if (getLTOCommonName(Name)) |
49 | return true; |
50 | |
51 | return false; |
52 | } |
53 | |
54 | /// Return standard escaped name of the function possibly renamed by BOLT. |
55 | std::string normalizeName(StringRef NameRef) { |
56 | // Strip "PG." prefix used for globalized locals. |
57 | NameRef = NameRef.starts_with(Prefix: "PG.") ? NameRef.substr(Start: 2) : NameRef; |
58 | return getEscapedName(Name: NameRef); |
59 | } |
60 | |
61 | } // anonymous namespace |
62 | |
63 | raw_ostream &operator<<(raw_ostream &OS, const Location &Loc) { |
64 | if (Loc.IsSymbol) { |
65 | OS << Loc.Name; |
66 | if (Loc.Offset) |
67 | OS << "+"<< Twine::utohexstr(Val: Loc.Offset); |
68 | } else { |
69 | OS << Twine::utohexstr(Val: Loc.Offset); |
70 | } |
71 | return OS; |
72 | } |
73 | |
74 | void FuncBranchData::appendFrom(const FuncBranchData &FBD, uint64_t Offset) { |
75 | Data.insert(position: Data.end(), first: FBD.Data.begin(), last: FBD.Data.end()); |
76 | for (auto I = Data.begin(), E = Data.end(); I != E; ++I) { |
77 | if (I->From.Name == FBD.Name) { |
78 | I->From.Name = this->Name; |
79 | I->From.Offset += Offset; |
80 | } |
81 | if (I->To.Name == FBD.Name) { |
82 | I->To.Name = this->Name; |
83 | I->To.Offset += Offset; |
84 | } |
85 | } |
86 | llvm::stable_sort(Range&: Data); |
87 | ExecutionCount += FBD.ExecutionCount; |
88 | ExternEntryCount += FBD.ExternEntryCount; |
89 | for (auto I = FBD.EntryData.begin(), E = FBD.EntryData.end(); I != E; ++I) { |
90 | assert(I->To.Name == FBD.Name); |
91 | auto NewElmt = EntryData.insert(position: EntryData.end(), x: *I); |
92 | NewElmt->To.Name = this->Name; |
93 | NewElmt->To.Offset += Offset; |
94 | } |
95 | } |
96 | |
97 | uint64_t FuncBranchData::getNumExecutedBranches() const { |
98 | uint64_t ExecutedBranches = 0; |
99 | for (const BranchInfo &BI : Data) { |
100 | int64_t BranchCount = BI.Branches; |
101 | assert(BranchCount >= 0 && "branch execution count should not be negative"); |
102 | ExecutedBranches += BranchCount; |
103 | } |
104 | return ExecutedBranches; |
105 | } |
106 | |
107 | void BasicSampleInfo::mergeWith(const BasicSampleInfo &SI) { Hits += SI.Hits; } |
108 | |
109 | void BasicSampleInfo::print(raw_ostream &OS) const { |
110 | OS << Loc.IsSymbol << " "<< Loc.Name << " "<< Twine::utohexstr(Val: Loc.Offset) |
111 | << " "<< Hits << "\n"; |
112 | } |
113 | |
114 | uint64_t FuncBasicSampleData::getSamples(uint64_t Start, uint64_t End) const { |
115 | assert(llvm::is_sorted(Data)); |
116 | struct Compare { |
117 | bool operator()(const BasicSampleInfo &SI, const uint64_t Val) const { |
118 | return SI.Loc.Offset < Val; |
119 | } |
120 | bool operator()(const uint64_t Val, const BasicSampleInfo &SI) const { |
121 | return Val < SI.Loc.Offset; |
122 | } |
123 | }; |
124 | uint64_t Result = 0; |
125 | for (auto I = llvm::lower_bound(Range: Data, Value&: Start, C: Compare()), |
126 | E = llvm::lower_bound(Range: Data, Value&: End, C: Compare()); |
127 | I != E; ++I) |
128 | Result += I->Hits; |
129 | return Result; |
130 | } |
131 | |
132 | uint64_t FuncBasicSampleData::getSamples() const { |
133 | uint64_t Result = 0; |
134 | for (const BasicSampleInfo &I : Data) |
135 | Result += I.Hits; |
136 | return Result; |
137 | } |
138 | |
139 | void FuncBasicSampleData::bumpCount(uint64_t Offset, uint64_t Count) { |
140 | auto Iter = Index.find(Val: Offset); |
141 | if (Iter == Index.end()) { |
142 | Data.emplace_back(args: Location(true, Name, Offset), args&: Count); |
143 | Index[Offset] = Data.size() - 1; |
144 | return; |
145 | } |
146 | BasicSampleInfo &SI = Data[Iter->second]; |
147 | SI.Hits += Count; |
148 | } |
149 | |
150 | void FuncBranchData::bumpBranchCount(uint64_t OffsetFrom, uint64_t OffsetTo, |
151 | uint64_t Count, uint64_t Mispreds) { |
152 | auto Iter = IntraIndex[OffsetFrom].find(Val: OffsetTo); |
153 | if (Iter == IntraIndex[OffsetFrom].end()) { |
154 | Data.emplace_back(args: Location(true, Name, OffsetFrom), |
155 | args: Location(true, Name, OffsetTo), args&: Mispreds, args&: Count); |
156 | IntraIndex[OffsetFrom][OffsetTo] = Data.size() - 1; |
157 | return; |
158 | } |
159 | BranchInfo &BI = Data[Iter->second]; |
160 | BI.Branches += Count; |
161 | BI.Mispreds += Mispreds; |
162 | } |
163 | |
164 | void FuncBranchData::bumpCallCount(uint64_t OffsetFrom, const Location &To, |
165 | uint64_t Count, uint64_t Mispreds) { |
166 | auto Iter = InterIndex[OffsetFrom].find(Val: To); |
167 | if (Iter == InterIndex[OffsetFrom].end()) { |
168 | Data.emplace_back(args: Location(true, Name, OffsetFrom), args: To, args&: Mispreds, args&: Count); |
169 | InterIndex[OffsetFrom][To] = Data.size() - 1; |
170 | return; |
171 | } |
172 | BranchInfo &BI = Data[Iter->second]; |
173 | BI.Branches += Count; |
174 | BI.Mispreds += Mispreds; |
175 | } |
176 | |
177 | void FuncBranchData::bumpEntryCount(const Location &From, uint64_t OffsetTo, |
178 | uint64_t Count, uint64_t Mispreds) { |
179 | auto Iter = EntryIndex[OffsetTo].find(Val: From); |
180 | if (Iter == EntryIndex[OffsetTo].end()) { |
181 | EntryData.emplace_back(args: From, args: Location(true, Name, OffsetTo), args&: Mispreds, |
182 | args&: Count); |
183 | EntryIndex[OffsetTo][From] = EntryData.size() - 1; |
184 | return; |
185 | } |
186 | BranchInfo &BI = EntryData[Iter->second]; |
187 | BI.Branches += Count; |
188 | BI.Mispreds += Mispreds; |
189 | } |
190 | |
191 | void BranchInfo::mergeWith(const BranchInfo &BI) { |
192 | Branches += BI.Branches; |
193 | Mispreds += BI.Mispreds; |
194 | } |
195 | |
196 | void BranchInfo::print(raw_ostream &OS) const { |
197 | OS << From.IsSymbol << " "<< From.Name << " " |
198 | << Twine::utohexstr(Val: From.Offset) << " "<< To.IsSymbol << " "<< To.Name |
199 | << " "<< Twine::utohexstr(Val: To.Offset) << " "<< Mispreds << " "<< Branches |
200 | << '\n'; |
201 | } |
202 | |
203 | ErrorOr<const BranchInfo &> FuncBranchData::getBranch(uint64_t From, |
204 | uint64_t To) const { |
205 | for (const BranchInfo &I : Data) |
206 | if (I.From.Offset == From && I.To.Offset == To && I.From.Name == I.To.Name) |
207 | return I; |
208 | |
209 | return make_error_code(E: llvm::errc::invalid_argument); |
210 | } |
211 | |
212 | ErrorOr<const BranchInfo &> |
213 | FuncBranchData::getDirectCallBranch(uint64_t From) const { |
214 | // Commented out because it can be expensive. |
215 | // assert(std::is_sorted(Data.begin(), Data.end())); |
216 | struct Compare { |
217 | bool operator()(const BranchInfo &BI, const uint64_t Val) const { |
218 | return BI.From.Offset < Val; |
219 | } |
220 | bool operator()(const uint64_t Val, const BranchInfo &BI) const { |
221 | return Val < BI.From.Offset; |
222 | } |
223 | }; |
224 | auto Range = std::equal_range(first: Data.begin(), last: Data.end(), val: From, comp: Compare()); |
225 | for (const auto &RI : llvm::make_range(p: Range)) |
226 | if (RI.From.Name != RI.To.Name) |
227 | return RI; |
228 | |
229 | return make_error_code(E: llvm::errc::invalid_argument); |
230 | } |
231 | |
232 | void MemInfo::print(raw_ostream &OS) const { |
233 | OS << (Offset.IsSymbol + 3) << " "<< Offset.Name << " " |
234 | << Twine::utohexstr(Val: Offset.Offset) << " "<< (Addr.IsSymbol + 3) << " " |
235 | << Addr.Name << " "<< Twine::utohexstr(Val: Addr.Offset) << " "<< Count |
236 | << "\n"; |
237 | } |
238 | |
239 | void MemInfo::prettyPrint(raw_ostream &OS) const { |
240 | OS << "(PC: "<< Offset << ", M: "<< Addr << ", C: "<< Count << ")"; |
241 | } |
242 | |
243 | void FuncMemData::update(const Location &Offset, const Location &Addr) { |
244 | auto Iter = EventIndex[Offset.Offset].find(Val: Addr); |
245 | if (Iter == EventIndex[Offset.Offset].end()) { |
246 | Data.emplace_back(args: MemInfo(Offset, Addr, 1)); |
247 | EventIndex[Offset.Offset][Addr] = Data.size() - 1; |
248 | return; |
249 | } |
250 | ++Data[Iter->second].Count; |
251 | } |
252 | |
253 | Error DataReader::preprocessProfile(BinaryContext &BC) { |
254 | if (std::error_code EC = parseInput()) |
255 | return errorCodeToError(EC); |
256 | |
257 | if (opts::DumpData) |
258 | dump(); |
259 | |
260 | if (collectedInBoltedBinary()) |
261 | outs() << "BOLT-INFO: profile collection done on a binary already " |
262 | "processed by BOLT\n"; |
263 | |
264 | for (auto &BFI : BC.getBinaryFunctions()) { |
265 | BinaryFunction &Function = BFI.second; |
266 | if (FuncMemData *MemData = getMemDataForNames(FuncNames: Function.getNames())) { |
267 | setMemData(BF: Function, FMD: MemData); |
268 | MemData->Used = true; |
269 | } |
270 | if (FuncBranchData *FuncData = getBranchDataForNames(FuncNames: Function.getNames())) { |
271 | setBranchData(BF: Function, FBD: FuncData); |
272 | Function.ExecutionCount = FuncData->ExecutionCount; |
273 | Function.ExternEntryCount = FuncData->ExternEntryCount; |
274 | FuncData->Used = true; |
275 | } |
276 | } |
277 | |
278 | for (auto &BFI : BC.getBinaryFunctions()) { |
279 | BinaryFunction &Function = BFI.second; |
280 | matchProfileMemData(BF&: Function); |
281 | } |
282 | |
283 | return Error::success(); |
284 | } |
285 | |
286 | Error DataReader::readProfilePreCFG(BinaryContext &BC) { |
287 | for (auto &BFI : BC.getBinaryFunctions()) { |
288 | BinaryFunction &Function = BFI.second; |
289 | FuncMemData *MemoryData = getMemData(BF: Function); |
290 | if (!MemoryData) |
291 | continue; |
292 | |
293 | for (MemInfo &MI : MemoryData->Data) { |
294 | const uint64_t Offset = MI.Offset.Offset; |
295 | auto II = Function.Instructions.find(x: Offset); |
296 | if (II == Function.Instructions.end()) { |
297 | // Ignore bad instruction address. |
298 | continue; |
299 | } |
300 | |
301 | auto &MemAccessProfile = |
302 | BC.MIB->getOrCreateAnnotationAs<MemoryAccessProfile>( |
303 | Inst&: II->second, Name: "MemoryAccessProfile"); |
304 | BinaryData *BD = nullptr; |
305 | if (MI.Addr.IsSymbol) |
306 | BD = BC.getBinaryDataByName(Name: MI.Addr.Name); |
307 | MemAccessProfile.AddressAccessInfo.push_back( |
308 | Elt: {.MemoryObject: BD, .Offset: MI.Addr.Offset, .Count: MI.Count}); |
309 | auto NextII = std::next(x: II); |
310 | if (NextII == Function.Instructions.end()) |
311 | MemAccessProfile.NextInstrOffset = Function.getSize(); |
312 | else |
313 | MemAccessProfile.NextInstrOffset = II->first; |
314 | } |
315 | Function.HasMemoryProfile = true; |
316 | } |
317 | |
318 | return Error::success(); |
319 | } |
320 | |
321 | Error DataReader::readProfile(BinaryContext &BC) { |
322 | for (auto &BFI : BC.getBinaryFunctions()) { |
323 | BinaryFunction &Function = BFI.second; |
324 | readProfile(BF&: Function); |
325 | } |
326 | |
327 | uint64_t NumUnused = 0; |
328 | for (const auto &KV : NamesToBranches) { |
329 | const FuncBranchData &FBD = KV.second; |
330 | if (!FBD.Used) |
331 | ++NumUnused; |
332 | } |
333 | BC.setNumUnusedProfiledObjects(NumUnused); |
334 | |
335 | return Error::success(); |
336 | } |
337 | |
338 | std::error_code DataReader::parseInput() { |
339 | ErrorOr<std::unique_ptr<MemoryBuffer>> MB = |
340 | MemoryBuffer::getFileOrSTDIN(Filename); |
341 | if (std::error_code EC = MB.getError()) { |
342 | Diag << "cannot open "<< Filename << ": "<< EC.message() << "\n"; |
343 | return EC; |
344 | } |
345 | FileBuf = std::move(MB.get()); |
346 | ParsingBuf = FileBuf->getBuffer(); |
347 | if (std::error_code EC = parse()) |
348 | return EC; |
349 | if (!ParsingBuf.empty()) |
350 | Diag << "WARNING: invalid profile data detected at line "<< Line |
351 | << ". Possibly corrupted profile.\n"; |
352 | |
353 | buildLTONameMaps(); |
354 | |
355 | return std::error_code(); |
356 | } |
357 | |
358 | void DataReader::readProfile(BinaryFunction &BF) { |
359 | if (BF.empty()) |
360 | return; |
361 | |
362 | if (!hasLBR()) { |
363 | BF.ProfileFlags = BinaryFunction::PF_BASIC; |
364 | readBasicSampleData(BF); |
365 | return; |
366 | } |
367 | |
368 | BF.ProfileFlags = BinaryFunction::PF_BRANCH; |
369 | |
370 | // Possibly assign/re-assign branch profile data. |
371 | matchProfileData(BF); |
372 | |
373 | FuncBranchData *FBD = getBranchData(BF); |
374 | if (!FBD) |
375 | return; |
376 | |
377 | // Assign basic block counts to function entry points. These only include |
378 | // counts for outside entries. |
379 | // |
380 | // There is a slight skew introduced here as branches originated from RETs |
381 | // may be accounted for in the execution count of an entry block if the last |
382 | // instruction in a predecessor fall-through block is a call. This situation |
383 | // should rarely happen because there are few multiple-entry functions. |
384 | for (const BranchInfo &BI : FBD->EntryData) { |
385 | BinaryBasicBlock *BB = BF.getBasicBlockAtOffset(Offset: BI.To.Offset); |
386 | if (BB && (BB->isEntryPoint() || BB->isLandingPad())) { |
387 | uint64_t Count = BB->getExecutionCount(); |
388 | if (Count == BinaryBasicBlock::COUNT_NO_PROFILE) |
389 | Count = 0; |
390 | BB->setExecutionCount(Count + BI.Branches); |
391 | } |
392 | } |
393 | |
394 | for (const BranchInfo &BI : FBD->Data) { |
395 | if (BI.From.Name != BI.To.Name) |
396 | continue; |
397 | |
398 | if (!recordBranch(BF, From: BI.From.Offset, To: BI.To.Offset, Count: BI.Branches, |
399 | Mispreds: BI.Mispreds)) { |
400 | LLVM_DEBUG(dbgs() << "bad branch : "<< BI.From.Offset << " -> " |
401 | << BI.To.Offset << '\n'); |
402 | } |
403 | } |
404 | |
405 | // Convert branch data into annotations. |
406 | convertBranchData(BF); |
407 | } |
408 | |
409 | void DataReader::matchProfileData(BinaryFunction &BF) { |
410 | // This functionality is available for LBR-mode only |
411 | // TODO: Implement evaluateProfileData() for samples, checking whether |
412 | // sample addresses match instruction addresses in the function |
413 | if (!hasLBR()) |
414 | return; |
415 | |
416 | FuncBranchData *FBD = getBranchData(BF); |
417 | if (FBD) { |
418 | BF.ProfileMatchRatio = evaluateProfileData(BF, BranchData: *FBD); |
419 | BF.RawSampleCount = FBD->getNumExecutedBranches(); |
420 | if (BF.ProfileMatchRatio == 1.0f) { |
421 | if (fetchProfileForOtherEntryPoints(BF)) { |
422 | BF.ProfileMatchRatio = evaluateProfileData(BF, BranchData: *FBD); |
423 | BF.ExecutionCount = FBD->ExecutionCount; |
424 | BF.ExternEntryCount = FBD->ExternEntryCount; |
425 | BF.RawSampleCount = FBD->getNumExecutedBranches(); |
426 | } |
427 | return; |
428 | } |
429 | } |
430 | |
431 | // Check if the function name can fluctuate between several compilations |
432 | // possibly triggered by minor unrelated code changes in the source code |
433 | // of the input binary. |
434 | if (!hasVolatileName(BF)) |
435 | return; |
436 | |
437 | // Check for a profile that matches with 100% confidence. |
438 | const std::vector<FuncBranchData *> AllBranchData = |
439 | getBranchDataForNamesRegex(FuncNames: BF.getNames()); |
440 | for (FuncBranchData *NewBranchData : AllBranchData) { |
441 | // Prevent functions from sharing the same profile. |
442 | if (NewBranchData->Used) |
443 | continue; |
444 | |
445 | if (evaluateProfileData(BF, BranchData: *NewBranchData) != 1.0f) |
446 | continue; |
447 | |
448 | if (FBD) |
449 | FBD->Used = false; |
450 | |
451 | // Update function profile data with the new set. |
452 | setBranchData(BF, FBD: NewBranchData); |
453 | NewBranchData->Used = true; |
454 | BF.ExecutionCount = NewBranchData->ExecutionCount; |
455 | BF.ExternEntryCount = NewBranchData->ExternEntryCount; |
456 | BF.ProfileMatchRatio = 1.0f; |
457 | break; |
458 | } |
459 | } |
460 | |
461 | void DataReader::matchProfileMemData(BinaryFunction &BF) { |
462 | const std::vector<FuncMemData *> AllMemData = |
463 | getMemDataForNamesRegex(FuncNames: BF.getNames()); |
464 | for (FuncMemData *NewMemData : AllMemData) { |
465 | // Prevent functions from sharing the same profile. |
466 | if (NewMemData->Used) |
467 | continue; |
468 | |
469 | if (FuncMemData *MD = getMemData(BF)) |
470 | MD->Used = false; |
471 | |
472 | // Update function profile data with the new set. |
473 | setMemData(BF, FMD: NewMemData); |
474 | NewMemData->Used = true; |
475 | break; |
476 | } |
477 | } |
478 | |
479 | bool DataReader::fetchProfileForOtherEntryPoints(BinaryFunction &BF) { |
480 | BinaryContext &BC = BF.getBinaryContext(); |
481 | |
482 | FuncBranchData *FBD = getBranchData(BF); |
483 | if (!FBD) |
484 | return false; |
485 | |
486 | // Check if we are missing profiling data for secondary entry points |
487 | bool First = true; |
488 | bool Updated = false; |
489 | for (BinaryBasicBlock *BB : BF.BasicBlocks) { |
490 | if (First) { |
491 | First = false; |
492 | continue; |
493 | } |
494 | if (BB->isEntryPoint()) { |
495 | uint64_t EntryAddress = BB->getOffset() + BF.getAddress(); |
496 | // Look for branch data associated with this entry point |
497 | if (BinaryData *BD = BC.getBinaryDataAtAddress(Address: EntryAddress)) { |
498 | if (FuncBranchData *Data = getBranchDataForSymbols(Symbols: BD->getSymbols())) { |
499 | FBD->appendFrom(FBD: *Data, Offset: BB->getOffset()); |
500 | Data->Used = true; |
501 | Updated = true; |
502 | } |
503 | } |
504 | } |
505 | } |
506 | |
507 | return Updated; |
508 | } |
509 | |
510 | float DataReader::evaluateProfileData(BinaryFunction &BF, |
511 | const FuncBranchData &BranchData) const { |
512 | BinaryContext &BC = BF.getBinaryContext(); |
513 | |
514 | // Until we define a minimal profile, we consider an empty branch data to be |
515 | // a valid profile. It could happen to a function without branches when we |
516 | // still have an EntryData for the execution count. |
517 | if (BranchData.Data.empty()) |
518 | return 1.0f; |
519 | |
520 | uint64_t NumMatchedBranches = 0; |
521 | for (const BranchInfo &BI : BranchData.Data) { |
522 | bool IsValid = false; |
523 | if (BI.From.Name == BI.To.Name) { |
524 | // Try to record information with 0 count. |
525 | IsValid = recordBranch(BF, From: BI.From.Offset, To: BI.To.Offset, Count: 0); |
526 | } else if (collectedInBoltedBinary()) { |
527 | // We can't check branch source for collections in bolted binaries because |
528 | // the source of the branch may be mapped to the first instruction in a BB |
529 | // instead of the original branch (which may not exist in the source bin). |
530 | IsValid = true; |
531 | } else { |
532 | // The branch has to originate from this function. |
533 | // Check for calls, tail calls, rets and indirect branches. |
534 | // When matching profiling info, we did not reach the stage |
535 | // when we identify tail calls, so they are still represented |
536 | // by regular branch instructions and we need isBranch() here. |
537 | MCInst *Instr = BF.getInstructionAtOffset(Offset: BI.From.Offset); |
538 | // If it's a prefix - skip it. |
539 | if (Instr && BC.MIB->isPrefix(Inst: *Instr)) |
540 | Instr = BF.getInstructionAtOffset(Offset: BI.From.Offset + 1); |
541 | if (Instr && (BC.MIB->isCall(Inst: *Instr) || BC.MIB->isBranch(Inst: *Instr) || |
542 | BC.MIB->isReturn(Inst: *Instr))) |
543 | IsValid = true; |
544 | } |
545 | |
546 | if (IsValid) { |
547 | ++NumMatchedBranches; |
548 | continue; |
549 | } |
550 | |
551 | LLVM_DEBUG(dbgs() << "\tinvalid branch in "<< BF << " : 0x" |
552 | << Twine::utohexstr(BI.From.Offset) << " -> "; |
553 | if (BI.From.Name == BI.To.Name) dbgs() |
554 | << "0x"<< Twine::utohexstr(BI.To.Offset) << '\n'; |
555 | else dbgs() << "<outbounds>\n";); |
556 | } |
557 | |
558 | const float MatchRatio = (float)NumMatchedBranches / BranchData.Data.size(); |
559 | if (opts::Verbosity >= 2 && NumMatchedBranches < BranchData.Data.size()) |
560 | errs() << "BOLT-WARNING: profile branches match only " |
561 | << format(Fmt: "%.1f%%", Vals: MatchRatio * 100.0f) << " (" |
562 | << NumMatchedBranches << '/' << BranchData.Data.size() |
563 | << ") for function "<< BF << '\n'; |
564 | |
565 | return MatchRatio; |
566 | } |
567 | |
568 | void DataReader::readBasicSampleData(BinaryFunction &BF) { |
569 | FuncBasicSampleData *SampleDataOrErr = getFuncBasicSampleData(FuncNames: BF.getNames()); |
570 | if (!SampleDataOrErr) |
571 | return; |
572 | |
573 | // Basic samples mode territory (without LBR info) |
574 | // First step is to assign BB execution count based on samples from perf |
575 | BF.ProfileMatchRatio = 1.0f; |
576 | BF.removeTagsFromProfile(); |
577 | bool NormalizeByInsnCount = usesEvent(Name: "cycles") || usesEvent(Name: "instructions"); |
578 | bool NormalizeByCalls = usesEvent(Name: "branches"); |
579 | static bool NagUser = true; |
580 | if (NagUser) { |
581 | outs() |
582 | << "BOLT-INFO: operating with basic samples profiling data (no LBR).\n"; |
583 | if (NormalizeByInsnCount) |
584 | outs() << "BOLT-INFO: normalizing samples by instruction count.\n"; |
585 | else if (NormalizeByCalls) |
586 | outs() << "BOLT-INFO: normalizing samples by branches.\n"; |
587 | |
588 | NagUser = false; |
589 | } |
590 | uint64_t LastOffset = BF.getSize(); |
591 | uint64_t TotalEntryCount = 0; |
592 | for (BinaryFunction::BasicBlockOffset &BBOffset : |
593 | llvm::reverse(C&: BF.BasicBlockOffsets)) { |
594 | uint64_t CurOffset = BBOffset.first; |
595 | // Always work with samples multiplied by 1000 to avoid losing them if we |
596 | // later need to normalize numbers |
597 | uint64_t NumSamples = |
598 | SampleDataOrErr->getSamples(Start: CurOffset, End: LastOffset) * 1000; |
599 | if (NormalizeByInsnCount && BBOffset.second->getNumNonPseudos()) { |
600 | NumSamples /= BBOffset.second->getNumNonPseudos(); |
601 | } else if (NormalizeByCalls) { |
602 | uint32_t NumCalls = BBOffset.second->getNumCalls(); |
603 | NumSamples /= NumCalls + 1; |
604 | } |
605 | BBOffset.second->setExecutionCount(NumSamples); |
606 | if (BBOffset.second->isEntryPoint()) |
607 | TotalEntryCount += NumSamples; |
608 | LastOffset = CurOffset; |
609 | } |
610 | |
611 | BF.ExecutionCount = TotalEntryCount; |
612 | } |
613 | |
614 | void DataReader::convertBranchData(BinaryFunction &BF) const { |
615 | BinaryContext &BC = BF.getBinaryContext(); |
616 | |
617 | if (BF.empty()) |
618 | return; |
619 | |
620 | FuncBranchData *FBD = getBranchData(BF); |
621 | if (!FBD) |
622 | return; |
623 | |
624 | // Profile information for calls. |
625 | // |
626 | // There are 3 cases that we annotate differently: |
627 | // 1) Conditional tail calls that could be mispredicted. |
628 | // 2) Indirect calls to multiple destinations with mispredictions. |
629 | // Before we validate CFG we have to handle indirect branches here too. |
630 | // 3) Regular direct calls. The count could be different from containing |
631 | // basic block count. Keep this data in case we find it useful. |
632 | // |
633 | for (BranchInfo &BI : FBD->Data) { |
634 | // Ignore internal branches. |
635 | if (BI.To.IsSymbol && BI.To.Name == BI.From.Name && BI.To.Offset != 0) |
636 | continue; |
637 | |
638 | MCInst *Instr = BF.getInstructionAtOffset(Offset: BI.From.Offset); |
639 | if (!Instr || |
640 | (!BC.MIB->isCall(Inst: *Instr) && !BC.MIB->isIndirectBranch(Inst: *Instr))) |
641 | continue; |
642 | |
643 | auto setOrUpdateAnnotation = [&](StringRef Name, uint64_t Count) { |
644 | if (opts::Verbosity >= 1 && BC.MIB->hasAnnotation(Inst: *Instr, Name)) |
645 | errs() << "BOLT-WARNING: duplicate "<< Name << " info for offset 0x" |
646 | << Twine::utohexstr(Val: BI.From.Offset) << " in function "<< BF |
647 | << '\n'; |
648 | auto &Value = BC.MIB->getOrCreateAnnotationAs<uint64_t>(Inst&: *Instr, Name); |
649 | Value += Count; |
650 | }; |
651 | |
652 | if (BC.MIB->isIndirectCall(Inst: *Instr) || BC.MIB->isIndirectBranch(Inst: *Instr)) { |
653 | IndirectCallSiteProfile &CSP = |
654 | BC.MIB->getOrCreateAnnotationAs<IndirectCallSiteProfile>( |
655 | Inst&: *Instr, Name: "CallProfile"); |
656 | MCSymbol *CalleeSymbol = nullptr; |
657 | if (BI.To.IsSymbol) { |
658 | if (BinaryData *BD = BC.getBinaryDataByName(Name: BI.To.Name)) |
659 | CalleeSymbol = BD->getSymbol(); |
660 | } |
661 | CSP.emplace_back(Args&: CalleeSymbol, Args&: BI.Branches, Args&: BI.Mispreds); |
662 | } else if (BC.MIB->getConditionalTailCall(Inst: *Instr)) { |
663 | setOrUpdateAnnotation("CTCTakenCount", BI.Branches); |
664 | setOrUpdateAnnotation("CTCMispredCount", BI.Mispreds); |
665 | } else { |
666 | setOrUpdateAnnotation("Count", BI.Branches); |
667 | } |
668 | } |
669 | } |
670 | |
671 | bool DataReader::recordBranch(BinaryFunction &BF, uint64_t From, uint64_t To, |
672 | uint64_t Count, uint64_t Mispreds) const { |
673 | BinaryContext &BC = BF.getBinaryContext(); |
674 | |
675 | BinaryBasicBlock *FromBB = BF.getBasicBlockContainingOffset(Offset: From); |
676 | const BinaryBasicBlock *ToBB = BF.getBasicBlockContainingOffset(Offset: To); |
677 | |
678 | if (!FromBB || !ToBB) { |
679 | LLVM_DEBUG(dbgs() << "failed to get block for recorded branch\n"); |
680 | return false; |
681 | } |
682 | |
683 | // Could be bad LBR data; ignore the branch. In the case of data collected |
684 | // in binaries optimized by BOLT, a source BB may be mapped to two output |
685 | // BBs as a result of optimizations. In that case, a branch between these |
686 | // two will be recorded as a branch from A going to A in the source address |
687 | // space. Keep processing. |
688 | if (From == To) |
689 | return true; |
690 | |
691 | // Return from a tail call. |
692 | if (FromBB->succ_size() == 0) |
693 | return true; |
694 | |
695 | // Very rarely we will see ignored branches. Do a linear check. |
696 | for (std::pair<uint32_t, uint32_t> &Branch : BF.IgnoredBranches) |
697 | if (Branch == |
698 | std::make_pair(x: static_cast<uint32_t>(From), y: static_cast<uint32_t>(To))) |
699 | return true; |
700 | |
701 | bool OffsetMatches = !!(To == ToBB->getOffset()); |
702 | if (!OffsetMatches) { |
703 | // Skip the nops to support old .fdata |
704 | uint64_t Offset = ToBB->getOffset(); |
705 | for (const MCInst &Instr : *ToBB) { |
706 | if (!BC.MIB->isNoop(Inst: Instr)) |
707 | break; |
708 | |
709 | if (std::optional<uint32_t> Size = BC.MIB->getSize(Inst: Instr)) |
710 | Offset += *Size; |
711 | } |
712 | |
713 | if (To == Offset) |
714 | OffsetMatches = true; |
715 | } |
716 | |
717 | if (!OffsetMatches) { |
718 | // "To" could be referring to nop instructions in between 2 basic blocks. |
719 | // While building the CFG we make sure these nops are attributed to the |
720 | // previous basic block, thus we check if the destination belongs to the |
721 | // gap past the last instruction. |
722 | const MCInst *LastInstr = ToBB->getLastNonPseudoInstr(); |
723 | if (LastInstr) { |
724 | const uint32_t LastInstrOffset = |
725 | BC.MIB->getOffsetWithDefault(Inst: *LastInstr, Default: 0); |
726 | |
727 | // With old .fdata we are getting FT branches for "jcc,jmp" sequences. |
728 | if (To == LastInstrOffset && BC.MIB->isUnconditionalBranch(Inst: *LastInstr)) |
729 | return true; |
730 | |
731 | if (To <= LastInstrOffset) { |
732 | LLVM_DEBUG(dbgs() << "branch recorded into the middle of the block" |
733 | << " in "<< BF << " : "<< From << " -> "<< To |
734 | << '\n'); |
735 | return false; |
736 | } |
737 | } |
738 | |
739 | // The real destination is the layout successor of the detected ToBB. |
740 | if (ToBB == BF.getLayout().block_back()) |
741 | return false; |
742 | const BinaryBasicBlock *NextBB = |
743 | BF.getLayout().getBlock(Index: ToBB->getIndex() + 1); |
744 | assert((NextBB && NextBB->getOffset() > ToBB->getOffset()) && "bad layout"); |
745 | ToBB = NextBB; |
746 | } |
747 | |
748 | // If there's no corresponding instruction for 'From', we have probably |
749 | // discarded it as a FT from __builtin_unreachable. |
750 | MCInst *FromInstruction = BF.getInstructionAtOffset(Offset: From); |
751 | if (!FromInstruction) { |
752 | // If the data was collected in a bolted binary, the From addresses may be |
753 | // translated to the first instruction of the source BB if BOLT inserted |
754 | // a new branch that did not exist in the source (we can't map it to the |
755 | // source instruction, so we map it to the first instr of source BB). |
756 | // We do not keep offsets for random instructions. So the check above will |
757 | // evaluate to true if the first instr is not a branch (call/jmp/ret/etc) |
758 | if (collectedInBoltedBinary()) { |
759 | if (FromBB->getInputOffset() != From) { |
760 | LLVM_DEBUG(dbgs() << "offset "<< From << " does not match a BB in " |
761 | << BF << '\n'); |
762 | return false; |
763 | } |
764 | FromInstruction = nullptr; |
765 | } else { |
766 | LLVM_DEBUG(dbgs() << "no instruction for offset "<< From << " in "<< BF |
767 | << '\n'); |
768 | return false; |
769 | } |
770 | } |
771 | |
772 | if (!FromBB->getSuccessor(Label: ToBB->getLabel())) { |
773 | // Check if this is a recursive call or a return from a recursive call. |
774 | if (FromInstruction && ToBB->isEntryPoint() && |
775 | (BC.MIB->isCall(Inst: *FromInstruction) || |
776 | BC.MIB->isIndirectBranch(Inst: *FromInstruction))) { |
777 | // Execution count is already accounted for. |
778 | return true; |
779 | } |
780 | // For data collected in a bolted binary, we may have created two output BBs |
781 | // that map to one original block. Branches between these two blocks will |
782 | // appear here as one BB jumping to itself, even though it has no loop |
783 | // edges. Ignore these. |
784 | if (collectedInBoltedBinary() && FromBB == ToBB) |
785 | return true; |
786 | |
787 | // Allow passthrough blocks. |
788 | BinaryBasicBlock *FTSuccessor = FromBB->getConditionalSuccessor(Condition: false); |
789 | if (FTSuccessor && FTSuccessor->succ_size() == 1 && |
790 | FTSuccessor->getSuccessor(Label: ToBB->getLabel())) { |
791 | BinaryBasicBlock::BinaryBranchInfo &FTBI = |
792 | FTSuccessor->getBranchInfo(Succ: *ToBB); |
793 | FTBI.Count += Count; |
794 | if (Count) |
795 | FTBI.MispredictedCount += Mispreds; |
796 | ToBB = FTSuccessor; |
797 | } else { |
798 | LLVM_DEBUG(dbgs() << "invalid branch in "<< BF |
799 | << formatv(": {0:x} -> {1:x}\n", From, To)); |
800 | return false; |
801 | } |
802 | } |
803 | |
804 | BinaryBasicBlock::BinaryBranchInfo &BI = FromBB->getBranchInfo(Succ: *ToBB); |
805 | BI.Count += Count; |
806 | // Only update mispredicted count if it the count was real. |
807 | if (Count) { |
808 | BI.MispredictedCount += Mispreds; |
809 | } |
810 | |
811 | return true; |
812 | } |
813 | |
814 | void DataReader::reportError(StringRef ErrorMsg) { |
815 | Diag << "Error reading BOLT data input file: line "<< Line << ", column " |
816 | << Col << ": "<< ErrorMsg << '\n'; |
817 | } |
818 | |
819 | bool DataReader::expectAndConsumeFS() { |
820 | if (ParsingBuf[0] != FieldSeparator) { |
821 | reportError(ErrorMsg: "expected field separator"); |
822 | return false; |
823 | } |
824 | ParsingBuf = ParsingBuf.drop_front(N: 1); |
825 | Col += 1; |
826 | return true; |
827 | } |
828 | |
829 | void DataReader::consumeAllRemainingFS() { |
830 | while (ParsingBuf[0] == FieldSeparator) { |
831 | ParsingBuf = ParsingBuf.drop_front(N: 1); |
832 | Col += 1; |
833 | } |
834 | } |
835 | |
836 | bool DataReader::checkAndConsumeNewLine() { |
837 | if (ParsingBuf[0] != '\n') |
838 | return false; |
839 | |
840 | ParsingBuf = ParsingBuf.drop_front(N: 1); |
841 | Col = 0; |
842 | Line += 1; |
843 | return true; |
844 | } |
845 | |
846 | ErrorOr<StringRef> DataReader::parseString(char EndChar, bool EndNl) { |
847 | if (EndChar == '\\') { |
848 | reportError(ErrorMsg: "EndChar could not be backslash"); |
849 | return make_error_code(E: llvm::errc::io_error); |
850 | } |
851 | |
852 | std::string EndChars(1, EndChar); |
853 | EndChars.push_back(c: '\\'); |
854 | if (EndNl) |
855 | EndChars.push_back(c: '\n'); |
856 | |
857 | size_t StringEnd = 0; |
858 | do { |
859 | StringEnd = ParsingBuf.find_first_of(Chars: EndChars, From: StringEnd); |
860 | if (StringEnd == StringRef::npos || |
861 | (StringEnd == 0 && ParsingBuf[StringEnd] != '\\')) { |
862 | reportError(ErrorMsg: "malformed field"); |
863 | return make_error_code(E: llvm::errc::io_error); |
864 | } |
865 | |
866 | if (ParsingBuf[StringEnd] != '\\') |
867 | break; |
868 | |
869 | StringEnd += 2; |
870 | } while (true); |
871 | |
872 | StringRef Str = ParsingBuf.substr(Start: 0, N: StringEnd); |
873 | |
874 | // If EndNl was set and nl was found instead of EndChar, do not consume the |
875 | // new line. |
876 | bool EndNlInsteadOfEndChar = ParsingBuf[StringEnd] == '\n' && EndChar != '\n'; |
877 | unsigned End = EndNlInsteadOfEndChar ? StringEnd : StringEnd + 1; |
878 | |
879 | ParsingBuf = ParsingBuf.drop_front(N: End); |
880 | if (EndChar == '\n') { |
881 | Col = 0; |
882 | Line += 1; |
883 | } else { |
884 | Col += End; |
885 | } |
886 | return Str; |
887 | } |
888 | |
889 | ErrorOr<int64_t> DataReader::parseNumberField(char EndChar, bool EndNl) { |
890 | ErrorOr<StringRef> NumStrRes = parseString(EndChar, EndNl); |
891 | if (std::error_code EC = NumStrRes.getError()) |
892 | return EC; |
893 | StringRef NumStr = NumStrRes.get(); |
894 | int64_t Num; |
895 | if (NumStr.getAsInteger(Radix: 10, Result&: Num)) { |
896 | reportError(ErrorMsg: "expected decimal number"); |
897 | Diag << "Found: "<< NumStr << "\n"; |
898 | return make_error_code(E: llvm::errc::io_error); |
899 | } |
900 | return Num; |
901 | } |
902 | |
903 | ErrorOr<uint64_t> DataReader::parseHexField(char EndChar, bool EndNl) { |
904 | ErrorOr<StringRef> NumStrRes = parseString(EndChar, EndNl); |
905 | if (std::error_code EC = NumStrRes.getError()) |
906 | return EC; |
907 | StringRef NumStr = NumStrRes.get(); |
908 | uint64_t Num; |
909 | if (NumStr.getAsInteger(Radix: 16, Result&: Num)) { |
910 | reportError(ErrorMsg: "expected hexidecimal number"); |
911 | Diag << "Found: "<< NumStr << "\n"; |
912 | return make_error_code(E: llvm::errc::io_error); |
913 | } |
914 | return Num; |
915 | } |
916 | |
917 | ErrorOr<Location> DataReader::parseLocation(char EndChar, bool EndNl, |
918 | bool ExpectMemLoc) { |
919 | // Read whether the location of the branch should be DSO or a symbol |
920 | // 0 means it is a DSO. 1 means it is a global symbol. 2 means it is a local |
921 | // symbol. |
922 | // The symbol flag is also used to tag memory load events by adding 3 to the |
923 | // base values, i.e. 3 not a symbol, 4 global symbol and 5 local symbol. |
924 | if (!ExpectMemLoc && ParsingBuf[0] != '0' && ParsingBuf[0] != '1' && |
925 | ParsingBuf[0] != '2') { |
926 | reportError(ErrorMsg: "expected 0, 1 or 2"); |
927 | return make_error_code(E: llvm::errc::io_error); |
928 | } |
929 | |
930 | if (ExpectMemLoc && ParsingBuf[0] != '3' && ParsingBuf[0] != '4' && |
931 | ParsingBuf[0] != '5') { |
932 | reportError(ErrorMsg: "expected 3, 4 or 5"); |
933 | return make_error_code(E: llvm::errc::io_error); |
934 | } |
935 | |
936 | bool IsSymbol = |
937 | (!ExpectMemLoc && (ParsingBuf[0] == '1' || ParsingBuf[0] == '2')) || |
938 | (ExpectMemLoc && (ParsingBuf[0] == '4' || ParsingBuf[0] == '5')); |
939 | ParsingBuf = ParsingBuf.drop_front(N: 1); |
940 | Col += 1; |
941 | |
942 | if (!expectAndConsumeFS()) |
943 | return make_error_code(E: llvm::errc::io_error); |
944 | consumeAllRemainingFS(); |
945 | |
946 | // Read the string containing the symbol or the DSO name |
947 | ErrorOr<StringRef> NameRes = parseString(EndChar: FieldSeparator); |
948 | if (std::error_code EC = NameRes.getError()) |
949 | return EC; |
950 | StringRef Name = NameRes.get(); |
951 | consumeAllRemainingFS(); |
952 | |
953 | // Read the offset |
954 | ErrorOr<uint64_t> Offset = parseHexField(EndChar, EndNl); |
955 | if (std::error_code EC = Offset.getError()) |
956 | return EC; |
957 | |
958 | return Location(IsSymbol, Name, Offset.get()); |
959 | } |
960 | |
961 | ErrorOr<BranchInfo> DataReader::parseBranchInfo() { |
962 | ErrorOr<Location> Res = parseLocation(EndChar: FieldSeparator); |
963 | if (std::error_code EC = Res.getError()) |
964 | return EC; |
965 | Location From = Res.get(); |
966 | |
967 | consumeAllRemainingFS(); |
968 | Res = parseLocation(EndChar: FieldSeparator); |
969 | if (std::error_code EC = Res.getError()) |
970 | return EC; |
971 | Location To = Res.get(); |
972 | |
973 | consumeAllRemainingFS(); |
974 | ErrorOr<int64_t> MRes = parseNumberField(EndChar: FieldSeparator); |
975 | if (std::error_code EC = MRes.getError()) |
976 | return EC; |
977 | int64_t NumMispreds = MRes.get(); |
978 | |
979 | consumeAllRemainingFS(); |
980 | ErrorOr<int64_t> BRes = parseNumberField(EndChar: FieldSeparator, /* EndNl = */ true); |
981 | if (std::error_code EC = BRes.getError()) |
982 | return EC; |
983 | int64_t NumBranches = BRes.get(); |
984 | |
985 | consumeAllRemainingFS(); |
986 | if (!checkAndConsumeNewLine()) { |
987 | reportError(ErrorMsg: "expected end of line"); |
988 | return make_error_code(E: llvm::errc::io_error); |
989 | } |
990 | |
991 | return BranchInfo(std::move(From), std::move(To), NumMispreds, NumBranches); |
992 | } |
993 | |
994 | ErrorOr<MemInfo> DataReader::parseMemInfo() { |
995 | ErrorOr<Location> Res = parseMemLocation(EndChar: FieldSeparator); |
996 | if (std::error_code EC = Res.getError()) |
997 | return EC; |
998 | Location Offset = Res.get(); |
999 | |
1000 | consumeAllRemainingFS(); |
1001 | Res = parseMemLocation(EndChar: FieldSeparator); |
1002 | if (std::error_code EC = Res.getError()) |
1003 | return EC; |
1004 | Location Addr = Res.get(); |
1005 | |
1006 | consumeAllRemainingFS(); |
1007 | ErrorOr<int64_t> CountRes = parseNumberField(EndChar: FieldSeparator, EndNl: true); |
1008 | if (std::error_code EC = CountRes.getError()) |
1009 | return EC; |
1010 | |
1011 | consumeAllRemainingFS(); |
1012 | if (!checkAndConsumeNewLine()) { |
1013 | reportError(ErrorMsg: "expected end of line"); |
1014 | return make_error_code(E: llvm::errc::io_error); |
1015 | } |
1016 | |
1017 | return MemInfo(Offset, Addr, CountRes.get()); |
1018 | } |
1019 | |
1020 | ErrorOr<BasicSampleInfo> DataReader::parseSampleInfo() { |
1021 | ErrorOr<Location> Res = parseLocation(EndChar: FieldSeparator); |
1022 | if (std::error_code EC = Res.getError()) |
1023 | return EC; |
1024 | Location Address = Res.get(); |
1025 | |
1026 | consumeAllRemainingFS(); |
1027 | ErrorOr<int64_t> BRes = parseNumberField(EndChar: FieldSeparator, /* EndNl = */ true); |
1028 | if (std::error_code EC = BRes.getError()) |
1029 | return EC; |
1030 | int64_t Occurrences = BRes.get(); |
1031 | |
1032 | consumeAllRemainingFS(); |
1033 | if (!checkAndConsumeNewLine()) { |
1034 | reportError(ErrorMsg: "expected end of line"); |
1035 | return make_error_code(E: llvm::errc::io_error); |
1036 | } |
1037 | |
1038 | return BasicSampleInfo(std::move(Address), Occurrences); |
1039 | } |
1040 | |
1041 | ErrorOr<bool> DataReader::maybeParseNoLBRFlag() { |
1042 | if (!ParsingBuf.consume_front(Prefix: "no_lbr")) |
1043 | return false; |
1044 | Col += 6; |
1045 | |
1046 | if (ParsingBuf.size() > 0 && ParsingBuf[0] == ' ') |
1047 | ParsingBuf = ParsingBuf.drop_front(N: 1); |
1048 | |
1049 | while (ParsingBuf.size() > 0 && ParsingBuf[0] != '\n') { |
1050 | ErrorOr<StringRef> EventName = parseString(EndChar: ' ', EndNl: true); |
1051 | if (!EventName) |
1052 | return make_error_code(E: llvm::errc::io_error); |
1053 | EventNames.insert(key: EventName.get()); |
1054 | } |
1055 | |
1056 | if (!checkAndConsumeNewLine()) { |
1057 | reportError(ErrorMsg: "malformed no_lbr line"); |
1058 | return make_error_code(E: llvm::errc::io_error); |
1059 | } |
1060 | return true; |
1061 | } |
1062 | |
1063 | ErrorOr<bool> DataReader::maybeParseBATFlag() { |
1064 | if (!ParsingBuf.consume_front(Prefix: "boltedcollection")) |
1065 | return false; |
1066 | Col += 16; |
1067 | |
1068 | if (!checkAndConsumeNewLine()) { |
1069 | reportError(ErrorMsg: "malformed boltedcollection line"); |
1070 | return make_error_code(E: llvm::errc::io_error); |
1071 | } |
1072 | return true; |
1073 | } |
1074 | |
1075 | bool DataReader::hasBranchData() { |
1076 | if (ParsingBuf.size() == 0) |
1077 | return false; |
1078 | |
1079 | if (ParsingBuf[0] == '0' || ParsingBuf[0] == '1' || ParsingBuf[0] == '2') |
1080 | return true; |
1081 | return false; |
1082 | } |
1083 | |
1084 | bool DataReader::hasMemData() { |
1085 | if (ParsingBuf.size() == 0) |
1086 | return false; |
1087 | |
1088 | if (ParsingBuf[0] == '3' || ParsingBuf[0] == '4' || ParsingBuf[0] == '5') |
1089 | return true; |
1090 | return false; |
1091 | } |
1092 | |
1093 | std::error_code DataReader::parseInNoLBRMode() { |
1094 | auto GetOrCreateFuncEntry = [&](StringRef Name) { |
1095 | return NamesToBasicSamples.try_emplace(k: Name, args&: Name).first; |
1096 | }; |
1097 | |
1098 | auto GetOrCreateFuncMemEntry = [&](StringRef Name) { |
1099 | return NamesToMemEvents.try_emplace(k: Name, args&: Name).first; |
1100 | }; |
1101 | |
1102 | while (hasBranchData()) { |
1103 | ErrorOr<BasicSampleInfo> Res = parseSampleInfo(); |
1104 | if (std::error_code EC = Res.getError()) |
1105 | return EC; |
1106 | |
1107 | BasicSampleInfo SI = Res.get(); |
1108 | |
1109 | // Ignore samples not involving known locations |
1110 | if (!SI.Loc.IsSymbol) |
1111 | continue; |
1112 | |
1113 | auto I = GetOrCreateFuncEntry(SI.Loc.Name); |
1114 | I->second.Data.emplace_back(args: std::move(SI)); |
1115 | } |
1116 | |
1117 | while (hasMemData()) { |
1118 | ErrorOr<MemInfo> Res = parseMemInfo(); |
1119 | if (std::error_code EC = Res.getError()) |
1120 | return EC; |
1121 | |
1122 | MemInfo MI = Res.get(); |
1123 | |
1124 | // Ignore memory events not involving known pc. |
1125 | if (!MI.Offset.IsSymbol) |
1126 | continue; |
1127 | |
1128 | auto I = GetOrCreateFuncMemEntry(MI.Offset.Name); |
1129 | I->second.Data.emplace_back(args: std::move(MI)); |
1130 | } |
1131 | |
1132 | for (auto &FuncBasicSamples : NamesToBasicSamples) |
1133 | llvm::stable_sort(Range&: FuncBasicSamples.second.Data); |
1134 | |
1135 | for (auto &MemEvents : NamesToMemEvents) |
1136 | llvm::stable_sort(Range&: MemEvents.second.Data); |
1137 | |
1138 | return std::error_code(); |
1139 | } |
1140 | |
1141 | std::error_code DataReader::parse() { |
1142 | auto GetOrCreateFuncEntry = [&](StringRef Name) { |
1143 | return NamesToBranches.try_emplace(k: Name, args&: Name).first; |
1144 | }; |
1145 | |
1146 | auto GetOrCreateFuncMemEntry = [&](StringRef Name) { |
1147 | return NamesToMemEvents.try_emplace(k: Name, args&: Name).first; |
1148 | }; |
1149 | |
1150 | Col = 0; |
1151 | Line = 1; |
1152 | ErrorOr<bool> FlagOrErr = maybeParseNoLBRFlag(); |
1153 | if (!FlagOrErr) |
1154 | return FlagOrErr.getError(); |
1155 | NoLBRMode = *FlagOrErr; |
1156 | |
1157 | ErrorOr<bool> BATFlagOrErr = maybeParseBATFlag(); |
1158 | if (!BATFlagOrErr) |
1159 | return BATFlagOrErr.getError(); |
1160 | BATMode = *BATFlagOrErr; |
1161 | |
1162 | if (!hasBranchData() && !hasMemData()) { |
1163 | Diag << "ERROR: no valid profile data found\n"; |
1164 | return make_error_code(E: llvm::errc::io_error); |
1165 | } |
1166 | |
1167 | if (NoLBRMode) |
1168 | return parseInNoLBRMode(); |
1169 | |
1170 | while (hasBranchData()) { |
1171 | ErrorOr<BranchInfo> Res = parseBranchInfo(); |
1172 | if (std::error_code EC = Res.getError()) |
1173 | return EC; |
1174 | |
1175 | BranchInfo BI = Res.get(); |
1176 | |
1177 | // Ignore branches not involving known location. |
1178 | if (!BI.From.IsSymbol && !BI.To.IsSymbol) |
1179 | continue; |
1180 | |
1181 | auto I = GetOrCreateFuncEntry(BI.From.Name); |
1182 | I->second.Data.emplace_back(args: std::move(BI)); |
1183 | |
1184 | // Add entry data for branches to another function or branches |
1185 | // to entry points (including recursive calls) |
1186 | if (BI.To.IsSymbol && (BI.From.Name != BI.To.Name || BI.To.Offset == 0)) { |
1187 | I = GetOrCreateFuncEntry(BI.To.Name); |
1188 | I->second.EntryData.emplace_back(args: std::move(BI)); |
1189 | } |
1190 | |
1191 | // If destination is the function start - update execution count. |
1192 | // NB: the data is skewed since we cannot tell tail recursion from |
1193 | // branches to the function start. |
1194 | if (BI.To.IsSymbol && BI.To.Offset == 0) { |
1195 | I = GetOrCreateFuncEntry(BI.To.Name); |
1196 | I->second.ExecutionCount += BI.Branches; |
1197 | if (!BI.From.IsSymbol) |
1198 | I->second.ExternEntryCount += BI.Branches; |
1199 | } |
1200 | } |
1201 | |
1202 | while (hasMemData()) { |
1203 | ErrorOr<MemInfo> Res = parseMemInfo(); |
1204 | if (std::error_code EC = Res.getError()) |
1205 | return EC; |
1206 | |
1207 | MemInfo MI = Res.get(); |
1208 | |
1209 | // Ignore memory events not involving known pc. |
1210 | if (!MI.Offset.IsSymbol) |
1211 | continue; |
1212 | |
1213 | auto I = GetOrCreateFuncMemEntry(MI.Offset.Name); |
1214 | I->second.Data.emplace_back(args: std::move(MI)); |
1215 | } |
1216 | |
1217 | for (auto &FuncBranches : NamesToBranches) |
1218 | llvm::stable_sort(Range&: FuncBranches.second.Data); |
1219 | |
1220 | for (auto &MemEvents : NamesToMemEvents) |
1221 | llvm::stable_sort(Range&: MemEvents.second.Data); |
1222 | |
1223 | return std::error_code(); |
1224 | } |
1225 | |
1226 | void DataReader::buildLTONameMaps() { |
1227 | for (auto &FuncData : NamesToBranches) { |
1228 | const StringRef FuncName = FuncData.first; |
1229 | const std::optional<StringRef> CommonName = getLTOCommonName(Name: FuncName); |
1230 | if (CommonName) |
1231 | LTOCommonNameMap[*CommonName].push_back(x: &FuncData.second); |
1232 | } |
1233 | |
1234 | for (auto &FuncData : NamesToMemEvents) { |
1235 | const StringRef FuncName = FuncData.first; |
1236 | const std::optional<StringRef> CommonName = getLTOCommonName(Name: FuncName); |
1237 | if (CommonName) |
1238 | LTOCommonNameMemMap[*CommonName].push_back(x: &FuncData.second); |
1239 | } |
1240 | } |
1241 | |
1242 | template <typename MapTy> |
1243 | static typename MapTy::mapped_type * |
1244 | fetchMapEntry(MapTy &Map, const std::vector<MCSymbol *> &Symbols) { |
1245 | // Do a reverse order iteration since the name in profile has a higher chance |
1246 | // of matching a name at the end of the list. |
1247 | for (const MCSymbol *Symbol : llvm::reverse(C: Symbols)) { |
1248 | auto I = Map.find(normalizeName(NameRef: Symbol->getName())); |
1249 | if (I != Map.end()) |
1250 | return &I->second; |
1251 | } |
1252 | return nullptr; |
1253 | } |
1254 | |
1255 | template <typename MapTy> |
1256 | static typename MapTy::mapped_type * |
1257 | fetchMapEntry(MapTy &Map, const std::vector<StringRef> &FuncNames) { |
1258 | // Do a reverse order iteration since the name in profile has a higher chance |
1259 | // of matching a name at the end of the list. |
1260 | for (StringRef Name : llvm::reverse(C: FuncNames)) { |
1261 | auto I = Map.find(normalizeName(NameRef: Name)); |
1262 | if (I != Map.end()) |
1263 | return &I->second; |
1264 | } |
1265 | return nullptr; |
1266 | } |
1267 | |
1268 | template <typename MapTy> |
1269 | static std::vector<typename MapTy::mapped_type *> |
1270 | fetchMapEntriesRegex(MapTy &Map, |
1271 | const StringMap<std::vector<typename MapTy::mapped_type *>> |
1272 | <OCommonNameMap, |
1273 | const std::vector<StringRef> &FuncNames) { |
1274 | std::vector<typename MapTy::mapped_type *> AllData; |
1275 | // Do a reverse order iteration since the name in profile has a higher chance |
1276 | // of matching a name at the end of the list. |
1277 | for (StringRef FuncName : llvm::reverse(C: FuncNames)) { |
1278 | std::string Name = normalizeName(NameRef: FuncName); |
1279 | const std::optional<StringRef> LTOCommonName = getLTOCommonName(Name); |
1280 | if (LTOCommonName) { |
1281 | auto I = LTOCommonNameMap.find(*LTOCommonName); |
1282 | if (I != LTOCommonNameMap.end()) { |
1283 | const std::vector<typename MapTy::mapped_type *> &CommonData = |
1284 | I->second; |
1285 | AllData.insert(AllData.end(), CommonData.begin(), CommonData.end()); |
1286 | } |
1287 | } else { |
1288 | auto I = Map.find(Name); |
1289 | if (I != Map.end()) |
1290 | return {&I->second}; |
1291 | } |
1292 | } |
1293 | return AllData; |
1294 | } |
1295 | |
1296 | bool DataReader::mayHaveProfileData(const BinaryFunction &Function) { |
1297 | if (getBranchData(BF: Function) || getMemData(BF: Function)) |
1298 | return true; |
1299 | |
1300 | if (getFuncBasicSampleData(FuncNames: Function.getNames()) || |
1301 | getBranchDataForNames(FuncNames: Function.getNames()) || |
1302 | getMemDataForNames(FuncNames: Function.getNames())) |
1303 | return true; |
1304 | |
1305 | if (!hasVolatileName(BF: Function)) |
1306 | return false; |
1307 | |
1308 | const std::vector<FuncBranchData *> AllBranchData = |
1309 | getBranchDataForNamesRegex(FuncNames: Function.getNames()); |
1310 | if (!AllBranchData.empty()) |
1311 | return true; |
1312 | |
1313 | const std::vector<FuncMemData *> AllMemData = |
1314 | getMemDataForNamesRegex(FuncNames: Function.getNames()); |
1315 | if (!AllMemData.empty()) |
1316 | return true; |
1317 | |
1318 | return false; |
1319 | } |
1320 | |
1321 | FuncBranchData * |
1322 | DataReader::getBranchDataForNames(const std::vector<StringRef> &FuncNames) { |
1323 | return fetchMapEntry<NamesToBranchesMapTy>(Map&: NamesToBranches, FuncNames); |
1324 | } |
1325 | |
1326 | FuncBranchData * |
1327 | DataReader::getBranchDataForSymbols(const std::vector<MCSymbol *> &Symbols) { |
1328 | return fetchMapEntry<NamesToBranchesMapTy>(Map&: NamesToBranches, Symbols); |
1329 | } |
1330 | |
1331 | FuncMemData * |
1332 | DataReader::getMemDataForNames(const std::vector<StringRef> &FuncNames) { |
1333 | return fetchMapEntry<NamesToMemEventsMapTy>(Map&: NamesToMemEvents, FuncNames); |
1334 | } |
1335 | |
1336 | FuncBasicSampleData * |
1337 | DataReader::getFuncBasicSampleData(const std::vector<StringRef> &FuncNames) { |
1338 | return fetchMapEntry<NamesToBasicSamplesMapTy>(Map&: NamesToBasicSamples, |
1339 | FuncNames); |
1340 | } |
1341 | |
1342 | std::vector<FuncBranchData *> DataReader::getBranchDataForNamesRegex( |
1343 | const std::vector<StringRef> &FuncNames) { |
1344 | return fetchMapEntriesRegex(Map&: NamesToBranches, LTOCommonNameMap, FuncNames); |
1345 | } |
1346 | |
1347 | std::vector<FuncMemData *> |
1348 | DataReader::getMemDataForNamesRegex(const std::vector<StringRef> &FuncNames) { |
1349 | return fetchMapEntriesRegex(Map&: NamesToMemEvents, LTOCommonNameMap: LTOCommonNameMemMap, FuncNames); |
1350 | } |
1351 | |
1352 | bool DataReader::hasLocalsWithFileName() const { |
1353 | for (const auto &Func : NamesToBranches) { |
1354 | const StringRef &FuncName = Func.first; |
1355 | if (FuncName.count(C: '/') == 2 && FuncName[0] != '/') |
1356 | return true; |
1357 | } |
1358 | return false; |
1359 | } |
1360 | |
1361 | void DataReader::dump() const { |
1362 | for (const auto &KV : NamesToBranches) { |
1363 | const StringRef Name = KV.first; |
1364 | const FuncBranchData &FBD = KV.second; |
1365 | Diag << Name << " branches:\n"; |
1366 | for (const BranchInfo &BI : FBD.Data) |
1367 | Diag << BI.From.Name << " "<< BI.From.Offset << " "<< BI.To.Name << " " |
1368 | << BI.To.Offset << " "<< BI.Mispreds << " "<< BI.Branches << "\n"; |
1369 | Diag << Name << " entry points:\n"; |
1370 | for (const BranchInfo &BI : FBD.EntryData) |
1371 | Diag << BI.From.Name << " "<< BI.From.Offset << " "<< BI.To.Name << " " |
1372 | << BI.To.Offset << " "<< BI.Mispreds << " "<< BI.Branches << "\n"; |
1373 | } |
1374 | |
1375 | for (auto I = EventNames.begin(), E = EventNames.end(); I != E; ++I) { |
1376 | StringRef Event = I->getKey(); |
1377 | Diag << "Data was collected with event: "<< Event << "\n"; |
1378 | } |
1379 | for (const auto &KV : NamesToBasicSamples) { |
1380 | const StringRef Name = KV.first; |
1381 | const FuncBasicSampleData &FSD = KV.second; |
1382 | Diag << Name << " samples:\n"; |
1383 | for (const BasicSampleInfo &SI : FSD.Data) |
1384 | Diag << SI.Loc.Name << " "<< SI.Loc.Offset << " "<< SI.Hits << "\n"; |
1385 | } |
1386 | |
1387 | for (const auto &KV : NamesToMemEvents) { |
1388 | const StringRef Name = KV.first; |
1389 | const FuncMemData &FMD = KV.second; |
1390 | Diag << "Memory events for "<< Name; |
1391 | Location LastOffset(0); |
1392 | for (const MemInfo &MI : FMD.Data) { |
1393 | if (MI.Offset == LastOffset) |
1394 | Diag << ", "<< MI.Addr << "/"<< MI.Count; |
1395 | else |
1396 | Diag << "\n"<< MI.Offset << ": "<< MI.Addr << "/"<< MI.Count; |
1397 | LastOffset = MI.Offset; |
1398 | } |
1399 | Diag << "\n"; |
1400 | } |
1401 | } |
1402 | |
1403 | } // namespace bolt |
1404 | } // namespace llvm |
1405 |
Definitions
- DumpData
- hasVolatileName
- normalizeName
- operator<<
- appendFrom
- getNumExecutedBranches
- mergeWith
- getSamples
- getSamples
- bumpCount
- bumpBranchCount
- bumpCallCount
- bumpEntryCount
- mergeWith
- getBranch
- getDirectCallBranch
- prettyPrint
- update
- preprocessProfile
- readProfilePreCFG
- readProfile
- parseInput
- readProfile
- matchProfileData
- matchProfileMemData
- fetchProfileForOtherEntryPoints
- evaluateProfileData
- readBasicSampleData
- convertBranchData
- recordBranch
- reportError
- expectAndConsumeFS
- consumeAllRemainingFS
- checkAndConsumeNewLine
- parseString
- parseNumberField
- parseHexField
- parseLocation
- parseBranchInfo
- parseMemInfo
- parseSampleInfo
- maybeParseNoLBRFlag
- maybeParseBATFlag
- hasBranchData
- hasMemData
- parseInNoLBRMode
- parse
- buildLTONameMaps
- fetchMapEntry
- fetchMapEntry
- fetchMapEntriesRegex
- mayHaveProfileData
- getBranchDataForNames
- getBranchDataForSymbols
- getMemDataForNames
- getFuncBasicSampleData
- getBranchDataForNamesRegex
- getMemDataForNamesRegex
- hasLocalsWithFileName
Learn to use CMake with our Intro Training
Find out more