1//===- bolt/tools/merge-fdata/merge-fdata.cpp -----------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Tool for merging profile in fdata format:
10//
11// $ merge-fdata 1.fdata 2.fdata 3.fdata > merged.fdata
12//
13//===----------------------------------------------------------------------===//
14
15#include "bolt/Profile/ProfileYAMLMapping.h"
16#include "llvm/ADT/STLExtras.h"
17#include "llvm/ADT/StringMap.h"
18#include "llvm/Support/CommandLine.h"
19#include "llvm/Support/FileSystem.h"
20#include "llvm/Support/ManagedStatic.h"
21#include "llvm/Support/PrettyStackTrace.h"
22#include "llvm/Support/Signals.h"
23#include "llvm/Support/ThreadPool.h"
24#include <algorithm>
25#include <mutex>
26#include <unordered_map>
27
28using namespace llvm;
29using namespace llvm::yaml::bolt;
30
31namespace opts {
32
33cl::OptionCategory MergeFdataCategory("merge-fdata options");
34
35enum SortType : char {
36 ST_NONE,
37 ST_EXEC_COUNT, /// Sort based on function execution count.
38 ST_TOTAL_BRANCHES, /// Sort based on all branches in the function.
39};
40
41static cl::list<std::string>
42InputDataFilenames(
43 cl::Positional,
44 cl::CommaSeparated,
45 cl::desc("<fdata1> [<fdata2>]..."),
46 cl::OneOrMore,
47 cl::cat(MergeFdataCategory));
48
49static cl::opt<SortType>
50PrintFunctionList("print",
51 cl::desc("print the list of objects with count to stderr"),
52 cl::init(Val: ST_NONE),
53 cl::values(clEnumValN(ST_NONE,
54 "none",
55 "do not print objects/functions"),
56 clEnumValN(ST_EXEC_COUNT,
57 "exec",
58 "print functions sorted by execution count"),
59 clEnumValN(ST_TOTAL_BRANCHES,
60 "branches",
61 "print functions sorted by total branch count")),
62 cl::cat(MergeFdataCategory));
63
64static cl::opt<bool>
65SuppressMergedDataOutput("q",
66 cl::desc("do not print merged data to stdout"),
67 cl::init(Val: false),
68 cl::Optional,
69 cl::cat(MergeFdataCategory));
70
71static cl::opt<std::string>
72OutputFilePath("o",
73 cl::value_desc("file"),
74 cl::desc("Write output to <file>"),
75 cl::cat(MergeFdataCategory));
76
77} // namespace opts
78
79namespace {
80
81static StringRef ToolName;
82
83static void report_error(StringRef Message, std::error_code EC) {
84 assert(EC);
85 errs() << ToolName << ": '" << Message << "': " << EC.message() << ".\n";
86 exit(status: 1);
87}
88
89static void report_error(Twine Message, StringRef CustomError) {
90 errs() << ToolName << ": '" << Message << "': " << CustomError << ".\n";
91 exit(status: 1);
92}
93
94static raw_fd_ostream &output() {
95 if (opts::OutputFilePath.empty() || opts::OutputFilePath == "-")
96 return outs();
97 else {
98 std::error_code EC;
99 static raw_fd_ostream Output(opts::OutputFilePath, EC);
100 if (EC)
101 report_error(Message: opts::OutputFilePath, EC);
102 return Output;
103 }
104}
105
106void mergeProfileHeaders(BinaryProfileHeader &MergedHeader,
107 const BinaryProfileHeader &Header) {
108 if (MergedHeader.FileName.empty())
109 MergedHeader.FileName = Header.FileName;
110
111 if (!MergedHeader.FileName.empty() &&
112 MergedHeader.FileName != Header.FileName)
113 errs() << "WARNING: merging profile from a binary for " << Header.FileName
114 << " into a profile for binary " << MergedHeader.FileName << '\n';
115
116 if (MergedHeader.Id.empty())
117 MergedHeader.Id = Header.Id;
118
119 if (!MergedHeader.Id.empty() && (MergedHeader.Id != Header.Id))
120 errs() << "WARNING: build-ids in merged profiles do not match\n";
121
122 // Cannot merge samples profile with LBR profile.
123 if (!MergedHeader.Flags)
124 MergedHeader.Flags = Header.Flags;
125
126 constexpr auto Mask = llvm::bolt::BinaryFunction::PF_LBR |
127 llvm::bolt::BinaryFunction::PF_SAMPLE;
128 if ((MergedHeader.Flags & Mask) != (Header.Flags & Mask)) {
129 errs() << "ERROR: cannot merge LBR profile with non-LBR profile\n";
130 exit(status: 1);
131 }
132 MergedHeader.Flags = MergedHeader.Flags | Header.Flags;
133
134 if (!Header.Origin.empty()) {
135 if (MergedHeader.Origin.empty())
136 MergedHeader.Origin = Header.Origin;
137 else if (MergedHeader.Origin != Header.Origin)
138 MergedHeader.Origin += "; " + Header.Origin;
139 }
140
141 if (MergedHeader.EventNames.empty())
142 MergedHeader.EventNames = Header.EventNames;
143
144 if (MergedHeader.EventNames != Header.EventNames) {
145 errs() << "WARNING: merging profiles with different sampling events\n";
146 MergedHeader.EventNames += "," + Header.EventNames;
147 }
148}
149
150void mergeBasicBlockProfile(BinaryBasicBlockProfile &MergedBB,
151 BinaryBasicBlockProfile &&BB,
152 const BinaryFunctionProfile &BF) {
153 // Verify that the blocks match.
154 if (BB.NumInstructions != MergedBB.NumInstructions)
155 report_error(Message: BF.Name + " : BB #" + Twine(BB.Index),
156 CustomError: "number of instructions in block mismatch");
157 if (BB.Hash != MergedBB.Hash)
158 report_error(Message: BF.Name + " : BB #" + Twine(BB.Index),
159 CustomError: "basic block hash mismatch");
160
161 // Update the execution count.
162 MergedBB.ExecCount += BB.ExecCount;
163
164 // Update the event count.
165 MergedBB.EventCount += BB.EventCount;
166
167 // Merge calls sites.
168 std::unordered_map<uint32_t, CallSiteInfo *> CSByOffset;
169 for (CallSiteInfo &CS : BB.CallSites)
170 CSByOffset.emplace(args: std::make_pair(x&: CS.Offset, y: &CS));
171
172 for (CallSiteInfo &MergedCS : MergedBB.CallSites) {
173 auto CSI = CSByOffset.find(x: MergedCS.Offset);
174 if (CSI == CSByOffset.end())
175 continue;
176 yaml::bolt::CallSiteInfo &CS = *CSI->second;
177 if (CS != MergedCS)
178 continue;
179
180 MergedCS.Count += CS.Count;
181 MergedCS.Mispreds += CS.Mispreds;
182
183 CSByOffset.erase(position: CSI);
184 }
185
186 // Append the rest of call sites.
187 for (std::pair<const uint32_t, CallSiteInfo *> CSI : CSByOffset)
188 MergedBB.CallSites.emplace_back(args: std::move(*CSI.second));
189
190 // Merge successor info.
191 std::vector<SuccessorInfo *> SIByIndex(BF.NumBasicBlocks);
192 for (SuccessorInfo &SI : BB.Successors) {
193 if (SI.Index >= BF.NumBasicBlocks)
194 report_error(Message: BF.Name, CustomError: "bad successor index");
195 SIByIndex[SI.Index] = &SI;
196 }
197 for (SuccessorInfo &MergedSI : MergedBB.Successors) {
198 if (!SIByIndex[MergedSI.Index])
199 continue;
200 SuccessorInfo &SI = *SIByIndex[MergedSI.Index];
201
202 MergedSI.Count += SI.Count;
203 MergedSI.Mispreds += SI.Mispreds;
204
205 SIByIndex[MergedSI.Index] = nullptr;
206 }
207 for (SuccessorInfo *SI : SIByIndex)
208 if (SI)
209 MergedBB.Successors.emplace_back(args: std::move(*SI));
210}
211
212void mergeFunctionProfile(BinaryFunctionProfile &MergedBF,
213 BinaryFunctionProfile &&BF) {
214 // Validate that we are merging the correct function.
215 if (BF.NumBasicBlocks != MergedBF.NumBasicBlocks)
216 report_error(Message: BF.Name, CustomError: "number of basic blocks mismatch");
217 if (BF.Id != MergedBF.Id)
218 report_error(Message: BF.Name, CustomError: "ID mismatch");
219 if (BF.Hash != MergedBF.Hash)
220 report_error(Message: BF.Name, CustomError: "hash mismatch");
221
222 // Update the execution count.
223 MergedBF.ExecCount += BF.ExecCount;
224
225 // Merge basic blocks profile.
226 std::vector<BinaryBasicBlockProfile *> BlockByIndex(BF.NumBasicBlocks);
227 for (BinaryBasicBlockProfile &BB : BF.Blocks) {
228 if (BB.Index >= BF.NumBasicBlocks)
229 report_error(Message: BF.Name + " : BB #" + Twine(BB.Index),
230 CustomError: "bad basic block index");
231 BlockByIndex[BB.Index] = &BB;
232 }
233 for (BinaryBasicBlockProfile &MergedBB : MergedBF.Blocks) {
234 if (!BlockByIndex[MergedBB.Index])
235 continue;
236 BinaryBasicBlockProfile &BB = *BlockByIndex[MergedBB.Index];
237
238 mergeBasicBlockProfile(MergedBB, BB: std::move(BB), BF: MergedBF);
239
240 // Ignore this block in the future.
241 BlockByIndex[MergedBB.Index] = nullptr;
242 }
243
244 // Append blocks unique to BF (i.e. those that are not in MergedBF).
245 for (BinaryBasicBlockProfile *BB : BlockByIndex)
246 if (BB)
247 MergedBF.Blocks.emplace_back(args: std::move(*BB));
248}
249
250bool isYAML(const StringRef Filename) {
251 ErrorOr<std::unique_ptr<MemoryBuffer>> MB =
252 MemoryBuffer::getFileOrSTDIN(Filename);
253 if (std::error_code EC = MB.getError())
254 report_error(Message: Filename, EC);
255 StringRef Buffer = MB.get()->getBuffer();
256 if (Buffer.starts_with(Prefix: "---\n"))
257 return true;
258 return false;
259}
260
261void mergeLegacyProfiles(const SmallVectorImpl<std::string> &Filenames) {
262 errs() << "Using legacy profile format.\n";
263 std::optional<bool> BoltedCollection;
264 std::mutex BoltedCollectionMutex;
265 typedef StringMap<uint64_t> ProfileTy;
266
267 auto ParseProfile = [&](const std::string &Filename, auto &Profiles) {
268 const llvm::thread::id tid = llvm::this_thread::get_id();
269
270 if (isYAML(Filename))
271 report_error(Message: Filename, CustomError: "cannot mix YAML and legacy formats");
272 ErrorOr<std::unique_ptr<MemoryBuffer>> MB =
273 MemoryBuffer::getFileOrSTDIN(Filename);
274 if (std::error_code EC = MB.getError())
275 report_error(Message: Filename, EC);
276
277 StringRef Buf = MB.get()->getBuffer();
278 ProfileTy *Profile;
279 {
280 std::lock_guard<std::mutex> Lock(BoltedCollectionMutex);
281 // Check if the string "boltedcollection" is in the first line
282 if (Buf.starts_with(Prefix: "boltedcollection\n")) {
283 if (!BoltedCollection.value_or(u: true))
284 report_error(
285 Message: Filename,
286 CustomError: "cannot mix profile collected in BOLT and non-BOLT deployments");
287 BoltedCollection = true;
288 Buf = Buf.drop_front(N: 17);
289 } else {
290 if (BoltedCollection.value_or(u: false))
291 report_error(
292 Message: Filename,
293 CustomError: "cannot mix profile collected in BOLT and non-BOLT deployments");
294 BoltedCollection = false;
295 }
296
297 Profile = &Profiles[tid];
298 }
299
300 SmallVector<StringRef> Lines;
301 SplitString(Source: Buf, OutFragments&: Lines, Delimiters: "\n");
302 for (StringRef Line : Lines) {
303 size_t Pos = Line.rfind(Str: " ");
304 if (Pos == StringRef::npos)
305 report_error(Message: Filename, CustomError: "Malformed / corrupted profile");
306 StringRef Signature = Line.substr(Start: 0, N: Pos);
307 uint64_t Count;
308 if (Line.substr(Start: Pos + 1, N: Line.size() - Pos).getAsInteger(Radix: 10, Result&: Count))
309 report_error(Message: Filename, CustomError: "Malformed / corrupted profile counter");
310 Count += Profile->lookup(Key: Signature);
311 Profile->insert_or_assign(Key: Signature, Val&: Count);
312 }
313 };
314
315 // The final reduction has non-trivial cost, make sure each thread has at
316 // least 4 tasks.
317 ThreadPoolStrategy S = optimal_concurrency(
318 TaskCount: std::max(a: Filenames.size() / 4, b: static_cast<size_t>(1)));
319 DefaultThreadPool Pool(S);
320 DenseMap<llvm::thread::id, ProfileTy> ParsedProfiles(
321 Pool.getMaxConcurrency());
322 for (const auto &Filename : Filenames)
323 Pool.async(F&: ParseProfile, ArgList: std::cref(t: Filename), ArgList: std::ref(t&: ParsedProfiles));
324 Pool.wait();
325
326 ProfileTy MergedProfile;
327 for (const auto &[Thread, Profile] : ParsedProfiles)
328 for (const auto &[Key, Value] : Profile) {
329 uint64_t Count = MergedProfile.lookup(Key) + Value;
330 MergedProfile.insert_or_assign(Key, Val&: Count);
331 }
332
333 if (BoltedCollection.value_or(u: false))
334 output() << "boltedcollection\n";
335 for (const auto &[Key, Value] : MergedProfile)
336 output() << Key << " " << Value << "\n";
337
338 errs() << "Profile from " << Filenames.size() << " files merged.\n";
339}
340
341} // anonymous namespace
342
343int main(int argc, char **argv) {
344 // Print a stack trace if we signal out.
345 sys::PrintStackTraceOnErrorSignal(Argv0: argv[0]);
346 PrettyStackTraceProgram X(argc, argv);
347
348 llvm_shutdown_obj Y; // Call llvm_shutdown() on exit.
349
350 cl::HideUnrelatedOptions(Category&: opts::MergeFdataCategory);
351
352 cl::ParseCommandLineOptions(argc, argv,
353 Overview: "merge multiple fdata into a single file");
354
355 ToolName = argv[0];
356
357 // Recursively expand input directories into input file lists.
358 SmallVector<std::string> Inputs;
359 for (std::string &InputDataFilename : opts::InputDataFilenames) {
360 if (!llvm::sys::fs::exists(Path: InputDataFilename))
361 report_error(Message: InputDataFilename,
362 EC: std::make_error_code(e: std::errc::no_such_file_or_directory));
363 if (llvm::sys::fs::is_regular_file(Path: InputDataFilename))
364 Inputs.emplace_back(Args&: InputDataFilename);
365 else if (llvm::sys::fs::is_directory(Path: InputDataFilename)) {
366 std::error_code EC;
367 for (llvm::sys::fs::recursive_directory_iterator F(InputDataFilename, EC),
368 E;
369 F != E && !EC; F.increment(ec&: EC))
370 if (llvm::sys::fs::is_regular_file(Path: F->path()))
371 Inputs.emplace_back(Args: F->path());
372 if (EC)
373 report_error(Message: InputDataFilename, EC);
374 }
375 }
376
377 if (!isYAML(Filename: Inputs.front())) {
378 mergeLegacyProfiles(Filenames: Inputs);
379 return 0;
380 }
381
382 // Merged header.
383 BinaryProfileHeader MergedHeader;
384 MergedHeader.Version = 1;
385
386 // Merged information for all functions.
387 StringMap<BinaryFunctionProfile> MergedBFs;
388
389 for (std::string &InputDataFilename : Inputs) {
390 ErrorOr<std::unique_ptr<MemoryBuffer>> MB =
391 MemoryBuffer::getFileOrSTDIN(Filename: InputDataFilename);
392 if (std::error_code EC = MB.getError())
393 report_error(Message: InputDataFilename, EC);
394 yaml::Input YamlInput(MB.get()->getBuffer());
395
396 errs() << "Merging data from " << InputDataFilename << "...\n";
397
398 BinaryProfile BP;
399 YamlInput >> BP;
400 if (YamlInput.error())
401 report_error(Message: InputDataFilename, EC: YamlInput.error());
402
403 // Sanity check.
404 if (BP.Header.Version != 1) {
405 errs() << "Unable to merge data from profile using version "
406 << BP.Header.Version << '\n';
407 exit(status: 1);
408 }
409
410 // Merge the header.
411 mergeProfileHeaders(MergedHeader, Header: BP.Header);
412
413 // Do the function merge.
414 for (BinaryFunctionProfile &BF : BP.Functions) {
415 if (!MergedBFs.count(Key: BF.Name)) {
416 MergedBFs.insert(KV: std::make_pair(x&: BF.Name, y&: BF));
417 continue;
418 }
419
420 BinaryFunctionProfile &MergedBF = MergedBFs.find(Key: BF.Name)->second;
421 mergeFunctionProfile(MergedBF, BF: std::move(BF));
422 }
423 }
424
425 if (!opts::SuppressMergedDataOutput) {
426 yaml::Output YamlOut(output());
427
428 BinaryProfile MergedProfile;
429 MergedProfile.Header = MergedHeader;
430 MergedProfile.Functions.resize(new_size: MergedBFs.size());
431 llvm::copy(Range: llvm::make_second_range(c&: MergedBFs),
432 Out: MergedProfile.Functions.begin());
433
434 // For consistency, sort functions by their IDs.
435 llvm::sort(C&: MergedProfile.Functions,
436 Comp: [](const BinaryFunctionProfile &A,
437 const BinaryFunctionProfile &B) { return A.Id < B.Id; });
438
439 YamlOut << MergedProfile;
440 }
441
442 errs() << "Data for " << MergedBFs.size()
443 << " unique objects successfully merged.\n";
444
445 if (opts::PrintFunctionList != opts::ST_NONE) {
446 // List of function names with execution count.
447 std::vector<std::pair<uint64_t, StringRef>> FunctionList(MergedBFs.size());
448 using CountFuncType = std::function<std::pair<uint64_t, StringRef>(
449 const StringMapEntry<BinaryFunctionProfile> &)>;
450 CountFuncType ExecCountFunc =
451 [](const StringMapEntry<BinaryFunctionProfile> &V) {
452 return std::make_pair(x: V.second.ExecCount, y: StringRef(V.second.Name));
453 };
454 CountFuncType BranchCountFunc =
455 [](const StringMapEntry<BinaryFunctionProfile> &V) {
456 // Return total branch count.
457 uint64_t BranchCount = 0;
458 for (const BinaryBasicBlockProfile &BI : V.second.Blocks)
459 for (const SuccessorInfo &SI : BI.Successors)
460 BranchCount += SI.Count;
461 return std::make_pair(x&: BranchCount, y: StringRef(V.second.Name));
462 };
463
464 CountFuncType CountFunc = (opts::PrintFunctionList == opts::ST_EXEC_COUNT)
465 ? ExecCountFunc
466 : BranchCountFunc;
467 llvm::transform(Range&: MergedBFs, d_first: FunctionList.begin(), F: CountFunc);
468 llvm::stable_sort(Range: reverse(C&: FunctionList));
469 errs() << "Functions sorted by "
470 << (opts::PrintFunctionList == opts::ST_EXEC_COUNT ? "execution"
471 : "total branch")
472 << " count:\n";
473 for (std::pair<uint64_t, StringRef> &FI : FunctionList)
474 errs() << FI.second << " : " << FI.first << '\n';
475 }
476
477 return 0;
478}
479

source code of bolt/tools/merge-fdata/merge-fdata.cpp