1//===- InstrProf.cpp - Instrumented profiling format support --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains support for clang's instrumentation based PGO and
10// coverage.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/ProfileData/InstrProf.h"
15#include "llvm/ADT/ArrayRef.h"
16#include "llvm/ADT/SetVector.h"
17#include "llvm/ADT/SmallVector.h"
18#include "llvm/ADT/StringExtras.h"
19#include "llvm/ADT/StringRef.h"
20#include "llvm/Config/config.h"
21#include "llvm/IR/Constant.h"
22#include "llvm/IR/Constants.h"
23#include "llvm/IR/Function.h"
24#include "llvm/IR/GlobalValue.h"
25#include "llvm/IR/GlobalVariable.h"
26#include "llvm/IR/Instruction.h"
27#include "llvm/IR/LLVMContext.h"
28#include "llvm/IR/MDBuilder.h"
29#include "llvm/IR/Metadata.h"
30#include "llvm/IR/Module.h"
31#include "llvm/IR/Type.h"
32#include "llvm/ProfileData/InstrProfReader.h"
33#include "llvm/Support/Casting.h"
34#include "llvm/Support/CommandLine.h"
35#include "llvm/Support/Compiler.h"
36#include "llvm/Support/Compression.h"
37#include "llvm/Support/Debug.h"
38#include "llvm/Support/Endian.h"
39#include "llvm/Support/Error.h"
40#include "llvm/Support/ErrorHandling.h"
41#include "llvm/Support/LEB128.h"
42#include "llvm/Support/MathExtras.h"
43#include "llvm/Support/Path.h"
44#include "llvm/Support/SwapByteOrder.h"
45#include "llvm/Support/VirtualFileSystem.h"
46#include "llvm/TargetParser/Triple.h"
47#include <algorithm>
48#include <cassert>
49#include <cstddef>
50#include <cstdint>
51#include <cstring>
52#include <memory>
53#include <string>
54#include <system_error>
55#include <type_traits>
56#include <utility>
57#include <vector>
58
59using namespace llvm;
60
61#define DEBUG_TYPE "instrprof"
62
63static cl::opt<bool> StaticFuncFullModulePrefix(
64 "static-func-full-module-prefix", cl::init(Val: true), cl::Hidden,
65 cl::desc("Use full module build paths in the profile counter names for "
66 "static functions."));
67
68// This option is tailored to users that have different top-level directory in
69// profile-gen and profile-use compilation. Users need to specific the number
70// of levels to strip. A value larger than the number of directories in the
71// source file will strip all the directory names and only leave the basename.
72//
73// Note current ThinLTO module importing for the indirect-calls assumes
74// the source directory name not being stripped. A non-zero option value here
75// can potentially prevent some inter-module indirect-call-promotions.
76static cl::opt<unsigned> StaticFuncStripDirNamePrefix(
77 "static-func-strip-dirname-prefix", cl::init(Val: 0), cl::Hidden,
78 cl::desc("Strip specified level of directory name from source path in "
79 "the profile counter name for static functions."));
80
81static std::string getInstrProfErrString(instrprof_error Err,
82 const std::string &ErrMsg = "") {
83 std::string Msg;
84 raw_string_ostream OS(Msg);
85
86 switch (Err) {
87 case instrprof_error::success:
88 OS << "success";
89 break;
90 case instrprof_error::eof:
91 OS << "end of File";
92 break;
93 case instrprof_error::unrecognized_format:
94 OS << "unrecognized instrumentation profile encoding format";
95 break;
96 case instrprof_error::bad_magic:
97 OS << "invalid instrumentation profile data (bad magic)";
98 break;
99 case instrprof_error::bad_header:
100 OS << "invalid instrumentation profile data (file header is corrupt)";
101 break;
102 case instrprof_error::unsupported_version:
103 OS << "unsupported instrumentation profile format version";
104 break;
105 case instrprof_error::unsupported_hash_type:
106 OS << "unsupported instrumentation profile hash type";
107 break;
108 case instrprof_error::too_large:
109 OS << "too much profile data";
110 break;
111 case instrprof_error::truncated:
112 OS << "truncated profile data";
113 break;
114 case instrprof_error::malformed:
115 OS << "malformed instrumentation profile data";
116 break;
117 case instrprof_error::missing_correlation_info:
118 OS << "debug info/binary for correlation is required";
119 break;
120 case instrprof_error::unexpected_correlation_info:
121 OS << "debug info/binary for correlation is not necessary";
122 break;
123 case instrprof_error::unable_to_correlate_profile:
124 OS << "unable to correlate profile";
125 break;
126 case instrprof_error::invalid_prof:
127 OS << "invalid profile created. Please file a bug "
128 "at: " BUG_REPORT_URL
129 " and include the profraw files that caused this error.";
130 break;
131 case instrprof_error::unknown_function:
132 OS << "no profile data available for function";
133 break;
134 case instrprof_error::hash_mismatch:
135 OS << "function control flow change detected (hash mismatch)";
136 break;
137 case instrprof_error::count_mismatch:
138 OS << "function basic block count change detected (counter mismatch)";
139 break;
140 case instrprof_error::bitmap_mismatch:
141 OS << "function bitmap size change detected (bitmap size mismatch)";
142 break;
143 case instrprof_error::counter_overflow:
144 OS << "counter overflow";
145 break;
146 case instrprof_error::value_site_count_mismatch:
147 OS << "function value site count change detected (counter mismatch)";
148 break;
149 case instrprof_error::compress_failed:
150 OS << "failed to compress data (zlib)";
151 break;
152 case instrprof_error::uncompress_failed:
153 OS << "failed to uncompress data (zlib)";
154 break;
155 case instrprof_error::empty_raw_profile:
156 OS << "empty raw profile file";
157 break;
158 case instrprof_error::zlib_unavailable:
159 OS << "profile uses zlib compression but the profile reader was built "
160 "without zlib support";
161 break;
162 case instrprof_error::raw_profile_version_mismatch:
163 OS << "raw profile version mismatch";
164 break;
165 case instrprof_error::counter_value_too_large:
166 OS << "excessively large counter value suggests corrupted profile data";
167 break;
168 }
169
170 // If optional error message is not empty, append it to the message.
171 if (!ErrMsg.empty())
172 OS << ": " << ErrMsg;
173
174 return OS.str();
175}
176
177namespace {
178
179// FIXME: This class is only here to support the transition to llvm::Error. It
180// will be removed once this transition is complete. Clients should prefer to
181// deal with the Error value directly, rather than converting to error_code.
182class InstrProfErrorCategoryType : public std::error_category {
183 const char *name() const noexcept override { return "llvm.instrprof"; }
184
185 std::string message(int IE) const override {
186 return getInstrProfErrString(Err: static_cast<instrprof_error>(IE));
187 }
188};
189
190} // end anonymous namespace
191
192const std::error_category &llvm::instrprof_category() {
193 static InstrProfErrorCategoryType ErrorCategory;
194 return ErrorCategory;
195}
196
197namespace {
198
199const char *InstrProfSectNameCommon[] = {
200#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
201 SectNameCommon,
202#include "llvm/ProfileData/InstrProfData.inc"
203};
204
205const char *InstrProfSectNameCoff[] = {
206#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
207 SectNameCoff,
208#include "llvm/ProfileData/InstrProfData.inc"
209};
210
211const char *InstrProfSectNamePrefix[] = {
212#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
213 Prefix,
214#include "llvm/ProfileData/InstrProfData.inc"
215};
216
217} // namespace
218
219namespace llvm {
220
221cl::opt<bool> DoInstrProfNameCompression(
222 "enable-name-compression",
223 cl::desc("Enable name/filename string compression"), cl::init(Val: true));
224
225cl::opt<bool> EnableVTableValueProfiling(
226 "enable-vtable-value-profiling", cl::init(Val: false),
227 cl::desc("If true, the virtual table address will be instrumented to know "
228 "the types of a C++ pointer. The information is used in indirect "
229 "call promotion to do selective vtable-based comparison."));
230
231std::string getInstrProfSectionName(InstrProfSectKind IPSK,
232 Triple::ObjectFormatType OF,
233 bool AddSegmentInfo) {
234 std::string SectName;
235
236 if (OF == Triple::MachO && AddSegmentInfo)
237 SectName = InstrProfSectNamePrefix[IPSK];
238
239 if (OF == Triple::COFF)
240 SectName += InstrProfSectNameCoff[IPSK];
241 else
242 SectName += InstrProfSectNameCommon[IPSK];
243
244 if (OF == Triple::MachO && IPSK == IPSK_data && AddSegmentInfo)
245 SectName += ",regular,live_support";
246
247 return SectName;
248}
249
250std::string InstrProfError::message() const {
251 return getInstrProfErrString(Err, ErrMsg: Msg);
252}
253
254char InstrProfError::ID = 0;
255
256std::string getPGOFuncName(StringRef Name, GlobalValue::LinkageTypes Linkage,
257 StringRef FileName,
258 uint64_t Version LLVM_ATTRIBUTE_UNUSED) {
259 // Value names may be prefixed with a binary '1' to indicate
260 // that the backend should not modify the symbols due to any platform
261 // naming convention. Do not include that '1' in the PGO profile name.
262 if (Name[0] == '\1')
263 Name = Name.substr(Start: 1);
264
265 std::string NewName = std::string(Name);
266 if (llvm::GlobalValue::isLocalLinkage(Linkage)) {
267 // For local symbols, prepend the main file name to distinguish them.
268 // Do not include the full path in the file name since there's no guarantee
269 // that it will stay the same, e.g., if the files are checked out from
270 // version control in different locations.
271 if (FileName.empty())
272 NewName = NewName.insert(pos: 0, s: "<unknown>:");
273 else
274 NewName = NewName.insert(pos1: 0, str: FileName.str() + ":");
275 }
276 return NewName;
277}
278
279// Strip NumPrefix level of directory name from PathNameStr. If the number of
280// directory separators is less than NumPrefix, strip all the directories and
281// leave base file name only.
282static StringRef stripDirPrefix(StringRef PathNameStr, uint32_t NumPrefix) {
283 uint32_t Count = NumPrefix;
284 uint32_t Pos = 0, LastPos = 0;
285 for (auto & CI : PathNameStr) {
286 ++Pos;
287 if (llvm::sys::path::is_separator(value: CI)) {
288 LastPos = Pos;
289 --Count;
290 }
291 if (Count == 0)
292 break;
293 }
294 return PathNameStr.substr(Start: LastPos);
295}
296
297static StringRef getStrippedSourceFileName(const GlobalObject &GO) {
298 StringRef FileName(GO.getParent()->getSourceFileName());
299 uint32_t StripLevel = StaticFuncFullModulePrefix ? 0 : (uint32_t)-1;
300 if (StripLevel < StaticFuncStripDirNamePrefix)
301 StripLevel = StaticFuncStripDirNamePrefix;
302 if (StripLevel)
303 FileName = stripDirPrefix(PathNameStr: FileName, NumPrefix: StripLevel);
304 return FileName;
305}
306
307// The PGO name has the format [<filepath>;]<mangled-name> where <filepath>; is
308// provided if linkage is local and is used to discriminate possibly identical
309// mangled names. ";" is used because it is unlikely to be found in either
310// <filepath> or <mangled-name>.
311//
312// Older compilers used getPGOFuncName() which has the format
313// [<filepath>:]<mangled-name>. This caused trouble for Objective-C functions
314// which commonly have :'s in their names. We still need to compute this name to
315// lookup functions from profiles built by older compilers.
316static std::string
317getIRPGONameForGlobalObject(const GlobalObject &GO,
318 GlobalValue::LinkageTypes Linkage,
319 StringRef FileName) {
320 return GlobalValue::getGlobalIdentifier(Name: GO.getName(), Linkage, FileName);
321}
322
323static std::optional<std::string> lookupPGONameFromMetadata(MDNode *MD) {
324 if (MD != nullptr) {
325 StringRef S = cast<MDString>(Val: MD->getOperand(I: 0))->getString();
326 return S.str();
327 }
328 return {};
329}
330
331// Returns the PGO object name. This function has some special handling
332// when called in LTO optimization. The following only applies when calling in
333// LTO passes (when \c InLTO is true): LTO's internalization privatizes many
334// global linkage symbols. This happens after value profile annotation, but
335// those internal linkage functions should not have a source prefix.
336// Additionally, for ThinLTO mode, exported internal functions are promoted
337// and renamed. We need to ensure that the original internal PGO name is
338// used when computing the GUID that is compared against the profiled GUIDs.
339// To differentiate compiler generated internal symbols from original ones,
340// PGOFuncName meta data are created and attached to the original internal
341// symbols in the value profile annotation step
342// (PGOUseFunc::annotateIndirectCallSites). If a symbol does not have the meta
343// data, its original linkage must be non-internal.
344static std::string getIRPGOObjectName(const GlobalObject &GO, bool InLTO,
345 MDNode *PGONameMetadata) {
346 if (!InLTO) {
347 auto FileName = getStrippedSourceFileName(GO);
348 return getIRPGONameForGlobalObject(GO, Linkage: GO.getLinkage(), FileName);
349 }
350
351 // In LTO mode (when InLTO is true), first check if there is a meta data.
352 if (auto IRPGOFuncName = lookupPGONameFromMetadata(MD: PGONameMetadata))
353 return *IRPGOFuncName;
354
355 // If there is no meta data, the function must be a global before the value
356 // profile annotation pass. Its current linkage may be internal if it is
357 // internalized in LTO mode.
358 return getIRPGONameForGlobalObject(GO, Linkage: GlobalValue::ExternalLinkage, FileName: "");
359}
360
361// Returns the IRPGO function name and does special handling when called
362// in LTO optimization. See the comments of `getIRPGOObjectName` for details.
363std::string getIRPGOFuncName(const Function &F, bool InLTO) {
364 return getIRPGOObjectName(GO: F, InLTO, PGONameMetadata: getPGOFuncNameMetadata(F));
365}
366
367// Please use getIRPGOFuncName for LLVM IR instrumentation. This function is
368// for front-end (Clang, etc) instrumentation.
369// The implementation is kept for profile matching from older profiles.
370// This is similar to `getIRPGOFuncName` except that this function calls
371// 'getPGOFuncName' to get a name and `getIRPGOFuncName` calls
372// 'getIRPGONameForGlobalObject'. See the difference between two callees in the
373// comments of `getIRPGONameForGlobalObject`.
374std::string getPGOFuncName(const Function &F, bool InLTO, uint64_t Version) {
375 if (!InLTO) {
376 auto FileName = getStrippedSourceFileName(GO: F);
377 return getPGOFuncName(Name: F.getName(), Linkage: F.getLinkage(), FileName, Version);
378 }
379
380 // In LTO mode (when InLTO is true), first check if there is a meta data.
381 if (auto PGOFuncName = lookupPGONameFromMetadata(MD: getPGOFuncNameMetadata(F)))
382 return *PGOFuncName;
383
384 // If there is no meta data, the function must be a global before the value
385 // profile annotation pass. Its current linkage may be internal if it is
386 // internalized in LTO mode.
387 return getPGOFuncName(Name: F.getName(), Linkage: GlobalValue::ExternalLinkage, FileName: "");
388}
389
390std::string getPGOName(const GlobalVariable &V, bool InLTO) {
391 // PGONameMetadata should be set by compiler at profile use time
392 // and read by symtab creation to look up symbols corresponding to
393 // a MD5 hash.
394 return getIRPGOObjectName(GO: V, InLTO, /*PGONameMetadata=*/nullptr);
395}
396
397// See getIRPGOObjectName() for a discription of the format.
398std::pair<StringRef, StringRef> getParsedIRPGOName(StringRef IRPGOName) {
399 auto [FileName, MangledName] = IRPGOName.split(Separator: kGlobalIdentifierDelimiter);
400 if (MangledName.empty())
401 return std::make_pair(x: StringRef(), y&: IRPGOName);
402 return std::make_pair(x&: FileName, y&: MangledName);
403}
404
405StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName) {
406 if (FileName.empty())
407 return PGOFuncName;
408 // Drop the file name including ':' or ';'. See getIRPGONameForGlobalObject as
409 // well.
410 if (PGOFuncName.starts_with(Prefix: FileName))
411 PGOFuncName = PGOFuncName.drop_front(N: FileName.size() + 1);
412 return PGOFuncName;
413}
414
415// \p FuncName is the string used as profile lookup key for the function. A
416// symbol is created to hold the name. Return the legalized symbol name.
417std::string getPGOFuncNameVarName(StringRef FuncName,
418 GlobalValue::LinkageTypes Linkage) {
419 std::string VarName = std::string(getInstrProfNameVarPrefix());
420 VarName += FuncName;
421
422 if (!GlobalValue::isLocalLinkage(Linkage))
423 return VarName;
424
425 // Now fix up illegal chars in local VarName that may upset the assembler.
426 const char InvalidChars[] = "-:;<>/\"'";
427 size_t found = VarName.find_first_of(s: InvalidChars);
428 while (found != std::string::npos) {
429 VarName[found] = '_';
430 found = VarName.find_first_of(s: InvalidChars, pos: found + 1);
431 }
432 return VarName;
433}
434
435GlobalVariable *createPGOFuncNameVar(Module &M,
436 GlobalValue::LinkageTypes Linkage,
437 StringRef PGOFuncName) {
438 // We generally want to match the function's linkage, but available_externally
439 // and extern_weak both have the wrong semantics, and anything that doesn't
440 // need to link across compilation units doesn't need to be visible at all.
441 if (Linkage == GlobalValue::ExternalWeakLinkage)
442 Linkage = GlobalValue::LinkOnceAnyLinkage;
443 else if (Linkage == GlobalValue::AvailableExternallyLinkage)
444 Linkage = GlobalValue::LinkOnceODRLinkage;
445 else if (Linkage == GlobalValue::InternalLinkage ||
446 Linkage == GlobalValue::ExternalLinkage)
447 Linkage = GlobalValue::PrivateLinkage;
448
449 auto *Value =
450 ConstantDataArray::getString(Context&: M.getContext(), Initializer: PGOFuncName, AddNull: false);
451 auto FuncNameVar =
452 new GlobalVariable(M, Value->getType(), true, Linkage, Value,
453 getPGOFuncNameVarName(FuncName: PGOFuncName, Linkage));
454
455 // Hide the symbol so that we correctly get a copy for each executable.
456 if (!GlobalValue::isLocalLinkage(Linkage: FuncNameVar->getLinkage()))
457 FuncNameVar->setVisibility(GlobalValue::HiddenVisibility);
458
459 return FuncNameVar;
460}
461
462GlobalVariable *createPGOFuncNameVar(Function &F, StringRef PGOFuncName) {
463 return createPGOFuncNameVar(M&: *F.getParent(), Linkage: F.getLinkage(), PGOFuncName);
464}
465
466Error InstrProfSymtab::create(Module &M, bool InLTO) {
467 for (Function &F : M) {
468 // Function may not have a name: like using asm("") to overwrite the name.
469 // Ignore in this case.
470 if (!F.hasName())
471 continue;
472 if (Error E = addFuncWithName(F, PGOFuncName: getIRPGOFuncName(F, InLTO)))
473 return E;
474 // Also use getPGOFuncName() so that we can find records from older profiles
475 if (Error E = addFuncWithName(F, PGOFuncName: getPGOFuncName(F, InLTO)))
476 return E;
477 }
478
479 Sorted = false;
480 finalizeSymtab();
481 return Error::success();
482}
483
484/// \c NameStrings is a string composed of one of more possibly encoded
485/// sub-strings. The substrings are separated by 0 or more zero bytes. This
486/// method decodes the string and calls `NameCallback` for each substring.
487static Error
488readAndDecodeStrings(StringRef NameStrings,
489 std::function<Error(StringRef)> NameCallback) {
490 const uint8_t *P = NameStrings.bytes_begin();
491 const uint8_t *EndP = NameStrings.bytes_end();
492 while (P < EndP) {
493 uint32_t N;
494 uint64_t UncompressedSize = decodeULEB128(p: P, n: &N);
495 P += N;
496 uint64_t CompressedSize = decodeULEB128(p: P, n: &N);
497 P += N;
498 bool isCompressed = (CompressedSize != 0);
499 SmallVector<uint8_t, 128> UncompressedNameStrings;
500 StringRef NameStrings;
501 if (isCompressed) {
502 if (!llvm::compression::zlib::isAvailable())
503 return make_error<InstrProfError>(Args: instrprof_error::zlib_unavailable);
504
505 if (Error E = compression::zlib::decompress(Input: ArrayRef(P, CompressedSize),
506 Output&: UncompressedNameStrings,
507 UncompressedSize)) {
508 consumeError(Err: std::move(E));
509 return make_error<InstrProfError>(Args: instrprof_error::uncompress_failed);
510 }
511 P += CompressedSize;
512 NameStrings = toStringRef(Input: UncompressedNameStrings);
513 } else {
514 NameStrings =
515 StringRef(reinterpret_cast<const char *>(P), UncompressedSize);
516 P += UncompressedSize;
517 }
518 // Now parse the name strings.
519 SmallVector<StringRef, 0> Names;
520 NameStrings.split(A&: Names, Separator: getInstrProfNameSeparator());
521 for (StringRef &Name : Names)
522 if (Error E = NameCallback(Name))
523 return E;
524
525 while (P < EndP && *P == 0)
526 P++;
527 }
528 return Error::success();
529}
530
531Error InstrProfSymtab::create(StringRef NameStrings) {
532 return readAndDecodeStrings(
533 NameStrings,
534 NameCallback: std::bind(f: &InstrProfSymtab::addFuncName, args: this, args: std::placeholders::_1));
535}
536
537Error InstrProfSymtab::create(StringRef FuncNameStrings,
538 StringRef VTableNameStrings) {
539 if (Error E = readAndDecodeStrings(NameStrings: FuncNameStrings,
540 NameCallback: std::bind(f: &InstrProfSymtab::addFuncName,
541 args: this, args: std::placeholders::_1)))
542 return E;
543
544 return readAndDecodeStrings(
545 NameStrings: VTableNameStrings,
546 NameCallback: std::bind(f: &InstrProfSymtab::addVTableName, args: this, args: std::placeholders::_1));
547}
548
549Error InstrProfSymtab::initVTableNamesFromCompressedStrings(
550 StringRef CompressedVTableStrings) {
551 return readAndDecodeStrings(
552 NameStrings: CompressedVTableStrings,
553 NameCallback: std::bind(f: &InstrProfSymtab::addVTableName, args: this, args: std::placeholders::_1));
554}
555
556StringRef InstrProfSymtab::getCanonicalName(StringRef PGOName) {
557 // In ThinLTO, local function may have been promoted to global and have
558 // suffix ".llvm." added to the function name. We need to add the
559 // stripped function name to the symbol table so that we can find a match
560 // from profile.
561 //
562 // ".__uniq." suffix is used to differentiate internal linkage functions in
563 // different modules and should be kept. This is the only suffix with the
564 // pattern ".xxx" which is kept before matching, other suffixes similar as
565 // ".llvm." will be stripped.
566 const std::string UniqSuffix = ".__uniq.";
567 size_t pos = PGOName.find(Str: UniqSuffix);
568 if (pos != StringRef::npos)
569 pos += UniqSuffix.length();
570 else
571 pos = 0;
572
573 // Search '.' after ".__uniq." if ".__uniq." exists, otherwise search '.' from
574 // the beginning.
575 pos = PGOName.find(C: '.', From: pos);
576 if (pos != StringRef::npos && pos != 0)
577 return PGOName.substr(Start: 0, N: pos);
578
579 return PGOName;
580}
581
582Error InstrProfSymtab::addFuncWithName(Function &F, StringRef PGOFuncName) {
583 auto mapName = [&](StringRef Name) -> Error {
584 if (Error E = addFuncName(FuncName: Name))
585 return E;
586 MD5FuncMap.emplace_back(args: Function::getGUID(GlobalName: Name), args: &F);
587 return Error::success();
588 };
589 if (Error E = mapName(PGOFuncName))
590 return E;
591
592 StringRef CanonicalFuncName = getCanonicalName(PGOName: PGOFuncName);
593 if (CanonicalFuncName != PGOFuncName)
594 return mapName(CanonicalFuncName);
595
596 return Error::success();
597}
598
599uint64_t InstrProfSymtab::getVTableHashFromAddress(uint64_t Address) {
600 // Given a runtime address, look up the hash value in the interval map, and
601 // fallback to value 0 if a hash value is not found.
602 return VTableAddrMap.lookup(x: Address, NotFound: 0);
603}
604
605uint64_t InstrProfSymtab::getFunctionHashFromAddress(uint64_t Address) {
606 finalizeSymtab();
607 auto It = partition_point(Range&: AddrToMD5Map, P: [=](std::pair<uint64_t, uint64_t> A) {
608 return A.first < Address;
609 });
610 // Raw function pointer collected by value profiler may be from
611 // external functions that are not instrumented. They won't have
612 // mapping data to be used by the deserializer. Force the value to
613 // be 0 in this case.
614 if (It != AddrToMD5Map.end() && It->first == Address)
615 return (uint64_t)It->second;
616 return 0;
617}
618
619void InstrProfSymtab::dumpNames(raw_ostream &OS) const {
620 SmallVector<StringRef, 0> Sorted(NameTab.keys());
621 llvm::sort(C&: Sorted);
622 for (StringRef S : Sorted)
623 OS << S << '\n';
624}
625
626Error collectGlobalObjectNameStrings(ArrayRef<std::string> NameStrs,
627 bool doCompression, std::string &Result) {
628 assert(!NameStrs.empty() && "No name data to emit");
629
630 uint8_t Header[20], *P = Header;
631 std::string UncompressedNameStrings =
632 join(Begin: NameStrs.begin(), End: NameStrs.end(), Separator: getInstrProfNameSeparator());
633
634 assert(StringRef(UncompressedNameStrings)
635 .count(getInstrProfNameSeparator()) == (NameStrs.size() - 1) &&
636 "PGO name is invalid (contains separator token)");
637
638 unsigned EncLen = encodeULEB128(Value: UncompressedNameStrings.length(), p: P);
639 P += EncLen;
640
641 auto WriteStringToResult = [&](size_t CompressedLen, StringRef InputStr) {
642 EncLen = encodeULEB128(Value: CompressedLen, p: P);
643 P += EncLen;
644 char *HeaderStr = reinterpret_cast<char *>(&Header[0]);
645 unsigned HeaderLen = P - &Header[0];
646 Result.append(s: HeaderStr, n: HeaderLen);
647 Result += InputStr;
648 return Error::success();
649 };
650
651 if (!doCompression) {
652 return WriteStringToResult(0, UncompressedNameStrings);
653 }
654
655 SmallVector<uint8_t, 128> CompressedNameStrings;
656 compression::zlib::compress(Input: arrayRefFromStringRef(Input: UncompressedNameStrings),
657 CompressedBuffer&: CompressedNameStrings,
658 Level: compression::zlib::BestSizeCompression);
659
660 return WriteStringToResult(CompressedNameStrings.size(),
661 toStringRef(Input: CompressedNameStrings));
662}
663
664StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar) {
665 auto *Arr = cast<ConstantDataArray>(Val: NameVar->getInitializer());
666 StringRef NameStr =
667 Arr->isCString() ? Arr->getAsCString() : Arr->getAsString();
668 return NameStr;
669}
670
671Error collectPGOFuncNameStrings(ArrayRef<GlobalVariable *> NameVars,
672 std::string &Result, bool doCompression) {
673 std::vector<std::string> NameStrs;
674 for (auto *NameVar : NameVars) {
675 NameStrs.push_back(x: std::string(getPGOFuncNameVarInitializer(NameVar)));
676 }
677 return collectGlobalObjectNameStrings(
678 NameStrs, doCompression: compression::zlib::isAvailable() && doCompression, Result);
679}
680
681Error collectVTableStrings(ArrayRef<GlobalVariable *> VTables,
682 std::string &Result, bool doCompression) {
683 std::vector<std::string> VTableNameStrs;
684 for (auto *VTable : VTables)
685 VTableNameStrs.push_back(x: getPGOName(V: *VTable));
686 return collectGlobalObjectNameStrings(
687 NameStrs: VTableNameStrs, doCompression: compression::zlib::isAvailable() && doCompression,
688 Result);
689}
690
691void InstrProfRecord::accumulateCounts(CountSumOrPercent &Sum) const {
692 uint64_t FuncSum = 0;
693 Sum.NumEntries += Counts.size();
694 for (uint64_t Count : Counts)
695 FuncSum += Count;
696 Sum.CountSum += FuncSum;
697
698 for (uint32_t VK = IPVK_First; VK <= IPVK_Last; ++VK) {
699 uint64_t KindSum = 0;
700 uint32_t NumValueSites = getNumValueSites(ValueKind: VK);
701 for (size_t I = 0; I < NumValueSites; ++I) {
702 uint32_t NV = getNumValueDataForSite(ValueKind: VK, Site: I);
703 std::unique_ptr<InstrProfValueData[]> VD = getValueForSite(ValueKind: VK, Site: I);
704 for (uint32_t V = 0; V < NV; V++)
705 KindSum += VD[V].Count;
706 }
707 Sum.ValueCounts[VK] += KindSum;
708 }
709}
710
711void InstrProfValueSiteRecord::overlap(InstrProfValueSiteRecord &Input,
712 uint32_t ValueKind,
713 OverlapStats &Overlap,
714 OverlapStats &FuncLevelOverlap) {
715 this->sortByTargetValues();
716 Input.sortByTargetValues();
717 double Score = 0.0f, FuncLevelScore = 0.0f;
718 auto I = ValueData.begin();
719 auto IE = ValueData.end();
720 auto J = Input.ValueData.begin();
721 auto JE = Input.ValueData.end();
722 while (I != IE && J != JE) {
723 if (I->Value == J->Value) {
724 Score += OverlapStats::score(Val1: I->Count, Val2: J->Count,
725 Sum1: Overlap.Base.ValueCounts[ValueKind],
726 Sum2: Overlap.Test.ValueCounts[ValueKind]);
727 FuncLevelScore += OverlapStats::score(
728 Val1: I->Count, Val2: J->Count, Sum1: FuncLevelOverlap.Base.ValueCounts[ValueKind],
729 Sum2: FuncLevelOverlap.Test.ValueCounts[ValueKind]);
730 ++I;
731 } else if (I->Value < J->Value) {
732 ++I;
733 continue;
734 }
735 ++J;
736 }
737 Overlap.Overlap.ValueCounts[ValueKind] += Score;
738 FuncLevelOverlap.Overlap.ValueCounts[ValueKind] += FuncLevelScore;
739}
740
741// Return false on mismatch.
742void InstrProfRecord::overlapValueProfData(uint32_t ValueKind,
743 InstrProfRecord &Other,
744 OverlapStats &Overlap,
745 OverlapStats &FuncLevelOverlap) {
746 uint32_t ThisNumValueSites = getNumValueSites(ValueKind);
747 assert(ThisNumValueSites == Other.getNumValueSites(ValueKind));
748 if (!ThisNumValueSites)
749 return;
750
751 std::vector<InstrProfValueSiteRecord> &ThisSiteRecords =
752 getOrCreateValueSitesForKind(ValueKind);
753 MutableArrayRef<InstrProfValueSiteRecord> OtherSiteRecords =
754 Other.getValueSitesForKind(ValueKind);
755 for (uint32_t I = 0; I < ThisNumValueSites; I++)
756 ThisSiteRecords[I].overlap(Input&: OtherSiteRecords[I], ValueKind, Overlap,
757 FuncLevelOverlap);
758}
759
760void InstrProfRecord::overlap(InstrProfRecord &Other, OverlapStats &Overlap,
761 OverlapStats &FuncLevelOverlap,
762 uint64_t ValueCutoff) {
763 // FuncLevel CountSum for other should already computed and nonzero.
764 assert(FuncLevelOverlap.Test.CountSum >= 1.0f);
765 accumulateCounts(Sum&: FuncLevelOverlap.Base);
766 bool Mismatch = (Counts.size() != Other.Counts.size());
767
768 // Check if the value profiles mismatch.
769 if (!Mismatch) {
770 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) {
771 uint32_t ThisNumValueSites = getNumValueSites(ValueKind: Kind);
772 uint32_t OtherNumValueSites = Other.getNumValueSites(ValueKind: Kind);
773 if (ThisNumValueSites != OtherNumValueSites) {
774 Mismatch = true;
775 break;
776 }
777 }
778 }
779 if (Mismatch) {
780 Overlap.addOneMismatch(MismatchFunc: FuncLevelOverlap.Test);
781 return;
782 }
783
784 // Compute overlap for value counts.
785 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
786 overlapValueProfData(ValueKind: Kind, Other, Overlap, FuncLevelOverlap);
787
788 double Score = 0.0;
789 uint64_t MaxCount = 0;
790 // Compute overlap for edge counts.
791 for (size_t I = 0, E = Other.Counts.size(); I < E; ++I) {
792 Score += OverlapStats::score(Val1: Counts[I], Val2: Other.Counts[I],
793 Sum1: Overlap.Base.CountSum, Sum2: Overlap.Test.CountSum);
794 MaxCount = std::max(a: Other.Counts[I], b: MaxCount);
795 }
796 Overlap.Overlap.CountSum += Score;
797 Overlap.Overlap.NumEntries += 1;
798
799 if (MaxCount >= ValueCutoff) {
800 double FuncScore = 0.0;
801 for (size_t I = 0, E = Other.Counts.size(); I < E; ++I)
802 FuncScore += OverlapStats::score(Val1: Counts[I], Val2: Other.Counts[I],
803 Sum1: FuncLevelOverlap.Base.CountSum,
804 Sum2: FuncLevelOverlap.Test.CountSum);
805 FuncLevelOverlap.Overlap.CountSum = FuncScore;
806 FuncLevelOverlap.Overlap.NumEntries = Other.Counts.size();
807 FuncLevelOverlap.Valid = true;
808 }
809}
810
811void InstrProfValueSiteRecord::merge(InstrProfValueSiteRecord &Input,
812 uint64_t Weight,
813 function_ref<void(instrprof_error)> Warn) {
814 this->sortByTargetValues();
815 Input.sortByTargetValues();
816 auto I = ValueData.begin();
817 auto IE = ValueData.end();
818 for (const InstrProfValueData &J : Input.ValueData) {
819 while (I != IE && I->Value < J.Value)
820 ++I;
821 if (I != IE && I->Value == J.Value) {
822 bool Overflowed;
823 I->Count = SaturatingMultiplyAdd(X: J.Count, Y: Weight, A: I->Count, ResultOverflowed: &Overflowed);
824 if (Overflowed)
825 Warn(instrprof_error::counter_overflow);
826 ++I;
827 continue;
828 }
829 ValueData.insert(position: I, x: J);
830 }
831}
832
833void InstrProfValueSiteRecord::scale(uint64_t N, uint64_t D,
834 function_ref<void(instrprof_error)> Warn) {
835 for (InstrProfValueData &I : ValueData) {
836 bool Overflowed;
837 I.Count = SaturatingMultiply(X: I.Count, Y: N, ResultOverflowed: &Overflowed) / D;
838 if (Overflowed)
839 Warn(instrprof_error::counter_overflow);
840 }
841}
842
843// Merge Value Profile data from Src record to this record for ValueKind.
844// Scale merged value counts by \p Weight.
845void InstrProfRecord::mergeValueProfData(
846 uint32_t ValueKind, InstrProfRecord &Src, uint64_t Weight,
847 function_ref<void(instrprof_error)> Warn) {
848 uint32_t ThisNumValueSites = getNumValueSites(ValueKind);
849 uint32_t OtherNumValueSites = Src.getNumValueSites(ValueKind);
850 if (ThisNumValueSites != OtherNumValueSites) {
851 Warn(instrprof_error::value_site_count_mismatch);
852 return;
853 }
854 if (!ThisNumValueSites)
855 return;
856 std::vector<InstrProfValueSiteRecord> &ThisSiteRecords =
857 getOrCreateValueSitesForKind(ValueKind);
858 MutableArrayRef<InstrProfValueSiteRecord> OtherSiteRecords =
859 Src.getValueSitesForKind(ValueKind);
860 for (uint32_t I = 0; I < ThisNumValueSites; I++)
861 ThisSiteRecords[I].merge(Input&: OtherSiteRecords[I], Weight, Warn);
862}
863
864void InstrProfRecord::merge(InstrProfRecord &Other, uint64_t Weight,
865 function_ref<void(instrprof_error)> Warn) {
866 // If the number of counters doesn't match we either have bad data
867 // or a hash collision.
868 if (Counts.size() != Other.Counts.size()) {
869 Warn(instrprof_error::count_mismatch);
870 return;
871 }
872
873 // Special handling of the first count as the PseudoCount.
874 CountPseudoKind OtherKind = Other.getCountPseudoKind();
875 CountPseudoKind ThisKind = getCountPseudoKind();
876 if (OtherKind != NotPseudo || ThisKind != NotPseudo) {
877 // We don't allow the merge of a profile with pseudo counts and
878 // a normal profile (i.e. without pesudo counts).
879 // Profile supplimenation should be done after the profile merge.
880 if (OtherKind == NotPseudo || ThisKind == NotPseudo) {
881 Warn(instrprof_error::count_mismatch);
882 return;
883 }
884 if (OtherKind == PseudoHot || ThisKind == PseudoHot)
885 setPseudoCount(PseudoHot);
886 else
887 setPseudoCount(PseudoWarm);
888 return;
889 }
890
891 for (size_t I = 0, E = Other.Counts.size(); I < E; ++I) {
892 bool Overflowed;
893 uint64_t Value =
894 SaturatingMultiplyAdd(X: Other.Counts[I], Y: Weight, A: Counts[I], ResultOverflowed: &Overflowed);
895 if (Value > getInstrMaxCountValue()) {
896 Value = getInstrMaxCountValue();
897 Overflowed = true;
898 }
899 Counts[I] = Value;
900 if (Overflowed)
901 Warn(instrprof_error::counter_overflow);
902 }
903
904 // If the number of bitmap bytes doesn't match we either have bad data
905 // or a hash collision.
906 if (BitmapBytes.size() != Other.BitmapBytes.size()) {
907 Warn(instrprof_error::bitmap_mismatch);
908 return;
909 }
910
911 // Bitmap bytes are merged by simply ORing them together.
912 for (size_t I = 0, E = Other.BitmapBytes.size(); I < E; ++I) {
913 BitmapBytes[I] = Other.BitmapBytes[I] | BitmapBytes[I];
914 }
915
916 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
917 mergeValueProfData(ValueKind: Kind, Src&: Other, Weight, Warn);
918}
919
920void InstrProfRecord::scaleValueProfData(
921 uint32_t ValueKind, uint64_t N, uint64_t D,
922 function_ref<void(instrprof_error)> Warn) {
923 for (auto &R : getValueSitesForKind(ValueKind))
924 R.scale(N, D, Warn);
925}
926
927void InstrProfRecord::scale(uint64_t N, uint64_t D,
928 function_ref<void(instrprof_error)> Warn) {
929 assert(D != 0 && "D cannot be 0");
930 for (auto &Count : this->Counts) {
931 bool Overflowed;
932 Count = SaturatingMultiply(X: Count, Y: N, ResultOverflowed: &Overflowed) / D;
933 if (Count > getInstrMaxCountValue()) {
934 Count = getInstrMaxCountValue();
935 Overflowed = true;
936 }
937 if (Overflowed)
938 Warn(instrprof_error::counter_overflow);
939 }
940 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
941 scaleValueProfData(ValueKind: Kind, N, D, Warn);
942}
943
944// Map indirect call target name hash to name string.
945uint64_t InstrProfRecord::remapValue(uint64_t Value, uint32_t ValueKind,
946 InstrProfSymtab *SymTab) {
947 if (!SymTab)
948 return Value;
949
950 if (ValueKind == IPVK_IndirectCallTarget)
951 return SymTab->getFunctionHashFromAddress(Address: Value);
952
953 if (ValueKind == IPVK_VTableTarget)
954 return SymTab->getVTableHashFromAddress(Address: Value);
955
956 return Value;
957}
958
959void InstrProfRecord::addValueData(uint32_t ValueKind, uint32_t Site,
960 InstrProfValueData *VData, uint32_t N,
961 InstrProfSymtab *ValueMap) {
962 for (uint32_t I = 0; I < N; I++) {
963 VData[I].Value = remapValue(Value: VData[I].Value, ValueKind, SymTab: ValueMap);
964 }
965 std::vector<InstrProfValueSiteRecord> &ValueSites =
966 getOrCreateValueSitesForKind(ValueKind);
967 if (N == 0)
968 ValueSites.emplace_back();
969 else
970 ValueSites.emplace_back(args&: VData, args: VData + N);
971}
972
973std::vector<BPFunctionNode> TemporalProfTraceTy::createBPFunctionNodes(
974 ArrayRef<TemporalProfTraceTy> Traces) {
975 using IDT = BPFunctionNode::IDT;
976 using UtilityNodeT = BPFunctionNode::UtilityNodeT;
977 // Collect all function IDs ordered by their smallest timestamp. This will be
978 // used as the initial FunctionNode order.
979 SetVector<IDT> FunctionIds;
980 size_t LargestTraceSize = 0;
981 for (auto &Trace : Traces)
982 LargestTraceSize =
983 std::max(a: LargestTraceSize, b: Trace.FunctionNameRefs.size());
984 for (size_t Timestamp = 0; Timestamp < LargestTraceSize; Timestamp++)
985 for (auto &Trace : Traces)
986 if (Timestamp < Trace.FunctionNameRefs.size())
987 FunctionIds.insert(X: Trace.FunctionNameRefs[Timestamp]);
988
989 const int N = Log2_64(Value: LargestTraceSize) + 1;
990
991 // TODO: We need to use the Trace.Weight field to give more weight to more
992 // important utilities
993 DenseMap<IDT, SmallVector<UtilityNodeT, 4>> FuncGroups;
994 for (size_t TraceIdx = 0; TraceIdx < Traces.size(); TraceIdx++) {
995 auto &Trace = Traces[TraceIdx].FunctionNameRefs;
996 for (size_t Timestamp = 0; Timestamp < Trace.size(); Timestamp++) {
997 for (int I = Log2_64(Value: Timestamp + 1); I < N; I++) {
998 auto FunctionId = Trace[Timestamp];
999 UtilityNodeT GroupId = TraceIdx * N + I;
1000 FuncGroups[FunctionId].push_back(Elt: GroupId);
1001 }
1002 }
1003 }
1004
1005 std::vector<BPFunctionNode> Nodes;
1006 for (auto Id : FunctionIds) {
1007 auto &UNs = FuncGroups[Id];
1008 llvm::sort(C&: UNs);
1009 UNs.erase(CS: std::unique(first: UNs.begin(), last: UNs.end()), CE: UNs.end());
1010 Nodes.emplace_back(args&: Id, args&: UNs);
1011 }
1012 return Nodes;
1013}
1014
1015#define INSTR_PROF_COMMON_API_IMPL
1016#include "llvm/ProfileData/InstrProfData.inc"
1017
1018/*!
1019 * ValueProfRecordClosure Interface implementation for InstrProfRecord
1020 * class. These C wrappers are used as adaptors so that C++ code can be
1021 * invoked as callbacks.
1022 */
1023uint32_t getNumValueKindsInstrProf(const void *Record) {
1024 return reinterpret_cast<const InstrProfRecord *>(Record)->getNumValueKinds();
1025}
1026
1027uint32_t getNumValueSitesInstrProf(const void *Record, uint32_t VKind) {
1028 return reinterpret_cast<const InstrProfRecord *>(Record)
1029 ->getNumValueSites(ValueKind: VKind);
1030}
1031
1032uint32_t getNumValueDataInstrProf(const void *Record, uint32_t VKind) {
1033 return reinterpret_cast<const InstrProfRecord *>(Record)
1034 ->getNumValueData(ValueKind: VKind);
1035}
1036
1037uint32_t getNumValueDataForSiteInstrProf(const void *R, uint32_t VK,
1038 uint32_t S) {
1039 return reinterpret_cast<const InstrProfRecord *>(R)
1040 ->getNumValueDataForSite(ValueKind: VK, Site: S);
1041}
1042
1043void getValueForSiteInstrProf(const void *R, InstrProfValueData *Dst,
1044 uint32_t K, uint32_t S) {
1045 reinterpret_cast<const InstrProfRecord *>(R)->getValueForSite(Dest: Dst, ValueKind: K, Site: S);
1046}
1047
1048ValueProfData *allocValueProfDataInstrProf(size_t TotalSizeInBytes) {
1049 ValueProfData *VD =
1050 (ValueProfData *)(new (::operator new(TotalSizeInBytes)) ValueProfData());
1051 memset(s: VD, c: 0, n: TotalSizeInBytes);
1052 return VD;
1053}
1054
1055static ValueProfRecordClosure InstrProfRecordClosure = {
1056 .Record: nullptr,
1057 .GetNumValueKinds: getNumValueKindsInstrProf,
1058 .GetNumValueSites: getNumValueSitesInstrProf,
1059 .GetNumValueData: getNumValueDataInstrProf,
1060 .GetNumValueDataForSite: getNumValueDataForSiteInstrProf,
1061 .RemapValueData: nullptr,
1062 .GetValueForSite: getValueForSiteInstrProf,
1063 .AllocValueProfData: allocValueProfDataInstrProf};
1064
1065// Wrapper implementation using the closure mechanism.
1066uint32_t ValueProfData::getSize(const InstrProfRecord &Record) {
1067 auto Closure = InstrProfRecordClosure;
1068 Closure.Record = &Record;
1069 return getValueProfDataSize(Closure: &Closure);
1070}
1071
1072// Wrapper implementation using the closure mechanism.
1073std::unique_ptr<ValueProfData>
1074ValueProfData::serializeFrom(const InstrProfRecord &Record) {
1075 InstrProfRecordClosure.Record = &Record;
1076
1077 std::unique_ptr<ValueProfData> VPD(
1078 serializeValueProfDataFrom(Closure: &InstrProfRecordClosure, DstData: nullptr));
1079 return VPD;
1080}
1081
1082void ValueProfRecord::deserializeTo(InstrProfRecord &Record,
1083 InstrProfSymtab *SymTab) {
1084 Record.reserveSites(ValueKind: Kind, NumValueSites);
1085
1086 InstrProfValueData *ValueData = getValueProfRecordValueData(This: this);
1087 for (uint64_t VSite = 0; VSite < NumValueSites; ++VSite) {
1088 uint8_t ValueDataCount = this->SiteCountArray[VSite];
1089 Record.addValueData(ValueKind: Kind, Site: VSite, VData: ValueData, N: ValueDataCount, ValueMap: SymTab);
1090 ValueData += ValueDataCount;
1091 }
1092}
1093
1094// For writing/serializing, Old is the host endianness, and New is
1095// byte order intended on disk. For Reading/deserialization, Old
1096// is the on-disk source endianness, and New is the host endianness.
1097void ValueProfRecord::swapBytes(llvm::endianness Old, llvm::endianness New) {
1098 using namespace support;
1099
1100 if (Old == New)
1101 return;
1102
1103 if (llvm::endianness::native != Old) {
1104 sys::swapByteOrder<uint32_t>(Value&: NumValueSites);
1105 sys::swapByteOrder<uint32_t>(Value&: Kind);
1106 }
1107 uint32_t ND = getValueProfRecordNumValueData(This: this);
1108 InstrProfValueData *VD = getValueProfRecordValueData(This: this);
1109
1110 // No need to swap byte array: SiteCountArrray.
1111 for (uint32_t I = 0; I < ND; I++) {
1112 sys::swapByteOrder<uint64_t>(Value&: VD[I].Value);
1113 sys::swapByteOrder<uint64_t>(Value&: VD[I].Count);
1114 }
1115 if (llvm::endianness::native == Old) {
1116 sys::swapByteOrder<uint32_t>(Value&: NumValueSites);
1117 sys::swapByteOrder<uint32_t>(Value&: Kind);
1118 }
1119}
1120
1121void ValueProfData::deserializeTo(InstrProfRecord &Record,
1122 InstrProfSymtab *SymTab) {
1123 if (NumValueKinds == 0)
1124 return;
1125
1126 ValueProfRecord *VR = getFirstValueProfRecord(This: this);
1127 for (uint32_t K = 0; K < NumValueKinds; K++) {
1128 VR->deserializeTo(Record, SymTab);
1129 VR = getValueProfRecordNext(This: VR);
1130 }
1131}
1132
1133template <class T>
1134static T swapToHostOrder(const unsigned char *&D, llvm::endianness Orig) {
1135 using namespace support;
1136
1137 if (Orig == llvm::endianness::little)
1138 return endian::readNext<T, llvm::endianness::little>(D);
1139 else
1140 return endian::readNext<T, llvm::endianness::big>(D);
1141}
1142
1143static std::unique_ptr<ValueProfData> allocValueProfData(uint32_t TotalSize) {
1144 return std::unique_ptr<ValueProfData>(new (::operator new(TotalSize))
1145 ValueProfData());
1146}
1147
1148Error ValueProfData::checkIntegrity() {
1149 if (NumValueKinds > IPVK_Last + 1)
1150 return make_error<InstrProfError>(
1151 Args: instrprof_error::malformed, Args: "number of value profile kinds is invalid");
1152 // Total size needs to be multiple of quadword size.
1153 if (TotalSize % sizeof(uint64_t))
1154 return make_error<InstrProfError>(
1155 Args: instrprof_error::malformed, Args: "total size is not multiples of quardword");
1156
1157 ValueProfRecord *VR = getFirstValueProfRecord(This: this);
1158 for (uint32_t K = 0; K < this->NumValueKinds; K++) {
1159 if (VR->Kind > IPVK_Last)
1160 return make_error<InstrProfError>(Args: instrprof_error::malformed,
1161 Args: "value kind is invalid");
1162 VR = getValueProfRecordNext(This: VR);
1163 if ((char *)VR - (char *)this > (ptrdiff_t)TotalSize)
1164 return make_error<InstrProfError>(
1165 Args: instrprof_error::malformed,
1166 Args: "value profile address is greater than total size");
1167 }
1168 return Error::success();
1169}
1170
1171Expected<std::unique_ptr<ValueProfData>>
1172ValueProfData::getValueProfData(const unsigned char *D,
1173 const unsigned char *const BufferEnd,
1174 llvm::endianness Endianness) {
1175 using namespace support;
1176
1177 if (D + sizeof(ValueProfData) > BufferEnd)
1178 return make_error<InstrProfError>(Args: instrprof_error::truncated);
1179
1180 const unsigned char *Header = D;
1181 uint32_t TotalSize = swapToHostOrder<uint32_t>(D&: Header, Orig: Endianness);
1182 if (D + TotalSize > BufferEnd)
1183 return make_error<InstrProfError>(Args: instrprof_error::too_large);
1184
1185 std::unique_ptr<ValueProfData> VPD = allocValueProfData(TotalSize);
1186 memcpy(dest: VPD.get(), src: D, n: TotalSize);
1187 // Byte swap.
1188 VPD->swapBytesToHost(Endianness);
1189
1190 Error E = VPD->checkIntegrity();
1191 if (E)
1192 return std::move(E);
1193
1194 return std::move(VPD);
1195}
1196
1197void ValueProfData::swapBytesToHost(llvm::endianness Endianness) {
1198 using namespace support;
1199
1200 if (Endianness == llvm::endianness::native)
1201 return;
1202
1203 sys::swapByteOrder<uint32_t>(Value&: TotalSize);
1204 sys::swapByteOrder<uint32_t>(Value&: NumValueKinds);
1205
1206 ValueProfRecord *VR = getFirstValueProfRecord(This: this);
1207 for (uint32_t K = 0; K < NumValueKinds; K++) {
1208 VR->swapBytes(Old: Endianness, New: llvm::endianness::native);
1209 VR = getValueProfRecordNext(This: VR);
1210 }
1211}
1212
1213void ValueProfData::swapBytesFromHost(llvm::endianness Endianness) {
1214 using namespace support;
1215
1216 if (Endianness == llvm::endianness::native)
1217 return;
1218
1219 ValueProfRecord *VR = getFirstValueProfRecord(This: this);
1220 for (uint32_t K = 0; K < NumValueKinds; K++) {
1221 ValueProfRecord *NVR = getValueProfRecordNext(This: VR);
1222 VR->swapBytes(Old: llvm::endianness::native, New: Endianness);
1223 VR = NVR;
1224 }
1225 sys::swapByteOrder<uint32_t>(Value&: TotalSize);
1226 sys::swapByteOrder<uint32_t>(Value&: NumValueKinds);
1227}
1228
1229void annotateValueSite(Module &M, Instruction &Inst,
1230 const InstrProfRecord &InstrProfR,
1231 InstrProfValueKind ValueKind, uint32_t SiteIdx,
1232 uint32_t MaxMDCount) {
1233 uint32_t NV = InstrProfR.getNumValueDataForSite(ValueKind, Site: SiteIdx);
1234 if (!NV)
1235 return;
1236
1237 uint64_t Sum = 0;
1238 std::unique_ptr<InstrProfValueData[]> VD =
1239 InstrProfR.getValueForSite(ValueKind, Site: SiteIdx, TotalC: &Sum);
1240
1241 ArrayRef<InstrProfValueData> VDs(VD.get(), NV);
1242 annotateValueSite(M, Inst, VDs, Sum, ValueKind, MaxMDCount);
1243}
1244
1245void annotateValueSite(Module &M, Instruction &Inst,
1246 ArrayRef<InstrProfValueData> VDs,
1247 uint64_t Sum, InstrProfValueKind ValueKind,
1248 uint32_t MaxMDCount) {
1249 if (VDs.empty())
1250 return;
1251 LLVMContext &Ctx = M.getContext();
1252 MDBuilder MDHelper(Ctx);
1253 SmallVector<Metadata *, 3> Vals;
1254 // Tag
1255 Vals.push_back(Elt: MDHelper.createString(Str: "VP"));
1256 // Value Kind
1257 Vals.push_back(Elt: MDHelper.createConstant(
1258 C: ConstantInt::get(Ty: Type::getInt32Ty(C&: Ctx), V: ValueKind)));
1259 // Total Count
1260 Vals.push_back(
1261 Elt: MDHelper.createConstant(C: ConstantInt::get(Ty: Type::getInt64Ty(C&: Ctx), V: Sum)));
1262
1263 // Value Profile Data
1264 uint32_t MDCount = MaxMDCount;
1265 for (auto &VD : VDs) {
1266 Vals.push_back(Elt: MDHelper.createConstant(
1267 C: ConstantInt::get(Ty: Type::getInt64Ty(C&: Ctx), V: VD.Value)));
1268 Vals.push_back(Elt: MDHelper.createConstant(
1269 C: ConstantInt::get(Ty: Type::getInt64Ty(C&: Ctx), V: VD.Count)));
1270 if (--MDCount == 0)
1271 break;
1272 }
1273 Inst.setMetadata(KindID: LLVMContext::MD_prof, Node: MDNode::get(Context&: Ctx, MDs: Vals));
1274}
1275
1276MDNode *mayHaveValueProfileOfKind(const Instruction &Inst,
1277 InstrProfValueKind ValueKind) {
1278 MDNode *MD = Inst.getMetadata(KindID: LLVMContext::MD_prof);
1279 if (!MD)
1280 return nullptr;
1281
1282 if (MD->getNumOperands() < 5)
1283 return nullptr;
1284
1285 MDString *Tag = cast<MDString>(Val: MD->getOperand(I: 0));
1286 if (!Tag || !Tag->getString().equals(RHS: "VP"))
1287 return nullptr;
1288
1289 // Now check kind:
1290 ConstantInt *KindInt = mdconst::dyn_extract<ConstantInt>(MD: MD->getOperand(I: 1));
1291 if (!KindInt)
1292 return nullptr;
1293 if (KindInt->getZExtValue() != ValueKind)
1294 return nullptr;
1295
1296 return MD;
1297}
1298
1299static bool getValueProfDataFromInstImpl(const MDNode *const MD,
1300 const uint32_t MaxNumDataWant,
1301 InstrProfValueData ValueData[],
1302 uint32_t &ActualNumValueData,
1303 uint64_t &TotalC, bool GetNoICPValue) {
1304 const unsigned NOps = MD->getNumOperands();
1305 // Get total count
1306 ConstantInt *TotalCInt = mdconst::dyn_extract<ConstantInt>(MD: MD->getOperand(I: 2));
1307 if (!TotalCInt)
1308 return false;
1309 TotalC = TotalCInt->getZExtValue();
1310 ActualNumValueData = 0;
1311
1312 for (unsigned I = 3; I < NOps; I += 2) {
1313 if (ActualNumValueData >= MaxNumDataWant)
1314 break;
1315 ConstantInt *Value = mdconst::dyn_extract<ConstantInt>(MD: MD->getOperand(I));
1316 ConstantInt *Count =
1317 mdconst::dyn_extract<ConstantInt>(MD: MD->getOperand(I: I + 1));
1318 if (!Value || !Count)
1319 return false;
1320 uint64_t CntValue = Count->getZExtValue();
1321 if (!GetNoICPValue && (CntValue == NOMORE_ICP_MAGICNUM))
1322 continue;
1323 ValueData[ActualNumValueData].Value = Value->getZExtValue();
1324 ValueData[ActualNumValueData].Count = CntValue;
1325 ActualNumValueData++;
1326 }
1327 return true;
1328}
1329
1330std::unique_ptr<InstrProfValueData[]>
1331getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind,
1332 uint32_t MaxNumValueData, uint32_t &ActualNumValueData,
1333 uint64_t &TotalC, bool GetNoICPValue) {
1334 MDNode *MD = mayHaveValueProfileOfKind(Inst, ValueKind);
1335 if (!MD)
1336 return nullptr;
1337 auto ValueDataArray = std::make_unique<InstrProfValueData[]>(num: MaxNumValueData);
1338 if (!getValueProfDataFromInstImpl(MD, MaxNumDataWant: MaxNumValueData, ValueData: ValueDataArray.get(),
1339 ActualNumValueData, TotalC, GetNoICPValue))
1340 return nullptr;
1341 return ValueDataArray;
1342}
1343
1344// FIXME: Migrate existing callers to the function above that returns an
1345// array.
1346bool getValueProfDataFromInst(const Instruction &Inst,
1347 InstrProfValueKind ValueKind,
1348 uint32_t MaxNumValueData,
1349 InstrProfValueData ValueData[],
1350 uint32_t &ActualNumValueData, uint64_t &TotalC,
1351 bool GetNoICPValue) {
1352 MDNode *MD = mayHaveValueProfileOfKind(Inst, ValueKind);
1353 if (!MD)
1354 return false;
1355 return getValueProfDataFromInstImpl(MD, MaxNumDataWant: MaxNumValueData, ValueData,
1356 ActualNumValueData, TotalC,
1357 GetNoICPValue);
1358}
1359
1360MDNode *getPGOFuncNameMetadata(const Function &F) {
1361 return F.getMetadata(Kind: getPGOFuncNameMetadataName());
1362}
1363
1364void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName) {
1365 // Only for internal linkage functions.
1366 if (PGOFuncName == F.getName())
1367 return;
1368 // Don't create duplicated meta-data.
1369 if (getPGOFuncNameMetadata(F))
1370 return;
1371 LLVMContext &C = F.getContext();
1372 MDNode *N = MDNode::get(Context&: C, MDs: MDString::get(Context&: C, Str: PGOFuncName));
1373 F.setMetadata(Kind: getPGOFuncNameMetadataName(), Node: N);
1374}
1375
1376bool needsComdatForCounter(const GlobalObject &GO, const Module &M) {
1377 if (GO.hasComdat())
1378 return true;
1379
1380 if (!Triple(M.getTargetTriple()).supportsCOMDAT())
1381 return false;
1382
1383 // See createPGOFuncNameVar for more details. To avoid link errors, profile
1384 // counters for function with available_externally linkage needs to be changed
1385 // to linkonce linkage. On ELF based systems, this leads to weak symbols to be
1386 // created. Without using comdat, duplicate entries won't be removed by the
1387 // linker leading to increased data segement size and raw profile size. Even
1388 // worse, since the referenced counter from profile per-function data object
1389 // will be resolved to the common strong definition, the profile counts for
1390 // available_externally functions will end up being duplicated in raw profile
1391 // data. This can result in distorted profile as the counts of those dups
1392 // will be accumulated by the profile merger.
1393 GlobalValue::LinkageTypes Linkage = GO.getLinkage();
1394 if (Linkage != GlobalValue::ExternalWeakLinkage &&
1395 Linkage != GlobalValue::AvailableExternallyLinkage)
1396 return false;
1397
1398 return true;
1399}
1400
1401// Check if INSTR_PROF_RAW_VERSION_VAR is defined.
1402bool isIRPGOFlagSet(const Module *M) {
1403 auto IRInstrVar =
1404 M->getNamedGlobal(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR));
1405 if (!IRInstrVar || IRInstrVar->hasLocalLinkage())
1406 return false;
1407
1408 // For CSPGO+LTO, this variable might be marked as non-prevailing and we only
1409 // have the decl.
1410 if (IRInstrVar->isDeclaration())
1411 return true;
1412
1413 // Check if the flag is set.
1414 if (!IRInstrVar->hasInitializer())
1415 return false;
1416
1417 auto *InitVal = dyn_cast_or_null<ConstantInt>(Val: IRInstrVar->getInitializer());
1418 if (!InitVal)
1419 return false;
1420 return (InitVal->getZExtValue() & VARIANT_MASK_IR_PROF) != 0;
1421}
1422
1423// Check if we can safely rename this Comdat function.
1424bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken) {
1425 if (F.getName().empty())
1426 return false;
1427 if (!needsComdatForCounter(GO: F, M: *(F.getParent())))
1428 return false;
1429 // Unsafe to rename the address-taken function (which can be used in
1430 // function comparison).
1431 if (CheckAddressTaken && F.hasAddressTaken())
1432 return false;
1433 // Only safe to do if this function may be discarded if it is not used
1434 // in the compilation unit.
1435 if (!GlobalValue::isDiscardableIfUnused(Linkage: F.getLinkage()))
1436 return false;
1437
1438 // For AvailableExternallyLinkage functions.
1439 if (!F.hasComdat()) {
1440 assert(F.getLinkage() == GlobalValue::AvailableExternallyLinkage);
1441 return true;
1442 }
1443 return true;
1444}
1445
1446// Create the variable for the profile file name.
1447void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput) {
1448 if (InstrProfileOutput.empty())
1449 return;
1450 Constant *ProfileNameConst =
1451 ConstantDataArray::getString(Context&: M.getContext(), Initializer: InstrProfileOutput, AddNull: true);
1452 GlobalVariable *ProfileNameVar = new GlobalVariable(
1453 M, ProfileNameConst->getType(), true, GlobalValue::WeakAnyLinkage,
1454 ProfileNameConst, INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR));
1455 ProfileNameVar->setVisibility(GlobalValue::HiddenVisibility);
1456 Triple TT(M.getTargetTriple());
1457 if (TT.supportsCOMDAT()) {
1458 ProfileNameVar->setLinkage(GlobalValue::ExternalLinkage);
1459 ProfileNameVar->setComdat(M.getOrInsertComdat(
1460 Name: StringRef(INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR))));
1461 }
1462}
1463
1464Error OverlapStats::accumulateCounts(const std::string &BaseFilename,
1465 const std::string &TestFilename,
1466 bool IsCS) {
1467 auto getProfileSum = [IsCS](const std::string &Filename,
1468 CountSumOrPercent &Sum) -> Error {
1469 // This function is only used from llvm-profdata that doesn't use any kind
1470 // of VFS. Just create a default RealFileSystem to read profiles.
1471 auto FS = vfs::getRealFileSystem();
1472 auto ReaderOrErr = InstrProfReader::create(Path: Filename, FS&: *FS);
1473 if (Error E = ReaderOrErr.takeError()) {
1474 return E;
1475 }
1476 auto Reader = std::move(ReaderOrErr.get());
1477 Reader->accumulateCounts(Sum, IsCS);
1478 return Error::success();
1479 };
1480 auto Ret = getProfileSum(BaseFilename, Base);
1481 if (Ret)
1482 return Ret;
1483 Ret = getProfileSum(TestFilename, Test);
1484 if (Ret)
1485 return Ret;
1486 this->BaseFilename = &BaseFilename;
1487 this->TestFilename = &TestFilename;
1488 Valid = true;
1489 return Error::success();
1490}
1491
1492void OverlapStats::addOneMismatch(const CountSumOrPercent &MismatchFunc) {
1493 Mismatch.NumEntries += 1;
1494 Mismatch.CountSum += MismatchFunc.CountSum / Test.CountSum;
1495 for (unsigned I = 0; I < IPVK_Last - IPVK_First + 1; I++) {
1496 if (Test.ValueCounts[I] >= 1.0f)
1497 Mismatch.ValueCounts[I] +=
1498 MismatchFunc.ValueCounts[I] / Test.ValueCounts[I];
1499 }
1500}
1501
1502void OverlapStats::addOneUnique(const CountSumOrPercent &UniqueFunc) {
1503 Unique.NumEntries += 1;
1504 Unique.CountSum += UniqueFunc.CountSum / Test.CountSum;
1505 for (unsigned I = 0; I < IPVK_Last - IPVK_First + 1; I++) {
1506 if (Test.ValueCounts[I] >= 1.0f)
1507 Unique.ValueCounts[I] += UniqueFunc.ValueCounts[I] / Test.ValueCounts[I];
1508 }
1509}
1510
1511void OverlapStats::dump(raw_fd_ostream &OS) const {
1512 if (!Valid)
1513 return;
1514
1515 const char *EntryName =
1516 (Level == ProgramLevel ? "functions" : "edge counters");
1517 if (Level == ProgramLevel) {
1518 OS << "Profile overlap infomation for base_profile: " << *BaseFilename
1519 << " and test_profile: " << *TestFilename << "\nProgram level:\n";
1520 } else {
1521 OS << "Function level:\n"
1522 << " Function: " << FuncName << " (Hash=" << FuncHash << ")\n";
1523 }
1524
1525 OS << " # of " << EntryName << " overlap: " << Overlap.NumEntries << "\n";
1526 if (Mismatch.NumEntries)
1527 OS << " # of " << EntryName << " mismatch: " << Mismatch.NumEntries
1528 << "\n";
1529 if (Unique.NumEntries)
1530 OS << " # of " << EntryName
1531 << " only in test_profile: " << Unique.NumEntries << "\n";
1532
1533 OS << " Edge profile overlap: " << format(Fmt: "%.3f%%", Vals: Overlap.CountSum * 100)
1534 << "\n";
1535 if (Mismatch.NumEntries)
1536 OS << " Mismatched count percentage (Edge): "
1537 << format(Fmt: "%.3f%%", Vals: Mismatch.CountSum * 100) << "\n";
1538 if (Unique.NumEntries)
1539 OS << " Percentage of Edge profile only in test_profile: "
1540 << format(Fmt: "%.3f%%", Vals: Unique.CountSum * 100) << "\n";
1541 OS << " Edge profile base count sum: " << format(Fmt: "%.0f", Vals: Base.CountSum)
1542 << "\n"
1543 << " Edge profile test count sum: " << format(Fmt: "%.0f", Vals: Test.CountSum)
1544 << "\n";
1545
1546 for (unsigned I = 0; I < IPVK_Last - IPVK_First + 1; I++) {
1547 if (Base.ValueCounts[I] < 1.0f && Test.ValueCounts[I] < 1.0f)
1548 continue;
1549 char ProfileKindName[20] = {0};
1550 switch (I) {
1551 case IPVK_IndirectCallTarget:
1552 strncpy(dest: ProfileKindName, src: "IndirectCall", n: 19);
1553 break;
1554 case IPVK_MemOPSize:
1555 strncpy(dest: ProfileKindName, src: "MemOP", n: 19);
1556 break;
1557 case IPVK_VTableTarget:
1558 strncpy(dest: ProfileKindName, src: "VTable", n: 19);
1559 break;
1560 default:
1561 snprintf(s: ProfileKindName, maxlen: 19, format: "VP[%d]", I);
1562 break;
1563 }
1564 OS << " " << ProfileKindName
1565 << " profile overlap: " << format(Fmt: "%.3f%%", Vals: Overlap.ValueCounts[I] * 100)
1566 << "\n";
1567 if (Mismatch.NumEntries)
1568 OS << " Mismatched count percentage (" << ProfileKindName
1569 << "): " << format(Fmt: "%.3f%%", Vals: Mismatch.ValueCounts[I] * 100) << "\n";
1570 if (Unique.NumEntries)
1571 OS << " Percentage of " << ProfileKindName
1572 << " profile only in test_profile: "
1573 << format(Fmt: "%.3f%%", Vals: Unique.ValueCounts[I] * 100) << "\n";
1574 OS << " " << ProfileKindName
1575 << " profile base count sum: " << format(Fmt: "%.0f", Vals: Base.ValueCounts[I])
1576 << "\n"
1577 << " " << ProfileKindName
1578 << " profile test count sum: " << format(Fmt: "%.0f", Vals: Test.ValueCounts[I])
1579 << "\n";
1580 }
1581}
1582
1583namespace IndexedInstrProf {
1584// A C++14 compatible version of the offsetof macro.
1585template <typename T1, typename T2>
1586inline size_t constexpr offsetOf(T1 T2::*Member) {
1587 constexpr T2 Object{};
1588 return size_t(&(Object.*Member)) - size_t(&Object);
1589}
1590
1591static inline uint64_t read(const unsigned char *Buffer, size_t Offset) {
1592 return *reinterpret_cast<const uint64_t *>(Buffer + Offset);
1593}
1594
1595uint64_t Header::formatVersion() const {
1596 using namespace support;
1597 return endian::byte_swap<uint64_t, llvm::endianness::little>(value: Version);
1598}
1599
1600Expected<Header> Header::readFromBuffer(const unsigned char *Buffer) {
1601 using namespace support;
1602 static_assert(std::is_standard_layout_v<Header>,
1603 "The header should be standard layout type since we use offset "
1604 "of fields to read.");
1605 Header H;
1606
1607 H.Magic = read(Buffer, Offset: offsetOf(Member: &Header::Magic));
1608 // Check the magic number.
1609 uint64_t Magic =
1610 endian::byte_swap<uint64_t, llvm::endianness::little>(value: H.Magic);
1611 if (Magic != IndexedInstrProf::Magic)
1612 return make_error<InstrProfError>(Args: instrprof_error::bad_magic);
1613
1614 // Read the version.
1615 H.Version = read(Buffer, Offset: offsetOf(Member: &Header::Version));
1616 if (GET_VERSION(H.formatVersion()) >
1617 IndexedInstrProf::ProfVersion::CurrentVersion)
1618 return make_error<InstrProfError>(Args: instrprof_error::unsupported_version);
1619
1620 switch (GET_VERSION(H.formatVersion())) {
1621 // When a new field is added in the header add a case statement here to
1622 // populate it.
1623 static_assert(
1624 IndexedInstrProf::ProfVersion::CurrentVersion == Version12,
1625 "Please update the reading code below if a new field has been added, "
1626 "if not add a case statement to fall through to the latest version.");
1627 case 12ull:
1628 H.VTableNamesOffset = read(Buffer, Offset: offsetOf(Member: &Header::VTableNamesOffset));
1629 [[fallthrough]];
1630 case 11ull:
1631 [[fallthrough]];
1632 case 10ull:
1633 H.TemporalProfTracesOffset =
1634 read(Buffer, Offset: offsetOf(Member: &Header::TemporalProfTracesOffset));
1635 [[fallthrough]];
1636 case 9ull:
1637 H.BinaryIdOffset = read(Buffer, Offset: offsetOf(Member: &Header::BinaryIdOffset));
1638 [[fallthrough]];
1639 case 8ull:
1640 H.MemProfOffset = read(Buffer, Offset: offsetOf(Member: &Header::MemProfOffset));
1641 [[fallthrough]];
1642 default: // Version7 (when the backwards compatible header was introduced).
1643 H.HashType = read(Buffer, Offset: offsetOf(Member: &Header::HashType));
1644 H.HashOffset = read(Buffer, Offset: offsetOf(Member: &Header::HashOffset));
1645 }
1646
1647 return H;
1648}
1649
1650size_t Header::size() const {
1651 switch (GET_VERSION(formatVersion())) {
1652 // When a new field is added to the header add a case statement here to
1653 // compute the size as offset of the new field + size of the new field. This
1654 // relies on the field being added to the end of the list.
1655 static_assert(IndexedInstrProf::ProfVersion::CurrentVersion == Version12,
1656 "Please update the size computation below if a new field has "
1657 "been added to the header, if not add a case statement to "
1658 "fall through to the latest version.");
1659 case 12ull:
1660 return offsetOf(Member: &Header::VTableNamesOffset) +
1661 sizeof(Header::VTableNamesOffset);
1662 case 11ull:
1663 [[fallthrough]];
1664 case 10ull:
1665 return offsetOf(Member: &Header::TemporalProfTracesOffset) +
1666 sizeof(Header::TemporalProfTracesOffset);
1667 case 9ull:
1668 return offsetOf(Member: &Header::BinaryIdOffset) + sizeof(Header::BinaryIdOffset);
1669 case 8ull:
1670 return offsetOf(Member: &Header::MemProfOffset) + sizeof(Header::MemProfOffset);
1671 default: // Version7 (when the backwards compatible header was introduced).
1672 return offsetOf(Member: &Header::HashOffset) + sizeof(Header::HashOffset);
1673 }
1674}
1675
1676} // namespace IndexedInstrProf
1677
1678} // end namespace llvm
1679

source code of llvm/lib/ProfileData/InstrProf.cpp