1//===- InstrProf.h - Instrumented profiling format support ------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Instrumentation-based profiling data is generated by instrumented
10// binaries through library functions in compiler-rt, and read by the clang
11// frontend to feed PGO.
12//
13//===----------------------------------------------------------------------===//
14
15#ifndef LLVM_PROFILEDATA_INSTRPROF_H
16#define LLVM_PROFILEDATA_INSTRPROF_H
17
18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/BitmaskEnum.h"
20#include "llvm/ADT/IntervalMap.h"
21#include "llvm/ADT/STLExtras.h"
22#include "llvm/ADT/StringRef.h"
23#include "llvm/ADT/StringSet.h"
24#include "llvm/IR/GlobalValue.h"
25#include "llvm/IR/ProfileSummary.h"
26#include "llvm/ProfileData/InstrProfData.inc"
27#include "llvm/Support/BalancedPartitioning.h"
28#include "llvm/Support/CommandLine.h"
29#include "llvm/Support/Compiler.h"
30#include "llvm/Support/Error.h"
31#include "llvm/Support/ErrorHandling.h"
32#include "llvm/Support/MD5.h"
33#include "llvm/Support/MathExtras.h"
34#include "llvm/Support/raw_ostream.h"
35#include "llvm/TargetParser/Host.h"
36#include "llvm/TargetParser/Triple.h"
37#include <algorithm>
38#include <cassert>
39#include <cstddef>
40#include <cstdint>
41#include <cstring>
42#include <list>
43#include <memory>
44#include <string>
45#include <system_error>
46#include <utility>
47#include <vector>
48
49namespace llvm {
50
51class Function;
52class GlobalVariable;
53struct InstrProfRecord;
54class InstrProfSymtab;
55class Instruction;
56class MDNode;
57class Module;
58
59enum InstrProfSectKind {
60#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) Kind,
61#include "llvm/ProfileData/InstrProfData.inc"
62};
63
64/// Return the max count value. We reserver a few large values for special use.
65inline uint64_t getInstrMaxCountValue() {
66 return std::numeric_limits<uint64_t>::max() - 2;
67}
68
69/// Return the name of the profile section corresponding to \p IPSK.
70///
71/// The name of the section depends on the object format type \p OF. If
72/// \p AddSegmentInfo is true, a segment prefix and additional linker hints may
73/// be added to the section name (this is the default).
74std::string getInstrProfSectionName(InstrProfSectKind IPSK,
75 Triple::ObjectFormatType OF,
76 bool AddSegmentInfo = true);
77
78/// Return the name profile runtime entry point to do value profiling
79/// for a given site.
80inline StringRef getInstrProfValueProfFuncName() {
81 return INSTR_PROF_VALUE_PROF_FUNC_STR;
82}
83
84/// Return the name profile runtime entry point to do memop size value
85/// profiling.
86inline StringRef getInstrProfValueProfMemOpFuncName() {
87 return INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR;
88}
89
90/// Return the name prefix of variables containing instrumented function names.
91inline StringRef getInstrProfNameVarPrefix() { return "__profn_"; }
92
93/// Return the name prefix of variables containing virtual table profile data.
94inline StringRef getInstrProfVTableVarPrefix() { return "__profvt_"; }
95
96/// Return the name prefix of variables containing per-function control data.
97inline StringRef getInstrProfDataVarPrefix() { return "__profd_"; }
98
99/// Return the name prefix of profile counter variables.
100inline StringRef getInstrProfCountersVarPrefix() { return "__profc_"; }
101
102/// Return the name prefix of profile bitmap variables.
103inline StringRef getInstrProfBitmapVarPrefix() { return "__profbm_"; }
104
105/// Return the name prefix of value profile variables.
106inline StringRef getInstrProfValuesVarPrefix() { return "__profvp_"; }
107
108/// Return the name of value profile node array variables:
109inline StringRef getInstrProfVNodesVarName() { return "__llvm_prf_vnodes"; }
110
111/// Return the name of the variable holding the strings (possibly compressed)
112/// of all function's PGO names.
113inline StringRef getInstrProfNamesVarName() { return "__llvm_prf_nm"; }
114
115inline StringRef getInstrProfVTableNamesVarName() { return "__llvm_prf_vnm"; }
116
117/// Return the name of a covarage mapping variable (internal linkage)
118/// for each instrumented source module. Such variables are allocated
119/// in the __llvm_covmap section.
120inline StringRef getCoverageMappingVarName() {
121 return "__llvm_coverage_mapping";
122}
123
124/// Return the name of the internal variable recording the array
125/// of PGO name vars referenced by the coverage mapping. The owning
126/// functions of those names are not emitted by FE (e.g, unused inline
127/// functions.)
128inline StringRef getCoverageUnusedNamesVarName() {
129 return "__llvm_coverage_names";
130}
131
132/// Return the name of function that registers all the per-function control
133/// data at program startup time by calling __llvm_register_function. This
134/// function has internal linkage and is called by __llvm_profile_init
135/// runtime method. This function is not generated for these platforms:
136/// Darwin, Linux, and FreeBSD.
137inline StringRef getInstrProfRegFuncsName() {
138 return "__llvm_profile_register_functions";
139}
140
141/// Return the name of the runtime interface that registers per-function control
142/// data for one instrumented function.
143inline StringRef getInstrProfRegFuncName() {
144 return "__llvm_profile_register_function";
145}
146
147/// Return the name of the runtime interface that registers the PGO name
148/// strings.
149inline StringRef getInstrProfNamesRegFuncName() {
150 return "__llvm_profile_register_names_function";
151}
152
153/// Return the name of the runtime initialization method that is generated by
154/// the compiler. The function calls __llvm_profile_register_functions and
155/// __llvm_profile_override_default_filename functions if needed. This function
156/// has internal linkage and invoked at startup time via init_array.
157inline StringRef getInstrProfInitFuncName() { return "__llvm_profile_init"; }
158
159/// Return the name of the hook variable defined in profile runtime library.
160/// A reference to the variable causes the linker to link in the runtime
161/// initialization module (which defines the hook variable).
162inline StringRef getInstrProfRuntimeHookVarName() {
163 return INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_RUNTIME_VAR);
164}
165
166/// Return the name of the compiler generated function that references the
167/// runtime hook variable. The function is a weak global.
168inline StringRef getInstrProfRuntimeHookVarUseFuncName() {
169 return "__llvm_profile_runtime_user";
170}
171
172inline StringRef getInstrProfCounterBiasVarName() {
173 return INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_COUNTER_BIAS_VAR);
174}
175
176/// Return the marker used to separate PGO names during serialization.
177inline StringRef getInstrProfNameSeparator() { return "\01"; }
178
179/// Please use getIRPGOFuncName for LLVM IR instrumentation. This function is
180/// for front-end (Clang, etc) instrumentation.
181/// Return the modified name for function \c F suitable to be
182/// used the key for profile lookup. Variable \c InLTO indicates if this
183/// is called in LTO optimization passes.
184std::string getPGOFuncName(const Function &F, bool InLTO = false,
185 uint64_t Version = INSTR_PROF_INDEX_VERSION);
186
187/// Return the modified name for a function suitable to be
188/// used the key for profile lookup. The function's original
189/// name is \c RawFuncName and has linkage of type \c Linkage.
190/// The function is defined in module \c FileName.
191std::string getPGOFuncName(StringRef RawFuncName,
192 GlobalValue::LinkageTypes Linkage,
193 StringRef FileName,
194 uint64_t Version = INSTR_PROF_INDEX_VERSION);
195
196/// \return the modified name for function \c F suitable to be
197/// used as the key for IRPGO profile lookup. \c InLTO indicates if this is
198/// called from LTO optimization passes.
199std::string getIRPGOFuncName(const Function &F, bool InLTO = false);
200
201/// \return the filename and the function name parsed from the output of
202/// \c getIRPGOFuncName()
203std::pair<StringRef, StringRef> getParsedIRPGOName(StringRef IRPGOName);
204
205/// Return the name of the global variable used to store a function
206/// name in PGO instrumentation. \c FuncName is the IRPGO function name
207/// (returned by \c getIRPGOFuncName) for LLVM IR instrumentation and PGO
208/// function name (returned by \c getPGOFuncName) for front-end instrumentation.
209std::string getPGOFuncNameVarName(StringRef FuncName,
210 GlobalValue::LinkageTypes Linkage);
211
212/// Create and return the global variable for function name used in PGO
213/// instrumentation. \c FuncName is the IRPGO function name (returned by
214/// \c getIRPGOFuncName) for LLVM IR instrumentation and PGO function name
215/// (returned by \c getPGOFuncName) for front-end instrumentation.
216GlobalVariable *createPGOFuncNameVar(Function &F, StringRef PGOFuncName);
217
218/// Create and return the global variable for function name used in PGO
219/// instrumentation. \c FuncName is the IRPGO function name (returned by
220/// \c getIRPGOFuncName) for LLVM IR instrumentation and PGO function name
221/// (returned by \c getPGOFuncName) for front-end instrumentation.
222GlobalVariable *createPGOFuncNameVar(Module &M,
223 GlobalValue::LinkageTypes Linkage,
224 StringRef PGOFuncName);
225
226/// Return the initializer in string of the PGO name var \c NameVar.
227StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar);
228
229/// Given a PGO function name, remove the filename prefix and return
230/// the original (static) function name.
231StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName,
232 StringRef FileName = "<unknown>");
233
234/// Given a vector of strings (names of global objects like functions or,
235/// virtual tables) \c NameStrs, the method generates a combined string \c
236/// Result that is ready to be serialized. The \c Result string is comprised of
237/// three fields: The first field is the length of the uncompressed strings, and
238/// the the second field is the length of the zlib-compressed string. Both
239/// fields are encoded in ULEB128. If \c doCompress is false, the
240/// third field is the uncompressed strings; otherwise it is the
241/// compressed string. When the string compression is off, the
242/// second field will have value zero.
243Error collectGlobalObjectNameStrings(ArrayRef<std::string> NameStrs,
244 bool doCompression, std::string &Result);
245
246/// Produce \c Result string with the same format described above. The input
247/// is vector of PGO function name variables that are referenced.
248/// The global variable element in 'NameVars' is a string containing the pgo
249/// name of a function. See `createPGOFuncNameVar` that creates these global
250/// variables.
251Error collectPGOFuncNameStrings(ArrayRef<GlobalVariable *> NameVars,
252 std::string &Result, bool doCompression = true);
253
254Error collectVTableStrings(ArrayRef<GlobalVariable *> VTables,
255 std::string &Result, bool doCompression);
256
257/// Check if INSTR_PROF_RAW_VERSION_VAR is defined. This global is only being
258/// set in IR PGO compilation.
259bool isIRPGOFlagSet(const Module *M);
260
261/// Check if we can safely rename this Comdat function. Instances of the same
262/// comdat function may have different control flows thus can not share the
263/// same counter variable.
264bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken = false);
265
266enum InstrProfValueKind : uint32_t {
267#define VALUE_PROF_KIND(Enumerator, Value, Descr) Enumerator = Value,
268#include "llvm/ProfileData/InstrProfData.inc"
269};
270
271/// Get the value profile data for value site \p SiteIdx from \p InstrProfR
272/// and annotate the instruction \p Inst with the value profile meta data.
273/// Annotate up to \p MaxMDCount (default 3) number of records per value site.
274void annotateValueSite(Module &M, Instruction &Inst,
275 const InstrProfRecord &InstrProfR,
276 InstrProfValueKind ValueKind, uint32_t SiteIndx,
277 uint32_t MaxMDCount = 3);
278
279/// Same as the above interface but using an ArrayRef, as well as \p Sum.
280/// This function will not annotate !prof metadata on the instruction if the
281/// referenced array is empty.
282void annotateValueSite(Module &M, Instruction &Inst,
283 ArrayRef<InstrProfValueData> VDs, uint64_t Sum,
284 InstrProfValueKind ValueKind, uint32_t MaxMDCount);
285
286/// Extract the value profile data from \p Inst which is annotated with
287/// value profile meta data. Return false if there is no value data annotated,
288/// otherwise return true.
289bool getValueProfDataFromInst(const Instruction &Inst,
290 InstrProfValueKind ValueKind,
291 uint32_t MaxNumValueData,
292 InstrProfValueData ValueData[],
293 uint32_t &ActualNumValueData, uint64_t &TotalC,
294 bool GetNoICPValue = false);
295
296/// Extract the value profile data from \p Inst and returns them if \p Inst is
297/// annotated with value profile data. Returns nullptr otherwise. It's similar
298/// to `getValueProfDataFromInst` above except that an array is allocated only
299/// after a preliminary checking that the value profiles of kind `ValueKind`
300/// exist.
301std::unique_ptr<InstrProfValueData[]>
302getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind,
303 uint32_t MaxNumValueData, uint32_t &ActualNumValueData,
304 uint64_t &TotalC, bool GetNoICPValue = false);
305
306inline StringRef getPGOFuncNameMetadataName() { return "PGOFuncName"; }
307
308/// Return the PGOFuncName meta data associated with a function.
309MDNode *getPGOFuncNameMetadata(const Function &F);
310
311std::string getPGOName(const GlobalVariable &V, bool InLTO = false);
312
313/// Create the PGOFuncName meta data if PGOFuncName is different from
314/// function's raw name. This should only apply to internal linkage functions
315/// declared by users only.
316void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName);
317
318/// Check if we can use Comdat for profile variables. This will eliminate
319/// the duplicated profile variables for Comdat functions.
320bool needsComdatForCounter(const GlobalObject &GV, const Module &M);
321
322/// An enum describing the attributes of an instrumented profile.
323enum class InstrProfKind {
324 Unknown = 0x0,
325 // A frontend clang profile, incompatible with other attrs.
326 FrontendInstrumentation = 0x1,
327 // An IR-level profile (default when -fprofile-generate is used).
328 IRInstrumentation = 0x2,
329 // A profile with entry basic block instrumentation.
330 FunctionEntryInstrumentation = 0x4,
331 // A context sensitive IR-level profile.
332 ContextSensitive = 0x8,
333 // Use single byte probes for coverage.
334 SingleByteCoverage = 0x10,
335 // Only instrument the function entry basic block.
336 FunctionEntryOnly = 0x20,
337 // A memory profile collected using -fprofile=memory.
338 MemProf = 0x40,
339 // A temporal profile.
340 TemporalProfile = 0x80,
341 LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/TemporalProfile)
342};
343
344const std::error_category &instrprof_category();
345
346enum class instrprof_error {
347 success = 0,
348 eof,
349 unrecognized_format,
350 bad_magic,
351 bad_header,
352 unsupported_version,
353 unsupported_hash_type,
354 too_large,
355 truncated,
356 malformed,
357 missing_correlation_info,
358 unexpected_correlation_info,
359 unable_to_correlate_profile,
360 unknown_function,
361 invalid_prof,
362 hash_mismatch,
363 count_mismatch,
364 bitmap_mismatch,
365 counter_overflow,
366 value_site_count_mismatch,
367 compress_failed,
368 uncompress_failed,
369 empty_raw_profile,
370 zlib_unavailable,
371 raw_profile_version_mismatch,
372 counter_value_too_large,
373};
374
375/// An ordered list of functions identified by their NameRef found in
376/// INSTR_PROF_DATA
377struct TemporalProfTraceTy {
378 std::vector<uint64_t> FunctionNameRefs;
379 uint64_t Weight;
380 TemporalProfTraceTy(std::initializer_list<uint64_t> Trace = {},
381 uint64_t Weight = 1)
382 : FunctionNameRefs(Trace), Weight(Weight) {}
383
384 /// Use a set of temporal profile traces to create a list of balanced
385 /// partitioning function nodes used by BalancedPartitioning to generate a
386 /// function order that reduces page faults during startup
387 static std::vector<BPFunctionNode>
388 createBPFunctionNodes(ArrayRef<TemporalProfTraceTy> Traces);
389};
390
391inline std::error_code make_error_code(instrprof_error E) {
392 return std::error_code(static_cast<int>(E), instrprof_category());
393}
394
395class InstrProfError : public ErrorInfo<InstrProfError> {
396public:
397 InstrProfError(instrprof_error Err, const Twine &ErrStr = Twine())
398 : Err(Err), Msg(ErrStr.str()) {
399 assert(Err != instrprof_error::success && "Not an error");
400 }
401
402 std::string message() const override;
403
404 void log(raw_ostream &OS) const override { OS << message(); }
405
406 std::error_code convertToErrorCode() const override {
407 return make_error_code(E: Err);
408 }
409
410 instrprof_error get() const { return Err; }
411 const std::string &getMessage() const { return Msg; }
412
413 /// Consume an Error and return the raw enum value contained within it, and
414 /// the optional error message. The Error must either be a success value, or
415 /// contain a single InstrProfError.
416 static std::pair<instrprof_error, std::string> take(Error E) {
417 auto Err = instrprof_error::success;
418 std::string Msg = "";
419 handleAllErrors(E: std::move(E), Handlers: [&Err, &Msg](const InstrProfError &IPE) {
420 assert(Err == instrprof_error::success && "Multiple errors encountered");
421 Err = IPE.get();
422 Msg = IPE.getMessage();
423 });
424 return {Err, Msg};
425 }
426
427 static char ID;
428
429private:
430 instrprof_error Err;
431 std::string Msg;
432};
433
434namespace object {
435
436class SectionRef;
437
438} // end namespace object
439
440namespace IndexedInstrProf {
441
442uint64_t ComputeHash(StringRef K);
443
444} // end namespace IndexedInstrProf
445
446/// A symbol table used for function [IR]PGO name look-up with keys
447/// (such as pointers, md5hash values) to the function. A function's
448/// [IR]PGO name or name's md5hash are used in retrieving the profile
449/// data of the function. See \c getIRPGOFuncName() and \c getPGOFuncName
450/// methods for details how [IR]PGO name is formed.
451class InstrProfSymtab {
452public:
453 using AddrHashMap = std::vector<std::pair<uint64_t, uint64_t>>;
454
455private:
456 using AddrIntervalMap =
457 IntervalMap<uint64_t, uint64_t, 4, IntervalMapHalfOpenInfo<uint64_t>>;
458 StringRef Data;
459 uint64_t Address = 0;
460 // Unique name strings. Used to ensure entries in MD5NameMap (a vector that's
461 // going to be sorted) has unique MD5 keys in the first place.
462 StringSet<> NameTab;
463 // Records the unique virtual table names. This is used by InstrProfWriter to
464 // write out an on-disk chained hash table of virtual table names.
465 // InstrProfWriter stores per function profile data (keyed by function names)
466 // so it doesn't use a StringSet for function names.
467 StringSet<> VTableNames;
468 // A map from MD5 keys to function name strings.
469 std::vector<std::pair<uint64_t, StringRef>> MD5NameMap;
470 // A map from MD5 keys to function define. We only populate this map
471 // when build the Symtab from a Module.
472 std::vector<std::pair<uint64_t, Function *>> MD5FuncMap;
473 // A map from function runtime address to function name MD5 hash.
474 // This map is only populated and used by raw instr profile reader.
475 AddrHashMap AddrToMD5Map;
476
477 AddrIntervalMap::Allocator VTableAddrMapAllocator;
478 // This map is only populated and used by raw instr profile reader.
479 AddrIntervalMap VTableAddrMap;
480 bool Sorted = false;
481
482 static StringRef getExternalSymbol() { return "** External Symbol **"; }
483
484 // Returns the canonial name of the given PGOName. In a canonical name, all
485 // suffixes that begins with "." except ".__uniq." are stripped.
486 // FIXME: Unify this with `FunctionSamples::getCanonicalFnName`.
487 static StringRef getCanonicalName(StringRef PGOName);
488
489 // Add the function into the symbol table, by creating the following
490 // map entries:
491 // name-set = {PGOFuncName} + {getCanonicalName(PGOFuncName)} if the canonical
492 // name is different from pgo name
493 // - In MD5NameMap: <MD5Hash(name), name> for name in name-set
494 // - In MD5FuncMap: <MD5Hash(name), &F> for name in name-set
495 Error addFuncWithName(Function &F, StringRef PGOFuncName);
496
497 // If the symtab is created by a series of calls to \c addFuncName, \c
498 // finalizeSymtab needs to be called before looking up function names.
499 // This is required because the underlying map is a vector (for space
500 // efficiency) which needs to be sorted.
501 inline void finalizeSymtab();
502
503public:
504 InstrProfSymtab() : VTableAddrMap(VTableAddrMapAllocator) {}
505
506 // Not copyable or movable.
507 // Consider std::unique_ptr for move.
508 InstrProfSymtab(const InstrProfSymtab &) = delete;
509 InstrProfSymtab &operator=(const InstrProfSymtab &) = delete;
510 InstrProfSymtab(InstrProfSymtab &&) = delete;
511 InstrProfSymtab &operator=(InstrProfSymtab &&) = delete;
512
513 /// Create InstrProfSymtab from an object file section which
514 /// contains function PGO names. When section may contain raw
515 /// string data or string data in compressed form. This method
516 /// only initialize the symtab with reference to the data and
517 /// the section base address. The decompression will be delayed
518 /// until before it is used. See also \c create(StringRef) method.
519 Error create(object::SectionRef &Section);
520
521 /// \c NameStrings is a string composed of one of more sub-strings
522 /// encoded in the format described in \c collectPGOFuncNameStrings.
523 /// This method is a wrapper to \c readAndDecodeStrings method.
524 Error create(StringRef NameStrings);
525
526 /// Initialize symtab states with function names and vtable names. \c
527 /// FuncNameStrings is a string composed of one or more encoded function name
528 /// strings, and \c VTableNameStrings composes of one or more encoded vtable
529 /// names. This interface is solely used by raw profile reader.
530 Error create(StringRef FuncNameStrings, StringRef VTableNameStrings);
531
532 /// Initialize 'this' with the set of vtable names encoded in
533 /// \c CompressedVTableNames.
534 Error initVTableNamesFromCompressedStrings(StringRef CompressedVTableNames);
535
536 /// This interface is used by reader of CoverageMapping test
537 /// format.
538 inline Error create(StringRef D, uint64_t BaseAddr);
539
540 /// A wrapper interface to populate the PGO symtab with functions
541 /// decls from module \c M. This interface is used by transformation
542 /// passes such as indirect function call promotion. Variable \c InLTO
543 /// indicates if this is called from LTO optimization passes.
544 Error create(Module &M, bool InLTO = false);
545
546 /// Create InstrProfSymtab from a set of names iteratable from
547 /// \p IterRange. This interface is used by IndexedProfReader.
548 template <typename NameIterRange>
549 Error create(const NameIterRange &IterRange);
550
551 /// Create InstrProfSymtab from a set of function names and vtable
552 /// names iteratable from \p IterRange. This interface is used by
553 /// IndexedProfReader.
554 template <typename FuncNameIterRange, typename VTableNameIterRange>
555 Error create(const FuncNameIterRange &FuncIterRange,
556 const VTableNameIterRange &VTableIterRange);
557
558 Error addSymbolName(StringRef SymbolName) {
559 if (SymbolName.empty())
560 return make_error<InstrProfError>(Args: instrprof_error::malformed,
561 Args: "symbol name is empty");
562
563 // Insert into NameTab so that MD5NameMap (a vector that will be sorted)
564 // won't have duplicated entries in the first place.
565 auto Ins = NameTab.insert(key: SymbolName);
566 if (Ins.second) {
567 MD5NameMap.push_back(x: std::make_pair(
568 x: IndexedInstrProf::ComputeHash(K: SymbolName), y: Ins.first->getKey()));
569 Sorted = false;
570 }
571 return Error::success();
572 }
573
574 /// The method name is kept since there are many callers.
575 /// It just forwards to 'addSymbolName'.
576 Error addFuncName(StringRef FuncName) { return addSymbolName(SymbolName: FuncName); }
577
578 /// Adds VTableName as a known symbol, and inserts it to a map that
579 /// tracks all vtable names.
580 Error addVTableName(StringRef VTableName) {
581 if (Error E = addSymbolName(SymbolName: VTableName))
582 return E;
583
584 // Record VTableName. InstrProfWriter uses this set. The comment around
585 // class member explains why.
586 VTableNames.insert(key: VTableName);
587 return Error::success();
588 }
589
590 const StringSet<> &getVTableNames() const { return VTableNames; }
591
592 /// Map a function address to its name's MD5 hash. This interface
593 /// is only used by the raw profiler reader.
594 void mapAddress(uint64_t Addr, uint64_t MD5Val) {
595 AddrToMD5Map.push_back(x: std::make_pair(x&: Addr, y&: MD5Val));
596 }
597
598 /// Map the address range (i.e., [start_address, end_address)) of a variable
599 /// to its names' MD5 hash. This interface is only used by the raw profile
600 /// reader.
601 void mapVTableAddress(uint64_t StartAddr, uint64_t EndAddr, uint64_t MD5Val) {
602 VTableAddrMap.insert(a: StartAddr, b: EndAddr, y: MD5Val);
603 }
604
605 /// Return a function's hash, or 0, if the function isn't in this SymTab.
606 uint64_t getFunctionHashFromAddress(uint64_t Address);
607
608 /// Return a vtable's hash, or 0 if the vtable doesn't exist in this SymTab.
609 uint64_t getVTableHashFromAddress(uint64_t Address);
610
611 /// Return function's PGO name from the function name's symbol
612 /// address in the object file. If an error occurs, return
613 /// an empty string.
614 StringRef getFuncName(uint64_t FuncNameAddress, size_t NameSize);
615
616 /// Return name of functions or global variables from the name's md5 hash
617 /// value. If not found, return an empty string.
618 inline StringRef getFuncOrVarName(uint64_t ValMD5Hash);
619
620 /// Just like getFuncOrVarName, except that it will return literal string
621 /// 'External Symbol' if the function or global variable is external to
622 /// this symbol table.
623 inline StringRef getFuncOrVarNameIfDefined(uint64_t ValMD5Hash);
624
625 /// True if Symbol is the value used to represent external symbols.
626 static bool isExternalSymbol(const StringRef &Symbol) {
627 return Symbol == InstrProfSymtab::getExternalSymbol();
628 }
629
630 /// Return function from the name's md5 hash. Return nullptr if not found.
631 inline Function *getFunction(uint64_t FuncMD5Hash);
632
633 /// Return the name section data.
634 inline StringRef getNameData() const { return Data; }
635
636 /// Dump the symbols in this table.
637 void dumpNames(raw_ostream &OS) const;
638};
639
640Error InstrProfSymtab::create(StringRef D, uint64_t BaseAddr) {
641 Data = D;
642 Address = BaseAddr;
643 return Error::success();
644}
645
646template <typename NameIterRange>
647Error InstrProfSymtab::create(const NameIterRange &IterRange) {
648 for (auto Name : IterRange)
649 if (Error E = addFuncName(FuncName: Name))
650 return E;
651
652 finalizeSymtab();
653 return Error::success();
654}
655
656template <typename FuncNameIterRange, typename VTableNameIterRange>
657Error InstrProfSymtab::create(const FuncNameIterRange &FuncIterRange,
658 const VTableNameIterRange &VTableIterRange) {
659 // Iterate elements by StringRef rather than by const reference.
660 // StringRef is small enough, so the loop is efficient whether
661 // element in the range is std::string or StringRef.
662 for (StringRef Name : FuncIterRange)
663 if (Error E = addFuncName(FuncName: Name))
664 return E;
665
666 for (StringRef VTableName : VTableIterRange)
667 if (Error E = addVTableName(VTableName))
668 return E;
669
670 finalizeSymtab();
671 return Error::success();
672}
673
674void InstrProfSymtab::finalizeSymtab() {
675 if (Sorted)
676 return;
677 llvm::sort(C&: MD5NameMap, Comp: less_first());
678 llvm::sort(C&: MD5FuncMap, Comp: less_first());
679 llvm::sort(C&: AddrToMD5Map, Comp: less_first());
680 AddrToMD5Map.erase(first: std::unique(first: AddrToMD5Map.begin(), last: AddrToMD5Map.end()),
681 last: AddrToMD5Map.end());
682 Sorted = true;
683}
684
685StringRef InstrProfSymtab::getFuncOrVarNameIfDefined(uint64_t MD5Hash) {
686 StringRef ret = getFuncOrVarName(ValMD5Hash: MD5Hash);
687 if (ret.empty())
688 return InstrProfSymtab::getExternalSymbol();
689 return ret;
690}
691
692StringRef InstrProfSymtab::getFuncOrVarName(uint64_t MD5Hash) {
693 finalizeSymtab();
694 auto Result = llvm::lower_bound(Range&: MD5NameMap, Value&: MD5Hash,
695 C: [](const std::pair<uint64_t, StringRef> &LHS,
696 uint64_t RHS) { return LHS.first < RHS; });
697 if (Result != MD5NameMap.end() && Result->first == MD5Hash)
698 return Result->second;
699 return StringRef();
700}
701
702Function* InstrProfSymtab::getFunction(uint64_t FuncMD5Hash) {
703 finalizeSymtab();
704 auto Result = llvm::lower_bound(Range&: MD5FuncMap, Value&: FuncMD5Hash,
705 C: [](const std::pair<uint64_t, Function *> &LHS,
706 uint64_t RHS) { return LHS.first < RHS; });
707 if (Result != MD5FuncMap.end() && Result->first == FuncMD5Hash)
708 return Result->second;
709 return nullptr;
710}
711
712// To store the sums of profile count values, or the percentage of
713// the sums of the total count values.
714struct CountSumOrPercent {
715 uint64_t NumEntries;
716 double CountSum;
717 double ValueCounts[IPVK_Last - IPVK_First + 1];
718 CountSumOrPercent() : NumEntries(0), CountSum(0.0f), ValueCounts() {}
719 void reset() {
720 NumEntries = 0;
721 CountSum = 0.0f;
722 for (double &VC : ValueCounts)
723 VC = 0.0f;
724 }
725};
726
727// Function level or program level overlap information.
728struct OverlapStats {
729 enum OverlapStatsLevel { ProgramLevel, FunctionLevel };
730 // Sum of the total count values for the base profile.
731 CountSumOrPercent Base;
732 // Sum of the total count values for the test profile.
733 CountSumOrPercent Test;
734 // Overlap lap score. Should be in range of [0.0f to 1.0f].
735 CountSumOrPercent Overlap;
736 CountSumOrPercent Mismatch;
737 CountSumOrPercent Unique;
738 OverlapStatsLevel Level;
739 const std::string *BaseFilename;
740 const std::string *TestFilename;
741 StringRef FuncName;
742 uint64_t FuncHash;
743 bool Valid;
744
745 OverlapStats(OverlapStatsLevel L = ProgramLevel)
746 : Level(L), BaseFilename(nullptr), TestFilename(nullptr), FuncHash(0),
747 Valid(false) {}
748
749 void dump(raw_fd_ostream &OS) const;
750
751 void setFuncInfo(StringRef Name, uint64_t Hash) {
752 FuncName = Name;
753 FuncHash = Hash;
754 }
755
756 Error accumulateCounts(const std::string &BaseFilename,
757 const std::string &TestFilename, bool IsCS);
758 void addOneMismatch(const CountSumOrPercent &MismatchFunc);
759 void addOneUnique(const CountSumOrPercent &UniqueFunc);
760
761 static inline double score(uint64_t Val1, uint64_t Val2, double Sum1,
762 double Sum2) {
763 if (Sum1 < 1.0f || Sum2 < 1.0f)
764 return 0.0f;
765 return std::min(a: Val1 / Sum1, b: Val2 / Sum2);
766 }
767};
768
769// This is used to filter the functions whose overlap information
770// to be output.
771struct OverlapFuncFilters {
772 uint64_t ValueCutoff;
773 const std::string NameFilter;
774};
775
776struct InstrProfValueSiteRecord {
777 /// Value profiling data pairs at a given value site.
778 std::list<InstrProfValueData> ValueData;
779
780 InstrProfValueSiteRecord() { ValueData.clear(); }
781 template <class InputIterator>
782 InstrProfValueSiteRecord(InputIterator F, InputIterator L)
783 : ValueData(F, L) {}
784
785 /// Sort ValueData ascending by Value
786 void sortByTargetValues() {
787 ValueData.sort(
788 comp: [](const InstrProfValueData &left, const InstrProfValueData &right) {
789 return left.Value < right.Value;
790 });
791 }
792 /// Sort ValueData Descending by Count
793 inline void sortByCount();
794
795 /// Merge data from another InstrProfValueSiteRecord
796 /// Optionally scale merged counts by \p Weight.
797 void merge(InstrProfValueSiteRecord &Input, uint64_t Weight,
798 function_ref<void(instrprof_error)> Warn);
799 /// Scale up value profile data counts by N (Numerator) / D (Denominator).
800 void scale(uint64_t N, uint64_t D, function_ref<void(instrprof_error)> Warn);
801
802 /// Compute the overlap b/w this record and Input record.
803 void overlap(InstrProfValueSiteRecord &Input, uint32_t ValueKind,
804 OverlapStats &Overlap, OverlapStats &FuncLevelOverlap);
805};
806
807/// Profiling information for a single function.
808struct InstrProfRecord {
809 std::vector<uint64_t> Counts;
810 std::vector<uint8_t> BitmapBytes;
811
812 InstrProfRecord() = default;
813 InstrProfRecord(std::vector<uint64_t> Counts) : Counts(std::move(Counts)) {}
814 InstrProfRecord(std::vector<uint64_t> Counts,
815 std::vector<uint8_t> BitmapBytes)
816 : Counts(std::move(Counts)), BitmapBytes(std::move(BitmapBytes)) {}
817 InstrProfRecord(InstrProfRecord &&) = default;
818 InstrProfRecord(const InstrProfRecord &RHS)
819 : Counts(RHS.Counts), BitmapBytes(RHS.BitmapBytes),
820 ValueData(RHS.ValueData
821 ? std::make_unique<ValueProfData>(args&: *RHS.ValueData)
822 : nullptr) {}
823 InstrProfRecord &operator=(InstrProfRecord &&) = default;
824 InstrProfRecord &operator=(const InstrProfRecord &RHS) {
825 Counts = RHS.Counts;
826 BitmapBytes = RHS.BitmapBytes;
827 if (!RHS.ValueData) {
828 ValueData = nullptr;
829 return *this;
830 }
831 if (!ValueData)
832 ValueData = std::make_unique<ValueProfData>(args&: *RHS.ValueData);
833 else
834 *ValueData = *RHS.ValueData;
835 return *this;
836 }
837
838 /// Return the number of value profile kinds with non-zero number
839 /// of profile sites.
840 inline uint32_t getNumValueKinds() const;
841 /// Return the number of instrumented sites for ValueKind.
842 inline uint32_t getNumValueSites(uint32_t ValueKind) const;
843
844 /// Return the total number of ValueData for ValueKind.
845 inline uint32_t getNumValueData(uint32_t ValueKind) const;
846
847 /// Return the number of value data collected for ValueKind at profiling
848 /// site: Site.
849 inline uint32_t getNumValueDataForSite(uint32_t ValueKind,
850 uint32_t Site) const;
851
852 /// Return the array of profiled values at \p Site. If \p TotalC
853 /// is not null, the total count of all target values at this site
854 /// will be stored in \c *TotalC.
855 inline std::unique_ptr<InstrProfValueData[]>
856 getValueForSite(uint32_t ValueKind, uint32_t Site,
857 uint64_t *TotalC = nullptr) const;
858
859 /// Get the target value/counts of kind \p ValueKind collected at site
860 /// \p Site and store the result in array \p Dest. Return the total
861 /// counts of all target values at this site.
862 inline uint64_t getValueForSite(InstrProfValueData Dest[], uint32_t ValueKind,
863 uint32_t Site) const;
864
865 /// Reserve space for NumValueSites sites.
866 inline void reserveSites(uint32_t ValueKind, uint32_t NumValueSites);
867
868 /// Add ValueData for ValueKind at value Site.
869 void addValueData(uint32_t ValueKind, uint32_t Site,
870 InstrProfValueData *VData, uint32_t N,
871 InstrProfSymtab *SymTab);
872
873 /// Merge the counts in \p Other into this one.
874 /// Optionally scale merged counts by \p Weight.
875 void merge(InstrProfRecord &Other, uint64_t Weight,
876 function_ref<void(instrprof_error)> Warn);
877
878 /// Scale up profile counts (including value profile data) by
879 /// a factor of (N / D).
880 void scale(uint64_t N, uint64_t D, function_ref<void(instrprof_error)> Warn);
881
882 /// Sort value profile data (per site) by count.
883 void sortValueData() {
884 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
885 for (auto &SR : getValueSitesForKind(ValueKind: Kind))
886 SR.sortByCount();
887 }
888
889 /// Clear value data entries and edge counters.
890 void Clear() {
891 Counts.clear();
892 clearValueData();
893 }
894
895 /// Clear value data entries
896 void clearValueData() { ValueData = nullptr; }
897
898 /// Compute the sums of all counts and store in Sum.
899 void accumulateCounts(CountSumOrPercent &Sum) const;
900
901 /// Compute the overlap b/w this IntrprofRecord and Other.
902 void overlap(InstrProfRecord &Other, OverlapStats &Overlap,
903 OverlapStats &FuncLevelOverlap, uint64_t ValueCutoff);
904
905 /// Compute the overlap of value profile counts.
906 void overlapValueProfData(uint32_t ValueKind, InstrProfRecord &Src,
907 OverlapStats &Overlap,
908 OverlapStats &FuncLevelOverlap);
909
910 enum CountPseudoKind {
911 NotPseudo = 0,
912 PseudoHot,
913 PseudoWarm,
914 };
915 enum PseudoCountVal {
916 HotFunctionVal = -1,
917 WarmFunctionVal = -2,
918 };
919 CountPseudoKind getCountPseudoKind() const {
920 uint64_t FirstCount = Counts[0];
921 if (FirstCount == (uint64_t)HotFunctionVal)
922 return PseudoHot;
923 if (FirstCount == (uint64_t)WarmFunctionVal)
924 return PseudoWarm;
925 return NotPseudo;
926 }
927 void setPseudoCount(CountPseudoKind Kind) {
928 if (Kind == PseudoHot)
929 Counts[0] = (uint64_t)HotFunctionVal;
930 else if (Kind == PseudoWarm)
931 Counts[0] = (uint64_t)WarmFunctionVal;
932 }
933
934private:
935 struct ValueProfData {
936 std::vector<InstrProfValueSiteRecord> IndirectCallSites;
937 std::vector<InstrProfValueSiteRecord> MemOPSizes;
938 std::vector<InstrProfValueSiteRecord> VTableTargets;
939 };
940 std::unique_ptr<ValueProfData> ValueData;
941
942 MutableArrayRef<InstrProfValueSiteRecord>
943 getValueSitesForKind(uint32_t ValueKind) {
944 // Cast to /add/ const (should be an implicit_cast, ideally, if that's ever
945 // implemented in LLVM) to call the const overload of this function, then
946 // cast away the constness from the result.
947 auto AR = const_cast<const InstrProfRecord *>(this)->getValueSitesForKind(
948 ValueKind);
949 return MutableArrayRef(
950 const_cast<InstrProfValueSiteRecord *>(AR.data()), AR.size());
951 }
952 ArrayRef<InstrProfValueSiteRecord>
953 getValueSitesForKind(uint32_t ValueKind) const {
954 if (!ValueData)
955 return std::nullopt;
956 switch (ValueKind) {
957 case IPVK_IndirectCallTarget:
958 return ValueData->IndirectCallSites;
959 case IPVK_MemOPSize:
960 return ValueData->MemOPSizes;
961 case IPVK_VTableTarget:
962 return ValueData->VTableTargets;
963 default:
964 llvm_unreachable("Unknown value kind!");
965 }
966 }
967
968 std::vector<InstrProfValueSiteRecord> &
969 getOrCreateValueSitesForKind(uint32_t ValueKind) {
970 if (!ValueData)
971 ValueData = std::make_unique<ValueProfData>();
972 switch (ValueKind) {
973 case IPVK_IndirectCallTarget:
974 return ValueData->IndirectCallSites;
975 case IPVK_MemOPSize:
976 return ValueData->MemOPSizes;
977 case IPVK_VTableTarget:
978 return ValueData->VTableTargets;
979 default:
980 llvm_unreachable("Unknown value kind!");
981 }
982 }
983
984 // Map indirect call target name hash to name string.
985 uint64_t remapValue(uint64_t Value, uint32_t ValueKind,
986 InstrProfSymtab *SymTab);
987
988 // Merge Value Profile data from Src record to this record for ValueKind.
989 // Scale merged value counts by \p Weight.
990 void mergeValueProfData(uint32_t ValkeKind, InstrProfRecord &Src,
991 uint64_t Weight,
992 function_ref<void(instrprof_error)> Warn);
993
994 // Scale up value profile data count by N (Numerator) / D (Denominator).
995 void scaleValueProfData(uint32_t ValueKind, uint64_t N, uint64_t D,
996 function_ref<void(instrprof_error)> Warn);
997};
998
999struct NamedInstrProfRecord : InstrProfRecord {
1000 StringRef Name;
1001 uint64_t Hash;
1002
1003 // We reserve this bit as the flag for context sensitive profile record.
1004 static const int CS_FLAG_IN_FUNC_HASH = 60;
1005
1006 NamedInstrProfRecord() = default;
1007 NamedInstrProfRecord(StringRef Name, uint64_t Hash,
1008 std::vector<uint64_t> Counts)
1009 : InstrProfRecord(std::move(Counts)), Name(Name), Hash(Hash) {}
1010 NamedInstrProfRecord(StringRef Name, uint64_t Hash,
1011 std::vector<uint64_t> Counts,
1012 std::vector<uint8_t> BitmapBytes)
1013 : InstrProfRecord(std::move(Counts), std::move(BitmapBytes)), Name(Name),
1014 Hash(Hash) {}
1015
1016 static bool hasCSFlagInHash(uint64_t FuncHash) {
1017 return ((FuncHash >> CS_FLAG_IN_FUNC_HASH) & 1);
1018 }
1019 static void setCSFlagInHash(uint64_t &FuncHash) {
1020 FuncHash |= ((uint64_t)1 << CS_FLAG_IN_FUNC_HASH);
1021 }
1022};
1023
1024uint32_t InstrProfRecord::getNumValueKinds() const {
1025 uint32_t NumValueKinds = 0;
1026 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
1027 NumValueKinds += !(getValueSitesForKind(ValueKind: Kind).empty());
1028 return NumValueKinds;
1029}
1030
1031uint32_t InstrProfRecord::getNumValueData(uint32_t ValueKind) const {
1032 uint32_t N = 0;
1033 for (const auto &SR : getValueSitesForKind(ValueKind))
1034 N += SR.ValueData.size();
1035 return N;
1036}
1037
1038uint32_t InstrProfRecord::getNumValueSites(uint32_t ValueKind) const {
1039 return getValueSitesForKind(ValueKind).size();
1040}
1041
1042uint32_t InstrProfRecord::getNumValueDataForSite(uint32_t ValueKind,
1043 uint32_t Site) const {
1044 return getValueSitesForKind(ValueKind)[Site].ValueData.size();
1045}
1046
1047std::unique_ptr<InstrProfValueData[]>
1048InstrProfRecord::getValueForSite(uint32_t ValueKind, uint32_t Site,
1049 uint64_t *TotalC) const {
1050 uint64_t Dummy = 0;
1051 uint64_t &TotalCount = (TotalC == nullptr ? Dummy : *TotalC);
1052 uint32_t N = getNumValueDataForSite(ValueKind, Site);
1053 if (N == 0) {
1054 TotalCount = 0;
1055 return std::unique_ptr<InstrProfValueData[]>(nullptr);
1056 }
1057
1058 auto VD = std::make_unique<InstrProfValueData[]>(num: N);
1059 TotalCount = getValueForSite(Dest: VD.get(), ValueKind, Site);
1060
1061 return VD;
1062}
1063
1064uint64_t InstrProfRecord::getValueForSite(InstrProfValueData Dest[],
1065 uint32_t ValueKind,
1066 uint32_t Site) const {
1067 uint32_t I = 0;
1068 uint64_t TotalCount = 0;
1069 for (auto V : getValueSitesForKind(ValueKind)[Site].ValueData) {
1070 Dest[I].Value = V.Value;
1071 Dest[I].Count = V.Count;
1072 TotalCount = SaturatingAdd(X: TotalCount, Y: V.Count);
1073 I++;
1074 }
1075 return TotalCount;
1076}
1077
1078void InstrProfRecord::reserveSites(uint32_t ValueKind, uint32_t NumValueSites) {
1079 if (!NumValueSites)
1080 return;
1081 getOrCreateValueSitesForKind(ValueKind).reserve(n: NumValueSites);
1082}
1083
1084// Include definitions for value profile data
1085#define INSTR_PROF_VALUE_PROF_DATA
1086#include "llvm/ProfileData/InstrProfData.inc"
1087
1088void InstrProfValueSiteRecord::sortByCount() {
1089 ValueData.sort(
1090 comp: [](const InstrProfValueData &left, const InstrProfValueData &right) {
1091 return left.Count > right.Count;
1092 });
1093 // Now truncate
1094 size_t max_s = INSTR_PROF_MAX_NUM_VAL_PER_SITE;
1095 if (ValueData.size() > max_s)
1096 ValueData.resize(new_size: max_s);
1097}
1098
1099namespace IndexedInstrProf {
1100
1101enum class HashT : uint32_t {
1102 MD5,
1103 Last = MD5
1104};
1105
1106inline uint64_t ComputeHash(HashT Type, StringRef K) {
1107 switch (Type) {
1108 case HashT::MD5:
1109 return MD5Hash(Str: K);
1110 }
1111 llvm_unreachable("Unhandled hash type");
1112}
1113
1114const uint64_t Magic = 0x8169666f72706cff; // "\xfflprofi\x81"
1115
1116enum ProfVersion {
1117 // Version 1 is the first version. In this version, the value of
1118 // a key/value pair can only include profile data of a single function.
1119 // Due to this restriction, the number of block counters for a given
1120 // function is not recorded but derived from the length of the value.
1121 Version1 = 1,
1122 // The version 2 format supports recording profile data of multiple
1123 // functions which share the same key in one value field. To support this,
1124 // the number block counters is recorded as an uint64_t field right after the
1125 // function structural hash.
1126 Version2 = 2,
1127 // Version 3 supports value profile data. The value profile data is expected
1128 // to follow the block counter profile data.
1129 Version3 = 3,
1130 // In this version, profile summary data \c IndexedInstrProf::Summary is
1131 // stored after the profile header.
1132 Version4 = 4,
1133 // In this version, the frontend PGO stable hash algorithm defaults to V2.
1134 Version5 = 5,
1135 // In this version, the frontend PGO stable hash algorithm got fixed and
1136 // may produce hashes different from Version5.
1137 Version6 = 6,
1138 // An additional counter is added around logical operators.
1139 Version7 = 7,
1140 // An additional (optional) memory profile type is added.
1141 Version8 = 8,
1142 // Binary ids are added.
1143 Version9 = 9,
1144 // An additional (optional) temporal profile traces section is added.
1145 Version10 = 10,
1146 // An additional field is used for bitmap bytes.
1147 Version11 = 11,
1148 // VTable profiling,
1149 Version12 = 12,
1150 // The current version is 12.
1151 CurrentVersion = INSTR_PROF_INDEX_VERSION
1152};
1153const uint64_t Version = ProfVersion::CurrentVersion;
1154
1155const HashT HashType = HashT::MD5;
1156
1157inline uint64_t ComputeHash(StringRef K) { return ComputeHash(Type: HashType, K); }
1158
1159// This structure defines the file header of the LLVM profile
1160// data file in indexed-format. Please update llvm/docs/InstrProfileFormat.rst
1161// as appropriate when updating the indexed profile format.
1162struct Header {
1163 uint64_t Magic;
1164 // The lower 32 bits specify the version of the indexed profile.
1165 // The most significant 32 bits are reserved to specify the variant types of
1166 // the profile.
1167 uint64_t Version;
1168 uint64_t Unused; // Becomes unused since version 4
1169 uint64_t HashType;
1170 // This field records the offset of this hash table's metadata (i.e., the
1171 // number of buckets and entries), which follows right after the payload of
1172 // the entire hash table.
1173 uint64_t HashOffset;
1174 uint64_t MemProfOffset;
1175 uint64_t BinaryIdOffset;
1176 uint64_t TemporalProfTracesOffset;
1177 uint64_t VTableNamesOffset;
1178 // New fields should only be added at the end to ensure that the size
1179 // computation is correct. The methods below need to be updated to ensure that
1180 // the new field is read correctly.
1181
1182 // Reads a header struct from the buffer.
1183 static Expected<Header> readFromBuffer(const unsigned char *Buffer);
1184
1185 // Returns the size of the header in bytes for all valid fields based on the
1186 // version. I.e a older version header will return a smaller size.
1187 size_t size() const;
1188
1189 // Returns the format version in little endian. The header retains the version
1190 // in native endian of the compiler runtime.
1191 uint64_t formatVersion() const;
1192};
1193
1194// Profile summary data recorded in the profile data file in indexed
1195// format. It is introduced in version 4. The summary data follows
1196// right after the profile file header.
1197struct Summary {
1198 struct Entry {
1199 uint64_t Cutoff; ///< The required percentile of total execution count.
1200 uint64_t
1201 MinBlockCount; ///< The minimum execution count for this percentile.
1202 uint64_t NumBlocks; ///< Number of blocks >= the minumum execution count.
1203 };
1204 // The field kind enumerator to assigned value mapping should remain
1205 // unchanged when a new kind is added or an old kind gets deleted in
1206 // the future.
1207 enum SummaryFieldKind {
1208 /// The total number of functions instrumented.
1209 TotalNumFunctions = 0,
1210 /// Total number of instrumented blocks/edges.
1211 TotalNumBlocks = 1,
1212 /// The maximal execution count among all functions.
1213 /// This field does not exist for profile data from IR based
1214 /// instrumentation.
1215 MaxFunctionCount = 2,
1216 /// Max block count of the program.
1217 MaxBlockCount = 3,
1218 /// Max internal block count of the program (excluding entry blocks).
1219 MaxInternalBlockCount = 4,
1220 /// The sum of all instrumented block counts.
1221 TotalBlockCount = 5,
1222 NumKinds = TotalBlockCount + 1
1223 };
1224
1225 // The number of summmary fields following the summary header.
1226 uint64_t NumSummaryFields;
1227 // The number of Cutoff Entries (Summary::Entry) following summary fields.
1228 uint64_t NumCutoffEntries;
1229
1230 Summary() = delete;
1231 Summary(uint32_t Size) { memset(s: this, c: 0, n: Size); }
1232
1233 void operator delete(void *ptr) { ::operator delete(ptr); }
1234
1235 static uint32_t getSize(uint32_t NumSumFields, uint32_t NumCutoffEntries) {
1236 return sizeof(Summary) + NumCutoffEntries * sizeof(Entry) +
1237 NumSumFields * sizeof(uint64_t);
1238 }
1239
1240 const uint64_t *getSummaryDataBase() const {
1241 return reinterpret_cast<const uint64_t *>(this + 1);
1242 }
1243
1244 uint64_t *getSummaryDataBase() {
1245 return reinterpret_cast<uint64_t *>(this + 1);
1246 }
1247
1248 const Entry *getCutoffEntryBase() const {
1249 return reinterpret_cast<const Entry *>(
1250 &getSummaryDataBase()[NumSummaryFields]);
1251 }
1252
1253 Entry *getCutoffEntryBase() {
1254 return reinterpret_cast<Entry *>(&getSummaryDataBase()[NumSummaryFields]);
1255 }
1256
1257 uint64_t get(SummaryFieldKind K) const {
1258 return getSummaryDataBase()[K];
1259 }
1260
1261 void set(SummaryFieldKind K, uint64_t V) {
1262 getSummaryDataBase()[K] = V;
1263 }
1264
1265 const Entry &getEntry(uint32_t I) const { return getCutoffEntryBase()[I]; }
1266
1267 void setEntry(uint32_t I, const ProfileSummaryEntry &E) {
1268 Entry &ER = getCutoffEntryBase()[I];
1269 ER.Cutoff = E.Cutoff;
1270 ER.MinBlockCount = E.MinCount;
1271 ER.NumBlocks = E.NumCounts;
1272 }
1273};
1274
1275inline std::unique_ptr<Summary> allocSummary(uint32_t TotalSize) {
1276 return std::unique_ptr<Summary>(new (::operator new(TotalSize))
1277 Summary(TotalSize));
1278}
1279
1280} // end namespace IndexedInstrProf
1281
1282namespace RawInstrProf {
1283
1284// Version 1: First version
1285// Version 2: Added value profile data section. Per-function control data
1286// struct has more fields to describe value profile information.
1287// Version 3: Compressed name section support. Function PGO name reference
1288// from control data struct is changed from raw pointer to Name's MD5 value.
1289// Version 4: ValueDataBegin and ValueDataSizes fields are removed from the
1290// raw header.
1291// Version 5: Bit 60 of FuncHash is reserved for the flag for the context
1292// sensitive records.
1293// Version 6: Added binary id.
1294// Version 7: Reorder binary id and include version in signature.
1295// Version 8: Use relative counter pointer.
1296// Version 9: Added relative bitmap bytes pointer and count used by MC/DC.
1297// Version 10: Added vtable, a new type of value profile data.
1298const uint64_t Version = INSTR_PROF_RAW_VERSION;
1299
1300template <class IntPtrT> inline uint64_t getMagic();
1301template <> inline uint64_t getMagic<uint64_t>() {
1302 return INSTR_PROF_RAW_MAGIC_64;
1303}
1304
1305template <> inline uint64_t getMagic<uint32_t>() {
1306 return INSTR_PROF_RAW_MAGIC_32;
1307}
1308
1309// Per-function profile data header/control structure.
1310// The definition should match the structure defined in
1311// compiler-rt/lib/profile/InstrProfiling.h.
1312// It should also match the synthesized type in
1313// Transforms/Instrumentation/InstrProfiling.cpp:getOrCreateRegionCounters.
1314template <class IntPtrT> struct alignas(8) ProfileData {
1315#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Type Name;
1316#include "llvm/ProfileData/InstrProfData.inc"
1317};
1318
1319template <class IntPtrT> struct alignas(8) VTableProfileData {
1320#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Init) Type Name;
1321#include "llvm/ProfileData/InstrProfData.inc"
1322};
1323
1324// File header structure of the LLVM profile data in raw format.
1325// The definition should match the header referenced in
1326// compiler-rt/lib/profile/InstrProfilingFile.c and
1327// InstrProfilingBuffer.c.
1328struct Header {
1329#define INSTR_PROF_RAW_HEADER(Type, Name, Init) const Type Name;
1330#include "llvm/ProfileData/InstrProfData.inc"
1331};
1332
1333} // end namespace RawInstrProf
1334
1335// Create the variable for the profile file name.
1336void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput);
1337
1338// Whether to compress function names in profile records, and filenames in
1339// code coverage mappings. Used by the Instrumentation library and unit tests.
1340extern cl::opt<bool> DoInstrProfNameCompression;
1341
1342} // end namespace llvm
1343#endif // LLVM_PROFILEDATA_INSTRPROF_H
1344

source code of llvm/include/llvm/ProfileData/InstrProf.h