1 | //===- InstrProf.h - Instrumented profiling format support ------*- C++ -*-===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // Instrumentation-based profiling data is generated by instrumented |
10 | // binaries through library functions in compiler-rt, and read by the clang |
11 | // frontend to feed PGO. |
12 | // |
13 | //===----------------------------------------------------------------------===// |
14 | |
15 | #ifndef LLVM_PROFILEDATA_INSTRPROF_H |
16 | #define LLVM_PROFILEDATA_INSTRPROF_H |
17 | |
18 | #include "llvm/ADT/ArrayRef.h" |
19 | #include "llvm/ADT/BitmaskEnum.h" |
20 | #include "llvm/ADT/IntervalMap.h" |
21 | #include "llvm/ADT/STLExtras.h" |
22 | #include "llvm/ADT/StringRef.h" |
23 | #include "llvm/ADT/StringSet.h" |
24 | #include "llvm/IR/GlobalValue.h" |
25 | #include "llvm/IR/ProfileSummary.h" |
26 | #include "llvm/ProfileData/InstrProfData.inc" |
27 | #include "llvm/Support/BalancedPartitioning.h" |
28 | #include "llvm/Support/CommandLine.h" |
29 | #include "llvm/Support/Compiler.h" |
30 | #include "llvm/Support/Error.h" |
31 | #include "llvm/Support/ErrorHandling.h" |
32 | #include "llvm/Support/MD5.h" |
33 | #include "llvm/Support/MathExtras.h" |
34 | #include "llvm/Support/raw_ostream.h" |
35 | #include "llvm/TargetParser/Host.h" |
36 | #include "llvm/TargetParser/Triple.h" |
37 | #include <algorithm> |
38 | #include <cassert> |
39 | #include <cstddef> |
40 | #include <cstdint> |
41 | #include <cstring> |
42 | #include <list> |
43 | #include <memory> |
44 | #include <string> |
45 | #include <system_error> |
46 | #include <utility> |
47 | #include <vector> |
48 | |
49 | namespace llvm { |
50 | |
51 | class Function; |
52 | class GlobalVariable; |
53 | struct InstrProfRecord; |
54 | class InstrProfSymtab; |
55 | class Instruction; |
56 | class MDNode; |
57 | class Module; |
58 | |
59 | enum InstrProfSectKind { |
60 | #define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) Kind, |
61 | #include "llvm/ProfileData/InstrProfData.inc" |
62 | }; |
63 | |
64 | /// Return the max count value. We reserver a few large values for special use. |
65 | inline uint64_t getInstrMaxCountValue() { |
66 | return std::numeric_limits<uint64_t>::max() - 2; |
67 | } |
68 | |
69 | /// Return the name of the profile section corresponding to \p IPSK. |
70 | /// |
71 | /// The name of the section depends on the object format type \p OF. If |
72 | /// \p AddSegmentInfo is true, a segment prefix and additional linker hints may |
73 | /// be added to the section name (this is the default). |
74 | std::string getInstrProfSectionName(InstrProfSectKind IPSK, |
75 | Triple::ObjectFormatType OF, |
76 | bool AddSegmentInfo = true); |
77 | |
78 | /// Return the name profile runtime entry point to do value profiling |
79 | /// for a given site. |
80 | inline StringRef getInstrProfValueProfFuncName() { |
81 | return INSTR_PROF_VALUE_PROF_FUNC_STR; |
82 | } |
83 | |
84 | /// Return the name profile runtime entry point to do memop size value |
85 | /// profiling. |
86 | inline StringRef getInstrProfValueProfMemOpFuncName() { |
87 | return INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR; |
88 | } |
89 | |
90 | /// Return the name prefix of variables containing instrumented function names. |
91 | inline StringRef getInstrProfNameVarPrefix() { return "__profn_" ; } |
92 | |
93 | /// Return the name prefix of variables containing virtual table profile data. |
94 | inline StringRef getInstrProfVTableVarPrefix() { return "__profvt_" ; } |
95 | |
96 | /// Return the name prefix of variables containing per-function control data. |
97 | inline StringRef getInstrProfDataVarPrefix() { return "__profd_" ; } |
98 | |
99 | /// Return the name prefix of profile counter variables. |
100 | inline StringRef getInstrProfCountersVarPrefix() { return "__profc_" ; } |
101 | |
102 | /// Return the name prefix of profile bitmap variables. |
103 | inline StringRef getInstrProfBitmapVarPrefix() { return "__profbm_" ; } |
104 | |
105 | /// Return the name prefix of value profile variables. |
106 | inline StringRef getInstrProfValuesVarPrefix() { return "__profvp_" ; } |
107 | |
108 | /// Return the name of value profile node array variables: |
109 | inline StringRef getInstrProfVNodesVarName() { return "__llvm_prf_vnodes" ; } |
110 | |
111 | /// Return the name of the variable holding the strings (possibly compressed) |
112 | /// of all function's PGO names. |
113 | inline StringRef getInstrProfNamesVarName() { return "__llvm_prf_nm" ; } |
114 | |
115 | inline StringRef getInstrProfVTableNamesVarName() { return "__llvm_prf_vnm" ; } |
116 | |
117 | /// Return the name of a covarage mapping variable (internal linkage) |
118 | /// for each instrumented source module. Such variables are allocated |
119 | /// in the __llvm_covmap section. |
120 | inline StringRef getCoverageMappingVarName() { |
121 | return "__llvm_coverage_mapping" ; |
122 | } |
123 | |
124 | /// Return the name of the internal variable recording the array |
125 | /// of PGO name vars referenced by the coverage mapping. The owning |
126 | /// functions of those names are not emitted by FE (e.g, unused inline |
127 | /// functions.) |
128 | inline StringRef getCoverageUnusedNamesVarName() { |
129 | return "__llvm_coverage_names" ; |
130 | } |
131 | |
132 | /// Return the name of function that registers all the per-function control |
133 | /// data at program startup time by calling __llvm_register_function. This |
134 | /// function has internal linkage and is called by __llvm_profile_init |
135 | /// runtime method. This function is not generated for these platforms: |
136 | /// Darwin, Linux, and FreeBSD. |
137 | inline StringRef getInstrProfRegFuncsName() { |
138 | return "__llvm_profile_register_functions" ; |
139 | } |
140 | |
141 | /// Return the name of the runtime interface that registers per-function control |
142 | /// data for one instrumented function. |
143 | inline StringRef getInstrProfRegFuncName() { |
144 | return "__llvm_profile_register_function" ; |
145 | } |
146 | |
147 | /// Return the name of the runtime interface that registers the PGO name |
148 | /// strings. |
149 | inline StringRef getInstrProfNamesRegFuncName() { |
150 | return "__llvm_profile_register_names_function" ; |
151 | } |
152 | |
153 | /// Return the name of the runtime initialization method that is generated by |
154 | /// the compiler. The function calls __llvm_profile_register_functions and |
155 | /// __llvm_profile_override_default_filename functions if needed. This function |
156 | /// has internal linkage and invoked at startup time via init_array. |
157 | inline StringRef getInstrProfInitFuncName() { return "__llvm_profile_init" ; } |
158 | |
159 | /// Return the name of the hook variable defined in profile runtime library. |
160 | /// A reference to the variable causes the linker to link in the runtime |
161 | /// initialization module (which defines the hook variable). |
162 | inline StringRef getInstrProfRuntimeHookVarName() { |
163 | return INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_RUNTIME_VAR); |
164 | } |
165 | |
166 | /// Return the name of the compiler generated function that references the |
167 | /// runtime hook variable. The function is a weak global. |
168 | inline StringRef getInstrProfRuntimeHookVarUseFuncName() { |
169 | return "__llvm_profile_runtime_user" ; |
170 | } |
171 | |
172 | inline StringRef getInstrProfCounterBiasVarName() { |
173 | return INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_COUNTER_BIAS_VAR); |
174 | } |
175 | |
176 | /// Return the marker used to separate PGO names during serialization. |
177 | inline StringRef getInstrProfNameSeparator() { return "\01" ; } |
178 | |
179 | /// Please use getIRPGOFuncName for LLVM IR instrumentation. This function is |
180 | /// for front-end (Clang, etc) instrumentation. |
181 | /// Return the modified name for function \c F suitable to be |
182 | /// used the key for profile lookup. Variable \c InLTO indicates if this |
183 | /// is called in LTO optimization passes. |
184 | std::string getPGOFuncName(const Function &F, bool InLTO = false, |
185 | uint64_t Version = INSTR_PROF_INDEX_VERSION); |
186 | |
187 | /// Return the modified name for a function suitable to be |
188 | /// used the key for profile lookup. The function's original |
189 | /// name is \c RawFuncName and has linkage of type \c Linkage. |
190 | /// The function is defined in module \c FileName. |
191 | std::string getPGOFuncName(StringRef RawFuncName, |
192 | GlobalValue::LinkageTypes Linkage, |
193 | StringRef FileName, |
194 | uint64_t Version = INSTR_PROF_INDEX_VERSION); |
195 | |
196 | /// \return the modified name for function \c F suitable to be |
197 | /// used as the key for IRPGO profile lookup. \c InLTO indicates if this is |
198 | /// called from LTO optimization passes. |
199 | std::string getIRPGOFuncName(const Function &F, bool InLTO = false); |
200 | |
201 | /// \return the filename and the function name parsed from the output of |
202 | /// \c getIRPGOFuncName() |
203 | std::pair<StringRef, StringRef> getParsedIRPGOName(StringRef IRPGOName); |
204 | |
205 | /// Return the name of the global variable used to store a function |
206 | /// name in PGO instrumentation. \c FuncName is the IRPGO function name |
207 | /// (returned by \c getIRPGOFuncName) for LLVM IR instrumentation and PGO |
208 | /// function name (returned by \c getPGOFuncName) for front-end instrumentation. |
209 | std::string getPGOFuncNameVarName(StringRef FuncName, |
210 | GlobalValue::LinkageTypes Linkage); |
211 | |
212 | /// Create and return the global variable for function name used in PGO |
213 | /// instrumentation. \c FuncName is the IRPGO function name (returned by |
214 | /// \c getIRPGOFuncName) for LLVM IR instrumentation and PGO function name |
215 | /// (returned by \c getPGOFuncName) for front-end instrumentation. |
216 | GlobalVariable *createPGOFuncNameVar(Function &F, StringRef PGOFuncName); |
217 | |
218 | /// Create and return the global variable for function name used in PGO |
219 | /// instrumentation. \c FuncName is the IRPGO function name (returned by |
220 | /// \c getIRPGOFuncName) for LLVM IR instrumentation and PGO function name |
221 | /// (returned by \c getPGOFuncName) for front-end instrumentation. |
222 | GlobalVariable *createPGOFuncNameVar(Module &M, |
223 | GlobalValue::LinkageTypes Linkage, |
224 | StringRef PGOFuncName); |
225 | |
226 | /// Return the initializer in string of the PGO name var \c NameVar. |
227 | StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar); |
228 | |
229 | /// Given a PGO function name, remove the filename prefix and return |
230 | /// the original (static) function name. |
231 | StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, |
232 | StringRef FileName = "<unknown>" ); |
233 | |
234 | /// Given a vector of strings (names of global objects like functions or, |
235 | /// virtual tables) \c NameStrs, the method generates a combined string \c |
236 | /// Result that is ready to be serialized. The \c Result string is comprised of |
237 | /// three fields: The first field is the length of the uncompressed strings, and |
238 | /// the the second field is the length of the zlib-compressed string. Both |
239 | /// fields are encoded in ULEB128. If \c doCompress is false, the |
240 | /// third field is the uncompressed strings; otherwise it is the |
241 | /// compressed string. When the string compression is off, the |
242 | /// second field will have value zero. |
243 | Error collectGlobalObjectNameStrings(ArrayRef<std::string> NameStrs, |
244 | bool doCompression, std::string &Result); |
245 | |
246 | /// Produce \c Result string with the same format described above. The input |
247 | /// is vector of PGO function name variables that are referenced. |
248 | /// The global variable element in 'NameVars' is a string containing the pgo |
249 | /// name of a function. See `createPGOFuncNameVar` that creates these global |
250 | /// variables. |
251 | Error collectPGOFuncNameStrings(ArrayRef<GlobalVariable *> NameVars, |
252 | std::string &Result, bool doCompression = true); |
253 | |
254 | Error collectVTableStrings(ArrayRef<GlobalVariable *> VTables, |
255 | std::string &Result, bool doCompression); |
256 | |
257 | /// Check if INSTR_PROF_RAW_VERSION_VAR is defined. This global is only being |
258 | /// set in IR PGO compilation. |
259 | bool isIRPGOFlagSet(const Module *M); |
260 | |
261 | /// Check if we can safely rename this Comdat function. Instances of the same |
262 | /// comdat function may have different control flows thus can not share the |
263 | /// same counter variable. |
264 | bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken = false); |
265 | |
266 | enum InstrProfValueKind : uint32_t { |
267 | #define VALUE_PROF_KIND(Enumerator, Value, Descr) Enumerator = Value, |
268 | #include "llvm/ProfileData/InstrProfData.inc" |
269 | }; |
270 | |
271 | /// Get the value profile data for value site \p SiteIdx from \p InstrProfR |
272 | /// and annotate the instruction \p Inst with the value profile meta data. |
273 | /// Annotate up to \p MaxMDCount (default 3) number of records per value site. |
274 | void annotateValueSite(Module &M, Instruction &Inst, |
275 | const InstrProfRecord &InstrProfR, |
276 | InstrProfValueKind ValueKind, uint32_t SiteIndx, |
277 | uint32_t MaxMDCount = 3); |
278 | |
279 | /// Same as the above interface but using an ArrayRef, as well as \p Sum. |
280 | /// This function will not annotate !prof metadata on the instruction if the |
281 | /// referenced array is empty. |
282 | void annotateValueSite(Module &M, Instruction &Inst, |
283 | ArrayRef<InstrProfValueData> VDs, uint64_t Sum, |
284 | InstrProfValueKind ValueKind, uint32_t MaxMDCount); |
285 | |
286 | /// Extract the value profile data from \p Inst which is annotated with |
287 | /// value profile meta data. Return false if there is no value data annotated, |
288 | /// otherwise return true. |
289 | bool getValueProfDataFromInst(const Instruction &Inst, |
290 | InstrProfValueKind ValueKind, |
291 | uint32_t MaxNumValueData, |
292 | InstrProfValueData ValueData[], |
293 | uint32_t &ActualNumValueData, uint64_t &TotalC, |
294 | bool GetNoICPValue = false); |
295 | |
296 | /// Extract the value profile data from \p Inst and returns them if \p Inst is |
297 | /// annotated with value profile data. Returns nullptr otherwise. It's similar |
298 | /// to `getValueProfDataFromInst` above except that an array is allocated only |
299 | /// after a preliminary checking that the value profiles of kind `ValueKind` |
300 | /// exist. |
301 | std::unique_ptr<InstrProfValueData[]> |
302 | getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind, |
303 | uint32_t MaxNumValueData, uint32_t &ActualNumValueData, |
304 | uint64_t &TotalC, bool GetNoICPValue = false); |
305 | |
306 | inline StringRef getPGOFuncNameMetadataName() { return "PGOFuncName" ; } |
307 | |
308 | /// Return the PGOFuncName meta data associated with a function. |
309 | MDNode *getPGOFuncNameMetadata(const Function &F); |
310 | |
311 | std::string getPGOName(const GlobalVariable &V, bool InLTO = false); |
312 | |
313 | /// Create the PGOFuncName meta data if PGOFuncName is different from |
314 | /// function's raw name. This should only apply to internal linkage functions |
315 | /// declared by users only. |
316 | void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName); |
317 | |
318 | /// Check if we can use Comdat for profile variables. This will eliminate |
319 | /// the duplicated profile variables for Comdat functions. |
320 | bool needsComdatForCounter(const GlobalObject &GV, const Module &M); |
321 | |
322 | /// An enum describing the attributes of an instrumented profile. |
323 | enum class InstrProfKind { |
324 | Unknown = 0x0, |
325 | // A frontend clang profile, incompatible with other attrs. |
326 | FrontendInstrumentation = 0x1, |
327 | // An IR-level profile (default when -fprofile-generate is used). |
328 | IRInstrumentation = 0x2, |
329 | // A profile with entry basic block instrumentation. |
330 | FunctionEntryInstrumentation = 0x4, |
331 | // A context sensitive IR-level profile. |
332 | ContextSensitive = 0x8, |
333 | // Use single byte probes for coverage. |
334 | SingleByteCoverage = 0x10, |
335 | // Only instrument the function entry basic block. |
336 | FunctionEntryOnly = 0x20, |
337 | // A memory profile collected using -fprofile=memory. |
338 | MemProf = 0x40, |
339 | // A temporal profile. |
340 | TemporalProfile = 0x80, |
341 | LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/TemporalProfile) |
342 | }; |
343 | |
344 | const std::error_category &instrprof_category(); |
345 | |
346 | enum class instrprof_error { |
347 | success = 0, |
348 | eof, |
349 | unrecognized_format, |
350 | bad_magic, |
351 | , |
352 | unsupported_version, |
353 | unsupported_hash_type, |
354 | too_large, |
355 | truncated, |
356 | malformed, |
357 | missing_correlation_info, |
358 | unexpected_correlation_info, |
359 | unable_to_correlate_profile, |
360 | unknown_function, |
361 | invalid_prof, |
362 | hash_mismatch, |
363 | count_mismatch, |
364 | bitmap_mismatch, |
365 | counter_overflow, |
366 | value_site_count_mismatch, |
367 | compress_failed, |
368 | uncompress_failed, |
369 | empty_raw_profile, |
370 | zlib_unavailable, |
371 | raw_profile_version_mismatch, |
372 | counter_value_too_large, |
373 | }; |
374 | |
375 | /// An ordered list of functions identified by their NameRef found in |
376 | /// INSTR_PROF_DATA |
377 | struct TemporalProfTraceTy { |
378 | std::vector<uint64_t> FunctionNameRefs; |
379 | uint64_t Weight; |
380 | TemporalProfTraceTy(std::initializer_list<uint64_t> Trace = {}, |
381 | uint64_t Weight = 1) |
382 | : FunctionNameRefs(Trace), Weight(Weight) {} |
383 | |
384 | /// Use a set of temporal profile traces to create a list of balanced |
385 | /// partitioning function nodes used by BalancedPartitioning to generate a |
386 | /// function order that reduces page faults during startup |
387 | static std::vector<BPFunctionNode> |
388 | createBPFunctionNodes(ArrayRef<TemporalProfTraceTy> Traces); |
389 | }; |
390 | |
391 | inline std::error_code make_error_code(instrprof_error E) { |
392 | return std::error_code(static_cast<int>(E), instrprof_category()); |
393 | } |
394 | |
395 | class InstrProfError : public ErrorInfo<InstrProfError> { |
396 | public: |
397 | InstrProfError(instrprof_error Err, const Twine &ErrStr = Twine()) |
398 | : Err(Err), Msg(ErrStr.str()) { |
399 | assert(Err != instrprof_error::success && "Not an error" ); |
400 | } |
401 | |
402 | std::string message() const override; |
403 | |
404 | void log(raw_ostream &OS) const override { OS << message(); } |
405 | |
406 | std::error_code convertToErrorCode() const override { |
407 | return make_error_code(E: Err); |
408 | } |
409 | |
410 | instrprof_error get() const { return Err; } |
411 | const std::string &getMessage() const { return Msg; } |
412 | |
413 | /// Consume an Error and return the raw enum value contained within it, and |
414 | /// the optional error message. The Error must either be a success value, or |
415 | /// contain a single InstrProfError. |
416 | static std::pair<instrprof_error, std::string> take(Error E) { |
417 | auto Err = instrprof_error::success; |
418 | std::string Msg = "" ; |
419 | handleAllErrors(E: std::move(E), Handlers: [&Err, &Msg](const InstrProfError &IPE) { |
420 | assert(Err == instrprof_error::success && "Multiple errors encountered" ); |
421 | Err = IPE.get(); |
422 | Msg = IPE.getMessage(); |
423 | }); |
424 | return {Err, Msg}; |
425 | } |
426 | |
427 | static char ID; |
428 | |
429 | private: |
430 | instrprof_error Err; |
431 | std::string Msg; |
432 | }; |
433 | |
434 | namespace object { |
435 | |
436 | class SectionRef; |
437 | |
438 | } // end namespace object |
439 | |
440 | namespace IndexedInstrProf { |
441 | |
442 | uint64_t ComputeHash(StringRef K); |
443 | |
444 | } // end namespace IndexedInstrProf |
445 | |
446 | /// A symbol table used for function [IR]PGO name look-up with keys |
447 | /// (such as pointers, md5hash values) to the function. A function's |
448 | /// [IR]PGO name or name's md5hash are used in retrieving the profile |
449 | /// data of the function. See \c getIRPGOFuncName() and \c getPGOFuncName |
450 | /// methods for details how [IR]PGO name is formed. |
451 | class InstrProfSymtab { |
452 | public: |
453 | using AddrHashMap = std::vector<std::pair<uint64_t, uint64_t>>; |
454 | |
455 | private: |
456 | using AddrIntervalMap = |
457 | IntervalMap<uint64_t, uint64_t, 4, IntervalMapHalfOpenInfo<uint64_t>>; |
458 | StringRef Data; |
459 | uint64_t Address = 0; |
460 | // Unique name strings. Used to ensure entries in MD5NameMap (a vector that's |
461 | // going to be sorted) has unique MD5 keys in the first place. |
462 | StringSet<> NameTab; |
463 | // Records the unique virtual table names. This is used by InstrProfWriter to |
464 | // write out an on-disk chained hash table of virtual table names. |
465 | // InstrProfWriter stores per function profile data (keyed by function names) |
466 | // so it doesn't use a StringSet for function names. |
467 | StringSet<> VTableNames; |
468 | // A map from MD5 keys to function name strings. |
469 | std::vector<std::pair<uint64_t, StringRef>> MD5NameMap; |
470 | // A map from MD5 keys to function define. We only populate this map |
471 | // when build the Symtab from a Module. |
472 | std::vector<std::pair<uint64_t, Function *>> MD5FuncMap; |
473 | // A map from function runtime address to function name MD5 hash. |
474 | // This map is only populated and used by raw instr profile reader. |
475 | AddrHashMap AddrToMD5Map; |
476 | |
477 | AddrIntervalMap::Allocator VTableAddrMapAllocator; |
478 | // This map is only populated and used by raw instr profile reader. |
479 | AddrIntervalMap VTableAddrMap; |
480 | bool Sorted = false; |
481 | |
482 | static StringRef getExternalSymbol() { return "** External Symbol **" ; } |
483 | |
484 | // Returns the canonial name of the given PGOName. In a canonical name, all |
485 | // suffixes that begins with "." except ".__uniq." are stripped. |
486 | // FIXME: Unify this with `FunctionSamples::getCanonicalFnName`. |
487 | static StringRef getCanonicalName(StringRef PGOName); |
488 | |
489 | // Add the function into the symbol table, by creating the following |
490 | // map entries: |
491 | // name-set = {PGOFuncName} + {getCanonicalName(PGOFuncName)} if the canonical |
492 | // name is different from pgo name |
493 | // - In MD5NameMap: <MD5Hash(name), name> for name in name-set |
494 | // - In MD5FuncMap: <MD5Hash(name), &F> for name in name-set |
495 | Error addFuncWithName(Function &F, StringRef PGOFuncName); |
496 | |
497 | // If the symtab is created by a series of calls to \c addFuncName, \c |
498 | // finalizeSymtab needs to be called before looking up function names. |
499 | // This is required because the underlying map is a vector (for space |
500 | // efficiency) which needs to be sorted. |
501 | inline void finalizeSymtab(); |
502 | |
503 | public: |
504 | InstrProfSymtab() : VTableAddrMap(VTableAddrMapAllocator) {} |
505 | |
506 | // Not copyable or movable. |
507 | // Consider std::unique_ptr for move. |
508 | InstrProfSymtab(const InstrProfSymtab &) = delete; |
509 | InstrProfSymtab &operator=(const InstrProfSymtab &) = delete; |
510 | InstrProfSymtab(InstrProfSymtab &&) = delete; |
511 | InstrProfSymtab &operator=(InstrProfSymtab &&) = delete; |
512 | |
513 | /// Create InstrProfSymtab from an object file section which |
514 | /// contains function PGO names. When section may contain raw |
515 | /// string data or string data in compressed form. This method |
516 | /// only initialize the symtab with reference to the data and |
517 | /// the section base address. The decompression will be delayed |
518 | /// until before it is used. See also \c create(StringRef) method. |
519 | Error create(object::SectionRef &Section); |
520 | |
521 | /// \c NameStrings is a string composed of one of more sub-strings |
522 | /// encoded in the format described in \c collectPGOFuncNameStrings. |
523 | /// This method is a wrapper to \c readAndDecodeStrings method. |
524 | Error create(StringRef NameStrings); |
525 | |
526 | /// Initialize symtab states with function names and vtable names. \c |
527 | /// FuncNameStrings is a string composed of one or more encoded function name |
528 | /// strings, and \c VTableNameStrings composes of one or more encoded vtable |
529 | /// names. This interface is solely used by raw profile reader. |
530 | Error create(StringRef FuncNameStrings, StringRef VTableNameStrings); |
531 | |
532 | /// Initialize 'this' with the set of vtable names encoded in |
533 | /// \c CompressedVTableNames. |
534 | Error initVTableNamesFromCompressedStrings(StringRef CompressedVTableNames); |
535 | |
536 | /// This interface is used by reader of CoverageMapping test |
537 | /// format. |
538 | inline Error create(StringRef D, uint64_t BaseAddr); |
539 | |
540 | /// A wrapper interface to populate the PGO symtab with functions |
541 | /// decls from module \c M. This interface is used by transformation |
542 | /// passes such as indirect function call promotion. Variable \c InLTO |
543 | /// indicates if this is called from LTO optimization passes. |
544 | Error create(Module &M, bool InLTO = false); |
545 | |
546 | /// Create InstrProfSymtab from a set of names iteratable from |
547 | /// \p IterRange. This interface is used by IndexedProfReader. |
548 | template <typename NameIterRange> |
549 | Error create(const NameIterRange &IterRange); |
550 | |
551 | /// Create InstrProfSymtab from a set of function names and vtable |
552 | /// names iteratable from \p IterRange. This interface is used by |
553 | /// IndexedProfReader. |
554 | template <typename FuncNameIterRange, typename VTableNameIterRange> |
555 | Error create(const FuncNameIterRange &FuncIterRange, |
556 | const VTableNameIterRange &VTableIterRange); |
557 | |
558 | Error addSymbolName(StringRef SymbolName) { |
559 | if (SymbolName.empty()) |
560 | return make_error<InstrProfError>(Args: instrprof_error::malformed, |
561 | Args: "symbol name is empty" ); |
562 | |
563 | // Insert into NameTab so that MD5NameMap (a vector that will be sorted) |
564 | // won't have duplicated entries in the first place. |
565 | auto Ins = NameTab.insert(key: SymbolName); |
566 | if (Ins.second) { |
567 | MD5NameMap.push_back(x: std::make_pair( |
568 | x: IndexedInstrProf::ComputeHash(K: SymbolName), y: Ins.first->getKey())); |
569 | Sorted = false; |
570 | } |
571 | return Error::success(); |
572 | } |
573 | |
574 | /// The method name is kept since there are many callers. |
575 | /// It just forwards to 'addSymbolName'. |
576 | Error addFuncName(StringRef FuncName) { return addSymbolName(SymbolName: FuncName); } |
577 | |
578 | /// Adds VTableName as a known symbol, and inserts it to a map that |
579 | /// tracks all vtable names. |
580 | Error addVTableName(StringRef VTableName) { |
581 | if (Error E = addSymbolName(SymbolName: VTableName)) |
582 | return E; |
583 | |
584 | // Record VTableName. InstrProfWriter uses this set. The comment around |
585 | // class member explains why. |
586 | VTableNames.insert(key: VTableName); |
587 | return Error::success(); |
588 | } |
589 | |
590 | const StringSet<> &getVTableNames() const { return VTableNames; } |
591 | |
592 | /// Map a function address to its name's MD5 hash. This interface |
593 | /// is only used by the raw profiler reader. |
594 | void mapAddress(uint64_t Addr, uint64_t MD5Val) { |
595 | AddrToMD5Map.push_back(x: std::make_pair(x&: Addr, y&: MD5Val)); |
596 | } |
597 | |
598 | /// Map the address range (i.e., [start_address, end_address)) of a variable |
599 | /// to its names' MD5 hash. This interface is only used by the raw profile |
600 | /// reader. |
601 | void mapVTableAddress(uint64_t StartAddr, uint64_t EndAddr, uint64_t MD5Val) { |
602 | VTableAddrMap.insert(a: StartAddr, b: EndAddr, y: MD5Val); |
603 | } |
604 | |
605 | /// Return a function's hash, or 0, if the function isn't in this SymTab. |
606 | uint64_t getFunctionHashFromAddress(uint64_t Address); |
607 | |
608 | /// Return a vtable's hash, or 0 if the vtable doesn't exist in this SymTab. |
609 | uint64_t getVTableHashFromAddress(uint64_t Address); |
610 | |
611 | /// Return function's PGO name from the function name's symbol |
612 | /// address in the object file. If an error occurs, return |
613 | /// an empty string. |
614 | StringRef getFuncName(uint64_t FuncNameAddress, size_t NameSize); |
615 | |
616 | /// Return name of functions or global variables from the name's md5 hash |
617 | /// value. If not found, return an empty string. |
618 | inline StringRef getFuncOrVarName(uint64_t ValMD5Hash); |
619 | |
620 | /// Just like getFuncOrVarName, except that it will return literal string |
621 | /// 'External Symbol' if the function or global variable is external to |
622 | /// this symbol table. |
623 | inline StringRef getFuncOrVarNameIfDefined(uint64_t ValMD5Hash); |
624 | |
625 | /// True if Symbol is the value used to represent external symbols. |
626 | static bool isExternalSymbol(const StringRef &Symbol) { |
627 | return Symbol == InstrProfSymtab::getExternalSymbol(); |
628 | } |
629 | |
630 | /// Return function from the name's md5 hash. Return nullptr if not found. |
631 | inline Function *getFunction(uint64_t FuncMD5Hash); |
632 | |
633 | /// Return the name section data. |
634 | inline StringRef getNameData() const { return Data; } |
635 | |
636 | /// Dump the symbols in this table. |
637 | void dumpNames(raw_ostream &OS) const; |
638 | }; |
639 | |
640 | Error InstrProfSymtab::create(StringRef D, uint64_t BaseAddr) { |
641 | Data = D; |
642 | Address = BaseAddr; |
643 | return Error::success(); |
644 | } |
645 | |
646 | template <typename NameIterRange> |
647 | Error InstrProfSymtab::create(const NameIterRange &IterRange) { |
648 | for (auto Name : IterRange) |
649 | if (Error E = addFuncName(FuncName: Name)) |
650 | return E; |
651 | |
652 | finalizeSymtab(); |
653 | return Error::success(); |
654 | } |
655 | |
656 | template <typename FuncNameIterRange, typename VTableNameIterRange> |
657 | Error InstrProfSymtab::create(const FuncNameIterRange &FuncIterRange, |
658 | const VTableNameIterRange &VTableIterRange) { |
659 | // Iterate elements by StringRef rather than by const reference. |
660 | // StringRef is small enough, so the loop is efficient whether |
661 | // element in the range is std::string or StringRef. |
662 | for (StringRef Name : FuncIterRange) |
663 | if (Error E = addFuncName(FuncName: Name)) |
664 | return E; |
665 | |
666 | for (StringRef VTableName : VTableIterRange) |
667 | if (Error E = addVTableName(VTableName)) |
668 | return E; |
669 | |
670 | finalizeSymtab(); |
671 | return Error::success(); |
672 | } |
673 | |
674 | void InstrProfSymtab::finalizeSymtab() { |
675 | if (Sorted) |
676 | return; |
677 | llvm::sort(C&: MD5NameMap, Comp: less_first()); |
678 | llvm::sort(C&: MD5FuncMap, Comp: less_first()); |
679 | llvm::sort(C&: AddrToMD5Map, Comp: less_first()); |
680 | AddrToMD5Map.erase(first: std::unique(first: AddrToMD5Map.begin(), last: AddrToMD5Map.end()), |
681 | last: AddrToMD5Map.end()); |
682 | Sorted = true; |
683 | } |
684 | |
685 | StringRef InstrProfSymtab::getFuncOrVarNameIfDefined(uint64_t MD5Hash) { |
686 | StringRef ret = getFuncOrVarName(ValMD5Hash: MD5Hash); |
687 | if (ret.empty()) |
688 | return InstrProfSymtab::getExternalSymbol(); |
689 | return ret; |
690 | } |
691 | |
692 | StringRef InstrProfSymtab::getFuncOrVarName(uint64_t MD5Hash) { |
693 | finalizeSymtab(); |
694 | auto Result = llvm::lower_bound(Range&: MD5NameMap, Value&: MD5Hash, |
695 | C: [](const std::pair<uint64_t, StringRef> &LHS, |
696 | uint64_t RHS) { return LHS.first < RHS; }); |
697 | if (Result != MD5NameMap.end() && Result->first == MD5Hash) |
698 | return Result->second; |
699 | return StringRef(); |
700 | } |
701 | |
702 | Function* InstrProfSymtab::getFunction(uint64_t FuncMD5Hash) { |
703 | finalizeSymtab(); |
704 | auto Result = llvm::lower_bound(Range&: MD5FuncMap, Value&: FuncMD5Hash, |
705 | C: [](const std::pair<uint64_t, Function *> &LHS, |
706 | uint64_t RHS) { return LHS.first < RHS; }); |
707 | if (Result != MD5FuncMap.end() && Result->first == FuncMD5Hash) |
708 | return Result->second; |
709 | return nullptr; |
710 | } |
711 | |
712 | // To store the sums of profile count values, or the percentage of |
713 | // the sums of the total count values. |
714 | struct CountSumOrPercent { |
715 | uint64_t NumEntries; |
716 | double CountSum; |
717 | double ValueCounts[IPVK_Last - IPVK_First + 1]; |
718 | CountSumOrPercent() : NumEntries(0), CountSum(0.0f), ValueCounts() {} |
719 | void reset() { |
720 | NumEntries = 0; |
721 | CountSum = 0.0f; |
722 | for (double &VC : ValueCounts) |
723 | VC = 0.0f; |
724 | } |
725 | }; |
726 | |
727 | // Function level or program level overlap information. |
728 | struct OverlapStats { |
729 | enum OverlapStatsLevel { ProgramLevel, FunctionLevel }; |
730 | // Sum of the total count values for the base profile. |
731 | CountSumOrPercent Base; |
732 | // Sum of the total count values for the test profile. |
733 | CountSumOrPercent Test; |
734 | // Overlap lap score. Should be in range of [0.0f to 1.0f]. |
735 | CountSumOrPercent Overlap; |
736 | CountSumOrPercent Mismatch; |
737 | CountSumOrPercent Unique; |
738 | OverlapStatsLevel Level; |
739 | const std::string *BaseFilename; |
740 | const std::string *TestFilename; |
741 | StringRef FuncName; |
742 | uint64_t FuncHash; |
743 | bool Valid; |
744 | |
745 | OverlapStats(OverlapStatsLevel L = ProgramLevel) |
746 | : Level(L), BaseFilename(nullptr), TestFilename(nullptr), FuncHash(0), |
747 | Valid(false) {} |
748 | |
749 | void dump(raw_fd_ostream &OS) const; |
750 | |
751 | void setFuncInfo(StringRef Name, uint64_t Hash) { |
752 | FuncName = Name; |
753 | FuncHash = Hash; |
754 | } |
755 | |
756 | Error accumulateCounts(const std::string &BaseFilename, |
757 | const std::string &TestFilename, bool IsCS); |
758 | void addOneMismatch(const CountSumOrPercent &MismatchFunc); |
759 | void addOneUnique(const CountSumOrPercent &UniqueFunc); |
760 | |
761 | static inline double score(uint64_t Val1, uint64_t Val2, double Sum1, |
762 | double Sum2) { |
763 | if (Sum1 < 1.0f || Sum2 < 1.0f) |
764 | return 0.0f; |
765 | return std::min(a: Val1 / Sum1, b: Val2 / Sum2); |
766 | } |
767 | }; |
768 | |
769 | // This is used to filter the functions whose overlap information |
770 | // to be output. |
771 | struct OverlapFuncFilters { |
772 | uint64_t ValueCutoff; |
773 | const std::string NameFilter; |
774 | }; |
775 | |
776 | struct InstrProfValueSiteRecord { |
777 | /// Value profiling data pairs at a given value site. |
778 | std::list<InstrProfValueData> ValueData; |
779 | |
780 | InstrProfValueSiteRecord() { ValueData.clear(); } |
781 | template <class InputIterator> |
782 | InstrProfValueSiteRecord(InputIterator F, InputIterator L) |
783 | : ValueData(F, L) {} |
784 | |
785 | /// Sort ValueData ascending by Value |
786 | void sortByTargetValues() { |
787 | ValueData.sort( |
788 | comp: [](const InstrProfValueData &left, const InstrProfValueData &right) { |
789 | return left.Value < right.Value; |
790 | }); |
791 | } |
792 | /// Sort ValueData Descending by Count |
793 | inline void sortByCount(); |
794 | |
795 | /// Merge data from another InstrProfValueSiteRecord |
796 | /// Optionally scale merged counts by \p Weight. |
797 | void merge(InstrProfValueSiteRecord &Input, uint64_t Weight, |
798 | function_ref<void(instrprof_error)> Warn); |
799 | /// Scale up value profile data counts by N (Numerator) / D (Denominator). |
800 | void scale(uint64_t N, uint64_t D, function_ref<void(instrprof_error)> Warn); |
801 | |
802 | /// Compute the overlap b/w this record and Input record. |
803 | void overlap(InstrProfValueSiteRecord &Input, uint32_t ValueKind, |
804 | OverlapStats &Overlap, OverlapStats &FuncLevelOverlap); |
805 | }; |
806 | |
807 | /// Profiling information for a single function. |
808 | struct InstrProfRecord { |
809 | std::vector<uint64_t> Counts; |
810 | std::vector<uint8_t> BitmapBytes; |
811 | |
812 | InstrProfRecord() = default; |
813 | InstrProfRecord(std::vector<uint64_t> Counts) : Counts(std::move(Counts)) {} |
814 | InstrProfRecord(std::vector<uint64_t> Counts, |
815 | std::vector<uint8_t> BitmapBytes) |
816 | : Counts(std::move(Counts)), BitmapBytes(std::move(BitmapBytes)) {} |
817 | InstrProfRecord(InstrProfRecord &&) = default; |
818 | InstrProfRecord(const InstrProfRecord &RHS) |
819 | : Counts(RHS.Counts), BitmapBytes(RHS.BitmapBytes), |
820 | ValueData(RHS.ValueData |
821 | ? std::make_unique<ValueProfData>(args&: *RHS.ValueData) |
822 | : nullptr) {} |
823 | InstrProfRecord &operator=(InstrProfRecord &&) = default; |
824 | InstrProfRecord &operator=(const InstrProfRecord &RHS) { |
825 | Counts = RHS.Counts; |
826 | BitmapBytes = RHS.BitmapBytes; |
827 | if (!RHS.ValueData) { |
828 | ValueData = nullptr; |
829 | return *this; |
830 | } |
831 | if (!ValueData) |
832 | ValueData = std::make_unique<ValueProfData>(args&: *RHS.ValueData); |
833 | else |
834 | *ValueData = *RHS.ValueData; |
835 | return *this; |
836 | } |
837 | |
838 | /// Return the number of value profile kinds with non-zero number |
839 | /// of profile sites. |
840 | inline uint32_t getNumValueKinds() const; |
841 | /// Return the number of instrumented sites for ValueKind. |
842 | inline uint32_t getNumValueSites(uint32_t ValueKind) const; |
843 | |
844 | /// Return the total number of ValueData for ValueKind. |
845 | inline uint32_t getNumValueData(uint32_t ValueKind) const; |
846 | |
847 | /// Return the number of value data collected for ValueKind at profiling |
848 | /// site: Site. |
849 | inline uint32_t getNumValueDataForSite(uint32_t ValueKind, |
850 | uint32_t Site) const; |
851 | |
852 | /// Return the array of profiled values at \p Site. If \p TotalC |
853 | /// is not null, the total count of all target values at this site |
854 | /// will be stored in \c *TotalC. |
855 | inline std::unique_ptr<InstrProfValueData[]> |
856 | getValueForSite(uint32_t ValueKind, uint32_t Site, |
857 | uint64_t *TotalC = nullptr) const; |
858 | |
859 | /// Get the target value/counts of kind \p ValueKind collected at site |
860 | /// \p Site and store the result in array \p Dest. Return the total |
861 | /// counts of all target values at this site. |
862 | inline uint64_t getValueForSite(InstrProfValueData Dest[], uint32_t ValueKind, |
863 | uint32_t Site) const; |
864 | |
865 | /// Reserve space for NumValueSites sites. |
866 | inline void reserveSites(uint32_t ValueKind, uint32_t NumValueSites); |
867 | |
868 | /// Add ValueData for ValueKind at value Site. |
869 | void addValueData(uint32_t ValueKind, uint32_t Site, |
870 | InstrProfValueData *VData, uint32_t N, |
871 | InstrProfSymtab *SymTab); |
872 | |
873 | /// Merge the counts in \p Other into this one. |
874 | /// Optionally scale merged counts by \p Weight. |
875 | void merge(InstrProfRecord &Other, uint64_t Weight, |
876 | function_ref<void(instrprof_error)> Warn); |
877 | |
878 | /// Scale up profile counts (including value profile data) by |
879 | /// a factor of (N / D). |
880 | void scale(uint64_t N, uint64_t D, function_ref<void(instrprof_error)> Warn); |
881 | |
882 | /// Sort value profile data (per site) by count. |
883 | void sortValueData() { |
884 | for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) |
885 | for (auto &SR : getValueSitesForKind(ValueKind: Kind)) |
886 | SR.sortByCount(); |
887 | } |
888 | |
889 | /// Clear value data entries and edge counters. |
890 | void Clear() { |
891 | Counts.clear(); |
892 | clearValueData(); |
893 | } |
894 | |
895 | /// Clear value data entries |
896 | void clearValueData() { ValueData = nullptr; } |
897 | |
898 | /// Compute the sums of all counts and store in Sum. |
899 | void accumulateCounts(CountSumOrPercent &Sum) const; |
900 | |
901 | /// Compute the overlap b/w this IntrprofRecord and Other. |
902 | void overlap(InstrProfRecord &Other, OverlapStats &Overlap, |
903 | OverlapStats &FuncLevelOverlap, uint64_t ValueCutoff); |
904 | |
905 | /// Compute the overlap of value profile counts. |
906 | void overlapValueProfData(uint32_t ValueKind, InstrProfRecord &Src, |
907 | OverlapStats &Overlap, |
908 | OverlapStats &FuncLevelOverlap); |
909 | |
910 | enum CountPseudoKind { |
911 | NotPseudo = 0, |
912 | PseudoHot, |
913 | PseudoWarm, |
914 | }; |
915 | enum PseudoCountVal { |
916 | HotFunctionVal = -1, |
917 | WarmFunctionVal = -2, |
918 | }; |
919 | CountPseudoKind getCountPseudoKind() const { |
920 | uint64_t FirstCount = Counts[0]; |
921 | if (FirstCount == (uint64_t)HotFunctionVal) |
922 | return PseudoHot; |
923 | if (FirstCount == (uint64_t)WarmFunctionVal) |
924 | return PseudoWarm; |
925 | return NotPseudo; |
926 | } |
927 | void setPseudoCount(CountPseudoKind Kind) { |
928 | if (Kind == PseudoHot) |
929 | Counts[0] = (uint64_t)HotFunctionVal; |
930 | else if (Kind == PseudoWarm) |
931 | Counts[0] = (uint64_t)WarmFunctionVal; |
932 | } |
933 | |
934 | private: |
935 | struct ValueProfData { |
936 | std::vector<InstrProfValueSiteRecord> IndirectCallSites; |
937 | std::vector<InstrProfValueSiteRecord> MemOPSizes; |
938 | std::vector<InstrProfValueSiteRecord> VTableTargets; |
939 | }; |
940 | std::unique_ptr<ValueProfData> ValueData; |
941 | |
942 | MutableArrayRef<InstrProfValueSiteRecord> |
943 | getValueSitesForKind(uint32_t ValueKind) { |
944 | // Cast to /add/ const (should be an implicit_cast, ideally, if that's ever |
945 | // implemented in LLVM) to call the const overload of this function, then |
946 | // cast away the constness from the result. |
947 | auto AR = const_cast<const InstrProfRecord *>(this)->getValueSitesForKind( |
948 | ValueKind); |
949 | return MutableArrayRef( |
950 | const_cast<InstrProfValueSiteRecord *>(AR.data()), AR.size()); |
951 | } |
952 | ArrayRef<InstrProfValueSiteRecord> |
953 | getValueSitesForKind(uint32_t ValueKind) const { |
954 | if (!ValueData) |
955 | return std::nullopt; |
956 | switch (ValueKind) { |
957 | case IPVK_IndirectCallTarget: |
958 | return ValueData->IndirectCallSites; |
959 | case IPVK_MemOPSize: |
960 | return ValueData->MemOPSizes; |
961 | case IPVK_VTableTarget: |
962 | return ValueData->VTableTargets; |
963 | default: |
964 | llvm_unreachable("Unknown value kind!" ); |
965 | } |
966 | } |
967 | |
968 | std::vector<InstrProfValueSiteRecord> & |
969 | getOrCreateValueSitesForKind(uint32_t ValueKind) { |
970 | if (!ValueData) |
971 | ValueData = std::make_unique<ValueProfData>(); |
972 | switch (ValueKind) { |
973 | case IPVK_IndirectCallTarget: |
974 | return ValueData->IndirectCallSites; |
975 | case IPVK_MemOPSize: |
976 | return ValueData->MemOPSizes; |
977 | case IPVK_VTableTarget: |
978 | return ValueData->VTableTargets; |
979 | default: |
980 | llvm_unreachable("Unknown value kind!" ); |
981 | } |
982 | } |
983 | |
984 | // Map indirect call target name hash to name string. |
985 | uint64_t remapValue(uint64_t Value, uint32_t ValueKind, |
986 | InstrProfSymtab *SymTab); |
987 | |
988 | // Merge Value Profile data from Src record to this record for ValueKind. |
989 | // Scale merged value counts by \p Weight. |
990 | void mergeValueProfData(uint32_t ValkeKind, InstrProfRecord &Src, |
991 | uint64_t Weight, |
992 | function_ref<void(instrprof_error)> Warn); |
993 | |
994 | // Scale up value profile data count by N (Numerator) / D (Denominator). |
995 | void scaleValueProfData(uint32_t ValueKind, uint64_t N, uint64_t D, |
996 | function_ref<void(instrprof_error)> Warn); |
997 | }; |
998 | |
999 | struct NamedInstrProfRecord : InstrProfRecord { |
1000 | StringRef Name; |
1001 | uint64_t Hash; |
1002 | |
1003 | // We reserve this bit as the flag for context sensitive profile record. |
1004 | static const int CS_FLAG_IN_FUNC_HASH = 60; |
1005 | |
1006 | NamedInstrProfRecord() = default; |
1007 | NamedInstrProfRecord(StringRef Name, uint64_t Hash, |
1008 | std::vector<uint64_t> Counts) |
1009 | : InstrProfRecord(std::move(Counts)), Name(Name), Hash(Hash) {} |
1010 | NamedInstrProfRecord(StringRef Name, uint64_t Hash, |
1011 | std::vector<uint64_t> Counts, |
1012 | std::vector<uint8_t> BitmapBytes) |
1013 | : InstrProfRecord(std::move(Counts), std::move(BitmapBytes)), Name(Name), |
1014 | Hash(Hash) {} |
1015 | |
1016 | static bool hasCSFlagInHash(uint64_t FuncHash) { |
1017 | return ((FuncHash >> CS_FLAG_IN_FUNC_HASH) & 1); |
1018 | } |
1019 | static void setCSFlagInHash(uint64_t &FuncHash) { |
1020 | FuncHash |= ((uint64_t)1 << CS_FLAG_IN_FUNC_HASH); |
1021 | } |
1022 | }; |
1023 | |
1024 | uint32_t InstrProfRecord::getNumValueKinds() const { |
1025 | uint32_t NumValueKinds = 0; |
1026 | for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) |
1027 | NumValueKinds += !(getValueSitesForKind(ValueKind: Kind).empty()); |
1028 | return NumValueKinds; |
1029 | } |
1030 | |
1031 | uint32_t InstrProfRecord::getNumValueData(uint32_t ValueKind) const { |
1032 | uint32_t N = 0; |
1033 | for (const auto &SR : getValueSitesForKind(ValueKind)) |
1034 | N += SR.ValueData.size(); |
1035 | return N; |
1036 | } |
1037 | |
1038 | uint32_t InstrProfRecord::getNumValueSites(uint32_t ValueKind) const { |
1039 | return getValueSitesForKind(ValueKind).size(); |
1040 | } |
1041 | |
1042 | uint32_t InstrProfRecord::getNumValueDataForSite(uint32_t ValueKind, |
1043 | uint32_t Site) const { |
1044 | return getValueSitesForKind(ValueKind)[Site].ValueData.size(); |
1045 | } |
1046 | |
1047 | std::unique_ptr<InstrProfValueData[]> |
1048 | InstrProfRecord::getValueForSite(uint32_t ValueKind, uint32_t Site, |
1049 | uint64_t *TotalC) const { |
1050 | uint64_t Dummy = 0; |
1051 | uint64_t &TotalCount = (TotalC == nullptr ? Dummy : *TotalC); |
1052 | uint32_t N = getNumValueDataForSite(ValueKind, Site); |
1053 | if (N == 0) { |
1054 | TotalCount = 0; |
1055 | return std::unique_ptr<InstrProfValueData[]>(nullptr); |
1056 | } |
1057 | |
1058 | auto VD = std::make_unique<InstrProfValueData[]>(num: N); |
1059 | TotalCount = getValueForSite(Dest: VD.get(), ValueKind, Site); |
1060 | |
1061 | return VD; |
1062 | } |
1063 | |
1064 | uint64_t InstrProfRecord::getValueForSite(InstrProfValueData Dest[], |
1065 | uint32_t ValueKind, |
1066 | uint32_t Site) const { |
1067 | uint32_t I = 0; |
1068 | uint64_t TotalCount = 0; |
1069 | for (auto V : getValueSitesForKind(ValueKind)[Site].ValueData) { |
1070 | Dest[I].Value = V.Value; |
1071 | Dest[I].Count = V.Count; |
1072 | TotalCount = SaturatingAdd(X: TotalCount, Y: V.Count); |
1073 | I++; |
1074 | } |
1075 | return TotalCount; |
1076 | } |
1077 | |
1078 | void InstrProfRecord::reserveSites(uint32_t ValueKind, uint32_t NumValueSites) { |
1079 | if (!NumValueSites) |
1080 | return; |
1081 | getOrCreateValueSitesForKind(ValueKind).reserve(n: NumValueSites); |
1082 | } |
1083 | |
1084 | // Include definitions for value profile data |
1085 | #define INSTR_PROF_VALUE_PROF_DATA |
1086 | #include "llvm/ProfileData/InstrProfData.inc" |
1087 | |
1088 | void InstrProfValueSiteRecord::sortByCount() { |
1089 | ValueData.sort( |
1090 | comp: [](const InstrProfValueData &left, const InstrProfValueData &right) { |
1091 | return left.Count > right.Count; |
1092 | }); |
1093 | // Now truncate |
1094 | size_t max_s = INSTR_PROF_MAX_NUM_VAL_PER_SITE; |
1095 | if (ValueData.size() > max_s) |
1096 | ValueData.resize(new_size: max_s); |
1097 | } |
1098 | |
1099 | namespace IndexedInstrProf { |
1100 | |
1101 | enum class HashT : uint32_t { |
1102 | MD5, |
1103 | Last = MD5 |
1104 | }; |
1105 | |
1106 | inline uint64_t ComputeHash(HashT Type, StringRef K) { |
1107 | switch (Type) { |
1108 | case HashT::MD5: |
1109 | return MD5Hash(Str: K); |
1110 | } |
1111 | llvm_unreachable("Unhandled hash type" ); |
1112 | } |
1113 | |
1114 | const uint64_t Magic = 0x8169666f72706cff; // "\xfflprofi\x81" |
1115 | |
1116 | enum ProfVersion { |
1117 | // Version 1 is the first version. In this version, the value of |
1118 | // a key/value pair can only include profile data of a single function. |
1119 | // Due to this restriction, the number of block counters for a given |
1120 | // function is not recorded but derived from the length of the value. |
1121 | Version1 = 1, |
1122 | // The version 2 format supports recording profile data of multiple |
1123 | // functions which share the same key in one value field. To support this, |
1124 | // the number block counters is recorded as an uint64_t field right after the |
1125 | // function structural hash. |
1126 | Version2 = 2, |
1127 | // Version 3 supports value profile data. The value profile data is expected |
1128 | // to follow the block counter profile data. |
1129 | Version3 = 3, |
1130 | // In this version, profile summary data \c IndexedInstrProf::Summary is |
1131 | // stored after the profile header. |
1132 | Version4 = 4, |
1133 | // In this version, the frontend PGO stable hash algorithm defaults to V2. |
1134 | Version5 = 5, |
1135 | // In this version, the frontend PGO stable hash algorithm got fixed and |
1136 | // may produce hashes different from Version5. |
1137 | Version6 = 6, |
1138 | // An additional counter is added around logical operators. |
1139 | Version7 = 7, |
1140 | // An additional (optional) memory profile type is added. |
1141 | Version8 = 8, |
1142 | // Binary ids are added. |
1143 | Version9 = 9, |
1144 | // An additional (optional) temporal profile traces section is added. |
1145 | Version10 = 10, |
1146 | // An additional field is used for bitmap bytes. |
1147 | Version11 = 11, |
1148 | // VTable profiling, |
1149 | Version12 = 12, |
1150 | // The current version is 12. |
1151 | CurrentVersion = INSTR_PROF_INDEX_VERSION |
1152 | }; |
1153 | const uint64_t Version = ProfVersion::CurrentVersion; |
1154 | |
1155 | const HashT HashType = HashT::MD5; |
1156 | |
1157 | inline uint64_t ComputeHash(StringRef K) { return ComputeHash(Type: HashType, K); } |
1158 | |
1159 | // This structure defines the file header of the LLVM profile |
1160 | // data file in indexed-format. Please update llvm/docs/InstrProfileFormat.rst |
1161 | // as appropriate when updating the indexed profile format. |
1162 | struct { |
1163 | uint64_t ; |
1164 | // The lower 32 bits specify the version of the indexed profile. |
1165 | // The most significant 32 bits are reserved to specify the variant types of |
1166 | // the profile. |
1167 | uint64_t ; |
1168 | uint64_t ; // Becomes unused since version 4 |
1169 | uint64_t ; |
1170 | // This field records the offset of this hash table's metadata (i.e., the |
1171 | // number of buckets and entries), which follows right after the payload of |
1172 | // the entire hash table. |
1173 | uint64_t ; |
1174 | uint64_t ; |
1175 | uint64_t ; |
1176 | uint64_t ; |
1177 | uint64_t ; |
1178 | // New fields should only be added at the end to ensure that the size |
1179 | // computation is correct. The methods below need to be updated to ensure that |
1180 | // the new field is read correctly. |
1181 | |
1182 | // Reads a header struct from the buffer. |
1183 | static Expected<Header> (const unsigned char *Buffer); |
1184 | |
1185 | // Returns the size of the header in bytes for all valid fields based on the |
1186 | // version. I.e a older version header will return a smaller size. |
1187 | size_t () const; |
1188 | |
1189 | // Returns the format version in little endian. The header retains the version |
1190 | // in native endian of the compiler runtime. |
1191 | uint64_t () const; |
1192 | }; |
1193 | |
1194 | // Profile summary data recorded in the profile data file in indexed |
1195 | // format. It is introduced in version 4. The summary data follows |
1196 | // right after the profile file header. |
1197 | struct Summary { |
1198 | struct Entry { |
1199 | uint64_t Cutoff; ///< The required percentile of total execution count. |
1200 | uint64_t |
1201 | MinBlockCount; ///< The minimum execution count for this percentile. |
1202 | uint64_t NumBlocks; ///< Number of blocks >= the minumum execution count. |
1203 | }; |
1204 | // The field kind enumerator to assigned value mapping should remain |
1205 | // unchanged when a new kind is added or an old kind gets deleted in |
1206 | // the future. |
1207 | enum SummaryFieldKind { |
1208 | /// The total number of functions instrumented. |
1209 | TotalNumFunctions = 0, |
1210 | /// Total number of instrumented blocks/edges. |
1211 | TotalNumBlocks = 1, |
1212 | /// The maximal execution count among all functions. |
1213 | /// This field does not exist for profile data from IR based |
1214 | /// instrumentation. |
1215 | MaxFunctionCount = 2, |
1216 | /// Max block count of the program. |
1217 | MaxBlockCount = 3, |
1218 | /// Max internal block count of the program (excluding entry blocks). |
1219 | MaxInternalBlockCount = 4, |
1220 | /// The sum of all instrumented block counts. |
1221 | TotalBlockCount = 5, |
1222 | NumKinds = TotalBlockCount + 1 |
1223 | }; |
1224 | |
1225 | // The number of summmary fields following the summary header. |
1226 | uint64_t NumSummaryFields; |
1227 | // The number of Cutoff Entries (Summary::Entry) following summary fields. |
1228 | uint64_t NumCutoffEntries; |
1229 | |
1230 | Summary() = delete; |
1231 | Summary(uint32_t Size) { memset(s: this, c: 0, n: Size); } |
1232 | |
1233 | void operator delete(void *ptr) { ::operator delete(ptr); } |
1234 | |
1235 | static uint32_t getSize(uint32_t NumSumFields, uint32_t NumCutoffEntries) { |
1236 | return sizeof(Summary) + NumCutoffEntries * sizeof(Entry) + |
1237 | NumSumFields * sizeof(uint64_t); |
1238 | } |
1239 | |
1240 | const uint64_t *getSummaryDataBase() const { |
1241 | return reinterpret_cast<const uint64_t *>(this + 1); |
1242 | } |
1243 | |
1244 | uint64_t *getSummaryDataBase() { |
1245 | return reinterpret_cast<uint64_t *>(this + 1); |
1246 | } |
1247 | |
1248 | const Entry *getCutoffEntryBase() const { |
1249 | return reinterpret_cast<const Entry *>( |
1250 | &getSummaryDataBase()[NumSummaryFields]); |
1251 | } |
1252 | |
1253 | Entry *getCutoffEntryBase() { |
1254 | return reinterpret_cast<Entry *>(&getSummaryDataBase()[NumSummaryFields]); |
1255 | } |
1256 | |
1257 | uint64_t get(SummaryFieldKind K) const { |
1258 | return getSummaryDataBase()[K]; |
1259 | } |
1260 | |
1261 | void set(SummaryFieldKind K, uint64_t V) { |
1262 | getSummaryDataBase()[K] = V; |
1263 | } |
1264 | |
1265 | const Entry &getEntry(uint32_t I) const { return getCutoffEntryBase()[I]; } |
1266 | |
1267 | void setEntry(uint32_t I, const ProfileSummaryEntry &E) { |
1268 | Entry &ER = getCutoffEntryBase()[I]; |
1269 | ER.Cutoff = E.Cutoff; |
1270 | ER.MinBlockCount = E.MinCount; |
1271 | ER.NumBlocks = E.NumCounts; |
1272 | } |
1273 | }; |
1274 | |
1275 | inline std::unique_ptr<Summary> allocSummary(uint32_t TotalSize) { |
1276 | return std::unique_ptr<Summary>(new (::operator new(TotalSize)) |
1277 | Summary(TotalSize)); |
1278 | } |
1279 | |
1280 | } // end namespace IndexedInstrProf |
1281 | |
1282 | namespace RawInstrProf { |
1283 | |
1284 | // Version 1: First version |
1285 | // Version 2: Added value profile data section. Per-function control data |
1286 | // struct has more fields to describe value profile information. |
1287 | // Version 3: Compressed name section support. Function PGO name reference |
1288 | // from control data struct is changed from raw pointer to Name's MD5 value. |
1289 | // Version 4: ValueDataBegin and ValueDataSizes fields are removed from the |
1290 | // raw header. |
1291 | // Version 5: Bit 60 of FuncHash is reserved for the flag for the context |
1292 | // sensitive records. |
1293 | // Version 6: Added binary id. |
1294 | // Version 7: Reorder binary id and include version in signature. |
1295 | // Version 8: Use relative counter pointer. |
1296 | // Version 9: Added relative bitmap bytes pointer and count used by MC/DC. |
1297 | // Version 10: Added vtable, a new type of value profile data. |
1298 | const uint64_t Version = INSTR_PROF_RAW_VERSION; |
1299 | |
1300 | template <class IntPtrT> inline uint64_t getMagic(); |
1301 | template <> inline uint64_t getMagic<uint64_t>() { |
1302 | return INSTR_PROF_RAW_MAGIC_64; |
1303 | } |
1304 | |
1305 | template <> inline uint64_t getMagic<uint32_t>() { |
1306 | return INSTR_PROF_RAW_MAGIC_32; |
1307 | } |
1308 | |
1309 | // Per-function profile data header/control structure. |
1310 | // The definition should match the structure defined in |
1311 | // compiler-rt/lib/profile/InstrProfiling.h. |
1312 | // It should also match the synthesized type in |
1313 | // Transforms/Instrumentation/InstrProfiling.cpp:getOrCreateRegionCounters. |
1314 | template <class IntPtrT> struct alignas(8) ProfileData { |
1315 | #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Type Name; |
1316 | #include "llvm/ProfileData/InstrProfData.inc" |
1317 | }; |
1318 | |
1319 | template <class IntPtrT> struct alignas(8) VTableProfileData { |
1320 | #define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Init) Type Name; |
1321 | #include "llvm/ProfileData/InstrProfData.inc" |
1322 | }; |
1323 | |
1324 | // File header structure of the LLVM profile data in raw format. |
1325 | // The definition should match the header referenced in |
1326 | // compiler-rt/lib/profile/InstrProfilingFile.c and |
1327 | // InstrProfilingBuffer.c. |
1328 | struct { |
1329 | #define (Type, Name, Init) const Type Name; |
1330 | #include "llvm/ProfileData/InstrProfData.inc" |
1331 | }; |
1332 | |
1333 | } // end namespace RawInstrProf |
1334 | |
1335 | // Create the variable for the profile file name. |
1336 | void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput); |
1337 | |
1338 | // Whether to compress function names in profile records, and filenames in |
1339 | // code coverage mappings. Used by the Instrumentation library and unit tests. |
1340 | extern cl::opt<bool> DoInstrProfNameCompression; |
1341 | |
1342 | } // end namespace llvm |
1343 | #endif // LLVM_PROFILEDATA_INSTRPROF_H |
1344 | |