1 | /*===-- InstrProfData.inc - instr profiling runtime structures -*- C++ -*-=== *\ |
2 | |* |
3 | |* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | |* See https://llvm.org/LICENSE.txt for license information. |
5 | |* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | |* |
7 | \*===----------------------------------------------------------------------===*/ |
8 | /* |
9 | * This is the main file that defines all the data structure, signature, |
10 | * constant literals that are shared across profiling runtime library, |
11 | * compiler (instrumentation), and host tools (reader/writer). The entities |
12 | * defined in this file affect the profile runtime ABI, the raw profile format, |
13 | * or both. |
14 | * |
15 | * The file has two identical copies. The primary copy lives in LLVM and |
16 | * the other one sits in compiler-rt/lib/profile directory. To make changes |
17 | * in this file, first modify the primary copy and copy it over to compiler-rt. |
18 | * Testing of any change in this file can start only after the two copies are |
19 | * synced up. |
20 | * |
21 | * The first part of the file includes macros that defines types, names, and |
22 | * initializers for the member fields of the core data structures. The field |
23 | * declarations for one structure is enabled by defining the field activation |
24 | * macro associated with that structure. Only one field activation record |
25 | * can be defined at one time and the rest definitions will be filtered out by |
26 | * the preprocessor. |
27 | * |
28 | * Examples of how the template is used to instantiate structure definition: |
29 | * 1. To declare a structure: |
30 | * |
31 | * struct ProfData { |
32 | * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ |
33 | * Type Name; |
34 | * #include "llvm/ProfileData/InstrProfData.inc" |
35 | * }; |
36 | * |
37 | * 2. To construct LLVM type arrays for the struct type: |
38 | * |
39 | * Type *DataTypes[] = { |
40 | * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ |
41 | * LLVMType, |
42 | * #include "llvm/ProfileData/InstrProfData.inc" |
43 | * }; |
44 | * |
45 | * 4. To construct constant array for the initializers: |
46 | * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ |
47 | * Initializer, |
48 | * Constant *ConstantVals[] = { |
49 | * #include "llvm/ProfileData/InstrProfData.inc" |
50 | * }; |
51 | * |
52 | * |
53 | * The second part of the file includes definitions all other entities that |
54 | * are related to runtime ABI and format. When no field activation macro is |
55 | * defined, this file can be included to introduce the definitions. |
56 | * |
57 | \*===----------------------------------------------------------------------===*/ |
58 | |
59 | /* Functions marked with INSTR_PROF_VISIBILITY must have hidden visibility in |
60 | * the compiler runtime. */ |
61 | #ifndef INSTR_PROF_VISIBILITY |
62 | #define INSTR_PROF_VISIBILITY |
63 | #endif |
64 | |
65 | // clang-format off:consider re-enabling clang-format if auto-formatted C macros |
66 | // are readable (e.g., after `issue #82426` is fixed) |
67 | /* INSTR_PROF_DATA start. */ |
68 | /* Definition of member fields of the per-function control structure. */ |
69 | #ifndef INSTR_PROF_DATA |
70 | #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) |
71 | #else |
72 | #define INSTR_PROF_DATA_DEFINED |
73 | #endif |
74 | INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \ |
75 | ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ |
76 | IndexedInstrProf::ComputeHash(getPGOFuncNameVarInitializer(Inc->getName())))) |
77 | INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \ |
78 | ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ |
79 | Inc->getHash()->getZExtValue())) |
80 | INSTR_PROF_DATA(const IntPtrT, IntPtrTy, CounterPtr, RelativeCounterPtr) |
81 | INSTR_PROF_DATA(const IntPtrT, IntPtrTy, BitmapPtr, RelativeBitmapPtr) |
82 | /* This is used to map function pointers for the indirect call targets to |
83 | * function name hashes during the conversion from raw to merged profile |
84 | * data. |
85 | */ |
86 | INSTR_PROF_DATA(const IntPtrT, llvm::PointerType::getUnqual(Ctx), FunctionPointer, \ |
87 | FunctionAddr) |
88 | INSTR_PROF_DATA(IntPtrT, llvm::PointerType::getUnqual(Ctx), Values, \ |
89 | ValuesPtrExpr) |
90 | INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumCounters, \ |
91 | ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumCounters)) |
92 | INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \ |
93 | ConstantArray::get(Int16ArrayTy, Int16ArrayVals)) \ |
94 | INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumBitmapBytes, \ |
95 | ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumBitmapBytes)) |
96 | #undef INSTR_PROF_DATA |
97 | /* INSTR_PROF_DATA end. */ |
98 | |
99 | /* For a virtual table object, record the name hash to associate profiled |
100 | * addresses with global variables, and record {starting address, size in bytes} |
101 | * to map the profiled virtual table (which usually have an offset from the |
102 | * starting address) back to a virtual table object. */ |
103 | #ifndef INSTR_PROF_VTABLE_DATA |
104 | #define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Initializer) |
105 | #else |
106 | #define INSTR_PROF_VTABLE_DATA_DEFINED |
107 | #endif |
108 | INSTR_PROF_VTABLE_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), \ |
109 | VTableNameHash, ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ |
110 | IndexedInstrProf::ComputeHash(PGOVTableName))) |
111 | INSTR_PROF_VTABLE_DATA(const IntPtrT, llvm::PointerType::getUnqual(Ctx), \ |
112 | VTablePointer, VTableAddr) |
113 | INSTR_PROF_VTABLE_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), VTableSize, \ |
114 | ConstantInt::get(llvm::Type::getInt32Ty(Ctx), \ |
115 | VTableSizeVal)) |
116 | #undef INSTR_PROF_VTABLE_DATA |
117 | /* INSTR_PROF_VTABLE_DATA end. */ |
118 | |
119 | /* This is an internal data structure used by value profiler. It |
120 | * is defined here to allow serialization code sharing by LLVM |
121 | * to be used in unit test. |
122 | * |
123 | * typedef struct ValueProfNode { |
124 | * // InstrProfValueData VData; |
125 | * uint64_t Value; |
126 | * uint64_t Count; |
127 | * struct ValueProfNode *Next; |
128 | * } ValueProfNode; |
129 | */ |
130 | /* INSTR_PROF_VALUE_NODE start. */ |
131 | #ifndef INSTR_PROF_VALUE_NODE |
132 | #define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Initializer) |
133 | #else |
134 | #define INSTR_PROF_DATA_DEFINED |
135 | #endif |
136 | INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Value, \ |
137 | ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0)) |
138 | INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Count, \ |
139 | ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0)) |
140 | INSTR_PROF_VALUE_NODE(PtrToNodeT, llvm::PointerType::getUnqual(Ctx), Next, \ |
141 | ConstantInt::get(llvm::PointerType::getUnqual(Ctx), 0)) |
142 | #undef INSTR_PROF_VALUE_NODE |
143 | /* INSTR_PROF_VALUE_NODE end. */ |
144 | |
145 | /* INSTR_PROF_RAW_HEADER start */ |
146 | /* Definition of member fields of the raw profile header data structure. */ |
147 | /* Please update llvm/docs/InstrProfileFormat.rst as appropriate when updating |
148 | raw profile format. */ |
149 | #ifndef INSTR_PROF_RAW_HEADER |
150 | #define (Type, Name, Initializer) |
151 | #else |
152 | #define INSTR_PROF_DATA_DEFINED |
153 | #endif |
154 | INSTR_PROF_RAW_HEADER(uint64_t, Magic, __llvm_profile_get_magic()) |
155 | INSTR_PROF_RAW_HEADER(uint64_t, Version, __llvm_profile_get_version()) |
156 | INSTR_PROF_RAW_HEADER(uint64_t, BinaryIdsSize, __llvm_write_binary_ids(NULL)) |
157 | INSTR_PROF_RAW_HEADER(uint64_t, NumData, NumData) |
158 | INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesBeforeCounters, PaddingBytesBeforeCounters) |
159 | INSTR_PROF_RAW_HEADER(uint64_t, NumCounters, NumCounters) |
160 | INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterCounters, PaddingBytesAfterCounters) |
161 | INSTR_PROF_RAW_HEADER(uint64_t, NumBitmapBytes, NumBitmapBytes) |
162 | INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterBitmapBytes, PaddingBytesAfterBitmapBytes) |
163 | INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize) |
164 | INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, |
165 | (uintptr_t)CountersBegin - (uintptr_t)DataBegin) |
166 | INSTR_PROF_RAW_HEADER(uint64_t, BitmapDelta, |
167 | (uintptr_t)BitmapBegin - (uintptr_t)DataBegin) |
168 | INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) |
169 | INSTR_PROF_RAW_HEADER(uint64_t, NumVTables, NumVTables) |
170 | INSTR_PROF_RAW_HEADER(uint64_t, VNamesSize, VNamesSize) |
171 | INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) |
172 | #undef INSTR_PROF_RAW_HEADER |
173 | /* INSTR_PROF_RAW_HEADER end */ |
174 | |
175 | /* VALUE_PROF_FUNC_PARAM start */ |
176 | /* Definition of parameter types of the runtime API used to do value profiling |
177 | * for a given value site. |
178 | */ |
179 | #ifndef VALUE_PROF_FUNC_PARAM |
180 | #define VALUE_PROF_FUNC_PARAM(ArgType, ArgName, ArgLLVMType) |
181 | #define INSTR_PROF_COMMA |
182 | #else |
183 | #define INSTR_PROF_DATA_DEFINED |
184 | #define INSTR_PROF_COMMA , |
185 | #endif |
186 | VALUE_PROF_FUNC_PARAM(uint64_t, TargetValue, Type::getInt64Ty(Ctx)) \ |
187 | INSTR_PROF_COMMA |
188 | VALUE_PROF_FUNC_PARAM(void *, Data, PointerType::getUnqual(Ctx)) INSTR_PROF_COMMA |
189 | VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx)) |
190 | #undef VALUE_PROF_FUNC_PARAM |
191 | #undef INSTR_PROF_COMMA |
192 | /* VALUE_PROF_FUNC_PARAM end */ |
193 | |
194 | /* VALUE_PROF_KIND start */ |
195 | #ifndef VALUE_PROF_KIND |
196 | #define VALUE_PROF_KIND(Enumerator, Value, Descr) |
197 | #else |
198 | #define INSTR_PROF_DATA_DEFINED |
199 | #endif |
200 | /* For indirect function call value profiling, the addresses of the target |
201 | * functions are profiled by the instrumented code. The target addresses are |
202 | * written in the raw profile data and converted to target function name's MD5 |
203 | * hash by the profile reader during deserialization. Typically, this happens |
204 | * when the raw profile data is read during profile merging. |
205 | * |
206 | * For this remapping the ProfData is used. ProfData contains both the function |
207 | * name hash and the function address. |
208 | */ |
209 | VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0, "indirect call target" ) |
210 | /* For memory intrinsic functions size profiling. */ |
211 | VALUE_PROF_KIND(IPVK_MemOPSize, 1, "memory intrinsic functions size" ) |
212 | /* For virtual table address profiling, the address point of the virtual table |
213 | * (i.e., the address contained in objects pointing to a virtual table) are |
214 | * profiled. Note this may not be the address of the per C++ class virtual table |
215 | * object (e.g., there might be an offset). |
216 | * |
217 | * The profiled addresses are stored in raw profile, together with the following |
218 | * two types of information. |
219 | * 1. The (starting and ending) addresses of per C++ class virtual table objects. |
220 | * 2. The (compressed) virtual table object names. |
221 | * RawInstrProfReader converts profiled virtual table addresses to virtual table |
222 | * objects' MD5 hash. |
223 | */ |
224 | VALUE_PROF_KIND(IPVK_VTableTarget, 2, "The profiled address point of the vtable" ) |
225 | /* These two kinds must be the last to be |
226 | * declared. This is to make sure the string |
227 | * array created with the template can be |
228 | * indexed with the kind value. |
229 | */ |
230 | VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget, "first" ) |
231 | VALUE_PROF_KIND(IPVK_Last, IPVK_VTableTarget, "last" ) |
232 | |
233 | #undef VALUE_PROF_KIND |
234 | /* VALUE_PROF_KIND end */ |
235 | |
236 | #undef COVMAP_V2_OR_V3 |
237 | #ifdef COVMAP_V2 |
238 | #define COVMAP_V2_OR_V3 |
239 | #endif |
240 | #ifdef COVMAP_V3 |
241 | #define COVMAP_V2_OR_V3 |
242 | #endif |
243 | |
244 | /* COVMAP_FUNC_RECORD start */ |
245 | /* Definition of member fields of the function record structure in coverage |
246 | * map. |
247 | */ |
248 | #ifndef COVMAP_FUNC_RECORD |
249 | #define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Initializer) |
250 | #else |
251 | #define INSTR_PROF_DATA_DEFINED |
252 | #endif |
253 | #ifdef COVMAP_V1 |
254 | COVMAP_FUNC_RECORD(const IntPtrT, llvm::PointerType::getUnqual(Ctx), \ |
255 | NamePtr, llvm::ConstantExpr::getBitCast(NamePtr, \ |
256 | llvm::PointerType::getUnqual(Ctx))) |
257 | COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), NameSize, \ |
258 | llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), \ |
259 | NameValue.size())) |
260 | #endif |
261 | #ifdef COVMAP_V2_OR_V3 |
262 | COVMAP_FUNC_RECORD(const int64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \ |
263 | llvm::ConstantInt::get( \ |
264 | llvm::Type::getInt64Ty(Ctx), NameHash)) |
265 | #endif |
266 | COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), DataSize, \ |
267 | llvm::ConstantInt::get( \ |
268 | llvm::Type::getInt32Ty(Ctx), CoverageMapping.size())) |
269 | COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \ |
270 | llvm::ConstantInt::get( \ |
271 | llvm::Type::getInt64Ty(Ctx), FuncHash)) |
272 | #ifdef COVMAP_V3 |
273 | COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FilenamesRef, \ |
274 | llvm::ConstantInt::get( \ |
275 | llvm::Type::getInt64Ty(Ctx), FilenamesRef)) |
276 | COVMAP_FUNC_RECORD(const char, \ |
277 | llvm::ArrayType::get(llvm::Type::getInt8Ty(Ctx), \ |
278 | CoverageMapping.size()), \ |
279 | CoverageMapping, |
280 | llvm::ConstantDataArray::getRaw( \ |
281 | CoverageMapping, CoverageMapping.size(), \ |
282 | llvm::Type::getInt8Ty(Ctx))) |
283 | #endif |
284 | #undef COVMAP_FUNC_RECORD |
285 | /* COVMAP_FUNC_RECORD end. */ |
286 | |
287 | /* COVMAP_HEADER start */ |
288 | /* Definition of member fields of coverage map header. |
289 | */ |
290 | #ifndef COVMAP_HEADER |
291 | #define (Type, LLVMType, Name, Initializer) |
292 | #else |
293 | #define INSTR_PROF_DATA_DEFINED |
294 | #endif |
295 | COVMAP_HEADER(uint32_t, Int32Ty, NRecords, \ |
296 | llvm::ConstantInt::get(Int32Ty, NRecords)) |
297 | COVMAP_HEADER(uint32_t, Int32Ty, FilenamesSize, \ |
298 | llvm::ConstantInt::get(Int32Ty, FilenamesSize)) |
299 | COVMAP_HEADER(uint32_t, Int32Ty, CoverageSize, \ |
300 | llvm::ConstantInt::get(Int32Ty, CoverageMappingSize)) |
301 | COVMAP_HEADER(uint32_t, Int32Ty, Version, \ |
302 | llvm::ConstantInt::get(Int32Ty, CovMapVersion::CurrentVersion)) |
303 | #undef COVMAP_HEADER |
304 | /* COVMAP_HEADER end. */ |
305 | |
306 | /* COVINIT_FUNC start */ |
307 | #ifndef COVINIT_FUNC |
308 | #define COVINIT_FUNC(Type, LLVMType, Name, Initializer) |
309 | #else |
310 | #define INSTR_PROF_DATA_DEFINED |
311 | #endif |
312 | COVINIT_FUNC(IntPtrT, llvm::PointerType::getUnqual(Ctx), WriteoutFunction, \ |
313 | WriteoutF) |
314 | COVINIT_FUNC(IntPtrT, llvm::PointerType::getUnqual(Ctx), ResetFunction, \ |
315 | ResetF) |
316 | #undef COVINIT_FUNC |
317 | /* COVINIT_FUNC end */ |
318 | |
319 | #ifdef INSTR_PROF_SECT_ENTRY |
320 | #define INSTR_PROF_DATA_DEFINED |
321 | INSTR_PROF_SECT_ENTRY(IPSK_data, \ |
322 | INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON), \ |
323 | INSTR_PROF_DATA_COFF, "__DATA," ) |
324 | INSTR_PROF_SECT_ENTRY(IPSK_cnts, \ |
325 | INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON), \ |
326 | INSTR_PROF_CNTS_COFF, "__DATA," ) |
327 | INSTR_PROF_SECT_ENTRY(IPSK_bitmap, \ |
328 | INSTR_PROF_QUOTE(INSTR_PROF_BITS_COMMON), \ |
329 | INSTR_PROF_BITS_COFF, "__DATA," ) |
330 | INSTR_PROF_SECT_ENTRY(IPSK_name, \ |
331 | INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \ |
332 | INSTR_PROF_NAME_COFF, "__DATA," ) |
333 | INSTR_PROF_SECT_ENTRY(IPSK_vname, \ |
334 | INSTR_PROF_QUOTE(INSTR_PROF_VNAME_COMMON), \ |
335 | INSTR_PROF_VNAME_COFF, "__DATA," ) |
336 | INSTR_PROF_SECT_ENTRY(IPSK_vals, \ |
337 | INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON), \ |
338 | INSTR_PROF_VALS_COFF, "__DATA," ) |
339 | INSTR_PROF_SECT_ENTRY(IPSK_vnodes, \ |
340 | INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON), \ |
341 | INSTR_PROF_VNODES_COFF, "__DATA," ) |
342 | INSTR_PROF_SECT_ENTRY(IPSK_vtab, \ |
343 | INSTR_PROF_QUOTE(INSTR_PROF_VTAB_COMMON), \ |
344 | INSTR_PROF_VTAB_COFF, "__DATA," ) |
345 | INSTR_PROF_SECT_ENTRY(IPSK_covmap, \ |
346 | INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON), \ |
347 | INSTR_PROF_COVMAP_COFF, "__LLVM_COV," ) |
348 | INSTR_PROF_SECT_ENTRY(IPSK_covfun, \ |
349 | INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON), \ |
350 | INSTR_PROF_COVFUN_COFF, "__LLVM_COV," ) |
351 | INSTR_PROF_SECT_ENTRY(IPSK_covdata, \ |
352 | INSTR_PROF_QUOTE(INSTR_PROF_COVDATA_COMMON), \ |
353 | INSTR_PROF_COVDATA_COFF, "__LLVM_COV," ) |
354 | INSTR_PROF_SECT_ENTRY(IPSK_covname, \ |
355 | INSTR_PROF_QUOTE(INSTR_PROF_COVNAME_COMMON), \ |
356 | INSTR_PROF_COVNAME_COFF, "__LLVM_COV," ) |
357 | INSTR_PROF_SECT_ENTRY(IPSK_covinit, \ |
358 | INSTR_PROF_QUOTE(INSTR_PROF_COVINIT_COMMON), \ |
359 | INSTR_PROF_COVINIT_COFF, "__LLVM_COV," ) |
360 | |
361 | #undef INSTR_PROF_SECT_ENTRY |
362 | #endif |
363 | |
364 | |
365 | #ifdef INSTR_PROF_VALUE_PROF_DATA |
366 | #define INSTR_PROF_DATA_DEFINED |
367 | |
368 | #define INSTR_PROF_MAX_NUM_VAL_PER_SITE 255 |
369 | /*! |
370 | * This is the header of the data structure that defines the on-disk |
371 | * layout of the value profile data of a particular kind for one function. |
372 | */ |
373 | typedef struct ValueProfRecord { |
374 | /* The kind of the value profile record. */ |
375 | uint32_t Kind; |
376 | /* |
377 | * The number of value profile sites. It is guaranteed to be non-zero; |
378 | * otherwise the record for this kind won't be emitted. |
379 | */ |
380 | uint32_t NumValueSites; |
381 | /* |
382 | * The first element of the array that stores the number of profiled |
383 | * values for each value site. The size of the array is NumValueSites. |
384 | * Since NumValueSites is greater than zero, there is at least one |
385 | * element in the array. |
386 | */ |
387 | uint8_t SiteCountArray[1]; |
388 | |
389 | /* |
390 | * The fake declaration is for documentation purpose only. |
391 | * Align the start of next field to be on 8 byte boundaries. |
392 | uint8_t Padding[X]; |
393 | */ |
394 | |
395 | /* The array of value profile data. The size of the array is the sum |
396 | * of all elements in SiteCountArray[]. |
397 | InstrProfValueData ValueData[]; |
398 | */ |
399 | |
400 | #ifdef __cplusplus |
401 | /*! |
402 | * Return the number of value sites. |
403 | */ |
404 | uint32_t getNumValueSites() const { return NumValueSites; } |
405 | /*! |
406 | * Read data from this record and save it to Record. |
407 | */ |
408 | LLVM_ABI void deserializeTo(InstrProfRecord &Record, |
409 | InstrProfSymtab *SymTab); |
410 | /* |
411 | * In-place byte swap: |
412 | * Do byte swap for this instance. \c Old is the original order before |
413 | * the swap, and \c New is the New byte order. |
414 | */ |
415 | LLVM_ABI void swapBytes(llvm::endianness Old, llvm::endianness New); |
416 | #endif |
417 | } ValueProfRecord; |
418 | |
419 | /*! |
420 | * Per-function header/control data structure for value profiling |
421 | * data in indexed format. |
422 | */ |
423 | typedef struct ValueProfData { |
424 | /* |
425 | * Total size in bytes including this field. It must be a multiple |
426 | * of sizeof(uint64_t). |
427 | */ |
428 | uint32_t TotalSize; |
429 | /* |
430 | *The number of value profile kinds that has value profile data. |
431 | * In this implementation, a value profile kind is considered to |
432 | * have profile data if the number of value profile sites for the |
433 | * kind is not zero. More aggressively, the implementation can |
434 | * choose to check the actual data value: if none of the value sites |
435 | * has any profiled values, the kind can be skipped. |
436 | */ |
437 | uint32_t NumValueKinds; |
438 | |
439 | /* |
440 | * Following are a sequence of variable length records. The prefix/header |
441 | * of each record is defined by ValueProfRecord type. The number of |
442 | * records is NumValueKinds. |
443 | * ValueProfRecord Record_1; |
444 | * ValueProfRecord Record_N; |
445 | */ |
446 | |
447 | #if __cplusplus |
448 | /*! |
449 | * Return the total size in bytes of the on-disk value profile data |
450 | * given the data stored in Record. |
451 | */ |
452 | LLVM_ABI static uint32_t getSize(const InstrProfRecord &Record); |
453 | /*! |
454 | * Return a pointer to \c ValueProfData instance ready to be streamed. |
455 | */ |
456 | LLVM_ABI static std::unique_ptr<ValueProfData> |
457 | serializeFrom(const InstrProfRecord &Record); |
458 | /*! |
459 | * Check the integrity of the record. |
460 | */ |
461 | LLVM_ABI Error checkIntegrity(); |
462 | /*! |
463 | * Return a pointer to \c ValueProfileData instance ready to be read. |
464 | * All data in the instance are properly byte swapped. The input |
465 | * data is assumed to be in little endian order. |
466 | */ |
467 | LLVM_ABI static Expected<std::unique_ptr<ValueProfData>> |
468 | getValueProfData(const unsigned char *SrcBuffer, |
469 | const unsigned char *const SrcBufferEnd, |
470 | llvm::endianness SrcDataEndianness); |
471 | /*! |
472 | * Swap byte order from \c Endianness order to host byte order. |
473 | */ |
474 | LLVM_ABI void swapBytesToHost(llvm::endianness Endianness); |
475 | /*! |
476 | * Swap byte order from host byte order to \c Endianness order. |
477 | */ |
478 | LLVM_ABI void swapBytesFromHost(llvm::endianness Endianness); |
479 | /*! |
480 | * Return the total size of \c ValueProfileData. |
481 | */ |
482 | LLVM_ABI uint32_t getSize() const { return TotalSize; } |
483 | /*! |
484 | * Read data from this data and save it to \c Record. |
485 | */ |
486 | LLVM_ABI void deserializeTo(InstrProfRecord &Record, |
487 | InstrProfSymtab *SymTab); |
488 | void operator delete(void *ptr) { ::operator delete(ptr); } |
489 | #endif |
490 | } ValueProfData; |
491 | |
492 | /* |
493 | * The closure is designed to abstract away two types of value profile data: |
494 | * - InstrProfRecord which is the primary data structure used to |
495 | * represent profile data in host tools (reader, writer, and profile-use) |
496 | * - value profile runtime data structure suitable to be used by C |
497 | * runtime library. |
498 | * |
499 | * Both sources of data need to serialize to disk/memory-buffer in common |
500 | * format: ValueProfData. The abstraction allows compiler-rt's raw profiler |
501 | * writer to share the same format and code with indexed profile writer. |
502 | * |
503 | * For documentation of the member methods below, refer to corresponding methods |
504 | * in class InstrProfRecord. |
505 | */ |
506 | typedef struct ValueProfRecordClosure { |
507 | const void *Record; |
508 | uint32_t (*GetNumValueKinds)(const void *Record); |
509 | uint32_t (*GetNumValueSites)(const void *Record, uint32_t VKind); |
510 | uint32_t (*GetNumValueData)(const void *Record, uint32_t VKind); |
511 | uint32_t (*GetNumValueDataForSite)(const void *R, uint32_t VK, uint32_t S); |
512 | |
513 | /* |
514 | * After extracting the value profile data from the value profile record, |
515 | * this method is used to map the in-memory value to on-disk value. If |
516 | * the method is null, value will be written out untranslated. |
517 | */ |
518 | uint64_t (*RemapValueData)(uint32_t, uint64_t Value); |
519 | void (*GetValueForSite)(const void *R, InstrProfValueData *Dst, uint32_t K, |
520 | uint32_t S); |
521 | ValueProfData *(*AllocValueProfData)(size_t TotalSizeInBytes); |
522 | } ValueProfRecordClosure; |
523 | |
524 | INSTR_PROF_VISIBILITY ValueProfRecord * |
525 | getFirstValueProfRecord(ValueProfData *VPD); |
526 | INSTR_PROF_VISIBILITY ValueProfRecord * |
527 | getValueProfRecordNext(ValueProfRecord *VPR); |
528 | INSTR_PROF_VISIBILITY InstrProfValueData * |
529 | getValueProfRecordValueData(ValueProfRecord *VPR); |
530 | INSTR_PROF_VISIBILITY uint32_t |
531 | getValueProfRecordHeaderSize(uint32_t NumValueSites); |
532 | |
533 | #undef INSTR_PROF_VALUE_PROF_DATA |
534 | #endif /* INSTR_PROF_VALUE_PROF_DATA */ |
535 | |
536 | |
537 | #ifdef INSTR_PROF_COMMON_API_IMPL |
538 | #define INSTR_PROF_DATA_DEFINED |
539 | #ifdef __cplusplus |
540 | #define INSTR_PROF_INLINE inline |
541 | #define INSTR_PROF_NULLPTR nullptr |
542 | #else |
543 | #define INSTR_PROF_INLINE |
544 | #define INSTR_PROF_NULLPTR NULL |
545 | #endif |
546 | |
547 | #ifndef offsetof |
548 | #define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER) |
549 | #endif |
550 | |
551 | // clang-format on |
552 | |
553 | /*! |
554 | * Return the \c ValueProfRecord header size including the |
555 | * padding bytes. |
556 | */ |
557 | INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint32_t |
558 | getValueProfRecordHeaderSize(uint32_t NumValueSites) { |
559 | uint32_t Size = offsetof(ValueProfRecord, SiteCountArray) + |
560 | sizeof(uint8_t) * NumValueSites; |
561 | /* Round the size to multiple of 8 bytes. */ |
562 | Size = (Size + 7) & ~7; |
563 | return Size; |
564 | } |
565 | |
566 | /*! |
567 | * Return the total size of the value profile record including the |
568 | * header and the value data. |
569 | */ |
570 | INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint32_t |
571 | getValueProfRecordSize(uint32_t NumValueSites, uint32_t NumValueData) { |
572 | return getValueProfRecordHeaderSize(NumValueSites) + |
573 | sizeof(InstrProfValueData) * NumValueData; |
574 | } |
575 | |
576 | /*! |
577 | * Return the pointer to the start of value data array. |
578 | */ |
579 | INSTR_PROF_VISIBILITY INSTR_PROF_INLINE InstrProfValueData * |
580 | getValueProfRecordValueData(ValueProfRecord *This) { |
581 | return (InstrProfValueData *)((char *)This + getValueProfRecordHeaderSize( |
582 | This->NumValueSites)); |
583 | } |
584 | |
585 | /*! |
586 | * Return the total number of value data for \c This record. |
587 | */ |
588 | INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint32_t |
589 | getValueProfRecordNumValueData(ValueProfRecord *This) { |
590 | uint32_t NumValueData = 0; |
591 | uint32_t I; |
592 | for (I = 0; I < This->NumValueSites; I++) |
593 | NumValueData += This->SiteCountArray[I]; |
594 | return NumValueData; |
595 | } |
596 | |
597 | /*! |
598 | * Use this method to advance to the next \c This \c ValueProfRecord. |
599 | */ |
600 | INSTR_PROF_VISIBILITY INSTR_PROF_INLINE ValueProfRecord * |
601 | getValueProfRecordNext(ValueProfRecord *This) { |
602 | uint32_t NumValueData = getValueProfRecordNumValueData(This); |
603 | return (ValueProfRecord *)((char *)This + |
604 | getValueProfRecordSize(This->NumValueSites, |
605 | NumValueData)); |
606 | } |
607 | |
608 | /*! |
609 | * Return the first \c ValueProfRecord instance. |
610 | */ |
611 | INSTR_PROF_VISIBILITY INSTR_PROF_INLINE ValueProfRecord * |
612 | getFirstValueProfRecord(ValueProfData *This) { |
613 | return (ValueProfRecord *)((char *)This + sizeof(ValueProfData)); |
614 | } |
615 | |
616 | /* Closure based interfaces. */ |
617 | |
618 | /*! |
619 | * Return the total size in bytes of the on-disk value profile data |
620 | * given the data stored in Record. |
621 | */ |
622 | INSTR_PROF_VISIBILITY uint32_t |
623 | getValueProfDataSize(ValueProfRecordClosure *Closure) { |
624 | uint32_t Kind; |
625 | uint32_t TotalSize = sizeof(ValueProfData); |
626 | const void *Record = Closure->Record; |
627 | |
628 | for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) { |
629 | uint32_t NumValueSites = Closure->GetNumValueSites(Record, Kind); |
630 | if (!NumValueSites) |
631 | continue; |
632 | TotalSize += getValueProfRecordSize(NumValueSites, |
633 | Closure->GetNumValueData(Record, Kind)); |
634 | } |
635 | return TotalSize; |
636 | } |
637 | |
638 | /*! |
639 | * Extract value profile data of a function for the profile kind \c ValueKind |
640 | * from the \c Closure and serialize the data into \c This record instance. |
641 | */ |
642 | INSTR_PROF_VISIBILITY void |
643 | serializeValueProfRecordFrom(ValueProfRecord *This, |
644 | ValueProfRecordClosure *Closure, |
645 | uint32_t ValueKind, uint32_t NumValueSites) { |
646 | uint32_t S; |
647 | const void *Record = Closure->Record; |
648 | This->Kind = ValueKind; |
649 | This->NumValueSites = NumValueSites; |
650 | InstrProfValueData *DstVD = getValueProfRecordValueData(This); |
651 | |
652 | for (S = 0; S < NumValueSites; S++) { |
653 | uint32_t ND = Closure->GetNumValueDataForSite(Record, ValueKind, S); |
654 | This->SiteCountArray[S] = ND; |
655 | Closure->GetValueForSite(Record, DstVD, ValueKind, S); |
656 | DstVD += ND; |
657 | } |
658 | } |
659 | |
660 | /*! |
661 | * Extract value profile data of a function from the \c Closure |
662 | * and serialize the data into \c DstData if it is not NULL or heap |
663 | * memory allocated by the \c Closure's allocator method. If \c |
664 | * DstData is not null, the caller is expected to set the TotalSize |
665 | * in DstData. |
666 | */ |
667 | INSTR_PROF_VISIBILITY ValueProfData * |
668 | serializeValueProfDataFrom(ValueProfRecordClosure *Closure, |
669 | ValueProfData *DstData) { |
670 | uint32_t Kind; |
671 | uint32_t TotalSize = |
672 | DstData ? DstData->TotalSize : getValueProfDataSize(Closure); |
673 | |
674 | ValueProfData *VPD = |
675 | DstData ? DstData : Closure->AllocValueProfData(TotalSize); |
676 | |
677 | VPD->TotalSize = TotalSize; |
678 | VPD->NumValueKinds = Closure->GetNumValueKinds(Closure->Record); |
679 | ValueProfRecord *VR = getFirstValueProfRecord(VPD); |
680 | for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) { |
681 | uint32_t NumValueSites = Closure->GetNumValueSites(Closure->Record, Kind); |
682 | if (!NumValueSites) |
683 | continue; |
684 | serializeValueProfRecordFrom(VR, Closure, Kind, NumValueSites); |
685 | VR = getValueProfRecordNext(VR); |
686 | } |
687 | return VPD; |
688 | } |
689 | |
690 | #undef INSTR_PROF_COMMON_API_IMPL |
691 | #endif /* INSTR_PROF_COMMON_API_IMPL */ |
692 | |
693 | /*============================================================================*/ |
694 | |
695 | // clang-format off:consider re-enabling clang-format if auto-formatted C macros |
696 | // are readable (e.g., after `issue #82426` is fixed) |
697 | #ifndef INSTR_PROF_DATA_DEFINED |
698 | |
699 | #ifndef INSTR_PROF_DATA_INC |
700 | #define INSTR_PROF_DATA_INC |
701 | |
702 | /* Helper macros. */ |
703 | #define INSTR_PROF_SIMPLE_QUOTE(x) #x |
704 | #define INSTR_PROF_QUOTE(x) INSTR_PROF_SIMPLE_QUOTE(x) |
705 | #define INSTR_PROF_SIMPLE_CONCAT(x,y) x ## y |
706 | #define INSTR_PROF_CONCAT(x,y) INSTR_PROF_SIMPLE_CONCAT(x,y) |
707 | |
708 | /* Magic number to detect file format and endianness. |
709 | * Use 255 at one end, since no UTF-8 file can use that character. Avoid 0, |
710 | * so that utilities, like strings, don't grab it as a string. 129 is also |
711 | * invalid UTF-8, and high enough to be interesting. |
712 | * Use "lprofr" in the centre to stand for "LLVM Profile Raw", or "lprofR" |
713 | * for 32-bit platforms. |
714 | */ |
715 | #define INSTR_PROF_RAW_MAGIC_64 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \ |
716 | (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \ |
717 | (uint64_t)'f' << 16 | (uint64_t)'r' << 8 | (uint64_t)129 |
718 | #define INSTR_PROF_RAW_MAGIC_32 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \ |
719 | (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \ |
720 | (uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129 |
721 | |
722 | /* Raw profile format version (start from 1). */ |
723 | #define INSTR_PROF_RAW_VERSION 10 |
724 | /* Indexed profile format version (start from 1). */ |
725 | #define INSTR_PROF_INDEX_VERSION 12 |
726 | /* Coverage mapping format version (start from 0). */ |
727 | #define INSTR_PROF_COVMAP_VERSION 6 |
728 | |
729 | /* Profile version is always of type uint64_t. Reserve the upper 32 bits in the |
730 | * version for other variants of profile. We set the 8th most significant bit |
731 | * (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentation |
732 | * generated profile, and 0 if this is a Clang FE generated profile. |
733 | * 1 in bit 57 indicates there are context-sensitive records in the profile. |
734 | * The 54th bit indicates whether to always instrument loop entry blocks. |
735 | * The 58th bit indicates whether to always instrument function entry blocks. |
736 | * The 59th bit indicates whether to use debug info to correlate profiles. |
737 | * The 60th bit indicates single byte coverage instrumentation. |
738 | * The 61st bit indicates function entry instrumentation only. |
739 | * The 62nd bit indicates whether memory profile information is present. |
740 | * The 63rd bit indicates if this is a temporal profile. |
741 | */ |
742 | #define VARIANT_MASKS_ALL 0xffffffff00000000ULL |
743 | #define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL) |
744 | #define VARIANT_MASK_INSTR_LOOP_ENTRIES (0x1ULL << 55) |
745 | #define VARIANT_MASK_IR_PROF (0x1ULL << 56) |
746 | #define VARIANT_MASK_CSIR_PROF (0x1ULL << 57) |
747 | #define VARIANT_MASK_INSTR_ENTRY (0x1ULL << 58) |
748 | #define VARIANT_MASK_DBG_CORRELATE (0x1ULL << 59) |
749 | #define VARIANT_MASK_BYTE_COVERAGE (0x1ULL << 60) |
750 | #define VARIANT_MASK_FUNCTION_ENTRY_ONLY (0x1ULL << 61) |
751 | #define VARIANT_MASK_MEMPROF (0x1ULL << 62) |
752 | #define VARIANT_MASK_TEMPORAL_PROF (0x1ULL << 63) |
753 | #define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version |
754 | #define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime |
755 | #define INSTR_PROF_PROFILE_COUNTER_BIAS_VAR __llvm_profile_counter_bias |
756 | #define INSTR_PROF_PROFILE_BITMAP_BIAS_VAR __llvm_profile_bitmap_bias |
757 | #define INSTR_PROF_PROFILE_SET_TIMESTAMP __llvm_profile_set_timestamp |
758 | #define INSTR_PROF_PROFILE_SAMPLING_VAR __llvm_profile_sampling |
759 | |
760 | /* The variable that holds the name of the profile data |
761 | * specified via command line. */ |
762 | #define INSTR_PROF_PROFILE_NAME_VAR __llvm_profile_filename |
763 | |
764 | /* section name strings common to all targets other |
765 | than WIN32 */ |
766 | #define INSTR_PROF_DATA_COMMON __llvm_prf_data |
767 | #define INSTR_PROF_NAME_COMMON __llvm_prf_names |
768 | #define INSTR_PROF_VNAME_COMMON __llvm_prf_vns |
769 | #define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts |
770 | #define INSTR_PROF_BITS_COMMON __llvm_prf_bits |
771 | #define INSTR_PROF_VALS_COMMON __llvm_prf_vals |
772 | #define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds |
773 | #define INSTR_PROF_VTAB_COMMON __llvm_prf_vtab |
774 | #define INSTR_PROF_COVMAP_COMMON __llvm_covmap |
775 | #define INSTR_PROF_COVFUN_COMMON __llvm_covfun |
776 | #define INSTR_PROF_COVDATA_COMMON __llvm_covdata |
777 | #define INSTR_PROF_COVNAME_COMMON __llvm_covnames |
778 | #define INSTR_PROF_COVINIT_COMMON __llvm_covinit |
779 | |
780 | /* Windows section names. Because these section names contain dollar characters, |
781 | * they must be quoted. |
782 | */ |
783 | #define INSTR_PROF_DATA_COFF ".lprfd$M" |
784 | #define INSTR_PROF_NAME_COFF ".lprfn$M" |
785 | #define INSTR_PROF_VNAME_COFF ".lprfvn$M" |
786 | #define INSTR_PROF_CNTS_COFF ".lprfc$M" |
787 | #define INSTR_PROF_BITS_COFF ".lprfb$M" |
788 | #define INSTR_PROF_VALS_COFF ".lprfv$M" |
789 | #define INSTR_PROF_VNODES_COFF ".lprfnd$M" |
790 | #define INSTR_PROF_VTAB_COFF ".lprfvt$M" |
791 | #define INSTR_PROF_COVMAP_COFF ".lcovmap$M" |
792 | #define INSTR_PROF_COVFUN_COFF ".lcovfun$M" |
793 | /* Since cov data and cov names sections are not allocated, we don't need to |
794 | * access them at runtime. |
795 | */ |
796 | #define INSTR_PROF_COVDATA_COFF ".lcovd" |
797 | #define INSTR_PROF_COVNAME_COFF ".lcovn" |
798 | |
799 | // FIXME: Placeholder for Windows. Windows currently does not initialize |
800 | // the GCOV functions in the runtime. |
801 | #define INSTR_PROF_COVINIT_COFF ".lcovd$M" |
802 | |
803 | #ifdef _WIN32 |
804 | /* Runtime section names and name strings. */ |
805 | #define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_DATA_COFF |
806 | #define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_NAME_COFF |
807 | #define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_CNTS_COFF |
808 | #define INSTR_PROF_BITS_SECT_NAME INSTR_PROF_BITS_COFF |
809 | #define INSTR_PROF_VTAB_SECT_NAME INSTR_PROF_VTAB_COFF |
810 | #define INSTR_PROF_VNAME_SECT_NAME INSTR_PROF_VNAME_COFF |
811 | /* Array of pointers. Each pointer points to a list |
812 | * of value nodes associated with one value site. |
813 | */ |
814 | #define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_VALS_COFF |
815 | /* Value profile nodes section. */ |
816 | #define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COFF |
817 | #define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COFF |
818 | #define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_COVFUN_COFF |
819 | #define INSTR_PROF_COVDATA_SECT_NAME INSTR_PROF_COVDATA_COFF |
820 | #define INSTR_PROF_COVNAME_SECT_NAME INSTR_PROF_COVNAME_COFF |
821 | #define INSTR_PROF_COVINIT_SECT_NAME INSTR_PROF_COVINIT_COFF |
822 | #else |
823 | /* Runtime section names and name strings. */ |
824 | #define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON) |
825 | #define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON) |
826 | #define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON) |
827 | #define INSTR_PROF_BITS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_BITS_COMMON) |
828 | #define INSTR_PROF_VTAB_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VTAB_COMMON) |
829 | #define INSTR_PROF_VNAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNAME_COMMON) |
830 | /* Array of pointers. Each pointer points to a list |
831 | * of value nodes associated with one value site. |
832 | */ |
833 | #define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON) |
834 | /* Value profile nodes section. */ |
835 | #define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON) |
836 | #define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON) |
837 | #define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON) |
838 | #define INSTR_PROF_COVDATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVDATA_COMMON) |
839 | #define INSTR_PROF_COVNAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVNAME_COMMON) |
840 | #define INSTR_PROF_COVINIT_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVINIT_COMMON) |
841 | #endif |
842 | |
843 | /* Macros to define start/stop section symbol for a given |
844 | * section on Linux. For instance |
845 | * INSTR_PROF_SECT_START(INSTR_PROF_DATA_SECT_NAME) will |
846 | * expand to __start___llvm_prof_data |
847 | */ |
848 | #define INSTR_PROF_SECT_START(Sect) \ |
849 | INSTR_PROF_CONCAT(__start_,Sect) |
850 | #define INSTR_PROF_SECT_STOP(Sect) \ |
851 | INSTR_PROF_CONCAT(__stop_,Sect) |
852 | |
853 | /* Value Profiling API linkage name. */ |
854 | #define INSTR_PROF_VALUE_PROF_FUNC __llvm_profile_instrument_target |
855 | #define INSTR_PROF_VALUE_PROF_FUNC_STR \ |
856 | INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_FUNC) |
857 | #define INSTR_PROF_VALUE_PROF_MEMOP_FUNC __llvm_profile_instrument_memop |
858 | #define INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR \ |
859 | INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_MEMOP_FUNC) |
860 | |
861 | /* InstrProfile per-function control data alignment. */ |
862 | #define INSTR_PROF_DATA_ALIGNMENT 8 |
863 | |
864 | /* The data structure that represents a tracked value by the |
865 | * value profiler. |
866 | */ |
867 | typedef struct InstrProfValueData { |
868 | /* Profiled value. */ |
869 | uint64_t Value; |
870 | /* Number of times the value appears in the training run. */ |
871 | uint64_t Count; |
872 | } InstrProfValueData; |
873 | |
874 | #endif /* INSTR_PROF_DATA_INC */ |
875 | |
876 | #else |
877 | #undef INSTR_PROF_DATA_DEFINED |
878 | #endif |
879 | |
880 | #undef COVMAP_V2_OR_V3 |
881 | |
882 | #ifdef INSTR_PROF_VALUE_PROF_MEMOP_API |
883 | |
884 | #ifdef __cplusplus |
885 | #define INSTR_PROF_INLINE inline |
886 | #else |
887 | #define INSTR_PROF_INLINE |
888 | #endif |
889 | |
890 | /* The value range buckets (22 buckets) for the memop size value profiling looks |
891 | * like: |
892 | * |
893 | * [0, 0] |
894 | * [1, 1] |
895 | * [2, 2] |
896 | * [3, 3] |
897 | * [4, 4] |
898 | * [5, 5] |
899 | * [6, 6] |
900 | * [7, 7] |
901 | * [8, 8] |
902 | * [9, 15] |
903 | * [16, 16] |
904 | * [17, 31] |
905 | * [32, 32] |
906 | * [33, 63] |
907 | * [64, 64] |
908 | * [65, 127] |
909 | * [128, 128] |
910 | * [129, 255] |
911 | * [256, 256] |
912 | * [257, 511] |
913 | * [512, 512] |
914 | * [513, UINT64_MAX] |
915 | * |
916 | * Each range has a 'representative value' which is the lower end value of the |
917 | * range and used to store in the runtime profile data records and the VP |
918 | * metadata. For example, it's 2 for [2, 2] and 64 for [65, 127]. |
919 | */ |
920 | #define INSTR_PROF_NUM_BUCKETS 22 |
921 | |
922 | /* |
923 | * Clz and Popcount. This code was copied from |
924 | * compiler-rt/lib/fuzzer/{FuzzerBuiltins.h,FuzzerBuiltinsMsvc.h} and |
925 | * llvm/include/llvm/Support/MathExtras.h. |
926 | */ |
927 | #if defined(_MSC_VER) && !defined(__clang__) |
928 | |
929 | #include <intrin.h> |
930 | INSTR_PROF_VISIBILITY INSTR_PROF_INLINE |
931 | int InstProfClzll(unsigned long long X) { |
932 | unsigned long LeadZeroIdx = 0; |
933 | #if !defined(_M_ARM64) && !defined(_M_X64) |
934 | // Scan the high 32 bits. |
935 | if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X >> 32))) |
936 | return (int)(63 - (LeadZeroIdx + 32)); // Create a bit offset |
937 | // from the MSB. |
938 | // Scan the low 32 bits. |
939 | if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X))) |
940 | return (int)(63 - LeadZeroIdx); |
941 | #else |
942 | if (_BitScanReverse64(&LeadZeroIdx, X)) return 63 - LeadZeroIdx; |
943 | #endif |
944 | return 64; |
945 | } |
946 | INSTR_PROF_VISIBILITY INSTR_PROF_INLINE |
947 | int InstProfPopcountll(unsigned long long X) { |
948 | // This code originates from https://reviews.llvm.org/rG30626254510f. |
949 | unsigned long long v = X; |
950 | v = v - ((v >> 1) & 0x5555555555555555ULL); |
951 | v = (v & 0x3333333333333333ULL) + ((v >> 2) & 0x3333333333333333ULL); |
952 | v = (v + (v >> 4)) & 0x0F0F0F0F0F0F0F0FULL; |
953 | return (int)((unsigned long long)(v * 0x0101010101010101ULL) >> 56); |
954 | } |
955 | |
956 | #else |
957 | |
958 | INSTR_PROF_VISIBILITY INSTR_PROF_INLINE |
959 | int InstProfClzll(unsigned long long X) { return __builtin_clzll(X); } |
960 | INSTR_PROF_VISIBILITY INSTR_PROF_INLINE |
961 | int InstProfPopcountll(unsigned long long X) { return __builtin_popcountll(X); } |
962 | |
963 | #endif /* defined(_MSC_VER) && !defined(__clang__) */ |
964 | |
965 | // clang-format on |
966 | |
967 | /* Map an (observed) memop size value to the representative value of its range. |
968 | * For example, 5 -> 5, 22 -> 17, 99 -> 65, 256 -> 256, 1001 -> 513. */ |
969 | INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint64_t |
970 | InstrProfGetRangeRepValue(uint64_t Value) { |
971 | if (Value <= 8) |
972 | // The first ranges are individually tracked. Use the value as is. |
973 | return Value; |
974 | else if (Value >= 513) |
975 | // The last range is mapped to its lowest value. |
976 | return 513; |
977 | else if (InstProfPopcountll(Value) == 1) |
978 | // If it's a power of two, use it as is. |
979 | return Value; |
980 | else |
981 | // Otherwise, take to the previous power of two + 1. |
982 | return (UINT64_C(1) << (64 - InstProfClzll(Value) - 1)) + 1; |
983 | } |
984 | |
985 | /* Return true if the range that an (observed) memop size value belongs to has |
986 | * only a single value in the range. For example, 0 -> true, 8 -> true, 10 -> |
987 | * false, 64 -> true, 100 -> false, 513 -> false. */ |
988 | INSTR_PROF_VISIBILITY INSTR_PROF_INLINE unsigned |
989 | InstrProfIsSingleValRange(uint64_t Value) { |
990 | if (Value <= 8) |
991 | // The first ranges are individually tracked. |
992 | return 1; |
993 | else if (InstProfPopcountll(Value) == 1) |
994 | // If it's a power of two, there's only one value. |
995 | return 1; |
996 | else |
997 | // Otherwise, there's more than one value in the range. |
998 | return 0; |
999 | } |
1000 | |
1001 | #endif /* INSTR_PROF_VALUE_PROF_MEMOP_API */ |
1002 | |