1/*===- InstrProfilingMerge.c - Profile in-process Merging ---------------===*\
2|*
3|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4|* See https://llvm.org/LICENSE.txt for license information.
5|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6|*
7|*===----------------------------------------------------------------------===*
8|* This file defines the API needed for in-process merging of profile data
9|* stored in memory buffer.
10\*===---------------------------------------------------------------------===*/
11
12#include "InstrProfiling.h"
13#include "InstrProfilingInternal.h"
14#include "InstrProfilingUtil.h"
15
16#define INSTR_PROF_VALUE_PROF_DATA
17#include "profile/InstrProfData.inc"
18
19COMPILER_RT_VISIBILITY
20void (*VPMergeHook)(ValueProfData *, __llvm_profile_data *);
21
22COMPILER_RT_VISIBILITY
23uint64_t lprofGetLoadModuleSignature(void) {
24 /* A very fast way to compute a module signature. */
25 uint64_t Version = __llvm_profile_get_version();
26 uint64_t NumCounters = __llvm_profile_get_num_counters(
27 Begin: __llvm_profile_begin_counters(), End: __llvm_profile_end_counters());
28 uint64_t NumData = __llvm_profile_get_num_data(Begin: __llvm_profile_begin_data(),
29 End: __llvm_profile_end_data());
30 uint64_t NamesSize =
31 (uint64_t)(__llvm_profile_end_names() - __llvm_profile_begin_names());
32 uint64_t NumVnodes =
33 (uint64_t)(__llvm_profile_end_vnodes() - __llvm_profile_begin_vnodes());
34 const __llvm_profile_data *FirstD = __llvm_profile_begin_data();
35
36 return (NamesSize << 40) + (NumCounters << 30) + (NumData << 20) +
37 (NumVnodes << 10) + (NumData > 0 ? FirstD->NameRef : 0) + Version +
38 __llvm_profile_get_magic();
39}
40
41#ifdef __GNUC__
42#pragma GCC diagnostic push
43#pragma GCC diagnostic ignored "-Wcast-qual"
44#elif defined(__clang__)
45#pragma clang diagnostic push
46#pragma clang diagnostic ignored "-Wcast-qual"
47#endif
48
49/* Returns 1 if profile is not structurally compatible. */
50COMPILER_RT_VISIBILITY
51int __llvm_profile_check_compatibility(const char *ProfileData,
52 uint64_t ProfileSize) {
53 __llvm_profile_header *Header = (__llvm_profile_header *)ProfileData;
54 __llvm_profile_data *SrcDataStart, *SrcDataEnd, *SrcData, *DstData;
55 SrcDataStart =
56 (__llvm_profile_data *)(ProfileData + sizeof(__llvm_profile_header) +
57 Header->BinaryIdsSize);
58 SrcDataEnd = SrcDataStart + Header->NumData;
59
60 if (ProfileSize < sizeof(__llvm_profile_header))
61 return 1;
62
63 /* Check the header first. */
64 if (Header->Magic != __llvm_profile_get_magic() ||
65 Header->Version != __llvm_profile_get_version() ||
66 Header->NumData !=
67 __llvm_profile_get_num_data(Begin: __llvm_profile_begin_data(),
68 End: __llvm_profile_end_data()) ||
69 Header->NumCounters !=
70 __llvm_profile_get_num_counters(Begin: __llvm_profile_begin_counters(),
71 End: __llvm_profile_end_counters()) ||
72 Header->NumBitmapBytes !=
73 __llvm_profile_get_num_bitmap_bytes(Begin: __llvm_profile_begin_bitmap(),
74 End: __llvm_profile_end_bitmap()) ||
75 Header->NamesSize !=
76 __llvm_profile_get_name_size(Begin: __llvm_profile_begin_names(),
77 End: __llvm_profile_end_names()) ||
78 Header->ValueKindLast != IPVK_Last)
79 return 1;
80
81 if (ProfileSize <
82 sizeof(__llvm_profile_header) + Header->BinaryIdsSize +
83 Header->NumData * sizeof(__llvm_profile_data) + Header->NamesSize +
84 Header->NumCounters * __llvm_profile_counter_entry_size() +
85 Header->NumBitmapBytes)
86 return 1;
87
88 for (SrcData = SrcDataStart,
89 DstData = (__llvm_profile_data *)__llvm_profile_begin_data();
90 SrcData < SrcDataEnd; ++SrcData, ++DstData) {
91 if (SrcData->NameRef != DstData->NameRef ||
92 SrcData->FuncHash != DstData->FuncHash ||
93 SrcData->NumCounters != DstData->NumCounters ||
94 SrcData->NumBitmapBytes != DstData->NumBitmapBytes)
95 return 1;
96 }
97
98 /* Matched! */
99 return 0;
100}
101
102static uintptr_t signextIfWin64(void *V) {
103#ifdef _WIN64
104 return (uintptr_t)(int32_t)(uintptr_t)V;
105#else
106 return (uintptr_t)V;
107#endif
108}
109
110// Skip names section, vtable profile data section and vtable names section
111// for runtime profile merge. To merge runtime addresses from multiple
112// profiles collected from the same instrumented binary, the binary should be
113// loaded at fixed base address (e.g., build with -no-pie, or run with ASLR
114// disabled). In this set-up these three sections remain unchanged.
115static uint64_t
116getDistanceFromCounterToValueProf(const __llvm_profile_header *const Header) {
117 const uint64_t VTableSectionSize =
118 Header->NumVTables * sizeof(VTableProfData);
119 const uint64_t PaddingBytesAfterVTableSection =
120 __llvm_profile_get_num_padding_bytes(SizeInBytes: VTableSectionSize);
121 const uint64_t VNamesSize = Header->VNamesSize;
122 const uint64_t PaddingBytesAfterVNamesSize =
123 __llvm_profile_get_num_padding_bytes(SizeInBytes: VNamesSize);
124 return Header->NamesSize +
125 __llvm_profile_get_num_padding_bytes(SizeInBytes: Header->NamesSize) +
126 VTableSectionSize + PaddingBytesAfterVTableSection + VNamesSize +
127 PaddingBytesAfterVNamesSize;
128}
129
130COMPILER_RT_VISIBILITY
131int __llvm_profile_merge_from_buffer(const char *ProfileData,
132 uint64_t ProfileSize) {
133 if (__llvm_profile_get_version() & VARIANT_MASK_TEMPORAL_PROF) {
134 PROF_ERR("%s\n",
135 "Temporal profiles do not support profile merging at runtime. "
136 "Instead, merge raw profiles using the llvm-profdata tool.");
137 return 1;
138 }
139
140 __llvm_profile_data *SrcDataStart, *SrcDataEnd, *SrcData, *DstData;
141 __llvm_profile_header *Header = (__llvm_profile_header *)ProfileData;
142 char *SrcCountersStart, *DstCounter;
143 const char *SrcCountersEnd, *SrcCounter;
144 const char *SrcBitmapStart;
145 const char *SrcNameStart;
146 const char *SrcValueProfDataStart, *SrcValueProfData;
147 uintptr_t CountersDelta = Header->CountersDelta;
148 uintptr_t BitmapDelta = Header->BitmapDelta;
149
150 SrcDataStart =
151 (__llvm_profile_data *)(ProfileData + sizeof(__llvm_profile_header) +
152 Header->BinaryIdsSize);
153 SrcDataEnd = SrcDataStart + Header->NumData;
154 SrcCountersStart = (char *)SrcDataEnd;
155 SrcCountersEnd = SrcCountersStart +
156 Header->NumCounters * __llvm_profile_counter_entry_size();
157 SrcBitmapStart = SrcCountersEnd;
158 SrcNameStart = SrcBitmapStart + Header->NumBitmapBytes;
159 SrcValueProfDataStart =
160 SrcNameStart + getDistanceFromCounterToValueProf(Header);
161 if (SrcNameStart < SrcCountersStart || SrcNameStart < SrcBitmapStart)
162 return 1;
163
164 // Merge counters by iterating the entire counter section when data section is
165 // empty due to correlation.
166 if (Header->NumData == 0) {
167 for (SrcCounter = SrcCountersStart,
168 DstCounter = __llvm_profile_begin_counters();
169 SrcCounter < SrcCountersEnd;) {
170 if (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE) {
171 *DstCounter &= *SrcCounter;
172 } else {
173 *(uint64_t *)DstCounter += *(uint64_t *)SrcCounter;
174 }
175 SrcCounter += __llvm_profile_counter_entry_size();
176 DstCounter += __llvm_profile_counter_entry_size();
177 }
178 return 0;
179 }
180
181 for (SrcData = SrcDataStart,
182 DstData = (__llvm_profile_data *)__llvm_profile_begin_data(),
183 SrcValueProfData = SrcValueProfDataStart;
184 SrcData < SrcDataEnd; ++SrcData, ++DstData) {
185 // For the in-memory destination, CounterPtr is the distance from the start
186 // address of the data to the start address of the counter. On WIN64,
187 // CounterPtr is a truncated 32-bit value due to COFF limitation. Sign
188 // extend CounterPtr to get the original value.
189 char *DstCounters =
190 (char *)((uintptr_t)DstData + signextIfWin64(V: DstData->CounterPtr));
191 char *DstBitmap =
192 (char *)((uintptr_t)DstData + signextIfWin64(V: DstData->BitmapPtr));
193 unsigned NVK = 0;
194
195 // SrcData is a serialized representation of the memory image. We need to
196 // compute the in-buffer counter offset from the in-memory address distance.
197 // The initial CountersDelta is the in-memory address difference
198 // start(__llvm_prf_cnts)-start(__llvm_prf_data), so SrcData->CounterPtr -
199 // CountersDelta computes the offset into the in-buffer counter section.
200 //
201 // On WIN64, CountersDelta is truncated as well, so no need for signext.
202 char *SrcCounters =
203 SrcCountersStart + ((uintptr_t)SrcData->CounterPtr - CountersDelta);
204 // CountersDelta needs to be decreased as we advance to the next data
205 // record.
206 CountersDelta -= sizeof(*SrcData);
207 unsigned NC = SrcData->NumCounters;
208 if (NC == 0)
209 return 1;
210 if (SrcCounters < SrcCountersStart || SrcCounters >= SrcNameStart ||
211 (SrcCounters + __llvm_profile_counter_entry_size() * NC) > SrcNameStart)
212 return 1;
213 for (unsigned I = 0; I < NC; I++) {
214 if (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE) {
215 // A value of zero signifies the function is covered.
216 DstCounters[I] &= SrcCounters[I];
217 } else {
218 ((uint64_t *)DstCounters)[I] += ((uint64_t *)SrcCounters)[I];
219 }
220 }
221
222 const char *SrcBitmap =
223 SrcBitmapStart + ((uintptr_t)SrcData->BitmapPtr - BitmapDelta);
224 // BitmapDelta also needs to be decreased as we advance to the next data
225 // record.
226 BitmapDelta -= sizeof(*SrcData);
227 unsigned NB = SrcData->NumBitmapBytes;
228 // NumBitmapBytes may legitimately be 0. Just keep going.
229 if (NB != 0) {
230 if (SrcBitmap < SrcBitmapStart || (SrcBitmap + NB) > SrcNameStart)
231 return 1;
232 // Merge Src and Dst Bitmap bytes by simply ORing them together.
233 for (unsigned I = 0; I < NB; I++)
234 DstBitmap[I] |= SrcBitmap[I];
235 }
236
237 /* Now merge value profile data. */
238 if (!VPMergeHook)
239 continue;
240
241 for (unsigned I = 0; I <= IPVK_Last; I++)
242 NVK += (SrcData->NumValueSites[I] != 0);
243
244 if (!NVK)
245 continue;
246
247 if (SrcValueProfData >= ProfileData + ProfileSize)
248 return 1;
249 VPMergeHook((ValueProfData *)SrcValueProfData, DstData);
250 SrcValueProfData =
251 SrcValueProfData + ((ValueProfData *)SrcValueProfData)->TotalSize;
252 }
253
254 return 0;
255}
256
257#ifdef __GNUC__
258#pragma GCC diagnostic pop
259#elif defined(__clang__)
260#pragma clang diagnostic pop
261#endif
262

source code of compiler-rt/lib/profile/InstrProfilingMerge.c