1#include <stdint.h>
2#include <stdlib.h>
3#include <string.h>
4
5#include "memprof_rawprofile.h"
6#include "profile/MemProfData.inc"
7#include "sanitizer_common/sanitizer_allocator_internal.h"
8#include "sanitizer_common/sanitizer_array_ref.h"
9#include "sanitizer_common/sanitizer_common.h"
10#include "sanitizer_common/sanitizer_linux.h"
11#include "sanitizer_common/sanitizer_procmaps.h"
12#include "sanitizer_common/sanitizer_stackdepot.h"
13#include "sanitizer_common/sanitizer_stackdepotbase.h"
14#include "sanitizer_common/sanitizer_stacktrace.h"
15#include "sanitizer_common/sanitizer_vector.h"
16
17namespace __memprof {
18using ::__sanitizer::Vector;
19using ::llvm::memprof::MemInfoBlock;
20using SegmentEntry = ::llvm::memprof::SegmentEntry;
21using Header = ::llvm::memprof::Header;
22
23namespace {
24template <class T> char *WriteBytes(const T &Pod, char *Buffer) {
25 *(T *)Buffer = Pod;
26 return Buffer + sizeof(T);
27}
28
29void RecordStackId(const uptr Key, UNUSED LockedMemInfoBlock *const &MIB,
30 void *Arg) {
31 // No need to touch the MIB value here since we are only recording the key.
32 auto *StackIds = reinterpret_cast<Vector<u64> *>(Arg);
33 StackIds->PushBack(v: Key);
34}
35} // namespace
36
37u64 SegmentSizeBytes(ArrayRef<LoadedModule> Modules) {
38 u64 NumSegmentsToRecord = 0;
39 for (const auto &Module : Modules) {
40 for (const auto &Segment : Module.ranges()) {
41 if (Segment.executable)
42 NumSegmentsToRecord++;
43 }
44 }
45
46 return sizeof(u64) // A header which stores the number of records.
47 + sizeof(SegmentEntry) * NumSegmentsToRecord;
48}
49
50// The segment section uses the following format:
51// ---------- Segment Info
52// Num Entries
53// ---------- Segment Entry
54// Start
55// End
56// Offset
57// UuidSize
58// Uuid 32B
59// ----------
60// ...
61void SerializeSegmentsToBuffer(ArrayRef<LoadedModule> Modules,
62 const u64 ExpectedNumBytes, char *&Buffer) {
63 char *Ptr = Buffer;
64 // Reserve space for the final count.
65 Ptr += sizeof(u64);
66
67 u64 NumSegmentsRecorded = 0;
68
69 for (const auto &Module : Modules) {
70 for (const auto &Segment : Module.ranges()) {
71 if (Segment.executable) {
72 SegmentEntry Entry(Segment.beg, Segment.end, Module.base_address());
73 CHECK(Module.uuid_size() <= MEMPROF_BUILDID_MAX_SIZE);
74 Entry.BuildIdSize = Module.uuid_size();
75 memcpy(dest: Entry.BuildId, src: Module.uuid(), n: Module.uuid_size());
76 memcpy(dest: Ptr, src: &Entry, n: sizeof(SegmentEntry));
77 Ptr += sizeof(SegmentEntry);
78 NumSegmentsRecorded++;
79 }
80 }
81 }
82 // Store the number of segments we recorded in the space we reserved.
83 *((u64 *)Buffer) = NumSegmentsRecorded;
84 CHECK(ExpectedNumBytes >= static_cast<u64>(Ptr - Buffer) &&
85 "Expected num bytes != actual bytes written");
86}
87
88u64 StackSizeBytes(const Vector<u64> &StackIds) {
89 u64 NumBytesToWrite = sizeof(u64);
90
91 const u64 NumIds = StackIds.Size();
92 for (unsigned k = 0; k < NumIds; ++k) {
93 const u64 Id = StackIds[k];
94 // One entry for the id and then one more for the number of stack pcs.
95 NumBytesToWrite += 2 * sizeof(u64);
96 const StackTrace St = StackDepotGet(id: Id);
97
98 CHECK(St.trace != nullptr && St.size > 0 && "Empty stack trace");
99 for (uptr i = 0; i < St.size && St.trace[i] != 0; i++) {
100 NumBytesToWrite += sizeof(u64);
101 }
102 }
103 return NumBytesToWrite;
104}
105
106// The stack info section uses the following format:
107//
108// ---------- Stack Info
109// Num Entries
110// ---------- Stack Entry
111// Num Stacks
112// PC1
113// PC2
114// ...
115// ----------
116void SerializeStackToBuffer(const Vector<u64> &StackIds,
117 const u64 ExpectedNumBytes, char *&Buffer) {
118 const u64 NumIds = StackIds.Size();
119 char *Ptr = Buffer;
120 Ptr = WriteBytes(Pod: static_cast<u64>(NumIds), Buffer: Ptr);
121
122 for (unsigned k = 0; k < NumIds; ++k) {
123 const u64 Id = StackIds[k];
124 Ptr = WriteBytes(Pod: Id, Buffer: Ptr);
125 Ptr += sizeof(u64); // Bump it by u64, we will fill this in later.
126 u64 Count = 0;
127 const StackTrace St = StackDepotGet(id: Id);
128 for (uptr i = 0; i < St.size && St.trace[i] != 0; i++) {
129 // PCs in stack traces are actually the return addresses, that is,
130 // addresses of the next instructions after the call.
131 uptr pc = StackTrace::GetPreviousInstructionPc(pc: St.trace[i]);
132 Ptr = WriteBytes(Pod: static_cast<u64>(pc), Buffer: Ptr);
133 ++Count;
134 }
135 // Store the count in the space we reserved earlier.
136 *(u64 *)(Ptr - (Count + 1) * sizeof(u64)) = Count;
137 }
138
139 CHECK(ExpectedNumBytes >= static_cast<u64>(Ptr - Buffer) &&
140 "Expected num bytes != actual bytes written");
141}
142
143// The MIB section has the following format:
144// ---------- MIB Info
145// Num Entries
146// ---------- MIB Entry 0
147// Alloc Count
148// ...
149// ---------- MIB Entry 1
150// Alloc Count
151// ...
152// ----------
153void SerializeMIBInfoToBuffer(MIBMapTy &MIBMap, const Vector<u64> &StackIds,
154 const u64 ExpectedNumBytes, char *&Buffer) {
155 char *Ptr = Buffer;
156 const u64 NumEntries = StackIds.Size();
157 Ptr = WriteBytes(Pod: NumEntries, Buffer: Ptr);
158
159 for (u64 i = 0; i < NumEntries; i++) {
160 const u64 Key = StackIds[i];
161 MIBMapTy::Handle h(&MIBMap, Key, /*remove=*/true, /*create=*/false);
162 CHECK(h.exists());
163 Ptr = WriteBytes(Pod: Key, Buffer: Ptr);
164 Ptr = WriteBytes(Pod: (*h)->mib, Buffer: Ptr);
165 }
166
167 CHECK(ExpectedNumBytes >= static_cast<u64>(Ptr - Buffer) &&
168 "Expected num bytes != actual bytes written");
169}
170
171// Format
172// ---------- Header
173// Magic
174// Version
175// Total Size
176// Segment Offset
177// MIB Info Offset
178// Stack Offset
179// ---------- Segment Info
180// Num Entries
181// ---------- Segment Entry
182// Start
183// End
184// Offset
185// BuildID 32B
186// ----------
187// ...
188// ----------
189// Optional Padding Bytes
190// ---------- MIB Info
191// Num Entries
192// ---------- MIB Entry
193// Alloc Count
194// ...
195// ----------
196// Optional Padding Bytes
197// ---------- Stack Info
198// Num Entries
199// ---------- Stack Entry
200// Num Stacks
201// PC1
202// PC2
203// ...
204// ----------
205// Optional Padding Bytes
206// ...
207u64 SerializeToRawProfile(MIBMapTy &MIBMap, ArrayRef<LoadedModule> Modules,
208 char *&Buffer) {
209 // Each section size is rounded up to 8b since the first entry in each section
210 // is a u64 which holds the number of entries in the section by convention.
211 const u64 NumSegmentBytes = RoundUpTo(size: SegmentSizeBytes(Modules), boundary: 8);
212
213 Vector<u64> StackIds;
214 MIBMap.ForEach(cb: RecordStackId, arg: reinterpret_cast<void *>(&StackIds));
215 // The first 8b are for the total number of MIB records. Each MIB record is
216 // preceded by a 8b stack id which is associated with stack frames in the next
217 // section.
218 const u64 NumMIBInfoBytes = RoundUpTo(
219 size: sizeof(u64) + StackIds.Size() * (sizeof(u64) + sizeof(MemInfoBlock)), boundary: 8);
220
221 const u64 NumStackBytes = RoundUpTo(size: StackSizeBytes(StackIds), boundary: 8);
222
223 // Ensure that the profile is 8b aligned. We allow for some optional padding
224 // at the end so that any subsequent profile serialized to the same file does
225 // not incur unaligned accesses.
226 const u64 TotalSizeBytes = RoundUpTo(
227 size: sizeof(Header) + NumSegmentBytes + NumStackBytes + NumMIBInfoBytes, boundary: 8);
228
229 // Allocate the memory for the entire buffer incl. info blocks.
230 Buffer = (char *)InternalAlloc(size: TotalSizeBytes);
231 char *Ptr = Buffer;
232
233 Header header{MEMPROF_RAW_MAGIC_64,
234 MEMPROF_RAW_VERSION,
235 .TotalSize: static_cast<u64>(TotalSizeBytes),
236 .SegmentOffset: sizeof(Header),
237 .MIBOffset: sizeof(Header) + NumSegmentBytes,
238 .StackOffset: sizeof(Header) + NumSegmentBytes + NumMIBInfoBytes};
239 Ptr = WriteBytes(Pod: header, Buffer: Ptr);
240
241 SerializeSegmentsToBuffer(Modules, ExpectedNumBytes: NumSegmentBytes, Buffer&: Ptr);
242 Ptr += NumSegmentBytes;
243
244 SerializeMIBInfoToBuffer(MIBMap, StackIds, ExpectedNumBytes: NumMIBInfoBytes, Buffer&: Ptr);
245 Ptr += NumMIBInfoBytes;
246
247 SerializeStackToBuffer(StackIds, ExpectedNumBytes: NumStackBytes, Buffer&: Ptr);
248
249 return TotalSizeBytes;
250}
251
252} // namespace __memprof
253

source code of compiler-rt/lib/memprof/memprof_rawprofile.cpp