1 | //===- SampleProfWriter.h - Write LLVM sample profile data ------*- C++ -*-===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This file contains definitions needed for writing sample profiles. |
10 | // |
11 | //===----------------------------------------------------------------------===// |
12 | #ifndef LLVM_PROFILEDATA_SAMPLEPROFWRITER_H |
13 | #define LLVM_PROFILEDATA_SAMPLEPROFWRITER_H |
14 | |
15 | #include "llvm/ADT/MapVector.h" |
16 | #include "llvm/ADT/StringRef.h" |
17 | #include "llvm/IR/ProfileSummary.h" |
18 | #include "llvm/ProfileData/SampleProf.h" |
19 | #include "llvm/Support/ErrorOr.h" |
20 | #include "llvm/Support/raw_ostream.h" |
21 | #include <cstdint> |
22 | #include <memory> |
23 | #include <set> |
24 | #include <system_error> |
25 | |
26 | namespace llvm { |
27 | namespace sampleprof { |
28 | |
29 | enum SectionLayout { |
30 | DefaultLayout, |
31 | // The layout splits profile with context information from profile without |
32 | // context information. When Thinlto is enabled, ThinLTO postlink phase only |
33 | // has to load profile with context information and can skip the other part. |
34 | CtxSplitLayout, |
35 | NumOfLayout, |
36 | }; |
37 | |
38 | /// When writing a profile with size limit, user may want to use a different |
39 | /// strategy to reduce function count other than dropping functions with fewest |
40 | /// samples first. In this case a class implementing the same interfaces should |
41 | /// be provided to SampleProfileWriter::writeWithSizeLimit(). |
42 | class FunctionPruningStrategy { |
43 | protected: |
44 | SampleProfileMap &ProfileMap; |
45 | size_t OutputSizeLimit; |
46 | |
47 | public: |
48 | /// \p ProfileMap A reference to the original profile map. It will be modified |
49 | /// by Erase(). |
50 | /// \p OutputSizeLimit Size limit in bytes of the output profile. This is |
51 | /// necessary to estimate how many functions to remove. |
52 | FunctionPruningStrategy(SampleProfileMap &ProfileMap, size_t OutputSizeLimit) |
53 | : ProfileMap(ProfileMap), OutputSizeLimit(OutputSizeLimit) {} |
54 | |
55 | virtual ~FunctionPruningStrategy() = default; |
56 | |
57 | /// SampleProfileWriter::writeWithSizeLimit() calls this after every write |
58 | /// iteration if the output size still exceeds the limit. This function |
59 | /// should erase some functions from the profile map so that the writer tries |
60 | /// to write the profile again with fewer functions. At least 1 entry from the |
61 | /// profile map must be erased. |
62 | /// |
63 | /// \p CurrentOutputSize Number of bytes in the output if current profile map |
64 | /// is written. |
65 | virtual void Erase(size_t CurrentOutputSize) = 0; |
66 | }; |
67 | |
68 | class DefaultFunctionPruningStrategy : public FunctionPruningStrategy { |
69 | std::vector<NameFunctionSamples> SortedFunctions; |
70 | |
71 | public: |
72 | DefaultFunctionPruningStrategy(SampleProfileMap &ProfileMap, |
73 | size_t OutputSizeLimit); |
74 | |
75 | /// In this default implementation, functions with fewest samples are dropped |
76 | /// first. Since the exact size of the output cannot be easily calculated due |
77 | /// to compression, we use a heuristic to remove as many functions as |
78 | /// necessary but not too many, aiming to minimize the number of write |
79 | /// iterations. |
80 | /// Empirically, functions with larger total sample count contain linearly |
81 | /// more sample entries, meaning it takes linearly more space to write them. |
82 | /// The cumulative length is therefore quadratic if all functions are sorted |
83 | /// by total sample count. |
84 | /// TODO: Find better heuristic. |
85 | void Erase(size_t CurrentOutputSize) override; |
86 | }; |
87 | |
88 | /// Sample-based profile writer. Base class. |
89 | class SampleProfileWriter { |
90 | public: |
91 | virtual ~SampleProfileWriter() = default; |
92 | |
93 | /// Write sample profiles in \p S. |
94 | /// |
95 | /// \returns status code of the file update operation. |
96 | virtual std::error_code writeSample(const FunctionSamples &S) = 0; |
97 | |
98 | /// Write all the sample profiles in the given map of samples. |
99 | /// |
100 | /// \returns status code of the file update operation. |
101 | virtual std::error_code write(const SampleProfileMap &ProfileMap); |
102 | |
103 | /// Write sample profiles up to given size limit, using the pruning strategy |
104 | /// to drop some functions if necessary. |
105 | /// |
106 | /// \returns status code of the file update operation. |
107 | template <typename FunctionPruningStrategy = DefaultFunctionPruningStrategy> |
108 | std::error_code writeWithSizeLimit(SampleProfileMap &ProfileMap, |
109 | size_t OutputSizeLimit) { |
110 | FunctionPruningStrategy Strategy(ProfileMap, OutputSizeLimit); |
111 | return writeWithSizeLimitInternal(ProfileMap, OutputSizeLimit, Strategy: &Strategy); |
112 | } |
113 | |
114 | raw_ostream &getOutputStream() { return *OutputStream; } |
115 | |
116 | /// Profile writer factory. |
117 | /// |
118 | /// Create a new file writer based on the value of \p Format. |
119 | static ErrorOr<std::unique_ptr<SampleProfileWriter>> |
120 | create(StringRef Filename, SampleProfileFormat Format); |
121 | |
122 | /// Create a new stream writer based on the value of \p Format. |
123 | /// For testing. |
124 | static ErrorOr<std::unique_ptr<SampleProfileWriter>> |
125 | create(std::unique_ptr<raw_ostream> &OS, SampleProfileFormat Format); |
126 | |
127 | virtual void setProfileSymbolList(ProfileSymbolList *PSL) {} |
128 | virtual void setToCompressAllSections() {} |
129 | virtual void setUseMD5() {} |
130 | virtual void setPartialProfile() {} |
131 | virtual void resetSecLayout(SectionLayout SL) {} |
132 | |
133 | protected: |
134 | SampleProfileWriter(std::unique_ptr<raw_ostream> &OS) |
135 | : OutputStream(std::move(OS)) {} |
136 | |
137 | /// Write a file header for the profile file. |
138 | virtual std::error_code (const SampleProfileMap &ProfileMap) = 0; |
139 | |
140 | // Write function profiles to the profile file. |
141 | virtual std::error_code writeFuncProfiles(const SampleProfileMap &ProfileMap); |
142 | |
143 | std::error_code writeWithSizeLimitInternal(SampleProfileMap &ProfileMap, |
144 | size_t OutputSizeLimit, |
145 | FunctionPruningStrategy *Strategy); |
146 | |
147 | /// For writeWithSizeLimit in text mode, each newline takes 1 additional byte |
148 | /// on Windows when actually written to the file, but not written to a memory |
149 | /// buffer. This needs to be accounted for when rewriting the profile. |
150 | size_t LineCount; |
151 | |
152 | /// Output stream where to emit the profile to. |
153 | std::unique_ptr<raw_ostream> OutputStream; |
154 | |
155 | /// Profile summary. |
156 | std::unique_ptr<ProfileSummary> Summary; |
157 | |
158 | /// Compute summary for this profile. |
159 | void computeSummary(const SampleProfileMap &ProfileMap); |
160 | |
161 | /// Profile format. |
162 | SampleProfileFormat Format = SPF_None; |
163 | }; |
164 | |
165 | /// Sample-based profile writer (text format). |
166 | class SampleProfileWriterText : public SampleProfileWriter { |
167 | public: |
168 | std::error_code writeSample(const FunctionSamples &S) override; |
169 | |
170 | protected: |
171 | SampleProfileWriterText(std::unique_ptr<raw_ostream> &OS) |
172 | : SampleProfileWriter(OS), Indent(0) {} |
173 | |
174 | std::error_code (const SampleProfileMap &ProfileMap) override { |
175 | LineCount = 0; |
176 | return sampleprof_error::success; |
177 | } |
178 | |
179 | private: |
180 | /// Indent level to use when writing. |
181 | /// |
182 | /// This is used when printing inlined callees. |
183 | unsigned Indent; |
184 | |
185 | friend ErrorOr<std::unique_ptr<SampleProfileWriter>> |
186 | SampleProfileWriter::create(std::unique_ptr<raw_ostream> &OS, |
187 | SampleProfileFormat Format); |
188 | }; |
189 | |
190 | /// Sample-based profile writer (binary format). |
191 | class SampleProfileWriterBinary : public SampleProfileWriter { |
192 | public: |
193 | SampleProfileWriterBinary(std::unique_ptr<raw_ostream> &OS) |
194 | : SampleProfileWriter(OS) {} |
195 | |
196 | std::error_code writeSample(const FunctionSamples &S) override; |
197 | |
198 | protected: |
199 | virtual MapVector<FunctionId, uint32_t> &getNameTable() { return NameTable; } |
200 | virtual std::error_code writeMagicIdent(SampleProfileFormat Format); |
201 | virtual std::error_code writeNameTable(); |
202 | std::error_code (const SampleProfileMap &ProfileMap) override; |
203 | std::error_code writeSummary(); |
204 | virtual std::error_code writeContextIdx(const SampleContext &Context); |
205 | std::error_code writeNameIdx(FunctionId FName); |
206 | std::error_code writeBody(const FunctionSamples &S); |
207 | inline void stablizeNameTable(MapVector<FunctionId, uint32_t> &NameTable, |
208 | std::set<FunctionId> &V); |
209 | |
210 | MapVector<FunctionId, uint32_t> NameTable; |
211 | |
212 | void addName(FunctionId FName); |
213 | virtual void addContext(const SampleContext &Context); |
214 | void addNames(const FunctionSamples &S); |
215 | |
216 | private: |
217 | friend ErrorOr<std::unique_ptr<SampleProfileWriter>> |
218 | SampleProfileWriter::create(std::unique_ptr<raw_ostream> &OS, |
219 | SampleProfileFormat Format); |
220 | }; |
221 | |
222 | class SampleProfileWriterRawBinary : public SampleProfileWriterBinary { |
223 | using SampleProfileWriterBinary::SampleProfileWriterBinary; |
224 | }; |
225 | |
226 | const std::array<SmallVector<SecHdrTableEntry, 8>, NumOfLayout> |
227 | ExtBinaryHdrLayoutTable = { |
228 | // Note that SecFuncOffsetTable section is written after SecLBRProfile |
229 | // in the profile, but is put before SecLBRProfile in SectionHdrLayout. |
230 | // This is because sample reader follows the order in SectionHdrLayout |
231 | // to read each section. To read function profiles on demand, sample |
232 | // reader need to get the offset of each function profile first. |
233 | // |
234 | // DefaultLayout |
235 | SmallVector<SecHdrTableEntry, 8>({{.Type: SecProfSummary, .Flags: 0, .Offset: 0, .Size: 0, .LayoutIndex: 0}, |
236 | {.Type: SecNameTable, .Flags: 0, .Offset: 0, .Size: 0, .LayoutIndex: 0}, |
237 | {.Type: SecCSNameTable, .Flags: 0, .Offset: 0, .Size: 0, .LayoutIndex: 0}, |
238 | {.Type: SecFuncOffsetTable, .Flags: 0, .Offset: 0, .Size: 0, .LayoutIndex: 0}, |
239 | {.Type: SecLBRProfile, .Flags: 0, .Offset: 0, .Size: 0, .LayoutIndex: 0}, |
240 | {.Type: SecProfileSymbolList, .Flags: 0, .Offset: 0, .Size: 0, .LayoutIndex: 0}, |
241 | {.Type: SecFuncMetadata, .Flags: 0, .Offset: 0, .Size: 0, .LayoutIndex: 0}}), |
242 | // CtxSplitLayout |
243 | SmallVector<SecHdrTableEntry, 8>({{.Type: SecProfSummary, .Flags: 0, .Offset: 0, .Size: 0, .LayoutIndex: 0}, |
244 | {.Type: SecNameTable, .Flags: 0, .Offset: 0, .Size: 0, .LayoutIndex: 0}, |
245 | // profile with context |
246 | // for next two sections |
247 | {.Type: SecFuncOffsetTable, .Flags: 0, .Offset: 0, .Size: 0, .LayoutIndex: 0}, |
248 | {.Type: SecLBRProfile, .Flags: 0, .Offset: 0, .Size: 0, .LayoutIndex: 0}, |
249 | // profile without context |
250 | // for next two sections |
251 | {.Type: SecFuncOffsetTable, .Flags: 0, .Offset: 0, .Size: 0, .LayoutIndex: 0}, |
252 | {.Type: SecLBRProfile, .Flags: 0, .Offset: 0, .Size: 0, .LayoutIndex: 0}, |
253 | {.Type: SecProfileSymbolList, .Flags: 0, .Offset: 0, .Size: 0, .LayoutIndex: 0}, |
254 | {.Type: SecFuncMetadata, .Flags: 0, .Offset: 0, .Size: 0, .LayoutIndex: 0}}), |
255 | }; |
256 | |
257 | class SampleProfileWriterExtBinaryBase : public SampleProfileWriterBinary { |
258 | using SampleProfileWriterBinary::SampleProfileWriterBinary; |
259 | public: |
260 | std::error_code write(const SampleProfileMap &ProfileMap) override; |
261 | |
262 | void setToCompressAllSections() override; |
263 | void setToCompressSection(SecType Type); |
264 | std::error_code writeSample(const FunctionSamples &S) override; |
265 | |
266 | // Set to use MD5 to represent string in NameTable. |
267 | void setUseMD5() override { |
268 | UseMD5 = true; |
269 | addSectionFlag(Type: SecNameTable, Flag: SecNameTableFlags::SecFlagMD5Name); |
270 | // MD5 will be stored as plain uint64_t instead of variable-length |
271 | // quantity format in NameTable section. |
272 | addSectionFlag(Type: SecNameTable, Flag: SecNameTableFlags::SecFlagFixedLengthMD5); |
273 | } |
274 | |
275 | // Set the profile to be partial. It means the profile is for |
276 | // common/shared code. The common profile is usually merged from |
277 | // profiles collected from running other targets. |
278 | void setPartialProfile() override { |
279 | addSectionFlag(Type: SecProfSummary, Flag: SecProfSummaryFlags::SecFlagPartial); |
280 | } |
281 | |
282 | void setProfileSymbolList(ProfileSymbolList *PSL) override { |
283 | ProfSymList = PSL; |
284 | }; |
285 | |
286 | void resetSecLayout(SectionLayout SL) override { |
287 | verifySecLayout(SL); |
288 | #ifndef NDEBUG |
289 | // Make sure resetSecLayout is called before any flag setting. |
290 | for (auto &Entry : SectionHdrLayout) { |
291 | assert(Entry.Flags == 0 && |
292 | "resetSecLayout has to be called before any flag setting" ); |
293 | } |
294 | #endif |
295 | SecLayout = SL; |
296 | SectionHdrLayout = ExtBinaryHdrLayoutTable[SL]; |
297 | } |
298 | |
299 | protected: |
300 | uint64_t markSectionStart(SecType Type, uint32_t LayoutIdx); |
301 | std::error_code addNewSection(SecType Sec, uint32_t LayoutIdx, |
302 | uint64_t SectionStart); |
303 | template <class SecFlagType> |
304 | void addSectionFlag(SecType Type, SecFlagType Flag) { |
305 | for (auto &Entry : SectionHdrLayout) { |
306 | if (Entry.Type == Type) |
307 | addSecFlag(Entry, Flag); |
308 | } |
309 | } |
310 | template <class SecFlagType> |
311 | void addSectionFlag(uint32_t SectionIdx, SecFlagType Flag) { |
312 | addSecFlag(SectionHdrLayout[SectionIdx], Flag); |
313 | } |
314 | |
315 | void addContext(const SampleContext &Context) override; |
316 | |
317 | // placeholder for subclasses to dispatch their own section writers. |
318 | virtual std::error_code writeCustomSection(SecType Type) = 0; |
319 | // Verify the SecLayout is supported by the format. |
320 | virtual void verifySecLayout(SectionLayout SL) = 0; |
321 | |
322 | // specify the order to write sections. |
323 | virtual std::error_code writeSections(const SampleProfileMap &ProfileMap) = 0; |
324 | |
325 | // Dispatch section writer for each section. \p LayoutIdx is the sequence |
326 | // number indicating where the section is located in SectionHdrLayout. |
327 | virtual std::error_code writeOneSection(SecType Type, uint32_t LayoutIdx, |
328 | const SampleProfileMap &ProfileMap); |
329 | |
330 | // Helper function to write name table. |
331 | std::error_code writeNameTable() override; |
332 | std::error_code writeContextIdx(const SampleContext &Context) override; |
333 | std::error_code writeCSNameIdx(const SampleContext &Context); |
334 | std::error_code writeCSNameTableSection(); |
335 | |
336 | std::error_code writeFuncMetadata(const SampleProfileMap &Profiles); |
337 | std::error_code writeFuncMetadata(const FunctionSamples &Profile); |
338 | |
339 | // Functions to write various kinds of sections. |
340 | std::error_code writeNameTableSection(const SampleProfileMap &ProfileMap); |
341 | std::error_code writeFuncOffsetTable(); |
342 | std::error_code writeProfileSymbolListSection(); |
343 | |
344 | SectionLayout SecLayout = DefaultLayout; |
345 | // Specifiy the order of sections in section header table. Note |
346 | // the order of sections in SecHdrTable may be different that the |
347 | // order in SectionHdrLayout. sample Reader will follow the order |
348 | // in SectionHdrLayout to read each section. |
349 | SmallVector<SecHdrTableEntry, 8> SectionHdrLayout = |
350 | ExtBinaryHdrLayoutTable[DefaultLayout]; |
351 | |
352 | // Save the start of SecLBRProfile so we can compute the offset to the |
353 | // start of SecLBRProfile for each Function's Profile and will keep it |
354 | // in FuncOffsetTable. |
355 | uint64_t SecLBRProfileStart = 0; |
356 | |
357 | private: |
358 | void allocSecHdrTable(); |
359 | std::error_code writeSecHdrTable(); |
360 | std::error_code (const SampleProfileMap &ProfileMap) override; |
361 | std::error_code compressAndOutput(); |
362 | |
363 | // We will swap the raw_ostream held by LocalBufStream and that |
364 | // held by OutputStream if we try to add a section which needs |
365 | // compression. After the swap, all the data written to output |
366 | // will be temporarily buffered into the underlying raw_string_ostream |
367 | // originally held by LocalBufStream. After the data writing for the |
368 | // section is completed, compress the data in the local buffer, |
369 | // swap the raw_ostream back and write the compressed data to the |
370 | // real output. |
371 | std::unique_ptr<raw_ostream> LocalBufStream; |
372 | // The location where the output stream starts. |
373 | uint64_t FileStart; |
374 | // The location in the output stream where the SecHdrTable should be |
375 | // written to. |
376 | uint64_t SecHdrTableOffset; |
377 | // The table contains SecHdrTableEntry entries in order of how they are |
378 | // populated in the writer. It may be different from the order in |
379 | // SectionHdrLayout which specifies the sequence in which sections will |
380 | // be read. |
381 | std::vector<SecHdrTableEntry> SecHdrTable; |
382 | |
383 | // FuncOffsetTable maps function context to its profile offset in |
384 | // SecLBRProfile section. It is used to load function profile on demand. |
385 | MapVector<SampleContext, uint64_t> FuncOffsetTable; |
386 | // Whether to use MD5 to represent string. |
387 | bool UseMD5 = false; |
388 | |
389 | /// CSNameTable maps function context to its offset in SecCSNameTable section. |
390 | /// The offset will be used everywhere where the context is referenced. |
391 | MapVector<SampleContext, uint32_t> CSNameTable; |
392 | |
393 | ProfileSymbolList *ProfSymList = nullptr; |
394 | }; |
395 | |
396 | class SampleProfileWriterExtBinary : public SampleProfileWriterExtBinaryBase { |
397 | public: |
398 | SampleProfileWriterExtBinary(std::unique_ptr<raw_ostream> &OS) |
399 | : SampleProfileWriterExtBinaryBase(OS) {} |
400 | |
401 | private: |
402 | std::error_code writeDefaultLayout(const SampleProfileMap &ProfileMap); |
403 | std::error_code writeCtxSplitLayout(const SampleProfileMap &ProfileMap); |
404 | |
405 | std::error_code writeSections(const SampleProfileMap &ProfileMap) override; |
406 | |
407 | std::error_code writeCustomSection(SecType Type) override { |
408 | return sampleprof_error::success; |
409 | }; |
410 | |
411 | void verifySecLayout(SectionLayout SL) override { |
412 | assert((SL == DefaultLayout || SL == CtxSplitLayout) && |
413 | "Unsupported layout" ); |
414 | } |
415 | }; |
416 | |
417 | } // end namespace sampleprof |
418 | } // end namespace llvm |
419 | |
420 | #endif // LLVM_PROFILEDATA_SAMPLEPROFWRITER_H |
421 | |