1	//===- unittest/ProfileData/InstrProfTest.cpp -------------------*- C++ -*-===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8
9	#include "llvm/IR/Function.h"
10	#include "llvm/IR/IRBuilder.h"
11	#include "llvm/IR/LLVMContext.h"
12	#include "llvm/IR/Module.h"
13	#include "llvm/ProfileData/InstrProfReader.h"
14	#include "llvm/ProfileData/InstrProfWriter.h"
15	#include "llvm/ProfileData/MemProf.h"
16	#include "llvm/ProfileData/MemProfData.inc"
17	#include "llvm/Support/Compression.h"
18	#include "llvm/Support/raw_ostream.h"
19	#include "llvm/Testing/Support/Error.h"
20	#include "gtest/gtest.h"
21	#include <cstdarg>
22	#include <optional>
23
24	using namespace llvm;
25	using ::testing::EndsWith;
26	using ::testing::IsSubsetOf;
27	using ::testing::SizeIs;
28	using ::testing::UnorderedElementsAre;
29
30	[[nodiscard]] static ::testing::AssertionResult
31	ErrorEquals(instrprof_error Expected, Error E) {
32	instrprof_error Found;
33	std::string FoundMsg;
34	handleAllErrors(E: std::move(E), Handlers: [&](const InstrProfError &IPE) {
35	Found = IPE.get();
36	FoundMsg = IPE.message();
37	});
38	if (Expected == Found)
39	return ::testing::AssertionSuccess();
40	return ::testing::AssertionFailure() << "error: " << FoundMsg << "\n";
41	}
42
43	namespace llvm {
44	bool operator==(const TemporalProfTraceTy &lhs,
45	const TemporalProfTraceTy &rhs) {
46	return lhs.Weight == rhs.Weight &&
47	lhs.FunctionNameRefs == rhs.FunctionNameRefs;
48	}
49	} // end namespace llvm
50
51	namespace {
52
53	struct InstrProfTest : ::testing::Test {
54	InstrProfWriter Writer;
55	std::unique_ptr<IndexedInstrProfReader> Reader;
56
57	void SetUp() override { Writer.setOutputSparse(false); }
58
59	void readProfile(std::unique_ptr<MemoryBuffer> Profile,
60	std::unique_ptr<MemoryBuffer> Remapping = nullptr) {
61	auto ReaderOrErr = IndexedInstrProfReader::create(Buffer: std::move(Profile),
62	RemappingBuffer: std::move(Remapping));
63	EXPECT_THAT_ERROR(ReaderOrErr.takeError(), Succeeded());
64	Reader = std::move(ReaderOrErr.get());
65	}
66	};
67
68	struct SparseInstrProfTest : public InstrProfTest {
69	void SetUp() override { Writer.setOutputSparse(true); }
70	};
71
72	struct InstrProfReaderWriterTest
73	: public InstrProfTest,
74	public ::testing::WithParamInterface<
75	std::tuple<bool, uint64_t, llvm::endianness>> {
76	void SetUp() override { Writer.setOutputSparse(std::get<0>(t: GetParam())); }
77	void TearDown() override {
78	// Reset writer value profile data endianness after each test case. Note
79	// it's not necessary to reset reader value profile endianness for each test
80	// case. Each test case creates a new reader; at reader initialization time,
81	// it uses the endianness from hash table object (which is little by
82	// default).
83	Writer.setValueProfDataEndianness(llvm::endianness::little);
84	}
85
86	uint64_t getProfWeight() const { return std::get<1>(t: GetParam()); }
87
88	llvm::endianness getEndianness() const { return std::get<2>(t: GetParam()); }
89	};
90
91	struct MaybeSparseInstrProfTest : public InstrProfTest,
92	public ::testing::WithParamInterface<bool> {
93	void SetUp() override { Writer.setOutputSparse(GetParam()); }
94	};
95
96	TEST_P(MaybeSparseInstrProfTest, write_and_read_empty_profile) {
97	auto Profile = Writer.writeBuffer();
98	readProfile(Profile: std::move(Profile));
99	ASSERT_TRUE(Reader->begin() == Reader->end());
100	}
101
102	static const auto Err = [](Error E) {
103	consumeError(Err: std::move(E));
104	FAIL();
105	};
106
107	TEST_P(MaybeSparseInstrProfTest, write_and_read_one_function) {
108	Writer.addRecord(I: {"foo", 0x1234, {1, 2, 3, 4}}, Warn: Err);
109	auto Profile = Writer.writeBuffer();
110	readProfile(Profile: std::move(Profile));
111
112	auto I = Reader->begin(), E = Reader->end();
113	ASSERT_TRUE(I != E);
114	ASSERT_EQ(StringRef("foo"), I->Name);
115	ASSERT_EQ(0x1234U, I->Hash);
116	ASSERT_EQ(4U, I->Counts.size());
117	ASSERT_EQ(1U, I->Counts[0]);
118	ASSERT_EQ(2U, I->Counts[1]);
119	ASSERT_EQ(3U, I->Counts[2]);
120	ASSERT_EQ(4U, I->Counts[3]);
121	ASSERT_TRUE(++I == E);
122	}
123
124	TEST_P(MaybeSparseInstrProfTest, get_instr_prof_record) {
125	Writer.addRecord(I: {"foo", 0x1234, {1, 2}}, Warn: Err);
126	Writer.addRecord(I: {"foo", 0x1235, {3, 4}}, Warn: Err);
127	auto Profile = Writer.writeBuffer();
128	readProfile(Profile: std::move(Profile));
129
130	Expected<InstrProfRecord> R = Reader->getInstrProfRecord(FuncName: "foo", FuncHash: 0x1234);
131	EXPECT_THAT_ERROR(R.takeError(), Succeeded());
132	ASSERT_EQ(2U, R->Counts.size());
133	ASSERT_EQ(1U, R->Counts[0]);
134	ASSERT_EQ(2U, R->Counts[1]);
135
136	R = Reader->getInstrProfRecord(FuncName: "foo", FuncHash: 0x1235);
137	EXPECT_THAT_ERROR(R.takeError(), Succeeded());
138	ASSERT_EQ(2U, R->Counts.size());
139	ASSERT_EQ(3U, R->Counts[0]);
140	ASSERT_EQ(4U, R->Counts[1]);
141
142	R = Reader->getInstrProfRecord(FuncName: "foo", FuncHash: 0x5678);
143	ASSERT_TRUE(ErrorEquals(instrprof_error::hash_mismatch, R.takeError()));
144
145	R = Reader->getInstrProfRecord(FuncName: "bar", FuncHash: 0x1234);
146	ASSERT_TRUE(ErrorEquals(instrprof_error::unknown_function, R.takeError()));
147	}
148
149	TEST_P(MaybeSparseInstrProfTest, get_function_counts) {
150	Writer.addRecord(I: {"foo", 0x1234, {1, 2}}, Warn: Err);
151	Writer.addRecord(I: {"foo", 0x1235, {3, 4}}, Warn: Err);
152	auto Profile = Writer.writeBuffer();
153	readProfile(Profile: std::move(Profile));
154
155	std::vector<uint64_t> Counts;
156	EXPECT_THAT_ERROR(Reader->getFunctionCounts("foo", 0x1234, Counts),
157	Succeeded());
158	ASSERT_EQ(2U, Counts.size());
159	ASSERT_EQ(1U, Counts[0]);
160	ASSERT_EQ(2U, Counts[1]);
161
162	EXPECT_THAT_ERROR(Reader->getFunctionCounts("foo", 0x1235, Counts),
163	Succeeded());
164	ASSERT_EQ(2U, Counts.size());
165	ASSERT_EQ(3U, Counts[0]);
166	ASSERT_EQ(4U, Counts[1]);
167
168	Error E1 = Reader->getFunctionCounts(FuncName: "foo", FuncHash: 0x5678, Counts);
169	ASSERT_TRUE(ErrorEquals(instrprof_error::hash_mismatch, std::move(E1)));
170
171	Error E2 = Reader->getFunctionCounts(FuncName: "bar", FuncHash: 0x1234, Counts);
172	ASSERT_TRUE(ErrorEquals(instrprof_error::unknown_function, std::move(E2)));
173	}
174
175	// Profile data is copied from general.proftext
176	TEST_F(InstrProfTest, get_profile_summary) {
177	Writer.addRecord(I: {"func1", 0x1234, {97531}}, Warn: Err);
178	Writer.addRecord(I: {"func2", 0x1234, {0, 0}}, Warn: Err);
179	Writer.addRecord(
180	I: {"func3",
181	0x1234,
182	{2305843009213693952, 1152921504606846976, 576460752303423488,
183	288230376151711744, 144115188075855872, 72057594037927936}},
184	Warn: Err);
185	Writer.addRecord(I: {"func4", 0x1234, {0}}, Warn: Err);
186	auto Profile = Writer.writeBuffer();
187	readProfile(Profile: std::move(Profile));
188
189	auto VerifySummary = [](ProfileSummary &IPS) mutable {
190	ASSERT_EQ(ProfileSummary::PSK_Instr, IPS.getKind());
191	ASSERT_EQ(2305843009213693952U, IPS.getMaxFunctionCount());
192	ASSERT_EQ(2305843009213693952U, IPS.getMaxCount());
193	ASSERT_EQ(10U, IPS.getNumCounts());
194	ASSERT_EQ(4539628424389557499U, IPS.getTotalCount());
195	const std::vector<ProfileSummaryEntry> &Details = IPS.getDetailedSummary();
196	uint32_t Cutoff = 800000;
197	auto Predicate = [&Cutoff](const ProfileSummaryEntry &PE) {
198	return PE.Cutoff == Cutoff;
199	};
200	auto EightyPerc = find_if(Range: Details, P: Predicate);
201	Cutoff = 900000;
202	auto NinetyPerc = find_if(Range: Details, P: Predicate);
203	Cutoff = 950000;
204	auto NinetyFivePerc = find_if(Range: Details, P: Predicate);
205	Cutoff = 990000;
206	auto NinetyNinePerc = find_if(Range: Details, P: Predicate);
207	ASSERT_EQ(576460752303423488U, EightyPerc->MinCount);
208	ASSERT_EQ(288230376151711744U, NinetyPerc->MinCount);
209	ASSERT_EQ(288230376151711744U, NinetyFivePerc->MinCount);
210	ASSERT_EQ(72057594037927936U, NinetyNinePerc->MinCount);
211	};
212	ProfileSummary &PS = Reader->getSummary(/* IsCS */ UseCS: false);
213	VerifySummary(PS);
214
215	// Test that conversion of summary to and from Metadata works.
216	LLVMContext Context;
217	Metadata *MD = PS.getMD(Context);
218	ASSERT_TRUE(MD);
219	ProfileSummary *PSFromMD = ProfileSummary::getFromMD(MD);
220	ASSERT_TRUE(PSFromMD);
221	VerifySummary(*PSFromMD);
222	delete PSFromMD;
223
224	// Test that summary can be attached to and read back from module.
225	Module M("my_module", Context);
226	M.setProfileSummary(M: MD, Kind: ProfileSummary::PSK_Instr);
227	MD = M.getProfileSummary(/* IsCS */ false);
228	ASSERT_TRUE(MD);
229	PSFromMD = ProfileSummary::getFromMD(MD);
230	ASSERT_TRUE(PSFromMD);
231	VerifySummary(*PSFromMD);
232	delete PSFromMD;
233	}
234
235	TEST_F(InstrProfTest, test_writer_merge) {
236	Writer.addRecord(I: {"func1", 0x1234, {42}}, Warn: Err);
237
238	InstrProfWriter Writer2;
239	Writer2.addRecord(I: {"func2", 0x1234, {0, 0}}, Warn: Err);
240
241	Writer.mergeRecordsFromWriter(IPW: std::move(Writer2), Warn: Err);
242
243	auto Profile = Writer.writeBuffer();
244	readProfile(Profile: std::move(Profile));
245
246	Expected<InstrProfRecord> R = Reader->getInstrProfRecord(FuncName: "func1", FuncHash: 0x1234);
247	EXPECT_THAT_ERROR(R.takeError(), Succeeded());
248	ASSERT_EQ(1U, R->Counts.size());
249	ASSERT_EQ(42U, R->Counts[0]);
250
251	R = Reader->getInstrProfRecord(FuncName: "func2", FuncHash: 0x1234);
252	EXPECT_THAT_ERROR(R.takeError(), Succeeded());
253	ASSERT_EQ(2U, R->Counts.size());
254	ASSERT_EQ(0U, R->Counts[0]);
255	ASSERT_EQ(0U, R->Counts[1]);
256	}
257
258	TEST_F(InstrProfTest, test_merge_temporal_prof_traces_truncated) {
259	uint64_t ReservoirSize = 10;
260	uint64_t MaxTraceLength = 2;
261	InstrProfWriter Writer(/*Sparse=*/false, ReservoirSize, MaxTraceLength);
262	ASSERT_THAT_ERROR(Writer.mergeProfileKind(InstrProfKind::TemporalProfile),
263	Succeeded());
264
265	TemporalProfTraceTy LargeTrace, SmallTrace;
266	LargeTrace.FunctionNameRefs = {IndexedInstrProf::ComputeHash(K: "foo"),
267	IndexedInstrProf::ComputeHash(K: "bar"),
268	IndexedInstrProf::ComputeHash(K: "goo")};
269	SmallTrace.FunctionNameRefs = {IndexedInstrProf::ComputeHash(K: "foo"),
270	IndexedInstrProf::ComputeHash(K: "bar")};
271
272	SmallVector<TemporalProfTraceTy, 4> Traces = {LargeTrace, SmallTrace};
273	Writer.addTemporalProfileTraces(SrcTraces&: Traces, SrcStreamSize: 2);
274
275	auto Profile = Writer.writeBuffer();
276	readProfile(Profile: std::move(Profile));
277
278	ASSERT_TRUE(Reader->hasTemporalProfile());
279	EXPECT_EQ(Reader->getTemporalProfTraceStreamSize(), 2U);
280	EXPECT_THAT(Reader->getTemporalProfTraces(),
281	UnorderedElementsAre(SmallTrace, SmallTrace));
282	}
283
284	TEST_F(InstrProfTest, test_merge_traces_from_writer) {
285	uint64_t ReservoirSize = 10;
286	uint64_t MaxTraceLength = 10;
287	InstrProfWriter Writer(/*Sparse=*/false, ReservoirSize, MaxTraceLength);
288	InstrProfWriter Writer2(/*Sparse=*/false, ReservoirSize, MaxTraceLength);
289	ASSERT_THAT_ERROR(Writer.mergeProfileKind(InstrProfKind::TemporalProfile),
290	Succeeded());
291	ASSERT_THAT_ERROR(Writer2.mergeProfileKind(InstrProfKind::TemporalProfile),
292	Succeeded());
293
294	TemporalProfTraceTy FooTrace, BarTrace;
295	FooTrace.FunctionNameRefs = {IndexedInstrProf::ComputeHash(K: "foo")};
296	BarTrace.FunctionNameRefs = {IndexedInstrProf::ComputeHash(K: "bar")};
297
298	SmallVector<TemporalProfTraceTy, 4> Traces1({FooTrace}), Traces2({BarTrace});
299	Writer.addTemporalProfileTraces(SrcTraces&: Traces1, SrcStreamSize: 1);
300	Writer2.addTemporalProfileTraces(SrcTraces&: Traces2, SrcStreamSize: 1);
301	Writer.mergeRecordsFromWriter(IPW: std::move(Writer2), Warn: Err);
302
303	auto Profile = Writer.writeBuffer();
304	readProfile(Profile: std::move(Profile));
305
306	ASSERT_TRUE(Reader->hasTemporalProfile());
307	EXPECT_EQ(Reader->getTemporalProfTraceStreamSize(), 2U);
308	EXPECT_THAT(Reader->getTemporalProfTraces(),
309	UnorderedElementsAre(FooTrace, BarTrace));
310	}
311
312	TEST_F(InstrProfTest, test_merge_traces_sampled) {
313	uint64_t ReservoirSize = 3;
314	uint64_t MaxTraceLength = 10;
315	InstrProfWriter Writer(/*Sparse=*/false, ReservoirSize, MaxTraceLength);
316	ASSERT_THAT_ERROR(Writer.mergeProfileKind(InstrProfKind::TemporalProfile),
317	Succeeded());
318
319	TemporalProfTraceTy FooTrace, BarTrace, GooTrace;
320	FooTrace.FunctionNameRefs = {IndexedInstrProf::ComputeHash(K: "foo")};
321	BarTrace.FunctionNameRefs = {IndexedInstrProf::ComputeHash(K: "bar")};
322	GooTrace.FunctionNameRefs = {IndexedInstrProf::ComputeHash(K: "Goo")};
323
324	// Add some sampled traces
325	SmallVector<TemporalProfTraceTy, 4> SampledTraces = {FooTrace, BarTrace,
326	GooTrace};
327	Writer.addTemporalProfileTraces(SrcTraces&: SampledTraces, SrcStreamSize: 5);
328	// Add some unsampled traces
329	SmallVector<TemporalProfTraceTy, 4> UnsampledTraces = {BarTrace, GooTrace};
330	Writer.addTemporalProfileTraces(SrcTraces&: UnsampledTraces, SrcStreamSize: 2);
331	UnsampledTraces = {FooTrace};
332	Writer.addTemporalProfileTraces(SrcTraces&: UnsampledTraces, SrcStreamSize: 1);
333
334	auto Profile = Writer.writeBuffer();
335	readProfile(Profile: std::move(Profile));
336
337	ASSERT_TRUE(Reader->hasTemporalProfile());
338	EXPECT_EQ(Reader->getTemporalProfTraceStreamSize(), 8U);
339	// Check that we have a subset of all the traces we added
340	EXPECT_THAT(Reader->getTemporalProfTraces(), SizeIs(ReservoirSize));
341	EXPECT_THAT(
342	Reader->getTemporalProfTraces(),
343	IsSubsetOf({FooTrace, BarTrace, GooTrace, BarTrace, GooTrace, FooTrace}));
344	}
345
346	using ::llvm::memprof::IndexedMemProfRecord;
347	using ::llvm::memprof::MemInfoBlock;
348	using FrameIdMapTy =
349	llvm::DenseMap<::llvm::memprof::FrameId, ::llvm::memprof::Frame>;
350	using CallStackIdMapTy =
351	llvm::DenseMap<::llvm::memprof::CallStackId,
352	::llvm::SmallVector<::llvm::memprof::FrameId>>;
353
354	static FrameIdMapTy getFrameMapping() {
355	FrameIdMapTy Mapping;
356	Mapping.insert(KV: {0, {0x123, 1, 2, false}});
357	Mapping.insert(KV: {1, {0x345, 3, 4, true}});
358	Mapping.insert(KV: {2, {0x125, 5, 6, false}});
359	Mapping.insert(KV: {3, {0x567, 7, 8, true}});
360	Mapping.insert(KV: {4, {0x124, 5, 6, false}});
361	Mapping.insert(KV: {5, {0x789, 8, 9, true}});
362	return Mapping;
363	}
364
365	static CallStackIdMapTy getCallStackMapping() {
366	CallStackIdMapTy Mapping;
367	Mapping.insert(KV: {0x111, {0, 1}});
368	Mapping.insert(KV: {0x222, {2, 3}});
369	Mapping.insert(KV: {0x333, {4, 5}});
370	return Mapping;
371	}
372
373	IndexedMemProfRecord makeRecord(
374	std::initializer_list<std::initializer_list<::llvm::memprof::FrameId>>
375	AllocFrames,
376	std::initializer_list<std::initializer_list<::llvm::memprof::FrameId>>
377	CallSiteFrames,
378	const MemInfoBlock &Block = MemInfoBlock()) {
379	llvm::memprof::IndexedMemProfRecord MR;
380	for (const auto &Frames : AllocFrames)
381	MR.AllocSites.emplace_back(Args: Frames, Args: llvm::memprof::hashCallStack(CS: Frames),
382	Args: Block);
383	for (const auto &Frames : CallSiteFrames)
384	MR.CallSites.push_back(Elt: Frames);
385	return MR;
386	}
387
388	IndexedMemProfRecord
389	makeRecordV2(std::initializer_list<::llvm::memprof::CallStackId> AllocFrames,
390	std::initializer_list<::llvm::memprof::CallStackId> CallSiteFrames,
391	const MemInfoBlock &Block = MemInfoBlock()) {
392	llvm::memprof::IndexedMemProfRecord MR;
393	for (const auto &CSId : AllocFrames)
394	// We don't populate IndexedAllocationInfo::CallStack because we use it only
395	// in Version0 and Version1.
396	MR.AllocSites.emplace_back(Args: ::llvm::SmallVector<memprof::FrameId>(), Args: CSId,
397	Args: Block);
398	for (const auto &CSId : CallSiteFrames)
399	MR.CallSiteIds.push_back(Elt: CSId);
400	return MR;
401	}
402
403	MATCHER_P(EqualsRecord, Want, "") {
404	const memprof::MemProfRecord &Got = arg;
405
406	auto PrintAndFail = [&]() {
407	std::string Buffer;
408	llvm::raw_string_ostream OS(Buffer);
409	OS << "Want:\n";
410	Want.print(OS);
411	OS << "Got:\n";
412	Got.print(OS);
413	OS.flush();
414	*result_listener << "MemProf Record differs!\n" << Buffer;
415	return false;
416	};
417
418	if (Want.AllocSites.size() != Got.AllocSites.size())
419	return PrintAndFail();
420	if (Want.CallSites.size() != Got.CallSites.size())
421	return PrintAndFail();
422
423	for (size_t I = 0; I < Got.AllocSites.size(); I++) {
424	if (Want.AllocSites[I].Info != Got.AllocSites[I].Info)
425	return PrintAndFail();
426	if (Want.AllocSites[I].CallStack != Got.AllocSites[I].CallStack)
427	return PrintAndFail();
428	}
429
430	for (size_t I = 0; I < Got.CallSites.size(); I++) {
431	if (Want.CallSites[I] != Got.CallSites[I])
432	return PrintAndFail();
433	}
434	return true;
435	}
436
437	TEST_F(InstrProfTest, test_memprof_v0) {
438	ASSERT_THAT_ERROR(Writer.mergeProfileKind(InstrProfKind::MemProf),
439	Succeeded());
440
441	const IndexedMemProfRecord IndexedMR = makeRecord(
442	/*AllocFrames=*/
443	{
444	{0, 1},
445	{2, 3},
446	},
447	/*CallSiteFrames=*/{
448	{4, 5},
449	});
450	const FrameIdMapTy IdToFrameMap = getFrameMapping();
451	for (const auto &I : IdToFrameMap) {
452	Writer.addMemProfFrame(I.first, F: I.getSecond(), Warn: Err);
453	}
454	Writer.addMemProfRecord(/*Id=*/0x9999, Record: IndexedMR);
455
456	auto Profile = Writer.writeBuffer();
457	readProfile(Profile: std::move(Profile));
458
459	auto RecordOr = Reader->getMemProfRecord(FuncNameHash: 0x9999);
460	ASSERT_THAT_ERROR(RecordOr.takeError(), Succeeded());
461	const memprof::MemProfRecord &Record = RecordOr.get();
462
463	std::optional<memprof::FrameId> LastUnmappedFrameId;
464	auto IdToFrameCallback = [&](const memprof::FrameId Id) {
465	auto Iter = IdToFrameMap.find(Val: Id);
466	if (Iter == IdToFrameMap.end()) {
467	LastUnmappedFrameId = Id;
468	return memprof::Frame(0, 0, 0, false);
469	}
470	return Iter->second;
471	};
472
473	const memprof::MemProfRecord WantRecord(IndexedMR, IdToFrameCallback);
474	ASSERT_EQ(LastUnmappedFrameId, std::nullopt)
475	<< "could not map frame id: " << *LastUnmappedFrameId;
476	EXPECT_THAT(WantRecord, EqualsRecord(Record));
477	}
478
479	TEST_F(InstrProfTest, test_memprof_v2) {
480	Writer.setMemProfVersionRequested(memprof::Version2);
481
482	ASSERT_THAT_ERROR(Writer.mergeProfileKind(InstrProfKind::MemProf),
483	Succeeded());
484
485	const IndexedMemProfRecord IndexedMR = makeRecordV2(
486	/*AllocFrames=*/{0x111, 0x222},
487	/*CallSiteFrames=*/{0x333});
488	const FrameIdMapTy IdToFrameMap = getFrameMapping();
489	const auto CSIdToCallStackMap = getCallStackMapping();
490	for (const auto &I : IdToFrameMap) {
491	Writer.addMemProfFrame(I.first, F: I.getSecond(), Warn: Err);
492	}
493	for (const auto &I : CSIdToCallStackMap) {
494	Writer.addMemProfCallStack(CSId: I.first, CallStack: I.getSecond(), Warn: Err);
495	}
496	Writer.addMemProfRecord(/*Id=*/0x9999, Record: IndexedMR);
497
498	auto Profile = Writer.writeBuffer();
499	readProfile(Profile: std::move(Profile));
500
501	auto RecordOr = Reader->getMemProfRecord(FuncNameHash: 0x9999);
502	ASSERT_THAT_ERROR(RecordOr.takeError(), Succeeded());
503	const memprof::MemProfRecord &Record = RecordOr.get();
504
505	std::optional<memprof::FrameId> LastUnmappedFrameId;
506	auto IdToFrameCallback = [&](const memprof::FrameId Id) {
507	auto Iter = IdToFrameMap.find(Val: Id);
508	if (Iter == IdToFrameMap.end()) {
509	LastUnmappedFrameId = Id;
510	return memprof::Frame(0, 0, 0, false);
511	}
512	return Iter->second;
513	};
514
515	std::optional<::llvm::memprof::CallStackId> LastUnmappedCSId;
516	auto CSIdToCallStackCallback = [&](::llvm::memprof::CallStackId CSId) {
517	llvm::SmallVector<memprof::Frame> Frames;
518	auto CSIter = CSIdToCallStackMap.find(Val: CSId);
519	if (CSIter == CSIdToCallStackMap.end()) {
520	LastUnmappedCSId = CSId;
521	} else {
522	const ::llvm::SmallVector<::llvm::memprof::FrameId> &CS =
523	CSIter->getSecond();
524	Frames.reserve(N: CS.size());
525	for (::llvm::memprof::FrameId Id : CS)
526	Frames.push_back(Elt: IdToFrameCallback(Id));
527	}
528	return Frames;
529	};
530
531	const ::llvm::memprof::MemProfRecord WantRecord =
532	IndexedMR.toMemProfRecord(Callback: CSIdToCallStackCallback);
533	ASSERT_EQ(LastUnmappedFrameId, std::nullopt)
534	<< "could not map frame id: " << *LastUnmappedFrameId;
535	ASSERT_EQ(LastUnmappedCSId, std::nullopt)
536	<< "could not map call stack id: " << *LastUnmappedCSId;
537	EXPECT_THAT(WantRecord, EqualsRecord(Record));
538	}
539
540	TEST_F(InstrProfTest, test_memprof_getrecord_error) {
541	ASSERT_THAT_ERROR(Writer.mergeProfileKind(InstrProfKind::MemProf),
542	Succeeded());
543
544	const IndexedMemProfRecord IndexedMR = makeRecord(
545	/*AllocFrames=*/
546	{
547	{0, 1},
548	{2, 3},
549	},
550	/*CallSiteFrames=*/{
551	{4, 5},
552	});
553	// We skip adding the frame mappings here unlike the test_memprof unit test
554	// above to exercise the failure path when getMemProfRecord is invoked.
555	Writer.addMemProfRecord(/*Id=*/0x9999, Record: IndexedMR);
556
557	auto Profile = Writer.writeBuffer();
558	readProfile(Profile: std::move(Profile));
559
560	// Missing frames give a hash_mismatch error.
561	auto RecordOr = Reader->getMemProfRecord(FuncNameHash: 0x9999);
562	ASSERT_TRUE(
563	ErrorEquals(instrprof_error::hash_mismatch, RecordOr.takeError()));
564
565	// Missing functions give a unknown_function error.
566	RecordOr = Reader->getMemProfRecord(FuncNameHash: 0x1111);
567	ASSERT_TRUE(
568	ErrorEquals(instrprof_error::unknown_function, RecordOr.takeError()));
569	}
570
571	TEST_F(InstrProfTest, test_memprof_merge) {
572	Writer.addRecord(I: {"func1", 0x1234, {42}}, Warn: Err);
573
574	InstrProfWriter Writer2;
575	ASSERT_THAT_ERROR(Writer2.mergeProfileKind(InstrProfKind::MemProf),
576	Succeeded());
577
578	const IndexedMemProfRecord IndexedMR = makeRecord(
579	/*AllocFrames=*/
580	{
581	{0, 1},
582	{2, 3},
583	},
584	/*CallSiteFrames=*/{
585	{4, 5},
586	});
587
588	const FrameIdMapTy IdToFrameMap = getFrameMapping();
589	for (const auto &I : IdToFrameMap) {
590	Writer.addMemProfFrame(I.first, F: I.getSecond(), Warn: Err);
591	}
592	Writer2.addMemProfRecord(/*Id=*/0x9999, Record: IndexedMR);
593
594	ASSERT_THAT_ERROR(Writer.mergeProfileKind(Writer2.getProfileKind()),
595	Succeeded());
596	Writer.mergeRecordsFromWriter(IPW: std::move(Writer2), Warn: Err);
597
598	auto Profile = Writer.writeBuffer();
599	readProfile(Profile: std::move(Profile));
600
601	Expected<InstrProfRecord> R = Reader->getInstrProfRecord(FuncName: "func1", FuncHash: 0x1234);
602	EXPECT_THAT_ERROR(R.takeError(), Succeeded());
603	ASSERT_EQ(1U, R->Counts.size());
604	ASSERT_EQ(42U, R->Counts[0]);
605
606	auto RecordOr = Reader->getMemProfRecord(FuncNameHash: 0x9999);
607	ASSERT_THAT_ERROR(RecordOr.takeError(), Succeeded());
608	const memprof::MemProfRecord &Record = RecordOr.get();
609
610	std::optional<memprof::FrameId> LastUnmappedFrameId;
611
612	auto IdToFrameCallback = [&](const memprof::FrameId Id) {
613	auto Iter = IdToFrameMap.find(Val: Id);
614	if (Iter == IdToFrameMap.end()) {
615	LastUnmappedFrameId = Id;
616	return memprof::Frame(0, 0, 0, false);
617	}
618	return Iter->second;
619	};
620
621	const memprof::MemProfRecord WantRecord(IndexedMR, IdToFrameCallback);
622	ASSERT_EQ(LastUnmappedFrameId, std::nullopt)
623	<< "could not map frame id: " << *LastUnmappedFrameId;
624	EXPECT_THAT(WantRecord, EqualsRecord(Record));
625	}
626
627	TEST_F(InstrProfTest, test_irpgo_function_name) {
628	LLVMContext Ctx;
629	auto M = std::make_unique<Module>(args: "MyModule.cpp", args&: Ctx);
630	auto *FTy = FunctionType::get(Result: Type::getVoidTy(C&: Ctx), /*isVarArg=*/false);
631
632	std::vector<std::tuple<StringRef, Function::LinkageTypes, StringRef>> Data;
633	Data.emplace_back(args: "ExternalFoo", args: Function::ExternalLinkage, args: "ExternalFoo");
634	Data.emplace_back(args: "InternalFoo", args: Function::InternalLinkage,
635	args: "MyModule.cpp;InternalFoo");
636	Data.emplace_back(args: "\01-[C dynamicFoo:]", args: Function::ExternalLinkage,
637	args: "-[C dynamicFoo:]");
638	Data.emplace_back(args: "\01-[C internalFoo:]", args: Function::InternalLinkage,
639	args: "MyModule.cpp;-[C internalFoo:]");
640
641	for (auto &[Name, Linkage, ExpectedIRPGOFuncName] : Data)
642	Function::Create(Ty: FTy, Linkage, N: Name, M: M.get());
643
644	for (auto &[Name, Linkage, ExpectedIRPGOFuncName] : Data) {
645	auto *F = M->getFunction(Name);
646	auto IRPGOFuncName = getIRPGOFuncName(F: *F);
647	EXPECT_EQ(IRPGOFuncName, ExpectedIRPGOFuncName);
648
649	auto [Filename, ParsedIRPGOFuncName] = getParsedIRPGOName(IRPGOName: IRPGOFuncName);
650	StringRef ExpectedParsedIRPGOFuncName = IRPGOFuncName;
651	if (ExpectedParsedIRPGOFuncName.consume_front(Prefix: "MyModule.cpp;")) {
652	EXPECT_EQ(Filename, "MyModule.cpp");
653	} else {
654	EXPECT_EQ(Filename, "");
655	}
656	EXPECT_EQ(ParsedIRPGOFuncName, ExpectedParsedIRPGOFuncName);
657	}
658	}
659
660	TEST_F(InstrProfTest, test_pgo_function_name) {
661	LLVMContext Ctx;
662	auto M = std::make_unique<Module>(args: "MyModule.cpp", args&: Ctx);
663	auto *FTy = FunctionType::get(Result: Type::getVoidTy(C&: Ctx), /*isVarArg=*/false);
664
665	std::vector<std::tuple<StringRef, Function::LinkageTypes, StringRef>> Data;
666	Data.emplace_back(args: "ExternalFoo", args: Function::ExternalLinkage, args: "ExternalFoo");
667	Data.emplace_back(args: "InternalFoo", args: Function::InternalLinkage,
668	args: "MyModule.cpp:InternalFoo");
669	Data.emplace_back(args: "\01-[C externalFoo:]", args: Function::ExternalLinkage,
670	args: "-[C externalFoo:]");
671	Data.emplace_back(args: "\01-[C internalFoo:]", args: Function::InternalLinkage,
672	args: "MyModule.cpp:-[C internalFoo:]");
673
674	for (auto &[Name, Linkage, ExpectedPGOFuncName] : Data)
675	Function::Create(Ty: FTy, Linkage, N: Name, M: M.get());
676
677	for (auto &[Name, Linkage, ExpectedPGOFuncName] : Data) {
678	auto *F = M->getFunction(Name);
679	EXPECT_EQ(getPGOFuncName(*F), ExpectedPGOFuncName);
680	}
681	}
682
683	TEST_F(InstrProfTest, test_irpgo_read_deprecated_names) {
684	LLVMContext Ctx;
685	auto M = std::make_unique<Module>(args: "MyModule.cpp", args&: Ctx);
686	auto *FTy = FunctionType::get(Result: Type::getVoidTy(C&: Ctx), /*isVarArg=*/false);
687	auto *InternalFooF =
688	Function::Create(Ty: FTy, Linkage: Function::InternalLinkage, N: "InternalFoo", M: M.get());
689	auto *ExternalFooF =
690	Function::Create(Ty: FTy, Linkage: Function::ExternalLinkage, N: "ExternalFoo", M: M.get());
691
692	auto *InternalBarF =
693	Function::Create(Ty: FTy, Linkage: Function::InternalLinkage, N: "InternalBar", M: M.get());
694	auto *ExternalBarF =
695	Function::Create(Ty: FTy, Linkage: Function::ExternalLinkage, N: "ExternalBar", M: M.get());
696
697	Writer.addRecord(I: {getIRPGOFuncName(F: *InternalFooF), 0x1234, {1}}, Warn: Err);
698	Writer.addRecord(I: {getIRPGOFuncName(F: *ExternalFooF), 0x5678, {1}}, Warn: Err);
699	// Write a record with a deprecated name
700	Writer.addRecord(I: {getPGOFuncName(F: *InternalBarF), 0x1111, {2}}, Warn: Err);
701	Writer.addRecord(I: {getPGOFuncName(F: *ExternalBarF), 0x2222, {2}}, Warn: Err);
702
703	auto Profile = Writer.writeBuffer();
704	readProfile(Profile: std::move(Profile));
705
706	EXPECT_THAT_EXPECTED(
707	Reader->getInstrProfRecord(getIRPGOFuncName(*InternalFooF), 0x1234,
708	getPGOFuncName(*InternalFooF)),
709	Succeeded());
710	EXPECT_THAT_EXPECTED(
711	Reader->getInstrProfRecord(getIRPGOFuncName(*ExternalFooF), 0x5678,
712	getPGOFuncName(*ExternalFooF)),
713	Succeeded());
714	// Ensure we can still read this old record name
715	EXPECT_THAT_EXPECTED(
716	Reader->getInstrProfRecord(getIRPGOFuncName(*InternalBarF), 0x1111,
717	getPGOFuncName(*InternalBarF)),
718	Succeeded());
719	EXPECT_THAT_EXPECTED(
720	Reader->getInstrProfRecord(getIRPGOFuncName(*ExternalBarF), 0x2222,
721	getPGOFuncName(*ExternalBarF)),
722	Succeeded());
723	}
724
725	// callee1 to callee6 are from vtable1 to vtable6 respectively.
726	static const char callee1[] = "callee1";
727	static const char callee2[] = "callee2";
728	static const char callee3[] = "callee3";
729	static const char callee4[] = "callee4";
730	static const char callee5[] = "callee5";
731	static const char callee6[] = "callee6";
732	// callee7 and callee8 are not from any vtables.
733	static const char callee7[] = "callee7";
734	static const char callee8[] = "callee8";
735	// 'callee' is primarily used to create multiple-element vtables.
736	static const char callee[] = "callee";
737	static const uint64_t vtable1[] = {uint64_t(callee), uint64_t(callee1)};
738	static const uint64_t vtable2[] = {uint64_t(callee2), uint64_t(callee)};
739	static const uint64_t vtable3[] = {
740	uint64_t(callee),
741	uint64_t(callee3),
742	};
743	static const uint64_t vtable4[] = {uint64_t(callee4), uint64_t(callee)};
744	static const uint64_t vtable5[] = {uint64_t(callee5), uint64_t(callee)};
745	static const uint64_t vtable6[] = {uint64_t(callee6), uint64_t(callee)};
746
747	// Returns the address of callee with a numbered suffix in vtable.
748	static uint64_t getCalleeAddress(const uint64_t *vtableAddr) {
749	uint64_t CalleeAddr;
750	// Callee with a numbered suffix is the 2nd element in vtable1 and vtable3,
751	// and the 1st element in the rest of vtables.
752	if (vtableAddr == vtable1 || vtableAddr == vtable3)
753	CalleeAddr = uint64_t(vtableAddr) + 8;
754	else
755	CalleeAddr = uint64_t(vtableAddr);
756	return CalleeAddr;
757	}
758
759	TEST_P(InstrProfReaderWriterTest, icall_and_vtable_data_read_write) {
760	NamedInstrProfRecord Record1("caller", 0x1234, {1, 2});
761
762	// 4 indirect call value sites.
763	{
764	Record1.reserveSites(ValueKind: IPVK_IndirectCallTarget, NumValueSites: 4);
765	InstrProfValueData VD0[] = {
766	{.Value: (uint64_t)callee1, .Count: 1}, {.Value: (uint64_t)callee2, .Count: 2}, {.Value: (uint64_t)callee3, .Count: 3}};
767	Record1.addValueData(ValueKind: IPVK_IndirectCallTarget, Site: 0, VData: VD0, N: 3, SymTab: nullptr);
768	// No value profile data at the second site.
769	Record1.addValueData(ValueKind: IPVK_IndirectCallTarget, Site: 1, VData: nullptr, N: 0, SymTab: nullptr);
770	InstrProfValueData VD2[] = {{.Value: (uint64_t)callee1, .Count: 1}, {.Value: (uint64_t)callee2, .Count: 2}};
771	Record1.addValueData(ValueKind: IPVK_IndirectCallTarget, Site: 2, VData: VD2, N: 2, SymTab: nullptr);
772	InstrProfValueData VD3[] = {{.Value: (uint64_t)callee7, .Count: 1}, {.Value: (uint64_t)callee8, .Count: 2}};
773	Record1.addValueData(ValueKind: IPVK_IndirectCallTarget, Site: 3, VData: VD3, N: 2, SymTab: nullptr);
774	}
775
776	// 2 vtable value sites.
777	{
778	InstrProfValueData VD0[] = {
779	{.Value: getCalleeAddress(vtableAddr: vtable1), .Count: 1},
780	{.Value: getCalleeAddress(vtableAddr: vtable2), .Count: 2},
781	{.Value: getCalleeAddress(vtableAddr: vtable3), .Count: 3},
782	};
783	InstrProfValueData VD2[] = {
784	{.Value: getCalleeAddress(vtableAddr: vtable1), .Count: 1},
785	{.Value: getCalleeAddress(vtableAddr: vtable2), .Count: 2},
786	};
787	Record1.addValueData(ValueKind: IPVK_VTableTarget, Site: 0, VData: VD0, N: 3, SymTab: nullptr);
788	Record1.addValueData(ValueKind: IPVK_VTableTarget, Site: 2, VData: VD2, N: 2, SymTab: nullptr);
789	}
790
791	Writer.addRecord(I: std::move(Record1), Weight: getProfWeight(), Warn: Err);
792	Writer.addRecord(I: {"callee1", 0x1235, {3, 4}}, Warn: Err);
793	Writer.addRecord(I: {"callee2", 0x1235, {3, 4}}, Warn: Err);
794	Writer.addRecord(I: {"callee3", 0x1235, {3, 4}}, Warn: Err);
795	Writer.addRecord(I: {"callee7", 0x1235, {3, 4}}, Warn: Err);
796	Writer.addRecord(I: {"callee8", 0x1235, {3, 4}}, Warn: Err);
797
798	// Set writer value prof data endianness.
799	Writer.setValueProfDataEndianness(getEndianness());
800
801	auto Profile = Writer.writeBuffer();
802	readProfile(Profile: std::move(Profile));
803
804	// Set reader value prof data endianness.
805	Reader->setValueProfDataEndianness(getEndianness());
806
807	Expected<InstrProfRecord> R = Reader->getInstrProfRecord(FuncName: "caller", FuncHash: 0x1234);
808	ASSERT_THAT_ERROR(R.takeError(), Succeeded());
809
810	// Test the number of instrumented indirect call sites and the number of
811	// profiled values at each site.
812	ASSERT_EQ(4U, R->getNumValueSites(IPVK_IndirectCallTarget));
813	EXPECT_EQ(3U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 0));
814	EXPECT_EQ(0U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 1));
815	EXPECT_EQ(2U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 2));
816	EXPECT_EQ(2U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 3));
817
818	// Test the number of instrumented vtable sites and the number of profiled
819	// values at each site.
820	ASSERT_EQ(R->getNumValueSites(IPVK_VTableTarget), 2U);
821	EXPECT_EQ(R->getNumValueDataForSite(IPVK_VTableTarget, 0), 3U);
822	EXPECT_EQ(R->getNumValueDataForSite(IPVK_VTableTarget, 1), 2U);
823
824	// First indirect site.
825	{
826	uint64_t TotalC;
827	auto VD = R->getValueForSite(ValueKind: IPVK_IndirectCallTarget, Site: 0, TotalC: &TotalC);
828
829	EXPECT_EQ(VD[0].Count, 3U * getProfWeight());
830	EXPECT_EQ(VD[1].Count, 2U * getProfWeight());
831	EXPECT_EQ(VD[2].Count, 1U * getProfWeight());
832	EXPECT_EQ(TotalC, 6U * getProfWeight());
833
834	EXPECT_STREQ((const char *)VD[0].Value, "callee3");
835	EXPECT_STREQ((const char *)VD[1].Value, "callee2");
836	EXPECT_STREQ((const char *)VD[2].Value, "callee1");
837	}
838
839	// First vtable site.
840	{
841	uint64_t TotalC;
842	auto VD = R->getValueForSite(ValueKind: IPVK_VTableTarget, Site: 0, TotalC: &TotalC);
843
844	EXPECT_EQ(VD[0].Count, 3U * getProfWeight());
845	EXPECT_EQ(VD[1].Count, 2U * getProfWeight());
846	EXPECT_EQ(VD[2].Count, 1U * getProfWeight());
847	EXPECT_EQ(TotalC, 6U * getProfWeight());
848
849	EXPECT_EQ(VD[0].Value, getCalleeAddress(vtable3));
850	EXPECT_EQ(VD[1].Value, getCalleeAddress(vtable2));
851	EXPECT_EQ(VD[2].Value, getCalleeAddress(vtable1));
852	}
853
854	// Second vtable site.
855	{
856	uint64_t TotalC;
857	auto VD = R->getValueForSite(ValueKind: IPVK_VTableTarget, Site: 1, TotalC: &TotalC);
858
859	EXPECT_EQ(VD[0].Count, 2U * getProfWeight());
860	EXPECT_EQ(VD[1].Count, 1U * getProfWeight());
861	EXPECT_EQ(TotalC, 3U * getProfWeight());
862
863	EXPECT_EQ(VD[0].Value, getCalleeAddress(vtable2));
864	EXPECT_EQ(VD[1].Value, getCalleeAddress(vtable1));
865	}
866	}
867
868	INSTANTIATE_TEST_SUITE_P(
869	WeightAndEndiannessTest, InstrProfReaderWriterTest,
870	::testing::Combine(
871	::testing::Bool(), /* Sparse */
872	::testing::Values(1U, 10U), /* ProfWeight */
873	::testing::Values(llvm::endianness::big,
874	llvm::endianness::little) /* Endianness */
875	));
876
877	TEST_P(MaybeSparseInstrProfTest, annotate_vp_data) {
878	NamedInstrProfRecord Record("caller", 0x1234, {1, 2});
879	Record.reserveSites(ValueKind: IPVK_IndirectCallTarget, NumValueSites: 1);
880	InstrProfValueData VD0[] = {{.Value: 1000, .Count: 1}, {.Value: 2000, .Count: 2}, {.Value: 3000, .Count: 3}, {.Value: 5000, .Count: 5},
881	{.Value: 4000, .Count: 4}, {.Value: 6000, .Count: 6}};
882	Record.addValueData(ValueKind: IPVK_IndirectCallTarget, Site: 0, VData: VD0, N: 6, SymTab: nullptr);
883	Writer.addRecord(I: std::move(Record), Warn: Err);
884	auto Profile = Writer.writeBuffer();
885	readProfile(Profile: std::move(Profile));
886	Expected<InstrProfRecord> R = Reader->getInstrProfRecord(FuncName: "caller", FuncHash: 0x1234);
887	EXPECT_THAT_ERROR(R.takeError(), Succeeded());
888
889	LLVMContext Ctx;
890	std::unique_ptr<Module> M(new Module("MyModule", Ctx));
891	FunctionType *FTy = FunctionType::get(Result: Type::getVoidTy(C&: Ctx),
892	/*isVarArg=*/false);
893	Function *F =
894	Function::Create(Ty: FTy, Linkage: Function::ExternalLinkage, N: "caller", M: M.get());
895	BasicBlock *BB = BasicBlock::Create(Context&: Ctx, Name: "", Parent: F);
896
897	IRBuilder<> Builder(BB);
898	BasicBlock *TBB = BasicBlock::Create(Context&: Ctx, Name: "", Parent: F);
899	BasicBlock *FBB = BasicBlock::Create(Context&: Ctx, Name: "", Parent: F);
900
901	// Use branch instruction to annotate with value profile data for simplicity
902	Instruction *Inst = Builder.CreateCondBr(Cond: Builder.getTrue(), True: TBB, False: FBB);
903	Instruction *Inst2 = Builder.CreateCondBr(Cond: Builder.getTrue(), True: TBB, False: FBB);
904	annotateValueSite(M&: *M, Inst&: *Inst, InstrProfR: R.get(), ValueKind: IPVK_IndirectCallTarget, SiteIndx: 0);
905
906	InstrProfValueData ValueData[5];
907	uint32_t N;
908	uint64_t T;
909	bool Res = getValueProfDataFromInst(Inst: *Inst, ValueKind: IPVK_IndirectCallTarget, MaxNumValueData: 5,
910	ValueData, ActualNumValueData&: N, TotalC&: T);
911	ASSERT_TRUE(Res);
912	ASSERT_EQ(3U, N);
913	ASSERT_EQ(21U, T);
914	// The result should be sorted already:
915	ASSERT_EQ(6000U, ValueData[0].Value);
916	ASSERT_EQ(6U, ValueData[0].Count);
917	ASSERT_EQ(5000U, ValueData[1].Value);
918	ASSERT_EQ(5U, ValueData[1].Count);
919	ASSERT_EQ(4000U, ValueData[2].Value);
920	ASSERT_EQ(4U, ValueData[2].Count);
921	Res = getValueProfDataFromInst(Inst: *Inst, ValueKind: IPVK_IndirectCallTarget, MaxNumValueData: 1, ValueData,
922	ActualNumValueData&: N, TotalC&: T);
923	ASSERT_TRUE(Res);
924	ASSERT_EQ(1U, N);
925	ASSERT_EQ(21U, T);
926
927	Res = getValueProfDataFromInst(Inst: *Inst2, ValueKind: IPVK_IndirectCallTarget, MaxNumValueData: 5, ValueData,
928	ActualNumValueData&: N, TotalC&: T);
929	ASSERT_FALSE(Res);
930
931	// Remove the MD_prof metadata
932	Inst->setMetadata(KindID: LLVMContext::MD_prof, Node: 0);
933	// Annotate 5 records this time.
934	annotateValueSite(M&: *M, Inst&: *Inst, InstrProfR: R.get(), ValueKind: IPVK_IndirectCallTarget, SiteIndx: 0, MaxMDCount: 5);
935	Res = getValueProfDataFromInst(Inst: *Inst, ValueKind: IPVK_IndirectCallTarget, MaxNumValueData: 5,
936	ValueData, ActualNumValueData&: N, TotalC&: T);
937	ASSERT_TRUE(Res);
938	ASSERT_EQ(5U, N);
939	ASSERT_EQ(21U, T);
940	ASSERT_EQ(6000U, ValueData[0].Value);
941	ASSERT_EQ(6U, ValueData[0].Count);
942	ASSERT_EQ(5000U, ValueData[1].Value);
943	ASSERT_EQ(5U, ValueData[1].Count);
944	ASSERT_EQ(4000U, ValueData[2].Value);
945	ASSERT_EQ(4U, ValueData[2].Count);
946	ASSERT_EQ(3000U, ValueData[3].Value);
947	ASSERT_EQ(3U, ValueData[3].Count);
948	ASSERT_EQ(2000U, ValueData[4].Value);
949	ASSERT_EQ(2U, ValueData[4].Count);
950
951	// Remove the MD_prof metadata
952	Inst->setMetadata(KindID: LLVMContext::MD_prof, Node: 0);
953	// Annotate with 4 records.
954	InstrProfValueData VD0Sorted[] = {{.Value: 1000, .Count: 6}, {.Value: 2000, .Count: 5}, {.Value: 3000, .Count: 4}, {.Value: 4000, .Count: 3},
955	{.Value: 5000, .Count: 2}, {.Value: 6000, .Count: 1}};
956	annotateValueSite(M&: *M, Inst&: *Inst, VDs: ArrayRef(VD0Sorted).slice(N: 2), Sum: 10,
957	ValueKind: IPVK_IndirectCallTarget, MaxMDCount: 5);
958	Res = getValueProfDataFromInst(Inst: *Inst, ValueKind: IPVK_IndirectCallTarget, MaxNumValueData: 5,
959	ValueData, ActualNumValueData&: N, TotalC&: T);
960	ASSERT_TRUE(Res);
961	ASSERT_EQ(4U, N);
962	ASSERT_EQ(10U, T);
963	ASSERT_EQ(3000U, ValueData[0].Value);
964	ASSERT_EQ(4U, ValueData[0].Count);
965	ASSERT_EQ(4000U, ValueData[1].Value);
966	ASSERT_EQ(3U, ValueData[1].Count);
967	ASSERT_EQ(5000U, ValueData[2].Value);
968	ASSERT_EQ(2U, ValueData[2].Count);
969	ASSERT_EQ(6000U, ValueData[3].Value);
970	ASSERT_EQ(1U, ValueData[3].Count);
971	}
972
973	TEST_P(MaybeSparseInstrProfTest, icall_and_vtable_data_merge) {
974	static const char caller[] = "caller";
975	NamedInstrProfRecord Record11(caller, 0x1234, {1, 2});
976	NamedInstrProfRecord Record12(caller, 0x1234, {1, 2});
977
978	// 5 value sites for indirect calls.
979	{
980	Record11.reserveSites(ValueKind: IPVK_IndirectCallTarget, NumValueSites: 5);
981	InstrProfValueData VD0[] = {{.Value: uint64_t(callee1), .Count: 1},
982	{.Value: uint64_t(callee2), .Count: 2},
983	{.Value: uint64_t(callee3), .Count: 3},
984	{.Value: uint64_t(callee4), .Count: 4}};
985	Record11.addValueData(ValueKind: IPVK_IndirectCallTarget, Site: 0, VData: VD0, N: 4, SymTab: nullptr);
986
987	// No value profile data at the second site.
988	Record11.addValueData(ValueKind: IPVK_IndirectCallTarget, Site: 1, VData: nullptr, N: 0, SymTab: nullptr);
989
990	InstrProfValueData VD2[] = {
991	{.Value: uint64_t(callee1), .Count: 1}, {.Value: uint64_t(callee2), .Count: 2}, {.Value: uint64_t(callee3), .Count: 3}};
992	Record11.addValueData(ValueKind: IPVK_IndirectCallTarget, Site: 2, VData: VD2, N: 3, SymTab: nullptr);
993
994	InstrProfValueData VD3[] = {{.Value: uint64_t(callee7), .Count: 1}, {.Value: uint64_t(callee8), .Count: 2}};
995	Record11.addValueData(ValueKind: IPVK_IndirectCallTarget, Site: 3, VData: VD3, N: 2, SymTab: nullptr);
996
997	InstrProfValueData VD4[] = {
998	{.Value: uint64_t(callee1), .Count: 1}, {.Value: uint64_t(callee2), .Count: 2}, {.Value: uint64_t(callee3), .Count: 3}};
999	Record11.addValueData(ValueKind: IPVK_IndirectCallTarget, Site: 4, VData: VD4, N: 3, SymTab: nullptr);
1000	}
1001	// 3 value sites for vtables.
1002	{
1003	Record11.reserveSites(ValueKind: IPVK_VTableTarget, NumValueSites: 3);
1004	InstrProfValueData VD0[] = {{.Value: getCalleeAddress(vtableAddr: vtable1), .Count: 1},
1005	{.Value: getCalleeAddress(vtableAddr: vtable2), .Count: 2},
1006	{.Value: getCalleeAddress(vtableAddr: vtable3), .Count: 3},
1007	{.Value: getCalleeAddress(vtableAddr: vtable4), .Count: 4}};
1008	Record11.addValueData(ValueKind: IPVK_VTableTarget, Site: 0, VData: VD0, N: 4, SymTab: nullptr);
1009
1010	InstrProfValueData VD2[] = {{.Value: getCalleeAddress(vtableAddr: vtable1), .Count: 1},
1011	{.Value: getCalleeAddress(vtableAddr: vtable2), .Count: 2},
1012	{.Value: getCalleeAddress(vtableAddr: vtable3), .Count: 3}};
1013	Record11.addValueData(ValueKind: IPVK_VTableTarget, Site: 1, VData: VD2, N: 3, SymTab: nullptr);
1014
1015	InstrProfValueData VD4[] = {{.Value: getCalleeAddress(vtableAddr: vtable1), .Count: 1},
1016	{.Value: getCalleeAddress(vtableAddr: vtable2), .Count: 2},
1017	{.Value: getCalleeAddress(vtableAddr: vtable3), .Count: 3}};
1018	Record11.addValueData(ValueKind: IPVK_VTableTarget, Site: 3, VData: VD4, N: 3, SymTab: nullptr);
1019	}
1020
1021	// A different record for the same caller.
1022	Record12.reserveSites(ValueKind: IPVK_IndirectCallTarget, NumValueSites: 5);
1023	InstrProfValueData VD02[] = {{.Value: uint64_t(callee2), .Count: 5}, {.Value: uint64_t(callee3), .Count: 3}};
1024	Record12.addValueData(ValueKind: IPVK_IndirectCallTarget, Site: 0, VData: VD02, N: 2, SymTab: nullptr);
1025
1026	// No value profile data at the second site.
1027	Record12.addValueData(ValueKind: IPVK_IndirectCallTarget, Site: 1, VData: nullptr, N: 0, SymTab: nullptr);
1028
1029	InstrProfValueData VD22[] = {
1030	{.Value: uint64_t(callee2), .Count: 1}, {.Value: uint64_t(callee3), .Count: 3}, {.Value: uint64_t(callee4), .Count: 4}};
1031	Record12.addValueData(ValueKind: IPVK_IndirectCallTarget, Site: 2, VData: VD22, N: 3, SymTab: nullptr);
1032
1033	Record12.addValueData(ValueKind: IPVK_IndirectCallTarget, Site: 3, VData: nullptr, N: 0, SymTab: nullptr);
1034
1035	InstrProfValueData VD42[] = {
1036	{.Value: uint64_t(callee1), .Count: 1}, {.Value: uint64_t(callee2), .Count: 2}, {.Value: uint64_t(callee3), .Count: 3}};
1037	Record12.addValueData(ValueKind: IPVK_IndirectCallTarget, Site: 4, VData: VD42, N: 3, SymTab: nullptr);
1038
1039	// 3 value sites for vtables.
1040	{
1041	Record12.reserveSites(ValueKind: IPVK_VTableTarget, NumValueSites: 3);
1042	InstrProfValueData VD0[] = {{.Value: getCalleeAddress(vtableAddr: vtable2), .Count: 5},
1043	{.Value: getCalleeAddress(vtableAddr: vtable3), .Count: 3}};
1044	Record12.addValueData(ValueKind: IPVK_VTableTarget, Site: 0, VData: VD0, N: 2, SymTab: nullptr);
1045
1046	InstrProfValueData VD2[] = {{.Value: getCalleeAddress(vtableAddr: vtable2), .Count: 1},
1047	{.Value: getCalleeAddress(vtableAddr: vtable3), .Count: 3},
1048	{.Value: getCalleeAddress(vtableAddr: vtable4), .Count: 4}};
1049	Record12.addValueData(ValueKind: IPVK_VTableTarget, Site: 1, VData: VD2, N: 3, SymTab: nullptr);
1050
1051	InstrProfValueData VD4[] = {{.Value: getCalleeAddress(vtableAddr: vtable1), .Count: 1},
1052	{.Value: getCalleeAddress(vtableAddr: vtable2), .Count: 2},
1053	{.Value: getCalleeAddress(vtableAddr: vtable3), .Count: 3}};
1054	Record12.addValueData(ValueKind: IPVK_VTableTarget, Site: 3, VData: VD4, N: 3, SymTab: nullptr);
1055	}
1056
1057	Writer.addRecord(I: std::move(Record11), Warn: Err);
1058	// Merge profile data.
1059	Writer.addRecord(I: std::move(Record12), Warn: Err);
1060
1061	Writer.addRecord(I: {callee1, 0x1235, {3, 4}}, Warn: Err);
1062	Writer.addRecord(I: {callee2, 0x1235, {3, 4}}, Warn: Err);
1063	Writer.addRecord(I: {callee3, 0x1235, {3, 4}}, Warn: Err);
1064	Writer.addRecord(I: {callee3, 0x1235, {3, 4}}, Warn: Err);
1065	Writer.addRecord(I: {callee4, 0x1235, {3, 5}}, Warn: Err);
1066	Writer.addRecord(I: {callee7, 0x1235, {3, 5}}, Warn: Err);
1067	Writer.addRecord(I: {callee8, 0x1235, {3, 5}}, Warn: Err);
1068	auto Profile = Writer.writeBuffer();
1069	readProfile(Profile: std::move(Profile));
1070
1071	// Test the number of instrumented value sites and the number of profiled
1072	// values for each site.
1073	Expected<InstrProfRecord> R = Reader->getInstrProfRecord(FuncName: "caller", FuncHash: 0x1234);
1074	EXPECT_THAT_ERROR(R.takeError(), Succeeded());
1075	// For indirect calls.
1076	ASSERT_EQ(5U, R->getNumValueSites(IPVK_IndirectCallTarget));
1077	ASSERT_EQ(4U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 0));
1078	ASSERT_EQ(0U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 1));
1079	ASSERT_EQ(4U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 2));
1080	ASSERT_EQ(2U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 3));
1081	ASSERT_EQ(3U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 4));
1082	// For vtables.
1083	ASSERT_EQ(R->getNumValueSites(IPVK_VTableTarget), 3U);
1084	ASSERT_EQ(R->getNumValueDataForSite(IPVK_VTableTarget, 0), 4U);
1085	ASSERT_EQ(R->getNumValueDataForSite(IPVK_VTableTarget, 1), 4U);
1086	ASSERT_EQ(R->getNumValueDataForSite(IPVK_VTableTarget, 2), 3U);
1087
1088	// Test the merged values for indirect calls.
1089	{
1090	auto VD = R->getValueForSite(ValueKind: IPVK_IndirectCallTarget, Site: 0);
1091	EXPECT_STREQ((const char *)VD[0].Value, "callee2");
1092	EXPECT_EQ(VD[0].Count, 7U);
1093	EXPECT_STREQ((const char *)VD[1].Value, "callee3");
1094	EXPECT_EQ(VD[1].Count, 6U);
1095	EXPECT_STREQ((const char *)VD[2].Value, "callee4");
1096	EXPECT_EQ(VD[2].Count, 4U);
1097	EXPECT_STREQ((const char *)VD[3].Value, "callee1");
1098	EXPECT_EQ(VD[3].Count, 1U);
1099
1100	auto VD_2(R->getValueForSite(ValueKind: IPVK_IndirectCallTarget, Site: 2));
1101	EXPECT_STREQ((const char *)VD_2[0].Value, "callee3");
1102	EXPECT_EQ(VD_2[0].Count, 6U);
1103	EXPECT_STREQ((const char *)VD_2[1].Value, "callee4");
1104	EXPECT_EQ(VD_2[1].Count, 4U);
1105	EXPECT_STREQ((const char *)VD_2[2].Value, "callee2");
1106	EXPECT_EQ(VD_2[2].Count, 3U);
1107	EXPECT_STREQ((const char *)VD_2[3].Value, "callee1");
1108	EXPECT_EQ(VD_2[3].Count, 1U);
1109
1110	auto VD_3(R->getValueForSite(ValueKind: IPVK_IndirectCallTarget, Site: 3));
1111	EXPECT_STREQ((const char *)VD_3[0].Value, "callee8");
1112	EXPECT_EQ(VD_3[0].Count, 2U);
1113	EXPECT_STREQ((const char *)VD_3[1].Value, "callee7");
1114	EXPECT_EQ(VD_3[1].Count, 1U);
1115
1116	auto VD_4(R->getValueForSite(ValueKind: IPVK_IndirectCallTarget, Site: 4));
1117	EXPECT_STREQ((const char *)VD_4[0].Value, "callee3");
1118	EXPECT_EQ(VD_4[0].Count, 6U);
1119	EXPECT_STREQ((const char *)VD_4[1].Value, "callee2");
1120	EXPECT_EQ(VD_4[1].Count, 4U);
1121	EXPECT_STREQ((const char *)VD_4[2].Value, "callee1");
1122	EXPECT_EQ(VD_4[2].Count, 2U);
1123	}
1124
1125	// Test the merged values for vtables
1126	{
1127	auto VD0 = R->getValueForSite(ValueKind: IPVK_VTableTarget, Site: 0);
1128	EXPECT_EQ(VD0[0].Value, getCalleeAddress(vtable2));
1129	EXPECT_EQ(VD0[0].Count, 7U);
1130	EXPECT_EQ(VD0[1].Value, getCalleeAddress(vtable3));
1131	EXPECT_EQ(VD0[1].Count, 6U);
1132	EXPECT_EQ(VD0[2].Value, getCalleeAddress(vtable4));
1133	EXPECT_EQ(VD0[2].Count, 4U);
1134	EXPECT_EQ(VD0[3].Value, getCalleeAddress(vtable1));
1135	EXPECT_EQ(VD0[3].Count, 1U);
1136
1137	auto VD1 = R->getValueForSite(ValueKind: IPVK_VTableTarget, Site: 1);
1138	EXPECT_EQ(VD1[0].Value, getCalleeAddress(vtable3));
1139	EXPECT_EQ(VD1[0].Count, 6U);
1140	EXPECT_EQ(VD1[1].Value, getCalleeAddress(vtable4));
1141	EXPECT_EQ(VD1[1].Count, 4U);
1142	EXPECT_EQ(VD1[2].Value, getCalleeAddress(vtable2));
1143	EXPECT_EQ(VD1[2].Count, 3U);
1144	EXPECT_EQ(VD1[3].Value, getCalleeAddress(vtable1));
1145	EXPECT_EQ(VD1[3].Count, 1U);
1146
1147	auto VD2 = R->getValueForSite(ValueKind: IPVK_VTableTarget, Site: 2);
1148	EXPECT_EQ(VD2[0].Value, getCalleeAddress(vtable3));
1149	EXPECT_EQ(VD2[0].Count, 6U);
1150	EXPECT_EQ(VD2[1].Value, getCalleeAddress(vtable2));
1151	EXPECT_EQ(VD2[1].Count, 4U);
1152	EXPECT_EQ(VD2[2].Value, getCalleeAddress(vtable1));
1153	EXPECT_EQ(VD2[2].Count, 2U);
1154	}
1155	}
1156
1157	struct ValueProfileMergeEdgeCaseTest
1158	: public InstrProfTest,
1159	public ::testing::WithParamInterface<std::tuple<bool, uint32_t>> {
1160	void SetUp() override { Writer.setOutputSparse(std::get<0>(t: GetParam())); }
1161
1162	uint32_t getValueProfileKind() const { return std::get<1>(t: GetParam()); }
1163	};
1164
1165	TEST_P(ValueProfileMergeEdgeCaseTest, value_profile_data_merge_saturation) {
1166	const uint32_t ValueKind = getValueProfileKind();
1167	static const char bar[] = "bar";
1168	const uint64_t ProfiledValue = 0x5678;
1169
1170	const uint64_t MaxValCount = std::numeric_limits<uint64_t>::max();
1171	const uint64_t MaxEdgeCount = getInstrMaxCountValue();
1172
1173	instrprof_error Result;
1174	auto Err = [&](Error E) {
1175	Result = std::get<0>(in: InstrProfError::take(E: std::move(E)));
1176	};
1177	Result = instrprof_error::success;
1178	Writer.addRecord(I: {"foo", 0x1234, {1}}, Warn: Err);
1179	ASSERT_EQ(Result, instrprof_error::success);
1180
1181	// Verify counter overflow.
1182	Result = instrprof_error::success;
1183	Writer.addRecord(I: {"foo", 0x1234, {MaxEdgeCount}}, Warn: Err);
1184	ASSERT_EQ(Result, instrprof_error::counter_overflow);
1185
1186	Result = instrprof_error::success;
1187	Writer.addRecord(I: {bar, 0x9012, {8}}, Warn: Err);
1188	ASSERT_EQ(Result, instrprof_error::success);
1189
1190	NamedInstrProfRecord Record4("baz", 0x5678, {3, 4});
1191	Record4.reserveSites(ValueKind, NumValueSites: 1);
1192	InstrProfValueData VD4[] = {{.Value: ProfiledValue, .Count: 1}};
1193	Record4.addValueData(ValueKind, Site: 0, VData: VD4, N: 1, SymTab: nullptr);
1194	Result = instrprof_error::success;
1195	Writer.addRecord(I: std::move(Record4), Warn: Err);
1196	ASSERT_EQ(Result, instrprof_error::success);
1197
1198	// Verify value data counter overflow.
1199	NamedInstrProfRecord Record5("baz", 0x5678, {5, 6});
1200	Record5.reserveSites(ValueKind, NumValueSites: 1);
1201	InstrProfValueData VD5[] = {{.Value: ProfiledValue, .Count: MaxValCount}};
1202	Record5.addValueData(ValueKind, Site: 0, VData: VD5, N: 1, SymTab: nullptr);
1203	Result = instrprof_error::success;
1204	Writer.addRecord(I: std::move(Record5), Warn: Err);
1205	ASSERT_EQ(Result, instrprof_error::counter_overflow);
1206
1207	auto Profile = Writer.writeBuffer();
1208	readProfile(Profile: std::move(Profile));
1209
1210	// Verify saturation of counts.
1211	Expected<InstrProfRecord> ReadRecord1 =
1212	Reader->getInstrProfRecord(FuncName: "foo", FuncHash: 0x1234);
1213	ASSERT_THAT_ERROR(ReadRecord1.takeError(), Succeeded());
1214	EXPECT_EQ(MaxEdgeCount, ReadRecord1->Counts[0]);
1215
1216	Expected<InstrProfRecord> ReadRecord2 =
1217	Reader->getInstrProfRecord(FuncName: "baz", FuncHash: 0x5678);
1218	ASSERT_TRUE(bool(ReadRecord2));
1219	ASSERT_EQ(1U, ReadRecord2->getNumValueSites(ValueKind));
1220	std::unique_ptr<InstrProfValueData[]> VD =
1221	ReadRecord2->getValueForSite(ValueKind, Site: 0);
1222	EXPECT_EQ(ProfiledValue, VD[0].Value);
1223	EXPECT_EQ(MaxValCount, VD[0].Count);
1224	}
1225
1226	// This test tests that when there are too many values for a given site, the
1227	// merged results are properly truncated.
1228	TEST_P(ValueProfileMergeEdgeCaseTest, value_profile_data_merge_site_trunc) {
1229	const uint32_t ValueKind = getValueProfileKind();
1230	static const char caller[] = "caller";
1231
1232	NamedInstrProfRecord Record11(caller, 0x1234, {1, 2});
1233	NamedInstrProfRecord Record12(caller, 0x1234, {1, 2});
1234
1235	// 2 value sites.
1236	Record11.reserveSites(ValueKind, NumValueSites: 2);
1237	InstrProfValueData VD0[255];
1238	for (int I = 0; I < 255; I++) {
1239	VD0[I].Value = 2 * I;
1240	VD0[I].Count = 2 * I + 1000;
1241	}
1242
1243	Record11.addValueData(ValueKind, Site: 0, VData: VD0, N: 255, SymTab: nullptr);
1244	Record11.addValueData(ValueKind, Site: 1, VData: nullptr, N: 0, SymTab: nullptr);
1245
1246	Record12.reserveSites(ValueKind, NumValueSites: 2);
1247	InstrProfValueData VD1[255];
1248	for (int I = 0; I < 255; I++) {
1249	VD1[I].Value = 2 * I + 1;
1250	VD1[I].Count = 2 * I + 1001;
1251	}
1252
1253	Record12.addValueData(ValueKind, Site: 0, VData: VD1, N: 255, SymTab: nullptr);
1254	Record12.addValueData(ValueKind, Site: 1, VData: nullptr, N: 0, SymTab: nullptr);
1255
1256	Writer.addRecord(I: std::move(Record11), Warn: Err);
1257	// Merge profile data.
1258	Writer.addRecord(I: std::move(Record12), Warn: Err);
1259
1260	auto Profile = Writer.writeBuffer();
1261	readProfile(Profile: std::move(Profile));
1262
1263	Expected<InstrProfRecord> R = Reader->getInstrProfRecord(FuncName: "caller", FuncHash: 0x1234);
1264	ASSERT_THAT_ERROR(R.takeError(), Succeeded());
1265	std::unique_ptr<InstrProfValueData[]> VD(R->getValueForSite(ValueKind, Site: 0));
1266	ASSERT_EQ(2U, R->getNumValueSites(ValueKind));
1267	EXPECT_EQ(255U, R->getNumValueDataForSite(ValueKind, 0));
1268	for (unsigned I = 0; I < 255; I++) {
1269	EXPECT_EQ(VD[I].Value, 509 - I);
1270	EXPECT_EQ(VD[I].Count, 1509 - I);
1271	}
1272	}
1273
1274	INSTANTIATE_TEST_SUITE_P(
1275	EdgeCaseTest, ValueProfileMergeEdgeCaseTest,
1276	::testing::Combine(::testing::Bool(), /* Sparse */
1277	::testing::Values(IPVK_IndirectCallTarget,
1278	IPVK_MemOPSize,
1279	IPVK_VTableTarget) /* ValueKind */
1280	));
1281
1282	static void addValueProfData(InstrProfRecord &Record) {
1283	// Add test data for indirect calls.
1284	{
1285	Record.reserveSites(ValueKind: IPVK_IndirectCallTarget, NumValueSites: 6);
1286	InstrProfValueData VD0[] = {{.Value: uint64_t(callee1), .Count: 400},
1287	{.Value: uint64_t(callee2), .Count: 1000},
1288	{.Value: uint64_t(callee3), .Count: 500},
1289	{.Value: uint64_t(callee4), .Count: 300},
1290	{.Value: uint64_t(callee5), .Count: 100}};
1291	Record.addValueData(ValueKind: IPVK_IndirectCallTarget, Site: 0, VData: VD0, N: 5, SymTab: nullptr);
1292	InstrProfValueData VD1[] = {{.Value: uint64_t(callee5), .Count: 800},
1293	{.Value: uint64_t(callee3), .Count: 1000},
1294	{.Value: uint64_t(callee2), .Count: 2500},
1295	{.Value: uint64_t(callee1), .Count: 1300}};
1296	Record.addValueData(ValueKind: IPVK_IndirectCallTarget, Site: 1, VData: VD1, N: 4, SymTab: nullptr);
1297	InstrProfValueData VD2[] = {{.Value: uint64_t(callee6), .Count: 800},
1298	{.Value: uint64_t(callee3), .Count: 1000},
1299	{.Value: uint64_t(callee4), .Count: 5500}};
1300	Record.addValueData(ValueKind: IPVK_IndirectCallTarget, Site: 2, VData: VD2, N: 3, SymTab: nullptr);
1301	InstrProfValueData VD3[] = {{.Value: uint64_t(callee2), .Count: 1800},
1302	{.Value: uint64_t(callee3), .Count: 2000}};
1303	Record.addValueData(ValueKind: IPVK_IndirectCallTarget, Site: 3, VData: VD3, N: 2, SymTab: nullptr);
1304	Record.addValueData(ValueKind: IPVK_IndirectCallTarget, Site: 4, VData: nullptr, N: 0, SymTab: nullptr);
1305	InstrProfValueData VD5[] = {{.Value: uint64_t(callee7), .Count: 1234},
1306	{.Value: uint64_t(callee8), .Count: 5678}};
1307	Record.addValueData(ValueKind: IPVK_IndirectCallTarget, Site: 5, VData: VD5, N: 2, SymTab: nullptr);
1308	}
1309
1310	// Add test data for vtables
1311	{
1312	Record.reserveSites(ValueKind: IPVK_VTableTarget, NumValueSites: 4);
1313	InstrProfValueData VD0[] = {
1314	{.Value: getCalleeAddress(vtableAddr: vtable1), .Count: 400}, {.Value: getCalleeAddress(vtableAddr: vtable2), .Count: 1000},
1315	{.Value: getCalleeAddress(vtableAddr: vtable3), .Count: 500}, {.Value: getCalleeAddress(vtableAddr: vtable4), .Count: 300},
1316	{.Value: getCalleeAddress(vtableAddr: vtable5), .Count: 100},
1317	};
1318	InstrProfValueData VD1[] = {{.Value: getCalleeAddress(vtableAddr: vtable5), .Count: 800},
1319	{.Value: getCalleeAddress(vtableAddr: vtable3), .Count: 1000},
1320	{.Value: getCalleeAddress(vtableAddr: vtable2), .Count: 2500},
1321	{.Value: getCalleeAddress(vtableAddr: vtable1), .Count: 1300}};
1322	InstrProfValueData VD2[] = {
1323	{.Value: getCalleeAddress(vtableAddr: vtable6), .Count: 800},
1324	{.Value: getCalleeAddress(vtableAddr: vtable3), .Count: 1000},
1325	{.Value: getCalleeAddress(vtableAddr: vtable4), .Count: 5500},
1326	};
1327	InstrProfValueData VD3[] = {{.Value: getCalleeAddress(vtableAddr: vtable2), .Count: 1800},
1328	{.Value: getCalleeAddress(vtableAddr: vtable3), .Count: 2000}};
1329	Record.addValueData(ValueKind: IPVK_VTableTarget, Site: 0, VData: VD0, N: 5, SymTab: nullptr);
1330	Record.addValueData(ValueKind: IPVK_VTableTarget, Site: 1, VData: VD1, N: 4, SymTab: nullptr);
1331	Record.addValueData(ValueKind: IPVK_VTableTarget, Site: 2, VData: VD2, N: 3, SymTab: nullptr);
1332	Record.addValueData(ValueKind: IPVK_VTableTarget, Site: 3, VData: VD3, N: 2, SymTab: nullptr);
1333	}
1334	}
1335
1336	TEST(ValueProfileReadWriteTest, value_prof_data_read_write) {
1337	InstrProfRecord SrcRecord({1ULL << 31, 2});
1338	addValueProfData(Record&: SrcRecord);
1339	std::unique_ptr<ValueProfData> VPData =
1340	ValueProfData::serializeFrom(Record: SrcRecord);
1341
1342	InstrProfRecord Record({1ULL << 31, 2});
1343	VPData->deserializeTo(Record, SymTab: nullptr);
1344
1345	// Now read data from Record and sanity check the data
1346	ASSERT_EQ(6U, Record.getNumValueSites(IPVK_IndirectCallTarget));
1347	ASSERT_EQ(5U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 0));
1348	ASSERT_EQ(4U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 1));
1349	ASSERT_EQ(3U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 2));
1350	ASSERT_EQ(2U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 3));
1351	ASSERT_EQ(0U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 4));
1352	ASSERT_EQ(2U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 5));
1353
1354	auto Cmp = [](const InstrProfValueData &VD1, const InstrProfValueData &VD2) {
1355	return VD1.Count > VD2.Count;
1356	};
1357
1358	std::unique_ptr<InstrProfValueData[]> VD_0(
1359	Record.getValueForSite(ValueKind: IPVK_IndirectCallTarget, Site: 0));
1360	llvm::sort(Start: &VD_0[0], End: &VD_0[5], Comp: Cmp);
1361	EXPECT_STREQ((const char *)VD_0[0].Value, "callee2");
1362	EXPECT_EQ(1000U, VD_0[0].Count);
1363	EXPECT_STREQ((const char *)VD_0[1].Value, "callee3");
1364	EXPECT_EQ(500U, VD_0[1].Count);
1365	EXPECT_STREQ((const char *)VD_0[2].Value, "callee1");
1366	EXPECT_EQ(400U, VD_0[2].Count);
1367	EXPECT_STREQ((const char *)VD_0[3].Value, "callee4");
1368	EXPECT_EQ(300U, VD_0[3].Count);
1369	EXPECT_STREQ((const char *)VD_0[4].Value, "callee5");
1370	EXPECT_EQ(100U, VD_0[4].Count);
1371
1372	std::unique_ptr<InstrProfValueData[]> VD_1(
1373	Record.getValueForSite(ValueKind: IPVK_IndirectCallTarget, Site: 1));
1374	llvm::sort(Start: &VD_1[0], End: &VD_1[4], Comp: Cmp);
1375	EXPECT_STREQ((const char *)VD_1[0].Value, "callee2");
1376	EXPECT_EQ(VD_1[0].Count, 2500U);
1377	EXPECT_STREQ((const char *)VD_1[1].Value, "callee1");
1378	EXPECT_EQ(VD_1[1].Count, 1300U);
1379	EXPECT_STREQ((const char *)VD_1[2].Value, "callee3");
1380	EXPECT_EQ(VD_1[2].Count, 1000U);
1381	EXPECT_STREQ((const char *)VD_1[3].Value, "callee5");
1382	EXPECT_EQ(VD_1[3].Count, 800U);
1383
1384	std::unique_ptr<InstrProfValueData[]> VD_2(
1385	Record.getValueForSite(ValueKind: IPVK_IndirectCallTarget, Site: 2));
1386	llvm::sort(Start: &VD_2[0], End: &VD_2[3], Comp: Cmp);
1387	EXPECT_STREQ((const char *)VD_2[0].Value, "callee4");
1388	EXPECT_EQ(VD_2[0].Count, 5500U);
1389	EXPECT_STREQ((const char *)VD_2[1].Value, "callee3");
1390	EXPECT_EQ(VD_2[1].Count, 1000U);
1391	EXPECT_STREQ((const char *)VD_2[2].Value, "callee6");
1392	EXPECT_EQ(VD_2[2].Count, 800U);
1393
1394	std::unique_ptr<InstrProfValueData[]> VD_3(
1395	Record.getValueForSite(ValueKind: IPVK_IndirectCallTarget, Site: 3));
1396	llvm::sort(Start: &VD_3[0], End: &VD_3[2], Comp: Cmp);
1397	EXPECT_STREQ((const char *)VD_3[0].Value, "callee3");
1398	EXPECT_EQ(VD_3[0].Count, 2000U);
1399	EXPECT_STREQ((const char *)VD_3[1].Value, "callee2");
1400	EXPECT_EQ(VD_3[1].Count, 1800U);
1401
1402	ASSERT_EQ(Record.getNumValueSites(IPVK_VTableTarget), 4U);
1403	ASSERT_EQ(Record.getNumValueDataForSite(IPVK_VTableTarget, 0), 5U);
1404	ASSERT_EQ(Record.getNumValueDataForSite(IPVK_VTableTarget, 1), 4U);
1405	ASSERT_EQ(Record.getNumValueDataForSite(IPVK_VTableTarget, 2), 3U);
1406	ASSERT_EQ(Record.getNumValueDataForSite(IPVK_VTableTarget, 3), 2U);
1407
1408	auto VD0(Record.getValueForSite(ValueKind: IPVK_VTableTarget, Site: 0));
1409	llvm::sort(Start: &VD0[0], End: &VD0[5], Comp: Cmp);
1410	EXPECT_EQ(VD0[0].Value, getCalleeAddress(vtable2));
1411	EXPECT_EQ(VD0[0].Count, 1000U);
1412	EXPECT_EQ(VD0[1].Value, getCalleeAddress(vtable3));
1413	EXPECT_EQ(VD0[1].Count, 500U);
1414	EXPECT_EQ(VD0[2].Value, getCalleeAddress(vtable1));
1415	EXPECT_EQ(VD0[2].Count, 400U);
1416	EXPECT_EQ(VD0[3].Value, getCalleeAddress(vtable4));
1417	EXPECT_EQ(VD0[3].Count, 300U);
1418	EXPECT_EQ(VD0[4].Value, getCalleeAddress(vtable5));
1419	EXPECT_EQ(VD0[4].Count, 100U);
1420
1421	auto VD1(Record.getValueForSite(ValueKind: IPVK_VTableTarget, Site: 1));
1422	llvm::sort(Start: &VD1[0], End: &VD1[4], Comp: Cmp);
1423	EXPECT_EQ(VD1[0].Value, getCalleeAddress(vtable2));
1424	EXPECT_EQ(VD1[0].Count, 2500U);
1425	EXPECT_EQ(VD1[1].Value, getCalleeAddress(vtable1));
1426	EXPECT_EQ(VD1[1].Count, 1300U);
1427	EXPECT_EQ(VD1[2].Value, getCalleeAddress(vtable3));
1428	EXPECT_EQ(VD1[2].Count, 1000U);
1429	EXPECT_EQ(VD1[3].Value, getCalleeAddress(vtable5));
1430	EXPECT_EQ(VD1[3].Count, 800U);
1431
1432	auto VD2(Record.getValueForSite(ValueKind: IPVK_VTableTarget, Site: 2));
1433	llvm::sort(Start: &VD2[0], End: &VD2[3], Comp: Cmp);
1434	EXPECT_EQ(VD2[0].Value, getCalleeAddress(vtable4));
1435	EXPECT_EQ(VD2[0].Count, 5500U);
1436	EXPECT_EQ(VD2[1].Value, getCalleeAddress(vtable3));
1437	EXPECT_EQ(VD2[1].Count, 1000U);
1438	EXPECT_EQ(VD2[2].Value, getCalleeAddress(vtable6));
1439	EXPECT_EQ(VD2[2].Count, 800U);
1440
1441	auto VD3(Record.getValueForSite(ValueKind: IPVK_VTableTarget, Site: 3));
1442	llvm::sort(Start: &VD3[0], End: &VD3[2], Comp: Cmp);
1443	EXPECT_EQ(VD3[0].Value, getCalleeAddress(vtable3));
1444	EXPECT_EQ(VD3[0].Count, 2000U);
1445	EXPECT_EQ(VD3[1].Value, getCalleeAddress(vtable2));
1446	EXPECT_EQ(VD3[1].Count, 1800U);
1447	}
1448
1449	TEST(ValueProfileReadWriteTest, symtab_mapping) {
1450	NamedInstrProfRecord SrcRecord("caller", 0x1234, {1ULL << 31, 2});
1451	addValueProfData(Record&: SrcRecord);
1452	std::unique_ptr<ValueProfData> VPData =
1453	ValueProfData::serializeFrom(Record: SrcRecord);
1454
1455	NamedInstrProfRecord Record("caller", 0x1234, {1ULL << 31, 2});
1456	InstrProfSymtab Symtab;
1457	Symtab.mapAddress(Addr: uint64_t(callee1), MD5Val: 0x1000ULL);
1458	Symtab.mapAddress(Addr: uint64_t(callee2), MD5Val: 0x2000ULL);
1459	Symtab.mapAddress(Addr: uint64_t(callee3), MD5Val: 0x3000ULL);
1460	Symtab.mapAddress(Addr: uint64_t(callee4), MD5Val: 0x4000ULL);
1461	// Missing mapping for callee5
1462
1463	auto getVTableStartAddr = [](const uint64_t *vtable) -> uint64_t {
1464	return uint64_t(vtable);
1465	};
1466	auto getVTableEndAddr = [](const uint64_t *vtable) -> uint64_t {
1467	return uint64_t(vtable) + 16;
1468	};
1469	auto getVTableMidAddr = [](const uint64_t *vtable) -> uint64_t {
1470	return uint64_t(vtable) + 8;
1471	};
1472	// vtable1, vtable2, vtable3, vtable4 get mapped; vtable5, vtable6 are not
1473	// mapped.
1474	Symtab.mapVTableAddress(StartAddr: getVTableStartAddr(vtable1),
1475	EndAddr: getVTableEndAddr(vtable1), MD5Val: MD5Hash(Str: "vtable1"));
1476	Symtab.mapVTableAddress(StartAddr: getVTableStartAddr(vtable2),
1477	EndAddr: getVTableEndAddr(vtable2), MD5Val: MD5Hash(Str: "vtable2"));
1478	Symtab.mapVTableAddress(StartAddr: getVTableStartAddr(vtable3),
1479	EndAddr: getVTableEndAddr(vtable3), MD5Val: MD5Hash(Str: "vtable3"));
1480	Symtab.mapVTableAddress(StartAddr: getVTableStartAddr(vtable4),
1481	EndAddr: getVTableEndAddr(vtable4), MD5Val: MD5Hash(Str: "vtable4"));
1482
1483	VPData->deserializeTo(Record, SymTab: &Symtab);
1484
1485	// Now read data from Record and sanity check the data
1486	ASSERT_EQ(Record.getNumValueSites(IPVK_IndirectCallTarget), 6U);
1487	ASSERT_EQ(Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 0), 5U);
1488
1489	// Look up the value correpsonding to the middle of a vtable in symtab and
1490	// test that it's the hash of the name.
1491	EXPECT_EQ(Symtab.getVTableHashFromAddress(getVTableMidAddr(vtable1)),
1492	MD5Hash("vtable1"));
1493	EXPECT_EQ(Symtab.getVTableHashFromAddress(getVTableMidAddr(vtable2)),
1494	MD5Hash("vtable2"));
1495	EXPECT_EQ(Symtab.getVTableHashFromAddress(getVTableMidAddr(vtable3)),
1496	MD5Hash("vtable3"));
1497	EXPECT_EQ(Symtab.getVTableHashFromAddress(getVTableMidAddr(vtable4)),
1498	MD5Hash("vtable4"));
1499
1500	auto Cmp = [](const InstrProfValueData &VD1, const InstrProfValueData &VD2) {
1501	return VD1.Count > VD2.Count;
1502	};
1503	auto VD_0(Record.getValueForSite(ValueKind: IPVK_IndirectCallTarget, Site: 0));
1504	llvm::sort(Start: &VD_0[0], End: &VD_0[5], Comp: Cmp);
1505	ASSERT_EQ(VD_0[0].Value, 0x2000ULL);
1506	ASSERT_EQ(VD_0[0].Count, 1000U);
1507	ASSERT_EQ(VD_0[1].Value, 0x3000ULL);
1508	ASSERT_EQ(VD_0[1].Count, 500U);
1509	ASSERT_EQ(VD_0[2].Value, 0x1000ULL);
1510	ASSERT_EQ(VD_0[2].Count, 400U);
1511
1512	// callee5 does not have a mapped value -- default to 0.
1513	ASSERT_EQ(VD_0[4].Value, 0ULL);
1514
1515	// Sanity check the vtable value data
1516	ASSERT_EQ(Record.getNumValueSites(IPVK_VTableTarget), 4U);
1517
1518	{
1519	// The first vtable site.
1520	auto VD(Record.getValueForSite(ValueKind: IPVK_VTableTarget, Site: 0));
1521	ASSERT_EQ(Record.getNumValueDataForSite(IPVK_VTableTarget, 0), 5U);
1522	llvm::sort(Start: &VD[0], End: &VD[5], Comp: Cmp);
1523	EXPECT_EQ(VD[0].Count, 1000U);
1524	EXPECT_EQ(VD[0].Value, MD5Hash("vtable2"));
1525	EXPECT_EQ(VD[1].Count, 500U);
1526	EXPECT_EQ(VD[1].Value, MD5Hash("vtable3"));
1527	EXPECT_EQ(VD[2].Value, MD5Hash("vtable1"));
1528	EXPECT_EQ(VD[2].Count, 400U);
1529	EXPECT_EQ(VD[3].Value, MD5Hash("vtable4"));
1530	EXPECT_EQ(VD[3].Count, 300U);
1531
1532	// vtable5 isn't mapped -- default to 0.
1533	EXPECT_EQ(VD[4].Value, 0U);
1534	EXPECT_EQ(VD[4].Count, 100U);
1535	}
1536
1537	{
1538	// The second vtable site.
1539	auto VD(Record.getValueForSite(ValueKind: IPVK_VTableTarget, Site: 1));
1540	ASSERT_EQ(Record.getNumValueDataForSite(IPVK_VTableTarget, 1), 4U);
1541	llvm::sort(Start: &VD[0], End: &VD[4], Comp: Cmp);
1542	EXPECT_EQ(VD[0].Value, MD5Hash("vtable2"));
1543	EXPECT_EQ(VD[0].Count, 2500U);
1544	EXPECT_EQ(VD[1].Value, MD5Hash("vtable1"));
1545	EXPECT_EQ(VD[1].Count, 1300U);
1546
1547	EXPECT_EQ(VD[2].Value, MD5Hash("vtable3"));
1548	EXPECT_EQ(VD[2].Count, 1000U);
1549	// vtable5 isn't mapped -- default to 0.
1550	EXPECT_EQ(VD[3].Value, 0U);
1551	EXPECT_EQ(VD[3].Count, 800U);
1552	}
1553
1554	{
1555	// The third vtable site.
1556	auto VD(Record.getValueForSite(ValueKind: IPVK_VTableTarget, Site: 2));
1557	ASSERT_EQ(Record.getNumValueDataForSite(IPVK_VTableTarget, 2), 3U);
1558	llvm::sort(Start: &VD[0], End: &VD[3], Comp: Cmp);
1559	EXPECT_EQ(VD[0].Count, 5500U);
1560	EXPECT_EQ(VD[0].Value, MD5Hash("vtable4"));
1561	EXPECT_EQ(VD[1].Count, 1000U);
1562	EXPECT_EQ(VD[1].Value, MD5Hash("vtable3"));
1563	// vtable6 isn't mapped -- default to 0.
1564	EXPECT_EQ(VD[2].Value, 0U);
1565	EXPECT_EQ(VD[2].Count, 800U);
1566	}
1567
1568	{
1569	// The fourth vtable site.
1570	auto VD(Record.getValueForSite(ValueKind: IPVK_VTableTarget, Site: 3));
1571	ASSERT_EQ(Record.getNumValueDataForSite(IPVK_VTableTarget, 3), 2U);
1572	llvm::sort(Start: &VD[0], End: &VD[2], Comp: Cmp);
1573	EXPECT_EQ(VD[0].Count, 2000U);
1574	EXPECT_EQ(VD[0].Value, MD5Hash("vtable3"));
1575	EXPECT_EQ(VD[1].Count, 1800U);
1576	EXPECT_EQ(VD[1].Value, MD5Hash("vtable2"));
1577	}
1578	}
1579
1580	TEST_P(MaybeSparseInstrProfTest, get_max_function_count) {
1581	Writer.addRecord(I: {"foo", 0x1234, {1ULL << 31, 2}}, Warn: Err);
1582	Writer.addRecord(I: {"bar", 0, {1ULL << 63}}, Warn: Err);
1583	Writer.addRecord(I: {"baz", 0x5678, {0, 0, 0, 0}}, Warn: Err);
1584	auto Profile = Writer.writeBuffer();
1585	readProfile(Profile: std::move(Profile));
1586
1587	ASSERT_EQ(1ULL << 63, Reader->getMaximumFunctionCount(/* IsCS */ false));
1588	}
1589
1590	TEST_P(MaybeSparseInstrProfTest, get_weighted_function_counts) {
1591	Writer.addRecord(I: {"foo", 0x1234, {1, 2}}, Weight: 3, Warn: Err);
1592	Writer.addRecord(I: {"foo", 0x1235, {3, 4}}, Weight: 5, Warn: Err);
1593	auto Profile = Writer.writeBuffer();
1594	readProfile(Profile: std::move(Profile));
1595
1596	std::vector<uint64_t> Counts;
1597	EXPECT_THAT_ERROR(Reader->getFunctionCounts("foo", 0x1234, Counts),
1598	Succeeded());
1599	ASSERT_EQ(2U, Counts.size());
1600	ASSERT_EQ(3U, Counts[0]);
1601	ASSERT_EQ(6U, Counts[1]);
1602
1603	EXPECT_THAT_ERROR(Reader->getFunctionCounts("foo", 0x1235, Counts),
1604	Succeeded());
1605	ASSERT_EQ(2U, Counts.size());
1606	ASSERT_EQ(15U, Counts[0]);
1607	ASSERT_EQ(20U, Counts[1]);
1608	}
1609
1610	// Testing symtab creator interface used by indexed profile reader.
1611	TEST(SymtabTest, instr_prof_symtab_test) {
1612	std::vector<StringRef> FuncNames;
1613	FuncNames.push_back(x: "func1");
1614	FuncNames.push_back(x: "func2");
1615	FuncNames.push_back(x: "func3");
1616	FuncNames.push_back(x: "bar1");
1617	FuncNames.push_back(x: "bar2");
1618	FuncNames.push_back(x: "bar3");
1619	InstrProfSymtab Symtab;
1620	EXPECT_THAT_ERROR(Symtab.create(FuncNames), Succeeded());
1621	StringRef R = Symtab.getFuncOrVarName(MD5Hash: IndexedInstrProf::ComputeHash(K: "func1"));
1622	ASSERT_EQ(StringRef("func1"), R);
1623	R = Symtab.getFuncOrVarName(MD5Hash: IndexedInstrProf::ComputeHash(K: "func2"));
1624	ASSERT_EQ(StringRef("func2"), R);
1625	R = Symtab.getFuncOrVarName(MD5Hash: IndexedInstrProf::ComputeHash(K: "func3"));
1626	ASSERT_EQ(StringRef("func3"), R);
1627	R = Symtab.getFuncOrVarName(MD5Hash: IndexedInstrProf::ComputeHash(K: "bar1"));
1628	ASSERT_EQ(StringRef("bar1"), R);
1629	R = Symtab.getFuncOrVarName(MD5Hash: IndexedInstrProf::ComputeHash(K: "bar2"));
1630	ASSERT_EQ(StringRef("bar2"), R);
1631	R = Symtab.getFuncOrVarName(MD5Hash: IndexedInstrProf::ComputeHash(K: "bar3"));
1632	ASSERT_EQ(StringRef("bar3"), R);
1633
1634	// negative tests
1635	R = Symtab.getFuncOrVarName(MD5Hash: IndexedInstrProf::ComputeHash(K: "bar4"));
1636	ASSERT_EQ(StringRef(), R);
1637	R = Symtab.getFuncOrVarName(MD5Hash: IndexedInstrProf::ComputeHash(K: "foo4"));
1638	ASSERT_EQ(StringRef(), R);
1639
1640	// Now incrementally update the symtab
1641	EXPECT_THAT_ERROR(Symtab.addFuncName("blah_1"), Succeeded());
1642	EXPECT_THAT_ERROR(Symtab.addFuncName("blah_2"), Succeeded());
1643	EXPECT_THAT_ERROR(Symtab.addFuncName("blah_3"), Succeeded());
1644
1645	// Check again
1646	R = Symtab.getFuncOrVarName(MD5Hash: IndexedInstrProf::ComputeHash(K: "blah_1"));
1647	ASSERT_EQ(StringRef("blah_1"), R);
1648	R = Symtab.getFuncOrVarName(MD5Hash: IndexedInstrProf::ComputeHash(K: "blah_2"));
1649	ASSERT_EQ(StringRef("blah_2"), R);
1650	R = Symtab.getFuncOrVarName(MD5Hash: IndexedInstrProf::ComputeHash(K: "blah_3"));
1651	ASSERT_EQ(StringRef("blah_3"), R);
1652	R = Symtab.getFuncOrVarName(MD5Hash: IndexedInstrProf::ComputeHash(K: "func1"));
1653	ASSERT_EQ(StringRef("func1"), R);
1654	R = Symtab.getFuncOrVarName(MD5Hash: IndexedInstrProf::ComputeHash(K: "func2"));
1655	ASSERT_EQ(StringRef("func2"), R);
1656	R = Symtab.getFuncOrVarName(MD5Hash: IndexedInstrProf::ComputeHash(K: "func3"));
1657	ASSERT_EQ(StringRef("func3"), R);
1658	R = Symtab.getFuncOrVarName(MD5Hash: IndexedInstrProf::ComputeHash(K: "bar1"));
1659	ASSERT_EQ(StringRef("bar1"), R);
1660	R = Symtab.getFuncOrVarName(MD5Hash: IndexedInstrProf::ComputeHash(K: "bar2"));
1661	ASSERT_EQ(StringRef("bar2"), R);
1662	R = Symtab.getFuncOrVarName(MD5Hash: IndexedInstrProf::ComputeHash(K: "bar3"));
1663	ASSERT_EQ(StringRef("bar3"), R);
1664	}
1665
1666	// Test that we get an error when creating a bogus symtab.
1667	TEST(SymtabTest, instr_prof_bogus_symtab_empty_func_name) {
1668	InstrProfSymtab Symtab;
1669	EXPECT_TRUE(ErrorEquals(instrprof_error::malformed, Symtab.addFuncName("")));
1670	}
1671
1672	// Testing symtab creator interface used by value profile transformer.
1673	TEST(SymtabTest, instr_prof_symtab_module_test) {
1674	LLVMContext Ctx;
1675	std::unique_ptr<Module> M = std::make_unique<Module>(args: "MyModule.cpp", args&: Ctx);
1676	FunctionType *FTy = FunctionType::get(Result: Type::getVoidTy(C&: Ctx),
1677	/*isVarArg=*/false);
1678	Function::Create(Ty: FTy, Linkage: Function::ExternalLinkage, N: "Gfoo", M: M.get());
1679	Function::Create(Ty: FTy, Linkage: Function::ExternalLinkage, N: "Gblah", M: M.get());
1680	Function::Create(Ty: FTy, Linkage: Function::ExternalLinkage, N: "Gbar", M: M.get());
1681	Function::Create(Ty: FTy, Linkage: Function::InternalLinkage, N: "Ifoo", M: M.get());
1682	Function::Create(Ty: FTy, Linkage: Function::InternalLinkage, N: "Iblah", M: M.get());
1683	Function::Create(Ty: FTy, Linkage: Function::InternalLinkage, N: "Ibar", M: M.get());
1684	Function::Create(Ty: FTy, Linkage: Function::PrivateLinkage, N: "Pfoo", M: M.get());
1685	Function::Create(Ty: FTy, Linkage: Function::PrivateLinkage, N: "Pblah", M: M.get());
1686	Function::Create(Ty: FTy, Linkage: Function::PrivateLinkage, N: "Pbar", M: M.get());
1687	Function::Create(Ty: FTy, Linkage: Function::WeakODRLinkage, N: "Wfoo", M: M.get());
1688	Function::Create(Ty: FTy, Linkage: Function::WeakODRLinkage, N: "Wblah", M: M.get());
1689	Function::Create(Ty: FTy, Linkage: Function::WeakODRLinkage, N: "Wbar", M: M.get());
1690
1691	InstrProfSymtab ProfSymtab;
1692	EXPECT_THAT_ERROR(ProfSymtab.create(*M), Succeeded());
1693
1694	StringRef Funcs[] = {"Gfoo", "Gblah", "Gbar", "Ifoo", "Iblah", "Ibar",
1695	"Pfoo", "Pblah", "Pbar", "Wfoo", "Wblah", "Wbar"};
1696
1697	for (unsigned I = 0; I < std::size(Funcs); I++) {
1698	Function *F = M->getFunction(Name: Funcs[I]);
1699
1700	std::string IRPGOName = getIRPGOFuncName(F: *F);
1701	auto IRPGOFuncName =
1702	ProfSymtab.getFuncOrVarName(MD5Hash: IndexedInstrProf::ComputeHash(K: IRPGOName));
1703	EXPECT_EQ(IRPGOName, IRPGOFuncName);
1704	EXPECT_EQ(Funcs[I], getParsedIRPGOName(IRPGOFuncName).second);
1705	// Ensure we can still read this old record name.
1706	std::string PGOName = getPGOFuncName(F: *F);
1707	auto PGOFuncName =
1708	ProfSymtab.getFuncOrVarName(MD5Hash: IndexedInstrProf::ComputeHash(K: PGOName));
1709	EXPECT_EQ(PGOName, PGOFuncName);
1710	EXPECT_THAT(PGOFuncName.str(), EndsWith(Funcs[I].str()));
1711	}
1712	}
1713
1714	// Testing symtab serialization and creator/deserialization interface
1715	// used by coverage map reader, and raw profile reader.
1716	TEST(SymtabTest, instr_prof_symtab_compression_test) {
1717	std::vector<std::string> FuncNames1;
1718	std::vector<std::string> FuncNames2;
1719	for (int I = 0; I < 3; I++) {
1720	std::string str;
1721	raw_string_ostream OS(str);
1722	OS << "func_" << I;
1723	FuncNames1.push_back(x: OS.str());
1724	str.clear();
1725	OS << "f oooooooooooooo_" << I;
1726	FuncNames1.push_back(x: OS.str());
1727	str.clear();
1728	OS << "BAR_" << I;
1729	FuncNames2.push_back(x: OS.str());
1730	str.clear();
1731	OS << "BlahblahBlahblahBar_" << I;
1732	FuncNames2.push_back(x: OS.str());
1733	}
1734
1735	for (bool DoCompression : {false, true}) {
1736	// Compressing:
1737	std::string FuncNameStrings1;
1738	EXPECT_THAT_ERROR(collectGlobalObjectNameStrings(
1739	FuncNames1,
1740	(DoCompression && compression::zlib::isAvailable()),
1741	FuncNameStrings1),
1742	Succeeded());
1743
1744	// Compressing:
1745	std::string FuncNameStrings2;
1746	EXPECT_THAT_ERROR(collectGlobalObjectNameStrings(
1747	FuncNames2,
1748	(DoCompression && compression::zlib::isAvailable()),
1749	FuncNameStrings2),
1750	Succeeded());
1751
1752	for (int Padding = 0; Padding < 2; Padding++) {
1753	// Join with paddings :
1754	std::string FuncNameStrings = FuncNameStrings1;
1755	for (int P = 0; P < Padding; P++) {
1756	FuncNameStrings.push_back(c: '\0');
1757	}
1758	FuncNameStrings += FuncNameStrings2;
1759
1760	// Now decompress:
1761	InstrProfSymtab Symtab;
1762	EXPECT_THAT_ERROR(Symtab.create(StringRef(FuncNameStrings)), Succeeded());
1763
1764	// Now do the checks:
1765	// First sampling some data points:
1766	StringRef R =
1767	Symtab.getFuncOrVarName(MD5Hash: IndexedInstrProf::ComputeHash(K: FuncNames1[0]));
1768	ASSERT_EQ(StringRef("func_0"), R);
1769	R = Symtab.getFuncOrVarName(MD5Hash: IndexedInstrProf::ComputeHash(K: FuncNames1[1]));
1770	ASSERT_EQ(StringRef("f oooooooooooooo_0"), R);
1771	for (int I = 0; I < 3; I++) {
1772	std::string N[4];
1773	N[0] = FuncNames1[2 * I];
1774	N[1] = FuncNames1[2 * I + 1];
1775	N[2] = FuncNames2[2 * I];
1776	N[3] = FuncNames2[2 * I + 1];
1777	for (int J = 0; J < 4; J++) {
1778	StringRef R =
1779	Symtab.getFuncOrVarName(MD5Hash: IndexedInstrProf::ComputeHash(K: N[J]));
1780	ASSERT_EQ(StringRef(N[J]), R);
1781	}
1782	}
1783	}
1784	}
1785	}
1786
1787	TEST_P(MaybeSparseInstrProfTest, remapping_test) {
1788	Writer.addRecord(I: {"_Z3fooi", 0x1234, {1, 2, 3, 4}}, Warn: Err);
1789	Writer.addRecord(I: {"file;_Z3barf", 0x567, {5, 6, 7}}, Warn: Err);
1790	auto Profile = Writer.writeBuffer();
1791	readProfile(Profile: std::move(Profile), Remapping: llvm::MemoryBuffer::getMemBuffer(InputData: R"(
1792	type i l
1793	name 3bar 4quux
1794	)"));
1795
1796	std::vector<uint64_t> Counts;
1797	for (StringRef FooName : {"_Z3fooi", "_Z3fool"}) {
1798	EXPECT_THAT_ERROR(Reader->getFunctionCounts(FooName, 0x1234, Counts),
1799	Succeeded());
1800	ASSERT_EQ(4u, Counts.size());
1801	EXPECT_EQ(1u, Counts[0]);
1802	EXPECT_EQ(2u, Counts[1]);
1803	EXPECT_EQ(3u, Counts[2]);
1804	EXPECT_EQ(4u, Counts[3]);
1805	}
1806
1807	for (StringRef BarName : {"file;_Z3barf", "file;_Z4quuxf"}) {
1808	EXPECT_THAT_ERROR(Reader->getFunctionCounts(BarName, 0x567, Counts),
1809	Succeeded());
1810	ASSERT_EQ(3u, Counts.size());
1811	EXPECT_EQ(5u, Counts[0]);
1812	EXPECT_EQ(6u, Counts[1]);
1813	EXPECT_EQ(7u, Counts[2]);
1814	}
1815
1816	for (StringRef BadName : {"_Z3foof", "_Z4quuxi", "_Z3barl", "", "_ZZZ",
1817	"_Z3barf", "otherfile:_Z4quuxf"}) {
1818	EXPECT_THAT_ERROR(Reader->getFunctionCounts(BadName, 0x1234, Counts),
1819	Failed());
1820	EXPECT_THAT_ERROR(Reader->getFunctionCounts(BadName, 0x567, Counts),
1821	Failed());
1822	}
1823	}
1824
1825	TEST_F(SparseInstrProfTest, preserve_no_records) {
1826	Writer.addRecord(I: {"foo", 0x1234, {0}}, Warn: Err);
1827	Writer.addRecord(I: {"bar", 0x4321, {0, 0}}, Warn: Err);
1828	Writer.addRecord(I: {"baz", 0x4321, {0, 0, 0}}, Warn: Err);
1829
1830	auto Profile = Writer.writeBuffer();
1831	readProfile(Profile: std::move(Profile));
1832
1833	auto I = Reader->begin(), E = Reader->end();
1834	ASSERT_TRUE(I == E);
1835	}
1836
1837	INSTANTIATE_TEST_SUITE_P(MaybeSparse, MaybeSparseInstrProfTest,
1838	::testing::Bool());
1839
1840	#if defined(_LP64) && defined(EXPENSIVE_CHECKS)
1841	TEST(ProfileReaderTest, ReadsLargeFiles) {
1842	const size_t LargeSize = 1ULL << 32; // 4GB
1843
1844	auto RawProfile = WritableMemoryBuffer::getNewUninitMemBuffer(LargeSize);
1845	if (!RawProfile)
1846	GTEST_SKIP();
1847	auto RawProfileReaderOrErr = InstrProfReader::create(std::move(RawProfile));
1848	ASSERT_TRUE(
1849	std::get<0>(InstrProfError::take(RawProfileReaderOrErr.takeError())) ==
1850	instrprof_error::unrecognized_format);
1851
1852	auto IndexedProfile = WritableMemoryBuffer::getNewUninitMemBuffer(LargeSize);
1853	if (!IndexedProfile)
1854	GTEST_SKIP();
1855	auto IndexedReaderOrErr =
1856	IndexedInstrProfReader::create(std::move(IndexedProfile), nullptr);
1857	ASSERT_TRUE(
1858	std::get<0>(InstrProfError::take(IndexedReaderOrErr.takeError())) ==
1859	instrprof_error::bad_magic);
1860	}
1861	#endif
1862
1863	} // end anonymous namespace
1864