1//===-- Automemcpy Json Results Analyzer Test ----------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "automemcpy/ResultAnalyzer.h"
10#include "gmock/gmock.h"
11#include "gtest/gtest.h"
12
13using testing::DoubleNear;
14using testing::ElementsAre;
15using testing::Pair;
16using testing::SizeIs;
17
18namespace llvm {
19namespace automemcpy {
20namespace {
21
22TEST(AutomemcpyJsonResultsAnalyzer, getThroughputsOneSample) {
23 static constexpr FunctionId Foo1 = {"memcpy1", FunctionType::MEMCPY};
24 static constexpr DistributionId DistA = {{"A"}};
25 static constexpr SampleId Id = {Foo1, DistA};
26 static constexpr Sample kSamples[] = {
27 Sample{Id, SampleType::ITERATION, 4},
28 Sample{Id, SampleType::AGGREGATE, -1}, // Aggegates gets discarded
29 };
30
31 const std::vector<FunctionData> Data = getThroughputs(kSamples);
32 EXPECT_THAT(Data, SizeIs(1));
33 EXPECT_THAT(Data[0].Id, Foo1);
34 EXPECT_THAT(Data[0].PerDistributionData, SizeIs(1));
35 // A single value is provided.
36 const auto &DistributionData = Data[0].PerDistributionData.lookup(DistA.Name);
37 EXPECT_THAT(DistributionData.BytesPerSecondMedian, 4);
38 EXPECT_THAT(DistributionData.BytesPerSecondMean, 4);
39 EXPECT_THAT(DistributionData.BytesPerSecondVariance, 0);
40}
41
42TEST(AutomemcpyJsonResultsAnalyzer, getThroughputsManySamplesSameBucket) {
43 static constexpr FunctionId Foo1 = {"memcpy1", FunctionType::MEMCPY};
44 static constexpr DistributionId DistA = {{"A"}};
45 static constexpr SampleId Id = {Foo1, DistA};
46 static constexpr Sample kSamples[] = {Sample{Id, SampleType::ITERATION, 4},
47 Sample{Id, SampleType::ITERATION, 5},
48 Sample{Id, SampleType::ITERATION, 5}};
49
50 const std::vector<FunctionData> Data = getThroughputs(kSamples);
51 EXPECT_THAT(Data, SizeIs(1));
52 EXPECT_THAT(Data[0].Id, Foo1);
53 EXPECT_THAT(Data[0].PerDistributionData, SizeIs(1));
54 // When multiple values are provided we pick the median one (here median of 4,
55 // 5, 5).
56 const auto &DistributionData = Data[0].PerDistributionData.lookup(DistA.Name);
57 EXPECT_THAT(DistributionData.BytesPerSecondMedian, 5);
58 EXPECT_THAT(DistributionData.BytesPerSecondMean, DoubleNear(4.6, 0.1));
59 EXPECT_THAT(DistributionData.BytesPerSecondVariance, DoubleNear(0.33, 0.01));
60}
61
62TEST(AutomemcpyJsonResultsAnalyzer, getThroughputsServeralFunctionAndDist) {
63 static constexpr FunctionId Foo1 = {"memcpy1", FunctionType::MEMCPY};
64 static constexpr DistributionId DistA = {{"A"}};
65 static constexpr FunctionId Foo2 = {"memcpy2", FunctionType::MEMCPY};
66 static constexpr DistributionId DistB = {{"B"}};
67 static constexpr Sample kSamples[] = {
68 Sample{{Foo1, DistA}, SampleType::ITERATION, 1},
69 Sample{{Foo1, DistB}, SampleType::ITERATION, 2},
70 Sample{{Foo2, DistA}, SampleType::ITERATION, 3},
71 Sample{{Foo2, DistB}, SampleType::ITERATION, 4}};
72 // Data is aggregated per function.
73 const std::vector<FunctionData> Data = getThroughputs(kSamples);
74 EXPECT_THAT(Data, SizeIs(2)); // 2 functions Foo1 and Foo2.
75 // Each function has data for both distributions DistA and DistB.
76 EXPECT_THAT(Data[0].PerDistributionData, SizeIs(2));
77 EXPECT_THAT(Data[1].PerDistributionData, SizeIs(2));
78}
79
80TEST(AutomemcpyJsonResultsAnalyzer, getScore) {
81 static constexpr FunctionId Foo1 = {"memcpy1", FunctionType::MEMCPY};
82 static constexpr FunctionId Foo2 = {"memcpy2", FunctionType::MEMCPY};
83 static constexpr FunctionId Foo3 = {"memcpy3", FunctionType::MEMCPY};
84 static constexpr DistributionId Dist = {{"A"}};
85 static constexpr Sample kSamples[] = {
86 Sample{{Foo1, Dist}, SampleType::ITERATION, 1},
87 Sample{{Foo2, Dist}, SampleType::ITERATION, 2},
88 Sample{{Foo3, Dist}, SampleType::ITERATION, 3}};
89
90 // Data is aggregated per function.
91 std::vector<FunctionData> Data = getThroughputs(kSamples);
92
93 // Sort Data by function name so we can test them.
94 std::sort(
95 Data.begin(), Data.end(),
96 [](const FunctionData &A, const FunctionData &B) { return A.Id < B.Id; });
97
98 EXPECT_THAT(Data[0].Id, Foo1);
99 EXPECT_THAT(Data[0].PerDistributionData.lookup("A").BytesPerSecondMedian, 1);
100 EXPECT_THAT(Data[1].Id, Foo2);
101 EXPECT_THAT(Data[1].PerDistributionData.lookup("A").BytesPerSecondMedian, 2);
102 EXPECT_THAT(Data[2].Id, Foo3);
103 EXPECT_THAT(Data[2].PerDistributionData.lookup("A").BytesPerSecondMedian, 3);
104
105 // Normalizes throughput per distribution.
106 fillScores(Data);
107 EXPECT_THAT(Data[0].PerDistributionData.lookup("A").Score, 0);
108 EXPECT_THAT(Data[1].PerDistributionData.lookup("A").Score, 0.5);
109 EXPECT_THAT(Data[2].PerDistributionData.lookup("A").Score, 1);
110}
111
112TEST(AutomemcpyJsonResultsAnalyzer, castVotes) {
113 static constexpr double kAbsErr = 0.01;
114
115 static constexpr FunctionId Foo1 = {"memcpy1", FunctionType::MEMCPY};
116 static constexpr FunctionId Foo2 = {"memcpy2", FunctionType::MEMCPY};
117 static constexpr FunctionId Foo3 = {"memcpy3", FunctionType::MEMCPY};
118 static constexpr DistributionId DistA = {{"A"}};
119 static constexpr DistributionId DistB = {{"B"}};
120 static constexpr Sample kSamples[] = {
121 Sample{{Foo1, DistA}, SampleType::ITERATION, 0},
122 Sample{{Foo1, DistB}, SampleType::ITERATION, 30},
123 Sample{{Foo2, DistA}, SampleType::ITERATION, 1},
124 Sample{{Foo2, DistB}, SampleType::ITERATION, 100},
125 Sample{{Foo3, DistA}, SampleType::ITERATION, 7},
126 Sample{{Foo3, DistB}, SampleType::ITERATION, 100},
127 };
128
129 // DistA Thoughput ranges from 0 to 7.
130 // DistB Thoughput ranges from 30 to 100.
131
132 // Data is aggregated per function.
133 std::vector<FunctionData> Data = getThroughputs(kSamples);
134
135 // Sort Data by function name so we can test them.
136 std::sort(
137 Data.begin(), Data.end(),
138 [](const FunctionData &A, const FunctionData &B) { return A.Id < B.Id; });
139
140 // Normalizes throughput per distribution.
141 fillScores(Data);
142
143 // Cast votes
144 castVotes(Data);
145
146 EXPECT_THAT(Data[0].Id, Foo1);
147 EXPECT_THAT(Data[1].Id, Foo2);
148 EXPECT_THAT(Data[2].Id, Foo3);
149
150 const auto GetDistData = [&Data](size_t Index, StringRef Name) {
151 return Data[Index].PerDistributionData.lookup(Name);
152 };
153
154 // Distribution A
155 // Throughput is 0, 1 and 7, so normalized scores are 0, 1/7 and 1.
156 EXPECT_THAT(GetDistData(0, "A").Score, DoubleNear(0, kAbsErr));
157 EXPECT_THAT(GetDistData(1, "A").Score, DoubleNear(1. / 7, kAbsErr));
158 EXPECT_THAT(GetDistData(2, "A").Score, DoubleNear(1, kAbsErr));
159 // which are turned into grades BAD, MEDIOCRE and EXCELLENT.
160 EXPECT_THAT(GetDistData(0, "A").Grade, Grade::BAD);
161 EXPECT_THAT(GetDistData(1, "A").Grade, Grade::MEDIOCRE);
162 EXPECT_THAT(GetDistData(2, "A").Grade, Grade::EXCELLENT);
163
164 // Distribution B
165 // Throughput is 30, 100 and 100, so normalized scores are 0, 1 and 1.
166 EXPECT_THAT(GetDistData(0, "B").Score, DoubleNear(0, kAbsErr));
167 EXPECT_THAT(GetDistData(1, "B").Score, DoubleNear(1, kAbsErr));
168 EXPECT_THAT(GetDistData(2, "B").Score, DoubleNear(1, kAbsErr));
169 // which are turned into grades BAD, EXCELLENT and EXCELLENT.
170 EXPECT_THAT(GetDistData(0, "B").Grade, Grade::BAD);
171 EXPECT_THAT(GetDistData(1, "B").Grade, Grade::EXCELLENT);
172 EXPECT_THAT(GetDistData(2, "B").Grade, Grade::EXCELLENT);
173
174 // Now looking from the functions point of view.
175 EXPECT_THAT(Data[0].ScoresGeoMean, DoubleNear(0, kAbsErr));
176 EXPECT_THAT(Data[1].ScoresGeoMean, DoubleNear(1. * (1. / 7), kAbsErr));
177 EXPECT_THAT(Data[2].ScoresGeoMean, DoubleNear(1, kAbsErr));
178
179 // Note the array is indexed by GradeEnum values (EXCELLENT=0 / BAD = 6)
180 EXPECT_THAT(Data[0].GradeHisto, ElementsAre(0, 0, 0, 0, 0, 0, 2));
181 EXPECT_THAT(Data[1].GradeHisto, ElementsAre(1, 0, 0, 0, 0, 1, 0));
182 EXPECT_THAT(Data[2].GradeHisto, ElementsAre(2, 0, 0, 0, 0, 0, 0));
183
184 EXPECT_THAT(Data[0].FinalGrade, Grade::BAD);
185 EXPECT_THAT(Data[1].FinalGrade, Grade::MEDIOCRE);
186 EXPECT_THAT(Data[2].FinalGrade, Grade::EXCELLENT);
187}
188
189} // namespace
190} // namespace automemcpy
191} // namespace llvm
192

source code of libc/benchmarks/automemcpy/unittests/ResultAnalyzerTest.cpp