1//===-- Benchmark ---------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "JSON.h"
10#include "LibcBenchmark.h"
11#include "LibcMemoryBenchmark.h"
12#include "MemorySizeDistributions.h"
13#include "src/__support/macros/config.h"
14#include "llvm/Support/CommandLine.h"
15#include "llvm/Support/ErrorHandling.h"
16#include "llvm/Support/FileSystem.h"
17#include "llvm/Support/JSON.h"
18#include "llvm/Support/MathExtras.h"
19#include "llvm/Support/MemoryBuffer.h"
20#include "llvm/Support/raw_ostream.h"
21
22#include <cstring>
23#include <unistd.h>
24
25namespace LIBC_NAMESPACE_DECL {
26
27extern void *memcpy(void *__restrict, const void *__restrict, size_t);
28extern void *memmove(void *, const void *, size_t);
29extern void *memset(void *, int, size_t);
30extern void bzero(void *, size_t);
31extern int memcmp(const void *, const void *, size_t);
32extern int bcmp(const void *, const void *, size_t);
33
34} // namespace LIBC_NAMESPACE_DECL
35
36namespace llvm {
37namespace libc_benchmarks {
38
39static cl::opt<std::string>
40 StudyName("study-name", cl::desc("The name for this study"), cl::Required);
41
42static cl::opt<std::string>
43 SizeDistributionName("size-distribution-name",
44 cl::desc("The name of the distribution to use"));
45
46static cl::opt<bool> SweepMode(
47 "sweep-mode",
48 cl::desc(
49 "If set, benchmark all sizes from sweep-min-size to sweep-max-size"));
50
51static cl::opt<uint32_t>
52 SweepMinSize("sweep-min-size",
53 cl::desc("The minimum size to use in sweep-mode"),
54 cl::init(Val: 0));
55
56static cl::opt<uint32_t>
57 SweepMaxSize("sweep-max-size",
58 cl::desc("The maximum size to use in sweep-mode"),
59 cl::init(Val: 256));
60
61static cl::opt<uint32_t>
62 AlignedAccess("aligned-access",
63 cl::desc("The alignment to use when accessing the buffers\n"
64 "Default is unaligned\n"
65 "Use 0 to disable address randomization"),
66 cl::init(Val: 1));
67
68static cl::opt<std::string> Output("output",
69 cl::desc("Specify output filename"),
70 cl::value_desc("filename"), cl::init(Val: "-"));
71
72static cl::opt<uint32_t>
73 NumTrials("num-trials", cl::desc("The number of benchmarks run to perform"),
74 cl::init(Val: 1));
75
76#if defined(LIBC_BENCHMARK_FUNCTION_MEMCPY)
77#define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_MEMCPY
78using BenchmarkSetup = CopySetup;
79#elif defined(LIBC_BENCHMARK_FUNCTION_MEMMOVE)
80#define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_MEMMOVE
81using BenchmarkSetup = MoveSetup;
82#elif defined(LIBC_BENCHMARK_FUNCTION_MEMSET)
83#define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_MEMSET
84using BenchmarkSetup = SetSetup;
85#elif defined(LIBC_BENCHMARK_FUNCTION_BZERO)
86#define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_BZERO
87using BenchmarkSetup = SetSetup;
88#elif defined(LIBC_BENCHMARK_FUNCTION_MEMCMP)
89#define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_MEMCMP
90using BenchmarkSetup = ComparisonSetup;
91#elif defined(LIBC_BENCHMARK_FUNCTION_BCMP)
92#define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_BCMP
93using BenchmarkSetup = ComparisonSetup;
94#else
95#error "Missing LIBC_BENCHMARK_FUNCTION_XXX definition"
96#endif
97
98struct MemfunctionBenchmarkBase : public BenchmarkSetup {
99 MemfunctionBenchmarkBase() : ReportProgress(isatty(fd: fileno(stdout))) {}
100 virtual ~MemfunctionBenchmarkBase() {}
101
102 virtual Study run() = 0;
103
104 CircularArrayRef<ParameterBatch::ParameterType>
105 generateBatch(size_t Iterations) {
106 randomize();
107 return cycle(ArrayRef(Parameters), Iterations);
108 }
109
110protected:
111 Study createStudy() {
112 Study Study;
113 // Setup study.
114 Study.StudyName = StudyName;
115 Runtime &RI = Study.Runtime;
116 RI.Host = HostState::get();
117 RI.BufferSize = BufferSize;
118 RI.BatchParameterCount = BatchSize;
119
120 BenchmarkOptions &BO = RI.BenchmarkOptions;
121 BO.MinDuration = std::chrono::milliseconds(1);
122 BO.MaxDuration = std::chrono::seconds(1);
123 BO.MaxIterations = 10'000'000U;
124 BO.MinSamples = 4;
125 BO.MaxSamples = 1000;
126 BO.Epsilon = 0.01; // 1%
127 BO.ScalingFactor = 1.4;
128
129 StudyConfiguration &SC = Study.Configuration;
130 SC.NumTrials = NumTrials;
131 SC.IsSweepMode = SweepMode;
132 SC.AccessAlignment = MaybeAlign(AlignedAccess);
133 SC.Function = LIBC_BENCHMARK_FUNCTION_NAME;
134 return Study;
135 }
136
137 void runTrials(const BenchmarkOptions &Options,
138 std::vector<Duration> &Measurements) {
139 for (size_t i = 0; i < NumTrials; ++i) {
140 const BenchmarkResult Result = benchmark(
141 Options, PP&: *this, foo: [this](ParameterBatch::ParameterType Parameter) {
142 return Call(Parameter, LIBC_BENCHMARK_FUNCTION);
143 });
144 Measurements.push_back(x: Result.BestGuess);
145 reportProgress(Measurements);
146 }
147 }
148
149 virtual void randomize() = 0;
150
151private:
152 bool ReportProgress;
153
154 void reportProgress(const std::vector<Duration> &Measurements) {
155 if (!ReportProgress)
156 return;
157 static size_t LastPercent = -1;
158 const size_t TotalSteps = Measurements.capacity();
159 const size_t Steps = Measurements.size();
160 const size_t Percent = 100 * Steps / TotalSteps;
161 if (Percent == LastPercent)
162 return;
163 LastPercent = Percent;
164 size_t I = 0;
165 errs() << '[';
166 for (; I <= Percent; ++I)
167 errs() << '#';
168 for (; I <= 100; ++I)
169 errs() << '_';
170 errs() << "] " << Percent << '%' << '\r';
171 }
172};
173
174struct MemfunctionBenchmarkSweep final : public MemfunctionBenchmarkBase {
175 MemfunctionBenchmarkSweep()
176 : OffsetSampler(MemfunctionBenchmarkBase::BufferSize, SweepMaxSize,
177 MaybeAlign(AlignedAccess)) {}
178
179 virtual void randomize() override {
180 for (auto &P : Parameters) {
181 P.OffsetBytes = OffsetSampler(Gen);
182 P.SizeBytes = CurrentSweepSize;
183 checkValid(P);
184 }
185 }
186
187 virtual Study run() override {
188 Study Study = createStudy();
189 Study.Configuration.SweepModeMaxSize = SweepMaxSize;
190 BenchmarkOptions &BO = Study.Runtime.BenchmarkOptions;
191 BO.MinDuration = std::chrono::milliseconds(1);
192 BO.InitialIterations = 100;
193 auto &Measurements = Study.Measurements;
194 Measurements.reserve(NumTrials * SweepMaxSize);
195 for (size_t Size = SweepMinSize; Size <= SweepMaxSize; ++Size) {
196 CurrentSweepSize = Size;
197 runTrials(Options: BO, Measurements&: Measurements);
198 }
199 return Study;
200 }
201
202private:
203 size_t CurrentSweepSize = 0;
204 OffsetDistribution OffsetSampler;
205 std::mt19937_64 Gen;
206};
207
208struct MemfunctionBenchmarkDistribution final
209 : public MemfunctionBenchmarkBase {
210 MemfunctionBenchmarkDistribution(MemorySizeDistribution Distribution)
211 : Distribution(Distribution), Probabilities(Distribution.Probabilities),
212 SizeSampler(Probabilities.begin(), Probabilities.end()),
213 OffsetSampler(MemfunctionBenchmarkBase::BufferSize,
214 Probabilities.size() - 1, MaybeAlign(AlignedAccess)) {}
215
216 virtual void randomize() override {
217 for (auto &P : Parameters) {
218 P.OffsetBytes = OffsetSampler(Gen);
219 P.SizeBytes = SizeSampler(Gen);
220 checkValid(P);
221 }
222 }
223
224 virtual Study run() override {
225 Study Study = createStudy();
226 Study.Configuration.SizeDistributionName = Distribution.Name.str();
227 BenchmarkOptions &BO = Study.Runtime.BenchmarkOptions;
228 BO.MinDuration = std::chrono::milliseconds(10);
229 BO.InitialIterations = BatchSize * 10;
230 auto &Measurements = Study.Measurements;
231 Measurements.reserve(NumTrials);
232 runTrials(Options: BO, Measurements&: Measurements);
233 return Study;
234 }
235
236private:
237 MemorySizeDistribution Distribution;
238 ArrayRef<double> Probabilities;
239 std::discrete_distribution<unsigned> SizeSampler;
240 OffsetDistribution OffsetSampler;
241 std::mt19937_64 Gen;
242};
243
244void writeStudy(const Study &S) {
245 std::error_code EC;
246 raw_fd_ostream FOS(Output, EC);
247 if (EC)
248 report_fatal_error(reason: Twine("Could not open file: ")
249 .concat(Suffix: EC.message())
250 .concat(Suffix: ", ")
251 .concat(Suffix: Output));
252 json::OStream JOS(FOS);
253 serializeToJson(S, JOS);
254 FOS << "\n";
255}
256
257void main() {
258 checkRequirements();
259 if (!isPowerOf2_32(Value: AlignedAccess))
260 report_fatal_error(reason: AlignedAccess.ArgStr +
261 Twine(" must be a power of two or zero"));
262
263 const bool HasDistributionName = !SizeDistributionName.empty();
264 if (SweepMode && HasDistributionName)
265 report_fatal_error(reason: "Select only one of `--" + Twine(SweepMode.ArgStr) +
266 "` or `--" + Twine(SizeDistributionName.ArgStr) + "`");
267
268 std::unique_ptr<MemfunctionBenchmarkBase> Benchmark;
269 if (SweepMode)
270 Benchmark.reset(new MemfunctionBenchmarkSweep());
271 else
272 Benchmark.reset(new MemfunctionBenchmarkDistribution(getDistributionOrDie(
273 BenchmarkSetup::getDistributions(), SizeDistributionName)));
274 writeStudy(S: Benchmark->run());
275}
276
277} // namespace libc_benchmarks
278} // namespace llvm
279
280#ifndef NDEBUG
281#error For reproducibility benchmarks should not be compiled in DEBUG mode.
282#endif
283
284int main(int argc, char **argv) {
285 llvm::cl::ParseCommandLineOptions(argc, argv);
286 llvm::libc_benchmarks::main();
287 return EXIT_SUCCESS;
288}
289

source code of libc/benchmarks/LibcMemoryBenchmarkMain.cpp