1//===-- Benchmark ---------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "JSON.h"
10#include "LibcBenchmark.h"
11#include "LibcMemoryBenchmark.h"
12#include "MemorySizeDistributions.h"
13#include "llvm/Support/CommandLine.h"
14#include "llvm/Support/ErrorHandling.h"
15#include "llvm/Support/FileSystem.h"
16#include "llvm/Support/JSON.h"
17#include "llvm/Support/MathExtras.h"
18#include "llvm/Support/MemoryBuffer.h"
19#include "llvm/Support/raw_ostream.h"
20
21#include <cstring>
22#include <unistd.h>
23
24namespace LIBC_NAMESPACE {
25
26extern void *memcpy(void *__restrict, const void *__restrict, size_t);
27extern void *memmove(void *, const void *, size_t);
28extern void *memset(void *, int, size_t);
29extern void bzero(void *, size_t);
30extern int memcmp(const void *, const void *, size_t);
31extern int bcmp(const void *, const void *, size_t);
32
33} // namespace LIBC_NAMESPACE
34
35namespace llvm {
36namespace libc_benchmarks {
37
38static cl::opt<std::string>
39 StudyName("study-name", cl::desc("The name for this study"), cl::Required);
40
41static cl::opt<std::string>
42 SizeDistributionName("size-distribution-name",
43 cl::desc("The name of the distribution to use"));
44
45static cl::opt<bool> SweepMode(
46 "sweep-mode",
47 cl::desc(
48 "If set, benchmark all sizes from sweep-min-size to sweep-max-size"));
49
50static cl::opt<uint32_t>
51 SweepMinSize("sweep-min-size",
52 cl::desc("The minimum size to use in sweep-mode"),
53 cl::init(0));
54
55static cl::opt<uint32_t>
56 SweepMaxSize("sweep-max-size",
57 cl::desc("The maximum size to use in sweep-mode"),
58 cl::init(256));
59
60static cl::opt<uint32_t>
61 AlignedAccess("aligned-access",
62 cl::desc("The alignment to use when accessing the buffers\n"
63 "Default is unaligned\n"
64 "Use 0 to disable address randomization"),
65 cl::init(1));
66
67static cl::opt<std::string> Output("output",
68 cl::desc("Specify output filename"),
69 cl::value_desc("filename"), cl::init("-"));
70
71static cl::opt<uint32_t>
72 NumTrials("num-trials", cl::desc("The number of benchmarks run to perform"),
73 cl::init(1));
74
75#if defined(LIBC_BENCHMARK_FUNCTION_MEMCPY)
76#define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_MEMCPY
77using BenchmarkSetup = CopySetup;
78#elif defined(LIBC_BENCHMARK_FUNCTION_MEMMOVE)
79#define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_MEMMOVE
80using BenchmarkSetup = MoveSetup;
81#elif defined(LIBC_BENCHMARK_FUNCTION_MEMSET)
82#define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_MEMSET
83using BenchmarkSetup = SetSetup;
84#elif defined(LIBC_BENCHMARK_FUNCTION_BZERO)
85#define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_BZERO
86using BenchmarkSetup = SetSetup;
87#elif defined(LIBC_BENCHMARK_FUNCTION_MEMCMP)
88#define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_MEMCMP
89using BenchmarkSetup = ComparisonSetup;
90#elif defined(LIBC_BENCHMARK_FUNCTION_BCMP)
91#define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_BCMP
92using BenchmarkSetup = ComparisonSetup;
93#else
94#error "Missing LIBC_BENCHMARK_FUNCTION_XXX definition"
95#endif
96
97struct MemfunctionBenchmarkBase : public BenchmarkSetup {
98 MemfunctionBenchmarkBase() : ReportProgress(isatty(fd: fileno(stdout))) {}
99 virtual ~MemfunctionBenchmarkBase() {}
100
101 virtual Study run() = 0;
102
103 CircularArrayRef<ParameterBatch::ParameterType>
104 generateBatch(size_t Iterations) {
105 randomize();
106 return cycle(ArrayRef(Parameters), Iterations);
107 }
108
109protected:
110 Study createStudy() {
111 Study Study;
112 // Setup study.
113 Study.StudyName = StudyName;
114 Runtime &RI = Study.Runtime;
115 RI.Host = HostState::get();
116 RI.BufferSize = BufferSize;
117 RI.BatchParameterCount = BatchSize;
118
119 BenchmarkOptions &BO = RI.BenchmarkOptions;
120 BO.MinDuration = std::chrono::milliseconds(1);
121 BO.MaxDuration = std::chrono::seconds(1);
122 BO.MaxIterations = 10'000'000U;
123 BO.MinSamples = 4;
124 BO.MaxSamples = 1000;
125 BO.Epsilon = 0.01; // 1%
126 BO.ScalingFactor = 1.4;
127
128 StudyConfiguration &SC = Study.Configuration;
129 SC.NumTrials = NumTrials;
130 SC.IsSweepMode = SweepMode;
131 SC.AccessAlignment = MaybeAlign(AlignedAccess);
132 SC.Function = LIBC_BENCHMARK_FUNCTION_NAME;
133 return Study;
134 }
135
136 void runTrials(const BenchmarkOptions &Options,
137 std::vector<Duration> &Measurements) {
138 for (size_t i = 0; i < NumTrials; ++i) {
139 const BenchmarkResult Result = benchmark(
140 Options, PP&: *this, foo: [this](ParameterBatch::ParameterType Parameter) {
141 return Call(Parameter, LIBC_BENCHMARK_FUNCTION);
142 });
143 Measurements.push_back(x: Result.BestGuess);
144 reportProgress(Measurements);
145 }
146 }
147
148 virtual void randomize() = 0;
149
150private:
151 bool ReportProgress;
152
153 void reportProgress(const std::vector<Duration> &Measurements) {
154 if (!ReportProgress)
155 return;
156 static size_t LastPercent = -1;
157 const size_t TotalSteps = Measurements.capacity();
158 const size_t Steps = Measurements.size();
159 const size_t Percent = 100 * Steps / TotalSteps;
160 if (Percent == LastPercent)
161 return;
162 LastPercent = Percent;
163 size_t I = 0;
164 errs() << '[';
165 for (; I <= Percent; ++I)
166 errs() << '#';
167 for (; I <= 100; ++I)
168 errs() << '_';
169 errs() << "] " << Percent << '%' << '\r';
170 }
171};
172
173struct MemfunctionBenchmarkSweep final : public MemfunctionBenchmarkBase {
174 MemfunctionBenchmarkSweep()
175 : OffsetSampler(MemfunctionBenchmarkBase::BufferSize, SweepMaxSize,
176 MaybeAlign(AlignedAccess)) {}
177
178 virtual void randomize() override {
179 for (auto &P : Parameters) {
180 P.OffsetBytes = OffsetSampler(Gen);
181 P.SizeBytes = CurrentSweepSize;
182 checkValid(P);
183 }
184 }
185
186 virtual Study run() override {
187 Study Study = createStudy();
188 Study.Configuration.SweepModeMaxSize = SweepMaxSize;
189 BenchmarkOptions &BO = Study.Runtime.BenchmarkOptions;
190 BO.MinDuration = std::chrono::milliseconds(1);
191 BO.InitialIterations = 100;
192 auto &Measurements = Study.Measurements;
193 Measurements.reserve(n: NumTrials * SweepMaxSize);
194 for (size_t Size = SweepMinSize; Size <= SweepMaxSize; ++Size) {
195 CurrentSweepSize = Size;
196 runTrials(Options: BO, Measurements);
197 }
198 return Study;
199 }
200
201private:
202 size_t CurrentSweepSize = 0;
203 OffsetDistribution OffsetSampler;
204 std::mt19937_64 Gen;
205};
206
207struct MemfunctionBenchmarkDistribution final
208 : public MemfunctionBenchmarkBase {
209 MemfunctionBenchmarkDistribution(MemorySizeDistribution Distribution)
210 : Distribution(Distribution), Probabilities(Distribution.Probabilities),
211 SizeSampler(Probabilities.begin(), Probabilities.end()),
212 OffsetSampler(MemfunctionBenchmarkBase::BufferSize,
213 Probabilities.size() - 1, MaybeAlign(AlignedAccess)) {}
214
215 virtual void randomize() override {
216 for (auto &P : Parameters) {
217 P.OffsetBytes = OffsetSampler(Gen);
218 P.SizeBytes = SizeSampler(Gen);
219 checkValid(P);
220 }
221 }
222
223 virtual Study run() override {
224 Study Study = createStudy();
225 Study.Configuration.SizeDistributionName = Distribution.Name.str();
226 BenchmarkOptions &BO = Study.Runtime.BenchmarkOptions;
227 BO.MinDuration = std::chrono::milliseconds(10);
228 BO.InitialIterations = BatchSize * 10;
229 auto &Measurements = Study.Measurements;
230 Measurements.reserve(NumTrials);
231 runTrials(Options: BO, Measurements&: Measurements);
232 return Study;
233 }
234
235private:
236 MemorySizeDistribution Distribution;
237 ArrayRef<double> Probabilities;
238 std::discrete_distribution<unsigned> SizeSampler;
239 OffsetDistribution OffsetSampler;
240 std::mt19937_64 Gen;
241};
242
243void writeStudy(const Study &S) {
244 std::error_code EC;
245 raw_fd_ostream FOS(Output, EC);
246 if (EC)
247 report_fatal_error(Twine("Could not open file: ")
248 .concat(EC.message())
249 .concat(", ")
250 .concat(Output));
251 json::OStream JOS(FOS);
252 serializeToJson(S, JOS);
253 FOS << "\n";
254}
255
256void main() {
257 checkRequirements();
258 if (!isPowerOf2_32(AlignedAccess))
259 report_fatal_error(AlignedAccess.ArgStr +
260 Twine(" must be a power of two or zero"));
261
262 const bool HasDistributionName = !SizeDistributionName.empty();
263 if (SweepMode && HasDistributionName)
264 report_fatal_error("Select only one of `--" + Twine(SweepMode.ArgStr) +
265 "` or `--" + Twine(SizeDistributionName.ArgStr) + "`");
266
267 std::unique_ptr<MemfunctionBenchmarkBase> Benchmark;
268 if (SweepMode)
269 Benchmark.reset(new MemfunctionBenchmarkSweep());
270 else
271 Benchmark.reset(new MemfunctionBenchmarkDistribution(getDistributionOrDie(
272 BenchmarkSetup::getDistributions(), SizeDistributionName)));
273 writeStudy(Benchmark->run());
274}
275
276} // namespace libc_benchmarks
277} // namespace llvm
278
279#ifndef NDEBUG
280#error For reproducibility benchmarks should not be compiled in DEBUG mode.
281#endif
282
283int main(int argc, char **argv) {
284 llvm::cl::ParseCommandLineOptions(argc, argv);
285 llvm::libc_benchmarks::main();
286 return EXIT_SUCCESS;
287}
288

source code of libc/benchmarks/LibcMemoryBenchmarkMain.cpp