1#include "LibcBenchmark.h"
2#include "LibcMemoryBenchmark.h"
3#include "MemorySizeDistributions.h"
4#include "benchmark/benchmark.h"
5#include "llvm/ADT/ArrayRef.h"
6#include "llvm/ADT/Twine.h"
7#include <chrono>
8#include <cstdint>
9#include <random>
10#include <vector>
11
12using llvm::Align;
13using llvm::ArrayRef;
14using llvm::Twine;
15using llvm::libc_benchmarks::BzeroConfiguration;
16using llvm::libc_benchmarks::ComparisonSetup;
17using llvm::libc_benchmarks::CopySetup;
18using llvm::libc_benchmarks::MemcmpOrBcmpConfiguration;
19using llvm::libc_benchmarks::MemcpyConfiguration;
20using llvm::libc_benchmarks::MemmoveConfiguration;
21using llvm::libc_benchmarks::MemorySizeDistribution;
22using llvm::libc_benchmarks::MemsetConfiguration;
23using llvm::libc_benchmarks::MoveSetup;
24using llvm::libc_benchmarks::OffsetDistribution;
25using llvm::libc_benchmarks::SetSetup;
26
27// Alignment to use for when accessing the buffers.
28static constexpr Align kBenchmarkAlignment = Align::Constant<1>();
29
30static std::mt19937_64 &getGenerator() {
31 static std::mt19937_64 Generator(
32 std::chrono::system_clock::now().time_since_epoch().count());
33 return Generator;
34}
35
36template <typename SetupType, typename ConfigurationType> struct Runner {
37 Runner(benchmark::State &S, llvm::ArrayRef<ConfigurationType> Configurations)
38 : State(S), Distribution(SetupType::getDistributions()[State.range(0)]),
39 Probabilities(Distribution.Probabilities),
40 SizeSampler(Probabilities.begin(), Probabilities.end()),
41 OffsetSampler(Setup.BufferSize, Probabilities.size() - 1,
42 kBenchmarkAlignment),
43 Configuration(Configurations[State.range(1)]) {
44 for (auto &P : Setup.Parameters) {
45 P.OffsetBytes = OffsetSampler(getGenerator());
46 P.SizeBytes = SizeSampler(getGenerator());
47 Setup.checkValid(P);
48 }
49 }
50
51 ~Runner() {
52 const size_t TotalBytes =
53 (State.iterations() * Setup.getBatchBytes()) / Setup.BatchSize;
54 State.SetBytesProcessed(TotalBytes);
55 State.SetItemsProcessed(State.iterations());
56 State.SetLabel((Twine(Configuration.Name) + "," + Distribution.Name).str());
57 State.counters["bytes_per_cycle"] = benchmark::Counter(
58 TotalBytes / benchmark::CPUInfo::Get().cycles_per_second,
59 benchmark::Counter::kIsRate);
60 }
61
62 inline void runBatch() {
63 for (const auto &P : Setup.Parameters)
64 benchmark::DoNotOptimize(Setup.Call(P, Configuration.Function));
65 }
66
67 size_t getBatchSize() const { return Setup.BatchSize; }
68
69private:
70 SetupType Setup;
71 benchmark::State &State;
72 MemorySizeDistribution Distribution;
73 ArrayRef<double> Probabilities;
74 std::discrete_distribution<unsigned> SizeSampler;
75 OffsetDistribution OffsetSampler;
76 ConfigurationType Configuration;
77};
78
79#define BENCHMARK_MEMORY_FUNCTION(BM_NAME, SETUP, CONFIGURATION_TYPE, \
80 CONFIGURATION_ARRAY_REF) \
81 void BM_NAME(benchmark::State &State) { \
82 Runner<SETUP, CONFIGURATION_TYPE> Setup(State, CONFIGURATION_ARRAY_REF); \
83 const size_t BatchSize = Setup.getBatchSize(); \
84 while (State.KeepRunningBatch(BatchSize)) \
85 Setup.runBatch(); \
86 } \
87 BENCHMARK(BM_NAME)->Apply([](benchmark::internal::Benchmark *benchmark) { \
88 const int64_t DistributionSize = SETUP::getDistributions().size(); \
89 const int64_t ConfigurationSize = CONFIGURATION_ARRAY_REF.size(); \
90 for (int64_t DistIndex = 0; DistIndex < DistributionSize; ++DistIndex) \
91 for (int64_t ConfIndex = 0; ConfIndex < ConfigurationSize; ++ConfIndex) \
92 benchmark->Args({DistIndex, ConfIndex}); \
93 })
94
95extern llvm::ArrayRef<MemcpyConfiguration> getMemcpyConfigurations();
96BENCHMARK_MEMORY_FUNCTION(BM_Memcpy, CopySetup, MemcpyConfiguration,
97 getMemcpyConfigurations());
98
99extern llvm::ArrayRef<MemmoveConfiguration> getMemmoveConfigurations();
100BENCHMARK_MEMORY_FUNCTION(BM_Memmove, MoveSetup, MemmoveConfiguration,
101 getMemmoveConfigurations());
102
103extern llvm::ArrayRef<MemcmpOrBcmpConfiguration> getMemcmpConfigurations();
104BENCHMARK_MEMORY_FUNCTION(BM_Memcmp, ComparisonSetup, MemcmpOrBcmpConfiguration,
105 getMemcmpConfigurations());
106
107extern llvm::ArrayRef<MemcmpOrBcmpConfiguration> getBcmpConfigurations();
108BENCHMARK_MEMORY_FUNCTION(BM_Bcmp, ComparisonSetup, MemcmpOrBcmpConfiguration,
109 getBcmpConfigurations());
110
111extern llvm::ArrayRef<MemsetConfiguration> getMemsetConfigurations();
112BENCHMARK_MEMORY_FUNCTION(BM_Memset, SetSetup, MemsetConfiguration,
113 getMemsetConfigurations());
114
115extern llvm::ArrayRef<BzeroConfiguration> getBzeroConfigurations();
116BENCHMARK_MEMORY_FUNCTION(BM_Bzero, SetSetup, BzeroConfiguration,
117 getBzeroConfigurations());
118

source code of libc/benchmarks/LibcMemoryGoogleBenchmarkMain.cpp