1#include <cstdio>
2#include <cstring>
3#include <fstream>
4#include <iostream>
5#include <map>
6#include <memory>
7#include <random>
8#include <sstream>
9#include <streambuf>
10
11#include "../src/benchmark_api_internal.h"
12#include "../src/check.h" // NOTE: check.h is for internal use only!
13#include "../src/log.h" // NOTE: log.h is for internal use only
14#include "../src/re.h" // NOTE: re.h is for internal use only
15#include "output_test.h"
16
17// ========================================================================= //
18// ------------------------------ Internals -------------------------------- //
19// ========================================================================= //
20namespace internal {
21namespace {
22
23using TestCaseList = std::vector<TestCase>;
24
25// Use a vector because the order elements are added matters during iteration.
26// std::map/unordered_map don't guarantee that.
27// For example:
28// SetSubstitutions({{"%HelloWorld", "Hello"}, {"%Hello", "Hi"}});
29// Substitute("%HelloWorld") // Always expands to Hello.
30using SubMap = std::vector<std::pair<std::string, std::string>>;
31
32TestCaseList& GetTestCaseList(TestCaseID ID) {
33 // Uses function-local statics to ensure initialization occurs
34 // before first use.
35 static TestCaseList lists[TC_NumID];
36 return lists[ID];
37}
38
39SubMap& GetSubstitutions() {
40 // Don't use 'dec_re' from header because it may not yet be initialized.
41 // clang-format off
42 static std::string safe_dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?";
43 static std::string time_re = "([0-9]+[.])?[0-9]+";
44 static std::string percentage_re = "[0-9]+[.][0-9]{2}";
45 static SubMap map = {
46 {"%float", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"},
47 // human-readable float
48 {"%hrfloat", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?[kKMGTPEZYmunpfazy]?i?"},
49 {"%percentage", percentage_re},
50 {"%int", "[ ]*[0-9]+"},
51 {" %s ", "[ ]+"},
52 {"%time", "[ ]*" + time_re + "[ ]+ns"},
53 {"%console_report", "[ ]*" + time_re + "[ ]+ns [ ]*" + time_re + "[ ]+ns [ ]*[0-9]+"},
54 {"%console_percentage_report", "[ ]*" + percentage_re + "[ ]+% [ ]*" + percentage_re + "[ ]+% [ ]*[0-9]+"},
55 {"%console_us_report", "[ ]*" + time_re + "[ ]+us [ ]*" + time_re + "[ ]+us [ ]*[0-9]+"},
56 {"%console_ms_report", "[ ]*" + time_re + "[ ]+ms [ ]*" + time_re + "[ ]+ms [ ]*[0-9]+"},
57 {"%console_s_report", "[ ]*" + time_re + "[ ]+s [ ]*" + time_re + "[ ]+s [ ]*[0-9]+"},
58 {"%console_time_only_report", "[ ]*" + time_re + "[ ]+ns [ ]*" + time_re + "[ ]+ns"},
59 {"%console_us_report", "[ ]*" + time_re + "[ ]+us [ ]*" + time_re + "[ ]+us [ ]*[0-9]+"},
60 {"%console_us_time_only_report", "[ ]*" + time_re + "[ ]+us [ ]*" + time_re + "[ ]+us"},
61 {"%csv_header",
62 "name,iterations,real_time,cpu_time,time_unit,bytes_per_second,"
63 "items_per_second,label,error_occurred,error_message"},
64 {"%csv_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,,,,,"},
65 {"%csv_us_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",us,,,,,"},
66 {"%csv_ms_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ms,,,,,"},
67 {"%csv_s_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",s,,,,,"},
68 {"%csv_cv_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",,,,,,"},
69 {"%csv_bytes_report",
70 "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns," + safe_dec_re + ",,,,"},
71 {"%csv_items_report",
72 "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,," + safe_dec_re + ",,,"},
73 {"%csv_bytes_items_report",
74 "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns," + safe_dec_re +
75 "," + safe_dec_re + ",,,"},
76 {"%csv_label_report_begin", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,,,"},
77 {"%csv_label_report_end", ",,"}};
78 // clang-format on
79 return map;
80}
81
82std::string PerformSubstitutions(std::string source) {
83 SubMap const& subs = GetSubstitutions();
84 using SizeT = std::string::size_type;
85 for (auto const& KV : subs) {
86 SizeT pos;
87 SizeT next_start = 0;
88 while ((pos = source.find(str: KV.first, pos: next_start)) != std::string::npos) {
89 next_start = pos + KV.second.size();
90 source.replace(pos: pos, n: KV.first.size(), str: KV.second);
91 }
92 }
93 return source;
94}
95
96void CheckCase(std::stringstream& remaining_output, TestCase const& TC,
97 TestCaseList const& not_checks) {
98 std::string first_line;
99 bool on_first = true;
100 std::string line;
101 while (remaining_output.eof() == false) {
102 BM_CHECK(remaining_output.good());
103 std::getline(is&: remaining_output, str&: line);
104 if (on_first) {
105 first_line = line;
106 on_first = false;
107 }
108 for (const auto& NC : not_checks) {
109 BM_CHECK(!NC.regex->Match(line))
110 << "Unexpected match for line \"" << line << "\" for MR_Not regex \""
111 << NC.regex_str << "\""
112 << "\n actual regex string \"" << TC.substituted_regex << "\""
113 << "\n started matching near: " << first_line;
114 }
115 if (TC.regex->Match(str: line)) return;
116 BM_CHECK(TC.match_rule != MR_Next)
117 << "Expected line \"" << line << "\" to match regex \"" << TC.regex_str
118 << "\""
119 << "\n actual regex string \"" << TC.substituted_regex << "\""
120 << "\n started matching near: " << first_line;
121 }
122 BM_CHECK(remaining_output.eof() == false)
123 << "End of output reached before match for regex \"" << TC.regex_str
124 << "\" was found"
125 << "\n actual regex string \"" << TC.substituted_regex << "\""
126 << "\n started matching near: " << first_line;
127}
128
129void CheckCases(TestCaseList const& checks, std::stringstream& output) {
130 std::vector<TestCase> not_checks;
131 for (size_t i = 0; i < checks.size(); ++i) {
132 const auto& TC = checks[i];
133 if (TC.match_rule == MR_Not) {
134 not_checks.push_back(x: TC);
135 continue;
136 }
137 CheckCase(remaining_output&: output, TC, not_checks);
138 not_checks.clear();
139 }
140}
141
142class TestReporter : public benchmark::BenchmarkReporter {
143 public:
144 TestReporter(std::vector<benchmark::BenchmarkReporter*> reps)
145 : reporters_(std::move(reps)) {}
146
147 bool ReportContext(const Context& context) override {
148 bool last_ret = false;
149 bool first = true;
150 for (auto rep : reporters_) {
151 bool new_ret = rep->ReportContext(context);
152 BM_CHECK(first || new_ret == last_ret)
153 << "Reports return different values for ReportContext";
154 first = false;
155 last_ret = new_ret;
156 }
157 (void)first;
158 return last_ret;
159 }
160
161 void ReportRuns(const std::vector<Run>& report) override {
162 for (auto rep : reporters_) rep->ReportRuns(report);
163 }
164 void Finalize() override {
165 for (auto rep : reporters_) rep->Finalize();
166 }
167
168 private:
169 std::vector<benchmark::BenchmarkReporter*> reporters_;
170};
171} // namespace
172
173} // end namespace internal
174
175// ========================================================================= //
176// -------------------------- Results checking ----------------------------- //
177// ========================================================================= //
178
179namespace internal {
180
181// Utility class to manage subscribers for checking benchmark results.
182// It works by parsing the CSV output to read the results.
183class ResultsChecker {
184 public:
185 struct PatternAndFn : public TestCase { // reusing TestCase for its regexes
186 PatternAndFn(const std::string& rx, ResultsCheckFn fn_)
187 : TestCase(rx), fn(std::move(fn_)) {}
188 ResultsCheckFn fn;
189 };
190
191 std::vector<PatternAndFn> check_patterns;
192 std::vector<Results> results;
193 std::vector<std::string> field_names;
194
195 void Add(const std::string& entry_pattern, const ResultsCheckFn& fn);
196
197 void CheckResults(std::stringstream& output);
198
199 private:
200 void SetHeader_(const std::string& csv_header);
201 void SetValues_(const std::string& entry_csv_line);
202
203 std::vector<std::string> SplitCsv_(const std::string& line);
204};
205
206// store the static ResultsChecker in a function to prevent initialization
207// order problems
208ResultsChecker& GetResultsChecker() {
209 static ResultsChecker rc;
210 return rc;
211}
212
213// add a results checker for a benchmark
214void ResultsChecker::Add(const std::string& entry_pattern,
215 const ResultsCheckFn& fn) {
216 check_patterns.emplace_back(args: entry_pattern, args: fn);
217}
218
219// check the results of all subscribed benchmarks
220void ResultsChecker::CheckResults(std::stringstream& output) {
221 // first reset the stream to the start
222 {
223 auto start = std::stringstream::pos_type(0);
224 // clear before calling tellg()
225 output.clear();
226 // seek to zero only when needed
227 if (output.tellg() > start) output.seekg(start);
228 // and just in case
229 output.clear();
230 }
231 // now go over every line and publish it to the ResultsChecker
232 std::string line;
233 bool on_first = true;
234 while (output.eof() == false) {
235 BM_CHECK(output.good());
236 std::getline(is&: output, str&: line);
237 if (on_first) {
238 SetHeader_(line); // this is important
239 on_first = false;
240 continue;
241 }
242 SetValues_(line);
243 }
244 // finally we can call the subscribed check functions
245 for (const auto& p : check_patterns) {
246 BM_VLOG(2) << "--------------------------------\n";
247 BM_VLOG(2) << "checking for benchmarks matching " << p.regex_str << "...\n";
248 for (const auto& r : results) {
249 if (!p.regex->Match(str: r.name)) {
250 BM_VLOG(2) << p.regex_str << " is not matched by " << r.name << "\n";
251 continue;
252 }
253 BM_VLOG(2) << p.regex_str << " is matched by " << r.name << "\n";
254 BM_VLOG(1) << "Checking results of " << r.name << ": ... \n";
255 p.fn(r);
256 BM_VLOG(1) << "Checking results of " << r.name << ": OK.\n";
257 }
258 }
259}
260
261// prepare for the names in this header
262void ResultsChecker::SetHeader_(const std::string& csv_header) {
263 field_names = SplitCsv_(line: csv_header);
264}
265
266// set the values for a benchmark
267void ResultsChecker::SetValues_(const std::string& entry_csv_line) {
268 if (entry_csv_line.empty()) return; // some lines are empty
269 BM_CHECK(!field_names.empty());
270 auto vals = SplitCsv_(line: entry_csv_line);
271 BM_CHECK_EQ(vals.size(), field_names.size());
272 results.emplace_back(args&: vals[0]); // vals[0] is the benchmark name
273 auto& entry = results.back();
274 for (size_t i = 1, e = vals.size(); i < e; ++i) {
275 entry.values[field_names[i]] = vals[i];
276 }
277}
278
279// a quick'n'dirty csv splitter (eliminating quotes)
280std::vector<std::string> ResultsChecker::SplitCsv_(const std::string& line) {
281 std::vector<std::string> out;
282 if (line.empty()) return out;
283 if (!field_names.empty()) out.reserve(n: field_names.size());
284 size_t prev = 0, pos = line.find_first_of(c: ','), curr = pos;
285 while (pos != line.npos) {
286 BM_CHECK(curr > 0);
287 if (line[prev] == '"') ++prev;
288 if (line[curr - 1] == '"') --curr;
289 out.push_back(x: line.substr(pos: prev, n: curr - prev));
290 prev = pos + 1;
291 pos = line.find_first_of(c: ',', pos: pos + 1);
292 curr = pos;
293 }
294 curr = line.size();
295 if (line[prev] == '"') ++prev;
296 if (line[curr - 1] == '"') --curr;
297 out.push_back(x: line.substr(pos: prev, n: curr - prev));
298 return out;
299}
300
301} // end namespace internal
302
303size_t AddChecker(const std::string& bm_name, const ResultsCheckFn& fn) {
304 auto& rc = internal::GetResultsChecker();
305 rc.Add(entry_pattern: bm_name, fn);
306 return rc.results.size();
307}
308
309int Results::NumThreads() const {
310 auto pos = name.find(s: "/threads:");
311 if (pos == name.npos) return 1;
312 auto end = name.find(c: '/', pos: pos + 9);
313 std::stringstream ss;
314 ss << name.substr(pos: pos + 9, n: end);
315 int num = 1;
316 ss >> num;
317 BM_CHECK(!ss.fail());
318 return num;
319}
320
321double Results::NumIterations() const { return GetAs<double>(entry_name: "iterations"); }
322
323double Results::GetTime(BenchmarkTime which) const {
324 BM_CHECK(which == kCpuTime || which == kRealTime);
325 const char* which_str = which == kCpuTime ? "cpu_time" : "real_time";
326 double val = GetAs<double>(entry_name: which_str);
327 auto unit = Get(entry_name: "time_unit");
328 BM_CHECK(unit);
329 if (*unit == "ns") {
330 return val * 1.e-9;
331 }
332 if (*unit == "us") {
333 return val * 1.e-6;
334 }
335 if (*unit == "ms") {
336 return val * 1.e-3;
337 }
338 if (*unit == "s") {
339 return val;
340 }
341 BM_CHECK(1 == 0) << "unknown time unit: " << *unit;
342 return 0;
343}
344
345// ========================================================================= //
346// -------------------------- Public API Definitions------------------------ //
347// ========================================================================= //
348
349TestCase::TestCase(std::string re, int rule)
350 : regex_str(std::move(re)),
351 match_rule(rule),
352 substituted_regex(internal::PerformSubstitutions(source: regex_str)),
353 regex(std::make_shared<benchmark::Regex>()) {
354 std::string err_str;
355 regex->Init(spec: substituted_regex, error: &err_str);
356 BM_CHECK(err_str.empty())
357 << "Could not construct regex \"" << substituted_regex << "\""
358 << "\n originally \"" << regex_str << "\""
359 << "\n got error: " << err_str;
360}
361
362int AddCases(TestCaseID ID, std::initializer_list<TestCase> il) {
363 auto& L = internal::GetTestCaseList(ID);
364 L.insert(position: L.end(), l: il);
365 return 0;
366}
367
368int SetSubstitutions(
369 std::initializer_list<std::pair<std::string, std::string>> il) {
370 auto& subs = internal::GetSubstitutions();
371 for (auto KV : il) {
372 bool exists = false;
373 KV.second = internal::PerformSubstitutions(source: KV.second);
374 for (auto& EKV : subs) {
375 if (EKV.first == KV.first) {
376 EKV.second = std::move(KV.second);
377 exists = true;
378 break;
379 }
380 }
381 if (!exists) subs.push_back(x: std::move(KV));
382 }
383 return 0;
384}
385
386// Disable deprecated warnings temporarily because we need to reference
387// CSVReporter but don't want to trigger -Werror=-Wdeprecated-declarations
388BENCHMARK_DISABLE_DEPRECATED_WARNING
389
390void RunOutputTests(int argc, char* argv[]) {
391 using internal::GetTestCaseList;
392 benchmark::Initialize(argc: &argc, argv);
393 auto options = benchmark::internal::GetOutputOptions(/*force_no_color*/ true);
394 benchmark::ConsoleReporter CR(options);
395 benchmark::JSONReporter JR;
396 benchmark::CSVReporter CSVR;
397 struct ReporterTest {
398 std::string name;
399 std::vector<TestCase>& output_cases;
400 std::vector<TestCase>& error_cases;
401 benchmark::BenchmarkReporter& reporter;
402 std::stringstream out_stream;
403 std::stringstream err_stream;
404
405 ReporterTest(const std::string& n, std::vector<TestCase>& out_tc,
406 std::vector<TestCase>& err_tc,
407 benchmark::BenchmarkReporter& br)
408 : name(n), output_cases(out_tc), error_cases(err_tc), reporter(br) {
409 reporter.SetOutputStream(&out_stream);
410 reporter.SetErrorStream(&err_stream);
411 }
412 } TestCases[] = {
413 {std::string("ConsoleReporter"), GetTestCaseList(ID: TC_ConsoleOut),
414 GetTestCaseList(ID: TC_ConsoleErr), CR},
415 {std::string("JSONReporter"), GetTestCaseList(ID: TC_JSONOut),
416 GetTestCaseList(ID: TC_JSONErr), JR},
417 {std::string("CSVReporter"), GetTestCaseList(ID: TC_CSVOut),
418 GetTestCaseList(ID: TC_CSVErr), CSVR},
419 };
420
421 // Create the test reporter and run the benchmarks.
422 std::cout << "Running benchmarks...\n";
423 internal::TestReporter test_rep({&CR, &JR, &CSVR});
424 benchmark::RunSpecifiedBenchmarks(display_reporter: &test_rep);
425
426 for (auto& rep_test : TestCases) {
427 std::string msg =
428 std::string("\nTesting ") + rep_test.name + std::string(" Output\n");
429 std::string banner(msg.size() - 1, '-');
430 std::cout << banner << msg << banner << "\n";
431
432 std::cerr << rep_test.err_stream.str();
433 std::cout << rep_test.out_stream.str();
434
435 internal::CheckCases(checks: rep_test.error_cases, output&: rep_test.err_stream);
436 internal::CheckCases(checks: rep_test.output_cases, output&: rep_test.out_stream);
437
438 std::cout << "\n";
439 }
440
441 // now that we know the output is as expected, we can dispatch
442 // the checks to subscribees.
443 auto& csv = TestCases[2];
444 // would use == but gcc spits a warning
445 BM_CHECK(csv.name == std::string("CSVReporter"));
446 internal::GetResultsChecker().CheckResults(output&: csv.out_stream);
447}
448
449BENCHMARK_RESTORE_DEPRECATED_WARNING
450
451int SubstrCnt(const std::string& haystack, const std::string& pat) {
452 if (pat.length() == 0) return 0;
453 int count = 0;
454 for (size_t offset = haystack.find(str: pat); offset != std::string::npos;
455 offset = haystack.find(str: pat, pos: offset + pat.length()))
456 ++count;
457 return count;
458}
459
460static char ToHex(int ch) {
461 return ch < 10 ? static_cast<char>('0' + ch)
462 : static_cast<char>('a' + (ch - 10));
463}
464
465static char RandomHexChar() {
466 static std::mt19937 rd{std::random_device{}()};
467 static std::uniform_int_distribution<int> mrand{0, 15};
468 return ToHex(ch: mrand(rd));
469}
470
471static std::string GetRandomFileName() {
472 std::string model = "test.%%%%%%";
473 for (auto& ch : model) {
474 if (ch == '%') ch = RandomHexChar();
475 }
476 return model;
477}
478
479static bool FileExists(std::string const& name) {
480 std::ifstream in(name.c_str());
481 return in.good();
482}
483
484static std::string GetTempFileName() {
485 // This function attempts to avoid race conditions where two tests
486 // create the same file at the same time. However, it still introduces races
487 // similar to tmpnam.
488 int retries = 3;
489 while (--retries) {
490 std::string name = GetRandomFileName();
491 if (!FileExists(name)) return name;
492 }
493 std::cerr << "Failed to create unique temporary file name" << std::endl;
494 std::abort();
495}
496
497std::string GetFileReporterOutput(int argc, char* argv[]) {
498 std::vector<char*> new_argv(argv, argv + argc);
499 assert(static_cast<decltype(new_argv)::size_type>(argc) == new_argv.size());
500
501 std::string tmp_file_name = GetTempFileName();
502 std::cout << "Will be using this as the tmp file: " << tmp_file_name << '\n';
503
504 std::string tmp = "--benchmark_out=";
505 tmp += tmp_file_name;
506 new_argv.emplace_back(args: const_cast<char*>(tmp.c_str()));
507
508 argc = int(new_argv.size());
509
510 benchmark::Initialize(argc: &argc, argv: new_argv.data());
511 benchmark::RunSpecifiedBenchmarks();
512
513 // Read the output back from the file, and delete the file.
514 std::ifstream tmp_stream(tmp_file_name);
515 std::string output = std::string((std::istreambuf_iterator<char>(tmp_stream)),
516 std::istreambuf_iterator<char>());
517 std::remove(filename: tmp_file_name.c_str());
518
519 return output;
520}
521

source code of third-party/benchmark/test/output_test_helper.cc