1
2#undef NDEBUG
3#include <numeric>
4#include <utility>
5
6#include "benchmark/benchmark.h"
7#include "output_test.h"
8
9// ========================================================================= //
10// ---------------------- Testing Prologue Output -------------------------- //
11// ========================================================================= //
12
13ADD_CASES(TC_ConsoleOut, {{"^[-]+$", MR_Next},
14 {"^Benchmark %s Time %s CPU %s Iterations$", MR_Next},
15 {"^[-]+$", MR_Next}});
16static int AddContextCases() {
17 AddCases(ID: TC_ConsoleErr,
18 il: {
19 {"^%int-%int-%intT%int:%int:%int[-+]%int:%int$", MR_Default},
20 {"Running .*(/|\\\\)reporter_output_test(\\.exe)?$", MR_Next},
21 {"Run on \\(%int X %float MHz CPU s?\\)", MR_Next},
22 });
23 AddCases(ID: TC_JSONOut,
24 il: {{"^\\{", MR_Default},
25 {"\"context\":", MR_Next},
26 {"\"date\": \"", MR_Next},
27 {"\"host_name\":", MR_Next},
28 {"\"executable\": \".*(/|\\\\)reporter_output_test(\\.exe)?\",",
29 MR_Next},
30 {"\"num_cpus\": %int,$", MR_Next},
31 {"\"mhz_per_cpu\": %float,$", MR_Next},
32 {"\"caches\": \\[$", MR_Default}});
33 auto const& Info = benchmark::CPUInfo::Get();
34 auto const& Caches = Info.caches;
35 if (!Caches.empty()) {
36 AddCases(ID: TC_ConsoleErr, il: {{"CPU Caches:$", MR_Next}});
37 }
38 for (size_t I = 0; I < Caches.size(); ++I) {
39 std::string num_caches_str =
40 Caches[I].num_sharing != 0 ? " \\(x%int\\)$" : "$";
41 AddCases(ID: TC_ConsoleErr,
42 il: {{"L%int (Data|Instruction|Unified) %int KiB" + num_caches_str,
43 MR_Next}});
44 AddCases(ID: TC_JSONOut, il: {{"\\{$", MR_Next},
45 {"\"type\": \"", MR_Next},
46 {"\"level\": %int,$", MR_Next},
47 {"\"size\": %int,$", MR_Next},
48 {"\"num_sharing\": %int$", MR_Next},
49 {"}[,]{0,1}$", MR_Next}});
50 }
51 AddCases(ID: TC_JSONOut, il: {{"],$"}});
52 auto const& LoadAvg = Info.load_avg;
53 if (!LoadAvg.empty()) {
54 AddCases(ID: TC_ConsoleErr,
55 il: {{"Load Average: (%float, ){0,2}%float$", MR_Next}});
56 }
57 AddCases(ID: TC_JSONOut, il: {{"\"load_avg\": \\[(%float,?){0,3}],$", MR_Next}});
58 AddCases(ID: TC_JSONOut, il: {{"\"library_version\": \".*\",$", MR_Next}});
59 AddCases(ID: TC_JSONOut, il: {{"\"library_build_type\": \".*\",$", MR_Next}});
60 AddCases(ID: TC_JSONOut, il: {{"\"json_schema_version\": 1$", MR_Next}});
61 return 0;
62}
63int dummy_register = AddContextCases();
64ADD_CASES(TC_CSVOut, {{"%csv_header"}});
65
66// ========================================================================= //
67// ------------------------ Testing Basic Output --------------------------- //
68// ========================================================================= //
69
70void BM_basic(benchmark::State& state) {
71 for (auto _ : state) {
72 }
73}
74BENCHMARK(BM_basic);
75
76ADD_CASES(TC_ConsoleOut, {{"^BM_basic %console_report$"}});
77ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_basic\",$"},
78 {"\"family_index\": 0,$", MR_Next},
79 {"\"per_family_instance_index\": 0,$", MR_Next},
80 {"\"run_name\": \"BM_basic\",$", MR_Next},
81 {"\"run_type\": \"iteration\",$", MR_Next},
82 {"\"repetitions\": 1,$", MR_Next},
83 {"\"repetition_index\": 0,$", MR_Next},
84 {"\"threads\": 1,$", MR_Next},
85 {"\"iterations\": %int,$", MR_Next},
86 {"\"real_time\": %float,$", MR_Next},
87 {"\"cpu_time\": %float,$", MR_Next},
88 {"\"time_unit\": \"ns\"$", MR_Next},
89 {"}", MR_Next}});
90ADD_CASES(TC_CSVOut, {{"^\"BM_basic\",%csv_report$"}});
91
92// ========================================================================= //
93// ------------------------ Testing Bytes per Second Output ---------------- //
94// ========================================================================= //
95
96void BM_bytes_per_second(benchmark::State& state) {
97 for (auto _ : state) {
98 // This test requires a non-zero CPU time to avoid divide-by-zero
99 auto iterations = double(state.iterations()) * double(state.iterations());
100 benchmark::DoNotOptimize(value&: iterations);
101 }
102 state.SetBytesProcessed(1);
103}
104BENCHMARK(BM_bytes_per_second);
105
106ADD_CASES(TC_ConsoleOut, {{"^BM_bytes_per_second %console_report "
107 "bytes_per_second=%float[kM]{0,1}/s$"}});
108ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_bytes_per_second\",$"},
109 {"\"family_index\": 1,$", MR_Next},
110 {"\"per_family_instance_index\": 0,$", MR_Next},
111 {"\"run_name\": \"BM_bytes_per_second\",$", MR_Next},
112 {"\"run_type\": \"iteration\",$", MR_Next},
113 {"\"repetitions\": 1,$", MR_Next},
114 {"\"repetition_index\": 0,$", MR_Next},
115 {"\"threads\": 1,$", MR_Next},
116 {"\"iterations\": %int,$", MR_Next},
117 {"\"real_time\": %float,$", MR_Next},
118 {"\"cpu_time\": %float,$", MR_Next},
119 {"\"time_unit\": \"ns\",$", MR_Next},
120 {"\"bytes_per_second\": %float$", MR_Next},
121 {"}", MR_Next}});
122ADD_CASES(TC_CSVOut, {{"^\"BM_bytes_per_second\",%csv_bytes_report$"}});
123
124// ========================================================================= //
125// ------------------------ Testing Items per Second Output ---------------- //
126// ========================================================================= //
127
128void BM_items_per_second(benchmark::State& state) {
129 for (auto _ : state) {
130 // This test requires a non-zero CPU time to avoid divide-by-zero
131 auto iterations = double(state.iterations()) * double(state.iterations());
132 benchmark::DoNotOptimize(value&: iterations);
133 }
134 state.SetItemsProcessed(1);
135}
136BENCHMARK(BM_items_per_second);
137
138ADD_CASES(TC_ConsoleOut, {{"^BM_items_per_second %console_report "
139 "items_per_second=%float[kM]{0,1}/s$"}});
140ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_items_per_second\",$"},
141 {"\"family_index\": 2,$", MR_Next},
142 {"\"per_family_instance_index\": 0,$", MR_Next},
143 {"\"run_name\": \"BM_items_per_second\",$", MR_Next},
144 {"\"run_type\": \"iteration\",$", MR_Next},
145 {"\"repetitions\": 1,$", MR_Next},
146 {"\"repetition_index\": 0,$", MR_Next},
147 {"\"threads\": 1,$", MR_Next},
148 {"\"iterations\": %int,$", MR_Next},
149 {"\"real_time\": %float,$", MR_Next},
150 {"\"cpu_time\": %float,$", MR_Next},
151 {"\"time_unit\": \"ns\",$", MR_Next},
152 {"\"items_per_second\": %float$", MR_Next},
153 {"}", MR_Next}});
154ADD_CASES(TC_CSVOut, {{"^\"BM_items_per_second\",%csv_items_report$"}});
155
156// ========================================================================= //
157// ------------------------ Testing Label Output --------------------------- //
158// ========================================================================= //
159
160void BM_label(benchmark::State& state) {
161 for (auto _ : state) {
162 }
163 state.SetLabel("some label");
164}
165BENCHMARK(BM_label);
166
167ADD_CASES(TC_ConsoleOut, {{"^BM_label %console_report some label$"}});
168ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_label\",$"},
169 {"\"family_index\": 3,$", MR_Next},
170 {"\"per_family_instance_index\": 0,$", MR_Next},
171 {"\"run_name\": \"BM_label\",$", MR_Next},
172 {"\"run_type\": \"iteration\",$", MR_Next},
173 {"\"repetitions\": 1,$", MR_Next},
174 {"\"repetition_index\": 0,$", MR_Next},
175 {"\"threads\": 1,$", MR_Next},
176 {"\"iterations\": %int,$", MR_Next},
177 {"\"real_time\": %float,$", MR_Next},
178 {"\"cpu_time\": %float,$", MR_Next},
179 {"\"time_unit\": \"ns\",$", MR_Next},
180 {"\"label\": \"some label\"$", MR_Next},
181 {"}", MR_Next}});
182ADD_CASES(TC_CSVOut, {{"^\"BM_label\",%csv_label_report_begin\"some "
183 "label\"%csv_label_report_end$"}});
184
185// ========================================================================= //
186// ------------------------ Testing Time Label Output ---------------------- //
187// ========================================================================= //
188
189void BM_time_label_nanosecond(benchmark::State& state) {
190 for (auto _ : state) {
191 }
192}
193BENCHMARK(BM_time_label_nanosecond)->Unit(unit: benchmark::kNanosecond);
194
195ADD_CASES(TC_ConsoleOut, {{"^BM_time_label_nanosecond %console_report$"}});
196ADD_CASES(TC_JSONOut,
197 {{"\"name\": \"BM_time_label_nanosecond\",$"},
198 {"\"family_index\": 4,$", MR_Next},
199 {"\"per_family_instance_index\": 0,$", MR_Next},
200 {"\"run_name\": \"BM_time_label_nanosecond\",$", MR_Next},
201 {"\"run_type\": \"iteration\",$", MR_Next},
202 {"\"repetitions\": 1,$", MR_Next},
203 {"\"repetition_index\": 0,$", MR_Next},
204 {"\"threads\": 1,$", MR_Next},
205 {"\"iterations\": %int,$", MR_Next},
206 {"\"real_time\": %float,$", MR_Next},
207 {"\"cpu_time\": %float,$", MR_Next},
208 {"\"time_unit\": \"ns\"$", MR_Next},
209 {"}", MR_Next}});
210ADD_CASES(TC_CSVOut, {{"^\"BM_time_label_nanosecond\",%csv_report$"}});
211
212void BM_time_label_microsecond(benchmark::State& state) {
213 for (auto _ : state) {
214 }
215}
216BENCHMARK(BM_time_label_microsecond)->Unit(unit: benchmark::kMicrosecond);
217
218ADD_CASES(TC_ConsoleOut, {{"^BM_time_label_microsecond %console_us_report$"}});
219ADD_CASES(TC_JSONOut,
220 {{"\"name\": \"BM_time_label_microsecond\",$"},
221 {"\"family_index\": 5,$", MR_Next},
222 {"\"per_family_instance_index\": 0,$", MR_Next},
223 {"\"run_name\": \"BM_time_label_microsecond\",$", MR_Next},
224 {"\"run_type\": \"iteration\",$", MR_Next},
225 {"\"repetitions\": 1,$", MR_Next},
226 {"\"repetition_index\": 0,$", MR_Next},
227 {"\"threads\": 1,$", MR_Next},
228 {"\"iterations\": %int,$", MR_Next},
229 {"\"real_time\": %float,$", MR_Next},
230 {"\"cpu_time\": %float,$", MR_Next},
231 {"\"time_unit\": \"us\"$", MR_Next},
232 {"}", MR_Next}});
233ADD_CASES(TC_CSVOut, {{"^\"BM_time_label_microsecond\",%csv_us_report$"}});
234
235void BM_time_label_millisecond(benchmark::State& state) {
236 for (auto _ : state) {
237 }
238}
239BENCHMARK(BM_time_label_millisecond)->Unit(unit: benchmark::kMillisecond);
240
241ADD_CASES(TC_ConsoleOut, {{"^BM_time_label_millisecond %console_ms_report$"}});
242ADD_CASES(TC_JSONOut,
243 {{"\"name\": \"BM_time_label_millisecond\",$"},
244 {"\"family_index\": 6,$", MR_Next},
245 {"\"per_family_instance_index\": 0,$", MR_Next},
246 {"\"run_name\": \"BM_time_label_millisecond\",$", MR_Next},
247 {"\"run_type\": \"iteration\",$", MR_Next},
248 {"\"repetitions\": 1,$", MR_Next},
249 {"\"repetition_index\": 0,$", MR_Next},
250 {"\"threads\": 1,$", MR_Next},
251 {"\"iterations\": %int,$", MR_Next},
252 {"\"real_time\": %float,$", MR_Next},
253 {"\"cpu_time\": %float,$", MR_Next},
254 {"\"time_unit\": \"ms\"$", MR_Next},
255 {"}", MR_Next}});
256ADD_CASES(TC_CSVOut, {{"^\"BM_time_label_millisecond\",%csv_ms_report$"}});
257
258void BM_time_label_second(benchmark::State& state) {
259 for (auto _ : state) {
260 }
261}
262BENCHMARK(BM_time_label_second)->Unit(unit: benchmark::kSecond);
263
264ADD_CASES(TC_ConsoleOut, {{"^BM_time_label_second %console_s_report$"}});
265ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_time_label_second\",$"},
266 {"\"family_index\": 7,$", MR_Next},
267 {"\"per_family_instance_index\": 0,$", MR_Next},
268 {"\"run_name\": \"BM_time_label_second\",$", MR_Next},
269 {"\"run_type\": \"iteration\",$", MR_Next},
270 {"\"repetitions\": 1,$", MR_Next},
271 {"\"repetition_index\": 0,$", MR_Next},
272 {"\"threads\": 1,$", MR_Next},
273 {"\"iterations\": %int,$", MR_Next},
274 {"\"real_time\": %float,$", MR_Next},
275 {"\"cpu_time\": %float,$", MR_Next},
276 {"\"time_unit\": \"s\"$", MR_Next},
277 {"}", MR_Next}});
278ADD_CASES(TC_CSVOut, {{"^\"BM_time_label_second\",%csv_s_report$"}});
279
280// ========================================================================= //
281// ------------------------ Testing Error Output --------------------------- //
282// ========================================================================= //
283
284void BM_error(benchmark::State& state) {
285 state.SkipWithError(msg: "message");
286 for (auto _ : state) {
287 }
288}
289BENCHMARK(BM_error);
290ADD_CASES(TC_ConsoleOut, {{"^BM_error[ ]+ERROR OCCURRED: 'message'$"}});
291ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_error\",$"},
292 {"\"family_index\": 8,$", MR_Next},
293 {"\"per_family_instance_index\": 0,$", MR_Next},
294 {"\"run_name\": \"BM_error\",$", MR_Next},
295 {"\"run_type\": \"iteration\",$", MR_Next},
296 {"\"repetitions\": 1,$", MR_Next},
297 {"\"repetition_index\": 0,$", MR_Next},
298 {"\"threads\": 1,$", MR_Next},
299 {"\"error_occurred\": true,$", MR_Next},
300 {"\"error_message\": \"message\",$", MR_Next}});
301
302ADD_CASES(TC_CSVOut, {{"^\"BM_error\",,,,,,,,true,\"message\"$"}});
303
304// ========================================================================= //
305// ------------------------ Testing No Arg Name Output -----------------------
306// //
307// ========================================================================= //
308
309void BM_no_arg_name(benchmark::State& state) {
310 for (auto _ : state) {
311 }
312}
313BENCHMARK(BM_no_arg_name)->Arg(x: 3);
314ADD_CASES(TC_ConsoleOut, {{"^BM_no_arg_name/3 %console_report$"}});
315ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_no_arg_name/3\",$"},
316 {"\"family_index\": 9,$", MR_Next},
317 {"\"per_family_instance_index\": 0,$", MR_Next},
318 {"\"run_name\": \"BM_no_arg_name/3\",$", MR_Next},
319 {"\"run_type\": \"iteration\",$", MR_Next},
320 {"\"repetitions\": 1,$", MR_Next},
321 {"\"repetition_index\": 0,$", MR_Next},
322 {"\"threads\": 1,$", MR_Next}});
323ADD_CASES(TC_CSVOut, {{"^\"BM_no_arg_name/3\",%csv_report$"}});
324
325// ========================================================================= //
326// ------------------------ Testing Arg Name Output ------------------------ //
327// ========================================================================= //
328
329void BM_arg_name(benchmark::State& state) {
330 for (auto _ : state) {
331 }
332}
333BENCHMARK(BM_arg_name)->ArgName(name: "first")->Arg(x: 3);
334ADD_CASES(TC_ConsoleOut, {{"^BM_arg_name/first:3 %console_report$"}});
335ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_name/first:3\",$"},
336 {"\"family_index\": 10,$", MR_Next},
337 {"\"per_family_instance_index\": 0,$", MR_Next},
338 {"\"run_name\": \"BM_arg_name/first:3\",$", MR_Next},
339 {"\"run_type\": \"iteration\",$", MR_Next},
340 {"\"repetitions\": 1,$", MR_Next},
341 {"\"repetition_index\": 0,$", MR_Next},
342 {"\"threads\": 1,$", MR_Next}});
343ADD_CASES(TC_CSVOut, {{"^\"BM_arg_name/first:3\",%csv_report$"}});
344
345// ========================================================================= //
346// ------------------------ Testing Arg Names Output ----------------------- //
347// ========================================================================= //
348
349void BM_arg_names(benchmark::State& state) {
350 for (auto _ : state) {
351 }
352}
353BENCHMARK(BM_arg_names)->Args(args: {2, 5, 4})->ArgNames(names: {"first", "", "third"});
354ADD_CASES(TC_ConsoleOut,
355 {{"^BM_arg_names/first:2/5/third:4 %console_report$"}});
356ADD_CASES(TC_JSONOut,
357 {{"\"name\": \"BM_arg_names/first:2/5/third:4\",$"},
358 {"\"family_index\": 11,$", MR_Next},
359 {"\"per_family_instance_index\": 0,$", MR_Next},
360 {"\"run_name\": \"BM_arg_names/first:2/5/third:4\",$", MR_Next},
361 {"\"run_type\": \"iteration\",$", MR_Next},
362 {"\"repetitions\": 1,$", MR_Next},
363 {"\"repetition_index\": 0,$", MR_Next},
364 {"\"threads\": 1,$", MR_Next}});
365ADD_CASES(TC_CSVOut, {{"^\"BM_arg_names/first:2/5/third:4\",%csv_report$"}});
366
367// ========================================================================= //
368// ------------------------ Testing Name Output ---------------------------- //
369// ========================================================================= //
370
371void BM_name(benchmark::State& state) {
372 for (auto _ : state) {
373 }
374}
375BENCHMARK(BM_name)->Name(name: "BM_custom_name");
376
377ADD_CASES(TC_ConsoleOut, {{"^BM_custom_name %console_report$"}});
378ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_custom_name\",$"},
379 {"\"family_index\": 12,$", MR_Next},
380 {"\"per_family_instance_index\": 0,$", MR_Next},
381 {"\"run_name\": \"BM_custom_name\",$", MR_Next},
382 {"\"run_type\": \"iteration\",$", MR_Next},
383 {"\"repetitions\": 1,$", MR_Next},
384 {"\"repetition_index\": 0,$", MR_Next},
385 {"\"threads\": 1,$", MR_Next},
386 {"\"iterations\": %int,$", MR_Next},
387 {"\"real_time\": %float,$", MR_Next},
388 {"\"cpu_time\": %float,$", MR_Next},
389 {"\"time_unit\": \"ns\"$", MR_Next},
390 {"}", MR_Next}});
391ADD_CASES(TC_CSVOut, {{"^\"BM_custom_name\",%csv_report$"}});
392
393// ========================================================================= //
394// ------------------------ Testing Big Args Output ------------------------ //
395// ========================================================================= //
396
397void BM_BigArgs(benchmark::State& state) {
398 for (auto _ : state) {
399 }
400}
401BENCHMARK(BM_BigArgs)->RangeMultiplier(multiplier: 2)->Range(start: 1U << 30U, limit: 1U << 31U);
402ADD_CASES(TC_ConsoleOut, {{"^BM_BigArgs/1073741824 %console_report$"},
403 {"^BM_BigArgs/2147483648 %console_report$"}});
404
405// ========================================================================= //
406// ----------------------- Testing Complexity Output ----------------------- //
407// ========================================================================= //
408
409void BM_Complexity_O1(benchmark::State& state) {
410 for (auto _ : state) {
411 // This test requires a non-zero CPU time to avoid divide-by-zero
412 auto iterations = double(state.iterations()) * double(state.iterations());
413 benchmark::DoNotOptimize(value&: iterations);
414 }
415 state.SetComplexityN(state.range(pos: 0));
416}
417BENCHMARK(BM_Complexity_O1)->Range(start: 1, limit: 1 << 18)->Complexity(complexity: benchmark::o1);
418SET_SUBSTITUTIONS({{"%bigOStr", "[ ]* %float \\([0-9]+\\)"},
419 {"%RMS", "[ ]*[0-9]+ %"}});
420ADD_CASES(TC_ConsoleOut, {{"^BM_Complexity_O1_BigO %bigOStr %bigOStr[ ]*$"},
421 {"^BM_Complexity_O1_RMS %RMS %RMS[ ]*$"}});
422
423// ========================================================================= //
424// ----------------------- Testing Aggregate Output ------------------------ //
425// ========================================================================= //
426
427// Test that non-aggregate data is printed by default
428void BM_Repeat(benchmark::State& state) {
429 for (auto _ : state) {
430 }
431}
432// need two repetitions min to be able to output any aggregate output
433BENCHMARK(BM_Repeat)->Repetitions(n: 2);
434ADD_CASES(TC_ConsoleOut,
435 {{"^BM_Repeat/repeats:2 %console_report$"},
436 {"^BM_Repeat/repeats:2 %console_report$"},
437 {"^BM_Repeat/repeats:2_mean %console_time_only_report [ ]*2$"},
438 {"^BM_Repeat/repeats:2_median %console_time_only_report [ ]*2$"},
439 {"^BM_Repeat/repeats:2_stddev %console_time_only_report [ ]*2$"}});
440ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:2\",$"},
441 {"\"family_index\": 15,$", MR_Next},
442 {"\"per_family_instance_index\": 0,$", MR_Next},
443 {"\"run_name\": \"BM_Repeat/repeats:2\"", MR_Next},
444 {"\"run_type\": \"iteration\",$", MR_Next},
445 {"\"repetitions\": 2,$", MR_Next},
446 {"\"repetition_index\": 0,$", MR_Next},
447 {"\"threads\": 1,$", MR_Next},
448 {"\"name\": \"BM_Repeat/repeats:2\",$"},
449 {"\"family_index\": 15,$", MR_Next},
450 {"\"per_family_instance_index\": 0,$", MR_Next},
451 {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
452 {"\"run_type\": \"iteration\",$", MR_Next},
453 {"\"repetitions\": 2,$", MR_Next},
454 {"\"repetition_index\": 1,$", MR_Next},
455 {"\"threads\": 1,$", MR_Next},
456 {"\"name\": \"BM_Repeat/repeats:2_mean\",$"},
457 {"\"family_index\": 15,$", MR_Next},
458 {"\"per_family_instance_index\": 0,$", MR_Next},
459 {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
460 {"\"run_type\": \"aggregate\",$", MR_Next},
461 {"\"repetitions\": 2,$", MR_Next},
462 {"\"threads\": 1,$", MR_Next},
463 {"\"aggregate_name\": \"mean\",$", MR_Next},
464 {"\"aggregate_unit\": \"time\",$", MR_Next},
465 {"\"iterations\": 2,$", MR_Next},
466 {"\"name\": \"BM_Repeat/repeats:2_median\",$"},
467 {"\"family_index\": 15,$", MR_Next},
468 {"\"per_family_instance_index\": 0,$", MR_Next},
469 {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
470 {"\"run_type\": \"aggregate\",$", MR_Next},
471 {"\"repetitions\": 2,$", MR_Next},
472 {"\"threads\": 1,$", MR_Next},
473 {"\"aggregate_name\": \"median\",$", MR_Next},
474 {"\"aggregate_unit\": \"time\",$", MR_Next},
475 {"\"iterations\": 2,$", MR_Next},
476 {"\"name\": \"BM_Repeat/repeats:2_stddev\",$"},
477 {"\"family_index\": 15,$", MR_Next},
478 {"\"per_family_instance_index\": 0,$", MR_Next},
479 {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
480 {"\"run_type\": \"aggregate\",$", MR_Next},
481 {"\"repetitions\": 2,$", MR_Next},
482 {"\"threads\": 1,$", MR_Next},
483 {"\"aggregate_name\": \"stddev\",$", MR_Next},
484 {"\"aggregate_unit\": \"time\",$", MR_Next},
485 {"\"iterations\": 2,$", MR_Next}});
486ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:2\",%csv_report$"},
487 {"^\"BM_Repeat/repeats:2\",%csv_report$"},
488 {"^\"BM_Repeat/repeats:2_mean\",%csv_report$"},
489 {"^\"BM_Repeat/repeats:2_median\",%csv_report$"},
490 {"^\"BM_Repeat/repeats:2_stddev\",%csv_report$"}});
491// but for two repetitions, mean and median is the same, so let's repeat..
492BENCHMARK(BM_Repeat)->Repetitions(n: 3);
493ADD_CASES(TC_ConsoleOut,
494 {{"^BM_Repeat/repeats:3 %console_report$"},
495 {"^BM_Repeat/repeats:3 %console_report$"},
496 {"^BM_Repeat/repeats:3 %console_report$"},
497 {"^BM_Repeat/repeats:3_mean %console_time_only_report [ ]*3$"},
498 {"^BM_Repeat/repeats:3_median %console_time_only_report [ ]*3$"},
499 {"^BM_Repeat/repeats:3_stddev %console_time_only_report [ ]*3$"}});
500ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"},
501 {"\"family_index\": 16,$", MR_Next},
502 {"\"per_family_instance_index\": 0,$", MR_Next},
503 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
504 {"\"run_type\": \"iteration\",$", MR_Next},
505 {"\"repetitions\": 3,$", MR_Next},
506 {"\"repetition_index\": 0,$", MR_Next},
507 {"\"threads\": 1,$", MR_Next},
508 {"\"name\": \"BM_Repeat/repeats:3\",$"},
509 {"\"family_index\": 16,$", MR_Next},
510 {"\"per_family_instance_index\": 0,$", MR_Next},
511 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
512 {"\"run_type\": \"iteration\",$", MR_Next},
513 {"\"repetitions\": 3,$", MR_Next},
514 {"\"repetition_index\": 1,$", MR_Next},
515 {"\"threads\": 1,$", MR_Next},
516 {"\"name\": \"BM_Repeat/repeats:3\",$"},
517 {"\"family_index\": 16,$", MR_Next},
518 {"\"per_family_instance_index\": 0,$", MR_Next},
519 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
520 {"\"run_type\": \"iteration\",$", MR_Next},
521 {"\"repetitions\": 3,$", MR_Next},
522 {"\"repetition_index\": 2,$", MR_Next},
523 {"\"threads\": 1,$", MR_Next},
524 {"\"name\": \"BM_Repeat/repeats:3_mean\",$"},
525 {"\"family_index\": 16,$", MR_Next},
526 {"\"per_family_instance_index\": 0,$", MR_Next},
527 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
528 {"\"run_type\": \"aggregate\",$", MR_Next},
529 {"\"repetitions\": 3,$", MR_Next},
530 {"\"threads\": 1,$", MR_Next},
531 {"\"aggregate_name\": \"mean\",$", MR_Next},
532 {"\"aggregate_unit\": \"time\",$", MR_Next},
533 {"\"iterations\": 3,$", MR_Next},
534 {"\"name\": \"BM_Repeat/repeats:3_median\",$"},
535 {"\"family_index\": 16,$", MR_Next},
536 {"\"per_family_instance_index\": 0,$", MR_Next},
537 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
538 {"\"run_type\": \"aggregate\",$", MR_Next},
539 {"\"repetitions\": 3,$", MR_Next},
540 {"\"threads\": 1,$", MR_Next},
541 {"\"aggregate_name\": \"median\",$", MR_Next},
542 {"\"aggregate_unit\": \"time\",$", MR_Next},
543 {"\"iterations\": 3,$", MR_Next},
544 {"\"name\": \"BM_Repeat/repeats:3_stddev\",$"},
545 {"\"family_index\": 16,$", MR_Next},
546 {"\"per_family_instance_index\": 0,$", MR_Next},
547 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
548 {"\"run_type\": \"aggregate\",$", MR_Next},
549 {"\"repetitions\": 3,$", MR_Next},
550 {"\"threads\": 1,$", MR_Next},
551 {"\"aggregate_name\": \"stddev\",$", MR_Next},
552 {"\"aggregate_unit\": \"time\",$", MR_Next},
553 {"\"iterations\": 3,$", MR_Next}});
554ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:3\",%csv_report$"},
555 {"^\"BM_Repeat/repeats:3\",%csv_report$"},
556 {"^\"BM_Repeat/repeats:3\",%csv_report$"},
557 {"^\"BM_Repeat/repeats:3_mean\",%csv_report$"},
558 {"^\"BM_Repeat/repeats:3_median\",%csv_report$"},
559 {"^\"BM_Repeat/repeats:3_stddev\",%csv_report$"}});
560// median differs between even/odd number of repetitions, so just to be sure
561BENCHMARK(BM_Repeat)->Repetitions(n: 4);
562ADD_CASES(TC_ConsoleOut,
563 {{"^BM_Repeat/repeats:4 %console_report$"},
564 {"^BM_Repeat/repeats:4 %console_report$"},
565 {"^BM_Repeat/repeats:4 %console_report$"},
566 {"^BM_Repeat/repeats:4 %console_report$"},
567 {"^BM_Repeat/repeats:4_mean %console_time_only_report [ ]*4$"},
568 {"^BM_Repeat/repeats:4_median %console_time_only_report [ ]*4$"},
569 {"^BM_Repeat/repeats:4_stddev %console_time_only_report [ ]*4$"}});
570ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:4\",$"},
571 {"\"family_index\": 17,$", MR_Next},
572 {"\"per_family_instance_index\": 0,$", MR_Next},
573 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
574 {"\"run_type\": \"iteration\",$", MR_Next},
575 {"\"repetitions\": 4,$", MR_Next},
576 {"\"repetition_index\": 0,$", MR_Next},
577 {"\"threads\": 1,$", MR_Next},
578 {"\"name\": \"BM_Repeat/repeats:4\",$"},
579 {"\"family_index\": 17,$", MR_Next},
580 {"\"per_family_instance_index\": 0,$", MR_Next},
581 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
582 {"\"run_type\": \"iteration\",$", MR_Next},
583 {"\"repetitions\": 4,$", MR_Next},
584 {"\"repetition_index\": 1,$", MR_Next},
585 {"\"threads\": 1,$", MR_Next},
586 {"\"name\": \"BM_Repeat/repeats:4\",$"},
587 {"\"family_index\": 17,$", MR_Next},
588 {"\"per_family_instance_index\": 0,$", MR_Next},
589 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
590 {"\"run_type\": \"iteration\",$", MR_Next},
591 {"\"repetitions\": 4,$", MR_Next},
592 {"\"repetition_index\": 2,$", MR_Next},
593 {"\"threads\": 1,$", MR_Next},
594 {"\"name\": \"BM_Repeat/repeats:4\",$"},
595 {"\"family_index\": 17,$", MR_Next},
596 {"\"per_family_instance_index\": 0,$", MR_Next},
597 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
598 {"\"run_type\": \"iteration\",$", MR_Next},
599 {"\"repetitions\": 4,$", MR_Next},
600 {"\"repetition_index\": 3,$", MR_Next},
601 {"\"threads\": 1,$", MR_Next},
602 {"\"name\": \"BM_Repeat/repeats:4_mean\",$"},
603 {"\"family_index\": 17,$", MR_Next},
604 {"\"per_family_instance_index\": 0,$", MR_Next},
605 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
606 {"\"run_type\": \"aggregate\",$", MR_Next},
607 {"\"repetitions\": 4,$", MR_Next},
608 {"\"threads\": 1,$", MR_Next},
609 {"\"aggregate_name\": \"mean\",$", MR_Next},
610 {"\"aggregate_unit\": \"time\",$", MR_Next},
611 {"\"iterations\": 4,$", MR_Next},
612 {"\"name\": \"BM_Repeat/repeats:4_median\",$"},
613 {"\"family_index\": 17,$", MR_Next},
614 {"\"per_family_instance_index\": 0,$", MR_Next},
615 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
616 {"\"run_type\": \"aggregate\",$", MR_Next},
617 {"\"repetitions\": 4,$", MR_Next},
618 {"\"threads\": 1,$", MR_Next},
619 {"\"aggregate_name\": \"median\",$", MR_Next},
620 {"\"aggregate_unit\": \"time\",$", MR_Next},
621 {"\"iterations\": 4,$", MR_Next},
622 {"\"name\": \"BM_Repeat/repeats:4_stddev\",$"},
623 {"\"family_index\": 17,$", MR_Next},
624 {"\"per_family_instance_index\": 0,$", MR_Next},
625 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
626 {"\"run_type\": \"aggregate\",$", MR_Next},
627 {"\"repetitions\": 4,$", MR_Next},
628 {"\"threads\": 1,$", MR_Next},
629 {"\"aggregate_name\": \"stddev\",$", MR_Next},
630 {"\"aggregate_unit\": \"time\",$", MR_Next},
631 {"\"iterations\": 4,$", MR_Next}});
632ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:4\",%csv_report$"},
633 {"^\"BM_Repeat/repeats:4\",%csv_report$"},
634 {"^\"BM_Repeat/repeats:4\",%csv_report$"},
635 {"^\"BM_Repeat/repeats:4\",%csv_report$"},
636 {"^\"BM_Repeat/repeats:4_mean\",%csv_report$"},
637 {"^\"BM_Repeat/repeats:4_median\",%csv_report$"},
638 {"^\"BM_Repeat/repeats:4_stddev\",%csv_report$"}});
639
640// Test that a non-repeated test still prints non-aggregate results even when
641// only-aggregate reports have been requested
642void BM_RepeatOnce(benchmark::State& state) {
643 for (auto _ : state) {
644 }
645}
646BENCHMARK(BM_RepeatOnce)->Repetitions(n: 1)->ReportAggregatesOnly();
647ADD_CASES(TC_ConsoleOut, {{"^BM_RepeatOnce/repeats:1 %console_report$"}});
648ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_RepeatOnce/repeats:1\",$"},
649 {"\"family_index\": 18,$", MR_Next},
650 {"\"per_family_instance_index\": 0,$", MR_Next},
651 {"\"run_name\": \"BM_RepeatOnce/repeats:1\",$", MR_Next},
652 {"\"run_type\": \"iteration\",$", MR_Next},
653 {"\"repetitions\": 1,$", MR_Next},
654 {"\"repetition_index\": 0,$", MR_Next},
655 {"\"threads\": 1,$", MR_Next}});
656ADD_CASES(TC_CSVOut, {{"^\"BM_RepeatOnce/repeats:1\",%csv_report$"}});
657
658// Test that non-aggregate data is not reported
659void BM_SummaryRepeat(benchmark::State& state) {
660 for (auto _ : state) {
661 }
662}
663BENCHMARK(BM_SummaryRepeat)->Repetitions(n: 3)->ReportAggregatesOnly();
664ADD_CASES(
665 TC_ConsoleOut,
666 {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
667 {"^BM_SummaryRepeat/repeats:3_mean %console_time_only_report [ ]*3$"},
668 {"^BM_SummaryRepeat/repeats:3_median %console_time_only_report [ ]*3$"},
669 {"^BM_SummaryRepeat/repeats:3_stddev %console_time_only_report [ ]*3$"}});
670ADD_CASES(TC_JSONOut,
671 {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
672 {"\"name\": \"BM_SummaryRepeat/repeats:3_mean\",$"},
673 {"\"family_index\": 19,$", MR_Next},
674 {"\"per_family_instance_index\": 0,$", MR_Next},
675 {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next},
676 {"\"run_type\": \"aggregate\",$", MR_Next},
677 {"\"repetitions\": 3,$", MR_Next},
678 {"\"threads\": 1,$", MR_Next},
679 {"\"aggregate_name\": \"mean\",$", MR_Next},
680 {"\"aggregate_unit\": \"time\",$", MR_Next},
681 {"\"iterations\": 3,$", MR_Next},
682 {"\"name\": \"BM_SummaryRepeat/repeats:3_median\",$"},
683 {"\"family_index\": 19,$", MR_Next},
684 {"\"per_family_instance_index\": 0,$", MR_Next},
685 {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next},
686 {"\"run_type\": \"aggregate\",$", MR_Next},
687 {"\"repetitions\": 3,$", MR_Next},
688 {"\"threads\": 1,$", MR_Next},
689 {"\"aggregate_name\": \"median\",$", MR_Next},
690 {"\"aggregate_unit\": \"time\",$", MR_Next},
691 {"\"iterations\": 3,$", MR_Next},
692 {"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\",$"},
693 {"\"family_index\": 19,$", MR_Next},
694 {"\"per_family_instance_index\": 0,$", MR_Next},
695 {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next},
696 {"\"run_type\": \"aggregate\",$", MR_Next},
697 {"\"repetitions\": 3,$", MR_Next},
698 {"\"threads\": 1,$", MR_Next},
699 {"\"aggregate_name\": \"stddev\",$", MR_Next},
700 {"\"aggregate_unit\": \"time\",$", MR_Next},
701 {"\"iterations\": 3,$", MR_Next}});
702ADD_CASES(TC_CSVOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
703 {"^\"BM_SummaryRepeat/repeats:3_mean\",%csv_report$"},
704 {"^\"BM_SummaryRepeat/repeats:3_median\",%csv_report$"},
705 {"^\"BM_SummaryRepeat/repeats:3_stddev\",%csv_report$"}});
706
707// Test that non-aggregate data is not displayed.
708// NOTE: this test is kinda bad. we are only testing the display output.
709// But we don't check that the file output still contains everything...
710void BM_SummaryDisplay(benchmark::State& state) {
711 for (auto _ : state) {
712 }
713}
714BENCHMARK(BM_SummaryDisplay)->Repetitions(n: 2)->DisplayAggregatesOnly();
715ADD_CASES(
716 TC_ConsoleOut,
717 {{".*BM_SummaryDisplay/repeats:2 ", MR_Not},
718 {"^BM_SummaryDisplay/repeats:2_mean %console_time_only_report [ ]*2$"},
719 {"^BM_SummaryDisplay/repeats:2_median %console_time_only_report [ ]*2$"},
720 {"^BM_SummaryDisplay/repeats:2_stddev %console_time_only_report [ ]*2$"}});
721ADD_CASES(TC_JSONOut,
722 {{".*BM_SummaryDisplay/repeats:2 ", MR_Not},
723 {"\"name\": \"BM_SummaryDisplay/repeats:2_mean\",$"},
724 {"\"family_index\": 20,$", MR_Next},
725 {"\"per_family_instance_index\": 0,$", MR_Next},
726 {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next},
727 {"\"run_type\": \"aggregate\",$", MR_Next},
728 {"\"repetitions\": 2,$", MR_Next},
729 {"\"threads\": 1,$", MR_Next},
730 {"\"aggregate_name\": \"mean\",$", MR_Next},
731 {"\"aggregate_unit\": \"time\",$", MR_Next},
732 {"\"iterations\": 2,$", MR_Next},
733 {"\"name\": \"BM_SummaryDisplay/repeats:2_median\",$"},
734 {"\"family_index\": 20,$", MR_Next},
735 {"\"per_family_instance_index\": 0,$", MR_Next},
736 {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next},
737 {"\"run_type\": \"aggregate\",$", MR_Next},
738 {"\"repetitions\": 2,$", MR_Next},
739 {"\"threads\": 1,$", MR_Next},
740 {"\"aggregate_name\": \"median\",$", MR_Next},
741 {"\"aggregate_unit\": \"time\",$", MR_Next},
742 {"\"iterations\": 2,$", MR_Next},
743 {"\"name\": \"BM_SummaryDisplay/repeats:2_stddev\",$"},
744 {"\"family_index\": 20,$", MR_Next},
745 {"\"per_family_instance_index\": 0,$", MR_Next},
746 {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next},
747 {"\"run_type\": \"aggregate\",$", MR_Next},
748 {"\"repetitions\": 2,$", MR_Next},
749 {"\"threads\": 1,$", MR_Next},
750 {"\"aggregate_name\": \"stddev\",$", MR_Next},
751 {"\"aggregate_unit\": \"time\",$", MR_Next},
752 {"\"iterations\": 2,$", MR_Next}});
753ADD_CASES(TC_CSVOut,
754 {{".*BM_SummaryDisplay/repeats:2 ", MR_Not},
755 {"^\"BM_SummaryDisplay/repeats:2_mean\",%csv_report$"},
756 {"^\"BM_SummaryDisplay/repeats:2_median\",%csv_report$"},
757 {"^\"BM_SummaryDisplay/repeats:2_stddev\",%csv_report$"}});
758
759// Test repeats with custom time unit.
760void BM_RepeatTimeUnit(benchmark::State& state) {
761 for (auto _ : state) {
762 }
763}
764BENCHMARK(BM_RepeatTimeUnit)
765 ->Repetitions(n: 3)
766 ->ReportAggregatesOnly()
767 ->Unit(unit: benchmark::kMicrosecond);
768ADD_CASES(
769 TC_ConsoleOut,
770 {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
771 {"^BM_RepeatTimeUnit/repeats:3_mean %console_us_time_only_report [ ]*3$"},
772 {"^BM_RepeatTimeUnit/repeats:3_median %console_us_time_only_report [ "
773 "]*3$"},
774 {"^BM_RepeatTimeUnit/repeats:3_stddev %console_us_time_only_report [ "
775 "]*3$"}});
776ADD_CASES(TC_JSONOut,
777 {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
778 {"\"name\": \"BM_RepeatTimeUnit/repeats:3_mean\",$"},
779 {"\"family_index\": 21,$", MR_Next},
780 {"\"per_family_instance_index\": 0,$", MR_Next},
781 {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next},
782 {"\"run_type\": \"aggregate\",$", MR_Next},
783 {"\"repetitions\": 3,$", MR_Next},
784 {"\"threads\": 1,$", MR_Next},
785 {"\"aggregate_name\": \"mean\",$", MR_Next},
786 {"\"aggregate_unit\": \"time\",$", MR_Next},
787 {"\"iterations\": 3,$", MR_Next},
788 {"\"time_unit\": \"us\",?$"},
789 {"\"name\": \"BM_RepeatTimeUnit/repeats:3_median\",$"},
790 {"\"family_index\": 21,$", MR_Next},
791 {"\"per_family_instance_index\": 0,$", MR_Next},
792 {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next},
793 {"\"run_type\": \"aggregate\",$", MR_Next},
794 {"\"repetitions\": 3,$", MR_Next},
795 {"\"threads\": 1,$", MR_Next},
796 {"\"aggregate_name\": \"median\",$", MR_Next},
797 {"\"aggregate_unit\": \"time\",$", MR_Next},
798 {"\"iterations\": 3,$", MR_Next},
799 {"\"time_unit\": \"us\",?$"},
800 {"\"name\": \"BM_RepeatTimeUnit/repeats:3_stddev\",$"},
801 {"\"family_index\": 21,$", MR_Next},
802 {"\"per_family_instance_index\": 0,$", MR_Next},
803 {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next},
804 {"\"run_type\": \"aggregate\",$", MR_Next},
805 {"\"repetitions\": 3,$", MR_Next},
806 {"\"threads\": 1,$", MR_Next},
807 {"\"aggregate_name\": \"stddev\",$", MR_Next},
808 {"\"aggregate_unit\": \"time\",$", MR_Next},
809 {"\"iterations\": 3,$", MR_Next},
810 {"\"time_unit\": \"us\",?$"}});
811ADD_CASES(TC_CSVOut,
812 {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
813 {"^\"BM_RepeatTimeUnit/repeats:3_mean\",%csv_us_report$"},
814 {"^\"BM_RepeatTimeUnit/repeats:3_median\",%csv_us_report$"},
815 {"^\"BM_RepeatTimeUnit/repeats:3_stddev\",%csv_us_report$"}});
816
817// ========================================================================= //
818// -------------------- Testing user-provided statistics ------------------- //
819// ========================================================================= //
820
821const auto UserStatistics = [](const std::vector<double>& v) {
822 return v.back();
823};
824void BM_UserStats(benchmark::State& state) {
825 for (auto _ : state) {
826 state.SetIterationTime(150 / 10e8);
827 }
828}
829// clang-format off
830BENCHMARK(BM_UserStats)
831 ->Repetitions(n: 3)
832 ->Iterations(n: 5)
833 ->UseManualTime()
834 ->ComputeStatistics(name: "", statistics: UserStatistics);
835// clang-format on
836
837// check that user-provided stats is calculated, and is after the default-ones
838// empty string as name is intentional, it would sort before anything else
839ADD_CASES(TC_ConsoleOut, {{"^BM_UserStats/iterations:5/repeats:3/manual_time [ "
840 "]* 150 ns %time [ ]*5$"},
841 {"^BM_UserStats/iterations:5/repeats:3/manual_time [ "
842 "]* 150 ns %time [ ]*5$"},
843 {"^BM_UserStats/iterations:5/repeats:3/manual_time [ "
844 "]* 150 ns %time [ ]*5$"},
845 {"^BM_UserStats/iterations:5/repeats:3/"
846 "manual_time_mean [ ]* 150 ns %time [ ]*3$"},
847 {"^BM_UserStats/iterations:5/repeats:3/"
848 "manual_time_median [ ]* 150 ns %time [ ]*3$"},
849 {"^BM_UserStats/iterations:5/repeats:3/"
850 "manual_time_stddev [ ]* 0.000 ns %time [ ]*3$"},
851 {"^BM_UserStats/iterations:5/repeats:3/manual_time_ "
852 "[ ]* 150 ns %time [ ]*3$"}});
853ADD_CASES(
854 TC_JSONOut,
855 {{"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"},
856 {"\"family_index\": 22,$", MR_Next},
857 {"\"per_family_instance_index\": 0,$", MR_Next},
858 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
859 MR_Next},
860 {"\"run_type\": \"iteration\",$", MR_Next},
861 {"\"repetitions\": 3,$", MR_Next},
862 {"\"repetition_index\": 0,$", MR_Next},
863 {"\"threads\": 1,$", MR_Next},
864 {"\"iterations\": 5,$", MR_Next},
865 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
866 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"},
867 {"\"family_index\": 22,$", MR_Next},
868 {"\"per_family_instance_index\": 0,$", MR_Next},
869 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
870 MR_Next},
871 {"\"run_type\": \"iteration\",$", MR_Next},
872 {"\"repetitions\": 3,$", MR_Next},
873 {"\"repetition_index\": 1,$", MR_Next},
874 {"\"threads\": 1,$", MR_Next},
875 {"\"iterations\": 5,$", MR_Next},
876 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
877 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"},
878 {"\"family_index\": 22,$", MR_Next},
879 {"\"per_family_instance_index\": 0,$", MR_Next},
880 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
881 MR_Next},
882 {"\"run_type\": \"iteration\",$", MR_Next},
883 {"\"repetitions\": 3,$", MR_Next},
884 {"\"repetition_index\": 2,$", MR_Next},
885 {"\"threads\": 1,$", MR_Next},
886 {"\"iterations\": 5,$", MR_Next},
887 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
888 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_mean\",$"},
889 {"\"family_index\": 22,$", MR_Next},
890 {"\"per_family_instance_index\": 0,$", MR_Next},
891 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
892 MR_Next},
893 {"\"run_type\": \"aggregate\",$", MR_Next},
894 {"\"repetitions\": 3,$", MR_Next},
895 {"\"threads\": 1,$", MR_Next},
896 {"\"aggregate_name\": \"mean\",$", MR_Next},
897 {"\"aggregate_unit\": \"time\",$", MR_Next},
898 {"\"iterations\": 3,$", MR_Next},
899 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
900 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_median\",$"},
901 {"\"family_index\": 22,$", MR_Next},
902 {"\"per_family_instance_index\": 0,$", MR_Next},
903 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
904 MR_Next},
905 {"\"run_type\": \"aggregate\",$", MR_Next},
906 {"\"repetitions\": 3,$", MR_Next},
907 {"\"threads\": 1,$", MR_Next},
908 {"\"aggregate_name\": \"median\",$", MR_Next},
909 {"\"aggregate_unit\": \"time\",$", MR_Next},
910 {"\"iterations\": 3,$", MR_Next},
911 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
912 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_stddev\",$"},
913 {"\"family_index\": 22,$", MR_Next},
914 {"\"per_family_instance_index\": 0,$", MR_Next},
915 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
916 MR_Next},
917 {"\"run_type\": \"aggregate\",$", MR_Next},
918 {"\"repetitions\": 3,$", MR_Next},
919 {"\"threads\": 1,$", MR_Next},
920 {"\"aggregate_name\": \"stddev\",$", MR_Next},
921 {"\"aggregate_unit\": \"time\",$", MR_Next},
922 {"\"iterations\": 3,$", MR_Next},
923 {"\"real_time\": %float,$", MR_Next},
924 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_\",$"},
925 {"\"family_index\": 22,$", MR_Next},
926 {"\"per_family_instance_index\": 0,$", MR_Next},
927 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
928 MR_Next},
929 {"\"run_type\": \"aggregate\",$", MR_Next},
930 {"\"repetitions\": 3,$", MR_Next},
931 {"\"threads\": 1,$", MR_Next},
932 {"\"aggregate_name\": \"\",$", MR_Next},
933 {"\"aggregate_unit\": \"time\",$", MR_Next},
934 {"\"iterations\": 3,$", MR_Next},
935 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}});
936ADD_CASES(
937 TC_CSVOut,
938 {{"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"},
939 {"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"},
940 {"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"},
941 {"^\"BM_UserStats/iterations:5/repeats:3/manual_time_mean\",%csv_report$"},
942 {"^\"BM_UserStats/iterations:5/repeats:3/"
943 "manual_time_median\",%csv_report$"},
944 {"^\"BM_UserStats/iterations:5/repeats:3/"
945 "manual_time_stddev\",%csv_report$"},
946 {"^\"BM_UserStats/iterations:5/repeats:3/manual_time_\",%csv_report$"}});
947
948// ========================================================================= //
949// ------------- Testing relative standard deviation statistics ------------ //
950// ========================================================================= //
951
952const auto UserPercentStatistics = [](const std::vector<double>&) {
953 return 1. / 100.;
954};
955void BM_UserPercentStats(benchmark::State& state) {
956 for (auto _ : state) {
957 state.SetIterationTime(150 / 10e8);
958 }
959}
960// clang-format off
961BENCHMARK(BM_UserPercentStats)
962 ->Repetitions(n: 3)
963 ->Iterations(n: 5)
964 ->UseManualTime()
965 ->Unit(unit: benchmark::TimeUnit::kNanosecond)
966 ->ComputeStatistics(name: "", statistics: UserPercentStatistics, unit: benchmark::StatisticUnit::kPercentage);
967// clang-format on
968
969// check that UserPercent-provided stats is calculated, and is after the
970// default-ones empty string as name is intentional, it would sort before
971// anything else
972ADD_CASES(TC_ConsoleOut,
973 {{"^BM_UserPercentStats/iterations:5/repeats:3/manual_time [ "
974 "]* 150 ns %time [ ]*5$"},
975 {"^BM_UserPercentStats/iterations:5/repeats:3/manual_time [ "
976 "]* 150 ns %time [ ]*5$"},
977 {"^BM_UserPercentStats/iterations:5/repeats:3/manual_time [ "
978 "]* 150 ns %time [ ]*5$"},
979 {"^BM_UserPercentStats/iterations:5/repeats:3/"
980 "manual_time_mean [ ]* 150 ns %time [ ]*3$"},
981 {"^BM_UserPercentStats/iterations:5/repeats:3/"
982 "manual_time_median [ ]* 150 ns %time [ ]*3$"},
983 {"^BM_UserPercentStats/iterations:5/repeats:3/"
984 "manual_time_stddev [ ]* 0.000 ns %time [ ]*3$"},
985 {"^BM_UserPercentStats/iterations:5/repeats:3/manual_time_ "
986 "[ ]* 1.00 % [ ]* 1.00 %[ ]*3$"}});
987ADD_CASES(
988 TC_JSONOut,
989 {{"\"name\": \"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$"},
990 {"\"family_index\": 23,$", MR_Next},
991 {"\"per_family_instance_index\": 0,$", MR_Next},
992 {"\"run_name\": "
993 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
994 MR_Next},
995 {"\"run_type\": \"iteration\",$", MR_Next},
996 {"\"repetitions\": 3,$", MR_Next},
997 {"\"repetition_index\": 0,$", MR_Next},
998 {"\"threads\": 1,$", MR_Next},
999 {"\"iterations\": 5,$", MR_Next},
1000 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
1001 {"\"name\": \"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$"},
1002 {"\"family_index\": 23,$", MR_Next},
1003 {"\"per_family_instance_index\": 0,$", MR_Next},
1004 {"\"run_name\": "
1005 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
1006 MR_Next},
1007 {"\"run_type\": \"iteration\",$", MR_Next},
1008 {"\"repetitions\": 3,$", MR_Next},
1009 {"\"repetition_index\": 1,$", MR_Next},
1010 {"\"threads\": 1,$", MR_Next},
1011 {"\"iterations\": 5,$", MR_Next},
1012 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
1013 {"\"name\": \"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$"},
1014 {"\"family_index\": 23,$", MR_Next},
1015 {"\"per_family_instance_index\": 0,$", MR_Next},
1016 {"\"run_name\": "
1017 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
1018 MR_Next},
1019 {"\"run_type\": \"iteration\",$", MR_Next},
1020 {"\"repetitions\": 3,$", MR_Next},
1021 {"\"repetition_index\": 2,$", MR_Next},
1022 {"\"threads\": 1,$", MR_Next},
1023 {"\"iterations\": 5,$", MR_Next},
1024 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
1025 {"\"name\": "
1026 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time_mean\",$"},
1027 {"\"family_index\": 23,$", MR_Next},
1028 {"\"per_family_instance_index\": 0,$", MR_Next},
1029 {"\"run_name\": "
1030 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
1031 MR_Next},
1032 {"\"run_type\": \"aggregate\",$", MR_Next},
1033 {"\"repetitions\": 3,$", MR_Next},
1034 {"\"threads\": 1,$", MR_Next},
1035 {"\"aggregate_name\": \"mean\",$", MR_Next},
1036 {"\"aggregate_unit\": \"time\",$", MR_Next},
1037 {"\"iterations\": 3,$", MR_Next},
1038 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
1039 {"\"name\": "
1040 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time_median\",$"},
1041 {"\"family_index\": 23,$", MR_Next},
1042 {"\"per_family_instance_index\": 0,$", MR_Next},
1043 {"\"run_name\": "
1044 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
1045 MR_Next},
1046 {"\"run_type\": \"aggregate\",$", MR_Next},
1047 {"\"repetitions\": 3,$", MR_Next},
1048 {"\"threads\": 1,$", MR_Next},
1049 {"\"aggregate_name\": \"median\",$", MR_Next},
1050 {"\"aggregate_unit\": \"time\",$", MR_Next},
1051 {"\"iterations\": 3,$", MR_Next},
1052 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
1053 {"\"name\": "
1054 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time_stddev\",$"},
1055 {"\"family_index\": 23,$", MR_Next},
1056 {"\"per_family_instance_index\": 0,$", MR_Next},
1057 {"\"run_name\": "
1058 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
1059 MR_Next},
1060 {"\"run_type\": \"aggregate\",$", MR_Next},
1061 {"\"repetitions\": 3,$", MR_Next},
1062 {"\"threads\": 1,$", MR_Next},
1063 {"\"aggregate_name\": \"stddev\",$", MR_Next},
1064 {"\"aggregate_unit\": \"time\",$", MR_Next},
1065 {"\"iterations\": 3,$", MR_Next},
1066 {"\"real_time\": %float,$", MR_Next},
1067 {"\"name\": "
1068 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time_\",$"},
1069 {"\"family_index\": 23,$", MR_Next},
1070 {"\"per_family_instance_index\": 0,$", MR_Next},
1071 {"\"run_name\": "
1072 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
1073 MR_Next},
1074 {"\"run_type\": \"aggregate\",$", MR_Next},
1075 {"\"repetitions\": 3,$", MR_Next},
1076 {"\"threads\": 1,$", MR_Next},
1077 {"\"aggregate_name\": \"\",$", MR_Next},
1078 {"\"aggregate_unit\": \"percentage\",$", MR_Next},
1079 {"\"iterations\": 3,$", MR_Next},
1080 {"\"real_time\": 1\\.(0)*e-(0)*2,$", MR_Next}});
1081ADD_CASES(TC_CSVOut, {{"^\"BM_UserPercentStats/iterations:5/repeats:3/"
1082 "manual_time\",%csv_report$"},
1083 {"^\"BM_UserPercentStats/iterations:5/repeats:3/"
1084 "manual_time\",%csv_report$"},
1085 {"^\"BM_UserPercentStats/iterations:5/repeats:3/"
1086 "manual_time\",%csv_report$"},
1087 {"^\"BM_UserPercentStats/iterations:5/repeats:3/"
1088 "manual_time_mean\",%csv_report$"},
1089 {"^\"BM_UserPercentStats/iterations:5/repeats:3/"
1090 "manual_time_median\",%csv_report$"},
1091 {"^\"BM_UserPercentStats/iterations:5/repeats:3/"
1092 "manual_time_stddev\",%csv_report$"},
1093 {"^\"BM_UserPercentStats/iterations:5/repeats:3/"
1094 "manual_time_\",%csv_cv_report$"}});
1095
1096// ========================================================================= //
1097// ------------------------- Testing StrEscape JSON ------------------------ //
1098// ========================================================================= //
1099#if 0 // enable when csv testing code correctly handles multi-line fields
1100void BM_JSON_Format(benchmark::State& state) {
1101 state.SkipWithError("val\b\f\n\r\t\\\"with\"es,capes");
1102 for (auto _ : state) {
1103 }
1104}
1105BENCHMARK(BM_JSON_Format);
1106ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_JSON_Format\",$"},
1107 {"\"family_index\": 23,$", MR_Next},
1108{"\"per_family_instance_index\": 0,$", MR_Next},
1109 {"\"run_name\": \"BM_JSON_Format\",$", MR_Next},
1110 {"\"run_type\": \"iteration\",$", MR_Next},
1111 {"\"repetitions\": 1,$", MR_Next},
1112 {"\"repetition_index\": 0,$", MR_Next},
1113 {"\"threads\": 1,$", MR_Next},
1114 {"\"error_occurred\": true,$", MR_Next},
1115 {R"("error_message": "val\\b\\f\\n\\r\\t\\\\\\"with\\"es,capes",$)", MR_Next}});
1116#endif
1117// ========================================================================= //
1118// -------------------------- Testing CsvEscape ---------------------------- //
1119// ========================================================================= //
1120
1121void BM_CSV_Format(benchmark::State& state) {
1122 state.SkipWithError(msg: "\"freedom\"");
1123 for (auto _ : state) {
1124 }
1125}
1126BENCHMARK(BM_CSV_Format);
1127ADD_CASES(TC_CSVOut, {{"^\"BM_CSV_Format\",,,,,,,,true,\"\"\"freedom\"\"\"$"}});
1128
1129// ========================================================================= //
1130// --------------------------- TEST CASES END ------------------------------ //
1131// ========================================================================= //
1132
1133int main(int argc, char* argv[]) { RunOutputTests(argc, argv); }
1134

source code of third-party/benchmark/test/reporter_output_test.cc