1 | |
2 | #undef NDEBUG |
3 | |
4 | #include "benchmark/benchmark.h" |
5 | #include "output_test.h" |
6 | |
7 | // @todo: <jpmag> this checks the full output at once; the rule for |
8 | // CounterSet1 was failing because it was not matching "^[-]+$". |
9 | // @todo: <jpmag> check that the counters are vertically aligned. |
10 | ADD_CASES(TC_ConsoleOut, |
11 | { |
12 | // keeping these lines long improves readability, so: |
13 | // clang-format off |
14 | {"^[-]+$" , MR_Next}, |
15 | {"^Benchmark %s Time %s CPU %s Iterations %s Bar %s Bat %s Baz %s Foo %s Frob %s Lob$" , MR_Next}, |
16 | {"^[-]+$" , MR_Next}, |
17 | {"^BM_Counters_Tabular/repeats:2/threads:1 %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$" , MR_Next}, |
18 | {"^BM_Counters_Tabular/repeats:2/threads:1 %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$" , MR_Next}, |
19 | {"^BM_Counters_Tabular/repeats:2/threads:1_mean %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$" , MR_Next}, |
20 | {"^BM_Counters_Tabular/repeats:2/threads:1_median %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$" , MR_Next}, |
21 | {"^BM_Counters_Tabular/repeats:2/threads:1_stddev %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$" , MR_Next}, |
22 | {"^BM_Counters_Tabular/repeats:2/threads:1_cv %console_percentage_report [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*%$" , MR_Next}, |
23 | {"^BM_Counters_Tabular/repeats:2/threads:2 %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$" , MR_Next}, |
24 | {"^BM_Counters_Tabular/repeats:2/threads:2 %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$" , MR_Next}, |
25 | {"^BM_Counters_Tabular/repeats:2/threads:2_mean %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$" , MR_Next}, |
26 | {"^BM_Counters_Tabular/repeats:2/threads:2_median %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$" , MR_Next}, |
27 | {"^BM_Counters_Tabular/repeats:2/threads:2_stddev %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$" , MR_Next}, |
28 | {"^BM_Counters_Tabular/repeats:2/threads:2_cv %console_percentage_report [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*%$" , MR_Next}, |
29 | {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$" , MR_Next}, |
30 | {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$" , MR_Next}, |
31 | {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$" , MR_Next}, |
32 | {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$" , MR_Next}, |
33 | {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$" , MR_Next}, |
34 | {"^[-]+$" , MR_Next}, |
35 | {"^Benchmark %s Time %s CPU %s Iterations %s Bar %s Baz %s Foo$" , MR_Next}, |
36 | {"^[-]+$" , MR_Next}, |
37 | {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$" , MR_Next}, |
38 | {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$" , MR_Next}, |
39 | {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$" , MR_Next}, |
40 | {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$" , MR_Next}, |
41 | {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$" , MR_Next}, |
42 | {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$" , MR_Next}, |
43 | {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$" , MR_Next}, |
44 | {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$" , MR_Next}, |
45 | {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$" , MR_Next}, |
46 | {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$" , MR_Next}, |
47 | {"^[-]+$" , MR_Next}, |
48 | {"^Benchmark %s Time %s CPU %s Iterations %s Bat %s Baz %s Foo$" , MR_Next}, |
49 | {"^[-]+$" , MR_Next}, |
50 | {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$" , MR_Next}, |
51 | {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$" , MR_Next}, |
52 | {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$" , MR_Next}, |
53 | {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$" , MR_Next}, |
54 | {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$" }, |
55 | // clang-format on |
56 | }); |
57 | ADD_CASES(TC_CSVOut, {{"%csv_header," |
58 | "\"Bar\",\"Bat\",\"Baz\",\"Foo\",\"Frob\",\"Lob\"" }}); |
59 | |
60 | // ========================================================================= // |
61 | // ------------------------- Tabular Counters Output ----------------------- // |
62 | // ========================================================================= // |
63 | |
64 | void BM_Counters_Tabular(benchmark::State& state) { |
65 | for (auto _ : state) { |
66 | // This test requires a non-zero CPU time to avoid divide-by-zero |
67 | auto iterations = double(state.iterations()) * double(state.iterations()); |
68 | benchmark::DoNotOptimize(value&: iterations); |
69 | } |
70 | namespace bm = benchmark; |
71 | state.counters.insert(list: { |
72 | {"Foo" , {1, bm::Counter::kAvgThreads}}, |
73 | {"Bar" , {2, bm::Counter::kAvgThreads}}, |
74 | {"Baz" , {4, bm::Counter::kAvgThreads}}, |
75 | {"Bat" , {8, bm::Counter::kAvgThreads}}, |
76 | {"Frob" , {16, bm::Counter::kAvgThreads}}, |
77 | {"Lob" , {32, bm::Counter::kAvgThreads}}, |
78 | }); |
79 | } |
80 | BENCHMARK(BM_Counters_Tabular)->ThreadRange(min_threads: 1, max_threads: 2)->Repetitions(n: 2); |
81 | ADD_CASES(TC_JSONOut, |
82 | {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$" }, |
83 | {"\"family_index\": 0,$" , MR_Next}, |
84 | {"\"per_family_instance_index\": 0,$" , MR_Next}, |
85 | {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$" , |
86 | MR_Next}, |
87 | {"\"run_type\": \"iteration\",$" , MR_Next}, |
88 | {"\"repetitions\": 2,$" , MR_Next}, |
89 | {"\"repetition_index\": 0,$" , MR_Next}, |
90 | {"\"threads\": 1,$" , MR_Next}, |
91 | {"\"iterations\": %int,$" , MR_Next}, |
92 | {"\"real_time\": %float,$" , MR_Next}, |
93 | {"\"cpu_time\": %float,$" , MR_Next}, |
94 | {"\"time_unit\": \"ns\",$" , MR_Next}, |
95 | {"\"Bar\": %float,$" , MR_Next}, |
96 | {"\"Bat\": %float,$" , MR_Next}, |
97 | {"\"Baz\": %float,$" , MR_Next}, |
98 | {"\"Foo\": %float,$" , MR_Next}, |
99 | {"\"Frob\": %float,$" , MR_Next}, |
100 | {"\"Lob\": %float$" , MR_Next}, |
101 | {"}" , MR_Next}}); |
102 | ADD_CASES(TC_JSONOut, |
103 | {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$" }, |
104 | {"\"family_index\": 0,$" , MR_Next}, |
105 | {"\"per_family_instance_index\": 0,$" , MR_Next}, |
106 | {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$" , |
107 | MR_Next}, |
108 | {"\"run_type\": \"iteration\",$" , MR_Next}, |
109 | {"\"repetitions\": 2,$" , MR_Next}, |
110 | {"\"repetition_index\": 1,$" , MR_Next}, |
111 | {"\"threads\": 1,$" , MR_Next}, |
112 | {"\"iterations\": %int,$" , MR_Next}, |
113 | {"\"real_time\": %float,$" , MR_Next}, |
114 | {"\"cpu_time\": %float,$" , MR_Next}, |
115 | {"\"time_unit\": \"ns\",$" , MR_Next}, |
116 | {"\"Bar\": %float,$" , MR_Next}, |
117 | {"\"Bat\": %float,$" , MR_Next}, |
118 | {"\"Baz\": %float,$" , MR_Next}, |
119 | {"\"Foo\": %float,$" , MR_Next}, |
120 | {"\"Frob\": %float,$" , MR_Next}, |
121 | {"\"Lob\": %float$" , MR_Next}, |
122 | {"}" , MR_Next}}); |
123 | ADD_CASES(TC_JSONOut, |
124 | {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:1_mean\",$" }, |
125 | {"\"family_index\": 0,$" , MR_Next}, |
126 | {"\"per_family_instance_index\": 0,$" , MR_Next}, |
127 | {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$" , |
128 | MR_Next}, |
129 | {"\"run_type\": \"aggregate\",$" , MR_Next}, |
130 | {"\"repetitions\": 2,$" , MR_Next}, |
131 | {"\"threads\": 1,$" , MR_Next}, |
132 | {"\"aggregate_name\": \"mean\",$" , MR_Next}, |
133 | {"\"aggregate_unit\": \"time\",$" , MR_Next}, |
134 | {"\"iterations\": %int,$" , MR_Next}, |
135 | {"\"real_time\": %float,$" , MR_Next}, |
136 | {"\"cpu_time\": %float,$" , MR_Next}, |
137 | {"\"time_unit\": \"ns\",$" , MR_Next}, |
138 | {"\"Bar\": %float,$" , MR_Next}, |
139 | {"\"Bat\": %float,$" , MR_Next}, |
140 | {"\"Baz\": %float,$" , MR_Next}, |
141 | {"\"Foo\": %float,$" , MR_Next}, |
142 | {"\"Frob\": %float,$" , MR_Next}, |
143 | {"\"Lob\": %float$" , MR_Next}, |
144 | {"}" , MR_Next}}); |
145 | ADD_CASES(TC_JSONOut, |
146 | {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:1_median\",$" }, |
147 | {"\"family_index\": 0,$" , MR_Next}, |
148 | {"\"per_family_instance_index\": 0,$" , MR_Next}, |
149 | {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$" , |
150 | MR_Next}, |
151 | {"\"run_type\": \"aggregate\",$" , MR_Next}, |
152 | {"\"repetitions\": 2,$" , MR_Next}, |
153 | {"\"threads\": 1,$" , MR_Next}, |
154 | {"\"aggregate_name\": \"median\",$" , MR_Next}, |
155 | {"\"aggregate_unit\": \"time\",$" , MR_Next}, |
156 | {"\"iterations\": %int,$" , MR_Next}, |
157 | {"\"real_time\": %float,$" , MR_Next}, |
158 | {"\"cpu_time\": %float,$" , MR_Next}, |
159 | {"\"time_unit\": \"ns\",$" , MR_Next}, |
160 | {"\"Bar\": %float,$" , MR_Next}, |
161 | {"\"Bat\": %float,$" , MR_Next}, |
162 | {"\"Baz\": %float,$" , MR_Next}, |
163 | {"\"Foo\": %float,$" , MR_Next}, |
164 | {"\"Frob\": %float,$" , MR_Next}, |
165 | {"\"Lob\": %float$" , MR_Next}, |
166 | {"}" , MR_Next}}); |
167 | ADD_CASES(TC_JSONOut, |
168 | {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:1_stddev\",$" }, |
169 | {"\"family_index\": 0,$" , MR_Next}, |
170 | {"\"per_family_instance_index\": 0,$" , MR_Next}, |
171 | {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$" , |
172 | MR_Next}, |
173 | {"\"run_type\": \"aggregate\",$" , MR_Next}, |
174 | {"\"repetitions\": 2,$" , MR_Next}, |
175 | {"\"threads\": 1,$" , MR_Next}, |
176 | {"\"aggregate_name\": \"stddev\",$" , MR_Next}, |
177 | {"\"aggregate_unit\": \"time\",$" , MR_Next}, |
178 | {"\"iterations\": %int,$" , MR_Next}, |
179 | {"\"real_time\": %float,$" , MR_Next}, |
180 | {"\"cpu_time\": %float,$" , MR_Next}, |
181 | {"\"time_unit\": \"ns\",$" , MR_Next}, |
182 | {"\"Bar\": %float,$" , MR_Next}, |
183 | {"\"Bat\": %float,$" , MR_Next}, |
184 | {"\"Baz\": %float,$" , MR_Next}, |
185 | {"\"Foo\": %float,$" , MR_Next}, |
186 | {"\"Frob\": %float,$" , MR_Next}, |
187 | {"\"Lob\": %float$" , MR_Next}, |
188 | {"}" , MR_Next}}); |
189 | ADD_CASES(TC_JSONOut, |
190 | {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:1_cv\",$" }, |
191 | {"\"family_index\": 0,$" , MR_Next}, |
192 | {"\"per_family_instance_index\": 0,$" , MR_Next}, |
193 | {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$" , |
194 | MR_Next}, |
195 | {"\"run_type\": \"aggregate\",$" , MR_Next}, |
196 | {"\"repetitions\": 2,$" , MR_Next}, |
197 | {"\"threads\": 1,$" , MR_Next}, |
198 | {"\"aggregate_name\": \"cv\",$" , MR_Next}, |
199 | {"\"aggregate_unit\": \"percentage\",$" , MR_Next}, |
200 | {"\"iterations\": %int,$" , MR_Next}, |
201 | {"\"real_time\": %float,$" , MR_Next}, |
202 | {"\"cpu_time\": %float,$" , MR_Next}, |
203 | {"\"time_unit\": \"ns\",$" , MR_Next}, |
204 | {"\"Bar\": %float,$" , MR_Next}, |
205 | {"\"Bat\": %float,$" , MR_Next}, |
206 | {"\"Baz\": %float,$" , MR_Next}, |
207 | {"\"Foo\": %float,$" , MR_Next}, |
208 | {"\"Frob\": %float,$" , MR_Next}, |
209 | {"\"Lob\": %float$" , MR_Next}, |
210 | {"}" , MR_Next}}); |
211 | |
212 | ADD_CASES(TC_JSONOut, |
213 | {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:2\",$" }, |
214 | {"\"family_index\": 0,$" , MR_Next}, |
215 | {"\"per_family_instance_index\": 1,$" , MR_Next}, |
216 | {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:2\",$" , |
217 | MR_Next}, |
218 | {"\"run_type\": \"iteration\",$" , MR_Next}, |
219 | {"\"repetitions\": 2,$" , MR_Next}, |
220 | {"\"repetition_index\": 0,$" , MR_Next}, |
221 | {"\"threads\": 2,$" , MR_Next}, |
222 | {"\"iterations\": %int,$" , MR_Next}, |
223 | {"\"real_time\": %float,$" , MR_Next}, |
224 | {"\"cpu_time\": %float,$" , MR_Next}, |
225 | {"\"time_unit\": \"ns\",$" , MR_Next}, |
226 | {"\"Bar\": %float,$" , MR_Next}, |
227 | {"\"Bat\": %float,$" , MR_Next}, |
228 | {"\"Baz\": %float,$" , MR_Next}, |
229 | {"\"Foo\": %float,$" , MR_Next}, |
230 | {"\"Frob\": %float,$" , MR_Next}, |
231 | {"\"Lob\": %float$" , MR_Next}, |
232 | {"}" , MR_Next}}); |
233 | ADD_CASES(TC_JSONOut, |
234 | {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:2\",$" }, |
235 | {"\"family_index\": 0,$" , MR_Next}, |
236 | {"\"per_family_instance_index\": 1,$" , MR_Next}, |
237 | {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:2\",$" , |
238 | MR_Next}, |
239 | {"\"run_type\": \"iteration\",$" , MR_Next}, |
240 | {"\"repetitions\": 2,$" , MR_Next}, |
241 | {"\"repetition_index\": 1,$" , MR_Next}, |
242 | {"\"threads\": 2,$" , MR_Next}, |
243 | {"\"iterations\": %int,$" , MR_Next}, |
244 | {"\"real_time\": %float,$" , MR_Next}, |
245 | {"\"cpu_time\": %float,$" , MR_Next}, |
246 | {"\"time_unit\": \"ns\",$" , MR_Next}, |
247 | {"\"Bar\": %float,$" , MR_Next}, |
248 | {"\"Bat\": %float,$" , MR_Next}, |
249 | {"\"Baz\": %float,$" , MR_Next}, |
250 | {"\"Foo\": %float,$" , MR_Next}, |
251 | {"\"Frob\": %float,$" , MR_Next}, |
252 | {"\"Lob\": %float$" , MR_Next}, |
253 | {"}" , MR_Next}}); |
254 | ADD_CASES(TC_JSONOut, |
255 | {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:2_median\",$" }, |
256 | {"\"family_index\": 0,$" , MR_Next}, |
257 | {"\"per_family_instance_index\": 1,$" , MR_Next}, |
258 | {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:2\",$" , |
259 | MR_Next}, |
260 | {"\"run_type\": \"aggregate\",$" , MR_Next}, |
261 | {"\"repetitions\": 2,$" , MR_Next}, |
262 | {"\"threads\": 2,$" , MR_Next}, |
263 | {"\"aggregate_name\": \"median\",$" , MR_Next}, |
264 | {"\"aggregate_unit\": \"time\",$" , MR_Next}, |
265 | {"\"iterations\": %int,$" , MR_Next}, |
266 | {"\"real_time\": %float,$" , MR_Next}, |
267 | {"\"cpu_time\": %float,$" , MR_Next}, |
268 | {"\"time_unit\": \"ns\",$" , MR_Next}, |
269 | {"\"Bar\": %float,$" , MR_Next}, |
270 | {"\"Bat\": %float,$" , MR_Next}, |
271 | {"\"Baz\": %float,$" , MR_Next}, |
272 | {"\"Foo\": %float,$" , MR_Next}, |
273 | {"\"Frob\": %float,$" , MR_Next}, |
274 | {"\"Lob\": %float$" , MR_Next}, |
275 | {"}" , MR_Next}}); |
276 | ADD_CASES(TC_JSONOut, |
277 | {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:2_stddev\",$" }, |
278 | {"\"family_index\": 0,$" , MR_Next}, |
279 | {"\"per_family_instance_index\": 1,$" , MR_Next}, |
280 | {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:2\",$" , |
281 | MR_Next}, |
282 | {"\"run_type\": \"aggregate\",$" , MR_Next}, |
283 | {"\"repetitions\": 2,$" , MR_Next}, |
284 | {"\"threads\": 2,$" , MR_Next}, |
285 | {"\"aggregate_name\": \"stddev\",$" , MR_Next}, |
286 | {"\"aggregate_unit\": \"time\",$" , MR_Next}, |
287 | {"\"iterations\": %int,$" , MR_Next}, |
288 | {"\"real_time\": %float,$" , MR_Next}, |
289 | {"\"cpu_time\": %float,$" , MR_Next}, |
290 | {"\"time_unit\": \"ns\",$" , MR_Next}, |
291 | {"\"Bar\": %float,$" , MR_Next}, |
292 | {"\"Bat\": %float,$" , MR_Next}, |
293 | {"\"Baz\": %float,$" , MR_Next}, |
294 | {"\"Foo\": %float,$" , MR_Next}, |
295 | {"\"Frob\": %float,$" , MR_Next}, |
296 | {"\"Lob\": %float$" , MR_Next}, |
297 | {"}" , MR_Next}}); |
298 | ADD_CASES(TC_JSONOut, |
299 | {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:2_cv\",$" }, |
300 | {"\"family_index\": 0,$" , MR_Next}, |
301 | {"\"per_family_instance_index\": 1,$" , MR_Next}, |
302 | {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:2\",$" , |
303 | MR_Next}, |
304 | {"\"run_type\": \"aggregate\",$" , MR_Next}, |
305 | {"\"repetitions\": 2,$" , MR_Next}, |
306 | {"\"threads\": 2,$" , MR_Next}, |
307 | {"\"aggregate_name\": \"cv\",$" , MR_Next}, |
308 | {"\"aggregate_unit\": \"percentage\",$" , MR_Next}, |
309 | {"\"iterations\": %int,$" , MR_Next}, |
310 | {"\"real_time\": %float,$" , MR_Next}, |
311 | {"\"cpu_time\": %float,$" , MR_Next}, |
312 | {"\"time_unit\": \"ns\",$" , MR_Next}, |
313 | {"\"Bar\": %float,$" , MR_Next}, |
314 | {"\"Bat\": %float,$" , MR_Next}, |
315 | {"\"Baz\": %float,$" , MR_Next}, |
316 | {"\"Foo\": %float,$" , MR_Next}, |
317 | {"\"Frob\": %float,$" , MR_Next}, |
318 | {"\"Lob\": %float$" , MR_Next}, |
319 | {"}" , MR_Next}}); |
320 | ADD_CASES(TC_CSVOut, |
321 | {{"^\"BM_Counters_Tabular/repeats:2/threads:1\",%csv_report," |
322 | "%float,%float,%float,%float,%float,%float$" }}); |
323 | ADD_CASES(TC_CSVOut, |
324 | {{"^\"BM_Counters_Tabular/repeats:2/threads:1\",%csv_report," |
325 | "%float,%float,%float,%float,%float,%float$" }}); |
326 | ADD_CASES(TC_CSVOut, |
327 | {{"^\"BM_Counters_Tabular/repeats:2/threads:1_mean\",%csv_report," |
328 | "%float,%float,%float,%float,%float,%float$" }}); |
329 | ADD_CASES(TC_CSVOut, |
330 | {{"^\"BM_Counters_Tabular/repeats:2/threads:1_median\",%csv_report," |
331 | "%float,%float,%float,%float,%float,%float$" }}); |
332 | ADD_CASES(TC_CSVOut, |
333 | {{"^\"BM_Counters_Tabular/repeats:2/threads:1_stddev\",%csv_report," |
334 | "%float,%float,%float,%float,%float,%float$" }}); |
335 | ADD_CASES(TC_CSVOut, |
336 | {{"^\"BM_Counters_Tabular/repeats:2/threads:1_cv\",%csv_cv_report," |
337 | "%float,%float,%float,%float,%float,%float$" }}); |
338 | ADD_CASES(TC_CSVOut, |
339 | {{"^\"BM_Counters_Tabular/repeats:2/threads:2\",%csv_report," |
340 | "%float,%float,%float,%float,%float,%float$" }}); |
341 | ADD_CASES(TC_CSVOut, |
342 | {{"^\"BM_Counters_Tabular/repeats:2/threads:2\",%csv_report," |
343 | "%float,%float,%float,%float,%float,%float$" }}); |
344 | ADD_CASES(TC_CSVOut, |
345 | {{"^\"BM_Counters_Tabular/repeats:2/threads:2_mean\",%csv_report," |
346 | "%float,%float,%float,%float,%float,%float$" }}); |
347 | ADD_CASES(TC_CSVOut, |
348 | {{"^\"BM_Counters_Tabular/repeats:2/threads:2_median\",%csv_report," |
349 | "%float,%float,%float,%float,%float,%float$" }}); |
350 | ADD_CASES(TC_CSVOut, |
351 | {{"^\"BM_Counters_Tabular/repeats:2/threads:2_stddev\",%csv_report," |
352 | "%float,%float,%float,%float,%float,%float$" }}); |
353 | ADD_CASES(TC_CSVOut, |
354 | {{"^\"BM_Counters_Tabular/repeats:2/threads:2_cv\",%csv_cv_report," |
355 | "%float,%float,%float,%float,%float,%float$" }}); |
356 | // VS2013 does not allow this function to be passed as a lambda argument |
357 | // to CHECK_BENCHMARK_RESULTS() |
358 | void CheckTabular(Results const& e) { |
359 | CHECK_COUNTER_VALUE(e, int, "Foo" , EQ, 1); |
360 | CHECK_COUNTER_VALUE(e, int, "Bar" , EQ, 2); |
361 | CHECK_COUNTER_VALUE(e, int, "Baz" , EQ, 4); |
362 | CHECK_COUNTER_VALUE(e, int, "Bat" , EQ, 8); |
363 | CHECK_COUNTER_VALUE(e, int, "Frob" , EQ, 16); |
364 | CHECK_COUNTER_VALUE(e, int, "Lob" , EQ, 32); |
365 | } |
366 | CHECK_BENCHMARK_RESULTS("BM_Counters_Tabular/repeats:2/threads:1$" , |
367 | &CheckTabular); |
368 | CHECK_BENCHMARK_RESULTS("BM_Counters_Tabular/repeats:2/threads:2$" , |
369 | &CheckTabular); |
370 | |
371 | // ========================================================================= // |
372 | // -------------------- Tabular+Rate Counters Output ----------------------- // |
373 | // ========================================================================= // |
374 | |
375 | void BM_CounterRates_Tabular(benchmark::State& state) { |
376 | for (auto _ : state) { |
377 | // This test requires a non-zero CPU time to avoid divide-by-zero |
378 | auto iterations = double(state.iterations()) * double(state.iterations()); |
379 | benchmark::DoNotOptimize(value&: iterations); |
380 | } |
381 | namespace bm = benchmark; |
382 | state.counters.insert(list: { |
383 | {"Foo" , {1, bm::Counter::kAvgThreadsRate}}, |
384 | {"Bar" , {2, bm::Counter::kAvgThreadsRate}}, |
385 | {"Baz" , {4, bm::Counter::kAvgThreadsRate}}, |
386 | {"Bat" , {8, bm::Counter::kAvgThreadsRate}}, |
387 | {"Frob" , {16, bm::Counter::kAvgThreadsRate}}, |
388 | {"Lob" , {32, bm::Counter::kAvgThreadsRate}}, |
389 | }); |
390 | } |
391 | BENCHMARK(BM_CounterRates_Tabular)->ThreadRange(min_threads: 1, max_threads: 16); |
392 | ADD_CASES(TC_JSONOut, |
393 | {{"\"name\": \"BM_CounterRates_Tabular/threads:%int\",$" }, |
394 | {"\"family_index\": 1,$" , MR_Next}, |
395 | {"\"per_family_instance_index\": 0,$" , MR_Next}, |
396 | {"\"run_name\": \"BM_CounterRates_Tabular/threads:%int\",$" , |
397 | MR_Next}, |
398 | {"\"run_type\": \"iteration\",$" , MR_Next}, |
399 | {"\"repetitions\": 1,$" , MR_Next}, |
400 | {"\"repetition_index\": 0,$" , MR_Next}, |
401 | {"\"threads\": 1,$" , MR_Next}, |
402 | {"\"iterations\": %int,$" , MR_Next}, |
403 | {"\"real_time\": %float,$" , MR_Next}, |
404 | {"\"cpu_time\": %float,$" , MR_Next}, |
405 | {"\"time_unit\": \"ns\",$" , MR_Next}, |
406 | {"\"Bar\": %float,$" , MR_Next}, |
407 | {"\"Bat\": %float,$" , MR_Next}, |
408 | {"\"Baz\": %float,$" , MR_Next}, |
409 | {"\"Foo\": %float,$" , MR_Next}, |
410 | {"\"Frob\": %float,$" , MR_Next}, |
411 | {"\"Lob\": %float$" , MR_Next}, |
412 | {"}" , MR_Next}}); |
413 | ADD_CASES(TC_CSVOut, {{"^\"BM_CounterRates_Tabular/threads:%int\",%csv_report," |
414 | "%float,%float,%float,%float,%float,%float$" }}); |
415 | // VS2013 does not allow this function to be passed as a lambda argument |
416 | // to CHECK_BENCHMARK_RESULTS() |
417 | void CheckTabularRate(Results const& e) { |
418 | double t = e.DurationCPUTime(); |
419 | CHECK_FLOAT_COUNTER_VALUE(e, "Foo" , EQ, 1. / t, 0.001); |
420 | CHECK_FLOAT_COUNTER_VALUE(e, "Bar" , EQ, 2. / t, 0.001); |
421 | CHECK_FLOAT_COUNTER_VALUE(e, "Baz" , EQ, 4. / t, 0.001); |
422 | CHECK_FLOAT_COUNTER_VALUE(e, "Bat" , EQ, 8. / t, 0.001); |
423 | CHECK_FLOAT_COUNTER_VALUE(e, "Frob" , EQ, 16. / t, 0.001); |
424 | CHECK_FLOAT_COUNTER_VALUE(e, "Lob" , EQ, 32. / t, 0.001); |
425 | } |
426 | CHECK_BENCHMARK_RESULTS("BM_CounterRates_Tabular/threads:%int" , |
427 | &CheckTabularRate); |
428 | |
429 | // ========================================================================= // |
430 | // ------------------------- Tabular Counters Output ----------------------- // |
431 | // ========================================================================= // |
432 | |
433 | // set only some of the counters |
434 | void BM_CounterSet0_Tabular(benchmark::State& state) { |
435 | for (auto _ : state) { |
436 | } |
437 | namespace bm = benchmark; |
438 | state.counters.insert(list: { |
439 | {"Foo" , {10, bm::Counter::kAvgThreads}}, |
440 | {"Bar" , {20, bm::Counter::kAvgThreads}}, |
441 | {"Baz" , {40, bm::Counter::kAvgThreads}}, |
442 | }); |
443 | } |
444 | BENCHMARK(BM_CounterSet0_Tabular)->ThreadRange(min_threads: 1, max_threads: 16); |
445 | ADD_CASES(TC_JSONOut, |
446 | {{"\"name\": \"BM_CounterSet0_Tabular/threads:%int\",$" }, |
447 | {"\"family_index\": 2,$" , MR_Next}, |
448 | {"\"per_family_instance_index\": 0,$" , MR_Next}, |
449 | {"\"run_name\": \"BM_CounterSet0_Tabular/threads:%int\",$" , MR_Next}, |
450 | {"\"run_type\": \"iteration\",$" , MR_Next}, |
451 | {"\"repetitions\": 1,$" , MR_Next}, |
452 | {"\"repetition_index\": 0,$" , MR_Next}, |
453 | {"\"threads\": 1,$" , MR_Next}, |
454 | {"\"iterations\": %int,$" , MR_Next}, |
455 | {"\"real_time\": %float,$" , MR_Next}, |
456 | {"\"cpu_time\": %float,$" , MR_Next}, |
457 | {"\"time_unit\": \"ns\",$" , MR_Next}, |
458 | {"\"Bar\": %float,$" , MR_Next}, |
459 | {"\"Baz\": %float,$" , MR_Next}, |
460 | {"\"Foo\": %float$" , MR_Next}, |
461 | {"}" , MR_Next}}); |
462 | ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet0_Tabular/threads:%int\",%csv_report," |
463 | "%float,,%float,%float,," }}); |
464 | // VS2013 does not allow this function to be passed as a lambda argument |
465 | // to CHECK_BENCHMARK_RESULTS() |
466 | void CheckSet0(Results const& e) { |
467 | CHECK_COUNTER_VALUE(e, int, "Foo" , EQ, 10); |
468 | CHECK_COUNTER_VALUE(e, int, "Bar" , EQ, 20); |
469 | CHECK_COUNTER_VALUE(e, int, "Baz" , EQ, 40); |
470 | } |
471 | CHECK_BENCHMARK_RESULTS("BM_CounterSet0_Tabular" , &CheckSet0); |
472 | |
473 | // again. |
474 | void BM_CounterSet1_Tabular(benchmark::State& state) { |
475 | for (auto _ : state) { |
476 | } |
477 | namespace bm = benchmark; |
478 | state.counters.insert(list: { |
479 | {"Foo" , {15, bm::Counter::kAvgThreads}}, |
480 | {"Bar" , {25, bm::Counter::kAvgThreads}}, |
481 | {"Baz" , {45, bm::Counter::kAvgThreads}}, |
482 | }); |
483 | } |
484 | BENCHMARK(BM_CounterSet1_Tabular)->ThreadRange(min_threads: 1, max_threads: 16); |
485 | ADD_CASES(TC_JSONOut, |
486 | {{"\"name\": \"BM_CounterSet1_Tabular/threads:%int\",$" }, |
487 | {"\"family_index\": 3,$" , MR_Next}, |
488 | {"\"per_family_instance_index\": 0,$" , MR_Next}, |
489 | {"\"run_name\": \"BM_CounterSet1_Tabular/threads:%int\",$" , MR_Next}, |
490 | {"\"run_type\": \"iteration\",$" , MR_Next}, |
491 | {"\"repetitions\": 1,$" , MR_Next}, |
492 | {"\"repetition_index\": 0,$" , MR_Next}, |
493 | {"\"threads\": 1,$" , MR_Next}, |
494 | {"\"iterations\": %int,$" , MR_Next}, |
495 | {"\"real_time\": %float,$" , MR_Next}, |
496 | {"\"cpu_time\": %float,$" , MR_Next}, |
497 | {"\"time_unit\": \"ns\",$" , MR_Next}, |
498 | {"\"Bar\": %float,$" , MR_Next}, |
499 | {"\"Baz\": %float,$" , MR_Next}, |
500 | {"\"Foo\": %float$" , MR_Next}, |
501 | {"}" , MR_Next}}); |
502 | ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet1_Tabular/threads:%int\",%csv_report," |
503 | "%float,,%float,%float,," }}); |
504 | // VS2013 does not allow this function to be passed as a lambda argument |
505 | // to CHECK_BENCHMARK_RESULTS() |
506 | void CheckSet1(Results const& e) { |
507 | CHECK_COUNTER_VALUE(e, int, "Foo" , EQ, 15); |
508 | CHECK_COUNTER_VALUE(e, int, "Bar" , EQ, 25); |
509 | CHECK_COUNTER_VALUE(e, int, "Baz" , EQ, 45); |
510 | } |
511 | CHECK_BENCHMARK_RESULTS("BM_CounterSet1_Tabular/threads:%int" , &CheckSet1); |
512 | |
513 | // ========================================================================= // |
514 | // ------------------------- Tabular Counters Output ----------------------- // |
515 | // ========================================================================= // |
516 | |
517 | // set only some of the counters, different set now. |
518 | void BM_CounterSet2_Tabular(benchmark::State& state) { |
519 | for (auto _ : state) { |
520 | } |
521 | namespace bm = benchmark; |
522 | state.counters.insert(list: { |
523 | {"Foo" , {10, bm::Counter::kAvgThreads}}, |
524 | {"Bat" , {30, bm::Counter::kAvgThreads}}, |
525 | {"Baz" , {40, bm::Counter::kAvgThreads}}, |
526 | }); |
527 | } |
528 | BENCHMARK(BM_CounterSet2_Tabular)->ThreadRange(min_threads: 1, max_threads: 16); |
529 | ADD_CASES(TC_JSONOut, |
530 | {{"\"name\": \"BM_CounterSet2_Tabular/threads:%int\",$" }, |
531 | {"\"family_index\": 4,$" , MR_Next}, |
532 | {"\"per_family_instance_index\": 0,$" , MR_Next}, |
533 | {"\"run_name\": \"BM_CounterSet2_Tabular/threads:%int\",$" , MR_Next}, |
534 | {"\"run_type\": \"iteration\",$" , MR_Next}, |
535 | {"\"repetitions\": 1,$" , MR_Next}, |
536 | {"\"repetition_index\": 0,$" , MR_Next}, |
537 | {"\"threads\": 1,$" , MR_Next}, |
538 | {"\"iterations\": %int,$" , MR_Next}, |
539 | {"\"real_time\": %float,$" , MR_Next}, |
540 | {"\"cpu_time\": %float,$" , MR_Next}, |
541 | {"\"time_unit\": \"ns\",$" , MR_Next}, |
542 | {"\"Bat\": %float,$" , MR_Next}, |
543 | {"\"Baz\": %float,$" , MR_Next}, |
544 | {"\"Foo\": %float$" , MR_Next}, |
545 | {"}" , MR_Next}}); |
546 | ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet2_Tabular/threads:%int\",%csv_report," |
547 | ",%float,%float,%float,," }}); |
548 | // VS2013 does not allow this function to be passed as a lambda argument |
549 | // to CHECK_BENCHMARK_RESULTS() |
550 | void CheckSet2(Results const& e) { |
551 | CHECK_COUNTER_VALUE(e, int, "Foo" , EQ, 10); |
552 | CHECK_COUNTER_VALUE(e, int, "Bat" , EQ, 30); |
553 | CHECK_COUNTER_VALUE(e, int, "Baz" , EQ, 40); |
554 | } |
555 | CHECK_BENCHMARK_RESULTS("BM_CounterSet2_Tabular" , &CheckSet2); |
556 | |
557 | // ========================================================================= // |
558 | // --------------------------- TEST CASES END ------------------------------ // |
559 | // ========================================================================= // |
560 | |
561 | int main(int argc, char* argv[]) { RunOutputTests(argc, argv); } |
562 | |