1
2 #undef NDEBUG
3
4 #include "benchmark/benchmark.h"
5 #include "output_test.h"
6
7 // @todo: <jpmag> this checks the full output at once; the rule for
8 // CounterSet1 was failing because it was not matching "^[-]+$".
9 // @todo: <jpmag> check that the counters are vertically aligned.
10 ADD_CASES(TC_ConsoleOut,
11 {
12 // keeping these lines long improves readability, so:
13 // clang-format off
14 {"^[-]+$", MR_Next},
15 {"^Benchmark %s Time %s CPU %s Iterations %s Bar %s Bat %s Baz %s Foo %s Frob %s Lob$", MR_Next},
16 {"^[-]+$", MR_Next},
17 {"^BM_Counters_Tabular/repeats:2/threads:1 %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
18 {"^BM_Counters_Tabular/repeats:2/threads:1 %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
19 {"^BM_Counters_Tabular/repeats:2/threads:1_mean %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
20 {"^BM_Counters_Tabular/repeats:2/threads:1_median %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
21 {"^BM_Counters_Tabular/repeats:2/threads:1_stddev %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
22 {"^BM_Counters_Tabular/repeats:2/threads:1_cv %console_percentage_report [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*%$", MR_Next},
23 {"^BM_Counters_Tabular/repeats:2/threads:2 %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
24 {"^BM_Counters_Tabular/repeats:2/threads:2 %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
25 {"^BM_Counters_Tabular/repeats:2/threads:2_mean %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
26 {"^BM_Counters_Tabular/repeats:2/threads:2_median %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
27 {"^BM_Counters_Tabular/repeats:2/threads:2_stddev %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
28 {"^BM_Counters_Tabular/repeats:2/threads:2_cv %console_percentage_report [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*%$", MR_Next},
29 {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
30 {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
31 {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
32 {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
33 {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
34 {"^[-]+$", MR_Next},
35 {"^Benchmark %s Time %s CPU %s Iterations %s Bar %s Baz %s Foo$", MR_Next},
36 {"^[-]+$", MR_Next},
37 {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
38 {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
39 {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
40 {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
41 {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
42 {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
43 {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
44 {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
45 {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
46 {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
47 {"^[-]+$", MR_Next},
48 {"^Benchmark %s Time %s CPU %s Iterations %s Bat %s Baz %s Foo$", MR_Next},
49 {"^[-]+$", MR_Next},
50 {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
51 {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
52 {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
53 {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
54 {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$"},
55 // clang-format on
56 });
57 ADD_CASES(TC_CSVOut, {{"%csv_header,"
58 "\"Bar\",\"Bat\",\"Baz\",\"Foo\",\"Frob\",\"Lob\""}});
59
60 // ========================================================================= //
61 // ------------------------- Tabular Counters Output ----------------------- //
62 // ========================================================================= //
63
BM_Counters_Tabular(benchmark::State & state)64 void BM_Counters_Tabular(benchmark::State& state) {
65 for (auto _ : state) {
66 // This test requires a non-zero CPU time to avoid divide-by-zero
67 auto iterations = double(state.iterations()) * double(state.iterations());
68 benchmark::DoNotOptimize(iterations);
69 }
70 namespace bm = benchmark;
71 state.counters.insert({
72 {"Foo", {1, bm::Counter::kAvgThreads}},
73 {"Bar", {2, bm::Counter::kAvgThreads}},
74 {"Baz", {4, bm::Counter::kAvgThreads}},
75 {"Bat", {8, bm::Counter::kAvgThreads}},
76 {"Frob", {16, bm::Counter::kAvgThreads}},
77 {"Lob", {32, bm::Counter::kAvgThreads}},
78 });
79 }
80 BENCHMARK(BM_Counters_Tabular)->ThreadRange(1, 2)->Repetitions(2);
81 ADD_CASES(TC_JSONOut,
82 {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$"},
83 {"\"family_index\": 0,$", MR_Next},
84 {"\"per_family_instance_index\": 0,$", MR_Next},
85 {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$",
86 MR_Next},
87 {"\"run_type\": \"iteration\",$", MR_Next},
88 {"\"repetitions\": 2,$", MR_Next},
89 {"\"repetition_index\": 0,$", MR_Next},
90 {"\"threads\": 1,$", MR_Next},
91 {"\"iterations\": %int,$", MR_Next},
92 {"\"real_time\": %float,$", MR_Next},
93 {"\"cpu_time\": %float,$", MR_Next},
94 {"\"time_unit\": \"ns\",$", MR_Next},
95 {"\"Bar\": %float,$", MR_Next},
96 {"\"Bat\": %float,$", MR_Next},
97 {"\"Baz\": %float,$", MR_Next},
98 {"\"Foo\": %float,$", MR_Next},
99 {"\"Frob\": %float,$", MR_Next},
100 {"\"Lob\": %float$", MR_Next},
101 {"}", MR_Next}});
102 ADD_CASES(TC_JSONOut,
103 {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$"},
104 {"\"family_index\": 0,$", MR_Next},
105 {"\"per_family_instance_index\": 0,$", MR_Next},
106 {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$",
107 MR_Next},
108 {"\"run_type\": \"iteration\",$", MR_Next},
109 {"\"repetitions\": 2,$", MR_Next},
110 {"\"repetition_index\": 1,$", MR_Next},
111 {"\"threads\": 1,$", MR_Next},
112 {"\"iterations\": %int,$", MR_Next},
113 {"\"real_time\": %float,$", MR_Next},
114 {"\"cpu_time\": %float,$", MR_Next},
115 {"\"time_unit\": \"ns\",$", MR_Next},
116 {"\"Bar\": %float,$", MR_Next},
117 {"\"Bat\": %float,$", MR_Next},
118 {"\"Baz\": %float,$", MR_Next},
119 {"\"Foo\": %float,$", MR_Next},
120 {"\"Frob\": %float,$", MR_Next},
121 {"\"Lob\": %float$", MR_Next},
122 {"}", MR_Next}});
123 ADD_CASES(TC_JSONOut,
124 {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:1_mean\",$"},
125 {"\"family_index\": 0,$", MR_Next},
126 {"\"per_family_instance_index\": 0,$", MR_Next},
127 {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$",
128 MR_Next},
129 {"\"run_type\": \"aggregate\",$", MR_Next},
130 {"\"repetitions\": 2,$", MR_Next},
131 {"\"threads\": 1,$", MR_Next},
132 {"\"aggregate_name\": \"mean\",$", MR_Next},
133 {"\"aggregate_unit\": \"time\",$", MR_Next},
134 {"\"iterations\": %int,$", MR_Next},
135 {"\"real_time\": %float,$", MR_Next},
136 {"\"cpu_time\": %float,$", MR_Next},
137 {"\"time_unit\": \"ns\",$", MR_Next},
138 {"\"Bar\": %float,$", MR_Next},
139 {"\"Bat\": %float,$", MR_Next},
140 {"\"Baz\": %float,$", MR_Next},
141 {"\"Foo\": %float,$", MR_Next},
142 {"\"Frob\": %float,$", MR_Next},
143 {"\"Lob\": %float$", MR_Next},
144 {"}", MR_Next}});
145 ADD_CASES(TC_JSONOut,
146 {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:1_median\",$"},
147 {"\"family_index\": 0,$", MR_Next},
148 {"\"per_family_instance_index\": 0,$", MR_Next},
149 {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$",
150 MR_Next},
151 {"\"run_type\": \"aggregate\",$", MR_Next},
152 {"\"repetitions\": 2,$", MR_Next},
153 {"\"threads\": 1,$", MR_Next},
154 {"\"aggregate_name\": \"median\",$", MR_Next},
155 {"\"aggregate_unit\": \"time\",$", MR_Next},
156 {"\"iterations\": %int,$", MR_Next},
157 {"\"real_time\": %float,$", MR_Next},
158 {"\"cpu_time\": %float,$", MR_Next},
159 {"\"time_unit\": \"ns\",$", MR_Next},
160 {"\"Bar\": %float,$", MR_Next},
161 {"\"Bat\": %float,$", MR_Next},
162 {"\"Baz\": %float,$", MR_Next},
163 {"\"Foo\": %float,$", MR_Next},
164 {"\"Frob\": %float,$", MR_Next},
165 {"\"Lob\": %float$", MR_Next},
166 {"}", MR_Next}});
167 ADD_CASES(TC_JSONOut,
168 {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:1_stddev\",$"},
169 {"\"family_index\": 0,$", MR_Next},
170 {"\"per_family_instance_index\": 0,$", MR_Next},
171 {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$",
172 MR_Next},
173 {"\"run_type\": \"aggregate\",$", MR_Next},
174 {"\"repetitions\": 2,$", MR_Next},
175 {"\"threads\": 1,$", MR_Next},
176 {"\"aggregate_name\": \"stddev\",$", MR_Next},
177 {"\"aggregate_unit\": \"time\",$", MR_Next},
178 {"\"iterations\": %int,$", MR_Next},
179 {"\"real_time\": %float,$", MR_Next},
180 {"\"cpu_time\": %float,$", MR_Next},
181 {"\"time_unit\": \"ns\",$", MR_Next},
182 {"\"Bar\": %float,$", MR_Next},
183 {"\"Bat\": %float,$", MR_Next},
184 {"\"Baz\": %float,$", MR_Next},
185 {"\"Foo\": %float,$", MR_Next},
186 {"\"Frob\": %float,$", MR_Next},
187 {"\"Lob\": %float$", MR_Next},
188 {"}", MR_Next}});
189 ADD_CASES(TC_JSONOut,
190 {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:1_cv\",$"},
191 {"\"family_index\": 0,$", MR_Next},
192 {"\"per_family_instance_index\": 0,$", MR_Next},
193 {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$",
194 MR_Next},
195 {"\"run_type\": \"aggregate\",$", MR_Next},
196 {"\"repetitions\": 2,$", MR_Next},
197 {"\"threads\": 1,$", MR_Next},
198 {"\"aggregate_name\": \"cv\",$", MR_Next},
199 {"\"aggregate_unit\": \"percentage\",$", MR_Next},
200 {"\"iterations\": %int,$", MR_Next},
201 {"\"real_time\": %float,$", MR_Next},
202 {"\"cpu_time\": %float,$", MR_Next},
203 {"\"time_unit\": \"ns\",$", MR_Next},
204 {"\"Bar\": %float,$", MR_Next},
205 {"\"Bat\": %float,$", MR_Next},
206 {"\"Baz\": %float,$", MR_Next},
207 {"\"Foo\": %float,$", MR_Next},
208 {"\"Frob\": %float,$", MR_Next},
209 {"\"Lob\": %float$", MR_Next},
210 {"}", MR_Next}});
211
212 ADD_CASES(TC_JSONOut,
213 {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:2\",$"},
214 {"\"family_index\": 0,$", MR_Next},
215 {"\"per_family_instance_index\": 1,$", MR_Next},
216 {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:2\",$",
217 MR_Next},
218 {"\"run_type\": \"iteration\",$", MR_Next},
219 {"\"repetitions\": 2,$", MR_Next},
220 {"\"repetition_index\": 0,$", MR_Next},
221 {"\"threads\": 2,$", MR_Next},
222 {"\"iterations\": %int,$", MR_Next},
223 {"\"real_time\": %float,$", MR_Next},
224 {"\"cpu_time\": %float,$", MR_Next},
225 {"\"time_unit\": \"ns\",$", MR_Next},
226 {"\"Bar\": %float,$", MR_Next},
227 {"\"Bat\": %float,$", MR_Next},
228 {"\"Baz\": %float,$", MR_Next},
229 {"\"Foo\": %float,$", MR_Next},
230 {"\"Frob\": %float,$", MR_Next},
231 {"\"Lob\": %float$", MR_Next},
232 {"}", MR_Next}});
233 ADD_CASES(TC_JSONOut,
234 {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:2\",$"},
235 {"\"family_index\": 0,$", MR_Next},
236 {"\"per_family_instance_index\": 1,$", MR_Next},
237 {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:2\",$",
238 MR_Next},
239 {"\"run_type\": \"iteration\",$", MR_Next},
240 {"\"repetitions\": 2,$", MR_Next},
241 {"\"repetition_index\": 1,$", MR_Next},
242 {"\"threads\": 2,$", MR_Next},
243 {"\"iterations\": %int,$", MR_Next},
244 {"\"real_time\": %float,$", MR_Next},
245 {"\"cpu_time\": %float,$", MR_Next},
246 {"\"time_unit\": \"ns\",$", MR_Next},
247 {"\"Bar\": %float,$", MR_Next},
248 {"\"Bat\": %float,$", MR_Next},
249 {"\"Baz\": %float,$", MR_Next},
250 {"\"Foo\": %float,$", MR_Next},
251 {"\"Frob\": %float,$", MR_Next},
252 {"\"Lob\": %float$", MR_Next},
253 {"}", MR_Next}});
254 ADD_CASES(TC_JSONOut,
255 {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:2_median\",$"},
256 {"\"family_index\": 0,$", MR_Next},
257 {"\"per_family_instance_index\": 1,$", MR_Next},
258 {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:2\",$",
259 MR_Next},
260 {"\"run_type\": \"aggregate\",$", MR_Next},
261 {"\"repetitions\": 2,$", MR_Next},
262 {"\"threads\": 2,$", MR_Next},
263 {"\"aggregate_name\": \"median\",$", MR_Next},
264 {"\"aggregate_unit\": \"time\",$", MR_Next},
265 {"\"iterations\": %int,$", MR_Next},
266 {"\"real_time\": %float,$", MR_Next},
267 {"\"cpu_time\": %float,$", MR_Next},
268 {"\"time_unit\": \"ns\",$", MR_Next},
269 {"\"Bar\": %float,$", MR_Next},
270 {"\"Bat\": %float,$", MR_Next},
271 {"\"Baz\": %float,$", MR_Next},
272 {"\"Foo\": %float,$", MR_Next},
273 {"\"Frob\": %float,$", MR_Next},
274 {"\"Lob\": %float$", MR_Next},
275 {"}", MR_Next}});
276 ADD_CASES(TC_JSONOut,
277 {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:2_stddev\",$"},
278 {"\"family_index\": 0,$", MR_Next},
279 {"\"per_family_instance_index\": 1,$", MR_Next},
280 {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:2\",$",
281 MR_Next},
282 {"\"run_type\": \"aggregate\",$", MR_Next},
283 {"\"repetitions\": 2,$", MR_Next},
284 {"\"threads\": 2,$", MR_Next},
285 {"\"aggregate_name\": \"stddev\",$", MR_Next},
286 {"\"aggregate_unit\": \"time\",$", MR_Next},
287 {"\"iterations\": %int,$", MR_Next},
288 {"\"real_time\": %float,$", MR_Next},
289 {"\"cpu_time\": %float,$", MR_Next},
290 {"\"time_unit\": \"ns\",$", MR_Next},
291 {"\"Bar\": %float,$", MR_Next},
292 {"\"Bat\": %float,$", MR_Next},
293 {"\"Baz\": %float,$", MR_Next},
294 {"\"Foo\": %float,$", MR_Next},
295 {"\"Frob\": %float,$", MR_Next},
296 {"\"Lob\": %float$", MR_Next},
297 {"}", MR_Next}});
298 ADD_CASES(TC_JSONOut,
299 {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:2_cv\",$"},
300 {"\"family_index\": 0,$", MR_Next},
301 {"\"per_family_instance_index\": 1,$", MR_Next},
302 {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:2\",$",
303 MR_Next},
304 {"\"run_type\": \"aggregate\",$", MR_Next},
305 {"\"repetitions\": 2,$", MR_Next},
306 {"\"threads\": 2,$", MR_Next},
307 {"\"aggregate_name\": \"cv\",$", MR_Next},
308 {"\"aggregate_unit\": \"percentage\",$", MR_Next},
309 {"\"iterations\": %int,$", MR_Next},
310 {"\"real_time\": %float,$", MR_Next},
311 {"\"cpu_time\": %float,$", MR_Next},
312 {"\"time_unit\": \"ns\",$", MR_Next},
313 {"\"Bar\": %float,$", MR_Next},
314 {"\"Bat\": %float,$", MR_Next},
315 {"\"Baz\": %float,$", MR_Next},
316 {"\"Foo\": %float,$", MR_Next},
317 {"\"Frob\": %float,$", MR_Next},
318 {"\"Lob\": %float$", MR_Next},
319 {"}", MR_Next}});
320 ADD_CASES(TC_CSVOut,
321 {{"^\"BM_Counters_Tabular/repeats:2/threads:1\",%csv_report,"
322 "%float,%float,%float,%float,%float,%float$"}});
323 ADD_CASES(TC_CSVOut,
324 {{"^\"BM_Counters_Tabular/repeats:2/threads:1\",%csv_report,"
325 "%float,%float,%float,%float,%float,%float$"}});
326 ADD_CASES(TC_CSVOut,
327 {{"^\"BM_Counters_Tabular/repeats:2/threads:1_mean\",%csv_report,"
328 "%float,%float,%float,%float,%float,%float$"}});
329 ADD_CASES(TC_CSVOut,
330 {{"^\"BM_Counters_Tabular/repeats:2/threads:1_median\",%csv_report,"
331 "%float,%float,%float,%float,%float,%float$"}});
332 ADD_CASES(TC_CSVOut,
333 {{"^\"BM_Counters_Tabular/repeats:2/threads:1_stddev\",%csv_report,"
334 "%float,%float,%float,%float,%float,%float$"}});
335 ADD_CASES(TC_CSVOut,
336 {{"^\"BM_Counters_Tabular/repeats:2/threads:1_cv\",%csv_cv_report,"
337 "%float,%float,%float,%float,%float,%float$"}});
338 ADD_CASES(TC_CSVOut,
339 {{"^\"BM_Counters_Tabular/repeats:2/threads:2\",%csv_report,"
340 "%float,%float,%float,%float,%float,%float$"}});
341 ADD_CASES(TC_CSVOut,
342 {{"^\"BM_Counters_Tabular/repeats:2/threads:2\",%csv_report,"
343 "%float,%float,%float,%float,%float,%float$"}});
344 ADD_CASES(TC_CSVOut,
345 {{"^\"BM_Counters_Tabular/repeats:2/threads:2_mean\",%csv_report,"
346 "%float,%float,%float,%float,%float,%float$"}});
347 ADD_CASES(TC_CSVOut,
348 {{"^\"BM_Counters_Tabular/repeats:2/threads:2_median\",%csv_report,"
349 "%float,%float,%float,%float,%float,%float$"}});
350 ADD_CASES(TC_CSVOut,
351 {{"^\"BM_Counters_Tabular/repeats:2/threads:2_stddev\",%csv_report,"
352 "%float,%float,%float,%float,%float,%float$"}});
353 ADD_CASES(TC_CSVOut,
354 {{"^\"BM_Counters_Tabular/repeats:2/threads:2_cv\",%csv_cv_report,"
355 "%float,%float,%float,%float,%float,%float$"}});
356 // VS2013 does not allow this function to be passed as a lambda argument
357 // to CHECK_BENCHMARK_RESULTS()
CheckTabular(Results const & e)358 void CheckTabular(Results const& e) {
359 CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 1);
360 CHECK_COUNTER_VALUE(e, int, "Bar", EQ, 2);
361 CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 4);
362 CHECK_COUNTER_VALUE(e, int, "Bat", EQ, 8);
363 CHECK_COUNTER_VALUE(e, int, "Frob", EQ, 16);
364 CHECK_COUNTER_VALUE(e, int, "Lob", EQ, 32);
365 }
366 CHECK_BENCHMARK_RESULTS("BM_Counters_Tabular/repeats:2/threads:1$",
367 &CheckTabular);
368 CHECK_BENCHMARK_RESULTS("BM_Counters_Tabular/repeats:2/threads:2$",
369 &CheckTabular);
370
371 // ========================================================================= //
372 // -------------------- Tabular+Rate Counters Output ----------------------- //
373 // ========================================================================= //
374
BM_CounterRates_Tabular(benchmark::State & state)375 void BM_CounterRates_Tabular(benchmark::State& state) {
376 for (auto _ : state) {
377 // This test requires a non-zero CPU time to avoid divide-by-zero
378 auto iterations = double(state.iterations()) * double(state.iterations());
379 benchmark::DoNotOptimize(iterations);
380 }
381 namespace bm = benchmark;
382 state.counters.insert({
383 {"Foo", {1, bm::Counter::kAvgThreadsRate}},
384 {"Bar", {2, bm::Counter::kAvgThreadsRate}},
385 {"Baz", {4, bm::Counter::kAvgThreadsRate}},
386 {"Bat", {8, bm::Counter::kAvgThreadsRate}},
387 {"Frob", {16, bm::Counter::kAvgThreadsRate}},
388 {"Lob", {32, bm::Counter::kAvgThreadsRate}},
389 });
390 }
391 BENCHMARK(BM_CounterRates_Tabular)->ThreadRange(1, 16);
392 ADD_CASES(TC_JSONOut,
393 {{"\"name\": \"BM_CounterRates_Tabular/threads:%int\",$"},
394 {"\"family_index\": 1,$", MR_Next},
395 {"\"per_family_instance_index\": 0,$", MR_Next},
396 {"\"run_name\": \"BM_CounterRates_Tabular/threads:%int\",$",
397 MR_Next},
398 {"\"run_type\": \"iteration\",$", MR_Next},
399 {"\"repetitions\": 1,$", MR_Next},
400 {"\"repetition_index\": 0,$", MR_Next},
401 {"\"threads\": 1,$", MR_Next},
402 {"\"iterations\": %int,$", MR_Next},
403 {"\"real_time\": %float,$", MR_Next},
404 {"\"cpu_time\": %float,$", MR_Next},
405 {"\"time_unit\": \"ns\",$", MR_Next},
406 {"\"Bar\": %float,$", MR_Next},
407 {"\"Bat\": %float,$", MR_Next},
408 {"\"Baz\": %float,$", MR_Next},
409 {"\"Foo\": %float,$", MR_Next},
410 {"\"Frob\": %float,$", MR_Next},
411 {"\"Lob\": %float$", MR_Next},
412 {"}", MR_Next}});
413 ADD_CASES(TC_CSVOut, {{"^\"BM_CounterRates_Tabular/threads:%int\",%csv_report,"
414 "%float,%float,%float,%float,%float,%float$"}});
415 // VS2013 does not allow this function to be passed as a lambda argument
416 // to CHECK_BENCHMARK_RESULTS()
CheckTabularRate(Results const & e)417 void CheckTabularRate(Results const& e) {
418 double t = e.DurationCPUTime();
419 CHECK_FLOAT_COUNTER_VALUE(e, "Foo", EQ, 1. / t, 0.001);
420 CHECK_FLOAT_COUNTER_VALUE(e, "Bar", EQ, 2. / t, 0.001);
421 CHECK_FLOAT_COUNTER_VALUE(e, "Baz", EQ, 4. / t, 0.001);
422 CHECK_FLOAT_COUNTER_VALUE(e, "Bat", EQ, 8. / t, 0.001);
423 CHECK_FLOAT_COUNTER_VALUE(e, "Frob", EQ, 16. / t, 0.001);
424 CHECK_FLOAT_COUNTER_VALUE(e, "Lob", EQ, 32. / t, 0.001);
425 }
426 CHECK_BENCHMARK_RESULTS("BM_CounterRates_Tabular/threads:%int",
427 &CheckTabularRate);
428
429 // ========================================================================= //
430 // ------------------------- Tabular Counters Output ----------------------- //
431 // ========================================================================= //
432
433 // set only some of the counters
BM_CounterSet0_Tabular(benchmark::State & state)434 void BM_CounterSet0_Tabular(benchmark::State& state) {
435 for (auto _ : state) {
436 }
437 namespace bm = benchmark;
438 state.counters.insert({
439 {"Foo", {10, bm::Counter::kAvgThreads}},
440 {"Bar", {20, bm::Counter::kAvgThreads}},
441 {"Baz", {40, bm::Counter::kAvgThreads}},
442 });
443 }
444 BENCHMARK(BM_CounterSet0_Tabular)->ThreadRange(1, 16);
445 ADD_CASES(TC_JSONOut,
446 {{"\"name\": \"BM_CounterSet0_Tabular/threads:%int\",$"},
447 {"\"family_index\": 2,$", MR_Next},
448 {"\"per_family_instance_index\": 0,$", MR_Next},
449 {"\"run_name\": \"BM_CounterSet0_Tabular/threads:%int\",$", MR_Next},
450 {"\"run_type\": \"iteration\",$", MR_Next},
451 {"\"repetitions\": 1,$", MR_Next},
452 {"\"repetition_index\": 0,$", MR_Next},
453 {"\"threads\": 1,$", MR_Next},
454 {"\"iterations\": %int,$", MR_Next},
455 {"\"real_time\": %float,$", MR_Next},
456 {"\"cpu_time\": %float,$", MR_Next},
457 {"\"time_unit\": \"ns\",$", MR_Next},
458 {"\"Bar\": %float,$", MR_Next},
459 {"\"Baz\": %float,$", MR_Next},
460 {"\"Foo\": %float$", MR_Next},
461 {"}", MR_Next}});
462 ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet0_Tabular/threads:%int\",%csv_report,"
463 "%float,,%float,%float,,"}});
464 // VS2013 does not allow this function to be passed as a lambda argument
465 // to CHECK_BENCHMARK_RESULTS()
CheckSet0(Results const & e)466 void CheckSet0(Results const& e) {
467 CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 10);
468 CHECK_COUNTER_VALUE(e, int, "Bar", EQ, 20);
469 CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 40);
470 }
471 CHECK_BENCHMARK_RESULTS("BM_CounterSet0_Tabular", &CheckSet0);
472
473 // again.
BM_CounterSet1_Tabular(benchmark::State & state)474 void BM_CounterSet1_Tabular(benchmark::State& state) {
475 for (auto _ : state) {
476 }
477 namespace bm = benchmark;
478 state.counters.insert({
479 {"Foo", {15, bm::Counter::kAvgThreads}},
480 {"Bar", {25, bm::Counter::kAvgThreads}},
481 {"Baz", {45, bm::Counter::kAvgThreads}},
482 });
483 }
484 BENCHMARK(BM_CounterSet1_Tabular)->ThreadRange(1, 16);
485 ADD_CASES(TC_JSONOut,
486 {{"\"name\": \"BM_CounterSet1_Tabular/threads:%int\",$"},
487 {"\"family_index\": 3,$", MR_Next},
488 {"\"per_family_instance_index\": 0,$", MR_Next},
489 {"\"run_name\": \"BM_CounterSet1_Tabular/threads:%int\",$", MR_Next},
490 {"\"run_type\": \"iteration\",$", MR_Next},
491 {"\"repetitions\": 1,$", MR_Next},
492 {"\"repetition_index\": 0,$", MR_Next},
493 {"\"threads\": 1,$", MR_Next},
494 {"\"iterations\": %int,$", MR_Next},
495 {"\"real_time\": %float,$", MR_Next},
496 {"\"cpu_time\": %float,$", MR_Next},
497 {"\"time_unit\": \"ns\",$", MR_Next},
498 {"\"Bar\": %float,$", MR_Next},
499 {"\"Baz\": %float,$", MR_Next},
500 {"\"Foo\": %float$", MR_Next},
501 {"}", MR_Next}});
502 ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet1_Tabular/threads:%int\",%csv_report,"
503 "%float,,%float,%float,,"}});
504 // VS2013 does not allow this function to be passed as a lambda argument
505 // to CHECK_BENCHMARK_RESULTS()
CheckSet1(Results const & e)506 void CheckSet1(Results const& e) {
507 CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 15);
508 CHECK_COUNTER_VALUE(e, int, "Bar", EQ, 25);
509 CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 45);
510 }
511 CHECK_BENCHMARK_RESULTS("BM_CounterSet1_Tabular/threads:%int", &CheckSet1);
512
513 // ========================================================================= //
514 // ------------------------- Tabular Counters Output ----------------------- //
515 // ========================================================================= //
516
517 // set only some of the counters, different set now.
BM_CounterSet2_Tabular(benchmark::State & state)518 void BM_CounterSet2_Tabular(benchmark::State& state) {
519 for (auto _ : state) {
520 }
521 namespace bm = benchmark;
522 state.counters.insert({
523 {"Foo", {10, bm::Counter::kAvgThreads}},
524 {"Bat", {30, bm::Counter::kAvgThreads}},
525 {"Baz", {40, bm::Counter::kAvgThreads}},
526 });
527 }
528 BENCHMARK(BM_CounterSet2_Tabular)->ThreadRange(1, 16);
529 ADD_CASES(TC_JSONOut,
530 {{"\"name\": \"BM_CounterSet2_Tabular/threads:%int\",$"},
531 {"\"family_index\": 4,$", MR_Next},
532 {"\"per_family_instance_index\": 0,$", MR_Next},
533 {"\"run_name\": \"BM_CounterSet2_Tabular/threads:%int\",$", MR_Next},
534 {"\"run_type\": \"iteration\",$", MR_Next},
535 {"\"repetitions\": 1,$", MR_Next},
536 {"\"repetition_index\": 0,$", MR_Next},
537 {"\"threads\": 1,$", MR_Next},
538 {"\"iterations\": %int,$", MR_Next},
539 {"\"real_time\": %float,$", MR_Next},
540 {"\"cpu_time\": %float,$", MR_Next},
541 {"\"time_unit\": \"ns\",$", MR_Next},
542 {"\"Bat\": %float,$", MR_Next},
543 {"\"Baz\": %float,$", MR_Next},
544 {"\"Foo\": %float$", MR_Next},
545 {"}", MR_Next}});
546 ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet2_Tabular/threads:%int\",%csv_report,"
547 ",%float,%float,%float,,"}});
548 // VS2013 does not allow this function to be passed as a lambda argument
549 // to CHECK_BENCHMARK_RESULTS()
CheckSet2(Results const & e)550 void CheckSet2(Results const& e) {
551 CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 10);
552 CHECK_COUNTER_VALUE(e, int, "Bat", EQ, 30);
553 CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 40);
554 }
555 CHECK_BENCHMARK_RESULTS("BM_CounterSet2_Tabular", &CheckSet2);
556
557 // ========================================================================= //
558 // --------------------------- TEST CASES END ------------------------------ //
559 // ========================================================================= //
560
main(int argc,char * argv[])561 int main(int argc, char* argv[]) { RunOutputTests(argc, argv); }
562