1 
2 #undef NDEBUG
3 
4 #include "benchmark/benchmark.h"
5 #include "output_test.h"
6 
7 // @todo: <jpmag> this checks the full output at once; the rule for
8 // CounterSet1 was failing because it was not matching "^[-]+$".
9 // @todo: <jpmag> check that the counters are vertically aligned.
10 ADD_CASES(TC_ConsoleOut, {
11 // keeping these lines long improves readability, so:
12 // clang-format off
13     {"^[-]+$", MR_Next},
14     {"^Benchmark %s Time %s CPU %s Iterations %s Bar %s Bat %s Baz %s Foo %s Frob %s Lob$", MR_Next},
15     {"^[-]+$", MR_Next},
16     {"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
17     {"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
18     {"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
19     {"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
20     {"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
21     {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
22     {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
23     {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
24     {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
25     {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
26     {"^[-]+$", MR_Next},
27     {"^Benchmark %s Time %s CPU %s Iterations %s Bar %s Baz %s Foo$", MR_Next},
28     {"^[-]+$", MR_Next},
29     {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
30     {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
31     {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
32     {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
33     {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
34     {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
35     {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
36     {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
37     {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
38     {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
39     {"^[-]+$", MR_Next},
40     {"^Benchmark %s Time %s CPU %s Iterations %s Bat %s Baz %s Foo$", MR_Next},
41     {"^[-]+$", MR_Next},
42     {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
43     {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
44     {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
45     {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
46     {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$"},
47 // clang-format on
48 });
49 ADD_CASES(TC_CSVOut, {{"%csv_header,"
50                        "\"Bar\",\"Bat\",\"Baz\",\"Foo\",\"Frob\",\"Lob\""}});
51 
52 // ========================================================================= //
53 // ------------------------- Tabular Counters Output ----------------------- //
54 // ========================================================================= //
55 
BM_Counters_Tabular(benchmark::State & state)56 void BM_Counters_Tabular(benchmark::State& state) {
57   for (auto _ : state) {
58   }
59   namespace bm = benchmark;
60   state.counters.insert({
61     {"Foo",  { 1, bm::Counter::kAvgThreads}},
62     {"Bar",  { 2, bm::Counter::kAvgThreads}},
63     {"Baz",  { 4, bm::Counter::kAvgThreads}},
64     {"Bat",  { 8, bm::Counter::kAvgThreads}},
65     {"Frob", {16, bm::Counter::kAvgThreads}},
66     {"Lob",  {32, bm::Counter::kAvgThreads}},
67   });
68 }
69 BENCHMARK(BM_Counters_Tabular)->ThreadRange(1, 16);
70 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Tabular/threads:%int\",$"},
71                        {"\"iterations\": %int,$", MR_Next},
72                        {"\"real_time\": %float,$", MR_Next},
73                        {"\"cpu_time\": %float,$", MR_Next},
74                        {"\"time_unit\": \"ns\",$", MR_Next},
75                        {"\"Bar\": %float,$", MR_Next},
76                        {"\"Bat\": %float,$", MR_Next},
77                        {"\"Baz\": %float,$", MR_Next},
78                        {"\"Foo\": %float,$", MR_Next},
79                        {"\"Frob\": %float,$", MR_Next},
80                        {"\"Lob\": %float$", MR_Next},
81                        {"}", MR_Next}});
82 ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_Tabular/threads:%int\",%csv_report,"
83                        "%float,%float,%float,%float,%float,%float$"}});
84 // VS2013 does not allow this function to be passed as a lambda argument
85 // to CHECK_BENCHMARK_RESULTS()
CheckTabular(Results const & e)86 void CheckTabular(Results const& e) {
87   CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 1);
88   CHECK_COUNTER_VALUE(e, int, "Bar", EQ, 2);
89   CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 4);
90   CHECK_COUNTER_VALUE(e, int, "Bat", EQ, 8);
91   CHECK_COUNTER_VALUE(e, int, "Frob", EQ, 16);
92   CHECK_COUNTER_VALUE(e, int, "Lob", EQ, 32);
93 }
94 CHECK_BENCHMARK_RESULTS("BM_Counters_Tabular/threads:%int", &CheckTabular);
95 
96 // ========================================================================= //
97 // -------------------- Tabular+Rate Counters Output ----------------------- //
98 // ========================================================================= //
99 
BM_CounterRates_Tabular(benchmark::State & state)100 void BM_CounterRates_Tabular(benchmark::State& state) {
101   for (auto _ : state) {
102   }
103   namespace bm = benchmark;
104   state.counters.insert({
105     {"Foo",  { 1, bm::Counter::kAvgThreadsRate}},
106     {"Bar",  { 2, bm::Counter::kAvgThreadsRate}},
107     {"Baz",  { 4, bm::Counter::kAvgThreadsRate}},
108     {"Bat",  { 8, bm::Counter::kAvgThreadsRate}},
109     {"Frob", {16, bm::Counter::kAvgThreadsRate}},
110     {"Lob",  {32, bm::Counter::kAvgThreadsRate}},
111   });
112 }
113 BENCHMARK(BM_CounterRates_Tabular)->ThreadRange(1, 16);
114 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterRates_Tabular/threads:%int\",$"},
115                        {"\"iterations\": %int,$", MR_Next},
116                        {"\"real_time\": %float,$", MR_Next},
117                        {"\"cpu_time\": %float,$", MR_Next},
118                        {"\"time_unit\": \"ns\",$", MR_Next},
119                        {"\"Bar\": %float,$", MR_Next},
120                        {"\"Bat\": %float,$", MR_Next},
121                        {"\"Baz\": %float,$", MR_Next},
122                        {"\"Foo\": %float,$", MR_Next},
123                        {"\"Frob\": %float,$", MR_Next},
124                        {"\"Lob\": %float$", MR_Next},
125                        {"}", MR_Next}});
126 ADD_CASES(TC_CSVOut, {{"^\"BM_CounterRates_Tabular/threads:%int\",%csv_report,"
127                        "%float,%float,%float,%float,%float,%float$"}});
128 // VS2013 does not allow this function to be passed as a lambda argument
129 // to CHECK_BENCHMARK_RESULTS()
CheckTabularRate(Results const & e)130 void CheckTabularRate(Results const& e) {
131   double t = e.DurationCPUTime();
132   CHECK_FLOAT_COUNTER_VALUE(e, "Foo", EQ, 1./t, 0.001);
133   CHECK_FLOAT_COUNTER_VALUE(e, "Bar", EQ, 2./t, 0.001);
134   CHECK_FLOAT_COUNTER_VALUE(e, "Baz", EQ, 4./t, 0.001);
135   CHECK_FLOAT_COUNTER_VALUE(e, "Bat", EQ, 8./t, 0.001);
136   CHECK_FLOAT_COUNTER_VALUE(e, "Frob", EQ, 16./t, 0.001);
137   CHECK_FLOAT_COUNTER_VALUE(e, "Lob", EQ, 32./t, 0.001);
138 }
139 CHECK_BENCHMARK_RESULTS("BM_CounterRates_Tabular/threads:%int",
140                         &CheckTabularRate);
141 
142 // ========================================================================= //
143 // ------------------------- Tabular Counters Output ----------------------- //
144 // ========================================================================= //
145 
146 // set only some of the counters
BM_CounterSet0_Tabular(benchmark::State & state)147 void BM_CounterSet0_Tabular(benchmark::State& state) {
148   for (auto _ : state) {
149   }
150   namespace bm = benchmark;
151   state.counters.insert({
152     {"Foo", {10, bm::Counter::kAvgThreads}},
153     {"Bar", {20, bm::Counter::kAvgThreads}},
154     {"Baz", {40, bm::Counter::kAvgThreads}},
155   });
156 }
157 BENCHMARK(BM_CounterSet0_Tabular)->ThreadRange(1, 16);
158 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet0_Tabular/threads:%int\",$"},
159                        {"\"iterations\": %int,$", MR_Next},
160                        {"\"real_time\": %float,$", MR_Next},
161                        {"\"cpu_time\": %float,$", MR_Next},
162                        {"\"time_unit\": \"ns\",$", MR_Next},
163                        {"\"Bar\": %float,$", MR_Next},
164                        {"\"Baz\": %float,$", MR_Next},
165                        {"\"Foo\": %float$", MR_Next},
166                        {"}", MR_Next}});
167 ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet0_Tabular/threads:%int\",%csv_report,"
168                        "%float,,%float,%float,,"}});
169 // VS2013 does not allow this function to be passed as a lambda argument
170 // to CHECK_BENCHMARK_RESULTS()
CheckSet0(Results const & e)171 void CheckSet0(Results const& e) {
172   CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 10);
173   CHECK_COUNTER_VALUE(e, int, "Bar", EQ, 20);
174   CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 40);
175 }
176 CHECK_BENCHMARK_RESULTS("BM_CounterSet0_Tabular", &CheckSet0);
177 
178 // again.
BM_CounterSet1_Tabular(benchmark::State & state)179 void BM_CounterSet1_Tabular(benchmark::State& state) {
180   for (auto _ : state) {
181   }
182   namespace bm = benchmark;
183   state.counters.insert({
184     {"Foo", {15, bm::Counter::kAvgThreads}},
185     {"Bar", {25, bm::Counter::kAvgThreads}},
186     {"Baz", {45, bm::Counter::kAvgThreads}},
187   });
188 }
189 BENCHMARK(BM_CounterSet1_Tabular)->ThreadRange(1, 16);
190 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet1_Tabular/threads:%int\",$"},
191                        {"\"iterations\": %int,$", MR_Next},
192                        {"\"real_time\": %float,$", MR_Next},
193                        {"\"cpu_time\": %float,$", MR_Next},
194                        {"\"time_unit\": \"ns\",$", MR_Next},
195                        {"\"Bar\": %float,$", MR_Next},
196                        {"\"Baz\": %float,$", MR_Next},
197                        {"\"Foo\": %float$", MR_Next},
198                        {"}", MR_Next}});
199 ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet1_Tabular/threads:%int\",%csv_report,"
200                        "%float,,%float,%float,,"}});
201 // VS2013 does not allow this function to be passed as a lambda argument
202 // to CHECK_BENCHMARK_RESULTS()
CheckSet1(Results const & e)203 void CheckSet1(Results const& e) {
204   CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 15);
205   CHECK_COUNTER_VALUE(e, int, "Bar", EQ, 25);
206   CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 45);
207 }
208 CHECK_BENCHMARK_RESULTS("BM_CounterSet1_Tabular/threads:%int", &CheckSet1);
209 
210 // ========================================================================= //
211 // ------------------------- Tabular Counters Output ----------------------- //
212 // ========================================================================= //
213 
214 // set only some of the counters, different set now.
BM_CounterSet2_Tabular(benchmark::State & state)215 void BM_CounterSet2_Tabular(benchmark::State& state) {
216   for (auto _ : state) {
217   }
218   namespace bm = benchmark;
219   state.counters.insert({
220     {"Foo", {10, bm::Counter::kAvgThreads}},
221     {"Bat", {30, bm::Counter::kAvgThreads}},
222     {"Baz", {40, bm::Counter::kAvgThreads}},
223   });
224 }
225 BENCHMARK(BM_CounterSet2_Tabular)->ThreadRange(1, 16);
226 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet2_Tabular/threads:%int\",$"},
227                        {"\"iterations\": %int,$", MR_Next},
228                        {"\"real_time\": %float,$", MR_Next},
229                        {"\"cpu_time\": %float,$", MR_Next},
230                        {"\"time_unit\": \"ns\",$", MR_Next},
231                        {"\"Bat\": %float,$", MR_Next},
232                        {"\"Baz\": %float,$", MR_Next},
233                        {"\"Foo\": %float$", MR_Next},
234                        {"}", MR_Next}});
235 ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet2_Tabular/threads:%int\",%csv_report,"
236                        ",%float,%float,%float,,"}});
237 // VS2013 does not allow this function to be passed as a lambda argument
238 // to CHECK_BENCHMARK_RESULTS()
CheckSet2(Results const & e)239 void CheckSet2(Results const& e) {
240   CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 10);
241   CHECK_COUNTER_VALUE(e, int, "Bat", EQ, 30);
242   CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 40);
243 }
244 CHECK_BENCHMARK_RESULTS("BM_CounterSet2_Tabular", &CheckSet2);
245 
246 // ========================================================================= //
247 // --------------------------- TEST CASES END ------------------------------ //
248 // ========================================================================= //
249 
main(int argc,char * argv[])250 int main(int argc, char* argv[]) { RunOutputTests(argc, argv); }
251