1 #ifndef TEST_OUTPUT_TEST_H
2 #define TEST_OUTPUT_TEST_H
3
4 #undef NDEBUG
5 #include <initializer_list>
6 #include <memory>
7 #include <string>
8 #include <utility>
9 #include <vector>
10 #include <functional>
11 #include <sstream>
12
13 #include "../src/re.h"
14 #include "benchmark/benchmark.h"
15
16 #define CONCAT2(x, y) x##y
17 #define CONCAT(x, y) CONCAT2(x, y)
18
19 #define ADD_CASES(...) int CONCAT(dummy, __LINE__) = ::AddCases(__VA_ARGS__)
20
21 #define SET_SUBSTITUTIONS(...) \
22 int CONCAT(dummy, __LINE__) = ::SetSubstitutions(__VA_ARGS__)
23
24 enum MatchRules {
25 MR_Default, // Skip non-matching lines until a match is found.
26 MR_Next, // Match must occur on the next line.
27 MR_Not // No line between the current position and the next match matches
28 // the regex
29 };
30
31 struct TestCase {
32 TestCase(std::string re, int rule = MR_Default);
33
34 std::string regex_str;
35 int match_rule;
36 std::string substituted_regex;
37 std::shared_ptr<benchmark::Regex> regex;
38 };
39
40 enum TestCaseID {
41 TC_ConsoleOut,
42 TC_ConsoleErr,
43 TC_JSONOut,
44 TC_JSONErr,
45 TC_CSVOut,
46 TC_CSVErr,
47
48 TC_NumID // PRIVATE
49 };
50
51 // Add a list of test cases to be run against the output specified by
52 // 'ID'
53 int AddCases(TestCaseID ID, std::initializer_list<TestCase> il);
54
55 // Add or set a list of substitutions to be performed on constructed regex's
56 // See 'output_test_helper.cc' for a list of default substitutions.
57 int SetSubstitutions(
58 std::initializer_list<std::pair<std::string, std::string>> il);
59
60 // Run all output tests.
61 void RunOutputTests(int argc, char* argv[]);
62
63 // ========================================================================= //
64 // ------------------------- Results checking ------------------------------ //
65 // ========================================================================= //
66
67 // Call this macro to register a benchmark for checking its results. This
68 // should be all that's needed. It subscribes a function to check the (CSV)
69 // results of a benchmark. This is done only after verifying that the output
70 // strings are really as expected.
71 // bm_name_pattern: a name or a regex pattern which will be matched against
72 // all the benchmark names. Matching benchmarks
73 // will be the subject of a call to checker_function
74 // checker_function: should be of type ResultsCheckFn (see below)
75 #define CHECK_BENCHMARK_RESULTS(bm_name_pattern, checker_function) \
76 size_t CONCAT(dummy, __LINE__) = AddChecker(bm_name_pattern, checker_function)
77
78 struct Results;
79 typedef std::function< void(Results const&) > ResultsCheckFn;
80
81 size_t AddChecker(const char* bm_name_pattern, ResultsCheckFn fn);
82
83 // Class holding the results of a benchmark.
84 // It is passed in calls to checker functions.
85 struct Results {
86
87 // the benchmark name
88 std::string name;
89 // the benchmark fields
90 std::map< std::string, std::string > values;
91
ResultsResults92 Results(const std::string& n) : name(n) {}
93
94 int NumThreads() const;
95
96 typedef enum { kCpuTime, kRealTime } BenchmarkTime;
97
98 // get cpu_time or real_time in seconds
99 double GetTime(BenchmarkTime which) const;
100
101 // get the real_time duration of the benchmark in seconds.
102 // it is better to use fuzzy float checks for this, as the float
103 // ASCII formatting is lossy.
DurationRealTimeResults104 double DurationRealTime() const {
105 return GetAs< double >("iterations") * GetTime(kRealTime);
106 }
107 // get the cpu_time duration of the benchmark in seconds
DurationCPUTimeResults108 double DurationCPUTime() const {
109 return GetAs< double >("iterations") * GetTime(kCpuTime);
110 }
111
112 // get the string for a result by name, or nullptr if the name
113 // is not found
GetResults114 const std::string* Get(const char* entry_name) const {
115 auto it = values.find(entry_name);
116 if(it == values.end()) return nullptr;
117 return &it->second;
118 }
119
120 // get a result by name, parsed as a specific type.
121 // NOTE: for counters, use GetCounterAs instead.
122 template <class T>
123 T GetAs(const char* entry_name) const;
124
125 // counters are written as doubles, so they have to be read first
126 // as a double, and only then converted to the asked type.
127 template <class T>
GetCounterAsResults128 T GetCounterAs(const char* entry_name) const {
129 double dval = GetAs< double >(entry_name);
130 T tval = static_cast< T >(dval);
131 return tval;
132 }
133 };
134
135 template <class T>
GetAs(const char * entry_name)136 T Results::GetAs(const char* entry_name) const {
137 auto *sv = Get(entry_name);
138 CHECK(sv != nullptr && !sv->empty());
139 std::stringstream ss;
140 ss << *sv;
141 T out;
142 ss >> out;
143 CHECK(!ss.fail());
144 return out;
145 }
146
147 //----------------------------------
148 // Macros to help in result checking. Do not use them with arguments causing
149 // side-effects.
150
151 #define _CHECK_RESULT_VALUE(entry, getfn, var_type, var_name, relationship, value) \
152 CONCAT(CHECK_, relationship) \
153 (entry.getfn< var_type >(var_name), (value)) << "\n" \
154 << __FILE__ << ":" << __LINE__ << ": " << (entry).name << ":\n" \
155 << __FILE__ << ":" << __LINE__ << ": " \
156 << "expected (" << #var_type << ")" << (var_name) \
157 << "=" << (entry).getfn< var_type >(var_name) \
158 << " to be " #relationship " to " << (value) << "\n"
159
160 // check with tolerance. eps_factor is the tolerance window, which is
161 // interpreted relative to value (eg, 0.1 means 10% of value).
162 #define _CHECK_FLOAT_RESULT_VALUE(entry, getfn, var_type, var_name, relationship, value, eps_factor) \
163 CONCAT(CHECK_FLOAT_, relationship) \
164 (entry.getfn< var_type >(var_name), (value), (eps_factor) * (value)) << "\n" \
165 << __FILE__ << ":" << __LINE__ << ": " << (entry).name << ":\n" \
166 << __FILE__ << ":" << __LINE__ << ": " \
167 << "expected (" << #var_type << ")" << (var_name) \
168 << "=" << (entry).getfn< var_type >(var_name) \
169 << " to be " #relationship " to " << (value) << "\n" \
170 << __FILE__ << ":" << __LINE__ << ": " \
171 << "with tolerance of " << (eps_factor) * (value) \
172 << " (" << (eps_factor)*100. << "%), " \
173 << "but delta was " << ((entry).getfn< var_type >(var_name) - (value)) \
174 << " (" << (((entry).getfn< var_type >(var_name) - (value)) \
175 / \
176 ((value) > 1.e-5 || value < -1.e-5 ? value : 1.e-5)*100.) \
177 << "%)"
178
179 #define CHECK_RESULT_VALUE(entry, var_type, var_name, relationship, value) \
180 _CHECK_RESULT_VALUE(entry, GetAs, var_type, var_name, relationship, value)
181
182 #define CHECK_COUNTER_VALUE(entry, var_type, var_name, relationship, value) \
183 _CHECK_RESULT_VALUE(entry, GetCounterAs, var_type, var_name, relationship, value)
184
185 #define CHECK_FLOAT_RESULT_VALUE(entry, var_name, relationship, value, eps_factor) \
186 _CHECK_FLOAT_RESULT_VALUE(entry, GetAs, double, var_name, relationship, value, eps_factor)
187
188 #define CHECK_FLOAT_COUNTER_VALUE(entry, var_name, relationship, value, eps_factor) \
189 _CHECK_FLOAT_RESULT_VALUE(entry, GetCounterAs, double, var_name, relationship, value, eps_factor)
190
191 // ========================================================================= //
192 // --------------------------- Misc Utilities ------------------------------ //
193 // ========================================================================= //
194
195 namespace {
196
197 const char* const dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?";
198
199 } // end namespace
200
201 #endif // TEST_OUTPUT_TEST_H
202