1 #include <cstdio>
2 #include <iostream>
3 #include <memory>
4 #include <stdexcept>
5 #include <string>
6 #include <sstream>
7 #include <array>
8 #include <algorithm>
9 #include <vector>
10 #include <cmath>
11 
12 #ifdef _WIN32
13 #define popen _popen
14 #define pclose _pclose
15 #endif
16 
closepipe(FILE * pipe)17 int closepipe(FILE *pipe) {
18     int exit_code = pclose(pipe);
19     if (exit_code != EXIT_SUCCESS) {
20         std::cerr << "Error " << exit_code << " running benchmark command!" << std::endl;
21         exit(EXIT_FAILURE);
22     };
23     return exit_code;
24 }
25 
exec(const char * cmd)26 std::string exec(const char* cmd) {
27     std::cerr << cmd << std::endl;
28     std::array<char, 128> buffer;
29     std::string result;
30     std::unique_ptr<FILE, decltype(&closepipe)> pipe(popen(cmd, "r"), closepipe);
31     if (!pipe) {
32         std::cerr << "popen() failed!" << std::endl;
33         abort();
34     }
35     while (fgets(buffer.data(), int(buffer.size()), pipe.get()) != nullptr) {
36         result += buffer.data();
37     }
38     return result;
39 }
40 
readThroughput(std::string parseOutput)41 double readThroughput(std::string parseOutput) {
42     std::istringstream output(parseOutput);
43     std::string line;
44     double result = 0;
45     int numResults = 0;
46     while (std::getline(output, line)) {
47         std::string::size_type pos = 0;
48         for (int i=0; i<5; i++) {
49             pos = line.find('\t', pos);
50             if (pos == std::string::npos) {
51                 std::cerr << "Command printed out a line with less than 5 fields in it:\n" << line << std::endl;
52             }
53             pos++;
54         }
55         result += std::stod(line.substr(pos));
56         numResults++;
57     }
58     if (numResults == 0) {
59         std::cerr << "No results returned from benchmark command!" << std::endl;
60         exit(EXIT_FAILURE);
61     }
62     return result / numResults;
63 }
64 
65 const double INTERLEAVED_ATTEMPTS = 7;
66 
main(int argc,const char * argv[])67 int main(int argc, const char *argv[]) {
68     if (argc < 3) {
69         std::cerr << "Usage: " << argv[0] << " <old parse exe> <new parse exe> [<parse arguments>]" << std::endl;
70         return 1;
71     }
72 
73     std::string newCommand = argv[1];
74     std::string refCommand = argv[2];
75     for (int i=3; i<argc; i++) {
76         newCommand += " ";
77         newCommand += argv[i];
78         refCommand += " ";
79         refCommand += argv[i];
80     }
81 
82     std::vector<double> ref;
83     std::vector<double> newcode;
84     for (int attempt=0; attempt < INTERLEAVED_ATTEMPTS; attempt++) {
85         std::cout << "Attempt #" << (attempt+1) << " of " << INTERLEAVED_ATTEMPTS << std::endl;
86 
87         // Read new throughput
88         double newThroughput = readThroughput(exec(newCommand.c_str()));
89         std::cout << "New throughput: " << newThroughput << std::endl;
90         newcode.push_back(newThroughput);
91 
92         // Read reference throughput
93         double referenceThroughput = readThroughput(exec(refCommand.c_str()));
94         std::cout << "Ref throughput: " << referenceThroughput << std::endl;
95         ref.push_back(referenceThroughput);
96     }
97     // we check if the maximum of newcode is lower than minimum of ref, if so we have a problem so fail!
98     double worseref = *std::min_element(ref.begin(), ref.end());
99     double bestnewcode =  *std::max_element(newcode.begin(), newcode.end());
100     double bestref = *std::max_element(ref.begin(), ref.end());
101     double worsenewcode =  *std::min_element(newcode.begin(), newcode.end());
102     std::cout << "The new code has a throughput in       " << worsenewcode << " -- " << bestnewcode << std::endl;
103     std::cout << "The reference code has a throughput in " << worseref << " -- " << bestref << std::endl;
104     if(bestnewcode < worseref) {
105       std::cerr << "You probably have a performance degradation." << std::endl;
106       return EXIT_FAILURE;
107     }
108     if(bestnewcode < worseref) {
109       std::cout << "You probably have a performance gain." << std::endl;
110       return EXIT_SUCCESS;
111     }
112     std::cout << "There is no obvious performance difference. A manual check might be needed." << std::endl;
113     return EXIT_SUCCESS;
114 }
115