1#!/usr/bin/env python3
2
3""" Automatically generate API headers from a specification of the JSON
4    messsages that are part of Measurement Kit's FFI API. """
5
6import jinja2
7import json
8import re
9import subprocess
10import sys
11
12class Attribute(object):
13    """ Attribute of a class. """
14
15    def __init__(self, cxx_type, key, value=None):
16        self.cxx_type = cxx_type
17        self.key = key
18        self.value = value if value is not None else {
19            "bool": "false",
20            "double": "0.0",
21            "int64_t": "0",
22            "std::string": json.dumps(""),
23        }.get(cxx_type, "{}")
24        # We need to know the JSON type to tell our JSON library what is
25        # the expected type of specific settings or options.
26        self.json_type = {
27            "Options": "object",
28            "bool": "boolean",
29            "double": "number_float",
30            "int64_t": "number_integer",
31            "std::map<std::string, std::string>": "object",
32            "std::string": "string",
33            "std::vector<std::string>": "array",
34        }[cxx_type]
35
36class Setting(Attribute):
37    """ Attribute of the Setting class. """
38
39    def __init__(self, cxx_type, key, value=None, mandatory=False):
40        Attribute.__init__(self, cxx_type, key, value)
41        self.mandatory = mandatory
42
43class Event(object):
44    """ Event emitted by FFI API while running tests. """
45
46    def __init__(self, key, *attributes):
47        self.key = key
48        self.cxx_class_name = "".join([e.capitalize() for e in key.replace(".", "_").split("_")])
49        self.attributes = list(attributes)
50
51class Nettest(object):
52    """ Network test runnable using the FFI API. """
53
54    def __init__(self, snake_case_name, *attributes):
55        self.snake_case_name = snake_case_name
56        self.name = "".join([e.capitalize() for e in snake_case_name.replace(".", "_").split("_")])
57        self.attributes = list(attributes)
58
59def main():
60    """ Main function """
61
62    events = [Event("bug.json_dump",
63                    Attribute("std::string", "failure"),
64                    Attribute("std::string", "orig_key")),
65
66              Event("failure.asn_lookup",
67                    Attribute("std::string", "failure")),
68
69              Event("failure.cc_lookup",
70                    Attribute("std::string", "failure")),
71
72              Event("failure.ip_lookup",
73                    Attribute("std::string", "failure")),
74
75              Event("failure.measurement",
76                    Attribute("std::string", "failure")),
77
78              Event("failure.measurement_submission",
79                    Attribute("std::string", "failure"),
80                    Attribute("int64_t", "idx"),
81                    Attribute("std::string", "json_str")),
82
83              Event("failure.network_name_lookup",
84                    Attribute("std::string", "failure")),
85
86              Event("failure.report_create",
87                    Attribute("std::string", "failure")),
88
89              Event("failure.report_close",
90                    Attribute("std::string", "failure")),
91
92              Event("failure.resolver_lookup",
93                    Attribute("std::string", "failure")),
94
95              Event("failure.startup",
96                    Attribute("std::string", "failure")),
97
98              Event("log",
99                    Attribute("std::string", "log_level"),
100                    Attribute("std::string", "message")),
101
102              Event("measurement",
103                    Attribute("int64_t", "idx"),
104                    Attribute("std::string", "json_str")),
105
106              Event("status.end",
107                    Attribute("double", "downloaded_kb"),
108                    Attribute("double", "uploaded_kb"),
109                    Attribute("std::string", "failure")),
110
111              Event("status.geoip_lookup",
112                    Attribute("std::string", "probe_ip"),
113                    Attribute("std::string", "probe_asn"),
114                    Attribute("std::string", "probe_cc"),
115                    Attribute("std::string", "probe_network_name")),
116
117              Event("status.progress",
118                    Attribute("double", "percentage"),
119                    Attribute("std::string", "message")),
120
121              Event("status.queued"),
122
123              Event("status.measurement_start",
124                    Attribute("int64_t", "idx"),
125                    Attribute("std::string", "input")),
126
127              Event("status.measurement_submission",
128                    Attribute("int64_t", "idx")),
129
130              Event("status.measurement_done",
131                    Attribute("int64_t", "idx")),
132
133              Event("status.report_close",
134                    Attribute("std::string", "report_id")),
135
136              Event("status.report_create",
137                    Attribute("std::string", "report_id")),
138
139              Event("status.resolver_lookup",
140                    Attribute("std::string", "ip_address")),
141
142              Event("status.started"),
143
144              Event("status.update.performance",
145                    Attribute("std::string", "direction"),
146                    Attribute("double", "elapsed"),
147                    Attribute("int64_t", "num_streams"),
148                    Attribute("double", "speed_kbps")),
149
150              Event("status.update.websites",
151                    Attribute("std::string", "url"),
152                    Attribute("std::string", "status")),
153
154              Event("task_terminated")]
155
156    log_levels = ["err", "warning", "info", "debug", "debug2"]
157
158    settings = [Setting("std::map<std::string, std::string>", "annotations"),
159
160                Setting("std::vector<std::string>", "disabled_events"),
161
162                Setting("std::vector<std::string>", "inputs"),
163
164                Setting("std::vector<std::string>", "input_filepaths"),
165
166                Setting("std::string", "log_filepath"),
167
168                Setting("std::string", "log_level", "log_level_err"),
169
170                Setting("std::string", "name", mandatory=True),
171
172                Setting("Options", "options"),
173
174                Setting("std::string", "output_filepath")]
175
176    options = [Attribute("std::string", "address"),
177               Attribute("bool", "all_endpoints", "false"),
178               Attribute("std::string", "backend"),
179               Attribute("std::string", "bouncer_base_url", json.dumps("https://ps1.ooni.io")),
180               Attribute("std::string", "collector_base_url"),
181               Attribute("int64_t", "constant_bitrate", "0"),
182               Attribute("std::string", "dns/nameserver"),
183               Attribute("std::string", "dns/engine", json.dumps("system")),
184               Attribute("std::string", "expected_body"),
185               Attribute("std::string", "geoip_asn_path"),
186               Attribute("std::string", "geoip_country_path"),
187               Attribute("std::string", "hostname"),
188               Attribute("bool", "ignore_bouncer_error", "true"),
189               Attribute("bool", "ignore_open_report_error", "true"),
190               Attribute("int64_t", "max_runtime", "-1"),
191               Attribute("std::string", "mlabns/address_family"),
192               Attribute("std::string", "mlabns/base_url"),
193               Attribute("std::string", "mlabns/country"),
194               Attribute("std::string", "mlabns/metro"),
195               Attribute("std::string", "mlabns/policy"),
196               Attribute("std::string", "mlabns_tool_name"),
197               Attribute("std::string", "net/ca_bundle_path"),
198               Attribute("double", "net/timeout", "10.0"),
199               Attribute("bool", "no_bouncer", "false"),
200               Attribute("bool", "no_collector", "false"),
201               Attribute("bool", "no_file_report", "false"),
202               Attribute("bool", "no_geoip", "false"),
203               Attribute("bool", "no_resolver_lookup", "false"),
204               Attribute("int64_t", "port", "0"),
205               Attribute("std::string", "probe_ip"),
206               Attribute("std::string", "probe_asn"),
207               Attribute("std::string", "probe_cc"),
208               Attribute("std::string", "probe_network_name"),
209               Attribute("bool", "randomize_input", "true"),
210               Attribute("bool", "save_real_probe_asn", "true"),
211               Attribute("bool", "save_real_probe_cc", "true"),
212               Attribute("bool", "save_real_probe_ip", "false"),
213               Attribute("bool", "save_real_probe_network_name", "false"),
214               Attribute("bool", "save_real_resolver_ip", "true"),
215               Attribute("std::string", "server", ""),
216               Attribute("std::string", "software_name"),
217               Attribute("std::string", "software_version"),
218               Attribute("int64_t", "test_suite"),
219               Attribute("std::string", "uuid")]
220
221    nettests = [Nettest("captive_portal"),
222                Nettest("dash"),
223                Nettest("dns_injection"),
224                Nettest("facebook_messenger"),
225                Nettest("http_header_field_manipulation"),
226                Nettest("http_invalid_request_line"),
227                Nettest("meek_fronted_requests"),
228                Nettest("ndt"),
229                Nettest("tcp_connect"),
230                Nettest("telegram"),
231                Nettest("web_connectivity"),
232                Nettest("whatsapp",
233                        Attribute("bool", "all_endpoints", "false"))]
234
235    with open("./script/autoapi/nettests.hpp.j2", "r") as sourcefp:
236        tpl = jinja2.Template(sourcefp.read())
237        render = tpl.render(events=events, log_levels=log_levels,
238                            settings=settings, options=options,
239                            nettests=nettests)
240        with open("./include/measurement_kit/nettests.hpp", "w") as destfp:
241            destfp.write(render)
242
243    with open("./script/autoapi/engine.cpp.j2", "r") as sourcefp:
244        tpl = jinja2.Template(sourcefp.read())
245        render = tpl.render(events=events, log_levels=log_levels,
246                            settings=settings, options=options,
247                            nettests=nettests)
248        with open("./src/libmeasurement_kit/engine/autoapi.cpp", "w") as destfp:
249            destfp.write(render)
250
251    for arg in sys.argv[1:]:
252        if arg == "-c":
253            result = subprocess.check_output(["git", "diff"])
254            if result != "":
255                raise RuntimeError("Need to commit changes\n\n%s" % result)
256
257if __name__ == "__main__":
258    main()
259