1#!/usr/bin/env python3 2 3""" Automatically generate API headers from a specification of the JSON 4 messsages that are part of Measurement Kit's FFI API. """ 5 6import jinja2 7import json 8import re 9import subprocess 10import sys 11 12class Attribute(object): 13 """ Attribute of a class. """ 14 15 def __init__(self, cxx_type, key, value=None): 16 self.cxx_type = cxx_type 17 self.key = key 18 self.value = value if value is not None else { 19 "bool": "false", 20 "double": "0.0", 21 "int64_t": "0", 22 "std::string": json.dumps(""), 23 }.get(cxx_type, "{}") 24 # We need to know the JSON type to tell our JSON library what is 25 # the expected type of specific settings or options. 26 self.json_type = { 27 "Options": "object", 28 "bool": "boolean", 29 "double": "number_float", 30 "int64_t": "number_integer", 31 "std::map<std::string, std::string>": "object", 32 "std::string": "string", 33 "std::vector<std::string>": "array", 34 }[cxx_type] 35 36class Setting(Attribute): 37 """ Attribute of the Setting class. """ 38 39 def __init__(self, cxx_type, key, value=None, mandatory=False): 40 Attribute.__init__(self, cxx_type, key, value) 41 self.mandatory = mandatory 42 43class Event(object): 44 """ Event emitted by FFI API while running tests. """ 45 46 def __init__(self, key, *attributes): 47 self.key = key 48 self.cxx_class_name = "".join([e.capitalize() for e in key.replace(".", "_").split("_")]) 49 self.attributes = list(attributes) 50 51class Nettest(object): 52 """ Network test runnable using the FFI API. """ 53 54 def __init__(self, snake_case_name, *attributes): 55 self.snake_case_name = snake_case_name 56 self.name = "".join([e.capitalize() for e in snake_case_name.replace(".", "_").split("_")]) 57 self.attributes = list(attributes) 58 59def main(): 60 """ Main function """ 61 62 events = [Event("bug.json_dump", 63 Attribute("std::string", "failure"), 64 Attribute("std::string", "orig_key")), 65 66 Event("failure.asn_lookup", 67 Attribute("std::string", "failure")), 68 69 Event("failure.cc_lookup", 70 Attribute("std::string", "failure")), 71 72 Event("failure.ip_lookup", 73 Attribute("std::string", "failure")), 74 75 Event("failure.measurement", 76 Attribute("std::string", "failure")), 77 78 Event("failure.measurement_submission", 79 Attribute("std::string", "failure"), 80 Attribute("int64_t", "idx"), 81 Attribute("std::string", "json_str")), 82 83 Event("failure.network_name_lookup", 84 Attribute("std::string", "failure")), 85 86 Event("failure.report_create", 87 Attribute("std::string", "failure")), 88 89 Event("failure.report_close", 90 Attribute("std::string", "failure")), 91 92 Event("failure.resolver_lookup", 93 Attribute("std::string", "failure")), 94 95 Event("failure.startup", 96 Attribute("std::string", "failure")), 97 98 Event("log", 99 Attribute("std::string", "log_level"), 100 Attribute("std::string", "message")), 101 102 Event("measurement", 103 Attribute("int64_t", "idx"), 104 Attribute("std::string", "json_str")), 105 106 Event("status.end", 107 Attribute("double", "downloaded_kb"), 108 Attribute("double", "uploaded_kb"), 109 Attribute("std::string", "failure")), 110 111 Event("status.geoip_lookup", 112 Attribute("std::string", "probe_ip"), 113 Attribute("std::string", "probe_asn"), 114 Attribute("std::string", "probe_cc"), 115 Attribute("std::string", "probe_network_name")), 116 117 Event("status.progress", 118 Attribute("double", "percentage"), 119 Attribute("std::string", "message")), 120 121 Event("status.queued"), 122 123 Event("status.measurement_start", 124 Attribute("int64_t", "idx"), 125 Attribute("std::string", "input")), 126 127 Event("status.measurement_submission", 128 Attribute("int64_t", "idx")), 129 130 Event("status.measurement_done", 131 Attribute("int64_t", "idx")), 132 133 Event("status.report_close", 134 Attribute("std::string", "report_id")), 135 136 Event("status.report_create", 137 Attribute("std::string", "report_id")), 138 139 Event("status.resolver_lookup", 140 Attribute("std::string", "ip_address")), 141 142 Event("status.started"), 143 144 Event("status.update.performance", 145 Attribute("std::string", "direction"), 146 Attribute("double", "elapsed"), 147 Attribute("int64_t", "num_streams"), 148 Attribute("double", "speed_kbps")), 149 150 Event("status.update.websites", 151 Attribute("std::string", "url"), 152 Attribute("std::string", "status")), 153 154 Event("task_terminated")] 155 156 log_levels = ["err", "warning", "info", "debug", "debug2"] 157 158 settings = [Setting("std::map<std::string, std::string>", "annotations"), 159 160 Setting("std::vector<std::string>", "disabled_events"), 161 162 Setting("std::vector<std::string>", "inputs"), 163 164 Setting("std::vector<std::string>", "input_filepaths"), 165 166 Setting("std::string", "log_filepath"), 167 168 Setting("std::string", "log_level", "log_level_err"), 169 170 Setting("std::string", "name", mandatory=True), 171 172 Setting("Options", "options"), 173 174 Setting("std::string", "output_filepath")] 175 176 options = [Attribute("std::string", "address"), 177 Attribute("bool", "all_endpoints", "false"), 178 Attribute("std::string", "backend"), 179 Attribute("std::string", "bouncer_base_url", json.dumps("https://ps1.ooni.io")), 180 Attribute("std::string", "collector_base_url"), 181 Attribute("int64_t", "constant_bitrate", "0"), 182 Attribute("std::string", "dns/nameserver"), 183 Attribute("std::string", "dns/engine", json.dumps("system")), 184 Attribute("std::string", "expected_body"), 185 Attribute("std::string", "geoip_asn_path"), 186 Attribute("std::string", "geoip_country_path"), 187 Attribute("std::string", "hostname"), 188 Attribute("bool", "ignore_bouncer_error", "true"), 189 Attribute("bool", "ignore_open_report_error", "true"), 190 Attribute("int64_t", "max_runtime", "-1"), 191 Attribute("std::string", "mlabns/address_family"), 192 Attribute("std::string", "mlabns/base_url"), 193 Attribute("std::string", "mlabns/country"), 194 Attribute("std::string", "mlabns/metro"), 195 Attribute("std::string", "mlabns/policy"), 196 Attribute("std::string", "mlabns_tool_name"), 197 Attribute("std::string", "net/ca_bundle_path"), 198 Attribute("double", "net/timeout", "10.0"), 199 Attribute("bool", "no_bouncer", "false"), 200 Attribute("bool", "no_collector", "false"), 201 Attribute("bool", "no_file_report", "false"), 202 Attribute("bool", "no_geoip", "false"), 203 Attribute("bool", "no_resolver_lookup", "false"), 204 Attribute("int64_t", "port", "0"), 205 Attribute("std::string", "probe_ip"), 206 Attribute("std::string", "probe_asn"), 207 Attribute("std::string", "probe_cc"), 208 Attribute("std::string", "probe_network_name"), 209 Attribute("bool", "randomize_input", "true"), 210 Attribute("bool", "save_real_probe_asn", "true"), 211 Attribute("bool", "save_real_probe_cc", "true"), 212 Attribute("bool", "save_real_probe_ip", "false"), 213 Attribute("bool", "save_real_probe_network_name", "false"), 214 Attribute("bool", "save_real_resolver_ip", "true"), 215 Attribute("std::string", "server", ""), 216 Attribute("std::string", "software_name"), 217 Attribute("std::string", "software_version"), 218 Attribute("int64_t", "test_suite"), 219 Attribute("std::string", "uuid")] 220 221 nettests = [Nettest("captive_portal"), 222 Nettest("dash"), 223 Nettest("dns_injection"), 224 Nettest("facebook_messenger"), 225 Nettest("http_header_field_manipulation"), 226 Nettest("http_invalid_request_line"), 227 Nettest("meek_fronted_requests"), 228 Nettest("ndt"), 229 Nettest("tcp_connect"), 230 Nettest("telegram"), 231 Nettest("web_connectivity"), 232 Nettest("whatsapp", 233 Attribute("bool", "all_endpoints", "false"))] 234 235 with open("./script/autoapi/nettests.hpp.j2", "r") as sourcefp: 236 tpl = jinja2.Template(sourcefp.read()) 237 render = tpl.render(events=events, log_levels=log_levels, 238 settings=settings, options=options, 239 nettests=nettests) 240 with open("./include/measurement_kit/nettests.hpp", "w") as destfp: 241 destfp.write(render) 242 243 with open("./script/autoapi/engine.cpp.j2", "r") as sourcefp: 244 tpl = jinja2.Template(sourcefp.read()) 245 render = tpl.render(events=events, log_levels=log_levels, 246 settings=settings, options=options, 247 nettests=nettests) 248 with open("./src/libmeasurement_kit/engine/autoapi.cpp", "w") as destfp: 249 destfp.write(render) 250 251 for arg in sys.argv[1:]: 252 if arg == "-c": 253 result = subprocess.check_output(["git", "diff"]) 254 if result != "": 255 raise RuntimeError("Need to commit changes\n\n%s" % result) 256 257if __name__ == "__main__": 258 main() 259