1# This Source Code Form is subject to the terms of the Mozilla Public 2# License, v. 2.0. If a copy of the MPL was not distributed with this 3# file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 4from __future__ import print_function, unicode_literals, division 5 6import os 7import posixpath 8import sys 9import tempfile 10 11from datetime import timedelta 12from mozdevice import ADBDevice, ADBError, ADBTimeoutError, ADBProcessError 13 14from .results import TestOutput, escape_cmdline 15from .adaptor import xdr_annotate 16from .remote import init_device 17 18TESTS_LIB_DIR = os.path.dirname(os.path.abspath(__file__)) 19JS_DIR = os.path.dirname(os.path.dirname(TESTS_LIB_DIR)) 20JS_TESTS_DIR = posixpath.join(JS_DIR, "tests") 21TEST_DIR = os.path.join(JS_DIR, "jit-test", "tests") 22 23 24def aggregate_script_stdout(stdout_lines, prefix, tempdir, uniq_tag, tests, options): 25 test = None 26 tStart = None 27 cmd = "" 28 stdout = "" 29 30 # Use to debug this script in case of assertion failure. 31 meta_history = [] 32 last_line = "" 33 34 # Assert that the streamed content is not interrupted. 35 ended = False 36 37 # Check if the tag is present, if so, this is controlled output 38 # produced by the test runner, otherwise this is stdout content. 39 try: 40 for line in stdout_lines: 41 last_line = line 42 if line.startswith(uniq_tag): 43 meta = line[len(uniq_tag) :].strip() 44 meta_history.append(meta) 45 if meta.startswith("START="): 46 assert test is None 47 params = meta[len("START=") :].split(",") 48 test_idx = int(params[0]) 49 test = tests[test_idx] 50 tStart = timedelta(seconds=float(params[1])) 51 cmd = test.command( 52 prefix, 53 posixpath.join(options.remote_test_root, "lib/"), 54 posixpath.join(options.remote_test_root, "modules/"), 55 tempdir, 56 posixpath.join(options.remote_test_root, "tests"), 57 ) 58 stdout = "" 59 if options.show_cmd: 60 print(escape_cmdline(cmd)) 61 elif meta.startswith("STOP="): 62 assert test is not None 63 params = meta[len("STOP=") :].split(",") 64 exitcode = int(params[0]) 65 dt = timedelta(seconds=float(params[1])) - tStart 66 yield TestOutput( 67 test, 68 cmd, 69 stdout, 70 # NOTE: mozdevice fuse stdout and stderr. Thus, we are 71 # using stdout for both stdout and stderr. So far, 72 # doing so did not cause any issues. 73 stdout, 74 exitcode, 75 dt.total_seconds(), 76 dt > timedelta(seconds=int(options.timeout)), 77 ) 78 stdout = "" 79 cmd = "" 80 test = None 81 elif meta.startswith("RETRY="): 82 # On timeout, we discard the first timeout to avoid a 83 # random hang on pthread_join. 84 assert test is not None 85 stdout = "" 86 cmd = "" 87 test = None 88 else: 89 assert meta.startswith("THE_END") 90 ended = True 91 else: 92 assert uniq_tag not in line 93 stdout += line 94 95 # This assertion fails if the streamed content is interrupted, either 96 # by unplugging the phone or some adb failures. 97 assert ended 98 except AssertionError as e: 99 sys.stderr.write("Metadata history:\n{}\n".format("\n".join(meta_history))) 100 sys.stderr.write("Last line: {}\n".format(last_line)) 101 raise e 102 103 104def setup_device(prefix, options): 105 try: 106 device = init_device(options) 107 108 def replace_lib_file(path, name): 109 localfile = os.path.join(JS_TESTS_DIR, *path) 110 remotefile = posixpath.join(options.remote_test_root, "lib", name) 111 device.push(localfile, remotefile, timeout=10) 112 113 prefix[0] = posixpath.join(options.remote_test_root, "bin", "js") 114 tempdir = posixpath.join(options.remote_test_root, "tmp") 115 116 # Push tests & lib directories. 117 device.push(os.path.dirname(TEST_DIR), options.remote_test_root, timeout=600) 118 119 # Substitute lib files which are aliasing non262 files. 120 replace_lib_file(["non262", "shell.js"], "non262.js") 121 replace_lib_file(["non262", "reflect-parse", "Match.js"], "match.js") 122 replace_lib_file(["non262", "Math", "shell.js"], "math.js") 123 device.chmod(options.remote_test_root, recursive=True) 124 125 print("tasks_adb_remote.py : Device initialization completed") 126 return device, tempdir 127 except (ADBError, ADBTimeoutError): 128 print( 129 "TEST-UNEXPECTED-FAIL | tasks_adb_remote.py : " 130 + "Device initialization failed" 131 ) 132 raise 133 134 135def script_preamble(tag, prefix, options): 136 timeout = int(options.timeout) 137 retry = int(options.timeout_retry) 138 lib_path = os.path.dirname(prefix[0]) 139 return """ 140export LD_LIBRARY_PATH={lib_path} 141 142do_test() 143{{ 144 local idx=$1; shift; 145 local attempt=$1; shift; 146 147 # Read 10ms timestamp in seconds using shell builtins and /proc/uptime. 148 local time; 149 local unused; 150 151 # When printing the tag, we prefix by a new line, in case the 152 # previous command output did not contain any new line. 153 read time unused < /proc/uptime 154 echo '\\n{tag}START='$idx,$time 155 timeout {timeout}s "$@" 156 local rc=$? 157 read time unused < /proc/uptime 158 159 # Retry on timeout, to mute unlikely pthread_join hang issue. 160 # 161 # The timeout command send a SIGTERM signal, which should return 143 162 # (=128+15). However, due to a bug in tinybox, it returns 142. 163 if test \( $rc -eq 143 -o $rc -eq 142 \) -a $attempt -lt {retry}; then 164 echo '\\n{tag}RETRY='$rc,$time 165 attempt=$((attempt + 1)) 166 do_test $idx $attempt "$@" 167 else 168 echo '\\n{tag}STOP='$rc,$time 169 fi 170}} 171 172do_end() 173{{ 174 echo '\\n{tag}THE_END' 175}} 176""".format( 177 tag=tag, lib_path=lib_path, timeout=timeout, retry=retry 178 ) 179 180 181def setup_script(device, prefix, tempdir, options, uniq_tag, tests): 182 timeout = int(options.timeout) 183 script_timeout = 0 184 try: 185 print("tasks_adb_remote.py : Create batch script") 186 tmpf = tempfile.NamedTemporaryFile(mode="w", delete=False) 187 tmpf.write(script_preamble(uniq_tag, prefix, options)) 188 for i, test in enumerate(tests): 189 # This test is common to all tasks_*.py files, however, jit-test do 190 # not provide the `run_skipped` option, and all tests are always 191 # enabled. 192 assert test.enable # and not options.run_skipped 193 if options.test_reflect_stringify: 194 raise ValueError("can't run Reflect.stringify tests remotely") 195 196 cmd = test.command( 197 prefix, 198 posixpath.join(options.remote_test_root, "lib/"), 199 posixpath.join(options.remote_test_root, "modules/"), 200 tempdir, 201 posixpath.join(options.remote_test_root, "tests"), 202 ) 203 204 # replace with shlex.join when move to Python 3.8+ 205 cmd = ADBDevice._escape_command_line(cmd) 206 207 env = {} 208 if test.tz_pacific: 209 env["TZ"] = "PST8PDT" 210 envStr = "".join(key + "='" + val + "' " for key, val in env.items()) 211 212 tmpf.write("{}do_test {} 0 {};\n".format(envStr, i, cmd)) 213 script_timeout += timeout 214 tmpf.write("do_end;\n") 215 tmpf.close() 216 script = posixpath.join(options.remote_test_root, "test_manifest.sh") 217 device.push(tmpf.name, script) 218 device.chmod(script) 219 print("tasks_adb_remote.py : Batch script created") 220 except Exception as e: 221 print("tasks_adb_remote.py : Batch script failed") 222 raise e 223 finally: 224 if tmpf: 225 os.unlink(tmpf.name) 226 return script, script_timeout 227 228 229def start_script( 230 device, prefix, tempdir, script, uniq_tag, script_timeout, tests, options 231): 232 env = {} 233 234 # Allow ADBError or ADBTimeoutError to terminate the test run, but handle 235 # ADBProcessError in order to support the use of non-zero exit codes in the 236 # JavaScript shell tests. 237 # 238 # The stdout_callback will aggregate each output line, and reconstruct the 239 # output produced by each test, and queue TestOutput in the qResult queue. 240 try: 241 adb_process = device.shell( 242 "sh {}".format(script), 243 env=env, 244 cwd=options.remote_test_root, 245 timeout=script_timeout, 246 yield_stdout=True, 247 ) 248 for test_output in aggregate_script_stdout( 249 adb_process, prefix, tempdir, uniq_tag, tests, options 250 ): 251 yield test_output 252 print("tasks_adb_remote.py : Finished") 253 except ADBProcessError as e: 254 # After a device error, the device is typically in a 255 # state where all further tests will fail so there is no point in 256 # continuing here. 257 sys.stderr.write("Error running remote tests: {}".format(repr(e))) 258 259 260def get_remote_results(tests, prefix, pb, options): 261 """Create a script which batches the run of all tests, and spawn a thread to 262 reconstruct the TestOutput for each test. This is made to avoid multiple 263 `adb.shell` commands which has a high latency. 264 """ 265 device, tempdir = setup_device(prefix, options) 266 267 # Tests are sequentially executed in a batch. The first test executed is in 268 # charge of creating the xdr file for the self-hosted code. 269 if options.use_xdr: 270 tests = xdr_annotate(tests, options) 271 272 # We need tests to be subscriptable to find the test structure matching the 273 # index within the generated script. 274 tests = list(tests) 275 276 # Create a script which spawn each test one after the other, and upload the 277 # script 278 uniq_tag = "@@@TASKS_ADB_REMOTE@@@" 279 script, script_timeout = setup_script( 280 device, prefix, tempdir, options, uniq_tag, tests 281 ) 282 283 for test_output in start_script( 284 device, prefix, tempdir, script, uniq_tag, script_timeout, tests, options 285 ): 286 yield test_output 287