1defmodule ExUnit.Runner do 2 @moduledoc false 3 4 alias ExUnit.EventManager, as: EM 5 6 @rand_algorithm :exs1024 7 @current_key __MODULE__ 8 9 def run(opts, load_us) when (is_integer(load_us) or is_nil(load_us)) and is_list(opts) do 10 runner = self() 11 id = {__MODULE__, runner} 12 13 try do 14 # It may fail on Windows, so we ignore the result. 15 _ = 16 System.trap_signal(:sigquit, id, fn -> 17 ref = Process.monitor(runner) 18 send(runner, {ref, self(), :sigquit}) 19 20 receive do 21 ^ref -> :ok 22 {:DOWN, ^ref, _, _, _} -> :ok 23 after 24 5_000 -> :ok 25 end 26 27 Process.demonitor(ref, [:flush]) 28 :ok 29 end) 30 31 run_with_trap(opts, load_us) 32 after 33 System.untrap_signal(:sigquit, id) 34 end 35 end 36 37 defp run_with_trap(opts, load_us) do 38 opts = normalize_opts(opts) 39 {:ok, manager} = EM.start_link() 40 {:ok, stats_pid} = EM.add_handler(manager, ExUnit.RunnerStats, opts) 41 config = configure(opts, manager, self(), stats_pid) 42 :erlang.system_flag(:backtrace_depth, Keyword.fetch!(opts, :stacktrace_depth)) 43 44 start_time = System.monotonic_time() 45 EM.suite_started(config.manager, opts) 46 async_stop_time = async_loop(config, %{}, false) 47 stop_time = System.monotonic_time() 48 49 if max_failures_reached?(config) do 50 EM.max_failures_reached(config.manager) 51 end 52 53 async_us = 54 async_stop_time && 55 System.convert_time_unit(async_stop_time - start_time, :native, :microsecond) 56 57 run_us = System.convert_time_unit(stop_time - start_time, :native, :microsecond) 58 times_us = %{async: async_us, load: load_us, run: run_us} 59 EM.suite_finished(config.manager, times_us) 60 61 stats = ExUnit.RunnerStats.stats(stats_pid) 62 EM.stop(config.manager) 63 after_suite_callbacks = Application.fetch_env!(:ex_unit, :after_suite) 64 Enum.each(after_suite_callbacks, fn callback -> callback.(stats) end) 65 stats 66 end 67 68 defp configure(opts, manager, runner_pid, stats_pid) do 69 Enum.each(opts[:formatters], &EM.add_handler(manager, &1, opts)) 70 71 %{ 72 capture_log: opts[:capture_log], 73 exclude: opts[:exclude], 74 include: opts[:include], 75 manager: manager, 76 max_cases: opts[:max_cases], 77 max_failures: opts[:max_failures], 78 only_test_ids: opts[:only_test_ids], 79 runner_pid: runner_pid, 80 seed: opts[:seed], 81 stats_pid: stats_pid, 82 timeout: opts[:timeout], 83 trace: opts[:trace] 84 } 85 end 86 87 defp normalize_opts(opts) do 88 {include, exclude} = ExUnit.Filters.normalize(opts[:include], opts[:exclude]) 89 90 opts 91 |> Keyword.put(:exclude, exclude) 92 |> Keyword.put(:include, include) 93 end 94 95 defp async_loop(config, running, async_once?) do 96 available = config.max_cases - map_size(running) 97 98 cond do 99 # No modules available, wait for one 100 available <= 0 -> 101 running = wait_until_available(config, running) 102 async_loop(config, running, async_once?) 103 104 # Slots are available, start with async modules 105 modules = ExUnit.Server.take_async_modules(available) -> 106 running = spawn_modules(config, modules, running) 107 async_loop(config, running, true) 108 109 true -> 110 sync_modules = ExUnit.Server.take_sync_modules() 111 112 # Wait for all async modules 113 0 = 114 running 115 |> Enum.reduce(running, fn _, acc -> wait_until_available(config, acc) end) 116 |> map_size() 117 118 async_stop_time = if async_once?, do: System.monotonic_time(), else: nil 119 120 # Run all sync modules directly 121 for module <- sync_modules do 122 running = spawn_modules(config, [module], %{}) 123 running != %{} and wait_until_available(config, running) 124 end 125 126 async_stop_time 127 end 128 end 129 130 # Expect down messages from the spawned modules. 131 # 132 # We first look at the sigquit signal because we don't want 133 # to spawn new test cases when we know we will have to handle 134 # sigquit next. 135 # 136 # Otherwise, whenever a module has finished executing, update 137 # the running modules and attempt to spawn new ones. 138 defp wait_until_available(config, running) do 139 receive do 140 {ref, pid, :sigquit} -> 141 sigquit(config, ref, pid, running) 142 after 143 0 -> 144 receive do 145 {ref, pid, :sigquit} -> 146 sigquit(config, ref, pid, running) 147 148 {:DOWN, ref, _, _, _} when is_map_key(running, ref) -> 149 Map.delete(running, ref) 150 end 151 end 152 end 153 154 defp spawn_modules(_config, [], running) do 155 running 156 end 157 158 defp spawn_modules(config, [module | modules], running) do 159 if max_failures_reached?(config) do 160 running 161 else 162 {pid, ref} = spawn_monitor(fn -> run_module(config, module) end) 163 spawn_modules(config, modules, Map.put(running, ref, pid)) 164 end 165 end 166 167 ## Stacktrace 168 169 # Assertions can pop-up in the middle of the stack 170 def prune_stacktrace([{ExUnit.Assertions, _, _, _} | t]), do: prune_stacktrace(t) 171 172 # As soon as we see a Runner, it is time to ignore the stacktrace 173 def prune_stacktrace([{ExUnit.Runner, _, _, _} | _]), do: [] 174 175 # All other cases 176 def prune_stacktrace([h | t]), do: [h | prune_stacktrace(t)] 177 def prune_stacktrace([]), do: [] 178 179 ## sigquit 180 181 defp sigquit(config, ref, pid, running) do 182 # Stop all child processes from running and get their current state. 183 # We need to stop these processes because they may invoke the event 184 # manager and we must stop the event manager to guarantee the sigquit 185 # data has been flushed. 186 current = 187 Enum.map(running, fn {ref, pid} -> 188 current = safe_pdict_current(pid) 189 Process.exit(pid, :shutdown) 190 191 receive do 192 {:DOWN, ^ref, _, _, _} -> current 193 end 194 end) 195 196 EM.sigquit(config.manager, Enum.reject(current, &is_nil/1)) 197 EM.stop(config.manager) 198 199 # Reply to the event manager and wait until it shuts down the VM. 200 send(pid, ref) 201 Process.sleep(:infinity) 202 end 203 204 defp safe_pdict_current(pid) do 205 with {:dictionary, dictionary} <- Process.info(pid, :dictionary), 206 {@current_key, current} <- List.keyfind(dictionary, @current_key, 0), 207 do: current 208 rescue 209 _ -> nil 210 end 211 212 ## Running modules 213 214 defp run_module(config, module) do 215 test_module = module.__ex_unit__() 216 EM.module_started(config.manager, test_module) 217 218 # Prepare tests, selecting which ones should be run or skipped 219 tests = prepare_tests(config, test_module.tests) 220 {excluded_and_skipped_tests, to_run_tests} = Enum.split_with(tests, & &1.state) 221 222 for excluded_or_skipped_test <- excluded_and_skipped_tests do 223 EM.test_started(config.manager, excluded_or_skipped_test) 224 EM.test_finished(config.manager, excluded_or_skipped_test) 225 end 226 227 {test_module, invalid_tests, finished_tests} = run_module(config, test_module, to_run_tests) 228 229 pending_tests = 230 case process_max_failures(config, test_module) do 231 :no -> 232 invalid_tests 233 234 {:reached, n} -> 235 Enum.take(invalid_tests, n) 236 237 :surpassed -> 238 nil 239 end 240 241 # If pending_tests is [], EM.module_finished is still called. 242 # Only if process_max_failures/2 returns :surpassed it is not. 243 if pending_tests do 244 for pending_test <- pending_tests do 245 EM.test_started(config.manager, pending_test) 246 EM.test_finished(config.manager, pending_test) 247 end 248 249 test_module = %{test_module | tests: Enum.reverse(finished_tests, pending_tests)} 250 EM.module_finished(config.manager, test_module) 251 end 252 end 253 254 defp prepare_tests(config, tests) do 255 tests = shuffle(config, tests) 256 include = config.include 257 exclude = config.exclude 258 test_ids = config.only_test_ids 259 260 for test <- tests, include_test?(test_ids, test) do 261 tags = Map.merge(test.tags, %{test: test.name, module: test.module}) 262 263 case ExUnit.Filters.eval(include, exclude, tags, tests) do 264 :ok -> %{test | tags: tags} 265 excluded_or_skipped -> %{test | state: excluded_or_skipped} 266 end 267 end 268 end 269 270 defp include_test?(test_ids, test) do 271 test_ids == nil or MapSet.member?(test_ids, {test.module, test.name}) 272 end 273 274 defp run_module(_config, test_module, []) do 275 {test_module, [], []} 276 end 277 278 defp run_module(config, test_module, tests) do 279 {module_pid, module_ref} = run_setup_all(test_module, self()) 280 281 {test_module, invalid_tests, finished_tests} = 282 receive do 283 {^module_pid, :setup_all, {:ok, context}} -> 284 finished_tests = 285 if max_failures_reached?(config), do: [], else: run_tests(config, tests, context) 286 287 :ok = exit_setup_all(module_pid, module_ref) 288 {test_module, [], finished_tests} 289 290 {^module_pid, :setup_all, {:error, test_module}} -> 291 invalid_tests = Enum.map(tests, &%{&1 | state: {:invalid, test_module}}) 292 :ok = exit_setup_all(module_pid, module_ref) 293 {test_module, invalid_tests, []} 294 295 {:DOWN, ^module_ref, :process, ^module_pid, error} -> 296 test_module = %{test_module | state: failed({:EXIT, module_pid}, error, [])} 297 {test_module, [], []} 298 end 299 300 timeout = get_timeout(config, %{}) 301 {exec_on_exit(test_module, module_pid, timeout), invalid_tests, finished_tests} 302 end 303 304 defp run_setup_all(%ExUnit.TestModule{name: module} = test_module, parent_pid) do 305 Process.put(@current_key, test_module) 306 307 spawn_monitor(fn -> 308 ExUnit.OnExitHandler.register(self()) 309 310 result = 311 try do 312 {:ok, module.__ex_unit__(:setup_all, %{module: module, case: module})} 313 catch 314 kind, error -> 315 failed = failed(kind, error, prune_stacktrace(__STACKTRACE__)) 316 {:error, %{test_module | state: failed}} 317 end 318 319 send(parent_pid, {self(), :setup_all, result}) 320 321 # We keep the process alive so all of its resources 322 # stay alive until we run all tests in this case. 323 ref = Process.monitor(parent_pid) 324 325 receive do 326 {^parent_pid, :exit} -> :ok 327 {:DOWN, ^ref, _, _, _} -> :ok 328 end 329 end) 330 end 331 332 defp exit_setup_all(pid, ref) do 333 send(pid, {self(), :exit}) 334 335 receive do 336 {:DOWN, ^ref, _, _, _} -> :ok 337 end 338 end 339 340 defp run_tests(config, tests, context) do 341 Enum.reduce_while(tests, [], fn test, acc -> 342 Process.put(@current_key, test) 343 344 case run_test(config, test, context) do 345 {:ok, test} -> {:cont, [test | acc]} 346 :max_failures_reached -> {:halt, acc} 347 end 348 end) 349 end 350 351 defp run_test(config, %{tags: tags} = test, context) do 352 EM.test_started(config.manager, test) 353 354 context = maybe_create_tmp_dir(test, context, tags) 355 capture_log = Map.get(tags, :capture_log, config.capture_log) 356 test = run_test_with_capture_log(capture_log, config, test, Map.merge(tags, context)) 357 358 case process_max_failures(config, test) do 359 :no -> 360 EM.test_finished(config.manager, test) 361 {:ok, test} 362 363 {:reached, 1} -> 364 EM.test_finished(config.manager, test) 365 :max_failures_reached 366 367 :surpassed -> 368 :max_failures_reached 369 end 370 end 371 372 defp maybe_create_tmp_dir(test, context, %{tmp_dir: true}) do 373 create_tmp_dir!(test, "", context) 374 end 375 376 defp maybe_create_tmp_dir(test, context, %{tmp_dir: path}) when is_binary(path) do 377 create_tmp_dir!(test, path, context) 378 end 379 380 defp maybe_create_tmp_dir(_, context, %{tmp_dir: false}) do 381 context 382 end 383 384 defp maybe_create_tmp_dir(_, _, %{tmp_dir: other}) do 385 raise ArgumentError, "expected :tmp_dir to be a boolean or a string, got: #{inspect(other)}" 386 end 387 388 defp maybe_create_tmp_dir(_, context, _) do 389 context 390 end 391 392 defp create_tmp_dir!(test, extra_path, context) do 393 module = escape_path(inspect(test.module)) 394 name = escape_path(to_string(test.name)) 395 path = ["tmp", module, name, extra_path] |> Path.join() |> Path.expand() 396 File.rm_rf!(path) 397 File.mkdir_p!(path) 398 Map.put(context, :tmp_dir, path) 399 end 400 401 @escape Enum.map(' [~#%&*{}\\:<>?/+|"]', &<<&1::utf8>>) 402 403 defp escape_path(path) do 404 String.replace(path, @escape, "-") 405 end 406 407 defp run_test_with_capture_log(true, config, test, context) do 408 run_test_with_capture_log([], config, test, context) 409 end 410 411 defp run_test_with_capture_log(false, config, test, context) do 412 spawn_test(config, test, context) 413 end 414 415 defp run_test_with_capture_log(capture_log_opts, config, test, context) do 416 ref = make_ref() 417 418 try do 419 ExUnit.CaptureLog.capture_log(capture_log_opts, fn -> 420 send(self(), {ref, spawn_test(config, test, context)}) 421 end) 422 catch 423 :exit, :noproc -> 424 message = 425 "could not run test, it uses @tag :capture_log" <> 426 " but the :logger application is not running" 427 428 %{test | state: failed(:error, RuntimeError.exception(message), [])} 429 else 430 logged -> 431 receive do 432 {^ref, test} -> %{test | logs: logged} 433 end 434 end 435 end 436 437 defp spawn_test(config, test, context) do 438 parent_pid = self() 439 timeout = get_timeout(config, test.tags) 440 {test_pid, test_ref} = spawn_test_monitor(config, test, parent_pid, context) 441 test = receive_test_reply(test, test_pid, test_ref, timeout) 442 exec_on_exit(test, test_pid, timeout) 443 end 444 445 defp spawn_test_monitor(%{seed: seed}, test, parent_pid, context) do 446 spawn_monitor(fn -> 447 ExUnit.OnExitHandler.register(self()) 448 generate_test_seed(seed, test) 449 450 {time, test} = 451 :timer.tc(fn -> 452 case exec_test_setup(test, context) do 453 {:ok, test} -> exec_test(test) 454 {:error, test} -> test 455 end 456 end) 457 458 send(parent_pid, {self(), :test_finished, %{test | time: time}}) 459 exit(:shutdown) 460 end) 461 end 462 463 defp receive_test_reply(test, test_pid, test_ref, timeout) do 464 receive do 465 {^test_pid, :test_finished, test} -> 466 Process.demonitor(test_ref, [:flush]) 467 test 468 469 {:DOWN, ^test_ref, :process, ^test_pid, error} -> 470 %{test | state: failed({:EXIT, test_pid}, error, [])} 471 after 472 timeout -> 473 case Process.info(test_pid, :current_stacktrace) do 474 {:current_stacktrace, stacktrace} -> 475 Process.demonitor(test_ref, [:flush]) 476 Process.exit(test_pid, :kill) 477 478 exception = 479 ExUnit.TimeoutError.exception( 480 timeout: timeout, 481 type: Atom.to_string(test.tags.test_type) 482 ) 483 484 %{test | state: failed(:error, exception, stacktrace)} 485 486 nil -> 487 receive_test_reply(test, test_pid, test_ref, timeout) 488 end 489 end 490 end 491 492 defp exec_test_setup(%ExUnit.Test{module: module} = test, context) do 493 {:ok, %{test | tags: module.__ex_unit__(:setup, context)}} 494 catch 495 kind, error -> 496 {:error, %{test | state: failed(kind, error, prune_stacktrace(__STACKTRACE__))}} 497 end 498 499 defp exec_test(%ExUnit.Test{module: module, name: name, tags: context} = test) do 500 apply(module, name, [context]) 501 test 502 catch 503 kind, error -> 504 %{test | state: failed(kind, error, prune_stacktrace(__STACKTRACE__))} 505 end 506 507 defp exec_on_exit(test_or_case, pid, timeout) do 508 case ExUnit.OnExitHandler.run(pid, timeout) do 509 :ok -> 510 test_or_case 511 512 {kind, reason, stack} -> 513 state = test_or_case.state || failed(kind, reason, prune_stacktrace(stack)) 514 %{test_or_case | state: state} 515 end 516 end 517 518 ## Helpers 519 520 defp generate_test_seed(seed, %ExUnit.Test{module: module, name: name}) do 521 :rand.seed(@rand_algorithm, {:erlang.phash2(module), :erlang.phash2(name), seed}) 522 end 523 524 defp process_max_failures(%{max_failures: :infinity}, _), do: :no 525 526 defp process_max_failures(config, %ExUnit.TestModule{state: {:failed, _}, tests: tests}) do 527 process_max_failures(config.stats_pid, config.max_failures, length(tests)) 528 end 529 530 defp process_max_failures(config, %ExUnit.Test{state: {:failed, _}}) do 531 process_max_failures(config.stats_pid, config.max_failures, 1) 532 end 533 534 defp process_max_failures(config, _test_module_or_test) do 535 if max_failures_reached?(config), do: :surpassed, else: :no 536 end 537 538 defp process_max_failures(stats_pid, max_failures, bump) do 539 previous = ExUnit.RunnerStats.increment_failure_counter(stats_pid, bump) 540 541 cond do 542 previous >= max_failures -> :surpassed 543 previous + bump < max_failures -> :no 544 true -> {:reached, max_failures - previous} 545 end 546 end 547 548 defp max_failures_reached?(%{stats_pid: stats_pid, max_failures: max_failures}) do 549 max_failures != :infinity and 550 ExUnit.RunnerStats.get_failure_counter(stats_pid) >= max_failures 551 end 552 553 defp get_timeout(config, tags) do 554 if config.trace do 555 :infinity 556 else 557 Map.get(tags, :timeout, config.timeout) 558 end 559 end 560 561 defp shuffle(%{seed: 0}, list) do 562 Enum.reverse(list) 563 end 564 565 defp shuffle(%{seed: seed}, list) do 566 _ = :rand.seed(@rand_algorithm, {seed, seed, seed}) 567 Enum.shuffle(list) 568 end 569 570 defp failed(:error, %ExUnit.MultiError{errors: errors}, _stack) do 571 errors = 572 Enum.map(errors, fn {kind, reason, stack} -> 573 {kind, Exception.normalize(kind, reason, stack), prune_stacktrace(stack)} 574 end) 575 576 {:failed, errors} 577 end 578 579 defp failed(kind, reason, stack) do 580 {:failed, [{kind, Exception.normalize(kind, reason, stack), stack}]} 581 end 582end 583