10 def test_help(testdir):  argument
11     result = testdir.runpytest_subprocess('--help')
49 def test_groups(testdir):  argument
50     test = testdir.makepyfile('''"""
75     result = testdir.runpytest_subprocess('-vv', '--doctest-modules', test)
159 def test_group_by_name(testdir):  argument
160     test_x = testdir.makepyfile(test_x=GROUPING_TEST)
161     test_y = testdir.makepyfile(test_y=GROUPING_TEST)
162 …result = testdir.runpytest_subprocess('--benchmark-max-time=0.0000001', '--benchmark-group-by', 'n…
196 def test_group_by_func(testdir):  argument
197     test_x = testdir.makepyfile(test_x=GROUPING_TEST)
198     test_y = testdir.makepyfile(test_y=GROUPING_TEST)
199 …result = testdir.runpytest_subprocess('--benchmark-max-time=0.0000001', '--benchmark-group-by', 'f…
225 def test_group_by_fullfunc(testdir):  argument
226     test_x = testdir.makepyfile(test_x=GROUPING_TEST)
227     test_y = testdir.makepyfile(test_y=GROUPING_TEST)
228 …result = testdir.runpytest_subprocess('--benchmark-max-time=0.0000001', '--benchmark-group-by', 'f…
266 def test_group_by_param_all(testdir):  argument
267     test_x = testdir.makepyfile(test_x=GROUPING_TEST)
268     test_y = testdir.makepyfile(test_y=GROUPING_TEST)
269 …result = testdir.runpytest_subprocess('--benchmark-max-time=0.0000001', '--benchmark-group-by', 'p…
298 def test_group_by_param_select(testdir):  argument
299     test_x = testdir.makepyfile(test_x=GROUPING_PARAMS_TEST)
300     result = testdir.runpytest_subprocess('--benchmark-max-time=0.0000001',
331 def test_group_by_param_select_multiple(testdir):  argument
332     test_x = testdir.makepyfile(test_x=GROUPING_PARAMS_TEST)
333     result = testdir.runpytest_subprocess('--benchmark-max-time=0.0000001',
374 def test_group_by_fullname(testdir):  argument
375     test_x = testdir.makepyfile(test_x=GROUPING_TEST)
376     test_y = testdir.makepyfile(test_y=GROUPING_TEST)
377 …result = testdir.runpytest_subprocess('--benchmark-max-time=0.0000001', '--benchmark-group-by', 'f…
392 def test_double_use(testdir):  argument
393     test = testdir.makepyfile('''
402     result = testdir.runpytest_subprocess(test, '--tb=line')
409 def test_only_override_skip(testdir):  argument
410     test = testdir.makepyfile(SIMPLE_TEST)
411     result = testdir.runpytest_subprocess('--benchmark-only', '--benchmark-skip', test)
425 def test_fixtures_also_skipped(testdir):  argument
426     test = testdir.makepyfile(FIXTURES_ALSO_SKIPPED_TEST)
427     result = testdir.runpytest_subprocess('--benchmark-only', '-s', test)
435 def test_conflict_between_only_and_disable(testdir):  argument
436     test = testdir.makepyfile(SIMPLE_TEST)
437     result = testdir.runpytest_subprocess('--benchmark-only', '--benchmark-disable', test)
444 def test_max_time_min_rounds(testdir):  argument
445     test = testdir.makepyfile(SIMPLE_TEST)
446     result = testdir.runpytest_subprocess('--doctest-modules', '--benchmark-max-time=0.000001',
461 def test_max_time(testdir):  argument
462     test = testdir.makepyfile(SIMPLE_TEST)
463 …  result = testdir.runpytest_subprocess('--doctest-modules', '--benchmark-max-time=0.000001', test)
477 def test_bogus_max_time(testdir):  argument
478     test = testdir.makepyfile(SIMPLE_TEST)
479     result = testdir.runpytest_subprocess('--doctest-modules', '--benchmark-max-time=bogus', test)
487 def test_pep418_timer(testdir):  argument
488     test = testdir.makepyfile(SIMPLE_TEST)
489     result = testdir.runpytest_subprocess('--benchmark-max-time=0.0000001', '--doctest-modules',
496 def test_bad_save(testdir):  argument
497     test = testdir.makepyfile(SIMPLE_TEST)
498     result = testdir.runpytest_subprocess('--doctest-modules', '--benchmark-save=asd:f?', test)
505 def test_bad_save_2(testdir):  argument
506     test = testdir.makepyfile(SIMPLE_TEST)
507     result = testdir.runpytest_subprocess('--doctest-modules', '--benchmark-save=', test)
514 def test_bad_compare_fail(testdir):  argument
515     test = testdir.makepyfile(SIMPLE_TEST)
516     result = testdir.runpytest_subprocess('--doctest-modules', '--benchmark-compare-fail=?', test)
523 def test_bad_rounds(testdir):  argument
524     test = testdir.makepyfile(SIMPLE_TEST)
525     result = testdir.runpytest_subprocess('--doctest-modules', '--benchmark-min-rounds=asd', test)
532 def test_bad_rounds_2(testdir):  argument
533     test = testdir.makepyfile(SIMPLE_TEST)
534     result = testdir.runpytest_subprocess('--doctest-modules', '--benchmark-min-rounds=0', test)
541 def test_compare(testdir):  argument
542     test = testdir.makepyfile(SIMPLE_TEST)
543 …testdir.runpytest_subprocess('--benchmark-max-time=0.0000001', '--doctest-modules', '--benchmark-a…
544     result = testdir.runpytest_subprocess('--benchmark-max-time=0.0000001', '--doctest-modules',
550     result = testdir.runpytest_subprocess('--benchmark-max-time=0.0000001', '--doctest-modules',
558 def test_compare_last(testdir):  argument
559     test = testdir.makepyfile(SIMPLE_TEST)
560 …testdir.runpytest_subprocess('--benchmark-max-time=0.0000001', '--doctest-modules', '--benchmark-a…
561 …result = testdir.runpytest_subprocess('--benchmark-max-time=0.0000001', '--doctest-modules', '--be…
566 …result = testdir.runpytest_subprocess('--benchmark-max-time=0.0000001', '--doctest-modules', '--be…
573 def test_compare_non_existing(testdir):  argument
574     test = testdir.makepyfile(SIMPLE_TEST)
575 …testdir.runpytest_subprocess('--benchmark-max-time=0.0000001', '--doctest-modules', '--benchmark-a…
576     result = testdir.runpytest_subprocess('--benchmark-max-time=0.0000001', '--doctest-modules',
584 def test_compare_non_existing_verbose(testdir):  argument
585     test = testdir.makepyfile(SIMPLE_TEST)
586 …testdir.runpytest_subprocess('--benchmark-max-time=0.0000001', '--doctest-modules', '--benchmark-a…
587     result = testdir.runpytest_subprocess('--benchmark-max-time=0.0000001', '--doctest-modules',
595 def test_compare_no_files(testdir):  argument
596     test = testdir.makepyfile(SIMPLE_TEST)
597 …result = testdir.runpytest_subprocess('--benchmark-max-time=0.0000001', '--doctest-modules', '-rw',
604 def test_compare_no_files_verbose(testdir):  argument
605     test = testdir.makepyfile(SIMPLE_TEST)
606     result = testdir.runpytest_subprocess('--benchmark-max-time=0.0000001', '--doctest-modules',
614 def test_compare_no_files_match(testdir):  argument
615     test = testdir.makepyfile(SIMPLE_TEST)
616 …result = testdir.runpytest_subprocess('--benchmark-max-time=0.0000001', '--doctest-modules', '-rw',
623 def test_compare_no_files_match_verbose(testdir):  argument
624     test = testdir.makepyfile(SIMPLE_TEST)
625     result = testdir.runpytest_subprocess('--benchmark-max-time=0.0000001', '--doctest-modules',
632 def test_verbose(testdir):  argument
633     test = testdir.makepyfile(SIMPLE_TEST)
634 …result = testdir.runpytest_subprocess('--benchmark-max-time=0.0000001', '--doctest-modules', '--be…
644 def test_save(testdir):  argument
645     test = testdir.makepyfile(SIMPLE_TEST)
646     result = testdir.runpytest_subprocess('--doctest-modules', '--benchmark-save=foobar',
651     json.loads(testdir.tmpdir.join('.benchmarks').listdir()[0].join('0001_foobar.json').read())
654 def test_save_extra_info(testdir):  argument
655     test = testdir.makepyfile("""
660     result = testdir.runpytest_subprocess('--doctest-modules', '--benchmark-save=foobar',
665 … info = json.loads(testdir.tmpdir.join('.benchmarks').listdir()[0].join('0001_foobar.json').read())
671 def test_update_machine_info_hook_detection(testdir):  argument
695     testdir.makepyfile(**{
703         testdir.runpytest_subprocess(
709         benchmark_json = json.loads(testdir.tmpdir.join('benchmark.json').read())
725 def test_histogram(testdir):  argument
726     test = testdir.makepyfile(SIMPLE_TEST)
727     result = testdir.runpytest_subprocess('--doctest-modules', '--benchmark-histogram=foobar',
732     assert [f.basename for f in testdir.tmpdir.listdir("*.svg", sort=True)] == [
737 def test_autosave(testdir):  argument
738     test = testdir.makepyfile(SIMPLE_TEST)
739     result = testdir.runpytest_subprocess('--doctest-modules', '--benchmark-autosave',
744     json.loads(testdir.tmpdir.join('.benchmarks').listdir()[0].listdir('0001_*.json')[0].read())
747 def test_bogus_min_time(testdir):  argument
748     test = testdir.makepyfile(SIMPLE_TEST)
749     result = testdir.runpytest_subprocess('--doctest-modules', '--benchmark-min-time=bogus', test)
756 def test_disable_gc(testdir):  argument
757     test = testdir.makepyfile(SIMPLE_TEST)
758     result = testdir.runpytest_subprocess('--benchmark-disable-gc', test)
772 def test_custom_timer(testdir):  argument
773     test = testdir.makepyfile(SIMPLE_TEST)
774     result = testdir.runpytest_subprocess('--benchmark-timer=time.time', test)
788 def test_bogus_timer(testdir):  argument
789     test = testdir.makepyfile(SIMPLE_TEST)
790     result = testdir.runpytest_subprocess('--benchmark-timer=bogus', test)
798 def test_sort_by_mean(testdir):  argument
799     test = testdir.makepyfile(SIMPLE_TEST)
800     result = testdir.runpytest_subprocess('--benchmark-sort=mean', test)
814 def test_bogus_sort(testdir):  argument
815     test = testdir.makepyfile(SIMPLE_TEST)
816     result = testdir.runpytest_subprocess('--benchmark-sort=bogus', test)
824 def test_xdist(testdir):  argument
826     test = testdir.makepyfile(SIMPLE_TEST)
827     result = testdir.runpytest_subprocess('--doctest-modules', '-n', '1', '-rw', test)
834 def test_xdist_verbose(testdir):  argument
836     test = testdir.makepyfile(SIMPLE_TEST)
837 … result = testdir.runpytest_subprocess('--doctest-modules', '-n', '1', '--benchmark-verbose', test)
846 def test_cprofile(testdir):  argument
847     test = testdir.makepyfile(SIMPLE_TEST)
848     result = testdir.runpytest_subprocess('--benchmark-cprofile=cumtime', test)
865 def test_disabled_and_cprofile(testdir):  argument
866     test = testdir.makepyfile(SIMPLE_TEST)
867 … result = testdir.runpytest_subprocess('--benchmark-disable', '--benchmark-cprofile=cumtime', test)
873 def test_abort_broken(testdir):  argument
877     test = testdir.makepyfile('''
907     result = testdir.runpytest_subprocess('-vv', test)
1025 def test_basic(testdir):  argument
1026     test = testdir.makepyfile(BASIC_TEST)
1027     result = testdir.runpytest_subprocess('-vv', '--doctest-modules', test)
1049 def test_skip(testdir):  argument
1050     test = testdir.makepyfile(BASIC_TEST)
1051     result = testdir.runpytest_subprocess('-vv', '--doctest-modules', '--benchmark-skip', test)
1063 def test_disable(testdir):  argument
1064     test = testdir.makepyfile(BASIC_TEST)
1065     result = testdir.runpytest_subprocess('-vv', '--doctest-modules', '--benchmark-disable', test)
1077 def test_mark_selection(testdir):  argument
1078     test = testdir.makepyfile(BASIC_TEST)
1079     result = testdir.runpytest_subprocess('-vv', '--doctest-modules', '-m', 'benchmark', test)
1092 def test_only_benchmarks(testdir):  argument
1093     test = testdir.makepyfile(BASIC_TEST)
1094     result = testdir.runpytest_subprocess('-vv', '--doctest-modules', '--benchmark-only', test)
1114 def test_columns(testdir):  argument
1115     test = testdir.makepyfile(SIMPLE_TEST)
1116 …result = testdir.runpytest_subprocess('--doctest-modules', '--benchmark-columns=max,iterations,min…