1import pytest
2
3
4@pytest.fixture
5def stepwise_testdir(testdir):
6    # Rather than having to modify our testfile between tests, we introduce
7    # a flag for whether or not the second test should fail.
8    testdir.makeconftest(
9        """
10def pytest_addoption(parser):
11    group = parser.getgroup('general')
12    group.addoption('--fail', action='store_true', dest='fail')
13    group.addoption('--fail-last', action='store_true', dest='fail_last')
14"""
15    )
16
17    # Create a simple test suite.
18    testdir.makepyfile(
19        test_a="""
20def test_success_before_fail():
21    assert 1
22
23def test_fail_on_flag(request):
24    assert not request.config.getvalue('fail')
25
26def test_success_after_fail():
27    assert 1
28
29def test_fail_last_on_flag(request):
30    assert not request.config.getvalue('fail_last')
31
32def test_success_after_last_fail():
33    assert 1
34"""
35    )
36
37    testdir.makepyfile(
38        test_b="""
39def test_success():
40    assert 1
41"""
42    )
43
44    # customize cache directory so we don't use the tox's cache directory, which makes tests in this module flaky
45    testdir.makeini(
46        """
47        [pytest]
48        cache_dir = .cache
49    """
50    )
51
52    return testdir
53
54
55@pytest.fixture
56def error_testdir(testdir):
57    testdir.makepyfile(
58        test_a="""
59def test_error(nonexisting_fixture):
60    assert 1
61
62def test_success_after_fail():
63    assert 1
64"""
65    )
66
67    return testdir
68
69
70@pytest.fixture
71def broken_testdir(testdir):
72    testdir.makepyfile(
73        working_testfile="def test_proper(): assert 1", broken_testfile="foobar"
74    )
75    return testdir
76
77
78def _strip_resource_warnings(lines):
79    # Strip unreliable ResourceWarnings, so no-output assertions on stderr can work.
80    # (https://github.com/pytest-dev/pytest/issues/5088)
81    return [
82        x
83        for x in lines
84        if not x.startswith(("Exception ignored in:", "ResourceWarning"))
85    ]
86
87
88def test_run_without_stepwise(stepwise_testdir):
89    result = stepwise_testdir.runpytest("-v", "--strict-markers", "--fail")
90
91    result.stdout.fnmatch_lines(["*test_success_before_fail PASSED*"])
92    result.stdout.fnmatch_lines(["*test_fail_on_flag FAILED*"])
93    result.stdout.fnmatch_lines(["*test_success_after_fail PASSED*"])
94
95
96def test_fail_and_continue_with_stepwise(stepwise_testdir):
97    # Run the tests with a failing second test.
98    result = stepwise_testdir.runpytest(
99        "-v", "--strict-markers", "--stepwise", "--fail"
100    )
101    assert _strip_resource_warnings(result.stderr.lines) == []
102
103    stdout = result.stdout.str()
104    # Make sure we stop after first failing test.
105    assert "test_success_before_fail PASSED" in stdout
106    assert "test_fail_on_flag FAILED" in stdout
107    assert "test_success_after_fail" not in stdout
108
109    # "Fix" the test that failed in the last run and run it again.
110    result = stepwise_testdir.runpytest("-v", "--strict-markers", "--stepwise")
111    assert _strip_resource_warnings(result.stderr.lines) == []
112
113    stdout = result.stdout.str()
114    # Make sure the latest failing test runs and then continues.
115    assert "test_success_before_fail" not in stdout
116    assert "test_fail_on_flag PASSED" in stdout
117    assert "test_success_after_fail PASSED" in stdout
118
119
120def test_run_with_skip_option(stepwise_testdir):
121    result = stepwise_testdir.runpytest(
122        "-v",
123        "--strict-markers",
124        "--stepwise",
125        "--stepwise-skip",
126        "--fail",
127        "--fail-last",
128    )
129    assert _strip_resource_warnings(result.stderr.lines) == []
130
131    stdout = result.stdout.str()
132    # Make sure first fail is ignore and second fail stops the test run.
133    assert "test_fail_on_flag FAILED" in stdout
134    assert "test_success_after_fail PASSED" in stdout
135    assert "test_fail_last_on_flag FAILED" in stdout
136    assert "test_success_after_last_fail" not in stdout
137
138
139def test_fail_on_errors(error_testdir):
140    result = error_testdir.runpytest("-v", "--strict-markers", "--stepwise")
141
142    assert _strip_resource_warnings(result.stderr.lines) == []
143    stdout = result.stdout.str()
144
145    assert "test_error ERROR" in stdout
146    assert "test_success_after_fail" not in stdout
147
148
149def test_change_testfile(stepwise_testdir):
150    result = stepwise_testdir.runpytest(
151        "-v", "--strict-markers", "--stepwise", "--fail", "test_a.py"
152    )
153    assert _strip_resource_warnings(result.stderr.lines) == []
154
155    stdout = result.stdout.str()
156    assert "test_fail_on_flag FAILED" in stdout
157
158    # Make sure the second test run starts from the beginning, since the
159    # test to continue from does not exist in testfile_b.
160    result = stepwise_testdir.runpytest(
161        "-v", "--strict-markers", "--stepwise", "test_b.py"
162    )
163    assert _strip_resource_warnings(result.stderr.lines) == []
164
165    stdout = result.stdout.str()
166    assert "test_success PASSED" in stdout
167
168
169@pytest.mark.parametrize("broken_first", [True, False])
170def test_stop_on_collection_errors(broken_testdir, broken_first):
171    """Stop during collection errors. Broken test first or broken test last
172    actually surfaced a bug (#5444), so we test both situations."""
173    files = ["working_testfile.py", "broken_testfile.py"]
174    if broken_first:
175        files.reverse()
176    result = broken_testdir.runpytest("-v", "--strict-markers", "--stepwise", *files)
177    result.stdout.fnmatch_lines("*error during collection*")
178
179
180def test_xfail_handling(testdir, monkeypatch):
181    """Ensure normal xfail is ignored, and strict xfail interrupts the session in sw mode
182
183    (#5547)
184    """
185    monkeypatch.setattr("sys.dont_write_bytecode", True)
186
187    contents = """
188        import pytest
189        def test_a(): pass
190
191        @pytest.mark.xfail(strict={strict})
192        def test_b(): assert {assert_value}
193
194        def test_c(): pass
195        def test_d(): pass
196    """
197    testdir.makepyfile(contents.format(assert_value="0", strict="False"))
198    result = testdir.runpytest("--sw", "-v")
199    result.stdout.fnmatch_lines(
200        [
201            "*::test_a PASSED *",
202            "*::test_b XFAIL *",
203            "*::test_c PASSED *",
204            "*::test_d PASSED *",
205            "* 3 passed, 1 xfailed in *",
206        ]
207    )
208
209    testdir.makepyfile(contents.format(assert_value="1", strict="True"))
210    result = testdir.runpytest("--sw", "-v")
211    result.stdout.fnmatch_lines(
212        [
213            "*::test_a PASSED *",
214            "*::test_b FAILED *",
215            "* Interrupted*",
216            "* 1 failed, 1 passed in *",
217        ]
218    )
219
220    testdir.makepyfile(contents.format(assert_value="0", strict="True"))
221    result = testdir.runpytest("--sw", "-v")
222    result.stdout.fnmatch_lines(
223        [
224            "*::test_b XFAIL *",
225            "*::test_c PASSED *",
226            "*::test_d PASSED *",
227            "* 2 passed, 1 deselected, 1 xfailed in *",
228        ]
229    )
230