1import pytest 2 3 4@pytest.fixture 5def stepwise_testdir(testdir): 6 # Rather than having to modify our testfile between tests, we introduce 7 # a flag for whether or not the second test should fail. 8 testdir.makeconftest( 9 """ 10def pytest_addoption(parser): 11 group = parser.getgroup('general') 12 group.addoption('--fail', action='store_true', dest='fail') 13 group.addoption('--fail-last', action='store_true', dest='fail_last') 14""" 15 ) 16 17 # Create a simple test suite. 18 testdir.makepyfile( 19 test_a=""" 20def test_success_before_fail(): 21 assert 1 22 23def test_fail_on_flag(request): 24 assert not request.config.getvalue('fail') 25 26def test_success_after_fail(): 27 assert 1 28 29def test_fail_last_on_flag(request): 30 assert not request.config.getvalue('fail_last') 31 32def test_success_after_last_fail(): 33 assert 1 34""" 35 ) 36 37 testdir.makepyfile( 38 test_b=""" 39def test_success(): 40 assert 1 41""" 42 ) 43 44 # customize cache directory so we don't use the tox's cache directory, which makes tests in this module flaky 45 testdir.makeini( 46 """ 47 [pytest] 48 cache_dir = .cache 49 """ 50 ) 51 52 return testdir 53 54 55@pytest.fixture 56def error_testdir(testdir): 57 testdir.makepyfile( 58 test_a=""" 59def test_error(nonexisting_fixture): 60 assert 1 61 62def test_success_after_fail(): 63 assert 1 64""" 65 ) 66 67 return testdir 68 69 70@pytest.fixture 71def broken_testdir(testdir): 72 testdir.makepyfile( 73 working_testfile="def test_proper(): assert 1", broken_testfile="foobar" 74 ) 75 return testdir 76 77 78def _strip_resource_warnings(lines): 79 # Strip unreliable ResourceWarnings, so no-output assertions on stderr can work. 80 # (https://github.com/pytest-dev/pytest/issues/5088) 81 return [ 82 x 83 for x in lines 84 if not x.startswith(("Exception ignored in:", "ResourceWarning")) 85 ] 86 87 88def test_run_without_stepwise(stepwise_testdir): 89 result = stepwise_testdir.runpytest("-v", "--strict-markers", "--fail") 90 91 result.stdout.fnmatch_lines(["*test_success_before_fail PASSED*"]) 92 result.stdout.fnmatch_lines(["*test_fail_on_flag FAILED*"]) 93 result.stdout.fnmatch_lines(["*test_success_after_fail PASSED*"]) 94 95 96def test_fail_and_continue_with_stepwise(stepwise_testdir): 97 # Run the tests with a failing second test. 98 result = stepwise_testdir.runpytest( 99 "-v", "--strict-markers", "--stepwise", "--fail" 100 ) 101 assert _strip_resource_warnings(result.stderr.lines) == [] 102 103 stdout = result.stdout.str() 104 # Make sure we stop after first failing test. 105 assert "test_success_before_fail PASSED" in stdout 106 assert "test_fail_on_flag FAILED" in stdout 107 assert "test_success_after_fail" not in stdout 108 109 # "Fix" the test that failed in the last run and run it again. 110 result = stepwise_testdir.runpytest("-v", "--strict-markers", "--stepwise") 111 assert _strip_resource_warnings(result.stderr.lines) == [] 112 113 stdout = result.stdout.str() 114 # Make sure the latest failing test runs and then continues. 115 assert "test_success_before_fail" not in stdout 116 assert "test_fail_on_flag PASSED" in stdout 117 assert "test_success_after_fail PASSED" in stdout 118 119 120def test_run_with_skip_option(stepwise_testdir): 121 result = stepwise_testdir.runpytest( 122 "-v", 123 "--strict-markers", 124 "--stepwise", 125 "--stepwise-skip", 126 "--fail", 127 "--fail-last", 128 ) 129 assert _strip_resource_warnings(result.stderr.lines) == [] 130 131 stdout = result.stdout.str() 132 # Make sure first fail is ignore and second fail stops the test run. 133 assert "test_fail_on_flag FAILED" in stdout 134 assert "test_success_after_fail PASSED" in stdout 135 assert "test_fail_last_on_flag FAILED" in stdout 136 assert "test_success_after_last_fail" not in stdout 137 138 139def test_fail_on_errors(error_testdir): 140 result = error_testdir.runpytest("-v", "--strict-markers", "--stepwise") 141 142 assert _strip_resource_warnings(result.stderr.lines) == [] 143 stdout = result.stdout.str() 144 145 assert "test_error ERROR" in stdout 146 assert "test_success_after_fail" not in stdout 147 148 149def test_change_testfile(stepwise_testdir): 150 result = stepwise_testdir.runpytest( 151 "-v", "--strict-markers", "--stepwise", "--fail", "test_a.py" 152 ) 153 assert _strip_resource_warnings(result.stderr.lines) == [] 154 155 stdout = result.stdout.str() 156 assert "test_fail_on_flag FAILED" in stdout 157 158 # Make sure the second test run starts from the beginning, since the 159 # test to continue from does not exist in testfile_b. 160 result = stepwise_testdir.runpytest( 161 "-v", "--strict-markers", "--stepwise", "test_b.py" 162 ) 163 assert _strip_resource_warnings(result.stderr.lines) == [] 164 165 stdout = result.stdout.str() 166 assert "test_success PASSED" in stdout 167 168 169@pytest.mark.parametrize("broken_first", [True, False]) 170def test_stop_on_collection_errors(broken_testdir, broken_first): 171 """Stop during collection errors. Broken test first or broken test last 172 actually surfaced a bug (#5444), so we test both situations.""" 173 files = ["working_testfile.py", "broken_testfile.py"] 174 if broken_first: 175 files.reverse() 176 result = broken_testdir.runpytest("-v", "--strict-markers", "--stepwise", *files) 177 result.stdout.fnmatch_lines("*error during collection*") 178 179 180def test_xfail_handling(testdir, monkeypatch): 181 """Ensure normal xfail is ignored, and strict xfail interrupts the session in sw mode 182 183 (#5547) 184 """ 185 monkeypatch.setattr("sys.dont_write_bytecode", True) 186 187 contents = """ 188 import pytest 189 def test_a(): pass 190 191 @pytest.mark.xfail(strict={strict}) 192 def test_b(): assert {assert_value} 193 194 def test_c(): pass 195 def test_d(): pass 196 """ 197 testdir.makepyfile(contents.format(assert_value="0", strict="False")) 198 result = testdir.runpytest("--sw", "-v") 199 result.stdout.fnmatch_lines( 200 [ 201 "*::test_a PASSED *", 202 "*::test_b XFAIL *", 203 "*::test_c PASSED *", 204 "*::test_d PASSED *", 205 "* 3 passed, 1 xfailed in *", 206 ] 207 ) 208 209 testdir.makepyfile(contents.format(assert_value="1", strict="True")) 210 result = testdir.runpytest("--sw", "-v") 211 result.stdout.fnmatch_lines( 212 [ 213 "*::test_a PASSED *", 214 "*::test_b FAILED *", 215 "* Interrupted*", 216 "* 1 failed, 1 passed in *", 217 ] 218 ) 219 220 testdir.makepyfile(contents.format(assert_value="0", strict="True")) 221 result = testdir.runpytest("--sw", "-v") 222 result.stdout.fnmatch_lines( 223 [ 224 "*::test_b XFAIL *", 225 "*::test_c PASSED *", 226 "*::test_d PASSED *", 227 "* 2 passed, 1 deselected, 1 xfailed in *", 228 ] 229 ) 230