1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
|
from unittest import TestCase, skip
# pylint:disable=import-error
import pytest
# pylint:enable=import-error
from flaky import flaky
# This is an end-to-end example of the flaky package in action. Consider it
# a live tutorial, showing the various features in action.
@flaky
def test_something_flaky(dummy_list=[]):
# pylint:disable=dangerous-default-value
dummy_list.append(0)
assert len(dummy_list) > 1
@pytest.fixture(scope='function')
def failing_setup_fixture():
assert False
@flaky
@pytest.mark.xfail(strict=True)
@pytest.mark.usefixtures("failing_setup_fixture")
def test_something_good_with_failing_setup_fixture():
assert True
class TestExample:
_threshold = -1
def test_non_flaky_thing(self):
"""Flaky will not interact with this test"""
@pytest.mark.xfail
def test_non_flaky_failing_thing(self):
"""Flaky will also not interact with this test"""
assert self == 1
@flaky(3, 2)
def test_flaky_thing_that_fails_then_succeeds(self):
"""
Flaky will run this test 3 times.
It will fail once and then succeed twice.
"""
# pylint:disable=no-self-use
TestExample._threshold += 1
assert TestExample._threshold >= 1
@flaky(3, 2)
def test_flaky_thing_that_succeeds_then_fails_then_succeeds(self):
"""
Flaky will run this test 3 times.
It will succeed once, fail once, and then succeed one more time.
"""
# pylint:disable=no-self-use
TestExample._threshold += 1
assert TestExample._threshold != 1
@flaky(2, 2)
def test_flaky_thing_that_always_passes(self):
"""Flaky will run this test twice. Both will succeed."""
@pytest.mark.skipif(
'True',
reason="This really fails! Remove skipif to see the test failure."
)
@flaky()
def test_flaky_thing_that_always_fails(self):
"""Flaky will run this test twice. Both will fail."""
assert self is None
@flaky
class TestExampleFlakyTests:
_threshold = -1
@staticmethod
def test_flaky_thing_that_fails_then_succeeds():
"""
Flaky will run this test twice.
It will fail once and then succeed.
"""
TestExampleFlakyTests._threshold += 1
assert TestExampleFlakyTests._threshold >= 1
@flaky
class TestExampleFlakyTestCase(TestCase):
_threshold = -1
@staticmethod
def test_flaky_thing_that_fails_then_succeeds():
"""
Flaky will run this test twice.
It will fail once and then succeed.
"""
TestExampleFlakyTestCase._threshold += 1
assert TestExampleFlakyTestCase._threshold >= 1
class TestFlakySubclass(TestExampleFlakyTestCase):
pass
@pytest.mark.flaky
class TestMarkedClass:
_threshold = -1
@staticmethod
def test_flaky_thing_that_fails_then_succeeds():
"""
Flaky will run this test twice.
It will fail once and then succeed.
"""
TestMarkedClass._threshold += 1
assert TestMarkedClass._threshold >= 1
def _test_flaky_doctest():
"""
Flaky ignores doctests. This test wouldn't be rerun if it failed.
>>> _test_flaky_doctest()
True
"""
return True
@pytest.fixture
def my_fixture():
return 42
@flaky
def test_requiring_my_fixture(my_fixture, dummy_list=[]):
# pylint:disable=dangerous-default-value,unused-argument
dummy_list.append(0)
assert len(dummy_list) > 1
def _rerun_filter(err, name, test, plugin):
# pylint:disable=unused-argument
return issubclass(err[0], AssertionError)
class TestExampleRerunFilter:
_threshold = -1
@flaky(rerun_filter=_rerun_filter)
def test_something_flaky(self):
# pylint:disable=no-self-use
TestExampleRerunFilter._threshold += 1
assert TestExampleRerunFilter._threshold >= 1
@skip('This test always fails')
@flaky
def test_something_that_always_fails_but_should_be_skipped():
assert 0
|
