1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
|
# Copyright 2017 The Abseil Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tests for test randomization."""
import random
import subprocess
from absl import flags
from absl.testing import _bazelize_command
from absl.testing import absltest
from absl.testing import parameterized
from absl.testing.tests import absltest_env
FLAGS = flags.FLAGS
class TestOrderRandomizationTest(parameterized.TestCase):
"""Integration tests: Runs a py_test binary with randomization.
This is done by setting flags and environment variables.
"""
def setUp(self):
super(TestOrderRandomizationTest, self).setUp()
self._test_name = 'absl/testing/tests/absltest_randomization_testcase'
def _run_test(self, extra_argv, extra_env):
"""Runs the py_test binary in a subprocess, with the given args or env.
Args:
extra_argv: extra args to pass to the test
extra_env: extra env vars to set when running the test
Returns:
(stdout, test_cases, exit_code) tuple of (str, list of strs, int).
"""
env = absltest_env.inherited_env()
# If *this* test is being run with this flag, we don't want to
# automatically set it for all tests we run.
env.pop('TEST_RANDOMIZE_ORDERING_SEED', '')
if extra_env is not None:
env.update(extra_env)
command = (
[_bazelize_command.get_executable_path(self._test_name)] + extra_argv)
proc = subprocess.Popen(
args=command,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
universal_newlines=True)
stdout, _ = proc.communicate()
test_lines = [l for l in stdout.splitlines() if l.startswith('class ')]
return stdout, test_lines, proc.wait()
def test_no_args(self):
output, tests, exit_code = self._run_test([], None)
self.assertEqual(0, exit_code, msg='command output: ' + output)
self.assertNotIn('Randomizing test order with seed:', output)
cases = ['class A test ' + t for t in ('A', 'B', 'C')]
self.assertEqual(cases, tests)
@parameterized.parameters(
{
'argv': ['--test_randomize_ordering_seed=random'],
'env': None,
},
{
'argv': [],
'env': {
'TEST_RANDOMIZE_ORDERING_SEED': 'random',
},
},)
def test_simple_randomization(self, argv, env):
output, tests, exit_code = self._run_test(argv, env)
self.assertEqual(0, exit_code, msg='command output: ' + output)
self.assertIn('Randomizing test order with seed: ', output)
cases = ['class A test ' + t for t in ('A', 'B', 'C')]
# This may come back in any order; we just know it'll be the same
# set of elements.
self.assertSameElements(cases, tests)
@parameterized.parameters(
{
'argv': ['--test_randomize_ordering_seed=1'],
'env': None,
},
{
'argv': [],
'env': {
'TEST_RANDOMIZE_ORDERING_SEED': '1'
},
},
{
'argv': [],
'env': {
'LATE_SET_TEST_RANDOMIZE_ORDERING_SEED': '1'
},
},
)
def test_fixed_seed(self, argv, env):
output, tests, exit_code = self._run_test(argv, env)
self.assertEqual(0, exit_code, msg='command output: ' + output)
self.assertIn('Randomizing test order with seed: 1', output)
# Even though we know the seed, we need to shuffle the tests here, since
# this behaves differently in Python2 vs Python3.
shuffled_cases = ['A', 'B', 'C']
random.Random(1).shuffle(shuffled_cases)
cases = ['class A test ' + t for t in shuffled_cases]
# We know what order this will come back for the random seed we've
# specified.
self.assertEqual(cases, tests)
@parameterized.parameters(
{
'argv': ['--test_randomize_ordering_seed=0'],
'env': {
'TEST_RANDOMIZE_ORDERING_SEED': 'random'
},
},
{
'argv': [],
'env': {
'TEST_RANDOMIZE_ORDERING_SEED': '0'
},
},)
def test_disabling_randomization(self, argv, env):
output, tests, exit_code = self._run_test(argv, env)
self.assertEqual(0, exit_code, msg='command output: ' + output)
self.assertNotIn('Randomizing test order with seed:', output)
cases = ['class A test ' + t for t in ('A', 'B', 'C')]
self.assertEqual(cases, tests)
if __name__ == '__main__':
absltest.main()
|
