File: test_utils.py

package info (click to toggle)
chromium 139.0.7258.127-1
  • links: PTS, VCS
  • area: main
  • in suites:
  • size: 6,122,068 kB
  • sloc: cpp: 35,100,771; ansic: 7,163,530; javascript: 4,103,002; python: 1,436,920; asm: 946,517; xml: 746,709; pascal: 187,653; perl: 88,691; sh: 88,436; objc: 79,953; sql: 51,488; cs: 44,583; fortran: 24,137; makefile: 22,147; tcl: 15,277; php: 13,980; yacc: 8,984; ruby: 7,485; awk: 3,720; lisp: 3,096; lex: 1,327; ada: 727; jsp: 228; sed: 36
file content (308 lines) | stat: -rw-r--r-- 9,317 bytes parent folder | download | duplicates (13)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
# Copyright 2022 the V8 project authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.


import contextlib
import json
import os
import shutil
import sys
import tempfile
import unittest

from contextlib import contextmanager
from dataclasses import dataclass
from io import StringIO
from mock import patch
from pathlib import Path

from testrunner.local.command import BaseCommand
from testrunner.objects import output
from testrunner.local.context import DefaultOSContext
from testrunner.local.pool import SingleThreadedExecutionPool
from testrunner.local.variants import REQUIRED_BUILD_VARIABLES

TOOLS_ROOT = Path(__file__).resolve().parent.parent.parent

TEST_DATA_ROOT = TOOLS_ROOT / 'testrunner' / 'testdata'
BUILD_CONFIG_BASE = TEST_DATA_ROOT / 'v8_build_config.json'

from testrunner.local import command
from testrunner.local import pool

@contextlib.contextmanager
def temp_dir():
  """Wrapper making a temporary directory available."""
  path = None
  try:
    path = Path(tempfile.mkdtemp('_v8_test'))
    yield path
  finally:
    if path:
      shutil.rmtree(path)


@contextlib.contextmanager
def temp_base(baseroot='testroot1'):
  """Wrapper that sets up a temporary V8 test root.

  Args:
    baseroot: The folder with the test root blueprint. All files will be
        copied to the temporary test root, to guarantee a fresh setup with no
        dirty state.
  """
  basedir = TEST_DATA_ROOT / baseroot
  with temp_dir() as tempbase:
    if basedir.exists():
      shutil.copytree(basedir, tempbase, dirs_exist_ok=True)
    yield tempbase


@contextlib.contextmanager
def capture():
  """Wrapper that replaces system stdout/stderr an provides the streams."""
  oldout = sys.stdout
  olderr = sys.stderr
  try:
    stdout=StringIO()
    stderr=StringIO()
    sys.stdout = stdout
    sys.stderr = stderr
    yield stdout, stderr
  finally:
    sys.stdout = oldout
    sys.stderr = olderr


def with_json_output(basedir):
  """ Function used as a placeholder where we need to resolve the value in the
  context of a temporary test configuration"""
  return basedir / 'out.json'

def clean_json_output(json_path, basedir):
  # Extract relevant properties of the json output.
  if not json_path:
    return None
  if not json_path.exists():
    return '--file-does-not-exists--'
  with open(json_path) as f:
    json_output = json.load(f)

  # Replace duration in actual output as it's non-deterministic. Also
  # replace the python executable prefix as it has a different absolute
  # path dependent on where this runs.
  def replace_variable_data(data):
    data['duration'] = 1
    data['max_rss'] = 1
    data['max_vms'] = 1
    data['command'] = ' '.join(
        ['/usr/bin/python'] + data['command'].split()[1:])
    data['command'] = data['command'].replace(f'{basedir}/', '')
  for container in [
      'max_rss_tests', 'max_vms_tests','slowest_tests', 'results']:
    for data in json_output[container]:
      replace_variable_data(data)
  json_output['duration_mean'] = 1
  # We need lexicographic sorting here to avoid non-deterministic behaviour
  # The original sorting key is duration or memory, but in our fake test we
  # have non-deterministic values before we reset them to 1.
  def sort_key(x):
    return str(sorted(x.items()))
  for container in [
      'max_rss_tests', 'max_vms_tests','slowest_tests']:
    json_output[container].sort(key=sort_key)
  return json_output


def test_schedule_log(json_path):
  if not json_path:
    return None
  with open(json_path.parent / 'test_schedule.log') as f:
    return f.read()


def setup_build_config(basedir, outdir):
  """Ensure a build config file exists - default or from test root."""
  path = basedir / outdir / 'build' / 'v8_build_config.json'
  if path.exists():
    return

  # Use default build-config blueprint.
  with open(BUILD_CONFIG_BASE) as f:
    config = json.load(f)

  # Add defaults for all variables used in variant configs.
  for key in REQUIRED_BUILD_VARIABLES:
    config[key] = False

  os.makedirs(path.parent, exist_ok=True)
  with open(path, 'w') as f:
    json.dump(config, f)

def override_build_config(basedir, **kwargs):
  """Override the build config with new values provided as kwargs."""
  if not kwargs:
    return
  path = basedir / 'out' / 'build' / 'v8_build_config.json'
  with open(path) as f:
    config = json.load(f)
    config.update(kwargs)
  with open(path, 'w') as f:
    json.dump(config, f)

@dataclass
class TestResult():
  stdout: str
  stderr: str
  returncode: int
  json: str
  test_schedule: str
  current_test_case: unittest.TestCase

  def __str__(self):
    return f'\nReturncode: {self.returncode}\nStdout:\n{self.stdout}\nStderr:\n{self.stderr}\n'

  def has_returncode(self, code):
    self.current_test_case.assertEqual(code, self.returncode, self)

  def stdout_includes(self, content):
    self.current_test_case.assertIn(content, self.stdout, self)

  def stdout_excludes(self, content):
    self.current_test_case.assertNotIn(content, self.stdout, self)

  def stderr_includes(self, content):
    self.current_test_case.assertIn(content, self.stderr, self)

  def stderr_excludes(self, content):
    self.current_test_case.assertNotIn(content, self.stderr, self)

  def json_content_equals(self, expected_results_file):
    with open(TEST_DATA_ROOT / expected_results_file) as f:
      expected_test_results = json.load(f)

    pretty_json = json.dumps(self.json, indent=2, sort_keys=True)
    msg = None  # Set to pretty_json for bootstrapping.
    self.current_test_case.assertDictEqual(self.json, expected_test_results, msg)


class TestRunnerTest(unittest.TestCase):
  @classmethod
  def setUpClass(cls):
    command.setup_testing()
    pool.setup_testing()

  def run_tests(
      self, *args, baseroot='testroot1', config_overrides=None,
      with_build_config=True, outdir='out', **kwargs):
    """Executes the test runner with captured output."""
    with temp_base(baseroot=baseroot) as basedir:
      if with_build_config:
        setup_build_config(basedir, outdir)
      override_build_config(basedir, **(config_overrides or {}))
      json_out_path = None
      def resolve_arg(arg):
        """Some arguments come as function objects to be called (resolved)
        in the context of a temporary test configuration"""
        nonlocal json_out_path
        if arg == with_json_output:
          json_out_path = with_json_output(basedir)
          return json_out_path
        return arg
      resolved_args = [resolve_arg(arg) for arg in args]
      with capture() as (stdout, stderr):
        sys_args = ['--command-prefix', sys.executable] + resolved_args
        if kwargs.get('infra_staging', False):
          sys_args.append('--infra-staging')
        else:
          sys_args.append('--no-infra-staging')
        runner = self.get_runner_class()(basedir=basedir)
        code = runner.execute(sys_args)
        json_out = clean_json_output(json_out_path, basedir)
        test_schedule = test_schedule_log(json_out_path)
        return TestResult(
            stdout.getvalue(), stderr.getvalue(), code, json_out, test_schedule, self)

  def get_runner_options(self, baseroot='testroot1'):
    """Returns a list of all flags parsed by the test runner."""
    with temp_base(baseroot=baseroot) as basedir:
      runner = self.get_runner_class()(basedir=basedir)
      parser = runner._create_parser()
      return [i.get_opt_string() for i in parser.option_list]

  def get_runner_class():
    """Implement to return the runner class"""
    return None

  @contextmanager
  def with_fake_rdb(self):
    records = []

    def fake_sink():
      return True

    class Fake_RPC:

      def __init__(self, sink):
        pass

      def send(self, r):
        records.append(r)

    with patch('testrunner.testproc.progress.rdb_sink', fake_sink), \
        patch('testrunner.testproc.resultdb.ResultDB_RPC', Fake_RPC):
      yield records


class FakeOSContext(DefaultOSContext):

  def __init__(self):
    super(FakeOSContext, self).__init__(FakeCommand,
                                        SingleThreadedExecutionPool())

  @contextmanager
  def handle_context(self, options):
    print("===>Starting stuff")
    yield
    print("<===Stopping stuff")

  def on_load(self):
    print("<==>Loading stuff")


class FakeCommand(BaseCommand):
  counter = 0

  def __init__(self,
               shell,
               args=None,
               cmd_prefix=None,
               timeout=60,
               env=None,
               verbose=False,
               test_case=None,
               handle_sigterm=False,
               log_process_stats=False):
    f_prefix = ['fake_wrapper'] + cmd_prefix
    super(FakeCommand, self).__init__(
        shell,
        args=args,
        cmd_prefix=f_prefix,
        timeout=timeout,
        env=env,
        verbose=verbose,
        handle_sigterm=handle_sigterm,
        log_process_stats=log_process_stats)

  def execute(self):
    FakeCommand.counter += 1
    return output.Output(
        0,  #return_code,
        False,  # TODO: Figure out timeouts.
        f'fake stdout {FakeCommand.counter}',
        f'fake stderr {FakeCommand.counter}',
        -1,  # No pid available.
        start_time=1,
        end_time=100,
    )