import json import platform import pytest pytest_plugins = ('pytester',) platform # noqa: B018 def test_help(testdir): result = testdir.runpytest_subprocess('--help') result.stdout.fnmatch_lines( [ '*', '*', 'benchmark:', ' --benchmark-min-time=SECONDS', " *Default: '0.000005'", ' --benchmark-max-time=SECONDS', " *Default: '1.0'", ' --benchmark-min-rounds=NUM', ' *Default: 5', ' --benchmark-timer=FUNC', ' --benchmark-calibration-precision=NUM', ' *Default: 10', ' --benchmark-warmup=[KIND]', ' --benchmark-warmup-iterations=NUM', ' *Default: 100000', ' --benchmark-disable-gc', ' --benchmark-skip *', ' --benchmark-only *', ' --benchmark-save=NAME', ' --benchmark-autosave *', ' --benchmark-save-data', ' --benchmark-json=PATH', ' --benchmark-compare=[NUM|_ID]', ' --benchmark-compare-fail=EXPR?[[]EXPR?...[]]', ' --benchmark-cprofile=COLUMN', ' --benchmark-storage=URI', " *Default: 'file://./.benchmarks'.", ' --benchmark-verbose *', ' --benchmark-sort=COL *', ' --benchmark-group-by=LABEL', " *Default: 'group'", ' --benchmark-columns=LABELS', ' --benchmark-histogram=[FILENAME-PREFIX]', '*', ] ) def test_groups(testdir): test = testdir.makepyfile( '''""" >>> print('Yay, doctests!') Yay, doctests! """ import time import pytest def test_fast(benchmark): benchmark(lambda: time.sleep(0.000001)) assert 1 == 1 def test_slow(benchmark): benchmark(lambda: time.sleep(0.001)) assert 1 == 1 @pytest.mark.benchmark(group="A") def test_slower(benchmark): benchmark(lambda: time.sleep(0.01)) assert 1 == 1 @pytest.mark.benchmark(group="A", warmup=True) def test_xfast(benchmark): benchmark(lambda: None) assert 1 == 1 ''' ) result = testdir.runpytest_subprocess('-vv', '--doctest-modules', test) result.stdout.fnmatch_lines( [ '*collected 5 items', '*', 'test_groups.py::*test_groups PASSED*', 'test_groups.py::test_fast PASSED*', 'test_groups.py::test_slow PASSED*', 'test_groups.py::test_slower PASSED*', 'test_groups.py::test_xfast PASSED*', '*', '* benchmark: 2 tests *', '*', "* benchmark 'A': 2 tests *", '*', '*====== 5 passed * ======*', ] ) SIMPLE_TEST = ''' """ >>> print('Yay, doctests!') Yay, doctests! """ import time import pytest def test_fast(benchmark): @benchmark def result(): return time.sleep(0.000001) assert result == None def test_slow(benchmark): benchmark(lambda: time.sleep(0.1)) assert 1 == 1 ''' FIXTURES_ALSO_SKIPPED_TEST = ''' import pytest @pytest.fixture def dep(): print(""" dep created""") def test_normal(dep): pass def test_bench(benchmark): benchmark.pedantic(lambda: None) ''' GROUPING_TEST = """ import pytest @pytest.mark.parametrize("foo", range(2)) @pytest.mark.benchmark(group="A") def test_a(benchmark, foo): benchmark(str) @pytest.mark.parametrize("foo", range(2)) @pytest.mark.benchmark(group="B") def test_b(benchmark, foo): benchmark(int) """ GROUPING_PARAMS_TEST = """ import pytest @pytest.mark.parametrize("bar", ["bar1", "bar2"]) @pytest.mark.parametrize("foo", ["foo1", "foo2"]) @pytest.mark.benchmark(group="A") def test_a(benchmark, foo, bar): benchmark(str) @pytest.mark.parametrize("bar", ["bar1", "bar2"]) @pytest.mark.parametrize("foo", ["foo1", "foo2"]) @pytest.mark.benchmark(group="B") def test_b(benchmark, foo, bar): benchmark(int) """ def test_group_by_name(testdir): test_x = testdir.makepyfile(test_x=GROUPING_TEST) test_y = testdir.makepyfile(test_y=GROUPING_TEST) result = testdir.runpytest_subprocess('--benchmark-max-time=0.0000001', '--benchmark-group-by', 'name', test_x, test_y) result.stdout.fnmatch_lines( [ '*', '*', '*', '*', '*', "* benchmark 'test_a[[]0[]]': 2 tests *", 'Name (time in ?s) *', '----------------------*', 'test_a[[]0[]] *', 'test_a[[]0[]] *', '----------------------*', '*', "* benchmark 'test_a[[]1[]]': 2 tests *", 'Name (time in ?s) *', '----------------------*', 'test_a[[]1[]] *', 'test_a[[]1[]] *', '----------------------*', '*', "* benchmark 'test_b[[]0[]]': 2 tests *", 'Name (time in ?s) *', '----------------------*', 'test_b[[]0[]] *', 'test_b[[]0[]] *', '----------------------*', '*', "* benchmark 'test_b[[]1[]]': 2 tests *", 'Name (time in ?s) *', '----------------------*', 'test_b[[]1[]] *', 'test_b[[]1[]] *', '----------------------*', ] ) def test_group_by_func(testdir): test_x = testdir.makepyfile(test_x=GROUPING_TEST) test_y = testdir.makepyfile(test_y=GROUPING_TEST) result = testdir.runpytest_subprocess('--benchmark-max-time=0.0000001', '--benchmark-group-by', 'func', test_x, test_y) result.stdout.fnmatch_lines( [ '*', '*', '*', '*', "* benchmark 'test_a': 4 tests *", 'Name (time in ?s) *', '----------------------*', 'test_a[[]*[]] *', 'test_a[[]*[]] *', 'test_a[[]*[]] *', 'test_a[[]*[]] *', '----------------------*', '*', "* benchmark 'test_b': 4 tests *", 'Name (time in ?s) *', '----------------------*', 'test_b[[]*[]] *', 'test_b[[]*[]] *', 'test_b[[]*[]] *', 'test_b[[]*[]] *', '----------------------*', '*', '*', '============* 8 passed * ============*', ] ) def test_group_by_fullfunc(testdir): test_x = testdir.makepyfile(test_x=GROUPING_TEST) test_y = testdir.makepyfile(test_y=GROUPING_TEST) result = testdir.runpytest_subprocess('--benchmark-max-time=0.0000001', '--benchmark-group-by', 'fullfunc', test_x, test_y) result.stdout.fnmatch_lines( [ '*', '*', '*', '*', '*', "* benchmark 'test_x.py::test_a': 2 tests *", 'Name (time in ?s) *', '------------------*', 'test_a[[]*[]] *', 'test_a[[]*[]] *', '------------------*', '', "* benchmark 'test_x.py::test_b': 2 tests *", 'Name (time in ?s) *', '------------------*', 'test_b[[]*[]] *', 'test_b[[]*[]] *', '------------------*', '', "* benchmark 'test_y.py::test_a': 2 tests *", 'Name (time in ?s) *', '------------------*', 'test_a[[]*[]] *', 'test_a[[]*[]] *', '------------------*', '', "* benchmark 'test_y.py::test_b': 2 tests *", 'Name (time in ?s) *', '------------------*', 'test_b[[]*[]] *', 'test_b[[]*[]] *', '------------------*', '', 'Legend:', ' Outliers: 1 Standard Deviation from M*', '============* 8 passed * ============*', ] ) def test_group_by_param_all(testdir): test_x = testdir.makepyfile(test_x=GROUPING_TEST) test_y = testdir.makepyfile(test_y=GROUPING_TEST) result = testdir.runpytest_subprocess('--benchmark-max-time=0.0000001', '--benchmark-group-by', 'param', test_x, test_y) result.stdout.fnmatch_lines( [ '*', '*', '*', '*', '*', "* benchmark '0': 4 tests *", 'Name (time in ?s) *', '-------------------*', 'test_*[[]0[]] *', 'test_*[[]0[]] *', 'test_*[[]0[]] *', 'test_*[[]0[]] *', '-------------------*', '', "* benchmark '1': 4 tests *", 'Name (time in ?s) *', '------------------*', 'test_*[[]1[]] *', 'test_*[[]1[]] *', 'test_*[[]1[]] *', 'test_*[[]1[]] *', '------------------*', '', 'Legend:', ' Outliers: 1 Standard Deviation from Mean; 1.5 IQR (InterQuartile Range) from 1st Quartile and 3rd ' 'Quartile.', '============* 8 passed * ============*', ] ) def test_group_by_param_select(testdir): test_x = testdir.makepyfile(test_x=GROUPING_PARAMS_TEST) result = testdir.runpytest_subprocess( '--benchmark-max-time=0.0000001', '--benchmark-group-by', 'param:foo', '--benchmark-sort', 'fullname', test_x ) result.stdout.fnmatch_lines( [ '*', '*', '*', '*', '*', "* benchmark 'foo=foo1': 4 tests *", 'Name (time in ?s) *', '-------------------*', 'test_a[[]foo1-bar1[]] *', 'test_a[[]foo1-bar2[]] *', 'test_b[[]foo1-bar1[]] *', 'test_b[[]foo1-bar2[]] *', '-------------------*', '', "* benchmark 'foo=foo2': 4 tests *", 'Name (time in ?s) *', '------------------*', 'test_a[[]foo2-bar1[]] *', 'test_a[[]foo2-bar2[]] *', 'test_b[[]foo2-bar1[]] *', 'test_b[[]foo2-bar2[]] *', '------------------*', '', 'Legend:', ' Outliers: 1 Standard Deviation from Mean; 1.5 IQR (InterQuartile Range) from 1st Quartile and 3rd ' 'Quartile.', '============* 8 passed * ============*', ] ) def test_group_by_param_select_multiple(testdir): test_x = testdir.makepyfile(test_x=GROUPING_PARAMS_TEST) result = testdir.runpytest_subprocess( '--benchmark-max-time=0.0000001', '--benchmark-group-by', 'param:foo,param:bar', '--benchmark-sort', 'fullname', test_x ) result.stdout.fnmatch_lines( [ '*', '*', '*', '*', '*', "* benchmark 'foo=foo1 bar=bar1': 2 tests *", 'Name (time in ?s) *', '-------------------*', 'test_a[[]foo1-bar1[]] *', 'test_b[[]foo1-bar1[]] *', '-------------------*', '', "* benchmark 'foo=foo1 bar=bar2': 2 tests *", 'Name (time in ?s) *', '-------------------*', 'test_a[[]foo1-bar2[]] *', 'test_b[[]foo1-bar2[]] *', '-------------------*', '', "* benchmark 'foo=foo2 bar=bar1': 2 tests *", 'Name (time in ?s) *', '------------------*', 'test_a[[]foo2-bar1[]] *', 'test_b[[]foo2-bar1[]] *', '-------------------*', '', "* benchmark 'foo=foo2 bar=bar2': 2 tests *", 'Name (time in ?s) *', '-------------------*', 'test_a[[]foo2-bar2[]] *', 'test_b[[]foo2-bar2[]] *', '------------------*', '', 'Legend:', ' Outliers: 1 Standard Deviation from Mean; 1.5 IQR (InterQuartile Range) from 1st Quartile and 3rd ' 'Quartile.', '============* 8 passed * ============*', ] ) def test_group_by_fullname(testdir): test_x = testdir.makepyfile(test_x=GROUPING_TEST) test_y = testdir.makepyfile(test_y=GROUPING_TEST) result = testdir.runpytest_subprocess('--benchmark-max-time=0.0000001', '--benchmark-group-by', 'fullname', test_x, test_y) result.stdout.fnmatch_lines_random( [ "* benchmark 'test_x.py::test_a[[]0[]]': 1 tests *", "* benchmark 'test_x.py::test_a[[]1[]]': 1 tests *", "* benchmark 'test_x.py::test_b[[]0[]]': 1 tests *", "* benchmark 'test_x.py::test_b[[]1[]]': 1 tests *", "* benchmark 'test_y.py::test_a[[]0[]]': 1 tests *", "* benchmark 'test_y.py::test_a[[]1[]]': 1 tests *", "* benchmark 'test_y.py::test_b[[]0[]]': 1 tests *", "* benchmark 'test_y.py::test_b[[]1[]]': 1 tests *", '============* 8 passed * ============*', ] ) def test_double_use(testdir): test = testdir.makepyfile( """ def test_a(benchmark): benchmark(lambda: None) benchmark.pedantic(lambda: None) def test_b(benchmark): benchmark.pedantic(lambda: None) benchmark(lambda: None) """ ) result = testdir.runpytest_subprocess(test, '--tb=line') result.stdout.fnmatch_lines( [ '*FixtureAlreadyUsed: Fixture can only be used once. Previously it was used in benchmark(...) mode.', '*FixtureAlreadyUsed: Fixture can only be used once. Previously it was used in benchmark.pedantic(...) mode.', ] ) def test_only_override_skip(testdir): test = testdir.makepyfile(SIMPLE_TEST) result = testdir.runpytest_subprocess('--benchmark-only', '--benchmark-skip', test) result.stdout.fnmatch_lines( [ '*collected 2 items', 'test_only_override_skip.py ..*', '* benchmark: 2 tests *', 'Name (time in ?s) * Min * Max * Mean * StdDev * Rounds * Iterations', '------*', 'test_fast *', 'test_slow *', '------*', '*====== 2 passed * ======*', ] ) def test_fixtures_also_skipped(testdir): test = testdir.makepyfile(FIXTURES_ALSO_SKIPPED_TEST) result = testdir.runpytest_subprocess('--benchmark-only', '-s', test) result.stdout.fnmatch_lines( [ '*collected 2 items', '*====== 1 passed, 1 skipped in * ======*', ] ) assert 'dep created' not in result.stdout.lines def test_conflict_between_only_and_disable(testdir): test = testdir.makepyfile(SIMPLE_TEST) result = testdir.runpytest_subprocess('--benchmark-only', '--benchmark-disable', test) result.stderr.fnmatch_lines( [ "ERROR: Can't have both --benchmark-only and --benchmark-disable options. Note that --benchmark-disable is " "automatically activated if xdist is on or you're missing the statistics dependency." ] ) def test_max_time_min_rounds(testdir): test = testdir.makepyfile(SIMPLE_TEST) result = testdir.runpytest_subprocess('--doctest-modules', '--benchmark-max-time=0.000001', '--benchmark-min-rounds=1', test) result.stdout.fnmatch_lines( [ '*collected 3 items', 'test_max_time_min_rounds.py ...*', '* benchmark: 2 tests *', 'Name (time in ?s) * Min * Max * Mean * StdDev * Rounds * Iterations', '------*', 'test_fast * 1 *', 'test_slow * 1 *', '------*', '*====== 3 passed * ======*', ] ) def test_max_time(testdir): test = testdir.makepyfile(SIMPLE_TEST) result = testdir.runpytest_subprocess('--doctest-modules', '--benchmark-max-time=0.000001', test) result.stdout.fnmatch_lines( [ '*collected 3 items', 'test_max_time.py ...*', '* benchmark: 2 tests *', 'Name (time in ?s) * Min * Max * Mean * StdDev * Rounds * Iterations', '------*', 'test_fast * 5 *', 'test_slow * 5 *', '------*', '*====== 3 passed * ======*', ] ) def test_bogus_max_time(testdir): test = testdir.makepyfile(SIMPLE_TEST) result = testdir.runpytest_subprocess('--doctest-modules', '--benchmark-max-time=bogus', test) result.stderr.fnmatch_lines( [ '*usage: *', "*: error: argument --benchmark-max-time: Invalid decimal value 'bogus': InvalidOperation*", ] ) @pytest.mark.skipif("platform.python_implementation() == 'PyPy'") def test_pep418_timer(testdir): test = testdir.makepyfile(SIMPLE_TEST) result = testdir.runpytest_subprocess( '--benchmark-max-time=0.0000001', '--doctest-modules', '--benchmark-timer=pep418.perf_counter', test ) result.stdout.fnmatch_lines( [ '* (defaults: timer=*.perf_counter*', ] ) def test_bad_save(testdir): test = testdir.makepyfile(SIMPLE_TEST) result = testdir.runpytest_subprocess('--doctest-modules', '--benchmark-save=asd:f?', test) result.stderr.fnmatch_lines( [ '*usage: *', "*: error: argument --benchmark-save: Must not contain any of these characters: /:*?<>|\\ (it has ':?')", ] ) def test_bad_save_2(testdir): test = testdir.makepyfile(SIMPLE_TEST) result = testdir.runpytest_subprocess('--doctest-modules', '--benchmark-save=', test) result.stderr.fnmatch_lines( [ '*usage: *', "*: error: argument --benchmark-save: Can't be empty.", ] ) def test_bad_compare_fail(testdir): test = testdir.makepyfile(SIMPLE_TEST) result = testdir.runpytest_subprocess('--doctest-modules', '--benchmark-compare-fail=?', test) result.stderr.fnmatch_lines( [ '*usage: *', "*: error: argument --benchmark-compare-fail: Could not parse value: '?'.", ] ) def test_bad_rounds(testdir): test = testdir.makepyfile(SIMPLE_TEST) result = testdir.runpytest_subprocess('--doctest-modules', '--benchmark-min-rounds=asd', test) result.stderr.fnmatch_lines( [ '*usage: *', "*: error: argument --benchmark-min-rounds: invalid literal for int() with base 10: 'asd'", ] ) def test_bad_rounds_2(testdir): test = testdir.makepyfile(SIMPLE_TEST) result = testdir.runpytest_subprocess('--doctest-modules', '--benchmark-min-rounds=0', test) result.stderr.fnmatch_lines( [ '*usage: *', '*: error: argument --benchmark-min-rounds: Value for --benchmark-rounds must be at least 1.', ] ) def test_compare(testdir): test = testdir.makepyfile(SIMPLE_TEST) testdir.runpytest_subprocess('--benchmark-max-time=0.0000001', '--doctest-modules', '--benchmark-autosave', test) result = testdir.runpytest_subprocess( '--benchmark-max-time=0.0000001', '--doctest-modules', '--benchmark-compare=0001', '--benchmark-compare-fail=min:0.1', test ) result.stderr.fnmatch_lines( [ 'Comparing against benchmarks from: *0001_unversioned_*.json', ] ) result = testdir.runpytest_subprocess( '--benchmark-max-time=0.0000001', '--doctest-modules', '--benchmark-compare=0001', '--benchmark-compare-fail=min:1%', test ) result.stderr.fnmatch_lines( [ 'Comparing against benchmarks from: *0001_unversioned_*.json', ] ) def test_compare_last(testdir): test = testdir.makepyfile(SIMPLE_TEST) testdir.runpytest_subprocess('--benchmark-max-time=0.0000001', '--doctest-modules', '--benchmark-autosave', test) result = testdir.runpytest_subprocess( '--benchmark-max-time=0.0000001', '--doctest-modules', '--benchmark-compare', '--benchmark-compare-fail=min:0.1', test ) result.stderr.fnmatch_lines( [ 'Comparing against benchmarks from: *0001_unversioned_*.json', ] ) result = testdir.runpytest_subprocess( '--benchmark-max-time=0.0000001', '--doctest-modules', '--benchmark-compare', '--benchmark-compare-fail=min:1%', test ) result.stderr.fnmatch_lines( [ 'Comparing against benchmarks from: *0001_unversioned_*.json', ] ) def test_compare_non_existing(testdir): test = testdir.makepyfile(SIMPLE_TEST) testdir.runpytest_subprocess('--benchmark-max-time=0.0000001', '--doctest-modules', '--benchmark-autosave', test) result = testdir.runpytest_subprocess('--benchmark-max-time=0.0000001', '--doctest-modules', '--benchmark-compare=0002', '-rw', test) result.stderr.fnmatch_lines( [ "* PytestBenchmarkWarning: Can't compare. No benchmark files * '0002'.", ] ) def test_compare_non_existing_verbose(testdir): test = testdir.makepyfile(SIMPLE_TEST) testdir.runpytest_subprocess('--benchmark-max-time=0.0000001', '--doctest-modules', '--benchmark-autosave', test) result = testdir.runpytest_subprocess( '--benchmark-max-time=0.0000001', '--doctest-modules', '--benchmark-compare=0002', test, '--benchmark-verbose' ) result.stderr.fnmatch_lines( [ " WARNING: Can't compare. No benchmark files * '0002'.", ] ) def test_compare_no_files(testdir): test = testdir.makepyfile(SIMPLE_TEST) result = testdir.runpytest_subprocess('--benchmark-max-time=0.0000001', '--doctest-modules', '-rw', test, '--benchmark-compare') result.stderr.fnmatch_lines(["* PytestBenchmarkWarning: Can't compare. No benchmark files in '*'. Can't load the previous benchmark."]) def test_compare_no_files_verbose(testdir): test = testdir.makepyfile(SIMPLE_TEST) result = testdir.runpytest_subprocess( '--benchmark-max-time=0.0000001', '--doctest-modules', test, '--benchmark-compare', '--benchmark-verbose' ) result.stderr.fnmatch_lines([" WARNING: Can't compare. No benchmark files in '*'." " Can't load the previous benchmark."]) def test_compare_no_files_match(testdir): test = testdir.makepyfile(SIMPLE_TEST) result = testdir.runpytest_subprocess('--benchmark-max-time=0.0000001', '--doctest-modules', '-rw', test, '--benchmark-compare=1') result.stderr.fnmatch_lines(["* PytestBenchmarkWarning: Can't compare. No benchmark files in '*' match '1'."]) def test_compare_no_files_match_verbose(testdir): test = testdir.makepyfile(SIMPLE_TEST) result = testdir.runpytest_subprocess( '--benchmark-max-time=0.0000001', '--doctest-modules', test, '--benchmark-compare=1', '--benchmark-verbose' ) result.stderr.fnmatch_lines([" WARNING: Can't compare. No benchmark files in '*' match '1'."]) def test_verbose(testdir): test = testdir.makepyfile(SIMPLE_TEST) result = testdir.runpytest_subprocess('--benchmark-max-time=0.0000001', '--doctest-modules', '--benchmark-verbose', '-vv', test) result.stderr.fnmatch_lines( [ ' Calibrating to target round *s; will estimate when reaching *s (using: *, precision: *).', ' Measured * iterations: *s.', ' Running * rounds x * iterations ...', ' Ran for *s.', ] ) def test_save(testdir): test = testdir.makepyfile(SIMPLE_TEST) result = testdir.runpytest_subprocess('--doctest-modules', '--benchmark-save=foobar', '--benchmark-max-time=0.0000001', test) result.stderr.fnmatch_lines( [ 'Saved benchmark data in: *', ] ) json.loads(testdir.tmpdir.join('.benchmarks').listdir()[0].join('0001_foobar.json').read()) def test_save_extra_info(testdir): test = testdir.makepyfile( """ def test_extra(benchmark): benchmark.extra_info['foo'] = 'bar' benchmark(lambda: None) """ ) result = testdir.runpytest_subprocess('--doctest-modules', '--benchmark-save=foobar', '--benchmark-max-time=0.0000001', test) result.stderr.fnmatch_lines( [ 'Saved benchmark data in: *', ] ) info = json.loads(testdir.tmpdir.join('.benchmarks').listdir()[0].join('0001_foobar.json').read()) bench_info = info['benchmarks'][0] assert bench_info['name'] == 'test_extra' assert bench_info['extra_info'] == {'foo': 'bar'} def test_update_machine_info_hook_detection(testdir): """Tests detection and execution and update_machine_info_hooks. Verifies that machine info hooks are detected and executed in nested `conftest.py`s. """ record_path_conftest = """ import os def pytest_benchmark_update_machine_info(config, machine_info): machine_info["conftest_path"] = ( machine_info.get("conftest_path", []) + [os.path.relpath(__file__)] ) """ simple_test = """ def test_simple(benchmark): @benchmark def resuilt(): 1+1 """ testdir.makepyfile( **{ 'conftest': record_path_conftest, 'test_module/conftest': record_path_conftest, 'test_module/tests/conftest': record_path_conftest, 'test_module/tests/simple_test.py': simple_test, } ) def run_verify_pytest(*args): testdir.runpytest_subprocess('--benchmark-json=benchmark.json', '--benchmark-max-time=0.0000001', *args) benchmark_json = json.loads(testdir.tmpdir.join('benchmark.json').read()) machine_info = benchmark_json['machine_info'] assert sorted(i.replace('\\', '/') for i in machine_info['conftest_path']) == sorted( [ 'conftest.py', 'test_module/conftest.py', 'test_module/tests/conftest.py', ] ) run_verify_pytest('test_module/tests') run_verify_pytest('test_module') run_verify_pytest('.') def test_histogram(testdir): test = testdir.makepyfile(SIMPLE_TEST) result = testdir.runpytest_subprocess('--doctest-modules', '--benchmark-histogram=foobar', '--benchmark-max-time=0.0000001', test) result.stderr.fnmatch_lines( [ 'Generated histogram: *foobar.svg', ] ) assert [f.basename for f in testdir.tmpdir.listdir('*.svg', sort=True)] == [ 'foobar.svg', ] def test_autosave(testdir): test = testdir.makepyfile(SIMPLE_TEST) result = testdir.runpytest_subprocess('--doctest-modules', '--benchmark-autosave', '--benchmark-max-time=0.0000001', test) result.stderr.fnmatch_lines( [ 'Saved benchmark data in: *', ] ) json.loads(testdir.tmpdir.join('.benchmarks').listdir()[0].listdir('0001_*.json')[0].read()) def test_bogus_min_time(testdir): test = testdir.makepyfile(SIMPLE_TEST) result = testdir.runpytest_subprocess('--doctest-modules', '--benchmark-min-time=bogus', test) result.stderr.fnmatch_lines( [ '*usage: *', "*: error: argument --benchmark-min-time: Invalid decimal value 'bogus': InvalidOperation*", ] ) def test_disable_gc(testdir): test = testdir.makepyfile(SIMPLE_TEST) result = testdir.runpytest_subprocess('--benchmark-disable-gc', test) result.stdout.fnmatch_lines( [ '*collected 2 items', 'test_disable_gc.py ..*', '* benchmark: 2 tests *', 'Name (time in ?s) * Min * Max * Mean * StdDev * Rounds * Iterations', '------*', 'test_fast *', 'test_slow *', '------*', '*====== 2 passed * ======*', ] ) def test_custom_timer(testdir): test = testdir.makepyfile(SIMPLE_TEST) result = testdir.runpytest_subprocess('--benchmark-timer=time.time', test) result.stdout.fnmatch_lines( [ '*collected 2 items', 'test_custom_timer.py ..*', '* benchmark: 2 tests *', 'Name (time in ?s) * Min * Max * Mean * StdDev * Rounds * Iterations', '------*', 'test_fast *', 'test_slow *', '------*', '*====== 2 passed * ======*', ] ) def test_bogus_timer(testdir): test = testdir.makepyfile(SIMPLE_TEST) result = testdir.runpytest_subprocess('--benchmark-timer=bogus', test) result.stderr.fnmatch_lines( [ '*usage: *', '*: error: argument --benchmark-timer: Value for --benchmark-timer must be in dotted form. Eg: ' "'module.attr'.", ] ) def test_sort_by_mean(testdir): test = testdir.makepyfile(SIMPLE_TEST) result = testdir.runpytest_subprocess('--benchmark-sort=mean', test) result.stdout.fnmatch_lines( [ '*collected 2 items', 'test_sort_by_mean.py ..*', '* benchmark: 2 tests *', 'Name (time in ?s) * Min * Max * Mean * StdDev * Rounds * Iterations', '------*', 'test_fast *', 'test_slow *', '------*', '*====== 2 passed * ======*', ] ) def test_bogus_sort(testdir): test = testdir.makepyfile(SIMPLE_TEST) result = testdir.runpytest_subprocess('--benchmark-sort=bogus', test) result.stderr.fnmatch_lines( [ '*usage: *', "*: error: argument --benchmark-sort: Unacceptable value: 'bogus'. Value for --benchmark-sort must be one " "of: 'min', 'max', 'mean', 'stddev', 'name', 'fullname'.", ] ) def test_xdist(testdir): pytest.importorskip('xdist') test = testdir.makepyfile(SIMPLE_TEST) result = testdir.runpytest_subprocess('--doctest-modules', '-n', '1', '-rw', test) result.stderr.fnmatch_lines( [ '* Benchmarks are automatically disabled because xdist plugin is active.Benchmarks cannot be ' 'performed reliably in a parallelized environment.', ] ) def test_xdist_verbose(testdir): pytest.importorskip('xdist') test = testdir.makepyfile(SIMPLE_TEST) result = testdir.runpytest_subprocess('--doctest-modules', '-n', '1', '--benchmark-verbose', test) result.stderr.fnmatch_lines( [ '------*', ' WARNING: Benchmarks are automatically disabled because xdist plugin is active.Benchmarks cannot be performed ' 'reliably in a parallelized environment.', '------*', ] ) def test_cprofile(testdir): test = testdir.makepyfile(SIMPLE_TEST) result = testdir.runpytest_subprocess('--benchmark-cprofile=cumtime', test) result.stdout.fnmatch_lines( [ '------------*----------- cProfile (time in s) ------------*-----------', 'test_cprofile.py::test_fast', 'ncalls tottime percall cumtime percall filename:lineno(function)', # "1 0.0000 0.0000 0.0001 0.0001 test_cprofile0/test_cprofile.py:9(result)", # "1 0.0001 0.0001 0.0001 0.0001 ~:0()", # "1 0.0000 0.0000 0.0000 0.0000 ~:0()", '', 'test_cprofile.py::test_slow', 'ncalls tottime percall cumtime percall filename:lineno(function)', # "1 0.0000 0.0000 0.1002 0.1002 test_cprofile0/test_cprofile.py:15()", # "1 0.1002 0.1002 0.1002 0.1002 ~:0()", # "1 0.0000 0.0000 0.0000 0.0000 ~:0()", ] ) def test_disabled_and_cprofile(testdir): test = testdir.makepyfile(SIMPLE_TEST) result = testdir.runpytest_subprocess('--benchmark-disable', '--benchmark-cprofile=cumtime', test) result.stdout.fnmatch_lines( [ '*==== 2 passed*', ] ) def test_abort_broken(testdir): """ Test that we don't benchmark code that raises exceptions. """ test = testdir.makepyfile( ''' """ >>> print('Yay, doctests!') Yay, doctests! """ import time import pytest def test_bad(benchmark): @benchmark def result(): raise Exception() assert 1 == 1 def test_bad2(benchmark): @benchmark def result(): time.sleep(0.1) assert 1 == 0 @pytest.fixture(params=['a', 'b', 'c']) def bad_fixture(request): raise ImportError() def test_ok(benchmark, bad_fixture): @benchmark def result(): time.sleep(0.1) assert 1 == 0 ''' ) result = testdir.runpytest_subprocess('-vv', test) result.stdout.fnmatch_lines( [ '*collected 5 items', '*', 'test_abort_broken.py::test_bad FAILED*', 'test_abort_broken.py::test_bad2 FAILED*', 'test_abort_broken.py::test_ok*a* ERROR*', 'test_abort_broken.py::test_ok*b* ERROR*', 'test_abort_broken.py::test_ok*c* ERROR*', '*====== ERRORS ======*', '*______ ERROR at setup of test_ok[[]a[]] ______*', "request = >", " @pytest.fixture(params=['a', 'b', 'c'])", ' def bad_fixture(request):', '> raise ImportError()', 'E ImportError', 'test_abort_broken.py:22: ImportError', '*______ ERROR at setup of test_ok[[]b[]] ______*', "request = >", " @pytest.fixture(params=['a', 'b', 'c'])", ' def bad_fixture(request):', '> raise ImportError()', 'E ImportError', 'test_abort_broken.py:22: ImportError', '*______ ERROR at setup of test_ok[[]c[]] ______*', "request = >", " @pytest.fixture(params=['a', 'b', 'c'])", ' def bad_fixture(request):', '> raise ImportError()', 'E ImportError', 'test_abort_broken.py:22: ImportError', '*====== FAILURES ======*', '*______ test_bad ______*', 'benchmark = ', ' def test_bad(benchmark):', '? @benchmark', 'test_abort_broken.py:*', '_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _*', '*', '_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _*', ' @benchmark', ' def result():', '> raise Exception()', 'E Exception', 'test_abort_broken.py:11: Exception', '*______ test_bad2 ______*', 'benchmark = ', ' def test_bad2(benchmark):', ' @benchmark', ' def result():', ' time.sleep(0.1)', '> assert 1 == 0', 'E assert 1 == 0', 'test_abort_broken.py:18: AssertionError', ] ) result.stdout.fnmatch_lines( [ '* benchmark: 1 tests *', 'Name (time in ?s) * Min * Max * Mean * StdDev * Rounds * Iterations', '------*', 'test_bad2 *', '------*', '*====== 2 failed*, 3 error* ======*', ] ) BASIC_TEST = ''' """ Just to make sure the plugin doesn't choke on doctests:: >>> print('Yay, doctests!') Yay, doctests! """ import time from functools import partial import pytest def test_fast(benchmark): @benchmark def result(): return time.sleep(0.000001) assert result is None def test_slow(benchmark): assert benchmark(partial(time.sleep, 0.001)) is None def test_slower(benchmark): benchmark(lambda: time.sleep(0.01)) @pytest.mark.benchmark(min_rounds=2) def test_xfast(benchmark): benchmark(str) def test_fast(benchmark): benchmark(int) ''' def test_basic(testdir): test = testdir.makepyfile(BASIC_TEST) result = testdir.runpytest_subprocess('-vv', '--doctest-modules', test) result.stdout.fnmatch_lines( [ '*collected 5 items', 'test_basic.py::*test_basic PASSED*', 'test_basic.py::test_fast PASSED*', 'test_basic.py::test_slow PASSED*', 'test_basic.py::test_slower PASSED*', 'test_basic.py::test_xfast PASSED*', '', '* benchmark: 4 tests *', 'Name (time in ?s) * Min * Max * Mean * StdDev * Rounds * Iterations', '------*', 'test_* *', 'test_* *', 'test_* *', 'test_* *', '------*', '', '*====== 5 passed * ======*', ] ) def test_skip(testdir): test = testdir.makepyfile(BASIC_TEST) result = testdir.runpytest_subprocess('-vv', '--doctest-modules', '--benchmark-skip', test) result.stdout.fnmatch_lines( [ '*collected 5 items', 'test_skip.py::*test_skip PASSED*', 'test_skip.py::test_fast SKIPPED*', 'test_skip.py::test_slow SKIPPED*', 'test_skip.py::test_slower SKIPPED*', 'test_skip.py::test_xfast SKIPPED*', '*====== 1 passed, 4 skipped * ======*', ] ) def test_disable(testdir): test = testdir.makepyfile(BASIC_TEST) result = testdir.runpytest_subprocess('-vv', '--doctest-modules', '--benchmark-disable', test) result.stdout.fnmatch_lines( [ '*collected 5 items', 'test_disable.py::*test_disable PASSED*', 'test_disable.py::test_fast PASSED*', 'test_disable.py::test_slow PASSED*', 'test_disable.py::test_slower PASSED*', 'test_disable.py::test_xfast PASSED*', '*====== 5 passed * ======*', ] ) def test_mark_selection(testdir): test = testdir.makepyfile(BASIC_TEST) result = testdir.runpytest_subprocess('-vv', '--doctest-modules', '-m', 'benchmark', test) result.stdout.fnmatch_lines( [ '*collected 5 items*', 'test_mark_selection.py::test_xfast PASSED*', '* benchmark: 1 tests *', 'Name (time in ?s) * Min * Max * Mean * StdDev * Rounds * Iterations', '------*', 'test_xfast *', '------*', '*====== 1 passed, 4 deselected * ======*', ] ) def test_only_benchmarks(testdir): test = testdir.makepyfile(BASIC_TEST) result = testdir.runpytest_subprocess('-vv', '--doctest-modules', '--benchmark-only', test) result.stdout.fnmatch_lines( [ '*collected 5 items', 'test_only_benchmarks.py::*test_only_benchmarks SKIPPED*', 'test_only_benchmarks.py::test_fast PASSED*', 'test_only_benchmarks.py::test_slow PASSED*', 'test_only_benchmarks.py::test_slower PASSED*', 'test_only_benchmarks.py::test_xfast PASSED*', '* benchmark: 4 tests *', 'Name (time in ?s) * Min * Max * Mean * StdDev * Rounds * Iterations', '------*', 'test_* *', 'test_* *', 'test_* *', 'test_* *', '------*', '*====== 4 passed, 1 skipped * ======*', ] ) def test_columns(testdir): test = testdir.makepyfile(SIMPLE_TEST) result = testdir.runpytest_subprocess('--doctest-modules', '--benchmark-columns=max,iterations,min', test) result.stdout.fnmatch_lines( [ '*collected 3 items', 'test_columns.py ...*', '* benchmark: 2 tests *', 'Name (time in ?s) * Max * Iterations * Min *', '------*', ] )