# -*- coding: utf-8 -*- import unittest from typing import Dict, List from tools.stats import print_test_stats from tools.stats.s3_stat_parser import ( Commit, Report, ReportMetaMeta, Status, Version1Case, Version1Report, Version2Case, Version2Report, ) def fakehash(char: str) -> str: return char * 40 def dummy_meta_meta() -> ReportMetaMeta: return { "build_pr": "", "build_tag": "", "build_sha1": "", "build_base_commit": "", "build_branch": "", "build_job": "", "build_workflow_id": "", "build_start_time_epoch": "", } def makecase( name: str, seconds: float, *, errored: bool = False, failed: bool = False, skipped: bool = False, ) -> Version1Case: return { "name": name, "seconds": seconds, "errored": errored, "failed": failed, "skipped": skipped, } def make_report_v1(tests: Dict[str, List[Version1Case]]) -> Version1Report: suites = { suite_name: { "total_seconds": sum(case["seconds"] for case in cases), "cases": cases, } for suite_name, cases in tests.items() } return { **dummy_meta_meta(), # type: ignore[misc] "total_seconds": sum(s["total_seconds"] for s in suites.values()), "suites": suites, } def make_case_v2(seconds: float, status: Status = None) -> Version2Case: return { "seconds": seconds, "status": status, } def make_report_v2( tests: Dict[str, Dict[str, Dict[str, Version2Case]]] ) -> Version2Report: files = {} for file_name, file_suites in tests.items(): suites = { suite_name: { "total_seconds": sum(case["seconds"] for case in cases.values()), "cases": cases, } for suite_name, cases in file_suites.items() } files[file_name] = { "suites": suites, "total_seconds": sum(suite["total_seconds"] for suite in suites.values()), # type: ignore[type-var] } return { **dummy_meta_meta(), # type: ignore[misc] "format_version": 2, "total_seconds": sum(s["total_seconds"] for s in files.values()), "files": files, } maxDiff = None class TestPrintTestStats(unittest.TestCase): version1_report: Version1Report = make_report_v1( { # input ordering of the suites is ignored "Grault": [ # not printed: status same and time similar makecase("test_grault0", 4.78, failed=True), # status same, but time increased a lot makecase("test_grault2", 1.473, errored=True), ], # individual tests times changed, not overall suite "Qux": [ # input ordering of the test cases is ignored makecase("test_qux1", 0.001, skipped=True), makecase("test_qux6", 0.002, skipped=True), # time in bounds, but status changed makecase("test_qux4", 7.158, failed=True), # not printed because it's the same as before makecase("test_qux7", 0.003, skipped=True), makecase("test_qux5", 11.968), makecase("test_qux3", 23.496), ], # new test suite "Bar": [ makecase("test_bar2", 3.742, failed=True), makecase("test_bar1", 50.447), ], # overall suite time changed but no individual tests "Norf": [ makecase("test_norf1", 3), makecase("test_norf2", 3), makecase("test_norf3", 3), makecase("test_norf4", 3), ], # suite doesn't show up if it doesn't change enough "Foo": [ makecase("test_foo1", 42), makecase("test_foo2", 56), ], } ) version2_report: Version2Report = make_report_v2( { "test_a": { "Grault": { "test_grault0": make_case_v2(4.78, "failed"), "test_grault2": make_case_v2(1.473, "errored"), }, "Qux": { "test_qux1": make_case_v2(0.001, "skipped"), "test_qux6": make_case_v2(0.002, "skipped"), "test_qux4": make_case_v2(7.158, "failed"), "test_qux7": make_case_v2(0.003, "skipped"), "test_qux8": make_case_v2(11.968), "test_qux3": make_case_v2(23.496), }, }, "test_b": { "Bar": { "test_bar2": make_case_v2(3.742, "failed"), "test_bar1": make_case_v2(50.447), }, # overall suite time changed but no individual tests "Norf": { "test_norf1": make_case_v2(3), "test_norf2": make_case_v2(3), "test_norf3": make_case_v2(3), "test_norf4": make_case_v2(3), }, }, "test_c": { "Foo": { "test_foo1": make_case_v2(42), "test_foo2": make_case_v2(56), }, }, } ) def test_simplify(self) -> None: self.assertEqual( { "": { "Bar": { "test_bar1": {"seconds": 50.447, "status": None}, "test_bar2": {"seconds": 3.742, "status": "failed"}, }, "Foo": { "test_foo1": {"seconds": 42, "status": None}, "test_foo2": {"seconds": 56, "status": None}, }, "Grault": { "test_grault0": {"seconds": 4.78, "status": "failed"}, "test_grault2": {"seconds": 1.473, "status": "errored"}, }, "Norf": { "test_norf1": {"seconds": 3, "status": None}, "test_norf3": {"seconds": 3, "status": None}, "test_norf2": {"seconds": 3, "status": None}, "test_norf4": {"seconds": 3, "status": None}, }, "Qux": { "test_qux1": {"seconds": 0.001, "status": "skipped"}, "test_qux3": {"seconds": 23.496, "status": None}, "test_qux4": {"seconds": 7.158, "status": "failed"}, "test_qux5": {"seconds": 11.968, "status": None}, "test_qux6": {"seconds": 0.002, "status": "skipped"}, "test_qux7": {"seconds": 0.003, "status": "skipped"}, }, }, }, print_test_stats.simplify(self.version1_report), ) self.assertEqual( { "test_a": { "Grault": { "test_grault0": {"seconds": 4.78, "status": "failed"}, "test_grault2": {"seconds": 1.473, "status": "errored"}, }, "Qux": { "test_qux1": {"seconds": 0.001, "status": "skipped"}, "test_qux3": {"seconds": 23.496, "status": None}, "test_qux4": {"seconds": 7.158, "status": "failed"}, "test_qux6": {"seconds": 0.002, "status": "skipped"}, "test_qux7": {"seconds": 0.003, "status": "skipped"}, "test_qux8": {"seconds": 11.968, "status": None}, }, }, "test_b": { "Bar": { "test_bar1": {"seconds": 50.447, "status": None}, "test_bar2": {"seconds": 3.742, "status": "failed"}, }, "Norf": { "test_norf1": {"seconds": 3, "status": None}, "test_norf2": {"seconds": 3, "status": None}, "test_norf3": {"seconds": 3, "status": None}, "test_norf4": {"seconds": 3, "status": None}, }, }, "test_c": { "Foo": { "test_foo1": {"seconds": 42, "status": None}, "test_foo2": {"seconds": 56, "status": None}, }, }, }, print_test_stats.simplify(self.version2_report), ) def test_analysis(self) -> None: head_report = self.version1_report base_reports: Dict[Commit, List[Report]] = { # bbbb has no reports, so base is cccc instead fakehash("b"): [], fakehash("c"): [ make_report_v1( { "Baz": [ makecase("test_baz2", 13.605), # no recent suites have & skip this test makecase("test_baz1", 0.004, skipped=True), ], "Foo": [ makecase("test_foo1", 43), # test added since dddd makecase("test_foo2", 57), ], "Grault": [ makecase("test_grault0", 4.88, failed=True), makecase("test_grault1", 11.967, failed=True), makecase("test_grault2", 0.395, errored=True), makecase("test_grault3", 30.460), ], "Norf": [ makecase("test_norf1", 2), makecase("test_norf2", 2), makecase("test_norf3", 2), makecase("test_norf4", 2), ], "Qux": [ makecase("test_qux3", 4.978, errored=True), makecase("test_qux7", 0.002, skipped=True), makecase("test_qux2", 5.618), makecase("test_qux4", 7.766, errored=True), makecase("test_qux6", 23.589, failed=True), ], } ), ], fakehash("d"): [ make_report_v1( { "Foo": [ makecase("test_foo1", 40), # removed in cccc makecase("test_foo3", 17), ], "Baz": [ # not skipped, so not included in stdev makecase("test_baz1", 3.14), ], "Qux": [ makecase("test_qux7", 0.004, skipped=True), makecase("test_qux2", 6.02), makecase("test_qux4", 20.932), ], "Norf": [ makecase("test_norf1", 3), makecase("test_norf2", 3), makecase("test_norf3", 3), makecase("test_norf4", 3), ], "Grault": [ makecase("test_grault0", 5, failed=True), makecase("test_grault1", 14.325, failed=True), makecase("test_grault2", 0.31, errored=True), ], } ), ], fakehash("e"): [], fakehash("f"): [ make_report_v1( { "Foo": [ makecase("test_foo3", 24), makecase("test_foo1", 43), ], "Baz": [ makecase("test_baz2", 16.857), ], "Qux": [ makecase("test_qux2", 6.422), makecase("test_qux4", 6.382, errored=True), ], "Norf": [ makecase("test_norf1", 0.9), makecase("test_norf3", 0.9), makecase("test_norf2", 0.9), makecase("test_norf4", 0.9), ], "Grault": [ makecase("test_grault0", 4.7, failed=True), makecase("test_grault1", 13.146, failed=True), makecase("test_grault2", 0.48, errored=True), ], } ), ], } simpler_head = print_test_stats.simplify(head_report) simpler_base = {} for commit, reports in base_reports.items(): simpler_base[commit] = [print_test_stats.simplify(r) for r in reports] analysis = print_test_stats.analyze( head_report=simpler_head, base_reports=simpler_base, ) self.assertEqual( """\ - class Baz: - # was 15.23s ± 2.30s - - def test_baz1: ... - # was 0.004s (skipped) - - def test_baz2: ... - # was 15.231s ± 2.300s class Grault: # was 48.86s ± 1.19s # now 6.25s - def test_grault1: ... - # was 13.146s ± 1.179s (failed) - def test_grault3: ... - # was 30.460s class Qux: # was 41.66s ± 1.06s # now 42.63s - def test_qux2: ... - # was 6.020s ± 0.402s ! def test_qux3: ... ! # was 4.978s (errored) ! # now 23.496s ! def test_qux4: ... ! # was 7.074s ± 0.979s (errored) ! # now 7.158s (failed) ! def test_qux6: ... ! # was 23.589s (failed) ! # now 0.002s (skipped) + def test_qux1: ... + # now 0.001s (skipped) + def test_qux5: ... + # now 11.968s + class Bar: + # now 54.19s + + def test_bar1: ... + # now 50.447s + + def test_bar2: ... + # now 3.742s (failed) """, print_test_stats.anomalies(analysis), ) def test_graph(self) -> None: # HEAD is on master self.assertEqual( """\ Commit graph (base is most recent master ancestor with at least one S3 report): : (master) | * aaaaaaaaaa (HEAD) total time 502.99s * bbbbbbbbbb (base) 1 report, total time 47.84s * cccccccccc 1 report, total time 332.50s * dddddddddd 0 reports | : """, print_test_stats.graph( head_sha=fakehash("a"), head_seconds=502.99, base_seconds={ fakehash("b"): [47.84], fakehash("c"): [332.50], fakehash("d"): [], }, on_master=True, ), ) self.assertEqual( """\ Commit graph (base is most recent master ancestor with at least one S3 report): : (master) | | * aaaaaaaaaa (HEAD) total time 9988.77s |/ * bbbbbbbbbb (base) 121 reports, total time 7654.32s ± 55.55s * cccccccccc 20 reports, total time 5555.55s ± 253.19s * dddddddddd 1 report, total time 1234.56s | : """, print_test_stats.graph( head_sha=fakehash("a"), head_seconds=9988.77, base_seconds={ fakehash("b"): [7598.77] * 60 + [7654.32] + [7709.87] * 60, fakehash("c"): [5308.77] * 10 + [5802.33] * 10, fakehash("d"): [1234.56], }, on_master=False, ), ) self.assertEqual( """\ Commit graph (base is most recent master ancestor with at least one S3 report): : (master) | | * aaaaaaaaaa (HEAD) total time 25.52s | | | : (5 commits) |/ * bbbbbbbbbb 0 reports * cccccccccc 0 reports * dddddddddd (base) 15 reports, total time 58.92s ± 25.82s | : """, print_test_stats.graph( head_sha=fakehash("a"), head_seconds=25.52, base_seconds={ fakehash("b"): [], fakehash("c"): [], fakehash("d"): [52.25] * 14 + [152.26], }, on_master=False, ancestry_path=5, ), ) self.assertEqual( """\ Commit graph (base is most recent master ancestor with at least one S3 report): : (master) | | * aaaaaaaaaa (HEAD) total time 0.08s |/| | : (1 commit) | * bbbbbbbbbb 0 reports * cccccccccc (base) 1 report, total time 0.09s * dddddddddd 3 reports, total time 0.10s ± 0.05s | : """, print_test_stats.graph( head_sha=fakehash("a"), head_seconds=0.08, base_seconds={ fakehash("b"): [], fakehash("c"): [0.09], fakehash("d"): [0.05, 0.10, 0.15], }, on_master=False, other_ancestors=1, ), ) self.assertEqual( """\ Commit graph (base is most recent master ancestor with at least one S3 report): : (master) | | * aaaaaaaaaa (HEAD) total time 5.98s | | | : (1 commit) |/| | : (7 commits) | * bbbbbbbbbb (base) 2 reports, total time 6.02s ± 1.71s * cccccccccc 0 reports * dddddddddd 10 reports, total time 5.84s ± 0.92s | : """, print_test_stats.graph( head_sha=fakehash("a"), head_seconds=5.98, base_seconds={ fakehash("b"): [4.81, 7.23], fakehash("c"): [], fakehash("d"): [4.97] * 5 + [6.71] * 5, }, on_master=False, ancestry_path=1, other_ancestors=7, ), ) def test_regression_info(self) -> None: self.assertEqual( """\ ----- Historic stats comparison result ------ job: foo_job commit: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa Commit graph (base is most recent master ancestor with at least one S3 report): : (master) | | * aaaaaaaaaa (HEAD) total time 3.02s |/ * bbbbbbbbbb (base) 1 report, total time 41.00s * cccccccccc 1 report, total time 43.00s | : Removed (across 1 suite) 1 test, totaling - 1.00s Modified (across 1 suite) 1 test, totaling - 41.48s ± 2.12s Added (across 1 suite) 1 test, totaling + 3.00s """, print_test_stats.regression_info( head_sha=fakehash("a"), head_report=make_report_v1( { "Foo": [ makecase("test_foo", 0.02, skipped=True), makecase("test_baz", 3), ] } ), base_reports={ fakehash("b"): [ make_report_v1( { "Foo": [ makecase("test_foo", 40), makecase("test_bar", 1), ], } ), ], fakehash("c"): [ make_report_v1( { "Foo": [ makecase("test_foo", 43), ], } ), ], }, job_name="foo_job", on_master=False, ancestry_path=0, other_ancestors=0, ), ) def test_regression_info_new_job(self) -> None: self.assertEqual( """\ ----- Historic stats comparison result ------ job: foo_job commit: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa Commit graph (base is most recent master ancestor with at least one S3 report): : (master) | | * aaaaaaaaaa (HEAD) total time 3.02s | | | : (3 commits) |/| | : (2 commits) | * bbbbbbbbbb 0 reports * cccccccccc 0 reports | : Removed (across 0 suites) 0 tests, totaling 0.00s Modified (across 0 suites) 0 tests, totaling 0.00s Added (across 1 suite) 2 tests, totaling + 3.02s """, print_test_stats.regression_info( head_sha=fakehash("a"), head_report=make_report_v1( { "Foo": [ makecase("test_foo", 0.02, skipped=True), makecase("test_baz", 3), ] } ), base_reports={ fakehash("b"): [], fakehash("c"): [], }, job_name="foo_job", on_master=False, ancestry_path=3, other_ancestors=2, ), ) if __name__ == "__main__": unittest.main()