# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0
# For details: https://github.com/nedbat/coveragepy/blob/master/NOTICE.txt
"""Tests of the helpers in goldtest.py"""
from __future__ import annotations
import os.path
import re
import pytest
from tests.coveragetest import CoverageTest, TESTS_DIR
from tests.goldtest import compare, gold_path
from tests.goldtest import contains, contains_any, contains_rx, doesnt_contain
from tests.helpers import os_sep, re_line, remove_tree
GOOD_GETTY = """\
Four score and seven years ago our fathers brought forth upon this continent, a
new nation, conceived in Liberty, and dedicated to the proposition that all men
are created equal.
11/19/9999, Gettysburg, Pennsylvania
"""
BAD_GETTY = """\
Five score and seven years ago our fathers brought forth upon this continent, a
new nation, conceived in Liberty, and dedicated to the proposition that all men
are created equal.
333/4444/55555, Gabcdef, Pennsylvania
"""
SCRUBS = [
# Numbers don't matter when comparing.
(r'\d+', 'D'),
(r'G\w+', 'Gxxx'),
]
def path_regex(path: str) -> str:
"""Convert a file path into a regex that will match that path on any OS."""
return re.sub(r"[/\\]", r"[/\\\\]", path.replace(".", "[.]"))
ACTUAL_DIR = os.path.join(TESTS_DIR, "actual/testing")
ACTUAL_GETTY_FILE = os.path.join(ACTUAL_DIR, "getty/gettysburg.txt")
GOLD_GETTY_FILE = os.path.join(TESTS_DIR, "gold/testing/getty/gettysburg.txt")
GOLD_GETTY_FILE_RX = path_regex(GOLD_GETTY_FILE)
GOLD_PATH_RX = path_regex("/tests/gold/testing/getty/gettysburg.txt")
OUT_PATH_RX = path_regex("out/gettysburg.txt")
class CompareTest(CoverageTest):
"""Tests of goldtest.py:compare()"""
def setUp(self) -> None:
super().setUp()
self.addCleanup(remove_tree, ACTUAL_DIR)
def test_good(self) -> None:
self.make_file("out/gettysburg.txt", GOOD_GETTY)
compare(gold_path("testing/getty"), "out", scrubs=SCRUBS)
self.assert_doesnt_exist(ACTUAL_GETTY_FILE)
def test_bad(self) -> None:
self.make_file("out/gettysburg.txt", BAD_GETTY)
# compare() raises an assertion.
msg = fr"Files differ: .*{GOLD_PATH_RX} != {OUT_PATH_RX}"
with pytest.raises(AssertionError, match=msg):
compare(gold_path("testing/getty"), "out", scrubs=SCRUBS)
# Stdout has a description of the diff. The diff shows the scrubbed content.
stdout = self.stdout()
assert "- Four score" in stdout
assert "+ Five score" in stdout
assert re_line(fr"^:::: diff '.*{GOLD_PATH_RX}' and '{OUT_PATH_RX}'", stdout)
assert re_line(fr"^:::: end diff '.*{GOLD_PATH_RX}' and '{OUT_PATH_RX}'", stdout)
assert (
os_sep(f"Saved actual output to '{ACTUAL_GETTY_FILE}': see tests/gold/README.rst")
in os_sep(stdout)
)
assert " D/D/D, Gxxx, Pennsylvania" in stdout
# The actual file was saved.
with open(ACTUAL_GETTY_FILE) as f:
saved = f.read()
assert saved == BAD_GETTY
def test_good_needs_scrubs(self) -> None:
# Comparing the "good" result without scrubbing the variable parts will fail.
self.make_file("out/gettysburg.txt", GOOD_GETTY)
# compare() raises an assertion.
msg = fr"Files differ: .*{GOLD_PATH_RX} != {OUT_PATH_RX}"
with pytest.raises(AssertionError, match=msg):
compare(gold_path("testing/getty"), "out")
stdout = self.stdout()
assert "- 11/19/1863, Gettysburg, Pennsylvania" in stdout
assert "+ 11/19/9999, Gettysburg, Pennsylvania" in stdout
def test_actual_extra(self) -> None:
self.make_file("out/gettysburg.txt", GOOD_GETTY)
self.make_file("out/another.more", "hi")
# Extra files in the output are ok with actual_extra=True.
compare(gold_path("testing/getty"), "out", scrubs=SCRUBS, actual_extra=True)
# But not without it:
# (test output is in files like /tmp/pytest-of-user/pytest-0/popen-gw3/t76/out)
msg = r"Files in .*[/\\]t\d+[/\\]out only: \['another.more'\]"
with pytest.raises(AssertionError, match=msg):
compare(gold_path("testing/getty"), "out", scrubs=SCRUBS)
self.assert_exists(os.path.join(TESTS_DIR, "actual/testing/getty/another.more"))
# But only the files matching the file_pattern are considered.
compare(gold_path("testing/getty"), "out", file_pattern="*.txt", scrubs=SCRUBS)
def test_xml_good(self) -> None:
self.make_file("out/output.xml", """\
Goodie
""")
compare(gold_path("testing/xml"), "out", scrubs=SCRUBS)
def test_xml_bad(self) -> None:
self.make_file("out/output.xml", """\
Goodbye
""")
# compare() raises an exception.
gold_rx = path_regex(gold_path("testing/xml/output.xml"))
out_rx = path_regex("out/output.xml")
msg = fr"Files differ: .*{gold_rx} != {out_rx}"
with pytest.raises(AssertionError, match=msg):
compare(gold_path("testing/xml"), "out", scrubs=SCRUBS)
# Stdout has a description of the diff. The diff shows the
# canonicalized and scrubbed content.
stdout = self.stdout()
assert '- ' in stdout
assert '+ ' in stdout
class ContainsTest(CoverageTest):
"""Tests of the various "contains" functions in goldtest.py"""
run_in_temp_dir = False
def test_contains(self) -> None:
contains(GOLD_GETTY_FILE, "Four", "fathers", "dedicated")
msg = fr"Missing content in {GOLD_GETTY_FILE_RX}: 'xyzzy'"
with pytest.raises(AssertionError, match=msg):
contains(GOLD_GETTY_FILE, "Four", "fathers", "xyzzy", "dedicated")
def test_contains_rx(self) -> None:
contains_rx(GOLD_GETTY_FILE, r"Fo.r", r"f[abc]thers", "dedi[cdef]ated")
msg = fr"Missing regex in {GOLD_GETTY_FILE_RX}: r'm\[opq\]thers'"
with pytest.raises(AssertionError, match=msg):
contains_rx(GOLD_GETTY_FILE, r"Fo.r", r"m[opq]thers")
def test_contains_any(self) -> None:
contains_any(GOLD_GETTY_FILE, "Five", "Four", "Three")
msg = fr"Missing content in {GOLD_GETTY_FILE_RX}: 'One' \[1 of 3\]"
with pytest.raises(AssertionError, match=msg):
contains_any(GOLD_GETTY_FILE, "One", "Two", "Three")
def test_doesnt_contain(self) -> None:
doesnt_contain(GOLD_GETTY_FILE, "One", "Two", "Three")
msg = fr"Forbidden content in {GOLD_GETTY_FILE_RX}: 'Four'"
with pytest.raises(AssertionError, match=msg):
doesnt_contain(GOLD_GETTY_FILE, "Three", "Four", "Five")