File: helpers.py

package info (click to toggle)
pytest-mpl 0.17.0-2
  • links: PTS, VCS
  • area: main
  • in suites: forky, sid
  • size: 2,132 kB
  • sloc: python: 2,514; javascript: 179; makefile: 16
file content (344 lines) | stat: -rw-r--r-- 11,940 bytes parent folder | download | duplicates (3)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
import os
import re
import json
from pathlib import Path

from PIL import Image, ImageDraw

__all__ = ['diff_summary', 'assert_existence', 'patch_summary', 'apply_regex',
           'remove_specific_hashes', 'transform_hashes', 'transform_images']


class MatchError(Exception):
    pass


def diff_summary(baseline, result, baseline_hash_library=None, result_hash_library=None,
                 generating_hashes=False):
    """Diff a pytest-mpl summary dictionary.

    Parameters
    ----------
    baseline : dict
        Baseline pytest-mpl summary.
    result : dict
        Generated result pytest-mpl summary.
    baseline_hash_library : Path, optional, default=None
        Path to the baseline hash library.
        Baseline hashes in the baseline summary are updated to these values
        to handle different Matplotlib versions.
    result_hash_library : Path, optional, default=None
        Path to the "baseline" image hash library.
        Result hashes in the baseline summary are updated to these values
        to handle different Matplotlib versions.
    generating_hashes : bool, optional, default=False
        Whether `--mpl-generate-hash-library` was specified and
        both of `--mpl-hash-library` and `hash_library=` were not.
    """
    if baseline_hash_library and baseline_hash_library.exists():
        # Load "correct" baseline hashes
        with open(baseline_hash_library, 'r') as f:
            baseline_hash_library = json.load(f)
    else:
        baseline_hash_library = {}
    if result_hash_library and result_hash_library.exists():
        # Load "correct" result hashes
        with open(result_hash_library, 'r') as f:
            result_hash_library = json.load(f)
    else:
        result_hash_library = {}

    # Get test names
    baseline_tests = set(baseline.keys())
    result_tests = set(result.keys())

    # Test names must be identical
    diff_set(baseline_tests, result_tests, error='Test names are not identical.')

    item_match_errors = []  # Raise a MatchError for all mismatched values at the end

    for test in baseline_tests:

        # Get baseline and result summary for the specific test
        baseline_summary = baseline[test]
        result_summary = result[test]

        # Swap the baseline and result hashes in the summary
        # for the corresponding hashes in each hash library
        if baseline_hash_library and test in baseline_hash_library and not generating_hashes:
            baseline_summary = replace_hash(baseline_summary, 'baseline_hash',
                                            baseline_hash_library[test])
        if result_hash_library:
            if generating_hashes:  # Newly generate result will appear as baseline_hash
                baseline_summary = replace_hash(baseline_summary, 'baseline_hash',
                                                result_hash_library[test])
            baseline_summary = replace_hash(baseline_summary, 'result_hash',
                                            result_hash_library[test])

        # Get keys of recorded items
        baseline_keys = set(baseline_summary.keys())
        result_keys = set(result_summary.keys())

        # Summaries must have the same keys
        diff_set(baseline_keys, result_keys, error=f'Summary for {test} is not identical.')

        for key in baseline_keys:
            error = f'Summary item {key} for {test} does not match.\n'
            try:
                diff_dict_item(baseline_summary[key], result_summary[key], error=error)
            except MatchError as e:
                item_match_errors.append(str(e))

    if len(item_match_errors) > 0:
        raise MatchError('\n\n----------\n\n'.join(item_match_errors))


def diff_set(baseline, result, error=''):
    """Raise and show the difference between Python sets."""
    if baseline != result:
        missing_from_result = baseline - result
        missing_from_baseline = result - baseline
        if len(missing_from_result) > 0:
            error += f'\nKeys {sorted(missing_from_result)} missing from the result.'
        if len(missing_from_baseline) > 0:
            error += f'\nKeys {sorted(missing_from_baseline)} missing from the baseline.'
        raise MatchError(error)


def diff_dict_item(baseline, result, error=''):
    """Diff a specific item in a pytest-mpl summary dictionary."""
    # Comparison makes the following (good) assumptions
    expected_types = (str, int, float, bool, type(None))
    assert isinstance(baseline, expected_types)
    assert isinstance(result, expected_types)

    # Prepare error message
    error += f'Baseline:\n"{baseline}"\n\n'
    error += f'Result:\n"{result}"\n'

    # Matching items must have the same type
    if type(baseline) is not type(result):
        raise MatchError(error + '\nTypes are not equal.\n')

    # Handle regex in baseline string (so things like paths can be ignored)
    if isinstance(baseline, str) and baseline.startswith('REGEX:'):
        if re.fullmatch(baseline[6:], result) is not None:
            return

    # Handle bool and NoneType
    if isinstance(baseline, (bool, type(None))) and baseline is result:
        return

    # Handle float
    if isinstance(baseline, float) and abs(baseline - result) < 1e-4:
        return

    # Handle str and int
    if baseline == result:
        return

    raise MatchError(error)


def patch_summary(summary, patch_file):
    """Replace in `summary` any items defined in `patch_file`."""
    # By only applying patches, changes between MPL versions are more obvious.
    with open(patch_file, 'r') as f:
        patch = json.load(f)
    for test, test_summary in patch.items():
        for k, v in test_summary.items():
            summary[test][k] = v
    return summary


def replace_hash(summary, hash_key, new_hash):
    """Replace a hash in a pytest-mpl summary with a different hash.

    Parameters
    ----------
    summary : dict
        A single test from a pytest-mpl summary.
    hash_key : str
        Key of the hash. Either `baseline_hash` or `result_hash`.
    new_hash : str
        The new hash.
    """
    assert isinstance(new_hash, str)
    old_hash = summary[hash_key]
    if not isinstance(old_hash, str) or old_hash == new_hash:
        return summary  # Either already correct or missing

    # Update the hash
    summary[hash_key] = new_hash
    summary['status_msg'] = summary['status_msg'].replace(old_hash, new_hash)

    return summary


def assert_existence(summary, items=('baseline_image', 'diff_image', 'result_image'), path=''):
    """Assert that images included in a pytest-mpl summary exist.

    Parameters
    ----------
    summary : dict
        The pytest-mpl summary dictionary to check.
    items : tuple or list, optional
        The image keys to check if reported.
    path : str or path_like, optional, default=''
        Path to results directory. Defaults to current directory.
    """
    for test in summary.values():
        for item in items:
            if test[item] is not None:
                assert (Path(path) / test[item]).exists()


def _escape_regex(msg):
    if not msg.startswith('REGEX:'):
        msg = msg.replace('.', r'\.').replace('(', r'\(').replace(')', r'\)')
        msg = 'REGEX:' + msg
    return msg


def _escape_path(msg, path):
    pattern = (rf"({path}[A-Za-z0-9_\-\/.\\]*)" +
               r"(baseline\\.png|result-failed-diff\\.png|result\\.png|\\.json)")
    msg = re.sub(pattern, r".*\2", msg)
    pattern = rf"({path}[A-Za-z0-9_\-\/.\\]*)"
    msg = re.sub(pattern, r".*", msg)
    return msg


def _escape_float(msg, key):
    pattern = rf"({key}[0-9]+\\\.[0-9]{{1}})([0-9]+)"
    msg = re.sub(pattern, r"\1[0-9]*", msg)
    return msg


def apply_regex(file, regex_paths, regex_strs):
    """Convert all `status_msg` entries in JSON summary file to regex.

    Use in your own script to assist with updating baseline summaries.

    Parameters
    ----------
    file : Path
        JSON summary file to convert `status_msg` to regex in. Overwritten.
    regex_paths : list of str
        List of path beginnings to identify paths that need to be converted to regex.
        E.g. `['/home/user/']`
        Does: `aaa /home/user/pytest/tmp/result\\.png bbb` -> `aaa .*result\\.png bbb`
    regex_strs : list of str
        List of keys to convert following floats to 1 d.p.
        E.g. ['RMS Value: ']
        Does: `aaa RMS Value: 12\\.432644 bbb` -> `aaa RMS Value: 12\\.4[0-9]* bbb`
    """

    with open(file, 'r') as f:
        summary = json.load(f)

    for test in summary.keys():

        msg = summary[test]['status_msg']

        for signal in [*regex_paths, *regex_strs]:
            if signal in msg:
                msg = _escape_regex(msg)
        if not msg.startswith('REGEX:'):
            continue

        for signal in regex_paths:
            if signal in msg:
                msg = _escape_path(msg, path=signal)

        for signal in regex_strs:
            if signal in msg:
                msg = _escape_float(msg, key=signal)

        summary[test]['status_msg'] = msg

    with open(file, 'w') as f:
        json.dump(summary, f, indent=2)


def remove_specific_hashes(summary_file):
    """Replace all hashes in a summary file with placeholder values.

    This is done because the actual hashes used for testing are taken from
    separate files for each specific matplotlib version.
    """

    baseline_placeholder = "###_BASELINE_HASH_###"
    result_placeholder = "###_RESULT_HASH_###"

    with open(summary_file, "r") as f:
        summary = json.load(f)

    for test in summary.keys():

        # Get actual hashes
        baseline = summary[test]["baseline_hash"]
        result = summary[test]["result_hash"]

        # Replace with placeholders (if summary has hashes)
        if baseline is not None:
            summary[test]["baseline_hash"] = baseline_placeholder
            summary[test]["status_msg"] = \
                summary[test]["status_msg"].replace(baseline, baseline_placeholder)
        if result is not None:
            summary[test]["result_hash"] = result_placeholder
            summary[test]["status_msg"] = \
                summary[test]["status_msg"].replace(result, result_placeholder)

    with open(summary_file, "w") as f:
        json.dump(summary, f, indent=2)


def transform_hashes(hash_file):
    """Make hash comparison tests fail correctly.

    Makes hashes of tests *hdiff* in hash_file fail hash comparison
    and remove *hmissing* hashes that should be missing.
    """

    with open(hash_file, "r") as f:
        hashes = json.load(f)

    for test in list(hashes.keys()):
        h = hashes[test]
        if "hdiff" in test and h is not None:
            # Replace first four letters with d1ff to force mismatch
            hashes[test] = "d1ff" + h[4:]
        if "hmissing" in test and h is not None:
            # Remove hashes that should be missing
            del hashes[test]

    with open(hash_file, "w") as f:
        json.dump(hashes, f, indent=2)


def transform_images(baseline_path):
    """Make image comparison tests fail correctly.

    Makes images of tests *idiff* under baseline_path fail image comparison
    and deletes images for *imissing* tests.
    """

    # Delete imissing files
    for file in baseline_path.glob("**/*imissing*.png"):
        file.unlink()

    # Add red cross to idiff files
    for file in baseline_path.glob("**/*idiff*.png"):
        with Image.open(file) as im:
            draw = ImageDraw.Draw(im)
            draw.line((0, 0) + im.size, "#f00", 3)
            draw.line((0, im.size[1], im.size[0], 0), "#f00", 3)
            im.save(file)

    # Resize idiffshape files
    for file in baseline_path.glob("**/*idiffshape*.png"):
        with Image.open(file) as im:
            (width, height) = (im.width // 2, im.height // 2)
            im_resized = im.resize((width, height))
            im_resized.save(file)