#!/usr/bin/env python3 # SPDX-License-Identifier: Apache-2.0 # ----------------------------------------------------------------------------- # Copyright 2020-2021 Arm Limited # # Licensed under the Apache License, Version 2.0 (the "License"); you may not # use this file except in compliance with the License. You may obtain a copy # of the License at: # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. # ----------------------------------------------------------------------------- """ The ``astc_test_result_report.py`` script consolidates all current sets of reference results into a single report giving PSNR diffs (absolute) and performance diffs (relative speedup, 1 = no change). """ import re import os import sys import testlib.resultset as trs from collections import defaultdict as ddict CONFIG_FILTER = [ re.compile(r"^.*1\.7.*$"), re.compile(r"^.*sse.*$") ] TESTSET_FILTER = [ re.compile(r"^Small$"), re.compile(r"^Frymire$"), ] QUALITY_FILTER = [ ] BLOCKSIZE_FILTER = [ re.compile(r"^12x12$") ] def find_reference_results(): """ Scrape the Test/Images directory for result CSV files and return an mapping of the result sets. Returns: Returns a three deep tree of dictionaries, with the final dict pointing at a `ResultSet` object. The hierarchy is: imageSet => quality => encoder => result """ scriptDir = os.path.dirname(__file__) imageDir = os.path.join(scriptDir, "Images") # Pattern for extracting useful data from the CSV file name filePat = re.compile(r"astc_reference-(.+)_(.+)_results\.csv") # Build a three level dictionary we can write into results = ddict(lambda: ddict(lambda: ddict())) # Final all CSVs, load them and store them in the dict tree for root, dirs, files in os.walk(imageDir): for name in files: match = filePat.match(name) if match: # Skip results set in the filter skip = [1 for filt in CONFIG_FILTER if filt.match(name)] if skip: continue fullPath = os.path.join(root, name) encoder = match.group(1) quality = match.group(2) imageSet = os.path.basename(root) # Skip results set in the filter skip = [1 for filt in TESTSET_FILTER if filt.match(imageSet)] if skip: continue # Skip results set in the filter skip = [1 for filt in QUALITY_FILTER if filt.match(quality)] if skip: continue testRef = trs.ResultSet(imageSet) testRef.load_from_file(fullPath) patchedRef = trs.ResultSet(imageSet) for result in testRef.records: skip = [1 for filt in BLOCKSIZE_FILTER if filt.match(result.blkSz)] if not skip: patchedRef.add_record(result) results[imageSet][quality]["ref-%s" % encoder] = patchedRef return results class DeltaRecord(): """ Record a single image result from N different encoders. Attributes: imageSet: The image set this cme from. quality: The compressor quality used. encoders: The names of the encoders used. The first encoder in this list will be used as the reference result. records: The raw records for the encoders. The order of records in this list matches the order of the `encoders` list. """ def __init__(self, imageSet, quality, encoders, records): self.imageSet = imageSet self.quality = quality self.encoders = list(encoders) self.records = list(records) assert(len(self.encoders) == len(self.records)) def get_delta_header(self, tag): """ Get the delta encoding header. Args: tag: The field name to include in the tag. Return: The array of strings, providing the header names. """ result = [] for encoder in self.encoders[1:]: result.append("%s %s" % (tag, encoder)) return result def get_abs_delta(self, field): """ Get an absolute delta result. Args: field: The Record attribute name to diff. Return: The array of float delta values. """ result = [] root = self.records[0] for record in self.records[1:]: result.append(getattr(record, field) - getattr(root, field)) return result def get_rel_delta(self, field): """ Get an relative delta result (score / ref). Args: field: The Record attribute name to diff. Return: The array of float delta values. """ result = [] root = self.records[0] for record in self.records[1:]: result.append(getattr(record, field) / getattr(root, field)) return result def get_irel_delta(self, field): """ Get an inverse relative delta result (ref / score). Args: field: The Record attribute name to diff. Return: The array of float delta values. """ return [1.0 / x for x in self.get_rel_delta(field)] def get_full_row_header_csv(self): """ Get a CSV encoded delta record header. Return: The string for the row. """ rows = [ "Image Set", "Quality", "Size", "Name" ] rows.append("") rows.extend(self.get_delta_header("PSNR")) rows.append("") rows.extend(self.get_delta_header("Speed")) return ",".join(rows) def get_full_row_csv(self): """ Get a CSV encoded delta record. Return: The string for the row. """ rows = [ self.imageSet, self.quality, self.records[0].name, self.records[0].blkSz ] rows.append("") data = ["%0.3f" % x for x in self.get_abs_delta("psnr")] rows.extend(data) rows.append("") data = ["%0.3f" % x for x in self.get_irel_delta("cTime")] rows.extend(data) return ",".join(rows) def print_result_set(imageSet, quality, encoders, results, printHeader): """ Attributes: imageSet: The image set name. quality: The compressor quality used. encoders: The names of the encoders used. The first encoder in this list will be used as the reference result. results: The dict of results, indexed by encoder. printHeader: True if the table header should be printed, else False. """ results = [results[x] for x in encoders] recordSizes = [len(x.records) for x in results] # Skip result sets that are not the same size # TODO: We can take the set intersection here to report what we can if min(recordSizes) != max(recordSizes): return # Interleave all result records recordSets = zip(*[x.records for x in results]) # Iterate each image for recordSet in recordSets: base = recordSet[0] # Sanity check consistency for record in recordSet[1:]: assert(record.blkSz == base.blkSz) assert(record.name == base.name) dr = DeltaRecord(imageSet, quality, encoders, recordSet) if printHeader: print(dr.get_full_row_header_csv()) printHeader = False print(dr.get_full_row_csv()) def main(): """ The main function. Returns: int: The process return code. """ results = find_reference_results() imageSet = sorted(results.keys()) first = True for image in imageSet: qualityTree = results[image] qualitySet = sorted(qualityTree.keys()) for qual in qualitySet: encoderTree = qualityTree[qual] encoderSet = sorted(encoderTree.keys()) if len(encoderSet) > 1: print_result_set(image, qual, encoderSet, encoderTree, first) first = False return 0 if __name__ == "__main__": sys.exit(main())