1#! /usr/bin/env python3 2# 3# Copyright 2023 The ANGLE Project Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6# 7''' 8compare_trace_screenshots.py 9 10This script will cycle through screenshots from traces and compare them in useful ways. 11 12It can run in multiple ways. 13 14* `versus_native` 15 16 This mode expects to be run in a directory full of two sets of screenshots 17 18 angle_trace_tests --run-to-key-frame --screenshot-dir /tmp/screenshots 19 angle_trace_tests --run-to-key-frame --screenshot-dir /tmp/screenshots --use-gl=native 20 python3 compare_trace_screenshots.py versus_native --screenshot-dir /tmp/screenshots --trace-list-path ~/angle/src/tests/restricted_traces/ 21 22* `versus_upgrade` 23 24 This mode expects to be pointed to two directories of identical images (same names and pixel contents) 25 26 python3 compare_trace_screenshots.py versus_upgrade --before /my/trace/before --after /my/trace/after --out /my/trace/compare 27 28Prerequisites 29sudo apt-get install imagemagick 30''' 31 32import argparse 33import json 34import logging 35import os 36import subprocess 37import sys 38 39DEFAULT_LOG_LEVEL = 'info' 40 41EXIT_SUCCESS = 0 42EXIT_FAILURE = 1 43 44 45def versus_native(args): 46 47 # Get a list of all PNG files in the directory 48 png_files = os.listdir(args.screenshot_dir) 49 50 # Build a set of unique trace names 51 traces = set() 52 53 def get_traces_from_images(): 54 # Iterate through the PNG files 55 for png_file in sorted(png_files): 56 if png_file.startswith("angle_native") or png_file.startswith("angle_vulkan"): 57 # Strip the prefix and the PNG extension from the file name 58 trace_name = png_file.replace("angle_vulkan_", 59 "").replace("swiftshader_", 60 "").replace("angle_native_", 61 "").replace(".png", "") 62 traces.add(trace_name) 63 64 def get_traces_from_file(restricted_traces_path): 65 with open(os.path.join(restricted_traces_path, "restricted_traces.json")) as f: 66 trace_data = json.load(f) 67 68 # Have to split the 'trace version' thing up 69 trace_and_version = trace_data['traces'] 70 for i in trace_and_version: 71 traces.add(i.split(' ',)[0]) 72 73 def get_trace_key_frame(restricted_traces_path, trace): 74 with open(os.path.join(restricted_traces_path, trace, trace + ".json")) as f: 75 single_trace_data = json.load(f) 76 77 metadata = single_trace_data['TraceMetadata'] 78 keyframe = "" 79 if 'KeyFrames' in metadata: 80 keyframe = metadata['KeyFrames'][0] 81 return keyframe 82 83 if args.trace_list_path != None: 84 get_traces_from_file(args.trace_list_path) 85 else: 86 get_traces_from_images() 87 88 for trace in sorted(traces): 89 if args.trace_list_path != None: 90 keyframe = get_trace_key_frame(args.trace_list_path, trace) 91 frame = "" 92 if keyframe != "": 93 frame = "_frame" + str(keyframe) 94 95 native_file = "angle_native_" + trace + frame + ".png" 96 native_file = os.path.join(args.screenshot_dir, native_file) 97 if not os.path.isfile(native_file): 98 native_file = "MISSING_EXT.png" 99 100 vulkan_file = "angle_vulkan_" + trace + frame + ".png" 101 vulkan_file = os.path.join(args.screenshot_dir, vulkan_file) 102 if not os.path.isfile(vulkan_file): 103 vulkan_file = "angle_vulkan_swiftshader_" + trace + frame + ".png" 104 vulkan_file = os.path.join(args.screenshot_dir, vulkan_file) 105 if not os.path.isfile(vulkan_file): 106 vulkan_file = "MISSING_EXT.png" 107 108 # Compare each of the images with different fuzz factors so we can see how each is doing 109 # `compare -metric AE -fuzz ${FUZZ} ${VULKAN} ${NATIVE} ${TRACE}_fuzz${FUZZ}_diff.png` 110 results = [] 111 for fuzz in {0, 1, 2, 5, 10, 20}: 112 diff_file = trace + "_fuzz" + str(fuzz) + "%_TEST_diff.png" 113 diff_file = os.path.join(args.screenshot_dir, diff_file) 114 command = "compare -metric AE -fuzz " + str( 115 fuzz) + "% " + vulkan_file + " " + native_file + " " + diff_file 116 logging.debug("Running " + command) 117 diff = subprocess.run(command, shell=True, capture_output=True) 118 for line in diff.stderr.splitlines(): 119 if "unable to open image".encode('UTF-8') in line: 120 results.append("NA".encode('UTF-8')) 121 else: 122 results.append(diff.stderr) 123 logging.debug(" for " + trace + " " + str(fuzz) + "%") 124 125 print(trace, os.path.basename(vulkan_file), os.path.basename(native_file), 126 results[0].decode('UTF-8'), results[1].decode('UTF-8'), results[2].decode('UTF-8'), 127 results[3].decode('UTF-8'), results[4].decode('UTF-8'), results[5].decode('UTF-8')) 128 129 130def versus_upgrade(args): 131 132 # Get a list of all the files in before 133 before_files = sorted(os.listdir(args.before)) 134 135 # Get a list of all the files in after 136 after_files = sorted(os.listdir(args.after)) 137 138 # If either list is missing files, this is a fail! 139 if before_files != after_files: 140 before_minus_after = list(sorted(set(before_files) - set(after_files))) 141 after_minus_before = list(sorted(set(after_files) - set(before_files))) 142 print("File lists don't match!") 143 if before_minus_after is not []: 144 print("Extra before files: %s" % before_minus_after) 145 if after_minus_before is not []: 146 print("Extra after files: %s" % after_minus_before) 147 exit(1) 148 149 # Walk through the before list and compare it with after 150 for before_image, after_image in zip(sorted(before_files), sorted(after_files)): 151 152 # Compare each of the images using root mean squared, no fuzz factor 153 # `compare -metric RMSE ${BEFORE} ${AFTER} ${TRACE}_RMSE_diff.png;` 154 155 results = [] 156 diff_file = args.outdir + "/" + before_image + "_TEST_diff.png" 157 command = "compare -metric RMSE " + os.path.join( 158 args.before, before_image) + " " + os.path.join(args.after, 159 after_image) + " " + diff_file 160 diff = subprocess.run(command, shell=True, capture_output=True) 161 for line in diff.stderr.splitlines(): 162 if "unable to open image".encode('UTF-8') in line: 163 results.append("NA".encode('UTF-8')) 164 else: 165 # If the last element of the diff isn't zero, there was a pixel diff 166 if line.split()[-1] != b'(0)': 167 print(before_image, diff.stderr.decode('UTF-8')) 168 print("Pixel diff detected!") 169 exit(1) 170 else: 171 results.append(diff.stderr) 172 173 print(before_image, results[0].decode('UTF-8')) 174 175 print("Test completed successfully, no diffs detected") 176 177 178def main(): 179 parser = argparse.ArgumentParser() 180 parser.add_argument('-l', '--log', help='Logging level.', default=DEFAULT_LOG_LEVEL) 181 182 # Create commands for different modes of using this script 183 subparsers = parser.add_subparsers(dest='command', required=True, help='Command to run.') 184 185 # This mode will compare images of two runs, vulkan vs. native, and give you fuzzy comparison results 186 versus_native_parser = subparsers.add_parser( 187 'versus_native', help='Compares vulkan vs. native images.') 188 versus_native_parser.add_argument( 189 '--screenshot-dir', help='Directory containing two sets of screenshots', required=True) 190 versus_native_parser.add_argument( 191 '--trace-list-path', help='Path to dir containing restricted_traces.json') 192 193 # This mode will compare before and after images when upgrading a trace 194 versus_upgrade_parser = subparsers.add_parser( 195 'versus_upgrade', help='Compare images before and after an upgrade') 196 versus_upgrade_parser.add_argument( 197 '--before', help='Full path to dir containing *before* screenshots', required=True) 198 versus_upgrade_parser.add_argument( 199 '--after', help='Full path to dir containing *after* screenshots', required=True) 200 versus_upgrade_parser.add_argument('--outdir', help='Where to write output files', default='.') 201 202 args = parser.parse_args() 203 204 try: 205 if args.command == 'versus_native': 206 return versus_native(args) 207 elif args.command == 'versus_upgrade': 208 return versus_upgrade(args) 209 else: 210 logging.fatal('Unknown command: %s' % args.command) 211 return EXIT_FAILURE 212 except subprocess.CalledProcessError as e: 213 logging.exception('There was an exception: %s', e.output.decode()) 214 return EXIT_FAILURE 215 216 217if __name__ == '__main__': 218 sys.exit(main()) 219