1#! /usr/bin/env vpython3 2# 3# Copyright 2020 The ANGLE Project Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6# 7# restricted_trace_gold_tests.py: 8# Uses Skia Gold (https://skia.org/dev/testing/skiagold) to run pixel tests with ANGLE traces. 9# 10# Requires vpython to run standalone. Run with --help for usage instructions. 11 12import argparse 13import contextlib 14import json 15import logging 16import os 17import pathlib 18import platform 19import re 20import shutil 21import sys 22import tempfile 23import time 24import traceback 25 26 27PY_UTILS = str(pathlib.Path(__file__).resolve().parents[1] / 'py_utils') 28if PY_UTILS not in sys.path: 29 os.stat(PY_UTILS) and sys.path.insert(0, PY_UTILS) 30import android_helper 31import angle_path_util 32import angle_test_util 33from skia_gold import angle_skia_gold_properties 34from skia_gold import angle_skia_gold_session_manager 35 36angle_path_util.AddDepsDirToPath('testing/scripts') 37import common 38 39 40DEFAULT_TEST_SUITE = angle_test_util.ANGLE_TRACE_TEST_SUITE 41DEFAULT_TEST_PREFIX = 'TraceTest.' 42DEFAULT_SCREENSHOT_PREFIX = 'angle_vulkan_' 43SWIFTSHADER_SCREENSHOT_PREFIX = 'angle_vulkan_swiftshader_' 44DEFAULT_BATCH_SIZE = 5 45DEFAULT_LOG = 'info' 46DEFAULT_GOLD_INSTANCE = 'angle' 47 48# Test expectations 49FAIL = 'FAIL' 50PASS = 'PASS' 51SKIP = 'SKIP' 52 53 54@contextlib.contextmanager 55def temporary_dir(prefix=''): 56 path = tempfile.mkdtemp(prefix=prefix) 57 try: 58 yield path 59 finally: 60 logging.info("Removing temporary directory: %s" % path) 61 shutil.rmtree(path) 62 63 64def add_skia_gold_args(parser): 65 group = parser.add_argument_group('Skia Gold Arguments') 66 group.add_argument('--git-revision', help='Revision being tested.', default=None) 67 group.add_argument( 68 '--gerrit-issue', help='For Skia Gold integration. Gerrit issue ID.', default='') 69 group.add_argument( 70 '--gerrit-patchset', 71 help='For Skia Gold integration. Gerrit patch set number.', 72 default='') 73 group.add_argument( 74 '--buildbucket-id', help='For Skia Gold integration. Buildbucket build ID.', default='') 75 group.add_argument( 76 '--bypass-skia-gold-functionality', 77 action='store_true', 78 default=False, 79 help='Bypass all interaction with Skia Gold, effectively disabling the ' 80 'image comparison portion of any tests that use Gold. Only meant to ' 81 'be used in case a Gold outage occurs and cannot be fixed quickly.') 82 local_group = group.add_mutually_exclusive_group() 83 local_group.add_argument( 84 '--local-pixel-tests', 85 action='store_true', 86 default=None, 87 help='Specifies to run the test harness in local run mode or not. When ' 88 'run in local mode, uploading to Gold is disabled and links to ' 89 'help with local debugging are output. Running in local mode also ' 90 'implies --no-luci-auth. If both this and --no-local-pixel-tests are ' 91 'left unset, the test harness will attempt to detect whether it is ' 92 'running on a workstation or not and set this option accordingly.') 93 local_group.add_argument( 94 '--no-local-pixel-tests', 95 action='store_false', 96 dest='local_pixel_tests', 97 help='Specifies to run the test harness in non-local (bot) mode. When ' 98 'run in this mode, data is actually uploaded to Gold and triage links ' 99 'arge generated. If both this and --local-pixel-tests are left unset, ' 100 'the test harness will attempt to detect whether it is running on a ' 101 'workstation or not and set this option accordingly.') 102 group.add_argument( 103 '--no-luci-auth', 104 action='store_true', 105 default=False, 106 help='Don\'t use the service account provided by LUCI for ' 107 'authentication for Skia Gold, instead relying on gsutil to be ' 108 'pre-authenticated. Meant for testing locally instead of on the bots.') 109 110 111def run_angle_system_info_test(sysinfo_args, args, env): 112 with temporary_dir() as temp_dir: 113 sysinfo_args += ['--render-test-output-dir=' + temp_dir] 114 115 result, _, _ = angle_test_util.RunTestSuite( 116 'angle_system_info_test', sysinfo_args, env, use_xvfb=args.xvfb) 117 if result != 0: 118 raise Exception('Error getting system info.') 119 120 with open(os.path.join(temp_dir, 'angle_system_info.json')) as f: 121 return json.load(f) 122 123 124def to_hex(num): 125 return hex(int(num)) 126 127 128def to_hex_or_none(num): 129 return 'None' if num == None else to_hex(num) 130 131 132def to_non_empty_string_or_none(val): 133 return 'None' if val == '' else str(val) 134 135 136def to_non_empty_string_or_none_dict(d, key): 137 return 'None' if not key in d else to_non_empty_string_or_none(d[key]) 138 139 140def get_skia_gold_keys(args, env): 141 """Get all the JSON metadata that will be passed to golctl.""" 142 # All values need to be strings, otherwise goldctl fails. 143 144 # Only call this method one time 145 if hasattr(get_skia_gold_keys, 'called') and get_skia_gold_keys.called: 146 logging.exception('get_skia_gold_keys may only be called once') 147 get_skia_gold_keys.called = True 148 149 sysinfo_args = ['--vulkan', '-v'] 150 if args.swiftshader: 151 sysinfo_args.append('--swiftshader') 152 153 if angle_test_util.IsAndroid(): 154 json_data = android_helper.AngleSystemInfo(sysinfo_args) 155 logging.info(json_data) 156 os_name = 'Android' 157 os_version = android_helper.GetBuildFingerprint() 158 else: 159 json_data = run_angle_system_info_test(sysinfo_args, args, env) 160 os_name = to_non_empty_string_or_none(platform.system()) 161 os_version = to_non_empty_string_or_none(platform.version()) 162 163 if len(json_data.get('gpus', [])) == 0 or not 'activeGPUIndex' in json_data: 164 raise Exception('Error getting system info.') 165 166 active_gpu = json_data['gpus'][json_data['activeGPUIndex']] 167 168 angle_keys = { 169 'vendor_id': to_hex_or_none(active_gpu['vendorId']), 170 'device_id': to_hex_or_none(active_gpu['deviceId']), 171 'model_name': to_non_empty_string_or_none_dict(active_gpu, 'machineModelVersion'), 172 'manufacturer_name': to_non_empty_string_or_none_dict(active_gpu, 'machineManufacturer'), 173 'os': os_name, 174 'os_version': os_version, 175 'driver_version': to_non_empty_string_or_none_dict(active_gpu, 'driverVersion'), 176 'driver_vendor': to_non_empty_string_or_none_dict(active_gpu, 'driverVendor'), 177 } 178 179 return angle_keys 180 181 182def output_diff_local_files(gold_session, image_name): 183 """Logs the local diff image files from the given SkiaGoldSession. 184 185 Args: 186 gold_session: A skia_gold_session.SkiaGoldSession instance to pull files 187 from. 188 image_name: A string containing the name of the image/test that was 189 compared. 190 """ 191 given_file = gold_session.GetGivenImageLink(image_name) 192 closest_file = gold_session.GetClosestImageLink(image_name) 193 diff_file = gold_session.GetDiffImageLink(image_name) 194 failure_message = 'Unable to retrieve link' 195 logging.error('Generated image: %s', given_file or failure_message) 196 logging.error('Closest image: %s', closest_file or failure_message) 197 logging.error('Diff image: %s', diff_file or failure_message) 198 199 200def get_trace_key_frame(trace): 201 # read trace info 202 json_name = os.path.join(angle_path_util.ANGLE_ROOT_DIR, 'src', 'tests', 'restricted_traces', 203 trace, trace + '.json') 204 with open(json_name) as fp: 205 trace_info = json.load(fp) 206 207 # Check its metadata for a keyframe 208 keyframe = '' 209 if 'KeyFrames' in trace_info['TraceMetadata']: 210 # KeyFrames is an array, but we only use the first value for now 211 keyframe = str(trace_info['TraceMetadata']['KeyFrames'][0]) 212 logging.info('trace %s is using a keyframe of %s' % (trace, keyframe)) 213 214 return keyframe 215 216 217def upload_test_result_to_skia_gold(args, gold_session_manager, gold_session, gold_properties, 218 screenshot_dir, trace, artifacts): 219 """Compares the given image using Skia Gold and uploads the result. 220 221 No uploading is done if the test is being run in local run mode. Compares 222 the given screenshot to baselines provided by Gold, raising an Exception if 223 a match is not found. 224 225 Args: 226 args: Command line options. 227 gold_session_manager: Skia Gold session manager. 228 gold_session: Skia Gold session. 229 gold_properties: Skia Gold properties. 230 screenshot_dir: directory where the test stores screenshots. 231 trace: base name of the trace being checked. 232 artifacts: dictionary of JSON artifacts to pass to the result merger. 233 """ 234 235 use_luci = not (gold_properties.local_pixel_tests or gold_properties.no_luci_auth) 236 237 # Determine if this trace is using a keyframe 238 image_name = trace 239 keyframe = get_trace_key_frame(trace) 240 if keyframe != '': 241 image_name = trace + '_frame' + keyframe 242 logging.debug('Using %s as image_name for upload' % image_name) 243 244 # Note: this would be better done by iterating the screenshot directory. 245 prefix = SWIFTSHADER_SCREENSHOT_PREFIX if args.swiftshader else DEFAULT_SCREENSHOT_PREFIX 246 png_file_name = os.path.join(screenshot_dir, prefix + image_name + '.png') 247 248 if not os.path.isfile(png_file_name): 249 raise Exception('Screenshot not found: ' + png_file_name) 250 251 if args.use_permissive_pixel_comparison: 252 # These arguments cause Gold to use the sample area inexact matching 253 # algorithm. It is set to allow any of up to 3 pixels in each 4x4 group 254 # of pixels to differ by any amount. Pixels that differ by a max of 1 255 # on all channels (i.e. have differences that can be attributed to 256 # rounding errors) do not count towards this limit. 257 # 258 # An image that passes due to this logic is auto-approved as a new good 259 # image. 260 inexact_matching_args = [ 261 '--add-test-optional-key', 262 'image_matching_algorithm:sample_area', 263 '--add-test-optional-key', 264 'sample_area_width:4', 265 '--add-test-optional-key', 266 'sample_area_max_different_pixels_per_area:3', 267 '--add-test-optional-key', 268 'sample_area_channel_delta_threshold:1', 269 ] 270 else: 271 # These arguments cause Gold to use the fuzzy inexact matching 272 # algorithm. It is set to allow up to 20k pixels to differ by 1 on all 273 # channels, which is meant to help reduce triage overhead caused by new 274 # images from rounding differences. 275 # 276 # The max number of pixels is fairly arbitrary, but the diff threshold 277 # is intentional since we don't want to let in any changes that can't be 278 # attributed to rounding errors. 279 # 280 # An image that passes due to this logic is auto-approved as a new good 281 # image. 282 inexact_matching_args = [ 283 '--add-test-optional-key', 284 'image_matching_algorithm:fuzzy', 285 '--add-test-optional-key', 286 'fuzzy_max_different_pixels:20000', 287 '--add-test-optional-key', 288 'fuzzy_pixel_per_channel_delta_threshold:1', 289 ] 290 291 status, error = gold_session.RunComparison( 292 name=image_name, 293 png_file=png_file_name, 294 use_luci=use_luci, 295 inexact_matching_args=inexact_matching_args) 296 297 artifact_name = os.path.basename(png_file_name) 298 artifacts[artifact_name] = [artifact_name] 299 300 if not status: 301 return PASS 302 303 status_codes = gold_session_manager.GetSessionClass().StatusCodes 304 if status == status_codes.AUTH_FAILURE: 305 logging.error('Gold authentication failed with output %s', error) 306 elif status == status_codes.INIT_FAILURE: 307 logging.error('Gold initialization failed with output %s', error) 308 elif status == status_codes.COMPARISON_FAILURE_REMOTE: 309 _, triage_link = gold_session.GetTriageLinks(image_name) 310 if not triage_link: 311 logging.error('Failed to get triage link for %s, raw output: %s', image_name, error) 312 logging.error('Reason for no triage link: %s', 313 gold_session.GetTriageLinkOmissionReason(image_name)) 314 if gold_properties.IsTryjobRun(): 315 # Pick "show all results" so we can see the tryjob images by default. 316 triage_link += '&master=true' 317 artifacts['triage_link_for_entire_cl'] = [triage_link] 318 else: 319 artifacts['gold_triage_link'] = [triage_link] 320 elif status == status_codes.COMPARISON_FAILURE_LOCAL: 321 logging.error('Local comparison failed. Local diff files:') 322 output_diff_local_files(gold_session, image_name) 323 elif status == status_codes.LOCAL_DIFF_FAILURE: 324 logging.error( 325 'Local comparison failed and an error occurred during diff ' 326 'generation: %s', error) 327 # There might be some files, so try outputting them. 328 logging.error('Local diff files:') 329 output_diff_local_files(gold_session, image_name) 330 else: 331 logging.error('Given unhandled SkiaGoldSession StatusCode %s with error %s', status, error) 332 333 return FAIL 334 335 336def _get_batches(traces, batch_size): 337 for i in range(0, len(traces), batch_size): 338 yield traces[i:i + batch_size] 339 340 341def _get_gtest_filter_for_batch(args, batch): 342 expanded = ['%s%s' % (DEFAULT_TEST_PREFIX, trace) for trace in batch] 343 return '--gtest_filter=%s' % ':'.join(expanded) 344 345 346def _run_tests(args, tests, extra_flags, env, screenshot_dir, results, test_results): 347 keys = get_skia_gold_keys(args, env) 348 349 with temporary_dir('angle_skia_gold_') as skia_gold_temp_dir: 350 gold_properties = angle_skia_gold_properties.ANGLESkiaGoldProperties(args) 351 gold_session_manager = angle_skia_gold_session_manager.ANGLESkiaGoldSessionManager( 352 skia_gold_temp_dir, gold_properties) 353 gold_session = gold_session_manager.GetSkiaGoldSession(keys, instance=args.instance) 354 355 traces = [trace.split(' ')[0] for trace in tests] 356 357 if args.isolated_script_test_filter: 358 traces = angle_test_util.FilterTests(traces, args.isolated_script_test_filter) 359 assert traces, 'Test filter did not match any tests' 360 361 if angle_test_util.IsAndroid(): 362 # On Android, screen orientation changes between traces can result in small pixel diffs 363 # making results depend on the ordering of traces. Disable batching. 364 batch_size = 1 365 else: 366 batch_size = args.batch_size 367 368 batches = _get_batches(traces, batch_size) 369 370 for batch in batches: 371 if angle_test_util.IsAndroid(): 372 android_helper.PrepareRestrictedTraces(batch) 373 374 for iteration in range(0, args.flaky_retries + 1): 375 # This is how we signal early exit 376 if not batch: 377 logging.debug('All tests in batch completed.') 378 break 379 if iteration > 0: 380 logging.info('Test run failed, running retry #%d...' % iteration) 381 382 gtest_filter = _get_gtest_filter_for_batch(args, batch) 383 cmd_args = [ 384 gtest_filter, 385 '--run-to-key-frame', 386 '--verbose-logging', 387 '--render-test-output-dir=%s' % screenshot_dir, 388 '--save-screenshots', 389 ] + extra_flags 390 if args.swiftshader: 391 cmd_args += ['--use-angle=swiftshader'] 392 393 logging.info('Running batch with args: %s' % cmd_args) 394 result, _, json_results = angle_test_util.RunTestSuite( 395 args.test_suite, cmd_args, env, use_xvfb=args.xvfb) 396 if result == 0: 397 batch_result = PASS 398 else: 399 batch_result = FAIL 400 logging.error('Batch FAIL! json_results: %s' % 401 json.dumps(json_results, indent=2)) 402 403 next_batch = [] 404 for trace in batch: 405 artifacts = {} 406 407 if batch_result == PASS: 408 test_name = DEFAULT_TEST_PREFIX + trace 409 if json_results['tests'][test_name]['actual'] == 'SKIP': 410 logging.info('Test skipped by suite: %s' % test_name) 411 result = SKIP 412 else: 413 logging.debug('upload test result: %s' % trace) 414 result = upload_test_result_to_skia_gold(args, gold_session_manager, 415 gold_session, gold_properties, 416 screenshot_dir, trace, 417 artifacts) 418 else: 419 result = batch_result 420 421 expected_result = SKIP if result == SKIP else PASS 422 test_results[trace] = {'expected': expected_result, 'actual': result} 423 if len(artifacts) > 0: 424 test_results[trace]['artifacts'] = artifacts 425 if result == FAIL: 426 next_batch.append(trace) 427 batch = next_batch 428 429 # These properties are recorded after iteration to ensure they only happen once. 430 for _, trace_results in test_results.items(): 431 result = trace_results['actual'] 432 results['num_failures_by_type'][result] += 1 433 if result == FAIL: 434 trace_results['is_unexpected'] = True 435 436 return results['num_failures_by_type'][FAIL] == 0 437 438 439def _shard_tests(tests, shard_count, shard_index): 440 return [tests[index] for index in range(shard_index, len(tests), shard_count)] 441 442 443def main(): 444 parser = argparse.ArgumentParser() 445 parser.add_argument('--isolated-script-test-output', type=str) 446 parser.add_argument('--isolated-script-test-perf-output', type=str) 447 parser.add_argument('-f', '--isolated-script-test-filter', '--filter', type=str) 448 parser.add_argument('--test-suite', help='Test suite to run.', default=DEFAULT_TEST_SUITE) 449 parser.add_argument('--render-test-output-dir', help='Directory to store screenshots') 450 parser.add_argument('--xvfb', help='Start xvfb.', action='store_true') 451 parser.add_argument( 452 '--flaky-retries', help='Number of times to retry failed tests.', type=int, default=0) 453 parser.add_argument( 454 '--shard-count', 455 help='Number of shards for test splitting. Default is 1.', 456 type=int, 457 default=1) 458 parser.add_argument( 459 '--shard-index', 460 help='Index of the current shard for test splitting. Default is 0.', 461 type=int, 462 default=0) 463 parser.add_argument( 464 '--batch-size', 465 help='Number of tests to run in a group. Default: %d (disabled on Android)' % 466 DEFAULT_BATCH_SIZE, 467 type=int, 468 default=DEFAULT_BATCH_SIZE) 469 parser.add_argument( 470 '-l', '--log', help='Log output level. Default is %s.' % DEFAULT_LOG, default=DEFAULT_LOG) 471 parser.add_argument('--swiftshader', help='Test with SwiftShader.', action='store_true') 472 parser.add_argument( 473 '-i', 474 '--instance', 475 '--gold-instance', 476 '--skia-gold-instance', 477 help='Skia Gold instance. Default is "%s".' % DEFAULT_GOLD_INSTANCE, 478 default=DEFAULT_GOLD_INSTANCE) 479 parser.add_argument( 480 '--use-permissive-pixel-comparison', 481 type=int, 482 help='Use a more permissive pixel comparison algorithm than the ' 483 'default "allow rounding errors" one. This is intended for use on CLs ' 484 'that are likely to cause differences in many tests, e.g. SwiftShader ' 485 'or driver changes. Can be enabled on bots by adding a ' 486 '"Use-Permissive-Angle-Pixel-Comparison: True" footer.') 487 488 add_skia_gold_args(parser) 489 490 args, extra_flags = parser.parse_known_args() 491 angle_test_util.SetupLogging(args.log.upper()) 492 493 env = os.environ.copy() 494 495 if angle_test_util.HasGtestShardsAndIndex(env): 496 args.shard_count, args.shard_index = angle_test_util.PopGtestShardsAndIndex(env) 497 498 angle_test_util.Initialize(args.test_suite) 499 500 results = { 501 'tests': {}, 502 'interrupted': False, 503 'seconds_since_epoch': time.time(), 504 'path_delimiter': '.', 505 'version': 3, 506 'num_failures_by_type': { 507 FAIL: 0, 508 PASS: 0, 509 SKIP: 0, 510 }, 511 } 512 513 test_results = {} 514 515 rc = 0 516 517 try: 518 # read test set 519 json_name = os.path.join(angle_path_util.ANGLE_ROOT_DIR, 'src', 'tests', 520 'restricted_traces', 'restricted_traces.json') 521 with open(json_name) as fp: 522 tests = json.load(fp) 523 524 # Split tests according to sharding 525 sharded_tests = _shard_tests(tests['traces'], args.shard_count, args.shard_index) 526 527 if args.render_test_output_dir: 528 if not _run_tests(args, sharded_tests, extra_flags, env, args.render_test_output_dir, 529 results, test_results): 530 rc = 1 531 elif 'ISOLATED_OUTDIR' in env: 532 if not _run_tests(args, sharded_tests, extra_flags, env, env['ISOLATED_OUTDIR'], 533 results, test_results): 534 rc = 1 535 else: 536 with temporary_dir('angle_trace_') as temp_dir: 537 if not _run_tests(args, sharded_tests, extra_flags, env, temp_dir, results, 538 test_results): 539 rc = 1 540 541 except Exception: 542 traceback.print_exc() 543 results['interrupted'] = True 544 rc = 1 545 546 if test_results: 547 results['tests']['angle_restricted_trace_gold_tests'] = test_results 548 549 if args.isolated_script_test_output: 550 with open(args.isolated_script_test_output, 'w') as out_file: 551 out_file.write(json.dumps(results, indent=2)) 552 553 if args.isolated_script_test_perf_output: 554 with open(args.isolated_script_test_perf_output, 'w') as out_file: 555 out_file.write(json.dumps({})) 556 557 return rc 558 559 560if __name__ == '__main__': 561 sys.exit(main()) 562