1#! /usr/bin/env python3 2# 3# Copyright 2021 The ANGLE Project Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6# 7''' 8Pixel 6 (ARM based) specific script that measures the following for each restricted_trace: 9- Wall time per frame 10- GPU time per frame (if run with --vsync) 11- CPU time per frame 12- GPU power per frame 13- CPU power per frame 14- GPU memory per frame 15 16Setup: 17 18 autoninja -C out/<config> angle_trace_perf_tests angle_apks 19 20Recommended command to run: 21 22 out/<config>/restricted_trace_perf --fixedtime 10 --power --memory --output-tag android.$(date '+%Y%m%d') --loop-count 5 23 24Alternatively, you can pass the build directory and run from anywhere: 25 26 python3 restricted_trace_perf.py --fixedtime 10 --power --output-tag android.$(date '+%Y%m%d') --loop-count 5 --build-dir ../../../out/<config> 27 28- This will run through all the traces 5 times with the native driver, then 5 times with vulkan (via ANGLE) 29- 10 second run time with one warmup loop 30 31Output will be printed to the terminal as it is collected. 32 33Of the 5 runs, the high and low for each data point will be dropped, average of the remaining three will be tracked in the summary spreadsheet. 34''' 35 36import argparse 37import contextlib 38import copy 39import csv 40import fnmatch 41import json 42import logging 43import os 44import pathlib 45import re 46import statistics 47import subprocess 48import sys 49import threading 50import time 51 52from collections import defaultdict, namedtuple 53from datetime import datetime 54 55PY_UTILS = str(pathlib.Path(__file__).resolve().parents[1] / 'py_utils') 56if PY_UTILS not in sys.path: 57 os.stat(PY_UTILS) and sys.path.insert(0, PY_UTILS) 58import android_helper 59import angle_path_util 60import angle_test_util 61 62DEFAULT_TEST_DIR = '.' 63DEFAULT_LOG_LEVEL = 'info' 64DEFAULT_ANGLE_PACKAGE = 'com.android.angle.test' 65 66Result = namedtuple('Result', ['stdout', 'stderr', 'time']) 67 68 69class _global(object): 70 current_user = None 71 storage_dir = None 72 cache_dir = None 73 74 75def init(): 76 _global.current_user = run_adb_shell_command('am get-current-user').strip() 77 _global.storage_dir = '/data/user/' + _global.current_user + '/com.android.angle.test/files' 78 _global.cache_dir = '/data/user_de/' + _global.current_user + '/com.android.angle.test/cache' 79 logging.debug('Running with user %s, storage %s, cache %s', _global.current_user, 80 _global.storage_dir, _global.cache_dir) 81 82 83def run_async_adb_shell_command(cmd): 84 logging.debug('Kicking off subprocess %s' % (cmd)) 85 86 try: 87 async_process = subprocess.Popen([android_helper.FindAdb(), 'shell', cmd], 88 stdout=subprocess.PIPE, 89 stderr=subprocess.STDOUT) 90 except subprocess.CalledProcessError as e: 91 raise RuntimeError("command '{}' return with error (code {}): {}".format( 92 e.cmd, e.returncode, e.output)) 93 94 logging.debug('Done launching subprocess') 95 96 return async_process 97 98 99def run_adb_command(args): 100 return android_helper._AdbRun(args).decode() 101 102 103def run_adb_shell_command(cmd): 104 return android_helper._AdbShell(cmd).decode() 105 106 107def run_async_adb_command(args): 108 return run_async_command('adb ' + args) 109 110 111def cleanup(): 112 run_adb_shell_command('rm -f ' + _global.storage_dir + '/out.txt ' + _global.storage_dir + 113 '/gpumem.txt') 114 115 116def clear_blob_cache(): 117 run_adb_shell_command('run-as com.android.angle.test rm -rf ' + _global.cache_dir) 118 119 120def select_device(device_arg): 121 # The output from 'adb devices' always includes a header and a new line at the end. 122 result_dev_out = run_adb_command(['devices']).strip() 123 124 result_header_end = result_dev_out.find('\n') 125 result_dev_out = '' if result_header_end == -1 else result_dev_out[result_header_end:] 126 result_dev_out = result_dev_out.split()[0::2] 127 128 def print_device_list(): 129 print('\nList of available devices:\n{}'.format('\n'.join(result_dev_out))) 130 131 num_connected_devices = len(result_dev_out) 132 133 # Check the device arg first. If there is none, use the ANDROID SERIAL env var. 134 android_serial_env = os.environ.get('ANDROID_SERIAL') 135 device_serial = device_arg if device_arg != '' else android_serial_env 136 137 # Check for device exceptions 138 if num_connected_devices == 0: 139 logging.error('DeviceError: No devices detected. Please connect a device and try again.') 140 exit() 141 142 if num_connected_devices > 1 and device_serial is None: 143 logging.error( 144 'DeviceError: More than one device detected. Please specify a target device\n' 145 'through either the --device option or $ANDROID_SERIAL, and try again.') 146 print_device_list() 147 exit() 148 149 if device_serial is not None and device_serial not in result_dev_out: 150 logging.error('DeviceError: Device with serial {} not detected.'.format(device_serial)) 151 if device_arg != '': 152 logging.error('Please update the --device input and try again.') 153 else: 154 logging.error('Please update $ANDROID_SERIAL and try again.') 155 print_device_list() 156 exit() 157 158 # Select device 159 if device_serial is not None: 160 logging.info('Device with serial {} selected.'.format(device_serial)) 161 os.environ['ANDROID_SERIAL'] = device_serial 162 163 else: 164 logging.info('Default device ({}) selected.'.format(result_dev_out[0])) 165 166 167def get_mode(args): 168 mode = '' 169 if args.vsync: 170 mode = 'vsync' 171 elif args.offscreen: 172 mode = 'offscreen' 173 174 return mode 175 176 177def get_trace_width(mode): 178 width = 40 179 if mode == 'vsync': 180 width += 5 181 elif mode == 'offscreen': 182 width += 10 183 184 return width 185 186 187# This function changes to the target directory, then 'yield' passes execution to the inner part of 188# the 'with' block that invoked it. The 'finally' block is executed at the end of the 'with' block, 189# including when exceptions are raised. 190@contextlib.contextmanager 191def run_from_dir(dir): 192 # If not set, just run the command and return 193 if not dir: 194 yield 195 return 196 # Otherwise, change directories 197 cwd = os.getcwd() 198 os.chdir(dir) 199 try: 200 yield 201 finally: 202 os.chdir(cwd) 203 204 205def run_trace(trace, args): 206 mode = get_mode(args) 207 208 # Kick off a subprocess that collects peak gpu memory periodically 209 # Note the 0.25 below is the delay (in seconds) between memory checks 210 if args.memory: 211 run_adb_command([ 212 'push', 213 os.path.join(angle_path_util.ANGLE_ROOT_DIR, 'src', 'tests', 'restricted_traces', 214 'gpumem.sh'), '/data/local/tmp' 215 ]) 216 memory_command = 'sh /data/local/tmp/gpumem.sh 0.25 ' + _global.storage_dir 217 memory_process = run_async_adb_shell_command(memory_command) 218 219 flags = [ 220 '--gtest_filter=TraceTest.' + trace, '--use-gl=native', '--verbose', '--verbose-logging' 221 ] 222 if mode != '': 223 flags.append('--' + mode) 224 if args.maxsteps != '': 225 flags += ['--max-steps-performed', args.maxsteps] 226 if args.fixedtime != '': 227 flags += ['--fixed-test-time-with-warmup', args.fixedtime] 228 if args.minimizegpuwork: 229 flags.append('--minimize-gpu-work') 230 231 # Build a command that can be run directly over ADB, for example: 232 r''' 233adb shell am instrument -w \ 234 -e org.chromium.native_test.NativeTestInstrumentationTestRunner.StdoutFile /data/user/0/com.android.angle.test/files/out.txt \ 235 -e org.chromium.native_test.NativeTest.CommandLineFlags \ 236 "--gtest_filter=TraceTest.empires_and_puzzles\ --use-angle=vulkan\ --screenshot-dir\ /data/user/0/com.android.angle.test/files\ --screenshot-frame\ 2\ --max-steps-performed\ 2\ --no-warmup" \ 237 -e org.chromium.native_test.NativeTestInstrumentationTestRunner.ShardNanoTimeout "1000000000000000000" \ 238 -e org.chromium.native_test.NativeTestInstrumentationTestRunner.NativeTestActivity com.android.angle.test.AngleUnitTestActivity \ 239 com.android.angle.test/org.chromium.build.gtest_apk.NativeTestInstrumentationTestRunner 240 ''' 241 shell_command = r''' 242am instrument -w \ 243 -e org.chromium.native_test.NativeTestInstrumentationTestRunner.StdoutFile {storage}/out.txt \ 244 -e org.chromium.native_test.NativeTest.CommandLineFlags "{flags}" \ 245 -e org.chromium.native_test.NativeTestInstrumentationTestRunner.ShardNanoTimeout "1000000000000000000" \ 246 -e org.chromium.native_test.NativeTestInstrumentationTestRunner.NativeTestActivity \ 247 com.android.angle.test.AngleUnitTestActivity \ 248 com.android.angle.test/org.chromium.build.gtest_apk.NativeTestInstrumentationTestRunner 249 '''.format( 250 flags=r' '.join(flags), 251 storage=_global.storage_dir).strip() # Note: space escaped due to subprocess shell=True 252 253 start_time = time.time() 254 result = run_adb_shell_command(shell_command) 255 time_elapsed = time.time() - start_time 256 257 if args.memory: 258 logging.debug('Killing gpumem subprocess') 259 memory_process.kill() 260 261 return time_elapsed 262 263 264def get_test_time(): 265 # Pull the results from the device and parse 266 result = run_adb_shell_command('cat ' + _global.storage_dir + 267 '/out.txt | grep -v Error | grep -v Frame') 268 269 measured_time = None 270 271 for line in result.splitlines(): 272 logging.debug('Checking line: %s' % line) 273 274 # Look for this line and grab the second to last entry: 275 # Mean result time: 1.2793 ms 276 if "Mean result time" in line: 277 measured_time = line.split()[-2] 278 break 279 280 # Check for skipped tests 281 if "Test skipped due to missing extension" in line: 282 missing_ext = line.split()[-1] 283 logging.debug('Skipping test due to missing extension: %s' % missing_ext) 284 measured_time = missing_ext 285 break 286 287 if measured_time is None: 288 if '[ PASSED ]' in result.stdout: 289 measured_time = 'missing' 290 else: 291 measured_time = 'crashed' 292 293 return measured_time 294 295 296def get_gpu_memory(trace_duration): 297 # Pull the results from the device and parse 298 result = run_adb_shell_command('cat ' + _global.storage_dir + '/gpumem.txt | awk "NF"') 299 300 # The gpumem script grabs snapshots of memory per process 301 # Output looks like this, repeated once per sleep_duration of the test: 302 # 303 # time_elapsed: 9 304 # com.android.angle.test:test_process 16513 305 # Memory snapshot for GPU 0: 306 # Global total: 516833280 307 # Proc 504 total: 170385408 308 # Proc 1708 total: 33767424 309 # Proc 2011 total: 17018880 310 # Proc 16513 total: 348479488 311 # Proc 27286 total: 20877312 312 # Proc 27398 total: 1028096 313 314 # Gather the memory at each snapshot 315 time_elapsed = '' 316 test_process = '' 317 gpu_mem = [] 318 gpu_mem_sustained = [] 319 for line in result.splitlines(): 320 logging.debug('Checking line: %s' % line) 321 322 if "time_elapsed" in line: 323 time_elapsed = line.split()[-1] 324 logging.debug('time_elapsed: %s' % time_elapsed) 325 continue 326 327 # Look for this line and grab the last entry: 328 # com.android.angle.test:test_process 31933 329 if "com.android.angle.test" in line: 330 test_process = line.split()[-1] 331 logging.debug('test_process: %s' % test_process) 332 continue 333 334 # If we don't know test process yet, break 335 if test_process == '': 336 continue 337 338 # If we made it this far, we have the process id 339 340 # Look for this line and grab the last entry: 341 # Proc 31933 total: 235184128 342 if test_process in line: 343 gpu_mem_entry = line.split()[-1] 344 logging.debug('Adding: %s to gpu_mem' % gpu_mem_entry) 345 gpu_mem.append(int(gpu_mem_entry)) 346 # logging.debug('gpu_mem contains: %i' % ' '.join(gpu_mem)) 347 if safe_cast_float(time_elapsed) >= (safe_cast_float(trace_duration) / 2): 348 # Start tracking sustained memory usage at the half way point 349 logging.debug('Adding: %s to gpu_mem_sustained' % gpu_mem_entry) 350 gpu_mem_sustained.append(int(gpu_mem_entry)) 351 continue 352 353 gpu_mem_max = 0 354 if len(gpu_mem) != 0: 355 gpu_mem_max = max(gpu_mem) 356 357 gpu_mem_average = 0 358 if len(gpu_mem_sustained) != 0: 359 gpu_mem_average = statistics.mean(gpu_mem_sustained) 360 361 return gpu_mem_average, gpu_mem_max 362 363 364def get_proc_memory(): 365 # Pull the results from the device and parse 366 result = run_adb_shell_command('cat ' + _global.storage_dir + '/out.txt') 367 memory_median = '' 368 memory_max = '' 369 370 for line in result.splitlines(): 371 # Look for "memory_max" in the line and grab the second to last entry: 372 logging.debug('Checking line: %s' % line) 373 374 if "memory_median" in line: 375 memory_median = line.split()[-2] 376 continue 377 378 if "memory_max" in line: 379 memory_max = line.split()[-2] 380 continue 381 382 return safe_cast_int(memory_max), safe_cast_int(memory_median) 383 384 385def get_gpu_time(): 386 # Pull the results from the device and parse 387 result = run_adb_shell_command('cat ' + _global.storage_dir + '/out.txt') 388 gpu_time = '0' 389 390 for line in result.splitlines(): 391 # Look for "gpu_time" in the line and grab the second to last entry: 392 logging.debug('Checking line: %s' % line) 393 394 if "gpu_time" in line: 395 gpu_time = line.split()[-2] 396 break 397 398 return gpu_time 399 400 401def get_cpu_time(): 402 # Pull the results from the device and parse 403 result = run_adb_shell_command('cat ' + _global.storage_dir + '/out.txt') 404 cpu_time = '0' 405 406 for line in result.splitlines(): 407 # Look for "cpu_time" in the line and grab the second to last entry: 408 logging.debug('Checking line: %s' % line) 409 410 if "cpu_time" in line: 411 cpu_time = line.split()[-2] 412 break 413 414 return cpu_time 415 416 417def get_frame_count(): 418 # Pull the results from the device and parse 419 result = run_adb_shell_command('cat ' + _global.storage_dir + 420 '/out.txt | grep -v Error | grep -v Frame') 421 422 frame_count = 0 423 424 for line in result.splitlines(): 425 logging.debug('Checking line: %s' % line) 426 if "trial_steps" in line: 427 frame_count = line.split()[-2] 428 break 429 430 logging.debug('Frame count: %s' % frame_count) 431 return frame_count 432 433 434class GPUPowerStats(): 435 436 # GPU power measured in uWs 437 438 def __init__(self): 439 self.power = {'gpu': 0, 'big_cpu': 0, 'mid_cpu': 0, 'little_cpu': 0} 440 441 def gpu_delta(self, starting): 442 return self.power['gpu'] - starting.power['gpu'] 443 444 def cpu_delta(self, starting): 445 big = self.power['big_cpu'] - starting.power['big_cpu'] 446 mid = self.power['mid_cpu'] - starting.power['mid_cpu'] 447 little = self.power['little_cpu'] - starting.power['little_cpu'] 448 return big + mid + little 449 450 def get_power_data(self): 451 energy_value_result = run_adb_shell_command( 452 'cat /sys/bus/iio/devices/iio:device*/energy_value') 453 # Output like this (ordering doesn't matter) 454 # 455 # t=251741617 456 # CH0(T=251741617)[VSYS_PWR_WLAN_BT], 192612469095 457 # CH1(T=251741617)[L2S_VDD_AOC_RET], 1393792850 458 # CH2(T=251741617)[S9S_VDD_AOC], 16816975638 459 # CH3(T=251741617)[S5S_VDDQ_MEM], 2720913855 460 # CH4(T=251741617)[S10S_VDD2L], 3656592412 461 # CH5(T=251741617)[S4S_VDD2H_MEM], 4597271837 462 # CH6(T=251741617)[S2S_VDD_G3D], 3702041607 463 # CH7(T=251741617)[L9S_GNSS_CORE], 88535064 464 # t=16086645 465 # CH0(T=16086645)[PPVAR_VSYS_PWR_DISP], 611687448 466 # CH1(T=16086645)[VSYS_PWR_MODEM], 179646995 467 # CH2(T=16086645)[VSYS_PWR_RFFE], 0 468 # CH3(T=16086645)[S2M_VDD_CPUCL2], 124265856 469 # CH4(T=16086645)[S3M_VDD_CPUCL1], 170096352 470 # CH5(T=16086645)[S4M_VDD_CPUCL0], 289995530 471 # CH6(T=16086645)[S5M_VDD_INT], 190164699 472 # CH7(T=16086645)[S1M_VDD_MIF], 196512891 473 474 id_map = { 475 r'S2S_VDD_G3D\b|S2S_VDD_GPU\b': 'gpu', 476 r'S\d+M_VDD_CPUCL2\b|S2M_VDD_CPU2\b': 'big_cpu', 477 r'S\d+M_VDD_CPUCL1\b|S3M_VDD_CPU1\b': 'mid_cpu', 478 r'S\d+M_VDD_CPUCL0\b|S4M_VDD_CPU\b': 'little_cpu', 479 } 480 481 for m in id_map.values(): 482 self.power[m] = 0 # Set to 0 to check for missing values and dupes below 483 484 for line in energy_value_result.splitlines(): 485 for mid, m in id_map.items(): 486 if re.search(mid, line): 487 value = int(line.split()[1]) 488 logging.debug('Power metric %s (%s): %d', mid, m, value) 489 assert self.power[m] == 0, 'Duplicate power metric: %s (%s)' % (mid, m) 490 self.power[m] = value 491 492 for mid, m in id_map.items(): 493 assert self.power[m] != 0, 'Power metric not found: %s (%s)' % (mid, m) 494 495 496def wait_for_test_warmup(done_event): 497 p = subprocess.Popen(['adb', 'logcat', '*:S', 'ANGLE:I'], 498 stdout=subprocess.PIPE, 499 text=True, 500 bufsize=1) # line-buffered 501 os.set_blocking(p.stdout.fileno(), False) 502 503 start_time = time.time() 504 while True: 505 line = p.stdout.readline() # non-blocking as per set_blocking above 506 507 # Look for text logged by the harness when warmup is complete and a test is starting 508 if 'running test name' in line: 509 p.kill() 510 break 511 if done_event.is_set(): 512 logging.warning('Test finished without logging to logcat') 513 p.kill() 514 break 515 516 time.sleep(0.05) 517 518 p.poll() 519 if p.returncode != None: 520 logging.warning('Logcat terminated unexpectedly') 521 return 522 523 524def collect_power(done_event, test_fixedtime, results): 525 # Starting point is post test warmup as there are spikes during warmup 526 wait_for_test_warmup(done_event) 527 528 starting_power = GPUPowerStats() 529 starting_power.get_power_data() 530 logging.debug('Starting power: %s' % starting_power.power) 531 532 duration = test_fixedtime - 1.0 # Avoid measuring through test termination 533 start_time = time.time() 534 while time.time() - start_time < duration: 535 if done_event.is_set(): 536 logging.warning('Test finished earlier than expected by collect_power') 537 break 538 time.sleep(0.05) 539 540 ending_power = GPUPowerStats() 541 ending_power.get_power_data() 542 dt = time.time() - start_time 543 logging.debug('Ending power: %s' % ending_power.power) 544 545 results.update({ 546 # 1e6 for uW -> W 547 'cpu': ending_power.cpu_delta(starting_power) / dt / 1e6, 548 'gpu': ending_power.gpu_delta(starting_power) / dt / 1e6, 549 }) 550 551 552def get_thermal_info(): 553 out = run_adb_shell_command('dumpsys android.hardware.thermal.IThermal/default') 554 result = [] 555 for line in out.splitlines(): 556 if 'ThrottlingStatus:' in line: 557 name = re.search('Name: ([^ ]*)', line).group(1) 558 if ('VIRTUAL-SKIN' in name and 559 '-CHARGE-' not in name and # only supposed to affect charging speed 560 'MODEL' not in name.split('-')): # different units and not used for throttling 561 result.append(line) 562 563 if not result: 564 logging.error('Unexpected dumpsys IThermal response:\n%s', out) 565 raise RuntimeError('Unexpected dumpsys IThermal response, logged above') 566 return result 567 568 569def set_vendor_thermal_control(disabled=None): 570 if disabled: 571 # When disabling, first wait for vendor throttling to end to reset all state 572 waiting = True 573 while waiting: 574 waiting = False 575 for line in get_thermal_info(): 576 if 'ThrottlingStatus: NONE' not in line: 577 logging.info('Waiting for vendor throttling to finish: %s', line.strip()) 578 time.sleep(10) 579 waiting = True 580 break 581 582 run_adb_shell_command('setprop persist.vendor.disable.thermal.control %d' % disabled) 583 584 585def sleep_until_temps_below(limit_temp): 586 waiting = True 587 while waiting: 588 waiting = False 589 for line in get_thermal_info(): 590 v = float(re.search('CurrentValue: ([^ ]*)', line).group(1)) 591 if v > limit_temp: 592 logging.info('Waiting for device temps below %.1f: %s', limit_temp, line.strip()) 593 time.sleep(5) 594 waiting = True 595 break 596 597 598def sleep_until_temps_below_thermalservice(limit_temp): 599 waiting = True 600 while waiting: 601 waiting = False 602 lines = run_adb_shell_command('dumpsys thermalservice').splitlines() 603 assert 'HAL Ready: true' in lines 604 for l in lines[lines.index('Current temperatures from HAL:') + 1:]: 605 if 'Temperature{' not in l: 606 break 607 v = re.search(r'mValue=([^,}]+)', l).group(1) 608 # Note: on some Pixel devices odd component temps are also reported here 609 # but some other key ones are not (e.g. CPU ODPM controlling cpu freqs) 610 if float(v) > limit_temp: 611 logging.info('Waiting for device temps below %.1f: %s', limit_temp, l.strip()) 612 time.sleep(5) 613 waiting = True 614 break 615 616 617def sleep_until_battery_level(min_battery_level): 618 while True: 619 level = int(run_adb_shell_command('dumpsys battery get level').strip()) 620 if level >= min_battery_level: 621 break 622 logging.info('Waiting for device battery level to reach %d. Current level: %d', 623 min_battery_level, level) 624 time.sleep(10) 625 626 627def drop_high_low_and_average(values): 628 if len(values) >= 3: 629 values.remove(min(values)) 630 values.remove(max(values)) 631 632 average = statistics.mean(values) 633 634 variance = 0 635 if len(values) >= 2 and average != 0: 636 variance = statistics.stdev(values) / average 637 638 print(average, variance) 639 640 return float(average), float(variance) 641 642 643def get_angle_version(): 644 angle_version = android_helper._Run('git rev-parse HEAD'.split(' ')).decode().strip() 645 origin_main_version = android_helper._Run( 646 'git rev-parse origin/main'.split(' ')).decode().strip() 647 if origin_main_version != angle_version: 648 angle_version += ' (origin/main %s)' % origin_main_version 649 return angle_version 650 651 652def safe_divide(x, y): 653 if y == 0: 654 return 0 655 return x / y 656 657 658def safe_cast_float(x): 659 if x == '': 660 return 0 661 return float(x) 662 663 664def safe_cast_int(x): 665 if x == '': 666 return 0 667 return int(x) 668 669 670def percent(x): 671 return "%.2f%%" % (x * 100) 672 673 674def main(): 675 parser = argparse.ArgumentParser() 676 parser.add_argument('-f', '--filter', help='Trace filter. Defaults to all.', default='*') 677 parser.add_argument('-l', '--log', help='Logging level.', default=DEFAULT_LOG_LEVEL) 678 parser.add_argument( 679 '--renderer', 680 help='Which renderer to use: native, vulkan (via ANGLE), or default (' + 681 'GLES driver selected by system). Providing no option will run twice, native and vulkan', 682 default='both') 683 parser.add_argument( 684 '--walltimeonly', 685 help='Limit output to just wall time', 686 action='store_true', 687 default=False) 688 parser.add_argument( 689 '--power', help='Include CPU/GPU power used per trace', action='store_true', default=False) 690 parser.add_argument( 691 '--memory', 692 help='Include CPU/GPU memory used per trace', 693 action='store_true', 694 default=False) 695 parser.add_argument('--maxsteps', help='Run for fixed set of frames', default='') 696 parser.add_argument('--fixedtime', help='Run for fixed set of time', default='') 697 parser.add_argument( 698 '--minimizegpuwork', 699 help='Whether to run with minimized GPU work', 700 action='store_true', 701 default=False) 702 parser.add_argument('--output-tag', help='Tag for output files.', required=True) 703 parser.add_argument('--angle-version', help='Specify ANGLE version string.', default='') 704 parser.add_argument( 705 '--loop-count', help='How many times to loop through the traces', default=5) 706 parser.add_argument( 707 '--device', help='Which device to run the tests on (use serial)', default='') 708 parser.add_argument( 709 '--sleep', help='Add a sleep of this many seconds between each test)', type=int, default=0) 710 parser.add_argument( 711 '--custom-throttling-temp', 712 help='Custom thermal throttling with limit set to this temperature (off by default)', 713 type=float) 714 parser.add_argument( 715 '--custom-throttling-thermalservice-temp', 716 help='Custom thermal throttling (thermalservice) with limit set to this temperature (off by default)', 717 type=float) 718 parser.add_argument( 719 '--min-battery-level', 720 help='Sleep between tests if battery level drops below this value (off by default)', 721 type=int) 722 parser.add_argument( 723 '--angle-package', 724 help='Where to load ANGLE libraries from. This will load from the test APK by default, ' + 725 'but you can point to any APK that contains ANGLE. Specify \'system\' to use libraries ' + 726 'already on the device', 727 default=DEFAULT_ANGLE_PACKAGE) 728 729 group = parser.add_mutually_exclusive_group() 730 group.add_argument( 731 '--vsync', 732 help='Whether to run the trace in vsync mode', 733 action='store_true', 734 default=False) 735 group.add_argument( 736 '--offscreen', 737 help='Whether to run the trace in offscreen mode', 738 action='store_true', 739 default=False) 740 parser.add_argument( 741 '--build-dir', 742 help='Where to find the APK on the host, i.e. out/Android. If unset, it is assumed you ' + 743 'are running from the build dir already, or are using the wrapper script ' + 744 'out/<config>/restricted_trace_perf.', 745 default='') 746 747 args = parser.parse_args() 748 749 angle_test_util.SetupLogging(args.log.upper()) 750 751 with run_from_dir(args.build_dir): 752 android_helper.Initialize("angle_trace_tests") # includes adb root 753 754 # Determine some starting parameters 755 init() 756 757 try: 758 if args.custom_throttling_temp: 759 set_vendor_thermal_control(disabled=1) 760 run_traces(args) 761 finally: 762 if args.custom_throttling_temp: 763 set_vendor_thermal_control(disabled=0) 764 # Clean up settings, including in case of exceptions (including Ctrl-C) 765 run_adb_shell_command('settings delete global angle_debug_package') 766 run_adb_shell_command('settings delete global angle_gl_driver_selection_pkgs') 767 run_adb_shell_command('settings delete global angle_gl_driver_selection_values') 768 769 return 0 770 771 772def logged_args(): 773 parser = argparse.ArgumentParser() 774 parser.add_argument('--output-tag') 775 _, extra_args = parser.parse_known_args() 776 return ' '.join(extra_args) 777 778 779def run_traces(args): 780 # Load trace names 781 test_json = os.path.join(args.build_dir, 'gen/trace_list.json') 782 with open(os.path.join(DEFAULT_TEST_DIR, test_json)) as f: 783 traces = json.loads(f.read()) 784 785 failures = [] 786 787 mode = get_mode(args) 788 trace_width = get_trace_width(mode) 789 790 # Select the target device 791 select_device(args.device) 792 793 # Add an underscore to the mode for use in loop below 794 if mode != '': 795 mode = mode + '_' 796 797 # Create output CSV 798 raw_data_filename = "raw_data." + args.output_tag + ".csv" 799 output_file = open(raw_data_filename, 'w', newline='') 800 output_writer = csv.writer(output_file) 801 802 # Set some widths that allow easily reading the values, but fit on smaller monitors. 803 column_width = { 804 'trace': trace_width, 805 'wall_time': 15, 806 'gpu_time': 15, 807 'cpu_time': 15, 808 'gpu_power': 10, 809 'cpu_power': 10, 810 'gpu_mem_sustained': 20, 811 'gpu_mem_peak': 15, 812 'proc_mem_median': 20, 813 'proc_mem_peak': 15 814 } 815 816 if args.walltimeonly: 817 print('%-*s' % (trace_width, 'wall_time_per_frame')) 818 else: 819 print('%-*s %-*s %-*s %-*s %-*s %-*s %-*s %-*s %-*s %-*s' % 820 (column_width['trace'], 'trace', column_width['wall_time'], 'wall_time', 821 column_width['gpu_time'], 'gpu_time', column_width['cpu_time'], 'cpu_time', 822 column_width['gpu_power'], 'gpu_power', column_width['cpu_power'], 'cpu_power', 823 column_width['gpu_mem_sustained'], 'gpu_mem_sustained', 824 column_width['gpu_mem_peak'], 'gpu_mem_peak', column_width['proc_mem_median'], 825 'proc_mem_median', column_width['proc_mem_peak'], 'proc_mem_peak')) 826 output_writer.writerow([ 827 'trace', 'wall_time(ms)', 'gpu_time(ms)', 'cpu_time(ms)', 'gpu_power(W)', 828 'cpu_power(W)', 'gpu_mem_sustained', 'gpu_mem_peak', 'proc_mem_median', 'proc_mem_peak' 829 ]) 830 831 if args.power: 832 starting_power = GPUPowerStats() 833 ending_power = GPUPowerStats() 834 835 renderers = [] 836 if args.renderer != "both": 837 renderers.append(args.renderer) 838 else: 839 renderers = ("native", "vulkan") 840 841 wall_times = defaultdict(dict) 842 gpu_times = defaultdict(dict) 843 cpu_times = defaultdict(dict) 844 gpu_powers = defaultdict(dict) 845 cpu_powers = defaultdict(dict) 846 gpu_mem_sustaineds = defaultdict(dict) 847 gpu_mem_peaks = defaultdict(dict) 848 proc_mem_medians = defaultdict(dict) 849 proc_mem_peaks = defaultdict(dict) 850 851 # Organize the data for writing out 852 rows = defaultdict(dict) 853 854 def populate_row(rows, name, results): 855 if len(rows[name]) == 0: 856 rows[name] = defaultdict(list) 857 for renderer, wtimes in results.items(): 858 average, variance = drop_high_low_and_average(wtimes) 859 rows[name][renderer].append(average) 860 rows[name][renderer].append(variance) 861 862 # Generate the SUMMARY output 863 summary_file = open("summary." + args.output_tag + ".csv", 'w', newline='') 864 summary_writer = csv.writer(summary_file) 865 866 android_version = run_adb_shell_command('getprop ro.build.fingerprint').strip() 867 angle_version = args.angle_version or get_angle_version() 868 # test_time = run_command('date \"+%Y%m%d\"').stdout.read().strip() 869 870 summary_writer.writerow([ 871 "\"Android: " + android_version + "\n" + "ANGLE: " + angle_version + "\n" + 872 # "Date: " + test_time + "\n" + 873 "Source: " + raw_data_filename + "\n" + "Args: " + logged_args() + "\"" 874 ]) 875 876 trace_number = 0 877 878 if (len(renderers) == 1) and (renderers[0] != "both"): 879 renderer_name = renderers[0] 880 summary_writer.writerow([ 881 "#", "\"Trace\"", f"\"{renderer_name}\nwall\ntime\nper\nframe\n(ms)\"", 882 f"\"{renderer_name}\nwall\ntime\nvariance\"", 883 f"\"{renderer_name}\nGPU\ntime\nper\nframe\n(ms)\"", 884 f"\"{renderer_name}\nGPU\ntime\nvariance\"", 885 f"\"{renderer_name}\nCPU\ntime\nper\nframe\n(ms)\"", 886 f"\"{renderer_name}\nCPU\ntime\nvariance\"", f"\"{renderer_name}\nGPU\npower\n(W)\"", 887 f"\"{renderer_name}\nGPU\npower\nvariance\"", f"\"{renderer_name}\nCPU\npower\n(W)\"", 888 f"\"{renderer_name}\nCPU\npower\nvariance\"", f"\"{renderer_name}\nGPU\nmem\n(B)\"", 889 f"\"{renderer_name}\nGPU\nmem\nvariance\"", 890 f"\"{renderer_name}\npeak\nGPU\nmem\n(B)\"", 891 f"\"{renderer_name}\npeak\nGPU\nmem\nvariance\"", 892 f"\"{renderer_name}\nprocess\nmem\n(B)\"", 893 f"\"{renderer_name}\nprocess\nmem\nvariance\"", 894 f"\"{renderer_name}\npeak\nprocess\nmem\n(B)\"", 895 f"\"{renderer_name}\npeak\nprocess\nmem\nvariance\"" 896 ]) 897 else: 898 summary_writer.writerow([ 899 "#", "\"Trace\"", "\"Native\nwall\ntime\nper\nframe\n(ms)\"", 900 "\"Native\nwall\ntime\nvariance\"", "\"ANGLE\nwall\ntime\nper\nframe\n(ms)\"", 901 "\"ANGLE\nwall\ntime\nvariance\"", "\"wall\ntime\ncompare\"", 902 "\"Native\nGPU\ntime\nper\nframe\n(ms)\"", "\"Native\nGPU\ntime\nvariance\"", 903 "\"ANGLE\nGPU\ntime\nper\nframe\n(ms)\"", "\"ANGLE\nGPU\ntime\nvariance\"", 904 "\"GPU\ntime\ncompare\"", "\"Native\nCPU\ntime\nper\nframe\n(ms)\"", 905 "\"Native\nCPU\ntime\nvariance\"", "\"ANGLE\nCPU\ntime\nper\nframe\n(ms)\"", 906 "\"ANGLE\nCPU\ntime\nvariance\"", "\"CPU\ntime\ncompare\"", 907 "\"Native\nGPU\npower\n(W)\"", "\"Native\nGPU\npower\nvariance\"", 908 "\"ANGLE\nGPU\npower\n(W)\"", "\"ANGLE\nGPU\npower\nvariance\"", 909 "\"GPU\npower\ncompare\"", "\"Native\nCPU\npower\n(W)\"", 910 "\"Native\nCPU\npower\nvariance\"", "\"ANGLE\nCPU\npower\n(W)\"", 911 "\"ANGLE\nCPU\npower\nvariance\"", "\"CPU\npower\ncompare\"", 912 "\"Native\nGPU\nmem\n(B)\"", "\"Native\nGPU\nmem\nvariance\"", 913 "\"ANGLE\nGPU\nmem\n(B)\"", "\"ANGLE\nGPU\nmem\nvariance\"", "\"GPU\nmem\ncompare\"", 914 "\"Native\npeak\nGPU\nmem\n(B)\"", "\"Native\npeak\nGPU\nmem\nvariance\"", 915 "\"ANGLE\npeak\nGPU\nmem\n(B)\"", "\"ANGLE\npeak\nGPU\nmem\nvariance\"", 916 "\"GPU\npeak\nmem\ncompare\"", "\"Native\nprocess\nmem\n(B)\"", 917 "\"Native\nprocess\nmem\nvariance\"", "\"ANGLE\nprocess\nmem\n(B)\"", 918 "\"ANGLE\nprocess\nmem\nvariance\"", "\"process\nmem\ncompare\"", 919 "\"Native\npeak\nprocess\nmem\n(B)\"", "\"Native\npeak\nprocess\nmem\nvariance\"", 920 "\"ANGLE\npeak\nprocess\nmem\n(B)\"", "\"ANGLE\npeak\nprocess\nmem\nvariance\"", 921 "\"process\npeak\nmem\ncompare\"" 922 ]) 923 924 with run_from_dir(args.build_dir): 925 android_helper.PrepareTestSuite("angle_trace_tests") 926 927 for trace in fnmatch.filter(traces, args.filter): 928 929 print( 930 "\nStarting run for %s loopcount %i with %s at %s\n" % 931 (trace, int(args.loop_count), renderers, datetime.now().strftime('%Y-%m-%d %H:%M:%S'))) 932 933 with run_from_dir(args.build_dir): 934 android_helper.PrepareRestrictedTraces([trace]) 935 936 # Start with clean data containers for each trace 937 rows.clear() 938 wall_times.clear() 939 gpu_times.clear() 940 cpu_times.clear() 941 gpu_powers.clear() 942 cpu_powers.clear() 943 gpu_mem_sustaineds.clear() 944 gpu_mem_peaks.clear() 945 proc_mem_medians.clear() 946 proc_mem_peaks.clear() 947 948 for i in range(int(args.loop_count)): 949 950 for renderer in renderers: 951 952 if renderer == "native": 953 # Force the settings to native 954 run_adb_shell_command( 955 'settings put global angle_gl_driver_selection_pkgs com.android.angle.test' 956 ) 957 run_adb_shell_command( 958 'settings put global angle_gl_driver_selection_values native') 959 elif renderer == "vulkan": 960 # Force the settings to ANGLE 961 run_adb_shell_command( 962 'settings put global angle_gl_driver_selection_pkgs com.android.angle.test' 963 ) 964 run_adb_shell_command( 965 'settings put global angle_gl_driver_selection_values angle') 966 elif renderer == "default": 967 logging.info( 968 'Deleting Android settings for forcing selection of GLES driver, ' + 969 'allowing system to load the default') 970 run_adb_shell_command('settings delete global angle_debug_package') 971 run_adb_shell_command('settings delete global angle_gl_driver_all_angle') 972 run_adb_shell_command('settings delete global angle_gl_driver_selection_pkgs') 973 run_adb_shell_command( 974 'settings delete global angle_gl_driver_selection_values') 975 else: 976 logging.error('Unsupported renderer {}'.format(renderer)) 977 exit() 978 979 if args.angle_package == 'system': 980 # Clear the debug package so ANGLE will be loaded from /system/lib64 981 run_adb_shell_command('settings delete global angle_debug_package') 982 else: 983 # Otherwise, load ANGLE from the specified APK 984 run_adb_shell_command('settings put global angle_debug_package ' + 985 args.angle_package) 986 987 # Remove any previous perf results 988 cleanup() 989 # Clear blob cache to avoid post-warmup cache eviction b/298028816 990 clear_blob_cache() 991 992 test = trace.split(' ')[0] 993 994 if args.power: 995 assert args.fixedtime, '--power requires --fixedtime' 996 done_event = threading.Event() 997 run_adb_command(['logcat', '-c']) # needed for wait_for_test_warmup 998 power_results = {} # output arg 999 power_thread = threading.Thread( 1000 target=collect_power, 1001 args=(done_event, float(args.fixedtime), power_results)) 1002 power_thread.daemon = True 1003 power_thread.start() 1004 1005 logging.debug('Running %s' % test) 1006 test_time = run_trace(test, args) 1007 1008 gpu_power, cpu_power = 0, 0 1009 if args.power: 1010 done_event.set() 1011 power_thread.join(timeout=2) 1012 if power_thread.is_alive(): 1013 logging.warning('collect_power thread did not terminate') 1014 else: 1015 gpu_power = power_results['gpu'] 1016 cpu_power = power_results['cpu'] 1017 1018 wall_time = get_test_time() 1019 1020 gpu_time = get_gpu_time() if args.vsync else '0' 1021 1022 cpu_time = get_cpu_time() 1023 1024 gpu_mem_sustained, gpu_mem_peak = 0, 0 1025 proc_mem_peak, proc_mem_median = 0, 0 1026 if args.memory: 1027 gpu_mem_sustained, gpu_mem_peak = get_gpu_memory(test_time) 1028 logging.debug( 1029 '%s = %i, %s = %i' % 1030 ('gpu_mem_sustained', gpu_mem_sustained, 'gpu_mem_peak', gpu_mem_peak)) 1031 1032 proc_mem_peak, proc_mem_median = get_proc_memory() 1033 1034 trace_name = mode + renderer + '_' + test 1035 1036 if len(wall_times[test]) == 0: 1037 wall_times[test] = defaultdict(list) 1038 try: 1039 wt = safe_cast_float(wall_time) 1040 except ValueError: # e.g. 'crashed' 1041 wt = -1 1042 wall_times[test][renderer].append(wt) 1043 1044 if len(gpu_times[test]) == 0: 1045 gpu_times[test] = defaultdict(list) 1046 gpu_times[test][renderer].append(safe_cast_float(gpu_time)) 1047 1048 if len(cpu_times[test]) == 0: 1049 cpu_times[test] = defaultdict(list) 1050 cpu_times[test][renderer].append(safe_cast_float(cpu_time)) 1051 1052 if len(gpu_powers[test]) == 0: 1053 gpu_powers[test] = defaultdict(list) 1054 gpu_powers[test][renderer].append(safe_cast_float(gpu_power)) 1055 1056 if len(cpu_powers[test]) == 0: 1057 cpu_powers[test] = defaultdict(list) 1058 cpu_powers[test][renderer].append(safe_cast_float(cpu_power)) 1059 1060 if len(gpu_mem_sustaineds[test]) == 0: 1061 gpu_mem_sustaineds[test] = defaultdict(list) 1062 gpu_mem_sustaineds[test][renderer].append(safe_cast_int(gpu_mem_sustained)) 1063 1064 if len(gpu_mem_peaks[test]) == 0: 1065 gpu_mem_peaks[test] = defaultdict(list) 1066 gpu_mem_peaks[test][renderer].append(safe_cast_int(gpu_mem_peak)) 1067 1068 if len(proc_mem_medians[test]) == 0: 1069 proc_mem_medians[test] = defaultdict(list) 1070 proc_mem_medians[test][renderer].append(safe_cast_int(proc_mem_median)) 1071 1072 if len(proc_mem_peaks[test]) == 0: 1073 proc_mem_peaks[test] = defaultdict(list) 1074 proc_mem_peaks[test][renderer].append(safe_cast_int(proc_mem_peak)) 1075 1076 if args.walltimeonly: 1077 print('%-*s' % (trace_width, wall_time)) 1078 else: 1079 print( 1080 '%-*s %-*s %-*s %-*s %-*s %-*s %-*i %-*i %-*i %-*i' % 1081 (column_width['trace'], trace_name, column_width['wall_time'], wall_time, 1082 column_width['gpu_time'], gpu_time, column_width['cpu_time'], cpu_time, 1083 column_width['gpu_power'], '%.3f' % gpu_power, column_width['cpu_power'], 1084 '%.3f' % cpu_power, column_width['gpu_mem_sustained'], gpu_mem_sustained, 1085 column_width['gpu_mem_peak'], gpu_mem_peak, 1086 column_width['proc_mem_median'], proc_mem_median, 1087 column_width['proc_mem_peak'], proc_mem_peak)) 1088 output_writer.writerow([ 1089 mode + renderer + '_' + test, wall_time, gpu_time, cpu_time, gpu_power, 1090 cpu_power, gpu_mem_sustained, gpu_mem_peak, proc_mem_median, proc_mem_peak 1091 ]) 1092 1093 1094 # Early exit for testing 1095 #exit() 1096 1097 # Depending on workload, sleeps might be needed to dissipate heat or recharge battery 1098 if args.sleep != 0: 1099 time.sleep(args.sleep) 1100 1101 if args.custom_throttling_temp: 1102 sleep_until_temps_below(args.custom_throttling_temp) 1103 1104 if args.custom_throttling_thermalservice_temp: 1105 sleep_until_temps_below_thermalservice( 1106 args.custom_throttling_thermalservice_temp) 1107 1108 if args.min_battery_level: 1109 sleep_until_battery_level(args.min_battery_level) 1110 1111 print() # New line for readability 1112 1113 for name, results in wall_times.items(): 1114 populate_row(rows, name, results) 1115 1116 for name, results in gpu_times.items(): 1117 populate_row(rows, name, results) 1118 1119 for name, results in cpu_times.items(): 1120 populate_row(rows, name, results) 1121 1122 for name, results in gpu_powers.items(): 1123 populate_row(rows, name, results) 1124 1125 for name, results in cpu_powers.items(): 1126 populate_row(rows, name, results) 1127 1128 for name, results in gpu_mem_sustaineds.items(): 1129 populate_row(rows, name, results) 1130 1131 for name, results in gpu_mem_peaks.items(): 1132 populate_row(rows, name, results) 1133 1134 for name, results in proc_mem_medians.items(): 1135 populate_row(rows, name, results) 1136 1137 for name, results in proc_mem_peaks.items(): 1138 populate_row(rows, name, results) 1139 1140 if (len(renderers) == 1) and (renderers[0] != "both"): 1141 renderer_name = renderers[0] 1142 for name, data in rows.items(): 1143 trace_number += 1 1144 summary_writer.writerow([ 1145 trace_number, 1146 name, 1147 # wall_time 1148 "%.3f" % data[renderer_name][0], 1149 percent(data[renderer_name][1]), 1150 # GPU time 1151 "%.3f" % data[renderer_name][2], 1152 percent(data[renderer_name][3]), 1153 # CPU time 1154 "%.3f" % data[renderer_name][4], 1155 percent(data[renderer_name][5]), 1156 # GPU power 1157 "%.3f" % data[renderer_name][6], 1158 percent(data[renderer_name][7]), 1159 # CPU power 1160 "%.3f" % data[renderer_name][8], 1161 percent(data[renderer_name][9]), 1162 # GPU mem 1163 int(data[renderer_name][10]), 1164 percent(data[renderer_name][11]), 1165 # GPU peak mem 1166 int(data[renderer_name][12]), 1167 percent(data[renderer_name][13]), 1168 # process mem 1169 int(data[renderer_name][14]), 1170 percent(data[renderer_name][15]), 1171 # process peak mem 1172 int(data[renderer_name][16]), 1173 percent(data[renderer_name][17]), 1174 ]) 1175 else: 1176 for name, data in rows.items(): 1177 trace_number += 1 1178 summary_writer.writerow([ 1179 trace_number, 1180 name, 1181 # wall_time 1182 "%.3f" % data["native"][0], 1183 percent(data["native"][1]), 1184 "%.3f" % data["vulkan"][0], 1185 percent(data["vulkan"][1]), 1186 percent(safe_divide(data["native"][0], data["vulkan"][0])), 1187 # GPU time 1188 "%.3f" % data["native"][2], 1189 percent(data["native"][3]), 1190 "%.3f" % data["vulkan"][2], 1191 percent(data["vulkan"][3]), 1192 percent(safe_divide(data["native"][2], data["vulkan"][2])), 1193 # CPU time 1194 "%.3f" % data["native"][4], 1195 percent(data["native"][5]), 1196 "%.3f" % data["vulkan"][4], 1197 percent(data["vulkan"][5]), 1198 percent(safe_divide(data["native"][4], data["vulkan"][4])), 1199 # GPU power 1200 "%.3f" % data["native"][6], 1201 percent(data["native"][7]), 1202 "%.3f" % data["vulkan"][6], 1203 percent(data["vulkan"][7]), 1204 percent(safe_divide(data["native"][6], data["vulkan"][6])), 1205 # CPU power 1206 "%.3f" % data["native"][8], 1207 percent(data["native"][9]), 1208 "%.3f" % data["vulkan"][8], 1209 percent(data["vulkan"][9]), 1210 percent(safe_divide(data["native"][8], data["vulkan"][8])), 1211 # GPU mem 1212 int(data["native"][10]), 1213 percent(data["native"][11]), 1214 int(data["vulkan"][10]), 1215 percent(data["vulkan"][11]), 1216 percent(safe_divide(data["native"][10], data["vulkan"][10])), 1217 # GPU peak mem 1218 int(data["native"][12]), 1219 percent(data["native"][13]), 1220 int(data["vulkan"][12]), 1221 percent(data["vulkan"][13]), 1222 percent(safe_divide(data["native"][12], data["vulkan"][12])), 1223 # process mem 1224 int(data["native"][14]), 1225 percent(data["native"][15]), 1226 int(data["vulkan"][14]), 1227 percent(data["vulkan"][15]), 1228 percent(safe_divide(data["native"][14], data["vulkan"][14])), 1229 # process peak mem 1230 int(data["native"][16]), 1231 percent(data["native"][17]), 1232 int(data["vulkan"][16]), 1233 percent(data["vulkan"][17]), 1234 percent(safe_divide(data["native"][16], data["vulkan"][16])) 1235 ]) 1236 1237 1238if __name__ == '__main__': 1239 sys.exit(main()) 1240