1# Copyright 2019 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5# Recipe which runs DM with trace flag on lottie files and then parses the 6# trace output into output JSON files to ingest to perf.skia.org. 7# Design doc: go/skottie-tracing 8 9 10import calendar 11import json 12import re 13import string 14 15PYTHON_VERSION_COMPATIBILITY = "PY3" 16 17DEPS = [ 18 'flavor', 19 'infra', 20 'recipe_engine/context', 21 'recipe_engine/file', 22 'recipe_engine/json', 23 'recipe_engine/path', 24 'recipe_engine/step', 25 'recipe_engine/time', 26 'recipe_engine/properties', 27 'recipe_engine/raw_io', 28 'run', 29 'vars', 30] 31 32SEEK_TRACE_NAME = 'skottie::Animation::seek' 33RENDER_TRACE_NAME = 'skottie::Animation::render' 34EXPECTED_DM_FRAMES = 25 35 36 37def perf_steps(api): 38 """Run DM on lottie files with tracing turned on and then parse the output.""" 39 api.flavor.create_clean_device_dir( 40 api.flavor.device_dirs.dm_dir) 41 lotties_host = api.path.start_dir.joinpath('lotties_with_assets') 42 lotties_device = api.path.start_dir.joinpath('lotties_with_assets') 43 if 'Android' in api.vars.builder_cfg.get('extra_config'): 44 # Due to http://b/72366966 and the fact that CIPD symlinks files in by default, we ran into 45 # a strange "Function not implemented" error when trying to copy folders that contained 46 # symlinked files. It is not easy to change the CIPD "InstallMode" from symlink to copy, so 47 # we use shutil (file.copytree) to make a local copy of the files on the host, which removes 48 # the symlinks and adb push --sync works as expected. 49 lotties_device = api.flavor.device_path_join(api.flavor.device_dirs.tmp_dir, 'lotties_with_assets') 50 api.flavor.create_clean_device_dir(lotties_device) 51 52 # Make a temp directory and then copy to a *non-existing* subfolder (otherwise copytree crashes). 53 lotties_no_symlinks = api.path.mkdtemp('lwa').joinpath('nosymlinks') 54 api.file.copytree('Copying files on host to remove symlinks', lotties_host, lotties_no_symlinks) 55 lotties_host = lotties_no_symlinks 56 api.flavor.copy_directory_contents_to_device(lotties_host, lotties_device) 57 58 # We expect this to be a bunch of folders that contain a data.json and optionally 59 # an images/ subfolder with image assets required. 60 lottie_files = api.file.listdir( 61 'list lottie files', lotties_host, 62 test_data=['skottie_asset_000', 'skottie_asset_001', 'skottie_asset_002']) 63 perf_results = {} 64 # Run DM on each lottie file and parse the trace files. 65 for idx, lottie_file in enumerate(lottie_files): 66 lottie_name = api.path.basename(lottie_file) 67 lottie_folder = api.flavor.device_path_join(lotties_device, lottie_name) 68 69 trace_output_path = api.flavor.device_path_join( 70 api.flavor.device_dirs.dm_dir, '%s.json' % (idx + 1)) 71 # See go/skottie-tracing for how these flags were selected. 72 dm_args = [ 73 'dm', 74 '--resourcePath', api.flavor.device_dirs.resource_dir, 75 '--lotties', lottie_folder, 76 '--src', 'lottie', 77 '--nonativeFonts', 78 '--verbose', 79 '--traceMatch', 'skottie', # recipe can OOM without this. 80 '--trace', trace_output_path, 81 '--match', get_trace_match( 82 'data.json', 'Android' in api.properties['buildername']), 83 ] 84 if api.vars.builder_cfg.get('cpu_or_gpu') == 'GPU': 85 dm_args.extend(['--config', 'gles', '--nocpu']) 86 elif api.vars.builder_cfg.get('cpu_or_gpu') == 'CPU': 87 dm_args.extend(['--config', '8888', '--nogpu']) 88 api.run(api.flavor.step, 'dm', cmd=dm_args, abort_on_failure=False) 89 90 trace_test_data = api.properties.get('trace_test_data', '{}') 91 trace_file_content = api.flavor.read_file_on_device(trace_output_path) 92 if not trace_file_content and trace_test_data: 93 trace_file_content = trace_test_data 94 95 key = 'gles' 96 if api.vars.builder_cfg.get('cpu_or_gpu') == 'CPU': 97 key = '8888' 98 perf_results[lottie_name] = { 99 key: parse_trace(trace_file_content, lottie_name, api), 100 } 101 api.flavor.remove_file_on_device(trace_output_path) 102 103 # Construct contents of the output JSON. 104 perf_json = { 105 'gitHash': api.properties['revision'], 106 'swarming_bot_id': api.vars.swarming_bot_id, 107 'swarming_task_id': api.vars.swarming_task_id, 108 'renderer': 'skottie', 109 'key': { 110 'bench_type': 'tracing', 111 'source_type': 'skottie', 112 }, 113 'results': perf_results, 114 } 115 if api.vars.is_trybot: 116 perf_json['issue'] = api.vars.issue 117 perf_json['patchset'] = api.vars.patchset 118 perf_json['patch_storage'] = api.vars.patch_storage 119 # Add tokens from the builder name to the key. 120 reg = re.compile('Perf-(?P<os>[A-Za-z0-9_]+)-' 121 '(?P<compiler>[A-Za-z0-9_]+)-' 122 '(?P<model>[A-Za-z0-9_]+)-' 123 '(?P<cpu_or_gpu>[A-Z]+)-' 124 '(?P<cpu_or_gpu_value>[A-Za-z0-9_]+)-' 125 '(?P<arch>[A-Za-z0-9_]+)-' 126 '(?P<configuration>[A-Za-z0-9_]+)-' 127 'All(-(?P<extra_config>[A-Za-z0-9_]+)|)') 128 m = reg.match(api.properties['buildername']) 129 keys = ['os', 'compiler', 'model', 'cpu_or_gpu', 'cpu_or_gpu_value', 'arch', 130 'configuration', 'extra_config'] 131 for k in keys: 132 perf_json['key'][k] = m.group(k) 133 134 # Create the output JSON file in perf_data_dir for the Upload task to upload. 135 api.file.ensure_directory( 136 'makedirs perf_dir', 137 api.flavor.host_dirs.perf_data_dir) 138 now = api.time.utcnow() 139 ts = int(calendar.timegm(now.utctimetuple())) 140 json_path = api.flavor.host_dirs.perf_data_dir.joinpath( 141 'perf_%s_%d.json' % (api.properties['revision'], ts)) 142 json_contents = json.dumps( 143 perf_json, indent=4, sort_keys=True, separators=(',', ': ')) 144 api.file.write_text('write output JSON', json_path, json_contents) 145 146 147def get_trace_match(lottie_filename, is_android): 148 """Returns the DM regex to match the specified lottie file name.""" 149 trace_match = '^%s$' % lottie_filename 150 if is_android and ' ' not in trace_match: 151 # Punctuation characters confuse DM when shelled out over adb, so escape 152 # them. Do not need to do this when there is a space in the match because 153 # subprocess.list2cmdline automatically adds quotes in that case. 154 for sp_char in string.punctuation: 155 if sp_char == '\\': 156 # No need to escape the escape char. 157 continue 158 trace_match = trace_match.replace(sp_char, '\%s' % sp_char) 159 return trace_match 160 161 162def parse_trace(trace_json, lottie_filename, api): 163 """parse_trace parses the specified trace JSON. 164 165 Parses the trace JSON and calculates the time of a single frame. Frame time is 166 considered the same as seek time + render time. 167 Note: The first seek is ignored because it is a constructor call. 168 169 A dictionary is returned that has the following structure: 170 { 171 'frame_max_us': 100, 172 'frame_min_us': 90, 173 'frame_avg_us': 95, 174 } 175 """ 176 script = api.infra.resource('parse_skottie_trace.py') 177 step_result = api.run( 178 api.step, 179 'parse %s trace' % lottie_filename, 180 cmd=['python3', script, trace_json, lottie_filename, api.json.output(), 181 SEEK_TRACE_NAME, RENDER_TRACE_NAME, EXPECTED_DM_FRAMES]) 182 183 # Sanitize float outputs to 2 precision points. 184 output = dict(step_result.json.output) 185 output['frame_max_us'] = float("%.2f" % output['frame_max_us']) 186 output['frame_min_us'] = float("%.2f" % output['frame_min_us']) 187 output['frame_avg_us'] = float("%.2f" % output['frame_avg_us']) 188 return output 189 190 191def RunSteps(api): 192 api.vars.setup() 193 api.file.ensure_directory('makedirs tmp_dir', api.vars.tmp_dir) 194 api.flavor.setup('dm') 195 196 with api.context(): 197 try: 198 api.flavor.install(resources=True, lotties=True) 199 perf_steps(api) 200 finally: 201 api.flavor.cleanup_steps() 202 api.run.check_failure() 203 204 205def GenTests(api): 206 trace_output = """ 207[{"ph":"X","name":"void skottie::Animation::seek(SkScalar)","ts":452,"dur":2.57,"tid":1,"pid":0},{"ph":"X","name":"void SkCanvas::drawPaint(const SkPaint &)","ts":473,"dur":2.67e+03,"tid":1,"pid":0},{"ph":"X","name":"void skottie::Animation::seek(SkScalar)","ts":3.15e+03,"dur":2.25,"tid":1,"pid":0},{"ph":"X","name":"void skottie::Animation::render(SkCanvas *, const SkRect *, RenderFlags) const","ts":3.15e+03,"dur":216,"tid":1,"pid":0},{"ph":"X","name":"void SkCanvas::drawPath(const SkPath &, const SkPaint &)","ts":3.35e+03,"dur":15.1,"tid":1,"pid":0},{"ph":"X","name":"void skottie::Animation::seek(SkScalar)","ts":3.37e+03,"dur":1.17,"tid":1,"pid":0},{"ph":"X","name":"void skottie::Animation::render(SkCanvas *, const SkRect *, RenderFlags) const","ts":3.37e+03,"dur":140,"tid":1,"pid":0}] 208""" 209 dm_json_test_data = """ 210{ 211 "gitHash": "bac53f089dbc473862bc5a2e328ba7600e0ed9c4", 212 "swarming_bot_id": "skia-rpi-094", 213 "swarming_task_id": "438f11c0e19eab11", 214 "key": { 215 "arch": "arm", 216 "compiler": "Clang", 217 "cpu_or_gpu": "GPU", 218 "cpu_or_gpu_value": "Mali400MP2", 219 "extra_config": "Android", 220 "model": "AndroidOne", 221 "os": "Android" 222 }, 223 "results": { 224 } 225} 226""" 227 parse_trace_json = { 228 'frame_avg_us': 179.71, 229 'frame_min_us': 141.17, 230 'frame_max_us': 218.25 231 } 232 android_buildername = ('Perf-Android-Clang-AndroidOne-GPU-Mali400MP2-arm-' 233 'Release-All-Android_SkottieTracing') 234 gpu_buildername = ('Perf-Debian10-Clang-NUC7i5BNK-GPU-IntelIris640-x86_64-' 235 'Release-All-SkottieTracing') 236 cpu_buildername = ('Perf-Debian10-Clang-GCE-CPU-AVX2-x86_64-Release-All-' 237 'SkottieTracing') 238 yield ( 239 api.test(android_buildername) + 240 api.properties(buildername=android_buildername, 241 repository='https://skia.googlesource.com/skia.git', 242 revision='abc123', 243 task_id='abc123', 244 trace_test_data=trace_output, 245 dm_json_test_data=dm_json_test_data, 246 path_config='kitchen', 247 swarm_out_dir='[SWARM_OUT_DIR]') + 248 api.step_data('parse skottie_asset_000 trace', 249 api.json.output(parse_trace_json)) + 250 api.step_data('parse skottie_asset_001 trace', 251 api.json.output(parse_trace_json)) + 252 api.step_data('parse skottie_asset_002 trace', 253 api.json.output(parse_trace_json)) 254 ) 255 yield ( 256 api.test(gpu_buildername) + 257 api.properties(buildername=gpu_buildername, 258 repository='https://skia.googlesource.com/skia.git', 259 revision='abc123', 260 task_id='abc123', 261 trace_test_data=trace_output, 262 dm_json_test_data=dm_json_test_data, 263 path_config='kitchen', 264 swarm_out_dir='[SWARM_OUT_DIR]') + 265 api.step_data('parse skottie_asset_000 trace', 266 api.json.output(parse_trace_json)) + 267 api.step_data('parse skottie_asset_001 trace', 268 api.json.output(parse_trace_json)) + 269 api.step_data('parse skottie_asset_002 trace', 270 api.json.output(parse_trace_json)) 271 ) 272 yield ( 273 api.test(cpu_buildername) + 274 api.properties(buildername=cpu_buildername, 275 repository='https://skia.googlesource.com/skia.git', 276 revision='abc123', 277 task_id='abc123', 278 trace_test_data=trace_output, 279 dm_json_test_data=dm_json_test_data, 280 path_config='kitchen', 281 swarm_out_dir='[SWARM_OUT_DIR]') + 282 api.step_data('parse skottie_asset_000 trace', 283 api.json.output(parse_trace_json)) + 284 api.step_data('parse skottie_asset_001 trace', 285 api.json.output(parse_trace_json)) + 286 api.step_data('parse skottie_asset_002 trace', 287 api.json.output(parse_trace_json)) 288 ) 289 yield ( 290 api.test('skottietracing_parse_trace_error') + 291 api.properties(buildername=android_buildername, 292 repository='https://skia.googlesource.com/skia.git', 293 revision='abc123', 294 task_id='abc123', 295 trace_test_data=trace_output, 296 dm_json_test_data=dm_json_test_data, 297 path_config='kitchen', 298 swarm_out_dir='[SWARM_OUT_DIR]') + 299 api.step_data('parse skottie_asset_000 trace', 300 api.json.output(parse_trace_json), retcode=1) 301 ) 302 yield ( 303 api.test('skottietracing_trybot') + 304 api.properties(buildername=android_buildername, 305 repository='https://skia.googlesource.com/skia.git', 306 revision='abc123', 307 task_id='abc123', 308 trace_test_data=trace_output, 309 dm_json_test_data=dm_json_test_data, 310 path_config='kitchen', 311 swarm_out_dir='[SWARM_OUT_DIR]', 312 patch_ref='89/456789/12', 313 patch_repo='https://skia.googlesource.com/skia.git', 314 patch_storage='gerrit', 315 patch_set=7, 316 patch_issue=1234, 317 gerrit_project='skia', 318 gerrit_url='https://skia-review.googlesource.com/') + 319 api.step_data('parse skottie_asset_000 trace', 320 api.json.output(parse_trace_json)) + 321 api.step_data('parse skottie_asset_001 trace', 322 api.json.output(parse_trace_json)) + 323 api.step_data('parse skottie_asset_002 trace', 324 api.json.output(parse_trace_json)) 325 ) 326