1# Copyright 2018 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5# Recipe which analyzes a compiled binary for information (e.g. file size) 6 7import ast 8import json 9 10PYTHON_VERSION_COMPATIBILITY = "PY3" 11 12DEPS = [ 13 'checkout', 14 'env', 15 'recipe_engine/context', 16 'recipe_engine/file', 17 'recipe_engine/path', 18 'recipe_engine/properties', 19 'recipe_engine/raw_io', 20 'recipe_engine/step', 21 'run', 22 'vars', 23] 24 25 26MAGIC_SEPERATOR = '#$%^&*' 27TOTAL_SIZE_BYTES_KEY = "total_size_bytes" 28 29 30def add_binary_size_output_property(result, source, binary_size): 31 result.presentation.properties['binary_size_%s' % source] = binary_size 32 33 34def RunSteps(api): 35 api.vars.setup() 36 37 checkout_root = api.checkout.default_checkout_root 38 api.checkout.bot_update(checkout_root=checkout_root) 39 40 out_dir = api.vars.swarming_out_dir 41 # Any binaries to scan should be here. 42 bin_dir = api.vars.build_dir 43 44 api.file.ensure_directory('mkdirs out_dir', out_dir, mode=0o777) 45 46 analyzed = 0 47 with api.context(cwd=bin_dir): 48 files = api.file.glob_paths( 49 'find WASM binaries', 50 bin_dir, 51 '*.wasm', 52 test_data=['pathkit.wasm']) 53 analyzed += len(files) 54 if files: 55 analyze_wasm_file(api, checkout_root, out_dir, files) 56 57 files = api.file.glob_paths( 58 'find JS files', 59 bin_dir, 60 '*.js', 61 test_data=['pathkit.js']) 62 analyzed += len(files) 63 if files: 64 analyze_web_file(api, checkout_root, out_dir, files) 65 66 files = api.file.glob_paths( 67 'find JS mem files', 68 bin_dir, 69 '*.js.mem', 70 test_data=['pathkit.js.mem']) 71 analyzed += len(files) 72 if files: 73 analyze_web_file(api, checkout_root, out_dir, files) 74 75 files = api.file.glob_paths( 76 'find flutter library', 77 bin_dir, 78 'libflutter.so', 79 test_data=['libflutter.so']) 80 analyzed += len(files) 81 if files: 82 analyze_flutter_lib(api, checkout_root, out_dir, files) 83 84 files = api.file.glob_paths( 85 'find skia library', 86 bin_dir, 87 'libskia.so', 88 test_data=['libskia.so']) 89 analyzed += len(files) 90 if files: 91 analyze_cpp_lib(api, checkout_root, out_dir, files) 92 93 files = api.file.glob_paths( 94 'find skottie_tool', 95 bin_dir, 96 'skottie_tool', 97 test_data=['skottie_tool']) 98 analyzed += len(files) 99 if files: 100 make_treemap(api, checkout_root, out_dir, files) 101 102 files = api.file.glob_paths( 103 'find dm', 104 bin_dir, 105 'dm', 106 test_data=['dm']) 107 analyzed += len(files) 108 if files: 109 make_treemap(api, checkout_root, out_dir, files) 110 111 if not analyzed: # pragma: nocover 112 raise Exception('No files were analyzed!') 113 114 115def keys_and_props(api): 116 keys = [] 117 for k in sorted(api.vars.builder_cfg.keys()): 118 if not k in ['role']: 119 keys.extend([k, api.vars.builder_cfg[k]]) 120 keystr = ' '.join(keys) 121 122 props = [ 123 'gitHash', api.properties['revision'], 124 'swarming_bot_id', api.vars.swarming_bot_id, 125 'swarming_task_id', api.vars.swarming_task_id, 126 ] 127 128 if api.vars.is_trybot: 129 props.extend([ 130 'issue', api.vars.issue, 131 'patchset', api.vars.patchset, 132 'patch_storage', api.vars.patch_storage, 133 ]) 134 propstr = ' '.join(str(prop) for prop in props) 135 return (keystr, propstr) 136 137 138# Get the raw and gzipped size of the given file 139def analyze_web_file(api, checkout_root, out_dir, files): 140 (keystr, propstr) = keys_and_props(api) 141 142 for f in files: 143 skia_dir = checkout_root.joinpath('skia') 144 with api.context(cwd=skia_dir): 145 script = skia_dir.joinpath('infra', 'bots', 'buildstats', 146 'buildstats_web.py') 147 step_data = api.run(api.step, 'Analyze %s' % f, 148 cmd=['python3', script, f, out_dir, keystr, propstr, 149 TOTAL_SIZE_BYTES_KEY, MAGIC_SEPERATOR], 150 stdout=api.raw_io.output()) 151 if step_data and step_data.stdout: 152 sections = step_data.stdout.decode('utf-8').split(MAGIC_SEPERATOR) 153 result = api.step.active_result 154 logs = result.presentation.logs 155 logs['perf_json'] = sections[1].split('\n') 156 157 add_binary_size_output_property(result, api.path.basename(f), ( 158 ast.literal_eval(sections[1]) 159 .get('results', {}) 160 .get(api.path.basename(f), {}) 161 .get('default', {}) 162 .get(TOTAL_SIZE_BYTES_KEY, {}))) 163 164 165# Get the raw size and a few metrics from bloaty 166def analyze_cpp_lib(api, checkout_root, out_dir, files): 167 (keystr, propstr) = keys_and_props(api) 168 bloaty_exe = api.path.start_dir.joinpath('bloaty', 'bloaty') 169 170 for f in files: 171 skia_dir = checkout_root.joinpath('skia') 172 with api.context(cwd=skia_dir): 173 script = skia_dir.joinpath('infra', 'bots', 'buildstats', 174 'buildstats_cpp.py') 175 step_data = api.run(api.step, 'Analyze %s' % f, 176 cmd=['python3', script, f, out_dir, keystr, propstr, bloaty_exe, 177 TOTAL_SIZE_BYTES_KEY, MAGIC_SEPERATOR], 178 stdout=api.raw_io.output()) 179 if step_data and step_data.stdout: 180 sections = step_data.stdout.decode('utf-8').split(MAGIC_SEPERATOR) 181 result = api.step.active_result 182 logs = result.presentation.logs 183 logs['perf_json'] = sections[2].split('\n') 184 185 add_binary_size_output_property(result, api.path.basename(f), ( 186 ast.literal_eval(sections[2]) 187 .get('results', {}) 188 .get(api.path.basename(f), {}) 189 .get('default', {}) 190 .get(TOTAL_SIZE_BYTES_KEY, {}))) 191 192 193# Get the size of skia in flutter and a few metrics from bloaty 194def analyze_flutter_lib(api, checkout_root, out_dir, files): 195 (keystr, propstr) = keys_and_props(api) 196 bloaty_exe = api.path.start_dir.joinpath('bloaty', 'bloaty') 197 198 for f in files: 199 200 skia_dir = checkout_root.joinpath('skia') 201 with api.context(cwd=skia_dir): 202 stripped = api.vars.build_dir.joinpath('libflutter_stripped.so') 203 script = skia_dir.joinpath('infra', 'bots', 'buildstats', 204 'buildstats_flutter.py') 205 config = "skia_in_flutter" 206 lib_name = "libflutter.so" 207 step_data = api.run(api.step, 'Analyze flutter', 208 cmd=['python3', script, stripped, out_dir, keystr, propstr, 209 bloaty_exe, f, config, TOTAL_SIZE_BYTES_KEY, lib_name, 210 MAGIC_SEPERATOR], 211 stdout=api.raw_io.output()) 212 if step_data and step_data.stdout: 213 sections = step_data.stdout.decode('utf-8').split(MAGIC_SEPERATOR) 214 result = api.step.active_result 215 logs = result.presentation.logs 216 # Skip section 0 because it's everything before first print, 217 # which is probably the empty string. 218 logs['bloaty_file_symbol_short'] = sections[1].split('\n') 219 logs['bloaty_file_symbol_full'] = sections[2].split('\n') 220 logs['bloaty_symbol_file_short'] = sections[3].split('\n') 221 logs['bloaty_symbol_file_full'] = sections[4].split('\n') 222 logs['perf_json'] = sections[5].split('\n') 223 224 add_binary_size_output_property(result, lib_name, ( 225 ast.literal_eval(sections[5]) 226 .get('results', {}) 227 .get(lib_name, {}) 228 .get(config, {}) 229 .get(TOTAL_SIZE_BYTES_KEY, {}))) 230 231 232# Get the size of skia in flutter and a few metrics from bloaty 233def analyze_wasm_file(api, checkout_root, out_dir, files): 234 (keystr, propstr) = keys_and_props(api) 235 bloaty_exe = api.path.start_dir.joinpath('bloaty', 'bloaty') 236 237 for f in files: 238 239 skia_dir = checkout_root.joinpath('skia') 240 with api.context(cwd=skia_dir): 241 script = skia_dir.joinpath('infra', 'bots', 'buildstats', 242 'buildstats_wasm.py') 243 step_data = api.run(api.step, 'Analyze wasm', 244 cmd=['python3', script, f, out_dir, keystr, propstr, bloaty_exe, 245 TOTAL_SIZE_BYTES_KEY, MAGIC_SEPERATOR], 246 stdout=api.raw_io.output()) 247 if step_data and step_data.stdout: 248 sections = step_data.stdout.decode('utf-8').split(MAGIC_SEPERATOR) 249 result = api.step.active_result 250 logs = result.presentation.logs 251 # Skip section 0 because it's everything before first print, 252 # which is probably the empty string. 253 logs['bloaty_symbol_short'] = sections[1].split('\n') 254 logs['bloaty_symbol_full'] = sections[2].split('\n') 255 logs['perf_json'] = sections[3].split('\n') 256 add_binary_size_output_property(result, api.path.basename(f), ( 257 ast.literal_eval(str(sections[3])) 258 .get('results', {}) 259 .get(api.path.basename(f), {}) 260 .get('default', {}) 261 .get(TOTAL_SIZE_BYTES_KEY, {}))) 262 263 264# make a zip file containing an HTML treemap of the files 265def make_treemap(api, checkout_root, out_dir, files): 266 for f in files: 267 env = {'DOCKER_CONFIG': '/home/chrome-bot/.docker'} 268 with api.env(env): 269 skia_dir = checkout_root.joinpath('skia') 270 with api.context(cwd=skia_dir): 271 script = skia_dir.joinpath('infra', 'bots', 'buildstats', 272 'make_treemap.py') 273 api.run(api.step, 'Make code size treemap %s' % f, 274 cmd=['python3', script, f, out_dir], 275 stdout=api.raw_io.output()) 276 277 278def GenTests(api): 279 builder = 'BuildStats-Debian10-EMCC-wasm-Release-PathKit' 280 yield ( 281 api.test('normal_bot') + 282 api.properties(buildername=builder, 283 repository='https://skia.googlesource.com/skia.git', 284 revision='abc123', 285 swarm_out_dir='[SWARM_OUT_DIR]', 286 path_config='kitchen') + 287 api.step_data('get swarming bot id', 288 stdout=api.raw_io.output('skia-bot-123')) + 289 api.step_data('get swarming task id', 290 stdout=api.raw_io.output('123456abc')) + 291 api.step_data('Analyze [START_DIR]/build/pathkit.js.mem', 292 stdout=api.raw_io.output(sample_web)) + 293 api.step_data('Analyze [START_DIR]/build/libskia.so', 294 stdout=api.raw_io.output(sample_cpp)) + 295 api.step_data('Analyze wasm', 296 stdout=api.raw_io.output(sample_wasm)) + 297 api.step_data('Analyze flutter', 298 stdout=api.raw_io.output(sample_flutter)) 299 ) 300 301 yield ( 302 api.test('trybot') + 303 api.properties(buildername=builder, 304 repository='https://skia.googlesource.com/skia.git', 305 revision='abc123', 306 swarm_out_dir='[SWARM_OUT_DIR]', 307 patch_repo='https://skia.googlesource.com/skia.git', 308 path_config='kitchen') + 309 api.step_data('get swarming bot id', 310 stdout=api.raw_io.output('skia-bot-123')) + 311 api.step_data('get swarming task id', 312 stdout=api.raw_io.output('123456abc')) + 313 api.properties(patch_storage='gerrit') + 314 api.properties.tryserver( 315 buildername=builder, 316 gerrit_project='skia', 317 gerrit_url='https://skia-review.googlesource.com/', 318 ) + 319 api.step_data('Analyze [START_DIR]/build/pathkit.js.mem', 320 stdout=api.raw_io.output(sample_web)) + 321 api.step_data('Analyze [START_DIR]/build/libskia.so', 322 stdout=api.raw_io.output(sample_cpp)) + 323 api.step_data('Analyze wasm', 324 stdout=api.raw_io.output(sample_wasm)) + 325 api.step_data('Analyze flutter', 326 stdout=api.raw_io.output(sample_flutter)) 327 ) 328 329sample_web = """ 330Report A 331 Total size: 50 bytes 332#$%^&* 333{ 334 "some": "json", 335 "results": { 336 "pathkit.js.mem": { 337 "default": { 338 "total_size_bytes": 7391117, 339 "gzip_size_bytes": 2884841 340 } 341 } 342 } 343} 344""" 345 346sample_cpp = """ 347#$%^&* 348Report A 349 Total size: 50 bytes 350#$%^&* 351{ 352 "some": "json", 353 "results": { 354 "libskia.so": { 355 "default": { 356 "total_size_bytes": 7391117, 357 "gzip_size_bytes": 2884841 358 } 359 } 360 } 361} 362""" 363 364sample_wasm = """ 365#$%^&* 366Report A 367 Total size: 50 bytes 368#$%^&* 369Report B 370 Total size: 60 bytes 371#$%^&* 372{ 373 "some": "json", 374 "results": { 375 "pathkit.wasm": { 376 "default": { 377 "total_size_bytes": 7391117, 378 "gzip_size_bytes": 2884841 379 } 380 } 381 } 382} 383""" 384 385sample_flutter = """ 386#$%^&* 387Report A 388 Total size: 50 bytes 389#$%^&* 390Report B 391 Total size: 60 bytes 392#$%^&* 393Report C 394 Total size: 70 bytes 395#$%^&* 396Report D 397 Total size: 80 bytes 398#$%^&* 399{ 400 "some": "json", 401 "results": { 402 "libflutter.so": { 403 "skia_in_flutter": { 404 "total_size_bytes": 1256676 405 } 406 } 407 } 408} 409""" 410