1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3# Copyright 2016 The ChromiumOS Authors 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7"""Given a specially-formatted JSON object, generates results report(s). 8 9The JSON object should look like: 10{"data": BenchmarkData, "platforms": BenchmarkPlatforms} 11 12BenchmarkPlatforms is a [str], each of which names a platform the benchmark 13 was run on (e.g. peppy, shamu, ...). Note that the order of this list is 14 related with the order of items in BenchmarkData. 15 16BenchmarkData is a {str: [PlatformData]}. The str is the name of the benchmark, 17and a PlatformData is a set of data for a given platform. There must be one 18PlatformData for each benchmark, for each element in BenchmarkPlatforms. 19 20A PlatformData is a [{str: float}], where each str names a metric we recorded, 21and the float is the value for that metric. Each element is considered to be 22the metrics collected from an independent run of this benchmark. NOTE: Each 23PlatformData is expected to have a "retval" key, with the return value of 24the benchmark. If the benchmark is successful, said return value should be 0. 25Otherwise, this will break some of our JSON functionality. 26 27Putting it all together, a JSON object will end up looking like: 28 { "platforms": ["peppy", "peppy-new-crosstool"], 29 "data": { 30 "bench_draw_line": [ 31 [{"time (ms)": 1.321, "memory (mb)": 128.1, "retval": 0}, 32 {"time (ms)": 1.920, "memory (mb)": 128.4, "retval": 0}], 33 [{"time (ms)": 1.221, "memory (mb)": 124.3, "retval": 0}, 34 {"time (ms)": 1.423, "memory (mb)": 123.9, "retval": 0}] 35 ] 36 } 37 } 38 39Which says that we ran a benchmark on platforms named peppy, and 40 peppy-new-crosstool. 41We ran one benchmark, named bench_draw_line. 42It was run twice on each platform. 43Peppy's runs took 1.321ms and 1.920ms, while peppy-new-crosstool's took 1.221ms 44 and 1.423ms. None of the runs failed to complete. 45""" 46 47 48import argparse 49import functools 50import json 51import os 52import sys 53import traceback 54 55from results_report import BenchmarkResults 56from results_report import HTMLResultsReport 57from results_report import JSONResultsReport 58from results_report import TextResultsReport 59 60 61def CountBenchmarks(benchmark_runs): 62 """Counts the number of iterations for each benchmark in benchmark_runs.""" 63 64 # Example input for benchmark_runs: 65 # {"bench": [[run1, run2, run3], [run1, run2, run3, run4]]} 66 def _MaxLen(results): 67 return 0 if not results else max(len(r) for r in results) 68 69 return [ 70 (name, _MaxLen(results)) for name, results in benchmark_runs.items() 71 ] 72 73 74def CutResultsInPlace(results, max_keys=50, complain_on_update=True): 75 """Limits the given benchmark results to max_keys keys in-place. 76 77 This takes the `data` field from the benchmark input, and mutates each 78 benchmark run to contain `max_keys` elements (ignoring special elements, like 79 "retval"). At the moment, it just selects the first `max_keys` keyvals, 80 alphabetically. 81 82 If complain_on_update is true, this will print a message noting that a 83 truncation occurred. 84 85 This returns the `results` object that was passed in, for convenience. 86 87 e.g. 88 >>> benchmark_data = { 89 ... "bench_draw_line": [ 90 ... [{"time (ms)": 1.321, "memory (mb)": 128.1, "retval": 0}, 91 ... {"time (ms)": 1.920, "memory (mb)": 128.4, "retval": 0}], 92 ... [{"time (ms)": 1.221, "memory (mb)": 124.3, "retval": 0}, 93 ... {"time (ms)": 1.423, "memory (mb)": 123.9, "retval": 0}] 94 ... ] 95 ... } 96 >>> CutResultsInPlace(benchmark_data, max_keys=1, complain_on_update=False) 97 { 98 'bench_draw_line': [ 99 [{'memory (mb)': 128.1, 'retval': 0}, 100 {'memory (mb)': 128.4, 'retval': 0}], 101 [{'memory (mb)': 124.3, 'retval': 0}, 102 {'memory (mb)': 123.9, 'retval': 0}] 103 ] 104 } 105 """ 106 actually_updated = False 107 for bench_results in results.values(): 108 for platform_results in bench_results: 109 for i, result in enumerate(platform_results): 110 # Keep the keys that come earliest when sorted alphabetically. 111 # Forcing alphabetical order is arbitrary, but necessary; otherwise, 112 # the keyvals we'd emit would depend on our iteration order through a 113 # map. 114 removable_keys = sorted(k for k in result if k != "retval") 115 retained_keys = removable_keys[:max_keys] 116 platform_results[i] = {k: result[k] for k in retained_keys} 117 # retval needs to be passed through all of the time. 118 retval = result.get("retval") 119 if retval is not None: 120 platform_results[i]["retval"] = retval 121 actually_updated = actually_updated or len( 122 retained_keys 123 ) != len(removable_keys) 124 125 if actually_updated and complain_on_update: 126 print( 127 "Warning: Some benchmark keyvals have been truncated.", 128 file=sys.stderr, 129 ) 130 return results 131 132 133def _PositiveInt(s): 134 i = int(s) 135 if i < 0: 136 raise argparse.ArgumentTypeError("%d is not a positive integer." % (i,)) 137 return i 138 139 140def _AccumulateActions(args): 141 """Given program arguments, determines what actions we want to run. 142 143 Returns [(ResultsReportCtor, str)], where ResultsReportCtor can construct a 144 ResultsReport, and the str is the file extension for the given report. 145 """ 146 results = [] 147 # The order of these is arbitrary. 148 if args.json: 149 results.append((JSONResultsReport, "json")) 150 if args.text: 151 results.append((TextResultsReport, "txt")) 152 if args.email: 153 email_ctor = functools.partial(TextResultsReport, email=True) 154 results.append((email_ctor, "email")) 155 # We emit HTML if nothing else was specified. 156 if args.html or not results: 157 results.append((HTMLResultsReport, "html")) 158 return results 159 160 161# Note: get_contents is a function, because it may be expensive (generating some 162# HTML reports takes O(seconds) on my machine, depending on the size of the 163# input data). 164def WriteFile(output_prefix, extension, get_contents, overwrite, verbose): 165 """Writes `contents` to a file named "${output_prefix}.${extension}". 166 167 get_contents should be a zero-args function that returns a string (of the 168 contents to write). 169 If output_prefix == '-', this writes to stdout. 170 If overwrite is False, this will not overwrite files. 171 """ 172 if output_prefix == "-": 173 if verbose: 174 print("Writing %s report to stdout" % (extension,), file=sys.stderr) 175 sys.stdout.write(get_contents()) 176 return 177 178 file_name = "%s.%s" % (output_prefix, extension) 179 if not overwrite and os.path.exists(file_name): 180 raise IOError( 181 "Refusing to write %s -- it already exists" % (file_name,) 182 ) 183 184 with open(file_name, "w") as out_file: 185 if verbose: 186 print( 187 "Writing %s report to %s" % (extension, file_name), 188 file=sys.stderr, 189 ) 190 out_file.write(get_contents()) 191 192 193def RunActions(actions, benchmark_results, output_prefix, overwrite, verbose): 194 """Runs `actions`, returning True if all succeeded.""" 195 failed = False 196 197 report_ctor = None # Make the linter happy 198 for report_ctor, extension in actions: 199 try: 200 get_contents = lambda: report_ctor(benchmark_results).GetReport() 201 WriteFile( 202 output_prefix, extension, get_contents, overwrite, verbose 203 ) 204 except Exception: 205 # Complain and move along; we may have more actions that might complete 206 # successfully. 207 failed = True 208 traceback.print_exc() 209 return not failed 210 211 212def PickInputFile(input_name): 213 """Given program arguments, returns file to read for benchmark input.""" 214 return sys.stdin if input_name == "-" else open(input_name) 215 216 217def _NoPerfReport(_label_name, _benchmark_name, _benchmark_iteration): 218 return {} 219 220 221def _ParseArgs(argv): 222 parser = argparse.ArgumentParser( 223 description="Turns JSON into results " "report(s)." 224 ) 225 parser.add_argument( 226 "-v", 227 "--verbose", 228 action="store_true", 229 help="Be a tiny bit more verbose.", 230 ) 231 parser.add_argument( 232 "-f", 233 "--force", 234 action="store_true", 235 help="Overwrite existing results files.", 236 ) 237 parser.add_argument( 238 "-o", 239 "--output", 240 default="report", 241 type=str, 242 help="Prefix of the output filename (default: report). " 243 "- means stdout.", 244 ) 245 parser.add_argument( 246 "-i", 247 "--input", 248 required=True, 249 type=str, 250 help="Where to read the JSON from. - means stdin.", 251 ) 252 parser.add_argument( 253 "-l", 254 "--statistic-limit", 255 default=0, 256 type=_PositiveInt, 257 help="The maximum number of benchmark statistics to " 258 "display from a single run. 0 implies unlimited.", 259 ) 260 parser.add_argument( 261 "--json", action="store_true", help="Output a JSON report." 262 ) 263 parser.add_argument( 264 "--text", action="store_true", help="Output a text report." 265 ) 266 parser.add_argument( 267 "--email", 268 action="store_true", 269 help="Output a text report suitable for email.", 270 ) 271 parser.add_argument( 272 "--html", 273 action="store_true", 274 help="Output an HTML report (this is the default if no " 275 "other output format is specified).", 276 ) 277 return parser.parse_args(argv) 278 279 280def Main(argv): 281 args = _ParseArgs(argv) 282 with PickInputFile(args.input) as in_file: 283 raw_results = json.load(in_file) 284 285 platform_names = raw_results["platforms"] 286 results = raw_results["data"] 287 if args.statistic_limit: 288 results = CutResultsInPlace(results, max_keys=args.statistic_limit) 289 benches = CountBenchmarks(results) 290 # In crosperf, a label is essentially a platform+configuration. So, a name of 291 # a label and a name of a platform are equivalent for our purposes. 292 bench_results = BenchmarkResults( 293 label_names=platform_names, 294 benchmark_names_and_iterations=benches, 295 run_keyvals=results, 296 read_perf_report=_NoPerfReport, 297 ) 298 actions = _AccumulateActions(args) 299 ok = RunActions( 300 actions, bench_results, args.output, args.force, args.verbose 301 ) 302 return 0 if ok else 1 303 304 305if __name__ == "__main__": 306 sys.exit(Main(sys.argv[1:])) 307