1# This import depends on the automake rule protoc_middleman, please make sure 2# protoc_middleman has been built before run this file. 3import argparse 4import json 5import re 6import os.path 7# BEGIN OPENSOURCE 8import sys 9sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir)) 10# END OPENSOURCE 11import tmp.benchmarks_pb2 as benchmarks_pb2 12 13__file_size_map = {} 14 15def __get_data_size(filename): 16 if filename[0] != '/': 17 filename = os.path.dirname(os.path.abspath(__file__)) + "/../" + filename 18 if filename in __file_size_map: 19 return __file_size_map[filename] 20 benchmark_dataset = benchmarks_pb2.BenchmarkDataset() 21 benchmark_dataset.ParseFromString( 22 open(filename, "rb").read()) 23 size = 0 24 count = 0 25 for payload in benchmark_dataset.payload: 26 size += len(payload) 27 count += 1 28 __file_size_map[filename] = (size, 1.0 * size / count) 29 return size, 1.0 * size / count 30 31 32def __extract_file_name(file_name): 33 name_list = re.split(r"[/\.]", file_name) 34 short_file_name = "" 35 for name in name_list: 36 if name[:14] == "google_message": 37 short_file_name = name 38 return short_file_name 39 40 41__results = [] 42 43 44# CPP results example: 45# [ 46# "benchmarks": [ 47# { 48# "bytes_per_second": int, 49# "cpu_time_ns": double, 50# "iterations": int, 51# "name: string, 52# "real_time_ns: double, 53# ... 54# }, 55# ... 56# ], 57# ... 58# ] 59def __parse_cpp_result(filename): 60 if filename == "": 61 return 62 if filename[0] != '/': 63 filename = os.path.dirname(os.path.abspath(__file__)) + '/' + filename 64 with open(filename, encoding="utf-8") as f: 65 results = json.loads(f.read()) 66 for benchmark in results["benchmarks"]: 67 data_filename = "".join( 68 re.split("(_parse_|_serialize)", benchmark["name"])[0]) 69 behavior = benchmark["name"][len(data_filename) + 1:] 70 if data_filename[:2] == "BM": 71 data_filename = data_filename[3:] 72 __results.append({ 73 "language": "cpp", 74 "dataFilename": data_filename, 75 "behavior": behavior, 76 "throughput": benchmark["bytes_per_second"] / 2.0 ** 20 77 }) 78 79 80# Synthetic benchmark results example: 81# [ 82# "benchmarks": [ 83# { 84# "cpu_time_ns": double, 85# "iterations": int, 86# "name: string, 87# "real_time_ns: double, 88# ... 89# }, 90# ... 91# ], 92# ... 93# ] 94def __parse_synthetic_result(filename): 95 if filename == "": 96 return 97 if filename[0] != "/": 98 filename = os.path.dirname(os.path.abspath(__file__)) + "/" + filename 99 with open(filename, encoding="utf-8") as f: 100 results = json.loads(f.read()) 101 for benchmark in results["benchmarks"]: 102 __results.append({ 103 "language": "cpp", 104 "dataFilename": "", 105 "behavior": "synthetic", 106 "throughput": 10.0**9 / benchmark["cpu_time_ns"] 107 }) 108 109 110# Python results example: 111# [ 112# [ 113# { 114# "filename": string, 115# "benchmarks": { 116# behavior: results, 117# ... 118# }, 119# }, 120# ... 121# ], #pure-python 122# ... 123# ] 124def __parse_python_result(filename): 125 if filename == "": 126 return 127 if filename[0] != '/': 128 filename = os.path.dirname(os.path.abspath(__file__)) + '/' + filename 129 with open(filename, encoding="utf-8") as f: 130 results_list = json.loads(f.read()) 131 for results in results_list: 132 for result in results: 133 _, avg_size = __get_data_size(result["filename"]) 134 for behavior in result["benchmarks"]: 135 __results.append({ 136 "language": "python", 137 "dataFilename": __extract_file_name(result["filename"]), 138 "behavior": behavior, 139 "throughput": result["benchmarks"][behavior] 140 }) 141 142 143# Java results example: 144# [ 145# { 146# "id": string, 147# "instrumentSpec": {...}, 148# "measurements": [ 149# { 150# "weight": float, 151# "value": { 152# "magnitude": float, 153# "unit": string 154# }, 155# ... 156# }, 157# ... 158# ], 159# "run": {...}, 160# "scenario": { 161# "benchmarkSpec": { 162# "methodName": string, 163# "parameters": { 164# defined parameters in the benchmark: parameters value 165# }, 166# ... 167# }, 168# ... 169# } 170# 171# }, 172# ... 173# ] 174def __parse_java_result(filename): 175 if filename == "": 176 return 177 if filename[0] != '/': 178 filename = os.path.dirname(os.path.abspath(__file__)) + '/' + filename 179 with open(filename, encoding="utf-8") as f: 180 results = json.loads(f.read()) 181 for result in results: 182 total_weight = 0 183 total_value = 0 184 for measurement in result["measurements"]: 185 total_weight += measurement["weight"] 186 total_value += measurement["value"]["magnitude"] 187 avg_time = total_value * 1.0 / total_weight 188 total_size, _ = __get_data_size( 189 result["scenario"]["benchmarkSpec"]["parameters"]["dataFile"]) 190 __results.append({ 191 "language": "java", 192 "throughput": total_size / avg_time * 1e9 / 2 ** 20, 193 "behavior": result["scenario"]["benchmarkSpec"]["methodName"], 194 "dataFilename": __extract_file_name( 195 result["scenario"]["benchmarkSpec"]["parameters"]["dataFile"]) 196 }) 197 198 199# Go benchmark results: 200# 201# goos: linux 202# goarch: amd64 203# Benchmark/.././datasets/google_message2/dataset.google_message2.pb/Unmarshal-12 3000 705784 ns/op 204# Benchmark/.././datasets/google_message2/dataset.google_message2.pb/Marshal-12 2000 634648 ns/op 205# Benchmark/.././datasets/google_message2/dataset.google_message2.pb/Size-12 5000 244174 ns/op 206# Benchmark/.././datasets/google_message2/dataset.google_message2.pb/Clone-12 300 4120954 ns/op 207# Benchmark/.././datasets/google_message2/dataset.google_message2.pb/Merge-12 300 4108632 ns/op 208# PASS 209# ok _/usr/local/google/home/yilunchong/mygit/protobuf/benchmarks 124.173s 210def __parse_go_result(filename): 211 if filename == "": 212 return 213 if filename[0] != '/': 214 filename = os.path.dirname(os.path.abspath(__file__)) + '/' + filename 215 with open(filename, encoding="utf-8") as f: 216 for line in f: 217 result_list = re.split(r"[\ \t]+", line) 218 if result_list[0][:9] != "Benchmark": 219 continue 220 first_slash_index = result_list[0].find('/') 221 last_slash_index = result_list[0].rfind('/') 222 full_filename = result_list[0][first_slash_index+1:last_slash_index] 223 total_bytes, _ = __get_data_size(full_filename) 224 behavior_with_suffix = result_list[0][last_slash_index+1:] 225 last_dash = behavior_with_suffix.rfind("-") 226 if last_dash == -1: 227 behavior = behavior_with_suffix 228 else: 229 behavior = behavior_with_suffix[:last_dash] 230 __results.append({ 231 "dataFilename": __extract_file_name(full_filename), 232 "throughput": total_bytes / float(result_list[2]) * 1e9 / 2 ** 20, 233 "behavior": behavior, 234 "language": "go" 235 }) 236 237 238# Self built json results example: 239# 240# [ 241# { 242# "filename": string, 243# "benchmarks": { 244# behavior: results, 245# ... 246# }, 247# }, 248# ... 249# ] 250def __parse_custom_result(filename, language): 251 if filename == "": 252 return 253 if filename[0] != '/': 254 filename = os.path.dirname(os.path.abspath(__file__)) + '/' + filename 255 with open(filename, encoding="utf-8") as f: 256 results = json.loads(f.read()) 257 for result in results: 258 _, avg_size = __get_data_size(result["filename"]) 259 for behavior in result["benchmarks"]: 260 __results.append({ 261 "language": language, 262 "dataFilename": __extract_file_name(result["filename"]), 263 "behavior": behavior, 264 "throughput": result["benchmarks"][behavior] 265 }) 266 267 268def __parse_js_result(filename, language): 269 return __parse_custom_result(filename, language) 270 271def __parse_php_result(filename, language): 272 return __parse_custom_result(filename, language) 273 274 275def get_result_from_file(cpp_file="", 276 java_file="", 277 python_file="", 278 go_file="", 279 synthetic_file="", 280 node_file="", 281 php_c_file="", 282 php_file=""): 283 results = {} 284 if cpp_file != "": 285 __parse_cpp_result(cpp_file) 286 if java_file != "": 287 __parse_java_result(java_file) 288 if python_file != "": 289 __parse_python_result(python_file) 290 if go_file != "": 291 __parse_go_result(go_file) 292 if synthetic_file != "": 293 __parse_synthetic_result(synthetic_file) 294 if node_file != "": 295 __parse_js_result(node_file, "node") 296 if php_file != "": 297 __parse_php_result(php_file, "php") 298 if php_c_file != "": 299 __parse_php_result(php_c_file, "php") 300 301 return __results 302 303 304if __name__ == "__main__": 305 parser = argparse.ArgumentParser() 306 parser.add_argument( 307 "-cpp", 308 "--cpp_input_file", 309 help="The CPP benchmark result file's name", 310 default="") 311 parser.add_argument( 312 "-java", 313 "--java_input_file", 314 help="The Java benchmark result file's name", 315 default="") 316 parser.add_argument( 317 "-python", 318 "--python_input_file", 319 help="The Python benchmark result file's name", 320 default="") 321 parser.add_argument( 322 "-go", 323 "--go_input_file", 324 help="The golang benchmark result file's name", 325 default="") 326 parser.add_argument( 327 "-node", 328 "--node_input_file", 329 help="The node.js benchmark result file's name", 330 default="") 331 parser.add_argument( 332 "-php", 333 "--php_input_file", 334 help="The pure php benchmark result file's name", 335 default="") 336 parser.add_argument( 337 "-php_c", 338 "--php_c_input_file", 339 help="The php with c ext benchmark result file's name", 340 default="") 341 args = parser.parse_args() 342 343 results = get_result_from_file( 344 cpp_file=args.cpp_input_file, 345 java_file=args.java_input_file, 346 python_file=args.python_input_file, 347 go_file=args.go_input_file, 348 node_file=args.node_input_file, 349 php_file=args.php_input_file, 350 php_c_file=args.php_c_input_file, 351 ) 352 print(json.dumps(results, indent=2)) 353