1*9e94795aSAndroid Build Coastguard Worker#!/usr/bin/env python3 2*9e94795aSAndroid Build Coastguard Worker# Copyright (C) 2023 The Android Open Source Project 3*9e94795aSAndroid Build Coastguard Worker# 4*9e94795aSAndroid Build Coastguard Worker# Licensed under the Apache License, Version 2.0 (the "License"); 5*9e94795aSAndroid Build Coastguard Worker# you may not use this file except in compliance with the License. 6*9e94795aSAndroid Build Coastguard Worker# You may obtain a copy of the License at 7*9e94795aSAndroid Build Coastguard Worker# 8*9e94795aSAndroid Build Coastguard Worker# http://www.apache.org/licenses/LICENSE-2.0 9*9e94795aSAndroid Build Coastguard Worker# 10*9e94795aSAndroid Build Coastguard Worker# Unless required by applicable law or agreed to in writing, software 11*9e94795aSAndroid Build Coastguard Worker# distributed under the License is distributed on an "AS IS" BASIS, 12*9e94795aSAndroid Build Coastguard Worker# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13*9e94795aSAndroid Build Coastguard Worker# See the License for the specific language governing permissions and 14*9e94795aSAndroid Build Coastguard Worker# limitations under the License. 15*9e94795aSAndroid Build Coastguard Worker 16*9e94795aSAndroid Build Coastguard Workerimport sys 17*9e94795aSAndroid Build Coastguard Workerif __name__ == "__main__": 18*9e94795aSAndroid Build Coastguard Worker sys.dont_write_bytecode = True 19*9e94795aSAndroid Build Coastguard Worker 20*9e94795aSAndroid Build Coastguard Workerimport argparse 21*9e94795aSAndroid Build Coastguard Workerimport dataclasses 22*9e94795aSAndroid Build Coastguard Workerimport datetime 23*9e94795aSAndroid Build Coastguard Workerimport json 24*9e94795aSAndroid Build Coastguard Workerimport os 25*9e94795aSAndroid Build Coastguard Workerimport pathlib 26*9e94795aSAndroid Build Coastguard Workerimport statistics 27*9e94795aSAndroid Build Coastguard Workerimport zoneinfo 28*9e94795aSAndroid Build Coastguard Workerimport csv 29*9e94795aSAndroid Build Coastguard Worker 30*9e94795aSAndroid Build Coastguard Workerimport pretty 31*9e94795aSAndroid Build Coastguard Workerimport utils 32*9e94795aSAndroid Build Coastguard Worker 33*9e94795aSAndroid Build Coastguard Worker# TODO: 34*9e94795aSAndroid Build Coastguard Worker# - Flag if the last postroll build was more than 15 seconds or something. That's 35*9e94795aSAndroid Build Coastguard Worker# an indicator that something is amiss. 36*9e94795aSAndroid Build Coastguard Worker# - Add a mode to print all of the values for multi-iteration runs 37*9e94795aSAndroid Build Coastguard Worker# - Add a flag to reorder the tags 38*9e94795aSAndroid Build Coastguard Worker# - Add a flag to reorder the headers in order to show grouping more clearly. 39*9e94795aSAndroid Build Coastguard Worker 40*9e94795aSAndroid Build Coastguard Worker 41*9e94795aSAndroid Build Coastguard Workerdef FindSummaries(args): 42*9e94795aSAndroid Build Coastguard Worker def find_summaries(directory): 43*9e94795aSAndroid Build Coastguard Worker return [str(p.resolve()) for p in pathlib.Path(directory).glob("**/summary.json")] 44*9e94795aSAndroid Build Coastguard Worker if not args: 45*9e94795aSAndroid Build Coastguard Worker # If they didn't give an argument, use the default dir 46*9e94795aSAndroid Build Coastguard Worker root = utils.get_root() 47*9e94795aSAndroid Build Coastguard Worker if not root: 48*9e94795aSAndroid Build Coastguard Worker return [] 49*9e94795aSAndroid Build Coastguard Worker return find_summaries(root.joinpath("..", utils.DEFAULT_REPORT_DIR)) 50*9e94795aSAndroid Build Coastguard Worker results = list() 51*9e94795aSAndroid Build Coastguard Worker for arg in args: 52*9e94795aSAndroid Build Coastguard Worker if os.path.isfile(arg): 53*9e94795aSAndroid Build Coastguard Worker # If it's a file add that 54*9e94795aSAndroid Build Coastguard Worker results.append(arg) 55*9e94795aSAndroid Build Coastguard Worker elif os.path.isdir(arg): 56*9e94795aSAndroid Build Coastguard Worker # If it's a directory, find all of the files there 57*9e94795aSAndroid Build Coastguard Worker results += find_summaries(arg) 58*9e94795aSAndroid Build Coastguard Worker else: 59*9e94795aSAndroid Build Coastguard Worker sys.stderr.write(f"Invalid summary argument: {arg}\n") 60*9e94795aSAndroid Build Coastguard Worker sys.exit(1) 61*9e94795aSAndroid Build Coastguard Worker return sorted(list(results)) 62*9e94795aSAndroid Build Coastguard Worker 63*9e94795aSAndroid Build Coastguard Worker 64*9e94795aSAndroid Build Coastguard Workerdef LoadSummary(filename): 65*9e94795aSAndroid Build Coastguard Worker with open(filename) as f: 66*9e94795aSAndroid Build Coastguard Worker return json.load(f) 67*9e94795aSAndroid Build Coastguard Worker 68*9e94795aSAndroid Build Coastguard Worker# Columns: 69*9e94795aSAndroid Build Coastguard Worker# Date 70*9e94795aSAndroid Build Coastguard Worker# Branch 71*9e94795aSAndroid Build Coastguard Worker# Tag 72*9e94795aSAndroid Build Coastguard Worker# -- 73*9e94795aSAndroid Build Coastguard Worker# Lunch 74*9e94795aSAndroid Build Coastguard Worker# Rows: 75*9e94795aSAndroid Build Coastguard Worker# Benchmark 76*9e94795aSAndroid Build Coastguard Worker 77*9e94795aSAndroid Build Coastguard Workerdef lunch_str(d): 78*9e94795aSAndroid Build Coastguard Worker "Convert a lunch dict to a string" 79*9e94795aSAndroid Build Coastguard Worker return f"{d['TARGET_PRODUCT']}-{d['TARGET_RELEASE']}-{d['TARGET_BUILD_VARIANT']}" 80*9e94795aSAndroid Build Coastguard Worker 81*9e94795aSAndroid Build Coastguard Workerdef group_by(l, key): 82*9e94795aSAndroid Build Coastguard Worker "Return a list of tuples, grouped by key, sorted by key" 83*9e94795aSAndroid Build Coastguard Worker result = {} 84*9e94795aSAndroid Build Coastguard Worker for item in l: 85*9e94795aSAndroid Build Coastguard Worker result.setdefault(key(item), []).append(item) 86*9e94795aSAndroid Build Coastguard Worker return [(k, v) for k, v in result.items()] 87*9e94795aSAndroid Build Coastguard Worker 88*9e94795aSAndroid Build Coastguard Worker 89*9e94795aSAndroid Build Coastguard Workerclass Table: 90*9e94795aSAndroid Build Coastguard Worker def __init__(self, row_title, fixed_titles=[]): 91*9e94795aSAndroid Build Coastguard Worker self._data = {} 92*9e94795aSAndroid Build Coastguard Worker self._rows = [] 93*9e94795aSAndroid Build Coastguard Worker self._cols = [] 94*9e94795aSAndroid Build Coastguard Worker self._fixed_cols = {} 95*9e94795aSAndroid Build Coastguard Worker self._titles = [row_title] + fixed_titles 96*9e94795aSAndroid Build Coastguard Worker 97*9e94795aSAndroid Build Coastguard Worker def Set(self, column_key, row_key, data): 98*9e94795aSAndroid Build Coastguard Worker self._data[(column_key, row_key)] = data 99*9e94795aSAndroid Build Coastguard Worker if not column_key in self._cols: 100*9e94795aSAndroid Build Coastguard Worker self._cols.append(column_key) 101*9e94795aSAndroid Build Coastguard Worker if not row_key in self._rows: 102*9e94795aSAndroid Build Coastguard Worker self._rows.append(row_key) 103*9e94795aSAndroid Build Coastguard Worker 104*9e94795aSAndroid Build Coastguard Worker def SetFixedCol(self, row_key, columns): 105*9e94795aSAndroid Build Coastguard Worker self._fixed_cols[row_key] = columns 106*9e94795aSAndroid Build Coastguard Worker 107*9e94795aSAndroid Build Coastguard Worker def Write(self, out, fmt): 108*9e94795aSAndroid Build Coastguard Worker table = [] 109*9e94795aSAndroid Build Coastguard Worker # Expand the column items 110*9e94795aSAndroid Build Coastguard Worker for row in zip(*self._cols): 111*9e94795aSAndroid Build Coastguard Worker if row.count(row[0]) == len(row): 112*9e94795aSAndroid Build Coastguard Worker continue 113*9e94795aSAndroid Build Coastguard Worker table.append([""] * len(self._titles) + [col for col in row]) 114*9e94795aSAndroid Build Coastguard Worker if table: 115*9e94795aSAndroid Build Coastguard Worker # Update the last row of the header with title and add separator 116*9e94795aSAndroid Build Coastguard Worker for i in range(len(self._titles)): 117*9e94795aSAndroid Build Coastguard Worker table[len(table)-1][i] = self._titles[i] 118*9e94795aSAndroid Build Coastguard Worker if fmt == "table": 119*9e94795aSAndroid Build Coastguard Worker table.append(pretty.SEPARATOR) 120*9e94795aSAndroid Build Coastguard Worker # Populate the data 121*9e94795aSAndroid Build Coastguard Worker for row in self._rows: 122*9e94795aSAndroid Build Coastguard Worker table.append([str(row)] 123*9e94795aSAndroid Build Coastguard Worker + self._fixed_cols[row] 124*9e94795aSAndroid Build Coastguard Worker + [str(self._data.get((col, row), "")) for col in self._cols]) 125*9e94795aSAndroid Build Coastguard Worker if fmt == "csv": 126*9e94795aSAndroid Build Coastguard Worker csv.writer(sys.stdout, quoting=csv.QUOTE_MINIMAL).writerows(table) 127*9e94795aSAndroid Build Coastguard Worker else: 128*9e94795aSAndroid Build Coastguard Worker out.write(pretty.FormatTable(table, alignments="LL")) 129*9e94795aSAndroid Build Coastguard Worker 130*9e94795aSAndroid Build Coastguard Worker 131*9e94795aSAndroid Build Coastguard Workerdef format_duration_sec(ns, fmt_sec): 132*9e94795aSAndroid Build Coastguard Worker "Format a duration in ns to second precision" 133*9e94795aSAndroid Build Coastguard Worker sec = round(ns / 1000000000) 134*9e94795aSAndroid Build Coastguard Worker if fmt_sec: 135*9e94795aSAndroid Build Coastguard Worker return f"{sec}" 136*9e94795aSAndroid Build Coastguard Worker else: 137*9e94795aSAndroid Build Coastguard Worker h, sec = divmod(sec, 60*60) 138*9e94795aSAndroid Build Coastguard Worker m, sec = divmod(sec, 60) 139*9e94795aSAndroid Build Coastguard Worker result = "" 140*9e94795aSAndroid Build Coastguard Worker if h > 0: 141*9e94795aSAndroid Build Coastguard Worker result += f"{h:2d}h " 142*9e94795aSAndroid Build Coastguard Worker if h > 0 or m > 0: 143*9e94795aSAndroid Build Coastguard Worker result += f"{m:2d}m " 144*9e94795aSAndroid Build Coastguard Worker return result + f"{sec:2d}s" 145*9e94795aSAndroid Build Coastguard Worker 146*9e94795aSAndroid Build Coastguard Worker 147*9e94795aSAndroid Build Coastguard Workerdef main(argv): 148*9e94795aSAndroid Build Coastguard Worker parser = argparse.ArgumentParser( 149*9e94795aSAndroid Build Coastguard Worker prog="format_benchmarks", 150*9e94795aSAndroid Build Coastguard Worker allow_abbrev=False, # Don't let people write unsupportable scripts. 151*9e94795aSAndroid Build Coastguard Worker description="Print analysis tables for benchmarks") 152*9e94795aSAndroid Build Coastguard Worker 153*9e94795aSAndroid Build Coastguard Worker parser.add_argument("--csv", action="store_true", 154*9e94795aSAndroid Build Coastguard Worker help="Print in CSV instead of table.") 155*9e94795aSAndroid Build Coastguard Worker 156*9e94795aSAndroid Build Coastguard Worker parser.add_argument("--sec", action="store_true", 157*9e94795aSAndroid Build Coastguard Worker help="Print in seconds instead of minutes and seconds") 158*9e94795aSAndroid Build Coastguard Worker 159*9e94795aSAndroid Build Coastguard Worker parser.add_argument("--tags", nargs="*", 160*9e94795aSAndroid Build Coastguard Worker help="The tags to print, in order.") 161*9e94795aSAndroid Build Coastguard Worker 162*9e94795aSAndroid Build Coastguard Worker parser.add_argument("summaries", nargs="*", 163*9e94795aSAndroid Build Coastguard Worker help="A summary.json file or a directory in which to look for summaries.") 164*9e94795aSAndroid Build Coastguard Worker 165*9e94795aSAndroid Build Coastguard Worker args = parser.parse_args() 166*9e94795aSAndroid Build Coastguard Worker 167*9e94795aSAndroid Build Coastguard Worker # Load the summaries 168*9e94795aSAndroid Build Coastguard Worker summaries = [(s, LoadSummary(s)) for s in FindSummaries(args.summaries)] 169*9e94795aSAndroid Build Coastguard Worker 170*9e94795aSAndroid Build Coastguard Worker # Convert to MTV time 171*9e94795aSAndroid Build Coastguard Worker for filename, s in summaries: 172*9e94795aSAndroid Build Coastguard Worker dt = datetime.datetime.fromisoformat(s["start_time"]) 173*9e94795aSAndroid Build Coastguard Worker dt = dt.astimezone(zoneinfo.ZoneInfo("America/Los_Angeles")) 174*9e94795aSAndroid Build Coastguard Worker s["datetime"] = dt 175*9e94795aSAndroid Build Coastguard Worker s["date"] = datetime.date(dt.year, dt.month, dt.day) 176*9e94795aSAndroid Build Coastguard Worker 177*9e94795aSAndroid Build Coastguard Worker # Filter out tags we don't want 178*9e94795aSAndroid Build Coastguard Worker if args.tags: 179*9e94795aSAndroid Build Coastguard Worker summaries = [(f, s) for f, s in summaries if s.get("tag", "") in args.tags] 180*9e94795aSAndroid Build Coastguard Worker 181*9e94795aSAndroid Build Coastguard Worker # If they supplied tags, sort in that order, otherwise sort by tag 182*9e94795aSAndroid Build Coastguard Worker if args.tags: 183*9e94795aSAndroid Build Coastguard Worker tagsort = lambda tag: args.tags.index(tag) 184*9e94795aSAndroid Build Coastguard Worker else: 185*9e94795aSAndroid Build Coastguard Worker tagsort = lambda tag: tag 186*9e94795aSAndroid Build Coastguard Worker 187*9e94795aSAndroid Build Coastguard Worker # Sort the summaries 188*9e94795aSAndroid Build Coastguard Worker summaries.sort(key=lambda s: (s[1]["date"], s[1]["branch"], tagsort(s[1]["tag"]))) 189*9e94795aSAndroid Build Coastguard Worker 190*9e94795aSAndroid Build Coastguard Worker # group the benchmarks by column and iteration 191*9e94795aSAndroid Build Coastguard Worker def bm_key(b): 192*9e94795aSAndroid Build Coastguard Worker return ( 193*9e94795aSAndroid Build Coastguard Worker lunch_str(b["lunch"]), 194*9e94795aSAndroid Build Coastguard Worker ) 195*9e94795aSAndroid Build Coastguard Worker for filename, summary in summaries: 196*9e94795aSAndroid Build Coastguard Worker summary["columns"] = [(key, group_by(bms, lambda b: b["id"])) for key, bms 197*9e94795aSAndroid Build Coastguard Worker in group_by(summary["benchmarks"], bm_key)] 198*9e94795aSAndroid Build Coastguard Worker 199*9e94795aSAndroid Build Coastguard Worker # Build the table 200*9e94795aSAndroid Build Coastguard Worker table = Table("Benchmark", ["Rebuild"]) 201*9e94795aSAndroid Build Coastguard Worker for filename, summary in summaries: 202*9e94795aSAndroid Build Coastguard Worker for key, column in summary["columns"]: 203*9e94795aSAndroid Build Coastguard Worker for id, cell in column: 204*9e94795aSAndroid Build Coastguard Worker duration_ns = statistics.median([b["duration_ns"] for b in cell]) 205*9e94795aSAndroid Build Coastguard Worker modules = cell[0]["modules"] 206*9e94795aSAndroid Build Coastguard Worker if not modules: 207*9e94795aSAndroid Build Coastguard Worker modules = ["---"] 208*9e94795aSAndroid Build Coastguard Worker table.SetFixedCol(cell[0]["title"], [" ".join(modules)]) 209*9e94795aSAndroid Build Coastguard Worker table.Set(tuple([summary["date"].strftime("%Y-%m-%d"), 210*9e94795aSAndroid Build Coastguard Worker summary["branch"], 211*9e94795aSAndroid Build Coastguard Worker summary["tag"]] 212*9e94795aSAndroid Build Coastguard Worker + list(key)), 213*9e94795aSAndroid Build Coastguard Worker cell[0]["title"], format_duration_sec(duration_ns, args.sec)) 214*9e94795aSAndroid Build Coastguard Worker 215*9e94795aSAndroid Build Coastguard Worker table.Write(sys.stdout, "csv" if args.csv else "table") 216*9e94795aSAndroid Build Coastguard Worker 217*9e94795aSAndroid Build Coastguard Workerif __name__ == "__main__": 218*9e94795aSAndroid Build Coastguard Worker main(sys.argv) 219*9e94795aSAndroid Build Coastguard Worker 220