xref: /aosp_15_r20/build/make/tools/perf/format_benchmarks (revision 9e94795a3d4ef5c1d47486f9a02bb378756cea8a)
1*9e94795aSAndroid Build Coastguard Worker#!/usr/bin/env python3
2*9e94795aSAndroid Build Coastguard Worker# Copyright (C) 2023 The Android Open Source Project
3*9e94795aSAndroid Build Coastguard Worker#
4*9e94795aSAndroid Build Coastguard Worker# Licensed under the Apache License, Version 2.0 (the "License");
5*9e94795aSAndroid Build Coastguard Worker# you may not use this file except in compliance with the License.
6*9e94795aSAndroid Build Coastguard Worker# You may obtain a copy of the License at
7*9e94795aSAndroid Build Coastguard Worker#
8*9e94795aSAndroid Build Coastguard Worker#      http://www.apache.org/licenses/LICENSE-2.0
9*9e94795aSAndroid Build Coastguard Worker#
10*9e94795aSAndroid Build Coastguard Worker# Unless required by applicable law or agreed to in writing, software
11*9e94795aSAndroid Build Coastguard Worker# distributed under the License is distributed on an "AS IS" BASIS,
12*9e94795aSAndroid Build Coastguard Worker# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13*9e94795aSAndroid Build Coastguard Worker# See the License for the specific language governing permissions and
14*9e94795aSAndroid Build Coastguard Worker# limitations under the License.
15*9e94795aSAndroid Build Coastguard Worker
16*9e94795aSAndroid Build Coastguard Workerimport sys
17*9e94795aSAndroid Build Coastguard Workerif __name__ == "__main__":
18*9e94795aSAndroid Build Coastguard Worker    sys.dont_write_bytecode = True
19*9e94795aSAndroid Build Coastguard Worker
20*9e94795aSAndroid Build Coastguard Workerimport argparse
21*9e94795aSAndroid Build Coastguard Workerimport dataclasses
22*9e94795aSAndroid Build Coastguard Workerimport datetime
23*9e94795aSAndroid Build Coastguard Workerimport json
24*9e94795aSAndroid Build Coastguard Workerimport os
25*9e94795aSAndroid Build Coastguard Workerimport pathlib
26*9e94795aSAndroid Build Coastguard Workerimport statistics
27*9e94795aSAndroid Build Coastguard Workerimport zoneinfo
28*9e94795aSAndroid Build Coastguard Workerimport csv
29*9e94795aSAndroid Build Coastguard Worker
30*9e94795aSAndroid Build Coastguard Workerimport pretty
31*9e94795aSAndroid Build Coastguard Workerimport utils
32*9e94795aSAndroid Build Coastguard Worker
33*9e94795aSAndroid Build Coastguard Worker# TODO:
34*9e94795aSAndroid Build Coastguard Worker# - Flag if the last postroll build was more than 15 seconds or something. That's
35*9e94795aSAndroid Build Coastguard Worker#   an indicator that something is amiss.
36*9e94795aSAndroid Build Coastguard Worker# - Add a mode to print all of the values for multi-iteration runs
37*9e94795aSAndroid Build Coastguard Worker# - Add a flag to reorder the tags
38*9e94795aSAndroid Build Coastguard Worker# - Add a flag to reorder the headers in order to show grouping more clearly.
39*9e94795aSAndroid Build Coastguard Worker
40*9e94795aSAndroid Build Coastguard Worker
41*9e94795aSAndroid Build Coastguard Workerdef FindSummaries(args):
42*9e94795aSAndroid Build Coastguard Worker    def find_summaries(directory):
43*9e94795aSAndroid Build Coastguard Worker        return [str(p.resolve()) for p in pathlib.Path(directory).glob("**/summary.json")]
44*9e94795aSAndroid Build Coastguard Worker    if not args:
45*9e94795aSAndroid Build Coastguard Worker        # If they didn't give an argument, use the default dir
46*9e94795aSAndroid Build Coastguard Worker        root = utils.get_root()
47*9e94795aSAndroid Build Coastguard Worker        if not root:
48*9e94795aSAndroid Build Coastguard Worker            return []
49*9e94795aSAndroid Build Coastguard Worker        return find_summaries(root.joinpath("..", utils.DEFAULT_REPORT_DIR))
50*9e94795aSAndroid Build Coastguard Worker    results = list()
51*9e94795aSAndroid Build Coastguard Worker    for arg in args:
52*9e94795aSAndroid Build Coastguard Worker        if os.path.isfile(arg):
53*9e94795aSAndroid Build Coastguard Worker            # If it's a file add that
54*9e94795aSAndroid Build Coastguard Worker            results.append(arg)
55*9e94795aSAndroid Build Coastguard Worker        elif os.path.isdir(arg):
56*9e94795aSAndroid Build Coastguard Worker            # If it's a directory, find all of the files there
57*9e94795aSAndroid Build Coastguard Worker            results += find_summaries(arg)
58*9e94795aSAndroid Build Coastguard Worker        else:
59*9e94795aSAndroid Build Coastguard Worker            sys.stderr.write(f"Invalid summary argument: {arg}\n")
60*9e94795aSAndroid Build Coastguard Worker            sys.exit(1)
61*9e94795aSAndroid Build Coastguard Worker    return sorted(list(results))
62*9e94795aSAndroid Build Coastguard Worker
63*9e94795aSAndroid Build Coastguard Worker
64*9e94795aSAndroid Build Coastguard Workerdef LoadSummary(filename):
65*9e94795aSAndroid Build Coastguard Worker    with open(filename) as f:
66*9e94795aSAndroid Build Coastguard Worker        return json.load(f)
67*9e94795aSAndroid Build Coastguard Worker
68*9e94795aSAndroid Build Coastguard Worker# Columns:
69*9e94795aSAndroid Build Coastguard Worker#   Date
70*9e94795aSAndroid Build Coastguard Worker#   Branch
71*9e94795aSAndroid Build Coastguard Worker#   Tag
72*9e94795aSAndroid Build Coastguard Worker#   --
73*9e94795aSAndroid Build Coastguard Worker#   Lunch
74*9e94795aSAndroid Build Coastguard Worker# Rows:
75*9e94795aSAndroid Build Coastguard Worker#   Benchmark
76*9e94795aSAndroid Build Coastguard Worker
77*9e94795aSAndroid Build Coastguard Workerdef lunch_str(d):
78*9e94795aSAndroid Build Coastguard Worker    "Convert a lunch dict to a string"
79*9e94795aSAndroid Build Coastguard Worker    return f"{d['TARGET_PRODUCT']}-{d['TARGET_RELEASE']}-{d['TARGET_BUILD_VARIANT']}"
80*9e94795aSAndroid Build Coastguard Worker
81*9e94795aSAndroid Build Coastguard Workerdef group_by(l, key):
82*9e94795aSAndroid Build Coastguard Worker    "Return a list of tuples, grouped by key, sorted by key"
83*9e94795aSAndroid Build Coastguard Worker    result = {}
84*9e94795aSAndroid Build Coastguard Worker    for item in l:
85*9e94795aSAndroid Build Coastguard Worker        result.setdefault(key(item), []).append(item)
86*9e94795aSAndroid Build Coastguard Worker    return [(k, v) for k, v in result.items()]
87*9e94795aSAndroid Build Coastguard Worker
88*9e94795aSAndroid Build Coastguard Worker
89*9e94795aSAndroid Build Coastguard Workerclass Table:
90*9e94795aSAndroid Build Coastguard Worker    def __init__(self, row_title, fixed_titles=[]):
91*9e94795aSAndroid Build Coastguard Worker        self._data = {}
92*9e94795aSAndroid Build Coastguard Worker        self._rows = []
93*9e94795aSAndroid Build Coastguard Worker        self._cols = []
94*9e94795aSAndroid Build Coastguard Worker        self._fixed_cols = {}
95*9e94795aSAndroid Build Coastguard Worker        self._titles = [row_title] + fixed_titles
96*9e94795aSAndroid Build Coastguard Worker
97*9e94795aSAndroid Build Coastguard Worker    def Set(self, column_key, row_key, data):
98*9e94795aSAndroid Build Coastguard Worker        self._data[(column_key, row_key)] = data
99*9e94795aSAndroid Build Coastguard Worker        if not column_key in self._cols:
100*9e94795aSAndroid Build Coastguard Worker            self._cols.append(column_key)
101*9e94795aSAndroid Build Coastguard Worker        if not row_key in self._rows:
102*9e94795aSAndroid Build Coastguard Worker            self._rows.append(row_key)
103*9e94795aSAndroid Build Coastguard Worker
104*9e94795aSAndroid Build Coastguard Worker    def SetFixedCol(self, row_key, columns):
105*9e94795aSAndroid Build Coastguard Worker        self._fixed_cols[row_key] = columns
106*9e94795aSAndroid Build Coastguard Worker
107*9e94795aSAndroid Build Coastguard Worker    def Write(self, out, fmt):
108*9e94795aSAndroid Build Coastguard Worker        table = []
109*9e94795aSAndroid Build Coastguard Worker        # Expand the column items
110*9e94795aSAndroid Build Coastguard Worker        for row in zip(*self._cols):
111*9e94795aSAndroid Build Coastguard Worker            if row.count(row[0]) == len(row):
112*9e94795aSAndroid Build Coastguard Worker                continue
113*9e94795aSAndroid Build Coastguard Worker            table.append([""] * len(self._titles) + [col for col in row])
114*9e94795aSAndroid Build Coastguard Worker        if table:
115*9e94795aSAndroid Build Coastguard Worker            # Update the last row of the header with title and add separator
116*9e94795aSAndroid Build Coastguard Worker            for i in range(len(self._titles)):
117*9e94795aSAndroid Build Coastguard Worker                table[len(table)-1][i] = self._titles[i]
118*9e94795aSAndroid Build Coastguard Worker            if fmt == "table":
119*9e94795aSAndroid Build Coastguard Worker                table.append(pretty.SEPARATOR)
120*9e94795aSAndroid Build Coastguard Worker        # Populate the data
121*9e94795aSAndroid Build Coastguard Worker        for row in self._rows:
122*9e94795aSAndroid Build Coastguard Worker            table.append([str(row)]
123*9e94795aSAndroid Build Coastguard Worker                         + self._fixed_cols[row]
124*9e94795aSAndroid Build Coastguard Worker                         + [str(self._data.get((col, row), "")) for col in self._cols])
125*9e94795aSAndroid Build Coastguard Worker        if fmt == "csv":
126*9e94795aSAndroid Build Coastguard Worker            csv.writer(sys.stdout, quoting=csv.QUOTE_MINIMAL).writerows(table)
127*9e94795aSAndroid Build Coastguard Worker        else:
128*9e94795aSAndroid Build Coastguard Worker            out.write(pretty.FormatTable(table, alignments="LL"))
129*9e94795aSAndroid Build Coastguard Worker
130*9e94795aSAndroid Build Coastguard Worker
131*9e94795aSAndroid Build Coastguard Workerdef format_duration_sec(ns, fmt_sec):
132*9e94795aSAndroid Build Coastguard Worker    "Format a duration in ns to second precision"
133*9e94795aSAndroid Build Coastguard Worker    sec = round(ns / 1000000000)
134*9e94795aSAndroid Build Coastguard Worker    if fmt_sec:
135*9e94795aSAndroid Build Coastguard Worker        return f"{sec}"
136*9e94795aSAndroid Build Coastguard Worker    else:
137*9e94795aSAndroid Build Coastguard Worker        h, sec = divmod(sec, 60*60)
138*9e94795aSAndroid Build Coastguard Worker        m, sec = divmod(sec, 60)
139*9e94795aSAndroid Build Coastguard Worker        result = ""
140*9e94795aSAndroid Build Coastguard Worker        if h > 0:
141*9e94795aSAndroid Build Coastguard Worker            result += f"{h:2d}h "
142*9e94795aSAndroid Build Coastguard Worker        if h > 0 or m > 0:
143*9e94795aSAndroid Build Coastguard Worker            result += f"{m:2d}m "
144*9e94795aSAndroid Build Coastguard Worker        return result + f"{sec:2d}s"
145*9e94795aSAndroid Build Coastguard Worker
146*9e94795aSAndroid Build Coastguard Worker
147*9e94795aSAndroid Build Coastguard Workerdef main(argv):
148*9e94795aSAndroid Build Coastguard Worker    parser = argparse.ArgumentParser(
149*9e94795aSAndroid Build Coastguard Worker            prog="format_benchmarks",
150*9e94795aSAndroid Build Coastguard Worker            allow_abbrev=False, # Don't let people write unsupportable scripts.
151*9e94795aSAndroid Build Coastguard Worker            description="Print analysis tables for benchmarks")
152*9e94795aSAndroid Build Coastguard Worker
153*9e94795aSAndroid Build Coastguard Worker    parser.add_argument("--csv", action="store_true",
154*9e94795aSAndroid Build Coastguard Worker                        help="Print in CSV instead of table.")
155*9e94795aSAndroid Build Coastguard Worker
156*9e94795aSAndroid Build Coastguard Worker    parser.add_argument("--sec", action="store_true",
157*9e94795aSAndroid Build Coastguard Worker                        help="Print in seconds instead of minutes and seconds")
158*9e94795aSAndroid Build Coastguard Worker
159*9e94795aSAndroid Build Coastguard Worker    parser.add_argument("--tags", nargs="*",
160*9e94795aSAndroid Build Coastguard Worker                        help="The tags to print, in order.")
161*9e94795aSAndroid Build Coastguard Worker
162*9e94795aSAndroid Build Coastguard Worker    parser.add_argument("summaries", nargs="*",
163*9e94795aSAndroid Build Coastguard Worker                        help="A summary.json file or a directory in which to look for summaries.")
164*9e94795aSAndroid Build Coastguard Worker
165*9e94795aSAndroid Build Coastguard Worker    args = parser.parse_args()
166*9e94795aSAndroid Build Coastguard Worker
167*9e94795aSAndroid Build Coastguard Worker    # Load the summaries
168*9e94795aSAndroid Build Coastguard Worker    summaries = [(s, LoadSummary(s)) for s in FindSummaries(args.summaries)]
169*9e94795aSAndroid Build Coastguard Worker
170*9e94795aSAndroid Build Coastguard Worker    # Convert to MTV time
171*9e94795aSAndroid Build Coastguard Worker    for filename, s in summaries:
172*9e94795aSAndroid Build Coastguard Worker        dt = datetime.datetime.fromisoformat(s["start_time"])
173*9e94795aSAndroid Build Coastguard Worker        dt = dt.astimezone(zoneinfo.ZoneInfo("America/Los_Angeles"))
174*9e94795aSAndroid Build Coastguard Worker        s["datetime"] = dt
175*9e94795aSAndroid Build Coastguard Worker        s["date"] = datetime.date(dt.year, dt.month, dt.day)
176*9e94795aSAndroid Build Coastguard Worker
177*9e94795aSAndroid Build Coastguard Worker    # Filter out tags we don't want
178*9e94795aSAndroid Build Coastguard Worker    if args.tags:
179*9e94795aSAndroid Build Coastguard Worker        summaries = [(f, s) for f, s in summaries if s.get("tag", "") in args.tags]
180*9e94795aSAndroid Build Coastguard Worker
181*9e94795aSAndroid Build Coastguard Worker    # If they supplied tags, sort in that order, otherwise sort by tag
182*9e94795aSAndroid Build Coastguard Worker    if args.tags:
183*9e94795aSAndroid Build Coastguard Worker        tagsort = lambda tag: args.tags.index(tag)
184*9e94795aSAndroid Build Coastguard Worker    else:
185*9e94795aSAndroid Build Coastguard Worker        tagsort = lambda tag: tag
186*9e94795aSAndroid Build Coastguard Worker
187*9e94795aSAndroid Build Coastguard Worker    # Sort the summaries
188*9e94795aSAndroid Build Coastguard Worker    summaries.sort(key=lambda s: (s[1]["date"], s[1]["branch"], tagsort(s[1]["tag"])))
189*9e94795aSAndroid Build Coastguard Worker
190*9e94795aSAndroid Build Coastguard Worker    # group the benchmarks by column and iteration
191*9e94795aSAndroid Build Coastguard Worker    def bm_key(b):
192*9e94795aSAndroid Build Coastguard Worker        return (
193*9e94795aSAndroid Build Coastguard Worker            lunch_str(b["lunch"]),
194*9e94795aSAndroid Build Coastguard Worker        )
195*9e94795aSAndroid Build Coastguard Worker    for filename, summary in summaries:
196*9e94795aSAndroid Build Coastguard Worker        summary["columns"] = [(key, group_by(bms, lambda b: b["id"])) for key, bms
197*9e94795aSAndroid Build Coastguard Worker                              in group_by(summary["benchmarks"], bm_key)]
198*9e94795aSAndroid Build Coastguard Worker
199*9e94795aSAndroid Build Coastguard Worker    # Build the table
200*9e94795aSAndroid Build Coastguard Worker    table = Table("Benchmark", ["Rebuild"])
201*9e94795aSAndroid Build Coastguard Worker    for filename, summary in summaries:
202*9e94795aSAndroid Build Coastguard Worker        for key, column in summary["columns"]:
203*9e94795aSAndroid Build Coastguard Worker            for id, cell in column:
204*9e94795aSAndroid Build Coastguard Worker                duration_ns = statistics.median([b["duration_ns"] for b in cell])
205*9e94795aSAndroid Build Coastguard Worker                modules = cell[0]["modules"]
206*9e94795aSAndroid Build Coastguard Worker                if not modules:
207*9e94795aSAndroid Build Coastguard Worker                    modules = ["---"]
208*9e94795aSAndroid Build Coastguard Worker                table.SetFixedCol(cell[0]["title"], [" ".join(modules)])
209*9e94795aSAndroid Build Coastguard Worker                table.Set(tuple([summary["date"].strftime("%Y-%m-%d"),
210*9e94795aSAndroid Build Coastguard Worker                                 summary["branch"],
211*9e94795aSAndroid Build Coastguard Worker                                 summary["tag"]]
212*9e94795aSAndroid Build Coastguard Worker                                + list(key)),
213*9e94795aSAndroid Build Coastguard Worker                          cell[0]["title"], format_duration_sec(duration_ns, args.sec))
214*9e94795aSAndroid Build Coastguard Worker
215*9e94795aSAndroid Build Coastguard Worker    table.Write(sys.stdout, "csv" if args.csv else "table")
216*9e94795aSAndroid Build Coastguard Worker
217*9e94795aSAndroid Build Coastguard Workerif __name__ == "__main__":
218*9e94795aSAndroid Build Coastguard Worker    main(sys.argv)
219*9e94795aSAndroid Build Coastguard Worker
220