xref: /aosp_15_r20/external/pytorch/benchmarks/dynamo/check_memory_compression_ratio.py (revision da0073e96a02ea20f0ac840b70461e3646d07c45) 
1import argparse
2import sys
3import textwrap
4
5import pandas as pd
6
7
8def main(args):
9    actual = pd.read_csv(args.actual)
10    expected = pd.read_csv(args.expected)
11    failed = []
12
13    for name in actual["name"]:
14        actual_memory_compression = float(
15            actual.loc[actual["name"] == name]["compression_ratio"]
16        )
17        try:
18            expected_memory_compression = float(
19                expected.loc[expected["name"] == name]["compression_ratio"]
20            )
21        except TypeError:
22            print(f"{name:34} is missing from {args.expected}")
23            continue
24        if actual_memory_compression >= expected_memory_compression * 0.95:
25            status = "PASS"
26        else:
27            status = "FAIL"
28            failed.append(name)
29        print(
30            f"""
31            {name:34}:
32                actual_memory_compression={actual_memory_compression:.2f},
33                expected_memory_compression={expected_memory_compression:.2f},
34                {status}
35            """
36        )
37
38    if failed:
39        print(
40            textwrap.dedent(
41                f"""
42                Error: {len(failed)} models below expected memory compression ratio:
43                    {' '.join(failed)}
44                If this drop is expected, you can update `{args.expected}`.
45                """
46            )
47        )
48        sys.exit(1)
49
50
51parser = argparse.ArgumentParser()
52parser.add_argument("--actual", type=str, required=True)
53parser.add_argument("--expected", type=str, required=True)
54args = parser.parse_args()
55
56if __name__ == "__main__":
57    main(args)
58