xref: /aosp_15_r20/external/grpc-grpc/tools/profiling/memory/memory_diff.py (revision cc02d7e222339f7a4f6ba5f422e6413f4bd931f2) 
1#!/usr/bin/env python3
2#
3# Copyright 2022 gRPC authors.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#     http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17import argparse
18import csv
19import glob
20import math
21import multiprocessing
22import os
23import pathlib
24import re
25import shutil
26import subprocess
27import sys
28
29sys.path.append(
30    os.path.join(
31        os.path.dirname(sys.argv[0]), "..", "..", "run_tests", "python_utils"
32    )
33)
34import check_on_pr
35
36argp = argparse.ArgumentParser(description="Perform diff on memory benchmarks")
37
38argp.add_argument(
39    "-d",
40    "--diff_base",
41    type=str,
42    help="Commit or branch to compare the current one to",
43)
44
45argp.add_argument("-j", "--jobs", type=int, default=multiprocessing.cpu_count())
46
47args = argp.parse_args()
48
49_INTERESTING = {
50    "call/client": (
51        rb"client call memory usage: ([0-9\.]+) bytes per call",
52        float,
53    ),
54    "call/server": (
55        rb"server call memory usage: ([0-9\.]+) bytes per call",
56        float,
57    ),
58    "channel/client": (
59        rb"client channel memory usage: ([0-9\.]+) bytes per channel",
60        float,
61    ),
62    "channel/server": (
63        rb"server channel memory usage: ([0-9\.]+) bytes per channel",
64        float,
65    ),
66    "call/xds_client": (
67        rb"xds client call memory usage: ([0-9\.]+) bytes per call",
68        float,
69    ),
70    "call/xds_server": (
71        rb"xds server call memory usage: ([0-9\.]+) bytes per call",
72        float,
73    ),
74    "channel/xds_client": (
75        rb"xds client channel memory usage: ([0-9\.]+) bytes per channel",
76        float,
77    ),
78    "channel/xds_server": (
79        rb"xds server channel memory usage: ([0-9\.]+) bytes per channel",
80        float,
81    ),
82    "channel_multi_address/xds_client": (
83        rb"xds multi_address client channel memory usage: ([0-9\.]+) bytes per channel",
84        float,
85    ),
86}
87
88_SCENARIOS = {
89    "default": [],
90    "minstack": ["--scenario_config=minstack"],
91}
92
93_BENCHMARKS = {
94    "call": ["--benchmark_names=call", "--size=50000"],
95    "channel": ["--benchmark_names=channel", "--size=10000"],
96    "channel_multi_address": [
97        "--benchmark_names=channel_multi_address",
98        "--size=10000",
99    ],
100}
101
102
103def _run():
104    """Build with Bazel, then run, and extract interesting lines from the output."""
105    subprocess.check_call(
106        [
107            "tools/bazel",
108            "build",
109            "-c",
110            "opt",
111            "test/core/memory_usage/memory_usage_test",
112        ]
113    )
114    ret = {}
115    for name, benchmark_args in _BENCHMARKS.items():
116        for use_xds in (False, True):
117            for scenario, extra_args in _SCENARIOS.items():
118                # TODO(chenancy) Remove when minstack is implemented for channel
119                if name == "channel" and scenario == "minstack":
120                    continue
121                if name == "channel_multi_address" and not use_xds:
122                    continue
123                argv = (
124                    ["bazel-bin/test/core/memory_usage/memory_usage_test"]
125                    + benchmark_args
126                    + extra_args
127                )
128                if use_xds:
129                    argv.append("--use_xds")
130                try:
131                    output = subprocess.check_output(argv)
132                except subprocess.CalledProcessError as e:
133                    print("Error running benchmark:", e)
134                    continue
135                for line in output.splitlines():
136                    for key, (pattern, conversion) in _INTERESTING.items():
137                        m = re.match(pattern, line)
138                        if m:
139                            ret[scenario + ": " + key] = conversion(m.group(1))
140    return ret
141
142
143cur = _run()
144old = None
145
146if args.diff_base:
147    where_am_i = (
148        subprocess.check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"])
149        .decode()
150        .strip()
151    )
152    # checkout the diff base (="old")
153    subprocess.check_call(["git", "checkout", args.diff_base])
154    try:
155        old = _run()
156    finally:
157        # restore the original revision (="cur")
158        subprocess.check_call(["git", "checkout", where_am_i])
159
160text = ""
161if old is None:
162    print(cur)
163    for key, value in sorted(cur.items()):
164        text += "{}: {}\n".format(key, value)
165else:
166    print(cur, old)
167    call_diff_size = 0
168    channel_diff_size = 0
169    for scenario in _SCENARIOS.keys():
170        for key, value in sorted(_INTERESTING.items()):
171            key = scenario + ": " + key
172            if key in cur:
173                if key not in old:
174                    text += "{}: {}\n".format(key, cur[key])
175                else:
176                    text += "{}: {} -> {}\n".format(key, old[key], cur[key])
177                    if "call" in key:
178                        call_diff_size += cur[key] - old[key]
179                    else:
180                        channel_diff_size += cur[key] - old[key]
181
182    print("CALL_DIFF_SIZE: %f" % call_diff_size)
183    print("CHANNEL_DIFF_SIZE: %f" % channel_diff_size)
184    check_on_pr.label_increase_decrease_on_pr(
185        "per-call-memory", call_diff_size, 64
186    )
187    check_on_pr.label_increase_decrease_on_pr(
188        "per-channel-memory", channel_diff_size, 1000
189    )
190    # TODO(chennancy)Change significant value when minstack also runs for channel
191
192print(text)
193check_on_pr.check_on_pr("Memory Difference", "```\n%s\n```" % text)
194