xref: /aosp_15_r20/tools/netsim/scripts/utils.py (revision cf78ab8cffb8fc9207af348f23af247fb04370a6) 
1#!/usr/bin/env python
2#
3# Copyright 2021 - The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the',  help='License');
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#     http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an',  help='AS IS' BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16import glob
17import json
18import logging
19import os
20from pathlib import Path
21import platform
22import shutil
23import socket
24import subprocess
25import sys
26from threading import currentThread
27
28from time_formatter import TimeFormatter
29
30if sys.version_info[0] == 3:
31  from queue import Queue
32else:
33  from Queue import Queue
34
35from threading import Thread, currentThread
36
37AOSP_ROOT = Path(__file__).absolute().parents[3]
38TOOLS = Path(AOSP_ROOT, "tools")
39EMULATOR_ARTIFACT_PATH = Path(AOSP_ROOT, "tools", "netsim", "emulator_tmp")
40PYTHON_EXE = sys.executable or "python3"
41TARGET_MAP = {
42    "windows": "windows_msvc-x86_64",
43    "windows_x64": "windows_msvc-x86_64",
44    "windows_x86_64": "windows_msvc-x86_64",
45    "linux": "linux-x86_64",
46    "linux_x64": "linux-x86_64",
47    "linux_x86_64": "linux-x86_64",
48    "linux_aarch64": "linux-aarch64",
49    "darwin": "darwin-x86_64",
50    "darwin_x64": "darwin-x86_64",
51    "darwin_x86_64": "darwin-x86_64",
52    "darwin_aarch64": "darwin-aarch64",
53}
54
55AVAILABLE = {
56    "windows_msvc-x86_64": "toolchain-windows_msvc-x86_64.cmake",
57    "linux-x86_64": "toolchain-linux-x86_64.cmake",
58    "darwin-x86_64": "toolchain-darwin-x86_64.cmake",
59    "linux-aarch64": "toolchain-linux-aarch64.cmake",
60    "darwin-aarch64": "toolchain-darwin-aarch64.cmake",
61}
62
63CMAKE = shutil.which(
64    "cmake",
65    path=str(
66        AOSP_ROOT
67        / "prebuilts"
68        / "cmake"
69        / f"{platform.system().lower()}-x86"
70        / "bin"
71    ),
72)
73
74
75def rust_version() -> str:
76  """Returns rust version"""
77  with open(
78      AOSP_ROOT / "external" / "qemu" / "android" / "build" / "toolchains.json",
79      encoding="utf-8",
80  ) as f:
81    return json.load(f)["rust"]
82
83
84def default_target() -> str:
85  """Returns default value for target"""
86  # If Mac M1, the default target should be 'darwin-aarch64'
87  if platform.system() == "Darwin" and platform.machine() == "arm64":
88    return "darwin-aarch64"
89  return platform.system()
90
91
92def create_emulator_artifact_path():
93  """Refresh or construct EMULATOR_ARTIFACT_PATH"""
94  if EMULATOR_ARTIFACT_PATH.exists():
95    shutil.rmtree(EMULATOR_ARTIFACT_PATH)
96  EMULATOR_ARTIFACT_PATH.mkdir(exist_ok=True, parents=True)
97
98
99def fetch_build_chaining_artifacts(out_dir, presubmit):
100  """Fetch the Emulator prebuilts for build_bots (go/build_chaining)"""
101  try:
102    out = Path(out_dir)
103    prebuilt_path = out / "prebuilt_cached" / "artifacts"
104    files = glob.glob(str(prebuilt_path / f"*.zip"))
105    for file in files:
106      shutil.copy2(prebuilt_path / file, EMULATOR_ARTIFACT_PATH)
107  except Exception as e:
108    if presubmit:
109      raise e
110    else:
111      logging.warn(
112          f"An error occurred during fetch_build_chaining_artifacts: {e}"
113      )
114
115
116def binary_extension(filename):
117  """Appends exe extension in case of Windows"""
118  if platform.system() == "Windows":
119    return filename + ".exe"
120  return filename
121
122
123def platform_to_cmake_target(target):
124  """Translates platform to cmake target"""
125  return TARGET_MAP[target.replace("-", "_")]
126
127
128def cmake_toolchain(target) -> str:
129  """Returns the path to the cmake toolchain file."""
130  return (
131      AOSP_ROOT
132      / "external"
133      / "qemu"
134      / "android"
135      / "build"
136      / "cmake"
137      / AVAILABLE[TARGET_MAP[target.replace("-", "_")]]
138  )
139
140
141def is_presubmit(build_id):
142  return build_id.startswith("P")
143
144
145def get_host_and_ip():
146  """Try to get my hostname and ip address."""
147  st = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
148  try:
149    st.connect(("10.255.255.255", 1))
150    my_ip = st.getsockname()[0]
151  except Exception:
152    my_ip = "127.0.0.1"
153  finally:
154    st.close()
155
156  try:
157    hostname = socket.gethostname()
158  except Exception:
159    hostname = "Unkwown"
160
161  return hostname, my_ip
162
163
164class LogBelowLevel(logging.Filter):
165
166  def __init__(self, exclusive_maximum, name=""):
167    super(LogBelowLevel, self).__init__(name)
168    self.max_level = exclusive_maximum
169
170  def filter(self, record):
171    return True if record.levelno < self.max_level else False
172
173
174def config_logging():
175  logging_handler_out = logging.StreamHandler(sys.stdout)
176  logging_handler_out.setFormatter(
177      TimeFormatter("%(asctime)s %(threadName)s | %(message)s")
178  )
179  logging_handler_out.setLevel(logging.DEBUG)
180  logging_handler_out.addFilter(LogBelowLevel(logging.WARNING))
181
182  logging_handler_err = logging.StreamHandler(sys.stderr)
183  logging_handler_err.setFormatter(
184      TimeFormatter("%(asctime)s %(threadName)s | %(message)s")
185  )
186  logging_handler_err.setLevel(logging.WARNING)
187
188  logging.root = logging.getLogger("build")
189  logging.root.setLevel(logging.INFO)
190  logging.root.addHandler(logging_handler_out)
191  logging.root.addHandler(logging_handler_err)
192
193  currentThread().setName("inf")
194
195
196def log_system_info():
197  """Log some useful system information."""
198  version = "{0[0]}.{0[1]}.{0[2]}".format(sys.version_info)
199  hostname, my_ip = get_host_and_ip()
200
201  logging.info(
202      "Hello from %s (%s). I'm a %s build bot",
203      hostname,
204      my_ip,
205      platform.system(),
206  )
207  logging.info("My uname is: %s", platform.uname())
208  logging.info(
209      "I'm happy to build the emulator using Python %s (%s)",
210      PYTHON_EXE,
211      version,
212  )
213
214
215def run(cmd, env, log_prefix, cwd=AOSP_ROOT, throw_on_failure=True):
216  currentThread().setName(log_prefix)
217  cmd_env = os.environ.copy()
218  cmd_env.update(env)
219  is_windows = platform.system() == "Windows"
220
221  cmd = [str(x) for x in cmd]
222  # logging.info("=" * 140)
223  # logging.info(json.dumps(cmd_env, sort_keys=True))
224  logging.info("%s $> %s", cwd, " ".join(cmd))
225  # logging.info("=" * 140)
226
227  proc = subprocess.Popen(
228      cmd,
229      stdout=subprocess.PIPE,
230      stderr=subprocess.PIPE,
231      shell=is_windows,  # Make sure windows propagates ENV vars properly.
232      cwd=cwd,
233      env=cmd_env,
234  )
235
236  _log_proc(proc, log_prefix)
237  proc.wait()
238  if proc.returncode != 0 and throw_on_failure:
239    raise Exception("Failed to run %s - %s" % (" ".join(cmd), proc.returncode))
240
241
242def log_to_queue(q, line):
243  """Logs the output of the given process."""
244  if q.full():
245    q.get()
246
247  strip = line.strip()
248  logging.info(strip)
249  q.put(strip)
250
251
252def _reader(pipe, logfn):
253  try:
254    with pipe:
255      for line in iter(pipe.readline, b""):
256        lg = line[:-1]
257        try:
258          lg = lg.decode("utf-8")
259        except Exception as e:
260          logfn("Failed to utf-8 decode line, {}".format(e))
261          lg = str(lg)
262        logfn(lg.strip())
263  finally:
264    pass
265
266
267def _log_proc(proc, log_prefix):
268  """Logs the output of the given process."""
269  q = Queue()
270  for args in [[proc.stdout, logging.info], [proc.stderr, logging.error]]:
271    t = Thread(target=_reader, args=args)
272    t.setName(log_prefix)
273    t.start()
274
275  return q
276