xref: /aosp_15_r20/external/perfetto/tools/install-build-deps (revision 6dbdd20afdafa5e3ca9b8809fa73465d530080dc) 
1#!/usr/bin/env python3
2# Copyright (C) 2017 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#      http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16import argparse
17import dataclasses as dc
18import hashlib
19import logging
20import os
21import shutil
22import subprocess
23import stat
24import sys
25import tempfile
26import time
27import zipfile
28import bz2
29
30from collections import namedtuple
31from platform import system, machine
32
33
34# The format for the deps below is the following:
35# (target_folder, source_url, sha1, target_os, target_arch)
36# |source_url| can be either a git repo or a http url.
37# If a git repo, |checksum| is the SHA1 committish that will be checked out.
38# If a http url, |checksum| is the SHA256 of the downloaded file.
39# If the url is a .zip, .tgz, or .tbz2 file it will be automatically deflated under
40# |target_folder|, taking care of stripping the root folder if it's a single
41# root (to avoid ending up with buildtools/protobuf/protobuf-1.2.3/... and have
42# instead just buildtools/protobuf).
43# |target_os| is either 'darwin', 'linux', 'windows' or 'all'
44# |target_arch| is either 'x64', 'arm64' or 'all'
45# in both cases the dep only applies on matching platforms
46# |target_arch| can be 'all' when 'target_os' is not 'all' for example in the
47# case of MacOS universal binaries.
48@dc.dataclass
49class Dependency:
50  target_folder: str
51  source_url: str
52  checksum: str
53  target_os: str
54  target_arch: str
55  submodules: bool = False
56
57
58# This is to remove old directories when build tools get {re,}moved. This is to
59# avoid accidentally referring to stale dir in custom user scripts.
60CLEANUP_OLD_DIRS = [
61    'buildtools/nodejs',  # Moved to buildtools/{mac,linux64}/nodejs
62    'buildtools/emsdk',  # Moved to buildtools/{mac,linux64}/emsdk
63    'buildtools/test_data',  # Moved to test/data by r.android.com/1539381 .
64    'buildtools/d8',  # Removed by r.android.com/1424334 .
65
66    # Build tools moved to third_party/ by r.android.com/2327602 .
67    'buildtools/mac/clang-format',
68    'buildtools/mac/gn',
69    'buildtools/mac/ninja',
70    'buildtools/linux64/clang-format',
71    'buildtools/linux64/gn',
72    'buildtools/linux64/ninja',
73    'buildtools/win/clang-format.exe',
74    'buildtools/win/gn.exe',
75    'buildtools/win/ninja.exe',
76]
77
78# Dependencies required to build code on the host or when targeting desktop OS.
79BUILD_DEPS_TOOLCHAIN_HOST = [
80    # GN. From https://chrome-infra-packages.appspot.com/dl/gn/gn/.
81    # git_revision:0725d7827575b239594fbc8fd5192873a1d62f44 .
82    Dependency(
83        'third_party/gn/gn',
84        'https://storage.googleapis.com/perfetto/gn-mac-1968-0725d782',
85        '9ced623a664560bba38bbadb9b91158ca4186358c847e17ab7d982b351373c2e',
86        'darwin', 'x64'),
87    Dependency(
88        'third_party/gn/gn',
89        'https://storage.googleapis.com/perfetto/gn-mac-arm64-1968-0725d782',
90        'd22336b5210b4dad5e36e8c28ce81187f491822cf4d8fd0a257b30d6bee3fd3f',
91        'darwin', 'arm64'),
92    Dependency(
93        'third_party/gn/gn',
94        'https://storage.googleapis.com/perfetto/gn-linux64-1968-0725d782',
95        'f706aaa0676e3e22f5fc9ca482295d7caee8535d1869f99efa2358177b64f5cd',
96        'linux', 'x64'),
97    Dependency(
98        'third_party/gn/gn',
99        'https://storage.googleapis.com/perfetto/gn-linux-arm64-1968-0725d782',
100        'c2a372cd4f911028d8bc351fbf24835c9b1194fcc92beadf6c5a2b3addae973c',
101        'linux', 'arm64'),
102    Dependency(
103        'third_party/gn/gn.exe',
104        'https://storage.googleapis.com/perfetto/gn-win-1968-0725d782',
105        '001f777f023c7a6959c778fb3a6b6cfc63f6baef953410ecdeaec350fb12285b',
106        'windows', 'x64'),
107
108    # clang-format
109    # From
110    # https://chromium.googlesource.com/chromium/src/buildtools/+/refs/heads/master/mac/clang-format.arm64.sha1
111    Dependency(
112        'third_party/clang-format/clang-format',
113        'https://storage.googleapis.com/chromium-clang-format/5553d7a3d912b7d49381ad68c9a56740601a57a0',
114        'e077990b2ea6807f6abc71b4cf1e487719d7e40574baddd2b51187fdcc8db803',
115        'darwin', 'arm64'),
116    # From
117    # https://chromium.googlesource.com/chromium/src/buildtools/+/refs/heads/master/mac/clang-format.x64.sha1
118    Dependency(
119        'third_party/clang-format/clang-format',
120        'https://storage.googleapis.com/chromium-clang-format/87d69aeff220c916b85b5d6d162fa5668aa9d64f',
121        '71179a8788a009cfd589636d50f0eb9f95f58b0cfda4351430bed7c0a48c080b',
122        'darwin', 'x64'),
123    # From https://chromium.googlesource.com/chromium/src/buildtools/+/refs/heads/master/linux64/clang-format.sha1
124    Dependency(
125        'third_party/clang-format/clang-format',
126        'https://storage.googleapis.com/chromium-clang-format/1facab3101fc6da6b9467543f27f0622b966bc19',
127        '5e459118d8ac825452e9e1f2717e4de5a36399dc6cc6aec7ec483ad27a0c927e',
128        'linux', 'x64'),
129    # From https://chromium.googlesource.com/chromium/src/buildtools/+/refs/heads/master/win/clang-format.exe.sha1
130    Dependency(
131        'third_party/clang-format/clang-format.exe',
132        'https://storage.googleapis.com/chromium-clang-format/2e569921b9884daf157021d6ae9e21991cd6cf81',
133        '2227376ada89ea832995b832222b722a27c4d5d8d59e9c4d7842877f99a1e30d',
134        'windows', 'x64'),
135
136    # Keep the SHA1 in sync with |clang_format_rev| in chromium //buildtools/DEPS.
137    Dependency(
138        'buildtools/clang_format/script',
139        'https://chromium.googlesource.com/external/github.com/llvm/llvm-project/clang/tools/clang-format.git',
140        'f97059df7f8b205064625cdb5f97b56668a125ef', 'all', 'all'),
141
142    # Ninja
143    Dependency(
144        'third_party/ninja/ninja',
145        'https://storage.googleapis.com/perfetto/ninja-mac-x64_and_arm64-182',
146        '36e8b7aaa06911e1334feb664dd731a1cd69a15eb916a231a3d10ff65fca2c73',
147        'darwin', 'all'),
148    Dependency(
149        'third_party/ninja/ninja',
150        'https://storage.googleapis.com/perfetto/ninja-linux64-182',
151        '54ac6a01362190aaabf4cf276f9c8982cdf11b225438940fdde3339be0f2ecdc',
152        'linux', 'x64'),
153    Dependency(
154        'third_party/ninja/ninja.exe',
155        'https://storage.googleapis.com/perfetto/ninja-win-182',
156        '09ced0fcd1a4dec7d1b798a2cf9ce5d20e5d2fbc2337343827f192ce47d0f491',
157        'windows', 'x64'),
158    Dependency(
159        'third_party/ninja/ninja',
160        'https://storage.googleapis.com/perfetto/ninja-linux-arm64-1111',
161        '05031a734ec4310a51b2cfe9f0096b26fce25ab4ff19e5b51abe6371de066cc5',
162        'linux', 'arm64'),
163
164    # Keep the revision in sync with Chrome's PACKAGE_VERSION in
165    # tools/clang/scripts/update.py.
166    Dependency(
167        'buildtools/linux64/clang.tgz',
168        'https://commondatastorage.googleapis.com/chromium-browser-clang/Linux_x64/clang-llvmorg-19-init-2941-ga0b3dbaf-22.tgz',
169        '6741cc1083f935795330b6e04617ac891a7b5d2b5647b664c5b0fccc354adb43',
170        'linux', 'x64'),
171    Dependency(
172        'buildtools/win/clang.tgz',
173        'https://commondatastorage.googleapis.com/chromium-browser-clang/Win/clang-llvmorg-19-init-2941-ga0b3dbaf-22.tgz',
174        'f627080ed53d4c156f089323e04fa3690c8bb459110b62cd1952b0e1f0755987',
175        'windows', 'x64'),
176]
177
178BUILD_DEPS_HOST = [
179    # Keep in sync with Android's //external/googletest/METADATA.
180    Dependency(
181        'buildtools/googletest',
182        'https://android.googlesource.com/platform/external/googletest.git',
183        '609281088cfefc76f9d0ce82e1ff6c30cc3591e5', 'all', 'all'),
184
185    # Keep in sync with Chromium's //third_party/protobuf.
186    Dependency(
187        'buildtools/protobuf',
188        # If you revert the below version back to an earlier version of
189        # protobuf, make sure to revert the changes to
190        # //gn/standalone/protoc.py as well.
191        #
192        # This comment can be removed with protobuf is next upreved.
193        'https://chromium.googlesource.com/external/github.com/protocolbuffers/protobuf.git',
194        'f0dc78d7e6e331b8c6bb2d5283e06aa26883ca7c',  # refs/tags/v21.12
195        'all',
196        'all'),
197
198    # libc++, libc++abi and libunwind for Linux where we need to rebuild the C++
199    # lib from sources. Keep the SHA1s in sync with Chrome's src/buildtools/DEPS.
200    Dependency(
201        'buildtools/libcxx',
202        'https://chromium.googlesource.com/external/github.com/llvm/llvm-project/libcxx.git',
203        '852bc6746f45add53fec19f3a29280e69e358d44', 'all', 'all'),
204    Dependency(
205        'buildtools/libcxxabi',
206        'https://chromium.googlesource.com/external/github.com/llvm/llvm-project/libcxxabi.git',
207        'a37a3aa431f132b02a58656f13984d51098330a2', 'all', 'all'),
208    Dependency(
209        'buildtools/libunwind',
210        'https://chromium.googlesource.com/external/github.com/llvm/llvm-project/libunwind.git',
211        '419b03c0b8f20d6da9ddcb0d661a94a97cdd7dad', 'all', 'all'),
212
213    # Keep in sync with chromium DEPS.
214    Dependency(
215        'buildtools/libfuzzer',
216        'https://chromium.googlesource.com/chromium/llvm-project/compiler-rt/lib/fuzzer.git',
217        'debe7d2d1982e540fbd6bd78604bf001753f9e74', 'linux', 'all'),
218
219    # Benchmarking tool.
220    Dependency(
221        'buildtools/benchmark',
222        'https://chromium.googlesource.com/external/github.com/google/benchmark.git',
223        'e991355c02b93fe17713efe04cbc2e278e00fdbd', 'all', 'all'),
224
225    # Libbacktrace, for stacktraces in Linux/Android debug builds.
226    # From https://github.com/ianlancetaylor/libbacktrace/archive/177940370e4a6b2509e92a0aaa9749184e64af43.zip
227    Dependency(
228        'buildtools/libbacktrace.zip',
229        'https://storage.googleapis.com/perfetto/libbacktrace-14818b7783eeb9a56c3f0fca78cefd3143f8c5f6.zip',
230        '0d09295938155aa84d9a6049f63df8cd2def3a28302b3550ea3ead9100b3d086',
231        'all', 'all'),
232
233    # Sqlite for the trace processing library.
234    # This is the amalgamated source whose compiled output is meant to be faster.
235    # We still pull the full source for the extensions (which are not available
236    # in the amalgamation).
237    # If updating the version, also update bazel/deps.bzl.
238    Dependency(
239        'buildtools/sqlite.zip',
240        'https://storage.googleapis.com/perfetto/sqlite-amalgamation-3440200.zip',
241        '833be89b53b3be8b40a2e3d5fedb635080e3edb204957244f3d6987c2bb2345f',
242        'all', 'all'),
243    Dependency(
244        'buildtools/sqlite_src',
245        'https://chromium.googlesource.com/external/github.com/sqlite/sqlite.git',
246        'c8f9803dc32bfee78a9ca2b1abbe39499729219b',  # refs/tags/version-3.44.2.
247        'all',
248        'all'),
249
250    # JsonCpp for legacy json import. Used only by the trace processor in
251    # standalone builds.
252    # If updating the version, also update bazel/deps.bzl.
253    Dependency(
254        'buildtools/jsoncpp',
255        'https://chromium.googlesource.com/external/github.com/open-source-parsers/jsoncpp.git',
256        '6aba23f4a8628d599a9ef7fa4811c4ff6e4070e2',  # refs/tags/1.9.3.
257        'all',
258        'all'),
259
260    # Libexpat for Instruments XML import.
261    # If updating the version, also update bazel/deps.bzl.
262    Dependency(
263        'buildtools/expat/src',
264        'https://chromium.googlesource.com/external/github.com/libexpat/libexpat.git',
265        'fa75b96546c069d17b8f80d91e0f4ef0cde3790d',  # refs/tags/upstream/R_2_6_2.
266        'all',
267        'all'),
268
269    # Archive with only the demangling sources from llvm-project.
270    # See tools/repackage_llvm_demangler.sh on how to update this.
271    # File suffix is the git reference to the commit at which we rearchived the
272    # sources, as hosted on https://llvm.googlesource.com/llvm-project.
273    # If updating the version, also update bazel/deps.bzl.
274    Dependency(
275        'buildtools/llvm-project.tgz',
276        'https://storage.googleapis.com/perfetto/llvm-project-617a15a9eac96088ae5e9134248d8236e34b91b1.tgz',
277        '7e2541446a27f2a09a84520da7bc93cd71749ba0f17318f2d5291fbf45b97956',
278        'all', 'all'),
279
280    # These dependencies are for libunwindstack, which is used by src/profiling.
281    Dependency('buildtools/android-core',
282               'https://android.googlesource.com/platform/system/core.git',
283               '9e6cef7f07d8c11b3ea820938aeb7ff2e9dbaa52', 'all', 'all'),
284    Dependency(
285        'buildtools/android-unwinding',
286        'https://android.googlesource.com/platform/system/unwinding.git',
287        '4b59ea8471e89d01300481a92de3230b79b6d7c7', 'all', 'all'),
288    Dependency('buildtools/android-logging',
289               'https://android.googlesource.com/platform/system/logging.git',
290               '7b36b566c9113fc703d68f76e8f40c0c2432481c', 'all', 'all'),
291    Dependency('buildtools/android-libbase',
292               'https://android.googlesource.com/platform/system/libbase.git',
293               '78f1c2f83e625bdf66d55b48bdb3a301c20d2fb3', 'all', 'all'),
294    Dependency(
295        'buildtools/android-libprocinfo',
296        'https://android.googlesource.com/platform/system/libprocinfo.git',
297        'fd214c13ededecae97a3b15b5fccc8925a749a84', 'all', 'all'),
298    Dependency('buildtools/lzma',
299               'https://android.googlesource.com/platform/external/lzma.git',
300               '7851dce6f4ca17f5caa1c93a4e0a45686b1d56c3', 'all', 'all'),
301    Dependency('buildtools/zstd',
302               'https://android.googlesource.com/platform/external/zstd.git',
303               '77211fcc5e08c781734a386402ada93d0d18d093', 'all', 'all'),
304    Dependency('buildtools/bionic',
305               'https://android.googlesource.com/platform/bionic.git',
306               'a0d0355105cb9d4a4b5384897448676133d7b8e2', 'all', 'all'),
307
308    # Zlib used both in the tracing binaries, as well as the trace processor and
309    # assorted tools.
310    # If updating the version, also update bazel/deps.bzl.
311    Dependency('buildtools/zlib',
312               'https://android.googlesource.com/platform/external/zlib.git',
313               '6d3f6aa0f87c9791ca7724c279ef61384f331dfd', 'all', 'all'),
314
315    # Linenoise, used only by trace_processor in standalone builds.
316    # If updating the version, also update bazel/deps.bzl.
317    Dependency('buildtools/linenoise',
318               'https://fuchsia.googlesource.com/third_party/linenoise.git',
319               'c894b9e59f02203dbe4e2be657572cf88c4230c3', 'all', 'all'),
320
321    # Bloaty, used to investigate binary size
322    Dependency(
323        'buildtools/bloaty.zip',
324        'https://storage.googleapis.com/perfetto/bloaty-1.1-b3b829de35babc2fe831b9488ad2e50bca939412-mac.zip',
325        '2d301bd72a20e3f42888c9274ceb4dca76c103608053572322412c2c65ab8cb8',
326        'darwin', 'all'),
327
328    Dependency('buildtools/open_csd',
329            'https://android.googlesource.com/platform/external/OpenCSD.git',
330            '0ce01e934f95efb6a216a6efa35af1245151c779', 'all', 'all'),
331]
332
333# Dependencies required to build Android code.
334# URLs and SHA1s taken from:
335# - https://dl.google.com/android/repository/repository-11.xml
336# - https://dl.google.com/android/repository/sys-img/android/sys-img.xml
337BUILD_DEPS_ANDROID = [
338    # Android NDK
339    Dependency(
340        'buildtools/ndk.zip',
341        'https://dl.google.com/android/repository/android-ndk-r26c-darwin.zip',
342        '312756dfcbdbf389d35d651e17ca98683bd36cb83cc7bf7ad51cac5c06bd064b',
343        'darwin', 'all'),
344    Dependency(
345        'buildtools/ndk.zip',
346        'https://dl.google.com/android/repository/android-ndk-r26c-linux.zip',
347        '6d6e659834d28bb24ba7ae66148ad05115ebbad7dabed1af9b3265674774fcf6',
348        'linux', 'x64'),
349]
350
351# Dependencies required to run Android tests.
352TEST_DEPS_ANDROID = [
353    # Android emulator images.
354    Dependency(
355        'buildtools/aosp-arm.zip',
356        'https://storage.googleapis.com/perfetto/aosp-02022018-arm.zip',
357        'f5c7a3a22ad7aa0bd14ba467e8697e1e917d306699bd25622aa4419a413b9b67',
358        'all', 'all'),
359
360    # platform-tools.zip contains adb binaries.
361    Dependency(
362        'buildtools/android_sdk/platform-tools.zip',
363        'https://dl.google.com/android/repository/platform-tools_r26.0.0-darwin.zip',
364        '98d392cbd21ca20d643c7e1605760cc49075611e317c534096b5564053f4ac8e',
365        'darwin', 'all'),
366    Dependency(
367        'buildtools/android_sdk/platform-tools.zip',
368        'https://dl.google.com/android/repository/platform-tools_r26.0.0-linux.zip',
369        '90208207521d85abf0d46e3374aa4e04b7aff74e4f355c792ac334de7a77e50b',
370        'linux', 'x64'),
371
372    # Android emulator binaries.
373    Dependency(
374        'buildtools/emulator',
375        'https://android.googlesource.com/platform/prebuilts/android-emulator.git',
376        '4b260028dc27bc92c39bee9129cb2ba839970956', 'all', 'x64'),
377]
378
379# This variable is updated by tools/roll-catapult-trace-viewer.
380CATAPULT_SHA256 = 'b30108e05268ce6c65bb4126b65f6bfac165d17f5c1fd285046e7e6fd76c209f'
381
382TYPEFACES_SHA256 = '1065172aaf0e9c22bc4f206ed9fdf5f1b4355d233fb21f9f26a89cd66c941940'
383
384UI_DEPS = [
385    Dependency(
386        'buildtools/mac/nodejs.tgz',
387        'https://storage.googleapis.com/chromium-nodejs/20.11.0/5b5681e12a21cda986410f69e03e6220a21dd4d2',
388        'cecb99fbb369a9090dddc27e228b66335cd72555b44fa8839ef78e56c51682c5',
389        'darwin', 'arm64'),
390    Dependency(
391        'buildtools/mac/nodejs.tgz',
392        'https://storage.googleapis.com/chromium-nodejs/20.11.0/e3c0fd53caae857309815f3f8de7c2dce49d7bca',
393        '20affacca2480c368b75a1d91ec1a2720604b325207ef0cf39cfef3c235dad19',
394        'darwin', 'x64'),
395    Dependency(
396        'buildtools/linux64/nodejs.tgz',
397        'https://storage.googleapis.com/chromium-nodejs/20.11.0/f9a337cfa0e2b92d3e5c671c26b454bd8e99769e',
398        '0ba9cc91698c1f833a1fdc1fe0cb392d825ad484c71b0d84388ac80bfd3d6079',
399        'linux', 'x64'),
400    Dependency(
401        'buildtools/mac/emsdk.tgz',
402        'https://storage.googleapis.com/perfetto/emscripten-2.0.12-mac.tgz',
403        'aa125f8c8ff8a386d43e18c8ea0c98c875cc19160a899403e8967a5478f96f31',
404        'darwin', 'all'),
405    Dependency(
406        'buildtools/linux64/emsdk.tgz',
407        'https://storage.googleapis.com/perfetto/emscripten-2.0.12-linux.tgz',
408        'bfff9fb0326363c12e19b542f27a5f12cedbfc310f30621dc497c9af51d2d2e3',
409        'linux', 'x64'),
410    Dependency(
411        'buildtools/catapult_trace_viewer.tgz',
412        'https://storage.googleapis.com/perfetto/catapult_trace_viewer-%s.tar.gz'
413        % CATAPULT_SHA256, CATAPULT_SHA256, 'all', 'all'),
414    Dependency(
415        'buildtools/typefaces.tgz',
416        'https://storage.googleapis.com/perfetto/typefaces-%s.tar.gz' %
417        TYPEFACES_SHA256, TYPEFACES_SHA256, 'all', 'all'),
418
419    Dependency(
420        'third_party/pnpm/pnpm',
421        'https://storage.googleapis.com/perfetto/pnpm-linux-arm64-8.6.3',
422        'ac76e9ab6a770479f93c1a2bf978d72636dbcb02608554378cf30075a78a22ac',
423        'linux', 'arm64'),
424    Dependency(
425        'third_party/pnpm/pnpm',
426        'https://storage.googleapis.com/perfetto/pnpm-linux-x64-8.6.3',
427        '5a58ccd78d44faac138d901976a7a8917c0f2a2f83743cfdd895fcd0bb6aa135',
428        'linux', 'x64'),
429    Dependency(
430        'third_party/pnpm/pnpm',
431        'https://storage.googleapis.com/perfetto/pnpm-macos-arm64-8.6.3',
432        'f527713d3183e30cfbfd7fd6403ceed730831c53649e50c979961eab3b2cf866',
433        'darwin', 'arm64'),
434    Dependency(
435        'third_party/pnpm/pnpm',
436        'https://storage.googleapis.com/perfetto/pnpm-macos-x64-8.6.3',
437        '6b425f7f0342341e9ee9427a9a2be2c89936c4a04efe6125f7af667eb02b10c1',
438        'darwin', 'x64'),
439]
440
441# Dependencies to build gRPC.
442BIGTRACE_DEPS = [
443    Dependency(
444        'buildtools/grpc/src',
445        'https://chromium.googlesource.com/external/github.com/grpc/grpc.git',
446        '4795c5e69b25e8c767b498bea784da0ef8c96fd5', 'all', 'all', True),
447    Dependency(
448      'buildtools/cpp-httplib',
449      'https://github.com/yhirose/cpp-httplib.git',
450      '6c3e8482f7b4e3b307bb42afbb85fd8771da86b8',
451      'all', 'all', True
452    )
453]
454
455# Sysroots required to cross-compile Linux targets (linux-arm{,64}).
456# These are taken from Chromium's build/linux/sysroot_scripts/sysroots.json.
457BUILD_DEPS_LINUX_CROSS_SYSROOTS = [
458    Dependency(
459        'buildtools/debian_sid_arm-sysroot.tgz',
460        'https://commondatastorage.googleapis.com/chrome-linux-sysroot/toolchain/11d6f690ca49e8ba01a1d8c5346cedad2cf308fd/debian_sid_arm_sysroot.tar.xz',
461        'ff192fe073d140d836c9ca1e68f7200d558bb9aa6c5c8f4f76f794f82890f99a',
462        'linux', 'all'),
463    Dependency(
464        'buildtools/debian_sid_arm64-sysroot.tgz',
465        'https://commondatastorage.googleapis.com/chrome-linux-sysroot/toolchain/2befe8ce3e88be6080e4fb7e6d412278ea6a7625/debian_sid_arm64_sysroot.tar.xz',
466        'e4389eab2fe363f3fbdfa4d3ce9d94457d78fd2c0e62171a7534867623eadc90',
467        'linux', 'all'),
468]
469
470ALL_DEPS = (
471    BUILD_DEPS_HOST + BUILD_DEPS_ANDROID + BUILD_DEPS_LINUX_CROSS_SYSROOTS +
472    TEST_DEPS_ANDROID + UI_DEPS)
473
474ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
475UI_DIR = os.path.join(ROOT_DIR, 'ui')
476TOOLS_DIR = os.path.join(ROOT_DIR, 'tools')
477NODE_MODULES_STATUS_FILE = os.path.join(UI_DIR, 'node_modules', '.last_install')
478TEST_DATA_SCRIPT = os.path.join(TOOLS_DIR, 'test_data')
479
480
481def CheckCallRetry(*args, **kwargs):
482  """ Like subprocess.check_call, with retries up to 5 times. """
483  MAX_ATTEMPTS = 5
484  for attempt in range(1, MAX_ATTEMPTS + 1):
485    try:
486      return subprocess.check_call(*args, **kwargs)
487    except subprocess.CalledProcessError as error:
488      if attempt == MAX_ATTEMPTS:
489        raise error
490      else:
491        logging.error(error)
492        time.sleep(attempt * 3)
493
494
495def DownloadURL(url, out_file):
496  CheckCallRetry(['curl', '-L', '-#', '-o', out_file, url])
497
498
499def GetArch():
500  arch = machine()
501  if arch == 'arm64':
502    return 'arm64'
503  elif arch == 'aarch64':
504    return 'arm64'
505  else:
506    # Assume everything else is x64 matching previous behaviour.
507    return 'x64'
508
509
510def ReadFile(path):
511  if not os.path.exists(path):
512    return None
513  with open(path) as f:
514    return f.read().strip()
515
516
517def MkdirRecursive(path):
518  # Works with both relative and absolute paths
519  cwd = '/' if path.startswith('/') else ROOT_DIR
520  for part in path.split('/'):
521    cwd = os.path.join(cwd, part)
522    if not os.path.exists(cwd):
523      os.makedirs(cwd)
524    else:
525      assert (os.path.isdir(cwd))
526
527
528def HashLocalFile(path):
529  if not os.path.exists(path):
530    return None
531  with open(path, 'rb') as f:
532    return hashlib.sha256(f.read()).hexdigest()
533
534
535def ExtractZipfilePreservePermissions(zf, info, path):
536  target_path = os.path.join(path, info.filename)
537  mode = info.external_attr >> 16
538  S_IFLNK  = 0o120000  # symbolic link
539  if (mode & S_IFLNK) == S_IFLNK:
540    dst = zf.read(info).decode()
541    os.symlink(dst, target_path)
542    return
543  zf.extract(info.filename, path=path)
544  min_acls = 0o755 if info.filename.endswith('/') else 0o644
545  os.chmod(target_path, mode | min_acls)
546
547
548def IsGitRepoCheckoutOutAtRevision(path, revision):
549  return ReadFile(os.path.join(path, '.git', 'HEAD')) == revision
550
551
552def RmtreeIfExists(path):
553  # Git creates read-only files on windows, which cause failures with rmtree.
554  # This seems the socially accepted way to deal with it.
555  # See https://bugs.python.org/issue19643 .
556  def del_read_only_for_windows(_action, name, _exc):
557    os.chmod(name, stat.S_IWRITE)
558    os.remove(name)
559
560  if not os.path.exists(path):
561    return
562  third_party_path = os.path.abspath(os.path.join(ROOT_DIR, 'third_party'))
563  buildtools_path = os.path.abspath(os.path.join(ROOT_DIR, 'buildtools'))
564  test_path = os.path.abspath(os.path.join(ROOT_DIR, 'test', 'data'))
565  if (not os.path.abspath(path).startswith(buildtools_path) and
566      not os.path.abspath(path).startswith(test_path) and
567      not os.path.abspath(path).startswith(third_party_path)):
568    # Safety check to prevent that some merge confilct ends up doing some
569    # rm -rf / or similar.
570    logging.fatal(
571        'Cannot remove %s: outside of {buildtools, test/data, third_party}',
572        path)
573    sys.exit(1)
574  logging.info('Removing %s' % path)
575  if os.path.isdir(path):
576    shutil.rmtree(path, onerror=del_read_only_for_windows)
577  else:
578    os.remove(path)
579
580
581def CheckoutGitRepo(path, git_url, revision, check_only):
582  if IsGitRepoCheckoutOutAtRevision(path, revision):
583    return False
584  if check_only:
585    return True
586  path = path.replace('/', os.sep)
587  RmtreeIfExists(path)
588  MkdirRecursive(path)
589  logging.info('Fetching %s @ %s into %s', git_url, revision, path)
590  subprocess.check_call(['git', 'init', path], cwd=path)
591  CheckCallRetry(['git', 'fetch', '--quiet', '--depth', '1', git_url, revision],
592                 cwd=path)
593  subprocess.check_call(['git', 'checkout', revision, '--quiet'], cwd=path)
594  CheckCallRetry(
595      ['git', 'submodule', 'update', '--init', '--recursive', '--quiet'],
596      cwd=path)
597  assert (IsGitRepoCheckoutOutAtRevision(path, revision))
598  return True
599
600
601def InstallNodeModules(force_clean=False):
602  if force_clean:
603    node_modules = os.path.join(UI_DIR, 'node_modules')
604    logging.info('Clearing %s', node_modules)
605    subprocess.check_call(['git', 'clean', '-qxffd', node_modules],
606                          cwd=ROOT_DIR)
607  logging.info("Running `pnpm install --shamefully-hoist --frozen-lockfile` in {0}".format(UI_DIR))
608
609  # Some node modules have postinstall scripts (already bad) but worse
610  # sometimes they are in the form: "postinstall: 'node ./scripts/foo'"
611  # so here we need to ensure that our hermetic node is available in
612  # PATH.
613  env = os.environ.copy()
614  env['PATH'] = TOOLS_DIR + ':' + env['PATH']
615
616  subprocess.check_call([
617    os.path.join(TOOLS_DIR, 'pnpm'),
618    'install',
619    '--shamefully-hoist',
620    '--frozen-lockfile'],
621    cwd=UI_DIR,
622    env=env)
623  # pbjs has the bad habit of installing extra packages on its first
624  # run. Run it here, so we avoid fetches while building.
625  pbjs = ['node_modules/.bin/pbjs', '/dev/null', '-o', '/dev/null']
626  subprocess.call(pbjs, cwd=UI_DIR, env=env)
627  with open(NODE_MODULES_STATUS_FILE, 'w') as f:
628    f.write(HashLocalFile(os.path.join(UI_DIR, 'pnpm-lock.yaml')))
629
630
631def CheckNodeModules():
632  """Returns True if the modules are up-to-date.
633
634  There doesn't seem to be an easy way to check node modules versions. Instead
635  just check if pnpm-lock.json changed since the last `pnpm install` call.
636  """
637  if not os.path.exists(NODE_MODULES_STATUS_FILE):
638    return False
639  with open(NODE_MODULES_STATUS_FILE, 'r') as f:
640    actual = f.read()
641  expected = HashLocalFile(os.path.join(UI_DIR, 'pnpm-lock.yaml'))
642  return expected == actual
643
644
645def CheckHashes():
646  for dep in ALL_DEPS:
647    if dep.source_url.endswith('.git'):
648      continue
649    logging.info('Downloading %s for %s-%s', dep.source_url, dep.target_os,
650                 dep.target_arch)
651    with tempfile.NamedTemporaryFile(delete=False) as f:
652      f.close()
653      DownloadURL(dep.source_url, f.name)
654      actual_checksum = HashLocalFile(f.name)
655      os.unlink(f.name)
656      if (actual_checksum != dep.checksum):
657        logging.fatal('SHA-256 mismatch for {} expected {} was {}'.format(
658            dep.source_url, dep.checksum, actual_checksum))
659
660
661def CheckDepotToolsIsRecent():
662  gn_py_path = shutil.which('gn.py')
663  if gn_py_path is None:
664    return True  # depot_tools doesn't seem to be installed in the PATH.
665  dt_dir = os.path.abspath(os.path.dirname(gn_py_path))
666  cmd = ['git', '-C', dt_dir, 'merge-base', '--is-ancestor', 'a0cf4321', 'HEAD']
667  git_ret = subprocess.call(cmd, stderr=subprocess.DEVNULL)
668  if git_ret == 0:
669    return True
670  print('\033[91mYour depot_tools revision is too old. Please run:\033[0m')
671  print('git -C %s fetch origin && git -C %s checkout -B main -t origin/main' %
672        (dt_dir, dt_dir))
673  return False
674
675
676def Main():
677  parser = argparse.ArgumentParser()
678  parser.add_argument(
679      '--android',
680      action='store_true',
681      help='NDK and emulator images target_os="android"')
682  parser.add_argument(
683      '--linux-arm',
684      action='store_true',
685      help='Debian sysroots for target_os="linux" target_cpu="arm|arm64"')
686  parser.add_argument(
687      '--ui',
688      action='store_true',
689      help='Node and NPM packages to Build the Web-based UI via ./ui/build')
690  parser.add_argument(
691      '--grpc', action='store_true', help='Packages to build gRPC')
692  parser.add_argument('--check-only')
693  parser.add_argument('--filter', action='append')
694  parser.add_argument('--verify', help='Check all URLs', action='store_true')
695  parser.add_argument(
696      '--no-toolchain', help='Do not download toolchain', action='store_true')
697  parser.add_argument(
698      '--build-os',
699      default=system().lower(),
700      choices=['windows', 'darwin', 'linux'],
701      help='Override the autodetected build operating system')
702  parser.add_argument(
703      '--build-arch',
704      default=GetArch(),
705      choices=['arm64', 'x64'],
706      help='Override the autodetected build CPU architecture')
707  args = parser.parse_args()
708  if args.verify:
709    CheckHashes()
710    return 0
711
712  target_os = args.build_os
713  if args.ui and target_os == 'windows':
714    print('Building the UI on Windows is unsupported')
715    return 1
716
717  if not CheckDepotToolsIsRecent():
718    return 1
719
720  deps = BUILD_DEPS_HOST
721  if not args.no_toolchain:
722    deps += BUILD_DEPS_TOOLCHAIN_HOST
723  if args.android:
724    deps += BUILD_DEPS_ANDROID + TEST_DEPS_ANDROID
725  if args.linux_arm:
726    deps += BUILD_DEPS_LINUX_CROSS_SYSROOTS
727  if args.ui:
728    deps += UI_DEPS
729  # TODO(b/360084012) Change the arg name to bigtrace
730  if args.grpc:
731    deps += BIGTRACE_DEPS
732  deps_updated = False
733  nodejs_updated = False
734
735  for old_dir in CLEANUP_OLD_DIRS:
736    RmtreeIfExists(os.path.join(ROOT_DIR, old_dir))
737
738  for dep in deps:
739    target_arch = args.build_arch
740    matches_os = dep.target_os == 'all' or target_os == dep.target_os
741    matches_arch = dep.target_arch == 'all' or target_arch == dep.target_arch
742    if not matches_os or not matches_arch:
743      continue
744    if args.filter and not any(f in dep.target_folder for f in args.filter):
745      continue
746    local_path = os.path.join(ROOT_DIR, dep.target_folder)
747    if dep.source_url.endswith('.git'):
748      deps_updated |= CheckoutGitRepo(local_path, dep.source_url, dep.checksum,
749                                      args.check_only)
750      continue
751    is_compressed = any([local_path.endswith(ext) for ext in ['.zip', '.tgz', '.tbz2']])
752    compressed_target_dir = os.path.splitext(local_path)[0] if is_compressed else None
753    compressed_dir_stamp = os.path.join(compressed_target_dir, '.stamp') if is_compressed else None
754
755    if ((not is_compressed and HashLocalFile(local_path) == dep.checksum) or
756        (is_compressed and ReadFile(compressed_dir_stamp) == dep.checksum)):
757      continue
758    deps_updated = True
759    if args.check_only:
760      continue
761    MkdirRecursive(os.path.dirname(dep.target_folder))
762    if HashLocalFile(local_path) != dep.checksum:
763      download_path = local_path + '.tmp'
764      logging.info('Downloading %s from %s', local_path, dep.source_url)
765      DownloadURL(dep.source_url, download_path)
766      os.chmod(download_path, 0o755)
767      actual_checksum = HashLocalFile(download_path)
768      if (actual_checksum != dep.checksum):
769        os.remove(download_path)
770        logging.fatal('SHA-256 mismatch for {} expected {} was {}'.format(
771            download_path, dep.checksum, actual_checksum))
772        return 1
773      shutil.move(download_path, local_path)
774      if 'nodejs' in dep.target_folder:
775        nodejs_updated = True
776
777    assert (HashLocalFile(local_path) == dep.checksum)
778
779    if is_compressed:
780      logging.info('Extracting %s into %s' % (local_path, compressed_target_dir))
781      assert (os.path.commonprefix((ROOT_DIR, compressed_target_dir)) == ROOT_DIR)
782      RmtreeIfExists(compressed_target_dir)
783
784      # Decompress the archive.
785      if local_path.endswith('.tgz'):
786        MkdirRecursive(compressed_target_dir)
787        subprocess.check_call(['tar', '-oxf', local_path], cwd=compressed_target_dir)
788      elif local_path.endswith('.zip'):
789        with zipfile.ZipFile(local_path, 'r') as zf:
790          for info in zf.infolist():
791            ExtractZipfilePreservePermissions(zf, info, compressed_target_dir)
792      elif local_path.endswith('.tbz2'):
793        tar_path = '{}.tar.tmp'.format(local_path)
794        with open(tar_path, 'w') as f:
795          with bz2.open(local_path, 'r') as bf:
796            f.write(bf.read())
797        MkdirRecursive(compressed_target_dir)
798        subprocess.check_call(['tar', '-oxf', tar_path], cwd=compressed_target_dir)
799
800      # If the zip contains one root folder, rebase one level up moving all
801      # its sub files and folders inside |target_dir|.
802      subdir = os.listdir(compressed_target_dir)
803      if len(subdir) == 1:
804        subdir = os.path.join(compressed_target_dir, subdir[0])
805        if os.path.isdir(subdir):
806          for subf in os.listdir(subdir):
807            shutil.move(os.path.join(subdir, subf), compressed_target_dir)
808          os.rmdir(subdir)
809
810      # Create stamp and remove the archive.
811      with open(compressed_dir_stamp, 'w') as stamp_file:
812        stamp_file.write(dep.checksum)
813      os.remove(local_path)
814
815  if args.ui:
816    # Needs to happen after nodejs is installed above.
817    if args.check_only:
818      deps_updated |= not CheckNodeModules()
819    else:
820      InstallNodeModules(force_clean=nodejs_updated)
821
822  cur_python_interpreter = sys.executable
823  test_data_synced = 0 == subprocess.call([
824      cur_python_interpreter, TEST_DATA_SCRIPT, 'status', '--quiet',
825      '--ignore-new'
826  ])
827  if args.check_only:
828    if not deps_updated and test_data_synced:
829      with open(args.check_only, 'w') as f:
830        f.write('OK')  # The content is irrelevant, just keep GN happy.
831      return 0
832    argz = ' '.join(
833        [x for x in sys.argv[1:] if not x.startswith('--check-only')])
834    print('\033[91mBuild deps are stale. ' +
835          'Please run tools/install-build-deps %s\033[0m' % argz)
836    if not test_data_synced:
837      print('//test/data/ is out of sync. `tools/test_data status` for details')
838    return 1
839
840  if not test_data_synced:
841    cmd = [cur_python_interpreter, TEST_DATA_SCRIPT, 'download', '--overwrite']
842    if not sys.stdout.isatty():
843      cmd += ['--verbose']  # For CI bots
844    subprocess.check_call(cmd)
845
846  if deps_updated:
847    # Stale binary files may be compiled against old sysroot headers that aren't
848    # tracked by gn.
849    logging.warning('Remember to run "gn clean <output_directory>" ' +
850                    'to avoid stale binary files.')
851
852
853if __name__ == '__main__':
854  logging.basicConfig(level=logging.INFO)
855  sys.exit(Main())
856