1#!/usr/bin/env python3 2# Copyright (C) 2021 The Android Open Source Project 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15""" Builds all the revisions in channels.json and deploys them if --upload. 16 17See go/perfetto-ui-autopush for docs on how this works end-to-end. 18""" 19 20import argparse 21import json 22import os 23import re 24import shutil 25import subprocess 26import sys 27 28from os.path import dirname 29 30pjoin = os.path.join 31 32BUCKET_NAME = 'ui.perfetto.dev' 33CUR_DIR = dirname(os.path.abspath(__file__)) 34ROOT_DIR = dirname(dirname(CUR_DIR)) 35 36 37def check_call_and_log(args): 38 print(' '.join(args)) 39 subprocess.check_call(args) 40 41 42def check_output(args): 43 return subprocess.check_output(args).decode().strip() 44 45 46def version_exists(version): 47 url = 'https://commondatastorage.googleapis.com/%s/%s/manifest.json' % ( 48 BUCKET_NAME, version) 49 return 0 == subprocess.call(['curl', '-fLs', '-o', '/dev/null', url]) 50 51 52def build_git_revision(channel, git_ref, tmp_dir): 53 workdir = pjoin(tmp_dir, channel) 54 check_call_and_log(['rm', '-rf', workdir]) 55 check_call_and_log(['git', 'clone', '--quiet', '--shared', ROOT_DIR, workdir]) 56 old_cwd = os.getcwd() 57 os.chdir(workdir) 58 try: 59 check_call_and_log(['git', 'reset', '--hard', git_ref]) 60 check_call_and_log(['git', 'clean', '-dfx']) 61 git_sha = check_output(['git', 'rev-parse', 'HEAD']) 62 print('===================================================================') 63 print('Building UI for channel %s @ %s (%s)' % (channel, git_ref, git_sha)) 64 print('===================================================================') 65 version = check_output(['tools/write_version_header.py', '--stdout']) 66 check_call_and_log(['tools/install-build-deps', '--ui']) 67 check_call_and_log(['ui/build']) 68 return version, pjoin(workdir, 'ui/out/dist') 69 finally: 70 os.chdir(old_cwd) 71 72 73def build_all_channels(channels, tmp_dir, merged_dist_dir): 74 channel_map = {} 75 for chan in channels: 76 channel = chan['name'] 77 git_ref = chan['rev'] 78 # version here is something like "v1.2.3". 79 version, dist_dir = build_git_revision(channel, git_ref, tmp_dir) 80 channel_map[channel] = version 81 check_call_and_log(['cp', '-an', pjoin(dist_dir, version), merged_dist_dir]) 82 if channel != 'stable': 83 continue 84 # Copy also the /index.html and /service_worker.*, but only for the stable 85 # channel. The /index.html and SW must be shared between all channels, 86 # because they are all reachable through ui.perfetto.dev/. Both the index 87 # and the SQ are supposed to be version-independent (go/perfetto-channels). 88 # If an accidental incompatibility bug sneaks in, we should much rather 89 # crash canary (or any other channel) rather than stable. Hence why we copy 90 # the index+sw from the stable channel. 91 for fname in os.listdir(dist_dir): 92 fpath = pjoin(dist_dir, fname) 93 if os.path.isfile(fpath): 94 check_call_and_log(['cp', '-an', fpath, merged_dist_dir]) 95 return channel_map 96 97 98def main(): 99 parser = argparse.ArgumentParser() 100 parser.add_argument('--upload', action='store_true') 101 parser.add_argument('--tmp', default='/tmp/perfetto_ui') 102 parser.add_argument('--branch_only') 103 104 args = parser.parse_args() 105 106 # Read the releases.json, which maps channel names to git refs, e.g.: 107 # {name:'stable', rev:'a0b1c2...0}, {name:'canary', rev:'HEAD'} 108 channels = [] 109 with open(pjoin(CUR_DIR, 'channels.json')) as f: 110 channels = json.load(f)['channels'] 111 112 if args.branch_only: 113 channels = [{'name': 'branch', 'rev': args.branch_only}] 114 115 merged_dist_dir = pjoin(args.tmp, 'dist') 116 check_call_and_log(['rm', '-rf', merged_dist_dir]) 117 shutil.os.makedirs(merged_dist_dir) 118 channel_map = build_all_channels(channels, args.tmp, merged_dist_dir) 119 120 if not args.branch_only: 121 print('Updating index in ' + merged_dist_dir) 122 with open(pjoin(merged_dist_dir, 'index.html'), 'r+') as f: 123 index_html = f.read() 124 f.seek(0, 0) 125 f.truncate() 126 index_html = re.sub( 127 r"data-perfetto_version='[^']*'", 128 "data-perfetto_version='%s'" % json.dumps(channel_map), index_html) 129 f.write(index_html) 130 131 if not args.upload: 132 return 133 134 print('===================================================================') 135 print('Uploading to gs://%s' % BUCKET_NAME) 136 print('===================================================================') 137 # TODO(primiano): re-enable caching once the gzip-related outage is restored. 138 # cache_hdr = 'Cache-Control:public, max-age=3600' 139 cache_hdr = 'Cache-Control:no-cache' 140 cp_cmd = ['gsutil', '-m', '-h', cache_hdr, 'cp', '-j', 'html,js,css,wasm,map'] 141 for name in os.listdir(merged_dist_dir): 142 path = pjoin(merged_dist_dir, name) 143 if os.path.isdir(path): 144 if version_exists(name): 145 print('Skipping upload of %s because it already exists on GCS' % name) 146 continue 147 check_call_and_log(cp_cmd + ['-r', path, 'gs://%s/' % BUCKET_NAME]) 148 else: 149 # /index.html or /service_worker.js{,.map} 150 check_call_and_log(cp_cmd + [path, 'gs://%s/%s' % (BUCKET_NAME, name)]) 151 152 153if __name__ == '__main__': 154 sys.exit(main()) 155