1# Copyright (C) 2018 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14# 15"""Downloads prebuilt from the build server.""" 16 17import argparse 18import logging 19import os 20import os.path 21import shutil 22import stat 23import sys 24import textwrap 25 26 27class InstallEntry(object): 28 def __init__(self, target, name, install_path, 29 need_strip=False, need_exec=False, need_unzip=False, 30 install_unzipped=False): 31 self.target = target 32 self.name = name 33 self.install_path = install_path 34 self.need_strip = need_strip 35 self.need_exec = need_exec 36 37 # Installs a zip file, and also unzips it into the same directory. The 38 # unzipped contents are not automatically installed. 39 self.need_unzip = need_unzip 40 41 # Install the unzipped contents of a zip file into install_path, but not 42 # the file itself. All old content in install_path is removed first. 43 self.install_unzipped = install_unzipped 44 45def logger(): 46 """Returns the main logger for this module.""" 47 return logging.getLogger(__name__) 48 49 50def check_call(cmd): 51 """Proxy for subprocess.check_call with logging.""" 52 import subprocess 53 logger().debug('check_call `%s`', ' '.join(cmd)) 54 subprocess.check_call(cmd) 55 56 57def fetch_artifact(branch, build, target, pattern): 58 """Fetches artifact from the build server.""" 59 logger().info('Fetching %s from %s %s (artifacts matching %s)', build, 60 target, branch, pattern) 61 if target.startswith('local:'): 62 shutil.copyfile(target[6:], pattern) 63 return 64 fetch_artifact_path = '/google/data/ro/projects/android/fetch_artifact' 65 cmd = [fetch_artifact_path, '--branch', branch, '--target', target, 66 '--bid', build, pattern] 67 check_call(cmd) 68 69 70def copy_artifact(local_dist, target, name): 71 """Copies artifact from a local dist directory.""" 72 source_path = (target[6:] if target.startswith('local:') 73 else os.path.join(local_dist, name)) 74 logger().info('Copying from %s', source_path) 75 shutil.copyfile(source_path, os.path.basename(name)) 76 77 78def start_branch(build): 79 """Creates a new branch in the project.""" 80 branch_name = 'update-' + (build or 'latest') 81 logger().info('Creating branch %s', branch_name) 82 check_call(['repo', 'start', branch_name, '.']) 83 84 85def commit(prebuilts, branch, build, add_paths, commit_message_note): 86 """Commits the new prebuilts.""" 87 logger().info('Making commit') 88 check_call(['git', 'add'] + add_paths) 89 if build: 90 message = textwrap.dedent("""\ 91 Update {prebuilts} prebuilts to build {build}. 92 93 Taken from branch {branch}.""").format( 94 prebuilts=prebuilts, branch=branch, build=build) 95 else: 96 message = ( 97 'DO NOT SUBMIT: Update {prebuilts} prebuilts from local build.' 98 .format(prebuilts=prebuilts)) 99 if commit_message_note: 100 message += "\n\n" + commit_message_note 101 check_call(['git', 'commit', '-m', message]) 102 103 104def list_installed_files(install_list, extracted_list): 105 """List all prebuilts in current directory.""" 106 result = [] 107 for entry in install_list: 108 result += [entry.install_path] 109 for entry in extracted_list: 110 result += [entry.install_path] 111 return result 112 113 114def remove_old_files(install_list, extracted_list): 115 """Removes the old files.""" 116 old_files = list_installed_files(install_list, extracted_list) 117 if not old_files: 118 return 119 logger().info('Removing old files %s', old_files) 120 check_call(['git', 'rm', '-qrf', '--ignore-unmatch'] + old_files) 121 122 # Need to check again because git won't remove directories if they have 123 # non-git files in them. 124 check_call(['rm', '-rf'] + old_files) 125 126 127def install_new_files(branch, build, local_dist, install_list, extracted_list): 128 """Installs the new release.""" 129 for entry in install_list: 130 install_entry(branch, build, local_dist, entry) 131 for entry in extracted_list: 132 if entry.need_strip: 133 check_call(['strip', entry.name]) 134 135 136def install_entry(branch, build, local_dist, entry): 137 """Installs one file specified by entry.""" 138 target = entry.target 139 name = entry.name 140 install_path = entry.install_path 141 need_strip = entry.need_strip 142 need_exec = entry.need_exec 143 need_unzip = entry.need_unzip 144 install_unzipped = entry.install_unzipped 145 146 if build: 147 fetch_artifact(branch, build, target, name) 148 else: 149 copy_artifact(local_dist, target, name) 150 if need_strip: 151 check_call(['strip', name]) 152 if need_exec: 153 check_call(['chmod', 'a+x', name]) 154 155 if install_unzipped: 156 os.makedirs(install_path) 157 zip_file = os.path.basename(name) 158 unzip(zip_file, install_path) 159 check_call(['rm', zip_file]) 160 else: 161 dir = os.path.dirname(install_path) 162 if dir and not os.path.isdir(dir): 163 os.makedirs(dir) 164 shutil.move(os.path.basename(name), install_path) 165 if need_unzip: 166 unzip(install_path, os.path.dirname(install_path)) 167 168def unzip(zip_file, unzip_path): 169 # Add -DD to not extract timestamps that may confuse the build system. 170 check_call(['unzip', '-DD', zip_file, '-d', unzip_path]) 171 172 173def parse_args(parser_modifier=None): 174 """Parses and returns command line arguments.""" 175 parser = argparse.ArgumentParser( 176 epilog='Either --build or --local-dist is required.') 177 178 parser.add_argument( 179 '-b', '--branch', default='aosp-master', 180 help='Branch to pull build from.') 181 parser.add_argument('--build', help='Build number to pull.') 182 parser.add_argument('--local-dist', 183 help='Take prebuilts from this local dist dir instead of ' 184 'using fetch_artifact') 185 parser.add_argument( 186 '--use-current-branch', action='store_true', 187 help='Perform the update in the current branch. Do not repo start.') 188 parser.add_argument( 189 '-v', '--verbose', action='count', default=0, 190 help='Increase output verbosity.') 191 192 if parser_modifier: 193 parser_modifier(parser) 194 195 args = parser.parse_args() 196 if ((not args.build and not args.local_dist) or 197 (args.build and args.local_dist)): 198 sys.exit(parser.format_help()) 199 return args 200 201 202def main(args, work_dir, prebuilts, install_list, extracted_list, commit_message_note=None): 203 """Program entry point.""" 204 205 verbose_map = (logging.WARNING, logging.INFO, logging.DEBUG) 206 verbosity = args.verbose 207 if verbosity > 2: 208 verbosity = 2 209 logging.basicConfig(level=verbose_map[verbosity]) 210 211 local_dist = args.local_dist 212 if local_dist: 213 local_dist = os.path.abspath(local_dist) 214 215 os.chdir(work_dir) 216 217 if not args.use_current_branch: 218 start_branch(args.build) 219 remove_old_files(install_list, extracted_list) 220 install_new_files(args.branch, args.build, local_dist, install_list, extracted_list) 221 files = list_installed_files(install_list, extracted_list) 222 commit(prebuilts, args.branch, args.build, files, commit_message_note) 223