1# 2# Copyright (C) 2018 The Android Open Source Project 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15"""A commandline tool to check and update packages in external/ 16 17Example usage: 18updater.sh checkall 19updater.sh update kotlinc 20updater.sh update --refresh --keep_date rust/crates/libc 21""" 22 23import argparse 24from collections.abc import Iterable 25import json 26import logging 27import os 28import subprocess 29import textwrap 30import time 31from typing import Dict, Iterator, List, Union, Tuple, Type 32from pathlib import Path 33 34from base_updater import Updater 35from color import Color, color_string 36from crates_updater import CratesUpdater 37from git_updater import GitUpdater 38from github_archive_updater import GithubArchiveUpdater 39import fileutils 40import git_utils 41# pylint: disable=import-error 42import metadata_pb2 # type: ignore 43import updater_utils 44 45UPDATERS: List[Type[Updater]] = [ 46 CratesUpdater, 47 GithubArchiveUpdater, 48 GitUpdater, 49] 50 51TMP_BRANCH_NAME = 'tmp_auto_upgrade' 52 53 54def build_updater(proj_path: Path) -> Tuple[Updater, metadata_pb2.MetaData]: 55 """Build updater for a project specified by proj_path. 56 57 Reads and parses METADATA file. And builds updater based on the information. 58 59 Args: 60 proj_path: Absolute or relative path to the project. 61 62 Returns: 63 The updater object built. None if there's any error. 64 """ 65 66 proj_path = fileutils.get_absolute_project_path(proj_path) 67 metadata = fileutils.read_metadata(proj_path) 68 metadata = fileutils.convert_url_to_identifier(metadata) 69 updater = updater_utils.create_updater(metadata, proj_path, UPDATERS) 70 return updater, metadata 71 72 73def commit_message_generator(project_name: str, version: str, path: str, bug: int | None = None) -> str: 74 header = f"Upgrade {project_name} to {version}\n" 75 body = textwrap.dedent(f""" 76 This project was upgraded with external_updater. 77 Usage: tools/external_updater/updater.sh update external/{path} 78 For more info, check https://cs.android.com/android/platform/superproject/main/+/main:tools/external_updater/README.md\n\n""") 79 if bug is None: 80 footer = "Test: TreeHugger" 81 else: 82 footer = f"Bug: {bug}\nTest: TreeHugger" 83 return header + body + footer 84 85 86def _do_update(args: argparse.Namespace, updater: Updater, 87 metadata: metadata_pb2.MetaData) -> None: 88 full_path = updater.project_path 89 90 if not args.keep_local_changes: 91 git_utils.detach_to_android_head(full_path) 92 if TMP_BRANCH_NAME in git_utils.list_local_branches(full_path): 93 git_utils.delete_branch(full_path, TMP_BRANCH_NAME) 94 git_utils.reset_hard(full_path) 95 git_utils.clean(full_path) 96 git_utils.start_branch(full_path, TMP_BRANCH_NAME) 97 98 try: 99 updater.update() 100 101 updated_metadata = updater.update_metadata(metadata) 102 fileutils.write_metadata(full_path, updated_metadata, args.keep_date) 103 git_utils.add_file(full_path, 'METADATA') 104 105 try: 106 rel_proj_path = str(fileutils.get_relative_project_path(full_path)) 107 except ValueError: 108 # Absolute paths to other trees will not be relative to our tree. There are 109 # not portable instructions for upgrading that project, since the path will 110 # differ between machines (or checkouts). 111 rel_proj_path = "<absolute path to project>" 112 commit_message = commit_message_generator(metadata.name, updater.latest_version, rel_proj_path, args.bug) 113 git_utils.remove_gitmodules(full_path) 114 git_utils.add_file(full_path, '*') 115 git_utils.commit(full_path, commit_message, args.no_verify) 116 117 if not args.skip_post_update: 118 updater_utils.run_post_update(full_path, full_path) 119 git_utils.add_file(full_path, '*') 120 git_utils.commit_amend(full_path) 121 122 if args.build: 123 try: 124 updater_utils.build(full_path) 125 except subprocess.CalledProcessError: 126 logging.exception("Build failed, aborting upload") 127 return 128 except Exception as err: 129 if updater.rollback(): 130 print('Rolled back.') 131 raise err 132 133 if not args.no_upload: 134 git_utils.push(full_path, args.remote_name, updater.has_errors) 135 136 137def has_new_version(updater: Updater) -> bool: 138 """Checks if a newer version of the project is available.""" 139 if updater.latest_version is not None and updater.current_version != updater.latest_version: 140 return True 141 return False 142 143 144def print_project_status(updater: Updater) -> None: 145 """Prints the current status of the project on console.""" 146 147 current_version = updater.current_version 148 latest_version = updater.latest_version 149 alternative_latest_version = updater.alternative_latest_version 150 151 print(f'Current version: {current_version}') 152 print('Latest version: ', end='') 153 if not latest_version: 154 print(color_string('Not available', Color.STALE)) 155 else: 156 print(latest_version) 157 if alternative_latest_version is not None: 158 print(f'Alternative latest version: {alternative_latest_version}') 159 if has_new_version(updater): 160 print(color_string('Out of date!', Color.STALE)) 161 else: 162 print(color_string('Up to date.', Color.FRESH)) 163 164 165def find_ver_types(current_version: str) -> Tuple[str, str]: 166 if git_utils.is_commit(current_version): 167 alternative_ver_type = 'tag' 168 latest_ver_type = 'sha' 169 else: 170 alternative_ver_type = 'sha' 171 latest_ver_type = 'tag' 172 return latest_ver_type, alternative_ver_type 173 174 175def use_alternative_version(updater: Updater) -> bool: 176 """This function only runs when there is an alternative version available.""" 177 178 latest_ver_type, alternative_ver_type = find_ver_types(updater.current_version) 179 latest_version = updater.latest_version 180 alternative_version = updater.alternative_latest_version 181 new_version_available = has_new_version(updater) 182 183 out_of_date_question = f'Would you like to upgrade to {alternative_ver_type} {alternative_version} instead of {latest_ver_type} {latest_version}? (yes/no)\n' 184 up_to_date_question = f'Would you like to upgrade to {alternative_ver_type} {alternative_version}? (yes/no)\n' 185 recom_message = color_string(f'We recommend upgrading to {alternative_ver_type} {alternative_version} instead. ', Color.FRESH) 186 not_recom_message = color_string(f'We DO NOT recommend upgrading to {alternative_ver_type} {alternative_version}. ', Color.STALE) 187 188 # If alternative_version is not None, there are four possible 189 # scenarios: 190 # Scenario 1, out of date, we recommend switching to tag: 191 # Current version: sha1 192 # Latest version: sha2 193 # Alternative latest version: tag 194 195 # Scenario 2, up to date, we DO NOT recommend switching to sha. 196 # Current version: tag1 197 # Latest version: tag1 198 # Alternative latest version: sha 199 200 # Scenario 3, out of date, we DO NOT recommend switching to sha. 201 # Current version: tag1 202 # Latest version: tag2 203 # Alternative latest version: sha 204 205 # Scenario 4, out of date, no recommendations at all 206 # Current version: sha1 207 # Latest version: No tag found or a tag that doesn't belong to any branch 208 # Alternative latest version: sha 209 210 if alternative_ver_type == 'tag': 211 warning = out_of_date_question + recom_message 212 else: 213 if not new_version_available: 214 warning = up_to_date_question + not_recom_message 215 else: 216 if not latest_version: 217 warning = up_to_date_question 218 else: 219 warning = out_of_date_question + not_recom_message 220 221 answer = input(warning) 222 if "yes".startswith(answer.lower()): 223 return True 224 elif answer.lower().startswith("no"): 225 return False 226 # If user types something that is not "yes" or "no" or something similar, abort. 227 else: 228 raise ValueError(f"Invalid input: {answer}") 229 230 231def check_and_update(args: argparse.Namespace, 232 proj_path: Path, 233 update_lib=False) -> Union[Updater, str]: 234 """Checks updates for a project. 235 236 Args: 237 args: commandline arguments 238 proj_path: Absolute or relative path to the project. 239 update_lib: If false, will only check for new version, but not update. 240 """ 241 242 try: 243 canonical_path = fileutils.canonicalize_project_path(proj_path) 244 print(f'Checking {canonical_path}...') 245 updater, metadata = build_updater(proj_path) 246 updater.check() 247 248 new_version_available = has_new_version(updater) 249 print_project_status(updater) 250 251 if update_lib: 252 if args.custom_version is not None: 253 updater.set_custom_version(args.custom_version) 254 print(f"Upgrading to custom version {args.custom_version}") 255 elif args.refresh: 256 updater.refresh_without_upgrading() 257 elif new_version_available: 258 if updater.alternative_latest_version is not None: 259 if use_alternative_version(updater): 260 updater.set_new_version(updater.alternative_latest_version) 261 else: 262 return updater 263 _do_update(args, updater, metadata) 264 return updater 265 266 # pylint: disable=broad-except 267 except Exception as err: 268 logging.exception("Failed to check or update %s", proj_path) 269 return str(err) 270 271 272def check_and_update_path(args: argparse.Namespace, paths: Iterable[Path], 273 update_lib: bool, 274 delay: int) -> Dict[str, Dict[str, str]]: 275 results = {} 276 for path in paths: 277 res = {} 278 updater = check_and_update(args, path, update_lib) 279 if isinstance(updater, str): 280 res['error'] = updater 281 else: 282 res['current'] = updater.current_version 283 res['latest'] = updater.latest_version 284 results[str(fileutils.canonicalize_project_path(path))] = res 285 time.sleep(delay) 286 return results 287 288 289def _list_all_metadata() -> Iterator[str]: 290 for path, dirs, files in os.walk(fileutils.external_path()): 291 if fileutils.METADATA_FILENAME in files: 292 # Skip sub directories. 293 dirs[:] = [] 294 yield path 295 dirs.sort(key=lambda d: d.lower()) 296 297 298def write_json(json_file: str, results: Dict[str, Dict[str, str]]) -> None: 299 """Output a JSON report.""" 300 with Path(json_file).open('w', encoding='utf-8') as res_file: 301 json.dump(results, res_file, sort_keys=True, indent=4) 302 303 304def validate(args: argparse.Namespace) -> None: 305 """Handler for validate command.""" 306 paths = fileutils.resolve_command_line_paths(args.paths) 307 try: 308 canonical_path = fileutils.canonicalize_project_path(paths[0]) 309 print(f'Validating {canonical_path}') 310 updater, _ = build_updater(paths[0]) 311 print(updater.validate()) 312 except Exception: # pylint: disable=broad-exception-caught 313 logging.exception("Failed to check or update %s", paths) 314 315 316def check(args: argparse.Namespace) -> None: 317 """Handler for check command.""" 318 if args.all: 319 paths = [Path(p) for p in _list_all_metadata()] 320 else: 321 paths = fileutils.resolve_command_line_paths(args.paths) 322 results = check_and_update_path(args, paths, False, args.delay) 323 324 if args.json_output is not None: 325 write_json(args.json_output, results) 326 327 328def update(args: argparse.Namespace) -> None: 329 """Handler for update command.""" 330 all_paths = fileutils.resolve_command_line_paths(args.paths) 331 # Remove excluded paths. 332 excludes = set() if args.exclude is None else set(args.exclude) 333 filtered_paths = [path for path in all_paths 334 if not path.name in excludes] 335 # Now we can update each path. 336 results = check_and_update_path(args, filtered_paths, True, 0) 337 338 if args.json_output is not None: 339 write_json(args.json_output, results) 340 341 342def parse_args() -> argparse.Namespace: 343 """Parses commandline arguments.""" 344 345 parser = argparse.ArgumentParser( 346 description='Check updates for third party projects in external/.') 347 subparsers = parser.add_subparsers(dest='cmd') 348 subparsers.required = True 349 350 diff_parser = subparsers.add_parser('validate', 351 help='Check if aosp version is what it claims to be.') 352 diff_parser.add_argument( 353 'paths', 354 nargs='*', 355 help='Paths of the project. ' 356 'Relative paths will be resolved from external/.') 357 diff_parser.set_defaults(func=validate) 358 359 # Creates parser for check command. 360 check_parser = subparsers.add_parser('check', 361 help='Check update for one project.') 362 check_parser.add_argument( 363 'paths', 364 nargs='*', 365 help='Paths of the project. ' 366 'Relative paths will be resolved from external/.') 367 check_parser.add_argument('--json-output', 368 help='Path of a json file to write result to.') 369 check_parser.add_argument( 370 '--all', 371 action='store_true', 372 help='If set, check updates for all supported projects.') 373 check_parser.add_argument( 374 '--delay', 375 default=0, 376 type=int, 377 help='Time in seconds to wait between checking two projects.') 378 check_parser.set_defaults(func=check) 379 380 # Creates parser for update command. 381 update_parser = subparsers.add_parser('update', help='Update one project.') 382 update_parser.add_argument( 383 'paths', 384 nargs='*', 385 help='Paths of the project as globs. ' 386 'Relative paths will be resolved from external/.') 387 update_parser.add_argument('--json-output', 388 help='Path of a json file to write result to.') 389 update_parser.add_argument( 390 '--refresh', 391 help='Run update and refresh to the current version.', 392 action='store_true') 393 update_parser.add_argument( 394 '--keep-date', 395 help='Run update and do not change date in METADATA.', 396 action='store_true') 397 update_parser.add_argument('--no-upload', 398 action='store_true', 399 help='Does not upload to Gerrit after upgrade') 400 update_parser.add_argument('--keep-local-changes', 401 action='store_true', 402 help='Updates the current branch') 403 update_parser.add_argument('--skip-post-update', 404 action='store_true', 405 help='Skip post_update script') 406 update_parser.add_argument('--no-build', 407 action='store_false', 408 dest='build', 409 help='Skip building') 410 update_parser.add_argument('--no-verify', 411 action='store_true', 412 help='Pass --no-verify to git commit') 413 update_parser.add_argument('--bug', 414 type=int, 415 help='Bug number for this update') 416 update_parser.add_argument('--custom-version', 417 type=str, 418 help='Custom version we want to upgrade to.') 419 update_parser.add_argument('--remote-name', 420 default='aosp', 421 required=False, 422 help='Upstream remote name.') 423 update_parser.add_argument('--exclude', 424 action='append', 425 help='Names of projects to exclude. ' 426 'These are just the final part of the path ' 427 'with no directories.') 428 update_parser.set_defaults(func=update) 429 430 return parser.parse_args() 431 432 433def main() -> None: 434 """The main entry.""" 435 436 args = parse_args() 437 args.func(args) 438 439 440if __name__ == '__main__': 441 main() 442