1#!/usr/bin/env vpython3 2# Copyright 2022 The Chromium Authors 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5"""Updates the Fuchsia product bundles to the given revision. Should be used 6in a 'hooks_os' entry so that it only runs when .gclient's target_os includes 7'fuchsia'.""" 8 9import argparse 10import json 11import logging 12import os 13import sys 14 15sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), 16 'test'))) 17 18import common 19import update_sdk 20from compatible_utils import running_unattended 21 22 23# TODO(crbug/1361089): Remove when the old scripts have been deprecated. 24_IMAGE_TO_PRODUCT_BUNDLE = { 25 'qemu.arm64': 26 'terminal.qemu-arm64', 27 'qemu.x64': 28 'terminal.x64', 29} 30 31 32# TODO(crbug/1361089): Remove when the old scripts have been deprecated. 33def convert_to_products(images_list): 34 """Convert image names in the SDK to product bundle names.""" 35 36 product_bundle_list = [] 37 for image in images_list: 38 if image in _IMAGE_TO_PRODUCT_BUNDLE: 39 logging.warning(f'Image name {image} has been deprecated. Use ' 40 f'{_IMAGE_TO_PRODUCT_BUNDLE.get(image)} instead.') 41 product_bundle_list.append(_IMAGE_TO_PRODUCT_BUNDLE[image]) 42 else: 43 if image.endswith('-release'): 44 image = image[:-len('-release')] 45 logging.warning(f'Image name {image}-release has been deprecated. Use ' 46 f'{image} instead.') 47 product_bundle_list.append(image) 48 return product_bundle_list 49 50 51def remove_repositories(repo_names_to_remove): 52 """Removes given repos from repo list. 53 Repo MUST be present in list to succeed. 54 55 Args: 56 repo_names_to_remove: List of repo names (as strings) to remove. 57 """ 58 for repo_name in repo_names_to_remove: 59 common.run_ffx_command(cmd=('repository', 'remove', repo_name), check=True) 60 61 62def get_repositories(): 63 """Lists repositories that are available on disk. 64 65 Also prunes repositories that are listed, but do not have an actual packages 66 directory. 67 68 Returns: 69 List of dictionaries containing info about the repositories. They have the 70 following structure: 71 { 72 'name': <repo name>, 73 'spec': { 74 'type': <type, usually pm>, 75 'path': <path to packages directory> 76 }, 77 } 78 """ 79 80 repos = json.loads( 81 common.run_ffx_command(cmd=('--machine', 'json', 'repository', 'list'), 82 check=True, 83 capture_output=True).stdout.strip()) 84 to_prune = set() 85 sdk_root_abspath = os.path.abspath(os.path.dirname(common.SDK_ROOT)) 86 for repo in repos: 87 # Confirm the path actually exists. If not, prune list. 88 # Also assert the product-bundle repository is for the current repo 89 # (IE within the same directory). 90 if not os.path.exists(repo['spec']['path']): 91 to_prune.add(repo['name']) 92 93 if not repo['spec']['path'].startswith(sdk_root_abspath): 94 to_prune.add(repo['name']) 95 96 repos = [repo for repo in repos if repo['name'] not in to_prune] 97 98 remove_repositories(to_prune) 99 return repos 100 101 102def get_current_signature(image_dir): 103 """Determines the current version of the image, if it exists. 104 105 Returns: 106 The current version, or None if the image is non-existent. 107 """ 108 109 version_file = os.path.join(image_dir, 'product_bundle.json') 110 if os.path.exists(version_file): 111 with open(version_file) as f: 112 return json.load(f)['product_version'] 113 return None 114 115 116# VisibleForTesting 117def internal_hash(): 118 hash_filename = os.path.join(os.path.dirname(__file__), 119 'linux_internal.sdk.sha1') 120 return (open(hash_filename, 'r').read().strip() 121 if os.path.exists(hash_filename) else '') 122 123 124def main(): 125 parser = argparse.ArgumentParser() 126 parser.add_argument('--verbose', 127 '-v', 128 action='store_true', 129 help='Enable debug-level logging.') 130 parser.add_argument( 131 'products', 132 type=str, 133 help='List of product bundles to download, represented as a comma ' 134 'separated list.') 135 parser.add_argument( 136 '--internal', 137 action='store_true', 138 help='Whether the images are coming from internal, it impacts version ' 139 'file, bucket and download location.') 140 args = parser.parse_args() 141 142 logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO) 143 144 # Check whether there's Fuchsia support for this platform. 145 common.get_host_os() 146 147 new_products = convert_to_products(args.products.split(',')) 148 logging.debug('Searching for the following products: %s', str(new_products)) 149 150 logging.debug('Getting new SDK hash') 151 if args.internal: 152 new_hash = internal_hash() 153 else: 154 new_hash = common.get_hash_from_sdk() 155 156 auth_args = [ 157 '--auth', 158 os.path.join(os.path.dirname(__file__), 'get_auth_token.py') 159 ] if args.internal and running_unattended() else [] 160 if args.internal and not running_unattended(): 161 print('*** product bundle v2 requires authentication with your account and ' 162 'it should already open a browser window to do it if you have not ' 163 'granted the permission yet.') 164 for product in new_products: 165 prod, board = product.split('.', 1) 166 if prod.startswith('smart_display_') and board in [ 167 'astro', 'sherlock', 'nelson' 168 ]: 169 # This is a hacky way of keeping the files into the folders matching 170 # the original image name, since the definition is unfortunately in 171 # src-internal. Likely we can download two copies for a smooth 172 # transition, but it would be easier to keep it as-is during the ffx 173 # product v2 migration. 174 # TODO(crbug.com/1496426): Migrate the image download folder away from the 175 # following hack. 176 prod, board = board + '-release', prod 177 if args.internal: 178 # sdk_override.txt does not work for internal images. 179 override_url = None 180 image_dir = os.path.join(common.INTERNAL_IMAGES_ROOT, prod, board) 181 else: 182 override_url = update_sdk.GetSDKOverrideGCSPath() 183 if override_url: 184 logging.debug('Using override file') 185 # TODO(zijiehe): Convert to removesuffix once python 3.9 is supported. 186 if override_url.endswith('/sdk'): 187 override_url = override_url[:-len('/sdk')] 188 image_dir = os.path.join(common.IMAGES_ROOT, prod, board) 189 curr_signature = get_current_signature(image_dir) 190 191 if not override_url and curr_signature == new_hash: 192 continue 193 194 common.make_clean_directory(image_dir) 195 base_url = 'gs://{bucket}/development/{new_hash}'.format( 196 bucket='fuchsia-sdk' if args.internal else 'fuchsia', new_hash=new_hash) 197 lookup_output = common.run_ffx_command(cmd=[ 198 '--machine', 'json', 'product', 'lookup', product, new_hash, 199 '--base-url', override_url or base_url 200 ] + auth_args, 201 check=True, 202 capture_output=True).stdout.strip() 203 download_url = json.loads(lookup_output)['transfer_manifest_url'] 204 # The download_url is purely a timestamp based gs location and is fairly 205 # meaningless, so we log the base_url instead which contains the sdk version 206 # if it's not coming from the sdk_override.txt file. 207 logging.info(f'Downloading {product} from {base_url} and {download_url}.') 208 common.run_ffx_command( 209 cmd=['product', 'download', download_url, image_dir] + auth_args, 210 check=True) 211 212 return 0 213 214 215if __name__ == '__main__': 216 sys.exit(main()) 217