1#!/usr/bin/env python3 2# Copyright 2020 The Chromium Authors 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5"""Updates the Fuchsia images to the given revision. Should be used in a 6'hooks_os' entry so that it only runs when .gclient's target_os includes 7'fuchsia'. Note, for a smooth transition, this file automatically adds 8'-release' at the end of the image gcs file name to eliminate the difference 9between product bundle v2 and gce files.""" 10 11# TODO(crbug.com/1496426): Remove this file. 12 13import argparse 14import itertools 15import logging 16import os 17import re 18import subprocess 19import sys 20from typing import Dict, Optional 21 22sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), 23 'test'))) 24 25from common import DIR_SRC_ROOT, IMAGES_ROOT, get_host_os, \ 26 make_clean_directory 27 28from gcs_download import DownloadAndUnpackFromCloudStorage 29 30from update_sdk import GetSDKOverrideGCSPath 31 32IMAGE_SIGNATURE_FILE = '.hash' 33 34 35# TODO(crbug.com/1138433): Investigate whether we can deprecate 36# use of sdk_bucket.txt. 37def GetOverrideCloudStorageBucket(): 38 """Read bucket entry from sdk_bucket.txt""" 39 return ReadFile('sdk-bucket.txt').strip() 40 41 42def ReadFile(filename): 43 """Read a file in this directory.""" 44 with open(os.path.join(os.path.dirname(__file__), filename), 'r') as f: 45 return f.read() 46 47 48def StrExpansion(): 49 return lambda str_value: str_value 50 51 52def VarLookup(local_scope): 53 return lambda var_name: local_scope['vars'][var_name] 54 55 56def GetImageHashList(bucket): 57 """Read filename entries from sdk-hash-files.list (one per line), substitute 58 {platform} in each entry if present, and read from each filename.""" 59 assert (get_host_os() == 'linux') 60 filenames = [ 61 line.strip() for line in ReadFile('sdk-hash-files.list').replace( 62 '{platform}', 'linux_internal').splitlines() 63 ] 64 image_hashes = [ReadFile(filename).strip() for filename in filenames] 65 return image_hashes 66 67 68def ParseDepsDict(deps_content): 69 local_scope = {} 70 global_scope = { 71 'Str': StrExpansion(), 72 'Var': VarLookup(local_scope), 73 'deps_os': {}, 74 } 75 exec(deps_content, global_scope, local_scope) 76 return local_scope 77 78 79def ParseDepsFile(filename): 80 with open(filename, 'rb') as f: 81 deps_content = f.read() 82 return ParseDepsDict(deps_content) 83 84 85def GetImageHash(bucket): 86 """Gets the hash identifier of the newest generation of images.""" 87 if bucket == 'fuchsia-sdk': 88 hashes = GetImageHashList(bucket) 89 return max(hashes) 90 deps_file = os.path.join(DIR_SRC_ROOT, 'DEPS') 91 return ParseDepsFile(deps_file)['vars']['fuchsia_version'].split(':')[1] 92 93 94def GetImageSignature(image_hash, boot_images): 95 return 'gn:{image_hash}:{boot_images}:'.format(image_hash=image_hash, 96 boot_images=boot_images) 97 98 99def GetAllImages(boot_image_names): 100 if not boot_image_names: 101 return 102 103 all_device_types = ['generic', 'qemu'] 104 all_archs = ['x64', 'arm64'] 105 106 images_to_download = set() 107 108 for boot_image in boot_image_names.split(','): 109 components = boot_image.split('.') 110 if len(components) != 2: 111 continue 112 113 device_type, arch = components 114 device_images = all_device_types if device_type == '*' else [device_type] 115 arch_images = all_archs if arch == '*' else [arch] 116 images_to_download.update(itertools.product(device_images, arch_images)) 117 return images_to_download 118 119 120def DownloadBootImages(bucket, image_hash, boot_image_names, image_root_dir): 121 images_to_download = GetAllImages(boot_image_names) 122 for image_to_download in images_to_download: 123 device_type = image_to_download[0] 124 arch = image_to_download[1] 125 image_output_dir = os.path.join(image_root_dir, arch, device_type) 126 if os.path.exists(image_output_dir): 127 continue 128 129 logging.info('Downloading Fuchsia boot images for %s.%s...', device_type, 130 arch) 131 132 # Legacy images use different naming conventions. See fxbug.dev/85552. 133 legacy_delimiter_device_types = ['qemu', 'generic'] 134 if bucket == 'fuchsia-sdk' or \ 135 device_type not in legacy_delimiter_device_types: 136 type_arch_connector = '.' 137 else: 138 type_arch_connector = '-' 139 140 images_tarball_url = 'gs://{bucket}/development/{image_hash}/images/'\ 141 '{device_type}{type_arch_connector}{arch}.tgz'.format( 142 bucket=bucket, image_hash=image_hash, device_type=device_type, 143 type_arch_connector=type_arch_connector, arch=arch) 144 try: 145 DownloadAndUnpackFromCloudStorage(images_tarball_url, image_output_dir) 146 except subprocess.CalledProcessError as e: 147 logging.exception('Failed to download image %s from URL: %s', 148 image_to_download, images_tarball_url) 149 raise e 150 151 152def _GetImageOverrideInfo() -> Optional[Dict[str, str]]: 153 """Get the bucket location from sdk_override.txt.""" 154 location = GetSDKOverrideGCSPath() 155 if not location: 156 return None 157 158 m = re.match(r'gs://([^/]+)/development/([^/]+)/?(?:sdk)?', location) 159 if not m: 160 raise ValueError('Badly formatted image override location %s' % location) 161 162 return { 163 'bucket': m.group(1), 164 'image_hash': m.group(2), 165 } 166 167 168def GetImageLocationInfo(default_bucket: str, 169 allow_override: bool = True) -> Dict[str, str]: 170 """Figures out where to pull the image from. 171 172 Defaults to the provided default bucket and generates the hash from defaults. 173 If sdk_override.txt exists (and is allowed) it uses that bucket instead. 174 175 Args: 176 default_bucket: a given default for what bucket to use 177 allow_override: allow SDK override to be used. 178 179 Returns: 180 A dictionary containing the bucket and image_hash 181 """ 182 # if sdk_override.txt exists (and is allowed) use the image from that bucket. 183 if allow_override: 184 override = _GetImageOverrideInfo() 185 if override: 186 return override 187 188 # Use the bucket in sdk-bucket.txt if an entry exists. 189 # Otherwise use the default bucket. 190 bucket = GetOverrideCloudStorageBucket() or default_bucket 191 return { 192 'bucket': bucket, 193 'image_hash': GetImageHash(bucket), 194 } 195 196 197def main(): 198 parser = argparse.ArgumentParser() 199 parser.add_argument('--verbose', 200 '-v', 201 action='store_true', 202 help='Enable debug-level logging.') 203 parser.add_argument( 204 '--boot-images', 205 type=str, 206 required=True, 207 help='List of boot images to download, represented as a comma separated ' 208 'list. Wildcards are allowed. ') 209 parser.add_argument( 210 '--default-bucket', 211 type=str, 212 default='fuchsia', 213 help='The Google Cloud Storage bucket in which the Fuchsia images are ' 214 'stored. Entry in sdk-bucket.txt will override this flag.') 215 parser.add_argument( 216 '--image-root-dir', 217 default=IMAGES_ROOT, 218 help='Specify the root directory of the downloaded images. Optional') 219 parser.add_argument( 220 '--allow-override', 221 action='store_true', 222 help='Whether sdk_override.txt can be used for fetching the image, if ' 223 'it exists.') 224 args = parser.parse_args() 225 226 logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO) 227 228 # If no boot images need to be downloaded, exit. 229 if not args.boot_images: 230 return 0 231 232 for index, item in enumerate(args.boot_images): 233 # The gclient configuration is in the src-internal and cannot be changed 234 # atomically with the test script in src. So the update_images.py needs to 235 # support both scenarios before being fully deprecated for older milestones. 236 if not item.endswith('-release'): 237 args.boot_images[index] = item + '-release' 238 239 # Check whether there's Fuchsia support for this platform. 240 get_host_os() 241 image_info = GetImageLocationInfo(args.default_bucket, args.allow_override) 242 243 bucket = image_info['bucket'] 244 image_hash = image_info['image_hash'] 245 246 if not image_hash: 247 return 1 248 249 signature_filename = os.path.join(args.image_root_dir, IMAGE_SIGNATURE_FILE) 250 current_signature = (open(signature_filename, 'r').read().strip() 251 if os.path.exists(signature_filename) else '') 252 new_signature = GetImageSignature(image_hash, args.boot_images) 253 if current_signature != new_signature: 254 logging.info('Downloading Fuchsia images %s from bucket %s...', image_hash, 255 bucket) 256 make_clean_directory(args.image_root_dir) 257 258 try: 259 DownloadBootImages(bucket, image_hash, args.boot_images, 260 args.image_root_dir) 261 with open(signature_filename, 'w') as f: 262 f.write(new_signature) 263 except subprocess.CalledProcessError as e: 264 logging.exception("command '%s' failed with status %d.%s", 265 ' '.join(e.cmd), e.returncode, 266 ' Details: ' + e.output if e.output else '') 267 raise e 268 else: 269 logging.info('Signatures matched! Got %s', new_signature) 270 271 return 0 272 273 274if __name__ == '__main__': 275 sys.exit(main()) 276