1#!/usr/bin/python3
2
3import argparse
4import glob
5import json
6import os
7import re
8import shlex
9import shutil
10import subprocess
11import sys
12import tempfile
13import zipfile
14
15from collections import defaultdict
16from pathlib import Path
17
18# See go/fetch_artifact for details on this script.
19FETCH_ARTIFACT = '/google/data/ro/projects/android/fetch_artifact'
20COMPAT_REPO = Path('prebuilts/sdk')
21COMPAT_README = Path('extensions/README.md')
22# This build target is used when fetching from a train build (TXXXXXXXX)
23BUILD_TARGET_TRAIN = 'train_build'
24# This build target is used when fetching from a non-train build (XXXXXXXX)
25BUILD_TARGET_CONTINUOUS = 'mainline_modules_sdks-userdebug'
26BUILD_TARGET_CONTINUOUS_MAIN = 'mainline_modules_sdks-{release_config}-userdebug'
27# The glob of sdk artifacts to fetch from remote build
28ARTIFACT_PATTERN = 'mainline-sdks/for-next-build/current/{module_name}/sdk/*.zip'
29# The glob of sdk artifacts to fetch from local build
30ARTIFACT_LOCAL_PATTERN = 'out/dist/mainline-sdks/for-next-build/current/{module_name}/sdk/*.zip'
31ARTIFACT_MODULES_INFO = 'mainline-modules-info.json'
32ARTIFACT_LOCAL_MODULES_INFO = 'out/dist/mainline-modules-info.json'
33COMMIT_TEMPLATE = """Finalize artifacts for extension SDK %d
34
35Import from build id %s.
36
37Generated with:
38$ %s
39
40Bug: %d
41Test: presubmit"""
42
43def fail(*args, **kwargs):
44    print(*args, file=sys.stderr, **kwargs)
45    sys.exit(1)
46
47def fetch_artifacts(build_id, target, artifact_path, dest):
48    print('Fetching %s from %s ...' % (artifact_path, target))
49    fetch_cmd = [FETCH_ARTIFACT]
50    fetch_cmd.extend(['--bid', str(build_id)])
51    fetch_cmd.extend(['--target', target])
52    fetch_cmd.append(artifact_path)
53    fetch_cmd.append(str(dest))
54    print("Running: " + ' '.join(fetch_cmd))
55    try:
56        subprocess.check_output(fetch_cmd, stderr=subprocess.STDOUT)
57    except subprocess.CalledProcessError as e:
58        fail(
59            'FAIL: Unable to retrieve %s artifact for build ID %s for %s target\n Error: %s'
60            % (artifact_path, build_id, target, e.output.decode())
61        )
62
63def fetch_mainline_modules_info_artifact(target, build_id):
64    tmpdir = Path(tempfile.TemporaryDirectory().name)
65    tmpdir.mkdir()
66    if args.local_mode:
67        artifact_path = ARTIFACT_LOCAL_MODULES_INFO
68        print('Copying %s to %s ...' % (artifact_path, tmpdir))
69        shutil.copy(artifact_path, tmpdir)
70    else:
71        artifact_path = ARTIFACT_MODULES_INFO
72        fetch_artifacts(build_id, target, artifact_path, tmpdir)
73    return tmpdir / ARTIFACT_MODULES_INFO
74
75def fetch_module_sdk_artifacts(target, build_id, module_name):
76    tmpdir = Path(tempfile.TemporaryDirectory().name)
77    tmpdir.mkdir()
78    if args.local_mode:
79        artifact_path = ARTIFACT_LOCAL_PATTERN.format(module_name='*')
80        print('Copying %s to %s ...' % (artifact_path, tmpdir))
81        for file in glob.glob(artifact_path):
82            shutil.copy(file, tmpdir)
83    else:
84        artifact_path = ARTIFACT_PATTERN.format(module_name=module_name)
85        fetch_artifacts(build_id, target, artifact_path, tmpdir)
86    return tmpdir
87
88def repo_for_sdk(sdk_filename, mainline_modules_info):
89    for module in mainline_modules_info:
90        if mainline_modules_info[module]["sdk_name"] in sdk_filename:
91            project_path = Path(mainline_modules_info[module]["module_sdk_project"])
92            if args.gantry_download_dir:
93                project_path = args.gantry_download_dir / project_path
94                os.makedirs(project_path , exist_ok = True, mode = 0o777)
95            print(f"module_sdk_path for {module}: {project_path}")
96            return project_path
97
98    fail('"%s" has no valid mapping to any mainline module.' % sdk_filename)
99
100def dir_for_sdk(filename, version):
101    base = str(version)
102    if 'test-exports' in filename:
103        return os.path.join(base, 'test-exports')
104    if 'host-exports' in filename:
105        return os.path.join(base, 'host-exports')
106    return base
107
108def is_ignored(file):
109    # Conscrypt has some legacy API tracking files that we don't consider for extensions.
110    bad_stem_prefixes = ['conscrypt.module.intra.core.api', 'conscrypt.module.platform.api']
111    return any([file.stem.startswith(p) for p in bad_stem_prefixes])
112
113
114def maybe_tweak_compat_stem(file):
115    # For legacy reasons, art and conscrypt txt file names in the SDKs (*.module.public.api)
116    # do not match their expected filename in prebuilts/sdk (art, conscrypt). So rename them
117    # to match.
118    new_stem = file.stem
119    new_stem = new_stem.replace('art.module.public.api', 'art')
120    new_stem = new_stem.replace('conscrypt.module.public.api', 'conscrypt')
121
122    # The stub jar artifacts from official builds are named '*-stubs.jar', but
123    # the convention for the copies in prebuilts/sdk is just '*.jar'. Fix that.
124    new_stem = new_stem.replace('-stubs', '')
125
126    return file.with_stem(new_stem)
127
128parser = argparse.ArgumentParser(description=('Finalize an extension SDK with prebuilts'))
129parser.add_argument('-f', '--finalize_sdk', type=int, required=True, help='The numbered SDK to finalize.')
130parser.add_argument('-c', '--release_config', type=str, help='The release config to use to finalize.')
131parser.add_argument('-b', '--bug', type=int, required=True, help='The bug number to add to the commit message.')
132parser.add_argument('-r', '--readme', required=True, help='Version history entry to add to %s' % (COMPAT_REPO / COMPAT_README))
133parser.add_argument('-a', '--amend_last_commit', action="store_true", help='Amend current HEAD commits instead of making new commits.')
134parser.add_argument('-m', '--modules', action='append', help='Modules to include. Can be provided multiple times, or not at all for all modules.')
135parser.add_argument('-l', '--local_mode', action="store_true", help='Local mode: use locally built artifacts and don\'t upload the result to Gerrit.')
136# This flag is only required when executed via Gantry. It points to the downloaded directory to be used.
137parser.add_argument('-g', '--gantry_download_dir', type=str, help=argparse.SUPPRESS)
138parser.add_argument('bid', help='Build server build ID')
139args = parser.parse_args()
140
141if not os.path.isdir('build/soong') and not args.gantry_download_dir:
142    fail("This script must be run from the top of an Android source tree.")
143
144if args.release_config:
145    BUILD_TARGET_CONTINUOUS = BUILD_TARGET_CONTINUOUS_MAIN.format(release_config=args.release_config)
146build_target = BUILD_TARGET_TRAIN if args.bid[0] == 'T' else BUILD_TARGET_CONTINUOUS
147branch_name = 'finalize-%d' % args.finalize_sdk
148cmdline = shlex.join([x for x in sys.argv if x not in ['-a', '--amend_last_commit', '-l', '--local_mode']])
149commit_message = COMMIT_TEMPLATE % (args.finalize_sdk, args.bid, cmdline, args.bug)
150module_names = args.modules or ['*']
151
152if args.gantry_download_dir:
153    args.gantry_download_dir = Path(args.gantry_download_dir)
154    COMPAT_REPO = args.gantry_download_dir / COMPAT_REPO
155    mainline_modules_info_file = args.gantry_download_dir / ARTIFACT_MODULES_INFO
156else:
157    mainline_modules_info_file = fetch_mainline_modules_info_artifact(build_target, args.bid)
158
159compat_dir = COMPAT_REPO.joinpath('extensions/%d' % args.finalize_sdk)
160if compat_dir.is_dir():
161    print('Removing existing dir %s' % compat_dir)
162    shutil.rmtree(compat_dir)
163
164created_dirs = defaultdict(set)
165with open(mainline_modules_info_file, "r", encoding="utf8",) as file:
166    mainline_modules_info = json.load(file)
167
168for m in module_names:
169    if args.gantry_download_dir:
170        tmpdir = args.gantry_download_dir / "sdk_artifacts"
171    else:
172        tmpdir = fetch_module_sdk_artifacts(build_target, args.bid, m)
173    for f in tmpdir.iterdir():
174        repo = repo_for_sdk(f.name, mainline_modules_info)
175        dir = dir_for_sdk(f.name, args.finalize_sdk)
176        target_dir = repo.joinpath(dir)
177        if target_dir.is_dir():
178            print('Removing existing dir %s' % target_dir)
179            shutil.rmtree(target_dir)
180        with zipfile.ZipFile(tmpdir.joinpath(f)) as zipFile:
181            zipFile.extractall(target_dir)
182
183        # Disable the Android.bp, but keep it for reference / potential future use.
184        shutil.move(target_dir.joinpath('Android.bp'), target_dir.joinpath('Android.bp.auto'))
185
186        print('Created %s' % target_dir)
187        created_dirs[repo].add(dir)
188
189        # Copy api txt files to compat tracking dir
190        src_files = [Path(p) for p in glob.glob(os.path.join(target_dir, 'sdk_library/*/*.txt')) + glob.glob(os.path.join(target_dir, 'sdk_library/*/*.jar'))]
191        for src_file in src_files:
192            if is_ignored(src_file):
193                continue
194            api_type = src_file.parts[-2]
195            dest_dir = compat_dir.joinpath(api_type, 'api') if src_file.suffix == '.txt' else compat_dir.joinpath(api_type)
196            dest_file = maybe_tweak_compat_stem(dest_dir.joinpath(src_file.name))
197            os.makedirs(dest_dir, exist_ok = True)
198            shutil.copy(src_file, dest_file)
199            created_dirs[COMPAT_REPO].add(dest_dir.relative_to(COMPAT_REPO))
200
201if args.local_mode:
202    print('Updated prebuilts using locally built artifacts. Don\'t submit or use for anything besides local testing.')
203    sys.exit(0)
204
205# Do not commit any changes when the script is executed via Gantry.
206if args.gantry_download_dir:
207    sys.exit(0)
208
209subprocess.check_output(['repo', 'start', branch_name] + list(created_dirs.keys()))
210print('Running git commit')
211for repo in created_dirs:
212    git = ['git', '-C', str(repo)]
213    subprocess.check_output(git + ['add'] + list(created_dirs[repo]))
214
215    if repo == COMPAT_REPO:
216        with open(COMPAT_REPO / COMPAT_README, "a") as readme:
217            readme.write(f"- {args.finalize_sdk}: {args.readme}\n")
218        subprocess.check_output(git + ['add', COMPAT_README])
219
220    if args.amend_last_commit:
221        change_id_match = re.search(r'Change-Id: [^\\n]+', str(subprocess.check_output(git + ['log', '-1'])))
222        if change_id_match:
223            change_id = '\n' + change_id_match.group(0)
224        else:
225            fail('FAIL: Unable to find change_id of the last commit.')
226        subprocess.check_output(git + ['commit', '--amend', '-m', commit_message + change_id])
227    else:
228        subprocess.check_output(git + ['commit', '-m', commit_message])
229