xref: /aosp_15_r20/external/clang/build.py (revision 67e74705e28f6214e480b399dd47ea732279e315)
1*67e74705SXin Li#!/usr/bin/env python
2*67e74705SXin Li#
3*67e74705SXin Li# Copyright (C) 2015 The Android Open Source Project
4*67e74705SXin Li#
5*67e74705SXin Li# Licensed under the Apache License, Version 2.0 (the "License");
6*67e74705SXin Li# you may not use this file except in compliance with the License.
7*67e74705SXin Li# You may obtain a copy of the License at
8*67e74705SXin Li#
9*67e74705SXin Li#      http://www.apache.org/licenses/LICENSE-2.0
10*67e74705SXin Li#
11*67e74705SXin Li# Unless required by applicable law or agreed to in writing, software
12*67e74705SXin Li# distributed under the License is distributed on an "AS IS" BASIS,
13*67e74705SXin Li# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14*67e74705SXin Li# See the License for the specific language governing permissions and
15*67e74705SXin Li# limitations under the License.
16*67e74705SXin Li#
17*67e74705SXin Li"""Builds the Android Clang toolchain."""
18*67e74705SXin Liimport argparse
19*67e74705SXin Liimport glob
20*67e74705SXin Liimport logging
21*67e74705SXin Liimport multiprocessing
22*67e74705SXin Liimport os
23*67e74705SXin Liimport pprint
24*67e74705SXin Liimport subprocess
25*67e74705SXin Liimport sys
26*67e74705SXin Li
27*67e74705SXin Liimport version
28*67e74705SXin Li
29*67e74705SXin Li
30*67e74705SXin Li# Disable all the "too many/few methods/parameters" warnings and the like.
31*67e74705SXin Li# pylint: disable=design
32*67e74705SXin Li
33*67e74705SXin Li# Disable lint warnings for todo comments and the like.
34*67e74705SXin Li# pylint: disable=fixme
35*67e74705SXin Li
36*67e74705SXin Li# TODO: Add docstrings?
37*67e74705SXin Li# pylint: disable=missing-docstring
38*67e74705SXin Li
39*67e74705SXin Li
40*67e74705SXin LiTHIS_DIR = os.path.realpath(os.path.dirname(__file__))
41*67e74705SXin LiORIG_ENV = dict(os.environ)
42*67e74705SXin Li
43*67e74705SXin Li
44*67e74705SXin Liclass Config(object):
45*67e74705SXin Li    """Container for global configuration options."""
46*67e74705SXin Li
47*67e74705SXin Li    # Set True to skip all actions (log only). Controlled by --dry-run.
48*67e74705SXin Li    dry_run = False
49*67e74705SXin Li
50*67e74705SXin Li
51*67e74705SXin Lidef logger():
52*67e74705SXin Li    """Returns the default logger for the module."""
53*67e74705SXin Li    return logging.getLogger(__name__)
54*67e74705SXin Li
55*67e74705SXin Li
56*67e74705SXin Lidef android_path(*args):
57*67e74705SXin Li    return os.path.realpath(os.path.join(THIS_DIR, '../..', *args))
58*67e74705SXin Li
59*67e74705SXin Li
60*67e74705SXin Lidef build_path(*args):
61*67e74705SXin Li    # Our multistage build directories will be placed under OUT_DIR if it is in
62*67e74705SXin Li    # the environment. By default they will be placed under
63*67e74705SXin Li    # $ANDROID_BUILD_TOP/out.
64*67e74705SXin Li    top_out = ORIG_ENV.get('OUT_DIR', 'out')
65*67e74705SXin Li    return os.path.join(top_out, *args)
66*67e74705SXin Li
67*67e74705SXin Li
68*67e74705SXin Lidef short_version():
69*67e74705SXin Li    return '.'.join([version.major, version.minor])
70*67e74705SXin Li
71*67e74705SXin Li
72*67e74705SXin Lidef long_version():
73*67e74705SXin Li    return '.'.join([version.major, version.minor, version.patch])
74*67e74705SXin Li
75*67e74705SXin Li
76*67e74705SXin Lidef check_call(cmd, *args, **kwargs):
77*67e74705SXin Li    """Proxy for subprocess.check_call with logging and dry-run support."""
78*67e74705SXin Li    import subprocess
79*67e74705SXin Li    logger().info('check_call: %s', ' '.join(cmd))
80*67e74705SXin Li    if 'env' in kwargs:
81*67e74705SXin Li        # Rather than dump the whole environment to the terminal every time,
82*67e74705SXin Li        # just print the difference between this call and our environment.
83*67e74705SXin Li        # Note that this will not include environment that was *removed* from
84*67e74705SXin Li        # os.environ.
85*67e74705SXin Li        extra_env = dict(set(kwargs['env'].items()) - set(os.environ.items()))
86*67e74705SXin Li        if len(extra_env) > 0:
87*67e74705SXin Li            logger().info('check_call additional env:\n%s',
88*67e74705SXin Li                          pprint.pformat(extra_env))
89*67e74705SXin Li    if not Config.dry_run:
90*67e74705SXin Li        subprocess.check_call(cmd, *args, **kwargs)
91*67e74705SXin Li
92*67e74705SXin Li
93*67e74705SXin Lidef install_file(src, dst):
94*67e74705SXin Li    """Proxy for shutil.copy2 with logging and dry-run support."""
95*67e74705SXin Li    import shutil
96*67e74705SXin Li    logger().info('copy %s %s', src, dst)
97*67e74705SXin Li    if not Config.dry_run:
98*67e74705SXin Li        shutil.copy2(src, dst)
99*67e74705SXin Li
100*67e74705SXin Li
101*67e74705SXin Lidef install_directory(src, dst):
102*67e74705SXin Li    """Proxy for shutil.copytree with logging and dry-run support."""
103*67e74705SXin Li    import shutil
104*67e74705SXin Li    logger().info('copytree %s %s', src, dst)
105*67e74705SXin Li    if not Config.dry_run:
106*67e74705SXin Li        shutil.copytree(src, dst)
107*67e74705SXin Li
108*67e74705SXin Li
109*67e74705SXin Lidef rmtree(path):
110*67e74705SXin Li    """Proxy for shutil.rmtree with logging and dry-run support."""
111*67e74705SXin Li    import shutil
112*67e74705SXin Li    logger().info('rmtree %s', path)
113*67e74705SXin Li    if not Config.dry_run:
114*67e74705SXin Li        shutil.rmtree(path)
115*67e74705SXin Li
116*67e74705SXin Li
117*67e74705SXin Lidef rename(src, dst):
118*67e74705SXin Li    """Proxy for os.rename with logging and dry-run support."""
119*67e74705SXin Li    logger().info('rename %s %s', src, dst)
120*67e74705SXin Li    if not Config.dry_run:
121*67e74705SXin Li        os.rename(src, dst)
122*67e74705SXin Li
123*67e74705SXin Li
124*67e74705SXin Lidef makedirs(path):
125*67e74705SXin Li    """Proxy for os.makedirs with logging and dry-run support."""
126*67e74705SXin Li    logger().info('makedirs %s', path)
127*67e74705SXin Li    if not Config.dry_run:
128*67e74705SXin Li        os.makedirs(path)
129*67e74705SXin Li
130*67e74705SXin Li
131*67e74705SXin Lidef symlink(src, dst):
132*67e74705SXin Li    """Proxy for os.symlink with logging and dry-run support."""
133*67e74705SXin Li    logger().info('symlink %s %s', src, dst)
134*67e74705SXin Li    if not Config.dry_run:
135*67e74705SXin Li        os.symlink(src, dst)
136*67e74705SXin Li
137*67e74705SXin Li
138*67e74705SXin Lidef build(out_dir, prebuilts_path=None, prebuilts_version=None,
139*67e74705SXin Li          build_all_clang_tools=None, build_all_llvm_tools=None,
140*67e74705SXin Li          debug_clang=None, max_jobs=multiprocessing.cpu_count()):
141*67e74705SXin Li    products = (
142*67e74705SXin Li        'aosp_arm',
143*67e74705SXin Li        'aosp_arm64',
144*67e74705SXin Li        'aosp_mips',
145*67e74705SXin Li        'aosp_mips64',
146*67e74705SXin Li        'aosp_x86',
147*67e74705SXin Li        'aosp_x86_64',
148*67e74705SXin Li    )
149*67e74705SXin Li    for product in products:
150*67e74705SXin Li        build_product(out_dir, product, prebuilts_path, prebuilts_version,
151*67e74705SXin Li                      build_all_clang_tools, build_all_llvm_tools, debug_clang,
152*67e74705SXin Li                      max_jobs)
153*67e74705SXin Li
154*67e74705SXin Li
155*67e74705SXin Lidef build_product(out_dir, product, prebuilts_path, prebuilts_version,
156*67e74705SXin Li                  build_all_clang_tools, build_all_llvm_tools, debug_clang,
157*67e74705SXin Li                  max_jobs):
158*67e74705SXin Li    env = dict(ORIG_ENV)
159*67e74705SXin Li    env['DISABLE_LLVM_DEVICE_BUILDS'] = 'true'
160*67e74705SXin Li    env['DISABLE_RELOCATION_PACKER'] = 'true'
161*67e74705SXin Li    env['FORCE_BUILD_LLVM_COMPONENTS'] = 'true'
162*67e74705SXin Li    env['FORCE_BUILD_SANITIZER_SHARED_OBJECTS'] = 'true'
163*67e74705SXin Li    env['OUT_DIR'] = out_dir
164*67e74705SXin Li    env['SKIP_LLVM_TESTS'] = 'true'
165*67e74705SXin Li    env['SOONG_ALLOW_MISSING_DEPENDENCIES'] = 'true'
166*67e74705SXin Li    env['TARGET_BUILD_VARIANT'] = 'userdebug'
167*67e74705SXin Li    env['TARGET_PRODUCT'] = product
168*67e74705SXin Li
169*67e74705SXin Li    if debug_clang:
170*67e74705SXin Li        env['FORCE_BUILD_LLVM_DEBUG'] = 'true'
171*67e74705SXin Li        env['FORCE_BUILD_LLVM_DISABLE_NDEBUG'] = 'true'
172*67e74705SXin Li
173*67e74705SXin Li    overrides = []
174*67e74705SXin Li    if prebuilts_path is not None:
175*67e74705SXin Li        overrides.append('LLVM_PREBUILTS_BASE={}'.format(prebuilts_path))
176*67e74705SXin Li    if prebuilts_version is not None:
177*67e74705SXin Li        overrides.append('LLVM_PREBUILTS_VERSION={}'.format(prebuilts_version))
178*67e74705SXin Li
179*67e74705SXin Li    # Use at least 1 and at most all available CPUs (sanitize the user input).
180*67e74705SXin Li    jobs_arg = '-j{}'.format(
181*67e74705SXin Li        max(1, min(max_jobs, multiprocessing.cpu_count())))
182*67e74705SXin Li
183*67e74705SXin Li    targets = ['clang-toolchain-minimal']
184*67e74705SXin Li    if build_all_clang_tools:
185*67e74705SXin Li        targets += ['clang-toolchain-full']
186*67e74705SXin Li    if build_all_llvm_tools:
187*67e74705SXin Li        targets += ['llvm-tools']
188*67e74705SXin Li    check_call(['make', jobs_arg] + overrides + targets,
189*67e74705SXin Li               cwd=android_path(), env=env)
190*67e74705SXin Li
191*67e74705SXin Li
192*67e74705SXin Lidef package_toolchain(build_dir, build_name, host, dist_dir):
193*67e74705SXin Li    package_name = 'clang-' + build_name
194*67e74705SXin Li    install_host_dir = build_path('install', host)
195*67e74705SXin Li    install_dir = os.path.join(install_host_dir, package_name)
196*67e74705SXin Li
197*67e74705SXin Li    # Remove any previously installed toolchain so it doesn't pollute the
198*67e74705SXin Li    # build.
199*67e74705SXin Li    if os.path.exists(install_host_dir):
200*67e74705SXin Li        rmtree(install_host_dir)
201*67e74705SXin Li
202*67e74705SXin Li    install_toolchain(build_dir, install_dir, host, True)
203*67e74705SXin Li
204*67e74705SXin Li    version_file_path = os.path.join(install_dir, 'AndroidVersion.txt')
205*67e74705SXin Li    with open(version_file_path, 'w') as version_file:
206*67e74705SXin Li        version_file.write('{}.{}.{}\n'.format(
207*67e74705SXin Li            version.major, version.minor, version.patch))
208*67e74705SXin Li
209*67e74705SXin Li    tarball_name = package_name + '-' + host
210*67e74705SXin Li    package_path = os.path.join(dist_dir, tarball_name) + '.tar.bz2'
211*67e74705SXin Li    logger().info('Packaging %s', package_path)
212*67e74705SXin Li    args = [
213*67e74705SXin Li        'tar', '-cjC', install_host_dir, '-f', package_path, package_name
214*67e74705SXin Li    ]
215*67e74705SXin Li    check_call(args)
216*67e74705SXin Li
217*67e74705SXin Li
218*67e74705SXin Lidef install_minimal_toolchain(build_dir, install_dir, host, strip):
219*67e74705SXin Li    install_built_host_files(build_dir, install_dir, host, strip, minimal=True)
220*67e74705SXin Li    install_headers(build_dir, install_dir, host)
221*67e74705SXin Li    install_sanitizers(build_dir, install_dir, host)
222*67e74705SXin Li
223*67e74705SXin Li
224*67e74705SXin Lidef install_toolchain(build_dir, install_dir, host, strip):
225*67e74705SXin Li    install_built_host_files(build_dir, install_dir, host, strip)
226*67e74705SXin Li    install_compiler_wrapper(install_dir, host)
227*67e74705SXin Li    install_sanitizer_scripts(install_dir)
228*67e74705SXin Li    install_scan_scripts(install_dir)
229*67e74705SXin Li    install_analyzer_scripts(install_dir)
230*67e74705SXin Li    install_headers(build_dir, install_dir, host)
231*67e74705SXin Li    install_profile_rt(build_dir, install_dir, host)
232*67e74705SXin Li    install_sanitizers(build_dir, install_dir, host)
233*67e74705SXin Li    install_sanitizer_tests(build_dir, install_dir, host)
234*67e74705SXin Li    install_libomp(build_dir, install_dir, host)
235*67e74705SXin Li    install_license_files(install_dir)
236*67e74705SXin Li    install_repo_prop(install_dir)
237*67e74705SXin Li
238*67e74705SXin Li
239*67e74705SXin Lidef get_built_host_files(host, minimal):
240*67e74705SXin Li    is_windows = host.startswith('windows')
241*67e74705SXin Li    is_darwin = host.startswith('darwin-x86')
242*67e74705SXin Li    bin_ext = '.exe' if is_windows else ''
243*67e74705SXin Li
244*67e74705SXin Li    if is_windows:
245*67e74705SXin Li        lib_ext = '.dll'
246*67e74705SXin Li    elif is_darwin:
247*67e74705SXin Li        lib_ext = '.dylib'
248*67e74705SXin Li    else:
249*67e74705SXin Li        lib_ext = '.so'
250*67e74705SXin Li
251*67e74705SXin Li    built_files = [
252*67e74705SXin Li        'bin/clang' + bin_ext,
253*67e74705SXin Li        'bin/clang++' + bin_ext,
254*67e74705SXin Li    ]
255*67e74705SXin Li    if not is_windows:
256*67e74705SXin Li        built_files.extend(['lib64/libc++' + lib_ext])
257*67e74705SXin Li
258*67e74705SXin Li    if minimal:
259*67e74705SXin Li        return built_files
260*67e74705SXin Li
261*67e74705SXin Li    built_files.extend([
262*67e74705SXin Li        'bin/clang-format' + bin_ext,
263*67e74705SXin Li        'bin/clang-tidy' + bin_ext,
264*67e74705SXin Li    ])
265*67e74705SXin Li
266*67e74705SXin Li    if is_windows:
267*67e74705SXin Li        built_files.extend([
268*67e74705SXin Li            'bin/clang_32' + bin_ext,
269*67e74705SXin Li        ])
270*67e74705SXin Li    else:
271*67e74705SXin Li        built_files.extend([
272*67e74705SXin Li            'bin/FileCheck' + bin_ext,
273*67e74705SXin Li            'bin/llvm-as' + bin_ext,
274*67e74705SXin Li            'bin/llvm-dis' + bin_ext,
275*67e74705SXin Li            'bin/llvm-link' + bin_ext,
276*67e74705SXin Li            'bin/llvm-symbolizer' + bin_ext,
277*67e74705SXin Li            'lib64/libLLVM' + lib_ext,
278*67e74705SXin Li            'lib64/LLVMgold' + lib_ext,
279*67e74705SXin Li        ])
280*67e74705SXin Li    return built_files
281*67e74705SXin Li
282*67e74705SXin Li
283*67e74705SXin Lidef install_built_host_files(build_dir, install_dir, host, strip, minimal=None):
284*67e74705SXin Li    built_files = get_built_host_files(host, minimal)
285*67e74705SXin Li    for built_file in built_files:
286*67e74705SXin Li        dirname = os.path.dirname(built_file)
287*67e74705SXin Li        install_path = os.path.join(install_dir, dirname)
288*67e74705SXin Li        if not os.path.exists(install_path):
289*67e74705SXin Li            makedirs(install_path)
290*67e74705SXin Li
291*67e74705SXin Li        built_path = os.path.join(build_dir, 'host', host, built_file)
292*67e74705SXin Li        install_file(built_path, install_path)
293*67e74705SXin Li
294*67e74705SXin Li        file_name = os.path.basename(built_file)
295*67e74705SXin Li
296*67e74705SXin Li        # Only strip bin files (not libs) on darwin.
297*67e74705SXin Li        is_darwin = host.startswith('darwin-x86')
298*67e74705SXin Li        if strip and (not is_darwin or built_file.startswith('bin/')):
299*67e74705SXin Li            check_call(['strip', os.path.join(install_path, file_name)])
300*67e74705SXin Li
301*67e74705SXin Li
302*67e74705SXin Lidef install_sanitizer_scripts(install_dir):
303*67e74705SXin Li    script_path = android_path(
304*67e74705SXin Li        'external/compiler-rt/lib/asan/scripts/asan_device_setup')
305*67e74705SXin Li    install_file(script_path, os.path.join(install_dir, 'bin'))
306*67e74705SXin Li
307*67e74705SXin Li
308*67e74705SXin Lidef install_analyzer_scripts(install_dir):
309*67e74705SXin Li    """Create and install bash scripts for invoking Clang for analysis."""
310*67e74705SXin Li    analyzer_text = (
311*67e74705SXin Li        '#!/bin/bash\n'
312*67e74705SXin Li        'if [ "$1" != "-cc1" ]; then\n'
313*67e74705SXin Li        '    `dirname $0`/../clang{clang_suffix} -target {target} "$@"\n'
314*67e74705SXin Li        'else\n'
315*67e74705SXin Li        '    # target/triple already spelled out.\n'
316*67e74705SXin Li        '    `dirname $0`/../clang{clang_suffix} "$@"\n'
317*67e74705SXin Li        'fi\n'
318*67e74705SXin Li    )
319*67e74705SXin Li
320*67e74705SXin Li    arch_target_pairs = (
321*67e74705SXin Li        ('arm64-v8a', 'aarch64-none-linux-android'),
322*67e74705SXin Li        ('armeabi', 'armv5te-none-linux-androideabi'),
323*67e74705SXin Li        ('armeabi-v7a', 'armv7-none-linux-androideabi'),
324*67e74705SXin Li        ('armeabi-v7a-hard', 'armv7-none-linux-androideabi'),
325*67e74705SXin Li        ('mips', 'mipsel-none-linux-android'),
326*67e74705SXin Li        ('mips64', 'mips64el-none-linux-android'),
327*67e74705SXin Li        ('x86', 'i686-none-linux-android'),
328*67e74705SXin Li        ('x86_64', 'x86_64-none-linux-android'),
329*67e74705SXin Li    )
330*67e74705SXin Li
331*67e74705SXin Li    for arch, target in arch_target_pairs:
332*67e74705SXin Li        arch_path = os.path.join(install_dir, 'bin', arch)
333*67e74705SXin Li        makedirs(arch_path)
334*67e74705SXin Li
335*67e74705SXin Li        analyzer_file_path = os.path.join(arch_path, 'analyzer')
336*67e74705SXin Li        logger().info('Creating %s', analyzer_file_path)
337*67e74705SXin Li        with open(analyzer_file_path, 'w') as analyzer_file:
338*67e74705SXin Li            analyzer_file.write(
339*67e74705SXin Li                analyzer_text.format(clang_suffix='', target=target))
340*67e74705SXin Li        subprocess.check_call(['chmod', 'a+x', analyzer_file_path])
341*67e74705SXin Li
342*67e74705SXin Li        analyzerpp_file_path = os.path.join(arch_path, 'analyzer++')
343*67e74705SXin Li        logger().info('Creating %s', analyzerpp_file_path)
344*67e74705SXin Li        with open(analyzerpp_file_path, 'w') as analyzerpp_file:
345*67e74705SXin Li            analyzerpp_file.write(
346*67e74705SXin Li                analyzer_text.format(clang_suffix='++', target=target))
347*67e74705SXin Li        subprocess.check_call(['chmod', 'a+x', analyzerpp_file_path])
348*67e74705SXin Li
349*67e74705SXin Li
350*67e74705SXin Lidef install_scan_scripts(install_dir):
351*67e74705SXin Li    tools_install_dir = os.path.join(install_dir, 'tools')
352*67e74705SXin Li    makedirs(tools_install_dir)
353*67e74705SXin Li    tools = ('scan-build', 'scan-view')
354*67e74705SXin Li    tools_dir = android_path('external/clang/tools')
355*67e74705SXin Li    for tool in tools:
356*67e74705SXin Li        tool_path = os.path.join(tools_dir, tool)
357*67e74705SXin Li        install_path = os.path.join(install_dir, 'tools', tool)
358*67e74705SXin Li        install_directory(tool_path, install_path)
359*67e74705SXin Li
360*67e74705SXin Li
361*67e74705SXin Lidef install_headers(build_dir, install_dir, host):
362*67e74705SXin Li    def should_copy(path):
363*67e74705SXin Li        if os.path.basename(path) in ('Makefile', 'CMakeLists.txt'):
364*67e74705SXin Li            return False
365*67e74705SXin Li        _, ext = os.path.splitext(path)
366*67e74705SXin Li        if ext == '.mk':
367*67e74705SXin Li            return False
368*67e74705SXin Li        return True
369*67e74705SXin Li
370*67e74705SXin Li    headers_src = android_path('external/clang/lib/Headers')
371*67e74705SXin Li    headers_dst = os.path.join(
372*67e74705SXin Li        install_dir, 'lib64/clang', short_version(), 'include')
373*67e74705SXin Li    makedirs(headers_dst)
374*67e74705SXin Li    for header in os.listdir(headers_src):
375*67e74705SXin Li        if not should_copy(header):
376*67e74705SXin Li            continue
377*67e74705SXin Li        install_file(os.path.join(headers_src, header), headers_dst)
378*67e74705SXin Li
379*67e74705SXin Li    install_file(android_path('bionic/libc/include/stdatomic.h'), headers_dst)
380*67e74705SXin Li
381*67e74705SXin Li    # arm_neon.h gets produced as part of external/clang/Android.bp.
382*67e74705SXin Li    # We must bundle the resulting file as part of the official Clang headers.
383*67e74705SXin Li    arm_neon_h = os.path.join(
384*67e74705SXin Li        build_dir, 'soong/.intermediates/external/clang/clang-gen-arm-neon/gen/clang/Basic/arm_neon.h')
385*67e74705SXin Li    install_file(arm_neon_h, headers_dst)
386*67e74705SXin Li
387*67e74705SXin Li    symlink(short_version(),
388*67e74705SXin Li            os.path.join(install_dir, 'lib64/clang', long_version()))
389*67e74705SXin Li
390*67e74705SXin Li
391*67e74705SXin Lidef install_profile_rt(build_dir, install_dir, host):
392*67e74705SXin Li    lib_dir = os.path.join(
393*67e74705SXin Li        install_dir, 'lib64/clang', short_version(), 'lib/linux')
394*67e74705SXin Li    makedirs(lib_dir)
395*67e74705SXin Li
396*67e74705SXin Li    install_target_profile_rt(build_dir, lib_dir)
397*67e74705SXin Li
398*67e74705SXin Li    # We only support profiling libs for Linux and Android.
399*67e74705SXin Li    if host == 'linux-x86':
400*67e74705SXin Li        install_host_profile_rt(build_dir, host, lib_dir)
401*67e74705SXin Li
402*67e74705SXin Li
403*67e74705SXin Lidef install_target_profile_rt(build_dir, lib_dir):
404*67e74705SXin Li    product_to_arch = {
405*67e74705SXin Li        'generic': 'arm',
406*67e74705SXin Li        'generic_arm64': 'aarch64',
407*67e74705SXin Li        'generic_mips': 'mipsel',
408*67e74705SXin Li        'generic_mips64': 'mips64el',
409*67e74705SXin Li        'generic_x86': 'i686',
410*67e74705SXin Li        'generic_x86_64': 'x86_64',
411*67e74705SXin Li    }
412*67e74705SXin Li
413*67e74705SXin Li    for product, arch in product_to_arch.items():
414*67e74705SXin Li        product_dir = os.path.join(build_dir, 'target/product', product)
415*67e74705SXin Li        static_libs = os.path.join(product_dir, 'obj/STATIC_LIBRARIES')
416*67e74705SXin Li        built_lib = os.path.join(
417*67e74705SXin Li            static_libs, 'libprofile_rt_intermediates/libprofile_rt.a')
418*67e74705SXin Li        lib_name = 'libclang_rt.profile-{}-android.a'.format(arch)
419*67e74705SXin Li        install_file(built_lib, os.path.join(lib_dir, lib_name))
420*67e74705SXin Li
421*67e74705SXin Li
422*67e74705SXin Lidef install_host_profile_rt(build_dir, host, lib_dir):
423*67e74705SXin Li    arch_to_obj_dir = {
424*67e74705SXin Li        'i686': 'obj32',
425*67e74705SXin Li        'x86_64': 'obj',
426*67e74705SXin Li    }
427*67e74705SXin Li
428*67e74705SXin Li    for arch, obj_dir in arch_to_obj_dir.items():
429*67e74705SXin Li        static_libs = os.path.join(
430*67e74705SXin Li            build_dir, 'host', host, obj_dir, 'STATIC_LIBRARIES')
431*67e74705SXin Li        built_lib = os.path.join(
432*67e74705SXin Li            static_libs, 'libprofile_rt_intermediates/libprofile_rt.a')
433*67e74705SXin Li        lib_name = 'libclang_rt.profile-{}.a'.format(arch)
434*67e74705SXin Li        install_file(built_lib, os.path.join(lib_dir, lib_name))
435*67e74705SXin Li
436*67e74705SXin Li
437*67e74705SXin Lidef install_libomp(build_dir, install_dir, host):
438*67e74705SXin Li    # libomp is not built for Darwin
439*67e74705SXin Li    if host == 'darwin-x86':
440*67e74705SXin Li        return
441*67e74705SXin Li
442*67e74705SXin Li    lib_dir = os.path.join(
443*67e74705SXin Li        install_dir, 'lib64/clang', short_version(), 'lib/linux')
444*67e74705SXin Li    if not os.path.isdir(lib_dir):
445*67e74705SXin Li        makedirs(lib_dir)
446*67e74705SXin Li
447*67e74705SXin Li    product_to_arch = {
448*67e74705SXin Li        'generic': 'arm',
449*67e74705SXin Li        'generic_arm64': 'arm64',
450*67e74705SXin Li        'generic_x86': 'x86',
451*67e74705SXin Li        'generic_x86_64': 'x86_64',
452*67e74705SXin Li    }
453*67e74705SXin Li
454*67e74705SXin Li    for product, arch in product_to_arch.items():
455*67e74705SXin Li        module = 'libomp-' + arch
456*67e74705SXin Li        product_dir = os.path.join(build_dir, 'target/product', product)
457*67e74705SXin Li        shared_libs = os.path.join(product_dir, 'obj/SHARED_LIBRARIES')
458*67e74705SXin Li        built_lib = os.path.join(
459*67e74705SXin Li            shared_libs,
460*67e74705SXin Li            '{}_intermediates/PACKED/{}.so'.format(module, module))
461*67e74705SXin Li        install_file(built_lib, os.path.join(lib_dir, module + '.so'))
462*67e74705SXin Li
463*67e74705SXin Li
464*67e74705SXin Lidef install_sanitizers(build_dir, install_dir, host):
465*67e74705SXin Li    headers_src = android_path('external/compiler-rt/include/sanitizer')
466*67e74705SXin Li    clang_lib = os.path.join(install_dir, 'lib64/clang', short_version())
467*67e74705SXin Li    headers_dst = os.path.join(clang_lib, 'include/sanitizer')
468*67e74705SXin Li    lib_dst = os.path.join(clang_lib, 'lib/linux')
469*67e74705SXin Li    install_directory(headers_src, headers_dst)
470*67e74705SXin Li
471*67e74705SXin Li    if not os.path.exists(lib_dst):
472*67e74705SXin Li        makedirs(lib_dst)
473*67e74705SXin Li
474*67e74705SXin Li    if host == 'linux-x86':
475*67e74705SXin Li        install_host_sanitizers(build_dir, host, lib_dst)
476*67e74705SXin Li
477*67e74705SXin Li    # Tuples of (product, arch)
478*67e74705SXin Li    product_to_arch = (
479*67e74705SXin Li        ('generic', 'arm'),
480*67e74705SXin Li        ('generic_arm64', 'aarch64'),
481*67e74705SXin Li        ('generic_x86', 'i686'),
482*67e74705SXin Li        ('generic_mips', 'mips'),
483*67e74705SXin Li        ('generic_mips64', 'mips64'),
484*67e74705SXin Li    )
485*67e74705SXin Li
486*67e74705SXin Li    sanitizers = ('asan', 'ubsan_standalone')
487*67e74705SXin Li
488*67e74705SXin Li    for product, arch in product_to_arch:
489*67e74705SXin Li        for sanitizer in sanitizers:
490*67e74705SXin Li            module = 'libclang_rt.{}-{}-android'.format(sanitizer, arch)
491*67e74705SXin Li            product_dir = os.path.join(build_dir, 'target/product', product)
492*67e74705SXin Li            lib_dir = os.path.join(product_dir, 'obj/SHARED_LIBRARIES',
493*67e74705SXin Li                                   '{}_intermediates'.format(module))
494*67e74705SXin Li            lib_name = '{}.so'.format(module)
495*67e74705SXin Li            built_lib = os.path.join(lib_dir, 'PACKED', lib_name)
496*67e74705SXin Li            install_file(built_lib, lib_dst)
497*67e74705SXin Li
498*67e74705SXin Li
499*67e74705SXin Li# Also install the asan_test binaries. We need to do this because the
500*67e74705SXin Li# platform sources for compiler-rt are potentially different from our
501*67e74705SXin Li# toolchain sources. The only way to ensure that this test builds
502*67e74705SXin Li# correctly is to make it a prebuilt based on our latest toolchain
503*67e74705SXin Li# sources. Note that this is only created/compiled by the previous
504*67e74705SXin Li# stage (usually stage1) compiler. We are not doing a subsequent
505*67e74705SXin Li# compile with our stage2 binaries to construct any further
506*67e74705SXin Li# device-targeted objects.
507*67e74705SXin Lidef install_sanitizer_tests(build_dir, install_dir, host):
508*67e74705SXin Li    # Tuples of (product, arch)
509*67e74705SXin Li    product_to_arch = (
510*67e74705SXin Li        ('generic', 'arm'),
511*67e74705SXin Li        ('generic_arm64', 'aarch64'),
512*67e74705SXin Li        ('generic_x86', 'i686'),
513*67e74705SXin Li        ('generic_mips', 'mips'),
514*67e74705SXin Li        ('generic_mips64', 'mips64'),
515*67e74705SXin Li    )
516*67e74705SXin Li
517*67e74705SXin Li    for product, arch in product_to_arch:
518*67e74705SXin Li        product_dir = os.path.join(build_dir, 'target/product', product)
519*67e74705SXin Li        test_module = 'asan_test'
520*67e74705SXin Li        test_dir = os.path.join(product_dir, 'obj/EXECUTABLES',
521*67e74705SXin Li                                '{}_intermediates'.format(test_module))
522*67e74705SXin Li        built_test = os.path.join(test_dir, 'PACKED', test_module)
523*67e74705SXin Li        test_dst = os.path.join(install_dir, 'test', arch, 'bin')
524*67e74705SXin Li        makedirs(test_dst)
525*67e74705SXin Li        install_file(built_test, test_dst)
526*67e74705SXin Li
527*67e74705SXin Li
528*67e74705SXin Lidef install_host_sanitizers(build_dir, host, lib_dst):
529*67e74705SXin Li    # Tuples of (name, multilib).
530*67e74705SXin Li    libs = (
531*67e74705SXin Li        ('asan', True),
532*67e74705SXin Li        ('asan_cxx', True),
533*67e74705SXin Li        ('ubsan_standalone', True),
534*67e74705SXin Li        ('ubsan_standalone_cxx', True),
535*67e74705SXin Li        ('tsan', False),
536*67e74705SXin Li        ('tsan_cxx', False),
537*67e74705SXin Li    )
538*67e74705SXin Li
539*67e74705SXin Li    obj32 = os.path.join(build_dir, 'host', host, 'obj32/STATIC_LIBRARIES')
540*67e74705SXin Li    obj64 = os.path.join(build_dir, 'host', host, 'obj/STATIC_LIBRARIES')
541*67e74705SXin Li    for lib, is_multilib in libs:
542*67e74705SXin Li        built_lib_name = 'lib{}.a'.format(lib)
543*67e74705SXin Li
544*67e74705SXin Li        obj64_dir = os.path.join(obj64, 'lib{}_intermediates'.format(lib))
545*67e74705SXin Li        lib64_name = 'libclang_rt.{}-x86_64.a'.format(lib)
546*67e74705SXin Li        built_lib64 = os.path.join(obj64_dir, built_lib_name)
547*67e74705SXin Li        install_file(built_lib64, os.path.join(lib_dst, lib64_name))
548*67e74705SXin Li        if is_multilib:
549*67e74705SXin Li            obj32_dir = os.path.join(obj32, 'lib{}_intermediates'.format(lib))
550*67e74705SXin Li            lib32_name = 'libclang_rt.{}-i686.a'.format(lib)
551*67e74705SXin Li            built_lib32 = os.path.join(obj32_dir, built_lib_name)
552*67e74705SXin Li            install_file(built_lib32, os.path.join(lib_dst, lib32_name))
553*67e74705SXin Li
554*67e74705SXin Li
555*67e74705SXin Lidef install_license_files(install_dir):
556*67e74705SXin Li    projects = (
557*67e74705SXin Li        'clang',
558*67e74705SXin Li        'clang-tools-extra',
559*67e74705SXin Li        'compiler-rt',
560*67e74705SXin Li        'libcxx',
561*67e74705SXin Li        'libcxxabi',
562*67e74705SXin Li        'libunwind_llvm',
563*67e74705SXin Li        'llvm',
564*67e74705SXin Li        'openmp_llvm'
565*67e74705SXin Li    )
566*67e74705SXin Li
567*67e74705SXin Li    notices = []
568*67e74705SXin Li    for project in projects:
569*67e74705SXin Li        project_path = android_path('external', project)
570*67e74705SXin Li        license_pattern = os.path.join(project_path, 'MODULE_LICENSE_*')
571*67e74705SXin Li        for license_file in glob.glob(license_pattern):
572*67e74705SXin Li            install_file(license_file, install_dir)
573*67e74705SXin Li        with open(os.path.join(project_path, 'NOTICE')) as notice_file:
574*67e74705SXin Li            notices.append(notice_file.read())
575*67e74705SXin Li    with open(os.path.join(install_dir, 'NOTICE'), 'w') as notice_file:
576*67e74705SXin Li        notice_file.write('\n'.join(notices))
577*67e74705SXin Li
578*67e74705SXin Li
579*67e74705SXin Lidef install_repo_prop(install_dir):
580*67e74705SXin Li    file_name = 'repo.prop'
581*67e74705SXin Li
582*67e74705SXin Li    dist_dir = os.environ.get('DIST_DIR')
583*67e74705SXin Li    if dist_dir is not None:
584*67e74705SXin Li        dist_repo_prop = os.path.join(dist_dir, file_name)
585*67e74705SXin Li        install_file(dist_repo_prop, install_dir)
586*67e74705SXin Li    else:
587*67e74705SXin Li        out_file = os.path.join(install_dir, file_name)
588*67e74705SXin Li        with open(out_file, 'w') as prop_file:
589*67e74705SXin Li            cmd = [
590*67e74705SXin Li                'repo', 'forall', '-c',
591*67e74705SXin Li                'echo $REPO_PROJECT $(git rev-parse HEAD)',
592*67e74705SXin Li            ]
593*67e74705SXin Li            check_call(cmd, stdout=prop_file)
594*67e74705SXin Li
595*67e74705SXin Li
596*67e74705SXin Lidef install_compiler_wrapper(install_dir, host):
597*67e74705SXin Li    is_windows = host.startswith('windows')
598*67e74705SXin Li    bin_ext = '.exe' if is_windows else ''
599*67e74705SXin Li
600*67e74705SXin Li    built_files = [
601*67e74705SXin Li        'bin/clang' + bin_ext,
602*67e74705SXin Li        'bin/clang++' + bin_ext,
603*67e74705SXin Li    ]
604*67e74705SXin Li
605*67e74705SXin Li    if is_windows:
606*67e74705SXin Li        built_files.extend([
607*67e74705SXin Li            'bin/clang_32' + bin_ext,
608*67e74705SXin Li        ])
609*67e74705SXin Li
610*67e74705SXin Li    wrapper_dir = android_path('external/clang')
611*67e74705SXin Li    wrapper = os.path.join(wrapper_dir, 'compiler_wrapper')
612*67e74705SXin Li
613*67e74705SXin Li    for built_file in built_files:
614*67e74705SXin Li        old_file = os.path.join(install_dir, built_file)
615*67e74705SXin Li        new_file = os.path.join(install_dir, built_file + ".real")
616*67e74705SXin Li        rename(old_file, new_file)
617*67e74705SXin Li        install_file(wrapper, old_file)
618*67e74705SXin Li
619*67e74705SXin Li
620*67e74705SXin Lidef parse_args():
621*67e74705SXin Li    parser = argparse.ArgumentParser()
622*67e74705SXin Li
623*67e74705SXin Li    parser.add_argument('-j', action='store', dest='jobs', type=int,
624*67e74705SXin Li                        default=multiprocessing.cpu_count(),
625*67e74705SXin Li                        help='Specify number of executed jobs')
626*67e74705SXin Li
627*67e74705SXin Li    parser.add_argument(
628*67e74705SXin Li        '--build-name', default='dev', help='Release name for the package.')
629*67e74705SXin Li    parser.add_argument(
630*67e74705SXin Li        '--dry-run', action='store_true', default=False,
631*67e74705SXin Li        help='Skip running commands; just print.')
632*67e74705SXin Li    parser.add_argument(
633*67e74705SXin Li        '-v', '--verbose', action='store_true', default=False,
634*67e74705SXin Li        help='Print debug output.')
635*67e74705SXin Li
636*67e74705SXin Li    multi_stage_group = parser.add_mutually_exclusive_group()
637*67e74705SXin Li    multi_stage_group.add_argument(
638*67e74705SXin Li        '--multi-stage', action='store_true', default=True,
639*67e74705SXin Li        help='Perform multi-stage build (enabled by default).')
640*67e74705SXin Li    multi_stage_group.add_argument(
641*67e74705SXin Li        '--no-multi-stage', action='store_false', dest='multi_stage',
642*67e74705SXin Li        help='Do not perform multi-stage build.')
643*67e74705SXin Li
644*67e74705SXin Li    build_all_llvm_tools_group = parser.add_mutually_exclusive_group()
645*67e74705SXin Li    build_all_llvm_tools_group.add_argument(
646*67e74705SXin Li        '--build-all-llvm-tools', action='store_true', default=True,
647*67e74705SXin Li        help='Build all the LLVM tools/utilities.')
648*67e74705SXin Li    build_all_llvm_tools_group.add_argument(
649*67e74705SXin Li        '--no-build-all-llvm-tools', action='store_false',
650*67e74705SXin Li        dest='build_all_llvm_tools',
651*67e74705SXin Li        help='Do not build all the LLVM tools/utilities.')
652*67e74705SXin Li
653*67e74705SXin Li    build_debug_clang_group = parser.add_mutually_exclusive_group()
654*67e74705SXin Li    build_debug_clang_group.add_argument(
655*67e74705SXin Li        '--debug-clang', action='store_true', default=True,
656*67e74705SXin Li        help='Also generate a debug version of clang (enabled by default).')
657*67e74705SXin Li    build_debug_clang_group.add_argument(
658*67e74705SXin Li        '--no-debug-clang', action='store_false',
659*67e74705SXin Li        dest='debug_clang',
660*67e74705SXin Li        help='Skip generating a debug version of clang.')
661*67e74705SXin Li
662*67e74705SXin Li    return parser.parse_args()
663*67e74705SXin Li
664*67e74705SXin Li
665*67e74705SXin Lidef main():
666*67e74705SXin Li    args = parse_args()
667*67e74705SXin Li    log_level = logging.INFO
668*67e74705SXin Li    if args.verbose:
669*67e74705SXin Li        log_level = logging.DEBUG
670*67e74705SXin Li    logging.basicConfig(level=log_level)
671*67e74705SXin Li
672*67e74705SXin Li    logger().info('chdir %s', android_path())
673*67e74705SXin Li    os.chdir(android_path())
674*67e74705SXin Li
675*67e74705SXin Li    Config.dry_run = args.dry_run
676*67e74705SXin Li
677*67e74705SXin Li    if sys.platform.startswith('linux'):
678*67e74705SXin Li        hosts = ['linux-x86', 'windows-x86']
679*67e74705SXin Li    elif sys.platform == 'darwin':
680*67e74705SXin Li        hosts = ['darwin-x86']
681*67e74705SXin Li    else:
682*67e74705SXin Li        raise RuntimeError('Unsupported host: {}'.format(sys.platform))
683*67e74705SXin Li
684*67e74705SXin Li    stage_1_out_dir = build_path('stage1')
685*67e74705SXin Li
686*67e74705SXin Li    # For a multi-stage build, build a minimum clang for the first stage that is
687*67e74705SXin Li    # just enough to build the second stage.
688*67e74705SXin Li    is_stage1_final = not args.multi_stage
689*67e74705SXin Li    build(out_dir=stage_1_out_dir,
690*67e74705SXin Li          build_all_clang_tools=is_stage1_final,
691*67e74705SXin Li          build_all_llvm_tools=(is_stage1_final and args.build_all_llvm_tools),
692*67e74705SXin Li          max_jobs=args.jobs)
693*67e74705SXin Li    final_out_dir = stage_1_out_dir
694*67e74705SXin Li    if args.multi_stage:
695*67e74705SXin Li        stage_1_install_dir = build_path('stage1-install')
696*67e74705SXin Li        for host in hosts:
697*67e74705SXin Li            package_name = 'clang-' + args.build_name
698*67e74705SXin Li            install_host_dir = os.path.join(stage_1_install_dir, host)
699*67e74705SXin Li            install_dir = os.path.join(install_host_dir, package_name)
700*67e74705SXin Li
701*67e74705SXin Li            # Remove any previously installed toolchain so it doesn't pollute
702*67e74705SXin Li            # the build.
703*67e74705SXin Li            if os.path.exists(install_host_dir):
704*67e74705SXin Li                rmtree(install_host_dir)
705*67e74705SXin Li
706*67e74705SXin Li            install_minimal_toolchain(stage_1_out_dir, install_dir, host, True)
707*67e74705SXin Li
708*67e74705SXin Li        stage_2_out_dir = build_path('stage2')
709*67e74705SXin Li        build(out_dir=stage_2_out_dir, prebuilts_path=stage_1_install_dir,
710*67e74705SXin Li              prebuilts_version=package_name,
711*67e74705SXin Li              build_all_clang_tools=True,
712*67e74705SXin Li              build_all_llvm_tools=args.build_all_llvm_tools,
713*67e74705SXin Li              max_jobs=args.jobs)
714*67e74705SXin Li        final_out_dir = stage_2_out_dir
715*67e74705SXin Li
716*67e74705SXin Li        if args.debug_clang:
717*67e74705SXin Li            debug_clang_out_dir = build_path('debug')
718*67e74705SXin Li            build(out_dir=debug_clang_out_dir,
719*67e74705SXin Li                  prebuilts_path=stage_1_install_dir,
720*67e74705SXin Li                  prebuilts_version=package_name,
721*67e74705SXin Li                  build_all_clang_tools=True,
722*67e74705SXin Li                  build_all_llvm_tools=args.build_all_llvm_tools,
723*67e74705SXin Li                  debug_clang=args.debug_clang,
724*67e74705SXin Li                  max_jobs=args.jobs)
725*67e74705SXin Li            # Install the actual debug toolchain somewhere, so it is easier to
726*67e74705SXin Li            # use.
727*67e74705SXin Li            debug_package_name = 'clang-debug'
728*67e74705SXin Li            base_debug_install_dir = build_path('debug-install')
729*67e74705SXin Li            for host in hosts:
730*67e74705SXin Li                debug_install_host_dir = os.path.join(
731*67e74705SXin Li                    base_debug_install_dir, host)
732*67e74705SXin Li                debug_install_dir = os.path.join(
733*67e74705SXin Li                    debug_install_host_dir, debug_package_name)
734*67e74705SXin Li                if os.path.exists(debug_install_host_dir):
735*67e74705SXin Li                    rmtree(debug_install_host_dir)
736*67e74705SXin Li                install_toolchain(
737*67e74705SXin Li                    debug_clang_out_dir, debug_install_dir, host, False)
738*67e74705SXin Li
739*67e74705SXin Li    dist_dir = ORIG_ENV.get('DIST_DIR', final_out_dir)
740*67e74705SXin Li    for host in hosts:
741*67e74705SXin Li        package_toolchain(final_out_dir, args.build_name, host, dist_dir)
742*67e74705SXin Li
743*67e74705SXin Li
744*67e74705SXin Liif __name__ == '__main__':
745*67e74705SXin Li    print 'This script and llvm branch are deprecated and unmaintained.'
746*67e74705SXin Li    print 'Use the llvm-toolchain branch (repo init ... -b llvm-toolchain).'
747*67e74705SXin Li    print 'https://android.googlesource.com/toolchain/llvm_android/+/master'
748*67e74705SXin Li    sys.exit(0)
749