xref: /aosp_15_r20/external/angle/build/3pp_common/common.py (revision 8975f5c5ed3d1c378011245431ada316dfb6f244)

# Copyright 2024 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import argparse
import logging
import os
import pathlib
import shlex
import shutil
import subprocess
import sys
import tarfile
import tempfile
import time
import urllib.request

import scripthash

_THIS_DIR = pathlib.Path(__file__).resolve().parent
_SRC_ROOT = _THIS_DIR.parents[1]
_CHECKOUT_SRC_ROOT_SUBDIR = '.3pp/chromium'


def parse_args():
    parser = argparse.ArgumentParser()
    # TODO(agrieve): Add required=True once 3pp builds with > python3.6.
    subparsers = parser.add_subparsers()

    subparser = subparsers.add_parser(
        'latest', help='Prints the version as $LATEST.$RUNTIME_DEPS_HASH')
    subparser.set_defaults(action='latest')

    subparser = subparsers.add_parser(
        'checkout', help='Copies files into the workdir used by docker')
    subparser.add_argument('checkout_dir')
    subparser.add_argument('--version', help='Output from "latest"')
    subparser.set_defaults(action='checkout')

    subparser = subparsers.add_parser(
        'install',
        help=('Run from workdir, inside docker container. '
              'Builds & copies outputs into |output_prefix| directory'))
    subparser.add_argument('output_prefix',
                           help='The path to install the compiled package to.')
    subparser.add_argument('deps_prefix',
                           help='The path to a directory containing all deps.')
    subparser.add_argument('--version', help='Output from "latest"')
    subparser.add_argument('--checkout-dir', help='Directory to use as CWD')
    subparser.set_defaults(action='install')

    subparser = subparsers.add_parser(
        'local-test', help='Run latest / checkout / install locally')
    subparser.add_argument('--checkout-dir',
                           default='3pp_workdir',
                           help='Workdir to use')
    subparser.add_argument('--output-prefix',
                           default='3pp_out',
                           help='Directory for final artifacts')
    subparser.set_defaults(action='local-test')

    args = parser.parse_args()
    if not hasattr(args, 'action'):
        parser.print_help()
        sys.exit(1)

    if hasattr(args, 'version'):
        args.version = args.version or os.environ.get('_3PP_VERSION')
        if not args.version:
            parser.error('Must set --version or _3PP_VERSION')
    if hasattr(args, 'output_prefix') and args.output_prefix:
        args.output_prefix = os.path.abspath(args.output_prefix)
    if hasattr(args, 'checkout_dir') and args.checkout_dir:
        args.checkout_dir = os.path.abspath(args.checkout_dir)

    if args.action == 'checkout':
        # 3pp bot recipe does this, so needed only when running locally.
        os.makedirs(args.checkout_dir, exist_ok=True)

    if args.action == 'install':
        if args.checkout_dir:
            logging.info('Setting CWD=%s', args.checkout_dir)
            os.chdir(args.checkout_dir)

        if not os.path.exists(_CHECKOUT_SRC_ROOT_SUBDIR):
            parser.error(f'Does not exist: {_CHECKOUT_SRC_ROOT_SUBDIR}.'
                         f' Use --checkout-dir?')

        # 3pp bot recipe does this, so needed only when running locally.
        os.makedirs(args.output_prefix, exist_ok=True)

    return args


def path_within_checkout(subpath):
    return os.path.abspath(os.path.join(_CHECKOUT_SRC_ROOT_SUBDIR, subpath))


def _all_files(path):
    if os.path.isfile(path):
        return [path]
    assert os.path.isdir(path), 'Not a file or dir: ' + path
    all_paths = pathlib.Path(path).glob('**/*')
    return [str(f) for f in all_paths if f.is_file()]


def _resolve_runtime_deps(runtime_deps):
    ret = []
    for p in runtime_deps:
        if p.startswith('//'):
            ret.append(os.path.relpath(str(_SRC_ROOT / p[2:])))
        elif os.path.isabs(p):
            ret.append(os.path.relpath(p))
        else:
            ret.append(p)
    return ret


def copy_runtime_deps(checkout_dir, runtime_deps):
    # Make 3pp_common scripts available in the docker container install.py
    # will run in.
    dest_dir = os.path.join(checkout_dir, _CHECKOUT_SRC_ROOT_SUBDIR)

    for src_path in _resolve_runtime_deps(runtime_deps):
        relpath = os.path.relpath(src_path, _SRC_ROOT)
        dest_path = os.path.join(dest_dir, relpath)
        os.makedirs(os.path.dirname(dest_path), exist_ok=True)
        if os.path.isfile(src_path):
            shutil.copy(src_path, dest_path)
        else:
            shutil.copytree(src_path,
                            dest_path,
                            ignore=shutil.ignore_patterns('.*', '__pycache__'))
    logging.info('Runtime deps:')
    sys.stderr.write('\n'.join(_all_files(checkout_dir)) + '\n')


def download_file(url, dest):
    logging.info('Downloading %s', url)
    with urllib.request.urlopen(url) as r:
        with open(dest, 'wb') as f:
            shutil.copyfileobj(r, f)


def extract_tar(path, dest):
    logging.info('Extracting %s to %s', path, dest)
    with tarfile.open(path) as f:
        f.extractall(dest)


def run_cmd(cmd, check=True, *args, **kwargs):
    logging.info('Running: %s', shlex.join(cmd))
    return subprocess.run(cmd, check=check, *args, **kwargs)


def apply_patches(patches_dir, checkout_dir):
    for path in sorted(pathlib.Path(patches_dir).glob('*.patch')):
        cmd = ['git', 'apply', '-v', str(path)]
        run_cmd(cmd, cwd=checkout_dir)


def main(*, do_latest, do_install, runtime_deps):
    logging.basicConfig(
        level=logging.DEBUG,
        format='%(levelname).1s %(relativeCreated)6d %(message)s')
    args = parse_args()
    runtime_deps = [str(_THIS_DIR)] + runtime_deps

    if args.action == 'local-test':
        logging.warning('Will use work dir: %s', args.checkout_dir)
        logging.warning('Will use output dir: %s', args.output_prefix)
        if os.path.exists(args.checkout_dir) and os.listdir(args.checkout_dir):
            logging.warning(
                '*** Work dir not empty. This often causes failures. ***')
            time.sleep(4)
        # Approximates what 3pp recipe does for minimal configs.
        # https://source.chromium.org/search?q=symbol:Chromium3ppApi.execute&ss=chromium
        prog = os.path.abspath(sys.argv[0])
        cmd = [prog, 'latest']
        version = run_cmd(cmd, stdout=subprocess.PIPE, text=True).stdout
        os.environ['_3PP_VERSION'] = version
        checkout_dir = args.checkout_dir
        run_cmd([prog, 'checkout', checkout_dir])
        run_cmd([prog, 'install', args.output_prefix, 'UNUSED-DEPS-DIR'],
                cwd=checkout_dir)
        logging.warning('Local test complete.')
        return

    if args.action == 'latest':
        version = do_latest()
        assert version, 'do_latest() returned ' + repr(version)
        extra_paths = []
        for p in _resolve_runtime_deps(runtime_deps):
            extra_paths += _all_files(p)
        deps_hash = scripthash.compute(extra_paths=extra_paths)
        print(f'{version}.{deps_hash}')
        return

    # Remove the hash at the end: 30.4.0-alpha05.HASH => 30.4.0-alpha05
    args.version = args.version.rsplit('.', 1)[0]
    if args.action == 'checkout':
        copy_runtime_deps(args.checkout_dir, runtime_deps)
        return

    assert args.action == 'install'
    do_install(args)
    prefix_len = len(args.output_prefix) + 1
    logging.info(
        'Contents of %s: \n%s\n', args.output_prefix,
        '\n'.join(p[prefix_len:] for p in _all_files(args.output_prefix)))