xref: /aosp_15_r20/external/pigweed/pw_build/py/pw_build/generate_python_wheel_cache.py (revision 61c4878ac05f98d0ceed94b57d316916de578985) 
1# Copyright 2023 The Pigweed Authors
2#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may not
4# use this file except in compliance with the License. You may obtain a copy of
5# the License at
6#
7#     https://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations under
13# the License.
14"""Download all Python packages required for a pw_python_venv."""
15
16import argparse
17import logging
18from pathlib import Path
19import shlex
20import subprocess
21import sys
22
23
24_LOG = logging.getLogger('pw_build.generate_python_wheel_cache')
25
26
27def _parse_args() -> argparse.Namespace:
28    parser = argparse.ArgumentParser(description=__doc__)
29    parser.add_argument(
30        '--pip-download-log',
31        type=Path,
32        required=True,
33        help='Path to the root gn build dir.',
34    )
35    parser.add_argument(
36        '--wheel-dir',
37        type=Path,
38        required=True,
39        help='Path to save wheel files.',
40    )
41    parser.add_argument(
42        '--download-all-platforms',
43        action='store_true',
44        help='Download Python precompiled wheels for all platforms.',
45    )
46    parser.add_argument(
47        '-r',
48        '--requirement',
49        type=Path,
50        nargs='+',
51        required=True,
52        help='Requirements files',
53    )
54    return parser.parse_args()
55
56
57def main(
58    pip_download_log: Path,
59    wheel_dir: Path,
60    requirement: list[Path],
61    download_all_platforms: bool = False,
62) -> int:
63    """Download all Python packages required for a pw_python_venv."""
64
65    # Delete existing wheels from the out dir, there may be stale versions.
66    # shutil.rmtree(wheel_dir, ignore_errors=True)
67    wheel_dir.mkdir(parents=True, exist_ok=True)
68
69    pip_download_args = [
70        sys.executable,
71        "-m",
72        "pip",
73        "--log",
74        str(pip_download_log),
75        "download",
76        "--dest",
77        str(wheel_dir),
78    ]
79    for req in requirement:
80        pip_download_args.extend(["--requirement", str(req)])
81
82    if not download_all_platforms:
83        # Download for the current platform only.
84        quoted_pip_download_args = ' '.join(
85            shlex.quote(arg) for arg in pip_download_args
86        )
87        _LOG.info('Run ==> %s', quoted_pip_download_args)
88        # Download packages
89        subprocess.run(pip_download_args, check=True)
90        return 0
91
92    # DOCSTAG: [wheel-platform-args]
93    # These platform args are derived from the cffi pypi package:
94    #   https://pypi.org/project/cffi/#files
95    # See also these pages on Python wheel filename format:
96    #   https://peps.python.org/pep-0491/#file-name-convention
97    # and Platform compatibility tags:
98    #   https://packaging.python.org/en/latest/specifications/
99    #      platform-compatibility-tags/
100    platform_args = [
101        '--platform=any',
102        '--platform=macosx_10_9_universal2',
103        '--platform=macosx_10_9_x86_64',
104        '--platform=macosx_11_0_arm64',
105        '--platform=manylinux2010_x86_64',
106        '--platform=manylinux2014_aarch64',
107        '--platform=manylinux2014_x86_64',
108        '--platform=manylinux_2_17_aarch64',
109        '--platform=manylinux_2_17_x86_64',
110        '--platform=musllinux_1_1_x86_64',
111        '--platform=win_amd64',
112        # Note: These 32bit platforms are omitted
113        # '--platform=manylinux2010_i686',
114        # '--platform=manylinux2014_i686',
115        # '--platform=manylinux_2_12_i686'
116        # '--platform=musllinux_1_1_i686',
117        # '--platform=win32',
118    ]
119
120    # Pigweed supports Python 3.8 and up.
121    python_version_args = [
122        [
123            '--implementation=py3',
124            '--abi=none',
125        ],
126        [
127            '--implementation=cp',
128            '--python-version=3.8',
129            '--abi=cp38',
130        ],
131        [
132            '--implementation=cp',
133            '--python-version=3.9',
134            '--abi=cp39',
135        ],
136        [
137            '--implementation=cp',
138            '--python-version=3.10',
139            '--abi=cp310',
140        ],
141        [
142            '--implementation=cp',
143            '--python-version=3.11',
144            '--abi=cp311',
145        ],
146    ]
147    # DOCSTAG: [wheel-platform-args]
148
149    # --no-deps is required when downloading binary packages for different
150    # platforms other than the current one. The requirements.txt files already
151    # has the fully expanded deps list using pip-compile so this is not a
152    # problem.
153    pip_download_args.append('--no-deps')
154
155    # Run pip download once for each Python version. Multiple platform args can
156    # be added to the same command.
157    for py_version_args in python_version_args:
158        for platform_arg in platform_args:
159            final_pip_download_args = list(pip_download_args)
160            final_pip_download_args.append(platform_arg)
161            final_pip_download_args.extend(py_version_args)
162            quoted_pip_download_args = ' '.join(
163                shlex.quote(arg) for arg in final_pip_download_args
164            )
165            _LOG.info('')
166            _LOG.info('Fetching packages for:')
167            _LOG.info(
168                'Python %s and Platforms: %s', py_version_args, platform_args
169            )
170            _LOG.info('Run ==> %s', quoted_pip_download_args)
171
172            # Download packages
173            subprocess.run(final_pip_download_args, check=True)
174
175    return 0
176
177
178if __name__ == '__main__':
179    logging.basicConfig(format='%(message)s', level=logging.DEBUG)
180    sys.exit(main(**vars(_parse_args())))
181