1#!/usr/bin/env python3 2# 3# Copyright (C) 2021 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16"""A script to copy outputs from Bazel rules to a user specified dist directory. 17 18This script is only meant to be executed with `bazel run`. `bazel build <this 19script>` doesn't actually copy the files, you'd have to `bazel run` a 20copy_to_dist_dir target. 21 22This script copies files from Bazel's output tree into a directory specified by 23the user. It does not check if the dist dir already contains the file, and will 24simply overwrite it. 25 26One approach is to wipe the dist dir every time this script runs, but that may 27be overly destructive and best left to an explicit rm -rf call outside of this 28script. 29 30Another approach is to error out if the file being copied already exist in the 31dist dir, or perform some kind of content hash checking. 32""" 33 34import argparse 35import collections 36import fnmatch 37import glob 38import logging 39import os 40import pathlib 41import shutil 42import sys 43import tarfile 44import textwrap 45 46 47_CMDLINE_FLAGS_SENTINEL = "CMDLINE_FLAGS_SENTINEL" 48 49# Arguments that should not be specified in the command line, but only 50# in BUILD files. 51_DEPRECATED_CMDLINE_OPTIONS = { 52 "--dist_dir": "Use --destdir instead.", 53 "--log": "Use -q instead.", 54 "--archive_prefix": "", 55 "--flat": "Specify it in the BUILD file (e.g. copy_to_dist_dir(flat=True))", 56 "--strip_components": "Specify it in the BUILD file " 57 "(e.g. copy_to_dist_dir(strip_components=1))", 58 "--prefix": "Specify it in the BUILD file " 59 "(e.g. copy_to_dist_dir(prefix='prefix'))", 60 "--wipe_dist_dir": "Specify it in the BUILD file " 61 "(e.g. copy_to_dist_dir(wipe_dist_dir=True))", 62 "--allow_duplicate_filenames": 63 "Specify it in the BUILD file " 64 "(e.g. copy_to_dist_dir(allow_duplicate_filenames=True))", 65 "--mode_override": 66 "Specify it in the BUILD file " 67 "(e.g. copy_to_dist_dir(mode_overrides=[('*.sh', '755')]))", 68} 69 70 71def copy_with_modes(src, dst, mode_overrides): 72 mode_override = None 73 for (pattern, mode) in mode_overrides: 74 if fnmatch.fnmatch(src, pattern): 75 mode_override = mode 76 break 77 78 # Remove destination file that may be write-protected 79 pathlib.Path(dst).unlink(missing_ok=True) 80 81 # Copy the file with copy2 to preserve whatever permissions are set on src 82 shutil.copy2(os.path.abspath(src), dst, follow_symlinks=True) 83 84 if mode_override: 85 os.chmod(dst, mode_override) 86 87 88def ensure_unique_filenames(files): 89 basename_to_srcs_map = collections.defaultdict(list) 90 for f in files: 91 basename_to_srcs_map[os.path.basename(f)].append(f) 92 93 duplicates_exist = False 94 for (basename, srcs) in basename_to_srcs_map.items(): 95 if len(srcs) > 1: 96 duplicates_exist = True 97 logging.error('Destination filename "%s" has multiple possible sources: %s', 98 basename, srcs) 99 100 if duplicates_exist: 101 sys.exit(1) 102 103 104def files_to_dist(pattern): 105 # Assume that dist.bzl is in the same package as dist.py 106 runfiles_directory = os.path.dirname(__file__) 107 dist_manifests = glob.glob( 108 os.path.join(runfiles_directory, pattern)) 109 if not dist_manifests: 110 logging.warning("Could not find a file with pattern %s" 111 " in the runfiles directory: %s", pattern, runfiles_directory) 112 files_to_dist = [] 113 for dist_manifest in dist_manifests: 114 with open(dist_manifest, "r") as f: 115 files_to_dist += [line.strip() for line in f] 116 return files_to_dist 117 118 119def copy_files_to_dist_dir(files, archives, mode_overrides, dist_dir, flat, prefix, 120 strip_components, archive_prefix, wipe_dist_dir, allow_duplicate_filenames, **ignored): 121 122 if flat and not allow_duplicate_filenames: 123 ensure_unique_filenames(files) 124 125 if wipe_dist_dir and os.path.exists(dist_dir): 126 shutil.rmtree(dist_dir) 127 128 logging.info("Copying to %s", dist_dir) 129 130 for src in files: 131 if flat: 132 src_relpath = os.path.basename(src) 133 elif strip_components > 0: 134 src_relpath = src.split('/', strip_components)[-1] 135 else: 136 src_relpath = src 137 138 src_relpath = os.path.join(prefix, src_relpath) 139 140 dst = os.path.join(dist_dir, src_relpath) 141 if os.path.isfile(src): 142 dst_dirname = os.path.dirname(dst) 143 logging.debug("Copying file: %s" % dst) 144 if not os.path.exists(dst_dirname): 145 os.makedirs(dst_dirname) 146 147 copy_with_modes(src, dst, mode_overrides) 148 elif os.path.isdir(src): 149 logging.debug("Copying dir: %s" % dst) 150 if os.path.exists(dst): 151 # make the directory temporary writable, then 152 # shutil.copytree will restore correct permissions. 153 os.chmod(dst, 750) 154 shutil.copytree( 155 os.path.abspath(src), 156 dst, 157 copy_function=lambda s, d: copy_with_modes(s, d, mode_overrides), 158 dirs_exist_ok=True, 159 ) 160 161 for archive in archives: 162 try: 163 with tarfile.open(archive) as tf: 164 dst_dirname = os.path.join(dist_dir, archive_prefix) 165 logging.debug("Extracting archive: %s -> %s", archive, dst_dirname) 166 tf.extractall(dst_dirname) 167 except tarfile.TarError: 168 # toybox does not support creating empty tar files, hence the build 169 # system may use empty files as empty archives. 170 if os.path.getsize(archive) == 0: 171 logging.warning("Skipping empty tar file: %s", archive) 172 continue 173 # re-raise if we do not know anything about this error 174 logging.exception("Unknown TarError.") 175 raise 176 177 178def config_logging(log_level_str): 179 level = getattr(logging, log_level_str.upper(), None) 180 if not isinstance(level, int): 181 sys.stderr.write("ERROR: Invalid --log {}\n".format(log_level_str)) 182 sys.exit(1) 183 logging.basicConfig(level=level, format="[dist] %(levelname)s: %(message)s") 184 185 186class CheckDeprecationAction(argparse.Action): 187 """Checks if a deprecated option is used, then do nothing.""" 188 def __call__(self, parser, namespace, values, option_string=None): 189 if option_string in _DEPRECATED_CMDLINE_OPTIONS: 190 logging.warning("%s is deprecated! %s", option_string, 191 _DEPRECATED_CMDLINE_OPTIONS[option_string]) 192 193 194class StoreAndCheckDeprecationAction(CheckDeprecationAction): 195 """Sotres the value, and checks if a deprecated option is used.""" 196 def __call__(self, parser, namespace, values, option_string=None): 197 super().__call__(parser, namespace, values, option_string) 198 setattr(namespace, self.dest, values) 199 200 201class StoreTrueAndCheckDeprecationAction(CheckDeprecationAction): 202 """Sotres true, and checks if a deprecated option is used.""" 203 def __call__(self, parser, namespace, values, option_string=None): 204 super().__call__(parser, namespace, values, option_string) 205 setattr(namespace, self.dest, True) 206 207 208class AppendAndCheckDeprecationAction(CheckDeprecationAction): 209 """Appends the value, and checks if a deprecated option is used.""" 210 def __call__(self, parser, namespace, values, option_string=None): 211 super().__call__(parser, namespace, values, option_string) 212 if not values: 213 return 214 metavar_len = len(self.metavar)if self.metavar else 1 215 value_groups = [values[i:i + metavar_len] 216 for i in range(0, len(values), metavar_len)] 217 setattr(namespace, self.dest, 218 getattr(namespace, self.dest, []) + value_groups) 219 220 221def _get_parser(cmdline=False) -> argparse.ArgumentParser: 222 parser = argparse.ArgumentParser( 223 description="Dist Bazel output files into a custom directory.", 224 formatter_class=argparse.RawTextHelpFormatter) 225 deprecated = parser.add_argument_group( 226 "Deprecated command line options", 227 description=textwrap.dedent("""\ 228 List of command line options that are deprecated. 229 Most of them should be specified in the BUILD file instead. 230 """)) 231 parser.add_argument( 232 "--destdir", "--dist_dir", required=not cmdline, dest="dist_dir", 233 help=textwrap.dedent("""\ 234 path to the dist dir. 235 If relative, it is interpreted as relative to Bazel workspace root 236 set by the BUILD_WORKSPACE_DIRECTORY environment variable, or 237 PWD if BUILD_WORKSPACE_DIRECTORY is not set. 238 239 Note: --dist_dir is deprecated; use --destdir instead."""), 240 action=StoreAndCheckDeprecationAction if cmdline else "store") 241 deprecated.add_argument( 242 "--flat", 243 action=StoreTrueAndCheckDeprecationAction if cmdline else "store_true", 244 help="ignore subdirectories in the manifest") 245 deprecated.add_argument( 246 "--strip_components", type=int, default=0, 247 help="number of leading components to strip from paths before applying --prefix", 248 action=StoreAndCheckDeprecationAction if cmdline else "store") 249 deprecated.add_argument( 250 "--prefix", default="", 251 help="path prefix to apply within dist_dir for copied files", 252 action=StoreAndCheckDeprecationAction if cmdline else "store") 253 deprecated.add_argument( 254 "--archive_prefix", default="", 255 help="Path prefix to apply within dist_dir for extracted archives. " + 256 "Supported archives: tar.", 257 action=StoreAndCheckDeprecationAction if cmdline else "store") 258 deprecated.add_argument("--log", help="Log level (debug, info, warning, error)", 259 default="debug", 260 action=StoreAndCheckDeprecationAction if cmdline else "store") 261 parser.add_argument("-q", "--quiet", action="store_const", default=False, 262 help="Same as --log=error", const="error", dest="log") 263 deprecated.add_argument( 264 "--wipe_dist_dir", 265 action=StoreTrueAndCheckDeprecationAction if cmdline else "store_true", 266 help="remove existing dist_dir prior to running", 267 ) 268 deprecated.add_argument( 269 "--allow_duplicate_filenames", 270 action=StoreTrueAndCheckDeprecationAction if cmdline else "store_true", 271 help="allow multiple files with the same name to be copied to dist_dir (overwriting)" 272 ) 273 deprecated.add_argument( 274 "--mode_override", 275 metavar=("PATTERN", "MODE"), 276 action=AppendAndCheckDeprecationAction if cmdline else "append", 277 nargs=2, 278 default=[], 279 help='glob pattern and mode to set on files matching pattern (e.g. --mode_override "*.sh" "755")' 280 ) 281 return parser 282 283def main(): 284 args = sys.argv[1:] 285 args.remove(_CMDLINE_FLAGS_SENTINEL) 286 args = _get_parser().parse_args(args) 287 288 config_logging(args.log) 289 290 # Warn about arguments that should not be set in command line. 291 _get_parser(cmdline=True).parse_args( 292 sys.argv[sys.argv.index(_CMDLINE_FLAGS_SENTINEL) + 1:]) 293 294 mode_overrides = [] 295 for (pattern, mode) in args.mode_override: 296 try: 297 mode_overrides.append((pattern, int(mode, 8))) 298 except ValueError: 299 logging.error("invalid octal permissions: %s", mode) 300 sys.exit(1) 301 302 if not os.path.isabs(args.dist_dir): 303 # BUILD_WORKSPACE_DIRECTORY is the root of the Bazel workspace containing 304 # this binary target. 305 # https://docs.bazel.build/versions/main/user-manual.html#run 306 args.dist_dir = os.path.join( 307 os.environ.get("BUILD_WORKSPACE_DIRECTORY"), args.dist_dir) 308 309 files = files_to_dist("*_dist_manifest.txt") 310 archives = files_to_dist("*_dist_archives_manifest.txt") 311 copy_files_to_dist_dir(files, archives, mode_overrides, **vars(args)) 312 313 314if __name__ == "__main__": 315 main() 316