1#!/usr/bin/env python3 2 3# Copyright 2016 gRPC authors. 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17from __future__ import print_function 18 19import argparse 20import errno 21import filecmp 22import glob 23import os 24import os.path 25import pprint 26import shutil 27import subprocess 28import sys 29import traceback 30import uuid 31 32# the template for the content of protoc_lib_deps.py 33DEPS_FILE_CONTENT = """ 34# Copyright 2017 gRPC authors. 35# 36# Licensed under the Apache License, Version 2.0 (the "License"); 37# you may not use this file except in compliance with the License. 38# You may obtain a copy of the License at 39# 40# http://www.apache.org/licenses/LICENSE-2.0 41# 42# Unless required by applicable law or agreed to in writing, software 43# distributed under the License is distributed on an "AS IS" BASIS, 44# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 45# See the License for the specific language governing permissions and 46# limitations under the License. 47 48# AUTO-GENERATED BY make_grpcio_tools.py! 49CC_FILES={cc_files} 50 51PROTO_FILES={proto_files} 52 53CC_INCLUDES={cc_includes} 54PROTO_INCLUDE={proto_include} 55 56{commit_hash_expr} 57""" 58 59# expose commit hash suffix and prefix for check_grpcio_tools.py 60COMMIT_HASH_PREFIX = 'PROTOBUF_SUBMODULE_VERSION="' 61COMMIT_HASH_SUFFIX = '"' 62 63EXTERNAL_LINKS = [ 64 ("@com_google_absl//", "third_party/abseil-cpp/"), 65 ("@com_google_protobuf//", "third_party/protobuf/"), 66 ("@utf8_range//", "third_party/protobuf/third_party/utf8_range"), 67] 68 69PROTOBUF_PROTO_PREFIX = "@com_google_protobuf//src/" 70 71# will be added to include path when building grpcio_tools 72CC_INCLUDES = [ 73 os.path.join("third_party", "abseil-cpp"), 74 os.path.join("third_party", "protobuf"), 75 os.path.join("third_party", "protobuf", "src"), 76 os.path.join("third_party", "protobuf", "upb"), 77 os.path.join("third_party", "protobuf", "third_party", "utf8_range"), 78] 79 80# include path for .proto files 81PROTO_INCLUDE = os.path.join("third_party", "protobuf", "src") 82 83# the target directory is relative to the grpcio_tools package root. 84GRPCIO_TOOLS_ROOT_PREFIX = "tools/distrib/python/grpcio_tools/" 85 86# Pairs of (source, target) directories to copy 87# from the grpc repo root to the grpcio_tools build root. 88COPY_FILES_SOURCE_TARGET_PAIRS = [ 89 ("include", "grpc_root/include"), 90 ("src/compiler", "grpc_root/src/compiler"), 91 ("third_party/abseil-cpp/absl", "third_party/abseil-cpp/absl"), 92 ("third_party/protobuf/src", "third_party/protobuf/src"), 93 ("third_party/protobuf/upb", "third_party/protobuf/upb"), 94 ( 95 "third_party/protobuf/upb_generator", 96 "third_party/protobuf/upb_generator", 97 ), 98 ( 99 "third_party/protobuf/third_party/utf8_range", 100 "third_party/protobuf/third_party/utf8_range", 101 ), 102] 103 104DELETE_TARGETS_ON_CLEANUP = ["third_party"] 105 106# grpc repo root 107GRPC_ROOT = os.path.abspath( 108 os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..", "..") 109) 110 111# the directory under which to probe for the current protobuf commit SHA 112GRPC_PROTOBUF_SUBMODULE_ROOT = os.path.join( 113 GRPC_ROOT, "third_party", "protobuf" 114) 115 116# the file to generate 117GRPC_PYTHON_PROTOC_LIB_DEPS = os.path.join( 118 GRPC_ROOT, 119 "tools", 120 "distrib", 121 "python", 122 "grpcio_tools", 123 "protoc_lib_deps.py", 124) 125 126# the script to run for getting dependencies 127BAZEL_DEPS = os.path.join( 128 GRPC_ROOT, "tools", "distrib", "python", "bazel_deps.sh" 129) 130 131# the bazel target to scrape to get list of sources for the build 132BAZEL_DEPS_PROTOC_LIB_QUERY = "@com_google_protobuf//:protoc_lib" 133 134BAZEL_DEPS_COMMON_PROTOS_QUERIES = [ 135 "@com_google_protobuf//:well_known_type_protos", 136 # has both plugin.proto and descriptor.proto 137 "@com_google_protobuf//:compiler_plugin_proto", 138] 139 140 141def protobuf_submodule_commit_hash(): 142 """Gets the commit hash for the HEAD of the protobuf submodule currently 143 checked out.""" 144 cwd = os.getcwd() 145 os.chdir(GRPC_PROTOBUF_SUBMODULE_ROOT) 146 output = subprocess.check_output(["git", "rev-parse", "HEAD"]) 147 os.chdir(cwd) 148 return output.decode("ascii").splitlines()[0].strip() 149 150 151def _bazel_query(query): 152 """Runs 'bazel query' to collect source file info.""" 153 print('Running "bazel query %s"' % query) 154 output = subprocess.check_output([BAZEL_DEPS, query]) 155 return output.decode("ascii").splitlines() 156 157 158def _pretty_print_list(items): 159 """Pretty print python list""" 160 formatted = pprint.pformat(items, indent=4) 161 # add newline after opening bracket (and fix indent of the next line) 162 if formatted.startswith("["): 163 formatted = formatted[0] + "\n " + formatted[1:] 164 # add newline before closing bracket 165 if formatted.endswith("]"): 166 formatted = formatted[:-1] + "\n" + formatted[-1] 167 return formatted 168 169 170def _bazel_name_to_file_path(name): 171 """Transform bazel reference to source file name.""" 172 for link in EXTERNAL_LINKS: 173 if name.startswith(link[0]): 174 filepath = link[1] + name[len(link[0]) :].replace(":", "/") 175 176 # For some reason, the WKT sources (such as wrappers.pb.cc) 177 # end up being reported by bazel as having an extra 'wkt/google/protobuf' 178 # in path. Removing it makes the compilation pass. 179 # TODO(jtattermusch) Get dir of this hack. 180 return filepath.replace("wkt/google/protobuf/", "") 181 return None 182 183 184def _generate_deps_file_content(): 185 """Returns the data structure with dependencies of protoc as python code.""" 186 cc_files_output = _bazel_query(BAZEL_DEPS_PROTOC_LIB_QUERY) 187 188 # Collect .cc files (that will be later included in the native extension build) 189 cc_files = [] 190 for name in cc_files_output: 191 if name.endswith(".c") or name.endswith(".cc"): 192 filepath = _bazel_name_to_file_path(name) 193 if filepath: 194 cc_files.append(filepath) 195 196 # Collect list of .proto files that will be bundled in the grpcio_tools package. 197 raw_proto_files = [] 198 for target in BAZEL_DEPS_COMMON_PROTOS_QUERIES: 199 raw_proto_files += _bazel_query(target) 200 proto_files = [ 201 name[len(PROTOBUF_PROTO_PREFIX) :].replace(":", "/") 202 for name in raw_proto_files 203 if name.endswith(".proto") and name.startswith(PROTOBUF_PROTO_PREFIX) 204 ] 205 206 commit_hash = protobuf_submodule_commit_hash() 207 commit_hash_expr = COMMIT_HASH_PREFIX + commit_hash + COMMIT_HASH_SUFFIX 208 209 deps_file_content = DEPS_FILE_CONTENT.format( 210 cc_files=_pretty_print_list(sorted(cc_files)), 211 proto_files=_pretty_print_list(sorted(set(proto_files))), 212 cc_includes=_pretty_print_list(CC_INCLUDES), 213 proto_include=repr(PROTO_INCLUDE), 214 commit_hash_expr=commit_hash_expr, 215 ) 216 return deps_file_content 217 218 219def _copy_source_tree(source, target): 220 """Copies source directory to a given target directory.""" 221 print("Copying contents of %s to %s" % (source, target)) 222 # TODO(jtattermusch): It is unclear why this legacy code needs to copy 223 # the source directory to the target via the following boilerplate. 224 # Should this code be simplified? 225 for source_dir, _, files in os.walk(source): 226 target_dir = os.path.abspath( 227 os.path.join(target, os.path.relpath(source_dir, source)) 228 ) 229 try: 230 os.makedirs(target_dir) 231 except OSError as error: 232 if error.errno != errno.EEXIST: 233 raise 234 for relative_file in files: 235 source_file = os.path.abspath( 236 os.path.join(source_dir, relative_file) 237 ) 238 target_file = os.path.abspath( 239 os.path.join(target_dir, relative_file) 240 ) 241 shutil.copyfile(source_file, target_file) 242 243 244def _delete_source_tree(target): 245 """Deletes the copied target directory.""" 246 target = GRPCIO_TOOLS_ROOT_PREFIX + target 247 target_abs = os.path.join(*target.split("/")) 248 print("Deleting copied folder %s" % (target_abs)) 249 shutil.rmtree(target_abs, ignore_errors=True) 250 251 252def main(): 253 parser = argparse.ArgumentParser() 254 # In Step 1 below, the third_party folder is copied to a location required 255 # by the build scripts. This folder does not need to be committed to the 256 # repo, so you can pass `--cleanup_third_party` in automated scripts to 257 # ensure that the temporary folders are deleted after the script runs. 258 # See Jan's TODO in _copy_source_tree above. 259 parser.add_argument( 260 "--cleanup_third_party", 261 default=False, 262 action="store_true", 263 help="Delete the temporary third_party folder", 264 ) 265 args = parser.parse_args() 266 os.chdir(GRPC_ROOT) 267 268 # Step 1: 269 # In order to be able to build the grpcio_tools package, we need the source code for the codegen plugins 270 # and its dependencies to be available under the build root of the grpcio_tools package. 271 # So we simply copy all the necessary files where the build will expect them to be. 272 for source, target in COPY_FILES_SOURCE_TARGET_PAIRS: 273 # convert the slashes in the relative path to platform-specific path dividers. 274 # All paths are relative to GRPC_ROOT 275 source_abs = os.path.join(GRPC_ROOT, os.path.join(*source.split("/"))) 276 # for targets, add grpcio_tools root prefix 277 target = GRPCIO_TOOLS_ROOT_PREFIX + target 278 target_abs = os.path.join(GRPC_ROOT, os.path.join(*target.split("/"))) 279 280 _copy_source_tree(source_abs, target_abs) 281 print( 282 "The necessary source files were copied under the grpcio_tools package" 283 " root." 284 ) 285 print() 286 287 # Step 2: 288 # Extract build metadata from bazel build (by running "bazel query") 289 # and populate the protoc_lib_deps.py file with python-readable data structure 290 # that will be used by grpcio_tools's setup.py (so it knows how to configure 291 # the native build for the codegen plugin) 292 try: 293 print('Invoking "bazel query" to gather the protobuf dependencies.') 294 protoc_lib_deps_content = _generate_deps_file_content() 295 except Exception as error: 296 # We allow this script to succeed even if we couldn't get the dependencies, 297 # as then we can assume that even without a successful bazel run the 298 # dependencies currently in source control are 'good enough'. 299 sys.stderr.write("Got non-fatal error:\n") 300 traceback.print_exc(file=sys.stderr) 301 return 302 # If we successfully got the dependencies, truncate and rewrite the deps file. 303 with open(GRPC_PYTHON_PROTOC_LIB_DEPS, "w") as deps_file: 304 deps_file.write(protoc_lib_deps_content) 305 print('File "%s" updated.' % GRPC_PYTHON_PROTOC_LIB_DEPS) 306 if args.cleanup_third_party: 307 for target in DELETE_TARGETS_ON_CLEANUP: 308 _delete_source_tree(target) 309 print("Done.") 310 311 312if __name__ == "__main__": 313 main() 314