1#!/usr/bin/env python3 2# Copyright 2022 The gRPC Authors 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16# Fake protobuf compiler for use in the Grpc.Tools MSBuild integration 17# unit tests. Its purpose is to be called from the Grpc.Tools 18# Google.Protobuf.Tools.targets MSBuild file instead of the actual protoc 19# compiler. This script: 20# - parses the command line arguments 21# - generates expected dependencies file 22# - generates dummy .cs files that are expected by the tests 23# - writes a JSON results file containing the arguments passed in 24 25# Configuration is done via environment variables as it is not possible 26# to pass additional argument when called from the MSBuild scripts under test. 27# 28# Environment variables: 29# FAKEPROTOC_PROJECTDIR - project directory 30# FAKEPROTOC_OUTDIR - output directory for generated files and output file 31# FAKEPROTOC_GENERATE_EXPECTED - list of expected generated files in format: 32# file1.proto:csfile1.cs;csfile2.cs|file2.proto:csfile3.cs;csfile4.cs|... 33 34import datetime 35import hashlib 36import json 37import os 38import sys 39 40# Set to True to write out debug messages from this script 41_dbg = True 42# file to which write the debug log 43_dbgfile = None 44 45 46def _open_debug_log(filename): 47 """Create debug file for this script.""" 48 global _dbgfile 49 if _dbg: 50 # append mode since this script may be called multiple times 51 # during one build/test 52 _dbgfile = open(filename, "a") 53 54 55def _close_debug_log(): 56 """Close the debug log file.""" 57 if _dbgfile: 58 _dbgfile.close() 59 60 61def _write_debug(msg): 62 """Write to the debug log file if debug is enabled.""" 63 if _dbg and _dbgfile: 64 print(msg, file=_dbgfile, flush=True) 65 66 67def _read_protoc_arguments(): 68 """ 69 Get the protoc argument from the command line and 70 any response files specified on the command line. 71 72 Returns the list of arguments. 73 """ 74 _write_debug("\nread_protoc_arguments") 75 result = [] 76 for arg in sys.argv[1:]: 77 _write_debug(" arg: " + arg) 78 if arg.startswith("@"): 79 # TODO(jtattermusch): inserting a "commented out" argument feels hacky 80 result.append("# RSP file: %s" % arg) 81 rsp_file_name = arg[1:] 82 result.extend(_read_rsp_file(rsp_file_name)) 83 else: 84 result.append(arg) 85 return result 86 87 88def _read_rsp_file(rspfile): 89 """ 90 Returns list of arguments from a response file. 91 """ 92 _write_debug("\nread_rsp_file: " + rspfile) 93 result = [] 94 with open(rspfile, "r") as rsp: 95 for line in rsp: 96 line = line.strip() 97 _write_debug(" line: " + line) 98 result.append(line) 99 return result 100 101 102def _parse_protoc_arguments(protoc_args, projectdir): 103 """ 104 Parse the protoc arguments from the provided list 105 """ 106 107 _write_debug("\nparse_protoc_arguments") 108 arg_dict = {} 109 for arg in protoc_args: 110 _write_debug("Parsing: %s" % arg) 111 112 # All arguments containing file or directory paths are 113 # normalized by converting all '\' and changed to '/' 114 if arg.startswith("--"): 115 # Assumes that cmdline arguments are always passed in the 116 # "--somearg=argvalue", which happens to be the form that 117 # msbuild integration uses, but it's not the only way. 118 (name, value) = arg.split("=", 1) 119 120 if ( 121 name == "--dependency_out" 122 or name == "--grpc_out" 123 or name == "--csharp_out" 124 ): 125 # For args that contain a path, make the path absolute and normalize it 126 # to make it easier to assert equality in tests. 127 value = _normalized_absolute_path(value) 128 129 if name == "--proto_path": 130 # for simplicity keep this one as relative path rather than absolute path 131 # since it is an input file that is always be near the project file 132 value = _normalized_relative_to_projectdir(value, projectdir) 133 134 _add_protoc_arg_to_dict(arg_dict, name, value) 135 136 elif arg.startswith("#"): 137 pass # ignore 138 else: 139 # arg represents a proto file name 140 arg = _normalized_relative_to_projectdir(arg, projectdir) 141 _add_protoc_arg_to_dict(arg_dict, "protofile", arg) 142 return arg_dict 143 144 145def _add_protoc_arg_to_dict(arg_dict, name, value): 146 """ 147 Add the arguments with name/value to a multi-dictionary of arguments 148 """ 149 if name not in arg_dict: 150 arg_dict[name] = [] 151 152 arg_dict[name].append(value) 153 154 155def _normalized_relative_to_projectdir(file, projectdir): 156 """Convert a file path to one relative to the project directory.""" 157 try: 158 return _normalize_slashes( 159 os.path.relpath(os.path.abspath(file), projectdir) 160 ) 161 except ValueError: 162 # On Windows if the paths are on different drives then we get this error 163 # Just return the absolute path 164 return _normalize_slashes(os.path.abspath(file)) 165 166 167def _normalized_absolute_path(file): 168 """Returns normalized absolute path to file.""" 169 return _normalize_slashes(os.path.abspath(file)) 170 171 172def _normalize_slashes(path): 173 """Change all backslashes to forward slashes to make comparing path strings easier.""" 174 return path.replace("\\", "/") 175 176 177def _write_or_update_results_json(log_dir, protofile, protoc_arg_dict): 178 """Write or update the results JSON file""" 179 180 # Read existing json. 181 # Since protoc may be called more than once each build/test if there is 182 # more than one protoc file, we read the existing data to add to it. 183 fname = os.path.abspath("%s/results.json" % log_dir) 184 if os.path.isfile(fname): 185 # Load the original contents. 186 with open(fname, "r") as forig: 187 results_json = json.load(forig) 188 else: 189 results_json = {} 190 results_json["Files"] = {} 191 192 results_json["Files"][protofile] = protoc_arg_dict 193 results_json["Metadata"] = {"timestamp": str(datetime.datetime.now())} 194 195 with open(fname, "w") as fout: 196 json.dump(results_json, fout, indent=4) 197 198 199def _parse_generate_expected(generate_expected_str): 200 """ 201 Parse FAKEPROTOC_GENERATE_EXPECTED that specifies the proto files 202 and the cs files to generate. We rely on the test to say what is 203 expected rather than trying to work it out in this script. 204 205 The format of the input is: 206 file1.proto:csfile1.cs;csfile2.cs|file2.proto:csfile3.cs;csfile4.cs|... 207 """ 208 _write_debug("\nparse_generate_expected") 209 210 result = {} 211 entries = generate_expected_str.split("|") 212 for entry in entries: 213 parts = entry.split(":") 214 pfile = _normalize_slashes(parts[0]) 215 csfiles = parts[1].split(";") 216 result[pfile] = csfiles 217 _write_debug(pfile + " : " + str(csfiles)) 218 return result 219 220 221def _get_cs_files_to_generate(protofile, proto_to_generated): 222 """Returns list of .cs files to generated based on FAKEPROTOC_GENERATE_EXPECTED env.""" 223 protoname_normalized = _normalize_slashes(protofile) 224 cs_files_to_generate = proto_to_generated.get(protoname_normalized) 225 return cs_files_to_generate 226 227 228def _is_grpc_out_file(csfile): 229 """Return true if the file is one that would be generated by gRPC plugin""" 230 # This is using the heuristics of checking that the name of the file 231 # matches *Grpc.cs which is the name that the gRPC plugin would produce. 232 return csfile.endswith("Grpc.cs") 233 234 235def _generate_cs_files( 236 protofile, cs_files_to_generate, grpc_out_dir, csharp_out_dir, projectdir 237): 238 """Create expected cs files.""" 239 _write_debug("\ngenerate_cs_files") 240 241 if not cs_files_to_generate: 242 _write_debug("No .cs files matching proto file name %s" % protofile) 243 return 244 245 if not os.path.isabs(grpc_out_dir): 246 # if not absolute, it is relative to project directory 247 grpc_out_dir = os.path.abspath("%s/%s" % (projectdir, grpc_out_dir)) 248 249 if not os.path.isabs(csharp_out_dir): 250 # if not absolute, it is relative to project directory 251 csharp_out_dir = os.path.abspath("%s/%s" % (projectdir, csharp_out_dir)) 252 253 # Ensure directories exist 254 if not os.path.isdir(grpc_out_dir): 255 os.makedirs(grpc_out_dir) 256 257 if not os.path.isdir(csharp_out_dir): 258 os.makedirs(csharp_out_dir) 259 260 timestamp = str(datetime.datetime.now()) 261 for csfile in cs_files_to_generate: 262 if csfile.endswith("Grpc.cs"): 263 csfile_fullpath = "%s/%s" % (grpc_out_dir, csfile) 264 else: 265 csfile_fullpath = "%s/%s" % (csharp_out_dir, csfile) 266 _write_debug("Creating: %s" % csfile_fullpath) 267 with open(csfile_fullpath, "w") as fout: 268 print("// Generated by fake protoc: %s" % timestamp, file=fout) 269 270 271def _create_dependency_file( 272 protofile, 273 cs_files_to_generate, 274 dependencyfile, 275 grpc_out_dir, 276 csharp_out_dir, 277): 278 """Create the expected dependency file.""" 279 _write_debug("\ncreate_dependency_file") 280 281 if not dependencyfile: 282 _write_debug("dependencyfile is not set.") 283 return 284 285 if not cs_files_to_generate: 286 _write_debug("No .cs files matching proto file name %s" % protofile) 287 return 288 289 _write_debug("Creating dependency file: %s" % dependencyfile) 290 with open(dependencyfile, "w") as out: 291 nfiles = len(cs_files_to_generate) 292 for i in range(0, nfiles): 293 csfile = cs_files_to_generate[i] 294 if csfile.endswith("Grpc.cs"): 295 cs_filename = os.path.join(grpc_out_dir, csfile) 296 else: 297 cs_filename = os.path.join(csharp_out_dir, csfile) 298 if i == nfiles - 1: 299 print("%s: %s" % (cs_filename, protofile), file=out) 300 else: 301 print("%s \\" % cs_filename, file=out) 302 303 304def _getenv(name): 305 # Note there is a bug in .NET core 3.x that lowercases the environment 306 # variable names when they are added via Process.StartInfo, so we need to 307 # check both cases here (only an issue on Linux which is case sensitive) 308 value = os.getenv(name) 309 if value is None: 310 value = os.getenv(name.lower()) 311 return value 312 313 314def _get_argument_last_occurrence_or_none(protoc_arg_dict, name): 315 # If argument was passed multiple times, take the last occurrence. 316 # If the value does not exist then return None 317 values = protoc_arg_dict.get(name) 318 if values is not None: 319 return values[-1] 320 return None 321 322 323def main(): 324 # Check environment variables for the additional arguments used in the tests. 325 326 projectdir = _getenv("FAKEPROTOC_PROJECTDIR") 327 if not projectdir: 328 print("FAKEPROTOC_PROJECTDIR not set") 329 sys.exit(1) 330 projectdir = os.path.abspath(projectdir) 331 332 # Output directory for generated files and output file 333 protoc_outdir = _getenv("FAKEPROTOC_OUTDIR") 334 if not protoc_outdir: 335 print("FAKEPROTOC_OUTDIR not set") 336 sys.exit(1) 337 protoc_outdir = os.path.abspath(protoc_outdir) 338 339 # Get list of expected generated files from env variable 340 generate_expected = _getenv("FAKEPROTOC_GENERATE_EXPECTED") 341 if not generate_expected: 342 print("FAKEPROTOC_GENERATE_EXPECTED not set") 343 sys.exit(1) 344 345 # Prepare the debug log 346 log_dir = os.path.join(protoc_outdir, "log") 347 if not os.path.isdir(log_dir): 348 os.makedirs(log_dir) 349 _open_debug_log("%s/fakeprotoc_log.txt" % log_dir) 350 351 _write_debug( 352 ( 353 "##### fakeprotoc called at %s\n" 354 + "FAKEPROTOC_PROJECTDIR = %s\n" 355 + "FAKEPROTOC_GENERATE_EXPECTED = %s\n" 356 ) 357 % (datetime.datetime.now(), projectdir, generate_expected) 358 ) 359 360 proto_to_generated = _parse_generate_expected(generate_expected) 361 protoc_args = _read_protoc_arguments() 362 protoc_arg_dict = _parse_protoc_arguments(protoc_args, projectdir) 363 364 # If argument was passed multiple times, take the last occurrence of it. 365 # TODO(jtattermusch): handle multiple occurrences of the same argument 366 dependencyfile = _get_argument_last_occurrence_or_none( 367 protoc_arg_dict, "--dependency_out" 368 ) 369 grpcout = _get_argument_last_occurrence_or_none( 370 protoc_arg_dict, "--grpc_out" 371 ) 372 csharpout = _get_argument_last_occurrence_or_none( 373 protoc_arg_dict, "--csharp_out" 374 ) 375 376 # --grpc_out might not be set in which case use --csharp_out 377 if grpcout is None: 378 grpcout = csharpout 379 380 if len(protoc_arg_dict.get("protofile")) != 1: 381 # regular protoc can process multiple .proto files passed at once, but we know 382 # the Grpc.Tools msbuild integration only ever passes one .proto file per invocation. 383 print( 384 "Expecting to get exactly one .proto file argument per fakeprotoc" 385 " invocation." 386 ) 387 sys.exit(1) 388 protofile = protoc_arg_dict.get("protofile")[0] 389 390 cs_files_to_generate = _get_cs_files_to_generate( 391 protofile=protofile, proto_to_generated=proto_to_generated 392 ) 393 394 _create_dependency_file( 395 protofile=protofile, 396 cs_files_to_generate=cs_files_to_generate, 397 dependencyfile=dependencyfile, 398 grpc_out_dir=grpcout, 399 csharp_out_dir=csharpout, 400 ) 401 402 _generate_cs_files( 403 protofile=protofile, 404 cs_files_to_generate=cs_files_to_generate, 405 grpc_out_dir=grpcout, 406 csharp_out_dir=csharpout, 407 projectdir=projectdir, 408 ) 409 410 _write_or_update_results_json( 411 log_dir=log_dir, protofile=protofile, protoc_arg_dict=protoc_arg_dict 412 ) 413 414 _close_debug_log() 415 416 417if __name__ == "__main__": 418 main() 419