1# Copyright 2019 Google LLC 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# https://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15import json 16import os 17from pathlib import Path 18import re 19import subprocess 20import sys 21 22import click 23import templates 24 25 26@click.group(invoke_without_command=False) 27@click.pass_context 28@click.version_option(message="%(version)s") 29def main(ctx): 30 pass 31 32@main.command() 33@click.option( 34 "--api_shortname", 35 required=True, 36 type=str, 37 prompt="Service name? (e.g. automl)", 38 help="Name for the new directory name and (default) artifact name" 39) 40@click.option( 41 "--name-pretty", 42 required=True, 43 type=str, 44 prompt="Pretty name? (e.g. 'Cloud AutoML')", 45 help="The human-friendly name that appears in README.md" 46) 47@click.option( 48 "--product-docs", 49 required=True, 50 type=str, 51 prompt="Product Documentation URL", 52 help="Documentation URL that appears in README.md" 53) 54@click.option( 55 "--api-description", 56 required=True, 57 type=str, 58 prompt="Description for README. The first sentence is prefixed by the " 59 "pretty name", 60 help="Description that appears in README.md" 61) 62@click.option( 63 "--release-level", 64 type=click.Choice(["stable", "preview"]), 65 default="preview", 66 show_default=True, 67 help="A label that appears in repo-metadata.json. The first library " 68 "generation is always 'preview'." 69) 70@click.option( 71 "--transport", 72 type=click.Choice(["grpc", "http", "both"]), 73 default="grpc", 74 show_default=True, 75 help="A label that appears in repo-metadata.json" 76) 77@click.option("--language", type=str, default="java", show_default=True) 78@click.option( 79 "--distribution-name", 80 type=str, 81 help="Maven coordinates of the generated library. By default it's " 82 "com.google.cloud:google-cloud-<api_shortname>" 83) 84@click.option( 85 "--api-id", 86 type=str, 87 help="The value of the apiid parameter used in README.md It has link to " 88 "https://console.cloud.google.com/flows/enableapi?apiid=<api_id>" 89) 90@click.option( 91 "--requires-billing", 92 type=bool, 93 default=True, 94 show_default=True, 95 help="Based on this value, README.md explains whether billing setup is " 96 "needed or not." 97) 98@click.option( 99 "--destination-name", 100 type=str, 101 default=None, 102 help="The directory name of the new library. By default it's " 103 "java-<api_shortname>" 104) 105@click.option( 106 "--proto-path", 107 required=True, 108 type=str, 109 default=None, 110 help="Path to proto file from the root of the googleapis repository to the" 111 "directory that contains the proto files (without the version)." 112 "For example, to generate the library for 'google/maps/routing/v2', " 113 "then you specify this value as 'google/maps/routing'" 114) 115@click.option( 116 "--cloud-api", 117 type=bool, 118 default=True, 119 show_default=True, 120 help="If true, the artifact ID of the library is 'google-cloud-'; " 121 "otherwise 'google-'" 122) 123@click.option( 124 "--group-id", 125 type=str, 126 default="com.google.cloud", 127 show_default=True, 128 help="The group ID of the artifact when distribution name is not set" 129) 130@click.option( 131 "--owlbot-image", 132 type=str, 133 default="gcr.io/cloud-devrel-public-resources/owlbot-java", 134 show_default=True, 135 help="The owlbot container image used in OwlBot.yaml" 136) 137@click.option( 138 "--library-type", 139 type=str, 140 default="GAPIC_AUTO", 141 show_default=True, 142 help="A label that appear in repo-metadata.json to tell how the library is " 143 "maintained or generated" 144) 145@click.option( 146 "--googleapis-gen-url", 147 type=str, 148 default="https://github.com/googleapis/googleapis-gen.git", 149 show_default=True, 150 help="The URL of the repository that has generated Java code from proto " 151 "service definition" 152) 153def generate( 154 api_shortname, 155 name_pretty, 156 product_docs, 157 api_description, 158 release_level, 159 distribution_name, 160 api_id, 161 requires_billing, 162 transport, 163 language, 164 destination_name, 165 proto_path, 166 cloud_api, 167 group_id, 168 owlbot_image, 169 library_type, 170 googleapis_gen_url, 171): 172 cloud_prefix = "cloud-" if cloud_api else "" 173 174 output_name = destination_name if destination_name else api_shortname 175 if distribution_name is None: 176 distribution_name = f"{group_id}:google-{cloud_prefix}{output_name}" 177 178 distribution_name_short = re.split(r"[:\/]", distribution_name)[-1] 179 180 if api_id is None: 181 api_id = f"{api_shortname}.googleapis.com" 182 183 if not product_docs.startswith("https"): 184 sys.exit("product_docs must starts with 'https://'") 185 186 client_documentation = ( 187 f"https://cloud.google.com/{language}/docs/reference/{distribution_name_short}/latest/overview" 188 ) 189 190 if proto_path is None: 191 proto_path = f"/google/cloud/{api_shortname}" 192 193 if api_shortname == "": 194 sys.exit("api_shortname is empty") 195 196 repo_metadata = { 197 "api_shortname": api_shortname, 198 "name_pretty": name_pretty, 199 "product_documentation": product_docs, 200 "api_description": api_description, 201 "client_documentation": client_documentation, 202 "release_level": release_level, 203 "transport": transport, 204 "language": language, 205 "repo": f"googleapis/{language}-{output_name}", 206 "repo_short": f"{language}-{output_name}", 207 "distribution_name": distribution_name, 208 "api_id": api_id, 209 "library_type": library_type, 210 } 211 if requires_billing: 212 repo_metadata["requires_billing"] = True 213 214 # Initialize workdir 215 workdir = Path(f"{sys.path[0]}/../../java-{output_name}").resolve() 216 if os.path.isdir(workdir): 217 sys.exit( 218 "Couldn't create the module because " 219 f"the module {workdir} already exists. In Java client library " 220 "generation, a new API version of an existing module does not " 221 "require new-client.py invocation. " 222 "See go/yoshi-java-new-client#adding-a-new-service-version-by-owlbot." 223 ) 224 print(f"Creating a new module {workdir}") 225 os.makedirs(workdir, exist_ok=False) 226 # write .repo-metadata.json file 227 with open(workdir / ".repo-metadata.json", "w") as fp: 228 json.dump(repo_metadata, fp, indent=2) 229 230 # create owlbot.py 231 templates.render( 232 template_name="owlbot.py.j2", 233 output_name=str(workdir / "owlbot.py"), 234 should_include_templates=True, 235 template_excludes=[], 236 ) 237 238 # In monorepo, .OwlBot.yaml needs to be in the directory of the module. 239 owlbot_yaml_location_from_module = ".OwlBot.yaml" 240 # create owlbot config 241 templates.render( 242 template_name="owlbot.yaml.monorepo.j2", 243 output_name=str(workdir / owlbot_yaml_location_from_module), 244 artifact_name=distribution_name_short, 245 proto_path=proto_path, 246 module_name=f"java-{output_name}", 247 api_shortname=api_shortname 248 ) 249 250 # get the sha256 digets for the owlbot image 251 subprocess.check_call(["docker", "pull", "-q", owlbot_image]) 252 owlbot_image_digest = ( 253 subprocess.check_output( 254 ["docker", "inspect", "--format='{{index .RepoDigests 0}}", owlbot_image,], 255 encoding="utf-8", 256 ) 257 .strip() 258 .split("@")[-1] 259 ) 260 261 user = subprocess.check_output(["id", "-u"], encoding="utf8").strip() 262 group = subprocess.check_output(["id", "-g"], encoding="utf8").strip() 263 264 # run owlbot copy 265 print("Cloning googleapis-gen...") 266 subprocess.check_call(["git", "clone", "-q", googleapis_gen_url, "./gen/googleapis-gen"], cwd=workdir) 267 subprocess.check_call(["docker", "pull", "gcr.io/cloud-devrel-public-resources/owlbot-cli:latest"]) 268 copy_code_parameters = [ 269 "docker", 270 "run", 271 "--rm", 272 "--user", 273 f"{user}:{group}", 274 "-v", 275 f"{workdir}:/repo", 276 "-v", 277 ""f"{workdir}""/gen/googleapis-gen:/googleapis-gen", 278 "-w", 279 "/repo", 280 "--env", "HOME=/tmp", 281 "gcr.io/cloud-devrel-public-resources/owlbot-cli:latest", 282 "copy-code", 283 "--source-repo=/googleapis-gen", 284 f"--config-file={owlbot_yaml_location_from_module}" 285 ] 286 print("Running copy-code: " + str(copy_code_parameters)) 287 print(" in directory: " + str(workdir)) 288 subprocess.check_call( 289 copy_code_parameters, 290 cwd=workdir, 291 ) 292 293 print("Removing googleapis-gen...") 294 subprocess.check_call(["rm", "-fr", "gen"], cwd=workdir) 295 296 # Bringing owl-bot-staging from the new module's directory to the root 297 # directory so that owlbot-java can process them. 298 subprocess.check_call( 299 [ 300 "mv", 301 "owl-bot-staging", 302 "../" 303 ], 304 cwd=workdir, 305 ) 306 monorepo_root=(workdir / '..').resolve() 307 print("monorepo_root=",monorepo_root) 308 print("Running the post-processor...") 309 subprocess.check_call( 310 [ 311 "docker", 312 "run", 313 "--rm", 314 "-v", 315 f"{monorepo_root}:/workspace", 316 "--user", 317 f"{user}:{group}", 318 owlbot_image, 319 ], 320 cwd=monorepo_root, 321 ) 322 323 # Remove irrelevant files from templates 324 subprocess.check_call( 325 ["bash", "generation/update_owlbot_postprocessor_config.sh"], 326 cwd=monorepo_root 327 ) 328 subprocess.check_call( 329 ["bash", "generation/delete_non_generated_samples.sh"], 330 cwd=monorepo_root 331 ) 332 333 print("Regenerating the BOM") 334 subprocess.check_call( 335 [ 336 "bash", "generation/generate_gapic_bom.sh", 337 ], 338 cwd=monorepo_root, 339 ) 340 341 print("Regenerating root pom.xml") 342 343 # This script takes care of updating the root pom.xml 344 os.system(f"cd {monorepo_root} && generation/generate_root_pom.sh") 345 346 print("Consolidating configurations") 347 subprocess.check_call( 348 [ 349 "bash", "generation/consolidate_config.sh" 350 ], 351 cwd=monorepo_root, 352 ) 353 print("Setting parent poms") 354 subprocess.check_call( 355 [ 356 "bash", "generation/set_parent_pom.sh" 357 ], 358 cwd=monorepo_root, 359 ) 360 361 print("Applying the versions") 362 subprocess.check_call( 363 [ 364 "bash", "generation/apply_current_versions.sh" 365 ], 366 cwd=monorepo_root, 367 ) 368 369 print("Adding annotations in readme") 370 subprocess.check_call( 371 [ 372 "bash", "generation/readme_update.sh" 373 ], 374 cwd=monorepo_root, 375 ) 376 377 print(f"Prepared new library in {workdir}") 378 print(f"Please create a pull request:\n" 379 f" $ git checkout -b new_module_java-{output_name}\n" 380 f" $ git add .\n" 381 f" $ git commit -m 'feat: [{api_shortname}] new module for {api_shortname}'\n" 382 f" $ gh pr create --title 'feat: [{api_shortname}] new module for {api_shortname}'") 383 384if __name__ == "__main__": 385 main() 386