# Copyright 2019 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import os from pathlib import Path import re import subprocess import sys import click import templates @click.group(invoke_without_command=False) @click.pass_context @click.version_option(message="%(version)s") def main(ctx): pass @main.command() @click.option( "--api_shortname", required=True, type=str, prompt="Service name? (e.g. automl)", help="Name for the new directory name and (default) artifact name" ) @click.option( "--name-pretty", required=True, type=str, prompt="Pretty name? (e.g. 'Cloud AutoML')", help="The human-friendly name that appears in README.md" ) @click.option( "--product-docs", required=True, type=str, prompt="Product Documentation URL", help="Documentation URL that appears in README.md" ) @click.option( "--api-description", required=True, type=str, prompt="Description for README. The first sentence is prefixed by the " "pretty name", help="Description that appears in README.md" ) @click.option( "--release-level", type=click.Choice(["stable", "preview"]), default="preview", show_default=True, help="A label that appears in repo-metadata.json. The first library " "generation is always 'preview'." ) @click.option( "--transport", type=click.Choice(["grpc", "http", "both"]), default="grpc", show_default=True, help="A label that appears in repo-metadata.json" ) @click.option("--language", type=str, default="java", show_default=True) @click.option( "--distribution-name", type=str, help="Maven coordinates of the generated library. By default it's " "com.google.cloud:google-cloud-" ) @click.option( "--api-id", type=str, help="The value of the apiid parameter used in README.md It has link to " "https://console.cloud.google.com/flows/enableapi?apiid=" ) @click.option( "--requires-billing", type=bool, default=True, show_default=True, help="Based on this value, README.md explains whether billing setup is " "needed or not." ) @click.option( "--destination-name", type=str, default=None, help="The directory name of the new library. By default it's " "java-" ) @click.option( "--proto-path", required=True, type=str, default=None, help="Path to proto file from the root of the googleapis repository to the" "directory that contains the proto files (without the version)." "For example, to generate the library for 'google/maps/routing/v2', " "then you specify this value as 'google/maps/routing'" ) @click.option( "--cloud-api", type=bool, default=True, show_default=True, help="If true, the artifact ID of the library is 'google-cloud-'; " "otherwise 'google-'" ) @click.option( "--group-id", type=str, default="com.google.cloud", show_default=True, help="The group ID of the artifact when distribution name is not set" ) @click.option( "--owlbot-image", type=str, default="gcr.io/cloud-devrel-public-resources/owlbot-java", show_default=True, help="The owlbot container image used in OwlBot.yaml" ) @click.option( "--library-type", type=str, default="GAPIC_AUTO", show_default=True, help="A label that appear in repo-metadata.json to tell how the library is " "maintained or generated" ) @click.option( "--googleapis-gen-url", type=str, default="https://github.com/googleapis/googleapis-gen.git", show_default=True, help="The URL of the repository that has generated Java code from proto " "service definition" ) def generate( api_shortname, name_pretty, product_docs, api_description, release_level, distribution_name, api_id, requires_billing, transport, language, destination_name, proto_path, cloud_api, group_id, owlbot_image, library_type, googleapis_gen_url, ): cloud_prefix = "cloud-" if cloud_api else "" output_name = destination_name if destination_name else api_shortname if distribution_name is None: distribution_name = f"{group_id}:google-{cloud_prefix}{output_name}" distribution_name_short = re.split(r"[:\/]", distribution_name)[-1] if api_id is None: api_id = f"{api_shortname}.googleapis.com" if not product_docs.startswith("https"): sys.exit("product_docs must starts with 'https://'") client_documentation = ( f"https://cloud.google.com/{language}/docs/reference/{distribution_name_short}/latest/overview" ) if proto_path is None: proto_path = f"/google/cloud/{api_shortname}" if api_shortname == "": sys.exit("api_shortname is empty") repo_metadata = { "api_shortname": api_shortname, "name_pretty": name_pretty, "product_documentation": product_docs, "api_description": api_description, "client_documentation": client_documentation, "release_level": release_level, "transport": transport, "language": language, "repo": f"googleapis/{language}-{output_name}", "repo_short": f"{language}-{output_name}", "distribution_name": distribution_name, "api_id": api_id, "library_type": library_type, } if requires_billing: repo_metadata["requires_billing"] = True # Initialize workdir workdir = Path(f"{sys.path[0]}/../../java-{output_name}").resolve() if os.path.isdir(workdir): sys.exit( "Couldn't create the module because " f"the module {workdir} already exists. In Java client library " "generation, a new API version of an existing module does not " "require new-client.py invocation. " "See go/yoshi-java-new-client#adding-a-new-service-version-by-owlbot." ) print(f"Creating a new module {workdir}") os.makedirs(workdir, exist_ok=False) # write .repo-metadata.json file with open(workdir / ".repo-metadata.json", "w") as fp: json.dump(repo_metadata, fp, indent=2) # create owlbot.py templates.render( template_name="owlbot.py.j2", output_name=str(workdir / "owlbot.py"), should_include_templates=True, template_excludes=[], ) # In monorepo, .OwlBot.yaml needs to be in the directory of the module. owlbot_yaml_location_from_module = ".OwlBot.yaml" # create owlbot config templates.render( template_name="owlbot.yaml.monorepo.j2", output_name=str(workdir / owlbot_yaml_location_from_module), artifact_name=distribution_name_short, proto_path=proto_path, module_name=f"java-{output_name}", api_shortname=api_shortname ) # get the sha256 digets for the owlbot image subprocess.check_call(["docker", "pull", "-q", owlbot_image]) owlbot_image_digest = ( subprocess.check_output( ["docker", "inspect", "--format='{{index .RepoDigests 0}}", owlbot_image,], encoding="utf-8", ) .strip() .split("@")[-1] ) user = subprocess.check_output(["id", "-u"], encoding="utf8").strip() group = subprocess.check_output(["id", "-g"], encoding="utf8").strip() # run owlbot copy print("Cloning googleapis-gen...") subprocess.check_call(["git", "clone", "-q", googleapis_gen_url, "./gen/googleapis-gen"], cwd=workdir) subprocess.check_call(["docker", "pull", "gcr.io/cloud-devrel-public-resources/owlbot-cli:latest"]) copy_code_parameters = [ "docker", "run", "--rm", "--user", f"{user}:{group}", "-v", f"{workdir}:/repo", "-v", ""f"{workdir}""/gen/googleapis-gen:/googleapis-gen", "-w", "/repo", "--env", "HOME=/tmp", "gcr.io/cloud-devrel-public-resources/owlbot-cli:latest", "copy-code", "--source-repo=/googleapis-gen", f"--config-file={owlbot_yaml_location_from_module}" ] print("Running copy-code: " + str(copy_code_parameters)) print(" in directory: " + str(workdir)) subprocess.check_call( copy_code_parameters, cwd=workdir, ) print("Removing googleapis-gen...") subprocess.check_call(["rm", "-fr", "gen"], cwd=workdir) # Bringing owl-bot-staging from the new module's directory to the root # directory so that owlbot-java can process them. subprocess.check_call( [ "mv", "owl-bot-staging", "../" ], cwd=workdir, ) monorepo_root=(workdir / '..').resolve() print("monorepo_root=",monorepo_root) print("Running the post-processor...") subprocess.check_call( [ "docker", "run", "--rm", "-v", f"{monorepo_root}:/workspace", "--user", f"{user}:{group}", owlbot_image, ], cwd=monorepo_root, ) # Remove irrelevant files from templates subprocess.check_call( ["bash", "generation/update_owlbot_postprocessor_config.sh"], cwd=monorepo_root ) subprocess.check_call( ["bash", "generation/delete_non_generated_samples.sh"], cwd=monorepo_root ) print("Regenerating the BOM") subprocess.check_call( [ "bash", "generation/generate_gapic_bom.sh", ], cwd=monorepo_root, ) print("Regenerating root pom.xml") # This script takes care of updating the root pom.xml os.system(f"cd {monorepo_root} && generation/generate_root_pom.sh") print("Consolidating configurations") subprocess.check_call( [ "bash", "generation/consolidate_config.sh" ], cwd=monorepo_root, ) print("Setting parent poms") subprocess.check_call( [ "bash", "generation/set_parent_pom.sh" ], cwd=monorepo_root, ) print("Applying the versions") subprocess.check_call( [ "bash", "generation/apply_current_versions.sh" ], cwd=monorepo_root, ) print("Adding annotations in readme") subprocess.check_call( [ "bash", "generation/readme_update.sh" ], cwd=monorepo_root, ) print(f"Prepared new library in {workdir}") print(f"Please create a pull request:\n" f" $ git checkout -b new_module_java-{output_name}\n" f" $ git add .\n" f" $ git commit -m 'feat: [{api_shortname}] new module for {api_shortname}'\n" f" $ gh pr create --title 'feat: [{api_shortname}] new module for {api_shortname}'") if __name__ == "__main__": main()