1// Copyright 2023 Google LLC 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15syntax = "proto3"; 16 17package google.cloud.aiplatform.v1; 18 19import "google/api/field_behavior.proto"; 20import "google/api/resource.proto"; 21import "google/cloud/aiplatform/v1/encryption_spec.proto"; 22import "google/cloud/aiplatform/v1/env_var.proto"; 23import "google/cloud/aiplatform/v1/io.proto"; 24import "google/cloud/aiplatform/v1/job_state.proto"; 25import "google/cloud/aiplatform/v1/machine_resources.proto"; 26import "google/protobuf/duration.proto"; 27import "google/protobuf/timestamp.proto"; 28import "google/rpc/status.proto"; 29 30option csharp_namespace = "Google.Cloud.AIPlatform.V1"; 31option go_package = "cloud.google.com/go/aiplatform/apiv1/aiplatformpb;aiplatformpb"; 32option java_multiple_files = true; 33option java_outer_classname = "CustomJobProto"; 34option java_package = "com.google.cloud.aiplatform.v1"; 35option php_namespace = "Google\\Cloud\\AIPlatform\\V1"; 36option ruby_package = "Google::Cloud::AIPlatform::V1"; 37 38// Represents a job that runs custom workloads such as a Docker container or a 39// Python package. A CustomJob can have multiple worker pools and each worker 40// pool can have its own machine and input spec. A CustomJob will be cleaned up 41// once the job enters terminal state (failed or succeeded). 42message CustomJob { 43 option (google.api.resource) = { 44 type: "aiplatform.googleapis.com/CustomJob" 45 pattern: "projects/{project}/locations/{location}/customJobs/{custom_job}" 46 }; 47 48 // Output only. Resource name of a CustomJob. 49 string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; 50 51 // Required. The display name of the CustomJob. 52 // The name can be up to 128 characters long and can consist of any UTF-8 53 // characters. 54 string display_name = 2 [(google.api.field_behavior) = REQUIRED]; 55 56 // Required. Job spec. 57 CustomJobSpec job_spec = 4 [(google.api.field_behavior) = REQUIRED]; 58 59 // Output only. The detailed state of the job. 60 JobState state = 5 [(google.api.field_behavior) = OUTPUT_ONLY]; 61 62 // Output only. Time when the CustomJob was created. 63 google.protobuf.Timestamp create_time = 6 64 [(google.api.field_behavior) = OUTPUT_ONLY]; 65 66 // Output only. Time when the CustomJob for the first time entered the 67 // `JOB_STATE_RUNNING` state. 68 google.protobuf.Timestamp start_time = 7 69 [(google.api.field_behavior) = OUTPUT_ONLY]; 70 71 // Output only. Time when the CustomJob entered any of the following states: 72 // `JOB_STATE_SUCCEEDED`, `JOB_STATE_FAILED`, `JOB_STATE_CANCELLED`. 73 google.protobuf.Timestamp end_time = 8 74 [(google.api.field_behavior) = OUTPUT_ONLY]; 75 76 // Output only. Time when the CustomJob was most recently updated. 77 google.protobuf.Timestamp update_time = 9 78 [(google.api.field_behavior) = OUTPUT_ONLY]; 79 80 // Output only. Only populated when job's state is `JOB_STATE_FAILED` or 81 // `JOB_STATE_CANCELLED`. 82 google.rpc.Status error = 10 [(google.api.field_behavior) = OUTPUT_ONLY]; 83 84 // The labels with user-defined metadata to organize CustomJobs. 85 // 86 // Label keys and values can be no longer than 64 characters 87 // (Unicode codepoints), can only contain lowercase letters, numeric 88 // characters, underscores and dashes. International characters are allowed. 89 // 90 // See https://goo.gl/xmQnxf for more information and examples of labels. 91 map<string, string> labels = 11; 92 93 // Customer-managed encryption key options for a CustomJob. If this is set, 94 // then all resources created by the CustomJob will be encrypted with the 95 // provided encryption key. 96 EncryptionSpec encryption_spec = 12; 97 98 // Output only. URIs for accessing [interactive 99 // shells](https://cloud.google.com/vertex-ai/docs/training/monitor-debug-interactive-shell) 100 // (one URI for each training node). Only available if 101 // [job_spec.enable_web_access][google.cloud.aiplatform.v1.CustomJobSpec.enable_web_access] 102 // is `true`. 103 // 104 // The keys are names of each node in the training job; for example, 105 // `workerpool0-0` for the primary node, `workerpool1-0` for the first node in 106 // the second worker pool, and `workerpool1-1` for the second node in the 107 // second worker pool. 108 // 109 // The values are the URIs for each node's interactive shell. 110 map<string, string> web_access_uris = 16 111 [(google.api.field_behavior) = OUTPUT_ONLY]; 112} 113 114// Represents the spec of a CustomJob. 115message CustomJobSpec { 116 // Required. The spec of the worker pools including machine type and Docker 117 // image. All worker pools except the first one are optional and can be 118 // skipped by providing an empty value. 119 repeated WorkerPoolSpec worker_pool_specs = 1 120 [(google.api.field_behavior) = REQUIRED]; 121 122 // Scheduling options for a CustomJob. 123 Scheduling scheduling = 3; 124 125 // Specifies the service account for workload run-as account. 126 // Users submitting jobs must have act-as permission on this run-as account. 127 // If unspecified, the [Vertex AI Custom Code Service 128 // Agent](https://cloud.google.com/vertex-ai/docs/general/access-control#service-agents) 129 // for the CustomJob's project is used. 130 string service_account = 4; 131 132 // Optional. The full name of the Compute Engine 133 // [network](/compute/docs/networks-and-firewalls#networks) to which the Job 134 // should be peered. For example, `projects/12345/global/networks/myVPC`. 135 // [Format](/compute/docs/reference/rest/v1/networks/insert) 136 // is of the form `projects/{project}/global/networks/{network}`. 137 // Where {project} is a project number, as in `12345`, and {network} is a 138 // network name. 139 // 140 // To specify this field, you must have already [configured VPC Network 141 // Peering for Vertex 142 // AI](https://cloud.google.com/vertex-ai/docs/general/vpc-peering). 143 // 144 // If this field is left unspecified, the job is not peered with any network. 145 string network = 5 [ 146 (google.api.field_behavior) = OPTIONAL, 147 (google.api.resource_reference) = { type: "compute.googleapis.com/Network" } 148 ]; 149 150 // Optional. A list of names for the reserved ip ranges under the VPC network 151 // that can be used for this job. 152 // 153 // If set, we will deploy the job within the provided ip ranges. Otherwise, 154 // the job will be deployed to any ip ranges under the provided VPC 155 // network. 156 // 157 // Example: ['vertex-ai-ip-range']. 158 repeated string reserved_ip_ranges = 13 159 [(google.api.field_behavior) = OPTIONAL]; 160 161 // The Cloud Storage location to store the output of this CustomJob or 162 // HyperparameterTuningJob. For HyperparameterTuningJob, 163 // the baseOutputDirectory of 164 // each child CustomJob backing a Trial is set to a subdirectory of name 165 // [id][google.cloud.aiplatform.v1.Trial.id] under its parent 166 // HyperparameterTuningJob's baseOutputDirectory. 167 // 168 // The following Vertex AI environment variables will be passed to 169 // containers or python modules when this field is set: 170 // 171 // For CustomJob: 172 // 173 // * AIP_MODEL_DIR = `<base_output_directory>/model/` 174 // * AIP_CHECKPOINT_DIR = `<base_output_directory>/checkpoints/` 175 // * AIP_TENSORBOARD_LOG_DIR = `<base_output_directory>/logs/` 176 // 177 // For CustomJob backing a Trial of HyperparameterTuningJob: 178 // 179 // * AIP_MODEL_DIR = `<base_output_directory>/<trial_id>/model/` 180 // * AIP_CHECKPOINT_DIR = `<base_output_directory>/<trial_id>/checkpoints/` 181 // * AIP_TENSORBOARD_LOG_DIR = `<base_output_directory>/<trial_id>/logs/` 182 GcsDestination base_output_directory = 6; 183 184 // Optional. The name of a Vertex AI 185 // [Tensorboard][google.cloud.aiplatform.v1.Tensorboard] resource to which 186 // this CustomJob will upload Tensorboard logs. Format: 187 // `projects/{project}/locations/{location}/tensorboards/{tensorboard}` 188 string tensorboard = 7 [ 189 (google.api.field_behavior) = OPTIONAL, 190 (google.api.resource_reference) = { 191 type: "aiplatform.googleapis.com/Tensorboard" 192 } 193 ]; 194 195 // Optional. Whether you want Vertex AI to enable [interactive shell 196 // access](https://cloud.google.com/vertex-ai/docs/training/monitor-debug-interactive-shell) 197 // to training containers. 198 // 199 // If set to `true`, you can access interactive shells at the URIs given 200 // by 201 // [CustomJob.web_access_uris][google.cloud.aiplatform.v1.CustomJob.web_access_uris] 202 // or 203 // [Trial.web_access_uris][google.cloud.aiplatform.v1.Trial.web_access_uris] 204 // (within 205 // [HyperparameterTuningJob.trials][google.cloud.aiplatform.v1.HyperparameterTuningJob.trials]). 206 bool enable_web_access = 10 [(google.api.field_behavior) = OPTIONAL]; 207 208 // Optional. Whether you want Vertex AI to enable access to the customized 209 // dashboard in training chief container. 210 // 211 // If set to `true`, you can access the dashboard at the URIs given 212 // by 213 // [CustomJob.web_access_uris][google.cloud.aiplatform.v1.CustomJob.web_access_uris] 214 // or 215 // [Trial.web_access_uris][google.cloud.aiplatform.v1.Trial.web_access_uris] 216 // (within 217 // [HyperparameterTuningJob.trials][google.cloud.aiplatform.v1.HyperparameterTuningJob.trials]). 218 bool enable_dashboard_access = 16 [(google.api.field_behavior) = OPTIONAL]; 219 220 // Optional. The Experiment associated with this job. 221 // Format: 222 // `projects/{project}/locations/{location}/metadataStores/{metadataStores}/contexts/{experiment-name}` 223 string experiment = 17 [ 224 (google.api.field_behavior) = OPTIONAL, 225 (google.api.resource_reference) = { 226 type: "aiplatform.googleapis.com/Context" 227 } 228 ]; 229 230 // Optional. The Experiment Run associated with this job. 231 // Format: 232 // `projects/{project}/locations/{location}/metadataStores/{metadataStores}/contexts/{experiment-name}-{experiment-run-name}` 233 string experiment_run = 18 [ 234 (google.api.field_behavior) = OPTIONAL, 235 (google.api.resource_reference) = { 236 type: "aiplatform.googleapis.com/Context" 237 } 238 ]; 239} 240 241// Represents the spec of a worker pool in a job. 242message WorkerPoolSpec { 243 // The custom task to be executed in this worker pool. 244 oneof task { 245 // The custom container task. 246 ContainerSpec container_spec = 6; 247 248 // The Python packaged task. 249 PythonPackageSpec python_package_spec = 7; 250 } 251 252 // Optional. Immutable. The specification of a single machine. 253 MachineSpec machine_spec = 1 [ 254 (google.api.field_behavior) = OPTIONAL, 255 (google.api.field_behavior) = IMMUTABLE 256 ]; 257 258 // Optional. The number of worker replicas to use for this worker pool. 259 int64 replica_count = 2 [(google.api.field_behavior) = OPTIONAL]; 260 261 // Optional. List of NFS mount spec. 262 repeated NfsMount nfs_mounts = 4 [(google.api.field_behavior) = OPTIONAL]; 263 264 // Disk spec. 265 DiskSpec disk_spec = 5; 266} 267 268// The spec of a Container. 269message ContainerSpec { 270 // Required. The URI of a container image in the Container Registry that is to 271 // be run on each worker replica. 272 string image_uri = 1 [(google.api.field_behavior) = REQUIRED]; 273 274 // The command to be invoked when the container is started. 275 // It overrides the entrypoint instruction in Dockerfile when provided. 276 repeated string command = 2; 277 278 // The arguments to be passed when starting the container. 279 repeated string args = 3; 280 281 // Environment variables to be passed to the container. 282 // Maximum limit is 100. 283 repeated EnvVar env = 4; 284} 285 286// The spec of a Python packaged code. 287message PythonPackageSpec { 288 // Required. The URI of a container image in Artifact Registry that will run 289 // the provided Python package. Vertex AI provides a wide range of executor 290 // images with pre-installed packages to meet users' various use cases. See 291 // the list of [pre-built containers for 292 // training](https://cloud.google.com/vertex-ai/docs/training/pre-built-containers). 293 // You must use an image from this list. 294 string executor_image_uri = 1 [(google.api.field_behavior) = REQUIRED]; 295 296 // Required. The Google Cloud Storage location of the Python package files 297 // which are the training program and its dependent packages. The maximum 298 // number of package URIs is 100. 299 repeated string package_uris = 2 [(google.api.field_behavior) = REQUIRED]; 300 301 // Required. The Python module name to run after installing the packages. 302 string python_module = 3 [(google.api.field_behavior) = REQUIRED]; 303 304 // Command line arguments to be passed to the Python task. 305 repeated string args = 4; 306 307 // Environment variables to be passed to the python module. 308 // Maximum limit is 100. 309 repeated EnvVar env = 5; 310} 311 312// All parameters related to queuing and scheduling of custom jobs. 313message Scheduling { 314 // The maximum job running time. The default is 7 days. 315 google.protobuf.Duration timeout = 1; 316 317 // Restarts the entire CustomJob if a worker gets restarted. 318 // This feature can be used by distributed training jobs that are not 319 // resilient to workers leaving and joining a job. 320 bool restart_job_on_worker_restart = 3; 321} 322