1// Copyright 2023 Google LLC 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15syntax = "proto3"; 16 17package google.cloud.aiplatform.v1; 18 19import "google/api/field_behavior.proto"; 20import "google/api/resource.proto"; 21import "google/cloud/aiplatform/v1/explanation.proto"; 22import "google/protobuf/struct.proto"; 23import "google/protobuf/timestamp.proto"; 24 25option csharp_namespace = "Google.Cloud.AIPlatform.V1"; 26option go_package = "cloud.google.com/go/aiplatform/apiv1/aiplatformpb;aiplatformpb"; 27option java_multiple_files = true; 28option java_outer_classname = "ModelEvaluationProto"; 29option java_package = "com.google.cloud.aiplatform.v1"; 30option php_namespace = "Google\\Cloud\\AIPlatform\\V1"; 31option ruby_package = "Google::Cloud::AIPlatform::V1"; 32 33// A collection of metrics calculated by comparing Model's predictions on all of 34// the test data against annotations from the test data. 35message ModelEvaluation { 36 option (google.api.resource) = { 37 type: "aiplatform.googleapis.com/ModelEvaluation" 38 pattern: "projects/{project}/locations/{location}/models/{model}/evaluations/{evaluation}" 39 }; 40 41 message ModelEvaluationExplanationSpec { 42 // Explanation type. 43 // 44 // For AutoML Image Classification models, possible values are: 45 // 46 // * `image-integrated-gradients` 47 // * `image-xrai` 48 string explanation_type = 1; 49 50 // Explanation spec details. 51 ExplanationSpec explanation_spec = 2; 52 } 53 54 // Output only. The resource name of the ModelEvaluation. 55 string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; 56 57 // The display name of the ModelEvaluation. 58 string display_name = 10; 59 60 // Points to a YAML file stored on Google Cloud Storage describing the 61 // [metrics][google.cloud.aiplatform.v1.ModelEvaluation.metrics] of this 62 // ModelEvaluation. The schema is defined as an OpenAPI 3.0.2 [Schema 63 // Object](https://github.com/OAI/OpenAPI-Specification/blob/main/versions/3.0.2.md#schemaObject). 64 string metrics_schema_uri = 2; 65 66 // Evaluation metrics of the Model. The schema of the metrics is stored in 67 // [metrics_schema_uri][google.cloud.aiplatform.v1.ModelEvaluation.metrics_schema_uri] 68 google.protobuf.Value metrics = 3; 69 70 // Output only. Timestamp when this ModelEvaluation was created. 71 google.protobuf.Timestamp create_time = 4 72 [(google.api.field_behavior) = OUTPUT_ONLY]; 73 74 // All possible [dimensions][ModelEvaluationSlice.slice.dimension] of 75 // ModelEvaluationSlices. The dimensions can be used as the filter of the 76 // [ModelService.ListModelEvaluationSlices][google.cloud.aiplatform.v1.ModelService.ListModelEvaluationSlices] 77 // request, in the form of `slice.dimension = <dimension>`. 78 repeated string slice_dimensions = 5; 79 80 // Points to a YAML file stored on Google Cloud Storage describing 81 // [EvaluatedDataItemView.data_item_payload][] and 82 // [EvaluatedAnnotation.data_item_payload][google.cloud.aiplatform.v1.EvaluatedAnnotation.data_item_payload]. 83 // The schema is defined as an OpenAPI 3.0.2 [Schema 84 // Object](https://github.com/OAI/OpenAPI-Specification/blob/main/versions/3.0.2.md#schemaObject). 85 // 86 // This field is not populated if there are neither EvaluatedDataItemViews nor 87 // EvaluatedAnnotations under this ModelEvaluation. 88 string data_item_schema_uri = 6; 89 90 // Points to a YAML file stored on Google Cloud Storage describing 91 // [EvaluatedDataItemView.predictions][], 92 // [EvaluatedDataItemView.ground_truths][], 93 // [EvaluatedAnnotation.predictions][google.cloud.aiplatform.v1.EvaluatedAnnotation.predictions], 94 // and 95 // [EvaluatedAnnotation.ground_truths][google.cloud.aiplatform.v1.EvaluatedAnnotation.ground_truths]. 96 // The schema is defined as an OpenAPI 3.0.2 [Schema 97 // Object](https://github.com/OAI/OpenAPI-Specification/blob/main/versions/3.0.2.md#schemaObject). 98 // 99 // This field is not populated if there are neither EvaluatedDataItemViews nor 100 // EvaluatedAnnotations under this ModelEvaluation. 101 string annotation_schema_uri = 7; 102 103 // Aggregated explanation metrics for the Model's prediction output over the 104 // data this ModelEvaluation uses. This field is populated only if the Model 105 // is evaluated with explanations, and only for AutoML tabular Models. 106 // 107 ModelExplanation model_explanation = 8; 108 109 // Describes the values of 110 // [ExplanationSpec][google.cloud.aiplatform.v1.ExplanationSpec] that are used 111 // for explaining the predicted values on the evaluated data. 112 repeated ModelEvaluationExplanationSpec explanation_specs = 9; 113 114 // The metadata of the ModelEvaluation. 115 // For the ModelEvaluation uploaded from Managed Pipeline, metadata contains a 116 // structured value with keys of "pipeline_job_id", "evaluation_dataset_type", 117 // "evaluation_dataset_path". 118 google.protobuf.Value metadata = 11; 119} 120