1// Copyright 2023 Google LLC
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15syntax = "proto3";
16
17package google.cloud.aiplatform.v1;
18
19import "google/api/field_behavior.proto";
20import "google/api/resource.proto";
21import "google/cloud/aiplatform/v1/explanation.proto";
22import "google/protobuf/struct.proto";
23import "google/protobuf/timestamp.proto";
24
25option csharp_namespace = "Google.Cloud.AIPlatform.V1";
26option go_package = "cloud.google.com/go/aiplatform/apiv1/aiplatformpb;aiplatformpb";
27option java_multiple_files = true;
28option java_outer_classname = "ModelEvaluationProto";
29option java_package = "com.google.cloud.aiplatform.v1";
30option php_namespace = "Google\\Cloud\\AIPlatform\\V1";
31option ruby_package = "Google::Cloud::AIPlatform::V1";
32
33// A collection of metrics calculated by comparing Model's predictions on all of
34// the test data against annotations from the test data.
35message ModelEvaluation {
36  option (google.api.resource) = {
37    type: "aiplatform.googleapis.com/ModelEvaluation"
38    pattern: "projects/{project}/locations/{location}/models/{model}/evaluations/{evaluation}"
39  };
40
41  message ModelEvaluationExplanationSpec {
42    // Explanation type.
43    //
44    // For AutoML Image Classification models, possible values are:
45    //
46    //   * `image-integrated-gradients`
47    //   * `image-xrai`
48    string explanation_type = 1;
49
50    // Explanation spec details.
51    ExplanationSpec explanation_spec = 2;
52  }
53
54  // Output only. The resource name of the ModelEvaluation.
55  string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
56
57  // The display name of the ModelEvaluation.
58  string display_name = 10;
59
60  // Points to a YAML file stored on Google Cloud Storage describing the
61  // [metrics][google.cloud.aiplatform.v1.ModelEvaluation.metrics] of this
62  // ModelEvaluation. The schema is defined as an OpenAPI 3.0.2 [Schema
63  // Object](https://github.com/OAI/OpenAPI-Specification/blob/main/versions/3.0.2.md#schemaObject).
64  string metrics_schema_uri = 2;
65
66  // Evaluation metrics of the Model. The schema of the metrics is stored in
67  // [metrics_schema_uri][google.cloud.aiplatform.v1.ModelEvaluation.metrics_schema_uri]
68  google.protobuf.Value metrics = 3;
69
70  // Output only. Timestamp when this ModelEvaluation was created.
71  google.protobuf.Timestamp create_time = 4
72      [(google.api.field_behavior) = OUTPUT_ONLY];
73
74  // All possible [dimensions][ModelEvaluationSlice.slice.dimension] of
75  // ModelEvaluationSlices. The dimensions can be used as the filter of the
76  // [ModelService.ListModelEvaluationSlices][google.cloud.aiplatform.v1.ModelService.ListModelEvaluationSlices]
77  // request, in the form of `slice.dimension = <dimension>`.
78  repeated string slice_dimensions = 5;
79
80  // Points to a YAML file stored on Google Cloud Storage describing
81  // [EvaluatedDataItemView.data_item_payload][] and
82  // [EvaluatedAnnotation.data_item_payload][google.cloud.aiplatform.v1.EvaluatedAnnotation.data_item_payload].
83  // The schema is defined as an OpenAPI 3.0.2 [Schema
84  // Object](https://github.com/OAI/OpenAPI-Specification/blob/main/versions/3.0.2.md#schemaObject).
85  //
86  // This field is not populated if there are neither EvaluatedDataItemViews nor
87  // EvaluatedAnnotations under this ModelEvaluation.
88  string data_item_schema_uri = 6;
89
90  // Points to a YAML file stored on Google Cloud Storage describing
91  // [EvaluatedDataItemView.predictions][],
92  // [EvaluatedDataItemView.ground_truths][],
93  // [EvaluatedAnnotation.predictions][google.cloud.aiplatform.v1.EvaluatedAnnotation.predictions],
94  // and
95  // [EvaluatedAnnotation.ground_truths][google.cloud.aiplatform.v1.EvaluatedAnnotation.ground_truths].
96  // The schema is defined as an OpenAPI 3.0.2 [Schema
97  // Object](https://github.com/OAI/OpenAPI-Specification/blob/main/versions/3.0.2.md#schemaObject).
98  //
99  // This field is not populated if there are neither EvaluatedDataItemViews nor
100  // EvaluatedAnnotations under this ModelEvaluation.
101  string annotation_schema_uri = 7;
102
103  // Aggregated explanation metrics for the Model's prediction output over the
104  // data this ModelEvaluation uses. This field is populated only if the Model
105  // is evaluated with explanations, and only for AutoML tabular Models.
106  //
107  ModelExplanation model_explanation = 8;
108
109  // Describes the values of
110  // [ExplanationSpec][google.cloud.aiplatform.v1.ExplanationSpec] that are used
111  // for explaining the predicted values on the evaluated data.
112  repeated ModelEvaluationExplanationSpec explanation_specs = 9;
113
114  // The metadata of the ModelEvaluation.
115  // For the ModelEvaluation uploaded from Managed Pipeline, metadata contains a
116  // structured value with keys of "pipeline_job_id", "evaluation_dataset_type",
117  // "evaluation_dataset_path".
118  google.protobuf.Value metadata = 11;
119}
120