1// Copyright 2021 Google LLC
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15syntax = "proto3";
16
17package google.cloud.automl.v1;
18
19option csharp_namespace = "Google.Cloud.AutoML.V1";
20option go_package = "cloud.google.com/go/automl/apiv1/automlpb;automlpb";
21option java_multiple_files = true;
22option java_outer_classname = "ClassificationProto";
23option java_package = "com.google.cloud.automl.v1";
24option php_namespace = "Google\\Cloud\\AutoMl\\V1";
25option ruby_package = "Google::Cloud::AutoML::V1";
26
27// Type of the classification problem.
28enum ClassificationType {
29  // An un-set value of this enum.
30  CLASSIFICATION_TYPE_UNSPECIFIED = 0;
31
32  // At most one label is allowed per example.
33  MULTICLASS = 1;
34
35  // Multiple labels are allowed for one example.
36  MULTILABEL = 2;
37}
38
39// Contains annotation details specific to classification.
40message ClassificationAnnotation {
41  // Output only. A confidence estimate between 0.0 and 1.0. A higher value
42  // means greater confidence that the annotation is positive. If a user
43  // approves an annotation as negative or positive, the score value remains
44  // unchanged. If a user creates an annotation, the score is 0 for negative or
45  // 1 for positive.
46  float score = 1;
47}
48
49// Model evaluation metrics for classification problems.
50// Note: For Video Classification this metrics only describe quality of the
51// Video Classification predictions of "segment_classification" type.
52message ClassificationEvaluationMetrics {
53  // Metrics for a single confidence threshold.
54  message ConfidenceMetricsEntry {
55    // Output only. Metrics are computed with an assumption that the model
56    // never returns predictions with score lower than this value.
57    float confidence_threshold = 1;
58
59    // Output only. Metrics are computed with an assumption that the model
60    // always returns at most this many predictions (ordered by their score,
61    // descendingly), but they all still need to meet the confidence_threshold.
62    int32 position_threshold = 14;
63
64    // Output only. Recall (True Positive Rate) for the given confidence
65    // threshold.
66    float recall = 2;
67
68    // Output only. Precision for the given confidence threshold.
69    float precision = 3;
70
71    // Output only. False Positive Rate for the given confidence threshold.
72    float false_positive_rate = 8;
73
74    // Output only. The harmonic mean of recall and precision.
75    float f1_score = 4;
76
77    // Output only. The Recall (True Positive Rate) when only considering the
78    // label that has the highest prediction score and not below the confidence
79    // threshold for each example.
80    float recall_at1 = 5;
81
82    // Output only. The precision when only considering the label that has the
83    // highest prediction score and not below the confidence threshold for each
84    // example.
85    float precision_at1 = 6;
86
87    // Output only. The False Positive Rate when only considering the label that
88    // has the highest prediction score and not below the confidence threshold
89    // for each example.
90    float false_positive_rate_at1 = 9;
91
92    // Output only. The harmonic mean of [recall_at1][google.cloud.automl.v1.ClassificationEvaluationMetrics.ConfidenceMetricsEntry.recall_at1] and [precision_at1][google.cloud.automl.v1.ClassificationEvaluationMetrics.ConfidenceMetricsEntry.precision_at1].
93    float f1_score_at1 = 7;
94
95    // Output only. The number of model created labels that match a ground truth
96    // label.
97    int64 true_positive_count = 10;
98
99    // Output only. The number of model created labels that do not match a
100    // ground truth label.
101    int64 false_positive_count = 11;
102
103    // Output only. The number of ground truth labels that are not matched
104    // by a model created label.
105    int64 false_negative_count = 12;
106
107    // Output only. The number of labels that were not created by the model,
108    // but if they would, they would not match a ground truth label.
109    int64 true_negative_count = 13;
110  }
111
112  // Confusion matrix of the model running the classification.
113  message ConfusionMatrix {
114    // Output only. A row in the confusion matrix.
115    message Row {
116      // Output only. Value of the specific cell in the confusion matrix.
117      // The number of values each row has (i.e. the length of the row) is equal
118      // to the length of the `annotation_spec_id` field or, if that one is not
119      // populated, length of the [display_name][google.cloud.automl.v1.ClassificationEvaluationMetrics.ConfusionMatrix.display_name] field.
120      repeated int32 example_count = 1;
121    }
122
123    // Output only. IDs of the annotation specs used in the confusion matrix.
124    // For Tables CLASSIFICATION
125    // [prediction_type][google.cloud.automl.v1p1beta.TablesModelMetadata.prediction_type]
126    // only list of [annotation_spec_display_name-s][] is populated.
127    repeated string annotation_spec_id = 1;
128
129    // Output only. Display name of the annotation specs used in the confusion
130    // matrix, as they were at the moment of the evaluation. For Tables
131    // CLASSIFICATION
132    // [prediction_type-s][google.cloud.automl.v1p1beta.TablesModelMetadata.prediction_type],
133    // distinct values of the target column at the moment of the model
134    // evaluation are populated here.
135    repeated string display_name = 3;
136
137    // Output only. Rows in the confusion matrix. The number of rows is equal to
138    // the size of `annotation_spec_id`.
139    // `row[i].example_count[j]` is the number of examples that have ground
140    // truth of the `annotation_spec_id[i]` and are predicted as
141    // `annotation_spec_id[j]` by the model being evaluated.
142    repeated Row row = 2;
143  }
144
145  // Output only. The Area Under Precision-Recall Curve metric. Micro-averaged
146  // for the overall evaluation.
147  float au_prc = 1;
148
149  // Output only. The Area Under Receiver Operating Characteristic curve metric.
150  // Micro-averaged for the overall evaluation.
151  float au_roc = 6;
152
153  // Output only. The Log Loss metric.
154  float log_loss = 7;
155
156  // Output only. Metrics for each confidence_threshold in
157  // 0.00,0.05,0.10,...,0.95,0.96,0.97,0.98,0.99 and
158  // position_threshold = INT32_MAX_VALUE.
159  // ROC and precision-recall curves, and other aggregated metrics are derived
160  // from them. The confidence metrics entries may also be supplied for
161  // additional values of position_threshold, but from these no aggregated
162  // metrics are computed.
163  repeated ConfidenceMetricsEntry confidence_metrics_entry = 3;
164
165  // Output only. Confusion matrix of the evaluation.
166  // Only set for MULTICLASS classification problems where number
167  // of labels is no more than 10.
168  // Only set for model level evaluation, not for evaluation per label.
169  ConfusionMatrix confusion_matrix = 4;
170
171  // Output only. The annotation spec ids used for this evaluation.
172  repeated string annotation_spec_id = 5;
173}
174