1// Copyright 2020 Google LLC
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15syntax = "proto3";
16
17package google.cloud.automl.v1beta1;
18
19import "google/cloud/automl/v1beta1/geometry.proto";
20import "google/protobuf/duration.proto";
21
22option go_package = "cloud.google.com/go/automl/apiv1beta1/automlpb;automlpb";
23option java_multiple_files = true;
24option java_package = "com.google.cloud.automl.v1beta1";
25option php_namespace = "Google\\Cloud\\AutoMl\\V1beta1";
26option ruby_package = "Google::Cloud::AutoML::V1beta1";
27
28// Annotation details for image object detection.
29message ImageObjectDetectionAnnotation {
30  // Output only. The rectangle representing the object location.
31  BoundingPoly bounding_box = 1;
32
33  // Output only. The confidence that this annotation is positive for the parent example,
34  // value in [0, 1], higher means higher positivity confidence.
35  float score = 2;
36}
37
38// Annotation details for video object tracking.
39message VideoObjectTrackingAnnotation {
40  // Optional. The instance of the object, expressed as a positive integer. Used to tell
41  // apart objects of the same type (i.e. AnnotationSpec) when multiple are
42  // present on a single example.
43  // NOTE: Instance ID prediction quality is not a part of model evaluation and
44  // is done as best effort. Especially in cases when an entity goes
45  // off-screen for a longer time (minutes), when it comes back it may be given
46  // a new instance ID.
47  string instance_id = 1;
48
49  // Required. A time (frame) of a video to which this annotation pertains.
50  // Represented as the duration since the video's start.
51  google.protobuf.Duration time_offset = 2;
52
53  // Required. The rectangle representing the object location on the frame (i.e.
54  // at the time_offset of the video).
55  BoundingPoly bounding_box = 3;
56
57  // Output only. The confidence that this annotation is positive for the video at
58  // the time_offset, value in [0, 1], higher means higher positivity
59  // confidence. For annotations created by the user the score is 1. When
60  // user approves an annotation, the original float score is kept (and not
61  // changed to 1).
62  float score = 4;
63}
64
65// Bounding box matching model metrics for a single intersection-over-union
66// threshold and multiple label match confidence thresholds.
67message BoundingBoxMetricsEntry {
68  // Metrics for a single confidence threshold.
69  message ConfidenceMetricsEntry {
70    // Output only. The confidence threshold value used to compute the metrics.
71    float confidence_threshold = 1;
72
73    // Output only. Recall under the given confidence threshold.
74    float recall = 2;
75
76    // Output only. Precision under the given confidence threshold.
77    float precision = 3;
78
79    // Output only. The harmonic mean of recall and precision.
80    float f1_score = 4;
81  }
82
83  // Output only. The intersection-over-union threshold value used to compute
84  // this metrics entry.
85  float iou_threshold = 1;
86
87  // Output only. The mean average precision, most often close to au_prc.
88  float mean_average_precision = 2;
89
90  // Output only. Metrics for each label-match confidence_threshold from
91  // 0.05,0.10,...,0.95,0.96,0.97,0.98,0.99. Precision-recall curve is
92  // derived from them.
93  repeated ConfidenceMetricsEntry confidence_metrics_entries = 3;
94}
95
96// Model evaluation metrics for image object detection problems.
97// Evaluates prediction quality of labeled bounding boxes.
98message ImageObjectDetectionEvaluationMetrics {
99  // Output only. The total number of bounding boxes (i.e. summed over all
100  // images) the ground truth used to create this evaluation had.
101  int32 evaluated_bounding_box_count = 1;
102
103  // Output only. The bounding boxes match metrics for each
104  // Intersection-over-union threshold 0.05,0.10,...,0.95,0.96,0.97,0.98,0.99
105  // and each label confidence threshold 0.05,0.10,...,0.95,0.96,0.97,0.98,0.99
106  // pair.
107  repeated BoundingBoxMetricsEntry bounding_box_metrics_entries = 2;
108
109  // Output only. The single metric for bounding boxes evaluation:
110  // the mean_average_precision averaged over all bounding_box_metrics_entries.
111  float bounding_box_mean_average_precision = 3;
112}
113
114// Model evaluation metrics for video object tracking problems.
115// Evaluates prediction quality of both labeled bounding boxes and labeled
116// tracks (i.e. series of bounding boxes sharing same label and instance ID).
117message VideoObjectTrackingEvaluationMetrics {
118  // Output only. The number of video frames used to create this evaluation.
119  int32 evaluated_frame_count = 1;
120
121  // Output only. The total number of bounding boxes (i.e. summed over all
122  // frames) the ground truth used to create this evaluation had.
123  int32 evaluated_bounding_box_count = 2;
124
125  // Output only. The bounding boxes match metrics for each
126  // Intersection-over-union threshold 0.05,0.10,...,0.95,0.96,0.97,0.98,0.99
127  // and each label confidence threshold 0.05,0.10,...,0.95,0.96,0.97,0.98,0.99
128  // pair.
129  repeated BoundingBoxMetricsEntry bounding_box_metrics_entries = 4;
130
131  // Output only. The single metric for bounding boxes evaluation:
132  // the mean_average_precision averaged over all bounding_box_metrics_entries.
133  float bounding_box_mean_average_precision = 6;
134}
135