1 /*
2 * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10 #include "api/test/metrics/metrics_accumulator.h"
11
12 #include <map>
13 #include <vector>
14
15 #include "api/test/metrics/metric.h"
16 #include "api/units/timestamp.h"
17 #include "test/gmock.h"
18 #include "test/gtest.h"
19
20 namespace webrtc {
21 namespace test {
22 namespace {
23
24 using ::testing::Eq;
25 using ::testing::IsEmpty;
26 using ::testing::SizeIs;
27
TEST(MetricsAccumulatorTest,AddSampleToTheNewMetricWillCreateOne)28 TEST(MetricsAccumulatorTest, AddSampleToTheNewMetricWillCreateOne) {
29 MetricsAccumulator accumulator;
30 ASSERT_TRUE(accumulator.AddSample(
31 "metric_name", "test_case_name",
32 /*value=*/10, Timestamp::Seconds(1),
33 /*point_metadata=*/std::map<std::string, std::string>{{"key", "value"}}));
34
35 std::vector<Metric> metrics = accumulator.GetCollectedMetrics();
36 ASSERT_THAT(metrics, SizeIs(1));
37 const Metric& metric = metrics[0];
38 EXPECT_THAT(metric.name, Eq("metric_name"));
39 EXPECT_THAT(metric.test_case, Eq("test_case_name"));
40 EXPECT_THAT(metric.unit, Eq(Unit::kUnitless));
41 EXPECT_THAT(metric.improvement_direction,
42 Eq(ImprovementDirection::kNeitherIsBetter));
43 EXPECT_THAT(metric.metric_metadata, IsEmpty());
44 ASSERT_THAT(metric.time_series.samples, SizeIs(1));
45 EXPECT_THAT(metric.time_series.samples[0].value, Eq(10.0));
46 EXPECT_THAT(metric.time_series.samples[0].timestamp,
47 Eq(Timestamp::Seconds(1)));
48 EXPECT_THAT(metric.time_series.samples[0].sample_metadata,
49 Eq(std::map<std::string, std::string>{{"key", "value"}}));
50 ASSERT_THAT(metric.stats.mean, absl::optional<double>(10.0));
51 ASSERT_THAT(metric.stats.stddev, absl::optional<double>(0.0));
52 ASSERT_THAT(metric.stats.min, absl::optional<double>(10.0));
53 ASSERT_THAT(metric.stats.max, absl::optional<double>(10.0));
54 }
55
TEST(MetricsAccumulatorTest,AddSamplesToExistingMetricWontCreateNewOne)56 TEST(MetricsAccumulatorTest, AddSamplesToExistingMetricWontCreateNewOne) {
57 MetricsAccumulator accumulator;
58 ASSERT_TRUE(accumulator.AddSample(
59 "metric_name", "test_case_name",
60 /*value=*/10, Timestamp::Seconds(1),
61 /*point_metadata=*/
62 std::map<std::string, std::string>{{"key1", "value1"}}));
63 ASSERT_FALSE(accumulator.AddSample(
64 "metric_name", "test_case_name",
65 /*value=*/20, Timestamp::Seconds(2),
66 /*point_metadata=*/
67 std::map<std::string, std::string>{{"key2", "value2"}}));
68
69 std::vector<Metric> metrics = accumulator.GetCollectedMetrics();
70 ASSERT_THAT(metrics, SizeIs(1));
71 const Metric& metric = metrics[0];
72 EXPECT_THAT(metric.name, Eq("metric_name"));
73 EXPECT_THAT(metric.test_case, Eq("test_case_name"));
74 EXPECT_THAT(metric.unit, Eq(Unit::kUnitless));
75 EXPECT_THAT(metric.improvement_direction,
76 Eq(ImprovementDirection::kNeitherIsBetter));
77 EXPECT_THAT(metric.metric_metadata, IsEmpty());
78 ASSERT_THAT(metric.time_series.samples, SizeIs(2));
79 EXPECT_THAT(metric.time_series.samples[0].value, Eq(10.0));
80 EXPECT_THAT(metric.time_series.samples[0].timestamp,
81 Eq(Timestamp::Seconds(1)));
82 EXPECT_THAT(metric.time_series.samples[0].sample_metadata,
83 Eq(std::map<std::string, std::string>{{"key1", "value1"}}));
84 EXPECT_THAT(metric.time_series.samples[1].value, Eq(20.0));
85 EXPECT_THAT(metric.time_series.samples[1].timestamp,
86 Eq(Timestamp::Seconds(2)));
87 EXPECT_THAT(metric.time_series.samples[1].sample_metadata,
88 Eq(std::map<std::string, std::string>{{"key2", "value2"}}));
89 ASSERT_THAT(metric.stats.mean, absl::optional<double>(15.0));
90 ASSERT_THAT(metric.stats.stddev, absl::optional<double>(5.0));
91 ASSERT_THAT(metric.stats.min, absl::optional<double>(10.0));
92 ASSERT_THAT(metric.stats.max, absl::optional<double>(20.0));
93 }
94
TEST(MetricsAccumulatorTest,AddSampleToDifferentMetricsWillCreateBoth)95 TEST(MetricsAccumulatorTest, AddSampleToDifferentMetricsWillCreateBoth) {
96 MetricsAccumulator accumulator;
97 ASSERT_TRUE(accumulator.AddSample(
98 "metric_name1", "test_case_name1",
99 /*value=*/10, Timestamp::Seconds(1),
100 /*point_metadata=*/
101 std::map<std::string, std::string>{{"key1", "value1"}}));
102 ASSERT_TRUE(accumulator.AddSample(
103 "metric_name2", "test_case_name2",
104 /*value=*/20, Timestamp::Seconds(2),
105 /*point_metadata=*/
106 std::map<std::string, std::string>{{"key2", "value2"}}));
107
108 std::vector<Metric> metrics = accumulator.GetCollectedMetrics();
109 ASSERT_THAT(metrics, SizeIs(2));
110 EXPECT_THAT(metrics[0].name, Eq("metric_name1"));
111 EXPECT_THAT(metrics[0].test_case, Eq("test_case_name1"));
112 EXPECT_THAT(metrics[0].unit, Eq(Unit::kUnitless));
113 EXPECT_THAT(metrics[0].improvement_direction,
114 Eq(ImprovementDirection::kNeitherIsBetter));
115 EXPECT_THAT(metrics[0].metric_metadata, IsEmpty());
116 ASSERT_THAT(metrics[0].time_series.samples, SizeIs(1));
117 EXPECT_THAT(metrics[0].time_series.samples[0].value, Eq(10.0));
118 EXPECT_THAT(metrics[0].time_series.samples[0].timestamp,
119 Eq(Timestamp::Seconds(1)));
120 EXPECT_THAT(metrics[0].time_series.samples[0].sample_metadata,
121 Eq(std::map<std::string, std::string>{{"key1", "value1"}}));
122 ASSERT_THAT(metrics[0].stats.mean, absl::optional<double>(10.0));
123 ASSERT_THAT(metrics[0].stats.stddev, absl::optional<double>(0.0));
124 ASSERT_THAT(metrics[0].stats.min, absl::optional<double>(10.0));
125 ASSERT_THAT(metrics[0].stats.max, absl::optional<double>(10.0));
126 EXPECT_THAT(metrics[1].name, Eq("metric_name2"));
127 EXPECT_THAT(metrics[1].test_case, Eq("test_case_name2"));
128 EXPECT_THAT(metrics[1].unit, Eq(Unit::kUnitless));
129 EXPECT_THAT(metrics[1].improvement_direction,
130 Eq(ImprovementDirection::kNeitherIsBetter));
131 EXPECT_THAT(metrics[1].metric_metadata, IsEmpty());
132 ASSERT_THAT(metrics[1].time_series.samples, SizeIs(1));
133 EXPECT_THAT(metrics[1].time_series.samples[0].value, Eq(20.0));
134 EXPECT_THAT(metrics[1].time_series.samples[0].timestamp,
135 Eq(Timestamp::Seconds(2)));
136 EXPECT_THAT(metrics[1].time_series.samples[0].sample_metadata,
137 Eq(std::map<std::string, std::string>{{"key2", "value2"}}));
138 ASSERT_THAT(metrics[1].stats.mean, absl::optional<double>(20.0));
139 ASSERT_THAT(metrics[1].stats.stddev, absl::optional<double>(0.0));
140 ASSERT_THAT(metrics[1].stats.min, absl::optional<double>(20.0));
141 ASSERT_THAT(metrics[1].stats.max, absl::optional<double>(20.0));
142 }
143
TEST(MetricsAccumulatorTest,AddMetadataToTheNewMetricWillCreateOne)144 TEST(MetricsAccumulatorTest, AddMetadataToTheNewMetricWillCreateOne) {
145 MetricsAccumulator accumulator;
146 ASSERT_TRUE(accumulator.AddMetricMetadata(
147 "metric_name", "test_case_name", Unit::kMilliseconds,
148 ImprovementDirection::kBiggerIsBetter,
149 /*metric_metadata=*/
150 std::map<std::string, std::string>{{"key", "value"}}));
151
152 std::vector<Metric> metrics = accumulator.GetCollectedMetrics();
153 ASSERT_THAT(metrics, SizeIs(1));
154 const Metric& metric = metrics[0];
155 EXPECT_THAT(metric.name, Eq("metric_name"));
156 EXPECT_THAT(metric.test_case, Eq("test_case_name"));
157 EXPECT_THAT(metric.unit, Eq(Unit::kMilliseconds));
158 EXPECT_THAT(metric.improvement_direction,
159 Eq(ImprovementDirection::kBiggerIsBetter));
160 EXPECT_THAT(metric.metric_metadata,
161 Eq(std::map<std::string, std::string>{{"key", "value"}}));
162 ASSERT_THAT(metric.time_series.samples, IsEmpty());
163 ASSERT_THAT(metric.stats.mean, absl::nullopt);
164 ASSERT_THAT(metric.stats.stddev, absl::nullopt);
165 ASSERT_THAT(metric.stats.min, absl::nullopt);
166 ASSERT_THAT(metric.stats.max, absl::nullopt);
167 }
168
TEST(MetricsAccumulatorTest,AddMetadataToTheExistingMetricWillOverwriteValues)169 TEST(MetricsAccumulatorTest,
170 AddMetadataToTheExistingMetricWillOverwriteValues) {
171 MetricsAccumulator accumulator;
172 ASSERT_TRUE(accumulator.AddMetricMetadata(
173 "metric_name", "test_case_name", Unit::kMilliseconds,
174 ImprovementDirection::kBiggerIsBetter,
175 /*metric_metadata=*/
176 std::map<std::string, std::string>{{"key1", "value1"}}));
177
178 ASSERT_FALSE(accumulator.AddMetricMetadata(
179 "metric_name", "test_case_name", Unit::kBytes,
180 ImprovementDirection::kSmallerIsBetter,
181 /*metric_metadata=*/
182 std::map<std::string, std::string>{{"key2", "value2"}}));
183
184 std::vector<Metric> metrics = accumulator.GetCollectedMetrics();
185 ASSERT_THAT(metrics, SizeIs(1));
186 const Metric& metric = metrics[0];
187 EXPECT_THAT(metric.name, Eq("metric_name"));
188 EXPECT_THAT(metric.test_case, Eq("test_case_name"));
189 EXPECT_THAT(metric.unit, Eq(Unit::kBytes));
190 EXPECT_THAT(metric.improvement_direction,
191 Eq(ImprovementDirection::kSmallerIsBetter));
192 EXPECT_THAT(metric.metric_metadata,
193 Eq(std::map<std::string, std::string>{{"key2", "value2"}}));
194 ASSERT_THAT(metric.time_series.samples, IsEmpty());
195 ASSERT_THAT(metric.stats.mean, absl::nullopt);
196 ASSERT_THAT(metric.stats.stddev, absl::nullopt);
197 ASSERT_THAT(metric.stats.min, absl::nullopt);
198 ASSERT_THAT(metric.stats.max, absl::nullopt);
199 }
200
TEST(MetricsAccumulatorTest,AddMetadataToDifferentMetricsWillCreateBoth)201 TEST(MetricsAccumulatorTest, AddMetadataToDifferentMetricsWillCreateBoth) {
202 MetricsAccumulator accumulator;
203 ASSERT_TRUE(accumulator.AddMetricMetadata(
204 "metric_name1", "test_case_name1", Unit::kMilliseconds,
205 ImprovementDirection::kBiggerIsBetter,
206 /*metric_metadata=*/
207 std::map<std::string, std::string>{{"key1", "value1"}}));
208
209 ASSERT_TRUE(accumulator.AddMetricMetadata(
210 "metric_name2", "test_case_name2", Unit::kBytes,
211 ImprovementDirection::kSmallerIsBetter,
212 /*metric_metadata=*/
213 std::map<std::string, std::string>{{"key2", "value2"}}));
214
215 std::vector<Metric> metrics = accumulator.GetCollectedMetrics();
216 ASSERT_THAT(metrics, SizeIs(2));
217 EXPECT_THAT(metrics[0].name, Eq("metric_name1"));
218 EXPECT_THAT(metrics[0].test_case, Eq("test_case_name1"));
219 EXPECT_THAT(metrics[0].unit, Eq(Unit::kMilliseconds));
220 EXPECT_THAT(metrics[0].improvement_direction,
221 Eq(ImprovementDirection::kBiggerIsBetter));
222 EXPECT_THAT(metrics[0].metric_metadata,
223 Eq(std::map<std::string, std::string>{{"key1", "value1"}}));
224 ASSERT_THAT(metrics[0].time_series.samples, IsEmpty());
225 ASSERT_THAT(metrics[0].stats.mean, absl::nullopt);
226 ASSERT_THAT(metrics[0].stats.stddev, absl::nullopt);
227 ASSERT_THAT(metrics[0].stats.min, absl::nullopt);
228 ASSERT_THAT(metrics[0].stats.max, absl::nullopt);
229 EXPECT_THAT(metrics[1].name, Eq("metric_name2"));
230 EXPECT_THAT(metrics[1].test_case, Eq("test_case_name2"));
231 EXPECT_THAT(metrics[1].unit, Eq(Unit::kBytes));
232 EXPECT_THAT(metrics[1].improvement_direction,
233 Eq(ImprovementDirection::kSmallerIsBetter));
234 EXPECT_THAT(metrics[1].metric_metadata,
235 Eq(std::map<std::string, std::string>{{"key2", "value2"}}));
236 ASSERT_THAT(metrics[1].time_series.samples, IsEmpty());
237 ASSERT_THAT(metrics[1].stats.mean, absl::nullopt);
238 ASSERT_THAT(metrics[1].stats.stddev, absl::nullopt);
239 ASSERT_THAT(metrics[1].stats.min, absl::nullopt);
240 ASSERT_THAT(metrics[1].stats.max, absl::nullopt);
241 }
242
TEST(MetricsAccumulatorTest,AddMetadataAfterAddingSampleWontCreateNewMetric)243 TEST(MetricsAccumulatorTest, AddMetadataAfterAddingSampleWontCreateNewMetric) {
244 MetricsAccumulator accumulator;
245 ASSERT_TRUE(accumulator.AddSample(
246 "metric_name", "test_case_name",
247 /*value=*/10, Timestamp::Seconds(1),
248 /*point_metadata=*/
249 std::map<std::string, std::string>{{"key_s", "value_s"}}));
250 ASSERT_FALSE(accumulator.AddMetricMetadata(
251 "metric_name", "test_case_name", Unit::kMilliseconds,
252 ImprovementDirection::kBiggerIsBetter,
253 /*metric_metadata=*/
254 std::map<std::string, std::string>{{"key_m", "value_m"}}));
255
256 std::vector<Metric> metrics = accumulator.GetCollectedMetrics();
257 ASSERT_THAT(metrics, SizeIs(1));
258 const Metric& metric = metrics[0];
259 EXPECT_THAT(metric.name, Eq("metric_name"));
260 EXPECT_THAT(metric.test_case, Eq("test_case_name"));
261 EXPECT_THAT(metric.unit, Eq(Unit::kMilliseconds));
262 EXPECT_THAT(metric.improvement_direction,
263 Eq(ImprovementDirection::kBiggerIsBetter));
264 EXPECT_THAT(metric.metric_metadata,
265 Eq(std::map<std::string, std::string>{{"key_m", "value_m"}}));
266 ASSERT_THAT(metric.time_series.samples, SizeIs(1));
267 EXPECT_THAT(metric.time_series.samples[0].value, Eq(10.0));
268 EXPECT_THAT(metric.time_series.samples[0].timestamp,
269 Eq(Timestamp::Seconds(1)));
270 EXPECT_THAT(metric.time_series.samples[0].sample_metadata,
271 Eq(std::map<std::string, std::string>{{"key_s", "value_s"}}));
272 ASSERT_THAT(metric.stats.mean, absl::optional<double>(10.0));
273 ASSERT_THAT(metric.stats.stddev, absl::optional<double>(0.0));
274 ASSERT_THAT(metric.stats.min, absl::optional<double>(10.0));
275 ASSERT_THAT(metric.stats.max, absl::optional<double>(10.0));
276 }
277
TEST(MetricsAccumulatorTest,AddSampleAfterAddingMetadataWontCreateNewMetric)278 TEST(MetricsAccumulatorTest, AddSampleAfterAddingMetadataWontCreateNewMetric) {
279 MetricsAccumulator accumulator;
280 ASSERT_TRUE(accumulator.AddMetricMetadata(
281 "metric_name", "test_case_name", Unit::kMilliseconds,
282 ImprovementDirection::kBiggerIsBetter,
283 /*metric_metadata=*/
284 std::map<std::string, std::string>{{"key_m", "value_m"}}));
285 ASSERT_FALSE(accumulator.AddSample(
286 "metric_name", "test_case_name",
287 /*value=*/10, Timestamp::Seconds(1),
288 /*point_metadata=*/
289 std::map<std::string, std::string>{{"key_s", "value_s"}}));
290
291 std::vector<Metric> metrics = accumulator.GetCollectedMetrics();
292 ASSERT_THAT(metrics, SizeIs(1));
293 const Metric& metric = metrics[0];
294 EXPECT_THAT(metric.name, Eq("metric_name"));
295 EXPECT_THAT(metric.test_case, Eq("test_case_name"));
296 EXPECT_THAT(metric.unit, Eq(Unit::kMilliseconds));
297 EXPECT_THAT(metric.improvement_direction,
298 Eq(ImprovementDirection::kBiggerIsBetter));
299 EXPECT_THAT(metric.metric_metadata,
300 Eq(std::map<std::string, std::string>{{"key_m", "value_m"}}));
301 ASSERT_THAT(metric.time_series.samples, SizeIs(1));
302 EXPECT_THAT(metric.time_series.samples[0].value, Eq(10.0));
303 EXPECT_THAT(metric.time_series.samples[0].timestamp,
304 Eq(Timestamp::Seconds(1)));
305 EXPECT_THAT(metric.time_series.samples[0].sample_metadata,
306 Eq(std::map<std::string, std::string>{{"key_s", "value_s"}}));
307 ASSERT_THAT(metric.stats.mean, absl::optional<double>(10.0));
308 ASSERT_THAT(metric.stats.stddev, absl::optional<double>(0.0));
309 ASSERT_THAT(metric.stats.min, absl::optional<double>(10.0));
310 ASSERT_THAT(metric.stats.max, absl::optional<double>(10.0));
311 }
312
313 } // namespace
314 } // namespace test
315 } // namespace webrtc
316