1 /*
2 * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10 #include "api/test/metrics/metrics_logger.h"
11
12 #include <map>
13 #include <memory>
14 #include <string>
15 #include <vector>
16
17 #include "absl/types/optional.h"
18 #include "api/numerics/samples_stats_counter.h"
19 #include "api/test/metrics/metric.h"
20 #include "system_wrappers/include/clock.h"
21 #include "test/gmock.h"
22 #include "test/gtest.h"
23
24 namespace webrtc {
25 namespace test {
26 namespace {
27
28 using ::testing::Eq;
29 using ::testing::IsEmpty;
30 using ::testing::SizeIs;
31
DefaultMetadata()32 std::map<std::string, std::string> DefaultMetadata() {
33 return std::map<std::string, std::string>{{"key", "value"}};
34 }
35
TEST(DefaultMetricsLoggerTest,LogSingleValueMetricRecordsMetric)36 TEST(DefaultMetricsLoggerTest, LogSingleValueMetricRecordsMetric) {
37 DefaultMetricsLogger logger(Clock::GetRealTimeClock());
38 logger.LogSingleValueMetric(
39 "metric_name", "test_case_name",
40 /*value=*/10, Unit::kMilliseconds, ImprovementDirection::kBiggerIsBetter,
41 std::map<std::string, std::string>{{"key", "value"}});
42
43 std::vector<Metric> metrics = logger.GetCollectedMetrics();
44 ASSERT_THAT(metrics, SizeIs(1));
45 const Metric& metric = metrics[0];
46 EXPECT_THAT(metric.name, Eq("metric_name"));
47 EXPECT_THAT(metric.test_case, Eq("test_case_name"));
48 EXPECT_THAT(metric.unit, Eq(Unit::kMilliseconds));
49 EXPECT_THAT(metric.improvement_direction,
50 Eq(ImprovementDirection::kBiggerIsBetter));
51 EXPECT_THAT(metric.metric_metadata,
52 Eq(std::map<std::string, std::string>{{"key", "value"}}));
53 ASSERT_THAT(metric.time_series.samples, SizeIs(1));
54 EXPECT_THAT(metric.time_series.samples[0].value, Eq(10.0));
55 EXPECT_THAT(metric.time_series.samples[0].sample_metadata,
56 Eq(std::map<std::string, std::string>{}));
57 ASSERT_THAT(metric.stats.mean, absl::optional<double>(10.0));
58 ASSERT_THAT(metric.stats.stddev, absl::nullopt);
59 ASSERT_THAT(metric.stats.min, absl::optional<double>(10.0));
60 ASSERT_THAT(metric.stats.max, absl::optional<double>(10.0));
61 }
62
TEST(DefaultMetricsLoggerTest,LogMetricWithSamplesStatsCounterRecordsMetric)63 TEST(DefaultMetricsLoggerTest, LogMetricWithSamplesStatsCounterRecordsMetric) {
64 DefaultMetricsLogger logger(Clock::GetRealTimeClock());
65
66 SamplesStatsCounter values;
67 values.AddSample(SamplesStatsCounter::StatsSample{
68 .value = 10,
69 .time = Clock::GetRealTimeClock()->CurrentTime(),
70 .metadata =
71 std::map<std::string, std::string>{{"point_key1", "value1"}}});
72 values.AddSample(SamplesStatsCounter::StatsSample{
73 .value = 20,
74 .time = Clock::GetRealTimeClock()->CurrentTime(),
75 .metadata =
76 std::map<std::string, std::string>{{"point_key2", "value2"}}});
77 logger.LogMetric("metric_name", "test_case_name", values, Unit::kMilliseconds,
78 ImprovementDirection::kBiggerIsBetter,
79 std::map<std::string, std::string>{{"key", "value"}});
80
81 std::vector<Metric> metrics = logger.GetCollectedMetrics();
82 ASSERT_THAT(metrics, SizeIs(1));
83 const Metric& metric = metrics[0];
84 EXPECT_THAT(metric.name, Eq("metric_name"));
85 EXPECT_THAT(metric.test_case, Eq("test_case_name"));
86 EXPECT_THAT(metric.unit, Eq(Unit::kMilliseconds));
87 EXPECT_THAT(metric.improvement_direction,
88 Eq(ImprovementDirection::kBiggerIsBetter));
89 EXPECT_THAT(metric.metric_metadata,
90 Eq(std::map<std::string, std::string>{{"key", "value"}}));
91 ASSERT_THAT(metric.time_series.samples, SizeIs(2));
92 EXPECT_THAT(metric.time_series.samples[0].value, Eq(10.0));
93 EXPECT_THAT(metric.time_series.samples[0].sample_metadata,
94 Eq(std::map<std::string, std::string>{{"point_key1", "value1"}}));
95 EXPECT_THAT(metric.time_series.samples[1].value, Eq(20.0));
96 EXPECT_THAT(metric.time_series.samples[1].sample_metadata,
97 Eq(std::map<std::string, std::string>{{"point_key2", "value2"}}));
98 ASSERT_THAT(metric.stats.mean, absl::optional<double>(15.0));
99 ASSERT_THAT(metric.stats.stddev, absl::optional<double>(5.0));
100 ASSERT_THAT(metric.stats.min, absl::optional<double>(10.0));
101 ASSERT_THAT(metric.stats.max, absl::optional<double>(20.0));
102 }
103
TEST(DefaultMetricsLoggerTest,LogMetricWithEmptySamplesStatsCounterRecordsEmptyMetric)104 TEST(DefaultMetricsLoggerTest,
105 LogMetricWithEmptySamplesStatsCounterRecordsEmptyMetric) {
106 DefaultMetricsLogger logger(Clock::GetRealTimeClock());
107 SamplesStatsCounter values;
108 logger.LogMetric("metric_name", "test_case_name", values, Unit::kUnitless,
109 ImprovementDirection::kBiggerIsBetter, DefaultMetadata());
110
111 std::vector<Metric> metrics = logger.GetCollectedMetrics();
112 ASSERT_THAT(metrics, SizeIs(1));
113 EXPECT_THAT(metrics[0].name, Eq("metric_name"));
114 EXPECT_THAT(metrics[0].test_case, Eq("test_case_name"));
115 EXPECT_THAT(metrics[0].time_series.samples, IsEmpty());
116 ASSERT_THAT(metrics[0].stats.mean, Eq(absl::nullopt));
117 ASSERT_THAT(metrics[0].stats.stddev, Eq(absl::nullopt));
118 ASSERT_THAT(metrics[0].stats.min, Eq(absl::nullopt));
119 ASSERT_THAT(metrics[0].stats.max, Eq(absl::nullopt));
120 }
121
TEST(DefaultMetricsLoggerTest,LogMetricWithStatsRecordsMetric)122 TEST(DefaultMetricsLoggerTest, LogMetricWithStatsRecordsMetric) {
123 DefaultMetricsLogger logger(Clock::GetRealTimeClock());
124 Metric::Stats metric_stats{.mean = 15, .stddev = 5, .min = 10, .max = 20};
125 logger.LogMetric("metric_name", "test_case_name", metric_stats,
126 Unit::kMilliseconds, ImprovementDirection::kBiggerIsBetter,
127 std::map<std::string, std::string>{{"key", "value"}});
128
129 std::vector<Metric> metrics = logger.GetCollectedMetrics();
130 ASSERT_THAT(metrics, SizeIs(1));
131 const Metric& metric = metrics[0];
132 EXPECT_THAT(metric.name, Eq("metric_name"));
133 EXPECT_THAT(metric.test_case, Eq("test_case_name"));
134 EXPECT_THAT(metric.unit, Eq(Unit::kMilliseconds));
135 EXPECT_THAT(metric.improvement_direction,
136 Eq(ImprovementDirection::kBiggerIsBetter));
137 EXPECT_THAT(metric.metric_metadata,
138 Eq(std::map<std::string, std::string>{{"key", "value"}}));
139 ASSERT_THAT(metric.time_series.samples, IsEmpty());
140 ASSERT_THAT(metric.stats.mean, absl::optional<double>(15.0));
141 ASSERT_THAT(metric.stats.stddev, absl::optional<double>(5.0));
142 ASSERT_THAT(metric.stats.min, absl::optional<double>(10.0));
143 ASSERT_THAT(metric.stats.max, absl::optional<double>(20.0));
144 }
145
TEST(DefaultMetricsLoggerTest,LogSingleValueMetricRecordsMultipleMetrics)146 TEST(DefaultMetricsLoggerTest, LogSingleValueMetricRecordsMultipleMetrics) {
147 DefaultMetricsLogger logger(Clock::GetRealTimeClock());
148
149 logger.LogSingleValueMetric("metric_name1", "test_case_name1",
150 /*value=*/10, Unit::kMilliseconds,
151 ImprovementDirection::kBiggerIsBetter,
152 DefaultMetadata());
153 logger.LogSingleValueMetric("metric_name2", "test_case_name2",
154 /*value=*/10, Unit::kMilliseconds,
155 ImprovementDirection::kBiggerIsBetter,
156 DefaultMetadata());
157
158 std::vector<Metric> metrics = logger.GetCollectedMetrics();
159 ASSERT_THAT(metrics, SizeIs(2));
160 EXPECT_THAT(metrics[0].name, Eq("metric_name1"));
161 EXPECT_THAT(metrics[0].test_case, Eq("test_case_name1"));
162 EXPECT_THAT(metrics[1].name, Eq("metric_name2"));
163 EXPECT_THAT(metrics[1].test_case, Eq("test_case_name2"));
164 }
165
TEST(DefaultMetricsLoggerTest,LogMetricWithSamplesStatsCounterRecordsMultipleMetrics)166 TEST(DefaultMetricsLoggerTest,
167 LogMetricWithSamplesStatsCounterRecordsMultipleMetrics) {
168 DefaultMetricsLogger logger(Clock::GetRealTimeClock());
169 SamplesStatsCounter values;
170 values.AddSample(SamplesStatsCounter::StatsSample{
171 .value = 10,
172 .time = Clock::GetRealTimeClock()->CurrentTime(),
173 .metadata = DefaultMetadata()});
174 values.AddSample(SamplesStatsCounter::StatsSample{
175 .value = 20,
176 .time = Clock::GetRealTimeClock()->CurrentTime(),
177 .metadata = DefaultMetadata()});
178
179 logger.LogMetric("metric_name1", "test_case_name1", values,
180 Unit::kMilliseconds, ImprovementDirection::kBiggerIsBetter,
181 DefaultMetadata());
182 logger.LogMetric("metric_name2", "test_case_name2", values,
183 Unit::kMilliseconds, ImprovementDirection::kBiggerIsBetter,
184 DefaultMetadata());
185
186 std::vector<Metric> metrics = logger.GetCollectedMetrics();
187 ASSERT_THAT(metrics, SizeIs(2));
188 EXPECT_THAT(metrics[0].name, Eq("metric_name1"));
189 EXPECT_THAT(metrics[0].test_case, Eq("test_case_name1"));
190 EXPECT_THAT(metrics[1].name, Eq("metric_name2"));
191 EXPECT_THAT(metrics[1].test_case, Eq("test_case_name2"));
192 }
193
TEST(DefaultMetricsLoggerTest,LogMetricWithStatsRecordsMultipleMetrics)194 TEST(DefaultMetricsLoggerTest, LogMetricWithStatsRecordsMultipleMetrics) {
195 DefaultMetricsLogger logger(Clock::GetRealTimeClock());
196 Metric::Stats metric_stats{.mean = 15, .stddev = 5, .min = 10, .max = 20};
197
198 logger.LogMetric("metric_name1", "test_case_name1", metric_stats,
199 Unit::kMilliseconds, ImprovementDirection::kBiggerIsBetter,
200 DefaultMetadata());
201 logger.LogMetric("metric_name2", "test_case_name2", metric_stats,
202 Unit::kMilliseconds, ImprovementDirection::kBiggerIsBetter,
203 DefaultMetadata());
204
205 std::vector<Metric> metrics = logger.GetCollectedMetrics();
206 ASSERT_THAT(metrics, SizeIs(2));
207 EXPECT_THAT(metrics[0].name, Eq("metric_name1"));
208 EXPECT_THAT(metrics[0].test_case, Eq("test_case_name1"));
209 EXPECT_THAT(metrics[1].name, Eq("metric_name2"));
210 EXPECT_THAT(metrics[1].test_case, Eq("test_case_name2"));
211 }
212
TEST(DefaultMetricsLoggerTest,LogMetricThroughtAllMethodsAccumulateAllMetrics)213 TEST(DefaultMetricsLoggerTest,
214 LogMetricThroughtAllMethodsAccumulateAllMetrics) {
215 DefaultMetricsLogger logger(Clock::GetRealTimeClock());
216 SamplesStatsCounter values;
217 values.AddSample(SamplesStatsCounter::StatsSample{
218 .value = 10,
219 .time = Clock::GetRealTimeClock()->CurrentTime(),
220 .metadata = DefaultMetadata()});
221 values.AddSample(SamplesStatsCounter::StatsSample{
222 .value = 20,
223 .time = Clock::GetRealTimeClock()->CurrentTime(),
224 .metadata = DefaultMetadata()});
225 Metric::Stats metric_stats{.mean = 15, .stddev = 5, .min = 10, .max = 20};
226
227 logger.LogSingleValueMetric("metric_name1", "test_case_name1",
228 /*value=*/10, Unit::kMilliseconds,
229 ImprovementDirection::kBiggerIsBetter,
230 DefaultMetadata());
231 logger.LogMetric("metric_name2", "test_case_name2", values,
232 Unit::kMilliseconds, ImprovementDirection::kBiggerIsBetter,
233 DefaultMetadata());
234 logger.LogMetric("metric_name3", "test_case_name3", metric_stats,
235 Unit::kMilliseconds, ImprovementDirection::kBiggerIsBetter,
236 DefaultMetadata());
237
238 std::vector<Metric> metrics = logger.GetCollectedMetrics();
239 ASSERT_THAT(metrics.size(), Eq(3lu));
240 EXPECT_THAT(metrics[0].name, Eq("metric_name1"));
241 EXPECT_THAT(metrics[0].test_case, Eq("test_case_name1"));
242 EXPECT_THAT(metrics[1].name, Eq("metric_name2"));
243 EXPECT_THAT(metrics[1].test_case, Eq("test_case_name2"));
244 EXPECT_THAT(metrics[2].name, Eq("metric_name3"));
245 EXPECT_THAT(metrics[2].test_case, Eq("test_case_name3"));
246 }
247
TEST(DefaultMetricsLoggerTest,AccumulatedMetricsReturnedInCollectedMetrics)248 TEST(DefaultMetricsLoggerTest, AccumulatedMetricsReturnedInCollectedMetrics) {
249 DefaultMetricsLogger logger(Clock::GetRealTimeClock());
250 logger.GetMetricsAccumulator()->AddSample(
251 "metric_name", "test_case_name",
252 /*value=*/10, Timestamp::Seconds(1),
253 /*point_metadata=*/std::map<std::string, std::string>{{"key", "value"}});
254
255 std::vector<Metric> metrics = logger.GetCollectedMetrics();
256 ASSERT_THAT(metrics, SizeIs(1));
257 const Metric& metric = metrics[0];
258 EXPECT_THAT(metric.name, Eq("metric_name"));
259 EXPECT_THAT(metric.test_case, Eq("test_case_name"));
260 EXPECT_THAT(metric.unit, Eq(Unit::kUnitless));
261 EXPECT_THAT(metric.improvement_direction,
262 Eq(ImprovementDirection::kNeitherIsBetter));
263 EXPECT_THAT(metric.metric_metadata, IsEmpty());
264 ASSERT_THAT(metric.time_series.samples, SizeIs(1));
265 EXPECT_THAT(metric.time_series.samples[0].value, Eq(10.0));
266 EXPECT_THAT(metric.time_series.samples[0].timestamp,
267 Eq(Timestamp::Seconds(1)));
268 EXPECT_THAT(metric.time_series.samples[0].sample_metadata,
269 Eq(std::map<std::string, std::string>{{"key", "value"}}));
270 ASSERT_THAT(metric.stats.mean, absl::optional<double>(10.0));
271 ASSERT_THAT(metric.stats.stddev, absl::optional<double>(0.0));
272 ASSERT_THAT(metric.stats.min, absl::optional<double>(10.0));
273 ASSERT_THAT(metric.stats.max, absl::optional<double>(10.0));
274 }
275
TEST(DefaultMetricsLoggerTest,AccumulatedMetricsReturnedTogetherWithLoggedMetrics)276 TEST(DefaultMetricsLoggerTest,
277 AccumulatedMetricsReturnedTogetherWithLoggedMetrics) {
278 DefaultMetricsLogger logger(Clock::GetRealTimeClock());
279 logger.LogSingleValueMetric(
280 "metric_name1", "test_case_name1",
281 /*value=*/10, Unit::kMilliseconds, ImprovementDirection::kBiggerIsBetter,
282 std::map<std::string, std::string>{{"key_m", "value_m"}});
283 logger.GetMetricsAccumulator()->AddSample(
284 "metric_name2", "test_case_name2",
285 /*value=*/10, Timestamp::Seconds(1),
286 /*point_metadata=*/
287 std::map<std::string, std::string>{{"key_s", "value_s"}});
288
289 std::vector<Metric> metrics = logger.GetCollectedMetrics();
290 ASSERT_THAT(metrics, SizeIs(2));
291 EXPECT_THAT(metrics[0].name, Eq("metric_name2"));
292 EXPECT_THAT(metrics[0].test_case, Eq("test_case_name2"));
293 EXPECT_THAT(metrics[0].unit, Eq(Unit::kUnitless));
294 EXPECT_THAT(metrics[0].improvement_direction,
295 Eq(ImprovementDirection::kNeitherIsBetter));
296 EXPECT_THAT(metrics[0].metric_metadata, IsEmpty());
297 ASSERT_THAT(metrics[0].time_series.samples, SizeIs(1));
298 EXPECT_THAT(metrics[0].time_series.samples[0].value, Eq(10.0));
299 EXPECT_THAT(metrics[0].time_series.samples[0].timestamp,
300 Eq(Timestamp::Seconds(1)));
301 EXPECT_THAT(metrics[0].time_series.samples[0].sample_metadata,
302 Eq(std::map<std::string, std::string>{{"key_s", "value_s"}}));
303 ASSERT_THAT(metrics[0].stats.mean, absl::optional<double>(10.0));
304 ASSERT_THAT(metrics[0].stats.stddev, absl::optional<double>(0.0));
305 ASSERT_THAT(metrics[0].stats.min, absl::optional<double>(10.0));
306 ASSERT_THAT(metrics[0].stats.max, absl::optional<double>(10.0));
307 EXPECT_THAT(metrics[1].name, Eq("metric_name1"));
308 EXPECT_THAT(metrics[1].test_case, Eq("test_case_name1"));
309 EXPECT_THAT(metrics[1].unit, Eq(Unit::kMilliseconds));
310 EXPECT_THAT(metrics[1].improvement_direction,
311 Eq(ImprovementDirection::kBiggerIsBetter));
312 EXPECT_THAT(metrics[1].metric_metadata,
313 Eq(std::map<std::string, std::string>{{"key_m", "value_m"}}));
314 ASSERT_THAT(metrics[1].time_series.samples, SizeIs(1));
315 EXPECT_THAT(metrics[1].time_series.samples[0].value, Eq(10.0));
316 EXPECT_THAT(metrics[1].time_series.samples[0].sample_metadata,
317 Eq(std::map<std::string, std::string>{}));
318 ASSERT_THAT(metrics[1].stats.mean, absl::optional<double>(10.0));
319 ASSERT_THAT(metrics[1].stats.stddev, absl::nullopt);
320 ASSERT_THAT(metrics[1].stats.min, absl::optional<double>(10.0));
321 ASSERT_THAT(metrics[1].stats.max, absl::optional<double>(10.0));
322 }
323
324 } // namespace
325 } // namespace test
326 } // namespace webrtc
327