1 /*
2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10 #include "rtc_tools/frame_analyzer/video_quality_analysis.h"
11
12 #include <string>
13 #include <vector>
14
15 #include "api/test/metrics/metric.h"
16 #include "api/test/metrics/metrics_logger.h"
17 #include "system_wrappers/include/clock.h"
18 #include "test/gmock.h"
19 #include "test/gtest.h"
20 #include "test/testsupport/file_utils.h"
21
22 namespace webrtc {
23 namespace test {
24 namespace {
25
26 using ::testing::IsSupersetOf;
27
28 // Metric fields to assert on
29 struct MetricValidationInfo {
30 std::string test_case;
31 std::string name;
32 Unit unit;
33 ImprovementDirection improvement_direction;
34 double mean;
35 };
36
operator ==(const MetricValidationInfo & a,const MetricValidationInfo & b)37 bool operator==(const MetricValidationInfo& a, const MetricValidationInfo& b) {
38 return a.name == b.name && a.test_case == b.test_case && a.unit == b.unit &&
39 a.improvement_direction == b.improvement_direction;
40 }
41
operator <<(std::ostream & os,const MetricValidationInfo & m)42 std::ostream& operator<<(std::ostream& os, const MetricValidationInfo& m) {
43 os << "{ test_case=" << m.test_case << "; name=" << m.name
44 << "; unit=" << test::ToString(m.unit)
45 << "; improvement_direction=" << test::ToString(m.improvement_direction)
46 << " }";
47 return os;
48 }
49
ToValidationInfo(const std::vector<Metric> & metrics)50 std::vector<MetricValidationInfo> ToValidationInfo(
51 const std::vector<Metric>& metrics) {
52 std::vector<MetricValidationInfo> out;
53 for (const Metric& m : metrics) {
54 out.push_back(
55 MetricValidationInfo{.test_case = m.test_case,
56 .name = m.name,
57 .unit = m.unit,
58 .improvement_direction = m.improvement_direction,
59 .mean = *m.stats.mean});
60 }
61 return out;
62 }
63
TEST(VideoQualityAnalysisTest,PrintAnalysisResultsEmpty)64 TEST(VideoQualityAnalysisTest, PrintAnalysisResultsEmpty) {
65 ResultsContainer result;
66 DefaultMetricsLogger logger(Clock::GetRealTimeClock());
67 PrintAnalysisResults("Empty", result, logger);
68 }
69
TEST(VideoQualityAnalysisTest,PrintAnalysisResultsOneFrame)70 TEST(VideoQualityAnalysisTest, PrintAnalysisResultsOneFrame) {
71 ResultsContainer result;
72 result.frames.push_back(AnalysisResult(0, 35.0, 0.9));
73 DefaultMetricsLogger logger(Clock::GetRealTimeClock());
74 PrintAnalysisResults("OneFrame", result, logger);
75 }
76
TEST(VideoQualityAnalysisTest,PrintAnalysisResultsThreeFrames)77 TEST(VideoQualityAnalysisTest, PrintAnalysisResultsThreeFrames) {
78 ResultsContainer result;
79 result.frames.push_back(AnalysisResult(0, 35.0, 0.9));
80 result.frames.push_back(AnalysisResult(1, 34.0, 0.8));
81 result.frames.push_back(AnalysisResult(2, 33.0, 0.7));
82 DefaultMetricsLogger logger(Clock::GetRealTimeClock());
83 PrintAnalysisResults("ThreeFrames", result, logger);
84 }
85
TEST(VideoQualityAnalysisTest,PrintMaxRepeatedAndSkippedFramesSkippedFrames)86 TEST(VideoQualityAnalysisTest, PrintMaxRepeatedAndSkippedFramesSkippedFrames) {
87 ResultsContainer result;
88
89 result.max_repeated_frames = 2;
90 result.max_skipped_frames = 2;
91 result.total_skipped_frames = 3;
92 result.decode_errors_ref = 0;
93 result.decode_errors_test = 0;
94
95 DefaultMetricsLogger logger(Clock::GetRealTimeClock());
96 PrintAnalysisResults("NormalStatsFile", result, logger);
97
98 std::vector<MetricValidationInfo> metrics =
99 ToValidationInfo(logger.GetCollectedMetrics());
100 EXPECT_THAT(
101 metrics,
102 IsSupersetOf(
103 {MetricValidationInfo{
104 .test_case = "NormalStatsFile",
105 .name = "Max_repeated",
106 .unit = Unit::kUnitless,
107 .improvement_direction = ImprovementDirection::kNeitherIsBetter,
108 .mean = 2},
109 MetricValidationInfo{
110 .test_case = "NormalStatsFile",
111 .name = "Max_skipped",
112 .unit = Unit::kUnitless,
113 .improvement_direction = ImprovementDirection::kNeitherIsBetter,
114 .mean = 2},
115 MetricValidationInfo{
116 .test_case = "NormalStatsFile",
117 .name = "Total_skipped",
118 .unit = Unit::kUnitless,
119 .improvement_direction = ImprovementDirection::kNeitherIsBetter,
120 .mean = 3},
121 MetricValidationInfo{
122 .test_case = "NormalStatsFile",
123 .name = "Decode_errors_reference",
124 .unit = Unit::kUnitless,
125 .improvement_direction = ImprovementDirection::kNeitherIsBetter,
126 .mean = 0},
127 MetricValidationInfo{
128 .test_case = "NormalStatsFile",
129 .name = "Decode_errors_test",
130 .unit = Unit::kUnitless,
131 .improvement_direction = ImprovementDirection::kNeitherIsBetter,
132 .mean = 0}}));
133 }
134
TEST(VideoQualityAnalysisTest,PrintMaxRepeatedAndSkippedFramesDecodeErrorInTest)135 TEST(VideoQualityAnalysisTest,
136 PrintMaxRepeatedAndSkippedFramesDecodeErrorInTest) {
137 ResultsContainer result;
138
139 std::string log_filename =
140 TempFilename(webrtc::test::OutputPath(), "log.log");
141 FILE* logfile = fopen(log_filename.c_str(), "w");
142 ASSERT_TRUE(logfile != NULL);
143
144 result.max_repeated_frames = 1;
145 result.max_skipped_frames = 0;
146 result.total_skipped_frames = 0;
147 result.decode_errors_ref = 0;
148 result.decode_errors_test = 3;
149
150 DefaultMetricsLogger logger(Clock::GetRealTimeClock());
151 PrintAnalysisResults("NormalStatsFile", result, logger);
152
153 std::vector<MetricValidationInfo> metrics =
154 ToValidationInfo(logger.GetCollectedMetrics());
155 EXPECT_THAT(
156 metrics,
157 IsSupersetOf(
158 {MetricValidationInfo{
159 .test_case = "NormalStatsFile",
160 .name = "Max_repeated",
161 .unit = Unit::kUnitless,
162 .improvement_direction = ImprovementDirection::kNeitherIsBetter,
163 .mean = 1},
164 MetricValidationInfo{
165 .test_case = "NormalStatsFile",
166 .name = "Max_skipped",
167 .unit = Unit::kUnitless,
168 .improvement_direction = ImprovementDirection::kNeitherIsBetter,
169 .mean = 0},
170 MetricValidationInfo{
171 .test_case = "NormalStatsFile",
172 .name = "Total_skipped",
173 .unit = Unit::kUnitless,
174 .improvement_direction = ImprovementDirection::kNeitherIsBetter,
175 .mean = 0},
176 MetricValidationInfo{
177 .test_case = "NormalStatsFile",
178 .name = "Decode_errors_reference",
179 .unit = Unit::kUnitless,
180 .improvement_direction = ImprovementDirection::kNeitherIsBetter,
181 .mean = 0},
182 MetricValidationInfo{
183 .test_case = "NormalStatsFile",
184 .name = "Decode_errors_test",
185 .unit = Unit::kUnitless,
186 .improvement_direction = ImprovementDirection::kNeitherIsBetter,
187 .mean = 3}}));
188 }
189
TEST(VideoQualityAnalysisTest,CalculateFrameClustersOneValue)190 TEST(VideoQualityAnalysisTest, CalculateFrameClustersOneValue) {
191 const std::vector<Cluster> result = CalculateFrameClusters({1});
192 EXPECT_EQ(1u, result.size());
193 EXPECT_EQ(1u, result[0].index);
194 EXPECT_EQ(1, result[0].number_of_repeated_frames);
195 }
196
TEST(VideoQualityAnalysisTest,GetMaxRepeatedFramesOneValue)197 TEST(VideoQualityAnalysisTest, GetMaxRepeatedFramesOneValue) {
198 EXPECT_EQ(1, GetMaxRepeatedFrames(CalculateFrameClusters({1})));
199 }
200
TEST(VideoQualityAnalysisTest,GetMaxSkippedFramesOneValue)201 TEST(VideoQualityAnalysisTest, GetMaxSkippedFramesOneValue) {
202 EXPECT_EQ(0, GetMaxSkippedFrames(CalculateFrameClusters({1})));
203 }
204
TEST(VideoQualityAnalysisTest,GetTotalNumberOfSkippedFramesOneValue)205 TEST(VideoQualityAnalysisTest, GetTotalNumberOfSkippedFramesOneValue) {
206 EXPECT_EQ(0, GetTotalNumberOfSkippedFrames(CalculateFrameClusters({1})));
207 }
208
TEST(VideoQualityAnalysisTest,CalculateFrameClustersOneOneTwo)209 TEST(VideoQualityAnalysisTest, CalculateFrameClustersOneOneTwo) {
210 const std::vector<Cluster> result = CalculateFrameClusters({1, 1, 2});
211 EXPECT_EQ(2u, result.size());
212 EXPECT_EQ(1u, result[0].index);
213 EXPECT_EQ(2, result[0].number_of_repeated_frames);
214 EXPECT_EQ(2u, result[1].index);
215 EXPECT_EQ(1, result[1].number_of_repeated_frames);
216 }
217
TEST(VideoQualityAnalysisTest,GetMaxRepeatedFramesOneOneTwo)218 TEST(VideoQualityAnalysisTest, GetMaxRepeatedFramesOneOneTwo) {
219 EXPECT_EQ(2, GetMaxRepeatedFrames(CalculateFrameClusters({1, 1, 2})));
220 }
221
TEST(VideoQualityAnalysisTest,GetMaxSkippedFramesOneOneTwo)222 TEST(VideoQualityAnalysisTest, GetMaxSkippedFramesOneOneTwo) {
223 EXPECT_EQ(0, GetMaxSkippedFrames(CalculateFrameClusters({1, 1, 2})));
224 }
225
TEST(VideoQualityAnalysisTest,GetTotalNumberOfSkippedFramesOneOneTwo)226 TEST(VideoQualityAnalysisTest, GetTotalNumberOfSkippedFramesOneOneTwo) {
227 EXPECT_EQ(0,
228 GetTotalNumberOfSkippedFrames(CalculateFrameClusters({1, 1, 2})));
229 }
230
TEST(VideoQualityAnalysisTest,CalculateFrameClustersEmpty)231 TEST(VideoQualityAnalysisTest, CalculateFrameClustersEmpty) {
232 EXPECT_TRUE(CalculateFrameClusters({}).empty());
233 }
234
TEST(VideoQualityAnalysisTest,GetMaxRepeatedFramesEmpty)235 TEST(VideoQualityAnalysisTest, GetMaxRepeatedFramesEmpty) {
236 EXPECT_EQ(0, GetMaxRepeatedFrames({}));
237 }
238
TEST(VideoQualityAnalysisTest,GetMaxSkippedFramesEmpty)239 TEST(VideoQualityAnalysisTest, GetMaxSkippedFramesEmpty) {
240 EXPECT_EQ(0, GetMaxSkippedFrames({}));
241 }
242
TEST(VideoQualityAnalysisTest,GetTotalNumberOfSkippedFramesEmpty)243 TEST(VideoQualityAnalysisTest, GetTotalNumberOfSkippedFramesEmpty) {
244 EXPECT_EQ(0, GetTotalNumberOfSkippedFrames({}));
245 }
246
247 } // namespace
248 } // namespace test
249 } // namespace webrtc
250