1 /*
2 * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.h"
12
13 #include <map>
14 #include <string>
15 #include <vector>
16
17 #include "api/test/create_frame_generator.h"
18 #include "api/units/timestamp.h"
19 #include "rtc_base/strings/string_builder.h"
20 #include "system_wrappers/include/clock.h"
21 #include "test/gmock.h"
22 #include "test/gtest.h"
23 #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_cpu_measurer.h"
24 #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h"
25
26 namespace webrtc {
27 namespace {
28
29 using ::testing::Contains;
30 using ::testing::Eq;
31 using ::testing::IsEmpty;
32 using ::testing::Pair;
33
34 using StatsSample = ::webrtc::SamplesStatsCounter::StatsSample;
35
36 constexpr int kMaxFramesInFlightPerStream = 10;
37
AnalyzerOptionsForTest()38 DefaultVideoQualityAnalyzerOptions AnalyzerOptionsForTest() {
39 DefaultVideoQualityAnalyzerOptions options;
40 options.compute_psnr = false;
41 options.compute_ssim = false;
42 options.adjust_cropping_before_comparing_frames = false;
43 options.max_frames_in_flight_per_stream_count = kMaxFramesInFlightPerStream;
44 return options;
45 }
46
CreateFrame(uint16_t frame_id,int width,int height,Timestamp timestamp)47 VideoFrame CreateFrame(uint16_t frame_id,
48 int width,
49 int height,
50 Timestamp timestamp) {
51 std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
52 test::CreateSquareFrameGenerator(width, height,
53 /*type=*/absl::nullopt,
54 /*num_squares=*/absl::nullopt);
55 test::FrameGeneratorInterface::VideoFrameData frame_data =
56 frame_generator->NextFrame();
57 return VideoFrame::Builder()
58 .set_id(frame_id)
59 .set_video_frame_buffer(frame_data.buffer)
60 .set_update_rect(frame_data.update_rect)
61 .set_timestamp_us(timestamp.us())
62 .build();
63 }
64
Vp8CodecForOneFrame(uint16_t frame_id,Timestamp time)65 StreamCodecInfo Vp8CodecForOneFrame(uint16_t frame_id, Timestamp time) {
66 StreamCodecInfo info;
67 info.codec_name = "VP8";
68 info.first_frame_id = frame_id;
69 info.last_frame_id = frame_id;
70 info.switched_on_at = time;
71 info.switched_from_at = time;
72 return info;
73 }
74
FrameStatsWith10msDeltaBetweenPhasesAnd10x10Frame(uint16_t frame_id,Timestamp captured_time)75 FrameStats FrameStatsWith10msDeltaBetweenPhasesAnd10x10Frame(
76 uint16_t frame_id,
77 Timestamp captured_time) {
78 FrameStats frame_stats(frame_id, captured_time);
79 frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10);
80 frame_stats.encoded_time = captured_time + TimeDelta::Millis(20);
81 frame_stats.received_time = captured_time + TimeDelta::Millis(30);
82 frame_stats.decode_start_time = captured_time + TimeDelta::Millis(40);
83 // Decode time is in microseconds.
84 frame_stats.decode_end_time = captured_time + TimeDelta::Micros(40010);
85 frame_stats.rendered_time = captured_time + TimeDelta::Millis(60);
86 frame_stats.used_encoder = Vp8CodecForOneFrame(1, frame_stats.encoded_time);
87 frame_stats.used_decoder =
88 Vp8CodecForOneFrame(1, frame_stats.decode_end_time);
89 frame_stats.decoded_frame_width = 10;
90 frame_stats.decoded_frame_height = 10;
91 return frame_stats;
92 }
93
ShiftStatsOn(const FrameStats & stats,TimeDelta delta)94 FrameStats ShiftStatsOn(const FrameStats& stats, TimeDelta delta) {
95 FrameStats frame_stats(stats.frame_id, stats.captured_time + delta);
96 frame_stats.pre_encode_time = stats.pre_encode_time + delta;
97 frame_stats.encoded_time = stats.encoded_time + delta;
98 frame_stats.received_time = stats.received_time + delta;
99 frame_stats.decode_start_time = stats.decode_start_time + delta;
100 frame_stats.decode_end_time = stats.decode_end_time + delta;
101 frame_stats.rendered_time = stats.rendered_time + delta;
102
103 frame_stats.used_encoder = stats.used_encoder;
104 frame_stats.used_decoder = stats.used_decoder;
105 frame_stats.decoded_frame_width = stats.decoded_frame_width;
106 frame_stats.decoded_frame_height = stats.decoded_frame_height;
107
108 return frame_stats;
109 }
110
StatsCounter(const std::vector<std::pair<double,Timestamp>> & samples)111 SamplesStatsCounter StatsCounter(
112 const std::vector<std::pair<double, Timestamp>>& samples) {
113 SamplesStatsCounter counter;
114 for (const std::pair<double, Timestamp>& sample : samples) {
115 counter.AddSample(SamplesStatsCounter::StatsSample{.value = sample.first,
116 .time = sample.second});
117 }
118 return counter;
119 }
120
GetFirstOrDie(const SamplesStatsCounter & counter)121 double GetFirstOrDie(const SamplesStatsCounter& counter) {
122 EXPECT_FALSE(counter.IsEmpty()) << "Counter has to be not empty";
123 return counter.GetSamples()[0];
124 }
125
AssertFirstMetadataHasField(const SamplesStatsCounter & counter,const std::string & field_name,const std::string & field_value)126 void AssertFirstMetadataHasField(const SamplesStatsCounter& counter,
127 const std::string& field_name,
128 const std::string& field_value) {
129 EXPECT_FALSE(counter.IsEmpty()) << "Coutner has to be not empty";
130 EXPECT_THAT(counter.GetTimedSamples()[0].metadata,
131 Contains(Pair(field_name, field_value)));
132 }
133
ToString(const SamplesStatsCounter & counter)134 std::string ToString(const SamplesStatsCounter& counter) {
135 rtc::StringBuilder out;
136 for (const StatsSample& s : counter.GetTimedSamples()) {
137 out << "{ time_ms=" << s.time.ms() << "; value=" << s.value << "}, ";
138 }
139 return out.str();
140 }
141
expectEmpty(const SamplesStatsCounter & counter)142 void expectEmpty(const SamplesStatsCounter& counter) {
143 EXPECT_TRUE(counter.IsEmpty())
144 << "Expected empty SamplesStatsCounter, but got " << ToString(counter);
145 }
146
expectEmpty(const SamplesRateCounter & counter)147 void expectEmpty(const SamplesRateCounter& counter) {
148 EXPECT_TRUE(counter.IsEmpty())
149 << "Expected empty SamplesRateCounter, but got "
150 << counter.GetEventsPerSecond();
151 }
152
TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,StatsPresentedAfterAddingOneComparison)153 TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,
154 StatsPresentedAfterAddingOneComparison) {
155 DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer;
156 DefaultVideoQualityAnalyzerFramesComparator comparator(
157 Clock::GetRealTimeClock(), cpu_measurer, AnalyzerOptionsForTest());
158
159 Timestamp stream_start_time = Clock::GetRealTimeClock()->CurrentTime();
160 size_t stream = 0;
161 size_t sender = 0;
162 size_t receiver = 1;
163 size_t peers_count = 2;
164 InternalStatsKey stats_key(stream, sender, receiver);
165
166 FrameStats frame_stats = FrameStatsWith10msDeltaBetweenPhasesAnd10x10Frame(
167 /*frame_id=*/1, stream_start_time);
168
169 comparator.Start(/*max_threads_count=*/1);
170 comparator.EnsureStatsForStream(stream, sender, peers_count,
171 stream_start_time, stream_start_time);
172 comparator.AddComparison(stats_key,
173 /*captured=*/absl::nullopt,
174 /*rendered=*/absl::nullopt,
175 FrameComparisonType::kRegular, frame_stats);
176 comparator.Stop(/*last_rendered_frame_times=*/{});
177
178 std::map<InternalStatsKey, StreamStats> stats = comparator.stream_stats();
179 EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.at(stats_key).transport_time_ms), 20.0);
180 EXPECT_DOUBLE_EQ(
181 GetFirstOrDie(stats.at(stats_key).total_delay_incl_transport_ms), 60.0);
182 EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.at(stats_key).encode_time_ms), 10.0);
183 EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.at(stats_key).decode_time_ms), 0.01);
184 EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.at(stats_key).receive_to_render_time_ms),
185 30.0);
186 EXPECT_DOUBLE_EQ(
187 GetFirstOrDie(stats.at(stats_key).resolution_of_decoded_frame), 100.0);
188 }
189
TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,MultiFrameStatsPresentedWithMetadataAfterAddingTwoComparisonWith10msDelay)190 TEST(
191 DefaultVideoQualityAnalyzerFramesComparatorTest,
192 MultiFrameStatsPresentedWithMetadataAfterAddingTwoComparisonWith10msDelay) {
193 DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer;
194 DefaultVideoQualityAnalyzerFramesComparator comparator(
195 Clock::GetRealTimeClock(), cpu_measurer, AnalyzerOptionsForTest());
196
197 Timestamp stream_start_time = Clock::GetRealTimeClock()->CurrentTime();
198 size_t stream = 0;
199 size_t sender = 0;
200 size_t receiver = 1;
201 size_t peers_count = 2;
202 InternalStatsKey stats_key(stream, sender, receiver);
203
204 FrameStats frame_stats1 = FrameStatsWith10msDeltaBetweenPhasesAnd10x10Frame(
205 /*frame_id=*/1, stream_start_time);
206 FrameStats frame_stats2 = FrameStatsWith10msDeltaBetweenPhasesAnd10x10Frame(
207 /*frame_id=*/2, stream_start_time + TimeDelta::Millis(15));
208 frame_stats2.prev_frame_rendered_time = frame_stats1.rendered_time;
209
210 comparator.Start(/*max_threads_count=*/1);
211 comparator.EnsureStatsForStream(stream, sender, peers_count,
212 stream_start_time, stream_start_time);
213 comparator.AddComparison(stats_key,
214 /*captured=*/absl::nullopt,
215 /*rendered=*/absl::nullopt,
216 FrameComparisonType::kRegular, frame_stats1);
217 comparator.AddComparison(stats_key,
218 /*captured=*/absl::nullopt,
219 /*rendered=*/absl::nullopt,
220 FrameComparisonType::kRegular, frame_stats2);
221 comparator.Stop(/*last_rendered_frame_times=*/{});
222
223 std::map<InternalStatsKey, StreamStats> stats = comparator.stream_stats();
224 EXPECT_DOUBLE_EQ(
225 GetFirstOrDie(stats.at(stats_key).time_between_rendered_frames_ms), 15.0);
226 AssertFirstMetadataHasField(
227 stats.at(stats_key).time_between_rendered_frames_ms, "frame_id", "2");
228 EXPECT_DOUBLE_EQ(stats.at(stats_key).encode_frame_rate.GetEventsPerSecond(),
229 2.0 / 15 * 1000)
230 << "There should be 2 events with interval of 15 ms";
231 }
232
TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,FrameInFlightStatsAreHandledCorrectly)233 TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,
234 FrameInFlightStatsAreHandledCorrectly) {
235 DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer;
236 DefaultVideoQualityAnalyzerFramesComparator comparator(
237 Clock::GetRealTimeClock(), cpu_measurer, AnalyzerOptionsForTest());
238
239 Timestamp stream_start_time = Clock::GetRealTimeClock()->CurrentTime();
240 size_t stream = 0;
241 size_t sender = 0;
242 size_t receiver = 1;
243 size_t peers_count = 2;
244 InternalStatsKey stats_key(stream, sender, receiver);
245
246 // There are 7 different timings inside frame stats: captured, pre_encode,
247 // encoded, received, decode_start, decode_end, rendered. captured is always
248 // set and received is set together with decode_start. So we create 6
249 // different frame stats with interval of 15 ms, where for each stat next
250 // timings will be set
251 // * 1st - captured
252 // * 2nd - captured, pre_encode
253 // * 3rd - captured, pre_encode, encoded
254 // * 4th - captured, pre_encode, encoded, received, decode_start
255 // * 5th - captured, pre_encode, encoded, received, decode_start, decode_end
256 // * 6th - all of them set
257 std::vector<FrameStats> stats;
258 // 1st stat
259 FrameStats frame_stats(/*frame_id=*/1, stream_start_time);
260 stats.push_back(frame_stats);
261 // 2nd stat
262 frame_stats = ShiftStatsOn(frame_stats, TimeDelta::Millis(15));
263 frame_stats.frame_id = 2;
264 frame_stats.pre_encode_time =
265 frame_stats.captured_time + TimeDelta::Millis(10);
266 stats.push_back(frame_stats);
267 // 3rd stat
268 frame_stats = ShiftStatsOn(frame_stats, TimeDelta::Millis(15));
269 frame_stats.frame_id = 3;
270 frame_stats.encoded_time = frame_stats.captured_time + TimeDelta::Millis(20);
271 frame_stats.used_encoder = Vp8CodecForOneFrame(1, frame_stats.encoded_time);
272 stats.push_back(frame_stats);
273 // 4th stat
274 frame_stats = ShiftStatsOn(frame_stats, TimeDelta::Millis(15));
275 frame_stats.frame_id = 4;
276 frame_stats.received_time = frame_stats.captured_time + TimeDelta::Millis(30);
277 frame_stats.decode_start_time =
278 frame_stats.captured_time + TimeDelta::Millis(40);
279 stats.push_back(frame_stats);
280 // 5th stat
281 frame_stats = ShiftStatsOn(frame_stats, TimeDelta::Millis(15));
282 frame_stats.frame_id = 5;
283 frame_stats.decode_end_time =
284 frame_stats.captured_time + TimeDelta::Millis(50);
285 frame_stats.used_decoder =
286 Vp8CodecForOneFrame(1, frame_stats.decode_end_time);
287 frame_stats.decoded_frame_width = 10;
288 frame_stats.decoded_frame_height = 10;
289 stats.push_back(frame_stats);
290 // 6th stat
291 frame_stats = ShiftStatsOn(frame_stats, TimeDelta::Millis(15));
292 frame_stats.frame_id = 6;
293 frame_stats.rendered_time = frame_stats.captured_time + TimeDelta::Millis(60);
294 stats.push_back(frame_stats);
295
296 comparator.Start(/*max_threads_count=*/1);
297 comparator.EnsureStatsForStream(stream, sender, peers_count,
298 stream_start_time, stream_start_time);
299 for (size_t i = 0; i < stats.size() - 1; ++i) {
300 comparator.AddComparison(stats_key,
301 /*captured=*/absl::nullopt,
302 /*rendered=*/absl::nullopt,
303 FrameComparisonType::kFrameInFlight, stats[i]);
304 }
305 comparator.AddComparison(stats_key,
306 /*captured=*/absl::nullopt,
307 /*rendered=*/absl::nullopt,
308 FrameComparisonType::kRegular,
309 stats[stats.size() - 1]);
310 comparator.Stop(/*last_rendered_frame_times=*/{});
311
312 EXPECT_EQ(comparator.stream_stats().size(), 1lu);
313 StreamStats result_stats = comparator.stream_stats().at(stats_key);
314
315 EXPECT_DOUBLE_EQ(result_stats.transport_time_ms.GetAverage(), 20.0)
316 << ToString(result_stats.transport_time_ms);
317 EXPECT_EQ(result_stats.transport_time_ms.NumSamples(), 3);
318
319 EXPECT_DOUBLE_EQ(result_stats.total_delay_incl_transport_ms.GetAverage(),
320 60.0)
321 << ToString(result_stats.total_delay_incl_transport_ms);
322 EXPECT_EQ(result_stats.total_delay_incl_transport_ms.NumSamples(), 1);
323
324 EXPECT_DOUBLE_EQ(result_stats.encode_time_ms.GetAverage(), 10)
325 << ToString(result_stats.encode_time_ms);
326 EXPECT_EQ(result_stats.encode_time_ms.NumSamples(), 4);
327
328 EXPECT_DOUBLE_EQ(result_stats.decode_time_ms.GetAverage(), 10)
329 << ToString(result_stats.decode_time_ms);
330 EXPECT_EQ(result_stats.decode_time_ms.NumSamples(), 2);
331
332 EXPECT_DOUBLE_EQ(result_stats.receive_to_render_time_ms.GetAverage(), 30)
333 << ToString(result_stats.receive_to_render_time_ms);
334 EXPECT_EQ(result_stats.receive_to_render_time_ms.NumSamples(), 1);
335
336 EXPECT_DOUBLE_EQ(result_stats.resolution_of_decoded_frame.GetAverage(), 100)
337 << ToString(result_stats.resolution_of_decoded_frame);
338 EXPECT_EQ(result_stats.resolution_of_decoded_frame.NumSamples(), 2);
339
340 EXPECT_DOUBLE_EQ(result_stats.encode_frame_rate.GetEventsPerSecond(),
341 4.0 / 45 * 1000)
342 << "There should be 4 events with interval of 15 ms";
343 }
344
345 // Tests to validate that stats for each possible input frame are computed
346 // correctly.
347 // Frame in flight start
TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,CapturedOnlyInFlightFrameAccountedInStats)348 TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,
349 CapturedOnlyInFlightFrameAccountedInStats) {
350 DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer;
351 DefaultVideoQualityAnalyzerFramesComparator comparator(
352 Clock::GetRealTimeClock(), cpu_measurer,
353 DefaultVideoQualityAnalyzerOptions());
354
355 Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime();
356 size_t stream = 0;
357 size_t sender = 0;
358 size_t receiver = 1;
359 InternalStatsKey stats_key(stream, sender, receiver);
360
361 // Frame captured
362 FrameStats frame_stats(/*frame_id=*/1, captured_time);
363
364 comparator.Start(/*max_threads_count=*/1);
365 comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2,
366 captured_time, captured_time);
367 comparator.AddComparison(stats_key,
368 /*captured=*/absl::nullopt,
369 /*rendered=*/absl::nullopt,
370 FrameComparisonType::kFrameInFlight, frame_stats);
371 comparator.Stop(/*last_rendered_frame_times=*/{});
372
373 EXPECT_EQ(comparator.stream_stats().size(), 1lu);
374 StreamStats stats = comparator.stream_stats().at(stats_key);
375 EXPECT_EQ(stats.stream_started_time, captured_time);
376 expectEmpty(stats.psnr);
377 expectEmpty(stats.ssim);
378 expectEmpty(stats.transport_time_ms);
379 expectEmpty(stats.total_delay_incl_transport_ms);
380 expectEmpty(stats.time_between_rendered_frames_ms);
381 expectEmpty(stats.encode_frame_rate);
382 expectEmpty(stats.encode_time_ms);
383 expectEmpty(stats.decode_time_ms);
384 expectEmpty(stats.receive_to_render_time_ms);
385 expectEmpty(stats.skipped_between_rendered);
386 expectEmpty(stats.freeze_time_ms);
387 expectEmpty(stats.time_between_freezes_ms);
388 expectEmpty(stats.resolution_of_decoded_frame);
389 expectEmpty(stats.target_encode_bitrate);
390 expectEmpty(stats.qp);
391 expectEmpty(stats.recv_key_frame_size_bytes);
392 expectEmpty(stats.recv_delta_frame_size_bytes);
393 EXPECT_EQ(stats.total_encoded_images_payload, 0);
394 EXPECT_EQ(stats.num_send_key_frames, 0);
395 EXPECT_EQ(stats.num_recv_key_frames, 0);
396 EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{
397 {FrameDropPhase::kBeforeEncoder, 0},
398 {FrameDropPhase::kByEncoder, 0},
399 {FrameDropPhase::kTransport, 0},
400 {FrameDropPhase::kByDecoder, 0},
401 {FrameDropPhase::kAfterDecoder, 0}}));
402 EXPECT_THAT(stats.encoders, IsEmpty());
403 EXPECT_THAT(stats.decoders, IsEmpty());
404 }
405
TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,PreEncodedInFlightFrameAccountedInStats)406 TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,
407 PreEncodedInFlightFrameAccountedInStats) {
408 DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer;
409 DefaultVideoQualityAnalyzerFramesComparator comparator(
410 Clock::GetRealTimeClock(), cpu_measurer,
411 DefaultVideoQualityAnalyzerOptions());
412
413 Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime();
414 size_t stream = 0;
415 size_t sender = 0;
416 size_t receiver = 1;
417 InternalStatsKey stats_key(stream, sender, receiver);
418
419 // Frame captured
420 FrameStats frame_stats(/*frame_id=*/1, captured_time);
421 // Frame pre encoded
422 frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10);
423
424 comparator.Start(/*max_threads_count=*/1);
425 comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2,
426 captured_time, captured_time);
427 comparator.AddComparison(stats_key,
428 /*captured=*/absl::nullopt,
429 /*rendered=*/absl::nullopt,
430 FrameComparisonType::kFrameInFlight, frame_stats);
431 comparator.Stop(/*last_rendered_frame_times=*/{});
432
433 EXPECT_EQ(comparator.stream_stats().size(), 1lu);
434 StreamStats stats = comparator.stream_stats().at(stats_key);
435 EXPECT_EQ(stats.stream_started_time, captured_time);
436 expectEmpty(stats.psnr);
437 expectEmpty(stats.ssim);
438 expectEmpty(stats.transport_time_ms);
439 expectEmpty(stats.total_delay_incl_transport_ms);
440 expectEmpty(stats.time_between_rendered_frames_ms);
441 expectEmpty(stats.encode_frame_rate);
442 expectEmpty(stats.encode_time_ms);
443 expectEmpty(stats.decode_time_ms);
444 expectEmpty(stats.receive_to_render_time_ms);
445 expectEmpty(stats.skipped_between_rendered);
446 expectEmpty(stats.freeze_time_ms);
447 expectEmpty(stats.time_between_freezes_ms);
448 expectEmpty(stats.resolution_of_decoded_frame);
449 expectEmpty(stats.target_encode_bitrate);
450 expectEmpty(stats.qp);
451 expectEmpty(stats.recv_key_frame_size_bytes);
452 expectEmpty(stats.recv_delta_frame_size_bytes);
453 EXPECT_EQ(stats.total_encoded_images_payload, 0);
454 EXPECT_EQ(stats.num_send_key_frames, 0);
455 EXPECT_EQ(stats.num_recv_key_frames, 0);
456 EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{
457 {FrameDropPhase::kBeforeEncoder, 0},
458 {FrameDropPhase::kByEncoder, 0},
459 {FrameDropPhase::kTransport, 0},
460 {FrameDropPhase::kByDecoder, 0},
461 {FrameDropPhase::kAfterDecoder, 0}}));
462 EXPECT_THAT(stats.encoders, IsEmpty());
463 EXPECT_THAT(stats.decoders, IsEmpty());
464 }
465
TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,EncodedInFlightKeyFrameAccountedInStats)466 TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,
467 EncodedInFlightKeyFrameAccountedInStats) {
468 DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer;
469 DefaultVideoQualityAnalyzerFramesComparator comparator(
470 Clock::GetRealTimeClock(), cpu_measurer,
471 DefaultVideoQualityAnalyzerOptions());
472
473 Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime();
474 uint16_t frame_id = 1;
475 size_t stream = 0;
476 size_t sender = 0;
477 size_t receiver = 1;
478 InternalStatsKey stats_key(stream, sender, receiver);
479
480 // Frame captured
481 FrameStats frame_stats(/*frame_id=*/1, captured_time);
482 // Frame pre encoded
483 frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10);
484 // Frame encoded
485 frame_stats.encoded_time = captured_time + TimeDelta::Millis(20);
486 frame_stats.used_encoder =
487 Vp8CodecForOneFrame(frame_id, frame_stats.encoded_time);
488 frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameKey;
489 frame_stats.encoded_image_size = DataSize::Bytes(1000);
490 frame_stats.target_encode_bitrate = 2000;
491 frame_stats.qp_values = StatsCounter(
492 /*samples=*/{{5, Timestamp::Seconds(1)}, {5, Timestamp::Seconds(2)}});
493
494 comparator.Start(/*max_threads_count=*/1);
495 comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2,
496 captured_time, captured_time);
497 comparator.AddComparison(stats_key,
498 /*captured=*/absl::nullopt,
499 /*rendered=*/absl::nullopt,
500 FrameComparisonType::kFrameInFlight, frame_stats);
501 comparator.Stop(/*last_rendered_frame_times=*/{});
502
503 EXPECT_EQ(comparator.stream_stats().size(), 1lu);
504 StreamStats stats = comparator.stream_stats().at(stats_key);
505 EXPECT_EQ(stats.stream_started_time, captured_time);
506 expectEmpty(stats.psnr);
507 expectEmpty(stats.ssim);
508 expectEmpty(stats.transport_time_ms);
509 expectEmpty(stats.total_delay_incl_transport_ms);
510 expectEmpty(stats.time_between_rendered_frames_ms);
511 expectEmpty(stats.encode_frame_rate);
512 EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.encode_time_ms), 10.0);
513 expectEmpty(stats.decode_time_ms);
514 expectEmpty(stats.receive_to_render_time_ms);
515 expectEmpty(stats.skipped_between_rendered);
516 expectEmpty(stats.freeze_time_ms);
517 expectEmpty(stats.time_between_freezes_ms);
518 expectEmpty(stats.resolution_of_decoded_frame);
519 EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.target_encode_bitrate), 2000.0);
520 EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.qp), 5.0);
521 expectEmpty(stats.recv_key_frame_size_bytes);
522 expectEmpty(stats.recv_delta_frame_size_bytes);
523 EXPECT_EQ(stats.total_encoded_images_payload, 1000);
524 EXPECT_EQ(stats.num_send_key_frames, 1);
525 EXPECT_EQ(stats.num_recv_key_frames, 0);
526 EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{
527 {FrameDropPhase::kBeforeEncoder, 0},
528 {FrameDropPhase::kByEncoder, 0},
529 {FrameDropPhase::kTransport, 0},
530 {FrameDropPhase::kByDecoder, 0},
531 {FrameDropPhase::kAfterDecoder, 0}}));
532 EXPECT_EQ(stats.encoders,
533 std::vector<StreamCodecInfo>{*frame_stats.used_encoder});
534 EXPECT_THAT(stats.decoders, IsEmpty());
535 }
536
TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,EncodedInFlightDeltaFrameAccountedInStats)537 TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,
538 EncodedInFlightDeltaFrameAccountedInStats) {
539 DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer;
540 DefaultVideoQualityAnalyzerFramesComparator comparator(
541 Clock::GetRealTimeClock(), cpu_measurer,
542 DefaultVideoQualityAnalyzerOptions());
543
544 Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime();
545 uint16_t frame_id = 1;
546 size_t stream = 0;
547 size_t sender = 0;
548 size_t receiver = 1;
549 InternalStatsKey stats_key(stream, sender, receiver);
550
551 // Frame captured
552 FrameStats frame_stats(/*frame_id=*/1, captured_time);
553 // Frame pre encoded
554 frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10);
555 // Frame encoded
556 frame_stats.encoded_time = captured_time + TimeDelta::Millis(20);
557 frame_stats.used_encoder =
558 Vp8CodecForOneFrame(frame_id, frame_stats.encoded_time);
559 frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameDelta;
560 frame_stats.encoded_image_size = DataSize::Bytes(1000);
561 frame_stats.target_encode_bitrate = 2000;
562 frame_stats.qp_values = StatsCounter(
563 /*samples=*/{{5, Timestamp::Seconds(1)}, {5, Timestamp::Seconds(2)}});
564
565 comparator.Start(/*max_threads_count=*/1);
566 comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2,
567 captured_time, captured_time);
568 comparator.AddComparison(stats_key,
569 /*captured=*/absl::nullopt,
570 /*rendered=*/absl::nullopt,
571 FrameComparisonType::kFrameInFlight, frame_stats);
572 comparator.Stop(/*last_rendered_frame_times=*/{});
573
574 EXPECT_EQ(comparator.stream_stats().size(), 1lu);
575 StreamStats stats = comparator.stream_stats().at(stats_key);
576 EXPECT_EQ(stats.stream_started_time, captured_time);
577 expectEmpty(stats.psnr);
578 expectEmpty(stats.ssim);
579 expectEmpty(stats.transport_time_ms);
580 expectEmpty(stats.total_delay_incl_transport_ms);
581 expectEmpty(stats.time_between_rendered_frames_ms);
582 expectEmpty(stats.encode_frame_rate);
583 EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.encode_time_ms), 10.0);
584 expectEmpty(stats.decode_time_ms);
585 expectEmpty(stats.receive_to_render_time_ms);
586 expectEmpty(stats.skipped_between_rendered);
587 expectEmpty(stats.freeze_time_ms);
588 expectEmpty(stats.time_between_freezes_ms);
589 expectEmpty(stats.resolution_of_decoded_frame);
590 EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.target_encode_bitrate), 2000.0);
591 EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.qp), 5.0);
592 expectEmpty(stats.recv_key_frame_size_bytes);
593 expectEmpty(stats.recv_delta_frame_size_bytes);
594 EXPECT_EQ(stats.total_encoded_images_payload, 1000);
595 EXPECT_EQ(stats.num_send_key_frames, 0);
596 EXPECT_EQ(stats.num_recv_key_frames, 0);
597 EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{
598 {FrameDropPhase::kBeforeEncoder, 0},
599 {FrameDropPhase::kByEncoder, 0},
600 {FrameDropPhase::kTransport, 0},
601 {FrameDropPhase::kByDecoder, 0},
602 {FrameDropPhase::kAfterDecoder, 0}}));
603 EXPECT_EQ(stats.encoders,
604 std::vector<StreamCodecInfo>{*frame_stats.used_encoder});
605 EXPECT_THAT(stats.decoders, IsEmpty());
606 }
607
TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,PreDecodedInFlightKeyFrameAccountedInStats)608 TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,
609 PreDecodedInFlightKeyFrameAccountedInStats) {
610 DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer;
611 DefaultVideoQualityAnalyzerFramesComparator comparator(
612 Clock::GetRealTimeClock(), cpu_measurer,
613 DefaultVideoQualityAnalyzerOptions());
614
615 Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime();
616 uint16_t frame_id = 1;
617 size_t stream = 0;
618 size_t sender = 0;
619 size_t receiver = 1;
620 InternalStatsKey stats_key(stream, sender, receiver);
621
622 // Frame captured
623 FrameStats frame_stats(/*frame_id=*/1, captured_time);
624 // Frame pre encoded
625 frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10);
626 // Frame encoded
627 frame_stats.encoded_time = captured_time + TimeDelta::Millis(20);
628 frame_stats.used_encoder =
629 Vp8CodecForOneFrame(frame_id, frame_stats.encoded_time);
630 frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameKey;
631 frame_stats.encoded_image_size = DataSize::Bytes(1000);
632 frame_stats.target_encode_bitrate = 2000;
633 frame_stats.qp_values = StatsCounter(
634 /*samples=*/{{5, Timestamp::Seconds(1)}, {5, Timestamp::Seconds(2)}});
635 // Frame pre decoded
636 frame_stats.pre_decoded_frame_type = VideoFrameType::kVideoFrameKey;
637 frame_stats.pre_decoded_image_size = DataSize::Bytes(500);
638 frame_stats.received_time = captured_time + TimeDelta::Millis(30);
639 frame_stats.decode_start_time = captured_time + TimeDelta::Millis(40);
640
641 comparator.Start(/*max_threads_count=*/1);
642 comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2,
643 captured_time, captured_time);
644 comparator.AddComparison(stats_key,
645 /*captured=*/absl::nullopt,
646 /*rendered=*/absl::nullopt,
647 FrameComparisonType::kFrameInFlight, frame_stats);
648 comparator.Stop(/*last_rendered_frame_times=*/{});
649
650 EXPECT_EQ(comparator.stream_stats().size(), 1lu);
651 StreamStats stats = comparator.stream_stats().at(stats_key);
652 EXPECT_EQ(stats.stream_started_time, captured_time);
653 expectEmpty(stats.psnr);
654 expectEmpty(stats.ssim);
655 EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.transport_time_ms), 20.0);
656 expectEmpty(stats.total_delay_incl_transport_ms);
657 expectEmpty(stats.time_between_rendered_frames_ms);
658 expectEmpty(stats.encode_frame_rate);
659 EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.encode_time_ms), 10.0);
660 expectEmpty(stats.decode_time_ms);
661 expectEmpty(stats.receive_to_render_time_ms);
662 expectEmpty(stats.skipped_between_rendered);
663 expectEmpty(stats.freeze_time_ms);
664 expectEmpty(stats.time_between_freezes_ms);
665 expectEmpty(stats.resolution_of_decoded_frame);
666 EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.target_encode_bitrate), 2000.0);
667 EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.qp), 5.0);
668 EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.recv_key_frame_size_bytes), 500.0);
669 expectEmpty(stats.recv_delta_frame_size_bytes);
670 EXPECT_EQ(stats.total_encoded_images_payload, 1000);
671 EXPECT_EQ(stats.num_send_key_frames, 1);
672 EXPECT_EQ(stats.num_recv_key_frames, 1);
673 EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{
674 {FrameDropPhase::kBeforeEncoder, 0},
675 {FrameDropPhase::kByEncoder, 0},
676 {FrameDropPhase::kTransport, 0},
677 {FrameDropPhase::kByDecoder, 0},
678 {FrameDropPhase::kAfterDecoder, 0}}));
679 EXPECT_EQ(stats.encoders,
680 std::vector<StreamCodecInfo>{*frame_stats.used_encoder});
681 EXPECT_THAT(stats.decoders, IsEmpty());
682 }
683
TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,DecodedInFlightKeyFrameAccountedInStats)684 TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,
685 DecodedInFlightKeyFrameAccountedInStats) {
686 DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer;
687 DefaultVideoQualityAnalyzerFramesComparator comparator(
688 Clock::GetRealTimeClock(), cpu_measurer,
689 DefaultVideoQualityAnalyzerOptions());
690
691 Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime();
692 uint16_t frame_id = 1;
693 size_t stream = 0;
694 size_t sender = 0;
695 size_t receiver = 1;
696 InternalStatsKey stats_key(stream, sender, receiver);
697
698 // Frame captured
699 FrameStats frame_stats(/*frame_id=*/1, captured_time);
700 // Frame pre encoded
701 frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10);
702 // Frame encoded
703 frame_stats.encoded_time = captured_time + TimeDelta::Millis(20);
704 frame_stats.used_encoder =
705 Vp8CodecForOneFrame(frame_id, frame_stats.encoded_time);
706 frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameKey;
707 frame_stats.encoded_image_size = DataSize::Bytes(1000);
708 frame_stats.target_encode_bitrate = 2000;
709 frame_stats.qp_values = StatsCounter(
710 /*samples=*/{{5, Timestamp::Seconds(1)}, {5, Timestamp::Seconds(2)}});
711 // Frame pre decoded
712 frame_stats.pre_decoded_frame_type = VideoFrameType::kVideoFrameKey;
713 frame_stats.pre_decoded_image_size = DataSize::Bytes(500);
714 frame_stats.received_time = captured_time + TimeDelta::Millis(30);
715 frame_stats.decode_start_time = captured_time + TimeDelta::Millis(40);
716 // Frame decoded
717 frame_stats.decode_end_time = captured_time + TimeDelta::Millis(50);
718 frame_stats.decoded_frame_width = 200;
719 frame_stats.decoded_frame_height = 100;
720
721 frame_stats.used_decoder =
722 Vp8CodecForOneFrame(frame_id, frame_stats.decode_end_time);
723
724 comparator.Start(/*max_threads_count=*/1);
725 comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2,
726 captured_time, captured_time);
727 comparator.AddComparison(stats_key,
728 /*captured=*/absl::nullopt,
729 /*rendered=*/absl::nullopt,
730 FrameComparisonType::kFrameInFlight, frame_stats);
731 comparator.Stop(/*last_rendered_frame_times=*/{});
732
733 EXPECT_EQ(comparator.stream_stats().size(), 1lu);
734 StreamStats stats = comparator.stream_stats().at(stats_key);
735 EXPECT_EQ(stats.stream_started_time, captured_time);
736 expectEmpty(stats.psnr);
737 expectEmpty(stats.ssim);
738 EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.transport_time_ms), 20.0);
739 expectEmpty(stats.total_delay_incl_transport_ms);
740 expectEmpty(stats.time_between_rendered_frames_ms);
741 expectEmpty(stats.encode_frame_rate);
742 EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.encode_time_ms), 10.0);
743 EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.decode_time_ms), 10.0);
744 expectEmpty(stats.receive_to_render_time_ms);
745 expectEmpty(stats.skipped_between_rendered);
746 expectEmpty(stats.freeze_time_ms);
747 expectEmpty(stats.time_between_freezes_ms);
748 EXPECT_GE(GetFirstOrDie(stats.resolution_of_decoded_frame), 200 * 100.0);
749 EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.target_encode_bitrate), 2000.0);
750 EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.qp), 5.0);
751 EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.recv_key_frame_size_bytes), 500.0);
752 expectEmpty(stats.recv_delta_frame_size_bytes);
753 EXPECT_EQ(stats.total_encoded_images_payload, 1000);
754 EXPECT_EQ(stats.num_send_key_frames, 1);
755 EXPECT_EQ(stats.num_recv_key_frames, 1);
756 EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{
757 {FrameDropPhase::kBeforeEncoder, 0},
758 {FrameDropPhase::kByEncoder, 0},
759 {FrameDropPhase::kTransport, 0},
760 {FrameDropPhase::kByDecoder, 0},
761 {FrameDropPhase::kAfterDecoder, 0}}));
762 EXPECT_EQ(stats.encoders,
763 std::vector<StreamCodecInfo>{*frame_stats.used_encoder});
764 EXPECT_EQ(stats.decoders,
765 std::vector<StreamCodecInfo>{*frame_stats.used_decoder});
766 }
767
TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,DecoderFailureOnInFlightKeyFrameAccountedInStats)768 TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,
769 DecoderFailureOnInFlightKeyFrameAccountedInStats) {
770 DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer;
771 DefaultVideoQualityAnalyzerFramesComparator comparator(
772 Clock::GetRealTimeClock(), cpu_measurer,
773 DefaultVideoQualityAnalyzerOptions());
774
775 Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime();
776 uint16_t frame_id = 1;
777 size_t stream = 0;
778 size_t sender = 0;
779 size_t receiver = 1;
780 InternalStatsKey stats_key(stream, sender, receiver);
781
782 // Frame captured
783 FrameStats frame_stats(/*frame_id=*/1, captured_time);
784 // Frame pre encoded
785 frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10);
786 // Frame encoded
787 frame_stats.encoded_time = captured_time + TimeDelta::Millis(20);
788 frame_stats.used_encoder =
789 Vp8CodecForOneFrame(frame_id, frame_stats.encoded_time);
790 frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameKey;
791 frame_stats.encoded_image_size = DataSize::Bytes(1000);
792 frame_stats.target_encode_bitrate = 2000;
793 frame_stats.qp_values = StatsCounter(
794 /*samples=*/{{5, Timestamp::Seconds(1)}, {5, Timestamp::Seconds(2)}});
795 // Frame pre decoded
796 frame_stats.pre_decoded_frame_type = VideoFrameType::kVideoFrameKey;
797 frame_stats.pre_decoded_image_size = DataSize::Bytes(500);
798 frame_stats.received_time = captured_time + TimeDelta::Millis(30);
799 frame_stats.decode_start_time = captured_time + TimeDelta::Millis(40);
800 // Frame decoded
801 frame_stats.decoder_failed = true;
802 frame_stats.used_decoder =
803 Vp8CodecForOneFrame(frame_id, frame_stats.decode_end_time);
804
805 comparator.Start(/*max_threads_count=*/1);
806 comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2,
807 captured_time, captured_time);
808 comparator.AddComparison(stats_key,
809 /*captured=*/absl::nullopt,
810 /*rendered=*/absl::nullopt,
811 FrameComparisonType::kFrameInFlight, frame_stats);
812 comparator.Stop(/*last_rendered_frame_times=*/{});
813
814 EXPECT_EQ(comparator.stream_stats().size(), 1lu);
815 StreamStats stats = comparator.stream_stats().at(stats_key);
816 EXPECT_EQ(stats.stream_started_time, captured_time);
817 expectEmpty(stats.psnr);
818 expectEmpty(stats.ssim);
819 EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.transport_time_ms), 20.0);
820 expectEmpty(stats.total_delay_incl_transport_ms);
821 expectEmpty(stats.time_between_rendered_frames_ms);
822 expectEmpty(stats.encode_frame_rate);
823 EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.encode_time_ms), 10.0);
824 expectEmpty(stats.decode_time_ms);
825 expectEmpty(stats.receive_to_render_time_ms);
826 expectEmpty(stats.skipped_between_rendered);
827 expectEmpty(stats.freeze_time_ms);
828 expectEmpty(stats.time_between_freezes_ms);
829 expectEmpty(stats.resolution_of_decoded_frame);
830 EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.target_encode_bitrate), 2000.0);
831 EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.qp), 5.0);
832 EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.recv_key_frame_size_bytes), 500.0);
833 expectEmpty(stats.recv_delta_frame_size_bytes);
834 EXPECT_EQ(stats.total_encoded_images_payload, 1000);
835 EXPECT_EQ(stats.num_send_key_frames, 1);
836 EXPECT_EQ(stats.num_recv_key_frames, 1);
837 // All frame in flight are not considered as dropped.
838 EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{
839 {FrameDropPhase::kBeforeEncoder, 0},
840 {FrameDropPhase::kByEncoder, 0},
841 {FrameDropPhase::kTransport, 0},
842 {FrameDropPhase::kByDecoder, 0},
843 {FrameDropPhase::kAfterDecoder, 0}}));
844 EXPECT_EQ(stats.encoders,
845 std::vector<StreamCodecInfo>{*frame_stats.used_encoder});
846 EXPECT_EQ(stats.decoders,
847 std::vector<StreamCodecInfo>{*frame_stats.used_decoder});
848 }
849 // Frame in flight end
850
851 // Dropped frame start
TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,CapturedOnlyDroppedFrameAccountedInStats)852 TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,
853 CapturedOnlyDroppedFrameAccountedInStats) {
854 DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer;
855 DefaultVideoQualityAnalyzerFramesComparator comparator(
856 Clock::GetRealTimeClock(), cpu_measurer,
857 DefaultVideoQualityAnalyzerOptions());
858
859 Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime();
860 size_t stream = 0;
861 size_t sender = 0;
862 size_t receiver = 1;
863 InternalStatsKey stats_key(stream, sender, receiver);
864
865 // Frame captured
866 FrameStats frame_stats(/*frame_id=*/1, captured_time);
867
868 comparator.Start(/*max_threads_count=*/1);
869 comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2,
870 captured_time, captured_time);
871 comparator.AddComparison(stats_key,
872 /*captured=*/absl::nullopt,
873 /*rendered=*/absl::nullopt,
874 FrameComparisonType::kDroppedFrame, frame_stats);
875 comparator.Stop(/*last_rendered_frame_times=*/{});
876
877 EXPECT_EQ(comparator.stream_stats().size(), 1lu);
878 StreamStats stats = comparator.stream_stats().at(stats_key);
879 EXPECT_EQ(stats.stream_started_time, captured_time);
880 expectEmpty(stats.psnr);
881 expectEmpty(stats.ssim);
882 expectEmpty(stats.transport_time_ms);
883 expectEmpty(stats.total_delay_incl_transport_ms);
884 expectEmpty(stats.time_between_rendered_frames_ms);
885 expectEmpty(stats.encode_frame_rate);
886 expectEmpty(stats.encode_time_ms);
887 expectEmpty(stats.decode_time_ms);
888 expectEmpty(stats.receive_to_render_time_ms);
889 expectEmpty(stats.skipped_between_rendered);
890 expectEmpty(stats.freeze_time_ms);
891 expectEmpty(stats.time_between_freezes_ms);
892 expectEmpty(stats.resolution_of_decoded_frame);
893 expectEmpty(stats.target_encode_bitrate);
894 expectEmpty(stats.qp);
895 expectEmpty(stats.recv_key_frame_size_bytes);
896 expectEmpty(stats.recv_delta_frame_size_bytes);
897 EXPECT_EQ(stats.total_encoded_images_payload, 0);
898 EXPECT_EQ(stats.num_send_key_frames, 0);
899 EXPECT_EQ(stats.num_recv_key_frames, 0);
900 EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{
901 {FrameDropPhase::kBeforeEncoder, 1},
902 {FrameDropPhase::kByEncoder, 0},
903 {FrameDropPhase::kTransport, 0},
904 {FrameDropPhase::kByDecoder, 0},
905 {FrameDropPhase::kAfterDecoder, 0}}));
906 EXPECT_THAT(stats.encoders, IsEmpty());
907 EXPECT_THAT(stats.decoders, IsEmpty());
908 }
909
TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,PreEncodedDroppedFrameAccountedInStats)910 TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,
911 PreEncodedDroppedFrameAccountedInStats) {
912 DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer;
913 DefaultVideoQualityAnalyzerFramesComparator comparator(
914 Clock::GetRealTimeClock(), cpu_measurer,
915 DefaultVideoQualityAnalyzerOptions());
916
917 Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime();
918 size_t stream = 0;
919 size_t sender = 0;
920 size_t receiver = 1;
921 InternalStatsKey stats_key(stream, sender, receiver);
922
923 // Frame captured
924 FrameStats frame_stats(/*frame_id=*/1, captured_time);
925 // Frame pre encoded
926 frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10);
927
928 comparator.Start(/*max_threads_count=*/1);
929 comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2,
930 captured_time, captured_time);
931 comparator.AddComparison(stats_key,
932 /*captured=*/absl::nullopt,
933 /*rendered=*/absl::nullopt,
934 FrameComparisonType::kDroppedFrame, frame_stats);
935 comparator.Stop(/*last_rendered_frame_times=*/{});
936
937 EXPECT_EQ(comparator.stream_stats().size(), 1lu);
938 StreamStats stats = comparator.stream_stats().at(stats_key);
939 EXPECT_EQ(stats.stream_started_time, captured_time);
940 expectEmpty(stats.psnr);
941 expectEmpty(stats.ssim);
942 expectEmpty(stats.transport_time_ms);
943 expectEmpty(stats.total_delay_incl_transport_ms);
944 expectEmpty(stats.time_between_rendered_frames_ms);
945 expectEmpty(stats.encode_frame_rate);
946 expectEmpty(stats.encode_time_ms);
947 expectEmpty(stats.decode_time_ms);
948 expectEmpty(stats.receive_to_render_time_ms);
949 expectEmpty(stats.skipped_between_rendered);
950 expectEmpty(stats.freeze_time_ms);
951 expectEmpty(stats.time_between_freezes_ms);
952 expectEmpty(stats.resolution_of_decoded_frame);
953 expectEmpty(stats.target_encode_bitrate);
954 expectEmpty(stats.qp);
955 expectEmpty(stats.recv_key_frame_size_bytes);
956 expectEmpty(stats.recv_delta_frame_size_bytes);
957 EXPECT_EQ(stats.total_encoded_images_payload, 0);
958 EXPECT_EQ(stats.num_send_key_frames, 0);
959 EXPECT_EQ(stats.num_recv_key_frames, 0);
960 EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{
961 {FrameDropPhase::kBeforeEncoder, 0},
962 {FrameDropPhase::kByEncoder, 1},
963 {FrameDropPhase::kTransport, 0},
964 {FrameDropPhase::kByDecoder, 0},
965 {FrameDropPhase::kAfterDecoder, 0}}));
966 EXPECT_THAT(stats.encoders, IsEmpty());
967 EXPECT_THAT(stats.decoders, IsEmpty());
968 }
969
TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,EncodedDroppedKeyFrameAccountedInStats)970 TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,
971 EncodedDroppedKeyFrameAccountedInStats) {
972 DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer;
973 DefaultVideoQualityAnalyzerFramesComparator comparator(
974 Clock::GetRealTimeClock(), cpu_measurer,
975 DefaultVideoQualityAnalyzerOptions());
976
977 Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime();
978 uint16_t frame_id = 1;
979 size_t stream = 0;
980 size_t sender = 0;
981 size_t receiver = 1;
982 InternalStatsKey stats_key(stream, sender, receiver);
983
984 // Frame captured
985 FrameStats frame_stats(/*frame_id=*/1, captured_time);
986 // Frame pre encoded
987 frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10);
988 // Frame encoded
989 frame_stats.encoded_time = captured_time + TimeDelta::Millis(20);
990 frame_stats.used_encoder =
991 Vp8CodecForOneFrame(frame_id, frame_stats.encoded_time);
992 frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameKey;
993 frame_stats.encoded_image_size = DataSize::Bytes(1000);
994 frame_stats.target_encode_bitrate = 2000;
995 frame_stats.qp_values = StatsCounter(
996 /*samples=*/{{5, Timestamp::Seconds(1)}, {5, Timestamp::Seconds(2)}});
997
998 comparator.Start(/*max_threads_count=*/1);
999 comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2,
1000 captured_time, captured_time);
1001 comparator.AddComparison(stats_key,
1002 /*captured=*/absl::nullopt,
1003 /*rendered=*/absl::nullopt,
1004 FrameComparisonType::kDroppedFrame, frame_stats);
1005 comparator.Stop(/*last_rendered_frame_times=*/{});
1006
1007 EXPECT_EQ(comparator.stream_stats().size(), 1lu);
1008 StreamStats stats = comparator.stream_stats().at(stats_key);
1009 EXPECT_EQ(stats.stream_started_time, captured_time);
1010 expectEmpty(stats.psnr);
1011 expectEmpty(stats.ssim);
1012 expectEmpty(stats.transport_time_ms);
1013 expectEmpty(stats.total_delay_incl_transport_ms);
1014 expectEmpty(stats.time_between_rendered_frames_ms);
1015 expectEmpty(stats.encode_frame_rate);
1016 EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.encode_time_ms), 10.0);
1017 expectEmpty(stats.decode_time_ms);
1018 expectEmpty(stats.receive_to_render_time_ms);
1019 expectEmpty(stats.skipped_between_rendered);
1020 expectEmpty(stats.freeze_time_ms);
1021 expectEmpty(stats.time_between_freezes_ms);
1022 expectEmpty(stats.resolution_of_decoded_frame);
1023 EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.target_encode_bitrate), 2000.0);
1024 EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.qp), 5.0);
1025 expectEmpty(stats.recv_key_frame_size_bytes);
1026 expectEmpty(stats.recv_delta_frame_size_bytes);
1027 EXPECT_EQ(stats.total_encoded_images_payload, 1000);
1028 EXPECT_EQ(stats.num_send_key_frames, 1);
1029 EXPECT_EQ(stats.num_recv_key_frames, 0);
1030 EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{
1031 {FrameDropPhase::kBeforeEncoder, 0},
1032 {FrameDropPhase::kByEncoder, 0},
1033 {FrameDropPhase::kTransport, 1},
1034 {FrameDropPhase::kByDecoder, 0},
1035 {FrameDropPhase::kAfterDecoder, 0}}));
1036 EXPECT_EQ(stats.encoders,
1037 std::vector<StreamCodecInfo>{*frame_stats.used_encoder});
1038 EXPECT_THAT(stats.decoders, IsEmpty());
1039 }
1040
TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,EncodedDroppedDeltaFrameAccountedInStats)1041 TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,
1042 EncodedDroppedDeltaFrameAccountedInStats) {
1043 DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer;
1044 DefaultVideoQualityAnalyzerFramesComparator comparator(
1045 Clock::GetRealTimeClock(), cpu_measurer,
1046 DefaultVideoQualityAnalyzerOptions());
1047
1048 Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime();
1049 uint16_t frame_id = 1;
1050 size_t stream = 0;
1051 size_t sender = 0;
1052 size_t receiver = 1;
1053 InternalStatsKey stats_key(stream, sender, receiver);
1054
1055 // Frame captured
1056 FrameStats frame_stats(/*frame_id=*/1, captured_time);
1057 // Frame pre encoded
1058 frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10);
1059 // Frame encoded
1060 frame_stats.encoded_time = captured_time + TimeDelta::Millis(20);
1061 frame_stats.used_encoder =
1062 Vp8CodecForOneFrame(frame_id, frame_stats.encoded_time);
1063 frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameDelta;
1064 frame_stats.encoded_image_size = DataSize::Bytes(1000);
1065 frame_stats.target_encode_bitrate = 2000;
1066 frame_stats.qp_values = StatsCounter(
1067 /*samples=*/{{5, Timestamp::Seconds(1)}, {5, Timestamp::Seconds(2)}});
1068
1069 comparator.Start(/*max_threads_count=*/1);
1070 comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2,
1071 captured_time, captured_time);
1072 comparator.AddComparison(stats_key,
1073 /*captured=*/absl::nullopt,
1074 /*rendered=*/absl::nullopt,
1075 FrameComparisonType::kDroppedFrame, frame_stats);
1076 comparator.Stop(/*last_rendered_frame_times=*/{});
1077
1078 EXPECT_EQ(comparator.stream_stats().size(), 1lu);
1079 StreamStats stats = comparator.stream_stats().at(stats_key);
1080 EXPECT_EQ(stats.stream_started_time, captured_time);
1081 expectEmpty(stats.psnr);
1082 expectEmpty(stats.ssim);
1083 expectEmpty(stats.transport_time_ms);
1084 expectEmpty(stats.total_delay_incl_transport_ms);
1085 expectEmpty(stats.time_between_rendered_frames_ms);
1086 expectEmpty(stats.encode_frame_rate);
1087 EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.encode_time_ms), 10.0);
1088 expectEmpty(stats.decode_time_ms);
1089 expectEmpty(stats.receive_to_render_time_ms);
1090 expectEmpty(stats.skipped_between_rendered);
1091 expectEmpty(stats.freeze_time_ms);
1092 expectEmpty(stats.time_between_freezes_ms);
1093 expectEmpty(stats.resolution_of_decoded_frame);
1094 EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.target_encode_bitrate), 2000.0);
1095 EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.qp), 5.0);
1096 expectEmpty(stats.recv_key_frame_size_bytes);
1097 expectEmpty(stats.recv_delta_frame_size_bytes);
1098 EXPECT_EQ(stats.total_encoded_images_payload, 1000);
1099 EXPECT_EQ(stats.num_send_key_frames, 0);
1100 EXPECT_EQ(stats.num_recv_key_frames, 0);
1101 EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{
1102 {FrameDropPhase::kBeforeEncoder, 0},
1103 {FrameDropPhase::kByEncoder, 0},
1104 {FrameDropPhase::kTransport, 1},
1105 {FrameDropPhase::kByDecoder, 0},
1106 {FrameDropPhase::kAfterDecoder, 0}}));
1107 EXPECT_EQ(stats.encoders,
1108 std::vector<StreamCodecInfo>{*frame_stats.used_encoder});
1109 EXPECT_THAT(stats.decoders, IsEmpty());
1110 }
1111
TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,PreDecodedDroppedKeyFrameAccountedInStats)1112 TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,
1113 PreDecodedDroppedKeyFrameAccountedInStats) {
1114 DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer;
1115 DefaultVideoQualityAnalyzerFramesComparator comparator(
1116 Clock::GetRealTimeClock(), cpu_measurer,
1117 DefaultVideoQualityAnalyzerOptions());
1118
1119 Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime();
1120 uint16_t frame_id = 1;
1121 size_t stream = 0;
1122 size_t sender = 0;
1123 size_t receiver = 1;
1124 InternalStatsKey stats_key(stream, sender, receiver);
1125
1126 // Frame captured
1127 FrameStats frame_stats(/*frame_id=*/1, captured_time);
1128 // Frame pre encoded
1129 frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10);
1130 // Frame encoded
1131 frame_stats.encoded_time = captured_time + TimeDelta::Millis(20);
1132 frame_stats.used_encoder =
1133 Vp8CodecForOneFrame(frame_id, frame_stats.encoded_time);
1134 frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameKey;
1135 frame_stats.encoded_image_size = DataSize::Bytes(1000);
1136 frame_stats.target_encode_bitrate = 2000;
1137 // Frame pre decoded
1138 frame_stats.pre_decoded_frame_type = VideoFrameType::kVideoFrameKey;
1139 frame_stats.pre_decoded_image_size = DataSize::Bytes(500);
1140 frame_stats.received_time = captured_time + TimeDelta::Millis(30);
1141 frame_stats.decode_start_time = captured_time + TimeDelta::Millis(40);
1142
1143 comparator.Start(/*max_threads_count=*/1);
1144 comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2,
1145 captured_time, captured_time);
1146 comparator.AddComparison(stats_key,
1147 /*captured=*/absl::nullopt,
1148 /*rendered=*/absl::nullopt,
1149 FrameComparisonType::kDroppedFrame, frame_stats);
1150 comparator.Stop(/*last_rendered_frame_times=*/{});
1151
1152 EXPECT_EQ(comparator.stream_stats().size(), 1lu);
1153 StreamStats stats = comparator.stream_stats().at(stats_key);
1154 EXPECT_EQ(stats.stream_started_time, captured_time);
1155 expectEmpty(stats.psnr);
1156 expectEmpty(stats.ssim);
1157 expectEmpty(stats.transport_time_ms);
1158 expectEmpty(stats.total_delay_incl_transport_ms);
1159 expectEmpty(stats.time_between_rendered_frames_ms);
1160 expectEmpty(stats.encode_frame_rate);
1161 EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.encode_time_ms), 10.0);
1162 expectEmpty(stats.decode_time_ms);
1163 expectEmpty(stats.receive_to_render_time_ms);
1164 expectEmpty(stats.skipped_between_rendered);
1165 expectEmpty(stats.freeze_time_ms);
1166 expectEmpty(stats.time_between_freezes_ms);
1167 expectEmpty(stats.resolution_of_decoded_frame);
1168 EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.target_encode_bitrate), 2000.0);
1169 expectEmpty(stats.recv_key_frame_size_bytes);
1170 expectEmpty(stats.recv_delta_frame_size_bytes);
1171 EXPECT_EQ(stats.total_encoded_images_payload, 1000);
1172 EXPECT_EQ(stats.num_send_key_frames, 1);
1173 EXPECT_EQ(stats.num_recv_key_frames, 0);
1174 EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{
1175 {FrameDropPhase::kBeforeEncoder, 0},
1176 {FrameDropPhase::kByEncoder, 0},
1177 {FrameDropPhase::kTransport, 0},
1178 {FrameDropPhase::kByDecoder, 1},
1179 {FrameDropPhase::kAfterDecoder, 0}}));
1180 EXPECT_EQ(stats.encoders,
1181 std::vector<StreamCodecInfo>{*frame_stats.used_encoder});
1182 EXPECT_THAT(stats.decoders, IsEmpty());
1183 }
1184
TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,DecodedDroppedKeyFrameAccountedInStats)1185 TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,
1186 DecodedDroppedKeyFrameAccountedInStats) {
1187 // We don't really drop frames after decoder, so it's a bit unclear what is
1188 // correct way to account such frames in stats, so this test just fixes some
1189 // current way.
1190 DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer;
1191 DefaultVideoQualityAnalyzerFramesComparator comparator(
1192 Clock::GetRealTimeClock(), cpu_measurer,
1193 DefaultVideoQualityAnalyzerOptions());
1194
1195 Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime();
1196 uint16_t frame_id = 1;
1197 size_t stream = 0;
1198 size_t sender = 0;
1199 size_t receiver = 1;
1200 InternalStatsKey stats_key(stream, sender, receiver);
1201
1202 // Frame captured
1203 FrameStats frame_stats(/*frame_id=*/1, captured_time);
1204 // Frame pre encoded
1205 frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10);
1206 // Frame encoded
1207 frame_stats.encoded_time = captured_time + TimeDelta::Millis(20);
1208 frame_stats.used_encoder =
1209 Vp8CodecForOneFrame(frame_id, frame_stats.encoded_time);
1210 frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameKey;
1211 frame_stats.encoded_image_size = DataSize::Bytes(1000);
1212 frame_stats.target_encode_bitrate = 2000;
1213 frame_stats.qp_values = StatsCounter(
1214 /*samples=*/{{5, Timestamp::Seconds(1)}, {5, Timestamp::Seconds(2)}});
1215 // Frame pre decoded
1216 frame_stats.pre_decoded_frame_type = VideoFrameType::kVideoFrameKey;
1217 frame_stats.pre_decoded_image_size = DataSize::Bytes(500);
1218 frame_stats.received_time = captured_time + TimeDelta::Millis(30);
1219 frame_stats.decode_start_time = captured_time + TimeDelta::Millis(40);
1220 // Frame decoded
1221 frame_stats.decode_end_time = captured_time + TimeDelta::Millis(50);
1222 frame_stats.used_decoder =
1223 Vp8CodecForOneFrame(frame_id, frame_stats.decode_end_time);
1224 frame_stats.decoded_frame_width = 200;
1225 frame_stats.decoded_frame_height = 100;
1226
1227 comparator.Start(/*max_threads_count=*/1);
1228 comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2,
1229 captured_time, captured_time);
1230 comparator.AddComparison(stats_key,
1231 /*captured=*/absl::nullopt,
1232 /*rendered=*/absl::nullopt,
1233 FrameComparisonType::kDroppedFrame, frame_stats);
1234 comparator.Stop(/*last_rendered_frame_times=*/{});
1235
1236 EXPECT_EQ(comparator.stream_stats().size(), 1lu);
1237 StreamStats stats = comparator.stream_stats().at(stats_key);
1238 EXPECT_EQ(stats.stream_started_time, captured_time);
1239 expectEmpty(stats.psnr);
1240 expectEmpty(stats.ssim);
1241 expectEmpty(stats.transport_time_ms);
1242 expectEmpty(stats.total_delay_incl_transport_ms);
1243 expectEmpty(stats.time_between_rendered_frames_ms);
1244 expectEmpty(stats.encode_frame_rate);
1245 EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.encode_time_ms), 10.0);
1246 expectEmpty(stats.decode_time_ms);
1247 expectEmpty(stats.receive_to_render_time_ms);
1248 expectEmpty(stats.skipped_between_rendered);
1249 expectEmpty(stats.freeze_time_ms);
1250 expectEmpty(stats.time_between_freezes_ms);
1251 expectEmpty(stats.resolution_of_decoded_frame);
1252 EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.target_encode_bitrate), 2000.0);
1253 EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.qp), 5.0);
1254 expectEmpty(stats.recv_key_frame_size_bytes);
1255 expectEmpty(stats.recv_delta_frame_size_bytes);
1256 EXPECT_EQ(stats.total_encoded_images_payload, 1000);
1257 EXPECT_EQ(stats.num_send_key_frames, 1);
1258 EXPECT_EQ(stats.num_recv_key_frames, 0);
1259 EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{
1260 {FrameDropPhase::kBeforeEncoder, 0},
1261 {FrameDropPhase::kByEncoder, 0},
1262 {FrameDropPhase::kTransport, 0},
1263 {FrameDropPhase::kByDecoder, 0},
1264 {FrameDropPhase::kAfterDecoder, 1}}));
1265 EXPECT_EQ(stats.encoders,
1266 std::vector<StreamCodecInfo>{*frame_stats.used_encoder});
1267 EXPECT_EQ(stats.decoders,
1268 std::vector<StreamCodecInfo>{*frame_stats.used_decoder});
1269 }
1270
TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,DecoderFailedDroppedKeyFrameAccountedInStats)1271 TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,
1272 DecoderFailedDroppedKeyFrameAccountedInStats) {
1273 DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer;
1274 DefaultVideoQualityAnalyzerFramesComparator comparator(
1275 Clock::GetRealTimeClock(), cpu_measurer,
1276 DefaultVideoQualityAnalyzerOptions());
1277
1278 Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime();
1279 uint16_t frame_id = 1;
1280 size_t stream = 0;
1281 size_t sender = 0;
1282 size_t receiver = 1;
1283 InternalStatsKey stats_key(stream, sender, receiver);
1284
1285 // Frame captured
1286 FrameStats frame_stats(/*frame_id=*/1, captured_time);
1287 // Frame pre encoded
1288 frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10);
1289 // Frame encoded
1290 frame_stats.encoded_time = captured_time + TimeDelta::Millis(20);
1291 frame_stats.used_encoder =
1292 Vp8CodecForOneFrame(frame_id, frame_stats.encoded_time);
1293 frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameKey;
1294 frame_stats.encoded_image_size = DataSize::Bytes(1000);
1295 frame_stats.target_encode_bitrate = 2000;
1296 frame_stats.qp_values = StatsCounter(
1297 /*samples=*/{{5, Timestamp::Seconds(1)}, {5, Timestamp::Seconds(2)}});
1298 // Frame pre decoded
1299 frame_stats.pre_decoded_frame_type = VideoFrameType::kVideoFrameKey;
1300 frame_stats.pre_decoded_image_size = DataSize::Bytes(500);
1301 frame_stats.received_time = captured_time + TimeDelta::Millis(30);
1302 frame_stats.decode_start_time = captured_time + TimeDelta::Millis(40);
1303 // Frame decoded
1304 frame_stats.decoder_failed = true;
1305 frame_stats.used_decoder =
1306 Vp8CodecForOneFrame(frame_id, frame_stats.decode_end_time);
1307
1308 comparator.Start(/*max_threads_count=*/1);
1309 comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2,
1310 captured_time, captured_time);
1311 comparator.AddComparison(stats_key,
1312 /*captured=*/absl::nullopt,
1313 /*rendered=*/absl::nullopt,
1314 FrameComparisonType::kDroppedFrame, frame_stats);
1315 comparator.Stop(/*last_rendered_frame_times=*/{});
1316
1317 EXPECT_EQ(comparator.stream_stats().size(), 1lu);
1318 StreamStats stats = comparator.stream_stats().at(stats_key);
1319 EXPECT_EQ(stats.stream_started_time, captured_time);
1320 expectEmpty(stats.psnr);
1321 expectEmpty(stats.ssim);
1322 EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.transport_time_ms), 20.0);
1323 expectEmpty(stats.total_delay_incl_transport_ms);
1324 expectEmpty(stats.time_between_rendered_frames_ms);
1325 expectEmpty(stats.encode_frame_rate);
1326 EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.encode_time_ms), 10.0);
1327 expectEmpty(stats.decode_time_ms);
1328 expectEmpty(stats.receive_to_render_time_ms);
1329 expectEmpty(stats.skipped_between_rendered);
1330 expectEmpty(stats.freeze_time_ms);
1331 expectEmpty(stats.time_between_freezes_ms);
1332 expectEmpty(stats.resolution_of_decoded_frame);
1333 EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.target_encode_bitrate), 2000.0);
1334 EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.qp), 5.0);
1335 EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.recv_key_frame_size_bytes), 500.0);
1336 expectEmpty(stats.recv_delta_frame_size_bytes);
1337 EXPECT_EQ(stats.total_encoded_images_payload, 1000);
1338 EXPECT_EQ(stats.num_send_key_frames, 1);
1339 EXPECT_EQ(stats.num_recv_key_frames, 1);
1340 EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{
1341 {FrameDropPhase::kBeforeEncoder, 0},
1342 {FrameDropPhase::kByEncoder, 0},
1343 {FrameDropPhase::kTransport, 0},
1344 {FrameDropPhase::kByDecoder, 1},
1345 {FrameDropPhase::kAfterDecoder, 0}}));
1346 EXPECT_EQ(stats.encoders,
1347 std::vector<StreamCodecInfo>{*frame_stats.used_encoder});
1348 EXPECT_EQ(stats.decoders,
1349 std::vector<StreamCodecInfo>{*frame_stats.used_decoder});
1350 }
1351 // Dropped frame end
1352
1353 // Regular frame start
TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,RenderedKeyFrameAccountedInStats)1354 TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,
1355 RenderedKeyFrameAccountedInStats) {
1356 DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer;
1357 DefaultVideoQualityAnalyzerFramesComparator comparator(
1358 Clock::GetRealTimeClock(), cpu_measurer,
1359 DefaultVideoQualityAnalyzerOptions());
1360
1361 Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime();
1362 uint16_t frame_id = 1;
1363 size_t stream = 0;
1364 size_t sender = 0;
1365 size_t receiver = 1;
1366 InternalStatsKey stats_key(stream, sender, receiver);
1367
1368 // Frame captured
1369 VideoFrame frame =
1370 CreateFrame(frame_id, /*width=*/320, /*height=*/180, captured_time);
1371 FrameStats frame_stats(/*frame_id=*/1, captured_time);
1372 // Frame pre encoded
1373 frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10);
1374 // Frame encoded
1375 frame_stats.encoded_time = captured_time + TimeDelta::Millis(20);
1376 frame_stats.used_encoder =
1377 Vp8CodecForOneFrame(frame_id, frame_stats.encoded_time);
1378 frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameKey;
1379 frame_stats.encoded_image_size = DataSize::Bytes(1000);
1380 frame_stats.target_encode_bitrate = 2000;
1381 frame_stats.qp_values = StatsCounter(
1382 /*samples=*/{{5, Timestamp::Seconds(1)}, {5, Timestamp::Seconds(2)}});
1383 // Frame pre decoded
1384 frame_stats.pre_decoded_frame_type = VideoFrameType::kVideoFrameKey;
1385 frame_stats.pre_decoded_image_size = DataSize::Bytes(500);
1386 frame_stats.received_time = captured_time + TimeDelta::Millis(30);
1387 frame_stats.decode_start_time = captured_time + TimeDelta::Millis(40);
1388 // Frame decoded
1389 frame_stats.decode_end_time = captured_time + TimeDelta::Millis(50);
1390 frame_stats.used_decoder =
1391 Vp8CodecForOneFrame(frame_id, frame_stats.decode_end_time);
1392 frame_stats.decoded_frame_width = 200;
1393 frame_stats.decoded_frame_height = 100;
1394 // Frame rendered
1395 frame_stats.rendered_time = captured_time + TimeDelta::Millis(60);
1396
1397 comparator.Start(/*max_threads_count=*/1);
1398 comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2,
1399 captured_time, captured_time);
1400 comparator.AddComparison(stats_key,
1401 /*captured=*/frame,
1402 /*rendered=*/frame, FrameComparisonType::kRegular,
1403 frame_stats);
1404 comparator.Stop(/*last_rendered_frame_times=*/{});
1405
1406 EXPECT_EQ(comparator.stream_stats().size(), 1lu);
1407 StreamStats stats = comparator.stream_stats().at(stats_key);
1408 EXPECT_EQ(stats.stream_started_time, captured_time);
1409 EXPECT_GE(GetFirstOrDie(stats.psnr), 20);
1410 EXPECT_GE(GetFirstOrDie(stats.ssim), 0.5);
1411 EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.transport_time_ms), 20.0);
1412 EXPECT_GE(GetFirstOrDie(stats.total_delay_incl_transport_ms), 60.0);
1413 expectEmpty(stats.time_between_rendered_frames_ms);
1414 expectEmpty(stats.encode_frame_rate);
1415 EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.encode_time_ms), 10.0);
1416 EXPECT_GE(GetFirstOrDie(stats.decode_time_ms), 10.0);
1417 EXPECT_GE(GetFirstOrDie(stats.receive_to_render_time_ms), 30.0);
1418 expectEmpty(stats.skipped_between_rendered);
1419 expectEmpty(stats.freeze_time_ms);
1420 expectEmpty(stats.time_between_freezes_ms);
1421 EXPECT_GE(GetFirstOrDie(stats.resolution_of_decoded_frame), 200 * 100.0);
1422 EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.target_encode_bitrate), 2000.0);
1423 EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.qp), 5.0);
1424 EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.recv_key_frame_size_bytes), 500.0);
1425 expectEmpty(stats.recv_delta_frame_size_bytes);
1426 EXPECT_EQ(stats.total_encoded_images_payload, 1000);
1427 EXPECT_EQ(stats.num_send_key_frames, 1);
1428 EXPECT_EQ(stats.num_recv_key_frames, 1);
1429 EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{
1430 {FrameDropPhase::kBeforeEncoder, 0},
1431 {FrameDropPhase::kByEncoder, 0},
1432 {FrameDropPhase::kTransport, 0},
1433 {FrameDropPhase::kByDecoder, 0},
1434 {FrameDropPhase::kAfterDecoder, 0}}));
1435 EXPECT_EQ(stats.encoders,
1436 std::vector<StreamCodecInfo>{*frame_stats.used_encoder});
1437 EXPECT_EQ(stats.decoders,
1438 std::vector<StreamCodecInfo>{*frame_stats.used_decoder});
1439 }
1440
TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,AllStatsHaveMetadataSet)1441 TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, AllStatsHaveMetadataSet) {
1442 DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer;
1443 DefaultVideoQualityAnalyzerFramesComparator comparator(
1444 Clock::GetRealTimeClock(), cpu_measurer,
1445 DefaultVideoQualityAnalyzerOptions());
1446
1447 Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime();
1448 uint16_t frame_id = 1;
1449 size_t stream = 0;
1450 size_t sender = 0;
1451 size_t receiver = 1;
1452 InternalStatsKey stats_key(stream, sender, receiver);
1453
1454 // Frame captured
1455 VideoFrame frame =
1456 CreateFrame(frame_id, /*width=*/320, /*height=*/180, captured_time);
1457 FrameStats frame_stats(/*frame_id=*/1, captured_time);
1458 // Frame pre encoded
1459 frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10);
1460 // Frame encoded
1461 frame_stats.encoded_time = captured_time + TimeDelta::Millis(20);
1462 frame_stats.used_encoder =
1463 Vp8CodecForOneFrame(frame_id, frame_stats.encoded_time);
1464 frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameKey;
1465 frame_stats.encoded_image_size = DataSize::Bytes(1000);
1466 frame_stats.target_encode_bitrate = 2000;
1467 frame_stats.qp_values = StatsCounter(
1468 /*samples=*/{{5, Timestamp::Seconds(1)}, {5, Timestamp::Seconds(2)}});
1469 // Frame pre decoded
1470 frame_stats.pre_decoded_frame_type = VideoFrameType::kVideoFrameKey;
1471 frame_stats.pre_decoded_image_size = DataSize::Bytes(500);
1472 frame_stats.received_time = captured_time + TimeDelta::Millis(30);
1473 frame_stats.decode_start_time = captured_time + TimeDelta::Millis(40);
1474 // Frame decoded
1475 frame_stats.decode_end_time = captured_time + TimeDelta::Millis(50);
1476 frame_stats.used_decoder =
1477 Vp8CodecForOneFrame(frame_id, frame_stats.decode_end_time);
1478 // Frame rendered
1479 frame_stats.rendered_time = captured_time + TimeDelta::Millis(60);
1480 frame_stats.decoded_frame_width = 200;
1481 frame_stats.decoded_frame_height = 100;
1482
1483 comparator.Start(/*max_threads_count=*/1);
1484 comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2,
1485 captured_time, captured_time);
1486 comparator.AddComparison(stats_key,
1487 /*captured=*/frame,
1488 /*rendered=*/frame, FrameComparisonType::kRegular,
1489 frame_stats);
1490 comparator.Stop(/*last_rendered_frame_times=*/{});
1491
1492 EXPECT_EQ(comparator.stream_stats().size(), 1lu);
1493 StreamStats stats = comparator.stream_stats().at(stats_key);
1494 AssertFirstMetadataHasField(stats.psnr, "frame_id", "1");
1495 AssertFirstMetadataHasField(stats.ssim, "frame_id", "1");
1496 AssertFirstMetadataHasField(stats.transport_time_ms, "frame_id", "1");
1497 AssertFirstMetadataHasField(stats.total_delay_incl_transport_ms, "frame_id",
1498 "1");
1499 AssertFirstMetadataHasField(stats.encode_time_ms, "frame_id", "1");
1500 AssertFirstMetadataHasField(stats.decode_time_ms, "frame_id", "1");
1501 AssertFirstMetadataHasField(stats.receive_to_render_time_ms, "frame_id", "1");
1502 AssertFirstMetadataHasField(stats.resolution_of_decoded_frame, "frame_id",
1503 "1");
1504 AssertFirstMetadataHasField(stats.target_encode_bitrate, "frame_id", "1");
1505 AssertFirstMetadataHasField(stats.qp, "frame_id", "1");
1506 AssertFirstMetadataHasField(stats.recv_key_frame_size_bytes, "frame_id", "1");
1507
1508 expectEmpty(stats.recv_delta_frame_size_bytes);
1509 }
1510 // Regular frame end
1511
TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,FreezeStatsPresentedWithMetadataAfterAddFrameWithSkippedAndDelay)1512 TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,
1513 FreezeStatsPresentedWithMetadataAfterAddFrameWithSkippedAndDelay) {
1514 DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer;
1515 DefaultVideoQualityAnalyzerFramesComparator comparator(
1516 Clock::GetRealTimeClock(), cpu_measurer, AnalyzerOptionsForTest());
1517
1518 Timestamp stream_start_time = Clock::GetRealTimeClock()->CurrentTime();
1519 size_t stream = 0;
1520 size_t sender = 0;
1521 size_t receiver = 1;
1522 size_t peers_count = 2;
1523 InternalStatsKey stats_key(stream, sender, receiver);
1524
1525 comparator.Start(/*max_threads_count=*/1);
1526 comparator.EnsureStatsForStream(stream, sender, peers_count,
1527 stream_start_time, stream_start_time);
1528
1529 // Add 5 frames which were rendered with 30 fps (~30ms between frames)
1530 // Frame ids are in [1..5] and last frame is with 120ms offset from first.
1531 Timestamp prev_frame_rendered_time = Timestamp::MinusInfinity();
1532 for (int i = 0; i < 5; ++i) {
1533 FrameStats frame_stats = FrameStatsWith10msDeltaBetweenPhasesAnd10x10Frame(
1534 /*frame_id=*/i + 1, stream_start_time + TimeDelta::Millis(30 * i));
1535 frame_stats.prev_frame_rendered_time = prev_frame_rendered_time;
1536 prev_frame_rendered_time = frame_stats.rendered_time;
1537
1538 comparator.AddComparison(stats_key,
1539 /*captured=*/absl::nullopt,
1540 /*rendered=*/absl::nullopt,
1541 FrameComparisonType::kRegular, frame_stats);
1542 }
1543
1544 // Next frame was rendered with 4 frames skipped and delay 300ms after last
1545 // frame.
1546 FrameStats freeze_frame_stats =
1547 FrameStatsWith10msDeltaBetweenPhasesAnd10x10Frame(
1548 /*frame_id=*/10, stream_start_time + TimeDelta::Millis(120 + 300));
1549 freeze_frame_stats.prev_frame_rendered_time = prev_frame_rendered_time;
1550
1551 comparator.AddComparison(stats_key,
1552 /*skipped_between_rendered=*/4,
1553 /*captured=*/absl::nullopt,
1554 /*rendered=*/absl::nullopt,
1555 FrameComparisonType::kRegular, freeze_frame_stats);
1556 comparator.Stop(/*last_rendered_frame_times=*/{});
1557
1558 StreamStats stats = comparator.stream_stats().at(stats_key);
1559 ASSERT_THAT(GetFirstOrDie(stats.skipped_between_rendered), Eq(4));
1560 AssertFirstMetadataHasField(stats.skipped_between_rendered, "frame_id", "10");
1561 ASSERT_THAT(GetFirstOrDie(stats.freeze_time_ms), Eq(300));
1562 AssertFirstMetadataHasField(stats.freeze_time_ms, "frame_id", "10");
1563 // 180ms is time from the stream start to the rendered time of the last frame
1564 // among first 5 frames which were received before freeze.
1565 ASSERT_THAT(GetFirstOrDie(stats.time_between_freezes_ms), Eq(180));
1566 AssertFirstMetadataHasField(stats.time_between_freezes_ms, "frame_id", "10");
1567 }
1568 // Stats validation tests end.
1569
1570 } // namespace
1571 } // namespace webrtc
1572