xref: /aosp_15_r20/external/webrtc/video/video_stream_buffer_controller_unittest.cc (revision d9f758449e529ab9291ac668be2861e7a55c2422) 
1 /*
2  *  Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #include "video/video_stream_buffer_controller.h"
12 
13 #include <stdint.h>
14 
15 #include <limits>
16 #include <memory>
17 #include <string>
18 #include <tuple>
19 #include <utility>
20 #include <vector>
21 
22 #include "absl/types/optional.h"
23 #include "absl/types/variant.h"
24 #include "api/metronome/test/fake_metronome.h"
25 #include "api/units/frequency.h"
26 #include "api/units/time_delta.h"
27 #include "api/units/timestamp.h"
28 #include "api/video/video_content_type.h"
29 #include "api/video/video_timing.h"
30 #include "rtc_base/checks.h"
31 #include "test/fake_encoded_frame.h"
32 #include "test/gmock.h"
33 #include "test/gtest.h"
34 #include "test/scoped_key_value_config.h"
35 #include "test/time_controller/simulated_time_controller.h"
36 #include "video/decode_synchronizer.h"
37 #include "video/task_queue_frame_decode_scheduler.h"
38 
39 using ::testing::_;
40 using ::testing::AllOf;
41 using ::testing::Contains;
42 using ::testing::Each;
43 using ::testing::Eq;
44 using ::testing::IsEmpty;
45 using ::testing::Matches;
46 using ::testing::Ne;
47 using ::testing::Not;
48 using ::testing::Optional;
49 using ::testing::Pointee;
50 using ::testing::SizeIs;
51 using ::testing::VariantWith;
52 
53 namespace webrtc {
54 
55 namespace {
56 
57 constexpr size_t kFrameSize = 10;
58 constexpr uint32_t kFps30Rtp = 90000 / 30;
59 constexpr TimeDelta kFps30Delay = 1 / Frequency::Hertz(30);
60 const VideoPlayoutDelay kZeroPlayoutDelay = {0, 0};
61 constexpr Timestamp kClockStart = Timestamp::Millis(1000);
62 
TimedOut()63 auto TimedOut() {
64   return Optional(VariantWith<TimeDelta>(_));
65 }
66 
Frame(testing::Matcher<EncodedFrame> m)67 auto Frame(testing::Matcher<EncodedFrame> m) {
68   return Optional(VariantWith<std::unique_ptr<EncodedFrame>>(Pointee(m)));
69 }
70 
WithReceiveTimeFromRtpTimestamp(std::unique_ptr<test::FakeEncodedFrame> frame)71 std::unique_ptr<test::FakeEncodedFrame> WithReceiveTimeFromRtpTimestamp(
72     std::unique_ptr<test::FakeEncodedFrame> frame) {
73   if (frame->Timestamp() == 0) {
74     frame->SetReceivedTime(kClockStart.ms());
75   } else {
76     frame->SetReceivedTime(
77         TimeDelta::Seconds(frame->Timestamp() / 90000.0).ms() +
78         kClockStart.ms());
79   }
80   return frame;
81 }
82 
83 class VCMTimingTest : public VCMTiming {
84  public:
85   using VCMTiming::VCMTiming;
IncomingTimestamp(uint32_t rtp_timestamp,Timestamp last_packet_time)86   void IncomingTimestamp(uint32_t rtp_timestamp,
87                          Timestamp last_packet_time) override {
88     IncomingTimestampMocked(rtp_timestamp, last_packet_time);
89     VCMTiming::IncomingTimestamp(rtp_timestamp, last_packet_time);
90   }
91 
92   MOCK_METHOD(void,
93               IncomingTimestampMocked,
94               (uint32_t rtp_timestamp, Timestamp last_packet_time),
95               ());
96 };
97 
98 class VCMReceiveStatisticsCallbackMock : public VCMReceiveStatisticsCallback {
99  public:
100   MOCK_METHOD(void,
101               OnCompleteFrame,
102               (bool is_keyframe,
103                size_t size_bytes,
104                VideoContentType content_type),
105               (override));
106   MOCK_METHOD(void, OnDroppedFrames, (uint32_t num_dropped), (override));
107   MOCK_METHOD(void,
108               OnFrameBufferTimingsUpdated,
109               (int max_decode_ms,
110                int current_delay_ms,
111                int target_delay_ms,
112                int jitter_buffer_ms,
113                int min_playout_delay_ms,
114                int render_delay_ms),
115               (override));
116   MOCK_METHOD(void,
117               OnTimingFrameInfoUpdated,
118               (const TimingFrameInfo& info),
119               (override));
120 };
121 
122 }  // namespace
123 
124 constexpr auto kMaxWaitForKeyframe = TimeDelta::Millis(500);
125 constexpr auto kMaxWaitForFrame = TimeDelta::Millis(1500);
126 class VideoStreamBufferControllerFixture
127     : public ::testing::WithParamInterface<std::tuple<bool, std::string>>,
128       public FrameSchedulingReceiver {
129  public:
VideoStreamBufferControllerFixture()130   VideoStreamBufferControllerFixture()
131       : sync_decoding_(std::get<0>(GetParam())),
132         field_trials_(std::get<1>(GetParam())),
133         time_controller_(kClockStart),
134         clock_(time_controller_.GetClock()),
135         fake_metronome_(TimeDelta::Millis(16)),
136         decode_sync_(clock_,
137                      &fake_metronome_,
138                      time_controller_.GetMainThread()),
139         timing_(clock_, field_trials_),
140         buffer_(std::make_unique<VideoStreamBufferController>(
141             clock_,
142             time_controller_.GetMainThread(),
143             &timing_,
144             &stats_callback_,
145             this,
146             kMaxWaitForKeyframe,
147             kMaxWaitForFrame,
148             sync_decoding_ ? decode_sync_.CreateSynchronizedFrameScheduler()
149                            : std::make_unique<TaskQueueFrameDecodeScheduler>(
150                                  clock_,
151                                  time_controller_.GetMainThread()),
152             field_trials_)) {
153     // Avoid starting with negative render times.
154     timing_.set_min_playout_delay(TimeDelta::Millis(10));
155 
156     ON_CALL(stats_callback_, OnDroppedFrames)
157         .WillByDefault(
158             [this](auto num_dropped) { dropped_frames_ += num_dropped; });
159   }
160 
~VideoStreamBufferControllerFixture()161   ~VideoStreamBufferControllerFixture() override {
162     if (buffer_) {
163       buffer_->Stop();
164     }
165     time_controller_.AdvanceTime(TimeDelta::Zero());
166   }
167 
OnEncodedFrame(std::unique_ptr<EncodedFrame> frame)168   void OnEncodedFrame(std::unique_ptr<EncodedFrame> frame) override {
169     RTC_DCHECK(frame);
170     SetWaitResult(std::move(frame));
171   }
172 
OnDecodableFrameTimeout(TimeDelta wait_time)173   void OnDecodableFrameTimeout(TimeDelta wait_time) override {
174     SetWaitResult(wait_time);
175   }
176 
177   using WaitResult =
178       absl::variant<std::unique_ptr<EncodedFrame>, TimeDelta /*wait_time*/>;
179 
WaitForFrameOrTimeout(TimeDelta wait)180   absl::optional<WaitResult> WaitForFrameOrTimeout(TimeDelta wait) {
181     if (wait_result_) {
182       return std::move(wait_result_);
183     }
184     time_controller_.AdvanceTime(TimeDelta::Zero());
185     if (wait_result_) {
186       return std::move(wait_result_);
187     }
188 
189     Timestamp now = clock_->CurrentTime();
190     // TODO(bugs.webrtc.org/13756): Remove this when rtc::Thread uses uses
191     // Timestamp instead of an integer milliseconds. This extra wait is needed
192     // for some tests that use the metronome. This is due to rounding
193     // milliseconds, affecting the precision of simulated time controller uses
194     // when posting tasks from threads.
195     TimeDelta potential_extra_wait =
196         Timestamp::Millis((now + wait).ms()) - (now + wait);
197 
198     time_controller_.AdvanceTime(wait);
199     if (potential_extra_wait > TimeDelta::Zero()) {
200       time_controller_.AdvanceTime(potential_extra_wait);
201     }
202     return std::move(wait_result_);
203   }
204 
StartNextDecode()205   void StartNextDecode() {
206     ResetLastResult();
207     buffer_->StartNextDecode(false);
208   }
209 
StartNextDecodeForceKeyframe()210   void StartNextDecodeForceKeyframe() {
211     ResetLastResult();
212     buffer_->StartNextDecode(true);
213   }
214 
ResetLastResult()215   void ResetLastResult() { wait_result_.reset(); }
216 
dropped_frames() const217   int dropped_frames() const { return dropped_frames_; }
218 
219  protected:
220   const bool sync_decoding_;
221   test::ScopedKeyValueConfig field_trials_;
222   GlobalSimulatedTimeController time_controller_;
223   Clock* const clock_;
224   test::FakeMetronome fake_metronome_;
225   DecodeSynchronizer decode_sync_;
226 
227   ::testing::NiceMock<VCMTimingTest> timing_;
228   ::testing::NiceMock<VCMReceiveStatisticsCallbackMock> stats_callback_;
229   std::unique_ptr<VideoStreamBufferController> buffer_;
230 
231  private:
SetWaitResult(WaitResult result)232   void SetWaitResult(WaitResult result) {
233     RTC_DCHECK(!wait_result_);
234     if (absl::holds_alternative<std::unique_ptr<EncodedFrame>>(result)) {
235       RTC_DCHECK(absl::get<std::unique_ptr<EncodedFrame>>(result));
236     }
237     wait_result_.emplace(std::move(result));
238   }
239 
240   uint32_t dropped_frames_ = 0;
241   absl::optional<WaitResult> wait_result_;
242 };
243 
244 class VideoStreamBufferControllerTest
245     : public ::testing::Test,
246       public VideoStreamBufferControllerFixture {};
247 
TEST_P(VideoStreamBufferControllerTest,InitialTimeoutAfterKeyframeTimeoutPeriod)248 TEST_P(VideoStreamBufferControllerTest,
249        InitialTimeoutAfterKeyframeTimeoutPeriod) {
250   StartNextDecodeForceKeyframe();
251   // No frame inserted. Timeout expected.
252   EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForKeyframe), TimedOut());
253 
254   // No new timeout set since receiver has not started new decode.
255   ResetLastResult();
256   EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForKeyframe), Eq(absl::nullopt));
257 
258   // Now that receiver has asked for new frame, a new timeout can occur.
259   StartNextDecodeForceKeyframe();
260   EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForKeyframe), TimedOut());
261 }
262 
TEST_P(VideoStreamBufferControllerTest,KeyFramesAreScheduled)263 TEST_P(VideoStreamBufferControllerTest, KeyFramesAreScheduled) {
264   StartNextDecodeForceKeyframe();
265   time_controller_.AdvanceTime(TimeDelta::Millis(50));
266 
267   auto frame = test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build();
268   buffer_->InsertFrame(std::move(frame));
269 
270   EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0)));
271 }
272 
TEST_P(VideoStreamBufferControllerTest,DeltaFrameTimeoutAfterKeyframeExtracted)273 TEST_P(VideoStreamBufferControllerTest,
274        DeltaFrameTimeoutAfterKeyframeExtracted) {
275   StartNextDecodeForceKeyframe();
276 
277   time_controller_.AdvanceTime(TimeDelta::Millis(50));
278   auto frame = test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build();
279   buffer_->InsertFrame(std::move(frame));
280   EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForKeyframe),
281               Frame(test::WithId(0)));
282 
283   StartNextDecode();
284   time_controller_.AdvanceTime(TimeDelta::Millis(50));
285 
286   // Timeouts should now happen at the normal frequency.
287   const int expected_timeouts = 5;
288   for (int i = 0; i < expected_timeouts; ++i) {
289     EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForFrame), TimedOut());
290     StartNextDecode();
291   }
292 }
293 
TEST_P(VideoStreamBufferControllerTest,DependantFramesAreScheduled)294 TEST_P(VideoStreamBufferControllerTest, DependantFramesAreScheduled) {
295   StartNextDecodeForceKeyframe();
296   buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build());
297   EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0)));
298 
299   StartNextDecode();
300 
301   time_controller_.AdvanceTime(kFps30Delay);
302   buffer_->InsertFrame(test::FakeFrameBuilder()
303                            .Id(1)
304                            .Time(kFps30Rtp)
305                            .AsLast()
306                            .Refs({0})
307                            .Build());
308   EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(1)));
309 }
310 
TEST_P(VideoStreamBufferControllerTest,SpatialLayersAreScheduled)311 TEST_P(VideoStreamBufferControllerTest, SpatialLayersAreScheduled) {
312   StartNextDecodeForceKeyframe();
313   buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(
314       test::FakeFrameBuilder().Id(0).SpatialLayer(0).Time(0).Build()));
315   buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(
316       test::FakeFrameBuilder().Id(1).SpatialLayer(1).Time(0).Build()));
317   buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(
318       test::FakeFrameBuilder().Id(2).SpatialLayer(2).Time(0).AsLast().Build()));
319   EXPECT_THAT(
320       WaitForFrameOrTimeout(TimeDelta::Zero()),
321       Frame(AllOf(test::WithId(0), test::FrameWithSize(3 * kFrameSize))));
322 
323   buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(
324       test::FakeFrameBuilder().Id(3).Time(kFps30Rtp).SpatialLayer(0).Build()));
325   buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(
326       test::FakeFrameBuilder().Id(4).Time(kFps30Rtp).SpatialLayer(1).Build()));
327   buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(test::FakeFrameBuilder()
328                                                            .Id(5)
329                                                            .Time(kFps30Rtp)
330                                                            .SpatialLayer(2)
331                                                            .AsLast()
332                                                            .Build()));
333 
334   StartNextDecode();
335   EXPECT_THAT(
336       WaitForFrameOrTimeout(kFps30Delay * 10),
337       Frame(AllOf(test::WithId(3), test::FrameWithSize(3 * kFrameSize))));
338 }
339 
TEST_P(VideoStreamBufferControllerTest,OutstandingFrameTasksAreCancelledAfterDeletion)340 TEST_P(VideoStreamBufferControllerTest,
341        OutstandingFrameTasksAreCancelledAfterDeletion) {
342   StartNextDecodeForceKeyframe();
343   buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(
344       test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build()));
345   // Get keyframe. Delta frame should now be scheduled.
346   EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0)));
347 
348   StartNextDecode();
349   buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(test::FakeFrameBuilder()
350                                                            .Id(1)
351                                                            .Time(kFps30Rtp)
352                                                            .AsLast()
353                                                            .Refs({0})
354                                                            .Build()));
355   buffer_->Stop();
356   // Wait for 2x max wait time. Since we stopped, this should cause no timeouts
357   // or frame-ready callbacks.
358   EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForFrame * 2), Eq(absl::nullopt));
359 }
360 
TEST_P(VideoStreamBufferControllerTest,FramesWaitForDecoderToComplete)361 TEST_P(VideoStreamBufferControllerTest, FramesWaitForDecoderToComplete) {
362   StartNextDecodeForceKeyframe();
363 
364   // Start with a keyframe.
365   buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build());
366   EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0)));
367 
368   ResetLastResult();
369   // Insert a delta frame.
370   buffer_->InsertFrame(test::FakeFrameBuilder()
371                            .Id(1)
372                            .Time(kFps30Rtp)
373                            .AsLast()
374                            .Refs({0})
375                            .Build());
376 
377   // Advancing time should not result in a frame since the scheduler has not
378   // been signalled that we are ready.
379   EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Eq(absl::nullopt));
380   // Signal ready.
381   StartNextDecode();
382   EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(1)));
383 }
384 
TEST_P(VideoStreamBufferControllerTest,LateFrameDropped)385 TEST_P(VideoStreamBufferControllerTest, LateFrameDropped) {
386   StartNextDecodeForceKeyframe();
387   //   F1
388   //   /
389   // F0 --> F2
390   buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build());
391   // Start with a keyframe.
392   EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0)));
393 
394   StartNextDecode();
395 
396   // Simulate late F1 which arrives after F2.
397   time_controller_.AdvanceTime(kFps30Delay * 2);
398   buffer_->InsertFrame(test::FakeFrameBuilder()
399                            .Id(2)
400                            .Time(2 * kFps30Rtp)
401                            .AsLast()
402                            .Refs({0})
403                            .Build());
404   EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(2)));
405 
406   StartNextDecode();
407 
408   buffer_->InsertFrame(test::FakeFrameBuilder()
409                            .Id(1)
410                            .Time(1 * kFps30Rtp)
411                            .AsLast()
412                            .Refs({0})
413                            .Build());
414   // Confirm frame 1 is never scheduled by timing out.
415   EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForFrame), TimedOut());
416 }
417 
TEST_P(VideoStreamBufferControllerTest,FramesFastForwardOnSystemHalt)418 TEST_P(VideoStreamBufferControllerTest, FramesFastForwardOnSystemHalt) {
419   StartNextDecodeForceKeyframe();
420   //   F1
421   //   /
422   // F0 --> F2
423   buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build());
424 
425   // Start with a keyframe.
426   EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0)));
427 
428   time_controller_.AdvanceTime(kFps30Delay);
429   buffer_->InsertFrame(test::FakeFrameBuilder()
430                            .Id(1)
431                            .Time(kFps30Rtp)
432                            .AsLast()
433                            .Refs({0})
434                            .Build());
435   time_controller_.AdvanceTime(kFps30Delay);
436   buffer_->InsertFrame(test::FakeFrameBuilder()
437                            .Id(2)
438                            .Time(2 * kFps30Rtp)
439                            .AsLast()
440                            .Refs({0})
441                            .Build());
442 
443   // Halting time should result in F1 being skipped.
444   time_controller_.AdvanceTime(kFps30Delay * 2);
445   StartNextDecode();
446   EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(2)));
447   EXPECT_EQ(dropped_frames(), 1);
448 }
449 
TEST_P(VideoStreamBufferControllerTest,ForceKeyFrame)450 TEST_P(VideoStreamBufferControllerTest, ForceKeyFrame) {
451   StartNextDecodeForceKeyframe();
452   // Initial keyframe.
453   buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build());
454   EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0)));
455 
456   StartNextDecodeForceKeyframe();
457 
458   // F2 is the next keyframe, and should be extracted since a keyframe was
459   // forced.
460   buffer_->InsertFrame(test::FakeFrameBuilder()
461                            .Id(1)
462                            .Time(kFps30Rtp)
463                            .AsLast()
464                            .Refs({0})
465                            .Build());
466   buffer_->InsertFrame(
467       test::FakeFrameBuilder().Id(2).Time(kFps30Rtp * 2).AsLast().Build());
468 
469   EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay * 3), Frame(test::WithId(2)));
470 }
471 
TEST_P(VideoStreamBufferControllerTest,SlowDecoderDropsTemporalLayers)472 TEST_P(VideoStreamBufferControllerTest, SlowDecoderDropsTemporalLayers) {
473   StartNextDecodeForceKeyframe();
474   // 2 temporal layers, at 15fps per layer to make 30fps total.
475   // Decoder is slower than 30fps, so last_frame() will be skipped.
476   //   F1 --> F3 --> F5
477   //   /      /     /
478   // F0 --> F2 --> F4
479   buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build());
480   // Keyframe received.
481   // Don't start next decode until slow delay.
482   EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0)));
483 
484   time_controller_.AdvanceTime(kFps30Delay);
485   buffer_->InsertFrame(test::FakeFrameBuilder()
486                            .Id(1)
487                            .Time(1 * kFps30Rtp)
488                            .Refs({0})
489                            .AsLast()
490                            .Build());
491   time_controller_.AdvanceTime(kFps30Delay);
492   buffer_->InsertFrame(test::FakeFrameBuilder()
493                            .Id(2)
494                            .Time(2 * kFps30Rtp)
495                            .Refs({0})
496                            .AsLast()
497                            .Build());
498 
499   // Simulate decode taking 3x FPS rate.
500   time_controller_.AdvanceTime(kFps30Delay * 1.5);
501   StartNextDecode();
502   // F2 is the best frame since decoding was so slow that F1 is too old.
503   EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay * 2), Frame(test::WithId(2)));
504   EXPECT_EQ(dropped_frames(), 1);
505   time_controller_.AdvanceTime(kFps30Delay / 2);
506 
507   buffer_->InsertFrame(test::FakeFrameBuilder()
508                            .Id(3)
509                            .Time(3 * kFps30Rtp)
510                            .Refs({1, 2})
511                            .AsLast()
512                            .Build());
513   time_controller_.AdvanceTime(kFps30Delay / 2);
514   buffer_->InsertFrame(test::FakeFrameBuilder()
515                            .Id(4)
516                            .Time(4 * kFps30Rtp)
517                            .Refs({2})
518                            .AsLast()
519                            .Build());
520   time_controller_.AdvanceTime(kFps30Delay / 2);
521 
522   // F4 is the best frame since decoding was so slow that F1 is too old.
523   time_controller_.AdvanceTime(kFps30Delay);
524   StartNextDecode();
525   EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(4)));
526 
527   buffer_->InsertFrame(test::FakeFrameBuilder()
528                            .Id(5)
529                            .Time(5 * kFps30Rtp)
530                            .Refs({3, 4})
531                            .AsLast()
532                            .Build());
533   time_controller_.AdvanceTime(kFps30Delay / 2);
534 
535   // F5 is not decodable since F4 was decoded, so a timeout is expected.
536   time_controller_.AdvanceTime(TimeDelta::Millis(10));
537   StartNextDecode();
538   EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForFrame), TimedOut());
539   // TODO(bugs.webrtc.org/13343): This should be 2 dropped frames since frames 1
540   // and 3 were dropped. However, frame_buffer2 does not mark frame 3 as dropped
541   // which is a bug. Uncomment below when that is fixed for frame_buffer2 is
542   // deleted.
543   // EXPECT_EQ(dropped_frames(), 2);
544 }
545 
TEST_P(VideoStreamBufferControllerTest,NewFrameInsertedWhileWaitingToReleaseFrame)546 TEST_P(VideoStreamBufferControllerTest,
547        NewFrameInsertedWhileWaitingToReleaseFrame) {
548   StartNextDecodeForceKeyframe();
549   // Initial keyframe.
550   buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(
551       test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build()));
552   EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0)));
553 
554   time_controller_.AdvanceTime(kFps30Delay / 2);
555   buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(test::FakeFrameBuilder()
556                                                            .Id(1)
557                                                            .Time(kFps30Rtp)
558                                                            .Refs({0})
559                                                            .AsLast()
560                                                            .Build()));
561   StartNextDecode();
562   EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Eq(absl::nullopt));
563 
564   // Scheduler is waiting to deliver Frame 1 now. Insert Frame 2. Frame 1 should
565   // be delivered still.
566   buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(test::FakeFrameBuilder()
567                                                            .Id(2)
568                                                            .Time(kFps30Rtp * 2)
569                                                            .Refs({0})
570                                                            .AsLast()
571                                                            .Build()));
572   EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(1)));
573 }
574 
TEST_P(VideoStreamBufferControllerTest,SameFrameNotScheduledTwice)575 TEST_P(VideoStreamBufferControllerTest, SameFrameNotScheduledTwice) {
576   // A frame could be scheduled twice if last_frame() arrive out-of-order but
577   // the older frame is old enough to be fast forwarded.
578   //
579   // 1. F2 arrives and is scheduled.
580   // 2. F3 arrives, but scheduling will not change since F2 is next.
581   // 3. F1 arrives late and scheduling is checked since it is before F2. F1
582   // fast-forwarded since it is older.
583   //
584   // F2 is the best frame, but should only be scheduled once, followed by F3.
585   StartNextDecodeForceKeyframe();
586 
587   // First keyframe.
588   buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(
589       test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build()));
590   EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Millis(15)),
591               Frame(test::WithId(0)));
592 
593   StartNextDecode();
594 
595   // F2 arrives and is scheduled.
596   buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(
597       test::FakeFrameBuilder().Id(2).Time(2 * kFps30Rtp).AsLast().Build()));
598 
599   // F3 arrives before F2 is extracted.
600   time_controller_.AdvanceTime(kFps30Delay);
601   buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(
602       test::FakeFrameBuilder().Id(3).Time(3 * kFps30Rtp).AsLast().Build()));
603 
604   // F1 arrives and is fast-forwarded since it is too late.
605   // F2 is already scheduled and should not be rescheduled.
606   time_controller_.AdvanceTime(kFps30Delay / 2);
607   buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(
608       test::FakeFrameBuilder().Id(1).Time(1 * kFps30Rtp).AsLast().Build()));
609 
610   EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(2)));
611   StartNextDecode();
612 
613   EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(3)));
614   StartNextDecode();
615   EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForFrame), TimedOut());
616   EXPECT_EQ(dropped_frames(), 1);
617 }
618 
TEST_P(VideoStreamBufferControllerTest,TestStatsCallback)619 TEST_P(VideoStreamBufferControllerTest, TestStatsCallback) {
620   EXPECT_CALL(stats_callback_,
621               OnCompleteFrame(true, kFrameSize, VideoContentType::UNSPECIFIED));
622   EXPECT_CALL(stats_callback_, OnFrameBufferTimingsUpdated);
623 
624   // Fake timing having received decoded frame.
625   timing_.StopDecodeTimer(TimeDelta::Millis(1), clock_->CurrentTime());
626   StartNextDecodeForceKeyframe();
627   buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build());
628   EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0)));
629 
630   // Flush stats posted on the decode queue.
631   time_controller_.AdvanceTime(TimeDelta::Zero());
632 }
633 
TEST_P(VideoStreamBufferControllerTest,FrameCompleteCalledOnceForDuplicateFrame)634 TEST_P(VideoStreamBufferControllerTest,
635        FrameCompleteCalledOnceForDuplicateFrame) {
636   EXPECT_CALL(stats_callback_,
637               OnCompleteFrame(true, kFrameSize, VideoContentType::UNSPECIFIED))
638       .Times(1);
639 
640   StartNextDecodeForceKeyframe();
641   buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build());
642   buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build());
643   // Flush stats posted on the decode queue.
644   time_controller_.AdvanceTime(TimeDelta::Zero());
645 }
646 
TEST_P(VideoStreamBufferControllerTest,FrameCompleteCalledOnceForSingleTemporalUnit)647 TEST_P(VideoStreamBufferControllerTest,
648        FrameCompleteCalledOnceForSingleTemporalUnit) {
649   StartNextDecodeForceKeyframe();
650 
651   // `OnCompleteFrame` should not be called for the first two frames since they
652   // do not complete the temporal layer.
653   EXPECT_CALL(stats_callback_, OnCompleteFrame(_, _, _)).Times(0);
654   buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).Build());
655   buffer_->InsertFrame(
656       test::FakeFrameBuilder().Id(1).Time(0).Refs({0}).Build());
657   time_controller_.AdvanceTime(TimeDelta::Zero());
658   // Flush stats posted on the decode queue.
659   ::testing::Mock::VerifyAndClearExpectations(&stats_callback_);
660 
661   // Note that this frame is not marked as a keyframe since the last spatial
662   // layer has dependencies.
663   EXPECT_CALL(stats_callback_,
664               OnCompleteFrame(false, kFrameSize, VideoContentType::UNSPECIFIED))
665       .Times(1);
666   buffer_->InsertFrame(
667       test::FakeFrameBuilder().Id(2).Time(0).Refs({0, 1}).AsLast().Build());
668   // Flush stats posted on the decode queue.
669   time_controller_.AdvanceTime(TimeDelta::Zero());
670 }
671 
TEST_P(VideoStreamBufferControllerTest,FrameCompleteCalledOnceForCompleteTemporalUnit)672 TEST_P(VideoStreamBufferControllerTest,
673        FrameCompleteCalledOnceForCompleteTemporalUnit) {
674   // FrameBuffer2 logs the complete frame on the arrival of the last layer.
675   StartNextDecodeForceKeyframe();
676 
677   // `OnCompleteFrame` should not be called for the first two frames since they
678   // do not complete the temporal layer. Frame 1 arrives later, at which time
679   // this frame can finally be considered complete.
680   EXPECT_CALL(stats_callback_, OnCompleteFrame(_, _, _)).Times(0);
681   buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).Build());
682   buffer_->InsertFrame(
683       test::FakeFrameBuilder().Id(2).Time(0).Refs({0, 1}).AsLast().Build());
684   time_controller_.AdvanceTime(TimeDelta::Zero());
685   // Flush stats posted on the decode queue.
686   ::testing::Mock::VerifyAndClearExpectations(&stats_callback_);
687 
688   EXPECT_CALL(stats_callback_,
689               OnCompleteFrame(false, kFrameSize, VideoContentType::UNSPECIFIED))
690       .Times(1);
691   buffer_->InsertFrame(
692       test::FakeFrameBuilder().Id(1).Time(0).Refs({0}).Build());
693   // Flush stats posted on the decode queue.
694   time_controller_.AdvanceTime(TimeDelta::Zero());
695 }
696 
697 // Note: This test takes a long time to run if the fake metronome is active.
698 // Since the test needs to wait for the timestamp to rollover, it has a fake
699 // delay of around 6.5 hours. Even though time is simulated, this will be
700 // around 1,500,000 metronome tick invocations.
TEST_P(VideoStreamBufferControllerTest,NextFrameWithOldTimestamp)701 TEST_P(VideoStreamBufferControllerTest, NextFrameWithOldTimestamp) {
702   // Test inserting 31 frames and pause the stream for a long time before
703   // frame 32.
704   StartNextDecodeForceKeyframe();
705   constexpr uint32_t kBaseRtp = std::numeric_limits<uint32_t>::max() / 2;
706 
707   // First keyframe. The receive time must be explicitly set in this test since
708   // the RTP derived time used in all tests does not work when the long pause
709   // happens later in the test.
710   buffer_->InsertFrame(test::FakeFrameBuilder()
711                            .Id(0)
712                            .Time(kBaseRtp)
713                            .ReceivedTime(clock_->CurrentTime())
714                            .AsLast()
715                            .Build());
716   EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(0)));
717 
718   // 1 more frame to warmup VCMTiming for 30fps.
719   StartNextDecode();
720   buffer_->InsertFrame(test::FakeFrameBuilder()
721                            .Id(1)
722                            .Time(kBaseRtp + kFps30Rtp)
723                            .ReceivedTime(clock_->CurrentTime())
724                            .AsLast()
725                            .Build());
726   EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(1)));
727 
728   // Pause the stream for such a long time it incurs an RTP timestamp rollover
729   // by over half.
730   constexpr uint32_t kLastRtp = kBaseRtp + kFps30Rtp;
731   constexpr uint32_t kRolloverRtp =
732       kLastRtp + std::numeric_limits<uint32_t>::max() / 2 + 1;
733   constexpr Frequency kRtpHz = Frequency::KiloHertz(90);
734   // Pause for corresponding delay such that RTP timestamp would increase this
735   // much at 30fps.
736   constexpr TimeDelta kRolloverDelay =
737       (std::numeric_limits<uint32_t>::max() / 2 + 1) / kRtpHz;
738 
739   // Avoid timeout being set while waiting for the frame and before the receiver
740   // is ready.
741   ResetLastResult();
742   EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForFrame), Eq(absl::nullopt));
743   time_controller_.AdvanceTime(kRolloverDelay - kMaxWaitForFrame);
744   StartNextDecode();
745   buffer_->InsertFrame(test::FakeFrameBuilder()
746                            .Id(2)
747                            .Time(kRolloverRtp)
748                            .ReceivedTime(clock_->CurrentTime())
749                            .AsLast()
750                            .Build());
751   // FrameBuffer2 drops the frame, while FrameBuffer3 will continue the stream.
752   EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(2)));
753 }
754 
TEST_P(VideoStreamBufferControllerTest,FrameNotSetForDecodedIfFrameBufferBecomesNonDecodable)755 TEST_P(VideoStreamBufferControllerTest,
756        FrameNotSetForDecodedIfFrameBufferBecomesNonDecodable) {
757   // This can happen if the frame buffer receives non-standard input. This test
758   // will simply clear the frame buffer to replicate this.
759   StartNextDecodeForceKeyframe();
760   // Initial keyframe.
761   buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(
762       test::FakeFrameBuilder().Id(0).Time(0).SpatialLayer(1).AsLast().Build()));
763   EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0)));
764 
765   // Insert a frame that will become non-decodable.
766   buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(test::FakeFrameBuilder()
767                                                            .Id(11)
768                                                            .Time(kFps30Rtp)
769                                                            .Refs({0})
770                                                            .SpatialLayer(1)
771                                                            .AsLast()
772                                                            .Build()));
773   StartNextDecode();
774   // Second layer inserted after last layer for the same frame out-of-order.
775   // This second frame requires some older frame to be decoded and so now the
776   // super-frame is no longer decodable despite already being scheduled.
777   buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(test::FakeFrameBuilder()
778                                                            .Id(10)
779                                                            .Time(kFps30Rtp)
780                                                            .SpatialLayer(0)
781                                                            .Refs({2})
782                                                            .Build()));
783   EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForFrame), TimedOut());
784 
785   // Ensure that this frame can be decoded later.
786   StartNextDecode();
787   buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(test::FakeFrameBuilder()
788                                                            .Id(2)
789                                                            .Time(kFps30Rtp / 2)
790                                                            .SpatialLayer(0)
791                                                            .Refs({0})
792                                                            .AsLast()
793                                                            .Build()));
794   EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(2)));
795   StartNextDecode();
796   EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(10)));
797 }
798 
799 INSTANTIATE_TEST_SUITE_P(VideoStreamBufferController,
800                          VideoStreamBufferControllerTest,
801                          ::testing::Combine(::testing::Bool(),
802                                             ::testing::Values("")),
__anone543b9140302(const auto& info) 803                          [](const auto& info) {
804                            return std::get<0>(info.param) ? "SyncDecoding"
805                                                           : "UnsyncedDecoding";
806                          });
807 
808 class LowLatencyVideoStreamBufferControllerTest
809     : public ::testing::Test,
810       public VideoStreamBufferControllerFixture {};
811 
TEST_P(LowLatencyVideoStreamBufferControllerTest,FramesDecodedInstantlyWithLowLatencyRendering)812 TEST_P(LowLatencyVideoStreamBufferControllerTest,
813        FramesDecodedInstantlyWithLowLatencyRendering) {
814   // Initial keyframe.
815   StartNextDecodeForceKeyframe();
816   timing_.set_min_playout_delay(TimeDelta::Zero());
817   timing_.set_max_playout_delay(TimeDelta::Millis(10));
818   auto frame = test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build();
819   // Playout delay of 0 implies low-latency rendering.
820   frame->SetPlayoutDelay({0, 10});
821   buffer_->InsertFrame(std::move(frame));
822   EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0)));
823 
824   // Delta frame would normally wait here, but should decode at the pacing rate
825   // in low-latency mode.
826   StartNextDecode();
827   frame = test::FakeFrameBuilder().Id(1).Time(kFps30Rtp).AsLast().Build();
828   frame->SetPlayoutDelay({0, 10});
829   buffer_->InsertFrame(std::move(frame));
830   // Pacing is set to 16ms in the field trial so we should not decode yet.
831   EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Eq(absl::nullopt));
832   time_controller_.AdvanceTime(TimeDelta::Millis(16));
833   EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(1)));
834 }
835 
TEST_P(LowLatencyVideoStreamBufferControllerTest,ZeroPlayoutDelayFullQueue)836 TEST_P(LowLatencyVideoStreamBufferControllerTest, ZeroPlayoutDelayFullQueue) {
837   // Initial keyframe.
838   StartNextDecodeForceKeyframe();
839   timing_.set_min_playout_delay(TimeDelta::Zero());
840   timing_.set_max_playout_delay(TimeDelta::Millis(10));
841   auto frame = test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build();
842   // Playout delay of 0 implies low-latency rendering.
843   frame->SetPlayoutDelay({0, 10});
844   buffer_->InsertFrame(std::move(frame));
845   EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0)));
846 
847   // Queue up 5 frames (configured max queue size for 0-playout delay pacing).
848   for (int id = 1; id <= 6; ++id) {
849     frame =
850         test::FakeFrameBuilder().Id(id).Time(kFps30Rtp * id).AsLast().Build();
851     frame->SetPlayoutDelay({0, 10});
852     buffer_->InsertFrame(std::move(frame));
853   }
854 
855   // The queue is at its max size for zero playout delay pacing, so the pacing
856   // should be ignored and the next frame should be decoded instantly.
857   StartNextDecode();
858   EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(1)));
859 }
860 
TEST_P(LowLatencyVideoStreamBufferControllerTest,MinMaxDelayZeroLowLatencyMode)861 TEST_P(LowLatencyVideoStreamBufferControllerTest,
862        MinMaxDelayZeroLowLatencyMode) {
863   // Initial keyframe.
864   StartNextDecodeForceKeyframe();
865   timing_.set_min_playout_delay(TimeDelta::Zero());
866   timing_.set_max_playout_delay(TimeDelta::Zero());
867   auto frame = test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build();
868   // Playout delay of 0 implies low-latency rendering.
869   frame->SetPlayoutDelay({0, 0});
870   buffer_->InsertFrame(std::move(frame));
871   EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0)));
872 
873   // Delta frame would normally wait here, but should decode at the pacing rate
874   // in low-latency mode.
875   StartNextDecode();
876   frame = test::FakeFrameBuilder().Id(1).Time(kFps30Rtp).AsLast().Build();
877   frame->SetPlayoutDelay({0, 0});
878   buffer_->InsertFrame(std::move(frame));
879   // The min/max=0 version of low-latency rendering will result in a large
880   // negative decode wait time, so the frame should be ready right away.
881   EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(1)));
882 }
883 
884 INSTANTIATE_TEST_SUITE_P(
885     VideoStreamBufferController,
886     LowLatencyVideoStreamBufferControllerTest,
887     ::testing::Combine(
888         ::testing::Bool(),
889         ::testing::Values(
890             "WebRTC-ZeroPlayoutDelay/min_pacing:16ms,max_decode_queue_size:5/",
891             "WebRTC-ZeroPlayoutDelay/"
892             "min_pacing:16ms,max_decode_queue_size:5/")));
893 
894 class IncomingTimestampVideoStreamBufferControllerTest
895     : public ::testing::Test,
896       public VideoStreamBufferControllerFixture {};
897 
TEST_P(IncomingTimestampVideoStreamBufferControllerTest,IncomingTimestampOnMarkerBitOnly)898 TEST_P(IncomingTimestampVideoStreamBufferControllerTest,
899        IncomingTimestampOnMarkerBitOnly) {
900   StartNextDecodeForceKeyframe();
901   EXPECT_CALL(timing_, IncomingTimestampMocked)
902       .Times(field_trials_.IsDisabled("WebRTC-IncomingTimestampOnMarkerBitOnly")
903                  ? 3
904                  : 1);
905   buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(
906       test::FakeFrameBuilder().Id(0).SpatialLayer(0).Time(0).Build()));
907   buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(
908       test::FakeFrameBuilder().Id(1).SpatialLayer(1).Time(0).Build()));
909   buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(
910       test::FakeFrameBuilder().Id(2).SpatialLayer(2).Time(0).AsLast().Build()));
911 }
912 
913 INSTANTIATE_TEST_SUITE_P(
914     VideoStreamBufferController,
915     IncomingTimestampVideoStreamBufferControllerTest,
916     ::testing::Combine(
917         ::testing::Bool(),
918         ::testing::Values(
919             "WebRTC-IncomingTimestampOnMarkerBitOnly/Enabled/",
920             "WebRTC-IncomingTimestampOnMarkerBitOnly/Disabled/")));
921 
922 }  // namespace webrtc
923