xref: /aosp_15_r20/external/webrtc/modules/video_coding/receiver_unittest.cc (revision d9f758449e529ab9291ac668be2861e7a55c2422)

/*  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include "modules/video_coding/receiver.h"

#include <string.h>

#include <cstdint>
#include <memory>
#include <queue>
#include <vector>

#include "modules/video_coding/encoded_frame.h"
#include "modules/video_coding/jitter_buffer_common.h"
#include "modules/video_coding/packet.h"
#include "modules/video_coding/test/stream_generator.h"
#include "modules/video_coding/timing/timing.h"
#include "rtc_base/checks.h"
#include "system_wrappers/include/clock.h"
#include "test/gtest.h"
#include "test/scoped_key_value_config.h"

namespace webrtc {

class TestVCMReceiver : public ::testing::Test {
 protected:
  TestVCMReceiver()
      : clock_(0),
        timing_(&clock_, field_trials_),
        receiver_(&timing_, &clock_, field_trials_),
        stream_generator_(0, clock_.TimeInMilliseconds()) {}

  int32_t InsertPacket(int index) {
    VCMPacket packet;
    bool packet_available = stream_generator_.GetPacket(&packet, index);
    EXPECT_TRUE(packet_available);
    if (!packet_available)
      return kGeneralError;  // Return here to avoid crashes below.
    return receiver_.InsertPacket(packet);
  }

  int32_t InsertPacketAndPop(int index) {
    VCMPacket packet;
    bool packet_available = stream_generator_.PopPacket(&packet, index);
    EXPECT_TRUE(packet_available);
    if (!packet_available)
      return kGeneralError;  // Return here to avoid crashes below.
    return receiver_.InsertPacket(packet);
  }

  int32_t InsertFrame(VideoFrameType frame_type, bool complete) {
    int num_of_packets = complete ? 1 : 2;
    stream_generator_.GenerateFrame(
        frame_type,
        (frame_type != VideoFrameType::kEmptyFrame) ? num_of_packets : 0,
        (frame_type == VideoFrameType::kEmptyFrame) ? 1 : 0,
        clock_.TimeInMilliseconds());
    int32_t ret = InsertPacketAndPop(0);
    if (!complete) {
      // Drop the second packet.
      VCMPacket packet;
      stream_generator_.PopPacket(&packet, 0);
    }
    clock_.AdvanceTimeMilliseconds(kDefaultFramePeriodMs);
    return ret;
  }

  bool DecodeNextFrame() {
    VCMEncodedFrame* frame = receiver_.FrameForDecoding(0, false);
    if (!frame)
      return false;
    receiver_.ReleaseFrame(frame);
    return true;
  }

  test::ScopedKeyValueConfig field_trials_;
  SimulatedClock clock_;
  VCMTiming timing_;
  VCMReceiver receiver_;
  StreamGenerator stream_generator_;
};

TEST_F(TestVCMReceiver, NonDecodableDuration_Empty) {
  const size_t kMaxNackListSize = 1000;
  const int kMaxPacketAgeToNack = 1000;
  const int kMaxNonDecodableDuration = 500;
  const int kMinDelayMs = 500;
  receiver_.SetNackSettings(kMaxNackListSize, kMaxPacketAgeToNack,
                            kMaxNonDecodableDuration);
  EXPECT_GE(InsertFrame(VideoFrameType::kVideoFrameKey, true), kNoError);
  // Advance time until it's time to decode the key frame.
  clock_.AdvanceTimeMilliseconds(kMinDelayMs);
  EXPECT_TRUE(DecodeNextFrame());
  bool request_key_frame = false;
  std::vector<uint16_t> nack_list = receiver_.NackList(&request_key_frame);
  EXPECT_FALSE(request_key_frame);
}

TEST_F(TestVCMReceiver, NonDecodableDuration_NoKeyFrame) {
  const size_t kMaxNackListSize = 1000;
  const int kMaxPacketAgeToNack = 1000;
  const int kMaxNonDecodableDuration = 500;
  receiver_.SetNackSettings(kMaxNackListSize, kMaxPacketAgeToNack,
                            kMaxNonDecodableDuration);
  const int kNumFrames = kDefaultFrameRate * kMaxNonDecodableDuration / 1000;
  for (int i = 0; i < kNumFrames; ++i) {
    EXPECT_GE(InsertFrame(VideoFrameType::kVideoFrameDelta, true), kNoError);
  }
  bool request_key_frame = false;
  std::vector<uint16_t> nack_list = receiver_.NackList(&request_key_frame);
  EXPECT_TRUE(request_key_frame);
}

TEST_F(TestVCMReceiver, NonDecodableDuration_OneIncomplete) {
  const size_t kMaxNackListSize = 1000;
  const int kMaxPacketAgeToNack = 1000;
  const int kMaxNonDecodableDuration = 500;
  const int kMaxNonDecodableDurationFrames =
      (kDefaultFrameRate * kMaxNonDecodableDuration + 500) / 1000;
  const int kMinDelayMs = 500;
  receiver_.SetNackSettings(kMaxNackListSize, kMaxPacketAgeToNack,
                            kMaxNonDecodableDuration);
  timing_.set_min_playout_delay(TimeDelta::Millis(kMinDelayMs));
  int64_t key_frame_inserted = clock_.TimeInMilliseconds();
  EXPECT_GE(InsertFrame(VideoFrameType::kVideoFrameKey, true), kNoError);
  // Insert an incomplete frame.
  EXPECT_GE(InsertFrame(VideoFrameType::kVideoFrameDelta, false), kNoError);
  // Insert enough frames to have too long non-decodable sequence.
  for (int i = 0; i < kMaxNonDecodableDurationFrames; ++i) {
    EXPECT_GE(InsertFrame(VideoFrameType::kVideoFrameDelta, true), kNoError);
  }
  // Advance time until it's time to decode the key frame.
  clock_.AdvanceTimeMilliseconds(kMinDelayMs - clock_.TimeInMilliseconds() -
                                 key_frame_inserted);
  EXPECT_TRUE(DecodeNextFrame());
  // Make sure we get a key frame request.
  bool request_key_frame = false;
  std::vector<uint16_t> nack_list = receiver_.NackList(&request_key_frame);
  EXPECT_TRUE(request_key_frame);
}

TEST_F(TestVCMReceiver, NonDecodableDuration_NoTrigger) {
  const size_t kMaxNackListSize = 1000;
  const int kMaxPacketAgeToNack = 1000;
  const int kMaxNonDecodableDuration = 500;
  const int kMaxNonDecodableDurationFrames =
      (kDefaultFrameRate * kMaxNonDecodableDuration + 500) / 1000;
  const int kMinDelayMs = 500;
  receiver_.SetNackSettings(kMaxNackListSize, kMaxPacketAgeToNack,
                            kMaxNonDecodableDuration);
  timing_.set_min_playout_delay(TimeDelta::Millis(kMinDelayMs));
  int64_t key_frame_inserted = clock_.TimeInMilliseconds();
  EXPECT_GE(InsertFrame(VideoFrameType::kVideoFrameKey, true), kNoError);
  // Insert an incomplete frame.
  EXPECT_GE(InsertFrame(VideoFrameType::kVideoFrameDelta, false), kNoError);
  // Insert all but one frame to not trigger a key frame request due to
  // too long duration of non-decodable frames.
  for (int i = 0; i < kMaxNonDecodableDurationFrames - 1; ++i) {
    EXPECT_GE(InsertFrame(VideoFrameType::kVideoFrameDelta, true), kNoError);
  }
  // Advance time until it's time to decode the key frame.
  clock_.AdvanceTimeMilliseconds(kMinDelayMs - clock_.TimeInMilliseconds() -
                                 key_frame_inserted);
  EXPECT_TRUE(DecodeNextFrame());
  // Make sure we don't get a key frame request since we haven't generated
  // enough frames.
  bool request_key_frame = false;
  std::vector<uint16_t> nack_list = receiver_.NackList(&request_key_frame);
  EXPECT_FALSE(request_key_frame);
}

TEST_F(TestVCMReceiver, NonDecodableDuration_NoTrigger2) {
  const size_t kMaxNackListSize = 1000;
  const int kMaxPacketAgeToNack = 1000;
  const int kMaxNonDecodableDuration = 500;
  const int kMaxNonDecodableDurationFrames =
      (kDefaultFrameRate * kMaxNonDecodableDuration + 500) / 1000;
  const int kMinDelayMs = 500;
  receiver_.SetNackSettings(kMaxNackListSize, kMaxPacketAgeToNack,
                            kMaxNonDecodableDuration);
  timing_.set_min_playout_delay(TimeDelta::Millis(kMinDelayMs));
  int64_t key_frame_inserted = clock_.TimeInMilliseconds();
  EXPECT_GE(InsertFrame(VideoFrameType::kVideoFrameKey, true), kNoError);
  // Insert enough frames to have too long non-decodable sequence, except that
  // we don't have any losses.
  for (int i = 0; i < kMaxNonDecodableDurationFrames; ++i) {
    EXPECT_GE(InsertFrame(VideoFrameType::kVideoFrameDelta, true), kNoError);
  }
  // Insert an incomplete frame.
  EXPECT_GE(InsertFrame(VideoFrameType::kVideoFrameDelta, false), kNoError);
  // Advance time until it's time to decode the key frame.
  clock_.AdvanceTimeMilliseconds(kMinDelayMs - clock_.TimeInMilliseconds() -
                                 key_frame_inserted);
  EXPECT_TRUE(DecodeNextFrame());
  // Make sure we don't get a key frame request since the non-decodable duration
  // is only one frame.
  bool request_key_frame = false;
  std::vector<uint16_t> nack_list = receiver_.NackList(&request_key_frame);
  EXPECT_FALSE(request_key_frame);
}

TEST_F(TestVCMReceiver, NonDecodableDuration_KeyFrameAfterIncompleteFrames) {
  const size_t kMaxNackListSize = 1000;
  const int kMaxPacketAgeToNack = 1000;
  const int kMaxNonDecodableDuration = 500;
  const int kMaxNonDecodableDurationFrames =
      (kDefaultFrameRate * kMaxNonDecodableDuration + 500) / 1000;
  const int kMinDelayMs = 500;
  receiver_.SetNackSettings(kMaxNackListSize, kMaxPacketAgeToNack,
                            kMaxNonDecodableDuration);
  timing_.set_min_playout_delay(TimeDelta::Millis(kMinDelayMs));
  int64_t key_frame_inserted = clock_.TimeInMilliseconds();
  EXPECT_GE(InsertFrame(VideoFrameType::kVideoFrameKey, true), kNoError);
  // Insert an incomplete frame.
  EXPECT_GE(InsertFrame(VideoFrameType::kVideoFrameDelta, false), kNoError);
  // Insert enough frames to have too long non-decodable sequence.
  for (int i = 0; i < kMaxNonDecodableDurationFrames; ++i) {
    EXPECT_GE(InsertFrame(VideoFrameType::kVideoFrameDelta, true), kNoError);
  }
  EXPECT_GE(InsertFrame(VideoFrameType::kVideoFrameKey, true), kNoError);
  // Advance time until it's time to decode the key frame.
  clock_.AdvanceTimeMilliseconds(kMinDelayMs - clock_.TimeInMilliseconds() -
                                 key_frame_inserted);
  EXPECT_TRUE(DecodeNextFrame());
  // Make sure we don't get a key frame request since we have a key frame
  // in the list.
  bool request_key_frame = false;
  std::vector<uint16_t> nack_list = receiver_.NackList(&request_key_frame);
  EXPECT_FALSE(request_key_frame);
}

// A simulated clock, when time elapses, will insert frames into the jitter
// buffer, based on initial settings.
class SimulatedClockWithFrames : public SimulatedClock {
 public:
  SimulatedClockWithFrames(StreamGenerator* stream_generator,
                           VCMReceiver* receiver)
      : SimulatedClock(0),
        stream_generator_(stream_generator),
        receiver_(receiver) {}
  virtual ~SimulatedClockWithFrames() {}

  // If `stop_on_frame` is true and next frame arrives between now and
  // now+`milliseconds`, the clock will be advanced to the arrival time of next
  // frame.
  // Otherwise, the clock will be advanced by `milliseconds`.
  //
  // For both cases, a frame will be inserted into the jitter buffer at the
  // instant when the clock time is timestamps_.front().arrive_time.
  //
  // Return true if some frame arrives between now and now+`milliseconds`.
  bool AdvanceTimeMilliseconds(int64_t milliseconds, bool stop_on_frame) {
    return AdvanceTimeMicroseconds(milliseconds * 1000, stop_on_frame);
  }

  bool AdvanceTimeMicroseconds(int64_t microseconds, bool stop_on_frame) {
    int64_t start_time = TimeInMicroseconds();
    int64_t end_time = start_time + microseconds;
    bool frame_injected = false;
    while (!timestamps_.empty() &&
           timestamps_.front().arrive_time <= end_time) {
      RTC_DCHECK_GE(timestamps_.front().arrive_time, start_time);

      SimulatedClock::AdvanceTimeMicroseconds(timestamps_.front().arrive_time -
                                              TimeInMicroseconds());
      GenerateAndInsertFrame((timestamps_.front().render_time + 500) / 1000);
      timestamps_.pop();
      frame_injected = true;

      if (stop_on_frame)
        return frame_injected;
    }

    if (TimeInMicroseconds() < end_time) {
      SimulatedClock::AdvanceTimeMicroseconds(end_time - TimeInMicroseconds());
    }
    return frame_injected;
  }

  // Input timestamps are in unit Milliseconds.
  // And `arrive_timestamps` must be positive and in increasing order.
  // `arrive_timestamps` determine when we are going to insert frames into the
  // jitter buffer.
  // `render_timestamps` are the timestamps on the frame.
  void SetFrames(const int64_t* arrive_timestamps,
                 const int64_t* render_timestamps,
                 size_t size) {
    int64_t previous_arrive_timestamp = 0;
    for (size_t i = 0; i < size; i++) {
      RTC_CHECK_GE(arrive_timestamps[i], previous_arrive_timestamp);
      timestamps_.push(TimestampPair(arrive_timestamps[i] * 1000,
                                     render_timestamps[i] * 1000));
      previous_arrive_timestamp = arrive_timestamps[i];
    }
  }

 private:
  struct TimestampPair {
    TimestampPair(int64_t arrive_timestamp, int64_t render_timestamp)
        : arrive_time(arrive_timestamp), render_time(render_timestamp) {}

    int64_t arrive_time;
    int64_t render_time;
  };

  void GenerateAndInsertFrame(int64_t render_timestamp_ms) {
    VCMPacket packet;
    stream_generator_->GenerateFrame(VideoFrameType::kVideoFrameKey,
                                     1,  // media packets
                                     0,  // empty packets
                                     render_timestamp_ms);

    bool packet_available = stream_generator_->PopPacket(&packet, 0);
    EXPECT_TRUE(packet_available);
    if (!packet_available)
      return;  // Return here to avoid crashes below.
    receiver_->InsertPacket(packet);
  }

  std::queue<TimestampPair> timestamps_;
  StreamGenerator* stream_generator_;
  VCMReceiver* receiver_;
};

// Use a SimulatedClockWithFrames
// Wait call will do either of these:
// 1. If `stop_on_frame` is true, the clock will be turned to the exact instant
// that the first frame comes and the frame will be inserted into the jitter
// buffer, or the clock will be turned to now + `max_time` if no frame comes in
// the window.
// 2. If `stop_on_frame` is false, the clock will be turn to now + `max_time`,
// and all the frames arriving between now and now + `max_time` will be
// inserted into the jitter buffer.
//
// This is used to simulate the JitterBuffer getting packets from internet as
// time elapses.

class FrameInjectEvent : public EventWrapper {
 public:
  FrameInjectEvent(SimulatedClockWithFrames* clock, bool stop_on_frame)
      : clock_(clock), stop_on_frame_(stop_on_frame) {}

  bool Set() override { return true; }

  EventTypeWrapper Wait(int max_time_ms) override {
    if (clock_->AdvanceTimeMilliseconds(max_time_ms, stop_on_frame_) &&
        stop_on_frame_) {
      return EventTypeWrapper::kEventSignaled;
    } else {
      return EventTypeWrapper::kEventTimeout;
    }
  }

 private:
  SimulatedClockWithFrames* clock_;
  bool stop_on_frame_;
};

class VCMReceiverTimingTest : public ::testing::Test {
 protected:
  VCMReceiverTimingTest()
      : clock_(&stream_generator_, &receiver_),
        stream_generator_(0, clock_.TimeInMilliseconds()),
        timing_(&clock_, field_trials_),
        receiver_(
            &timing_,
            &clock_,
            std::unique_ptr<EventWrapper>(new FrameInjectEvent(&clock_, false)),
            std::unique_ptr<EventWrapper>(new FrameInjectEvent(&clock_, true)),
            field_trials_) {}

  virtual void SetUp() {}

  test::ScopedKeyValueConfig field_trials_;
  SimulatedClockWithFrames clock_;
  StreamGenerator stream_generator_;
  VCMTiming timing_;
  VCMReceiver receiver_;
};

// Test whether VCMReceiver::FrameForDecoding handles parameter
// `max_wait_time_ms` correctly:
// 1. The function execution should never take more than `max_wait_time_ms`.
// 2. If the function exit before now + `max_wait_time_ms`, a frame must be
//    returned.
TEST_F(VCMReceiverTimingTest, FrameForDecoding) {
  const size_t kNumFrames = 100;
  const int kFramePeriod = 40;
  int64_t arrive_timestamps[kNumFrames];
  int64_t render_timestamps[kNumFrames];

  // Construct test samples.
  // render_timestamps are the timestamps stored in the Frame;
  // arrive_timestamps controls when the Frame packet got received.
  for (size_t i = 0; i < kNumFrames; i++) {
    // Preset frame rate to 25Hz.
    // But we add a reasonable deviation to arrive_timestamps to mimic Internet
    // fluctuation.
    arrive_timestamps[i] =
        (i + 1) * kFramePeriod + (i % 10) * ((i % 2) ? 1 : -1);
    render_timestamps[i] = (i + 1) * kFramePeriod;
  }

  clock_.SetFrames(arrive_timestamps, render_timestamps, kNumFrames);

  // Record how many frames we finally get out of the receiver.
  size_t num_frames_return = 0;

  const int64_t kMaxWaitTime = 30;

  // Ideally, we should get all frames that we input in InitializeFrames.
  // In the case that FrameForDecoding kills frames by error, we rely on the
  // build bot to kill the test.
  while (num_frames_return < kNumFrames) {
    int64_t start_time = clock_.TimeInMilliseconds();
    VCMEncodedFrame* frame = receiver_.FrameForDecoding(kMaxWaitTime, false);
    int64_t end_time = clock_.TimeInMilliseconds();

    // In any case the FrameForDecoding should not wait longer than
    // max_wait_time.
    // In the case that we did not get a frame, it should have been waiting for
    // exactly max_wait_time. (By the testing samples we constructed above, we
    // are sure there is no timing error, so the only case it returns with NULL
    // is that it runs out of time.)
    if (frame) {
      receiver_.ReleaseFrame(frame);
      ++num_frames_return;
      EXPECT_GE(kMaxWaitTime, end_time - start_time);
    } else {
      EXPECT_EQ(kMaxWaitTime, end_time - start_time);
    }
  }
}

// Test whether VCMReceiver::FrameForDecoding handles parameter
// `prefer_late_decoding` and `max_wait_time_ms` correctly:
// 1. The function execution should never take more than `max_wait_time_ms`.
// 2. If the function exit before now + `max_wait_time_ms`, a frame must be
//    returned and the end time must be equal to the render timestamp - delay
//    for decoding and rendering.
TEST_F(VCMReceiverTimingTest, FrameForDecodingPreferLateDecoding) {
  const size_t kNumFrames = 100;
  const int kFramePeriod = 40;

  int64_t arrive_timestamps[kNumFrames];
  int64_t render_timestamps[kNumFrames];

  auto timings = timing_.GetTimings();
  TimeDelta render_delay = timings.render_delay;
  TimeDelta max_decode = timings.max_decode_duration;

  // Construct test samples.
  // render_timestamps are the timestamps stored in the Frame;
  // arrive_timestamps controls when the Frame packet got received.
  for (size_t i = 0; i < kNumFrames; i++) {
    // Preset frame rate to 25Hz.
    // But we add a reasonable deviation to arrive_timestamps to mimic Internet
    // fluctuation.
    arrive_timestamps[i] =
        (i + 1) * kFramePeriod + (i % 10) * ((i % 2) ? 1 : -1);
    render_timestamps[i] = (i + 1) * kFramePeriod;
  }

  clock_.SetFrames(arrive_timestamps, render_timestamps, kNumFrames);

  // Record how many frames we finally get out of the receiver.
  size_t num_frames_return = 0;
  const int64_t kMaxWaitTime = 30;
  bool prefer_late_decoding = true;
  while (num_frames_return < kNumFrames) {
    int64_t start_time = clock_.TimeInMilliseconds();

    VCMEncodedFrame* frame =
        receiver_.FrameForDecoding(kMaxWaitTime, prefer_late_decoding);
    int64_t end_time = clock_.TimeInMilliseconds();
    if (frame) {
      EXPECT_EQ(frame->RenderTimeMs() - max_decode.ms() - render_delay.ms(),
                end_time);
      receiver_.ReleaseFrame(frame);
      ++num_frames_return;
    } else {
      EXPECT_EQ(kMaxWaitTime, end_time - start_time);
    }
  }
}

}  // namespace webrtc