1 /*
2 * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/rtp_rtcp/source/rtp_sender_video.h"
12
13 #include <memory>
14 #include <string>
15 #include <utility>
16 #include <vector>
17
18 #include "absl/memory/memory.h"
19 #include "api/field_trials_registry.h"
20 #include "api/frame_transformer_factory.h"
21 #include "api/rtp_headers.h"
22 #include "api/task_queue/task_queue_base.h"
23 #include "api/task_queue/task_queue_factory.h"
24 #include "api/test/mock_frame_encryptor.h"
25 #include "api/transport/field_trial_based_config.h"
26 #include "api/transport/rtp/dependency_descriptor.h"
27 #include "api/units/timestamp.h"
28 #include "api/video/video_codec_constants.h"
29 #include "api/video/video_timing.h"
30 #include "modules/rtp_rtcp/include/rtp_cvo.h"
31 #include "modules/rtp_rtcp/include/rtp_header_extension_map.h"
32 #include "modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.h"
33 #include "modules/rtp_rtcp/source/rtp_format_video_generic.h"
34 #include "modules/rtp_rtcp/source/rtp_generic_frame_descriptor.h"
35 #include "modules/rtp_rtcp/source/rtp_generic_frame_descriptor_extension.h"
36 #include "modules/rtp_rtcp/source/rtp_header_extensions.h"
37 #include "modules/rtp_rtcp/source/rtp_packet_received.h"
38 #include "modules/rtp_rtcp/source/rtp_rtcp_impl2.h"
39 #include "modules/rtp_rtcp/source/rtp_video_layers_allocation_extension.h"
40 #include "rtc_base/arraysize.h"
41 #include "rtc_base/checks.h"
42 #include "rtc_base/logging.h"
43 #include "rtc_base/rate_limiter.h"
44 #include "rtc_base/thread.h"
45 #include "test/gmock.h"
46 #include "test/gtest.h"
47 #include "test/mock_frame_transformer.h"
48 #include "test/time_controller/simulated_time_controller.h"
49
50 namespace webrtc {
51
52 namespace {
53
54 using ::testing::_;
55 using ::testing::ContainerEq;
56 using ::testing::ElementsAre;
57 using ::testing::ElementsAreArray;
58 using ::testing::IsEmpty;
59 using ::testing::NiceMock;
60 using ::testing::Return;
61 using ::testing::ReturnArg;
62 using ::testing::SaveArg;
63 using ::testing::SizeIs;
64 using ::testing::WithArgs;
65
66 enum : int { // The first valid value is 1.
67 kAbsoluteSendTimeExtensionId = 1,
68 kGenericDescriptorId,
69 kDependencyDescriptorId,
70 kTransmissionTimeOffsetExtensionId,
71 kTransportSequenceNumberExtensionId,
72 kVideoRotationExtensionId,
73 kVideoTimingExtensionId,
74 kAbsoluteCaptureTimeExtensionId,
75 kPlayoutDelayExtensionId,
76 kVideoLayersAllocationExtensionId,
77 };
78
79 constexpr int kPayload = 100;
80 constexpr VideoCodecType kType = VideoCodecType::kVideoCodecGeneric;
81 constexpr uint32_t kTimestamp = 10;
82 constexpr uint16_t kSeqNum = 33;
83 constexpr uint32_t kSsrc = 725242;
84 constexpr int kMaxPacketLength = 1500;
85 constexpr Timestamp kStartTime = Timestamp::Millis(123456789);
86 constexpr int64_t kDefaultExpectedRetransmissionTimeMs = 125;
87
88 class LoopbackTransportTest : public webrtc::Transport {
89 public:
LoopbackTransportTest()90 LoopbackTransportTest() {
91 receivers_extensions_.Register<TransmissionOffset>(
92 kTransmissionTimeOffsetExtensionId);
93 receivers_extensions_.Register<AbsoluteSendTime>(
94 kAbsoluteSendTimeExtensionId);
95 receivers_extensions_.Register<TransportSequenceNumber>(
96 kTransportSequenceNumberExtensionId);
97 receivers_extensions_.Register<VideoOrientation>(kVideoRotationExtensionId);
98 receivers_extensions_.Register<VideoTimingExtension>(
99 kVideoTimingExtensionId);
100 receivers_extensions_.Register<RtpGenericFrameDescriptorExtension00>(
101 kGenericDescriptorId);
102 receivers_extensions_.Register<RtpDependencyDescriptorExtension>(
103 kDependencyDescriptorId);
104 receivers_extensions_.Register<AbsoluteCaptureTimeExtension>(
105 kAbsoluteCaptureTimeExtensionId);
106 receivers_extensions_.Register<PlayoutDelayLimits>(
107 kPlayoutDelayExtensionId);
108 receivers_extensions_.Register<RtpVideoLayersAllocationExtension>(
109 kVideoLayersAllocationExtensionId);
110 }
111
SendRtp(const uint8_t * data,size_t len,const PacketOptions & options)112 bool SendRtp(const uint8_t* data,
113 size_t len,
114 const PacketOptions& options) override {
115 sent_packets_.push_back(RtpPacketReceived(&receivers_extensions_));
116 EXPECT_TRUE(sent_packets_.back().Parse(data, len));
117 return true;
118 }
SendRtcp(const uint8_t * data,size_t len)119 bool SendRtcp(const uint8_t* data, size_t len) override { return false; }
last_sent_packet()120 const RtpPacketReceived& last_sent_packet() { return sent_packets_.back(); }
packets_sent()121 int packets_sent() { return sent_packets_.size(); }
sent_packets() const122 const std::vector<RtpPacketReceived>& sent_packets() const {
123 return sent_packets_;
124 }
125
126 private:
127 RtpHeaderExtensionMap receivers_extensions_;
128 std::vector<RtpPacketReceived> sent_packets_;
129 };
130
131 class TestRtpSenderVideo : public RTPSenderVideo {
132 public:
TestRtpSenderVideo(Clock * clock,RTPSender * rtp_sender,const FieldTrialsView & field_trials)133 TestRtpSenderVideo(Clock* clock,
134 RTPSender* rtp_sender,
135 const FieldTrialsView& field_trials)
136 : RTPSenderVideo([&] {
137 Config config;
138 config.clock = clock;
139 config.rtp_sender = rtp_sender;
140 config.field_trials = &field_trials;
141 return config;
142 }()) {}
~TestRtpSenderVideo()143 ~TestRtpSenderVideo() override {}
144
AllowRetransmission(const RTPVideoHeader & header,int32_t retransmission_settings,int64_t expected_retransmission_time_ms)145 bool AllowRetransmission(const RTPVideoHeader& header,
146 int32_t retransmission_settings,
147 int64_t expected_retransmission_time_ms) {
148 return RTPSenderVideo::AllowRetransmission(GetTemporalId(header),
149 retransmission_settings,
150 expected_retransmission_time_ms);
151 }
152 };
153
154 class FieldTrials : public FieldTrialsRegistry {
155 public:
FieldTrials()156 FieldTrials() : include_capture_clock_offset_(false) {}
157
set_include_capture_clock_offset(bool include_capture_clock_offset)158 void set_include_capture_clock_offset(bool include_capture_clock_offset) {
159 include_capture_clock_offset_ = include_capture_clock_offset;
160 }
161
162 private:
GetValue(absl::string_view key) const163 std::string GetValue(absl::string_view key) const override {
164 if (key == "WebRTC-IncludeCaptureClockOffset") {
165 return include_capture_clock_offset_ ? "" : "Disabled";
166 }
167 return "";
168 }
169
170 bool include_capture_clock_offset_;
171 };
172
173 class RtpSenderVideoTest : public ::testing::Test {
174 public:
RtpSenderVideoTest()175 RtpSenderVideoTest()
176 : fake_clock_(kStartTime),
177 retransmission_rate_limiter_(&fake_clock_, 1000),
178 rtp_module_(ModuleRtpRtcpImpl2::Create([&] {
179 RtpRtcpInterface::Configuration config;
180 config.clock = &fake_clock_;
181 config.outgoing_transport = &transport_;
182 config.retransmission_rate_limiter = &retransmission_rate_limiter_;
183 config.field_trials = &field_trials_;
184 config.local_media_ssrc = kSsrc;
185 return config;
186 }())),
187 rtp_sender_video_(
188 std::make_unique<TestRtpSenderVideo>(&fake_clock_,
189 rtp_module_->RtpSender(),
190 field_trials_)) {
191 rtp_module_->SetSequenceNumber(kSeqNum);
192 rtp_module_->SetStartTimestamp(0);
193 }
194
195 void UsesMinimalVp8DescriptorWhenGenericFrameDescriptorExtensionIsUsed(
196 int version);
197
198 protected:
199 rtc::AutoThread main_thread_;
200 const RtpRtcpInterface::Configuration config_;
201 FieldTrials field_trials_;
202 SimulatedClock fake_clock_;
203 LoopbackTransportTest transport_;
204 RateLimiter retransmission_rate_limiter_;
205 std::unique_ptr<ModuleRtpRtcpImpl2> rtp_module_;
206 std::unique_ptr<TestRtpSenderVideo> rtp_sender_video_;
207 };
208
TEST_F(RtpSenderVideoTest,KeyFrameHasCVO)209 TEST_F(RtpSenderVideoTest, KeyFrameHasCVO) {
210 uint8_t kFrame[kMaxPacketLength];
211 rtp_module_->RegisterRtpHeaderExtension(VideoOrientation::Uri(),
212 kVideoRotationExtensionId);
213
214 RTPVideoHeader hdr;
215 hdr.rotation = kVideoRotation_0;
216 hdr.frame_type = VideoFrameType::kVideoFrameKey;
217 rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr,
218 kDefaultExpectedRetransmissionTimeMs);
219
220 VideoRotation rotation;
221 EXPECT_TRUE(
222 transport_.last_sent_packet().GetExtension<VideoOrientation>(&rotation));
223 EXPECT_EQ(kVideoRotation_0, rotation);
224 }
225
TEST_F(RtpSenderVideoTest,TimingFrameHasPacketizationTimstampSet)226 TEST_F(RtpSenderVideoTest, TimingFrameHasPacketizationTimstampSet) {
227 uint8_t kFrame[kMaxPacketLength];
228 const int64_t kPacketizationTimeMs = 100;
229 const int64_t kEncodeStartDeltaMs = 10;
230 const int64_t kEncodeFinishDeltaMs = 50;
231 rtp_module_->RegisterRtpHeaderExtension(VideoTimingExtension::Uri(),
232 kVideoTimingExtensionId);
233
234 const int64_t kCaptureTimestamp = fake_clock_.TimeInMilliseconds();
235
236 RTPVideoHeader hdr;
237 hdr.video_timing.flags = VideoSendTiming::kTriggeredByTimer;
238 hdr.video_timing.encode_start_delta_ms = kEncodeStartDeltaMs;
239 hdr.video_timing.encode_finish_delta_ms = kEncodeFinishDeltaMs;
240
241 fake_clock_.AdvanceTimeMilliseconds(kPacketizationTimeMs);
242 hdr.frame_type = VideoFrameType::kVideoFrameKey;
243 rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, kCaptureTimestamp,
244 kFrame, hdr,
245 kDefaultExpectedRetransmissionTimeMs);
246 VideoSendTiming timing;
247 EXPECT_TRUE(transport_.last_sent_packet().GetExtension<VideoTimingExtension>(
248 &timing));
249 EXPECT_EQ(kPacketizationTimeMs, timing.packetization_finish_delta_ms);
250 EXPECT_EQ(kEncodeStartDeltaMs, timing.encode_start_delta_ms);
251 EXPECT_EQ(kEncodeFinishDeltaMs, timing.encode_finish_delta_ms);
252 }
253
TEST_F(RtpSenderVideoTest,DeltaFrameHasCVOWhenChanged)254 TEST_F(RtpSenderVideoTest, DeltaFrameHasCVOWhenChanged) {
255 uint8_t kFrame[kMaxPacketLength];
256 rtp_module_->RegisterRtpHeaderExtension(VideoOrientation::Uri(),
257 kVideoRotationExtensionId);
258
259 RTPVideoHeader hdr;
260 hdr.rotation = kVideoRotation_90;
261 hdr.frame_type = VideoFrameType::kVideoFrameKey;
262 EXPECT_TRUE(
263 rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr,
264 kDefaultExpectedRetransmissionTimeMs));
265
266 hdr.rotation = kVideoRotation_0;
267 hdr.frame_type = VideoFrameType::kVideoFrameDelta;
268 EXPECT_TRUE(
269 rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp + 1, 0, kFrame,
270 hdr, kDefaultExpectedRetransmissionTimeMs));
271
272 VideoRotation rotation;
273 EXPECT_TRUE(
274 transport_.last_sent_packet().GetExtension<VideoOrientation>(&rotation));
275 EXPECT_EQ(kVideoRotation_0, rotation);
276 }
277
TEST_F(RtpSenderVideoTest,DeltaFrameHasCVOWhenNonZero)278 TEST_F(RtpSenderVideoTest, DeltaFrameHasCVOWhenNonZero) {
279 uint8_t kFrame[kMaxPacketLength];
280 rtp_module_->RegisterRtpHeaderExtension(VideoOrientation::Uri(),
281 kVideoRotationExtensionId);
282
283 RTPVideoHeader hdr;
284 hdr.rotation = kVideoRotation_90;
285 hdr.frame_type = VideoFrameType::kVideoFrameKey;
286 EXPECT_TRUE(
287 rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr,
288 kDefaultExpectedRetransmissionTimeMs));
289
290 hdr.frame_type = VideoFrameType::kVideoFrameDelta;
291 EXPECT_TRUE(
292 rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp + 1, 0, kFrame,
293 hdr, kDefaultExpectedRetransmissionTimeMs));
294
295 VideoRotation rotation;
296 EXPECT_TRUE(
297 transport_.last_sent_packet().GetExtension<VideoOrientation>(&rotation));
298 EXPECT_EQ(kVideoRotation_90, rotation);
299 }
300
301 // Make sure rotation is parsed correctly when the Camera (C) and Flip (F) bits
302 // are set in the CVO byte.
TEST_F(RtpSenderVideoTest,SendVideoWithCameraAndFlipCVO)303 TEST_F(RtpSenderVideoTest, SendVideoWithCameraAndFlipCVO) {
304 // Test extracting rotation when Camera (C) and Flip (F) bits are zero.
305 EXPECT_EQ(kVideoRotation_0, ConvertCVOByteToVideoRotation(0));
306 EXPECT_EQ(kVideoRotation_90, ConvertCVOByteToVideoRotation(1));
307 EXPECT_EQ(kVideoRotation_180, ConvertCVOByteToVideoRotation(2));
308 EXPECT_EQ(kVideoRotation_270, ConvertCVOByteToVideoRotation(3));
309 // Test extracting rotation when Camera (C) and Flip (F) bits are set.
310 const int flip_bit = 1 << 2;
311 const int camera_bit = 1 << 3;
312 EXPECT_EQ(kVideoRotation_0,
313 ConvertCVOByteToVideoRotation(flip_bit | camera_bit | 0));
314 EXPECT_EQ(kVideoRotation_90,
315 ConvertCVOByteToVideoRotation(flip_bit | camera_bit | 1));
316 EXPECT_EQ(kVideoRotation_180,
317 ConvertCVOByteToVideoRotation(flip_bit | camera_bit | 2));
318 EXPECT_EQ(kVideoRotation_270,
319 ConvertCVOByteToVideoRotation(flip_bit | camera_bit | 3));
320 }
321
TEST_F(RtpSenderVideoTest,RetransmissionTypesGeneric)322 TEST_F(RtpSenderVideoTest, RetransmissionTypesGeneric) {
323 RTPVideoHeader header;
324 header.codec = kVideoCodecGeneric;
325
326 EXPECT_FALSE(rtp_sender_video_->AllowRetransmission(
327 header, kRetransmitOff, kDefaultExpectedRetransmissionTimeMs));
328 EXPECT_TRUE(rtp_sender_video_->AllowRetransmission(
329 header, kRetransmitBaseLayer, kDefaultExpectedRetransmissionTimeMs));
330 EXPECT_TRUE(rtp_sender_video_->AllowRetransmission(
331 header, kRetransmitHigherLayers, kDefaultExpectedRetransmissionTimeMs));
332 EXPECT_TRUE(rtp_sender_video_->AllowRetransmission(
333 header, kConditionallyRetransmitHigherLayers,
334 kDefaultExpectedRetransmissionTimeMs));
335 }
336
TEST_F(RtpSenderVideoTest,RetransmissionTypesH264)337 TEST_F(RtpSenderVideoTest, RetransmissionTypesH264) {
338 RTPVideoHeader header;
339 header.video_type_header.emplace<RTPVideoHeaderH264>().packetization_mode =
340 H264PacketizationMode::NonInterleaved;
341 header.codec = kVideoCodecH264;
342
343 EXPECT_FALSE(rtp_sender_video_->AllowRetransmission(
344 header, kRetransmitOff, kDefaultExpectedRetransmissionTimeMs));
345 EXPECT_TRUE(rtp_sender_video_->AllowRetransmission(
346 header, kRetransmitBaseLayer, kDefaultExpectedRetransmissionTimeMs));
347 EXPECT_TRUE(rtp_sender_video_->AllowRetransmission(
348 header, kRetransmitHigherLayers, kDefaultExpectedRetransmissionTimeMs));
349 EXPECT_TRUE(rtp_sender_video_->AllowRetransmission(
350 header, kConditionallyRetransmitHigherLayers,
351 kDefaultExpectedRetransmissionTimeMs));
352 }
353
TEST_F(RtpSenderVideoTest,RetransmissionTypesVP8BaseLayer)354 TEST_F(RtpSenderVideoTest, RetransmissionTypesVP8BaseLayer) {
355 RTPVideoHeader header;
356 header.codec = kVideoCodecVP8;
357 auto& vp8_header = header.video_type_header.emplace<RTPVideoHeaderVP8>();
358 vp8_header.temporalIdx = 0;
359
360 EXPECT_FALSE(rtp_sender_video_->AllowRetransmission(
361 header, kRetransmitOff, kDefaultExpectedRetransmissionTimeMs));
362 EXPECT_TRUE(rtp_sender_video_->AllowRetransmission(
363 header, kRetransmitBaseLayer, kDefaultExpectedRetransmissionTimeMs));
364 EXPECT_FALSE(rtp_sender_video_->AllowRetransmission(
365 header, kRetransmitHigherLayers, kDefaultExpectedRetransmissionTimeMs));
366 EXPECT_TRUE(rtp_sender_video_->AllowRetransmission(
367 header, kRetransmitHigherLayers | kRetransmitBaseLayer,
368 kDefaultExpectedRetransmissionTimeMs));
369 EXPECT_FALSE(rtp_sender_video_->AllowRetransmission(
370 header, kConditionallyRetransmitHigherLayers,
371 kDefaultExpectedRetransmissionTimeMs));
372 EXPECT_TRUE(rtp_sender_video_->AllowRetransmission(
373 header, kRetransmitBaseLayer | kConditionallyRetransmitHigherLayers,
374 kDefaultExpectedRetransmissionTimeMs));
375 }
376
TEST_F(RtpSenderVideoTest,RetransmissionTypesVP8HigherLayers)377 TEST_F(RtpSenderVideoTest, RetransmissionTypesVP8HigherLayers) {
378 RTPVideoHeader header;
379 header.codec = kVideoCodecVP8;
380
381 auto& vp8_header = header.video_type_header.emplace<RTPVideoHeaderVP8>();
382 for (int tid = 1; tid <= kMaxTemporalStreams; ++tid) {
383 vp8_header.temporalIdx = tid;
384
385 EXPECT_FALSE(rtp_sender_video_->AllowRetransmission(
386 header, kRetransmitOff, kDefaultExpectedRetransmissionTimeMs));
387 EXPECT_FALSE(rtp_sender_video_->AllowRetransmission(
388 header, kRetransmitBaseLayer, kDefaultExpectedRetransmissionTimeMs));
389 EXPECT_TRUE(rtp_sender_video_->AllowRetransmission(
390 header, kRetransmitHigherLayers, kDefaultExpectedRetransmissionTimeMs));
391 EXPECT_TRUE(rtp_sender_video_->AllowRetransmission(
392 header, kRetransmitHigherLayers | kRetransmitBaseLayer,
393 kDefaultExpectedRetransmissionTimeMs));
394 }
395 }
396
TEST_F(RtpSenderVideoTest,RetransmissionTypesVP9)397 TEST_F(RtpSenderVideoTest, RetransmissionTypesVP9) {
398 RTPVideoHeader header;
399 header.codec = kVideoCodecVP9;
400
401 auto& vp9_header = header.video_type_header.emplace<RTPVideoHeaderVP9>();
402 for (int tid = 1; tid <= kMaxTemporalStreams; ++tid) {
403 vp9_header.temporal_idx = tid;
404
405 EXPECT_FALSE(rtp_sender_video_->AllowRetransmission(
406 header, kRetransmitOff, kDefaultExpectedRetransmissionTimeMs));
407 EXPECT_FALSE(rtp_sender_video_->AllowRetransmission(
408 header, kRetransmitBaseLayer, kDefaultExpectedRetransmissionTimeMs));
409 EXPECT_TRUE(rtp_sender_video_->AllowRetransmission(
410 header, kRetransmitHigherLayers, kDefaultExpectedRetransmissionTimeMs));
411 EXPECT_TRUE(rtp_sender_video_->AllowRetransmission(
412 header, kRetransmitHigherLayers | kRetransmitBaseLayer,
413 kDefaultExpectedRetransmissionTimeMs));
414 }
415 }
416
TEST_F(RtpSenderVideoTest,ConditionalRetransmit)417 TEST_F(RtpSenderVideoTest, ConditionalRetransmit) {
418 const int64_t kFrameIntervalMs = 33;
419 const int64_t kRttMs = (kFrameIntervalMs * 3) / 2;
420 const uint8_t kSettings =
421 kRetransmitBaseLayer | kConditionallyRetransmitHigherLayers;
422
423 // Insert VP8 frames for all temporal layers, but stop before the final index.
424 RTPVideoHeader header;
425 header.codec = kVideoCodecVP8;
426
427 // Fill averaging window to prevent rounding errors.
428 constexpr int kNumRepetitions =
429 (RTPSenderVideo::kTLRateWindowSizeMs + (kFrameIntervalMs / 2)) /
430 kFrameIntervalMs;
431 constexpr int kPattern[] = {0, 2, 1, 2};
432 auto& vp8_header = header.video_type_header.emplace<RTPVideoHeaderVP8>();
433 for (size_t i = 0; i < arraysize(kPattern) * kNumRepetitions; ++i) {
434 vp8_header.temporalIdx = kPattern[i % arraysize(kPattern)];
435 rtp_sender_video_->AllowRetransmission(header, kSettings, kRttMs);
436 fake_clock_.AdvanceTimeMilliseconds(kFrameIntervalMs);
437 }
438
439 // Since we're at the start of the pattern, the next expected frame in TL0 is
440 // right now. We will wait at most one expected retransmission time before
441 // acknowledging that it did not arrive, which means this frame and the next
442 // will not be retransmitted.
443 vp8_header.temporalIdx = 1;
444 EXPECT_FALSE(
445 rtp_sender_video_->AllowRetransmission(header, kSettings, kRttMs));
446 fake_clock_.AdvanceTimeMilliseconds(kFrameIntervalMs);
447 EXPECT_FALSE(
448 rtp_sender_video_->AllowRetransmission(header, kSettings, kRttMs));
449 fake_clock_.AdvanceTimeMilliseconds(kFrameIntervalMs);
450
451 // The TL0 frame did not arrive. So allow retransmission.
452 EXPECT_TRUE(
453 rtp_sender_video_->AllowRetransmission(header, kSettings, kRttMs));
454 fake_clock_.AdvanceTimeMilliseconds(kFrameIntervalMs);
455
456 // Insert a frame for TL2. We just had frame in TL1, so the next one there is
457 // in three frames away. TL0 is still too far in the past. So, allow
458 // retransmission.
459 vp8_header.temporalIdx = 2;
460 EXPECT_TRUE(
461 rtp_sender_video_->AllowRetransmission(header, kSettings, kRttMs));
462 fake_clock_.AdvanceTimeMilliseconds(kFrameIntervalMs);
463
464 // Another TL2, next in TL1 is two frames away. Allow again.
465 EXPECT_TRUE(
466 rtp_sender_video_->AllowRetransmission(header, kSettings, kRttMs));
467 fake_clock_.AdvanceTimeMilliseconds(kFrameIntervalMs);
468
469 // Yet another TL2, next in TL1 is now only one frame away, so don't store
470 // for retransmission.
471 EXPECT_FALSE(
472 rtp_sender_video_->AllowRetransmission(header, kSettings, kRttMs));
473 }
474
TEST_F(RtpSenderVideoTest,ConditionalRetransmitLimit)475 TEST_F(RtpSenderVideoTest, ConditionalRetransmitLimit) {
476 const int64_t kFrameIntervalMs = 200;
477 const int64_t kRttMs = (kFrameIntervalMs * 3) / 2;
478 const int32_t kSettings =
479 kRetransmitBaseLayer | kConditionallyRetransmitHigherLayers;
480
481 // Insert VP8 frames for all temporal layers, but stop before the final index.
482 RTPVideoHeader header;
483 header.codec = kVideoCodecVP8;
484
485 // Fill averaging window to prevent rounding errors.
486 constexpr int kNumRepetitions =
487 (RTPSenderVideo::kTLRateWindowSizeMs + (kFrameIntervalMs / 2)) /
488 kFrameIntervalMs;
489 constexpr int kPattern[] = {0, 2, 2, 2};
490 auto& vp8_header = header.video_type_header.emplace<RTPVideoHeaderVP8>();
491 for (size_t i = 0; i < arraysize(kPattern) * kNumRepetitions; ++i) {
492 vp8_header.temporalIdx = kPattern[i % arraysize(kPattern)];
493
494 rtp_sender_video_->AllowRetransmission(header, kSettings, kRttMs);
495 fake_clock_.AdvanceTimeMilliseconds(kFrameIntervalMs);
496 }
497
498 // Since we're at the start of the pattern, the next expected frame will be
499 // right now in TL0. Put it in TL1 instead. Regular rules would dictate that
500 // we don't store for retransmission because we expect a frame in a lower
501 // layer, but that last frame in TL1 was a long time ago in absolute terms,
502 // so allow retransmission anyway.
503 vp8_header.temporalIdx = 1;
504 EXPECT_TRUE(
505 rtp_sender_video_->AllowRetransmission(header, kSettings, kRttMs));
506 }
507
TEST_F(RtpSenderVideoTest,SendsDependencyDescriptorWhenVideoStructureIsSet)508 TEST_F(RtpSenderVideoTest, SendsDependencyDescriptorWhenVideoStructureIsSet) {
509 const int64_t kFrameId = 100000;
510 uint8_t kFrame[100];
511 rtp_module_->RegisterRtpHeaderExtension(
512 RtpDependencyDescriptorExtension::Uri(), kDependencyDescriptorId);
513 FrameDependencyStructure video_structure;
514 video_structure.num_decode_targets = 2;
515 video_structure.templates = {
516 FrameDependencyTemplate().S(0).T(0).Dtis("SS"),
517 FrameDependencyTemplate().S(1).T(0).Dtis("-S"),
518 FrameDependencyTemplate().S(1).T(1).Dtis("-D"),
519 };
520 rtp_sender_video_->SetVideoStructure(&video_structure);
521
522 // Send key frame.
523 RTPVideoHeader hdr;
524 RTPVideoHeader::GenericDescriptorInfo& generic = hdr.generic.emplace();
525 generic.frame_id = kFrameId;
526 generic.temporal_index = 0;
527 generic.spatial_index = 0;
528 generic.decode_target_indications = {DecodeTargetIndication::kSwitch,
529 DecodeTargetIndication::kSwitch};
530 hdr.frame_type = VideoFrameType::kVideoFrameKey;
531 rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr,
532 kDefaultExpectedRetransmissionTimeMs);
533
534 ASSERT_EQ(transport_.packets_sent(), 1);
535 DependencyDescriptor descriptor_key;
536 ASSERT_TRUE(transport_.last_sent_packet()
537 .GetExtension<RtpDependencyDescriptorExtension>(
538 nullptr, &descriptor_key));
539 ASSERT_TRUE(descriptor_key.attached_structure);
540 EXPECT_EQ(descriptor_key.attached_structure->num_decode_targets, 2);
541 EXPECT_THAT(descriptor_key.attached_structure->templates, SizeIs(3));
542 EXPECT_EQ(descriptor_key.frame_number, kFrameId & 0xFFFF);
543 EXPECT_EQ(descriptor_key.frame_dependencies.spatial_id, 0);
544 EXPECT_EQ(descriptor_key.frame_dependencies.temporal_id, 0);
545 EXPECT_EQ(descriptor_key.frame_dependencies.decode_target_indications,
546 generic.decode_target_indications);
547 EXPECT_THAT(descriptor_key.frame_dependencies.frame_diffs, IsEmpty());
548
549 // Send delta frame.
550 generic.frame_id = kFrameId + 1;
551 generic.temporal_index = 1;
552 generic.spatial_index = 1;
553 generic.dependencies = {kFrameId, kFrameId - 500};
554 generic.decode_target_indications = {DecodeTargetIndication::kNotPresent,
555 DecodeTargetIndication::kRequired};
556 hdr.frame_type = VideoFrameType::kVideoFrameDelta;
557 rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr,
558 kDefaultExpectedRetransmissionTimeMs);
559
560 EXPECT_EQ(transport_.packets_sent(), 2);
561 DependencyDescriptor descriptor_delta;
562 ASSERT_TRUE(
563 transport_.last_sent_packet()
564 .GetExtension<RtpDependencyDescriptorExtension>(
565 descriptor_key.attached_structure.get(), &descriptor_delta));
566 EXPECT_EQ(descriptor_delta.attached_structure, nullptr);
567 EXPECT_EQ(descriptor_delta.frame_number, (kFrameId + 1) & 0xFFFF);
568 EXPECT_EQ(descriptor_delta.frame_dependencies.spatial_id, 1);
569 EXPECT_EQ(descriptor_delta.frame_dependencies.temporal_id, 1);
570 EXPECT_EQ(descriptor_delta.frame_dependencies.decode_target_indications,
571 generic.decode_target_indications);
572 EXPECT_THAT(descriptor_delta.frame_dependencies.frame_diffs,
573 ElementsAre(1, 501));
574 }
575
TEST_F(RtpSenderVideoTest,SkipsDependencyDescriptorOnDeltaFrameWhenFailedToAttachToKeyFrame)576 TEST_F(RtpSenderVideoTest,
577 SkipsDependencyDescriptorOnDeltaFrameWhenFailedToAttachToKeyFrame) {
578 const int64_t kFrameId = 100000;
579 uint8_t kFrame[100];
580 rtp_module_->RegisterRtpHeaderExtension(
581 RtpDependencyDescriptorExtension::Uri(), kDependencyDescriptorId);
582 rtp_module_->SetExtmapAllowMixed(false);
583 FrameDependencyStructure video_structure;
584 video_structure.num_decode_targets = 2;
585 // Use many templates so that key dependency descriptor would be too large
586 // to fit into 16 bytes (max size of one byte header rtp header extension)
587 video_structure.templates = {
588 FrameDependencyTemplate().S(0).T(0).Dtis("SS"),
589 FrameDependencyTemplate().S(1).T(0).Dtis("-S"),
590 FrameDependencyTemplate().S(1).T(1).Dtis("-D").FrameDiffs({1, 2, 3, 4}),
591 FrameDependencyTemplate().S(1).T(1).Dtis("-D").FrameDiffs({2, 3, 4, 5}),
592 FrameDependencyTemplate().S(1).T(1).Dtis("-D").FrameDiffs({3, 4, 5, 6}),
593 FrameDependencyTemplate().S(1).T(1).Dtis("-D").FrameDiffs({4, 5, 6, 7}),
594 };
595 rtp_sender_video_->SetVideoStructure(&video_structure);
596
597 // Send key frame.
598 RTPVideoHeader hdr;
599 RTPVideoHeader::GenericDescriptorInfo& generic = hdr.generic.emplace();
600 generic.frame_id = kFrameId;
601 generic.temporal_index = 0;
602 generic.spatial_index = 0;
603 generic.decode_target_indications = {DecodeTargetIndication::kSwitch,
604 DecodeTargetIndication::kSwitch};
605 hdr.frame_type = VideoFrameType::kVideoFrameKey;
606 rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr,
607 kDefaultExpectedRetransmissionTimeMs);
608
609 ASSERT_EQ(transport_.packets_sent(), 1);
610 DependencyDescriptor descriptor_key;
611 ASSERT_FALSE(transport_.last_sent_packet()
612 .HasExtension<RtpDependencyDescriptorExtension>());
613
614 // Send delta frame.
615 generic.frame_id = kFrameId + 1;
616 generic.temporal_index = 1;
617 generic.spatial_index = 1;
618 generic.dependencies = {kFrameId, kFrameId - 500};
619 generic.decode_target_indications = {DecodeTargetIndication::kNotPresent,
620 DecodeTargetIndication::kRequired};
621 hdr.frame_type = VideoFrameType::kVideoFrameDelta;
622 rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr,
623 kDefaultExpectedRetransmissionTimeMs);
624
625 EXPECT_EQ(transport_.packets_sent(), 2);
626 EXPECT_FALSE(transport_.last_sent_packet()
627 .HasExtension<RtpDependencyDescriptorExtension>());
628 }
629
TEST_F(RtpSenderVideoTest,PropagatesChainDiffsIntoDependencyDescriptor)630 TEST_F(RtpSenderVideoTest, PropagatesChainDiffsIntoDependencyDescriptor) {
631 const int64_t kFrameId = 100000;
632 uint8_t kFrame[100];
633 rtp_module_->RegisterRtpHeaderExtension(
634 RtpDependencyDescriptorExtension::Uri(), kDependencyDescriptorId);
635 FrameDependencyStructure video_structure;
636 video_structure.num_decode_targets = 2;
637 video_structure.num_chains = 1;
638 video_structure.decode_target_protected_by_chain = {0, 0};
639 video_structure.templates = {
640 FrameDependencyTemplate().S(0).T(0).Dtis("SS").ChainDiffs({1}),
641 };
642 rtp_sender_video_->SetVideoStructure(&video_structure);
643
644 RTPVideoHeader hdr;
645 RTPVideoHeader::GenericDescriptorInfo& generic = hdr.generic.emplace();
646 generic.frame_id = kFrameId;
647 generic.decode_target_indications = {DecodeTargetIndication::kSwitch,
648 DecodeTargetIndication::kSwitch};
649 generic.chain_diffs = {2};
650 hdr.frame_type = VideoFrameType::kVideoFrameKey;
651 rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr,
652 kDefaultExpectedRetransmissionTimeMs);
653
654 ASSERT_EQ(transport_.packets_sent(), 1);
655 DependencyDescriptor descriptor_key;
656 ASSERT_TRUE(transport_.last_sent_packet()
657 .GetExtension<RtpDependencyDescriptorExtension>(
658 nullptr, &descriptor_key));
659 EXPECT_THAT(descriptor_key.frame_dependencies.chain_diffs,
660 ContainerEq(generic.chain_diffs));
661 }
662
TEST_F(RtpSenderVideoTest,PropagatesActiveDecodeTargetsIntoDependencyDescriptor)663 TEST_F(RtpSenderVideoTest,
664 PropagatesActiveDecodeTargetsIntoDependencyDescriptor) {
665 const int64_t kFrameId = 100000;
666 uint8_t kFrame[100];
667 rtp_module_->RegisterRtpHeaderExtension(
668 RtpDependencyDescriptorExtension::Uri(), kDependencyDescriptorId);
669 FrameDependencyStructure video_structure;
670 video_structure.num_decode_targets = 2;
671 video_structure.num_chains = 1;
672 video_structure.decode_target_protected_by_chain = {0, 0};
673 video_structure.templates = {
674 FrameDependencyTemplate().S(0).T(0).Dtis("SS").ChainDiffs({1}),
675 };
676 rtp_sender_video_->SetVideoStructure(&video_structure);
677
678 RTPVideoHeader hdr;
679 RTPVideoHeader::GenericDescriptorInfo& generic = hdr.generic.emplace();
680 generic.frame_id = kFrameId;
681 generic.decode_target_indications = {DecodeTargetIndication::kSwitch,
682 DecodeTargetIndication::kSwitch};
683 generic.active_decode_targets = 0b01;
684 generic.chain_diffs = {1};
685 hdr.frame_type = VideoFrameType::kVideoFrameKey;
686 rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr,
687 kDefaultExpectedRetransmissionTimeMs);
688
689 ASSERT_EQ(transport_.packets_sent(), 1);
690 DependencyDescriptor descriptor_key;
691 ASSERT_TRUE(transport_.last_sent_packet()
692 .GetExtension<RtpDependencyDescriptorExtension>(
693 nullptr, &descriptor_key));
694 EXPECT_EQ(descriptor_key.active_decode_targets_bitmask, 0b01u);
695 }
696
TEST_F(RtpSenderVideoTest,SetDiffentVideoStructureAvoidsCollisionWithThePreviousStructure)697 TEST_F(RtpSenderVideoTest,
698 SetDiffentVideoStructureAvoidsCollisionWithThePreviousStructure) {
699 const int64_t kFrameId = 100000;
700 uint8_t kFrame[100];
701 rtp_module_->RegisterRtpHeaderExtension(
702 RtpDependencyDescriptorExtension::Uri(), kDependencyDescriptorId);
703 FrameDependencyStructure video_structure1;
704 video_structure1.num_decode_targets = 2;
705 video_structure1.templates = {
706 FrameDependencyTemplate().S(0).T(0).Dtis("SS"),
707 FrameDependencyTemplate().S(0).T(1).Dtis("D-"),
708 };
709 FrameDependencyStructure video_structure2;
710 video_structure2.num_decode_targets = 2;
711 video_structure2.templates = {
712 FrameDependencyTemplate().S(0).T(0).Dtis("SS"),
713 FrameDependencyTemplate().S(0).T(1).Dtis("R-"),
714 };
715
716 // Send 1st key frame.
717 RTPVideoHeader hdr;
718 RTPVideoHeader::GenericDescriptorInfo& generic = hdr.generic.emplace();
719 generic.frame_id = kFrameId;
720 generic.decode_target_indications = {DecodeTargetIndication::kSwitch,
721 DecodeTargetIndication::kSwitch};
722 hdr.frame_type = VideoFrameType::kVideoFrameKey;
723 rtp_sender_video_->SetVideoStructure(&video_structure1);
724 rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr,
725 kDefaultExpectedRetransmissionTimeMs);
726 // Parse 1st extension.
727 ASSERT_EQ(transport_.packets_sent(), 1);
728 DependencyDescriptor descriptor_key1;
729 ASSERT_TRUE(transport_.last_sent_packet()
730 .GetExtension<RtpDependencyDescriptorExtension>(
731 nullptr, &descriptor_key1));
732 ASSERT_TRUE(descriptor_key1.attached_structure);
733
734 // Send the delta frame.
735 generic.frame_id = kFrameId + 1;
736 generic.temporal_index = 1;
737 generic.decode_target_indications = {DecodeTargetIndication::kDiscardable,
738 DecodeTargetIndication::kNotPresent};
739 hdr.frame_type = VideoFrameType::kVideoFrameDelta;
740 rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr,
741 kDefaultExpectedRetransmissionTimeMs);
742
743 ASSERT_EQ(transport_.packets_sent(), 2);
744 RtpPacket delta_packet = transport_.last_sent_packet();
745
746 // Send 2nd key frame.
747 generic.frame_id = kFrameId + 2;
748 generic.decode_target_indications = {DecodeTargetIndication::kSwitch,
749 DecodeTargetIndication::kSwitch};
750 hdr.frame_type = VideoFrameType::kVideoFrameKey;
751 rtp_sender_video_->SetVideoStructure(&video_structure2);
752 rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr,
753 kDefaultExpectedRetransmissionTimeMs);
754 // Parse the 2nd key frame.
755 ASSERT_EQ(transport_.packets_sent(), 3);
756 DependencyDescriptor descriptor_key2;
757 ASSERT_TRUE(transport_.last_sent_packet()
758 .GetExtension<RtpDependencyDescriptorExtension>(
759 nullptr, &descriptor_key2));
760 ASSERT_TRUE(descriptor_key2.attached_structure);
761
762 // Try to parse the 1st delta frame. It should parseble using the structure
763 // from the 1st key frame, but not using the structure from the 2nd key frame.
764 DependencyDescriptor descriptor_delta;
765 EXPECT_TRUE(delta_packet.GetExtension<RtpDependencyDescriptorExtension>(
766 descriptor_key1.attached_structure.get(), &descriptor_delta));
767 EXPECT_FALSE(delta_packet.GetExtension<RtpDependencyDescriptorExtension>(
768 descriptor_key2.attached_structure.get(), &descriptor_delta));
769 }
770
TEST_F(RtpSenderVideoTest,AuthenticateVideoHeaderWhenDependencyDescriptorExtensionIsUsed)771 TEST_F(RtpSenderVideoTest,
772 AuthenticateVideoHeaderWhenDependencyDescriptorExtensionIsUsed) {
773 static constexpr size_t kFrameSize = 100;
774 uint8_t kFrame[kFrameSize] = {1, 2, 3, 4};
775
776 rtp_module_->RegisterRtpHeaderExtension(
777 RtpDependencyDescriptorExtension::Uri(), kDependencyDescriptorId);
778 auto encryptor = rtc::make_ref_counted<NiceMock<MockFrameEncryptor>>();
779 ON_CALL(*encryptor, GetMaxCiphertextByteSize).WillByDefault(ReturnArg<1>());
780 ON_CALL(*encryptor, Encrypt)
781 .WillByDefault(WithArgs<3, 5>(
782 [](rtc::ArrayView<const uint8_t> frame, size_t* bytes_written) {
783 *bytes_written = frame.size();
784 return 0;
785 }));
786 RTPSenderVideo::Config config;
787 config.clock = &fake_clock_;
788 config.rtp_sender = rtp_module_->RtpSender();
789 config.field_trials = &field_trials_;
790 config.frame_encryptor = encryptor.get();
791 RTPSenderVideo rtp_sender_video(config);
792
793 FrameDependencyStructure video_structure;
794 video_structure.num_decode_targets = 1;
795 video_structure.templates = {FrameDependencyTemplate().Dtis("S")};
796 rtp_sender_video.SetVideoStructure(&video_structure);
797
798 // Send key frame.
799 RTPVideoHeader hdr;
800 hdr.frame_type = VideoFrameType::kVideoFrameKey;
801 hdr.generic.emplace().decode_target_indications =
802 video_structure.templates[0].decode_target_indications;
803
804 EXPECT_CALL(*encryptor,
805 Encrypt(_, _, Not(IsEmpty()), ElementsAreArray(kFrame), _, _));
806 rtp_sender_video.SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr,
807 kDefaultExpectedRetransmissionTimeMs);
808 // Double check packet with the dependency descriptor is sent.
809 ASSERT_EQ(transport_.packets_sent(), 1);
810 EXPECT_TRUE(transport_.last_sent_packet()
811 .HasExtension<RtpDependencyDescriptorExtension>());
812 }
813
TEST_F(RtpSenderVideoTest,PopulateGenericFrameDescriptor)814 TEST_F(RtpSenderVideoTest, PopulateGenericFrameDescriptor) {
815 const int64_t kFrameId = 100000;
816 uint8_t kFrame[100];
817 rtp_module_->RegisterRtpHeaderExtension(
818 RtpGenericFrameDescriptorExtension00::Uri(), kGenericDescriptorId);
819
820 RTPVideoHeader hdr;
821 RTPVideoHeader::GenericDescriptorInfo& generic = hdr.generic.emplace();
822 generic.frame_id = kFrameId;
823 generic.temporal_index = 3;
824 generic.spatial_index = 2;
825 generic.dependencies.push_back(kFrameId - 1);
826 generic.dependencies.push_back(kFrameId - 500);
827 hdr.frame_type = VideoFrameType::kVideoFrameDelta;
828 rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr,
829 kDefaultExpectedRetransmissionTimeMs);
830
831 RtpGenericFrameDescriptor descriptor_wire;
832 EXPECT_EQ(1, transport_.packets_sent());
833 ASSERT_TRUE(transport_.last_sent_packet()
834 .GetExtension<RtpGenericFrameDescriptorExtension00>(
835 &descriptor_wire));
836 EXPECT_EQ(static_cast<uint16_t>(generic.frame_id), descriptor_wire.FrameId());
837 EXPECT_EQ(generic.temporal_index, descriptor_wire.TemporalLayer());
838 EXPECT_THAT(descriptor_wire.FrameDependenciesDiffs(), ElementsAre(1, 500));
839 EXPECT_EQ(descriptor_wire.SpatialLayersBitmask(), 0b0000'0100);
840 }
841
842 void RtpSenderVideoTest::
UsesMinimalVp8DescriptorWhenGenericFrameDescriptorExtensionIsUsed(int version)843 UsesMinimalVp8DescriptorWhenGenericFrameDescriptorExtensionIsUsed(
844 int version) {
845 const int64_t kFrameId = 100000;
846 const size_t kFrameSize = 100;
847 uint8_t kFrame[kFrameSize];
848
849 rtp_module_->RegisterRtpHeaderExtension(
850 RtpGenericFrameDescriptorExtension00::Uri(), kGenericDescriptorId);
851
852 RTPVideoHeader hdr;
853 hdr.codec = kVideoCodecVP8;
854 RTPVideoHeaderVP8& vp8 = hdr.video_type_header.emplace<RTPVideoHeaderVP8>();
855 vp8.pictureId = kFrameId % 0X7FFF;
856 vp8.tl0PicIdx = 13;
857 vp8.temporalIdx = 1;
858 vp8.keyIdx = 2;
859 RTPVideoHeader::GenericDescriptorInfo& generic = hdr.generic.emplace();
860 generic.frame_id = kFrameId;
861 hdr.frame_type = VideoFrameType::kVideoFrameDelta;
862 rtp_sender_video_->SendVideo(kPayload, VideoCodecType::kVideoCodecVP8,
863 kTimestamp, 0, kFrame, hdr,
864 kDefaultExpectedRetransmissionTimeMs);
865
866 ASSERT_EQ(transport_.packets_sent(), 1);
867 // Expect only minimal 1-byte vp8 descriptor was generated.
868 EXPECT_EQ(transport_.last_sent_packet().payload_size(), 1 + kFrameSize);
869 }
870
TEST_F(RtpSenderVideoTest,UsesMinimalVp8DescriptorWhenGenericFrameDescriptorExtensionIsUsed00)871 TEST_F(RtpSenderVideoTest,
872 UsesMinimalVp8DescriptorWhenGenericFrameDescriptorExtensionIsUsed00) {
873 UsesMinimalVp8DescriptorWhenGenericFrameDescriptorExtensionIsUsed(0);
874 }
875
TEST_F(RtpSenderVideoTest,UsesMinimalVp8DescriptorWhenGenericFrameDescriptorExtensionIsUsed01)876 TEST_F(RtpSenderVideoTest,
877 UsesMinimalVp8DescriptorWhenGenericFrameDescriptorExtensionIsUsed01) {
878 UsesMinimalVp8DescriptorWhenGenericFrameDescriptorExtensionIsUsed(1);
879 }
880
TEST_F(RtpSenderVideoTest,VideoLayersAllocationWithResolutionSentOnKeyFrames)881 TEST_F(RtpSenderVideoTest, VideoLayersAllocationWithResolutionSentOnKeyFrames) {
882 const size_t kFrameSize = 100;
883 uint8_t kFrame[kFrameSize];
884 rtp_module_->RegisterRtpHeaderExtension(
885 RtpVideoLayersAllocationExtension::Uri(),
886 kVideoLayersAllocationExtensionId);
887
888 VideoLayersAllocation allocation;
889 VideoLayersAllocation::SpatialLayer layer;
890 layer.width = 360;
891 layer.height = 180;
892 layer.target_bitrate_per_temporal_layer.push_back(
893 DataRate::KilobitsPerSec(50));
894 allocation.resolution_and_frame_rate_is_valid = true;
895 allocation.active_spatial_layers.push_back(layer);
896 rtp_sender_video_->SetVideoLayersAllocation(allocation);
897
898 RTPVideoHeader hdr;
899 hdr.frame_type = VideoFrameType::kVideoFrameKey;
900 rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr,
901 kDefaultExpectedRetransmissionTimeMs);
902
903 VideoLayersAllocation sent_allocation;
904 EXPECT_TRUE(
905 transport_.last_sent_packet()
906 .GetExtension<RtpVideoLayersAllocationExtension>(&sent_allocation));
907 EXPECT_THAT(sent_allocation.active_spatial_layers, ElementsAre(layer));
908
909 // Next key frame also have the allocation.
910 rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr,
911 kDefaultExpectedRetransmissionTimeMs);
912 EXPECT_TRUE(
913 transport_.last_sent_packet()
914 .GetExtension<RtpVideoLayersAllocationExtension>(&sent_allocation));
915 }
916
TEST_F(RtpSenderVideoTest,VideoLayersAllocationWithoutResolutionSentOnDeltaWhenUpdated)917 TEST_F(RtpSenderVideoTest,
918 VideoLayersAllocationWithoutResolutionSentOnDeltaWhenUpdated) {
919 const size_t kFrameSize = 100;
920 uint8_t kFrame[kFrameSize];
921 rtp_module_->RegisterRtpHeaderExtension(
922 RtpVideoLayersAllocationExtension::Uri(),
923 kVideoLayersAllocationExtensionId);
924
925 VideoLayersAllocation allocation;
926 VideoLayersAllocation::SpatialLayer layer;
927 layer.width = 360;
928 layer.height = 180;
929 allocation.resolution_and_frame_rate_is_valid = true;
930 layer.target_bitrate_per_temporal_layer.push_back(
931 DataRate::KilobitsPerSec(50));
932 allocation.active_spatial_layers.push_back(layer);
933 rtp_sender_video_->SetVideoLayersAllocation(allocation);
934
935 RTPVideoHeader hdr;
936 hdr.frame_type = VideoFrameType::kVideoFrameKey;
937 rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr,
938 kDefaultExpectedRetransmissionTimeMs);
939 EXPECT_TRUE(transport_.last_sent_packet()
940 .HasExtension<RtpVideoLayersAllocationExtension>());
941
942 // No allocation sent on delta frame unless it has been updated.
943 hdr.frame_type = VideoFrameType::kVideoFrameDelta;
944 rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr,
945 kDefaultExpectedRetransmissionTimeMs);
946 EXPECT_FALSE(transport_.last_sent_packet()
947 .HasExtension<RtpVideoLayersAllocationExtension>());
948
949 // Update the allocation.
950 rtp_sender_video_->SetVideoLayersAllocation(allocation);
951 rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr,
952 kDefaultExpectedRetransmissionTimeMs);
953
954 VideoLayersAllocation sent_allocation;
955 EXPECT_TRUE(
956 transport_.last_sent_packet()
957 .GetExtension<RtpVideoLayersAllocationExtension>(&sent_allocation));
958 ASSERT_THAT(sent_allocation.active_spatial_layers, SizeIs(1));
959 EXPECT_FALSE(sent_allocation.resolution_and_frame_rate_is_valid);
960 EXPECT_THAT(sent_allocation.active_spatial_layers[0]
961 .target_bitrate_per_temporal_layer,
962 SizeIs(1));
963 }
964
TEST_F(RtpSenderVideoTest,VideoLayersAllocationWithResolutionSentOnDeltaWhenSpatialLayerAdded)965 TEST_F(RtpSenderVideoTest,
966 VideoLayersAllocationWithResolutionSentOnDeltaWhenSpatialLayerAdded) {
967 const size_t kFrameSize = 100;
968 uint8_t kFrame[kFrameSize];
969 rtp_module_->RegisterRtpHeaderExtension(
970 RtpVideoLayersAllocationExtension::Uri(),
971 kVideoLayersAllocationExtensionId);
972
973 VideoLayersAllocation allocation;
974 allocation.resolution_and_frame_rate_is_valid = true;
975 VideoLayersAllocation::SpatialLayer layer;
976 layer.width = 360;
977 layer.height = 180;
978 layer.spatial_id = 0;
979 layer.target_bitrate_per_temporal_layer.push_back(
980 DataRate::KilobitsPerSec(50));
981 allocation.active_spatial_layers.push_back(layer);
982 rtp_sender_video_->SetVideoLayersAllocation(allocation);
983
984 RTPVideoHeader hdr;
985 hdr.frame_type = VideoFrameType::kVideoFrameKey;
986 rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr,
987 kDefaultExpectedRetransmissionTimeMs);
988 ASSERT_TRUE(transport_.last_sent_packet()
989 .HasExtension<RtpVideoLayersAllocationExtension>());
990
991 // Update the allocation.
992 layer.width = 640;
993 layer.height = 320;
994 layer.spatial_id = 1;
995 layer.target_bitrate_per_temporal_layer.push_back(
996 DataRate::KilobitsPerSec(100));
997 allocation.active_spatial_layers.push_back(layer);
998 rtp_sender_video_->SetVideoLayersAllocation(allocation);
999 hdr.frame_type = VideoFrameType::kVideoFrameDelta;
1000 rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr,
1001 kDefaultExpectedRetransmissionTimeMs);
1002
1003 VideoLayersAllocation sent_allocation;
1004 EXPECT_TRUE(
1005 transport_.last_sent_packet()
1006 .GetExtension<RtpVideoLayersAllocationExtension>(&sent_allocation));
1007 EXPECT_THAT(sent_allocation.active_spatial_layers, SizeIs(2));
1008 EXPECT_TRUE(sent_allocation.resolution_and_frame_rate_is_valid);
1009 }
1010
TEST_F(RtpSenderVideoTest,VideoLayersAllocationWithResolutionSentOnLargeFrameRateChange)1011 TEST_F(RtpSenderVideoTest,
1012 VideoLayersAllocationWithResolutionSentOnLargeFrameRateChange) {
1013 const size_t kFrameSize = 100;
1014 uint8_t kFrame[kFrameSize];
1015 rtp_module_->RegisterRtpHeaderExtension(
1016 RtpVideoLayersAllocationExtension::Uri(),
1017 kVideoLayersAllocationExtensionId);
1018
1019 VideoLayersAllocation allocation;
1020 allocation.resolution_and_frame_rate_is_valid = true;
1021 VideoLayersAllocation::SpatialLayer layer;
1022 layer.width = 360;
1023 layer.height = 180;
1024 layer.spatial_id = 0;
1025 layer.frame_rate_fps = 10;
1026 layer.target_bitrate_per_temporal_layer.push_back(
1027 DataRate::KilobitsPerSec(50));
1028 allocation.active_spatial_layers.push_back(layer);
1029 rtp_sender_video_->SetVideoLayersAllocation(allocation);
1030
1031 RTPVideoHeader hdr;
1032 hdr.frame_type = VideoFrameType::kVideoFrameKey;
1033 rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr,
1034 kDefaultExpectedRetransmissionTimeMs);
1035 ASSERT_TRUE(transport_.last_sent_packet()
1036 .HasExtension<RtpVideoLayersAllocationExtension>());
1037
1038 // Update frame rate only.
1039 allocation.active_spatial_layers[0].frame_rate_fps = 20;
1040 rtp_sender_video_->SetVideoLayersAllocation(allocation);
1041 hdr.frame_type = VideoFrameType::kVideoFrameDelta;
1042 rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr,
1043 kDefaultExpectedRetransmissionTimeMs);
1044
1045 VideoLayersAllocation sent_allocation;
1046 EXPECT_TRUE(
1047 transport_.last_sent_packet()
1048 .GetExtension<RtpVideoLayersAllocationExtension>(&sent_allocation));
1049 ASSERT_TRUE(sent_allocation.resolution_and_frame_rate_is_valid);
1050 EXPECT_EQ(sent_allocation.active_spatial_layers[0].frame_rate_fps, 20);
1051 }
1052
TEST_F(RtpSenderVideoTest,VideoLayersAllocationWithoutResolutionSentOnSmallFrameRateChange)1053 TEST_F(RtpSenderVideoTest,
1054 VideoLayersAllocationWithoutResolutionSentOnSmallFrameRateChange) {
1055 const size_t kFrameSize = 100;
1056 uint8_t kFrame[kFrameSize];
1057 rtp_module_->RegisterRtpHeaderExtension(
1058 RtpVideoLayersAllocationExtension::Uri(),
1059 kVideoLayersAllocationExtensionId);
1060
1061 VideoLayersAllocation allocation;
1062 allocation.resolution_and_frame_rate_is_valid = true;
1063 VideoLayersAllocation::SpatialLayer layer;
1064 layer.width = 360;
1065 layer.height = 180;
1066 layer.spatial_id = 0;
1067 layer.frame_rate_fps = 10;
1068 layer.target_bitrate_per_temporal_layer.push_back(
1069 DataRate::KilobitsPerSec(50));
1070 allocation.active_spatial_layers.push_back(layer);
1071 rtp_sender_video_->SetVideoLayersAllocation(allocation);
1072
1073 RTPVideoHeader hdr;
1074 hdr.frame_type = VideoFrameType::kVideoFrameKey;
1075 rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr,
1076 kDefaultExpectedRetransmissionTimeMs);
1077 ASSERT_TRUE(transport_.last_sent_packet()
1078 .HasExtension<RtpVideoLayersAllocationExtension>());
1079
1080 // Update frame rate slightly.
1081 allocation.active_spatial_layers[0].frame_rate_fps = 9;
1082 rtp_sender_video_->SetVideoLayersAllocation(allocation);
1083 hdr.frame_type = VideoFrameType::kVideoFrameDelta;
1084 rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr,
1085 kDefaultExpectedRetransmissionTimeMs);
1086
1087 VideoLayersAllocation sent_allocation;
1088 EXPECT_TRUE(
1089 transport_.last_sent_packet()
1090 .GetExtension<RtpVideoLayersAllocationExtension>(&sent_allocation));
1091 EXPECT_FALSE(sent_allocation.resolution_and_frame_rate_is_valid);
1092 }
1093
TEST_F(RtpSenderVideoTest,VideoLayersAllocationSentOnDeltaFramesOnlyOnUpdate)1094 TEST_F(RtpSenderVideoTest, VideoLayersAllocationSentOnDeltaFramesOnlyOnUpdate) {
1095 const size_t kFrameSize = 100;
1096 uint8_t kFrame[kFrameSize];
1097 rtp_module_->RegisterRtpHeaderExtension(
1098 RtpVideoLayersAllocationExtension::Uri(),
1099 kVideoLayersAllocationExtensionId);
1100
1101 VideoLayersAllocation allocation;
1102 VideoLayersAllocation::SpatialLayer layer;
1103 layer.width = 360;
1104 layer.height = 180;
1105 layer.target_bitrate_per_temporal_layer.push_back(
1106 DataRate::KilobitsPerSec(50));
1107 allocation.active_spatial_layers.push_back(layer);
1108 rtp_sender_video_->SetVideoLayersAllocation(allocation);
1109
1110 RTPVideoHeader hdr;
1111 hdr.frame_type = VideoFrameType::kVideoFrameDelta;
1112 rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr,
1113 kDefaultExpectedRetransmissionTimeMs);
1114
1115 VideoLayersAllocation sent_allocation;
1116 EXPECT_TRUE(
1117 transport_.last_sent_packet()
1118 .GetExtension<RtpVideoLayersAllocationExtension>(&sent_allocation));
1119 EXPECT_THAT(sent_allocation.active_spatial_layers, SizeIs(1));
1120
1121 // VideoLayersAllocation not sent on the next delta frame.
1122 rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr,
1123 kDefaultExpectedRetransmissionTimeMs);
1124 EXPECT_FALSE(transport_.last_sent_packet()
1125 .HasExtension<RtpVideoLayersAllocationExtension>());
1126
1127 // Update allocation. VideoLayesAllocation should be sent on the next frame.
1128 rtp_sender_video_->SetVideoLayersAllocation(allocation);
1129 rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr,
1130 kDefaultExpectedRetransmissionTimeMs);
1131 EXPECT_TRUE(
1132 transport_.last_sent_packet()
1133 .GetExtension<RtpVideoLayersAllocationExtension>(&sent_allocation));
1134 }
1135
TEST_F(RtpSenderVideoTest,VideoLayersAllocationNotSentOnHigherTemporalLayers)1136 TEST_F(RtpSenderVideoTest, VideoLayersAllocationNotSentOnHigherTemporalLayers) {
1137 const size_t kFrameSize = 100;
1138 uint8_t kFrame[kFrameSize];
1139 rtp_module_->RegisterRtpHeaderExtension(
1140 RtpVideoLayersAllocationExtension::Uri(),
1141 kVideoLayersAllocationExtensionId);
1142
1143 VideoLayersAllocation allocation;
1144 allocation.resolution_and_frame_rate_is_valid = true;
1145 VideoLayersAllocation::SpatialLayer layer;
1146 layer.width = 360;
1147 layer.height = 180;
1148 layer.target_bitrate_per_temporal_layer.push_back(
1149 DataRate::KilobitsPerSec(50));
1150 allocation.active_spatial_layers.push_back(layer);
1151 rtp_sender_video_->SetVideoLayersAllocation(allocation);
1152
1153 RTPVideoHeader hdr;
1154 hdr.frame_type = VideoFrameType::kVideoFrameDelta;
1155 hdr.codec = VideoCodecType::kVideoCodecVP8;
1156 auto& vp8_header = hdr.video_type_header.emplace<RTPVideoHeaderVP8>();
1157 vp8_header.temporalIdx = 1;
1158
1159 rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr,
1160 kDefaultExpectedRetransmissionTimeMs);
1161 EXPECT_FALSE(transport_.last_sent_packet()
1162 .HasExtension<RtpVideoLayersAllocationExtension>());
1163
1164 // Send a delta frame on tl0.
1165 vp8_header.temporalIdx = 0;
1166 rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr,
1167 kDefaultExpectedRetransmissionTimeMs);
1168 EXPECT_TRUE(transport_.last_sent_packet()
1169 .HasExtension<RtpVideoLayersAllocationExtension>());
1170 }
1171
TEST_F(RtpSenderVideoTest,AbsoluteCaptureTime)1172 TEST_F(RtpSenderVideoTest, AbsoluteCaptureTime) {
1173 constexpr int64_t kAbsoluteCaptureTimestampMs = 12345678;
1174 uint8_t kFrame[kMaxPacketLength];
1175 rtp_module_->RegisterRtpHeaderExtension(AbsoluteCaptureTimeExtension::Uri(),
1176 kAbsoluteCaptureTimeExtensionId);
1177
1178 RTPVideoHeader hdr;
1179 hdr.frame_type = VideoFrameType::kVideoFrameKey;
1180 rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp,
1181 kAbsoluteCaptureTimestampMs, kFrame, hdr,
1182 kDefaultExpectedRetransmissionTimeMs);
1183
1184 absl::optional<AbsoluteCaptureTime> absolute_capture_time;
1185
1186 // It is expected that one and only one of the packets sent on this video
1187 // frame has absolute capture time header extension.
1188 for (const RtpPacketReceived& packet : transport_.sent_packets()) {
1189 if (absolute_capture_time.has_value()) {
1190 EXPECT_FALSE(packet.HasExtension<AbsoluteCaptureTimeExtension>());
1191 } else {
1192 absolute_capture_time =
1193 packet.GetExtension<AbsoluteCaptureTimeExtension>();
1194 }
1195 }
1196
1197 // Verify the capture timestamp and that the clock offset is not set.
1198 ASSERT_TRUE(absolute_capture_time.has_value());
1199 EXPECT_EQ(
1200 absolute_capture_time->absolute_capture_timestamp,
1201 Int64MsToUQ32x32(fake_clock_.ConvertTimestampToNtpTimeInMilliseconds(
1202 kAbsoluteCaptureTimestampMs)));
1203 EXPECT_FALSE(
1204 absolute_capture_time->estimated_capture_clock_offset.has_value());
1205 }
1206
1207 // Essentially the same test as AbsoluteCaptureTime but with a field trial.
1208 // After the field trial is experimented, we will remove AbsoluteCaptureTime.
TEST_F(RtpSenderVideoTest,AbsoluteCaptureTimeWithCaptureClockOffset)1209 TEST_F(RtpSenderVideoTest, AbsoluteCaptureTimeWithCaptureClockOffset) {
1210 field_trials_.set_include_capture_clock_offset(true);
1211 rtp_sender_video_ = std::make_unique<TestRtpSenderVideo>(
1212 &fake_clock_, rtp_module_->RtpSender(), field_trials_);
1213
1214 constexpr int64_t kAbsoluteCaptureTimestampMs = 12345678;
1215 uint8_t kFrame[kMaxPacketLength];
1216 rtp_module_->RegisterRtpHeaderExtension(AbsoluteCaptureTimeExtension::Uri(),
1217 kAbsoluteCaptureTimeExtensionId);
1218
1219 RTPVideoHeader hdr;
1220 hdr.frame_type = VideoFrameType::kVideoFrameKey;
1221 rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp,
1222 kAbsoluteCaptureTimestampMs, kFrame, hdr,
1223 kDefaultExpectedRetransmissionTimeMs);
1224
1225 absl::optional<AbsoluteCaptureTime> absolute_capture_time;
1226
1227 // It is expected that one and only one of the packets sent on this video
1228 // frame has absolute capture time header extension.
1229 for (const RtpPacketReceived& packet : transport_.sent_packets()) {
1230 if (absolute_capture_time.has_value()) {
1231 EXPECT_FALSE(packet.HasExtension<AbsoluteCaptureTimeExtension>());
1232 } else {
1233 absolute_capture_time =
1234 packet.GetExtension<AbsoluteCaptureTimeExtension>();
1235 }
1236 }
1237
1238 // Verify the capture timestamp and that the clock offset is set to zero.
1239 ASSERT_TRUE(absolute_capture_time.has_value());
1240 EXPECT_EQ(
1241 absolute_capture_time->absolute_capture_timestamp,
1242 Int64MsToUQ32x32(fake_clock_.ConvertTimestampToNtpTimeInMilliseconds(
1243 kAbsoluteCaptureTimestampMs)));
1244 EXPECT_EQ(absolute_capture_time->estimated_capture_clock_offset, 0);
1245 }
1246
TEST_F(RtpSenderVideoTest,AbsoluteCaptureTimeWithExtensionProvided)1247 TEST_F(RtpSenderVideoTest, AbsoluteCaptureTimeWithExtensionProvided) {
1248 constexpr AbsoluteCaptureTime kAbsoluteCaptureTime = {
1249 123,
1250 absl::optional<int64_t>(456),
1251 };
1252 uint8_t kFrame[kMaxPacketLength];
1253 rtp_module_->RegisterRtpHeaderExtension(AbsoluteCaptureTimeExtension::Uri(),
1254 kAbsoluteCaptureTimeExtensionId);
1255
1256 RTPVideoHeader hdr;
1257 hdr.frame_type = VideoFrameType::kVideoFrameKey;
1258 hdr.absolute_capture_time = kAbsoluteCaptureTime;
1259 rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp,
1260 /*capture_time_ms=*/789, kFrame, hdr,
1261 kDefaultExpectedRetransmissionTimeMs);
1262
1263 absl::optional<AbsoluteCaptureTime> absolute_capture_time;
1264
1265 // It is expected that one and only one of the packets sent on this video
1266 // frame has absolute capture time header extension.
1267 for (const RtpPacketReceived& packet : transport_.sent_packets()) {
1268 if (absolute_capture_time.has_value()) {
1269 EXPECT_FALSE(packet.HasExtension<AbsoluteCaptureTimeExtension>());
1270 } else {
1271 absolute_capture_time =
1272 packet.GetExtension<AbsoluteCaptureTimeExtension>();
1273 }
1274 }
1275
1276 // Verify the extension.
1277 EXPECT_EQ(absolute_capture_time, kAbsoluteCaptureTime);
1278 }
1279
TEST_F(RtpSenderVideoTest,PopulatesPlayoutDelay)1280 TEST_F(RtpSenderVideoTest, PopulatesPlayoutDelay) {
1281 // Single packet frames.
1282 constexpr size_t kPacketSize = 123;
1283 uint8_t kFrame[kPacketSize];
1284 rtp_module_->RegisterRtpHeaderExtension(PlayoutDelayLimits::Uri(),
1285 kPlayoutDelayExtensionId);
1286 const VideoPlayoutDelay kExpectedDelay = {10, 20};
1287
1288 // Send initial key-frame without playout delay.
1289 RTPVideoHeader hdr;
1290 hdr.frame_type = VideoFrameType::kVideoFrameKey;
1291 hdr.codec = VideoCodecType::kVideoCodecVP8;
1292 auto& vp8_header = hdr.video_type_header.emplace<RTPVideoHeaderVP8>();
1293 vp8_header.temporalIdx = 0;
1294
1295 rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr,
1296 kDefaultExpectedRetransmissionTimeMs);
1297 EXPECT_FALSE(
1298 transport_.last_sent_packet().HasExtension<PlayoutDelayLimits>());
1299
1300 // Set playout delay on a discardable frame.
1301 hdr.playout_delay = kExpectedDelay;
1302 hdr.frame_type = VideoFrameType::kVideoFrameDelta;
1303 vp8_header.temporalIdx = 1;
1304 rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr,
1305 kDefaultExpectedRetransmissionTimeMs);
1306 VideoPlayoutDelay received_delay = VideoPlayoutDelay();
1307 ASSERT_TRUE(transport_.last_sent_packet().GetExtension<PlayoutDelayLimits>(
1308 &received_delay));
1309 EXPECT_EQ(received_delay, kExpectedDelay);
1310
1311 // Set playout delay on a non-discardable frame, the extension should still
1312 // be populated since dilvery wasn't guaranteed on the last one.
1313 hdr.playout_delay = VideoPlayoutDelay(); // Indicates "no change".
1314 vp8_header.temporalIdx = 0;
1315 rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr,
1316 kDefaultExpectedRetransmissionTimeMs);
1317 ASSERT_TRUE(transport_.last_sent_packet().GetExtension<PlayoutDelayLimits>(
1318 &received_delay));
1319 EXPECT_EQ(received_delay, kExpectedDelay);
1320
1321 // The next frame does not need the extensions since it's delivery has
1322 // already been guaranteed.
1323 rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr,
1324 kDefaultExpectedRetransmissionTimeMs);
1325 EXPECT_FALSE(
1326 transport_.last_sent_packet().HasExtension<PlayoutDelayLimits>());
1327
1328 // Insert key-frame, we need to refresh the state here.
1329 hdr.frame_type = VideoFrameType::kVideoFrameKey;
1330 rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr,
1331 kDefaultExpectedRetransmissionTimeMs);
1332 ASSERT_TRUE(transport_.last_sent_packet().GetExtension<PlayoutDelayLimits>(
1333 &received_delay));
1334 EXPECT_EQ(received_delay, kExpectedDelay);
1335 }
1336
TEST_F(RtpSenderVideoTest,SendGenericVideo)1337 TEST_F(RtpSenderVideoTest, SendGenericVideo) {
1338 const uint8_t kPayloadType = 127;
1339 const VideoCodecType kCodecType = VideoCodecType::kVideoCodecGeneric;
1340 const uint8_t kPayload[] = {47, 11, 32, 93, 89};
1341
1342 // Send keyframe.
1343 RTPVideoHeader video_header;
1344 video_header.frame_type = VideoFrameType::kVideoFrameKey;
1345 ASSERT_TRUE(rtp_sender_video_->SendVideo(kPayloadType, kCodecType, 1234, 4321,
1346 kPayload, video_header,
1347 absl::nullopt));
1348
1349 rtc::ArrayView<const uint8_t> sent_payload =
1350 transport_.last_sent_packet().payload();
1351 uint8_t generic_header = sent_payload[0];
1352 EXPECT_TRUE(generic_header & RtpFormatVideoGeneric::kKeyFrameBit);
1353 EXPECT_TRUE(generic_header & RtpFormatVideoGeneric::kFirstPacketBit);
1354 EXPECT_THAT(sent_payload.subview(1), ElementsAreArray(kPayload));
1355
1356 // Send delta frame.
1357 const uint8_t kDeltaPayload[] = {13, 42, 32, 93, 13};
1358 video_header.frame_type = VideoFrameType::kVideoFrameDelta;
1359 ASSERT_TRUE(rtp_sender_video_->SendVideo(kPayloadType, kCodecType, 1234, 4321,
1360 kDeltaPayload, video_header,
1361 absl::nullopt));
1362
1363 sent_payload = sent_payload = transport_.last_sent_packet().payload();
1364 generic_header = sent_payload[0];
1365 EXPECT_FALSE(generic_header & RtpFormatVideoGeneric::kKeyFrameBit);
1366 EXPECT_TRUE(generic_header & RtpFormatVideoGeneric::kFirstPacketBit);
1367 EXPECT_THAT(sent_payload.subview(1), ElementsAreArray(kDeltaPayload));
1368 }
1369
TEST_F(RtpSenderVideoTest,SendRawVideo)1370 TEST_F(RtpSenderVideoTest, SendRawVideo) {
1371 const uint8_t kPayloadType = 111;
1372 const uint8_t kPayload[] = {11, 22, 33, 44, 55};
1373
1374 // Send a frame.
1375 RTPVideoHeader video_header;
1376 video_header.frame_type = VideoFrameType::kVideoFrameKey;
1377 ASSERT_TRUE(rtp_sender_video_->SendVideo(kPayloadType, absl::nullopt, 1234,
1378 4321, kPayload, video_header,
1379 absl::nullopt));
1380
1381 rtc::ArrayView<const uint8_t> sent_payload =
1382 transport_.last_sent_packet().payload();
1383 EXPECT_THAT(sent_payload, ElementsAreArray(kPayload));
1384 }
1385
1386 class RtpSenderVideoWithFrameTransformerTest : public ::testing::Test {
1387 public:
RtpSenderVideoWithFrameTransformerTest()1388 RtpSenderVideoWithFrameTransformerTest()
1389 : time_controller_(kStartTime),
1390 retransmission_rate_limiter_(time_controller_.GetClock(), 1000),
1391 rtp_module_(ModuleRtpRtcpImpl2::Create([&] {
1392 RtpRtcpInterface::Configuration config;
1393 config.clock = time_controller_.GetClock();
1394 config.outgoing_transport = &transport_;
1395 config.retransmission_rate_limiter = &retransmission_rate_limiter_;
1396 config.field_trials = &field_trials_;
1397 config.local_media_ssrc = kSsrc;
1398 return config;
1399 }())) {
1400 rtp_module_->SetSequenceNumber(kSeqNum);
1401 rtp_module_->SetStartTimestamp(0);
1402 }
1403
CreateSenderWithFrameTransformer(rtc::scoped_refptr<FrameTransformerInterface> transformer)1404 std::unique_ptr<RTPSenderVideo> CreateSenderWithFrameTransformer(
1405 rtc::scoped_refptr<FrameTransformerInterface> transformer) {
1406 RTPSenderVideo::Config config;
1407 config.clock = time_controller_.GetClock();
1408 config.rtp_sender = rtp_module_->RtpSender();
1409 config.field_trials = &field_trials_;
1410 config.frame_transformer = transformer;
1411 config.task_queue_factory = time_controller_.GetTaskQueueFactory();
1412 return std::make_unique<RTPSenderVideo>(config);
1413 }
1414
1415 protected:
1416 GlobalSimulatedTimeController time_controller_;
1417 FieldTrialBasedConfig field_trials_;
1418 LoopbackTransportTest transport_;
1419 RateLimiter retransmission_rate_limiter_;
1420 std::unique_ptr<ModuleRtpRtcpImpl2> rtp_module_;
1421 };
1422
CreateDefaultEncodedImage()1423 std::unique_ptr<EncodedImage> CreateDefaultEncodedImage() {
1424 const uint8_t data[] = {1, 2, 3, 4};
1425 auto encoded_image = std::make_unique<EncodedImage>();
1426 encoded_image->SetEncodedData(
1427 webrtc::EncodedImageBuffer::Create(data, sizeof(data)));
1428 return encoded_image;
1429 }
1430
TEST_F(RtpSenderVideoWithFrameTransformerTest,CreateSenderRegistersFrameTransformer)1431 TEST_F(RtpSenderVideoWithFrameTransformerTest,
1432 CreateSenderRegistersFrameTransformer) {
1433 auto mock_frame_transformer =
1434 rtc::make_ref_counted<NiceMock<MockFrameTransformer>>();
1435 EXPECT_CALL(*mock_frame_transformer,
1436 RegisterTransformedFrameSinkCallback(_, kSsrc));
1437 std::unique_ptr<RTPSenderVideo> rtp_sender_video =
1438 CreateSenderWithFrameTransformer(mock_frame_transformer);
1439 }
1440
TEST_F(RtpSenderVideoWithFrameTransformerTest,DestroySenderUnregistersFrameTransformer)1441 TEST_F(RtpSenderVideoWithFrameTransformerTest,
1442 DestroySenderUnregistersFrameTransformer) {
1443 auto mock_frame_transformer =
1444 rtc::make_ref_counted<NiceMock<MockFrameTransformer>>();
1445 std::unique_ptr<RTPSenderVideo> rtp_sender_video =
1446 CreateSenderWithFrameTransformer(mock_frame_transformer);
1447 EXPECT_CALL(*mock_frame_transformer,
1448 UnregisterTransformedFrameSinkCallback(kSsrc));
1449 rtp_sender_video = nullptr;
1450 }
1451
TEST_F(RtpSenderVideoWithFrameTransformerTest,SendEncodedImageTransformsFrame)1452 TEST_F(RtpSenderVideoWithFrameTransformerTest,
1453 SendEncodedImageTransformsFrame) {
1454 auto mock_frame_transformer =
1455 rtc::make_ref_counted<NiceMock<MockFrameTransformer>>();
1456 std::unique_ptr<RTPSenderVideo> rtp_sender_video =
1457 CreateSenderWithFrameTransformer(mock_frame_transformer);
1458 auto encoded_image = CreateDefaultEncodedImage();
1459 RTPVideoHeader video_header;
1460
1461 EXPECT_CALL(*mock_frame_transformer, Transform);
1462 rtp_sender_video->SendEncodedImage(kPayload, kType, kTimestamp,
1463 *encoded_image, video_header,
1464 kDefaultExpectedRetransmissionTimeMs);
1465 }
1466
1467 #if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID)
TEST_F(RtpSenderVideoWithFrameTransformerTest,ValidPayloadTypes)1468 TEST_F(RtpSenderVideoWithFrameTransformerTest, ValidPayloadTypes) {
1469 auto mock_frame_transformer =
1470 rtc::make_ref_counted<NiceMock<MockFrameTransformer>>();
1471 std::unique_ptr<RTPSenderVideo> rtp_sender_video =
1472 CreateSenderWithFrameTransformer(mock_frame_transformer);
1473 auto encoded_image = CreateDefaultEncodedImage();
1474 RTPVideoHeader video_header;
1475
1476 EXPECT_TRUE(rtp_sender_video->SendEncodedImage(
1477 0, kType, kTimestamp, *encoded_image, video_header,
1478 kDefaultExpectedRetransmissionTimeMs));
1479 EXPECT_TRUE(rtp_sender_video->SendEncodedImage(
1480 127, kType, kTimestamp, *encoded_image, video_header,
1481 kDefaultExpectedRetransmissionTimeMs));
1482 EXPECT_DEATH(rtp_sender_video->SendEncodedImage(
1483 -1, kType, kTimestamp, *encoded_image, video_header,
1484 kDefaultExpectedRetransmissionTimeMs),
1485 "");
1486 EXPECT_DEATH(rtp_sender_video->SendEncodedImage(
1487 128, kType, kTimestamp, *encoded_image, video_header,
1488 kDefaultExpectedRetransmissionTimeMs),
1489 "");
1490 }
1491 #endif
1492
TEST_F(RtpSenderVideoWithFrameTransformerTest,OnTransformedFrameSendsVideo)1493 TEST_F(RtpSenderVideoWithFrameTransformerTest, OnTransformedFrameSendsVideo) {
1494 auto mock_frame_transformer =
1495 rtc::make_ref_counted<NiceMock<MockFrameTransformer>>();
1496 rtc::scoped_refptr<TransformedFrameCallback> callback;
1497 EXPECT_CALL(*mock_frame_transformer, RegisterTransformedFrameSinkCallback)
1498 .WillOnce(SaveArg<0>(&callback));
1499 std::unique_ptr<RTPSenderVideo> rtp_sender_video =
1500 CreateSenderWithFrameTransformer(mock_frame_transformer);
1501 ASSERT_TRUE(callback);
1502
1503 auto encoded_image = CreateDefaultEncodedImage();
1504 RTPVideoHeader video_header;
1505 video_header.frame_type = VideoFrameType::kVideoFrameKey;
1506 ON_CALL(*mock_frame_transformer, Transform)
1507 .WillByDefault(
1508 [&callback](std::unique_ptr<TransformableFrameInterface> frame) {
1509 callback->OnTransformedFrame(std::move(frame));
1510 });
1511 auto encoder_queue = time_controller_.GetTaskQueueFactory()->CreateTaskQueue(
1512 "encoder_queue", TaskQueueFactory::Priority::NORMAL);
1513 encoder_queue->PostTask([&] {
1514 rtp_sender_video->SendEncodedImage(kPayload, kType, kTimestamp,
1515 *encoded_image, video_header,
1516 kDefaultExpectedRetransmissionTimeMs);
1517 });
1518 time_controller_.AdvanceTime(TimeDelta::Zero());
1519 EXPECT_EQ(transport_.packets_sent(), 1);
1520 encoder_queue->PostTask([&] {
1521 rtp_sender_video->SendEncodedImage(kPayload, kType, kTimestamp,
1522 *encoded_image, video_header,
1523 kDefaultExpectedRetransmissionTimeMs);
1524 });
1525 time_controller_.AdvanceTime(TimeDelta::Zero());
1526 EXPECT_EQ(transport_.packets_sent(), 2);
1527 }
1528
TEST_F(RtpSenderVideoWithFrameTransformerTest,TransformableFrameMetadataHasCorrectValue)1529 TEST_F(RtpSenderVideoWithFrameTransformerTest,
1530 TransformableFrameMetadataHasCorrectValue) {
1531 auto mock_frame_transformer =
1532 rtc::make_ref_counted<NiceMock<MockFrameTransformer>>();
1533 std::unique_ptr<RTPSenderVideo> rtp_sender_video =
1534 CreateSenderWithFrameTransformer(mock_frame_transformer);
1535 auto encoded_image = CreateDefaultEncodedImage();
1536 RTPVideoHeader video_header;
1537 video_header.width = 1280u;
1538 video_header.height = 720u;
1539 RTPVideoHeader::GenericDescriptorInfo& generic =
1540 video_header.generic.emplace();
1541 generic.frame_id = 10;
1542 generic.temporal_index = 3;
1543 generic.spatial_index = 2;
1544 generic.decode_target_indications = {DecodeTargetIndication::kSwitch};
1545 generic.dependencies = {5};
1546
1547 // Check that the transformable frame passed to the frame transformer has the
1548 // correct metadata.
1549 EXPECT_CALL(*mock_frame_transformer, Transform)
1550 .WillOnce(
1551 [](std::unique_ptr<TransformableFrameInterface> transformable_frame) {
1552 auto frame =
1553 absl::WrapUnique(static_cast<TransformableVideoFrameInterface*>(
1554 transformable_frame.release()));
1555 ASSERT_TRUE(frame);
1556 auto metadata = frame->GetMetadata();
1557 EXPECT_EQ(metadata.GetWidth(), 1280u);
1558 EXPECT_EQ(metadata.GetHeight(), 720u);
1559 EXPECT_EQ(metadata.GetFrameId(), 10);
1560 EXPECT_EQ(metadata.GetTemporalIndex(), 3);
1561 EXPECT_EQ(metadata.GetSpatialIndex(), 2);
1562 EXPECT_THAT(metadata.GetFrameDependencies(), ElementsAre(5));
1563 EXPECT_THAT(metadata.GetDecodeTargetIndications(),
1564 ElementsAre(DecodeTargetIndication::kSwitch));
1565 });
1566 rtp_sender_video->SendEncodedImage(kPayload, kType, kTimestamp,
1567 *encoded_image, video_header,
1568 kDefaultExpectedRetransmissionTimeMs);
1569 }
1570
TEST_F(RtpSenderVideoWithFrameTransformerTest,OnTransformedFrameSendsVideoWhenCloned)1571 TEST_F(RtpSenderVideoWithFrameTransformerTest,
1572 OnTransformedFrameSendsVideoWhenCloned) {
1573 auto mock_frame_transformer =
1574 rtc::make_ref_counted<NiceMock<MockFrameTransformer>>();
1575 rtc::scoped_refptr<TransformedFrameCallback> callback;
1576 EXPECT_CALL(*mock_frame_transformer, RegisterTransformedFrameSinkCallback)
1577 .WillOnce(SaveArg<0>(&callback));
1578 std::unique_ptr<RTPSenderVideo> rtp_sender_video =
1579 CreateSenderWithFrameTransformer(mock_frame_transformer);
1580 ASSERT_TRUE(callback);
1581
1582 auto encoded_image = CreateDefaultEncodedImage();
1583 RTPVideoHeader video_header;
1584 video_header.frame_type = VideoFrameType::kVideoFrameKey;
1585 ON_CALL(*mock_frame_transformer, Transform)
1586 .WillByDefault(
1587 [&callback](std::unique_ptr<TransformableFrameInterface> frame) {
1588 auto clone = CloneVideoFrame(
1589 static_cast<TransformableVideoFrameInterface*>(frame.get()));
1590 EXPECT_TRUE(clone);
1591 callback->OnTransformedFrame(std::move(clone));
1592 });
1593 auto encoder_queue = time_controller_.GetTaskQueueFactory()->CreateTaskQueue(
1594 "encoder_queue", TaskQueueFactory::Priority::NORMAL);
1595 encoder_queue->PostTask([&] {
1596 rtp_sender_video->SendEncodedImage(kPayload, kType, kTimestamp,
1597 *encoded_image, video_header,
1598 kDefaultExpectedRetransmissionTimeMs);
1599 });
1600 time_controller_.AdvanceTime(TimeDelta::Zero());
1601 EXPECT_EQ(transport_.packets_sent(), 1);
1602 encoder_queue->PostTask([&] {
1603 rtp_sender_video->SendEncodedImage(kPayload, kType, kTimestamp,
1604 *encoded_image, video_header,
1605 kDefaultExpectedRetransmissionTimeMs);
1606 });
1607 time_controller_.AdvanceTime(TimeDelta::Zero());
1608 EXPECT_EQ(transport_.packets_sent(), 2);
1609 }
1610
1611 } // namespace
1612 } // namespace webrtc
1613