1 /*
2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10 #include <algorithm> // max
11 #include <memory>
12 #include <vector>
13
14 #include "absl/algorithm/container.h"
15 #include "absl/strings/match.h"
16 #include "api/sequence_checker.h"
17 #include "api/task_queue/default_task_queue_factory.h"
18 #include "api/task_queue/task_queue_base.h"
19 #include "api/test/metrics/global_metrics_logger_and_exporter.h"
20 #include "api/test/metrics/metric.h"
21 #include "api/test/simulated_network.h"
22 #include "api/video/builtin_video_bitrate_allocator_factory.h"
23 #include "api/video/encoded_image.h"
24 #include "api/video/video_bitrate_allocation.h"
25 #include "api/video_codecs/video_encoder.h"
26 #include "call/call.h"
27 #include "call/fake_network_pipe.h"
28 #include "call/rtp_transport_controller_send.h"
29 #include "call/simulated_network.h"
30 #include "call/video_send_stream.h"
31 #include "media/engine/internal_encoder_factory.h"
32 #include "media/engine/simulcast_encoder_adapter.h"
33 #include "media/engine/webrtc_video_engine.h"
34 #include "modules/rtp_rtcp/include/rtp_header_extension_map.h"
35 #include "modules/rtp_rtcp/source/create_video_rtp_depacketizer.h"
36 #include "modules/rtp_rtcp/source/rtcp_sender.h"
37 #include "modules/rtp_rtcp/source/rtp_header_extensions.h"
38 #include "modules/rtp_rtcp/source/rtp_packet.h"
39 #include "modules/rtp_rtcp/source/rtp_rtcp_impl2.h"
40 #include "modules/rtp_rtcp/source/rtp_util.h"
41 #include "modules/rtp_rtcp/source/video_rtp_depacketizer_vp9.h"
42 #include "modules/video_coding/codecs/interface/common_constants.h"
43 #include "modules/video_coding/codecs/vp8/include/vp8.h"
44 #include "modules/video_coding/codecs/vp9/include/vp9.h"
45 #include "modules/video_coding/svc/create_scalability_structure.h"
46 #include "modules/video_coding/svc/scalability_mode_util.h"
47 #include "modules/video_coding/svc/scalable_video_controller.h"
48 #include "rtc_base/checks.h"
49 #include "rtc_base/event.h"
50 #include "rtc_base/experiments/alr_experiment.h"
51 #include "rtc_base/logging.h"
52 #include "rtc_base/platform_thread.h"
53 #include "rtc_base/rate_limiter.h"
54 #include "rtc_base/strings/string_builder.h"
55 #include "rtc_base/synchronization/mutex.h"
56 #include "rtc_base/task_queue_for_test.h"
57 #include "rtc_base/time_utils.h"
58 #include "rtc_base/unique_id_generator.h"
59 #include "system_wrappers/include/sleep.h"
60 #include "test/call_test.h"
61 #include "test/configurable_frame_size_encoder.h"
62 #include "test/fake_encoder.h"
63 #include "test/fake_texture_frame.h"
64 #include "test/frame_forwarder.h"
65 #include "test/frame_generator_capturer.h"
66 #include "test/frame_utils.h"
67 #include "test/gmock.h"
68 #include "test/gtest.h"
69 #include "test/null_transport.h"
70 #include "test/rtcp_packet_parser.h"
71 #include "test/video_encoder_proxy_factory.h"
72 #include "video/config/encoder_stream_factory.h"
73 #include "video/send_statistics_proxy.h"
74 #include "video/transport_adapter.h"
75 #include "video/video_send_stream.h"
76
77 namespace webrtc {
78 namespace test {
79 class VideoSendStreamPeer {
80 public:
VideoSendStreamPeer(webrtc::VideoSendStream * base_class_stream)81 explicit VideoSendStreamPeer(webrtc::VideoSendStream* base_class_stream)
82 : internal_stream_(
83 static_cast<internal::VideoSendStream*>(base_class_stream)) {}
GetPacingFactorOverride() const84 absl::optional<float> GetPacingFactorOverride() const {
85 return internal_stream_->GetPacingFactorOverride();
86 }
87
88 private:
89 internal::VideoSendStream const* const internal_stream_;
90 };
91 } // namespace test
92
93 namespace {
94 enum : int { // The first valid value is 1.
95 kAbsSendTimeExtensionId = 1,
96 kTimestampOffsetExtensionId,
97 kTransportSequenceNumberExtensionId,
98 kVideoContentTypeExtensionId,
99 kVideoRotationExtensionId,
100 kVideoTimingExtensionId,
101 };
102
103 // Readability convenience enum for `WaitBitrateChanged()`.
104 enum class WaitUntil : bool { kZero = false, kNonZero = true };
105
106 constexpr int64_t kRtcpIntervalMs = 1000;
107
108 enum VideoFormat {
109 kGeneric,
110 kVP8,
111 };
112
113 struct Vp9TestParams {
114 std::string scalability_mode;
115 uint8_t num_spatial_layers;
116 uint8_t num_temporal_layers;
117 InterLayerPredMode inter_layer_pred;
118 };
119
120 using ParameterizationType = std::tuple<Vp9TestParams, bool>;
121
ParamInfoToStr(const testing::TestParamInfo<ParameterizationType> & info)122 std::string ParamInfoToStr(
123 const testing::TestParamInfo<ParameterizationType>& info) {
124 rtc::StringBuilder sb;
125 sb << std::get<0>(info.param).scalability_mode << "_"
126 << (std::get<1>(info.param) ? "WithIdentifier" : "WithoutIdentifier");
127 return sb.str();
128 }
129
130 } // namespace
131
132 class VideoSendStreamTest : public test::CallTest {
133 public:
VideoSendStreamTest()134 VideoSendStreamTest() {
135 RegisterRtpExtension(RtpExtension(RtpExtension::kTransportSequenceNumberUri,
136 kTransportSequenceNumberExtensionId));
137 }
138
139 protected:
140 void TestNackRetransmission(uint32_t retransmit_ssrc,
141 uint8_t retransmit_payload_type);
142 void TestPacketFragmentationSize(VideoFormat format, bool with_fec);
143
144 void TestVp9NonFlexMode(const Vp9TestParams& params,
145 bool use_scalability_mode_identifier);
146
147 void TestRequestSourceRotateVideo(bool support_orientation_ext);
148
149 void TestTemporalLayers(VideoEncoderFactory* encoder_factory,
150 const std::string& payload_name,
151 const std::vector<int>& num_temporal_layers,
152 const std::vector<ScalabilityMode>& scalability_mode);
153 };
154
TEST_F(VideoSendStreamTest,CanStartStartedStream)155 TEST_F(VideoSendStreamTest, CanStartStartedStream) {
156 SendTask(task_queue(), [this]() {
157 CreateSenderCall();
158
159 test::NullTransport transport;
160 CreateSendConfig(1, 0, 0, &transport);
161 CreateVideoStreams();
162 GetVideoSendStream()->Start();
163 GetVideoSendStream()->Start();
164 DestroyStreams();
165 DestroyCalls();
166 });
167 }
168
TEST_F(VideoSendStreamTest,CanStopStoppedStream)169 TEST_F(VideoSendStreamTest, CanStopStoppedStream) {
170 SendTask(task_queue(), [this]() {
171 CreateSenderCall();
172
173 test::NullTransport transport;
174 CreateSendConfig(1, 0, 0, &transport);
175 CreateVideoStreams();
176 GetVideoSendStream()->Stop();
177 GetVideoSendStream()->Stop();
178 DestroyStreams();
179 DestroyCalls();
180 });
181 }
182
TEST_F(VideoSendStreamTest,SupportsCName)183 TEST_F(VideoSendStreamTest, SupportsCName) {
184 static std::string kCName = "PjQatC14dGfbVwGPUOA9IH7RlsFDbWl4AhXEiDsBizo=";
185 class CNameObserver : public test::SendTest {
186 public:
187 CNameObserver() : SendTest(kDefaultTimeout) {}
188
189 private:
190 Action OnSendRtcp(const uint8_t* packet, size_t length) override {
191 test::RtcpPacketParser parser;
192 EXPECT_TRUE(parser.Parse(packet, length));
193 if (parser.sdes()->num_packets() > 0) {
194 EXPECT_EQ(1u, parser.sdes()->chunks().size());
195 EXPECT_EQ(kCName, parser.sdes()->chunks()[0].cname);
196
197 observation_complete_.Set();
198 }
199
200 return SEND_PACKET;
201 }
202
203 void ModifyVideoConfigs(
204 VideoSendStream::Config* send_config,
205 std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
206 VideoEncoderConfig* encoder_config) override {
207 send_config->rtp.c_name = kCName;
208 }
209
210 void PerformTest() override {
211 EXPECT_TRUE(Wait()) << "Timed out while waiting for RTCP with CNAME.";
212 }
213 } test;
214
215 RunBaseTest(&test);
216 }
217
TEST_F(VideoSendStreamTest,SupportsAbsoluteSendTime)218 TEST_F(VideoSendStreamTest, SupportsAbsoluteSendTime) {
219 class AbsoluteSendTimeObserver : public test::SendTest {
220 public:
221 AbsoluteSendTimeObserver() : SendTest(kDefaultTimeout) {
222 extensions_.Register<AbsoluteSendTime>(kAbsSendTimeExtensionId);
223 }
224
225 Action OnSendRtp(const uint8_t* packet, size_t length) override {
226 RtpPacket rtp_packet(&extensions_);
227 EXPECT_TRUE(rtp_packet.Parse(packet, length));
228
229 uint32_t abs_send_time = 0;
230 EXPECT_FALSE(rtp_packet.HasExtension<TransmissionOffset>());
231 EXPECT_TRUE(rtp_packet.GetExtension<AbsoluteSendTime>(&abs_send_time));
232 if (abs_send_time != 0) {
233 // Wait for at least one packet with a non-zero send time. The send time
234 // is a 16-bit value derived from the system clock, and it is valid
235 // for a packet to have a zero send time. To tell that from an
236 // unpopulated value we'll wait for a packet with non-zero send time.
237 observation_complete_.Set();
238 } else {
239 RTC_LOG(LS_WARNING)
240 << "Got a packet with zero absoluteSendTime, waiting"
241 " for another packet...";
242 }
243
244 return SEND_PACKET;
245 }
246
247 void ModifyVideoConfigs(
248 VideoSendStream::Config* send_config,
249 std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
250 VideoEncoderConfig* encoder_config) override {
251 send_config->rtp.extensions.clear();
252 send_config->rtp.extensions.push_back(
253 RtpExtension(RtpExtension::kAbsSendTimeUri, kAbsSendTimeExtensionId));
254 }
255
256 void PerformTest() override {
257 EXPECT_TRUE(Wait()) << "Timed out while waiting for single RTP packet.";
258 }
259
260 private:
261 RtpHeaderExtensionMap extensions_;
262 } test;
263
264 RunBaseTest(&test);
265 }
266
TEST_F(VideoSendStreamTest,SupportsTransmissionTimeOffset)267 TEST_F(VideoSendStreamTest, SupportsTransmissionTimeOffset) {
268 static const int kEncodeDelayMs = 5;
269 class TransmissionTimeOffsetObserver : public test::SendTest {
270 public:
271 TransmissionTimeOffsetObserver()
272 : SendTest(kDefaultTimeout), encoder_factory_([]() {
273 return std::make_unique<test::DelayedEncoder>(
274 Clock::GetRealTimeClock(), kEncodeDelayMs);
275 }) {
276 extensions_.Register<TransmissionOffset>(kTimestampOffsetExtensionId);
277 }
278
279 private:
280 Action OnSendRtp(const uint8_t* packet, size_t length) override {
281 RtpPacket rtp_packet(&extensions_);
282 EXPECT_TRUE(rtp_packet.Parse(packet, length));
283
284 int32_t toffset = 0;
285 EXPECT_TRUE(rtp_packet.GetExtension<TransmissionOffset>(&toffset));
286 EXPECT_FALSE(rtp_packet.HasExtension<AbsoluteSendTime>());
287 EXPECT_GT(toffset, 0);
288 observation_complete_.Set();
289
290 return SEND_PACKET;
291 }
292
293 void ModifyVideoConfigs(
294 VideoSendStream::Config* send_config,
295 std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
296 VideoEncoderConfig* encoder_config) override {
297 send_config->encoder_settings.encoder_factory = &encoder_factory_;
298 send_config->rtp.extensions.clear();
299 send_config->rtp.extensions.push_back(RtpExtension(
300 RtpExtension::kTimestampOffsetUri, kTimestampOffsetExtensionId));
301 }
302
303 void PerformTest() override {
304 EXPECT_TRUE(Wait()) << "Timed out while waiting for a single RTP packet.";
305 }
306
307 test::FunctionVideoEncoderFactory encoder_factory_;
308 RtpHeaderExtensionMap extensions_;
309 } test;
310
311 RunBaseTest(&test);
312 }
313
TEST_F(VideoSendStreamTest,SupportsTransportWideSequenceNumbers)314 TEST_F(VideoSendStreamTest, SupportsTransportWideSequenceNumbers) {
315 static const uint8_t kExtensionId = kTransportSequenceNumberExtensionId;
316 class TransportWideSequenceNumberObserver : public test::SendTest {
317 public:
318 TransportWideSequenceNumberObserver()
319 : SendTest(kDefaultTimeout), encoder_factory_([]() {
320 return std::make_unique<test::FakeEncoder>(
321 Clock::GetRealTimeClock());
322 }) {
323 extensions_.Register<TransportSequenceNumber>(kExtensionId);
324 }
325
326 private:
327 Action OnSendRtp(const uint8_t* packet, size_t length) override {
328 RtpPacket rtp_packet(&extensions_);
329 EXPECT_TRUE(rtp_packet.Parse(packet, length));
330
331 EXPECT_TRUE(rtp_packet.HasExtension<TransportSequenceNumber>());
332 EXPECT_FALSE(rtp_packet.HasExtension<TransmissionOffset>());
333 EXPECT_FALSE(rtp_packet.HasExtension<AbsoluteSendTime>());
334
335 observation_complete_.Set();
336
337 return SEND_PACKET;
338 }
339
340 void ModifyVideoConfigs(
341 VideoSendStream::Config* send_config,
342 std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
343 VideoEncoderConfig* encoder_config) override {
344 send_config->encoder_settings.encoder_factory = &encoder_factory_;
345 }
346
347 void PerformTest() override {
348 EXPECT_TRUE(Wait()) << "Timed out while waiting for a single RTP packet.";
349 }
350
351 test::FunctionVideoEncoderFactory encoder_factory_;
352 RtpHeaderExtensionMap extensions_;
353 } test;
354
355 RunBaseTest(&test);
356 }
357
TEST_F(VideoSendStreamTest,SupportsVideoRotation)358 TEST_F(VideoSendStreamTest, SupportsVideoRotation) {
359 class VideoRotationObserver : public test::SendTest {
360 public:
361 VideoRotationObserver() : SendTest(kDefaultTimeout) {
362 extensions_.Register<VideoOrientation>(kVideoRotationExtensionId);
363 }
364
365 Action OnSendRtp(const uint8_t* packet, size_t length) override {
366 RtpPacket rtp_packet(&extensions_);
367 EXPECT_TRUE(rtp_packet.Parse(packet, length));
368 // Only the last packet of the frame is required to have the extension.
369 if (!rtp_packet.Marker())
370 return SEND_PACKET;
371 EXPECT_EQ(rtp_packet.GetExtension<VideoOrientation>(), kVideoRotation_90);
372 observation_complete_.Set();
373 return SEND_PACKET;
374 }
375
376 void ModifyVideoConfigs(
377 VideoSendStream::Config* send_config,
378 std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
379 VideoEncoderConfig* encoder_config) override {
380 send_config->rtp.extensions.clear();
381 send_config->rtp.extensions.push_back(RtpExtension(
382 RtpExtension::kVideoRotationUri, kVideoRotationExtensionId));
383 }
384
385 void OnFrameGeneratorCapturerCreated(
386 test::FrameGeneratorCapturer* frame_generator_capturer) override {
387 frame_generator_capturer->SetFakeRotation(kVideoRotation_90);
388 }
389
390 void PerformTest() override {
391 EXPECT_TRUE(Wait()) << "Timed out while waiting for single RTP packet.";
392 }
393
394 private:
395 RtpHeaderExtensionMap extensions_;
396 } test;
397
398 RunBaseTest(&test);
399 }
400
TEST_F(VideoSendStreamTest,SupportsVideoContentType)401 TEST_F(VideoSendStreamTest, SupportsVideoContentType) {
402 class VideoContentTypeObserver : public test::SendTest {
403 public:
404 VideoContentTypeObserver()
405 : SendTest(kDefaultTimeout), first_frame_sent_(false) {
406 extensions_.Register<VideoContentTypeExtension>(
407 kVideoContentTypeExtensionId);
408 }
409
410 Action OnSendRtp(const uint8_t* packet, size_t length) override {
411 RtpPacket rtp_packet(&extensions_);
412 EXPECT_TRUE(rtp_packet.Parse(packet, length));
413 // Only the last packet of the key-frame must have extension.
414 if (!rtp_packet.Marker() || first_frame_sent_)
415 return SEND_PACKET;
416 // First marker bit seen means that the first frame is sent.
417 first_frame_sent_ = true;
418 VideoContentType type;
419 EXPECT_TRUE(rtp_packet.GetExtension<VideoContentTypeExtension>(&type));
420 EXPECT_TRUE(videocontenttypehelpers::IsScreenshare(type));
421 observation_complete_.Set();
422 return SEND_PACKET;
423 }
424
425 void ModifyVideoConfigs(
426 VideoSendStream::Config* send_config,
427 std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
428 VideoEncoderConfig* encoder_config) override {
429 send_config->rtp.extensions.clear();
430 send_config->rtp.extensions.push_back(RtpExtension(
431 RtpExtension::kVideoContentTypeUri, kVideoContentTypeExtensionId));
432 encoder_config->content_type = VideoEncoderConfig::ContentType::kScreen;
433 }
434
435 void PerformTest() override {
436 EXPECT_TRUE(Wait()) << "Timed out while waiting for single RTP packet.";
437 }
438
439 private:
440 bool first_frame_sent_;
441 RtpHeaderExtensionMap extensions_;
442 } test;
443
444 RunBaseTest(&test);
445 }
446
TEST_F(VideoSendStreamTest,SupportsVideoTimingFrames)447 TEST_F(VideoSendStreamTest, SupportsVideoTimingFrames) {
448 class VideoTimingObserver : public test::SendTest {
449 public:
450 VideoTimingObserver()
451 : SendTest(kDefaultTimeout), first_frame_sent_(false) {
452 extensions_.Register<VideoTimingExtension>(kVideoTimingExtensionId);
453 }
454
455 Action OnSendRtp(const uint8_t* packet, size_t length) override {
456 RtpPacket rtp_packet(&extensions_);
457 EXPECT_TRUE(rtp_packet.Parse(packet, length));
458 // Only the last packet of the frame must have extension.
459 // Also don't check packets of the second frame if they happen to get
460 // through before the test terminates.
461 if (!rtp_packet.Marker() || first_frame_sent_)
462 return SEND_PACKET;
463 EXPECT_TRUE(rtp_packet.HasExtension<VideoTimingExtension>());
464 observation_complete_.Set();
465 first_frame_sent_ = true;
466 return SEND_PACKET;
467 }
468
469 void ModifyVideoConfigs(
470 VideoSendStream::Config* send_config,
471 std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
472 VideoEncoderConfig* encoder_config) override {
473 send_config->rtp.extensions.clear();
474 send_config->rtp.extensions.push_back(
475 RtpExtension(RtpExtension::kVideoTimingUri, kVideoTimingExtensionId));
476 }
477
478 void PerformTest() override {
479 EXPECT_TRUE(Wait()) << "Timed out while waiting for timing frames.";
480 }
481
482 private:
483 RtpHeaderExtensionMap extensions_;
484 bool first_frame_sent_;
485 } test;
486
487 RunBaseTest(&test);
488 }
489
490 class FakeReceiveStatistics : public ReceiveStatisticsProvider {
491 public:
FakeReceiveStatistics(uint32_t send_ssrc,uint32_t last_sequence_number,uint32_t cumulative_lost,uint8_t fraction_lost)492 FakeReceiveStatistics(uint32_t send_ssrc,
493 uint32_t last_sequence_number,
494 uint32_t cumulative_lost,
495 uint8_t fraction_lost) {
496 stat_.SetMediaSsrc(send_ssrc);
497 stat_.SetExtHighestSeqNum(last_sequence_number);
498 stat_.SetCumulativeLost(cumulative_lost);
499 stat_.SetFractionLost(fraction_lost);
500 }
501
RtcpReportBlocks(size_t max_blocks)502 std::vector<rtcp::ReportBlock> RtcpReportBlocks(size_t max_blocks) override {
503 EXPECT_GE(max_blocks, 1u);
504 return {stat_};
505 }
506
507 private:
508 rtcp::ReportBlock stat_;
509 };
510
511 class UlpfecObserver : public test::EndToEndTest {
512 public:
513 // Some of the test cases are expected to time out.
514 // Use a shorter timeout window than the default one for those.
515 static constexpr TimeDelta kReducedTimeout = TimeDelta::Seconds(10);
516
UlpfecObserver(bool header_extensions_enabled,bool use_nack,bool expect_red,bool expect_ulpfec,const std::string & codec,VideoEncoderFactory * encoder_factory)517 UlpfecObserver(bool header_extensions_enabled,
518 bool use_nack,
519 bool expect_red,
520 bool expect_ulpfec,
521 const std::string& codec,
522 VideoEncoderFactory* encoder_factory)
523 : EndToEndTest(expect_ulpfec ? VideoSendStreamTest::kDefaultTimeout
524 : kReducedTimeout),
525 encoder_factory_(encoder_factory),
526 payload_name_(codec),
527 use_nack_(use_nack),
528 expect_red_(expect_red),
529 expect_ulpfec_(expect_ulpfec),
530 sent_media_(false),
531 sent_ulpfec_(false),
532 header_extensions_enabled_(header_extensions_enabled) {
533 extensions_.Register<AbsoluteSendTime>(kAbsSendTimeExtensionId);
534 extensions_.Register<TransportSequenceNumber>(
535 kTransportSequenceNumberExtensionId);
536 }
537
538 private:
OnSendRtp(const uint8_t * packet,size_t length)539 Action OnSendRtp(const uint8_t* packet, size_t length) override {
540 RtpPacket rtp_packet(&extensions_);
541 EXPECT_TRUE(rtp_packet.Parse(packet, length));
542
543 int encapsulated_payload_type = -1;
544 if (rtp_packet.PayloadType() == VideoSendStreamTest::kRedPayloadType) {
545 EXPECT_TRUE(expect_red_);
546 encapsulated_payload_type = rtp_packet.payload()[0];
547 if (encapsulated_payload_type !=
548 VideoSendStreamTest::kFakeVideoSendPayloadType) {
549 EXPECT_EQ(VideoSendStreamTest::kUlpfecPayloadType,
550 encapsulated_payload_type);
551 }
552 } else {
553 EXPECT_EQ(VideoSendStreamTest::kFakeVideoSendPayloadType,
554 rtp_packet.PayloadType());
555 if (rtp_packet.payload_size() > 0) {
556 // Not padding-only, media received outside of RED.
557 EXPECT_FALSE(expect_red_);
558 sent_media_ = true;
559 }
560 }
561
562 if (header_extensions_enabled_) {
563 uint32_t abs_send_time;
564 EXPECT_TRUE(rtp_packet.GetExtension<AbsoluteSendTime>(&abs_send_time));
565 uint16_t transport_seq_num;
566 EXPECT_TRUE(
567 rtp_packet.GetExtension<TransportSequenceNumber>(&transport_seq_num));
568 if (!first_packet_) {
569 uint32_t kHalf24BitsSpace = 0xFFFFFF / 2;
570 if (abs_send_time <= kHalf24BitsSpace &&
571 prev_abs_send_time_ > kHalf24BitsSpace) {
572 // 24 bits wrap.
573 EXPECT_GT(prev_abs_send_time_, abs_send_time);
574 } else {
575 EXPECT_GE(abs_send_time, prev_abs_send_time_);
576 }
577
578 uint16_t seq_num_diff = transport_seq_num - prev_transport_seq_num_;
579 EXPECT_EQ(1, seq_num_diff);
580 }
581 first_packet_ = false;
582 prev_abs_send_time_ = abs_send_time;
583 prev_transport_seq_num_ = transport_seq_num;
584 }
585
586 if (encapsulated_payload_type != -1) {
587 if (encapsulated_payload_type ==
588 VideoSendStreamTest::kUlpfecPayloadType) {
589 EXPECT_TRUE(expect_ulpfec_);
590 sent_ulpfec_ = true;
591 } else {
592 sent_media_ = true;
593 }
594 }
595
596 if (sent_media_ && sent_ulpfec_) {
597 observation_complete_.Set();
598 }
599
600 return SEND_PACKET;
601 }
602
CreateSendTransport(TaskQueueBase * task_queue,Call * sender_call)603 std::unique_ptr<test::PacketTransport> CreateSendTransport(
604 TaskQueueBase* task_queue,
605 Call* sender_call) override {
606 // At low RTT (< kLowRttNackMs) -> NACK only, no FEC.
607 // Configure some network delay.
608 const int kNetworkDelayMs = 100;
609 BuiltInNetworkBehaviorConfig config;
610 config.loss_percent = 5;
611 config.queue_delay_ms = kNetworkDelayMs;
612 return std::make_unique<test::PacketTransport>(
613 task_queue, sender_call, this, test::PacketTransport::kSender,
614 VideoSendStreamTest::payload_type_map_,
615 std::make_unique<FakeNetworkPipe>(
616 Clock::GetRealTimeClock(),
617 std::make_unique<SimulatedNetwork>(config)));
618 }
619
ModifyVideoConfigs(VideoSendStream::Config * send_config,std::vector<VideoReceiveStreamInterface::Config> * receive_configs,VideoEncoderConfig * encoder_config)620 void ModifyVideoConfigs(
621 VideoSendStream::Config* send_config,
622 std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
623 VideoEncoderConfig* encoder_config) override {
624 if (use_nack_) {
625 send_config->rtp.nack.rtp_history_ms =
626 (*receive_configs)[0].rtp.nack.rtp_history_ms =
627 VideoSendStreamTest::kNackRtpHistoryMs;
628 }
629 send_config->encoder_settings.encoder_factory = encoder_factory_;
630 send_config->rtp.payload_name = payload_name_;
631 send_config->rtp.ulpfec.red_payload_type =
632 VideoSendStreamTest::kRedPayloadType;
633 send_config->rtp.ulpfec.ulpfec_payload_type =
634 VideoSendStreamTest::kUlpfecPayloadType;
635 if (!header_extensions_enabled_) {
636 send_config->rtp.extensions.clear();
637 } else {
638 send_config->rtp.extensions.push_back(
639 RtpExtension(RtpExtension::kAbsSendTimeUri, kAbsSendTimeExtensionId));
640 }
641 (*receive_configs)[0].rtp.extensions = send_config->rtp.extensions;
642 encoder_config->codec_type = PayloadStringToCodecType(payload_name_);
643 (*receive_configs)[0].rtp.red_payload_type =
644 send_config->rtp.ulpfec.red_payload_type;
645 (*receive_configs)[0].rtp.ulpfec_payload_type =
646 send_config->rtp.ulpfec.ulpfec_payload_type;
647 }
648
PerformTest()649 void PerformTest() override {
650 EXPECT_EQ(expect_ulpfec_, Wait())
651 << "Timed out waiting for ULPFEC and/or media packets.";
652 }
653
654 VideoEncoderFactory* encoder_factory_;
655 RtpHeaderExtensionMap extensions_;
656 const std::string payload_name_;
657 const bool use_nack_;
658 const bool expect_red_;
659 const bool expect_ulpfec_;
660 bool sent_media_;
661 bool sent_ulpfec_;
662 const bool header_extensions_enabled_;
663 bool first_packet_ = true;
664 uint32_t prev_abs_send_time_ = 0;
665 uint16_t prev_transport_seq_num_ = 0;
666 };
667
TEST_F(VideoSendStreamTest,SupportsUlpfecWithExtensions)668 TEST_F(VideoSendStreamTest, SupportsUlpfecWithExtensions) {
669 test::FunctionVideoEncoderFactory encoder_factory(
670 []() { return VP8Encoder::Create(); });
671 UlpfecObserver test(true, false, true, true, "VP8", &encoder_factory);
672 RunBaseTest(&test);
673 }
674
TEST_F(VideoSendStreamTest,SupportsUlpfecWithoutExtensions)675 TEST_F(VideoSendStreamTest, SupportsUlpfecWithoutExtensions) {
676 test::FunctionVideoEncoderFactory encoder_factory(
677 []() { return VP8Encoder::Create(); });
678 UlpfecObserver test(false, false, true, true, "VP8", &encoder_factory);
679 RunBaseTest(&test);
680 }
681
682 class VideoSendStreamWithoutUlpfecTest : public test::CallTest {
683 protected:
VideoSendStreamWithoutUlpfecTest()684 VideoSendStreamWithoutUlpfecTest()
685 : field_trial_(field_trials_, "WebRTC-DisableUlpFecExperiment/Enabled/") {
686 }
687
688 test::ScopedKeyValueConfig field_trial_;
689 };
690
TEST_F(VideoSendStreamWithoutUlpfecTest,NoUlpfecIfDisabledThroughFieldTrial)691 TEST_F(VideoSendStreamWithoutUlpfecTest, NoUlpfecIfDisabledThroughFieldTrial) {
692 test::FunctionVideoEncoderFactory encoder_factory(
693 []() { return VP8Encoder::Create(); });
694 UlpfecObserver test(false, false, false, false, "VP8", &encoder_factory);
695 RunBaseTest(&test);
696 }
697
698 // The FEC scheme used is not efficient for H264, so we should not use RED/FEC
699 // since we'll still have to re-request FEC packets, effectively wasting
700 // bandwidth since the receiver has to wait for FEC retransmissions to determine
701 // that the received state is actually decodable.
TEST_F(VideoSendStreamTest,DoesNotUtilizeUlpfecForH264WithNackEnabled)702 TEST_F(VideoSendStreamTest, DoesNotUtilizeUlpfecForH264WithNackEnabled) {
703 test::FunctionVideoEncoderFactory encoder_factory([]() {
704 return std::make_unique<test::FakeH264Encoder>(Clock::GetRealTimeClock());
705 });
706 UlpfecObserver test(false, true, false, false, "H264", &encoder_factory);
707 RunBaseTest(&test);
708 }
709
710 // Without retransmissions FEC for H264 is fine.
TEST_F(VideoSendStreamTest,DoesUtilizeUlpfecForH264WithoutNackEnabled)711 TEST_F(VideoSendStreamTest, DoesUtilizeUlpfecForH264WithoutNackEnabled) {
712 test::FunctionVideoEncoderFactory encoder_factory([]() {
713 return std::make_unique<test::FakeH264Encoder>(Clock::GetRealTimeClock());
714 });
715 UlpfecObserver test(false, false, true, true, "H264", &encoder_factory);
716 RunBaseTest(&test);
717 }
718
TEST_F(VideoSendStreamTest,DoesUtilizeUlpfecForVp8WithNackEnabled)719 TEST_F(VideoSendStreamTest, DoesUtilizeUlpfecForVp8WithNackEnabled) {
720 test::FunctionVideoEncoderFactory encoder_factory(
721 []() { return VP8Encoder::Create(); });
722 UlpfecObserver test(false, true, true, true, "VP8", &encoder_factory);
723 RunBaseTest(&test);
724 }
725
726 #if defined(RTC_ENABLE_VP9)
TEST_F(VideoSendStreamTest,DoesUtilizeUlpfecForVp9WithNackEnabled)727 TEST_F(VideoSendStreamTest, DoesUtilizeUlpfecForVp9WithNackEnabled) {
728 test::FunctionVideoEncoderFactory encoder_factory(
729 []() { return VP9Encoder::Create(); });
730 UlpfecObserver test(false, true, true, true, "VP9", &encoder_factory);
731 RunBaseTest(&test);
732 }
733 #endif // defined(RTC_ENABLE_VP9)
734
TEST_F(VideoSendStreamTest,SupportsUlpfecWithMultithreadedH264)735 TEST_F(VideoSendStreamTest, SupportsUlpfecWithMultithreadedH264) {
736 std::unique_ptr<TaskQueueFactory> task_queue_factory =
737 CreateDefaultTaskQueueFactory();
738 test::FunctionVideoEncoderFactory encoder_factory([&]() {
739 return std::make_unique<test::MultithreadedFakeH264Encoder>(
740 Clock::GetRealTimeClock(), task_queue_factory.get());
741 });
742 UlpfecObserver test(false, false, true, true, "H264", &encoder_factory);
743 RunBaseTest(&test);
744 }
745
746 // TODO(brandtr): Move these FlexFEC tests when we have created
747 // FlexfecSendStream.
748 class FlexfecObserver : public test::EndToEndTest {
749 public:
FlexfecObserver(bool header_extensions_enabled,bool use_nack,const std::string & codec,VideoEncoderFactory * encoder_factory,size_t num_video_streams)750 FlexfecObserver(bool header_extensions_enabled,
751 bool use_nack,
752 const std::string& codec,
753 VideoEncoderFactory* encoder_factory,
754 size_t num_video_streams)
755 : EndToEndTest(VideoSendStreamTest::kDefaultTimeout),
756 encoder_factory_(encoder_factory),
757 payload_name_(codec),
758 use_nack_(use_nack),
759 sent_media_(false),
760 sent_flexfec_(false),
761 header_extensions_enabled_(header_extensions_enabled),
762 num_video_streams_(num_video_streams) {
763 extensions_.Register<AbsoluteSendTime>(kAbsSendTimeExtensionId);
764 extensions_.Register<TransmissionOffset>(kTimestampOffsetExtensionId);
765 extensions_.Register<TransportSequenceNumber>(
766 kTransportSequenceNumberExtensionId);
767 }
768
GetNumFlexfecStreams() const769 size_t GetNumFlexfecStreams() const override { return 1; }
GetNumVideoStreams() const770 size_t GetNumVideoStreams() const override { return num_video_streams_; }
771
772 private:
OnSendRtp(const uint8_t * packet,size_t length)773 Action OnSendRtp(const uint8_t* packet, size_t length) override {
774 RtpPacket rtp_packet(&extensions_);
775 EXPECT_TRUE(rtp_packet.Parse(packet, length));
776
777 if (rtp_packet.PayloadType() == VideoSendStreamTest::kFlexfecPayloadType) {
778 EXPECT_EQ(VideoSendStreamTest::kFlexfecSendSsrc, rtp_packet.Ssrc());
779 sent_flexfec_ = true;
780 } else {
781 EXPECT_EQ(VideoSendStreamTest::kFakeVideoSendPayloadType,
782 rtp_packet.PayloadType());
783 EXPECT_THAT(::testing::make_tuple(VideoSendStreamTest::kVideoSendSsrcs,
784 num_video_streams_),
785 ::testing::Contains(rtp_packet.Ssrc()));
786 sent_media_ = true;
787 }
788
789 if (header_extensions_enabled_) {
790 EXPECT_TRUE(rtp_packet.HasExtension<AbsoluteSendTime>());
791 EXPECT_TRUE(rtp_packet.HasExtension<TransmissionOffset>());
792 EXPECT_TRUE(rtp_packet.HasExtension<TransportSequenceNumber>());
793 }
794
795 if (sent_media_ && sent_flexfec_) {
796 observation_complete_.Set();
797 }
798
799 return SEND_PACKET;
800 }
801
CreateSendTransport(TaskQueueBase * task_queue,Call * sender_call)802 std::unique_ptr<test::PacketTransport> CreateSendTransport(
803 TaskQueueBase* task_queue,
804 Call* sender_call) override {
805 // At low RTT (< kLowRttNackMs) -> NACK only, no FEC.
806 // Therefore we need some network delay.
807 const int kNetworkDelayMs = 100;
808 BuiltInNetworkBehaviorConfig config;
809 config.loss_percent = 5;
810 config.queue_delay_ms = kNetworkDelayMs;
811 return std::make_unique<test::PacketTransport>(
812 task_queue, sender_call, this, test::PacketTransport::kSender,
813 VideoSendStreamTest::payload_type_map_,
814 std::make_unique<FakeNetworkPipe>(
815 Clock::GetRealTimeClock(),
816 std::make_unique<SimulatedNetwork>(config)));
817 }
818
CreateReceiveTransport(TaskQueueBase * task_queue)819 std::unique_ptr<test::PacketTransport> CreateReceiveTransport(
820 TaskQueueBase* task_queue) override {
821 // We need the RTT to be >200 ms to send FEC and the network delay for the
822 // send transport is 100 ms, so add 100 ms (but no loss) on the return link.
823 BuiltInNetworkBehaviorConfig config;
824 config.loss_percent = 0;
825 config.queue_delay_ms = 100;
826 return std::make_unique<test::PacketTransport>(
827 task_queue, nullptr, this, test::PacketTransport::kReceiver,
828 VideoSendStreamTest::payload_type_map_,
829 std::make_unique<FakeNetworkPipe>(
830 Clock::GetRealTimeClock(),
831 std::make_unique<SimulatedNetwork>(config)));
832 }
833
ModifyVideoConfigs(VideoSendStream::Config * send_config,std::vector<VideoReceiveStreamInterface::Config> * receive_configs,VideoEncoderConfig * encoder_config)834 void ModifyVideoConfigs(
835 VideoSendStream::Config* send_config,
836 std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
837 VideoEncoderConfig* encoder_config) override {
838 if (use_nack_) {
839 send_config->rtp.nack.rtp_history_ms =
840 (*receive_configs)[0].rtp.nack.rtp_history_ms =
841 VideoSendStreamTest::kNackRtpHistoryMs;
842 }
843 send_config->encoder_settings.encoder_factory = encoder_factory_;
844 send_config->rtp.payload_name = payload_name_;
845 if (header_extensions_enabled_) {
846 send_config->rtp.extensions.push_back(
847 RtpExtension(RtpExtension::kAbsSendTimeUri, kAbsSendTimeExtensionId));
848 send_config->rtp.extensions.push_back(RtpExtension(
849 RtpExtension::kTimestampOffsetUri, kTimestampOffsetExtensionId));
850 } else {
851 send_config->rtp.extensions.clear();
852 }
853 (*receive_configs)[0].rtp.extensions = send_config->rtp.extensions;
854 encoder_config->codec_type = PayloadStringToCodecType(payload_name_);
855 }
856
PerformTest()857 void PerformTest() override {
858 EXPECT_TRUE(Wait())
859 << "Timed out waiting for FlexFEC and/or media packets.";
860 }
861
862 VideoEncoderFactory* encoder_factory_;
863 RtpHeaderExtensionMap extensions_;
864 const std::string payload_name_;
865 const bool use_nack_;
866 bool sent_media_;
867 bool sent_flexfec_;
868 const bool header_extensions_enabled_;
869 const size_t num_video_streams_;
870 };
871
TEST_F(VideoSendStreamTest,SupportsFlexfecVp8)872 TEST_F(VideoSendStreamTest, SupportsFlexfecVp8) {
873 test::FunctionVideoEncoderFactory encoder_factory(
874 []() { return VP8Encoder::Create(); });
875 FlexfecObserver test(false, false, "VP8", &encoder_factory, 1);
876 RunBaseTest(&test);
877 }
878
TEST_F(VideoSendStreamTest,SupportsFlexfecSimulcastVp8)879 TEST_F(VideoSendStreamTest, SupportsFlexfecSimulcastVp8) {
880 test::FunctionVideoEncoderFactory encoder_factory(
881 []() { return VP8Encoder::Create(); });
882 FlexfecObserver test(false, false, "VP8", &encoder_factory, 2);
883 RunBaseTest(&test);
884 }
885
TEST_F(VideoSendStreamTest,SupportsFlexfecWithNackVp8)886 TEST_F(VideoSendStreamTest, SupportsFlexfecWithNackVp8) {
887 test::FunctionVideoEncoderFactory encoder_factory(
888 []() { return VP8Encoder::Create(); });
889 FlexfecObserver test(false, true, "VP8", &encoder_factory, 1);
890 RunBaseTest(&test);
891 }
892
TEST_F(VideoSendStreamTest,SupportsFlexfecWithRtpExtensionsVp8)893 TEST_F(VideoSendStreamTest, SupportsFlexfecWithRtpExtensionsVp8) {
894 test::FunctionVideoEncoderFactory encoder_factory(
895 []() { return VP8Encoder::Create(); });
896 FlexfecObserver test(true, false, "VP8", &encoder_factory, 1);
897 RunBaseTest(&test);
898 }
899
900 #if defined(RTC_ENABLE_VP9)
TEST_F(VideoSendStreamTest,SupportsFlexfecVp9)901 TEST_F(VideoSendStreamTest, SupportsFlexfecVp9) {
902 test::FunctionVideoEncoderFactory encoder_factory(
903 []() { return VP9Encoder::Create(); });
904 FlexfecObserver test(false, false, "VP9", &encoder_factory, 1);
905 RunBaseTest(&test);
906 }
907
TEST_F(VideoSendStreamTest,SupportsFlexfecWithNackVp9)908 TEST_F(VideoSendStreamTest, SupportsFlexfecWithNackVp9) {
909 test::FunctionVideoEncoderFactory encoder_factory(
910 []() { return VP9Encoder::Create(); });
911 FlexfecObserver test(false, true, "VP9", &encoder_factory, 1);
912 RunBaseTest(&test);
913 }
914 #endif // defined(RTC_ENABLE_VP9)
915
TEST_F(VideoSendStreamTest,SupportsFlexfecH264)916 TEST_F(VideoSendStreamTest, SupportsFlexfecH264) {
917 test::FunctionVideoEncoderFactory encoder_factory([]() {
918 return std::make_unique<test::FakeH264Encoder>(Clock::GetRealTimeClock());
919 });
920 FlexfecObserver test(false, false, "H264", &encoder_factory, 1);
921 RunBaseTest(&test);
922 }
923
TEST_F(VideoSendStreamTest,SupportsFlexfecWithNackH264)924 TEST_F(VideoSendStreamTest, SupportsFlexfecWithNackH264) {
925 test::FunctionVideoEncoderFactory encoder_factory([]() {
926 return std::make_unique<test::FakeH264Encoder>(Clock::GetRealTimeClock());
927 });
928 FlexfecObserver test(false, true, "H264", &encoder_factory, 1);
929 RunBaseTest(&test);
930 }
931
TEST_F(VideoSendStreamTest,SupportsFlexfecWithMultithreadedH264)932 TEST_F(VideoSendStreamTest, SupportsFlexfecWithMultithreadedH264) {
933 std::unique_ptr<TaskQueueFactory> task_queue_factory =
934 CreateDefaultTaskQueueFactory();
935 test::FunctionVideoEncoderFactory encoder_factory([&]() {
936 return std::make_unique<test::MultithreadedFakeH264Encoder>(
937 Clock::GetRealTimeClock(), task_queue_factory.get());
938 });
939
940 FlexfecObserver test(false, false, "H264", &encoder_factory, 1);
941 RunBaseTest(&test);
942 }
943
TestNackRetransmission(uint32_t retransmit_ssrc,uint8_t retransmit_payload_type)944 void VideoSendStreamTest::TestNackRetransmission(
945 uint32_t retransmit_ssrc,
946 uint8_t retransmit_payload_type) {
947 class NackObserver : public test::SendTest {
948 public:
949 explicit NackObserver(uint32_t retransmit_ssrc,
950 uint8_t retransmit_payload_type)
951 : SendTest(kDefaultTimeout),
952 send_count_(0),
953 retransmit_count_(0),
954 retransmit_ssrc_(retransmit_ssrc),
955 retransmit_payload_type_(retransmit_payload_type) {}
956
957 private:
958 Action OnSendRtp(const uint8_t* packet, size_t length) override {
959 RtpPacket rtp_packet;
960 EXPECT_TRUE(rtp_packet.Parse(packet, length));
961
962 // NACK packets two times at some arbitrary points.
963 const int kNackedPacketsAtOnceCount = 3;
964 const int kRetransmitTarget = kNackedPacketsAtOnceCount * 2;
965
966 // Skip padding packets because they will never be retransmitted.
967 if (rtp_packet.payload_size() == 0) {
968 return SEND_PACKET;
969 }
970
971 ++send_count_;
972
973 // NACK packets at arbitrary points.
974 if (send_count_ % 25 == 0) {
975 RTCPSender::Configuration config;
976 config.clock = Clock::GetRealTimeClock();
977 config.outgoing_transport = transport_adapter_.get();
978 config.rtcp_report_interval = TimeDelta::Millis(kRtcpIntervalMs);
979 config.local_media_ssrc = kReceiverLocalVideoSsrc;
980 RTCPSender rtcp_sender(config);
981
982 rtcp_sender.SetRTCPStatus(RtcpMode::kReducedSize);
983 rtcp_sender.SetRemoteSSRC(kVideoSendSsrcs[0]);
984
985 RTCPSender::FeedbackState feedback_state;
986 uint16_t nack_sequence_numbers[kNackedPacketsAtOnceCount];
987 int nack_count = 0;
988 for (uint16_t sequence_number :
989 sequence_numbers_pending_retransmission_) {
990 if (nack_count < kNackedPacketsAtOnceCount) {
991 nack_sequence_numbers[nack_count++] = sequence_number;
992 } else {
993 break;
994 }
995 }
996
997 EXPECT_EQ(0, rtcp_sender.SendRTCP(feedback_state, kRtcpNack, nack_count,
998 nack_sequence_numbers));
999 }
1000
1001 uint16_t sequence_number = rtp_packet.SequenceNumber();
1002 if (rtp_packet.Ssrc() == retransmit_ssrc_ &&
1003 retransmit_ssrc_ != kVideoSendSsrcs[0]) {
1004 // Not kVideoSendSsrcs[0], assume correct RTX packet. Extract sequence
1005 // number.
1006 const uint8_t* rtx_header = rtp_packet.payload().data();
1007 sequence_number = (rtx_header[0] << 8) + rtx_header[1];
1008 }
1009
1010 auto it = sequence_numbers_pending_retransmission_.find(sequence_number);
1011 if (it == sequence_numbers_pending_retransmission_.end()) {
1012 // Not currently pending retransmission. Add it to retransmission queue
1013 // if media and limit not reached.
1014 if (rtp_packet.Ssrc() == kVideoSendSsrcs[0] &&
1015 rtp_packet.payload_size() > 0 &&
1016 retransmit_count_ +
1017 sequence_numbers_pending_retransmission_.size() <
1018 kRetransmitTarget) {
1019 sequence_numbers_pending_retransmission_.insert(sequence_number);
1020 }
1021 } else {
1022 // Packet is a retransmission, remove it from queue and check if done.
1023 sequence_numbers_pending_retransmission_.erase(it);
1024 if (++retransmit_count_ == kRetransmitTarget) {
1025 EXPECT_EQ(retransmit_ssrc_, rtp_packet.Ssrc());
1026 EXPECT_EQ(retransmit_payload_type_, rtp_packet.PayloadType());
1027 observation_complete_.Set();
1028 }
1029 }
1030
1031 return SEND_PACKET;
1032 }
1033
1034 void ModifyVideoConfigs(
1035 VideoSendStream::Config* send_config,
1036 std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
1037 VideoEncoderConfig* encoder_config) override {
1038 transport_adapter_.reset(
1039 new internal::TransportAdapter(send_config->send_transport));
1040 transport_adapter_->Enable();
1041 send_config->rtp.nack.rtp_history_ms = kNackRtpHistoryMs;
1042 send_config->rtp.rtx.payload_type = retransmit_payload_type_;
1043 if (retransmit_ssrc_ != kVideoSendSsrcs[0])
1044 send_config->rtp.rtx.ssrcs.push_back(retransmit_ssrc_);
1045 }
1046
1047 void PerformTest() override {
1048 EXPECT_TRUE(Wait()) << "Timed out while waiting for NACK retransmission.";
1049 }
1050
1051 std::unique_ptr<internal::TransportAdapter> transport_adapter_;
1052 int send_count_;
1053 int retransmit_count_;
1054 const uint32_t retransmit_ssrc_;
1055 const uint8_t retransmit_payload_type_;
1056 std::set<uint16_t> sequence_numbers_pending_retransmission_;
1057 } test(retransmit_ssrc, retransmit_payload_type);
1058
1059 RunBaseTest(&test);
1060 }
1061
TEST_F(VideoSendStreamTest,RetransmitsNack)1062 TEST_F(VideoSendStreamTest, RetransmitsNack) {
1063 // Normal NACKs should use the send SSRC.
1064 TestNackRetransmission(kVideoSendSsrcs[0], kFakeVideoSendPayloadType);
1065 }
1066
TEST_F(VideoSendStreamTest,RetransmitsNackOverRtx)1067 TEST_F(VideoSendStreamTest, RetransmitsNackOverRtx) {
1068 // NACKs over RTX should use a separate SSRC.
1069 TestNackRetransmission(kSendRtxSsrcs[0], kSendRtxPayloadType);
1070 }
1071
TestPacketFragmentationSize(VideoFormat format,bool with_fec)1072 void VideoSendStreamTest::TestPacketFragmentationSize(VideoFormat format,
1073 bool with_fec) {
1074 // Use a fake encoder to output a frame of every size in the range [90, 290],
1075 // for each size making sure that the exact number of payload bytes received
1076 // is correct and that packets are fragmented to respect max packet size.
1077 static const size_t kMaxPacketSize = 128;
1078 static const size_t start = 90;
1079 static const size_t stop = 290;
1080
1081 // Observer that verifies that the expected number of packets and bytes
1082 // arrive for each frame size, from start_size to stop_size.
1083 class FrameFragmentationTest : public test::SendTest {
1084 public:
1085 FrameFragmentationTest(size_t max_packet_size,
1086 size_t start_size,
1087 size_t stop_size,
1088 bool test_generic_packetization,
1089 bool use_fec)
1090 : SendTest(kLongTimeout),
1091 encoder_(stop),
1092 encoder_factory_(&encoder_),
1093 max_packet_size_(max_packet_size),
1094 stop_size_(stop_size),
1095 test_generic_packetization_(test_generic_packetization),
1096 use_fec_(use_fec),
1097 packet_count_(0),
1098 packets_lost_(0),
1099 last_packet_count_(0),
1100 last_packets_lost_(0),
1101 accumulated_size_(0),
1102 accumulated_payload_(0),
1103 fec_packet_received_(false),
1104 current_size_rtp_(start_size),
1105 current_size_frame_(static_cast<int>(start_size)) {
1106 // Fragmentation required, this test doesn't make sense without it.
1107 encoder_.SetFrameSize(start_size);
1108 RTC_DCHECK_GT(stop_size, max_packet_size);
1109 if (!test_generic_packetization_)
1110 encoder_.SetCodecType(kVideoCodecVP8);
1111 }
1112
1113 private:
1114 Action OnSendRtp(const uint8_t* packet, size_t size) override {
1115 size_t length = size;
1116 RtpPacket rtp_packet;
1117 EXPECT_TRUE(rtp_packet.Parse(packet, length));
1118
1119 EXPECT_LE(length, max_packet_size_);
1120
1121 if (use_fec_ && rtp_packet.payload_size() > 0) {
1122 uint8_t payload_type = rtp_packet.payload()[0];
1123 bool is_fec = rtp_packet.PayloadType() == kRedPayloadType &&
1124 payload_type == kUlpfecPayloadType;
1125 if (is_fec) {
1126 fec_packet_received_ = true;
1127 return SEND_PACKET;
1128 }
1129 }
1130
1131 accumulated_size_ += length;
1132
1133 if (use_fec_)
1134 TriggerLossReport(rtp_packet);
1135
1136 if (test_generic_packetization_) {
1137 size_t overhead = rtp_packet.headers_size() + rtp_packet.padding_size();
1138 // Only remove payload header and RED header if the packet actually
1139 // contains payload.
1140 if (length > overhead) {
1141 overhead += (1 /* Generic header */);
1142 if (use_fec_)
1143 overhead += 1; // RED for FEC header.
1144 }
1145 EXPECT_GE(length, overhead);
1146 accumulated_payload_ += length - overhead;
1147 }
1148
1149 // Marker bit set indicates last packet of a frame.
1150 if (rtp_packet.Marker()) {
1151 if (use_fec_ && accumulated_payload_ == current_size_rtp_ - 1) {
1152 // With FEC enabled, frame size is incremented asynchronously, so
1153 // "old" frames one byte too small may arrive. Accept, but don't
1154 // increase expected frame size.
1155 accumulated_size_ = 0;
1156 accumulated_payload_ = 0;
1157 return SEND_PACKET;
1158 }
1159
1160 EXPECT_GE(accumulated_size_, current_size_rtp_);
1161 if (test_generic_packetization_) {
1162 EXPECT_EQ(current_size_rtp_, accumulated_payload_);
1163 }
1164
1165 // Last packet of frame; reset counters.
1166 accumulated_size_ = 0;
1167 accumulated_payload_ = 0;
1168 if (current_size_rtp_ == stop_size_) {
1169 // Done! (Don't increase size again, might arrive more @ stop_size).
1170 observation_complete_.Set();
1171 } else {
1172 // Increase next expected frame size. If testing with FEC, make sure
1173 // a FEC packet has been received for this frame size before
1174 // proceeding, to make sure that redundancy packets don't exceed
1175 // size limit.
1176 if (!use_fec_) {
1177 ++current_size_rtp_;
1178 } else if (fec_packet_received_) {
1179 fec_packet_received_ = false;
1180 ++current_size_rtp_;
1181
1182 MutexLock lock(&mutex_);
1183 ++current_size_frame_;
1184 }
1185 }
1186 }
1187
1188 return SEND_PACKET;
1189 }
1190
1191 void TriggerLossReport(const RtpPacket& rtp_packet) {
1192 // Send lossy receive reports to trigger FEC enabling.
1193 const int kLossPercent = 5;
1194 if (++packet_count_ % (100 / kLossPercent) == 0) {
1195 packets_lost_++;
1196 int loss_delta = packets_lost_ - last_packets_lost_;
1197 int packets_delta = packet_count_ - last_packet_count_;
1198 last_packet_count_ = packet_count_;
1199 last_packets_lost_ = packets_lost_;
1200 uint8_t loss_ratio =
1201 static_cast<uint8_t>(loss_delta * 255 / packets_delta);
1202 FakeReceiveStatistics lossy_receive_stats(
1203 kVideoSendSsrcs[0], rtp_packet.SequenceNumber(),
1204 packets_lost_, // Cumulative lost.
1205 loss_ratio); // Loss percent.
1206 RTCPSender::Configuration config;
1207 config.clock = Clock::GetRealTimeClock();
1208 config.receive_statistics = &lossy_receive_stats;
1209 config.outgoing_transport = transport_adapter_.get();
1210 config.rtcp_report_interval = TimeDelta::Millis(kRtcpIntervalMs);
1211 config.local_media_ssrc = kVideoSendSsrcs[0];
1212 RTCPSender rtcp_sender(config);
1213
1214 rtcp_sender.SetRTCPStatus(RtcpMode::kReducedSize);
1215 rtcp_sender.SetRemoteSSRC(kVideoSendSsrcs[0]);
1216
1217 RTCPSender::FeedbackState feedback_state;
1218
1219 EXPECT_EQ(0, rtcp_sender.SendRTCP(feedback_state, kRtcpRr));
1220 }
1221 }
1222
1223 void UpdateConfiguration() {
1224 MutexLock lock(&mutex_);
1225 // Increase frame size for next encoded frame, in the context of the
1226 // encoder thread.
1227 if (!use_fec_ && current_size_frame_ < static_cast<int32_t>(stop_size_)) {
1228 ++current_size_frame_;
1229 }
1230 encoder_.SetFrameSize(static_cast<size_t>(current_size_frame_));
1231 }
1232 void ModifySenderBitrateConfig(
1233 BitrateConstraints* bitrate_config) override {
1234 const int kMinBitrateBps = 300000;
1235 bitrate_config->min_bitrate_bps = kMinBitrateBps;
1236 }
1237
1238 void ModifyVideoConfigs(
1239 VideoSendStream::Config* send_config,
1240 std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
1241 VideoEncoderConfig* encoder_config) override {
1242 transport_adapter_.reset(
1243 new internal::TransportAdapter(send_config->send_transport));
1244 transport_adapter_->Enable();
1245 if (use_fec_) {
1246 send_config->rtp.ulpfec.red_payload_type = kRedPayloadType;
1247 send_config->rtp.ulpfec.ulpfec_payload_type = kUlpfecPayloadType;
1248 }
1249
1250 if (!test_generic_packetization_)
1251 send_config->rtp.payload_name = "VP8";
1252
1253 send_config->encoder_settings.encoder_factory = &encoder_factory_;
1254 send_config->rtp.max_packet_size = kMaxPacketSize;
1255 encoder_.RegisterPostEncodeCallback([this]() { UpdateConfiguration(); });
1256
1257 // Make sure there is at least one extension header, to make the RTP
1258 // header larger than the base length of 12 bytes.
1259 EXPECT_FALSE(send_config->rtp.extensions.empty());
1260
1261 // Setup screen content disables frame dropping which makes this easier.
1262 EXPECT_EQ(1u, encoder_config->simulcast_layers.size());
1263 encoder_config->simulcast_layers[0].num_temporal_layers = 2;
1264 encoder_config->content_type = VideoEncoderConfig::ContentType::kScreen;
1265 }
1266
1267 void PerformTest() override {
1268 EXPECT_TRUE(Wait()) << "Timed out while observing incoming RTP packets.";
1269 }
1270
1271 std::unique_ptr<internal::TransportAdapter> transport_adapter_;
1272 test::ConfigurableFrameSizeEncoder encoder_;
1273 test::VideoEncoderProxyFactory encoder_factory_;
1274
1275 const size_t max_packet_size_;
1276 const size_t stop_size_;
1277 const bool test_generic_packetization_;
1278 const bool use_fec_;
1279
1280 uint32_t packet_count_;
1281 uint32_t packets_lost_;
1282 uint32_t last_packet_count_;
1283 uint32_t last_packets_lost_;
1284 size_t accumulated_size_;
1285 size_t accumulated_payload_;
1286 bool fec_packet_received_;
1287
1288 size_t current_size_rtp_;
1289 Mutex mutex_;
1290 int current_size_frame_ RTC_GUARDED_BY(mutex_);
1291 };
1292
1293 // Don't auto increment if FEC is used; continue sending frame size until
1294 // a FEC packet has been received.
1295 FrameFragmentationTest test(kMaxPacketSize, start, stop, format == kGeneric,
1296 with_fec);
1297
1298 RunBaseTest(&test);
1299 }
1300
1301 // TODO(sprang): Is there any way of speeding up these tests?
TEST_F(VideoSendStreamTest,FragmentsGenericAccordingToMaxPacketSize)1302 TEST_F(VideoSendStreamTest, FragmentsGenericAccordingToMaxPacketSize) {
1303 TestPacketFragmentationSize(kGeneric, false);
1304 }
1305
TEST_F(VideoSendStreamTest,FragmentsGenericAccordingToMaxPacketSizeWithFec)1306 TEST_F(VideoSendStreamTest, FragmentsGenericAccordingToMaxPacketSizeWithFec) {
1307 TestPacketFragmentationSize(kGeneric, true);
1308 }
1309
TEST_F(VideoSendStreamTest,FragmentsVp8AccordingToMaxPacketSize)1310 TEST_F(VideoSendStreamTest, FragmentsVp8AccordingToMaxPacketSize) {
1311 TestPacketFragmentationSize(kVP8, false);
1312 }
1313
TEST_F(VideoSendStreamTest,FragmentsVp8AccordingToMaxPacketSizeWithFec)1314 TEST_F(VideoSendStreamTest, FragmentsVp8AccordingToMaxPacketSizeWithFec) {
1315 TestPacketFragmentationSize(kVP8, true);
1316 }
1317
1318 // This test that padding stops being send after a while if the Camera stops
1319 // producing video frames and that padding resumes if the camera restarts.
TEST_F(VideoSendStreamTest,NoPaddingWhenVideoIsMuted)1320 TEST_F(VideoSendStreamTest, NoPaddingWhenVideoIsMuted) {
1321 class NoPaddingWhenVideoIsMuted : public test::SendTest {
1322 public:
1323 NoPaddingWhenVideoIsMuted()
1324 : SendTest(kDefaultTimeout),
1325 clock_(Clock::GetRealTimeClock()),
1326 capturer_(nullptr) {}
1327
1328 private:
1329 Action OnSendRtp(const uint8_t* packet, size_t length) override {
1330 MutexLock lock(&mutex_);
1331 last_packet_time_ms_ = clock_->TimeInMilliseconds();
1332
1333 RtpPacket rtp_packet;
1334 rtp_packet.Parse(packet, length);
1335 const bool only_padding = rtp_packet.payload_size() == 0;
1336
1337 if (test_state_ == kBeforeStopCapture) {
1338 // Packets are flowing, stop camera.
1339 capturer_->Stop();
1340 test_state_ = kWaitingForPadding;
1341 } else if (test_state_ == kWaitingForPadding && only_padding) {
1342 // We're still getting padding, after stopping camera.
1343 test_state_ = kWaitingForNoPackets;
1344 } else if (test_state_ == kWaitingForMediaAfterCameraRestart &&
1345 !only_padding) {
1346 // Media packets are flowing again, stop camera a second time.
1347 capturer_->Stop();
1348 test_state_ = kWaitingForPaddingAfterCameraStopsAgain;
1349 } else if (test_state_ == kWaitingForPaddingAfterCameraStopsAgain &&
1350 only_padding) {
1351 // Padding is still flowing, test ok.
1352 observation_complete_.Set();
1353 }
1354 return SEND_PACKET;
1355 }
1356
1357 Action OnSendRtcp(const uint8_t* packet, size_t length) override {
1358 MutexLock lock(&mutex_);
1359 const int kNoPacketsThresholdMs = 2000;
1360 if (test_state_ == kWaitingForNoPackets &&
1361 (last_packet_time_ms_ &&
1362 clock_->TimeInMilliseconds() - last_packet_time_ms_.value() >
1363 kNoPacketsThresholdMs)) {
1364 // No packets seen for `kNoPacketsThresholdMs`, restart camera.
1365 capturer_->Start();
1366 test_state_ = kWaitingForMediaAfterCameraRestart;
1367 }
1368 return SEND_PACKET;
1369 }
1370
1371 void ModifyVideoConfigs(
1372 VideoSendStream::Config* send_config,
1373 std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
1374 VideoEncoderConfig* encoder_config) override {
1375 // Make sure padding is sent if encoder is not producing media.
1376 encoder_config->min_transmit_bitrate_bps = 50000;
1377 }
1378
1379 void OnFrameGeneratorCapturerCreated(
1380 test::FrameGeneratorCapturer* frame_generator_capturer) override {
1381 MutexLock lock(&mutex_);
1382 capturer_ = frame_generator_capturer;
1383 }
1384
1385 void PerformTest() override {
1386 EXPECT_TRUE(Wait())
1387 << "Timed out while waiting for RTP packets to stop being sent.";
1388 }
1389
1390 enum TestState {
1391 kBeforeStopCapture,
1392 kWaitingForPadding,
1393 kWaitingForNoPackets,
1394 kWaitingForMediaAfterCameraRestart,
1395 kWaitingForPaddingAfterCameraStopsAgain
1396 };
1397
1398 TestState test_state_ = kBeforeStopCapture;
1399 Clock* const clock_;
1400 Mutex mutex_;
1401 absl::optional<int64_t> last_packet_time_ms_ RTC_GUARDED_BY(mutex_);
1402 test::FrameGeneratorCapturer* capturer_ RTC_GUARDED_BY(mutex_);
1403 } test;
1404
1405 RunBaseTest(&test);
1406 }
1407
TEST_F(VideoSendStreamTest,PaddingIsPrimarilyRetransmissions)1408 TEST_F(VideoSendStreamTest, PaddingIsPrimarilyRetransmissions) {
1409 const int kCapacityKbps = 10000; // 10 Mbps
1410 class PaddingIsPrimarilyRetransmissions : public test::EndToEndTest {
1411 public:
1412 PaddingIsPrimarilyRetransmissions()
1413 : EndToEndTest(kDefaultTimeout),
1414 clock_(Clock::GetRealTimeClock()),
1415 padding_length_(0),
1416 total_length_(0),
1417 call_(nullptr) {}
1418
1419 private:
1420 void OnCallsCreated(Call* sender_call, Call* receiver_call) override {
1421 call_ = sender_call;
1422 }
1423
1424 Action OnSendRtp(const uint8_t* packet, size_t length) override {
1425 MutexLock lock(&mutex_);
1426
1427 RtpPacket rtp_packet;
1428 rtp_packet.Parse(packet, length);
1429 padding_length_ += rtp_packet.padding_size();
1430 total_length_ += length;
1431 return SEND_PACKET;
1432 }
1433
1434 std::unique_ptr<test::PacketTransport> CreateSendTransport(
1435 TaskQueueBase* task_queue,
1436 Call* sender_call) override {
1437 const int kNetworkDelayMs = 50;
1438 BuiltInNetworkBehaviorConfig config;
1439 config.loss_percent = 10;
1440 config.link_capacity_kbps = kCapacityKbps;
1441 config.queue_delay_ms = kNetworkDelayMs;
1442 return std::make_unique<test::PacketTransport>(
1443 task_queue, sender_call, this, test::PacketTransport::kSender,
1444 payload_type_map_,
1445 std::make_unique<FakeNetworkPipe>(
1446 Clock::GetRealTimeClock(),
1447 std::make_unique<SimulatedNetwork>(config)));
1448 }
1449
1450 void ModifyVideoConfigs(
1451 VideoSendStream::Config* send_config,
1452 std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
1453 VideoEncoderConfig* encoder_config) override {
1454 // Turn on RTX.
1455 send_config->rtp.rtx.payload_type = kFakeVideoSendPayloadType;
1456 send_config->rtp.rtx.ssrcs.push_back(kSendRtxSsrcs[0]);
1457 }
1458
1459 void PerformTest() override {
1460 // TODO(isheriff): Some platforms do not ramp up as expected to full
1461 // capacity due to packet scheduling delays. Fix that before getting
1462 // rid of this.
1463 SleepMs(5000);
1464 {
1465 MutexLock lock(&mutex_);
1466 // Expect padding to be a small percentage of total bytes sent.
1467 EXPECT_LT(padding_length_, .1 * total_length_);
1468 }
1469 }
1470
1471 Mutex mutex_;
1472 Clock* const clock_;
1473 size_t padding_length_ RTC_GUARDED_BY(mutex_);
1474 size_t total_length_ RTC_GUARDED_BY(mutex_);
1475 Call* call_;
1476 } test;
1477
1478 RunBaseTest(&test);
1479 }
1480
1481 // This test first observes "high" bitrate use at which point it sends a REMB to
1482 // indicate that it should be lowered significantly. The test then observes that
1483 // the bitrate observed is sinking well below the min-transmit-bitrate threshold
1484 // to verify that the min-transmit bitrate respects incoming REMB.
1485 //
1486 // Note that the test starts at "high" bitrate and does not ramp up to "higher"
1487 // bitrate since no receiver block or remb is sent in the initial phase.
TEST_F(VideoSendStreamTest,MinTransmitBitrateRespectsRemb)1488 TEST_F(VideoSendStreamTest, MinTransmitBitrateRespectsRemb) {
1489 static const int kMinTransmitBitrateBps = 400000;
1490 static const int kHighBitrateBps = 150000;
1491 static const int kRembBitrateBps = 80000;
1492 static const int kRembRespectedBitrateBps = 100000;
1493 class BitrateObserver : public test::SendTest {
1494 public:
1495 explicit BitrateObserver(TaskQueueBase* task_queue)
1496 : SendTest(kDefaultTimeout),
1497 task_queue_(task_queue),
1498 retranmission_rate_limiter_(Clock::GetRealTimeClock(), 1000),
1499 stream_(nullptr),
1500 bitrate_capped_(false),
1501 task_safety_flag_(PendingTaskSafetyFlag::CreateDetached()) {}
1502
1503 private:
1504 Action OnSendRtp(const uint8_t* packet, size_t length) override {
1505 if (IsRtcpPacket(rtc::MakeArrayView(packet, length)))
1506 return DROP_PACKET;
1507
1508 RtpPacket rtp_packet;
1509 RTC_CHECK(rtp_packet.Parse(packet, length));
1510 const uint32_t ssrc = rtp_packet.Ssrc();
1511 RTC_DCHECK(stream_);
1512
1513 task_queue_->PostTask(SafeTask(task_safety_flag_, [this, ssrc]() {
1514 VideoSendStream::Stats stats = stream_->GetStats();
1515 if (!stats.substreams.empty()) {
1516 EXPECT_EQ(1u, stats.substreams.size());
1517 int total_bitrate_bps =
1518 stats.substreams.begin()->second.total_bitrate_bps;
1519 test::GetGlobalMetricsLogger()->LogSingleValueMetric(
1520 "bitrate_stats_min_transmit_bitrate_low_remb", "bitrate_bps",
1521 static_cast<size_t>(total_bitrate_bps) / 1000.0,
1522 test::Unit::kKilobitsPerSecond,
1523 test::ImprovementDirection::kNeitherIsBetter);
1524 if (total_bitrate_bps > kHighBitrateBps) {
1525 rtp_rtcp_->SetRemb(kRembBitrateBps, {ssrc});
1526 bitrate_capped_ = true;
1527 } else if (bitrate_capped_ &&
1528 total_bitrate_bps < kRembRespectedBitrateBps) {
1529 observation_complete_.Set();
1530 }
1531 }
1532 }));
1533
1534 // Packets don't have to be delivered since the test is the receiver.
1535 return DROP_PACKET;
1536 }
1537
1538 void OnVideoStreamsCreated(VideoSendStream* send_stream,
1539 const std::vector<VideoReceiveStreamInterface*>&
1540 receive_streams) override {
1541 stream_ = send_stream;
1542 RtpRtcpInterface::Configuration config;
1543 config.clock = Clock::GetRealTimeClock();
1544 config.outgoing_transport = feedback_transport_.get();
1545 config.retransmission_rate_limiter = &retranmission_rate_limiter_;
1546 rtp_rtcp_ = ModuleRtpRtcpImpl2::Create(config);
1547 rtp_rtcp_->SetRTCPStatus(RtcpMode::kReducedSize);
1548 }
1549
1550 void ModifyVideoConfigs(
1551 VideoSendStream::Config* send_config,
1552 std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
1553 VideoEncoderConfig* encoder_config) override {
1554 feedback_transport_.reset(
1555 new internal::TransportAdapter(send_config->send_transport));
1556 feedback_transport_->Enable();
1557 encoder_config->min_transmit_bitrate_bps = kMinTransmitBitrateBps;
1558 }
1559
1560 void OnStreamsStopped() override {
1561 task_safety_flag_->SetNotAlive();
1562 rtp_rtcp_.reset();
1563 }
1564
1565 void PerformTest() override {
1566 EXPECT_TRUE(Wait())
1567 << "Timeout while waiting for low bitrate stats after REMB.";
1568 }
1569
1570 TaskQueueBase* const task_queue_;
1571 std::unique_ptr<ModuleRtpRtcpImpl2> rtp_rtcp_;
1572 std::unique_ptr<internal::TransportAdapter> feedback_transport_;
1573 RateLimiter retranmission_rate_limiter_;
1574 VideoSendStream* stream_;
1575 bool bitrate_capped_;
1576 rtc::scoped_refptr<PendingTaskSafetyFlag> task_safety_flag_;
1577 } test(task_queue());
1578
1579 RunBaseTest(&test);
1580 }
1581
TEST_F(VideoSendStreamTest,ChangingNetworkRoute)1582 TEST_F(VideoSendStreamTest, ChangingNetworkRoute) {
1583 static const int kStartBitrateBps = 300000;
1584 static const int kNewMaxBitrateBps = 1234567;
1585 static const uint8_t kExtensionId = kTransportSequenceNumberExtensionId;
1586 class ChangingNetworkRouteTest : public test::EndToEndTest {
1587 public:
1588 explicit ChangingNetworkRouteTest(TaskQueueBase* task_queue)
1589 : EndToEndTest(test::CallTest::kDefaultTimeout),
1590 task_queue_(task_queue),
1591 call_(nullptr) {
1592 module_process_thread_.Detach();
1593 task_queue_thread_.Detach();
1594 extensions_.Register<TransportSequenceNumber>(kExtensionId);
1595 }
1596
1597 ~ChangingNetworkRouteTest() {
1598 // Block until all already posted tasks run to avoid 'use after free'
1599 // when such task accesses `this`.
1600 SendTask(task_queue_, [] {});
1601 }
1602
1603 void OnCallsCreated(Call* sender_call, Call* receiver_call) override {
1604 RTC_DCHECK_RUN_ON(&task_queue_thread_);
1605 RTC_DCHECK(!call_);
1606 call_ = sender_call;
1607 }
1608
1609 void ModifyVideoConfigs(
1610 VideoSendStream::Config* send_config,
1611 std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
1612 VideoEncoderConfig* encoder_config) override {
1613 RTC_DCHECK_RUN_ON(&task_queue_thread_);
1614 send_config->rtp.extensions.clear();
1615 send_config->rtp.extensions.push_back(RtpExtension(
1616 RtpExtension::kTransportSequenceNumberUri, kExtensionId));
1617 (*receive_configs)[0].rtp.extensions = send_config->rtp.extensions;
1618 (*receive_configs)[0].rtp.transport_cc = true;
1619 }
1620
1621 void ModifyAudioConfigs(AudioSendStream::Config* send_config,
1622 std::vector<AudioReceiveStreamInterface::Config>*
1623 receive_configs) override {
1624 RTC_DCHECK_RUN_ON(&task_queue_thread_);
1625 send_config->rtp.extensions.clear();
1626 send_config->rtp.extensions.push_back(RtpExtension(
1627 RtpExtension::kTransportSequenceNumberUri, kExtensionId));
1628 (*receive_configs)[0].rtp.extensions.clear();
1629 (*receive_configs)[0].rtp.extensions = send_config->rtp.extensions;
1630 (*receive_configs)[0].rtp.transport_cc = true;
1631 }
1632
1633 Action OnSendRtp(const uint8_t* packet, size_t length) override {
1634 RTC_DCHECK_RUN_ON(&module_process_thread_);
1635 task_queue_->PostTask([this]() {
1636 RTC_DCHECK_RUN_ON(&task_queue_thread_);
1637 if (!call_)
1638 return;
1639 Call::Stats stats = call_->GetStats();
1640 if (stats.send_bandwidth_bps > kStartBitrateBps)
1641 observation_complete_.Set();
1642 });
1643 return SEND_PACKET;
1644 }
1645
1646 void OnStreamsStopped() override {
1647 RTC_DCHECK_RUN_ON(&task_queue_thread_);
1648 call_ = nullptr;
1649 }
1650
1651 void PerformTest() override {
1652 rtc::NetworkRoute new_route;
1653 new_route.connected = true;
1654 new_route.local = rtc::RouteEndpoint::CreateWithNetworkId(10);
1655 new_route.remote = rtc::RouteEndpoint::CreateWithNetworkId(20);
1656 BitrateConstraints bitrate_config;
1657
1658 SendTask(task_queue_,
1659 [this, &new_route, &bitrate_config]() {
1660 RTC_DCHECK_RUN_ON(&task_queue_thread_);
1661 call_->GetTransportControllerSend()->OnNetworkRouteChanged(
1662 "transport", new_route);
1663 bitrate_config.start_bitrate_bps = kStartBitrateBps;
1664 call_->GetTransportControllerSend()->SetSdpBitrateParameters(
1665 bitrate_config);
1666 });
1667
1668 EXPECT_TRUE(Wait())
1669 << "Timed out while waiting for start bitrate to be exceeded.";
1670
1671 SendTask(
1672 task_queue_, [this, &new_route, &bitrate_config]() {
1673 RTC_DCHECK_RUN_ON(&task_queue_thread_);
1674 bitrate_config.start_bitrate_bps = -1;
1675 bitrate_config.max_bitrate_bps = kNewMaxBitrateBps;
1676 call_->GetTransportControllerSend()->SetSdpBitrateParameters(
1677 bitrate_config);
1678 // TODO(holmer): We should set the last sent packet id here and
1679 // verify that we correctly ignore any packet loss reported prior to
1680 // that id.
1681 new_route.local = rtc::RouteEndpoint::CreateWithNetworkId(
1682 new_route.local.network_id() + 1);
1683 call_->GetTransportControllerSend()->OnNetworkRouteChanged(
1684 "transport", new_route);
1685 EXPECT_GE(call_->GetStats().send_bandwidth_bps, kStartBitrateBps);
1686 });
1687 }
1688
1689 private:
1690 webrtc::SequenceChecker module_process_thread_;
1691 webrtc::SequenceChecker task_queue_thread_;
1692 TaskQueueBase* const task_queue_;
1693 RtpHeaderExtensionMap extensions_;
1694 Call* call_ RTC_GUARDED_BY(task_queue_thread_);
1695 } test(task_queue());
1696
1697 RunBaseTest(&test);
1698 }
1699
1700 // Test that if specified, relay cap is lifted on transition to direct
1701 // connection.
1702 // TODO(https://bugs.webrtc.org/13353): Test disabled due to flakiness.
TEST_F(VideoSendStreamTest,DISABLED_RelayToDirectRoute)1703 TEST_F(VideoSendStreamTest, DISABLED_RelayToDirectRoute) {
1704 static const int kStartBitrateBps = 300000;
1705 static const int kRelayBandwidthCapBps = 800000;
1706 static const int kMinPacketsToSend = 100;
1707 webrtc::test::ScopedKeyValueConfig field_trials(
1708 field_trials_, "WebRTC-Bwe-NetworkRouteConstraints/relay_cap:" +
1709 std::to_string(kRelayBandwidthCapBps) + "bps/");
1710
1711 class RelayToDirectRouteTest : public test::EndToEndTest {
1712 public:
1713 explicit RelayToDirectRouteTest(TaskQueueBase* task_queue)
1714 : EndToEndTest(test::CallTest::kDefaultTimeout),
1715 task_queue_(task_queue),
1716 call_(nullptr),
1717 packets_sent_(0),
1718 relayed_phase_(true) {
1719 module_process_thread_.Detach();
1720 task_queue_thread_.Detach();
1721 }
1722
1723 ~RelayToDirectRouteTest() {
1724 // Block until all already posted tasks run to avoid 'use after free'
1725 // when such task accesses `this`.
1726 SendTask(task_queue_, [] {});
1727 }
1728
1729 void OnCallsCreated(Call* sender_call, Call* receiver_call) override {
1730 RTC_DCHECK_RUN_ON(&task_queue_thread_);
1731 RTC_DCHECK(!call_);
1732 call_ = sender_call;
1733 }
1734
1735 Action OnSendRtp(const uint8_t* packet, size_t length) override {
1736 RTC_DCHECK_RUN_ON(&module_process_thread_);
1737 task_queue_->PostTask([this]() {
1738 RTC_DCHECK_RUN_ON(&task_queue_thread_);
1739 if (!call_)
1740 return;
1741 bool had_time_to_exceed_cap_in_relayed_phase =
1742 relayed_phase_ && ++packets_sent_ > kMinPacketsToSend;
1743 bool did_exceed_cap =
1744 call_->GetStats().send_bandwidth_bps > kRelayBandwidthCapBps;
1745 if (did_exceed_cap || had_time_to_exceed_cap_in_relayed_phase)
1746 observation_complete_.Set();
1747 });
1748 return SEND_PACKET;
1749 }
1750
1751 void OnStreamsStopped() override {
1752 RTC_DCHECK_RUN_ON(&task_queue_thread_);
1753 call_ = nullptr;
1754 }
1755
1756 void PerformTest() override {
1757 rtc::NetworkRoute route;
1758 route.connected = true;
1759 route.local = rtc::RouteEndpoint::CreateWithNetworkId(10);
1760 route.remote = rtc::RouteEndpoint::CreateWithNetworkId(20);
1761
1762 SendTask(task_queue_, [this, &route]() {
1763 RTC_DCHECK_RUN_ON(&task_queue_thread_);
1764 relayed_phase_ = true;
1765 route.remote = route.remote.CreateWithTurn(true);
1766 call_->GetTransportControllerSend()->OnNetworkRouteChanged("transport",
1767 route);
1768 BitrateConstraints bitrate_config;
1769 bitrate_config.start_bitrate_bps = kStartBitrateBps;
1770
1771 call_->GetTransportControllerSend()->SetSdpBitrateParameters(
1772 bitrate_config);
1773 });
1774
1775 EXPECT_TRUE(Wait())
1776 << "Timeout waiting for sufficient packets sent count.";
1777
1778 SendTask(task_queue_, [this, &route]() {
1779 RTC_DCHECK_RUN_ON(&task_queue_thread_);
1780 EXPECT_LE(call_->GetStats().send_bandwidth_bps, kRelayBandwidthCapBps);
1781
1782 route.remote = route.remote.CreateWithTurn(false);
1783 call_->GetTransportControllerSend()->OnNetworkRouteChanged("transport",
1784 route);
1785 relayed_phase_ = false;
1786 observation_complete_.Reset();
1787 });
1788
1789 EXPECT_TRUE(Wait())
1790 << "Timeout while waiting for bandwidth to outgrow relay cap.";
1791 }
1792
1793 private:
1794 webrtc::SequenceChecker module_process_thread_;
1795 webrtc::SequenceChecker task_queue_thread_;
1796 TaskQueueBase* const task_queue_;
1797 Call* call_ RTC_GUARDED_BY(task_queue_thread_);
1798 int packets_sent_ RTC_GUARDED_BY(task_queue_thread_);
1799 bool relayed_phase_ RTC_GUARDED_BY(task_queue_thread_);
1800 } test(task_queue());
1801
1802 RunBaseTest(&test);
1803 }
1804
TEST_F(VideoSendStreamTest,ChangingTransportOverhead)1805 TEST_F(VideoSendStreamTest, ChangingTransportOverhead) {
1806 class ChangingTransportOverheadTest : public test::EndToEndTest {
1807 public:
1808 explicit ChangingTransportOverheadTest(TaskQueueBase* task_queue)
1809 : EndToEndTest(test::CallTest::kDefaultTimeout),
1810 task_queue_(task_queue),
1811 call_(nullptr),
1812 packets_sent_(0),
1813 transport_overhead_(0) {}
1814
1815 void OnCallsCreated(Call* sender_call, Call* receiver_call) override {
1816 call_ = sender_call;
1817 }
1818
1819 Action OnSendRtp(const uint8_t* packet, size_t length) override {
1820 EXPECT_LE(length, kMaxRtpPacketSize);
1821 MutexLock lock(&lock_);
1822 if (++packets_sent_ < 100)
1823 return SEND_PACKET;
1824 observation_complete_.Set();
1825 return SEND_PACKET;
1826 }
1827
1828 void ModifyVideoConfigs(
1829 VideoSendStream::Config* send_config,
1830 std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
1831 VideoEncoderConfig* encoder_config) override {
1832 send_config->rtp.max_packet_size = kMaxRtpPacketSize;
1833 }
1834
1835 void PerformTest() override {
1836 SendTask(task_queue_, [this]() {
1837 transport_overhead_ = 100;
1838 call_->GetTransportControllerSend()->OnTransportOverheadChanged(
1839 transport_overhead_);
1840 });
1841
1842 EXPECT_TRUE(Wait());
1843
1844 {
1845 MutexLock lock(&lock_);
1846 packets_sent_ = 0;
1847 }
1848
1849 SendTask(task_queue_, [this]() {
1850 transport_overhead_ = 500;
1851 call_->GetTransportControllerSend()->OnTransportOverheadChanged(
1852 transport_overhead_);
1853 });
1854
1855 EXPECT_TRUE(Wait());
1856 }
1857
1858 private:
1859 TaskQueueBase* const task_queue_;
1860 Call* call_;
1861 Mutex lock_;
1862 int packets_sent_ RTC_GUARDED_BY(lock_);
1863 int transport_overhead_;
1864 const size_t kMaxRtpPacketSize = 1000;
1865 } test(task_queue());
1866
1867 RunBaseTest(&test);
1868 }
1869
1870 // Test class takes takes as argument a switch selecting if type switch should
1871 // occur and a function pointer to reset the send stream. This is necessary
1872 // since you cannot change the content type of a VideoSendStream, you need to
1873 // recreate it. Stopping and recreating the stream can only be done on the main
1874 // thread and in the context of VideoSendStreamTest (not BaseTest).
1875 template <typename T>
1876 class MaxPaddingSetTest : public test::SendTest {
1877 public:
1878 static const uint32_t kMinTransmitBitrateBps = 400000;
1879 static const uint32_t kActualEncodeBitrateBps = 40000;
1880 static const uint32_t kMinPacketsToSend = 50;
1881
MaxPaddingSetTest(bool test_switch_content_type,T * stream_reset_fun,TaskQueueBase * task_queue)1882 MaxPaddingSetTest(bool test_switch_content_type,
1883 T* stream_reset_fun,
1884 TaskQueueBase* task_queue)
1885 : SendTest(test::CallTest::kDefaultTimeout),
1886 running_without_padding_(test_switch_content_type),
1887 stream_resetter_(stream_reset_fun),
1888 task_queue_(task_queue) {
1889 RTC_DCHECK(stream_resetter_);
1890 module_process_thread_.Detach();
1891 task_queue_thread_.Detach();
1892 }
1893
~MaxPaddingSetTest()1894 ~MaxPaddingSetTest() {
1895 // Block until all already posted tasks run to avoid 'use after free'
1896 // when such task accesses `this`.
1897 SendTask(task_queue_, [] {});
1898 }
1899
ModifyVideoConfigs(VideoSendStream::Config * send_config,std::vector<VideoReceiveStreamInterface::Config> * receive_configs,VideoEncoderConfig * encoder_config)1900 void ModifyVideoConfigs(
1901 VideoSendStream::Config* send_config,
1902 std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
1903 VideoEncoderConfig* encoder_config) override {
1904 RTC_DCHECK_RUN_ON(&task_queue_thread_);
1905 RTC_DCHECK_EQ(1, encoder_config->number_of_streams);
1906 if (running_without_padding_) {
1907 encoder_config->min_transmit_bitrate_bps = 0;
1908 encoder_config->content_type =
1909 VideoEncoderConfig::ContentType::kRealtimeVideo;
1910 } else {
1911 encoder_config->min_transmit_bitrate_bps = kMinTransmitBitrateBps;
1912 encoder_config->content_type = VideoEncoderConfig::ContentType::kScreen;
1913 }
1914 send_stream_config_ = send_config->Copy();
1915 encoder_config_ = encoder_config->Copy();
1916 }
1917
OnCallsCreated(Call * sender_call,Call * receiver_call)1918 void OnCallsCreated(Call* sender_call, Call* receiver_call) override {
1919 RTC_DCHECK_RUN_ON(&task_queue_thread_);
1920 RTC_DCHECK(task_queue_->IsCurrent());
1921 RTC_DCHECK(!call_);
1922 RTC_DCHECK(sender_call);
1923 call_ = sender_call;
1924 }
1925
1926 // Called on the pacer thread.
OnSendRtp(const uint8_t * packet,size_t length)1927 Action OnSendRtp(const uint8_t* packet, size_t length) override {
1928 RTC_DCHECK_RUN_ON(&module_process_thread_);
1929
1930 // Check the stats on the correct thread and signal the 'complete' flag
1931 // once we detect that we're done.
1932
1933 task_queue_->PostTask([this]() {
1934 RTC_DCHECK_RUN_ON(&task_queue_thread_);
1935 // In case we get a callback during teardown.
1936 // When this happens, OnStreamsStopped() has been called already,
1937 // `call_` is null and the streams are being torn down.
1938 if (!call_)
1939 return;
1940
1941 ++packets_sent_;
1942
1943 Call::Stats stats = call_->GetStats();
1944 if (running_without_padding_) {
1945 EXPECT_EQ(0, stats.max_padding_bitrate_bps);
1946
1947 // Wait until at least kMinPacketsToSend frames have been encoded, so
1948 // that we have reliable data.
1949 if (packets_sent_ < kMinPacketsToSend)
1950 return;
1951
1952 // We've sent kMinPacketsToSend packets with default configuration,
1953 // switch to enabling screen content and setting min transmit bitrate.
1954 // Note that we need to recreate the stream if changing content type.
1955 packets_sent_ = 0;
1956
1957 encoder_config_.min_transmit_bitrate_bps = kMinTransmitBitrateBps;
1958 encoder_config_.content_type = VideoEncoderConfig::ContentType::kScreen;
1959
1960 running_without_padding_ = false;
1961 (*stream_resetter_)(send_stream_config_, encoder_config_);
1962 } else {
1963 // Make sure the pacer has been configured with a min transmit bitrate.
1964 if (stats.max_padding_bitrate_bps > 0) {
1965 observation_complete_.Set();
1966 }
1967 }
1968 });
1969
1970 return SEND_PACKET;
1971 }
1972
1973 // Called on `task_queue_`
OnStreamsStopped()1974 void OnStreamsStopped() override {
1975 RTC_DCHECK_RUN_ON(&task_queue_thread_);
1976 RTC_DCHECK(task_queue_->IsCurrent());
1977 call_ = nullptr;
1978 }
1979
PerformTest()1980 void PerformTest() override {
1981 ASSERT_TRUE(Wait()) << "Timed out waiting for a valid padding bitrate.";
1982 }
1983
1984 private:
1985 webrtc::SequenceChecker task_queue_thread_;
1986 Call* call_ RTC_GUARDED_BY(task_queue_thread_) = nullptr;
1987 VideoSendStream::Config send_stream_config_{nullptr};
1988 VideoEncoderConfig encoder_config_;
1989 webrtc::SequenceChecker module_process_thread_;
1990 uint32_t packets_sent_ RTC_GUARDED_BY(task_queue_thread_) = 0;
1991 bool running_without_padding_ RTC_GUARDED_BY(task_queue_thread_);
1992 T* const stream_resetter_;
1993 TaskQueueBase* const task_queue_;
1994 };
1995
TEST_F(VideoSendStreamTest,RespectsMinTransmitBitrate)1996 TEST_F(VideoSendStreamTest, RespectsMinTransmitBitrate) {
1997 auto reset_fun = [](const VideoSendStream::Config& send_stream_config,
1998 const VideoEncoderConfig& encoder_config) {};
1999 MaxPaddingSetTest<decltype(reset_fun)> test(false, &reset_fun, task_queue());
2000 RunBaseTest(&test);
2001 }
2002
TEST_F(VideoSendStreamTest,RespectsMinTransmitBitrateAfterContentSwitch)2003 TEST_F(VideoSendStreamTest, RespectsMinTransmitBitrateAfterContentSwitch) {
2004 // Function for removing and recreating the send stream with a new config.
2005 auto reset_fun = [this](const VideoSendStream::Config& send_stream_config,
2006 const VideoEncoderConfig& encoder_config) {
2007 RTC_DCHECK(task_queue()->IsCurrent());
2008 Stop();
2009 DestroyVideoSendStreams();
2010 SetVideoSendConfig(send_stream_config);
2011 SetVideoEncoderConfig(encoder_config);
2012 CreateVideoSendStreams();
2013 SetVideoDegradation(DegradationPreference::MAINTAIN_RESOLUTION);
2014 Start();
2015 };
2016 MaxPaddingSetTest<decltype(reset_fun)> test(true, &reset_fun, task_queue());
2017 RunBaseTest(&test);
2018 }
2019
2020 // This test verifies that new frame sizes reconfigures encoders even though not
2021 // (yet) sending. The purpose of this is to permit encoding as quickly as
2022 // possible once we start sending. Likely the frames being input are from the
2023 // same source that will be sent later, which just means that we're ready
2024 // earlier.
TEST_F(VideoSendStreamTest,EncoderReconfigureOnResolutionChangeWhenNotSending)2025 TEST_F(VideoSendStreamTest,
2026 EncoderReconfigureOnResolutionChangeWhenNotSending) {
2027 class EncoderObserver : public test::FakeEncoder {
2028 public:
2029 EncoderObserver()
2030 : FakeEncoder(Clock::GetRealTimeClock()),
2031 last_initialized_frame_width_(0),
2032 last_initialized_frame_height_(0) {}
2033
2034 void WaitForResolution(int width, int height) {
2035 {
2036 MutexLock lock(&mutex_);
2037 if (last_initialized_frame_width_ == width &&
2038 last_initialized_frame_height_ == height) {
2039 return;
2040 }
2041 }
2042 EXPECT_TRUE(
2043 init_encode_called_.Wait(VideoSendStreamTest::kDefaultTimeout));
2044 {
2045 MutexLock lock(&mutex_);
2046 EXPECT_EQ(width, last_initialized_frame_width_);
2047 EXPECT_EQ(height, last_initialized_frame_height_);
2048 }
2049 }
2050
2051 private:
2052 int32_t InitEncode(const VideoCodec* config,
2053 const Settings& settings) override {
2054 MutexLock lock(&mutex_);
2055 last_initialized_frame_width_ = config->width;
2056 last_initialized_frame_height_ = config->height;
2057 init_encode_called_.Set();
2058 return FakeEncoder::InitEncode(config, settings);
2059 }
2060
2061 int32_t Encode(const VideoFrame& input_image,
2062 const std::vector<VideoFrameType>* frame_types) override {
2063 ADD_FAILURE()
2064 << "Unexpected Encode call since the send stream is not started";
2065 return 0;
2066 }
2067
2068 Mutex mutex_;
2069 rtc::Event init_encode_called_;
2070 int last_initialized_frame_width_ RTC_GUARDED_BY(&mutex_);
2071 int last_initialized_frame_height_ RTC_GUARDED_BY(&mutex_);
2072 };
2073
2074 test::NullTransport transport;
2075 EncoderObserver encoder;
2076 test::VideoEncoderProxyFactory encoder_factory(&encoder);
2077
2078 SendTask(task_queue(), [this, &transport, &encoder_factory]() {
2079 CreateSenderCall();
2080 CreateSendConfig(1, 0, 0, &transport);
2081 GetVideoSendConfig()->encoder_settings.encoder_factory = &encoder_factory;
2082 CreateVideoStreams();
2083 CreateFrameGeneratorCapturer(kDefaultFramerate, kDefaultWidth,
2084 kDefaultHeight);
2085 frame_generator_capturer_->Start();
2086 });
2087
2088 encoder.WaitForResolution(kDefaultWidth, kDefaultHeight);
2089
2090 SendTask(task_queue(), [this]() {
2091 frame_generator_capturer_->ChangeResolution(kDefaultWidth * 2,
2092 kDefaultHeight * 2);
2093 });
2094
2095 encoder.WaitForResolution(kDefaultWidth * 2, kDefaultHeight * 2);
2096
2097 SendTask(task_queue(), [this]() {
2098 DestroyStreams();
2099 DestroyCalls();
2100 });
2101 }
2102
TEST_F(VideoSendStreamTest,CanReconfigureToUseStartBitrateAbovePreviousMax)2103 TEST_F(VideoSendStreamTest, CanReconfigureToUseStartBitrateAbovePreviousMax) {
2104 class StartBitrateObserver : public test::FakeEncoder {
2105 public:
2106 StartBitrateObserver()
2107 : FakeEncoder(Clock::GetRealTimeClock()), start_bitrate_kbps_(0) {}
2108 int32_t InitEncode(const VideoCodec* config,
2109 const Settings& settings) override {
2110 MutexLock lock(&mutex_);
2111 start_bitrate_kbps_ = config->startBitrate;
2112 start_bitrate_changed_.Set();
2113 return FakeEncoder::InitEncode(config, settings);
2114 }
2115
2116 void SetRates(const RateControlParameters& parameters) override {
2117 MutexLock lock(&mutex_);
2118 start_bitrate_kbps_ = parameters.bitrate.get_sum_kbps();
2119 start_bitrate_changed_.Set();
2120 FakeEncoder::SetRates(parameters);
2121 }
2122
2123 int GetStartBitrateKbps() const {
2124 MutexLock lock(&mutex_);
2125 return start_bitrate_kbps_;
2126 }
2127
2128 bool WaitForStartBitrate() {
2129 return start_bitrate_changed_.Wait(VideoSendStreamTest::kDefaultTimeout);
2130 }
2131
2132 private:
2133 mutable Mutex mutex_;
2134 rtc::Event start_bitrate_changed_;
2135 int start_bitrate_kbps_ RTC_GUARDED_BY(mutex_);
2136 };
2137
2138 CreateSenderCall();
2139
2140 test::NullTransport transport;
2141 CreateSendConfig(1, 0, 0, &transport);
2142
2143 BitrateConstraints bitrate_config;
2144 bitrate_config.start_bitrate_bps =
2145 2 * GetVideoEncoderConfig()->max_bitrate_bps;
2146 sender_call_->GetTransportControllerSend()->SetSdpBitrateParameters(
2147 bitrate_config);
2148
2149 StartBitrateObserver encoder;
2150 test::VideoEncoderProxyFactory encoder_factory(&encoder);
2151 GetVideoSendConfig()->encoder_settings.encoder_factory = &encoder_factory;
2152
2153 CreateVideoStreams();
2154
2155 // Start capturing and encoding frames to force encoder reconfiguration.
2156 CreateFrameGeneratorCapturer(kDefaultFramerate, kDefaultWidth,
2157 kDefaultHeight);
2158 frame_generator_capturer_->Start();
2159 // TODO(crbug/1255737): Added manual current thread message processing because
2160 // the test code context is interpreted as the worker thread and we assume
2161 // progress on it. The test should probably be ported to use simulated time
2162 // instead (ported to a scenario test perhaps?).
2163 rtc::Thread::Current()->ProcessMessages(5000);
2164
2165 EXPECT_TRUE(encoder.WaitForStartBitrate());
2166 EXPECT_EQ(GetVideoEncoderConfig()->max_bitrate_bps / 1000,
2167 encoder.GetStartBitrateKbps());
2168
2169 GetVideoEncoderConfig()->max_bitrate_bps =
2170 2 * bitrate_config.start_bitrate_bps;
2171 GetVideoSendStream()->ReconfigureVideoEncoder(
2172 GetVideoEncoderConfig()->Copy());
2173 // TODO(crbug/1255737): Added manual current thread message processing because
2174 // the test code context is interpreted as the worker thread and we assume
2175 // progress on it. The test should probably be ported to use simulated time
2176 // instead (ported to a scenario test perhaps?).
2177 rtc::Thread::Current()->ProcessMessages(5000);
2178
2179 // New bitrate should be reconfigured above the previous max. As there's no
2180 // network connection this shouldn't be flaky, as no bitrate should've been
2181 // reported in between.
2182 EXPECT_TRUE(encoder.WaitForStartBitrate());
2183 EXPECT_EQ(bitrate_config.start_bitrate_bps / 1000,
2184 encoder.GetStartBitrateKbps());
2185
2186 DestroyStreams();
2187 }
2188
2189 class StartStopBitrateObserver : public test::FakeEncoder {
2190 public:
StartStopBitrateObserver()2191 StartStopBitrateObserver() : FakeEncoder(Clock::GetRealTimeClock()) {}
InitEncode(const VideoCodec * config,const Settings & settings)2192 int32_t InitEncode(const VideoCodec* config,
2193 const Settings& settings) override {
2194 MutexLock lock(&mutex_);
2195 encoder_init_.Set();
2196 return FakeEncoder::InitEncode(config, settings);
2197 }
2198
SetRates(const RateControlParameters & parameters)2199 void SetRates(const RateControlParameters& parameters) override {
2200 MutexLock lock(&mutex_);
2201 bitrate_kbps_ = parameters.bitrate.get_sum_kbps();
2202 bitrate_changed_.Set();
2203 FakeEncoder::SetRates(parameters);
2204 }
2205
WaitForEncoderInit()2206 bool WaitForEncoderInit() {
2207 return encoder_init_.Wait(VideoSendStreamTest::kDefaultTimeout);
2208 }
2209
WaitBitrateChanged(WaitUntil until)2210 bool WaitBitrateChanged(WaitUntil until) {
2211 do {
2212 absl::optional<int> bitrate_kbps;
2213 {
2214 MutexLock lock(&mutex_);
2215 bitrate_kbps = bitrate_kbps_;
2216 }
2217 if (!bitrate_kbps)
2218 continue;
2219
2220 if ((until == WaitUntil::kNonZero && *bitrate_kbps > 0) ||
2221 (until == WaitUntil::kZero && *bitrate_kbps == 0)) {
2222 return true;
2223 }
2224 } while (bitrate_changed_.Wait(VideoSendStreamTest::kDefaultTimeout));
2225 return false;
2226 }
2227
2228 private:
2229 Mutex mutex_;
2230 rtc::Event encoder_init_;
2231 rtc::Event bitrate_changed_;
2232 absl::optional<int> bitrate_kbps_ RTC_GUARDED_BY(mutex_);
2233 };
2234
TEST_F(VideoSendStreamTest,EncoderIsProperlyInitializedAndDestroyed)2235 TEST_F(VideoSendStreamTest, EncoderIsProperlyInitializedAndDestroyed) {
2236 class EncoderStateObserver : public test::SendTest, public VideoEncoder {
2237 public:
2238 explicit EncoderStateObserver(TaskQueueBase* task_queue)
2239 : SendTest(kDefaultTimeout),
2240 task_queue_(task_queue),
2241 stream_(nullptr),
2242 initialized_(false),
2243 callback_registered_(false),
2244 num_releases_(0),
2245 released_(false),
2246 encoder_factory_(this) {}
2247
2248 bool IsReleased() RTC_LOCKS_EXCLUDED(mutex_) {
2249 MutexLock lock(&mutex_);
2250 return released_;
2251 }
2252
2253 bool IsReadyForEncode() RTC_LOCKS_EXCLUDED(mutex_) {
2254 MutexLock lock(&mutex_);
2255 return IsReadyForEncodeLocked();
2256 }
2257
2258 size_t num_releases() RTC_LOCKS_EXCLUDED(mutex_) {
2259 MutexLock lock(&mutex_);
2260 return num_releases_;
2261 }
2262
2263 private:
2264 bool IsReadyForEncodeLocked() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_) {
2265 return initialized_ && callback_registered_;
2266 }
2267
2268 void SetFecControllerOverride(
2269 FecControllerOverride* fec_controller_override) override {
2270 // Ignored.
2271 }
2272
2273 int32_t InitEncode(const VideoCodec* codecSettings,
2274 const Settings& settings) override
2275 RTC_LOCKS_EXCLUDED(mutex_) {
2276 MutexLock lock(&mutex_);
2277 EXPECT_FALSE(initialized_);
2278 initialized_ = true;
2279 released_ = false;
2280 return 0;
2281 }
2282
2283 int32_t Encode(const VideoFrame& inputImage,
2284 const std::vector<VideoFrameType>* frame_types) override {
2285 EXPECT_TRUE(IsReadyForEncode());
2286
2287 observation_complete_.Set();
2288 return 0;
2289 }
2290
2291 int32_t RegisterEncodeCompleteCallback(
2292 EncodedImageCallback* callback) override RTC_LOCKS_EXCLUDED(mutex_) {
2293 MutexLock lock(&mutex_);
2294 EXPECT_TRUE(initialized_);
2295 callback_registered_ = true;
2296 return 0;
2297 }
2298
2299 int32_t Release() override RTC_LOCKS_EXCLUDED(mutex_) {
2300 MutexLock lock(&mutex_);
2301 EXPECT_TRUE(IsReadyForEncodeLocked());
2302 EXPECT_FALSE(released_);
2303 initialized_ = false;
2304 callback_registered_ = false;
2305 released_ = true;
2306 ++num_releases_;
2307 return 0;
2308 }
2309
2310 void SetRates(const RateControlParameters& parameters) override {
2311 EXPECT_TRUE(IsReadyForEncode());
2312 }
2313
2314 void OnVideoStreamsCreated(VideoSendStream* send_stream,
2315 const std::vector<VideoReceiveStreamInterface*>&
2316 receive_streams) override {
2317 stream_ = send_stream;
2318 }
2319
2320 void ModifyVideoConfigs(
2321 VideoSendStream::Config* send_config,
2322 std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
2323 VideoEncoderConfig* encoder_config) override {
2324 send_config->encoder_settings.encoder_factory = &encoder_factory_;
2325 encoder_config_ = encoder_config->Copy();
2326 }
2327
2328 void PerformTest() override {
2329 EXPECT_TRUE(Wait()) << "Timed out while waiting for Encode.";
2330
2331 SendTask(task_queue_, [this]() {
2332 EXPECT_EQ(0u, num_releases());
2333 stream_->ReconfigureVideoEncoder(std::move(encoder_config_));
2334 EXPECT_EQ(0u, num_releases());
2335 stream_->Stop();
2336 // Encoder should not be released before destroying the VideoSendStream.
2337 EXPECT_FALSE(IsReleased());
2338 EXPECT_TRUE(IsReadyForEncode());
2339 stream_->Start();
2340 });
2341
2342 // Sanity check, make sure we still encode frames with this encoder.
2343 EXPECT_TRUE(Wait()) << "Timed out while waiting for Encode.";
2344 }
2345
2346 TaskQueueBase* const task_queue_;
2347 Mutex mutex_;
2348 VideoSendStream* stream_;
2349 bool initialized_ RTC_GUARDED_BY(mutex_);
2350 bool callback_registered_ RTC_GUARDED_BY(mutex_);
2351 size_t num_releases_ RTC_GUARDED_BY(mutex_);
2352 bool released_ RTC_GUARDED_BY(mutex_);
2353 test::VideoEncoderProxyFactory encoder_factory_;
2354 VideoEncoderConfig encoder_config_;
2355 } test_encoder(task_queue());
2356
2357 RunBaseTest(&test_encoder);
2358
2359 EXPECT_TRUE(test_encoder.IsReleased());
2360 EXPECT_EQ(1u, test_encoder.num_releases());
2361 }
2362
2363 static const size_t kVideoCodecConfigObserverNumberOfTemporalLayers = 3;
2364 template <typename T>
2365 class VideoCodecConfigObserver : public test::SendTest,
2366 public test::FakeEncoder {
2367 public:
VideoCodecConfigObserver(VideoCodecType video_codec_type,TaskQueueBase * task_queue)2368 VideoCodecConfigObserver(VideoCodecType video_codec_type,
2369 TaskQueueBase* task_queue)
2370 : SendTest(VideoSendStreamTest::kDefaultTimeout),
2371 FakeEncoder(Clock::GetRealTimeClock()),
2372 video_codec_type_(video_codec_type),
2373 stream_(nullptr),
2374 encoder_factory_(this),
2375 task_queue_(task_queue) {
2376 InitCodecSpecifics();
2377 }
2378
2379 private:
ModifyVideoConfigs(VideoSendStream::Config * send_config,std::vector<VideoReceiveStreamInterface::Config> * receive_configs,VideoEncoderConfig * encoder_config)2380 void ModifyVideoConfigs(
2381 VideoSendStream::Config* send_config,
2382 std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
2383 VideoEncoderConfig* encoder_config) override {
2384 send_config->encoder_settings.encoder_factory = &encoder_factory_;
2385 send_config->rtp.payload_name = CodecTypeToPayloadString(video_codec_type_);
2386
2387 encoder_config->codec_type = video_codec_type_;
2388 encoder_config->encoder_specific_settings = GetEncoderSpecificSettings();
2389 EXPECT_EQ(1u, encoder_config->simulcast_layers.size());
2390 encoder_config->simulcast_layers[0].num_temporal_layers =
2391 kVideoCodecConfigObserverNumberOfTemporalLayers;
2392 encoder_config_ = encoder_config->Copy();
2393 }
2394
OnVideoStreamsCreated(VideoSendStream * send_stream,const std::vector<VideoReceiveStreamInterface * > & receive_streams)2395 void OnVideoStreamsCreated(VideoSendStream* send_stream,
2396 const std::vector<VideoReceiveStreamInterface*>&
2397 receive_streams) override {
2398 stream_ = send_stream;
2399 }
2400
InitEncode(const VideoCodec * config,const Settings & settings)2401 int32_t InitEncode(const VideoCodec* config,
2402 const Settings& settings) override {
2403 EXPECT_EQ(video_codec_type_, config->codecType);
2404 VerifyCodecSpecifics(*config);
2405 int ret = FakeEncoder::InitEncode(config, settings);
2406 init_encode_event_.Set();
2407 return ret;
2408 }
2409
2410 void InitCodecSpecifics();
2411 void VerifyCodecSpecifics(const VideoCodec& config) const;
2412 rtc::scoped_refptr<VideoEncoderConfig::EncoderSpecificSettings>
2413 GetEncoderSpecificSettings() const;
2414
PerformTest()2415 void PerformTest() override {
2416 EXPECT_TRUE(init_encode_event_.Wait(VideoSendStreamTest::kDefaultTimeout));
2417 ASSERT_EQ(1, FakeEncoder::GetNumInitializations())
2418 << "VideoEncoder not initialized.";
2419
2420 // Change encoder settings to actually trigger reconfiguration.
2421 encoder_config_.frame_drop_enabled = !encoder_config_.frame_drop_enabled;
2422 encoder_config_.encoder_specific_settings = GetEncoderSpecificSettings();
2423 SendTask(task_queue_, [&]() {
2424 stream_->ReconfigureVideoEncoder(std::move(encoder_config_));
2425 });
2426 ASSERT_TRUE(init_encode_event_.Wait(VideoSendStreamTest::kDefaultTimeout));
2427 EXPECT_EQ(2, FakeEncoder::GetNumInitializations())
2428 << "ReconfigureVideoEncoder did not reinitialize the encoder with "
2429 "new encoder settings.";
2430 }
2431
Encode(const VideoFrame & input_image,const std::vector<VideoFrameType> * frame_types)2432 int32_t Encode(const VideoFrame& input_image,
2433 const std::vector<VideoFrameType>* frame_types) override {
2434 // Silently skip the encode, FakeEncoder::Encode doesn't produce VP8.
2435 return 0;
2436 }
2437
2438 T encoder_settings_;
2439 const VideoCodecType video_codec_type_;
2440 rtc::Event init_encode_event_;
2441 VideoSendStream* stream_;
2442 test::VideoEncoderProxyFactory encoder_factory_;
2443 VideoEncoderConfig encoder_config_;
2444 TaskQueueBase* task_queue_;
2445 };
2446
2447 template <>
InitCodecSpecifics()2448 void VideoCodecConfigObserver<VideoCodecH264>::InitCodecSpecifics() {}
2449
2450 template <>
VerifyCodecSpecifics(const VideoCodec & config) const2451 void VideoCodecConfigObserver<VideoCodecH264>::VerifyCodecSpecifics(
2452 const VideoCodec& config) const {
2453 // Check that the number of temporal layers has propagated properly to
2454 // VideoCodec.
2455 EXPECT_EQ(kVideoCodecConfigObserverNumberOfTemporalLayers,
2456 config.H264().numberOfTemporalLayers);
2457
2458 for (unsigned char i = 0; i < config.numberOfSimulcastStreams; ++i) {
2459 EXPECT_EQ(kVideoCodecConfigObserverNumberOfTemporalLayers,
2460 config.simulcastStream[i].numberOfTemporalLayers);
2461 }
2462
2463 // Set expected temporal layers as they should have been set when
2464 // reconfiguring the encoder and not match the set config.
2465 VideoCodecH264 encoder_settings = VideoEncoder::GetDefaultH264Settings();
2466 encoder_settings.numberOfTemporalLayers =
2467 kVideoCodecConfigObserverNumberOfTemporalLayers;
2468 EXPECT_EQ(config.H264(), encoder_settings);
2469 }
2470
2471 template <>
2472 rtc::scoped_refptr<VideoEncoderConfig::EncoderSpecificSettings>
GetEncoderSpecificSettings() const2473 VideoCodecConfigObserver<VideoCodecH264>::GetEncoderSpecificSettings() const {
2474 return nullptr;
2475 }
2476
2477 template <>
InitCodecSpecifics()2478 void VideoCodecConfigObserver<VideoCodecVP8>::InitCodecSpecifics() {
2479 encoder_settings_ = VideoEncoder::GetDefaultVp8Settings();
2480 }
2481
2482 template <>
VerifyCodecSpecifics(const VideoCodec & config) const2483 void VideoCodecConfigObserver<VideoCodecVP8>::VerifyCodecSpecifics(
2484 const VideoCodec& config) const {
2485 // Check that the number of temporal layers has propagated properly to
2486 // VideoCodec.
2487 EXPECT_EQ(kVideoCodecConfigObserverNumberOfTemporalLayers,
2488 config.VP8().numberOfTemporalLayers);
2489
2490 for (unsigned char i = 0; i < config.numberOfSimulcastStreams; ++i) {
2491 EXPECT_EQ(kVideoCodecConfigObserverNumberOfTemporalLayers,
2492 config.simulcastStream[i].numberOfTemporalLayers);
2493 }
2494
2495 // Set expected temporal layers as they should have been set when
2496 // reconfiguring the encoder and not match the set config.
2497 VideoCodecVP8 encoder_settings = encoder_settings_;
2498 encoder_settings.numberOfTemporalLayers =
2499 kVideoCodecConfigObserverNumberOfTemporalLayers;
2500 EXPECT_EQ(
2501 0, memcmp(&config.VP8(), &encoder_settings, sizeof(encoder_settings_)));
2502 }
2503
2504 template <>
2505 rtc::scoped_refptr<VideoEncoderConfig::EncoderSpecificSettings>
GetEncoderSpecificSettings() const2506 VideoCodecConfigObserver<VideoCodecVP8>::GetEncoderSpecificSettings() const {
2507 return rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
2508 encoder_settings_);
2509 }
2510
2511 template <>
InitCodecSpecifics()2512 void VideoCodecConfigObserver<VideoCodecVP9>::InitCodecSpecifics() {
2513 encoder_settings_ = VideoEncoder::GetDefaultVp9Settings();
2514 }
2515
2516 template <>
VerifyCodecSpecifics(const VideoCodec & config) const2517 void VideoCodecConfigObserver<VideoCodecVP9>::VerifyCodecSpecifics(
2518 const VideoCodec& config) const {
2519 // Check that the number of temporal layers has propagated properly to
2520 // VideoCodec.
2521 EXPECT_EQ(kVideoCodecConfigObserverNumberOfTemporalLayers,
2522 config.VP9().numberOfTemporalLayers);
2523
2524 for (unsigned char i = 0; i < config.numberOfSimulcastStreams; ++i) {
2525 EXPECT_EQ(kVideoCodecConfigObserverNumberOfTemporalLayers,
2526 config.simulcastStream[i].numberOfTemporalLayers);
2527 }
2528
2529 // Set expected temporal layers as they should have been set when
2530 // reconfiguring the encoder and not match the set config.
2531 VideoCodecVP9 encoder_settings = encoder_settings_;
2532 encoder_settings.numberOfTemporalLayers =
2533 kVideoCodecConfigObserverNumberOfTemporalLayers;
2534 EXPECT_EQ(
2535 0, memcmp(&(config.VP9()), &encoder_settings, sizeof(encoder_settings_)));
2536 }
2537
2538 template <>
2539 rtc::scoped_refptr<VideoEncoderConfig::EncoderSpecificSettings>
GetEncoderSpecificSettings() const2540 VideoCodecConfigObserver<VideoCodecVP9>::GetEncoderSpecificSettings() const {
2541 return rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
2542 encoder_settings_);
2543 }
2544
TEST_F(VideoSendStreamTest,EncoderSetupPropagatesVp8Config)2545 TEST_F(VideoSendStreamTest, EncoderSetupPropagatesVp8Config) {
2546 VideoCodecConfigObserver<VideoCodecVP8> test(kVideoCodecVP8, task_queue());
2547 RunBaseTest(&test);
2548 }
2549
TEST_F(VideoSendStreamTest,EncoderSetupPropagatesVp9Config)2550 TEST_F(VideoSendStreamTest, EncoderSetupPropagatesVp9Config) {
2551 VideoCodecConfigObserver<VideoCodecVP9> test(kVideoCodecVP9, task_queue());
2552 RunBaseTest(&test);
2553 }
2554
2555 // Fails on MSAN: https://bugs.chromium.org/p/webrtc/issues/detail?id=11376.
2556 #if defined(MEMORY_SANITIZER)
2557 #define MAYBE_EncoderSetupPropagatesH264Config \
2558 DISABLED_EncoderSetupPropagatesH264Config
2559 #else
2560 #define MAYBE_EncoderSetupPropagatesH264Config EncoderSetupPropagatesH264Config
2561 #endif
TEST_F(VideoSendStreamTest,MAYBE_EncoderSetupPropagatesH264Config)2562 TEST_F(VideoSendStreamTest, MAYBE_EncoderSetupPropagatesH264Config) {
2563 VideoCodecConfigObserver<VideoCodecH264> test(kVideoCodecH264, task_queue());
2564 RunBaseTest(&test);
2565 }
2566
TEST_F(VideoSendStreamTest,RtcpSenderReportContainsMediaBytesSent)2567 TEST_F(VideoSendStreamTest, RtcpSenderReportContainsMediaBytesSent) {
2568 class RtcpSenderReportTest : public test::SendTest {
2569 public:
2570 RtcpSenderReportTest()
2571 : SendTest(kDefaultTimeout),
2572 rtp_packets_sent_(0),
2573 media_bytes_sent_(0) {}
2574
2575 private:
2576 Action OnSendRtp(const uint8_t* packet, size_t length) override {
2577 MutexLock lock(&mutex_);
2578 RtpPacket rtp_packet;
2579 EXPECT_TRUE(rtp_packet.Parse(packet, length));
2580 ++rtp_packets_sent_;
2581 media_bytes_sent_ += rtp_packet.payload_size();
2582 return SEND_PACKET;
2583 }
2584
2585 Action OnSendRtcp(const uint8_t* packet, size_t length) override {
2586 MutexLock lock(&mutex_);
2587 test::RtcpPacketParser parser;
2588 EXPECT_TRUE(parser.Parse(packet, length));
2589
2590 if (parser.sender_report()->num_packets() > 0) {
2591 // Only compare sent media bytes if SenderPacketCount matches the
2592 // number of sent rtp packets (a new rtp packet could be sent before
2593 // the rtcp packet).
2594 if (parser.sender_report()->sender_octet_count() > 0 &&
2595 parser.sender_report()->sender_packet_count() ==
2596 rtp_packets_sent_) {
2597 EXPECT_EQ(media_bytes_sent_,
2598 parser.sender_report()->sender_octet_count());
2599 observation_complete_.Set();
2600 }
2601 }
2602
2603 return SEND_PACKET;
2604 }
2605
2606 void PerformTest() override {
2607 EXPECT_TRUE(Wait()) << "Timed out while waiting for RTCP sender report.";
2608 }
2609
2610 Mutex mutex_;
2611 size_t rtp_packets_sent_ RTC_GUARDED_BY(&mutex_);
2612 size_t media_bytes_sent_ RTC_GUARDED_BY(&mutex_);
2613 } test;
2614
2615 RunBaseTest(&test);
2616 }
2617
TEST_F(VideoSendStreamTest,TranslatesTwoLayerScreencastToTargetBitrate)2618 TEST_F(VideoSendStreamTest, TranslatesTwoLayerScreencastToTargetBitrate) {
2619 static const int kScreencastMaxTargetBitrateDeltaKbps = 1;
2620
2621 class VideoStreamFactory
2622 : public VideoEncoderConfig::VideoStreamFactoryInterface {
2623 public:
2624 VideoStreamFactory() {}
2625
2626 private:
2627 std::vector<VideoStream> CreateEncoderStreams(
2628 int frame_width,
2629 int frame_height,
2630 const VideoEncoderConfig& encoder_config) override {
2631 std::vector<VideoStream> streams =
2632 test::CreateVideoStreams(frame_width, frame_height, encoder_config);
2633 RTC_CHECK_GT(streams[0].max_bitrate_bps,
2634 kScreencastMaxTargetBitrateDeltaKbps);
2635 streams[0].target_bitrate_bps =
2636 streams[0].max_bitrate_bps -
2637 kScreencastMaxTargetBitrateDeltaKbps * 1000;
2638 return streams;
2639 }
2640 };
2641
2642 class ScreencastTargetBitrateTest : public test::SendTest,
2643 public test::FakeEncoder {
2644 public:
2645 ScreencastTargetBitrateTest()
2646 : SendTest(kDefaultTimeout),
2647 test::FakeEncoder(Clock::GetRealTimeClock()),
2648 encoder_factory_(this) {}
2649
2650 private:
2651 int32_t InitEncode(const VideoCodec* config,
2652 const Settings& settings) override {
2653 EXPECT_EQ(config->numberOfSimulcastStreams, 1);
2654 EXPECT_EQ(static_cast<unsigned int>(kScreencastMaxTargetBitrateDeltaKbps),
2655 config->simulcastStream[0].maxBitrate -
2656 config->simulcastStream[0].targetBitrate);
2657 observation_complete_.Set();
2658 return test::FakeEncoder::InitEncode(config, settings);
2659 }
2660
2661 void ModifyVideoConfigs(
2662 VideoSendStream::Config* send_config,
2663 std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
2664 VideoEncoderConfig* encoder_config) override {
2665 send_config->encoder_settings.encoder_factory = &encoder_factory_;
2666 EXPECT_EQ(1u, encoder_config->number_of_streams);
2667 encoder_config->video_stream_factory =
2668 rtc::make_ref_counted<VideoStreamFactory>();
2669 EXPECT_EQ(1u, encoder_config->simulcast_layers.size());
2670 encoder_config->simulcast_layers[0].num_temporal_layers = 2;
2671 encoder_config->content_type = VideoEncoderConfig::ContentType::kScreen;
2672 }
2673
2674 void PerformTest() override {
2675 EXPECT_TRUE(Wait())
2676 << "Timed out while waiting for the encoder to be initialized.";
2677 }
2678 test::VideoEncoderProxyFactory encoder_factory_;
2679 } test;
2680
2681 RunBaseTest(&test);
2682 }
2683
TEST_F(VideoSendStreamTest,ReconfigureBitratesSetsEncoderBitratesCorrectly)2684 TEST_F(VideoSendStreamTest, ReconfigureBitratesSetsEncoderBitratesCorrectly) {
2685 // These are chosen to be "kind of odd" to not be accidentally checked against
2686 // default values.
2687 static const int kMinBitrateKbps = 137;
2688 static const int kStartBitrateKbps = 345;
2689 static const int kLowerMaxBitrateKbps = 312;
2690 static const int kMaxBitrateKbps = 413;
2691 static const int kIncreasedStartBitrateKbps = 451;
2692 static const int kIncreasedMaxBitrateKbps = 597;
2693 // TODO(bugs.webrtc.org/12058): If these fields trial are on, we get lower
2694 // bitrates than expected by this test, due to encoder pushback and subtracted
2695 // overhead.
2696 webrtc::test::ScopedKeyValueConfig field_trials(
2697 field_trials_, "WebRTC-VideoRateControl/bitrate_adjuster:false/");
2698
2699 class EncoderBitrateThresholdObserver : public test::SendTest,
2700 public VideoBitrateAllocatorFactory,
2701 public test::FakeEncoder {
2702 public:
2703 explicit EncoderBitrateThresholdObserver(TaskQueueBase* task_queue)
2704 : SendTest(kDefaultTimeout),
2705 FakeEncoder(Clock::GetRealTimeClock()),
2706 task_queue_(task_queue),
2707 target_bitrate_(0),
2708 num_rate_allocator_creations_(0),
2709 num_encoder_initializations_(0),
2710 call_(nullptr),
2711 send_stream_(nullptr),
2712 encoder_factory_(this),
2713 bitrate_allocator_factory_(
2714 CreateBuiltinVideoBitrateAllocatorFactory()) {}
2715
2716 private:
2717 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
2718 const VideoCodec& codec) override {
2719 EXPECT_GE(codec.startBitrate, codec.minBitrate);
2720 EXPECT_LE(codec.startBitrate, codec.maxBitrate);
2721 if (num_rate_allocator_creations_ == 0) {
2722 EXPECT_EQ(static_cast<unsigned int>(kMinBitrateKbps), codec.minBitrate);
2723 EXPECT_NEAR(static_cast<unsigned int>(kStartBitrateKbps),
2724 codec.startBitrate, 10);
2725 EXPECT_EQ(static_cast<unsigned int>(kMaxBitrateKbps), codec.maxBitrate);
2726 } else if (num_rate_allocator_creations_ == 1) {
2727 EXPECT_EQ(static_cast<unsigned int>(kLowerMaxBitrateKbps),
2728 codec.maxBitrate);
2729 // The start bitrate should be kept (-1) and capped to the max bitrate.
2730 // Since this is not an end-to-end call no receiver should have been
2731 // returning a REMB that could lower this estimate.
2732 EXPECT_EQ(codec.startBitrate, codec.maxBitrate);
2733 } else if (num_rate_allocator_creations_ == 2) {
2734 EXPECT_EQ(static_cast<unsigned int>(kIncreasedMaxBitrateKbps),
2735 codec.maxBitrate);
2736 // The start bitrate will be whatever the rate BitRateController has
2737 // currently configured but in the span of the set max and min bitrate.
2738 }
2739 ++num_rate_allocator_creations_;
2740 create_rate_allocator_event_.Set();
2741
2742 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
2743 }
2744
2745 int32_t InitEncode(const VideoCodec* codecSettings,
2746 const Settings& settings) override {
2747 EXPECT_EQ(0, num_encoder_initializations_);
2748 EXPECT_EQ(static_cast<unsigned int>(kMinBitrateKbps),
2749 codecSettings->minBitrate);
2750 EXPECT_NEAR(static_cast<unsigned int>(kStartBitrateKbps),
2751 codecSettings->startBitrate, 10);
2752 EXPECT_EQ(static_cast<unsigned int>(kMaxBitrateKbps),
2753 codecSettings->maxBitrate);
2754
2755 ++num_encoder_initializations_;
2756
2757 observation_complete_.Set();
2758 init_encode_event_.Set();
2759
2760 return FakeEncoder::InitEncode(codecSettings, settings);
2761 }
2762
2763 void SetRates(const RateControlParameters& parameters) override {
2764 {
2765 MutexLock lock(&mutex_);
2766 if (target_bitrate_ == parameters.bitrate.get_sum_kbps()) {
2767 FakeEncoder::SetRates(parameters);
2768 return;
2769 }
2770 target_bitrate_ = parameters.bitrate.get_sum_kbps();
2771 }
2772 bitrate_changed_event_.Set();
2773 FakeEncoder::SetRates(parameters);
2774 }
2775
2776 void WaitForSetRates(uint32_t expected_bitrate, int abs_error) {
2777 // Wait for the expected rate to be set. In some cases there can be
2778 // more than one update pending, in which case we keep waiting
2779 // until the correct value has been observed.
2780 // The target_bitrate_ is reduced by the calculated packet overhead.
2781 const int64_t start_time = rtc::TimeMillis();
2782 do {
2783 MutexLock lock(&mutex_);
2784
2785 int error = target_bitrate_ - expected_bitrate;
2786 if ((error < 0 && error >= -abs_error) ||
2787 (error >= 0 && error <= abs_error)) {
2788 return;
2789 }
2790 } while (bitrate_changed_event_.Wait(
2791 std::max(TimeDelta::Millis(1),
2792 VideoSendStreamTest::kDefaultTimeout -
2793 TimeDelta::Millis(rtc::TimeMillis() - start_time))));
2794 MutexLock lock(&mutex_);
2795 EXPECT_NEAR(target_bitrate_, expected_bitrate, abs_error)
2796 << "Timed out while waiting encoder rate to be set.";
2797 }
2798
2799 void ModifySenderBitrateConfig(
2800 BitrateConstraints* bitrate_config) override {
2801 bitrate_config->min_bitrate_bps = kMinBitrateKbps * 1000;
2802 bitrate_config->start_bitrate_bps = kStartBitrateKbps * 1000;
2803 bitrate_config->max_bitrate_bps = kMaxBitrateKbps * 1000;
2804 }
2805
2806 void ModifyVideoConfigs(
2807 VideoSendStream::Config* send_config,
2808 std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
2809 VideoEncoderConfig* encoder_config) override {
2810 send_config->encoder_settings.encoder_factory = &encoder_factory_;
2811 send_config->encoder_settings.bitrate_allocator_factory = this;
2812 // Set bitrates lower/higher than min/max to make sure they are properly
2813 // capped.
2814 encoder_config->max_bitrate_bps = kMaxBitrateKbps * 1000;
2815 EXPECT_EQ(1u, encoder_config->simulcast_layers.size());
2816 encoder_config->simulcast_layers[0].min_bitrate_bps =
2817 kMinBitrateKbps * 1000;
2818 encoder_config_ = encoder_config->Copy();
2819 }
2820
2821 void OnCallsCreated(Call* sender_call, Call* receiver_call) override {
2822 call_ = sender_call;
2823 }
2824
2825 void OnVideoStreamsCreated(VideoSendStream* send_stream,
2826 const std::vector<VideoReceiveStreamInterface*>&
2827 receive_streams) override {
2828 send_stream_ = send_stream;
2829 }
2830
2831 void PerformTest() override {
2832 ASSERT_TRUE(create_rate_allocator_event_.Wait(
2833 VideoSendStreamTest::kDefaultTimeout))
2834 << "Timed out while waiting for rate allocator to be created.";
2835 ASSERT_TRUE(init_encode_event_.Wait(VideoSendStreamTest::kDefaultTimeout))
2836 << "Timed out while waiting for encoder to be configured.";
2837 WaitForSetRates(kStartBitrateKbps, 80);
2838 BitrateConstraints bitrate_config;
2839 bitrate_config.start_bitrate_bps = kIncreasedStartBitrateKbps * 1000;
2840 bitrate_config.max_bitrate_bps = kIncreasedMaxBitrateKbps * 1000;
2841 SendTask(task_queue_, [this, &bitrate_config]() {
2842 call_->GetTransportControllerSend()->SetSdpBitrateParameters(
2843 bitrate_config);
2844 });
2845 // Encoder rate is capped by EncoderConfig max_bitrate_bps.
2846 WaitForSetRates(kMaxBitrateKbps, 10);
2847 encoder_config_.max_bitrate_bps = kLowerMaxBitrateKbps * 1000;
2848 SendTask(task_queue_, [&]() {
2849 send_stream_->ReconfigureVideoEncoder(encoder_config_.Copy());
2850 });
2851 ASSERT_TRUE(create_rate_allocator_event_.Wait(
2852 VideoSendStreamTest::kDefaultTimeout));
2853 EXPECT_EQ(2, num_rate_allocator_creations_)
2854 << "Rate allocator should have been recreated.";
2855
2856 WaitForSetRates(kLowerMaxBitrateKbps, 10);
2857 EXPECT_EQ(1, num_encoder_initializations_);
2858
2859 encoder_config_.max_bitrate_bps = kIncreasedMaxBitrateKbps * 1000;
2860 SendTask(task_queue_, [&]() {
2861 send_stream_->ReconfigureVideoEncoder(encoder_config_.Copy());
2862 });
2863 ASSERT_TRUE(create_rate_allocator_event_.Wait(
2864 VideoSendStreamTest::kDefaultTimeout));
2865 EXPECT_EQ(3, num_rate_allocator_creations_)
2866 << "Rate allocator should have been recreated.";
2867
2868 // Expected target bitrate is the start bitrate set in the call to
2869 // call_->GetTransportControllerSend()->SetSdpBitrateParameters.
2870 WaitForSetRates(kIncreasedStartBitrateKbps, 10);
2871 EXPECT_EQ(1, num_encoder_initializations_);
2872 }
2873
2874 TaskQueueBase* const task_queue_;
2875 rtc::Event create_rate_allocator_event_;
2876 rtc::Event init_encode_event_;
2877 rtc::Event bitrate_changed_event_;
2878 Mutex mutex_;
2879 uint32_t target_bitrate_ RTC_GUARDED_BY(&mutex_);
2880
2881 int num_rate_allocator_creations_;
2882 int num_encoder_initializations_;
2883 webrtc::Call* call_;
2884 webrtc::VideoSendStream* send_stream_;
2885 test::VideoEncoderProxyFactory encoder_factory_;
2886 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
2887 webrtc::VideoEncoderConfig encoder_config_;
2888 } test(task_queue());
2889
2890 RunBaseTest(&test);
2891 }
2892
TEST_F(VideoSendStreamTest,ReportsSentResolution)2893 TEST_F(VideoSendStreamTest, ReportsSentResolution) {
2894 static const size_t kNumStreams = 3;
2895 // Unusual resolutions to make sure that they are the ones being reported.
2896 static const struct {
2897 int width;
2898 int height;
2899 } kEncodedResolution[kNumStreams] = {{241, 181}, {300, 121}, {121, 221}};
2900 class ScreencastTargetBitrateTest : public test::SendTest,
2901 public test::FakeEncoder {
2902 public:
2903 explicit ScreencastTargetBitrateTest(TaskQueueBase* task_queue)
2904 : SendTest(kDefaultTimeout),
2905 test::FakeEncoder(Clock::GetRealTimeClock()),
2906 send_stream_(nullptr),
2907 encoder_factory_(this),
2908 task_queue_(task_queue) {}
2909
2910 private:
2911 int32_t Encode(const VideoFrame& input_image,
2912 const std::vector<VideoFrameType>* frame_types) override {
2913 CodecSpecificInfo specifics;
2914 specifics.codecType = kVideoCodecGeneric;
2915
2916 EncodedImage encoded;
2917 auto buffer = EncodedImageBuffer::Create(16);
2918 memset(buffer->data(), 0, 16);
2919 encoded.SetEncodedData(buffer);
2920 encoded.SetTimestamp(input_image.timestamp());
2921 encoded.capture_time_ms_ = input_image.render_time_ms();
2922
2923 for (size_t i = 0; i < kNumStreams; ++i) {
2924 encoded._frameType = (*frame_types)[i];
2925 encoded._encodedWidth = kEncodedResolution[i].width;
2926 encoded._encodedHeight = kEncodedResolution[i].height;
2927 encoded.SetSpatialIndex(i);
2928 EncodedImageCallback* callback;
2929 {
2930 MutexLock lock(&mutex_);
2931 callback = callback_;
2932 }
2933 RTC_DCHECK(callback);
2934 if (callback->OnEncodedImage(encoded, &specifics).error !=
2935 EncodedImageCallback::Result::OK) {
2936 return -1;
2937 }
2938 }
2939
2940 observation_complete_.Set();
2941 return 0;
2942 }
2943 void ModifyVideoConfigs(
2944 VideoSendStream::Config* send_config,
2945 std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
2946 VideoEncoderConfig* encoder_config) override {
2947 send_config->encoder_settings.encoder_factory = &encoder_factory_;
2948 EXPECT_EQ(kNumStreams, encoder_config->number_of_streams);
2949 }
2950
2951 size_t GetNumVideoStreams() const override { return kNumStreams; }
2952
2953 void PerformTest() override {
2954 EXPECT_TRUE(Wait())
2955 << "Timed out while waiting for the encoder to send one frame.";
2956 VideoSendStream::Stats stats;
2957 SendTask(task_queue_, [&]() { stats = send_stream_->GetStats(); });
2958
2959 for (size_t i = 0; i < kNumStreams; ++i) {
2960 ASSERT_TRUE(stats.substreams.find(kVideoSendSsrcs[i]) !=
2961 stats.substreams.end())
2962 << "No stats for SSRC: " << kVideoSendSsrcs[i]
2963 << ", stats should exist as soon as frames have been encoded.";
2964 VideoSendStream::StreamStats ssrc_stats =
2965 stats.substreams[kVideoSendSsrcs[i]];
2966 EXPECT_EQ(kEncodedResolution[i].width, ssrc_stats.width);
2967 EXPECT_EQ(kEncodedResolution[i].height, ssrc_stats.height);
2968 }
2969 }
2970
2971 void OnVideoStreamsCreated(VideoSendStream* send_stream,
2972 const std::vector<VideoReceiveStreamInterface*>&
2973 receive_streams) override {
2974 send_stream_ = send_stream;
2975 }
2976
2977 VideoSendStream* send_stream_;
2978 test::VideoEncoderProxyFactory encoder_factory_;
2979 TaskQueueBase* const task_queue_;
2980 } test(task_queue());
2981
2982 RunBaseTest(&test);
2983 }
2984
2985 #if defined(RTC_ENABLE_VP9)
2986 class Vp9HeaderObserver : public test::SendTest {
2987 public:
Vp9HeaderObserver(const Vp9TestParams & params)2988 explicit Vp9HeaderObserver(const Vp9TestParams& params)
2989 : SendTest(VideoSendStreamTest::kLongTimeout),
2990 encoder_factory_([]() { return VP9Encoder::Create(); }),
2991 params_(params),
2992 vp9_settings_(VideoEncoder::GetDefaultVp9Settings()) {}
2993
ModifyVideoConfigsHook(VideoSendStream::Config * send_config,std::vector<VideoReceiveStreamInterface::Config> * receive_configs,VideoEncoderConfig * encoder_config)2994 virtual void ModifyVideoConfigsHook(
2995 VideoSendStream::Config* send_config,
2996 std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
2997 VideoEncoderConfig* encoder_config) {}
2998
2999 virtual void InspectHeader(const RTPVideoHeaderVP9& vp9) = 0;
3000
3001 private:
3002 const int kVp9PayloadType = test::CallTest::kVideoSendPayloadType;
3003
ModifyVideoConfigs(VideoSendStream::Config * send_config,std::vector<VideoReceiveStreamInterface::Config> * receive_configs,VideoEncoderConfig * encoder_config)3004 void ModifyVideoConfigs(
3005 VideoSendStream::Config* send_config,
3006 std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
3007 VideoEncoderConfig* encoder_config) override {
3008 send_config->encoder_settings.encoder_factory = &encoder_factory_;
3009 send_config->rtp.payload_name = "VP9";
3010 send_config->rtp.payload_type = kVp9PayloadType;
3011 ModifyVideoConfigsHook(send_config, receive_configs, encoder_config);
3012 encoder_config->encoder_specific_settings =
3013 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
3014 vp9_settings_);
3015 EXPECT_EQ(1u, encoder_config->number_of_streams);
3016 EXPECT_EQ(1u, encoder_config->simulcast_layers.size());
3017 encoder_config_ = encoder_config->Copy();
3018 }
3019
ModifyVideoCaptureStartResolution(int * width,int * height,int * frame_rate)3020 void ModifyVideoCaptureStartResolution(int* width,
3021 int* height,
3022 int* frame_rate) override {
3023 expected_width_ = *width;
3024 expected_height_ = *height;
3025 }
3026
PerformTest()3027 void PerformTest() override {
3028 bool wait = Wait();
3029 {
3030 // In case of time out, OnSendRtp might still access frames_sent_;
3031 MutexLock lock(&mutex_);
3032 EXPECT_TRUE(wait) << "Test timed out waiting for VP9 packet, num frames "
3033 << frames_sent_;
3034 }
3035 }
3036
OnSendRtp(const uint8_t * packet,size_t length)3037 Action OnSendRtp(const uint8_t* packet, size_t length) override {
3038 RtpPacket rtp_packet;
3039 EXPECT_TRUE(rtp_packet.Parse(packet, length));
3040
3041 EXPECT_EQ(kVp9PayloadType, rtp_packet.PayloadType());
3042 rtc::ArrayView<const uint8_t> rtp_payload = rtp_packet.payload();
3043
3044 bool new_packet = !last_packet_sequence_number_.has_value() ||
3045 IsNewerSequenceNumber(rtp_packet.SequenceNumber(),
3046 *last_packet_sequence_number_);
3047 if (!rtp_payload.empty() && new_packet) {
3048 RTPVideoHeader video_header;
3049 EXPECT_NE(
3050 VideoRtpDepacketizerVp9::ParseRtpPayload(rtp_payload, &video_header),
3051 0);
3052 EXPECT_EQ(VideoCodecType::kVideoCodecVP9, video_header.codec);
3053 // Verify common fields for all configurations.
3054 const auto& vp9_header =
3055 absl::get<RTPVideoHeaderVP9>(video_header.video_type_header);
3056 VerifyCommonHeader(vp9_header);
3057 CompareConsecutiveFrames(rtp_packet, video_header);
3058 // Verify configuration specific settings.
3059 InspectHeader(vp9_header);
3060
3061 if (rtp_packet.Marker()) {
3062 MutexLock lock(&mutex_);
3063 ++frames_sent_;
3064 }
3065 last_packet_marker_ = rtp_packet.Marker();
3066 last_packet_sequence_number_ = rtp_packet.SequenceNumber();
3067 last_packet_timestamp_ = rtp_packet.Timestamp();
3068 last_vp9_ = vp9_header;
3069 last_temporal_idx_by_spatial_idx_[vp9_header.spatial_idx] =
3070 vp9_header.temporal_idx;
3071 }
3072 return SEND_PACKET;
3073 }
3074
3075 protected:
ContinuousPictureId(const RTPVideoHeaderVP9 & vp9) const3076 bool ContinuousPictureId(const RTPVideoHeaderVP9& vp9) const {
3077 if (last_vp9_.picture_id > vp9.picture_id) {
3078 return vp9.picture_id == 0; // Wrap.
3079 } else {
3080 return vp9.picture_id == last_vp9_.picture_id + 1;
3081 }
3082 }
3083
IsTemporalShiftEnabled() const3084 bool IsTemporalShiftEnabled() const {
3085 return params_.scalability_mode.find("_SHIFT") != std::string::npos;
3086 }
3087
VerifySpatialIdxWithinFrame(const RTPVideoHeaderVP9 & vp9) const3088 void VerifySpatialIdxWithinFrame(const RTPVideoHeaderVP9& vp9) const {
3089 bool new_layer = vp9.spatial_idx != last_vp9_.spatial_idx;
3090 EXPECT_EQ(new_layer, vp9.beginning_of_frame);
3091 EXPECT_EQ(new_layer, last_vp9_.end_of_frame);
3092 EXPECT_EQ(new_layer ? last_vp9_.spatial_idx + 1 : last_vp9_.spatial_idx,
3093 vp9.spatial_idx);
3094 }
3095
VerifyTemporalIdxWithinFrame(const RTPVideoHeaderVP9 & vp9) const3096 void VerifyTemporalIdxWithinFrame(const RTPVideoHeaderVP9& vp9) const {
3097 if (!IsTemporalShiftEnabled()) {
3098 EXPECT_EQ(vp9.temporal_idx, last_vp9_.temporal_idx);
3099 return;
3100 }
3101 // Temporal shift.
3102 EXPECT_EQ(params_.num_temporal_layers, 2);
3103 if (vp9.spatial_idx == params_.num_spatial_layers - 1) {
3104 // Lower spatial layers should be shifted.
3105 int expected_tid =
3106 (!vp9.inter_pic_predicted || vp9.temporal_idx == 1) ? 0 : 1;
3107 for (int i = 0; i < vp9.spatial_idx; ++i) {
3108 EXPECT_EQ(last_temporal_idx_by_spatial_idx_.at(i), expected_tid);
3109 }
3110 }
3111 // Same within spatial layer.
3112 bool new_layer = vp9.spatial_idx != last_vp9_.spatial_idx;
3113 if (!new_layer) {
3114 EXPECT_EQ(vp9.temporal_idx, last_vp9_.temporal_idx);
3115 }
3116 }
3117
VerifyFixedTemporalLayerStructure(const RTPVideoHeaderVP9 & vp9,uint8_t num_layers) const3118 void VerifyFixedTemporalLayerStructure(const RTPVideoHeaderVP9& vp9,
3119 uint8_t num_layers) const {
3120 switch (num_layers) {
3121 case 0:
3122 VerifyTemporalLayerStructure0(vp9);
3123 break;
3124 case 1:
3125 VerifyTemporalLayerStructure1(vp9);
3126 break;
3127 case 2:
3128 VerifyTemporalLayerStructure2(vp9);
3129 break;
3130 case 3:
3131 VerifyTemporalLayerStructure3(vp9);
3132 break;
3133 default:
3134 RTC_DCHECK_NOTREACHED();
3135 }
3136 }
3137
VerifyTemporalLayerStructure0(const RTPVideoHeaderVP9 & vp9) const3138 void VerifyTemporalLayerStructure0(const RTPVideoHeaderVP9& vp9) const {
3139 EXPECT_EQ(kNoTl0PicIdx, vp9.tl0_pic_idx);
3140 EXPECT_EQ(kNoTemporalIdx, vp9.temporal_idx); // no tid
3141 // Technically true, but layer indices not available.
3142 EXPECT_FALSE(vp9.temporal_up_switch);
3143 }
3144
VerifyTemporalLayerStructure1(const RTPVideoHeaderVP9 & vp9) const3145 void VerifyTemporalLayerStructure1(const RTPVideoHeaderVP9& vp9) const {
3146 EXPECT_NE(kNoTl0PicIdx, vp9.tl0_pic_idx);
3147 EXPECT_EQ(0, vp9.temporal_idx); // 0,0,0,...
3148 }
3149
VerifyTemporalLayerStructure2(const RTPVideoHeaderVP9 & vp9) const3150 void VerifyTemporalLayerStructure2(const RTPVideoHeaderVP9& vp9) const {
3151 EXPECT_NE(kNoTl0PicIdx, vp9.tl0_pic_idx);
3152 EXPECT_GE(vp9.temporal_idx, 0); // 0,1,0,1,... (tid reset on I-frames).
3153 EXPECT_LE(vp9.temporal_idx, 1);
3154 EXPECT_TRUE(vp9.temporal_up_switch);
3155 // Verify temporal structure for the highest spatial layer (the structure
3156 // may be shifted for lower spatial layer if temporal shift is configured).
3157 if (IsHighestSpatialLayer(vp9) && vp9.beginning_of_frame) {
3158 int expected_tid =
3159 (!vp9.inter_pic_predicted ||
3160 last_temporal_idx_by_spatial_idx_.at(vp9.spatial_idx) == 1)
3161 ? 0
3162 : 1;
3163 EXPECT_EQ(vp9.temporal_idx, expected_tid);
3164 }
3165 }
3166
VerifyTemporalLayerStructure3(const RTPVideoHeaderVP9 & vp9) const3167 void VerifyTemporalLayerStructure3(const RTPVideoHeaderVP9& vp9) const {
3168 EXPECT_NE(kNoTl0PicIdx, vp9.tl0_pic_idx);
3169 EXPECT_GE(vp9.temporal_idx, 0); // 0,2,1,2,... (tid reset on I-frames).
3170 EXPECT_LE(vp9.temporal_idx, 2);
3171 if (IsNewPictureId(vp9) && vp9.inter_pic_predicted) {
3172 EXPECT_NE(vp9.temporal_idx, last_vp9_.temporal_idx);
3173 EXPECT_TRUE(vp9.temporal_up_switch);
3174 switch (vp9.temporal_idx) {
3175 case 0:
3176 EXPECT_EQ(last_vp9_.temporal_idx, 2);
3177 break;
3178 case 1:
3179 EXPECT_EQ(last_vp9_.temporal_idx, 2);
3180 break;
3181 case 2:
3182 EXPECT_LT(last_vp9_.temporal_idx, 2);
3183 break;
3184 }
3185 }
3186 }
3187
VerifyTl0Idx(const RTPVideoHeaderVP9 & vp9) const3188 void VerifyTl0Idx(const RTPVideoHeaderVP9& vp9) const {
3189 if (vp9.tl0_pic_idx == kNoTl0PicIdx)
3190 return;
3191
3192 uint8_t expected_tl0_idx = last_vp9_.tl0_pic_idx;
3193 if (vp9.temporal_idx == 0)
3194 ++expected_tl0_idx;
3195 EXPECT_EQ(expected_tl0_idx, vp9.tl0_pic_idx);
3196 }
3197
IsNewPictureId(const RTPVideoHeaderVP9 & vp9) const3198 bool IsNewPictureId(const RTPVideoHeaderVP9& vp9) const {
3199 return frames_sent_ > 0 && (vp9.picture_id != last_vp9_.picture_id);
3200 }
3201
IsHighestSpatialLayer(const RTPVideoHeaderVP9 & vp9) const3202 bool IsHighestSpatialLayer(const RTPVideoHeaderVP9& vp9) const {
3203 return vp9.spatial_idx == params_.num_spatial_layers - 1 ||
3204 vp9.spatial_idx == kNoSpatialIdx;
3205 }
3206
3207 // Flexible mode (F=1): Non-flexible mode (F=0):
3208 //
3209 // +-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+
3210 // |I|P|L|F|B|E|V|-| |I|P|L|F|B|E|V|-|
3211 // +-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+
3212 // I: |M| PICTURE ID | I: |M| PICTURE ID |
3213 // +-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+
3214 // M: | EXTENDED PID | M: | EXTENDED PID |
3215 // +-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+
3216 // L: | T |U| S |D| L: | T |U| S |D|
3217 // +-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+
3218 // P,F: | P_DIFF |X|N| | TL0PICIDX |
3219 // +-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+
3220 // X: |EXTENDED P_DIFF| V: | SS .. |
3221 // +-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+
3222 // V: | SS .. |
3223 // +-+-+-+-+-+-+-+-+
VerifyCommonHeader(const RTPVideoHeaderVP9 & vp9) const3224 void VerifyCommonHeader(const RTPVideoHeaderVP9& vp9) const {
3225 EXPECT_EQ(kMaxTwoBytePictureId, vp9.max_picture_id); // M:1
3226 EXPECT_NE(kNoPictureId, vp9.picture_id); // I:1
3227 EXPECT_EQ(vp9_settings_.flexibleMode, vp9.flexible_mode); // F
3228
3229 if (params_.num_spatial_layers > 1) {
3230 EXPECT_LT(vp9.spatial_idx, params_.num_spatial_layers);
3231 } else if (params_.num_temporal_layers > 1) {
3232 EXPECT_EQ(vp9.spatial_idx, 0);
3233 } else {
3234 EXPECT_EQ(vp9.spatial_idx, kNoSpatialIdx);
3235 }
3236
3237 if (params_.num_temporal_layers > 1) {
3238 EXPECT_LT(vp9.temporal_idx, params_.num_temporal_layers);
3239 } else if (params_.num_spatial_layers > 1) {
3240 EXPECT_EQ(vp9.temporal_idx, 0);
3241 } else {
3242 EXPECT_EQ(vp9.temporal_idx, kNoTemporalIdx);
3243 }
3244
3245 if (vp9.ss_data_available) // V
3246 VerifySsData(vp9);
3247
3248 if (frames_sent_ == 0)
3249 EXPECT_FALSE(vp9.inter_pic_predicted); // P
3250
3251 if (!vp9.inter_pic_predicted) {
3252 if (vp9.temporal_idx == kNoTemporalIdx) {
3253 EXPECT_FALSE(vp9.temporal_up_switch);
3254 } else {
3255 EXPECT_EQ(vp9.temporal_idx, 0);
3256 EXPECT_TRUE(vp9.temporal_up_switch);
3257 }
3258 }
3259 }
3260
3261 // Scalability structure (SS).
3262 //
3263 // +-+-+-+-+-+-+-+-+
3264 // V: | N_S |Y|G|-|-|-|
3265 // +-+-+-+-+-+-+-+-+
3266 // Y: | WIDTH | N_S + 1 times
3267 // +-+-+-+-+-+-+-+-+
3268 // | HEIGHT |
3269 // +-+-+-+-+-+-+-+-+
3270 // G: | N_G |
3271 // +-+-+-+-+-+-+-+-+
3272 // N_G: | T |U| R |-|-| N_G times
3273 // +-+-+-+-+-+-+-+-+
3274 // | P_DIFF | R times
3275 // +-+-+-+-+-+-+-+-+
VerifySsData(const RTPVideoHeaderVP9 & vp9) const3276 void VerifySsData(const RTPVideoHeaderVP9& vp9) const {
3277 EXPECT_TRUE(vp9.ss_data_available); // V
3278 EXPECT_EQ(params_.num_spatial_layers, // N_S + 1
3279 vp9.num_spatial_layers);
3280 EXPECT_TRUE(vp9.spatial_layer_resolution_present); // Y:1
3281
3282 ScalableVideoController::StreamLayersConfig config = GetScalabilityConfig();
3283 for (int i = config.num_spatial_layers - 1; i >= 0; --i) {
3284 double ratio = static_cast<double>(config.scaling_factor_num[i]) /
3285 config.scaling_factor_den[i];
3286 EXPECT_EQ(expected_width_ * ratio, vp9.width[i]); // WIDTH
3287 EXPECT_EQ(expected_height_ * ratio, vp9.height[i]); // HEIGHT
3288 }
3289 }
3290
CompareConsecutiveFrames(const RtpPacket & rtp_packet,const RTPVideoHeader & video) const3291 void CompareConsecutiveFrames(const RtpPacket& rtp_packet,
3292 const RTPVideoHeader& video) const {
3293 const auto& vp9_header =
3294 absl::get<RTPVideoHeaderVP9>(video.video_type_header);
3295
3296 const bool new_temporal_unit =
3297 !last_packet_timestamp_.has_value() ||
3298 IsNewerTimestamp(rtp_packet.Timestamp(), *last_packet_timestamp_);
3299 const bool new_frame =
3300 new_temporal_unit || last_vp9_.spatial_idx != vp9_header.spatial_idx;
3301
3302 EXPECT_EQ(new_frame, video.is_first_packet_in_frame);
3303 if (!new_temporal_unit) {
3304 EXPECT_FALSE(last_packet_marker_);
3305 EXPECT_EQ(*last_packet_timestamp_, rtp_packet.Timestamp());
3306 EXPECT_EQ(last_vp9_.picture_id, vp9_header.picture_id);
3307 EXPECT_EQ(last_vp9_.tl0_pic_idx, vp9_header.tl0_pic_idx);
3308 VerifySpatialIdxWithinFrame(vp9_header);
3309 VerifyTemporalIdxWithinFrame(vp9_header);
3310 return;
3311 }
3312 // New frame.
3313 EXPECT_TRUE(vp9_header.beginning_of_frame);
3314
3315 // Compare with last packet in previous frame.
3316 if (frames_sent_ == 0)
3317 return;
3318 EXPECT_TRUE(last_vp9_.end_of_frame);
3319 EXPECT_TRUE(last_packet_marker_);
3320 EXPECT_TRUE(ContinuousPictureId(vp9_header));
3321 VerifyTl0Idx(vp9_header);
3322 }
3323
GetScalabilityConfig() const3324 ScalableVideoController::StreamLayersConfig GetScalabilityConfig() const {
3325 absl::optional<ScalabilityMode> scalability_mode =
3326 ScalabilityModeFromString(params_.scalability_mode);
3327 EXPECT_TRUE(scalability_mode.has_value());
3328 absl::optional<ScalableVideoController::StreamLayersConfig> config =
3329 ScalabilityStructureConfig(*scalability_mode);
3330 EXPECT_TRUE(config.has_value());
3331 EXPECT_EQ(config->num_spatial_layers, params_.num_spatial_layers);
3332 return *config;
3333 }
3334
3335 test::FunctionVideoEncoderFactory encoder_factory_;
3336 const Vp9TestParams params_;
3337 VideoCodecVP9 vp9_settings_;
3338 webrtc::VideoEncoderConfig encoder_config_;
3339 bool last_packet_marker_ = false;
3340 absl::optional<uint16_t> last_packet_sequence_number_;
3341 absl::optional<uint32_t> last_packet_timestamp_;
3342 RTPVideoHeaderVP9 last_vp9_;
3343 std::map<int, int> last_temporal_idx_by_spatial_idx_;
3344 Mutex mutex_;
3345 size_t frames_sent_ = 0;
3346 int expected_width_ = 0;
3347 int expected_height_ = 0;
3348 };
3349
3350 class Vp9Test : public VideoSendStreamTest,
3351 public ::testing::WithParamInterface<ParameterizationType> {
3352 public:
Vp9Test()3353 Vp9Test()
3354 : params_(::testing::get<Vp9TestParams>(GetParam())),
3355 use_scalability_mode_identifier_(::testing::get<bool>(GetParam())) {}
3356
3357 protected:
3358 const Vp9TestParams params_;
3359 const bool use_scalability_mode_identifier_;
3360 };
3361
3362 INSTANTIATE_TEST_SUITE_P(
3363 ScalabilityMode,
3364 Vp9Test,
3365 ::testing::Combine(
3366 ::testing::ValuesIn<Vp9TestParams>(
3367 {{"L1T1", 1, 1, InterLayerPredMode::kOn},
3368 {"L1T2", 1, 2, InterLayerPredMode::kOn},
3369 {"L1T3", 1, 3, InterLayerPredMode::kOn},
3370 {"L2T1", 2, 1, InterLayerPredMode::kOn},
3371 {"L2T1_KEY", 2, 1, InterLayerPredMode::kOnKeyPic},
3372 {"L2T2", 2, 2, InterLayerPredMode::kOn},
3373 {"L2T2_KEY", 2, 2, InterLayerPredMode::kOnKeyPic},
3374 {"L2T3", 2, 3, InterLayerPredMode::kOn},
3375 {"L2T3_KEY", 2, 3, InterLayerPredMode::kOnKeyPic},
3376 {"L3T1", 3, 1, InterLayerPredMode::kOn},
3377 {"L3T1_KEY", 3, 1, InterLayerPredMode::kOnKeyPic},
3378 {"L3T2", 3, 2, InterLayerPredMode::kOn},
3379 {"L3T2_KEY", 3, 2, InterLayerPredMode::kOnKeyPic},
3380 {"L3T3", 3, 3, InterLayerPredMode::kOn},
3381 {"L3T3_KEY", 3, 3, InterLayerPredMode::kOnKeyPic},
3382 {"S2T1", 2, 1, InterLayerPredMode::kOff},
3383 {"S2T2", 2, 2, InterLayerPredMode::kOff},
3384 {"S2T3", 2, 3, InterLayerPredMode::kOff},
3385 {"S3T1", 3, 1, InterLayerPredMode::kOff},
3386 {"S3T2", 3, 2, InterLayerPredMode::kOff},
3387 {"S3T3", 3, 3, InterLayerPredMode::kOff}}),
3388 ::testing::Values(false, true)), // use_scalability_mode_identifier
3389 ParamInfoToStr);
3390
3391 INSTANTIATE_TEST_SUITE_P(
3392 ScalabilityModeOn,
3393 Vp9Test,
3394 ::testing::Combine(
3395 ::testing::ValuesIn<Vp9TestParams>(
3396 {{"L2T1h", 2, 1, InterLayerPredMode::kOn},
3397 {"L2T2h", 2, 2, InterLayerPredMode::kOn},
3398 {"L2T3h", 2, 3, InterLayerPredMode::kOn},
3399 {"L2T2_KEY_SHIFT", 2, 2, InterLayerPredMode::kOnKeyPic},
3400 {"L3T1h", 3, 1, InterLayerPredMode::kOn},
3401 {"L3T2h", 3, 2, InterLayerPredMode::kOn},
3402 {"L3T3h", 3, 3, InterLayerPredMode::kOn},
3403 {"S2T1h", 2, 1, InterLayerPredMode::kOff},
3404 {"S2T2h", 2, 2, InterLayerPredMode::kOff},
3405 {"S2T3h", 2, 3, InterLayerPredMode::kOff},
3406 {"S3T1h", 3, 1, InterLayerPredMode::kOff},
3407 {"S3T2h", 3, 2, InterLayerPredMode::kOff},
3408 {"S3T3h", 3, 3, InterLayerPredMode::kOff}}),
3409 ::testing::Values(true)), // use_scalability_mode_identifier
3410 ParamInfoToStr);
3411
TEST_P(Vp9Test,NonFlexMode)3412 TEST_P(Vp9Test, NonFlexMode) {
3413 TestVp9NonFlexMode(params_, use_scalability_mode_identifier_);
3414 }
3415
TestVp9NonFlexMode(const Vp9TestParams & params,bool use_scalability_mode_identifier)3416 void VideoSendStreamTest::TestVp9NonFlexMode(
3417 const Vp9TestParams& params,
3418 bool use_scalability_mode_identifier) {
3419 static const size_t kNumFramesToSend = 100;
3420 // Set to < kNumFramesToSend and coprime to length of temporal layer
3421 // structures to verify temporal id reset on key frame.
3422 static const int kKeyFrameInterval = 31;
3423
3424 static const int kWidth = kMinVp9SpatialLayerLongSideLength;
3425 static const int kHeight = kMinVp9SpatialLayerShortSideLength;
3426 static const float kGoodBitsPerPixel = 0.1f;
3427 class NonFlexibleMode : public Vp9HeaderObserver {
3428 public:
3429 NonFlexibleMode(const Vp9TestParams& params,
3430 bool use_scalability_mode_identifier)
3431 : Vp9HeaderObserver(params),
3432 use_scalability_mode_identifier_(use_scalability_mode_identifier),
3433 l_field_(params.num_temporal_layers > 1 ||
3434 params.num_spatial_layers > 1) {}
3435
3436 void ModifyVideoConfigsHook(
3437 VideoSendStream::Config* send_config,
3438 std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
3439 VideoEncoderConfig* encoder_config) override {
3440 encoder_config->codec_type = kVideoCodecVP9;
3441 int bitrate_bps = 0;
3442 for (int sl_idx = 0; sl_idx < params_.num_spatial_layers; ++sl_idx) {
3443 const int width = kWidth << sl_idx;
3444 const int height = kHeight << sl_idx;
3445 const float bpp = kGoodBitsPerPixel / (1 << sl_idx);
3446 bitrate_bps += static_cast<int>(width * height * bpp * 30);
3447 }
3448 encoder_config->max_bitrate_bps = bitrate_bps * 2;
3449
3450 encoder_config->frame_drop_enabled = false;
3451
3452 vp9_settings_.flexibleMode = false;
3453 vp9_settings_.automaticResizeOn = false;
3454 vp9_settings_.keyFrameInterval = kKeyFrameInterval;
3455 if (!use_scalability_mode_identifier_) {
3456 vp9_settings_.numberOfTemporalLayers = params_.num_temporal_layers;
3457 vp9_settings_.numberOfSpatialLayers = params_.num_spatial_layers;
3458 vp9_settings_.interLayerPred = params_.inter_layer_pred;
3459 } else {
3460 absl::optional<ScalabilityMode> mode =
3461 ScalabilityModeFromString(params_.scalability_mode);
3462 encoder_config->simulcast_layers[0].scalability_mode = mode;
3463 EXPECT_TRUE(mode.has_value());
3464 }
3465 }
3466
3467 int GetRequiredDivisibility() const {
3468 ScalableVideoController::StreamLayersConfig config =
3469 GetScalabilityConfig();
3470 int required_divisibility = 1;
3471 for (int sl_idx = 0; sl_idx < config.num_spatial_layers; ++sl_idx) {
3472 required_divisibility = cricket::LeastCommonMultiple(
3473 required_divisibility, config.scaling_factor_den[sl_idx]);
3474 }
3475 return required_divisibility;
3476 }
3477
3478 void ModifyVideoCaptureStartResolution(int* width,
3479 int* height,
3480 int* frame_rate) override {
3481 expected_width_ = kWidth << (params_.num_spatial_layers - 1);
3482 expected_height_ = kHeight << (params_.num_spatial_layers - 1);
3483 *width = expected_width_;
3484 *height = expected_height_;
3485 // Top layer may be adjusted to ensure evenly divided layers.
3486 int divisibility = GetRequiredDivisibility();
3487 expected_width_ -= (expected_width_ % divisibility);
3488 expected_height_ -= (expected_height_ % divisibility);
3489 }
3490
3491 void InspectHeader(const RTPVideoHeaderVP9& vp9) override {
3492 bool ss_data_expected = !vp9.inter_pic_predicted &&
3493 vp9.beginning_of_frame &&
3494 !vp9.inter_layer_predicted;
3495 EXPECT_EQ(ss_data_expected, vp9.ss_data_available);
3496
3497 bool is_key_frame = frames_sent_ % kKeyFrameInterval == 0;
3498 if (params_.num_spatial_layers > 1) {
3499 switch (params_.inter_layer_pred) {
3500 case InterLayerPredMode::kOff:
3501 EXPECT_FALSE(vp9.inter_layer_predicted);
3502 break;
3503 case InterLayerPredMode::kOn:
3504 EXPECT_EQ(vp9.spatial_idx > 0, vp9.inter_layer_predicted);
3505 break;
3506 case InterLayerPredMode::kOnKeyPic:
3507 EXPECT_EQ(is_key_frame && vp9.spatial_idx > 0,
3508 vp9.inter_layer_predicted);
3509 break;
3510 }
3511 } else {
3512 EXPECT_FALSE(vp9.inter_layer_predicted);
3513 }
3514
3515 EXPECT_EQ(is_key_frame, !vp9.inter_pic_predicted);
3516
3517 if (IsNewPictureId(vp9)) {
3518 if (params_.num_temporal_layers == 1 &&
3519 params_.num_spatial_layers == 1) {
3520 EXPECT_EQ(kNoSpatialIdx, vp9.spatial_idx);
3521 } else {
3522 EXPECT_EQ(0, vp9.spatial_idx);
3523 }
3524 if (params_.num_spatial_layers > 1)
3525 EXPECT_EQ(params_.num_spatial_layers - 1, last_vp9_.spatial_idx);
3526 }
3527
3528 VerifyFixedTemporalLayerStructure(
3529 vp9, l_field_ ? params_.num_temporal_layers : 0);
3530
3531 if (frames_sent_ > kNumFramesToSend)
3532 observation_complete_.Set();
3533 }
3534 const bool use_scalability_mode_identifier_;
3535 const bool l_field_;
3536
3537 private:
3538 void ModifySenderBitrateConfig(
3539 BitrateConstraints* bitrate_config) override {
3540 const int kBitrateBps = 800000;
3541 bitrate_config->min_bitrate_bps = kBitrateBps;
3542 bitrate_config->start_bitrate_bps = kBitrateBps;
3543 }
3544 } test(params, use_scalability_mode_identifier);
3545
3546 RunBaseTest(&test);
3547 }
3548
TEST_F(VideoSendStreamTest,Vp9NonFlexModeSmallResolution)3549 TEST_F(VideoSendStreamTest, Vp9NonFlexModeSmallResolution) {
3550 static const size_t kNumFramesToSend = 50;
3551 static const int kWidth = 4;
3552 static const int kHeight = 4;
3553 class NonFlexibleModeResolution : public Vp9HeaderObserver {
3554 public:
3555 explicit NonFlexibleModeResolution(const Vp9TestParams& params)
3556 : Vp9HeaderObserver(params) {}
3557
3558 private:
3559 void ModifyVideoConfigsHook(
3560 VideoSendStream::Config* send_config,
3561 std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
3562 VideoEncoderConfig* encoder_config) override {
3563 encoder_config->codec_type = kVideoCodecVP9;
3564 vp9_settings_.flexibleMode = false;
3565 vp9_settings_.numberOfTemporalLayers = params_.num_temporal_layers;
3566 vp9_settings_.numberOfSpatialLayers = params_.num_spatial_layers;
3567 vp9_settings_.interLayerPred = params_.inter_layer_pred;
3568 }
3569
3570 void InspectHeader(const RTPVideoHeaderVP9& vp9_header) override {
3571 if (frames_sent_ > kNumFramesToSend)
3572 observation_complete_.Set();
3573 }
3574
3575 void ModifyVideoCaptureStartResolution(int* width,
3576 int* height,
3577 int* frame_rate) override {
3578 expected_width_ = kWidth;
3579 expected_height_ = kHeight;
3580 *width = kWidth;
3581 *height = kHeight;
3582 }
3583 };
3584
3585 Vp9TestParams params{"L1T1", 1, 1, InterLayerPredMode::kOn};
3586 NonFlexibleModeResolution test(params);
3587
3588 RunBaseTest(&test);
3589 }
3590
3591 #if defined(WEBRTC_ANDROID)
3592 // Crashes on Android; bugs.webrtc.org/7401
3593 #define MAYBE_Vp9FlexModeRefCount DISABLED_Vp9FlexModeRefCount
3594 #else
3595 // TODO(webrtc:9270): Support of flexible mode is temporarily disabled. Enable
3596 // the test after webrtc:9270 is implemented.
3597 #define MAYBE_Vp9FlexModeRefCount DISABLED_Vp9FlexModeRefCount
3598 // #define MAYBE_Vp9FlexModeRefCount Vp9FlexModeRefCount
3599 #endif
TEST_F(VideoSendStreamTest,MAYBE_Vp9FlexModeRefCount)3600 TEST_F(VideoSendStreamTest, MAYBE_Vp9FlexModeRefCount) {
3601 class FlexibleMode : public Vp9HeaderObserver {
3602 public:
3603 explicit FlexibleMode(const Vp9TestParams& params)
3604 : Vp9HeaderObserver(params) {}
3605
3606 private:
3607 void ModifyVideoConfigsHook(
3608 VideoSendStream::Config* send_config,
3609 std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
3610 VideoEncoderConfig* encoder_config) override {
3611 encoder_config->codec_type = kVideoCodecVP9;
3612 encoder_config->content_type = VideoEncoderConfig::ContentType::kScreen;
3613 vp9_settings_.flexibleMode = true;
3614 vp9_settings_.numberOfTemporalLayers = params_.num_temporal_layers;
3615 vp9_settings_.numberOfSpatialLayers = params_.num_spatial_layers;
3616 vp9_settings_.interLayerPred = params_.inter_layer_pred;
3617 }
3618
3619 void InspectHeader(const RTPVideoHeaderVP9& vp9_header) override {
3620 EXPECT_TRUE(vp9_header.flexible_mode);
3621 EXPECT_EQ(kNoTl0PicIdx, vp9_header.tl0_pic_idx);
3622 if (vp9_header.inter_pic_predicted) {
3623 EXPECT_GT(vp9_header.num_ref_pics, 0u);
3624 observation_complete_.Set();
3625 }
3626 }
3627 };
3628
3629 Vp9TestParams params{"L2T1", 2, 1, InterLayerPredMode::kOn};
3630 FlexibleMode test(params);
3631
3632 RunBaseTest(&test);
3633 }
3634 #endif // defined(RTC_ENABLE_VP9)
3635
TestRequestSourceRotateVideo(bool support_orientation_ext)3636 void VideoSendStreamTest::TestRequestSourceRotateVideo(
3637 bool support_orientation_ext) {
3638 CreateSenderCall();
3639
3640 test::NullTransport transport;
3641 CreateSendConfig(1, 0, 0, &transport);
3642 GetVideoSendConfig()->rtp.extensions.clear();
3643 if (support_orientation_ext) {
3644 GetVideoSendConfig()->rtp.extensions.push_back(
3645 RtpExtension(RtpExtension::kVideoRotationUri, 1));
3646 }
3647
3648 CreateVideoStreams();
3649 test::FrameForwarder forwarder;
3650 GetVideoSendStream()->SetSource(&forwarder,
3651 DegradationPreference::MAINTAIN_FRAMERATE);
3652
3653 EXPECT_TRUE(forwarder.sink_wants().rotation_applied !=
3654 support_orientation_ext);
3655
3656 DestroyStreams();
3657 }
3658
TEST_F(VideoSendStreamTest,RequestSourceRotateIfVideoOrientationExtensionNotSupported)3659 TEST_F(VideoSendStreamTest,
3660 RequestSourceRotateIfVideoOrientationExtensionNotSupported) {
3661 TestRequestSourceRotateVideo(false);
3662 }
3663
TEST_F(VideoSendStreamTest,DoNotRequestsRotationIfVideoOrientationExtensionSupported)3664 TEST_F(VideoSendStreamTest,
3665 DoNotRequestsRotationIfVideoOrientationExtensionSupported) {
3666 TestRequestSourceRotateVideo(true);
3667 }
3668
TEST_F(VideoSendStreamTest,EncoderConfigMaxFramerateReportedToSource)3669 TEST_F(VideoSendStreamTest, EncoderConfigMaxFramerateReportedToSource) {
3670 static const int kMaxFps = 22;
3671 class FpsObserver : public test::SendTest,
3672 public test::FrameGeneratorCapturer::SinkWantsObserver {
3673 public:
3674 FpsObserver() : SendTest(kDefaultTimeout) {}
3675
3676 void OnFrameGeneratorCapturerCreated(
3677 test::FrameGeneratorCapturer* frame_generator_capturer) override {
3678 frame_generator_capturer->SetSinkWantsObserver(this);
3679 }
3680
3681 void OnSinkWantsChanged(rtc::VideoSinkInterface<VideoFrame>* sink,
3682 const rtc::VideoSinkWants& wants) override {
3683 if (wants.max_framerate_fps == kMaxFps)
3684 observation_complete_.Set();
3685 }
3686
3687 void ModifyVideoConfigs(
3688 VideoSendStream::Config* send_config,
3689 std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
3690 VideoEncoderConfig* encoder_config) override {
3691 encoder_config->simulcast_layers[0].max_framerate = kMaxFps;
3692 }
3693
3694 void PerformTest() override {
3695 EXPECT_TRUE(Wait()) << "Timed out while waiting for fps to be reported.";
3696 }
3697 } test;
3698
3699 RunBaseTest(&test);
3700 }
3701
3702 // This test verifies that overhead is removed from the bandwidth estimate by
3703 // testing that the maximum possible target payload rate is smaller than the
3704 // maximum bandwidth estimate by the overhead rate.
TEST_F(VideoSendStreamTest,RemoveOverheadFromBandwidth)3705 TEST_F(VideoSendStreamTest, RemoveOverheadFromBandwidth) {
3706 class RemoveOverheadFromBandwidthTest : public test::EndToEndTest,
3707 public test::FakeEncoder {
3708 public:
3709 explicit RemoveOverheadFromBandwidthTest(TaskQueueBase* task_queue)
3710 : EndToEndTest(test::CallTest::kDefaultTimeout),
3711 FakeEncoder(Clock::GetRealTimeClock()),
3712 task_queue_(task_queue),
3713 encoder_factory_(this),
3714 call_(nullptr),
3715 max_bitrate_bps_(0),
3716 first_packet_sent_(false) {}
3717
3718 void SetRates(const RateControlParameters& parameters) override {
3719 MutexLock lock(&mutex_);
3720 // Wait for the first sent packet so that videosendstream knows
3721 // rtp_overhead.
3722 if (first_packet_sent_) {
3723 max_bitrate_bps_ = parameters.bitrate.get_sum_bps();
3724 bitrate_changed_event_.Set();
3725 }
3726 return FakeEncoder::SetRates(parameters);
3727 }
3728
3729 void OnCallsCreated(Call* sender_call, Call* receiver_call) override {
3730 call_ = sender_call;
3731 }
3732
3733 void ModifyVideoConfigs(
3734 VideoSendStream::Config* send_config,
3735 std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
3736 VideoEncoderConfig* encoder_config) override {
3737 send_config->rtp.max_packet_size = 1200;
3738 send_config->encoder_settings.encoder_factory = &encoder_factory_;
3739 EXPECT_FALSE(send_config->rtp.extensions.empty());
3740 }
3741
3742 Action OnSendRtp(const uint8_t* packet, size_t length) override {
3743 MutexLock lock(&mutex_);
3744 first_packet_sent_ = true;
3745 return SEND_PACKET;
3746 }
3747
3748 void PerformTest() override {
3749 BitrateConstraints bitrate_config;
3750 constexpr int kStartBitrateBps = 60000;
3751 constexpr int kMaxBitrateBps = 60000;
3752 constexpr int kMinBitrateBps = 10000;
3753 bitrate_config.start_bitrate_bps = kStartBitrateBps;
3754 bitrate_config.max_bitrate_bps = kMaxBitrateBps;
3755 bitrate_config.min_bitrate_bps = kMinBitrateBps;
3756 SendTask(task_queue_, [this, &bitrate_config]() {
3757 call_->GetTransportControllerSend()->SetSdpBitrateParameters(
3758 bitrate_config);
3759 call_->GetTransportControllerSend()->OnTransportOverheadChanged(40);
3760 });
3761
3762 // At a bitrate of 60kbps with a packet size of 1200B video and an
3763 // overhead of 40B per packet video produces 2240bps overhead.
3764 // So the encoder BW should be set to 57760bps.
3765 EXPECT_TRUE(
3766 bitrate_changed_event_.Wait(VideoSendStreamTest::kDefaultTimeout));
3767 {
3768 MutexLock lock(&mutex_);
3769 EXPECT_LE(max_bitrate_bps_, 57760u);
3770 }
3771 }
3772
3773 private:
3774 TaskQueueBase* const task_queue_;
3775 test::VideoEncoderProxyFactory encoder_factory_;
3776 Call* call_;
3777 Mutex mutex_;
3778 uint32_t max_bitrate_bps_ RTC_GUARDED_BY(&mutex_);
3779 bool first_packet_sent_ RTC_GUARDED_BY(&mutex_);
3780 rtc::Event bitrate_changed_event_;
3781 } test(task_queue());
3782 RunBaseTest(&test);
3783 }
3784
3785 class PacingFactorObserver : public test::SendTest {
3786 public:
PacingFactorObserver(bool configure_send_side,absl::optional<float> expected_pacing_factor)3787 PacingFactorObserver(bool configure_send_side,
3788 absl::optional<float> expected_pacing_factor)
3789 : test::SendTest(VideoSendStreamTest::kDefaultTimeout),
3790 configure_send_side_(configure_send_side),
3791 expected_pacing_factor_(expected_pacing_factor) {}
3792
ModifyVideoConfigs(VideoSendStream::Config * send_config,std::vector<VideoReceiveStreamInterface::Config> * receive_configs,VideoEncoderConfig * encoder_config)3793 void ModifyVideoConfigs(
3794 VideoSendStream::Config* send_config,
3795 std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
3796 VideoEncoderConfig* encoder_config) override {
3797 // Check if send-side bwe extension is already present, and remove it if
3798 // it is not desired.
3799 bool has_send_side = false;
3800 for (auto it = send_config->rtp.extensions.begin();
3801 it != send_config->rtp.extensions.end(); ++it) {
3802 if (it->uri == RtpExtension::kTransportSequenceNumberUri) {
3803 if (configure_send_side_) {
3804 has_send_side = true;
3805 } else {
3806 send_config->rtp.extensions.erase(it);
3807 }
3808 break;
3809 }
3810 }
3811
3812 if (configure_send_side_ && !has_send_side) {
3813 rtc::UniqueNumberGenerator<int> unique_id_generator;
3814 unique_id_generator.AddKnownId(0); // First valid RTP extension ID is 1.
3815 for (const RtpExtension& extension : send_config->rtp.extensions) {
3816 unique_id_generator.AddKnownId(extension.id);
3817 }
3818 // Want send side, not present by default, so add it.
3819 send_config->rtp.extensions.emplace_back(
3820 RtpExtension::kTransportSequenceNumberUri, unique_id_generator());
3821 }
3822
3823 // ALR only enabled for screenshare.
3824 encoder_config->content_type = VideoEncoderConfig::ContentType::kScreen;
3825 }
3826
OnVideoStreamsCreated(VideoSendStream * send_stream,const std::vector<VideoReceiveStreamInterface * > & receive_streams)3827 void OnVideoStreamsCreated(VideoSendStream* send_stream,
3828 const std::vector<VideoReceiveStreamInterface*>&
3829 receive_streams) override {
3830 auto internal_send_peer = test::VideoSendStreamPeer(send_stream);
3831 // Video streams created, check that pacing factor is correctly configured.
3832 EXPECT_EQ(expected_pacing_factor_,
3833 internal_send_peer.GetPacingFactorOverride());
3834 observation_complete_.Set();
3835 }
3836
PerformTest()3837 void PerformTest() override {
3838 EXPECT_TRUE(Wait()) << "Timed out while waiting for stream creation.";
3839 }
3840
3841 private:
3842 const bool configure_send_side_;
3843 const absl::optional<float> expected_pacing_factor_;
3844 };
3845
GetAlrProbingExperimentString()3846 std::string GetAlrProbingExperimentString() {
3847 return std::string(
3848 AlrExperimentSettings::kScreenshareProbingBweExperimentName) +
3849 "/1.0,2875,80,40,-60,3/";
3850 }
3851 const float kAlrProbingExperimentPaceMultiplier = 1.0f;
3852
TEST_F(VideoSendStreamTest,AlrConfiguredWhenSendSideOn)3853 TEST_F(VideoSendStreamTest, AlrConfiguredWhenSendSideOn) {
3854 test::ScopedFieldTrials alr_experiment(GetAlrProbingExperimentString());
3855 // Send-side bwe on, use pacing factor from `kAlrProbingExperiment` above.
3856 PacingFactorObserver test_with_send_side(true,
3857 kAlrProbingExperimentPaceMultiplier);
3858 RunBaseTest(&test_with_send_side);
3859 }
3860
TEST_F(VideoSendStreamTest,AlrNotConfiguredWhenSendSideOff)3861 TEST_F(VideoSendStreamTest, AlrNotConfiguredWhenSendSideOff) {
3862 test::ScopedFieldTrials alr_experiment(GetAlrProbingExperimentString());
3863 // Send-side bwe off, use configuration should not be overridden.
3864 PacingFactorObserver test_without_send_side(false, absl::nullopt);
3865 RunBaseTest(&test_without_send_side);
3866 }
3867
3868 // Test class takes as argument a function pointer to reset the send
3869 // stream and call OnVideoStreamsCreated. This is necessary since you cannot
3870 // change the content type of a VideoSendStream, you need to recreate it.
3871 // Stopping and recreating the stream can only be done on the main thread and in
3872 // the context of VideoSendStreamTest (not BaseTest). The test switches from
3873 // realtime to screenshare and back.
3874 template <typename T>
3875 class ContentSwitchTest : public test::SendTest {
3876 public:
3877 enum class StreamState {
3878 kBeforeSwitch = 0,
3879 kInScreenshare = 1,
3880 kAfterSwitchBack = 2,
3881 };
3882 static const uint32_t kMinPacketsToSend = 50;
3883
ContentSwitchTest(T * stream_reset_fun,TaskQueueBase * task_queue)3884 explicit ContentSwitchTest(T* stream_reset_fun, TaskQueueBase* task_queue)
3885 : SendTest(test::CallTest::kDefaultTimeout),
3886 call_(nullptr),
3887 state_(StreamState::kBeforeSwitch),
3888 send_stream_(nullptr),
3889 send_stream_config_(nullptr),
3890 packets_sent_(0),
3891 stream_resetter_(stream_reset_fun),
3892 task_queue_(task_queue) {
3893 RTC_DCHECK(stream_resetter_);
3894 }
3895
OnVideoStreamsCreated(VideoSendStream * send_stream,const std::vector<VideoReceiveStreamInterface * > & receive_streams)3896 void OnVideoStreamsCreated(VideoSendStream* send_stream,
3897 const std::vector<VideoReceiveStreamInterface*>&
3898 receive_streams) override {
3899 MutexLock lock(&mutex_);
3900 send_stream_ = send_stream;
3901 }
3902
ModifyVideoConfigs(VideoSendStream::Config * send_config,std::vector<VideoReceiveStreamInterface::Config> * receive_configs,VideoEncoderConfig * encoder_config)3903 void ModifyVideoConfigs(
3904 VideoSendStream::Config* send_config,
3905 std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
3906 VideoEncoderConfig* encoder_config) override {
3907 RTC_DCHECK_EQ(1, encoder_config->number_of_streams);
3908 encoder_config->min_transmit_bitrate_bps = 0;
3909 encoder_config->content_type =
3910 VideoEncoderConfig::ContentType::kRealtimeVideo;
3911 send_stream_config_ = send_config->Copy();
3912 encoder_config_ = encoder_config->Copy();
3913 }
3914
OnCallsCreated(Call * sender_call,Call * receiver_call)3915 void OnCallsCreated(Call* sender_call, Call* receiver_call) override {
3916 call_ = sender_call;
3917 }
3918
OnStreamsStopped()3919 void OnStreamsStopped() override {
3920 MutexLock lock(&mutex_);
3921 done_ = true;
3922 }
3923
OnSendRtp(const uint8_t * packet,size_t length)3924 Action OnSendRtp(const uint8_t* packet, size_t length) override {
3925 task_queue_->PostTask([this]() {
3926 MutexLock lock(&mutex_);
3927 if (done_)
3928 return;
3929
3930 auto internal_send_peer = test::VideoSendStreamPeer(send_stream_);
3931 float pacing_factor =
3932 internal_send_peer.GetPacingFactorOverride().value_or(0.0f);
3933 float expected_pacing_factor = 1.1; // Strict pacing factor.
3934 VideoSendStream::Stats stats = send_stream_->GetStats();
3935 if (stats.content_type == webrtc::VideoContentType::SCREENSHARE) {
3936 expected_pacing_factor = 1.0f; // Currently used pacing factor in ALR.
3937 }
3938
3939 EXPECT_NEAR(expected_pacing_factor, pacing_factor, 1e-6);
3940
3941 // Wait until at least kMinPacketsToSend packets to be sent, so that
3942 // some frames would be encoded.
3943 if (++packets_sent_ < kMinPacketsToSend)
3944 return;
3945
3946 if (state_ != StreamState::kAfterSwitchBack) {
3947 // We've sent kMinPacketsToSend packets, switch the content type and
3948 // move move to the next state. Note that we need to recreate the stream
3949 // if changing content type.
3950 packets_sent_ = 0;
3951 if (encoder_config_.content_type ==
3952 VideoEncoderConfig::ContentType::kRealtimeVideo) {
3953 encoder_config_.content_type =
3954 VideoEncoderConfig::ContentType::kScreen;
3955 } else {
3956 encoder_config_.content_type =
3957 VideoEncoderConfig::ContentType::kRealtimeVideo;
3958 }
3959 switch (state_) {
3960 case StreamState::kBeforeSwitch:
3961 state_ = StreamState::kInScreenshare;
3962 break;
3963 case StreamState::kInScreenshare:
3964 state_ = StreamState::kAfterSwitchBack;
3965 break;
3966 case StreamState::kAfterSwitchBack:
3967 RTC_DCHECK_NOTREACHED();
3968 break;
3969 }
3970 content_switch_event_.Set();
3971 return;
3972 }
3973 observation_complete_.Set();
3974 });
3975
3976 return SEND_PACKET;
3977 }
3978
PerformTest()3979 void PerformTest() override {
3980 while (GetStreamState() != StreamState::kAfterSwitchBack) {
3981 ASSERT_TRUE(content_switch_event_.Wait(test::CallTest::kDefaultTimeout));
3982 (*stream_resetter_)(send_stream_config_, encoder_config_, this);
3983 }
3984
3985 ASSERT_TRUE(Wait())
3986 << "Timed out waiting for a frame sent after switch back";
3987 }
3988
3989 private:
GetStreamState()3990 StreamState GetStreamState() {
3991 MutexLock lock(&mutex_);
3992 return state_;
3993 }
3994
3995 Mutex mutex_;
3996 rtc::Event content_switch_event_;
3997 Call* call_;
3998 bool done_ RTC_GUARDED_BY(mutex_) = false;
3999 StreamState state_ RTC_GUARDED_BY(mutex_);
4000 VideoSendStream* send_stream_ RTC_GUARDED_BY(mutex_);
4001 VideoSendStream::Config send_stream_config_;
4002 VideoEncoderConfig encoder_config_;
4003 uint32_t packets_sent_ RTC_GUARDED_BY(mutex_);
4004 T* stream_resetter_;
4005 TaskQueueBase* task_queue_;
4006 };
4007
TEST_F(VideoSendStreamTest,SwitchesToScreenshareAndBack)4008 TEST_F(VideoSendStreamTest, SwitchesToScreenshareAndBack) {
4009 auto reset_fun = [this](const VideoSendStream::Config& send_stream_config,
4010 const VideoEncoderConfig& encoder_config,
4011 test::BaseTest* test) {
4012 SendTask(task_queue(),
4013 [this, &send_stream_config, &encoder_config, &test]() {
4014 Stop();
4015 DestroyVideoSendStreams();
4016 SetVideoSendConfig(send_stream_config);
4017 SetVideoEncoderConfig(encoder_config);
4018 CreateVideoSendStreams();
4019 SetVideoDegradation(DegradationPreference::MAINTAIN_RESOLUTION);
4020 test->OnVideoStreamsCreated(GetVideoSendStream(),
4021 video_receive_streams_);
4022 Start();
4023 });
4024 };
4025 ContentSwitchTest<decltype(reset_fun)> test(&reset_fun, task_queue());
4026 RunBaseTest(&test);
4027 }
4028
TestTemporalLayers(VideoEncoderFactory * encoder_factory,const std::string & payload_name,const std::vector<int> & num_temporal_layers,const std::vector<ScalabilityMode> & scalability_mode)4029 void VideoSendStreamTest::TestTemporalLayers(
4030 VideoEncoderFactory* encoder_factory,
4031 const std::string& payload_name,
4032 const std::vector<int>& num_temporal_layers,
4033 const std::vector<ScalabilityMode>& scalability_mode) {
4034 static constexpr int kMaxBitrateBps = 1000000;
4035 static constexpr int kMinFramesToObservePerStream = 8;
4036
4037 class TemporalLayerObserver
4038 : public test::EndToEndTest,
4039 public test::FrameGeneratorCapturer::SinkWantsObserver {
4040 public:
4041 TemporalLayerObserver(VideoEncoderFactory* encoder_factory,
4042 const std::string& payload_name,
4043 const std::vector<int>& num_temporal_layers,
4044 const std::vector<ScalabilityMode>& scalability_mode)
4045 : EndToEndTest(kDefaultTimeout),
4046 encoder_factory_(encoder_factory),
4047 payload_name_(payload_name),
4048 num_temporal_layers_(num_temporal_layers),
4049 scalability_mode_(scalability_mode),
4050 depacketizer_(CreateVideoRtpDepacketizer(
4051 PayloadStringToCodecType(payload_name))) {}
4052
4053 private:
4054 void OnFrameGeneratorCapturerCreated(
4055 test::FrameGeneratorCapturer* frame_generator_capturer) override {
4056 frame_generator_capturer->ChangeResolution(640, 360);
4057 }
4058
4059 void OnSinkWantsChanged(rtc::VideoSinkInterface<VideoFrame>* sink,
4060 const rtc::VideoSinkWants& wants) override {}
4061
4062 void ModifySenderBitrateConfig(
4063 BitrateConstraints* bitrate_config) override {
4064 bitrate_config->start_bitrate_bps = kMaxBitrateBps / 2;
4065 }
4066
4067 size_t GetNumVideoStreams() const override {
4068 if (scalability_mode_.empty()) {
4069 return num_temporal_layers_.size();
4070 } else {
4071 return scalability_mode_.size();
4072 }
4073 }
4074
4075 void ModifyVideoConfigs(
4076 VideoSendStream::Config* send_config,
4077 std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
4078 VideoEncoderConfig* encoder_config) override {
4079 webrtc::VideoEncoder::EncoderInfo encoder_info;
4080 send_config->encoder_settings.encoder_factory = encoder_factory_;
4081 send_config->rtp.payload_name = payload_name_;
4082 send_config->rtp.payload_type = test::CallTest::kVideoSendPayloadType;
4083 encoder_config->video_format.name = payload_name_;
4084 encoder_config->codec_type = PayloadStringToCodecType(payload_name_);
4085 encoder_config->video_stream_factory =
4086 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
4087 payload_name_, /*max_qp=*/56, /*is_screenshare=*/false,
4088 /*conference_mode=*/false, encoder_info);
4089 encoder_config->max_bitrate_bps = kMaxBitrateBps;
4090 if (absl::EqualsIgnoreCase(payload_name_, "VP9")) {
4091 encoder_config->encoder_specific_settings = rtc::make_ref_counted<
4092 VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4093 VideoEncoder::GetDefaultVp9Settings());
4094 }
4095 if (scalability_mode_.empty()) {
4096 for (size_t i = 0; i < num_temporal_layers_.size(); ++i) {
4097 VideoStream& stream = encoder_config->simulcast_layers[i];
4098 stream.num_temporal_layers = num_temporal_layers_[i];
4099 configured_num_temporal_layers_[send_config->rtp.ssrcs[i]] =
4100 num_temporal_layers_[i];
4101 }
4102 } else {
4103 for (size_t i = 0; i < scalability_mode_.size(); ++i) {
4104 VideoStream& stream = encoder_config->simulcast_layers[i];
4105 stream.scalability_mode = scalability_mode_[i];
4106
4107 configured_num_temporal_layers_[send_config->rtp.ssrcs[i]] =
4108 ScalabilityModeToNumTemporalLayers(scalability_mode_[i]);
4109 }
4110 }
4111 }
4112
4113 struct ParsedPacket {
4114 uint32_t timestamp;
4115 uint32_t ssrc;
4116 int temporal_idx;
4117 };
4118
4119 bool ParsePayload(const uint8_t* packet,
4120 size_t length,
4121 ParsedPacket& parsed) const {
4122 RtpPacket rtp_packet;
4123 EXPECT_TRUE(rtp_packet.Parse(packet, length));
4124
4125 if (rtp_packet.payload_size() == 0) {
4126 return false; // Padding packet.
4127 }
4128 parsed.timestamp = rtp_packet.Timestamp();
4129 parsed.ssrc = rtp_packet.Ssrc();
4130
4131 absl::optional<VideoRtpDepacketizer::ParsedRtpPayload> parsed_payload =
4132 depacketizer_->Parse(rtp_packet.PayloadBuffer());
4133 EXPECT_TRUE(parsed_payload);
4134
4135 if (const auto* vp8_header = absl::get_if<RTPVideoHeaderVP8>(
4136 &parsed_payload->video_header.video_type_header)) {
4137 parsed.temporal_idx = vp8_header->temporalIdx;
4138 } else if (const auto* vp9_header = absl::get_if<RTPVideoHeaderVP9>(
4139 &parsed_payload->video_header.video_type_header)) {
4140 parsed.temporal_idx = vp9_header->temporal_idx;
4141 } else {
4142 RTC_DCHECK_NOTREACHED();
4143 }
4144 return true;
4145 }
4146
4147 Action OnSendRtp(const uint8_t* packet, size_t length) override {
4148 ParsedPacket parsed;
4149 if (!ParsePayload(packet, length, parsed))
4150 return SEND_PACKET;
4151
4152 uint32_t ssrc = parsed.ssrc;
4153 int temporal_idx =
4154 parsed.temporal_idx == kNoTemporalIdx ? 0 : parsed.temporal_idx;
4155 max_observed_tl_idxs_[ssrc] =
4156 std::max(temporal_idx, max_observed_tl_idxs_[ssrc]);
4157
4158 if (last_observed_packet_.count(ssrc) == 0 ||
4159 parsed.timestamp != last_observed_packet_[ssrc].timestamp) {
4160 num_observed_frames_[ssrc]++;
4161 }
4162 last_observed_packet_[ssrc] = parsed;
4163
4164 if (HighestTemporalLayerSentPerStream())
4165 observation_complete_.Set();
4166
4167 return SEND_PACKET;
4168 }
4169
4170 bool HighestTemporalLayerSentPerStream() const {
4171 if (num_observed_frames_.size() !=
4172 configured_num_temporal_layers_.size()) {
4173 return false;
4174 }
4175 for (const auto& num_frames : num_observed_frames_) {
4176 if (num_frames.second < kMinFramesToObservePerStream) {
4177 return false;
4178 }
4179 }
4180 if (max_observed_tl_idxs_.size() !=
4181 configured_num_temporal_layers_.size()) {
4182 return false;
4183 }
4184 for (const auto& max_tl_idx : max_observed_tl_idxs_) {
4185 uint32_t ssrc = max_tl_idx.first;
4186 int configured_num_tls =
4187 configured_num_temporal_layers_.find(ssrc)->second;
4188 if (max_tl_idx.second != configured_num_tls - 1)
4189 return false;
4190 }
4191 return true;
4192 }
4193
4194 void PerformTest() override { EXPECT_TRUE(Wait()); }
4195
4196 VideoEncoderFactory* const encoder_factory_;
4197 const std::string payload_name_;
4198 const std::vector<int> num_temporal_layers_;
4199 const std::vector<ScalabilityMode> scalability_mode_;
4200 const std::unique_ptr<VideoRtpDepacketizer> depacketizer_;
4201 // Mapped by SSRC.
4202 std::map<uint32_t, int> configured_num_temporal_layers_;
4203 std::map<uint32_t, int> max_observed_tl_idxs_;
4204 std::map<uint32_t, int> num_observed_frames_;
4205 std::map<uint32_t, ParsedPacket> last_observed_packet_;
4206 } test(encoder_factory, payload_name, num_temporal_layers, scalability_mode);
4207
4208 RunBaseTest(&test);
4209 }
4210
TEST_F(VideoSendStreamTest,TestTemporalLayersVp8)4211 TEST_F(VideoSendStreamTest, TestTemporalLayersVp8) {
4212 InternalEncoderFactory internal_encoder_factory;
4213 test::FunctionVideoEncoderFactory encoder_factory(
4214 [&internal_encoder_factory]() {
4215 return std::make_unique<SimulcastEncoderAdapter>(
4216 &internal_encoder_factory, SdpVideoFormat("VP8"));
4217 });
4218
4219 TestTemporalLayers(&encoder_factory, "VP8",
4220 /*num_temporal_layers=*/{2},
4221 /*scalability_mode=*/{});
4222 }
4223
TEST_F(VideoSendStreamTest,TestTemporalLayersVp8Simulcast)4224 TEST_F(VideoSendStreamTest, TestTemporalLayersVp8Simulcast) {
4225 InternalEncoderFactory internal_encoder_factory;
4226 test::FunctionVideoEncoderFactory encoder_factory(
4227 [&internal_encoder_factory]() {
4228 return std::make_unique<SimulcastEncoderAdapter>(
4229 &internal_encoder_factory, SdpVideoFormat("VP8"));
4230 });
4231
4232 TestTemporalLayers(&encoder_factory, "VP8",
4233 /*num_temporal_layers=*/{2, 2},
4234 /*scalability_mode=*/{});
4235 }
4236
TEST_F(VideoSendStreamTest,TestTemporalLayersVp8SimulcastWithDifferentNumTls)4237 TEST_F(VideoSendStreamTest, TestTemporalLayersVp8SimulcastWithDifferentNumTls) {
4238 InternalEncoderFactory internal_encoder_factory;
4239 test::FunctionVideoEncoderFactory encoder_factory(
4240 [&internal_encoder_factory]() {
4241 return std::make_unique<SimulcastEncoderAdapter>(
4242 &internal_encoder_factory, SdpVideoFormat("VP8"));
4243 });
4244
4245 TestTemporalLayers(&encoder_factory, "VP8",
4246 /*num_temporal_layers=*/{3, 1},
4247 /*scalability_mode=*/{});
4248 }
4249
TEST_F(VideoSendStreamTest,TestTemporalLayersVp8SimulcastWithoutSimAdapter)4250 TEST_F(VideoSendStreamTest, TestTemporalLayersVp8SimulcastWithoutSimAdapter) {
4251 test::FunctionVideoEncoderFactory encoder_factory(
4252 []() { return VP8Encoder::Create(); });
4253
4254 TestTemporalLayers(&encoder_factory, "VP8",
4255 /*num_temporal_layers=*/{2, 2},
4256 /*scalability_mode=*/{});
4257 }
4258
TEST_F(VideoSendStreamTest,TestScalabilityModeVp8L1T2)4259 TEST_F(VideoSendStreamTest, TestScalabilityModeVp8L1T2) {
4260 InternalEncoderFactory internal_encoder_factory;
4261 test::FunctionVideoEncoderFactory encoder_factory(
4262 [&internal_encoder_factory]() {
4263 return std::make_unique<SimulcastEncoderAdapter>(
4264 &internal_encoder_factory, SdpVideoFormat("VP8"));
4265 });
4266
4267 TestTemporalLayers(&encoder_factory, "VP8",
4268 /*num_temporal_layers=*/{}, {ScalabilityMode::kL1T2});
4269 }
4270
TEST_F(VideoSendStreamTest,TestScalabilityModeVp8Simulcast)4271 TEST_F(VideoSendStreamTest, TestScalabilityModeVp8Simulcast) {
4272 InternalEncoderFactory internal_encoder_factory;
4273 test::FunctionVideoEncoderFactory encoder_factory(
4274 [&internal_encoder_factory]() {
4275 return std::make_unique<SimulcastEncoderAdapter>(
4276 &internal_encoder_factory, SdpVideoFormat("VP8"));
4277 });
4278
4279 TestTemporalLayers(&encoder_factory, "VP8",
4280 /*num_temporal_layers=*/{},
4281 {ScalabilityMode::kL1T2, ScalabilityMode::kL1T2});
4282 }
4283
TEST_F(VideoSendStreamTest,TestScalabilityModeVp8SimulcastWithDifferentMode)4284 TEST_F(VideoSendStreamTest, TestScalabilityModeVp8SimulcastWithDifferentMode) {
4285 InternalEncoderFactory internal_encoder_factory;
4286 test::FunctionVideoEncoderFactory encoder_factory(
4287 [&internal_encoder_factory]() {
4288 return std::make_unique<SimulcastEncoderAdapter>(
4289 &internal_encoder_factory, SdpVideoFormat("VP8"));
4290 });
4291
4292 TestTemporalLayers(&encoder_factory, "VP8",
4293 /*num_temporal_layers=*/{},
4294 {ScalabilityMode::kL1T3, ScalabilityMode::kL1T1});
4295 }
4296
TEST_F(VideoSendStreamTest,TestScalabilityModeVp8SimulcastWithoutSimAdapter)4297 TEST_F(VideoSendStreamTest, TestScalabilityModeVp8SimulcastWithoutSimAdapter) {
4298 test::FunctionVideoEncoderFactory encoder_factory(
4299 []() { return VP8Encoder::Create(); });
4300
4301 TestTemporalLayers(&encoder_factory, "VP8",
4302 /*num_temporal_layers=*/{},
4303 {ScalabilityMode::kL1T2, ScalabilityMode::kL1T2});
4304 }
4305
TEST_F(VideoSendStreamTest,TestTemporalLayersVp9)4306 TEST_F(VideoSendStreamTest, TestTemporalLayersVp9) {
4307 test::FunctionVideoEncoderFactory encoder_factory(
4308 []() { return VP9Encoder::Create(); });
4309
4310 TestTemporalLayers(&encoder_factory, "VP9",
4311 /*num_temporal_layers=*/{2},
4312 /*scalability_mode=*/{});
4313 }
4314
4315 } // namespace webrtc
4316