xref: /aosp_15_r20/external/webrtc/video/video_send_stream_tests.cc (revision d9f758449e529ab9291ac668be2861e7a55c2422)
1 /*
2  *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 #include <algorithm>  // max
11 #include <memory>
12 #include <vector>
13 
14 #include "absl/algorithm/container.h"
15 #include "absl/strings/match.h"
16 #include "api/sequence_checker.h"
17 #include "api/task_queue/default_task_queue_factory.h"
18 #include "api/task_queue/task_queue_base.h"
19 #include "api/test/metrics/global_metrics_logger_and_exporter.h"
20 #include "api/test/metrics/metric.h"
21 #include "api/test/simulated_network.h"
22 #include "api/video/builtin_video_bitrate_allocator_factory.h"
23 #include "api/video/encoded_image.h"
24 #include "api/video/video_bitrate_allocation.h"
25 #include "api/video_codecs/video_encoder.h"
26 #include "call/call.h"
27 #include "call/fake_network_pipe.h"
28 #include "call/rtp_transport_controller_send.h"
29 #include "call/simulated_network.h"
30 #include "call/video_send_stream.h"
31 #include "media/engine/internal_encoder_factory.h"
32 #include "media/engine/simulcast_encoder_adapter.h"
33 #include "media/engine/webrtc_video_engine.h"
34 #include "modules/rtp_rtcp/include/rtp_header_extension_map.h"
35 #include "modules/rtp_rtcp/source/create_video_rtp_depacketizer.h"
36 #include "modules/rtp_rtcp/source/rtcp_sender.h"
37 #include "modules/rtp_rtcp/source/rtp_header_extensions.h"
38 #include "modules/rtp_rtcp/source/rtp_packet.h"
39 #include "modules/rtp_rtcp/source/rtp_rtcp_impl2.h"
40 #include "modules/rtp_rtcp/source/rtp_util.h"
41 #include "modules/rtp_rtcp/source/video_rtp_depacketizer_vp9.h"
42 #include "modules/video_coding/codecs/interface/common_constants.h"
43 #include "modules/video_coding/codecs/vp8/include/vp8.h"
44 #include "modules/video_coding/codecs/vp9/include/vp9.h"
45 #include "modules/video_coding/svc/create_scalability_structure.h"
46 #include "modules/video_coding/svc/scalability_mode_util.h"
47 #include "modules/video_coding/svc/scalable_video_controller.h"
48 #include "rtc_base/checks.h"
49 #include "rtc_base/event.h"
50 #include "rtc_base/experiments/alr_experiment.h"
51 #include "rtc_base/logging.h"
52 #include "rtc_base/platform_thread.h"
53 #include "rtc_base/rate_limiter.h"
54 #include "rtc_base/strings/string_builder.h"
55 #include "rtc_base/synchronization/mutex.h"
56 #include "rtc_base/task_queue_for_test.h"
57 #include "rtc_base/time_utils.h"
58 #include "rtc_base/unique_id_generator.h"
59 #include "system_wrappers/include/sleep.h"
60 #include "test/call_test.h"
61 #include "test/configurable_frame_size_encoder.h"
62 #include "test/fake_encoder.h"
63 #include "test/fake_texture_frame.h"
64 #include "test/frame_forwarder.h"
65 #include "test/frame_generator_capturer.h"
66 #include "test/frame_utils.h"
67 #include "test/gmock.h"
68 #include "test/gtest.h"
69 #include "test/null_transport.h"
70 #include "test/rtcp_packet_parser.h"
71 #include "test/video_encoder_proxy_factory.h"
72 #include "video/config/encoder_stream_factory.h"
73 #include "video/send_statistics_proxy.h"
74 #include "video/transport_adapter.h"
75 #include "video/video_send_stream.h"
76 
77 namespace webrtc {
78 namespace test {
79 class VideoSendStreamPeer {
80  public:
VideoSendStreamPeer(webrtc::VideoSendStream * base_class_stream)81   explicit VideoSendStreamPeer(webrtc::VideoSendStream* base_class_stream)
82       : internal_stream_(
83             static_cast<internal::VideoSendStream*>(base_class_stream)) {}
GetPacingFactorOverride() const84   absl::optional<float> GetPacingFactorOverride() const {
85     return internal_stream_->GetPacingFactorOverride();
86   }
87 
88  private:
89   internal::VideoSendStream const* const internal_stream_;
90 };
91 }  // namespace test
92 
93 namespace {
94 enum : int {  // The first valid value is 1.
95   kAbsSendTimeExtensionId = 1,
96   kTimestampOffsetExtensionId,
97   kTransportSequenceNumberExtensionId,
98   kVideoContentTypeExtensionId,
99   kVideoRotationExtensionId,
100   kVideoTimingExtensionId,
101 };
102 
103 // Readability convenience enum for `WaitBitrateChanged()`.
104 enum class WaitUntil : bool { kZero = false, kNonZero = true };
105 
106 constexpr int64_t kRtcpIntervalMs = 1000;
107 
108 enum VideoFormat {
109   kGeneric,
110   kVP8,
111 };
112 
113 struct Vp9TestParams {
114   std::string scalability_mode;
115   uint8_t num_spatial_layers;
116   uint8_t num_temporal_layers;
117   InterLayerPredMode inter_layer_pred;
118 };
119 
120 using ParameterizationType = std::tuple<Vp9TestParams, bool>;
121 
ParamInfoToStr(const testing::TestParamInfo<ParameterizationType> & info)122 std::string ParamInfoToStr(
123     const testing::TestParamInfo<ParameterizationType>& info) {
124   rtc::StringBuilder sb;
125   sb << std::get<0>(info.param).scalability_mode << "_"
126      << (std::get<1>(info.param) ? "WithIdentifier" : "WithoutIdentifier");
127   return sb.str();
128 }
129 
130 }  // namespace
131 
132 class VideoSendStreamTest : public test::CallTest {
133  public:
VideoSendStreamTest()134   VideoSendStreamTest() {
135     RegisterRtpExtension(RtpExtension(RtpExtension::kTransportSequenceNumberUri,
136                                       kTransportSequenceNumberExtensionId));
137   }
138 
139  protected:
140   void TestNackRetransmission(uint32_t retransmit_ssrc,
141                               uint8_t retransmit_payload_type);
142   void TestPacketFragmentationSize(VideoFormat format, bool with_fec);
143 
144   void TestVp9NonFlexMode(const Vp9TestParams& params,
145                           bool use_scalability_mode_identifier);
146 
147   void TestRequestSourceRotateVideo(bool support_orientation_ext);
148 
149   void TestTemporalLayers(VideoEncoderFactory* encoder_factory,
150                           const std::string& payload_name,
151                           const std::vector<int>& num_temporal_layers,
152                           const std::vector<ScalabilityMode>& scalability_mode);
153 };
154 
TEST_F(VideoSendStreamTest,CanStartStartedStream)155 TEST_F(VideoSendStreamTest, CanStartStartedStream) {
156   SendTask(task_queue(), [this]() {
157     CreateSenderCall();
158 
159     test::NullTransport transport;
160     CreateSendConfig(1, 0, 0, &transport);
161     CreateVideoStreams();
162     GetVideoSendStream()->Start();
163     GetVideoSendStream()->Start();
164     DestroyStreams();
165     DestroyCalls();
166   });
167 }
168 
TEST_F(VideoSendStreamTest,CanStopStoppedStream)169 TEST_F(VideoSendStreamTest, CanStopStoppedStream) {
170   SendTask(task_queue(), [this]() {
171     CreateSenderCall();
172 
173     test::NullTransport transport;
174     CreateSendConfig(1, 0, 0, &transport);
175     CreateVideoStreams();
176     GetVideoSendStream()->Stop();
177     GetVideoSendStream()->Stop();
178     DestroyStreams();
179     DestroyCalls();
180   });
181 }
182 
TEST_F(VideoSendStreamTest,SupportsCName)183 TEST_F(VideoSendStreamTest, SupportsCName) {
184   static std::string kCName = "PjQatC14dGfbVwGPUOA9IH7RlsFDbWl4AhXEiDsBizo=";
185   class CNameObserver : public test::SendTest {
186    public:
187     CNameObserver() : SendTest(kDefaultTimeout) {}
188 
189    private:
190     Action OnSendRtcp(const uint8_t* packet, size_t length) override {
191       test::RtcpPacketParser parser;
192       EXPECT_TRUE(parser.Parse(packet, length));
193       if (parser.sdes()->num_packets() > 0) {
194         EXPECT_EQ(1u, parser.sdes()->chunks().size());
195         EXPECT_EQ(kCName, parser.sdes()->chunks()[0].cname);
196 
197         observation_complete_.Set();
198       }
199 
200       return SEND_PACKET;
201     }
202 
203     void ModifyVideoConfigs(
204         VideoSendStream::Config* send_config,
205         std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
206         VideoEncoderConfig* encoder_config) override {
207       send_config->rtp.c_name = kCName;
208     }
209 
210     void PerformTest() override {
211       EXPECT_TRUE(Wait()) << "Timed out while waiting for RTCP with CNAME.";
212     }
213   } test;
214 
215   RunBaseTest(&test);
216 }
217 
TEST_F(VideoSendStreamTest,SupportsAbsoluteSendTime)218 TEST_F(VideoSendStreamTest, SupportsAbsoluteSendTime) {
219   class AbsoluteSendTimeObserver : public test::SendTest {
220    public:
221     AbsoluteSendTimeObserver() : SendTest(kDefaultTimeout) {
222       extensions_.Register<AbsoluteSendTime>(kAbsSendTimeExtensionId);
223     }
224 
225     Action OnSendRtp(const uint8_t* packet, size_t length) override {
226       RtpPacket rtp_packet(&extensions_);
227       EXPECT_TRUE(rtp_packet.Parse(packet, length));
228 
229       uint32_t abs_send_time = 0;
230       EXPECT_FALSE(rtp_packet.HasExtension<TransmissionOffset>());
231       EXPECT_TRUE(rtp_packet.GetExtension<AbsoluteSendTime>(&abs_send_time));
232       if (abs_send_time != 0) {
233         // Wait for at least one packet with a non-zero send time. The send time
234         // is a 16-bit value derived from the system clock, and it is valid
235         // for a packet to have a zero send time. To tell that from an
236         // unpopulated value we'll wait for a packet with non-zero send time.
237         observation_complete_.Set();
238       } else {
239         RTC_LOG(LS_WARNING)
240             << "Got a packet with zero absoluteSendTime, waiting"
241                " for another packet...";
242       }
243 
244       return SEND_PACKET;
245     }
246 
247     void ModifyVideoConfigs(
248         VideoSendStream::Config* send_config,
249         std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
250         VideoEncoderConfig* encoder_config) override {
251       send_config->rtp.extensions.clear();
252       send_config->rtp.extensions.push_back(
253           RtpExtension(RtpExtension::kAbsSendTimeUri, kAbsSendTimeExtensionId));
254     }
255 
256     void PerformTest() override {
257       EXPECT_TRUE(Wait()) << "Timed out while waiting for single RTP packet.";
258     }
259 
260    private:
261     RtpHeaderExtensionMap extensions_;
262   } test;
263 
264   RunBaseTest(&test);
265 }
266 
TEST_F(VideoSendStreamTest,SupportsTransmissionTimeOffset)267 TEST_F(VideoSendStreamTest, SupportsTransmissionTimeOffset) {
268   static const int kEncodeDelayMs = 5;
269   class TransmissionTimeOffsetObserver : public test::SendTest {
270    public:
271     TransmissionTimeOffsetObserver()
272         : SendTest(kDefaultTimeout), encoder_factory_([]() {
273             return std::make_unique<test::DelayedEncoder>(
274                 Clock::GetRealTimeClock(), kEncodeDelayMs);
275           }) {
276       extensions_.Register<TransmissionOffset>(kTimestampOffsetExtensionId);
277     }
278 
279    private:
280     Action OnSendRtp(const uint8_t* packet, size_t length) override {
281       RtpPacket rtp_packet(&extensions_);
282       EXPECT_TRUE(rtp_packet.Parse(packet, length));
283 
284       int32_t toffset = 0;
285       EXPECT_TRUE(rtp_packet.GetExtension<TransmissionOffset>(&toffset));
286       EXPECT_FALSE(rtp_packet.HasExtension<AbsoluteSendTime>());
287       EXPECT_GT(toffset, 0);
288       observation_complete_.Set();
289 
290       return SEND_PACKET;
291     }
292 
293     void ModifyVideoConfigs(
294         VideoSendStream::Config* send_config,
295         std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
296         VideoEncoderConfig* encoder_config) override {
297       send_config->encoder_settings.encoder_factory = &encoder_factory_;
298       send_config->rtp.extensions.clear();
299       send_config->rtp.extensions.push_back(RtpExtension(
300           RtpExtension::kTimestampOffsetUri, kTimestampOffsetExtensionId));
301     }
302 
303     void PerformTest() override {
304       EXPECT_TRUE(Wait()) << "Timed out while waiting for a single RTP packet.";
305     }
306 
307     test::FunctionVideoEncoderFactory encoder_factory_;
308     RtpHeaderExtensionMap extensions_;
309   } test;
310 
311   RunBaseTest(&test);
312 }
313 
TEST_F(VideoSendStreamTest,SupportsTransportWideSequenceNumbers)314 TEST_F(VideoSendStreamTest, SupportsTransportWideSequenceNumbers) {
315   static const uint8_t kExtensionId = kTransportSequenceNumberExtensionId;
316   class TransportWideSequenceNumberObserver : public test::SendTest {
317    public:
318     TransportWideSequenceNumberObserver()
319         : SendTest(kDefaultTimeout), encoder_factory_([]() {
320             return std::make_unique<test::FakeEncoder>(
321                 Clock::GetRealTimeClock());
322           }) {
323       extensions_.Register<TransportSequenceNumber>(kExtensionId);
324     }
325 
326    private:
327     Action OnSendRtp(const uint8_t* packet, size_t length) override {
328       RtpPacket rtp_packet(&extensions_);
329       EXPECT_TRUE(rtp_packet.Parse(packet, length));
330 
331       EXPECT_TRUE(rtp_packet.HasExtension<TransportSequenceNumber>());
332       EXPECT_FALSE(rtp_packet.HasExtension<TransmissionOffset>());
333       EXPECT_FALSE(rtp_packet.HasExtension<AbsoluteSendTime>());
334 
335       observation_complete_.Set();
336 
337       return SEND_PACKET;
338     }
339 
340     void ModifyVideoConfigs(
341         VideoSendStream::Config* send_config,
342         std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
343         VideoEncoderConfig* encoder_config) override {
344       send_config->encoder_settings.encoder_factory = &encoder_factory_;
345     }
346 
347     void PerformTest() override {
348       EXPECT_TRUE(Wait()) << "Timed out while waiting for a single RTP packet.";
349     }
350 
351     test::FunctionVideoEncoderFactory encoder_factory_;
352     RtpHeaderExtensionMap extensions_;
353   } test;
354 
355   RunBaseTest(&test);
356 }
357 
TEST_F(VideoSendStreamTest,SupportsVideoRotation)358 TEST_F(VideoSendStreamTest, SupportsVideoRotation) {
359   class VideoRotationObserver : public test::SendTest {
360    public:
361     VideoRotationObserver() : SendTest(kDefaultTimeout) {
362       extensions_.Register<VideoOrientation>(kVideoRotationExtensionId);
363     }
364 
365     Action OnSendRtp(const uint8_t* packet, size_t length) override {
366       RtpPacket rtp_packet(&extensions_);
367       EXPECT_TRUE(rtp_packet.Parse(packet, length));
368       // Only the last packet of the frame is required to have the extension.
369       if (!rtp_packet.Marker())
370         return SEND_PACKET;
371       EXPECT_EQ(rtp_packet.GetExtension<VideoOrientation>(), kVideoRotation_90);
372       observation_complete_.Set();
373       return SEND_PACKET;
374     }
375 
376     void ModifyVideoConfigs(
377         VideoSendStream::Config* send_config,
378         std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
379         VideoEncoderConfig* encoder_config) override {
380       send_config->rtp.extensions.clear();
381       send_config->rtp.extensions.push_back(RtpExtension(
382           RtpExtension::kVideoRotationUri, kVideoRotationExtensionId));
383     }
384 
385     void OnFrameGeneratorCapturerCreated(
386         test::FrameGeneratorCapturer* frame_generator_capturer) override {
387       frame_generator_capturer->SetFakeRotation(kVideoRotation_90);
388     }
389 
390     void PerformTest() override {
391       EXPECT_TRUE(Wait()) << "Timed out while waiting for single RTP packet.";
392     }
393 
394    private:
395     RtpHeaderExtensionMap extensions_;
396   } test;
397 
398   RunBaseTest(&test);
399 }
400 
TEST_F(VideoSendStreamTest,SupportsVideoContentType)401 TEST_F(VideoSendStreamTest, SupportsVideoContentType) {
402   class VideoContentTypeObserver : public test::SendTest {
403    public:
404     VideoContentTypeObserver()
405         : SendTest(kDefaultTimeout), first_frame_sent_(false) {
406       extensions_.Register<VideoContentTypeExtension>(
407           kVideoContentTypeExtensionId);
408     }
409 
410     Action OnSendRtp(const uint8_t* packet, size_t length) override {
411       RtpPacket rtp_packet(&extensions_);
412       EXPECT_TRUE(rtp_packet.Parse(packet, length));
413       // Only the last packet of the key-frame must have extension.
414       if (!rtp_packet.Marker() || first_frame_sent_)
415         return SEND_PACKET;
416       // First marker bit seen means that the first frame is sent.
417       first_frame_sent_ = true;
418       VideoContentType type;
419       EXPECT_TRUE(rtp_packet.GetExtension<VideoContentTypeExtension>(&type));
420       EXPECT_TRUE(videocontenttypehelpers::IsScreenshare(type));
421       observation_complete_.Set();
422       return SEND_PACKET;
423     }
424 
425     void ModifyVideoConfigs(
426         VideoSendStream::Config* send_config,
427         std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
428         VideoEncoderConfig* encoder_config) override {
429       send_config->rtp.extensions.clear();
430       send_config->rtp.extensions.push_back(RtpExtension(
431           RtpExtension::kVideoContentTypeUri, kVideoContentTypeExtensionId));
432       encoder_config->content_type = VideoEncoderConfig::ContentType::kScreen;
433     }
434 
435     void PerformTest() override {
436       EXPECT_TRUE(Wait()) << "Timed out while waiting for single RTP packet.";
437     }
438 
439    private:
440     bool first_frame_sent_;
441     RtpHeaderExtensionMap extensions_;
442   } test;
443 
444   RunBaseTest(&test);
445 }
446 
TEST_F(VideoSendStreamTest,SupportsVideoTimingFrames)447 TEST_F(VideoSendStreamTest, SupportsVideoTimingFrames) {
448   class VideoTimingObserver : public test::SendTest {
449    public:
450     VideoTimingObserver()
451         : SendTest(kDefaultTimeout), first_frame_sent_(false) {
452       extensions_.Register<VideoTimingExtension>(kVideoTimingExtensionId);
453     }
454 
455     Action OnSendRtp(const uint8_t* packet, size_t length) override {
456       RtpPacket rtp_packet(&extensions_);
457       EXPECT_TRUE(rtp_packet.Parse(packet, length));
458       // Only the last packet of the frame must have extension.
459       // Also don't check packets of the second frame if they happen to get
460       // through before the test terminates.
461       if (!rtp_packet.Marker() || first_frame_sent_)
462         return SEND_PACKET;
463       EXPECT_TRUE(rtp_packet.HasExtension<VideoTimingExtension>());
464       observation_complete_.Set();
465       first_frame_sent_ = true;
466       return SEND_PACKET;
467     }
468 
469     void ModifyVideoConfigs(
470         VideoSendStream::Config* send_config,
471         std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
472         VideoEncoderConfig* encoder_config) override {
473       send_config->rtp.extensions.clear();
474       send_config->rtp.extensions.push_back(
475           RtpExtension(RtpExtension::kVideoTimingUri, kVideoTimingExtensionId));
476     }
477 
478     void PerformTest() override {
479       EXPECT_TRUE(Wait()) << "Timed out while waiting for timing frames.";
480     }
481 
482    private:
483     RtpHeaderExtensionMap extensions_;
484     bool first_frame_sent_;
485   } test;
486 
487   RunBaseTest(&test);
488 }
489 
490 class FakeReceiveStatistics : public ReceiveStatisticsProvider {
491  public:
FakeReceiveStatistics(uint32_t send_ssrc,uint32_t last_sequence_number,uint32_t cumulative_lost,uint8_t fraction_lost)492   FakeReceiveStatistics(uint32_t send_ssrc,
493                         uint32_t last_sequence_number,
494                         uint32_t cumulative_lost,
495                         uint8_t fraction_lost) {
496     stat_.SetMediaSsrc(send_ssrc);
497     stat_.SetExtHighestSeqNum(last_sequence_number);
498     stat_.SetCumulativeLost(cumulative_lost);
499     stat_.SetFractionLost(fraction_lost);
500   }
501 
RtcpReportBlocks(size_t max_blocks)502   std::vector<rtcp::ReportBlock> RtcpReportBlocks(size_t max_blocks) override {
503     EXPECT_GE(max_blocks, 1u);
504     return {stat_};
505   }
506 
507  private:
508   rtcp::ReportBlock stat_;
509 };
510 
511 class UlpfecObserver : public test::EndToEndTest {
512  public:
513   // Some of the test cases are expected to time out.
514   // Use a shorter timeout window than the default one for those.
515   static constexpr TimeDelta kReducedTimeout = TimeDelta::Seconds(10);
516 
UlpfecObserver(bool header_extensions_enabled,bool use_nack,bool expect_red,bool expect_ulpfec,const std::string & codec,VideoEncoderFactory * encoder_factory)517   UlpfecObserver(bool header_extensions_enabled,
518                  bool use_nack,
519                  bool expect_red,
520                  bool expect_ulpfec,
521                  const std::string& codec,
522                  VideoEncoderFactory* encoder_factory)
523       : EndToEndTest(expect_ulpfec ? VideoSendStreamTest::kDefaultTimeout
524                                    : kReducedTimeout),
525         encoder_factory_(encoder_factory),
526         payload_name_(codec),
527         use_nack_(use_nack),
528         expect_red_(expect_red),
529         expect_ulpfec_(expect_ulpfec),
530         sent_media_(false),
531         sent_ulpfec_(false),
532         header_extensions_enabled_(header_extensions_enabled) {
533     extensions_.Register<AbsoluteSendTime>(kAbsSendTimeExtensionId);
534     extensions_.Register<TransportSequenceNumber>(
535         kTransportSequenceNumberExtensionId);
536   }
537 
538  private:
OnSendRtp(const uint8_t * packet,size_t length)539   Action OnSendRtp(const uint8_t* packet, size_t length) override {
540     RtpPacket rtp_packet(&extensions_);
541     EXPECT_TRUE(rtp_packet.Parse(packet, length));
542 
543     int encapsulated_payload_type = -1;
544     if (rtp_packet.PayloadType() == VideoSendStreamTest::kRedPayloadType) {
545       EXPECT_TRUE(expect_red_);
546       encapsulated_payload_type = rtp_packet.payload()[0];
547       if (encapsulated_payload_type !=
548           VideoSendStreamTest::kFakeVideoSendPayloadType) {
549         EXPECT_EQ(VideoSendStreamTest::kUlpfecPayloadType,
550                   encapsulated_payload_type);
551       }
552     } else {
553       EXPECT_EQ(VideoSendStreamTest::kFakeVideoSendPayloadType,
554                 rtp_packet.PayloadType());
555       if (rtp_packet.payload_size() > 0) {
556         // Not padding-only, media received outside of RED.
557         EXPECT_FALSE(expect_red_);
558         sent_media_ = true;
559       }
560     }
561 
562     if (header_extensions_enabled_) {
563       uint32_t abs_send_time;
564       EXPECT_TRUE(rtp_packet.GetExtension<AbsoluteSendTime>(&abs_send_time));
565       uint16_t transport_seq_num;
566       EXPECT_TRUE(
567           rtp_packet.GetExtension<TransportSequenceNumber>(&transport_seq_num));
568       if (!first_packet_) {
569         uint32_t kHalf24BitsSpace = 0xFFFFFF / 2;
570         if (abs_send_time <= kHalf24BitsSpace &&
571             prev_abs_send_time_ > kHalf24BitsSpace) {
572           // 24 bits wrap.
573           EXPECT_GT(prev_abs_send_time_, abs_send_time);
574         } else {
575           EXPECT_GE(abs_send_time, prev_abs_send_time_);
576         }
577 
578         uint16_t seq_num_diff = transport_seq_num - prev_transport_seq_num_;
579         EXPECT_EQ(1, seq_num_diff);
580       }
581       first_packet_ = false;
582       prev_abs_send_time_ = abs_send_time;
583       prev_transport_seq_num_ = transport_seq_num;
584     }
585 
586     if (encapsulated_payload_type != -1) {
587       if (encapsulated_payload_type ==
588           VideoSendStreamTest::kUlpfecPayloadType) {
589         EXPECT_TRUE(expect_ulpfec_);
590         sent_ulpfec_ = true;
591       } else {
592         sent_media_ = true;
593       }
594     }
595 
596     if (sent_media_ && sent_ulpfec_) {
597       observation_complete_.Set();
598     }
599 
600     return SEND_PACKET;
601   }
602 
CreateSendTransport(TaskQueueBase * task_queue,Call * sender_call)603   std::unique_ptr<test::PacketTransport> CreateSendTransport(
604       TaskQueueBase* task_queue,
605       Call* sender_call) override {
606     // At low RTT (< kLowRttNackMs) -> NACK only, no FEC.
607     // Configure some network delay.
608     const int kNetworkDelayMs = 100;
609     BuiltInNetworkBehaviorConfig config;
610     config.loss_percent = 5;
611     config.queue_delay_ms = kNetworkDelayMs;
612     return std::make_unique<test::PacketTransport>(
613         task_queue, sender_call, this, test::PacketTransport::kSender,
614         VideoSendStreamTest::payload_type_map_,
615         std::make_unique<FakeNetworkPipe>(
616             Clock::GetRealTimeClock(),
617             std::make_unique<SimulatedNetwork>(config)));
618   }
619 
ModifyVideoConfigs(VideoSendStream::Config * send_config,std::vector<VideoReceiveStreamInterface::Config> * receive_configs,VideoEncoderConfig * encoder_config)620   void ModifyVideoConfigs(
621       VideoSendStream::Config* send_config,
622       std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
623       VideoEncoderConfig* encoder_config) override {
624     if (use_nack_) {
625       send_config->rtp.nack.rtp_history_ms =
626           (*receive_configs)[0].rtp.nack.rtp_history_ms =
627               VideoSendStreamTest::kNackRtpHistoryMs;
628     }
629     send_config->encoder_settings.encoder_factory = encoder_factory_;
630     send_config->rtp.payload_name = payload_name_;
631     send_config->rtp.ulpfec.red_payload_type =
632         VideoSendStreamTest::kRedPayloadType;
633     send_config->rtp.ulpfec.ulpfec_payload_type =
634         VideoSendStreamTest::kUlpfecPayloadType;
635     if (!header_extensions_enabled_) {
636       send_config->rtp.extensions.clear();
637     } else {
638       send_config->rtp.extensions.push_back(
639           RtpExtension(RtpExtension::kAbsSendTimeUri, kAbsSendTimeExtensionId));
640     }
641     (*receive_configs)[0].rtp.extensions = send_config->rtp.extensions;
642     encoder_config->codec_type = PayloadStringToCodecType(payload_name_);
643     (*receive_configs)[0].rtp.red_payload_type =
644         send_config->rtp.ulpfec.red_payload_type;
645     (*receive_configs)[0].rtp.ulpfec_payload_type =
646         send_config->rtp.ulpfec.ulpfec_payload_type;
647   }
648 
PerformTest()649   void PerformTest() override {
650     EXPECT_EQ(expect_ulpfec_, Wait())
651         << "Timed out waiting for ULPFEC and/or media packets.";
652   }
653 
654   VideoEncoderFactory* encoder_factory_;
655   RtpHeaderExtensionMap extensions_;
656   const std::string payload_name_;
657   const bool use_nack_;
658   const bool expect_red_;
659   const bool expect_ulpfec_;
660   bool sent_media_;
661   bool sent_ulpfec_;
662   const bool header_extensions_enabled_;
663   bool first_packet_ = true;
664   uint32_t prev_abs_send_time_ = 0;
665   uint16_t prev_transport_seq_num_ = 0;
666 };
667 
TEST_F(VideoSendStreamTest,SupportsUlpfecWithExtensions)668 TEST_F(VideoSendStreamTest, SupportsUlpfecWithExtensions) {
669   test::FunctionVideoEncoderFactory encoder_factory(
670       []() { return VP8Encoder::Create(); });
671   UlpfecObserver test(true, false, true, true, "VP8", &encoder_factory);
672   RunBaseTest(&test);
673 }
674 
TEST_F(VideoSendStreamTest,SupportsUlpfecWithoutExtensions)675 TEST_F(VideoSendStreamTest, SupportsUlpfecWithoutExtensions) {
676   test::FunctionVideoEncoderFactory encoder_factory(
677       []() { return VP8Encoder::Create(); });
678   UlpfecObserver test(false, false, true, true, "VP8", &encoder_factory);
679   RunBaseTest(&test);
680 }
681 
682 class VideoSendStreamWithoutUlpfecTest : public test::CallTest {
683  protected:
VideoSendStreamWithoutUlpfecTest()684   VideoSendStreamWithoutUlpfecTest()
685       : field_trial_(field_trials_, "WebRTC-DisableUlpFecExperiment/Enabled/") {
686   }
687 
688   test::ScopedKeyValueConfig field_trial_;
689 };
690 
TEST_F(VideoSendStreamWithoutUlpfecTest,NoUlpfecIfDisabledThroughFieldTrial)691 TEST_F(VideoSendStreamWithoutUlpfecTest, NoUlpfecIfDisabledThroughFieldTrial) {
692   test::FunctionVideoEncoderFactory encoder_factory(
693       []() { return VP8Encoder::Create(); });
694   UlpfecObserver test(false, false, false, false, "VP8", &encoder_factory);
695   RunBaseTest(&test);
696 }
697 
698 // The FEC scheme used is not efficient for H264, so we should not use RED/FEC
699 // since we'll still have to re-request FEC packets, effectively wasting
700 // bandwidth since the receiver has to wait for FEC retransmissions to determine
701 // that the received state is actually decodable.
TEST_F(VideoSendStreamTest,DoesNotUtilizeUlpfecForH264WithNackEnabled)702 TEST_F(VideoSendStreamTest, DoesNotUtilizeUlpfecForH264WithNackEnabled) {
703   test::FunctionVideoEncoderFactory encoder_factory([]() {
704     return std::make_unique<test::FakeH264Encoder>(Clock::GetRealTimeClock());
705   });
706   UlpfecObserver test(false, true, false, false, "H264", &encoder_factory);
707   RunBaseTest(&test);
708 }
709 
710 // Without retransmissions FEC for H264 is fine.
TEST_F(VideoSendStreamTest,DoesUtilizeUlpfecForH264WithoutNackEnabled)711 TEST_F(VideoSendStreamTest, DoesUtilizeUlpfecForH264WithoutNackEnabled) {
712   test::FunctionVideoEncoderFactory encoder_factory([]() {
713     return std::make_unique<test::FakeH264Encoder>(Clock::GetRealTimeClock());
714   });
715   UlpfecObserver test(false, false, true, true, "H264", &encoder_factory);
716   RunBaseTest(&test);
717 }
718 
TEST_F(VideoSendStreamTest,DoesUtilizeUlpfecForVp8WithNackEnabled)719 TEST_F(VideoSendStreamTest, DoesUtilizeUlpfecForVp8WithNackEnabled) {
720   test::FunctionVideoEncoderFactory encoder_factory(
721       []() { return VP8Encoder::Create(); });
722   UlpfecObserver test(false, true, true, true, "VP8", &encoder_factory);
723   RunBaseTest(&test);
724 }
725 
726 #if defined(RTC_ENABLE_VP9)
TEST_F(VideoSendStreamTest,DoesUtilizeUlpfecForVp9WithNackEnabled)727 TEST_F(VideoSendStreamTest, DoesUtilizeUlpfecForVp9WithNackEnabled) {
728   test::FunctionVideoEncoderFactory encoder_factory(
729       []() { return VP9Encoder::Create(); });
730   UlpfecObserver test(false, true, true, true, "VP9", &encoder_factory);
731   RunBaseTest(&test);
732 }
733 #endif  // defined(RTC_ENABLE_VP9)
734 
TEST_F(VideoSendStreamTest,SupportsUlpfecWithMultithreadedH264)735 TEST_F(VideoSendStreamTest, SupportsUlpfecWithMultithreadedH264) {
736   std::unique_ptr<TaskQueueFactory> task_queue_factory =
737       CreateDefaultTaskQueueFactory();
738   test::FunctionVideoEncoderFactory encoder_factory([&]() {
739     return std::make_unique<test::MultithreadedFakeH264Encoder>(
740         Clock::GetRealTimeClock(), task_queue_factory.get());
741   });
742   UlpfecObserver test(false, false, true, true, "H264", &encoder_factory);
743   RunBaseTest(&test);
744 }
745 
746 // TODO(brandtr): Move these FlexFEC tests when we have created
747 // FlexfecSendStream.
748 class FlexfecObserver : public test::EndToEndTest {
749  public:
FlexfecObserver(bool header_extensions_enabled,bool use_nack,const std::string & codec,VideoEncoderFactory * encoder_factory,size_t num_video_streams)750   FlexfecObserver(bool header_extensions_enabled,
751                   bool use_nack,
752                   const std::string& codec,
753                   VideoEncoderFactory* encoder_factory,
754                   size_t num_video_streams)
755       : EndToEndTest(VideoSendStreamTest::kDefaultTimeout),
756         encoder_factory_(encoder_factory),
757         payload_name_(codec),
758         use_nack_(use_nack),
759         sent_media_(false),
760         sent_flexfec_(false),
761         header_extensions_enabled_(header_extensions_enabled),
762         num_video_streams_(num_video_streams) {
763     extensions_.Register<AbsoluteSendTime>(kAbsSendTimeExtensionId);
764     extensions_.Register<TransmissionOffset>(kTimestampOffsetExtensionId);
765     extensions_.Register<TransportSequenceNumber>(
766         kTransportSequenceNumberExtensionId);
767   }
768 
GetNumFlexfecStreams() const769   size_t GetNumFlexfecStreams() const override { return 1; }
GetNumVideoStreams() const770   size_t GetNumVideoStreams() const override { return num_video_streams_; }
771 
772  private:
OnSendRtp(const uint8_t * packet,size_t length)773   Action OnSendRtp(const uint8_t* packet, size_t length) override {
774     RtpPacket rtp_packet(&extensions_);
775     EXPECT_TRUE(rtp_packet.Parse(packet, length));
776 
777     if (rtp_packet.PayloadType() == VideoSendStreamTest::kFlexfecPayloadType) {
778       EXPECT_EQ(VideoSendStreamTest::kFlexfecSendSsrc, rtp_packet.Ssrc());
779       sent_flexfec_ = true;
780     } else {
781       EXPECT_EQ(VideoSendStreamTest::kFakeVideoSendPayloadType,
782                 rtp_packet.PayloadType());
783       EXPECT_THAT(::testing::make_tuple(VideoSendStreamTest::kVideoSendSsrcs,
784                                         num_video_streams_),
785                   ::testing::Contains(rtp_packet.Ssrc()));
786       sent_media_ = true;
787     }
788 
789     if (header_extensions_enabled_) {
790       EXPECT_TRUE(rtp_packet.HasExtension<AbsoluteSendTime>());
791       EXPECT_TRUE(rtp_packet.HasExtension<TransmissionOffset>());
792       EXPECT_TRUE(rtp_packet.HasExtension<TransportSequenceNumber>());
793     }
794 
795     if (sent_media_ && sent_flexfec_) {
796       observation_complete_.Set();
797     }
798 
799     return SEND_PACKET;
800   }
801 
CreateSendTransport(TaskQueueBase * task_queue,Call * sender_call)802   std::unique_ptr<test::PacketTransport> CreateSendTransport(
803       TaskQueueBase* task_queue,
804       Call* sender_call) override {
805     // At low RTT (< kLowRttNackMs) -> NACK only, no FEC.
806     // Therefore we need some network delay.
807     const int kNetworkDelayMs = 100;
808     BuiltInNetworkBehaviorConfig config;
809     config.loss_percent = 5;
810     config.queue_delay_ms = kNetworkDelayMs;
811     return std::make_unique<test::PacketTransport>(
812         task_queue, sender_call, this, test::PacketTransport::kSender,
813         VideoSendStreamTest::payload_type_map_,
814         std::make_unique<FakeNetworkPipe>(
815             Clock::GetRealTimeClock(),
816             std::make_unique<SimulatedNetwork>(config)));
817   }
818 
CreateReceiveTransport(TaskQueueBase * task_queue)819   std::unique_ptr<test::PacketTransport> CreateReceiveTransport(
820       TaskQueueBase* task_queue) override {
821     // We need the RTT to be >200 ms to send FEC and the network delay for the
822     // send transport is 100 ms, so add 100 ms (but no loss) on the return link.
823     BuiltInNetworkBehaviorConfig config;
824     config.loss_percent = 0;
825     config.queue_delay_ms = 100;
826     return std::make_unique<test::PacketTransport>(
827         task_queue, nullptr, this, test::PacketTransport::kReceiver,
828         VideoSendStreamTest::payload_type_map_,
829         std::make_unique<FakeNetworkPipe>(
830             Clock::GetRealTimeClock(),
831             std::make_unique<SimulatedNetwork>(config)));
832   }
833 
ModifyVideoConfigs(VideoSendStream::Config * send_config,std::vector<VideoReceiveStreamInterface::Config> * receive_configs,VideoEncoderConfig * encoder_config)834   void ModifyVideoConfigs(
835       VideoSendStream::Config* send_config,
836       std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
837       VideoEncoderConfig* encoder_config) override {
838     if (use_nack_) {
839       send_config->rtp.nack.rtp_history_ms =
840           (*receive_configs)[0].rtp.nack.rtp_history_ms =
841               VideoSendStreamTest::kNackRtpHistoryMs;
842     }
843     send_config->encoder_settings.encoder_factory = encoder_factory_;
844     send_config->rtp.payload_name = payload_name_;
845     if (header_extensions_enabled_) {
846       send_config->rtp.extensions.push_back(
847           RtpExtension(RtpExtension::kAbsSendTimeUri, kAbsSendTimeExtensionId));
848       send_config->rtp.extensions.push_back(RtpExtension(
849           RtpExtension::kTimestampOffsetUri, kTimestampOffsetExtensionId));
850     } else {
851       send_config->rtp.extensions.clear();
852     }
853     (*receive_configs)[0].rtp.extensions = send_config->rtp.extensions;
854     encoder_config->codec_type = PayloadStringToCodecType(payload_name_);
855   }
856 
PerformTest()857   void PerformTest() override {
858     EXPECT_TRUE(Wait())
859         << "Timed out waiting for FlexFEC and/or media packets.";
860   }
861 
862   VideoEncoderFactory* encoder_factory_;
863   RtpHeaderExtensionMap extensions_;
864   const std::string payload_name_;
865   const bool use_nack_;
866   bool sent_media_;
867   bool sent_flexfec_;
868   const bool header_extensions_enabled_;
869   const size_t num_video_streams_;
870 };
871 
TEST_F(VideoSendStreamTest,SupportsFlexfecVp8)872 TEST_F(VideoSendStreamTest, SupportsFlexfecVp8) {
873   test::FunctionVideoEncoderFactory encoder_factory(
874       []() { return VP8Encoder::Create(); });
875   FlexfecObserver test(false, false, "VP8", &encoder_factory, 1);
876   RunBaseTest(&test);
877 }
878 
TEST_F(VideoSendStreamTest,SupportsFlexfecSimulcastVp8)879 TEST_F(VideoSendStreamTest, SupportsFlexfecSimulcastVp8) {
880   test::FunctionVideoEncoderFactory encoder_factory(
881       []() { return VP8Encoder::Create(); });
882   FlexfecObserver test(false, false, "VP8", &encoder_factory, 2);
883   RunBaseTest(&test);
884 }
885 
TEST_F(VideoSendStreamTest,SupportsFlexfecWithNackVp8)886 TEST_F(VideoSendStreamTest, SupportsFlexfecWithNackVp8) {
887   test::FunctionVideoEncoderFactory encoder_factory(
888       []() { return VP8Encoder::Create(); });
889   FlexfecObserver test(false, true, "VP8", &encoder_factory, 1);
890   RunBaseTest(&test);
891 }
892 
TEST_F(VideoSendStreamTest,SupportsFlexfecWithRtpExtensionsVp8)893 TEST_F(VideoSendStreamTest, SupportsFlexfecWithRtpExtensionsVp8) {
894   test::FunctionVideoEncoderFactory encoder_factory(
895       []() { return VP8Encoder::Create(); });
896   FlexfecObserver test(true, false, "VP8", &encoder_factory, 1);
897   RunBaseTest(&test);
898 }
899 
900 #if defined(RTC_ENABLE_VP9)
TEST_F(VideoSendStreamTest,SupportsFlexfecVp9)901 TEST_F(VideoSendStreamTest, SupportsFlexfecVp9) {
902   test::FunctionVideoEncoderFactory encoder_factory(
903       []() { return VP9Encoder::Create(); });
904   FlexfecObserver test(false, false, "VP9", &encoder_factory, 1);
905   RunBaseTest(&test);
906 }
907 
TEST_F(VideoSendStreamTest,SupportsFlexfecWithNackVp9)908 TEST_F(VideoSendStreamTest, SupportsFlexfecWithNackVp9) {
909   test::FunctionVideoEncoderFactory encoder_factory(
910       []() { return VP9Encoder::Create(); });
911   FlexfecObserver test(false, true, "VP9", &encoder_factory, 1);
912   RunBaseTest(&test);
913 }
914 #endif  // defined(RTC_ENABLE_VP9)
915 
TEST_F(VideoSendStreamTest,SupportsFlexfecH264)916 TEST_F(VideoSendStreamTest, SupportsFlexfecH264) {
917   test::FunctionVideoEncoderFactory encoder_factory([]() {
918     return std::make_unique<test::FakeH264Encoder>(Clock::GetRealTimeClock());
919   });
920   FlexfecObserver test(false, false, "H264", &encoder_factory, 1);
921   RunBaseTest(&test);
922 }
923 
TEST_F(VideoSendStreamTest,SupportsFlexfecWithNackH264)924 TEST_F(VideoSendStreamTest, SupportsFlexfecWithNackH264) {
925   test::FunctionVideoEncoderFactory encoder_factory([]() {
926     return std::make_unique<test::FakeH264Encoder>(Clock::GetRealTimeClock());
927   });
928   FlexfecObserver test(false, true, "H264", &encoder_factory, 1);
929   RunBaseTest(&test);
930 }
931 
TEST_F(VideoSendStreamTest,SupportsFlexfecWithMultithreadedH264)932 TEST_F(VideoSendStreamTest, SupportsFlexfecWithMultithreadedH264) {
933   std::unique_ptr<TaskQueueFactory> task_queue_factory =
934       CreateDefaultTaskQueueFactory();
935   test::FunctionVideoEncoderFactory encoder_factory([&]() {
936     return std::make_unique<test::MultithreadedFakeH264Encoder>(
937         Clock::GetRealTimeClock(), task_queue_factory.get());
938   });
939 
940   FlexfecObserver test(false, false, "H264", &encoder_factory, 1);
941   RunBaseTest(&test);
942 }
943 
TestNackRetransmission(uint32_t retransmit_ssrc,uint8_t retransmit_payload_type)944 void VideoSendStreamTest::TestNackRetransmission(
945     uint32_t retransmit_ssrc,
946     uint8_t retransmit_payload_type) {
947   class NackObserver : public test::SendTest {
948    public:
949     explicit NackObserver(uint32_t retransmit_ssrc,
950                           uint8_t retransmit_payload_type)
951         : SendTest(kDefaultTimeout),
952           send_count_(0),
953           retransmit_count_(0),
954           retransmit_ssrc_(retransmit_ssrc),
955           retransmit_payload_type_(retransmit_payload_type) {}
956 
957    private:
958     Action OnSendRtp(const uint8_t* packet, size_t length) override {
959       RtpPacket rtp_packet;
960       EXPECT_TRUE(rtp_packet.Parse(packet, length));
961 
962       // NACK packets two times at some arbitrary points.
963       const int kNackedPacketsAtOnceCount = 3;
964       const int kRetransmitTarget = kNackedPacketsAtOnceCount * 2;
965 
966       // Skip padding packets because they will never be retransmitted.
967       if (rtp_packet.payload_size() == 0) {
968         return SEND_PACKET;
969       }
970 
971       ++send_count_;
972 
973       // NACK packets at arbitrary points.
974       if (send_count_ % 25 == 0) {
975         RTCPSender::Configuration config;
976         config.clock = Clock::GetRealTimeClock();
977         config.outgoing_transport = transport_adapter_.get();
978         config.rtcp_report_interval = TimeDelta::Millis(kRtcpIntervalMs);
979         config.local_media_ssrc = kReceiverLocalVideoSsrc;
980         RTCPSender rtcp_sender(config);
981 
982         rtcp_sender.SetRTCPStatus(RtcpMode::kReducedSize);
983         rtcp_sender.SetRemoteSSRC(kVideoSendSsrcs[0]);
984 
985         RTCPSender::FeedbackState feedback_state;
986         uint16_t nack_sequence_numbers[kNackedPacketsAtOnceCount];
987         int nack_count = 0;
988         for (uint16_t sequence_number :
989              sequence_numbers_pending_retransmission_) {
990           if (nack_count < kNackedPacketsAtOnceCount) {
991             nack_sequence_numbers[nack_count++] = sequence_number;
992           } else {
993             break;
994           }
995         }
996 
997         EXPECT_EQ(0, rtcp_sender.SendRTCP(feedback_state, kRtcpNack, nack_count,
998                                           nack_sequence_numbers));
999       }
1000 
1001       uint16_t sequence_number = rtp_packet.SequenceNumber();
1002       if (rtp_packet.Ssrc() == retransmit_ssrc_ &&
1003           retransmit_ssrc_ != kVideoSendSsrcs[0]) {
1004         // Not kVideoSendSsrcs[0], assume correct RTX packet. Extract sequence
1005         // number.
1006         const uint8_t* rtx_header = rtp_packet.payload().data();
1007         sequence_number = (rtx_header[0] << 8) + rtx_header[1];
1008       }
1009 
1010       auto it = sequence_numbers_pending_retransmission_.find(sequence_number);
1011       if (it == sequence_numbers_pending_retransmission_.end()) {
1012         // Not currently pending retransmission. Add it to retransmission queue
1013         // if media and limit not reached.
1014         if (rtp_packet.Ssrc() == kVideoSendSsrcs[0] &&
1015             rtp_packet.payload_size() > 0 &&
1016             retransmit_count_ +
1017                     sequence_numbers_pending_retransmission_.size() <
1018                 kRetransmitTarget) {
1019           sequence_numbers_pending_retransmission_.insert(sequence_number);
1020         }
1021       } else {
1022         // Packet is a retransmission, remove it from queue and check if done.
1023         sequence_numbers_pending_retransmission_.erase(it);
1024         if (++retransmit_count_ == kRetransmitTarget) {
1025           EXPECT_EQ(retransmit_ssrc_, rtp_packet.Ssrc());
1026           EXPECT_EQ(retransmit_payload_type_, rtp_packet.PayloadType());
1027           observation_complete_.Set();
1028         }
1029       }
1030 
1031       return SEND_PACKET;
1032     }
1033 
1034     void ModifyVideoConfigs(
1035         VideoSendStream::Config* send_config,
1036         std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
1037         VideoEncoderConfig* encoder_config) override {
1038       transport_adapter_.reset(
1039           new internal::TransportAdapter(send_config->send_transport));
1040       transport_adapter_->Enable();
1041       send_config->rtp.nack.rtp_history_ms = kNackRtpHistoryMs;
1042       send_config->rtp.rtx.payload_type = retransmit_payload_type_;
1043       if (retransmit_ssrc_ != kVideoSendSsrcs[0])
1044         send_config->rtp.rtx.ssrcs.push_back(retransmit_ssrc_);
1045     }
1046 
1047     void PerformTest() override {
1048       EXPECT_TRUE(Wait()) << "Timed out while waiting for NACK retransmission.";
1049     }
1050 
1051     std::unique_ptr<internal::TransportAdapter> transport_adapter_;
1052     int send_count_;
1053     int retransmit_count_;
1054     const uint32_t retransmit_ssrc_;
1055     const uint8_t retransmit_payload_type_;
1056     std::set<uint16_t> sequence_numbers_pending_retransmission_;
1057   } test(retransmit_ssrc, retransmit_payload_type);
1058 
1059   RunBaseTest(&test);
1060 }
1061 
TEST_F(VideoSendStreamTest,RetransmitsNack)1062 TEST_F(VideoSendStreamTest, RetransmitsNack) {
1063   // Normal NACKs should use the send SSRC.
1064   TestNackRetransmission(kVideoSendSsrcs[0], kFakeVideoSendPayloadType);
1065 }
1066 
TEST_F(VideoSendStreamTest,RetransmitsNackOverRtx)1067 TEST_F(VideoSendStreamTest, RetransmitsNackOverRtx) {
1068   // NACKs over RTX should use a separate SSRC.
1069   TestNackRetransmission(kSendRtxSsrcs[0], kSendRtxPayloadType);
1070 }
1071 
TestPacketFragmentationSize(VideoFormat format,bool with_fec)1072 void VideoSendStreamTest::TestPacketFragmentationSize(VideoFormat format,
1073                                                       bool with_fec) {
1074   // Use a fake encoder to output a frame of every size in the range [90, 290],
1075   // for each size making sure that the exact number of payload bytes received
1076   // is correct and that packets are fragmented to respect max packet size.
1077   static const size_t kMaxPacketSize = 128;
1078   static const size_t start = 90;
1079   static const size_t stop = 290;
1080 
1081   // Observer that verifies that the expected number of packets and bytes
1082   // arrive for each frame size, from start_size to stop_size.
1083   class FrameFragmentationTest : public test::SendTest {
1084    public:
1085     FrameFragmentationTest(size_t max_packet_size,
1086                            size_t start_size,
1087                            size_t stop_size,
1088                            bool test_generic_packetization,
1089                            bool use_fec)
1090         : SendTest(kLongTimeout),
1091           encoder_(stop),
1092           encoder_factory_(&encoder_),
1093           max_packet_size_(max_packet_size),
1094           stop_size_(stop_size),
1095           test_generic_packetization_(test_generic_packetization),
1096           use_fec_(use_fec),
1097           packet_count_(0),
1098           packets_lost_(0),
1099           last_packet_count_(0),
1100           last_packets_lost_(0),
1101           accumulated_size_(0),
1102           accumulated_payload_(0),
1103           fec_packet_received_(false),
1104           current_size_rtp_(start_size),
1105           current_size_frame_(static_cast<int>(start_size)) {
1106       // Fragmentation required, this test doesn't make sense without it.
1107       encoder_.SetFrameSize(start_size);
1108       RTC_DCHECK_GT(stop_size, max_packet_size);
1109       if (!test_generic_packetization_)
1110         encoder_.SetCodecType(kVideoCodecVP8);
1111     }
1112 
1113    private:
1114     Action OnSendRtp(const uint8_t* packet, size_t size) override {
1115       size_t length = size;
1116       RtpPacket rtp_packet;
1117       EXPECT_TRUE(rtp_packet.Parse(packet, length));
1118 
1119       EXPECT_LE(length, max_packet_size_);
1120 
1121       if (use_fec_ && rtp_packet.payload_size() > 0) {
1122         uint8_t payload_type = rtp_packet.payload()[0];
1123         bool is_fec = rtp_packet.PayloadType() == kRedPayloadType &&
1124                       payload_type == kUlpfecPayloadType;
1125         if (is_fec) {
1126           fec_packet_received_ = true;
1127           return SEND_PACKET;
1128         }
1129       }
1130 
1131       accumulated_size_ += length;
1132 
1133       if (use_fec_)
1134         TriggerLossReport(rtp_packet);
1135 
1136       if (test_generic_packetization_) {
1137         size_t overhead = rtp_packet.headers_size() + rtp_packet.padding_size();
1138         // Only remove payload header and RED header if the packet actually
1139         // contains payload.
1140         if (length > overhead) {
1141           overhead += (1 /* Generic header */);
1142           if (use_fec_)
1143             overhead += 1;  // RED for FEC header.
1144         }
1145         EXPECT_GE(length, overhead);
1146         accumulated_payload_ += length - overhead;
1147       }
1148 
1149       // Marker bit set indicates last packet of a frame.
1150       if (rtp_packet.Marker()) {
1151         if (use_fec_ && accumulated_payload_ == current_size_rtp_ - 1) {
1152           // With FEC enabled, frame size is incremented asynchronously, so
1153           // "old" frames one byte too small may arrive. Accept, but don't
1154           // increase expected frame size.
1155           accumulated_size_ = 0;
1156           accumulated_payload_ = 0;
1157           return SEND_PACKET;
1158         }
1159 
1160         EXPECT_GE(accumulated_size_, current_size_rtp_);
1161         if (test_generic_packetization_) {
1162           EXPECT_EQ(current_size_rtp_, accumulated_payload_);
1163         }
1164 
1165         // Last packet of frame; reset counters.
1166         accumulated_size_ = 0;
1167         accumulated_payload_ = 0;
1168         if (current_size_rtp_ == stop_size_) {
1169           // Done! (Don't increase size again, might arrive more @ stop_size).
1170           observation_complete_.Set();
1171         } else {
1172           // Increase next expected frame size. If testing with FEC, make sure
1173           // a FEC packet has been received for this frame size before
1174           // proceeding, to make sure that redundancy packets don't exceed
1175           // size limit.
1176           if (!use_fec_) {
1177             ++current_size_rtp_;
1178           } else if (fec_packet_received_) {
1179             fec_packet_received_ = false;
1180             ++current_size_rtp_;
1181 
1182             MutexLock lock(&mutex_);
1183             ++current_size_frame_;
1184           }
1185         }
1186       }
1187 
1188       return SEND_PACKET;
1189     }
1190 
1191     void TriggerLossReport(const RtpPacket& rtp_packet) {
1192       // Send lossy receive reports to trigger FEC enabling.
1193       const int kLossPercent = 5;
1194       if (++packet_count_ % (100 / kLossPercent) == 0) {
1195         packets_lost_++;
1196         int loss_delta = packets_lost_ - last_packets_lost_;
1197         int packets_delta = packet_count_ - last_packet_count_;
1198         last_packet_count_ = packet_count_;
1199         last_packets_lost_ = packets_lost_;
1200         uint8_t loss_ratio =
1201             static_cast<uint8_t>(loss_delta * 255 / packets_delta);
1202         FakeReceiveStatistics lossy_receive_stats(
1203             kVideoSendSsrcs[0], rtp_packet.SequenceNumber(),
1204             packets_lost_,  // Cumulative lost.
1205             loss_ratio);    // Loss percent.
1206         RTCPSender::Configuration config;
1207         config.clock = Clock::GetRealTimeClock();
1208         config.receive_statistics = &lossy_receive_stats;
1209         config.outgoing_transport = transport_adapter_.get();
1210         config.rtcp_report_interval = TimeDelta::Millis(kRtcpIntervalMs);
1211         config.local_media_ssrc = kVideoSendSsrcs[0];
1212         RTCPSender rtcp_sender(config);
1213 
1214         rtcp_sender.SetRTCPStatus(RtcpMode::kReducedSize);
1215         rtcp_sender.SetRemoteSSRC(kVideoSendSsrcs[0]);
1216 
1217         RTCPSender::FeedbackState feedback_state;
1218 
1219         EXPECT_EQ(0, rtcp_sender.SendRTCP(feedback_state, kRtcpRr));
1220       }
1221     }
1222 
1223     void UpdateConfiguration() {
1224       MutexLock lock(&mutex_);
1225       // Increase frame size for next encoded frame, in the context of the
1226       // encoder thread.
1227       if (!use_fec_ && current_size_frame_ < static_cast<int32_t>(stop_size_)) {
1228         ++current_size_frame_;
1229       }
1230       encoder_.SetFrameSize(static_cast<size_t>(current_size_frame_));
1231     }
1232     void ModifySenderBitrateConfig(
1233         BitrateConstraints* bitrate_config) override {
1234       const int kMinBitrateBps = 300000;
1235       bitrate_config->min_bitrate_bps = kMinBitrateBps;
1236     }
1237 
1238     void ModifyVideoConfigs(
1239         VideoSendStream::Config* send_config,
1240         std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
1241         VideoEncoderConfig* encoder_config) override {
1242       transport_adapter_.reset(
1243           new internal::TransportAdapter(send_config->send_transport));
1244       transport_adapter_->Enable();
1245       if (use_fec_) {
1246         send_config->rtp.ulpfec.red_payload_type = kRedPayloadType;
1247         send_config->rtp.ulpfec.ulpfec_payload_type = kUlpfecPayloadType;
1248       }
1249 
1250       if (!test_generic_packetization_)
1251         send_config->rtp.payload_name = "VP8";
1252 
1253       send_config->encoder_settings.encoder_factory = &encoder_factory_;
1254       send_config->rtp.max_packet_size = kMaxPacketSize;
1255       encoder_.RegisterPostEncodeCallback([this]() { UpdateConfiguration(); });
1256 
1257       // Make sure there is at least one extension header, to make the RTP
1258       // header larger than the base length of 12 bytes.
1259       EXPECT_FALSE(send_config->rtp.extensions.empty());
1260 
1261       // Setup screen content disables frame dropping which makes this easier.
1262       EXPECT_EQ(1u, encoder_config->simulcast_layers.size());
1263       encoder_config->simulcast_layers[0].num_temporal_layers = 2;
1264       encoder_config->content_type = VideoEncoderConfig::ContentType::kScreen;
1265     }
1266 
1267     void PerformTest() override {
1268       EXPECT_TRUE(Wait()) << "Timed out while observing incoming RTP packets.";
1269     }
1270 
1271     std::unique_ptr<internal::TransportAdapter> transport_adapter_;
1272     test::ConfigurableFrameSizeEncoder encoder_;
1273     test::VideoEncoderProxyFactory encoder_factory_;
1274 
1275     const size_t max_packet_size_;
1276     const size_t stop_size_;
1277     const bool test_generic_packetization_;
1278     const bool use_fec_;
1279 
1280     uint32_t packet_count_;
1281     uint32_t packets_lost_;
1282     uint32_t last_packet_count_;
1283     uint32_t last_packets_lost_;
1284     size_t accumulated_size_;
1285     size_t accumulated_payload_;
1286     bool fec_packet_received_;
1287 
1288     size_t current_size_rtp_;
1289     Mutex mutex_;
1290     int current_size_frame_ RTC_GUARDED_BY(mutex_);
1291   };
1292 
1293   // Don't auto increment if FEC is used; continue sending frame size until
1294   // a FEC packet has been received.
1295   FrameFragmentationTest test(kMaxPacketSize, start, stop, format == kGeneric,
1296                               with_fec);
1297 
1298   RunBaseTest(&test);
1299 }
1300 
1301 // TODO(sprang): Is there any way of speeding up these tests?
TEST_F(VideoSendStreamTest,FragmentsGenericAccordingToMaxPacketSize)1302 TEST_F(VideoSendStreamTest, FragmentsGenericAccordingToMaxPacketSize) {
1303   TestPacketFragmentationSize(kGeneric, false);
1304 }
1305 
TEST_F(VideoSendStreamTest,FragmentsGenericAccordingToMaxPacketSizeWithFec)1306 TEST_F(VideoSendStreamTest, FragmentsGenericAccordingToMaxPacketSizeWithFec) {
1307   TestPacketFragmentationSize(kGeneric, true);
1308 }
1309 
TEST_F(VideoSendStreamTest,FragmentsVp8AccordingToMaxPacketSize)1310 TEST_F(VideoSendStreamTest, FragmentsVp8AccordingToMaxPacketSize) {
1311   TestPacketFragmentationSize(kVP8, false);
1312 }
1313 
TEST_F(VideoSendStreamTest,FragmentsVp8AccordingToMaxPacketSizeWithFec)1314 TEST_F(VideoSendStreamTest, FragmentsVp8AccordingToMaxPacketSizeWithFec) {
1315   TestPacketFragmentationSize(kVP8, true);
1316 }
1317 
1318 // This test that padding stops being send after a while if the Camera stops
1319 // producing video frames and that padding resumes if the camera restarts.
TEST_F(VideoSendStreamTest,NoPaddingWhenVideoIsMuted)1320 TEST_F(VideoSendStreamTest, NoPaddingWhenVideoIsMuted) {
1321   class NoPaddingWhenVideoIsMuted : public test::SendTest {
1322    public:
1323     NoPaddingWhenVideoIsMuted()
1324         : SendTest(kDefaultTimeout),
1325           clock_(Clock::GetRealTimeClock()),
1326           capturer_(nullptr) {}
1327 
1328    private:
1329     Action OnSendRtp(const uint8_t* packet, size_t length) override {
1330       MutexLock lock(&mutex_);
1331       last_packet_time_ms_ = clock_->TimeInMilliseconds();
1332 
1333       RtpPacket rtp_packet;
1334       rtp_packet.Parse(packet, length);
1335       const bool only_padding = rtp_packet.payload_size() == 0;
1336 
1337       if (test_state_ == kBeforeStopCapture) {
1338         // Packets are flowing, stop camera.
1339         capturer_->Stop();
1340         test_state_ = kWaitingForPadding;
1341       } else if (test_state_ == kWaitingForPadding && only_padding) {
1342         // We're still getting padding, after stopping camera.
1343         test_state_ = kWaitingForNoPackets;
1344       } else if (test_state_ == kWaitingForMediaAfterCameraRestart &&
1345                  !only_padding) {
1346         // Media packets are flowing again, stop camera a second time.
1347         capturer_->Stop();
1348         test_state_ = kWaitingForPaddingAfterCameraStopsAgain;
1349       } else if (test_state_ == kWaitingForPaddingAfterCameraStopsAgain &&
1350                  only_padding) {
1351         // Padding is still flowing, test ok.
1352         observation_complete_.Set();
1353       }
1354       return SEND_PACKET;
1355     }
1356 
1357     Action OnSendRtcp(const uint8_t* packet, size_t length) override {
1358       MutexLock lock(&mutex_);
1359       const int kNoPacketsThresholdMs = 2000;
1360       if (test_state_ == kWaitingForNoPackets &&
1361           (last_packet_time_ms_ &&
1362            clock_->TimeInMilliseconds() - last_packet_time_ms_.value() >
1363                kNoPacketsThresholdMs)) {
1364         // No packets seen for `kNoPacketsThresholdMs`, restart camera.
1365         capturer_->Start();
1366         test_state_ = kWaitingForMediaAfterCameraRestart;
1367       }
1368       return SEND_PACKET;
1369     }
1370 
1371     void ModifyVideoConfigs(
1372         VideoSendStream::Config* send_config,
1373         std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
1374         VideoEncoderConfig* encoder_config) override {
1375       // Make sure padding is sent if encoder is not producing media.
1376       encoder_config->min_transmit_bitrate_bps = 50000;
1377     }
1378 
1379     void OnFrameGeneratorCapturerCreated(
1380         test::FrameGeneratorCapturer* frame_generator_capturer) override {
1381       MutexLock lock(&mutex_);
1382       capturer_ = frame_generator_capturer;
1383     }
1384 
1385     void PerformTest() override {
1386       EXPECT_TRUE(Wait())
1387           << "Timed out while waiting for RTP packets to stop being sent.";
1388     }
1389 
1390     enum TestState {
1391       kBeforeStopCapture,
1392       kWaitingForPadding,
1393       kWaitingForNoPackets,
1394       kWaitingForMediaAfterCameraRestart,
1395       kWaitingForPaddingAfterCameraStopsAgain
1396     };
1397 
1398     TestState test_state_ = kBeforeStopCapture;
1399     Clock* const clock_;
1400     Mutex mutex_;
1401     absl::optional<int64_t> last_packet_time_ms_ RTC_GUARDED_BY(mutex_);
1402     test::FrameGeneratorCapturer* capturer_ RTC_GUARDED_BY(mutex_);
1403   } test;
1404 
1405   RunBaseTest(&test);
1406 }
1407 
TEST_F(VideoSendStreamTest,PaddingIsPrimarilyRetransmissions)1408 TEST_F(VideoSendStreamTest, PaddingIsPrimarilyRetransmissions) {
1409   const int kCapacityKbps = 10000;  // 10 Mbps
1410   class PaddingIsPrimarilyRetransmissions : public test::EndToEndTest {
1411    public:
1412     PaddingIsPrimarilyRetransmissions()
1413         : EndToEndTest(kDefaultTimeout),
1414           clock_(Clock::GetRealTimeClock()),
1415           padding_length_(0),
1416           total_length_(0),
1417           call_(nullptr) {}
1418 
1419    private:
1420     void OnCallsCreated(Call* sender_call, Call* receiver_call) override {
1421       call_ = sender_call;
1422     }
1423 
1424     Action OnSendRtp(const uint8_t* packet, size_t length) override {
1425       MutexLock lock(&mutex_);
1426 
1427       RtpPacket rtp_packet;
1428       rtp_packet.Parse(packet, length);
1429       padding_length_ += rtp_packet.padding_size();
1430       total_length_ += length;
1431       return SEND_PACKET;
1432     }
1433 
1434     std::unique_ptr<test::PacketTransport> CreateSendTransport(
1435         TaskQueueBase* task_queue,
1436         Call* sender_call) override {
1437       const int kNetworkDelayMs = 50;
1438       BuiltInNetworkBehaviorConfig config;
1439       config.loss_percent = 10;
1440       config.link_capacity_kbps = kCapacityKbps;
1441       config.queue_delay_ms = kNetworkDelayMs;
1442       return std::make_unique<test::PacketTransport>(
1443           task_queue, sender_call, this, test::PacketTransport::kSender,
1444           payload_type_map_,
1445           std::make_unique<FakeNetworkPipe>(
1446               Clock::GetRealTimeClock(),
1447               std::make_unique<SimulatedNetwork>(config)));
1448     }
1449 
1450     void ModifyVideoConfigs(
1451         VideoSendStream::Config* send_config,
1452         std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
1453         VideoEncoderConfig* encoder_config) override {
1454       // Turn on RTX.
1455       send_config->rtp.rtx.payload_type = kFakeVideoSendPayloadType;
1456       send_config->rtp.rtx.ssrcs.push_back(kSendRtxSsrcs[0]);
1457     }
1458 
1459     void PerformTest() override {
1460       // TODO(isheriff): Some platforms do not ramp up as expected to full
1461       // capacity due to packet scheduling delays. Fix that before getting
1462       // rid of this.
1463       SleepMs(5000);
1464       {
1465         MutexLock lock(&mutex_);
1466         // Expect padding to be a small percentage of total bytes sent.
1467         EXPECT_LT(padding_length_, .1 * total_length_);
1468       }
1469     }
1470 
1471     Mutex mutex_;
1472     Clock* const clock_;
1473     size_t padding_length_ RTC_GUARDED_BY(mutex_);
1474     size_t total_length_ RTC_GUARDED_BY(mutex_);
1475     Call* call_;
1476   } test;
1477 
1478   RunBaseTest(&test);
1479 }
1480 
1481 // This test first observes "high" bitrate use at which point it sends a REMB to
1482 // indicate that it should be lowered significantly. The test then observes that
1483 // the bitrate observed is sinking well below the min-transmit-bitrate threshold
1484 // to verify that the min-transmit bitrate respects incoming REMB.
1485 //
1486 // Note that the test starts at "high" bitrate and does not ramp up to "higher"
1487 // bitrate since no receiver block or remb is sent in the initial phase.
TEST_F(VideoSendStreamTest,MinTransmitBitrateRespectsRemb)1488 TEST_F(VideoSendStreamTest, MinTransmitBitrateRespectsRemb) {
1489   static const int kMinTransmitBitrateBps = 400000;
1490   static const int kHighBitrateBps = 150000;
1491   static const int kRembBitrateBps = 80000;
1492   static const int kRembRespectedBitrateBps = 100000;
1493   class BitrateObserver : public test::SendTest {
1494    public:
1495     explicit BitrateObserver(TaskQueueBase* task_queue)
1496         : SendTest(kDefaultTimeout),
1497           task_queue_(task_queue),
1498           retranmission_rate_limiter_(Clock::GetRealTimeClock(), 1000),
1499           stream_(nullptr),
1500           bitrate_capped_(false),
1501           task_safety_flag_(PendingTaskSafetyFlag::CreateDetached()) {}
1502 
1503    private:
1504     Action OnSendRtp(const uint8_t* packet, size_t length) override {
1505       if (IsRtcpPacket(rtc::MakeArrayView(packet, length)))
1506         return DROP_PACKET;
1507 
1508       RtpPacket rtp_packet;
1509       RTC_CHECK(rtp_packet.Parse(packet, length));
1510       const uint32_t ssrc = rtp_packet.Ssrc();
1511       RTC_DCHECK(stream_);
1512 
1513       task_queue_->PostTask(SafeTask(task_safety_flag_, [this, ssrc]() {
1514         VideoSendStream::Stats stats = stream_->GetStats();
1515         if (!stats.substreams.empty()) {
1516           EXPECT_EQ(1u, stats.substreams.size());
1517           int total_bitrate_bps =
1518               stats.substreams.begin()->second.total_bitrate_bps;
1519           test::GetGlobalMetricsLogger()->LogSingleValueMetric(
1520               "bitrate_stats_min_transmit_bitrate_low_remb", "bitrate_bps",
1521               static_cast<size_t>(total_bitrate_bps) / 1000.0,
1522               test::Unit::kKilobitsPerSecond,
1523               test::ImprovementDirection::kNeitherIsBetter);
1524           if (total_bitrate_bps > kHighBitrateBps) {
1525             rtp_rtcp_->SetRemb(kRembBitrateBps, {ssrc});
1526             bitrate_capped_ = true;
1527           } else if (bitrate_capped_ &&
1528                      total_bitrate_bps < kRembRespectedBitrateBps) {
1529             observation_complete_.Set();
1530           }
1531         }
1532       }));
1533 
1534       // Packets don't have to be delivered since the test is the receiver.
1535       return DROP_PACKET;
1536     }
1537 
1538     void OnVideoStreamsCreated(VideoSendStream* send_stream,
1539                                const std::vector<VideoReceiveStreamInterface*>&
1540                                    receive_streams) override {
1541       stream_ = send_stream;
1542       RtpRtcpInterface::Configuration config;
1543       config.clock = Clock::GetRealTimeClock();
1544       config.outgoing_transport = feedback_transport_.get();
1545       config.retransmission_rate_limiter = &retranmission_rate_limiter_;
1546       rtp_rtcp_ = ModuleRtpRtcpImpl2::Create(config);
1547       rtp_rtcp_->SetRTCPStatus(RtcpMode::kReducedSize);
1548     }
1549 
1550     void ModifyVideoConfigs(
1551         VideoSendStream::Config* send_config,
1552         std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
1553         VideoEncoderConfig* encoder_config) override {
1554       feedback_transport_.reset(
1555           new internal::TransportAdapter(send_config->send_transport));
1556       feedback_transport_->Enable();
1557       encoder_config->min_transmit_bitrate_bps = kMinTransmitBitrateBps;
1558     }
1559 
1560     void OnStreamsStopped() override {
1561       task_safety_flag_->SetNotAlive();
1562       rtp_rtcp_.reset();
1563     }
1564 
1565     void PerformTest() override {
1566       EXPECT_TRUE(Wait())
1567           << "Timeout while waiting for low bitrate stats after REMB.";
1568     }
1569 
1570     TaskQueueBase* const task_queue_;
1571     std::unique_ptr<ModuleRtpRtcpImpl2> rtp_rtcp_;
1572     std::unique_ptr<internal::TransportAdapter> feedback_transport_;
1573     RateLimiter retranmission_rate_limiter_;
1574     VideoSendStream* stream_;
1575     bool bitrate_capped_;
1576     rtc::scoped_refptr<PendingTaskSafetyFlag> task_safety_flag_;
1577   } test(task_queue());
1578 
1579   RunBaseTest(&test);
1580 }
1581 
TEST_F(VideoSendStreamTest,ChangingNetworkRoute)1582 TEST_F(VideoSendStreamTest, ChangingNetworkRoute) {
1583   static const int kStartBitrateBps = 300000;
1584   static const int kNewMaxBitrateBps = 1234567;
1585   static const uint8_t kExtensionId = kTransportSequenceNumberExtensionId;
1586   class ChangingNetworkRouteTest : public test::EndToEndTest {
1587    public:
1588     explicit ChangingNetworkRouteTest(TaskQueueBase* task_queue)
1589         : EndToEndTest(test::CallTest::kDefaultTimeout),
1590           task_queue_(task_queue),
1591           call_(nullptr) {
1592       module_process_thread_.Detach();
1593       task_queue_thread_.Detach();
1594       extensions_.Register<TransportSequenceNumber>(kExtensionId);
1595     }
1596 
1597     ~ChangingNetworkRouteTest() {
1598       // Block until all already posted tasks run to avoid 'use after free'
1599       // when such task accesses `this`.
1600       SendTask(task_queue_, [] {});
1601     }
1602 
1603     void OnCallsCreated(Call* sender_call, Call* receiver_call) override {
1604       RTC_DCHECK_RUN_ON(&task_queue_thread_);
1605       RTC_DCHECK(!call_);
1606       call_ = sender_call;
1607     }
1608 
1609     void ModifyVideoConfigs(
1610         VideoSendStream::Config* send_config,
1611         std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
1612         VideoEncoderConfig* encoder_config) override {
1613       RTC_DCHECK_RUN_ON(&task_queue_thread_);
1614       send_config->rtp.extensions.clear();
1615       send_config->rtp.extensions.push_back(RtpExtension(
1616           RtpExtension::kTransportSequenceNumberUri, kExtensionId));
1617       (*receive_configs)[0].rtp.extensions = send_config->rtp.extensions;
1618       (*receive_configs)[0].rtp.transport_cc = true;
1619     }
1620 
1621     void ModifyAudioConfigs(AudioSendStream::Config* send_config,
1622                             std::vector<AudioReceiveStreamInterface::Config>*
1623                                 receive_configs) override {
1624       RTC_DCHECK_RUN_ON(&task_queue_thread_);
1625       send_config->rtp.extensions.clear();
1626       send_config->rtp.extensions.push_back(RtpExtension(
1627           RtpExtension::kTransportSequenceNumberUri, kExtensionId));
1628       (*receive_configs)[0].rtp.extensions.clear();
1629       (*receive_configs)[0].rtp.extensions = send_config->rtp.extensions;
1630       (*receive_configs)[0].rtp.transport_cc = true;
1631     }
1632 
1633     Action OnSendRtp(const uint8_t* packet, size_t length) override {
1634       RTC_DCHECK_RUN_ON(&module_process_thread_);
1635       task_queue_->PostTask([this]() {
1636         RTC_DCHECK_RUN_ON(&task_queue_thread_);
1637         if (!call_)
1638           return;
1639         Call::Stats stats = call_->GetStats();
1640         if (stats.send_bandwidth_bps > kStartBitrateBps)
1641           observation_complete_.Set();
1642       });
1643       return SEND_PACKET;
1644     }
1645 
1646     void OnStreamsStopped() override {
1647       RTC_DCHECK_RUN_ON(&task_queue_thread_);
1648       call_ = nullptr;
1649     }
1650 
1651     void PerformTest() override {
1652       rtc::NetworkRoute new_route;
1653       new_route.connected = true;
1654       new_route.local = rtc::RouteEndpoint::CreateWithNetworkId(10);
1655       new_route.remote = rtc::RouteEndpoint::CreateWithNetworkId(20);
1656       BitrateConstraints bitrate_config;
1657 
1658       SendTask(task_queue_,
1659                [this, &new_route, &bitrate_config]() {
1660                  RTC_DCHECK_RUN_ON(&task_queue_thread_);
1661                  call_->GetTransportControllerSend()->OnNetworkRouteChanged(
1662                      "transport", new_route);
1663                  bitrate_config.start_bitrate_bps = kStartBitrateBps;
1664                  call_->GetTransportControllerSend()->SetSdpBitrateParameters(
1665                      bitrate_config);
1666                });
1667 
1668       EXPECT_TRUE(Wait())
1669           << "Timed out while waiting for start bitrate to be exceeded.";
1670 
1671       SendTask(
1672           task_queue_, [this, &new_route, &bitrate_config]() {
1673             RTC_DCHECK_RUN_ON(&task_queue_thread_);
1674             bitrate_config.start_bitrate_bps = -1;
1675             bitrate_config.max_bitrate_bps = kNewMaxBitrateBps;
1676             call_->GetTransportControllerSend()->SetSdpBitrateParameters(
1677                 bitrate_config);
1678             // TODO(holmer): We should set the last sent packet id here and
1679             // verify that we correctly ignore any packet loss reported prior to
1680             // that id.
1681             new_route.local = rtc::RouteEndpoint::CreateWithNetworkId(
1682                 new_route.local.network_id() + 1);
1683             call_->GetTransportControllerSend()->OnNetworkRouteChanged(
1684                 "transport", new_route);
1685             EXPECT_GE(call_->GetStats().send_bandwidth_bps, kStartBitrateBps);
1686           });
1687     }
1688 
1689    private:
1690     webrtc::SequenceChecker module_process_thread_;
1691     webrtc::SequenceChecker task_queue_thread_;
1692     TaskQueueBase* const task_queue_;
1693     RtpHeaderExtensionMap extensions_;
1694     Call* call_ RTC_GUARDED_BY(task_queue_thread_);
1695   } test(task_queue());
1696 
1697   RunBaseTest(&test);
1698 }
1699 
1700 // Test that if specified, relay cap is lifted on transition to direct
1701 // connection.
1702 // TODO(https://bugs.webrtc.org/13353): Test disabled  due to flakiness.
TEST_F(VideoSendStreamTest,DISABLED_RelayToDirectRoute)1703 TEST_F(VideoSendStreamTest, DISABLED_RelayToDirectRoute) {
1704   static const int kStartBitrateBps = 300000;
1705   static const int kRelayBandwidthCapBps = 800000;
1706   static const int kMinPacketsToSend = 100;
1707   webrtc::test::ScopedKeyValueConfig field_trials(
1708       field_trials_, "WebRTC-Bwe-NetworkRouteConstraints/relay_cap:" +
1709                          std::to_string(kRelayBandwidthCapBps) + "bps/");
1710 
1711   class RelayToDirectRouteTest : public test::EndToEndTest {
1712    public:
1713     explicit RelayToDirectRouteTest(TaskQueueBase* task_queue)
1714         : EndToEndTest(test::CallTest::kDefaultTimeout),
1715           task_queue_(task_queue),
1716           call_(nullptr),
1717           packets_sent_(0),
1718           relayed_phase_(true) {
1719       module_process_thread_.Detach();
1720       task_queue_thread_.Detach();
1721     }
1722 
1723     ~RelayToDirectRouteTest() {
1724       // Block until all already posted tasks run to avoid 'use after free'
1725       // when such task accesses `this`.
1726       SendTask(task_queue_, [] {});
1727     }
1728 
1729     void OnCallsCreated(Call* sender_call, Call* receiver_call) override {
1730       RTC_DCHECK_RUN_ON(&task_queue_thread_);
1731       RTC_DCHECK(!call_);
1732       call_ = sender_call;
1733     }
1734 
1735     Action OnSendRtp(const uint8_t* packet, size_t length) override {
1736       RTC_DCHECK_RUN_ON(&module_process_thread_);
1737       task_queue_->PostTask([this]() {
1738         RTC_DCHECK_RUN_ON(&task_queue_thread_);
1739         if (!call_)
1740           return;
1741         bool had_time_to_exceed_cap_in_relayed_phase =
1742             relayed_phase_ && ++packets_sent_ > kMinPacketsToSend;
1743         bool did_exceed_cap =
1744             call_->GetStats().send_bandwidth_bps > kRelayBandwidthCapBps;
1745         if (did_exceed_cap || had_time_to_exceed_cap_in_relayed_phase)
1746           observation_complete_.Set();
1747       });
1748       return SEND_PACKET;
1749     }
1750 
1751     void OnStreamsStopped() override {
1752       RTC_DCHECK_RUN_ON(&task_queue_thread_);
1753       call_ = nullptr;
1754     }
1755 
1756     void PerformTest() override {
1757       rtc::NetworkRoute route;
1758       route.connected = true;
1759       route.local = rtc::RouteEndpoint::CreateWithNetworkId(10);
1760       route.remote = rtc::RouteEndpoint::CreateWithNetworkId(20);
1761 
1762       SendTask(task_queue_, [this, &route]() {
1763         RTC_DCHECK_RUN_ON(&task_queue_thread_);
1764         relayed_phase_ = true;
1765         route.remote = route.remote.CreateWithTurn(true);
1766         call_->GetTransportControllerSend()->OnNetworkRouteChanged("transport",
1767                                                                    route);
1768         BitrateConstraints bitrate_config;
1769         bitrate_config.start_bitrate_bps = kStartBitrateBps;
1770 
1771         call_->GetTransportControllerSend()->SetSdpBitrateParameters(
1772             bitrate_config);
1773       });
1774 
1775       EXPECT_TRUE(Wait())
1776           << "Timeout waiting for sufficient packets sent count.";
1777 
1778       SendTask(task_queue_, [this, &route]() {
1779         RTC_DCHECK_RUN_ON(&task_queue_thread_);
1780         EXPECT_LE(call_->GetStats().send_bandwidth_bps, kRelayBandwidthCapBps);
1781 
1782         route.remote = route.remote.CreateWithTurn(false);
1783         call_->GetTransportControllerSend()->OnNetworkRouteChanged("transport",
1784                                                                    route);
1785         relayed_phase_ = false;
1786         observation_complete_.Reset();
1787       });
1788 
1789       EXPECT_TRUE(Wait())
1790           << "Timeout while waiting for bandwidth to outgrow relay cap.";
1791     }
1792 
1793    private:
1794     webrtc::SequenceChecker module_process_thread_;
1795     webrtc::SequenceChecker task_queue_thread_;
1796     TaskQueueBase* const task_queue_;
1797     Call* call_ RTC_GUARDED_BY(task_queue_thread_);
1798     int packets_sent_ RTC_GUARDED_BY(task_queue_thread_);
1799     bool relayed_phase_ RTC_GUARDED_BY(task_queue_thread_);
1800   } test(task_queue());
1801 
1802   RunBaseTest(&test);
1803 }
1804 
TEST_F(VideoSendStreamTest,ChangingTransportOverhead)1805 TEST_F(VideoSendStreamTest, ChangingTransportOverhead) {
1806   class ChangingTransportOverheadTest : public test::EndToEndTest {
1807    public:
1808     explicit ChangingTransportOverheadTest(TaskQueueBase* task_queue)
1809         : EndToEndTest(test::CallTest::kDefaultTimeout),
1810           task_queue_(task_queue),
1811           call_(nullptr),
1812           packets_sent_(0),
1813           transport_overhead_(0) {}
1814 
1815     void OnCallsCreated(Call* sender_call, Call* receiver_call) override {
1816       call_ = sender_call;
1817     }
1818 
1819     Action OnSendRtp(const uint8_t* packet, size_t length) override {
1820       EXPECT_LE(length, kMaxRtpPacketSize);
1821       MutexLock lock(&lock_);
1822       if (++packets_sent_ < 100)
1823         return SEND_PACKET;
1824       observation_complete_.Set();
1825       return SEND_PACKET;
1826     }
1827 
1828     void ModifyVideoConfigs(
1829         VideoSendStream::Config* send_config,
1830         std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
1831         VideoEncoderConfig* encoder_config) override {
1832       send_config->rtp.max_packet_size = kMaxRtpPacketSize;
1833     }
1834 
1835     void PerformTest() override {
1836       SendTask(task_queue_, [this]() {
1837         transport_overhead_ = 100;
1838         call_->GetTransportControllerSend()->OnTransportOverheadChanged(
1839             transport_overhead_);
1840       });
1841 
1842       EXPECT_TRUE(Wait());
1843 
1844       {
1845         MutexLock lock(&lock_);
1846         packets_sent_ = 0;
1847       }
1848 
1849       SendTask(task_queue_, [this]() {
1850         transport_overhead_ = 500;
1851         call_->GetTransportControllerSend()->OnTransportOverheadChanged(
1852             transport_overhead_);
1853       });
1854 
1855       EXPECT_TRUE(Wait());
1856     }
1857 
1858    private:
1859     TaskQueueBase* const task_queue_;
1860     Call* call_;
1861     Mutex lock_;
1862     int packets_sent_ RTC_GUARDED_BY(lock_);
1863     int transport_overhead_;
1864     const size_t kMaxRtpPacketSize = 1000;
1865   } test(task_queue());
1866 
1867   RunBaseTest(&test);
1868 }
1869 
1870 // Test class takes takes as argument a switch selecting if type switch should
1871 // occur and a function pointer to reset the send stream. This is necessary
1872 // since you cannot change the content type of a VideoSendStream, you need to
1873 // recreate it. Stopping and recreating the stream can only be done on the main
1874 // thread and in the context of VideoSendStreamTest (not BaseTest).
1875 template <typename T>
1876 class MaxPaddingSetTest : public test::SendTest {
1877  public:
1878   static const uint32_t kMinTransmitBitrateBps = 400000;
1879   static const uint32_t kActualEncodeBitrateBps = 40000;
1880   static const uint32_t kMinPacketsToSend = 50;
1881 
MaxPaddingSetTest(bool test_switch_content_type,T * stream_reset_fun,TaskQueueBase * task_queue)1882   MaxPaddingSetTest(bool test_switch_content_type,
1883                     T* stream_reset_fun,
1884                     TaskQueueBase* task_queue)
1885       : SendTest(test::CallTest::kDefaultTimeout),
1886         running_without_padding_(test_switch_content_type),
1887         stream_resetter_(stream_reset_fun),
1888         task_queue_(task_queue) {
1889     RTC_DCHECK(stream_resetter_);
1890     module_process_thread_.Detach();
1891     task_queue_thread_.Detach();
1892   }
1893 
~MaxPaddingSetTest()1894   ~MaxPaddingSetTest() {
1895     // Block until all already posted tasks run to avoid 'use after free'
1896     // when such task accesses `this`.
1897     SendTask(task_queue_, [] {});
1898   }
1899 
ModifyVideoConfigs(VideoSendStream::Config * send_config,std::vector<VideoReceiveStreamInterface::Config> * receive_configs,VideoEncoderConfig * encoder_config)1900   void ModifyVideoConfigs(
1901       VideoSendStream::Config* send_config,
1902       std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
1903       VideoEncoderConfig* encoder_config) override {
1904     RTC_DCHECK_RUN_ON(&task_queue_thread_);
1905     RTC_DCHECK_EQ(1, encoder_config->number_of_streams);
1906     if (running_without_padding_) {
1907       encoder_config->min_transmit_bitrate_bps = 0;
1908       encoder_config->content_type =
1909           VideoEncoderConfig::ContentType::kRealtimeVideo;
1910     } else {
1911       encoder_config->min_transmit_bitrate_bps = kMinTransmitBitrateBps;
1912       encoder_config->content_type = VideoEncoderConfig::ContentType::kScreen;
1913     }
1914     send_stream_config_ = send_config->Copy();
1915     encoder_config_ = encoder_config->Copy();
1916   }
1917 
OnCallsCreated(Call * sender_call,Call * receiver_call)1918   void OnCallsCreated(Call* sender_call, Call* receiver_call) override {
1919     RTC_DCHECK_RUN_ON(&task_queue_thread_);
1920     RTC_DCHECK(task_queue_->IsCurrent());
1921     RTC_DCHECK(!call_);
1922     RTC_DCHECK(sender_call);
1923     call_ = sender_call;
1924   }
1925 
1926   // Called on the pacer thread.
OnSendRtp(const uint8_t * packet,size_t length)1927   Action OnSendRtp(const uint8_t* packet, size_t length) override {
1928     RTC_DCHECK_RUN_ON(&module_process_thread_);
1929 
1930     // Check the stats on the correct thread and signal the 'complete' flag
1931     // once we detect that we're done.
1932 
1933     task_queue_->PostTask([this]() {
1934       RTC_DCHECK_RUN_ON(&task_queue_thread_);
1935       // In case we get a callback during teardown.
1936       // When this happens, OnStreamsStopped() has been called already,
1937       // `call_` is null and the streams are being torn down.
1938       if (!call_)
1939         return;
1940 
1941       ++packets_sent_;
1942 
1943       Call::Stats stats = call_->GetStats();
1944       if (running_without_padding_) {
1945         EXPECT_EQ(0, stats.max_padding_bitrate_bps);
1946 
1947         // Wait until at least kMinPacketsToSend frames have been encoded, so
1948         // that we have reliable data.
1949         if (packets_sent_ < kMinPacketsToSend)
1950           return;
1951 
1952         // We've sent kMinPacketsToSend packets with default configuration,
1953         // switch to enabling screen content and setting min transmit bitrate.
1954         // Note that we need to recreate the stream if changing content type.
1955         packets_sent_ = 0;
1956 
1957         encoder_config_.min_transmit_bitrate_bps = kMinTransmitBitrateBps;
1958         encoder_config_.content_type = VideoEncoderConfig::ContentType::kScreen;
1959 
1960         running_without_padding_ = false;
1961         (*stream_resetter_)(send_stream_config_, encoder_config_);
1962       } else {
1963         // Make sure the pacer has been configured with a min transmit bitrate.
1964         if (stats.max_padding_bitrate_bps > 0) {
1965           observation_complete_.Set();
1966         }
1967       }
1968     });
1969 
1970     return SEND_PACKET;
1971   }
1972 
1973   // Called on `task_queue_`
OnStreamsStopped()1974   void OnStreamsStopped() override {
1975     RTC_DCHECK_RUN_ON(&task_queue_thread_);
1976     RTC_DCHECK(task_queue_->IsCurrent());
1977     call_ = nullptr;
1978   }
1979 
PerformTest()1980   void PerformTest() override {
1981     ASSERT_TRUE(Wait()) << "Timed out waiting for a valid padding bitrate.";
1982   }
1983 
1984  private:
1985   webrtc::SequenceChecker task_queue_thread_;
1986   Call* call_ RTC_GUARDED_BY(task_queue_thread_) = nullptr;
1987   VideoSendStream::Config send_stream_config_{nullptr};
1988   VideoEncoderConfig encoder_config_;
1989   webrtc::SequenceChecker module_process_thread_;
1990   uint32_t packets_sent_ RTC_GUARDED_BY(task_queue_thread_) = 0;
1991   bool running_without_padding_ RTC_GUARDED_BY(task_queue_thread_);
1992   T* const stream_resetter_;
1993   TaskQueueBase* const task_queue_;
1994 };
1995 
TEST_F(VideoSendStreamTest,RespectsMinTransmitBitrate)1996 TEST_F(VideoSendStreamTest, RespectsMinTransmitBitrate) {
1997   auto reset_fun = [](const VideoSendStream::Config& send_stream_config,
1998                       const VideoEncoderConfig& encoder_config) {};
1999   MaxPaddingSetTest<decltype(reset_fun)> test(false, &reset_fun, task_queue());
2000   RunBaseTest(&test);
2001 }
2002 
TEST_F(VideoSendStreamTest,RespectsMinTransmitBitrateAfterContentSwitch)2003 TEST_F(VideoSendStreamTest, RespectsMinTransmitBitrateAfterContentSwitch) {
2004   // Function for removing and recreating the send stream with a new config.
2005   auto reset_fun = [this](const VideoSendStream::Config& send_stream_config,
2006                           const VideoEncoderConfig& encoder_config) {
2007     RTC_DCHECK(task_queue()->IsCurrent());
2008     Stop();
2009     DestroyVideoSendStreams();
2010     SetVideoSendConfig(send_stream_config);
2011     SetVideoEncoderConfig(encoder_config);
2012     CreateVideoSendStreams();
2013     SetVideoDegradation(DegradationPreference::MAINTAIN_RESOLUTION);
2014     Start();
2015   };
2016   MaxPaddingSetTest<decltype(reset_fun)> test(true, &reset_fun, task_queue());
2017   RunBaseTest(&test);
2018 }
2019 
2020 // This test verifies that new frame sizes reconfigures encoders even though not
2021 // (yet) sending. The purpose of this is to permit encoding as quickly as
2022 // possible once we start sending. Likely the frames being input are from the
2023 // same source that will be sent later, which just means that we're ready
2024 // earlier.
TEST_F(VideoSendStreamTest,EncoderReconfigureOnResolutionChangeWhenNotSending)2025 TEST_F(VideoSendStreamTest,
2026        EncoderReconfigureOnResolutionChangeWhenNotSending) {
2027   class EncoderObserver : public test::FakeEncoder {
2028    public:
2029     EncoderObserver()
2030         : FakeEncoder(Clock::GetRealTimeClock()),
2031           last_initialized_frame_width_(0),
2032           last_initialized_frame_height_(0) {}
2033 
2034     void WaitForResolution(int width, int height) {
2035       {
2036         MutexLock lock(&mutex_);
2037         if (last_initialized_frame_width_ == width &&
2038             last_initialized_frame_height_ == height) {
2039           return;
2040         }
2041       }
2042       EXPECT_TRUE(
2043           init_encode_called_.Wait(VideoSendStreamTest::kDefaultTimeout));
2044       {
2045         MutexLock lock(&mutex_);
2046         EXPECT_EQ(width, last_initialized_frame_width_);
2047         EXPECT_EQ(height, last_initialized_frame_height_);
2048       }
2049     }
2050 
2051    private:
2052     int32_t InitEncode(const VideoCodec* config,
2053                        const Settings& settings) override {
2054       MutexLock lock(&mutex_);
2055       last_initialized_frame_width_ = config->width;
2056       last_initialized_frame_height_ = config->height;
2057       init_encode_called_.Set();
2058       return FakeEncoder::InitEncode(config, settings);
2059     }
2060 
2061     int32_t Encode(const VideoFrame& input_image,
2062                    const std::vector<VideoFrameType>* frame_types) override {
2063       ADD_FAILURE()
2064           << "Unexpected Encode call since the send stream is not started";
2065       return 0;
2066     }
2067 
2068     Mutex mutex_;
2069     rtc::Event init_encode_called_;
2070     int last_initialized_frame_width_ RTC_GUARDED_BY(&mutex_);
2071     int last_initialized_frame_height_ RTC_GUARDED_BY(&mutex_);
2072   };
2073 
2074   test::NullTransport transport;
2075   EncoderObserver encoder;
2076   test::VideoEncoderProxyFactory encoder_factory(&encoder);
2077 
2078   SendTask(task_queue(), [this, &transport, &encoder_factory]() {
2079     CreateSenderCall();
2080     CreateSendConfig(1, 0, 0, &transport);
2081     GetVideoSendConfig()->encoder_settings.encoder_factory = &encoder_factory;
2082     CreateVideoStreams();
2083     CreateFrameGeneratorCapturer(kDefaultFramerate, kDefaultWidth,
2084                                  kDefaultHeight);
2085     frame_generator_capturer_->Start();
2086   });
2087 
2088   encoder.WaitForResolution(kDefaultWidth, kDefaultHeight);
2089 
2090   SendTask(task_queue(), [this]() {
2091     frame_generator_capturer_->ChangeResolution(kDefaultWidth * 2,
2092                                                 kDefaultHeight * 2);
2093   });
2094 
2095   encoder.WaitForResolution(kDefaultWidth * 2, kDefaultHeight * 2);
2096 
2097   SendTask(task_queue(), [this]() {
2098     DestroyStreams();
2099     DestroyCalls();
2100   });
2101 }
2102 
TEST_F(VideoSendStreamTest,CanReconfigureToUseStartBitrateAbovePreviousMax)2103 TEST_F(VideoSendStreamTest, CanReconfigureToUseStartBitrateAbovePreviousMax) {
2104   class StartBitrateObserver : public test::FakeEncoder {
2105    public:
2106     StartBitrateObserver()
2107         : FakeEncoder(Clock::GetRealTimeClock()), start_bitrate_kbps_(0) {}
2108     int32_t InitEncode(const VideoCodec* config,
2109                        const Settings& settings) override {
2110       MutexLock lock(&mutex_);
2111       start_bitrate_kbps_ = config->startBitrate;
2112       start_bitrate_changed_.Set();
2113       return FakeEncoder::InitEncode(config, settings);
2114     }
2115 
2116     void SetRates(const RateControlParameters& parameters) override {
2117       MutexLock lock(&mutex_);
2118       start_bitrate_kbps_ = parameters.bitrate.get_sum_kbps();
2119       start_bitrate_changed_.Set();
2120       FakeEncoder::SetRates(parameters);
2121     }
2122 
2123     int GetStartBitrateKbps() const {
2124       MutexLock lock(&mutex_);
2125       return start_bitrate_kbps_;
2126     }
2127 
2128     bool WaitForStartBitrate() {
2129       return start_bitrate_changed_.Wait(VideoSendStreamTest::kDefaultTimeout);
2130     }
2131 
2132    private:
2133     mutable Mutex mutex_;
2134     rtc::Event start_bitrate_changed_;
2135     int start_bitrate_kbps_ RTC_GUARDED_BY(mutex_);
2136   };
2137 
2138   CreateSenderCall();
2139 
2140   test::NullTransport transport;
2141   CreateSendConfig(1, 0, 0, &transport);
2142 
2143   BitrateConstraints bitrate_config;
2144   bitrate_config.start_bitrate_bps =
2145       2 * GetVideoEncoderConfig()->max_bitrate_bps;
2146   sender_call_->GetTransportControllerSend()->SetSdpBitrateParameters(
2147       bitrate_config);
2148 
2149   StartBitrateObserver encoder;
2150   test::VideoEncoderProxyFactory encoder_factory(&encoder);
2151   GetVideoSendConfig()->encoder_settings.encoder_factory = &encoder_factory;
2152 
2153   CreateVideoStreams();
2154 
2155   // Start capturing and encoding frames to force encoder reconfiguration.
2156   CreateFrameGeneratorCapturer(kDefaultFramerate, kDefaultWidth,
2157                                kDefaultHeight);
2158   frame_generator_capturer_->Start();
2159   // TODO(crbug/1255737): Added manual current thread message processing because
2160   // the test code context is interpreted as the worker thread and we assume
2161   // progress on it. The test should probably be ported to use simulated time
2162   // instead (ported to a scenario test perhaps?).
2163   rtc::Thread::Current()->ProcessMessages(5000);
2164 
2165   EXPECT_TRUE(encoder.WaitForStartBitrate());
2166   EXPECT_EQ(GetVideoEncoderConfig()->max_bitrate_bps / 1000,
2167             encoder.GetStartBitrateKbps());
2168 
2169   GetVideoEncoderConfig()->max_bitrate_bps =
2170       2 * bitrate_config.start_bitrate_bps;
2171   GetVideoSendStream()->ReconfigureVideoEncoder(
2172       GetVideoEncoderConfig()->Copy());
2173   // TODO(crbug/1255737): Added manual current thread message processing because
2174   // the test code context is interpreted as the worker thread and we assume
2175   // progress on it. The test should probably be ported to use simulated time
2176   // instead (ported to a scenario test perhaps?).
2177   rtc::Thread::Current()->ProcessMessages(5000);
2178 
2179   // New bitrate should be reconfigured above the previous max. As there's no
2180   // network connection this shouldn't be flaky, as no bitrate should've been
2181   // reported in between.
2182   EXPECT_TRUE(encoder.WaitForStartBitrate());
2183   EXPECT_EQ(bitrate_config.start_bitrate_bps / 1000,
2184             encoder.GetStartBitrateKbps());
2185 
2186   DestroyStreams();
2187 }
2188 
2189 class StartStopBitrateObserver : public test::FakeEncoder {
2190  public:
StartStopBitrateObserver()2191   StartStopBitrateObserver() : FakeEncoder(Clock::GetRealTimeClock()) {}
InitEncode(const VideoCodec * config,const Settings & settings)2192   int32_t InitEncode(const VideoCodec* config,
2193                      const Settings& settings) override {
2194     MutexLock lock(&mutex_);
2195     encoder_init_.Set();
2196     return FakeEncoder::InitEncode(config, settings);
2197   }
2198 
SetRates(const RateControlParameters & parameters)2199   void SetRates(const RateControlParameters& parameters) override {
2200     MutexLock lock(&mutex_);
2201     bitrate_kbps_ = parameters.bitrate.get_sum_kbps();
2202     bitrate_changed_.Set();
2203     FakeEncoder::SetRates(parameters);
2204   }
2205 
WaitForEncoderInit()2206   bool WaitForEncoderInit() {
2207     return encoder_init_.Wait(VideoSendStreamTest::kDefaultTimeout);
2208   }
2209 
WaitBitrateChanged(WaitUntil until)2210   bool WaitBitrateChanged(WaitUntil until) {
2211     do {
2212       absl::optional<int> bitrate_kbps;
2213       {
2214         MutexLock lock(&mutex_);
2215         bitrate_kbps = bitrate_kbps_;
2216       }
2217       if (!bitrate_kbps)
2218         continue;
2219 
2220       if ((until == WaitUntil::kNonZero && *bitrate_kbps > 0) ||
2221           (until == WaitUntil::kZero && *bitrate_kbps == 0)) {
2222         return true;
2223       }
2224     } while (bitrate_changed_.Wait(VideoSendStreamTest::kDefaultTimeout));
2225     return false;
2226   }
2227 
2228  private:
2229   Mutex mutex_;
2230   rtc::Event encoder_init_;
2231   rtc::Event bitrate_changed_;
2232   absl::optional<int> bitrate_kbps_ RTC_GUARDED_BY(mutex_);
2233 };
2234 
TEST_F(VideoSendStreamTest,EncoderIsProperlyInitializedAndDestroyed)2235 TEST_F(VideoSendStreamTest, EncoderIsProperlyInitializedAndDestroyed) {
2236   class EncoderStateObserver : public test::SendTest, public VideoEncoder {
2237    public:
2238     explicit EncoderStateObserver(TaskQueueBase* task_queue)
2239         : SendTest(kDefaultTimeout),
2240           task_queue_(task_queue),
2241           stream_(nullptr),
2242           initialized_(false),
2243           callback_registered_(false),
2244           num_releases_(0),
2245           released_(false),
2246           encoder_factory_(this) {}
2247 
2248     bool IsReleased() RTC_LOCKS_EXCLUDED(mutex_) {
2249       MutexLock lock(&mutex_);
2250       return released_;
2251     }
2252 
2253     bool IsReadyForEncode() RTC_LOCKS_EXCLUDED(mutex_) {
2254       MutexLock lock(&mutex_);
2255       return IsReadyForEncodeLocked();
2256     }
2257 
2258     size_t num_releases() RTC_LOCKS_EXCLUDED(mutex_) {
2259       MutexLock lock(&mutex_);
2260       return num_releases_;
2261     }
2262 
2263    private:
2264     bool IsReadyForEncodeLocked() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_) {
2265       return initialized_ && callback_registered_;
2266     }
2267 
2268     void SetFecControllerOverride(
2269         FecControllerOverride* fec_controller_override) override {
2270       // Ignored.
2271     }
2272 
2273     int32_t InitEncode(const VideoCodec* codecSettings,
2274                        const Settings& settings) override
2275         RTC_LOCKS_EXCLUDED(mutex_) {
2276       MutexLock lock(&mutex_);
2277       EXPECT_FALSE(initialized_);
2278       initialized_ = true;
2279       released_ = false;
2280       return 0;
2281     }
2282 
2283     int32_t Encode(const VideoFrame& inputImage,
2284                    const std::vector<VideoFrameType>* frame_types) override {
2285       EXPECT_TRUE(IsReadyForEncode());
2286 
2287       observation_complete_.Set();
2288       return 0;
2289     }
2290 
2291     int32_t RegisterEncodeCompleteCallback(
2292         EncodedImageCallback* callback) override RTC_LOCKS_EXCLUDED(mutex_) {
2293       MutexLock lock(&mutex_);
2294       EXPECT_TRUE(initialized_);
2295       callback_registered_ = true;
2296       return 0;
2297     }
2298 
2299     int32_t Release() override RTC_LOCKS_EXCLUDED(mutex_) {
2300       MutexLock lock(&mutex_);
2301       EXPECT_TRUE(IsReadyForEncodeLocked());
2302       EXPECT_FALSE(released_);
2303       initialized_ = false;
2304       callback_registered_ = false;
2305       released_ = true;
2306       ++num_releases_;
2307       return 0;
2308     }
2309 
2310     void SetRates(const RateControlParameters& parameters) override {
2311       EXPECT_TRUE(IsReadyForEncode());
2312     }
2313 
2314     void OnVideoStreamsCreated(VideoSendStream* send_stream,
2315                                const std::vector<VideoReceiveStreamInterface*>&
2316                                    receive_streams) override {
2317       stream_ = send_stream;
2318     }
2319 
2320     void ModifyVideoConfigs(
2321         VideoSendStream::Config* send_config,
2322         std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
2323         VideoEncoderConfig* encoder_config) override {
2324       send_config->encoder_settings.encoder_factory = &encoder_factory_;
2325       encoder_config_ = encoder_config->Copy();
2326     }
2327 
2328     void PerformTest() override {
2329       EXPECT_TRUE(Wait()) << "Timed out while waiting for Encode.";
2330 
2331       SendTask(task_queue_, [this]() {
2332         EXPECT_EQ(0u, num_releases());
2333         stream_->ReconfigureVideoEncoder(std::move(encoder_config_));
2334         EXPECT_EQ(0u, num_releases());
2335         stream_->Stop();
2336         // Encoder should not be released before destroying the VideoSendStream.
2337         EXPECT_FALSE(IsReleased());
2338         EXPECT_TRUE(IsReadyForEncode());
2339         stream_->Start();
2340       });
2341 
2342       // Sanity check, make sure we still encode frames with this encoder.
2343       EXPECT_TRUE(Wait()) << "Timed out while waiting for Encode.";
2344     }
2345 
2346     TaskQueueBase* const task_queue_;
2347     Mutex mutex_;
2348     VideoSendStream* stream_;
2349     bool initialized_ RTC_GUARDED_BY(mutex_);
2350     bool callback_registered_ RTC_GUARDED_BY(mutex_);
2351     size_t num_releases_ RTC_GUARDED_BY(mutex_);
2352     bool released_ RTC_GUARDED_BY(mutex_);
2353     test::VideoEncoderProxyFactory encoder_factory_;
2354     VideoEncoderConfig encoder_config_;
2355   } test_encoder(task_queue());
2356 
2357   RunBaseTest(&test_encoder);
2358 
2359   EXPECT_TRUE(test_encoder.IsReleased());
2360   EXPECT_EQ(1u, test_encoder.num_releases());
2361 }
2362 
2363 static const size_t kVideoCodecConfigObserverNumberOfTemporalLayers = 3;
2364 template <typename T>
2365 class VideoCodecConfigObserver : public test::SendTest,
2366                                  public test::FakeEncoder {
2367  public:
VideoCodecConfigObserver(VideoCodecType video_codec_type,TaskQueueBase * task_queue)2368   VideoCodecConfigObserver(VideoCodecType video_codec_type,
2369                            TaskQueueBase* task_queue)
2370       : SendTest(VideoSendStreamTest::kDefaultTimeout),
2371         FakeEncoder(Clock::GetRealTimeClock()),
2372         video_codec_type_(video_codec_type),
2373         stream_(nullptr),
2374         encoder_factory_(this),
2375         task_queue_(task_queue) {
2376     InitCodecSpecifics();
2377   }
2378 
2379  private:
ModifyVideoConfigs(VideoSendStream::Config * send_config,std::vector<VideoReceiveStreamInterface::Config> * receive_configs,VideoEncoderConfig * encoder_config)2380   void ModifyVideoConfigs(
2381       VideoSendStream::Config* send_config,
2382       std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
2383       VideoEncoderConfig* encoder_config) override {
2384     send_config->encoder_settings.encoder_factory = &encoder_factory_;
2385     send_config->rtp.payload_name = CodecTypeToPayloadString(video_codec_type_);
2386 
2387     encoder_config->codec_type = video_codec_type_;
2388     encoder_config->encoder_specific_settings = GetEncoderSpecificSettings();
2389     EXPECT_EQ(1u, encoder_config->simulcast_layers.size());
2390     encoder_config->simulcast_layers[0].num_temporal_layers =
2391         kVideoCodecConfigObserverNumberOfTemporalLayers;
2392     encoder_config_ = encoder_config->Copy();
2393   }
2394 
OnVideoStreamsCreated(VideoSendStream * send_stream,const std::vector<VideoReceiveStreamInterface * > & receive_streams)2395   void OnVideoStreamsCreated(VideoSendStream* send_stream,
2396                              const std::vector<VideoReceiveStreamInterface*>&
2397                                  receive_streams) override {
2398     stream_ = send_stream;
2399   }
2400 
InitEncode(const VideoCodec * config,const Settings & settings)2401   int32_t InitEncode(const VideoCodec* config,
2402                      const Settings& settings) override {
2403     EXPECT_EQ(video_codec_type_, config->codecType);
2404     VerifyCodecSpecifics(*config);
2405     int ret = FakeEncoder::InitEncode(config, settings);
2406     init_encode_event_.Set();
2407     return ret;
2408   }
2409 
2410   void InitCodecSpecifics();
2411   void VerifyCodecSpecifics(const VideoCodec& config) const;
2412   rtc::scoped_refptr<VideoEncoderConfig::EncoderSpecificSettings>
2413   GetEncoderSpecificSettings() const;
2414 
PerformTest()2415   void PerformTest() override {
2416     EXPECT_TRUE(init_encode_event_.Wait(VideoSendStreamTest::kDefaultTimeout));
2417     ASSERT_EQ(1, FakeEncoder::GetNumInitializations())
2418         << "VideoEncoder not initialized.";
2419 
2420     // Change encoder settings to actually trigger reconfiguration.
2421     encoder_config_.frame_drop_enabled = !encoder_config_.frame_drop_enabled;
2422     encoder_config_.encoder_specific_settings = GetEncoderSpecificSettings();
2423     SendTask(task_queue_, [&]() {
2424       stream_->ReconfigureVideoEncoder(std::move(encoder_config_));
2425     });
2426     ASSERT_TRUE(init_encode_event_.Wait(VideoSendStreamTest::kDefaultTimeout));
2427     EXPECT_EQ(2, FakeEncoder::GetNumInitializations())
2428         << "ReconfigureVideoEncoder did not reinitialize the encoder with "
2429            "new encoder settings.";
2430   }
2431 
Encode(const VideoFrame & input_image,const std::vector<VideoFrameType> * frame_types)2432   int32_t Encode(const VideoFrame& input_image,
2433                  const std::vector<VideoFrameType>* frame_types) override {
2434     // Silently skip the encode, FakeEncoder::Encode doesn't produce VP8.
2435     return 0;
2436   }
2437 
2438   T encoder_settings_;
2439   const VideoCodecType video_codec_type_;
2440   rtc::Event init_encode_event_;
2441   VideoSendStream* stream_;
2442   test::VideoEncoderProxyFactory encoder_factory_;
2443   VideoEncoderConfig encoder_config_;
2444   TaskQueueBase* task_queue_;
2445 };
2446 
2447 template <>
InitCodecSpecifics()2448 void VideoCodecConfigObserver<VideoCodecH264>::InitCodecSpecifics() {}
2449 
2450 template <>
VerifyCodecSpecifics(const VideoCodec & config) const2451 void VideoCodecConfigObserver<VideoCodecH264>::VerifyCodecSpecifics(
2452     const VideoCodec& config) const {
2453   // Check that the number of temporal layers has propagated properly to
2454   // VideoCodec.
2455   EXPECT_EQ(kVideoCodecConfigObserverNumberOfTemporalLayers,
2456             config.H264().numberOfTemporalLayers);
2457 
2458   for (unsigned char i = 0; i < config.numberOfSimulcastStreams; ++i) {
2459     EXPECT_EQ(kVideoCodecConfigObserverNumberOfTemporalLayers,
2460               config.simulcastStream[i].numberOfTemporalLayers);
2461   }
2462 
2463   // Set expected temporal layers as they should have been set when
2464   // reconfiguring the encoder and not match the set config.
2465   VideoCodecH264 encoder_settings = VideoEncoder::GetDefaultH264Settings();
2466   encoder_settings.numberOfTemporalLayers =
2467       kVideoCodecConfigObserverNumberOfTemporalLayers;
2468   EXPECT_EQ(config.H264(), encoder_settings);
2469 }
2470 
2471 template <>
2472 rtc::scoped_refptr<VideoEncoderConfig::EncoderSpecificSettings>
GetEncoderSpecificSettings() const2473 VideoCodecConfigObserver<VideoCodecH264>::GetEncoderSpecificSettings() const {
2474   return nullptr;
2475 }
2476 
2477 template <>
InitCodecSpecifics()2478 void VideoCodecConfigObserver<VideoCodecVP8>::InitCodecSpecifics() {
2479   encoder_settings_ = VideoEncoder::GetDefaultVp8Settings();
2480 }
2481 
2482 template <>
VerifyCodecSpecifics(const VideoCodec & config) const2483 void VideoCodecConfigObserver<VideoCodecVP8>::VerifyCodecSpecifics(
2484     const VideoCodec& config) const {
2485   // Check that the number of temporal layers has propagated properly to
2486   // VideoCodec.
2487   EXPECT_EQ(kVideoCodecConfigObserverNumberOfTemporalLayers,
2488             config.VP8().numberOfTemporalLayers);
2489 
2490   for (unsigned char i = 0; i < config.numberOfSimulcastStreams; ++i) {
2491     EXPECT_EQ(kVideoCodecConfigObserverNumberOfTemporalLayers,
2492               config.simulcastStream[i].numberOfTemporalLayers);
2493   }
2494 
2495   // Set expected temporal layers as they should have been set when
2496   // reconfiguring the encoder and not match the set config.
2497   VideoCodecVP8 encoder_settings = encoder_settings_;
2498   encoder_settings.numberOfTemporalLayers =
2499       kVideoCodecConfigObserverNumberOfTemporalLayers;
2500   EXPECT_EQ(
2501       0, memcmp(&config.VP8(), &encoder_settings, sizeof(encoder_settings_)));
2502 }
2503 
2504 template <>
2505 rtc::scoped_refptr<VideoEncoderConfig::EncoderSpecificSettings>
GetEncoderSpecificSettings() const2506 VideoCodecConfigObserver<VideoCodecVP8>::GetEncoderSpecificSettings() const {
2507   return rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
2508       encoder_settings_);
2509 }
2510 
2511 template <>
InitCodecSpecifics()2512 void VideoCodecConfigObserver<VideoCodecVP9>::InitCodecSpecifics() {
2513   encoder_settings_ = VideoEncoder::GetDefaultVp9Settings();
2514 }
2515 
2516 template <>
VerifyCodecSpecifics(const VideoCodec & config) const2517 void VideoCodecConfigObserver<VideoCodecVP9>::VerifyCodecSpecifics(
2518     const VideoCodec& config) const {
2519   // Check that the number of temporal layers has propagated properly to
2520   // VideoCodec.
2521   EXPECT_EQ(kVideoCodecConfigObserverNumberOfTemporalLayers,
2522             config.VP9().numberOfTemporalLayers);
2523 
2524   for (unsigned char i = 0; i < config.numberOfSimulcastStreams; ++i) {
2525     EXPECT_EQ(kVideoCodecConfigObserverNumberOfTemporalLayers,
2526               config.simulcastStream[i].numberOfTemporalLayers);
2527   }
2528 
2529   // Set expected temporal layers as they should have been set when
2530   // reconfiguring the encoder and not match the set config.
2531   VideoCodecVP9 encoder_settings = encoder_settings_;
2532   encoder_settings.numberOfTemporalLayers =
2533       kVideoCodecConfigObserverNumberOfTemporalLayers;
2534   EXPECT_EQ(
2535       0, memcmp(&(config.VP9()), &encoder_settings, sizeof(encoder_settings_)));
2536 }
2537 
2538 template <>
2539 rtc::scoped_refptr<VideoEncoderConfig::EncoderSpecificSettings>
GetEncoderSpecificSettings() const2540 VideoCodecConfigObserver<VideoCodecVP9>::GetEncoderSpecificSettings() const {
2541   return rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
2542       encoder_settings_);
2543 }
2544 
TEST_F(VideoSendStreamTest,EncoderSetupPropagatesVp8Config)2545 TEST_F(VideoSendStreamTest, EncoderSetupPropagatesVp8Config) {
2546   VideoCodecConfigObserver<VideoCodecVP8> test(kVideoCodecVP8, task_queue());
2547   RunBaseTest(&test);
2548 }
2549 
TEST_F(VideoSendStreamTest,EncoderSetupPropagatesVp9Config)2550 TEST_F(VideoSendStreamTest, EncoderSetupPropagatesVp9Config) {
2551   VideoCodecConfigObserver<VideoCodecVP9> test(kVideoCodecVP9, task_queue());
2552   RunBaseTest(&test);
2553 }
2554 
2555 // Fails on MSAN: https://bugs.chromium.org/p/webrtc/issues/detail?id=11376.
2556 #if defined(MEMORY_SANITIZER)
2557 #define MAYBE_EncoderSetupPropagatesH264Config \
2558   DISABLED_EncoderSetupPropagatesH264Config
2559 #else
2560 #define MAYBE_EncoderSetupPropagatesH264Config EncoderSetupPropagatesH264Config
2561 #endif
TEST_F(VideoSendStreamTest,MAYBE_EncoderSetupPropagatesH264Config)2562 TEST_F(VideoSendStreamTest, MAYBE_EncoderSetupPropagatesH264Config) {
2563   VideoCodecConfigObserver<VideoCodecH264> test(kVideoCodecH264, task_queue());
2564   RunBaseTest(&test);
2565 }
2566 
TEST_F(VideoSendStreamTest,RtcpSenderReportContainsMediaBytesSent)2567 TEST_F(VideoSendStreamTest, RtcpSenderReportContainsMediaBytesSent) {
2568   class RtcpSenderReportTest : public test::SendTest {
2569    public:
2570     RtcpSenderReportTest()
2571         : SendTest(kDefaultTimeout),
2572           rtp_packets_sent_(0),
2573           media_bytes_sent_(0) {}
2574 
2575    private:
2576     Action OnSendRtp(const uint8_t* packet, size_t length) override {
2577       MutexLock lock(&mutex_);
2578       RtpPacket rtp_packet;
2579       EXPECT_TRUE(rtp_packet.Parse(packet, length));
2580       ++rtp_packets_sent_;
2581       media_bytes_sent_ += rtp_packet.payload_size();
2582       return SEND_PACKET;
2583     }
2584 
2585     Action OnSendRtcp(const uint8_t* packet, size_t length) override {
2586       MutexLock lock(&mutex_);
2587       test::RtcpPacketParser parser;
2588       EXPECT_TRUE(parser.Parse(packet, length));
2589 
2590       if (parser.sender_report()->num_packets() > 0) {
2591         // Only compare sent media bytes if SenderPacketCount matches the
2592         // number of sent rtp packets (a new rtp packet could be sent before
2593         // the rtcp packet).
2594         if (parser.sender_report()->sender_octet_count() > 0 &&
2595             parser.sender_report()->sender_packet_count() ==
2596                 rtp_packets_sent_) {
2597           EXPECT_EQ(media_bytes_sent_,
2598                     parser.sender_report()->sender_octet_count());
2599           observation_complete_.Set();
2600         }
2601       }
2602 
2603       return SEND_PACKET;
2604     }
2605 
2606     void PerformTest() override {
2607       EXPECT_TRUE(Wait()) << "Timed out while waiting for RTCP sender report.";
2608     }
2609 
2610     Mutex mutex_;
2611     size_t rtp_packets_sent_ RTC_GUARDED_BY(&mutex_);
2612     size_t media_bytes_sent_ RTC_GUARDED_BY(&mutex_);
2613   } test;
2614 
2615   RunBaseTest(&test);
2616 }
2617 
TEST_F(VideoSendStreamTest,TranslatesTwoLayerScreencastToTargetBitrate)2618 TEST_F(VideoSendStreamTest, TranslatesTwoLayerScreencastToTargetBitrate) {
2619   static const int kScreencastMaxTargetBitrateDeltaKbps = 1;
2620 
2621   class VideoStreamFactory
2622       : public VideoEncoderConfig::VideoStreamFactoryInterface {
2623    public:
2624     VideoStreamFactory() {}
2625 
2626    private:
2627     std::vector<VideoStream> CreateEncoderStreams(
2628         int frame_width,
2629         int frame_height,
2630         const VideoEncoderConfig& encoder_config) override {
2631       std::vector<VideoStream> streams =
2632           test::CreateVideoStreams(frame_width, frame_height, encoder_config);
2633       RTC_CHECK_GT(streams[0].max_bitrate_bps,
2634                    kScreencastMaxTargetBitrateDeltaKbps);
2635       streams[0].target_bitrate_bps =
2636           streams[0].max_bitrate_bps -
2637           kScreencastMaxTargetBitrateDeltaKbps * 1000;
2638       return streams;
2639     }
2640   };
2641 
2642   class ScreencastTargetBitrateTest : public test::SendTest,
2643                                       public test::FakeEncoder {
2644    public:
2645     ScreencastTargetBitrateTest()
2646         : SendTest(kDefaultTimeout),
2647           test::FakeEncoder(Clock::GetRealTimeClock()),
2648           encoder_factory_(this) {}
2649 
2650    private:
2651     int32_t InitEncode(const VideoCodec* config,
2652                        const Settings& settings) override {
2653       EXPECT_EQ(config->numberOfSimulcastStreams, 1);
2654       EXPECT_EQ(static_cast<unsigned int>(kScreencastMaxTargetBitrateDeltaKbps),
2655                 config->simulcastStream[0].maxBitrate -
2656                     config->simulcastStream[0].targetBitrate);
2657       observation_complete_.Set();
2658       return test::FakeEncoder::InitEncode(config, settings);
2659     }
2660 
2661     void ModifyVideoConfigs(
2662         VideoSendStream::Config* send_config,
2663         std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
2664         VideoEncoderConfig* encoder_config) override {
2665       send_config->encoder_settings.encoder_factory = &encoder_factory_;
2666       EXPECT_EQ(1u, encoder_config->number_of_streams);
2667       encoder_config->video_stream_factory =
2668           rtc::make_ref_counted<VideoStreamFactory>();
2669       EXPECT_EQ(1u, encoder_config->simulcast_layers.size());
2670       encoder_config->simulcast_layers[0].num_temporal_layers = 2;
2671       encoder_config->content_type = VideoEncoderConfig::ContentType::kScreen;
2672     }
2673 
2674     void PerformTest() override {
2675       EXPECT_TRUE(Wait())
2676           << "Timed out while waiting for the encoder to be initialized.";
2677     }
2678     test::VideoEncoderProxyFactory encoder_factory_;
2679   } test;
2680 
2681   RunBaseTest(&test);
2682 }
2683 
TEST_F(VideoSendStreamTest,ReconfigureBitratesSetsEncoderBitratesCorrectly)2684 TEST_F(VideoSendStreamTest, ReconfigureBitratesSetsEncoderBitratesCorrectly) {
2685   // These are chosen to be "kind of odd" to not be accidentally checked against
2686   // default values.
2687   static const int kMinBitrateKbps = 137;
2688   static const int kStartBitrateKbps = 345;
2689   static const int kLowerMaxBitrateKbps = 312;
2690   static const int kMaxBitrateKbps = 413;
2691   static const int kIncreasedStartBitrateKbps = 451;
2692   static const int kIncreasedMaxBitrateKbps = 597;
2693   // TODO(bugs.webrtc.org/12058): If these fields trial are on, we get lower
2694   // bitrates than expected by this test, due to encoder pushback and subtracted
2695   // overhead.
2696   webrtc::test::ScopedKeyValueConfig field_trials(
2697       field_trials_, "WebRTC-VideoRateControl/bitrate_adjuster:false/");
2698 
2699   class EncoderBitrateThresholdObserver : public test::SendTest,
2700                                           public VideoBitrateAllocatorFactory,
2701                                           public test::FakeEncoder {
2702    public:
2703     explicit EncoderBitrateThresholdObserver(TaskQueueBase* task_queue)
2704         : SendTest(kDefaultTimeout),
2705           FakeEncoder(Clock::GetRealTimeClock()),
2706           task_queue_(task_queue),
2707           target_bitrate_(0),
2708           num_rate_allocator_creations_(0),
2709           num_encoder_initializations_(0),
2710           call_(nullptr),
2711           send_stream_(nullptr),
2712           encoder_factory_(this),
2713           bitrate_allocator_factory_(
2714               CreateBuiltinVideoBitrateAllocatorFactory()) {}
2715 
2716    private:
2717     std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
2718         const VideoCodec& codec) override {
2719       EXPECT_GE(codec.startBitrate, codec.minBitrate);
2720       EXPECT_LE(codec.startBitrate, codec.maxBitrate);
2721       if (num_rate_allocator_creations_ == 0) {
2722         EXPECT_EQ(static_cast<unsigned int>(kMinBitrateKbps), codec.minBitrate);
2723         EXPECT_NEAR(static_cast<unsigned int>(kStartBitrateKbps),
2724                     codec.startBitrate, 10);
2725         EXPECT_EQ(static_cast<unsigned int>(kMaxBitrateKbps), codec.maxBitrate);
2726       } else if (num_rate_allocator_creations_ == 1) {
2727         EXPECT_EQ(static_cast<unsigned int>(kLowerMaxBitrateKbps),
2728                   codec.maxBitrate);
2729         // The start bitrate should be kept (-1) and capped to the max bitrate.
2730         // Since this is not an end-to-end call no receiver should have been
2731         // returning a REMB that could lower this estimate.
2732         EXPECT_EQ(codec.startBitrate, codec.maxBitrate);
2733       } else if (num_rate_allocator_creations_ == 2) {
2734         EXPECT_EQ(static_cast<unsigned int>(kIncreasedMaxBitrateKbps),
2735                   codec.maxBitrate);
2736         // The start bitrate will be whatever the rate BitRateController has
2737         // currently configured but in the span of the set max and min bitrate.
2738       }
2739       ++num_rate_allocator_creations_;
2740       create_rate_allocator_event_.Set();
2741 
2742       return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
2743     }
2744 
2745     int32_t InitEncode(const VideoCodec* codecSettings,
2746                        const Settings& settings) override {
2747       EXPECT_EQ(0, num_encoder_initializations_);
2748       EXPECT_EQ(static_cast<unsigned int>(kMinBitrateKbps),
2749                 codecSettings->minBitrate);
2750       EXPECT_NEAR(static_cast<unsigned int>(kStartBitrateKbps),
2751                   codecSettings->startBitrate, 10);
2752       EXPECT_EQ(static_cast<unsigned int>(kMaxBitrateKbps),
2753                 codecSettings->maxBitrate);
2754 
2755       ++num_encoder_initializations_;
2756 
2757       observation_complete_.Set();
2758       init_encode_event_.Set();
2759 
2760       return FakeEncoder::InitEncode(codecSettings, settings);
2761     }
2762 
2763     void SetRates(const RateControlParameters& parameters) override {
2764       {
2765         MutexLock lock(&mutex_);
2766         if (target_bitrate_ == parameters.bitrate.get_sum_kbps()) {
2767           FakeEncoder::SetRates(parameters);
2768           return;
2769         }
2770         target_bitrate_ = parameters.bitrate.get_sum_kbps();
2771       }
2772       bitrate_changed_event_.Set();
2773       FakeEncoder::SetRates(parameters);
2774     }
2775 
2776     void WaitForSetRates(uint32_t expected_bitrate, int abs_error) {
2777       // Wait for the expected rate to be set. In some cases there can be
2778       // more than one update pending, in which case we keep waiting
2779       // until the correct value has been observed.
2780       // The target_bitrate_ is reduced by the calculated packet overhead.
2781       const int64_t start_time = rtc::TimeMillis();
2782       do {
2783         MutexLock lock(&mutex_);
2784 
2785         int error = target_bitrate_ - expected_bitrate;
2786         if ((error < 0 && error >= -abs_error) ||
2787             (error >= 0 && error <= abs_error)) {
2788           return;
2789         }
2790       } while (bitrate_changed_event_.Wait(
2791           std::max(TimeDelta::Millis(1),
2792                    VideoSendStreamTest::kDefaultTimeout -
2793                        TimeDelta::Millis(rtc::TimeMillis() - start_time))));
2794       MutexLock lock(&mutex_);
2795       EXPECT_NEAR(target_bitrate_, expected_bitrate, abs_error)
2796           << "Timed out while waiting encoder rate to be set.";
2797     }
2798 
2799     void ModifySenderBitrateConfig(
2800         BitrateConstraints* bitrate_config) override {
2801       bitrate_config->min_bitrate_bps = kMinBitrateKbps * 1000;
2802       bitrate_config->start_bitrate_bps = kStartBitrateKbps * 1000;
2803       bitrate_config->max_bitrate_bps = kMaxBitrateKbps * 1000;
2804     }
2805 
2806     void ModifyVideoConfigs(
2807         VideoSendStream::Config* send_config,
2808         std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
2809         VideoEncoderConfig* encoder_config) override {
2810       send_config->encoder_settings.encoder_factory = &encoder_factory_;
2811       send_config->encoder_settings.bitrate_allocator_factory = this;
2812       // Set bitrates lower/higher than min/max to make sure they are properly
2813       // capped.
2814       encoder_config->max_bitrate_bps = kMaxBitrateKbps * 1000;
2815       EXPECT_EQ(1u, encoder_config->simulcast_layers.size());
2816       encoder_config->simulcast_layers[0].min_bitrate_bps =
2817           kMinBitrateKbps * 1000;
2818       encoder_config_ = encoder_config->Copy();
2819     }
2820 
2821     void OnCallsCreated(Call* sender_call, Call* receiver_call) override {
2822       call_ = sender_call;
2823     }
2824 
2825     void OnVideoStreamsCreated(VideoSendStream* send_stream,
2826                                const std::vector<VideoReceiveStreamInterface*>&
2827                                    receive_streams) override {
2828       send_stream_ = send_stream;
2829     }
2830 
2831     void PerformTest() override {
2832       ASSERT_TRUE(create_rate_allocator_event_.Wait(
2833           VideoSendStreamTest::kDefaultTimeout))
2834           << "Timed out while waiting for rate allocator to be created.";
2835       ASSERT_TRUE(init_encode_event_.Wait(VideoSendStreamTest::kDefaultTimeout))
2836           << "Timed out while waiting for encoder to be configured.";
2837       WaitForSetRates(kStartBitrateKbps, 80);
2838       BitrateConstraints bitrate_config;
2839       bitrate_config.start_bitrate_bps = kIncreasedStartBitrateKbps * 1000;
2840       bitrate_config.max_bitrate_bps = kIncreasedMaxBitrateKbps * 1000;
2841       SendTask(task_queue_, [this, &bitrate_config]() {
2842         call_->GetTransportControllerSend()->SetSdpBitrateParameters(
2843             bitrate_config);
2844       });
2845       // Encoder rate is capped by EncoderConfig max_bitrate_bps.
2846       WaitForSetRates(kMaxBitrateKbps, 10);
2847       encoder_config_.max_bitrate_bps = kLowerMaxBitrateKbps * 1000;
2848       SendTask(task_queue_, [&]() {
2849         send_stream_->ReconfigureVideoEncoder(encoder_config_.Copy());
2850       });
2851       ASSERT_TRUE(create_rate_allocator_event_.Wait(
2852           VideoSendStreamTest::kDefaultTimeout));
2853       EXPECT_EQ(2, num_rate_allocator_creations_)
2854           << "Rate allocator should have been recreated.";
2855 
2856       WaitForSetRates(kLowerMaxBitrateKbps, 10);
2857       EXPECT_EQ(1, num_encoder_initializations_);
2858 
2859       encoder_config_.max_bitrate_bps = kIncreasedMaxBitrateKbps * 1000;
2860       SendTask(task_queue_, [&]() {
2861         send_stream_->ReconfigureVideoEncoder(encoder_config_.Copy());
2862       });
2863       ASSERT_TRUE(create_rate_allocator_event_.Wait(
2864           VideoSendStreamTest::kDefaultTimeout));
2865       EXPECT_EQ(3, num_rate_allocator_creations_)
2866           << "Rate allocator should have been recreated.";
2867 
2868       // Expected target bitrate is the start bitrate set in the call to
2869       // call_->GetTransportControllerSend()->SetSdpBitrateParameters.
2870       WaitForSetRates(kIncreasedStartBitrateKbps, 10);
2871       EXPECT_EQ(1, num_encoder_initializations_);
2872     }
2873 
2874     TaskQueueBase* const task_queue_;
2875     rtc::Event create_rate_allocator_event_;
2876     rtc::Event init_encode_event_;
2877     rtc::Event bitrate_changed_event_;
2878     Mutex mutex_;
2879     uint32_t target_bitrate_ RTC_GUARDED_BY(&mutex_);
2880 
2881     int num_rate_allocator_creations_;
2882     int num_encoder_initializations_;
2883     webrtc::Call* call_;
2884     webrtc::VideoSendStream* send_stream_;
2885     test::VideoEncoderProxyFactory encoder_factory_;
2886     std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
2887     webrtc::VideoEncoderConfig encoder_config_;
2888   } test(task_queue());
2889 
2890   RunBaseTest(&test);
2891 }
2892 
TEST_F(VideoSendStreamTest,ReportsSentResolution)2893 TEST_F(VideoSendStreamTest, ReportsSentResolution) {
2894   static const size_t kNumStreams = 3;
2895   // Unusual resolutions to make sure that they are the ones being reported.
2896   static const struct {
2897     int width;
2898     int height;
2899   } kEncodedResolution[kNumStreams] = {{241, 181}, {300, 121}, {121, 221}};
2900   class ScreencastTargetBitrateTest : public test::SendTest,
2901                                       public test::FakeEncoder {
2902    public:
2903     explicit ScreencastTargetBitrateTest(TaskQueueBase* task_queue)
2904         : SendTest(kDefaultTimeout),
2905           test::FakeEncoder(Clock::GetRealTimeClock()),
2906           send_stream_(nullptr),
2907           encoder_factory_(this),
2908           task_queue_(task_queue) {}
2909 
2910    private:
2911     int32_t Encode(const VideoFrame& input_image,
2912                    const std::vector<VideoFrameType>* frame_types) override {
2913       CodecSpecificInfo specifics;
2914       specifics.codecType = kVideoCodecGeneric;
2915 
2916       EncodedImage encoded;
2917       auto buffer = EncodedImageBuffer::Create(16);
2918       memset(buffer->data(), 0, 16);
2919       encoded.SetEncodedData(buffer);
2920       encoded.SetTimestamp(input_image.timestamp());
2921       encoded.capture_time_ms_ = input_image.render_time_ms();
2922 
2923       for (size_t i = 0; i < kNumStreams; ++i) {
2924         encoded._frameType = (*frame_types)[i];
2925         encoded._encodedWidth = kEncodedResolution[i].width;
2926         encoded._encodedHeight = kEncodedResolution[i].height;
2927         encoded.SetSpatialIndex(i);
2928         EncodedImageCallback* callback;
2929         {
2930           MutexLock lock(&mutex_);
2931           callback = callback_;
2932         }
2933         RTC_DCHECK(callback);
2934         if (callback->OnEncodedImage(encoded, &specifics).error !=
2935             EncodedImageCallback::Result::OK) {
2936           return -1;
2937         }
2938       }
2939 
2940       observation_complete_.Set();
2941       return 0;
2942     }
2943     void ModifyVideoConfigs(
2944         VideoSendStream::Config* send_config,
2945         std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
2946         VideoEncoderConfig* encoder_config) override {
2947       send_config->encoder_settings.encoder_factory = &encoder_factory_;
2948       EXPECT_EQ(kNumStreams, encoder_config->number_of_streams);
2949     }
2950 
2951     size_t GetNumVideoStreams() const override { return kNumStreams; }
2952 
2953     void PerformTest() override {
2954       EXPECT_TRUE(Wait())
2955           << "Timed out while waiting for the encoder to send one frame.";
2956       VideoSendStream::Stats stats;
2957       SendTask(task_queue_, [&]() { stats = send_stream_->GetStats(); });
2958 
2959       for (size_t i = 0; i < kNumStreams; ++i) {
2960         ASSERT_TRUE(stats.substreams.find(kVideoSendSsrcs[i]) !=
2961                     stats.substreams.end())
2962             << "No stats for SSRC: " << kVideoSendSsrcs[i]
2963             << ", stats should exist as soon as frames have been encoded.";
2964         VideoSendStream::StreamStats ssrc_stats =
2965             stats.substreams[kVideoSendSsrcs[i]];
2966         EXPECT_EQ(kEncodedResolution[i].width, ssrc_stats.width);
2967         EXPECT_EQ(kEncodedResolution[i].height, ssrc_stats.height);
2968       }
2969     }
2970 
2971     void OnVideoStreamsCreated(VideoSendStream* send_stream,
2972                                const std::vector<VideoReceiveStreamInterface*>&
2973                                    receive_streams) override {
2974       send_stream_ = send_stream;
2975     }
2976 
2977     VideoSendStream* send_stream_;
2978     test::VideoEncoderProxyFactory encoder_factory_;
2979     TaskQueueBase* const task_queue_;
2980   } test(task_queue());
2981 
2982   RunBaseTest(&test);
2983 }
2984 
2985 #if defined(RTC_ENABLE_VP9)
2986 class Vp9HeaderObserver : public test::SendTest {
2987  public:
Vp9HeaderObserver(const Vp9TestParams & params)2988   explicit Vp9HeaderObserver(const Vp9TestParams& params)
2989       : SendTest(VideoSendStreamTest::kLongTimeout),
2990         encoder_factory_([]() { return VP9Encoder::Create(); }),
2991         params_(params),
2992         vp9_settings_(VideoEncoder::GetDefaultVp9Settings()) {}
2993 
ModifyVideoConfigsHook(VideoSendStream::Config * send_config,std::vector<VideoReceiveStreamInterface::Config> * receive_configs,VideoEncoderConfig * encoder_config)2994   virtual void ModifyVideoConfigsHook(
2995       VideoSendStream::Config* send_config,
2996       std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
2997       VideoEncoderConfig* encoder_config) {}
2998 
2999   virtual void InspectHeader(const RTPVideoHeaderVP9& vp9) = 0;
3000 
3001  private:
3002   const int kVp9PayloadType = test::CallTest::kVideoSendPayloadType;
3003 
ModifyVideoConfigs(VideoSendStream::Config * send_config,std::vector<VideoReceiveStreamInterface::Config> * receive_configs,VideoEncoderConfig * encoder_config)3004   void ModifyVideoConfigs(
3005       VideoSendStream::Config* send_config,
3006       std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
3007       VideoEncoderConfig* encoder_config) override {
3008     send_config->encoder_settings.encoder_factory = &encoder_factory_;
3009     send_config->rtp.payload_name = "VP9";
3010     send_config->rtp.payload_type = kVp9PayloadType;
3011     ModifyVideoConfigsHook(send_config, receive_configs, encoder_config);
3012     encoder_config->encoder_specific_settings =
3013         rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
3014             vp9_settings_);
3015     EXPECT_EQ(1u, encoder_config->number_of_streams);
3016     EXPECT_EQ(1u, encoder_config->simulcast_layers.size());
3017     encoder_config_ = encoder_config->Copy();
3018   }
3019 
ModifyVideoCaptureStartResolution(int * width,int * height,int * frame_rate)3020   void ModifyVideoCaptureStartResolution(int* width,
3021                                          int* height,
3022                                          int* frame_rate) override {
3023     expected_width_ = *width;
3024     expected_height_ = *height;
3025   }
3026 
PerformTest()3027   void PerformTest() override {
3028     bool wait = Wait();
3029     {
3030       // In case of time out, OnSendRtp might still access frames_sent_;
3031       MutexLock lock(&mutex_);
3032       EXPECT_TRUE(wait) << "Test timed out waiting for VP9 packet, num frames "
3033                         << frames_sent_;
3034     }
3035   }
3036 
OnSendRtp(const uint8_t * packet,size_t length)3037   Action OnSendRtp(const uint8_t* packet, size_t length) override {
3038     RtpPacket rtp_packet;
3039     EXPECT_TRUE(rtp_packet.Parse(packet, length));
3040 
3041     EXPECT_EQ(kVp9PayloadType, rtp_packet.PayloadType());
3042     rtc::ArrayView<const uint8_t> rtp_payload = rtp_packet.payload();
3043 
3044     bool new_packet = !last_packet_sequence_number_.has_value() ||
3045                       IsNewerSequenceNumber(rtp_packet.SequenceNumber(),
3046                                             *last_packet_sequence_number_);
3047     if (!rtp_payload.empty() && new_packet) {
3048       RTPVideoHeader video_header;
3049       EXPECT_NE(
3050           VideoRtpDepacketizerVp9::ParseRtpPayload(rtp_payload, &video_header),
3051           0);
3052       EXPECT_EQ(VideoCodecType::kVideoCodecVP9, video_header.codec);
3053       // Verify common fields for all configurations.
3054       const auto& vp9_header =
3055           absl::get<RTPVideoHeaderVP9>(video_header.video_type_header);
3056       VerifyCommonHeader(vp9_header);
3057       CompareConsecutiveFrames(rtp_packet, video_header);
3058       // Verify configuration specific settings.
3059       InspectHeader(vp9_header);
3060 
3061       if (rtp_packet.Marker()) {
3062         MutexLock lock(&mutex_);
3063         ++frames_sent_;
3064       }
3065       last_packet_marker_ = rtp_packet.Marker();
3066       last_packet_sequence_number_ = rtp_packet.SequenceNumber();
3067       last_packet_timestamp_ = rtp_packet.Timestamp();
3068       last_vp9_ = vp9_header;
3069       last_temporal_idx_by_spatial_idx_[vp9_header.spatial_idx] =
3070           vp9_header.temporal_idx;
3071     }
3072     return SEND_PACKET;
3073   }
3074 
3075  protected:
ContinuousPictureId(const RTPVideoHeaderVP9 & vp9) const3076   bool ContinuousPictureId(const RTPVideoHeaderVP9& vp9) const {
3077     if (last_vp9_.picture_id > vp9.picture_id) {
3078       return vp9.picture_id == 0;  // Wrap.
3079     } else {
3080       return vp9.picture_id == last_vp9_.picture_id + 1;
3081     }
3082   }
3083 
IsTemporalShiftEnabled() const3084   bool IsTemporalShiftEnabled() const {
3085     return params_.scalability_mode.find("_SHIFT") != std::string::npos;
3086   }
3087 
VerifySpatialIdxWithinFrame(const RTPVideoHeaderVP9 & vp9) const3088   void VerifySpatialIdxWithinFrame(const RTPVideoHeaderVP9& vp9) const {
3089     bool new_layer = vp9.spatial_idx != last_vp9_.spatial_idx;
3090     EXPECT_EQ(new_layer, vp9.beginning_of_frame);
3091     EXPECT_EQ(new_layer, last_vp9_.end_of_frame);
3092     EXPECT_EQ(new_layer ? last_vp9_.spatial_idx + 1 : last_vp9_.spatial_idx,
3093               vp9.spatial_idx);
3094   }
3095 
VerifyTemporalIdxWithinFrame(const RTPVideoHeaderVP9 & vp9) const3096   void VerifyTemporalIdxWithinFrame(const RTPVideoHeaderVP9& vp9) const {
3097     if (!IsTemporalShiftEnabled()) {
3098       EXPECT_EQ(vp9.temporal_idx, last_vp9_.temporal_idx);
3099       return;
3100     }
3101     // Temporal shift.
3102     EXPECT_EQ(params_.num_temporal_layers, 2);
3103     if (vp9.spatial_idx == params_.num_spatial_layers - 1) {
3104       // Lower spatial layers should be shifted.
3105       int expected_tid =
3106           (!vp9.inter_pic_predicted || vp9.temporal_idx == 1) ? 0 : 1;
3107       for (int i = 0; i < vp9.spatial_idx; ++i) {
3108         EXPECT_EQ(last_temporal_idx_by_spatial_idx_.at(i), expected_tid);
3109       }
3110     }
3111     // Same within spatial layer.
3112     bool new_layer = vp9.spatial_idx != last_vp9_.spatial_idx;
3113     if (!new_layer) {
3114       EXPECT_EQ(vp9.temporal_idx, last_vp9_.temporal_idx);
3115     }
3116   }
3117 
VerifyFixedTemporalLayerStructure(const RTPVideoHeaderVP9 & vp9,uint8_t num_layers) const3118   void VerifyFixedTemporalLayerStructure(const RTPVideoHeaderVP9& vp9,
3119                                          uint8_t num_layers) const {
3120     switch (num_layers) {
3121       case 0:
3122         VerifyTemporalLayerStructure0(vp9);
3123         break;
3124       case 1:
3125         VerifyTemporalLayerStructure1(vp9);
3126         break;
3127       case 2:
3128         VerifyTemporalLayerStructure2(vp9);
3129         break;
3130       case 3:
3131         VerifyTemporalLayerStructure3(vp9);
3132         break;
3133       default:
3134         RTC_DCHECK_NOTREACHED();
3135     }
3136   }
3137 
VerifyTemporalLayerStructure0(const RTPVideoHeaderVP9 & vp9) const3138   void VerifyTemporalLayerStructure0(const RTPVideoHeaderVP9& vp9) const {
3139     EXPECT_EQ(kNoTl0PicIdx, vp9.tl0_pic_idx);
3140     EXPECT_EQ(kNoTemporalIdx, vp9.temporal_idx);  // no tid
3141     // Technically true, but layer indices not available.
3142     EXPECT_FALSE(vp9.temporal_up_switch);
3143   }
3144 
VerifyTemporalLayerStructure1(const RTPVideoHeaderVP9 & vp9) const3145   void VerifyTemporalLayerStructure1(const RTPVideoHeaderVP9& vp9) const {
3146     EXPECT_NE(kNoTl0PicIdx, vp9.tl0_pic_idx);
3147     EXPECT_EQ(0, vp9.temporal_idx);  // 0,0,0,...
3148   }
3149 
VerifyTemporalLayerStructure2(const RTPVideoHeaderVP9 & vp9) const3150   void VerifyTemporalLayerStructure2(const RTPVideoHeaderVP9& vp9) const {
3151     EXPECT_NE(kNoTl0PicIdx, vp9.tl0_pic_idx);
3152     EXPECT_GE(vp9.temporal_idx, 0);  // 0,1,0,1,... (tid reset on I-frames).
3153     EXPECT_LE(vp9.temporal_idx, 1);
3154     EXPECT_TRUE(vp9.temporal_up_switch);
3155     // Verify temporal structure for the highest spatial layer (the structure
3156     // may be shifted for lower spatial layer if temporal shift is configured).
3157     if (IsHighestSpatialLayer(vp9) && vp9.beginning_of_frame) {
3158       int expected_tid =
3159           (!vp9.inter_pic_predicted ||
3160            last_temporal_idx_by_spatial_idx_.at(vp9.spatial_idx) == 1)
3161               ? 0
3162               : 1;
3163       EXPECT_EQ(vp9.temporal_idx, expected_tid);
3164     }
3165   }
3166 
VerifyTemporalLayerStructure3(const RTPVideoHeaderVP9 & vp9) const3167   void VerifyTemporalLayerStructure3(const RTPVideoHeaderVP9& vp9) const {
3168     EXPECT_NE(kNoTl0PicIdx, vp9.tl0_pic_idx);
3169     EXPECT_GE(vp9.temporal_idx, 0);  // 0,2,1,2,... (tid reset on I-frames).
3170     EXPECT_LE(vp9.temporal_idx, 2);
3171     if (IsNewPictureId(vp9) && vp9.inter_pic_predicted) {
3172       EXPECT_NE(vp9.temporal_idx, last_vp9_.temporal_idx);
3173       EXPECT_TRUE(vp9.temporal_up_switch);
3174       switch (vp9.temporal_idx) {
3175         case 0:
3176           EXPECT_EQ(last_vp9_.temporal_idx, 2);
3177           break;
3178         case 1:
3179           EXPECT_EQ(last_vp9_.temporal_idx, 2);
3180           break;
3181         case 2:
3182           EXPECT_LT(last_vp9_.temporal_idx, 2);
3183           break;
3184       }
3185     }
3186   }
3187 
VerifyTl0Idx(const RTPVideoHeaderVP9 & vp9) const3188   void VerifyTl0Idx(const RTPVideoHeaderVP9& vp9) const {
3189     if (vp9.tl0_pic_idx == kNoTl0PicIdx)
3190       return;
3191 
3192     uint8_t expected_tl0_idx = last_vp9_.tl0_pic_idx;
3193     if (vp9.temporal_idx == 0)
3194       ++expected_tl0_idx;
3195     EXPECT_EQ(expected_tl0_idx, vp9.tl0_pic_idx);
3196   }
3197 
IsNewPictureId(const RTPVideoHeaderVP9 & vp9) const3198   bool IsNewPictureId(const RTPVideoHeaderVP9& vp9) const {
3199     return frames_sent_ > 0 && (vp9.picture_id != last_vp9_.picture_id);
3200   }
3201 
IsHighestSpatialLayer(const RTPVideoHeaderVP9 & vp9) const3202   bool IsHighestSpatialLayer(const RTPVideoHeaderVP9& vp9) const {
3203     return vp9.spatial_idx == params_.num_spatial_layers - 1 ||
3204            vp9.spatial_idx == kNoSpatialIdx;
3205   }
3206 
3207   // Flexible mode (F=1):    Non-flexible mode (F=0):
3208   //
3209   //      +-+-+-+-+-+-+-+-+     +-+-+-+-+-+-+-+-+
3210   //      |I|P|L|F|B|E|V|-|     |I|P|L|F|B|E|V|-|
3211   //      +-+-+-+-+-+-+-+-+     +-+-+-+-+-+-+-+-+
3212   // I:   |M| PICTURE ID  |  I: |M| PICTURE ID  |
3213   //      +-+-+-+-+-+-+-+-+     +-+-+-+-+-+-+-+-+
3214   // M:   | EXTENDED PID  |  M: | EXTENDED PID  |
3215   //      +-+-+-+-+-+-+-+-+     +-+-+-+-+-+-+-+-+
3216   // L:   |  T  |U|  S  |D|  L: |  T  |U|  S  |D|
3217   //      +-+-+-+-+-+-+-+-+     +-+-+-+-+-+-+-+-+
3218   // P,F: | P_DIFF    |X|N|     |   TL0PICIDX   |
3219   //      +-+-+-+-+-+-+-+-+     +-+-+-+-+-+-+-+-+
3220   // X:   |EXTENDED P_DIFF|  V: | SS  ..        |
3221   //      +-+-+-+-+-+-+-+-+     +-+-+-+-+-+-+-+-+
3222   // V:   | SS  ..        |
3223   //      +-+-+-+-+-+-+-+-+
VerifyCommonHeader(const RTPVideoHeaderVP9 & vp9) const3224   void VerifyCommonHeader(const RTPVideoHeaderVP9& vp9) const {
3225     EXPECT_EQ(kMaxTwoBytePictureId, vp9.max_picture_id);       // M:1
3226     EXPECT_NE(kNoPictureId, vp9.picture_id);                   // I:1
3227     EXPECT_EQ(vp9_settings_.flexibleMode, vp9.flexible_mode);  // F
3228 
3229     if (params_.num_spatial_layers > 1) {
3230       EXPECT_LT(vp9.spatial_idx, params_.num_spatial_layers);
3231     } else if (params_.num_temporal_layers > 1) {
3232       EXPECT_EQ(vp9.spatial_idx, 0);
3233     } else {
3234       EXPECT_EQ(vp9.spatial_idx, kNoSpatialIdx);
3235     }
3236 
3237     if (params_.num_temporal_layers > 1) {
3238       EXPECT_LT(vp9.temporal_idx, params_.num_temporal_layers);
3239     } else if (params_.num_spatial_layers > 1) {
3240       EXPECT_EQ(vp9.temporal_idx, 0);
3241     } else {
3242       EXPECT_EQ(vp9.temporal_idx, kNoTemporalIdx);
3243     }
3244 
3245     if (vp9.ss_data_available)  // V
3246       VerifySsData(vp9);
3247 
3248     if (frames_sent_ == 0)
3249       EXPECT_FALSE(vp9.inter_pic_predicted);  // P
3250 
3251     if (!vp9.inter_pic_predicted) {
3252       if (vp9.temporal_idx == kNoTemporalIdx) {
3253         EXPECT_FALSE(vp9.temporal_up_switch);
3254       } else {
3255         EXPECT_EQ(vp9.temporal_idx, 0);
3256         EXPECT_TRUE(vp9.temporal_up_switch);
3257       }
3258     }
3259   }
3260 
3261   // Scalability structure (SS).
3262   //
3263   //      +-+-+-+-+-+-+-+-+
3264   // V:   | N_S |Y|G|-|-|-|
3265   //      +-+-+-+-+-+-+-+-+
3266   // Y:   |    WIDTH      |  N_S + 1 times
3267   //      +-+-+-+-+-+-+-+-+
3268   //      |    HEIGHT     |
3269   //      +-+-+-+-+-+-+-+-+
3270   // G:   |      N_G      |
3271   //      +-+-+-+-+-+-+-+-+
3272   // N_G: |  T  |U| R |-|-|  N_G times
3273   //      +-+-+-+-+-+-+-+-+
3274   //      |    P_DIFF     |  R times
3275   //      +-+-+-+-+-+-+-+-+
VerifySsData(const RTPVideoHeaderVP9 & vp9) const3276   void VerifySsData(const RTPVideoHeaderVP9& vp9) const {
3277     EXPECT_TRUE(vp9.ss_data_available);             // V
3278     EXPECT_EQ(params_.num_spatial_layers,           // N_S + 1
3279               vp9.num_spatial_layers);
3280     EXPECT_TRUE(vp9.spatial_layer_resolution_present);  // Y:1
3281 
3282     ScalableVideoController::StreamLayersConfig config = GetScalabilityConfig();
3283     for (int i = config.num_spatial_layers - 1; i >= 0; --i) {
3284       double ratio = static_cast<double>(config.scaling_factor_num[i]) /
3285                      config.scaling_factor_den[i];
3286       EXPECT_EQ(expected_width_ * ratio, vp9.width[i]);    // WIDTH
3287       EXPECT_EQ(expected_height_ * ratio, vp9.height[i]);  // HEIGHT
3288     }
3289   }
3290 
CompareConsecutiveFrames(const RtpPacket & rtp_packet,const RTPVideoHeader & video) const3291   void CompareConsecutiveFrames(const RtpPacket& rtp_packet,
3292                                 const RTPVideoHeader& video) const {
3293     const auto& vp9_header =
3294         absl::get<RTPVideoHeaderVP9>(video.video_type_header);
3295 
3296     const bool new_temporal_unit =
3297         !last_packet_timestamp_.has_value() ||
3298         IsNewerTimestamp(rtp_packet.Timestamp(), *last_packet_timestamp_);
3299     const bool new_frame =
3300         new_temporal_unit || last_vp9_.spatial_idx != vp9_header.spatial_idx;
3301 
3302     EXPECT_EQ(new_frame, video.is_first_packet_in_frame);
3303     if (!new_temporal_unit) {
3304       EXPECT_FALSE(last_packet_marker_);
3305       EXPECT_EQ(*last_packet_timestamp_, rtp_packet.Timestamp());
3306       EXPECT_EQ(last_vp9_.picture_id, vp9_header.picture_id);
3307       EXPECT_EQ(last_vp9_.tl0_pic_idx, vp9_header.tl0_pic_idx);
3308       VerifySpatialIdxWithinFrame(vp9_header);
3309       VerifyTemporalIdxWithinFrame(vp9_header);
3310       return;
3311     }
3312     // New frame.
3313     EXPECT_TRUE(vp9_header.beginning_of_frame);
3314 
3315     // Compare with last packet in previous frame.
3316     if (frames_sent_ == 0)
3317       return;
3318     EXPECT_TRUE(last_vp9_.end_of_frame);
3319     EXPECT_TRUE(last_packet_marker_);
3320     EXPECT_TRUE(ContinuousPictureId(vp9_header));
3321     VerifyTl0Idx(vp9_header);
3322   }
3323 
GetScalabilityConfig() const3324   ScalableVideoController::StreamLayersConfig GetScalabilityConfig() const {
3325     absl::optional<ScalabilityMode> scalability_mode =
3326         ScalabilityModeFromString(params_.scalability_mode);
3327     EXPECT_TRUE(scalability_mode.has_value());
3328     absl::optional<ScalableVideoController::StreamLayersConfig> config =
3329         ScalabilityStructureConfig(*scalability_mode);
3330     EXPECT_TRUE(config.has_value());
3331     EXPECT_EQ(config->num_spatial_layers, params_.num_spatial_layers);
3332     return *config;
3333   }
3334 
3335   test::FunctionVideoEncoderFactory encoder_factory_;
3336   const Vp9TestParams params_;
3337   VideoCodecVP9 vp9_settings_;
3338   webrtc::VideoEncoderConfig encoder_config_;
3339   bool last_packet_marker_ = false;
3340   absl::optional<uint16_t> last_packet_sequence_number_;
3341   absl::optional<uint32_t> last_packet_timestamp_;
3342   RTPVideoHeaderVP9 last_vp9_;
3343   std::map<int, int> last_temporal_idx_by_spatial_idx_;
3344   Mutex mutex_;
3345   size_t frames_sent_ = 0;
3346   int expected_width_ = 0;
3347   int expected_height_ = 0;
3348 };
3349 
3350 class Vp9Test : public VideoSendStreamTest,
3351                 public ::testing::WithParamInterface<ParameterizationType> {
3352  public:
Vp9Test()3353   Vp9Test()
3354       : params_(::testing::get<Vp9TestParams>(GetParam())),
3355         use_scalability_mode_identifier_(::testing::get<bool>(GetParam())) {}
3356 
3357  protected:
3358   const Vp9TestParams params_;
3359   const bool use_scalability_mode_identifier_;
3360 };
3361 
3362 INSTANTIATE_TEST_SUITE_P(
3363     ScalabilityMode,
3364     Vp9Test,
3365     ::testing::Combine(
3366         ::testing::ValuesIn<Vp9TestParams>(
3367             {{"L1T1", 1, 1, InterLayerPredMode::kOn},
3368              {"L1T2", 1, 2, InterLayerPredMode::kOn},
3369              {"L1T3", 1, 3, InterLayerPredMode::kOn},
3370              {"L2T1", 2, 1, InterLayerPredMode::kOn},
3371              {"L2T1_KEY", 2, 1, InterLayerPredMode::kOnKeyPic},
3372              {"L2T2", 2, 2, InterLayerPredMode::kOn},
3373              {"L2T2_KEY", 2, 2, InterLayerPredMode::kOnKeyPic},
3374              {"L2T3", 2, 3, InterLayerPredMode::kOn},
3375              {"L2T3_KEY", 2, 3, InterLayerPredMode::kOnKeyPic},
3376              {"L3T1", 3, 1, InterLayerPredMode::kOn},
3377              {"L3T1_KEY", 3, 1, InterLayerPredMode::kOnKeyPic},
3378              {"L3T2", 3, 2, InterLayerPredMode::kOn},
3379              {"L3T2_KEY", 3, 2, InterLayerPredMode::kOnKeyPic},
3380              {"L3T3", 3, 3, InterLayerPredMode::kOn},
3381              {"L3T3_KEY", 3, 3, InterLayerPredMode::kOnKeyPic},
3382              {"S2T1", 2, 1, InterLayerPredMode::kOff},
3383              {"S2T2", 2, 2, InterLayerPredMode::kOff},
3384              {"S2T3", 2, 3, InterLayerPredMode::kOff},
3385              {"S3T1", 3, 1, InterLayerPredMode::kOff},
3386              {"S3T2", 3, 2, InterLayerPredMode::kOff},
3387              {"S3T3", 3, 3, InterLayerPredMode::kOff}}),
3388         ::testing::Values(false, true)),  // use_scalability_mode_identifier
3389     ParamInfoToStr);
3390 
3391 INSTANTIATE_TEST_SUITE_P(
3392     ScalabilityModeOn,
3393     Vp9Test,
3394     ::testing::Combine(
3395         ::testing::ValuesIn<Vp9TestParams>(
3396             {{"L2T1h", 2, 1, InterLayerPredMode::kOn},
3397              {"L2T2h", 2, 2, InterLayerPredMode::kOn},
3398              {"L2T3h", 2, 3, InterLayerPredMode::kOn},
3399              {"L2T2_KEY_SHIFT", 2, 2, InterLayerPredMode::kOnKeyPic},
3400              {"L3T1h", 3, 1, InterLayerPredMode::kOn},
3401              {"L3T2h", 3, 2, InterLayerPredMode::kOn},
3402              {"L3T3h", 3, 3, InterLayerPredMode::kOn},
3403              {"S2T1h", 2, 1, InterLayerPredMode::kOff},
3404              {"S2T2h", 2, 2, InterLayerPredMode::kOff},
3405              {"S2T3h", 2, 3, InterLayerPredMode::kOff},
3406              {"S3T1h", 3, 1, InterLayerPredMode::kOff},
3407              {"S3T2h", 3, 2, InterLayerPredMode::kOff},
3408              {"S3T3h", 3, 3, InterLayerPredMode::kOff}}),
3409         ::testing::Values(true)),  // use_scalability_mode_identifier
3410     ParamInfoToStr);
3411 
TEST_P(Vp9Test,NonFlexMode)3412 TEST_P(Vp9Test, NonFlexMode) {
3413   TestVp9NonFlexMode(params_, use_scalability_mode_identifier_);
3414 }
3415 
TestVp9NonFlexMode(const Vp9TestParams & params,bool use_scalability_mode_identifier)3416 void VideoSendStreamTest::TestVp9NonFlexMode(
3417     const Vp9TestParams& params,
3418     bool use_scalability_mode_identifier) {
3419   static const size_t kNumFramesToSend = 100;
3420   // Set to < kNumFramesToSend and coprime to length of temporal layer
3421   // structures to verify temporal id reset on key frame.
3422   static const int kKeyFrameInterval = 31;
3423 
3424   static const int kWidth = kMinVp9SpatialLayerLongSideLength;
3425   static const int kHeight = kMinVp9SpatialLayerShortSideLength;
3426   static const float kGoodBitsPerPixel = 0.1f;
3427   class NonFlexibleMode : public Vp9HeaderObserver {
3428    public:
3429     NonFlexibleMode(const Vp9TestParams& params,
3430                     bool use_scalability_mode_identifier)
3431         : Vp9HeaderObserver(params),
3432           use_scalability_mode_identifier_(use_scalability_mode_identifier),
3433           l_field_(params.num_temporal_layers > 1 ||
3434                    params.num_spatial_layers > 1) {}
3435 
3436     void ModifyVideoConfigsHook(
3437         VideoSendStream::Config* send_config,
3438         std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
3439         VideoEncoderConfig* encoder_config) override {
3440       encoder_config->codec_type = kVideoCodecVP9;
3441       int bitrate_bps = 0;
3442       for (int sl_idx = 0; sl_idx < params_.num_spatial_layers; ++sl_idx) {
3443         const int width = kWidth << sl_idx;
3444         const int height = kHeight << sl_idx;
3445         const float bpp = kGoodBitsPerPixel / (1 << sl_idx);
3446         bitrate_bps += static_cast<int>(width * height * bpp * 30);
3447       }
3448       encoder_config->max_bitrate_bps = bitrate_bps * 2;
3449 
3450       encoder_config->frame_drop_enabled = false;
3451 
3452       vp9_settings_.flexibleMode = false;
3453       vp9_settings_.automaticResizeOn = false;
3454       vp9_settings_.keyFrameInterval = kKeyFrameInterval;
3455       if (!use_scalability_mode_identifier_) {
3456         vp9_settings_.numberOfTemporalLayers = params_.num_temporal_layers;
3457         vp9_settings_.numberOfSpatialLayers = params_.num_spatial_layers;
3458         vp9_settings_.interLayerPred = params_.inter_layer_pred;
3459       } else {
3460         absl::optional<ScalabilityMode> mode =
3461             ScalabilityModeFromString(params_.scalability_mode);
3462         encoder_config->simulcast_layers[0].scalability_mode = mode;
3463         EXPECT_TRUE(mode.has_value());
3464       }
3465     }
3466 
3467     int GetRequiredDivisibility() const {
3468       ScalableVideoController::StreamLayersConfig config =
3469           GetScalabilityConfig();
3470       int required_divisibility = 1;
3471       for (int sl_idx = 0; sl_idx < config.num_spatial_layers; ++sl_idx) {
3472         required_divisibility = cricket::LeastCommonMultiple(
3473             required_divisibility, config.scaling_factor_den[sl_idx]);
3474       }
3475       return required_divisibility;
3476     }
3477 
3478     void ModifyVideoCaptureStartResolution(int* width,
3479                                            int* height,
3480                                            int* frame_rate) override {
3481       expected_width_ = kWidth << (params_.num_spatial_layers - 1);
3482       expected_height_ = kHeight << (params_.num_spatial_layers - 1);
3483       *width = expected_width_;
3484       *height = expected_height_;
3485       // Top layer may be adjusted to ensure evenly divided layers.
3486       int divisibility = GetRequiredDivisibility();
3487       expected_width_ -= (expected_width_ % divisibility);
3488       expected_height_ -= (expected_height_ % divisibility);
3489     }
3490 
3491     void InspectHeader(const RTPVideoHeaderVP9& vp9) override {
3492       bool ss_data_expected = !vp9.inter_pic_predicted &&
3493                               vp9.beginning_of_frame &&
3494                               !vp9.inter_layer_predicted;
3495       EXPECT_EQ(ss_data_expected, vp9.ss_data_available);
3496 
3497       bool is_key_frame = frames_sent_ % kKeyFrameInterval == 0;
3498       if (params_.num_spatial_layers > 1) {
3499         switch (params_.inter_layer_pred) {
3500           case InterLayerPredMode::kOff:
3501             EXPECT_FALSE(vp9.inter_layer_predicted);
3502             break;
3503           case InterLayerPredMode::kOn:
3504             EXPECT_EQ(vp9.spatial_idx > 0, vp9.inter_layer_predicted);
3505             break;
3506           case InterLayerPredMode::kOnKeyPic:
3507             EXPECT_EQ(is_key_frame && vp9.spatial_idx > 0,
3508                       vp9.inter_layer_predicted);
3509             break;
3510         }
3511       } else {
3512         EXPECT_FALSE(vp9.inter_layer_predicted);
3513       }
3514 
3515       EXPECT_EQ(is_key_frame, !vp9.inter_pic_predicted);
3516 
3517       if (IsNewPictureId(vp9)) {
3518         if (params_.num_temporal_layers == 1 &&
3519             params_.num_spatial_layers == 1) {
3520           EXPECT_EQ(kNoSpatialIdx, vp9.spatial_idx);
3521         } else {
3522           EXPECT_EQ(0, vp9.spatial_idx);
3523         }
3524         if (params_.num_spatial_layers > 1)
3525           EXPECT_EQ(params_.num_spatial_layers - 1, last_vp9_.spatial_idx);
3526       }
3527 
3528       VerifyFixedTemporalLayerStructure(
3529           vp9, l_field_ ? params_.num_temporal_layers : 0);
3530 
3531       if (frames_sent_ > kNumFramesToSend)
3532         observation_complete_.Set();
3533     }
3534     const bool use_scalability_mode_identifier_;
3535     const bool l_field_;
3536 
3537    private:
3538     void ModifySenderBitrateConfig(
3539         BitrateConstraints* bitrate_config) override {
3540       const int kBitrateBps = 800000;
3541       bitrate_config->min_bitrate_bps = kBitrateBps;
3542       bitrate_config->start_bitrate_bps = kBitrateBps;
3543     }
3544   } test(params, use_scalability_mode_identifier);
3545 
3546   RunBaseTest(&test);
3547 }
3548 
TEST_F(VideoSendStreamTest,Vp9NonFlexModeSmallResolution)3549 TEST_F(VideoSendStreamTest, Vp9NonFlexModeSmallResolution) {
3550   static const size_t kNumFramesToSend = 50;
3551   static const int kWidth = 4;
3552   static const int kHeight = 4;
3553   class NonFlexibleModeResolution : public Vp9HeaderObserver {
3554    public:
3555     explicit NonFlexibleModeResolution(const Vp9TestParams& params)
3556         : Vp9HeaderObserver(params) {}
3557 
3558    private:
3559     void ModifyVideoConfigsHook(
3560         VideoSendStream::Config* send_config,
3561         std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
3562         VideoEncoderConfig* encoder_config) override {
3563       encoder_config->codec_type = kVideoCodecVP9;
3564       vp9_settings_.flexibleMode = false;
3565       vp9_settings_.numberOfTemporalLayers = params_.num_temporal_layers;
3566       vp9_settings_.numberOfSpatialLayers = params_.num_spatial_layers;
3567       vp9_settings_.interLayerPred = params_.inter_layer_pred;
3568     }
3569 
3570     void InspectHeader(const RTPVideoHeaderVP9& vp9_header) override {
3571       if (frames_sent_ > kNumFramesToSend)
3572         observation_complete_.Set();
3573     }
3574 
3575     void ModifyVideoCaptureStartResolution(int* width,
3576                                            int* height,
3577                                            int* frame_rate) override {
3578       expected_width_ = kWidth;
3579       expected_height_ = kHeight;
3580       *width = kWidth;
3581       *height = kHeight;
3582     }
3583   };
3584 
3585   Vp9TestParams params{"L1T1", 1, 1, InterLayerPredMode::kOn};
3586   NonFlexibleModeResolution test(params);
3587 
3588   RunBaseTest(&test);
3589 }
3590 
3591 #if defined(WEBRTC_ANDROID)
3592 // Crashes on Android; bugs.webrtc.org/7401
3593 #define MAYBE_Vp9FlexModeRefCount DISABLED_Vp9FlexModeRefCount
3594 #else
3595 // TODO(webrtc:9270): Support of flexible mode is temporarily disabled. Enable
3596 // the test after webrtc:9270 is implemented.
3597 #define MAYBE_Vp9FlexModeRefCount DISABLED_Vp9FlexModeRefCount
3598 // #define MAYBE_Vp9FlexModeRefCount Vp9FlexModeRefCount
3599 #endif
TEST_F(VideoSendStreamTest,MAYBE_Vp9FlexModeRefCount)3600 TEST_F(VideoSendStreamTest, MAYBE_Vp9FlexModeRefCount) {
3601   class FlexibleMode : public Vp9HeaderObserver {
3602    public:
3603     explicit FlexibleMode(const Vp9TestParams& params)
3604         : Vp9HeaderObserver(params) {}
3605 
3606    private:
3607     void ModifyVideoConfigsHook(
3608         VideoSendStream::Config* send_config,
3609         std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
3610         VideoEncoderConfig* encoder_config) override {
3611       encoder_config->codec_type = kVideoCodecVP9;
3612       encoder_config->content_type = VideoEncoderConfig::ContentType::kScreen;
3613       vp9_settings_.flexibleMode = true;
3614       vp9_settings_.numberOfTemporalLayers = params_.num_temporal_layers;
3615       vp9_settings_.numberOfSpatialLayers = params_.num_spatial_layers;
3616       vp9_settings_.interLayerPred = params_.inter_layer_pred;
3617     }
3618 
3619     void InspectHeader(const RTPVideoHeaderVP9& vp9_header) override {
3620       EXPECT_TRUE(vp9_header.flexible_mode);
3621       EXPECT_EQ(kNoTl0PicIdx, vp9_header.tl0_pic_idx);
3622       if (vp9_header.inter_pic_predicted) {
3623         EXPECT_GT(vp9_header.num_ref_pics, 0u);
3624         observation_complete_.Set();
3625       }
3626     }
3627   };
3628 
3629   Vp9TestParams params{"L2T1", 2, 1, InterLayerPredMode::kOn};
3630   FlexibleMode test(params);
3631 
3632   RunBaseTest(&test);
3633 }
3634 #endif  // defined(RTC_ENABLE_VP9)
3635 
TestRequestSourceRotateVideo(bool support_orientation_ext)3636 void VideoSendStreamTest::TestRequestSourceRotateVideo(
3637     bool support_orientation_ext) {
3638   CreateSenderCall();
3639 
3640   test::NullTransport transport;
3641   CreateSendConfig(1, 0, 0, &transport);
3642   GetVideoSendConfig()->rtp.extensions.clear();
3643   if (support_orientation_ext) {
3644     GetVideoSendConfig()->rtp.extensions.push_back(
3645         RtpExtension(RtpExtension::kVideoRotationUri, 1));
3646   }
3647 
3648   CreateVideoStreams();
3649   test::FrameForwarder forwarder;
3650   GetVideoSendStream()->SetSource(&forwarder,
3651                                   DegradationPreference::MAINTAIN_FRAMERATE);
3652 
3653   EXPECT_TRUE(forwarder.sink_wants().rotation_applied !=
3654               support_orientation_ext);
3655 
3656   DestroyStreams();
3657 }
3658 
TEST_F(VideoSendStreamTest,RequestSourceRotateIfVideoOrientationExtensionNotSupported)3659 TEST_F(VideoSendStreamTest,
3660        RequestSourceRotateIfVideoOrientationExtensionNotSupported) {
3661   TestRequestSourceRotateVideo(false);
3662 }
3663 
TEST_F(VideoSendStreamTest,DoNotRequestsRotationIfVideoOrientationExtensionSupported)3664 TEST_F(VideoSendStreamTest,
3665        DoNotRequestsRotationIfVideoOrientationExtensionSupported) {
3666   TestRequestSourceRotateVideo(true);
3667 }
3668 
TEST_F(VideoSendStreamTest,EncoderConfigMaxFramerateReportedToSource)3669 TEST_F(VideoSendStreamTest, EncoderConfigMaxFramerateReportedToSource) {
3670   static const int kMaxFps = 22;
3671   class FpsObserver : public test::SendTest,
3672                       public test::FrameGeneratorCapturer::SinkWantsObserver {
3673    public:
3674     FpsObserver() : SendTest(kDefaultTimeout) {}
3675 
3676     void OnFrameGeneratorCapturerCreated(
3677         test::FrameGeneratorCapturer* frame_generator_capturer) override {
3678       frame_generator_capturer->SetSinkWantsObserver(this);
3679     }
3680 
3681     void OnSinkWantsChanged(rtc::VideoSinkInterface<VideoFrame>* sink,
3682                             const rtc::VideoSinkWants& wants) override {
3683       if (wants.max_framerate_fps == kMaxFps)
3684         observation_complete_.Set();
3685     }
3686 
3687     void ModifyVideoConfigs(
3688         VideoSendStream::Config* send_config,
3689         std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
3690         VideoEncoderConfig* encoder_config) override {
3691       encoder_config->simulcast_layers[0].max_framerate = kMaxFps;
3692     }
3693 
3694     void PerformTest() override {
3695       EXPECT_TRUE(Wait()) << "Timed out while waiting for fps to be reported.";
3696     }
3697   } test;
3698 
3699   RunBaseTest(&test);
3700 }
3701 
3702 // This test verifies that overhead is removed from the bandwidth estimate by
3703 // testing that the maximum possible target payload rate is smaller than the
3704 // maximum bandwidth estimate by the overhead rate.
TEST_F(VideoSendStreamTest,RemoveOverheadFromBandwidth)3705 TEST_F(VideoSendStreamTest, RemoveOverheadFromBandwidth) {
3706   class RemoveOverheadFromBandwidthTest : public test::EndToEndTest,
3707                                           public test::FakeEncoder {
3708    public:
3709     explicit RemoveOverheadFromBandwidthTest(TaskQueueBase* task_queue)
3710         : EndToEndTest(test::CallTest::kDefaultTimeout),
3711           FakeEncoder(Clock::GetRealTimeClock()),
3712           task_queue_(task_queue),
3713           encoder_factory_(this),
3714           call_(nullptr),
3715           max_bitrate_bps_(0),
3716           first_packet_sent_(false) {}
3717 
3718     void SetRates(const RateControlParameters& parameters) override {
3719       MutexLock lock(&mutex_);
3720       // Wait for the first sent packet so that videosendstream knows
3721       // rtp_overhead.
3722       if (first_packet_sent_) {
3723         max_bitrate_bps_ = parameters.bitrate.get_sum_bps();
3724         bitrate_changed_event_.Set();
3725       }
3726       return FakeEncoder::SetRates(parameters);
3727     }
3728 
3729     void OnCallsCreated(Call* sender_call, Call* receiver_call) override {
3730       call_ = sender_call;
3731     }
3732 
3733     void ModifyVideoConfigs(
3734         VideoSendStream::Config* send_config,
3735         std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
3736         VideoEncoderConfig* encoder_config) override {
3737       send_config->rtp.max_packet_size = 1200;
3738       send_config->encoder_settings.encoder_factory = &encoder_factory_;
3739       EXPECT_FALSE(send_config->rtp.extensions.empty());
3740     }
3741 
3742     Action OnSendRtp(const uint8_t* packet, size_t length) override {
3743       MutexLock lock(&mutex_);
3744       first_packet_sent_ = true;
3745       return SEND_PACKET;
3746     }
3747 
3748     void PerformTest() override {
3749       BitrateConstraints bitrate_config;
3750       constexpr int kStartBitrateBps = 60000;
3751       constexpr int kMaxBitrateBps = 60000;
3752       constexpr int kMinBitrateBps = 10000;
3753       bitrate_config.start_bitrate_bps = kStartBitrateBps;
3754       bitrate_config.max_bitrate_bps = kMaxBitrateBps;
3755       bitrate_config.min_bitrate_bps = kMinBitrateBps;
3756       SendTask(task_queue_, [this, &bitrate_config]() {
3757         call_->GetTransportControllerSend()->SetSdpBitrateParameters(
3758             bitrate_config);
3759         call_->GetTransportControllerSend()->OnTransportOverheadChanged(40);
3760       });
3761 
3762       // At a bitrate of 60kbps with a packet size of 1200B video and an
3763       // overhead of 40B per packet video produces 2240bps overhead.
3764       // So the encoder BW should be set to 57760bps.
3765       EXPECT_TRUE(
3766           bitrate_changed_event_.Wait(VideoSendStreamTest::kDefaultTimeout));
3767       {
3768         MutexLock lock(&mutex_);
3769         EXPECT_LE(max_bitrate_bps_, 57760u);
3770       }
3771     }
3772 
3773    private:
3774     TaskQueueBase* const task_queue_;
3775     test::VideoEncoderProxyFactory encoder_factory_;
3776     Call* call_;
3777     Mutex mutex_;
3778     uint32_t max_bitrate_bps_ RTC_GUARDED_BY(&mutex_);
3779     bool first_packet_sent_ RTC_GUARDED_BY(&mutex_);
3780     rtc::Event bitrate_changed_event_;
3781   } test(task_queue());
3782   RunBaseTest(&test);
3783 }
3784 
3785 class PacingFactorObserver : public test::SendTest {
3786  public:
PacingFactorObserver(bool configure_send_side,absl::optional<float> expected_pacing_factor)3787   PacingFactorObserver(bool configure_send_side,
3788                        absl::optional<float> expected_pacing_factor)
3789       : test::SendTest(VideoSendStreamTest::kDefaultTimeout),
3790         configure_send_side_(configure_send_side),
3791         expected_pacing_factor_(expected_pacing_factor) {}
3792 
ModifyVideoConfigs(VideoSendStream::Config * send_config,std::vector<VideoReceiveStreamInterface::Config> * receive_configs,VideoEncoderConfig * encoder_config)3793   void ModifyVideoConfigs(
3794       VideoSendStream::Config* send_config,
3795       std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
3796       VideoEncoderConfig* encoder_config) override {
3797     // Check if send-side bwe extension is already present, and remove it if
3798     // it is not desired.
3799     bool has_send_side = false;
3800     for (auto it = send_config->rtp.extensions.begin();
3801          it != send_config->rtp.extensions.end(); ++it) {
3802       if (it->uri == RtpExtension::kTransportSequenceNumberUri) {
3803         if (configure_send_side_) {
3804           has_send_side = true;
3805         } else {
3806           send_config->rtp.extensions.erase(it);
3807         }
3808         break;
3809       }
3810     }
3811 
3812     if (configure_send_side_ && !has_send_side) {
3813       rtc::UniqueNumberGenerator<int> unique_id_generator;
3814       unique_id_generator.AddKnownId(0);  // First valid RTP extension ID is 1.
3815       for (const RtpExtension& extension : send_config->rtp.extensions) {
3816         unique_id_generator.AddKnownId(extension.id);
3817       }
3818       // Want send side, not present by default, so add it.
3819       send_config->rtp.extensions.emplace_back(
3820           RtpExtension::kTransportSequenceNumberUri, unique_id_generator());
3821     }
3822 
3823     // ALR only enabled for screenshare.
3824     encoder_config->content_type = VideoEncoderConfig::ContentType::kScreen;
3825   }
3826 
OnVideoStreamsCreated(VideoSendStream * send_stream,const std::vector<VideoReceiveStreamInterface * > & receive_streams)3827   void OnVideoStreamsCreated(VideoSendStream* send_stream,
3828                              const std::vector<VideoReceiveStreamInterface*>&
3829                                  receive_streams) override {
3830     auto internal_send_peer = test::VideoSendStreamPeer(send_stream);
3831     // Video streams created, check that pacing factor is correctly configured.
3832     EXPECT_EQ(expected_pacing_factor_,
3833               internal_send_peer.GetPacingFactorOverride());
3834     observation_complete_.Set();
3835   }
3836 
PerformTest()3837   void PerformTest() override {
3838     EXPECT_TRUE(Wait()) << "Timed out while waiting for stream creation.";
3839   }
3840 
3841  private:
3842   const bool configure_send_side_;
3843   const absl::optional<float> expected_pacing_factor_;
3844 };
3845 
GetAlrProbingExperimentString()3846 std::string GetAlrProbingExperimentString() {
3847   return std::string(
3848              AlrExperimentSettings::kScreenshareProbingBweExperimentName) +
3849          "/1.0,2875,80,40,-60,3/";
3850 }
3851 const float kAlrProbingExperimentPaceMultiplier = 1.0f;
3852 
TEST_F(VideoSendStreamTest,AlrConfiguredWhenSendSideOn)3853 TEST_F(VideoSendStreamTest, AlrConfiguredWhenSendSideOn) {
3854   test::ScopedFieldTrials alr_experiment(GetAlrProbingExperimentString());
3855   // Send-side bwe on, use pacing factor from `kAlrProbingExperiment` above.
3856   PacingFactorObserver test_with_send_side(true,
3857                                            kAlrProbingExperimentPaceMultiplier);
3858   RunBaseTest(&test_with_send_side);
3859 }
3860 
TEST_F(VideoSendStreamTest,AlrNotConfiguredWhenSendSideOff)3861 TEST_F(VideoSendStreamTest, AlrNotConfiguredWhenSendSideOff) {
3862   test::ScopedFieldTrials alr_experiment(GetAlrProbingExperimentString());
3863   // Send-side bwe off, use configuration should not be overridden.
3864   PacingFactorObserver test_without_send_side(false, absl::nullopt);
3865   RunBaseTest(&test_without_send_side);
3866 }
3867 
3868 // Test class takes as argument a function pointer to reset the send
3869 // stream and call OnVideoStreamsCreated. This is necessary since you cannot
3870 // change the content type of a VideoSendStream, you need to recreate it.
3871 // Stopping and recreating the stream can only be done on the main thread and in
3872 // the context of VideoSendStreamTest (not BaseTest). The test switches from
3873 // realtime to screenshare and back.
3874 template <typename T>
3875 class ContentSwitchTest : public test::SendTest {
3876  public:
3877   enum class StreamState {
3878     kBeforeSwitch = 0,
3879     kInScreenshare = 1,
3880     kAfterSwitchBack = 2,
3881   };
3882   static const uint32_t kMinPacketsToSend = 50;
3883 
ContentSwitchTest(T * stream_reset_fun,TaskQueueBase * task_queue)3884   explicit ContentSwitchTest(T* stream_reset_fun, TaskQueueBase* task_queue)
3885       : SendTest(test::CallTest::kDefaultTimeout),
3886         call_(nullptr),
3887         state_(StreamState::kBeforeSwitch),
3888         send_stream_(nullptr),
3889         send_stream_config_(nullptr),
3890         packets_sent_(0),
3891         stream_resetter_(stream_reset_fun),
3892         task_queue_(task_queue) {
3893     RTC_DCHECK(stream_resetter_);
3894   }
3895 
OnVideoStreamsCreated(VideoSendStream * send_stream,const std::vector<VideoReceiveStreamInterface * > & receive_streams)3896   void OnVideoStreamsCreated(VideoSendStream* send_stream,
3897                              const std::vector<VideoReceiveStreamInterface*>&
3898                                  receive_streams) override {
3899     MutexLock lock(&mutex_);
3900     send_stream_ = send_stream;
3901   }
3902 
ModifyVideoConfigs(VideoSendStream::Config * send_config,std::vector<VideoReceiveStreamInterface::Config> * receive_configs,VideoEncoderConfig * encoder_config)3903   void ModifyVideoConfigs(
3904       VideoSendStream::Config* send_config,
3905       std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
3906       VideoEncoderConfig* encoder_config) override {
3907     RTC_DCHECK_EQ(1, encoder_config->number_of_streams);
3908     encoder_config->min_transmit_bitrate_bps = 0;
3909     encoder_config->content_type =
3910         VideoEncoderConfig::ContentType::kRealtimeVideo;
3911     send_stream_config_ = send_config->Copy();
3912     encoder_config_ = encoder_config->Copy();
3913   }
3914 
OnCallsCreated(Call * sender_call,Call * receiver_call)3915   void OnCallsCreated(Call* sender_call, Call* receiver_call) override {
3916     call_ = sender_call;
3917   }
3918 
OnStreamsStopped()3919   void OnStreamsStopped() override {
3920     MutexLock lock(&mutex_);
3921     done_ = true;
3922   }
3923 
OnSendRtp(const uint8_t * packet,size_t length)3924   Action OnSendRtp(const uint8_t* packet, size_t length) override {
3925     task_queue_->PostTask([this]() {
3926       MutexLock lock(&mutex_);
3927       if (done_)
3928         return;
3929 
3930       auto internal_send_peer = test::VideoSendStreamPeer(send_stream_);
3931       float pacing_factor =
3932           internal_send_peer.GetPacingFactorOverride().value_or(0.0f);
3933       float expected_pacing_factor = 1.1;  // Strict pacing factor.
3934       VideoSendStream::Stats stats = send_stream_->GetStats();
3935       if (stats.content_type == webrtc::VideoContentType::SCREENSHARE) {
3936         expected_pacing_factor = 1.0f;  // Currently used pacing factor in ALR.
3937       }
3938 
3939       EXPECT_NEAR(expected_pacing_factor, pacing_factor, 1e-6);
3940 
3941       // Wait until at least kMinPacketsToSend packets to be sent, so that
3942       // some frames would be encoded.
3943       if (++packets_sent_ < kMinPacketsToSend)
3944         return;
3945 
3946       if (state_ != StreamState::kAfterSwitchBack) {
3947         // We've sent kMinPacketsToSend packets, switch the content type and
3948         // move move to the next state. Note that we need to recreate the stream
3949         // if changing content type.
3950         packets_sent_ = 0;
3951         if (encoder_config_.content_type ==
3952             VideoEncoderConfig::ContentType::kRealtimeVideo) {
3953           encoder_config_.content_type =
3954               VideoEncoderConfig::ContentType::kScreen;
3955         } else {
3956           encoder_config_.content_type =
3957               VideoEncoderConfig::ContentType::kRealtimeVideo;
3958         }
3959         switch (state_) {
3960           case StreamState::kBeforeSwitch:
3961             state_ = StreamState::kInScreenshare;
3962             break;
3963           case StreamState::kInScreenshare:
3964             state_ = StreamState::kAfterSwitchBack;
3965             break;
3966           case StreamState::kAfterSwitchBack:
3967             RTC_DCHECK_NOTREACHED();
3968             break;
3969         }
3970         content_switch_event_.Set();
3971         return;
3972       }
3973       observation_complete_.Set();
3974     });
3975 
3976     return SEND_PACKET;
3977   }
3978 
PerformTest()3979   void PerformTest() override {
3980     while (GetStreamState() != StreamState::kAfterSwitchBack) {
3981       ASSERT_TRUE(content_switch_event_.Wait(test::CallTest::kDefaultTimeout));
3982       (*stream_resetter_)(send_stream_config_, encoder_config_, this);
3983     }
3984 
3985     ASSERT_TRUE(Wait())
3986         << "Timed out waiting for a frame sent after switch back";
3987   }
3988 
3989  private:
GetStreamState()3990   StreamState GetStreamState() {
3991     MutexLock lock(&mutex_);
3992     return state_;
3993   }
3994 
3995   Mutex mutex_;
3996   rtc::Event content_switch_event_;
3997   Call* call_;
3998   bool done_ RTC_GUARDED_BY(mutex_) = false;
3999   StreamState state_ RTC_GUARDED_BY(mutex_);
4000   VideoSendStream* send_stream_ RTC_GUARDED_BY(mutex_);
4001   VideoSendStream::Config send_stream_config_;
4002   VideoEncoderConfig encoder_config_;
4003   uint32_t packets_sent_ RTC_GUARDED_BY(mutex_);
4004   T* stream_resetter_;
4005   TaskQueueBase* task_queue_;
4006 };
4007 
TEST_F(VideoSendStreamTest,SwitchesToScreenshareAndBack)4008 TEST_F(VideoSendStreamTest, SwitchesToScreenshareAndBack) {
4009   auto reset_fun = [this](const VideoSendStream::Config& send_stream_config,
4010                           const VideoEncoderConfig& encoder_config,
4011                           test::BaseTest* test) {
4012     SendTask(task_queue(),
4013              [this, &send_stream_config, &encoder_config, &test]() {
4014                Stop();
4015                DestroyVideoSendStreams();
4016                SetVideoSendConfig(send_stream_config);
4017                SetVideoEncoderConfig(encoder_config);
4018                CreateVideoSendStreams();
4019                SetVideoDegradation(DegradationPreference::MAINTAIN_RESOLUTION);
4020                test->OnVideoStreamsCreated(GetVideoSendStream(),
4021                                            video_receive_streams_);
4022                Start();
4023              });
4024   };
4025   ContentSwitchTest<decltype(reset_fun)> test(&reset_fun, task_queue());
4026   RunBaseTest(&test);
4027 }
4028 
TestTemporalLayers(VideoEncoderFactory * encoder_factory,const std::string & payload_name,const std::vector<int> & num_temporal_layers,const std::vector<ScalabilityMode> & scalability_mode)4029 void VideoSendStreamTest::TestTemporalLayers(
4030     VideoEncoderFactory* encoder_factory,
4031     const std::string& payload_name,
4032     const std::vector<int>& num_temporal_layers,
4033     const std::vector<ScalabilityMode>& scalability_mode) {
4034   static constexpr int kMaxBitrateBps = 1000000;
4035   static constexpr int kMinFramesToObservePerStream = 8;
4036 
4037   class TemporalLayerObserver
4038       : public test::EndToEndTest,
4039         public test::FrameGeneratorCapturer::SinkWantsObserver {
4040    public:
4041     TemporalLayerObserver(VideoEncoderFactory* encoder_factory,
4042                           const std::string& payload_name,
4043                           const std::vector<int>& num_temporal_layers,
4044                           const std::vector<ScalabilityMode>& scalability_mode)
4045         : EndToEndTest(kDefaultTimeout),
4046           encoder_factory_(encoder_factory),
4047           payload_name_(payload_name),
4048           num_temporal_layers_(num_temporal_layers),
4049           scalability_mode_(scalability_mode),
4050           depacketizer_(CreateVideoRtpDepacketizer(
4051               PayloadStringToCodecType(payload_name))) {}
4052 
4053    private:
4054     void OnFrameGeneratorCapturerCreated(
4055         test::FrameGeneratorCapturer* frame_generator_capturer) override {
4056       frame_generator_capturer->ChangeResolution(640, 360);
4057     }
4058 
4059     void OnSinkWantsChanged(rtc::VideoSinkInterface<VideoFrame>* sink,
4060                             const rtc::VideoSinkWants& wants) override {}
4061 
4062     void ModifySenderBitrateConfig(
4063         BitrateConstraints* bitrate_config) override {
4064       bitrate_config->start_bitrate_bps = kMaxBitrateBps / 2;
4065     }
4066 
4067     size_t GetNumVideoStreams() const override {
4068       if (scalability_mode_.empty()) {
4069         return num_temporal_layers_.size();
4070       } else {
4071         return scalability_mode_.size();
4072       }
4073     }
4074 
4075     void ModifyVideoConfigs(
4076         VideoSendStream::Config* send_config,
4077         std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
4078         VideoEncoderConfig* encoder_config) override {
4079       webrtc::VideoEncoder::EncoderInfo encoder_info;
4080       send_config->encoder_settings.encoder_factory = encoder_factory_;
4081       send_config->rtp.payload_name = payload_name_;
4082       send_config->rtp.payload_type = test::CallTest::kVideoSendPayloadType;
4083       encoder_config->video_format.name = payload_name_;
4084       encoder_config->codec_type = PayloadStringToCodecType(payload_name_);
4085       encoder_config->video_stream_factory =
4086           rtc::make_ref_counted<cricket::EncoderStreamFactory>(
4087               payload_name_, /*max_qp=*/56, /*is_screenshare=*/false,
4088               /*conference_mode=*/false, encoder_info);
4089       encoder_config->max_bitrate_bps = kMaxBitrateBps;
4090       if (absl::EqualsIgnoreCase(payload_name_, "VP9")) {
4091         encoder_config->encoder_specific_settings = rtc::make_ref_counted<
4092             VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4093             VideoEncoder::GetDefaultVp9Settings());
4094       }
4095       if (scalability_mode_.empty()) {
4096         for (size_t i = 0; i < num_temporal_layers_.size(); ++i) {
4097           VideoStream& stream = encoder_config->simulcast_layers[i];
4098           stream.num_temporal_layers = num_temporal_layers_[i];
4099           configured_num_temporal_layers_[send_config->rtp.ssrcs[i]] =
4100               num_temporal_layers_[i];
4101         }
4102       } else {
4103         for (size_t i = 0; i < scalability_mode_.size(); ++i) {
4104           VideoStream& stream = encoder_config->simulcast_layers[i];
4105           stream.scalability_mode = scalability_mode_[i];
4106 
4107           configured_num_temporal_layers_[send_config->rtp.ssrcs[i]] =
4108               ScalabilityModeToNumTemporalLayers(scalability_mode_[i]);
4109         }
4110       }
4111     }
4112 
4113     struct ParsedPacket {
4114       uint32_t timestamp;
4115       uint32_t ssrc;
4116       int temporal_idx;
4117     };
4118 
4119     bool ParsePayload(const uint8_t* packet,
4120                       size_t length,
4121                       ParsedPacket& parsed) const {
4122       RtpPacket rtp_packet;
4123       EXPECT_TRUE(rtp_packet.Parse(packet, length));
4124 
4125       if (rtp_packet.payload_size() == 0) {
4126         return false;  // Padding packet.
4127       }
4128       parsed.timestamp = rtp_packet.Timestamp();
4129       parsed.ssrc = rtp_packet.Ssrc();
4130 
4131       absl::optional<VideoRtpDepacketizer::ParsedRtpPayload> parsed_payload =
4132           depacketizer_->Parse(rtp_packet.PayloadBuffer());
4133       EXPECT_TRUE(parsed_payload);
4134 
4135       if (const auto* vp8_header = absl::get_if<RTPVideoHeaderVP8>(
4136               &parsed_payload->video_header.video_type_header)) {
4137         parsed.temporal_idx = vp8_header->temporalIdx;
4138       } else if (const auto* vp9_header = absl::get_if<RTPVideoHeaderVP9>(
4139                      &parsed_payload->video_header.video_type_header)) {
4140         parsed.temporal_idx = vp9_header->temporal_idx;
4141       } else {
4142         RTC_DCHECK_NOTREACHED();
4143       }
4144       return true;
4145     }
4146 
4147     Action OnSendRtp(const uint8_t* packet, size_t length) override {
4148       ParsedPacket parsed;
4149       if (!ParsePayload(packet, length, parsed))
4150         return SEND_PACKET;
4151 
4152       uint32_t ssrc = parsed.ssrc;
4153       int temporal_idx =
4154           parsed.temporal_idx == kNoTemporalIdx ? 0 : parsed.temporal_idx;
4155       max_observed_tl_idxs_[ssrc] =
4156           std::max(temporal_idx, max_observed_tl_idxs_[ssrc]);
4157 
4158       if (last_observed_packet_.count(ssrc) == 0 ||
4159           parsed.timestamp != last_observed_packet_[ssrc].timestamp) {
4160         num_observed_frames_[ssrc]++;
4161       }
4162       last_observed_packet_[ssrc] = parsed;
4163 
4164       if (HighestTemporalLayerSentPerStream())
4165         observation_complete_.Set();
4166 
4167       return SEND_PACKET;
4168     }
4169 
4170     bool HighestTemporalLayerSentPerStream() const {
4171       if (num_observed_frames_.size() !=
4172           configured_num_temporal_layers_.size()) {
4173         return false;
4174       }
4175       for (const auto& num_frames : num_observed_frames_) {
4176         if (num_frames.second < kMinFramesToObservePerStream) {
4177           return false;
4178         }
4179       }
4180       if (max_observed_tl_idxs_.size() !=
4181           configured_num_temporal_layers_.size()) {
4182         return false;
4183       }
4184       for (const auto& max_tl_idx : max_observed_tl_idxs_) {
4185         uint32_t ssrc = max_tl_idx.first;
4186         int configured_num_tls =
4187             configured_num_temporal_layers_.find(ssrc)->second;
4188         if (max_tl_idx.second != configured_num_tls - 1)
4189           return false;
4190       }
4191       return true;
4192     }
4193 
4194     void PerformTest() override { EXPECT_TRUE(Wait()); }
4195 
4196     VideoEncoderFactory* const encoder_factory_;
4197     const std::string payload_name_;
4198     const std::vector<int> num_temporal_layers_;
4199     const std::vector<ScalabilityMode> scalability_mode_;
4200     const std::unique_ptr<VideoRtpDepacketizer> depacketizer_;
4201     // Mapped by SSRC.
4202     std::map<uint32_t, int> configured_num_temporal_layers_;
4203     std::map<uint32_t, int> max_observed_tl_idxs_;
4204     std::map<uint32_t, int> num_observed_frames_;
4205     std::map<uint32_t, ParsedPacket> last_observed_packet_;
4206   } test(encoder_factory, payload_name, num_temporal_layers, scalability_mode);
4207 
4208   RunBaseTest(&test);
4209 }
4210 
TEST_F(VideoSendStreamTest,TestTemporalLayersVp8)4211 TEST_F(VideoSendStreamTest, TestTemporalLayersVp8) {
4212   InternalEncoderFactory internal_encoder_factory;
4213   test::FunctionVideoEncoderFactory encoder_factory(
4214       [&internal_encoder_factory]() {
4215         return std::make_unique<SimulcastEncoderAdapter>(
4216             &internal_encoder_factory, SdpVideoFormat("VP8"));
4217       });
4218 
4219   TestTemporalLayers(&encoder_factory, "VP8",
4220                      /*num_temporal_layers=*/{2},
4221                      /*scalability_mode=*/{});
4222 }
4223 
TEST_F(VideoSendStreamTest,TestTemporalLayersVp8Simulcast)4224 TEST_F(VideoSendStreamTest, TestTemporalLayersVp8Simulcast) {
4225   InternalEncoderFactory internal_encoder_factory;
4226   test::FunctionVideoEncoderFactory encoder_factory(
4227       [&internal_encoder_factory]() {
4228         return std::make_unique<SimulcastEncoderAdapter>(
4229             &internal_encoder_factory, SdpVideoFormat("VP8"));
4230       });
4231 
4232   TestTemporalLayers(&encoder_factory, "VP8",
4233                      /*num_temporal_layers=*/{2, 2},
4234                      /*scalability_mode=*/{});
4235 }
4236 
TEST_F(VideoSendStreamTest,TestTemporalLayersVp8SimulcastWithDifferentNumTls)4237 TEST_F(VideoSendStreamTest, TestTemporalLayersVp8SimulcastWithDifferentNumTls) {
4238   InternalEncoderFactory internal_encoder_factory;
4239   test::FunctionVideoEncoderFactory encoder_factory(
4240       [&internal_encoder_factory]() {
4241         return std::make_unique<SimulcastEncoderAdapter>(
4242             &internal_encoder_factory, SdpVideoFormat("VP8"));
4243       });
4244 
4245   TestTemporalLayers(&encoder_factory, "VP8",
4246                      /*num_temporal_layers=*/{3, 1},
4247                      /*scalability_mode=*/{});
4248 }
4249 
TEST_F(VideoSendStreamTest,TestTemporalLayersVp8SimulcastWithoutSimAdapter)4250 TEST_F(VideoSendStreamTest, TestTemporalLayersVp8SimulcastWithoutSimAdapter) {
4251   test::FunctionVideoEncoderFactory encoder_factory(
4252       []() { return VP8Encoder::Create(); });
4253 
4254   TestTemporalLayers(&encoder_factory, "VP8",
4255                      /*num_temporal_layers=*/{2, 2},
4256                      /*scalability_mode=*/{});
4257 }
4258 
TEST_F(VideoSendStreamTest,TestScalabilityModeVp8L1T2)4259 TEST_F(VideoSendStreamTest, TestScalabilityModeVp8L1T2) {
4260   InternalEncoderFactory internal_encoder_factory;
4261   test::FunctionVideoEncoderFactory encoder_factory(
4262       [&internal_encoder_factory]() {
4263         return std::make_unique<SimulcastEncoderAdapter>(
4264             &internal_encoder_factory, SdpVideoFormat("VP8"));
4265       });
4266 
4267   TestTemporalLayers(&encoder_factory, "VP8",
4268                      /*num_temporal_layers=*/{}, {ScalabilityMode::kL1T2});
4269 }
4270 
TEST_F(VideoSendStreamTest,TestScalabilityModeVp8Simulcast)4271 TEST_F(VideoSendStreamTest, TestScalabilityModeVp8Simulcast) {
4272   InternalEncoderFactory internal_encoder_factory;
4273   test::FunctionVideoEncoderFactory encoder_factory(
4274       [&internal_encoder_factory]() {
4275         return std::make_unique<SimulcastEncoderAdapter>(
4276             &internal_encoder_factory, SdpVideoFormat("VP8"));
4277       });
4278 
4279   TestTemporalLayers(&encoder_factory, "VP8",
4280                      /*num_temporal_layers=*/{},
4281                      {ScalabilityMode::kL1T2, ScalabilityMode::kL1T2});
4282 }
4283 
TEST_F(VideoSendStreamTest,TestScalabilityModeVp8SimulcastWithDifferentMode)4284 TEST_F(VideoSendStreamTest, TestScalabilityModeVp8SimulcastWithDifferentMode) {
4285   InternalEncoderFactory internal_encoder_factory;
4286   test::FunctionVideoEncoderFactory encoder_factory(
4287       [&internal_encoder_factory]() {
4288         return std::make_unique<SimulcastEncoderAdapter>(
4289             &internal_encoder_factory, SdpVideoFormat("VP8"));
4290       });
4291 
4292   TestTemporalLayers(&encoder_factory, "VP8",
4293                      /*num_temporal_layers=*/{},
4294                      {ScalabilityMode::kL1T3, ScalabilityMode::kL1T1});
4295 }
4296 
TEST_F(VideoSendStreamTest,TestScalabilityModeVp8SimulcastWithoutSimAdapter)4297 TEST_F(VideoSendStreamTest, TestScalabilityModeVp8SimulcastWithoutSimAdapter) {
4298   test::FunctionVideoEncoderFactory encoder_factory(
4299       []() { return VP8Encoder::Create(); });
4300 
4301   TestTemporalLayers(&encoder_factory, "VP8",
4302                      /*num_temporal_layers=*/{},
4303                      {ScalabilityMode::kL1T2, ScalabilityMode::kL1T2});
4304 }
4305 
TEST_F(VideoSendStreamTest,TestTemporalLayersVp9)4306 TEST_F(VideoSendStreamTest, TestTemporalLayersVp9) {
4307   test::FunctionVideoEncoderFactory encoder_factory(
4308       []() { return VP9Encoder::Create(); });
4309 
4310   TestTemporalLayers(&encoder_factory, "VP9",
4311                      /*num_temporal_layers=*/{2},
4312                      /*scalability_mode=*/{});
4313 }
4314 
4315 }  // namespace webrtc
4316