1 /*
2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include <algorithm>
12 #include <limits>
13 #include <memory>
14 #include <string>
15
16 #include "absl/strings/string_view.h"
17 #include "api/audio_codecs/builtin_audio_encoder_factory.h"
18 #include "api/numerics/samples_stats_counter.h"
19 #include "api/rtc_event_log/rtc_event_log.h"
20 #include "api/task_queue/pending_task_safety_flag.h"
21 #include "api/task_queue/task_queue_base.h"
22 #include "api/test/metrics/global_metrics_logger_and_exporter.h"
23 #include "api/test/metrics/metric.h"
24 #include "api/test/simulated_network.h"
25 #include "api/video/builtin_video_bitrate_allocator_factory.h"
26 #include "api/video/video_bitrate_allocation.h"
27 #include "api/video_codecs/video_encoder.h"
28 #include "call/call.h"
29 #include "call/fake_network_pipe.h"
30 #include "call/simulated_network.h"
31 #include "media/engine/internal_encoder_factory.h"
32 #include "media/engine/simulcast_encoder_adapter.h"
33 #include "modules/audio_coding/include/audio_coding_module.h"
34 #include "modules/audio_device/include/test_audio_device.h"
35 #include "modules/audio_mixer/audio_mixer_impl.h"
36 #include "modules/rtp_rtcp/source/rtp_packet.h"
37 #include "rtc_base/checks.h"
38 #include "rtc_base/synchronization/mutex.h"
39 #include "rtc_base/task_queue_for_test.h"
40 #include "rtc_base/thread.h"
41 #include "rtc_base/thread_annotations.h"
42 #include "system_wrappers/include/metrics.h"
43 #include "test/call_test.h"
44 #include "test/direct_transport.h"
45 #include "test/drifting_clock.h"
46 #include "test/encoder_settings.h"
47 #include "test/fake_encoder.h"
48 #include "test/field_trial.h"
49 #include "test/frame_generator_capturer.h"
50 #include "test/gtest.h"
51 #include "test/null_transport.h"
52 #include "test/rtp_rtcp_observer.h"
53 #include "test/testsupport/file_utils.h"
54 #include "test/video_encoder_proxy_factory.h"
55 #include "video/config/video_encoder_config.h"
56 #include "video/transport_adapter.h"
57
58 using webrtc::test::DriftingClock;
59
60 namespace webrtc {
61 namespace {
62
63 using ::webrtc::test::GetGlobalMetricsLogger;
64 using ::webrtc::test::ImprovementDirection;
65 using ::webrtc::test::Unit;
66
67 enum : int { // The first valid value is 1.
68 kTransportSequenceNumberExtensionId = 1,
69 };
70
71 } // namespace
72
73 class CallPerfTest : public test::CallTest {
74 public:
CallPerfTest()75 CallPerfTest() {
76 RegisterRtpExtension(RtpExtension(RtpExtension::kTransportSequenceNumberUri,
77 kTransportSequenceNumberExtensionId));
78 }
79
80 protected:
81 enum class FecMode { kOn, kOff };
82 enum class CreateOrder { kAudioFirst, kVideoFirst };
83 void TestAudioVideoSync(FecMode fec,
84 CreateOrder create_first,
85 float video_ntp_speed,
86 float video_rtp_speed,
87 float audio_rtp_speed,
88 absl::string_view test_label);
89
90 void TestMinTransmitBitrate(bool pad_to_min_bitrate);
91
92 void TestCaptureNtpTime(const BuiltInNetworkBehaviorConfig& net_config,
93 int threshold_ms,
94 int start_time_ms,
95 int run_time_ms);
96 void TestMinAudioVideoBitrate(int test_bitrate_from,
97 int test_bitrate_to,
98 int test_bitrate_step,
99 int min_bwe,
100 int start_bwe,
101 int max_bwe);
102 void TestEncodeFramerate(VideoEncoderFactory* encoder_factory,
103 absl::string_view payload_name,
104 const std::vector<int>& max_framerates);
105 };
106
107 class VideoRtcpAndSyncObserver : public test::RtpRtcpObserver,
108 public rtc::VideoSinkInterface<VideoFrame> {
109 static const int kInSyncThresholdMs = 50;
110 static const int kStartupTimeMs = 2000;
111 static const int kMinRunTimeMs = 30000;
112
113 public:
VideoRtcpAndSyncObserver(TaskQueueBase * task_queue,Clock * clock,absl::string_view test_label)114 explicit VideoRtcpAndSyncObserver(TaskQueueBase* task_queue,
115 Clock* clock,
116 absl::string_view test_label)
117 : test::RtpRtcpObserver(CallPerfTest::kLongTimeout),
118 clock_(clock),
119 test_label_(test_label),
120 creation_time_ms_(clock_->TimeInMilliseconds()),
121 task_queue_(task_queue) {}
122
OnFrame(const VideoFrame & video_frame)123 void OnFrame(const VideoFrame& video_frame) override {
124 task_queue_->PostTask([this]() { CheckStats(); });
125 }
126
CheckStats()127 void CheckStats() {
128 if (!receive_stream_)
129 return;
130
131 VideoReceiveStreamInterface::Stats stats = receive_stream_->GetStats();
132 if (stats.sync_offset_ms == std::numeric_limits<int>::max())
133 return;
134
135 int64_t now_ms = clock_->TimeInMilliseconds();
136 int64_t time_since_creation = now_ms - creation_time_ms_;
137 // During the first couple of seconds audio and video can falsely be
138 // estimated as being synchronized. We don't want to trigger on those.
139 if (time_since_creation < kStartupTimeMs)
140 return;
141 if (std::abs(stats.sync_offset_ms) < kInSyncThresholdMs) {
142 if (first_time_in_sync_ == -1) {
143 first_time_in_sync_ = now_ms;
144 GetGlobalMetricsLogger()->LogSingleValueMetric(
145 "sync_convergence_time" + test_label_, "synchronization",
146 time_since_creation, Unit::kMilliseconds,
147 ImprovementDirection::kSmallerIsBetter);
148 }
149 if (time_since_creation > kMinRunTimeMs)
150 observation_complete_.Set();
151 }
152 if (first_time_in_sync_ != -1)
153 sync_offset_ms_list_.AddSample(stats.sync_offset_ms);
154 }
155
set_receive_stream(VideoReceiveStreamInterface * receive_stream)156 void set_receive_stream(VideoReceiveStreamInterface* receive_stream) {
157 RTC_DCHECK_EQ(task_queue_, TaskQueueBase::Current());
158 // Note that receive_stream may be nullptr.
159 receive_stream_ = receive_stream;
160 }
161
PrintResults()162 void PrintResults() {
163 GetGlobalMetricsLogger()->LogMetric(
164 "stream_offset" + test_label_, "synchronization", sync_offset_ms_list_,
165 Unit::kMilliseconds, ImprovementDirection::kNeitherIsBetter);
166 }
167
168 private:
169 Clock* const clock_;
170 const std::string test_label_;
171 const int64_t creation_time_ms_;
172 int64_t first_time_in_sync_ = -1;
173 VideoReceiveStreamInterface* receive_stream_ = nullptr;
174 SamplesStatsCounter sync_offset_ms_list_;
175 TaskQueueBase* const task_queue_;
176 };
177
TestAudioVideoSync(FecMode fec,CreateOrder create_first,float video_ntp_speed,float video_rtp_speed,float audio_rtp_speed,absl::string_view test_label)178 void CallPerfTest::TestAudioVideoSync(FecMode fec,
179 CreateOrder create_first,
180 float video_ntp_speed,
181 float video_rtp_speed,
182 float audio_rtp_speed,
183 absl::string_view test_label) {
184 const char* kSyncGroup = "av_sync";
185 const uint32_t kAudioSendSsrc = 1234;
186 const uint32_t kAudioRecvSsrc = 5678;
187
188 BuiltInNetworkBehaviorConfig audio_net_config;
189 audio_net_config.queue_delay_ms = 500;
190 audio_net_config.loss_percent = 5;
191
192 auto observer = std::make_unique<VideoRtcpAndSyncObserver>(
193 task_queue(), Clock::GetRealTimeClock(), test_label);
194
195 std::map<uint8_t, MediaType> audio_pt_map;
196 std::map<uint8_t, MediaType> video_pt_map;
197
198 std::unique_ptr<test::PacketTransport> audio_send_transport;
199 std::unique_ptr<test::PacketTransport> video_send_transport;
200 std::unique_ptr<test::PacketTransport> receive_transport;
201
202 AudioSendStream* audio_send_stream;
203 AudioReceiveStreamInterface* audio_receive_stream;
204 std::unique_ptr<DriftingClock> drifting_clock;
205
206 SendTask(task_queue(), [&]() {
207 metrics::Reset();
208 rtc::scoped_refptr<TestAudioDeviceModule> fake_audio_device =
209 TestAudioDeviceModule::Create(
210 task_queue_factory_.get(),
211 TestAudioDeviceModule::CreatePulsedNoiseCapturer(256, 48000),
212 TestAudioDeviceModule::CreateDiscardRenderer(48000),
213 audio_rtp_speed);
214 EXPECT_EQ(0, fake_audio_device->Init());
215
216 AudioState::Config send_audio_state_config;
217 send_audio_state_config.audio_mixer = AudioMixerImpl::Create();
218 send_audio_state_config.audio_processing =
219 AudioProcessingBuilder().Create();
220 send_audio_state_config.audio_device_module = fake_audio_device;
221 Call::Config sender_config(send_event_log_.get());
222
223 auto audio_state = AudioState::Create(send_audio_state_config);
224 fake_audio_device->RegisterAudioCallback(audio_state->audio_transport());
225 sender_config.audio_state = audio_state;
226 Call::Config receiver_config(recv_event_log_.get());
227 receiver_config.audio_state = audio_state;
228 CreateCalls(sender_config, receiver_config);
229
230 std::copy_if(std::begin(payload_type_map_), std::end(payload_type_map_),
231 std::inserter(audio_pt_map, audio_pt_map.end()),
232 [](const std::pair<const uint8_t, MediaType>& pair) {
233 return pair.second == MediaType::AUDIO;
234 });
235 std::copy_if(std::begin(payload_type_map_), std::end(payload_type_map_),
236 std::inserter(video_pt_map, video_pt_map.end()),
237 [](const std::pair<const uint8_t, MediaType>& pair) {
238 return pair.second == MediaType::VIDEO;
239 });
240
241 audio_send_transport = std::make_unique<test::PacketTransport>(
242 task_queue(), sender_call_.get(), observer.get(),
243 test::PacketTransport::kSender, audio_pt_map,
244 std::make_unique<FakeNetworkPipe>(
245 Clock::GetRealTimeClock(),
246 std::make_unique<SimulatedNetwork>(audio_net_config)));
247 audio_send_transport->SetReceiver(receiver_call_->Receiver());
248
249 video_send_transport = std::make_unique<test::PacketTransport>(
250 task_queue(), sender_call_.get(), observer.get(),
251 test::PacketTransport::kSender, video_pt_map,
252 std::make_unique<FakeNetworkPipe>(Clock::GetRealTimeClock(),
253 std::make_unique<SimulatedNetwork>(
254 BuiltInNetworkBehaviorConfig())));
255 video_send_transport->SetReceiver(receiver_call_->Receiver());
256
257 receive_transport = std::make_unique<test::PacketTransport>(
258 task_queue(), receiver_call_.get(), observer.get(),
259 test::PacketTransport::kReceiver, payload_type_map_,
260 std::make_unique<FakeNetworkPipe>(Clock::GetRealTimeClock(),
261 std::make_unique<SimulatedNetwork>(
262 BuiltInNetworkBehaviorConfig())));
263 receive_transport->SetReceiver(sender_call_->Receiver());
264
265 CreateSendConfig(1, 0, 0, video_send_transport.get());
266 CreateMatchingReceiveConfigs(receive_transport.get());
267
268 AudioSendStream::Config audio_send_config(audio_send_transport.get());
269 audio_send_config.rtp.ssrc = kAudioSendSsrc;
270 // TODO(bugs.webrtc.org/14683): Let the tests fail with invalid config.
271 audio_send_config.send_codec_spec = AudioSendStream::Config::SendCodecSpec(
272 kAudioSendPayloadType, {"OPUS", 48000, 2});
273 audio_send_config.min_bitrate_bps = 6000;
274 audio_send_config.max_bitrate_bps = 510000;
275 audio_send_config.encoder_factory = CreateBuiltinAudioEncoderFactory();
276 audio_send_stream = sender_call_->CreateAudioSendStream(audio_send_config);
277
278 GetVideoSendConfig()->rtp.nack.rtp_history_ms = kNackRtpHistoryMs;
279 if (fec == FecMode::kOn) {
280 GetVideoSendConfig()->rtp.ulpfec.red_payload_type = kRedPayloadType;
281 GetVideoSendConfig()->rtp.ulpfec.ulpfec_payload_type = kUlpfecPayloadType;
282 video_receive_configs_[0].rtp.red_payload_type = kRedPayloadType;
283 video_receive_configs_[0].rtp.ulpfec_payload_type = kUlpfecPayloadType;
284 }
285 video_receive_configs_[0].rtp.nack.rtp_history_ms = 1000;
286 video_receive_configs_[0].renderer = observer.get();
287 video_receive_configs_[0].sync_group = kSyncGroup;
288
289 AudioReceiveStreamInterface::Config audio_recv_config;
290 audio_recv_config.rtp.remote_ssrc = kAudioSendSsrc;
291 audio_recv_config.rtp.local_ssrc = kAudioRecvSsrc;
292 audio_recv_config.rtcp_send_transport = receive_transport.get();
293 audio_recv_config.sync_group = kSyncGroup;
294 audio_recv_config.decoder_factory = audio_decoder_factory_;
295 audio_recv_config.decoder_map = {
296 {kAudioSendPayloadType, {"OPUS", 48000, 2}}};
297
298 if (create_first == CreateOrder::kAudioFirst) {
299 audio_receive_stream =
300 receiver_call_->CreateAudioReceiveStream(audio_recv_config);
301 CreateVideoStreams();
302 } else {
303 CreateVideoStreams();
304 audio_receive_stream =
305 receiver_call_->CreateAudioReceiveStream(audio_recv_config);
306 }
307 EXPECT_EQ(1u, video_receive_streams_.size());
308 observer->set_receive_stream(video_receive_streams_[0]);
309 drifting_clock = std::make_unique<DriftingClock>(clock_, video_ntp_speed);
310 CreateFrameGeneratorCapturerWithDrift(drifting_clock.get(), video_rtp_speed,
311 kDefaultFramerate, kDefaultWidth,
312 kDefaultHeight);
313
314 Start();
315
316 audio_send_stream->Start();
317 audio_receive_stream->Start();
318 });
319
320 EXPECT_TRUE(observer->Wait())
321 << "Timed out while waiting for audio and video to be synchronized.";
322
323 SendTask(task_queue(), [&]() {
324 // Clear the pointer to the receive stream since it will now be deleted.
325 observer->set_receive_stream(nullptr);
326
327 audio_send_stream->Stop();
328 audio_receive_stream->Stop();
329
330 Stop();
331
332 DestroyStreams();
333
334 sender_call_->DestroyAudioSendStream(audio_send_stream);
335 receiver_call_->DestroyAudioReceiveStream(audio_receive_stream);
336
337 DestroyCalls();
338 // Call may post periodic rtcp packet to the transport on the process
339 // thread, thus transport should be destroyed after the call objects.
340 // Though transports keep pointers to the call objects, transports handle
341 // packets on the task_queue() and thus wouldn't create a race while current
342 // destruction happens in the same task as destruction of the call objects.
343 video_send_transport.reset();
344 audio_send_transport.reset();
345 receive_transport.reset();
346 });
347
348 observer->PrintResults();
349
350 // In quick test synchronization may not be achieved in time.
351 if (!field_trial::IsEnabled("WebRTC-QuickPerfTest")) {
352 // TODO(bugs.webrtc.org/10417): Reenable this for iOS
353 #if !defined(WEBRTC_IOS)
354 EXPECT_METRIC_EQ(1, metrics::NumSamples("WebRTC.Video.AVSyncOffsetInMs"));
355 #endif
356 }
357
358 task_queue()->PostTask(
359 [to_delete = observer.release()]() { delete to_delete; });
360 }
361
TEST_F(CallPerfTest,Synchronization_PlaysOutAudioAndVideoWithoutClockDrift)362 TEST_F(CallPerfTest, Synchronization_PlaysOutAudioAndVideoWithoutClockDrift) {
363 TestAudioVideoSync(FecMode::kOff, CreateOrder::kAudioFirst,
364 DriftingClock::kNoDrift, DriftingClock::kNoDrift,
365 DriftingClock::kNoDrift, "_video_no_drift");
366 }
367
TEST_F(CallPerfTest,Synchronization_PlaysOutAudioAndVideoWithVideoNtpDrift)368 TEST_F(CallPerfTest, Synchronization_PlaysOutAudioAndVideoWithVideoNtpDrift) {
369 TestAudioVideoSync(FecMode::kOff, CreateOrder::kAudioFirst,
370 DriftingClock::PercentsFaster(10.0f),
371 DriftingClock::kNoDrift, DriftingClock::kNoDrift,
372 "_video_ntp_drift");
373 }
374
TEST_F(CallPerfTest,Synchronization_PlaysOutAudioAndVideoWithAudioFasterThanVideoDrift)375 TEST_F(CallPerfTest,
376 Synchronization_PlaysOutAudioAndVideoWithAudioFasterThanVideoDrift) {
377 TestAudioVideoSync(FecMode::kOff, CreateOrder::kAudioFirst,
378 DriftingClock::kNoDrift,
379 DriftingClock::PercentsSlower(30.0f),
380 DriftingClock::PercentsFaster(30.0f), "_audio_faster");
381 }
382
TEST_F(CallPerfTest,Synchronization_PlaysOutAudioAndVideoWithVideoFasterThanAudioDrift)383 TEST_F(CallPerfTest,
384 Synchronization_PlaysOutAudioAndVideoWithVideoFasterThanAudioDrift) {
385 TestAudioVideoSync(FecMode::kOn, CreateOrder::kVideoFirst,
386 DriftingClock::kNoDrift,
387 DriftingClock::PercentsFaster(30.0f),
388 DriftingClock::PercentsSlower(30.0f), "_video_faster");
389 }
390
TestCaptureNtpTime(const BuiltInNetworkBehaviorConfig & net_config,int threshold_ms,int start_time_ms,int run_time_ms)391 void CallPerfTest::TestCaptureNtpTime(
392 const BuiltInNetworkBehaviorConfig& net_config,
393 int threshold_ms,
394 int start_time_ms,
395 int run_time_ms) {
396 class CaptureNtpTimeObserver : public test::EndToEndTest,
397 public rtc::VideoSinkInterface<VideoFrame> {
398 public:
399 CaptureNtpTimeObserver(const BuiltInNetworkBehaviorConfig& net_config,
400 int threshold_ms,
401 int start_time_ms,
402 int run_time_ms)
403 : EndToEndTest(kLongTimeout),
404 net_config_(net_config),
405 clock_(Clock::GetRealTimeClock()),
406 threshold_ms_(threshold_ms),
407 start_time_ms_(start_time_ms),
408 run_time_ms_(run_time_ms),
409 creation_time_ms_(clock_->TimeInMilliseconds()),
410 capturer_(nullptr),
411 rtp_start_timestamp_set_(false),
412 rtp_start_timestamp_(0) {}
413
414 private:
415 std::unique_ptr<test::PacketTransport> CreateSendTransport(
416 TaskQueueBase* task_queue,
417 Call* sender_call) override {
418 return std::make_unique<test::PacketTransport>(
419 task_queue, sender_call, this, test::PacketTransport::kSender,
420 payload_type_map_,
421 std::make_unique<FakeNetworkPipe>(
422 Clock::GetRealTimeClock(),
423 std::make_unique<SimulatedNetwork>(net_config_)));
424 }
425
426 std::unique_ptr<test::PacketTransport> CreateReceiveTransport(
427 TaskQueueBase* task_queue) override {
428 return std::make_unique<test::PacketTransport>(
429 task_queue, nullptr, this, test::PacketTransport::kReceiver,
430 payload_type_map_,
431 std::make_unique<FakeNetworkPipe>(
432 Clock::GetRealTimeClock(),
433 std::make_unique<SimulatedNetwork>(net_config_)));
434 }
435
436 void OnFrame(const VideoFrame& video_frame) override {
437 MutexLock lock(&mutex_);
438 if (video_frame.ntp_time_ms() <= 0) {
439 // Haven't got enough RTCP SR in order to calculate the capture ntp
440 // time.
441 return;
442 }
443
444 int64_t now_ms = clock_->TimeInMilliseconds();
445 int64_t time_since_creation = now_ms - creation_time_ms_;
446 if (time_since_creation < start_time_ms_) {
447 // Wait for `start_time_ms_` before start measuring.
448 return;
449 }
450
451 if (time_since_creation > run_time_ms_) {
452 observation_complete_.Set();
453 }
454
455 FrameCaptureTimeList::iterator iter =
456 capture_time_list_.find(video_frame.timestamp());
457 EXPECT_TRUE(iter != capture_time_list_.end());
458
459 // The real capture time has been wrapped to uint32_t before converted
460 // to rtp timestamp in the sender side. So here we convert the estimated
461 // capture time to a uint32_t 90k timestamp also for comparing.
462 uint32_t estimated_capture_timestamp =
463 90 * static_cast<uint32_t>(video_frame.ntp_time_ms());
464 uint32_t real_capture_timestamp = iter->second;
465 int time_offset_ms = real_capture_timestamp - estimated_capture_timestamp;
466 time_offset_ms = time_offset_ms / 90;
467 time_offset_ms_list_.AddSample(time_offset_ms);
468
469 EXPECT_TRUE(std::abs(time_offset_ms) < threshold_ms_);
470 }
471
472 Action OnSendRtp(const uint8_t* packet, size_t length) override {
473 MutexLock lock(&mutex_);
474 RtpPacket rtp_packet;
475 EXPECT_TRUE(rtp_packet.Parse(packet, length));
476
477 if (!rtp_start_timestamp_set_) {
478 // Calculate the rtp timestamp offset in order to calculate the real
479 // capture time.
480 uint32_t first_capture_timestamp =
481 90 * static_cast<uint32_t>(capturer_->first_frame_capture_time());
482 rtp_start_timestamp_ = rtp_packet.Timestamp() - first_capture_timestamp;
483 rtp_start_timestamp_set_ = true;
484 }
485
486 uint32_t capture_timestamp =
487 rtp_packet.Timestamp() - rtp_start_timestamp_;
488 capture_time_list_.insert(
489 capture_time_list_.end(),
490 std::make_pair(rtp_packet.Timestamp(), capture_timestamp));
491 return SEND_PACKET;
492 }
493
494 void OnFrameGeneratorCapturerCreated(
495 test::FrameGeneratorCapturer* frame_generator_capturer) override {
496 capturer_ = frame_generator_capturer;
497 }
498
499 void ModifyVideoConfigs(
500 VideoSendStream::Config* send_config,
501 std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
502 VideoEncoderConfig* encoder_config) override {
503 (*receive_configs)[0].renderer = this;
504 // Enable the receiver side rtt calculation.
505 (*receive_configs)[0].rtp.rtcp_xr.receiver_reference_time_report = true;
506 }
507
508 void PerformTest() override {
509 EXPECT_TRUE(Wait()) << "Timed out while waiting for estimated capture "
510 "NTP time to be within bounds.";
511 GetGlobalMetricsLogger()->LogMetric(
512 "capture_ntp_time", "real - estimated", time_offset_ms_list_,
513 Unit::kMilliseconds, ImprovementDirection::kNeitherIsBetter);
514 }
515
516 Mutex mutex_;
517 const BuiltInNetworkBehaviorConfig net_config_;
518 Clock* const clock_;
519 const int threshold_ms_;
520 const int start_time_ms_;
521 const int run_time_ms_;
522 const int64_t creation_time_ms_;
523 test::FrameGeneratorCapturer* capturer_;
524 bool rtp_start_timestamp_set_;
525 uint32_t rtp_start_timestamp_;
526 typedef std::map<uint32_t, uint32_t> FrameCaptureTimeList;
527 FrameCaptureTimeList capture_time_list_ RTC_GUARDED_BY(&mutex_);
528 SamplesStatsCounter time_offset_ms_list_;
529 } test(net_config, threshold_ms, start_time_ms, run_time_ms);
530
531 RunBaseTest(&test);
532 }
533
534 // Flaky tests, disabled on Mac and Windows due to webrtc:8291.
535 #if !(defined(WEBRTC_MAC) || defined(WEBRTC_WIN))
TEST_F(CallPerfTest,Real_Estimated_CaptureNtpTimeWithNetworkDelay)536 TEST_F(CallPerfTest, Real_Estimated_CaptureNtpTimeWithNetworkDelay) {
537 BuiltInNetworkBehaviorConfig net_config;
538 net_config.queue_delay_ms = 100;
539 // TODO(wu): lower the threshold as the calculation/estimation becomes more
540 // accurate.
541 const int kThresholdMs = 100;
542 const int kStartTimeMs = 10000;
543 const int kRunTimeMs = 20000;
544 TestCaptureNtpTime(net_config, kThresholdMs, kStartTimeMs, kRunTimeMs);
545 }
546
TEST_F(CallPerfTest,Real_Estimated_CaptureNtpTimeWithNetworkJitter)547 TEST_F(CallPerfTest, Real_Estimated_CaptureNtpTimeWithNetworkJitter) {
548 BuiltInNetworkBehaviorConfig net_config;
549 net_config.queue_delay_ms = 100;
550 net_config.delay_standard_deviation_ms = 10;
551 // TODO(wu): lower the threshold as the calculation/estimation becomes more
552 // accurate.
553 const int kThresholdMs = 100;
554 const int kStartTimeMs = 10000;
555 const int kRunTimeMs = 20000;
556 TestCaptureNtpTime(net_config, kThresholdMs, kStartTimeMs, kRunTimeMs);
557 }
558 #endif
559
TEST_F(CallPerfTest,ReceivesCpuOveruseAndUnderuse)560 TEST_F(CallPerfTest, ReceivesCpuOveruseAndUnderuse) {
561 // Minimal normal usage at the start, then 30s overuse to allow filter to
562 // settle, and then 80s underuse to allow plenty of time for rampup again.
563 test::ScopedFieldTrials fake_overuse_settings(
564 "WebRTC-ForceSimulatedOveruseIntervalMs/1-30000-80000/");
565
566 class LoadObserver : public test::SendTest,
567 public test::FrameGeneratorCapturer::SinkWantsObserver {
568 public:
569 LoadObserver() : SendTest(kLongTimeout), test_phase_(TestPhase::kInit) {}
570
571 void OnFrameGeneratorCapturerCreated(
572 test::FrameGeneratorCapturer* frame_generator_capturer) override {
573 frame_generator_capturer->SetSinkWantsObserver(this);
574 // Set a high initial resolution to be sure that we can scale down.
575 frame_generator_capturer->ChangeResolution(1920, 1080);
576 }
577
578 // OnSinkWantsChanged is called when FrameGeneratorCapturer::AddOrUpdateSink
579 // is called.
580 // TODO(sprang): Add integration test for maintain-framerate mode?
581 void OnSinkWantsChanged(rtc::VideoSinkInterface<VideoFrame>* sink,
582 const rtc::VideoSinkWants& wants) override {
583 // The sink wants can change either because an adaptation happened (i.e.
584 // the pixels or frame rate changed) or for other reasons, such as encoded
585 // resolutions being communicated (happens whenever we capture a new frame
586 // size). In this test, we only care about adaptations.
587 bool did_adapt =
588 last_wants_.max_pixel_count != wants.max_pixel_count ||
589 last_wants_.target_pixel_count != wants.target_pixel_count ||
590 last_wants_.max_framerate_fps != wants.max_framerate_fps;
591 last_wants_ = wants;
592 if (!did_adapt) {
593 return;
594 }
595 // At kStart expect CPU overuse. Then expect CPU underuse when the encoder
596 // delay has been decreased.
597 switch (test_phase_) {
598 case TestPhase::kInit:
599 // Max framerate should be set initially.
600 if (wants.max_framerate_fps != std::numeric_limits<int>::max() &&
601 wants.max_pixel_count == std::numeric_limits<int>::max()) {
602 test_phase_ = TestPhase::kStart;
603 } else {
604 ADD_FAILURE() << "Got unexpected adaptation request, max res = "
605 << wants.max_pixel_count << ", target res = "
606 << wants.target_pixel_count.value_or(-1)
607 << ", max fps = " << wants.max_framerate_fps;
608 }
609 break;
610 case TestPhase::kStart:
611 if (wants.max_pixel_count < std::numeric_limits<int>::max()) {
612 // On adapting down, VideoStreamEncoder::VideoSourceProxy will set
613 // only the max pixel count, leaving the target unset.
614 test_phase_ = TestPhase::kAdaptedDown;
615 } else {
616 ADD_FAILURE() << "Got unexpected adaptation request, max res = "
617 << wants.max_pixel_count << ", target res = "
618 << wants.target_pixel_count.value_or(-1)
619 << ", max fps = " << wants.max_framerate_fps;
620 }
621 break;
622 case TestPhase::kAdaptedDown:
623 // On adapting up, the adaptation counter will again be at zero, and
624 // so all constraints will be reset.
625 if (wants.max_pixel_count == std::numeric_limits<int>::max() &&
626 !wants.target_pixel_count) {
627 test_phase_ = TestPhase::kAdaptedUp;
628 observation_complete_.Set();
629 } else {
630 ADD_FAILURE() << "Got unexpected adaptation request, max res = "
631 << wants.max_pixel_count << ", target res = "
632 << wants.target_pixel_count.value_or(-1)
633 << ", max fps = " << wants.max_framerate_fps;
634 }
635 break;
636 case TestPhase::kAdaptedUp:
637 ADD_FAILURE() << "Got unexpected adaptation request, max res = "
638 << wants.max_pixel_count << ", target res = "
639 << wants.target_pixel_count.value_or(-1)
640 << ", max fps = " << wants.max_framerate_fps;
641 }
642 }
643
644 void ModifyVideoConfigs(
645 VideoSendStream::Config* send_config,
646 std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
647 VideoEncoderConfig* encoder_config) override {}
648
649 void PerformTest() override {
650 EXPECT_TRUE(Wait()) << "Timed out before receiving an overuse callback.";
651 }
652
653 enum class TestPhase {
654 kInit,
655 kStart,
656 kAdaptedDown,
657 kAdaptedUp
658 } test_phase_;
659
660 private:
661 rtc::VideoSinkWants last_wants_;
662 } test;
663
664 RunBaseTest(&test);
665 }
666
TestMinTransmitBitrate(bool pad_to_min_bitrate)667 void CallPerfTest::TestMinTransmitBitrate(bool pad_to_min_bitrate) {
668 static const int kMaxEncodeBitrateKbps = 30;
669 static const int kMinTransmitBitrateBps = 150000;
670 static const int kMinAcceptableTransmitBitrate = 130;
671 static const int kMaxAcceptableTransmitBitrate = 170;
672 static const int kNumBitrateObservationsInRange = 100;
673 static const int kAcceptableBitrateErrorMargin = 15; // +- 7
674 class BitrateObserver : public test::EndToEndTest {
675 public:
676 explicit BitrateObserver(bool using_min_transmit_bitrate,
677 TaskQueueBase* task_queue)
678 : EndToEndTest(kLongTimeout),
679 send_stream_(nullptr),
680 converged_(false),
681 pad_to_min_bitrate_(using_min_transmit_bitrate),
682 min_acceptable_bitrate_(using_min_transmit_bitrate
683 ? kMinAcceptableTransmitBitrate
684 : (kMaxEncodeBitrateKbps -
685 kAcceptableBitrateErrorMargin / 2)),
686 max_acceptable_bitrate_(using_min_transmit_bitrate
687 ? kMaxAcceptableTransmitBitrate
688 : (kMaxEncodeBitrateKbps +
689 kAcceptableBitrateErrorMargin / 2)),
690 num_bitrate_observations_in_range_(0),
691 task_queue_(task_queue),
692 task_safety_flag_(PendingTaskSafetyFlag::CreateDetached()) {}
693
694 private:
695 // TODO(holmer): Run this with a timer instead of once per packet.
696 Action OnSendRtp(const uint8_t* packet, size_t length) override {
697 task_queue_->PostTask(SafeTask(task_safety_flag_, [this]() {
698 VideoSendStream::Stats stats = send_stream_->GetStats();
699
700 if (!stats.substreams.empty()) {
701 RTC_DCHECK_EQ(1, stats.substreams.size());
702 int bitrate_kbps =
703 stats.substreams.begin()->second.total_bitrate_bps / 1000;
704 if (bitrate_kbps > min_acceptable_bitrate_ &&
705 bitrate_kbps < max_acceptable_bitrate_) {
706 converged_ = true;
707 ++num_bitrate_observations_in_range_;
708 if (num_bitrate_observations_in_range_ ==
709 kNumBitrateObservationsInRange)
710 observation_complete_.Set();
711 }
712 if (converged_)
713 bitrate_kbps_list_.AddSample(bitrate_kbps);
714 }
715 }));
716 return SEND_PACKET;
717 }
718
719 void OnVideoStreamsCreated(VideoSendStream* send_stream,
720 const std::vector<VideoReceiveStreamInterface*>&
721 receive_streams) override {
722 send_stream_ = send_stream;
723 }
724
725 void OnStreamsStopped() override { task_safety_flag_->SetNotAlive(); }
726
727 void ModifyVideoConfigs(
728 VideoSendStream::Config* send_config,
729 std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
730 VideoEncoderConfig* encoder_config) override {
731 if (pad_to_min_bitrate_) {
732 encoder_config->min_transmit_bitrate_bps = kMinTransmitBitrateBps;
733 } else {
734 RTC_DCHECK_EQ(0, encoder_config->min_transmit_bitrate_bps);
735 }
736 }
737
738 void PerformTest() override {
739 EXPECT_TRUE(Wait()) << "Timeout while waiting for send-bitrate stats.";
740 GetGlobalMetricsLogger()->LogMetric(
741 std::string("bitrate_stats_") +
742 (pad_to_min_bitrate_ ? "min_transmit_bitrate"
743 : "without_min_transmit_bitrate"),
744 "bitrate_kbps", bitrate_kbps_list_, Unit::kUnitless,
745 ImprovementDirection::kNeitherIsBetter);
746 }
747
748 VideoSendStream* send_stream_;
749 bool converged_;
750 const bool pad_to_min_bitrate_;
751 const int min_acceptable_bitrate_;
752 const int max_acceptable_bitrate_;
753 int num_bitrate_observations_in_range_;
754 SamplesStatsCounter bitrate_kbps_list_;
755 TaskQueueBase* task_queue_;
756 rtc::scoped_refptr<PendingTaskSafetyFlag> task_safety_flag_;
757 } test(pad_to_min_bitrate, task_queue());
758
759 fake_encoder_max_bitrate_ = kMaxEncodeBitrateKbps;
760 RunBaseTest(&test);
761 }
762
TEST_F(CallPerfTest,Bitrate_Kbps_PadsToMinTransmitBitrate)763 TEST_F(CallPerfTest, Bitrate_Kbps_PadsToMinTransmitBitrate) {
764 TestMinTransmitBitrate(true);
765 }
766
TEST_F(CallPerfTest,Bitrate_Kbps_NoPadWithoutMinTransmitBitrate)767 TEST_F(CallPerfTest, Bitrate_Kbps_NoPadWithoutMinTransmitBitrate) {
768 TestMinTransmitBitrate(false);
769 }
770
771 // TODO(bugs.webrtc.org/8878)
772 #if defined(WEBRTC_MAC)
773 #define MAYBE_KeepsHighBitrateWhenReconfiguringSender \
774 DISABLED_KeepsHighBitrateWhenReconfiguringSender
775 #else
776 #define MAYBE_KeepsHighBitrateWhenReconfiguringSender \
777 KeepsHighBitrateWhenReconfiguringSender
778 #endif
TEST_F(CallPerfTest,MAYBE_KeepsHighBitrateWhenReconfiguringSender)779 TEST_F(CallPerfTest, MAYBE_KeepsHighBitrateWhenReconfiguringSender) {
780 static const uint32_t kInitialBitrateKbps = 400;
781 static const uint32_t kInitialBitrateOverheadKpbs = 6;
782 static const uint32_t kReconfigureThresholdKbps = 600;
783
784 class VideoStreamFactory
785 : public VideoEncoderConfig::VideoStreamFactoryInterface {
786 public:
787 VideoStreamFactory() {}
788
789 private:
790 std::vector<VideoStream> CreateEncoderStreams(
791 int frame_width,
792 int frame_height,
793 const webrtc::VideoEncoderConfig& encoder_config) override {
794 std::vector<VideoStream> streams =
795 test::CreateVideoStreams(frame_width, frame_height, encoder_config);
796 streams[0].min_bitrate_bps = 50000;
797 streams[0].target_bitrate_bps = streams[0].max_bitrate_bps = 2000000;
798 return streams;
799 }
800 };
801
802 class BitrateObserver : public test::EndToEndTest, public test::FakeEncoder {
803 public:
804 explicit BitrateObserver(TaskQueueBase* task_queue)
805 : EndToEndTest(kDefaultTimeout),
806 FakeEncoder(Clock::GetRealTimeClock()),
807 encoder_inits_(0),
808 last_set_bitrate_kbps_(0),
809 send_stream_(nullptr),
810 frame_generator_(nullptr),
811 encoder_factory_(this),
812 bitrate_allocator_factory_(
813 CreateBuiltinVideoBitrateAllocatorFactory()),
814 task_queue_(task_queue) {}
815
816 int32_t InitEncode(const VideoCodec* config,
817 const VideoEncoder::Settings& settings) override {
818 ++encoder_inits_;
819 if (encoder_inits_ == 1) {
820 // First time initialization. Frame size is known.
821 // `expected_bitrate` is affected by bandwidth estimation before the
822 // first frame arrives to the encoder.
823 uint32_t expected_bitrate =
824 last_set_bitrate_kbps_ > 0
825 ? last_set_bitrate_kbps_
826 : kInitialBitrateKbps - kInitialBitrateOverheadKpbs;
827 EXPECT_EQ(expected_bitrate, config->startBitrate)
828 << "Encoder not initialized at expected bitrate.";
829 EXPECT_EQ(kDefaultWidth, config->width);
830 EXPECT_EQ(kDefaultHeight, config->height);
831 } else if (encoder_inits_ == 2) {
832 EXPECT_EQ(2 * kDefaultWidth, config->width);
833 EXPECT_EQ(2 * kDefaultHeight, config->height);
834 EXPECT_GE(last_set_bitrate_kbps_, kReconfigureThresholdKbps);
835 EXPECT_GT(config->startBitrate, kReconfigureThresholdKbps)
836 << "Encoder reconfigured with bitrate too far away from last set.";
837 observation_complete_.Set();
838 }
839 return FakeEncoder::InitEncode(config, settings);
840 }
841
842 void SetRates(const RateControlParameters& parameters) override {
843 last_set_bitrate_kbps_ = parameters.bitrate.get_sum_kbps();
844 if (encoder_inits_ == 1 &&
845 parameters.bitrate.get_sum_kbps() > kReconfigureThresholdKbps) {
846 time_to_reconfigure_.Set();
847 }
848 FakeEncoder::SetRates(parameters);
849 }
850
851 void ModifySenderBitrateConfig(
852 BitrateConstraints* bitrate_config) override {
853 bitrate_config->start_bitrate_bps = kInitialBitrateKbps * 1000;
854 }
855
856 void ModifyVideoConfigs(
857 VideoSendStream::Config* send_config,
858 std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
859 VideoEncoderConfig* encoder_config) override {
860 send_config->encoder_settings.encoder_factory = &encoder_factory_;
861 send_config->encoder_settings.bitrate_allocator_factory =
862 bitrate_allocator_factory_.get();
863 encoder_config->max_bitrate_bps = 2 * kReconfigureThresholdKbps * 1000;
864 encoder_config->video_stream_factory =
865 rtc::make_ref_counted<VideoStreamFactory>();
866
867 encoder_config_ = encoder_config->Copy();
868 }
869
870 void OnVideoStreamsCreated(VideoSendStream* send_stream,
871 const std::vector<VideoReceiveStreamInterface*>&
872 receive_streams) override {
873 send_stream_ = send_stream;
874 }
875
876 void OnFrameGeneratorCapturerCreated(
877 test::FrameGeneratorCapturer* frame_generator_capturer) override {
878 frame_generator_ = frame_generator_capturer;
879 }
880
881 void PerformTest() override {
882 ASSERT_TRUE(time_to_reconfigure_.Wait(kDefaultTimeout))
883 << "Timed out before receiving an initial high bitrate.";
884 frame_generator_->ChangeResolution(kDefaultWidth * 2, kDefaultHeight * 2);
885 SendTask(task_queue_, [&]() {
886 send_stream_->ReconfigureVideoEncoder(encoder_config_.Copy());
887 });
888 EXPECT_TRUE(Wait())
889 << "Timed out while waiting for a couple of high bitrate estimates "
890 "after reconfiguring the send stream.";
891 }
892
893 private:
894 rtc::Event time_to_reconfigure_;
895 int encoder_inits_;
896 uint32_t last_set_bitrate_kbps_;
897 VideoSendStream* send_stream_;
898 test::FrameGeneratorCapturer* frame_generator_;
899 test::VideoEncoderProxyFactory encoder_factory_;
900 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
901 VideoEncoderConfig encoder_config_;
902 TaskQueueBase* task_queue_;
903 } test(task_queue());
904
905 RunBaseTest(&test);
906 }
907
908 // Discovers the minimal supported audio+video bitrate. The test bitrate is
909 // considered supported if Rtt does not go above 400ms with the network
910 // contrained to the test bitrate.
911 //
912 // |test_bitrate_from test_bitrate_to| bitrate constraint range
913 // `test_bitrate_step` bitrate constraint update step during the test
914 // |min_bwe max_bwe| BWE range
915 // `start_bwe` initial BWE
TestMinAudioVideoBitrate(int test_bitrate_from,int test_bitrate_to,int test_bitrate_step,int min_bwe,int start_bwe,int max_bwe)916 void CallPerfTest::TestMinAudioVideoBitrate(int test_bitrate_from,
917 int test_bitrate_to,
918 int test_bitrate_step,
919 int min_bwe,
920 int start_bwe,
921 int max_bwe) {
922 static const std::string kAudioTrackId = "audio_track_0";
923 static constexpr int kOpusBitrateFbBps = 32000;
924 static constexpr int kBitrateStabilizationMs = 10000;
925 static constexpr int kBitrateMeasurements = 10;
926 static constexpr int kBitrateMeasurementMs = 1000;
927 static constexpr int kShortDelayMs = 10;
928 static constexpr int kMinGoodRttMs = 400;
929
930 class MinVideoAndAudioBitrateTester : public test::EndToEndTest {
931 public:
932 MinVideoAndAudioBitrateTester(int test_bitrate_from,
933 int test_bitrate_to,
934 int test_bitrate_step,
935 int min_bwe,
936 int start_bwe,
937 int max_bwe,
938 TaskQueueBase* task_queue)
939 : EndToEndTest(),
940 test_bitrate_from_(test_bitrate_from),
941 test_bitrate_to_(test_bitrate_to),
942 test_bitrate_step_(test_bitrate_step),
943 min_bwe_(min_bwe),
944 start_bwe_(start_bwe),
945 max_bwe_(max_bwe),
946 task_queue_(task_queue) {}
947
948 protected:
949 BuiltInNetworkBehaviorConfig GetFakeNetworkPipeConfig() {
950 BuiltInNetworkBehaviorConfig pipe_config;
951 pipe_config.link_capacity_kbps = test_bitrate_from_;
952 return pipe_config;
953 }
954
955 std::unique_ptr<test::PacketTransport> CreateSendTransport(
956 TaskQueueBase* task_queue,
957 Call* sender_call) override {
958 auto network =
959 std::make_unique<SimulatedNetwork>(GetFakeNetworkPipeConfig());
960 send_simulated_network_ = network.get();
961 return std::make_unique<test::PacketTransport>(
962 task_queue, sender_call, this, test::PacketTransport::kSender,
963 test::CallTest::payload_type_map_,
964 std::make_unique<FakeNetworkPipe>(Clock::GetRealTimeClock(),
965 std::move(network)));
966 }
967
968 std::unique_ptr<test::PacketTransport> CreateReceiveTransport(
969 TaskQueueBase* task_queue) override {
970 auto network =
971 std::make_unique<SimulatedNetwork>(GetFakeNetworkPipeConfig());
972 receive_simulated_network_ = network.get();
973 return std::make_unique<test::PacketTransport>(
974 task_queue, nullptr, this, test::PacketTransport::kReceiver,
975 test::CallTest::payload_type_map_,
976 std::make_unique<FakeNetworkPipe>(Clock::GetRealTimeClock(),
977 std::move(network)));
978 }
979
980 void PerformTest() override {
981 // Quick test mode, just to exercise all the code paths without actually
982 // caring about performance measurements.
983 const bool quick_perf_test =
984 field_trial::IsEnabled("WebRTC-QuickPerfTest");
985 int last_passed_test_bitrate = -1;
986 for (int test_bitrate = test_bitrate_from_;
987 test_bitrate_from_ < test_bitrate_to_
988 ? test_bitrate <= test_bitrate_to_
989 : test_bitrate >= test_bitrate_to_;
990 test_bitrate += test_bitrate_step_) {
991 BuiltInNetworkBehaviorConfig pipe_config;
992 pipe_config.link_capacity_kbps = test_bitrate;
993 send_simulated_network_->SetConfig(pipe_config);
994 receive_simulated_network_->SetConfig(pipe_config);
995
996 rtc::Thread::SleepMs(quick_perf_test ? kShortDelayMs
997 : kBitrateStabilizationMs);
998
999 int64_t avg_rtt = 0;
1000 for (int i = 0; i < kBitrateMeasurements; i++) {
1001 Call::Stats call_stats;
1002 SendTask(task_queue_, [this, &call_stats]() {
1003 call_stats = sender_call_->GetStats();
1004 });
1005 avg_rtt += call_stats.rtt_ms;
1006 rtc::Thread::SleepMs(quick_perf_test ? kShortDelayMs
1007 : kBitrateMeasurementMs);
1008 }
1009 avg_rtt = avg_rtt / kBitrateMeasurements;
1010 if (avg_rtt > kMinGoodRttMs) {
1011 break;
1012 } else {
1013 last_passed_test_bitrate = test_bitrate;
1014 }
1015 }
1016 EXPECT_GT(last_passed_test_bitrate, -1)
1017 << "Minimum supported bitrate out of the test scope";
1018 GetGlobalMetricsLogger()->LogSingleValueMetric(
1019 "min_test_bitrate_", "min_bitrate", last_passed_test_bitrate,
1020 Unit::kUnitless, ImprovementDirection::kNeitherIsBetter);
1021 }
1022
1023 void OnCallsCreated(Call* sender_call, Call* receiver_call) override {
1024 sender_call_ = sender_call;
1025 BitrateConstraints bitrate_config;
1026 bitrate_config.min_bitrate_bps = min_bwe_;
1027 bitrate_config.start_bitrate_bps = start_bwe_;
1028 bitrate_config.max_bitrate_bps = max_bwe_;
1029 sender_call->GetTransportControllerSend()->SetSdpBitrateParameters(
1030 bitrate_config);
1031 }
1032
1033 size_t GetNumVideoStreams() const override { return 1; }
1034
1035 size_t GetNumAudioStreams() const override { return 1; }
1036
1037 void ModifyAudioConfigs(AudioSendStream::Config* send_config,
1038 std::vector<AudioReceiveStreamInterface::Config>*
1039 receive_configs) override {
1040 send_config->send_codec_spec->target_bitrate_bps =
1041 absl::optional<int>(kOpusBitrateFbBps);
1042 }
1043
1044 private:
1045 const int test_bitrate_from_;
1046 const int test_bitrate_to_;
1047 const int test_bitrate_step_;
1048 const int min_bwe_;
1049 const int start_bwe_;
1050 const int max_bwe_;
1051 SimulatedNetwork* send_simulated_network_;
1052 SimulatedNetwork* receive_simulated_network_;
1053 Call* sender_call_;
1054 TaskQueueBase* const task_queue_;
1055 } test(test_bitrate_from, test_bitrate_to, test_bitrate_step, min_bwe,
1056 start_bwe, max_bwe, task_queue());
1057
1058 RunBaseTest(&test);
1059 }
1060
1061 // TODO(bugs.webrtc.org/8878)
1062 #if defined(WEBRTC_MAC)
1063 #define MAYBE_Min_Bitrate_VideoAndAudio DISABLED_Min_Bitrate_VideoAndAudio
1064 #else
1065 #define MAYBE_Min_Bitrate_VideoAndAudio Min_Bitrate_VideoAndAudio
1066 #endif
TEST_F(CallPerfTest,MAYBE_Min_Bitrate_VideoAndAudio)1067 TEST_F(CallPerfTest, MAYBE_Min_Bitrate_VideoAndAudio) {
1068 TestMinAudioVideoBitrate(110, 40, -10, 10000, 70000, 200000);
1069 }
1070
TestEncodeFramerate(VideoEncoderFactory * encoder_factory,absl::string_view payload_name,const std::vector<int> & max_framerates)1071 void CallPerfTest::TestEncodeFramerate(VideoEncoderFactory* encoder_factory,
1072 absl::string_view payload_name,
1073 const std::vector<int>& max_framerates) {
1074 static constexpr double kAllowedFpsDiff = 1.5;
1075 static constexpr TimeDelta kMinGetStatsInterval = TimeDelta::Millis(400);
1076 static constexpr TimeDelta kMinRunTime = TimeDelta::Seconds(15);
1077 static constexpr DataRate kMaxBitrate = DataRate::KilobitsPerSec(1000);
1078
1079 class FramerateObserver
1080 : public test::EndToEndTest,
1081 public test::FrameGeneratorCapturer::SinkWantsObserver {
1082 public:
1083 FramerateObserver(VideoEncoderFactory* encoder_factory,
1084 absl::string_view payload_name,
1085 const std::vector<int>& max_framerates,
1086 TaskQueueBase* task_queue)
1087 : EndToEndTest(kDefaultTimeout),
1088 clock_(Clock::GetRealTimeClock()),
1089 encoder_factory_(encoder_factory),
1090 payload_name_(payload_name),
1091 max_framerates_(max_framerates),
1092 task_queue_(task_queue),
1093 start_time_(clock_->CurrentTime()),
1094 last_getstats_time_(start_time_),
1095 send_stream_(nullptr) {}
1096
1097 void OnFrameGeneratorCapturerCreated(
1098 test::FrameGeneratorCapturer* frame_generator_capturer) override {
1099 frame_generator_capturer->ChangeResolution(640, 360);
1100 }
1101
1102 void OnSinkWantsChanged(rtc::VideoSinkInterface<VideoFrame>* sink,
1103 const rtc::VideoSinkWants& wants) override {}
1104
1105 void ModifySenderBitrateConfig(
1106 BitrateConstraints* bitrate_config) override {
1107 bitrate_config->start_bitrate_bps = kMaxBitrate.bps() / 2;
1108 }
1109
1110 void OnVideoStreamsCreated(VideoSendStream* send_stream,
1111 const std::vector<VideoReceiveStreamInterface*>&
1112 receive_streams) override {
1113 send_stream_ = send_stream;
1114 }
1115
1116 size_t GetNumVideoStreams() const override {
1117 return max_framerates_.size();
1118 }
1119
1120 void ModifyVideoConfigs(
1121 VideoSendStream::Config* send_config,
1122 std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
1123 VideoEncoderConfig* encoder_config) override {
1124 send_config->encoder_settings.encoder_factory = encoder_factory_;
1125 send_config->rtp.payload_name = payload_name_;
1126 send_config->rtp.payload_type = test::CallTest::kVideoSendPayloadType;
1127 encoder_config->video_format.name = payload_name_;
1128 encoder_config->codec_type = PayloadStringToCodecType(payload_name_);
1129 encoder_config->max_bitrate_bps = kMaxBitrate.bps();
1130 for (size_t i = 0; i < max_framerates_.size(); ++i) {
1131 encoder_config->simulcast_layers[i].max_framerate = max_framerates_[i];
1132 configured_framerates_[send_config->rtp.ssrcs[i]] = max_framerates_[i];
1133 }
1134 }
1135
1136 void PerformTest() override {
1137 EXPECT_TRUE(Wait()) << "Timeout while waiting for framerate stats.";
1138 }
1139
1140 void VerifyStats() const {
1141 double input_fps = 0.0;
1142 for (const auto& configured_framerate : configured_framerates_) {
1143 input_fps = std::max(configured_framerate.second, input_fps);
1144 }
1145 for (const auto& encode_frame_rate_list : encode_frame_rate_lists_) {
1146 const SamplesStatsCounter& values = encode_frame_rate_list.second;
1147 GetGlobalMetricsLogger()->LogMetric(
1148 "substream_fps", "encode_frame_rate", values, Unit::kUnitless,
1149 ImprovementDirection::kNeitherIsBetter);
1150 if (values.IsEmpty()) {
1151 continue;
1152 }
1153 double average_fps = values.GetAverage();
1154 uint32_t ssrc = encode_frame_rate_list.first;
1155 double expected_fps = configured_framerates_.find(ssrc)->second;
1156 if (expected_fps != input_fps)
1157 EXPECT_NEAR(expected_fps, average_fps, kAllowedFpsDiff);
1158 }
1159 }
1160
1161 Action OnSendRtp(const uint8_t* packet, size_t length) override {
1162 const Timestamp now = clock_->CurrentTime();
1163 if (now - last_getstats_time_ > kMinGetStatsInterval) {
1164 last_getstats_time_ = now;
1165 task_queue_->PostTask([this, now]() {
1166 VideoSendStream::Stats stats = send_stream_->GetStats();
1167 for (const auto& stat : stats.substreams) {
1168 encode_frame_rate_lists_[stat.first].AddSample(
1169 stat.second.encode_frame_rate);
1170 }
1171 if (now - start_time_ > kMinRunTime) {
1172 VerifyStats();
1173 observation_complete_.Set();
1174 }
1175 });
1176 }
1177 return SEND_PACKET;
1178 }
1179
1180 Clock* const clock_;
1181 VideoEncoderFactory* const encoder_factory_;
1182 const std::string payload_name_;
1183 const std::vector<int> max_framerates_;
1184 TaskQueueBase* const task_queue_;
1185 const Timestamp start_time_;
1186 Timestamp last_getstats_time_;
1187 VideoSendStream* send_stream_;
1188 std::map<uint32_t, SamplesStatsCounter> encode_frame_rate_lists_;
1189 std::map<uint32_t, double> configured_framerates_;
1190 } test(encoder_factory, payload_name, max_framerates, task_queue());
1191
1192 RunBaseTest(&test);
1193 }
1194
TEST_F(CallPerfTest,TestEncodeFramerateVp8Simulcast)1195 TEST_F(CallPerfTest, TestEncodeFramerateVp8Simulcast) {
1196 InternalEncoderFactory internal_encoder_factory;
1197 test::FunctionVideoEncoderFactory encoder_factory(
1198 [&internal_encoder_factory]() {
1199 return std::make_unique<SimulcastEncoderAdapter>(
1200 &internal_encoder_factory, SdpVideoFormat("VP8"));
1201 });
1202
1203 TestEncodeFramerate(&encoder_factory, "VP8",
1204 /*max_framerates=*/{20, 30});
1205 }
1206
TEST_F(CallPerfTest,TestEncodeFramerateVp8SimulcastLowerInputFps)1207 TEST_F(CallPerfTest, TestEncodeFramerateVp8SimulcastLowerInputFps) {
1208 InternalEncoderFactory internal_encoder_factory;
1209 test::FunctionVideoEncoderFactory encoder_factory(
1210 [&internal_encoder_factory]() {
1211 return std::make_unique<SimulcastEncoderAdapter>(
1212 &internal_encoder_factory, SdpVideoFormat("VP8"));
1213 });
1214
1215 TestEncodeFramerate(&encoder_factory, "VP8",
1216 /*max_framerates=*/{14, 20});
1217 }
1218
1219 } // namespace webrtc
1220