xref: /aosp_15_r20/external/webrtc/pc/peer_connection_rampup_tests.cc (revision d9f758449e529ab9291ac668be2861e7a55c2422)
1 /*
2  *  Copyright 2018 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #include <memory>
12 #include <string>
13 #include <utility>
14 #include <vector>
15 
16 #include "absl/types/optional.h"
17 #include "api/audio/audio_mixer.h"
18 #include "api/audio_codecs/builtin_audio_decoder_factory.h"
19 #include "api/audio_codecs/builtin_audio_encoder_factory.h"
20 #include "api/audio_options.h"
21 #include "api/create_peerconnection_factory.h"
22 #include "api/jsep.h"
23 #include "api/media_stream_interface.h"
24 #include "api/peer_connection_interface.h"
25 #include "api/rtc_error.h"
26 #include "api/scoped_refptr.h"
27 #include "api/stats/rtc_stats.h"
28 #include "api/stats/rtc_stats_report.h"
29 #include "api/stats/rtcstats_objects.h"
30 #include "api/test/metrics/global_metrics_logger_and_exporter.h"
31 #include "api/test/metrics/metric.h"
32 #include "api/video_codecs/builtin_video_decoder_factory.h"
33 #include "api/video_codecs/builtin_video_encoder_factory.h"
34 #include "modules/audio_device/include/audio_device.h"
35 #include "modules/audio_processing/include/audio_processing.h"
36 #include "p2p/base/port_allocator.h"
37 #include "p2p/base/port_interface.h"
38 #include "p2p/base/test_turn_server.h"
39 #include "p2p/client/basic_port_allocator.h"
40 #include "pc/peer_connection.h"
41 #include "pc/peer_connection_wrapper.h"
42 #include "pc/test/fake_audio_capture_module.h"
43 #include "pc/test/frame_generator_capturer_video_track_source.h"
44 #include "pc/test/mock_peer_connection_observers.h"
45 #include "rtc_base/checks.h"
46 #include "rtc_base/fake_network.h"
47 #include "rtc_base/firewall_socket_server.h"
48 #include "rtc_base/gunit.h"
49 #include "rtc_base/helpers.h"
50 #include "rtc_base/socket_address.h"
51 #include "rtc_base/socket_factory.h"
52 #include "rtc_base/ssl_certificate.h"
53 #include "rtc_base/task_queue_for_test.h"
54 #include "rtc_base/test_certificate_verifier.h"
55 #include "rtc_base/thread.h"
56 #include "rtc_base/virtual_socket_server.h"
57 #include "system_wrappers/include/clock.h"
58 #include "test/gtest.h"
59 
60 namespace webrtc {
61 namespace {
62 
63 using ::webrtc::test::GetGlobalMetricsLogger;
64 using ::webrtc::test::ImprovementDirection;
65 using ::webrtc::test::Unit;
66 
67 static const int kDefaultTestTimeMs = 15000;
68 static const int kRampUpTimeMs = 5000;
69 static const int kPollIntervalTimeMs = 50;
70 static const int kDefaultTimeoutMs = 10000;
71 static const rtc::SocketAddress kDefaultLocalAddress("1.1.1.1", 0);
72 static const char kTurnInternalAddress[] = "88.88.88.0";
73 static const char kTurnExternalAddress[] = "88.88.88.1";
74 static const int kTurnInternalPort = 3478;
75 static const int kTurnExternalPort = 0;
76 // The video's configured max bitrate in webrtcvideoengine.cc is 1.7 Mbps.
77 // Setting the network bandwidth to 1 Mbps allows the video's bitrate to push
78 // the network's limitations.
79 static const int kNetworkBandwidth = 1000000;
80 
81 }  // namespace
82 
83 using RTCConfiguration = PeerConnectionInterface::RTCConfiguration;
84 
85 // This is an end to end test to verify that BWE is functioning when setting
86 // up a one to one call at the PeerConnection level. The intention of the test
87 // is to catch potential regressions for different ICE path configurations. The
88 // test uses a VirtualSocketServer for it's underlying simulated network and
89 // fake audio and video sources. The test is based upon rampup_tests.cc, but
90 // instead is at the PeerConnection level and uses a different fake network
91 // (rampup_tests.cc uses SimulatedNetwork). In the future, this test could
92 // potentially test different network conditions and test video quality as well
93 // (video_quality_test.cc does this, but at the call level).
94 //
95 // The perf test results are printed using the perf test support. If the
96 // isolated_script_test_perf_output flag is specified in test_main.cc, then
97 // the results are written to a JSON formatted file for the Chrome perf
98 // dashboard. Since this test is a webrtc_perf_test, it will be run in the perf
99 // console every webrtc commit.
100 class PeerConnectionWrapperForRampUpTest : public PeerConnectionWrapper {
101  public:
102   using PeerConnectionWrapper::PeerConnectionWrapper;
103 
PeerConnectionWrapperForRampUpTest(rtc::scoped_refptr<PeerConnectionFactoryInterface> pc_factory,rtc::scoped_refptr<PeerConnectionInterface> pc,std::unique_ptr<MockPeerConnectionObserver> observer)104   PeerConnectionWrapperForRampUpTest(
105       rtc::scoped_refptr<PeerConnectionFactoryInterface> pc_factory,
106       rtc::scoped_refptr<PeerConnectionInterface> pc,
107       std::unique_ptr<MockPeerConnectionObserver> observer)
108       : PeerConnectionWrapper::PeerConnectionWrapper(pc_factory,
109                                                      pc,
110                                                      std::move(observer)) {}
111 
AddIceCandidates(std::vector<const IceCandidateInterface * > candidates)112   bool AddIceCandidates(std::vector<const IceCandidateInterface*> candidates) {
113     bool success = true;
114     for (const auto candidate : candidates) {
115       if (!pc()->AddIceCandidate(candidate)) {
116         success = false;
117       }
118     }
119     return success;
120   }
121 
CreateLocalVideoTrack(FrameGeneratorCapturerVideoTrackSource::Config config,Clock * clock)122   rtc::scoped_refptr<VideoTrackInterface> CreateLocalVideoTrack(
123       FrameGeneratorCapturerVideoTrackSource::Config config,
124       Clock* clock) {
125     video_track_sources_.emplace_back(
126         rtc::make_ref_counted<FrameGeneratorCapturerVideoTrackSource>(
127             config, clock, /*is_screencast=*/false));
128     video_track_sources_.back()->Start();
129     return rtc::scoped_refptr<VideoTrackInterface>(
130         pc_factory()->CreateVideoTrack(rtc::CreateRandomUuid(),
131                                        video_track_sources_.back().get()));
132   }
133 
CreateLocalAudioTrack(const cricket::AudioOptions options)134   rtc::scoped_refptr<AudioTrackInterface> CreateLocalAudioTrack(
135       const cricket::AudioOptions options) {
136     rtc::scoped_refptr<AudioSourceInterface> source =
137         pc_factory()->CreateAudioSource(options);
138     return pc_factory()->CreateAudioTrack(rtc::CreateRandomUuid(),
139                                           source.get());
140   }
141 
142  private:
143   std::vector<rtc::scoped_refptr<FrameGeneratorCapturerVideoTrackSource>>
144       video_track_sources_;
145 };
146 
147 // TODO(shampson): Paramaterize the test to run for both Plan B & Unified Plan.
148 class PeerConnectionRampUpTest : public ::testing::Test {
149  public:
PeerConnectionRampUpTest()150   PeerConnectionRampUpTest()
151       : clock_(Clock::GetRealTimeClock()),
152         virtual_socket_server_(new rtc::VirtualSocketServer()),
153         firewall_socket_server_(
154             new rtc::FirewallSocketServer(virtual_socket_server_.get())),
155         network_thread_(new rtc::Thread(firewall_socket_server_.get())),
156         worker_thread_(rtc::Thread::Create()) {
157     network_thread_->SetName("PCNetworkThread", this);
158     worker_thread_->SetName("PCWorkerThread", this);
159     RTC_CHECK(network_thread_->Start());
160     RTC_CHECK(worker_thread_->Start());
161 
162     virtual_socket_server_->set_bandwidth(kNetworkBandwidth / 8);
163     pc_factory_ = CreatePeerConnectionFactory(
164         network_thread_.get(), worker_thread_.get(), rtc::Thread::Current(),
165         rtc::scoped_refptr<AudioDeviceModule>(FakeAudioCaptureModule::Create()),
166         CreateBuiltinAudioEncoderFactory(), CreateBuiltinAudioDecoderFactory(),
167         CreateBuiltinVideoEncoderFactory(), CreateBuiltinVideoDecoderFactory(),
168         nullptr /* audio_mixer */, nullptr /* audio_processing */);
169   }
170 
~PeerConnectionRampUpTest()171   virtual ~PeerConnectionRampUpTest() {
172     SendTask(network_thread(), [this] { turn_servers_.clear(); });
173   }
174 
CreatePeerConnectionWrappers(const RTCConfiguration & caller_config,const RTCConfiguration & callee_config)175   bool CreatePeerConnectionWrappers(const RTCConfiguration& caller_config,
176                                     const RTCConfiguration& callee_config) {
177     caller_ = CreatePeerConnectionWrapper(caller_config);
178     callee_ = CreatePeerConnectionWrapper(callee_config);
179     return caller_ && callee_;
180   }
181 
182   std::unique_ptr<PeerConnectionWrapperForRampUpTest>
CreatePeerConnectionWrapper(const RTCConfiguration & config)183   CreatePeerConnectionWrapper(const RTCConfiguration& config) {
184     auto* fake_network_manager = new rtc::FakeNetworkManager();
185     fake_network_manager->AddInterface(kDefaultLocalAddress);
186     fake_network_managers_.emplace_back(fake_network_manager);
187 
188     auto observer = std::make_unique<MockPeerConnectionObserver>();
189     webrtc::PeerConnectionDependencies dependencies(observer.get());
190     cricket::BasicPortAllocator* port_allocator =
191         new cricket::BasicPortAllocator(
192             fake_network_manager,
193             std::make_unique<rtc::BasicPacketSocketFactory>(
194                 firewall_socket_server_.get()));
195     port_allocator->set_step_delay(cricket::kDefaultStepDelay);
196     dependencies.allocator =
197         std::unique_ptr<cricket::BasicPortAllocator>(port_allocator);
198     dependencies.tls_cert_verifier =
199         std::make_unique<rtc::TestCertificateVerifier>();
200 
201     auto result = pc_factory_->CreatePeerConnectionOrError(
202         config, std::move(dependencies));
203     if (!result.ok()) {
204       return nullptr;
205     }
206 
207     return std::make_unique<PeerConnectionWrapperForRampUpTest>(
208         pc_factory_, result.MoveValue(), std::move(observer));
209   }
210 
SetupOneWayCall()211   void SetupOneWayCall() {
212     ASSERT_TRUE(caller_);
213     ASSERT_TRUE(callee_);
214     FrameGeneratorCapturerVideoTrackSource::Config config;
215     caller_->AddTrack(caller_->CreateLocalVideoTrack(config, clock_));
216     // Disable highpass filter so that we can get all the test audio frames.
217     cricket::AudioOptions options;
218     options.highpass_filter = false;
219     caller_->AddTrack(caller_->CreateLocalAudioTrack(options));
220 
221     // Do the SDP negotiation, and also exchange ice candidates.
222     ASSERT_TRUE(caller_->ExchangeOfferAnswerWith(callee_.get()));
223     ASSERT_TRUE_WAIT(
224         caller_->signaling_state() == PeerConnectionInterface::kStable,
225         kDefaultTimeoutMs);
226     ASSERT_TRUE_WAIT(caller_->IsIceGatheringDone(), kDefaultTimeoutMs);
227     ASSERT_TRUE_WAIT(callee_->IsIceGatheringDone(), kDefaultTimeoutMs);
228 
229     // Connect an ICE candidate pairs.
230     ASSERT_TRUE(
231         callee_->AddIceCandidates(caller_->observer()->GetAllCandidates()));
232     ASSERT_TRUE(
233         caller_->AddIceCandidates(callee_->observer()->GetAllCandidates()));
234     // This means that ICE and DTLS are connected.
235     ASSERT_TRUE_WAIT(callee_->IsIceConnected(), kDefaultTimeoutMs);
236     ASSERT_TRUE_WAIT(caller_->IsIceConnected(), kDefaultTimeoutMs);
237   }
238 
CreateTurnServer(cricket::ProtocolType type,const std::string & common_name="test turn server")239   void CreateTurnServer(cricket::ProtocolType type,
240                         const std::string& common_name = "test turn server") {
241     rtc::Thread* thread = network_thread();
242     rtc::SocketFactory* factory = firewall_socket_server_.get();
243     std::unique_ptr<cricket::TestTurnServer> turn_server;
244     SendTask(network_thread_.get(), [&] {
245       static const rtc::SocketAddress turn_server_internal_address{
246           kTurnInternalAddress, kTurnInternalPort};
247       static const rtc::SocketAddress turn_server_external_address{
248           kTurnExternalAddress, kTurnExternalPort};
249       turn_server = std::make_unique<cricket::TestTurnServer>(
250           thread, factory, turn_server_internal_address,
251           turn_server_external_address, type, true /*ignore_bad_certs=*/,
252           common_name);
253     });
254     turn_servers_.push_back(std::move(turn_server));
255   }
256 
257   // First runs the call for kRampUpTimeMs to ramp up the bandwidth estimate.
258   // Then runs the test for the remaining test time, grabbing the bandwidth
259   // estimation stat, every kPollIntervalTimeMs. When finished, averages the
260   // bandwidth estimations and prints the bandwidth estimation result as a perf
261   // metric.
RunTest(const std::string & test_string)262   void RunTest(const std::string& test_string) {
263     rtc::Thread::Current()->ProcessMessages(kRampUpTimeMs);
264     int number_of_polls =
265         (kDefaultTestTimeMs - kRampUpTimeMs) / kPollIntervalTimeMs;
266     int total_bwe = 0;
267     for (int i = 0; i < number_of_polls; ++i) {
268       rtc::Thread::Current()->ProcessMessages(kPollIntervalTimeMs);
269       total_bwe += static_cast<int>(GetCallerAvailableBitrateEstimate());
270     }
271     double average_bandwidth_estimate = total_bwe / number_of_polls;
272     std::string value_description =
273         "bwe_after_" + std::to_string(kDefaultTestTimeMs / 1000) + "_seconds";
274     GetGlobalMetricsLogger()->LogSingleValueMetric(
275         "peerconnection_ramp_up_" + test_string, value_description,
276         average_bandwidth_estimate, Unit::kUnitless,
277         ImprovementDirection::kNeitherIsBetter);
278   }
279 
network_thread()280   rtc::Thread* network_thread() { return network_thread_.get(); }
281 
firewall_socket_server()282   rtc::FirewallSocketServer* firewall_socket_server() {
283     return firewall_socket_server_.get();
284   }
285 
caller()286   PeerConnectionWrapperForRampUpTest* caller() { return caller_.get(); }
287 
callee()288   PeerConnectionWrapperForRampUpTest* callee() { return callee_.get(); }
289 
290  private:
291   // Gets the caller's outgoing available bitrate from the stats. Returns 0 if
292   // something went wrong. It takes the outgoing bitrate from the current
293   // selected ICE candidate pair's stats.
GetCallerAvailableBitrateEstimate()294   double GetCallerAvailableBitrateEstimate() {
295     auto stats = caller_->GetStats();
296     auto transport_stats = stats->GetStatsOfType<RTCTransportStats>();
297     if (transport_stats.size() == 0u ||
298         !transport_stats[0]->selected_candidate_pair_id.is_defined()) {
299       return 0;
300     }
301     std::string selected_ice_id =
302         transport_stats[0]->selected_candidate_pair_id.ValueToString();
303     // Use the selected ICE candidate pair ID to get the appropriate ICE stats.
304     const RTCIceCandidatePairStats ice_candidate_pair_stats =
305         stats->Get(selected_ice_id)->cast_to<const RTCIceCandidatePairStats>();
306     if (ice_candidate_pair_stats.available_outgoing_bitrate.is_defined()) {
307       return *ice_candidate_pair_stats.available_outgoing_bitrate;
308     }
309     // We couldn't get the `available_outgoing_bitrate` for the active candidate
310     // pair.
311     return 0;
312   }
313 
314   Clock* const clock_;
315   // The turn servers should be accessed & deleted on the network thread to
316   // avoid a race with the socket read/write which occurs on the network thread.
317   std::vector<std::unique_ptr<cricket::TestTurnServer>> turn_servers_;
318   // `virtual_socket_server_` is used by `network_thread_` so it must be
319   // destroyed later.
320   // TODO(bugs.webrtc.org/7668): We would like to update the virtual network we
321   // use for this test. VirtualSocketServer isn't ideal because:
322   // 1) It uses the same queue & network capacity for both directions.
323   // 2) VirtualSocketServer implements how the network bandwidth affects the
324   //    send delay differently than the SimulatedNetwork, used by the
325   //    FakeNetworkPipe. It would be ideal if all of levels of virtual
326   //    networks used in testing were consistent.
327   // We would also like to update this test to record the time to ramp up,
328   // down, and back up (similar to in rampup_tests.cc). This is problematic with
329   // the VirtualSocketServer. The first ramp down time is very noisy and the
330   // second ramp up time can take up to 300 seconds, most likely due to a built
331   // up queue.
332   std::unique_ptr<rtc::VirtualSocketServer> virtual_socket_server_;
333   std::unique_ptr<rtc::FirewallSocketServer> firewall_socket_server_;
334   std::unique_ptr<rtc::Thread> network_thread_;
335   std::unique_ptr<rtc::Thread> worker_thread_;
336   // The `pc_factory` uses `network_thread_` & `worker_thread_`, so it must be
337   // destroyed first.
338   std::vector<std::unique_ptr<rtc::FakeNetworkManager>> fake_network_managers_;
339   rtc::scoped_refptr<PeerConnectionFactoryInterface> pc_factory_;
340   std::unique_ptr<PeerConnectionWrapperForRampUpTest> caller_;
341   std::unique_ptr<PeerConnectionWrapperForRampUpTest> callee_;
342 };
343 
TEST_F(PeerConnectionRampUpTest,Bwe_After_TurnOverTCP)344 TEST_F(PeerConnectionRampUpTest, Bwe_After_TurnOverTCP) {
345   CreateTurnServer(cricket::ProtocolType::PROTO_TCP);
346   PeerConnectionInterface::IceServer ice_server;
347   std::string ice_server_url = "turn:" + std::string(kTurnInternalAddress) +
348                                ":" + std::to_string(kTurnInternalPort) +
349                                "?transport=tcp";
350   ice_server.urls.push_back(ice_server_url);
351   ice_server.username = "test";
352   ice_server.password = "test";
353   PeerConnectionInterface::RTCConfiguration client_1_config;
354   client_1_config.sdp_semantics = SdpSemantics::kUnifiedPlan;
355   client_1_config.servers.push_back(ice_server);
356   client_1_config.type = PeerConnectionInterface::kRelay;
357   PeerConnectionInterface::RTCConfiguration client_2_config;
358   client_2_config.sdp_semantics = SdpSemantics::kUnifiedPlan;
359   client_2_config.servers.push_back(ice_server);
360   client_2_config.type = PeerConnectionInterface::kRelay;
361   ASSERT_TRUE(CreatePeerConnectionWrappers(client_1_config, client_2_config));
362 
363   SetupOneWayCall();
364   RunTest("turn_over_tcp");
365 }
366 
TEST_F(PeerConnectionRampUpTest,Bwe_After_TurnOverUDP)367 TEST_F(PeerConnectionRampUpTest, Bwe_After_TurnOverUDP) {
368   CreateTurnServer(cricket::ProtocolType::PROTO_UDP);
369   PeerConnectionInterface::IceServer ice_server;
370   std::string ice_server_url = "turn:" + std::string(kTurnInternalAddress) +
371                                ":" + std::to_string(kTurnInternalPort);
372 
373   ice_server.urls.push_back(ice_server_url);
374   ice_server.username = "test";
375   ice_server.password = "test";
376   PeerConnectionInterface::RTCConfiguration client_1_config;
377   client_1_config.sdp_semantics = SdpSemantics::kUnifiedPlan;
378   client_1_config.servers.push_back(ice_server);
379   client_1_config.type = PeerConnectionInterface::kRelay;
380   PeerConnectionInterface::RTCConfiguration client_2_config;
381   client_2_config.sdp_semantics = SdpSemantics::kUnifiedPlan;
382   client_2_config.servers.push_back(ice_server);
383   client_2_config.type = PeerConnectionInterface::kRelay;
384   ASSERT_TRUE(CreatePeerConnectionWrappers(client_1_config, client_2_config));
385 
386   SetupOneWayCall();
387   RunTest("turn_over_udp");
388 }
389 
TEST_F(PeerConnectionRampUpTest,Bwe_After_TurnOverTLS)390 TEST_F(PeerConnectionRampUpTest, Bwe_After_TurnOverTLS) {
391   CreateTurnServer(cricket::ProtocolType::PROTO_TLS, kTurnInternalAddress);
392   PeerConnectionInterface::IceServer ice_server;
393   std::string ice_server_url = "turns:" + std::string(kTurnInternalAddress) +
394                                ":" + std::to_string(kTurnInternalPort) +
395                                "?transport=tcp";
396   ice_server.urls.push_back(ice_server_url);
397   ice_server.username = "test";
398   ice_server.password = "test";
399   PeerConnectionInterface::RTCConfiguration client_1_config;
400   client_1_config.sdp_semantics = SdpSemantics::kUnifiedPlan;
401   client_1_config.servers.push_back(ice_server);
402   client_1_config.type = PeerConnectionInterface::kRelay;
403   PeerConnectionInterface::RTCConfiguration client_2_config;
404   client_2_config.sdp_semantics = SdpSemantics::kUnifiedPlan;
405   client_2_config.servers.push_back(ice_server);
406   client_2_config.type = PeerConnectionInterface::kRelay;
407 
408   ASSERT_TRUE(CreatePeerConnectionWrappers(client_1_config, client_2_config));
409 
410   SetupOneWayCall();
411   RunTest("turn_over_tls");
412 }
413 
TEST_F(PeerConnectionRampUpTest,Bwe_After_UDPPeerToPeer)414 TEST_F(PeerConnectionRampUpTest, Bwe_After_UDPPeerToPeer) {
415   PeerConnectionInterface::RTCConfiguration client_1_config;
416   client_1_config.sdp_semantics = SdpSemantics::kUnifiedPlan;
417   client_1_config.tcp_candidate_policy =
418       PeerConnection::kTcpCandidatePolicyDisabled;
419   PeerConnectionInterface::RTCConfiguration client_2_config;
420   client_2_config.sdp_semantics = SdpSemantics::kUnifiedPlan;
421   client_2_config.tcp_candidate_policy =
422       PeerConnection::kTcpCandidatePolicyDisabled;
423   ASSERT_TRUE(CreatePeerConnectionWrappers(client_1_config, client_2_config));
424 
425   SetupOneWayCall();
426   RunTest("udp_peer_to_peer");
427 }
428 
TEST_F(PeerConnectionRampUpTest,Bwe_After_TCPPeerToPeer)429 TEST_F(PeerConnectionRampUpTest, Bwe_After_TCPPeerToPeer) {
430   firewall_socket_server()->set_udp_sockets_enabled(false);
431   PeerConnectionInterface::RTCConfiguration config;
432   config.sdp_semantics = SdpSemantics::kUnifiedPlan;
433   ASSERT_TRUE(CreatePeerConnectionWrappers(config, config));
434 
435   SetupOneWayCall();
436   RunTest("tcp_peer_to_peer");
437 }
438 
439 }  // namespace webrtc
440