1 /*
2 * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/audio_coding/neteq/test/neteq_decoding_test.h"
12
13 #include "absl/strings/string_view.h"
14 #include "api/audio_codecs/builtin_audio_decoder_factory.h"
15 #include "api/rtp_headers.h"
16 #include "modules/audio_coding/neteq/default_neteq_factory.h"
17 #include "modules/audio_coding/neteq/test/result_sink.h"
18 #include "rtc_base/strings/string_builder.h"
19 #include "test/testsupport/file_utils.h"
20
21 #ifdef WEBRTC_NETEQ_UNITTEST_BITEXACT
22 RTC_PUSH_IGNORING_WUNDEF()
23 #ifdef WEBRTC_ANDROID_PLATFORM_BUILD
24 #include "external/webrtc/webrtc/modules/audio_coding/neteq/neteq_unittest.pb.h"
25 #else
26 #include "modules/audio_coding/neteq/neteq_unittest.pb.h"
27 #endif
28 RTC_POP_IGNORING_WUNDEF()
29 #endif
30
31 namespace webrtc {
32
33 namespace {
34
LoadDecoders(webrtc::NetEq * neteq)35 void LoadDecoders(webrtc::NetEq* neteq) {
36 ASSERT_EQ(true,
37 neteq->RegisterPayloadType(0, SdpAudioFormat("pcmu", 8000, 1)));
38 ASSERT_EQ(true,
39 neteq->RegisterPayloadType(8, SdpAudioFormat("pcma", 8000, 1)));
40 #ifdef WEBRTC_CODEC_ILBC
41 ASSERT_EQ(true,
42 neteq->RegisterPayloadType(102, SdpAudioFormat("ilbc", 8000, 1)));
43 #endif
44 #if defined(WEBRTC_CODEC_ISAC) || defined(WEBRTC_CODEC_ISACFX)
45 ASSERT_EQ(true,
46 neteq->RegisterPayloadType(103, SdpAudioFormat("isac", 16000, 1)));
47 #endif
48 #ifdef WEBRTC_CODEC_ISAC
49 ASSERT_EQ(true,
50 neteq->RegisterPayloadType(104, SdpAudioFormat("isac", 32000, 1)));
51 #endif
52 #ifdef WEBRTC_CODEC_OPUS
53 ASSERT_EQ(true,
54 neteq->RegisterPayloadType(
55 111, SdpAudioFormat("opus", 48000, 2, {{"stereo", "0"}})));
56 #endif
57 ASSERT_EQ(true,
58 neteq->RegisterPayloadType(93, SdpAudioFormat("L16", 8000, 1)));
59 ASSERT_EQ(true,
60 neteq->RegisterPayloadType(94, SdpAudioFormat("L16", 16000, 1)));
61 ASSERT_EQ(true,
62 neteq->RegisterPayloadType(95, SdpAudioFormat("L16", 32000, 1)));
63 ASSERT_EQ(true,
64 neteq->RegisterPayloadType(13, SdpAudioFormat("cn", 8000, 1)));
65 ASSERT_EQ(true,
66 neteq->RegisterPayloadType(98, SdpAudioFormat("cn", 16000, 1)));
67 }
68
69 } // namespace
70
71 const int NetEqDecodingTest::kTimeStepMs;
72 const size_t NetEqDecodingTest::kBlockSize8kHz;
73 const size_t NetEqDecodingTest::kBlockSize16kHz;
74 const size_t NetEqDecodingTest::kBlockSize32kHz;
75 const int NetEqDecodingTest::kInitSampleRateHz;
76
NetEqDecodingTest()77 NetEqDecodingTest::NetEqDecodingTest()
78 : clock_(0),
79 config_(),
80 output_sample_rate_(kInitSampleRateHz),
81 algorithmic_delay_ms_(0) {
82 config_.sample_rate_hz = kInitSampleRateHz;
83 }
84
SetUp()85 void NetEqDecodingTest::SetUp() {
86 auto decoder_factory = CreateBuiltinAudioDecoderFactory();
87 neteq_ = DefaultNetEqFactory().CreateNetEq(config_, decoder_factory, &clock_);
88 NetEqNetworkStatistics stat;
89 ASSERT_EQ(0, neteq_->NetworkStatistics(&stat));
90 algorithmic_delay_ms_ = stat.current_buffer_size_ms;
91 ASSERT_TRUE(neteq_);
92 LoadDecoders(neteq_.get());
93 }
94
TearDown()95 void NetEqDecodingTest::TearDown() {}
96
OpenInputFile(absl::string_view rtp_file)97 void NetEqDecodingTest::OpenInputFile(absl::string_view rtp_file) {
98 rtp_source_.reset(test::RtpFileSource::Create(rtp_file));
99 }
100
Process()101 void NetEqDecodingTest::Process() {
102 // Check if time to receive.
103 while (packet_ && clock_.TimeInMilliseconds() >= packet_->time_ms()) {
104 if (packet_->payload_length_bytes() > 0) {
105 #ifndef WEBRTC_CODEC_ISAC
106 // Ignore payload type 104 (iSAC-swb) if ISAC is not supported.
107 if (packet_->header().payloadType != 104)
108 #endif
109 ASSERT_EQ(
110 0, neteq_->InsertPacket(
111 packet_->header(),
112 rtc::ArrayView<const uint8_t>(
113 packet_->payload(), packet_->payload_length_bytes())));
114 }
115 // Get next packet.
116 packet_ = rtp_source_->NextPacket();
117 }
118
119 // Get audio from NetEq.
120 bool muted;
121 ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted));
122 ASSERT_FALSE(muted);
123 ASSERT_TRUE((out_frame_.samples_per_channel_ == kBlockSize8kHz) ||
124 (out_frame_.samples_per_channel_ == kBlockSize16kHz) ||
125 (out_frame_.samples_per_channel_ == kBlockSize32kHz) ||
126 (out_frame_.samples_per_channel_ == kBlockSize48kHz));
127 output_sample_rate_ = out_frame_.sample_rate_hz_;
128 EXPECT_EQ(output_sample_rate_, neteq_->last_output_sample_rate_hz());
129
130 // Increase time.
131 clock_.AdvanceTimeMilliseconds(kTimeStepMs);
132 }
133
DecodeAndCompare(absl::string_view rtp_file,absl::string_view output_checksum,absl::string_view network_stats_checksum,bool gen_ref)134 void NetEqDecodingTest::DecodeAndCompare(
135 absl::string_view rtp_file,
136 absl::string_view output_checksum,
137 absl::string_view network_stats_checksum,
138 bool gen_ref) {
139 OpenInputFile(rtp_file);
140
141 std::string ref_out_file =
142 gen_ref ? webrtc::test::OutputPath() + "neteq_universal_ref.pcm" : "";
143 ResultSink output(ref_out_file);
144
145 std::string stat_out_file =
146 gen_ref ? webrtc::test::OutputPath() + "neteq_network_stats.dat" : "";
147 ResultSink network_stats(stat_out_file);
148
149 packet_ = rtp_source_->NextPacket();
150 int i = 0;
151 uint64_t last_concealed_samples = 0;
152 uint64_t last_total_samples_received = 0;
153 while (packet_) {
154 rtc::StringBuilder ss;
155 ss << "Lap number " << i++ << " in DecodeAndCompare while loop";
156 SCOPED_TRACE(ss.str()); // Print out the parameter values on failure.
157 ASSERT_NO_FATAL_FAILURE(Process());
158 ASSERT_NO_FATAL_FAILURE(
159 output.AddResult(out_frame_.data(), out_frame_.samples_per_channel_));
160
161 // Query the network statistics API once per second
162 if (clock_.TimeInMilliseconds() % 1000 == 0) {
163 // Process NetworkStatistics.
164 NetEqNetworkStatistics current_network_stats;
165 ASSERT_EQ(0, neteq_->NetworkStatistics(¤t_network_stats));
166 ASSERT_NO_FATAL_FAILURE(network_stats.AddResult(current_network_stats));
167
168 // Verify that liftime stats and network stats report similar loss
169 // concealment rates.
170 auto lifetime_stats = neteq_->GetLifetimeStatistics();
171 const uint64_t delta_concealed_samples =
172 lifetime_stats.concealed_samples - last_concealed_samples;
173 last_concealed_samples = lifetime_stats.concealed_samples;
174 const uint64_t delta_total_samples_received =
175 lifetime_stats.total_samples_received - last_total_samples_received;
176 last_total_samples_received = lifetime_stats.total_samples_received;
177 // The tolerance is 1% but expressed in Q14.
178 EXPECT_NEAR(
179 (delta_concealed_samples << 14) / delta_total_samples_received,
180 current_network_stats.expand_rate, (2 << 14) / 100.0);
181 }
182 }
183
184 SCOPED_TRACE("Check output audio.");
185 output.VerifyChecksum(output_checksum);
186 SCOPED_TRACE("Check network stats.");
187 network_stats.VerifyChecksum(network_stats_checksum);
188 }
189
PopulateRtpInfo(int frame_index,int timestamp,RTPHeader * rtp_info)190 void NetEqDecodingTest::PopulateRtpInfo(int frame_index,
191 int timestamp,
192 RTPHeader* rtp_info) {
193 rtp_info->sequenceNumber = frame_index;
194 rtp_info->timestamp = timestamp;
195 rtp_info->ssrc = 0x1234; // Just an arbitrary SSRC.
196 rtp_info->payloadType = 94; // PCM16b WB codec.
197 rtp_info->markerBit = false;
198 }
199
PopulateCng(int frame_index,int timestamp,RTPHeader * rtp_info,uint8_t * payload,size_t * payload_len)200 void NetEqDecodingTest::PopulateCng(int frame_index,
201 int timestamp,
202 RTPHeader* rtp_info,
203 uint8_t* payload,
204 size_t* payload_len) {
205 rtp_info->sequenceNumber = frame_index;
206 rtp_info->timestamp = timestamp;
207 rtp_info->ssrc = 0x1234; // Just an arbitrary SSRC.
208 rtp_info->payloadType = 98; // WB CNG.
209 rtp_info->markerBit = false;
210 payload[0] = 64; // Noise level -64 dBov, quite arbitrarily chosen.
211 *payload_len = 1; // Only noise level, no spectral parameters.
212 }
213
WrapTest(uint16_t start_seq_no,uint32_t start_timestamp,const std::set<uint16_t> & drop_seq_numbers,bool expect_seq_no_wrap,bool expect_timestamp_wrap)214 void NetEqDecodingTest::WrapTest(uint16_t start_seq_no,
215 uint32_t start_timestamp,
216 const std::set<uint16_t>& drop_seq_numbers,
217 bool expect_seq_no_wrap,
218 bool expect_timestamp_wrap) {
219 uint16_t seq_no = start_seq_no;
220 uint32_t timestamp = start_timestamp;
221 const int kBlocksPerFrame = 3; // Number of 10 ms blocks per frame.
222 const int kFrameSizeMs = kBlocksPerFrame * kTimeStepMs;
223 const int kSamples = kBlockSize16kHz * kBlocksPerFrame;
224 const size_t kPayloadBytes = kSamples * sizeof(int16_t);
225 double next_input_time_ms = 0.0;
226
227 // Insert speech for 2 seconds.
228 const int kSpeechDurationMs = 2000;
229 uint16_t last_seq_no;
230 uint32_t last_timestamp;
231 bool timestamp_wrapped = false;
232 bool seq_no_wrapped = false;
233 for (double t_ms = 0; t_ms < kSpeechDurationMs; t_ms += 10) {
234 // Each turn in this for loop is 10 ms.
235 while (next_input_time_ms <= t_ms) {
236 // Insert one 30 ms speech frame.
237 uint8_t payload[kPayloadBytes] = {0};
238 RTPHeader rtp_info;
239 PopulateRtpInfo(seq_no, timestamp, &rtp_info);
240 if (drop_seq_numbers.find(seq_no) == drop_seq_numbers.end()) {
241 // This sequence number was not in the set to drop. Insert it.
242 ASSERT_EQ(0, neteq_->InsertPacket(rtp_info, payload));
243 }
244 NetEqNetworkStatistics network_stats;
245 ASSERT_EQ(0, neteq_->NetworkStatistics(&network_stats));
246
247 EXPECT_LE(network_stats.preferred_buffer_size_ms, 80);
248 EXPECT_LE(network_stats.current_buffer_size_ms,
249 80 + algorithmic_delay_ms_);
250 last_seq_no = seq_no;
251 last_timestamp = timestamp;
252
253 ++seq_no;
254 timestamp += kSamples;
255 next_input_time_ms += static_cast<double>(kFrameSizeMs);
256
257 seq_no_wrapped |= seq_no < last_seq_no;
258 timestamp_wrapped |= timestamp < last_timestamp;
259 }
260 // Pull out data once.
261 AudioFrame output;
262 bool muted;
263 ASSERT_EQ(0, neteq_->GetAudio(&output, &muted));
264 ASSERT_EQ(kBlockSize16kHz, output.samples_per_channel_);
265 ASSERT_EQ(1u, output.num_channels_);
266
267 // Expect delay (in samples) to be less than 2 packets.
268 absl::optional<uint32_t> playout_timestamp = neteq_->GetPlayoutTimestamp();
269 ASSERT_TRUE(playout_timestamp);
270 EXPECT_LE(timestamp - *playout_timestamp,
271 static_cast<uint32_t>(kSamples * 2));
272 }
273 // Make sure we have actually tested wrap-around.
274 ASSERT_EQ(expect_seq_no_wrap, seq_no_wrapped);
275 ASSERT_EQ(expect_timestamp_wrap, timestamp_wrapped);
276 }
277
LongCngWithClockDrift(double drift_factor,double network_freeze_ms,bool pull_audio_during_freeze,int delay_tolerance_ms,int max_time_to_speech_ms)278 void NetEqDecodingTest::LongCngWithClockDrift(double drift_factor,
279 double network_freeze_ms,
280 bool pull_audio_during_freeze,
281 int delay_tolerance_ms,
282 int max_time_to_speech_ms) {
283 uint16_t seq_no = 0;
284 uint32_t timestamp = 0;
285 const int kFrameSizeMs = 30;
286 const size_t kSamples = kFrameSizeMs * 16;
287 const size_t kPayloadBytes = kSamples * 2;
288 double next_input_time_ms = 0.0;
289 double t_ms;
290 bool muted;
291
292 // Insert speech for 5 seconds.
293 const int kSpeechDurationMs = 5000;
294 for (t_ms = 0; t_ms < kSpeechDurationMs; t_ms += 10) {
295 // Each turn in this for loop is 10 ms.
296 while (next_input_time_ms <= t_ms) {
297 // Insert one 30 ms speech frame.
298 uint8_t payload[kPayloadBytes] = {0};
299 RTPHeader rtp_info;
300 PopulateRtpInfo(seq_no, timestamp, &rtp_info);
301 ASSERT_EQ(0, neteq_->InsertPacket(rtp_info, payload));
302 ++seq_no;
303 timestamp += kSamples;
304 next_input_time_ms += static_cast<double>(kFrameSizeMs) * drift_factor;
305 }
306 // Pull out data once.
307 ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted));
308 ASSERT_EQ(kBlockSize16kHz, out_frame_.samples_per_channel_);
309 }
310
311 EXPECT_EQ(AudioFrame::kNormalSpeech, out_frame_.speech_type_);
312 absl::optional<uint32_t> playout_timestamp = neteq_->GetPlayoutTimestamp();
313 ASSERT_TRUE(playout_timestamp);
314 int32_t delay_before = timestamp - *playout_timestamp;
315
316 // Insert CNG for 1 minute (= 60000 ms).
317 const int kCngPeriodMs = 100;
318 const int kCngPeriodSamples = kCngPeriodMs * 16; // Period in 16 kHz samples.
319 const int kCngDurationMs = 60000;
320 for (; t_ms < kSpeechDurationMs + kCngDurationMs; t_ms += 10) {
321 // Each turn in this for loop is 10 ms.
322 while (next_input_time_ms <= t_ms) {
323 // Insert one CNG frame each 100 ms.
324 uint8_t payload[kPayloadBytes];
325 size_t payload_len;
326 RTPHeader rtp_info;
327 PopulateCng(seq_no, timestamp, &rtp_info, payload, &payload_len);
328 ASSERT_EQ(0, neteq_->InsertPacket(rtp_info, rtc::ArrayView<const uint8_t>(
329 payload, payload_len)));
330 ++seq_no;
331 timestamp += kCngPeriodSamples;
332 next_input_time_ms += static_cast<double>(kCngPeriodMs) * drift_factor;
333 }
334 // Pull out data once.
335 ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted));
336 ASSERT_EQ(kBlockSize16kHz, out_frame_.samples_per_channel_);
337 }
338
339 EXPECT_EQ(AudioFrame::kCNG, out_frame_.speech_type_);
340
341 if (network_freeze_ms > 0) {
342 // First keep pulling audio for `network_freeze_ms` without inserting
343 // any data, then insert CNG data corresponding to `network_freeze_ms`
344 // without pulling any output audio.
345 const double loop_end_time = t_ms + network_freeze_ms;
346 for (; t_ms < loop_end_time; t_ms += 10) {
347 // Pull out data once.
348 ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted));
349 ASSERT_EQ(kBlockSize16kHz, out_frame_.samples_per_channel_);
350 EXPECT_EQ(AudioFrame::kCNG, out_frame_.speech_type_);
351 }
352 bool pull_once = pull_audio_during_freeze;
353 // If `pull_once` is true, GetAudio will be called once half-way through
354 // the network recovery period.
355 double pull_time_ms = (t_ms + next_input_time_ms) / 2;
356 while (next_input_time_ms <= t_ms) {
357 if (pull_once && next_input_time_ms >= pull_time_ms) {
358 pull_once = false;
359 // Pull out data once.
360 ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted));
361 ASSERT_EQ(kBlockSize16kHz, out_frame_.samples_per_channel_);
362 EXPECT_EQ(AudioFrame::kCNG, out_frame_.speech_type_);
363 t_ms += 10;
364 }
365 // Insert one CNG frame each 100 ms.
366 uint8_t payload[kPayloadBytes];
367 size_t payload_len;
368 RTPHeader rtp_info;
369 PopulateCng(seq_no, timestamp, &rtp_info, payload, &payload_len);
370 ASSERT_EQ(0, neteq_->InsertPacket(rtp_info, rtc::ArrayView<const uint8_t>(
371 payload, payload_len)));
372 ++seq_no;
373 timestamp += kCngPeriodSamples;
374 next_input_time_ms += kCngPeriodMs * drift_factor;
375 }
376 }
377
378 // Insert speech again until output type is speech.
379 double speech_restart_time_ms = t_ms;
380 while (out_frame_.speech_type_ != AudioFrame::kNormalSpeech) {
381 // Each turn in this for loop is 10 ms.
382 while (next_input_time_ms <= t_ms) {
383 // Insert one 30 ms speech frame.
384 uint8_t payload[kPayloadBytes] = {0};
385 RTPHeader rtp_info;
386 PopulateRtpInfo(seq_no, timestamp, &rtp_info);
387 ASSERT_EQ(0, neteq_->InsertPacket(rtp_info, payload));
388 ++seq_no;
389 timestamp += kSamples;
390 next_input_time_ms += kFrameSizeMs * drift_factor;
391 }
392 // Pull out data once.
393 ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted));
394 ASSERT_EQ(kBlockSize16kHz, out_frame_.samples_per_channel_);
395 // Increase clock.
396 t_ms += 10;
397 }
398
399 // Check that the speech starts again within reasonable time.
400 double time_until_speech_returns_ms = t_ms - speech_restart_time_ms;
401 EXPECT_LT(time_until_speech_returns_ms, max_time_to_speech_ms);
402 playout_timestamp = neteq_->GetPlayoutTimestamp();
403 ASSERT_TRUE(playout_timestamp);
404 int32_t delay_after = timestamp - *playout_timestamp;
405 // Compare delay before and after, and make sure it differs less than 20 ms.
406 EXPECT_LE(delay_after, delay_before + delay_tolerance_ms * 16);
407 EXPECT_GE(delay_after, delay_before - delay_tolerance_ms * 16);
408 }
409
SetUp()410 void NetEqDecodingTestTwoInstances::SetUp() {
411 NetEqDecodingTest::SetUp();
412 config2_ = config_;
413 }
414
CreateSecondInstance()415 void NetEqDecodingTestTwoInstances::CreateSecondInstance() {
416 auto decoder_factory = CreateBuiltinAudioDecoderFactory();
417 neteq2_ =
418 DefaultNetEqFactory().CreateNetEq(config2_, decoder_factory, &clock_);
419 ASSERT_TRUE(neteq2_);
420 LoadDecoders(neteq2_.get());
421 }
422
423 } // namespace webrtc
424