1 /*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/audio_coding/neteq/neteq_impl.h"
12
13 #include <memory>
14 #include <utility>
15 #include <vector>
16
17 #include "api/audio_codecs/builtin_audio_decoder_factory.h"
18 #include "api/neteq/default_neteq_controller_factory.h"
19 #include "api/neteq/neteq.h"
20 #include "api/neteq/neteq_controller.h"
21 #include "modules/audio_coding/codecs/g711/audio_decoder_pcm.h"
22 #include "modules/audio_coding/neteq/accelerate.h"
23 #include "modules/audio_coding/neteq/decision_logic.h"
24 #include "modules/audio_coding/neteq/default_neteq_factory.h"
25 #include "modules/audio_coding/neteq/expand.h"
26 #include "modules/audio_coding/neteq/histogram.h"
27 #include "modules/audio_coding/neteq/mock/mock_decoder_database.h"
28 #include "modules/audio_coding/neteq/mock/mock_dtmf_buffer.h"
29 #include "modules/audio_coding/neteq/mock/mock_dtmf_tone_generator.h"
30 #include "modules/audio_coding/neteq/mock/mock_neteq_controller.h"
31 #include "modules/audio_coding/neteq/mock/mock_packet_buffer.h"
32 #include "modules/audio_coding/neteq/mock/mock_red_payload_splitter.h"
33 #include "modules/audio_coding/neteq/preemptive_expand.h"
34 #include "modules/audio_coding/neteq/statistics_calculator.h"
35 #include "modules/audio_coding/neteq/sync_buffer.h"
36 #include "modules/audio_coding/neteq/timestamp_scaler.h"
37 #include "rtc_base/numerics/safe_conversions.h"
38 #include "system_wrappers/include/clock.h"
39 #include "test/audio_decoder_proxy_factory.h"
40 #include "test/function_audio_decoder_factory.h"
41 #include "test/gmock.h"
42 #include "test/gtest.h"
43 #include "test/mock_audio_decoder.h"
44 #include "test/mock_audio_decoder_factory.h"
45
46 using ::testing::_;
47 using ::testing::AtLeast;
48 using ::testing::DoAll;
49 using ::testing::ElementsAre;
50 using ::testing::InSequence;
51 using ::testing::Invoke;
52 using ::testing::IsEmpty;
53 using ::testing::IsNull;
54 using ::testing::Pointee;
55 using ::testing::Return;
56 using ::testing::ReturnNull;
57 using ::testing::SetArgPointee;
58 using ::testing::SetArrayArgument;
59 using ::testing::SizeIs;
60 using ::testing::WithArg;
61
62 namespace webrtc {
63
64 // This function is called when inserting a packet list into the mock packet
65 // buffer. The purpose is to delete all inserted packets properly, to avoid
66 // memory leaks in the test.
DeletePacketsAndReturnOk(PacketList * packet_list)67 int DeletePacketsAndReturnOk(PacketList* packet_list) {
68 packet_list->clear();
69 return PacketBuffer::kOK;
70 }
71
72 class NetEqImplTest : public ::testing::Test {
73 protected:
NetEqImplTest()74 NetEqImplTest() : clock_(0) { config_.sample_rate_hz = 8000; }
75
CreateInstance(const rtc::scoped_refptr<AudioDecoderFactory> & decoder_factory)76 void CreateInstance(
77 const rtc::scoped_refptr<AudioDecoderFactory>& decoder_factory) {
78 ASSERT_TRUE(decoder_factory);
79 config_.enable_muted_state = enable_muted_state_;
80 NetEqImpl::Dependencies deps(config_, &clock_, decoder_factory,
81 DefaultNetEqControllerFactory());
82
83 // Get a local pointer to NetEq's TickTimer object.
84 tick_timer_ = deps.tick_timer.get();
85
86 if (use_mock_decoder_database_) {
87 std::unique_ptr<MockDecoderDatabase> mock(new MockDecoderDatabase);
88 mock_decoder_database_ = mock.get();
89 EXPECT_CALL(*mock_decoder_database_, GetActiveCngDecoder())
90 .WillOnce(ReturnNull());
91 deps.decoder_database = std::move(mock);
92 }
93 decoder_database_ = deps.decoder_database.get();
94
95 if (use_mock_dtmf_buffer_) {
96 std::unique_ptr<MockDtmfBuffer> mock(
97 new MockDtmfBuffer(config_.sample_rate_hz));
98 mock_dtmf_buffer_ = mock.get();
99 deps.dtmf_buffer = std::move(mock);
100 }
101 dtmf_buffer_ = deps.dtmf_buffer.get();
102
103 if (use_mock_dtmf_tone_generator_) {
104 std::unique_ptr<MockDtmfToneGenerator> mock(new MockDtmfToneGenerator);
105 mock_dtmf_tone_generator_ = mock.get();
106 deps.dtmf_tone_generator = std::move(mock);
107 }
108 dtmf_tone_generator_ = deps.dtmf_tone_generator.get();
109
110 if (use_mock_packet_buffer_) {
111 std::unique_ptr<MockPacketBuffer> mock(
112 new MockPacketBuffer(config_.max_packets_in_buffer, tick_timer_));
113 mock_packet_buffer_ = mock.get();
114 deps.packet_buffer = std::move(mock);
115 }
116 packet_buffer_ = deps.packet_buffer.get();
117
118 if (use_mock_neteq_controller_) {
119 std::unique_ptr<MockNetEqController> mock(new MockNetEqController());
120 mock_neteq_controller_ = mock.get();
121 deps.neteq_controller = std::move(mock);
122 } else {
123 deps.stats = std::make_unique<StatisticsCalculator>();
124 NetEqController::Config controller_config;
125 controller_config.tick_timer = tick_timer_;
126 controller_config.base_min_delay_ms = config_.min_delay_ms;
127 controller_config.allow_time_stretching = true;
128 controller_config.max_packets_in_buffer = config_.max_packets_in_buffer;
129 controller_config.clock = &clock_;
130 deps.neteq_controller =
131 std::make_unique<DecisionLogic>(std::move(controller_config));
132 }
133 neteq_controller_ = deps.neteq_controller.get();
134
135 if (use_mock_payload_splitter_) {
136 std::unique_ptr<MockRedPayloadSplitter> mock(new MockRedPayloadSplitter);
137 mock_payload_splitter_ = mock.get();
138 deps.red_payload_splitter = std::move(mock);
139 }
140 red_payload_splitter_ = deps.red_payload_splitter.get();
141
142 deps.timestamp_scaler = std::unique_ptr<TimestampScaler>(
143 new TimestampScaler(*deps.decoder_database.get()));
144
145 neteq_.reset(new NetEqImpl(config_, std::move(deps)));
146 ASSERT_TRUE(neteq_ != NULL);
147 }
148
CreateInstance()149 void CreateInstance() { CreateInstance(CreateBuiltinAudioDecoderFactory()); }
150
UseNoMocks()151 void UseNoMocks() {
152 ASSERT_TRUE(neteq_ == NULL) << "Must call UseNoMocks before CreateInstance";
153 use_mock_decoder_database_ = false;
154 use_mock_neteq_controller_ = false;
155 use_mock_dtmf_buffer_ = false;
156 use_mock_dtmf_tone_generator_ = false;
157 use_mock_packet_buffer_ = false;
158 use_mock_payload_splitter_ = false;
159 }
160
~NetEqImplTest()161 virtual ~NetEqImplTest() {
162 if (use_mock_decoder_database_) {
163 EXPECT_CALL(*mock_decoder_database_, Die()).Times(1);
164 }
165 if (use_mock_neteq_controller_) {
166 EXPECT_CALL(*mock_neteq_controller_, Die()).Times(1);
167 }
168 if (use_mock_dtmf_buffer_) {
169 EXPECT_CALL(*mock_dtmf_buffer_, Die()).Times(1);
170 }
171 if (use_mock_dtmf_tone_generator_) {
172 EXPECT_CALL(*mock_dtmf_tone_generator_, Die()).Times(1);
173 }
174 if (use_mock_packet_buffer_) {
175 EXPECT_CALL(*mock_packet_buffer_, Die()).Times(1);
176 }
177 }
178
TestDtmfPacket(int sample_rate_hz)179 void TestDtmfPacket(int sample_rate_hz) {
180 const size_t kPayloadLength = 4;
181 const uint8_t kPayloadType = 110;
182 const int kSampleRateHz = 16000;
183 config_.sample_rate_hz = kSampleRateHz;
184 UseNoMocks();
185 CreateInstance();
186 // Event: 2, E bit, Volume: 17, Length: 4336.
187 uint8_t payload[kPayloadLength] = {0x02, 0x80 + 0x11, 0x10, 0xF0};
188 RTPHeader rtp_header;
189 rtp_header.payloadType = kPayloadType;
190 rtp_header.sequenceNumber = 0x1234;
191 rtp_header.timestamp = 0x12345678;
192 rtp_header.ssrc = 0x87654321;
193
194 EXPECT_TRUE(neteq_->RegisterPayloadType(
195 kPayloadType, SdpAudioFormat("telephone-event", sample_rate_hz, 1)));
196
197 // Insert first packet.
198 EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload));
199
200 // Pull audio once.
201 const size_t kMaxOutputSize =
202 static_cast<size_t>(10 * kSampleRateHz / 1000);
203 AudioFrame output;
204 bool muted;
205 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted));
206 ASSERT_FALSE(muted);
207 ASSERT_EQ(kMaxOutputSize, output.samples_per_channel_);
208 EXPECT_EQ(1u, output.num_channels_);
209 EXPECT_EQ(AudioFrame::kNormalSpeech, output.speech_type_);
210
211 // DTMF packets are immediately consumed by `InsertPacket()` and won't be
212 // returned by `GetAudio()`.
213 EXPECT_THAT(output.packet_infos_, IsEmpty());
214
215 // Verify first 64 samples of actual output.
216 const std::vector<int16_t> kOutput(
217 {0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
218 -1578, -2816, -3460, -3403, -2709, -1594, -363, 671, 1269, 1328,
219 908, 202, -513, -964, -955, -431, 504, 1617, 2602, 3164,
220 3101, 2364, 1073, -511, -2047, -3198, -3721, -3525, -2688, -1440,
221 -99, 1015, 1663, 1744, 1319, 588, -171, -680, -747, -315,
222 515, 1512, 2378, 2828, 2674, 1877, 568, -986, -2446, -3482,
223 -3864, -3516, -2534, -1163});
224 ASSERT_GE(kMaxOutputSize, kOutput.size());
225 EXPECT_TRUE(std::equal(kOutput.begin(), kOutput.end(), output.data()));
226 }
227
228 std::unique_ptr<NetEqImpl> neteq_;
229 NetEq::Config config_;
230 SimulatedClock clock_;
231 TickTimer* tick_timer_ = nullptr;
232 MockDecoderDatabase* mock_decoder_database_ = nullptr;
233 DecoderDatabase* decoder_database_ = nullptr;
234 bool use_mock_decoder_database_ = true;
235 MockNetEqController* mock_neteq_controller_ = nullptr;
236 NetEqController* neteq_controller_ = nullptr;
237 bool use_mock_neteq_controller_ = true;
238 MockDtmfBuffer* mock_dtmf_buffer_ = nullptr;
239 DtmfBuffer* dtmf_buffer_ = nullptr;
240 bool use_mock_dtmf_buffer_ = true;
241 MockDtmfToneGenerator* mock_dtmf_tone_generator_ = nullptr;
242 DtmfToneGenerator* dtmf_tone_generator_ = nullptr;
243 bool use_mock_dtmf_tone_generator_ = true;
244 MockPacketBuffer* mock_packet_buffer_ = nullptr;
245 PacketBuffer* packet_buffer_ = nullptr;
246 bool use_mock_packet_buffer_ = true;
247 MockRedPayloadSplitter* mock_payload_splitter_ = nullptr;
248 RedPayloadSplitter* red_payload_splitter_ = nullptr;
249 bool use_mock_payload_splitter_ = true;
250 bool enable_muted_state_ = false;
251 };
252
253 // This tests the interface class NetEq.
254 // TODO(hlundin): Move to separate file?
TEST(NetEq,CreateAndDestroy)255 TEST(NetEq, CreateAndDestroy) {
256 NetEq::Config config;
257 SimulatedClock clock(0);
258 auto decoder_factory = CreateBuiltinAudioDecoderFactory();
259 std::unique_ptr<NetEq> neteq =
260 DefaultNetEqFactory().CreateNetEq(config, decoder_factory, &clock);
261 }
262
TEST_F(NetEqImplTest,RegisterPayloadType)263 TEST_F(NetEqImplTest, RegisterPayloadType) {
264 CreateInstance();
265 constexpr int rtp_payload_type = 0;
266 const SdpAudioFormat format("pcmu", 8000, 1);
267 EXPECT_CALL(*mock_decoder_database_,
268 RegisterPayload(rtp_payload_type, format));
269 neteq_->RegisterPayloadType(rtp_payload_type, format);
270 }
271
TEST_F(NetEqImplTest,RemovePayloadType)272 TEST_F(NetEqImplTest, RemovePayloadType) {
273 CreateInstance();
274 uint8_t rtp_payload_type = 0;
275 EXPECT_CALL(*mock_decoder_database_, Remove(rtp_payload_type))
276 .WillOnce(Return(DecoderDatabase::kDecoderNotFound));
277 // Check that kOK is returned when database returns kDecoderNotFound, because
278 // removing a payload type that was never registered is not an error.
279 EXPECT_EQ(NetEq::kOK, neteq_->RemovePayloadType(rtp_payload_type));
280 }
281
TEST_F(NetEqImplTest,RemoveAllPayloadTypes)282 TEST_F(NetEqImplTest, RemoveAllPayloadTypes) {
283 CreateInstance();
284 EXPECT_CALL(*mock_decoder_database_, RemoveAll()).WillOnce(Return());
285 neteq_->RemoveAllPayloadTypes();
286 }
287
TEST_F(NetEqImplTest,InsertPacket)288 TEST_F(NetEqImplTest, InsertPacket) {
289 using ::testing::AllOf;
290 using ::testing::Field;
291 CreateInstance();
292 const size_t kPayloadLength = 100;
293 const uint8_t kPayloadType = 0;
294 const uint16_t kFirstSequenceNumber = 0x1234;
295 const uint32_t kFirstTimestamp = 0x12345678;
296 const uint32_t kSsrc = 0x87654321;
297 uint8_t payload[kPayloadLength] = {0};
298 RTPHeader rtp_header;
299 rtp_header.payloadType = kPayloadType;
300 rtp_header.sequenceNumber = kFirstSequenceNumber;
301 rtp_header.timestamp = kFirstTimestamp;
302 rtp_header.ssrc = kSsrc;
303 Packet fake_packet;
304 fake_packet.payload_type = kPayloadType;
305 fake_packet.sequence_number = kFirstSequenceNumber;
306 fake_packet.timestamp = kFirstTimestamp;
307
308 auto mock_decoder_factory = rtc::make_ref_counted<MockAudioDecoderFactory>();
309 EXPECT_CALL(*mock_decoder_factory, MakeAudioDecoderMock(_, _, _))
310 .WillOnce(Invoke([&](const SdpAudioFormat& format,
311 absl::optional<AudioCodecPairId> codec_pair_id,
312 std::unique_ptr<AudioDecoder>* dec) {
313 EXPECT_EQ("pcmu", format.name);
314
315 std::unique_ptr<MockAudioDecoder> mock_decoder(new MockAudioDecoder);
316 EXPECT_CALL(*mock_decoder, Channels()).WillRepeatedly(Return(1));
317 EXPECT_CALL(*mock_decoder, SampleRateHz()).WillRepeatedly(Return(8000));
318 EXPECT_CALL(*mock_decoder, Die()).Times(1); // Called when deleted.
319
320 *dec = std::move(mock_decoder);
321 }));
322 DecoderDatabase::DecoderInfo info(SdpAudioFormat("pcmu", 8000, 1),
323 absl::nullopt, mock_decoder_factory.get());
324
325 // Expectations for decoder database.
326 EXPECT_CALL(*mock_decoder_database_, GetDecoderInfo(kPayloadType))
327 .WillRepeatedly(Return(&info));
328
329 // Expectations for packet buffer.
330 EXPECT_CALL(*mock_packet_buffer_, Empty())
331 .WillOnce(Return(false)); // Called once after first packet is inserted.
332 EXPECT_CALL(*mock_packet_buffer_, Flush(_)).Times(1);
333 EXPECT_CALL(*mock_packet_buffer_, InsertPacketList(_, _, _, _, _, _, _, _))
334 .Times(2)
335 .WillRepeatedly(DoAll(SetArgPointee<2>(kPayloadType),
336 WithArg<0>(Invoke(DeletePacketsAndReturnOk))));
337 // SetArgPointee<2>(kPayloadType) means that the third argument (zero-based
338 // index) is a pointer, and the variable pointed to is set to kPayloadType.
339 // Also invoke the function DeletePacketsAndReturnOk to properly delete all
340 // packets in the list (to avoid memory leaks in the test).
341 EXPECT_CALL(*mock_packet_buffer_, PeekNextPacket())
342 .Times(1)
343 .WillOnce(Return(&fake_packet));
344
345 // Expectations for DTMF buffer.
346 EXPECT_CALL(*mock_dtmf_buffer_, Flush()).Times(1);
347
348 // Expectations for delay manager.
349 {
350 // All expectations within this block must be called in this specific order.
351 InSequence sequence; // Dummy variable.
352 // Expectations when the first packet is inserted.
353 EXPECT_CALL(
354 *mock_neteq_controller_,
355 PacketArrived(
356 /*fs_hz*/ 8000,
357 /*should_update_stats*/ _,
358 /*info*/
359 AllOf(
360 Field(&NetEqController::PacketArrivedInfo::is_cng_or_dtmf,
361 false),
362 Field(&NetEqController::PacketArrivedInfo::main_sequence_number,
363 kFirstSequenceNumber),
364 Field(&NetEqController::PacketArrivedInfo::main_timestamp,
365 kFirstTimestamp))));
366 EXPECT_CALL(
367 *mock_neteq_controller_,
368 PacketArrived(
369 /*fs_hz*/ 8000,
370 /*should_update_stats*/ _,
371 /*info*/
372 AllOf(
373 Field(&NetEqController::PacketArrivedInfo::is_cng_or_dtmf,
374 false),
375 Field(&NetEqController::PacketArrivedInfo::main_sequence_number,
376 kFirstSequenceNumber + 1),
377 Field(&NetEqController::PacketArrivedInfo::main_timestamp,
378 kFirstTimestamp + 160))));
379 }
380
381 // Insert first packet.
382 neteq_->InsertPacket(rtp_header, payload);
383
384 // Insert second packet.
385 rtp_header.timestamp += 160;
386 rtp_header.sequenceNumber += 1;
387 neteq_->InsertPacket(rtp_header, payload);
388 }
389
TEST_F(NetEqImplTest,InsertPacketsUntilBufferIsFull)390 TEST_F(NetEqImplTest, InsertPacketsUntilBufferIsFull) {
391 UseNoMocks();
392 CreateInstance();
393
394 const int kPayloadLengthSamples = 80;
395 const size_t kPayloadLengthBytes = 2 * kPayloadLengthSamples; // PCM 16-bit.
396 const uint8_t kPayloadType = 17; // Just an arbitrary number.
397 uint8_t payload[kPayloadLengthBytes] = {0};
398 RTPHeader rtp_header;
399 rtp_header.payloadType = kPayloadType;
400 rtp_header.sequenceNumber = 0x1234;
401 rtp_header.timestamp = 0x12345678;
402 rtp_header.ssrc = 0x87654321;
403
404 EXPECT_TRUE(neteq_->RegisterPayloadType(kPayloadType,
405 SdpAudioFormat("l16", 8000, 1)));
406
407 // Insert packets. The buffer should not flush.
408 for (size_t i = 1; i <= config_.max_packets_in_buffer; ++i) {
409 EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload));
410 rtp_header.timestamp += kPayloadLengthSamples;
411 rtp_header.sequenceNumber += 1;
412 EXPECT_EQ(i, packet_buffer_->NumPacketsInBuffer());
413 }
414
415 // Insert one more packet and make sure the buffer got flushed. That is, it
416 // should only hold one single packet.
417 EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload));
418 EXPECT_EQ(1u, packet_buffer_->NumPacketsInBuffer());
419 const Packet* test_packet = packet_buffer_->PeekNextPacket();
420 EXPECT_EQ(rtp_header.timestamp, test_packet->timestamp);
421 EXPECT_EQ(rtp_header.sequenceNumber, test_packet->sequence_number);
422 }
423
TEST_F(NetEqImplTest,TestDtmfPacketAVT)424 TEST_F(NetEqImplTest, TestDtmfPacketAVT) {
425 TestDtmfPacket(8000);
426 }
427
TEST_F(NetEqImplTest,TestDtmfPacketAVT16kHz)428 TEST_F(NetEqImplTest, TestDtmfPacketAVT16kHz) {
429 TestDtmfPacket(16000);
430 }
431
TEST_F(NetEqImplTest,TestDtmfPacketAVT32kHz)432 TEST_F(NetEqImplTest, TestDtmfPacketAVT32kHz) {
433 TestDtmfPacket(32000);
434 }
435
TEST_F(NetEqImplTest,TestDtmfPacketAVT48kHz)436 TEST_F(NetEqImplTest, TestDtmfPacketAVT48kHz) {
437 TestDtmfPacket(48000);
438 }
439
440 // This test verifies that timestamps propagate from the incoming packets
441 // through to the sync buffer and to the playout timestamp.
TEST_F(NetEqImplTest,VerifyTimestampPropagation)442 TEST_F(NetEqImplTest, VerifyTimestampPropagation) {
443 const uint8_t kPayloadType = 17; // Just an arbitrary number.
444 const int kSampleRateHz = 8000;
445 const size_t kPayloadLengthSamples =
446 static_cast<size_t>(10 * kSampleRateHz / 1000); // 10 ms.
447 const size_t kPayloadLengthBytes = kPayloadLengthSamples;
448 uint8_t payload[kPayloadLengthBytes] = {0};
449 RTPHeader rtp_header;
450 rtp_header.payloadType = kPayloadType;
451 rtp_header.sequenceNumber = 0x1234;
452 rtp_header.timestamp = 0x12345678;
453 rtp_header.ssrc = 0x87654321;
454 rtp_header.numCSRCs = 3;
455 rtp_header.arrOfCSRCs[0] = 43;
456 rtp_header.arrOfCSRCs[1] = 65;
457 rtp_header.arrOfCSRCs[2] = 17;
458
459 // This is a dummy decoder that produces as many output samples as the input
460 // has bytes. The output is an increasing series, starting at 1 for the first
461 // sample, and then increasing by 1 for each sample.
462 class CountingSamplesDecoder : public AudioDecoder {
463 public:
464 CountingSamplesDecoder() : next_value_(1) {}
465
466 // Produce as many samples as input bytes (`encoded_len`).
467 int DecodeInternal(const uint8_t* encoded,
468 size_t encoded_len,
469 int /* sample_rate_hz */,
470 int16_t* decoded,
471 SpeechType* speech_type) override {
472 for (size_t i = 0; i < encoded_len; ++i) {
473 decoded[i] = next_value_++;
474 }
475 *speech_type = kSpeech;
476 return rtc::checked_cast<int>(encoded_len);
477 }
478
479 void Reset() override { next_value_ = 1; }
480
481 int SampleRateHz() const override { return kSampleRateHz; }
482
483 size_t Channels() const override { return 1; }
484
485 uint16_t next_value() const { return next_value_; }
486
487 private:
488 int16_t next_value_;
489 } decoder_;
490
491 auto decoder_factory =
492 rtc::make_ref_counted<test::AudioDecoderProxyFactory>(&decoder_);
493
494 UseNoMocks();
495 CreateInstance(decoder_factory);
496
497 EXPECT_TRUE(neteq_->RegisterPayloadType(kPayloadType,
498 SdpAudioFormat("L16", 8000, 1)));
499
500 // Insert one packet.
501 clock_.AdvanceTimeMilliseconds(123456);
502 Timestamp expected_receive_time = clock_.CurrentTime();
503 EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload));
504
505 // Pull audio once.
506 const size_t kMaxOutputSize = static_cast<size_t>(10 * kSampleRateHz / 1000);
507 AudioFrame output;
508 bool muted;
509 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted));
510 ASSERT_FALSE(muted);
511 ASSERT_EQ(kMaxOutputSize, output.samples_per_channel_);
512 EXPECT_EQ(1u, output.num_channels_);
513 EXPECT_EQ(AudioFrame::kNormalSpeech, output.speech_type_);
514
515 // Verify `output.packet_infos_`.
516 ASSERT_THAT(output.packet_infos_, SizeIs(1));
517 {
518 const auto& packet_info = output.packet_infos_[0];
519 EXPECT_EQ(packet_info.ssrc(), rtp_header.ssrc);
520 EXPECT_THAT(packet_info.csrcs(), ElementsAre(43, 65, 17));
521 EXPECT_EQ(packet_info.rtp_timestamp(), rtp_header.timestamp);
522 EXPECT_FALSE(packet_info.audio_level().has_value());
523 EXPECT_EQ(packet_info.receive_time(), expected_receive_time);
524 }
525
526 // Start with a simple check that the fake decoder is behaving as expected.
527 EXPECT_EQ(kPayloadLengthSamples,
528 static_cast<size_t>(decoder_.next_value() - 1));
529
530 // The value of the last of the output samples is the same as the number of
531 // samples played from the decoded packet. Thus, this number + the RTP
532 // timestamp should match the playout timestamp.
533 // Wrap the expected value in an absl::optional to compare them as such.
534 EXPECT_EQ(
535 absl::optional<uint32_t>(rtp_header.timestamp +
536 output.data()[output.samples_per_channel_ - 1]),
537 neteq_->GetPlayoutTimestamp());
538
539 // Check the timestamp for the last value in the sync buffer. This should
540 // be one full frame length ahead of the RTP timestamp.
541 const SyncBuffer* sync_buffer = neteq_->sync_buffer_for_test();
542 ASSERT_TRUE(sync_buffer != NULL);
543 EXPECT_EQ(rtp_header.timestamp + kPayloadLengthSamples,
544 sync_buffer->end_timestamp());
545
546 // Check that the number of samples still to play from the sync buffer add
547 // up with what was already played out.
548 EXPECT_EQ(
549 kPayloadLengthSamples - output.data()[output.samples_per_channel_ - 1],
550 sync_buffer->FutureLength());
551 }
552
TEST_F(NetEqImplTest,ReorderedPacket)553 TEST_F(NetEqImplTest, ReorderedPacket) {
554 UseNoMocks();
555
556 // Create a mock decoder object.
557 MockAudioDecoder mock_decoder;
558
559 CreateInstance(
560 rtc::make_ref_counted<test::AudioDecoderProxyFactory>(&mock_decoder));
561
562 const uint8_t kPayloadType = 17; // Just an arbitrary number.
563 const int kSampleRateHz = 8000;
564 const size_t kPayloadLengthSamples =
565 static_cast<size_t>(10 * kSampleRateHz / 1000); // 10 ms.
566 const size_t kPayloadLengthBytes = kPayloadLengthSamples;
567 uint8_t payload[kPayloadLengthBytes] = {0};
568 RTPHeader rtp_header;
569 rtp_header.payloadType = kPayloadType;
570 rtp_header.sequenceNumber = 0x1234;
571 rtp_header.timestamp = 0x12345678;
572 rtp_header.ssrc = 0x87654321;
573 rtp_header.extension.hasAudioLevel = true;
574 rtp_header.extension.audioLevel = 42;
575
576 EXPECT_CALL(mock_decoder, Reset()).WillRepeatedly(Return());
577 EXPECT_CALL(mock_decoder, SampleRateHz())
578 .WillRepeatedly(Return(kSampleRateHz));
579 EXPECT_CALL(mock_decoder, Channels()).WillRepeatedly(Return(1));
580 EXPECT_CALL(mock_decoder, PacketDuration(_, kPayloadLengthBytes))
581 .WillRepeatedly(Return(rtc::checked_cast<int>(kPayloadLengthSamples)));
582 int16_t dummy_output[kPayloadLengthSamples] = {0};
583 // The below expectation will make the mock decoder write
584 // `kPayloadLengthSamples` zeros to the output array, and mark it as speech.
585 EXPECT_CALL(mock_decoder, DecodeInternal(Pointee(0), kPayloadLengthBytes,
586 kSampleRateHz, _, _))
587 .WillOnce(DoAll(SetArrayArgument<3>(dummy_output,
588 dummy_output + kPayloadLengthSamples),
589 SetArgPointee<4>(AudioDecoder::kSpeech),
590 Return(rtc::checked_cast<int>(kPayloadLengthSamples))));
591 EXPECT_TRUE(neteq_->RegisterPayloadType(kPayloadType,
592 SdpAudioFormat("L16", 8000, 1)));
593
594 // Insert one packet.
595 clock_.AdvanceTimeMilliseconds(123456);
596 Timestamp expected_receive_time = clock_.CurrentTime();
597 EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload));
598
599 // Pull audio once.
600 const size_t kMaxOutputSize = static_cast<size_t>(10 * kSampleRateHz / 1000);
601 AudioFrame output;
602 bool muted;
603 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted));
604 ASSERT_EQ(kMaxOutputSize, output.samples_per_channel_);
605 EXPECT_EQ(1u, output.num_channels_);
606 EXPECT_EQ(AudioFrame::kNormalSpeech, output.speech_type_);
607
608 // Verify `output.packet_infos_`.
609 ASSERT_THAT(output.packet_infos_, SizeIs(1));
610 {
611 const auto& packet_info = output.packet_infos_[0];
612 EXPECT_EQ(packet_info.ssrc(), rtp_header.ssrc);
613 EXPECT_THAT(packet_info.csrcs(), IsEmpty());
614 EXPECT_EQ(packet_info.rtp_timestamp(), rtp_header.timestamp);
615 EXPECT_EQ(packet_info.audio_level(), rtp_header.extension.audioLevel);
616 EXPECT_EQ(packet_info.receive_time(), expected_receive_time);
617 }
618
619 // Insert two more packets. The first one is out of order, and is already too
620 // old, the second one is the expected next packet.
621 rtp_header.sequenceNumber -= 1;
622 rtp_header.timestamp -= kPayloadLengthSamples;
623 rtp_header.extension.audioLevel = 1;
624 payload[0] = 1;
625 clock_.AdvanceTimeMilliseconds(1000);
626 EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload));
627 rtp_header.sequenceNumber += 2;
628 rtp_header.timestamp += 2 * kPayloadLengthSamples;
629 rtp_header.extension.audioLevel = 2;
630 payload[0] = 2;
631 clock_.AdvanceTimeMilliseconds(2000);
632 expected_receive_time = clock_.CurrentTime();
633 EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload));
634
635 // Expect only the second packet to be decoded (the one with "2" as the first
636 // payload byte).
637 EXPECT_CALL(mock_decoder, DecodeInternal(Pointee(2), kPayloadLengthBytes,
638 kSampleRateHz, _, _))
639 .WillOnce(DoAll(SetArrayArgument<3>(dummy_output,
640 dummy_output + kPayloadLengthSamples),
641 SetArgPointee<4>(AudioDecoder::kSpeech),
642 Return(rtc::checked_cast<int>(kPayloadLengthSamples))));
643
644 // Pull audio once.
645 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted));
646 ASSERT_EQ(kMaxOutputSize, output.samples_per_channel_);
647 EXPECT_EQ(1u, output.num_channels_);
648 EXPECT_EQ(AudioFrame::kNormalSpeech, output.speech_type_);
649
650 // Now check the packet buffer, and make sure it is empty, since the
651 // out-of-order packet should have been discarded.
652 EXPECT_TRUE(packet_buffer_->Empty());
653
654 // NetEq `packets_discarded` should capture this packet discard.
655 EXPECT_EQ(1u, neteq_->GetLifetimeStatistics().packets_discarded);
656
657 // Verify `output.packet_infos_`. Expect to only see the second packet.
658 ASSERT_THAT(output.packet_infos_, SizeIs(1));
659 {
660 const auto& packet_info = output.packet_infos_[0];
661 EXPECT_EQ(packet_info.ssrc(), rtp_header.ssrc);
662 EXPECT_THAT(packet_info.csrcs(), IsEmpty());
663 EXPECT_EQ(packet_info.rtp_timestamp(), rtp_header.timestamp);
664 EXPECT_EQ(packet_info.audio_level(), rtp_header.extension.audioLevel);
665 EXPECT_EQ(packet_info.receive_time(), expected_receive_time);
666 }
667
668 EXPECT_CALL(mock_decoder, Die());
669 }
670
671 // This test verifies that NetEq can handle the situation where the first
672 // incoming packet is rejected.
TEST_F(NetEqImplTest,FirstPacketUnknown)673 TEST_F(NetEqImplTest, FirstPacketUnknown) {
674 UseNoMocks();
675 CreateInstance();
676
677 const uint8_t kPayloadType = 17; // Just an arbitrary number.
678 const int kSampleRateHz = 8000;
679 const size_t kPayloadLengthSamples =
680 static_cast<size_t>(10 * kSampleRateHz / 1000); // 10 ms.
681 const size_t kPayloadLengthBytes = kPayloadLengthSamples * 2;
682 uint8_t payload[kPayloadLengthBytes] = {0};
683 RTPHeader rtp_header;
684 rtp_header.payloadType = kPayloadType;
685 rtp_header.sequenceNumber = 0x1234;
686 rtp_header.timestamp = 0x12345678;
687 rtp_header.ssrc = 0x87654321;
688
689 // Insert one packet. Note that we have not registered any payload type, so
690 // this packet will be rejected.
691 EXPECT_EQ(NetEq::kFail, neteq_->InsertPacket(rtp_header, payload));
692
693 // Pull audio once.
694 const size_t kMaxOutputSize = static_cast<size_t>(10 * kSampleRateHz / 1000);
695 AudioFrame output;
696 bool muted;
697 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted));
698 ASSERT_LE(output.samples_per_channel_, kMaxOutputSize);
699 EXPECT_EQ(kMaxOutputSize, output.samples_per_channel_);
700 EXPECT_EQ(1u, output.num_channels_);
701 EXPECT_EQ(AudioFrame::kPLC, output.speech_type_);
702 EXPECT_THAT(output.packet_infos_, IsEmpty());
703
704 // Register the payload type.
705 EXPECT_TRUE(neteq_->RegisterPayloadType(kPayloadType,
706 SdpAudioFormat("l16", 8000, 1)));
707
708 // Insert 10 packets.
709 for (size_t i = 0; i < 10; ++i) {
710 rtp_header.sequenceNumber++;
711 rtp_header.timestamp += kPayloadLengthSamples;
712 EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload));
713 EXPECT_EQ(i + 1, packet_buffer_->NumPacketsInBuffer());
714 }
715
716 // Pull audio repeatedly and make sure we get normal output, that is not PLC.
717 for (size_t i = 0; i < 3; ++i) {
718 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted));
719 ASSERT_LE(output.samples_per_channel_, kMaxOutputSize);
720 EXPECT_EQ(kMaxOutputSize, output.samples_per_channel_);
721 EXPECT_EQ(1u, output.num_channels_);
722 EXPECT_EQ(AudioFrame::kNormalSpeech, output.speech_type_)
723 << "NetEq did not decode the packets as expected.";
724 EXPECT_THAT(output.packet_infos_, SizeIs(1));
725 }
726 }
727
728 // This test verifies that audio interruption is not logged for the initial
729 // PLC period before the first packet is deocoded.
730 // TODO(henrik.lundin) Maybe move this test to neteq_network_stats_unittest.cc.
731 // Make the test parametrized, so that we can test with different initial
732 // sample rates in NetEq.
733 class NetEqImplTestSampleRateParameter
734 : public NetEqImplTest,
735 public testing::WithParamInterface<int> {
736 protected:
NetEqImplTestSampleRateParameter()737 NetEqImplTestSampleRateParameter()
738 : NetEqImplTest(), initial_sample_rate_hz_(GetParam()) {
739 config_.sample_rate_hz = initial_sample_rate_hz_;
740 }
741
742 const int initial_sample_rate_hz_;
743 };
744
745 class NetEqImplTestSdpFormatParameter
746 : public NetEqImplTest,
747 public testing::WithParamInterface<SdpAudioFormat> {
748 protected:
NetEqImplTestSdpFormatParameter()749 NetEqImplTestSdpFormatParameter()
750 : NetEqImplTest(), sdp_format_(GetParam()) {}
751 const SdpAudioFormat sdp_format_;
752 };
753
754 // This test does the following:
755 // 0. Set up NetEq with initial sample rate given by test parameter, and a codec
756 // sample rate of 16000.
757 // 1. Start calling GetAudio before inserting any encoded audio. The audio
758 // produced will be PLC.
759 // 2. Insert a number of encoded audio packets.
760 // 3. Keep calling GetAudio and verify that no audio interruption was logged.
761 // Call GetAudio until NetEq runs out of data again; PLC starts.
762 // 4. Insert one more packet.
763 // 5. Call GetAudio until that packet is decoded and the PLC ends.
764
TEST_P(NetEqImplTestSampleRateParameter,NoAudioInterruptionLoggedBeforeFirstDecode)765 TEST_P(NetEqImplTestSampleRateParameter,
766 NoAudioInterruptionLoggedBeforeFirstDecode) {
767 UseNoMocks();
768 CreateInstance();
769
770 const uint8_t kPayloadType = 17; // Just an arbitrary number.
771 const int kPayloadSampleRateHz = 16000;
772 const size_t kPayloadLengthSamples =
773 static_cast<size_t>(10 * kPayloadSampleRateHz / 1000); // 10 ms.
774 const size_t kPayloadLengthBytes = kPayloadLengthSamples * 2;
775 uint8_t payload[kPayloadLengthBytes] = {0};
776 RTPHeader rtp_header;
777 rtp_header.payloadType = kPayloadType;
778 rtp_header.sequenceNumber = 0x1234;
779 rtp_header.timestamp = 0x12345678;
780 rtp_header.ssrc = 0x87654321;
781
782 // Register the payload type.
783 EXPECT_TRUE(neteq_->RegisterPayloadType(
784 kPayloadType, SdpAudioFormat("l16", kPayloadSampleRateHz, 1)));
785
786 // Pull audio several times. No packets have been inserted yet.
787 const size_t initial_output_size =
788 static_cast<size_t>(10 * initial_sample_rate_hz_ / 1000); // 10 ms
789 AudioFrame output;
790 bool muted;
791 for (int i = 0; i < 100; ++i) {
792 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted));
793 EXPECT_EQ(initial_output_size, output.samples_per_channel_);
794 EXPECT_EQ(1u, output.num_channels_);
795 EXPECT_NE(AudioFrame::kNormalSpeech, output.speech_type_);
796 EXPECT_THAT(output.packet_infos_, IsEmpty());
797 }
798
799 // Lambda for inserting packets.
800 auto insert_packet = [&]() {
801 rtp_header.sequenceNumber++;
802 rtp_header.timestamp += kPayloadLengthSamples;
803 EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload));
804 };
805 // Insert 10 packets.
806 for (size_t i = 0; i < 10; ++i) {
807 insert_packet();
808 EXPECT_EQ(i + 1, packet_buffer_->NumPacketsInBuffer());
809 }
810
811 // Pull audio repeatedly and make sure we get normal output, that is not PLC.
812 constexpr size_t kOutputSize =
813 static_cast<size_t>(10 * kPayloadSampleRateHz / 1000); // 10 ms
814 for (size_t i = 0; i < 3; ++i) {
815 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted));
816 EXPECT_EQ(kOutputSize, output.samples_per_channel_);
817 EXPECT_EQ(1u, output.num_channels_);
818 EXPECT_EQ(AudioFrame::kNormalSpeech, output.speech_type_)
819 << "NetEq did not decode the packets as expected.";
820 EXPECT_THAT(output.packet_infos_, SizeIs(1));
821 }
822
823 // Verify that no interruption was logged.
824 auto lifetime_stats = neteq_->GetLifetimeStatistics();
825 EXPECT_EQ(0, lifetime_stats.interruption_count);
826
827 // Keep pulling audio data until a new PLC period is started.
828 size_t count_loops = 0;
829 while (output.speech_type_ == AudioFrame::kNormalSpeech) {
830 // Make sure we don't hang the test if we never go to PLC.
831 ASSERT_LT(++count_loops, 100u);
832 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted));
833 }
834
835 // Insert a few packets to avoid postpone decoding after expand.
836 for (size_t i = 0; i < 5; ++i) {
837 insert_packet();
838 }
839
840 // Pull audio until the newly inserted packet is decoded and the PLC ends.
841 while (output.speech_type_ != AudioFrame::kNormalSpeech) {
842 // Make sure we don't hang the test if we never go to PLC.
843 ASSERT_LT(++count_loops, 100u);
844 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted));
845 }
846
847 // Verify that no interruption was logged.
848 lifetime_stats = neteq_->GetLifetimeStatistics();
849 EXPECT_EQ(0, lifetime_stats.interruption_count);
850 }
851
852 // This test does the following:
853 // 0. Set up NetEq with initial sample rate given by test parameter, and a codec
854 // sample rate of 16000.
855 // 1. Insert a number of encoded audio packets.
856 // 2. Call GetAudio and verify that decoded audio is produced.
857 // 3. Keep calling GetAudio until NetEq runs out of data; PLC starts.
858 // 4. Keep calling GetAudio until PLC has been produced for at least 150 ms.
859 // 5. Insert one more packet.
860 // 6. Call GetAudio until that packet is decoded and the PLC ends.
861 // 7. Verify that an interruption was logged.
862
TEST_P(NetEqImplTestSampleRateParameter,AudioInterruptionLogged)863 TEST_P(NetEqImplTestSampleRateParameter, AudioInterruptionLogged) {
864 UseNoMocks();
865 CreateInstance();
866
867 const uint8_t kPayloadType = 17; // Just an arbitrary number.
868 const int kPayloadSampleRateHz = 16000;
869 const size_t kPayloadLengthSamples =
870 static_cast<size_t>(10 * kPayloadSampleRateHz / 1000); // 10 ms.
871 const size_t kPayloadLengthBytes = kPayloadLengthSamples * 2;
872 uint8_t payload[kPayloadLengthBytes] = {0};
873 RTPHeader rtp_header;
874 rtp_header.payloadType = kPayloadType;
875 rtp_header.sequenceNumber = 0x1234;
876 rtp_header.timestamp = 0x12345678;
877 rtp_header.ssrc = 0x87654321;
878
879 // Register the payload type.
880 EXPECT_TRUE(neteq_->RegisterPayloadType(
881 kPayloadType, SdpAudioFormat("l16", kPayloadSampleRateHz, 1)));
882
883 // Lambda for inserting packets.
884 auto insert_packet = [&]() {
885 rtp_header.sequenceNumber++;
886 rtp_header.timestamp += kPayloadLengthSamples;
887 EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload));
888 };
889 // Insert 10 packets.
890 for (size_t i = 0; i < 10; ++i) {
891 insert_packet();
892 EXPECT_EQ(i + 1, packet_buffer_->NumPacketsInBuffer());
893 }
894
895 AudioFrame output;
896 bool muted;
897 // Keep pulling audio data until a new PLC period is started.
898 size_t count_loops = 0;
899 do {
900 // Make sure we don't hang the test if we never go to PLC.
901 ASSERT_LT(++count_loops, 100u);
902 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted));
903 } while (output.speech_type_ == AudioFrame::kNormalSpeech);
904
905 // Pull audio 15 times, which produces 150 ms of output audio. This should
906 // all be produced as PLC. The total length of the gap will then be 150 ms
907 // plus an initial fraction of 10 ms at the start and the end of the PLC
908 // period. In total, less than 170 ms.
909 for (size_t i = 0; i < 15; ++i) {
910 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted));
911 EXPECT_NE(AudioFrame::kNormalSpeech, output.speech_type_);
912 }
913
914 // Insert a few packets to avoid postpone decoding after expand.
915 for (size_t i = 0; i < 5; ++i) {
916 insert_packet();
917 }
918
919 // Pull audio until the newly inserted packet is decoded and the PLC ends.
920 while (output.speech_type_ != AudioFrame::kNormalSpeech) {
921 // Make sure we don't hang the test if we never go to PLC.
922 ASSERT_LT(++count_loops, 100u);
923 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted));
924 }
925
926 // Verify that the interruption was logged.
927 auto lifetime_stats = neteq_->GetLifetimeStatistics();
928 EXPECT_EQ(1, lifetime_stats.interruption_count);
929 EXPECT_GT(lifetime_stats.total_interruption_duration_ms, 150);
930 EXPECT_LT(lifetime_stats.total_interruption_duration_ms, 170);
931 }
932
933 INSTANTIATE_TEST_SUITE_P(SampleRates,
934 NetEqImplTestSampleRateParameter,
935 testing::Values(8000, 16000, 32000, 48000));
936
TEST_P(NetEqImplTestSdpFormatParameter,GetNackListScaledTimestamp)937 TEST_P(NetEqImplTestSdpFormatParameter, GetNackListScaledTimestamp) {
938 UseNoMocks();
939 CreateInstance();
940
941 neteq_->EnableNack(128);
942
943 const uint8_t kPayloadType = 17; // Just an arbitrary number.
944 const int kPayloadSampleRateHz = sdp_format_.clockrate_hz;
945 const size_t kPayloadLengthSamples =
946 static_cast<size_t>(10 * kPayloadSampleRateHz / 1000); // 10 ms.
947 const size_t kPayloadLengthBytes = kPayloadLengthSamples * 2;
948 std::vector<uint8_t> payload(kPayloadLengthBytes, 0);
949 RTPHeader rtp_header;
950 rtp_header.payloadType = kPayloadType;
951 rtp_header.sequenceNumber = 0x1234;
952 rtp_header.timestamp = 0x12345678;
953 rtp_header.ssrc = 0x87654321;
954
955 EXPECT_TRUE(neteq_->RegisterPayloadType(kPayloadType, sdp_format_));
956
957 auto insert_packet = [&](bool lost = false) {
958 rtp_header.sequenceNumber++;
959 rtp_header.timestamp += kPayloadLengthSamples;
960 if (!lost)
961 EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload));
962 };
963
964 // Insert and decode 10 packets.
965 for (size_t i = 0; i < 10; ++i) {
966 insert_packet();
967 }
968 AudioFrame output;
969 size_t count_loops = 0;
970 do {
971 bool muted;
972 // Make sure we don't hang the test if we never go to PLC.
973 ASSERT_LT(++count_loops, 100u);
974 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted));
975 } while (output.speech_type_ == AudioFrame::kNormalSpeech);
976
977 insert_packet();
978
979 insert_packet(/*lost=*/true);
980
981 // Ensure packet gets marked as missing.
982 for (int i = 0; i < 5; ++i) {
983 insert_packet();
984 }
985
986 // Missing packet recoverable with 5ms RTT.
987 EXPECT_THAT(neteq_->GetNackList(5), Not(IsEmpty()));
988
989 // No packets should have TimeToPlay > 500ms.
990 EXPECT_THAT(neteq_->GetNackList(500), IsEmpty());
991 }
992
993 INSTANTIATE_TEST_SUITE_P(GetNackList,
994 NetEqImplTestSdpFormatParameter,
995 testing::Values(SdpAudioFormat("g722", 8000, 1),
996 SdpAudioFormat("opus", 48000, 2)));
997
998 // This test verifies that NetEq can handle comfort noise and enters/quits codec
999 // internal CNG mode properly.
TEST_F(NetEqImplTest,CodecInternalCng)1000 TEST_F(NetEqImplTest, CodecInternalCng) {
1001 UseNoMocks();
1002 // Create a mock decoder object.
1003 MockAudioDecoder mock_decoder;
1004 CreateInstance(
1005 rtc::make_ref_counted<test::AudioDecoderProxyFactory>(&mock_decoder));
1006
1007 const uint8_t kPayloadType = 17; // Just an arbitrary number.
1008 const int kSampleRateKhz = 48;
1009 const size_t kPayloadLengthSamples =
1010 static_cast<size_t>(20 * kSampleRateKhz); // 20 ms.
1011 const size_t kPayloadLengthBytes = 10;
1012 uint8_t payload[kPayloadLengthBytes] = {0};
1013 int16_t dummy_output[kPayloadLengthSamples] = {0};
1014
1015 RTPHeader rtp_header;
1016 rtp_header.payloadType = kPayloadType;
1017 rtp_header.sequenceNumber = 0x1234;
1018 rtp_header.timestamp = 0x12345678;
1019 rtp_header.ssrc = 0x87654321;
1020
1021 EXPECT_CALL(mock_decoder, Reset()).WillRepeatedly(Return());
1022 EXPECT_CALL(mock_decoder, SampleRateHz())
1023 .WillRepeatedly(Return(kSampleRateKhz * 1000));
1024 EXPECT_CALL(mock_decoder, Channels()).WillRepeatedly(Return(1));
1025 EXPECT_CALL(mock_decoder, PacketDuration(_, kPayloadLengthBytes))
1026 .WillRepeatedly(Return(rtc::checked_cast<int>(kPayloadLengthSamples)));
1027 // Packed duration when asking the decoder for more CNG data (without a new
1028 // packet).
1029 EXPECT_CALL(mock_decoder, PacketDuration(nullptr, 0))
1030 .WillRepeatedly(Return(rtc::checked_cast<int>(kPayloadLengthSamples)));
1031
1032 EXPECT_TRUE(neteq_->RegisterPayloadType(kPayloadType,
1033 SdpAudioFormat("opus", 48000, 2)));
1034
1035 struct Packet {
1036 int sequence_number_delta;
1037 int timestamp_delta;
1038 AudioDecoder::SpeechType decoder_output_type;
1039 };
1040 std::vector<Packet> packets = {
1041 {0, 0, AudioDecoder::kSpeech},
1042 {1, kPayloadLengthSamples, AudioDecoder::kComfortNoise},
1043 {2, 2 * kPayloadLengthSamples, AudioDecoder::kSpeech},
1044 {1, kPayloadLengthSamples, AudioDecoder::kSpeech}};
1045
1046 for (size_t i = 0; i < packets.size(); ++i) {
1047 rtp_header.sequenceNumber += packets[i].sequence_number_delta;
1048 rtp_header.timestamp += packets[i].timestamp_delta;
1049 payload[0] = i;
1050 EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload));
1051
1052 // Pointee(x) verifies that first byte of the payload equals x, this makes
1053 // it possible to verify that the correct payload is fed to Decode().
1054 EXPECT_CALL(mock_decoder, DecodeInternal(Pointee(i), kPayloadLengthBytes,
1055 kSampleRateKhz * 1000, _, _))
1056 .WillOnce(DoAll(SetArrayArgument<3>(
1057 dummy_output, dummy_output + kPayloadLengthSamples),
1058 SetArgPointee<4>(packets[i].decoder_output_type),
1059 Return(rtc::checked_cast<int>(kPayloadLengthSamples))));
1060 }
1061
1062 // Expect comfort noise to be returned by the decoder.
1063 EXPECT_CALL(mock_decoder,
1064 DecodeInternal(IsNull(), 0, kSampleRateKhz * 1000, _, _))
1065 .WillOnce(DoAll(SetArrayArgument<3>(dummy_output,
1066 dummy_output + kPayloadLengthSamples),
1067 SetArgPointee<4>(AudioDecoder::kComfortNoise),
1068 Return(rtc::checked_cast<int>(kPayloadLengthSamples))));
1069
1070 std::vector<AudioFrame::SpeechType> expected_output = {
1071 AudioFrame::kNormalSpeech, AudioFrame::kCNG, AudioFrame::kNormalSpeech};
1072 size_t output_index = 0;
1073
1074 int timeout_counter = 0;
1075 while (!packet_buffer_->Empty()) {
1076 ASSERT_LT(timeout_counter++, 20) << "Test timed out";
1077 AudioFrame output;
1078 bool muted;
1079 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted));
1080 if (output_index + 1 < expected_output.size() &&
1081 output.speech_type_ == expected_output[output_index + 1]) {
1082 ++output_index;
1083 } else {
1084 EXPECT_EQ(output.speech_type_, expected_output[output_index]);
1085 }
1086 }
1087
1088 EXPECT_CALL(mock_decoder, Die());
1089 }
1090
TEST_F(NetEqImplTest,UnsupportedDecoder)1091 TEST_F(NetEqImplTest, UnsupportedDecoder) {
1092 UseNoMocks();
1093 ::testing::NiceMock<MockAudioDecoder> decoder;
1094
1095 CreateInstance(
1096 rtc::make_ref_counted<test::AudioDecoderProxyFactory>(&decoder));
1097 static const size_t kNetEqMaxFrameSize = 5760; // 120 ms @ 48 kHz.
1098 static const size_t kChannels = 2;
1099
1100 const uint8_t kPayloadType = 17; // Just an arbitrary number.
1101 const int kSampleRateHz = 8000;
1102
1103 const size_t kPayloadLengthSamples =
1104 static_cast<size_t>(10 * kSampleRateHz / 1000); // 10 ms.
1105 const size_t kPayloadLengthBytes = 1;
1106 uint8_t payload[kPayloadLengthBytes] = {0};
1107 int16_t dummy_output[kPayloadLengthSamples * kChannels] = {0};
1108 RTPHeader rtp_header;
1109 rtp_header.payloadType = kPayloadType;
1110 rtp_header.sequenceNumber = 0x1234;
1111 rtp_header.timestamp = 0x12345678;
1112 rtp_header.ssrc = 0x87654321;
1113
1114 const uint8_t kFirstPayloadValue = 1;
1115 const uint8_t kSecondPayloadValue = 2;
1116
1117 EXPECT_CALL(decoder,
1118 PacketDuration(Pointee(kFirstPayloadValue), kPayloadLengthBytes))
1119 .Times(AtLeast(1))
1120 .WillRepeatedly(Return(rtc::checked_cast<int>(kNetEqMaxFrameSize + 1)));
1121
1122 EXPECT_CALL(decoder, DecodeInternal(Pointee(kFirstPayloadValue), _, _, _, _))
1123 .Times(0);
1124
1125 EXPECT_CALL(decoder, DecodeInternal(Pointee(kSecondPayloadValue),
1126 kPayloadLengthBytes, kSampleRateHz, _, _))
1127 .Times(1)
1128 .WillOnce(DoAll(
1129 SetArrayArgument<3>(dummy_output,
1130 dummy_output + kPayloadLengthSamples * kChannels),
1131 SetArgPointee<4>(AudioDecoder::kSpeech),
1132 Return(static_cast<int>(kPayloadLengthSamples * kChannels))));
1133
1134 EXPECT_CALL(decoder,
1135 PacketDuration(Pointee(kSecondPayloadValue), kPayloadLengthBytes))
1136 .Times(AtLeast(1))
1137 .WillRepeatedly(Return(rtc::checked_cast<int>(kNetEqMaxFrameSize)));
1138
1139 EXPECT_CALL(decoder, SampleRateHz()).WillRepeatedly(Return(kSampleRateHz));
1140
1141 EXPECT_CALL(decoder, Channels()).WillRepeatedly(Return(kChannels));
1142
1143 EXPECT_TRUE(neteq_->RegisterPayloadType(kPayloadType,
1144 SdpAudioFormat("L16", 8000, 1)));
1145
1146 // Insert one packet.
1147 payload[0] = kFirstPayloadValue; // This will make Decode() fail.
1148 EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload));
1149
1150 // Insert another packet.
1151 payload[0] = kSecondPayloadValue; // This will make Decode() successful.
1152 rtp_header.sequenceNumber++;
1153 // The second timestamp needs to be at least 30 ms after the first to make
1154 // the second packet get decoded.
1155 rtp_header.timestamp += 3 * kPayloadLengthSamples;
1156 EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload));
1157
1158 AudioFrame output;
1159 bool muted;
1160 // First call to GetAudio will try to decode the "faulty" packet.
1161 // Expect kFail return value.
1162 EXPECT_EQ(NetEq::kFail, neteq_->GetAudio(&output, &muted));
1163 // Output size and number of channels should be correct.
1164 const size_t kExpectedOutputSize = 10 * (kSampleRateHz / 1000) * kChannels;
1165 EXPECT_EQ(kExpectedOutputSize, output.samples_per_channel_ * kChannels);
1166 EXPECT_EQ(kChannels, output.num_channels_);
1167 EXPECT_THAT(output.packet_infos_, IsEmpty());
1168
1169 // Second call to GetAudio will decode the packet that is ok. No errors are
1170 // expected.
1171 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted));
1172 EXPECT_EQ(kExpectedOutputSize, output.samples_per_channel_ * kChannels);
1173 EXPECT_EQ(kChannels, output.num_channels_);
1174 EXPECT_THAT(output.packet_infos_, SizeIs(1));
1175
1176 // Die isn't called through NiceMock (since it's called by the
1177 // MockAudioDecoder constructor), so it needs to be mocked explicitly.
1178 EXPECT_CALL(decoder, Die());
1179 }
1180
1181 // This test inserts packets until the buffer is flushed. After that, it asks
1182 // NetEq for the network statistics. The purpose of the test is to make sure
1183 // that even though the buffer size increment is negative (which it becomes when
1184 // the packet causing a flush is inserted), the packet length stored in the
1185 // decision logic remains valid.
TEST_F(NetEqImplTest,FloodBufferAndGetNetworkStats)1186 TEST_F(NetEqImplTest, FloodBufferAndGetNetworkStats) {
1187 UseNoMocks();
1188 CreateInstance();
1189
1190 const size_t kPayloadLengthSamples = 80;
1191 const size_t kPayloadLengthBytes = 2 * kPayloadLengthSamples; // PCM 16-bit.
1192 const uint8_t kPayloadType = 17; // Just an arbitrary number.
1193 uint8_t payload[kPayloadLengthBytes] = {0};
1194 RTPHeader rtp_header;
1195 rtp_header.payloadType = kPayloadType;
1196 rtp_header.sequenceNumber = 0x1234;
1197 rtp_header.timestamp = 0x12345678;
1198 rtp_header.ssrc = 0x87654321;
1199
1200 EXPECT_TRUE(neteq_->RegisterPayloadType(kPayloadType,
1201 SdpAudioFormat("l16", 8000, 1)));
1202
1203 // Insert packets until the buffer flushes.
1204 for (size_t i = 0; i <= config_.max_packets_in_buffer; ++i) {
1205 EXPECT_EQ(i, packet_buffer_->NumPacketsInBuffer());
1206 EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload));
1207 rtp_header.timestamp += rtc::checked_cast<uint32_t>(kPayloadLengthSamples);
1208 ++rtp_header.sequenceNumber;
1209 }
1210 EXPECT_EQ(1u, packet_buffer_->NumPacketsInBuffer());
1211
1212 // Ask for network statistics. This should not crash.
1213 NetEqNetworkStatistics stats;
1214 EXPECT_EQ(NetEq::kOK, neteq_->NetworkStatistics(&stats));
1215 }
1216
TEST_F(NetEqImplTest,DecodedPayloadTooShort)1217 TEST_F(NetEqImplTest, DecodedPayloadTooShort) {
1218 UseNoMocks();
1219 // Create a mock decoder object.
1220 MockAudioDecoder mock_decoder;
1221
1222 CreateInstance(
1223 rtc::make_ref_counted<test::AudioDecoderProxyFactory>(&mock_decoder));
1224
1225 const uint8_t kPayloadType = 17; // Just an arbitrary number.
1226 const int kSampleRateHz = 8000;
1227 const size_t kPayloadLengthSamples =
1228 static_cast<size_t>(10 * kSampleRateHz / 1000); // 10 ms.
1229 const size_t kPayloadLengthBytes = 2 * kPayloadLengthSamples;
1230 uint8_t payload[kPayloadLengthBytes] = {0};
1231 RTPHeader rtp_header;
1232 rtp_header.payloadType = kPayloadType;
1233 rtp_header.sequenceNumber = 0x1234;
1234 rtp_header.timestamp = 0x12345678;
1235 rtp_header.ssrc = 0x87654321;
1236
1237 EXPECT_CALL(mock_decoder, Reset()).WillRepeatedly(Return());
1238 EXPECT_CALL(mock_decoder, SampleRateHz())
1239 .WillRepeatedly(Return(kSampleRateHz));
1240 EXPECT_CALL(mock_decoder, Channels()).WillRepeatedly(Return(1));
1241 EXPECT_CALL(mock_decoder, PacketDuration(_, _))
1242 .WillRepeatedly(Return(rtc::checked_cast<int>(kPayloadLengthSamples)));
1243 int16_t dummy_output[kPayloadLengthSamples] = {0};
1244 // The below expectation will make the mock decoder write
1245 // `kPayloadLengthSamples` - 5 zeros to the output array, and mark it as
1246 // speech. That is, the decoded length is 5 samples shorter than the expected.
1247 EXPECT_CALL(mock_decoder,
1248 DecodeInternal(_, kPayloadLengthBytes, kSampleRateHz, _, _))
1249 .WillOnce(
1250 DoAll(SetArrayArgument<3>(dummy_output,
1251 dummy_output + kPayloadLengthSamples - 5),
1252 SetArgPointee<4>(AudioDecoder::kSpeech),
1253 Return(rtc::checked_cast<int>(kPayloadLengthSamples - 5))));
1254 EXPECT_TRUE(neteq_->RegisterPayloadType(kPayloadType,
1255 SdpAudioFormat("L16", 8000, 1)));
1256
1257 // Insert one packet.
1258 EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload));
1259
1260 EXPECT_EQ(5u, neteq_->sync_buffer_for_test()->FutureLength());
1261
1262 // Pull audio once.
1263 const size_t kMaxOutputSize = static_cast<size_t>(10 * kSampleRateHz / 1000);
1264 AudioFrame output;
1265 bool muted;
1266 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted));
1267 ASSERT_EQ(kMaxOutputSize, output.samples_per_channel_);
1268 EXPECT_EQ(1u, output.num_channels_);
1269 EXPECT_EQ(AudioFrame::kNormalSpeech, output.speech_type_);
1270 EXPECT_THAT(output.packet_infos_, SizeIs(1));
1271
1272 EXPECT_CALL(mock_decoder, Die());
1273 }
1274
1275 // This test checks the behavior of NetEq when audio decoder fails.
TEST_F(NetEqImplTest,DecodingError)1276 TEST_F(NetEqImplTest, DecodingError) {
1277 UseNoMocks();
1278 // Create a mock decoder object.
1279 MockAudioDecoder mock_decoder;
1280
1281 CreateInstance(
1282 rtc::make_ref_counted<test::AudioDecoderProxyFactory>(&mock_decoder));
1283
1284 const uint8_t kPayloadType = 17; // Just an arbitrary number.
1285 const int kSampleRateHz = 8000;
1286 const int kDecoderErrorCode = -97; // Any negative number.
1287
1288 // We let decoder return 5 ms each time, and therefore, 2 packets make 10 ms.
1289 const size_t kFrameLengthSamples =
1290 static_cast<size_t>(5 * kSampleRateHz / 1000);
1291
1292 const size_t kPayloadLengthBytes = 1; // This can be arbitrary.
1293
1294 uint8_t payload[kPayloadLengthBytes] = {0};
1295
1296 RTPHeader rtp_header;
1297 rtp_header.payloadType = kPayloadType;
1298 rtp_header.sequenceNumber = 0x1234;
1299 rtp_header.timestamp = 0x12345678;
1300 rtp_header.ssrc = 0x87654321;
1301
1302 EXPECT_CALL(mock_decoder, Reset()).WillRepeatedly(Return());
1303 EXPECT_CALL(mock_decoder, SampleRateHz())
1304 .WillRepeatedly(Return(kSampleRateHz));
1305 EXPECT_CALL(mock_decoder, Channels()).WillRepeatedly(Return(1));
1306 EXPECT_CALL(mock_decoder, PacketDuration(_, _))
1307 .WillRepeatedly(Return(rtc::checked_cast<int>(kFrameLengthSamples)));
1308 EXPECT_CALL(mock_decoder, ErrorCode()).WillOnce(Return(kDecoderErrorCode));
1309 EXPECT_CALL(mock_decoder, HasDecodePlc()).WillOnce(Return(false));
1310 int16_t dummy_output[kFrameLengthSamples] = {0};
1311
1312 {
1313 InSequence sequence; // Dummy variable.
1314 // Mock decoder works normally the first time.
1315 EXPECT_CALL(mock_decoder,
1316 DecodeInternal(_, kPayloadLengthBytes, kSampleRateHz, _, _))
1317 .Times(3)
1318 .WillRepeatedly(
1319 DoAll(SetArrayArgument<3>(dummy_output,
1320 dummy_output + kFrameLengthSamples),
1321 SetArgPointee<4>(AudioDecoder::kSpeech),
1322 Return(rtc::checked_cast<int>(kFrameLengthSamples))))
1323 .RetiresOnSaturation();
1324
1325 // Then mock decoder fails. A common reason for failure can be buffer being
1326 // too short
1327 EXPECT_CALL(mock_decoder,
1328 DecodeInternal(_, kPayloadLengthBytes, kSampleRateHz, _, _))
1329 .WillOnce(Return(-1))
1330 .RetiresOnSaturation();
1331
1332 // Mock decoder finally returns to normal.
1333 EXPECT_CALL(mock_decoder,
1334 DecodeInternal(_, kPayloadLengthBytes, kSampleRateHz, _, _))
1335 .Times(2)
1336 .WillRepeatedly(
1337 DoAll(SetArrayArgument<3>(dummy_output,
1338 dummy_output + kFrameLengthSamples),
1339 SetArgPointee<4>(AudioDecoder::kSpeech),
1340 Return(rtc::checked_cast<int>(kFrameLengthSamples))));
1341 }
1342
1343 EXPECT_TRUE(neteq_->RegisterPayloadType(kPayloadType,
1344 SdpAudioFormat("L16", 8000, 1)));
1345
1346 // Insert packets.
1347 for (int i = 0; i < 20; ++i) {
1348 rtp_header.sequenceNumber += 1;
1349 rtp_header.timestamp += kFrameLengthSamples;
1350 EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload));
1351 }
1352
1353 // Pull audio.
1354 const size_t kMaxOutputSize = static_cast<size_t>(10 * kSampleRateHz / 1000);
1355 AudioFrame output;
1356 bool muted;
1357 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted));
1358 EXPECT_EQ(kMaxOutputSize, output.samples_per_channel_);
1359 EXPECT_EQ(1u, output.num_channels_);
1360 EXPECT_EQ(AudioFrame::kNormalSpeech, output.speech_type_);
1361 EXPECT_THAT(output.packet_infos_, SizeIs(2)); // 5 ms packets vs 10 ms output
1362
1363 // Pull audio again. Decoder fails.
1364 EXPECT_EQ(NetEq::kFail, neteq_->GetAudio(&output, &muted));
1365 EXPECT_EQ(kMaxOutputSize, output.samples_per_channel_);
1366 EXPECT_EQ(1u, output.num_channels_);
1367 // We are not expecting anything for output.speech_type_, since an error was
1368 // returned.
1369
1370 // Pull audio again, should behave normal.
1371 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted));
1372 EXPECT_EQ(kMaxOutputSize, output.samples_per_channel_);
1373 EXPECT_EQ(1u, output.num_channels_);
1374 EXPECT_EQ(AudioFrame::kNormalSpeech, output.speech_type_);
1375 EXPECT_THAT(output.packet_infos_, SizeIs(2)); // 5 ms packets vs 10 ms output
1376
1377 EXPECT_CALL(mock_decoder, Die());
1378 }
1379
1380 // This test checks the behavior of NetEq when audio decoder fails during CNG.
TEST_F(NetEqImplTest,DecodingErrorDuringInternalCng)1381 TEST_F(NetEqImplTest, DecodingErrorDuringInternalCng) {
1382 UseNoMocks();
1383
1384 // Create a mock decoder object.
1385 MockAudioDecoder mock_decoder;
1386 CreateInstance(
1387 rtc::make_ref_counted<test::AudioDecoderProxyFactory>(&mock_decoder));
1388
1389 const uint8_t kPayloadType = 17; // Just an arbitrary number.
1390 const int kSampleRateHz = 8000;
1391 const int kDecoderErrorCode = -97; // Any negative number.
1392
1393 // We let decoder return 5 ms each time, and therefore, 2 packets make 10 ms.
1394 const size_t kFrameLengthSamples =
1395 static_cast<size_t>(5 * kSampleRateHz / 1000);
1396
1397 const size_t kPayloadLengthBytes = 1; // This can be arbitrary.
1398
1399 uint8_t payload[kPayloadLengthBytes] = {0};
1400
1401 RTPHeader rtp_header;
1402 rtp_header.payloadType = kPayloadType;
1403 rtp_header.sequenceNumber = 0x1234;
1404 rtp_header.timestamp = 0x12345678;
1405 rtp_header.ssrc = 0x87654321;
1406
1407 EXPECT_CALL(mock_decoder, Reset()).WillRepeatedly(Return());
1408 EXPECT_CALL(mock_decoder, SampleRateHz())
1409 .WillRepeatedly(Return(kSampleRateHz));
1410 EXPECT_CALL(mock_decoder, Channels()).WillRepeatedly(Return(1));
1411 EXPECT_CALL(mock_decoder, PacketDuration(_, _))
1412 .WillRepeatedly(Return(rtc::checked_cast<int>(kFrameLengthSamples)));
1413 EXPECT_CALL(mock_decoder, ErrorCode()).WillOnce(Return(kDecoderErrorCode));
1414 int16_t dummy_output[kFrameLengthSamples] = {0};
1415
1416 {
1417 InSequence sequence; // Dummy variable.
1418 // Mock decoder works normally the first 2 times.
1419 EXPECT_CALL(mock_decoder,
1420 DecodeInternal(_, kPayloadLengthBytes, kSampleRateHz, _, _))
1421 .Times(2)
1422 .WillRepeatedly(
1423 DoAll(SetArrayArgument<3>(dummy_output,
1424 dummy_output + kFrameLengthSamples),
1425 SetArgPointee<4>(AudioDecoder::kComfortNoise),
1426 Return(rtc::checked_cast<int>(kFrameLengthSamples))))
1427 .RetiresOnSaturation();
1428
1429 // Then mock decoder fails. A common reason for failure can be buffer being
1430 // too short
1431 EXPECT_CALL(mock_decoder, DecodeInternal(nullptr, 0, kSampleRateHz, _, _))
1432 .WillOnce(Return(-1))
1433 .RetiresOnSaturation();
1434
1435 // Mock decoder finally returns to normal.
1436 EXPECT_CALL(mock_decoder, DecodeInternal(nullptr, 0, kSampleRateHz, _, _))
1437 .Times(2)
1438 .WillRepeatedly(
1439 DoAll(SetArrayArgument<3>(dummy_output,
1440 dummy_output + kFrameLengthSamples),
1441 SetArgPointee<4>(AudioDecoder::kComfortNoise),
1442 Return(rtc::checked_cast<int>(kFrameLengthSamples))));
1443 }
1444
1445 EXPECT_TRUE(neteq_->RegisterPayloadType(kPayloadType,
1446 SdpAudioFormat("l16", 8000, 1)));
1447
1448 // Insert 2 packets. This will make netEq into codec internal CNG mode.
1449 for (int i = 0; i < 2; ++i) {
1450 rtp_header.sequenceNumber += 1;
1451 rtp_header.timestamp += kFrameLengthSamples;
1452 EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload));
1453 }
1454
1455 // Pull audio.
1456 const size_t kMaxOutputSize = static_cast<size_t>(10 * kSampleRateHz / 1000);
1457 AudioFrame output;
1458 bool muted;
1459 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted));
1460 EXPECT_EQ(kMaxOutputSize, output.samples_per_channel_);
1461 EXPECT_EQ(1u, output.num_channels_);
1462 EXPECT_EQ(AudioFrame::kCNG, output.speech_type_);
1463
1464 // Pull audio again. Decoder fails.
1465 EXPECT_EQ(NetEq::kFail, neteq_->GetAudio(&output, &muted));
1466 EXPECT_EQ(kMaxOutputSize, output.samples_per_channel_);
1467 EXPECT_EQ(1u, output.num_channels_);
1468 // We are not expecting anything for output.speech_type_, since an error was
1469 // returned.
1470
1471 // Pull audio again, should resume codec CNG.
1472 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted));
1473 EXPECT_EQ(kMaxOutputSize, output.samples_per_channel_);
1474 EXPECT_EQ(1u, output.num_channels_);
1475 EXPECT_EQ(AudioFrame::kCNG, output.speech_type_);
1476
1477 EXPECT_CALL(mock_decoder, Die());
1478 }
1479
1480 // Tests that the return value from last_output_sample_rate_hz() is equal to the
1481 // configured inital sample rate.
TEST_F(NetEqImplTest,InitialLastOutputSampleRate)1482 TEST_F(NetEqImplTest, InitialLastOutputSampleRate) {
1483 UseNoMocks();
1484 config_.sample_rate_hz = 48000;
1485 CreateInstance();
1486 EXPECT_EQ(48000, neteq_->last_output_sample_rate_hz());
1487 }
1488
TEST_F(NetEqImplTest,TickTimerIncrement)1489 TEST_F(NetEqImplTest, TickTimerIncrement) {
1490 UseNoMocks();
1491 CreateInstance();
1492 ASSERT_TRUE(tick_timer_);
1493 EXPECT_EQ(0u, tick_timer_->ticks());
1494 AudioFrame output;
1495 bool muted;
1496 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted));
1497 EXPECT_EQ(1u, tick_timer_->ticks());
1498 }
1499
TEST_F(NetEqImplTest,SetBaseMinimumDelay)1500 TEST_F(NetEqImplTest, SetBaseMinimumDelay) {
1501 UseNoMocks();
1502 use_mock_neteq_controller_ = true;
1503 CreateInstance();
1504
1505 EXPECT_CALL(*mock_neteq_controller_, SetBaseMinimumDelay(_))
1506 .WillOnce(Return(true))
1507 .WillOnce(Return(false));
1508
1509 const int delay_ms = 200;
1510
1511 EXPECT_EQ(true, neteq_->SetBaseMinimumDelayMs(delay_ms));
1512 EXPECT_EQ(false, neteq_->SetBaseMinimumDelayMs(delay_ms));
1513 }
1514
TEST_F(NetEqImplTest,GetBaseMinimumDelayMs)1515 TEST_F(NetEqImplTest, GetBaseMinimumDelayMs) {
1516 UseNoMocks();
1517 use_mock_neteq_controller_ = true;
1518 CreateInstance();
1519
1520 const int delay_ms = 200;
1521
1522 EXPECT_CALL(*mock_neteq_controller_, GetBaseMinimumDelay())
1523 .WillOnce(Return(delay_ms));
1524
1525 EXPECT_EQ(delay_ms, neteq_->GetBaseMinimumDelayMs());
1526 }
1527
TEST_F(NetEqImplTest,TargetDelayMs)1528 TEST_F(NetEqImplTest, TargetDelayMs) {
1529 UseNoMocks();
1530 use_mock_neteq_controller_ = true;
1531 CreateInstance();
1532 constexpr int kTargetLevelMs = 510;
1533 EXPECT_CALL(*mock_neteq_controller_, TargetLevelMs())
1534 .WillOnce(Return(kTargetLevelMs));
1535 EXPECT_EQ(510, neteq_->TargetDelayMs());
1536 }
1537
TEST_F(NetEqImplTest,InsertEmptyPacket)1538 TEST_F(NetEqImplTest, InsertEmptyPacket) {
1539 UseNoMocks();
1540 use_mock_neteq_controller_ = true;
1541 CreateInstance();
1542
1543 RTPHeader rtp_header;
1544 rtp_header.payloadType = 17;
1545 rtp_header.sequenceNumber = 0x1234;
1546 rtp_header.timestamp = 0x12345678;
1547 rtp_header.ssrc = 0x87654321;
1548
1549 EXPECT_CALL(*mock_neteq_controller_, RegisterEmptyPacket());
1550 neteq_->InsertEmptyPacket(rtp_header);
1551 }
1552
TEST_F(NetEqImplTest,NotifyControllerOfReorderedPacket)1553 TEST_F(NetEqImplTest, NotifyControllerOfReorderedPacket) {
1554 using ::testing::AllOf;
1555 using ::testing::Field;
1556 UseNoMocks();
1557 use_mock_neteq_controller_ = true;
1558 CreateInstance();
1559 EXPECT_CALL(*mock_neteq_controller_, GetDecision(_, _))
1560 .Times(1)
1561 .WillOnce(Return(NetEq::Operation::kNormal));
1562
1563 const int kPayloadLengthSamples = 80;
1564 const size_t kPayloadLengthBytes = 2 * kPayloadLengthSamples; // PCM 16-bit.
1565 const uint8_t kPayloadType = 17; // Just an arbitrary number.
1566 uint8_t payload[kPayloadLengthBytes] = {0};
1567 RTPHeader rtp_header;
1568 rtp_header.payloadType = kPayloadType;
1569 rtp_header.sequenceNumber = 0x1234;
1570 rtp_header.timestamp = 0x12345678;
1571 rtp_header.ssrc = 0x87654321;
1572
1573 EXPECT_TRUE(neteq_->RegisterPayloadType(kPayloadType,
1574 SdpAudioFormat("l16", 8000, 1)));
1575 EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload));
1576 AudioFrame output;
1577 bool muted;
1578 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted));
1579
1580 // Insert second packet that was sent before the first packet.
1581 rtp_header.sequenceNumber -= 1;
1582 rtp_header.timestamp -= kPayloadLengthSamples;
1583 EXPECT_CALL(
1584 *mock_neteq_controller_,
1585 PacketArrived(
1586 /*fs_hz*/ 8000,
1587 /*should_update_stats*/ true,
1588 /*info*/
1589 AllOf(
1590 Field(&NetEqController::PacketArrivedInfo::packet_length_samples,
1591 kPayloadLengthSamples),
1592 Field(&NetEqController::PacketArrivedInfo::main_sequence_number,
1593 rtp_header.sequenceNumber),
1594 Field(&NetEqController::PacketArrivedInfo::main_timestamp,
1595 rtp_header.timestamp))));
1596
1597 EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload));
1598 }
1599
1600 // When using a codec with 1000 channels, there should be no crashes.
TEST_F(NetEqImplTest,NoCrashWith1000Channels)1601 TEST_F(NetEqImplTest, NoCrashWith1000Channels) {
1602 using ::testing::AllOf;
1603 using ::testing::Field;
1604 UseNoMocks();
1605 use_mock_decoder_database_ = true;
1606 enable_muted_state_ = true;
1607 CreateInstance();
1608 const size_t kPayloadLength = 100;
1609 const uint8_t kPayloadType = 0;
1610 const uint16_t kFirstSequenceNumber = 0x1234;
1611 const uint32_t kFirstTimestamp = 0x12345678;
1612 const uint32_t kSsrc = 0x87654321;
1613 uint8_t payload[kPayloadLength] = {0};
1614 RTPHeader rtp_header;
1615 rtp_header.payloadType = kPayloadType;
1616 rtp_header.sequenceNumber = kFirstSequenceNumber;
1617 rtp_header.timestamp = kFirstTimestamp;
1618 rtp_header.ssrc = kSsrc;
1619 Packet fake_packet;
1620 fake_packet.payload_type = kPayloadType;
1621 fake_packet.sequence_number = kFirstSequenceNumber;
1622 fake_packet.timestamp = kFirstTimestamp;
1623
1624 AudioDecoder* decoder = nullptr;
1625
1626 auto mock_decoder_factory = rtc::make_ref_counted<MockAudioDecoderFactory>();
1627 EXPECT_CALL(*mock_decoder_factory, MakeAudioDecoderMock(_, _, _))
1628 .WillOnce(Invoke([&](const SdpAudioFormat& format,
1629 absl::optional<AudioCodecPairId> codec_pair_id,
1630 std::unique_ptr<AudioDecoder>* dec) {
1631 EXPECT_EQ("pcmu", format.name);
1632 *dec = std::make_unique<AudioDecoderPcmU>(1000);
1633 decoder = dec->get();
1634 }));
1635 DecoderDatabase::DecoderInfo info(SdpAudioFormat("pcmu", 8000, 1),
1636 absl::nullopt, mock_decoder_factory.get());
1637 // Expectations for decoder database.
1638 EXPECT_CALL(*mock_decoder_database_, GetDecoderInfo(kPayloadType))
1639 .WillRepeatedly(Return(&info));
1640 EXPECT_CALL(*mock_decoder_database_, GetActiveCngDecoder())
1641 .WillRepeatedly(ReturnNull());
1642 EXPECT_CALL(*mock_decoder_database_, GetActiveDecoder())
1643 .WillRepeatedly(Return(decoder));
1644 EXPECT_CALL(*mock_decoder_database_, SetActiveDecoder(_, _))
1645 .WillOnce(Invoke([](uint8_t rtp_payload_type, bool* new_decoder) {
1646 *new_decoder = true;
1647 return 0;
1648 }));
1649
1650 // Insert first packet.
1651 neteq_->InsertPacket(rtp_header, payload);
1652
1653 AudioFrame audio_frame;
1654 bool muted;
1655
1656 // Repeat 40 times to ensure we enter muted state.
1657 for (int i = 0; i < 40; i++) {
1658 // GetAudio should return an error, and not crash, even in muted state.
1659 EXPECT_NE(0, neteq_->GetAudio(&audio_frame, &muted));
1660 }
1661 }
1662
1663 class Decoder120ms : public AudioDecoder {
1664 public:
Decoder120ms(int sample_rate_hz,SpeechType speech_type)1665 Decoder120ms(int sample_rate_hz, SpeechType speech_type)
1666 : sample_rate_hz_(sample_rate_hz),
1667 next_value_(1),
1668 speech_type_(speech_type) {}
1669
DecodeInternal(const uint8_t * encoded,size_t encoded_len,int sample_rate_hz,int16_t * decoded,SpeechType * speech_type)1670 int DecodeInternal(const uint8_t* encoded,
1671 size_t encoded_len,
1672 int sample_rate_hz,
1673 int16_t* decoded,
1674 SpeechType* speech_type) override {
1675 EXPECT_EQ(sample_rate_hz_, sample_rate_hz);
1676 size_t decoded_len =
1677 rtc::CheckedDivExact(sample_rate_hz, 1000) * 120 * Channels();
1678 for (size_t i = 0; i < decoded_len; ++i) {
1679 decoded[i] = next_value_++;
1680 }
1681 *speech_type = speech_type_;
1682 return rtc::checked_cast<int>(decoded_len);
1683 }
1684
Reset()1685 void Reset() override { next_value_ = 1; }
SampleRateHz() const1686 int SampleRateHz() const override { return sample_rate_hz_; }
Channels() const1687 size_t Channels() const override { return 2; }
1688
1689 private:
1690 int sample_rate_hz_;
1691 int16_t next_value_;
1692 SpeechType speech_type_;
1693 };
1694
1695 class NetEqImplTest120ms : public NetEqImplTest {
1696 protected:
NetEqImplTest120ms()1697 NetEqImplTest120ms() : NetEqImplTest() {}
~NetEqImplTest120ms()1698 virtual ~NetEqImplTest120ms() {}
1699
CreateInstanceNoMocks()1700 void CreateInstanceNoMocks() {
1701 UseNoMocks();
1702 CreateInstance(decoder_factory_);
1703 EXPECT_TRUE(neteq_->RegisterPayloadType(
1704 kPayloadType, SdpAudioFormat("opus", 48000, 2, {{"stereo", "1"}})));
1705 }
1706
CreateInstanceWithDelayManagerMock()1707 void CreateInstanceWithDelayManagerMock() {
1708 UseNoMocks();
1709 use_mock_neteq_controller_ = true;
1710 CreateInstance(decoder_factory_);
1711 EXPECT_TRUE(neteq_->RegisterPayloadType(
1712 kPayloadType, SdpAudioFormat("opus", 48000, 2, {{"stereo", "1"}})));
1713 }
1714
timestamp_diff_between_packets() const1715 uint32_t timestamp_diff_between_packets() const {
1716 return rtc::CheckedDivExact(kSamplingFreq_, 1000u) * 120;
1717 }
1718
first_timestamp() const1719 uint32_t first_timestamp() const { return 10u; }
1720
GetFirstPacket()1721 void GetFirstPacket() {
1722 bool muted;
1723 for (int i = 0; i < 12; i++) {
1724 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_, &muted));
1725 EXPECT_FALSE(muted);
1726 }
1727 }
1728
InsertPacket(uint32_t timestamp)1729 void InsertPacket(uint32_t timestamp) {
1730 RTPHeader rtp_header;
1731 rtp_header.payloadType = kPayloadType;
1732 rtp_header.sequenceNumber = sequence_number_;
1733 rtp_header.timestamp = timestamp;
1734 rtp_header.ssrc = 15;
1735 const size_t kPayloadLengthBytes = 1; // This can be arbitrary.
1736 uint8_t payload[kPayloadLengthBytes] = {0};
1737 EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload));
1738 sequence_number_++;
1739 }
1740
Register120msCodec(AudioDecoder::SpeechType speech_type)1741 void Register120msCodec(AudioDecoder::SpeechType speech_type) {
1742 const uint32_t sampling_freq = kSamplingFreq_;
1743 decoder_factory_ = rtc::make_ref_counted<test::FunctionAudioDecoderFactory>(
1744 [sampling_freq, speech_type]() {
1745 std::unique_ptr<AudioDecoder> decoder =
1746 std::make_unique<Decoder120ms>(sampling_freq, speech_type);
1747 RTC_CHECK_EQ(2, decoder->Channels());
1748 return decoder;
1749 });
1750 }
1751
1752 rtc::scoped_refptr<AudioDecoderFactory> decoder_factory_;
1753 AudioFrame output_;
1754 const uint32_t kPayloadType = 17;
1755 const uint32_t kSamplingFreq_ = 48000;
1756 uint16_t sequence_number_ = 1;
1757 };
1758
TEST_F(NetEqImplTest120ms,CodecInternalCng)1759 TEST_F(NetEqImplTest120ms, CodecInternalCng) {
1760 Register120msCodec(AudioDecoder::kComfortNoise);
1761 CreateInstanceNoMocks();
1762
1763 InsertPacket(first_timestamp());
1764 GetFirstPacket();
1765
1766 bool muted;
1767 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_, &muted));
1768 EXPECT_EQ(NetEq::Operation::kCodecInternalCng,
1769 neteq_->last_operation_for_test());
1770 }
1771
TEST_F(NetEqImplTest120ms,Normal)1772 TEST_F(NetEqImplTest120ms, Normal) {
1773 Register120msCodec(AudioDecoder::kSpeech);
1774 CreateInstanceNoMocks();
1775
1776 InsertPacket(first_timestamp());
1777 GetFirstPacket();
1778
1779 EXPECT_EQ(NetEq::Operation::kNormal, neteq_->last_operation_for_test());
1780 }
1781
TEST_F(NetEqImplTest120ms,Merge)1782 TEST_F(NetEqImplTest120ms, Merge) {
1783 Register120msCodec(AudioDecoder::kSpeech);
1784 CreateInstanceWithDelayManagerMock();
1785
1786 EXPECT_CALL(*mock_neteq_controller_, CngOff()).WillRepeatedly(Return(true));
1787 InsertPacket(first_timestamp());
1788
1789 GetFirstPacket();
1790 bool muted;
1791 EXPECT_CALL(*mock_neteq_controller_, GetDecision(_, _))
1792 .WillOnce(Return(NetEq::Operation::kExpand));
1793 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_, &muted));
1794
1795 InsertPacket(first_timestamp() + 2 * timestamp_diff_between_packets());
1796
1797 EXPECT_CALL(*mock_neteq_controller_, GetDecision(_, _))
1798 .WillOnce(Return(NetEq::Operation::kMerge));
1799
1800 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_, &muted));
1801 EXPECT_EQ(NetEq::Operation::kMerge, neteq_->last_operation_for_test());
1802 }
1803
TEST_F(NetEqImplTest120ms,Expand)1804 TEST_F(NetEqImplTest120ms, Expand) {
1805 Register120msCodec(AudioDecoder::kSpeech);
1806 CreateInstanceNoMocks();
1807
1808 InsertPacket(first_timestamp());
1809 GetFirstPacket();
1810
1811 bool muted;
1812 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_, &muted));
1813 EXPECT_EQ(NetEq::Operation::kExpand, neteq_->last_operation_for_test());
1814 }
1815
TEST_F(NetEqImplTest120ms,FastAccelerate)1816 TEST_F(NetEqImplTest120ms, FastAccelerate) {
1817 Register120msCodec(AudioDecoder::kSpeech);
1818 CreateInstanceWithDelayManagerMock();
1819
1820 InsertPacket(first_timestamp());
1821 GetFirstPacket();
1822 InsertPacket(first_timestamp() + timestamp_diff_between_packets());
1823
1824 EXPECT_CALL(*mock_neteq_controller_, GetDecision(_, _))
1825 .Times(1)
1826 .WillOnce(Return(NetEq::Operation::kFastAccelerate));
1827
1828 bool muted;
1829 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_, &muted));
1830 EXPECT_EQ(NetEq::Operation::kFastAccelerate,
1831 neteq_->last_operation_for_test());
1832 }
1833
TEST_F(NetEqImplTest120ms,PreemptiveExpand)1834 TEST_F(NetEqImplTest120ms, PreemptiveExpand) {
1835 Register120msCodec(AudioDecoder::kSpeech);
1836 CreateInstanceWithDelayManagerMock();
1837
1838 InsertPacket(first_timestamp());
1839 GetFirstPacket();
1840
1841 InsertPacket(first_timestamp() + timestamp_diff_between_packets());
1842
1843 EXPECT_CALL(*mock_neteq_controller_, GetDecision(_, _))
1844 .Times(1)
1845 .WillOnce(Return(NetEq::Operation::kPreemptiveExpand));
1846
1847 bool muted;
1848 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_, &muted));
1849 EXPECT_EQ(NetEq::Operation::kPreemptiveExpand,
1850 neteq_->last_operation_for_test());
1851 }
1852
TEST_F(NetEqImplTest120ms,Accelerate)1853 TEST_F(NetEqImplTest120ms, Accelerate) {
1854 Register120msCodec(AudioDecoder::kSpeech);
1855 CreateInstanceWithDelayManagerMock();
1856
1857 InsertPacket(first_timestamp());
1858 GetFirstPacket();
1859
1860 InsertPacket(first_timestamp() + timestamp_diff_between_packets());
1861
1862 EXPECT_CALL(*mock_neteq_controller_, GetDecision(_, _))
1863 .Times(1)
1864 .WillOnce(Return(NetEq::Operation::kAccelerate));
1865
1866 bool muted;
1867 EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_, &muted));
1868 EXPECT_EQ(NetEq::Operation::kAccelerate, neteq_->last_operation_for_test());
1869 }
1870
1871 } // namespace webrtc
1872