1 /*
2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/audio_device/android/opensles_player.h"
12
13 #include <android/log.h>
14
15 #include <memory>
16
17 #include "api/array_view.h"
18 #include "modules/audio_device/android/audio_common.h"
19 #include "modules/audio_device/android/audio_manager.h"
20 #include "modules/audio_device/fine_audio_buffer.h"
21 #include "rtc_base/arraysize.h"
22 #include "rtc_base/checks.h"
23 #include "rtc_base/platform_thread.h"
24 #include "rtc_base/time_utils.h"
25
26 #define TAG "OpenSLESPlayer"
27 #define ALOGV(...) __android_log_print(ANDROID_LOG_VERBOSE, TAG, __VA_ARGS__)
28 #define ALOGD(...) __android_log_print(ANDROID_LOG_DEBUG, TAG, __VA_ARGS__)
29 #define ALOGE(...) __android_log_print(ANDROID_LOG_ERROR, TAG, __VA_ARGS__)
30 #define ALOGW(...) __android_log_print(ANDROID_LOG_WARN, TAG, __VA_ARGS__)
31 #define ALOGI(...) __android_log_print(ANDROID_LOG_INFO, TAG, __VA_ARGS__)
32
33 #define RETURN_ON_ERROR(op, ...) \
34 do { \
35 SLresult err = (op); \
36 if (err != SL_RESULT_SUCCESS) { \
37 ALOGE("%s failed: %s", #op, GetSLErrorString(err)); \
38 return __VA_ARGS__; \
39 } \
40 } while (0)
41
42 namespace webrtc {
43
OpenSLESPlayer(AudioManager * audio_manager)44 OpenSLESPlayer::OpenSLESPlayer(AudioManager* audio_manager)
45 : audio_manager_(audio_manager),
46 audio_parameters_(audio_manager->GetPlayoutAudioParameters()),
47 audio_device_buffer_(nullptr),
48 initialized_(false),
49 playing_(false),
50 buffer_index_(0),
51 engine_(nullptr),
52 player_(nullptr),
53 simple_buffer_queue_(nullptr),
54 volume_(nullptr),
55 last_play_time_(0) {
56 ALOGD("ctor[tid=%d]", rtc::CurrentThreadId());
57 // Use native audio output parameters provided by the audio manager and
58 // define the PCM format structure.
59 pcm_format_ = CreatePCMConfiguration(audio_parameters_.channels(),
60 audio_parameters_.sample_rate(),
61 audio_parameters_.bits_per_sample());
62 // Detach from this thread since we want to use the checker to verify calls
63 // from the internal audio thread.
64 thread_checker_opensles_.Detach();
65 }
66
~OpenSLESPlayer()67 OpenSLESPlayer::~OpenSLESPlayer() {
68 ALOGD("dtor[tid=%d]", rtc::CurrentThreadId());
69 RTC_DCHECK(thread_checker_.IsCurrent());
70 Terminate();
71 DestroyAudioPlayer();
72 DestroyMix();
73 engine_ = nullptr;
74 RTC_DCHECK(!engine_);
75 RTC_DCHECK(!output_mix_.Get());
76 RTC_DCHECK(!player_);
77 RTC_DCHECK(!simple_buffer_queue_);
78 RTC_DCHECK(!volume_);
79 }
80
Init()81 int OpenSLESPlayer::Init() {
82 ALOGD("Init[tid=%d]", rtc::CurrentThreadId());
83 RTC_DCHECK(thread_checker_.IsCurrent());
84 if (audio_parameters_.channels() == 2) {
85 ALOGW("Stereo mode is enabled");
86 }
87 return 0;
88 }
89
Terminate()90 int OpenSLESPlayer::Terminate() {
91 ALOGD("Terminate[tid=%d]", rtc::CurrentThreadId());
92 RTC_DCHECK(thread_checker_.IsCurrent());
93 StopPlayout();
94 return 0;
95 }
96
InitPlayout()97 int OpenSLESPlayer::InitPlayout() {
98 ALOGD("InitPlayout[tid=%d]", rtc::CurrentThreadId());
99 RTC_DCHECK(thread_checker_.IsCurrent());
100 RTC_DCHECK(!initialized_);
101 RTC_DCHECK(!playing_);
102 if (!ObtainEngineInterface()) {
103 ALOGE("Failed to obtain SL Engine interface");
104 return -1;
105 }
106 CreateMix();
107 initialized_ = true;
108 buffer_index_ = 0;
109 return 0;
110 }
111
StartPlayout()112 int OpenSLESPlayer::StartPlayout() {
113 ALOGD("StartPlayout[tid=%d]", rtc::CurrentThreadId());
114 RTC_DCHECK(thread_checker_.IsCurrent());
115 RTC_DCHECK(initialized_);
116 RTC_DCHECK(!playing_);
117 if (fine_audio_buffer_) {
118 fine_audio_buffer_->ResetPlayout();
119 }
120 // The number of lower latency audio players is limited, hence we create the
121 // audio player in Start() and destroy it in Stop().
122 CreateAudioPlayer();
123 // Fill up audio buffers to avoid initial glitch and to ensure that playback
124 // starts when mode is later changed to SL_PLAYSTATE_PLAYING.
125 // TODO(henrika): we can save some delay by only making one call to
126 // EnqueuePlayoutData. Most likely not worth the risk of adding a glitch.
127 last_play_time_ = rtc::Time();
128 for (int i = 0; i < kNumOfOpenSLESBuffers; ++i) {
129 EnqueuePlayoutData(true);
130 }
131 // Start streaming data by setting the play state to SL_PLAYSTATE_PLAYING.
132 // For a player object, when the object is in the SL_PLAYSTATE_PLAYING
133 // state, adding buffers will implicitly start playback.
134 RETURN_ON_ERROR((*player_)->SetPlayState(player_, SL_PLAYSTATE_PLAYING), -1);
135 playing_ = (GetPlayState() == SL_PLAYSTATE_PLAYING);
136 RTC_DCHECK(playing_);
137 return 0;
138 }
139
StopPlayout()140 int OpenSLESPlayer::StopPlayout() {
141 ALOGD("StopPlayout[tid=%d]", rtc::CurrentThreadId());
142 RTC_DCHECK(thread_checker_.IsCurrent());
143 if (!initialized_ || !playing_) {
144 return 0;
145 }
146 // Stop playing by setting the play state to SL_PLAYSTATE_STOPPED.
147 RETURN_ON_ERROR((*player_)->SetPlayState(player_, SL_PLAYSTATE_STOPPED), -1);
148 // Clear the buffer queue to flush out any remaining data.
149 RETURN_ON_ERROR((*simple_buffer_queue_)->Clear(simple_buffer_queue_), -1);
150 #if RTC_DCHECK_IS_ON
151 // Verify that the buffer queue is in fact cleared as it should.
152 SLAndroidSimpleBufferQueueState buffer_queue_state;
153 (*simple_buffer_queue_)->GetState(simple_buffer_queue_, &buffer_queue_state);
154 RTC_DCHECK_EQ(0, buffer_queue_state.count);
155 RTC_DCHECK_EQ(0, buffer_queue_state.index);
156 #endif
157 // The number of lower latency audio players is limited, hence we create the
158 // audio player in Start() and destroy it in Stop().
159 DestroyAudioPlayer();
160 thread_checker_opensles_.Detach();
161 initialized_ = false;
162 playing_ = false;
163 return 0;
164 }
165
SpeakerVolumeIsAvailable(bool & available)166 int OpenSLESPlayer::SpeakerVolumeIsAvailable(bool& available) {
167 available = false;
168 return 0;
169 }
170
MaxSpeakerVolume(uint32_t & maxVolume) const171 int OpenSLESPlayer::MaxSpeakerVolume(uint32_t& maxVolume) const {
172 return -1;
173 }
174
MinSpeakerVolume(uint32_t & minVolume) const175 int OpenSLESPlayer::MinSpeakerVolume(uint32_t& minVolume) const {
176 return -1;
177 }
178
SetSpeakerVolume(uint32_t volume)179 int OpenSLESPlayer::SetSpeakerVolume(uint32_t volume) {
180 return -1;
181 }
182
SpeakerVolume(uint32_t & volume) const183 int OpenSLESPlayer::SpeakerVolume(uint32_t& volume) const {
184 return -1;
185 }
186
AttachAudioBuffer(AudioDeviceBuffer * audioBuffer)187 void OpenSLESPlayer::AttachAudioBuffer(AudioDeviceBuffer* audioBuffer) {
188 ALOGD("AttachAudioBuffer");
189 RTC_DCHECK(thread_checker_.IsCurrent());
190 audio_device_buffer_ = audioBuffer;
191 const int sample_rate_hz = audio_parameters_.sample_rate();
192 ALOGD("SetPlayoutSampleRate(%d)", sample_rate_hz);
193 audio_device_buffer_->SetPlayoutSampleRate(sample_rate_hz);
194 const size_t channels = audio_parameters_.channels();
195 ALOGD("SetPlayoutChannels(%zu)", channels);
196 audio_device_buffer_->SetPlayoutChannels(channels);
197 RTC_CHECK(audio_device_buffer_);
198 AllocateDataBuffers();
199 }
200
AllocateDataBuffers()201 void OpenSLESPlayer::AllocateDataBuffers() {
202 ALOGD("AllocateDataBuffers");
203 RTC_DCHECK(thread_checker_.IsCurrent());
204 RTC_DCHECK(!simple_buffer_queue_);
205 RTC_CHECK(audio_device_buffer_);
206 // Create a modified audio buffer class which allows us to ask for any number
207 // of samples (and not only multiple of 10ms) to match the native OpenSL ES
208 // buffer size. The native buffer size corresponds to the
209 // PROPERTY_OUTPUT_FRAMES_PER_BUFFER property which is the number of audio
210 // frames that the HAL (Hardware Abstraction Layer) buffer can hold. It is
211 // recommended to construct audio buffers so that they contain an exact
212 // multiple of this number. If so, callbacks will occur at regular intervals,
213 // which reduces jitter.
214 const size_t buffer_size_in_samples =
215 audio_parameters_.frames_per_buffer() * audio_parameters_.channels();
216 ALOGD("native buffer size: %zu", buffer_size_in_samples);
217 ALOGD("native buffer size in ms: %.2f",
218 audio_parameters_.GetBufferSizeInMilliseconds());
219 fine_audio_buffer_ = std::make_unique<FineAudioBuffer>(audio_device_buffer_);
220 // Allocated memory for audio buffers.
221 for (int i = 0; i < kNumOfOpenSLESBuffers; ++i) {
222 audio_buffers_[i].reset(new SLint16[buffer_size_in_samples]);
223 }
224 }
225
ObtainEngineInterface()226 bool OpenSLESPlayer::ObtainEngineInterface() {
227 ALOGD("ObtainEngineInterface");
228 RTC_DCHECK(thread_checker_.IsCurrent());
229 if (engine_)
230 return true;
231 // Get access to (or create if not already existing) the global OpenSL Engine
232 // object.
233 SLObjectItf engine_object = audio_manager_->GetOpenSLEngine();
234 if (engine_object == nullptr) {
235 ALOGE("Failed to access the global OpenSL engine");
236 return false;
237 }
238 // Get the SL Engine Interface which is implicit.
239 RETURN_ON_ERROR(
240 (*engine_object)->GetInterface(engine_object, SL_IID_ENGINE, &engine_),
241 false);
242 return true;
243 }
244
CreateMix()245 bool OpenSLESPlayer::CreateMix() {
246 ALOGD("CreateMix");
247 RTC_DCHECK(thread_checker_.IsCurrent());
248 RTC_DCHECK(engine_);
249 if (output_mix_.Get())
250 return true;
251
252 // Create the ouput mix on the engine object. No interfaces will be used.
253 RETURN_ON_ERROR((*engine_)->CreateOutputMix(engine_, output_mix_.Receive(), 0,
254 nullptr, nullptr),
255 false);
256 RETURN_ON_ERROR(output_mix_->Realize(output_mix_.Get(), SL_BOOLEAN_FALSE),
257 false);
258 return true;
259 }
260
DestroyMix()261 void OpenSLESPlayer::DestroyMix() {
262 ALOGD("DestroyMix");
263 RTC_DCHECK(thread_checker_.IsCurrent());
264 if (!output_mix_.Get())
265 return;
266 output_mix_.Reset();
267 }
268
CreateAudioPlayer()269 bool OpenSLESPlayer::CreateAudioPlayer() {
270 ALOGD("CreateAudioPlayer");
271 RTC_DCHECK(thread_checker_.IsCurrent());
272 RTC_DCHECK(output_mix_.Get());
273 if (player_object_.Get())
274 return true;
275 RTC_DCHECK(!player_);
276 RTC_DCHECK(!simple_buffer_queue_);
277 RTC_DCHECK(!volume_);
278
279 // source: Android Simple Buffer Queue Data Locator is source.
280 SLDataLocator_AndroidSimpleBufferQueue simple_buffer_queue = {
281 SL_DATALOCATOR_ANDROIDSIMPLEBUFFERQUEUE,
282 static_cast<SLuint32>(kNumOfOpenSLESBuffers)};
283 SLDataSource audio_source = {&simple_buffer_queue, &pcm_format_};
284
285 // sink: OutputMix-based data is sink.
286 SLDataLocator_OutputMix locator_output_mix = {SL_DATALOCATOR_OUTPUTMIX,
287 output_mix_.Get()};
288 SLDataSink audio_sink = {&locator_output_mix, nullptr};
289
290 // Define interfaces that we indend to use and realize.
291 const SLInterfaceID interface_ids[] = {SL_IID_ANDROIDCONFIGURATION,
292 SL_IID_BUFFERQUEUE, SL_IID_VOLUME};
293 const SLboolean interface_required[] = {SL_BOOLEAN_TRUE, SL_BOOLEAN_TRUE,
294 SL_BOOLEAN_TRUE};
295
296 // Create the audio player on the engine interface.
297 RETURN_ON_ERROR(
298 (*engine_)->CreateAudioPlayer(
299 engine_, player_object_.Receive(), &audio_source, &audio_sink,
300 arraysize(interface_ids), interface_ids, interface_required),
301 false);
302
303 // Use the Android configuration interface to set platform-specific
304 // parameters. Should be done before player is realized.
305 SLAndroidConfigurationItf player_config;
306 RETURN_ON_ERROR(
307 player_object_->GetInterface(player_object_.Get(),
308 SL_IID_ANDROIDCONFIGURATION, &player_config),
309 false);
310 // Set audio player configuration to SL_ANDROID_STREAM_VOICE which
311 // corresponds to android.media.AudioManager.STREAM_VOICE_CALL.
312 SLint32 stream_type = SL_ANDROID_STREAM_VOICE;
313 RETURN_ON_ERROR(
314 (*player_config)
315 ->SetConfiguration(player_config, SL_ANDROID_KEY_STREAM_TYPE,
316 &stream_type, sizeof(SLint32)),
317 false);
318
319 // Realize the audio player object after configuration has been set.
320 RETURN_ON_ERROR(
321 player_object_->Realize(player_object_.Get(), SL_BOOLEAN_FALSE), false);
322
323 // Get the SLPlayItf interface on the audio player.
324 RETURN_ON_ERROR(
325 player_object_->GetInterface(player_object_.Get(), SL_IID_PLAY, &player_),
326 false);
327
328 // Get the SLAndroidSimpleBufferQueueItf interface on the audio player.
329 RETURN_ON_ERROR(
330 player_object_->GetInterface(player_object_.Get(), SL_IID_BUFFERQUEUE,
331 &simple_buffer_queue_),
332 false);
333
334 // Register callback method for the Android Simple Buffer Queue interface.
335 // This method will be called when the native audio layer needs audio data.
336 RETURN_ON_ERROR((*simple_buffer_queue_)
337 ->RegisterCallback(simple_buffer_queue_,
338 SimpleBufferQueueCallback, this),
339 false);
340
341 // Get the SLVolumeItf interface on the audio player.
342 RETURN_ON_ERROR(player_object_->GetInterface(player_object_.Get(),
343 SL_IID_VOLUME, &volume_),
344 false);
345
346 // TODO(henrika): might not be required to set volume to max here since it
347 // seems to be default on most devices. Might be required for unit tests.
348 // RETURN_ON_ERROR((*volume_)->SetVolumeLevel(volume_, 0), false);
349
350 return true;
351 }
352
DestroyAudioPlayer()353 void OpenSLESPlayer::DestroyAudioPlayer() {
354 ALOGD("DestroyAudioPlayer");
355 RTC_DCHECK(thread_checker_.IsCurrent());
356 if (!player_object_.Get())
357 return;
358 (*simple_buffer_queue_)
359 ->RegisterCallback(simple_buffer_queue_, nullptr, nullptr);
360 player_object_.Reset();
361 player_ = nullptr;
362 simple_buffer_queue_ = nullptr;
363 volume_ = nullptr;
364 }
365
366 // static
SimpleBufferQueueCallback(SLAndroidSimpleBufferQueueItf caller,void * context)367 void OpenSLESPlayer::SimpleBufferQueueCallback(
368 SLAndroidSimpleBufferQueueItf caller,
369 void* context) {
370 OpenSLESPlayer* stream = reinterpret_cast<OpenSLESPlayer*>(context);
371 stream->FillBufferQueue();
372 }
373
FillBufferQueue()374 void OpenSLESPlayer::FillBufferQueue() {
375 RTC_DCHECK(thread_checker_opensles_.IsCurrent());
376 SLuint32 state = GetPlayState();
377 if (state != SL_PLAYSTATE_PLAYING) {
378 ALOGW("Buffer callback in non-playing state!");
379 return;
380 }
381 EnqueuePlayoutData(false);
382 }
383
EnqueuePlayoutData(bool silence)384 void OpenSLESPlayer::EnqueuePlayoutData(bool silence) {
385 // Check delta time between two successive callbacks and provide a warning
386 // if it becomes very large.
387 // TODO(henrika): using 150ms as upper limit but this value is rather random.
388 const uint32_t current_time = rtc::Time();
389 const uint32_t diff = current_time - last_play_time_;
390 if (diff > 150) {
391 ALOGW("Bad OpenSL ES playout timing, dT=%u [ms]", diff);
392 }
393 last_play_time_ = current_time;
394 SLint8* audio_ptr8 =
395 reinterpret_cast<SLint8*>(audio_buffers_[buffer_index_].get());
396 if (silence) {
397 RTC_DCHECK(thread_checker_.IsCurrent());
398 // Avoid acquiring real audio data from WebRTC and fill the buffer with
399 // zeros instead. Used to prime the buffer with silence and to avoid asking
400 // for audio data from two different threads.
401 memset(audio_ptr8, 0, audio_parameters_.GetBytesPerBuffer());
402 } else {
403 RTC_DCHECK(thread_checker_opensles_.IsCurrent());
404 // Read audio data from the WebRTC source using the FineAudioBuffer object
405 // to adjust for differences in buffer size between WebRTC (10ms) and native
406 // OpenSL ES. Use hardcoded delay estimate since OpenSL ES does not support
407 // delay estimation.
408 fine_audio_buffer_->GetPlayoutData(
409 rtc::ArrayView<int16_t>(audio_buffers_[buffer_index_].get(),
410 audio_parameters_.frames_per_buffer() *
411 audio_parameters_.channels()),
412 25);
413 }
414 // Enqueue the decoded audio buffer for playback.
415 SLresult err = (*simple_buffer_queue_)
416 ->Enqueue(simple_buffer_queue_, audio_ptr8,
417 audio_parameters_.GetBytesPerBuffer());
418 if (SL_RESULT_SUCCESS != err) {
419 ALOGE("Enqueue failed: %d", err);
420 }
421 buffer_index_ = (buffer_index_ + 1) % kNumOfOpenSLESBuffers;
422 }
423
GetPlayState() const424 SLuint32 OpenSLESPlayer::GetPlayState() const {
425 RTC_DCHECK(player_);
426 SLuint32 state;
427 SLresult err = (*player_)->GetPlayState(player_, &state);
428 if (SL_RESULT_SUCCESS != err) {
429 ALOGE("GetPlayState failed: %d", err);
430 }
431 return state;
432 }
433
434 } // namespace webrtc
435