1 /* 2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef MODULES_AUDIO_DEVICE_ANDROID_OPENSLES_RECORDER_H_ 12 #define MODULES_AUDIO_DEVICE_ANDROID_OPENSLES_RECORDER_H_ 13 14 #include <SLES/OpenSLES.h> 15 #include <SLES/OpenSLES_Android.h> 16 #include <SLES/OpenSLES_AndroidConfiguration.h> 17 18 #include <memory> 19 20 #include "api/sequence_checker.h" 21 #include "modules/audio_device/android/audio_common.h" 22 #include "modules/audio_device/android/audio_manager.h" 23 #include "modules/audio_device/android/opensles_common.h" 24 #include "modules/audio_device/audio_device_generic.h" 25 #include "modules/audio_device/include/audio_device_defines.h" 26 #include "modules/utility/include/helpers_android.h" 27 28 namespace webrtc { 29 30 class FineAudioBuffer; 31 32 // Implements 16-bit mono PCM audio input support for Android using the 33 // C based OpenSL ES API. No calls from C/C++ to Java using JNI is done. 34 // 35 // An instance must be created and destroyed on one and the same thread. 36 // All public methods must also be called on the same thread. A thread checker 37 // will RTC_DCHECK if any method is called on an invalid thread. Recorded audio 38 // buffers are provided on a dedicated internal thread managed by the OpenSL 39 // ES layer. 40 // 41 // The existing design forces the user to call InitRecording() after 42 // StopRecording() to be able to call StartRecording() again. This is inline 43 // with how the Java-based implementation works. 44 // 45 // As of API level 21, lower latency audio input is supported on select devices. 46 // To take advantage of this feature, first confirm that lower latency output is 47 // available. The capability for lower latency output is a prerequisite for the 48 // lower latency input feature. Then, create an AudioRecorder with the same 49 // sample rate and buffer size as would be used for output. OpenSL ES interfaces 50 // for input effects preclude the lower latency path. 51 // See https://developer.android.com/ndk/guides/audio/opensl-prog-notes.html 52 // for more details. 53 class OpenSLESRecorder { 54 public: 55 // Beginning with API level 17 (Android 4.2), a buffer count of 2 or more is 56 // required for lower latency. Beginning with API level 18 (Android 4.3), a 57 // buffer count of 1 is sufficient for lower latency. In addition, the buffer 58 // size and sample rate must be compatible with the device's native input 59 // configuration provided via the audio manager at construction. 60 // TODO(henrika): perhaps set this value dynamically based on OS version. 61 static const int kNumOfOpenSLESBuffers = 2; 62 63 explicit OpenSLESRecorder(AudioManager* audio_manager); 64 ~OpenSLESRecorder(); 65 66 int Init(); 67 int Terminate(); 68 69 int InitRecording(); RecordingIsInitialized()70 bool RecordingIsInitialized() const { return initialized_; } 71 72 int StartRecording(); 73 int StopRecording(); Recording()74 bool Recording() const { return recording_; } 75 76 void AttachAudioBuffer(AudioDeviceBuffer* audio_buffer); 77 78 // TODO(henrika): add support using OpenSL ES APIs when available. 79 int EnableBuiltInAEC(bool enable); 80 int EnableBuiltInAGC(bool enable); 81 int EnableBuiltInNS(bool enable); 82 83 private: 84 // Obtaines the SL Engine Interface from the existing global Engine object. 85 // The interface exposes creation methods of all the OpenSL ES object types. 86 // This method defines the `engine_` member variable. 87 bool ObtainEngineInterface(); 88 89 // Creates/destroys the audio recorder and the simple-buffer queue object. 90 bool CreateAudioRecorder(); 91 void DestroyAudioRecorder(); 92 93 // Allocate memory for audio buffers which will be used to capture audio 94 // via the SLAndroidSimpleBufferQueueItf interface. 95 void AllocateDataBuffers(); 96 97 // These callback methods are called when data has been written to the input 98 // buffer queue. They are both called from an internal "OpenSL ES thread" 99 // which is not attached to the Dalvik VM. 100 static void SimpleBufferQueueCallback(SLAndroidSimpleBufferQueueItf caller, 101 void* context); 102 void ReadBufferQueue(); 103 104 // Wraps calls to SLAndroidSimpleBufferQueueState::Enqueue() and it can be 105 // called both on the main thread (but before recording has started) and from 106 // the internal audio thread while input streaming is active. It uses 107 // `simple_buffer_queue_` but no lock is needed since the initial calls from 108 // the main thread and the native callback thread are mutually exclusive. 109 bool EnqueueAudioBuffer(); 110 111 // Returns the current recorder state. 112 SLuint32 GetRecordState() const; 113 114 // Returns the current buffer queue state. 115 SLAndroidSimpleBufferQueueState GetBufferQueueState() const; 116 117 // Number of buffers currently in the queue. 118 SLuint32 GetBufferCount(); 119 120 // Prints a log message of the current queue state. Can be used for debugging 121 // purposes. 122 void LogBufferState() const; 123 124 // Ensures that methods are called from the same thread as this object is 125 // created on. 126 SequenceChecker thread_checker_; 127 128 // Stores thread ID in first call to SimpleBufferQueueCallback() from internal 129 // non-application thread which is not attached to the Dalvik JVM. 130 // Detached during construction of this object. 131 SequenceChecker thread_checker_opensles_; 132 133 // Raw pointer to the audio manager injected at construction. Used to cache 134 // audio parameters and to access the global SL engine object needed by the 135 // ObtainEngineInterface() method. The audio manager outlives any instance of 136 // this class. 137 AudioManager* const audio_manager_; 138 139 // Contains audio parameters provided to this class at construction by the 140 // AudioManager. 141 const AudioParameters audio_parameters_; 142 143 // Raw pointer handle provided to us in AttachAudioBuffer(). Owned by the 144 // AudioDeviceModuleImpl class and called by AudioDeviceModule::Create(). 145 AudioDeviceBuffer* audio_device_buffer_; 146 147 // PCM-type format definition. 148 // TODO(henrika): add support for SLAndroidDataFormat_PCM_EX (android-21) if 149 // 32-bit float representation is needed. 150 SLDataFormat_PCM pcm_format_; 151 152 bool initialized_; 153 bool recording_; 154 155 // This interface exposes creation methods for all the OpenSL ES object types. 156 // It is the OpenSL ES API entry point. 157 SLEngineItf engine_; 158 159 // The audio recorder media object records audio to the destination specified 160 // by the data sink capturing it from the input specified by the data source. 161 webrtc::ScopedSLObjectItf recorder_object_; 162 163 // This interface is supported on the audio recorder object and it controls 164 // the state of the audio recorder. 165 SLRecordItf recorder_; 166 167 // The Android Simple Buffer Queue interface is supported on the audio 168 // recorder. For recording, an app should enqueue empty buffers. When a 169 // registered callback sends notification that the system has finished writing 170 // data to the buffer, the app can read the buffer. 171 SLAndroidSimpleBufferQueueItf simple_buffer_queue_; 172 173 // Consumes audio of native buffer size and feeds the WebRTC layer with 10ms 174 // chunks of audio. 175 std::unique_ptr<FineAudioBuffer> fine_audio_buffer_; 176 177 // Queue of audio buffers to be used by the recorder object for capturing 178 // audio. They will be used in a Round-robin way and the size of each buffer 179 // is given by AudioParameters::frames_per_buffer(), i.e., it corresponds to 180 // the native OpenSL ES buffer size. 181 std::unique_ptr<std::unique_ptr<SLint16[]>[]> audio_buffers_; 182 183 // Keeps track of active audio buffer 'n' in the audio_buffers_[n] queue. 184 // Example (kNumOfOpenSLESBuffers = 2): counts 0, 1, 0, 1, ... 185 int buffer_index_; 186 187 // Last time the OpenSL ES layer delivered recorded audio data. 188 uint32_t last_rec_time_; 189 }; 190 191 } // namespace webrtc 192 193 #endif // MODULES_AUDIO_DEVICE_ANDROID_OPENSLES_RECORDER_H_ 194