1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef SDK_OBJC_NATIVE_SRC_AUDIO_AUDIO_DEVICE_IOS_H_ 12 #define SDK_OBJC_NATIVE_SRC_AUDIO_AUDIO_DEVICE_IOS_H_ 13 14 #include <atomic> 15 #include <memory> 16 17 #include "api/scoped_refptr.h" 18 #include "api/sequence_checker.h" 19 #include "api/task_queue/pending_task_safety_flag.h" 20 #include "audio_session_observer.h" 21 #include "modules/audio_device/audio_device_generic.h" 22 #include "rtc_base/buffer.h" 23 #include "rtc_base/thread.h" 24 #include "rtc_base/thread_annotations.h" 25 #include "sdk/objc/base/RTCMacros.h" 26 #include "voice_processing_audio_unit.h" 27 28 RTC_FWD_DECL_OBJC_CLASS(RTCNativeAudioSessionDelegateAdapter); 29 30 namespace webrtc { 31 32 class FineAudioBuffer; 33 34 namespace ios_adm { 35 36 // Implements full duplex 16-bit mono PCM audio support for iOS using a 37 // Voice-Processing (VP) I/O audio unit in Core Audio. The VP I/O audio unit 38 // supports audio echo cancellation. It also adds automatic gain control, 39 // adjustment of voice-processing quality and muting. 40 // 41 // An instance must be created and destroyed on one and the same thread. 42 // All supported public methods must also be called on the same thread. 43 // A thread checker will RTC_DCHECK if any supported method is called on an 44 // invalid thread. 45 // 46 // Recorded audio will be delivered on a real-time internal I/O thread in the 47 // audio unit. The audio unit will also ask for audio data to play out on this 48 // same thread. 49 class AudioDeviceIOS : public AudioDeviceGeneric, 50 public AudioSessionObserver, 51 public VoiceProcessingAudioUnitObserver { 52 public: 53 explicit AudioDeviceIOS(bool bypass_voice_processing); 54 ~AudioDeviceIOS() override; 55 56 void AttachAudioBuffer(AudioDeviceBuffer* audioBuffer) override; 57 58 InitStatus Init() override; 59 int32_t Terminate() override; 60 bool Initialized() const override; 61 62 int32_t InitPlayout() override; 63 bool PlayoutIsInitialized() const override; 64 65 int32_t InitRecording() override; 66 bool RecordingIsInitialized() const override; 67 68 int32_t StartPlayout() override; 69 int32_t StopPlayout() override; 70 bool Playing() const override; 71 72 int32_t StartRecording() override; 73 int32_t StopRecording() override; 74 bool Recording() const override; 75 76 // These methods returns hard-coded delay values and not dynamic delay 77 // estimates. The reason is that iOS supports a built-in AEC and the WebRTC 78 // AEC will always be disabled in the Libjingle layer to avoid running two 79 // AEC implementations at the same time. And, it saves resources to avoid 80 // updating these delay values continuously. 81 // TODO(henrika): it would be possible to mark these two methods as not 82 // implemented since they are only called for A/V-sync purposes today and 83 // A/V-sync is not supported on iOS. However, we avoid adding error messages 84 // the log by using these dummy implementations instead. 85 int32_t PlayoutDelay(uint16_t& delayMS) const override; 86 87 // No implementation for playout underrun on iOS. We override it to avoid a 88 // periodic log that it isn't available from the base class. GetPlayoutUnderrunCount()89 int32_t GetPlayoutUnderrunCount() const override { return -1; } 90 91 // Native audio parameters stored during construction. 92 // These methods are unique for the iOS implementation. 93 int GetPlayoutAudioParameters(AudioParameters* params) const override; 94 int GetRecordAudioParameters(AudioParameters* params) const override; 95 96 // These methods are currently not fully implemented on iOS: 97 98 // See audio_device_not_implemented.cc for trivial implementations. 99 int32_t ActiveAudioLayer( 100 AudioDeviceModule::AudioLayer& audioLayer) const override; 101 int32_t PlayoutIsAvailable(bool& available) override; 102 int32_t RecordingIsAvailable(bool& available) override; 103 int16_t PlayoutDevices() override; 104 int16_t RecordingDevices() override; 105 int32_t PlayoutDeviceName(uint16_t index, 106 char name[kAdmMaxDeviceNameSize], 107 char guid[kAdmMaxGuidSize]) override; 108 int32_t RecordingDeviceName(uint16_t index, 109 char name[kAdmMaxDeviceNameSize], 110 char guid[kAdmMaxGuidSize]) override; 111 int32_t SetPlayoutDevice(uint16_t index) override; 112 int32_t SetPlayoutDevice( 113 AudioDeviceModule::WindowsDeviceType device) override; 114 int32_t SetRecordingDevice(uint16_t index) override; 115 int32_t SetRecordingDevice( 116 AudioDeviceModule::WindowsDeviceType device) override; 117 int32_t InitSpeaker() override; 118 bool SpeakerIsInitialized() const override; 119 int32_t InitMicrophone() override; 120 bool MicrophoneIsInitialized() const override; 121 int32_t SpeakerVolumeIsAvailable(bool& available) override; 122 int32_t SetSpeakerVolume(uint32_t volume) override; 123 int32_t SpeakerVolume(uint32_t& volume) const override; 124 int32_t MaxSpeakerVolume(uint32_t& maxVolume) const override; 125 int32_t MinSpeakerVolume(uint32_t& minVolume) const override; 126 int32_t MicrophoneVolumeIsAvailable(bool& available) override; 127 int32_t SetMicrophoneVolume(uint32_t volume) override; 128 int32_t MicrophoneVolume(uint32_t& volume) const override; 129 int32_t MaxMicrophoneVolume(uint32_t& maxVolume) const override; 130 int32_t MinMicrophoneVolume(uint32_t& minVolume) const override; 131 int32_t MicrophoneMuteIsAvailable(bool& available) override; 132 int32_t SetMicrophoneMute(bool enable) override; 133 int32_t MicrophoneMute(bool& enabled) const override; 134 int32_t SpeakerMuteIsAvailable(bool& available) override; 135 int32_t SetSpeakerMute(bool enable) override; 136 int32_t SpeakerMute(bool& enabled) const override; 137 int32_t StereoPlayoutIsAvailable(bool& available) override; 138 int32_t SetStereoPlayout(bool enable) override; 139 int32_t StereoPlayout(bool& enabled) const override; 140 int32_t StereoRecordingIsAvailable(bool& available) override; 141 int32_t SetStereoRecording(bool enable) override; 142 int32_t StereoRecording(bool& enabled) const override; 143 144 // AudioSessionObserver methods. May be called from any thread. 145 void OnInterruptionBegin() override; 146 void OnInterruptionEnd() override; 147 void OnValidRouteChange() override; 148 void OnCanPlayOrRecordChange(bool can_play_or_record) override; 149 void OnChangedOutputVolume() override; 150 151 // VoiceProcessingAudioUnitObserver methods. 152 OSStatus OnDeliverRecordedData(AudioUnitRenderActionFlags* flags, 153 const AudioTimeStamp* time_stamp, 154 UInt32 bus_number, 155 UInt32 num_frames, 156 AudioBufferList* io_data) override; 157 OSStatus OnGetPlayoutData(AudioUnitRenderActionFlags* flags, 158 const AudioTimeStamp* time_stamp, 159 UInt32 bus_number, 160 UInt32 num_frames, 161 AudioBufferList* io_data) override; 162 163 bool IsInterrupted(); 164 165 private: 166 // Called by the relevant AudioSessionObserver methods on `thread_`. 167 void HandleInterruptionBegin(); 168 void HandleInterruptionEnd(); 169 void HandleValidRouteChange(); 170 void HandleCanPlayOrRecordChange(bool can_play_or_record); 171 void HandleSampleRateChange(); 172 void HandlePlayoutGlitchDetected(); 173 void HandleOutputVolumeChange(); 174 175 // Uses current `playout_parameters_` and `record_parameters_` to inform the 176 // audio device buffer (ADB) about our internal audio parameters. 177 void UpdateAudioDeviceBuffer(); 178 179 // Since the preferred audio parameters are only hints to the OS, the actual 180 // values may be different once the AVAudioSession has been activated. 181 // This method asks for the current hardware parameters and takes actions 182 // if they should differ from what we have asked for initially. It also 183 // defines `playout_parameters_` and `record_parameters_`. 184 void SetupAudioBuffersForActiveAudioSession(); 185 186 // Creates the audio unit. 187 bool CreateAudioUnit(); 188 189 // Updates the audio unit state based on current state. 190 void UpdateAudioUnit(bool can_play_or_record); 191 192 // Configures the audio session for WebRTC. 193 bool ConfigureAudioSession(); 194 195 // Like above, but requires caller to already hold session lock. 196 bool ConfigureAudioSessionLocked(); 197 198 // Unconfigures the audio session. 199 void UnconfigureAudioSession(); 200 201 // Activates our audio session, creates and initializes the voice-processing 202 // audio unit and verifies that we got the preferred native audio parameters. 203 bool InitPlayOrRecord(); 204 205 // Closes and deletes the voice-processing I/O unit. 206 void ShutdownPlayOrRecord(); 207 208 // Resets thread-checkers before a call is restarted. 209 void PrepareForNewStart(); 210 211 // Determines whether voice processing should be enabled or disabled. 212 const bool bypass_voice_processing_; 213 214 // Native I/O audio thread checker. 215 SequenceChecker io_thread_checker_; 216 217 // Thread that this object is created on. 218 rtc::Thread* thread_; 219 220 // Raw pointer handle provided to us in AttachAudioBuffer(). Owned by the 221 // AudioDeviceModuleImpl class and called by AudioDeviceModule::Create(). 222 // The AudioDeviceBuffer is a member of the AudioDeviceModuleImpl instance 223 // and therefore outlives this object. 224 AudioDeviceBuffer* audio_device_buffer_; 225 226 // Contains audio parameters (sample rate, #channels, buffer size etc.) for 227 // the playout and recording sides. These structure is set in two steps: 228 // first, native sample rate and #channels are defined in Init(). Next, the 229 // audio session is activated and we verify that the preferred parameters 230 // were granted by the OS. At this stage it is also possible to add a third 231 // component to the parameters; the native I/O buffer duration. 232 // A RTC_CHECK will be hit if we for some reason fail to open an audio session 233 // using the specified parameters. 234 AudioParameters playout_parameters_; 235 AudioParameters record_parameters_; 236 237 // The AudioUnit used to play and record audio. 238 std::unique_ptr<VoiceProcessingAudioUnit> audio_unit_; 239 240 // FineAudioBuffer takes an AudioDeviceBuffer which delivers audio data 241 // in chunks of 10ms. It then allows for this data to be pulled in 242 // a finer or coarser granularity. I.e. interacting with this class instead 243 // of directly with the AudioDeviceBuffer one can ask for any number of 244 // audio data samples. Is also supports a similar scheme for the recording 245 // side. 246 // Example: native buffer size can be 128 audio frames at 16kHz sample rate. 247 // WebRTC will provide 480 audio frames per 10ms but iOS asks for 128 248 // in each callback (one every 8ms). This class can then ask for 128 and the 249 // FineAudioBuffer will ask WebRTC for new data only when needed and also 250 // cache non-utilized audio between callbacks. On the recording side, iOS 251 // can provide audio data frames of size 128 and these are accumulated until 252 // enough data to supply one 10ms call exists. This 10ms chunk is then sent 253 // to WebRTC and the remaining part is stored. 254 std::unique_ptr<FineAudioBuffer> fine_audio_buffer_; 255 256 // Temporary storage for recorded data. AudioUnitRender() renders into this 257 // array as soon as a frame of the desired buffer size has been recorded. 258 // On real iOS devices, the size will be fixed and set once. For iOS 259 // simulators, the size can vary from callback to callback and the size 260 // will be changed dynamically to account for this behavior. 261 rtc::BufferT<int16_t> record_audio_buffer_; 262 263 // Set to 1 when recording is active and 0 otherwise. 264 std::atomic<int> recording_; 265 266 // Set to 1 when playout is active and 0 otherwise. 267 std::atomic<int> playing_; 268 269 // Set to true after successful call to Init(), false otherwise. 270 bool initialized_ RTC_GUARDED_BY(thread_); 271 272 // Set to true after successful call to InitRecording() or InitPlayout(), 273 // false otherwise. 274 bool audio_is_initialized_; 275 276 // Set to true if audio session is interrupted, false otherwise. 277 bool is_interrupted_; 278 279 // Audio interruption observer instance. 280 RTCNativeAudioSessionDelegateAdapter* audio_session_observer_ 281 RTC_GUARDED_BY(thread_); 282 283 // Set to true if we've activated the audio session. 284 bool has_configured_session_ RTC_GUARDED_BY(thread_); 285 286 // Counts number of detected audio glitches on the playout side. 287 int64_t num_detected_playout_glitches_ RTC_GUARDED_BY(thread_); 288 int64_t last_playout_time_ RTC_GUARDED_BY(io_thread_checker_); 289 290 // Counts number of playout callbacks per call. 291 // The value is updated on the native I/O thread and later read on the 292 // creating `thread_` but at this stage no audio is active. 293 // Hence, it is a "thread safe" design and no lock is needed. 294 int64_t num_playout_callbacks_; 295 296 // Contains the time for when the last output volume change was detected. 297 int64_t last_output_volume_change_time_ RTC_GUARDED_BY(thread_); 298 299 // Avoids running pending task after `this` is Terminated. 300 rtc::scoped_refptr<PendingTaskSafetyFlag> safety_ = 301 PendingTaskSafetyFlag::Create(); 302 }; 303 } // namespace ios_adm 304 } // namespace webrtc 305 306 #endif // SDK_OBJC_NATIVE_SRC_AUDIO_AUDIO_DEVICE_IOS_H_ 307