1 /* 2 * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef SDK_OBJC_NATIVE_SRC_OBJC_AUDIO_DEVICE_H_ 12 #define SDK_OBJC_NATIVE_SRC_OBJC_AUDIO_DEVICE_H_ 13 14 #include <memory> 15 16 #import "components/audio/RTCAudioDevice.h" 17 18 #include "modules/audio_device/audio_device_buffer.h" 19 #include "modules/audio_device/include/audio_device.h" 20 #include "rtc_base/thread.h" 21 22 @class ObjCAudioDeviceDelegate; 23 24 namespace webrtc { 25 26 class FineAudioBuffer; 27 28 namespace objc_adm { 29 30 class ObjCAudioDeviceModule : public AudioDeviceModule { 31 public: 32 explicit ObjCAudioDeviceModule(id<RTC_OBJC_TYPE(RTCAudioDevice)> audio_device); 33 ~ObjCAudioDeviceModule() override; 34 35 // Retrieve the currently utilized audio layer 36 int32_t ActiveAudioLayer(AudioLayer* audioLayer) const override; 37 38 // Full-duplex transportation of PCM audio 39 int32_t RegisterAudioCallback(AudioTransport* audioCallback) override; 40 41 // Main initialization and termination 42 int32_t Init() override; 43 int32_t Terminate() override; 44 bool Initialized() const override; 45 46 // Device enumeration 47 int16_t PlayoutDevices() override; 48 int16_t RecordingDevices() override; 49 int32_t PlayoutDeviceName(uint16_t index, 50 char name[kAdmMaxDeviceNameSize], 51 char guid[kAdmMaxGuidSize]) override; 52 int32_t RecordingDeviceName(uint16_t index, 53 char name[kAdmMaxDeviceNameSize], 54 char guid[kAdmMaxGuidSize]) override; 55 56 // Device selection 57 int32_t SetPlayoutDevice(uint16_t index) override; 58 int32_t SetPlayoutDevice(WindowsDeviceType device) override; 59 int32_t SetRecordingDevice(uint16_t index) override; 60 int32_t SetRecordingDevice(WindowsDeviceType device) override; 61 62 // Audio transport initialization 63 int32_t PlayoutIsAvailable(bool* available) override; 64 int32_t InitPlayout() override; 65 bool PlayoutIsInitialized() const override; 66 int32_t RecordingIsAvailable(bool* available) override; 67 int32_t InitRecording() override; 68 bool RecordingIsInitialized() const override; 69 70 // Audio transport control 71 int32_t StartPlayout() override; 72 int32_t StopPlayout() override; 73 bool Playing() const override; 74 int32_t StartRecording() override; 75 int32_t StopRecording() override; 76 bool Recording() const override; 77 78 // Audio mixer initialization 79 int32_t InitSpeaker() override; 80 bool SpeakerIsInitialized() const override; 81 int32_t InitMicrophone() override; 82 bool MicrophoneIsInitialized() const override; 83 84 // Speaker volume controls 85 int32_t SpeakerVolumeIsAvailable(bool* available) override; 86 int32_t SetSpeakerVolume(uint32_t volume) override; 87 int32_t SpeakerVolume(uint32_t* volume) const override; 88 int32_t MaxSpeakerVolume(uint32_t* maxVolume) const override; 89 int32_t MinSpeakerVolume(uint32_t* minVolume) const override; 90 91 // Microphone volume controls 92 int32_t MicrophoneVolumeIsAvailable(bool* available) override; 93 int32_t SetMicrophoneVolume(uint32_t volume) override; 94 int32_t MicrophoneVolume(uint32_t* volume) const override; 95 int32_t MaxMicrophoneVolume(uint32_t* maxVolume) const override; 96 int32_t MinMicrophoneVolume(uint32_t* minVolume) const override; 97 98 // Speaker mute control 99 int32_t SpeakerMuteIsAvailable(bool* available) override; 100 int32_t SetSpeakerMute(bool enable) override; 101 int32_t SpeakerMute(bool* enabled) const override; 102 103 // Microphone mute control 104 int32_t MicrophoneMuteIsAvailable(bool* available) override; 105 int32_t SetMicrophoneMute(bool enable) override; 106 int32_t MicrophoneMute(bool* enabled) const override; 107 108 // Stereo support 109 int32_t StereoPlayoutIsAvailable(bool* available) const override; 110 int32_t SetStereoPlayout(bool enable) override; 111 int32_t StereoPlayout(bool* enabled) const override; 112 int32_t StereoRecordingIsAvailable(bool* available) const override; 113 int32_t SetStereoRecording(bool enable) override; 114 int32_t StereoRecording(bool* enabled) const override; 115 116 // Playout delay 117 int32_t PlayoutDelay(uint16_t* delayMS) const override; 118 119 // Only supported on Android. 120 bool BuiltInAECIsAvailable() const override; 121 bool BuiltInAGCIsAvailable() const override; 122 bool BuiltInNSIsAvailable() const override; 123 124 // Enables the built-in audio effects. Only supported on Android. 125 int32_t EnableBuiltInAEC(bool enable) override; 126 int32_t EnableBuiltInAGC(bool enable) override; 127 int32_t EnableBuiltInNS(bool enable) override; 128 129 // Play underrun count. Only supported on Android. 130 int32_t GetPlayoutUnderrunCount() const override; 131 132 #if defined(WEBRTC_IOS) 133 int GetPlayoutAudioParameters(AudioParameters* params) const override; 134 int GetRecordAudioParameters(AudioParameters* params) const override; 135 #endif // WEBRTC_IOS 136 137 public: 138 OSStatus OnDeliverRecordedData(AudioUnitRenderActionFlags* flags, 139 const AudioTimeStamp* time_stamp, 140 NSInteger bus_number, 141 UInt32 num_frames, 142 const AudioBufferList* io_data, 143 void* render_context, 144 RTC_OBJC_TYPE(RTCAudioDeviceRenderRecordedDataBlock) render_block); 145 146 OSStatus OnGetPlayoutData(AudioUnitRenderActionFlags* flags, 147 const AudioTimeStamp* time_stamp, 148 NSInteger bus_number, 149 UInt32 num_frames, 150 AudioBufferList* io_data); 151 152 // Notifies `ObjCAudioDeviceModule` that at least one of the audio input 153 // parameters or audio input latency of `RTCAudioDevice` has changed. It necessary to 154 // update `record_parameters_` with current audio parameter of `RTCAudioDevice` 155 // via `UpdateAudioParameters` and if parameters are actually change then 156 // ADB parameters are updated with `UpdateInputAudioDeviceBuffer`. Audio input latency 157 // stored in `cached_recording_delay_ms_` is also updated with current latency 158 // of `RTCAudioDevice`. 159 void HandleAudioInputParametersChange(); 160 161 // Same as `HandleAudioInputParametersChange` but should be called when audio output 162 // parameters of `RTCAudioDevice` has changed. 163 void HandleAudioOutputParametersChange(); 164 165 // Notifies `ObjCAudioDeviceModule` about audio input interruption happen due to 166 // any reason so `ObjCAudioDeviceModule` is can prepare to restart of audio IO. 167 void HandleAudioInputInterrupted(); 168 169 // Same as `ObjCAudioDeviceModule` but should be called when audio output 170 // is interrupted. 171 void HandleAudioOutputInterrupted(); 172 173 private: 174 // Update our audio parameters if they are different from current device audio parameters 175 // Returns true when our parameters are update, false - otherwise. 176 // `ObjCAudioDeviceModule` has audio device buffer (ADB) which has audio parameters 177 // of playout & recording. The ADB is configured to work with specific sample rate & channel 178 // count. `ObjCAudioDeviceModule` stores audio parameters which were used to configure ADB in the 179 // fields `playout_parameters_` and `recording_parameters_`. 180 // `RTCAudioDevice` protocol has its own audio parameters exposed as individual properties. 181 // `RTCAudioDevice` audio parameters might change when playout/recording is already in progress, 182 // for example, when device is switched. `RTCAudioDevice` audio parameters must be kept in sync 183 // with ADB audio parameters. This method is invoked when `RTCAudioDevice` reports that it's audio 184 // parameters (`device_params`) are changed and it detects if there any difference with our 185 // current audio parameters (`params`). Our parameters are updated in case of actual change and 186 // method returns true. In case of actual change there is follow-up call to either 187 // `UpdateOutputAudioDeviceBuffer` or `UpdateInputAudioDeviceBuffer` to apply updated 188 // `playout_parameters_` or `recording_parameters_` to ADB. 189 190 bool UpdateAudioParameters(AudioParameters& params, const AudioParameters& device_params); 191 192 // Update our cached audio latency with device latency. Device latency is reported by 193 // `RTCAudioDevice` object. Whenever latency is changed, `RTCAudioDevice` is obliged to notify ADM 194 // about the change via `HandleAudioInputParametersChange` or `HandleAudioOutputParametersChange`. 195 // Current device IO latency is cached in the atomic field and used from audio IO thread 196 // to be reported to audio device buffer. It is highly recommended by Apple not to call any 197 // ObjC methods from audio IO thread, that is why implementation relies on caching latency 198 // into a field and being notified when latency is changed, which is the case when device 199 // is switched. 200 void UpdateAudioDelay(std::atomic<int>& delay_ms, const NSTimeInterval device_latency); 201 202 // Uses current `playout_parameters_` to inform the audio device buffer (ADB) 203 // about our internal audio parameters. 204 void UpdateOutputAudioDeviceBuffer(); 205 206 // Uses current `record_parameters_` to inform the audio device buffer (ADB) 207 // about our internal audio parameters. 208 void UpdateInputAudioDeviceBuffer(); 209 210 private: 211 id<RTC_OBJC_TYPE(RTCAudioDevice)> audio_device_; 212 213 const std::unique_ptr<TaskQueueFactory> task_queue_factory_; 214 215 // AudioDeviceBuffer is a buffer to consume audio recorded by `RTCAudioDevice` 216 // and provide audio to be played via `RTCAudioDevice`. 217 // Audio PCMs could have different sample rate and channels count, but expected 218 // to be in 16-bit integer interleaved linear PCM format. 219 // The current parameters ADB configured to work with is stored in field 220 // `playout_parameters_` for playout and `record_parameters_` for recording. 221 // These parameters and ADB must kept in sync with `RTCAudioDevice` audio parameters. 222 std::unique_ptr<AudioDeviceBuffer> audio_device_buffer_; 223 224 // Set to 1 when recording is active and 0 otherwise. 225 std::atomic<bool> recording_ = false; 226 227 // Set to 1 when playout is active and 0 otherwise. 228 std::atomic<bool> playing_ = false; 229 230 // Stores cached value of `RTCAudioDevice outputLatency` to be used from 231 // audio IO thread. Latency is updated on audio output parameters change. 232 std::atomic<int> cached_playout_delay_ms_ = 0; 233 234 // Same as `cached_playout_delay_ms_` but for audio input 235 std::atomic<int> cached_recording_delay_ms_ = 0; 236 237 // Thread that is initialized audio device module. 238 rtc::Thread* thread_; 239 240 // Ensures that methods are called from the same thread as this object is 241 // initialized on. 242 SequenceChecker thread_checker_; 243 244 // I/O audio thread checker. 245 SequenceChecker io_playout_thread_checker_; 246 SequenceChecker io_record_thread_checker_; 247 248 bool is_initialized_ RTC_GUARDED_BY(thread_checker_) = false; 249 bool is_playout_initialized_ RTC_GUARDED_BY(thread_checker_) = false; 250 bool is_recording_initialized_ RTC_GUARDED_BY(thread_checker_) = false; 251 252 // Contains audio parameters (sample rate, #channels, buffer size etc.) for 253 // the playout and recording sides. 254 AudioParameters playout_parameters_; 255 AudioParameters record_parameters_; 256 257 // `FineAudioBuffer` takes an `AudioDeviceBuffer` which delivers audio data 258 // in chunks of 10ms. `RTCAudioDevice` might deliver recorded data in 259 // chunks which are not 10ms long. `FineAudioBuffer` implements adaptation 260 // from undetermined chunk size to 10ms chunks. 261 std::unique_ptr<FineAudioBuffer> record_fine_audio_buffer_; 262 263 // Same as `record_fine_audio_buffer_` but for audio output. 264 std::unique_ptr<FineAudioBuffer> playout_fine_audio_buffer_; 265 266 // Temporary storage for recorded data. 267 rtc::BufferT<int16_t> record_audio_buffer_; 268 269 // Delegate object provided to RTCAudioDevice during initialization 270 ObjCAudioDeviceDelegate* audio_device_delegate_; 271 }; 272 273 } // namespace objc_adm 274 275 } // namespace webrtc 276 277 #endif // SDK_OBJC_NATIVE_SRC_OBJC_AUDIO_DEVICE_H_ 278