1 /* 2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef MODULES_AUDIO_PROCESSING_AEC3_ECHO_CANCELLER3_H_ 12 #define MODULES_AUDIO_PROCESSING_AEC3_ECHO_CANCELLER3_H_ 13 14 #include <stddef.h> 15 16 #include <atomic> 17 #include <memory> 18 #include <vector> 19 20 #include "absl/types/optional.h" 21 #include "api/array_view.h" 22 #include "api/audio/echo_canceller3_config.h" 23 #include "api/audio/echo_control.h" 24 #include "modules/audio_processing/aec3/api_call_jitter_metrics.h" 25 #include "modules/audio_processing/aec3/block_delay_buffer.h" 26 #include "modules/audio_processing/aec3/block_framer.h" 27 #include "modules/audio_processing/aec3/block_processor.h" 28 #include "modules/audio_processing/aec3/config_selector.h" 29 #include "modules/audio_processing/aec3/frame_blocker.h" 30 #include "modules/audio_processing/aec3/multi_channel_content_detector.h" 31 #include "modules/audio_processing/audio_buffer.h" 32 #include "modules/audio_processing/logging/apm_data_dumper.h" 33 #include "rtc_base/checks.h" 34 #include "rtc_base/race_checker.h" 35 #include "rtc_base/swap_queue.h" 36 #include "rtc_base/thread_annotations.h" 37 38 namespace webrtc { 39 40 // Method for adjusting config parameter dependencies. 41 // Only to be used externally to AEC3 for testing purposes. 42 // TODO(webrtc:5298): Move this to a separate file. 43 EchoCanceller3Config AdjustConfig(const EchoCanceller3Config& config); 44 45 // Functor for verifying the invariance of the frames being put into the render 46 // queue. 47 class Aec3RenderQueueItemVerifier { 48 public: Aec3RenderQueueItemVerifier(size_t num_bands,size_t num_channels,size_t frame_length)49 Aec3RenderQueueItemVerifier(size_t num_bands, 50 size_t num_channels, 51 size_t frame_length) 52 : num_bands_(num_bands), 53 num_channels_(num_channels), 54 frame_length_(frame_length) {} 55 operator()56 bool operator()(const std::vector<std::vector<std::vector<float>>>& v) const { 57 if (v.size() != num_bands_) { 58 return false; 59 } 60 for (const auto& band : v) { 61 if (band.size() != num_channels_) { 62 return false; 63 } 64 for (const auto& channel : band) { 65 if (channel.size() != frame_length_) { 66 return false; 67 } 68 } 69 } 70 return true; 71 } 72 73 private: 74 const size_t num_bands_; 75 const size_t num_channels_; 76 const size_t frame_length_; 77 }; 78 79 // Main class for the echo canceller3. 80 // It does 4 things: 81 // -Receives 10 ms frames of band-split audio. 82 // -Provides the lower level echo canceller functionality with 83 // blocks of 64 samples of audio data. 84 // -Partially handles the jitter in the render and capture API 85 // call sequence. 86 // 87 // The class is supposed to be used in a non-concurrent manner apart from the 88 // AnalyzeRender call which can be called concurrently with the other methods. 89 class EchoCanceller3 : public EchoControl { 90 public: 91 EchoCanceller3( 92 const EchoCanceller3Config& config, 93 const absl::optional<EchoCanceller3Config>& multichannel_config, 94 int sample_rate_hz, 95 size_t num_render_channels, 96 size_t num_capture_channels); 97 98 ~EchoCanceller3() override; 99 100 EchoCanceller3(const EchoCanceller3&) = delete; 101 EchoCanceller3& operator=(const EchoCanceller3&) = delete; 102 103 // Analyzes and stores an internal copy of the split-band domain render 104 // signal. AnalyzeRender(AudioBuffer * render)105 void AnalyzeRender(AudioBuffer* render) override { AnalyzeRender(*render); } 106 // Analyzes the full-band domain capture signal to detect signal saturation. AnalyzeCapture(AudioBuffer * capture)107 void AnalyzeCapture(AudioBuffer* capture) override { 108 AnalyzeCapture(*capture); 109 } 110 // Processes the split-band domain capture signal in order to remove any echo 111 // present in the signal. 112 void ProcessCapture(AudioBuffer* capture, bool level_change) override; 113 // As above, but also returns the linear filter output. 114 void ProcessCapture(AudioBuffer* capture, 115 AudioBuffer* linear_output, 116 bool level_change) override; 117 // Collect current metrics from the echo canceller. 118 Metrics GetMetrics() const override; 119 // Provides an optional external estimate of the audio buffer delay. 120 void SetAudioBufferDelay(int delay_ms) override; 121 122 // Specifies whether the capture output will be used. The purpose of this is 123 // to allow the echo controller to deactivate some of the processing when the 124 // resulting output is anyway not used, for instance when the endpoint is 125 // muted. 126 void SetCaptureOutputUsage(bool capture_output_used) override; 127 128 bool ActiveProcessing() const override; 129 130 // Signals whether an external detector has detected echo leakage from the 131 // echo canceller. 132 // Note that in the case echo leakage has been flagged, it should be unflagged 133 // once it is no longer occurring. UpdateEchoLeakageStatus(bool leakage_detected)134 void UpdateEchoLeakageStatus(bool leakage_detected) { 135 RTC_DCHECK_RUNS_SERIALIZED(&capture_race_checker_); 136 block_processor_->UpdateEchoLeakageStatus(leakage_detected); 137 } 138 139 // Produces a default configuration for multichannel. 140 static EchoCanceller3Config CreateDefaultMultichannelConfig(); 141 142 private: 143 friend class EchoCanceller3Tester; 144 FRIEND_TEST_ALL_PREFIXES(EchoCanceller3, DetectionOfProperStereo); 145 FRIEND_TEST_ALL_PREFIXES(EchoCanceller3, 146 DetectionOfProperStereoUsingThreshold); 147 FRIEND_TEST_ALL_PREFIXES(EchoCanceller3, 148 DetectionOfProperStereoUsingHysteresis); 149 FRIEND_TEST_ALL_PREFIXES(EchoCanceller3, 150 StereoContentDetectionForMonoSignals); 151 152 class RenderWriter; 153 154 // (Re-)Initializes the selected subset of the EchoCanceller3 fields, at 155 // creation as well as during reconfiguration. 156 void Initialize(); 157 158 // Only for testing. Replaces the internal block processor. 159 void SetBlockProcessorForTesting( 160 std::unique_ptr<BlockProcessor> block_processor); 161 162 // Only for testing. Returns whether stereo processing is active. StereoRenderProcessingActiveForTesting()163 bool StereoRenderProcessingActiveForTesting() const { 164 return multichannel_content_detector_.IsProperMultiChannelContentDetected(); 165 } 166 167 // Only for testing. GetActiveConfigForTesting()168 const EchoCanceller3Config& GetActiveConfigForTesting() const { 169 return config_selector_.active_config(); 170 } 171 172 // Empties the render SwapQueue. 173 void EmptyRenderQueue(); 174 175 // Analyzes and stores an internal copy of the split-band domain render 176 // signal. 177 void AnalyzeRender(const AudioBuffer& render); 178 // Analyzes the full-band domain capture signal to detect signal saturation. 179 void AnalyzeCapture(const AudioBuffer& capture); 180 181 rtc::RaceChecker capture_race_checker_; 182 rtc::RaceChecker render_race_checker_; 183 184 // State that is accessed by the AnalyzeRender call. 185 std::unique_ptr<RenderWriter> render_writer_ 186 RTC_GUARDED_BY(render_race_checker_); 187 188 // State that may be accessed by the capture thread. 189 static std::atomic<int> instance_count_; 190 std::unique_ptr<ApmDataDumper> data_dumper_; 191 const EchoCanceller3Config config_; 192 const int sample_rate_hz_; 193 const int num_bands_; 194 const size_t num_render_input_channels_; 195 size_t num_render_channels_to_aec_; 196 const size_t num_capture_channels_; 197 ConfigSelector config_selector_; 198 MultiChannelContentDetector multichannel_content_detector_; 199 std::unique_ptr<BlockFramer> linear_output_framer_ 200 RTC_GUARDED_BY(capture_race_checker_); 201 BlockFramer output_framer_ RTC_GUARDED_BY(capture_race_checker_); 202 FrameBlocker capture_blocker_ RTC_GUARDED_BY(capture_race_checker_); 203 std::unique_ptr<FrameBlocker> render_blocker_ 204 RTC_GUARDED_BY(capture_race_checker_); 205 SwapQueue<std::vector<std::vector<std::vector<float>>>, 206 Aec3RenderQueueItemVerifier> 207 render_transfer_queue_; 208 std::unique_ptr<BlockProcessor> block_processor_ 209 RTC_GUARDED_BY(capture_race_checker_); 210 std::vector<std::vector<std::vector<float>>> render_queue_output_frame_ 211 RTC_GUARDED_BY(capture_race_checker_); 212 bool saturated_microphone_signal_ RTC_GUARDED_BY(capture_race_checker_) = 213 false; 214 Block render_block_ RTC_GUARDED_BY(capture_race_checker_); 215 std::unique_ptr<Block> linear_output_block_ 216 RTC_GUARDED_BY(capture_race_checker_); 217 Block capture_block_ RTC_GUARDED_BY(capture_race_checker_); 218 std::vector<std::vector<rtc::ArrayView<float>>> render_sub_frame_view_ 219 RTC_GUARDED_BY(capture_race_checker_); 220 std::vector<std::vector<rtc::ArrayView<float>>> linear_output_sub_frame_view_ 221 RTC_GUARDED_BY(capture_race_checker_); 222 std::vector<std::vector<rtc::ArrayView<float>>> capture_sub_frame_view_ 223 RTC_GUARDED_BY(capture_race_checker_); 224 std::unique_ptr<BlockDelayBuffer> block_delay_buffer_ 225 RTC_GUARDED_BY(capture_race_checker_); 226 ApiCallJitterMetrics api_call_metrics_ RTC_GUARDED_BY(capture_race_checker_); 227 }; 228 } // namespace webrtc 229 230 #endif // MODULES_AUDIO_PROCESSING_AEC3_ECHO_CANCELLER3_H_ 231