1 /*
2 * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10 #include "modules/audio_processing/aec3/block_processor.h"
11
12 #include <stddef.h>
13
14 #include <atomic>
15 #include <memory>
16 #include <utility>
17 #include <vector>
18
19 #include "absl/types/optional.h"
20 #include "api/audio/echo_canceller3_config.h"
21 #include "api/audio/echo_control.h"
22 #include "modules/audio_processing/aec3/aec3_common.h"
23 #include "modules/audio_processing/aec3/block_processor_metrics.h"
24 #include "modules/audio_processing/aec3/delay_estimate.h"
25 #include "modules/audio_processing/aec3/echo_path_variability.h"
26 #include "modules/audio_processing/aec3/echo_remover.h"
27 #include "modules/audio_processing/aec3/render_delay_buffer.h"
28 #include "modules/audio_processing/aec3/render_delay_controller.h"
29 #include "modules/audio_processing/logging/apm_data_dumper.h"
30 #include "rtc_base/checks.h"
31 #include "rtc_base/logging.h"
32
33 namespace webrtc {
34 namespace {
35
36 enum class BlockProcessorApiCall { kCapture, kRender };
37
38 class BlockProcessorImpl final : public BlockProcessor {
39 public:
40 BlockProcessorImpl(const EchoCanceller3Config& config,
41 int sample_rate_hz,
42 size_t num_render_channels,
43 size_t num_capture_channels,
44 std::unique_ptr<RenderDelayBuffer> render_buffer,
45 std::unique_ptr<RenderDelayController> delay_controller,
46 std::unique_ptr<EchoRemover> echo_remover);
47
48 BlockProcessorImpl() = delete;
49
50 ~BlockProcessorImpl() override;
51
52 void ProcessCapture(bool echo_path_gain_change,
53 bool capture_signal_saturation,
54 Block* linear_output,
55 Block* capture_block) override;
56
57 void BufferRender(const Block& block) override;
58
59 void UpdateEchoLeakageStatus(bool leakage_detected) override;
60
61 void GetMetrics(EchoControl::Metrics* metrics) const override;
62
63 void SetAudioBufferDelay(int delay_ms) override;
64 void SetCaptureOutputUsage(bool capture_output_used) override;
65
66 private:
67 static std::atomic<int> instance_count_;
68 std::unique_ptr<ApmDataDumper> data_dumper_;
69 const EchoCanceller3Config config_;
70 bool capture_properly_started_ = false;
71 bool render_properly_started_ = false;
72 const size_t sample_rate_hz_;
73 std::unique_ptr<RenderDelayBuffer> render_buffer_;
74 std::unique_ptr<RenderDelayController> delay_controller_;
75 std::unique_ptr<EchoRemover> echo_remover_;
76 BlockProcessorMetrics metrics_;
77 RenderDelayBuffer::BufferingEvent render_event_;
78 size_t capture_call_counter_ = 0;
79 absl::optional<DelayEstimate> estimated_delay_;
80 };
81
82 std::atomic<int> BlockProcessorImpl::instance_count_(0);
83
BlockProcessorImpl(const EchoCanceller3Config & config,int sample_rate_hz,size_t num_render_channels,size_t num_capture_channels,std::unique_ptr<RenderDelayBuffer> render_buffer,std::unique_ptr<RenderDelayController> delay_controller,std::unique_ptr<EchoRemover> echo_remover)84 BlockProcessorImpl::BlockProcessorImpl(
85 const EchoCanceller3Config& config,
86 int sample_rate_hz,
87 size_t num_render_channels,
88 size_t num_capture_channels,
89 std::unique_ptr<RenderDelayBuffer> render_buffer,
90 std::unique_ptr<RenderDelayController> delay_controller,
91 std::unique_ptr<EchoRemover> echo_remover)
92 : data_dumper_(new ApmDataDumper(instance_count_.fetch_add(1) + 1)),
93 config_(config),
94 sample_rate_hz_(sample_rate_hz),
95 render_buffer_(std::move(render_buffer)),
96 delay_controller_(std::move(delay_controller)),
97 echo_remover_(std::move(echo_remover)),
98 render_event_(RenderDelayBuffer::BufferingEvent::kNone) {
99 RTC_DCHECK(ValidFullBandRate(sample_rate_hz_));
100 }
101
102 BlockProcessorImpl::~BlockProcessorImpl() = default;
103
ProcessCapture(bool echo_path_gain_change,bool capture_signal_saturation,Block * linear_output,Block * capture_block)104 void BlockProcessorImpl::ProcessCapture(bool echo_path_gain_change,
105 bool capture_signal_saturation,
106 Block* linear_output,
107 Block* capture_block) {
108 RTC_DCHECK(capture_block);
109 RTC_DCHECK_EQ(NumBandsForRate(sample_rate_hz_), capture_block->NumBands());
110
111 capture_call_counter_++;
112
113 data_dumper_->DumpRaw("aec3_processblock_call_order",
114 static_cast<int>(BlockProcessorApiCall::kCapture));
115 data_dumper_->DumpWav("aec3_processblock_capture_input",
116 capture_block->View(/*band=*/0, /*channel=*/0), 16000,
117 1);
118
119 if (render_properly_started_) {
120 if (!capture_properly_started_) {
121 capture_properly_started_ = true;
122 render_buffer_->Reset();
123 if (delay_controller_)
124 delay_controller_->Reset(true);
125 }
126 } else {
127 // If no render data has yet arrived, do not process the capture signal.
128 render_buffer_->HandleSkippedCaptureProcessing();
129 return;
130 }
131
132 EchoPathVariability echo_path_variability(
133 echo_path_gain_change, EchoPathVariability::DelayAdjustment::kNone,
134 false);
135
136 if (render_event_ == RenderDelayBuffer::BufferingEvent::kRenderOverrun &&
137 render_properly_started_) {
138 echo_path_variability.delay_change =
139 EchoPathVariability::DelayAdjustment::kBufferFlush;
140 if (delay_controller_)
141 delay_controller_->Reset(true);
142 RTC_LOG(LS_WARNING) << "Reset due to render buffer overrun at block "
143 << capture_call_counter_;
144 }
145 render_event_ = RenderDelayBuffer::BufferingEvent::kNone;
146
147 // Update the render buffers with any newly arrived render blocks and prepare
148 // the render buffers for reading the render data corresponding to the current
149 // capture block.
150 RenderDelayBuffer::BufferingEvent buffer_event =
151 render_buffer_->PrepareCaptureProcessing();
152 // Reset the delay controller at render buffer underrun.
153 if (buffer_event == RenderDelayBuffer::BufferingEvent::kRenderUnderrun) {
154 if (delay_controller_)
155 delay_controller_->Reset(false);
156 }
157
158 data_dumper_->DumpWav("aec3_processblock_capture_input2",
159 capture_block->View(/*band=*/0, /*channel=*/0), 16000,
160 1);
161
162 bool has_delay_estimator = !config_.delay.use_external_delay_estimator;
163 if (has_delay_estimator) {
164 RTC_DCHECK(delay_controller_);
165 // Compute and apply the render delay required to achieve proper signal
166 // alignment.
167 estimated_delay_ = delay_controller_->GetDelay(
168 render_buffer_->GetDownsampledRenderBuffer(), render_buffer_->Delay(),
169 *capture_block);
170
171 if (estimated_delay_) {
172 bool delay_change =
173 render_buffer_->AlignFromDelay(estimated_delay_->delay);
174 if (delay_change) {
175 rtc::LoggingSeverity log_level =
176 config_.delay.log_warning_on_delay_changes ? rtc::LS_WARNING
177 : rtc::LS_INFO;
178 RTC_LOG_V(log_level) << "Delay changed to " << estimated_delay_->delay
179 << " at block " << capture_call_counter_;
180 echo_path_variability.delay_change =
181 EchoPathVariability::DelayAdjustment::kNewDetectedDelay;
182 }
183 }
184
185 echo_path_variability.clock_drift = delay_controller_->HasClockdrift();
186
187 } else {
188 render_buffer_->AlignFromExternalDelay();
189 }
190
191 // Remove the echo from the capture signal.
192 if (has_delay_estimator || render_buffer_->HasReceivedBufferDelay()) {
193 echo_remover_->ProcessCapture(
194 echo_path_variability, capture_signal_saturation, estimated_delay_,
195 render_buffer_->GetRenderBuffer(), linear_output, capture_block);
196 }
197
198 // Update the metrics.
199 metrics_.UpdateCapture(false);
200 }
201
BufferRender(const Block & block)202 void BlockProcessorImpl::BufferRender(const Block& block) {
203 RTC_DCHECK_EQ(NumBandsForRate(sample_rate_hz_), block.NumBands());
204 data_dumper_->DumpRaw("aec3_processblock_call_order",
205 static_cast<int>(BlockProcessorApiCall::kRender));
206 data_dumper_->DumpWav("aec3_processblock_render_input",
207 block.View(/*band=*/0, /*channel=*/0), 16000, 1);
208
209 render_event_ = render_buffer_->Insert(block);
210
211 metrics_.UpdateRender(render_event_ !=
212 RenderDelayBuffer::BufferingEvent::kNone);
213
214 render_properly_started_ = true;
215 if (delay_controller_)
216 delay_controller_->LogRenderCall();
217 }
218
UpdateEchoLeakageStatus(bool leakage_detected)219 void BlockProcessorImpl::UpdateEchoLeakageStatus(bool leakage_detected) {
220 echo_remover_->UpdateEchoLeakageStatus(leakage_detected);
221 }
222
GetMetrics(EchoControl::Metrics * metrics) const223 void BlockProcessorImpl::GetMetrics(EchoControl::Metrics* metrics) const {
224 echo_remover_->GetMetrics(metrics);
225 constexpr int block_size_ms = 4;
226 absl::optional<size_t> delay = render_buffer_->Delay();
227 metrics->delay_ms = delay ? static_cast<int>(*delay) * block_size_ms : 0;
228 }
229
SetAudioBufferDelay(int delay_ms)230 void BlockProcessorImpl::SetAudioBufferDelay(int delay_ms) {
231 render_buffer_->SetAudioBufferDelay(delay_ms);
232 }
233
SetCaptureOutputUsage(bool capture_output_used)234 void BlockProcessorImpl::SetCaptureOutputUsage(bool capture_output_used) {
235 echo_remover_->SetCaptureOutputUsage(capture_output_used);
236 }
237
238 } // namespace
239
Create(const EchoCanceller3Config & config,int sample_rate_hz,size_t num_render_channels,size_t num_capture_channels)240 BlockProcessor* BlockProcessor::Create(const EchoCanceller3Config& config,
241 int sample_rate_hz,
242 size_t num_render_channels,
243 size_t num_capture_channels) {
244 std::unique_ptr<RenderDelayBuffer> render_buffer(
245 RenderDelayBuffer::Create(config, sample_rate_hz, num_render_channels));
246 std::unique_ptr<RenderDelayController> delay_controller;
247 if (!config.delay.use_external_delay_estimator) {
248 delay_controller.reset(RenderDelayController::Create(config, sample_rate_hz,
249 num_capture_channels));
250 }
251 std::unique_ptr<EchoRemover> echo_remover(EchoRemover::Create(
252 config, sample_rate_hz, num_render_channels, num_capture_channels));
253 return Create(config, sample_rate_hz, num_render_channels,
254 num_capture_channels, std::move(render_buffer),
255 std::move(delay_controller), std::move(echo_remover));
256 }
257
Create(const EchoCanceller3Config & config,int sample_rate_hz,size_t num_render_channels,size_t num_capture_channels,std::unique_ptr<RenderDelayBuffer> render_buffer)258 BlockProcessor* BlockProcessor::Create(
259 const EchoCanceller3Config& config,
260 int sample_rate_hz,
261 size_t num_render_channels,
262 size_t num_capture_channels,
263 std::unique_ptr<RenderDelayBuffer> render_buffer) {
264 std::unique_ptr<RenderDelayController> delay_controller;
265 if (!config.delay.use_external_delay_estimator) {
266 delay_controller.reset(RenderDelayController::Create(config, sample_rate_hz,
267 num_capture_channels));
268 }
269 std::unique_ptr<EchoRemover> echo_remover(EchoRemover::Create(
270 config, sample_rate_hz, num_render_channels, num_capture_channels));
271 return Create(config, sample_rate_hz, num_render_channels,
272 num_capture_channels, std::move(render_buffer),
273 std::move(delay_controller), std::move(echo_remover));
274 }
275
Create(const EchoCanceller3Config & config,int sample_rate_hz,size_t num_render_channels,size_t num_capture_channels,std::unique_ptr<RenderDelayBuffer> render_buffer,std::unique_ptr<RenderDelayController> delay_controller,std::unique_ptr<EchoRemover> echo_remover)276 BlockProcessor* BlockProcessor::Create(
277 const EchoCanceller3Config& config,
278 int sample_rate_hz,
279 size_t num_render_channels,
280 size_t num_capture_channels,
281 std::unique_ptr<RenderDelayBuffer> render_buffer,
282 std::unique_ptr<RenderDelayController> delay_controller,
283 std::unique_ptr<EchoRemover> echo_remover) {
284 return new BlockProcessorImpl(config, sample_rate_hz, num_render_channels,
285 num_capture_channels, std::move(render_buffer),
286 std::move(delay_controller),
287 std::move(echo_remover));
288 }
289
290 } // namespace webrtc
291