1 /*
2 * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/audio_processing/aec3/multi_channel_content_detector.h"
12
13 #include <cmath>
14
15 #include "rtc_base/checks.h"
16 #include "system_wrappers/include/metrics.h"
17
18 namespace webrtc {
19
20 namespace {
21
22 constexpr int kNumFramesPerSecond = 100;
23
24 // Compares the left and right channels in the render `frame` to determine
25 // whether the signal is a proper stereo signal. To allow for differences
26 // introduced by hardware drivers, a threshold `detection_threshold` is used for
27 // the detection.
HasStereoContent(const std::vector<std::vector<std::vector<float>>> & frame,float detection_threshold)28 bool HasStereoContent(const std::vector<std::vector<std::vector<float>>>& frame,
29 float detection_threshold) {
30 if (frame[0].size() < 2) {
31 return false;
32 }
33
34 for (size_t band = 0; band < frame.size(); ++band) {
35 for (size_t k = 0; k < frame[band][0].size(); ++k) {
36 if (std::fabs(frame[band][0][k] - frame[band][1][k]) >
37 detection_threshold) {
38 return true;
39 }
40 }
41 }
42 return false;
43 }
44
45 // In order to avoid logging metrics for very short lifetimes that are unlikely
46 // to reflect real calls and that may dilute the "real" data, logging is limited
47 // to lifetimes of at leats 5 seconds.
48 constexpr int kMinNumberOfFramesRequiredToLogMetrics = 500;
49
50 // Continuous metrics are logged every 10 seconds.
51 constexpr int kFramesPer10Seconds = 1000;
52
53 } // namespace
54
MetricsLogger()55 MultiChannelContentDetector::MetricsLogger::MetricsLogger() {}
56
~MetricsLogger()57 MultiChannelContentDetector::MetricsLogger::~MetricsLogger() {
58 if (frame_counter_ < kMinNumberOfFramesRequiredToLogMetrics)
59 return;
60
61 RTC_HISTOGRAM_BOOLEAN(
62 "WebRTC.Audio.EchoCanceller.PersistentMultichannelContentEverDetected",
63 any_multichannel_content_detected_ ? 1 : 0);
64 }
65
Update(bool persistent_multichannel_content_detected)66 void MultiChannelContentDetector::MetricsLogger::Update(
67 bool persistent_multichannel_content_detected) {
68 ++frame_counter_;
69 if (persistent_multichannel_content_detected) {
70 any_multichannel_content_detected_ = true;
71 ++persistent_multichannel_frame_counter_;
72 }
73
74 if (frame_counter_ < kMinNumberOfFramesRequiredToLogMetrics)
75 return;
76 if (frame_counter_ % kFramesPer10Seconds != 0)
77 return;
78 const bool mostly_multichannel_last_10_seconds =
79 (persistent_multichannel_frame_counter_ >= kFramesPer10Seconds / 2);
80 RTC_HISTOGRAM_BOOLEAN(
81 "WebRTC.Audio.EchoCanceller.ProcessingPersistentMultichannelContent",
82 mostly_multichannel_last_10_seconds ? 1 : 0);
83
84 persistent_multichannel_frame_counter_ = 0;
85 }
86
MultiChannelContentDetector(bool detect_stereo_content,int num_render_input_channels,float detection_threshold,int stereo_detection_timeout_threshold_seconds,float stereo_detection_hysteresis_seconds)87 MultiChannelContentDetector::MultiChannelContentDetector(
88 bool detect_stereo_content,
89 int num_render_input_channels,
90 float detection_threshold,
91 int stereo_detection_timeout_threshold_seconds,
92 float stereo_detection_hysteresis_seconds)
93 : detect_stereo_content_(detect_stereo_content),
94 detection_threshold_(detection_threshold),
95 detection_timeout_threshold_frames_(
96 stereo_detection_timeout_threshold_seconds > 0
97 ? absl::make_optional(stereo_detection_timeout_threshold_seconds *
98 kNumFramesPerSecond)
99 : absl::nullopt),
100 stereo_detection_hysteresis_frames_(static_cast<int>(
101 stereo_detection_hysteresis_seconds * kNumFramesPerSecond)),
102 metrics_logger_((detect_stereo_content && num_render_input_channels > 1)
103 ? std::make_unique<MetricsLogger>()
104 : nullptr),
105 persistent_multichannel_content_detected_(
106 !detect_stereo_content && num_render_input_channels > 1) {}
107
UpdateDetection(const std::vector<std::vector<std::vector<float>>> & frame)108 bool MultiChannelContentDetector::UpdateDetection(
109 const std::vector<std::vector<std::vector<float>>>& frame) {
110 if (!detect_stereo_content_) {
111 RTC_DCHECK_EQ(frame[0].size() > 1,
112 persistent_multichannel_content_detected_);
113 return false;
114 }
115
116 const bool previous_persistent_multichannel_content_detected =
117 persistent_multichannel_content_detected_;
118 const bool stereo_detected_in_frame =
119 HasStereoContent(frame, detection_threshold_);
120
121 consecutive_frames_with_stereo_ =
122 stereo_detected_in_frame ? consecutive_frames_with_stereo_ + 1 : 0;
123 frames_since_stereo_detected_last_ =
124 stereo_detected_in_frame ? 0 : frames_since_stereo_detected_last_ + 1;
125
126 // Detect persistent multichannel content.
127 if (consecutive_frames_with_stereo_ > stereo_detection_hysteresis_frames_) {
128 persistent_multichannel_content_detected_ = true;
129 }
130 if (detection_timeout_threshold_frames_.has_value() &&
131 frames_since_stereo_detected_last_ >=
132 *detection_timeout_threshold_frames_) {
133 persistent_multichannel_content_detected_ = false;
134 }
135
136 // Detect temporary multichannel content.
137 temporary_multichannel_content_detected_ =
138 persistent_multichannel_content_detected_ ? false
139 : stereo_detected_in_frame;
140
141 if (metrics_logger_)
142 metrics_logger_->Update(persistent_multichannel_content_detected_);
143
144 return previous_persistent_multichannel_content_detected !=
145 persistent_multichannel_content_detected_;
146 }
147
148 } // namespace webrtc
149