1 /*
2 * Copyright 2022 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 // #define LOG_NDEBUG 0
18 #define LOG_TAG "audio_utils_MelProcessor"
19 // #define VERY_VERY_VERBOSE_LOGGING
20 #ifdef VERY_VERY_VERBOSE_LOGGING
21 #define ALOGVV ALOGV
22 #else
23 #define ALOGVV(a...) do { } while(0)
24 #endif
25
26 #include <audio_utils/MelProcessor.h>
27
28 #include <audio_utils/format.h>
29 #include <audio_utils/mutex.h>
30 #include <audio_utils/power.h>
31 #include <chrono>
32 #include <log/log.h>
33 #include <sstream>
34 #include <unordered_map>
35 #include <utils/threads.h>
36
37 namespace android::audio_utils {
38
39 constexpr int32_t kSecondsPerMelValue = 1;
40 constexpr float kMelAdjustmentDb = -3.f;
41
42 // Estimated offset defined in Table39 of IEC62368-1 3rd edition
43 // -30dBFS, -10dBFS should correspond to 80dBSPL, 100dBSPL
44 constexpr float kMeldBFSTodBSPLOffset = 110.f;
45
46 constexpr float kRs1OutputdBFS = 80.f; // dBA
47
48 constexpr float kRs2LowerBound = 80.f; // dBA
49 constexpr float kRs2UpperBound = 100.f; // dBA
50
51 // The following arrays contain the IIR biquad filter coefficients for performing A-weighting as
52 // described in IEC 61672:2003 for multiple sample rates. The format is b0, b1, b2, a1, a2
53 constexpr std::array<std::array<float, kBiquadNumCoefs>, MelProcessor::kCascadeBiquadNumber>
54 kBqCoeffs8000 = {{{0.630301, 0.000000, -0.630301, 0.103818, -0.360417},
55 {1.000000, 0.000000, -1.000000, -0.264382, -0.601403},
56 {1.000000, -2.000000, 1.000000, -1.967903, 0.968160}}};
57 constexpr std::array<std::array<float, kBiquadNumCoefs>, MelProcessor::kCascadeBiquadNumber>
58 kBqCoeffs11025 = {{{0.601164, 1.202327, 0.601164, 1.106098, 0.305863},
59 {1.000000, -2.000000, 1.000000, -1.593019, 0.613701},
60 {1.000000, -2.000000, 1.000000, -1.976658, 0.976794}}};
61 constexpr std::array<std::array<float, kBiquadNumCoefs>, MelProcessor::kCascadeBiquadNumber>
62 kBqCoeffs12000 = {{{0.588344, 1.176688, 0.588344, 1.045901, 0.273477},
63 {1.000000, -2.000000, 1.000000, -1.621383, 0.639134},
64 {1.000000, -2.000000, 1.000000, -1.978544, 0.978660}}};
65 constexpr std::array<std::array<float, kBiquadNumCoefs>, MelProcessor::kCascadeBiquadNumber>
66 kBqCoeffs16000 = {{{0.531220, 1.062441, 0.531220, 0.821564, 0.168742},
67 {1.000000, -2.000000, 1.000000, -1.705510, 0.715988},
68 {1.000000, -2.000000, 1.000000, -1.983887, 0.983952}}};
69 constexpr std::array<std::array<float, kBiquadNumCoefs>, MelProcessor::kCascadeBiquadNumber>
70 kBqCoeffs22050 = {{{0.449072, 0.898144, 0.449072, 0.538750, 0.072563},
71 {1.000000, -2.000000, 1.000000, -1.779533, 0.785281},
72 {1.000000, -2.000000, 1.000000, -1.988295, 0.988329}}};
73 constexpr std::array<std::array<float, kBiquadNumCoefs>, MelProcessor::kCascadeBiquadNumber>
74 kBqCoeffs24000 = {{{0.425411, 0.850821, 0.425411, 0.459298, 0.052739},
75 {1.000000, -2.000000, 1.000000, -1.796051, 0.800946},
76 {1.000000, -2.000000, 1.000000, -1.989243, 0.989272}}};
77 constexpr std::array<std::array<float, kBiquadNumCoefs>, MelProcessor::kCascadeBiquadNumber>
78 kBqCoeffs32000 = {{{0.343284, 0.686569, 0.343284, 0.179472, 0.008053},
79 {1.000000, -2.000000, 1.000000, -1.843991, 0.846816},
80 {1.000000, -2.000000, 1.000000, -1.991927, 0.991943}}};
81 constexpr std::array<std::array<float, kBiquadNumCoefs>, MelProcessor::kCascadeBiquadNumber>
82 kBqCoeffs44100 = {{{0.255612, 0.511223, 0.255612, -0.140536, 0.004938},
83 {1.000000, -2.000000, 1.000000, -1.884901, 0.886421},
84 {1.000000, -2.000000, 1.000000, -1.994139, 0.994147}}};
85 constexpr std::array<std::array<float, kBiquadNumCoefs>, MelProcessor::kCascadeBiquadNumber>
86 kBqCoeffs48000 = {{{0.234183, 0.468366, 0.234183, -0.224558, 0.012607},
87 {1.000000, -2.000000, 1.000000, -1.893870, 0.895160},
88 {1.000000, -2.000000, 1.000000, -1.994614, 0.994622}}};
89 constexpr std::array<std::array<float, kBiquadNumCoefs>, MelProcessor::kCascadeBiquadNumber>
90 kBqCoeffs64000 = {{{0.169014, 0.338029, 0.169014, -0.502217, 0.063056},
91 {1.000000, -2.000000, 1.000000, -1.919579, 0.920314},
92 {1.000000, -2.000000, 1.000000, -1.995959, 0.995964}}};
93 constexpr std::array<std::array<float, kBiquadNumCoefs>, MelProcessor::kCascadeBiquadNumber>
94 kBqCoeffs88200 = {{{0.111831, 0.223662, 0.111831, -0.788729, 0.155523},
95 {1.000000, -2.000000, 1.000000, -1.941143, 0.941534},
96 {1.000000, -2.000000, 1.000000, -1.997067, 0.997069}}};
97 constexpr std::array<std::array<float, kBiquadNumCoefs>, MelProcessor::kCascadeBiquadNumber>
98 kBqCoeffs96000 = {{{0.099469, 0.198937, 0.099469, -0.859073, 0.184502},
99 {1.000000, -2.000000, 1.000000, -1.945825, 0.946156},
100 {1.000000, -2.000000, 1.000000, -1.997305, 0.997307}}};
101 constexpr std::array<std::array<float, kBiquadNumCoefs>, MelProcessor::kCascadeBiquadNumber>
102 kBqCoeffs128000 = {{{0.065337, 0.130674, 0.065337, -1.078602, 0.290845},
103 {1.000000, -2.000000, 1.000000, -1.959154, 0.959342},
104 {1.000000, -2.000000, 1.000000, -1.997979, 0.997980}}};
105 constexpr std::array<std::array<float, kBiquadNumCoefs>, MelProcessor::kCascadeBiquadNumber>
106 kBqCoeffs176400 = {{{0.039432, 0.078864, 0.039432, -1.286304, 0.413645},
107 {1.000000, -2.000000, 1.000000, -1.970232, 0.970331},
108 {1.000000, -2.000000, 1.000000, -1.998533, 0.998534}}};
109 constexpr std::array<std::array<float, kBiquadNumCoefs>, MelProcessor::kCascadeBiquadNumber>
110 kBqCoeffs192000 = {{{0.034315, 0.068629, 0.034315, -1.334647, 0.445320},
111 {1.000000, -2.000000, 1.000000, -1.972625, 0.972709},
112 {1.000000, -2.000000, 1.000000, -1.998652, 0.998653}}};
113
MelProcessor(uint32_t sampleRate,uint32_t channelCount,audio_format_t format,const sp<MelCallback> & callback,audio_port_handle_t deviceId,float rs2Value,size_t maxMelsCallback)114 MelProcessor::MelProcessor(uint32_t sampleRate,
115 uint32_t channelCount,
116 audio_format_t format,
117 const sp<MelCallback>& callback,
118 audio_port_handle_t deviceId,
119 float rs2Value,
120 size_t maxMelsCallback)
121 : mCallback(callback),
122 mMelWorker("MelWorker#" + pointerString(), mCallback),
123 mSampleRate(sampleRate),
124 mFramesPerMelValue(sampleRate * kSecondsPerMelValue),
125 mChannelCount(channelCount),
126 mFormat(format),
127 mAWeightSamples(mFramesPerMelValue * mChannelCount),
128 mFloatSamples(mFramesPerMelValue * mChannelCount),
129 mCurrentChannelEnergy(channelCount, 0.0f),
130 mMelValues(maxMelsCallback),
131 mDeviceId(deviceId),
132 mRs2UpperBound(rs2Value)
133 {
134 createBiquads_l();
135
136 mMelWorker.run();
137 }
138
139 static const std::unordered_map<uint32_t, const std::array<std::array<float, kBiquadNumCoefs>,
getSampleRateBiquadCoeffs()140 MelProcessor::kCascadeBiquadNumber>*>& getSampleRateBiquadCoeffs() {
141 static const std::unordered_map<uint32_t, const std::array<std::array<float, kBiquadNumCoefs>,
142 MelProcessor::kCascadeBiquadNumber>*> sampleRateBiquadCoeffs = {
143 {8000, &kBqCoeffs8000},
144 {11025, &kBqCoeffs11025},
145 {12000, &kBqCoeffs12000},
146 {16000, &kBqCoeffs16000},
147 {22050, &kBqCoeffs22050},
148 {24000, &kBqCoeffs24000},
149 {32000, &kBqCoeffs32000},
150 {44100, &kBqCoeffs44100},
151 {48000, &kBqCoeffs48000},
152 {64000, &kBqCoeffs64000},
153 {88200, &kBqCoeffs88200},
154 {96000, &kBqCoeffs96000},
155 {128000, &kBqCoeffs128000},
156 {176400, &kBqCoeffs176400},
157 {192000, &kBqCoeffs192000},
158 };
159 return sampleRateBiquadCoeffs;
160 }
161
isSampleRateSupported_l() const162 bool MelProcessor::isSampleRateSupported_l() const {
163 return getSampleRateBiquadCoeffs().count(mSampleRate) != 0;
164 }
165
createBiquads_l()166 void MelProcessor::createBiquads_l() {
167 if (!isSampleRateSupported_l()) {
168 return;
169 }
170
171 const auto& biquadCoeffs = getSampleRateBiquadCoeffs().at(mSampleRate); // checked above
172 mCascadedBiquads =
173 {std::make_unique<DefaultBiquadFilter>(mChannelCount, biquadCoeffs->at(0)),
174 std::make_unique<DefaultBiquadFilter>(mChannelCount, biquadCoeffs->at(1)),
175 std::make_unique<DefaultBiquadFilter>(mChannelCount, biquadCoeffs->at(2))};
176 }
177
setOutputRs2UpperBound(float rs2Value)178 status_t MelProcessor::setOutputRs2UpperBound(float rs2Value)
179 {
180 if (rs2Value < kRs2LowerBound || rs2Value > kRs2UpperBound) {
181 return BAD_VALUE;
182 }
183
184 mRs2UpperBound = rs2Value;
185
186 return NO_ERROR;
187 }
188
getOutputRs2UpperBound() const189 float MelProcessor::getOutputRs2UpperBound() const
190 {
191 return mRs2UpperBound;
192 }
193
setDeviceId(audio_port_handle_t deviceId)194 void MelProcessor::setDeviceId(audio_port_handle_t deviceId)
195 {
196 mDeviceId = deviceId;
197 }
198
getDeviceId()199 audio_port_handle_t MelProcessor::getDeviceId() {
200 return mDeviceId;
201 }
202
pause()203 void MelProcessor::pause()
204 {
205 ALOGV("%s", __func__);
206 mPaused = true;
207 }
208
resume()209 void MelProcessor::resume()
210 {
211 ALOGV("%s", __func__);
212 mPaused = false;
213 }
214
drain()215 void MelProcessor::drain()
216 {
217 ALOGV("%s", __func__);
218 mMelWorker.drain();
219 }
220
drainAndWait()221 void MelProcessor::drainAndWait() {
222 constexpr size_t kPollMs = 8;
223 while (!mMelWorker.ringBufferIsEmpty()) {
224 drain();
225 std::this_thread::sleep_for(std::chrono::milliseconds(kPollMs));
226 }
227 }
228
updateAudioFormat(uint32_t sampleRate,uint32_t channelCount,audio_format_t format)229 void MelProcessor::updateAudioFormat(uint32_t sampleRate,
230 uint32_t channelCount,
231 audio_format_t format) {
232 ALOGV("%s: update audio format %u, %u, %d", __func__, sampleRate, channelCount, format);
233
234 std::lock_guard l(mLock);
235
236 bool differentSampleRate = (mSampleRate != sampleRate);
237 bool differentChannelCount = (mChannelCount != channelCount);
238
239 mSampleRate = sampleRate;
240 mFramesPerMelValue = sampleRate * kSecondsPerMelValue;
241 mChannelCount = channelCount;
242 mFormat = format;
243
244 if (differentSampleRate || differentChannelCount) {
245 mAWeightSamples.resize(mFramesPerMelValue * mChannelCount);
246 mFloatSamples.resize(mFramesPerMelValue * mChannelCount);
247 }
248 if (differentChannelCount) {
249 mCurrentChannelEnergy.resize(channelCount);
250 }
251
252 createBiquads_l();
253 }
254
applyAWeight_l(const void * buffer,size_t samples)255 void MelProcessor::applyAWeight_l(const void* buffer, size_t samples)
256 {
257 memcpy_by_audio_format(mFloatSamples.data(), AUDIO_FORMAT_PCM_FLOAT, buffer, mFormat, samples);
258
259 float* tempFloat[2] = { mFloatSamples.data(), mAWeightSamples.data() };
260 int inIdx = 1, outIdx = 0;
261 const size_t frames = samples / mChannelCount;
262 for (const auto& biquad : mCascadedBiquads) {
263 outIdx ^= 1;
264 inIdx ^= 1;
265 biquad->process(tempFloat[outIdx], tempFloat[inIdx], frames);
266 }
267
268 // should not be the case since the size is odd
269 if (!(mCascadedBiquads.size() & 1)) {
270 std::swap(mFloatSamples, mAWeightSamples);
271 }
272 }
273
getCombinedChannelEnergy_l()274 float MelProcessor::getCombinedChannelEnergy_l() {
275 float combinedEnergy = 0.0f;
276 for (auto& energy: mCurrentChannelEnergy) {
277 combinedEnergy += energy;
278 energy = 0;
279 }
280
281 combinedEnergy /= (float) mFramesPerMelValue;
282 return combinedEnergy;
283 }
284
addMelValue_l(float mel)285 void MelProcessor::addMelValue_l(float mel) {
286 mMelValues[mCurrentIndex] = mel;
287 ALOGV("%s: writing MEL %f at index %d for device %d",
288 __func__,
289 mel,
290 mCurrentIndex,
291 mDeviceId.load());
292
293 bool notifyWorker = false;
294
295 if (mel > mRs2UpperBound) {
296 mMelWorker.momentaryExposure(mel, mDeviceId);
297 notifyWorker = true;
298 }
299
300 bool reportContinuousValues = false;
301 if ((mMelValues[mCurrentIndex] < kRs1OutputdBFS && mCurrentIndex > 0)) {
302 reportContinuousValues = true;
303 } else if (mMelValues[mCurrentIndex] >= kRs1OutputdBFS) {
304 // only store MEL that are above RS1
305 ++mCurrentIndex;
306 }
307
308 if (reportContinuousValues || (mCurrentIndex > mMelValues.size() - 1)) {
309 mMelWorker.newMelValues(mMelValues, mCurrentIndex, mDeviceId);
310 notifyWorker = true;
311 mCurrentIndex = 0;
312 }
313
314 if (notifyWorker) {
315 mMelWorker.notify();
316 }
317 }
318
process(const void * buffer,size_t bytes)319 int32_t MelProcessor::process(const void* buffer, size_t bytes) {
320 if (mPaused) {
321 return 0;
322 }
323
324 // should be uncontested and not block if process method is called from a single thread
325 std::lock_guard<std::mutex> guard(mLock);
326
327 if (!isSampleRateSupported_l()) {
328 return 0;
329 }
330
331 const size_t bytes_per_sample = audio_bytes_per_sample(mFormat);
332 size_t samples = bytes_per_sample > 0 ? bytes / bytes_per_sample : 0;
333 while (samples > 0) {
334 const size_t requiredSamples =
335 mFramesPerMelValue * mChannelCount - mCurrentSamples;
336 size_t processSamples = std::min(requiredSamples, samples);
337 processSamples -= processSamples % mChannelCount;
338
339 applyAWeight_l(buffer, processSamples);
340
341 audio_utils_accumulate_energy(mAWeightSamples.data(),
342 AUDIO_FORMAT_PCM_FLOAT,
343 processSamples,
344 mChannelCount,
345 mCurrentChannelEnergy.data());
346 mCurrentSamples += processSamples;
347
348 ALOGVV(
349 "required:%zu, process:%zu, mCurrentChannelEnergy[0]:%f, mCurrentSamples:%zu",
350 requiredSamples,
351 processSamples,
352 mCurrentChannelEnergy[0],
353 mCurrentSamples.load());
354 if (processSamples < requiredSamples) {
355 return static_cast<int32_t>(bytes);
356 }
357
358 addMelValue_l(fmaxf(
359 audio_utils_power_from_energy(getCombinedChannelEnergy_l())
360 + kMelAdjustmentDb
361 + kMeldBFSTodBSPLOffset
362 - mAttenuationDB, 0.0f));
363
364 samples -= processSamples;
365 buffer =
366 (const uint8_t*) buffer + processSamples * bytes_per_sample;
367 mCurrentSamples = 0;
368 }
369
370 return static_cast<int32_t>(bytes);
371 }
372
setAttenuation(float attenuationDB)373 void MelProcessor::setAttenuation(float attenuationDB) {
374 ALOGV("%s: setting the attenuation %f", __func__, attenuationDB);
375 mAttenuationDB = attenuationDB;
376 }
377
onLastStrongRef(const void * id)378 void MelProcessor::onLastStrongRef(const void* id __attribute__((unused))) {
379 mMelWorker.stop();
380 ALOGV("%s: Stopped thread: %s for device %d", __func__, mMelWorker.getThreadName().c_str(),
381 mDeviceId.load());
382 }
383
pointerString() const384 std::string MelProcessor::pointerString() const {
385 const void * address = static_cast<const void*>(this);
386 std::stringstream aStream;
387 aStream << address;
388 return aStream.str();
389 }
390
run()391 void MelProcessor::MelWorker::run() {
392 mThread = std::thread([&]{
393 // name the thread to help identification
394 androidSetThreadName(mThreadName.c_str());
395 ALOGV("%s::run(): Started thread", mThreadName.c_str());
396
397 audio_utils::unique_lock l(mCondVarMutex);
398 while (true) {
399 if (mStopRequested) {
400 return;
401 }
402 mCondVar.wait(l);
403 while (mRbReadPtr != mRbWritePtr && !mStopRequested) { // load-acquire mRbWritePtr
404 ALOGV("%s::run(): new callbacks, rb idx read=%zu, write=%zu",
405 mThreadName.c_str(),
406 mRbReadPtr.load(),
407 mRbWritePtr.load());
408 const auto callback = mCallback.promote();
409 if (callback == nullptr) {
410 ALOGW("%s::run(): MelCallback is null, quitting MelWorker",
411 mThreadName.c_str());
412 return;
413 }
414
415 const MelCallbackData& data = mCallbackRingBuffer[mRbReadPtr];
416 if (data.mMel != 0.f) {
417 l.unlock();
418 callback->onMomentaryExposure(data.mMel, data.mPort);
419 l.lock();
420 } else if (data.mMelsSize != 0) {
421 l.unlock();
422 callback->onNewMelValues(data.mMels, 0, data.mMelsSize,
423 data.mPort, /*attenuated=*/true);
424 l.lock();
425 } else {
426 ALOGE("%s::run(): Invalid MEL data. Skipping callback", mThreadName.c_str());
427 }
428 mRbReadPtr = nextRingBufferIndex(mRbReadPtr); // single reader updates this.
429 }
430 }
431 });
432 }
433
stop()434 void MelProcessor::MelWorker::stop() {
435 bool oldValue;
436 {
437 std::lock_guard l(mCondVarMutex);
438 oldValue = mStopRequested;
439 mStopRequested = true;
440 }
441 if (!oldValue) {
442 mCondVar.notify_one();
443 mThread.join();
444 }
445 }
446
drain()447 void MelProcessor::MelWorker::drain() {
448 std::lock_guard l(mCondVarMutex);
449 mCondVar.notify_one();
450 }
451
momentaryExposure(float mel,audio_port_handle_t port)452 void MelProcessor::MelWorker::momentaryExposure(float mel, audio_port_handle_t port) {
453 ALOGV("%s", __func__);
454
455 if (ringBufferIsFull()) {
456 ALOGW("%s: cannot add momentary exposure for port %d, MelWorker buffer is full", __func__,
457 port);
458 return;
459 }
460
461 // worker is thread-safe, no lock since there is only one writer and we take into account
462 // spurious wake-ups
463 mCallbackRingBuffer[mRbWritePtr].mMel = mel;
464 mCallbackRingBuffer[mRbWritePtr].mMelsSize = 0;
465 mCallbackRingBuffer[mRbWritePtr].mPort = port;
466
467 mRbWritePtr = nextRingBufferIndex(mRbWritePtr); // single writer, store-release.
468 }
469
newMelValues(const std::vector<float> & mels,size_t melsSize,audio_port_handle_t port)470 void MelProcessor::MelWorker::newMelValues(const std::vector<float>& mels,
471 size_t melsSize,
472 audio_port_handle_t port) {
473 ALOGV("%s", __func__);
474
475 if (ringBufferIsFull()) {
476 ALOGW("%s: cannot add %zu mel values for port %d, MelWorker buffer is full", __func__,
477 melsSize, port);
478 return;
479 }
480
481 // worker is thread-safe, no lock since there is only one writer and we take into account
482 // spurious wake-ups
483 std::copy_n(std::begin(mels), melsSize, mCallbackRingBuffer[mRbWritePtr].mMels.begin());
484 mCallbackRingBuffer[mRbWritePtr].mMelsSize = melsSize;
485 mCallbackRingBuffer[mRbWritePtr].mMel = 0.f;
486 mCallbackRingBuffer[mRbWritePtr].mPort = port;
487
488 mRbWritePtr = nextRingBufferIndex(mRbWritePtr); // single writer, store-release.
489 }
490
ringBufferIsFull() const491 bool MelProcessor::MelWorker::ringBufferIsFull() const {
492 return nextRingBufferIndex(mRbWritePtr) == mRbReadPtr;
493 }
494
495 } // namespace android
496