1 /*
2 **
3 ** Copyright 2021, The Android Open Source Project
4 **
5 ** Licensed under the Apache License, Version 2.0 (the "License");
6 ** you may not use this file except in compliance with the License.
7 ** You may obtain a copy of the License at
8 **
9 ** http://www.apache.org/licenses/LICENSE-2.0
10 **
11 ** Unless required by applicable law or agreed to in writing, software
12 ** distributed under the License is distributed on an "AS IS" BASIS,
13 ** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 ** See the License for the specific language governing permissions and
15 ** limitations under the License.
16 */
17
18 #include <string>
19 #define LOG_TAG "Spatializer"
20 //#define LOG_NDEBUG 0
21 #include <utils/Log.h>
22
23 #include <algorithm>
24 #include <inttypes.h>
25 #include <limits.h>
26 #include <stdint.h>
27 #include <sys/types.h>
28
29 #include <android/content/AttributionSourceState.h>
30 #include <android/sysprop/BluetoothProperties.sysprop.h>
31 #include <audio_utils/fixedfft.h>
32 #include <cutils/bitops.h>
33 #include <hardware/sensors.h>
34 #include <media/stagefright/foundation/AHandler.h>
35 #include <media/stagefright/foundation/AMessage.h>
36 #include <media/MediaMetricsItem.h>
37 #include <media/QuaternionUtil.h>
38 #include <media/ShmemCompat.h>
39 #include <mediautils/SchedulingPolicyService.h>
40 #include <mediautils/ServiceUtilities.h>
41 #include <utils/Thread.h>
42
43 #include "Spatializer.h"
44 #include "SpatializerHelper.h"
45
46 namespace android {
47
48 using aidl_utils::binderStatusFromStatusT;
49 using aidl_utils::statusTFromBinderStatus;
50 using android::content::AttributionSourceState;
51 using binder::Status;
52 using internal::ToString;
53 using media::HeadTrackingMode;
54 using media::Pose3f;
55 using media::SensorPoseProvider;
56 using media::audio::common::HeadTracking;
57 using media::audio::common::Spatialization;
58
59 using namespace std::chrono_literals;
60
61 #define VALUE_OR_RETURN_BINDER_STATUS(x) \
62 ({ auto _tmp = (x); \
63 if (!_tmp.ok()) return aidl_utils::binderStatusFromStatusT(_tmp.error()); \
64 std::move(_tmp.value()); })
65
getMaxChannelMask(const std::vector<audio_channel_mask_t> & masks,size_t channelLimit=SIZE_MAX)66 static audio_channel_mask_t getMaxChannelMask(
67 const std::vector<audio_channel_mask_t>& masks, size_t channelLimit = SIZE_MAX) {
68 uint32_t maxCount = 0;
69 audio_channel_mask_t maxMask = AUDIO_CHANNEL_NONE;
70 for (auto mask : masks) {
71 const size_t count = audio_channel_count_from_out_mask(mask);
72 if (count > channelLimit) continue; // ignore masks greater than channelLimit
73 if (count > maxCount) {
74 maxMask = mask;
75 maxCount = count;
76 }
77 }
78 return maxMask;
79 }
80
recordFromTranslationRotationVector(const std::vector<float> & trVector)81 static std::vector<float> recordFromTranslationRotationVector(
82 const std::vector<float>& trVector) {
83 auto headToStageOpt = Pose3f::fromVector(trVector);
84 if (!headToStageOpt) return {};
85
86 const auto stageToHead = headToStageOpt.value().inverse();
87 const auto stageToHeadTranslation = stageToHead.translation();
88 constexpr float RAD_TO_DEGREE = 180.f / M_PI;
89 std::vector<float> record{
90 stageToHeadTranslation[0], stageToHeadTranslation[1], stageToHeadTranslation[2],
91 0.f, 0.f, 0.f};
92 media::quaternionToAngles(stageToHead.rotation(), &record[3], &record[4], &record[5]);
93 record[3] *= RAD_TO_DEGREE;
94 record[4] *= RAD_TO_DEGREE;
95 record[5] *= RAD_TO_DEGREE;
96 return record;
97 }
98
99 template<typename T>
safe_clamp(const T & value,const T & low,const T & high)100 static constexpr const T& safe_clamp(const T& value, const T& low, const T& high) {
101 if constexpr (std::is_floating_point_v<T>) {
102 return value != value /* constexpr isnan */
103 ? low : std::clamp(value, low, high);
104 } else /* constexpr */ {
105 return std::clamp(value, low, high);
106 }
107 }
108
109 // ---------------------------------------------------------------------------
110
111 class Spatializer::EngineCallbackHandler : public AHandler {
112 public:
EngineCallbackHandler(wp<Spatializer> spatializer)113 EngineCallbackHandler(wp<Spatializer> spatializer)
114 : mSpatializer(spatializer) {
115 }
116
117 enum {
118 // Device state callbacks
119 kWhatOnFramesProcessed, // AudioEffect::EVENT_FRAMES_PROCESSED
120 kWhatOnHeadToStagePose, // SpatializerPoseController::Listener::onHeadToStagePose
121 kWhatOnActualModeChange, // SpatializerPoseController::Listener::onActualModeChange
122 kWhatOnLatencyModesChanged, // Spatializer::onSupportedLatencyModesChanged
123 };
124 static constexpr const char *kNumFramesKey = "numFrames";
125 static constexpr const char *kModeKey = "mode";
126 static constexpr const char *kTranslation0Key = "translation0";
127 static constexpr const char *kTranslation1Key = "translation1";
128 static constexpr const char *kTranslation2Key = "translation2";
129 static constexpr const char *kRotation0Key = "rotation0";
130 static constexpr const char *kRotation1Key = "rotation1";
131 static constexpr const char *kRotation2Key = "rotation2";
132 static constexpr const char *kLatencyModesKey = "latencyModes";
133
134 class LatencyModes : public RefBase {
135 public:
LatencyModes(audio_io_handle_t output,const std::vector<audio_latency_mode_t> & latencyModes)136 LatencyModes(audio_io_handle_t output,
137 const std::vector<audio_latency_mode_t>& latencyModes)
138 : mOutput(output), mLatencyModes(latencyModes) {}
139 ~LatencyModes() = default;
140
141 audio_io_handle_t mOutput;
142 std::vector<audio_latency_mode_t> mLatencyModes;
143 };
144
onMessageReceived(const sp<AMessage> & msg)145 void onMessageReceived(const sp<AMessage> &msg) override {
146 // No ALooper method to get the tid so update
147 // Spatializer priority on the first message received.
148 std::call_once(mPrioritySetFlag, [](){
149 const pid_t pid = getpid();
150 const pid_t tid = gettid();
151 (void)requestSpatializerPriority(pid, tid);
152 });
153
154 sp<Spatializer> spatializer = mSpatializer.promote();
155 if (spatializer == nullptr) {
156 ALOGW("%s: Cannot promote spatializer", __func__);
157 return;
158 }
159 switch (msg->what()) {
160 case kWhatOnFramesProcessed: {
161 int numFrames;
162 if (!msg->findInt32(kNumFramesKey, &numFrames)) {
163 ALOGE("%s: Cannot find num frames!", __func__);
164 return;
165 }
166 if (numFrames > 0) {
167 spatializer->calculateHeadPose();
168 }
169 } break;
170 case kWhatOnHeadToStagePose: {
171 std::vector<float> headToStage(sHeadPoseKeys.size());
172 for (size_t i = 0 ; i < sHeadPoseKeys.size(); i++) {
173 if (!msg->findFloat(sHeadPoseKeys[i], &headToStage[i])) {
174 ALOGE("%s: Cannot find kTranslation0Key!", __func__);
175 return;
176 }
177 }
178 spatializer->onHeadToStagePoseMsg(headToStage);
179 } break;
180 case kWhatOnActualModeChange: {
181 int mode;
182 if (!msg->findInt32(kModeKey, &mode)) {
183 ALOGE("%s: Cannot find actualMode!", __func__);
184 return;
185 }
186 spatializer->onActualModeChangeMsg(static_cast<HeadTrackingMode>(mode));
187 } break;
188
189 case kWhatOnLatencyModesChanged: {
190 sp<RefBase> object;
191 if (!msg->findObject(kLatencyModesKey, &object)) {
192 ALOGE("%s: Cannot find latency modes!", __func__);
193 return;
194 }
195 sp<LatencyModes> latencyModes = static_cast<LatencyModes*>(object.get());
196 spatializer->onSupportedLatencyModesChangedMsg(
197 latencyModes->mOutput, std::move(latencyModes->mLatencyModes));
198 } break;
199
200 default:
201 LOG_ALWAYS_FATAL("Invalid callback message %d", msg->what());
202 }
203 }
204 private:
205 wp<Spatializer> mSpatializer;
206 std::once_flag mPrioritySetFlag;
207 };
208
209 const std::vector<const char *> Spatializer::sHeadPoseKeys = {
210 Spatializer::EngineCallbackHandler::kTranslation0Key,
211 Spatializer::EngineCallbackHandler::kTranslation1Key,
212 Spatializer::EngineCallbackHandler::kTranslation2Key,
213 Spatializer::EngineCallbackHandler::kRotation0Key,
214 Spatializer::EngineCallbackHandler::kRotation1Key,
215 Spatializer::EngineCallbackHandler::kRotation2Key,
216 };
217
218 // Mapping table between strings read form property bluetooth.core.le.dsa_transport_preference
219 // and low latency modes emums.
220 //TODO b/273373363: use AIDL enum when available
221 const std::map<std::string, audio_latency_mode_t> Spatializer::sStringToLatencyModeMap = {
222 {"le-acl", AUDIO_LATENCY_MODE_LOW},
223 {"iso-sw", AUDIO_LATENCY_MODE_DYNAMIC_SPATIAL_AUDIO_SOFTWARE},
224 {"iso-hw", AUDIO_LATENCY_MODE_DYNAMIC_SPATIAL_AUDIO_HARDWARE},
225 };
226
loadOrderedLowLatencyModes()227 void Spatializer::loadOrderedLowLatencyModes() {
228 if (!com::android::media::audio::dsa_over_bt_le_audio()) {
229 return;
230 }
231 auto latencyModesStrs = android::sysprop::BluetoothProperties::dsa_transport_preference();
232 audio_utils::lock_guard lock(mMutex);
233 // First load preferred low latency modes ordered from the property
234 for (auto str : latencyModesStrs) {
235 if (!str.has_value()) continue;
236 if (auto it = sStringToLatencyModeMap.find(str.value());
237 it != sStringToLatencyModeMap.end()) {
238 mOrderedLowLatencyModes.push_back(it->second);
239 }
240 }
241 // Then add unlisted latency modes at the end of the ordered list
242 for (auto it : sStringToLatencyModeMap) {
243 if (std::find(mOrderedLowLatencyModes.begin(), mOrderedLowLatencyModes.end(), it.second)
244 == mOrderedLowLatencyModes.end()) {
245 mOrderedLowLatencyModes.push_back(it.second);
246 }
247 }
248 }
249
250 // ---------------------------------------------------------------------------
create(SpatializerPolicyCallback * callback,const sp<EffectsFactoryHalInterface> & effectsFactoryHal)251 sp<Spatializer> Spatializer::create(SpatializerPolicyCallback* callback,
252 const sp<EffectsFactoryHalInterface>& effectsFactoryHal) {
253 sp<Spatializer> spatializer;
254
255 if (effectsFactoryHal == nullptr) {
256 ALOGW("%s failed to create effect factory interface", __func__);
257 return spatializer;
258 }
259
260 std::vector<effect_descriptor_t> descriptors;
261 status_t status = effectsFactoryHal->getDescriptors(FX_IID_SPATIALIZER, &descriptors);
262 if (status != NO_ERROR) {
263 ALOGW("%s failed to get spatializer descriptor, error %d", __func__, status);
264 return spatializer;
265 }
266 ALOG_ASSERT(!descriptors.empty(),
267 "%s getDescriptors() returned no error but empty list", __func__);
268
269 // TODO: get supported spatialization modes from FX engine or descriptor
270 sp<EffectHalInterface> effect;
271 status = effectsFactoryHal->createEffect(&descriptors[0].uuid, AUDIO_SESSION_OUTPUT_STAGE,
272 AUDIO_IO_HANDLE_NONE, AUDIO_PORT_HANDLE_NONE, &effect);
273 ALOGI("%s FX create status %d effect %p", __func__, status, effect.get());
274
275 if (status == NO_ERROR && effect != nullptr) {
276 spatializer = new Spatializer(descriptors[0], callback);
277 if (spatializer->loadEngineConfiguration(effect) != NO_ERROR) {
278 spatializer.clear();
279 ALOGW("%s loadEngine error: %d effect %p", __func__, status, effect.get());
280 } else {
281 spatializer->loadOrderedLowLatencyModes();
282 spatializer->mLocalLog.log("%s with effect Id %p", __func__, effect.get());
283 }
284 }
285
286 return spatializer;
287 }
288
Spatializer(effect_descriptor_t engineDescriptor,SpatializerPolicyCallback * callback)289 Spatializer::Spatializer(effect_descriptor_t engineDescriptor, SpatializerPolicyCallback* callback)
290 : mEngineDescriptor(engineDescriptor),
291 mPolicyCallback(callback) {
292 ALOGV("%s", __func__);
293 setMinSchedulerPolicy(SCHED_NORMAL, ANDROID_PRIORITY_AUDIO);
294 setInheritRt(true);
295 }
296
onFirstRef()297 void Spatializer::onFirstRef() {
298 mLooper = new ALooper;
299 mLooper->setName("Spatializer-looper");
300 mLooper->start(
301 /*runOnCallingThread*/false,
302 /*canCallJava*/ false,
303 PRIORITY_URGENT_AUDIO);
304
305 mHandler = new EngineCallbackHandler(this);
306 mLooper->registerHandler(mHandler);
307 }
308
~Spatializer()309 Spatializer::~Spatializer() {
310 ALOGV("%s", __func__);
311 if (mLooper != nullptr) {
312 mLooper->stop();
313 mLooper->unregisterHandler(mHandler->id());
314 }
315 mLooper.clear();
316 mHandler.clear();
317 }
318
channelMaskVectorToString(const std::vector<audio_channel_mask_t> & masks)319 static std::string channelMaskVectorToString(
320 const std::vector<audio_channel_mask_t>& masks) {
321 std::stringstream ss;
322 for (const auto &mask : masks) {
323 if (ss.tellp() != 0) ss << "|";
324 ss << mask;
325 }
326 return ss.str();
327 }
328
loadEngineConfiguration(sp<EffectHalInterface> effect)329 status_t Spatializer::loadEngineConfiguration(sp<EffectHalInterface> effect) {
330 ALOGV("%s", __func__);
331
332 std::vector<bool> supportsHeadTracking;
333 status_t status = getHalParameter<false>(effect, SPATIALIZER_PARAM_HEADTRACKING_SUPPORTED,
334 &supportsHeadTracking);
335 if (status != NO_ERROR) {
336 ALOGW("%s: cannot get SPATIALIZER_PARAM_HEADTRACKING_SUPPORTED", __func__);
337 return status;
338 }
339 mSupportsHeadTracking = supportsHeadTracking[0];
340
341 std::vector<Spatialization::Level> spatializationLevels;
342 status = getHalParameter<true>(effect, SPATIALIZER_PARAM_SUPPORTED_LEVELS,
343 &spatializationLevels);
344 if (status != NO_ERROR) {
345 ALOGW("%s: cannot get SPATIALIZER_PARAM_SUPPORTED_LEVELS", __func__);
346 return status;
347 }
348 bool noneLevelFound = false;
349 bool activeLevelFound = false;
350 for (const auto spatializationLevel : spatializationLevels) {
351 if (!aidl_utils::isValidEnum(spatializationLevel)) {
352 ALOGW("%s: ignoring spatializationLevel:%s", __func__,
353 ToString(spatializationLevel).c_str());
354 continue;
355 }
356 if (spatializationLevel == Spatialization::Level::NONE) {
357 noneLevelFound = true;
358 } else {
359 activeLevelFound = true;
360 }
361 // we don't detect duplicates.
362 mLevels.emplace_back(spatializationLevel);
363 }
364 if (!noneLevelFound || !activeLevelFound) {
365 ALOGW("%s: SPATIALIZER_PARAM_SUPPORTED_LEVELS must include NONE"
366 " and another valid level", __func__);
367 return BAD_VALUE;
368 }
369
370 std::vector<Spatialization::Mode> spatializationModes;
371 status = getHalParameter<true>(effect, SPATIALIZER_PARAM_SUPPORTED_SPATIALIZATION_MODES,
372 &spatializationModes);
373 if (status != NO_ERROR) {
374 ALOGW("%s: cannot get SPATIALIZER_PARAM_SUPPORTED_SPATIALIZATION_MODES", __func__);
375 return status;
376 }
377
378 for (const auto spatializationMode : spatializationModes) {
379 if (!aidl_utils::isValidEnum(spatializationMode)) {
380 ALOGW("%s: ignoring spatializationMode:%s", __func__,
381 ToString(spatializationMode).c_str());
382 continue;
383 }
384 // we don't detect duplicates.
385 mSpatializationModes.emplace_back(spatializationMode);
386 }
387 if (mSpatializationModes.empty()) {
388 ALOGW("%s: SPATIALIZER_PARAM_SUPPORTED_SPATIALIZATION_MODES reports empty", __func__);
389 return BAD_VALUE;
390 }
391
392 std::vector<audio_channel_mask_t> channelMasks;
393 status = getHalParameter<true>(effect, SPATIALIZER_PARAM_SUPPORTED_CHANNEL_MASKS,
394 &channelMasks);
395 if (status != NO_ERROR) {
396 ALOGW("%s: cannot get SPATIALIZER_PARAM_SUPPORTED_CHANNEL_MASKS", __func__);
397 return status;
398 }
399 for (const auto channelMask : channelMasks) {
400 const bool channel_mask_spatialized =
401 SpatializerHelper::isStereoSpatializationFeatureEnabled()
402 ? audio_channel_mask_contains_stereo(channelMask)
403 : audio_is_channel_mask_spatialized(channelMask);
404 if (!channel_mask_spatialized) {
405 ALOGW("%s: ignoring channelMask:%#x", __func__, channelMask);
406 continue;
407 }
408 // we don't detect duplicates.
409 mChannelMasks.emplace_back(channelMask);
410 }
411 if (mChannelMasks.empty()) {
412 ALOGW("%s: SPATIALIZER_PARAM_SUPPORTED_CHANNEL_MASKS reports empty", __func__);
413 return BAD_VALUE;
414 }
415
416 std::vector<audio_channel_mask_t> spatializedChannelMasks;
417 status = getHalParameter<true>(effect, SPATIALIZER_PARAM_SPATIALIZED_CHANNEL_MASKS,
418 &spatializedChannelMasks);
419 if (status != NO_ERROR) {
420 ALOGW("%s: cannot get SPATIALIZER_PARAM_SPATIALIZED_CHANNEL_MASKS", __func__);
421 // do not return an error yet as spatializer implementations may not have been
422 // updated yet to support this parameter
423 }
424 if (spatializedChannelMasks.empty()) {
425 ALOGW("%s: SPATIALIZER_PARAM_SPATIALIZED_CHANNEL_MASKS reports empty", __func__);
426 // do not return an error yet as spatializer implementations may not have been
427 // updated yet to support this parameter
428 }
429 for (const audio_channel_mask_t spatializedMask : spatializedChannelMasks) {
430 // spatialized masks must be contained in the supported input masks
431 if (std::find(mChannelMasks.begin(), mChannelMasks.end(), spatializedMask)
432 != mChannelMasks.end()) {
433 mSpatializedChannelMasks.emplace_back(spatializedMask);
434 } else {
435 ALOGE("%s: spatialized mask %#x not in list reported by PARAM_SUPPORTED_CHANNEL_MASKS",
436 __func__, spatializedMask);
437 return BAD_VALUE;
438 }
439 }
440
441 if (com::android::media::audio::dsa_over_bt_le_audio()
442 && mSupportsHeadTracking) {
443 mHeadtrackingConnectionMode = HeadTracking::ConnectionMode::FRAMEWORK_PROCESSED;
444 std::vector<HeadTracking::ConnectionMode> headtrackingConnectionModes;
445 status = getHalParameter<true>(effect, SPATIALIZER_PARAM_SUPPORTED_HEADTRACKING_CONNECTION,
446 &headtrackingConnectionModes);
447 if (status == NO_ERROR) {
448 for (const auto htConnectionMode : headtrackingConnectionModes) {
449 if (htConnectionMode < HeadTracking::ConnectionMode::FRAMEWORK_PROCESSED ||
450 htConnectionMode > HeadTracking::ConnectionMode::DIRECT_TO_SENSOR_TUNNEL) {
451 ALOGW("%s: ignoring HT connection mode:%s", __func__,
452 ToString(htConnectionMode).c_str());
453 continue;
454 }
455 mSupportedHeadtrackingConnectionModes.insert(htConnectionMode);
456 }
457 ALOGW_IF(mSupportedHeadtrackingConnectionModes.find(
458 HeadTracking::ConnectionMode::FRAMEWORK_PROCESSED) ==
459 mSupportedHeadtrackingConnectionModes.end(),
460 "%s: Headtracking FRAMEWORK_PROCESSED not reported", __func__);
461 }
462 }
463
464 // Currently we expose only RELATIVE_WORLD.
465 // This is a limitation of the head tracking library based on a UX choice.
466 mHeadTrackingModes.push_back(HeadTracking::Mode::DISABLED);
467 if (mSupportsHeadTracking) {
468 mHeadTrackingModes.push_back(HeadTracking::Mode::RELATIVE_WORLD);
469 }
470 mediametrics::LogItem(mMetricsId)
471 .set(AMEDIAMETRICS_PROP_EVENT, AMEDIAMETRICS_PROP_EVENT_VALUE_CREATE)
472 .set(AMEDIAMETRICS_PROP_CHANNELMASKS, channelMaskVectorToString(mChannelMasks))
473 .set(AMEDIAMETRICS_PROP_LEVELS, aidl_utils::enumsToString(mLevels))
474 .set(AMEDIAMETRICS_PROP_MODES, aidl_utils::enumsToString(mSpatializationModes))
475 .set(AMEDIAMETRICS_PROP_HEADTRACKINGMODES, aidl_utils::enumsToString(mHeadTrackingModes))
476 .set(AMEDIAMETRICS_PROP_STATUS, (int32_t)status)
477 .record();
478 return NO_ERROR;
479 }
480
481 /* static */
sendEmptyCreateSpatializerMetricWithStatus(status_t status)482 void Spatializer::sendEmptyCreateSpatializerMetricWithStatus(status_t status) {
483 mediametrics::LogItem(kDefaultMetricsId)
484 .set(AMEDIAMETRICS_PROP_EVENT, AMEDIAMETRICS_PROP_EVENT_VALUE_CREATE)
485 .set(AMEDIAMETRICS_PROP_CHANNELMASKS, "")
486 .set(AMEDIAMETRICS_PROP_LEVELS, "")
487 .set(AMEDIAMETRICS_PROP_MODES, "")
488 .set(AMEDIAMETRICS_PROP_HEADTRACKINGMODES, "")
489 .set(AMEDIAMETRICS_PROP_STATUS, (int32_t)status)
490 .record();
491 }
492
493 /** Gets the channel mask, sampling rate and format set for the spatializer input. */
getAudioInConfig() const494 audio_config_base_t Spatializer::getAudioInConfig() const {
495 audio_utils::lock_guard lock(mMutex);
496 audio_config_base_t config = AUDIO_CONFIG_BASE_INITIALIZER;
497 // For now use highest supported channel count
498 config.channel_mask = getMaxChannelMask(mChannelMasks, FCC_LIMIT);
499 return config;
500 }
501
getSpatializedChannelMasks(std::vector<int> * masks)502 Status Spatializer::getSpatializedChannelMasks(std::vector<int>* masks) {
503 audio_utils::lock_guard lock(mMutex);
504 ALOGV("%s", __func__);
505 if (masks == nullptr) {
506 return binderStatusFromStatusT(BAD_VALUE);
507 }
508 masks->insert(masks->end(), mSpatializedChannelMasks.begin(),
509 mSpatializedChannelMasks.end());
510 return Status::ok();
511 }
512
registerCallback(const sp<media::INativeSpatializerCallback> & callback)513 status_t Spatializer::registerCallback(
514 const sp<media::INativeSpatializerCallback>& callback) {
515 audio_utils::lock_guard lock(mMutex);
516 if (callback == nullptr) {
517 return BAD_VALUE;
518 }
519
520 if (mSpatializerCallback != nullptr) {
521 if (IInterface::asBinder(callback) == IInterface::asBinder(mSpatializerCallback)) {
522 ALOGW("%s: Registering callback %p again",
523 __func__, mSpatializerCallback.get());
524 return NO_ERROR;
525 }
526 ALOGE("%s: Already one client registered with callback %p",
527 __func__, mSpatializerCallback.get());
528 return INVALID_OPERATION;
529 }
530
531 sp<IBinder> binder = IInterface::asBinder(callback);
532 status_t status = binder->linkToDeath(this);
533 if (status == NO_ERROR) {
534 mSpatializerCallback = callback;
535 }
536 ALOGV("%s status %d", __func__, status);
537 return status;
538 }
539
540 // IBinder::DeathRecipient
binderDied(__unused const wp<IBinder> & who)541 void Spatializer::binderDied(__unused const wp<IBinder> &who) {
542 {
543 audio_utils::lock_guard lock(mMutex);
544 mLevel = Spatialization::Level::NONE;
545 mSpatializerCallback.clear();
546 }
547 ALOGV("%s", __func__);
548 mPolicyCallback->onCheckSpatializer();
549 }
550
551 // ISpatializer
getSupportedLevels(std::vector<Spatialization::Level> * levels)552 Status Spatializer::getSupportedLevels(std::vector<Spatialization::Level> *levels) {
553 ALOGV("%s", __func__);
554 if (levels == nullptr) {
555 return binderStatusFromStatusT(BAD_VALUE);
556 }
557 // Spatialization::Level::NONE is already required from the effect or we don't load it.
558 levels->insert(levels->end(), mLevels.begin(), mLevels.end());
559 return Status::ok();
560 }
561
setLevel(Spatialization::Level level)562 Status Spatializer::setLevel(Spatialization::Level level) {
563 ALOGV("%s level %s", __func__, ToString(level).c_str());
564 mLocalLog.log("%s with %s", __func__, ToString(level).c_str());
565 if (level != Spatialization::Level::NONE
566 && std::find(mLevels.begin(), mLevels.end(), level) == mLevels.end()) {
567 return binderStatusFromStatusT(BAD_VALUE);
568 }
569 sp<media::INativeSpatializerCallback> callback;
570 bool levelChanged = false;
571 {
572 audio_utils::lock_guard lock(mMutex);
573 levelChanged = mLevel != level;
574 mLevel = level;
575 callback = mSpatializerCallback;
576
577 if (levelChanged && mEngine != nullptr) {
578 checkEngineState_l();
579 }
580 checkSensorsState_l();
581 }
582
583 if (levelChanged) {
584 mPolicyCallback->onCheckSpatializer();
585 if (callback != nullptr) {
586 callback->onLevelChanged(level);
587 }
588 }
589 return Status::ok();
590 }
591
getLevel(Spatialization::Level * level)592 Status Spatializer::getLevel(Spatialization::Level *level) {
593 if (level == nullptr) {
594 return binderStatusFromStatusT(BAD_VALUE);
595 }
596 audio_utils::lock_guard lock(mMutex);
597 *level = mLevel;
598 ALOGV("%s level %s", __func__, ToString(*level).c_str());
599 return Status::ok();
600 }
601
isHeadTrackingSupported(bool * supports)602 Status Spatializer::isHeadTrackingSupported(bool *supports) {
603 ALOGV("%s mSupportsHeadTracking %s", __func__, ToString(mSupportsHeadTracking).c_str());
604 if (supports == nullptr) {
605 return binderStatusFromStatusT(BAD_VALUE);
606 }
607 audio_utils::lock_guard lock(mMutex);
608 *supports = mSupportsHeadTracking;
609 return Status::ok();
610 }
611
getSupportedHeadTrackingModes(std::vector<HeadTracking::Mode> * modes)612 Status Spatializer::getSupportedHeadTrackingModes(
613 std::vector<HeadTracking::Mode>* modes) {
614 audio_utils::lock_guard lock(mMutex);
615 ALOGV("%s", __func__);
616 if (modes == nullptr) {
617 return binderStatusFromStatusT(BAD_VALUE);
618 }
619 modes->insert(modes->end(), mHeadTrackingModes.begin(), mHeadTrackingModes.end());
620 return Status::ok();
621 }
622
setDesiredHeadTrackingMode(HeadTracking::Mode mode)623 Status Spatializer::setDesiredHeadTrackingMode(HeadTracking::Mode mode) {
624 ALOGV("%s mode %s", __func__, ToString(mode).c_str());
625
626 if (!mSupportsHeadTracking) {
627 return binderStatusFromStatusT(INVALID_OPERATION);
628 }
629 mLocalLog.log("%s with %s", __func__, ToString(mode).c_str());
630 audio_utils::lock_guard lock(mMutex);
631 switch (mode) {
632 case HeadTracking::Mode::OTHER:
633 return binderStatusFromStatusT(BAD_VALUE);
634 case HeadTracking::Mode::DISABLED:
635 mDesiredHeadTrackingMode = HeadTrackingMode::STATIC;
636 break;
637 case HeadTracking::Mode::RELATIVE_WORLD:
638 mDesiredHeadTrackingMode = HeadTrackingMode::WORLD_RELATIVE;
639 break;
640 case HeadTracking::Mode::RELATIVE_SCREEN:
641 mDesiredHeadTrackingMode = HeadTrackingMode::SCREEN_RELATIVE;
642 break;
643 }
644
645 checkPoseController_l();
646 checkSensorsState_l();
647
648 return Status::ok();
649 }
650
getActualHeadTrackingMode(HeadTracking::Mode * mode)651 Status Spatializer::getActualHeadTrackingMode(HeadTracking::Mode *mode) {
652 if (mode == nullptr) {
653 return binderStatusFromStatusT(BAD_VALUE);
654 }
655 audio_utils::lock_guard lock(mMutex);
656 *mode = mActualHeadTrackingMode;
657 ALOGV("%s mode %d", __func__, (int)*mode);
658 return Status::ok();
659 }
660
recenterHeadTracker()661 Status Spatializer::recenterHeadTracker() {
662 if (!mSupportsHeadTracking) {
663 return binderStatusFromStatusT(INVALID_OPERATION);
664 }
665 audio_utils::lock_guard lock(mMutex);
666 if (mPoseController != nullptr) {
667 mPoseController->recenter();
668 }
669 return Status::ok();
670 }
671
setGlobalTransform(const std::vector<float> & screenToStage)672 Status Spatializer::setGlobalTransform(const std::vector<float>& screenToStage) {
673 ALOGV("%s", __func__);
674 if (!mSupportsHeadTracking) {
675 return binderStatusFromStatusT(INVALID_OPERATION);
676 }
677 std::optional<Pose3f> maybePose = Pose3f::fromVector(screenToStage);
678 if (!maybePose.has_value()) {
679 ALOGW("Invalid screenToStage vector.");
680 return binderStatusFromStatusT(BAD_VALUE);
681 }
682 audio_utils::lock_guard lock(mMutex);
683 if (mPoseController != nullptr) {
684 mLocalLog.log("%s with screenToStage %s", __func__,
685 media::VectorRecorder::toString<float>(screenToStage).c_str());
686 mPoseController->setScreenToStagePose(maybePose.value());
687 }
688 return Status::ok();
689 }
690
release()691 Status Spatializer::release() {
692 ALOGV("%s", __func__);
693 bool levelChanged = false;
694 {
695 audio_utils::lock_guard lock(mMutex);
696 if (mSpatializerCallback == nullptr) {
697 return binderStatusFromStatusT(INVALID_OPERATION);
698 }
699
700 sp<IBinder> binder = IInterface::asBinder(mSpatializerCallback);
701 binder->unlinkToDeath(this);
702 mSpatializerCallback.clear();
703
704 levelChanged = mLevel != Spatialization::Level::NONE;
705 mLevel = Spatialization::Level::NONE;
706 }
707
708 if (levelChanged) {
709 mPolicyCallback->onCheckSpatializer();
710 }
711 return Status::ok();
712 }
713
setHeadSensor(int sensorHandle)714 Status Spatializer::setHeadSensor(int sensorHandle) {
715 ALOGV("%s sensorHandle %d", __func__, sensorHandle);
716 if (!mSupportsHeadTracking) {
717 return binderStatusFromStatusT(INVALID_OPERATION);
718 }
719 audio_utils::lock_guard lock(mMutex);
720 if (mHeadSensor != sensorHandle) {
721 mLocalLog.log("%s with 0x%08x", __func__, sensorHandle);
722 mHeadSensor = sensorHandle;
723 checkPoseController_l();
724 checkSensorsState_l();
725 }
726 return Status::ok();
727 }
728
setScreenSensor(int sensorHandle)729 Status Spatializer::setScreenSensor(int sensorHandle) {
730 ALOGV("%s sensorHandle %d", __func__, sensorHandle);
731 if (!mSupportsHeadTracking) {
732 return binderStatusFromStatusT(INVALID_OPERATION);
733 }
734 audio_utils::lock_guard lock(mMutex);
735 if (mScreenSensor != sensorHandle) {
736 mLocalLog.log("%s with 0x%08x", __func__, sensorHandle);
737 mScreenSensor = sensorHandle;
738 // TODO: consider a new method setHeadAndScreenSensor()
739 // because we generally set both at the same time.
740 // This will avoid duplicated work and recentering.
741 checkSensorsState_l();
742 }
743 return Status::ok();
744 }
745
setDisplayOrientation(float physicalToLogicalAngle)746 Status Spatializer::setDisplayOrientation(float physicalToLogicalAngle) {
747 ALOGV("%s physicalToLogicalAngle %f", __func__, physicalToLogicalAngle);
748 mLocalLog.log("%s with %f", __func__, physicalToLogicalAngle);
749 const float angle = safe_clamp(physicalToLogicalAngle, 0.f, (float)(2. * M_PI));
750 // It is possible due to numerical inaccuracies to exceed the boundaries of 0 to 2 * M_PI.
751 ALOGI_IF(angle != physicalToLogicalAngle,
752 "%s: clamping %f to %f", __func__, physicalToLogicalAngle, angle);
753 audio_utils::lock_guard lock(mMutex);
754 mDisplayOrientation = angle;
755 if (mPoseController != nullptr) {
756 // This turns on the rate-limiter.
757 mPoseController->setDisplayOrientation(angle);
758 }
759 if (mEngine != nullptr) {
760 setEffectParameter_l(
761 SPATIALIZER_PARAM_DISPLAY_ORIENTATION, std::vector<float>{angle});
762 }
763 return Status::ok();
764 }
765
setHingeAngle(float hingeAngle)766 Status Spatializer::setHingeAngle(float hingeAngle) {
767 ALOGV("%s hingeAngle %f", __func__, hingeAngle);
768 mLocalLog.log("%s with %f", __func__, hingeAngle);
769 const float angle = safe_clamp(hingeAngle, 0.f, (float)(2. * M_PI));
770 // It is possible due to numerical inaccuracies to exceed the boundaries of 0 to 2 * M_PI.
771 ALOGI_IF(angle != hingeAngle,
772 "%s: clamping %f to %f", __func__, hingeAngle, angle);
773 audio_utils::lock_guard lock(mMutex);
774 mHingeAngle = angle;
775 if (mEngine != nullptr) {
776 setEffectParameter_l(SPATIALIZER_PARAM_HINGE_ANGLE, std::vector<float>{angle});
777 }
778 return Status::ok();
779 }
780
setFoldState(bool folded)781 Status Spatializer::setFoldState(bool folded) {
782 ALOGV("%s foldState %d", __func__, (int)folded);
783 mLocalLog.log("%s with %d", __func__, (int)folded);
784 audio_utils::lock_guard lock(mMutex);
785 mFoldedState = folded;
786 if (mEngine != nullptr) {
787 // we don't suppress multiple calls with the same folded state - that's
788 // done at the caller.
789 setEffectParameter_l(SPATIALIZER_PARAM_FOLD_STATE, std::vector<uint8_t>{mFoldedState});
790 }
791 return Status::ok();
792 }
793
getSupportedModes(std::vector<Spatialization::Mode> * modes)794 Status Spatializer::getSupportedModes(std::vector<Spatialization::Mode> *modes) {
795 ALOGV("%s", __func__);
796 if (modes == nullptr) {
797 return binderStatusFromStatusT(BAD_VALUE);
798 }
799 *modes = mSpatializationModes;
800 return Status::ok();
801 }
802
registerHeadTrackingCallback(const sp<media::ISpatializerHeadTrackingCallback> & callback)803 Status Spatializer::registerHeadTrackingCallback(
804 const sp<media::ISpatializerHeadTrackingCallback>& callback) {
805 ALOGV("%s callback %p", __func__, callback.get());
806 audio_utils::lock_guard lock(mMutex);
807 if (!mSupportsHeadTracking) {
808 return binderStatusFromStatusT(INVALID_OPERATION);
809 }
810 mHeadTrackingCallback = callback;
811 return Status::ok();
812 }
813
setParameter(int key,const std::vector<unsigned char> & value)814 Status Spatializer::setParameter(int key, const std::vector<unsigned char>& value) {
815 ALOGV("%s key %d", __func__, key);
816 audio_utils::lock_guard lock(mMutex);
817 status_t status = INVALID_OPERATION;
818 if (mEngine != nullptr) {
819 status = setEffectParameter_l(key, value);
820 }
821 return binderStatusFromStatusT(status);
822 }
823
getParameter(int key,std::vector<unsigned char> * value)824 Status Spatializer::getParameter(int key, std::vector<unsigned char> *value) {
825 ALOGV("%s key %d value size %d", __func__, key,
826 (value != nullptr ? (int)value->size() : -1));
827 if (value == nullptr) {
828 return binderStatusFromStatusT(BAD_VALUE);
829 }
830 audio_utils::lock_guard lock(mMutex);
831 status_t status = INVALID_OPERATION;
832 if (mEngine != nullptr) {
833 ALOGV("%s key %d mEngine %p", __func__, key, mEngine.get());
834 status = getEffectParameter_l(key, value);
835 }
836 return binderStatusFromStatusT(status);
837 }
838
getOutput(int * output)839 Status Spatializer::getOutput(int *output) {
840 ALOGV("%s", __func__);
841 if (output == nullptr) {
842 binderStatusFromStatusT(BAD_VALUE);
843 }
844 audio_utils::lock_guard lock(mMutex);
845 *output = VALUE_OR_RETURN_BINDER_STATUS(legacy2aidl_audio_io_handle_t_int32_t(mOutput));
846 ALOGV("%s got output %d", __func__, *output);
847 return Status::ok();
848 }
849
850 // SpatializerPoseController::Listener
onHeadToStagePose(const Pose3f & headToStage)851 void Spatializer::onHeadToStagePose(const Pose3f& headToStage) {
852 ALOGV("%s", __func__);
853 LOG_ALWAYS_FATAL_IF(!mSupportsHeadTracking,
854 "onHeadToStagePose() called with no head tracking support!");
855
856 auto vec = headToStage.toVector();
857 LOG_ALWAYS_FATAL_IF(vec.size() != sHeadPoseKeys.size(),
858 "%s invalid head to stage vector size %zu", __func__, vec.size());
859 sp<AMessage> msg =
860 new AMessage(EngineCallbackHandler::kWhatOnHeadToStagePose, mHandler);
861 for (size_t i = 0 ; i < sHeadPoseKeys.size(); i++) {
862 msg->setFloat(sHeadPoseKeys[i], vec[i]);
863 }
864 msg->post();
865 }
866
resetEngineHeadPose_l()867 void Spatializer::resetEngineHeadPose_l() {
868 ALOGV("%s mEngine %p", __func__, mEngine.get());
869 if (mEngine == nullptr) {
870 return;
871 }
872 const std::vector<float> headToStage(6, 0.0);
873 setEffectParameter_l(SPATIALIZER_PARAM_HEAD_TO_STAGE, headToStage);
874 setEffectParameter_l(SPATIALIZER_PARAM_HEADTRACKING_MODE,
875 std::vector<HeadTracking::Mode>{HeadTracking::Mode::DISABLED});
876 }
877
onHeadToStagePoseMsg(const std::vector<float> & headToStage)878 void Spatializer::onHeadToStagePoseMsg(const std::vector<float>& headToStage) {
879 ALOGV("%s", __func__);
880 sp<media::ISpatializerHeadTrackingCallback> callback;
881 {
882 audio_utils::lock_guard lock(mMutex);
883 callback = mHeadTrackingCallback;
884 if (mEngine != nullptr) {
885 setEffectParameter_l(SPATIALIZER_PARAM_HEAD_TO_STAGE, headToStage);
886 const auto record = recordFromTranslationRotationVector(headToStage);
887 mPoseRecorder.record(record);
888 mPoseDurableRecorder.record(record);
889 }
890 }
891
892 if (callback != nullptr) {
893 callback->onHeadToSoundStagePoseUpdated(headToStage);
894 }
895 }
896
onActualModeChange(HeadTrackingMode mode)897 void Spatializer::onActualModeChange(HeadTrackingMode mode) {
898 std::string modeStr = ToString(mode);
899 ALOGV("%s(%s)", __func__, modeStr.c_str());
900 sp<AMessage> msg = new AMessage(EngineCallbackHandler::kWhatOnActualModeChange, mHandler);
901 msg->setInt32(EngineCallbackHandler::kModeKey, static_cast<int>(mode));
902 msg->post();
903 }
904
onActualModeChangeMsg(HeadTrackingMode mode)905 void Spatializer::onActualModeChangeMsg(HeadTrackingMode mode) {
906 ALOGV("%s(%s)", __func__, ToString(mode).c_str());
907 sp<media::ISpatializerHeadTrackingCallback> callback;
908 HeadTracking::Mode spatializerMode;
909 {
910 audio_utils::lock_guard lock(mMutex);
911 if (!mSupportsHeadTracking) {
912 spatializerMode = HeadTracking::Mode::DISABLED;
913 } else {
914 switch (mode) {
915 case HeadTrackingMode::STATIC:
916 spatializerMode = HeadTracking::Mode::DISABLED;
917 break;
918 case HeadTrackingMode::WORLD_RELATIVE:
919 spatializerMode = HeadTracking::Mode::RELATIVE_WORLD;
920 break;
921 case HeadTrackingMode::SCREEN_RELATIVE:
922 spatializerMode = HeadTracking::Mode::RELATIVE_SCREEN;
923 break;
924 default:
925 LOG_ALWAYS_FATAL("Unknown mode: %s", ToString(mode).c_str());
926 }
927 }
928 mActualHeadTrackingMode = spatializerMode;
929 if (mEngine != nullptr) {
930 if (spatializerMode == HeadTracking::Mode::DISABLED) {
931 resetEngineHeadPose_l();
932 } else {
933 setEffectParameter_l(SPATIALIZER_PARAM_HEADTRACKING_MODE,
934 std::vector<HeadTracking::Mode>{spatializerMode});
935 setEngineHeadtrackingConnectionMode_l();
936 }
937 }
938 callback = mHeadTrackingCallback;
939 mLocalLog.log("%s: updating mode to %s", __func__, ToString(mode).c_str());
940 }
941 if (callback != nullptr) {
942 callback->onHeadTrackingModeChanged(spatializerMode);
943 }
944 }
945
setEngineHeadtrackingConnectionMode_l()946 void Spatializer::setEngineHeadtrackingConnectionMode_l() {
947 if (!com::android::media::audio::dsa_over_bt_le_audio()) {
948 return;
949 }
950 if (mActualHeadTrackingMode != HeadTracking::Mode::DISABLED
951 && !mSupportedHeadtrackingConnectionModes.empty()) {
952 setEffectParameter_l(SPATIALIZER_PARAM_HEADTRACKING_CONNECTION,
953 static_cast<uint8_t>(mHeadtrackingConnectionMode),
954 static_cast<uint32_t>(mHeadSensor));
955 }
956 }
957
sortSupportedLatencyModes_l()958 void Spatializer::sortSupportedLatencyModes_l() {
959 if (!com::android::media::audio::dsa_over_bt_le_audio()) {
960 return;
961 }
962 std::sort(mSupportedLatencyModes.begin(), mSupportedLatencyModes.end(),
963 [this](audio_latency_mode_t x, audio_latency_mode_t y) {
964 auto itX = std::find(mOrderedLowLatencyModes.begin(),
965 mOrderedLowLatencyModes.end(), x);
966 auto itY = std::find(mOrderedLowLatencyModes.begin(),
967 mOrderedLowLatencyModes.end(), y);
968 return itX < itY;
969 });
970 }
971
attachOutput(audio_io_handle_t output,const std::vector<audio_channel_mask_t> & activeTracksMasks)972 status_t Spatializer::attachOutput(audio_io_handle_t output,
973 const std::vector<audio_channel_mask_t>& activeTracksMasks) {
974 bool outputChanged = false;
975 sp<media::INativeSpatializerCallback> callback;
976
977 {
978 audio_utils::lock_guard lock(mMutex);
979 ALOGV("%s output %d mOutput %d", __func__, (int)output, (int)mOutput);
980 mLocalLog.log("%s with output %d tracks %zu (mOutput %d)", __func__, (int)output,
981 activeTracksMasks.size(), (int)mOutput);
982 if (mOutput != AUDIO_IO_HANDLE_NONE) {
983 LOG_ALWAYS_FATAL_IF(mEngine == nullptr, "%s output set without FX engine", __func__);
984 // remove FX instance
985 mEngine->setEnabled(false);
986 mEngine.clear();
987 mPoseController.reset();
988 AudioSystem::removeSupportedLatencyModesCallback(this);
989 }
990
991 // create FX instance on output
992 AttributionSourceState attributionSource = AttributionSourceState();
993 mEngine = new AudioEffect(attributionSource);
994 mEngine->set(nullptr /* type */, &mEngineDescriptor.uuid, 0 /* priority */,
995 wp<AudioEffect::IAudioEffectCallback>::fromExisting(this),
996 AUDIO_SESSION_OUTPUT_STAGE, output, {} /* device */, false /* probe */,
997 true /* notifyFramesProcessed */);
998 status_t status = mEngine->initCheck();
999 ALOGV("%s mEngine create status %d", __func__, (int)status);
1000 if (status != NO_ERROR) {
1001 return status;
1002 }
1003
1004 outputChanged = mOutput != output;
1005 mOutput = output;
1006 mActiveTracksMasks = activeTracksMasks;
1007 AudioSystem::addSupportedLatencyModesCallback(this);
1008
1009 std::vector<audio_latency_mode_t> latencyModes;
1010 status = AudioSystem::getSupportedLatencyModes(mOutput, &latencyModes);
1011 if (status == OK) {
1012 mSupportedLatencyModes = latencyModes;
1013 sortSupportedLatencyModes_l();
1014 }
1015
1016 checkEngineState_l();
1017 if (mSupportsHeadTracking) {
1018 checkPoseController_l();
1019 checkSensorsState_l();
1020 }
1021 callback = mSpatializerCallback;
1022
1023 // Restore common effect state.
1024 setEffectParameter_l(SPATIALIZER_PARAM_DISPLAY_ORIENTATION,
1025 std::vector<float>{mDisplayOrientation});
1026 setEffectParameter_l(SPATIALIZER_PARAM_FOLD_STATE,
1027 std::vector<uint8_t>{mFoldedState});
1028 setEffectParameter_l(SPATIALIZER_PARAM_HINGE_ANGLE,
1029 std::vector<float>{mHingeAngle});
1030 }
1031
1032 if (outputChanged && callback != nullptr) {
1033 callback->onOutputChanged(output);
1034 }
1035
1036 return NO_ERROR;
1037 }
1038
detachOutput()1039 audio_io_handle_t Spatializer::detachOutput() {
1040 audio_io_handle_t output = AUDIO_IO_HANDLE_NONE;
1041 sp<media::INativeSpatializerCallback> callback;
1042
1043 {
1044 audio_utils::lock_guard lock(mMutex);
1045 mLocalLog.log("%s with output %d num tracks %zu",
1046 __func__, (int)mOutput, mActiveTracksMasks.size());
1047 ALOGV("%s mOutput %d", __func__, (int)mOutput);
1048 if (mOutput == AUDIO_IO_HANDLE_NONE) {
1049 return output;
1050 }
1051 // remove FX instance
1052 mEngine->setEnabled(false);
1053 mEngine.clear();
1054 AudioSystem::removeSupportedLatencyModesCallback(this);
1055 output = mOutput;
1056 mOutput = AUDIO_IO_HANDLE_NONE;
1057 mPoseController.reset();
1058 callback = mSpatializerCallback;
1059 }
1060
1061 if (callback != nullptr) {
1062 callback->onOutputChanged(AUDIO_IO_HANDLE_NONE);
1063 }
1064 return output;
1065 }
1066
onSupportedLatencyModesChanged(audio_io_handle_t output,const std::vector<audio_latency_mode_t> & modes)1067 void Spatializer::onSupportedLatencyModesChanged(
1068 audio_io_handle_t output, const std::vector<audio_latency_mode_t>& modes) {
1069 ALOGV("%s output %d num modes %zu", __func__, (int)output, modes.size());
1070 sp<AMessage> msg =
1071 new AMessage(EngineCallbackHandler::kWhatOnLatencyModesChanged, mHandler);
1072 msg->setObject(EngineCallbackHandler::kLatencyModesKey,
1073 sp<EngineCallbackHandler::LatencyModes>::make(output, modes));
1074 msg->post();
1075 }
1076
onSupportedLatencyModesChangedMsg(audio_io_handle_t output,std::vector<audio_latency_mode_t> && modes)1077 void Spatializer::onSupportedLatencyModesChangedMsg(
1078 audio_io_handle_t output, std::vector<audio_latency_mode_t>&& modes) {
1079 audio_utils::lock_guard lock(mMutex);
1080 ALOGV("%s output %d mOutput %d num modes %zu",
1081 __func__, (int)output, (int)mOutput, modes.size());
1082 if (output == mOutput) {
1083 mSupportedLatencyModes = std::move(modes);
1084 sortSupportedLatencyModes_l();
1085 checkSensorsState_l();
1086 }
1087 }
1088
updateActiveTracks(const std::vector<audio_channel_mask_t> & activeTracksMasks)1089 void Spatializer::updateActiveTracks(
1090 const std::vector<audio_channel_mask_t>& activeTracksMasks) {
1091 audio_utils::lock_guard lock(mMutex);
1092 if (mActiveTracksMasks != activeTracksMasks) {
1093 mLocalLog.log("%s from %zu to %zu",
1094 __func__, mActiveTracksMasks.size(), activeTracksMasks.size());
1095 mActiveTracksMasks = activeTracksMasks;
1096 checkEngineState_l();
1097 checkSensorsState_l();
1098 }
1099 }
1100
selectHeadtrackingConnectionMode_l()1101 audio_latency_mode_t Spatializer::selectHeadtrackingConnectionMode_l() {
1102 if (!com::android::media::audio::dsa_over_bt_le_audio()) {
1103 return AUDIO_LATENCY_MODE_LOW;
1104 }
1105 // mSupportedLatencyModes is ordered according to system preferences loaded in
1106 // mOrderedLowLatencyModes
1107 mHeadtrackingConnectionMode = HeadTracking::ConnectionMode::FRAMEWORK_PROCESSED;
1108 audio_latency_mode_t requestedLatencyMode = mSupportedLatencyModes[0];
1109 if (requestedLatencyMode == AUDIO_LATENCY_MODE_DYNAMIC_SPATIAL_AUDIO_HARDWARE) {
1110 if (mSupportedHeadtrackingConnectionModes.find(
1111 HeadTracking::ConnectionMode::DIRECT_TO_SENSOR_TUNNEL)
1112 != mSupportedHeadtrackingConnectionModes.end()) {
1113 mHeadtrackingConnectionMode = HeadTracking::ConnectionMode::DIRECT_TO_SENSOR_TUNNEL;
1114 } else if (mSupportedHeadtrackingConnectionModes.find(
1115 HeadTracking::ConnectionMode::DIRECT_TO_SENSOR_SW)
1116 != mSupportedHeadtrackingConnectionModes.end()) {
1117 mHeadtrackingConnectionMode = HeadTracking::ConnectionMode::DIRECT_TO_SENSOR_SW;
1118 } else {
1119 // if the engine does not support direct reading of IMU data, do not allow
1120 // DYNAMIC_SPATIAL_AUDIO_HARDWARE mode and fallback to next mode
1121 if (mSupportedLatencyModes.size() > 1) {
1122 requestedLatencyMode = mSupportedLatencyModes[1];
1123 } else {
1124 // If only DYNAMIC_SPATIAL_AUDIO_HARDWARE mode is reported by the
1125 // HAL and the engine does not support it, assert as this is a
1126 // product configuration error
1127 LOG_ALWAYS_FATAL("%s: the audio HAL reported only low latency with"
1128 "HW HID tunneling but the spatializer does not support it",
1129 __func__);
1130 }
1131 }
1132 }
1133 return requestedLatencyMode;
1134 }
1135
checkSensorsState_l()1136 void Spatializer::checkSensorsState_l() {
1137 mRequestedLatencyMode = AUDIO_LATENCY_MODE_FREE;
1138 const bool supportsSetLatencyMode = !mSupportedLatencyModes.empty();
1139 bool supportsLowLatencyMode;
1140 if (com::android::media::audio::dsa_over_bt_le_audio()) {
1141 // mSupportedLatencyModes is ordered with MODE_FREE always at the end:
1142 // the first entry is never MODE_FREE if at least one low ltency mode is supported.
1143 supportsLowLatencyMode = supportsSetLatencyMode
1144 && mSupportedLatencyModes[0] != AUDIO_LATENCY_MODE_FREE;
1145 } else {
1146 supportsLowLatencyMode = supportsSetLatencyMode && std::find(
1147 mSupportedLatencyModes.begin(), mSupportedLatencyModes.end(),
1148 AUDIO_LATENCY_MODE_LOW) != mSupportedLatencyModes.end();
1149 }
1150 if (mSupportsHeadTracking) {
1151 if (mPoseController != nullptr) {
1152 // TODO(b/253297301, b/255433067) reenable low latency condition check
1153 // for Head Tracking after Bluetooth HAL supports it correctly.
1154 if (shouldUseHeadTracking_l() && mLevel != Spatialization::Level::NONE
1155 && mDesiredHeadTrackingMode != HeadTrackingMode::STATIC
1156 && mHeadSensor != SpatializerPoseController::INVALID_SENSOR) {
1157 if (supportsLowLatencyMode) {
1158 mRequestedLatencyMode = selectHeadtrackingConnectionMode_l();
1159 }
1160 if (mEngine != nullptr) {
1161 setEffectParameter_l(SPATIALIZER_PARAM_HEADTRACKING_MODE,
1162 std::vector<HeadTracking::Mode>{mActualHeadTrackingMode});
1163 setEngineHeadtrackingConnectionMode_l();
1164 }
1165 // TODO: b/307588546: configure mPoseController according to selected
1166 // mHeadtrackingConnectionMode
1167 mPoseController->setHeadSensor(mHeadSensor);
1168 mPoseController->setScreenSensor(mScreenSensor);
1169 } else {
1170 mPoseController->setHeadSensor(SpatializerPoseController::INVALID_SENSOR);
1171 mPoseController->setScreenSensor(SpatializerPoseController::INVALID_SENSOR);
1172 resetEngineHeadPose_l();
1173 }
1174 } else {
1175 resetEngineHeadPose_l();
1176 }
1177 }
1178 if (mOutput != AUDIO_IO_HANDLE_NONE && supportsSetLatencyMode) {
1179 const status_t status =
1180 AudioSystem::setRequestedLatencyMode(mOutput, mRequestedLatencyMode);
1181 ALOGD("%s: setRequestedLatencyMode for output thread(%d) to %s returned %d", __func__,
1182 mOutput, toString(mRequestedLatencyMode).c_str(), status);
1183 }
1184 }
1185
1186
1187 /* static */
containsImmersiveChannelMask(const std::vector<audio_channel_mask_t> & masks)1188 bool Spatializer::containsImmersiveChannelMask(
1189 const std::vector<audio_channel_mask_t>& masks)
1190 {
1191 for (auto mask : masks) {
1192 if (audio_is_channel_mask_spatialized(mask)) {
1193 return true;
1194 }
1195 }
1196 // Only non-immersive channel masks, e.g. AUDIO_CHANNEL_OUT_STEREO, are present.
1197 return false;
1198 }
1199
shouldUseHeadTracking_l() const1200 bool Spatializer::shouldUseHeadTracking_l() const {
1201 // Headtracking only available on immersive channel masks.
1202 return containsImmersiveChannelMask(mActiveTracksMasks);
1203 }
1204
checkEngineState_l()1205 void Spatializer::checkEngineState_l() {
1206 if (mEngine != nullptr) {
1207 if (mLevel != Spatialization::Level::NONE && mActiveTracksMasks.size() > 0) {
1208 mEngine->setEnabled(true);
1209 setEffectParameter_l(SPATIALIZER_PARAM_LEVEL,
1210 std::vector<Spatialization::Level>{mLevel});
1211 } else {
1212 setEffectParameter_l(SPATIALIZER_PARAM_LEVEL,
1213 std::vector<Spatialization::Level>{Spatialization::Level::NONE});
1214 mEngine->setEnabled(false);
1215 }
1216 }
1217 }
1218
checkPoseController_l()1219 void Spatializer::checkPoseController_l() {
1220 bool isControllerNeeded = mDesiredHeadTrackingMode != HeadTrackingMode::STATIC
1221 && mHeadSensor != SpatializerPoseController::INVALID_SENSOR;
1222
1223 if (isControllerNeeded && mPoseController == nullptr) {
1224 mPoseController = std::make_shared<SpatializerPoseController>(
1225 static_cast<SpatializerPoseController::Listener*>(this),
1226 10ms, std::nullopt);
1227 LOG_ALWAYS_FATAL_IF(mPoseController == nullptr,
1228 "%s could not allocate pose controller", __func__);
1229 mPoseController->setDisplayOrientation(mDisplayOrientation);
1230 } else if (!isControllerNeeded && mPoseController != nullptr) {
1231 mPoseController.reset();
1232 resetEngineHeadPose_l();
1233 }
1234 if (mPoseController != nullptr) {
1235 mPoseController->setDesiredMode(mDesiredHeadTrackingMode);
1236 }
1237 }
1238
calculateHeadPose()1239 void Spatializer::calculateHeadPose() {
1240 ALOGV("%s", __func__);
1241 audio_utils::lock_guard lock(mMutex);
1242 if (mPoseController != nullptr) {
1243 mPoseController->calculateAsync();
1244 }
1245 }
1246
onFramesProcessed(int32_t framesProcessed)1247 void Spatializer::onFramesProcessed(int32_t framesProcessed) {
1248 sp<AMessage> msg =
1249 new AMessage(EngineCallbackHandler::kWhatOnFramesProcessed, mHandler);
1250 msg->setInt32(EngineCallbackHandler::kNumFramesKey, framesProcessed);
1251 msg->post();
1252 }
1253
toString(unsigned level) const1254 std::string Spatializer::toString(unsigned level) const {
1255 std::string prefixSpace(level, ' ');
1256 std::string ss = prefixSpace + "Spatializer:\n";
1257 bool needUnlock = false;
1258
1259 prefixSpace += ' ';
1260 if (!mMutex.try_lock()) {
1261 // dumpsys even try_lock failed, information dump can be useful although may not accurate
1262 ss.append(prefixSpace).append("try_lock failed, dumpsys below maybe INACCURATE!\n");
1263 } else {
1264 needUnlock = true;
1265 }
1266
1267 // Spatializer class information.
1268 // 1. Capabilities (mLevels, mHeadTrackingModes, mSpatializationModes, mChannelMasks, etc)
1269 ss.append(prefixSpace).append("Supported levels: [");
1270 for (auto& level : mLevels) {
1271 base::StringAppendF(&ss, " %s", ToString(level).c_str());
1272 }
1273 base::StringAppendF(&ss, "], mLevel: %s", ToString(mLevel).c_str());
1274
1275 base::StringAppendF(&ss, "\n%smHeadTrackingModes: [", prefixSpace.c_str());
1276 for (auto& mode : mHeadTrackingModes) {
1277 base::StringAppendF(&ss, " %s", ToString(mode).c_str());
1278 }
1279 base::StringAppendF(&ss, "], Desired: %s, Actual %s\n",
1280 ToString(mDesiredHeadTrackingMode).c_str(),
1281 ToString(mActualHeadTrackingMode).c_str());
1282
1283 base::StringAppendF(&ss, "%smSpatializationModes: [", prefixSpace.c_str());
1284 for (auto& mode : mSpatializationModes) {
1285 base::StringAppendF(&ss, " %s", ToString(mode).c_str());
1286 }
1287 ss += "]\n";
1288
1289 base::StringAppendF(&ss, "%smChannelMasks: ", prefixSpace.c_str());
1290 for (auto& mask : mChannelMasks) {
1291 base::StringAppendF(&ss, "%s", audio_channel_out_mask_to_string(mask));
1292 }
1293 base::StringAppendF(&ss, "%smSpatializedChannelMasks: ", prefixSpace.c_str());
1294 for (auto& mask : mSpatializedChannelMasks) {
1295 base::StringAppendF(&ss, "%s", audio_channel_out_mask_to_string(mask));
1296 }
1297 base::StringAppendF(&ss, "\n%smSupportsHeadTracking: %s\n", prefixSpace.c_str(),
1298 mSupportsHeadTracking ? "true" : "false");
1299 // 2. Settings (Output, tracks)
1300 base::StringAppendF(&ss, "%sNum Active Tracks: %zu\n",
1301 prefixSpace.c_str(), mActiveTracksMasks.size());
1302 base::StringAppendF(&ss, "%sOutputStreamHandle: %d\n", prefixSpace.c_str(), (int)mOutput);
1303
1304 // 3. Sensors, Effect information.
1305 base::StringAppendF(&ss, "%sHeadSensorHandle: 0x%08x\n", prefixSpace.c_str(), mHeadSensor);
1306 base::StringAppendF(&ss, "%sScreenSensorHandle: 0x%08x\n", prefixSpace.c_str(), mScreenSensor);
1307 base::StringAppendF(&ss, "%sEffectHandle: %p\n", prefixSpace.c_str(), mEngine.get());
1308 base::StringAppendF(&ss, "%sDisplayOrientation: %f\n", prefixSpace.c_str(),
1309 mDisplayOrientation);
1310
1311 // 4. Show flag or property state.
1312 base::StringAppendF(
1313 &ss, "%sStereo Spatialization: %s\n", prefixSpace.c_str(),
1314 SpatializerHelper::isStereoSpatializationFeatureEnabled() ? "true" : "false");
1315
1316 ss.append(prefixSpace + "CommandLog:\n");
1317 ss += mLocalLog.dumpToString((prefixSpace + " ").c_str(), mMaxLocalLogLine);
1318
1319 // PostController dump.
1320 if (mPoseController != nullptr) {
1321 ss.append(mPoseController->toString(level + 1))
1322 .append(prefixSpace)
1323 .append("Pose (active stage-to-head) [tx, ty, tz : pitch, roll, yaw]:\n")
1324 .append(prefixSpace)
1325 .append(" PerMinuteHistory:\n")
1326 .append(mPoseDurableRecorder.toString(level + 3))
1327 .append(prefixSpace)
1328 .append(" PerSecondHistory:\n")
1329 .append(mPoseRecorder.toString(level + 3));
1330 } else {
1331 ss.append(prefixSpace).append("SpatializerPoseController not exist\n");
1332 }
1333
1334 if (needUnlock) {
1335 mMutex.unlock();
1336 }
1337 return ss;
1338 }
1339
1340 } // namespace android
1341