/** * Copyright 2024 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #define LOG_TAG "LegacyResampler" #include #include #include #include #include #include #include #include #include namespace android { namespace { const bool IS_DEBUGGABLE_BUILD = #if defined(__ANDROID__) android::base::GetBoolProperty("ro.debuggable", false); #else true; #endif /** * Log debug messages about timestamp and coordinates of event resampling. * Enable this via "adb shell setprop log.tag.LegacyResamplerResampling DEBUG" * (requires restart) */ bool debugResampling() { if (!IS_DEBUGGABLE_BUILD) { static const bool DEBUG_TRANSPORT_RESAMPLING = __android_log_is_loggable(ANDROID_LOG_DEBUG, LOG_TAG "Resampling", ANDROID_LOG_INFO); return DEBUG_TRANSPORT_RESAMPLING; } return __android_log_is_loggable(ANDROID_LOG_DEBUG, LOG_TAG "Resampling", ANDROID_LOG_INFO); } using std::chrono::nanoseconds; constexpr std::chrono::milliseconds RESAMPLE_LATENCY{5}; constexpr std::chrono::milliseconds RESAMPLE_MIN_DELTA{2}; constexpr std::chrono::milliseconds RESAMPLE_MAX_DELTA{20}; constexpr std::chrono::milliseconds RESAMPLE_MAX_PREDICTION{8}; bool canResampleTool(ToolType toolType) { return toolType == ToolType::FINGER || toolType == ToolType::MOUSE || toolType == ToolType::STYLUS || toolType == ToolType::UNKNOWN; } inline float lerp(float a, float b, float alpha) { return a + alpha * (b - a); } PointerCoords calculateResampledCoords(const PointerCoords& a, const PointerCoords& b, float alpha) { // We use the value of alpha to initialize resampledCoords with the latest sample information. PointerCoords resampledCoords = (alpha < 1.0f) ? a : b; resampledCoords.isResampled = true; resampledCoords.setAxisValue(AMOTION_EVENT_AXIS_X, lerp(a.getX(), b.getX(), alpha)); resampledCoords.setAxisValue(AMOTION_EVENT_AXIS_Y, lerp(a.getY(), b.getY(), alpha)); return resampledCoords; } bool equalXY(const PointerCoords& a, const PointerCoords& b) { return (a.getX() == b.getX()) && (a.getY() == b.getY()); } void setMotionEventPointerCoords(MotionEvent& motionEvent, size_t sampleIndex, size_t pointerIndex, const PointerCoords& pointerCoords) { // Ideally, we should not cast away const. In this particular case, it's safe to cast away const // and dereference getHistoricalRawPointerCoords returned pointer because motionEvent is a // nonconst reference to a MotionEvent object, so mutating the object should not be undefined // behavior; moreover, the invoked method guarantees to return a valid pointer. Otherwise, it // fatally logs. Alternatively, we could've created a new MotionEvent from scratch, but this // approach is simpler and more efficient. PointerCoords& motionEventCoords = const_cast( *(motionEvent.getHistoricalRawPointerCoords(pointerIndex, sampleIndex))); motionEventCoords.setAxisValue(AMOTION_EVENT_AXIS_X, pointerCoords.getX()); motionEventCoords.setAxisValue(AMOTION_EVENT_AXIS_Y, pointerCoords.getY()); motionEventCoords.isResampled = pointerCoords.isResampled; } std::ostream& operator<<(std::ostream& os, const PointerCoords& pointerCoords) { os << "(" << pointerCoords.getX() << ", " << pointerCoords.getY() << ")"; return os; } } // namespace void LegacyResampler::updateLatestSamples(const MotionEvent& motionEvent) { const size_t numSamples = motionEvent.getHistorySize() + 1; const size_t latestIndex = numSamples - 1; const size_t secondToLatestIndex = (latestIndex > 0) ? (latestIndex - 1) : 0; for (size_t sampleIndex = secondToLatestIndex; sampleIndex < numSamples; ++sampleIndex) { PointerMap pointerMap; for (size_t pointerIndex = 0; pointerIndex < motionEvent.getPointerCount(); ++pointerIndex) { pointerMap.insert(Pointer{*(motionEvent.getPointerProperties(pointerIndex)), *(motionEvent.getHistoricalRawPointerCoords(pointerIndex, sampleIndex))}); } mLatestSamples.pushBack( Sample{nanoseconds{motionEvent.getHistoricalEventTime(sampleIndex)}, pointerMap}); } } LegacyResampler::Sample LegacyResampler::messageToSample(const InputMessage& message) { PointerMap pointerMap; for (uint32_t i = 0; i < message.body.motion.pointerCount; ++i) { pointerMap.insert(Pointer{message.body.motion.pointers[i].properties, message.body.motion.pointers[i].coords}); } return Sample{nanoseconds{message.body.motion.eventTime}, pointerMap}; } bool LegacyResampler::pointerPropertiesResampleable(const Sample& target, const Sample& auxiliary) { for (const Pointer& pointer : target.pointerMap) { const std::optional auxiliaryPointer = auxiliary.pointerMap.find(PointerMap::PointerId{pointer.properties.id}); if (!auxiliaryPointer.has_value()) { LOG_IF(INFO, debugResampling()) << "Not resampled. Auxiliary sample does not contain all pointers from target."; return false; } if (pointer.properties.toolType != auxiliaryPointer->properties.toolType) { LOG_IF(INFO, debugResampling()) << "Not resampled. Pointer ToolType mismatch."; return false; } if (!canResampleTool(pointer.properties.toolType)) { LOG_IF(INFO, debugResampling()) << "Not resampled. Cannot resample " << ftl::enum_string(pointer.properties.toolType) << " ToolType."; return false; } } return true; } bool LegacyResampler::canInterpolate(const InputMessage& message) const { LOG_IF(FATAL, mLatestSamples.empty()) << "Not resampled. mLatestSamples must not be empty to interpolate."; const Sample& pastSample = *(mLatestSamples.end() - 1); const Sample& futureSample = messageToSample(message); if (!pointerPropertiesResampleable(pastSample, futureSample)) { return false; } const nanoseconds delta = futureSample.eventTime - pastSample.eventTime; if (delta < RESAMPLE_MIN_DELTA) { LOG_IF(INFO, debugResampling()) << "Not resampled. Delta is too small: " << std::setprecision(3) << std::chrono::duration{delta}.count() << "ms"; return false; } return true; } std::optional LegacyResampler::attemptInterpolation( nanoseconds resampleTime, const InputMessage& futureMessage) const { if (!canInterpolate(futureMessage)) { return std::nullopt; } LOG_IF(FATAL, mLatestSamples.empty()) << "Not resampled. mLatestSamples must not be empty to interpolate."; const Sample& pastSample = *(mLatestSamples.end() - 1); const Sample& futureSample = messageToSample(futureMessage); const nanoseconds delta = nanoseconds{futureSample.eventTime} - pastSample.eventTime; const float alpha = std::chrono::duration(resampleTime - pastSample.eventTime) / delta; PointerMap resampledPointerMap; for (const Pointer& pointer : pastSample.pointerMap) { if (std::optional futureSamplePointer = futureSample.pointerMap.find(PointerMap::PointerId{pointer.properties.id}); futureSamplePointer.has_value()) { const PointerCoords& resampledCoords = calculateResampledCoords(pointer.coords, futureSamplePointer->coords, alpha); resampledPointerMap.insert(Pointer{pointer.properties, resampledCoords}); } } return Sample{resampleTime, resampledPointerMap}; } bool LegacyResampler::canExtrapolate() const { if (mLatestSamples.size() < 2) { LOG_IF(INFO, debugResampling()) << "Not resampled. Not enough data."; return false; } const Sample& pastSample = *(mLatestSamples.end() - 2); const Sample& presentSample = *(mLatestSamples.end() - 1); if (!pointerPropertiesResampleable(presentSample, pastSample)) { return false; } const nanoseconds delta = presentSample.eventTime - pastSample.eventTime; if (delta < RESAMPLE_MIN_DELTA) { LOG_IF(INFO, debugResampling()) << "Not resampled. Delta is too small: " << std::setprecision(3) << std::chrono::duration{delta}.count() << "ms"; return false; } else if (delta > RESAMPLE_MAX_DELTA) { LOG_IF(INFO, debugResampling()) << "Not resampled. Delta is too large: " << std::setprecision(3) << std::chrono::duration{delta}.count() << "ms"; return false; } return true; } std::optional LegacyResampler::attemptExtrapolation( nanoseconds resampleTime) const { if (!canExtrapolate()) { return std::nullopt; } LOG_IF(FATAL, mLatestSamples.size() < 2) << "Not resampled. mLatestSamples must have at least two samples to extrapolate."; const Sample& pastSample = *(mLatestSamples.end() - 2); const Sample& presentSample = *(mLatestSamples.end() - 1); const nanoseconds delta = presentSample.eventTime - pastSample.eventTime; // The farthest future time to which we can extrapolate. If the given resampleTime exceeds this, // we use this value as the resample time target. const nanoseconds farthestPrediction = presentSample.eventTime + std::min(delta / 2, RESAMPLE_MAX_PREDICTION); const nanoseconds newResampleTime = (resampleTime > farthestPrediction) ? (farthestPrediction) : (resampleTime); LOG_IF(INFO, debugResampling() && newResampleTime == farthestPrediction) << "Resample time is too far in the future. Adjusting prediction from " << std::setprecision(3) << std::chrono::duration{resampleTime - presentSample.eventTime} .count() << "ms to " << std::chrono::duration{farthestPrediction - presentSample.eventTime} .count() << "ms"; const float alpha = std::chrono::duration(newResampleTime - pastSample.eventTime) / delta; PointerMap resampledPointerMap; for (const Pointer& pointer : presentSample.pointerMap) { if (std::optional pastSamplePointer = pastSample.pointerMap.find(PointerMap::PointerId{pointer.properties.id}); pastSamplePointer.has_value()) { const PointerCoords& resampledCoords = calculateResampledCoords(pastSamplePointer->coords, pointer.coords, alpha); resampledPointerMap.insert(Pointer{pointer.properties, resampledCoords}); } } return Sample{newResampleTime, resampledPointerMap}; } inline void LegacyResampler::addSampleToMotionEvent(const Sample& sample, MotionEvent& motionEvent) { motionEvent.addSample(sample.eventTime.count(), sample.asPointerCoords().data(), motionEvent.getId()); } nanoseconds LegacyResampler::getResampleLatency() const { return RESAMPLE_LATENCY; } /** * The resampler is unaware of ACTION_DOWN. Thus, it needs to constantly check for pointer IDs * occurrences. This problem could be fixed if the resampler has access to the entire stream of * MotionEvent actions. That way, both ACTION_DOWN and ACTION_UP will be visible; therefore, * facilitating pointer tracking between samples. */ void LegacyResampler::overwriteMotionEventSamples(MotionEvent& motionEvent) const { const size_t numSamples = motionEvent.getHistorySize() + 1; for (size_t sampleIndex = 0; sampleIndex < numSamples; ++sampleIndex) { overwriteStillPointers(motionEvent, sampleIndex); overwriteOldPointers(motionEvent, sampleIndex); } } void LegacyResampler::overwriteStillPointers(MotionEvent& motionEvent, size_t sampleIndex) const { if (!mLastRealSample.has_value() || !mPreviousPrediction.has_value()) { LOG_IF(INFO, debugResampling()) << "Still pointers not overwritten. Not enough data."; return; } for (size_t pointerIndex = 0; pointerIndex < motionEvent.getPointerCount(); ++pointerIndex) { const std::optional lastRealPointer = mLastRealSample->pointerMap.find( PointerMap::PointerId{motionEvent.getPointerId(pointerIndex)}); const std::optional previousPointer = mPreviousPrediction->pointerMap.find( PointerMap::PointerId{motionEvent.getPointerId(pointerIndex)}); // This could happen because resampler only receives ACTION_MOVE events. if (!lastRealPointer.has_value() || !previousPointer.has_value()) { continue; } const PointerCoords& pointerCoords = *(motionEvent.getHistoricalRawPointerCoords(pointerIndex, sampleIndex)); if (equalXY(pointerCoords, lastRealPointer->coords)) { LOG_IF(INFO, debugResampling()) << "Pointer ID: " << motionEvent.getPointerId(pointerIndex) << " did not move. Overwriting its coordinates from " << pointerCoords << " to " << previousPointer->coords; setMotionEventPointerCoords(motionEvent, sampleIndex, pointerIndex, previousPointer->coords); } } } void LegacyResampler::overwriteOldPointers(MotionEvent& motionEvent, size_t sampleIndex) const { if (!mPreviousPrediction.has_value()) { LOG_IF(INFO, debugResampling()) << "Old sample not overwritten. Not enough data."; return; } if (nanoseconds{motionEvent.getHistoricalEventTime(sampleIndex)} < mPreviousPrediction->eventTime) { LOG_IF(INFO, debugResampling()) << "Motion event sample older than predicted sample. Overwriting event time from " << std::setprecision(3) << std::chrono::duration{nanoseconds{motionEvent.getHistoricalEventTime( sampleIndex)}} .count() << "ms to " << std::chrono::duration{mPreviousPrediction->eventTime}.count() << "ms"; for (size_t pointerIndex = 0; pointerIndex < motionEvent.getPointerCount(); ++pointerIndex) { const std::optional previousPointer = mPreviousPrediction->pointerMap.find( PointerMap::PointerId{motionEvent.getPointerId(pointerIndex)}); // This could happen because resampler only receives ACTION_MOVE events. if (!previousPointer.has_value()) { continue; } setMotionEventPointerCoords(motionEvent, sampleIndex, pointerIndex, previousPointer->coords); } } } void LegacyResampler::resampleMotionEvent(nanoseconds frameTime, MotionEvent& motionEvent, const InputMessage* futureSample) { const nanoseconds resampleTime = frameTime - RESAMPLE_LATENCY; if (resampleTime.count() == motionEvent.getEventTime()) { LOG_IF(INFO, debugResampling()) << "Not resampled. Resample time equals motion event time."; return; } updateLatestSamples(motionEvent); const std::optional sample = (futureSample != nullptr) ? (attemptInterpolation(resampleTime, *futureSample)) : (attemptExtrapolation(resampleTime)); if (sample.has_value()) { addSampleToMotionEvent(*sample, motionEvent); if (mPreviousPrediction.has_value()) { overwriteMotionEventSamples(motionEvent); } // mPreviousPrediction is only updated whenever extrapolation occurs because extrapolation // is about predicting upcoming scenarios. if (futureSample == nullptr) { mPreviousPrediction = sample; } } LOG_IF(FATAL, mLatestSamples.empty()) << "mLatestSamples must contain at least one sample."; mLastRealSample = *(mLatestSamples.end() - 1); } // --- FilteredLegacyResampler --- FilteredLegacyResampler::FilteredLegacyResampler(float minCutoffFreq, float beta) : mResampler{}, mMinCutoffFreq{minCutoffFreq}, mBeta{beta} {} void FilteredLegacyResampler::resampleMotionEvent(std::chrono::nanoseconds requestedFrameTime, MotionEvent& motionEvent, const InputMessage* futureSample) { mResampler.resampleMotionEvent(requestedFrameTime, motionEvent, futureSample); const size_t numSamples = motionEvent.getHistorySize() + 1; for (size_t sampleIndex = 0; sampleIndex < numSamples; ++sampleIndex) { for (size_t pointerIndex = 0; pointerIndex < motionEvent.getPointerCount(); ++pointerIndex) { const int32_t pointerId = motionEvent.getPointerProperties(pointerIndex)->id; const nanoseconds eventTime = nanoseconds{motionEvent.getHistoricalEventTime(sampleIndex)}; // Refer to the static function `setMotionEventPointerCoords` for a justification of // casting away const. PointerCoords& pointerCoords = const_cast( *(motionEvent.getHistoricalRawPointerCoords(pointerIndex, sampleIndex))); const auto& [iter, _] = mFilteredPointers.try_emplace(pointerId, mMinCutoffFreq, mBeta); iter->second.filter(eventTime, pointerCoords); } } } std::chrono::nanoseconds FilteredLegacyResampler::getResampleLatency() const { return mResampler.getResampleLatency(); } } // namespace android