/* * Copyright (C) 2022 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #define LOG_TAG "ExtCamOfflnSsn" #include #include "ExternalCameraOfflineSession.h" #include #include #include #include #include #include #include #include #include #define HAVE_JPEG // required for libyuv.h to export MJPEG decode APIs #include namespace { // Size of request/result metadata fast message queue. Change to 0 to always use hwbinder buffer. constexpr size_t kMetadataMsgQueueSize = 1 << 18 /* 256kB */; } // anonymous namespace namespace android { namespace hardware { namespace camera { namespace device { namespace implementation { using ::aidl::android::hardware::camera::device::BufferStatus; using ::aidl::android::hardware::camera::device::ErrorMsg; using ::aidl::android::hardware::camera::device::ShutterMsg; using ::aidl::android::hardware::camera::device::StreamBuffer; // Static instance HandleImporter ExternalCameraOfflineSession::sHandleImporter; ExternalCameraOfflineSession::ExternalCameraOfflineSession( const CroppingType& croppingType, const common::V1_0::helper::CameraMetadata& chars, const std::string& cameraId, const std::string& exifMake, const std::string& exifModel, uint32_t blobBufferSize, bool afTrigger, const std::vector& offlineStreams, std::deque>& offlineReqs, const std::map& circulatingBuffers) : mCroppingType(croppingType), mChars(chars), mCameraId(cameraId), mExifMake(exifMake), mExifModel(exifModel), mBlobBufferSize(blobBufferSize), mAfTrigger(afTrigger), mOfflineStreams(offlineStreams), mOfflineReqs(offlineReqs), mCirculatingBuffers(circulatingBuffers) {} ExternalCameraOfflineSession::~ExternalCameraOfflineSession() { close(); } bool ExternalCameraOfflineSession::initialize() { mResultMetadataQueue = std::make_shared(kMetadataMsgQueueSize, false /* non blocking */); if (!mResultMetadataQueue->isValid()) { ALOGE("%s: invalid result fmq", __FUNCTION__); return true; } return false; } Status ExternalCameraOfflineSession::importBuffer(int32_t streamId, uint64_t bufId, buffer_handle_t buf, buffer_handle_t** outBufPtr) { Mutex::Autolock _l(mCbsLock); return importBufferImpl(mCirculatingBuffers, sHandleImporter, streamId, bufId, buf, outBufPtr); } Status ExternalCameraOfflineSession::processCaptureResult(std::shared_ptr& req) { ATRACE_CALL(); // Fill output buffers std::vector results; results.resize(1); CaptureResult& result = results[0]; result.frameNumber = req->frameNumber; result.partialResult = 1; result.inputBuffer.streamId = -1; result.outputBuffers.resize(req->buffers.size()); for (size_t i = 0; i < req->buffers.size(); i++) { StreamBuffer& outputBuffer = result.outputBuffers[i]; outputBuffer.streamId = req->buffers[i].streamId; outputBuffer.bufferId = req->buffers[i].bufferId; if (req->buffers[i].fenceTimeout) { outputBuffer.status = BufferStatus::ERROR; if (req->buffers[i].acquireFence >= 0) { native_handle_t* handle = native_handle_create(/*numFds*/ 1, /*numInts*/ 0); handle->data[0] = req->buffers[i].acquireFence; result.outputBuffers[i].releaseFence = android::dupToAidl(handle); native_handle_delete(handle); } notifyError(req->frameNumber, req->buffers[i].streamId, ErrorCode::ERROR_BUFFER); } else { result.outputBuffers[i].status = BufferStatus::OK; // TODO: refactor if (req->buffers[i].acquireFence >= 0) { native_handle_t* handle = native_handle_create(/*numFds*/ 1, /*numInts*/ 0); handle->data[0] = req->buffers[i].acquireFence; outputBuffer.releaseFence = android::dupToAidl(handle); native_handle_delete(handle); } } } // Fill capture result metadata fillCaptureResult(req->setting, req->shutterTs); const camera_metadata_t* rawResult = req->setting.getAndLock(); convertToAidl(rawResult, &result.result); req->setting.unlock(rawResult); // Callback into framework invokeProcessCaptureResultCallback(results, /* tryWriteFmq */ true); freeReleaseFences(results); return Status::OK; } #define UPDATE(md, tag, data, size) \ do { \ if ((md).update((tag), (data), (size))) { \ ALOGE("Update " #tag " failed!"); \ return BAD_VALUE; \ } \ } while (0) status_t ExternalCameraOfflineSession::fillCaptureResult(common::V1_0::helper::CameraMetadata md, nsecs_t timestamp) { bool afTrigger = false; { std::lock_guard lk(mAfTriggerLock); afTrigger = mAfTrigger; if (md.exists(ANDROID_CONTROL_AF_TRIGGER)) { camera_metadata_entry entry = md.find(ANDROID_CONTROL_AF_TRIGGER); if (entry.data.u8[0] == ANDROID_CONTROL_AF_TRIGGER_START) { mAfTrigger = afTrigger = true; } else if (entry.data.u8[0] == ANDROID_CONTROL_AF_TRIGGER_CANCEL) { mAfTrigger = afTrigger = false; } } } // For USB camera, the USB camera handles everything and we don't have control // over AF. We only simply fake the AF metadata based on the request // received here. uint8_t afState; if (afTrigger) { afState = ANDROID_CONTROL_AF_STATE_FOCUSED_LOCKED; } else { afState = ANDROID_CONTROL_AF_STATE_INACTIVE; } UPDATE(md, ANDROID_CONTROL_AF_STATE, &afState, 1); camera_metadata_ro_entry activeArraySize = mChars.find(ANDROID_SENSOR_INFO_ACTIVE_ARRAY_SIZE); return fillCaptureResultCommon(md, timestamp, activeArraySize); } void ExternalCameraOfflineSession::invokeProcessCaptureResultCallback( std::vector& results, bool tryWriteFmq) { if (mProcessCaptureResultLock.tryLock() != OK) { const nsecs_t NS_TO_SECOND = 1E9; ALOGV("%s: previous call is not finished! waiting 1s...", __FUNCTION__); if (mProcessCaptureResultLock.timedLock(/* 1s */ NS_TO_SECOND) != OK) { ALOGE("%s: cannot acquire lock in 1s, cannot proceed", __FUNCTION__); return; } } if (tryWriteFmq && mResultMetadataQueue->availableToWrite() > 0) { for (CaptureResult& result : results) { if (!result.result.metadata.empty()) { if (mResultMetadataQueue->write( reinterpret_cast(result.result.metadata.data()), result.result.metadata.size())) { result.fmqResultSize = result.result.metadata.size(); result.result.metadata.clear(); } else { ALOGW("%s: couldn't utilize fmq, fall back to hwbinder", __FUNCTION__); result.fmqResultSize = 0; } } else { result.fmqResultSize = 0; } } } auto status = mCallback->processCaptureResult(results); if (!status.isOk()) { ALOGE("%s: processCaptureResult ERROR : %d:%d", __FUNCTION__, status.getExceptionCode(), status.getServiceSpecificError()); } mProcessCaptureResultLock.unlock(); } Status ExternalCameraOfflineSession::processCaptureRequestError( const std::shared_ptr& req, std::vector* outMsgs, std::vector* outResults) { ATRACE_CALL(); if (outMsgs == nullptr) { notifyError(/*frameNum*/ req->frameNumber, /*stream*/ -1, ErrorCode::ERROR_REQUEST); } else { NotifyMsg shutter; shutter.set(ShutterMsg{ .frameNumber = req->frameNumber, .timestamp = req->shutterTs, }); NotifyMsg error; error.set(ErrorMsg{.frameNumber = req->frameNumber, .errorStreamId = -1, .errorCode = ErrorCode::ERROR_REQUEST}); outMsgs->push_back(shutter); outMsgs->push_back(error); } // Fill output buffers CaptureResult result; result.frameNumber = req->frameNumber; result.partialResult = 1; result.inputBuffer.streamId = -1; result.outputBuffers.resize(req->buffers.size()); for (size_t i = 0; i < req->buffers.size(); i++) { StreamBuffer& outputBuffer = result.outputBuffers[i]; outputBuffer.streamId = req->buffers[i].streamId; outputBuffer.bufferId = req->buffers[i].bufferId; outputBuffer.status = BufferStatus::ERROR; if (req->buffers[i].acquireFence >= 0) { native_handle_t* handle = native_handle_create(/*numFds*/ 1, /*numInts*/ 0); handle->data[0] = req->buffers[i].acquireFence; outputBuffer.releaseFence = dupToAidl(handle); native_handle_delete(handle); } } if (outResults == nullptr) { // Callback into framework std::vector results(1); results[0] = std::move(result); invokeProcessCaptureResultCallback(results, /* tryWriteFmq */ true); freeReleaseFences(results); } else { outResults->push_back(std::move(result)); } return Status::OK; } ssize_t ExternalCameraOfflineSession::getJpegBufferSize(int32_t, int32_t) const { // Empty implementation here as the jpeg buffer size is passed in by ctor return 0; } void ExternalCameraOfflineSession::notifyError(int32_t frameNumber, int32_t streamId, ErrorCode ec) { NotifyMsg msg; msg.set( ErrorMsg{.frameNumber = frameNumber, .errorStreamId = streamId, .errorCode = ec}); mCallback->notify({msg}); } ScopedAStatus ExternalCameraOfflineSession::setCallback( const std::shared_ptr& in_cb) { Mutex::Autolock _il(mInterfaceLock); if (mCallback != nullptr && in_cb != nullptr) { ALOGE("%s: callback must not be set twice!", __FUNCTION__); return fromStatus(Status::OK); } mCallback = in_cb; initOutputThread(); if (mOutputThread == nullptr) { ALOGE("%s: init OutputThread failed!", __FUNCTION__); } return fromStatus(Status::OK); } void ExternalCameraOfflineSession::initOutputThread() { if (mOutputThread != nullptr) { ALOGE("%s: OutputThread already exist!", __FUNCTION__); return; } // Grab a shared_ptr to 'this' from ndk::SharedRefBase::ref() std::shared_ptr thiz = ref(); mBufferRequestThread = std::make_shared( /*parent=*/thiz, mCallback); mBufferRequestThread->run(); mOutputThread = std::make_shared(/*parent=*/thiz, mCroppingType, mChars, mBufferRequestThread, mOfflineReqs); mOutputThread->setExifMakeModel(mExifMake, mExifModel); Size inputSize = {mOfflineReqs[0]->frameIn->mWidth, mOfflineReqs[0]->frameIn->mHeight}; Size maxThumbSize = getMaxThumbnailResolution(mChars); mOutputThread->allocateIntermediateBuffers(inputSize, maxThumbSize, mOfflineStreams, mBlobBufferSize); mOutputThread->run(); } ScopedAStatus ExternalCameraOfflineSession::getCaptureResultMetadataQueue( MQDescriptor* _aidl_return) { Mutex::Autolock _il(mInterfaceLock); *_aidl_return = mResultMetadataQueue->dupeDesc(); return fromStatus(Status::OK); } ScopedAStatus ExternalCameraOfflineSession::close() { Mutex::Autolock _il(mInterfaceLock); { Mutex::Autolock _l(mLock); if (mClosed) { ALOGW("%s: offline session already closed!", __FUNCTION__); return fromStatus(Status::OK); } } if (mBufferRequestThread != nullptr) { mBufferRequestThread->requestExitAndWait(); mBufferRequestThread.reset(); } if (mOutputThread) { mOutputThread->flush(); mOutputThread->requestExitAndWait(); mOutputThread.reset(); } Mutex::Autolock _l(mLock); // free all buffers { Mutex::Autolock _cbl(mCbsLock); for (auto& stream : mOfflineStreams) { cleanupBuffersLocked(stream.id); } } mCallback.reset(); mClosed = true; return fromStatus(Status::OK); } void ExternalCameraOfflineSession::cleanupBuffersLocked(int32_t id) { for (auto& pair : mCirculatingBuffers.at(id)) { sHandleImporter.freeBuffer(pair.second); } mCirculatingBuffers[id].clear(); mCirculatingBuffers.erase(id); } bool ExternalCameraOfflineSession::OutputThread::threadLoop() { auto parent = mParent.lock(); if (parent == nullptr) { ALOGE("%s: session has been disconnected!", __FUNCTION__); return false; } if (mOfflineReqs.empty()) { ALOGI("%s: all offline requests are processed. Stopping.", __FUNCTION__); return false; } std::shared_ptr req = mOfflineReqs.front(); mOfflineReqs.pop_front(); auto onDeviceError = [&](auto... args) { ALOGE(args...); parent->notifyError(req->frameNumber, /*stream*/ -1, ErrorCode::ERROR_DEVICE); signalRequestDone(); return false; }; if (req->frameIn->mFourcc != V4L2_PIX_FMT_MJPEG && req->frameIn->mFourcc != V4L2_PIX_FMT_Z16) { return onDeviceError("%s: do not support V4L2 format %c%c%c%c", __FUNCTION__, req->frameIn->mFourcc & 0xFF, (req->frameIn->mFourcc >> 8) & 0xFF, (req->frameIn->mFourcc >> 16) & 0xFF, (req->frameIn->mFourcc >> 24) & 0xFF); } int res = requestBufferStart(req->buffers); if (res != 0) { ALOGE("%s: send BufferRequest failed! res %d", __FUNCTION__, res); return onDeviceError("%s: failed to send buffer request!", __FUNCTION__); } std::unique_lock lk(mBufferLock); // Convert input V4L2 frame to YU12 of the same size // TODO: see if we can save some computation by converting to YV12 here uint8_t* inData; size_t inDataSize; if (req->frameIn->getData(&inData, &inDataSize) != 0) { lk.unlock(); return onDeviceError("%s: V4L2 buffer map failed", __FUNCTION__); } // TODO: in some special case maybe we can decode jpg directly to gralloc output? if (req->frameIn->mFourcc == V4L2_PIX_FMT_MJPEG) { ATRACE_BEGIN("MJPGtoI420"); int convRes = libyuv::MJPGToI420( inData, inDataSize, static_cast(mYu12FrameLayout.y), mYu12FrameLayout.yStride, static_cast(mYu12FrameLayout.cb), mYu12FrameLayout.cStride, static_cast(mYu12FrameLayout.cr), mYu12FrameLayout.cStride, mYu12Frame->mWidth, mYu12Frame->mHeight, mYu12Frame->mWidth, mYu12Frame->mHeight); ATRACE_END(); if (convRes != 0) { // For some webcam, the first few V4L2 frames might be malformed... ALOGE("%s: Convert V4L2 frame to YU12 failed! res %d", __FUNCTION__, convRes); lk.unlock(); Status st = parent->processCaptureRequestError(req); if (st != Status::OK) { return onDeviceError("%s: failed to process capture request error!", __FUNCTION__); } signalRequestDone(); return true; } } ATRACE_BEGIN("Wait for BufferRequest done"); res = waitForBufferRequestDone(&req->buffers); ATRACE_END(); if (res != 0) { ALOGE("%s: wait for BufferRequest done failed! res %d", __FUNCTION__, res); lk.unlock(); return onDeviceError("%s: failed to process buffer request error!", __FUNCTION__); } ALOGV("%s processing new request", __FUNCTION__); const int kSyncWaitTimeoutMs = 500; for (auto& halBuf : req->buffers) { if (*(halBuf.bufPtr) == nullptr) { ALOGW("%s: buffer for stream %d missing", __FUNCTION__, halBuf.streamId); halBuf.fenceTimeout = true; } else if (halBuf.acquireFence >= 0) { int ret = sync_wait(halBuf.acquireFence, kSyncWaitTimeoutMs); if (ret) { halBuf.fenceTimeout = true; } else { ::close(halBuf.acquireFence); halBuf.acquireFence = -1; } } if (halBuf.fenceTimeout) { continue; } // Gralloc lockYCbCr the buffer switch (halBuf.format) { case PixelFormat::BLOB: { int ret = createJpegLocked(halBuf, req->setting); if (ret != 0) { lk.unlock(); return onDeviceError("%s: createJpegLocked failed with %d", __FUNCTION__, ret); } } break; case PixelFormat::Y16: { void* outLayout = sHandleImporter.lock( *(halBuf.bufPtr), static_cast(halBuf.usage), inDataSize); std::memcpy(outLayout, inData, inDataSize); int relFence = sHandleImporter.unlock(*(halBuf.bufPtr)); if (relFence >= 0) { halBuf.acquireFence = relFence; } } break; case PixelFormat::YCBCR_420_888: case PixelFormat::YV12: { android::Rect outRect{0, 0, static_cast(halBuf.width), static_cast(halBuf.height)}; android_ycbcr result = sHandleImporter.lockYCbCr( *(halBuf.bufPtr), static_cast(halBuf.usage), outRect); ALOGV("%s: outLayout y %p cb %p cr %p y_str %zu c_str %zu c_step %zu", __FUNCTION__, result.y, result.cb, result.cr, result.ystride, result.cstride, result.chroma_step); if (result.ystride > UINT32_MAX || result.cstride > UINT32_MAX || result.chroma_step > UINT32_MAX) { return onDeviceError("%s: lockYCbCr failed. Unexpected values!", __FUNCTION__); } YCbCrLayout outLayout = {.y = result.y, .cb = result.cb, .cr = result.cr, .yStride = static_cast(result.ystride), .cStride = static_cast(result.cstride), .chromaStep = static_cast(result.chroma_step)}; // Convert to output buffer size/format uint32_t outputFourcc = getFourCcFromLayout(outLayout); ALOGV("%s: converting to format %c%c%c%c", __FUNCTION__, outputFourcc & 0xFF, (outputFourcc >> 8) & 0xFF, (outputFourcc >> 16) & 0xFF, (outputFourcc >> 24) & 0xFF); YCbCrLayout cropAndScaled; ATRACE_BEGIN("cropAndScaleLocked"); int ret = cropAndScaleLocked(mYu12Frame, Size{halBuf.width, halBuf.height}, &cropAndScaled); ATRACE_END(); if (ret != 0) { lk.unlock(); return onDeviceError("%s: crop and scale failed!", __FUNCTION__); } Size sz{halBuf.width, halBuf.height}; ATRACE_BEGIN("formatConvert"); ret = formatConvert(cropAndScaled, outLayout, sz, outputFourcc); ATRACE_END(); if (ret != 0) { lk.unlock(); return onDeviceError("%s: format coversion failed!", __FUNCTION__); } int relFence = sHandleImporter.unlock(*(halBuf.bufPtr)); if (relFence >= 0) { halBuf.acquireFence = relFence; } } break; default: lk.unlock(); return onDeviceError("%s: unknown output format %x", __FUNCTION__, halBuf.format); } } // for each buffer mScaledYu12Frames.clear(); // Don't hold the lock while calling back to parent lk.unlock(); Status st = parent->processCaptureResult(req); if (st != Status::OK) { return onDeviceError("%s: failed to process capture result!", __FUNCTION__); } signalRequestDone(); return true; } } // namespace implementation } // namespace device } // namespace camera } // namespace hardware } // namespace android