1 /* 2 * Copyright (C) 2013-2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef ANDROID_SERVERS_CAMERA3DEVICE_H 18 #define ANDROID_SERVERS_CAMERA3DEVICE_H 19 20 #include <utility> 21 #include <unordered_map> 22 #include <set> 23 #include <tuple> 24 25 #include <utils/Condition.h> 26 #include <utils/Errors.h> 27 #include <utils/List.h> 28 #include <utils/Mutex.h> 29 #include <utils/Thread.h> 30 #include <utils/KeyedVector.h> 31 #include <utils/Timers.h> 32 33 #include <camera/CaptureResult.h> 34 #include <gui/Flags.h> 35 36 #include "CameraServiceWatchdog.h" 37 #include <aidl/android/hardware/camera/device/CameraBlob.h> 38 39 #include "common/CameraDeviceBase.h" 40 #include "common/DepthPhotoProcessor.h" 41 #include "device3/BufferUtils.h" 42 #include "device3/StatusTracker.h" 43 #include "device3/Camera3BufferManager.h" 44 #include "device3/DistortionMapper.h" 45 #include "device3/ZoomRatioMapper.h" 46 #include "device3/RotateAndCropMapper.h" 47 #include "device3/UHRCropAndMeteringRegionMapper.h" 48 #include "device3/InFlightRequest.h" 49 #include "device3/Camera3OutputInterface.h" 50 #include "device3/Camera3OfflineSession.h" 51 #include "device3/Camera3StreamInterface.h" 52 #include "utils/AttributionAndPermissionUtils.h" 53 #include "utils/TagMonitor.h" 54 #include "utils/IPCTransport.h" 55 #include "utils/LatencyHistogram.h" 56 #include "utils/CameraServiceProxyWrapper.h" 57 #include <camera_metadata_hidden.h> 58 59 using android::camera3::camera_capture_request_t; 60 using android::camera3::camera_request_template; 61 using android::camera3::camera_stream_buffer_t; 62 using android::camera3::camera_stream_configuration_t; 63 using android::camera3::camera_stream_configuration_mode_t; 64 using android::camera3::CAMERA_TEMPLATE_COUNT; 65 using android::camera3::OutputStreamInfo; 66 using android::camera3::SurfaceHolder; 67 68 namespace android { 69 70 namespace camera3 { 71 72 class Camera3Stream; 73 class Camera3ZslStream; 74 class Camera3StreamInterface; 75 76 } // namespace camera3 77 78 /** 79 * CameraDevice for HAL devices with version CAMERA_DEVICE_API_VERSION_3_0 or higher. 80 */ 81 class Camera3Device : 82 public CameraDeviceBase, 83 public camera3::SetErrorInterface, 84 public camera3::InflightRequestUpdateInterface, 85 public camera3::RequestBufferInterface, 86 public camera3::FlushBufferInterface, 87 public AttributionAndPermissionUtilsEncapsulator { 88 friend class HidlCamera3Device; 89 friend class AidlCamera3Device; 90 public: 91 92 explicit Camera3Device(std::shared_ptr<CameraServiceProxyWrapper>& cameraServiceProxyWrapper, 93 std::shared_ptr<AttributionAndPermissionUtils> attributionAndPermissionUtils, 94 const std::string& id, bool overrideForPerfClass, int rotationOverride, 95 bool legacyClient = false); 96 97 virtual ~Camera3Device(); 98 // Delete and optionally close native handles and clear the input vector afterward 99 static void cleanupNativeHandles( 100 std::vector<native_handle_t*> *handles, bool closeFd = false); 101 getTransportType()102 virtual IPCTransport getTransportType() const override { 103 return mInterface->getTransportType(); 104 } 105 isHalBufferManagedStream(int32_t streamId)106 bool isHalBufferManagedStream(int32_t streamId) const { 107 return mInterface->isHalBufferManagedStream(streamId); 108 }; 109 110 /** 111 * CameraDeviceBase interface 112 */ 113 114 const std::string& getId() const override; 115 getVendorTagId()116 metadata_vendor_id_t getVendorTagId() const override { return mVendorTagId; } 117 118 // Watchdog thread 119 sp<CameraServiceWatchdog> mCameraServiceWatchdog; 120 121 // Transitions to idle state on success. 122 virtual status_t initialize(sp<CameraProviderManager> /*manager*/, 123 const std::string& /*monitorTags*/) = 0; 124 125 status_t disconnect() override; 126 status_t dump(int fd, const Vector<String16> &args) override; 127 status_t startWatchingTags(const std::string &tags) override; 128 status_t stopWatchingTags() override; 129 status_t dumpWatchedEventsToVector(std::vector<std::string> &out) override; 130 const CameraMetadata& info() const override; 131 const CameraMetadata& infoPhysical(const std::string& physicalId) const override; isCompositeJpegRDisabled()132 bool isCompositeJpegRDisabled() const override { return mIsCompositeJpegRDisabled; }; 133 134 // Capture and setStreamingRequest will configure streams if currently in 135 // idle state 136 status_t capture(CameraMetadata &request, int64_t *lastFrameNumber = NULL) override; 137 status_t captureList(const List<const PhysicalCameraSettingsList> &requestsList, 138 const std::list<SurfaceMap> &surfaceMaps, 139 int64_t *lastFrameNumber = NULL) override; 140 status_t setStreamingRequest(const CameraMetadata &request, 141 int64_t *lastFrameNumber = NULL) override; 142 status_t setStreamingRequestList(const List<const PhysicalCameraSettingsList> &requestsList, 143 const std::list<SurfaceMap> &surfaceMaps, 144 int64_t *lastFrameNumber = NULL) override; 145 status_t clearStreamingRequest(int64_t *lastFrameNumber = NULL) override; 146 147 status_t waitUntilRequestReceived(int32_t requestId, nsecs_t timeout) override; 148 149 // Actual stream creation/deletion is delayed until first request is submitted 150 // If adding streams while actively capturing, will pause device before adding 151 // stream, reconfiguring device, and unpausing. If the client create a stream 152 // with nullptr consumer surface, the client must then call setConsumers() 153 // and finish the stream configuration before starting output streaming. 154 status_t createStream(sp<Surface> consumer, 155 uint32_t width, uint32_t height, int format, 156 android_dataspace dataSpace, camera_stream_rotation_t rotation, int *id, 157 const std::string& physicalCameraId, 158 const std::unordered_set<int32_t> &sensorPixelModesUsed, 159 std::vector<int> *surfaceIds = nullptr, 160 int streamSetId = camera3::CAMERA3_STREAM_SET_ID_INVALID, 161 bool isShared = false, bool isMultiResolution = false, 162 uint64_t consumerUsage = 0, 163 int64_t dynamicRangeProfile = 164 ANDROID_REQUEST_AVAILABLE_DYNAMIC_RANGE_PROFILES_MAP_STANDARD, 165 int64_t streamUseCase = ANDROID_SCALER_AVAILABLE_STREAM_USE_CASES_DEFAULT, 166 int timestampBase = OutputConfiguration::TIMESTAMP_BASE_DEFAULT, 167 int mirrorMode = OutputConfiguration::MIRROR_MODE_AUTO, 168 int32_t colorSpace = ANDROID_REQUEST_AVAILABLE_COLOR_SPACE_PROFILES_MAP_UNSPECIFIED, 169 bool useReadoutTimestamp = false) 170 override; 171 172 status_t createStream(const std::vector<SurfaceHolder>& consumers, 173 bool hasDeferredConsumer, uint32_t width, uint32_t height, int format, 174 android_dataspace dataSpace, camera_stream_rotation_t rotation, int *id, 175 const std::string& physicalCameraId, 176 const std::unordered_set<int32_t> &sensorPixelModesUsed, 177 std::vector<int> *surfaceIds = nullptr, 178 int streamSetId = camera3::CAMERA3_STREAM_SET_ID_INVALID, 179 bool isShared = false, bool isMultiResolution = false, 180 uint64_t consumerUsage = 0, 181 int64_t dynamicRangeProfile = 182 ANDROID_REQUEST_AVAILABLE_DYNAMIC_RANGE_PROFILES_MAP_STANDARD, 183 int64_t streamUseCase = ANDROID_SCALER_AVAILABLE_STREAM_USE_CASES_DEFAULT, 184 int timestampBase = OutputConfiguration::TIMESTAMP_BASE_DEFAULT, 185 int32_t colorSpace = ANDROID_REQUEST_AVAILABLE_COLOR_SPACE_PROFILES_MAP_UNSPECIFIED, 186 bool useReadoutTimestamp = false) 187 override; 188 189 status_t createInputStream( 190 uint32_t width, uint32_t height, int format, bool isMultiResolution, 191 int *id) override; 192 193 status_t getStreamInfo(int id, StreamInfo *streamInfo) override; 194 status_t setStreamTransform(int id, int transform) override; 195 196 status_t deleteStream(int id) override; 197 beginConfigure()198 virtual status_t beginConfigure() override {return OK;}; 199 getSharedStreamId(const OutputConfiguration &,int *)200 virtual status_t getSharedStreamId(const OutputConfiguration& /*config*/, 201 int* /*streamId*/) override {return INVALID_OPERATION;}; 202 addSharedSurfaces(int,const std::vector<android::camera3::OutputStreamInfo> &,const std::vector<SurfaceHolder> &,std::vector<int> *)203 virtual status_t addSharedSurfaces(int /*streamId*/, 204 const std::vector<android::camera3::OutputStreamInfo>& /*outputInfo*/, 205 const std::vector<SurfaceHolder>& /*surfaces*/, 206 std::vector<int>* /*surfaceIds*/) override {return INVALID_OPERATION;}; 207 removeSharedSurfaces(int,const std::vector<size_t> &)208 virtual status_t removeSharedSurfaces(int /*streamId*/, 209 const std::vector<size_t>& /*surfaceIds*/) override {return INVALID_OPERATION;}; 210 211 status_t configureStreams(const CameraMetadata& sessionParams, 212 int operatingMode = 213 camera_stream_configuration_mode_t::CAMERA_STREAM_CONFIGURATION_NORMAL_MODE) override; 214 #if WB_CAMERA3_AND_PROCESSORS_WITH_DEPENDENCIES 215 status_t getInputSurface(sp<Surface> *surface) override; 216 #else 217 status_t getInputBufferProducer( 218 sp<IGraphicBufferProducer> *producer) override; 219 #endif 220 221 void getOfflineStreamIds(std::vector<int> *offlineStreamIds) override; 222 223 status_t createDefaultRequest(camera_request_template_t templateId, 224 CameraMetadata *request) override; 225 226 // Transitions to the idle state on success 227 status_t waitUntilDrained() override; 228 229 status_t setNotifyCallback(wp<NotificationListener> listener) override; 230 bool willNotify3A() override; 231 status_t waitForNextFrame(nsecs_t timeout) override; 232 status_t getNextResult(CaptureResult *frame) override; 233 234 status_t triggerAutofocus(uint32_t id) override; 235 status_t triggerCancelAutofocus(uint32_t id) override; 236 status_t triggerPrecaptureMetering(uint32_t id) override; 237 238 status_t flush(int64_t *lastFrameNumber = NULL) override; 239 240 status_t prepare(int streamId) override; 241 242 status_t tearDown(int streamId) override; 243 244 status_t addBufferListenerForStream(int streamId, 245 wp<camera3::Camera3StreamBufferListener> listener) override; 246 247 status_t prepare(int maxCount, int streamId) override; 248 249 ssize_t getJpegBufferSize(const CameraMetadata &info, uint32_t width, 250 uint32_t height) const override; 251 ssize_t getPointCloudBufferSize(const CameraMetadata &info) const; 252 ssize_t getRawOpaqueBufferSize(const CameraMetadata &info, int32_t width, int32_t height, 253 bool maxResolution) const; 254 255 // Methods called by subclasses 256 void notifyStatus(bool idle); // updates from StatusTracker 257 258 /** 259 * Set the deferred consumer surfaces to the output stream and finish the deferred 260 * consumer configuration. 261 */ 262 status_t setConsumerSurfaces( 263 int streamId, const std::vector<SurfaceHolder>& consumers, 264 std::vector<int> *surfaceIds /*out*/) override; 265 266 /** 267 * Update a given stream. 268 */ 269 status_t updateStream(int streamId, const std::vector<SurfaceHolder> &newSurfaces, 270 const std::vector<OutputStreamInfo> &outputInfo, 271 const std::vector<size_t> &removedSurfaceIds, 272 KeyedVector<sp<Surface>, size_t> *outputMap/*out*/); 273 274 /** 275 * Drop buffers for stream of streamId if dropping is true. If dropping is false, do not 276 * drop buffers for stream of streamId. 277 */ 278 status_t dropStreamBuffers(bool dropping, int streamId) override; 279 280 nsecs_t getExpectedInFlightDuration() override; 281 switchToOffline(const std::vector<int32_t> &,sp<CameraOfflineSessionBase> *)282 virtual status_t switchToOffline(const std::vector<int32_t>& , 283 /*out*/ sp<CameraOfflineSessionBase>* ) override { 284 return INVALID_OPERATION; 285 }; 286 287 // RequestBufferInterface 288 bool startRequestBuffer() override; 289 void endRequestBuffer() override; 290 nsecs_t getWaitDuration() override; 291 292 // FlushBufferInterface 293 void getInflightBufferKeys(std::vector<std::pair<int32_t, int32_t>>* out) override; 294 void getInflightRequestBufferKeys(std::vector<uint64_t>* out) override; 295 std::vector<sp<camera3::Camera3StreamInterface>> getAllStreams() override; 296 297 /** 298 * Set the current behavior for the ROTATE_AND_CROP control when in AUTO. 299 * 300 * The value must be one of the ROTATE_AND_CROP_* values besides AUTO, 301 * and defaults to NONE. 302 */ 303 status_t setRotateAndCropAutoBehavior( 304 camera_metadata_enum_android_scaler_rotate_and_crop_t rotateAndCropValue, bool fromHal); 305 306 /** 307 * Set the current behavior for the AUTOFRAMING control when in AUTO. 308 * 309 * The value must be one of the AUTOFRAMING_* values besides AUTO. 310 */ 311 status_t setAutoframingAutoBehavior( 312 camera_metadata_enum_android_control_autoframing_t autoframingValue); 313 314 /** 315 * Whether camera muting (producing black-only output) is supported. 316 * 317 * Calling setCameraMute(true) when this returns false will return an 318 * INVALID_OPERATION error. 319 */ 320 bool supportsCameraMute(); 321 322 /** 323 * Mute the camera. 324 * 325 * When muted, black image data is output on all output streams. 326 */ 327 status_t setCameraMute(bool enabled); 328 329 /** 330 * Mute the camera. 331 * 332 * When muted, black image data is output on all output streams. 333 * This method assumes the caller already acquired the 'mInterfaceLock' 334 * and 'mLock' locks. 335 */ 336 status_t setCameraMuteLocked(bool enabled); 337 338 /** 339 * Enables/disables camera service watchdog 340 */ 341 status_t setCameraServiceWatchdog(bool enabled); 342 343 // Set stream use case overrides 344 void setStreamUseCaseOverrides( 345 const std::vector<int64_t>& useCaseOverrides); 346 347 // Clear stream use case overrides 348 void clearStreamUseCaseOverrides(); 349 350 /** 351 * Whether the camera device supports zoom override. 352 */ 353 bool supportsZoomOverride(); 354 355 // Set/reset zoom override 356 status_t setZoomOverride(int32_t zoomOverride); 357 358 // Get the status trackeer for the camera device getStatusTracker()359 wp<camera3::StatusTracker> getStatusTracker() { return mStatusTracker; } 360 361 // Whether the device is in error state 362 bool hasDeviceError(); 363 364 /** 365 * The injection camera session to replace the internal camera 366 * session. 367 */ 368 status_t injectCamera(const std::string& injectedCamId, 369 sp<CameraProviderManager> manager); 370 371 /** 372 * Stop the injection camera and restore to internal camera session. 373 */ 374 status_t stopInjection(); 375 376 /** 377 * Inject session params into the current client. 378 */ 379 status_t injectSessionParams(const CameraMetadata& sessionParams); 380 381 protected: 382 status_t disconnectImpl(); 383 static status_t removeFwkOnlyKeys(CameraMetadata *request); 384 385 float getMaxPreviewFps(sp<camera3::Camera3OutputStreamInterface> stream); 386 387 static const size_t kDumpLockAttempts = 10; 388 static const size_t kDumpSleepDuration = 100000; // 0.10 sec 389 static const nsecs_t kActiveTimeout = 500000000; // 500 ms 390 static const nsecs_t kMinWarnInflightDuration = 5000000000; // 5 s 391 static const size_t kInFlightWarnLimit = 30; 392 static const size_t kInFlightWarnLimitHighSpeed = 256; // batch size 32 * pipe depth 8 393 static const nsecs_t kMinInflightDuration = 5000000000; // 5 s 394 static const nsecs_t kBaseGetBufferWait = 3000000000; // 3 sec. 395 396 struct RequestTrigger; 397 // minimal jpeg buffer size: 256KB + blob header 398 static const ssize_t kMinJpegBufferSize = camera3::MIN_JPEG_BUFFER_SIZE + 399 sizeof(aidl::android::hardware::camera::device::CameraBlob); 400 // Constant to use for stream ID when one doesn't exist 401 static const int NO_STREAM = -1; 402 403 std::shared_ptr<CameraServiceProxyWrapper> mCameraServiceProxyWrapper; 404 405 // A lock to enforce serialization on the input/configure side 406 // of the public interface. 407 // Only locked by public methods inherited from CameraDeviceBase. 408 // Not locked by methods guarded by mOutputLock, since they may act 409 // concurrently to the input/configure side of the interface. 410 // Must be locked before mLock if both will be locked by a method 411 Mutex mInterfaceLock; 412 413 // The main lock on internal state 414 Mutex mLock; 415 416 // Camera device ID 417 const std::string mId; 418 419 // Legacy camera client flag 420 bool mLegacyClient; 421 422 // Current stream configuration mode; 423 int mOperatingMode; 424 // Current session wide parameters 425 hardware::camera2::impl::CameraMetadataNative mSessionParams; 426 // Constant to use for no set operating mode 427 static const int NO_MODE = -1; 428 429 // Flag indicating is the current active stream configuration is constrained high speed. 430 bool mIsConstrainedHighSpeedConfiguration; 431 432 /**** Scope for mLock ****/ 433 434 class HalInterface : public camera3::Camera3StreamBufferFreedListener, 435 public camera3::BufferRecordsInterface { 436 public: HalInterface(bool useHalBufManager,bool supportOfflineProcessing)437 HalInterface(bool useHalBufManager, bool supportOfflineProcessing) : 438 mUseHalBufManager(useHalBufManager), 439 mIsReconfigurationQuerySupported(true), 440 mSupportOfflineProcessing(supportOfflineProcessing) 441 {}; 442 HalInterface(const HalInterface &other); 443 HalInterface(); 444 445 virtual IPCTransport getTransportType() const = 0; 446 447 // Returns true if constructed with a valid device or session, and not yet cleared 448 virtual bool valid() = 0; 449 450 // Reset this HalInterface object (does not call close()) 451 virtual void clear() = 0; 452 453 // Calls into the HAL interface 454 455 // Caller takes ownership of requestTemplate 456 virtual status_t constructDefaultRequestSettings(camera_request_template templateId, 457 /*out*/ camera_metadata_t **requestTemplate) = 0; 458 459 virtual status_t configureStreams(const camera_metadata_t * sessionParams, 460 /*inout*/ camera_stream_configuration_t * config, 461 const std::vector<uint32_t>& bufferSizes, int64_t logId) = 0; 462 463 // The injection camera configures the streams to hal. 464 virtual status_t configureInjectedStreams( 465 const camera_metadata_t* sessionParams, 466 /*inout*/ camera_stream_configuration_t* config, 467 const std::vector<uint32_t>& bufferSizes, 468 const CameraMetadata& cameraCharacteristics) = 0; 469 470 // When the call succeeds, the ownership of acquire fences in requests is transferred to 471 // HalInterface. More specifically, the current implementation will send the fence to 472 // HAL process and close the FD in cameraserver process. When the call fails, the ownership 473 // of the acquire fence still belongs to the caller. 474 virtual status_t processBatchCaptureRequests( 475 std::vector<camera_capture_request_t*>& requests, 476 /*out*/uint32_t* numRequestProcessed) = 0; 477 478 virtual status_t flush() = 0; 479 480 virtual status_t dump(int fd) = 0; 481 482 virtual status_t close() = 0; 483 484 virtual void signalPipelineDrain(const std::vector<int>& streamIds) = 0; 485 486 virtual bool isReconfigurationRequired(CameraMetadata& oldSessionParams, 487 CameraMetadata& newSessionParams) = 0; 488 489 virtual status_t repeatingRequestEnd(uint32_t frameNumber, 490 const std::vector<int32_t> &streamIds) = 0; 491 492 ///////////////////////////////////////////////////////////////////// 493 // Implements BufferRecordsInterface 494 495 std::pair<bool, uint64_t> getBufferId( 496 const buffer_handle_t& buf, int streamId) override; 497 498 uint64_t removeOneBufferCache(int streamId, const native_handle_t* handle) override; 499 500 status_t popInflightBuffer(int32_t frameNumber, int32_t streamId, 501 /*out*/ buffer_handle_t **buffer) override; 502 503 status_t pushInflightRequestBuffer( 504 uint64_t bufferId, buffer_handle_t* buf, int32_t streamId) override; 505 506 status_t popInflightRequestBuffer(uint64_t bufferId, 507 /*out*/ buffer_handle_t** buffer, 508 /*optional out*/ int32_t* streamId = nullptr) override; 509 510 ///////////////////////////////////////////////////////////////////// 511 512 //Check if a stream is hal buffer managed 513 bool isHalBufferManagedStream(int32_t streamId) const; 514 515 // Get a vector of (frameNumber, streamId) pair of currently inflight 516 // buffers 517 void getInflightBufferKeys(std::vector<std::pair<int32_t, int32_t>>* out); 518 519 // Get a vector of bufferId of currently inflight buffers 520 void getInflightRequestBufferKeys(std::vector<uint64_t>* out); 521 522 void onStreamReConfigured(int streamId); 523 524 protected: 525 526 // Return true if the input caches match what we have; otherwise false 527 bool verifyBufferIds(int32_t streamId, std::vector<uint64_t>& inBufIds); 528 529 template <typename OfflineSessionInfoT> verifyBufferCaches(const OfflineSessionInfoT * offlineSessionInfo,camera3::BufferRecords * bufferRecords)530 status_t verifyBufferCaches( 531 const OfflineSessionInfoT *offlineSessionInfo, camera3::BufferRecords *bufferRecords) { 532 // Validate buffer caches 533 std::vector<int32_t> streams; 534 streams.reserve(offlineSessionInfo->offlineStreams.size()); 535 for (auto offlineStream : offlineSessionInfo->offlineStreams) { 536 int32_t id = offlineStream.id; 537 streams.push_back(id); 538 // Verify buffer caches 539 std::vector<uint64_t> bufIds(offlineStream.circulatingBufferIds.begin(), 540 offlineStream.circulatingBufferIds.end()); 541 { 542 // Due to timing it is possible that we may not have any remaining pending 543 // capture requests that can update the caches on Hal side. This can result in 544 // buffer cache mismatch between the service and the Hal and must be accounted 545 // for. 546 std::lock_guard<std::mutex> l(mFreedBuffersLock); 547 for (const auto& it : mFreedBuffers) { 548 if (it.first == id) { 549 ALOGV("%s: stream ID %d buffer id %" PRIu64 " cache removal still " 550 "pending", __FUNCTION__, id, it.second); 551 const auto& cachedEntry = std::find(bufIds.begin(), bufIds.end(), 552 it.second); 553 if (cachedEntry != bufIds.end()) { 554 bufIds.erase(cachedEntry); 555 } else { 556 ALOGE("%s: stream ID %d buffer id %" PRIu64 " cache removal still " 557 "pending however buffer is no longer in the offline stream " 558 "info!", __FUNCTION__, id, it.second); 559 } 560 } 561 } 562 } 563 if (!verifyBufferIds(id, bufIds)) { 564 ALOGE("%s: stream ID %d buffer cache records mismatch!", __FUNCTION__, id); 565 return UNKNOWN_ERROR; 566 } 567 } 568 569 // Move buffer records 570 bufferRecords->takeBufferCaches(mBufferRecords, streams); 571 bufferRecords->takeInflightBufferMap(mBufferRecords); 572 bufferRecords->takeRequestedBufferMap(mBufferRecords); 573 return OK; 574 } 575 576 virtual void onBufferFreed(int streamId, const native_handle_t* handle) override; 577 578 std::mutex mFreedBuffersLock; 579 std::vector<std::pair<int, uint64_t>> mFreedBuffers; 580 581 // Keep track of buffer cache and inflight buffer records 582 camera3::BufferRecords mBufferRecords; 583 584 uint32_t mNextStreamConfigCounter = 1; 585 586 // TODO: This can be removed after flags::session_hal_buf_manager is removed 587 bool mUseHalBufManager = false; 588 std::set<int32_t > mHalBufManagedStreamIds; 589 bool mIsReconfigurationQuerySupported; 590 591 const bool mSupportOfflineProcessing; 592 }; // class HalInterface 593 594 sp<HalInterface> mInterface; 595 596 CameraMetadata mDeviceInfo; 597 bool mSupportNativeZoomRatio; 598 bool mIsCompositeJpegRDisabled; 599 std::unordered_map<std::string, CameraMetadata> mPhysicalDeviceInfoMap; 600 601 CameraMetadata mRequestTemplateCache[CAMERA_TEMPLATE_COUNT]; 602 603 struct Size { 604 uint32_t width; 605 uint32_t height; widthSize606 explicit Size(uint32_t w = 0, uint32_t h = 0) : width(w), height(h){} 607 }; 608 609 enum Status { 610 STATUS_ERROR, 611 STATUS_UNINITIALIZED, 612 STATUS_UNCONFIGURED, 613 STATUS_CONFIGURED, 614 STATUS_ACTIVE 615 } mStatus; 616 617 struct StatusInfo { 618 Status status; 619 bool isInternal; // status triggered by internal reconfigureCamera. 620 }; 621 622 bool mStatusIsInternal; 623 624 // Only clear mRecentStatusUpdates, mStatusWaiters from waitUntilStateThenRelock 625 Vector<StatusInfo> mRecentStatusUpdates; 626 int mStatusWaiters; 627 628 Condition mStatusChanged; 629 630 // Tracking cause of fatal errors when in STATUS_ERROR 631 std::string mErrorCause; 632 633 camera3::StreamSet mOutputStreams; 634 sp<camera3::Camera3Stream> mInputStream; 635 bool mIsInputStreamMultiResolution; 636 SessionStatsBuilder mSessionStatsBuilder; 637 // Map from stream group ID to physical cameras backing the stream group 638 std::map<int32_t, std::set<std::string>> mGroupIdPhysicalCameraMap; 639 640 int mNextStreamId; 641 bool mNeedConfig; 642 643 int mFakeStreamId; 644 645 // Whether to send state updates upstream 646 // Pause when doing transparent reconfiguration 647 bool mPauseStateNotify; 648 649 // Need to hold on to stream references until configure completes. 650 Vector<sp<camera3::Camera3StreamInterface> > mDeletedStreams; 651 652 // Whether the HAL will send partial result 653 bool mUsePartialResult; 654 655 // Number of partial results that will be delivered by the HAL. 656 uint32_t mNumPartialResults; 657 658 /**** End scope for mLock ****/ 659 660 bool mDeviceTimeBaseIsRealtime; 661 // The offset converting from clock domain of other subsystem 662 // (video/hardware composer) to that of camera. Assumption is that this 663 // offset won't change during the life cycle of the camera device. In other 664 // words, camera device shouldn't be open during CPU suspend. 665 nsecs_t mTimestampOffset; 666 667 class CaptureRequest : public LightRefBase<CaptureRequest> { 668 public: 669 PhysicalCameraSettingsList mSettingsList; 670 sp<camera3::Camera3Stream> mInputStream; 671 camera_stream_buffer_t mInputBuffer; 672 camera3::Size mInputBufferSize; 673 Vector<sp<camera3::Camera3OutputStreamInterface> > 674 mOutputStreams; 675 SurfaceMap mOutputSurfaces; 676 CaptureResultExtras mResultExtras; 677 // The number of requests that should be submitted to HAL at a time. 678 // For example, if batch size is 8, this request and the following 7 679 // requests will be submitted to HAL at a time. The batch size for 680 // the following 7 requests will be ignored by the request thread. 681 int mBatchSize; 682 // Whether this request is from a repeating or repeating burst. 683 bool mRepeating; 684 // Whether this request has ROTATE_AND_CROP_AUTO set, so needs both 685 // overriding of ROTATE_AND_CROP value and adjustment of coordinates 686 // in several other controls in both the request and the result 687 bool mRotateAndCropAuto; 688 // Indicates that the ROTATE_AND_CROP value within 'mSettingsList' was modified 689 // irrespective of the original value. 690 bool mRotateAndCropChanged = false; 691 // Whether this request has AUTOFRAMING_AUTO set, so need to override the AUTOFRAMING value 692 // in the capture request. 693 bool mAutoframingAuto; 694 // Indicates that the auto framing value within 'mSettingsList' was modified 695 bool mAutoframingChanged = false; 696 // Indicates that the camera test pattern setting is modified 697 bool mTestPatternChanged = false; 698 699 // Whether this capture request has its zoom ratio set to 1.0x before 700 // the framework overrides it for camera HAL consumption. 701 bool mZoomRatioIs1x; 702 // The systemTime timestamp when the request is created. 703 nsecs_t mRequestTimeNs; 704 705 // Whether this capture request's distortion correction update has 706 // been done. 707 bool mDistortionCorrectionUpdated = false; 708 // Whether this capture request's rotation and crop update has been 709 // done. 710 bool mRotationAndCropUpdated = false; 711 // Whether this capture request's autoframing has been done. 712 bool mAutoframingUpdated = false; 713 // Whether this capture request's zoom ratio update has been done. 714 bool mZoomRatioUpdated = false; 715 // Whether this max resolution capture request's crop / metering region update has been 716 // done. 717 bool mUHRCropAndMeteringRegionsUpdated = false; 718 }; 719 typedef List<sp<CaptureRequest> > RequestList; 720 721 status_t checkStatusOkToCaptureLocked(); 722 723 status_t convertMetadataListToRequestListLocked( 724 const List<const PhysicalCameraSettingsList> &metadataList, 725 const std::list<SurfaceMap> &surfaceMaps, 726 bool repeating, nsecs_t requestTimeNs, 727 /*out*/ 728 RequestList *requestList); 729 730 void convertToRequestList(List<const PhysicalCameraSettingsList>& requestsList, 731 std::list<SurfaceMap>& surfaceMaps, 732 const CameraMetadata& request); 733 734 status_t submitRequestsHelper(const List<const PhysicalCameraSettingsList> &requestsList, 735 const std::list<SurfaceMap> &surfaceMaps, 736 bool repeating, 737 int64_t *lastFrameNumber = NULL); 738 739 // lock to ensure only one processCaptureResult is called at a time. 740 Mutex mProcessCaptureResultLock; 741 742 /** 743 * Common initialization code shared by both HAL paths 744 * 745 * Must be called with mLock and mInterfaceLock held. 746 */ 747 status_t initializeCommonLocked(sp<CameraProviderManager> manager); 748 749 /** 750 * Update capture request list so that each batch size honors the batch_size_max report from 751 * the HAL. Set the batch size to output stream for buffer operations. 752 * 753 * Must be called with mLock held. 754 */ 755 virtual void applyMaxBatchSizeLocked( 756 RequestList* requestList, const sp<camera3::Camera3OutputStreamInterface>& stream) = 0; 757 758 struct LatestRequestInfo { 759 CameraMetadata requestSettings; 760 std::unordered_map<std::string, CameraMetadata> physicalRequestSettings; 761 int32_t inputStreamId = -1; 762 std::set<int32_t> outputStreamIds; 763 }; 764 765 /** 766 * Get the last request submitted to the hal by the request thread. 767 * 768 * Must be called with mLock held. 769 */ 770 virtual LatestRequestInfo getLatestRequestInfoLocked(); 771 772 virtual status_t injectionCameraInitialize(const std::string &injectCamId, 773 sp<CameraProviderManager> manager) = 0; 774 775 /** 776 * Update the current device status and wake all waiting threads. 777 * 778 * Must be called with mLock held. 779 */ 780 void internalUpdateStatusLocked(Status status); 781 782 /** 783 * Pause processing and flush everything, but don't tell the clients. 784 * This is for reconfiguring outputs transparently when according to the 785 * CameraDeviceBase interface we shouldn't need to. 786 * Must be called with mLock and mInterfaceLock both held. 787 */ 788 status_t internalPauseAndWaitLocked(nsecs_t maxExpectedDuration, 789 bool requestThreadInvocation); 790 791 /** 792 * Resume work after internalPauseAndWaitLocked() 793 * Must be called with mLock and mInterfaceLock both held. 794 */ 795 status_t internalResumeLocked(); 796 797 /** 798 * Wait until status tracker tells us we've transitioned to the target state 799 * set, which is either ACTIVE when active==true or IDLE (which is any 800 * non-ACTIVE state) when active==false. 801 * 802 * Needs to be called with mLock and mInterfaceLock held. This means there 803 * can ever only be one waiter at most. 804 * 805 * During the wait mLock is released. 806 * 807 */ 808 status_t waitUntilStateThenRelock(bool active, nsecs_t timeout, 809 bool requestThreadInvocation); 810 811 /** 812 * Implementation of waitUntilDrained. On success, will transition to IDLE state. 813 * 814 * Need to be called with mLock and mInterfaceLock held. 815 */ 816 status_t waitUntilDrainedLocked(nsecs_t maxExpectedDuration); 817 818 /** 819 * Do common work for setting up a streaming or single capture request. 820 * On success, will transition to ACTIVE if in IDLE. 821 */ 822 sp<CaptureRequest> setUpRequestLocked(const PhysicalCameraSettingsList &request, 823 const SurfaceMap &surfaceMap); 824 825 /** 826 * Build a CaptureRequest request from the CameraDeviceBase request 827 * settings. 828 */ 829 sp<CaptureRequest> createCaptureRequest(const PhysicalCameraSettingsList &request, 830 const SurfaceMap &surfaceMap); 831 832 /** 833 * Internally re-configure camera device using new session parameters. 834 * This will get triggered by the request thread. 835 */ 836 bool reconfigureCamera(const CameraMetadata& sessionParams, int clientStatusId); 837 838 /** 839 * Return true in case of any output or input abandoned streams, 840 * otherwise return false. 841 */ 842 bool checkAbandonedStreamsLocked(); 843 844 /** 845 * Filter stream session parameters and configure camera HAL. 846 */ 847 status_t filterParamsAndConfigureLocked(const CameraMetadata& sessionParams, 848 int operatingMode); 849 850 /** 851 * Take the currently-defined set of streams and configure the HAL to use 852 * them. This is a long-running operation (may be several hundered ms). 853 */ 854 status_t configureStreamsLocked(int operatingMode, 855 const CameraMetadata& sessionParams, bool notifyRequestThread = true); 856 857 /** 858 * Cancel stream configuration that did not finish successfully. 859 */ 860 void cancelStreamsConfigurationLocked(); 861 862 /** 863 * Add a fake stream to the current stream set as a workaround for 864 * not allowing 0 streams in the camera HAL spec. 865 */ 866 status_t addFakeStreamLocked(); 867 868 /** 869 * Remove a fake stream if the current config includes real streams. 870 */ 871 status_t tryRemoveFakeStreamLocked(); 872 873 /** 874 * Set device into an error state due to some fatal failure, and set an 875 * error message to indicate why. Only the first call's message will be 876 * used. The message is also sent to the log. 877 */ 878 void setErrorState(const char *fmt, ...) override; 879 void setErrorStateLocked(const char *fmt, ...) override; 880 void setErrorStateV(const char *fmt, va_list args); 881 void setErrorStateLockedV(const char *fmt, va_list args); 882 883 ///////////////////////////////////////////////////////////////////// 884 // Implements InflightRequestUpdateInterface 885 886 void onInflightEntryRemovedLocked(nsecs_t duration) override; 887 void checkInflightMapLengthLocked() override; 888 void onInflightMapFlushedLocked() override; 889 890 ///////////////////////////////////////////////////////////////////// 891 892 /** 893 * Debugging trylock/spin method 894 * Try to acquire a lock a few times with sleeps between before giving up. 895 */ 896 bool tryLockSpinRightRound(Mutex& lock); 897 898 /** 899 * Helper function to get the offset between MONOTONIC and BOOTTIME 900 * timestamp. 901 */ 902 static nsecs_t getMonoToBoottimeOffset(); 903 904 // Override rotate_and_crop control if needed 905 static bool overrideAutoRotateAndCrop(const sp<CaptureRequest> &request /*out*/, 906 int rotationOverride, 907 camera_metadata_enum_android_scaler_rotate_and_crop_t rotateAndCropOverride); 908 909 // Override auto framing control if needed 910 static bool overrideAutoframing(const sp<CaptureRequest> &request /*out*/, 911 camera_metadata_enum_android_control_autoframing_t autoframingOverride); 912 913 struct RequestTrigger { 914 // Metadata tag number, e.g. android.control.aePrecaptureTrigger 915 uint32_t metadataTag; 916 // Metadata value, e.g. 'START' or the trigger ID 917 int32_t entryValue; 918 919 // The last part of the fully qualified path, e.g. afTrigger getTagNameRequestTrigger920 const char *getTagName() const { 921 return get_camera_metadata_tag_name(metadataTag) ?: "NULL"; 922 } 923 924 // e.g. TYPE_BYTE, TYPE_INT32, etc. getTagTypeRequestTrigger925 int getTagType() const { 926 return get_camera_metadata_tag_type(metadataTag); 927 } 928 }; 929 930 /** 931 * Thread for managing capture request submission to HAL device. 932 */ 933 class RequestThread : public Thread { 934 935 public: 936 937 RequestThread(wp<Camera3Device> parent, 938 sp<camera3::StatusTracker> statusTracker, 939 sp<HalInterface> interface, 940 const Vector<int32_t>& sessionParamKeys, 941 bool useHalBufManager, 942 bool supportCameraMute, 943 int rotationOverride, 944 bool supportSettingsOverride); 945 ~RequestThread(); 946 947 void setNotificationListener(wp<NotificationListener> listener); 948 949 /** 950 * Call after stream (re)-configuration is completed. 951 */ 952 void configurationComplete(bool isConstrainedHighSpeed, 953 const CameraMetadata& sessionParams, 954 const std::map<int32_t, std::set<std::string>>& groupIdPhysicalCameraMap); 955 956 /** 957 * Set or clear the list of repeating requests. Does not block 958 * on either. Use waitUntilPaused to wait until request queue 959 * has emptied out. 960 */ 961 status_t setRepeatingRequests(const RequestList& requests, 962 /*out*/ 963 int64_t *lastFrameNumber = NULL); 964 status_t clearRepeatingRequests(/*out*/ 965 int64_t *lastFrameNumber = NULL); 966 967 status_t queueRequestList(List<sp<CaptureRequest> > &requests, 968 /*out*/ 969 int64_t *lastFrameNumber = NULL); 970 971 /** 972 * Remove all queued and repeating requests, and pending triggers 973 */ 974 status_t clear(/*out*/int64_t *lastFrameNumber = NULL); 975 976 /** 977 * Flush all pending requests in HAL. 978 */ 979 status_t flush(); 980 981 /** 982 * Queue a trigger to be dispatched with the next outgoing 983 * process_capture_request. The settings for that request only 984 * will be temporarily rewritten to add the trigger tag/value. 985 * Subsequent requests will not be rewritten (for this tag). 986 */ 987 status_t queueTrigger(RequestTrigger trigger[], size_t count); 988 989 /** 990 * Pause/unpause the capture thread. Doesn't block, so use 991 * waitUntilPaused to wait until the thread is paused. 992 */ 993 void setPaused(bool paused); 994 995 /** 996 * Set Hal buffer managed streams 997 * @param halBufferManagedStreams The streams for which hal buffer manager is enabled 998 * 999 */ 1000 void setHalBufferManagedStreams(const std::set<int32_t> &halBufferManagedStreams); 1001 1002 /** 1003 * Wait until thread processes the capture request with settings' 1004 * android.request.id == requestId. 1005 * 1006 * Returns TIMED_OUT in case the thread does not process the request 1007 * within the timeout. 1008 */ 1009 status_t waitUntilRequestProcessed(int32_t requestId, nsecs_t timeout); 1010 1011 /** 1012 * Shut down the thread. Shutdown is asynchronous, so thread may 1013 * still be running once this method returns. 1014 */ 1015 virtual void requestExit(); 1016 1017 /** 1018 * Get the latest request that was sent to the HAL 1019 * with process_capture_request. 1020 */ 1021 LatestRequestInfo getLatestRequestInfo() const; 1022 1023 /** 1024 * Returns true if the stream is a target of any queued or repeating 1025 * capture request 1026 */ 1027 bool isStreamPending(sp<camera3::Camera3StreamInterface>& stream); 1028 1029 /** 1030 * Returns true if the surface is a target of any queued or repeating 1031 * capture request 1032 */ 1033 bool isOutputSurfacePending(int streamId, size_t surfaceId); 1034 1035 // dump processCaptureRequest latency dumpCaptureRequestLatency(int fd,const char * name)1036 void dumpCaptureRequestLatency(int fd, const char* name) { 1037 mRequestLatency.dump(fd, name); 1038 } 1039 1040 void signalPipelineDrain(const std::vector<int>& streamIds); 1041 void resetPipelineDrain(); 1042 1043 void clearPreviousRequest(); 1044 1045 status_t setRotateAndCropAutoBehavior( 1046 camera_metadata_enum_android_scaler_rotate_and_crop_t rotateAndCropValue); 1047 1048 status_t setAutoframingAutoBehaviour( 1049 camera_metadata_enum_android_control_autoframing_t autoframingValue); 1050 1051 status_t setComposerSurface(bool composerSurfacePresent); 1052 1053 status_t setCameraMute(int32_t muteMode); 1054 1055 status_t setZoomOverride(int32_t zoomOverride); 1056 1057 status_t setHalInterface(sp<HalInterface> newHalInterface); 1058 1059 status_t setInjectedSessionParams(const CameraMetadata& sessionParams); 1060 1061 void injectSessionParams( 1062 const sp<CaptureRequest> &request, 1063 const CameraMetadata& injectedSessionParams); 1064 1065 /** 1066 * signal mLatestRequestmutex 1067 **/ 1068 void wakeupLatestRequest(bool latestRequestFailed, int32_t latestRequestId); 1069 1070 protected: 1071 1072 virtual bool threadLoop(); 1073 1074 static const std::string& getId(const wp<Camera3Device> &device); 1075 1076 status_t queueTriggerLocked(RequestTrigger trigger); 1077 // Mix-in queued triggers into this request 1078 int32_t insertTriggers(const sp<CaptureRequest> &request); 1079 // Purge the queued triggers from this request, 1080 // restoring the old field values for those tags. 1081 status_t removeTriggers(const sp<CaptureRequest> &request); 1082 1083 // HAL workaround: Make sure a trigger ID always exists if 1084 // a trigger does 1085 status_t addFakeTriggerIds(const sp<CaptureRequest> &request); 1086 1087 // Override rotate_and_crop control if needed; returns true if the current value was changed 1088 bool overrideAutoRotateAndCrop(const sp<CaptureRequest> &request /*out*/); 1089 1090 // Override autoframing control if needed; returns true if the current value was changed 1091 bool overrideAutoframing(const sp<CaptureRequest> &request); 1092 1093 // Override test_pattern control if needed for camera mute; returns true 1094 // if the current value was changed 1095 bool overrideTestPattern(const sp<CaptureRequest> &request); 1096 1097 // Override settings override if needed for lower zoom latency; return 1098 // true if the current value was changed 1099 bool overrideSettingsOverride(const sp<CaptureRequest> &request); 1100 1101 static const nsecs_t kRequestTimeout = 50e6; // 50 ms 1102 1103 // TODO: does this need to be adjusted for long exposure requests? 1104 static const nsecs_t kRequestSubmitTimeout = 500e6; // 500 ms 1105 1106 // Used to prepare a batch of requests. 1107 struct NextRequest { 1108 sp<CaptureRequest> captureRequest; 1109 camera_capture_request_t halRequest; 1110 Vector<camera_stream_buffer_t> outputBuffers; 1111 bool submitted; 1112 }; 1113 1114 // Wait for the next batch of requests and put them in mNextRequests. mNextRequests will 1115 // be empty if it times out. 1116 void waitForNextRequestBatch(); 1117 1118 // Waits for a request, or returns NULL if times out. Must be called with mRequestLock hold. 1119 sp<CaptureRequest> waitForNextRequestLocked(); 1120 1121 // Prepare HAL requests and output buffers in mNextRequests. Return TIMED_OUT if getting any 1122 // output buffer timed out. If an error is returned, the caller should clean up the pending 1123 // request batch. 1124 status_t prepareHalRequests(); 1125 1126 // Return buffers, etc, for requests in mNextRequests that couldn't be fully constructed and 1127 // send request errors if sendRequestError is true. The buffers will be returned in the 1128 // ERROR state to mark them as not having valid data. mNextRequests will be cleared. 1129 void cleanUpFailedRequests(bool sendRequestError); 1130 1131 // Stop the repeating request if any of its output streams is abandoned. 1132 void checkAndStopRepeatingRequest(); 1133 1134 // Release physical camera settings and camera id resources. 1135 void cleanupPhysicalSettings(sp<CaptureRequest> request, 1136 /*out*/camera_capture_request_t *halRequest); 1137 1138 // Pause handling 1139 bool waitIfPaused(); 1140 void unpauseForNewRequests(); 1141 1142 // Relay error to parent device object setErrorState 1143 void setErrorState(const char *fmt, ...); 1144 1145 // If the input request is in mRepeatingRequests. Must be called with mRequestLock hold 1146 bool isRepeatingRequestLocked(const sp<CaptureRequest>&); 1147 1148 // Clear repeating requests. Must be called with mRequestLock held. 1149 status_t clearRepeatingRequestsLocked(/*out*/ int64_t *lastFrameNumber = NULL); 1150 1151 // send request in mNextRequests to HAL in a batch. Return true = sucssess 1152 bool sendRequestsBatch(); 1153 1154 // Calculate the expected (minimum, maximum, isFixedFps) duration info for a request 1155 struct ExpectedDurationInfo { 1156 nsecs_t minDuration; 1157 nsecs_t maxDuration; 1158 bool isFixedFps; 1159 }; 1160 ExpectedDurationInfo calculateExpectedDurationRange( 1161 const camera_metadata_t *request); 1162 1163 // Check and update latest session parameters based on the current request settings. 1164 bool updateSessionParameters(const CameraMetadata& settings); 1165 1166 // Check whether FPS range session parameter re-configuration is needed in constrained 1167 // high speed recording camera sessions. 1168 bool skipHFRTargetFPSUpdate(int32_t tag, const camera_metadata_ro_entry_t& newEntry, 1169 const camera_metadata_entry_t& currentEntry); 1170 1171 // Update next request sent to HAL 1172 void updateNextRequest(NextRequest& nextRequest); 1173 1174 wp<Camera3Device> mParent; 1175 wp<camera3::StatusTracker> mStatusTracker; 1176 sp<HalInterface> mInterface; 1177 1178 wp<NotificationListener> mListener; 1179 1180 const std::string mId; // The camera ID 1181 int mStatusId; // The RequestThread's component ID for 1182 // status tracking 1183 1184 Mutex mRequestLock; 1185 Condition mRequestSignal; 1186 bool mRequestClearing; 1187 1188 Condition mRequestSubmittedSignal; 1189 RequestList mRequestQueue; 1190 RequestList mRepeatingRequests; 1191 bool mFirstRepeating; 1192 // The next batch of requests being prepped for submission to the HAL, no longer 1193 // on the request queue. Read-only even with mRequestLock held, outside 1194 // of threadLoop 1195 Vector<NextRequest> mNextRequests; 1196 1197 // To protect flush() and sending a request batch to HAL. 1198 Mutex mFlushLock; 1199 1200 bool mReconfigured; 1201 1202 // Used by waitIfPaused, waitForNextRequest, waitUntilPaused, and signalPipelineDrain 1203 Mutex mPauseLock; 1204 bool mDoPause; 1205 Condition mDoPauseSignal; 1206 bool mPaused; 1207 bool mNotifyPipelineDrain; 1208 std::vector<int> mStreamIdsToBeDrained; 1209 1210 sp<CaptureRequest> mPrevRequest; 1211 int32_t mPrevTriggers; 1212 std::set<std::string> mPrevCameraIdsWithZoom; 1213 1214 uint32_t mFrameNumber; 1215 1216 mutable Mutex mLatestRequestMutex; 1217 Condition mLatestRequestSignal; 1218 // android.request.id for latest process_capture_request 1219 int32_t mLatestRequestId; 1220 int32_t mLatestFailedRequestId; 1221 LatestRequestInfo mLatestRequestInfo; 1222 1223 typedef KeyedVector<uint32_t/*tag*/, RequestTrigger> TriggerMap; 1224 Mutex mTriggerMutex; 1225 TriggerMap mTriggerMap; 1226 TriggerMap mTriggerRemovedMap; 1227 TriggerMap mTriggerReplacedMap; 1228 uint32_t mCurrentAfTriggerId; 1229 uint32_t mCurrentPreCaptureTriggerId; 1230 camera_metadata_enum_android_scaler_rotate_and_crop_t mRotateAndCropOverride; 1231 camera_metadata_enum_android_control_autoframing_t mAutoframingOverride; 1232 bool mComposerOutput; 1233 int32_t mCameraMute; // 0 = no mute, otherwise the TEST_PATTERN_MODE to use 1234 int32_t mSettingsOverride; // -1 = use original, otherwise 1235 // the settings override to use. 1236 1237 int64_t mRepeatingLastFrameNumber; 1238 1239 // Flag indicating if we should prepare video stream for video requests. 1240 bool mPrepareVideoStream; 1241 1242 bool mConstrainedMode; 1243 1244 static const int32_t kRequestLatencyBinSize = 40; // in ms 1245 CameraLatencyHistogram mRequestLatency; 1246 1247 Vector<int32_t> mSessionParamKeys; 1248 CameraMetadata mLatestSessionParams; 1249 CameraMetadata mInjectedSessionParams; 1250 bool mForceNewRequestAfterReconfigure; 1251 1252 std::map<int32_t, std::set<std::string>> mGroupIdPhysicalCameraMap; 1253 1254 bool mUseHalBufManager = false; 1255 std::set<int32_t > mHalBufManagedStreamIds; 1256 const bool mSupportCameraMute; 1257 const bool mRotationOverride; 1258 const bool mSupportSettingsOverride; 1259 int32_t mVndkVersion = -1; 1260 }; 1261 1262 virtual sp<RequestThread> createNewRequestThread(wp<Camera3Device> /*parent*/, 1263 sp<camera3::StatusTracker> /*statusTracker*/, 1264 sp<HalInterface> /*interface*/, 1265 const Vector<int32_t>& /*sessionParamKeys*/, 1266 bool /*useHalBufManager*/, 1267 bool /*supportCameraMute*/, 1268 int /*rotationOverride*/, 1269 bool /*supportSettingsOverride*/) = 0; 1270 1271 sp<RequestThread> mRequestThread; 1272 1273 /** 1274 * In-flight queue for tracking completion of capture requests. 1275 */ 1276 std::mutex mInFlightLock; 1277 camera3::InFlightRequestMap mInFlightMap; 1278 nsecs_t mExpectedInflightDuration = 0; 1279 int64_t mLastCompletedRegularFrameNumber = -1; 1280 int64_t mLastCompletedReprocessFrameNumber = -1; 1281 int64_t mLastCompletedZslFrameNumber = -1; 1282 // End of mInFlightLock protection scope 1283 1284 int mInFlightStatusId; // const after initialize 1285 1286 status_t registerInFlight(uint32_t frameNumber, 1287 int32_t numBuffers, CaptureResultExtras resultExtras, bool hasInput, 1288 bool callback, nsecs_t minExpectedDuration, nsecs_t maxExpectedDuration, 1289 bool isFixedFps, const std::set<std::set<std::string>>& physicalCameraIds, 1290 bool isStillCapture, bool isZslCapture, bool rotateAndCropAuto, bool autoframingAuto, 1291 const std::set<std::string>& cameraIdsWithZoom, bool useZoomRatio, 1292 const SurfaceMap& outputSurfaces, nsecs_t requestTimeNs); 1293 1294 /** 1295 * Tracking for idle detection 1296 */ 1297 sp<camera3::StatusTracker> mStatusTracker; 1298 1299 /** 1300 * Graphic buffer manager for output streams. Each device has a buffer manager, which is used 1301 * by the output streams to get and return buffers if these streams are registered to this 1302 * buffer manager. 1303 */ 1304 sp<camera3::Camera3BufferManager> mBufferManager; 1305 1306 /** 1307 * Thread for preparing streams 1308 */ 1309 class PreparerThread : private Thread, public virtual RefBase { 1310 public: 1311 PreparerThread(); 1312 ~PreparerThread(); 1313 1314 void setNotificationListener(wp<NotificationListener> listener); 1315 1316 /** 1317 * Queue up a stream to be prepared. Streams are processed by a background thread in FIFO 1318 * order. Pre-allocate up to maxCount buffers for the stream, or the maximum number needed 1319 * for the pipeline if maxCount is ALLOCATE_PIPELINE_MAX. 1320 */ 1321 status_t prepare(int maxCount, sp<camera3::Camera3StreamInterface>& stream); 1322 1323 /** 1324 * Cancel all current and pending stream preparation 1325 */ 1326 status_t clear(); 1327 1328 /** 1329 * Pause all preparation activities 1330 */ 1331 void pause(); 1332 1333 /** 1334 * Resume preparation activities 1335 */ 1336 status_t resume(); 1337 1338 private: 1339 Mutex mLock; 1340 Condition mThreadActiveSignal; 1341 1342 virtual bool threadLoop(); 1343 1344 // Guarded by mLock 1345 1346 wp<NotificationListener> mListener; 1347 std::list<std::tuple<int, sp<camera3::Camera3StreamInterface>>> mPendingStreams; 1348 bool mActive; 1349 bool mCancelNow; 1350 1351 // Only accessed by threadLoop and the destructor 1352 1353 sp<camera3::Camera3StreamInterface> mCurrentStream; 1354 int mCurrentMaxCount; 1355 bool mCurrentPrepareComplete; 1356 }; 1357 sp<PreparerThread> mPreparerThread; 1358 1359 /** 1360 * Output result queue and current HAL device 3A state 1361 */ 1362 1363 // Lock for output side of device 1364 std::mutex mOutputLock; 1365 1366 /**** Scope for mOutputLock ****/ 1367 // the minimal frame number of the next non-reprocess result 1368 uint32_t mNextResultFrameNumber; 1369 // the minimal frame number of the next reprocess result 1370 uint32_t mNextReprocessResultFrameNumber; 1371 // the minimal frame number of the next ZSL still capture result 1372 uint32_t mNextZslStillResultFrameNumber; 1373 // the minimal frame number of the next non-reprocess shutter 1374 uint32_t mNextShutterFrameNumber; 1375 // the minimal frame number of the next reprocess shutter 1376 uint32_t mNextReprocessShutterFrameNumber; 1377 // the minimal frame number of the next ZSL still capture shutter 1378 uint32_t mNextZslStillShutterFrameNumber; 1379 std::list<CaptureResult> mResultQueue; 1380 std::condition_variable mResultSignal; 1381 wp<NotificationListener> mListener; 1382 1383 /**** End scope for mOutputLock ****/ 1384 1385 /**** Scope for mInFlightLock ****/ 1386 1387 // Remove the in-flight map entry of the given index from mInFlightMap. 1388 // It must only be called with mInFlightLock held. 1389 void removeInFlightMapEntryLocked(int idx); 1390 1391 // Remove all in-flight requests and return all buffers. 1392 // This is used after HAL interface is closed to cleanup any request/buffers 1393 // not returned by HAL. 1394 void flushInflightRequests(); 1395 1396 /**** End scope for mInFlightLock ****/ 1397 1398 /** 1399 * Distortion correction support 1400 */ 1401 // Map from camera IDs to its corresponding distortion mapper. Only contains 1402 // 1 ID if the device isn't a logical multi-camera. Otherwise contains both 1403 // logical camera and its physical subcameras. 1404 std::unordered_map<std::string, camera3::DistortionMapper> mDistortionMappers; 1405 1406 /** 1407 * Zoom ratio mapper support 1408 */ 1409 std::unordered_map<std::string, camera3::ZoomRatioMapper> mZoomRatioMappers; 1410 1411 /** 1412 * UHR request crop / metering region mapper support 1413 */ 1414 std::unordered_map<std::string, camera3::UHRCropAndMeteringRegionMapper> 1415 mUHRCropAndMeteringRegionMappers; 1416 1417 /** 1418 * RotateAndCrop mapper support 1419 */ 1420 std::unordered_map<std::string, camera3::RotateAndCropMapper> mRotateAndCropMappers; 1421 1422 // Debug tracker for metadata tag value changes 1423 // - Enabled with the -m <taglist> option to dumpsys, such as 1424 // dumpsys -m android.control.aeState,android.control.aeMode 1425 // - Disabled with -m off 1426 // - dumpsys -m 3a is a shortcut for ae/af/awbMode, State, and Triggers 1427 TagMonitor mTagMonitor; 1428 1429 void monitorMetadata(TagMonitor::eventSource source, int64_t frameNumber, 1430 nsecs_t timestamp, const CameraMetadata& metadata, 1431 const std::unordered_map<std::string, CameraMetadata>& physicalMetadata, 1432 const camera_stream_buffer_t *outputBuffers, uint32_t numOutputBuffers, 1433 int32_t inputStreamId); 1434 1435 // Collect any statistics that are based on the stream of capture requests sent 1436 // to the HAL 1437 void collectRequestStats(int64_t frameNumber, const CameraMetadata& request); 1438 1439 metadata_vendor_id_t mVendorTagId; 1440 1441 // Cached last requested template id 1442 int mLastTemplateId; 1443 1444 // Synchronizes access to status tracker between inflight updates and disconnect. 1445 // b/79972865 1446 Mutex mTrackerLock; 1447 1448 // Whether HAL request buffers through requestStreamBuffers API 1449 bool mUseHalBufManager = false; 1450 std::set<int32_t > mHalBufManagedStreamIds; 1451 bool mSessionHalBufManager = false; 1452 // Lock to ensure requestStreamBuffers() callbacks are serialized 1453 std::mutex mRequestBufferInterfaceLock; 1454 1455 // The state machine to control when requestStreamBuffers should allow 1456 // HAL to request buffers. 1457 enum RequestBufferState { 1458 /** 1459 * This is the initial state. 1460 * requestStreamBuffers call will return FAILED_CONFIGURING in this state. 1461 * Will switch to RB_STATUS_READY after a successful configureStreams or 1462 * processCaptureRequest call. 1463 */ 1464 RB_STATUS_STOPPED, 1465 1466 /** 1467 * requestStreamBuffers call will proceed in this state. 1468 * When device is asked to stay idle via waitUntilStateThenRelock() call: 1469 * - Switch to RB_STATUS_STOPPED if there is no inflight requests and 1470 * request thread is paused. 1471 * - Switch to RB_STATUS_PENDING_STOP otherwise 1472 */ 1473 RB_STATUS_READY, 1474 1475 /** 1476 * requestStreamBuffers call will proceed in this state. 1477 * Switch to RB_STATUS_STOPPED when all inflight requests are fulfilled 1478 * and request thread is paused 1479 */ 1480 RB_STATUS_PENDING_STOP, 1481 }; 1482 1483 class RequestBufferStateMachine { 1484 public: 1485 status_t initialize(sp<camera3::StatusTracker> statusTracker); 1486 1487 status_t deInit(); 1488 1489 // Return if the state machine currently allows for requestBuffers 1490 // If the state allows for it, mRequestBufferOngoing will be set to true 1491 // and caller must call endRequestBuffer() later to unset the flag 1492 bool startRequestBuffer(); 1493 void endRequestBuffer(); 1494 1495 // Events triggered by application API call 1496 void onStreamsConfigured(); 1497 void onWaitUntilIdle(); 1498 1499 // Events usually triggered by hwBinder processCaptureResult callback thread 1500 // But can also be triggered on request thread for failed request, or on 1501 // hwbinder notify callback thread for shutter/error callbacks 1502 void onInflightMapEmpty(); 1503 1504 // Events triggered by RequestThread 1505 void onSubmittingRequest(); 1506 void onRequestThreadPaused(); 1507 1508 // Events triggered by successful switchToOffline call 1509 // Return true is there is no ongoing requestBuffer call. 1510 bool onSwitchToOfflineSuccess(); 1511 1512 private: 1513 void notifyTrackerLocked(bool active); 1514 1515 // Switch to STOPPED state and return true if all conditions allows for it. 1516 // Otherwise do nothing and return false. 1517 bool checkSwitchToStopLocked(); 1518 1519 std::mutex mLock; 1520 RequestBufferState mStatus = RB_STATUS_STOPPED; 1521 1522 bool mRequestThreadPaused = true; 1523 bool mInflightMapEmpty = true; 1524 bool mRequestBufferOngoing = false; 1525 bool mSwitchedToOffline = false; 1526 1527 wp<camera3::StatusTracker> mStatusTracker; 1528 int mRequestBufferStatusId; 1529 } mRequestBufferSM; 1530 1531 // Fix up result metadata for monochrome camera. 1532 bool mNeedFixupMonochromeTags; 1533 1534 // Whether HAL supports offline processing capability. 1535 bool mSupportOfflineProcessing = false; 1536 1537 // Whether the HAL supports camera muting via test pattern 1538 bool mSupportCameraMute = false; 1539 // Whether the HAL supports SOLID_COLOR or BLACK if mSupportCameraMute is true 1540 bool mSupportTestPatternSolidColor = false; 1541 // Whether the HAL supports zoom settings override 1542 bool mSupportZoomOverride = false; 1543 1544 // Whether the camera framework overrides the device characteristics for 1545 // performance class. 1546 bool mOverrideForPerfClass; 1547 1548 // Whether the camera framework overrides the device characteristics for 1549 // app compatibility reasons. 1550 int mRotationOverride; 1551 camera_metadata_enum_android_scaler_rotate_and_crop_t mRotateAndCropOverride; 1552 bool mComposerOutput; 1553 1554 // Auto framing override value 1555 camera_metadata_enum_android_control_autoframing mAutoframingOverride; 1556 1557 // Initial camera mute state stored before the request thread 1558 // is active. 1559 bool mCameraMuteInitial = false; 1560 1561 // Settings override value 1562 int32_t mSettingsOverride; // -1 = use original, otherwise 1563 // the settings override to use. 1564 1565 // Current active physical id of the logical multi-camera, if any 1566 std::string mActivePhysicalId; 1567 1568 // The current minimum expected frame duration based on AE_TARGET_FPS_RANGE 1569 nsecs_t mMinExpectedDuration = 0; 1570 // Whether the camera device runs at fixed frame rate based on AE_MODE and 1571 // AE_TARGET_FPS_RANGE 1572 bool mIsFixedFps = false; 1573 1574 // Flag to indicate that we shouldn't forward extension related metadata 1575 bool mSupportsExtensionKeys = false; 1576 1577 // Injection camera related methods. 1578 class Camera3DeviceInjectionMethods : public virtual RefBase { 1579 public: 1580 Camera3DeviceInjectionMethods(wp<Camera3Device> parent); 1581 1582 ~Camera3DeviceInjectionMethods(); 1583 1584 // Injection camera will replace the internal camera and configure streams 1585 // when device is IDLE and request thread is paused. 1586 status_t injectCamera( 1587 camera3::camera_stream_configuration& injectionConfig, 1588 const std::vector<uint32_t>& injectionBufferSizes); 1589 1590 // Stop the injection camera and switch back to backup hal interface. 1591 status_t stopInjection(); 1592 1593 bool isInjecting(); 1594 1595 bool isStreamConfigCompleteButNotInjected(); 1596 1597 const std::string& getInjectedCamId() const; 1598 1599 void getInjectionConfig(/*out*/ camera3::camera_stream_configuration* injectionConfig, 1600 /*out*/ std::vector<uint32_t>* injectionBufferSizes); 1601 1602 // When the streaming configuration is completed and the camera device is active, but the 1603 // injection camera has not yet been injected, the streaming configuration of the internal 1604 // camera will be stored first. 1605 void storeInjectionConfig( 1606 const camera3::camera_stream_configuration& injectionConfig, 1607 const std::vector<uint32_t>& injectionBufferSizes); 1608 1609 protected: 1610 // Configure the streams of injection camera, it need wait until the 1611 // output streams are created and configured to the original camera before 1612 // proceeding. 1613 status_t injectionConfigureStreams( 1614 camera3::camera_stream_configuration& injectionConfig, 1615 const std::vector<uint32_t>& injectionBufferSizes); 1616 1617 // Disconnect the injection camera and delete the hal interface. 1618 void injectionDisconnectImpl(); 1619 1620 // Use injection camera hal interface to replace and backup original 1621 // camera hal interface. 1622 virtual status_t replaceHalInterface(sp<HalInterface> /*newHalInterface*/, 1623 bool /*keepBackup*/) = 0; 1624 1625 wp<Camera3Device> mParent; 1626 1627 // Backup of the original camera hal interface. 1628 sp<HalInterface> mBackupHalInterface; 1629 1630 // Generated injection camera hal interface. 1631 sp<HalInterface> mInjectedCamHalInterface; 1632 1633 // The flag indicates that the stream configuration is complete, the camera device is 1634 // active, but the injection camera has not yet been injected. 1635 bool mIsStreamConfigCompleteButNotInjected = false; 1636 1637 // Copy the configuration of the internal camera. 1638 camera3::camera_stream_configuration mInjectionConfig; 1639 1640 // Copy the streams of the internal camera. 1641 Vector<camera3::camera_stream_t*> mInjectionStreams; 1642 1643 // Copy the bufferSizes of the output streams of the internal camera. 1644 std::vector<uint32_t> mInjectionBufferSizes; 1645 1646 // Synchronizes access to injection camera between initialize and 1647 // disconnect. 1648 Mutex mInjectionLock; 1649 1650 // The injection camera ID. 1651 std::string mInjectedCamId; 1652 }; 1653 1654 virtual sp<Camera3DeviceInjectionMethods> 1655 createCamera3DeviceInjectionMethods(wp<Camera3Device>) = 0; 1656 1657 sp<Camera3DeviceInjectionMethods> mInjectionMethods; 1658 1659 void overrideStreamUseCaseLocked(); 1660 status_t deriveAndSetTransformLocked(camera3::Camera3OutputStreamInterface& stream, 1661 int mirrorMode, int surfaceId); 1662 1663 1664 }; // class Camera3Device 1665 1666 }; // namespace android 1667 1668 #endif 1669