xref: /aosp_15_r20/frameworks/av/services/camera/libcameraservice/device3/Camera3Device.h (revision ec779b8e0859a360c3d303172224686826e6e0e1)
1 /*
2  * Copyright (C) 2013-2018 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #ifndef ANDROID_SERVERS_CAMERA3DEVICE_H
18 #define ANDROID_SERVERS_CAMERA3DEVICE_H
19 
20 #include <utility>
21 #include <unordered_map>
22 #include <set>
23 #include <tuple>
24 
25 #include <utils/Condition.h>
26 #include <utils/Errors.h>
27 #include <utils/List.h>
28 #include <utils/Mutex.h>
29 #include <utils/Thread.h>
30 #include <utils/KeyedVector.h>
31 #include <utils/Timers.h>
32 
33 #include <camera/CaptureResult.h>
34 #include <gui/Flags.h>
35 
36 #include "CameraServiceWatchdog.h"
37 #include <aidl/android/hardware/camera/device/CameraBlob.h>
38 
39 #include "common/CameraDeviceBase.h"
40 #include "common/DepthPhotoProcessor.h"
41 #include "device3/BufferUtils.h"
42 #include "device3/StatusTracker.h"
43 #include "device3/Camera3BufferManager.h"
44 #include "device3/DistortionMapper.h"
45 #include "device3/ZoomRatioMapper.h"
46 #include "device3/RotateAndCropMapper.h"
47 #include "device3/UHRCropAndMeteringRegionMapper.h"
48 #include "device3/InFlightRequest.h"
49 #include "device3/Camera3OutputInterface.h"
50 #include "device3/Camera3OfflineSession.h"
51 #include "device3/Camera3StreamInterface.h"
52 #include "utils/AttributionAndPermissionUtils.h"
53 #include "utils/TagMonitor.h"
54 #include "utils/IPCTransport.h"
55 #include "utils/LatencyHistogram.h"
56 #include "utils/CameraServiceProxyWrapper.h"
57 #include <camera_metadata_hidden.h>
58 
59 using android::camera3::camera_capture_request_t;
60 using android::camera3::camera_request_template;
61 using android::camera3::camera_stream_buffer_t;
62 using android::camera3::camera_stream_configuration_t;
63 using android::camera3::camera_stream_configuration_mode_t;
64 using android::camera3::CAMERA_TEMPLATE_COUNT;
65 using android::camera3::OutputStreamInfo;
66 using android::camera3::SurfaceHolder;
67 
68 namespace android {
69 
70 namespace camera3 {
71 
72 class Camera3Stream;
73 class Camera3ZslStream;
74 class Camera3StreamInterface;
75 
76 } // namespace camera3
77 
78 /**
79  * CameraDevice for HAL devices with version CAMERA_DEVICE_API_VERSION_3_0 or higher.
80  */
81 class Camera3Device :
82             public CameraDeviceBase,
83             public camera3::SetErrorInterface,
84             public camera3::InflightRequestUpdateInterface,
85             public camera3::RequestBufferInterface,
86             public camera3::FlushBufferInterface,
87             public AttributionAndPermissionUtilsEncapsulator {
88   friend class HidlCamera3Device;
89   friend class AidlCamera3Device;
90   public:
91 
92     explicit Camera3Device(std::shared_ptr<CameraServiceProxyWrapper>& cameraServiceProxyWrapper,
93             std::shared_ptr<AttributionAndPermissionUtils> attributionAndPermissionUtils,
94             const std::string& id, bool overrideForPerfClass, int rotationOverride,
95             bool legacyClient = false);
96 
97     virtual ~Camera3Device();
98     // Delete and optionally close native handles and clear the input vector afterward
99     static void cleanupNativeHandles(
100             std::vector<native_handle_t*> *handles, bool closeFd = false);
101 
getTransportType()102     virtual IPCTransport getTransportType() const override {
103         return mInterface->getTransportType();
104     }
105 
isHalBufferManagedStream(int32_t streamId)106     bool isHalBufferManagedStream(int32_t streamId) const {
107         return mInterface->isHalBufferManagedStream(streamId);
108     };
109 
110     /**
111      * CameraDeviceBase interface
112      */
113 
114     const std::string& getId() const override;
115 
getVendorTagId()116     metadata_vendor_id_t getVendorTagId() const override { return mVendorTagId; }
117 
118     // Watchdog thread
119     sp<CameraServiceWatchdog> mCameraServiceWatchdog;
120 
121     // Transitions to idle state on success.
122     virtual status_t initialize(sp<CameraProviderManager> /*manager*/,
123             const std::string& /*monitorTags*/) = 0;
124 
125     status_t disconnect() override;
126     status_t dump(int fd, const Vector<String16> &args) override;
127     status_t startWatchingTags(const std::string &tags) override;
128     status_t stopWatchingTags() override;
129     status_t dumpWatchedEventsToVector(std::vector<std::string> &out) override;
130     const CameraMetadata& info() const override;
131     const CameraMetadata& infoPhysical(const std::string& physicalId) const override;
isCompositeJpegRDisabled()132     bool isCompositeJpegRDisabled() const override { return mIsCompositeJpegRDisabled; };
133 
134     // Capture and setStreamingRequest will configure streams if currently in
135     // idle state
136     status_t capture(CameraMetadata &request, int64_t *lastFrameNumber = NULL) override;
137     status_t captureList(const List<const PhysicalCameraSettingsList> &requestsList,
138             const std::list<SurfaceMap> &surfaceMaps,
139             int64_t *lastFrameNumber = NULL) override;
140     status_t setStreamingRequest(const CameraMetadata &request,
141             int64_t *lastFrameNumber = NULL) override;
142     status_t setStreamingRequestList(const List<const PhysicalCameraSettingsList> &requestsList,
143             const std::list<SurfaceMap> &surfaceMaps,
144             int64_t *lastFrameNumber = NULL) override;
145     status_t clearStreamingRequest(int64_t *lastFrameNumber = NULL) override;
146 
147     status_t waitUntilRequestReceived(int32_t requestId, nsecs_t timeout) override;
148 
149     // Actual stream creation/deletion is delayed until first request is submitted
150     // If adding streams while actively capturing, will pause device before adding
151     // stream, reconfiguring device, and unpausing. If the client create a stream
152     // with nullptr consumer surface, the client must then call setConsumers()
153     // and finish the stream configuration before starting output streaming.
154     status_t createStream(sp<Surface> consumer,
155             uint32_t width, uint32_t height, int format,
156             android_dataspace dataSpace, camera_stream_rotation_t rotation, int *id,
157             const std::string& physicalCameraId,
158             const std::unordered_set<int32_t> &sensorPixelModesUsed,
159             std::vector<int> *surfaceIds = nullptr,
160             int streamSetId = camera3::CAMERA3_STREAM_SET_ID_INVALID,
161             bool isShared = false, bool isMultiResolution = false,
162             uint64_t consumerUsage = 0,
163             int64_t dynamicRangeProfile =
164             ANDROID_REQUEST_AVAILABLE_DYNAMIC_RANGE_PROFILES_MAP_STANDARD,
165             int64_t streamUseCase = ANDROID_SCALER_AVAILABLE_STREAM_USE_CASES_DEFAULT,
166             int timestampBase = OutputConfiguration::TIMESTAMP_BASE_DEFAULT,
167             int mirrorMode = OutputConfiguration::MIRROR_MODE_AUTO,
168             int32_t colorSpace = ANDROID_REQUEST_AVAILABLE_COLOR_SPACE_PROFILES_MAP_UNSPECIFIED,
169             bool useReadoutTimestamp = false)
170             override;
171 
172     status_t createStream(const std::vector<SurfaceHolder>& consumers,
173             bool hasDeferredConsumer, uint32_t width, uint32_t height, int format,
174             android_dataspace dataSpace, camera_stream_rotation_t rotation, int *id,
175             const std::string& physicalCameraId,
176             const std::unordered_set<int32_t> &sensorPixelModesUsed,
177             std::vector<int> *surfaceIds = nullptr,
178             int streamSetId = camera3::CAMERA3_STREAM_SET_ID_INVALID,
179             bool isShared = false, bool isMultiResolution = false,
180             uint64_t consumerUsage = 0,
181             int64_t dynamicRangeProfile =
182             ANDROID_REQUEST_AVAILABLE_DYNAMIC_RANGE_PROFILES_MAP_STANDARD,
183             int64_t streamUseCase = ANDROID_SCALER_AVAILABLE_STREAM_USE_CASES_DEFAULT,
184             int timestampBase = OutputConfiguration::TIMESTAMP_BASE_DEFAULT,
185             int32_t colorSpace = ANDROID_REQUEST_AVAILABLE_COLOR_SPACE_PROFILES_MAP_UNSPECIFIED,
186             bool useReadoutTimestamp = false)
187             override;
188 
189     status_t createInputStream(
190             uint32_t width, uint32_t height, int format, bool isMultiResolution,
191             int *id) override;
192 
193     status_t getStreamInfo(int id, StreamInfo *streamInfo) override;
194     status_t setStreamTransform(int id, int transform) override;
195 
196     status_t deleteStream(int id) override;
197 
beginConfigure()198     virtual status_t beginConfigure() override {return OK;};
199 
getSharedStreamId(const OutputConfiguration &,int *)200     virtual status_t getSharedStreamId(const OutputConfiguration& /*config*/,
201             int* /*streamId*/) override {return INVALID_OPERATION;};
202 
addSharedSurfaces(int,const std::vector<android::camera3::OutputStreamInfo> &,const std::vector<SurfaceHolder> &,std::vector<int> *)203     virtual status_t addSharedSurfaces(int /*streamId*/,
204             const std::vector<android::camera3::OutputStreamInfo>& /*outputInfo*/,
205             const std::vector<SurfaceHolder>& /*surfaces*/,
206             std::vector<int>* /*surfaceIds*/) override {return INVALID_OPERATION;};
207 
removeSharedSurfaces(int,const std::vector<size_t> &)208     virtual status_t removeSharedSurfaces(int /*streamId*/,
209             const std::vector<size_t>& /*surfaceIds*/) override {return INVALID_OPERATION;};
210 
211     status_t configureStreams(const CameraMetadata& sessionParams,
212             int operatingMode =
213             camera_stream_configuration_mode_t::CAMERA_STREAM_CONFIGURATION_NORMAL_MODE) override;
214 #if WB_CAMERA3_AND_PROCESSORS_WITH_DEPENDENCIES
215     status_t getInputSurface(sp<Surface> *surface) override;
216 #else
217     status_t getInputBufferProducer(
218             sp<IGraphicBufferProducer> *producer) override;
219 #endif
220 
221     void getOfflineStreamIds(std::vector<int> *offlineStreamIds) override;
222 
223     status_t createDefaultRequest(camera_request_template_t templateId,
224             CameraMetadata *request) override;
225 
226     // Transitions to the idle state on success
227     status_t waitUntilDrained() override;
228 
229     status_t setNotifyCallback(wp<NotificationListener> listener) override;
230     bool     willNotify3A() override;
231     status_t waitForNextFrame(nsecs_t timeout) override;
232     status_t getNextResult(CaptureResult *frame) override;
233 
234     status_t triggerAutofocus(uint32_t id) override;
235     status_t triggerCancelAutofocus(uint32_t id) override;
236     status_t triggerPrecaptureMetering(uint32_t id) override;
237 
238     status_t flush(int64_t *lastFrameNumber = NULL) override;
239 
240     status_t prepare(int streamId) override;
241 
242     status_t tearDown(int streamId) override;
243 
244     status_t addBufferListenerForStream(int streamId,
245             wp<camera3::Camera3StreamBufferListener> listener) override;
246 
247     status_t prepare(int maxCount, int streamId) override;
248 
249     ssize_t getJpegBufferSize(const CameraMetadata &info, uint32_t width,
250             uint32_t height) const override;
251     ssize_t getPointCloudBufferSize(const CameraMetadata &info) const;
252     ssize_t getRawOpaqueBufferSize(const CameraMetadata &info, int32_t width, int32_t height,
253             bool maxResolution) const;
254 
255     // Methods called by subclasses
256     void             notifyStatus(bool idle); // updates from StatusTracker
257 
258     /**
259      * Set the deferred consumer surfaces to the output stream and finish the deferred
260      * consumer configuration.
261      */
262     status_t setConsumerSurfaces(
263             int streamId, const std::vector<SurfaceHolder>& consumers,
264             std::vector<int> *surfaceIds /*out*/) override;
265 
266     /**
267      * Update a given stream.
268      */
269     status_t updateStream(int streamId, const std::vector<SurfaceHolder> &newSurfaces,
270             const std::vector<OutputStreamInfo> &outputInfo,
271             const std::vector<size_t> &removedSurfaceIds,
272             KeyedVector<sp<Surface>, size_t> *outputMap/*out*/);
273 
274     /**
275      * Drop buffers for stream of streamId if dropping is true. If dropping is false, do not
276      * drop buffers for stream of streamId.
277      */
278     status_t dropStreamBuffers(bool dropping, int streamId) override;
279 
280     nsecs_t getExpectedInFlightDuration() override;
281 
switchToOffline(const std::vector<int32_t> &,sp<CameraOfflineSessionBase> *)282     virtual status_t switchToOffline(const std::vector<int32_t>& ,
283             /*out*/ sp<CameraOfflineSessionBase>* )  override {
284       return INVALID_OPERATION;
285     };
286 
287     // RequestBufferInterface
288     bool startRequestBuffer() override;
289     void endRequestBuffer() override;
290     nsecs_t getWaitDuration() override;
291 
292     // FlushBufferInterface
293     void getInflightBufferKeys(std::vector<std::pair<int32_t, int32_t>>* out) override;
294     void getInflightRequestBufferKeys(std::vector<uint64_t>* out) override;
295     std::vector<sp<camera3::Camera3StreamInterface>> getAllStreams() override;
296 
297     /**
298      * Set the current behavior for the ROTATE_AND_CROP control when in AUTO.
299      *
300      * The value must be one of the ROTATE_AND_CROP_* values besides AUTO,
301      * and defaults to NONE.
302      */
303     status_t setRotateAndCropAutoBehavior(
304             camera_metadata_enum_android_scaler_rotate_and_crop_t rotateAndCropValue, bool fromHal);
305 
306     /**
307      * Set the current behavior for the AUTOFRAMING control when in AUTO.
308      *
309      * The value must be one of the AUTOFRAMING_* values besides AUTO.
310      */
311     status_t setAutoframingAutoBehavior(
312             camera_metadata_enum_android_control_autoframing_t autoframingValue);
313 
314     /**
315      * Whether camera muting (producing black-only output) is supported.
316      *
317      * Calling setCameraMute(true) when this returns false will return an
318      * INVALID_OPERATION error.
319      */
320     bool supportsCameraMute();
321 
322     /**
323      * Mute the camera.
324      *
325      * When muted, black image data is output on all output streams.
326      */
327     status_t setCameraMute(bool enabled);
328 
329     /**
330      * Mute the camera.
331      *
332      * When muted, black image data is output on all output streams.
333      * This method assumes the caller already acquired the 'mInterfaceLock'
334      * and 'mLock' locks.
335      */
336     status_t setCameraMuteLocked(bool enabled);
337 
338     /**
339      * Enables/disables camera service watchdog
340      */
341     status_t setCameraServiceWatchdog(bool enabled);
342 
343     // Set stream use case overrides
344     void setStreamUseCaseOverrides(
345             const std::vector<int64_t>& useCaseOverrides);
346 
347     // Clear stream use case overrides
348     void clearStreamUseCaseOverrides();
349 
350     /**
351      * Whether the camera device supports zoom override.
352      */
353     bool supportsZoomOverride();
354 
355     // Set/reset zoom override
356     status_t setZoomOverride(int32_t zoomOverride);
357 
358     // Get the status trackeer for the camera device
getStatusTracker()359     wp<camera3::StatusTracker> getStatusTracker() { return mStatusTracker; }
360 
361     // Whether the device is in error state
362     bool hasDeviceError();
363 
364     /**
365      * The injection camera session to replace the internal camera
366      * session.
367      */
368     status_t injectCamera(const std::string& injectedCamId,
369                           sp<CameraProviderManager> manager);
370 
371     /**
372      * Stop the injection camera and restore to internal camera session.
373      */
374     status_t stopInjection();
375 
376     /**
377      * Inject session params into the current client.
378      */
379     status_t injectSessionParams(const CameraMetadata& sessionParams);
380 
381   protected:
382     status_t disconnectImpl();
383     static status_t removeFwkOnlyKeys(CameraMetadata *request);
384 
385     float getMaxPreviewFps(sp<camera3::Camera3OutputStreamInterface> stream);
386 
387     static const size_t        kDumpLockAttempts  = 10;
388     static const size_t        kDumpSleepDuration = 100000; // 0.10 sec
389     static const nsecs_t       kActiveTimeout     = 500000000;  // 500 ms
390     static const nsecs_t       kMinWarnInflightDuration = 5000000000; // 5 s
391     static const size_t        kInFlightWarnLimit = 30;
392     static const size_t        kInFlightWarnLimitHighSpeed = 256; // batch size 32 * pipe depth 8
393     static const nsecs_t       kMinInflightDuration = 5000000000; // 5 s
394     static const nsecs_t       kBaseGetBufferWait = 3000000000; // 3 sec.
395 
396     struct                     RequestTrigger;
397     // minimal jpeg buffer size: 256KB + blob header
398     static const ssize_t       kMinJpegBufferSize = camera3::MIN_JPEG_BUFFER_SIZE +
399             sizeof(aidl::android::hardware::camera::device::CameraBlob);
400     // Constant to use for stream ID when one doesn't exist
401     static const int           NO_STREAM = -1;
402 
403     std::shared_ptr<CameraServiceProxyWrapper> mCameraServiceProxyWrapper;
404 
405     // A lock to enforce serialization on the input/configure side
406     // of the public interface.
407     // Only locked by public methods inherited from CameraDeviceBase.
408     // Not locked by methods guarded by mOutputLock, since they may act
409     // concurrently to the input/configure side of the interface.
410     // Must be locked before mLock if both will be locked by a method
411     Mutex                      mInterfaceLock;
412 
413     // The main lock on internal state
414     Mutex                      mLock;
415 
416     // Camera device ID
417     const std::string          mId;
418 
419     // Legacy camera client flag
420     bool                       mLegacyClient;
421 
422     // Current stream configuration mode;
423     int                        mOperatingMode;
424     // Current session wide parameters
425     hardware::camera2::impl::CameraMetadataNative mSessionParams;
426     // Constant to use for no set operating mode
427     static const int           NO_MODE = -1;
428 
429     // Flag indicating is the current active stream configuration is constrained high speed.
430     bool                       mIsConstrainedHighSpeedConfiguration;
431 
432     /**** Scope for mLock ****/
433 
434     class HalInterface : public camera3::Camera3StreamBufferFreedListener,
435             public camera3::BufferRecordsInterface {
436       public:
HalInterface(bool useHalBufManager,bool supportOfflineProcessing)437         HalInterface(bool useHalBufManager, bool supportOfflineProcessing) :
438               mUseHalBufManager(useHalBufManager),
439               mIsReconfigurationQuerySupported(true),
440               mSupportOfflineProcessing(supportOfflineProcessing)
441                {};
442         HalInterface(const HalInterface &other);
443         HalInterface();
444 
445         virtual IPCTransport getTransportType() const = 0;
446 
447         // Returns true if constructed with a valid device or session, and not yet cleared
448         virtual bool valid() = 0;
449 
450         // Reset this HalInterface object (does not call close())
451         virtual void clear() = 0;
452 
453         // Calls into the HAL interface
454 
455         // Caller takes ownership of requestTemplate
456         virtual status_t constructDefaultRequestSettings(camera_request_template templateId,
457                 /*out*/ camera_metadata_t **requestTemplate) = 0;
458 
459         virtual status_t configureStreams(const camera_metadata_t * sessionParams,
460                 /*inout*/ camera_stream_configuration_t * config,
461                 const std::vector<uint32_t>& bufferSizes, int64_t logId) = 0;
462 
463         // The injection camera configures the streams to hal.
464         virtual status_t configureInjectedStreams(
465                 const camera_metadata_t* sessionParams,
466                 /*inout*/ camera_stream_configuration_t* config,
467                 const std::vector<uint32_t>& bufferSizes,
468                 const CameraMetadata& cameraCharacteristics) = 0;
469 
470         // When the call succeeds, the ownership of acquire fences in requests is transferred to
471         // HalInterface. More specifically, the current implementation will send the fence to
472         // HAL process and close the FD in cameraserver process. When the call fails, the ownership
473         // of the acquire fence still belongs to the caller.
474         virtual status_t processBatchCaptureRequests(
475                 std::vector<camera_capture_request_t*>& requests,
476                 /*out*/uint32_t* numRequestProcessed) = 0;
477 
478         virtual status_t flush() = 0;
479 
480         virtual status_t dump(int fd) = 0;
481 
482         virtual status_t close() = 0;
483 
484         virtual void signalPipelineDrain(const std::vector<int>& streamIds) = 0;
485 
486         virtual bool isReconfigurationRequired(CameraMetadata& oldSessionParams,
487                 CameraMetadata& newSessionParams) = 0;
488 
489         virtual status_t repeatingRequestEnd(uint32_t frameNumber,
490                 const std::vector<int32_t> &streamIds) = 0;
491 
492         /////////////////////////////////////////////////////////////////////
493         // Implements BufferRecordsInterface
494 
495         std::pair<bool, uint64_t> getBufferId(
496                 const buffer_handle_t& buf, int streamId) override;
497 
498         uint64_t removeOneBufferCache(int streamId, const native_handle_t* handle) override;
499 
500         status_t popInflightBuffer(int32_t frameNumber, int32_t streamId,
501                 /*out*/ buffer_handle_t **buffer) override;
502 
503         status_t pushInflightRequestBuffer(
504                 uint64_t bufferId, buffer_handle_t* buf, int32_t streamId) override;
505 
506         status_t popInflightRequestBuffer(uint64_t bufferId,
507                 /*out*/ buffer_handle_t** buffer,
508                 /*optional out*/ int32_t* streamId = nullptr) override;
509 
510         /////////////////////////////////////////////////////////////////////
511 
512         //Check if a stream is hal buffer managed
513         bool isHalBufferManagedStream(int32_t streamId) const;
514 
515         // Get a vector of (frameNumber, streamId) pair of currently inflight
516         // buffers
517         void getInflightBufferKeys(std::vector<std::pair<int32_t, int32_t>>* out);
518 
519         // Get a vector of bufferId of currently inflight buffers
520         void getInflightRequestBufferKeys(std::vector<uint64_t>* out);
521 
522         void onStreamReConfigured(int streamId);
523 
524       protected:
525 
526         // Return true if the input caches match what we have; otherwise false
527         bool verifyBufferIds(int32_t streamId, std::vector<uint64_t>& inBufIds);
528 
529         template <typename OfflineSessionInfoT>
verifyBufferCaches(const OfflineSessionInfoT * offlineSessionInfo,camera3::BufferRecords * bufferRecords)530         status_t verifyBufferCaches(
531             const OfflineSessionInfoT *offlineSessionInfo, camera3::BufferRecords *bufferRecords) {
532             // Validate buffer caches
533             std::vector<int32_t> streams;
534             streams.reserve(offlineSessionInfo->offlineStreams.size());
535             for (auto offlineStream : offlineSessionInfo->offlineStreams) {
536                 int32_t id = offlineStream.id;
537                 streams.push_back(id);
538                 // Verify buffer caches
539                 std::vector<uint64_t> bufIds(offlineStream.circulatingBufferIds.begin(),
540                         offlineStream.circulatingBufferIds.end());
541                 {
542                     // Due to timing it is possible that we may not have any remaining pending
543                     // capture requests that can update the caches on Hal side. This can result in
544                     // buffer cache mismatch between the service and the Hal and must be accounted
545                     // for.
546                     std::lock_guard<std::mutex> l(mFreedBuffersLock);
547                     for (const auto& it : mFreedBuffers) {
548                         if (it.first == id) {
549                             ALOGV("%s: stream ID %d buffer id %" PRIu64 " cache removal still "
550                                     "pending", __FUNCTION__, id, it.second);
551                             const auto& cachedEntry = std::find(bufIds.begin(), bufIds.end(),
552                                     it.second);
553                             if (cachedEntry != bufIds.end()) {
554                                 bufIds.erase(cachedEntry);
555                             } else {
556                                 ALOGE("%s: stream ID %d buffer id %" PRIu64 " cache removal still "
557                                         "pending however buffer is no longer in the offline stream "
558                                         "info!", __FUNCTION__, id, it.second);
559                             }
560                         }
561                     }
562                 }
563                 if (!verifyBufferIds(id, bufIds)) {
564                     ALOGE("%s: stream ID %d buffer cache records mismatch!", __FUNCTION__, id);
565                     return UNKNOWN_ERROR;
566                 }
567             }
568 
569             // Move buffer records
570             bufferRecords->takeBufferCaches(mBufferRecords, streams);
571             bufferRecords->takeInflightBufferMap(mBufferRecords);
572             bufferRecords->takeRequestedBufferMap(mBufferRecords);
573             return OK;
574         }
575 
576         virtual void onBufferFreed(int streamId, const native_handle_t* handle) override;
577 
578         std::mutex mFreedBuffersLock;
579         std::vector<std::pair<int, uint64_t>> mFreedBuffers;
580 
581         // Keep track of buffer cache and inflight buffer records
582         camera3::BufferRecords mBufferRecords;
583 
584         uint32_t mNextStreamConfigCounter = 1;
585 
586         // TODO: This can be removed after flags::session_hal_buf_manager is removed
587         bool mUseHalBufManager = false;
588         std::set<int32_t > mHalBufManagedStreamIds;
589         bool mIsReconfigurationQuerySupported;
590 
591         const bool mSupportOfflineProcessing;
592     }; // class HalInterface
593 
594     sp<HalInterface> mInterface;
595 
596     CameraMetadata             mDeviceInfo;
597     bool                       mSupportNativeZoomRatio;
598     bool                       mIsCompositeJpegRDisabled;
599     std::unordered_map<std::string, CameraMetadata> mPhysicalDeviceInfoMap;
600 
601     CameraMetadata             mRequestTemplateCache[CAMERA_TEMPLATE_COUNT];
602 
603     struct Size {
604         uint32_t width;
605         uint32_t height;
widthSize606         explicit Size(uint32_t w = 0, uint32_t h = 0) : width(w), height(h){}
607     };
608 
609     enum Status {
610         STATUS_ERROR,
611         STATUS_UNINITIALIZED,
612         STATUS_UNCONFIGURED,
613         STATUS_CONFIGURED,
614         STATUS_ACTIVE
615     }                          mStatus;
616 
617     struct StatusInfo {
618         Status status;
619         bool isInternal; // status triggered by internal reconfigureCamera.
620     };
621 
622     bool                       mStatusIsInternal;
623 
624     // Only clear mRecentStatusUpdates, mStatusWaiters from waitUntilStateThenRelock
625     Vector<StatusInfo>         mRecentStatusUpdates;
626     int                        mStatusWaiters;
627 
628     Condition                  mStatusChanged;
629 
630     // Tracking cause of fatal errors when in STATUS_ERROR
631     std::string                mErrorCause;
632 
633     camera3::StreamSet         mOutputStreams;
634     sp<camera3::Camera3Stream> mInputStream;
635     bool                       mIsInputStreamMultiResolution;
636     SessionStatsBuilder        mSessionStatsBuilder;
637     // Map from stream group ID to physical cameras backing the stream group
638     std::map<int32_t, std::set<std::string>> mGroupIdPhysicalCameraMap;
639 
640     int                        mNextStreamId;
641     bool                       mNeedConfig;
642 
643     int                        mFakeStreamId;
644 
645     // Whether to send state updates upstream
646     // Pause when doing transparent reconfiguration
647     bool                       mPauseStateNotify;
648 
649     // Need to hold on to stream references until configure completes.
650     Vector<sp<camera3::Camera3StreamInterface> > mDeletedStreams;
651 
652     // Whether the HAL will send partial result
653     bool                       mUsePartialResult;
654 
655     // Number of partial results that will be delivered by the HAL.
656     uint32_t                   mNumPartialResults;
657 
658     /**** End scope for mLock ****/
659 
660     bool                       mDeviceTimeBaseIsRealtime;
661     // The offset converting from clock domain of other subsystem
662     // (video/hardware composer) to that of camera. Assumption is that this
663     // offset won't change during the life cycle of the camera device. In other
664     // words, camera device shouldn't be open during CPU suspend.
665     nsecs_t                    mTimestampOffset;
666 
667     class CaptureRequest : public LightRefBase<CaptureRequest> {
668       public:
669         PhysicalCameraSettingsList          mSettingsList;
670         sp<camera3::Camera3Stream>          mInputStream;
671         camera_stream_buffer_t              mInputBuffer;
672         camera3::Size                       mInputBufferSize;
673         Vector<sp<camera3::Camera3OutputStreamInterface> >
674                                             mOutputStreams;
675         SurfaceMap                          mOutputSurfaces;
676         CaptureResultExtras                 mResultExtras;
677         // The number of requests that should be submitted to HAL at a time.
678         // For example, if batch size is 8, this request and the following 7
679         // requests will be submitted to HAL at a time. The batch size for
680         // the following 7 requests will be ignored by the request thread.
681         int                                 mBatchSize;
682         //  Whether this request is from a repeating or repeating burst.
683         bool                                mRepeating;
684         // Whether this request has ROTATE_AND_CROP_AUTO set, so needs both
685         // overriding of ROTATE_AND_CROP value and adjustment of coordinates
686         // in several other controls in both the request and the result
687         bool                                mRotateAndCropAuto;
688         // Indicates that the ROTATE_AND_CROP value within 'mSettingsList' was modified
689         // irrespective of the original value.
690         bool                                mRotateAndCropChanged = false;
691         // Whether this request has AUTOFRAMING_AUTO set, so need to override the AUTOFRAMING value
692         // in the capture request.
693         bool                                mAutoframingAuto;
694         // Indicates that the auto framing value within 'mSettingsList' was modified
695         bool                                mAutoframingChanged = false;
696         // Indicates that the camera test pattern setting is modified
697         bool                                mTestPatternChanged = false;
698 
699         // Whether this capture request has its zoom ratio set to 1.0x before
700         // the framework overrides it for camera HAL consumption.
701         bool                                mZoomRatioIs1x;
702         // The systemTime timestamp when the request is created.
703         nsecs_t                             mRequestTimeNs;
704 
705         // Whether this capture request's distortion correction update has
706         // been done.
707         bool                                mDistortionCorrectionUpdated = false;
708         // Whether this capture request's rotation and crop update has been
709         // done.
710         bool                                mRotationAndCropUpdated = false;
711         // Whether this capture request's autoframing has been done.
712         bool                                mAutoframingUpdated = false;
713         // Whether this capture request's zoom ratio update has been done.
714         bool                                mZoomRatioUpdated = false;
715         // Whether this max resolution capture request's  crop / metering region update has been
716         // done.
717         bool                                mUHRCropAndMeteringRegionsUpdated = false;
718     };
719     typedef List<sp<CaptureRequest> > RequestList;
720 
721     status_t checkStatusOkToCaptureLocked();
722 
723     status_t convertMetadataListToRequestListLocked(
724             const List<const PhysicalCameraSettingsList> &metadataList,
725             const std::list<SurfaceMap> &surfaceMaps,
726             bool repeating, nsecs_t requestTimeNs,
727             /*out*/
728             RequestList *requestList);
729 
730     void convertToRequestList(List<const PhysicalCameraSettingsList>& requestsList,
731             std::list<SurfaceMap>& surfaceMaps,
732             const CameraMetadata& request);
733 
734     status_t submitRequestsHelper(const List<const PhysicalCameraSettingsList> &requestsList,
735                                   const std::list<SurfaceMap> &surfaceMaps,
736                                   bool repeating,
737                                   int64_t *lastFrameNumber = NULL);
738 
739     // lock to ensure only one processCaptureResult is called at a time.
740     Mutex mProcessCaptureResultLock;
741 
742     /**
743      * Common initialization code shared by both HAL paths
744      *
745      * Must be called with mLock and mInterfaceLock held.
746      */
747     status_t initializeCommonLocked(sp<CameraProviderManager> manager);
748 
749     /**
750      * Update capture request list so that each batch size honors the batch_size_max report from
751      * the HAL. Set the batch size to output stream for buffer operations.
752      *
753      * Must be called with mLock held.
754      */
755     virtual void applyMaxBatchSizeLocked(
756             RequestList* requestList, const sp<camera3::Camera3OutputStreamInterface>& stream) = 0;
757 
758     struct LatestRequestInfo {
759         CameraMetadata requestSettings;
760         std::unordered_map<std::string, CameraMetadata> physicalRequestSettings;
761         int32_t inputStreamId = -1;
762         std::set<int32_t> outputStreamIds;
763     };
764 
765     /**
766      * Get the last request submitted to the hal by the request thread.
767      *
768      * Must be called with mLock held.
769      */
770     virtual LatestRequestInfo getLatestRequestInfoLocked();
771 
772     virtual status_t injectionCameraInitialize(const std::string &injectCamId,
773             sp<CameraProviderManager> manager) = 0;
774 
775     /**
776      * Update the current device status and wake all waiting threads.
777      *
778      * Must be called with mLock held.
779      */
780     void internalUpdateStatusLocked(Status status);
781 
782     /**
783      * Pause processing and flush everything, but don't tell the clients.
784      * This is for reconfiguring outputs transparently when according to the
785      * CameraDeviceBase interface we shouldn't need to.
786      * Must be called with mLock and mInterfaceLock both held.
787      */
788     status_t internalPauseAndWaitLocked(nsecs_t maxExpectedDuration,
789                      bool requestThreadInvocation);
790 
791     /**
792      * Resume work after internalPauseAndWaitLocked()
793      * Must be called with mLock and mInterfaceLock both held.
794      */
795     status_t internalResumeLocked();
796 
797     /**
798      * Wait until status tracker tells us we've transitioned to the target state
799      * set, which is either ACTIVE when active==true or IDLE (which is any
800      * non-ACTIVE state) when active==false.
801      *
802      * Needs to be called with mLock and mInterfaceLock held.  This means there
803      * can ever only be one waiter at most.
804      *
805      * During the wait mLock is released.
806      *
807      */
808     status_t waitUntilStateThenRelock(bool active, nsecs_t timeout,
809                      bool requestThreadInvocation);
810 
811     /**
812      * Implementation of waitUntilDrained. On success, will transition to IDLE state.
813      *
814      * Need to be called with mLock and mInterfaceLock held.
815      */
816     status_t waitUntilDrainedLocked(nsecs_t maxExpectedDuration);
817 
818     /**
819      * Do common work for setting up a streaming or single capture request.
820      * On success, will transition to ACTIVE if in IDLE.
821      */
822     sp<CaptureRequest> setUpRequestLocked(const PhysicalCameraSettingsList &request,
823                                           const SurfaceMap &surfaceMap);
824 
825     /**
826      * Build a CaptureRequest request from the CameraDeviceBase request
827      * settings.
828      */
829     sp<CaptureRequest> createCaptureRequest(const PhysicalCameraSettingsList &request,
830                                             const SurfaceMap &surfaceMap);
831 
832     /**
833      * Internally re-configure camera device using new session parameters.
834      * This will get triggered by the request thread.
835      */
836     bool reconfigureCamera(const CameraMetadata& sessionParams, int clientStatusId);
837 
838     /**
839      * Return true in case of any output or input abandoned streams,
840      * otherwise return false.
841      */
842     bool checkAbandonedStreamsLocked();
843 
844     /**
845      * Filter stream session parameters and configure camera HAL.
846      */
847     status_t filterParamsAndConfigureLocked(const CameraMetadata& sessionParams,
848             int operatingMode);
849 
850     /**
851      * Take the currently-defined set of streams and configure the HAL to use
852      * them. This is a long-running operation (may be several hundered ms).
853      */
854     status_t           configureStreamsLocked(int operatingMode,
855             const CameraMetadata& sessionParams, bool notifyRequestThread = true);
856 
857     /**
858      * Cancel stream configuration that did not finish successfully.
859      */
860     void               cancelStreamsConfigurationLocked();
861 
862     /**
863      * Add a fake stream to the current stream set as a workaround for
864      * not allowing 0 streams in the camera HAL spec.
865      */
866     status_t           addFakeStreamLocked();
867 
868     /**
869      * Remove a fake stream if the current config includes real streams.
870      */
871     status_t           tryRemoveFakeStreamLocked();
872 
873     /**
874      * Set device into an error state due to some fatal failure, and set an
875      * error message to indicate why. Only the first call's message will be
876      * used. The message is also sent to the log.
877      */
878     void               setErrorState(const char *fmt, ...) override;
879     void               setErrorStateLocked(const char *fmt, ...) override;
880     void               setErrorStateV(const char *fmt, va_list args);
881     void               setErrorStateLockedV(const char *fmt, va_list args);
882 
883     /////////////////////////////////////////////////////////////////////
884     // Implements InflightRequestUpdateInterface
885 
886     void onInflightEntryRemovedLocked(nsecs_t duration) override;
887     void checkInflightMapLengthLocked() override;
888     void onInflightMapFlushedLocked() override;
889 
890     /////////////////////////////////////////////////////////////////////
891 
892     /**
893      * Debugging trylock/spin method
894      * Try to acquire a lock a few times with sleeps between before giving up.
895      */
896     bool               tryLockSpinRightRound(Mutex& lock);
897 
898     /**
899      * Helper function to get the offset between MONOTONIC and BOOTTIME
900      * timestamp.
901      */
902     static nsecs_t getMonoToBoottimeOffset();
903 
904     // Override rotate_and_crop control if needed
905     static bool    overrideAutoRotateAndCrop(const sp<CaptureRequest> &request /*out*/,
906             int rotationOverride,
907             camera_metadata_enum_android_scaler_rotate_and_crop_t rotateAndCropOverride);
908 
909     // Override auto framing control if needed
910     static bool    overrideAutoframing(const sp<CaptureRequest> &request /*out*/,
911             camera_metadata_enum_android_control_autoframing_t autoframingOverride);
912 
913     struct RequestTrigger {
914         // Metadata tag number, e.g. android.control.aePrecaptureTrigger
915         uint32_t metadataTag;
916         // Metadata value, e.g. 'START' or the trigger ID
917         int32_t entryValue;
918 
919         // The last part of the fully qualified path, e.g. afTrigger
getTagNameRequestTrigger920         const char *getTagName() const {
921             return get_camera_metadata_tag_name(metadataTag) ?: "NULL";
922         }
923 
924         // e.g. TYPE_BYTE, TYPE_INT32, etc.
getTagTypeRequestTrigger925         int getTagType() const {
926             return get_camera_metadata_tag_type(metadataTag);
927         }
928     };
929 
930     /**
931      * Thread for managing capture request submission to HAL device.
932      */
933     class RequestThread : public Thread {
934 
935       public:
936 
937         RequestThread(wp<Camera3Device> parent,
938                 sp<camera3::StatusTracker> statusTracker,
939                 sp<HalInterface> interface,
940                 const Vector<int32_t>& sessionParamKeys,
941                 bool useHalBufManager,
942                 bool supportCameraMute,
943                 int rotationOverride,
944                 bool supportSettingsOverride);
945         ~RequestThread();
946 
947         void     setNotificationListener(wp<NotificationListener> listener);
948 
949         /**
950          * Call after stream (re)-configuration is completed.
951          */
952         void     configurationComplete(bool isConstrainedHighSpeed,
953                 const CameraMetadata& sessionParams,
954                 const std::map<int32_t, std::set<std::string>>& groupIdPhysicalCameraMap);
955 
956         /**
957          * Set or clear the list of repeating requests. Does not block
958          * on either. Use waitUntilPaused to wait until request queue
959          * has emptied out.
960          */
961         status_t setRepeatingRequests(const RequestList& requests,
962                                       /*out*/
963                                       int64_t *lastFrameNumber = NULL);
964         status_t clearRepeatingRequests(/*out*/
965                                         int64_t *lastFrameNumber = NULL);
966 
967         status_t queueRequestList(List<sp<CaptureRequest> > &requests,
968                                   /*out*/
969                                   int64_t *lastFrameNumber = NULL);
970 
971         /**
972          * Remove all queued and repeating requests, and pending triggers
973          */
974         status_t clear(/*out*/int64_t *lastFrameNumber = NULL);
975 
976         /**
977          * Flush all pending requests in HAL.
978          */
979         status_t flush();
980 
981         /**
982          * Queue a trigger to be dispatched with the next outgoing
983          * process_capture_request. The settings for that request only
984          * will be temporarily rewritten to add the trigger tag/value.
985          * Subsequent requests will not be rewritten (for this tag).
986          */
987         status_t queueTrigger(RequestTrigger trigger[], size_t count);
988 
989         /**
990          * Pause/unpause the capture thread. Doesn't block, so use
991          * waitUntilPaused to wait until the thread is paused.
992          */
993         void     setPaused(bool paused);
994 
995         /**
996          * Set Hal buffer managed streams
997          * @param halBufferManagedStreams The streams for which hal buffer manager is enabled
998          *
999          */
1000         void setHalBufferManagedStreams(const std::set<int32_t> &halBufferManagedStreams);
1001 
1002         /**
1003          * Wait until thread processes the capture request with settings'
1004          * android.request.id == requestId.
1005          *
1006          * Returns TIMED_OUT in case the thread does not process the request
1007          * within the timeout.
1008          */
1009         status_t waitUntilRequestProcessed(int32_t requestId, nsecs_t timeout);
1010 
1011         /**
1012          * Shut down the thread. Shutdown is asynchronous, so thread may
1013          * still be running once this method returns.
1014          */
1015         virtual void requestExit();
1016 
1017         /**
1018          * Get the latest request that was sent to the HAL
1019          * with process_capture_request.
1020          */
1021         LatestRequestInfo getLatestRequestInfo() const;
1022 
1023         /**
1024          * Returns true if the stream is a target of any queued or repeating
1025          * capture request
1026          */
1027         bool isStreamPending(sp<camera3::Camera3StreamInterface>& stream);
1028 
1029         /**
1030          * Returns true if the surface is a target of any queued or repeating
1031          * capture request
1032          */
1033         bool isOutputSurfacePending(int streamId, size_t surfaceId);
1034 
1035         // dump processCaptureRequest latency
dumpCaptureRequestLatency(int fd,const char * name)1036         void dumpCaptureRequestLatency(int fd, const char* name) {
1037             mRequestLatency.dump(fd, name);
1038         }
1039 
1040         void signalPipelineDrain(const std::vector<int>& streamIds);
1041         void resetPipelineDrain();
1042 
1043         void clearPreviousRequest();
1044 
1045         status_t setRotateAndCropAutoBehavior(
1046                 camera_metadata_enum_android_scaler_rotate_and_crop_t rotateAndCropValue);
1047 
1048         status_t setAutoframingAutoBehaviour(
1049                 camera_metadata_enum_android_control_autoframing_t autoframingValue);
1050 
1051         status_t setComposerSurface(bool composerSurfacePresent);
1052 
1053         status_t setCameraMute(int32_t muteMode);
1054 
1055         status_t setZoomOverride(int32_t zoomOverride);
1056 
1057         status_t setHalInterface(sp<HalInterface> newHalInterface);
1058 
1059         status_t setInjectedSessionParams(const CameraMetadata& sessionParams);
1060 
1061         void injectSessionParams(
1062             const sp<CaptureRequest> &request,
1063             const CameraMetadata& injectedSessionParams);
1064 
1065         /**
1066          * signal mLatestRequestmutex
1067          **/
1068         void wakeupLatestRequest(bool latestRequestFailed, int32_t latestRequestId);
1069 
1070       protected:
1071 
1072         virtual bool threadLoop();
1073 
1074         static const std::string& getId(const wp<Camera3Device> &device);
1075 
1076         status_t           queueTriggerLocked(RequestTrigger trigger);
1077         // Mix-in queued triggers into this request
1078         int32_t            insertTriggers(const sp<CaptureRequest> &request);
1079         // Purge the queued triggers from this request,
1080         //  restoring the old field values for those tags.
1081         status_t           removeTriggers(const sp<CaptureRequest> &request);
1082 
1083         // HAL workaround: Make sure a trigger ID always exists if
1084         // a trigger does
1085         status_t           addFakeTriggerIds(const sp<CaptureRequest> &request);
1086 
1087         // Override rotate_and_crop control if needed; returns true if the current value was changed
1088         bool               overrideAutoRotateAndCrop(const sp<CaptureRequest> &request /*out*/);
1089 
1090         // Override autoframing control if needed; returns true if the current value was changed
1091         bool               overrideAutoframing(const sp<CaptureRequest> &request);
1092 
1093         // Override test_pattern control if needed for camera mute; returns true
1094         // if the current value was changed
1095         bool               overrideTestPattern(const sp<CaptureRequest> &request);
1096 
1097         // Override settings override if needed for lower zoom latency; return
1098         // true if the current value was changed
1099         bool               overrideSettingsOverride(const sp<CaptureRequest> &request);
1100 
1101         static const nsecs_t kRequestTimeout = 50e6; // 50 ms
1102 
1103         // TODO: does this need to be adjusted for long exposure requests?
1104         static const nsecs_t kRequestSubmitTimeout = 500e6; // 500 ms
1105 
1106         // Used to prepare a batch of requests.
1107         struct NextRequest {
1108             sp<CaptureRequest>              captureRequest;
1109             camera_capture_request_t       halRequest;
1110             Vector<camera_stream_buffer_t> outputBuffers;
1111             bool                            submitted;
1112         };
1113 
1114         // Wait for the next batch of requests and put them in mNextRequests. mNextRequests will
1115         // be empty if it times out.
1116         void waitForNextRequestBatch();
1117 
1118         // Waits for a request, or returns NULL if times out. Must be called with mRequestLock hold.
1119         sp<CaptureRequest> waitForNextRequestLocked();
1120 
1121         // Prepare HAL requests and output buffers in mNextRequests. Return TIMED_OUT if getting any
1122         // output buffer timed out. If an error is returned, the caller should clean up the pending
1123         // request batch.
1124         status_t prepareHalRequests();
1125 
1126         // Return buffers, etc, for requests in mNextRequests that couldn't be fully constructed and
1127         // send request errors if sendRequestError is true. The buffers will be returned in the
1128         // ERROR state to mark them as not having valid data. mNextRequests will be cleared.
1129         void cleanUpFailedRequests(bool sendRequestError);
1130 
1131         // Stop the repeating request if any of its output streams is abandoned.
1132         void checkAndStopRepeatingRequest();
1133 
1134         // Release physical camera settings and camera id resources.
1135         void cleanupPhysicalSettings(sp<CaptureRequest> request,
1136                 /*out*/camera_capture_request_t *halRequest);
1137 
1138         // Pause handling
1139         bool               waitIfPaused();
1140         void               unpauseForNewRequests();
1141 
1142         // Relay error to parent device object setErrorState
1143         void               setErrorState(const char *fmt, ...);
1144 
1145         // If the input request is in mRepeatingRequests. Must be called with mRequestLock hold
1146         bool isRepeatingRequestLocked(const sp<CaptureRequest>&);
1147 
1148         // Clear repeating requests. Must be called with mRequestLock held.
1149         status_t clearRepeatingRequestsLocked(/*out*/ int64_t *lastFrameNumber = NULL);
1150 
1151         // send request in mNextRequests to HAL in a batch. Return true = sucssess
1152         bool sendRequestsBatch();
1153 
1154         // Calculate the expected (minimum, maximum, isFixedFps) duration info for a request
1155         struct ExpectedDurationInfo {
1156             nsecs_t minDuration;
1157             nsecs_t maxDuration;
1158             bool isFixedFps;
1159         };
1160         ExpectedDurationInfo calculateExpectedDurationRange(
1161                 const camera_metadata_t *request);
1162 
1163         // Check and update latest session parameters based on the current request settings.
1164         bool updateSessionParameters(const CameraMetadata& settings);
1165 
1166         // Check whether FPS range session parameter re-configuration is needed in constrained
1167         // high speed recording camera sessions.
1168         bool skipHFRTargetFPSUpdate(int32_t tag, const camera_metadata_ro_entry_t& newEntry,
1169                 const camera_metadata_entry_t& currentEntry);
1170 
1171         // Update next request sent to HAL
1172         void updateNextRequest(NextRequest& nextRequest);
1173 
1174         wp<Camera3Device>  mParent;
1175         wp<camera3::StatusTracker>  mStatusTracker;
1176         sp<HalInterface>   mInterface;
1177 
1178         wp<NotificationListener> mListener;
1179 
1180         const std::string  mId;       // The camera ID
1181         int                mStatusId; // The RequestThread's component ID for
1182                                       // status tracking
1183 
1184         Mutex              mRequestLock;
1185         Condition          mRequestSignal;
1186         bool               mRequestClearing;
1187 
1188         Condition          mRequestSubmittedSignal;
1189         RequestList        mRequestQueue;
1190         RequestList        mRepeatingRequests;
1191         bool               mFirstRepeating;
1192         // The next batch of requests being prepped for submission to the HAL, no longer
1193         // on the request queue. Read-only even with mRequestLock held, outside
1194         // of threadLoop
1195         Vector<NextRequest> mNextRequests;
1196 
1197         // To protect flush() and sending a request batch to HAL.
1198         Mutex              mFlushLock;
1199 
1200         bool               mReconfigured;
1201 
1202         // Used by waitIfPaused, waitForNextRequest, waitUntilPaused, and signalPipelineDrain
1203         Mutex              mPauseLock;
1204         bool               mDoPause;
1205         Condition          mDoPauseSignal;
1206         bool               mPaused;
1207         bool               mNotifyPipelineDrain;
1208         std::vector<int>   mStreamIdsToBeDrained;
1209 
1210         sp<CaptureRequest> mPrevRequest;
1211         int32_t            mPrevTriggers;
1212         std::set<std::string> mPrevCameraIdsWithZoom;
1213 
1214         uint32_t           mFrameNumber;
1215 
1216         mutable Mutex      mLatestRequestMutex;
1217         Condition          mLatestRequestSignal;
1218         // android.request.id for latest process_capture_request
1219         int32_t            mLatestRequestId;
1220         int32_t            mLatestFailedRequestId;
1221         LatestRequestInfo mLatestRequestInfo;
1222 
1223         typedef KeyedVector<uint32_t/*tag*/, RequestTrigger> TriggerMap;
1224         Mutex              mTriggerMutex;
1225         TriggerMap         mTriggerMap;
1226         TriggerMap         mTriggerRemovedMap;
1227         TriggerMap         mTriggerReplacedMap;
1228         uint32_t           mCurrentAfTriggerId;
1229         uint32_t           mCurrentPreCaptureTriggerId;
1230         camera_metadata_enum_android_scaler_rotate_and_crop_t mRotateAndCropOverride;
1231         camera_metadata_enum_android_control_autoframing_t mAutoframingOverride;
1232         bool               mComposerOutput;
1233         int32_t            mCameraMute; // 0 = no mute, otherwise the TEST_PATTERN_MODE to use
1234         int32_t            mSettingsOverride; // -1 = use original, otherwise
1235                                               // the settings override to use.
1236 
1237         int64_t            mRepeatingLastFrameNumber;
1238 
1239         // Flag indicating if we should prepare video stream for video requests.
1240         bool               mPrepareVideoStream;
1241 
1242         bool               mConstrainedMode;
1243 
1244         static const int32_t kRequestLatencyBinSize = 40; // in ms
1245         CameraLatencyHistogram mRequestLatency;
1246 
1247         Vector<int32_t>    mSessionParamKeys;
1248         CameraMetadata     mLatestSessionParams;
1249         CameraMetadata     mInjectedSessionParams;
1250         bool               mForceNewRequestAfterReconfigure;
1251 
1252         std::map<int32_t, std::set<std::string>> mGroupIdPhysicalCameraMap;
1253 
1254         bool               mUseHalBufManager = false;
1255         std::set<int32_t > mHalBufManagedStreamIds;
1256         const bool         mSupportCameraMute;
1257         const bool         mRotationOverride;
1258         const bool         mSupportSettingsOverride;
1259         int32_t            mVndkVersion = -1;
1260     };
1261 
1262     virtual sp<RequestThread> createNewRequestThread(wp<Camera3Device> /*parent*/,
1263                 sp<camera3::StatusTracker> /*statusTracker*/,
1264                 sp<HalInterface> /*interface*/,
1265                 const Vector<int32_t>& /*sessionParamKeys*/,
1266                 bool /*useHalBufManager*/,
1267                 bool /*supportCameraMute*/,
1268                 int /*rotationOverride*/,
1269                 bool /*supportSettingsOverride*/) = 0;
1270 
1271     sp<RequestThread> mRequestThread;
1272 
1273     /**
1274      * In-flight queue for tracking completion of capture requests.
1275      */
1276     std::mutex                    mInFlightLock;
1277     camera3::InFlightRequestMap   mInFlightMap;
1278     nsecs_t                       mExpectedInflightDuration = 0;
1279     int64_t                       mLastCompletedRegularFrameNumber = -1;
1280     int64_t                       mLastCompletedReprocessFrameNumber = -1;
1281     int64_t                       mLastCompletedZslFrameNumber = -1;
1282     // End of mInFlightLock protection scope
1283 
1284     int mInFlightStatusId; // const after initialize
1285 
1286     status_t registerInFlight(uint32_t frameNumber,
1287             int32_t numBuffers, CaptureResultExtras resultExtras, bool hasInput,
1288             bool callback, nsecs_t minExpectedDuration, nsecs_t maxExpectedDuration,
1289             bool isFixedFps, const std::set<std::set<std::string>>& physicalCameraIds,
1290             bool isStillCapture, bool isZslCapture, bool rotateAndCropAuto, bool autoframingAuto,
1291             const std::set<std::string>& cameraIdsWithZoom, bool useZoomRatio,
1292             const SurfaceMap& outputSurfaces, nsecs_t requestTimeNs);
1293 
1294     /**
1295      * Tracking for idle detection
1296      */
1297     sp<camera3::StatusTracker> mStatusTracker;
1298 
1299     /**
1300      * Graphic buffer manager for output streams. Each device has a buffer manager, which is used
1301      * by the output streams to get and return buffers if these streams are registered to this
1302      * buffer manager.
1303      */
1304     sp<camera3::Camera3BufferManager> mBufferManager;
1305 
1306     /**
1307      * Thread for preparing streams
1308      */
1309     class PreparerThread : private Thread, public virtual RefBase {
1310       public:
1311         PreparerThread();
1312         ~PreparerThread();
1313 
1314         void setNotificationListener(wp<NotificationListener> listener);
1315 
1316         /**
1317          * Queue up a stream to be prepared. Streams are processed by a background thread in FIFO
1318          * order.  Pre-allocate up to maxCount buffers for the stream, or the maximum number needed
1319          * for the pipeline if maxCount is ALLOCATE_PIPELINE_MAX.
1320          */
1321         status_t prepare(int maxCount, sp<camera3::Camera3StreamInterface>& stream);
1322 
1323         /**
1324          * Cancel all current and pending stream preparation
1325          */
1326         status_t clear();
1327 
1328         /**
1329          * Pause all preparation activities
1330          */
1331         void pause();
1332 
1333         /**
1334          * Resume preparation activities
1335          */
1336         status_t resume();
1337 
1338       private:
1339         Mutex mLock;
1340         Condition mThreadActiveSignal;
1341 
1342         virtual bool threadLoop();
1343 
1344         // Guarded by mLock
1345 
1346         wp<NotificationListener> mListener;
1347         std::list<std::tuple<int, sp<camera3::Camera3StreamInterface>>> mPendingStreams;
1348         bool mActive;
1349         bool mCancelNow;
1350 
1351         // Only accessed by threadLoop and the destructor
1352 
1353         sp<camera3::Camera3StreamInterface> mCurrentStream;
1354         int mCurrentMaxCount;
1355         bool mCurrentPrepareComplete;
1356     };
1357     sp<PreparerThread> mPreparerThread;
1358 
1359     /**
1360      * Output result queue and current HAL device 3A state
1361      */
1362 
1363     // Lock for output side of device
1364     std::mutex             mOutputLock;
1365 
1366     /**** Scope for mOutputLock ****/
1367     // the minimal frame number of the next non-reprocess result
1368     uint32_t               mNextResultFrameNumber;
1369     // the minimal frame number of the next reprocess result
1370     uint32_t               mNextReprocessResultFrameNumber;
1371     // the minimal frame number of the next ZSL still capture result
1372     uint32_t               mNextZslStillResultFrameNumber;
1373     // the minimal frame number of the next non-reprocess shutter
1374     uint32_t               mNextShutterFrameNumber;
1375     // the minimal frame number of the next reprocess shutter
1376     uint32_t               mNextReprocessShutterFrameNumber;
1377     // the minimal frame number of the next ZSL still capture shutter
1378     uint32_t               mNextZslStillShutterFrameNumber;
1379     std::list<CaptureResult>    mResultQueue;
1380     std::condition_variable  mResultSignal;
1381     wp<NotificationListener> mListener;
1382 
1383     /**** End scope for mOutputLock ****/
1384 
1385     /**** Scope for mInFlightLock ****/
1386 
1387     // Remove the in-flight map entry of the given index from mInFlightMap.
1388     // It must only be called with mInFlightLock held.
1389     void removeInFlightMapEntryLocked(int idx);
1390 
1391     // Remove all in-flight requests and return all buffers.
1392     // This is used after HAL interface is closed to cleanup any request/buffers
1393     // not returned by HAL.
1394     void flushInflightRequests();
1395 
1396     /**** End scope for mInFlightLock ****/
1397 
1398     /**
1399      * Distortion correction support
1400      */
1401     // Map from camera IDs to its corresponding distortion mapper. Only contains
1402     // 1 ID if the device isn't a logical multi-camera. Otherwise contains both
1403     // logical camera and its physical subcameras.
1404     std::unordered_map<std::string, camera3::DistortionMapper> mDistortionMappers;
1405 
1406     /**
1407      * Zoom ratio mapper support
1408      */
1409     std::unordered_map<std::string, camera3::ZoomRatioMapper> mZoomRatioMappers;
1410 
1411     /**
1412      * UHR request crop / metering region mapper support
1413      */
1414     std::unordered_map<std::string, camera3::UHRCropAndMeteringRegionMapper>
1415             mUHRCropAndMeteringRegionMappers;
1416 
1417     /**
1418      * RotateAndCrop mapper support
1419      */
1420     std::unordered_map<std::string, camera3::RotateAndCropMapper> mRotateAndCropMappers;
1421 
1422     // Debug tracker for metadata tag value changes
1423     // - Enabled with the -m <taglist> option to dumpsys, such as
1424     //   dumpsys -m android.control.aeState,android.control.aeMode
1425     // - Disabled with -m off
1426     // - dumpsys -m 3a is a shortcut for ae/af/awbMode, State, and Triggers
1427     TagMonitor mTagMonitor;
1428 
1429     void monitorMetadata(TagMonitor::eventSource source, int64_t frameNumber,
1430             nsecs_t timestamp, const CameraMetadata& metadata,
1431             const std::unordered_map<std::string, CameraMetadata>& physicalMetadata,
1432             const camera_stream_buffer_t *outputBuffers, uint32_t numOutputBuffers,
1433             int32_t inputStreamId);
1434 
1435     // Collect any statistics that are based on the stream of capture requests sent
1436     // to the HAL
1437     void collectRequestStats(int64_t frameNumber, const CameraMetadata& request);
1438 
1439     metadata_vendor_id_t mVendorTagId;
1440 
1441     // Cached last requested template id
1442     int mLastTemplateId;
1443 
1444     // Synchronizes access to status tracker between inflight updates and disconnect.
1445     // b/79972865
1446     Mutex mTrackerLock;
1447 
1448     // Whether HAL request buffers through requestStreamBuffers API
1449     bool mUseHalBufManager = false;
1450     std::set<int32_t > mHalBufManagedStreamIds;
1451     bool mSessionHalBufManager = false;
1452     // Lock to ensure requestStreamBuffers() callbacks are serialized
1453     std::mutex mRequestBufferInterfaceLock;
1454 
1455     // The state machine to control when requestStreamBuffers should allow
1456     // HAL to request buffers.
1457     enum RequestBufferState {
1458         /**
1459          * This is the initial state.
1460          * requestStreamBuffers call will return FAILED_CONFIGURING in this state.
1461          * Will switch to RB_STATUS_READY after a successful configureStreams or
1462          * processCaptureRequest call.
1463          */
1464         RB_STATUS_STOPPED,
1465 
1466         /**
1467          * requestStreamBuffers call will proceed in this state.
1468          * When device is asked to stay idle via waitUntilStateThenRelock() call:
1469          *     - Switch to RB_STATUS_STOPPED if there is no inflight requests and
1470          *       request thread is paused.
1471          *     - Switch to RB_STATUS_PENDING_STOP otherwise
1472          */
1473         RB_STATUS_READY,
1474 
1475         /**
1476          * requestStreamBuffers call will proceed in this state.
1477          * Switch to RB_STATUS_STOPPED when all inflight requests are fulfilled
1478          * and request thread is paused
1479          */
1480         RB_STATUS_PENDING_STOP,
1481     };
1482 
1483     class RequestBufferStateMachine {
1484       public:
1485         status_t initialize(sp<camera3::StatusTracker> statusTracker);
1486 
1487         status_t deInit();
1488 
1489         // Return if the state machine currently allows for requestBuffers
1490         // If the state allows for it, mRequestBufferOngoing will be set to true
1491         // and caller must call endRequestBuffer() later to unset the flag
1492         bool startRequestBuffer();
1493         void endRequestBuffer();
1494 
1495         // Events triggered by application API call
1496         void onStreamsConfigured();
1497         void onWaitUntilIdle();
1498 
1499         // Events usually triggered by hwBinder processCaptureResult callback thread
1500         // But can also be triggered on request thread for failed request, or on
1501         // hwbinder notify callback thread for shutter/error callbacks
1502         void onInflightMapEmpty();
1503 
1504         // Events triggered by RequestThread
1505         void onSubmittingRequest();
1506         void onRequestThreadPaused();
1507 
1508         // Events triggered by successful switchToOffline call
1509         // Return true is there is no ongoing requestBuffer call.
1510         bool onSwitchToOfflineSuccess();
1511 
1512       private:
1513         void notifyTrackerLocked(bool active);
1514 
1515         // Switch to STOPPED state and return true if all conditions allows for it.
1516         // Otherwise do nothing and return false.
1517         bool checkSwitchToStopLocked();
1518 
1519         std::mutex mLock;
1520         RequestBufferState mStatus = RB_STATUS_STOPPED;
1521 
1522         bool mRequestThreadPaused = true;
1523         bool mInflightMapEmpty = true;
1524         bool mRequestBufferOngoing = false;
1525         bool mSwitchedToOffline = false;
1526 
1527         wp<camera3::StatusTracker> mStatusTracker;
1528         int  mRequestBufferStatusId;
1529     } mRequestBufferSM;
1530 
1531     // Fix up result metadata for monochrome camera.
1532     bool mNeedFixupMonochromeTags;
1533 
1534     // Whether HAL supports offline processing capability.
1535     bool mSupportOfflineProcessing = false;
1536 
1537     // Whether the HAL supports camera muting via test pattern
1538     bool mSupportCameraMute = false;
1539     // Whether the HAL supports SOLID_COLOR or BLACK if mSupportCameraMute is true
1540     bool mSupportTestPatternSolidColor = false;
1541     // Whether the HAL supports zoom settings override
1542     bool mSupportZoomOverride = false;
1543 
1544     // Whether the camera framework overrides the device characteristics for
1545     // performance class.
1546     bool mOverrideForPerfClass;
1547 
1548     // Whether the camera framework overrides the device characteristics for
1549     // app compatibility reasons.
1550     int mRotationOverride;
1551     camera_metadata_enum_android_scaler_rotate_and_crop_t mRotateAndCropOverride;
1552     bool mComposerOutput;
1553 
1554     // Auto framing override value
1555     camera_metadata_enum_android_control_autoframing mAutoframingOverride;
1556 
1557     // Initial camera mute state stored before the request thread
1558     // is active.
1559     bool mCameraMuteInitial = false;
1560 
1561     // Settings override value
1562     int32_t mSettingsOverride; // -1 = use original, otherwise
1563                                // the settings override to use.
1564 
1565     // Current active physical id of the logical multi-camera, if any
1566     std::string mActivePhysicalId;
1567 
1568     // The current minimum expected frame duration based on AE_TARGET_FPS_RANGE
1569     nsecs_t mMinExpectedDuration = 0;
1570     // Whether the camera device runs at fixed frame rate based on AE_MODE and
1571     // AE_TARGET_FPS_RANGE
1572     bool mIsFixedFps = false;
1573 
1574     // Flag to indicate that we shouldn't forward extension related metadata
1575     bool mSupportsExtensionKeys = false;
1576 
1577     // Injection camera related methods.
1578     class Camera3DeviceInjectionMethods : public virtual RefBase {
1579       public:
1580         Camera3DeviceInjectionMethods(wp<Camera3Device> parent);
1581 
1582         ~Camera3DeviceInjectionMethods();
1583 
1584         // Injection camera will replace the internal camera and configure streams
1585         // when device is IDLE and request thread is paused.
1586         status_t injectCamera(
1587                 camera3::camera_stream_configuration& injectionConfig,
1588                 const std::vector<uint32_t>& injectionBufferSizes);
1589 
1590         // Stop the injection camera and switch back to backup hal interface.
1591         status_t stopInjection();
1592 
1593         bool isInjecting();
1594 
1595         bool isStreamConfigCompleteButNotInjected();
1596 
1597         const std::string& getInjectedCamId() const;
1598 
1599         void getInjectionConfig(/*out*/ camera3::camera_stream_configuration* injectionConfig,
1600                 /*out*/ std::vector<uint32_t>* injectionBufferSizes);
1601 
1602         // When the streaming configuration is completed and the camera device is active, but the
1603         // injection camera has not yet been injected, the streaming configuration of the internal
1604         // camera will be stored first.
1605         void storeInjectionConfig(
1606                 const camera3::camera_stream_configuration& injectionConfig,
1607                 const std::vector<uint32_t>& injectionBufferSizes);
1608 
1609       protected:
1610         // Configure the streams of injection camera, it need wait until the
1611         // output streams are created and configured to the original camera before
1612         // proceeding.
1613         status_t injectionConfigureStreams(
1614                 camera3::camera_stream_configuration& injectionConfig,
1615                 const std::vector<uint32_t>& injectionBufferSizes);
1616 
1617         // Disconnect the injection camera and delete the hal interface.
1618         void injectionDisconnectImpl();
1619 
1620         // Use injection camera hal interface to replace and backup original
1621         // camera hal interface.
1622         virtual status_t replaceHalInterface(sp<HalInterface> /*newHalInterface*/,
1623                 bool /*keepBackup*/) = 0;
1624 
1625         wp<Camera3Device> mParent;
1626 
1627         // Backup of the original camera hal interface.
1628         sp<HalInterface> mBackupHalInterface;
1629 
1630         // Generated injection camera hal interface.
1631         sp<HalInterface> mInjectedCamHalInterface;
1632 
1633         // The flag indicates that the stream configuration is complete, the camera device is
1634         // active, but the injection camera has not yet been injected.
1635         bool mIsStreamConfigCompleteButNotInjected = false;
1636 
1637         // Copy the configuration of the internal camera.
1638         camera3::camera_stream_configuration mInjectionConfig;
1639 
1640         // Copy the streams of the internal camera.
1641         Vector<camera3::camera_stream_t*> mInjectionStreams;
1642 
1643         // Copy the bufferSizes of the output streams of the internal camera.
1644         std::vector<uint32_t> mInjectionBufferSizes;
1645 
1646         // Synchronizes access to injection camera between initialize and
1647         // disconnect.
1648         Mutex mInjectionLock;
1649 
1650         // The injection camera ID.
1651         std::string mInjectedCamId;
1652     };
1653 
1654     virtual sp<Camera3DeviceInjectionMethods>
1655             createCamera3DeviceInjectionMethods(wp<Camera3Device>) = 0;
1656 
1657     sp<Camera3DeviceInjectionMethods> mInjectionMethods;
1658 
1659     void overrideStreamUseCaseLocked();
1660     status_t deriveAndSetTransformLocked(camera3::Camera3OutputStreamInterface& stream,
1661                                    int mirrorMode, int surfaceId);
1662 
1663 
1664 }; // class Camera3Device
1665 
1666 }; // namespace android
1667 
1668 #endif
1669