1 /* 2 * Copyright (C) 2019 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef ANDROID_SERVERS_CAMERA_CAMERA3_HEIC_COMPOSITE_STREAM_H 18 #define ANDROID_SERVERS_CAMERA_CAMERA3_HEIC_COMPOSITE_STREAM_H 19 20 #include <algorithm> 21 #include <android/data_space.h> 22 #include <memory> 23 #include <queue> 24 25 #include <gui/CpuConsumer.h> 26 27 #include <media/hardware/VideoAPI.h> 28 #include <media/MediaCodecBuffer.h> 29 #include <media/stagefright/foundation/ALooper.h> 30 #include <media/stagefright/foundation/AMessage.h> 31 #include <media/stagefright/MediaCodec.h> 32 #include <media/stagefright/MediaMuxer.h> 33 #include <ultrahdr/ultrahdrcommon.h> 34 #include <ultrahdr/gainmapmetadata.h> 35 36 #include "CompositeStream.h" 37 38 namespace android { 39 namespace camera3 { 40 41 class HeicCompositeStream : public CompositeStream, public Thread, 42 public CpuConsumer::FrameAvailableListener { 43 public: 44 HeicCompositeStream(sp<CameraDeviceBase> device, 45 wp<hardware::camera2::ICameraDeviceCallbacks> cb); 46 ~HeicCompositeStream() override; 47 48 static bool isHeicCompositeStream(const sp<Surface> &surface); 49 static bool isHeicCompositeStreamInfo(const OutputStreamInfo& streamInfo); 50 51 status_t createInternalStreams(const std::vector<SurfaceHolder>& consumers, 52 bool hasDeferredConsumer, uint32_t width, uint32_t height, int format, 53 camera_stream_rotation_t rotation, int *id, const std::string& physicalCameraId, 54 const std::unordered_set<int32_t> &sensorPixelModesUsed, 55 std::vector<int> *surfaceIds, int streamSetId, bool isShared, int32_t colorSpace, 56 int64_t dynamicProfile, int64_t streamUseCase, bool useReadoutTimestamp) override; 57 58 status_t deleteInternalStreams() override; 59 60 status_t configureStream() override; 61 62 status_t insertGbp(SurfaceMap* /*out*/outSurfaceMap, Vector<int32_t>* /*out*/outputStreamIds, 63 int32_t* /*out*/currentStreamId) override; 64 65 status_t insertCompositeStreamIds(std::vector<int32_t>* compositeStreamIds /*out*/) override; 66 67 void onShutter(const CaptureResultExtras& resultExtras, nsecs_t timestamp) override; 68 getStreamId()69 int getStreamId() override { return mMainImageStreamId; } 70 71 // Use onShutter to keep track of frame number <-> timestamp mapping. 72 void onBufferReleased(const BufferInfo& bufferInfo) override; 73 void onBufferRequestForFrameNumber(uint64_t frameNumber, int streamId, 74 const CameraMetadata& settings) override; 75 76 // CpuConsumer listener implementation 77 void onFrameAvailable(const BufferItem& item) override; 78 79 // Return stream information about the internal camera streams 80 static status_t getCompositeStreamInfo(const OutputStreamInfo &streamInfo, 81 const CameraMetadata& ch, std::vector<OutputStreamInfo>* compositeOutput /*out*/); 82 83 // Get composite stream stats getStreamStats(hardware::CameraStreamStats *)84 void getStreamStats(hardware::CameraStreamStats*) override {}; 85 86 static bool isSizeSupportedByHeifEncoder(int32_t width, int32_t height, 87 bool* useHeic, bool* useGrid, int64_t* stall, AString* hevcName = nullptr, 88 bool allowSWCodec = false); 89 static bool isInMemoryTempFileSupported(); 90 91 // HDR Gainmap subsampling 92 static const size_t kGainmapScale = 4; 93 94 protected: 95 96 bool threadLoop() override; 97 bool onStreamBufferError(const CaptureResultExtras& resultExtras) override; 98 void onResultError(const CaptureResultExtras& resultExtras) override; 99 void onRequestError(const CaptureResultExtras& resultExtras) override; 100 101 private: 102 // 103 // HEIC/HEVC Codec related structures, utility functions, and callbacks 104 // 105 struct CodecOutputBufferInfo { 106 int32_t index; 107 int32_t offset; 108 int32_t size; 109 int64_t timeUs; 110 uint32_t flags; 111 }; 112 113 struct CodecInputBufferInfo { 114 int32_t index; 115 int64_t timeUs; 116 size_t tileIndex; 117 }; 118 119 class CodecCallbackHandler : public AHandler { 120 public: 121 explicit CodecCallbackHandler(wp<HeicCompositeStream> parent, bool isGainmap = false) : mParent(parent)122 mParent(parent), mIsGainmap(isGainmap) {} 123 virtual void onMessageReceived(const sp<AMessage> &msg); 124 private: 125 wp<HeicCompositeStream> mParent; 126 bool mIsGainmap; 127 }; 128 129 enum { 130 kWhatCallbackNotify, 131 }; 132 133 bool mUseHeic; 134 sp<MediaCodec> mCodec; 135 sp<MediaCodec> mGainmapCodec; 136 sp<ALooper> mCodecLooper, mCallbackLooper, mGainmapCallbackLooper; 137 sp<CodecCallbackHandler> mCodecCallbackHandler, mGainmapCodecCallbackHandler; 138 sp<AMessage> mAsyncNotify, mGainmapAsyncNotify; 139 sp<AMessage> mFormat, mGainmapFormat; 140 size_t mNumOutputTiles, mNumGainmapOutputTiles; 141 142 int32_t mOutputWidth, mOutputHeight, mGainmapOutputWidth, mGainmapOutputHeight; 143 size_t mMaxHeicBufferSize; 144 int32_t mGridWidth, mGridHeight, mGainmapGridWidth, mGainmapGridHeight; 145 size_t mGridRows, mGridCols, mGainmapGridRows, mGainmapGridCols; 146 bool mUseGrid, mGainmapUseGrid; // Whether to use framework YUV frame tiling. 147 148 static const int64_t kNoFrameDropMaxPtsGap = -1000000; 149 static const int32_t kNoGridOpRate = 30; 150 static const int32_t kGridOpRate = 120; 151 152 void onHeicOutputFrameAvailable(const CodecOutputBufferInfo& bufferInfo, bool isGainmap); 153 void onHeicInputFrameAvailable(int32_t index, bool isGainmap);// Only called for YUV input mode. 154 void onHeicFormatChanged(sp<AMessage>& newFormat, bool isGainmap); 155 void onHeicGainmapFormatChanged(sp<AMessage>& newFormat); 156 void onHeicCodecError(); 157 158 status_t initializeCodec(uint32_t width, uint32_t height, 159 const sp<CameraDeviceBase>& cameraDevice); 160 void deinitCodec(); 161 status_t initializeGainmapCodec(); 162 void deinitGainmapCodec(); 163 164 // 165 // Composite stream related structures, utility functions and callbacks. 166 // 167 struct InputFrame { 168 int32_t orientation; 169 int32_t quality; 170 171 CpuConsumer::LockedBuffer appSegmentBuffer; 172 std::vector<CodecOutputBufferInfo> codecOutputBuffers, gainmapCodecOutputBuffers; 173 std::unique_ptr<CameraMetadata> result; 174 175 // Fields that are only applicable to HEVC tiling. 176 CpuConsumer::LockedBuffer yuvBuffer; 177 std::vector<CodecInputBufferInfo> codecInputBuffers, gainmapCodecInputBuffers; 178 179 bool error; // Main input image buffer error 180 bool exifError; // Exif/APP_SEGMENT buffer error 181 int64_t timestamp; 182 int32_t requestId; 183 184 sp<AMessage> format, gainmapFormat; 185 sp<MediaMuxer> muxer; 186 int fenceFd; 187 int fileFd; 188 ssize_t trackIndex, gainmapTrackIndex; 189 ANativeWindowBuffer *anb; 190 191 bool appSegmentWritten; 192 size_t pendingOutputTiles, gainmapPendingOutputTiles; 193 size_t codecInputCounter, gainmapCodecInputCounter; 194 195 std::unique_ptr<CpuConsumer::LockedBuffer> baseImage, gainmapImage; 196 std::unique_ptr<ultrahdr::uhdr_raw_image_ext> baseBuffer, gainmap; 197 std::unique_ptr<uint8_t[]> gainmapChroma; 198 std::vector<uint8_t> isoGainmapMetadata; 199 InputFrameInputFrame200 InputFrame() 201 : orientation(0), 202 quality(kDefaultJpegQuality), 203 error(false), 204 exifError(false), 205 timestamp(-1), 206 requestId(-1), 207 fenceFd(-1), 208 fileFd(-1), 209 trackIndex(-1), 210 gainmapTrackIndex(-1), 211 anb(nullptr), 212 appSegmentWritten(false), 213 pendingOutputTiles(0), 214 gainmapPendingOutputTiles(0), 215 codecInputCounter(0), 216 gainmapCodecInputCounter(0) {} 217 }; 218 219 void compilePendingInputLocked(); 220 // Find first complete and valid frame with smallest frame number 221 bool getNextReadyInputLocked(int64_t *frameNumber /*out*/); 222 // Find next failing frame number with smallest frame number and return respective frame number 223 int64_t getNextFailingInputLocked(); 224 225 status_t processInputFrame(int64_t frameNumber, InputFrame &inputFrame); 226 status_t processCodecInputFrame(InputFrame &inputFrame); 227 status_t processCodecGainmapInputFrame(InputFrame &inputFrame); 228 status_t startMuxerForInputFrame(int64_t frameNumber, InputFrame &inputFrame); 229 status_t processAppSegment(int64_t frameNumber, InputFrame &inputFrame); 230 status_t processOneCodecOutputFrame(int64_t frameNumber, InputFrame &inputFrame); 231 status_t processOneCodecGainmapOutputFrame(int64_t frameNumber, InputFrame &inputFrame); 232 status_t processCompletedInputFrame(int64_t frameNumber, InputFrame &inputFrame); 233 234 void releaseInputFrameLocked(int64_t frameNumber, InputFrame *inputFrame /*out*/); 235 void releaseInputFramesLocked(); 236 237 size_t findAppSegmentsSize(const uint8_t* appSegmentBuffer, size_t maxSize, 238 size_t* app1SegmentSize); 239 status_t copyOneYuvTile(sp<MediaCodecBuffer>& codecBuffer, 240 const CpuConsumer::LockedBuffer& yuvBuffer, 241 size_t top, size_t left, size_t width, size_t height); 242 void initCopyRowFunction(int32_t width); 243 static size_t calcAppSegmentMaxSize(const CameraMetadata& info); 244 void updateCodecQualityLocked(int32_t quality); 245 246 static const nsecs_t kWaitDuration = 10000000; // 10 ms 247 static const int32_t kDefaultJpegQuality = 99; 248 static const auto kJpegDataSpace = HAL_DATASPACE_V0_JFIF; 249 static const android_dataspace kAppSegmentDataSpace = 250 static_cast<android_dataspace>(HAL_DATASPACE_JPEG_APP_SEGMENTS); 251 static const android_dataspace kHeifDataSpace = 252 static_cast<android_dataspace>(HAL_DATASPACE_HEIF); 253 android_dataspace mInternalDataSpace = kHeifDataSpace; 254 // Use the limit of pipeline depth in the API sepc as maximum number of acquired 255 // app segment buffers. 256 static const uint32_t kMaxAcquiredAppSegment = 8; 257 258 int mAppSegmentStreamId, mAppSegmentSurfaceId; 259 sp<CpuConsumer> mAppSegmentConsumer; 260 sp<Surface> mAppSegmentSurface; 261 size_t mAppSegmentMaxSize; 262 std::queue<int64_t> mAppSegmentFrameNumbers; 263 CameraMetadata mStaticInfo; 264 265 int mMainImageStreamId, mMainImageSurfaceId; 266 sp<Surface> mMainImageSurface; 267 sp<CpuConsumer> mMainImageConsumer; // Only applicable for HEVC codec. 268 bool mYuvBufferAcquired; // Only applicable to HEVC codec 269 std::queue<int64_t> mMainImageFrameNumbers; 270 271 static const int32_t kMaxOutputSurfaceProducerCount = 1; 272 sp<Surface> mOutputSurface; 273 sp<StreamSurfaceListener> mStreamSurfaceListener; 274 int32_t mDequeuedOutputBufferCnt; 275 276 // Map from frame number to JPEG setting of orientation+quality 277 struct HeicSettings { 278 int32_t orientation; 279 int32_t quality; 280 int64_t timestamp; 281 int32_t requestId; 282 bool shutterNotified; 283 HeicSettingsHeicSettings284 HeicSettings() : orientation(0), quality(95), timestamp(0), 285 requestId(-1), shutterNotified(false) {} HeicSettingsHeicSettings286 HeicSettings(int32_t _orientation, int32_t _quality) : 287 orientation(_orientation), 288 quality(_quality), timestamp(0), 289 requestId(-1), shutterNotified(false) {} 290 291 }; 292 std::map<int64_t, HeicSettings> mSettingsByFrameNumber; 293 294 // Keep all incoming APP segment Blob buffer pending further processing. 295 std::vector<int64_t> mInputAppSegmentBuffers; 296 297 // Keep all incoming HEIC blob buffer pending further processing. 298 std::vector<CodecOutputBufferInfo> mCodecOutputBuffers, mGainmapCodecOutputBuffers; 299 std::queue<int64_t> mCodecOutputBufferFrameNumbers, mCodecGainmapOutputBufferFrameNumbers; 300 size_t mCodecOutputCounter, mCodecGainmapOutputCounter; 301 int32_t mQuality; 302 303 // Keep all incoming Yuv buffer pending tiling and encoding (for HEVC YUV tiling only) 304 std::vector<int64_t> mInputYuvBuffers; 305 // Keep all codec input buffers ready to be filled out (for HEVC YUV tiling only) 306 std::vector<int32_t> mCodecInputBuffers, mGainmapCodecInputBuffers; 307 308 // Artificial strictly incremental YUV grid timestamp to make encoder happy. 309 int64_t mGridTimestampUs; 310 311 // Indexed by frame number. In most common use case, entries are accessed in order. 312 std::map<int64_t, InputFrame> mPendingInputFrames; 313 314 // Function pointer of libyuv row copy. 315 void (*mFnCopyRow)(const uint8_t* src, uint8_t* dst, int width); 316 317 // A set of APP_SEGMENT error frame numbers 318 std::set<int64_t> mExifErrorFrameNumbers; 319 void flagAnExifErrorFrameNumber(int64_t frameNumber); 320 321 // The status id for tracking the active/idle status of this composite stream 322 int mStatusId; 323 void markTrackerIdle(); 324 325 //APP_SEGMENT stream supported 326 bool mAppSegmentSupported = false; 327 328 bool mHDRGainmapEnabled = false; 329 330 // UltraHDR tonemap color and format aspects 331 static const uhdr_img_fmt_t kUltraHdrInputFmt = UHDR_IMG_FMT_24bppYCbCrP010; 332 static const uhdr_color_gamut kUltraHdrInputGamut = UHDR_CG_BT_2100; 333 static const uhdr_color_transfer kUltraHdrInputTransfer = UHDR_CT_HLG; 334 static const uhdr_color_range kUltraHdrInputRange = UHDR_CR_FULL_RANGE; 335 336 static const uhdr_img_fmt_t kUltraHdrOutputFmt = UHDR_IMG_FMT_12bppYCbCr420; 337 static const uhdr_color_gamut kUltraHdrOutputGamut = UHDR_CG_DISPLAY_P3; 338 static const uhdr_color_transfer kUltraHdrOutputTransfer = UHDR_CT_SRGB; 339 static const uhdr_color_range kUltraHdrOutputRange = UHDR_CR_FULL_RANGE; 340 341 static const auto kUltraHDRDataSpace = 342 aidl::android::hardware::graphics::common::Dataspace::HEIF_ULTRAHDR; 343 344 // MediaMuxer/Codec color and format aspects for base image and gainmap metadata 345 static const int32_t kCodecColorFormat = COLOR_FormatYUV420Flexible; 346 static const ColorAspects::Primaries kCodecColorPrimaries = 347 ColorAspects::Primaries::PrimariesEG432; 348 static const ColorAspects::MatrixCoeffs kCodecColorMatrix = 349 ColorAspects::MatrixCoeffs::MatrixUnspecified; 350 static const ColorAspects::Transfer kCodecColorTransfer = 351 ColorAspects::Transfer::TransferSRGB; 352 static const ColorAspects::Range kCodecColorRange = 353 ColorAspects::Range::RangeFull; 354 355 // MediaMuxer/Codec color and format aspects for gainmap as per ISO 23008-12:2024 356 static const int32_t kCodecGainmapColorFormat = COLOR_FormatYUV420Flexible; 357 static const ColorAspects::Primaries kCodecGainmapColorPrimaries = 358 ColorAspects::Primaries::PrimariesUnspecified; 359 static const ColorAspects::MatrixCoeffs kCodecGainmapColorMatrix = 360 ColorAspects::MatrixCoeffs::MatrixUnspecified; 361 static const ColorAspects::Transfer kCodecGainmapColorTransfer = 362 ColorAspects::Transfer::TransferUnspecified; 363 static const ColorAspects::Range kCodecGainmapColorRange = 364 ColorAspects::Range::RangeFull; 365 366 367 status_t generateBaseImageAndGainmap(InputFrame &inputFrame); 368 }; 369 370 }; // namespace camera3 371 }; // namespace android 372 373 #endif //ANDROID_SERVERS_CAMERA_CAMERA3_HEIC_COMPOSITE_STREAM_H 374