1 /* 2 * Copyright 2017, The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef CCODEC_BUFFER_CHANNEL_H_ 18 19 #define CCODEC_BUFFER_CHANNEL_H_ 20 21 #include <deque> 22 #include <map> 23 #include <memory> 24 #include <vector> 25 26 #include <C2Buffer.h> 27 #include <C2Component.h> 28 #include <Codec2Mapper.h> 29 30 #include <codec2/hidl/client.h> 31 #include <media/stagefright/foundation/Mutexed.h> 32 #include <media/stagefright/CodecBase.h> 33 34 #include "CCodecBuffers.h" 35 #include "FrameReassembler.h" 36 #include "InputSurfaceWrapper.h" 37 #include "PipelineWatcher.h" 38 39 namespace android { 40 41 class MemoryDealer; 42 43 class CCodecCallback { 44 public: 45 virtual ~CCodecCallback() = default; 46 virtual void onError(status_t err, enum ActionCode actionCode) = 0; 47 virtual void onOutputFramesRendered(int64_t mediaTimeUs, nsecs_t renderTimeNs) = 0; 48 virtual void onOutputBuffersChanged() = 0; 49 virtual void onFirstTunnelFrameReady() = 0; 50 }; 51 52 /** 53 * BufferChannelBase implementation for CCodec. 54 */ 55 class CCodecBufferChannel 56 : public BufferChannelBase, public std::enable_shared_from_this<CCodecBufferChannel> { 57 public: 58 explicit CCodecBufferChannel(const std::shared_ptr<CCodecCallback> &callback); 59 virtual ~CCodecBufferChannel(); 60 61 // BufferChannelBase interface 62 void setCrypto(const sp<ICrypto> &crypto) override; 63 void setDescrambler(const sp<IDescrambler> &descrambler) override; 64 65 status_t queueInputBuffer(const sp<MediaCodecBuffer> &buffer) override; 66 status_t queueSecureInputBuffer( 67 const sp<MediaCodecBuffer> &buffer, 68 bool secure, 69 const uint8_t *key, 70 const uint8_t *iv, 71 CryptoPlugin::Mode mode, 72 CryptoPlugin::Pattern pattern, 73 const CryptoPlugin::SubSample *subSamples, 74 size_t numSubSamples, 75 AString *errorDetailMsg) override; 76 status_t queueSecureInputBuffers( 77 const sp<MediaCodecBuffer> &buffer, 78 bool secure, 79 AString *errorDetailMsg) override; 80 status_t attachBuffer( 81 const std::shared_ptr<C2Buffer> &c2Buffer, 82 const sp<MediaCodecBuffer> &buffer) override; 83 status_t attachEncryptedBuffer( 84 const sp<hardware::HidlMemory> &memory, 85 bool secure, 86 const uint8_t *key, 87 const uint8_t *iv, 88 CryptoPlugin::Mode mode, 89 CryptoPlugin::Pattern pattern, 90 size_t offset, 91 const CryptoPlugin::SubSample *subSamples, 92 size_t numSubSamples, 93 const sp<MediaCodecBuffer> &buffer, 94 AString* errorDetailMsg) override; 95 status_t attachEncryptedBuffers( 96 const sp<hardware::HidlMemory> &memory, 97 size_t offset, 98 const sp<MediaCodecBuffer> &buffer, 99 bool secure, 100 AString* errorDetailMsg) override; 101 status_t renderOutputBuffer( 102 const sp<MediaCodecBuffer> &buffer, int64_t timestampNs) override; 103 void pollForRenderedBuffers() override; 104 void onBufferReleasedFromOutputSurface(uint32_t generation) override; 105 void onBufferAttachedToOutputSurface(uint32_t generation) override; 106 status_t discardBuffer(const sp<MediaCodecBuffer> &buffer) override; 107 void getInputBufferArray(Vector<sp<MediaCodecBuffer>> *array) override; 108 void getOutputBufferArray(Vector<sp<MediaCodecBuffer>> *array) override; 109 110 // Methods below are interface for CCodec to use. 111 112 /** 113 * Set the component object for buffer processing. 114 */ 115 void setComponent(const std::shared_ptr<Codec2Client::Component> &component); 116 117 /** 118 * Set output graphic surface for rendering. 119 */ 120 status_t setSurface(const sp<Surface> &surface, uint32_t generation, bool pushBlankBuffer); 121 122 /** 123 * Set GraphicBufferSource object from which the component extracts input 124 * buffers. 125 */ 126 status_t setInputSurface(const std::shared_ptr<InputSurfaceWrapper> &surface); 127 128 /** 129 * Signal EOS to input surface. 130 */ 131 status_t signalEndOfInputStream(); 132 133 /** 134 * Set parameters. 135 */ 136 status_t setParameters(std::vector<std::unique_ptr<C2Param>> ¶ms); 137 138 /** 139 * Start queueing buffers to the component. This object should never queue 140 * buffers before this call has completed. 141 */ 142 status_t start( 143 const sp<AMessage> &inputFormat, 144 const sp<AMessage> &outputFormat, 145 bool buffersBoundToCodec); 146 147 /** 148 * Prepare initial input buffers to be filled by client. 149 * 150 * \param clientInputBuffers[out] pointer to slot index -> buffer map. 151 * On success, it contains prepared 152 * initial input buffers. 153 */ 154 status_t prepareInitialInputBuffers( 155 std::map<size_t, sp<MediaCodecBuffer>> *clientInputBuffers, 156 bool retry = false); 157 158 /** 159 * Request initial input buffers as prepared in clientInputBuffers. 160 * 161 * \param clientInputBuffers[in] slot index -> buffer map with prepared 162 * initial input buffers. 163 */ 164 status_t requestInitialInputBuffers( 165 std::map<size_t, sp<MediaCodecBuffer>> &&clientInputBuffers); 166 167 /** 168 * Stop using buffers of the current output surface for other Codec 169 * instances to use the surface safely. 170 * 171 * \param pushBlankBuffer[in] push a blank buffer at the end if true 172 */ 173 void stopUseOutputSurface(bool pushBlankBuffer); 174 175 /** 176 * Stop queueing buffers to the component. This object should never queue 177 * buffers after this call, until start() is called. 178 */ 179 void stop(); 180 181 /** 182 * Stop queueing buffers to the component and release all buffers. 183 */ 184 void reset(); 185 186 /** 187 * Release all resources. 188 */ 189 void release(); 190 191 void flush(const std::list<std::unique_ptr<C2Work>> &flushedWork); 192 193 /** 194 * Notify input client about work done. 195 * 196 * @param workItems finished work item. 197 * @param inputFormat input format 198 * @param outputFormat new output format if it has changed, otherwise nullptr 199 * @param initData new init data (CSD) if it has changed, otherwise nullptr 200 */ 201 void onWorkDone( 202 std::unique_ptr<C2Work> work, 203 const sp<AMessage> &inputFormat, 204 const sp<AMessage> &outputFormat, 205 const C2StreamInitDataInfo::output *initData); 206 207 /** 208 * Make an input buffer available for the client as it is no longer needed 209 * by the codec. 210 * 211 * @param frameIndex The index of input work 212 * @param arrayIndex The index of buffer in the input work buffers. 213 */ 214 void onInputBufferDone(uint64_t frameIndex, size_t arrayIndex); 215 216 PipelineWatcher::Clock::duration elapsed(); 217 218 enum MetaMode { 219 MODE_NONE, 220 MODE_ANW, 221 }; 222 223 void setMetaMode(MetaMode mode); 224 225 /** 226 * get pixel format from output buffers. 227 * 228 * @return 0 if no valid pixel format found. 229 */ 230 uint32_t getBuffersPixelFormat(bool isEncoder); 231 232 void resetBuffersPixelFormat(bool isEncoder); 233 234 /** 235 * Queue a C2 info buffer that will be sent to codec in the subsequent 236 * queueInputBuffer 237 * 238 * @param buffer C2 info buffer 239 */ 240 void setInfoBuffer(const std::shared_ptr<C2InfoBuffer> &buffer); 241 242 private: 243 uint32_t getInputBuffersPixelFormat(); 244 245 uint32_t getOutputBuffersPixelFormat(); 246 247 class QueueGuard; 248 249 /** 250 * Special mutex-like object with the following properties: 251 * 252 * - At STOPPED state (initial, or after stop()) 253 * - QueueGuard object gets created at STOPPED state, and the client is 254 * supposed to return immediately. 255 * - At RUNNING state (after start()) 256 * - Each QueueGuard object 257 */ 258 class QueueSync { 259 public: 260 /** 261 * At construction the sync object is in STOPPED state. 262 */ QueueSync()263 inline QueueSync() {} 264 ~QueueSync() = default; 265 266 /** 267 * Transition to RUNNING state when stopped. No-op if already in RUNNING 268 * state. 269 */ 270 void start(); 271 272 /** 273 * At RUNNING state, wait until all QueueGuard object created during 274 * RUNNING state are destroyed, and then transition to STOPPED state. 275 * No-op if already in STOPPED state. 276 */ 277 void stop(); 278 279 private: 280 Mutex mGuardLock; 281 282 struct Counter { CounterCounter283 inline Counter() : value(-1) {} 284 int32_t value; 285 Condition cond; 286 }; 287 Mutexed<Counter> mCount; 288 289 friend class CCodecBufferChannel::QueueGuard; 290 }; 291 292 class QueueGuard { 293 public: 294 QueueGuard(QueueSync &sync); 295 ~QueueGuard(); isRunning()296 inline bool isRunning() { return mRunning; } 297 298 private: 299 QueueSync &mSync; 300 bool mRunning; 301 }; 302 303 struct TrackedFrame { 304 uint64_t number; 305 int64_t mediaTimeUs; 306 int64_t desiredRenderTimeNs; 307 nsecs_t latchTime; 308 sp<Fence> presentFence; 309 }; 310 311 void feedInputBufferIfAvailable(); 312 void feedInputBufferIfAvailableInternal(); 313 status_t queueInputBufferInternal(sp<MediaCodecBuffer> buffer, 314 std::shared_ptr<C2LinearBlock> encryptedBlock = nullptr, 315 size_t blockSize = 0); 316 bool handleWork( 317 std::unique_ptr<C2Work> work, 318 const sp<AMessage> &inputFormat, 319 const sp<AMessage> &outputFormat, 320 const C2StreamInitDataInfo::output *initData); 321 void sendOutputBuffers(); 322 void ensureDecryptDestination(size_t size); 323 int32_t getHeapSeqNum(const sp<hardware::HidlMemory> &memory); 324 325 void initializeFrameTrackingFor(ANativeWindow * window); 326 void trackReleasedFrame(const IGraphicBufferProducer::QueueBufferOutput& qbo, 327 int64_t mediaTimeUs, int64_t desiredRenderTimeNs); 328 void processRenderedFrames(const FrameEventHistoryDelta& delta); 329 int64_t getRenderTimeNs(const TrackedFrame& frame); 330 331 QueueSync mSync; 332 sp<MemoryDealer> mDealer; 333 sp<IMemory> mDecryptDestination; 334 int32_t mHeapSeqNum; 335 std::map<wp<hardware::HidlMemory>, int32_t> mHeapSeqNumMap; 336 337 std::shared_ptr<Codec2Client::Component> mComponent; 338 std::string mComponentName; ///< component name for debugging 339 const char *mName; ///< C-string version of component name 340 std::shared_ptr<CCodecCallback> mCCodecCallback; 341 std::shared_ptr<C2BlockPool> mInputAllocator; 342 QueueSync mQueueSync; 343 std::vector<std::unique_ptr<C2Param>> mParamsToBeSet; 344 sp<AMessage> mOutputFormat; 345 346 struct Input { 347 Input(); 348 349 std::unique_ptr<InputBuffers> buffers; 350 size_t numSlots; 351 FlexBuffersImpl extraBuffers; 352 size_t numExtraSlots; 353 uint32_t inputDelay; 354 uint32_t pipelineDelay; 355 c2_cntr64_t lastFlushIndex; 356 357 FrameReassembler frameReassembler; 358 }; 359 Mutexed<Input> mInput; 360 struct Output { 361 std::unique_ptr<OutputBuffers> buffers; 362 size_t numSlots; 363 uint32_t outputDelay; 364 // true iff the underlying block pool is bounded --- for example, 365 // a BufferQueue-based block pool would be bounded by the BufferQueue. 366 bool bounded; 367 }; 368 Mutexed<Output> mOutput; 369 Mutexed<std::list<std::unique_ptr<C2Work>>> mFlushedConfigs; 370 371 std::atomic_uint64_t mFrameIndex; 372 std::atomic_uint64_t mFirstValidFrameIndex; 373 374 sp<MemoryDealer> makeMemoryDealer(size_t heapSize); 375 376 std::deque<TrackedFrame> mTrackedFrames; 377 bool mAreRenderMetricsEnabled; 378 bool mIsSurfaceToDisplay; 379 bool mHasPresentFenceTimes; 380 381 struct OutputSurface { 382 sp<Surface> surface; 383 uint32_t generation; 384 int maxDequeueBuffers; 385 std::map<uint64_t, int> rotation; 386 }; 387 Mutexed<OutputSurface> mOutputSurface; 388 int mRenderingDepth; 389 390 struct BlockPools { 391 C2Allocator::id_t inputAllocatorId; 392 std::shared_ptr<C2BlockPool> inputPool; 393 C2Allocator::id_t outputAllocatorId; 394 C2BlockPool::local_id_t outputPoolId; 395 std::shared_ptr<Codec2Client::Configurable> outputPoolIntf; 396 }; 397 Mutexed<BlockPools> mBlockPools; 398 399 std::atomic_bool mHasInputSurface; 400 struct InputSurface { 401 std::shared_ptr<InputSurfaceWrapper> surface; 402 // This variable tracks the number of buffers processing 403 // in the input surface and codec by counting the # of buffers to 404 // be filled in and queued from the input surface and the # of 405 // buffers generated from the codec. 406 // 407 // Note that this variable can go below 0, because it does not take 408 // account the number of buffers initially in the buffer queue at 409 // start. This is okay, as we only track how many more we allow 410 // from the initial state. 411 int64_t numProcessingBuffersBalance; 412 }; 413 Mutexed<InputSurface> mInputSurface; 414 415 MetaMode mMetaMode; 416 417 Mutexed<PipelineWatcher> mPipelineWatcher; 418 419 std::atomic_bool mInputMetEos; 420 std::once_flag mRenderWarningFlag; 421 422 sp<ICrypto> mCrypto; 423 sp<IDescrambler> mDescrambler; 424 hasCryptoOrDescrambler()425 inline bool hasCryptoOrDescrambler() { 426 return mCrypto != nullptr || mDescrambler != nullptr; 427 } 428 std::atomic_bool mSendEncryptedInfoBuffer; 429 430 std::atomic_bool mTunneled; 431 432 std::vector<std::shared_ptr<C2InfoBuffer>> mInfoBuffers; 433 }; 434 435 // Conversion of a c2_status_t value to a status_t value may depend on the 436 // operation that returns the c2_status_t value. 437 enum c2_operation_t { 438 C2_OPERATION_NONE, 439 C2_OPERATION_Component_connectToOmxInputSurface, 440 C2_OPERATION_Component_createBlockPool, 441 C2_OPERATION_Component_destroyBlockPool, 442 C2_OPERATION_Component_disconnectFromInputSurface, 443 C2_OPERATION_Component_drain, 444 C2_OPERATION_Component_flush, 445 C2_OPERATION_Component_queue, 446 C2_OPERATION_Component_release, 447 C2_OPERATION_Component_reset, 448 C2_OPERATION_Component_setOutputSurface, 449 C2_OPERATION_Component_start, 450 C2_OPERATION_Component_stop, 451 C2_OPERATION_ComponentStore_copyBuffer, 452 C2_OPERATION_ComponentStore_createComponent, 453 C2_OPERATION_ComponentStore_createInputSurface, 454 C2_OPERATION_ComponentStore_createInterface, 455 C2_OPERATION_Configurable_config, 456 C2_OPERATION_Configurable_query, 457 C2_OPERATION_Configurable_querySupportedParams, 458 C2_OPERATION_Configurable_querySupportedValues, 459 C2_OPERATION_InputSurface_connectToComponent, 460 C2_OPERATION_InputSurfaceConnection_disconnect, 461 }; 462 463 status_t toStatusT(c2_status_t c2s, c2_operation_t c2op = C2_OPERATION_NONE); 464 465 } // namespace android 466 467 #endif // CCODEC_BUFFER_CHANNEL_H_ 468