1 /* 2 * Copyright 2012, The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef MEDIA_CODEC_H_ 18 19 #define MEDIA_CODEC_H_ 20 21 #include <list> 22 #include <memory> 23 #include <vector> 24 25 #include <gui/IGraphicBufferProducer.h> 26 #include <media/hardware/CryptoAPI.h> 27 #include <media/MediaCodecInfo.h> 28 #include <media/MediaMetrics.h> 29 #include <media/MediaProfiles.h> 30 #include <media/stagefright/foundation/AHandler.h> 31 #include <media/stagefright/foundation/AMessage.h> 32 #include <media/stagefright/CodecErrorLog.h> 33 #include <media/stagefright/FrameRenderTracker.h> 34 #include <media/stagefright/MediaHistogram.h> 35 #include <media/stagefright/PlaybackDurationAccumulator.h> 36 #include <media/stagefright/VideoRenderQualityTracker.h> 37 #include <utils/Vector.h> 38 39 class C2Buffer; 40 class C2GraphicBlock; 41 class C2LinearBlock; 42 43 namespace aidl { 44 namespace android { 45 namespace media { 46 class MediaResourceParcel; 47 class ClientConfigParcel; 48 } // media 49 } // android 50 } // aidl 51 52 namespace android { 53 54 struct ABuffer; 55 struct AMessage; 56 struct AReplyToken; 57 struct AString; 58 struct BatteryChecker; 59 class BufferChannelBase; 60 struct AccessUnitInfo; 61 struct CodecBase; 62 struct CodecCryptoInfo; 63 struct CodecParameterDescriptor; 64 class IBatteryStats; 65 struct ICrypto; 66 class CryptoAsync; 67 class MediaCodecBuffer; 68 class IMemory; 69 struct PersistentSurface; 70 class RenderedFrameInfo; 71 class SoftwareRenderer; 72 class Surface; 73 namespace hardware { 74 namespace cas { 75 namespace native { 76 namespace V1_0 { 77 struct IDescrambler; 78 }}}} 79 80 using hardware::cas::native::V1_0::IDescrambler; 81 using aidl::android::media::MediaResourceParcel; 82 using aidl::android::media::ClientConfigParcel; 83 84 typedef WrapperObject<std::vector<AccessUnitInfo>> BufferInfosWrapper; 85 typedef WrapperObject<std::vector<std::unique_ptr<CodecCryptoInfo>>> CryptoInfosWrapper; 86 87 struct MediaCodec : public AHandler { 88 enum Domain { 89 DOMAIN_UNKNOWN = 0, 90 DOMAIN_VIDEO = 1, 91 DOMAIN_AUDIO = 2, 92 DOMAIN_IMAGE = 3 93 }; 94 95 enum ConfigureFlags { 96 CONFIGURE_FLAG_ENCODE = 1, 97 CONFIGURE_FLAG_USE_BLOCK_MODEL = 2, 98 CONFIGURE_FLAG_USE_CRYPTO_ASYNC = 4, 99 CONFIGURE_FLAG_DETACHED_SURFACE = 8, 100 }; 101 102 enum BufferFlags { 103 BUFFER_FLAG_SYNCFRAME = 1, 104 BUFFER_FLAG_CODECCONFIG = 2, 105 BUFFER_FLAG_EOS = 4, 106 BUFFER_FLAG_PARTIAL_FRAME = 8, 107 BUFFER_FLAG_MUXER_DATA = 16, 108 BUFFER_FLAG_DECODE_ONLY = 32, 109 }; 110 111 enum CVODegree { 112 CVO_DEGREE_0 = 0, 113 CVO_DEGREE_90 = 90, 114 CVO_DEGREE_180 = 180, 115 CVO_DEGREE_270 = 270, 116 }; 117 118 enum { 119 CB_INPUT_AVAILABLE = 1, 120 CB_OUTPUT_AVAILABLE = 2, 121 CB_ERROR = 3, 122 CB_OUTPUT_FORMAT_CHANGED = 4, 123 CB_RESOURCE_RECLAIMED = 5, 124 CB_CRYPTO_ERROR = 6, 125 CB_LARGE_FRAME_OUTPUT_AVAILABLE = 7, 126 127 /** Callback ID for when the metrics for this codec have been flushed 128 * due to the start of a new subsession. The associated AMessage will 129 * contain an sp<WrapperObject<std::unique_ptr<mediametrics::Item>>> 130 * Object at the "metrics" key. 131 */ 132 CB_METRICS_FLUSHED = 8, 133 134 /** Callback ID to notify the change in resource requirement 135 * for the codec component. 136 */ 137 CB_REQUIRED_RESOURCES_CHANGED = 9, 138 }; 139 140 static const pid_t kNoPid = -1; 141 static const uid_t kNoUid = -1; 142 143 static sp<MediaCodec> CreateByType( 144 const sp<ALooper> &looper, const AString &mime, bool encoder, status_t *err = NULL, 145 pid_t pid = kNoPid, uid_t uid = kNoUid); 146 147 static sp<MediaCodec> CreateByType( 148 const sp<ALooper> &looper, const AString &mime, bool encoder, status_t *err, 149 pid_t pid, uid_t uid, sp<AMessage> format); 150 151 static sp<MediaCodec> CreateByComponentName( 152 const sp<ALooper> &looper, const AString &name, status_t *err = NULL, 153 pid_t pid = kNoPid, uid_t uid = kNoUid); 154 155 static sp<PersistentSurface> CreatePersistentInputSurface(); 156 157 /** 158 * Abstraction for the Global Codec resources. 159 * This encapsulates all the available codec resources on the device. 160 */ 161 struct GlobalResourceInfo { 162 /** 163 * Name of the Resource type. 164 */ 165 std::string mName; 166 /** 167 * Total count/capacity of resources of this type. 168 */ 169 int mCapacity; 170 /** 171 * Available count of this resource type. 172 */ 173 int mAvailable; 174 GlobalResourceInfoMediaCodec::GlobalResourceInfo175 GlobalResourceInfo(const std::string& name, int capacity, int available) : 176 mName(name), 177 mCapacity(capacity), 178 mAvailable(available) {} 179 GlobalResourceInfoMediaCodec::GlobalResourceInfo180 GlobalResourceInfo(const GlobalResourceInfo& info) : 181 mName(info.mName), 182 mCapacity(info.mCapacity), 183 mAvailable(info.mAvailable) {} 184 }; 185 186 /** 187 * Abstraction for the resources associated with a codec instance. 188 * This encapsulates the required codec resources for a configured codec instance. 189 */ 190 struct InstanceResourceInfo { 191 /** 192 * Name of the Resource type. 193 */ 194 std::string mName; 195 /** 196 * Required resource count of this type. 197 */ 198 int mStaticCount; 199 /** 200 * Per frame resource requirement of this resource type. 201 */ 202 int mPerFrameCount; 203 InstanceResourceInfoMediaCodec::InstanceResourceInfo204 InstanceResourceInfo(const std::string& name, int staticCount, int perFrameCount) : 205 mName(name), 206 mStaticCount(staticCount), 207 mPerFrameCount(perFrameCount) {} 208 InstanceResourceInfoMediaCodec::InstanceResourceInfo209 InstanceResourceInfo(const InstanceResourceInfo& info) : 210 mName(info.mName), 211 mStaticCount(info.mStaticCount), 212 mPerFrameCount(info.mPerFrameCount) {} 213 }; 214 215 /** 216 * Get a list of Globally available device codec resources. 217 * 218 * It will return INVALID_OPERATION if: 219 * - HAL does not implement codec availability API 220 * - codec_availability feature flag isn't defined. 221 */ 222 static status_t getGloballyAvailableResources(std::vector<GlobalResourceInfo>& resources); 223 224 status_t configure( 225 const sp<AMessage> &format, 226 const sp<Surface> &nativeWindow, 227 const sp<ICrypto> &crypto, 228 uint32_t flags); 229 230 status_t configure( 231 const sp<AMessage> &format, 232 const sp<Surface> &nativeWindow, 233 const sp<ICrypto> &crypto, 234 const sp<IDescrambler> &descrambler, 235 uint32_t flags); 236 237 /** 238 * Get a list of required codec resources. 239 * 240 * This may only be called after configuring the codec. 241 * 242 * Calling this while the codec wasn't configured, will result in 243 * returning INVALID_OPERATION error code. 244 * It will also return INVALID_OPERATION if: 245 * - HAL does not implement codec availability API 246 * - codec_availability feature flag isn't defined. 247 */ 248 status_t getRequiredResources(std::vector<InstanceResourceInfo>& resources); 249 250 status_t releaseCrypto(); 251 252 status_t setCallback(const sp<AMessage> &callback); 253 254 status_t setOnFrameRenderedNotification(const sp<AMessage> ¬ify); 255 256 status_t setOnFirstTunnelFrameReadyNotification(const sp<AMessage> ¬ify); 257 258 status_t createInputSurface(sp<IGraphicBufferProducer>* bufferProducer); 259 260 status_t setInputSurface(const sp<PersistentSurface> &surface); 261 262 status_t start(); 263 264 // Returns to a state in which the component remains allocated but 265 // unconfigured. 266 status_t stop(); 267 268 // Resets the codec to the INITIALIZED state. Can be called after an error 269 // has occured to make the codec usable. 270 status_t reset(); 271 272 // Client MUST call release before releasing final reference to this 273 // object. 274 status_t release(); 275 276 status_t releaseAsync(const sp<AMessage> ¬ify); 277 278 status_t flush(); 279 280 status_t queueInputBuffer( 281 size_t index, 282 size_t offset, 283 size_t size, 284 int64_t presentationTimeUs, 285 uint32_t flags, 286 AString *errorDetailMsg = NULL); 287 288 status_t queueInputBuffers( 289 size_t index, 290 size_t offset, 291 size_t size, 292 const sp<BufferInfosWrapper> &accessUnitInfo, 293 AString *errorDetailMsg = NULL); 294 295 status_t queueSecureInputBuffer( 296 size_t index, 297 size_t offset, 298 const CryptoPlugin::SubSample *subSamples, 299 size_t numSubSamples, 300 const uint8_t key[16], 301 const uint8_t iv[16], 302 CryptoPlugin::Mode mode, 303 const CryptoPlugin::Pattern &pattern, 304 int64_t presentationTimeUs, 305 uint32_t flags, 306 AString *errorDetailMsg = NULL); 307 308 status_t queueSecureInputBuffers( 309 size_t index, 310 size_t offset, 311 size_t size, 312 const sp<BufferInfosWrapper> &accessUnitInfo, 313 const sp<CryptoInfosWrapper> &cryptoInfos, 314 AString *errorDetailMsg = NULL); 315 316 status_t queueBuffer( 317 size_t index, 318 const std::shared_ptr<C2Buffer> &buffer, 319 const sp<BufferInfosWrapper> &bufferInfos, 320 const sp<AMessage> &tunings, 321 AString *errorDetailMsg = NULL); 322 323 status_t queueEncryptedBuffer( 324 size_t index, 325 const sp<hardware::HidlMemory> &memory, 326 size_t offset, 327 size_t size, 328 const sp<BufferInfosWrapper> &bufferInfos, 329 const sp<CryptoInfosWrapper> &cryptoInfos, 330 const sp<AMessage> &tunings, 331 AString *errorDetailMsg = NULL); 332 333 std::shared_ptr<C2Buffer> decrypt( 334 const std::shared_ptr<C2Buffer> &buffer, 335 const CryptoPlugin::SubSample *subSamples, 336 size_t numSubSamples, 337 const uint8_t key[16], 338 const uint8_t iv[16], 339 CryptoPlugin::Mode mode, 340 const CryptoPlugin::Pattern &pattern); 341 342 status_t dequeueInputBuffer(size_t *index, int64_t timeoutUs = 0ll); 343 344 status_t dequeueOutputBuffer( 345 size_t *index, 346 size_t *offset, 347 size_t *size, 348 int64_t *presentationTimeUs, 349 uint32_t *flags, 350 int64_t timeoutUs = 0ll); 351 352 status_t renderOutputBufferAndRelease(size_t index, int64_t timestampNs); 353 status_t renderOutputBufferAndRelease(size_t index); 354 status_t releaseOutputBuffer(size_t index); 355 356 status_t signalEndOfInputStream(); 357 358 status_t getOutputFormat(sp<AMessage> *format) const; 359 status_t getInputFormat(sp<AMessage> *format) const; 360 361 status_t getInputBuffers(Vector<sp<MediaCodecBuffer> > *buffers) const; 362 status_t getOutputBuffers(Vector<sp<MediaCodecBuffer> > *buffers) const; 363 364 status_t getOutputBuffer(size_t index, sp<MediaCodecBuffer> *buffer); 365 status_t getOutputFormat(size_t index, sp<AMessage> *format); 366 status_t getInputBuffer(size_t index, sp<MediaCodecBuffer> *buffer); 367 368 status_t setSurface(const sp<Surface> &nativeWindow); 369 370 status_t detachOutputSurface(); 371 372 status_t requestIDRFrame(); 373 374 // Notification will be posted once there "is something to do", i.e. 375 // an input/output buffer has become available, a format change is 376 // pending, an error is pending. 377 void requestActivityNotification(const sp<AMessage> ¬ify); 378 379 status_t getName(AString *componentName) const; 380 381 status_t getCodecInfo(sp<MediaCodecInfo> *codecInfo) const; 382 383 status_t getMetrics(mediametrics_handle_t &reply); 384 385 status_t setParameters(const sp<AMessage> ¶ms); 386 387 status_t querySupportedVendorParameters(std::vector<std::string> *names); 388 status_t describeParameter(const std::string &name, CodecParameterDescriptor *desc); 389 status_t subscribeToVendorParameters(const std::vector<std::string> &names); 390 status_t unsubscribeFromVendorParameters(const std::vector<std::string> &names); 391 392 // Create a MediaCodec notification message from a list of rendered or dropped render infos 393 // by adding rendered frame information to a base notification message. Returns the number 394 // of frames that were rendered. 395 static size_t CreateFramesRenderedMessage( 396 const std::list<RenderedFrameInfo> &done, sp<AMessage> &msg); 397 static size_t CreateFramesRenderedMessage( 398 const std::list<FrameRenderTracker::Info> &done, sp<AMessage> &msg); 399 400 static status_t CanFetchLinearBlock( 401 const std::vector<std::string> &names, bool *isCompatible); 402 403 static std::shared_ptr<C2LinearBlock> FetchLinearBlock( 404 size_t capacity, const std::vector<std::string> &names); 405 406 static status_t CanFetchGraphicBlock( 407 const std::vector<std::string> &names, bool *isCompatible); 408 409 static std::shared_ptr<C2GraphicBlock> FetchGraphicBlock( 410 int32_t width, 411 int32_t height, 412 int32_t format, 413 uint64_t usage, 414 const std::vector<std::string> &names); 415 416 template <typename T> 417 struct WrapperObject : public RefBase { WrapperObjectMediaCodec::WrapperObject418 WrapperObject(const T& v) : value(v) {} WrapperObjectMediaCodec::WrapperObject419 WrapperObject(T&& v) : value(std::move(v)) {} 420 T value; 421 }; 422 getErrorLogMediaCodec423 inline CodecErrorLog &getErrorLog() { return mErrorLog; } 424 425 protected: 426 virtual ~MediaCodec(); 427 virtual void onMessageReceived(const sp<AMessage> &msg); 428 429 private: 430 // used by ResourceManagerClient 431 status_t reclaim(bool force = false); 432 friend struct ResourceManagerClient; 433 434 // to create the metrics associated with this codec. 435 // Any error in this function will be captured by the output argument err. 436 mediametrics_handle_t createMediaMetrics(const sp<AMessage>& format, 437 uint32_t flags, 438 status_t* err); 439 440 private: 441 enum State { 442 UNINITIALIZED, 443 INITIALIZING, 444 INITIALIZED, 445 CONFIGURING, 446 CONFIGURED, 447 STARTING, 448 STARTED, 449 FLUSHING, 450 FLUSHED, 451 STOPPING, 452 RELEASING, 453 }; 454 std::string stateString(State state); 455 std::string apiStateString(); 456 457 enum { 458 kPortIndexInput = 0, 459 kPortIndexOutput = 1, 460 }; 461 462 enum { 463 kWhatInit = 'init', 464 kWhatConfigure = 'conf', 465 kWhatSetSurface = 'sSur', 466 kWhatDetachSurface = 'dSur', 467 kWhatCreateInputSurface = 'cisf', 468 kWhatSetInputSurface = 'sisf', 469 kWhatStart = 'strt', 470 kWhatStop = 'stop', 471 kWhatRelease = 'rele', 472 kWhatDequeueInputBuffer = 'deqI', 473 kWhatQueueInputBuffer = 'queI', 474 kWhatDequeueOutputBuffer = 'deqO', 475 kWhatReleaseOutputBuffer = 'relO', 476 kWhatSignalEndOfInputStream = 'eois', 477 kWhatGetBuffers = 'getB', 478 kWhatFlush = 'flus', 479 kWhatGetOutputFormat = 'getO', 480 kWhatGetInputFormat = 'getI', 481 kWhatDequeueInputTimedOut = 'dITO', 482 kWhatDequeueOutputTimedOut = 'dOTO', 483 kWhatCodecNotify = 'codc', 484 kWhatRequestIDRFrame = 'ridr', 485 kWhatRequestActivityNotification = 'racN', 486 kWhatGetName = 'getN', 487 kWhatGetCodecInfo = 'gCoI', 488 kWhatSetParameters = 'setP', 489 kWhatSetCallback = 'setC', 490 kWhatSetNotification = 'setN', 491 kWhatDrmReleaseCrypto = 'rDrm', 492 kWhatCheckBatteryStats = 'chkB', 493 kWhatGetMetrics = 'getM', 494 }; 495 496 enum { 497 kFlagUsesSoftwareRenderer = 1, 498 kFlagOutputFormatChanged = 2, 499 kFlagOutputBuffersChanged = 4, 500 kFlagStickyError = 8, 501 kFlagDequeueInputPending = 16, 502 kFlagDequeueOutputPending = 32, 503 kFlagIsSecure = 64, 504 kFlagSawMediaServerDie = 128, 505 kFlagIsEncoder = 256, 506 // 512 skipped 507 kFlagIsAsync = 1024, 508 kFlagIsComponentAllocated = 2048, 509 kFlagPushBlankBuffersOnShutdown = 4096, 510 kFlagUseBlockModel = 8192, 511 kFlagUseCryptoAsync = 16384, 512 }; 513 514 struct BufferInfo { 515 BufferInfo(); 516 517 sp<MediaCodecBuffer> mData; 518 bool mOwnedByClient; 519 }; 520 521 // This type is used to track the tunnel mode video peek state machine: 522 // 523 // DisabledNoBuffer -> EnabledNoBuffer when tunnel-peek = true 524 // DisabledQueued -> EnabledQueued when tunnel-peek = true 525 // DisabledNoBuffer -> DisabledQueued when first frame queued 526 // EnabledNoBuffer -> DisabledNoBuffer when tunnel-peek = false 527 // EnabledQueued -> DisabledQueued when tunnel-peek = false 528 // EnabledNoBuffer -> EnabledQueued when first frame queued 529 // DisabledNoBuffer -> BufferDecoded when kWhatFirstTunnelFrameReady 530 // DisabledQueued -> BufferDecoded when kWhatFirstTunnelFrameReady 531 // EnabledNoBuffer -> BufferDecoded when kWhatFirstTunnelFrameReady 532 // EnabledQueued -> BufferDecoded when kWhatFirstTunnelFrameReady 533 // BufferDecoded -> BufferRendered when kWhatFrameRendered 534 // <all states> -> EnabledNoBuffer when flush 535 // <all states> -> EnabledNoBuffer when stop then configure then start 536 enum struct TunnelPeekState { 537 kLegacyMode, 538 kDisabledNoBuffer, 539 kEnabledNoBuffer, 540 kDisabledQueued, 541 kEnabledQueued, 542 kBufferDecoded, 543 kBufferRendered, 544 }; 545 546 enum class DequeueOutputResult { 547 kNoBuffer, 548 kDiscardedBuffer, 549 kRepliedWithError, 550 kSuccess, 551 }; 552 553 struct ResourceManagerServiceProxy; 554 555 State mState; 556 bool mReleasedByResourceManager; 557 sp<ALooper> mLooper; 558 sp<ALooper> mCodecLooper; 559 sp<CodecBase> mCodec; 560 AString mComponentName; 561 AString mOwnerName; 562 sp<MediaCodecInfo> mCodecInfo; 563 sp<AReplyToken> mReplyID; 564 std::string mLastReplyOrigin; 565 std::vector<sp<AMessage>> mDeferredMessages; 566 uint32_t mFlags; 567 int64_t mPresentationTimeUs = 0; 568 status_t mStickyError; 569 sp<Surface> mSurface; 570 uint32_t mSurfaceGeneration = 0; 571 SoftwareRenderer *mSoftRenderer; 572 573 // Get the detached BufferQueue surface for a video decoder, and create it 574 // if it did not yet exist. 575 sp<Surface> getOrCreateDetachedSurface(); 576 577 Mutex mMetricsLock; 578 mediametrics_handle_t mMetricsHandle = 0; 579 mediametrics_handle_t mLastMetricsHandle = 0; // only accessed from the looper or destructor 580 bool mMetricsToUpload = false; 581 nsecs_t mLifetimeStartNs = 0; 582 void initMediametrics(); 583 void updateMediametrics(); 584 void flushMediametrics(); 585 void resetMetricsFields(); 586 587 // Reset the metrics fields for a new subsession. 588 void resetSubsessionMetricsFields(); 589 590 // Start a new subsession (for metrics). This includes flushing the current 591 // metrics, notifying the client and resetting the session fields. 592 void handleStartingANewSubsession(); 593 594 void updateEphemeralMediametrics(mediametrics_handle_t item); 595 void updateLowLatency(const sp<AMessage> &msg); 596 void updateCodecImportance(const sp<AMessage>& msg); 597 void onGetMetrics(const sp<AMessage>& msg); 598 constexpr const char *asString(TunnelPeekState state, const char *default_string="?"); 599 void updateTunnelPeek(const sp<AMessage> &msg); 600 void processRenderedFrames(const sp<AMessage> &msg); 601 602 inline void initClientConfigParcel(ClientConfigParcel& clientConfig); 603 604 sp<AMessage> mOutputFormat; 605 sp<AMessage> mInputFormat; 606 sp<AMessage> mCallback; 607 sp<AMessage> mOnFrameRenderedNotification; 608 sp<AMessage> mAsyncReleaseCompleteNotification; 609 sp<AMessage> mOnFirstTunnelFrameReadyNotification; 610 611 std::shared_ptr<ResourceManagerServiceProxy> mResourceManagerProxy; 612 613 Domain mDomain; 614 AString mLogSessionId; 615 int32_t mWidth; 616 int32_t mHeight; 617 int32_t mRotationDegrees; 618 int32_t mAllowFrameDroppingBySurface; 619 620 enum { 621 kFlagHasHdrStaticInfo = 1, 622 kFlagHasHdr10PlusInfo = 2, 623 }; 624 uint32_t mHdrInfoFlags; 625 void updateHdrMetrics(bool isConfig); 626 hdr_format getHdrFormat(const AString &mime, const int32_t profile, 627 const int32_t colorTransfer); 628 hdr_format getHdrFormatForEncoder(const AString &mime, const int32_t profile, 629 const int32_t colorTransfer); 630 hdr_format getHdrFormatForDecoder(const AString &mime, const int32_t profile, 631 const int32_t colorTransfer); 632 bool profileSupport10Bits(const AString &mime, const int32_t profile); 633 634 struct ApiUsageMetrics { 635 bool isArrayMode; 636 enum OperationMode { 637 kUnknownMode = 0, 638 kSynchronousMode = 1, 639 kAsynchronousMode = 2, 640 kBlockMode = 3, 641 }; 642 OperationMode operationMode; 643 bool isUsingOutputSurface; 644 struct InputBufferSize { 645 int32_t appMax; // max size configured by the app 646 int32_t usedMax; // max size actually used 647 int32_t codecMax; // max size suggested by the codec 648 } inputBufferSize; 649 } mApiUsageMetrics; 650 struct ReliabilityContextMetrics { 651 int32_t flushCount; 652 int32_t setOutputSurfaceCount; 653 int32_t resolutionChangeCount; 654 } mReliabilityContextMetrics; 655 int32_t mSubsessionCount; 656 657 // initial create parameters 658 AString mInitName; 659 660 // configure parameter 661 sp<AMessage> mConfigureMsg; 662 663 // rewrites the format description during configure() for encoding. 664 // format and flags as they exist within configure() 665 // the (possibly) updated format is returned in place. 666 status_t shapeMediaFormat( 667 const sp<AMessage> &format, 668 uint32_t flags, 669 mediametrics_handle_t handle); 670 671 // populate the format shaper library with information for this codec encoding 672 // for the indicated media type 673 status_t setupFormatShaper(AString mediaType); 674 675 // Used only to synchronize asynchronous getBufferAndFormat 676 // across all the other (synchronous) buffer state change 677 // operations, such as de/queueIn/OutputBuffer, start and 678 // stop/flush/reset/release. 679 Mutex mBufferLock; 680 681 std::list<size_t> mAvailPortBuffers[2]; 682 std::vector<BufferInfo> mPortBuffers[2]; 683 684 int32_t mDequeueInputTimeoutGeneration; 685 sp<AReplyToken> mDequeueInputReplyID; 686 687 int32_t mDequeueOutputTimeoutGeneration; 688 sp<AReplyToken> mDequeueOutputReplyID; 689 690 sp<ICrypto> mCrypto; 691 692 int32_t mTunneledInputWidth; 693 int32_t mTunneledInputHeight; 694 bool mTunneled; 695 TunnelPeekState mTunnelPeekState; 696 bool mTunnelPeekEnabled; 697 698 sp<IDescrambler> mDescrambler; 699 700 std::list<sp<ABuffer> > mCSD; 701 702 sp<AMessage> mActivityNotify; 703 704 bool mHaveInputSurface; 705 bool mHavePendingInputBuffers; 706 bool mCpuBoostRequested; 707 708 std::shared_ptr<BufferChannelBase> mBufferChannel; 709 sp<CryptoAsync> mCryptoAsync; 710 sp<ALooper> mCryptoLooper; 711 712 bool mIsSurfaceToDisplay; 713 bool mAreRenderMetricsEnabled; 714 PlaybackDurationAccumulator mPlaybackDurationAccumulator; 715 VideoRenderQualityTracker mVideoRenderQualityTracker; 716 717 MediaCodec( 718 const sp<ALooper> &looper, pid_t pid, uid_t uid, 719 std::function<sp<CodecBase>(const AString &, const char *)> getCodecBase = nullptr, 720 std::function<status_t(const AString &, sp<MediaCodecInfo> *)> getCodecInfo = nullptr); 721 722 static sp<CodecBase> GetCodecBase(const AString &name, const char *owner = nullptr); 723 724 static status_t PostAndAwaitResponse( 725 const sp<AMessage> &msg, sp<AMessage> *response); 726 727 void PostReplyWithError(const sp<AMessage> &msg, int32_t err); 728 void PostReplyWithError(const sp<AReplyToken> &replyID, int32_t err); 729 730 status_t init(const AString &name); 731 732 void setState(State newState); 733 void returnBuffersToCodec(bool isReclaim = false); 734 void returnBuffersToCodecOnPort(int32_t portIndex, bool isReclaim = false); 735 size_t updateBuffers(int32_t portIndex, const sp<AMessage> &msg); 736 status_t onQueueInputBuffer(const sp<AMessage> &msg); 737 status_t onReleaseOutputBuffer(const sp<AMessage> &msg); 738 BufferInfo *peekNextPortBuffer(int32_t portIndex); 739 ssize_t dequeuePortBuffer(int32_t portIndex); 740 741 status_t getBufferAndFormat( 742 size_t portIndex, size_t index, 743 sp<MediaCodecBuffer> *buffer, sp<AMessage> *format); 744 745 bool handleDequeueInputBuffer(const sp<AReplyToken> &replyID, bool newRequest = false); 746 DequeueOutputResult handleDequeueOutputBuffer( 747 const sp<AReplyToken> &replyID, 748 bool newRequest = false); 749 void cancelPendingDequeueOperations(); 750 751 void extractCSD(const sp<AMessage> &format); 752 status_t queueCSDInputBuffer(size_t bufferIndex); 753 754 status_t handleSetSurface(const sp<Surface> &surface); 755 756 // Common reimplementation of changing the output surface. 757 // Handles setting null surface, which is used during configure and init. 758 // Set |callCodec| to true if the codec needs to be notified (e.g. during executing state). 759 // Setting |onShutdown| to true will avoid extra work, if this is used for detaching on 760 // delayed release. 761 status_t handleSetSurface(const sp<Surface> &surface, bool callCodec, bool onShutdown = false); 762 status_t connectToSurface(const sp<Surface> &surface, uint32_t *generation); 763 status_t disconnectFromSurface(); 764 hasCryptoOrDescramblerMediaCodec765 bool hasCryptoOrDescrambler() { 766 return mCrypto != NULL || mDescrambler != NULL; 767 } 768 769 void postActivityNotificationIfPossible(); 770 771 void onInputBufferAvailable(); 772 void onOutputBufferAvailable(); 773 void onCryptoError(const sp<AMessage> &msg); 774 void onError(status_t err, int32_t actionCode, const char *detail = NULL); 775 void onOutputFormatChanged(); 776 void onRequiredResourcesChanged(const std::vector<InstanceResourceInfo>& resourceInfo); 777 778 status_t onSetParameters(const sp<AMessage> ¶ms); 779 780 status_t amendOutputFormatWithCodecSpecificData(const sp<MediaCodecBuffer> &buffer); 781 void handleOutputFormatChangeIfNeeded(const sp<MediaCodecBuffer> &buffer); 782 bool isExecuting() const; 783 784 uint64_t getGraphicBufferSize(); 785 void requestCpuBoostIfNeeded(); 786 787 bool hasPendingBuffer(int portIndex); 788 bool hasPendingBuffer(); 789 790 void postPendingRepliesAndDeferredMessages(std::string origin, status_t err = OK); 791 void postPendingRepliesAndDeferredMessages(std::string origin, const sp<AMessage> &response); 792 793 /* called to get the last codec error when the sticky flag is set. 794 * if no such codec error is found, returns UNKNOWN_ERROR. 795 */ getStickyErrorMediaCodec796 inline status_t getStickyError() const { 797 return mStickyError != 0 ? mStickyError : UNKNOWN_ERROR; 798 } 799 setStickyErrorMediaCodec800 inline void setStickyError(status_t err) { 801 mFlags |= kFlagStickyError; 802 mStickyError = err; 803 } 804 805 void onReleaseCrypto(const sp<AMessage>& msg); 806 807 // managing time-of-flight aka latency 808 typedef struct { 809 int64_t presentationUs; 810 int64_t startedNs; 811 } BufferFlightTiming_t; 812 std::deque<BufferFlightTiming_t> mBuffersInFlight; 813 Mutex mLatencyLock; 814 int64_t mLatencyUnknown; // buffers for which we couldn't calculate latency 815 816 Mutex mOutputStatsLock; 817 int64_t mBytesEncoded = 0; 818 int64_t mEarliestEncodedPtsUs = INT64_MAX; 819 int64_t mLatestEncodedPtsUs = INT64_MIN; 820 int64_t mFramesEncoded = 0; 821 int64_t mBytesInput = 0; 822 int64_t mFramesInput = 0; 823 824 int64_t mNumLowLatencyEnables; // how many times low latency mode is enabled 825 int64_t mNumLowLatencyDisables; // how many times low latency mode is disabled 826 bool mIsLowLatencyModeOn; // is low latency mode on currently 827 int64_t mIndexOfFirstFrameWhenLowLatencyOn; // index of the first frame queued 828 // when low latency is on 829 int64_t mInputBufferCounter; // number of input buffers queued since last reset/flush 830 831 // A rescheduleable message that periodically polls for rendered buffers 832 sp<AMessage> mMsgPollForRenderedBuffers; 833 834 class ReleaseSurface; 835 std::unique_ptr<ReleaseSurface> mDetachedSurface; 836 837 std::list<sp<AMessage>> mLeftover; 838 status_t handleLeftover(size_t index); 839 840 sp<BatteryChecker> mBatteryChecker; 841 842 void statsBufferSent(int64_t presentationUs, const sp<MediaCodecBuffer> &buffer); 843 void statsBufferReceived(int64_t presentationUs, const sp<MediaCodecBuffer> &buffer); 844 bool discardDecodeOnlyOutputBuffer(size_t index); 845 846 enum { 847 // the default shape of our latency histogram buckets 848 // XXX: should these be configurable in some way? 849 kLatencyHistBuckets = 20, 850 kLatencyHistWidth = 2000, 851 kLatencyHistFloor = 2000, 852 853 // how many samples are in the 'recent latency' histogram 854 // 300 frames = 5 sec @ 60fps or ~12 sec @ 24fps 855 kRecentLatencyFrames = 300, 856 857 // how we initialize mRecentSamples 858 kRecentSampleInvalid = -1, 859 }; 860 861 int64_t mRecentSamples[kRecentLatencyFrames]; 862 int mRecentHead; 863 Mutex mRecentLock; 864 865 MediaHistogram<int64_t> mLatencyHist; 866 867 // An unique ID for the codec - Used by the metrics. 868 uint64_t mCodecId = 0; 869 bool mIsHardware = false; 870 871 std::function<sp<CodecBase>(const AString &, const char *)> mGetCodecBase; 872 std::function<status_t(const AString &, sp<MediaCodecInfo> *)> mGetCodecInfo; 873 friend class MediaTestHelper; 874 875 CodecErrorLog mErrorLog; 876 // Required resource info for this codec. 877 std::vector<InstanceResourceInfo> mRequiredResourceInfo; 878 879 DISALLOW_EVIL_CONSTRUCTORS(MediaCodec); 880 }; 881 882 } // namespace android 883 884 #endif // MEDIA_CODEC_H_ 885