1 /* 2 * Copyright 2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef CODEC2_HIDL_CLIENT_H 18 #define CODEC2_HIDL_CLIENT_H 19 20 #include <C2PlatformSupport.h> 21 #include <C2Component.h> 22 #include <C2Buffer.h> 23 #include <C2Param.h> 24 #include <C2.h> 25 26 #include <gui/FrameTimestamps.h> 27 #include <gui/IGraphicBufferProducer.h> 28 #include <hidl/HidlSupport.h> 29 #include <utils/StrongPointer.h> 30 31 #include <functional> 32 #include <map> 33 #include <memory> 34 #include <mutex> 35 36 /** 37 * This file contains minimal interfaces for the framework to access Codec2.0. 38 * 39 * Codec2Client is the main class that contains the following inner classes: 40 * - Listener 41 * - Configurable 42 * - Interface 43 * - Component 44 * 45 * Classes in Codec2Client, interfaces in Codec2.0, and HIDL interfaces are 46 * related as follows: 47 * - Codec2Client <==> C2ComponentStore <==> IComponentStore 48 * - Codec2Client::Listener <==> C2Component::Listener <==> IComponentListener 49 * - Codec2Client::Configurable <==> [No equivalent] <==> IConfigurable 50 * - Codec2Client::Interface <==> C2ComponentInterface <==> IComponentInterface 51 * - Codec2Client::Component <==> C2Component <==> IComponent 52 * 53 * The entry point is Codec2Client::CreateFromService(), which creates a 54 * Codec2Client object. From Codec2Client, Interface and Component objects can 55 * be created by calling createComponent() and createInterface(). 56 * 57 * createComponent() takes a Listener object, which must be implemented by the 58 * user. 59 * 60 * At the present, createBlockPool() is the only method that yields a 61 * Configurable object. Note, however, that Interface, Component and 62 * Codec2Client are all subclasses of Configurable. 63 */ 64 65 // Forward declaration of relevant HIDL interfaces 66 67 namespace android::hardware::media::c2::V1_0 { 68 struct IConfigurable; 69 struct IComponent; 70 struct IComponentInterface; 71 struct IComponentStore; 72 struct IInputSink; 73 struct IInputSurface; 74 struct IInputSurfaceConnection; 75 } // namespace android::hardware::media::c2::V1_0 76 77 namespace android::hardware::media::c2::V1_1 { 78 struct IComponent; 79 struct IComponentStore; 80 } // namespace android::hardware::media::c2::V1_1 81 82 namespace android::hardware::media::c2::V1_2 { 83 struct IComponent; 84 struct IComponentStore; 85 } // namespace android::hardware::media::c2::V1_2 86 87 namespace aidl::android::hardware::media::c2 { 88 class IComponent; 89 class IComponentInterface; 90 class IComponentStore; 91 class IConfigurable; 92 } // namespace aidl::android::hardware::media::c2 93 94 namespace android::hardware::media::bufferpool::V2_0 { 95 struct IClientManager; 96 } // namespace android::hardware::media::bufferpool::V2_0 97 98 namespace aidl::android::hardware::media::bufferpool2 { 99 class IClientManager; 100 } // namespace aidl::android::hardware::media::c2 101 102 103 namespace android::hardware::graphics::bufferqueue::V1_0 { 104 struct IGraphicBufferProducer; 105 } // android::hardware::graphics::bufferqueue::V1_0 106 107 namespace android::hardware::graphics::bufferqueue::V2_0 { 108 struct IGraphicBufferProducer; 109 } // android::hardware::graphics::bufferqueue::V2_0 110 111 namespace android::hardware::media::omx::V1_0 { 112 struct IGraphicBufferSource; 113 } // namespace android::hardware::media::omx::V1_0 114 115 struct ApexCodec_ComponentStore; 116 struct ApexCodec_Component; 117 struct ApexCodec_Configurable; 118 119 namespace android { 120 121 // This class is supposed to be called Codec2Client::Configurable, but forward 122 // declaration of an inner class is not possible. 123 struct Codec2ConfigurableClient { 124 125 typedef ::android::hardware::media::c2::V1_0::IConfigurable HidlBase; 126 typedef ::aidl::android::hardware::media::c2::IConfigurable AidlBase; 127 128 struct ImplBase { 129 virtual ~ImplBase() = default; 130 131 virtual const C2String& getName() const = 0; 132 133 virtual c2_status_t query( 134 const std::vector<C2Param*>& stackParams, 135 const std::vector<C2Param::Index> &heapParamIndices, 136 c2_blocking_t mayBlock, 137 std::vector<std::unique_ptr<C2Param>>* const heapParams) const = 0; 138 139 virtual c2_status_t config( 140 const std::vector<C2Param*> ¶ms, 141 c2_blocking_t mayBlock, 142 std::vector<std::unique_ptr<C2SettingResult>>* const failures) = 0; 143 144 virtual c2_status_t querySupportedParams( 145 std::vector<std::shared_ptr<C2ParamDescriptor>>* const params 146 ) const = 0; 147 148 virtual c2_status_t querySupportedValues( 149 std::vector<C2FieldSupportedValuesQuery>& fields, 150 c2_blocking_t mayBlock) const = 0; 151 }; 152 153 explicit Codec2ConfigurableClient(const sp<HidlBase> &hidlBase); 154 explicit Codec2ConfigurableClient(const std::shared_ptr<AidlBase> &aidlBase); 155 Codec2ConfigurableClient(ApexCodec_Configurable *base, const C2String &name); 156 157 const C2String& getName() const; 158 159 c2_status_t query( 160 const std::vector<C2Param*>& stackParams, 161 const std::vector<C2Param::Index> &heapParamIndices, 162 c2_blocking_t mayBlock, 163 std::vector<std::unique_ptr<C2Param>>* const heapParams) const; 164 165 c2_status_t config( 166 const std::vector<C2Param*> ¶ms, 167 c2_blocking_t mayBlock, 168 std::vector<std::unique_ptr<C2SettingResult>>* const failures); 169 170 c2_status_t querySupportedParams( 171 std::vector<std::shared_ptr<C2ParamDescriptor>>* const params 172 ) const; 173 174 c2_status_t querySupportedValues( 175 std::vector<C2FieldSupportedValuesQuery>& fields, 176 c2_blocking_t mayBlock) const; 177 private: 178 struct HidlImpl; 179 struct AidlImpl; 180 struct ApexImpl; 181 182 const std::unique_ptr<ImplBase> mImpl; 183 }; 184 185 struct Codec2Client : public Codec2ConfigurableClient { 186 187 typedef ::android::hardware::media::c2::V1_0::IComponentStore HidlBase1_0; 188 typedef ::android::hardware::media::c2::V1_1::IComponentStore HidlBase1_1; 189 typedef ::android::hardware::media::c2::V1_2::IComponentStore HidlBase1_2; 190 typedef HidlBase1_0 HidlBase; 191 192 typedef ::aidl::android::hardware::media::c2::IComponentStore AidlBase; 193 194 struct Listener; 195 196 typedef Codec2ConfigurableClient Configurable; 197 198 struct Component; 199 200 struct Interface; 201 202 struct InputSurface; 203 204 struct InputSurfaceConnection; 205 206 typedef Codec2Client Store; 207 208 sp<HidlBase> const& getHidlBase() const; 209 sp<HidlBase1_0> const& getHidlBase1_0() const; 210 sp<HidlBase1_1> const& getHidlBase1_1() const; 211 sp<HidlBase1_2> const& getHidlBase1_2() const; 212 ::ndk::SpAIBinder getAidlBase() const; 213 214 std::string const& getServiceName() const; 215 216 c2_status_t createComponent( 217 C2String const& name, 218 std::shared_ptr<Listener> const& listener, 219 std::shared_ptr<Component>* const component); 220 221 c2_status_t createInterface( 222 C2String const& name, 223 std::shared_ptr<Interface>* const interface); 224 225 c2_status_t createInputSurface( 226 std::shared_ptr<InputSurface>* const inputSurface); 227 228 std::vector<C2Component::Traits> const& listComponents() const; 229 230 c2_status_t copyBuffer( 231 std::shared_ptr<C2Buffer> const& src, 232 std::shared_ptr<C2Buffer> const& dst); 233 234 std::shared_ptr<C2ParamReflector> getParamReflector(); 235 236 // Returns the list of IComponentStore service names that are available on 237 // the device. This list is specified at the build time in manifest files. 238 // Note: A software service will have "_software" as a suffix. 239 static std::vector<std::string> const& GetServiceNames(); 240 241 // Create a client to a service with a given name. 242 // 243 // After a client to the service is successfully created, if 244 // setAsPreferredCodec2ComponentStore is true, the component store that the 245 // service hosts will be set as the preferred C2ComponentStore for this 246 // process. (See SetPreferredCodec2ComponentStore() for more information.) 247 static std::shared_ptr<Codec2Client> CreateFromService( 248 char const* name, 249 bool setAsPreferredCodec2ComponentStore = false); 250 251 // Get clients to all services. 252 static std::vector<std::shared_ptr<Codec2Client>> CreateFromAllServices(); 253 254 // Try to create a component with a given name from all known 255 // IComponentStore services. numberOfAttempts determines the number of times 256 // to retry the HIDL call if the transaction fails. 257 static c2_status_t CreateComponentByName( 258 char const* componentName, 259 std::shared_ptr<Listener> const& listener, 260 std::shared_ptr<Component>* component, 261 std::shared_ptr<Codec2Client>* owner = nullptr, 262 size_t numberOfAttempts = 10); 263 264 // Try to create a component interface with a given name from all known 265 // IComponentStore services. numberOfAttempts determines the number of times 266 // to retry the HIDL call if the transaction fails. 267 static std::shared_ptr<Interface> CreateInterfaceByName( 268 char const* interfaceName, 269 std::shared_ptr<Codec2Client>* owner = nullptr, 270 size_t numberOfAttempts = 10); 271 272 // List traits from all known IComponentStore services. 273 static std::vector<C2Component::Traits> const& ListComponents(); 274 275 // Create an input surface. 276 static std::shared_ptr<InputSurface> CreateInputSurface( 277 char const* serviceName = nullptr); 278 279 // Whether AIDL is selected. 280 static bool IsAidlSelected(); 281 282 // base and/or configurable cannot be null. 283 Codec2Client( 284 sp<HidlBase> const& base, 285 sp<Codec2ConfigurableClient::HidlBase> const& configurable, 286 size_t serviceIndex); 287 Codec2Client( 288 std::shared_ptr<AidlBase> const& base, 289 std::shared_ptr<Codec2ConfigurableClient::AidlBase> const& configurable, 290 size_t serviceIndex); 291 Codec2Client( 292 ApexCodec_ComponentStore* base, 293 size_t serviceIndex); 294 295 protected: 296 sp<HidlBase1_0> mHidlBase1_0; 297 sp<HidlBase1_1> mHidlBase1_1; 298 sp<HidlBase1_2> mHidlBase1_2; 299 std::shared_ptr<AidlBase> mAidlBase; 300 ApexCodec_ComponentStore* mApexBase{nullptr}; 301 302 // Finds the first store where the predicate returns C2_OK and returns the 303 // last predicate result. The predicate will be tried on all stores. The 304 // function will return C2_OK the first time the predicate returns C2_OK, 305 // or it will return the value from the last time that predicate is tried. 306 // (The latter case corresponds to a failure on every store.) The order of 307 // the stores to try is the same as the return value of GetServiceNames(). 308 // 309 // key is used to remember the last store with which the predicate last 310 // succeeded. If the last successful store is cached, it will be tried 311 // first before all the stores are tried. Note that the last successful 312 // store will be tried twice---first before all the stores, and another time 313 // with all the stores. 314 // 315 // If an attempt to evaluate the predicate results in a transaction failure, 316 // repeated attempts will be made until the predicate returns without a 317 // transaction failure or numberOfAttempts attempts have been made. 318 static c2_status_t ForAllServices( 319 const std::string& key, 320 size_t numberOfAttempts, 321 std::function<c2_status_t(std::shared_ptr<Codec2Client> const&)> 322 predicate); 323 324 size_t mServiceIndex; 325 mutable std::vector<C2Component::Traits> mTraitsList; 326 327 sp<::android::hardware::media::bufferpool::V2_0::IClientManager> 328 mHidlHostPoolManager; 329 std::shared_ptr<::aidl::android::hardware::media::bufferpool2::IClientManager> 330 mAidlHostPoolManager; 331 332 static std::vector<std::string> CacheServiceNames(); 333 static std::shared_ptr<Codec2Client> _CreateFromIndex(size_t index); 334 335 std::vector<C2Component::Traits> _listComponents(bool* success) const; 336 337 class Cache; 338 339 private: 340 c2_status_t createComponent_aidl( 341 C2String const& name, 342 std::shared_ptr<Listener> const& listener, 343 std::shared_ptr<Component>* const component); 344 c2_status_t createComponent_hidl( 345 C2String const& name, 346 std::shared_ptr<Listener> const& listener, 347 std::shared_ptr<Component>* const component); 348 c2_status_t createComponent_apex( 349 C2String const& name, 350 std::shared_ptr<Listener> const& listener, 351 std::shared_ptr<Component>* const component); 352 }; 353 354 struct Codec2Client::Interface : public Codec2Client::Configurable { 355 356 typedef ::android::hardware::media::c2::V1_0::IComponentInterface HidlBase; 357 typedef ::aidl::android::hardware::media::c2::IComponentInterface AidlBase; 358 359 Interface(const sp<HidlBase>& base); 360 Interface(const std::shared_ptr<AidlBase>& base); 361 362 protected: 363 sp<HidlBase> mHidlBase; 364 std::shared_ptr<AidlBase> mAidlBase; 365 }; 366 367 struct Codec2Client::Listener { 368 369 // This is called when the component produces some output. 370 virtual void onWorkDone( 371 const std::weak_ptr<Component>& comp, 372 std::list<std::unique_ptr<C2Work>>& workItems) = 0; 373 374 // This is called when the component goes into a tripped state. 375 virtual void onTripped( 376 const std::weak_ptr<Component>& comp, 377 const std::vector<std::shared_ptr<C2SettingResult>>& settingResults 378 ) = 0; 379 380 // This is called when the component encounters an error. 381 virtual void onError( 382 const std::weak_ptr<Component>& comp, 383 uint32_t errorCode) = 0; 384 385 // This is called when the process that hosts the component shuts down 386 // unexpectedly. 387 virtual void onDeath( 388 const std::weak_ptr<Component>& comp) = 0; 389 390 // This is called when an input buffer is no longer in use by the codec. 391 // Input buffers that have been returned by onWorkDone() or flush() will not 392 // trigger a call to this function. 393 virtual void onInputBufferDone( 394 uint64_t frameIndex, size_t arrayIndex) = 0; 395 396 // This is called when the component becomes aware of a frame being 397 // rendered. 398 virtual void onFrameRendered( 399 uint64_t bufferQueueId, 400 int32_t slotId, 401 int64_t timestampNs) = 0; 402 403 virtual ~Listener() = default; 404 }; 405 406 struct Codec2Client::Component : public Codec2Client::Configurable { 407 408 typedef ::android::hardware::media::c2::V1_0::IComponent HidlBase1_0; 409 typedef ::android::hardware::media::c2::V1_1::IComponent HidlBase1_1; 410 typedef ::android::hardware::media::c2::V1_2::IComponent HidlBase1_2; 411 typedef HidlBase1_0 HidlBase; 412 413 typedef ::aidl::android::hardware::media::c2::IComponent AidlBase; 414 415 c2_status_t createBlockPool( 416 C2Allocator::id_t id, 417 C2BlockPool::local_id_t* blockPoolId, 418 std::shared_ptr<Configurable>* configurable); 419 420 c2_status_t destroyBlockPool( 421 C2BlockPool::local_id_t localId); 422 423 c2_status_t queue( 424 std::list<std::unique_ptr<C2Work>>* const items); 425 426 c2_status_t flush( 427 C2Component::flush_mode_t mode, 428 std::list<std::unique_ptr<C2Work>>* const flushedWork); 429 430 c2_status_t drain(C2Component::drain_mode_t mode); 431 432 c2_status_t start(); 433 434 c2_status_t stop(); 435 436 c2_status_t reset(); 437 438 c2_status_t release(); 439 440 /** 441 * Use tunneling. 442 * 443 * On success, @p sidebandHandle will be a newly allocated native handle. 444 * File descriptors in @p sidebandHandle must be closed and 445 * @p sidebandHandle itself must be deleted afterwards. 446 */ 447 c2_status_t configureVideoTunnel( 448 uint32_t avSyncHwId, 449 native_handle_t** sidebandHandle); 450 451 typedef ::android:: 452 IGraphicBufferProducer IGraphicBufferProducer; 453 typedef IGraphicBufferProducer:: 454 QueueBufferInput QueueBufferInput; 455 typedef IGraphicBufferProducer:: 456 QueueBufferOutput QueueBufferOutput; 457 458 typedef ::android::hardware::graphics::bufferqueue::V1_0:: 459 IGraphicBufferProducer HGraphicBufferProducer1; 460 typedef ::android::hardware::graphics::bufferqueue::V2_0:: 461 IGraphicBufferProducer HGraphicBufferProducer2; 462 typedef ::android::hardware::media::omx::V1_0:: 463 IGraphicBufferSource HGraphicBufferSource; 464 465 // Set the output surface to be used with a blockpool previously created by 466 // createBlockPool(). 467 c2_status_t setOutputSurface( 468 C2BlockPool::local_id_t blockPoolId, 469 const sp<IGraphicBufferProducer>& surface, 470 uint32_t generation, 471 int maxDequeueBufferCount); 472 473 // Extract a slot number from of the block, then call 474 // IGraphicBufferProducer::queueBuffer(). 475 // 476 // If the output surface has not been set, NO_INIT will be returned. 477 // 478 // If the block does not come from a bufferqueue-based blockpool, 479 // attachBuffer() will be called, followed by queueBuffer(). 480 // 481 // If the block has a bqId that does not match the id of the output surface, 482 // DEAD_OBJECT will be returned. 483 // 484 // If the call to queueBuffer() is successful but the block cannot be 485 // associated to the output surface for automatic cancellation upon 486 // destruction, UNKNOWN_ERROR will be returned. 487 // 488 // Otherwise, the return value from queueBuffer() will be returned. 489 status_t queueToOutputSurface( 490 const C2ConstGraphicBlock& block, 491 const QueueBufferInput& input, 492 QueueBufferOutput* output); 493 494 // configure consumer usage. 495 uint64_t configConsumerUsage(const sp<IGraphicBufferProducer>& surface); 496 497 // Retrieve frame event history from the output surface. 498 void pollForRenderedFrames(FrameEventHistoryDelta* delta); 499 500 // Set max dequeue count for output surface. 501 void setOutputSurfaceMaxDequeueCount(int maxDequeueCount); 502 503 // Stop using the current output surface. 504 void stopUsingOutputSurface( 505 C2BlockPool::local_id_t blockPoolId); 506 507 // Notify a buffer is released from output surface. 508 void onBufferReleasedFromOutputSurface( 509 uint32_t generation); 510 511 // Notify a buffer is attached to output surface. 512 void onBufferAttachedToOutputSurface( 513 uint32_t generation); 514 515 // When the client received \p workList and the blocks inside 516 // \p workList are IGBA based graphic blocks, specify the owner 517 // as the current IGBA for the future operations. 518 // Future operations could be rendering the blocks to the surface 519 // or deallocating blocks to the surface. 520 void holdIgbaBlocks( 521 const std::list<std::unique_ptr<C2Work>>& workList); 522 523 // Connect to a given InputSurface. 524 c2_status_t connectToInputSurface( 525 const std::shared_ptr<InputSurface>& inputSurface, 526 std::shared_ptr<InputSurfaceConnection>* connection); 527 528 c2_status_t connectToOmxInputSurface( 529 const sp<HGraphicBufferProducer1>& producer, 530 const sp<HGraphicBufferSource>& source, 531 std::shared_ptr<InputSurfaceConnection>* connection); 532 533 c2_status_t disconnectFromInputSurface(); 534 535 c2_status_t initApexHandler( 536 const std::shared_ptr<Listener> &listener, 537 const std::shared_ptr<Component> &comp); 538 539 // base cannot be null. 540 Component(const sp<HidlBase>& base); 541 Component(const sp<HidlBase1_1>& base); 542 Component(const sp<HidlBase1_2>& base); 543 Component(const std::shared_ptr<AidlBase>& base); 544 Component(ApexCodec_Component* base, const C2String& name); 545 546 ~Component(); 547 548 protected: 549 sp<HidlBase1_0> mHidlBase1_0; 550 sp<HidlBase1_1> mHidlBase1_1; 551 sp<HidlBase1_2> mHidlBase1_2; 552 std::shared_ptr<AidlBase> mAidlBase; 553 ApexCodec_Component *mApexBase{nullptr}; 554 555 struct HidlBufferPoolSender; 556 struct AidlBufferPoolSender; 557 std::unique_ptr<HidlBufferPoolSender> mHidlBufferPoolSender; 558 std::unique_ptr<AidlBufferPoolSender> mAidlBufferPoolSender; 559 560 class ApexHandler; 561 std::unique_ptr<ApexHandler> mApexHandler; 562 563 struct OutputBufferQueue; 564 std::unique_ptr<OutputBufferQueue> mOutputBufferQueue; 565 566 // (b/202903117) Sometimes MediaCodec::setSurface races between normal 567 // setSurface and setSurface with ReleaseSurface due to timing issues. 568 // In order to prevent the race condition mutex is added. 569 std::mutex mOutputMutex; 570 571 struct GraphicBufferAllocators; 572 std::unique_ptr<GraphicBufferAllocators> mGraphicBufferAllocators; 573 574 class AidlDeathManager; 575 static AidlDeathManager *GetAidlDeathManager(); 576 std::optional<size_t> mAidlDeathSeq; 577 578 static c2_status_t setDeathListener( 579 const std::shared_ptr<Component>& component, 580 const std::shared_ptr<Listener>& listener); 581 sp<::android::hardware::hidl_death_recipient> mDeathRecipient; 582 583 // This is a map of block pools created for APEX components in the client. 584 // Note that the APEX codec API requires output buffers to be passed from the client, 585 // so the client creates and keeps track of the block pools here. 586 std::map<C2BlockPool::local_id_t, std::shared_ptr<C2BlockPool>> mBlockPools; 587 588 friend struct Codec2Client; 589 590 struct HidlListener; 591 struct AidlListener; 592 void handleOnWorkDone(const std::list<std::unique_ptr<C2Work>> &workItems); 593 }; 594 595 struct Codec2Client::InputSurface : public Codec2Client::Configurable { 596 public: 597 typedef ::android::hardware::media::c2::V1_0::IInputSurface Base; 598 599 typedef ::android::hardware::media::c2::V1_0::IInputSurfaceConnection ConnectionBase; 600 601 typedef Codec2Client::InputSurfaceConnection Connection; 602 603 typedef ::android::IGraphicBufferProducer IGraphicBufferProducer; 604 605 sp<IGraphicBufferProducer> getGraphicBufferProducer() const; 606 607 // Return the underlying IInputSurface. 608 sp<Base> getHalInterface() const; 609 610 // base cannot be null. 611 InputSurface(const sp<Base>& base); 612 613 protected: 614 sp<Base> mBase; 615 616 sp<IGraphicBufferProducer> mGraphicBufferProducer; 617 618 friend struct Codec2Client; 619 friend struct Component; 620 }; 621 622 struct Codec2Client::InputSurfaceConnection : public Codec2Client::Configurable { 623 624 typedef ::android::hardware::media::c2::V1_0::IInputSurfaceConnection Base; 625 626 c2_status_t disconnect(); 627 628 // base cannot be null. 629 InputSurfaceConnection(const sp<Base>& base); 630 631 protected: 632 sp<Base> mBase; 633 634 friend struct Codec2Client::InputSurface; 635 }; 636 637 } // namespace android 638 639 #endif // CODEC2_HIDL_CLIENT_H 640