1 #ifndef _VKTVIDEOFRAMEBUFFER_HPP 2 #define _VKTVIDEOFRAMEBUFFER_HPP 3 /*------------------------------------------------------------------------ 4 * Vulkan Conformance Tests 5 * ------------------------ 6 * 7 * Copyright (c) 2023 The Khronos Group Inc. 8 * 9 * Licensed under the Apache License, Version 2.0 (the "License"); 10 * you may not use this file except in compliance with the License. 11 * You may obtain a copy of the License at 12 * 13 * http://www.apache.org/licenses/LICENSE-2.0 14 * 15 * Unless required by applicable law or agreed to in writing, software 16 * distributed under the License is distributed on an "AS IS" BASIS, 17 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 * See the License for the specific language governing permissions and 19 * limitations under the License. 20 * 21 *//*! 22 * \file 23 * \brief Video framebuffer 24 *//*--------------------------------------------------------------------*/ 25 /* 26 * Copyright 2020 NVIDIA Corporation. 27 * 28 * Licensed under the Apache License, Version 2.0 (the "License"); 29 * you may not use this file except in compliance with the License. 30 * You may obtain a copy of the License at 31 * 32 * http://www.apache.org/licenses/LICENSE-2.0 33 * 34 * Unless required by applicable law or agreed to in writing, software 35 * distributed under the License is distributed on an "AS IS" BASIS, 36 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 37 * See the License for the specific language governing permissions and 38 * limitations under the License. 39 */ 40 #include "deDefs.hpp" 41 #include "vktVideoTestUtils.hpp" 42 #include "vktBitstreamBufferImpl.hpp" 43 44 #include "vkvideo_parser/VulkanVideoParser.h" // IVulkanVideoFrameBufferParserCb 45 #include "vkvideo_parser/VulkanVideoParserIf.h" // vkPicBuffBase VkParser* 46 47 namespace vkt 48 { 49 namespace video 50 { 51 52 using ImagePtr = de::MovePtr<ImageWithMemory>; 53 54 class VkImageResource : public VkVideoRefCountBase 55 { 56 public: 57 static VkResult Create(DeviceContext &vkDevCtx, const VkImageCreateInfo *pImageCreateInfo, 58 VkSharedBaseObj<VkImageResource> &imageResource); 59 AddRef()60 int32_t AddRef() override 61 { 62 return ++m_refCount; 63 } 64 Release()65 int32_t Release() override 66 { 67 uint32_t ret = --m_refCount; 68 // Destroy the device if ref-count reaches zero 69 if (ret == 0) 70 { 71 delete this; 72 } 73 return ret; 74 } 75 GetImage() const76 VkImage GetImage() const 77 { 78 return m_imageWithMemory->get(); 79 } 80 GetImageCreateInfo() const81 const VkImageCreateInfo &GetImageCreateInfo() const 82 { 83 return m_imageCreateInfo; 84 } 85 86 private: 87 std::atomic<int32_t> m_refCount; 88 const VkImageCreateInfo m_imageCreateInfo; 89 ImagePtr m_imageWithMemory; 90 VkImageResource(DeviceContext & vkDevCtx,const VkImageCreateInfo * pImageCreateInfo)91 VkImageResource(DeviceContext &vkDevCtx, const VkImageCreateInfo *pImageCreateInfo) 92 : m_refCount(0) 93 , m_imageCreateInfo(*pImageCreateInfo) 94 { 95 m_imageWithMemory = 96 ImagePtr(new ImageWithMemory(vkDevCtx.getDeviceDriver(), vkDevCtx.device, vkDevCtx.allocator(), 97 *pImageCreateInfo, MemoryRequirement::Local)); 98 } 99 }; 100 101 class VkImageResourceView : public VkVideoRefCountBase 102 { 103 public: 104 static VkResult Create(DeviceContext &vkDevCtx, VkSharedBaseObj<VkImageResource> &imageResource, 105 VkImageSubresourceRange &imageSubresourceRange, 106 VkSharedBaseObj<VkImageResourceView> &imageResourceView); 107 AddRef()108 virtual int32_t AddRef() 109 { 110 return ++m_refCount; 111 } 112 Release()113 virtual int32_t Release() 114 { 115 uint32_t ret = --m_refCount; 116 // Destroy the device if ref-count reaches zero 117 if (ret == 0) 118 { 119 delete this; 120 } 121 return ret; 122 } 123 operator VkImageView() const124 operator VkImageView() const 125 { 126 return m_imageView; 127 } GetImageView() const128 VkImageView GetImageView() const 129 { 130 return m_imageView; 131 } 132 GetImageResource()133 const VkSharedBaseObj<VkImageResource> &GetImageResource() 134 { 135 return m_imageResource; 136 } 137 138 private: 139 std::atomic<int32_t> m_refCount; 140 DeviceContext &m_vkDevCtx; 141 VkSharedBaseObj<VkImageResource> m_imageResource; 142 VkImageView m_imageView; 143 VkImageResourceView(DeviceContext & vkDevCtx,VkSharedBaseObj<VkImageResource> & imageResource,VkImageView imageView,VkImageSubresourceRange &)144 VkImageResourceView(DeviceContext &vkDevCtx, VkSharedBaseObj<VkImageResource> &imageResource, VkImageView imageView, 145 VkImageSubresourceRange & /*imageSubresourceRange*/) 146 : m_refCount(0) 147 , m_vkDevCtx(vkDevCtx) 148 , m_imageResource(imageResource) 149 , m_imageView(imageView) 150 { 151 } 152 153 virtual ~VkImageResourceView(); 154 }; 155 156 struct DecodedFrame 157 { 158 int32_t pictureIndex; 159 uint32_t imageLayerIndex; // The layer of a multi-layered images. Always "0" for single layered images 160 int32_t displayWidth; 161 int32_t displayHeight; 162 VkSharedBaseObj<VkImageResourceView> decodedImageView; 163 VkSharedBaseObj<VkImageResourceView> outputImageView; 164 VkFence frameCompleteFence; // If valid, the fence is signaled when the decoder is done decoding the frame. 165 VkFence 166 frameConsumerDoneFence; // If valid, the fence is signaled when the consumer (graphics, compute or display) is done using the frame. 167 VkSemaphore 168 frameCompleteSemaphore; // If valid, the semaphore is signaled when the decoder is done decoding the frame. 169 VkSemaphore 170 frameConsumerDoneSemaphore; // If valid, the semaphore is signaled when the consumer (graphics, compute or display) is done using the frame. 171 VkQueryPool queryPool; // queryPool handle used for the video queries. 172 int32_t startQueryId; // query Id used for the this frame. 173 uint32_t numQueries; // usually one query per frame 174 // If multiple queues are available, submittedVideoQueueIndex is the queue index that the video frame was submitted to. 175 // if only one queue is available, submittedVideoQueueIndex will always have a value of "0". 176 int32_t submittedVideoQueueIndex; 177 uint64_t timestamp; 178 uint32_t hasConsummerSignalFence : 1; 179 uint32_t hasConsummerSignalSemaphore : 1; 180 // For debugging 181 int32_t decodeOrder; 182 int32_t displayOrder; 183 Resetvkt::video::DecodedFrame184 void Reset() 185 { 186 pictureIndex = -1; 187 imageLayerIndex = 0; 188 displayWidth = 0; 189 displayHeight = 0; 190 decodedImageView = nullptr; 191 outputImageView = nullptr; 192 frameCompleteFence = VK_NULL_HANDLE; 193 frameConsumerDoneFence = VK_NULL_HANDLE; 194 frameCompleteSemaphore = VK_NULL_HANDLE; 195 frameConsumerDoneSemaphore = VK_NULL_HANDLE; 196 queryPool = VK_NULL_HANDLE; 197 startQueryId = 0; 198 numQueries = 0; 199 submittedVideoQueueIndex = 0; 200 timestamp = 0; 201 hasConsummerSignalFence = false; 202 hasConsummerSignalSemaphore = false; 203 // For debugging 204 decodeOrder = 0; 205 displayOrder = 0; 206 } 207 }; 208 209 struct DecodedFrameRelease 210 { 211 int32_t pictureIndex; 212 VkVideotimestamp timestamp; 213 uint32_t hasConsummerSignalFence : 1; 214 uint32_t hasConsummerSignalSemaphore : 1; 215 // For debugging 216 int32_t decodeOrder; 217 int32_t displayOrder; 218 }; 219 220 class VulkanVideoFrameBuffer : public IVulkanVideoFrameBufferParserCb 221 { 222 public: 223 // Synchronization 224 struct FrameSynchronizationInfo 225 { 226 VkFence frameCompleteFence{VK_NULL_HANDLE}; 227 VkSemaphore frameCompleteSemaphore{VK_NULL_HANDLE}; 228 VkFence frameConsumerDoneFence{VK_NULL_HANDLE}; 229 VkSemaphore frameConsumerDoneSemaphore{VK_NULL_HANDLE}; 230 VkQueryPool queryPool{VK_NULL_HANDLE}; 231 int32_t startQueryId; 232 uint32_t numQueries; 233 uint32_t hasFrameCompleteSignalFence : 1; 234 uint32_t hasFrameCompleteSignalSemaphore : 1; 235 }; 236 237 struct ReferencedObjectsInfo 238 { 239 240 // The bitstream Buffer 241 const VkVideoRefCountBase *pBitstreamData; 242 // PPS 243 const VkVideoRefCountBase *pStdPps; 244 // SPS 245 const VkVideoRefCountBase *pStdSps; 246 // VPS 247 const VkVideoRefCountBase *pStdVps; 248 249 // AV1 250 const VkVideoRefCountBase *pStdAV1Sps; 251 ReferencedObjectsInfovkt::video::VulkanVideoFrameBuffer::ReferencedObjectsInfo252 ReferencedObjectsInfo(const VkVideoRefCountBase *pBitstreamDataRef, const VkVideoRefCountBase *pStdPpsRef, 253 const VkVideoRefCountBase *pStdSpsRef, const VkVideoRefCountBase *pStdVpsRef, 254 const VkVideoRefCountBase *pStdAV1SpsRef) 255 : pBitstreamData(pBitstreamDataRef) 256 , pStdPps(pStdPpsRef) 257 , pStdSps(pStdSpsRef) 258 , pStdVps(pStdVpsRef) 259 , pStdAV1Sps(pStdAV1SpsRef) 260 { 261 } 262 }; 263 264 struct PictureResourceInfo 265 { 266 VkImage image; 267 VkFormat imageFormat; 268 VkImageLayout currentImageLayout; 269 }; 270 271 virtual int32_t InitImagePool(const VkVideoProfileInfoKHR *pDecodeProfile, uint32_t numImages, 272 VkFormat dpbImageFormat, VkFormat outImageFormat, const VkExtent2D &maxImageExtent, 273 VkImageUsageFlags dpbImageUsage, VkImageUsageFlags outImageUsage, 274 uint32_t queueFamilyIndex, bool useImageArray = false, bool useImageViewArray = false, 275 bool useSeparateOutputImage = false, bool useLinearOutput = false) = 0; 276 277 virtual int32_t QueuePictureForDecode(int8_t picId, VkParserDecodePictureInfo *pDecodePictureInfo, 278 ReferencedObjectsInfo *pReferencedObjectsInfo, 279 FrameSynchronizationInfo *pFrameSynchronizationInfo) = 0; 280 virtual int32_t DequeueDecodedPicture(DecodedFrame *pDecodedFrame) = 0; 281 virtual int32_t ReleaseDisplayedPicture(DecodedFrameRelease **pDecodedFramesRelease, 282 uint32_t numFramesToRelease) = 0; 283 virtual int32_t GetDpbImageResourcesByIndex( 284 uint32_t numResources, const int8_t *referenceSlotIndexes, VkVideoPictureResourceInfoKHR *pictureResources, 285 PictureResourceInfo *pictureResourcesInfo, 286 VkImageLayout newDpbImageLayerLayout = VK_IMAGE_LAYOUT_VIDEO_DECODE_DPB_KHR) = 0; 287 virtual int32_t GetCurrentImageResourceByIndex( 288 int8_t referenceSlotIndex, VkVideoPictureResourceInfoKHR *dpbPictureResource, 289 PictureResourceInfo *dpbPictureResourceInfo, 290 VkImageLayout newDpbImageLayerLayout = VK_IMAGE_LAYOUT_VIDEO_DECODE_DPB_KHR, 291 VkVideoPictureResourceInfoKHR *outputPictureResource = nullptr, 292 PictureResourceInfo *outputPictureResourceInfo = nullptr, 293 VkImageLayout newOutputImageLayerLayout = VK_IMAGE_LAYOUT_MAX_ENUM) = 0; 294 virtual int32_t ReleaseImageResources(uint32_t numResources, const uint32_t *indexes) = 0; 295 virtual int32_t SetPicNumInDecodeOrder(int32_t picId, int32_t picNumInDecodeOrder) = 0; 296 297 virtual int32_t SetPicNumInDisplayOrder(int32_t picId, int32_t picNumInDisplayOrder) = 0; 298 virtual size_t GetSize() = 0; 299 virtual size_t GetDisplayedFrameCount() const = 0; 300 ~VulkanVideoFrameBuffer()301 virtual ~VulkanVideoFrameBuffer() 302 { 303 } 304 305 static VkResult Create(DeviceContext *devCtx, bool supportsQueries, bool resourcesWithoutProfiles, 306 VkSharedBaseObj<VulkanVideoFrameBuffer> &vkVideoFrameBuffer); 307 }; 308 309 } // namespace video 310 } // namespace vkt 311 312 #endif // _VKTVIDEOFRAMEBUFFER_HPP 313