1 /* 2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef API_VIDEO_VIDEO_FRAME_BUFFER_H_ 12 #define API_VIDEO_VIDEO_FRAME_BUFFER_H_ 13 14 #include <stdint.h> 15 16 #include "api/array_view.h" 17 #include "api/scoped_refptr.h" 18 #include "rtc_base/ref_count.h" 19 #include "rtc_base/system/rtc_export.h" 20 21 namespace webrtc { 22 23 class I420BufferInterface; 24 class I420ABufferInterface; 25 class I422BufferInterface; 26 class I444BufferInterface; 27 class I010BufferInterface; 28 class I210BufferInterface; 29 class NV12BufferInterface; 30 31 // Base class for frame buffers of different types of pixel format and storage. 32 // The tag in type() indicates how the data is represented, and each type is 33 // implemented as a subclass. To access the pixel data, call the appropriate 34 // GetXXX() function, where XXX represents the type. There is also a function 35 // ToI420() that returns a frame buffer in I420 format, converting from the 36 // underlying representation if necessary. I420 is the most widely accepted 37 // format and serves as a fallback for video sinks that can only handle I420, 38 // e.g. the internal WebRTC software encoders. A special enum value 'kNative' is 39 // provided for external clients to implement their own frame buffer 40 // representations, e.g. as textures. The external client can produce such 41 // native frame buffers from custom video sources, and then cast it back to the 42 // correct subclass in custom video sinks. The purpose of this is to improve 43 // performance by providing an optimized path without intermediate conversions. 44 // Frame metadata such as rotation and timestamp are stored in 45 // webrtc::VideoFrame, and not here. 46 class RTC_EXPORT VideoFrameBuffer : public rtc::RefCountInterface { 47 public: 48 // New frame buffer types will be added conservatively when there is an 49 // opportunity to optimize the path between some pair of video source and 50 // video sink. 51 // GENERATED_JAVA_ENUM_PACKAGE: org.webrtc 52 // GENERATED_JAVA_CLASS_NAME_OVERRIDE: VideoFrameBufferType 53 enum class Type { 54 kNative, 55 kI420, 56 kI420A, 57 kI422, 58 kI444, 59 kI010, 60 kI210, 61 kNV12, 62 }; 63 64 // This function specifies in what pixel format the data is stored in. 65 virtual Type type() const = 0; 66 67 // The resolution of the frame in pixels. For formats where some planes are 68 // subsampled, this is the highest-resolution plane. 69 virtual int width() const = 0; 70 virtual int height() const = 0; 71 72 // Returns a memory-backed frame buffer in I420 format. If the pixel data is 73 // in another format, a conversion will take place. All implementations must 74 // provide a fallback to I420 for compatibility with e.g. the internal WebRTC 75 // software encoders. 76 // Conversion may fail, for example if reading the pixel data from a texture 77 // fails. If the conversion fails, nullptr is returned. 78 virtual rtc::scoped_refptr<I420BufferInterface> ToI420() = 0; 79 80 // GetI420() methods should return I420 buffer if conversion is trivial, i.e 81 // no change for binary data is needed. Otherwise these methods should return 82 // nullptr. One example of buffer with that property is 83 // WebrtcVideoFrameAdapter in Chrome - it's I420 buffer backed by a shared 84 // memory buffer. Therefore it must have type kNative. Yet, ToI420() 85 // doesn't affect binary data at all. Another example is any I420A buffer. 86 // TODO(https://crbug.com/webrtc/12021): Make this method non-virtual and 87 // behave as the other GetXXX methods below. 88 virtual const I420BufferInterface* GetI420() const; 89 90 // A format specific scale function. Default implementation works by 91 // converting to I420. But more efficient implementations may override it, 92 // especially for kNative. 93 // First, the image is cropped to `crop_width` and `crop_height` and then 94 // scaled to `scaled_width` and `scaled_height`. 95 virtual rtc::scoped_refptr<VideoFrameBuffer> CropAndScale(int offset_x, 96 int offset_y, 97 int crop_width, 98 int crop_height, 99 int scaled_width, 100 int scaled_height); 101 102 // Alias for common use case. Scale(int scaled_width,int scaled_height)103 rtc::scoped_refptr<VideoFrameBuffer> Scale(int scaled_width, 104 int scaled_height) { 105 return CropAndScale(0, 0, width(), height(), scaled_width, scaled_height); 106 } 107 108 // These functions should only be called if type() is of the correct type. 109 // Calling with a different type will result in a crash. 110 const I420ABufferInterface* GetI420A() const; 111 const I422BufferInterface* GetI422() const; 112 const I444BufferInterface* GetI444() const; 113 const I010BufferInterface* GetI010() const; 114 const I210BufferInterface* GetI210() const; 115 const NV12BufferInterface* GetNV12() const; 116 117 // From a kNative frame, returns a VideoFrameBuffer with a pixel format in 118 // the list of types that is in the main memory with a pixel perfect 119 // conversion for encoding with a software encoder. Returns nullptr if the 120 // frame type is not supported, mapping is not possible, or if the kNative 121 // frame has not implemented this method. Only callable if type() is kNative. 122 virtual rtc::scoped_refptr<VideoFrameBuffer> GetMappedFrameBuffer( 123 rtc::ArrayView<Type> types); 124 125 protected: ~VideoFrameBuffer()126 ~VideoFrameBuffer() override {} 127 }; 128 129 // Update when VideoFrameBuffer::Type is updated. 130 const char* VideoFrameBufferTypeToString(VideoFrameBuffer::Type type); 131 132 // This interface represents planar formats. 133 class PlanarYuvBuffer : public VideoFrameBuffer { 134 public: 135 virtual int ChromaWidth() const = 0; 136 virtual int ChromaHeight() const = 0; 137 138 // Returns the number of steps(in terms of Data*() return type) between 139 // successive rows for a given plane. 140 virtual int StrideY() const = 0; 141 virtual int StrideU() const = 0; 142 virtual int StrideV() const = 0; 143 144 protected: ~PlanarYuvBuffer()145 ~PlanarYuvBuffer() override {} 146 }; 147 148 // This interface represents 8-bit color depth formats: Type::kI420, 149 // Type::kI420A, Type::kI422 and Type::kI444. 150 class PlanarYuv8Buffer : public PlanarYuvBuffer { 151 public: 152 // Returns pointer to the pixel data for a given plane. The memory is owned by 153 // the VideoFrameBuffer object and must not be freed by the caller. 154 virtual const uint8_t* DataY() const = 0; 155 virtual const uint8_t* DataU() const = 0; 156 virtual const uint8_t* DataV() const = 0; 157 158 protected: ~PlanarYuv8Buffer()159 ~PlanarYuv8Buffer() override {} 160 }; 161 162 class RTC_EXPORT I420BufferInterface : public PlanarYuv8Buffer { 163 public: 164 Type type() const override; 165 166 int ChromaWidth() const final; 167 int ChromaHeight() const final; 168 169 rtc::scoped_refptr<I420BufferInterface> ToI420() final; 170 const I420BufferInterface* GetI420() const final; 171 172 protected: ~I420BufferInterface()173 ~I420BufferInterface() override {} 174 }; 175 176 class RTC_EXPORT I420ABufferInterface : public I420BufferInterface { 177 public: 178 Type type() const final; 179 virtual const uint8_t* DataA() const = 0; 180 virtual int StrideA() const = 0; 181 182 protected: ~I420ABufferInterface()183 ~I420ABufferInterface() override {} 184 }; 185 186 // Represents Type::kI422, 4:2:2 planar with 8 bits per pixel. 187 class I422BufferInterface : public PlanarYuv8Buffer { 188 public: 189 Type type() const final; 190 191 int ChromaWidth() const final; 192 int ChromaHeight() const final; 193 194 rtc::scoped_refptr<VideoFrameBuffer> CropAndScale(int offset_x, 195 int offset_y, 196 int crop_width, 197 int crop_height, 198 int scaled_width, 199 int scaled_height) override; 200 201 protected: ~I422BufferInterface()202 ~I422BufferInterface() override {} 203 }; 204 205 // Represents Type::kI444, 4:4:4 planar with 8 bits per pixel. 206 class I444BufferInterface : public PlanarYuv8Buffer { 207 public: 208 Type type() const final; 209 210 int ChromaWidth() const final; 211 int ChromaHeight() const final; 212 213 rtc::scoped_refptr<VideoFrameBuffer> CropAndScale(int offset_x, 214 int offset_y, 215 int crop_width, 216 int crop_height, 217 int scaled_width, 218 int scaled_height) override; 219 220 protected: ~I444BufferInterface()221 ~I444BufferInterface() override {} 222 }; 223 224 // This interface represents 8-bit to 16-bit color depth formats: Type::kI010 or 225 // Type::kI210 . 226 class PlanarYuv16BBuffer : public PlanarYuvBuffer { 227 public: 228 // Returns pointer to the pixel data for a given plane. The memory is owned by 229 // the VideoFrameBuffer object and must not be freed by the caller. 230 virtual const uint16_t* DataY() const = 0; 231 virtual const uint16_t* DataU() const = 0; 232 virtual const uint16_t* DataV() const = 0; 233 234 protected: ~PlanarYuv16BBuffer()235 ~PlanarYuv16BBuffer() override {} 236 }; 237 238 // Represents Type::kI010, allocates 16 bits per pixel and fills 10 least 239 // significant bits with color information. 240 class I010BufferInterface : public PlanarYuv16BBuffer { 241 public: 242 Type type() const override; 243 244 int ChromaWidth() const final; 245 int ChromaHeight() const final; 246 247 protected: ~I010BufferInterface()248 ~I010BufferInterface() override {} 249 }; 250 251 // Represents Type::kI210, allocates 16 bits per pixel and fills 10 least 252 // significant bits with color information. 253 class I210BufferInterface : public PlanarYuv16BBuffer { 254 public: 255 Type type() const override; 256 257 int ChromaWidth() const final; 258 int ChromaHeight() const final; 259 260 protected: ~I210BufferInterface()261 ~I210BufferInterface() override {} 262 }; 263 264 class BiplanarYuvBuffer : public VideoFrameBuffer { 265 public: 266 virtual int ChromaWidth() const = 0; 267 virtual int ChromaHeight() const = 0; 268 269 // Returns the number of steps(in terms of Data*() return type) between 270 // successive rows for a given plane. 271 virtual int StrideY() const = 0; 272 virtual int StrideUV() const = 0; 273 274 protected: ~BiplanarYuvBuffer()275 ~BiplanarYuvBuffer() override {} 276 }; 277 278 class BiplanarYuv8Buffer : public BiplanarYuvBuffer { 279 public: 280 virtual const uint8_t* DataY() const = 0; 281 virtual const uint8_t* DataUV() const = 0; 282 283 protected: ~BiplanarYuv8Buffer()284 ~BiplanarYuv8Buffer() override {} 285 }; 286 287 // Represents Type::kNV12. NV12 is full resolution Y and half-resolution 288 // interleved UV. 289 class RTC_EXPORT NV12BufferInterface : public BiplanarYuv8Buffer { 290 public: 291 Type type() const override; 292 293 int ChromaWidth() const final; 294 int ChromaHeight() const final; 295 296 rtc::scoped_refptr<VideoFrameBuffer> CropAndScale(int offset_x, 297 int offset_y, 298 int crop_width, 299 int crop_height, 300 int scaled_width, 301 int scaled_height) override; 302 303 protected: ~NV12BufferInterface()304 ~NV12BufferInterface() override {} 305 }; 306 307 } // namespace webrtc 308 309 #endif // API_VIDEO_VIDEO_FRAME_BUFFER_H_ 310