1 /* 2 * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef API_VIDEO_VIDEO_FRAME_H_ 12 #define API_VIDEO_VIDEO_FRAME_H_ 13 14 #include <stdint.h> 15 16 #include <utility> 17 18 #include "absl/types/optional.h" 19 #include "api/rtp_packet_infos.h" 20 #include "api/scoped_refptr.h" 21 #include "api/video/color_space.h" 22 #include "api/video/hdr_metadata.h" 23 #include "api/video/video_frame_buffer.h" 24 #include "api/video/video_rotation.h" 25 #include "rtc_base/checks.h" 26 #include "rtc_base/system/rtc_export.h" 27 28 namespace webrtc { 29 30 class RTC_EXPORT VideoFrame { 31 public: 32 // Value used to signal that `VideoFrame::id()` is not set. 33 static constexpr uint16_t kNotSetId = 0; 34 35 struct RTC_EXPORT UpdateRect { 36 int offset_x; 37 int offset_y; 38 int width; 39 int height; 40 41 // Makes this UpdateRect a bounding box of this and other rect. 42 void Union(const UpdateRect& other); 43 44 // Makes this UpdateRect an intersection of this and other rect. 45 void Intersect(const UpdateRect& other); 46 47 // Sets everything to 0, making this UpdateRect a zero-size (empty) update. 48 void MakeEmptyUpdate(); 49 50 bool IsEmpty() const; 51 52 // Per-member equality check. Empty rectangles with different offsets would 53 // be considered different. 54 bool operator==(const UpdateRect& other) const { 55 return other.offset_x == offset_x && other.offset_y == offset_y && 56 other.width == width && other.height == height; 57 } 58 59 bool operator!=(const UpdateRect& other) const { return !(*this == other); } 60 61 // Scales update_rect given original frame dimensions. 62 // Cropping is applied first, then rect is scaled down. 63 // Update rect is snapped to 2x2 grid due to possible UV subsampling and 64 // then expanded by additional 2 pixels in each direction to accommodate any 65 // possible scaling artifacts. 66 // Note, close but not equal update_rects on original frame may result in 67 // the same scaled update rects. 68 UpdateRect ScaleWithFrame(int frame_width, 69 int frame_height, 70 int crop_x, 71 int crop_y, 72 int crop_width, 73 int crop_height, 74 int scaled_width, 75 int scaled_height) const; 76 }; 77 78 struct RTC_EXPORT ProcessingTime { ElapsedProcessingTime79 TimeDelta Elapsed() const { return finish - start; } 80 Timestamp start; 81 Timestamp finish; 82 }; 83 84 struct RTC_EXPORT RenderParameters { 85 bool use_low_latency_rendering = false; 86 absl::optional<int32_t> max_composition_delay_in_frames; 87 88 bool operator==(const RenderParameters& other) const { 89 return other.use_low_latency_rendering == use_low_latency_rendering && 90 other.max_composition_delay_in_frames == 91 max_composition_delay_in_frames; 92 } 93 94 bool operator!=(const RenderParameters& other) const { 95 return !(*this == other); 96 } 97 }; 98 99 // Preferred way of building VideoFrame objects. 100 class RTC_EXPORT Builder { 101 public: 102 Builder(); 103 ~Builder(); 104 105 VideoFrame build(); 106 Builder& set_video_frame_buffer( 107 const rtc::scoped_refptr<VideoFrameBuffer>& buffer); 108 Builder& set_timestamp_ms(int64_t timestamp_ms); 109 Builder& set_timestamp_us(int64_t timestamp_us); 110 Builder& set_timestamp_rtp(uint32_t timestamp_rtp); 111 Builder& set_ntp_time_ms(int64_t ntp_time_ms); 112 Builder& set_rotation(VideoRotation rotation); 113 Builder& set_color_space(const absl::optional<ColorSpace>& color_space); 114 Builder& set_color_space(const ColorSpace* color_space); 115 Builder& set_id(uint16_t id); 116 Builder& set_update_rect(const absl::optional<UpdateRect>& update_rect); 117 Builder& set_packet_infos(RtpPacketInfos packet_infos); 118 119 private: 120 uint16_t id_ = kNotSetId; 121 rtc::scoped_refptr<webrtc::VideoFrameBuffer> video_frame_buffer_; 122 int64_t timestamp_us_ = 0; 123 uint32_t timestamp_rtp_ = 0; 124 int64_t ntp_time_ms_ = 0; 125 VideoRotation rotation_ = kVideoRotation_0; 126 absl::optional<ColorSpace> color_space_; 127 RenderParameters render_parameters_; 128 absl::optional<UpdateRect> update_rect_; 129 RtpPacketInfos packet_infos_; 130 }; 131 132 // To be deprecated. Migrate all use to Builder. 133 VideoFrame(const rtc::scoped_refptr<VideoFrameBuffer>& buffer, 134 webrtc::VideoRotation rotation, 135 int64_t timestamp_us); 136 VideoFrame(const rtc::scoped_refptr<VideoFrameBuffer>& buffer, 137 uint32_t timestamp_rtp, 138 int64_t render_time_ms, 139 VideoRotation rotation); 140 141 ~VideoFrame(); 142 143 // Support move and copy. 144 VideoFrame(const VideoFrame&); 145 VideoFrame(VideoFrame&&); 146 VideoFrame& operator=(const VideoFrame&); 147 VideoFrame& operator=(VideoFrame&&); 148 149 // Get frame width. 150 int width() const; 151 // Get frame height. 152 int height() const; 153 // Get frame size in pixels. 154 uint32_t size() const; 155 156 // Get frame ID. Returns `kNotSetId` if ID is not set. Not guaranteed to be 157 // transferred from the sender to the receiver, but preserved on the sender 158 // side. The id should be propagated between all frame modifications during 159 // its lifetime from capturing to sending as encoded image. It is intended to 160 // be unique over a time window of a few minutes for the peer connection to 161 // which the corresponding video stream belongs to. id()162 uint16_t id() const { return id_; } set_id(uint16_t id)163 void set_id(uint16_t id) { id_ = id; } 164 165 // System monotonic clock, same timebase as rtc::TimeMicros(). timestamp_us()166 int64_t timestamp_us() const { return timestamp_us_; } set_timestamp_us(int64_t timestamp_us)167 void set_timestamp_us(int64_t timestamp_us) { timestamp_us_ = timestamp_us; } 168 169 // Set frame timestamp (90kHz). set_timestamp(uint32_t timestamp)170 void set_timestamp(uint32_t timestamp) { timestamp_rtp_ = timestamp; } 171 172 // Get frame timestamp (90kHz). timestamp()173 uint32_t timestamp() const { return timestamp_rtp_; } 174 175 // Set capture ntp time in milliseconds. set_ntp_time_ms(int64_t ntp_time_ms)176 void set_ntp_time_ms(int64_t ntp_time_ms) { ntp_time_ms_ = ntp_time_ms; } 177 178 // Get capture ntp time in milliseconds. ntp_time_ms()179 int64_t ntp_time_ms() const { return ntp_time_ms_; } 180 181 // Naming convention for Coordination of Video Orientation. Please see 182 // http://www.etsi.org/deliver/etsi_ts/126100_126199/126114/12.07.00_60/ts_126114v120700p.pdf 183 // 184 // "pending rotation" or "pending" = a frame that has a VideoRotation > 0. 185 // 186 // "not pending" = a frame that has a VideoRotation == 0. 187 // 188 // "apply rotation" = modify a frame from being "pending" to being "not 189 // pending" rotation (a no-op for "unrotated"). 190 // rotation()191 VideoRotation rotation() const { return rotation_; } set_rotation(VideoRotation rotation)192 void set_rotation(VideoRotation rotation) { rotation_ = rotation; } 193 194 // Get color space when available. color_space()195 const absl::optional<ColorSpace>& color_space() const { return color_space_; } set_color_space(const absl::optional<ColorSpace> & color_space)196 void set_color_space(const absl::optional<ColorSpace>& color_space) { 197 color_space_ = color_space; 198 } 199 render_parameters()200 RenderParameters render_parameters() const { return render_parameters_; } set_render_parameters(const RenderParameters & render_parameters)201 void set_render_parameters(const RenderParameters& render_parameters) { 202 render_parameters_ = render_parameters; 203 } 204 205 // Deprecated in favor of render_parameters, will be removed once Chromium is 206 // updated. max_composition_delay_in_frames() is used in an experiment of a 207 // low-latency renderer algorithm see crbug.com/1138888. 208 [[deprecated("Use render_parameters() instead.")]] absl::optional<int32_t> max_composition_delay_in_frames()209 max_composition_delay_in_frames() const { 210 return render_parameters_.max_composition_delay_in_frames; 211 } 212 213 // Get render time in milliseconds. 214 int64_t render_time_ms() const; 215 216 // Return the underlying buffer. Never nullptr for a properly 217 // initialized VideoFrame. 218 rtc::scoped_refptr<webrtc::VideoFrameBuffer> video_frame_buffer() const; 219 220 void set_video_frame_buffer( 221 const rtc::scoped_refptr<VideoFrameBuffer>& buffer); 222 223 // Return true if the frame is stored in a texture. is_texture()224 bool is_texture() const { 225 return video_frame_buffer()->type() == VideoFrameBuffer::Type::kNative; 226 } 227 has_update_rect()228 bool has_update_rect() const { return update_rect_.has_value(); } 229 230 // Returns update_rect set by the builder or set_update_rect() or whole frame 231 // rect if no update rect is available. update_rect()232 UpdateRect update_rect() const { 233 return update_rect_.value_or(UpdateRect{0, 0, width(), height()}); 234 } 235 236 // Rectangle must be within the frame dimensions. set_update_rect(const VideoFrame::UpdateRect & update_rect)237 void set_update_rect(const VideoFrame::UpdateRect& update_rect) { 238 RTC_DCHECK_GE(update_rect.offset_x, 0); 239 RTC_DCHECK_GE(update_rect.offset_y, 0); 240 RTC_DCHECK_LE(update_rect.offset_x + update_rect.width, width()); 241 RTC_DCHECK_LE(update_rect.offset_y + update_rect.height, height()); 242 update_rect_ = update_rect; 243 } 244 clear_update_rect()245 void clear_update_rect() { update_rect_ = absl::nullopt; } 246 247 // Get information about packets used to assemble this video frame. Might be 248 // empty if the information isn't available. packet_infos()249 const RtpPacketInfos& packet_infos() const { return packet_infos_; } set_packet_infos(RtpPacketInfos value)250 void set_packet_infos(RtpPacketInfos value) { 251 packet_infos_ = std::move(value); 252 } 253 processing_time()254 const absl::optional<ProcessingTime> processing_time() const { 255 return processing_time_; 256 } set_processing_time(const ProcessingTime & processing_time)257 void set_processing_time(const ProcessingTime& processing_time) { 258 processing_time_ = processing_time; 259 } 260 261 private: 262 VideoFrame(uint16_t id, 263 const rtc::scoped_refptr<VideoFrameBuffer>& buffer, 264 int64_t timestamp_us, 265 uint32_t timestamp_rtp, 266 int64_t ntp_time_ms, 267 VideoRotation rotation, 268 const absl::optional<ColorSpace>& color_space, 269 const RenderParameters& render_parameters, 270 const absl::optional<UpdateRect>& update_rect, 271 RtpPacketInfos packet_infos); 272 273 uint16_t id_; 274 // An opaque reference counted handle that stores the pixel data. 275 rtc::scoped_refptr<webrtc::VideoFrameBuffer> video_frame_buffer_; 276 uint32_t timestamp_rtp_; 277 int64_t ntp_time_ms_; 278 int64_t timestamp_us_; 279 VideoRotation rotation_; 280 absl::optional<ColorSpace> color_space_; 281 // Contains parameters that affect have the frame should be rendered. 282 RenderParameters render_parameters_; 283 // Updated since the last frame area. If present it means that the bounding 284 // box of all the changes is within the rectangular area and is close to it. 285 // If absent, it means that there's no information about the change at all and 286 // update_rect() will return a rectangle corresponding to the entire frame. 287 absl::optional<UpdateRect> update_rect_; 288 // Information about packets used to assemble this video frame. This is needed 289 // by `SourceTracker` when the frame is delivered to the RTCRtpReceiver's 290 // MediaStreamTrack, in order to implement getContributingSources(). See: 291 // https://w3c.github.io/webrtc-pc/#dom-rtcrtpreceiver-getcontributingsources 292 RtpPacketInfos packet_infos_; 293 // Processing timestamps of the frame. For received video frames these are the 294 // timestamps when the frame is sent to the decoder and the decoded image 295 // returned from the decoder. 296 // Currently, not set for locally captured video frames. 297 absl::optional<ProcessingTime> processing_time_; 298 }; 299 300 } // namespace webrtc 301 302 #endif // API_VIDEO_VIDEO_FRAME_H_ 303