1 /* 2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef CALL_VIDEO_RECEIVE_STREAM_H_ 12 #define CALL_VIDEO_RECEIVE_STREAM_H_ 13 14 #include <limits> 15 #include <map> 16 #include <set> 17 #include <string> 18 #include <utility> 19 #include <vector> 20 21 #include "api/call/transport.h" 22 #include "api/crypto/crypto_options.h" 23 #include "api/rtp_headers.h" 24 #include "api/rtp_parameters.h" 25 #include "api/video/recordable_encoded_frame.h" 26 #include "api/video/video_content_type.h" 27 #include "api/video/video_frame.h" 28 #include "api/video/video_sink_interface.h" 29 #include "api/video/video_timing.h" 30 #include "api/video_codecs/sdp_video_format.h" 31 #include "call/receive_stream.h" 32 #include "call/rtp_config.h" 33 #include "common_video/frame_counts.h" 34 #include "modules/rtp_rtcp/include/rtcp_statistics.h" 35 #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" 36 37 namespace webrtc { 38 39 class RtpPacketSinkInterface; 40 class VideoDecoderFactory; 41 42 class VideoReceiveStreamInterface : public MediaReceiveStreamInterface { 43 public: 44 // Class for handling moving in/out recording state. 45 struct RecordingState { 46 RecordingState() = default; RecordingStateRecordingState47 explicit RecordingState( 48 std::function<void(const RecordableEncodedFrame&)> callback) 49 : callback(std::move(callback)) {} 50 51 // Callback stored from the VideoReceiveStreamInterface. The 52 // VideoReceiveStreamInterface client should not interpret the attribute. 53 std::function<void(const RecordableEncodedFrame&)> callback; 54 // Memento of when a keyframe request was last sent. The 55 // VideoReceiveStreamInterface client should not interpret the attribute. 56 absl::optional<int64_t> last_keyframe_request_ms; 57 }; 58 59 // TODO(mflodman) Move all these settings to VideoDecoder and move the 60 // declaration to common_types.h. 61 struct Decoder { 62 Decoder(SdpVideoFormat video_format, int payload_type); 63 Decoder(); 64 Decoder(const Decoder&); 65 ~Decoder(); 66 67 bool operator==(const Decoder& other) const; 68 69 std::string ToString() const; 70 71 SdpVideoFormat video_format; 72 73 // Received RTP packets with this payload type will be sent to this decoder 74 // instance. 75 int payload_type = 0; 76 }; 77 78 struct Stats { 79 Stats(); 80 ~Stats(); 81 std::string ToString(int64_t time_ms) const; 82 83 int network_frame_rate = 0; 84 int decode_frame_rate = 0; 85 int render_frame_rate = 0; 86 uint32_t frames_rendered = 0; 87 88 // Decoder stats. 89 std::string decoder_implementation_name = "unknown"; 90 absl::optional<bool> power_efficient_decoder; 91 FrameCounts frame_counts; 92 int decode_ms = 0; 93 int max_decode_ms = 0; 94 int current_delay_ms = 0; 95 int target_delay_ms = 0; 96 int jitter_buffer_ms = 0; 97 // https://w3c.github.io/webrtc-stats/#dom-rtcvideoreceiverstats-jitterbufferdelay 98 double jitter_buffer_delay_seconds = 0; 99 // https://w3c.github.io/webrtc-stats/#dom-rtcvideoreceiverstats-jitterbufferemittedcount 100 uint64_t jitter_buffer_emitted_count = 0; 101 int min_playout_delay_ms = 0; 102 int render_delay_ms = 10; 103 int64_t interframe_delay_max_ms = -1; 104 // Frames dropped due to decoding failures or if the system is too slow. 105 // https://www.w3.org/TR/webrtc-stats/#dom-rtcvideoreceiverstats-framesdropped 106 uint32_t frames_dropped = 0; 107 uint32_t frames_decoded = 0; 108 // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-totaldecodetime 109 TimeDelta total_decode_time = TimeDelta::Zero(); 110 // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-totalprocessingdelay 111 TimeDelta total_processing_delay = TimeDelta::Zero(); 112 // TODO(bugs.webrtc.org/13986): standardize 113 TimeDelta total_assembly_time = TimeDelta::Zero(); 114 uint32_t frames_assembled_from_multiple_packets = 0; 115 // Total inter frame delay in seconds. 116 // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-totalinterframedelay 117 double total_inter_frame_delay = 0; 118 // Total squared inter frame delay in seconds^2. 119 // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-totalsqauredinterframedelay 120 double total_squared_inter_frame_delay = 0; 121 int64_t first_frame_received_to_decoded_ms = -1; 122 absl::optional<uint64_t> qp_sum; 123 124 int current_payload_type = -1; 125 126 int total_bitrate_bps = 0; 127 128 int width = 0; 129 int height = 0; 130 131 uint32_t freeze_count = 0; 132 uint32_t pause_count = 0; 133 uint32_t total_freezes_duration_ms = 0; 134 uint32_t total_pauses_duration_ms = 0; 135 uint32_t total_frames_duration_ms = 0; 136 double sum_squared_frame_durations = 0.0; 137 138 VideoContentType content_type = VideoContentType::UNSPECIFIED; 139 140 // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-estimatedplayouttimestamp 141 absl::optional<int64_t> estimated_playout_ntp_timestamp_ms; 142 int sync_offset_ms = std::numeric_limits<int>::max(); 143 144 uint32_t ssrc = 0; 145 std::string c_name; 146 RtpReceiveStats rtp_stats; 147 RtcpPacketTypeCounter rtcp_packet_type_counts; 148 149 // Timing frame info: all important timestamps for a full lifetime of a 150 // single 'timing frame'. 151 absl::optional<webrtc::TimingFrameInfo> timing_frame_info; 152 }; 153 154 struct Config { 155 private: 156 // Access to the copy constructor is private to force use of the Copy() 157 // method for those exceptional cases where we do use it. 158 Config(const Config&); 159 160 public: 161 Config() = delete; 162 Config(Config&&); 163 Config(Transport* rtcp_send_transport, 164 VideoDecoderFactory* decoder_factory = nullptr); 165 Config& operator=(Config&&); 166 Config& operator=(const Config&) = delete; 167 ~Config(); 168 169 // Mostly used by tests. Avoid creating copies if you can. CopyConfig170 Config Copy() const { return Config(*this); } 171 172 std::string ToString() const; 173 174 // Decoders for every payload that we can receive. 175 std::vector<Decoder> decoders; 176 177 // Ownership stays with WebrtcVideoEngine (delegated from PeerConnection). 178 VideoDecoderFactory* decoder_factory = nullptr; 179 180 // Receive-stream specific RTP settings. 181 struct Rtp : public ReceiveStreamRtpConfig { 182 Rtp(); 183 Rtp(const Rtp&); 184 ~Rtp(); 185 std::string ToString() const; 186 187 // See NackConfig for description. 188 NackConfig nack; 189 190 // See RtcpMode for description. 191 RtcpMode rtcp_mode = RtcpMode::kCompound; 192 193 // Extended RTCP settings. 194 struct RtcpXr { 195 // True if RTCP Receiver Reference Time Report Block extension 196 // (RFC 3611) should be enabled. 197 bool receiver_reference_time_report = false; 198 } rtcp_xr; 199 200 // How to request keyframes from a remote sender. Applies only if lntf is 201 // disabled. 202 KeyFrameReqMethod keyframe_method = KeyFrameReqMethod::kPliRtcp; 203 204 // See LntfConfig for description. 205 LntfConfig lntf; 206 207 // Payload types for ULPFEC and RED, respectively. 208 int ulpfec_payload_type = -1; 209 int red_payload_type = -1; 210 211 // SSRC for retransmissions. 212 uint32_t rtx_ssrc = 0; 213 214 // Set if the stream is protected using FlexFEC. 215 bool protected_by_flexfec = false; 216 217 // Optional callback sink to support additional packet handlers such as 218 // FlexFec. 219 RtpPacketSinkInterface* packet_sink_ = nullptr; 220 221 // Map from rtx payload type -> media payload type. 222 // For RTX to be enabled, both an SSRC and this mapping are needed. 223 std::map<int, int> rtx_associated_payload_types; 224 225 // Payload types that should be depacketized using raw depacketizer 226 // (payload header will not be parsed and must not be present, additional 227 // meta data is expected to be present in generic frame descriptor 228 // RTP header extension). 229 std::set<int> raw_payload_types; 230 } rtp; 231 232 // Transport for outgoing packets (RTCP). 233 Transport* rtcp_send_transport = nullptr; 234 235 // Must always be set. 236 rtc::VideoSinkInterface<VideoFrame>* renderer = nullptr; 237 238 // Expected delay needed by the renderer, i.e. the frame will be delivered 239 // this many milliseconds, if possible, earlier than the ideal render time. 240 int render_delay_ms = 10; 241 242 // If false, pass frames on to the renderer as soon as they are 243 // available. 244 bool enable_prerenderer_smoothing = true; 245 246 // Identifier for an A/V synchronization group. Empty string to disable. 247 // TODO(pbos): Synchronize streams in a sync group, not just video streams 248 // to one of the audio streams. 249 std::string sync_group; 250 251 // An optional custom frame decryptor that allows the entire frame to be 252 // decrypted in whatever way the caller choses. This is not required by 253 // default. 254 rtc::scoped_refptr<webrtc::FrameDecryptorInterface> frame_decryptor; 255 256 // Per PeerConnection cryptography options. 257 CryptoOptions crypto_options; 258 259 rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer; 260 }; 261 262 // TODO(pbos): Add info on currently-received codec to Stats. 263 virtual Stats GetStats() const = 0; 264 265 // Sets a base minimum for the playout delay. Base minimum delay sets lower 266 // bound on minimum delay value determining lower bound on playout delay. 267 // 268 // Returns true if value was successfully set, false overwise. 269 virtual bool SetBaseMinimumPlayoutDelayMs(int delay_ms) = 0; 270 271 // Returns current value of base minimum delay in milliseconds. 272 virtual int GetBaseMinimumPlayoutDelayMs() const = 0; 273 274 // Sets and returns recording state. The old state is moved out 275 // of the video receive stream and returned to the caller, and `state` 276 // is moved in. If the state's callback is set, it will be called with 277 // recordable encoded frames as they arrive. 278 // If `generate_key_frame` is true, the method will generate a key frame. 279 // When the function returns, it's guaranteed that all old callouts 280 // to the returned callback has ceased. 281 // Note: the client should not interpret the returned state's attributes, but 282 // instead treat it as opaque data. 283 virtual RecordingState SetAndGetRecordingState(RecordingState state, 284 bool generate_key_frame) = 0; 285 286 // Cause eventual generation of a key frame from the sender. 287 virtual void GenerateKeyFrame() = 0; 288 289 virtual void SetRtcpMode(RtcpMode mode) = 0; 290 291 // Sets or clears a flexfec RTP sink. This affects `rtp.packet_sink_` and 292 // `rtp.protected_by_flexfec` parts of the configuration. Must be called on 293 // the packet delivery thread. 294 // TODO(bugs.webrtc.org/11993): Packet delivery thread today means `worker 295 // thread` but will be `network thread`. 296 virtual void SetFlexFecProtection(RtpPacketSinkInterface* flexfec_sink) = 0; 297 298 // Turns on/off loss notifications. Must be called on the packet delivery 299 // thread. 300 virtual void SetLossNotificationEnabled(bool enabled) = 0; 301 302 // Modify `rtp.nack.rtp_history_ms` post construction. Setting this value 303 // to 0 disables nack. 304 // Must be called on the packet delivery thread. 305 virtual void SetNackHistory(TimeDelta history) = 0; 306 307 virtual void SetProtectionPayloadTypes(int red_payload_type, 308 int ulpfec_payload_type) = 0; 309 310 virtual void SetRtcpXr(Config::Rtp::RtcpXr rtcp_xr) = 0; 311 312 virtual void SetAssociatedPayloadTypes( 313 std::map<int, int> associated_payload_types) = 0; 314 315 protected: ~VideoReceiveStreamInterface()316 virtual ~VideoReceiveStreamInterface() {} 317 }; 318 319 } // namespace webrtc 320 321 #endif // CALL_VIDEO_RECEIVE_STREAM_H_ 322