1 /*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/video_coding/receiver.h"
12
13
14 #include <cstdint>
15 #include <cstdlib>
16 #include <utility>
17 #include <vector>
18
19 #include "absl/memory/memory.h"
20 #include "api/video/encoded_image.h"
21 #include "modules/video_coding/encoded_frame.h"
22 #include "modules/video_coding/internal_defines.h"
23 #include "modules/video_coding/jitter_buffer_common.h"
24 #include "rtc_base/logging.h"
25 #include "rtc_base/numerics/safe_conversions.h"
26 #include "rtc_base/trace_event.h"
27 #include "system_wrappers/include/clock.h"
28
29 namespace webrtc {
30
31 enum { kMaxReceiverDelayMs = 10000 };
32
VCMReceiver(VCMTiming * timing,Clock * clock,const FieldTrialsView & field_trials)33 VCMReceiver::VCMReceiver(VCMTiming* timing,
34 Clock* clock,
35 const FieldTrialsView& field_trials)
36 : VCMReceiver::VCMReceiver(timing,
37 clock,
38 absl::WrapUnique(EventWrapper::Create()),
39 absl::WrapUnique(EventWrapper::Create()),
40 field_trials) {}
41
VCMReceiver(VCMTiming * timing,Clock * clock,std::unique_ptr<EventWrapper> receiver_event,std::unique_ptr<EventWrapper> jitter_buffer_event,const FieldTrialsView & field_trials)42 VCMReceiver::VCMReceiver(VCMTiming* timing,
43 Clock* clock,
44 std::unique_ptr<EventWrapper> receiver_event,
45 std::unique_ptr<EventWrapper> jitter_buffer_event,
46 const FieldTrialsView& field_trials)
47 : clock_(clock),
48 jitter_buffer_(clock_, std::move(jitter_buffer_event), field_trials),
49 timing_(timing),
50 render_wait_event_(std::move(receiver_event)),
51 max_video_delay_ms_(kMaxVideoDelayMs) {
52 jitter_buffer_.Start();
53 }
54
~VCMReceiver()55 VCMReceiver::~VCMReceiver() {
56 render_wait_event_->Set();
57 }
58
InsertPacket(const VCMPacket & packet)59 int32_t VCMReceiver::InsertPacket(const VCMPacket& packet) {
60 // Insert the packet into the jitter buffer. The packet can either be empty or
61 // contain media at this point.
62 bool retransmitted = false;
63 const VCMFrameBufferEnum ret =
64 jitter_buffer_.InsertPacket(packet, &retransmitted);
65 if (ret == kOldPacket) {
66 return VCM_OK;
67 } else if (ret == kFlushIndicator) {
68 return VCM_FLUSH_INDICATOR;
69 } else if (ret < 0) {
70 return VCM_JITTER_BUFFER_ERROR;
71 }
72 if (ret == kCompleteSession && !retransmitted) {
73 // We don't want to include timestamps which have suffered from
74 // retransmission here, since we compensate with extra retransmission
75 // delay within the jitter estimate.
76 timing_->IncomingTimestamp(packet.timestamp, clock_->CurrentTime());
77 }
78 return VCM_OK;
79 }
80
FrameForDecoding(uint16_t max_wait_time_ms,bool prefer_late_decoding)81 VCMEncodedFrame* VCMReceiver::FrameForDecoding(uint16_t max_wait_time_ms,
82 bool prefer_late_decoding) {
83 const int64_t start_time_ms = clock_->TimeInMilliseconds();
84 uint32_t frame_timestamp = 0;
85 int min_playout_delay_ms = -1;
86 int max_playout_delay_ms = -1;
87 int64_t render_time_ms = 0;
88 // Exhaust wait time to get a complete frame for decoding.
89 VCMEncodedFrame* found_frame =
90 jitter_buffer_.NextCompleteFrame(max_wait_time_ms);
91
92 if (found_frame) {
93 frame_timestamp = found_frame->Timestamp();
94 min_playout_delay_ms = found_frame->EncodedImage().playout_delay_.min_ms;
95 max_playout_delay_ms = found_frame->EncodedImage().playout_delay_.max_ms;
96 } else {
97 return nullptr;
98 }
99
100 if (min_playout_delay_ms >= 0)
101 timing_->set_min_playout_delay(TimeDelta::Millis(min_playout_delay_ms));
102
103 if (max_playout_delay_ms >= 0)
104 timing_->set_max_playout_delay(TimeDelta::Millis(max_playout_delay_ms));
105
106 // We have a frame - Set timing and render timestamp.
107 timing_->SetJitterDelay(
108 TimeDelta::Millis(jitter_buffer_.EstimatedJitterMs()));
109 const Timestamp now = clock_->CurrentTime();
110 const int64_t now_ms = now.ms();
111 timing_->UpdateCurrentDelay(frame_timestamp);
112 render_time_ms = timing_->RenderTime(frame_timestamp, now).ms();
113 // Check render timing.
114 bool timing_error = false;
115 // Assume that render timing errors are due to changes in the video stream.
116 if (render_time_ms < 0) {
117 timing_error = true;
118 } else if (std::abs(render_time_ms - now_ms) > max_video_delay_ms_) {
119 int frame_delay = static_cast<int>(std::abs(render_time_ms - now_ms));
120 RTC_LOG(LS_WARNING)
121 << "A frame about to be decoded is out of the configured "
122 "delay bounds ("
123 << frame_delay << " > " << max_video_delay_ms_
124 << "). Resetting the video jitter buffer.";
125 timing_error = true;
126 } else if (static_cast<int>(timing_->TargetVideoDelay().ms()) >
127 max_video_delay_ms_) {
128 RTC_LOG(LS_WARNING) << "The video target delay has grown larger than "
129 << max_video_delay_ms_
130 << " ms. Resetting jitter buffer.";
131 timing_error = true;
132 }
133
134 if (timing_error) {
135 // Timing error => reset timing and flush the jitter buffer.
136 jitter_buffer_.Flush();
137 timing_->Reset();
138 return NULL;
139 }
140
141 if (prefer_late_decoding) {
142 // Decode frame as close as possible to the render timestamp.
143 const int32_t available_wait_time =
144 max_wait_time_ms -
145 static_cast<int32_t>(clock_->TimeInMilliseconds() - start_time_ms);
146 uint16_t new_max_wait_time =
147 static_cast<uint16_t>(VCM_MAX(available_wait_time, 0));
148 uint32_t wait_time_ms = rtc::saturated_cast<uint32_t>(
149 timing_
150 ->MaxWaitingTime(Timestamp::Millis(render_time_ms),
151 clock_->CurrentTime(),
152 /*too_many_frames_queued=*/false)
153 .ms());
154 if (new_max_wait_time < wait_time_ms) {
155 // We're not allowed to wait until the frame is supposed to be rendered,
156 // waiting as long as we're allowed to avoid busy looping, and then return
157 // NULL. Next call to this function might return the frame.
158 render_wait_event_->Wait(new_max_wait_time);
159 return NULL;
160 }
161 // Wait until it's time to render.
162 render_wait_event_->Wait(wait_time_ms);
163 }
164
165 // Extract the frame from the jitter buffer and set the render time.
166 VCMEncodedFrame* frame = jitter_buffer_.ExtractAndSetDecode(frame_timestamp);
167 if (frame == NULL) {
168 return NULL;
169 }
170 frame->SetRenderTime(render_time_ms);
171 TRACE_EVENT_ASYNC_STEP1("webrtc", "Video", frame->Timestamp(), "SetRenderTS",
172 "render_time", frame->RenderTimeMs());
173 return frame;
174 }
175
ReleaseFrame(VCMEncodedFrame * frame)176 void VCMReceiver::ReleaseFrame(VCMEncodedFrame* frame) {
177 jitter_buffer_.ReleaseFrame(frame);
178 }
179
SetNackSettings(size_t max_nack_list_size,int max_packet_age_to_nack,int max_incomplete_time_ms)180 void VCMReceiver::SetNackSettings(size_t max_nack_list_size,
181 int max_packet_age_to_nack,
182 int max_incomplete_time_ms) {
183 jitter_buffer_.SetNackSettings(max_nack_list_size, max_packet_age_to_nack,
184 max_incomplete_time_ms);
185 }
186
NackList(bool * request_key_frame)187 std::vector<uint16_t> VCMReceiver::NackList(bool* request_key_frame) {
188 return jitter_buffer_.GetNackList(request_key_frame);
189 }
190
191 } // namespace webrtc
192