1 /*
2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/remote_bitrate_estimator/remote_estimator_proxy.h"
12
13 #include <algorithm>
14 #include <limits>
15 #include <memory>
16 #include <utility>
17
18 #include "api/units/data_size.h"
19 #include "modules/rtp_rtcp/source/rtcp_packet/remote_estimate.h"
20 #include "modules/rtp_rtcp/source/rtcp_packet/transport_feedback.h"
21 #include "rtc_base/checks.h"
22 #include "rtc_base/logging.h"
23 #include "rtc_base/numerics/safe_minmax.h"
24 #include "system_wrappers/include/clock.h"
25
26 namespace webrtc {
27 namespace {
28 // The maximum allowed value for a timestamp in milliseconds. This is lower
29 // than the numerical limit since we often convert to microseconds.
30 constexpr int64_t kMaxTimeMs = std::numeric_limits<int64_t>::max() / 1000;
31 constexpr TimeDelta kBackWindow = TimeDelta::Millis(500);
32 constexpr TimeDelta kMinInterval = TimeDelta::Millis(50);
33 constexpr TimeDelta kMaxInterval = TimeDelta::Millis(250);
34 constexpr TimeDelta kDefaultInterval = TimeDelta::Millis(100);
35
GetAbsoluteSendTimeDelta(uint32_t new_sendtime,uint32_t previous_sendtime)36 TimeDelta GetAbsoluteSendTimeDelta(uint32_t new_sendtime,
37 uint32_t previous_sendtime) {
38 static constexpr uint32_t kWrapAroundPeriod = 0x0100'0000;
39 RTC_DCHECK_LT(new_sendtime, kWrapAroundPeriod);
40 RTC_DCHECK_LT(previous_sendtime, kWrapAroundPeriod);
41 uint32_t delta = (new_sendtime - previous_sendtime) % kWrapAroundPeriod;
42 if (delta >= kWrapAroundPeriod / 2) {
43 // absolute send time wraps around, thus treat deltas larger than half of
44 // the wrap around period as negative. Ignore reordering of packets and
45 // treat them as they have approximately the same send time.
46 return TimeDelta::Zero();
47 }
48 return TimeDelta::Micros(int64_t{delta} * 1'000'000 / (1 << 18));
49 }
50 } // namespace
51
RemoteEstimatorProxy(TransportFeedbackSender feedback_sender,NetworkStateEstimator * network_state_estimator)52 RemoteEstimatorProxy::RemoteEstimatorProxy(
53 TransportFeedbackSender feedback_sender,
54 NetworkStateEstimator* network_state_estimator)
55 : feedback_sender_(std::move(feedback_sender)),
56 last_process_time_(Timestamp::MinusInfinity()),
57 network_state_estimator_(network_state_estimator),
58 media_ssrc_(0),
59 feedback_packet_count_(0),
60 packet_overhead_(DataSize::Zero()),
61 send_interval_(kDefaultInterval),
62 send_periodic_feedback_(true),
63 previous_abs_send_time_(0),
64 abs_send_timestamp_(Timestamp::Zero()) {
65 RTC_LOG(LS_INFO)
66 << "Maximum interval between transport feedback RTCP messages: "
67 << kMaxInterval;
68 }
69
~RemoteEstimatorProxy()70 RemoteEstimatorProxy::~RemoteEstimatorProxy() {}
71
MaybeCullOldPackets(int64_t sequence_number,Timestamp arrival_time)72 void RemoteEstimatorProxy::MaybeCullOldPackets(int64_t sequence_number,
73 Timestamp arrival_time) {
74 if (periodic_window_start_seq_ >=
75 packet_arrival_times_.end_sequence_number() &&
76 arrival_time - Timestamp::Zero() >= kBackWindow) {
77 // Start new feedback packet, cull old packets.
78 packet_arrival_times_.RemoveOldPackets(sequence_number,
79 arrival_time - kBackWindow);
80 }
81 }
82
IncomingPacket(int64_t arrival_time_ms,size_t payload_size,const RTPHeader & header)83 void RemoteEstimatorProxy::IncomingPacket(int64_t arrival_time_ms,
84 size_t payload_size,
85 const RTPHeader& header) {
86 if (arrival_time_ms < 0 || arrival_time_ms >= kMaxTimeMs) {
87 RTC_LOG(LS_WARNING) << "Arrival time out of bounds: " << arrival_time_ms;
88 return;
89 }
90 Packet packet = {.arrival_time = Timestamp::Millis(arrival_time_ms),
91 .size = DataSize::Bytes(header.headerLength + payload_size),
92 .ssrc = header.ssrc};
93 if (header.extension.hasTransportSequenceNumber) {
94 packet.transport_sequence_number = header.extension.transportSequenceNumber;
95 }
96 if (header.extension.hasAbsoluteSendTime) {
97 packet.absolute_send_time_24bits = header.extension.absoluteSendTime;
98 }
99 packet.feedback_request = header.extension.feedback_request;
100
101 IncomingPacket(packet);
102 }
103
IncomingPacket(Packet packet)104 void RemoteEstimatorProxy::IncomingPacket(Packet packet) {
105 MutexLock lock(&lock_);
106 media_ssrc_ = packet.ssrc;
107 int64_t seq = 0;
108
109 if (packet.transport_sequence_number.has_value()) {
110 seq = unwrapper_.Unwrap(*packet.transport_sequence_number);
111
112 if (send_periodic_feedback_) {
113 MaybeCullOldPackets(seq, packet.arrival_time);
114
115 if (!periodic_window_start_seq_ || seq < *periodic_window_start_seq_) {
116 periodic_window_start_seq_ = seq;
117 }
118 }
119
120 // We are only interested in the first time a packet is received.
121 if (packet_arrival_times_.has_received(seq)) {
122 return;
123 }
124
125 packet_arrival_times_.AddPacket(seq, packet.arrival_time);
126
127 // Limit the range of sequence numbers to send feedback for.
128 if (!periodic_window_start_seq_.has_value() ||
129 periodic_window_start_seq_.value() <
130 packet_arrival_times_.begin_sequence_number()) {
131 periodic_window_start_seq_ =
132 packet_arrival_times_.begin_sequence_number();
133 }
134
135 if (packet.feedback_request) {
136 // Send feedback packet immediately.
137 SendFeedbackOnRequest(seq, *packet.feedback_request);
138 }
139 }
140
141 if (network_state_estimator_ && packet.absolute_send_time_24bits) {
142 PacketResult packet_result;
143 packet_result.receive_time = packet.arrival_time;
144 abs_send_timestamp_ += GetAbsoluteSendTimeDelta(
145 *packet.absolute_send_time_24bits, previous_abs_send_time_);
146 previous_abs_send_time_ = *packet.absolute_send_time_24bits;
147 packet_result.sent_packet.send_time = abs_send_timestamp_;
148 packet_result.sent_packet.size = packet.size + packet_overhead_;
149 packet_result.sent_packet.sequence_number = seq;
150 network_state_estimator_->OnReceivedPacket(packet_result);
151 }
152 }
153
Process(Timestamp now)154 TimeDelta RemoteEstimatorProxy::Process(Timestamp now) {
155 MutexLock lock(&lock_);
156 if (!send_periodic_feedback_) {
157 return TimeDelta::PlusInfinity();
158 }
159 Timestamp next_process_time = last_process_time_ + send_interval_;
160 if (now >= next_process_time) {
161 last_process_time_ = now;
162 SendPeriodicFeedbacks();
163 return send_interval_;
164 }
165
166 return next_process_time - now;
167 }
168
OnBitrateChanged(int bitrate_bps)169 void RemoteEstimatorProxy::OnBitrateChanged(int bitrate_bps) {
170 // TwccReportSize = Ipv4(20B) + UDP(8B) + SRTP(10B) +
171 // AverageTwccReport(30B)
172 // TwccReport size at 50ms interval is 24 byte.
173 // TwccReport size at 250ms interval is 36 byte.
174 // AverageTwccReport = (TwccReport(50ms) + TwccReport(250ms)) / 2
175 constexpr DataSize kTwccReportSize = DataSize::Bytes(20 + 8 + 10 + 30);
176 constexpr DataRate kMinTwccRate = kTwccReportSize / kMaxInterval;
177
178 // Let TWCC reports occupy 5% of total bandwidth.
179 DataRate twcc_bitrate = DataRate::BitsPerSec(0.05 * bitrate_bps);
180
181 // Check upper send_interval bound by checking bitrate to avoid overflow when
182 // dividing by small bitrate, in particular avoid dividing by zero bitrate.
183 TimeDelta send_interval =
184 twcc_bitrate <= kMinTwccRate
185 ? kMaxInterval
186 : std::max(kTwccReportSize / twcc_bitrate, kMinInterval);
187
188 MutexLock lock(&lock_);
189 send_interval_ = send_interval;
190 }
191
SetSendPeriodicFeedback(bool send_periodic_feedback)192 void RemoteEstimatorProxy::SetSendPeriodicFeedback(
193 bool send_periodic_feedback) {
194 MutexLock lock(&lock_);
195 send_periodic_feedback_ = send_periodic_feedback;
196 }
197
SetTransportOverhead(DataSize overhead_per_packet)198 void RemoteEstimatorProxy::SetTransportOverhead(DataSize overhead_per_packet) {
199 MutexLock lock(&lock_);
200 packet_overhead_ = overhead_per_packet;
201 }
202
SendPeriodicFeedbacks()203 void RemoteEstimatorProxy::SendPeriodicFeedbacks() {
204 // `periodic_window_start_seq_` is the first sequence number to include in
205 // the current feedback packet. Some older may still be in the map, in case
206 // a reordering happens and we need to retransmit them.
207 if (!periodic_window_start_seq_)
208 return;
209
210 std::unique_ptr<rtcp::RemoteEstimate> remote_estimate;
211 if (network_state_estimator_) {
212 absl::optional<NetworkStateEstimate> state_estimate =
213 network_state_estimator_->GetCurrentEstimate();
214 if (state_estimate) {
215 remote_estimate = std::make_unique<rtcp::RemoteEstimate>();
216 remote_estimate->SetEstimate(state_estimate.value());
217 }
218 }
219
220 int64_t packet_arrival_times_end_seq =
221 packet_arrival_times_.end_sequence_number();
222 while (periodic_window_start_seq_ < packet_arrival_times_end_seq) {
223 auto feedback_packet = MaybeBuildFeedbackPacket(
224 /*include_timestamps=*/true, periodic_window_start_seq_.value(),
225 packet_arrival_times_end_seq,
226 /*is_periodic_update=*/true);
227
228 if (feedback_packet == nullptr) {
229 break;
230 }
231
232 RTC_DCHECK(feedback_sender_ != nullptr);
233
234 std::vector<std::unique_ptr<rtcp::RtcpPacket>> packets;
235 if (remote_estimate) {
236 packets.push_back(std::move(remote_estimate));
237 }
238 packets.push_back(std::move(feedback_packet));
239
240 feedback_sender_(std::move(packets));
241 // Note: Don't erase items from packet_arrival_times_ after sending, in
242 // case they need to be re-sent after a reordering. Removal will be
243 // handled by OnPacketArrival once packets are too old.
244 }
245 }
246
SendFeedbackOnRequest(int64_t sequence_number,const FeedbackRequest & feedback_request)247 void RemoteEstimatorProxy::SendFeedbackOnRequest(
248 int64_t sequence_number,
249 const FeedbackRequest& feedback_request) {
250 if (feedback_request.sequence_count == 0) {
251 return;
252 }
253
254 int64_t first_sequence_number =
255 sequence_number - feedback_request.sequence_count + 1;
256
257 auto feedback_packet = MaybeBuildFeedbackPacket(
258 feedback_request.include_timestamps, first_sequence_number,
259 sequence_number + 1, /*is_periodic_update=*/false);
260
261 // This is called when a packet has just been added.
262 RTC_DCHECK(feedback_packet != nullptr);
263
264 // Clear up to the first packet that is included in this feedback packet.
265 packet_arrival_times_.EraseTo(first_sequence_number);
266
267 RTC_DCHECK(feedback_sender_ != nullptr);
268 std::vector<std::unique_ptr<rtcp::RtcpPacket>> packets;
269 packets.push_back(std::move(feedback_packet));
270 feedback_sender_(std::move(packets));
271 }
272
273 std::unique_ptr<rtcp::TransportFeedback>
MaybeBuildFeedbackPacket(bool include_timestamps,int64_t begin_sequence_number_inclusive,int64_t end_sequence_number_exclusive,bool is_periodic_update)274 RemoteEstimatorProxy::MaybeBuildFeedbackPacket(
275 bool include_timestamps,
276 int64_t begin_sequence_number_inclusive,
277 int64_t end_sequence_number_exclusive,
278 bool is_periodic_update) {
279 RTC_DCHECK_LT(begin_sequence_number_inclusive, end_sequence_number_exclusive);
280
281 int64_t start_seq =
282 packet_arrival_times_.clamp(begin_sequence_number_inclusive);
283
284 int64_t end_seq = packet_arrival_times_.clamp(end_sequence_number_exclusive);
285
286 // Create the packet on demand, as it's not certain that there are packets
287 // in the range that have been received.
288 std::unique_ptr<rtcp::TransportFeedback> feedback_packet = nullptr;
289
290 int64_t next_sequence_number = begin_sequence_number_inclusive;
291
292 for (int64_t seq = start_seq; seq < end_seq; ++seq) {
293 PacketArrivalTimeMap::PacketArrivalTime packet =
294 packet_arrival_times_.FindNextAtOrAfter(seq);
295 seq = packet.sequence_number;
296 if (seq >= end_seq) {
297 break;
298 }
299
300 if (feedback_packet == nullptr) {
301 feedback_packet =
302 std::make_unique<rtcp::TransportFeedback>(include_timestamps);
303 feedback_packet->SetMediaSsrc(media_ssrc_);
304 // Base sequence number is the expected first sequence number. This is
305 // known, but we might not have actually received it, so the base time
306 // shall be the time of the first received packet in the feedback.
307 feedback_packet->SetBase(
308 static_cast<uint16_t>(begin_sequence_number_inclusive & 0xFFFF),
309 packet.arrival_time);
310 feedback_packet->SetFeedbackSequenceNumber(feedback_packet_count_++);
311 }
312
313 if (!feedback_packet->AddReceivedPacket(static_cast<uint16_t>(seq & 0xFFFF),
314 packet.arrival_time)) {
315 // Could not add timestamp, feedback packet might be full. Return and
316 // try again with a fresh packet.
317 break;
318 }
319
320 next_sequence_number = seq + 1;
321 }
322 if (is_periodic_update) {
323 periodic_window_start_seq_ = next_sequence_number;
324 }
325 return feedback_packet;
326 }
327
328 } // namespace webrtc
329