1 /*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include <cstring>
12 #include <limits>
13 #include <map>
14 #include <set>
15 #include <utility>
16 #include <vector>
17
18 #include "modules/video_coding/frame_object.h"
19 #include "modules/video_coding/packet_buffer.h"
20 #include "modules/video_coding/rtp_frame_reference_finder.h"
21 #include "rtc_base/random.h"
22 #include "rtc_base/ref_count.h"
23 #include "system_wrappers/include/clock.h"
24 #include "test/gtest.h"
25
26 namespace webrtc {
27
28 namespace {
CreateFrame(uint16_t seq_num_start,uint16_t seq_num_end,bool keyframe,VideoCodecType codec,const RTPVideoTypeHeader & video_type_header)29 std::unique_ptr<RtpFrameObject> CreateFrame(
30 uint16_t seq_num_start,
31 uint16_t seq_num_end,
32 bool keyframe,
33 VideoCodecType codec,
34 const RTPVideoTypeHeader& video_type_header) {
35 RTPVideoHeader video_header;
36 video_header.frame_type = keyframe ? VideoFrameType::kVideoFrameKey
37 : VideoFrameType::kVideoFrameDelta;
38 video_header.video_type_header = video_type_header;
39
40 // clang-format off
41 return std::make_unique<RtpFrameObject>(
42 seq_num_start,
43 seq_num_end,
44 /*markerBit=*/true,
45 /*times_nacked=*/0,
46 /*first_packet_received_time=*/0,
47 /*last_packet_received_time=*/0,
48 /*rtp_timestamp=*/0,
49 /*ntp_time_ms=*/0,
50 VideoSendTiming(),
51 /*payload_type=*/0,
52 codec,
53 kVideoRotation_0,
54 VideoContentType::UNSPECIFIED,
55 video_header,
56 /*color_space=*/absl::nullopt,
57 RtpPacketInfos(),
58 EncodedImageBuffer::Create(/*size=*/0));
59 // clang-format on
60 }
61 } // namespace
62
63 class TestRtpFrameReferenceFinder : public ::testing::Test {
64 protected:
TestRtpFrameReferenceFinder()65 TestRtpFrameReferenceFinder()
66 : rand_(0x8739211),
67 reference_finder_(std::make_unique<RtpFrameReferenceFinder>()),
68 frames_from_callback_(FrameComp()) {}
69
Rand()70 uint16_t Rand() { return rand_.Rand<uint16_t>(); }
71
OnCompleteFrames(RtpFrameReferenceFinder::ReturnVector frames)72 void OnCompleteFrames(RtpFrameReferenceFinder::ReturnVector frames) {
73 for (auto& frame : frames) {
74 int64_t pid = frame->Id();
75 uint16_t sidx = *frame->SpatialIndex();
76 auto frame_it = frames_from_callback_.find(std::make_pair(pid, sidx));
77 if (frame_it != frames_from_callback_.end()) {
78 ADD_FAILURE() << "Already received frame with (pid:sidx): (" << pid
79 << ":" << sidx << ")";
80 return;
81 }
82
83 frames_from_callback_.insert(
84 std::make_pair(std::make_pair(pid, sidx), std::move(frame)));
85 }
86 }
87
InsertGeneric(uint16_t seq_num_start,uint16_t seq_num_end,bool keyframe)88 void InsertGeneric(uint16_t seq_num_start,
89 uint16_t seq_num_end,
90 bool keyframe) {
91 std::unique_ptr<RtpFrameObject> frame =
92 CreateFrame(seq_num_start, seq_num_end, keyframe, kVideoCodecGeneric,
93 RTPVideoTypeHeader());
94
95 OnCompleteFrames(reference_finder_->ManageFrame(std::move(frame)));
96 }
97
InsertH264(uint16_t seq_num_start,uint16_t seq_num_end,bool keyframe)98 void InsertH264(uint16_t seq_num_start, uint16_t seq_num_end, bool keyframe) {
99 std::unique_ptr<RtpFrameObject> frame =
100 CreateFrame(seq_num_start, seq_num_end, keyframe, kVideoCodecH264,
101 RTPVideoTypeHeader());
102 OnCompleteFrames(reference_finder_->ManageFrame(std::move(frame)));
103 }
104
InsertPadding(uint16_t seq_num)105 void InsertPadding(uint16_t seq_num) {
106 OnCompleteFrames(reference_finder_->PaddingReceived(seq_num));
107 }
108
109 // Check if a frame with picture id `pid` and spatial index `sidx` has been
110 // delivered from the packet buffer, and if so, if it has the references
111 // specified by `refs`.
112 template <typename... T>
CheckReferences(int64_t picture_id_offset,uint16_t sidx,T...refs) const113 void CheckReferences(int64_t picture_id_offset,
114 uint16_t sidx,
115 T... refs) const {
116 int64_t pid = picture_id_offset;
117 auto frame_it = frames_from_callback_.find(std::make_pair(pid, sidx));
118 if (frame_it == frames_from_callback_.end()) {
119 ADD_FAILURE() << "Could not find frame with (pid:sidx): (" << pid << ":"
120 << sidx << ")";
121 return;
122 }
123
124 std::set<int64_t> actual_refs;
125 for (uint8_t r = 0; r < frame_it->second->num_references; ++r)
126 actual_refs.insert(frame_it->second->references[r]);
127
128 std::set<int64_t> expected_refs;
129 RefsToSet(&expected_refs, refs...);
130
131 ASSERT_EQ(expected_refs, actual_refs);
132 }
133
134 template <typename... T>
CheckReferencesGeneric(int64_t pid,T...refs) const135 void CheckReferencesGeneric(int64_t pid, T... refs) const {
136 CheckReferences(pid, 0, refs...);
137 }
138
139 template <typename... T>
CheckReferencesH264(int64_t pid,T...refs) const140 void CheckReferencesH264(int64_t pid, T... refs) const {
141 CheckReferences(pid, 0, refs...);
142 }
143
144 template <typename... T>
RefsToSet(std::set<int64_t> * m,int64_t ref,T...refs) const145 void RefsToSet(std::set<int64_t>* m, int64_t ref, T... refs) const {
146 m->insert(ref);
147 RefsToSet(m, refs...);
148 }
149
RefsToSet(std::set<int64_t> * m) const150 void RefsToSet(std::set<int64_t>* m) const {}
151
152 Random rand_;
153 std::unique_ptr<RtpFrameReferenceFinder> reference_finder_;
154 struct FrameComp {
operator ()webrtc::TestRtpFrameReferenceFinder::FrameComp155 bool operator()(const std::pair<int64_t, uint8_t> f1,
156 const std::pair<int64_t, uint8_t> f2) const {
157 if (f1.first == f2.first)
158 return f1.second < f2.second;
159 return f1.first < f2.first;
160 }
161 };
162 std::
163 map<std::pair<int64_t, uint8_t>, std::unique_ptr<EncodedFrame>, FrameComp>
164 frames_from_callback_;
165 };
166
TEST_F(TestRtpFrameReferenceFinder,PaddingPackets)167 TEST_F(TestRtpFrameReferenceFinder, PaddingPackets) {
168 uint16_t sn = Rand();
169
170 InsertGeneric(sn, sn, true);
171 InsertGeneric(sn + 2, sn + 2, false);
172 EXPECT_EQ(1UL, frames_from_callback_.size());
173 InsertPadding(sn + 1);
174 EXPECT_EQ(2UL, frames_from_callback_.size());
175 }
176
TEST_F(TestRtpFrameReferenceFinder,PaddingPacketsReordered)177 TEST_F(TestRtpFrameReferenceFinder, PaddingPacketsReordered) {
178 uint16_t sn = Rand();
179
180 InsertGeneric(sn, sn, true);
181 InsertPadding(sn + 1);
182 InsertPadding(sn + 4);
183 InsertGeneric(sn + 2, sn + 3, false);
184
185 EXPECT_EQ(2UL, frames_from_callback_.size());
186 CheckReferencesGeneric(sn);
187 CheckReferencesGeneric(sn + 3, sn + 0);
188 }
189
TEST_F(TestRtpFrameReferenceFinder,PaddingPacketsReorderedMultipleKeyframes)190 TEST_F(TestRtpFrameReferenceFinder, PaddingPacketsReorderedMultipleKeyframes) {
191 uint16_t sn = Rand();
192
193 InsertGeneric(sn, sn, true);
194 InsertPadding(sn + 1);
195 InsertPadding(sn + 4);
196 InsertGeneric(sn + 2, sn + 3, false);
197 InsertGeneric(sn + 5, sn + 5, true);
198 InsertPadding(sn + 6);
199 InsertPadding(sn + 9);
200 InsertGeneric(sn + 7, sn + 8, false);
201
202 EXPECT_EQ(4UL, frames_from_callback_.size());
203 }
204
TEST_F(TestRtpFrameReferenceFinder,AdvanceSavedKeyframe)205 TEST_F(TestRtpFrameReferenceFinder, AdvanceSavedKeyframe) {
206 uint16_t sn = Rand();
207
208 InsertGeneric(sn, sn, true);
209 InsertGeneric(sn + 1, sn + 1, true);
210 InsertGeneric(sn + 2, sn + 10000, false);
211 InsertGeneric(sn + 10001, sn + 20000, false);
212 InsertGeneric(sn + 20001, sn + 30000, false);
213 InsertGeneric(sn + 30001, sn + 40000, false);
214
215 EXPECT_EQ(6UL, frames_from_callback_.size());
216 }
217
TEST_F(TestRtpFrameReferenceFinder,ClearTo)218 TEST_F(TestRtpFrameReferenceFinder, ClearTo) {
219 uint16_t sn = Rand();
220
221 InsertGeneric(sn, sn + 1, true);
222 InsertGeneric(sn + 4, sn + 5, false); // stashed
223 EXPECT_EQ(1UL, frames_from_callback_.size());
224
225 InsertGeneric(sn + 6, sn + 7, true); // keyframe
226 EXPECT_EQ(2UL, frames_from_callback_.size());
227 reference_finder_->ClearTo(sn + 7);
228
229 InsertGeneric(sn + 8, sn + 9, false); // first frame after keyframe.
230 EXPECT_EQ(3UL, frames_from_callback_.size());
231
232 InsertGeneric(sn + 2, sn + 3, false); // late, cleared past this frame.
233 EXPECT_EQ(3UL, frames_from_callback_.size());
234 }
235
TEST_F(TestRtpFrameReferenceFinder,H264KeyFrameReferences)236 TEST_F(TestRtpFrameReferenceFinder, H264KeyFrameReferences) {
237 uint16_t sn = Rand();
238 InsertH264(sn, sn, true);
239
240 ASSERT_EQ(1UL, frames_from_callback_.size());
241 CheckReferencesH264(sn);
242 }
243
TEST_F(TestRtpFrameReferenceFinder,H264SequenceNumberWrap)244 TEST_F(TestRtpFrameReferenceFinder, H264SequenceNumberWrap) {
245 uint16_t sn = 0xFFFF;
246
247 InsertH264(sn - 1, sn - 1, true);
248 InsertH264(sn, sn, false);
249 InsertH264(sn + 1, sn + 1, false);
250 InsertH264(sn + 2, sn + 2, false);
251
252 ASSERT_EQ(4UL, frames_from_callback_.size());
253 CheckReferencesH264(sn - 1);
254 CheckReferencesH264(sn, sn - 1);
255 CheckReferencesH264(sn + 1, sn);
256 CheckReferencesH264(sn + 2, sn + 1);
257 }
258
TEST_F(TestRtpFrameReferenceFinder,H264Frames)259 TEST_F(TestRtpFrameReferenceFinder, H264Frames) {
260 uint16_t sn = Rand();
261
262 InsertH264(sn, sn, true);
263 InsertH264(sn + 1, sn + 1, false);
264 InsertH264(sn + 2, sn + 2, false);
265 InsertH264(sn + 3, sn + 3, false);
266
267 ASSERT_EQ(4UL, frames_from_callback_.size());
268 CheckReferencesH264(sn);
269 CheckReferencesH264(sn + 1, sn);
270 CheckReferencesH264(sn + 2, sn + 1);
271 CheckReferencesH264(sn + 3, sn + 2);
272 }
273
TEST_F(TestRtpFrameReferenceFinder,H264Reordering)274 TEST_F(TestRtpFrameReferenceFinder, H264Reordering) {
275 uint16_t sn = Rand();
276
277 InsertH264(sn, sn, true);
278 InsertH264(sn + 1, sn + 1, false);
279 InsertH264(sn + 3, sn + 3, false);
280 InsertH264(sn + 2, sn + 2, false);
281 InsertH264(sn + 5, sn + 5, false);
282 InsertH264(sn + 6, sn + 6, false);
283 InsertH264(sn + 4, sn + 4, false);
284
285 ASSERT_EQ(7UL, frames_from_callback_.size());
286 CheckReferencesH264(sn);
287 CheckReferencesH264(sn + 1, sn);
288 CheckReferencesH264(sn + 2, sn + 1);
289 CheckReferencesH264(sn + 3, sn + 2);
290 CheckReferencesH264(sn + 4, sn + 3);
291 CheckReferencesH264(sn + 5, sn + 4);
292 CheckReferencesH264(sn + 6, sn + 5);
293 }
294
TEST_F(TestRtpFrameReferenceFinder,H264SequenceNumberWrapMulti)295 TEST_F(TestRtpFrameReferenceFinder, H264SequenceNumberWrapMulti) {
296 uint16_t sn = 0xFFFF;
297
298 InsertH264(sn - 3, sn - 2, true);
299 InsertH264(sn - 1, sn + 1, false);
300 InsertH264(sn + 2, sn + 3, false);
301 InsertH264(sn + 4, sn + 7, false);
302
303 ASSERT_EQ(4UL, frames_from_callback_.size());
304 CheckReferencesH264(sn - 2);
305 CheckReferencesH264(sn + 1, sn - 2);
306 CheckReferencesH264(sn + 3, sn + 1);
307 CheckReferencesH264(sn + 7, sn + 3);
308 }
309
TEST_F(TestRtpFrameReferenceFinder,Av1FrameNoDependencyDescriptor)310 TEST_F(TestRtpFrameReferenceFinder, Av1FrameNoDependencyDescriptor) {
311 uint16_t sn = 0xFFFF;
312 std::unique_ptr<RtpFrameObject> frame =
313 CreateFrame(/*seq_num_start=*/sn, /*seq_num_end=*/sn, /*keyframe=*/true,
314 kVideoCodecAV1, RTPVideoTypeHeader());
315
316 OnCompleteFrames(reference_finder_->ManageFrame(std::move(frame)));
317
318 ASSERT_EQ(1UL, frames_from_callback_.size());
319 CheckReferencesGeneric(sn);
320 }
321
322 } // namespace webrtc
323