1 /*
2 * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/rtp_rtcp/source/rtp_packetizer_av1.h"
12
13 #include <stddef.h>
14 #include <stdint.h>
15
16 #include <initializer_list>
17 #include <utility>
18 #include <vector>
19
20 #include "api/array_view.h"
21 #include "api/scoped_refptr.h"
22 #include "api/video/encoded_image.h"
23 #include "modules/rtp_rtcp/source/rtp_packet_to_send.h"
24 #include "modules/rtp_rtcp/source/rtp_packetizer_av1_test_helper.h"
25 #include "modules/rtp_rtcp/source/video_rtp_depacketizer_av1.h"
26 #include "test/gmock.h"
27 #include "test/gtest.h"
28
29 namespace webrtc {
30 namespace {
31
32 using ::testing::Each;
33 using ::testing::ElementsAre;
34 using ::testing::ElementsAreArray;
35 using ::testing::Le;
36 using ::testing::SizeIs;
37
38 constexpr uint8_t kNewCodedVideoSequenceBit = 0b00'00'1000;
39
40 // Wrapper around rtp_packet to make it look like container of payload bytes.
41 struct RtpPayload {
42 using value_type = rtc::ArrayView<const uint8_t>::value_type;
43 using const_iterator = rtc::ArrayView<const uint8_t>::const_iterator;
44
RtpPayloadwebrtc::__anon148957830111::RtpPayload45 RtpPayload() : rtp_packet(/*extensions=*/nullptr) {}
46 RtpPayload& operator=(RtpPayload&&) = default;
47 RtpPayload(RtpPayload&&) = default;
48
beginwebrtc::__anon148957830111::RtpPayload49 const_iterator begin() const { return rtp_packet.payload().begin(); }
endwebrtc::__anon148957830111::RtpPayload50 const_iterator end() const { return rtp_packet.payload().end(); }
datawebrtc::__anon148957830111::RtpPayload51 const uint8_t* data() const { return rtp_packet.payload().data(); }
sizewebrtc::__anon148957830111::RtpPayload52 size_t size() const { return rtp_packet.payload().size(); }
53
aggregation_headerwebrtc::__anon148957830111::RtpPayload54 uint8_t aggregation_header() const { return rtp_packet.payload()[0]; }
55
56 RtpPacketToSend rtp_packet;
57 };
58
59 // Wrapper around frame pointer to make it look like container of bytes with
60 // nullptr frame look like empty container.
61 class Av1Frame {
62 public:
63 using value_type = uint8_t;
64 using const_iterator = const uint8_t*;
65
Av1Frame(rtc::scoped_refptr<EncodedImageBuffer> frame)66 explicit Av1Frame(rtc::scoped_refptr<EncodedImageBuffer> frame)
67 : frame_(std::move(frame)) {}
68
begin() const69 const_iterator begin() const { return frame_ ? frame_->data() : nullptr; }
end() const70 const_iterator end() const {
71 return frame_ ? (frame_->data() + frame_->size()) : nullptr;
72 }
73
74 private:
75 rtc::scoped_refptr<EncodedImageBuffer> frame_;
76 };
77
Packetize(rtc::ArrayView<const uint8_t> payload,RtpPacketizer::PayloadSizeLimits limits,VideoFrameType frame_type=VideoFrameType::kVideoFrameDelta,bool is_last_frame_in_picture=true)78 std::vector<RtpPayload> Packetize(
79 rtc::ArrayView<const uint8_t> payload,
80 RtpPacketizer::PayloadSizeLimits limits,
81 VideoFrameType frame_type = VideoFrameType::kVideoFrameDelta,
82 bool is_last_frame_in_picture = true) {
83 // Run code under test.
84 RtpPacketizerAv1 packetizer(payload, limits, frame_type,
85 is_last_frame_in_picture);
86 // Convert result into structure that is easier to run expectation against.
87 std::vector<RtpPayload> result(packetizer.NumPackets());
88 for (RtpPayload& rtp_payload : result) {
89 EXPECT_TRUE(packetizer.NextPacket(&rtp_payload.rtp_packet));
90 }
91 return result;
92 }
93
ReassembleFrame(rtc::ArrayView<const RtpPayload> rtp_payloads)94 Av1Frame ReassembleFrame(rtc::ArrayView<const RtpPayload> rtp_payloads) {
95 std::vector<rtc::ArrayView<const uint8_t>> payloads(rtp_payloads.size());
96 for (size_t i = 0; i < rtp_payloads.size(); ++i) {
97 payloads[i] = rtp_payloads[i];
98 }
99 return Av1Frame(VideoRtpDepacketizerAv1().AssembleFrame(payloads));
100 }
101
TEST(RtpPacketizerAv1Test,PacketizeOneObuWithoutSizeAndExtension)102 TEST(RtpPacketizerAv1Test, PacketizeOneObuWithoutSizeAndExtension) {
103 auto kFrame = BuildAv1Frame({Av1Obu(kAv1ObuTypeFrame)
104 .WithoutSize()
105 .WithPayload({1, 2, 3, 4, 5, 6, 7})});
106 EXPECT_THAT(Packetize(kFrame, {}),
107 ElementsAre(ElementsAre(0b00'01'0000, // aggregation header
108 kAv1ObuTypeFrame, 1, 2, 3, 4, 5, 6, 7)));
109 }
110
TEST(RtpPacketizerAv1Test,PacketizeOneObuWithoutSizeWithExtension)111 TEST(RtpPacketizerAv1Test, PacketizeOneObuWithoutSizeWithExtension) {
112 auto kFrame = BuildAv1Frame({Av1Obu(kAv1ObuTypeFrame)
113 .WithoutSize()
114 .WithExtension(kAv1ObuExtensionS1T1)
115 .WithPayload({2, 3, 4, 5, 6, 7})});
116 EXPECT_THAT(
117 Packetize(kFrame, {}),
118 ElementsAre(ElementsAre(0b00'01'0000, // aggregation header
119 kAv1ObuTypeFrame | kAv1ObuExtensionPresentBit,
120 kAv1ObuExtensionS1T1, 2, 3, 4, 5, 6, 7)));
121 }
122
TEST(RtpPacketizerAv1Test,RemovesObuSizeFieldWithoutExtension)123 TEST(RtpPacketizerAv1Test, RemovesObuSizeFieldWithoutExtension) {
124 auto kFrame = BuildAv1Frame(
125 {Av1Obu(kAv1ObuTypeFrame).WithPayload({11, 12, 13, 14, 15, 16, 17})});
126 EXPECT_THAT(
127 Packetize(kFrame, {}),
128 ElementsAre(ElementsAre(0b00'01'0000, // aggregation header
129 kAv1ObuTypeFrame, 11, 12, 13, 14, 15, 16, 17)));
130 }
131
TEST(RtpPacketizerAv1Test,RemovesObuSizeFieldWithExtension)132 TEST(RtpPacketizerAv1Test, RemovesObuSizeFieldWithExtension) {
133 auto kFrame = BuildAv1Frame({Av1Obu(kAv1ObuTypeFrame)
134 .WithExtension(kAv1ObuExtensionS1T1)
135 .WithPayload({1, 2, 3, 4, 5, 6, 7})});
136 EXPECT_THAT(
137 Packetize(kFrame, {}),
138 ElementsAre(ElementsAre(0b00'01'0000, // aggregation header
139 kAv1ObuTypeFrame | kAv1ObuExtensionPresentBit,
140 kAv1ObuExtensionS1T1, 1, 2, 3, 4, 5, 6, 7)));
141 }
142
TEST(RtpPacketizerAv1Test,OmitsSizeForLastObuWhenThreeObusFitsIntoThePacket)143 TEST(RtpPacketizerAv1Test, OmitsSizeForLastObuWhenThreeObusFitsIntoThePacket) {
144 auto kFrame = BuildAv1Frame(
145 {Av1Obu(kAv1ObuTypeSequenceHeader).WithPayload({1, 2, 3, 4, 5, 6}),
146 Av1Obu(kAv1ObuTypeMetadata).WithPayload({11, 12, 13, 14}),
147 Av1Obu(kAv1ObuTypeFrame).WithPayload({21, 22, 23, 24, 25, 26})});
148 EXPECT_THAT(Packetize(kFrame, {}),
149 ElementsAre(ElementsAre(
150 0b00'11'0000, // aggregation header
151 7, kAv1ObuTypeSequenceHeader, 1, 2, 3, 4, 5, 6, //
152 5, kAv1ObuTypeMetadata, 11, 12, 13, 14, //
153 kAv1ObuTypeFrame, 21, 22, 23, 24, 25, 26)));
154 }
155
TEST(RtpPacketizerAv1Test,UseSizeForAllObusWhenFourObusFitsIntoThePacket)156 TEST(RtpPacketizerAv1Test, UseSizeForAllObusWhenFourObusFitsIntoThePacket) {
157 auto kFrame = BuildAv1Frame(
158 {Av1Obu(kAv1ObuTypeSequenceHeader).WithPayload({1, 2, 3, 4, 5, 6}),
159 Av1Obu(kAv1ObuTypeMetadata).WithPayload({11, 12, 13, 14}),
160 Av1Obu(kAv1ObuTypeFrameHeader).WithPayload({21, 22, 23}),
161 Av1Obu(kAv1ObuTypeTileGroup).WithPayload({31, 32, 33, 34, 35, 36})});
162 EXPECT_THAT(Packetize(kFrame, {}),
163 ElementsAre(ElementsAre(
164 0b00'00'0000, // aggregation header
165 7, kAv1ObuTypeSequenceHeader, 1, 2, 3, 4, 5, 6, //
166 5, kAv1ObuTypeMetadata, 11, 12, 13, 14, //
167 4, kAv1ObuTypeFrameHeader, 21, 22, 23, //
168 7, kAv1ObuTypeTileGroup, 31, 32, 33, 34, 35, 36)));
169 }
170
TEST(RtpPacketizerAv1Test,DiscardsTemporalDelimiterAndTileListObu)171 TEST(RtpPacketizerAv1Test, DiscardsTemporalDelimiterAndTileListObu) {
172 auto kFrame = BuildAv1Frame(
173 {Av1Obu(kAv1ObuTypeTemporalDelimiter), Av1Obu(kAv1ObuTypeMetadata),
174 Av1Obu(kAv1ObuTypeTileList).WithPayload({1, 2, 3, 4, 5, 6}),
175 Av1Obu(kAv1ObuTypeFrameHeader).WithPayload({21, 22, 23}),
176 Av1Obu(kAv1ObuTypeTileGroup).WithPayload({31, 32, 33, 34, 35, 36})});
177
178 EXPECT_THAT(
179 Packetize(kFrame, {}),
180 ElementsAre(ElementsAre(0b00'11'0000, // aggregation header
181 1,
182 kAv1ObuTypeMetadata, //
183 4, kAv1ObuTypeFrameHeader, 21, 22,
184 23, //
185 kAv1ObuTypeTileGroup, 31, 32, 33, 34, 35, 36)));
186 }
187
TEST(RtpPacketizerAv1Test,SplitTwoObusIntoTwoPacketForceSplitObuHeader)188 TEST(RtpPacketizerAv1Test, SplitTwoObusIntoTwoPacketForceSplitObuHeader) {
189 // Craft expected payloads so that there is only one way to split original
190 // frame into two packets.
191 const uint8_t kExpectPayload1[6] = {
192 0b01'10'0000, // aggregation_header
193 3,
194 kAv1ObuTypeFrameHeader | kAv1ObuExtensionPresentBit,
195 kAv1ObuExtensionS1T1,
196 21, //
197 kAv1ObuTypeTileGroup | kAv1ObuExtensionPresentBit};
198 const uint8_t kExpectPayload2[6] = {0b10'01'0000, // aggregation_header
199 kAv1ObuExtensionS1T1, 11, 12, 13, 14};
200 auto kFrame = BuildAv1Frame({Av1Obu(kAv1ObuTypeFrameHeader)
201 .WithExtension(kAv1ObuExtensionS1T1)
202 .WithPayload({21}),
203 Av1Obu(kAv1ObuTypeTileGroup)
204 .WithExtension(kAv1ObuExtensionS1T1)
205 .WithPayload({11, 12, 13, 14})});
206
207 RtpPacketizer::PayloadSizeLimits limits;
208 limits.max_payload_len = 6;
209 auto payloads = Packetize(kFrame, limits);
210 EXPECT_THAT(payloads, ElementsAre(ElementsAreArray(kExpectPayload1),
211 ElementsAreArray(kExpectPayload2)));
212 }
213
TEST(RtpPacketizerAv1Test,SetsNbitAtTheFirstPacketOfAKeyFrameWithSequenceHeader)214 TEST(RtpPacketizerAv1Test,
215 SetsNbitAtTheFirstPacketOfAKeyFrameWithSequenceHeader) {
216 auto kFrame = BuildAv1Frame(
217 {Av1Obu(kAv1ObuTypeSequenceHeader).WithPayload({1, 2, 3, 4, 5, 6, 7})});
218 RtpPacketizer::PayloadSizeLimits limits;
219 limits.max_payload_len = 6;
220 auto packets = Packetize(kFrame, limits, VideoFrameType::kVideoFrameKey);
221 ASSERT_THAT(packets, SizeIs(2));
222 EXPECT_TRUE(packets[0].aggregation_header() & kNewCodedVideoSequenceBit);
223 EXPECT_FALSE(packets[1].aggregation_header() & kNewCodedVideoSequenceBit);
224 }
225
TEST(RtpPacketizerAv1Test,DoesntSetNbitAtThePacketsOfAKeyFrameWithoutSequenceHeader)226 TEST(RtpPacketizerAv1Test,
227 DoesntSetNbitAtThePacketsOfAKeyFrameWithoutSequenceHeader) {
228 auto kFrame = BuildAv1Frame(
229 {Av1Obu(kAv1ObuTypeFrame).WithPayload({1, 2, 3, 4, 5, 6, 7})});
230 RtpPacketizer::PayloadSizeLimits limits;
231 limits.max_payload_len = 6;
232 auto packets = Packetize(kFrame, limits, VideoFrameType::kVideoFrameKey);
233 ASSERT_THAT(packets, SizeIs(2));
234 EXPECT_FALSE(packets[0].aggregation_header() & kNewCodedVideoSequenceBit);
235 EXPECT_FALSE(packets[1].aggregation_header() & kNewCodedVideoSequenceBit);
236 }
237
TEST(RtpPacketizerAv1Test,DoesntSetNbitAtThePacketsOfADeltaFrame)238 TEST(RtpPacketizerAv1Test, DoesntSetNbitAtThePacketsOfADeltaFrame) {
239 // Even when that delta frame starts with a (redundant) sequence header.
240 auto kFrame = BuildAv1Frame(
241 {Av1Obu(kAv1ObuTypeSequenceHeader).WithPayload({1, 2, 3, 4, 5, 6, 7})});
242 RtpPacketizer::PayloadSizeLimits limits;
243 limits.max_payload_len = 6;
244 auto packets = Packetize(kFrame, limits, VideoFrameType::kVideoFrameDelta);
245 ASSERT_THAT(packets, SizeIs(2));
246 EXPECT_FALSE(packets[0].aggregation_header() & kNewCodedVideoSequenceBit);
247 EXPECT_FALSE(packets[1].aggregation_header() & kNewCodedVideoSequenceBit);
248 }
249
250 // There are multiple valid reasonable ways to split payload into multiple
251 // packets, do not validate current choice, instead use RtpDepacketizer
252 // to validate frame is reconstracted to the same one. Note: since
253 // RtpDepacketizer always inserts obu_size fields in the output, use frame where
254 // each obu has obu_size fields for more streight forward validation.
TEST(RtpPacketizerAv1Test,SplitSingleObuIntoTwoPackets)255 TEST(RtpPacketizerAv1Test, SplitSingleObuIntoTwoPackets) {
256 auto kFrame =
257 BuildAv1Frame({Av1Obu(kAv1ObuTypeFrame)
258 .WithPayload({11, 12, 13, 14, 15, 16, 17, 18, 19})});
259
260 RtpPacketizer::PayloadSizeLimits limits;
261 limits.max_payload_len = 8;
262 auto payloads = Packetize(kFrame, limits);
263 EXPECT_THAT(payloads, ElementsAre(SizeIs(Le(8u)), SizeIs(Le(8u))));
264
265 // Use RtpDepacketizer to validate the split.
266 EXPECT_THAT(ReassembleFrame(payloads), ElementsAreArray(kFrame));
267 }
268
TEST(RtpPacketizerAv1Test,SplitSingleObuIntoManyPackets)269 TEST(RtpPacketizerAv1Test, SplitSingleObuIntoManyPackets) {
270 auto kFrame = BuildAv1Frame(
271 {Av1Obu(kAv1ObuTypeFrame).WithPayload(std::vector<uint8_t>(1200, 27))});
272
273 RtpPacketizer::PayloadSizeLimits limits;
274 limits.max_payload_len = 100;
275 auto payloads = Packetize(kFrame, limits);
276 EXPECT_THAT(payloads, SizeIs(13u));
277 EXPECT_THAT(payloads, Each(SizeIs(Le(100u))));
278
279 // Use RtpDepacketizer to validate the split.
280 EXPECT_THAT(ReassembleFrame(payloads), ElementsAreArray(kFrame));
281 }
282
TEST(RtpPacketizerAv1Test,SetMarkerBitForLastPacketInEndOfPictureFrame)283 TEST(RtpPacketizerAv1Test, SetMarkerBitForLastPacketInEndOfPictureFrame) {
284 auto kFrame = BuildAv1Frame(
285 {Av1Obu(kAv1ObuTypeFrame).WithPayload(std::vector<uint8_t>(200, 27))});
286
287 RtpPacketizer::PayloadSizeLimits limits;
288 limits.max_payload_len = 100;
289 auto payloads = Packetize(kFrame, limits, VideoFrameType::kVideoFrameDelta,
290 /*is_last_frame_in_picture=*/true);
291 ASSERT_THAT(payloads, SizeIs(3u));
292 EXPECT_FALSE(payloads[0].rtp_packet.Marker());
293 EXPECT_FALSE(payloads[1].rtp_packet.Marker());
294 EXPECT_TRUE(payloads[2].rtp_packet.Marker());
295 }
296
TEST(RtpPacketizerAv1Test,DoesntSetMarkerBitForPacketsNotInEndOfPictureFrame)297 TEST(RtpPacketizerAv1Test, DoesntSetMarkerBitForPacketsNotInEndOfPictureFrame) {
298 auto kFrame = BuildAv1Frame(
299 {Av1Obu(kAv1ObuTypeFrame).WithPayload(std::vector<uint8_t>(200, 27))});
300
301 RtpPacketizer::PayloadSizeLimits limits;
302 limits.max_payload_len = 100;
303 auto payloads = Packetize(kFrame, limits, VideoFrameType::kVideoFrameDelta,
304 /*is_last_frame_in_picture=*/false);
305 ASSERT_THAT(payloads, SizeIs(3u));
306 EXPECT_FALSE(payloads[0].rtp_packet.Marker());
307 EXPECT_FALSE(payloads[1].rtp_packet.Marker());
308 EXPECT_FALSE(payloads[2].rtp_packet.Marker());
309 }
310
TEST(RtpPacketizerAv1Test,SplitTwoObusIntoTwoPackets)311 TEST(RtpPacketizerAv1Test, SplitTwoObusIntoTwoPackets) {
312 // 2nd OBU is too large to fit into one packet, so its head would be in the
313 // same packet as the 1st OBU.
314 auto kFrame = BuildAv1Frame(
315 {Av1Obu(kAv1ObuTypeSequenceHeader).WithPayload({11, 12}),
316 Av1Obu(kAv1ObuTypeFrame).WithPayload({1, 2, 3, 4, 5, 6, 7, 8, 9})});
317
318 RtpPacketizer::PayloadSizeLimits limits;
319 limits.max_payload_len = 8;
320 auto payloads = Packetize(kFrame, limits);
321 EXPECT_THAT(payloads, ElementsAre(SizeIs(Le(8u)), SizeIs(Le(8u))));
322
323 // Use RtpDepacketizer to validate the split.
324 EXPECT_THAT(ReassembleFrame(payloads), ElementsAreArray(kFrame));
325 }
326
TEST(RtpPacketizerAv1Test,SplitSingleObuIntoTwoPacketsBecauseOfSinglePacketLimit)327 TEST(RtpPacketizerAv1Test,
328 SplitSingleObuIntoTwoPacketsBecauseOfSinglePacketLimit) {
329 auto kFrame =
330 BuildAv1Frame({Av1Obu(kAv1ObuTypeFrame)
331 .WithPayload({11, 12, 13, 14, 15, 16, 17, 18, 19})});
332 RtpPacketizer::PayloadSizeLimits limits;
333 limits.max_payload_len = 10;
334 limits.single_packet_reduction_len = 8;
335 auto payloads = Packetize(kFrame, limits);
336 EXPECT_THAT(payloads, ElementsAre(SizeIs(Le(10u)), SizeIs(Le(10u))));
337
338 EXPECT_THAT(ReassembleFrame(payloads), ElementsAreArray(kFrame));
339 }
340
341 } // namespace
342 } // namespace webrtc
343