1 /*
2  *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 #include "modules/video_coding/svc/scalability_structure_l2t2_key_shift.h"
11 
12 #include <vector>
13 
14 #include "api/array_view.h"
15 #include "api/transport/rtp/dependency_descriptor.h"
16 #include "common_video/generic_frame_descriptor/generic_frame_info.h"
17 #include "modules/video_coding/svc/scalability_structure_test_helpers.h"
18 #include "test/gmock.h"
19 #include "test/gtest.h"
20 
21 namespace webrtc {
22 namespace {
23 
24 using ::testing::ElementsAre;
25 using ::testing::IsEmpty;
26 using ::testing::SizeIs;
27 
28 // S1T1     3   7
29 //         /   /
30 // S1T0   1---5---9
31 //        |
32 // S0T1   |   4   8
33 //        |  /   /
34 // S0T0   0-2---6
35 // Time-> 0 1 2 3 4
TEST(ScalabilityStructureL2T2KeyShiftTest,DecodeTargetsAreEnabledByDefault)36 TEST(ScalabilityStructureL2T2KeyShiftTest, DecodeTargetsAreEnabledByDefault) {
37   ScalabilityStructureL2T2KeyShift structure;
38   ScalabilityStructureWrapper wrapper(structure);
39   std::vector<GenericFrameInfo> frames;
40   wrapper.GenerateFrames(/*num_temporal_units=*/5, frames);
41   ASSERT_THAT(frames, SizeIs(10));
42 
43   EXPECT_EQ(frames[0].spatial_id, 0);
44   EXPECT_EQ(frames[1].spatial_id, 1);
45   EXPECT_EQ(frames[2].spatial_id, 0);
46   EXPECT_EQ(frames[3].spatial_id, 1);
47   EXPECT_EQ(frames[4].spatial_id, 0);
48   EXPECT_EQ(frames[5].spatial_id, 1);
49   EXPECT_EQ(frames[6].spatial_id, 0);
50   EXPECT_EQ(frames[7].spatial_id, 1);
51   EXPECT_EQ(frames[8].spatial_id, 0);
52   EXPECT_EQ(frames[9].spatial_id, 1);
53 
54   // spatial_id = 0 has the temporal shift.
55   EXPECT_EQ(frames[0].temporal_id, 0);
56   EXPECT_EQ(frames[2].temporal_id, 0);
57   EXPECT_EQ(frames[4].temporal_id, 1);
58   EXPECT_EQ(frames[6].temporal_id, 0);
59   EXPECT_EQ(frames[8].temporal_id, 1);
60 
61   // spatial_id = 1 hasn't temporal shift.
62   EXPECT_EQ(frames[1].temporal_id, 0);
63   EXPECT_EQ(frames[3].temporal_id, 1);
64   EXPECT_EQ(frames[5].temporal_id, 0);
65   EXPECT_EQ(frames[7].temporal_id, 1);
66   EXPECT_EQ(frames[9].temporal_id, 0);
67 
68   // Key frame diff.
69   EXPECT_THAT(frames[0].frame_diffs, IsEmpty());
70   EXPECT_THAT(frames[1].frame_diffs, ElementsAre(1));
71   // S0T0 frame diffs
72   EXPECT_THAT(frames[2].frame_diffs, ElementsAre(2));
73   EXPECT_THAT(frames[6].frame_diffs, ElementsAre(4));
74   // S1T0 frame diffs
75   EXPECT_THAT(frames[5].frame_diffs, ElementsAre(4));
76   EXPECT_THAT(frames[9].frame_diffs, ElementsAre(4));
77   // T1 frames refer T0 frame of same spatial layer which is 2 frame ids away.
78   EXPECT_THAT(frames[3].frame_diffs, ElementsAre(2));
79   EXPECT_THAT(frames[4].frame_diffs, ElementsAre(2));
80   EXPECT_THAT(frames[7].frame_diffs, ElementsAre(2));
81   EXPECT_THAT(frames[8].frame_diffs, ElementsAre(2));
82 }
83 
84 // S1T0   1---4---7
85 //        |
86 // S0T1   |   3   6
87 //        |  /   /
88 // S0T0   0-2---5--
89 // Time-> 0 1 2 3 4
TEST(ScalabilityStructureL2T2KeyShiftTest,DisableS1T1Layer)90 TEST(ScalabilityStructureL2T2KeyShiftTest, DisableS1T1Layer) {
91   ScalabilityStructureL2T2KeyShift structure;
92   structure.OnRatesUpdated(EnableTemporalLayers(/*s0=*/2, /*s1=*/1));
93   ScalabilityStructureWrapper wrapper(structure);
94   std::vector<GenericFrameInfo> frames;
95   wrapper.GenerateFrames(/*num_temporal_units=*/5, frames);
96   ASSERT_THAT(frames, SizeIs(8));
97 
98   EXPECT_EQ(frames[0].spatial_id, 0);
99   EXPECT_EQ(frames[1].spatial_id, 1);
100   EXPECT_EQ(frames[2].spatial_id, 0);
101   EXPECT_EQ(frames[3].spatial_id, 0);
102   EXPECT_EQ(frames[4].spatial_id, 1);
103   EXPECT_EQ(frames[5].spatial_id, 0);
104   EXPECT_EQ(frames[6].spatial_id, 0);
105   EXPECT_EQ(frames[7].spatial_id, 1);
106 
107   // spatial_id = 0 has the temporal shift.
108   EXPECT_EQ(frames[0].temporal_id, 0);
109   EXPECT_EQ(frames[2].temporal_id, 0);
110   EXPECT_EQ(frames[3].temporal_id, 1);
111   EXPECT_EQ(frames[5].temporal_id, 0);
112   EXPECT_EQ(frames[6].temporal_id, 1);
113 
114   // spatial_id = 1 has single temporal layer.
115   EXPECT_EQ(frames[1].temporal_id, 0);
116   EXPECT_EQ(frames[4].temporal_id, 0);
117   EXPECT_EQ(frames[5].temporal_id, 0);
118 }
119 
120 // S1T1     3  |
121 //         /   |
122 // S1T0   1---5+--7
123 //        |    |
124 // S0T1   |   4|
125 //        |  / |
126 // S0T0   0-2--+6---8
127 // Time-> 0 1 2 3 4 5
TEST(ScalabilityStructureL2T2KeyShiftTest,DisableT1LayersAfterFewFrames)128 TEST(ScalabilityStructureL2T2KeyShiftTest, DisableT1LayersAfterFewFrames) {
129   ScalabilityStructureL2T2KeyShift structure;
130   ScalabilityStructureWrapper wrapper(structure);
131   std::vector<GenericFrameInfo> frames;
132 
133   wrapper.GenerateFrames(/*num_temporal_units=*/3, frames);
134   EXPECT_THAT(frames, SizeIs(6));
135   structure.OnRatesUpdated(EnableTemporalLayers(/*s0=*/1, /*s1=*/1));
136   wrapper.GenerateFrames(/*num_temporal_units=*/3, frames);
137   ASSERT_THAT(frames, SizeIs(9));
138 
139   // Skip validation before T1 was disabled as that is covered by the test
140   // where no layers are disabled.
141   EXPECT_EQ(frames[6].spatial_id, 0);
142   EXPECT_EQ(frames[7].spatial_id, 1);
143   EXPECT_EQ(frames[8].spatial_id, 0);
144 
145   EXPECT_EQ(frames[6].temporal_id, 0);
146   EXPECT_EQ(frames[7].temporal_id, 0);
147   EXPECT_EQ(frames[8].temporal_id, 0);
148 
149   EXPECT_TRUE(wrapper.FrameReferencesAreValid(frames));
150 }
151 
152 // S1T1     1   3
153 //         /   /
154 // S1T0   0---2
155 // Time-> 0 1 2 3 4 5
TEST(ScalabilityStructureL2T2KeyShiftTest,DisableS0FromTheStart)156 TEST(ScalabilityStructureL2T2KeyShiftTest, DisableS0FromTheStart) {
157   ScalabilityStructureL2T2KeyShift structure;
158   ScalabilityStructureWrapper wrapper(structure);
159   std::vector<GenericFrameInfo> frames;
160 
161   structure.OnRatesUpdated(EnableTemporalLayers(/*s0=*/0, /*s1=*/2));
162   wrapper.GenerateFrames(/*num_temporal_units=*/4, frames);
163   EXPECT_THAT(frames, SizeIs(4));
164 
165   EXPECT_EQ(frames[0].spatial_id, 1);
166   EXPECT_EQ(frames[1].spatial_id, 1);
167   EXPECT_EQ(frames[2].spatial_id, 1);
168   EXPECT_EQ(frames[3].spatial_id, 1);
169 
170   EXPECT_EQ(frames[0].temporal_id, 0);
171   EXPECT_EQ(frames[1].temporal_id, 1);
172   EXPECT_EQ(frames[2].temporal_id, 0);
173   EXPECT_EQ(frames[3].temporal_id, 1);
174 
175   EXPECT_TRUE(wrapper.FrameReferencesAreValid(frames));
176 }
177 
178 // S1T1     3  |6   8
179 //         /   /   /
180 // S1T0   1---5+--7
181 //        |    |
182 // S0T1   |   4|
183 //        |  / |
184 // S0T0   0-2  |
185 // Time-> 0 1 2 3 4 5
TEST(ScalabilityStructureL2T2KeyShiftTest,DisableS0AfterFewFrames)186 TEST(ScalabilityStructureL2T2KeyShiftTest, DisableS0AfterFewFrames) {
187   ScalabilityStructureL2T2KeyShift structure;
188   ScalabilityStructureWrapper wrapper(structure);
189   std::vector<GenericFrameInfo> frames;
190 
191   wrapper.GenerateFrames(/*num_temporal_units=*/3, frames);
192   EXPECT_THAT(frames, SizeIs(6));
193   structure.OnRatesUpdated(EnableTemporalLayers(/*s0=*/0, /*s1=*/2));
194   wrapper.GenerateFrames(/*num_temporal_units=*/3, frames);
195   ASSERT_THAT(frames, SizeIs(9));
196 
197   // Expect frame[6] is delta frame.
198   EXPECT_THAT(frames[6].frame_diffs, ElementsAre(1));
199   // Skip validation before S0 was disabled as that should be covered by
200   // test where no layers are disabled.
201   EXPECT_EQ(frames[6].spatial_id, 1);
202   EXPECT_EQ(frames[7].spatial_id, 1);
203   EXPECT_EQ(frames[8].spatial_id, 1);
204 
205   EXPECT_EQ(frames[6].temporal_id, 1);
206   EXPECT_EQ(frames[7].temporal_id, 0);
207   EXPECT_EQ(frames[8].temporal_id, 1);
208 
209   EXPECT_TRUE(wrapper.FrameReferencesAreValid(frames));
210 }
211 
212 // S1T1     3| |  8
213 //         / | | /
214 // S1T0   1  | |6
215 //        |  | ||
216 // S0T1   |  |4||
217 //        |  / ||
218 // S0T0   0-2| |5-7
219 // Time-> 0 1 2 3 4 5
TEST(ScalabilityStructureL2T2KeyShiftTest,ReenableS1TriggersKeyFrame)220 TEST(ScalabilityStructureL2T2KeyShiftTest, ReenableS1TriggersKeyFrame) {
221   ScalabilityStructureL2T2KeyShift structure;
222   ScalabilityStructureWrapper wrapper(structure);
223   std::vector<GenericFrameInfo> frames;
224 
225   wrapper.GenerateFrames(/*num_temporal_units=*/2, frames);
226   EXPECT_THAT(frames, SizeIs(4));
227 
228   structure.OnRatesUpdated(EnableTemporalLayers(/*s0=*/2, /*s1=*/0));
229   wrapper.GenerateFrames(/*num_temporal_units=*/1, frames);
230   EXPECT_THAT(frames, SizeIs(5));
231 
232   structure.OnRatesUpdated(EnableTemporalLayers(/*s0=*/2, /*s1=*/2));
233   wrapper.GenerateFrames(/*num_temporal_units=*/2, frames);
234   ASSERT_THAT(frames, SizeIs(9));
235 
236   EXPECT_THAT(frames[4].spatial_id, 0);
237   EXPECT_THAT(frames[4].temporal_id, 1);
238 
239   // Expect frame[5] to be a key frame.
240   EXPECT_TRUE(wrapper.FrameReferencesAreValid(
241       rtc::MakeArrayView(frames.data() + 5, 4)));
242 
243   EXPECT_THAT(frames[5].spatial_id, 0);
244   EXPECT_THAT(frames[6].spatial_id, 1);
245   EXPECT_THAT(frames[7].spatial_id, 0);
246   EXPECT_THAT(frames[8].spatial_id, 1);
247 
248   // S0 should do temporal shift after the key frame.
249   EXPECT_THAT(frames[5].temporal_id, 0);
250   EXPECT_THAT(frames[7].temporal_id, 0);
251 
252   // No temporal shift for the top spatial layer.
253   EXPECT_THAT(frames[6].temporal_id, 0);
254   EXPECT_THAT(frames[8].temporal_id, 1);
255 }
256 
TEST(ScalabilityStructureL2T2KeyShiftTest,EnableOnlyS0T0FromTheStart)257 TEST(ScalabilityStructureL2T2KeyShiftTest, EnableOnlyS0T0FromTheStart) {
258   ScalabilityStructureL2T2KeyShift structure;
259   ScalabilityStructureWrapper wrapper(structure);
260   std::vector<GenericFrameInfo> frames;
261 
262   structure.OnRatesUpdated(EnableTemporalLayers(/*s0=*/1, /*s1=*/0));
263   wrapper.GenerateFrames(/*num_temporal_units=*/3, frames);
264   ASSERT_THAT(frames, SizeIs(3));
265 
266   EXPECT_EQ(frames[0].spatial_id, 0);
267   EXPECT_EQ(frames[1].spatial_id, 0);
268   EXPECT_EQ(frames[2].spatial_id, 0);
269 
270   EXPECT_EQ(frames[0].temporal_id, 0);
271   EXPECT_EQ(frames[1].temporal_id, 0);
272   EXPECT_EQ(frames[2].temporal_id, 0);
273 
274   EXPECT_TRUE(wrapper.FrameReferencesAreValid(frames));
275 }
276 
277 // S1T1     3|
278 //         / |
279 // S1T0   1  |
280 //        |  |
281 // S0T1   |  |
282 //        |  |
283 // S0T0   0-2+4-5-6
284 // Time-> 0 1 2 3 4
TEST(ScalabilityStructureL2T2KeyShiftTest,EnableOnlyS0T0AfterFewFrames)285 TEST(ScalabilityStructureL2T2KeyShiftTest, EnableOnlyS0T0AfterFewFrames) {
286   ScalabilityStructureL2T2KeyShift structure;
287   ScalabilityStructureWrapper wrapper(structure);
288   std::vector<GenericFrameInfo> frames;
289 
290   wrapper.GenerateFrames(/*num_temporal_units=*/2, frames);
291   EXPECT_THAT(frames, SizeIs(4));
292   structure.OnRatesUpdated(EnableTemporalLayers(/*s0=*/1, /*s1=*/0));
293   wrapper.GenerateFrames(/*num_temporal_units=*/3, frames);
294   ASSERT_THAT(frames, SizeIs(7));
295 
296   EXPECT_EQ(frames[4].spatial_id, 0);
297   EXPECT_EQ(frames[5].spatial_id, 0);
298   EXPECT_EQ(frames[6].spatial_id, 0);
299 
300   EXPECT_EQ(frames[4].temporal_id, 0);
301   EXPECT_EQ(frames[5].temporal_id, 0);
302   EXPECT_EQ(frames[6].temporal_id, 0);
303 
304   EXPECT_TRUE(wrapper.FrameReferencesAreValid(frames));
305 }
306 
TEST(ScalabilityStructureL2T2KeyShiftTest,EnableOnlyS1T0FromTheStart)307 TEST(ScalabilityStructureL2T2KeyShiftTest, EnableOnlyS1T0FromTheStart) {
308   ScalabilityStructureL2T2KeyShift structure;
309   ScalabilityStructureWrapper wrapper(structure);
310   std::vector<GenericFrameInfo> frames;
311 
312   structure.OnRatesUpdated(EnableTemporalLayers(/*s0=*/0, /*s1=*/1));
313   wrapper.GenerateFrames(/*num_temporal_units=*/3, frames);
314   ASSERT_THAT(frames, SizeIs(3));
315 
316   EXPECT_EQ(frames[0].spatial_id, 1);
317   EXPECT_EQ(frames[1].spatial_id, 1);
318   EXPECT_EQ(frames[2].spatial_id, 1);
319 
320   EXPECT_EQ(frames[0].temporal_id, 0);
321   EXPECT_EQ(frames[1].temporal_id, 0);
322   EXPECT_EQ(frames[2].temporal_id, 0);
323 
324   EXPECT_TRUE(wrapper.FrameReferencesAreValid(frames));
325 }
326 
327 // S1T1     3|
328 //         / |
329 // S1T0   1--+4-5-6
330 //        |  |
331 // S0T1   |  |
332 //        |  |
333 // S0T0   0-2|
334 // Time-> 0 1 2 3 4
TEST(ScalabilityStructureL2T2KeyShiftTest,EnableOnlyS1T0AfterFewFrames)335 TEST(ScalabilityStructureL2T2KeyShiftTest, EnableOnlyS1T0AfterFewFrames) {
336   ScalabilityStructureL2T2KeyShift structure;
337   ScalabilityStructureWrapper wrapper(structure);
338   std::vector<GenericFrameInfo> frames;
339 
340   wrapper.GenerateFrames(/*num_temporal_units=*/2, frames);
341   EXPECT_THAT(frames, SizeIs(4));
342   structure.OnRatesUpdated(EnableTemporalLayers(/*s0=*/0, /*s1=*/1));
343   wrapper.GenerateFrames(/*num_temporal_units=*/3, frames);
344   ASSERT_THAT(frames, SizeIs(7));
345 
346   EXPECT_EQ(frames[4].spatial_id, 1);
347   EXPECT_EQ(frames[5].spatial_id, 1);
348   EXPECT_EQ(frames[6].spatial_id, 1);
349 
350   EXPECT_EQ(frames[4].temporal_id, 0);
351   EXPECT_EQ(frames[5].temporal_id, 0);
352   EXPECT_EQ(frames[6].temporal_id, 0);
353 
354   EXPECT_TRUE(wrapper.FrameReferencesAreValid(frames));
355 }
356 
357 }  // namespace
358 }  // namespace webrtc
359