1 /*
2 * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include <stddef.h>
12 #include <stdint.h>
13
14 #include <memory>
15 #include <ostream>
16 #include <string>
17
18 #include "absl/types/optional.h"
19 #include "api/array_view.h"
20 #include "api/transport/rtp/dependency_descriptor.h"
21 #include "modules/video_coding/svc/create_scalability_structure.h"
22 #include "modules/video_coding/svc/scalability_mode_util.h"
23 #include "modules/video_coding/svc/scalability_structure_test_helpers.h"
24 #include "modules/video_coding/svc/scalable_video_controller.h"
25 #include "rtc_base/strings/string_builder.h"
26 #include "test/gmock.h"
27 #include "test/gtest.h"
28
29 namespace webrtc {
30 namespace {
31
32 using ::testing::AllOf;
33 using ::testing::Contains;
34 using ::testing::Each;
35 using ::testing::ElementsAreArray;
36 using ::testing::Field;
37 using ::testing::Ge;
38 using ::testing::IsEmpty;
39 using ::testing::Le;
40 using ::testing::Lt;
41 using ::testing::Not;
42 using ::testing::NotNull;
43 using ::testing::SizeIs;
44 using ::testing::TestWithParam;
45 using ::testing::Values;
46
FrameDependencyTemplateToString(const FrameDependencyTemplate & t)47 std::string FrameDependencyTemplateToString(const FrameDependencyTemplate& t) {
48 rtc::StringBuilder sb;
49 sb << "S" << t.spatial_id << "T" << t.temporal_id;
50 sb << ": dtis = ";
51 for (const auto dtis : t.decode_target_indications) {
52 switch (dtis) {
53 case DecodeTargetIndication::kNotPresent:
54 sb << "-";
55 break;
56 case DecodeTargetIndication::kDiscardable:
57 sb << "D";
58 break;
59 case DecodeTargetIndication::kSwitch:
60 sb << "S";
61 break;
62 case DecodeTargetIndication::kRequired:
63 sb << "R";
64 break;
65 default:
66 sb << "?";
67 break;
68 }
69 }
70 sb << ", frame diffs = { ";
71 for (int d : t.frame_diffs) {
72 sb << d << ", ";
73 }
74 sb << "}, chain diffs = { ";
75 for (int d : t.chain_diffs) {
76 sb << d << ", ";
77 }
78 sb << "}";
79 return sb.Release();
80 }
81
82 struct SvcTestParam {
operator <<(std::ostream & os,const SvcTestParam & param)83 friend std::ostream& operator<<(std::ostream& os, const SvcTestParam& param) {
84 return os << param.name;
85 }
86
GetScalabilityModewebrtc::__anonf9e96bcd0111::SvcTestParam87 ScalabilityMode GetScalabilityMode() const {
88 absl::optional<ScalabilityMode> scalability_mode =
89 ScalabilityModeFromString(name);
90 RTC_CHECK(scalability_mode.has_value());
91 return *scalability_mode;
92 }
93
94 std::string name;
95 int num_temporal_units;
96 };
97
98 class ScalabilityStructureTest : public TestWithParam<SvcTestParam> {};
99
TEST_P(ScalabilityStructureTest,StaticConfigMatchesConfigReturnedByController)100 TEST_P(ScalabilityStructureTest,
101 StaticConfigMatchesConfigReturnedByController) {
102 std::unique_ptr<ScalableVideoController> controller =
103 CreateScalabilityStructure(GetParam().GetScalabilityMode());
104 absl::optional<ScalableVideoController::StreamLayersConfig> static_config =
105 ScalabilityStructureConfig(GetParam().GetScalabilityMode());
106 ASSERT_THAT(controller, NotNull());
107 ASSERT_NE(static_config, absl::nullopt);
108 ScalableVideoController::StreamLayersConfig config =
109 controller->StreamConfig();
110 EXPECT_EQ(config.num_spatial_layers, static_config->num_spatial_layers);
111 EXPECT_EQ(config.num_temporal_layers, static_config->num_temporal_layers);
112 EXPECT_THAT(
113 rtc::MakeArrayView(config.scaling_factor_num, config.num_spatial_layers),
114 ElementsAreArray(static_config->scaling_factor_num,
115 static_config->num_spatial_layers));
116 EXPECT_THAT(
117 rtc::MakeArrayView(config.scaling_factor_den, config.num_spatial_layers),
118 ElementsAreArray(static_config->scaling_factor_den,
119 static_config->num_spatial_layers));
120 }
121
TEST_P(ScalabilityStructureTest,NumberOfDecodeTargetsAndChainsAreInRangeAndConsistent)122 TEST_P(ScalabilityStructureTest,
123 NumberOfDecodeTargetsAndChainsAreInRangeAndConsistent) {
124 FrameDependencyStructure structure =
125 CreateScalabilityStructure(GetParam().GetScalabilityMode())
126 ->DependencyStructure();
127 EXPECT_GT(structure.num_decode_targets, 0);
128 EXPECT_LE(structure.num_decode_targets,
129 DependencyDescriptor::kMaxDecodeTargets);
130 EXPECT_GE(structure.num_chains, 0);
131 EXPECT_LE(structure.num_chains, structure.num_decode_targets);
132 if (structure.num_chains == 0) {
133 EXPECT_THAT(structure.decode_target_protected_by_chain, IsEmpty());
134 } else {
135 EXPECT_THAT(structure.decode_target_protected_by_chain,
136 AllOf(SizeIs(structure.num_decode_targets), Each(Ge(0)),
137 Each(Lt(structure.num_chains))));
138 }
139 EXPECT_THAT(structure.templates,
140 SizeIs(Lt(size_t{DependencyDescriptor::kMaxTemplates})));
141 }
142
TEST_P(ScalabilityStructureTest,TemplatesAreSortedByLayerId)143 TEST_P(ScalabilityStructureTest, TemplatesAreSortedByLayerId) {
144 FrameDependencyStructure structure =
145 CreateScalabilityStructure(GetParam().GetScalabilityMode())
146 ->DependencyStructure();
147 ASSERT_THAT(structure.templates, Not(IsEmpty()));
148 const auto& first_templates = structure.templates.front();
149 EXPECT_EQ(first_templates.spatial_id, 0);
150 EXPECT_EQ(first_templates.temporal_id, 0);
151 for (size_t i = 1; i < structure.templates.size(); ++i) {
152 const auto& prev_template = structure.templates[i - 1];
153 const auto& next_template = structure.templates[i];
154 if (next_template.spatial_id == prev_template.spatial_id &&
155 next_template.temporal_id == prev_template.temporal_id) {
156 // Same layer, next_layer_idc == 0
157 } else if (next_template.spatial_id == prev_template.spatial_id &&
158 next_template.temporal_id == prev_template.temporal_id + 1) {
159 // Next temporal layer, next_layer_idc == 1
160 } else if (next_template.spatial_id == prev_template.spatial_id + 1 &&
161 next_template.temporal_id == 0) {
162 // Next spatial layer, next_layer_idc == 2
163 } else {
164 // everything else is invalid.
165 ADD_FAILURE() << "Invalid templates order. Template #" << i
166 << " with layer (" << next_template.spatial_id << ","
167 << next_template.temporal_id
168 << ") follows template with layer ("
169 << prev_template.spatial_id << ","
170 << prev_template.temporal_id << ").";
171 }
172 }
173 }
174
TEST_P(ScalabilityStructureTest,TemplatesMatchNumberOfDecodeTargetsAndChains)175 TEST_P(ScalabilityStructureTest, TemplatesMatchNumberOfDecodeTargetsAndChains) {
176 FrameDependencyStructure structure =
177 CreateScalabilityStructure(GetParam().GetScalabilityMode())
178 ->DependencyStructure();
179 EXPECT_THAT(
180 structure.templates,
181 Each(AllOf(Field(&FrameDependencyTemplate::decode_target_indications,
182 SizeIs(structure.num_decode_targets)),
183 Field(&FrameDependencyTemplate::chain_diffs,
184 SizeIs(structure.num_chains)))));
185 }
186
TEST_P(ScalabilityStructureTest,FrameInfoMatchesFrameDependencyStructure)187 TEST_P(ScalabilityStructureTest, FrameInfoMatchesFrameDependencyStructure) {
188 std::unique_ptr<ScalableVideoController> svc_controller =
189 CreateScalabilityStructure(GetParam().GetScalabilityMode());
190 FrameDependencyStructure structure = svc_controller->DependencyStructure();
191 std::vector<GenericFrameInfo> frame_infos =
192 ScalabilityStructureWrapper(*svc_controller)
193 .GenerateFrames(GetParam().num_temporal_units);
194 for (size_t frame_id = 0; frame_id < frame_infos.size(); ++frame_id) {
195 const auto& frame = frame_infos[frame_id];
196 EXPECT_GE(frame.spatial_id, 0) << " for frame " << frame_id;
197 EXPECT_GE(frame.temporal_id, 0) << " for frame " << frame_id;
198 EXPECT_THAT(frame.decode_target_indications,
199 SizeIs(structure.num_decode_targets))
200 << " for frame " << frame_id;
201 EXPECT_THAT(frame.part_of_chain, SizeIs(structure.num_chains))
202 << " for frame " << frame_id;
203 }
204 }
205
TEST_P(ScalabilityStructureTest,ThereIsAPerfectTemplateForEachFrame)206 TEST_P(ScalabilityStructureTest, ThereIsAPerfectTemplateForEachFrame) {
207 std::unique_ptr<ScalableVideoController> svc_controller =
208 CreateScalabilityStructure(GetParam().GetScalabilityMode());
209 FrameDependencyStructure structure = svc_controller->DependencyStructure();
210 std::vector<GenericFrameInfo> frame_infos =
211 ScalabilityStructureWrapper(*svc_controller)
212 .GenerateFrames(GetParam().num_temporal_units);
213 for (size_t frame_id = 0; frame_id < frame_infos.size(); ++frame_id) {
214 EXPECT_THAT(structure.templates, Contains(frame_infos[frame_id]))
215 << " for frame " << frame_id << ", Expected "
216 << FrameDependencyTemplateToString(frame_infos[frame_id]);
217 }
218 }
219
TEST_P(ScalabilityStructureTest,FrameDependsOnSameOrLowerLayer)220 TEST_P(ScalabilityStructureTest, FrameDependsOnSameOrLowerLayer) {
221 std::unique_ptr<ScalableVideoController> svc_controller =
222 CreateScalabilityStructure(GetParam().GetScalabilityMode());
223 std::vector<GenericFrameInfo> frame_infos =
224 ScalabilityStructureWrapper(*svc_controller)
225 .GenerateFrames(GetParam().num_temporal_units);
226 int64_t num_frames = frame_infos.size();
227
228 for (int64_t frame_id = 0; frame_id < num_frames; ++frame_id) {
229 const auto& frame = frame_infos[frame_id];
230 for (int frame_diff : frame.frame_diffs) {
231 int64_t base_frame_id = frame_id - frame_diff;
232 const auto& base_frame = frame_infos[base_frame_id];
233 EXPECT_GE(frame.spatial_id, base_frame.spatial_id)
234 << "Frame " << frame_id << " depends on frame " << base_frame_id;
235 EXPECT_GE(frame.temporal_id, base_frame.temporal_id)
236 << "Frame " << frame_id << " depends on frame " << base_frame_id;
237 }
238 }
239 }
240
TEST_P(ScalabilityStructureTest,NoFrameDependsOnDiscardableOrNotPresent)241 TEST_P(ScalabilityStructureTest, NoFrameDependsOnDiscardableOrNotPresent) {
242 std::unique_ptr<ScalableVideoController> svc_controller =
243 CreateScalabilityStructure(GetParam().GetScalabilityMode());
244 std::vector<GenericFrameInfo> frame_infos =
245 ScalabilityStructureWrapper(*svc_controller)
246 .GenerateFrames(GetParam().num_temporal_units);
247 int64_t num_frames = frame_infos.size();
248 FrameDependencyStructure structure = svc_controller->DependencyStructure();
249
250 for (int dt = 0; dt < structure.num_decode_targets; ++dt) {
251 for (int64_t frame_id = 0; frame_id < num_frames; ++frame_id) {
252 const auto& frame = frame_infos[frame_id];
253 if (frame.decode_target_indications[dt] ==
254 DecodeTargetIndication::kNotPresent) {
255 continue;
256 }
257 for (int frame_diff : frame.frame_diffs) {
258 int64_t base_frame_id = frame_id - frame_diff;
259 const auto& base_frame = frame_infos[base_frame_id];
260 EXPECT_NE(base_frame.decode_target_indications[dt],
261 DecodeTargetIndication::kNotPresent)
262 << "Frame " << frame_id << " depends on frame " << base_frame_id
263 << " that is not part of decode target#" << dt;
264 EXPECT_NE(base_frame.decode_target_indications[dt],
265 DecodeTargetIndication::kDiscardable)
266 << "Frame " << frame_id << " depends on frame " << base_frame_id
267 << " that is discardable for decode target#" << dt;
268 }
269 }
270 }
271 }
272
TEST_P(ScalabilityStructureTest,NoFrameDependsThroughSwitchIndication)273 TEST_P(ScalabilityStructureTest, NoFrameDependsThroughSwitchIndication) {
274 std::unique_ptr<ScalableVideoController> svc_controller =
275 CreateScalabilityStructure(GetParam().GetScalabilityMode());
276 FrameDependencyStructure structure = svc_controller->DependencyStructure();
277 std::vector<GenericFrameInfo> frame_infos =
278 ScalabilityStructureWrapper(*svc_controller)
279 .GenerateFrames(GetParam().num_temporal_units);
280 int64_t num_frames = frame_infos.size();
281 std::vector<std::set<int64_t>> full_deps(num_frames);
282
283 // For each frame calculate set of all frames it depends on, both directly and
284 // indirectly.
285 for (int64_t frame_id = 0; frame_id < num_frames; ++frame_id) {
286 std::set<int64_t> all_base_frames;
287 for (int frame_diff : frame_infos[frame_id].frame_diffs) {
288 int64_t base_frame_id = frame_id - frame_diff;
289 all_base_frames.insert(base_frame_id);
290 const auto& indirect = full_deps[base_frame_id];
291 all_base_frames.insert(indirect.begin(), indirect.end());
292 }
293 full_deps[frame_id] = std::move(all_base_frames);
294 }
295
296 // Now check the switch indication: frames after the switch indication mustn't
297 // depend on any addition frames before the switch indications.
298 for (int dt = 0; dt < structure.num_decode_targets; ++dt) {
299 for (int64_t switch_frame_id = 0; switch_frame_id < num_frames;
300 ++switch_frame_id) {
301 if (frame_infos[switch_frame_id].decode_target_indications[dt] !=
302 DecodeTargetIndication::kSwitch) {
303 continue;
304 }
305 for (int64_t later_frame_id = switch_frame_id + 1;
306 later_frame_id < num_frames; ++later_frame_id) {
307 if (frame_infos[later_frame_id].decode_target_indications[dt] ==
308 DecodeTargetIndication::kNotPresent) {
309 continue;
310 }
311 for (int frame_diff : frame_infos[later_frame_id].frame_diffs) {
312 int64_t early_frame_id = later_frame_id - frame_diff;
313 if (early_frame_id < switch_frame_id) {
314 EXPECT_THAT(full_deps[switch_frame_id], Contains(early_frame_id))
315 << "For decode target #" << dt << " frame " << later_frame_id
316 << " depends on the frame " << early_frame_id
317 << " that switch indication frame " << switch_frame_id
318 << " doesn't directly on indirectly depend on.";
319 }
320 }
321 }
322 }
323 }
324 }
325
TEST_P(ScalabilityStructureTest,ProduceNoFrameForDisabledLayers)326 TEST_P(ScalabilityStructureTest, ProduceNoFrameForDisabledLayers) {
327 std::unique_ptr<ScalableVideoController> svc_controller =
328 CreateScalabilityStructure(GetParam().GetScalabilityMode());
329 ScalableVideoController::StreamLayersConfig structure =
330 svc_controller->StreamConfig();
331
332 VideoBitrateAllocation all_bitrates;
333 for (int sid = 0; sid < structure.num_spatial_layers; ++sid) {
334 for (int tid = 0; tid < structure.num_temporal_layers; ++tid) {
335 all_bitrates.SetBitrate(sid, tid, 100'000);
336 }
337 }
338
339 svc_controller->OnRatesUpdated(all_bitrates);
340 ScalabilityStructureWrapper wrapper(*svc_controller);
341 std::vector<GenericFrameInfo> frames =
342 wrapper.GenerateFrames(GetParam().num_temporal_units);
343
344 for (int sid = 0; sid < structure.num_spatial_layers; ++sid) {
345 for (int tid = 0; tid < structure.num_temporal_layers; ++tid) {
346 // When all layers were enabled, expect there was a frame for each layer.
347 EXPECT_THAT(frames,
348 Contains(AllOf(Field(&GenericFrameInfo::spatial_id, sid),
349 Field(&GenericFrameInfo::temporal_id, tid))))
350 << "For layer (" << sid << "," << tid << ")";
351 // Restore bitrates for all layers before disabling single layer.
352 VideoBitrateAllocation bitrates = all_bitrates;
353 bitrates.SetBitrate(sid, tid, 0);
354 svc_controller->OnRatesUpdated(bitrates);
355 // With layer (sid, tid) disabled, expect no frames are produced for it.
356 EXPECT_THAT(
357 wrapper.GenerateFrames(GetParam().num_temporal_units),
358 Not(Contains(AllOf(Field(&GenericFrameInfo::spatial_id, sid),
359 Field(&GenericFrameInfo::temporal_id, tid)))))
360 << "For layer (" << sid << "," << tid << ")";
361 }
362 }
363 }
364
365 INSTANTIATE_TEST_SUITE_P(
366 Svc,
367 ScalabilityStructureTest,
368 Values(SvcTestParam{"L1T1", /*num_temporal_units=*/3},
369 SvcTestParam{"L1T2", /*num_temporal_units=*/4},
370 SvcTestParam{"L1T3", /*num_temporal_units=*/8},
371 SvcTestParam{"L2T1", /*num_temporal_units=*/3},
372 SvcTestParam{"L2T1_KEY", /*num_temporal_units=*/3},
373 SvcTestParam{"L3T1", /*num_temporal_units=*/3},
374 SvcTestParam{"L3T1_KEY", /*num_temporal_units=*/3},
375 SvcTestParam{"L3T3", /*num_temporal_units=*/8},
376 SvcTestParam{"S2T1", /*num_temporal_units=*/3},
377 SvcTestParam{"S2T2", /*num_temporal_units=*/4},
378 SvcTestParam{"S2T3", /*num_temporal_units=*/8},
379 SvcTestParam{"S3T1", /*num_temporal_units=*/3},
380 SvcTestParam{"S3T2", /*num_temporal_units=*/4},
381 SvcTestParam{"S3T3", /*num_temporal_units=*/8},
382 SvcTestParam{"L2T2", /*num_temporal_units=*/4},
383 SvcTestParam{"L2T2_KEY", /*num_temporal_units=*/4},
384 SvcTestParam{"L2T2_KEY_SHIFT", /*num_temporal_units=*/4},
385 SvcTestParam{"L2T3", /*num_temporal_units=*/8},
386 SvcTestParam{"L2T3_KEY", /*num_temporal_units=*/8},
387 SvcTestParam{"L3T2", /*num_temporal_units=*/4},
388 SvcTestParam{"L3T2_KEY", /*num_temporal_units=*/4},
389 SvcTestParam{"L3T3_KEY", /*num_temporal_units=*/8}),
__anonf9e96bcd0202(const testing::TestParamInfo<SvcTestParam>& info) 390 [](const testing::TestParamInfo<SvcTestParam>& info) {
391 return info.param.name;
392 });
393
394 } // namespace
395 } // namespace webrtc
396