1 /*
2 * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10 #include "modules/video_coding/svc/scalability_structure_simulcast.h"
11
12 #include <utility>
13 #include <vector>
14
15 #include "absl/base/macros.h"
16 #include "api/transport/rtp/dependency_descriptor.h"
17 #include "rtc_base/checks.h"
18 #include "rtc_base/logging.h"
19
20 namespace webrtc {
21 namespace {
22
23 DecodeTargetIndication
Dti(int sid,int tid,const ScalableVideoController::LayerFrameConfig & config)24 Dti(int sid, int tid, const ScalableVideoController::LayerFrameConfig& config) {
25 if (sid != config.SpatialId() || tid < config.TemporalId()) {
26 return DecodeTargetIndication::kNotPresent;
27 }
28 if (tid == 0) {
29 RTC_DCHECK_EQ(config.TemporalId(), 0);
30 return DecodeTargetIndication::kSwitch;
31 }
32 if (tid == config.TemporalId()) {
33 return DecodeTargetIndication::kDiscardable;
34 }
35 RTC_DCHECK_GT(tid, config.TemporalId());
36 return DecodeTargetIndication::kSwitch;
37 }
38
39 } // namespace
40
41 constexpr int ScalabilityStructureSimulcast::kMaxNumSpatialLayers;
42 constexpr int ScalabilityStructureSimulcast::kMaxNumTemporalLayers;
43
ScalabilityStructureSimulcast(int num_spatial_layers,int num_temporal_layers,ScalingFactor resolution_factor)44 ScalabilityStructureSimulcast::ScalabilityStructureSimulcast(
45 int num_spatial_layers,
46 int num_temporal_layers,
47 ScalingFactor resolution_factor)
48 : num_spatial_layers_(num_spatial_layers),
49 num_temporal_layers_(num_temporal_layers),
50 resolution_factor_(resolution_factor),
51 active_decode_targets_(
52 (uint32_t{1} << (num_spatial_layers * num_temporal_layers)) - 1) {
53 RTC_DCHECK_LE(num_spatial_layers, kMaxNumSpatialLayers);
54 RTC_DCHECK_LE(num_temporal_layers, kMaxNumTemporalLayers);
55 }
56
57 ScalabilityStructureSimulcast::~ScalabilityStructureSimulcast() = default;
58
59 ScalableVideoController::StreamLayersConfig
StreamConfig() const60 ScalabilityStructureSimulcast::StreamConfig() const {
61 StreamLayersConfig result;
62 result.num_spatial_layers = num_spatial_layers_;
63 result.num_temporal_layers = num_temporal_layers_;
64 result.scaling_factor_num[num_spatial_layers_ - 1] = 1;
65 result.scaling_factor_den[num_spatial_layers_ - 1] = 1;
66 for (int sid = num_spatial_layers_ - 1; sid > 0; --sid) {
67 result.scaling_factor_num[sid - 1] =
68 resolution_factor_.num * result.scaling_factor_num[sid];
69 result.scaling_factor_den[sid - 1] =
70 resolution_factor_.den * result.scaling_factor_den[sid];
71 }
72 result.uses_reference_scaling = false;
73 return result;
74 }
75
TemporalLayerIsActive(int tid) const76 bool ScalabilityStructureSimulcast::TemporalLayerIsActive(int tid) const {
77 if (tid >= num_temporal_layers_) {
78 return false;
79 }
80 for (int sid = 0; sid < num_spatial_layers_; ++sid) {
81 if (DecodeTargetIsActive(sid, tid)) {
82 return true;
83 }
84 }
85 return false;
86 }
87
88 ScalabilityStructureSimulcast::FramePattern
NextPattern() const89 ScalabilityStructureSimulcast::NextPattern() const {
90 switch (last_pattern_) {
91 case kNone:
92 case kDeltaT2B:
93 return kDeltaT0;
94 case kDeltaT2A:
95 if (TemporalLayerIsActive(1)) {
96 return kDeltaT1;
97 }
98 return kDeltaT0;
99 case kDeltaT1:
100 if (TemporalLayerIsActive(2)) {
101 return kDeltaT2B;
102 }
103 return kDeltaT0;
104 case kDeltaT0:
105 if (TemporalLayerIsActive(2)) {
106 return kDeltaT2A;
107 }
108 if (TemporalLayerIsActive(1)) {
109 return kDeltaT1;
110 }
111 return kDeltaT0;
112 }
113 RTC_DCHECK_NOTREACHED();
114 return kDeltaT0;
115 }
116
117 std::vector<ScalableVideoController::LayerFrameConfig>
NextFrameConfig(bool restart)118 ScalabilityStructureSimulcast::NextFrameConfig(bool restart) {
119 std::vector<LayerFrameConfig> configs;
120 if (active_decode_targets_.none()) {
121 last_pattern_ = kNone;
122 return configs;
123 }
124 configs.reserve(num_spatial_layers_);
125
126 if (last_pattern_ == kNone || restart) {
127 can_reference_t0_frame_for_spatial_id_.reset();
128 last_pattern_ = kNone;
129 }
130 FramePattern current_pattern = NextPattern();
131
132 switch (current_pattern) {
133 case kDeltaT0:
134 // Disallow temporal references cross T0 on higher temporal layers.
135 can_reference_t1_frame_for_spatial_id_.reset();
136 for (int sid = 0; sid < num_spatial_layers_; ++sid) {
137 if (!DecodeTargetIsActive(sid, /*tid=*/0)) {
138 // Next frame from the spatial layer `sid` shouldn't depend on
139 // potentially old previous frame from the spatial layer `sid`.
140 can_reference_t0_frame_for_spatial_id_.reset(sid);
141 continue;
142 }
143 configs.emplace_back();
144 ScalableVideoController::LayerFrameConfig& config = configs.back();
145 config.Id(current_pattern).S(sid).T(0);
146
147 if (can_reference_t0_frame_for_spatial_id_[sid]) {
148 config.ReferenceAndUpdate(BufferIndex(sid, /*tid=*/0));
149 } else {
150 config.Keyframe().Update(BufferIndex(sid, /*tid=*/0));
151 }
152 can_reference_t0_frame_for_spatial_id_.set(sid);
153 }
154 break;
155 case kDeltaT1:
156 for (int sid = 0; sid < num_spatial_layers_; ++sid) {
157 if (!DecodeTargetIsActive(sid, /*tid=*/1) ||
158 !can_reference_t0_frame_for_spatial_id_[sid]) {
159 continue;
160 }
161 configs.emplace_back();
162 ScalableVideoController::LayerFrameConfig& config = configs.back();
163 config.Id(current_pattern)
164 .S(sid)
165 .T(1)
166 .Reference(BufferIndex(sid, /*tid=*/0));
167 // Save frame only if there is a higher temporal layer that may need it.
168 if (num_temporal_layers_ > 2) {
169 config.Update(BufferIndex(sid, /*tid=*/1));
170 }
171 }
172 break;
173 case kDeltaT2A:
174 case kDeltaT2B:
175 for (int sid = 0; sid < num_spatial_layers_; ++sid) {
176 if (!DecodeTargetIsActive(sid, /*tid=*/2) ||
177 !can_reference_t0_frame_for_spatial_id_[sid]) {
178 continue;
179 }
180 configs.emplace_back();
181 ScalableVideoController::LayerFrameConfig& config = configs.back();
182 config.Id(current_pattern).S(sid).T(2);
183 if (can_reference_t1_frame_for_spatial_id_[sid]) {
184 config.Reference(BufferIndex(sid, /*tid=*/1));
185 } else {
186 config.Reference(BufferIndex(sid, /*tid=*/0));
187 }
188 }
189 break;
190 case kNone:
191 RTC_DCHECK_NOTREACHED();
192 break;
193 }
194
195 return configs;
196 }
197
OnEncodeDone(const LayerFrameConfig & config)198 GenericFrameInfo ScalabilityStructureSimulcast::OnEncodeDone(
199 const LayerFrameConfig& config) {
200 last_pattern_ = static_cast<FramePattern>(config.Id());
201 if (config.TemporalId() == 1) {
202 can_reference_t1_frame_for_spatial_id_.set(config.SpatialId());
203 }
204 GenericFrameInfo frame_info;
205 frame_info.spatial_id = config.SpatialId();
206 frame_info.temporal_id = config.TemporalId();
207 frame_info.encoder_buffers = config.Buffers();
208 frame_info.decode_target_indications.reserve(num_spatial_layers_ *
209 num_temporal_layers_);
210 for (int sid = 0; sid < num_spatial_layers_; ++sid) {
211 for (int tid = 0; tid < num_temporal_layers_; ++tid) {
212 frame_info.decode_target_indications.push_back(Dti(sid, tid, config));
213 }
214 }
215 frame_info.part_of_chain.assign(num_spatial_layers_, false);
216 if (config.TemporalId() == 0) {
217 frame_info.part_of_chain[config.SpatialId()] = true;
218 }
219 frame_info.active_decode_targets = active_decode_targets_;
220 return frame_info;
221 }
222
OnRatesUpdated(const VideoBitrateAllocation & bitrates)223 void ScalabilityStructureSimulcast::OnRatesUpdated(
224 const VideoBitrateAllocation& bitrates) {
225 for (int sid = 0; sid < num_spatial_layers_; ++sid) {
226 // Enable/disable spatial layers independetely.
227 bool active = true;
228 for (int tid = 0; tid < num_temporal_layers_; ++tid) {
229 // To enable temporal layer, require bitrates for lower temporal layers.
230 active = active && bitrates.GetBitrate(sid, tid) > 0;
231 SetDecodeTargetIsActive(sid, tid, active);
232 }
233 }
234 }
235
DependencyStructure() const236 FrameDependencyStructure ScalabilityStructureS2T1::DependencyStructure() const {
237 FrameDependencyStructure structure;
238 structure.num_decode_targets = 2;
239 structure.num_chains = 2;
240 structure.decode_target_protected_by_chain = {0, 1};
241 structure.templates.resize(4);
242 structure.templates[0].S(0).Dtis("S-").ChainDiffs({2, 1}).FrameDiffs({2});
243 structure.templates[1].S(0).Dtis("S-").ChainDiffs({0, 0});
244 structure.templates[2].S(1).Dtis("-S").ChainDiffs({1, 2}).FrameDiffs({2});
245 structure.templates[3].S(1).Dtis("-S").ChainDiffs({1, 0});
246 return structure;
247 }
248
DependencyStructure() const249 FrameDependencyStructure ScalabilityStructureS2T2::DependencyStructure() const {
250 FrameDependencyStructure structure;
251 structure.num_decode_targets = 4;
252 structure.num_chains = 2;
253 structure.decode_target_protected_by_chain = {0, 0, 1, 1};
254 auto& t = structure.templates;
255 t.resize(6);
256 t[1].S(0).T(0).Dtis("SS--").ChainDiffs({0, 0});
257 t[4].S(1).T(0).Dtis("--SS").ChainDiffs({1, 0});
258 t[2].S(0).T(1).Dtis("-D--").ChainDiffs({2, 1}).FrameDiffs({2});
259 t[5].S(1).T(1).Dtis("---D").ChainDiffs({3, 2}).FrameDiffs({2});
260 t[0].S(0).T(0).Dtis("SS--").ChainDiffs({4, 3}).FrameDiffs({4});
261 t[3].S(1).T(0).Dtis("--SS").ChainDiffs({1, 4}).FrameDiffs({4});
262 return structure;
263 }
264
DependencyStructure() const265 FrameDependencyStructure ScalabilityStructureS2T3::DependencyStructure() const {
266 FrameDependencyStructure structure;
267 structure.num_decode_targets = 6;
268 structure.num_chains = 2;
269 structure.decode_target_protected_by_chain = {0, 0, 0, 1, 1, 1};
270 auto& t = structure.templates;
271 t.resize(10);
272 t[1].S(0).T(0).Dtis("SSS---").ChainDiffs({0, 0});
273 t[6].S(1).T(0).Dtis("---SSS").ChainDiffs({1, 0});
274 t[3].S(0).T(2).Dtis("--D---").ChainDiffs({2, 1}).FrameDiffs({2});
275 t[8].S(1).T(2).Dtis("-----D").ChainDiffs({3, 2}).FrameDiffs({2});
276 t[2].S(0).T(1).Dtis("-DS---").ChainDiffs({4, 3}).FrameDiffs({4});
277 t[7].S(1).T(1).Dtis("----DS").ChainDiffs({5, 4}).FrameDiffs({4});
278 t[4].S(0).T(2).Dtis("--D---").ChainDiffs({6, 5}).FrameDiffs({2});
279 t[9].S(1).T(2).Dtis("-----D").ChainDiffs({7, 6}).FrameDiffs({2});
280 t[0].S(0).T(0).Dtis("SSS---").ChainDiffs({8, 7}).FrameDiffs({8});
281 t[5].S(1).T(0).Dtis("---SSS").ChainDiffs({1, 8}).FrameDiffs({8});
282 return structure;
283 }
284
DependencyStructure() const285 FrameDependencyStructure ScalabilityStructureS3T1::DependencyStructure() const {
286 FrameDependencyStructure structure;
287 structure.num_decode_targets = 3;
288 structure.num_chains = 3;
289 structure.decode_target_protected_by_chain = {0, 1, 2};
290 auto& t = structure.templates;
291 t.resize(6);
292 t[1].S(0).T(0).Dtis("S--").ChainDiffs({0, 0, 0});
293 t[3].S(1).T(0).Dtis("-S-").ChainDiffs({1, 0, 0});
294 t[5].S(2).T(0).Dtis("--S").ChainDiffs({2, 1, 0});
295 t[0].S(0).T(0).Dtis("S--").ChainDiffs({3, 2, 1}).FrameDiffs({3});
296 t[2].S(1).T(0).Dtis("-S-").ChainDiffs({1, 3, 2}).FrameDiffs({3});
297 t[4].S(2).T(0).Dtis("--S").ChainDiffs({2, 1, 3}).FrameDiffs({3});
298 return structure;
299 }
300
DependencyStructure() const301 FrameDependencyStructure ScalabilityStructureS3T2::DependencyStructure() const {
302 FrameDependencyStructure structure;
303 structure.num_decode_targets = 6;
304 structure.num_chains = 3;
305 structure.decode_target_protected_by_chain = {0, 0, 1, 1, 2, 2};
306 auto& t = structure.templates;
307 t.resize(9);
308 // Templates are shown in the order frames following them appear in the
309 // stream, but in `structure.templates` array templates are sorted by
310 // (`spatial_id`, `temporal_id`) since that is a dependency descriptor
311 // requirement.
312 t[1].S(0).T(0).Dtis("SS----").ChainDiffs({0, 0, 0});
313 t[4].S(1).T(0).Dtis("--SS--").ChainDiffs({1, 0, 0});
314 t[7].S(2).T(0).Dtis("----SS").ChainDiffs({2, 1, 0});
315 t[2].S(0).T(1).Dtis("-D----").ChainDiffs({3, 2, 1}).FrameDiffs({3});
316 t[5].S(1).T(1).Dtis("---D--").ChainDiffs({4, 3, 2}).FrameDiffs({3});
317 t[8].S(2).T(1).Dtis("-----D").ChainDiffs({5, 4, 3}).FrameDiffs({3});
318 t[0].S(0).T(0).Dtis("SS----").ChainDiffs({6, 5, 4}).FrameDiffs({6});
319 t[3].S(1).T(0).Dtis("--SS--").ChainDiffs({1, 6, 5}).FrameDiffs({6});
320 t[6].S(2).T(0).Dtis("----SS").ChainDiffs({2, 1, 6}).FrameDiffs({6});
321 return structure;
322 }
323
DependencyStructure() const324 FrameDependencyStructure ScalabilityStructureS3T3::DependencyStructure() const {
325 FrameDependencyStructure structure;
326 structure.num_decode_targets = 9;
327 structure.num_chains = 3;
328 structure.decode_target_protected_by_chain = {0, 0, 0, 1, 1, 1, 2, 2, 2};
329 auto& t = structure.templates;
330 t.resize(15);
331 // Templates are shown in the order frames following them appear in the
332 // stream, but in `structure.templates` array templates are sorted by
333 // (`spatial_id`, `temporal_id`) since that is a dependency descriptor
334 // requirement. Indexes are written in hex for nicer alignment.
335 t[0x1].S(0).T(0).Dtis("SSS------").ChainDiffs({0, 0, 0});
336 t[0x6].S(1).T(0).Dtis("---SSS---").ChainDiffs({1, 0, 0});
337 t[0xB].S(2).T(0).Dtis("------SSS").ChainDiffs({2, 1, 0});
338 t[0x3].S(0).T(2).Dtis("--D------").ChainDiffs({3, 2, 1}).FrameDiffs({3});
339 t[0x8].S(1).T(2).Dtis("-----D---").ChainDiffs({4, 3, 2}).FrameDiffs({3});
340 t[0xD].S(2).T(2).Dtis("--------D").ChainDiffs({5, 4, 3}).FrameDiffs({3});
341 t[0x2].S(0).T(1).Dtis("-DS------").ChainDiffs({6, 5, 4}).FrameDiffs({6});
342 t[0x7].S(1).T(1).Dtis("----DS---").ChainDiffs({7, 6, 5}).FrameDiffs({6});
343 t[0xC].S(2).T(1).Dtis("-------DS").ChainDiffs({8, 7, 6}).FrameDiffs({6});
344 t[0x4].S(0).T(2).Dtis("--D------").ChainDiffs({9, 8, 7}).FrameDiffs({3});
345 t[0x9].S(1).T(2).Dtis("-----D---").ChainDiffs({10, 9, 8}).FrameDiffs({3});
346 t[0xE].S(2).T(2).Dtis("--------D").ChainDiffs({11, 10, 9}).FrameDiffs({3});
347 t[0x0].S(0).T(0).Dtis("SSS------").ChainDiffs({12, 11, 10}).FrameDiffs({12});
348 t[0x5].S(1).T(0).Dtis("---SSS---").ChainDiffs({1, 12, 11}).FrameDiffs({12});
349 t[0xA].S(2).T(0).Dtis("------SSS").ChainDiffs({2, 1, 12}).FrameDiffs({12});
350 return structure;
351 }
352
353 } // namespace webrtc
354