1 /*
2 * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/video_coding/codecs/vp9/svc_config.h"
12
13 #include <algorithm>
14 #include <cmath>
15 #include <memory>
16 #include <vector>
17
18 #include "media/base/video_common.h"
19 #include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
20 #include "modules/video_coding/svc/create_scalability_structure.h"
21 #include "modules/video_coding/svc/scalability_mode_util.h"
22 #include "rtc_base/checks.h"
23 #include "rtc_base/logging.h"
24
25 namespace webrtc {
26
27 namespace {
28 const size_t kMinVp9SvcBitrateKbps = 30;
29
30 const size_t kMaxNumLayersForScreenSharing = 3;
31 const float kMaxScreenSharingLayerFramerateFps[] = {5.0, 10.0, 30.0};
32 const size_t kMinScreenSharingLayerBitrateKbps[] = {30, 200, 500};
33 const size_t kTargetScreenSharingLayerBitrateKbps[] = {150, 350, 950};
34 const size_t kMaxScreenSharingLayerBitrateKbps[] = {250, 500, 950};
35
36 // Gets limited number of layers for given resolution.
GetLimitedNumSpatialLayers(size_t width,size_t height)37 size_t GetLimitedNumSpatialLayers(size_t width, size_t height) {
38 const bool is_landscape = width >= height;
39 const size_t min_width = is_landscape ? kMinVp9SpatialLayerLongSideLength
40 : kMinVp9SpatialLayerShortSideLength;
41 const size_t min_height = is_landscape ? kMinVp9SpatialLayerShortSideLength
42 : kMinVp9SpatialLayerLongSideLength;
43 const size_t num_layers_fit_horz = static_cast<size_t>(
44 std::floor(1 + std::max(0.0f, std::log2(1.0f * width / min_width))));
45 const size_t num_layers_fit_vert = static_cast<size_t>(
46 std::floor(1 + std::max(0.0f, std::log2(1.0f * height / min_height))));
47 return std::min(num_layers_fit_horz, num_layers_fit_vert);
48 }
49 } // namespace
50
ConfigureSvcScreenSharing(size_t input_width,size_t input_height,float max_framerate_fps,size_t num_spatial_layers)51 std::vector<SpatialLayer> ConfigureSvcScreenSharing(size_t input_width,
52 size_t input_height,
53 float max_framerate_fps,
54 size_t num_spatial_layers) {
55 num_spatial_layers =
56 std::min(num_spatial_layers, kMaxNumLayersForScreenSharing);
57 std::vector<SpatialLayer> spatial_layers;
58
59 for (size_t sl_idx = 0; sl_idx < num_spatial_layers; ++sl_idx) {
60 SpatialLayer spatial_layer = {0};
61 spatial_layer.width = input_width;
62 spatial_layer.height = input_height;
63 spatial_layer.maxFramerate =
64 std::min(kMaxScreenSharingLayerFramerateFps[sl_idx], max_framerate_fps);
65 spatial_layer.numberOfTemporalLayers = 1;
66 spatial_layer.minBitrate =
67 static_cast<int>(kMinScreenSharingLayerBitrateKbps[sl_idx]);
68 spatial_layer.maxBitrate =
69 static_cast<int>(kMaxScreenSharingLayerBitrateKbps[sl_idx]);
70 spatial_layer.targetBitrate =
71 static_cast<int>(kTargetScreenSharingLayerBitrateKbps[sl_idx]);
72 spatial_layer.active = true;
73 spatial_layers.push_back(spatial_layer);
74 }
75
76 return spatial_layers;
77 }
78
ConfigureSvcNormalVideo(size_t input_width,size_t input_height,float max_framerate_fps,size_t first_active_layer,size_t num_spatial_layers,size_t num_temporal_layers,absl::optional<ScalableVideoController::StreamLayersConfig> config)79 std::vector<SpatialLayer> ConfigureSvcNormalVideo(
80 size_t input_width,
81 size_t input_height,
82 float max_framerate_fps,
83 size_t first_active_layer,
84 size_t num_spatial_layers,
85 size_t num_temporal_layers,
86 absl::optional<ScalableVideoController::StreamLayersConfig> config) {
87 RTC_DCHECK_LT(first_active_layer, num_spatial_layers);
88
89 // Limit number of layers for given resolution.
90 size_t limited_num_spatial_layers =
91 GetLimitedNumSpatialLayers(input_width, input_height);
92 if (limited_num_spatial_layers < num_spatial_layers) {
93 RTC_LOG(LS_WARNING) << "Reducing number of spatial layers from "
94 << num_spatial_layers << " to "
95 << limited_num_spatial_layers
96 << " due to low input resolution.";
97 num_spatial_layers = limited_num_spatial_layers;
98 }
99
100 // First active layer must be configured.
101 num_spatial_layers = std::max(num_spatial_layers, first_active_layer + 1);
102
103 // Ensure top layer is even enough.
104 int required_divisiblity = 1 << (num_spatial_layers - first_active_layer - 1);
105 if (config) {
106 required_divisiblity = 1;
107 for (size_t sl_idx = 0; sl_idx < num_spatial_layers; ++sl_idx) {
108 required_divisiblity = cricket::LeastCommonMultiple(
109 required_divisiblity, config->scaling_factor_den[sl_idx]);
110 }
111 }
112 input_width = input_width - input_width % required_divisiblity;
113 input_height = input_height - input_height % required_divisiblity;
114
115 std::vector<SpatialLayer> spatial_layers;
116 for (size_t sl_idx = first_active_layer; sl_idx < num_spatial_layers;
117 ++sl_idx) {
118 SpatialLayer spatial_layer = {0};
119 spatial_layer.width = input_width >> (num_spatial_layers - sl_idx - 1);
120 spatial_layer.height = input_height >> (num_spatial_layers - sl_idx - 1);
121 spatial_layer.maxFramerate = max_framerate_fps;
122 spatial_layer.numberOfTemporalLayers = num_temporal_layers;
123 spatial_layer.active = true;
124
125 if (config) {
126 spatial_layer.width = input_width * config->scaling_factor_num[sl_idx] /
127 config->scaling_factor_den[sl_idx];
128 spatial_layer.height = input_height * config->scaling_factor_num[sl_idx] /
129 config->scaling_factor_den[sl_idx];
130 }
131
132 // minBitrate and maxBitrate formulas were derived from
133 // subjective-quality data to determing bit rates below which video
134 // quality is unacceptable and above which additional bits do not provide
135 // benefit. The formulas express rate in units of kbps.
136
137 // TODO(ssilkin): Add to the comment PSNR/SSIM we get at encoding certain
138 // video to min/max bitrate specified by those formulas.
139 const size_t num_pixels = spatial_layer.width * spatial_layer.height;
140 int min_bitrate =
141 static_cast<int>((600. * std::sqrt(num_pixels) - 95000.) / 1000.);
142 min_bitrate = std::max(min_bitrate, 0);
143 spatial_layer.minBitrate =
144 std::max(static_cast<size_t>(min_bitrate), kMinVp9SvcBitrateKbps);
145 spatial_layer.maxBitrate =
146 static_cast<int>((1.6 * num_pixels + 50 * 1000) / 1000);
147 spatial_layer.targetBitrate =
148 (spatial_layer.minBitrate + spatial_layer.maxBitrate) / 2;
149 spatial_layers.push_back(spatial_layer);
150 }
151
152 // A workaround for situation when single HD layer is left with minBitrate
153 // about 500kbps. This would mean that there will always be at least 500kbps
154 // allocated to video regardless of how low is the actual BWE.
155 // Also, boost maxBitrate for the first layer to account for lost ability to
156 // predict from previous layers.
157 if (first_active_layer > 0) {
158 spatial_layers[0].minBitrate = kMinVp9SvcBitrateKbps;
159 // TODO(ilnik): tune this value or come up with a different formula to
160 // ensure that all singlecast configurations look good and not too much
161 // bitrate is added.
162 spatial_layers[0].maxBitrate *= 1.1;
163 }
164
165 return spatial_layers;
166 }
167
168 // Uses scalability mode to configure spatial layers.
GetVp9SvcConfig(VideoCodec & codec)169 std::vector<SpatialLayer> GetVp9SvcConfig(VideoCodec& codec) {
170 RTC_DCHECK_EQ(codec.codecType, kVideoCodecVP9);
171
172 absl::optional<ScalabilityMode> scalability_mode = codec.GetScalabilityMode();
173 RTC_DCHECK(scalability_mode.has_value());
174
175 // Limit number of spatial layers for given resolution.
176 int limited_num_spatial_layers =
177 GetLimitedNumSpatialLayers(codec.width, codec.height);
178 if (limited_num_spatial_layers <
179 ScalabilityModeToNumSpatialLayers(*scalability_mode)) {
180 ScalabilityMode limited_scalability_mode =
181 LimitNumSpatialLayers(*scalability_mode, limited_num_spatial_layers);
182 RTC_LOG(LS_WARNING)
183 << "Reducing number of spatial layers due to low input resolution: "
184 << ScalabilityModeToString(*scalability_mode) << " to "
185 << ScalabilityModeToString(limited_scalability_mode);
186 scalability_mode = limited_scalability_mode;
187 codec.SetScalabilityMode(limited_scalability_mode);
188 }
189
190 absl::optional<ScalableVideoController::StreamLayersConfig> info =
191 ScalabilityStructureConfig(*scalability_mode);
192 if (!info.has_value()) {
193 RTC_LOG(LS_WARNING) << "Failed to create structure "
194 << ScalabilityModeToString(*scalability_mode);
195 return {};
196 }
197
198 // TODO(bugs.webrtc.org/11607): Add support for screensharing.
199 std::vector<SpatialLayer> spatial_layers =
200 GetSvcConfig(codec.width, codec.height, codec.maxFramerate,
201 /*first_active_layer=*/0, info->num_spatial_layers,
202 info->num_temporal_layers, /*is_screen_sharing=*/false,
203 codec.GetScalabilityMode() ? info : absl::nullopt);
204 RTC_DCHECK(!spatial_layers.empty());
205
206 // Use codec bitrate limits if spatial layering is not requested.
207 if (info->num_spatial_layers == 1) {
208 spatial_layers.back().minBitrate = codec.minBitrate;
209 spatial_layers.back().targetBitrate = codec.maxBitrate;
210 spatial_layers.back().maxBitrate = codec.maxBitrate;
211 }
212
213 return spatial_layers;
214 }
215
GetSvcConfig(size_t input_width,size_t input_height,float max_framerate_fps,size_t first_active_layer,size_t num_spatial_layers,size_t num_temporal_layers,bool is_screen_sharing,absl::optional<ScalableVideoController::StreamLayersConfig> config)216 std::vector<SpatialLayer> GetSvcConfig(
217 size_t input_width,
218 size_t input_height,
219 float max_framerate_fps,
220 size_t first_active_layer,
221 size_t num_spatial_layers,
222 size_t num_temporal_layers,
223 bool is_screen_sharing,
224 absl::optional<ScalableVideoController::StreamLayersConfig> config) {
225 RTC_DCHECK_GT(input_width, 0);
226 RTC_DCHECK_GT(input_height, 0);
227 RTC_DCHECK_GT(num_spatial_layers, 0);
228 RTC_DCHECK_GT(num_temporal_layers, 0);
229
230 if (is_screen_sharing) {
231 return ConfigureSvcScreenSharing(input_width, input_height,
232 max_framerate_fps, num_spatial_layers);
233 } else {
234 return ConfigureSvcNormalVideo(input_width, input_height, max_framerate_fps,
235 first_active_layer, num_spatial_layers,
236 num_temporal_layers, config);
237 }
238 }
239
240 } // namespace webrtc
241