1 /*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/audio_coding/audio_network_adaptor/controller_manager.h"
12
13 #include <cmath>
14 #include <memory>
15 #include <string>
16 #include <utility>
17
18 #include "absl/strings/string_view.h"
19 #include "modules/audio_coding/audio_network_adaptor/bitrate_controller.h"
20 #include "modules/audio_coding/audio_network_adaptor/channel_controller.h"
21 #include "modules/audio_coding/audio_network_adaptor/debug_dump_writer.h"
22 #include "modules/audio_coding/audio_network_adaptor/dtx_controller.h"
23 #include "modules/audio_coding/audio_network_adaptor/fec_controller_plr_based.h"
24 #include "modules/audio_coding/audio_network_adaptor/frame_length_controller.h"
25 #include "modules/audio_coding/audio_network_adaptor/frame_length_controller_v2.h"
26 #include "modules/audio_coding/audio_network_adaptor/util/threshold_curve.h"
27 #include "rtc_base/ignore_wundef.h"
28 #include "rtc_base/logging.h"
29 #include "rtc_base/time_utils.h"
30
31 #if WEBRTC_ENABLE_PROTOBUF
32 RTC_PUSH_IGNORING_WUNDEF()
33 #ifdef WEBRTC_ANDROID_PLATFORM_BUILD
34 #include "external/webrtc/webrtc/modules/audio_coding/audio_network_adaptor/config.pb.h"
35 #else
36 #include "modules/audio_coding/audio_network_adaptor/config.pb.h"
37 #endif
38 RTC_POP_IGNORING_WUNDEF()
39 #endif
40
41 namespace webrtc {
42
43 namespace {
44
45 #if WEBRTC_ENABLE_PROTOBUF
46
CreateFecControllerPlrBased(const audio_network_adaptor::config::FecController & config,bool initial_fec_enabled)47 std::unique_ptr<FecControllerPlrBased> CreateFecControllerPlrBased(
48 const audio_network_adaptor::config::FecController& config,
49 bool initial_fec_enabled) {
50 RTC_CHECK(config.has_fec_enabling_threshold());
51 RTC_CHECK(config.has_fec_disabling_threshold());
52 RTC_CHECK(config.has_time_constant_ms());
53
54 auto& fec_enabling_threshold = config.fec_enabling_threshold();
55 RTC_CHECK(fec_enabling_threshold.has_low_bandwidth_bps());
56 RTC_CHECK(fec_enabling_threshold.has_low_bandwidth_packet_loss());
57 RTC_CHECK(fec_enabling_threshold.has_high_bandwidth_bps());
58 RTC_CHECK(fec_enabling_threshold.has_high_bandwidth_packet_loss());
59
60 auto& fec_disabling_threshold = config.fec_disabling_threshold();
61 RTC_CHECK(fec_disabling_threshold.has_low_bandwidth_bps());
62 RTC_CHECK(fec_disabling_threshold.has_low_bandwidth_packet_loss());
63 RTC_CHECK(fec_disabling_threshold.has_high_bandwidth_bps());
64 RTC_CHECK(fec_disabling_threshold.has_high_bandwidth_packet_loss());
65
66 return std::unique_ptr<FecControllerPlrBased>(
67 new FecControllerPlrBased(FecControllerPlrBased::Config(
68 initial_fec_enabled,
69 ThresholdCurve(fec_enabling_threshold.low_bandwidth_bps(),
70 fec_enabling_threshold.low_bandwidth_packet_loss(),
71 fec_enabling_threshold.high_bandwidth_bps(),
72 fec_enabling_threshold.high_bandwidth_packet_loss()),
73 ThresholdCurve(fec_disabling_threshold.low_bandwidth_bps(),
74 fec_disabling_threshold.low_bandwidth_packet_loss(),
75 fec_disabling_threshold.high_bandwidth_bps(),
76 fec_disabling_threshold.high_bandwidth_packet_loss()),
77 config.time_constant_ms())));
78 }
79
CreateFrameLengthController(const audio_network_adaptor::config::FrameLengthController & config,rtc::ArrayView<const int> encoder_frame_lengths_ms,int initial_frame_length_ms,int min_encoder_bitrate_bps)80 std::unique_ptr<FrameLengthController> CreateFrameLengthController(
81 const audio_network_adaptor::config::FrameLengthController& config,
82 rtc::ArrayView<const int> encoder_frame_lengths_ms,
83 int initial_frame_length_ms,
84 int min_encoder_bitrate_bps) {
85 RTC_CHECK(config.has_fl_increasing_packet_loss_fraction());
86 RTC_CHECK(config.has_fl_decreasing_packet_loss_fraction());
87
88 std::map<FrameLengthController::Config::FrameLengthChange, int>
89 fl_changing_bandwidths_bps;
90
91 if (config.has_fl_20ms_to_60ms_bandwidth_bps()) {
92 fl_changing_bandwidths_bps.insert(
93 std::make_pair(FrameLengthController::Config::FrameLengthChange(20, 60),
94 config.fl_20ms_to_60ms_bandwidth_bps()));
95 }
96
97 if (config.has_fl_60ms_to_20ms_bandwidth_bps()) {
98 fl_changing_bandwidths_bps.insert(
99 std::make_pair(FrameLengthController::Config::FrameLengthChange(60, 20),
100 config.fl_60ms_to_20ms_bandwidth_bps()));
101 }
102
103 if (config.has_fl_20ms_to_40ms_bandwidth_bps()) {
104 fl_changing_bandwidths_bps.insert(
105 std::make_pair(FrameLengthController::Config::FrameLengthChange(20, 40),
106 config.fl_20ms_to_40ms_bandwidth_bps()));
107 }
108
109 if (config.has_fl_40ms_to_20ms_bandwidth_bps()) {
110 fl_changing_bandwidths_bps.insert(
111 std::make_pair(FrameLengthController::Config::FrameLengthChange(40, 20),
112 config.fl_40ms_to_20ms_bandwidth_bps()));
113 }
114
115 if (config.has_fl_40ms_to_60ms_bandwidth_bps()) {
116 fl_changing_bandwidths_bps.insert(
117 std::make_pair(FrameLengthController::Config::FrameLengthChange(40, 60),
118 config.fl_40ms_to_60ms_bandwidth_bps()));
119 }
120
121 if (config.has_fl_60ms_to_40ms_bandwidth_bps()) {
122 fl_changing_bandwidths_bps.insert(
123 std::make_pair(FrameLengthController::Config::FrameLengthChange(60, 40),
124 config.fl_60ms_to_40ms_bandwidth_bps()));
125 }
126
127 if (config.has_fl_60ms_to_120ms_bandwidth_bps()) {
128 fl_changing_bandwidths_bps.insert(std::make_pair(
129 FrameLengthController::Config::FrameLengthChange(60, 120),
130 config.fl_60ms_to_120ms_bandwidth_bps()));
131 }
132
133 if (config.has_fl_120ms_to_60ms_bandwidth_bps()) {
134 fl_changing_bandwidths_bps.insert(std::make_pair(
135 FrameLengthController::Config::FrameLengthChange(120, 60),
136 config.fl_120ms_to_60ms_bandwidth_bps()));
137 }
138
139 int fl_increase_overhead_offset = 0;
140 if (config.has_fl_increase_overhead_offset()) {
141 fl_increase_overhead_offset = config.fl_increase_overhead_offset();
142 }
143 int fl_decrease_overhead_offset = 0;
144 if (config.has_fl_decrease_overhead_offset()) {
145 fl_decrease_overhead_offset = config.fl_decrease_overhead_offset();
146 }
147
148 FrameLengthController::Config ctor_config(
149 std::set<int>(), initial_frame_length_ms, min_encoder_bitrate_bps,
150 config.fl_increasing_packet_loss_fraction(),
151 config.fl_decreasing_packet_loss_fraction(), fl_increase_overhead_offset,
152 fl_decrease_overhead_offset, std::move(fl_changing_bandwidths_bps));
153
154 for (auto frame_length : encoder_frame_lengths_ms)
155 ctor_config.encoder_frame_lengths_ms.insert(frame_length);
156
157 return std::unique_ptr<FrameLengthController>(
158 new FrameLengthController(ctor_config));
159 }
160
CreateChannelController(const audio_network_adaptor::config::ChannelController & config,size_t num_encoder_channels,size_t intial_channels_to_encode)161 std::unique_ptr<ChannelController> CreateChannelController(
162 const audio_network_adaptor::config::ChannelController& config,
163 size_t num_encoder_channels,
164 size_t intial_channels_to_encode) {
165 RTC_CHECK(config.has_channel_1_to_2_bandwidth_bps());
166 RTC_CHECK(config.has_channel_2_to_1_bandwidth_bps());
167
168 return std::unique_ptr<ChannelController>(new ChannelController(
169 ChannelController::Config(num_encoder_channels, intial_channels_to_encode,
170 config.channel_1_to_2_bandwidth_bps(),
171 config.channel_2_to_1_bandwidth_bps())));
172 }
173
CreateDtxController(const audio_network_adaptor::config::DtxController & dtx_config,bool initial_dtx_enabled)174 std::unique_ptr<DtxController> CreateDtxController(
175 const audio_network_adaptor::config::DtxController& dtx_config,
176 bool initial_dtx_enabled) {
177 RTC_CHECK(dtx_config.has_dtx_enabling_bandwidth_bps());
178 RTC_CHECK(dtx_config.has_dtx_disabling_bandwidth_bps());
179
180 return std::unique_ptr<DtxController>(new DtxController(DtxController::Config(
181 initial_dtx_enabled, dtx_config.dtx_enabling_bandwidth_bps(),
182 dtx_config.dtx_disabling_bandwidth_bps())));
183 }
184
185 using audio_network_adaptor::BitrateController;
CreateBitrateController(const audio_network_adaptor::config::BitrateController & bitrate_config,int initial_bitrate_bps,int initial_frame_length_ms)186 std::unique_ptr<BitrateController> CreateBitrateController(
187 const audio_network_adaptor::config::BitrateController& bitrate_config,
188 int initial_bitrate_bps,
189 int initial_frame_length_ms) {
190 int fl_increase_overhead_offset = 0;
191 if (bitrate_config.has_fl_increase_overhead_offset()) {
192 fl_increase_overhead_offset = bitrate_config.fl_increase_overhead_offset();
193 }
194 int fl_decrease_overhead_offset = 0;
195 if (bitrate_config.has_fl_decrease_overhead_offset()) {
196 fl_decrease_overhead_offset = bitrate_config.fl_decrease_overhead_offset();
197 }
198 return std::unique_ptr<BitrateController>(
199 new BitrateController(BitrateController::Config(
200 initial_bitrate_bps, initial_frame_length_ms,
201 fl_increase_overhead_offset, fl_decrease_overhead_offset)));
202 }
203
CreateFrameLengthControllerV2(const audio_network_adaptor::config::FrameLengthControllerV2 & config,rtc::ArrayView<const int> encoder_frame_lengths_ms)204 std::unique_ptr<FrameLengthControllerV2> CreateFrameLengthControllerV2(
205 const audio_network_adaptor::config::FrameLengthControllerV2& config,
206 rtc::ArrayView<const int> encoder_frame_lengths_ms) {
207 return std::make_unique<FrameLengthControllerV2>(
208 encoder_frame_lengths_ms, config.min_payload_bitrate_bps(),
209 config.use_slow_adaptation());
210 }
211 #endif // WEBRTC_ENABLE_PROTOBUF
212
213 } // namespace
214
Config(int min_reordering_time_ms,float min_reordering_squared_distance)215 ControllerManagerImpl::Config::Config(int min_reordering_time_ms,
216 float min_reordering_squared_distance)
217 : min_reordering_time_ms(min_reordering_time_ms),
218 min_reordering_squared_distance(min_reordering_squared_distance) {}
219
220 ControllerManagerImpl::Config::~Config() = default;
221
Create(absl::string_view config_string,size_t num_encoder_channels,rtc::ArrayView<const int> encoder_frame_lengths_ms,int min_encoder_bitrate_bps,size_t intial_channels_to_encode,int initial_frame_length_ms,int initial_bitrate_bps,bool initial_fec_enabled,bool initial_dtx_enabled)222 std::unique_ptr<ControllerManager> ControllerManagerImpl::Create(
223 absl::string_view config_string,
224 size_t num_encoder_channels,
225 rtc::ArrayView<const int> encoder_frame_lengths_ms,
226 int min_encoder_bitrate_bps,
227 size_t intial_channels_to_encode,
228 int initial_frame_length_ms,
229 int initial_bitrate_bps,
230 bool initial_fec_enabled,
231 bool initial_dtx_enabled) {
232 return Create(config_string, num_encoder_channels, encoder_frame_lengths_ms,
233 min_encoder_bitrate_bps, intial_channels_to_encode,
234 initial_frame_length_ms, initial_bitrate_bps,
235 initial_fec_enabled, initial_dtx_enabled, nullptr);
236 }
237
Create(absl::string_view config_string,size_t num_encoder_channels,rtc::ArrayView<const int> encoder_frame_lengths_ms,int min_encoder_bitrate_bps,size_t intial_channels_to_encode,int initial_frame_length_ms,int initial_bitrate_bps,bool initial_fec_enabled,bool initial_dtx_enabled,DebugDumpWriter * debug_dump_writer)238 std::unique_ptr<ControllerManager> ControllerManagerImpl::Create(
239 absl::string_view config_string,
240 size_t num_encoder_channels,
241 rtc::ArrayView<const int> encoder_frame_lengths_ms,
242 int min_encoder_bitrate_bps,
243 size_t intial_channels_to_encode,
244 int initial_frame_length_ms,
245 int initial_bitrate_bps,
246 bool initial_fec_enabled,
247 bool initial_dtx_enabled,
248 DebugDumpWriter* debug_dump_writer) {
249 #if WEBRTC_ENABLE_PROTOBUF
250 audio_network_adaptor::config::ControllerManager controller_manager_config;
251 RTC_CHECK(
252 controller_manager_config.ParseFromString(std::string(config_string)));
253 if (debug_dump_writer)
254 debug_dump_writer->DumpControllerManagerConfig(controller_manager_config,
255 rtc::TimeMillis());
256
257 std::vector<std::unique_ptr<Controller>> controllers;
258 std::map<const Controller*, std::pair<int, float>> scoring_points;
259
260 for (int i = 0; i < controller_manager_config.controllers_size(); ++i) {
261 auto& controller_config = controller_manager_config.controllers(i);
262 std::unique_ptr<Controller> controller;
263 switch (controller_config.controller_case()) {
264 case audio_network_adaptor::config::Controller::kFecController:
265 controller = CreateFecControllerPlrBased(
266 controller_config.fec_controller(), initial_fec_enabled);
267 break;
268 case audio_network_adaptor::config::Controller::kFecControllerRplrBased:
269 // FecControllerRplrBased has been removed and can't be used anymore.
270 RTC_DCHECK_NOTREACHED();
271 continue;
272 case audio_network_adaptor::config::Controller::kFrameLengthController:
273 controller = CreateFrameLengthController(
274 controller_config.frame_length_controller(),
275 encoder_frame_lengths_ms, initial_frame_length_ms,
276 min_encoder_bitrate_bps);
277 break;
278 case audio_network_adaptor::config::Controller::kChannelController:
279 controller = CreateChannelController(
280 controller_config.channel_controller(), num_encoder_channels,
281 intial_channels_to_encode);
282 break;
283 case audio_network_adaptor::config::Controller::kDtxController:
284 controller = CreateDtxController(controller_config.dtx_controller(),
285 initial_dtx_enabled);
286 break;
287 case audio_network_adaptor::config::Controller::kBitrateController:
288 controller = CreateBitrateController(
289 controller_config.bitrate_controller(), initial_bitrate_bps,
290 initial_frame_length_ms);
291 break;
292 case audio_network_adaptor::config::Controller::kFrameLengthControllerV2:
293 controller = CreateFrameLengthControllerV2(
294 controller_config.frame_length_controller_v2(),
295 encoder_frame_lengths_ms);
296 break;
297 default:
298 RTC_DCHECK_NOTREACHED();
299 }
300 if (controller_config.has_scoring_point()) {
301 auto& scoring_point = controller_config.scoring_point();
302 RTC_CHECK(scoring_point.has_uplink_bandwidth_bps());
303 RTC_CHECK(scoring_point.has_uplink_packet_loss_fraction());
304 scoring_points[controller.get()] = std::make_pair<int, float>(
305 scoring_point.uplink_bandwidth_bps(),
306 scoring_point.uplink_packet_loss_fraction());
307 }
308 controllers.push_back(std::move(controller));
309 }
310
311 if (scoring_points.size() == 0) {
312 return std::unique_ptr<ControllerManagerImpl>(
313 new ControllerManagerImpl(ControllerManagerImpl::Config(0, 0),
314 std::move(controllers), scoring_points));
315 } else {
316 RTC_CHECK(controller_manager_config.has_min_reordering_time_ms());
317 RTC_CHECK(controller_manager_config.has_min_reordering_squared_distance());
318 return std::unique_ptr<ControllerManagerImpl>(new ControllerManagerImpl(
319 ControllerManagerImpl::Config(
320 controller_manager_config.min_reordering_time_ms(),
321 controller_manager_config.min_reordering_squared_distance()),
322 std::move(controllers), scoring_points));
323 }
324
325 #else
326 RTC_DCHECK_NOTREACHED();
327 return nullptr;
328 #endif // WEBRTC_ENABLE_PROTOBUF
329 }
330
ControllerManagerImpl(const Config & config)331 ControllerManagerImpl::ControllerManagerImpl(const Config& config)
332 : ControllerManagerImpl(
333 config,
334 std::vector<std::unique_ptr<Controller>>(),
335 std::map<const Controller*, std::pair<int, float>>()) {}
336
ControllerManagerImpl(const Config & config,std::vector<std::unique_ptr<Controller>> controllers,const std::map<const Controller *,std::pair<int,float>> & scoring_points)337 ControllerManagerImpl::ControllerManagerImpl(
338 const Config& config,
339 std::vector<std::unique_ptr<Controller>> controllers,
340 const std::map<const Controller*, std::pair<int, float>>& scoring_points)
341 : config_(config),
342 controllers_(std::move(controllers)),
343 last_reordering_time_ms_(absl::nullopt),
344 last_scoring_point_(0, 0.0) {
345 for (auto& controller : controllers_)
346 default_sorted_controllers_.push_back(controller.get());
347 sorted_controllers_ = default_sorted_controllers_;
348 for (auto& controller_point : scoring_points) {
349 controller_scoring_points_.insert(std::make_pair(
350 controller_point.first, ScoringPoint(controller_point.second.first,
351 controller_point.second.second)));
352 }
353 }
354
355 ControllerManagerImpl::~ControllerManagerImpl() = default;
356
GetSortedControllers(const Controller::NetworkMetrics & metrics)357 std::vector<Controller*> ControllerManagerImpl::GetSortedControllers(
358 const Controller::NetworkMetrics& metrics) {
359 if (controller_scoring_points_.size() == 0)
360 return default_sorted_controllers_;
361
362 if (!metrics.uplink_bandwidth_bps || !metrics.uplink_packet_loss_fraction)
363 return sorted_controllers_;
364
365 const int64_t now_ms = rtc::TimeMillis();
366 if (last_reordering_time_ms_ &&
367 now_ms - *last_reordering_time_ms_ < config_.min_reordering_time_ms)
368 return sorted_controllers_;
369
370 ScoringPoint scoring_point(*metrics.uplink_bandwidth_bps,
371 *metrics.uplink_packet_loss_fraction);
372
373 if (last_reordering_time_ms_ &&
374 last_scoring_point_.SquaredDistanceTo(scoring_point) <
375 config_.min_reordering_squared_distance)
376 return sorted_controllers_;
377
378 // Sort controllers according to the distances of `scoring_point` to the
379 // scoring points of controllers.
380 //
381 // A controller that does not associate with any scoring point
382 // are treated as if
383 // 1) they are less important than any controller that has a scoring point,
384 // 2) they are equally important to any controller that has no scoring point,
385 // and their relative order will follow `default_sorted_controllers_`.
386 std::vector<Controller*> sorted_controllers(default_sorted_controllers_);
387 std::stable_sort(
388 sorted_controllers.begin(), sorted_controllers.end(),
389 [this, &scoring_point](const Controller* lhs, const Controller* rhs) {
390 auto lhs_scoring_point = controller_scoring_points_.find(lhs);
391 auto rhs_scoring_point = controller_scoring_points_.find(rhs);
392
393 if (lhs_scoring_point == controller_scoring_points_.end())
394 return false;
395
396 if (rhs_scoring_point == controller_scoring_points_.end())
397 return true;
398
399 return lhs_scoring_point->second.SquaredDistanceTo(scoring_point) <
400 rhs_scoring_point->second.SquaredDistanceTo(scoring_point);
401 });
402
403 if (sorted_controllers_ != sorted_controllers) {
404 sorted_controllers_ = sorted_controllers;
405 last_reordering_time_ms_ = now_ms;
406 last_scoring_point_ = scoring_point;
407 }
408 return sorted_controllers_;
409 }
410
GetControllers() const411 std::vector<Controller*> ControllerManagerImpl::GetControllers() const {
412 return default_sorted_controllers_;
413 }
414
ScoringPoint(int uplink_bandwidth_bps,float uplink_packet_loss_fraction)415 ControllerManagerImpl::ScoringPoint::ScoringPoint(
416 int uplink_bandwidth_bps,
417 float uplink_packet_loss_fraction)
418 : uplink_bandwidth_bps(uplink_bandwidth_bps),
419 uplink_packet_loss_fraction(uplink_packet_loss_fraction) {}
420
421 namespace {
422
423 constexpr int kMinUplinkBandwidthBps = 0;
424 constexpr int kMaxUplinkBandwidthBps = 120000;
425
NormalizeUplinkBandwidth(int uplink_bandwidth_bps)426 float NormalizeUplinkBandwidth(int uplink_bandwidth_bps) {
427 uplink_bandwidth_bps =
428 std::min(kMaxUplinkBandwidthBps,
429 std::max(kMinUplinkBandwidthBps, uplink_bandwidth_bps));
430 return static_cast<float>(uplink_bandwidth_bps - kMinUplinkBandwidthBps) /
431 (kMaxUplinkBandwidthBps - kMinUplinkBandwidthBps);
432 }
433
NormalizePacketLossFraction(float uplink_packet_loss_fraction)434 float NormalizePacketLossFraction(float uplink_packet_loss_fraction) {
435 // `uplink_packet_loss_fraction` is seldom larger than 0.3, so we scale it up
436 // by 3.3333f.
437 return std::min(uplink_packet_loss_fraction * 3.3333f, 1.0f);
438 }
439
440 } // namespace
441
SquaredDistanceTo(const ScoringPoint & scoring_point) const442 float ControllerManagerImpl::ScoringPoint::SquaredDistanceTo(
443 const ScoringPoint& scoring_point) const {
444 float diff_normalized_bitrate_bps =
445 NormalizeUplinkBandwidth(scoring_point.uplink_bandwidth_bps) -
446 NormalizeUplinkBandwidth(uplink_bandwidth_bps);
447 float diff_normalized_packet_loss =
448 NormalizePacketLossFraction(scoring_point.uplink_packet_loss_fraction) -
449 NormalizePacketLossFraction(uplink_packet_loss_fraction);
450 return std::pow(diff_normalized_bitrate_bps, 2) +
451 std::pow(diff_normalized_packet_loss, 2);
452 }
453
454 } // namespace webrtc
455