1 /*
2 * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/video_coding/utility/quality_scaler.h"
12
13 #include <memory>
14 #include <utility>
15
16 #include "api/units/time_delta.h"
17 #include "api/video/video_adaptation_reason.h"
18 #include "rtc_base/checks.h"
19 #include "rtc_base/experiments/quality_scaler_settings.h"
20 #include "rtc_base/logging.h"
21 #include "rtc_base/numerics/exp_filter.h"
22 #include "rtc_base/weak_ptr.h"
23
24 namespace webrtc {
25
26 namespace {
27 // Threshold constant used until first downscale (to permit fast rampup).
28 static const int kMeasureMs = 2000;
29 static const float kSamplePeriodScaleFactor = 2.5;
30 static const int kFramedropPercentThreshold = 60;
31 static const size_t kMinFramesNeededToScale = 2 * 30;
32
33 } // namespace
34
35 class QualityScaler::QpSmoother {
36 public:
QpSmoother(float alpha)37 explicit QpSmoother(float alpha)
38 : alpha_(alpha),
39 // The initial value of last_sample_ms doesn't matter since the smoother
40 // will ignore the time delta for the first update.
41 last_sample_ms_(0),
42 smoother_(alpha) {}
43
GetAvg() const44 absl::optional<int> GetAvg() const {
45 float value = smoother_.filtered();
46 if (value == rtc::ExpFilter::kValueUndefined) {
47 return absl::nullopt;
48 }
49 return static_cast<int>(value);
50 }
51
Add(float sample,int64_t time_sent_us)52 void Add(float sample, int64_t time_sent_us) {
53 int64_t now_ms = time_sent_us / 1000;
54 smoother_.Apply(static_cast<float>(now_ms - last_sample_ms_), sample);
55 last_sample_ms_ = now_ms;
56 }
57
Reset()58 void Reset() { smoother_.Reset(alpha_); }
59
60 private:
61 const float alpha_;
62 int64_t last_sample_ms_;
63 rtc::ExpFilter smoother_;
64 };
65
66 // The QualityScaler checks for QP periodically by queuing CheckQpTasks. The
67 // task will either run to completion and trigger a new task being queued, or it
68 // will be destroyed because the QualityScaler is destroyed.
69 //
70 // When high or low QP is reported, the task will be pending until a callback is
71 // invoked. This lets the QualityScalerQpUsageHandlerInterface react to QP usage
72 // asynchronously and prevents checking for QP until the stream has potentially
73 // been reconfigured.
74 class QualityScaler::CheckQpTask {
75 public:
76 // The result of one CheckQpTask may influence the delay of the next
77 // CheckQpTask.
78 struct Result {
79 bool observed_enough_frames = false;
80 bool qp_usage_reported = false;
81 };
82
CheckQpTask(QualityScaler * quality_scaler,Result previous_task_result)83 CheckQpTask(QualityScaler* quality_scaler, Result previous_task_result)
84 : quality_scaler_(quality_scaler),
85 state_(State::kNotStarted),
86 previous_task_result_(previous_task_result),
87 weak_ptr_factory_(this) {}
88
StartDelayedTask()89 void StartDelayedTask() {
90 RTC_DCHECK_EQ(state_, State::kNotStarted);
91 state_ = State::kCheckingQp;
92 TaskQueueBase::Current()->PostDelayedTask(
93 [this_weak_ptr = weak_ptr_factory_.GetWeakPtr(), this] {
94 if (!this_weak_ptr) {
95 // The task has been cancelled through destruction.
96 return;
97 }
98 RTC_DCHECK_EQ(state_, State::kCheckingQp);
99 RTC_DCHECK_RUN_ON(&quality_scaler_->task_checker_);
100 switch (quality_scaler_->CheckQp()) {
101 case QualityScaler::CheckQpResult::kInsufficientSamples: {
102 result_.observed_enough_frames = false;
103 // After this line, `this` may be deleted.
104 break;
105 }
106 case QualityScaler::CheckQpResult::kNormalQp: {
107 result_.observed_enough_frames = true;
108 break;
109 }
110 case QualityScaler::CheckQpResult::kHighQp: {
111 result_.observed_enough_frames = true;
112 result_.qp_usage_reported = true;
113 quality_scaler_->fast_rampup_ = false;
114 quality_scaler_->handler_->OnReportQpUsageHigh();
115 quality_scaler_->ClearSamples();
116 break;
117 }
118 case QualityScaler::CheckQpResult::kLowQp: {
119 result_.observed_enough_frames = true;
120 result_.qp_usage_reported = true;
121 quality_scaler_->handler_->OnReportQpUsageLow();
122 quality_scaler_->ClearSamples();
123 break;
124 }
125 }
126 state_ = State::kCompleted;
127 // Starting the next task deletes the pending task. After this line,
128 // `this` has been deleted.
129 quality_scaler_->StartNextCheckQpTask();
130 },
131 TimeDelta::Millis(GetCheckingQpDelayMs()));
132 }
133
HasCompletedTask() const134 bool HasCompletedTask() const { return state_ == State::kCompleted; }
135
result() const136 Result result() const {
137 RTC_DCHECK(HasCompletedTask());
138 return result_;
139 }
140
141 private:
142 enum class State {
143 kNotStarted,
144 kCheckingQp,
145 kCompleted,
146 };
147
148 // Determines the sampling period of CheckQpTasks.
GetCheckingQpDelayMs() const149 int64_t GetCheckingQpDelayMs() const {
150 RTC_DCHECK_RUN_ON(&quality_scaler_->task_checker_);
151 if (quality_scaler_->fast_rampup_) {
152 return quality_scaler_->sampling_period_ms_;
153 }
154 if (quality_scaler_->experiment_enabled_ &&
155 !previous_task_result_.observed_enough_frames) {
156 // Use half the interval while waiting for enough frames.
157 return quality_scaler_->sampling_period_ms_ / 2;
158 }
159 if (quality_scaler_->scale_factor_ &&
160 !previous_task_result_.qp_usage_reported) {
161 // Last CheckQp did not call AdaptDown/Up, possibly reduce interval.
162 return quality_scaler_->sampling_period_ms_ *
163 quality_scaler_->scale_factor_.value();
164 }
165 return quality_scaler_->sampling_period_ms_ *
166 quality_scaler_->initial_scale_factor_;
167 }
168
169 QualityScaler* const quality_scaler_;
170 State state_;
171 const Result previous_task_result_;
172 Result result_;
173
174 rtc::WeakPtrFactory<CheckQpTask> weak_ptr_factory_;
175 };
176
QualityScaler(QualityScalerQpUsageHandlerInterface * handler,VideoEncoder::QpThresholds thresholds)177 QualityScaler::QualityScaler(QualityScalerQpUsageHandlerInterface* handler,
178 VideoEncoder::QpThresholds thresholds)
179 : QualityScaler(handler, thresholds, kMeasureMs) {}
180
181 // Protected ctor, should not be called directly.
QualityScaler(QualityScalerQpUsageHandlerInterface * handler,VideoEncoder::QpThresholds thresholds,int64_t default_sampling_period_ms)182 QualityScaler::QualityScaler(QualityScalerQpUsageHandlerInterface* handler,
183 VideoEncoder::QpThresholds thresholds,
184 int64_t default_sampling_period_ms)
185 : handler_(handler),
186 thresholds_(thresholds),
187 sampling_period_ms_(QualityScalerSettings::ParseFromFieldTrials()
188 .SamplingPeriodMs()
189 .value_or(default_sampling_period_ms)),
190 fast_rampup_(true),
191 // Arbitrarily choose size based on 30 fps for 5 seconds.
192 average_qp_(QualityScalerSettings::ParseFromFieldTrials()
193 .AverageQpWindow()
194 .value_or(5 * 30)),
195 framedrop_percent_media_opt_(5 * 30),
196 framedrop_percent_all_(5 * 30),
197 experiment_enabled_(QualityScalingExperiment::Enabled()),
198 min_frames_needed_(
199 QualityScalerSettings::ParseFromFieldTrials().MinFrames().value_or(
200 kMinFramesNeededToScale)),
201 initial_scale_factor_(QualityScalerSettings::ParseFromFieldTrials()
202 .InitialScaleFactor()
203 .value_or(kSamplePeriodScaleFactor)),
204 scale_factor_(
205 QualityScalerSettings::ParseFromFieldTrials().ScaleFactor()) {
206 RTC_DCHECK_RUN_ON(&task_checker_);
207 if (experiment_enabled_) {
208 config_ = QualityScalingExperiment::GetConfig();
209 qp_smoother_high_.reset(new QpSmoother(config_.alpha_high));
210 qp_smoother_low_.reset(new QpSmoother(config_.alpha_low));
211 }
212 RTC_DCHECK(handler_ != nullptr);
213 StartNextCheckQpTask();
214 RTC_LOG(LS_INFO) << "QP thresholds: low: " << thresholds_.low
215 << ", high: " << thresholds_.high;
216 }
217
~QualityScaler()218 QualityScaler::~QualityScaler() {
219 RTC_DCHECK_RUN_ON(&task_checker_);
220 }
221
StartNextCheckQpTask()222 void QualityScaler::StartNextCheckQpTask() {
223 RTC_DCHECK_RUN_ON(&task_checker_);
224 RTC_DCHECK(!pending_qp_task_ || pending_qp_task_->HasCompletedTask())
225 << "A previous CheckQpTask has not completed yet!";
226 CheckQpTask::Result previous_task_result;
227 if (pending_qp_task_) {
228 previous_task_result = pending_qp_task_->result();
229 }
230 pending_qp_task_ = std::make_unique<CheckQpTask>(this, previous_task_result);
231 pending_qp_task_->StartDelayedTask();
232 }
233
SetQpThresholds(VideoEncoder::QpThresholds thresholds)234 void QualityScaler::SetQpThresholds(VideoEncoder::QpThresholds thresholds) {
235 RTC_DCHECK_RUN_ON(&task_checker_);
236 thresholds_ = thresholds;
237 }
238
ReportDroppedFrameByMediaOpt()239 void QualityScaler::ReportDroppedFrameByMediaOpt() {
240 RTC_DCHECK_RUN_ON(&task_checker_);
241 framedrop_percent_media_opt_.AddSample(100);
242 framedrop_percent_all_.AddSample(100);
243 }
244
ReportDroppedFrameByEncoder()245 void QualityScaler::ReportDroppedFrameByEncoder() {
246 RTC_DCHECK_RUN_ON(&task_checker_);
247 framedrop_percent_all_.AddSample(100);
248 }
249
ReportQp(int qp,int64_t time_sent_us)250 void QualityScaler::ReportQp(int qp, int64_t time_sent_us) {
251 RTC_DCHECK_RUN_ON(&task_checker_);
252 framedrop_percent_media_opt_.AddSample(0);
253 framedrop_percent_all_.AddSample(0);
254 average_qp_.AddSample(qp);
255 if (qp_smoother_high_)
256 qp_smoother_high_->Add(qp, time_sent_us);
257 if (qp_smoother_low_)
258 qp_smoother_low_->Add(qp, time_sent_us);
259 }
260
QpFastFilterLow() const261 bool QualityScaler::QpFastFilterLow() const {
262 RTC_DCHECK_RUN_ON(&task_checker_);
263 size_t num_frames = config_.use_all_drop_reasons
264 ? framedrop_percent_all_.Size()
265 : framedrop_percent_media_opt_.Size();
266 const size_t kMinNumFrames = 10;
267 if (num_frames < kMinNumFrames) {
268 return false; // Wait for more frames before making a decision.
269 }
270 absl::optional<int> avg_qp_high = qp_smoother_high_
271 ? qp_smoother_high_->GetAvg()
272 : average_qp_.GetAverageRoundedDown();
273 return (avg_qp_high) ? (avg_qp_high.value() <= thresholds_.low) : false;
274 }
275
CheckQp() const276 QualityScaler::CheckQpResult QualityScaler::CheckQp() const {
277 RTC_DCHECK_RUN_ON(&task_checker_);
278 // Should be set through InitEncode -> Should be set by now.
279 RTC_DCHECK_GE(thresholds_.low, 0);
280
281 // If we have not observed at least this many frames we can't make a good
282 // scaling decision.
283 const size_t frames = config_.use_all_drop_reasons
284 ? framedrop_percent_all_.Size()
285 : framedrop_percent_media_opt_.Size();
286 if (frames < min_frames_needed_) {
287 return CheckQpResult::kInsufficientSamples;
288 }
289
290 // Check if we should scale down due to high frame drop.
291 const absl::optional<int> drop_rate =
292 config_.use_all_drop_reasons
293 ? framedrop_percent_all_.GetAverageRoundedDown()
294 : framedrop_percent_media_opt_.GetAverageRoundedDown();
295 if (drop_rate && *drop_rate >= kFramedropPercentThreshold) {
296 RTC_LOG(LS_INFO) << "Reporting high QP, framedrop percent " << *drop_rate;
297 return CheckQpResult::kHighQp;
298 }
299
300 // Check if we should scale up or down based on QP.
301 const absl::optional<int> avg_qp_high =
302 qp_smoother_high_ ? qp_smoother_high_->GetAvg()
303 : average_qp_.GetAverageRoundedDown();
304 const absl::optional<int> avg_qp_low =
305 qp_smoother_low_ ? qp_smoother_low_->GetAvg()
306 : average_qp_.GetAverageRoundedDown();
307 if (avg_qp_high && avg_qp_low) {
308 RTC_LOG(LS_INFO) << "Checking average QP " << *avg_qp_high << " ("
309 << *avg_qp_low << ").";
310 if (*avg_qp_high > thresholds_.high) {
311 return CheckQpResult::kHighQp;
312 }
313 if (*avg_qp_low <= thresholds_.low) {
314 // QP has been low. We want to try a higher resolution.
315 return CheckQpResult::kLowQp;
316 }
317 }
318 return CheckQpResult::kNormalQp;
319 }
320
ClearSamples()321 void QualityScaler::ClearSamples() {
322 RTC_DCHECK_RUN_ON(&task_checker_);
323 framedrop_percent_media_opt_.Reset();
324 framedrop_percent_all_.Reset();
325 average_qp_.Reset();
326 if (qp_smoother_high_)
327 qp_smoother_high_->Reset();
328 if (qp_smoother_low_)
329 qp_smoother_low_->Reset();
330 }
331
~QualityScalerQpUsageHandlerInterface()332 QualityScalerQpUsageHandlerInterface::~QualityScalerQpUsageHandlerInterface() {}
333
334 } // namespace webrtc
335