1 /*
2 * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/video_coding/codecs/vp8/libvpx_vp8_decoder.h"
12
13 #include <stdio.h>
14 #include <string.h>
15
16 #include <algorithm>
17 #include <memory>
18 #include <string>
19
20 #include "absl/types/optional.h"
21 #include "api/scoped_refptr.h"
22 #include "api/video/i420_buffer.h"
23 #include "api/video/video_frame.h"
24 #include "api/video/video_frame_buffer.h"
25 #include "api/video/video_rotation.h"
26 #include "modules/video_coding/codecs/vp8/include/vp8.h"
27 #include "modules/video_coding/include/video_error_codes.h"
28 #include "rtc_base/checks.h"
29 #include "rtc_base/numerics/exp_filter.h"
30 #include "rtc_base/time_utils.h"
31 #include "system_wrappers/include/field_trial.h"
32 #include "system_wrappers/include/metrics.h"
33 #include "third_party/libyuv/include/libyuv/convert.h"
34 #include "vpx/vp8.h"
35 #include "vpx/vp8dx.h"
36 #include "vpx/vpx_decoder.h"
37
38 namespace webrtc {
39 namespace {
40 constexpr int kVp8ErrorPropagationTh = 30;
41 // vpx_decoder.h documentation indicates decode deadline is time in us, with
42 // "Set to zero for unlimited.", but actual implementation requires this to be
43 // a mode with 0 meaning allow delay and 1 not allowing it.
44 constexpr long kDecodeDeadlineRealtime = 1; // NOLINT
45
46 const char kVp8PostProcArmFieldTrial[] = "WebRTC-VP8-Postproc-Config-Arm";
47 const char kVp8PostProcFieldTrial[] = "WebRTC-VP8-Postproc-Config";
48
49 #if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) || \
50 defined(WEBRTC_ANDROID)
51 constexpr bool kIsArm = true;
52 #else
53 constexpr bool kIsArm = false;
54 #endif
55
DefaultDeblockParams()56 absl::optional<LibvpxVp8Decoder::DeblockParams> DefaultDeblockParams() {
57 return LibvpxVp8Decoder::DeblockParams(/*max_level=*/8,
58 /*degrade_qp=*/60,
59 /*min_qp=*/30);
60 }
61
62 absl::optional<LibvpxVp8Decoder::DeblockParams>
GetPostProcParamsFromFieldTrialGroup()63 GetPostProcParamsFromFieldTrialGroup() {
64 std::string group = webrtc::field_trial::FindFullName(
65 kIsArm ? kVp8PostProcArmFieldTrial : kVp8PostProcFieldTrial);
66 if (group.empty()) {
67 return DefaultDeblockParams();
68 }
69
70 LibvpxVp8Decoder::DeblockParams params;
71 if (sscanf(group.c_str(), "Enabled-%d,%d,%d", ¶ms.max_level,
72 ¶ms.min_qp, ¶ms.degrade_qp) != 3) {
73 return DefaultDeblockParams();
74 }
75
76 if (params.max_level < 0 || params.max_level > 16) {
77 return DefaultDeblockParams();
78 }
79
80 if (params.min_qp < 0 || params.degrade_qp <= params.min_qp) {
81 return DefaultDeblockParams();
82 }
83
84 return params;
85 }
86
87 } // namespace
88
Create()89 std::unique_ptr<VideoDecoder> VP8Decoder::Create() {
90 return std::make_unique<LibvpxVp8Decoder>();
91 }
92
93 class LibvpxVp8Decoder::QpSmoother {
94 public:
QpSmoother()95 QpSmoother() : last_sample_ms_(rtc::TimeMillis()), smoother_(kAlpha) {}
96
GetAvg() const97 int GetAvg() const {
98 float value = smoother_.filtered();
99 return (value == rtc::ExpFilter::kValueUndefined) ? 0
100 : static_cast<int>(value);
101 }
102
Add(float sample)103 void Add(float sample) {
104 int64_t now_ms = rtc::TimeMillis();
105 smoother_.Apply(static_cast<float>(now_ms - last_sample_ms_), sample);
106 last_sample_ms_ = now_ms;
107 }
108
Reset()109 void Reset() { smoother_.Reset(kAlpha); }
110
111 private:
112 const float kAlpha = 0.95f;
113 int64_t last_sample_ms_;
114 rtc::ExpFilter smoother_;
115 };
116
LibvpxVp8Decoder()117 LibvpxVp8Decoder::LibvpxVp8Decoder()
118 : use_postproc_(
119 kIsArm ? webrtc::field_trial::IsEnabled(kVp8PostProcArmFieldTrial)
120 : true),
121 buffer_pool_(false, 300 /* max_number_of_buffers*/),
122 decode_complete_callback_(NULL),
123 inited_(false),
124 decoder_(NULL),
125 propagation_cnt_(-1),
126 last_frame_width_(0),
127 last_frame_height_(0),
128 key_frame_required_(true),
129 deblock_params_(use_postproc_ ? GetPostProcParamsFromFieldTrialGroup()
130 : absl::nullopt),
131 qp_smoother_(use_postproc_ ? new QpSmoother() : nullptr) {}
132
~LibvpxVp8Decoder()133 LibvpxVp8Decoder::~LibvpxVp8Decoder() {
134 inited_ = true; // in order to do the actual release
135 Release();
136 }
137
Configure(const Settings & settings)138 bool LibvpxVp8Decoder::Configure(const Settings& settings) {
139 if (Release() < 0) {
140 return false;
141 }
142 if (decoder_ == NULL) {
143 decoder_ = new vpx_codec_ctx_t;
144 memset(decoder_, 0, sizeof(*decoder_));
145 }
146 vpx_codec_dec_cfg_t cfg;
147 // Setting number of threads to a constant value (1)
148 cfg.threads = 1;
149 cfg.h = cfg.w = 0; // set after decode
150
151 vpx_codec_flags_t flags = use_postproc_ ? VPX_CODEC_USE_POSTPROC : 0;
152
153 if (vpx_codec_dec_init(decoder_, vpx_codec_vp8_dx(), &cfg, flags)) {
154 delete decoder_;
155 decoder_ = nullptr;
156 return false;
157 }
158
159 propagation_cnt_ = -1;
160 inited_ = true;
161
162 // Always start with a complete key frame.
163 key_frame_required_ = true;
164 if (absl::optional<int> buffer_pool_size = settings.buffer_pool_size()) {
165 if (!buffer_pool_.Resize(*buffer_pool_size)) {
166 return false;
167 }
168 }
169 return true;
170 }
171
Decode(const EncodedImage & input_image,bool missing_frames,int64_t)172 int LibvpxVp8Decoder::Decode(const EncodedImage& input_image,
173 bool missing_frames,
174 int64_t /*render_time_ms*/) {
175 if (!inited_) {
176 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
177 }
178 if (decode_complete_callback_ == NULL) {
179 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
180 }
181 if (input_image.data() == NULL && input_image.size() > 0) {
182 // Reset to avoid requesting key frames too often.
183 if (propagation_cnt_ > 0)
184 propagation_cnt_ = 0;
185 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
186 }
187
188 // Post process configurations.
189 if (use_postproc_) {
190 vp8_postproc_cfg_t ppcfg;
191 // MFQE enabled to reduce key frame popping.
192 ppcfg.post_proc_flag = VP8_MFQE;
193
194 if (kIsArm) {
195 RTC_DCHECK(deblock_params_.has_value());
196 }
197 if (deblock_params_.has_value()) {
198 // For low resolutions, use stronger deblocking filter.
199 int last_width_x_height = last_frame_width_ * last_frame_height_;
200 if (last_width_x_height > 0 && last_width_x_height <= 320 * 240) {
201 // Enable the deblock and demacroblocker based on qp thresholds.
202 RTC_DCHECK(qp_smoother_);
203 int qp = qp_smoother_->GetAvg();
204 if (qp > deblock_params_->min_qp) {
205 int level = deblock_params_->max_level;
206 if (qp < deblock_params_->degrade_qp) {
207 // Use lower level.
208 level = deblock_params_->max_level *
209 (qp - deblock_params_->min_qp) /
210 (deblock_params_->degrade_qp - deblock_params_->min_qp);
211 }
212 // Deblocking level only affects VP8_DEMACROBLOCK.
213 ppcfg.deblocking_level = std::max(level, 1);
214 ppcfg.post_proc_flag |= VP8_DEBLOCK | VP8_DEMACROBLOCK;
215 }
216 }
217 } else {
218 // Non-arm with no explicit deblock params set.
219 ppcfg.post_proc_flag |= VP8_DEBLOCK;
220 // For VGA resolutions and lower, enable the demacroblocker postproc.
221 if (last_frame_width_ * last_frame_height_ <= 640 * 360) {
222 ppcfg.post_proc_flag |= VP8_DEMACROBLOCK;
223 }
224 // Strength of deblocking filter. Valid range:[0,16]
225 ppcfg.deblocking_level = 3;
226 }
227
228 vpx_codec_control(decoder_, VP8_SET_POSTPROC, &ppcfg);
229 }
230
231 // Always start with a complete key frame.
232 if (key_frame_required_) {
233 if (input_image._frameType != VideoFrameType::kVideoFrameKey)
234 return WEBRTC_VIDEO_CODEC_ERROR;
235 key_frame_required_ = false;
236 }
237 // Restrict error propagation using key frame requests.
238 // Reset on a key frame refresh.
239 if (input_image._frameType == VideoFrameType::kVideoFrameKey) {
240 propagation_cnt_ = -1;
241 // Start count on first loss.
242 } else if (missing_frames && propagation_cnt_ == -1) {
243 propagation_cnt_ = 0;
244 }
245 if (propagation_cnt_ >= 0) {
246 propagation_cnt_++;
247 }
248
249 vpx_codec_iter_t iter = NULL;
250 vpx_image_t* img;
251 int ret;
252
253 // Check for missing frames.
254 if (missing_frames) {
255 // Call decoder with zero data length to signal missing frames.
256 if (vpx_codec_decode(decoder_, NULL, 0, 0, kDecodeDeadlineRealtime)) {
257 // Reset to avoid requesting key frames too often.
258 if (propagation_cnt_ > 0)
259 propagation_cnt_ = 0;
260 return WEBRTC_VIDEO_CODEC_ERROR;
261 }
262 img = vpx_codec_get_frame(decoder_, &iter);
263 iter = NULL;
264 }
265
266 const uint8_t* buffer = input_image.data();
267 if (input_image.size() == 0) {
268 buffer = NULL; // Triggers full frame concealment.
269 }
270 if (vpx_codec_decode(decoder_, buffer, input_image.size(), 0,
271 kDecodeDeadlineRealtime)) {
272 // Reset to avoid requesting key frames too often.
273 if (propagation_cnt_ > 0) {
274 propagation_cnt_ = 0;
275 }
276 return WEBRTC_VIDEO_CODEC_ERROR;
277 }
278
279 img = vpx_codec_get_frame(decoder_, &iter);
280 int qp;
281 vpx_codec_err_t vpx_ret =
282 vpx_codec_control(decoder_, VPXD_GET_LAST_QUANTIZER, &qp);
283 RTC_DCHECK_EQ(vpx_ret, VPX_CODEC_OK);
284 ret = ReturnFrame(img, input_image.Timestamp(), qp, input_image.ColorSpace());
285 if (ret != 0) {
286 // Reset to avoid requesting key frames too often.
287 if (ret < 0 && propagation_cnt_ > 0)
288 propagation_cnt_ = 0;
289 return ret;
290 }
291 // Check Vs. threshold
292 if (propagation_cnt_ > kVp8ErrorPropagationTh) {
293 // Reset to avoid requesting key frames too often.
294 propagation_cnt_ = 0;
295 return WEBRTC_VIDEO_CODEC_ERROR;
296 }
297 return WEBRTC_VIDEO_CODEC_OK;
298 }
299
ReturnFrame(const vpx_image_t * img,uint32_t timestamp,int qp,const webrtc::ColorSpace * explicit_color_space)300 int LibvpxVp8Decoder::ReturnFrame(
301 const vpx_image_t* img,
302 uint32_t timestamp,
303 int qp,
304 const webrtc::ColorSpace* explicit_color_space) {
305 if (img == NULL) {
306 // Decoder OK and NULL image => No show frame
307 return WEBRTC_VIDEO_CODEC_NO_OUTPUT;
308 }
309 if (qp_smoother_) {
310 if (last_frame_width_ != static_cast<int>(img->d_w) ||
311 last_frame_height_ != static_cast<int>(img->d_h)) {
312 qp_smoother_->Reset();
313 }
314 qp_smoother_->Add(qp);
315 }
316 last_frame_width_ = img->d_w;
317 last_frame_height_ = img->d_h;
318 // Allocate memory for decoded image.
319 rtc::scoped_refptr<VideoFrameBuffer> buffer;
320
321 rtc::scoped_refptr<I420Buffer> i420_buffer =
322 buffer_pool_.CreateI420Buffer(img->d_w, img->d_h);
323 buffer = i420_buffer;
324 if (i420_buffer.get()) {
325 libyuv::I420Copy(img->planes[VPX_PLANE_Y], img->stride[VPX_PLANE_Y],
326 img->planes[VPX_PLANE_U], img->stride[VPX_PLANE_U],
327 img->planes[VPX_PLANE_V], img->stride[VPX_PLANE_V],
328 i420_buffer->MutableDataY(), i420_buffer->StrideY(),
329 i420_buffer->MutableDataU(), i420_buffer->StrideU(),
330 i420_buffer->MutableDataV(), i420_buffer->StrideV(),
331 img->d_w, img->d_h);
332 }
333
334 if (!buffer.get()) {
335 // Pool has too many pending frames.
336 RTC_HISTOGRAM_BOOLEAN("WebRTC.Video.LibvpxVp8Decoder.TooManyPendingFrames",
337 1);
338 return WEBRTC_VIDEO_CODEC_NO_OUTPUT;
339 }
340
341 VideoFrame decoded_image = VideoFrame::Builder()
342 .set_video_frame_buffer(buffer)
343 .set_timestamp_rtp(timestamp)
344 .set_color_space(explicit_color_space)
345 .build();
346 decode_complete_callback_->Decoded(decoded_image, absl::nullopt, qp);
347
348 return WEBRTC_VIDEO_CODEC_OK;
349 }
350
RegisterDecodeCompleteCallback(DecodedImageCallback * callback)351 int LibvpxVp8Decoder::RegisterDecodeCompleteCallback(
352 DecodedImageCallback* callback) {
353 decode_complete_callback_ = callback;
354 return WEBRTC_VIDEO_CODEC_OK;
355 }
356
Release()357 int LibvpxVp8Decoder::Release() {
358 int ret_val = WEBRTC_VIDEO_CODEC_OK;
359
360 if (decoder_ != NULL) {
361 if (inited_) {
362 if (vpx_codec_destroy(decoder_)) {
363 ret_val = WEBRTC_VIDEO_CODEC_MEMORY;
364 }
365 }
366 delete decoder_;
367 decoder_ = NULL;
368 }
369 buffer_pool_.Release();
370 inited_ = false;
371 return ret_val;
372 }
373
GetDecoderInfo() const374 VideoDecoder::DecoderInfo LibvpxVp8Decoder::GetDecoderInfo() const {
375 DecoderInfo info;
376 info.implementation_name = "libvpx";
377 info.is_hardware_accelerated = false;
378 return info;
379 }
380
ImplementationName() const381 const char* LibvpxVp8Decoder::ImplementationName() const {
382 return "libvpx";
383 }
384 } // namespace webrtc
385