1 /*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "api/video_codecs/video_decoder_software_fallback_wrapper.h"
12
13 #include <stdint.h>
14
15 #include <memory>
16 #include <string>
17 #include <utility>
18
19 #include "api/video/encoded_image.h"
20 #include "api/video_codecs/video_decoder.h"
21 #include "modules/video_coding/include/video_error_codes.h"
22 #include "rtc_base/checks.h"
23 #include "rtc_base/logging.h"
24 #include "rtc_base/trace_event.h"
25 #include "system_wrappers/include/field_trial.h"
26 #include "system_wrappers/include/metrics.h"
27
28 namespace webrtc {
29
30 namespace {
31
32 constexpr size_t kMaxConsequtiveHwErrors = 4;
33
34 class VideoDecoderSoftwareFallbackWrapper final : public VideoDecoder {
35 public:
36 VideoDecoderSoftwareFallbackWrapper(
37 std::unique_ptr<VideoDecoder> sw_fallback_decoder,
38 std::unique_ptr<VideoDecoder> hw_decoder);
39 ~VideoDecoderSoftwareFallbackWrapper() override;
40
41 bool Configure(const Settings& settings) override;
42
43 int32_t Decode(const EncodedImage& input_image,
44 bool missing_frames,
45 int64_t render_time_ms) override;
46
47 int32_t RegisterDecodeCompleteCallback(
48 DecodedImageCallback* callback) override;
49
50 int32_t Release() override;
51
52 DecoderInfo GetDecoderInfo() const override;
53 const char* ImplementationName() const override;
54
55 private:
56 bool InitFallbackDecoder();
57 void UpdateFallbackDecoderHistograms();
58
59 bool InitHwDecoder();
60
61 VideoDecoder& active_decoder() const;
62
63 // Determines if we are trying to use the HW or SW decoder.
64 enum class DecoderType {
65 kNone,
66 kHardware,
67 kFallback,
68 } decoder_type_;
69 std::unique_ptr<VideoDecoder> hw_decoder_;
70
71 Settings decoder_settings_;
72 const std::unique_ptr<VideoDecoder> fallback_decoder_;
73 const std::string fallback_implementation_name_;
74 DecodedImageCallback* callback_;
75 int32_t hw_decoded_frames_since_last_fallback_;
76 size_t hw_consequtive_generic_errors_;
77 };
78
VideoDecoderSoftwareFallbackWrapper(std::unique_ptr<VideoDecoder> sw_fallback_decoder,std::unique_ptr<VideoDecoder> hw_decoder)79 VideoDecoderSoftwareFallbackWrapper::VideoDecoderSoftwareFallbackWrapper(
80 std::unique_ptr<VideoDecoder> sw_fallback_decoder,
81 std::unique_ptr<VideoDecoder> hw_decoder)
82 : decoder_type_(DecoderType::kNone),
83 hw_decoder_(std::move(hw_decoder)),
84 fallback_decoder_(std::move(sw_fallback_decoder)),
85 fallback_implementation_name_(
86 fallback_decoder_->GetDecoderInfo().implementation_name +
87 " (fallback from: " +
88 hw_decoder_->GetDecoderInfo().implementation_name + ")"),
89 callback_(nullptr),
90 hw_decoded_frames_since_last_fallback_(0),
91 hw_consequtive_generic_errors_(0) {}
92 VideoDecoderSoftwareFallbackWrapper::~VideoDecoderSoftwareFallbackWrapper() =
93 default;
94
Configure(const Settings & settings)95 bool VideoDecoderSoftwareFallbackWrapper::Configure(const Settings& settings) {
96 decoder_settings_ = settings;
97
98 if (webrtc::field_trial::IsEnabled("WebRTC-Video-ForcedSwDecoderFallback")) {
99 RTC_LOG(LS_INFO) << "Forced software decoder fallback enabled.";
100 RTC_DCHECK(decoder_type_ == DecoderType::kNone);
101 return InitFallbackDecoder();
102 }
103 if (InitHwDecoder()) {
104 return true;
105 }
106
107 RTC_DCHECK(decoder_type_ == DecoderType::kNone);
108 return InitFallbackDecoder();
109 }
110
InitHwDecoder()111 bool VideoDecoderSoftwareFallbackWrapper::InitHwDecoder() {
112 RTC_DCHECK(decoder_type_ == DecoderType::kNone);
113 if (!hw_decoder_->Configure(decoder_settings_)) {
114 return false;
115 }
116
117 decoder_type_ = DecoderType::kHardware;
118 if (callback_)
119 hw_decoder_->RegisterDecodeCompleteCallback(callback_);
120 return true;
121 }
122
InitFallbackDecoder()123 bool VideoDecoderSoftwareFallbackWrapper::InitFallbackDecoder() {
124 RTC_DCHECK(decoder_type_ == DecoderType::kNone ||
125 decoder_type_ == DecoderType::kHardware);
126 RTC_LOG(LS_WARNING) << "Decoder falling back to software decoding.";
127 if (!fallback_decoder_->Configure(decoder_settings_)) {
128 RTC_LOG(LS_ERROR) << "Failed to initialize software-decoder fallback.";
129 return false;
130 }
131
132 UpdateFallbackDecoderHistograms();
133
134 if (decoder_type_ == DecoderType::kHardware) {
135 hw_decoder_->Release();
136 }
137 decoder_type_ = DecoderType::kFallback;
138
139 if (callback_)
140 fallback_decoder_->RegisterDecodeCompleteCallback(callback_);
141 return true;
142 }
143
UpdateFallbackDecoderHistograms()144 void VideoDecoderSoftwareFallbackWrapper::UpdateFallbackDecoderHistograms() {
145 const std::string kFallbackHistogramsUmaPrefix =
146 "WebRTC.Video.HardwareDecodedFramesBetweenSoftwareFallbacks.";
147 // Each histogram needs its own code path for this to work otherwise the
148 // histogram names will be mixed up by the optimization that takes place.
149 switch (decoder_settings_.codec_type()) {
150 case kVideoCodecGeneric:
151 RTC_HISTOGRAM_COUNTS_100000(kFallbackHistogramsUmaPrefix + "Generic",
152 hw_decoded_frames_since_last_fallback_);
153 break;
154 case kVideoCodecVP8:
155 RTC_HISTOGRAM_COUNTS_100000(kFallbackHistogramsUmaPrefix + "Vp8",
156 hw_decoded_frames_since_last_fallback_);
157 break;
158 case kVideoCodecVP9:
159 RTC_HISTOGRAM_COUNTS_100000(kFallbackHistogramsUmaPrefix + "Vp9",
160 hw_decoded_frames_since_last_fallback_);
161 break;
162 case kVideoCodecAV1:
163 RTC_HISTOGRAM_COUNTS_100000(kFallbackHistogramsUmaPrefix + "Av1",
164 hw_decoded_frames_since_last_fallback_);
165 break;
166 case kVideoCodecH264:
167 RTC_HISTOGRAM_COUNTS_100000(kFallbackHistogramsUmaPrefix + "H264",
168 hw_decoded_frames_since_last_fallback_);
169 break;
170 case kVideoCodecMultiplex:
171 RTC_HISTOGRAM_COUNTS_100000(kFallbackHistogramsUmaPrefix + "Multiplex",
172 hw_decoded_frames_since_last_fallback_);
173 break;
174 }
175 }
176
Decode(const EncodedImage & input_image,bool missing_frames,int64_t render_time_ms)177 int32_t VideoDecoderSoftwareFallbackWrapper::Decode(
178 const EncodedImage& input_image,
179 bool missing_frames,
180 int64_t render_time_ms) {
181 TRACE_EVENT0("webrtc", "VideoDecoderSoftwareFallbackWrapper::Decode");
182 switch (decoder_type_) {
183 case DecoderType::kNone:
184 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
185 case DecoderType::kHardware: {
186 int32_t ret = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
187 ret = hw_decoder_->Decode(input_image, missing_frames, render_time_ms);
188 if (ret != WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE) {
189 if (ret != WEBRTC_VIDEO_CODEC_ERROR) {
190 ++hw_decoded_frames_since_last_fallback_;
191 hw_consequtive_generic_errors_ = 0;
192 return ret;
193 }
194 if (input_image._frameType == VideoFrameType::kVideoFrameKey) {
195 // Only count errors on key-frames, since generic errors can happen
196 // with hw decoder due to many arbitrary reasons.
197 // However, requesting a key-frame is supposed to fix the issue.
198 ++hw_consequtive_generic_errors_;
199 }
200 if (hw_consequtive_generic_errors_ < kMaxConsequtiveHwErrors) {
201 return ret;
202 }
203 }
204
205 // HW decoder returned WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE or
206 // too many generic errors on key-frames encountered.
207 if (!InitFallbackDecoder()) {
208 return ret;
209 }
210
211 // Fallback decoder initialized, fall-through.
212 [[fallthrough]];
213 }
214 case DecoderType::kFallback:
215 return fallback_decoder_->Decode(input_image, missing_frames,
216 render_time_ms);
217 default:
218 RTC_DCHECK_NOTREACHED();
219 return WEBRTC_VIDEO_CODEC_ERROR;
220 }
221 }
222
RegisterDecodeCompleteCallback(DecodedImageCallback * callback)223 int32_t VideoDecoderSoftwareFallbackWrapper::RegisterDecodeCompleteCallback(
224 DecodedImageCallback* callback) {
225 callback_ = callback;
226 return active_decoder().RegisterDecodeCompleteCallback(callback);
227 }
228
Release()229 int32_t VideoDecoderSoftwareFallbackWrapper::Release() {
230 int32_t status;
231 switch (decoder_type_) {
232 case DecoderType::kHardware:
233 status = hw_decoder_->Release();
234 break;
235 case DecoderType::kFallback:
236 RTC_LOG(LS_INFO) << "Releasing software fallback decoder.";
237 status = fallback_decoder_->Release();
238 break;
239 case DecoderType::kNone:
240 status = WEBRTC_VIDEO_CODEC_OK;
241 break;
242 default:
243 RTC_DCHECK_NOTREACHED();
244 status = WEBRTC_VIDEO_CODEC_ERROR;
245 }
246
247 decoder_type_ = DecoderType::kNone;
248 return status;
249 }
250
GetDecoderInfo() const251 VideoDecoder::DecoderInfo VideoDecoderSoftwareFallbackWrapper::GetDecoderInfo()
252 const {
253 DecoderInfo info = active_decoder().GetDecoderInfo();
254 if (decoder_type_ == DecoderType::kFallback) {
255 // Cached "A (fallback from B)" string.
256 info.implementation_name = fallback_implementation_name_;
257 }
258 return info;
259 }
260
ImplementationName() const261 const char* VideoDecoderSoftwareFallbackWrapper::ImplementationName() const {
262 if (decoder_type_ == DecoderType::kFallback) {
263 // Cached "A (fallback from B)" string.
264 return fallback_implementation_name_.c_str();
265 } else {
266 return hw_decoder_->ImplementationName();
267 }
268 }
269
active_decoder() const270 VideoDecoder& VideoDecoderSoftwareFallbackWrapper::active_decoder() const {
271 return decoder_type_ == DecoderType::kFallback ? *fallback_decoder_
272 : *hw_decoder_;
273 }
274
275 } // namespace
276
CreateVideoDecoderSoftwareFallbackWrapper(std::unique_ptr<VideoDecoder> sw_fallback_decoder,std::unique_ptr<VideoDecoder> hw_decoder)277 std::unique_ptr<VideoDecoder> CreateVideoDecoderSoftwareFallbackWrapper(
278 std::unique_ptr<VideoDecoder> sw_fallback_decoder,
279 std::unique_ptr<VideoDecoder> hw_decoder) {
280 return std::make_unique<VideoDecoderSoftwareFallbackWrapper>(
281 std::move(sw_fallback_decoder), std::move(hw_decoder));
282 }
283
284 } // namespace webrtc
285