1 /*
2 * Copyright 2017 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "sdk/android/src/jni/video_encoder_wrapper.h"
12
13 #include <utility>
14
15 #include "common_video/h264/h264_common.h"
16 #include "modules/video_coding/include/video_codec_interface.h"
17 #include "modules/video_coding/include/video_error_codes.h"
18 #include "modules/video_coding/svc/scalable_video_controller_no_layering.h"
19 #include "modules/video_coding/utility/vp8_header_parser.h"
20 #include "modules/video_coding/utility/vp9_uncompressed_header_parser.h"
21 #include "rtc_base/logging.h"
22 #include "rtc_base/time_utils.h"
23 #include "sdk/android/generated_video_jni/VideoEncoderWrapper_jni.h"
24 #include "sdk/android/generated_video_jni/VideoEncoder_jni.h"
25 #include "sdk/android/native_api/jni/class_loader.h"
26 #include "sdk/android/native_api/jni/java_types.h"
27 #include "sdk/android/src/jni/encoded_image.h"
28 #include "sdk/android/src/jni/video_codec_status.h"
29 #include "sdk/android/src/jni/video_frame.h"
30
31 namespace webrtc {
32 namespace jni {
33
VideoEncoderWrapper(JNIEnv * jni,const JavaRef<jobject> & j_encoder)34 VideoEncoderWrapper::VideoEncoderWrapper(JNIEnv* jni,
35 const JavaRef<jobject>& j_encoder)
36 : encoder_(jni, j_encoder), int_array_class_(GetClass(jni, "[I")) {
37 initialized_ = false;
38 num_resets_ = 0;
39
40 // Fetch and update encoder info.
41 UpdateEncoderInfo(jni);
42 }
43 VideoEncoderWrapper::~VideoEncoderWrapper() = default;
44
InitEncode(const VideoCodec * codec_settings,const Settings & settings)45 int VideoEncoderWrapper::InitEncode(const VideoCodec* codec_settings,
46 const Settings& settings) {
47 JNIEnv* jni = AttachCurrentThreadIfNeeded();
48
49 codec_settings_ = *codec_settings;
50 capabilities_ = settings.capabilities;
51 number_of_cores_ = settings.number_of_cores;
52 num_resets_ = 0;
53
54 return InitEncodeInternal(jni);
55 }
56
InitEncodeInternal(JNIEnv * jni)57 int32_t VideoEncoderWrapper::InitEncodeInternal(JNIEnv* jni) {
58 bool automatic_resize_on;
59 switch (codec_settings_.codecType) {
60 case kVideoCodecVP8:
61 automatic_resize_on = codec_settings_.VP8()->automaticResizeOn;
62 break;
63 case kVideoCodecVP9:
64 automatic_resize_on = codec_settings_.VP9()->automaticResizeOn;
65 gof_.SetGofInfoVP9(TemporalStructureMode::kTemporalStructureMode1);
66 gof_idx_ = 0;
67 break;
68 default:
69 automatic_resize_on = true;
70 }
71
72 RTC_DCHECK(capabilities_);
73 ScopedJavaLocalRef<jobject> capabilities =
74 Java_Capabilities_Constructor(jni, capabilities_->loss_notification);
75
76 ScopedJavaLocalRef<jobject> settings = Java_Settings_Constructor(
77 jni, number_of_cores_, codec_settings_.width, codec_settings_.height,
78 static_cast<int>(codec_settings_.startBitrate),
79 static_cast<int>(codec_settings_.maxFramerate),
80 static_cast<int>(codec_settings_.numberOfSimulcastStreams),
81 automatic_resize_on, capabilities);
82
83 ScopedJavaLocalRef<jobject> callback =
84 Java_VideoEncoderWrapper_createEncoderCallback(jni,
85 jlongFromPointer(this));
86
87 int32_t status = JavaToNativeVideoCodecStatus(
88 jni, Java_VideoEncoder_initEncode(jni, encoder_, settings, callback));
89 RTC_LOG(LS_INFO) << "initEncode: " << status;
90
91 // Some encoder's properties depend on settings and may change after
92 // initialization.
93 UpdateEncoderInfo(jni);
94
95 if (status == WEBRTC_VIDEO_CODEC_OK) {
96 initialized_ = true;
97 }
98 return status;
99 }
100
UpdateEncoderInfo(JNIEnv * jni)101 void VideoEncoderWrapper::UpdateEncoderInfo(JNIEnv* jni) {
102 encoder_info_.supports_native_handle = true;
103
104 encoder_info_.implementation_name = JavaToStdString(
105 jni, Java_VideoEncoder_getImplementationName(jni, encoder_));
106
107 encoder_info_.is_hardware_accelerated =
108 Java_VideoEncoder_isHardwareEncoder(jni, encoder_);
109
110 encoder_info_.scaling_settings = GetScalingSettingsInternal(jni);
111
112 encoder_info_.resolution_bitrate_limits = JavaToNativeResolutionBitrateLimits(
113 jni, Java_VideoEncoder_getResolutionBitrateLimits(jni, encoder_));
114
115 EncoderInfo info = GetEncoderInfoInternal(jni);
116 encoder_info_.requested_resolution_alignment =
117 info.requested_resolution_alignment;
118 encoder_info_.apply_alignment_to_all_simulcast_layers =
119 info.apply_alignment_to_all_simulcast_layers;
120 }
121
RegisterEncodeCompleteCallback(EncodedImageCallback * callback)122 int32_t VideoEncoderWrapper::RegisterEncodeCompleteCallback(
123 EncodedImageCallback* callback) {
124 callback_ = callback;
125 return WEBRTC_VIDEO_CODEC_OK;
126 }
127
Release()128 int32_t VideoEncoderWrapper::Release() {
129 JNIEnv* jni = AttachCurrentThreadIfNeeded();
130
131 int32_t status = JavaToNativeVideoCodecStatus(
132 jni, Java_VideoEncoder_release(jni, encoder_));
133 RTC_LOG(LS_INFO) << "release: " << status;
134 {
135 MutexLock lock(&frame_extra_infos_lock_);
136 frame_extra_infos_.clear();
137 }
138 initialized_ = false;
139
140 return status;
141 }
142
Encode(const VideoFrame & frame,const std::vector<VideoFrameType> * frame_types)143 int32_t VideoEncoderWrapper::Encode(
144 const VideoFrame& frame,
145 const std::vector<VideoFrameType>* frame_types) {
146 if (!initialized_) {
147 // Most likely initializing the codec failed.
148 return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
149 }
150
151 JNIEnv* jni = AttachCurrentThreadIfNeeded();
152
153 // Construct encode info.
154 ScopedJavaLocalRef<jobjectArray> j_frame_types =
155 NativeToJavaFrameTypeArray(jni, *frame_types);
156 ScopedJavaLocalRef<jobject> encode_info =
157 Java_EncodeInfo_Constructor(jni, j_frame_types);
158
159 FrameExtraInfo info;
160 info.capture_time_ns = frame.timestamp_us() * rtc::kNumNanosecsPerMicrosec;
161 info.timestamp_rtp = frame.timestamp();
162 {
163 MutexLock lock(&frame_extra_infos_lock_);
164 frame_extra_infos_.push_back(info);
165 }
166
167 ScopedJavaLocalRef<jobject> j_frame = NativeToJavaVideoFrame(jni, frame);
168 ScopedJavaLocalRef<jobject> ret =
169 Java_VideoEncoder_encode(jni, encoder_, j_frame, encode_info);
170 ReleaseJavaVideoFrame(jni, j_frame);
171 return HandleReturnCode(jni, ret, "encode");
172 }
173
SetRates(const RateControlParameters & rc_parameters)174 void VideoEncoderWrapper::SetRates(const RateControlParameters& rc_parameters) {
175 JNIEnv* jni = AttachCurrentThreadIfNeeded();
176
177 ScopedJavaLocalRef<jobject> j_rc_parameters =
178 ToJavaRateControlParameters(jni, rc_parameters);
179 ScopedJavaLocalRef<jobject> ret =
180 Java_VideoEncoder_setRates(jni, encoder_, j_rc_parameters);
181 HandleReturnCode(jni, ret, "setRates");
182 }
183
GetEncoderInfo() const184 VideoEncoder::EncoderInfo VideoEncoderWrapper::GetEncoderInfo() const {
185 return encoder_info_;
186 }
187
188 VideoEncoderWrapper::ScalingSettings
GetScalingSettingsInternal(JNIEnv * jni) const189 VideoEncoderWrapper::GetScalingSettingsInternal(JNIEnv* jni) const {
190 ScopedJavaLocalRef<jobject> j_scaling_settings =
191 Java_VideoEncoder_getScalingSettings(jni, encoder_);
192 bool isOn =
193 Java_VideoEncoderWrapper_getScalingSettingsOn(jni, j_scaling_settings);
194
195 if (!isOn)
196 return ScalingSettings::kOff;
197
198 absl::optional<int> low = JavaToNativeOptionalInt(
199 jni,
200 Java_VideoEncoderWrapper_getScalingSettingsLow(jni, j_scaling_settings));
201 absl::optional<int> high = JavaToNativeOptionalInt(
202 jni,
203 Java_VideoEncoderWrapper_getScalingSettingsHigh(jni, j_scaling_settings));
204
205 if (low && high)
206 return ScalingSettings(*low, *high);
207
208 switch (codec_settings_.codecType) {
209 case kVideoCodecVP8: {
210 // Same as in vp8_impl.cc.
211 static const int kLowVp8QpThreshold = 29;
212 static const int kHighVp8QpThreshold = 95;
213 return ScalingSettings(low.value_or(kLowVp8QpThreshold),
214 high.value_or(kHighVp8QpThreshold));
215 }
216 case kVideoCodecVP9: {
217 // QP is obtained from VP9-bitstream, so the QP corresponds to the
218 // bitstream range of [0, 255] and not the user-level range of [0,63].
219 static const int kLowVp9QpThreshold = 96;
220 static const int kHighVp9QpThreshold = 185;
221
222 return VideoEncoder::ScalingSettings(kLowVp9QpThreshold,
223 kHighVp9QpThreshold);
224 }
225 case kVideoCodecH264: {
226 // Same as in h264_encoder_impl.cc.
227 static const int kLowH264QpThreshold = 24;
228 static const int kHighH264QpThreshold = 37;
229 return ScalingSettings(low.value_or(kLowH264QpThreshold),
230 high.value_or(kHighH264QpThreshold));
231 }
232 default:
233 return ScalingSettings::kOff;
234 }
235 }
236
GetEncoderInfoInternal(JNIEnv * jni) const237 VideoEncoder::EncoderInfo VideoEncoderWrapper::GetEncoderInfoInternal(
238 JNIEnv* jni) const {
239 ScopedJavaLocalRef<jobject> j_encoder_info =
240 Java_VideoEncoder_getEncoderInfo(jni, encoder_);
241
242 jint requested_resolution_alignment =
243 Java_EncoderInfo_getRequestedResolutionAlignment(jni, j_encoder_info);
244
245 jboolean apply_alignment_to_all_simulcast_layers =
246 Java_EncoderInfo_getApplyAlignmentToAllSimulcastLayers(jni,
247 j_encoder_info);
248
249 VideoEncoder::EncoderInfo info;
250 info.requested_resolution_alignment = requested_resolution_alignment;
251 info.apply_alignment_to_all_simulcast_layers =
252 apply_alignment_to_all_simulcast_layers;
253
254 return info;
255 }
256
OnEncodedFrame(JNIEnv * jni,const JavaRef<jobject> & j_encoded_image)257 void VideoEncoderWrapper::OnEncodedFrame(
258 JNIEnv* jni,
259 const JavaRef<jobject>& j_encoded_image) {
260 EncodedImage frame = JavaToNativeEncodedImage(jni, j_encoded_image);
261 int64_t capture_time_ns =
262 GetJavaEncodedImageCaptureTimeNs(jni, j_encoded_image);
263
264 // Encoded frames are delivered in the order received, but some of them
265 // may be dropped, so remove records of frames older than the current
266 // one.
267 //
268 // NOTE: if the current frame is associated with Encoder A, in the time
269 // since this frame was received, Encoder A could have been
270 // Release()'ed, Encoder B InitEncode()'ed (due to reuse of Encoder A),
271 // and frames received by Encoder B. Thus there may be frame_extra_infos
272 // entries that don't belong to us, and we need to be careful not to
273 // remove them. Removing only those entries older than the current frame
274 // provides this guarantee.
275 FrameExtraInfo frame_extra_info;
276 {
277 MutexLock lock(&frame_extra_infos_lock_);
278 while (!frame_extra_infos_.empty() &&
279 frame_extra_infos_.front().capture_time_ns < capture_time_ns) {
280 frame_extra_infos_.pop_front();
281 }
282 if (frame_extra_infos_.empty() ||
283 frame_extra_infos_.front().capture_time_ns != capture_time_ns) {
284 RTC_LOG(LS_WARNING)
285 << "Java encoder produced an unexpected frame with timestamp: "
286 << capture_time_ns;
287 return;
288 }
289 frame_extra_info = frame_extra_infos_.front();
290 frame_extra_infos_.pop_front();
291 }
292
293 // This is a bit subtle. The `frame` variable from the lambda capture is
294 // const. Which implies that (i) we need to make a copy to be able to
295 // write to the metadata, and (ii) we should avoid using the .data()
296 // method (including implicit conversion to ArrayView) on the non-const
297 // copy, since that would trigget a copy operation on the underlying
298 // CopyOnWriteBuffer.
299 EncodedImage frame_copy = frame;
300
301 frame_copy.SetTimestamp(frame_extra_info.timestamp_rtp);
302 frame_copy.capture_time_ms_ = capture_time_ns / rtc::kNumNanosecsPerMillisec;
303
304 if (frame_copy.qp_ < 0)
305 frame_copy.qp_ = ParseQp(frame);
306
307 CodecSpecificInfo info(ParseCodecSpecificInfo(frame));
308
309 callback_->OnEncodedImage(frame_copy, &info);
310 }
311
HandleReturnCode(JNIEnv * jni,const JavaRef<jobject> & j_value,const char * method_name)312 int32_t VideoEncoderWrapper::HandleReturnCode(JNIEnv* jni,
313 const JavaRef<jobject>& j_value,
314 const char* method_name) {
315 int32_t value = JavaToNativeVideoCodecStatus(jni, j_value);
316 if (value >= 0) { // OK or NO_OUTPUT
317 return value;
318 }
319
320 RTC_LOG(LS_WARNING) << method_name << ": " << value;
321 if (value == WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE ||
322 value == WEBRTC_VIDEO_CODEC_UNINITIALIZED) { // Critical error.
323 RTC_LOG(LS_WARNING) << "Java encoder requested software fallback.";
324 return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
325 }
326
327 // Try resetting the codec.
328 if (Release() == WEBRTC_VIDEO_CODEC_OK &&
329 InitEncodeInternal(jni) == WEBRTC_VIDEO_CODEC_OK) {
330 RTC_LOG(LS_WARNING) << "Reset Java encoder.";
331 return WEBRTC_VIDEO_CODEC_ERROR;
332 }
333
334 RTC_LOG(LS_WARNING) << "Unable to reset Java encoder.";
335 return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
336 }
337
ParseQp(rtc::ArrayView<const uint8_t> buffer)338 int VideoEncoderWrapper::ParseQp(rtc::ArrayView<const uint8_t> buffer) {
339 int qp;
340 bool success;
341 switch (codec_settings_.codecType) {
342 case kVideoCodecVP8:
343 success = vp8::GetQp(buffer.data(), buffer.size(), &qp);
344 break;
345 case kVideoCodecVP9:
346 success = vp9::GetQp(buffer.data(), buffer.size(), &qp);
347 break;
348 case kVideoCodecH264:
349 h264_bitstream_parser_.ParseBitstream(buffer);
350 qp = h264_bitstream_parser_.GetLastSliceQp().value_or(-1);
351 success = (qp >= 0);
352 break;
353 default: // Default is to not provide QP.
354 success = false;
355 break;
356 }
357 return success ? qp : -1; // -1 means unknown QP.
358 }
359
ParseCodecSpecificInfo(const EncodedImage & frame)360 CodecSpecificInfo VideoEncoderWrapper::ParseCodecSpecificInfo(
361 const EncodedImage& frame) {
362 const bool key_frame = frame._frameType == VideoFrameType::kVideoFrameKey;
363
364 CodecSpecificInfo info;
365 // For stream with scalability, NextFrameConfig should be called before
366 // encoding and used to configure encoder, then passed here e.g. via
367 // FrameExtraInfo structure. But while this encoder wrapper uses only trivial
368 // scalability, NextFrameConfig can be called here.
369 auto layer_frames = svc_controller_.NextFrameConfig(/*reset=*/key_frame);
370 RTC_DCHECK_EQ(layer_frames.size(), 1);
371 info.generic_frame_info = svc_controller_.OnEncodeDone(layer_frames[0]);
372 if (key_frame) {
373 info.template_structure = svc_controller_.DependencyStructure();
374 info.template_structure->resolutions = {
375 RenderResolution(frame._encodedWidth, frame._encodedHeight)};
376 }
377
378 info.codecType = codec_settings_.codecType;
379
380 switch (codec_settings_.codecType) {
381 case kVideoCodecVP8:
382 info.codecSpecific.VP8.nonReference = false;
383 info.codecSpecific.VP8.temporalIdx = kNoTemporalIdx;
384 info.codecSpecific.VP8.layerSync = false;
385 info.codecSpecific.VP8.keyIdx = kNoKeyIdx;
386 break;
387 case kVideoCodecVP9:
388 if (key_frame) {
389 gof_idx_ = 0;
390 }
391 info.codecSpecific.VP9.inter_pic_predicted = key_frame ? false : true;
392 info.codecSpecific.VP9.flexible_mode = false;
393 info.codecSpecific.VP9.ss_data_available = key_frame ? true : false;
394 info.codecSpecific.VP9.temporal_idx = kNoTemporalIdx;
395 info.codecSpecific.VP9.temporal_up_switch = true;
396 info.codecSpecific.VP9.inter_layer_predicted = false;
397 info.codecSpecific.VP9.gof_idx =
398 static_cast<uint8_t>(gof_idx_++ % gof_.num_frames_in_gof);
399 info.codecSpecific.VP9.num_spatial_layers = 1;
400 info.codecSpecific.VP9.first_frame_in_picture = true;
401 info.codecSpecific.VP9.spatial_layer_resolution_present = false;
402 if (info.codecSpecific.VP9.ss_data_available) {
403 info.codecSpecific.VP9.spatial_layer_resolution_present = true;
404 info.codecSpecific.VP9.width[0] = frame._encodedWidth;
405 info.codecSpecific.VP9.height[0] = frame._encodedHeight;
406 info.codecSpecific.VP9.gof.CopyGofInfoVP9(gof_);
407 }
408 break;
409 default:
410 break;
411 }
412
413 return info;
414 }
415
ToJavaBitrateAllocation(JNIEnv * jni,const VideoBitrateAllocation & allocation)416 ScopedJavaLocalRef<jobject> VideoEncoderWrapper::ToJavaBitrateAllocation(
417 JNIEnv* jni,
418 const VideoBitrateAllocation& allocation) {
419 ScopedJavaLocalRef<jobjectArray> j_allocation_array(
420 jni, jni->NewObjectArray(kMaxSpatialLayers, int_array_class_.obj(),
421 nullptr /* initial */));
422 for (int spatial_i = 0; spatial_i < kMaxSpatialLayers; ++spatial_i) {
423 std::array<int32_t, kMaxTemporalStreams> spatial_layer;
424 for (int temporal_i = 0; temporal_i < kMaxTemporalStreams; ++temporal_i) {
425 spatial_layer[temporal_i] = allocation.GetBitrate(spatial_i, temporal_i);
426 }
427
428 ScopedJavaLocalRef<jintArray> j_array_spatial_layer =
429 NativeToJavaIntArray(jni, spatial_layer);
430 jni->SetObjectArrayElement(j_allocation_array.obj(), spatial_i,
431 j_array_spatial_layer.obj());
432 }
433 return Java_BitrateAllocation_Constructor(jni, j_allocation_array);
434 }
435
ToJavaRateControlParameters(JNIEnv * jni,const VideoEncoder::RateControlParameters & rc_parameters)436 ScopedJavaLocalRef<jobject> VideoEncoderWrapper::ToJavaRateControlParameters(
437 JNIEnv* jni,
438 const VideoEncoder::RateControlParameters& rc_parameters) {
439 ScopedJavaLocalRef<jobject> j_bitrate_allocation =
440 ToJavaBitrateAllocation(jni, rc_parameters.bitrate);
441
442 return Java_RateControlParameters_Constructor(jni, j_bitrate_allocation,
443 rc_parameters.framerate_fps);
444 }
445
JavaToNativeVideoEncoder(JNIEnv * jni,const JavaRef<jobject> & j_encoder)446 std::unique_ptr<VideoEncoder> JavaToNativeVideoEncoder(
447 JNIEnv* jni,
448 const JavaRef<jobject>& j_encoder) {
449 const jlong native_encoder =
450 Java_VideoEncoder_createNativeVideoEncoder(jni, j_encoder);
451 VideoEncoder* encoder;
452 if (native_encoder == 0) {
453 encoder = new VideoEncoderWrapper(jni, j_encoder);
454 } else {
455 encoder = reinterpret_cast<VideoEncoder*>(native_encoder);
456 }
457 return std::unique_ptr<VideoEncoder>(encoder);
458 }
459
460 std::vector<VideoEncoder::ResolutionBitrateLimits>
JavaToNativeResolutionBitrateLimits(JNIEnv * jni,const JavaRef<jobjectArray> & j_bitrate_limits_array)461 JavaToNativeResolutionBitrateLimits(
462 JNIEnv* jni,
463 const JavaRef<jobjectArray>& j_bitrate_limits_array) {
464 std::vector<VideoEncoder::ResolutionBitrateLimits> resolution_bitrate_limits;
465
466 const jsize array_length = jni->GetArrayLength(j_bitrate_limits_array.obj());
467 for (int i = 0; i < array_length; ++i) {
468 ScopedJavaLocalRef<jobject> j_bitrate_limits = ScopedJavaLocalRef<jobject>(
469 jni, jni->GetObjectArrayElement(j_bitrate_limits_array.obj(), i));
470
471 jint frame_size_pixels =
472 Java_ResolutionBitrateLimits_getFrameSizePixels(jni, j_bitrate_limits);
473 jint min_start_bitrate_bps =
474 Java_ResolutionBitrateLimits_getMinStartBitrateBps(jni,
475 j_bitrate_limits);
476 jint min_bitrate_bps =
477 Java_ResolutionBitrateLimits_getMinBitrateBps(jni, j_bitrate_limits);
478 jint max_bitrate_bps =
479 Java_ResolutionBitrateLimits_getMaxBitrateBps(jni, j_bitrate_limits);
480
481 resolution_bitrate_limits.push_back(VideoEncoder::ResolutionBitrateLimits(
482 frame_size_pixels, min_start_bitrate_bps, min_bitrate_bps,
483 max_bitrate_bps));
484 }
485
486 return resolution_bitrate_limits;
487 }
488
489 } // namespace jni
490 } // namespace webrtc
491