1 // Copyright 2023 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 //#define LOG_NDEBUG 0
6 #define LOG_TAG "V4L2EncodeInterface"
7
8 #include <v4l2_codec2/components/EncodeInterface.h>
9
10 #include <inttypes.h>
11 #include <algorithm>
12
13 #include <C2PlatformSupport.h>
14 #include <SimpleC2Interface.h>
15 #include <android/hardware/graphics/common/1.0/types.h>
16 #include <media/stagefright/MediaDefs.h>
17 #include <utils/Log.h>
18
19 #include <v4l2_codec2/common/Common.h>
20 #include <v4l2_codec2/common/H264.h>
21 #include <v4l2_codec2/common/VideoTypes.h>
22
23 using android::hardware::graphics::common::V1_0::BufferUsage;
24
25 namespace android {
26
27 namespace {
28
29 // Use basic linear block pool/allocator as default.
30 constexpr C2BlockPool::local_id_t kDefaultOutputBlockPool = C2BlockPool::BASIC_LINEAR;
31 // Default input and output allocators.
32 constexpr C2Allocator::id_t kDefaultInputAllocator = C2PlatformAllocatorStore::GRALLOC;
33 constexpr C2Allocator::id_t kDefaultOutputAllocator = C2PlatformAllocatorStore::BLOB;
34
35 // The default output framerate in frames per second.
36 // TODO: increase to 60 fps in the future.
37 constexpr float kDefaultFrameRate = 30.0;
38 // The default output bitrate in bits per second. Use the max bitrate of AVC Level1.0 as default.
39 constexpr uint32_t kDefaultBitrate = 64000;
40
41 // The maximal output bitrate in bits per second. It's the max bitrate of AVC Level4.1.
42 // TODO: increase this in the future for supporting higher level/resolution encoding.
43 constexpr uint32_t kMaxBitrate = 50000000;
44 } // namespace
45
46 //static
47 C2Config::level_t EncodeInterface::lowestConfigLevel = C2Config::LEVEL_UNUSED;
48
49 // static
H264ProfileLevelSetter(bool,C2P<C2StreamProfileLevelInfo::output> & info,const C2P<C2StreamPictureSizeInfo::input> & videoSize,const C2P<C2StreamFrameRateInfo::output> & frameRate,const C2P<C2StreamBitrateInfo::output> & bitrate)50 C2R EncodeInterface::H264ProfileLevelSetter(bool /*mayBlock*/,
51 C2P<C2StreamProfileLevelInfo::output>& info,
52 const C2P<C2StreamPictureSizeInfo::input>& videoSize,
53 const C2P<C2StreamFrameRateInfo::output>& frameRate,
54 const C2P<C2StreamBitrateInfo::output>& bitrate) {
55 // Adopt default minimal profile instead if the requested profile is not supported, or lower
56 // than the default minimal one.
57 constexpr C2Config::profile_t minProfile = C2Config::PROFILE_AVC_BASELINE;
58 if (!info.F(info.v.profile).supportsAtAll(info.v.profile) || info.v.profile < minProfile) {
59 if (info.F(info.v.profile).supportsAtAll(minProfile)) {
60 ALOGV("Set profile to default (%u) instead.", minProfile);
61 info.set().profile = minProfile;
62 } else {
63 ALOGE("Unable to set either requested profile (%u) or default profile (%u).",
64 info.v.profile, minProfile);
65 return C2R(C2SettingResultBuilder::BadValue(info.F(info.v.profile)));
66 }
67 }
68
69 uint64_t targetFS =
70 static_cast<uint64_t>((videoSize.v.width + 15) / 16) * ((videoSize.v.height + 15) / 16);
71 float targetMBPS = static_cast<float>(targetFS) * frameRate.v.value;
72
73 // Try the recorded lowest configed level. This level should become adaptable after input size,
74 // frame rate, and bitrate are all set.
75 if (lowestConfigLevel != C2Config::LEVEL_UNUSED && lowestConfigLevel < info.v.level) {
76 info.set().level = lowestConfigLevel;
77 }
78
79 // Check if the supplied level meets the requirements. If not, update the level with the lowest
80 // level meeting the requirements.
81
82 bool found = false;
83 bool needsUpdate = !info.F(info.v.level).supportsAtAll(info.v.level);
84 for (const H264LevelLimits& limit : kH264Limits) {
85 if (!info.F(info.v.level).supportsAtAll(limit.level)) {
86 continue;
87 }
88
89 // Table A-2 in spec
90 // The maximum bit rate for High Profile is 1.25 times that of the Base/Extended/Main
91 // Profiles, 3 times for Hi10P, and 4 times for Hi422P/Hi444PP.
92 uint32_t maxBR = limit.maxBR;
93 if (info.v.profile >= C2Config::PROFILE_AVC_HIGH_422) {
94 maxBR *= 4;
95 } else if (info.v.profile >= C2Config::PROFILE_AVC_HIGH_10) {
96 maxBR *= 3;
97 } else if (info.v.profile >= C2Config::PROFILE_AVC_HIGH) {
98 maxBR = maxBR * 5.0 / 4.0;
99 }
100
101 if (targetFS <= limit.maxFS && targetMBPS <= limit.maxMBPS && bitrate.v.value <= maxBR) {
102 // This is the lowest level that meets the requirements, and if
103 // we haven't seen the supplied level yet, that means we don't
104 // need the update.
105 if (needsUpdate) {
106 // Since current config update is sequential, there is an issue when we want to set
107 // lower level for small input size, frame rate, and bitrate, if we set level first,
108 // it will be adjusted to a higher one because the default input size or others are
109 // above the limit. Use lowestConfigLevel to record the level we have tried to
110 // config (but failed).
111 // TODO(johnylin): remove this solution after b/140407694 has proper fix.
112 lowestConfigLevel = info.v.level;
113
114 ALOGD("Given level %u does not cover current configuration: "
115 "adjusting to %u",
116 info.v.level, limit.level);
117 info.set().level = limit.level;
118 }
119 found = true;
120 break;
121 }
122 if (info.v.level <= limit.level) {
123 // We break out of the loop when the lowest feasible level is found. The fact that we're
124 // here means that our level doesn't meet the requirement and needs to be updated.
125 needsUpdate = true;
126 }
127 }
128 if (!found) {
129 ALOGE("Unable to find proper level with current config, requested level (%u).",
130 info.v.level);
131 return C2R(C2SettingResultBuilder::BadValue(info.F(info.v.level)));
132 }
133
134 return C2R::Ok();
135 }
136
VP9ProfileLevelSetter(bool,C2P<C2StreamProfileLevelInfo::output> & info,const C2P<C2StreamPictureSizeInfo::input> &,const C2P<C2StreamFrameRateInfo::output> &,const C2P<C2StreamBitrateInfo::output> &)137 C2R EncodeInterface::VP9ProfileLevelSetter(bool /*mayBlock*/,
138 C2P<C2StreamProfileLevelInfo::output>& info,
139 const C2P<C2StreamPictureSizeInfo::input>& /*videoSize*/,
140 const C2P<C2StreamFrameRateInfo::output>& /*frameRate*/,
141 const C2P<C2StreamBitrateInfo::output>& /*bitrate*/) {
142 // Adopt default minimal profile instead if the requested profile is not supported, or lower
143 // than the default minimal one.
144 constexpr C2Config::profile_t defaultMinProfile = C2Config::PROFILE_VP9_0;
145 if (!info.F(info.v.profile).supportsAtAll(info.v.profile) ||
146 info.v.profile < defaultMinProfile) {
147 if (info.F(info.v.profile).supportsAtAll(defaultMinProfile)) {
148 ALOGV("Set profile to default (%u) instead.", defaultMinProfile);
149 info.set().profile = defaultMinProfile;
150 } else {
151 ALOGE("Unable to set either requested profile (%u) or default profile (%u).",
152 info.v.profile, defaultMinProfile);
153 return C2R(C2SettingResultBuilder::BadValue(info.F(info.v.profile)));
154 }
155 }
156
157 return C2R::Ok();
158 }
159
160 // static
SizeSetter(bool mayBlock,C2P<C2StreamPictureSizeInfo::input> & videoSize)161 C2R EncodeInterface::SizeSetter(bool mayBlock, C2P<C2StreamPictureSizeInfo::input>& videoSize) {
162 (void)mayBlock;
163 // TODO: maybe apply block limit?
164 return videoSize.F(videoSize.v.width)
165 .validatePossible(videoSize.v.width)
166 .plus(videoSize.F(videoSize.v.height).validatePossible(videoSize.v.height));
167 }
168
169 // static
IntraRefreshPeriodSetter(bool mayBlock,C2P<C2StreamIntraRefreshTuning::output> & period)170 C2R EncodeInterface::IntraRefreshPeriodSetter(bool mayBlock,
171 C2P<C2StreamIntraRefreshTuning::output>& period) {
172 (void)mayBlock;
173 if (period.v.period < 1) {
174 period.set().mode = C2Config::INTRA_REFRESH_DISABLED;
175 period.set().period = 0;
176 } else {
177 // Only support arbitrary mode (cyclic in our case).
178 period.set().mode = C2Config::INTRA_REFRESH_ARBITRARY;
179 }
180 return C2R::Ok();
181 }
182
EncodeInterface(const C2String & name,std::shared_ptr<C2ReflectorHelper> helper,const SupportedCapabilities & caps)183 EncodeInterface::EncodeInterface(const C2String& name, std::shared_ptr<C2ReflectorHelper> helper,
184 const SupportedCapabilities& caps)
185 : C2InterfaceHelper(std::move(helper)) {
186 ALOGV("%s(%s)", __func__, name.c_str());
187
188 setDerivedInstance(this);
189
190 Initialize(name, caps);
191 }
192
Initialize(const C2String & name,const SupportedCapabilities & caps)193 void EncodeInterface::Initialize(const C2String& name, const SupportedCapabilities& caps) {
194 // Note: unsigned int is used here, since std::vector<C2Config::profile_t> cannot convert to
195 // std::vector<unsigned int> required by the c2 framework below.
196 std::vector<unsigned int> profiles;
197 ui::Size maxSize;
198 for (const auto& supportedProfile : caps.supportedProfiles) {
199 if (!isValidProfileForCodec(caps.codec, supportedProfile.profile)) {
200 continue; // Ignore unrecognizable or unsupported profiles.
201 }
202 ALOGV("Queried c2_profile = 0x%x : max_size = %d x %d", supportedProfile.profile,
203 supportedProfile.max_resolution.width, supportedProfile.max_resolution.height);
204 profiles.push_back(static_cast<unsigned int>(supportedProfile.profile));
205 maxSize.setWidth(std::max(maxSize.width, supportedProfile.max_resolution.width));
206 maxSize.setHeight(std::max(maxSize.height, supportedProfile.max_resolution.height));
207 }
208
209 if (profiles.empty()) {
210 ALOGE("No supported profiles");
211 mInitStatus = C2_BAD_VALUE;
212 return;
213 }
214
215 // Special note: the order of addParameter matters if your setters are dependent on other
216 // parameters. Please make sure the dependent parameters are added prior to the
217 // one needs the setter dependency.
218
219 addParameter(DefineParam(mKind, C2_PARAMKEY_COMPONENT_KIND)
220 .withConstValue(new C2ComponentKindSetting(C2Component::KIND_ENCODER))
221 .build());
222
223 addParameter(DefineParam(mInputVisibleSize, C2_PARAMKEY_PICTURE_SIZE)
224 .withDefault(new C2StreamPictureSizeInfo::input(0u, 320, 240))
225 .withFields({
226 C2F(mInputVisibleSize, width).inRange(2, maxSize.width, 2),
227 C2F(mInputVisibleSize, height).inRange(2, maxSize.height, 2),
228 })
229 .withSetter(SizeSetter)
230 .build());
231
232 addParameter(DefineParam(mFrameRate, C2_PARAMKEY_FRAME_RATE)
233 .withDefault(new C2StreamFrameRateInfo::output(0u, kDefaultFrameRate))
234 // TODO: More restriction?
235 .withFields({C2F(mFrameRate, value).greaterThan(0.)})
236 .withSetter(Setter<decltype(*mFrameRate)>::StrictValueWithNoDeps)
237 .build());
238
239 addParameter(DefineParam(mBitrate, C2_PARAMKEY_BITRATE)
240 .withDefault(new C2StreamBitrateInfo::output(0u, kDefaultBitrate))
241 .withFields({C2F(mBitrate, value).inRange(0, kMaxBitrate)})
242 .withSetter(Setter<decltype(*mBitrate)>::StrictValueWithNoDeps)
243 .build());
244
245 addParameter(
246 DefineParam(mBitrateMode, C2_PARAMKEY_BITRATE_MODE)
247 .withDefault(new C2StreamBitrateModeTuning::output(0u, C2Config::BITRATE_CONST))
248 .withFields(
249 {C2F(mBitrateMode, value)
250 .oneOf({C2Config::BITRATE_CONST, C2Config::BITRATE_VARIABLE})})
251 .withSetter(Setter<decltype(*mBitrateMode)>::StrictValueWithNoDeps)
252 .build());
253
254 std::string outputMime;
255 if (caps.codec == VideoCodec::H264) {
256 outputMime = MEDIA_MIMETYPE_VIDEO_AVC;
257 C2Config::profile_t minProfile = static_cast<C2Config::profile_t>(
258 *std::min_element(profiles.begin(), profiles.end()));
259 addParameter(
260 DefineParam(mProfileLevel, C2_PARAMKEY_PROFILE_LEVEL)
261 .withDefault(new C2StreamProfileLevelInfo::output(0u, minProfile,
262 C2Config::LEVEL_AVC_4_1))
263 .withFields(
264 {C2F(mProfileLevel, profile).oneOf(profiles),
265 C2F(mProfileLevel, level)
266 // TODO: query supported levels from adaptor.
267 .oneOf({C2Config::LEVEL_AVC_1, C2Config::LEVEL_AVC_1B,
268 C2Config::LEVEL_AVC_1_1, C2Config::LEVEL_AVC_1_2,
269 C2Config::LEVEL_AVC_1_3, C2Config::LEVEL_AVC_2,
270 C2Config::LEVEL_AVC_2_1, C2Config::LEVEL_AVC_2_2,
271 C2Config::LEVEL_AVC_3, C2Config::LEVEL_AVC_3_1,
272 C2Config::LEVEL_AVC_3_2, C2Config::LEVEL_AVC_4,
273 C2Config::LEVEL_AVC_4_1, C2Config::LEVEL_AVC_4_2,
274 C2Config::LEVEL_AVC_5, C2Config::LEVEL_AVC_5_1})})
275 .withSetter(H264ProfileLevelSetter, mInputVisibleSize, mFrameRate, mBitrate)
276 .build());
277 } else if (caps.codec == VideoCodec::VP8) {
278 outputMime = MEDIA_MIMETYPE_VIDEO_VP8;
279 // VP8 doesn't have conventional profiles, we'll use profile0 if the VP8 codec is requested.
280 addParameter(DefineParam(mProfileLevel, C2_PARAMKEY_PROFILE_LEVEL)
281 .withConstValue(new C2StreamProfileLevelInfo::output(
282 0u, C2Config::PROFILE_VP8_0, C2Config::LEVEL_UNUSED))
283 .build());
284 } else if (caps.codec == VideoCodec::VP9) {
285 outputMime = MEDIA_MIMETYPE_VIDEO_VP9;
286 C2Config::profile_t minProfile = static_cast<C2Config::profile_t>(
287 *std::min_element(profiles.begin(), profiles.end()));
288 addParameter(
289 DefineParam(mProfileLevel, C2_PARAMKEY_PROFILE_LEVEL)
290 .withDefault(new C2StreamProfileLevelInfo::output(0u, minProfile,
291 C2Config::LEVEL_VP9_1))
292 .withFields(
293 {C2F(mProfileLevel, profile).oneOf(profiles),
294 C2F(mProfileLevel, level)
295 // TODO(dstaessens) query supported levels from adaptor.
296 .oneOf({C2Config::LEVEL_VP9_1, C2Config::LEVEL_VP9_1_1,
297 C2Config::LEVEL_VP9_2, C2Config::LEVEL_VP9_2_1,
298 C2Config::LEVEL_VP9_3, C2Config::LEVEL_VP9_3_1,
299 C2Config::LEVEL_VP9_4, C2Config::LEVEL_VP9_4_1,
300 C2Config::LEVEL_VP9_5, C2Config::LEVEL_VP9_5_1,
301 C2Config::LEVEL_VP9_5_2, C2Config::LEVEL_VP9_6,
302 C2Config::LEVEL_VP9_6_1,
303 C2Config::LEVEL_VP9_6_2})})
304 .withSetter(VP9ProfileLevelSetter, mInputVisibleSize, mFrameRate, mBitrate)
305 .build());
306 } else {
307 ALOGE("Unsupported component name: %s", name.c_str());
308 mInitStatus = C2_BAD_VALUE;
309 return;
310 }
311
312 addParameter(
313 DefineParam(mInputFormat, C2_PARAMKEY_INPUT_STREAM_BUFFER_TYPE)
314 .withConstValue(new C2StreamBufferTypeSetting::input(0u, C2BufferData::GRAPHIC))
315 .build());
316
317 addParameter(DefineParam(mInputMemoryUsage, C2_PARAMKEY_INPUT_STREAM_USAGE)
318 .withConstValue(new C2StreamUsageTuning::input(
319 0u, static_cast<uint64_t>(BufferUsage::VIDEO_ENCODER)))
320 .build());
321
322 addParameter(
323 DefineParam(mOutputFormat, C2_PARAMKEY_OUTPUT_STREAM_BUFFER_TYPE)
324 .withConstValue(new C2StreamBufferTypeSetting::output(0u, C2BufferData::LINEAR))
325 .build());
326
327 addParameter(DefineParam(mInputMediaType, C2_PARAMKEY_INPUT_MEDIA_TYPE)
328 .withConstValue(AllocSharedString<C2PortMediaTypeSetting::input>(
329 MEDIA_MIMETYPE_VIDEO_RAW))
330 .build());
331
332 addParameter(DefineParam(mOutputMediaType, C2_PARAMKEY_OUTPUT_MEDIA_TYPE)
333 .withConstValue(AllocSharedString<C2PortMediaTypeSetting::output>(
334 outputMime.c_str()))
335 .build());
336
337 addParameter(DefineParam(mIntraRefreshPeriod, C2_PARAMKEY_INTRA_REFRESH)
338 .withDefault(new C2StreamIntraRefreshTuning::output(
339 0u, C2Config::INTRA_REFRESH_DISABLED, 0.))
340 .withFields({C2F(mIntraRefreshPeriod, mode)
341 .oneOf({C2Config::INTRA_REFRESH_DISABLED,
342 C2Config::INTRA_REFRESH_ARBITRARY}),
343 C2F(mIntraRefreshPeriod, period).any()})
344 .withSetter(IntraRefreshPeriodSetter)
345 .build());
346
347 addParameter(DefineParam(mRequestKeyFrame, C2_PARAMKEY_REQUEST_SYNC_FRAME)
348 .withDefault(new C2StreamRequestSyncFrameTuning::output(0u, C2_FALSE))
349 .withFields({C2F(mRequestKeyFrame, value).oneOf({C2_FALSE, C2_TRUE})})
350 .withSetter(Setter<decltype(*mRequestKeyFrame)>::NonStrictValueWithNoDeps)
351 .build());
352
353 addParameter(DefineParam(mKeyFramePeriodUs, C2_PARAMKEY_SYNC_FRAME_INTERVAL)
354 .withDefault(new C2StreamSyncFrameIntervalTuning::output(0u, 1000000))
355 .withFields({C2F(mKeyFramePeriodUs, value).any()})
356 .withSetter(Setter<decltype(*mKeyFramePeriodUs)>::StrictValueWithNoDeps)
357 .build());
358
359 C2Allocator::id_t inputAllocators[] = {kDefaultInputAllocator};
360
361 C2Allocator::id_t outputAllocators[] = {kDefaultOutputAllocator};
362
363 addParameter(
364 DefineParam(mInputAllocatorIds, C2_PARAMKEY_INPUT_ALLOCATORS)
365 .withConstValue(C2PortAllocatorsTuning::input::AllocShared(inputAllocators))
366 .build());
367
368 addParameter(
369 DefineParam(mOutputAllocatorIds, C2_PARAMKEY_OUTPUT_ALLOCATORS)
370 .withConstValue(C2PortAllocatorsTuning::output::AllocShared(outputAllocators))
371 .build());
372
373 C2BlockPool::local_id_t outputBlockPools[] = {kDefaultOutputBlockPool};
374
375 addParameter(
376 DefineParam(mOutputBlockPoolIds, C2_PARAMKEY_OUTPUT_BLOCK_POOLS)
377 .withDefault(C2PortBlockPoolsTuning::output::AllocShared(outputBlockPools))
378 .withFields({C2F(mOutputBlockPoolIds, m.values[0]).any(),
379 C2F(mOutputBlockPoolIds, m.values).inRange(0, 1)})
380 .withSetter(Setter<C2PortBlockPoolsTuning::output>::NonStrictValuesWithNoDeps)
381 .build());
382
383 mInitStatus = C2_OK;
384 }
385
getKeyFramePeriod() const386 uint32_t EncodeInterface::getKeyFramePeriod() const {
387 if (mKeyFramePeriodUs->value < 0 || mKeyFramePeriodUs->value == INT64_MAX) {
388 return 0;
389 }
390 double period = mKeyFramePeriodUs->value / 1e6 * mFrameRate->value;
391 return static_cast<uint32_t>(std::max(std::min(std::round(period), double(UINT32_MAX)), 1.));
392 }
393
394 } // namespace android
395