1 // Copyright 2019 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 //#define LOG_NDEBUG 0
6 #define ATRACE_TAG ATRACE_TAG_VIDEO
7 #define LOG_TAG "FormatConverter"
8
9 #include <v4l2_codec2/common/FormatConverter.h>
10
11 #include <inttypes.h>
12
13 #include <memory>
14 #include <string>
15
16 #include <C2AllocatorGralloc.h>
17 #include <C2PlatformSupport.h>
18 #include <android/hardware/graphics/common/1.0/types.h>
19 #include <inttypes.h>
20 #include <libyuv.h>
21 #include <ui/GraphicBuffer.h>
22 #include <utils/Log.h>
23 #include <utils/Trace.h>
24
25 #include <v4l2_codec2/common/VideoTypes.h> // for HalPixelFormat
26
27 using android::hardware::graphics::common::V1_0::BufferUsage;
28
29 namespace android {
30
31 namespace {
32 // The constant expression of mapping the pixel format conversion pair (src, dst) to a unique
33 // integer.
convertMap(VideoPixelFormat src,VideoPixelFormat dst)34 constexpr int convertMap(VideoPixelFormat src, VideoPixelFormat dst) {
35 return static_cast<int>(src) * (static_cast<int>(VideoPixelFormat::UNKNOWN) + 1) +
36 static_cast<int>(dst);
37 }
38
39 // The helper function to copy a plane pixel by pixel. It assumes bytesPerPixel is 1.
copyPlaneByPixel(const uint8_t * src,int srcStride,int srcColInc,uint8_t * dst,int dstStride,int dstColInc,int width,int height)40 void copyPlaneByPixel(const uint8_t* src, int srcStride, int srcColInc, uint8_t* dst, int dstStride,
41 int dstColInc, int width, int height) {
42 for (int row = 0; row < height; row++) {
43 const uint8_t* srcRow = src;
44 uint8_t* dstRow = dst;
45 for (int col = 0; col < width; col++) {
46 memcpy(dstRow, srcRow, 1);
47 srcRow += srcColInc;
48 dstRow += dstColInc;
49 }
50 src += srcStride;
51 dst += dstStride;
52 }
53 }
54
55 } // namespace
56
ImplDefinedToRGBXMap(sp<GraphicBuffer> buf,uint8_t * addr,int rowInc)57 ImplDefinedToRGBXMap::ImplDefinedToRGBXMap(sp<GraphicBuffer> buf, uint8_t* addr, int rowInc)
58 : mBuffer(std::move(buf)), mAddr(addr), mRowInc(rowInc) {}
59
~ImplDefinedToRGBXMap()60 ImplDefinedToRGBXMap::~ImplDefinedToRGBXMap() {
61 mBuffer->unlock();
62 }
63
64 // static
create(const C2ConstGraphicBlock & block)65 std::unique_ptr<ImplDefinedToRGBXMap> ImplDefinedToRGBXMap::create(
66 const C2ConstGraphicBlock& block) {
67 uint32_t width, height, format, stride, igbpSlot, generation;
68 uint64_t usage, igbpId;
69 android::_UnwrapNativeCodec2GrallocMetadata(block.handle(), &width, &height, &format, &usage,
70 &stride, &generation, &igbpId, &igbpSlot);
71
72 if (format != HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED) {
73 ALOGE("The original format (=%u) is not IMPLEMENTATION_DEFINED", format);
74 return nullptr;
75 }
76
77 native_handle_t* grallocHandle = android::UnwrapNativeCodec2GrallocHandle(block.handle());
78 sp<GraphicBuffer> buf = new GraphicBuffer(grallocHandle, GraphicBuffer::CLONE_HANDLE, width,
79 height, format, 1, usage, stride);
80 native_handle_delete(grallocHandle);
81
82 void* pointer = nullptr;
83 int32_t status = buf->lock(GRALLOC_USAGE_SW_READ_OFTEN, &pointer);
84 if (status != OK) {
85 ALOGE("Failed to lock buffer as IMPLEMENTATION_DEFINED format");
86 return nullptr;
87 }
88
89 uint8_t* addr = reinterpret_cast<uint8_t*>(pointer);
90 int rowInc = static_cast<int>(stride * 4); // RGBX 4-byte data per pixel
91 ALOGD("Parsed input format IMPLEMENTATION_DEFINED to RGBX_8888");
92 return std::unique_ptr<ImplDefinedToRGBXMap>(
93 new ImplDefinedToRGBXMap(std::move(buf), addr, rowInc));
94 }
95
96 // static
create(VideoPixelFormat outFormat,const ui::Size & visibleSize,uint32_t inputCount,const ui::Size & codedSize)97 std::unique_ptr<FormatConverter> FormatConverter::create(VideoPixelFormat outFormat,
98 const ui::Size& visibleSize,
99 uint32_t inputCount,
100 const ui::Size& codedSize) {
101 if (outFormat != VideoPixelFormat::I420 && outFormat != VideoPixelFormat::NV12) {
102 ALOGE("Unsupported output format: %d", static_cast<int32_t>(outFormat));
103 return nullptr;
104 }
105
106 std::unique_ptr<FormatConverter> converter(new FormatConverter);
107 if (converter->initialize(outFormat, visibleSize, inputCount, codedSize) != C2_OK) {
108 ALOGE("Failed to initialize FormatConverter");
109 return nullptr;
110 }
111 return converter;
112 }
113
initialize(VideoPixelFormat outFormat,const ui::Size & visibleSize,uint32_t inputCount,const ui::Size & codedSize)114 c2_status_t FormatConverter::initialize(VideoPixelFormat outFormat, const ui::Size& visibleSize,
115 uint32_t inputCount, const ui::Size& codedSize) {
116 ALOGV("initialize(out_format=%s, visible_size=%dx%d, input_count=%u, coded_size=%dx%d)",
117 videoPixelFormatToString(outFormat).c_str(), visibleSize.width, visibleSize.height,
118 inputCount, codedSize.width, codedSize.height);
119
120 mOutFormat = outFormat;
121 mVisibleSize = visibleSize;
122 mCodedSize = codedSize;
123
124 mTempPlaneU =
125 std::unique_ptr<uint8_t[]>(new uint8_t[mVisibleSize.width * mVisibleSize.height / 4]);
126 mTempPlaneV =
127 std::unique_ptr<uint8_t[]>(new uint8_t[mVisibleSize.width * mVisibleSize.height / 4]);
128
129 // Allocate graphic blocks for format conversion.
130 uint32_t requested_buffer_count = std::max(1u, inputCount);
131 c2_status_t status = allocateBuffers(requested_buffer_count);
132 if (status != C2_OK) {
133 ALOGE("Failed to allocate buffers (error: %d)", status);
134 return status;
135 }
136
137 return C2_OK;
138 }
139
allocateBuffers(uint32_t count)140 c2_status_t FormatConverter::allocateBuffers(uint32_t count) {
141 ALOGV("Allocating %u buffers (format: %s, visible size: %dx%d, coded size: %dx%d)", count,
142 videoPixelFormatToString(mOutFormat).c_str(), mVisibleSize.width, mVisibleSize.height,
143 mCodedSize.width, mCodedSize.height);
144
145 HalPixelFormat halFormat;
146 if (mOutFormat == VideoPixelFormat::I420) {
147 // Android HAL format doesn't have I420, we use YV12 instead and swap U/V while converting.
148 halFormat = HalPixelFormat::YV12;
149 } else {
150 halFormat = HalPixelFormat::YCBCR_420_888; // Will allocate NV12 in minigbm.
151 }
152
153 std::shared_ptr<C2BlockPool> pool;
154 c2_status_t status = GetCodec2BlockPool(C2BlockPool::BASIC_GRAPHIC, nullptr, &pool);
155 if (status != C2_OK) {
156 ALOGE("Failed to get basic graphic block pool (error: %d)", status);
157 return C2_NO_MEMORY;
158 }
159
160 for (uint32_t i = 0; i < count; i++) {
161 std::shared_ptr<C2GraphicBlock> block;
162 status = pool->fetchGraphicBlock(mCodedSize.width, mCodedSize.height,
163 static_cast<uint32_t>(halFormat),
164 {(C2MemoryUsage::CPU_READ | C2MemoryUsage::CPU_WRITE),
165 static_cast<uint64_t>(BufferUsage::VIDEO_ENCODER)},
166 &block);
167 if (status != C2_OK) {
168 ALOGE("Failed to fetch graphic block (error: %d)", status);
169 return C2_NO_MEMORY;
170 }
171 mGraphicBlocks.emplace_back(new BlockEntry(std::move(block)));
172 mAvailableQueue.push(mGraphicBlocks.back().get());
173 }
174
175 return C2_OK;
176 }
177
convertBlock(uint64_t frameIndex,const C2ConstGraphicBlock & inputBlock,C2ConstGraphicBlock * convertedBlock)178 c2_status_t FormatConverter::convertBlock(uint64_t frameIndex,
179 const C2ConstGraphicBlock& inputBlock,
180 C2ConstGraphicBlock* convertedBlock) {
181 ATRACE_CALL();
182 const C2GraphicView& inputView = inputBlock.map().get();
183 C2PlanarLayout inputLayout = inputView.layout();
184
185 // Determine the input buffer pixel format.
186 VideoPixelFormat inputFormat = VideoPixelFormat::UNKNOWN;
187 std::unique_ptr<ImplDefinedToRGBXMap> idMap;
188 if (inputLayout.type == C2PlanarLayout::TYPE_YUV) {
189 if (inputLayout.rootPlanes == 3) {
190 inputFormat = VideoPixelFormat::YV12;
191 } else if (inputLayout.rootPlanes == 2) {
192 const uint8_t* const* data = inputView.data();
193 inputFormat = (data[C2PlanarLayout::PLANE_V] > data[C2PlanarLayout::PLANE_U])
194 ? VideoPixelFormat::NV12
195 : VideoPixelFormat::NV21;
196 }
197 } else if (inputLayout.type == C2PlanarLayout::TYPE_RGB) {
198 inputFormat = VideoPixelFormat::ABGR;
199 } else if (static_cast<uint32_t>(inputLayout.type) == 0u) {
200 // The above layout() cannot fill layout information and sets it to 0 instead if the input
201 // format is IMPLEMENTATION_DEFINED and its backed format is RGB. We fill the layout by
202 // using ImplDefinedToRGBXMap in this case.
203 idMap = ImplDefinedToRGBXMap::create(inputBlock);
204 if (!idMap) {
205 ALOGE("Unable to parse RGBX_8888 from IMPLEMENTATION_DEFINED");
206 return C2_CORRUPTED;
207 }
208 // There is only RGBA_8888 specified in C2AllocationGralloc::map(), no BGRA_8888. Maybe
209 // BGRA_8888 is not used now?
210 inputFormat = VideoPixelFormat::ABGR;
211 inputLayout.type = C2PlanarLayout::TYPE_RGB;
212 } else {
213 ALOGE("Failed to determine input pixel format: %u", inputLayout.type);
214 return C2_CORRUPTED;
215 }
216
217 if (inputFormat == mOutFormat) {
218 ALOGV("Zero-Copy is applied");
219 mGraphicBlocks.emplace_back(new BlockEntry(frameIndex));
220 *convertedBlock = inputBlock;
221 return C2_OK;
222 }
223
224 if (!isReady()) {
225 ALOGV("There is no available block for conversion");
226 return C2_NO_MEMORY;
227 }
228
229 BlockEntry* entry = mAvailableQueue.front();
230 std::shared_ptr<C2GraphicBlock> outputBlock = entry->mBlock;
231
232 C2GraphicView outputView = outputBlock->map().get();
233 C2PlanarLayout outputLayout = outputView.layout();
234 uint8_t* dstY = outputView.data()[C2PlanarLayout::PLANE_Y];
235 uint8_t* dstU = outputView.data()[C2PlanarLayout::PLANE_V]; // only for I420
236 uint8_t* dstV = outputView.data()[C2PlanarLayout::PLANE_U]; // only for I420
237 uint8_t* dstUV = outputView.data()[C2PlanarLayout::PLANE_U]; // only for NV12
238 const int dstStrideY = outputLayout.planes[C2PlanarLayout::PLANE_Y].rowInc;
239 const int dstStrideU = outputLayout.planes[C2PlanarLayout::PLANE_V].rowInc; // only for I420
240 const int dstStrideV = outputLayout.planes[C2PlanarLayout::PLANE_U].rowInc; // only for I420
241 const int dstStrideUV = outputLayout.planes[C2PlanarLayout::PLANE_U].rowInc; // only for NV12
242
243 if (inputLayout.type == C2PlanarLayout::TYPE_YUV) {
244 const uint8_t* srcY = inputView.data()[C2PlanarLayout::PLANE_Y];
245 const uint8_t* srcU = inputView.data()[C2PlanarLayout::PLANE_U];
246 const uint8_t* srcV = inputView.data()[C2PlanarLayout::PLANE_V];
247 const int srcStrideY = inputLayout.planes[C2PlanarLayout::PLANE_Y].rowInc;
248 const int srcStrideU = inputLayout.planes[C2PlanarLayout::PLANE_U].rowInc;
249 const int srcStrideV = inputLayout.planes[C2PlanarLayout::PLANE_V].rowInc;
250
251 switch (convertMap(inputFormat, mOutFormat)) {
252 case convertMap(VideoPixelFormat::YV12, VideoPixelFormat::I420):
253 libyuv::I420Copy(srcY, srcStrideY, srcU, srcStrideU, srcV, srcStrideV, dstY, dstStrideY,
254 dstU, dstStrideU, dstV, dstStrideV, mVisibleSize.width,
255 mVisibleSize.height);
256 break;
257 case convertMap(VideoPixelFormat::YV12, VideoPixelFormat::NV12):
258 libyuv::I420ToNV12(srcY, srcStrideY, srcU, srcStrideU, srcV, srcStrideV, dstY,
259 dstStrideY, dstUV, dstStrideUV, mVisibleSize.width,
260 mVisibleSize.height);
261 break;
262 case convertMap(VideoPixelFormat::NV12, VideoPixelFormat::I420):
263 libyuv::NV12ToI420(srcY, srcStrideY, srcU, srcStrideU, dstY, dstStrideY, dstU,
264 dstStrideU, dstV, dstStrideV, mVisibleSize.width,
265 mVisibleSize.height);
266 break;
267 case convertMap(VideoPixelFormat::NV21, VideoPixelFormat::I420):
268 libyuv::NV21ToI420(srcY, srcStrideY, srcV, srcStrideV, dstY, dstStrideY, dstU,
269 dstStrideU, dstV, dstStrideV, mVisibleSize.width,
270 mVisibleSize.height);
271 break;
272 case convertMap(VideoPixelFormat::NV21, VideoPixelFormat::NV12):
273 ALOGV("%s(): Converting PIXEL_FORMAT_NV21 -> PIXEL_FORMAT_NV12", __func__);
274 libyuv::CopyPlane(srcY, srcStrideY, dstY, dstStrideY, mVisibleSize.width,
275 mVisibleSize.height);
276 copyPlaneByPixel(srcU, srcStrideU, 2, dstUV, dstStrideUV, 2, mVisibleSize.width / 2,
277 mVisibleSize.height / 2);
278 copyPlaneByPixel(srcV, srcStrideV, 2, dstUV + 1, dstStrideUV, 2, mVisibleSize.width / 2,
279 mVisibleSize.height / 2);
280 break;
281 default:
282 ALOGE("Unsupported pixel format conversion from %s to %s",
283 videoPixelFormatToString(inputFormat).c_str(),
284 videoPixelFormatToString(mOutFormat).c_str());
285 return C2_CORRUPTED;
286 }
287 } else if (inputLayout.type == C2PlanarLayout::TYPE_RGB) {
288 const uint8_t* srcRGB = (idMap) ? idMap->addr() : inputView.data()[C2PlanarLayout::PLANE_R];
289 const int srcStrideRGB =
290 (idMap) ? idMap->rowInc() : inputLayout.planes[C2PlanarLayout::PLANE_R].rowInc;
291
292 switch (convertMap(inputFormat, mOutFormat)) {
293 case convertMap(VideoPixelFormat::ABGR, VideoPixelFormat::I420):
294 libyuv::ABGRToI420(srcRGB, srcStrideRGB, dstY, dstStrideY, dstU, dstStrideU, dstV,
295 dstStrideV, mVisibleSize.width, mVisibleSize.height);
296 break;
297 case convertMap(VideoPixelFormat::ABGR, VideoPixelFormat::NV12): {
298 // There is no libyuv function to convert ABGR to NV12. Therefore, we first convert to
299 // I420 on dst-Y plane and temporary U/V plane. Then we copy U and V pixels from
300 // temporary planes to dst-UV interleavedly.
301 const int tempStride = mVisibleSize.width / 2;
302 libyuv::ABGRToI420(srcRGB, srcStrideRGB, dstY, dstStrideY, mTempPlaneU.get(),
303 tempStride, mTempPlaneV.get(), tempStride, mVisibleSize.width,
304 mVisibleSize.height);
305 libyuv::MergeUVPlane(mTempPlaneU.get(), tempStride, mTempPlaneV.get(), tempStride,
306 dstUV, dstStrideUV, mVisibleSize.width / 2,
307 mVisibleSize.height / 2);
308 break;
309 }
310 default:
311 ALOGE("Unsupported pixel format conversion from %s to %s",
312 videoPixelFormatToString(inputFormat).c_str(),
313 videoPixelFormatToString(mOutFormat).c_str());
314 return C2_CORRUPTED;
315 }
316 } else {
317 ALOGE("Unsupported input layout type");
318 return C2_CORRUPTED;
319 }
320
321 ALOGV("convertBlock(frame_index=%" PRIu64 ", format=%s)", frameIndex,
322 videoPixelFormatToString(inputFormat).c_str());
323 entry->mAssociatedFrameIndex = frameIndex;
324 mAvailableQueue.pop();
325
326 *convertedBlock =
327 outputBlock->share(C2Rect(mVisibleSize.width, mVisibleSize.height), C2Fence());
328 return C2_OK;
329 }
330
returnBlock(uint64_t frameIndex)331 c2_status_t FormatConverter::returnBlock(uint64_t frameIndex) {
332 ALOGV("returnBlock(frame_index=%" PRIu64 ")", frameIndex);
333
334 auto iter = std::find_if(mGraphicBlocks.begin(), mGraphicBlocks.end(),
335 [frameIndex](const std::unique_ptr<BlockEntry>& be) {
336 return be->mAssociatedFrameIndex == frameIndex;
337 });
338 if (iter == mGraphicBlocks.end()) {
339 ALOGE("Failed to find graphic block by converted/zero-copied frame index: %" PRIu64 "",
340 frameIndex);
341 return C2_BAD_INDEX;
342 }
343
344 if ((*iter)->mBlock) {
345 // Returned block is format converted.
346 (*iter)->mAssociatedFrameIndex = kNoFrameAssociated;
347 mAvailableQueue.push(iter->get());
348 } else {
349 // Returned block is zero-copied.
350 mGraphicBlocks.erase(iter);
351 }
352 return C2_OK;
353 }
354
355 } // namespace android
356