1 // Copyright (C) 2019 The Android Open Source Project
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14 
15 #include "host-common/MediaH264DecoderFfmpeg.h"
16 #include "aemu/base/system/System.h"
17 #include "host-common/H264NaluParser.h"
18 #include "host-common/H264PingInfoParser.h"
19 #include "host-common/YuvConverter.h"
20 
21 #include <cstdint>
22 #include <string>
23 #include <vector>
24 
25 #include <stdio.h>
26 #include <string.h>
27 
28 #define MEDIA_H264_DEBUG 0
29 
30 #if MEDIA_H264_DEBUG
31 #define H264_DPRINT(fmt,...) fprintf(stderr, "h264-ffmpeg-dec: %s:%d " fmt "\n", __func__, __LINE__, ##__VA_ARGS__);
32 #else
33 #define H264_DPRINT(fmt,...)
34 #endif
35 
36 namespace android {
37 namespace emulation {
38 
39 using InitContextParam = H264PingInfoParser::InitContextParam;
40 using DecodeFrameParam = H264PingInfoParser::DecodeFrameParam;
41 using ResetParam = H264PingInfoParser::ResetParam;
42 using GetImageParam = H264PingInfoParser::GetImageParam;
~MediaH264DecoderFfmpeg()43 MediaH264DecoderFfmpeg::~MediaH264DecoderFfmpeg() {
44     H264_DPRINT("destroyed MediaH264DecoderFfmpeg %p", this);
45     destroyH264Context();
46 }
47 
reset(void * ptr)48 void MediaH264DecoderFfmpeg::reset(void* ptr) {
49     destroyH264Context();
50     ResetParam param{};
51     mParser.parseResetParams(ptr, param);
52     initH264ContextInternal(param.width, param.height, param.outputWidth,
53                             param.outputHeight, param.outputPixelFormat);
54 }
55 
initH264Context(void * ptr)56 void MediaH264DecoderFfmpeg::initH264Context(void* ptr) {
57     InitContextParam param{};
58     mParser.parseInitContextParams(ptr, param);
59     initH264ContextInternal(param.width, param.height, param.outputWidth,
60                             param.outputHeight, param.outputPixelFormat);
61 }
62 
initH264ContextInternal(unsigned int width,unsigned int height,unsigned int outWidth,unsigned int outHeight,PixelFormat outPixFmt)63 void MediaH264DecoderFfmpeg::initH264ContextInternal(unsigned int width,
64                                                      unsigned int height,
65                                                      unsigned int outWidth,
66                                                      unsigned int outHeight,
67                                                      PixelFormat outPixFmt) {
68     H264_DPRINT("%s(w=%u h=%u out_w=%u out_h=%u pixfmt=%u)",
69                 __func__, width, height, outWidth, outHeight, (uint8_t)outPixFmt);
70     mWidth = width;
71     mHeight = height;
72     mOutputWidth = outWidth;
73     mOutputHeight = outHeight;
74     mOutPixFmt = outPixFmt;
75     mOutBufferSize = outWidth * outHeight * 3 / 2;
76 
77     mIsInFlush = false;
78 
79     mDecodedFrame.resize(mOutBufferSize);
80 
81     // standard ffmpeg codec stuff
82     avcodec_register_all();
83     if(0){
84         AVCodec* current_codec = NULL;
85 
86         current_codec = av_codec_next(current_codec);
87         while (current_codec != NULL)
88         {
89             if (av_codec_is_decoder(current_codec))
90             {
91                 H264_DPRINT("codec decoder found %s long name %s", current_codec->name, current_codec->long_name);
92             }
93             current_codec = av_codec_next(current_codec);
94         }
95     }
96 
97     mCodec = NULL;
98     auto useCuvidEnv = android::base::System::getEnvironmentVariable(
99             "ANDROID_EMU_CODEC_USE_FFMPEG_CUVID_DECODER");
100     if (useCuvidEnv != "") {
101         mCodec = avcodec_find_decoder_by_name("h264_cuvid");
102         if (mCodec) {
103             mIsSoftwareDecoder = false;
104             H264_DPRINT("Found h264_cuvid decoder, using it");
105         } else {
106             H264_DPRINT("Cannot find h264_cuvid decoder");
107         }
108     }
109     if (!mCodec) {
110         mCodec = avcodec_find_decoder(AV_CODEC_ID_H264);
111         H264_DPRINT("Using default software h264 decoder");
112     }
113     mCodecCtx = avcodec_alloc_context3(mCodec);
114 
115     mCodecCtx->thread_count = 4;
116     mCodecCtx->thread_type = FF_THREAD_FRAME;
117     avcodec_open2(mCodecCtx, mCodec, 0);
118     mFrame = av_frame_alloc();
119 
120     H264_DPRINT("Successfully created software h264 decoder context %p", mCodecCtx);
121 }
122 
clone()123 MediaH264DecoderPlugin* MediaH264DecoderFfmpeg::clone() {
124     H264_DPRINT("clone MediaH264DecoderFfmpeg %p with version %d", this,
125                 (int)mParser.version());
126     return new MediaH264DecoderFfmpeg(mId, mParser);
127 }
128 
MediaH264DecoderFfmpeg(uint64_t id,H264PingInfoParser parser)129 MediaH264DecoderFfmpeg::MediaH264DecoderFfmpeg(uint64_t id,
130                                                H264PingInfoParser parser)
131     : mId(id), mParser(parser) {
132     H264_DPRINT("allocated MediaH264DecoderFfmpeg %p with version %d", this,
133                 (int)mParser.version());
134 }
destroyH264Context()135 void MediaH264DecoderFfmpeg::destroyH264Context() {
136     H264_DPRINT("Destroy context %p", this);
137     if (mCodecCtx) {
138         avcodec_close(mCodecCtx);
139         av_free(mCodecCtx);
140         mCodecCtx = NULL;
141     }
142     if (mFrame) {
143         av_frame_free(&mFrame);
144         mFrame = NULL;
145     }
146 }
147 
resetDecoder()148 void MediaH264DecoderFfmpeg::resetDecoder() {
149     mNumDecodedFrame = 0;
150     avcodec_close(mCodecCtx);
151     av_free(mCodecCtx);
152     mCodecCtx = avcodec_alloc_context3(mCodec);
153     avcodec_open2(mCodecCtx, mCodec, 0);
154 }
155 
checkWhetherConfigChanged(const uint8_t * frame,size_t szBytes)156 bool MediaH264DecoderFfmpeg::checkWhetherConfigChanged(const uint8_t* frame, size_t szBytes) {
157     // get frame type
158     // if the frame is none SPS/PPS, return false
159     // otherwise, check both SPS/PPS and return true
160     const uint8_t* currNalu = H264NaluParser::getNextStartCodeHeader(frame, szBytes);
161     if (currNalu == nullptr) {
162         // should not happen
163         H264_DPRINT("Found bad frame");
164         return false;
165     }
166 
167     size_t remaining = szBytes - (currNalu - frame);
168     size_t currNaluSize = remaining;
169     H264NaluParser::H264NaluType currNaluType = H264NaluParser::getFrameNaluType(currNalu, currNaluSize, NULL);
170     if (currNaluType != H264NaluParser::H264NaluType::SPS) {
171         return false;
172     }
173 
174     H264_DPRINT("found SPS\n");
175 
176     const uint8_t* nextNalu = H264NaluParser::getNextStartCodeHeader(currNalu + 3, remaining - 3);
177 
178     if (nextNalu == nullptr) {
179         // only one nalu, cannot have configuration change
180         H264_DPRINT("frame has only one Nalu unit, cannot be configuration change\n");
181         return false;
182     }
183 
184     if (mNumDecodedFrame == 0) {
185         H264_DPRINT("have not decoded anything yet, cannot be config change");
186         return false;
187     }
188     // pretty sure it is config change
189     H264_DPRINT("\n\nDetected stream configuration change !!!\n\n");
190     return true;
191 }
192 
decodeFrameDirect(void * ptr,const uint8_t * frame,size_t szBytes,uint64_t pts)193 void MediaH264DecoderFfmpeg::decodeFrameDirect(void* ptr,
194                                                const uint8_t* frame,
195                                                size_t szBytes,
196                                                uint64_t pts) {
197     DecodeFrameParam param{};
198     mParser.parseDecodeFrameParams(ptr, param);
199     param.pData = (uint8_t*)frame;
200     param.pts = pts;
201     param.size = szBytes;
202     decodeFrameInternal(param);
203 }
204 
decodeFrame(void * ptr)205 void MediaH264DecoderFfmpeg::decodeFrame(void* ptr) {
206     DecodeFrameParam param{};
207     mParser.parseDecodeFrameParams(ptr, param);
208     decodeFrameInternal(param);
209 }
210 
decodeFrameInternal(DecodeFrameParam & param)211 void MediaH264DecoderFfmpeg::decodeFrameInternal(DecodeFrameParam& param) {
212     const uint8_t* frame = param.pData;
213     size_t szBytes = param.size;
214     uint64_t inputPts = param.pts;
215 
216     H264_DPRINT("%s(frame=%p, sz=%zu pts %lld)", __func__, frame, szBytes, (long long)inputPts);
217     Err h264Err = Err::NoErr;
218     // TODO: move this somewhere else
219     // First return parameter is the number of bytes processed,
220     // Second return parameter is the error code
221     size_t* retSzBytes = param.pConsumedBytes;
222     int32_t* retErr = param.pDecoderErrorCode;
223 
224     const bool enableSnapshot = true;
225     if (enableSnapshot) {
226         std::vector<uint8_t> v;
227         v.assign(frame, frame + szBytes);
228         bool hasSps = H264NaluParser::checkSpsFrame(frame, szBytes);
229         if (hasSps) {
230             mSnapshotState = SnapshotState{};
231             mSnapshotState.saveSps(v);
232         } else {
233             bool hasPps = H264NaluParser::checkPpsFrame(frame, szBytes);
234             if (hasPps) {
235                 mSnapshotState.savePps(v);
236                 mSnapshotState.savedPackets.clear();
237                 mSnapshotState.savedDecodedFrame.data.clear();
238             } else {
239                 bool isIFrame = H264NaluParser::checkIFrame(frame, szBytes);
240                 if (isIFrame) {
241                     mSnapshotState.savedPackets.clear();
242                 }
243                 const bool saveOK = mSnapshotState.savePacket(std::move(v), inputPts);
244                 if (saveOK) {
245                     H264_DPRINT("saving packet; total is %d",
246                             (int)(mSnapshotState.savedPackets.size()));
247                 } else {
248                     H264_DPRINT("saving packet; has duplicate, skip; total is %d",
249                             (int)(mSnapshotState.savedPackets.size()));
250                     *retSzBytes = szBytes;
251                     *retErr = (int32_t)h264Err;
252                     mImageReady = true;
253                     return;
254                 }
255             }
256         }
257     }
258 
259     if (!mIsSoftwareDecoder) {
260         bool configChanged = checkWhetherConfigChanged(frame, szBytes);
261         if (configChanged) {
262             resetDecoder();
263         }
264     }
265     av_init_packet(&mPacket);
266     mPacket.data = (unsigned char*)frame;
267     mPacket.size = szBytes;
268     mPacket.pts = inputPts;
269     avcodec_send_packet(mCodecCtx, &mPacket);
270     int retframe = avcodec_receive_frame(mCodecCtx, mFrame);
271     *retSzBytes = szBytes;
272     *retErr = (int32_t)h264Err;
273     mIsInFlush = false;
274     if (retframe != 0) {
275         H264_DPRINT("decodeFrame has nonzero return value %d", retframe);
276         if (retframe == AVERROR_EOF) {
277             H264_DPRINT("EOF returned from decoder");
278             H264_DPRINT("EOF returned from decoder reset context now");
279             resetDecoder();
280         } else if (retframe == AVERROR(EAGAIN)) {
281             H264_DPRINT("EAGAIN returned from decoder");
282         } else {
283             H264_DPRINT("unknown value %d", retframe);
284         }
285         return;
286     }
287     H264_DPRINT("new w %d new h %d, old w %d old h %d",
288                 mFrame->width, mFrame->height,
289                 mOutputWidth, mOutputHeight);
290     mFrameFormatChanged = false;
291     ++mNumDecodedFrame;
292     copyFrame();
293     H264_DPRINT("%s: got frame in decode mode", __func__);
294     mImageReady = true;
295 }
296 
copyFrame()297 void MediaH264DecoderFfmpeg::copyFrame() {
298     int w = mFrame->width;
299     int h = mFrame->height;
300     if (w != mOutputWidth || h != mOutputHeight) {
301         mOutputWidth = w;
302         mOutputHeight= h;
303         mOutBufferSize = mOutputWidth * mOutputHeight * 3 / 2;
304         mDecodedFrame.resize(mOutBufferSize);
305     }
306     H264_DPRINT("w %d h %d Y line size %d U line size %d V line size %d", w, h,
307             mFrame->linesize[0], mFrame->linesize[1], mFrame->linesize[2]);
308     for (int i = 0; i < h; ++i) {
309         memcpy(mDecodedFrame.data() + i * w,
310                mFrame->data[0] + i * mFrame->linesize[0], w);
311     }
312     H264_DPRINT("format is %d and NV21 is %d  12 is %d", mFrame->format, (int)AV_PIX_FMT_NV21,
313             (int)AV_PIX_FMT_NV12);
314     if (mFrame->format == AV_PIX_FMT_NV12) {
315         for (int i=0; i < h / 2; ++i) {
316             memcpy(w * h + mDecodedFrame.data() + i * w,
317                    mFrame->data[1] + i * mFrame->linesize[1], w);
318         }
319         YuvConverter<uint8_t> convert8(mOutputWidth, mOutputHeight);
320         convert8.UVInterleavedToPlanar(mDecodedFrame.data());
321     } else {
322         for (int i=0; i < h / 2; ++i) {
323             memcpy(w * h + mDecodedFrame.data() + i * w / 2,
324                    mFrame->data[1] + i * mFrame->linesize[1], w / 2);
325         }
326         for (int i=0; i < h / 2; ++i) {
327             memcpy(w * h + w * h / 4 + mDecodedFrame.data() + i * w / 2,
328                    mFrame->data[2] + i * mFrame->linesize[2], w / 2);
329         }
330     }
331     mColorPrimaries = mFrame->color_primaries;
332     mColorRange = mFrame->color_range;
333     mColorTransfer = mFrame->color_trc;
334     mColorSpace = mFrame->colorspace;
335     mOutputPts = mFrame->pts;
336     H264_DPRINT("copied Frame and it has presentation time at %lld", (long long)(mFrame->pts));
337     H264_DPRINT("Frame primary %d range %d transfer %d space %d", mFrame->color_primaries,
338             mFrame->color_range, mFrame->color_trc, mFrame->colorspace);
339 }
340 
flush(void * ptr)341 void MediaH264DecoderFfmpeg::flush(void* ptr) {
342     H264_DPRINT("Flushing...");
343     mIsInFlush = true;
344     mSnapshotState = SnapshotState{};
345     H264_DPRINT("Flushing done");
346 }
347 
getImage(void * ptr)348 void MediaH264DecoderFfmpeg::getImage(void* ptr) {
349     H264_DPRINT("getImage %p", ptr);
350     GetImageParam param{};
351     mParser.parseGetImageParams(ptr, param);
352 
353     int* retErr = param.pDecoderErrorCode;
354     uint32_t* retWidth = param.pRetWidth;
355     uint32_t* retHeight = param.pRetHeight;
356     uint64_t* retPts = param.pRetPts;
357     uint32_t* retColorPrimaries = param.pRetColorPrimaries;
358     uint32_t* retColorRange = param.pRetColorRange;
359     uint32_t* retColorTransfer = param.pRetColorTransfer;
360     uint32_t* retColorSpace = param.pRetColorSpace;
361 
362     static int numbers=0;
363     //H264_DPRINT("calling getImage %d", numbers++);
364     if (!mImageReady) {
365         if (mFrameFormatChanged) {
366             *retWidth = mOutputWidth;
367             *retHeight = mOutputHeight;
368             *retErr = static_cast<int>(Err::DecoderRestarted);
369             return;
370         }
371         if (mIsInFlush) {
372             // guest be in flush mode, so try to get a frame
373             avcodec_send_packet(mCodecCtx, NULL);
374             int retframe = avcodec_receive_frame(mCodecCtx, mFrame);
375             if (retframe == AVERROR(EAGAIN) || retframe == AVERROR_EOF) {
376                 H264_DPRINT("%s: frame is null", __func__);
377                 *retErr = static_cast<int>(Err::NoDecodedFrame);
378                 return;
379             }
380 
381             if (retframe != 0) {
382                 char tmp[1024];
383                 av_strerror(retframe, tmp, sizeof(tmp));
384                 H264_DPRINT("WARNING: some unknown error %d: %s", retframe,
385                             tmp);
386                 *retErr = static_cast<int>(Err::NoDecodedFrame);
387                 return;
388             }
389             H264_DPRINT("%s: got frame in flush mode retrun code %d", __func__, retframe);
390             //now copy to mDecodedFrame
391             copyFrame();
392             mImageReady = true;
393         } else {
394             H264_DPRINT("%s: no new frame yet", __func__);
395             *retErr = static_cast<int>(Err::NoDecodedFrame);
396             return;
397         }
398     }
399 
400     *retWidth = mOutputWidth;
401     *retHeight = mOutputHeight;
402     *retPts = mOutputPts;
403     *retColorPrimaries = mColorPrimaries;
404     *retColorRange = mColorRange;
405     *retColorTransfer = mColorTransfer;
406     *retColorSpace = mColorSpace;
407 
408     if (mParser.version() == 100) {
409         uint8_t* dst = param.pDecodedFrame;
410         memcpy(dst, mDecodedFrame.data(), mOutBufferSize);
411     } else if (mParser.version() == 200) {
412         if (param.hostColorBufferId >= 0) {
413             mRenderer.renderToHostColorBuffer(param.hostColorBufferId,
414                                               mOutputWidth, mOutputHeight,
415                                               mDecodedFrame.data());
416         } else {
417             uint8_t* dst = param.pDecodedFrame;
418             memcpy(dst, mDecodedFrame.data(), mOutBufferSize);
419         }
420     }
421 
422     mImageReady = false;
423     *retErr = mOutBufferSize;
424     H264_DPRINT("getImage %p done", ptr);
425 }
426 
save(base::Stream * stream) const427 void MediaH264DecoderFfmpeg::save(base::Stream* stream) const {
428     stream->putBe32(mParser.version());
429     stream->putBe32(mWidth);
430     stream->putBe32(mHeight);
431     stream->putBe32((int)mOutPixFmt);
432 
433     const int hasContext = mCodecCtx != nullptr ? 1 : 0;
434     stream->putBe32(hasContext);
435 
436     if (mImageReady) {
437         mSnapshotState.saveDecodedFrame(
438                 mDecodedFrame, mOutputWidth, mOutputHeight,
439                 ColorAspects{mColorPrimaries, mColorRange, mColorTransfer,
440                              mColorSpace},
441                 mOutputPts);
442     } else {
443         mSnapshotState.savedDecodedFrame.data.clear();
444     }
445     H264_DPRINT("saving packets now %d",
446                 (int)(mSnapshotState.savedPackets.size()));
447     mSnapshotState.save(stream);
448 }
449 
oneShotDecode(std::vector<uint8_t> & data,uint64_t pts)450 void MediaH264DecoderFfmpeg::oneShotDecode(std::vector<uint8_t>& data,
451                                            uint64_t pts) {
452     av_init_packet(&mPacket);
453     mPacket.data = (unsigned char*)(data.data());
454     mPacket.size = data.size();
455     mPacket.pts = pts;
456     H264_DPRINT("decoding pts %lld packet size %d", (long long)pts, (int)data.size());
457     avcodec_send_packet(mCodecCtx, &mPacket);
458     avcodec_receive_frame(mCodecCtx, mFrame);
459 }
460 
load(base::Stream * stream)461 bool MediaH264DecoderFfmpeg::load(base::Stream* stream) {
462     uint32_t version = stream->getBe32();
463     mParser = H264PingInfoParser{version};
464 
465     mWidth = stream->getBe32();
466     mHeight = stream->getBe32();
467     mOutPixFmt = (PixelFormat)stream->getBe32();
468 
469     const int hasContext = stream->getBe32();
470     if (hasContext) {
471         initH264ContextInternal(mWidth, mHeight, mWidth, mHeight, mOutPixFmt);
472         assert(mCodecCtx != nullptr);
473     }
474 
475     mSnapshotState.load(stream);
476 
477     H264_DPRINT("loaded packets %d, now restore decoder",
478                 (int)(mSnapshotState.savedPackets.size()));
479     if (hasContext && mCodecCtx != nullptr && mSnapshotState.sps.size() > 0) {
480         oneShotDecode(mSnapshotState.sps, 0);
481         if (mSnapshotState.pps.size() > 0) {
482             oneShotDecode(mSnapshotState.pps, 0);
483             if (mSnapshotState.savedPackets.size() > 0) {
484                 for (int i = 0; i < mSnapshotState.savedPackets.size(); ++i) {
485                     PacketInfo& pkt = mSnapshotState.savedPackets[i];
486                     oneShotDecode(pkt.data, pkt.pts);
487                 }
488                 copyFrame(); //save the last frame
489             }
490         }
491     }
492 
493     if (mSnapshotState.savedDecodedFrame.data.size() > 0) {
494         mDecodedFrame = mSnapshotState.savedDecodedFrame.data;
495         mOutBufferSize = mSnapshotState.savedDecodedFrame.data.size();
496         mOutputWidth = mSnapshotState.savedDecodedFrame.width;
497         mOutputHeight = mSnapshotState.savedDecodedFrame.height;
498         mColorPrimaries = mSnapshotState.savedDecodedFrame.color.primaries;
499         mColorRange = mSnapshotState.savedDecodedFrame.color.range;
500         mColorTransfer = mSnapshotState.savedDecodedFrame.color.transfer;
501         mColorSpace = mSnapshotState.savedDecodedFrame.color.space;
502         mOutputPts = mSnapshotState.savedDecodedFrame.pts;
503         mImageReady = true;
504     } else {
505         mImageReady = false;
506     }
507     H264_DPRINT("Done loading snapshots frames\n\n");
508     return true;
509 }
510 
511 }  // namespace emulation
512 }  // namespace android
513 
514