xref: /aosp_15_r20/cts/tests/tests/media/codec/src/android/media/codec/cts/MediaCodecTest.java (revision b7c941bb3fa97aba169d73cee0bed2de8ac964bf) 
1 /*
2  * Copyright (C) 2013 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package android.media.codec.cts;
18 
19 import static org.junit.Assert.assertEquals;
20 import static org.junit.Assert.assertFalse;
21 import static org.junit.Assert.assertNotNull;
22 import static org.junit.Assert.assertNull;
23 import static org.junit.Assert.assertSame;
24 import static org.junit.Assert.assertThrows;
25 import static org.junit.Assert.assertTrue;
26 import static org.junit.Assert.fail;
27 
28 import android.content.res.AssetFileDescriptor;
29 import android.hardware.HardwareBuffer;
30 import android.media.AudioFormat;
31 import android.media.AudioPresentation;
32 import android.media.MediaCodec;
33 import android.media.MediaCodec.BufferInfo;
34 import android.media.MediaCodec.CodecException;
35 import android.media.MediaCodec.CryptoException;
36 import android.media.MediaCodec.CryptoInfo;
37 import android.media.MediaCodec.CryptoInfo.Pattern;
38 import android.media.MediaCodecInfo;
39 import android.media.MediaCodecInfo.AudioCapabilities;
40 import android.media.MediaCodecInfo.CodecCapabilities;
41 import android.media.MediaCodecInfo.CodecProfileLevel;
42 import android.media.MediaCodecInfo.EncoderCapabilities;
43 import android.media.MediaCodecInfo.VideoCapabilities;
44 import android.media.MediaCodecList;
45 import android.media.MediaExtractor;
46 import android.media.MediaFormat;
47 import android.media.cts.InputSurface;
48 import android.media.cts.OutputSurface;
49 import android.media.cts.StreamUtils;
50 import android.media.cts.TestUtils;
51 import android.opengl.GLES20;
52 import android.os.Build;
53 import android.os.ConditionVariable;
54 import android.os.Handler;
55 import android.os.HandlerThread;
56 import android.os.ParcelFileDescriptor;
57 import android.os.PersistableBundle;
58 import android.platform.test.annotations.AppModeFull;
59 import android.platform.test.annotations.Presubmit;
60 import android.util.Log;
61 import android.util.Range;
62 import android.view.Surface;
63 
64 import androidx.test.ext.junit.runners.AndroidJUnit4;
65 import androidx.test.filters.MediumTest;
66 import androidx.test.filters.SmallTest;
67 
68 import com.android.compatibility.common.util.ApiLevelUtil;
69 import com.android.compatibility.common.util.ApiTest;
70 import com.android.compatibility.common.util.FrameworkSpecificTest;
71 import com.android.compatibility.common.util.MediaUtils;
72 import com.android.compatibility.common.util.Preconditions;
73 
74 import org.junit.Test;
75 import org.junit.runner.RunWith;
76 
77 import java.io.File;
78 import java.io.IOException;
79 import java.nio.ByteBuffer;
80 import java.util.ArrayList;
81 import java.util.Arrays;
82 import java.util.List;
83 import java.util.UUID;
84 import java.util.concurrent.CountDownLatch;
85 import java.util.concurrent.LinkedBlockingQueue;
86 import java.util.concurrent.TimeUnit;
87 import java.util.concurrent.atomic.AtomicBoolean;
88 import java.util.concurrent.atomic.AtomicInteger;
89 
90 /**
91  * General MediaCodec tests.
92  *
93  * In particular, check various API edge cases.
94  */
95 @Presubmit
96 @SmallTest
97 @AppModeFull(reason = "Instant apps cannot access the SD card")
98 @RunWith(AndroidJUnit4.class)
99 public class MediaCodecTest {
100     private static final String TAG = "MediaCodecTest";
101     private static final boolean VERBOSE = false;           // lots of logging
102 
103     static final String mInpPrefix = WorkDir.getMediaDirString();
104     // parameters for the video encoder
105                                                             // H.264 Advanced Video Coding
106     private static final String MIME_TYPE = MediaFormat.MIMETYPE_VIDEO_AVC;
107     private static final int BIT_RATE = 2000000;            // 2Mbps
108     private static final int FRAME_RATE = 15;               // 15fps
109     private static final int IFRAME_INTERVAL = 10;          // 10 seconds between I-frames
110     private static final int WIDTH = 1280;
111     private static final int HEIGHT = 720;
112     // parameters for the audio encoder
113     private static final String MIME_TYPE_AUDIO = MediaFormat.MIMETYPE_AUDIO_AAC;
114     private static final int AUDIO_SAMPLE_RATE = 44100;
115     private static final int AUDIO_AAC_PROFILE = 2; /* OMX_AUDIO_AACObjectLC */
116     private static final int AUDIO_CHANNEL_COUNT = 2; // mono
117     private static final int AUDIO_BIT_RATE = 128000;
118 
119     private static final int TIMEOUT_USEC = 100000;
120     private static final int TIMEOUT_USEC_SHORT = 100;
121 
122     private boolean mVideoEncoderHadError = false;
123     private boolean mAudioEncoderHadError = false;
124     private volatile boolean mVideoEncodingOngoing = false;
125 
126     private static final String INPUT_RESOURCE =
127             "video_480x360_mp4_h264_1350kbps_30fps_aac_stereo_192kbps_44100hz.mp4";
128 
129     // The test should fail if the decoder never produces output frames for the input.
130     // Time out decoding, as we have no way to query whether the decoder will produce output.
131     private static final int DECODING_TIMEOUT_MS = 10000;
132 
133     private static boolean mIsAtLeastR = ApiLevelUtil.isAtLeast(Build.VERSION_CODES.R);
134     private static boolean mIsAtLeastS = ApiLevelUtil.isAtLeast(Build.VERSION_CODES.S);
135     private static boolean mIsAtLeastU = ApiLevelUtil.isAfter(Build.VERSION_CODES.TIRAMISU);
136 
137     /**
138      * Tests:
139      * <br> Exceptions for MediaCodec factory methods
140      * <br> Exceptions for MediaCodec methods when called in the incorrect state.
141      *
142      * A selective test to ensure proper exceptions are thrown from MediaCodec
143      * methods when called in incorrect operational states.
144      */
145     @ApiTest(apis = {"MediaCodec#createByCodecName", "MediaCodec#createDecoderByType",
146             "MediaCodec#createEncoderByType", "MediaCodec#start", "MediaCodec#flush",
147             "MediaCodec#configure", "MediaCodec#dequeueInputBuffer",
148             "MediaCodec#dequeueOutputBuffer", "MediaCodec#createInputSurface",
149             "MediaCodec#getInputBuffers", "MediaCodec#getQueueRequest",
150             "MediaCodec#getOutputFrame", "MediaCodec#stop", "MediaCodec#release",
151             "MediaCodec#getCodecInfo", "MediaCodec#getSupportedVendorParameters",
152             "MediaCodec#getParameterDescriptor",
153             "MediaCodec#subscribeToVendorParameters",
154             "MediaCodec#unsubscribeFromVendorParameters",
155             "MediaCodec#getInputBuffer", "MediaCodec#getOutputBuffer",
156             "MediaCodec#setCallback", "MediaCodec#getName"})
157     @Test
testException()158     public void testException() throws Exception {
159         boolean tested = false;
160         // audio decoder (MP3 should be present on all Android devices)
161         MediaFormat format = MediaFormat.createAudioFormat(
162                 MediaFormat.MIMETYPE_AUDIO_MPEG, 44100 /* sampleRate */, 2 /* channelCount */);
163         tested = verifyException(format, false /* isEncoder */) || tested;
164 
165         // audio encoder (AMR-WB may not be present on some Android devices)
166         format = MediaFormat.createAudioFormat(
167                 MediaFormat.MIMETYPE_AUDIO_AMR_WB, 16000 /* sampleRate */, 1 /* channelCount */);
168         format.setInteger(MediaFormat.KEY_BIT_RATE, 19850);
169         tested = verifyException(format, true /* isEncoder */) || tested;
170 
171         // video decoder (H.264/AVC may not be present on some Android devices)
172         format = createMediaFormat();
173         tested = verifyException(format, false /* isEncoder */) || tested;
174 
175         // video encoder (H.264/AVC may not be present on some Android devices)
176         tested = verifyException(format, true /* isEncoder */) || tested;
177 
178         // signal test is skipped due to no device media codecs.
179         if (!tested) {
180             MediaUtils.skipTest(TAG, "cannot find any compatible device codecs");
181         }
182     }
183 
184     // wrap MediaCodec encoder and decoder creation
createCodecByType(String type, boolean isEncoder)185     private static MediaCodec createCodecByType(String type, boolean isEncoder)
186             throws IOException {
187         if (isEncoder) {
188             return MediaCodec.createEncoderByType(type);
189         }
190         return MediaCodec.createDecoderByType(type);
191     }
192 
logMediaCodecException(MediaCodec.CodecException ex)193     private static void logMediaCodecException(MediaCodec.CodecException ex) {
194         if (ex.isRecoverable()) {
195             Log.w(TAG, "CodecException Recoverable: " + ex.getErrorCode());
196         } else if (ex.isTransient()) {
197             Log.w(TAG, "CodecException Transient: " + ex.getErrorCode());
198         } else {
199             Log.w(TAG, "CodecException Fatal: " + ex.getErrorCode());
200         }
201     }
202 
verifyException(MediaFormat format, boolean isEncoder)203     private static boolean verifyException(MediaFormat format, boolean isEncoder)
204             throws IOException {
205         String mimeType = format.getString(MediaFormat.KEY_MIME);
206         if (!supportsCodec(mimeType, isEncoder)) {
207             Log.i(TAG, "No " + (isEncoder ? "encoder" : "decoder")
208                     + " found for mimeType= " + mimeType);
209             return false;
210         }
211 
212         final boolean isVideoEncoder = isEncoder && mimeType.startsWith("video/");
213 
214         if (isVideoEncoder) {
215             format = new MediaFormat(format);
216             format.setInteger(MediaFormat.KEY_COLOR_FORMAT,
217                     MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface);
218         }
219 
220         // create codec (enter Initialized State)
221         MediaCodec codec;
222 
223         // create improperly
224         final String methodName = isEncoder ? "createEncoderByType" : "createDecoderByType";
225         try {
226             codec = createCodecByType(null, isEncoder);
227             fail(methodName + " should return NullPointerException on null");
228         } catch (NullPointerException e) { // expected
229         }
230         try {
231             codec = createCodecByType("foobarplan9", isEncoder); // invalid type
232             fail(methodName + " should return IllegalArgumentException on invalid type");
233         } catch (IllegalArgumentException e) { // expected
234         }
235         try {
236             codec = MediaCodec.createByCodecName("foobarplan9"); // invalid name
237             fail(methodName + " should return IllegalArgumentException on invalid name");
238         } catch (IllegalArgumentException e) { // expected
239         }
240         // correct
241         codec = createCodecByType(format.getString(MediaFormat.KEY_MIME), isEncoder);
242 
243         // test a few commands
244         try {
245             codec.start();
246             fail("start should return IllegalStateException when in Initialized state");
247         } catch (MediaCodec.CodecException e) {
248             logMediaCodecException(e);
249             fail("start should not return MediaCodec.CodecException on wrong state");
250         } catch (IllegalStateException e) { // expected
251         }
252         try {
253             codec.flush();
254             fail("flush should return IllegalStateException when in Initialized state");
255         } catch (MediaCodec.CodecException e) {
256             logMediaCodecException(e);
257             fail("flush should not return MediaCodec.CodecException on wrong state");
258         } catch (IllegalStateException e) { // expected
259         }
260 
261         MediaCodecInfo codecInfo = codec.getCodecInfo(); // obtaining the codec info now is fine.
262         try {
263             int bufIndex = codec.dequeueInputBuffer(0);
264             fail("dequeueInputBuffer should return IllegalStateException"
265                     + " when in the Initialized state");
266         } catch (MediaCodec.CodecException e) {
267             logMediaCodecException(e);
268             fail("dequeueInputBuffer should not return MediaCodec.CodecException"
269                     + " on wrong state");
270         } catch (IllegalStateException e) { // expected
271         }
272         try {
273             MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
274             int bufIndex = codec.dequeueOutputBuffer(info, 0);
275             fail("dequeueOutputBuffer should return IllegalStateException"
276                     + " when in the Initialized state");
277         } catch (MediaCodec.CodecException e) {
278             logMediaCodecException(e);
279             fail("dequeueOutputBuffer should not return MediaCodec.CodecException"
280                     + " on wrong state");
281         } catch (IllegalStateException e) { // expected
282         }
283 
284         // configure (enter Configured State)
285 
286         // configure improperly
287         try {
288             codec.configure(format, null /* surface */, null /* crypto */,
289                     isEncoder ? 0 : MediaCodec.CONFIGURE_FLAG_ENCODE /* flags */);
290             fail("configure needs MediaCodec.CONFIGURE_FLAG_ENCODE for encoders only");
291         } catch (MediaCodec.CodecException e) { // expected
292             logMediaCodecException(e);
293         } catch (IllegalStateException e) {
294             fail("configure should not return IllegalStateException when improperly configured");
295         }
296         if (mIsAtLeastU) {
297             try {
298                 int flags = MediaCodec.CONFIGURE_FLAG_USE_CRYPTO_ASYNC |
299                         (isEncoder ? MediaCodec.CONFIGURE_FLAG_ENCODE : 0);
300                 codec.configure(format, null /* surface */, null /* crypto */, flags /* flags */);
301                 fail("At the minimum, CONFIGURE_FLAG_USE_CRYPTO_ASYNC requires setting callback");
302             } catch(IllegalStateException e) { //expected
303                 // Need to set callbacks
304             }
305         }
306         // correct
307         codec.configure(format, null /* surface */, null /* crypto */,
308                 isEncoder ? MediaCodec.CONFIGURE_FLAG_ENCODE : 0 /* flags */);
309 
310         // test a few commands
311         try {
312             codec.flush();
313             fail("flush should return IllegalStateException when in Configured state");
314         } catch (MediaCodec.CodecException e) {
315             logMediaCodecException(e);
316             fail("flush should not return MediaCodec.CodecException on wrong state");
317         } catch (IllegalStateException e) { // expected
318         }
319         try {
320             Surface surface = codec.createInputSurface();
321             if (!isEncoder) {
322                 fail("createInputSurface should not work on a decoder");
323             }
324         } catch (IllegalStateException |
325                  IllegalArgumentException e) { // expected for decoder and audio encoder
326             if (isVideoEncoder) {
327                 throw e;
328             }
329         }
330 
331         // test getInputBuffers before start()
332         try {
333             ByteBuffer[] buffers = codec.getInputBuffers();
334             fail("getInputBuffers called before start() should throw exception");
335         } catch (IllegalStateException e) { // expected
336         }
337 
338         // start codec (enter Executing state)
339         codec.start();
340 
341         // test getInputBuffers after start()
342         try {
343             ByteBuffer[] buffers = codec.getInputBuffers();
344             if (buffers == null) {
345                 fail("getInputBuffers called after start() should not return null");
346             }
347             if (isVideoEncoder && buffers.length > 0) {
348                 fail("getInputBuffers returned non-zero length array with input surface");
349             }
350         } catch (IllegalStateException e) {
351             fail("getInputBuffers called after start() shouldn't throw exception");
352         }
353 
354         // test a few commands
355         try {
356             codec.configure(format, null /* surface */, null /* crypto */, 0 /* flags */);
357             fail("configure should return IllegalStateException when in Executing state");
358         } catch (MediaCodec.CodecException e) {
359             logMediaCodecException(e);
360             // TODO: consider configuring after a flush.
361             fail("configure should not return MediaCodec.CodecException on wrong state");
362         } catch (IllegalStateException e) { // expected
363         }
364         if (mIsAtLeastR) {
365             try {
366                 codec.getQueueRequest(0);
367                 fail("getQueueRequest should throw IllegalStateException when not configured with " +
368                         "CONFIGURE_FLAG_USE_BLOCK_MODEL");
369             } catch (MediaCodec.CodecException e) {
370                 logMediaCodecException(e);
371                 fail("getQueueRequest should not return " +
372                         "MediaCodec.CodecException on wrong configuration");
373             } catch (IllegalStateException e) { // expected
374             }
375             try {
376                 codec.getOutputFrame(0);
377                 fail("getOutputFrame should throw IllegalStateException when not configured with " +
378                         "CONFIGURE_FLAG_USE_BLOCK_MODEL");
379             } catch (MediaCodec.CodecException e) {
380                 logMediaCodecException(e);
381                 fail("getOutputFrame should not return MediaCodec.CodecException on wrong " +
382                         "configuration");
383             } catch (IllegalStateException e) { // expected
384             }
385         }
386 
387         // two flushes should be fine.
388         codec.flush();
389         codec.flush();
390 
391         // stop codec (enter Initialized state)
392         // two stops should be fine.
393         codec.stop();
394         codec.stop();
395 
396         // release codec (enter Uninitialized state)
397         // two releases should be fine.
398         codec.release();
399         codec.release();
400 
401         try {
402             codecInfo = codec.getCodecInfo();
403             fail("getCodecInfo should should return IllegalStateException" +
404                     " when in Uninitialized state");
405         } catch (MediaCodec.CodecException e) {
406             logMediaCodecException(e);
407             fail("getCodecInfo should not return MediaCodec.CodecException on wrong state");
408         } catch (IllegalStateException e) { // expected
409         }
410         try {
411             codec.stop();
412             fail("stop should return IllegalStateException when in Uninitialized state");
413         } catch (MediaCodec.CodecException e) {
414             logMediaCodecException(e);
415             fail("stop should not return MediaCodec.CodecException on wrong state");
416         } catch (IllegalStateException e) { // expected
417         }
418 
419         if (mIsAtLeastS) {
420             try {
421                 codec.getSupportedVendorParameters();
422                 fail("getSupportedVendorParameters should throw IllegalStateException" +
423                         " when in Uninitialized state");
424             } catch (IllegalStateException e) { // expected
425             } catch (Exception e) {
426                 fail("unexpected exception: " + e.toString());
427             }
428             try {
429                 codec.getParameterDescriptor("");
430                 fail("getParameterDescriptor should throw IllegalStateException" +
431                         " when in Uninitialized state");
432             } catch (IllegalStateException e) { // expected
433             } catch (Exception e) {
434                 fail("unexpected exception: " + e.toString());
435             }
436             try {
437                 codec.subscribeToVendorParameters(List.of(""));
438                 fail("subscribeToVendorParameters should throw IllegalStateException" +
439                         " when in Uninitialized state");
440             } catch (IllegalStateException e) { // expected
441             } catch (Exception e) {
442                 fail("unexpected exception: " + e.toString());
443             }
444             try {
445                 codec.unsubscribeFromVendorParameters(List.of(""));
446                 fail("unsubscribeFromVendorParameters should throw IllegalStateException" +
447                         " when in Uninitialized state");
448             } catch (IllegalStateException e) { // expected
449             } catch (Exception e) {
450                 fail("unexpected exception: " + e.toString());
451             }
452         }
453 
454         if (mIsAtLeastR) {
455             // recreate
456             codec = createCodecByType(format.getString(MediaFormat.KEY_MIME), isEncoder);
457 
458             if (isVideoEncoder) {
459                 format.setInteger(MediaFormat.KEY_COLOR_FORMAT,
460                         MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible);
461             }
462 
463             // configure improperly
464             try {
465                 codec.configure(format, null /* surface */, null /* crypto */,
466                         MediaCodec.CONFIGURE_FLAG_USE_BLOCK_MODEL |
467                         (isEncoder ? MediaCodec.CONFIGURE_FLAG_ENCODE : 0) /* flags */);
468                 fail("configure with detached buffer mode should be done after setCallback");
469             } catch (MediaCodec.CodecException e) {
470                 logMediaCodecException(e);
471                 fail("configure should not return IllegalStateException when improperly configured");
472             } catch (IllegalStateException e) { // expected
473             }
474 
475             final LinkedBlockingQueue<Integer> inputQueue = new LinkedBlockingQueue<>();
476             codec.setCallback(new MediaCodec.Callback() {
477                 @Override
478                 public void onInputBufferAvailable(MediaCodec codec, int index) {
479                     inputQueue.offer(index);
480                 }
481                 @Override
482                 public void onOutputBufferAvailable(
483                         MediaCodec codec, int index, MediaCodec.BufferInfo info) { }
484                 @Override
485                 public void onOutputFormatChanged(MediaCodec codec, MediaFormat format) { }
486                 @Override
487                 public void onError(MediaCodec codec, CodecException e) { }
488             });
489 
490             // configure with CONFIGURE_FLAG_USE_BLOCK_MODEL (enter Configured State)
491             codec.configure(format, null /* surface */, null /* crypto */,
492                     MediaCodec.CONFIGURE_FLAG_USE_BLOCK_MODEL |
493                     (isEncoder ? MediaCodec.CONFIGURE_FLAG_ENCODE : 0) /* flags */);
494 
495             // start codec (enter Executing state)
496             codec.start();
497 
498             // grab input index (this should happen immediately)
499             Integer index = null;
500             try {
501                 index = inputQueue.poll(2, TimeUnit.SECONDS);
502             } catch (InterruptedException e) {
503             }
504             assertNotNull(index);
505 
506             // test a few commands
507             try {
508                 codec.getInputBuffers();
509                 fail("getInputBuffers called in detached buffer mode should throw exception");
510             } catch (MediaCodec.IncompatibleWithBlockModelException e) { // expected
511             }
512             try {
513                 codec.getOutputBuffers();
514                 fail("getOutputBuffers called in detached buffer mode should throw exception");
515             } catch (MediaCodec.IncompatibleWithBlockModelException e) { // expected
516             }
517             try {
518                 codec.getInputBuffer(index);
519                 fail("getInputBuffer called in detached buffer mode should throw exception");
520             } catch (MediaCodec.IncompatibleWithBlockModelException e) { // expected
521             }
522             try {
523                 codec.dequeueInputBuffer(0);
524                 fail("dequeueInputBuffer called in detached buffer mode should throw exception");
525             } catch (MediaCodec.IncompatibleWithBlockModelException e) { // expected
526             }
527             MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
528             try {
529                 codec.dequeueOutputBuffer(info, 0);
530                 fail("dequeueOutputBuffer called in detached buffer mode should throw exception");
531             } catch (MediaCodec.IncompatibleWithBlockModelException e) { // expected
532             }
533 
534             // test getQueueRequest
535             MediaCodec.QueueRequest request = codec.getQueueRequest(index);
536             try {
537                 request.queue();
538                 fail("QueueRequest should throw IllegalStateException when no buffer is set");
539             } catch (IllegalStateException e) { // expected
540             }
541             // setting a block
542             String[] names = new String[]{ codec.getName() };
543             request.setLinearBlock(MediaCodec.LinearBlock.obtain(1, names), 0, 0);
544             // setting additional block should fail
545             try (HardwareBuffer buffer = HardwareBuffer.create(
546                     16 /* width */,
547                     16 /* height */,
548                     HardwareBuffer.YCBCR_420_888,
549                     1 /* layers */,
550                     HardwareBuffer.USAGE_CPU_READ_OFTEN | HardwareBuffer.USAGE_CPU_WRITE_OFTEN)) {
551                 request.setHardwareBuffer(buffer);
552                 fail("QueueRequest should throw IllegalStateException multiple blocks are set.");
553             } catch (IllegalStateException e) { // expected
554             }
555         }
556 
557         // release codec
558         codec.release();
559 
560         return true;
561     }
562 
563     /**
564      * Tests:
565      * <br> calling createInputSurface() before configure() throws exception
566      * <br> calling createInputSurface() after start() throws exception
567      * <br> calling createInputSurface() with a non-Surface color format is not required to throw exception
568      */
569     @ApiTest(apis = "MediaCodec#createInputSurface")
570     @Test
571     @FrameworkSpecificTest // tests only the first MIME_TYPE codec, which is usually hardware
testCreateInputSurfaceErrors()572     public void testCreateInputSurfaceErrors() {
573         if (!supportsCodec(MIME_TYPE, true)) {
574             Log.i(TAG, "No encoder found for mimeType= " + MIME_TYPE);
575             return;
576         }
577 
578         MediaFormat format = createMediaFormat();
579         MediaCodec encoder = null;
580         Surface surface = null;
581 
582         // Replace color format with something that isn't COLOR_FormatSurface.
583         MediaCodecInfo codecInfo = selectCodec(MIME_TYPE);
584         int colorFormat = findNonSurfaceColorFormat(codecInfo, MIME_TYPE);
585         format.setInteger(MediaFormat.KEY_COLOR_FORMAT, colorFormat);
586 
587         try {
588             try {
589                 encoder = MediaCodec.createByCodecName(codecInfo.getName());
590             } catch (IOException e) {
591                 fail("failed to create codec " + codecInfo.getName());
592             }
593             try {
594                 surface = encoder.createInputSurface();
595                 fail("createInputSurface should not work pre-configure");
596             } catch (IllegalStateException ise) {
597                 // good
598             }
599             encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
600             encoder.start();
601             try {
602                 surface = encoder.createInputSurface();
603                 fail("createInputSurface should not work post-start");
604             } catch (IllegalStateException ise) {
605                 // good
606             }
607         } finally {
608             if (encoder != null) {
609                 encoder.stop();
610                 encoder.release();
611             }
612         }
613         assertNull(surface);
614     }
615 
616     /**
617      * Tests:
618      * <br> signaling end-of-stream before any data is sent works
619      * <br> signaling EOS twice throws exception
620      * <br> submitting a frame after EOS throws exception [TODO]
621      */
622     @ApiTest(apis = "MediaCodec#signalEndOfInputStream")
623     @Test
624     @FrameworkSpecificTest // tests only the first MIME_TYPE codec, which is usually hardware
testSignalSurfaceEOS()625     public void testSignalSurfaceEOS() {
626         if (!supportsCodec(MIME_TYPE, true)) {
627             Log.i(TAG, "No encoder found for mimeType= " + MIME_TYPE);
628             return;
629         }
630 
631         MediaFormat format = createMediaFormat();
632         MediaCodec encoder = null;
633         InputSurface inputSurface = null;
634 
635         try {
636             try {
637                 encoder = MediaCodec.createEncoderByType(MIME_TYPE);
638             } catch (IOException e) {
639                 fail("failed to create " + MIME_TYPE + " encoder");
640             }
641             encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
642             inputSurface = new InputSurface(encoder.createInputSurface());
643             inputSurface.makeCurrent();
644             encoder.start();
645 
646             // send an immediate EOS
647             encoder.signalEndOfInputStream();
648 
649             try {
650                 encoder.signalEndOfInputStream();
651                 fail("should not be able to signal EOS twice");
652             } catch (IllegalStateException ise) {
653                 // good
654             }
655 
656             // submit a frame post-EOS
657             GLES20.glClearColor(0.0f, 0.5f, 0.0f, 1.0f);
658             GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
659             try {
660                 inputSurface.swapBuffers();
661                 if (false) {    // TODO
662                     fail("should not be able to submit frame after EOS");
663                 }
664             } catch (Exception ex) {
665                 // good
666             }
667         } finally {
668             if (encoder != null) {
669                 encoder.stop();
670                 encoder.release();
671             }
672             if (inputSurface != null) {
673                 inputSurface.release();
674             }
675         }
676     }
677 
678     /**
679      * Tests:
680      * <br> stopping with buffers in flight doesn't crash or hang
681      */
682     @ApiTest(apis = "MediaCodec#stop")
683     @Test
684     @FrameworkSpecificTest // tests only the first MIME_TYPE codec, which is usually hardware
testAbruptStop()685     public void testAbruptStop() {
686         if (!supportsCodec(MIME_TYPE, true)) {
687             Log.i(TAG, "No encoder found for mimeType= " + MIME_TYPE);
688             return;
689         }
690 
691         // There appears to be a race, so run it several times with a short delay between runs
692         // to allow any previous activity to shut down.
693         for (int i = 0; i < 50; i++) {
694             Log.d(TAG, "testAbruptStop " + i);
695             doTestAbruptStop();
696             try { Thread.sleep(400); } catch (InterruptedException ignored) {}
697         }
698     }
doTestAbruptStop()699     private void doTestAbruptStop() {
700         MediaFormat format = createMediaFormat();
701         MediaCodec encoder = null;
702         InputSurface inputSurface = null;
703 
704         try {
705             try {
706                 encoder = MediaCodec.createEncoderByType(MIME_TYPE);
707             } catch (IOException e) {
708                 fail("failed to create " + MIME_TYPE + " encoder");
709             }
710             encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
711             inputSurface = new InputSurface(encoder.createInputSurface());
712             inputSurface.makeCurrent();
713             encoder.start();
714 
715             int totalBuffers = encoder.getOutputBuffers().length;
716             if (VERBOSE) Log.d(TAG, "Total buffers: " + totalBuffers);
717 
718             // Submit several frames quickly, without draining the encoder output, to try to
719             // ensure that we've got some queued up when we call stop().  If we do too many
720             // we'll block in swapBuffers().
721             for (int i = 0; i < totalBuffers; i++) {
722                 GLES20.glClearColor(0.0f, (i % 8) / 8.0f, 0.0f, 1.0f);
723                 GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
724                 inputSurface.swapBuffers();
725             }
726             Log.d(TAG, "stopping");
727             encoder.stop();
728             Log.d(TAG, "stopped");
729         } finally {
730             if (encoder != null) {
731                 encoder.stop();
732                 encoder.release();
733             }
734             if (inputSurface != null) {
735                 inputSurface.release();
736             }
737         }
738     }
739 
740     @ApiTest(apis = {"MediaCodec#flush", "MediaCodec#release"})
741     @Test
testReleaseAfterFlush()742     public void testReleaseAfterFlush() throws IOException, InterruptedException {
743         String mimes[] = new String[] { MIME_TYPE, MIME_TYPE_AUDIO};
744         for (String mime : mimes) {
745             if (!MediaUtils.checkEncoder(mime)) {
746                 continue;
747             }
748             testReleaseAfterFlush(mime);
749         }
750     }
751 
testReleaseAfterFlush(String mime)752     private void testReleaseAfterFlush(String mime) throws IOException, InterruptedException {
753         CountDownLatch buffersExhausted = null;
754         CountDownLatch codecFlushed = null;
755         AtomicInteger numBuffers = null;
756 
757         // sync flush from same thread
758         MediaCodec encoder = MediaCodec.createEncoderByType(mime);
759         runReleaseAfterFlush(mime, encoder, buffersExhausted, codecFlushed, numBuffers);
760 
761         // sync flush from different thread
762         encoder = MediaCodec.createEncoderByType(mime);
763         buffersExhausted = new CountDownLatch(1);
764         codecFlushed = new CountDownLatch(1);
765         numBuffers = new AtomicInteger();
766         Thread flushThread = new FlushThread(encoder, buffersExhausted, codecFlushed);
767         flushThread.start();
768         runReleaseAfterFlush(mime, encoder, buffersExhausted, codecFlushed, numBuffers);
769         flushThread.join();
770 
771         // async
772         // This value is calculated in getOutputBufferIndices by calling dequeueOutputBuffer
773         // with a fixed timeout until buffers are exhausted; it is possible that random timing
774         // in dequeueOutputBuffer can result in a smaller `nBuffs` than the max possible value.
775         int nBuffs = numBuffers.get();
776         HandlerThread callbackThread = new HandlerThread("ReleaseAfterFlushCallbackThread");
777         callbackThread.start();
778         Handler handler = new Handler(callbackThread.getLooper());
779 
780         // async flush from same thread
781         encoder = MediaCodec.createEncoderByType(mime);
782         buffersExhausted = null;
783         codecFlushed = null;
784         ReleaseAfterFlushCallback callback =
785                 new ReleaseAfterFlushCallback(mime, encoder, buffersExhausted, codecFlushed, nBuffs);
786         encoder.setCallback(callback, handler); // setCallback before configure, which is called in run
787         callback.run(); // drive input on main thread
788 
789         // async flush from different thread
790         encoder = MediaCodec.createEncoderByType(mime);
791         buffersExhausted = new CountDownLatch(1);
792         codecFlushed = new CountDownLatch(1);
793         callback = new ReleaseAfterFlushCallback(mime, encoder, buffersExhausted, codecFlushed, nBuffs);
794         encoder.setCallback(callback, handler);
795         flushThread = new FlushThread(encoder, buffersExhausted, codecFlushed);
796         flushThread.start();
797         callback.run();
798         flushThread.join();
799 
800         callbackThread.quitSafely();
801         callbackThread.join();
802     }
803 
804     @ApiTest(apis = {"MediaCodec#setCallback", "MediaCodec#flush", "MediaCodec#reset"})
805     @Test
testAsyncFlushAndReset()806     public void testAsyncFlushAndReset() throws Exception, InterruptedException {
807         testAsyncReset(false /* testStop */);
808     }
809 
810     @ApiTest(apis = {"MediaCodec#setCallback", "MediaCodec#stop", "MediaCodec#reset"})
811     @Test
testAsyncStopAndReset()812     public void testAsyncStopAndReset() throws Exception, InterruptedException {
813         testAsyncReset(true /* testStop */);
814     }
815 
testAsyncReset(boolean testStop)816     private void testAsyncReset(boolean testStop) throws Exception, InterruptedException {
817         // Test video and audio 10x each
818         for (int i = 0; i < 10; i++) {
819             testAsyncReset(false /* audio */, (i % 2) == 0 /* swap */, testStop);
820         }
821         for (int i = 0; i < 10; i++) {
822             testAsyncReset(true /* audio */, (i % 2) == 0 /* swap */, testStop);
823         }
824     }
825 
826     /*
827      * This method simulates a race between flush (or stop) and reset() called from
828      * two threads. Neither call should get stuck. This should be run multiple rounds.
829      */
testAsyncReset(boolean audio, boolean swap, final boolean testStop)830     private void testAsyncReset(boolean audio, boolean swap, final boolean testStop)
831             throws Exception, InterruptedException {
832         String mimeTypePrefix  = audio ? "audio/" : "video/";
833         final MediaExtractor mediaExtractor = getMediaExtractorForMimeType(
834                 INPUT_RESOURCE, mimeTypePrefix);
835         MediaFormat mediaFormat = mediaExtractor.getTrackFormat(
836                 mediaExtractor.getSampleTrackIndex());
837         if (!MediaUtils.checkDecoderForFormat(mediaFormat)) {
838             return; // skip
839         }
840 
841         OutputSurface outputSurface = audio ? null : new OutputSurface(1, 1);
842         final Surface surface = outputSurface == null ? null : outputSurface.getSurface();
843 
844         String mimeType = mediaFormat.getString(MediaFormat.KEY_MIME);
845         final MediaCodec mediaCodec = MediaCodec.createDecoderByType(mimeType);
846 
847         try {
848             mediaCodec.configure(mediaFormat, surface, null /* crypto */, 0 /* flags */);
849 
850             mediaCodec.start();
851 
852             assertTrue(runDecodeTillFirstOutput(mediaCodec, mediaExtractor));
853 
854             Thread flushingThread = new Thread(new Runnable() {
855                 @Override
856                 public void run() {
857                     try {
858                         if (testStop) {
859                             mediaCodec.stop();
860                         } else {
861                             mediaCodec.flush();
862                         }
863                     } catch (IllegalStateException e) {
864                         // This is okay, since we're simulating a race between flush and reset.
865                         // If reset executed first, flush could fail.
866                     }
867                 }
868             });
869 
870             Thread resettingThread = new Thread(new Runnable() {
871                 @Override
872                 public void run() {
873                     mediaCodec.reset();
874                 }
875             });
876 
877             // start flushing (or stopping) and resetting in two threads
878             if (swap) {
879                 flushingThread.start();
880                 resettingThread.start();
881             } else {
882                 resettingThread.start();
883                 flushingThread.start();
884             }
885 
886             // wait for at most 5 sec, and check if the thread exits properly
887             flushingThread.join(5000);
888             assertFalse(flushingThread.isAlive());
889 
890             resettingThread.join(5000);
891             assertFalse(resettingThread.isAlive());
892         } finally {
893             if (mediaCodec != null) {
894                 mediaCodec.release();
895             }
896             if (mediaExtractor != null) {
897                 mediaExtractor.release();
898             }
899             if (outputSurface != null) {
900                 outputSurface.release();
901             }
902         }
903     }
904 
905     private static class FlushThread extends Thread {
906         final MediaCodec mEncoder;
907         final CountDownLatch mBuffersExhausted;
908         final CountDownLatch mCodecFlushed;
909 
FlushThread(MediaCodec encoder, CountDownLatch buffersExhausted, CountDownLatch codecFlushed)910         FlushThread(MediaCodec encoder, CountDownLatch buffersExhausted,
911                 CountDownLatch codecFlushed) {
912             mEncoder = encoder;
913             mBuffersExhausted = buffersExhausted;
914             mCodecFlushed = codecFlushed;
915         }
916 
917         @Override
run()918         public void run() {
919             try {
920                 mBuffersExhausted.await();
921             } catch (InterruptedException e) {
922                 Thread.currentThread().interrupt();
923                 Log.w(TAG, "buffersExhausted wait interrupted; flushing immediately.", e);
924             }
925             mEncoder.flush();
926             mCodecFlushed.countDown();
927         }
928     }
929 
930     private static class ReleaseAfterFlushCallback extends MediaCodec.Callback implements Runnable {
931         final String mMime;
932         final MediaCodec mEncoder;
933         final CountDownLatch mBuffersExhausted, mCodecFlushed;
934         final int mNumBuffersBeforeFlush;
935 
936         CountDownLatch mStopInput = new CountDownLatch(1);
937         List<Integer> mInputBufferIndices = new ArrayList<>();
938         List<Integer> mOutputBufferIndices = new ArrayList<>();
939 
ReleaseAfterFlushCallback(String mime, MediaCodec encoder, CountDownLatch buffersExhausted, CountDownLatch codecFlushed, int numBuffersBeforeFlush)940         ReleaseAfterFlushCallback(String mime,
941                 MediaCodec encoder,
942                 CountDownLatch buffersExhausted,
943                 CountDownLatch codecFlushed,
944                 int numBuffersBeforeFlush) {
945             mMime = mime;
946             mEncoder = encoder;
947             mBuffersExhausted = buffersExhausted;
948             mCodecFlushed = codecFlushed;
949             mNumBuffersBeforeFlush = numBuffersBeforeFlush;
950         }
951 
952         @Override
onInputBufferAvailable(MediaCodec codec, int index)953         public void onInputBufferAvailable(MediaCodec codec, int index) {
954             assertTrue("video onInputBufferAvailable " + index, mMime.startsWith("audio/"));
955             synchronized (mInputBufferIndices) {
956                 mInputBufferIndices.add(index);
957             };
958         }
959 
960         @Override
onOutputBufferAvailable(MediaCodec codec, int index, BufferInfo info)961         public void onOutputBufferAvailable(MediaCodec codec, int index, BufferInfo info) {
962             mOutputBufferIndices.add(index);
963             if (mOutputBufferIndices.size() == mNumBuffersBeforeFlush) {
964                 releaseAfterFlush(codec, mOutputBufferIndices, mBuffersExhausted, mCodecFlushed);
965                 mStopInput.countDown();
966             }
967         }
968 
969         @Override
onError(MediaCodec codec, CodecException e)970         public void onError(MediaCodec codec, CodecException e) {
971             Log.e(TAG, codec + " onError", e);
972             fail(codec + " onError " + e.getMessage());
973         }
974 
975         @Override
onOutputFormatChanged(MediaCodec codec, MediaFormat format)976         public void onOutputFormatChanged(MediaCodec codec, MediaFormat format) {
977             Log.v(TAG, codec + " onOutputFormatChanged " + format);
978         }
979 
980         @Override
run()981         public void run() {
982             InputSurface inputSurface = null;
983             try {
984                 inputSurface = initCodecAndSurface(mMime, mEncoder);
985                 do {
986                     int inputIndex = -1;
987                     if (inputSurface == null) {
988                         // asynchronous audio codec
989                         synchronized (mInputBufferIndices) {
990                             if (mInputBufferIndices.isEmpty()) {
991                                 continue;
992                             } else {
993                                 inputIndex = mInputBufferIndices.remove(0);
994                             }
995                         }
996                     }
997                     feedEncoder(mEncoder, inputSurface, inputIndex);
998                 } while (!mStopInput.await(TIMEOUT_USEC, TimeUnit.MICROSECONDS));
999             } catch (InterruptedException e) {
1000                 Thread.currentThread().interrupt();
1001                 Log.w(TAG, "mEncoder input frames interrupted/stopped", e);
1002             } finally {
1003                 cleanupCodecAndSurface(mEncoder, inputSurface);
1004             }
1005         }
1006     }
1007 
runReleaseAfterFlush( String mime, MediaCodec encoder, CountDownLatch buffersExhausted, CountDownLatch codecFlushed, AtomicInteger numBuffers)1008     private static void runReleaseAfterFlush(
1009             String mime,
1010             MediaCodec encoder,
1011             CountDownLatch buffersExhausted,
1012             CountDownLatch codecFlushed,
1013             AtomicInteger numBuffers) {
1014         InputSurface inputSurface = null;
1015         try {
1016             inputSurface = initCodecAndSurface(mime, encoder);
1017             List<Integer> outputBufferIndices = getOutputBufferIndices(encoder, inputSurface);
1018             if (numBuffers != null) {
1019                 numBuffers.set(outputBufferIndices.size());
1020             }
1021             releaseAfterFlush(encoder, outputBufferIndices, buffersExhausted, codecFlushed);
1022         } finally {
1023             cleanupCodecAndSurface(encoder, inputSurface);
1024         }
1025     }
1026 
initCodecAndSurface(String mime, MediaCodec encoder)1027     private static InputSurface initCodecAndSurface(String mime, MediaCodec encoder) {
1028         MediaFormat format;
1029         InputSurface inputSurface = null;
1030         if (mime.startsWith("audio/")) {
1031             format = MediaFormat.createAudioFormat(mime, AUDIO_SAMPLE_RATE, AUDIO_CHANNEL_COUNT);
1032             format.setInteger(MediaFormat.KEY_AAC_PROFILE, AUDIO_AAC_PROFILE);
1033             format.setInteger(MediaFormat.KEY_BIT_RATE, AUDIO_BIT_RATE);
1034             encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
1035         } else if (MIME_TYPE.equals(mime)) {
1036             CodecInfo info = getAvcSupportedFormatInfo();
1037             format = MediaFormat.createVideoFormat(mime, info.mMaxW, info.mMaxH);
1038             format.setInteger(MediaFormat.KEY_COLOR_FORMAT,
1039                     MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface);
1040             format.setInteger(MediaFormat.KEY_BIT_RATE, info.mBitRate);
1041             format.setInteger(MediaFormat.KEY_FRAME_RATE, info.mFps);
1042             format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, IFRAME_INTERVAL);
1043             OutputSurface outputSurface = new OutputSurface(1, 1);
1044             encoder.configure(format, outputSurface.getSurface(), null, MediaCodec.CONFIGURE_FLAG_ENCODE);
1045             inputSurface = new InputSurface(encoder.createInputSurface());
1046             inputSurface.makeCurrent();
1047         } else {
1048             throw new IllegalArgumentException("unsupported mime type: " + mime);
1049         }
1050         encoder.start();
1051         return inputSurface;
1052     }
1053 
cleanupCodecAndSurface(MediaCodec encoder, InputSurface inputSurface)1054     private static void cleanupCodecAndSurface(MediaCodec encoder, InputSurface inputSurface) {
1055         if (encoder != null) {
1056             encoder.stop();
1057             encoder.release();
1058         }
1059 
1060         if (inputSurface != null) {
1061             inputSurface.release();
1062         }
1063     }
1064 
getOutputBufferIndices(MediaCodec encoder, InputSurface inputSurface)1065     private static List<Integer> getOutputBufferIndices(MediaCodec encoder, InputSurface inputSurface) {
1066         boolean feedMoreFrames;
1067         MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();
1068         List<Integer> indices = new ArrayList<>();
1069         do {
1070             feedMoreFrames = indices.isEmpty();
1071             feedEncoder(encoder, inputSurface, -1);
1072             // dequeue buffers until not available
1073             int index = encoder.dequeueOutputBuffer(bufferInfo, TIMEOUT_USEC);
1074             while (index >= 0) {
1075                 indices.add(index);
1076                 index = encoder.dequeueOutputBuffer(bufferInfo, TIMEOUT_USEC_SHORT);
1077             }
1078         } while (feedMoreFrames);
1079         assertFalse(indices.isEmpty());
1080         return indices;
1081     }
1082 
1083     /**
1084      * @param encoder audio/video encoder
1085      * @param inputSurface null for and only for audio encoders
1086      * @param inputIndex only used for audio; if -1 the function would attempt to dequeue from encoder;
1087      * do not use -1 for asynchronous encoders
1088      */
feedEncoder(MediaCodec encoder, InputSurface inputSurface, int inputIndex)1089     private static void feedEncoder(MediaCodec encoder, InputSurface inputSurface, int inputIndex) {
1090         if (inputSurface == null) {
1091             // audio
1092             while (inputIndex == -1) {
1093                 inputIndex = encoder.dequeueInputBuffer(TIMEOUT_USEC);
1094             }
1095             ByteBuffer inputBuffer = encoder.getInputBuffer(inputIndex);;
1096             for (int i = 0; i < inputBuffer.capacity() / 2; i++) {
1097                 inputBuffer.putShort((short)i);
1098             }
1099             encoder.queueInputBuffer(inputIndex, 0, inputBuffer.limit(), 0, 0);
1100         } else {
1101             // video
1102             GLES20.glClearColor(0.0f, 0.5f, 0.0f, 1.0f);
1103             GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
1104             inputSurface.swapBuffers();
1105         }
1106     }
1107 
releaseAfterFlush( MediaCodec encoder, List<Integer> outputBufferIndices, CountDownLatch buffersExhausted, CountDownLatch codecFlushed)1108     private static void releaseAfterFlush(
1109             MediaCodec encoder,
1110             List<Integer> outputBufferIndices,
1111             CountDownLatch buffersExhausted,
1112             CountDownLatch codecFlushed) {
1113         if (buffersExhausted == null) {
1114             // flush from same thread
1115             encoder.flush();
1116         } else {
1117             assertNotNull(codecFlushed);
1118             buffersExhausted.countDown();
1119             try {
1120                 codecFlushed.await();
1121             } catch (InterruptedException e) {
1122                 Thread.currentThread().interrupt();
1123                 Log.w(TAG, "codecFlushed wait interrupted; releasing buffers immediately.", e);
1124             }
1125         }
1126 
1127         for (int index : outputBufferIndices) {
1128             try {
1129                 encoder.releaseOutputBuffer(index, true);
1130                 fail("MediaCodec releaseOutputBuffer after flush() does not throw exception");
1131             } catch (MediaCodec.CodecException e) {
1132                 // Expected
1133             }
1134         }
1135     }
1136 
1137     /**
1138      * Tests:
1139      * <br> dequeueInputBuffer() fails when encoder configured with an input Surface
1140      */
1141     @ApiTest(apis = {"MediaCodec#dequeueInputBuffer", "MediaCodec#getMetrics"})
1142     @Test
1143     @FrameworkSpecificTest // tests only the first MIME_TYPE codec, which is usually hardware
testDequeueSurface()1144     public void testDequeueSurface() {
1145         if (!supportsCodec(MIME_TYPE, true)) {
1146             Log.i(TAG, "No encoder found for mimeType= " + MIME_TYPE);
1147             return;
1148         }
1149 
1150         MediaFormat format = createMediaFormat();
1151         MediaCodec encoder = null;
1152         Surface surface = null;
1153 
1154         try {
1155             try {
1156                 encoder = MediaCodec.createEncoderByType(MIME_TYPE);
1157             } catch (IOException e) {
1158                 fail("failed to create " + MIME_TYPE + " encoder");
1159             }
1160             encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
1161             surface = encoder.createInputSurface();
1162             encoder.start();
1163 
1164             try {
1165                 encoder.dequeueInputBuffer(-1);
1166                 fail("dequeueInputBuffer should fail on encoder with input surface");
1167             } catch (IllegalStateException ise) {
1168                 // good
1169             }
1170 
1171             PersistableBundle metrics = encoder.getMetrics();
1172             if (metrics == null) {
1173                 fail("getMetrics() returns null");
1174             } else if (metrics.isEmpty()) {
1175                 fail("getMetrics() returns empty results");
1176             }
1177             int encoding = metrics.getInt(MediaCodec.MetricsConstants.ENCODER, -1);
1178             if (encoding != 1) {
1179                 fail("getMetrics() returns bad encoder value " + encoding);
1180             }
1181             String theCodec = metrics.getString(MediaCodec.MetricsConstants.CODEC, null);
1182             if (theCodec == null) {
1183                 fail("getMetrics() returns null codec value ");
1184             }
1185 
1186         } finally {
1187             if (encoder != null) {
1188                 encoder.stop();
1189                 encoder.release();
1190             }
1191             if (surface != null) {
1192                 surface.release();
1193             }
1194         }
1195     }
1196 
1197     /**
1198      * Tests:
1199      * <br> configure() encoder with Surface, re-configure() without Surface works
1200      * <br> sending EOS with signalEndOfInputStream on non-Surface encoder fails
1201      */
1202     @ApiTest(apis = {"MediaCodec#configure", "MediaCodec#signalEndOfInputStream",
1203             "MediaCodec#getMetrics"})
1204     @Test
1205     @FrameworkSpecificTest // tests only the first MIME_TYPE codec, which is usually hardware
testReconfigureWithoutSurface()1206     public void testReconfigureWithoutSurface() {
1207         if (!supportsCodec(MIME_TYPE, true)) {
1208             Log.i(TAG, "No encoder found for mimeType= " + MIME_TYPE);
1209             return;
1210         }
1211 
1212         MediaFormat format = createMediaFormat();
1213         MediaCodec encoder = null;
1214         Surface surface = null;
1215 
1216         try {
1217             try {
1218                 encoder = MediaCodec.createEncoderByType(MIME_TYPE);
1219             } catch (IOException e) {
1220                 fail("failed to create " + MIME_TYPE + " encoder");
1221             }
1222             encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
1223             surface = encoder.createInputSurface();
1224             encoder.start();
1225 
1226             encoder.getOutputBuffers();
1227 
1228             // re-configure, this time without an input surface
1229             if (VERBOSE) Log.d(TAG, "reconfiguring");
1230             encoder.stop();
1231             // Use non-opaque color format for byte buffer mode.
1232             format.setInteger(MediaFormat.KEY_COLOR_FORMAT,
1233                     MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible);
1234             encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
1235             encoder.start();
1236             if (VERBOSE) Log.d(TAG, "reconfigured");
1237 
1238             encoder.getOutputBuffers();
1239             encoder.dequeueInputBuffer(-1);
1240 
1241             try {
1242                 encoder.signalEndOfInputStream();
1243                 fail("signalEndOfInputStream only works on surface input");
1244             } catch (IllegalStateException ise) {
1245                 // good
1246             }
1247 
1248             PersistableBundle metrics = encoder.getMetrics();
1249             if (metrics == null) {
1250                 fail("getMetrics() returns null");
1251             } else if (metrics.isEmpty()) {
1252                 fail("getMetrics() returns empty results");
1253             }
1254             int encoding = metrics.getInt(MediaCodec.MetricsConstants.ENCODER, -1);
1255             if (encoding != 1) {
1256                 fail("getMetrics() returns bad encoder value " + encoding);
1257             }
1258             String theCodec = metrics.getString(MediaCodec.MetricsConstants.CODEC, null);
1259             if (theCodec == null) {
1260                 fail("getMetrics() returns null codec value ");
1261             }
1262 
1263         } finally {
1264             if (encoder != null) {
1265                 encoder.stop();
1266                 encoder.release();
1267             }
1268             if (surface != null) {
1269                 surface.release();
1270             }
1271         }
1272     }
1273 
1274     @ApiTest(apis = "MediaCodec#flush")
1275     @Test
testDecodeAfterFlush()1276     public void testDecodeAfterFlush() throws InterruptedException {
1277         testDecodeAfterFlush(true /* audio */);
1278         testDecodeAfterFlush(false /* audio */);
1279     }
1280 
testDecodeAfterFlush(final boolean audio)1281     private void testDecodeAfterFlush(final boolean audio) throws InterruptedException {
1282         final AtomicBoolean completed = new AtomicBoolean(false);
1283         Thread decodingThread = new Thread(new Runnable() {
1284             @Override
1285             public void run() {
1286                 OutputSurface outputSurface = null;
1287                 MediaExtractor mediaExtractor = null;
1288                 MediaCodec mediaCodec = null;
1289                 try {
1290                     String mimeTypePrefix  = audio ? "audio/" : "video/";
1291                     if (!audio) {
1292                         outputSurface = new OutputSurface(1, 1);
1293                     }
1294                     mediaExtractor = getMediaExtractorForMimeType(INPUT_RESOURCE, mimeTypePrefix);
1295                     MediaFormat mediaFormat =
1296                             mediaExtractor.getTrackFormat(mediaExtractor.getSampleTrackIndex());
1297                     if (!MediaUtils.checkDecoderForFormat(mediaFormat)) {
1298                         completed.set(true);
1299                         return; // skip
1300                     }
1301                     String mimeType = mediaFormat.getString(MediaFormat.KEY_MIME);
1302                     mediaCodec = MediaCodec.createDecoderByType(mimeType);
1303                     mediaCodec.configure(mediaFormat, outputSurface == null ? null : outputSurface.getSurface(),
1304                             null /* crypto */, 0 /* flags */);
1305                     mediaCodec.start();
1306 
1307                     if (!runDecodeTillFirstOutput(mediaCodec, mediaExtractor)) {
1308                         throw new RuntimeException("decoder does not generate non-empty output.");
1309                     }
1310 
1311                     PersistableBundle metrics = mediaCodec.getMetrics();
1312                     if (metrics == null) {
1313                         fail("getMetrics() returns null");
1314                     } else if (metrics.isEmpty()) {
1315                         fail("getMetrics() returns empty results");
1316                     }
1317                     int encoder = metrics.getInt(MediaCodec.MetricsConstants.ENCODER, -1);
1318                     if (encoder != 0) {
1319                         fail("getMetrics() returns bad encoder value " + encoder);
1320                     }
1321                     String theCodec = metrics.getString(MediaCodec.MetricsConstants.CODEC, null);
1322                     if (theCodec == null) {
1323                         fail("getMetrics() returns null codec value ");
1324                     }
1325 
1326 
1327                     // simulate application flush.
1328                     mediaExtractor.seekTo(0, MediaExtractor.SEEK_TO_CLOSEST_SYNC);
1329                     mediaCodec.flush();
1330 
1331                     completed.set(runDecodeTillFirstOutput(mediaCodec, mediaExtractor));
1332                     metrics = mediaCodec.getMetrics();
1333                     if (metrics == null) {
1334                         fail("getMetrics() returns null");
1335                     } else if (metrics.isEmpty()) {
1336                         fail("getMetrics() returns empty results");
1337                     }
1338                     int encoding = metrics.getInt(MediaCodec.MetricsConstants.ENCODER, -1);
1339                     if (encoding != 0) {
1340                         fail("getMetrics() returns bad encoder value " + encoding);
1341                     }
1342                     String theCodec2 = metrics.getString(MediaCodec.MetricsConstants.CODEC, null);
1343                     if (theCodec2 == null) {
1344                         fail("getMetrics() returns null codec value ");
1345                     }
1346 
1347                 } catch (IOException e) {
1348                     throw new RuntimeException("error setting up decoding", e);
1349                 } finally {
1350                     if (mediaCodec != null) {
1351                         mediaCodec.stop();
1352 
1353                         PersistableBundle metrics = mediaCodec.getMetrics();
1354                         if (metrics == null) {
1355                             fail("getMetrics() returns null");
1356                         } else if (metrics.isEmpty()) {
1357                             fail("getMetrics() returns empty results");
1358                         }
1359                         int encoder = metrics.getInt(MediaCodec.MetricsConstants.ENCODER, -1);
1360                         if (encoder != 0) {
1361                             fail("getMetrics() returns bad encoder value " + encoder);
1362                         }
1363                         String theCodec = metrics.getString(MediaCodec.MetricsConstants.CODEC, null);
1364                         if (theCodec == null) {
1365                             fail("getMetrics() returns null codec value ");
1366                         }
1367 
1368                         mediaCodec.release();
1369                     }
1370                     if (mediaExtractor != null) {
1371                         mediaExtractor.release();
1372                     }
1373                     if (outputSurface != null) {
1374                         outputSurface.release();
1375                     }
1376                 }
1377             }
1378         });
1379         decodingThread.start();
1380         decodingThread.join(DECODING_TIMEOUT_MS);
1381         // In case it's timed out, need to stop the thread and have all resources released.
1382         decodingThread.interrupt();
1383         if (!completed.get()) {
1384             throw new RuntimeException("timed out decoding to end-of-stream");
1385         }
1386     }
1387 
1388     // Run the decoder till it generates an output buffer.
1389     // Return true when that output buffer is not empty, false otherwise.
runDecodeTillFirstOutput( MediaCodec mediaCodec, MediaExtractor mediaExtractor)1390     private static boolean runDecodeTillFirstOutput(
1391             MediaCodec mediaCodec, MediaExtractor mediaExtractor) {
1392         final int TIME_OUT_US = 10000;
1393 
1394         assertTrue("Wrong test stream which has no data.",
1395                 mediaExtractor.getSampleTrackIndex() != -1);
1396         boolean signaledEos = false;
1397         MediaCodec.BufferInfo outputBufferInfo = new MediaCodec.BufferInfo();
1398         while (!Thread.interrupted()) {
1399             // Try to feed more data into the codec.
1400             if (!signaledEos) {
1401                 int bufferIndex = mediaCodec.dequeueInputBuffer(TIME_OUT_US /* timeoutUs */);
1402                 if (bufferIndex != -1) {
1403                     ByteBuffer buffer = mediaCodec.getInputBuffer(bufferIndex);
1404                     int size = mediaExtractor.readSampleData(buffer, 0 /* offset */);
1405                     long timestampUs = mediaExtractor.getSampleTime();
1406                     mediaExtractor.advance();
1407                     signaledEos = mediaExtractor.getSampleTrackIndex() == -1;
1408                     mediaCodec.queueInputBuffer(bufferIndex,
1409                             0 /* offset */,
1410                             size,
1411                             timestampUs,
1412                             signaledEos ? MediaCodec.BUFFER_FLAG_END_OF_STREAM : 0);
1413                     Log.i("DEBUG", "queue with " + signaledEos);
1414                 }
1415             }
1416 
1417             int outputBufferIndex = mediaCodec.dequeueOutputBuffer(
1418                     outputBufferInfo, TIME_OUT_US /* timeoutUs */);
1419 
1420             if (outputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED
1421                     || outputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED
1422                     || outputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
1423                 continue;
1424             }
1425             assertTrue("Wrong output buffer index", outputBufferIndex >= 0);
1426 
1427             PersistableBundle metrics = mediaCodec.getMetrics();
1428             Log.d(TAG, "getMetrics after first buffer metrics says: " + metrics);
1429 
1430             int encoder = metrics.getInt(MediaCodec.MetricsConstants.ENCODER, -1);
1431             if (encoder != 0) {
1432                 fail("getMetrics() returns bad encoder value " + encoder);
1433             }
1434             String theCodec = metrics.getString(MediaCodec.MetricsConstants.CODEC, null);
1435             if (theCodec == null) {
1436                 fail("getMetrics() returns null codec value ");
1437             }
1438 
1439             mediaCodec.releaseOutputBuffer(outputBufferIndex, false /* render */);
1440             boolean eos = (outputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0;
1441             Log.i("DEBUG", "Got a frame with eos=" + eos);
1442             if (eos && outputBufferInfo.size == 0) {
1443                 return false;
1444             } else {
1445                 return true;
1446             }
1447         }
1448 
1449         return false;
1450     }
1451 
1452     /**
1453      * Tests whether decoding a short group-of-pictures succeeds. The test queues a few video frames
1454      * then signals end-of-stream. The test fails if the decoder doesn't output the queued frames.
1455      */
1456     @ApiTest(apis = {"MediaCodecInfo.CodecCapabilities#COLOR_FormatSurface"})
1457     @Test
testDecodeShortInput()1458     public void testDecodeShortInput() throws InterruptedException {
1459         // Input buffers from this input video are queued up to and including the video frame with
1460         // timestamp LAST_BUFFER_TIMESTAMP_US.
1461         final String INPUT_RESOURCE =
1462                 "video_480x360_mp4_h264_1350kbps_30fps_aac_stereo_192kbps_44100hz.mp4";
1463         final long LAST_BUFFER_TIMESTAMP_US = 166666;
1464 
1465         // The test should fail if the decoder never produces output frames for the truncated input.
1466         // Time out decoding, as we have no way to query whether the decoder will produce output.
1467         final int DECODING_TIMEOUT_MS = 2000;
1468 
1469         final AtomicBoolean completed = new AtomicBoolean();
1470         Thread videoDecodingThread = new Thread(new Runnable() {
1471             @Override
1472             public void run() {
1473                 completed.set(runDecodeShortInput(INPUT_RESOURCE, LAST_BUFFER_TIMESTAMP_US));
1474             }
1475         });
1476         videoDecodingThread.start();
1477         videoDecodingThread.join(DECODING_TIMEOUT_MS);
1478         if (!completed.get()) {
1479             throw new RuntimeException("timed out decoding to end-of-stream");
1480         }
1481     }
1482 
runDecodeShortInput(final String inputResource, long lastBufferTimestampUs)1483     private boolean runDecodeShortInput(final String inputResource, long lastBufferTimestampUs) {
1484         final int NO_BUFFER_INDEX = -1;
1485 
1486         OutputSurface outputSurface = null;
1487         MediaExtractor mediaExtractor = null;
1488         MediaCodec mediaCodec = null;
1489         try {
1490             outputSurface = new OutputSurface(1, 1);
1491             mediaExtractor = getMediaExtractorForMimeType(inputResource, "video/");
1492             MediaFormat mediaFormat =
1493                     mediaExtractor.getTrackFormat(mediaExtractor.getSampleTrackIndex());
1494             String mimeType = mediaFormat.getString(MediaFormat.KEY_MIME);
1495             if (!supportsCodec(mimeType, false)) {
1496                 Log.i(TAG, "No decoder found for mimeType= " + MIME_TYPE);
1497                 return true;
1498             }
1499             mediaCodec =
1500                     MediaCodec.createDecoderByType(mimeType);
1501             mediaCodec.configure(mediaFormat, outputSurface.getSurface(), null, 0);
1502             mediaCodec.start();
1503             boolean eos = false;
1504             boolean signaledEos = false;
1505             MediaCodec.BufferInfo outputBufferInfo = new MediaCodec.BufferInfo();
1506             int outputBufferIndex = NO_BUFFER_INDEX;
1507             while (!eos && !Thread.interrupted()) {
1508                 // Try to feed more data into the codec.
1509                 if (mediaExtractor.getSampleTrackIndex() != -1 && !signaledEos) {
1510                     int bufferIndex = mediaCodec.dequeueInputBuffer(0);
1511                     if (bufferIndex != NO_BUFFER_INDEX) {
1512                         ByteBuffer buffer = mediaCodec.getInputBuffers()[bufferIndex];
1513                         int size = mediaExtractor.readSampleData(buffer, 0);
1514                         long timestampUs = mediaExtractor.getSampleTime();
1515                         mediaExtractor.advance();
1516                         signaledEos = mediaExtractor.getSampleTrackIndex() == -1
1517                                 || timestampUs == lastBufferTimestampUs;
1518                         mediaCodec.queueInputBuffer(bufferIndex,
1519                                 0,
1520                                 size,
1521                                 timestampUs,
1522                                 signaledEos ? MediaCodec.BUFFER_FLAG_END_OF_STREAM : 0);
1523                     }
1524                 }
1525 
1526                 // If we don't have an output buffer, try to get one now.
1527                 if (outputBufferIndex == NO_BUFFER_INDEX) {
1528                     outputBufferIndex = mediaCodec.dequeueOutputBuffer(outputBufferInfo, 0);
1529                 }
1530 
1531                 if (outputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED
1532                         || outputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED
1533                         || outputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
1534                     outputBufferIndex = NO_BUFFER_INDEX;
1535                 } else if (outputBufferIndex != NO_BUFFER_INDEX) {
1536                     eos = (outputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0;
1537 
1538                     boolean render = outputBufferInfo.size > 0;
1539                     mediaCodec.releaseOutputBuffer(outputBufferIndex, render);
1540                     if (render) {
1541                         outputSurface.awaitNewImage();
1542                     }
1543 
1544                     outputBufferIndex = NO_BUFFER_INDEX;
1545                 }
1546             }
1547 
1548             return eos;
1549         } catch (IOException e) {
1550             throw new RuntimeException("error reading input resource", e);
1551         } finally {
1552             if (mediaCodec != null) {
1553                 mediaCodec.stop();
1554                 mediaCodec.release();
1555             }
1556             if (mediaExtractor != null) {
1557                 mediaExtractor.release();
1558             }
1559             if (outputSurface != null) {
1560                 outputSurface.release();
1561             }
1562         }
1563     }
1564 
1565     /**
1566      * Tests creating two decoders for {@link #MIME_TYPE_AUDIO} at the same time.
1567      */
1568     @ApiTest(apis = {"MediaCodec#createDecoderByType",
1569             "android.media.MediaFormat#KEY_MIME",
1570             "android.media.MediaFormat#KEY_SAMPLE_RATE",
1571             "android.media.MediaFormat#KEY_CHANNEL_COUNT"})
1572     @Test
testCreateTwoAudioDecoders()1573     public void testCreateTwoAudioDecoders() {
1574         final MediaFormat format = MediaFormat.createAudioFormat(
1575                 MIME_TYPE_AUDIO, AUDIO_SAMPLE_RATE, AUDIO_CHANNEL_COUNT);
1576 
1577         MediaCodec audioDecoderA = null;
1578         MediaCodec audioDecoderB = null;
1579         try {
1580             try {
1581                 audioDecoderA = MediaCodec.createDecoderByType(MIME_TYPE_AUDIO);
1582             } catch (IOException e) {
1583                 fail("failed to create first " + MIME_TYPE_AUDIO + " decoder");
1584             }
1585             audioDecoderA.configure(format, null, null, 0);
1586             audioDecoderA.start();
1587 
1588             try {
1589                 audioDecoderB = MediaCodec.createDecoderByType(MIME_TYPE_AUDIO);
1590             } catch (IOException e) {
1591                 fail("failed to create second " + MIME_TYPE_AUDIO + " decoder");
1592             }
1593             audioDecoderB.configure(format, null, null, 0);
1594             audioDecoderB.start();
1595         } finally {
1596             if (audioDecoderB != null) {
1597                 try {
1598                     audioDecoderB.stop();
1599                     audioDecoderB.release();
1600                 } catch (RuntimeException e) {
1601                     Log.w(TAG, "exception stopping/releasing codec", e);
1602                 }
1603             }
1604 
1605             if (audioDecoderA != null) {
1606                 try {
1607                     audioDecoderA.stop();
1608                     audioDecoderA.release();
1609                 } catch (RuntimeException e) {
1610                     Log.w(TAG, "exception stopping/releasing codec", e);
1611                 }
1612             }
1613         }
1614     }
1615 
1616     /**
1617      * Tests creating an encoder and decoder for {@link #MIME_TYPE_AUDIO} at the same time.
1618      */
1619     @ApiTest(apis = {"MediaCodec#createDecoderByType", "MediaCodec#createEncoderByType",
1620             "android.media.MediaFormat#KEY_MIME",
1621             "android.media.MediaFormat#KEY_SAMPLE_RATE",
1622             "android.media.MediaFormat#KEY_CHANNEL_COUNT"})
1623     @Test
testCreateAudioDecoderAndEncoder()1624     public void testCreateAudioDecoderAndEncoder() {
1625         if (!supportsCodec(MIME_TYPE_AUDIO, true)) {
1626             Log.i(TAG, "No encoder found for mimeType= " + MIME_TYPE_AUDIO);
1627             return;
1628         }
1629 
1630         if (!supportsCodec(MIME_TYPE_AUDIO, false)) {
1631             Log.i(TAG, "No decoder found for mimeType= " + MIME_TYPE_AUDIO);
1632             return;
1633         }
1634 
1635         final MediaFormat encoderFormat = MediaFormat.createAudioFormat(
1636                 MIME_TYPE_AUDIO, AUDIO_SAMPLE_RATE, AUDIO_CHANNEL_COUNT);
1637         encoderFormat.setInteger(MediaFormat.KEY_AAC_PROFILE, AUDIO_AAC_PROFILE);
1638         encoderFormat.setInteger(MediaFormat.KEY_BIT_RATE, AUDIO_BIT_RATE);
1639         final MediaFormat decoderFormat = MediaFormat.createAudioFormat(
1640                 MIME_TYPE_AUDIO, AUDIO_SAMPLE_RATE, AUDIO_CHANNEL_COUNT);
1641 
1642         MediaCodec audioEncoder = null;
1643         MediaCodec audioDecoder = null;
1644         try {
1645             try {
1646                 audioEncoder = MediaCodec.createEncoderByType(MIME_TYPE_AUDIO);
1647             } catch (IOException e) {
1648                 fail("failed to create " + MIME_TYPE_AUDIO + " encoder");
1649             }
1650             audioEncoder.configure(encoderFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
1651             audioEncoder.start();
1652 
1653             try {
1654                 audioDecoder = MediaCodec.createDecoderByType(MIME_TYPE_AUDIO);
1655             } catch (IOException e) {
1656                 fail("failed to create " + MIME_TYPE_AUDIO + " decoder");
1657             }
1658             audioDecoder.configure(decoderFormat, null, null, 0);
1659             audioDecoder.start();
1660         } finally {
1661             if (audioDecoder != null) {
1662                 try {
1663                     audioDecoder.stop();
1664                     audioDecoder.release();
1665                 } catch (RuntimeException e) {
1666                     Log.w(TAG, "exception stopping/releasing codec", e);
1667                 }
1668             }
1669 
1670             if (audioEncoder != null) {
1671                 try {
1672                     audioEncoder.stop();
1673                     audioEncoder.release();
1674                 } catch (RuntimeException e) {
1675                     Log.w(TAG, "exception stopping/releasing codec", e);
1676                 }
1677             }
1678         }
1679     }
1680 
1681     @ApiTest(apis = {"MediaCodec#createEncoderByType",
1682             "android.media.MediaFormat#KEY_MIME",
1683             "android.media.MediaFormat#KEY_SAMPLE_RATE",
1684             "android.media.MediaFormat#KEY_CHANNEL_COUNT",
1685             "android.media.MediaFormat#KEY_WIDTH",
1686             "android.media.MediaFormat#KEY_HEIGHT"})
1687     @Test
testConcurrentAudioVideoEncodings()1688     public void testConcurrentAudioVideoEncodings() throws InterruptedException {
1689         if (!supportsCodec(MIME_TYPE_AUDIO, true)) {
1690             Log.i(TAG, "No encoder found for mimeType= " + MIME_TYPE_AUDIO);
1691             return;
1692         }
1693 
1694         if (!supportsCodec(MIME_TYPE, true)) {
1695             Log.i(TAG, "No decoder found for mimeType= " + MIME_TYPE);
1696             return;
1697         }
1698 
1699         final int VIDEO_NUM_SWAPS = 100;
1700         // audio only checks this and stop
1701         mVideoEncodingOngoing = true;
1702         final CodecInfo info = getAvcSupportedFormatInfo();
1703         long start = System.currentTimeMillis();
1704         Thread videoEncodingThread = new Thread(new Runnable() {
1705             @Override
1706             public void run() {
1707                 runVideoEncoding(VIDEO_NUM_SWAPS, info);
1708             }
1709         });
1710         Thread audioEncodingThread = new Thread(new Runnable() {
1711             @Override
1712             public void run() {
1713                 runAudioEncoding();
1714             }
1715         });
1716         videoEncodingThread.start();
1717         audioEncodingThread.start();
1718         videoEncodingThread.join();
1719         mVideoEncodingOngoing = false;
1720         audioEncodingThread.join();
1721         assertFalse("Video encoding error. Chekc logcat", mVideoEncoderHadError);
1722         assertFalse("Audio encoding error. Chekc logcat", mAudioEncoderHadError);
1723         long end = System.currentTimeMillis();
1724         Log.w(TAG, "Concurrent AV encoding took " + (end - start) + " ms for " + VIDEO_NUM_SWAPS +
1725                 " video frames");
1726     }
1727 
1728     private static class CodecInfo {
1729         public int mMaxW;
1730         public int mMaxH;
1731         public int mFps;
1732         public int mBitRate;
1733     }
1734 
1735     @ApiTest(apis = {"MediaCodec#CryptoInfo", "MediaCodec#CryptoInfo#Pattern"})
1736     @Test
1737     @FrameworkSpecificTest // media mainline doesn't update crypto
testCryptoInfoPattern()1738     public void testCryptoInfoPattern() {
1739         CryptoInfo info = new CryptoInfo();
1740         Pattern pattern = new Pattern(1 /*blocksToEncrypt*/, 2 /*blocksToSkip*/);
1741         assertEquals(1, pattern.getEncryptBlocks());
1742         assertEquals(2, pattern.getSkipBlocks());
1743         pattern.set(3 /*blocksToEncrypt*/, 4 /*blocksToSkip*/);
1744         assertEquals(3, pattern.getEncryptBlocks());
1745         assertEquals(4, pattern.getSkipBlocks());
1746         info.setPattern(pattern);
1747         // Check that CryptoInfo does not leak access to the underlying pattern.
1748         if (mIsAtLeastS) {
1749             // getPattern() availability SDK>=S
1750             pattern.set(10, 10);
1751             info.getPattern().set(10, 10);
1752             assertSame(3, info.getPattern().getEncryptBlocks());
1753             assertSame(4, info.getPattern().getSkipBlocks());
1754         }
1755     }
1756 
getAvcSupportedFormatInfo()1757     private static CodecInfo getAvcSupportedFormatInfo() {
1758         MediaCodecInfo mediaCodecInfo = selectCodec(MIME_TYPE);
1759         CodecCapabilities cap = mediaCodecInfo.getCapabilitiesForType(MIME_TYPE);
1760         if (cap == null) { // not supported
1761             return null;
1762         }
1763         CodecInfo info = new CodecInfo();
1764         int highestLevel = 0;
1765         for (CodecProfileLevel lvl : cap.profileLevels) {
1766             if (lvl.level > highestLevel) {
1767                 highestLevel = lvl.level;
1768             }
1769         }
1770         int maxW = 0;
1771         int maxH = 0;
1772         int bitRate = 0;
1773         int fps = 0; // frame rate for the max resolution
1774         switch(highestLevel) {
1775             // Do not support Level 1 to 2.
1776             case CodecProfileLevel.AVCLevel1:
1777             case CodecProfileLevel.AVCLevel11:
1778             case CodecProfileLevel.AVCLevel12:
1779             case CodecProfileLevel.AVCLevel13:
1780             case CodecProfileLevel.AVCLevel1b:
1781             case CodecProfileLevel.AVCLevel2:
1782                 return null;
1783             case CodecProfileLevel.AVCLevel21:
1784                 maxW = 352;
1785                 maxH = 576;
1786                 bitRate = 4000000;
1787                 fps = 25;
1788                 break;
1789             case CodecProfileLevel.AVCLevel22:
1790                 maxW = 720;
1791                 maxH = 480;
1792                 bitRate = 4000000;
1793                 fps = 15;
1794                 break;
1795             case CodecProfileLevel.AVCLevel3:
1796                 maxW = 720;
1797                 maxH = 480;
1798                 bitRate = 10000000;
1799                 fps = 30;
1800                 break;
1801             case CodecProfileLevel.AVCLevel31:
1802                 maxW = 1280;
1803                 maxH = 720;
1804                 bitRate = 14000000;
1805                 fps = 30;
1806                 break;
1807             case CodecProfileLevel.AVCLevel32:
1808                 maxW = 1280;
1809                 maxH = 720;
1810                 bitRate = 20000000;
1811                 fps = 60;
1812                 break;
1813             case CodecProfileLevel.AVCLevel4: // only try up to 1080p
1814             default:
1815                 maxW = 1920;
1816                 maxH = 1080;
1817                 bitRate = 20000000;
1818                 fps = 30;
1819                 break;
1820         }
1821         info.mMaxW = maxW;
1822         info.mMaxH = maxH;
1823         info.mFps = fps;
1824         info.mBitRate = bitRate;
1825         Log.i(TAG, "AVC Level 0x" + Integer.toHexString(highestLevel) + " bit rate " + bitRate +
1826                 " fps " + info.mFps + " w " + maxW + " h " + maxH);
1827 
1828         return info;
1829     }
1830 
runVideoEncoding(int numSwap, CodecInfo info)1831     private void runVideoEncoding(int numSwap, CodecInfo info) {
1832         MediaFormat format = MediaFormat.createVideoFormat(MIME_TYPE, info.mMaxW, info.mMaxH);
1833         format.setInteger(MediaFormat.KEY_COLOR_FORMAT,
1834                 MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface);
1835         format.setInteger(MediaFormat.KEY_BIT_RATE, info.mBitRate);
1836         format.setInteger(MediaFormat.KEY_FRAME_RATE, info.mFps);
1837         format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, IFRAME_INTERVAL);
1838         MediaCodec encoder = null;
1839         InputSurface inputSurface = null;
1840         mVideoEncoderHadError = false;
1841         try {
1842             encoder = MediaCodec.createEncoderByType(MIME_TYPE);
1843             encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
1844             inputSurface = new InputSurface(encoder.createInputSurface());
1845             inputSurface.makeCurrent();
1846             MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();
1847             encoder.start();
1848             for (int i = 0; i < numSwap; i++) {
1849                 GLES20.glClearColor(0.0f, 0.5f, 0.0f, 1.0f);
1850                 GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
1851                 inputSurface.swapBuffers();
1852                 // dequeue buffers until not available
1853                 int index = encoder.dequeueOutputBuffer(bufferInfo, TIMEOUT_USEC);
1854                 while (index >= 0) {
1855                     encoder.releaseOutputBuffer(index, false);
1856                     // just throw away output
1857                     // allow shorter wait for 2nd round to move on quickly.
1858                     index = encoder.dequeueOutputBuffer(bufferInfo, TIMEOUT_USEC_SHORT);
1859                 }
1860             }
1861             encoder.signalEndOfInputStream();
1862         } catch (Throwable e) {
1863             Log.w(TAG, "runVideoEncoding got error: " + e);
1864             mVideoEncoderHadError = true;
1865         } finally {
1866             if (encoder != null) {
1867                 encoder.stop();
1868                 encoder.release();
1869             }
1870             if (inputSurface != null) {
1871                 inputSurface.release();
1872             }
1873         }
1874     }
1875 
runAudioEncoding()1876     private void runAudioEncoding() {
1877         MediaFormat format = MediaFormat.createAudioFormat(MIME_TYPE_AUDIO, AUDIO_SAMPLE_RATE,
1878                 AUDIO_CHANNEL_COUNT);
1879         format.setInteger(MediaFormat.KEY_AAC_PROFILE, AUDIO_AAC_PROFILE);
1880         format.setInteger(MediaFormat.KEY_BIT_RATE, AUDIO_BIT_RATE);
1881         MediaCodec encoder = null;
1882         mAudioEncoderHadError = false;
1883         try {
1884             encoder = MediaCodec.createEncoderByType(MIME_TYPE_AUDIO);
1885             encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
1886             MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
1887             encoder.start();
1888             ByteBuffer[] inputBuffers = encoder.getInputBuffers();
1889             ByteBuffer source = ByteBuffer.allocate(inputBuffers[0].capacity());
1890             for (int i = 0; i < source.capacity()/2; i++) {
1891                 source.putShort((short)i);
1892             }
1893             source.rewind();
1894             int currentInputBufferIndex = 0;
1895             long encodingLatencySum = 0;
1896             int totalEncoded = 0;
1897             int numRepeat = 0;
1898             while (mVideoEncodingOngoing) {
1899                 numRepeat++;
1900                 int inputIndex = encoder.dequeueInputBuffer(TIMEOUT_USEC);
1901                 while (inputIndex == -1) {
1902                     inputIndex = encoder.dequeueInputBuffer(TIMEOUT_USEC);
1903                 }
1904                 ByteBuffer inputBuffer = inputBuffers[inputIndex];
1905                 inputBuffer.rewind();
1906                 inputBuffer.put(source);
1907                 long start = System.currentTimeMillis();
1908                 totalEncoded += inputBuffers[inputIndex].limit();
1909                 encoder.queueInputBuffer(inputIndex, 0, inputBuffer.limit(), 0, 0);
1910                 source.rewind();
1911                 int index = encoder.dequeueOutputBuffer(info, TIMEOUT_USEC);
1912                 long end = System.currentTimeMillis();
1913                 encodingLatencySum += (end - start);
1914                 while (index >= 0) {
1915                     encoder.releaseOutputBuffer(index, false);
1916                     // just throw away output
1917                     // allow shorter wait for 2nd round to move on quickly.
1918                     index = encoder.dequeueOutputBuffer(info, TIMEOUT_USEC_SHORT);
1919                 }
1920             }
1921             Log.w(TAG, "Audio encoding average latency " + encodingLatencySum / numRepeat +
1922                     " ms for average write size " + totalEncoded / numRepeat +
1923                     " total latency " + encodingLatencySum + " ms for total bytes " + totalEncoded);
1924         } catch (Throwable e) {
1925             Log.w(TAG, "runAudioEncoding got error: " + e);
1926             mAudioEncoderHadError = true;
1927         } finally {
1928             if (encoder != null) {
1929                 encoder.stop();
1930                 encoder.release();
1931             }
1932         }
1933     }
1934 
1935     /**
1936      * Creates a MediaFormat with the basic set of values.
1937      */
createMediaFormat()1938     private static MediaFormat createMediaFormat() {
1939         MediaFormat format = MediaFormat.createVideoFormat(MIME_TYPE, WIDTH, HEIGHT);
1940         format.setInteger(MediaFormat.KEY_COLOR_FORMAT,
1941                 MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface);
1942         format.setInteger(MediaFormat.KEY_BIT_RATE, BIT_RATE);
1943         format.setInteger(MediaFormat.KEY_FRAME_RATE, FRAME_RATE);
1944         format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, IFRAME_INTERVAL);
1945         return format;
1946     }
1947 
1948     /**
1949      * Returns the first codec capable of encoding the specified MIME type, or null if no
1950      * match was found.
1951      */
selectCodec(String mimeType)1952     private static MediaCodecInfo selectCodec(String mimeType) {
1953         // FIXME: select codecs based on the complete use-case, not just the mime
1954         MediaCodecList mcl = new MediaCodecList(MediaCodecList.REGULAR_CODECS);
1955         for (MediaCodecInfo info : mcl.getCodecInfos()) {
1956             if (!info.isEncoder()) {
1957                 continue;
1958             }
1959 
1960             String[] types = info.getSupportedTypes();
1961             for (int j = 0; j < types.length; j++) {
1962                 if (types[j].equalsIgnoreCase(mimeType)) {
1963                     return info;
1964                 }
1965             }
1966         }
1967         return null;
1968     }
1969 
1970     /**
1971      * Returns a color format that is supported by the codec and isn't COLOR_FormatSurface.  Throws
1972      * an exception if none found.
1973      */
findNonSurfaceColorFormat(MediaCodecInfo codecInfo, String mimeType)1974     private static int findNonSurfaceColorFormat(MediaCodecInfo codecInfo, String mimeType) {
1975         MediaCodecInfo.CodecCapabilities capabilities = codecInfo.getCapabilitiesForType(mimeType);
1976         for (int i = 0; i < capabilities.colorFormats.length; i++) {
1977             int colorFormat = capabilities.colorFormats[i];
1978             if (colorFormat != MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface) {
1979                 return colorFormat;
1980             }
1981         }
1982         fail("couldn't find a good color format for " + codecInfo.getName() + " / " + MIME_TYPE);
1983         return 0;   // not reached
1984     }
1985 
getMediaExtractorForMimeType(final String resource, String mimeTypePrefix)1986     private MediaExtractor getMediaExtractorForMimeType(final String resource,
1987             String mimeTypePrefix) throws IOException {
1988         Preconditions.assertTestFileExists(mInpPrefix + resource);
1989         MediaExtractor mediaExtractor = new MediaExtractor();
1990         File inpFile = new File(mInpPrefix + resource);
1991         ParcelFileDescriptor parcelFD =
1992                 ParcelFileDescriptor.open(inpFile, ParcelFileDescriptor.MODE_READ_ONLY);
1993         AssetFileDescriptor afd = new AssetFileDescriptor(parcelFD, 0, parcelFD.getStatSize());
1994         try {
1995             mediaExtractor.setDataSource(
1996                     afd.getFileDescriptor(), afd.getStartOffset(), afd.getLength());
1997         } finally {
1998             afd.close();
1999         }
2000         int trackIndex;
2001         for (trackIndex = 0; trackIndex < mediaExtractor.getTrackCount(); trackIndex++) {
2002             MediaFormat trackMediaFormat = mediaExtractor.getTrackFormat(trackIndex);
2003             if (trackMediaFormat.getString(MediaFormat.KEY_MIME).startsWith(mimeTypePrefix)) {
2004                 mediaExtractor.selectTrack(trackIndex);
2005                 break;
2006             }
2007         }
2008         if (trackIndex == mediaExtractor.getTrackCount()) {
2009             throw new IllegalStateException("couldn't get a video track");
2010         }
2011 
2012         return mediaExtractor;
2013     }
2014 
supportsCodec(String mimeType, boolean encoder)2015     private static boolean supportsCodec(String mimeType, boolean encoder) {
2016         MediaCodecList list = new MediaCodecList(MediaCodecList.ALL_CODECS);
2017         for (MediaCodecInfo info : list.getCodecInfos()) {
2018             if (encoder != info.isEncoder()) {
2019                 continue;
2020             }
2021 
2022             for (String type : info.getSupportedTypes()) {
2023                 if (type.equalsIgnoreCase(mimeType)) {
2024                     return true;
2025                 }
2026             }
2027         }
2028         return false;
2029     }
2030 
2031     private static final UUID CLEARKEY_SCHEME_UUID =
2032             new UUID(0x1077efecc0b24d02L, 0xace33c1e52e2fb4bL);
2033 
2034     /**
2035      * Tests MediaCodec.CryptoException
2036      */
2037     @ApiTest(apis = "MediaCodec#CryptoException")
2038     @Test
2039     @FrameworkSpecificTest // media mainline doesn't update crypto
testCryptoException()2040     public void testCryptoException() {
2041         int errorCode = CryptoException.ERROR_KEY_EXPIRED;
2042         String errorMessage = "key_expired";
2043         CryptoException exception = new CryptoException(errorCode, errorMessage);
2044 
2045         assertEquals(errorCode, exception.getErrorCode());
2046         assertEquals(errorMessage, exception.getMessage());
2047     }
2048 
2049     /**
2050      * PCM encoding configuration test.
2051      *
2052      * If not specified in configure(), PCM encoding if it exists must be 16 bit.
2053      * If specified float in configure(), PCM encoding if it exists must be 16 bit, or float.
2054      *
2055      * As of Q, any codec of type "audio/raw" must support PCM encoding float.
2056      */
2057     @ApiTest(apis = {"android.media.AudioFormat#ENCODING_PCM_16BIT",
2058             "android.media.AudioFormat#ENCODING_PCM_FLOAT"})
2059     @MediumTest
2060     @Test
testPCMEncoding()2061     public void testPCMEncoding() throws Exception {
2062         final MediaCodecList mcl = new MediaCodecList(MediaCodecList.ALL_CODECS);
2063         for (MediaCodecInfo codecInfo : mcl.getCodecInfos()) {
2064             final boolean isEncoder = codecInfo.isEncoder();
2065             final String name = codecInfo.getName();
2066 
2067             for (String type : codecInfo.getSupportedTypes()) {
2068                 final MediaCodecInfo.CodecCapabilities ccaps =
2069                         codecInfo.getCapabilitiesForType(type);
2070                 final MediaCodecInfo.AudioCapabilities acaps =
2071                         ccaps.getAudioCapabilities();
2072                 if (acaps == null) {
2073                     break; // not an audio codec
2074                 }
2075 
2076                 // Deduce the minimum channel count (though prefer stereo over mono).
2077                 final int channelCount = Math.min(acaps.getMaxInputChannelCount(), 2);
2078 
2079                 // Deduce the minimum sample rate.
2080                 final int[] sampleRates = acaps.getSupportedSampleRates();
2081                 final Range<Integer>[] sampleRateRanges = acaps.getSupportedSampleRateRanges();
2082                 assertNotNull("supported sample rate ranges must be non-null", sampleRateRanges);
2083                 final int sampleRate = sampleRateRanges[0].getLower();
2084 
2085                 // If sample rate array exists (it may not),
2086                 // the minimum value must be equal with the minimum from the sample rate range.
2087                 if (sampleRates != null) {
2088                     assertEquals("sample rate range and array should have equal minimum",
2089                             sampleRate, sampleRates[0]);
2090                     Log.d(TAG, "codec: " + name + " type: " + type
2091                             + " has both sampleRate array and ranges");
2092                 } else {
2093                     Log.d(TAG, "codec: " + name + " type: " + type
2094                             + " returns null getSupportedSampleRates()");
2095                 }
2096 
2097                 // We create one format here for both tests below.
2098                 final MediaFormat format = MediaFormat.createAudioFormat(
2099                     type, sampleRate, channelCount);
2100 
2101                 // Bitrate field is mandatory for encoders (except FLAC).
2102                 if (isEncoder) {
2103                     final int bitRate = acaps.getBitrateRange().getLower();
2104                     format.setInteger(MediaFormat.KEY_BIT_RATE, bitRate);
2105                 }
2106 
2107                 // First test: An audio codec must be createable from a format
2108                 // with the minimum sample rate and channel count.
2109                 // The PCM encoding must be null (doesn't exist) or 16 bit.
2110                 {
2111                     // Check encoding of codec.
2112                     final Integer actualEncoding = encodingOfAudioCodec(name, format, isEncoder);
2113                     if (actualEncoding != null) {
2114                         assertEquals("returned audio encoding must be 16 bit for codec: "
2115                                 + name + " type: " + type + " encoding: " + actualEncoding,
2116                                 AudioFormat.ENCODING_PCM_16BIT, actualEncoding.intValue());
2117                     }
2118                 }
2119 
2120                 // Second test: An audio codec configured with PCM encoding float must return
2121                 // either an encoding of null (doesn't exist), 16 bit, or float.
2122                 {
2123                     // Reuse the original format, and add float specifier.
2124                     format.setInteger(
2125                             MediaFormat.KEY_PCM_ENCODING, AudioFormat.ENCODING_PCM_FLOAT);
2126 
2127                     // Check encoding of codec.
2128                     // The KEY_PCM_ENCODING key is advisory, so should not cause configuration
2129                     // failure.  The actual PCM encoding is returned from
2130                     // the input format (encoder) or output format (decoder).
2131                     final Integer actualEncoding = encodingOfAudioCodec(name, format, isEncoder);
2132                     if (actualEncoding != null) {
2133                         assertTrue(
2134                                 "returned audio encoding must be 16 bit or float for codec: "
2135                                 + name + " type: " + type + " encoding: " + actualEncoding,
2136                                 actualEncoding == AudioFormat.ENCODING_PCM_16BIT
2137                                 || actualEncoding == AudioFormat.ENCODING_PCM_FLOAT);
2138                         if (actualEncoding == AudioFormat.ENCODING_PCM_FLOAT) {
2139                             Log.d(TAG, "codec: " + name + " type: " + type + " supports float");
2140                         }
2141                     }
2142 
2143                     // As of Q, all codecs of type "audio/raw" must support float.
2144                     if (type.equals("audio/raw")) {
2145                         assertTrue(type + " must support float",
2146                                 actualEncoding != null &&
2147                                 actualEncoding.intValue() == AudioFormat.ENCODING_PCM_FLOAT);
2148                     }
2149                 }
2150             }
2151         }
2152     }
2153 
2154     /**
2155      * Returns the PCM encoding of an audio codec, or null if codec doesn't exist,
2156      * or not an audio codec, or PCM encoding key doesn't exist.
2157      */
encodingOfAudioCodec(String name, MediaFormat format, boolean encode)2158     private Integer encodingOfAudioCodec(String name, MediaFormat format, boolean encode)
2159             throws IOException {
2160         final int flagEncoder = encode ? MediaCodec.CONFIGURE_FLAG_ENCODE : 0;
2161         final MediaCodec codec = MediaCodec.createByCodecName(name);
2162         Integer actualEncoding = null;
2163 
2164         try {
2165             codec.configure(format, null /* surface */, null /* crypto */, flagEncoder);
2166 
2167             // Check input/output format - this must exist.
2168             final MediaFormat actualFormat =
2169                     encode ? codec.getInputFormat() : codec.getOutputFormat();
2170             assertNotNull("cannot get format for " + name, actualFormat);
2171 
2172             // Check actual encoding - this may or may not exist
2173             try {
2174                 actualEncoding = actualFormat.getInteger(MediaFormat.KEY_PCM_ENCODING);
2175             } catch (Exception e) {
2176                 ; // trying to get a non-existent key throws exception
2177             }
2178         } finally {
2179             codec.release();
2180         }
2181         return actualEncoding;
2182     }
2183 
2184     @ApiTest(apis = "android.media.AudioFormat#KEY_FLAC_COMPRESSION_LEVEL")
2185     @SmallTest
2186     @Test
testFlacIdentity()2187     public void testFlacIdentity() throws Exception {
2188         final int PCM_FRAMES = 1152 * 4; // FIXME: requires 4 flac frames to work with OMX codecs.
2189         final int SAMPLES = PCM_FRAMES * AUDIO_CHANNEL_COUNT;
2190         final int[] SAMPLE_RATES = {AUDIO_SAMPLE_RATE, 192000}; // ensure 192kHz supported.
2191 
2192         for (int sampleRate : SAMPLE_RATES) {
2193             final MediaFormat format = new MediaFormat();
2194             format.setString(MediaFormat.KEY_MIME, MediaFormat.MIMETYPE_AUDIO_FLAC);
2195             format.setInteger(MediaFormat.KEY_SAMPLE_RATE, sampleRate);
2196             format.setInteger(MediaFormat.KEY_CHANNEL_COUNT, AUDIO_CHANNEL_COUNT);
2197 
2198             Log.d(TAG, "Trying sample rate: " + sampleRate
2199                     + " channel count: " + AUDIO_CHANNEL_COUNT);
2200             // this key is only needed for encode, ignored for decode
2201             format.setInteger(MediaFormat.KEY_FLAC_COMPRESSION_LEVEL, 5);
2202 
2203             for (int i = 0; i < 2; ++i) {
2204                 final boolean useFloat = (i == 1);
2205                 final StreamUtils.PcmAudioBufferStream audioStream =
2206                     new StreamUtils.PcmAudioBufferStream(SAMPLES, sampleRate, 1000 /* frequency */,
2207                     100 /* sweep */, useFloat);
2208 
2209                 if (useFloat) {
2210                     format.setInteger(
2211                         MediaFormat.KEY_PCM_ENCODING, AudioFormat.ENCODING_PCM_FLOAT);
2212                 }
2213 
2214                 final StreamUtils.MediaCodecStream rawToFlac = new StreamUtils.MediaCodecStream(
2215                     new StreamUtils.ByteBufferInputStream(audioStream), format, true /* encode */);
2216                 final StreamUtils.MediaCodecStream flacToRaw = new StreamUtils.MediaCodecStream(
2217                     rawToFlac, format, false /* encode */);
2218 
2219                 if (useFloat) { // ensure float precision supported at the sample rate.
2220                     assertTrue("No float FLAC encoder at " + sampleRate,
2221                             rawToFlac.mIsFloat);
2222                     assertTrue("No float FLAC decoder at " + sampleRate,
2223                             flacToRaw.mIsFloat);
2224                 }
2225 
2226                 // Note: the existence of signed zero (as well as NAN) may make byte
2227                 // comparisons invalid for floating point output. In our case, since the
2228                 // floats come through integer to float conversion, it does not matter.
2229                 assertEquals("Audio data not identical after compression",
2230                     audioStream.sizeInBytes(),
2231                     StreamUtils.compareStreams(new StreamUtils.ByteBufferInputStream(flacToRaw),
2232                         new StreamUtils.ByteBufferInputStream(
2233                         new StreamUtils.PcmAudioBufferStream(audioStream))));
2234             }
2235         }
2236     }
2237 
2238     @ApiTest(apis = "MediaCodec#release")
2239     @Test
testAsyncRelease()2240     public void testAsyncRelease() throws Exception {
2241         OutputSurface outputSurface = new OutputSurface(1, 1);
2242         MediaExtractor mediaExtractor = getMediaExtractorForMimeType(INPUT_RESOURCE, "video/");
2243         MediaFormat mediaFormat =
2244                 mediaExtractor.getTrackFormat(mediaExtractor.getSampleTrackIndex());
2245         String mimeType = mediaFormat.getString(MediaFormat.KEY_MIME);
2246         for (int i = 0; i < 100; ++i) {
2247             final MediaCodec codec = MediaCodec.createDecoderByType(mimeType);
2248 
2249             try {
2250                 final ConditionVariable cv = new ConditionVariable();
2251                 Runnable first = null;
2252                 switch (i % 5) {
2253                     case 0:  // release
2254                         codec.configure(mediaFormat, outputSurface.getSurface(), null, 0);
2255                         codec.start();
2256                         first = () -> { cv.block(); codec.release(); };
2257                         break;
2258                     case 1:  // start
2259                         codec.configure(mediaFormat, outputSurface.getSurface(), null, 0);
2260                         first = () -> {
2261                             cv.block();
2262                             try {
2263                                 codec.start();
2264                             } catch (Exception e) {
2265                                 Log.i(TAG, "start failed", e);
2266                             }
2267                         };
2268                         break;
2269                     case 2:  // configure
2270                         first = () -> {
2271                             cv.block();
2272                             try {
2273                                 codec.configure(mediaFormat, outputSurface.getSurface(), null, 0);
2274                             } catch (Exception e) {
2275                                 Log.i(TAG, "configure failed", e);
2276                             }
2277                         };
2278                         break;
2279                     case 3:  // stop
2280                         codec.configure(mediaFormat, outputSurface.getSurface(), null, 0);
2281                         codec.start();
2282                         first = () -> {
2283                             cv.block();
2284                             try {
2285                                 codec.stop();
2286                             } catch (Exception e) {
2287                                 Log.i(TAG, "stop failed", e);
2288                             }
2289                         };
2290                         break;
2291                     case 4:  // flush
2292                         codec.configure(mediaFormat, outputSurface.getSurface(), null, 0);
2293                         codec.start();
2294                         codec.dequeueInputBuffer(0);
2295                         first = () -> {
2296                             cv.block();
2297                             try {
2298                                 codec.flush();
2299                             } catch (Exception e) {
2300                                 Log.i(TAG, "flush failed", e);
2301                             }
2302                         };
2303                         break;
2304                 }
2305 
2306                 Thread[] threads = new Thread[10];
2307                 threads[0] = new Thread(first);
2308                 for (int j = 1; j < threads.length; ++j) {
2309                     threads[j] = new Thread(() -> { cv.block(); codec.release(); });
2310                 }
2311                 for (Thread thread : threads) {
2312                     thread.start();
2313                 }
2314                 // Wait a little bit so that threads may reach block() call.
2315                 Thread.sleep(50);
2316                 cv.open();
2317                 for (Thread thread : threads) {
2318                     thread.join();
2319                 }
2320             } finally {
2321                 codec.release();
2322             }
2323         }
2324     }
2325 
2326     @ApiTest(apis = "MediaCodec#setAudioPresentation")
2327     @Test
testSetAudioPresentation()2328     public void testSetAudioPresentation() throws Exception {
2329         MediaFormat format = MediaFormat.createAudioFormat(
2330                 MediaFormat.MIMETYPE_AUDIO_MPEG, 44100 /* sampleRate */, 2 /* channelCount */);
2331         String mimeType = format.getString(MediaFormat.KEY_MIME);
2332         MediaCodec codec = createCodecByType(
2333                 format.getString(MediaFormat.KEY_MIME), false /* isEncoder */);
2334         assertNotNull(codec);
2335         assertThrows(NullPointerException.class, () -> {
2336             codec.setAudioPresentation(null);
2337         });
2338         codec.setAudioPresentation(
2339                 (new AudioPresentation.Builder(42 /* presentationId */)).build());
2340     }
2341 
2342     @ApiTest(apis = "android.media.MediaFormat#KEY_PREPEND_HEADER_TO_SYNC_FRAMES")
2343     @Test
testPrependHeadersToSyncFrames()2344     public void testPrependHeadersToSyncFrames() throws IOException {
2345         MediaCodecList mcl = new MediaCodecList(MediaCodecList.REGULAR_CODECS);
2346         for (MediaCodecInfo info : mcl.getCodecInfos()) {
2347             boolean isEncoder = info.isEncoder();
2348             for (String mime: info.getSupportedTypes()) {
2349                 CodecCapabilities caps = info.getCapabilitiesForType(mime);
2350                 boolean isVideo = (caps.getVideoCapabilities() != null);
2351                 boolean isAudio = (caps.getAudioCapabilities() != null);
2352 
2353                 MediaCodec codec = null;
2354                 MediaFormat format = null;
2355                 try {
2356                     codec = MediaCodec.createByCodecName(info.getName());
2357                     if (isVideo) {
2358                         VideoCapabilities vcaps = caps.getVideoCapabilities();
2359                         int minWidth = vcaps.getSupportedWidths().getLower();
2360                         int minHeight = vcaps.getSupportedHeightsFor(minWidth).getLower();
2361                         int minBitrate = vcaps.getBitrateRange().getLower();
2362                         int minFrameRate = Math.max(vcaps.getSupportedFrameRatesFor(
2363                                 minWidth, minHeight) .getLower().intValue(), 1);
2364                         format = MediaFormat.createVideoFormat(mime, minWidth, minHeight);
2365                         format.setInteger(MediaFormat.KEY_COLOR_FORMAT, caps.colorFormats[0]);
2366                         format.setInteger(MediaFormat.KEY_BIT_RATE, minBitrate);
2367                         format.setInteger(MediaFormat.KEY_FRAME_RATE, minFrameRate);
2368                         format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, IFRAME_INTERVAL);
2369                     } else if(isAudio){
2370                         AudioCapabilities acaps = caps.getAudioCapabilities();
2371                         int minSampleRate = acaps.getSupportedSampleRateRanges()[0].getLower();
2372                         int minChannelCount = 1;
2373                         if (mIsAtLeastS) {
2374                             minChannelCount = acaps.getMinInputChannelCount();
2375                         }
2376                         int minBitrate = acaps.getBitrateRange().getLower();
2377                         format = MediaFormat.createAudioFormat(mime, minSampleRate, minChannelCount);
2378                         format.setInteger(MediaFormat.KEY_BIT_RATE, minBitrate);
2379                     }
2380 
2381                     if (isVideo || isAudio) {
2382                         format.setInteger(MediaFormat.KEY_PREPEND_HEADER_TO_SYNC_FRAMES, 1);
2383 
2384                         codec.configure(format, null /* surface */, null /* crypto */,
2385                             isEncoder ? codec.CONFIGURE_FLAG_ENCODE : 0);
2386                     }
2387                     if (isVideo && isEncoder) {
2388                         Log.i(TAG, info.getName() + " supports KEY_PREPEND_HEADER_TO_SYNC_FRAMES");
2389                     } else {
2390                         Log.i(TAG, info.getName() + " is not a video encoder, so" +
2391                                 " KEY_PREPEND_HEADER_TO_SYNC_FRAMES is no-op, as expected");
2392                     }
2393                     // TODO: actually test encoders prepend the headers to sync frames.
2394                 } catch (IllegalArgumentException | CodecException e) {
2395                     if (isVideo && isEncoder) {
2396                         Log.i(TAG, info.getName() + " does not support" +
2397                                 " KEY_PREPEND_HEADER_TO_SYNC_FRAMES");
2398                     } else {
2399                         fail(info.getName() + " is not a video encoder," +
2400                                 " so it should not fail to configure.\n" + e.toString());
2401                     }
2402                 } finally {
2403                     if (codec != null) {
2404                         codec.release();
2405                     }
2406                 }
2407             }
2408         }
2409     }
2410 
2411     /**
2412      * Test if flushing early in the playback does not prevent client from getting the
2413      * latest configuration. Empirically, this happens most often when the
2414      * codec is flushed after the first buffer is queued, so this test walks
2415      * through the scenario.
2416      */
2417     @ApiTest(apis = "MediaCodec#flush")
2418     @Test
testFlushAfterFirstBuffer()2419     public void testFlushAfterFirstBuffer() throws Exception {
2420         if (MediaUtils.check(mIsAtLeastR, "test needs Android 11")) {
2421             for (int i = 0; i < 100; ++i) {
2422                 doFlushAfterFirstBuffer();
2423             }
2424         }
2425     }
2426 
doFlushAfterFirstBuffer()2427     private void doFlushAfterFirstBuffer() throws Exception {
2428         MediaExtractor extractor = null;
2429         MediaCodec codec = null;
2430 
2431         try {
2432             MediaFormat newFormat = null;
2433             extractor = getMediaExtractorForMimeType(
2434                     "noise_2ch_48khz_aot29_dr_sbr_sig2_mp4.m4a", "audio/");
2435             int trackIndex = extractor.getSampleTrackIndex();
2436             MediaFormat format = extractor.getTrackFormat(trackIndex);
2437             codec = createCodecByType(
2438                     format.getString(MediaFormat.KEY_MIME), false /* isEncoder */);
2439             codec.configure(format, null, null, 0);
2440             codec.start();
2441             int firstInputIndex = codec.dequeueInputBuffer(0);
2442             while (firstInputIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
2443                 firstInputIndex = codec.dequeueInputBuffer(5000);
2444             }
2445             assertTrue(firstInputIndex >= 0);
2446             extractor.readSampleData(codec.getInputBuffer(firstInputIndex), 0);
2447             codec.queueInputBuffer(
2448                     firstInputIndex, 0, Math.toIntExact(extractor.getSampleSize()),
2449                     extractor.getSampleTime(), extractor.getSampleFlags());
2450             // Don't advance, so the first buffer will be read again after flush
2451             codec.flush();
2452             ByteBuffer csd = format.getByteBuffer("csd-0");
2453             MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();
2454             // We don't need to decode many frames
2455             int numFrames = 10;
2456             boolean eos = false;
2457             while (!eos) {
2458                 if (numFrames > 0) {
2459                     int inputIndex = codec.dequeueInputBuffer(0);
2460                     if (inputIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
2461                         inputIndex = codec.dequeueInputBuffer(5000);
2462                     }
2463                     if (inputIndex >= 0) {
2464                         ByteBuffer inputBuffer = codec.getInputBuffer(inputIndex);
2465                         if (csd != null) {
2466                             inputBuffer.clear();
2467                             inputBuffer.put(csd);
2468                             codec.queueInputBuffer(
2469                                     inputIndex, 0, inputBuffer.position(), 0,
2470                                     MediaCodec.BUFFER_FLAG_CODEC_CONFIG);
2471                             csd = null;
2472                         } else {
2473                             int size = extractor.readSampleData(inputBuffer, 0);
2474                             if (size <= 0) {
2475                                 break;
2476                             }
2477                             int flags = extractor.getSampleFlags();
2478                             --numFrames;
2479                             if (numFrames <= 0) {
2480                                 flags |= MediaCodec.BUFFER_FLAG_END_OF_STREAM;
2481                             }
2482                             codec.queueInputBuffer(
2483                                     inputIndex, 0, size, extractor.getSampleTime(), flags);
2484                             extractor.advance();
2485                         }
2486                     }
2487                 }
2488 
2489                 int outputIndex = codec.dequeueOutputBuffer(bufferInfo, 0);
2490                 if (outputIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
2491                     outputIndex = codec.dequeueOutputBuffer(bufferInfo, 5000);
2492                 }
2493                 if (outputIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
2494                     newFormat = codec.getOutputFormat();
2495                 } else if (outputIndex >= 0) {
2496                     if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
2497                         eos = true;
2498                     }
2499                     codec.releaseOutputBuffer(outputIndex, false);
2500                 }
2501             }
2502             assertNotNull(newFormat);
2503             assertEquals(48000, newFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE));
2504         } finally {
2505             if (extractor != null) {
2506                 extractor.release();
2507             }
2508             if (codec != null) {
2509                 codec.stop();
2510                 codec.release();
2511             }
2512         }
2513     }
2514 
2515     @ApiTest(apis = {"MediaCodec#getSupportedVendorParameters",
2516             "MediaCodec#getParameterDescriptor",
2517             "MediaCodec#subscribeToVendorParameters",
2518             "MediaCodec#unsubscribeFromVendorParameters"})
2519     @Test
testVendorParameters()2520     public void testVendorParameters() {
2521         if (!MediaUtils.check(mIsAtLeastS, "test needs Android 12")) {
2522             return;
2523         }
2524         MediaCodecList mcl = new MediaCodecList(MediaCodecList.REGULAR_CODECS);
2525         for (MediaCodecInfo info : mcl.getCodecInfos()) {
2526             if (info.isAlias()) {
2527                 continue;
2528             }
2529             MediaCodec codec = null;
2530             if (!TestUtils.isTestableCodecInCurrentMode(info.getName())) {
2531                 Log.d(TAG, "skip testing codec " + info.getName() + " in current mode:"
2532                                 + TestUtils.currentTestModeName());
2533                 continue;
2534             }
2535             try {
2536                 codec = MediaCodec.createByCodecName(info.getName());
2537                 List<String> vendorParams = codec.getSupportedVendorParameters();
2538                 if (VERBOSE) {
2539                     Log.d(TAG, "vendor params supported by " + info.getName() + ": " +
2540                             vendorParams.toString());
2541                 }
2542                 for (String name : vendorParams) {
2543                     MediaCodec.ParameterDescriptor desc = codec.getParameterDescriptor(name);
2544                     assertNotNull(name + " is in the list of supported parameters, so the codec" +
2545                             " should be able to describe it.", desc);
2546                     assertEquals("name differs from the name in the descriptor",
2547                             name, desc.getName());
2548                     assertTrue("type in the descriptor cannot be TYPE_NULL",
2549                             MediaFormat.TYPE_NULL != desc.getType());
2550                     if (VERBOSE) {
2551                         Log.d(TAG, name + " is of type " + desc.getType());
2552                     }
2553                 }
2554                 codec.subscribeToVendorParameters(vendorParams);
2555 
2556                 // Build a MediaFormat that makes sense to the codec.
2557                 String type = info.getSupportedTypes()[0];
2558                 MediaFormat format = null;
2559                 CodecCapabilities caps = info.getCapabilitiesForType(type);
2560                 AudioCapabilities audioCaps = caps.getAudioCapabilities();
2561                 VideoCapabilities videoCaps = caps.getVideoCapabilities();
2562                 if (audioCaps != null) {
2563                     format = MediaFormat.createAudioFormat(
2564                             type,
2565                             audioCaps.getSupportedSampleRateRanges()[0].getLower(),
2566                             audioCaps.getMaxInputChannelCount());
2567                     if (info.isEncoder()) {
2568                         format.setInteger(MediaFormat.KEY_BIT_RATE, AUDIO_BIT_RATE);
2569                     }
2570                 } else if (videoCaps != null) {
2571                     int width = videoCaps.getSupportedWidths().getLower();
2572                     int height = videoCaps.getSupportedHeightsFor(width).getLower();
2573                     format = MediaFormat.createVideoFormat(type, width, height);
2574                     if (info.isEncoder()) {
2575                         EncoderCapabilities encCaps = caps.getEncoderCapabilities();
2576                         if (encCaps != null) {
2577                             int bitrateMode = -1;
2578                             List<Integer> candidates = Arrays.asList(
2579                                     EncoderCapabilities.BITRATE_MODE_VBR,
2580                                     EncoderCapabilities.BITRATE_MODE_CBR,
2581                                     EncoderCapabilities.BITRATE_MODE_CQ,
2582                                     EncoderCapabilities.BITRATE_MODE_CBR_FD);
2583                             for (int candidate : candidates) {
2584                                 if (encCaps.isBitrateModeSupported(candidate)) {
2585                                     bitrateMode = candidate;
2586                                     break;
2587                                 }
2588                             }
2589                             if (VERBOSE) {
2590                                 Log.d(TAG, "video encoder: bitrate mode = " + bitrateMode);
2591                             }
2592                             format.setInteger(MediaFormat.KEY_BITRATE_MODE, bitrateMode);
2593                             switch (bitrateMode) {
2594                             case EncoderCapabilities.BITRATE_MODE_VBR:
2595                             case EncoderCapabilities.BITRATE_MODE_CBR:
2596                             case EncoderCapabilities.BITRATE_MODE_CBR_FD:
2597                                 format.setInteger(MediaFormat.KEY_BIT_RATE, BIT_RATE);
2598                                 break;
2599                             case EncoderCapabilities.BITRATE_MODE_CQ:
2600                                 format.setInteger(
2601                                         MediaFormat.KEY_QUALITY,
2602                                         encCaps.getQualityRange().getLower());
2603                                 if (VERBOSE) {
2604                                     Log.d(TAG, "video encoder: quality = " +
2605                                             encCaps.getQualityRange().getLower());
2606                                 }
2607                                 break;
2608                             default:
2609                                 format.removeKey(MediaFormat.KEY_BITRATE_MODE);
2610                             }
2611                         }
2612                         format.setInteger(MediaFormat.KEY_FRAME_RATE, 30);
2613                         format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, IFRAME_INTERVAL);
2614                         format.setInteger(
2615                                 MediaFormat.KEY_COLOR_FORMAT,
2616                                 CodecCapabilities.COLOR_FormatSurface);
2617                     }
2618                 } else {
2619                     Log.i(TAG, info.getName() + " is in neither audio nor video domain; skipped");
2620                     codec.release();
2621                     continue;
2622                 }
2623                 codec.configure(
2624                         format, null, null,
2625                         info.isEncoder() ? MediaCodec.CONFIGURE_FLAG_ENCODE : 0);
2626                 Surface inputSurface = null;
2627                 if (videoCaps != null && info.isEncoder()) {
2628                     inputSurface = codec.createInputSurface();
2629                 }
2630                 codec.start();
2631                 codec.unsubscribeFromVendorParameters(vendorParams);
2632                 codec.stop();
2633             } catch (Exception e) {
2634                 throw new RuntimeException("codec name: " + info.getName(), e);
2635             } finally {
2636                 if (codec != null) {
2637                     codec.release();
2638                 }
2639             }
2640         }
2641     }
2642 }
2643