1 /*
2 * Copyright (C) 2009 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 //#define LOG_NDEBUG 0
18 #define LOG_TAG "MPEG4Writer"
19
20 #include <algorithm>
21
22 #include <arpa/inet.h>
23 #include <fcntl.h>
24 #include <inttypes.h>
25 #include <pthread.h>
26 #include <sys/prctl.h>
27 #include <sys/stat.h>
28 #include <sys/types.h>
29 #include <unistd.h>
30
31 #include <utils/Log.h>
32
33 #include <functional>
34
35 #include <media/stagefright/MediaSource.h>
36 #include <media/stagefright/foundation/ADebug.h>
37 #include <media/stagefright/foundation/AMessage.h>
38 #include <media/stagefright/foundation/ALookup.h>
39 #include <media/stagefright/foundation/AUtils.h>
40 #include <media/stagefright/foundation/ByteUtils.h>
41 #include <media/stagefright/foundation/ColorUtils.h>
42 #include <media/stagefright/foundation/avc_utils.h>
43 #include <media/stagefright/MPEG4Writer.h>
44 #include <media/stagefright/MediaBuffer.h>
45 #include <media/stagefright/MetaData.h>
46 #include <media/stagefright/MediaDefs.h>
47 #include <media/stagefright/MediaCodecConstants.h>
48 #include <media/stagefright/MediaErrors.h>
49 #include <media/stagefright/Utils.h>
50 #include <media/mediarecorder.h>
51 #include <cutils/properties.h>
52
53 #include <media/esds/ESDS.h>
54 #include "include/HevcUtils.h"
55
56 #include <com_android_internal_camera_flags.h>
57 #include <com_android_media_editing_flags.h>
58 namespace editing_flags = com::android::media::editing::flags;
59
60 #ifndef __predict_false
61 #define __predict_false(exp) __builtin_expect((exp) != 0, 0)
62 #endif
63
64 #define WARN_UNLESS(condition, message, ...) \
65 ( (__predict_false(condition)) ? false : ({ \
66 ALOGW("Condition %s failed " message, #condition, ##__VA_ARGS__); \
67 true; \
68 }))
69
70 namespace flags_camera = com::android::internal::camera::flags;
71
72 namespace android {
73
74 static const int64_t kMinStreamableFileSizeInBytes = 5 * 1024 * 1024;
75 static const uint8_t kNalUnitTypeSeqParamSet = 0x07;
76 static const uint8_t kNalUnitTypePicParamSet = 0x08;
77 static const int64_t kInitialDelayTimeUs = 700000LL;
78 static const int64_t kMaxMetadataSize = 0x4000000LL; // 64MB max per-frame metadata size
79 static const int64_t kMaxCttsOffsetTimeUs = 30 * 60 * 1000000LL; // 30 minutes
80 static const size_t kESDSScratchBufferSize = 10; // kMaxAtomSize in Mpeg4Extractor 64MB
81 // Allow up to 100 milli second, which is safely above the maximum delay observed in manual testing
82 // between posting from setNextFd and handling it
83 static const int64_t kFdCondWaitTimeoutNs = 100000000;
84
85 static const char kMetaKey_Version[] = "com.android.version";
86 static const char kMetaKey_Manufacturer[] = "com.android.manufacturer";
87 static const char kMetaKey_Model[] = "com.android.model";
88
89 #ifdef SHOW_BUILD
90 static const char kMetaKey_Build[] = "com.android.build";
91 #endif
92 static const char kMetaKey_CaptureFps[] = "com.android.capture.fps";
93 static const char kMetaKey_TemporalLayerCount[] = "com.android.video.temporal_layers_count";
94
95 static const int kTimestampDebugCount = 10;
96 static const int kItemIdBase = 10000;
97 static const char kExifHeader[] = {'E', 'x', 'i', 'f', '\0', '\0'};
98 static const char kGainmapMetaHeader[] = {'t', 'm', 'a', 'p', '\0', '\0'};
99 static const char kGainmapHeader[] = {'g', 'm', 'a', 'p', '\0', '\0'};
100 static const uint8_t kExifApp1Marker[] = {'E', 'x', 'i', 'f', 0xff, 0xe1};
101
102 static const uint8_t kMandatoryHevcNalUnitTypes[3] = {
103 kHevcNalUnitTypeVps,
104 kHevcNalUnitTypeSps,
105 kHevcNalUnitTypePps,
106 };
107 static const uint8_t kHevcNalUnitTypes[5] = {
108 kHevcNalUnitTypeVps,
109 kHevcNalUnitTypeSps,
110 kHevcNalUnitTypePps,
111 kHevcNalUnitTypePrefixSei,
112 kHevcNalUnitTypeSuffixSei,
113 };
114 /* uncomment to include build in meta */
115 //#define SHOW_MODEL_BUILD 1
116
117 class MPEG4Writer::Track {
118 struct TrackId {
TrackIdandroid::MPEG4Writer::Track::TrackId119 TrackId(uint32_t aId)
120 :mId(aId),
121 mTrackIdValid(false) {
122 }
isValidandroid::MPEG4Writer::Track::TrackId123 bool isValid(bool akKey4BitTrackIds) {
124 // trackId cannot be zero, ISO/IEC 14496-12 8.3.2.3
125 if (mId == 0) {
126 return false;
127 }
128 /* MediaRecorder uses only 4 bit to represent track ids during notifying clients.
129 * MediaMuxer's track ids are restricted by container allowed size only.
130 * MPEG4 Container defines unsigned int (32), ISO/IEC 14496-12 8.3.2.2
131 */
132 if (akKey4BitTrackIds && mId > 15) {
133 return false;
134 }
135 mTrackIdValid = true;
136 return true;
137 }
getIdandroid::MPEG4Writer::Track::TrackId138 uint32_t getId() const {
139 CHECK(mTrackIdValid);
140 return mId;
141 }
142 TrackId() = delete;
143 DISALLOW_EVIL_CONSTRUCTORS(TrackId);
144 private:
145 // unsigned int (32), ISO/IEC 14496-12 8.3.2.2
146 uint32_t mId;
147 bool mTrackIdValid;
148 };
149
150 public:
151 Track(MPEG4Writer *owner, const sp<MediaSource> &source, uint32_t aTrackId);
152
153 ~Track();
154
155 status_t start(MetaData *params);
156 status_t stop(bool stopSource = true);
157 status_t pause();
158 bool reachedEOS();
159
160 int64_t getDurationUs() const;
161 int64_t getEstimatedTrackSizeBytes() const;
162 int32_t getMetaSizeIncrease(int32_t angle, int32_t trackCount) const;
163 void writeTrackHeader();
164 int64_t getMinCttsOffsetTimeUs();
165 void bufferChunk(int64_t timestampUs);
isAvc() const166 bool isAvc() const { return mIsAvc; }
isHevc() const167 bool isHevc() const { return mIsHevc; }
isAv1() const168 bool isAv1() const { return mIsAv1; }
isApv() const169 bool isApv() const { return mIsApv; }
isHeic() const170 bool isHeic() const { return mIsHeic; }
isAvif() const171 bool isAvif() const { return mIsAvif; }
isHeif() const172 bool isHeif() const { return mIsHeif; }
isAudio() const173 bool isAudio() const { return mIsAudio; }
isMPEG4() const174 bool isMPEG4() const { return mIsMPEG4; }
usePrefix() const175 bool usePrefix() const { return mIsAvc || mIsHevc || mIsHeic || mIsDovi; }
176 bool isExifData(MediaBufferBase *buffer, uint32_t *tiffHdrOffset) const;
177 bool isGainmapMetaData(MediaBufferBase* buffer, uint32_t* offset) const;
178 bool isGainmapData(MediaBufferBase* buffer, uint32_t* offset) const;
179 void addChunkOffset(off64_t offset);
180 void addItemOffsetAndSize(off64_t offset, size_t size, bool isExif,
181 bool isGainmapMeta = false, bool isGainmap = false);
182 void flushItemRefs();
getTrackId()183 TrackId& getTrackId() { return mTrackId; }
184 status_t dump(int fd, const Vector<String16>& args) const;
185 static const char *getFourCCForMime(const char *mime);
186 const char *getDoviFourCC() const;
187 const char *getTrackType() const;
188 void resetInternal();
189 int64_t trackMetaDataSize();
190 bool isTimestampValid(int64_t timeUs);
getImageItemId()191 uint16_t getImageItemId() { return mImageItemId; };
getGainmapItemId()192 uint16_t getGainmapItemId() { return mGainmapItemId; };
getGainmapMetaItemId()193 uint16_t getGainmapMetaItemId() { return mGainmapMetadataItemId; };
194
195 private:
196 // A helper class to handle faster write box with table entries
197 template<class TYPE, unsigned ENTRY_SIZE>
198 // ENTRY_SIZE: # of values in each entry
199 struct ListTableEntries {
200 static_assert(ENTRY_SIZE > 0, "ENTRY_SIZE must be positive");
ListTableEntriesandroid::MPEG4Writer::Track::ListTableEntries201 ListTableEntries(uint32_t elementCapacity)
202 : mElementCapacity(elementCapacity),
203 mTotalNumTableEntries(0),
204 mNumValuesInCurrEntry(0),
205 mCurrTableEntriesElement(NULL) {
206 CHECK_GT(mElementCapacity, 0u);
207 // Ensure no integer overflow on allocation in add().
208 CHECK_LT(ENTRY_SIZE, UINT32_MAX / mElementCapacity);
209 }
210
211 // Free the allocated memory.
~ListTableEntriesandroid::MPEG4Writer::Track::ListTableEntries212 ~ListTableEntries() {
213 while (!mTableEntryList.empty()) {
214 typename List<TYPE *>::iterator it = mTableEntryList.begin();
215 delete[] (*it);
216 mTableEntryList.erase(it);
217 }
218 }
219
220 // Replace the value at the given position by the given value.
221 // There must be an existing value at the given position.
222 // @arg value must be in network byte order
223 // @arg pos location the value must be in.
setandroid::MPEG4Writer::Track::ListTableEntries224 void set(const TYPE& value, uint32_t pos) {
225 CHECK_LT(pos, mTotalNumTableEntries * ENTRY_SIZE);
226
227 typename List<TYPE *>::iterator it = mTableEntryList.begin();
228 uint32_t iterations = (pos / (mElementCapacity * ENTRY_SIZE));
229 while (it != mTableEntryList.end() && iterations > 0) {
230 ++it;
231 --iterations;
232 }
233 CHECK(it != mTableEntryList.end());
234 CHECK_EQ(iterations, 0u);
235
236 (*it)[(pos % (mElementCapacity * ENTRY_SIZE))] = value;
237 }
238
239 // Get the value at the given position by the given value.
240 // @arg value the retrieved value at the position in network byte order.
241 // @arg pos location the value must be in.
242 // @return true if a value is found.
getandroid::MPEG4Writer::Track::ListTableEntries243 bool get(TYPE& value, uint32_t pos) const {
244 if (pos >= mTotalNumTableEntries * ENTRY_SIZE) {
245 return false;
246 }
247
248 typename List<TYPE *>::iterator it = mTableEntryList.begin();
249 uint32_t iterations = (pos / (mElementCapacity * ENTRY_SIZE));
250 while (it != mTableEntryList.end() && iterations > 0) {
251 ++it;
252 --iterations;
253 }
254 CHECK(it != mTableEntryList.end());
255 CHECK_EQ(iterations, 0u);
256
257 value = (*it)[(pos % (mElementCapacity * ENTRY_SIZE))];
258 return true;
259 }
260
261 // adjusts all values by |adjust(value)|
adjustEntriesandroid::MPEG4Writer::Track::ListTableEntries262 void adjustEntries(
263 std::function<void(size_t /* ix */, TYPE(& /* entry */)[ENTRY_SIZE])> update) {
264 size_t nEntries = mTotalNumTableEntries + mNumValuesInCurrEntry / ENTRY_SIZE;
265 size_t ix = 0;
266 for (TYPE *entryArray : mTableEntryList) {
267 size_t num = std::min(nEntries, (size_t)mElementCapacity);
268 for (size_t i = 0; i < num; ++i) {
269 update(ix++, (TYPE(&)[ENTRY_SIZE])(*entryArray));
270 entryArray += ENTRY_SIZE;
271 }
272 nEntries -= num;
273 }
274 }
275
276 // Store a single value.
277 // @arg value must be in network byte order.
addandroid::MPEG4Writer::Track::ListTableEntries278 void add(const TYPE& value) {
279 CHECK_LT(mNumValuesInCurrEntry, mElementCapacity);
280 uint32_t nEntries = mTotalNumTableEntries % mElementCapacity;
281 uint32_t nValues = mNumValuesInCurrEntry % ENTRY_SIZE;
282 if (nEntries == 0 && nValues == 0) {
283 mCurrTableEntriesElement = new TYPE[ENTRY_SIZE * mElementCapacity];
284 CHECK(mCurrTableEntriesElement != NULL);
285 mTableEntryList.push_back(mCurrTableEntriesElement);
286 }
287
288 uint32_t pos = nEntries * ENTRY_SIZE + nValues;
289 mCurrTableEntriesElement[pos] = value;
290
291 ++mNumValuesInCurrEntry;
292 if ((mNumValuesInCurrEntry % ENTRY_SIZE) == 0) {
293 ++mTotalNumTableEntries;
294 mNumValuesInCurrEntry = 0;
295 }
296 }
297
298 // Write out the table entries:
299 // 1. the number of entries goes first
300 // 2. followed by the values in the table enties in order
301 // @arg writer the writer to actual write to the storage
writeandroid::MPEG4Writer::Track::ListTableEntries302 void write(MPEG4Writer *writer) const {
303 CHECK_EQ(mNumValuesInCurrEntry % ENTRY_SIZE, 0u);
304 uint32_t nEntries = mTotalNumTableEntries;
305 writer->writeInt32(nEntries);
306 for (typename List<TYPE *>::iterator it = mTableEntryList.begin();
307 it != mTableEntryList.end(); ++it) {
308 CHECK_GT(nEntries, 0u);
309 if (nEntries >= mElementCapacity) {
310 writer->write(*it, sizeof(TYPE) * ENTRY_SIZE, mElementCapacity);
311 nEntries -= mElementCapacity;
312 } else {
313 writer->write(*it, sizeof(TYPE) * ENTRY_SIZE, nEntries);
314 break;
315 }
316 }
317 }
318
319 // Return the number of entries in the table.
countandroid::MPEG4Writer::Track::ListTableEntries320 uint32_t count() const { return mTotalNumTableEntries; }
321
322 private:
323 uint32_t mElementCapacity; // # entries in an element
324 uint32_t mTotalNumTableEntries;
325 uint32_t mNumValuesInCurrEntry; // up to ENTRY_SIZE
326 TYPE *mCurrTableEntriesElement;
327 mutable List<TYPE *> mTableEntryList;
328
329 DISALLOW_EVIL_CONSTRUCTORS(ListTableEntries);
330 };
331
332
333
334 MPEG4Writer *mOwner;
335 sp<MetaData> mMeta;
336 sp<MediaSource> mSource;
337 volatile bool mDone;
338 volatile bool mPaused;
339 volatile bool mResumed;
340 volatile bool mStarted;
341 bool mIsAvc;
342 bool mIsHevc;
343 bool mIsAv1;
344 bool mIsApv;
345 bool mIsDovi;
346 bool mIsAudio;
347 bool mIsVideo;
348 bool mIsHeic;
349 bool mIsAvif;
350 bool mIsHeif;
351 bool mIsMPEG4;
352 bool mGotStartKeyFrame;
353 bool mIsMalformed;
354 TrackId mTrackId;
355 int64_t mTrackDurationUs;
356 int64_t mMaxChunkDurationUs;
357 int64_t mLastDecodingTimeUs;
358 int64_t mEstimatedTrackSizeBytes;
359 int64_t mMdatSizeBytes;
360 int32_t mTimeScale;
361
362 pthread_t mThread;
363
364 List<MediaBuffer *> mChunkSamples;
365
366 bool mSamplesHaveSameSize;
367 ListTableEntries<uint32_t, 1> *mStszTableEntries;
368 ListTableEntries<off64_t, 1> *mCo64TableEntries;
369 ListTableEntries<uint32_t, 3> *mStscTableEntries;
370 ListTableEntries<uint32_t, 1> *mStssTableEntries;
371 ListTableEntries<uint32_t, 2> *mSttsTableEntries;
372 ListTableEntries<uint32_t, 2> *mCttsTableEntries;
373 ListTableEntries<uint32_t, 3> *mElstTableEntries; // 3columns: segDuration, mediaTime, mediaRate
374
375 int64_t mMinCttsOffsetTimeUs;
376 int64_t mMinCttsOffsetTicks;
377 int64_t mMaxCttsOffsetTicks;
378
379 // Save the last 10 frames' timestamp and frame type for debug.
380 struct TimestampDebugHelperEntry {
381 int64_t pts;
382 int64_t dts;
383 std::string frameType;
384 };
385
386 std::list<TimestampDebugHelperEntry> mTimestampDebugHelper;
387
388 // Sequence parameter set or picture parameter set
389 struct AVCParamSet {
AVCParamSetandroid::MPEG4Writer::Track::AVCParamSet390 AVCParamSet(uint16_t length, const uint8_t *data)
391 : mLength(length), mData(data) {}
392
393 uint16_t mLength;
394 const uint8_t *mData;
395 };
396 List<AVCParamSet> mSeqParamSets;
397 List<AVCParamSet> mPicParamSets;
398 uint8_t mProfileIdc;
399 uint8_t mProfileCompatible;
400 uint8_t mLevelIdc;
401
402 int32_t mDoviProfile;
403
404 void *mCodecSpecificData;
405 size_t mCodecSpecificDataSize;
406 bool mGotAllCodecSpecificData;
407 bool mTrackingProgressStatus;
408
409 bool mReachedEOS;
410 int64_t mStartTimestampUs;
411 int64_t mStartTimeRealUs;
412 int64_t mFirstSampleTimeRealUs;
413 // Captures negative start offset of a track(track starttime < 0).
414 int64_t mFirstSampleStartOffsetUs;
415 int64_t mPreviousTrackTimeUs;
416 int64_t mTrackEveryTimeDurationUs;
417
418 int32_t mRotation;
419
420 Vector<uint16_t> mProperties;
421 ItemRefs mDimgRefs;
422 ItemRefs mGainmapDimgRefs;
423 Vector<uint16_t> mExifList;
424 uint16_t mImageItemId;
425 uint16_t mItemIdBase;
426 int32_t mIsPrimary;
427 int32_t mWidth, mHeight;
428 int32_t mTileWidth, mTileHeight;
429 int32_t mGridRows, mGridCols;
430 size_t mNumTiles, mTileIndex;
431 uint16_t mGainmapItemId, mGainmapMetadataItemId;
432 ColorAspects mColorAspects;
433 bool mColorAspectsValid;
434 Vector<uint8_t> mBitsPerChannel;
435
436 // Update the audio track's drift information.
437 void updateDriftTime(const sp<MetaData>& meta);
438
439 void dumpTimeStamps();
440
441 int64_t getStartTimeOffsetTimeUs() const;
442 int32_t getStartTimeOffsetScaledTime() const;
443
444 static void *ThreadWrapper(void *me);
445 status_t threadEntry();
446
447 const uint8_t *parseParamSet(
448 const uint8_t *data, size_t length, int type, size_t *paramSetLen);
449
450 status_t copyCodecSpecificData(const uint8_t *data, size_t size, size_t minLength = 0);
451
452 status_t makeAVCCodecSpecificData(const uint8_t *data, size_t size);
453 status_t copyAVCCodecSpecificData(const uint8_t *data, size_t size);
454 status_t parseAVCCodecSpecificData(const uint8_t *data, size_t size);
455
456 status_t makeHEVCCodecSpecificData(const uint8_t *data, size_t size);
457 status_t copyHEVCCodecSpecificData(const uint8_t *data, size_t size);
458 status_t parseHEVCCodecSpecificData(
459 const uint8_t *data, size_t size, HevcParameterSets ¶mSets);
460
461 status_t getDolbyVisionProfile();
462
463 // Track authoring progress status
464 void trackProgressStatus(int64_t timeUs, status_t err = OK);
465 void initTrackingProgressStatus(MetaData *params);
466
467 void getCodecSpecificDataFromInputFormatIfPossible();
468
469 // Determine the track time scale
470 // If it is an audio track, try to use the sampling rate as
471 // the time scale; however, if user chooses the overwrite
472 // value, the user-supplied time scale will be used.
473 void setTimeScale();
474
475 // Simple validation on the codec specific data
476 status_t checkCodecSpecificData() const;
477
478 void updateTrackSizeEstimate();
479 void addOneStscTableEntry(size_t chunkId, size_t sampleId);
480 void addOneStssTableEntry(size_t sampleId);
481 void addOneSttsTableEntry(size_t sampleCount, int32_t delta /* media time scale based */);
482 void addOneCttsTableEntry(size_t sampleCount, int32_t sampleOffset);
483 void addOneElstTableEntry(uint32_t segmentDuration, int32_t mediaTime,
484 int16_t mediaRate, int16_t mediaRateFraction);
485
486 bool isTrackMalFormed();
487 void sendTrackSummary(bool hasMultipleTracks);
488
489 // Write the boxes
490 void writeCo64Box();
491 void writeStscBox();
492 void writeStszBox();
493 void writeStssBox();
494 void writeSttsBox();
495 void writeCttsBox();
496 void writeD263Box();
497 void writePaspBox();
498 void writeAvccBox();
499 void writeHvccBox();
500 void writeAv1cBox();
501 void writeApvcBox();
502 void writeDoviConfigBox();
503 void writeUrlBox();
504 void writeDrefBox();
505 void writeDinfBox();
506 void writeDamrBox();
507 void writeMdhdBox(uint32_t now);
508 void writeSmhdBox();
509 void writeVmhdBox();
510 void writeNmhdBox();
511 void writeHdlrBox();
512 void writeTkhdBox(uint32_t now);
513 void writeColrBox();
514 void writeMdcvAndClliBoxes();
515 void writeMp4aEsdsBox();
516 void writeMp4vEsdsBox();
517 void writeAudioFourCCBox();
518 void writeVideoFourCCBox();
519 void writeMetadataFourCCBox();
520 void writeStblBox();
521 void writeEdtsBox();
522
523 Track(const Track &);
524 Track &operator=(const Track &);
525 };
526
MPEG4Writer(int fd)527 MPEG4Writer::MPEG4Writer(int fd) {
528 initInternal(dup(fd), true /*isFirstSession*/);
529 }
530
~MPEG4Writer()531 MPEG4Writer::~MPEG4Writer() {
532 reset();
533
534 while (!mTracks.empty()) {
535 List<Track *>::iterator it = mTracks.begin();
536 delete *it;
537 (*it) = NULL;
538 mTracks.erase(it);
539 }
540 mTracks.clear();
541
542 if (mNextFd != -1) {
543 close(mNextFd);
544 }
545 }
546
initInternal(int fd,bool isFirstSession)547 void MPEG4Writer::initInternal(int fd, bool isFirstSession) {
548 ALOGV("initInternal");
549 mFd = fd;
550 mNextFd = -1;
551 mInitCheck = mFd < 0? NO_INIT: OK;
552
553 mInterleaveDurationUs = 1000000;
554
555 mStartTimestampUs = -1LL;
556 mStartTimeOffsetMs = -1;
557 mStartTimeOffsetBFramesUs = 0;
558 mPaused = false;
559 mStarted = false;
560 mWriterThreadStarted = false;
561 mSendNotify = false;
562 mWriteSeekErr = false;
563 mFallocateErr = false;
564 // Reset following variables for all the sessions and they will be
565 // initialized in start(MetaData *param).
566 mIsRealTimeRecording = true;
567 mIsBackgroundMode = false;
568 mUse4ByteNalLength = true;
569 mOffset = 0;
570 mMaxOffsetAppend = 0;
571 mPreAllocateFileEndOffset = 0;
572 mMdatOffset = 0;
573 mMdatEndOffset = 0;
574 mInMemoryCache = NULL;
575 mInMemoryCacheOffset = 0;
576 mInMemoryCacheSize = 0;
577 mWriteBoxToMemory = false;
578 mFreeBoxOffset = 0;
579 mStreamableFile = false;
580 mTimeScale = -1;
581 mHasFileLevelMeta = false;
582 mIsAvif = false;
583 mHasGainmap = false;
584 mFileLevelMetaDataSize = 0;
585 mPrimaryItemId = 0;
586 mAssociationEntryCount = 0;
587 mNumGrids = 0;
588 mNextItemId = kItemIdBase;
589 mHasRefs = false;
590 mResetStatus = OK;
591 mPreAllocFirstTime = true;
592 mPrevAllTracksTotalMetaDataSizeEstimate = 0;
593 mIsFirstChunk = false;
594 mDone = false;
595 mThread = 0;
596 mDriftTimeUs = 0;
597 mHasDolbyVision = false;
598
599 // Following variables only need to be set for the first recording session.
600 // And they will stay the same for all the recording sessions.
601 if (isFirstSession) {
602 mMoovExtraSize = 0;
603 mHasMoovBox = false;
604 mMetaKeys = new AMessage();
605 addDeviceMeta();
606 mLatitudex10000 = 0;
607 mLongitudex10000 = 0;
608 mAreGeoTagsAvailable = false;
609 mSwitchPending = false;
610 mIsFileSizeLimitExplicitlyRequested = false;
611 }
612
613 // Verify mFd is seekable
614 off64_t off = lseek64(mFd, 0, SEEK_SET);
615 if (off < 0) {
616 ALOGE("cannot seek mFd: %s (%d) %lld", strerror(errno), errno, (long long)mFd);
617 release();
618 }
619
620 if (fallocate64(mFd, FALLOC_FL_KEEP_SIZE, 0, 1) == 0) {
621 ALOGD("PreAllocation enabled");
622 mPreAllocationEnabled = true;
623 } else {
624 ALOGD("PreAllocation disabled. fallocate : %s, %d", strerror(errno), errno);
625 mPreAllocationEnabled = false;
626 }
627
628 for (List<Track *>::iterator it = mTracks.begin();
629 it != mTracks.end(); ++it) {
630 (*it)->resetInternal();
631 }
632 }
633
dump(int fd,const Vector<String16> & args)634 status_t MPEG4Writer::dump(
635 int fd, const Vector<String16>& args) {
636 const size_t SIZE = 256;
637 char buffer[SIZE];
638 String8 result;
639 snprintf(buffer, SIZE, " MPEG4Writer %p\n", this);
640 result.append(buffer);
641 snprintf(buffer, SIZE, " mStarted: %s\n", mStarted? "true": "false");
642 result.append(buffer);
643 ::write(fd, result.c_str(), result.size());
644 for (List<Track *>::iterator it = mTracks.begin();
645 it != mTracks.end(); ++it) {
646 (*it)->dump(fd, args);
647 }
648 return OK;
649 }
650
dump(int fd,const Vector<String16> &) const651 status_t MPEG4Writer::Track::dump(
652 int fd, const Vector<String16>& /* args */) const {
653 const size_t SIZE = 256;
654 char buffer[SIZE];
655 String8 result;
656 snprintf(buffer, SIZE, " %s track\n", getTrackType());
657 result.append(buffer);
658 snprintf(buffer, SIZE, " reached EOS: %s\n",
659 mReachedEOS? "true": "false");
660 result.append(buffer);
661 snprintf(buffer, SIZE, " frames encoded : %d\n", mStszTableEntries->count());
662 result.append(buffer);
663 snprintf(buffer, SIZE, " duration encoded : %" PRId64 " us\n", mTrackDurationUs);
664 result.append(buffer);
665 ::write(fd, result.c_str(), result.size());
666 return OK;
667 }
668
getDoviFourCC() const669 const char *MPEG4Writer::Track::getDoviFourCC() const {
670 if (mDoviProfile == DolbyVisionProfileDvheStn) {
671 return "dvh1";
672 } else if (mDoviProfile == DolbyVisionProfileDvheSt) {
673 return "hvc1";
674 } else if (mDoviProfile == DolbyVisionProfileDvavSe) {
675 return "avc1";
676 }
677 return nullptr;
678 }
679
680 // static
getFourCCForMime(const char * mime)681 const char *MPEG4Writer::Track::getFourCCForMime(const char *mime) {
682 if (mime == NULL) {
683 return NULL;
684 }
685 if (!strncasecmp(mime, "audio/", 6)) {
686 if (!strcasecmp(MEDIA_MIMETYPE_AUDIO_AMR_NB, mime)) {
687 return "samr";
688 } else if (!strcasecmp(MEDIA_MIMETYPE_AUDIO_AMR_WB, mime)) {
689 return "sawb";
690 } else if (!strcasecmp(MEDIA_MIMETYPE_AUDIO_AAC, mime)) {
691 return "mp4a";
692 }
693 } else if (!strncasecmp(mime, "video/", 6)) {
694 if (!strcasecmp(MEDIA_MIMETYPE_VIDEO_MPEG4, mime)) {
695 return "mp4v";
696 } else if (!strcasecmp(MEDIA_MIMETYPE_VIDEO_H263, mime)) {
697 return "s263";
698 } else if (!strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime)) {
699 return "avc1";
700 } else if (!strcasecmp(MEDIA_MIMETYPE_VIDEO_HEVC, mime)) {
701 return "hvc1";
702 } else if (!strcasecmp(MEDIA_MIMETYPE_VIDEO_AV1, mime)) {
703 return "av01";
704 } else if (editing_flags::muxer_mp4_enable_apv() &&
705 !strcasecmp(MEDIA_MIMETYPE_VIDEO_APV, mime)) {
706 return "apv1";
707 }
708 } else if (!strncasecmp(mime, "application/", 12)) {
709 return "mett";
710 } else if (!strcasecmp(MEDIA_MIMETYPE_IMAGE_ANDROID_HEIC, mime)) {
711 return "heic";
712 } else if (!strcasecmp(MEDIA_MIMETYPE_IMAGE_AVIF, mime)) {
713 return "avif";
714 } else {
715 ALOGE("Track (%s) other than video/audio/metadata is not supported", mime);
716 }
717 return NULL;
718 }
719
addSource(const sp<MediaSource> & source)720 status_t MPEG4Writer::addSource(const sp<MediaSource> &source) {
721 Mutex::Autolock l(mLock);
722 if (mStarted) {
723 ALOGE("Attempt to add source AFTER recording is started");
724 return UNKNOWN_ERROR;
725 }
726
727 CHECK(source.get() != NULL);
728
729 const char *mime = NULL;
730 sp<MetaData> meta = source->getFormat();
731 meta->findCString(kKeyMIMEType, &mime);
732
733
734 // Background mode for media transcoding. If either audio or video track signal this is in
735 // background mode, we will set all the threads to run in background priority.
736 int32_t isBackgroundMode;
737 if (meta && meta->findInt32(kKeyBackgroundMode, &isBackgroundMode)) {
738 mIsBackgroundMode |= isBackgroundMode;
739 }
740
741 if (flags_camera::camera_heif_gainmap()) {
742 int32_t gainmap = 0;
743 if (meta && meta->findInt32(kKeyGainmap, &gainmap)) {
744 mHasGainmap |= gainmap;
745 }
746 }
747
748 if (!strcmp(mime, MEDIA_MIMETYPE_VIDEO_DOLBY_VISION)) {
749 // For MEDIA_MIMETYPE_VIDEO_DOLBY_VISION,
750 // getFourCCForMime() requires profile information
751 // to decide the final FourCC codes.
752 // So we let the creation of the new track now and
753 // assign FourCC codes later using getDoviFourCC()
754 ALOGV("Add source mime '%s'", mime);
755 mHasDolbyVision = true;
756 } else if (Track::getFourCCForMime(mime) == NULL) {
757 ALOGE("Unsupported mime '%s'", mime);
758 return ERROR_UNSUPPORTED;
759 }
760
761 // This is a metadata track or the first track of either audio or video
762 // Go ahead to add the track.
763 Track *track = new Track(this, source, 1 + mTracks.size());
764 mTracks.push_back(track);
765
766 mHasMoovBox |= !track->isHeif();
767 mHasFileLevelMeta |= track->isHeif();
768 mIsAvif |= track->isAvif();
769
770 return OK;
771 }
772
startTracks(MetaData * params)773 status_t MPEG4Writer::startTracks(MetaData *params) {
774 if (mTracks.empty()) {
775 ALOGE("No source added");
776 return INVALID_OPERATION;
777 }
778
779 for (List<Track *>::iterator it = mTracks.begin();
780 it != mTracks.end(); ++it) {
781 status_t err = (*it)->start(params);
782
783 if (err != OK) {
784 for (List<Track *>::iterator it2 = mTracks.begin();
785 it2 != it; ++it2) {
786 (*it2)->stop();
787 }
788
789 return err;
790 }
791 }
792 return OK;
793 }
794
addDeviceMeta()795 void MPEG4Writer::addDeviceMeta() {
796 // add device info and estimate space in 'moov'
797 char val[PROPERTY_VALUE_MAX];
798 size_t n;
799 // meta size is estimated by adding up the following:
800 // - meta header structures, which occur only once (total 66 bytes)
801 // - size for each key, which consists of a fixed header (32 bytes),
802 // plus key length and data length.
803 mMoovExtraSize += 66;
804 if (property_get("ro.build.version.release", val, NULL)
805 && (n = strlen(val)) > 0) {
806 mMetaKeys->setString(kMetaKey_Version, val, n + 1);
807 mMoovExtraSize += sizeof(kMetaKey_Version) + n + 32;
808 }
809
810 if (property_get_bool("media.recorder.show_manufacturer_and_model", false)) {
811 if (property_get("ro.product.manufacturer", val, NULL)
812 && (n = strlen(val)) > 0) {
813 mMetaKeys->setString(kMetaKey_Manufacturer, val, n + 1);
814 mMoovExtraSize += sizeof(kMetaKey_Manufacturer) + n + 32;
815 }
816 if (property_get("ro.product.model", val, NULL)
817 && (n = strlen(val)) > 0) {
818 mMetaKeys->setString(kMetaKey_Model, val, n + 1);
819 mMoovExtraSize += sizeof(kMetaKey_Model) + n + 32;
820 }
821 }
822 #ifdef SHOW_MODEL_BUILD
823 if (property_get("ro.build.display.id", val, NULL)
824 && (n = strlen(val)) > 0) {
825 mMetaKeys->setString(kMetaKey_Build, val, n + 1);
826 mMoovExtraSize += sizeof(kMetaKey_Build) + n + 32;
827 }
828 #endif
829 }
830
estimateFileLevelMetaSize(MetaData * params)831 int64_t MPEG4Writer::estimateFileLevelMetaSize(MetaData *params) {
832 int32_t rotation;
833 if (!params || !params->findInt32(kKeyRotation, &rotation)) {
834 rotation = 0;
835 }
836
837 // base meta size
838 int64_t metaSize = 12 // meta fullbox header
839 + 33 // hdlr box
840 + 14 // pitm box
841 + 16 // iloc box (fixed size portion)
842 + 14 // iinf box (fixed size portion)
843 + 32 // iprp box (fixed size protion)
844 + 8 // idat box (when empty)
845 + 12 // iref box (when empty)
846 ;
847
848 if (flags_camera::camera_heif_gainmap()) {
849 metaSize += 36; // grpl box (when empty)
850 }
851
852 for (List<Track *>::iterator it = mTracks.begin();
853 it != mTracks.end(); ++it) {
854 if ((*it)->isHeif()) {
855 metaSize += (*it)->getMetaSizeIncrease(rotation, mTracks.size());
856 }
857 }
858
859 ALOGV("estimated meta size: %lld", (long long) metaSize);
860
861 // Need at least 8-byte padding at the end, otherwise the left-over
862 // freebox may become malformed
863 return metaSize + 8;
864 }
865
estimateMoovBoxSize(int32_t bitRate)866 int64_t MPEG4Writer::estimateMoovBoxSize(int32_t bitRate) {
867 // This implementation is highly experimental/heurisitic.
868 //
869 // Statistical analysis shows that metadata usually accounts
870 // for a small portion of the total file size, usually < 0.6%.
871
872 // The default MIN_MOOV_BOX_SIZE is set to 0.6% x 1MB / 2,
873 // where 1MB is the common file size limit for MMS application.
874 // The default MAX _MOOV_BOX_SIZE value is based on about 3
875 // minute video recording with a bit rate about 3 Mbps, because
876 // statistics show that most captured videos are less than 3 minutes.
877
878 // If the estimation is wrong, we will pay the price of wasting
879 // some reserved space. This should not happen so often statistically.
880 static const int64_t MIN_MOOV_BOX_SIZE = 3 * 1024; // 3 KibiBytes
881 static const int64_t MAX_MOOV_BOX_SIZE = (180 * 3000000 * 6LL / 8000); // 395.5 KibiBytes
882 int64_t size = MIN_MOOV_BOX_SIZE;
883
884 // Max file size limit is set
885 if (mMaxFileSizeLimitBytes != 0 && mIsFileSizeLimitExplicitlyRequested) {
886 size = mMaxFileSizeLimitBytes * 6 / 1000;
887 }
888
889 // Max file duration limit is set
890 if (mMaxFileDurationLimitUs != 0) {
891 if (bitRate > 0) {
892 int64_t size2 =
893 ((mMaxFileDurationLimitUs / 1000) * bitRate * 6) / 8000000;
894 if (mMaxFileSizeLimitBytes != 0 && mIsFileSizeLimitExplicitlyRequested) {
895 // When both file size and duration limits are set,
896 // we use the smaller limit of the two.
897 if (size > size2) {
898 size = size2;
899 }
900 } else {
901 // Only max file duration limit is set
902 size = size2;
903 }
904 }
905 }
906
907 if (size < MIN_MOOV_BOX_SIZE) {
908 size = MIN_MOOV_BOX_SIZE;
909 }
910
911 // Any long duration recording will be probably end up with
912 // non-streamable mp4 file.
913 if (size > MAX_MOOV_BOX_SIZE) {
914 size = MAX_MOOV_BOX_SIZE;
915 }
916
917 // Account for the extra stuff (Geo, meta keys, etc.)
918 size += mMoovExtraSize;
919
920 ALOGI("limits: %" PRId64 "/%" PRId64 " bytes/us, bit rate: %d bps and the"
921 " estimated moov size %" PRId64 " bytes",
922 mMaxFileSizeLimitBytes, mMaxFileDurationLimitUs, bitRate, size);
923
924 return size;
925 }
926
validateAllTracksId(bool akKey4BitTrackIds)927 status_t MPEG4Writer::validateAllTracksId(bool akKey4BitTrackIds) {
928 for (List<Track *>::iterator it = mTracks.begin(); it != mTracks.end(); ++it) {
929 if (!(*it)->getTrackId().isValid(akKey4BitTrackIds)) {
930 return BAD_VALUE;
931 }
932 }
933 return OK;
934 }
935
start(MetaData * param)936 status_t MPEG4Writer::start(MetaData *param) {
937 if (mInitCheck != OK) {
938 return UNKNOWN_ERROR;
939 }
940 mStartMeta = param;
941
942 /*
943 * Check mMaxFileSizeLimitBytes at the beginning since mMaxFileSizeLimitBytes may be implicitly
944 * changed later as per filesizebits of filesystem even if user does not set it explicitly.
945 */
946 if (mMaxFileSizeLimitBytes != 0) {
947 mIsFileSizeLimitExplicitlyRequested = true;
948 }
949
950 /* mMaxFileSizeLimitBytes has to be set everytime fd is switched, hence the following code is
951 * appropriate in start() method.
952 */
953 int32_t fileSizeBits = fpathconf(mFd, _PC_FILESIZEBITS);
954 ALOGD("fpathconf _PC_FILESIZEBITS:%" PRId32, fileSizeBits);
955 fileSizeBits = std::min(fileSizeBits, 52 /* cap it below 4 peta bytes */);
956 int64_t maxFileSizeBytes = ((int64_t)1 << fileSizeBits) - 1;
957 if (mMaxFileSizeLimitBytes > maxFileSizeBytes) {
958 mMaxFileSizeLimitBytes = maxFileSizeBytes;
959 ALOGD("File size limit (%" PRId64 " bytes) too big. It is changed to %" PRId64 " bytes",
960 mMaxFileSizeLimitBytes, maxFileSizeBytes);
961 } else if (mMaxFileSizeLimitBytes == 0) {
962 mMaxFileSizeLimitBytes = maxFileSizeBytes;
963 ALOGD("File size limit set to %" PRId64 " bytes implicitly", maxFileSizeBytes);
964 }
965
966 int32_t use2ByteNalLength;
967 if (param &&
968 param->findInt32(kKey2ByteNalLength, &use2ByteNalLength) &&
969 use2ByteNalLength) {
970 mUse4ByteNalLength = false;
971 }
972
973 int32_t isRealTimeRecording;
974 if (param && param->findInt32(kKeyRealTimeRecording, &isRealTimeRecording)) {
975 mIsRealTimeRecording = isRealTimeRecording;
976 }
977
978 mStartTimestampUs = -1;
979
980 if (mStarted) {
981 if (mPaused) {
982 mPaused = false;
983 return startTracks(param);
984 }
985 return OK;
986 }
987
988 if (!param ||
989 !param->findInt32(kKeyTimeScale, &mTimeScale)) {
990 // Increased by a factor of 10 to improve precision of segment duration in edit list entry.
991 mTimeScale = 10000;
992 }
993 CHECK_GT(mTimeScale, 0);
994 ALOGV("movie time scale: %d", mTimeScale);
995
996 /*
997 * When the requested file size limit is small, the priority
998 * is to meet the file size limit requirement, rather than
999 * to make the file streamable. mStreamableFile does not tell
1000 * whether the actual recorded file is streamable or not.
1001 */
1002 mStreamableFile =
1003 (mMaxFileSizeLimitBytes != 0 &&
1004 mMaxFileSizeLimitBytes >= kMinStreamableFileSizeInBytes);
1005
1006 /*
1007 * mWriteBoxToMemory is true if the amount of data in a file-level meta or
1008 * moov box is smaller than the reserved free space at the beginning of a
1009 * file, AND when the content of the box is constructed. Note that video/
1010 * audio frame data is always written to the file but not in the memory.
1011 *
1012 * Before stop()/reset() is called, mWriteBoxToMemory is always
1013 * false. When reset() is called at the end of a recording session,
1014 * file-level meta and/or moov box needs to be constructed.
1015 *
1016 * 1) Right before the box is constructed, mWriteBoxToMemory to set to
1017 * mStreamableFile so that if the file is intended to be streamable, it
1018 * is set to true; otherwise, it is set to false. When the value is set
1019 * to false, all the content of that box is written immediately to
1020 * the end of the file. When the value is set to true, all the
1021 * content of that box is written to an in-memory cache,
1022 * mInMemoryCache, util the following condition happens. Note
1023 * that the size of the in-memory cache is the same as the
1024 * reserved free space at the beginning of the file.
1025 *
1026 * 2) While the data of the box is written to an in-memory
1027 * cache, the data size is checked against the reserved space.
1028 * If the data size surpasses the reserved space, subsequent box data
1029 * could no longer be hold in the in-memory cache. This also
1030 * indicates that the reserved space was too small. At this point,
1031 * _all_ subsequent box data must be written to the end of the file.
1032 * mWriteBoxToMemory must be set to false to direct the write
1033 * to the file.
1034 *
1035 * 3) If the data size in the box is smaller than the reserved
1036 * space after the box is completely constructed, the in-memory
1037 * cache copy of the box is written to the reserved free space.
1038 * mWriteBoxToMemory is always set to false after all boxes that
1039 * using the in-memory cache have been constructed.
1040 */
1041 mWriteBoxToMemory = false;
1042 mInMemoryCache = NULL;
1043 mInMemoryCacheOffset = 0;
1044
1045 status_t err = OK;
1046 int32_t is4bitTrackId = false;
1047 if (param && param->findInt32(kKey4BitTrackIds, &is4bitTrackId) && is4bitTrackId) {
1048 err = validateAllTracksId(true);
1049 } else {
1050 err = validateAllTracksId(false);
1051 }
1052 if (err != OK) {
1053 return err;
1054 }
1055
1056 ALOGV("muxer starting: mHasMoovBox %d, mHasFileLevelMeta %d, mIsAvif %d",
1057 mHasMoovBox, mHasFileLevelMeta, mIsAvif);
1058
1059 err = startWriterThread();
1060 if (err != OK) {
1061 return err;
1062 }
1063
1064 err = setupAndStartLooper();
1065 if (err != OK) {
1066 return err;
1067 }
1068
1069 writeFtypBox(param);
1070
1071 mFreeBoxOffset = mOffset;
1072
1073 if (mInMemoryCacheSize == 0) {
1074 int32_t bitRate = -1;
1075 if (mHasFileLevelMeta) {
1076 mFileLevelMetaDataSize = estimateFileLevelMetaSize(param);
1077 mInMemoryCacheSize += mFileLevelMetaDataSize;
1078 }
1079 if (mHasMoovBox) {
1080 if (param) {
1081 param->findInt32(kKeyBitRate, &bitRate);
1082 }
1083 mInMemoryCacheSize += estimateMoovBoxSize(bitRate);
1084 }
1085 }
1086 if (mStreamableFile) {
1087 // Reserve a 'free' box only for streamable file
1088 seekOrPostError(mFd, mFreeBoxOffset, SEEK_SET);
1089 writeInt32(mInMemoryCacheSize);
1090 write("free", 4);
1091 if (mInMemoryCacheSize >= 8) {
1092 off64_t bufSize = mInMemoryCacheSize - 8;
1093 char* zeroBuffer = new (std::nothrow) char[bufSize];
1094 if (zeroBuffer) {
1095 std::fill_n(zeroBuffer, bufSize, '0');
1096 writeOrPostError(mFd, zeroBuffer, bufSize);
1097 delete [] zeroBuffer;
1098 } else {
1099 ALOGW("freebox in file isn't initialized to 0");
1100 }
1101 } else {
1102 ALOGW("freebox size is less than 8:%" PRId64, mInMemoryCacheSize);
1103 }
1104 mMdatOffset = mFreeBoxOffset + mInMemoryCacheSize;
1105 } else {
1106 mMdatOffset = mOffset;
1107 }
1108
1109 mOffset = mMdatOffset;
1110 seekOrPostError(mFd, mMdatOffset, SEEK_SET);
1111 write("\x00\x00\x00\x01mdat????????", 16);
1112
1113 /* Confirm whether the writing of the initial file atoms, ftyp and free,
1114 * are written to the file properly by posting kWhatNoIOErrorSoFar to the
1115 * MP4WtrCtrlHlpLooper that's handling write and seek errors also. If there
1116 * was kWhatIOError, the following two scenarios should be handled.
1117 * 1) If kWhatIOError was delivered and processed, MP4WtrCtrlHlpLooper
1118 * would have stopped all threads gracefully already and posting
1119 * kWhatNoIOErrorSoFar would fail.
1120 * 2) If kWhatIOError wasn't delivered or getting processed,
1121 * kWhatNoIOErrorSoFar should get posted successfully. Wait for
1122 * response from MP4WtrCtrlHlpLooper.
1123 */
1124 sp<AMessage> msg = new AMessage(kWhatNoIOErrorSoFar, mReflector);
1125 sp<AMessage> response;
1126 err = msg->postAndAwaitResponse(&response);
1127 if (err != OK || !response->findInt32("err", &err) || err != OK) {
1128 return ERROR_IO;
1129 }
1130
1131 err = startTracks(param);
1132 if (err != OK) {
1133 return err;
1134 }
1135
1136 mStarted = true;
1137 return OK;
1138 }
1139
stop()1140 status_t MPEG4Writer::stop() {
1141 // If reset was in progress, wait for it to complete.
1142 return reset(true, true);
1143 }
1144
pause()1145 status_t MPEG4Writer::pause() {
1146 ALOGW("MPEG4Writer: pause is not supported");
1147 return ERROR_UNSUPPORTED;
1148 }
1149
stopWriterThread()1150 status_t MPEG4Writer::stopWriterThread() {
1151 ALOGV("Stopping writer thread");
1152 if (!mWriterThreadStarted) {
1153 ALOGD("Writer thread not started");
1154 return OK;
1155 }
1156 {
1157 Mutex::Autolock autolock(mLock);
1158 mDone = true;
1159 mChunkReadyCondition.signal();
1160 }
1161
1162 void *dummy;
1163 status_t err = OK;
1164 int retVal = pthread_join(mThread, &dummy);
1165 if (retVal == 0) {
1166 err = static_cast<status_t>(reinterpret_cast<uintptr_t>(dummy));
1167 ALOGD("WriterThread stopped. Status:%d", err);
1168 } else {
1169 ALOGE("stopWriterThread pthread_join status:%d", retVal);
1170 err = UNKNOWN_ERROR;
1171 }
1172 mWriterThreadStarted = false;
1173 return err;
1174 }
1175
1176 /*
1177 * MP4 file standard defines a composition matrix:
1178 * | a b u |
1179 * | c d v |
1180 * | x y w |
1181 *
1182 * the element in the matrix is stored in the following
1183 * order: {a, b, u, c, d, v, x, y, w},
1184 * where a, b, c, d, x, and y is in 16.16 format, while
1185 * u, v and w is in 2.30 format.
1186 */
writeCompositionMatrix(int degrees)1187 void MPEG4Writer::writeCompositionMatrix(int degrees) {
1188 ALOGV("writeCompositionMatrix");
1189 uint32_t a = 0x00010000;
1190 uint32_t b = 0;
1191 uint32_t c = 0;
1192 uint32_t d = 0x00010000;
1193 switch (degrees) {
1194 case 0:
1195 break;
1196 case 90:
1197 a = 0;
1198 b = 0x00010000;
1199 c = 0xFFFF0000;
1200 d = 0;
1201 break;
1202 case 180:
1203 a = 0xFFFF0000;
1204 d = 0xFFFF0000;
1205 break;
1206 case 270:
1207 a = 0;
1208 b = 0xFFFF0000;
1209 c = 0x00010000;
1210 d = 0;
1211 break;
1212 default:
1213 CHECK(!"Should never reach this unknown rotation");
1214 break;
1215 }
1216
1217 writeInt32(a); // a
1218 writeInt32(b); // b
1219 writeInt32(0); // u
1220 writeInt32(c); // c
1221 writeInt32(d); // d
1222 writeInt32(0); // v
1223 writeInt32(0); // x
1224 writeInt32(0); // y
1225 writeInt32(0x40000000); // w
1226 }
1227
printWriteDurations()1228 void MPEG4Writer::printWriteDurations() {
1229 if (mWriteDurationPQ.empty()) {
1230 return;
1231 }
1232 std::string writeDurationsString =
1233 "Top " + std::to_string(mWriteDurationPQ.size()) + " write durations(microseconds):";
1234 uint8_t i = 0;
1235 while (!mWriteDurationPQ.empty()) {
1236 writeDurationsString +=
1237 " #" + std::to_string(++i) + ":" + std::to_string(mWriteDurationPQ.top().count());
1238 mWriteDurationPQ.pop();
1239 }
1240 ALOGD("%s", writeDurationsString.c_str());
1241 }
1242
release()1243 status_t MPEG4Writer::release() {
1244 ALOGD("release()");
1245 status_t err = OK;
1246 if (!truncatePreAllocation()) {
1247 if (err == OK) { err = ERROR_IO; }
1248 }
1249
1250 // TODO(b/174770856) remove this measurement (and perhaps the fsync)
1251 nsecs_t sync_started = systemTime(SYSTEM_TIME_REALTIME);
1252 if (fsync(mFd) != 0) {
1253 ALOGW("(ignored)fsync err:%s(%d)", std::strerror(errno), errno);
1254 // Don't bubble up fsync error, b/157291505.
1255 // if (err == OK) { err = ERROR_IO; }
1256 }
1257 nsecs_t sync_finished = systemTime(SYSTEM_TIME_REALTIME);
1258 nsecs_t sync_elapsed_ns = sync_finished - sync_started;
1259 int64_t filesize = -1;
1260 struct stat statbuf;
1261 if (fstat(mFd, &statbuf) == 0) {
1262 filesize = statbuf.st_size;
1263 }
1264 ALOGD("final fsync() takes %" PRId64 " ms, file size %" PRId64,
1265 sync_elapsed_ns / 1000000, (int64_t) filesize);
1266
1267 if (close(mFd) != 0) {
1268 ALOGE("close err:%s(%d)", std::strerror(errno), errno);
1269 if (err == OK) { err = ERROR_IO; }
1270 }
1271 mFd = -1;
1272 if (mNextFd != -1) {
1273 if (close(mNextFd) != 0) {
1274 ALOGE("close(mNextFd) error:%s(%d)", std::strerror(errno), errno);
1275 }
1276 if (err == OK) { err = ERROR_IO; }
1277 mNextFd = -1;
1278 }
1279 stopAndReleaseLooper();
1280 mInitCheck = NO_INIT;
1281 mStarted = false;
1282 free(mInMemoryCache);
1283 mInMemoryCache = NULL;
1284
1285 printWriteDurations();
1286
1287 return err;
1288 }
1289
finishCurrentSession()1290 status_t MPEG4Writer::finishCurrentSession() {
1291 ALOGV("finishCurrentSession");
1292 /* Don't wait if reset is in progress already, that avoids deadlock
1293 * as finishCurrentSession() is called from control looper thread.
1294 */
1295 return reset(false, false);
1296 }
1297
switchFd()1298 status_t MPEG4Writer::switchFd() {
1299 ALOGV("switchFd");
1300 Mutex::Autolock l(mLock);
1301 if (mSwitchPending) {
1302 return OK;
1303 }
1304
1305 // Wait for the signal only if the new file is not available.
1306 if (mNextFd == -1) {
1307 status_t res = mFdCond.waitRelative(mLock, kFdCondWaitTimeoutNs);
1308 if (res != OK) {
1309 ALOGW("No FileDescriptor for next recording");
1310 return INVALID_OPERATION;
1311 }
1312 }
1313
1314 mSwitchPending = true;
1315 sp<AMessage> msg = new AMessage(kWhatSwitch, mReflector);
1316 status_t err = msg->post();
1317
1318 return err;
1319 }
1320
reset(bool stopSource,bool waitForAnyPreviousCallToComplete)1321 status_t MPEG4Writer::reset(bool stopSource, bool waitForAnyPreviousCallToComplete) {
1322 ALOGD("reset()");
1323 std::unique_lock<std::mutex> lk(mResetMutex, std::defer_lock);
1324 if (waitForAnyPreviousCallToComplete) {
1325 /* stop=>reset from client needs the return value of reset call, hence wait here
1326 * if a reset was in process already.
1327 */
1328 lk.lock();
1329 } else if (!lk.try_lock()) {
1330 /* Internal reset from control looper thread shouldn't wait for any reset in
1331 * process already.
1332 */
1333 return INVALID_OPERATION;
1334 }
1335
1336 if (mResetStatus != OK) {
1337 /* Don't have to proceed if reset has finished with an error before.
1338 * If there was no error before, proceeding reset would be harmless, as the
1339 * the call would return from the mInitCheck condition below.
1340 */
1341 return mResetStatus;
1342 }
1343
1344 if (mInitCheck != OK) {
1345 mResetStatus = OK;
1346 return mResetStatus;
1347 } else {
1348 if (!mWriterThreadStarted ||
1349 !mStarted) {
1350 status_t writerErr = OK;
1351 if (mWriterThreadStarted) {
1352 writerErr = stopWriterThread();
1353 }
1354 status_t retErr = release();
1355 if (writerErr != OK) {
1356 retErr = writerErr;
1357 }
1358 mResetStatus = retErr;
1359 return mResetStatus;
1360 }
1361 }
1362
1363 status_t err = OK;
1364 int64_t maxDurationUs = 0;
1365 int64_t minDurationUs = 0x7fffffffffffffffLL;
1366 int32_t nonImageTrackCount = 0;
1367 for (List<Track *>::iterator it = mTracks.begin();
1368 it != mTracks.end(); ++it) {
1369 status_t trackErr = (*it)->stop(stopSource);
1370 WARN_UNLESS(trackErr == OK, "%s track stopped with an error",
1371 (*it)->getTrackType());
1372 if (err == OK && trackErr != OK) {
1373 err = trackErr;
1374 }
1375
1376 // skip image tracks
1377 if ((*it)->isHeif()) continue;
1378 nonImageTrackCount++;
1379
1380 int64_t durationUs = (*it)->getDurationUs();
1381 if (durationUs > maxDurationUs) {
1382 maxDurationUs = durationUs;
1383 }
1384 if (durationUs < minDurationUs) {
1385 minDurationUs = durationUs;
1386 }
1387 }
1388
1389 if (nonImageTrackCount > 1) {
1390 ALOGD("Duration from tracks range is [%" PRId64 ", %" PRId64 "] us",
1391 minDurationUs, maxDurationUs);
1392 }
1393
1394 status_t writerErr = stopWriterThread();
1395
1396 // Propagating writer error
1397 if (err == OK && writerErr != OK) {
1398 err = writerErr;
1399 }
1400
1401 // Do not write out movie header on error except malformed track.
1402 // TODO: Remove samples of malformed tracks added in mdat.
1403 if (err != OK && err != ERROR_MALFORMED) {
1404 // Ignoring release() return value as there was an "err" already.
1405 release();
1406 mResetStatus = err;
1407 return mResetStatus;
1408 }
1409
1410 // Fix up the size of the 'mdat' chunk.
1411 seekOrPostError(mFd, mMdatOffset + 8, SEEK_SET);
1412 uint64_t size = mOffset - mMdatOffset;
1413 size = hton64(size);
1414 writeOrPostError(mFd, &size, 8);
1415 seekOrPostError(mFd, mOffset, SEEK_SET);
1416 mMdatEndOffset = mOffset;
1417
1418 // Construct file-level meta and moov box now
1419 mInMemoryCacheOffset = 0;
1420 mWriteBoxToMemory = mStreamableFile;
1421 if (mWriteBoxToMemory) {
1422 // There is no need to allocate in-memory cache
1423 // if the file is not streamable.
1424
1425 mInMemoryCache = (uint8_t *) malloc(mInMemoryCacheSize);
1426 CHECK(mInMemoryCache != NULL);
1427 }
1428
1429 if (mHasFileLevelMeta) {
1430 writeFileLevelMetaBox();
1431 if (mWriteBoxToMemory) {
1432 writeCachedBoxToFile("meta");
1433 } else {
1434 ALOGI("The file meta box is written at the end.");
1435 }
1436 }
1437
1438 if (mHasMoovBox) {
1439 writeMoovBox(maxDurationUs);
1440 // mWriteBoxToMemory could be set to false in
1441 // MPEG4Writer::write() method
1442 if (mWriteBoxToMemory) {
1443 writeCachedBoxToFile("moov");
1444 } else {
1445 ALOGI("The mp4 file will not be streamable.");
1446 }
1447 ALOGI("MOOV atom was written to the file");
1448 }
1449 mWriteBoxToMemory = false;
1450
1451 // Free in-memory cache for box writing
1452 if (mInMemoryCache != NULL) {
1453 free(mInMemoryCache);
1454 mInMemoryCache = NULL;
1455 mInMemoryCacheOffset = 0;
1456 }
1457
1458 CHECK(mBoxes.empty());
1459
1460 status_t errRelease = release();
1461 // Prioritize the error that occurred before release().
1462 if (err == OK) {
1463 err = errRelease;
1464 }
1465 mResetStatus = err;
1466 return mResetStatus;
1467 }
1468
1469 /*
1470 * Writes currently cached box into file.
1471 *
1472 * Must be called while mWriteBoxToMemory is true, and will not modify
1473 * mWriteBoxToMemory. After the call, remaining cache size will be
1474 * reduced and buffer offset will be set to the beginning of the cache.
1475 */
writeCachedBoxToFile(const char * type)1476 void MPEG4Writer::writeCachedBoxToFile(const char *type) {
1477 CHECK(mWriteBoxToMemory);
1478
1479 mWriteBoxToMemory = false;
1480 // Content of the box is saved in the cache, and the in-memory
1481 // box needs to be written to the file in a single shot.
1482
1483 CHECK_LE(mInMemoryCacheOffset + 8, mInMemoryCacheSize);
1484
1485 // Cached box
1486 seekOrPostError(mFd, mFreeBoxOffset, SEEK_SET);
1487 mOffset = mFreeBoxOffset;
1488 write(mInMemoryCache, 1, mInMemoryCacheOffset);
1489
1490 // Free box
1491 seekOrPostError(mFd, mOffset, SEEK_SET);
1492 mFreeBoxOffset = mOffset;
1493 writeInt32(mInMemoryCacheSize - mInMemoryCacheOffset);
1494 write("free", 4);
1495
1496 // Rewind buffering to the beginning, and restore mWriteBoxToMemory flag
1497 mInMemoryCacheSize -= mInMemoryCacheOffset;
1498 mInMemoryCacheOffset = 0;
1499 mWriteBoxToMemory = true;
1500
1501 ALOGV("dumped out %s box, estimated size remaining %lld",
1502 type, (long long)mInMemoryCacheSize);
1503 }
1504
getMpeg4Time()1505 uint32_t MPEG4Writer::getMpeg4Time() {
1506 time_t now = time(NULL);
1507 // MP4 file uses time counting seconds since midnight, Jan. 1, 1904
1508 // while time function returns Unix epoch values which starts
1509 // at 1970-01-01. Lets add the number of seconds between them
1510 static const uint32_t delta = (66 * 365 + 17) * (24 * 60 * 60);
1511 if (now < 0 || uint32_t(now) > UINT32_MAX - delta) {
1512 return 0;
1513 }
1514 uint32_t mpeg4Time = uint32_t(now) + delta;
1515 return mpeg4Time;
1516 }
1517
writeMvhdBox(int64_t durationUs)1518 void MPEG4Writer::writeMvhdBox(int64_t durationUs) {
1519 uint32_t now = getMpeg4Time();
1520 beginBox("mvhd");
1521 writeInt32(0); // version=0, flags=0
1522 writeInt32(now); // creation time
1523 writeInt32(now); // modification time
1524 writeInt32(mTimeScale); // mvhd timescale
1525 int32_t duration = (durationUs * mTimeScale + 5E5) / 1E6;
1526 writeInt32(duration);
1527 writeInt32(0x10000); // rate: 1.0
1528 writeInt16(0x100); // volume
1529 writeInt16(0); // reserved
1530 writeInt32(0); // reserved
1531 writeInt32(0); // reserved
1532 writeCompositionMatrix(0); // matrix
1533 writeInt32(0); // predefined
1534 writeInt32(0); // predefined
1535 writeInt32(0); // predefined
1536 writeInt32(0); // predefined
1537 writeInt32(0); // predefined
1538 writeInt32(0); // predefined
1539 writeInt32(mTracks.size() + 1); // nextTrackID
1540 endBox(); // mvhd
1541 }
1542
writeMoovBox(int64_t durationUs)1543 void MPEG4Writer::writeMoovBox(int64_t durationUs) {
1544 beginBox("moov");
1545 writeMvhdBox(durationUs);
1546 if (mAreGeoTagsAvailable) {
1547 writeUdtaBox();
1548 }
1549 writeMoovLevelMetaBox();
1550 // Loop through all the tracks to get the global time offset if there is
1551 // any ctts table appears in a video track.
1552 int64_t minCttsOffsetTimeUs = kMaxCttsOffsetTimeUs;
1553 for (List<Track *>::iterator it = mTracks.begin();
1554 it != mTracks.end(); ++it) {
1555 if (!(*it)->isHeif()) {
1556 minCttsOffsetTimeUs =
1557 std::min(minCttsOffsetTimeUs, (*it)->getMinCttsOffsetTimeUs());
1558 }
1559 }
1560 ALOGI("Adjust the moov start time from %lld us -> %lld us", (long long)mStartTimestampUs,
1561 (long long)(mStartTimestampUs + minCttsOffsetTimeUs - kMaxCttsOffsetTimeUs));
1562 // Adjust movie start time.
1563 mStartTimestampUs += minCttsOffsetTimeUs - kMaxCttsOffsetTimeUs;
1564
1565 // Add mStartTimeOffsetBFramesUs(-ve or zero) to the start offset of tracks.
1566 mStartTimeOffsetBFramesUs = minCttsOffsetTimeUs - kMaxCttsOffsetTimeUs;
1567 ALOGV("mStartTimeOffsetBFramesUs :%" PRId32, mStartTimeOffsetBFramesUs);
1568
1569 for (List<Track *>::iterator it = mTracks.begin();
1570 it != mTracks.end(); ++it) {
1571 if (!(*it)->isHeif()) {
1572 (*it)->writeTrackHeader();
1573 }
1574 }
1575 endBox(); // moov
1576 }
1577
writeFtypBox(MetaData * param)1578 void MPEG4Writer::writeFtypBox(MetaData *param) {
1579 beginBox("ftyp");
1580
1581 int32_t fileType;
1582 if (!param || !param->findInt32(kKeyFileType, &fileType)) {
1583 fileType = OUTPUT_FORMAT_MPEG_4;
1584 }
1585 if (fileType != OUTPUT_FORMAT_MPEG_4 && fileType != OUTPUT_FORMAT_HEIF) {
1586 writeFourcc("3gp4");
1587 writeInt32(0);
1588 writeFourcc("isom");
1589 writeFourcc("3gp4");
1590 } else {
1591 // Only write "heic"/"avif" as major brand if the client specified HEIF/AVIF
1592 // AND we indeed receive some image heic/avif tracks.
1593 if (fileType == OUTPUT_FORMAT_HEIF && mHasFileLevelMeta) {
1594 if (mIsAvif) {
1595 writeFourcc("avif");
1596 } else {
1597 writeFourcc("heic");
1598 }
1599 } else {
1600 writeFourcc("mp42");
1601 }
1602 writeInt32(0);
1603 if (mHasFileLevelMeta) {
1604 if (mIsAvif) {
1605 writeFourcc("mif1");
1606 writeFourcc("miaf");
1607 writeFourcc("avif");
1608 } else {
1609 writeFourcc("mif1");
1610 writeFourcc("heic");
1611 if (flags_camera::camera_heif_gainmap() && mHasGainmap) {
1612 writeFourcc("tmap");
1613 }
1614 }
1615 }
1616 if (mHasMoovBox) {
1617 writeFourcc("isom");
1618 writeFourcc("mp42");
1619 }
1620 // If an AV1 video track is present, write "av01" as one of the
1621 // compatible brands.
1622 for (List<Track *>::iterator it = mTracks.begin(); it != mTracks.end();
1623 ++it) {
1624 if ((*it)->isAv1()) {
1625 writeFourcc("av01");
1626 break;
1627 }
1628 }
1629 // The brand ‘dby1’ should be used in the compatible_brands field to indicate that the file
1630 // is compliant with all Dolby Extensions. For details, refer to
1631 // https://professional.dolby.com/siteassets/content-creation/dolby-vision-for-content-creators/dolby_vision_bitstreams_within_the_iso_base_media_file_format_dec2017.pdf
1632 // Chapter 7, Dolby Vision Files.
1633 if (fileType == OUTPUT_FORMAT_MPEG_4 && mHasDolbyVision) {
1634 writeFourcc("dby1");
1635 }
1636 }
1637
1638 endBox();
1639 }
1640
isTestModeEnabled()1641 static bool isTestModeEnabled() {
1642 #if (PROPERTY_VALUE_MAX < 5)
1643 #error "PROPERTY_VALUE_MAX must be at least 5"
1644 #endif
1645
1646 // Test mode is enabled only if rw.media.record.test system
1647 // property is enabled.
1648 if (property_get_bool("rw.media.record.test", false)) {
1649 return true;
1650 }
1651 return false;
1652 }
1653
sendSessionSummary()1654 void MPEG4Writer::sendSessionSummary() {
1655 // Send session summary only if test mode is enabled
1656 if (!isTestModeEnabled()) {
1657 return;
1658 }
1659
1660 for (List<ChunkInfo>::iterator it = mChunkInfos.begin();
1661 it != mChunkInfos.end(); ++it) {
1662 uint32_t trackNum = (it->mTrack->getTrackId().getId() << 28);
1663 notify(MEDIA_RECORDER_TRACK_EVENT_INFO,
1664 trackNum | MEDIA_RECORDER_TRACK_INTER_CHUNK_TIME_MS,
1665 it->mMaxInterChunkDurUs);
1666 }
1667 }
1668
setInterleaveDuration(uint32_t durationUs)1669 status_t MPEG4Writer::setInterleaveDuration(uint32_t durationUs) {
1670 mInterleaveDurationUs = durationUs;
1671 return OK;
1672 }
1673
lock()1674 void MPEG4Writer::lock() {
1675 mLock.lock();
1676 }
1677
unlock()1678 void MPEG4Writer::unlock() {
1679 mLock.unlock();
1680 }
1681
addSample_l(MediaBuffer * buffer,bool usePrefix,uint32_t tiffHdrOffset,size_t * bytesWritten)1682 off64_t MPEG4Writer::addSample_l(
1683 MediaBuffer *buffer, bool usePrefix,
1684 uint32_t tiffHdrOffset, size_t *bytesWritten) {
1685 off64_t old_offset = mOffset;
1686 int64_t offset;
1687 ALOGV("buffer->range_length:%lld", (long long)buffer->range_length());
1688 if (buffer->meta_data().findInt64(kKeySampleFileOffset, &offset)) {
1689 ALOGV("offset:%lld, old_offset:%lld", (long long)offset, (long long)old_offset);
1690 if (mMaxOffsetAppend > offset) {
1691 // This has already been appended, skip updating mOffset value.
1692 *bytesWritten = buffer->range_length();
1693 return offset;
1694 }
1695 if (old_offset == offset) {
1696 mOffset += buffer->range_length();
1697 } else {
1698 ALOGV("offset and old_offset are not equal! diff:%lld", (long long)offset - old_offset);
1699 mOffset = offset + buffer->range_length();
1700 // mOffset += buffer->range_length() + offset - old_offset;
1701 }
1702 *bytesWritten = buffer->range_length();
1703 ALOGV("mOffset:%lld, mMaxOffsetAppend:%lld, bytesWritten:%lld", (long long)mOffset,
1704 (long long)mMaxOffsetAppend, (long long)*bytesWritten);
1705 mMaxOffsetAppend = std::max(mOffset, mMaxOffsetAppend);
1706 seekOrPostError(mFd, mMaxOffsetAppend, SEEK_SET);
1707 return offset;
1708 }
1709
1710 ALOGV("mOffset:%lld, mMaxOffsetAppend:%lld", (long long)mOffset, (long long)mMaxOffsetAppend);
1711
1712 if (usePrefix) {
1713 addMultipleLengthPrefixedSamples_l(buffer);
1714 } else {
1715 if (tiffHdrOffset > 0) {
1716 tiffHdrOffset = htonl(tiffHdrOffset);
1717 writeOrPostError(mFd, &tiffHdrOffset, 4); // exif_tiff_header_offset field
1718 mOffset += 4;
1719 }
1720
1721 writeOrPostError(mFd, (const uint8_t*)buffer->data() + buffer->range_offset(),
1722 buffer->range_length());
1723
1724 mOffset += buffer->range_length();
1725 }
1726 *bytesWritten = mOffset - old_offset;
1727
1728 ALOGV("mOffset:%lld, old_offset:%lld, bytesWritten:%lld", (long long)mOffset,
1729 (long long)old_offset, (long long)*bytesWritten);
1730
1731 return old_offset;
1732 }
1733
StripStartcode(MediaBuffer * buffer)1734 static void StripStartcode(MediaBuffer *buffer) {
1735 if (buffer->range_length() < 4) {
1736 return;
1737 }
1738
1739 const uint8_t *ptr =
1740 (const uint8_t *)buffer->data() + buffer->range_offset();
1741
1742 if (!memcmp(ptr, "\x00\x00\x00\x01", 4)) {
1743 ALOGV("stripping start code");
1744 buffer->set_range(
1745 buffer->range_offset() + 4, buffer->range_length() - 4);
1746 }
1747 }
1748
addMultipleLengthPrefixedSamples_l(MediaBuffer * buffer)1749 void MPEG4Writer::addMultipleLengthPrefixedSamples_l(MediaBuffer *buffer) {
1750 const uint8_t *dataStart = (const uint8_t *)buffer->data() + buffer->range_offset();
1751 const uint8_t *currentNalStart = dataStart;
1752 const uint8_t *nextNalStart;
1753 const uint8_t *data = dataStart;
1754 size_t nextNalSize;
1755 size_t searchSize = buffer->range_length();
1756
1757 while (getNextNALUnit(&data, &searchSize, &nextNalStart,
1758 &nextNalSize, true) == OK) {
1759 size_t currentNalSize = nextNalStart - currentNalStart - 4 /* strip start-code */;
1760 MediaBuffer *nalBuf = new MediaBuffer((void *)currentNalStart, currentNalSize);
1761 addLengthPrefixedSample_l(nalBuf);
1762 nalBuf->release();
1763
1764 currentNalStart = nextNalStart;
1765 }
1766
1767 size_t currentNalOffset = currentNalStart - dataStart;
1768 buffer->set_range(buffer->range_offset() + currentNalOffset,
1769 buffer->range_length() - currentNalOffset);
1770 addLengthPrefixedSample_l(buffer);
1771 }
1772
addLengthPrefixedSample_l(MediaBuffer * buffer)1773 void MPEG4Writer::addLengthPrefixedSample_l(MediaBuffer *buffer) {
1774 ALOGV("alp:buffer->range_length:%lld", (long long)buffer->range_length());
1775 size_t length = buffer->range_length();
1776 if (mUse4ByteNalLength) {
1777 ALOGV("mUse4ByteNalLength");
1778 uint8_t x[4];
1779 x[0] = length >> 24;
1780 x[1] = (length >> 16) & 0xff;
1781 x[2] = (length >> 8) & 0xff;
1782 x[3] = length & 0xff;
1783 writeOrPostError(mFd, &x, 4);
1784 writeOrPostError(mFd, (const uint8_t*)buffer->data() + buffer->range_offset(), length);
1785 mOffset += length + 4;
1786 } else {
1787 ALOGV("mUse2ByteNalLength");
1788 CHECK_LT(length, 65536u);
1789
1790 uint8_t x[2];
1791 x[0] = length >> 8;
1792 x[1] = length & 0xff;
1793 writeOrPostError(mFd, &x, 2);
1794 writeOrPostError(mFd, (const uint8_t*)buffer->data() + buffer->range_offset(), length);
1795 mOffset += length + 2;
1796 }
1797 }
1798
write(const void * ptr,size_t size,size_t nmemb)1799 size_t MPEG4Writer::write(
1800 const void *ptr, size_t size, size_t nmemb) {
1801
1802 const size_t bytes = size * nmemb;
1803 if (mWriteBoxToMemory) {
1804
1805 off64_t boxSize = 8 + mInMemoryCacheOffset + bytes;
1806 if (boxSize > mInMemoryCacheSize) {
1807 // The reserved free space at the beginning of the file is not big
1808 // enough. Boxes should be written to the end of the file from now
1809 // on, but not to the in-memory cache.
1810
1811 // We write partial box that is in the memory to the file first.
1812 for (List<off64_t>::iterator it = mBoxes.begin();
1813 it != mBoxes.end(); ++it) {
1814 (*it) += mOffset;
1815 }
1816 seekOrPostError(mFd, mOffset, SEEK_SET);
1817 writeOrPostError(mFd, mInMemoryCache, mInMemoryCacheOffset);
1818 writeOrPostError(mFd, ptr, bytes);
1819 mOffset += (bytes + mInMemoryCacheOffset);
1820
1821 // All subsequent boxes will be written to the end of the file.
1822 mWriteBoxToMemory = false;
1823 } else {
1824 memcpy(mInMemoryCache + mInMemoryCacheOffset, ptr, bytes);
1825 mInMemoryCacheOffset += bytes;
1826 }
1827 } else {
1828 writeOrPostError(mFd, ptr, bytes);
1829 mOffset += bytes;
1830 }
1831 return bytes;
1832 }
1833
writeOrPostError(int fd,const void * buf,size_t count)1834 void MPEG4Writer::writeOrPostError(int fd, const void* buf, size_t count) {
1835 if (mWriteSeekErr == true)
1836 return;
1837
1838 auto beforeTP = std::chrono::high_resolution_clock::now();
1839 ssize_t bytesWritten = ::write(fd, buf, count);
1840 auto afterTP = std::chrono::high_resolution_clock::now();
1841 auto writeDuration =
1842 std::chrono::duration_cast<std::chrono::microseconds>(afterTP - beforeTP).count();
1843 mWriteDurationPQ.emplace(writeDuration);
1844 if (mWriteDurationPQ.size() > kWriteDurationsCount) {
1845 mWriteDurationPQ.pop();
1846 }
1847
1848 /* Write as much as possible during stop() execution when there was an error
1849 * (mWriteSeekErr == true) in the previous call to write() or lseek64().
1850 */
1851 if (bytesWritten == count)
1852 return;
1853 mWriteSeekErr = true;
1854 // Note that errno is not changed even when bytesWritten < count.
1855 ALOGE("writeOrPostError bytesWritten:%zd, count:%zu, error:%s(%d)", bytesWritten, count,
1856 std::strerror(errno), errno);
1857
1858 // Can't guarantee that file is usable or write would succeed anymore, hence signal to stop.
1859 sp<AMessage> msg = new AMessage(kWhatIOError, mReflector);
1860 msg->setInt32("err", ERROR_IO);
1861 WARN_UNLESS(msg->post() == OK, "writeOrPostError:error posting ERROR_IO");
1862 }
1863
seekOrPostError(int fd,off64_t offset,int whence)1864 void MPEG4Writer::seekOrPostError(int fd, off64_t offset, int whence) {
1865 if (mWriteSeekErr == true)
1866 return;
1867 off64_t resOffset = lseek64(fd, offset, whence);
1868 /* Allow to seek during stop() execution even when there was an error
1869 * (mWriteSeekErr == true) in the previous call to write() or lseek64().
1870 */
1871 if (resOffset == offset)
1872 return;
1873 mWriteSeekErr = true;
1874 ALOGE("seekOrPostError resOffset:%" PRIu64 ", offset:%" PRIu64 ", error:%s(%d)", resOffset,
1875 offset, std::strerror(errno), errno);
1876
1877 // Can't guarantee that file is usable or seek would succeed anymore, hence signal to stop.
1878 sp<AMessage> msg = new AMessage(kWhatIOError, mReflector);
1879 msg->setInt32("err", ERROR_IO);
1880 WARN_UNLESS(msg->post() == OK, "seekOrPostError:error posting ERROR_IO");
1881 }
1882
beginBox(uint32_t id)1883 void MPEG4Writer::beginBox(uint32_t id) {
1884 ALOGV("beginBox:%" PRIu32, id);
1885
1886 mBoxes.push_back(mWriteBoxToMemory?
1887 mInMemoryCacheOffset: mOffset);
1888
1889 writeInt32(0);
1890 writeInt32(id);
1891 }
1892
beginBox(const char * fourcc)1893 void MPEG4Writer::beginBox(const char *fourcc) {
1894 ALOGV("beginBox:%s", fourcc);
1895 CHECK_EQ(strlen(fourcc), 4u);
1896
1897 mBoxes.push_back(mWriteBoxToMemory?
1898 mInMemoryCacheOffset: mOffset);
1899
1900 writeInt32(0);
1901 writeFourcc(fourcc);
1902 }
1903
endBox()1904 void MPEG4Writer::endBox() {
1905 CHECK(!mBoxes.empty());
1906
1907 off64_t offset = *--mBoxes.end();
1908 mBoxes.erase(--mBoxes.end());
1909
1910 if (mWriteBoxToMemory) {
1911 int32_t x = htonl(mInMemoryCacheOffset - offset);
1912 memcpy(mInMemoryCache + offset, &x, 4);
1913 } else {
1914 seekOrPostError(mFd, offset, SEEK_SET);
1915 writeInt32(mOffset - offset);
1916 ALOGV("box size:%" PRIu64, mOffset - offset);
1917 mOffset -= 4;
1918 seekOrPostError(mFd, mOffset, SEEK_SET);
1919 }
1920 }
1921
writeInt8(int8_t x)1922 void MPEG4Writer::writeInt8(int8_t x) {
1923 write(&x, 1, 1);
1924 }
1925
writeInt16(int16_t x)1926 void MPEG4Writer::writeInt16(int16_t x) {
1927 x = htons(x);
1928 write(&x, 1, 2);
1929 }
1930
writeInt32(int32_t x)1931 void MPEG4Writer::writeInt32(int32_t x) {
1932 x = htonl(x);
1933 write(&x, 1, 4);
1934 }
1935
writeInt64(int64_t x)1936 void MPEG4Writer::writeInt64(int64_t x) {
1937 x = hton64(x);
1938 write(&x, 1, 8);
1939 }
1940
writeCString(const char * s)1941 void MPEG4Writer::writeCString(const char *s) {
1942 size_t n = strlen(s);
1943 write(s, 1, n + 1);
1944 }
1945
writeFourcc(const char * s)1946 void MPEG4Writer::writeFourcc(const char *s) {
1947 CHECK_EQ(strlen(s), 4u);
1948 write(s, 1, 4);
1949 }
1950
1951
1952 // Written in +/-DD.DDDD format
writeLatitude(int degreex10000)1953 void MPEG4Writer::writeLatitude(int degreex10000) {
1954 bool isNegative = (degreex10000 < 0);
1955 char sign = isNegative? '-': '+';
1956
1957 // Handle the whole part
1958 char str[9];
1959 int wholePart = degreex10000 / 10000;
1960 if (wholePart == 0) {
1961 snprintf(str, 5, "%c%.2d.", sign, wholePart);
1962 } else {
1963 snprintf(str, 5, "%+.2d.", wholePart);
1964 }
1965
1966 // Handle the fractional part
1967 int fractionalPart = degreex10000 - (wholePart * 10000);
1968 if (fractionalPart < 0) {
1969 fractionalPart = -fractionalPart;
1970 }
1971 snprintf(&str[4], 5, "%.4d", fractionalPart);
1972
1973 // Do not write the null terminator
1974 write(str, 1, 8);
1975 }
1976
1977 // Written in +/- DDD.DDDD format
writeLongitude(int degreex10000)1978 void MPEG4Writer::writeLongitude(int degreex10000) {
1979 bool isNegative = (degreex10000 < 0);
1980 char sign = isNegative? '-': '+';
1981
1982 // Handle the whole part
1983 char str[10];
1984 int wholePart = degreex10000 / 10000;
1985 if (wholePart == 0) {
1986 snprintf(str, 6, "%c%.3d.", sign, wholePart);
1987 } else {
1988 snprintf(str, 6, "%+.3d.", wholePart);
1989 }
1990
1991 // Handle the fractional part
1992 int fractionalPart = degreex10000 - (wholePart * 10000);
1993 if (fractionalPart < 0) {
1994 fractionalPart = -fractionalPart;
1995 }
1996 snprintf(&str[5], 5, "%.4d", fractionalPart);
1997
1998 // Do not write the null terminator
1999 write(str, 1, 9);
2000 }
2001
2002 /*
2003 * Geodata is stored according to ISO-6709 standard.
2004 * latitudex10000 is latitude in degrees times 10000, and
2005 * longitudex10000 is longitude in degrees times 10000.
2006 * The range for the latitude is in [-90, +90], and
2007 * The range for the longitude is in [-180, +180]
2008 */
setGeoData(int latitudex10000,int longitudex10000)2009 status_t MPEG4Writer::setGeoData(int latitudex10000, int longitudex10000) {
2010 // Is latitude or longitude out of range?
2011 if (latitudex10000 < -900000 || latitudex10000 > 900000 ||
2012 longitudex10000 < -1800000 || longitudex10000 > 1800000) {
2013 return BAD_VALUE;
2014 }
2015
2016 mLatitudex10000 = latitudex10000;
2017 mLongitudex10000 = longitudex10000;
2018 mAreGeoTagsAvailable = true;
2019 mMoovExtraSize += 30;
2020 return OK;
2021 }
2022
setCaptureRate(float captureFps)2023 status_t MPEG4Writer::setCaptureRate(float captureFps) {
2024 if (captureFps <= 0.0f) {
2025 return BAD_VALUE;
2026 }
2027
2028 // Increase moovExtraSize once only irrespective of how many times
2029 // setCaptureRate is called.
2030 bool containsCaptureFps = mMetaKeys->contains(kMetaKey_CaptureFps);
2031 mMetaKeys->setFloat(kMetaKey_CaptureFps, captureFps);
2032 if (!containsCaptureFps) {
2033 mMoovExtraSize += sizeof(kMetaKey_CaptureFps) + 4 + 32;
2034 }
2035
2036 return OK;
2037 }
2038
setTemporalLayerCount(uint32_t layerCount)2039 status_t MPEG4Writer::setTemporalLayerCount(uint32_t layerCount) {
2040 if (layerCount > 9) {
2041 return BAD_VALUE;
2042 }
2043
2044 if (layerCount > 0) {
2045 mMetaKeys->setInt32(kMetaKey_TemporalLayerCount, layerCount);
2046 mMoovExtraSize += sizeof(kMetaKey_TemporalLayerCount) + 4 + 32;
2047 }
2048
2049 return OK;
2050 }
2051
notifyApproachingLimit()2052 void MPEG4Writer::notifyApproachingLimit() {
2053 Mutex::Autolock autolock(mLock);
2054 // Only notify once.
2055 if (mSendNotify) {
2056 return;
2057 }
2058 ALOGW("Recorded file size is approaching limit %" PRId64 "bytes",
2059 mMaxFileSizeLimitBytes);
2060 notify(MEDIA_RECORDER_EVENT_INFO, MEDIA_RECORDER_INFO_MAX_FILESIZE_APPROACHING, 0);
2061 mSendNotify = true;
2062 }
2063
write(const void * data,size_t size)2064 void MPEG4Writer::write(const void *data, size_t size) {
2065 write(data, 1, size);
2066 }
2067
isFileStreamable() const2068 bool MPEG4Writer::isFileStreamable() const {
2069 return mStreamableFile;
2070 }
2071
preAllocate(uint64_t wantSize)2072 bool MPEG4Writer::preAllocate(uint64_t wantSize) {
2073 if (!mPreAllocationEnabled)
2074 return true;
2075
2076 std::lock_guard<std::mutex> l(mFallocMutex);
2077
2078 if (mFallocateErr == true)
2079 return false;
2080
2081 // approxMOOVHeadersSize has to be changed whenever its needed in the future.
2082 uint64_t approxMOOVHeadersSize = 500;
2083 // approxTrackHeadersSize has to be changed whenever its needed in the future.
2084 const uint64_t approxTrackHeadersSize = 800;
2085
2086 uint64_t approxMOOVBoxSize = 0;
2087 if (mPreAllocFirstTime) {
2088 mPreAllocFirstTime = false;
2089 approxMOOVBoxSize = approxMOOVHeadersSize + mFileLevelMetaDataSize + mMoovExtraSize +
2090 (approxTrackHeadersSize * numTracks());
2091 ALOGV("firstTimeAllocation approxMOOVBoxSize:%" PRIu64, approxMOOVBoxSize);
2092 }
2093
2094 uint64_t allTracksTotalMetaDataSizeEstimate = 0;
2095 for (List<Track *>::iterator it = mTracks.begin(); it != mTracks.end(); ++it) {
2096 allTracksTotalMetaDataSizeEstimate += ((*it)->trackMetaDataSize());
2097 }
2098 ALOGV(" allTracksTotalMetaDataSizeEstimate:%" PRIu64, allTracksTotalMetaDataSizeEstimate);
2099
2100 /* MOOVBoxSize will increase whenever a sample gets written to the file. Enough to allocate
2101 * the delta increase for each sample after the very first allocation.
2102 */
2103 uint64_t approxMetaDataSizeIncrease =
2104 allTracksTotalMetaDataSizeEstimate - mPrevAllTracksTotalMetaDataSizeEstimate;
2105 ALOGV("approxMetaDataSizeIncrease:%" PRIu64 " wantSize:%" PRIu64, approxMetaDataSizeIncrease,
2106 wantSize);
2107 mPrevAllTracksTotalMetaDataSizeEstimate = allTracksTotalMetaDataSizeEstimate;
2108 ALOGV("mPreAllocateFileEndOffset:%" PRIu64 " mOffset:%" PRIu64, mPreAllocateFileEndOffset,
2109 mOffset);
2110 off64_t lastFileEndOffset = std::max(mPreAllocateFileEndOffset, mOffset);
2111 uint64_t preAllocateSize = wantSize + approxMOOVBoxSize + approxMetaDataSizeIncrease;
2112 ALOGV("preAllocateSize :%" PRIu64 " lastFileEndOffset:%" PRIu64, preAllocateSize,
2113 lastFileEndOffset);
2114
2115 int res = fallocate64(mFd, FALLOC_FL_KEEP_SIZE, lastFileEndOffset, preAllocateSize);
2116 if (res == -1) {
2117 ALOGE("fallocate err:%s, %d, fd:%d", strerror(errno), errno, mFd);
2118 sp<AMessage> msg = new AMessage(kWhatFallocateError, mReflector);
2119 msg->setInt32("err", ERROR_IO);
2120 status_t err = msg->post();
2121 mFallocateErr = true;
2122 ALOGD("preAllocation post:%d", err);
2123 } else {
2124 mPreAllocateFileEndOffset = lastFileEndOffset + preAllocateSize;
2125 ALOGV("mPreAllocateFileEndOffset:%" PRIu64, mPreAllocateFileEndOffset);
2126 }
2127 return (res == -1) ? false : true;
2128 }
2129
truncatePreAllocation()2130 bool MPEG4Writer::truncatePreAllocation() {
2131 if (!mPreAllocationEnabled)
2132 return true;
2133
2134 bool status = true;
2135 off64_t endOffset = std::max(mMdatEndOffset, mOffset);
2136 /* if mPreAllocateFileEndOffset >= endOffset, then preallocation logic works good. (diff >= 0).
2137 * Otherwise, the logic needs to be modified.
2138 */
2139 ALOGD("ftruncate mPreAllocateFileEndOffset:%" PRId64 " mOffset:%" PRIu64
2140 " mMdatEndOffset:%" PRIu64 " diff:%" PRId64, mPreAllocateFileEndOffset, mOffset,
2141 mMdatEndOffset, mPreAllocateFileEndOffset - endOffset);
2142 if (ftruncate64(mFd, endOffset) == -1) {
2143 ALOGE("ftruncate err:%s, %d, fd:%d", strerror(errno), errno, mFd);
2144 status = false;
2145 /* No need to post and handle(stop & notify client) error like it's done in preAllocate(),
2146 * because ftruncate() is called during release() only and the error here would be
2147 * reported from there as this function is returning false on any error in ftruncate().
2148 */
2149 }
2150 return status;
2151 }
2152
exceedsFileSizeLimit()2153 bool MPEG4Writer::exceedsFileSizeLimit() {
2154 // No limit
2155 if (mMaxFileSizeLimitBytes == 0) {
2156 return false;
2157 }
2158 int64_t nTotalBytesEstimate = static_cast<int64_t>(mInMemoryCacheSize);
2159 for (List<Track *>::iterator it = mTracks.begin();
2160 it != mTracks.end(); ++it) {
2161 nTotalBytesEstimate += (*it)->getEstimatedTrackSizeBytes();
2162 }
2163
2164 if (!mStreamableFile) {
2165 // Add 1024 bytes as error tolerance
2166 return nTotalBytesEstimate + 1024 >= mMaxFileSizeLimitBytes;
2167 }
2168
2169 // Be conservative in the estimate: do not exceed 95% of
2170 // the target file limit. For small target file size limit, though,
2171 // this will not help.
2172 return (nTotalBytesEstimate >= (95 * mMaxFileSizeLimitBytes) / 100);
2173 }
2174
approachingFileSizeLimit()2175 bool MPEG4Writer::approachingFileSizeLimit() {
2176 // No limit
2177 if (mMaxFileSizeLimitBytes == 0) {
2178 return false;
2179 }
2180
2181 int64_t nTotalBytesEstimate = static_cast<int64_t>(mInMemoryCacheSize);
2182 for (List<Track *>::iterator it = mTracks.begin();
2183 it != mTracks.end(); ++it) {
2184 nTotalBytesEstimate += (*it)->getEstimatedTrackSizeBytes();
2185 }
2186
2187 if (!mStreamableFile) {
2188 // Add 1024 bytes as error tolerance
2189 return nTotalBytesEstimate + 1024 >= (90 * mMaxFileSizeLimitBytes) / 100;
2190 }
2191
2192 return (nTotalBytesEstimate >= (90 * mMaxFileSizeLimitBytes) / 100);
2193 }
2194
exceedsFileDurationLimit()2195 bool MPEG4Writer::exceedsFileDurationLimit() {
2196 // No limit
2197 if (mMaxFileDurationLimitUs == 0) {
2198 return false;
2199 }
2200
2201 for (List<Track *>::iterator it = mTracks.begin();
2202 it != mTracks.end(); ++it) {
2203 if (!(*it)->isHeif() &&
2204 (*it)->getDurationUs() >= mMaxFileDurationLimitUs) {
2205 return true;
2206 }
2207 }
2208 return false;
2209 }
2210
reachedEOS()2211 bool MPEG4Writer::reachedEOS() {
2212 bool allDone = true;
2213 for (List<Track *>::iterator it = mTracks.begin();
2214 it != mTracks.end(); ++it) {
2215 if (!(*it)->reachedEOS()) {
2216 allDone = false;
2217 break;
2218 }
2219 }
2220
2221 return allDone;
2222 }
2223
setStartTimestampUs(int64_t timeUs)2224 void MPEG4Writer::setStartTimestampUs(int64_t timeUs) {
2225 ALOGI("setStartTimestampUs: %" PRId64, timeUs);
2226 CHECK_GE(timeUs, 0LL);
2227 Mutex::Autolock autoLock(mLock);
2228 if (mStartTimestampUs < 0 || mStartTimestampUs > timeUs) {
2229 mStartTimestampUs = timeUs;
2230 ALOGI("Earliest track starting time: %" PRId64, mStartTimestampUs);
2231 }
2232 }
2233
getStartTimestampUs()2234 int64_t MPEG4Writer::getStartTimestampUs() {
2235 Mutex::Autolock autoLock(mLock);
2236 return mStartTimestampUs;
2237 }
2238
2239 /* Returns negative when reordering is needed because of BFrames or zero otherwise.
2240 * CTTS values for tracks with BFrames offsets this negative value.
2241 */
getStartTimeOffsetBFramesUs()2242 int32_t MPEG4Writer::getStartTimeOffsetBFramesUs() {
2243 Mutex::Autolock autoLock(mLock);
2244 return mStartTimeOffsetBFramesUs;
2245 }
2246
numTracks()2247 size_t MPEG4Writer::numTracks() {
2248 Mutex::Autolock autolock(mLock);
2249 return mTracks.size();
2250 }
2251
2252 ////////////////////////////////////////////////////////////////////////////////
2253
Track(MPEG4Writer * owner,const sp<MediaSource> & source,uint32_t aTrackId)2254 MPEG4Writer::Track::Track(MPEG4Writer* owner, const sp<MediaSource>& source, uint32_t aTrackId)
2255 : mOwner(owner),
2256 mMeta(source->getFormat()),
2257 mSource(source),
2258 mDone(false),
2259 mPaused(false),
2260 mResumed(false),
2261 mStarted(false),
2262 mGotStartKeyFrame(false),
2263 mIsMalformed(false),
2264 mTrackId(aTrackId),
2265 mTrackDurationUs(0),
2266 mEstimatedTrackSizeBytes(0),
2267 mSamplesHaveSameSize(true),
2268 mStszTableEntries(new ListTableEntries<uint32_t, 1>(1000)),
2269 mCo64TableEntries(new ListTableEntries<off64_t, 1>(1000)),
2270 mStscTableEntries(new ListTableEntries<uint32_t, 3>(1000)),
2271 mStssTableEntries(new ListTableEntries<uint32_t, 1>(1000)),
2272 mSttsTableEntries(new ListTableEntries<uint32_t, 2>(1000)),
2273 mCttsTableEntries(new ListTableEntries<uint32_t, 2>(1000)),
2274 mElstTableEntries(new ListTableEntries<uint32_t, 3>(3)), // Reserve 3 rows, a row has 3 items
2275 mMinCttsOffsetTimeUs(0),
2276 mMinCttsOffsetTicks(0),
2277 mMaxCttsOffsetTicks(0),
2278 mDoviProfile(0),
2279 mCodecSpecificData(NULL),
2280 mCodecSpecificDataSize(0),
2281 mGotAllCodecSpecificData(false),
2282 mReachedEOS(false),
2283 mStartTimestampUs(-1),
2284 mFirstSampleTimeRealUs(0),
2285 mFirstSampleStartOffsetUs(0),
2286 mRotation(0),
2287 mDimgRefs("dimg"),
2288 mGainmapDimgRefs("dimg"),
2289 mImageItemId(0),
2290 mItemIdBase(0),
2291 mIsPrimary(0),
2292 mWidth(0),
2293 mHeight(0),
2294 mTileWidth(0),
2295 mTileHeight(0),
2296 mGridRows(0),
2297 mGridCols(0),
2298 mNumTiles(1),
2299 mTileIndex(0),
2300 mGainmapItemId(0),
2301 mGainmapMetadataItemId(0),
2302 mColorAspectsValid(false) {
2303 getCodecSpecificDataFromInputFormatIfPossible();
2304
2305 const char *mime;
2306 mMeta->findCString(kKeyMIMEType, &mime);
2307 mIsAvc = !strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_AVC);
2308 mIsHevc = !strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_HEVC);
2309 mIsAv1 = !strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_AV1);
2310 mIsApv = editing_flags::muxer_mp4_enable_apv() && !strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_APV);
2311 mIsDovi = !strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_DOLBY_VISION);
2312 mIsAudio = !strncasecmp(mime, "audio/", 6);
2313 mIsVideo = !strncasecmp(mime, "video/", 6);
2314 mIsHeic = !strcasecmp(mime, MEDIA_MIMETYPE_IMAGE_ANDROID_HEIC);
2315 mIsAvif = !strcasecmp(mime, MEDIA_MIMETYPE_IMAGE_AVIF);
2316 mIsHeif = mIsHeic || mIsAvif;
2317 mIsMPEG4 = !strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_MPEG4) ||
2318 !strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_AAC);
2319
2320 // store temporal layer count
2321 if (mIsVideo) {
2322 int32_t count;
2323 if (mMeta->findInt32(kKeyTemporalLayerCount, &count) && count > 1) {
2324 mOwner->setTemporalLayerCount(count);
2325 }
2326 }
2327
2328 if (!mIsHeif) {
2329 setTimeScale();
2330 } else {
2331 CHECK(mMeta->findInt32(kKeyWidth, &mWidth) && (mWidth > 0));
2332 CHECK(mMeta->findInt32(kKeyHeight, &mHeight) && (mHeight > 0));
2333
2334 int32_t tileWidth, tileHeight, gridRows, gridCols;
2335 if (mMeta->findInt32(kKeyTileWidth, &tileWidth) && (tileWidth > 0) &&
2336 mMeta->findInt32(kKeyTileHeight, &tileHeight) && (tileHeight > 0) &&
2337 mMeta->findInt32(kKeyGridRows, &gridRows) && (gridRows > 0) &&
2338 mMeta->findInt32(kKeyGridCols, &gridCols) && (gridCols > 0)) {
2339 mTileWidth = tileWidth;
2340 mTileHeight = tileHeight;
2341 mGridRows = gridRows;
2342 mGridCols = gridCols;
2343 mNumTiles = gridRows * gridCols;
2344 }
2345 if (!mMeta->findInt32(kKeyTrackIsDefault, &mIsPrimary)) {
2346 mIsPrimary = false;
2347 }
2348 }
2349 }
2350
2351 // Clear all the internal states except the CSD data.
resetInternal()2352 void MPEG4Writer::Track::resetInternal() {
2353 mDone = false;
2354 mPaused = false;
2355 mResumed = false;
2356 mStarted = false;
2357 mGotStartKeyFrame = false;
2358 mIsMalformed = false;
2359 mTrackDurationUs = 0;
2360 mEstimatedTrackSizeBytes = 0;
2361 mSamplesHaveSameSize = false;
2362 if (mStszTableEntries != NULL) {
2363 delete mStszTableEntries;
2364 mStszTableEntries = new ListTableEntries<uint32_t, 1>(1000);
2365 }
2366 if (mCo64TableEntries != NULL) {
2367 delete mCo64TableEntries;
2368 mCo64TableEntries = new ListTableEntries<off64_t, 1>(1000);
2369 }
2370 if (mStscTableEntries != NULL) {
2371 delete mStscTableEntries;
2372 mStscTableEntries = new ListTableEntries<uint32_t, 3>(1000);
2373 }
2374 if (mStssTableEntries != NULL) {
2375 delete mStssTableEntries;
2376 mStssTableEntries = new ListTableEntries<uint32_t, 1>(1000);
2377 }
2378 if (mSttsTableEntries != NULL) {
2379 delete mSttsTableEntries;
2380 mSttsTableEntries = new ListTableEntries<uint32_t, 2>(1000);
2381 }
2382 if (mCttsTableEntries != NULL) {
2383 delete mCttsTableEntries;
2384 mCttsTableEntries = new ListTableEntries<uint32_t, 2>(1000);
2385 }
2386 if (mElstTableEntries != NULL) {
2387 delete mElstTableEntries;
2388 mElstTableEntries = new ListTableEntries<uint32_t, 3>(3);
2389 }
2390 mReachedEOS = false;
2391 }
2392
trackMetaDataSize()2393 int64_t MPEG4Writer::Track::trackMetaDataSize() {
2394 int64_t co64BoxSizeBytes = mCo64TableEntries->count() * 8;
2395 int64_t stszBoxSizeBytes = mStszTableEntries->count() * 4;
2396 int64_t trackMetaDataSize = mStscTableEntries->count() * 12 + // stsc box size
2397 mStssTableEntries->count() * 4 + // stss box size
2398 mSttsTableEntries->count() * 8 + // stts box size
2399 mCttsTableEntries->count() * 8 + // ctts box size
2400 mElstTableEntries->count() * 12 + // elst box size
2401 co64BoxSizeBytes + // stco box size
2402 stszBoxSizeBytes; // stsz box size
2403 return trackMetaDataSize;
2404 }
2405
2406
updateTrackSizeEstimate()2407 void MPEG4Writer::Track::updateTrackSizeEstimate() {
2408 mEstimatedTrackSizeBytes = mMdatSizeBytes; // media data size
2409 if (!isHeif() && !mOwner->isFileStreamable()) {
2410 mEstimatedTrackSizeBytes += trackMetaDataSize();
2411 }
2412 }
2413
addOneStscTableEntry(size_t chunkId,size_t sampleId)2414 void MPEG4Writer::Track::addOneStscTableEntry(
2415 size_t chunkId, size_t sampleId) {
2416 mStscTableEntries->add(htonl(chunkId));
2417 mStscTableEntries->add(htonl(sampleId));
2418 mStscTableEntries->add(htonl(1));
2419 }
2420
addOneStssTableEntry(size_t sampleId)2421 void MPEG4Writer::Track::addOneStssTableEntry(size_t sampleId) {
2422 mStssTableEntries->add(htonl(sampleId));
2423 }
2424
addOneSttsTableEntry(size_t sampleCount,int32_t delta)2425 void MPEG4Writer::Track::addOneSttsTableEntry(size_t sampleCount, int32_t delta) {
2426 if (delta == 0) {
2427 ALOGW("0-duration samples found: %zu", sampleCount);
2428 }
2429 mSttsTableEntries->add(htonl(sampleCount));
2430 mSttsTableEntries->add(htonl(delta));
2431 }
2432
addOneCttsTableEntry(size_t sampleCount,int32_t sampleOffset)2433 void MPEG4Writer::Track::addOneCttsTableEntry(size_t sampleCount, int32_t sampleOffset) {
2434 if (!mIsVideo) {
2435 return;
2436 }
2437 mCttsTableEntries->add(htonl(sampleCount));
2438 mCttsTableEntries->add(htonl(sampleOffset));
2439 }
2440
addOneElstTableEntry(uint32_t segmentDuration,int32_t mediaTime,int16_t mediaRate,int16_t mediaRateFraction)2441 void MPEG4Writer::Track::addOneElstTableEntry(
2442 uint32_t segmentDuration, int32_t mediaTime, int16_t mediaRate, int16_t mediaRateFraction) {
2443 ALOGV("segmentDuration:%u, mediaTime:%d", segmentDuration, mediaTime);
2444 ALOGV("mediaRate :%" PRId16 ", mediaRateFraction :%" PRId16 ", Ored %u", mediaRate,
2445 mediaRateFraction, ((((uint32_t)mediaRate) << 16) | ((uint32_t)mediaRateFraction)));
2446 mElstTableEntries->add(htonl(segmentDuration));
2447 mElstTableEntries->add(htonl(mediaTime));
2448 mElstTableEntries->add(htonl((((uint32_t)mediaRate) << 16) | (uint32_t)mediaRateFraction));
2449 }
2450
setupAndStartLooper()2451 status_t MPEG4Writer::setupAndStartLooper() {
2452 status_t err = OK;
2453 if (mLooper == nullptr) {
2454 mLooper = new ALooper;
2455 mLooper->setName("MP4WtrCtrlHlpLooper");
2456 if (mIsBackgroundMode) {
2457 err = mLooper->start(false, false, ANDROID_PRIORITY_BACKGROUND);
2458 } else {
2459 err = mLooper->start();
2460 }
2461 mReflector = new AHandlerReflector<MPEG4Writer>(this);
2462 mLooper->registerHandler(mReflector);
2463 }
2464 ALOGD("MP4WtrCtrlHlpLooper Started");
2465 return err;
2466 }
2467
stopAndReleaseLooper()2468 void MPEG4Writer::stopAndReleaseLooper() {
2469 if (mLooper != nullptr) {
2470 if (mReflector != nullptr) {
2471 mLooper->unregisterHandler(mReflector->id());
2472 mReflector.clear();
2473 }
2474 mLooper->stop();
2475 mLooper.clear();
2476 ALOGD("MP4WtrCtrlHlpLooper stopped");
2477 }
2478 }
2479
setNextFd(int fd)2480 status_t MPEG4Writer::setNextFd(int fd) {
2481 Mutex::Autolock l(mLock);
2482 if (mNextFd != -1) {
2483 // No need to set a new FD yet.
2484 return INVALID_OPERATION;
2485 }
2486 mNextFd = dup(fd);
2487 mFdCond.signal();
2488 return OK;
2489 }
2490
isGainmapMetaData(MediaBufferBase * buffer,uint32_t * offset) const2491 bool MPEG4Writer::Track::isGainmapMetaData(MediaBufferBase* buffer, uint32_t* offset) const {
2492 if (!mIsHeif) {
2493 return false;
2494 }
2495
2496 // Gainmap metadata block starting with 'tmap\0\0'
2497 size_t length = buffer->range_length();
2498 uint8_t *data = (uint8_t *)buffer->data() + buffer->range_offset();
2499 if ((length > sizeof(kGainmapMetaHeader)) &&
2500 !memcmp(data, kGainmapMetaHeader, sizeof(kGainmapMetaHeader))) {
2501 *offset = sizeof(kGainmapMetaHeader);
2502 return true;
2503 }
2504
2505 return false;
2506 }
2507
isGainmapData(MediaBufferBase * buffer,uint32_t * offset) const2508 bool MPEG4Writer::Track::isGainmapData(MediaBufferBase* buffer, uint32_t* offset) const {
2509 if (!mIsHeif) {
2510 return false;
2511 }
2512
2513 // Gainmap block starting with 'gmap\0\0'
2514 size_t length = buffer->range_length();
2515 uint8_t* data = (uint8_t*)buffer->data() + buffer->range_offset();
2516 if ((length > sizeof(kGainmapHeader)) &&
2517 !memcmp(data, kGainmapHeader, sizeof(kGainmapHeader))) {
2518 *offset = sizeof(kGainmapHeader);
2519 return true;
2520 }
2521
2522 return false;
2523 }
2524
isExifData(MediaBufferBase * buffer,uint32_t * tiffHdrOffset) const2525 bool MPEG4Writer::Track::isExifData(MediaBufferBase* buffer, uint32_t* tiffHdrOffset) const {
2526 if (!mIsHeif) {
2527 return false;
2528 }
2529
2530 // Exif block starting with 'Exif\0\0'
2531 size_t length = buffer->range_length();
2532 uint8_t* data = (uint8_t*)buffer->data() + buffer->range_offset();
2533 if ((length > sizeof(kExifHeader)) && !memcmp(data, kExifHeader, sizeof(kExifHeader))) {
2534 *tiffHdrOffset = sizeof(kExifHeader);
2535 return true;
2536 }
2537
2538 // Exif block starting with fourcc 'Exif' followed by APP1 marker
2539 if ((length > sizeof(kExifApp1Marker) + 2 + sizeof(kExifHeader)) &&
2540 !memcmp(data, kExifApp1Marker, sizeof(kExifApp1Marker)) &&
2541 !memcmp(data + sizeof(kExifApp1Marker) + 2, kExifHeader, sizeof(kExifHeader))) {
2542 // skip 'Exif' fourcc
2543 buffer->set_range(4, buffer->range_length() - 4);
2544
2545 // 2-byte APP1 + 2-byte size followed by kExifHeader
2546 *tiffHdrOffset = 2 + 2 + sizeof(kExifHeader);
2547 return true;
2548 }
2549
2550 return false;
2551 }
2552
addChunkOffset(off64_t offset)2553 void MPEG4Writer::Track::addChunkOffset(off64_t offset) {
2554 CHECK(!mIsHeif);
2555 mCo64TableEntries->add(hton64(offset));
2556 }
2557
addItemOffsetAndSize(off64_t offset,size_t size,bool isExif,bool isGainmapMeta,bool isGainmap)2558 void MPEG4Writer::Track::addItemOffsetAndSize(off64_t offset, size_t size, bool isExif,
2559 bool isGainmapMeta, bool isGainmap) {
2560 CHECK(mIsHeif);
2561
2562 if (offset > UINT32_MAX || size > UINT32_MAX) {
2563 ALOGE("offset or size is out of range: %lld, %lld",
2564 (long long) offset, (long long) size);
2565 mIsMalformed = true;
2566 }
2567 if (mIsMalformed) {
2568 return;
2569 }
2570
2571 if (isExif) {
2572 uint16_t exifItemId;
2573 if (mOwner->reserveItemId_l(1, &exifItemId) != OK) {
2574 return;
2575 }
2576
2577 mExifList.push_back(mOwner->addItem_l({
2578 .itemType = "Exif",
2579 .itemId = exifItemId,
2580 .isPrimary = false,
2581 .isHidden = false,
2582 .offset = (uint32_t)offset,
2583 .size = (uint32_t)size,
2584 }));
2585 return;
2586 }
2587
2588 bool hasGrid = (mTileWidth > 0);
2589
2590 if (isGainmapMeta && flags_camera::camera_heif_gainmap()) {
2591 uint16_t metaItemId;
2592 if (mOwner->reserveItemId_l(1, &metaItemId) != OK) {
2593 return;
2594 }
2595
2596 Vector<uint16_t> props;
2597 if (mColorAspectsValid) {
2598 ItemProperty property;
2599 property.type = FOURCC('c', 'o', 'l', 'r');
2600 ColorUtils::convertCodecColorAspectsToIsoAspects(
2601 mColorAspects, &property.colorPrimaries, &property.colorTransfer,
2602 &property.colorMatrix, &property.colorRange);
2603 props.push_back(mOwner->addProperty_l(property));
2604 }
2605 if (!mBitsPerChannel.empty()) {
2606 ItemProperty property;
2607 property.type = FOURCC('p', 'i', 'x', 'i');
2608 property.bitsPerChannel.appendVector(mBitsPerChannel);
2609 props.push_back(mOwner->addProperty_l(property));
2610 }
2611 props.push_back(mOwner->addProperty_l({
2612 .type = FOURCC('i', 's', 'p', 'e'),
2613 .width = hasGrid ? mTileWidth : mWidth,
2614 .height = hasGrid ? mTileHeight : mHeight,
2615 }));
2616 mGainmapMetadataItemId = mOwner->addItem_l({
2617 .itemType = "tmap",
2618 .itemId = metaItemId,
2619 .isPrimary = false,
2620 .isHidden = false,
2621 .offset = (uint32_t)offset,
2622 .size = (uint32_t)size,
2623 .properties = props,
2624 });
2625 return;
2626 }
2627
2628 if (mTileIndex >= mNumTiles) {
2629 ALOGW("Ignoring excess tiles!");
2630 return;
2631 }
2632
2633 // Rotation angle in HEIF is CCW, framework angle is CW.
2634 int32_t heifRotation = 0;
2635 switch(mRotation) {
2636 case 90: heifRotation = 3; break;
2637 case 180: heifRotation = 2; break;
2638 case 270: heifRotation = 1; break;
2639 default: break; // don't set if invalid
2640 }
2641
2642 if (mProperties.empty()) {
2643 mProperties.push_back(mOwner->addProperty_l({
2644 .type = static_cast<uint32_t>(mIsAvif ?
2645 FOURCC('a', 'v', '1', 'C') :
2646 FOURCC('h', 'v', 'c', 'C')),
2647 .data = ABuffer::CreateAsCopy(mCodecSpecificData, mCodecSpecificDataSize)
2648 }));
2649
2650 mProperties.push_back(mOwner->addProperty_l({
2651 .type = FOURCC('i', 's', 'p', 'e'),
2652 .width = hasGrid ? mTileWidth : mWidth,
2653 .height = hasGrid ? mTileHeight : mHeight,
2654 }));
2655
2656 if (!hasGrid && heifRotation > 0) {
2657 mProperties.push_back(mOwner->addProperty_l({
2658 .type = FOURCC('i', 'r', 'o', 't'),
2659 .rotation = heifRotation,
2660 }));
2661 }
2662 }
2663
2664 mTileIndex++;
2665 if (hasGrid) {
2666 uint16_t id = mOwner->addItem_l({
2667 .itemType = mIsAvif ? "av01" : "hvc1",
2668 .itemId = mItemIdBase++,
2669 .isPrimary = false,
2670 .isHidden = true,
2671 .offset = (uint32_t)offset,
2672 .size = (uint32_t)size,
2673 .properties = mProperties,
2674 });
2675 if (isGainmap && flags_camera::camera_heif_gainmap()) {
2676 mGainmapDimgRefs.value.push_back(id);
2677 } else {
2678 mDimgRefs.value.push_back(id);
2679 }
2680
2681 if (mTileIndex == mNumTiles) {
2682 mProperties.clear();
2683 mProperties.push_back(mOwner->addProperty_l({
2684 .type = FOURCC('i', 's', 'p', 'e'),
2685 .width = mWidth,
2686 .height = mHeight,
2687 }));
2688 if (heifRotation > 0) {
2689 mProperties.push_back(mOwner->addProperty_l({
2690 .type = FOURCC('i', 'r', 'o', 't'),
2691 .rotation = heifRotation,
2692 }));
2693 }
2694 if (mColorAspectsValid && flags_camera::camera_heif_gainmap()) {
2695 ItemProperty property;
2696 property.type = FOURCC('c', 'o', 'l', 'r');
2697 ColorUtils::convertCodecColorAspectsToIsoAspects(
2698 mColorAspects, &property.colorPrimaries, &property.colorTransfer,
2699 &property.colorMatrix, &property.colorRange);
2700 mProperties.push_back(mOwner->addProperty_l(property));
2701 }
2702 if (!mBitsPerChannel.empty() && flags_camera::camera_heif_gainmap()) {
2703 ItemProperty property;
2704 property.type = FOURCC('p', 'i', 'x', 'i');
2705 property.bitsPerChannel.appendVector(mBitsPerChannel);
2706 mProperties.push_back(mOwner->addProperty_l(property));
2707 }
2708 uint16_t itemId = mOwner->addItem_l({
2709 .itemType = "grid",
2710 .itemId = mItemIdBase++,
2711 .isPrimary = isGainmap && flags_camera::camera_heif_gainmap()
2712 ? false
2713 : (mIsPrimary != 0),
2714 .isHidden = false,
2715 .rows = (uint32_t)mGridRows,
2716 .cols = (uint32_t)mGridCols,
2717 .width = (uint32_t)mWidth,
2718 .height = (uint32_t)mHeight,
2719 .properties = mProperties,
2720 });
2721
2722 if (isGainmap && flags_camera::camera_heif_gainmap()) {
2723 mGainmapItemId = itemId;
2724 } else {
2725 mImageItemId = itemId;
2726 }
2727 }
2728 } else {
2729 if (mColorAspectsValid && flags_camera::camera_heif_gainmap()) {
2730 ItemProperty property;
2731 property.type = FOURCC('c', 'o', 'l', 'r');
2732 ColorUtils::convertCodecColorAspectsToIsoAspects(
2733 mColorAspects, &property.colorPrimaries, &property.colorTransfer,
2734 &property.colorMatrix, &property.colorRange);
2735 mProperties.push_back(mOwner->addProperty_l(property));
2736 }
2737 if (!mBitsPerChannel.empty() && flags_camera::camera_heif_gainmap()) {
2738 ItemProperty property;
2739 property.type = FOURCC('p', 'i', 'x', 'i');
2740 property.bitsPerChannel.appendVector(mBitsPerChannel);
2741 mProperties.push_back(mOwner->addProperty_l(property));
2742 }
2743 uint16_t itemId = mOwner->addItem_l({
2744 .itemType = mIsAvif ? "av01" : "hvc1",
2745 .itemId = mItemIdBase++,
2746 .isPrimary = (isGainmap && flags_camera::camera_heif_gainmap()) ? false
2747 : (mIsPrimary != 0),
2748 .isHidden = false,
2749 .offset = (uint32_t)offset,
2750 .size = (uint32_t)size,
2751 .properties = mProperties,
2752 });
2753
2754 if (isGainmap && flags_camera::camera_heif_gainmap()) {
2755 mGainmapItemId = itemId;
2756 } else {
2757 mImageItemId = itemId;
2758 }
2759 }
2760 }
2761
2762 // Flush out the item refs for this track. Note that it must be called after the
2763 // writer thread has stopped, because there might be pending items in the last
2764 // few chunks written by the writer thread (as opposed to the track). In particular,
2765 // it affects the 'dimg' refs for tiled image, as we only have the refs after the
2766 // last tile sample is written.
flushItemRefs()2767 void MPEG4Writer::Track::flushItemRefs() {
2768 CHECK(mIsHeif);
2769
2770 if (mImageItemId > 0) {
2771 mOwner->addRefs_l(mImageItemId, mDimgRefs);
2772
2773 if (!mExifList.empty()) {
2774 // The "cdsc" ref is from the metadata/exif item to the image item.
2775 // So the refs all contain the image item.
2776 ItemRefs cdscRefs("cdsc");
2777 cdscRefs.value.push_back(mImageItemId);
2778 for (uint16_t exifItem : mExifList) {
2779 mOwner->addRefs_l(exifItem, cdscRefs);
2780 }
2781 }
2782 }
2783
2784 if ((mGainmapItemId > 0) && flags_camera::camera_heif_gainmap()) {
2785 mOwner->addRefs_l(mGainmapItemId, mGainmapDimgRefs);
2786 }
2787 }
2788
setTimeScale()2789 void MPEG4Writer::Track::setTimeScale() {
2790 ALOGV("setTimeScale");
2791 // Default time scale
2792 mTimeScale = 90000;
2793
2794 if (mIsAudio) {
2795 // Use the sampling rate as the default time scale for audio track.
2796 int32_t sampleRate;
2797 bool success = mMeta->findInt32(kKeySampleRate, &sampleRate);
2798 CHECK(success);
2799 mTimeScale = sampleRate;
2800 }
2801
2802 // If someone would like to overwrite the timescale, use user-supplied value.
2803 int32_t timeScale;
2804 if (mMeta->findInt32(kKeyTimeScale, &timeScale)) {
2805 mTimeScale = timeScale;
2806 }
2807
2808 CHECK_GT(mTimeScale, 0);
2809 }
2810
onMessageReceived(const sp<AMessage> & msg)2811 void MPEG4Writer::onMessageReceived(const sp<AMessage> &msg) {
2812 switch (msg->what()) {
2813 case kWhatSwitch:
2814 {
2815 mLock.lock();
2816 int fd = mNextFd;
2817 mNextFd = -1;
2818 mLock.unlock();
2819 if (finishCurrentSession() == OK) {
2820 initInternal(fd, false /*isFirstSession*/);
2821 status_t status = start(mStartMeta.get());
2822 mSwitchPending = false;
2823 if (status == OK) {
2824 notify(MEDIA_RECORDER_EVENT_INFO,
2825 MEDIA_RECORDER_INFO_NEXT_OUTPUT_FILE_STARTED, 0);
2826 }
2827 }
2828 break;
2829 }
2830 /* ::write() or lseek64() wasn't a success, file could be malformed.
2831 * Or fallocate() failed. reset() and notify client on both the cases.
2832 */
2833 case kWhatFallocateError: // fallthrough
2834 case kWhatIOError: {
2835 int32_t err;
2836 CHECK(msg->findInt32("err", &err));
2837 // If reset already in process, don't wait for it complete to avoid deadlock.
2838 reset(true, false);
2839 //TODO: new MEDIA_RECORDER_ERROR_**** instead MEDIA_RECORDER_ERROR_UNKNOWN ?
2840 notify(MEDIA_RECORDER_EVENT_ERROR, MEDIA_RECORDER_ERROR_UNKNOWN, err);
2841 break;
2842 }
2843 /* Response to kWhatNoIOErrorSoFar would be OK always as of now.
2844 * Responding with other options could be added later if required.
2845 */
2846 case kWhatNoIOErrorSoFar: {
2847 ALOGV("kWhatNoIOErrorSoFar");
2848 sp<AMessage> response = new AMessage;
2849 response->setInt32("err", OK);
2850 sp<AReplyToken> replyID;
2851 CHECK(msg->senderAwaitsResponse(&replyID));
2852 response->postReply(replyID);
2853 break;
2854 }
2855 default:
2856 TRESPASS();
2857 }
2858 }
2859
getCodecSpecificDataFromInputFormatIfPossible()2860 void MPEG4Writer::Track::getCodecSpecificDataFromInputFormatIfPossible() {
2861 const char *mime;
2862
2863 CHECK(mMeta->findCString(kKeyMIMEType, &mime));
2864
2865 uint32_t type;
2866 const void *data = NULL;
2867 size_t size = 0;
2868 if (!strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_AVC)) {
2869 mMeta->findData(kKeyAVCC, &type, &data, &size);
2870 } else if (!strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_HEVC) ||
2871 !strcasecmp(mime, MEDIA_MIMETYPE_IMAGE_ANDROID_HEIC)) {
2872 mMeta->findData(kKeyHVCC, &type, &data, &size);
2873 } else if (!strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_AV1) ||
2874 !strcasecmp(mime, MEDIA_MIMETYPE_IMAGE_AVIF)) {
2875 mMeta->findData(kKeyAV1C, &type, &data, &size);
2876 } else if (editing_flags::muxer_mp4_enable_apv() &&
2877 !strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_APV)) {
2878 mMeta->findData(kKeyAPVC, &type, &data, &size);
2879 } else if (!strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_DOLBY_VISION)) {
2880 getDolbyVisionProfile();
2881 if (!mMeta->findData(kKeyAVCC, &type, &data, &size) &&
2882 !mMeta->findData(kKeyHVCC, &type, &data, &size)) {
2883 ALOGE("Failed: No HVCC/AVCC for Dolby Vision ..\n");
2884 return;
2885 }
2886 } else if (!strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_MPEG4) ||
2887 !strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_AAC)) {
2888 if (mMeta->findData(kKeyESDS, &type, &data, &size)) {
2889 ESDS esds(data, size);
2890 if (esds.getCodecSpecificInfo(&data, &size) == OK &&
2891 data != NULL &&
2892 copyCodecSpecificData((uint8_t*)data, size) == OK) {
2893 mGotAllCodecSpecificData = true;
2894 }
2895 return;
2896 }
2897 }
2898 if (data != NULL && copyCodecSpecificData((uint8_t *)data, size) == OK) {
2899 mGotAllCodecSpecificData = true;
2900 }
2901 }
2902
~Track()2903 MPEG4Writer::Track::~Track() {
2904 stop();
2905
2906 delete mStszTableEntries;
2907 delete mCo64TableEntries;
2908 delete mStscTableEntries;
2909 delete mSttsTableEntries;
2910 delete mStssTableEntries;
2911 delete mCttsTableEntries;
2912 delete mElstTableEntries;
2913
2914 mStszTableEntries = NULL;
2915 mCo64TableEntries = NULL;
2916 mStscTableEntries = NULL;
2917 mSttsTableEntries = NULL;
2918 mStssTableEntries = NULL;
2919 mCttsTableEntries = NULL;
2920 mElstTableEntries = NULL;
2921
2922 if (mCodecSpecificData != NULL) {
2923 free(mCodecSpecificData);
2924 mCodecSpecificData = NULL;
2925 }
2926
2927 }
2928
initTrackingProgressStatus(MetaData * params)2929 void MPEG4Writer::Track::initTrackingProgressStatus(MetaData *params) {
2930 ALOGV("initTrackingProgressStatus");
2931 mPreviousTrackTimeUs = -1;
2932 mTrackingProgressStatus = false;
2933 mTrackEveryTimeDurationUs = 0;
2934 {
2935 int64_t timeUs;
2936 if (params && params->findInt64(kKeyTrackTimeStatus, &timeUs)) {
2937 ALOGV("Receive request to track progress status for every %" PRId64 " us", timeUs);
2938 mTrackEveryTimeDurationUs = timeUs;
2939 mTrackingProgressStatus = true;
2940 }
2941 }
2942 }
2943
2944 // static
ThreadWrapper(void * me)2945 void *MPEG4Writer::ThreadWrapper(void *me) {
2946 ALOGV("ThreadWrapper: %p", me);
2947 MPEG4Writer *writer = static_cast<MPEG4Writer *>(me);
2948 writer->threadFunc();
2949 return NULL;
2950 }
2951
bufferChunk(const Chunk & chunk)2952 void MPEG4Writer::bufferChunk(const Chunk& chunk) {
2953 ALOGV("bufferChunk: %p", chunk.mTrack);
2954 Mutex::Autolock autolock(mLock);
2955 CHECK_EQ(mDone, false);
2956
2957 for (List<ChunkInfo>::iterator it = mChunkInfos.begin();
2958 it != mChunkInfos.end(); ++it) {
2959
2960 if (chunk.mTrack == it->mTrack) { // Found owner
2961 it->mChunks.push_back(chunk);
2962 mChunkReadyCondition.signal();
2963 return;
2964 }
2965 }
2966
2967 CHECK(!"Received a chunk for a unknown track");
2968 }
2969
writeChunkToFile(Chunk * chunk)2970 void MPEG4Writer::writeChunkToFile(Chunk* chunk) {
2971 ALOGV("writeChunkToFile: %" PRId64 " from %s track",
2972 chunk->mTimeStampUs, chunk->mTrack->getTrackType());
2973
2974 int32_t isFirstSample = true;
2975 while (!chunk->mSamples.empty()) {
2976 List<MediaBuffer *>::iterator it = chunk->mSamples.begin();
2977
2978 uint32_t tiffHdrOffset;
2979 if (!(*it)->meta_data().findInt32(
2980 kKeyExifTiffOffset, (int32_t*)&tiffHdrOffset)) {
2981 tiffHdrOffset = 0;
2982 }
2983 bool isExif = (tiffHdrOffset > 0);
2984 bool usePrefix = chunk->mTrack->usePrefix() && !isExif;
2985
2986 size_t bytesWritten;
2987 off64_t offset = addSample_l(*it, usePrefix, tiffHdrOffset, &bytesWritten);
2988
2989 if (chunk->mTrack->isHeif()) {
2990 chunk->mTrack->addItemOffsetAndSize(offset, bytesWritten, isExif);
2991 } else if (isFirstSample) {
2992 chunk->mTrack->addChunkOffset(offset);
2993 isFirstSample = false;
2994 }
2995
2996 (*it)->release();
2997 (*it) = NULL;
2998 chunk->mSamples.erase(it);
2999 }
3000 chunk->mSamples.clear();
3001 }
3002
writeAllChunks()3003 void MPEG4Writer::writeAllChunks() {
3004 ALOGV("writeAllChunks");
3005 size_t outstandingChunks = 0;
3006 Chunk chunk;
3007 while (findChunkToWrite(&chunk)) {
3008 writeChunkToFile(&chunk);
3009 ++outstandingChunks;
3010 }
3011
3012 sendSessionSummary();
3013
3014 mChunkInfos.clear();
3015 ALOGD("%zu chunks are written in the last batch", outstandingChunks);
3016 }
3017
findChunkToWrite(Chunk * chunk)3018 bool MPEG4Writer::findChunkToWrite(Chunk *chunk) {
3019 ALOGV("findChunkToWrite");
3020
3021 int64_t minTimestampUs = 0x7FFFFFFFFFFFFFFFLL;
3022 Track *track = NULL;
3023 for (List<ChunkInfo>::iterator it = mChunkInfos.begin();
3024 it != mChunkInfos.end(); ++it) {
3025 if (!it->mChunks.empty()) {
3026 List<Chunk>::iterator chunkIt = it->mChunks.begin();
3027 if (chunkIt->mTimeStampUs < minTimestampUs) {
3028 minTimestampUs = chunkIt->mTimeStampUs;
3029 track = it->mTrack;
3030 }
3031 }
3032 }
3033
3034 if (track == NULL) {
3035 ALOGV("Nothing to be written after all");
3036 return false;
3037 }
3038
3039 if (mIsFirstChunk) {
3040 mIsFirstChunk = false;
3041 }
3042
3043 for (List<ChunkInfo>::iterator it = mChunkInfos.begin();
3044 it != mChunkInfos.end(); ++it) {
3045 if (it->mTrack == track) {
3046 *chunk = *(it->mChunks.begin());
3047 it->mChunks.erase(it->mChunks.begin());
3048 CHECK_EQ(chunk->mTrack, track);
3049
3050 int64_t interChunkTimeUs =
3051 chunk->mTimeStampUs - it->mPrevChunkTimestampUs;
3052 if (interChunkTimeUs > it->mPrevChunkTimestampUs) {
3053 it->mMaxInterChunkDurUs = interChunkTimeUs;
3054 }
3055 return true;
3056 }
3057 }
3058
3059 return false;
3060 }
3061
threadFunc()3062 void MPEG4Writer::threadFunc() {
3063 ALOGV("threadFunc");
3064
3065 prctl(PR_SET_NAME, (unsigned long)"MPEG4Writer", 0, 0, 0);
3066
3067 if (mIsBackgroundMode) {
3068 // Background priority for media transcoding.
3069 androidSetThreadPriority(0 /* tid (0 = current) */, ANDROID_PRIORITY_BACKGROUND);
3070 }
3071
3072 Mutex::Autolock autoLock(mLock);
3073 while (!mDone) {
3074 Chunk chunk;
3075 bool chunkFound = false;
3076
3077 while (!mDone && !(chunkFound = findChunkToWrite(&chunk))) {
3078 mChunkReadyCondition.wait(mLock);
3079 }
3080
3081 // In real time recording mode, write without holding the lock in order
3082 // to reduce the blocking time for media track threads.
3083 // Otherwise, hold the lock until the existing chunks get written to the
3084 // file.
3085 if (chunkFound) {
3086 if (mIsRealTimeRecording) {
3087 mLock.unlock();
3088 }
3089 writeChunkToFile(&chunk);
3090 if (mIsRealTimeRecording) {
3091 mLock.lock();
3092 }
3093 }
3094 }
3095
3096 writeAllChunks();
3097 ALOGV("threadFunc mOffset:%lld, mMaxOffsetAppend:%lld", (long long)mOffset,
3098 (long long)mMaxOffsetAppend);
3099 mOffset = std::max(mOffset, mMaxOffsetAppend);
3100 }
3101
startWriterThread()3102 status_t MPEG4Writer::startWriterThread() {
3103 ALOGV("startWriterThread");
3104
3105 mDone = false;
3106 mIsFirstChunk = true;
3107 mDriftTimeUs = 0;
3108 for (List<Track *>::iterator it = mTracks.begin();
3109 it != mTracks.end(); ++it) {
3110 ChunkInfo info;
3111 info.mTrack = *it;
3112 info.mPrevChunkTimestampUs = 0;
3113 info.mMaxInterChunkDurUs = 0;
3114 mChunkInfos.push_back(info);
3115 }
3116
3117 pthread_attr_t attr;
3118 pthread_attr_init(&attr);
3119 pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE);
3120 pthread_create(&mThread, &attr, ThreadWrapper, this);
3121 pthread_attr_destroy(&attr);
3122 mWriterThreadStarted = true;
3123 return OK;
3124 }
3125
3126
start(MetaData * params)3127 status_t MPEG4Writer::Track::start(MetaData *params) {
3128 if (!mDone && mPaused) {
3129 mPaused = false;
3130 mResumed = true;
3131 return OK;
3132 }
3133
3134 int64_t startTimeUs;
3135 if (params == NULL || !params->findInt64(kKeyTime, &startTimeUs)) {
3136 startTimeUs = 0;
3137 }
3138 mStartTimeRealUs = startTimeUs;
3139
3140 int32_t rotationDegrees;
3141 if ((mIsVideo || mIsHeif) && params &&
3142 params->findInt32(kKeyRotation, &rotationDegrees)) {
3143 mRotation = rotationDegrees;
3144 }
3145 if (mIsHeif) {
3146 // Reserve the item ids, so that the item ids are ordered in the same
3147 // order that the image tracks are added.
3148 // If we leave the item ids to be assigned when the sample is written out,
3149 // the original track order may not be preserved, if two image tracks
3150 // have data around the same time. (This could happen especially when
3151 // we're encoding with single tile.) The reordering may be undesirable,
3152 // even if the file is well-formed and the primary picture is correct.
3153
3154 // Reserve item ids for samples + grid
3155 size_t numItemsToReserve = mNumTiles + (mNumTiles > 0);
3156 status_t err = mOwner->reserveItemId_l(numItemsToReserve, &mItemIdBase);
3157 if (err != OK) {
3158 return err;
3159 }
3160 }
3161
3162 initTrackingProgressStatus(params);
3163
3164 sp<MetaData> meta = new MetaData;
3165 if (mOwner->isRealTimeRecording() && mOwner->numTracks() > 1) {
3166 /*
3167 * This extra delay of accepting incoming audio/video signals
3168 * helps to align a/v start time at the beginning of a recording
3169 * session, and it also helps eliminate the "recording" sound for
3170 * camcorder applications.
3171 *
3172 * If client does not set the start time offset, we fall back to
3173 * use the default initial delay value.
3174 */
3175 int64_t startTimeOffsetUs = mOwner->getStartTimeOffsetMs() * 1000LL;
3176 if (startTimeOffsetUs < 0) { // Start time offset was not set
3177 startTimeOffsetUs = kInitialDelayTimeUs;
3178 }
3179 startTimeUs += startTimeOffsetUs;
3180 ALOGI("Start time offset: %" PRId64 " us", startTimeOffsetUs);
3181 }
3182
3183 meta->setInt64(kKeyTime, startTimeUs);
3184
3185 status_t err = mSource->start(meta.get());
3186 if (err != OK) {
3187 mDone = mReachedEOS = true;
3188 return err;
3189 }
3190
3191 pthread_attr_t attr;
3192 pthread_attr_init(&attr);
3193 pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE);
3194
3195 mDone = false;
3196 mStarted = true;
3197 mTrackDurationUs = 0;
3198 mReachedEOS = false;
3199 mEstimatedTrackSizeBytes = 0;
3200 mMdatSizeBytes = 0;
3201 mMaxChunkDurationUs = 0;
3202 mLastDecodingTimeUs = -1;
3203
3204 pthread_create(&mThread, &attr, ThreadWrapper, this);
3205 pthread_attr_destroy(&attr);
3206
3207 return OK;
3208 }
3209
pause()3210 status_t MPEG4Writer::Track::pause() {
3211 mPaused = true;
3212 return OK;
3213 }
3214
stop(bool stopSource)3215 status_t MPEG4Writer::Track::stop(bool stopSource) {
3216 ALOGD("%s track stopping. %s source", getTrackType(), stopSource ? "Stop" : "Not Stop");
3217 if (!mStarted) {
3218 ALOGE("Stop() called but track is not started or stopped");
3219 return ERROR_END_OF_STREAM;
3220 }
3221
3222 if (mDone) {
3223 return OK;
3224 }
3225
3226 if (stopSource) {
3227 ALOGD("%s track source stopping", getTrackType());
3228 mSource->stop();
3229 ALOGD("%s track source stopped", getTrackType());
3230 }
3231
3232 // Set mDone to be true after sucessfully stop mSource as mSource may be still outputting
3233 // buffers to the writer.
3234 mDone = true;
3235
3236 void *dummy;
3237 status_t err = OK;
3238 int retVal = pthread_join(mThread, &dummy);
3239 if (retVal == 0) {
3240 err = static_cast<status_t>(reinterpret_cast<uintptr_t>(dummy));
3241 ALOGD("%s track stopped. Status:%d. %s source",
3242 getTrackType(), err, stopSource ? "Stop" : "Not Stop");
3243 } else {
3244 ALOGE("track::stop: pthread_join retVal:%d", retVal);
3245 err = UNKNOWN_ERROR;
3246 }
3247 mStarted = false;
3248 return err;
3249 }
3250
reachedEOS()3251 bool MPEG4Writer::Track::reachedEOS() {
3252 return mReachedEOS;
3253 }
3254
3255 // static
ThreadWrapper(void * me)3256 void *MPEG4Writer::Track::ThreadWrapper(void *me) {
3257 Track *track = static_cast<Track *>(me);
3258
3259 status_t err = track->threadEntry();
3260 return (void *)(uintptr_t)err;
3261 }
3262
getNalUnitType(uint8_t byte,uint8_t * type)3263 static void getNalUnitType(uint8_t byte, uint8_t* type) {
3264 ALOGV("getNalUnitType: %d", byte);
3265
3266 // nal_unit_type: 5-bit unsigned integer
3267 *type = (byte & 0x1F);
3268 }
3269
parseParamSet(const uint8_t * data,size_t length,int type,size_t * paramSetLen)3270 const uint8_t *MPEG4Writer::Track::parseParamSet(
3271 const uint8_t *data, size_t length, int type, size_t *paramSetLen) {
3272
3273 ALOGV("parseParamSet");
3274 CHECK(type == kNalUnitTypeSeqParamSet ||
3275 type == kNalUnitTypePicParamSet);
3276
3277 const uint8_t *nextStartCode = findNextNalStartCode(data, length);
3278 *paramSetLen = nextStartCode - data;
3279 if (*paramSetLen == 0) {
3280 ALOGE("Param set is malformed, since its length is 0");
3281 return NULL;
3282 }
3283
3284 AVCParamSet paramSet(*paramSetLen, data);
3285 if (type == kNalUnitTypeSeqParamSet) {
3286 if (*paramSetLen < 4) {
3287 ALOGE("Seq parameter set malformed");
3288 return NULL;
3289 }
3290 if (mSeqParamSets.empty()) {
3291 mProfileIdc = data[1];
3292 mProfileCompatible = data[2];
3293 mLevelIdc = data[3];
3294 } else {
3295 if (mProfileIdc != data[1] ||
3296 mProfileCompatible != data[2] ||
3297 mLevelIdc != data[3]) {
3298 // COULD DO: set profile/level to the lowest required to support all SPSs
3299 ALOGE("Inconsistent profile/level found in seq parameter sets");
3300 return NULL;
3301 }
3302 }
3303 mSeqParamSets.push_back(paramSet);
3304 } else {
3305 mPicParamSets.push_back(paramSet);
3306 }
3307 return nextStartCode;
3308 }
3309
copyAVCCodecSpecificData(const uint8_t * data,size_t size)3310 status_t MPEG4Writer::Track::copyAVCCodecSpecificData(
3311 const uint8_t *data, size_t size) {
3312 ALOGV("copyAVCCodecSpecificData");
3313
3314 // 2 bytes for each of the parameter set length field
3315 // plus the 7 bytes for the header
3316 return copyCodecSpecificData(data, size, 4 + 7);
3317 }
3318
copyHEVCCodecSpecificData(const uint8_t * data,size_t size)3319 status_t MPEG4Writer::Track::copyHEVCCodecSpecificData(
3320 const uint8_t *data, size_t size) {
3321 ALOGV("copyHEVCCodecSpecificData");
3322
3323 // Min length of HEVC CSD is 23. (ISO/IEC 14496-15:2014 Chapter 8.3.3.1.2)
3324 return copyCodecSpecificData(data, size, 23);
3325 }
3326
copyCodecSpecificData(const uint8_t * data,size_t size,size_t minLength)3327 status_t MPEG4Writer::Track::copyCodecSpecificData(
3328 const uint8_t *data, size_t size, size_t minLength) {
3329 if (size < minLength) {
3330 ALOGE("Codec specific data length too short: %zu", size);
3331 return ERROR_MALFORMED;
3332 }
3333
3334 mCodecSpecificData = malloc(size);
3335 if (mCodecSpecificData == NULL) {
3336 ALOGE("Failed allocating codec specific data");
3337 return NO_MEMORY;
3338 }
3339 mCodecSpecificDataSize = size;
3340 memcpy(mCodecSpecificData, data, size);
3341 return OK;
3342 }
3343
parseAVCCodecSpecificData(const uint8_t * data,size_t size)3344 status_t MPEG4Writer::Track::parseAVCCodecSpecificData(
3345 const uint8_t *data, size_t size) {
3346
3347 ALOGV("parseAVCCodecSpecificData");
3348 // Data starts with a start code.
3349 // SPS and PPS are separated with start codes.
3350 // Also, SPS must come before PPS
3351 uint8_t type = kNalUnitTypeSeqParamSet;
3352 bool gotSps = false;
3353 bool gotPps = false;
3354 const uint8_t *tmp = data;
3355 const uint8_t *nextStartCode = data;
3356 size_t bytesLeft = size;
3357 size_t paramSetLen = 0;
3358 mCodecSpecificDataSize = 0;
3359 while (bytesLeft > 4 && !memcmp("\x00\x00\x00\x01", tmp, 4)) {
3360 getNalUnitType(*(tmp + 4), &type);
3361 if (type == kNalUnitTypeSeqParamSet) {
3362 if (gotPps) {
3363 ALOGE("SPS must come before PPS");
3364 return ERROR_MALFORMED;
3365 }
3366 if (!gotSps) {
3367 gotSps = true;
3368 }
3369 nextStartCode = parseParamSet(tmp + 4, bytesLeft - 4, type, ¶mSetLen);
3370 } else if (type == kNalUnitTypePicParamSet) {
3371 if (!gotSps) {
3372 ALOGE("SPS must come before PPS");
3373 return ERROR_MALFORMED;
3374 }
3375 if (!gotPps) {
3376 gotPps = true;
3377 }
3378 nextStartCode = parseParamSet(tmp + 4, bytesLeft - 4, type, ¶mSetLen);
3379 } else {
3380 ALOGE("Only SPS and PPS Nal units are expected");
3381 return ERROR_MALFORMED;
3382 }
3383
3384 if (nextStartCode == NULL) {
3385 ALOGE("nextStartCode is null");
3386 return ERROR_MALFORMED;
3387 }
3388
3389 // Move on to find the next parameter set
3390 bytesLeft -= nextStartCode - tmp;
3391 tmp = nextStartCode;
3392 mCodecSpecificDataSize += (2 + paramSetLen);
3393 }
3394
3395 {
3396 // Check on the number of seq parameter sets
3397 size_t nSeqParamSets = mSeqParamSets.size();
3398 if (nSeqParamSets == 0) {
3399 ALOGE("Cound not find sequence parameter set");
3400 return ERROR_MALFORMED;
3401 }
3402
3403 if (nSeqParamSets > 0x1F) {
3404 ALOGE("Too many seq parameter sets (%zu) found", nSeqParamSets);
3405 return ERROR_MALFORMED;
3406 }
3407 }
3408
3409 {
3410 // Check on the number of pic parameter sets
3411 size_t nPicParamSets = mPicParamSets.size();
3412 if (nPicParamSets == 0) {
3413 ALOGE("Cound not find picture parameter set");
3414 return ERROR_MALFORMED;
3415 }
3416 if (nPicParamSets > 0xFF) {
3417 ALOGE("Too many pic parameter sets (%zd) found", nPicParamSets);
3418 return ERROR_MALFORMED;
3419 }
3420 }
3421 // FIXME:
3422 // Add chromat_format_idc, bit depth values, etc for AVC/h264 high profile and above
3423 // and remove #if 0
3424 #if 0
3425 {
3426 // Check on the profiles
3427 // These profiles requires additional parameter set extensions
3428 if (mProfileIdc == 100 || mProfileIdc == 110 ||
3429 mProfileIdc == 122 || mProfileIdc == 144) {
3430 ALOGE("Sorry, no support for profile_idc: %d!", mProfileIdc);
3431 return BAD_VALUE;
3432 }
3433 }
3434 #endif
3435 return OK;
3436 }
3437
makeAVCCodecSpecificData(const uint8_t * data,size_t size)3438 status_t MPEG4Writer::Track::makeAVCCodecSpecificData(
3439 const uint8_t *data, size_t size) {
3440
3441 if (mCodecSpecificData != NULL) {
3442 ALOGE("Already have codec specific data");
3443 return ERROR_MALFORMED;
3444 }
3445
3446 if (size < 4) {
3447 ALOGE("Codec specific data length too short: %zu", size);
3448 return ERROR_MALFORMED;
3449 }
3450
3451 // Data is in the form of AVCCodecSpecificData
3452 if (memcmp("\x00\x00\x00\x01", data, 4)) {
3453 return copyAVCCodecSpecificData(data, size);
3454 }
3455
3456 if (parseAVCCodecSpecificData(data, size) != OK) {
3457 return ERROR_MALFORMED;
3458 }
3459
3460 // ISO 14496-15: AVC file format
3461 mCodecSpecificDataSize += 7; // 7 more bytes in the header
3462 mCodecSpecificData = malloc(mCodecSpecificDataSize);
3463 if (mCodecSpecificData == NULL) {
3464 mCodecSpecificDataSize = 0;
3465 ALOGE("Failed allocating codec specific data");
3466 return NO_MEMORY;
3467 }
3468 uint8_t *header = (uint8_t *)mCodecSpecificData;
3469 header[0] = 1; // version
3470 header[1] = mProfileIdc; // profile indication
3471 header[2] = mProfileCompatible; // profile compatibility
3472 header[3] = mLevelIdc;
3473
3474 // 6-bit '111111' followed by 2-bit to lengthSizeMinuusOne
3475 if (mOwner->useNalLengthFour()) {
3476 header[4] = 0xfc | 3; // length size == 4 bytes
3477 } else {
3478 header[4] = 0xfc | 1; // length size == 2 bytes
3479 }
3480
3481 // 3-bit '111' followed by 5-bit numSequenceParameterSets
3482 int nSequenceParamSets = mSeqParamSets.size();
3483 header[5] = 0xe0 | nSequenceParamSets;
3484 header += 6;
3485 for (List<AVCParamSet>::iterator it = mSeqParamSets.begin();
3486 it != mSeqParamSets.end(); ++it) {
3487 // 16-bit sequence parameter set length
3488 uint16_t seqParamSetLength = it->mLength;
3489 header[0] = seqParamSetLength >> 8;
3490 header[1] = seqParamSetLength & 0xff;
3491
3492 // SPS NAL unit (sequence parameter length bytes)
3493 memcpy(&header[2], it->mData, seqParamSetLength);
3494 header += (2 + seqParamSetLength);
3495 }
3496
3497 // 8-bit nPictureParameterSets
3498 int nPictureParamSets = mPicParamSets.size();
3499 header[0] = nPictureParamSets;
3500 header += 1;
3501 for (List<AVCParamSet>::iterator it = mPicParamSets.begin();
3502 it != mPicParamSets.end(); ++it) {
3503 // 16-bit picture parameter set length
3504 uint16_t picParamSetLength = it->mLength;
3505 header[0] = picParamSetLength >> 8;
3506 header[1] = picParamSetLength & 0xff;
3507
3508 // PPS Nal unit (picture parameter set length bytes)
3509 memcpy(&header[2], it->mData, picParamSetLength);
3510 header += (2 + picParamSetLength);
3511 }
3512
3513 return OK;
3514 }
3515
3516
parseHEVCCodecSpecificData(const uint8_t * data,size_t size,HevcParameterSets & paramSets)3517 status_t MPEG4Writer::Track::parseHEVCCodecSpecificData(
3518 const uint8_t *data, size_t size, HevcParameterSets ¶mSets) {
3519
3520 ALOGV("parseHEVCCodecSpecificData");
3521 const uint8_t *tmp = data;
3522 const uint8_t *nextStartCode = data;
3523 size_t bytesLeft = size;
3524 while (bytesLeft > 4 && !memcmp("\x00\x00\x00\x01", tmp, 4)) {
3525 nextStartCode = findNextNalStartCode(tmp + 4, bytesLeft - 4);
3526 status_t err = paramSets.addNalUnit(tmp + 4, (nextStartCode - tmp) - 4);
3527 if (err != OK) {
3528 return ERROR_MALFORMED;
3529 }
3530
3531 // Move on to find the next parameter set
3532 bytesLeft -= nextStartCode - tmp;
3533 tmp = nextStartCode;
3534 }
3535
3536 size_t csdSize = 23;
3537 const size_t numNalUnits = paramSets.getNumNalUnits();
3538 for (size_t i = 0; i < ARRAY_SIZE(kMandatoryHevcNalUnitTypes); ++i) {
3539 int type = kMandatoryHevcNalUnitTypes[i];
3540 size_t numParamSets = paramSets.getNumNalUnitsOfType(type);
3541 if (numParamSets == 0) {
3542 ALOGE("Cound not find NAL unit of type %d", type);
3543 return ERROR_MALFORMED;
3544 }
3545 }
3546 for (size_t i = 0; i < ARRAY_SIZE(kHevcNalUnitTypes); ++i) {
3547 int type = kHevcNalUnitTypes[i];
3548 size_t numParamSets = paramSets.getNumNalUnitsOfType(type);
3549 if (numParamSets > 0xffff) {
3550 ALOGE("Too many seq parameter sets (%zu) found", numParamSets);
3551 return ERROR_MALFORMED;
3552 }
3553 csdSize += 3;
3554 for (size_t j = 0; j < numNalUnits; ++j) {
3555 if (paramSets.getType(j) != type) {
3556 continue;
3557 }
3558 csdSize += 2 + paramSets.getSize(j);
3559 }
3560 }
3561 mCodecSpecificDataSize = csdSize;
3562 return OK;
3563 }
3564
makeHEVCCodecSpecificData(const uint8_t * data,size_t size)3565 status_t MPEG4Writer::Track::makeHEVCCodecSpecificData(
3566 const uint8_t *data, size_t size) {
3567
3568 if (mCodecSpecificData != NULL) {
3569 ALOGE("Already have codec specific data");
3570 return ERROR_MALFORMED;
3571 }
3572
3573 if (size < 4) {
3574 ALOGE("Codec specific data length too short: %zu", size);
3575 return ERROR_MALFORMED;
3576 }
3577
3578 // Data is in the form of HEVCCodecSpecificData
3579 if (memcmp("\x00\x00\x00\x01", data, 4)) {
3580 return copyHEVCCodecSpecificData(data, size);
3581 }
3582
3583 HevcParameterSets paramSets;
3584 if (parseHEVCCodecSpecificData(data, size, paramSets) != OK) {
3585 ALOGE("failed parsing codec specific data");
3586 return ERROR_MALFORMED;
3587 }
3588
3589 mCodecSpecificData = malloc(mCodecSpecificDataSize);
3590 if (mCodecSpecificData == NULL) {
3591 mCodecSpecificDataSize = 0;
3592 ALOGE("Failed allocating codec specific data");
3593 return NO_MEMORY;
3594 }
3595 status_t err = paramSets.makeHvcc((uint8_t *)mCodecSpecificData,
3596 &mCodecSpecificDataSize, mOwner->useNalLengthFour() ? 4 : 2);
3597 if (err != OK) {
3598 ALOGE("failed constructing HVCC atom");
3599 return err;
3600 }
3601
3602 return OK;
3603 }
3604
getDolbyVisionProfile()3605 status_t MPEG4Writer::Track::getDolbyVisionProfile() {
3606 uint32_t type;
3607 const void *data = NULL;
3608 size_t size = 0;
3609
3610 if (!mMeta->findData(kKeyDVCC, &type, &data, &size) &&
3611 !mMeta->findData(kKeyDVVC, &type, &data, &size) &&
3612 !mMeta->findData(kKeyDVWC, &type, &data, &size)) {
3613 ALOGE("Failed getting Dovi config for Dolby Vision %d", (int)size);
3614 return ERROR_MALFORMED;
3615 }
3616 static const ALookup<uint8_t, int32_t> dolbyVisionProfileMap = {
3617 {1, DolbyVisionProfileDvavPen},
3618 {3, DolbyVisionProfileDvheDen},
3619 {4, DolbyVisionProfileDvheDtr},
3620 {5, DolbyVisionProfileDvheStn},
3621 {6, DolbyVisionProfileDvheDth},
3622 {7, DolbyVisionProfileDvheDtb},
3623 {8, DolbyVisionProfileDvheSt},
3624 {9, DolbyVisionProfileDvavSe},
3625 {10, DolbyVisionProfileDvav110}
3626 };
3627
3628 // Dolby Vision profile information is extracted as per
3629 // https://dolby.my.salesforce.com/sfc/p/#700000009YuG/a/4u000000l6FB/076wHYEmyEfz09m0V1bo85_25hlUJjaiWTbzorNmYY4
3630 uint8_t dv_profile = ((((uint8_t *)data)[2] >> 1) & 0x7f);
3631
3632 if (!dolbyVisionProfileMap.map(dv_profile, &mDoviProfile)) {
3633 ALOGE("Failed to get Dolby Profile from DV Config data");
3634 return ERROR_MALFORMED;
3635 }
3636 return OK;
3637 }
3638
3639 /*
3640 * Updates the drift time from the audio track so that
3641 * the video track can get the updated drift time information
3642 * from the file writer. The fluctuation of the drift time of the audio
3643 * encoding path is smoothed out with a simple filter by giving a larger
3644 * weight to more recently drift time. The filter coefficients, 0.5 and 0.5,
3645 * are heuristically determined.
3646 */
updateDriftTime(const sp<MetaData> & meta)3647 void MPEG4Writer::Track::updateDriftTime(const sp<MetaData>& meta) {
3648 int64_t driftTimeUs = 0;
3649 if (meta->findInt64(kKeyDriftTime, &driftTimeUs)) {
3650 int64_t prevDriftTimeUs = mOwner->getDriftTimeUs();
3651 int64_t timeUs = (driftTimeUs + prevDriftTimeUs) >> 1;
3652 mOwner->setDriftTimeUs(timeUs);
3653 }
3654 }
3655
dumpTimeStamps()3656 void MPEG4Writer::Track::dumpTimeStamps() {
3657 if (!mTimestampDebugHelper.empty()) {
3658 std::string timeStampString = "Dumping " + std::string(getTrackType()) + " track's last " +
3659 std::to_string(mTimestampDebugHelper.size()) +
3660 " frames' timestamps(pts, dts) and frame type : ";
3661 for (const TimestampDebugHelperEntry& entry : mTimestampDebugHelper) {
3662 timeStampString += "\n(" + std::to_string(entry.pts) + "us, " +
3663 std::to_string(entry.dts) + "us " + entry.frameType + ") ";
3664 }
3665 ALOGE("%s", timeStampString.c_str());
3666 } else {
3667 ALOGE("0 frames to dump timeStamps in %s track ", getTrackType());
3668 }
3669 }
3670
threadEntry()3671 status_t MPEG4Writer::Track::threadEntry() {
3672 int32_t count = 0;
3673 const int64_t interleaveDurationUs = mOwner->interleaveDuration();
3674 const bool hasMultipleTracks = (mOwner->numTracks() > 1);
3675 int64_t chunkTimestampUs = 0;
3676 int32_t nChunks = 0;
3677 int32_t nActualFrames = 0; // frames containing non-CSD data (non-0 length)
3678 int32_t nZeroLengthFrames = 0;
3679 int64_t lastTimestampUs = 0; // Previous sample time stamp
3680 int64_t previousSampleTimestampWithoutFudgeUs = 0; // Timestamp received/without fudge for STTS
3681 int64_t lastDurationUs = 0; // Between the previous two samples
3682 int64_t currDurationTicks = 0; // Timescale based ticks
3683 int64_t lastDurationTicks = 0; // Timescale based ticks
3684 int32_t sampleCount = 1; // Sample count in the current stts table entry
3685 uint32_t previousSampleSize = 0; // Size of the previous sample
3686 int64_t previousPausedDurationUs = 0;
3687 int64_t timestampUs = 0;
3688 int64_t cttsOffsetTimeUs = 0;
3689 int64_t currCttsOffsetTimeTicks = 0; // Timescale based ticks
3690 int64_t lastCttsOffsetTimeTicks = -1; // Timescale based ticks
3691 int32_t cttsSampleCount = 0; // Sample count in the current ctts table entry
3692 uint32_t lastSamplesPerChunk = 0;
3693 int64_t lastSampleDurationUs = -1; // Duration calculated from EOS buffer and its timestamp
3694 int64_t lastSampleDurationTicks = -1; // Timescale based ticks
3695 int64_t sampleFileOffset = -1;
3696
3697 if (mIsAudio) {
3698 prctl(PR_SET_NAME, (unsigned long)"MP4WtrAudTrkThread", 0, 0, 0);
3699 } else if (mIsVideo) {
3700 prctl(PR_SET_NAME, (unsigned long)"MP4WtrVidTrkThread", 0, 0, 0);
3701 } else {
3702 prctl(PR_SET_NAME, (unsigned long)"MP4WtrMetaTrkThread", 0, 0, 0);
3703 }
3704
3705 if (mOwner->isRealTimeRecording()) {
3706 androidSetThreadPriority(0, ANDROID_PRIORITY_AUDIO);
3707 } else if (mOwner->isBackgroundMode()) {
3708 // Background priority for media transcoding.
3709 androidSetThreadPriority(0 /* tid (0 = current) */, ANDROID_PRIORITY_BACKGROUND);
3710 }
3711
3712 sp<MetaData> meta_data;
3713
3714 status_t err = OK;
3715 MediaBufferBase *buffer;
3716 const char *trackName = getTrackType();
3717 while (!mDone && (err = mSource->read(&buffer)) == OK) {
3718 ALOGV("read:buffer->range_length:%lld", (long long)buffer->range_length());
3719 int32_t isEOS = false;
3720 if (buffer->range_length() == 0) {
3721 if (buffer->meta_data().findInt32(kKeyIsEndOfStream, &isEOS) && isEOS) {
3722 int64_t eosSampleTimestampUs = -1;
3723 CHECK(buffer->meta_data().findInt64(kKeyTime, &eosSampleTimestampUs));
3724 if (eosSampleTimestampUs > 0) {
3725 lastSampleDurationUs = eosSampleTimestampUs -
3726 previousSampleTimestampWithoutFudgeUs -
3727 previousPausedDurationUs;
3728 if (lastSampleDurationUs >= 0) {
3729 lastSampleDurationTicks = (lastSampleDurationUs * mTimeScale + 500000LL) /
3730 1000000LL;
3731 } else {
3732 ALOGW("lastSampleDurationUs %" PRId64 " is negative", lastSampleDurationUs);
3733 }
3734 }
3735 buffer->release();
3736 buffer = nullptr;
3737 mSource->stop();
3738 break;
3739 } else {
3740 buffer->release();
3741 buffer = nullptr;
3742 ++nZeroLengthFrames;
3743 continue;
3744 }
3745 }
3746
3747 // If the codec specific data has not been received yet, delay pause.
3748 // After the codec specific data is received, discard what we received
3749 // when the track is to be paused.
3750 if (mPaused && !mResumed) {
3751 buffer->release();
3752 buffer = NULL;
3753 continue;
3754 }
3755
3756 ++count;
3757
3758 int32_t isCodecConfig;
3759 if (buffer->meta_data().findInt32(kKeyIsCodecConfig, &isCodecConfig)
3760 && isCodecConfig) {
3761 // if config format (at track addition) already had CSD, keep that
3762 // UNLESS we have not received any frames yet.
3763 // TODO: for now the entire CSD has to come in one frame for encoders, even though
3764 // they need to be spread out for decoders.
3765 if (mGotAllCodecSpecificData && nActualFrames > 0) {
3766 ALOGI("ignoring additional CSD for video track after first frame");
3767 } else {
3768 mMeta = mSource->getFormat(); // get output format after format change
3769 status_t err;
3770 if (mIsAvc) {
3771 err = makeAVCCodecSpecificData(
3772 (const uint8_t *)buffer->data()
3773 + buffer->range_offset(),
3774 buffer->range_length());
3775 } else if (mIsHevc || mIsHeic) {
3776 err = makeHEVCCodecSpecificData(
3777 (const uint8_t *)buffer->data()
3778 + buffer->range_offset(),
3779 buffer->range_length());
3780 } else if (mIsMPEG4 || mIsAv1 || mIsApv) {
3781 err = copyCodecSpecificData((const uint8_t *)buffer->data() + buffer->range_offset(),
3782 buffer->range_length());
3783 }
3784 if (mIsDovi) {
3785 err = getDolbyVisionProfile();
3786 if(err == OK) {
3787 const void *data = NULL;
3788 size_t size = 0;
3789 uint32_t type = 0;
3790 if (mDoviProfile == DolbyVisionProfileDvavSe) {
3791 mMeta->findData(kKeyAVCC, &type, &data, &size);
3792 } else if (mDoviProfile < DolbyVisionProfileDvavSe) {
3793 mMeta->findData(kKeyHVCC, &type, &data, &size);
3794 } else {
3795 ALOGW("DV Profiles > DolbyVisionProfileDvavSe are not supported");
3796 err = ERROR_MALFORMED;
3797 }
3798 if (err == OK && data != NULL &&
3799 copyCodecSpecificData((uint8_t *)data, size) == OK) {
3800 mGotAllCodecSpecificData = true;
3801 }
3802 }
3803 }
3804 }
3805 buffer->release();
3806 buffer = NULL;
3807 if (OK != err) {
3808 mSource->stop();
3809 mIsMalformed = true;
3810 uint32_t trackNum = (mTrackId.getId() << 28);
3811 mOwner->notify(MEDIA_RECORDER_TRACK_EVENT_ERROR,
3812 trackNum | MEDIA_RECORDER_TRACK_ERROR_GENERAL, err);
3813 break;
3814 }
3815
3816 mGotAllCodecSpecificData = true;
3817 continue;
3818 }
3819
3820 // Per-frame metadata sample's size must be smaller than max allowed.
3821 if (!mIsVideo && !mIsAudio && !mIsHeif &&
3822 buffer->range_length() >= kMaxMetadataSize) {
3823 ALOGW("Buffer size is %zu. Maximum metadata buffer size is %lld for %s track",
3824 buffer->range_length(), (long long)kMaxMetadataSize, trackName);
3825 buffer->release();
3826 mSource->stop();
3827 mIsMalformed = true;
3828 break;
3829 }
3830
3831 bool isGainmapMeta = false;
3832 bool isGainmap = false;
3833 bool isExif = false;
3834 uint32_t tiffHdrOffset = 0;
3835 uint32_t gainmapOffset = 0;
3836 int32_t isMuxerData;
3837 if (buffer->meta_data().findInt32(kKeyIsMuxerData, &isMuxerData) && isMuxerData) {
3838 if (flags_camera::camera_heif_gainmap()) {
3839 isGainmapMeta = isGainmapMetaData(buffer, &gainmapOffset);
3840 isGainmap = isGainmapData(buffer, &gainmapOffset);
3841 if ((isGainmap || isGainmapMeta) && (gainmapOffset > 0) &&
3842 (gainmapOffset < buffer->range_length())) {
3843 // Don't include the tmap/gmap header
3844 buffer->set_range(gainmapOffset, buffer->range_length() - gainmapOffset);
3845 }
3846 }
3847 isExif = isExifData(buffer, &tiffHdrOffset);
3848 if (!isExif && !isGainmap && !isGainmapMeta) {
3849 ALOGW("Ignoring bad muxer data block");
3850 buffer->release();
3851 buffer = NULL;
3852 continue;
3853 }
3854 }
3855 if (flags_camera::camera_heif_gainmap()) {
3856 int32_t val32;
3857 if (buffer->meta_data().findInt32(kKeyColorPrimaries, &val32)) {
3858 mColorAspects.mPrimaries = static_cast<ColorAspects::Primaries>(val32);
3859 mColorAspectsValid = true;
3860 } else {
3861 mColorAspectsValid = false;
3862 }
3863 if (buffer->meta_data().findInt32(kKeyTransferFunction, &val32)) {
3864 mColorAspects.mTransfer = static_cast<ColorAspects::Transfer>(val32);
3865 } else {
3866 mColorAspectsValid = false;
3867 }
3868 if (buffer->meta_data().findInt32(kKeyColorMatrix, &val32)) {
3869 mColorAspects.mMatrixCoeffs = static_cast<ColorAspects::MatrixCoeffs>(val32);
3870 } else {
3871 mColorAspectsValid = false;
3872 }
3873 if (buffer->meta_data().findInt32(kKeyColorRange, &val32)) {
3874 mColorAspects.mRange = static_cast<ColorAspects::Range>(val32);
3875 } else {
3876 mColorAspectsValid = false;
3877 }
3878 if (mBitsPerChannel.empty() && buffer->meta_data().findInt32(kKeyColorFormat, &val32)) {
3879 switch (val32) {
3880 case COLOR_FormatYUV420Flexible:
3881 case COLOR_FormatYUV420Planar:
3882 case COLOR_FormatYUV420SemiPlanar: {
3883 uint8_t bitsPerChannel[] = {8, 8, 8};
3884 mBitsPerChannel.appendArray(bitsPerChannel, sizeof(bitsPerChannel));
3885 }
3886 break;
3887 default:
3888 break;
3889 }
3890 }
3891 }
3892
3893 if (!buffer->meta_data().findInt64(kKeySampleFileOffset, &sampleFileOffset)) {
3894 sampleFileOffset = -1;
3895 }
3896 int64_t lastSample = -1;
3897 if (!buffer->meta_data().findInt64(kKeyLastSampleIndexInChunk, &lastSample)) {
3898 lastSample = -1;
3899 }
3900 ALOGV("sampleFileOffset:%lld", (long long)sampleFileOffset);
3901
3902 /*
3903 * Reserve space in the file for the current sample + to be written MOOV box. If reservation
3904 * for a new sample fails, preAllocate(...) stops muxing session completely. Stop() could
3905 * write MOOV box successfully as space for the same was reserved in the prior call.
3906 * Release the current buffer/sample here.
3907 */
3908 if (sampleFileOffset == -1 && !mOwner->preAllocate(buffer->range_length())) {
3909 buffer->release();
3910 buffer = nullptr;
3911 break;
3912 }
3913
3914 ++nActualFrames;
3915
3916 // Make a deep copy of the MediaBuffer and Metadata and release
3917 // the original as soon as we can
3918 MediaBuffer* copy = new MediaBuffer(buffer->range_length());
3919 if (sampleFileOffset != -1) {
3920 copy->meta_data().setInt64(kKeySampleFileOffset, sampleFileOffset);
3921 } else {
3922 memcpy(copy->data(), (uint8_t*)buffer->data() + buffer->range_offset(),
3923 buffer->range_length());
3924 }
3925 copy->set_range(0, buffer->range_length());
3926
3927 meta_data = new MetaData(buffer->meta_data());
3928 buffer->release();
3929 buffer = NULL;
3930 if (isExif) {
3931 copy->meta_data().setInt32(kKeyExifTiffOffset, tiffHdrOffset);
3932 }
3933 bool usePrefix = this->usePrefix() && !isExif && !isGainmapMeta;
3934 if (sampleFileOffset == -1 && usePrefix) {
3935 StripStartcode(copy);
3936 }
3937 size_t sampleSize = copy->range_length();
3938 if (sampleFileOffset == -1 && usePrefix) {
3939 if (mOwner->useNalLengthFour()) {
3940 ALOGV("nallength4");
3941 sampleSize += 4;
3942 } else {
3943 ALOGV("nallength2");
3944 sampleSize += 2;
3945 }
3946 }
3947
3948 // Max file size or duration handling
3949 mMdatSizeBytes += sampleSize;
3950 updateTrackSizeEstimate();
3951
3952 if (mOwner->exceedsFileSizeLimit()) {
3953 copy->release();
3954 if (mOwner->switchFd() != OK) {
3955 ALOGW("Recorded file size exceeds limit %" PRId64 "bytes",
3956 mOwner->mMaxFileSizeLimitBytes);
3957 mSource->stop();
3958 mOwner->notify(
3959 MEDIA_RECORDER_EVENT_INFO, MEDIA_RECORDER_INFO_MAX_FILESIZE_REACHED, 0);
3960 } else {
3961 ALOGV("%s Current recorded file size exceeds limit %" PRId64 "bytes. Switching output",
3962 getTrackType(), mOwner->mMaxFileSizeLimitBytes);
3963 }
3964 break;
3965 }
3966
3967 if (mOwner->exceedsFileDurationLimit()) {
3968 ALOGW("Recorded file duration exceeds limit %" PRId64 "microseconds",
3969 mOwner->mMaxFileDurationLimitUs);
3970 copy->release();
3971 mSource->stop();
3972 mOwner->notify(MEDIA_RECORDER_EVENT_INFO, MEDIA_RECORDER_INFO_MAX_DURATION_REACHED, 0);
3973 break;
3974 }
3975
3976 if (mOwner->approachingFileSizeLimit()) {
3977 mOwner->notifyApproachingLimit();
3978 }
3979 int32_t isSync = false;
3980 meta_data->findInt32(kKeyIsSyncFrame, &isSync);
3981 CHECK(meta_data->findInt64(kKeyTime, ×tampUs));
3982 timestampUs += mFirstSampleStartOffsetUs;
3983
3984 // For video, skip the first several non-key frames until getting the first key frame.
3985 if (mIsVideo && !mGotStartKeyFrame && !isSync) {
3986 ALOGD("Video skip non-key frame");
3987 copy->release();
3988 continue;
3989 }
3990 if (mIsVideo && isSync) {
3991 mGotStartKeyFrame = true;
3992 }
3993 ////////////////////////////////////////////////////////////////////////////////
3994 if (!mIsHeif) {
3995 if (mStszTableEntries->count() == 0) {
3996 mFirstSampleTimeRealUs = systemTime() / 1000;
3997 if (timestampUs < 0 && mFirstSampleStartOffsetUs == 0) {
3998 if (WARN_UNLESS(timestampUs != INT64_MIN, "for %s track", trackName)) {
3999 copy->release();
4000 mSource->stop();
4001 mIsMalformed = true;
4002 break;
4003 }
4004 mFirstSampleStartOffsetUs = -timestampUs;
4005 timestampUs = 0;
4006 }
4007 mOwner->setStartTimestampUs(timestampUs);
4008 mStartTimestampUs = timestampUs;
4009 previousPausedDurationUs = mStartTimestampUs;
4010 }
4011
4012 if (mResumed) {
4013 int64_t durExcludingEarlierPausesUs = timestampUs - previousPausedDurationUs;
4014 if (WARN_UNLESS(durExcludingEarlierPausesUs >= 0LL, "for %s track", trackName)) {
4015 copy->release();
4016 mSource->stop();
4017 mIsMalformed = true;
4018 break;
4019 }
4020
4021 int64_t pausedDurationUs = durExcludingEarlierPausesUs - mTrackDurationUs;
4022 if (WARN_UNLESS(pausedDurationUs >= lastDurationUs, "for %s track", trackName)) {
4023 copy->release();
4024 mSource->stop();
4025 mIsMalformed = true;
4026 break;
4027 }
4028
4029 previousPausedDurationUs += pausedDurationUs - lastDurationUs;
4030 mResumed = false;
4031 }
4032 TimestampDebugHelperEntry timestampDebugEntry;
4033 timestampUs -= previousPausedDurationUs;
4034 timestampDebugEntry.pts = timestampUs;
4035 if (WARN_UNLESS(timestampUs >= 0LL, "for %s track", trackName)) {
4036 copy->release();
4037 mSource->stop();
4038 mIsMalformed = true;
4039 break;
4040 }
4041
4042 if (mIsVideo) {
4043 /*
4044 * Composition time: timestampUs
4045 * Decoding time: decodingTimeUs
4046 * Composition time offset = composition time - decoding time
4047 */
4048 int64_t decodingTimeUs;
4049 CHECK(meta_data->findInt64(kKeyDecodingTime, &decodingTimeUs));
4050 decodingTimeUs -= previousPausedDurationUs;
4051
4052 // ensure non-negative, monotonic decoding time
4053 if (mLastDecodingTimeUs < 0) {
4054 decodingTimeUs = std::max((int64_t)0, decodingTimeUs);
4055 } else {
4056 // increase decoding time by at least the larger vaule of 1 tick and
4057 // 0.1 milliseconds. This needs to take into account the possible
4058 // delta adjustment in DurationTicks in below.
4059 decodingTimeUs = std::max(mLastDecodingTimeUs +
4060 std::max(100, divUp(1000000, mTimeScale)), decodingTimeUs);
4061 }
4062
4063 mLastDecodingTimeUs = decodingTimeUs;
4064 timestampDebugEntry.dts = decodingTimeUs;
4065 timestampDebugEntry.frameType = isSync ? "Key frame" : "Non-Key frame";
4066 // Insert the timestamp into the mTimestampDebugHelper
4067 if (mTimestampDebugHelper.size() >= kTimestampDebugCount) {
4068 mTimestampDebugHelper.pop_front();
4069 }
4070 mTimestampDebugHelper.push_back(timestampDebugEntry);
4071
4072 cttsOffsetTimeUs =
4073 timestampUs + kMaxCttsOffsetTimeUs - decodingTimeUs;
4074 if (WARN_UNLESS(cttsOffsetTimeUs >= 0LL, "for %s track", trackName)) {
4075 copy->release();
4076 mSource->stop();
4077 mIsMalformed = true;
4078 break;
4079 }
4080
4081 timestampUs = decodingTimeUs;
4082 ALOGV("decoding time: %" PRId64 " and ctts offset time: %" PRId64,
4083 timestampUs, cttsOffsetTimeUs);
4084
4085 // Update ctts box table if necessary
4086 currCttsOffsetTimeTicks =
4087 (cttsOffsetTimeUs * mTimeScale + 500000LL) / 1000000LL;
4088 if (WARN_UNLESS(currCttsOffsetTimeTicks <= 0x0FFFFFFFFLL, "for %s track", trackName)) {
4089 copy->release();
4090 mSource->stop();
4091 mIsMalformed = true;
4092 break;
4093 }
4094
4095 if (mStszTableEntries->count() == 0) {
4096 // Force the first ctts table entry to have one single entry
4097 // so that we can do adjustment for the initial track start
4098 // time offset easily in writeCttsBox().
4099 lastCttsOffsetTimeTicks = currCttsOffsetTimeTicks;
4100 addOneCttsTableEntry(1, currCttsOffsetTimeTicks);
4101 cttsSampleCount = 0; // No sample in ctts box is pending
4102 } else {
4103 if (currCttsOffsetTimeTicks != lastCttsOffsetTimeTicks) {
4104 addOneCttsTableEntry(cttsSampleCount, lastCttsOffsetTimeTicks);
4105 lastCttsOffsetTimeTicks = currCttsOffsetTimeTicks;
4106 cttsSampleCount = 1; // One sample in ctts box is pending
4107 } else {
4108 ++cttsSampleCount;
4109 }
4110 }
4111
4112 // Update ctts time offset range
4113 if (mStszTableEntries->count() == 0) {
4114 mMinCttsOffsetTicks = currCttsOffsetTimeTicks;
4115 mMaxCttsOffsetTicks = currCttsOffsetTimeTicks;
4116 } else {
4117 if (currCttsOffsetTimeTicks > mMaxCttsOffsetTicks) {
4118 mMaxCttsOffsetTicks = currCttsOffsetTimeTicks;
4119 } else if (currCttsOffsetTimeTicks < mMinCttsOffsetTicks) {
4120 mMinCttsOffsetTicks = currCttsOffsetTimeTicks;
4121 mMinCttsOffsetTimeUs = cttsOffsetTimeUs;
4122 }
4123 }
4124 }
4125
4126 if (mOwner->isRealTimeRecording()) {
4127 if (mIsAudio) {
4128 updateDriftTime(meta_data);
4129 }
4130 }
4131
4132 if (WARN_UNLESS(timestampUs >= 0LL, "for %s track", trackName)) {
4133 copy->release();
4134 mSource->stop();
4135 mIsMalformed = true;
4136 break;
4137 }
4138
4139 ALOGV("%s media time stamp: %" PRId64 " and previous paused duration %" PRId64,
4140 trackName, timestampUs, previousPausedDurationUs);
4141 if (timestampUs > mTrackDurationUs) {
4142 mTrackDurationUs = timestampUs;
4143 }
4144
4145 // We need to use the time scale based ticks, rather than the
4146 // timestamp itself to determine whether we have to use a new
4147 // stts entry, since we may have rounding errors.
4148 // The calculation is intended to reduce the accumulated
4149 // rounding errors.
4150 currDurationTicks =
4151 ((timestampUs * mTimeScale + 500000LL) / 1000000LL -
4152 (lastTimestampUs * mTimeScale + 500000LL) / 1000000LL);
4153 if (currDurationTicks < 0LL) {
4154 ALOGE("do not support out of order frames (timestamp: %lld < last: %lld for %s track",
4155 (long long)timestampUs, (long long)lastTimestampUs, trackName);
4156 copy->release();
4157 mSource->stop();
4158 mIsMalformed = true;
4159 break;
4160 }
4161
4162 previousSampleTimestampWithoutFudgeUs = timestampUs;
4163
4164 // if the duration is different for this sample, see if it is close enough to the previous
4165 // duration that we can fudge it and use the same value, to avoid filling the stts table
4166 // with lots of near-identical entries.
4167 // "close enough" here means that the current duration needs to be adjusted by less
4168 // than 0.1 milliseconds
4169 if (lastDurationTicks && (currDurationTicks != lastDurationTicks)) {
4170 int64_t deltaUs = ((lastDurationTicks - currDurationTicks) * 1000000LL
4171 + (mTimeScale / 2)) / mTimeScale;
4172 if (deltaUs > -100 && deltaUs < 100) {
4173 // use previous ticks, and adjust timestamp as if it was actually that number
4174 // of ticks
4175 currDurationTicks = lastDurationTicks;
4176 timestampUs += deltaUs;
4177 }
4178 }
4179 mStszTableEntries->add(htonl(sampleSize));
4180
4181 if (mStszTableEntries->count() > 2) {
4182
4183 // Force the first sample to have its own stts entry so that
4184 // we can adjust its value later to maintain the A/V sync.
4185 if (lastDurationTicks && currDurationTicks != lastDurationTicks) {
4186 addOneSttsTableEntry(sampleCount, lastDurationTicks);
4187 sampleCount = 1;
4188 } else {
4189 ++sampleCount;
4190 }
4191 }
4192 if (mSamplesHaveSameSize) {
4193 if (mStszTableEntries->count() >= 2 && previousSampleSize != sampleSize) {
4194 mSamplesHaveSameSize = false;
4195 }
4196 previousSampleSize = sampleSize;
4197 }
4198 ALOGV("%s timestampUs/lastTimestampUs: %" PRId64 "/%" PRId64,
4199 trackName, timestampUs, lastTimestampUs);
4200 lastDurationUs = timestampUs - lastTimestampUs;
4201 lastDurationTicks = currDurationTicks;
4202 lastTimestampUs = timestampUs;
4203
4204 if (isSync != 0) {
4205 addOneStssTableEntry(mStszTableEntries->count());
4206 }
4207
4208 if (mTrackingProgressStatus) {
4209 if (mPreviousTrackTimeUs <= 0) {
4210 mPreviousTrackTimeUs = mStartTimestampUs;
4211 }
4212 trackProgressStatus(timestampUs);
4213 }
4214 }
4215
4216 if (flags_camera::camera_heif_gainmap() && mOwner->mHasGainmap) {
4217 Mutex::Autolock lock(mOwner->mLock);
4218 size_t bytesWritten;
4219 off64_t offset = mOwner->addSample_l(copy, usePrefix, tiffHdrOffset, &bytesWritten);
4220 addItemOffsetAndSize(offset, bytesWritten, isExif, isGainmapMeta, isGainmap);
4221 copy->release();
4222 copy = NULL;
4223 continue;
4224 }
4225
4226 if (!hasMultipleTracks) {
4227 size_t bytesWritten;
4228 off64_t offset = mOwner->addSample_l(
4229 copy, usePrefix, tiffHdrOffset, &bytesWritten);
4230 if (mIsHeif) {
4231 addItemOffsetAndSize(offset, bytesWritten, isExif);
4232 } else {
4233 if (mCo64TableEntries->count() == 0) {
4234 addChunkOffset(offset);
4235 }
4236 }
4237 copy->release();
4238 copy = NULL;
4239 continue;
4240 }
4241
4242 mChunkSamples.push_back(copy);
4243 if (mIsHeif) {
4244 bufferChunk(0 /*timestampUs*/);
4245 ++nChunks;
4246 } else if (interleaveDurationUs == 0) {
4247 addOneStscTableEntry(++nChunks, 1);
4248 bufferChunk(timestampUs);
4249 } else {
4250 if (chunkTimestampUs == 0) {
4251 chunkTimestampUs = timestampUs;
4252 } else {
4253 int64_t chunkDurationUs = timestampUs - chunkTimestampUs;
4254 if (chunkDurationUs > interleaveDurationUs || lastSample > 1) {
4255 ALOGV("lastSample:%lld", (long long)lastSample);
4256 if (chunkDurationUs > mMaxChunkDurationUs) {
4257 mMaxChunkDurationUs = chunkDurationUs;
4258 }
4259 ++nChunks;
4260 if (nChunks == 1 || // First chunk
4261 lastSamplesPerChunk != mChunkSamples.size()) {
4262 lastSamplesPerChunk = mChunkSamples.size();
4263 addOneStscTableEntry(nChunks, lastSamplesPerChunk);
4264 }
4265 bufferChunk(timestampUs);
4266 chunkTimestampUs = timestampUs;
4267 }
4268 }
4269 }
4270 }
4271
4272 if (isTrackMalFormed()) {
4273 dumpTimeStamps();
4274 err = ERROR_MALFORMED;
4275 }
4276
4277 mOwner->trackProgressStatus(mTrackId.getId(), -1, err);
4278
4279 // Add final entries only for non-empty tracks.
4280 if (mStszTableEntries->count() > 0) {
4281 if (mIsHeif) {
4282 if (!mChunkSamples.empty()) {
4283 bufferChunk(0);
4284 ++nChunks;
4285 }
4286 } else {
4287 // Last chunk
4288 if (!hasMultipleTracks) {
4289 addOneStscTableEntry(1, mStszTableEntries->count());
4290 } else if (!mChunkSamples.empty()) {
4291 addOneStscTableEntry(++nChunks, mChunkSamples.size());
4292 bufferChunk(timestampUs);
4293 }
4294
4295 // We don't really know how long the last frame lasts, since
4296 // there is no frame time after it, just repeat the previous
4297 // frame's duration.
4298 if (mStszTableEntries->count() == 1) {
4299 if (lastSampleDurationUs >= 0) {
4300 addOneSttsTableEntry(sampleCount, lastSampleDurationTicks);
4301 } else {
4302 lastDurationUs = 0; // A single sample's duration
4303 lastDurationTicks = 0;
4304 addOneSttsTableEntry(sampleCount, lastDurationTicks);
4305 }
4306 } else if (lastSampleDurationUs >= 0) {
4307 addOneSttsTableEntry(sampleCount, lastDurationTicks);
4308 addOneSttsTableEntry(1, lastSampleDurationTicks);
4309 } else {
4310 ++sampleCount; // Count for the last sample
4311 addOneSttsTableEntry(sampleCount, lastDurationTicks);
4312 }
4313
4314 // The last ctts box entry may not have been written yet, and this
4315 // is to make sure that we write out the last ctts box entry.
4316 if (currCttsOffsetTimeTicks == lastCttsOffsetTimeTicks) {
4317 if (cttsSampleCount > 0) {
4318 addOneCttsTableEntry(cttsSampleCount, lastCttsOffsetTimeTicks);
4319 }
4320 }
4321 if (lastSampleDurationUs >= 0) {
4322 mTrackDurationUs += lastSampleDurationUs;
4323 } else {
4324 mTrackDurationUs += lastDurationUs;
4325 }
4326 }
4327 }
4328 mReachedEOS = true;
4329
4330 sendTrackSummary(hasMultipleTracks);
4331
4332 ALOGI("Received total/0-length (%d/%d) buffers and encoded %d frames. - %s",
4333 count, nZeroLengthFrames, mStszTableEntries->count(), trackName);
4334 if (mIsAudio) {
4335 ALOGI("Audio track drift time: %" PRId64 " us", mOwner->getDriftTimeUs());
4336 }
4337
4338 if (err == ERROR_END_OF_STREAM) {
4339 return OK;
4340 }
4341 return err;
4342 }
4343
isTrackMalFormed()4344 bool MPEG4Writer::Track::isTrackMalFormed() {
4345 if (mIsMalformed) {
4346 return true;
4347 }
4348
4349 int32_t emptyTrackMalformed = false;
4350 if (mOwner->mStartMeta &&
4351 mOwner->mStartMeta->findInt32(kKeyEmptyTrackMalFormed, &emptyTrackMalformed) &&
4352 emptyTrackMalformed) {
4353 // MediaRecorder(sets kKeyEmptyTrackMalFormed by default) report empty tracks as malformed.
4354 if (!mIsHeif && mStszTableEntries->count() == 0) { // no samples written
4355 ALOGE("The number of recorded samples is 0");
4356 mIsMalformed = true;
4357 return true;
4358 }
4359 if (mIsVideo && mStssTableEntries->count() == 0) { // no sync frames for video
4360 ALOGE("There are no sync frames for video track");
4361 mIsMalformed = true;
4362 return true;
4363 }
4364 } else {
4365 // Through MediaMuxer, empty tracks can be added. No sync frames for video.
4366 if (mIsVideo && mStszTableEntries->count() > 0 && mStssTableEntries->count() == 0) {
4367 ALOGE("There are no sync frames for video track");
4368 mIsMalformed = true;
4369 return true;
4370 }
4371 }
4372 // Don't check for CodecSpecificData when track is empty.
4373 if (mStszTableEntries->count() > 0 && OK != checkCodecSpecificData()) {
4374 // No codec specific data.
4375 mIsMalformed = true;
4376 return true;
4377 }
4378
4379 return false;
4380 }
4381
sendTrackSummary(bool hasMultipleTracks)4382 void MPEG4Writer::Track::sendTrackSummary(bool hasMultipleTracks) {
4383
4384 // Send track summary only if test mode is enabled.
4385 if (!isTestModeEnabled()) {
4386 return;
4387 }
4388
4389 uint32_t trackNum = (mTrackId.getId() << 28);
4390
4391 mOwner->notify(MEDIA_RECORDER_TRACK_EVENT_INFO,
4392 trackNum | MEDIA_RECORDER_TRACK_INFO_TYPE,
4393 mIsAudio ? 0: 1);
4394
4395 mOwner->notify(MEDIA_RECORDER_TRACK_EVENT_INFO,
4396 trackNum | MEDIA_RECORDER_TRACK_INFO_DURATION_MS,
4397 mTrackDurationUs / 1000);
4398
4399 mOwner->notify(MEDIA_RECORDER_TRACK_EVENT_INFO,
4400 trackNum | MEDIA_RECORDER_TRACK_INFO_ENCODED_FRAMES,
4401 mStszTableEntries->count());
4402
4403 {
4404 // The system delay time excluding the requested initial delay that
4405 // is used to eliminate the recording sound.
4406 int64_t startTimeOffsetUs = mOwner->getStartTimeOffsetMs() * 1000LL;
4407 if (startTimeOffsetUs < 0) { // Start time offset was not set
4408 startTimeOffsetUs = kInitialDelayTimeUs;
4409 }
4410 int64_t initialDelayUs =
4411 mFirstSampleTimeRealUs - mStartTimeRealUs - startTimeOffsetUs;
4412
4413 mOwner->notify(MEDIA_RECORDER_TRACK_EVENT_INFO,
4414 trackNum | MEDIA_RECORDER_TRACK_INFO_INITIAL_DELAY_MS,
4415 (initialDelayUs) / 1000);
4416 }
4417
4418 mOwner->notify(MEDIA_RECORDER_TRACK_EVENT_INFO,
4419 trackNum | MEDIA_RECORDER_TRACK_INFO_DATA_KBYTES,
4420 mMdatSizeBytes / 1024);
4421
4422 if (hasMultipleTracks) {
4423 mOwner->notify(MEDIA_RECORDER_TRACK_EVENT_INFO,
4424 trackNum | MEDIA_RECORDER_TRACK_INFO_MAX_CHUNK_DUR_MS,
4425 mMaxChunkDurationUs / 1000);
4426
4427 int64_t moovStartTimeUs = mOwner->getStartTimestampUs();
4428 if (mStartTimestampUs != moovStartTimeUs) {
4429 int64_t startTimeOffsetUs = mStartTimestampUs - moovStartTimeUs;
4430 mOwner->notify(MEDIA_RECORDER_TRACK_EVENT_INFO,
4431 trackNum | MEDIA_RECORDER_TRACK_INFO_START_OFFSET_MS,
4432 startTimeOffsetUs / 1000);
4433 }
4434 }
4435 }
4436
trackProgressStatus(int64_t timeUs,status_t err)4437 void MPEG4Writer::Track::trackProgressStatus(int64_t timeUs, status_t err) {
4438 ALOGV("trackProgressStatus: %" PRId64 " us", timeUs);
4439
4440 if (mTrackEveryTimeDurationUs > 0 &&
4441 timeUs - mPreviousTrackTimeUs >= mTrackEveryTimeDurationUs) {
4442 ALOGV("Fire time tracking progress status at %" PRId64 " us", timeUs);
4443 mOwner->trackProgressStatus(mTrackId.getId(), timeUs - mPreviousTrackTimeUs, err);
4444 mPreviousTrackTimeUs = timeUs;
4445 }
4446 }
4447
trackProgressStatus(uint32_t trackId,int64_t timeUs,status_t err)4448 void MPEG4Writer::trackProgressStatus(
4449 uint32_t trackId, int64_t timeUs, status_t err) {
4450 Mutex::Autolock lock(mLock);
4451 uint32_t trackNum = (trackId << 28);
4452
4453 // Error notification
4454 // Do not consider ERROR_END_OF_STREAM an error
4455 if (err != OK && err != ERROR_END_OF_STREAM) {
4456 notify(MEDIA_RECORDER_TRACK_EVENT_ERROR,
4457 trackNum | MEDIA_RECORDER_TRACK_ERROR_GENERAL,
4458 err);
4459 return;
4460 }
4461
4462 if (timeUs == -1) {
4463 // Send completion notification
4464 notify(MEDIA_RECORDER_TRACK_EVENT_INFO,
4465 trackNum | MEDIA_RECORDER_TRACK_INFO_COMPLETION_STATUS,
4466 err);
4467 } else {
4468 // Send progress status
4469 notify(MEDIA_RECORDER_TRACK_EVENT_INFO,
4470 trackNum | MEDIA_RECORDER_TRACK_INFO_PROGRESS_IN_TIME,
4471 timeUs / 1000);
4472 }
4473 }
4474
setDriftTimeUs(int64_t driftTimeUs)4475 void MPEG4Writer::setDriftTimeUs(int64_t driftTimeUs) {
4476 ALOGV("setDriftTimeUs: %" PRId64 " us", driftTimeUs);
4477 Mutex::Autolock autolock(mLock);
4478 mDriftTimeUs = driftTimeUs;
4479 }
4480
getDriftTimeUs()4481 int64_t MPEG4Writer::getDriftTimeUs() {
4482 ALOGV("getDriftTimeUs: %" PRId64 " us", mDriftTimeUs);
4483 Mutex::Autolock autolock(mLock);
4484 return mDriftTimeUs;
4485 }
4486
isRealTimeRecording() const4487 bool MPEG4Writer::isRealTimeRecording() const {
4488 return mIsRealTimeRecording;
4489 }
4490
isBackgroundMode() const4491 bool MPEG4Writer::isBackgroundMode() const {
4492 return mIsBackgroundMode;
4493 }
4494
useNalLengthFour()4495 bool MPEG4Writer::useNalLengthFour() {
4496 return mUse4ByteNalLength;
4497 }
4498
bufferChunk(int64_t timestampUs)4499 void MPEG4Writer::Track::bufferChunk(int64_t timestampUs) {
4500 ALOGV("bufferChunk");
4501
4502 Chunk chunk(this, timestampUs, mChunkSamples);
4503 mOwner->bufferChunk(chunk);
4504 mChunkSamples.clear();
4505 }
4506
getDurationUs() const4507 int64_t MPEG4Writer::Track::getDurationUs() const {
4508 return mTrackDurationUs + getStartTimeOffsetTimeUs() + mOwner->getStartTimeOffsetBFramesUs();
4509 }
4510
getEstimatedTrackSizeBytes() const4511 int64_t MPEG4Writer::Track::getEstimatedTrackSizeBytes() const {
4512 return mEstimatedTrackSizeBytes;
4513 }
4514
getMetaSizeIncrease(int32_t angle,int32_t trackCount) const4515 int32_t MPEG4Writer::Track::getMetaSizeIncrease(
4516 int32_t angle, int32_t trackCount) const {
4517 CHECK(mIsHeif);
4518
4519 int32_t grid = (mTileWidth > 0);
4520 int32_t rotate = (angle > 0);
4521
4522 // Note that the rotation angle is in the file meta, and we don't have
4523 // it until start, so here the calculation has to assume rotation.
4524
4525 // increase to ipco
4526 int32_t increase = 20 * (grid + 1) // 'ispe' property
4527 + (8 + mCodecSpecificDataSize) // 'hvcC' property
4528 ;
4529
4530 if (rotate) {
4531 increase += 9; // 'irot' property (worst case)
4532 }
4533
4534 if (flags_camera::camera_heif_gainmap()) {
4535 // assume we have HDR gainmap and associated metadata
4536 increase += (8 + mCodecSpecificDataSize) // 'hvcC' property (HDR gainmap)
4537 + (2 * 20) // 'ispe' property
4538 + (2 * 16) // 'pixi' property
4539 + (2 * 19) // 'colr' property
4540 ;
4541 }
4542
4543 // increase to iref and idat
4544 if (grid) {
4545 increase += (12 + mNumTiles * 2) // 'dimg' in iref
4546 + 12; // ImageGrid in 'idat' (worst case)
4547 }
4548
4549 increase += (12 + 2); // 'cdsc' in iref
4550
4551 // increase to iloc, iinf
4552 increase += (16 // increase to 'iloc'
4553 + 21) // increase to 'iinf'
4554 * (mNumTiles + grid + 1); // "+1" is for 'Exif'
4555
4556 if (flags_camera::camera_heif_gainmap()) {
4557 increase += (16 // increase to 'iloc'
4558 + 21) // increase to 'iinf'
4559 * 2; // "2" is for 'tmap', 'gmap'
4560 }
4561
4562 // When total # of properties is > 127, the properties id becomes 2-byte.
4563 // We write 4 properties at most for each image (2x'ispe', 1x'hvcC', 1x'irot').
4564 // Set the threshold to be 30.
4565 int32_t propBytes = trackCount > 30 ? 2 : 1;
4566
4567 // increase to ipma
4568 increase += (3 + 2 * propBytes) * mNumTiles // 'ispe' + 'hvcC'
4569 + grid * (3 + propBytes) // 'ispe' for grid
4570 + rotate * propBytes; // 'irot' (either on grid or tile)
4571
4572 return increase;
4573 }
4574
checkCodecSpecificData() const4575 status_t MPEG4Writer::Track::checkCodecSpecificData() const {
4576 const char *mime;
4577 CHECK(mMeta->findCString(kKeyMIMEType, &mime));
4578 if (!strcasecmp(MEDIA_MIMETYPE_AUDIO_AAC, mime) ||
4579 !strcasecmp(MEDIA_MIMETYPE_VIDEO_MPEG4, mime) ||
4580 !strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime) ||
4581 !strcasecmp(MEDIA_MIMETYPE_VIDEO_HEVC, mime) ||
4582 !strcasecmp(MEDIA_MIMETYPE_VIDEO_AV1, mime) ||
4583 (editing_flags::muxer_mp4_enable_apv() && !strcasecmp(MEDIA_MIMETYPE_VIDEO_APV, mime)) ||
4584 !strcasecmp(MEDIA_MIMETYPE_VIDEO_DOLBY_VISION, mime) ||
4585 !strcasecmp(MEDIA_MIMETYPE_IMAGE_ANDROID_HEIC, mime) ||
4586 !strcasecmp(MEDIA_MIMETYPE_IMAGE_AVIF, mime)) {
4587 if (!mCodecSpecificData ||
4588 mCodecSpecificDataSize <= 0) {
4589 ALOGE("Missing codec specific data");
4590 return ERROR_MALFORMED;
4591 }
4592 } else {
4593 if (mCodecSpecificData ||
4594 mCodecSpecificDataSize > 0) {
4595 ALOGE("Unexepected codec specific data found");
4596 return ERROR_MALFORMED;
4597 }
4598 }
4599 return OK;
4600 }
4601
getTrackType() const4602 const char *MPEG4Writer::Track::getTrackType() const {
4603 return mIsAudio ? "Audio" :
4604 mIsVideo ? "Video" :
4605 mIsHeif ? "Image" :
4606 "Metadata";
4607 }
4608
writeTrackHeader()4609 void MPEG4Writer::Track::writeTrackHeader() {
4610 uint32_t now = getMpeg4Time();
4611 mOwner->beginBox("trak");
4612 writeTkhdBox(now);
4613 writeEdtsBox();
4614 mOwner->beginBox("mdia");
4615 writeMdhdBox(now);
4616 writeHdlrBox();
4617 mOwner->beginBox("minf");
4618 if (mIsAudio) {
4619 writeSmhdBox();
4620 } else if (mIsVideo) {
4621 writeVmhdBox();
4622 } else {
4623 writeNmhdBox();
4624 }
4625 writeDinfBox();
4626 writeStblBox();
4627 mOwner->endBox(); // minf
4628 mOwner->endBox(); // mdia
4629 mOwner->endBox(); // trak
4630 }
4631
getMinCttsOffsetTimeUs()4632 int64_t MPEG4Writer::Track::getMinCttsOffsetTimeUs() {
4633 // For video tracks with ctts table, this should return the minimum ctts
4634 // offset in the table. For non-video tracks or video tracks without ctts
4635 // table, this will return kMaxCttsOffsetTimeUs.
4636 if (mMinCttsOffsetTicks == mMaxCttsOffsetTicks) {
4637 return kMaxCttsOffsetTimeUs;
4638 }
4639 return mMinCttsOffsetTimeUs;
4640 }
4641
writeStblBox()4642 void MPEG4Writer::Track::writeStblBox() {
4643 mOwner->beginBox("stbl");
4644 // Add subboxes for only non-empty and well-formed tracks.
4645 if (mStszTableEntries->count() > 0 && !isTrackMalFormed()) {
4646 mOwner->beginBox("stsd");
4647 mOwner->writeInt32(0); // version=0, flags=0
4648 mOwner->writeInt32(1); // entry count
4649 if (mIsAudio) {
4650 writeAudioFourCCBox();
4651 } else if (mIsVideo) {
4652 writeVideoFourCCBox();
4653 } else {
4654 writeMetadataFourCCBox();
4655 }
4656 mOwner->endBox(); // stsd
4657 writeSttsBox();
4658 if (mIsVideo) {
4659 writeCttsBox();
4660 writeStssBox();
4661 }
4662 writeStszBox();
4663 writeStscBox();
4664 writeCo64Box();
4665 }
4666 mOwner->endBox(); // stbl
4667 }
4668
writeMetadataFourCCBox()4669 void MPEG4Writer::Track::writeMetadataFourCCBox() {
4670 const char *mime;
4671 bool success = mMeta->findCString(kKeyMIMEType, &mime);
4672 CHECK(success);
4673 const char *fourcc = getFourCCForMime(mime);
4674 if (fourcc == NULL) {
4675 ALOGE("Unknown mime type '%s'.", mime);
4676 TRESPASS();
4677 }
4678 mOwner->beginBox(fourcc); // TextMetaDataSampleEntry
4679
4680 // HACK to make the metadata track compliant with the ISO standard.
4681 //
4682 // Metadata track is added from API 26 and the original implementation does not
4683 // fully followed the TextMetaDataSampleEntry specified in ISO/IEC 14496-12-2015
4684 // in that only the mime_format is written out. content_encoding and
4685 // data_reference_index have not been written out. This leads to the failure
4686 // when some MP4 parser tries to parse the metadata track according to the
4687 // standard. The hack here will make the metadata track compliant with the
4688 // standard while still maintaining backwards compatibility. This would enable
4689 // Android versions before API 29 to be able to read out the standard compliant
4690 // Metadata track generated with Android API 29 and upward. The trick is based
4691 // on the fact that the Metadata track must start with prefix “application/” and
4692 // those missing fields are not used in Android's Metadata track. By writting
4693 // out the mime_format twice, the first mime_format will be used to fill out the
4694 // missing reserved, data_reference_index and content encoding fields. On the
4695 // parser side, the extracter before API 29 will read out the first mime_format
4696 // correctly and drop the second mime_format. The extractor from API 29 will
4697 // check if the reserved, data_reference_index and content encoding are filled
4698 // with “application” to detect if this is a standard compliant metadata track
4699 // and read out the data accordingly.
4700 mOwner->writeCString(mime);
4701
4702 mOwner->writeCString(mime); // metadata mime_format
4703 mOwner->endBox(); // mett
4704 }
4705
writeVideoFourCCBox()4706 void MPEG4Writer::Track::writeVideoFourCCBox() {
4707 const char *mime;
4708 bool success = mMeta->findCString(kKeyMIMEType, &mime);
4709 CHECK(success);
4710 const char *fourcc;
4711 if (!strcmp(mime, MEDIA_MIMETYPE_VIDEO_DOLBY_VISION)) {
4712 fourcc = getDoviFourCC();
4713 } else {
4714 fourcc = getFourCCForMime(mime);
4715 }
4716
4717 if (fourcc == NULL) {
4718 ALOGE("Unknown mime type '%s'.", mime);
4719 TRESPASS();
4720 }
4721
4722 mOwner->beginBox(fourcc); // video format
4723 mOwner->writeInt32(0); // reserved
4724 mOwner->writeInt16(0); // reserved
4725 mOwner->writeInt16(1); // data ref index
4726 mOwner->writeInt16(0); // predefined
4727 mOwner->writeInt16(0); // reserved
4728 mOwner->writeInt32(0); // predefined
4729 mOwner->writeInt32(0); // predefined
4730 mOwner->writeInt32(0); // predefined
4731
4732 int32_t width, height;
4733 success = mMeta->findInt32(kKeyWidth, &width);
4734 success = success && mMeta->findInt32(kKeyHeight, &height);
4735 CHECK(success);
4736
4737 mOwner->writeInt16(width);
4738 mOwner->writeInt16(height);
4739 mOwner->writeInt32(0x480000); // horiz resolution
4740 mOwner->writeInt32(0x480000); // vert resolution
4741 mOwner->writeInt32(0); // reserved
4742 mOwner->writeInt16(1); // frame count
4743 mOwner->writeInt8(0); // compressor string length
4744 mOwner->write(" ", 31);
4745 mOwner->writeInt16(0x18); // depth
4746 mOwner->writeInt16(-1); // predefined
4747
4748 if (!strcasecmp(MEDIA_MIMETYPE_VIDEO_MPEG4, mime)) {
4749 writeMp4vEsdsBox();
4750 } else if (!strcasecmp(MEDIA_MIMETYPE_VIDEO_H263, mime)) {
4751 writeD263Box();
4752 } else if (!strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime)) {
4753 writeAvccBox();
4754 } else if (!strcasecmp(MEDIA_MIMETYPE_VIDEO_HEVC, mime)) {
4755 writeHvccBox();
4756 } else if (!strcasecmp(MEDIA_MIMETYPE_VIDEO_AV1, mime)) {
4757 writeAv1cBox();
4758 } else if (editing_flags::muxer_mp4_enable_apv() &&
4759 !strcasecmp(MEDIA_MIMETYPE_VIDEO_APV, mime)) {
4760 writeApvcBox();
4761 } else if (!strcasecmp(MEDIA_MIMETYPE_VIDEO_DOLBY_VISION, mime)) {
4762 if (mDoviProfile <= DolbyVisionProfileDvheSt) {
4763 writeHvccBox();
4764 } else if (mDoviProfile == DolbyVisionProfileDvavSe) {
4765 writeAvccBox();
4766 } else {
4767 TRESPASS("Unsupported Dolby Vision profile");
4768 }
4769 writeDoviConfigBox();
4770 }
4771
4772 writePaspBox();
4773 writeColrBox();
4774 writeMdcvAndClliBoxes();
4775 mOwner->endBox(); // mp4v, s263 or avc1
4776 }
4777
writeColrBox()4778 void MPEG4Writer::Track::writeColrBox() {
4779 ColorAspects aspects;
4780 memset(&aspects, 0, sizeof(aspects));
4781 // Color metadata may have changed.
4782 sp<MetaData> meta = mSource->getFormat();
4783 bool findPrimaries = meta->findInt32(kKeyColorPrimaries, (int32_t*)&aspects.mPrimaries);
4784 bool findTransfer = meta->findInt32(kKeyTransferFunction, (int32_t*)&aspects.mTransfer);
4785 bool findMatrix = meta->findInt32(kKeyColorMatrix, (int32_t*)&aspects.mMatrixCoeffs);
4786 bool findRange = meta->findInt32(kKeyColorRange, (int32_t*)&aspects.mRange);
4787 if (!findPrimaries && !findTransfer && !findMatrix && !findRange) {
4788 ALOGV("no color information");
4789 return;
4790 }
4791
4792 int32_t primaries, transfer, coeffs;
4793 bool fullRange;
4794 ALOGV("primaries=%s transfer=%s matrix=%s range=%s",
4795 asString(aspects.mPrimaries),
4796 asString(aspects.mTransfer),
4797 asString(aspects.mMatrixCoeffs),
4798 asString(aspects.mRange));
4799 ColorUtils::convertCodecColorAspectsToIsoAspects(
4800 aspects, &primaries, &transfer, &coeffs, &fullRange);
4801 mOwner->beginBox("colr");
4802 mOwner->writeFourcc("nclx");
4803 mOwner->writeInt16(primaries);
4804 mOwner->writeInt16(transfer);
4805 mOwner->writeInt16(coeffs);
4806 mOwner->writeInt8(int8_t(fullRange ? 0x80 : 0x0));
4807 mOwner->endBox(); // colr
4808 }
4809
writeMdcvAndClliBoxes()4810 void MPEG4Writer::Track::writeMdcvAndClliBoxes() {
4811 sp<MetaData> meta = mSource->getFormat();
4812 uint32_t type;
4813 const uint8_t* data;
4814 size_t size;
4815 bool found =
4816 meta->findData(kKeyHdrStaticInfo, &type, reinterpret_cast<const void**>(&data), &size);
4817 if (!found) {
4818 return; // Nothing to encode.
4819 }
4820 if (size != 25) {
4821 ALOGW("Ignoring HDR static info with unexpected size %d", (int)size);
4822 return;
4823 }
4824 uint16_t displayPrimariesRX = U16LE_AT(&data[1]);
4825 uint16_t displayPrimariesRY = U16LE_AT(&data[3]);
4826
4827 uint16_t displayPrimariesGX = U16LE_AT(&data[5]);
4828 uint16_t displayPrimariesGY = U16LE_AT(&data[7]);
4829
4830 uint16_t displayPrimariesBX = U16LE_AT(&data[9]);
4831 uint16_t displayPrimariesBY = U16LE_AT(&data[11]);
4832
4833 uint16_t whitePointX = U16LE_AT(&data[13]);
4834 uint16_t whitePointY = U16LE_AT(&data[15]);
4835
4836 uint16_t maxDisplayMasteringLuminance = U16LE_AT(&data[17]);
4837 uint16_t minDisplayMasteringLuminance = U16LE_AT(&data[19]);
4838
4839 uint16_t maxContentLightLevel = U16LE_AT(&data[21]);
4840 uint16_t maxPicAverageLightLevel = U16LE_AT(&data[23]);
4841
4842 mOwner->beginBox("mdcv");
4843 mOwner->writeInt16(displayPrimariesGX);
4844 mOwner->writeInt16(displayPrimariesGY);
4845 mOwner->writeInt16(displayPrimariesBX);
4846 mOwner->writeInt16(displayPrimariesBY);
4847 mOwner->writeInt16(displayPrimariesRX);
4848 mOwner->writeInt16(displayPrimariesRY);
4849 mOwner->writeInt16(whitePointX);
4850 mOwner->writeInt16(whitePointY);
4851 mOwner->writeInt32(maxDisplayMasteringLuminance * 10000);
4852 mOwner->writeInt32(minDisplayMasteringLuminance * 10000);
4853 mOwner->endBox(); // mdcv.
4854
4855 mOwner->beginBox("clli");
4856 mOwner->writeInt16(maxContentLightLevel);
4857 mOwner->writeInt16(maxPicAverageLightLevel);
4858 mOwner->endBox(); // clli.
4859 }
4860
writeAudioFourCCBox()4861 void MPEG4Writer::Track::writeAudioFourCCBox() {
4862 const char *mime;
4863 bool success = mMeta->findCString(kKeyMIMEType, &mime);
4864 CHECK(success);
4865 const char *fourcc = getFourCCForMime(mime);
4866 if (fourcc == NULL) {
4867 ALOGE("Unknown mime type '%s'.", mime);
4868 TRESPASS();
4869 }
4870
4871 mOwner->beginBox(fourcc); // audio format
4872 mOwner->writeInt32(0); // reserved
4873 mOwner->writeInt16(0); // reserved
4874 mOwner->writeInt16(0x1); // data ref index
4875 mOwner->writeInt32(0); // reserved
4876 mOwner->writeInt32(0); // reserved
4877 int32_t nChannels;
4878 CHECK_EQ(true, mMeta->findInt32(kKeyChannelCount, &nChannels));
4879 mOwner->writeInt16(nChannels); // channel count
4880 mOwner->writeInt16(16); // sample size
4881 mOwner->writeInt16(0); // predefined
4882 mOwner->writeInt16(0); // reserved
4883
4884 int32_t samplerate;
4885 success = mMeta->findInt32(kKeySampleRate, &samplerate);
4886 CHECK(success);
4887 mOwner->writeInt32(samplerate << 16);
4888 if (!strcasecmp(MEDIA_MIMETYPE_AUDIO_AAC, mime)) {
4889 writeMp4aEsdsBox();
4890 } else if (!strcasecmp(MEDIA_MIMETYPE_AUDIO_AMR_NB, mime) ||
4891 !strcasecmp(MEDIA_MIMETYPE_AUDIO_AMR_WB, mime)) {
4892 writeDamrBox();
4893 }
4894 mOwner->endBox();
4895 }
4896
generateEsdsSize(size_t dataLength,size_t * sizeGenerated,uint8_t * buffer)4897 static void generateEsdsSize(size_t dataLength, size_t* sizeGenerated, uint8_t* buffer) {
4898 size_t offset = 0, cur = 0;
4899 size_t more = 0x00;
4900 *sizeGenerated = 0;
4901 /* Start with the LSB(7 bits) of dataLength and build the byte sequence upto MSB.
4902 * Continuation flag(most significant bit) will be set on the first N-1 bytes.
4903 */
4904 do {
4905 buffer[cur++] = (dataLength & 0x7f) | more;
4906 dataLength >>= 7;
4907 more = 0x80;
4908 ++(*sizeGenerated);
4909 } while (dataLength > 0u);
4910 --cur;
4911 // Reverse the newly formed byte sequence.
4912 while (cur > offset) {
4913 uint8_t tmp = buffer[cur];
4914 buffer[cur--] = buffer[offset];
4915 buffer[offset++] = tmp;
4916 }
4917 }
4918
writeMp4aEsdsBox()4919 void MPEG4Writer::Track::writeMp4aEsdsBox() {
4920 CHECK(mCodecSpecificData);
4921 CHECK_GT(mCodecSpecificDataSize, 0u);
4922
4923 uint8_t sizeESDBuffer[kESDSScratchBufferSize];
4924 uint8_t sizeDCDBuffer[kESDSScratchBufferSize];
4925 uint8_t sizeDSIBuffer[kESDSScratchBufferSize];
4926 size_t sizeESD = 0;
4927 size_t sizeDCD = 0;
4928 size_t sizeDSI = 0;
4929 generateEsdsSize(mCodecSpecificDataSize, &sizeDSI, sizeDSIBuffer);
4930 generateEsdsSize(mCodecSpecificDataSize + sizeDSI + 14, &sizeDCD, sizeDCDBuffer);
4931 generateEsdsSize(mCodecSpecificDataSize + sizeDSI + sizeDCD + 21, &sizeESD, sizeESDBuffer);
4932
4933 mOwner->beginBox("esds");
4934
4935 mOwner->writeInt32(0); // version=0, flags=0
4936 mOwner->writeInt8(0x03); // ES_DescrTag
4937 mOwner->write(sizeESDBuffer, sizeESD);
4938 mOwner->writeInt16(0x0000);// ES_ID
4939 mOwner->writeInt8(0x00);
4940
4941 mOwner->writeInt8(0x04); // DecoderConfigDescrTag
4942 mOwner->write(sizeDCDBuffer, sizeDCD);
4943 mOwner->writeInt8(0x40); // objectTypeIndication ISO/IEC 14492-2
4944 mOwner->writeInt8(0x15); // streamType AudioStream
4945
4946 mOwner->writeInt16(0x03); // XXX
4947 mOwner->writeInt8(0x00); // buffer size 24-bit (0x300)
4948
4949 int32_t avgBitrate = 0;
4950 (void)mMeta->findInt32(kKeyBitRate, &avgBitrate);
4951 int32_t maxBitrate = 0;
4952 (void)mMeta->findInt32(kKeyMaxBitRate, &maxBitrate);
4953 mOwner->writeInt32(maxBitrate);
4954 mOwner->writeInt32(avgBitrate);
4955
4956 mOwner->writeInt8(0x05); // DecoderSpecificInfoTag
4957 mOwner->write(sizeDSIBuffer, sizeDSI);
4958 mOwner->write(mCodecSpecificData, mCodecSpecificDataSize);
4959
4960 static const uint8_t kData2[] = {
4961 0x06, // SLConfigDescriptorTag
4962 0x01,
4963 0x02
4964 };
4965 mOwner->write(kData2, sizeof(kData2));
4966
4967 mOwner->endBox(); // esds
4968 }
4969
writeMp4vEsdsBox()4970 void MPEG4Writer::Track::writeMp4vEsdsBox() {
4971 CHECK(mCodecSpecificData);
4972 CHECK_GT(mCodecSpecificDataSize, 0u);
4973
4974 uint8_t sizeESDBuffer[kESDSScratchBufferSize];
4975 uint8_t sizeDCDBuffer[kESDSScratchBufferSize];
4976 uint8_t sizeDSIBuffer[kESDSScratchBufferSize];
4977 size_t sizeESD = 0;
4978 size_t sizeDCD = 0;
4979 size_t sizeDSI = 0;
4980 generateEsdsSize(mCodecSpecificDataSize, &sizeDSI, sizeDSIBuffer);
4981 generateEsdsSize(mCodecSpecificDataSize + sizeDSI + 14, &sizeDCD, sizeDCDBuffer);
4982 generateEsdsSize(mCodecSpecificDataSize + sizeDSI + sizeDCD + 21, &sizeESD, sizeESDBuffer);
4983
4984 mOwner->beginBox("esds");
4985
4986 mOwner->writeInt32(0); // version=0, flags=0
4987
4988 mOwner->writeInt8(0x03); // ES_DescrTag
4989 mOwner->write(sizeESDBuffer, sizeESD);
4990 mOwner->writeInt16(0x0000); // ES_ID
4991 mOwner->writeInt8(0x1f);
4992
4993 mOwner->writeInt8(0x04); // DecoderConfigDescrTag
4994 mOwner->write(sizeDCDBuffer, sizeDCD);
4995 mOwner->writeInt8(0x20); // objectTypeIndication ISO/IEC 14492-2
4996 mOwner->writeInt8(0x11); // streamType VisualStream
4997
4998 static const uint8_t kData[] = {
4999 0x01, 0x77, 0x00, // buffer size 96000 bytes
5000 };
5001 mOwner->write(kData, sizeof(kData));
5002
5003 int32_t avgBitrate = 0;
5004 (void)mMeta->findInt32(kKeyBitRate, &avgBitrate);
5005 int32_t maxBitrate = 0;
5006 (void)mMeta->findInt32(kKeyMaxBitRate, &maxBitrate);
5007 mOwner->writeInt32(maxBitrate);
5008 mOwner->writeInt32(avgBitrate);
5009
5010 mOwner->writeInt8(0x05); // DecoderSpecificInfoTag
5011
5012 mOwner->write(sizeDSIBuffer, sizeDSI);
5013 mOwner->write(mCodecSpecificData, mCodecSpecificDataSize);
5014
5015 static const uint8_t kData2[] = {
5016 0x06, // SLConfigDescriptorTag
5017 0x01,
5018 0x02
5019 };
5020 mOwner->write(kData2, sizeof(kData2));
5021
5022 mOwner->endBox(); // esds
5023 }
5024
writeTkhdBox(uint32_t now)5025 void MPEG4Writer::Track::writeTkhdBox(uint32_t now) {
5026 mOwner->beginBox("tkhd");
5027 // Flags = 7 to indicate that the track is enabled, and
5028 // part of the presentation
5029 mOwner->writeInt32(0x07); // version=0, flags=7
5030 mOwner->writeInt32(now); // creation time
5031 mOwner->writeInt32(now); // modification time
5032 mOwner->writeInt32(mTrackId.getId()); // track id starts with 1
5033 mOwner->writeInt32(0); // reserved
5034 int64_t trakDurationUs = getDurationUs();
5035 int32_t mvhdTimeScale = mOwner->getTimeScale();
5036 int32_t tkhdDuration =
5037 (trakDurationUs * mvhdTimeScale + 5E5) / 1E6;
5038 mOwner->writeInt32(tkhdDuration); // in mvhd timescale
5039 mOwner->writeInt32(0); // reserved
5040 mOwner->writeInt32(0); // reserved
5041 mOwner->writeInt16(0); // layer
5042 mOwner->writeInt16(0); // alternate group
5043 mOwner->writeInt16(mIsAudio ? 0x100 : 0); // volume
5044 mOwner->writeInt16(0); // reserved
5045
5046 mOwner->writeCompositionMatrix(mRotation); // matrix
5047
5048 if (!mIsVideo) {
5049 mOwner->writeInt32(0);
5050 mOwner->writeInt32(0);
5051 } else {
5052 int32_t width, height;
5053 bool success = mMeta->findInt32(kKeyDisplayWidth, &width);
5054 success = success && mMeta->findInt32(kKeyDisplayHeight, &height);
5055
5056 // Use width/height if display width/height are not present.
5057 if (!success) {
5058 success = mMeta->findInt32(kKeyWidth, &width);
5059 success = success && mMeta->findInt32(kKeyHeight, &height);
5060 }
5061 CHECK(success);
5062
5063 mOwner->writeInt32(width << 16); // 32-bit fixed-point value
5064 mOwner->writeInt32(height << 16); // 32-bit fixed-point value
5065 }
5066 mOwner->endBox(); // tkhd
5067 }
5068
writeVmhdBox()5069 void MPEG4Writer::Track::writeVmhdBox() {
5070 mOwner->beginBox("vmhd");
5071 mOwner->writeInt32(0x01); // version=0, flags=1
5072 mOwner->writeInt16(0); // graphics mode
5073 mOwner->writeInt16(0); // opcolor
5074 mOwner->writeInt16(0);
5075 mOwner->writeInt16(0);
5076 mOwner->endBox();
5077 }
5078
writeSmhdBox()5079 void MPEG4Writer::Track::writeSmhdBox() {
5080 mOwner->beginBox("smhd");
5081 mOwner->writeInt32(0); // version=0, flags=0
5082 mOwner->writeInt16(0); // balance
5083 mOwner->writeInt16(0); // reserved
5084 mOwner->endBox();
5085 }
5086
writeNmhdBox()5087 void MPEG4Writer::Track::writeNmhdBox() {
5088 mOwner->beginBox("nmhd");
5089 mOwner->writeInt32(0); // version=0, flags=0
5090 mOwner->endBox();
5091 }
5092
writeHdlrBox()5093 void MPEG4Writer::Track::writeHdlrBox() {
5094 mOwner->beginBox("hdlr");
5095 mOwner->writeInt32(0); // version=0, flags=0
5096 mOwner->writeInt32(0); // component type: should be mhlr
5097 mOwner->writeFourcc(mIsAudio ? "soun" : (mIsVideo ? "vide" : "meta")); // component subtype
5098 mOwner->writeInt32(0); // reserved
5099 mOwner->writeInt32(0); // reserved
5100 mOwner->writeInt32(0); // reserved
5101 // Removing "r" for the name string just makes the string 4 byte aligned
5102 mOwner->writeCString(mIsAudio ? "SoundHandle": (mIsVideo ? "VideoHandle" : "MetadHandle"));
5103 mOwner->endBox();
5104 }
5105
writeEdtsBox()5106 void MPEG4Writer::Track::writeEdtsBox() {
5107 ALOGV("%s : getStartTimeOffsetTimeUs of track:%" PRId64 " us", getTrackType(),
5108 getStartTimeOffsetTimeUs());
5109
5110 int32_t mvhdTimeScale = mOwner->getTimeScale();
5111 ALOGV("mvhdTimeScale:%" PRId32, mvhdTimeScale);
5112 /* trackStartOffsetUs of this track is the sum of longest offset needed by a track among all
5113 * tracks with B frames in this movie and the start offset of this track.
5114 */
5115 int64_t trackStartOffsetUs = getStartTimeOffsetTimeUs();
5116 ALOGV("trackStartOffsetUs:%" PRIu64, trackStartOffsetUs);
5117
5118 // Longest offset needed by a track among all tracks with B frames.
5119 int32_t movieStartOffsetBFramesUs = mOwner->getStartTimeOffsetBFramesUs();
5120 ALOGV("movieStartOffsetBFramesUs:%" PRId32, movieStartOffsetBFramesUs);
5121
5122 // This media/track's real duration (sum of duration of all samples in this track).
5123 uint32_t tkhdDurationTicks = (mTrackDurationUs * mvhdTimeScale + 5E5) / 1E6;
5124 ALOGV("mTrackDurationUs:%" PRId64 "us", mTrackDurationUs);
5125
5126 int64_t movieStartTimeUs = mOwner->getStartTimestampUs();
5127 ALOGV("movieStartTimeUs:%" PRId64, movieStartTimeUs);
5128
5129 int64_t trackStartTimeUs = movieStartTimeUs + trackStartOffsetUs;
5130 ALOGV("trackStartTimeUs:%" PRId64, trackStartTimeUs);
5131
5132 if (movieStartOffsetBFramesUs == 0) {
5133 // No B frames in any tracks.
5134 if (trackStartOffsetUs > 0) {
5135 // Track with positive start offset.
5136 uint32_t segDuration = (trackStartOffsetUs * mvhdTimeScale + 5E5) / 1E6;
5137 ALOGV("segDuration:%" PRIu64 "us", trackStartOffsetUs);
5138 /* The first entry is an empty edit (indicated by media_time equal to -1), and its
5139 * duration (segment_duration) is equal to the difference of the presentation times of
5140 * the earliest media sample among all tracks and the earliest media sample of the track.
5141 */
5142 ALOGV("Empty edit list entry");
5143 addOneElstTableEntry(segDuration, -1, 1, 0);
5144 addOneElstTableEntry(tkhdDurationTicks, 0, 1, 0);
5145 } else if (mFirstSampleStartOffsetUs > 0) {
5146 // Track with start time < 0 / negative start offset.
5147 ALOGV("Normal edit list entry");
5148 int32_t mediaTime = (mFirstSampleStartOffsetUs * mTimeScale + 5E5) / 1E6;
5149 int32_t firstSampleOffsetTicks =
5150 (mFirstSampleStartOffsetUs * mvhdTimeScale + 5E5) / 1E6;
5151 if (tkhdDurationTicks >= firstSampleOffsetTicks) {
5152 // samples before 0 don't count in for duration, hence subtract
5153 // firstSampleOffsetTicks.
5154 addOneElstTableEntry(tkhdDurationTicks - firstSampleOffsetTicks, mediaTime, 1, 0);
5155 } else {
5156 ALOGW("The track header duration %" PRId64
5157 " is smaller than the first sample offset %" PRId64,
5158 mTrackDurationUs, mFirstSampleStartOffsetUs);
5159 }
5160 } else {
5161 // Track starting at zero.
5162 ALOGV("No edit list entry required for this track");
5163 }
5164 } else if (movieStartOffsetBFramesUs < 0) {
5165 // B frames present in at least one of the tracks.
5166 ALOGV("writeEdtsBox - Reordered frames(B frames) present");
5167 if (trackStartOffsetUs == std::abs(movieStartOffsetBFramesUs)) {
5168 // Track starting at 0, no start offset.
5169 // TODO : need to take care of mFirstSampleStartOffsetUs > 0 and trackStartOffsetUs > 0
5170 // separately
5171 if (mMinCttsOffsetTicks == mMaxCttsOffsetTicks) {
5172 // Video with no B frame or non-video track.
5173 if (mFirstSampleStartOffsetUs > 0) {
5174 // Track with start time < 0 / negative start offset.
5175 ALOGV("Normal edit list entry");
5176 ALOGV("mFirstSampleStartOffsetUs:%" PRId64 "us", mFirstSampleStartOffsetUs);
5177 int32_t mediaTimeTicks = (mFirstSampleStartOffsetUs * mTimeScale + 5E5) / 1E6;
5178 int32_t firstSampleOffsetTicks =
5179 (mFirstSampleStartOffsetUs * mvhdTimeScale + 5E5) / 1E6;
5180 // Samples before 0 don't count for duration, subtract firstSampleOffsetTicks.
5181 addOneElstTableEntry(tkhdDurationTicks - firstSampleOffsetTicks, mediaTimeTicks,
5182 1, 0);
5183 }
5184 } else {
5185 // Track with B Frames.
5186 int32_t mediaTimeTicks = (trackStartOffsetUs * mTimeScale + 5E5) / 1E6;
5187 ALOGV("mediaTime:%" PRId64 "us", trackStartOffsetUs);
5188 ALOGV("Normal edit list entry to negate start offset by B Frames in others tracks");
5189 addOneElstTableEntry(tkhdDurationTicks, mediaTimeTicks, 1, 0);
5190 }
5191 } else if (trackStartOffsetUs > std::abs(movieStartOffsetBFramesUs)) {
5192 // Track with start offset.
5193 ALOGV("Tracks starting > 0");
5194 int32_t editDurationTicks = 0;
5195 int32_t trackStartOffsetBFramesUs = getMinCttsOffsetTimeUs() - kMaxCttsOffsetTimeUs;
5196 ALOGV("trackStartOffsetBFramesUs:%" PRId32, trackStartOffsetBFramesUs);
5197 if (mMinCttsOffsetTicks == mMaxCttsOffsetTicks) {
5198 // Video with no B frame or non-video track.
5199 editDurationTicks =
5200 ((trackStartOffsetUs + movieStartOffsetBFramesUs) * mvhdTimeScale + 5E5) /
5201 1E6;
5202 ALOGV("editDuration:%" PRId64 "us", (trackStartOffsetUs + movieStartOffsetBFramesUs));
5203 } else {
5204 // Track with B frame.
5205 editDurationTicks =
5206 ((trackStartOffsetUs + movieStartOffsetBFramesUs +
5207 trackStartOffsetBFramesUs) * mvhdTimeScale + 5E5) / 1E6;
5208 ALOGV("editDuration:%" PRId64 "us", (trackStartOffsetUs + movieStartOffsetBFramesUs + trackStartOffsetBFramesUs));
5209 }
5210 ALOGV("editDurationTicks:%" PRIu32, editDurationTicks);
5211 if (editDurationTicks > 0) {
5212 ALOGV("Empty edit list entry");
5213 addOneElstTableEntry(editDurationTicks, -1, 1, 0);
5214 addOneElstTableEntry(tkhdDurationTicks, 0, 1, 0);
5215 } else if (editDurationTicks < 0) {
5216 // Only video tracks with B Frames would hit this case.
5217 ALOGV("Edit list entry to negate start offset by B frames in other tracks");
5218 if (com::android::media::editing::flags::
5219 stagefrightrecorder_enable_b_frames()) {
5220 int32_t mediaTimeTicks =
5221 ((trackStartOffsetUs + movieStartOffsetBFramesUs +
5222 trackStartOffsetBFramesUs) * mTimeScale - 5E5) / 1E6;
5223 addOneElstTableEntry(tkhdDurationTicks, std::abs(mediaTimeTicks), 1, 0);
5224 } else {
5225 addOneElstTableEntry(tkhdDurationTicks, std::abs(editDurationTicks), 1, 0);
5226 }
5227 } else {
5228 ALOGV("No edit list entry needed for this track");
5229 }
5230 } else {
5231 // Not expecting this case as we adjust negative start timestamps to zero.
5232 ALOGW("trackStartOffsetUs < std::abs(movieStartOffsetBFramesUs)");
5233 }
5234 } else {
5235 // Neither B frames present nor absent! or any other case?.
5236 ALOGW("movieStartOffsetBFramesUs > 0");
5237 }
5238
5239 if (mElstTableEntries->count() == 0) {
5240 return;
5241 }
5242
5243 mOwner->beginBox("edts");
5244 mOwner->beginBox("elst");
5245 mOwner->writeInt32(0); // version=0, flags=0
5246 mElstTableEntries->write(mOwner);
5247 mOwner->endBox(); // elst;
5248 mOwner->endBox(); // edts
5249 }
5250
writeMdhdBox(uint32_t now)5251 void MPEG4Writer::Track::writeMdhdBox(uint32_t now) {
5252 int64_t trakDurationUs = getDurationUs();
5253 int64_t mdhdDuration = (trakDurationUs * mTimeScale + 5E5) / 1E6;
5254 mOwner->beginBox("mdhd");
5255
5256 if (mdhdDuration > UINT32_MAX) {
5257 mOwner->writeInt32((1 << 24)); // version=1, flags=0
5258 mOwner->writeInt64((int64_t)now); // creation time
5259 mOwner->writeInt64((int64_t)now); // modification time
5260 mOwner->writeInt32(mTimeScale); // media timescale
5261 mOwner->writeInt64(mdhdDuration); // media timescale
5262 } else {
5263 mOwner->writeInt32(0); // version=0, flags=0
5264 mOwner->writeInt32(now); // creation time
5265 mOwner->writeInt32(now); // modification time
5266 mOwner->writeInt32(mTimeScale); // media timescale
5267 mOwner->writeInt32((int32_t)mdhdDuration); // use media timescale
5268 }
5269 // Language follows the three letter standard ISO-639-2/T
5270 // 'e', 'n', 'g' for "English", for instance.
5271 // Each character is packed as the difference between its ASCII value and 0x60.
5272 // For "English", these are 00101, 01110, 00111.
5273 // XXX: Where is the padding bit located: 0x15C7?
5274 const char *lang = NULL;
5275 int16_t langCode = 0;
5276 if (mMeta->findCString(kKeyMediaLanguage, &lang) && lang && strnlen(lang, 3) > 2) {
5277 langCode = ((lang[0] & 0x1f) << 10) | ((lang[1] & 0x1f) << 5) | (lang[2] & 0x1f);
5278 }
5279 mOwner->writeInt16(langCode); // language code
5280 mOwner->writeInt16(0); // predefined
5281 mOwner->endBox();
5282 }
5283
writeDamrBox()5284 void MPEG4Writer::Track::writeDamrBox() {
5285 // 3gpp2 Spec AMRSampleEntry fields
5286 mOwner->beginBox("damr");
5287 mOwner->writeCString(" "); // vendor: 4 bytes
5288 mOwner->writeInt8(0); // decoder version
5289 mOwner->writeInt16(0x83FF); // mode set: all enabled
5290 mOwner->writeInt8(0); // mode change period
5291 mOwner->writeInt8(1); // frames per sample
5292 mOwner->endBox();
5293 }
5294
writeUrlBox()5295 void MPEG4Writer::Track::writeUrlBox() {
5296 // The table index here refers to the sample description index
5297 // in the sample table entries.
5298 mOwner->beginBox("url ");
5299 mOwner->writeInt32(1); // version=0, flags=1 (self-contained)
5300 mOwner->endBox(); // url
5301 }
5302
writeDrefBox()5303 void MPEG4Writer::Track::writeDrefBox() {
5304 mOwner->beginBox("dref");
5305 mOwner->writeInt32(0); // version=0, flags=0
5306 mOwner->writeInt32(1); // entry count (either url or urn)
5307 writeUrlBox();
5308 mOwner->endBox(); // dref
5309 }
5310
writeDinfBox()5311 void MPEG4Writer::Track::writeDinfBox() {
5312 mOwner->beginBox("dinf");
5313 writeDrefBox();
5314 mOwner->endBox(); // dinf
5315 }
5316
writeAvccBox()5317 void MPEG4Writer::Track::writeAvccBox() {
5318 CHECK(mCodecSpecificData);
5319 CHECK_GE(mCodecSpecificDataSize, 5u);
5320
5321 // Patch avcc's lengthSize field to match the number
5322 // of bytes we use to indicate the size of a nal unit.
5323 uint8_t *ptr = (uint8_t *)mCodecSpecificData;
5324 ptr[4] = (ptr[4] & 0xfc) | (mOwner->useNalLengthFour() ? 3 : 1);
5325 mOwner->beginBox("avcC");
5326 mOwner->write(mCodecSpecificData, mCodecSpecificDataSize);
5327 mOwner->endBox(); // avcC
5328 }
5329
writeHvccBox()5330 void MPEG4Writer::Track::writeHvccBox() {
5331 CHECK(mCodecSpecificData);
5332 CHECK_GE(mCodecSpecificDataSize, 5u);
5333
5334 // Patch hvcc's lengthSize field to match the number
5335 // of bytes we use to indicate the size of a nal unit.
5336 uint8_t *ptr = (uint8_t *)mCodecSpecificData;
5337 ptr[21] = (ptr[21] & 0xfc) | (mOwner->useNalLengthFour() ? 3 : 1);
5338 mOwner->beginBox("hvcC");
5339 mOwner->write(mCodecSpecificData, mCodecSpecificDataSize);
5340 mOwner->endBox(); // hvcC
5341 }
5342
writeAv1cBox()5343 void MPEG4Writer::Track::writeAv1cBox() {
5344 CHECK(mCodecSpecificData);
5345 CHECK_GE(mCodecSpecificDataSize, 4u);
5346
5347 mOwner->beginBox("av1C");
5348 mOwner->write(mCodecSpecificData, mCodecSpecificDataSize);
5349 mOwner->endBox(); // av1C
5350 }
5351
writeApvcBox()5352 void MPEG4Writer::Track::writeApvcBox() {
5353 CHECK(mCodecSpecificData);
5354 CHECK_GE(mCodecSpecificDataSize, 4u);
5355
5356 mOwner->beginBox("apvC");
5357 mOwner->write(mCodecSpecificData, mCodecSpecificDataSize);
5358 mOwner->endBox(); // apvC
5359 }
5360
writeDoviConfigBox()5361 void MPEG4Writer::Track::writeDoviConfigBox() {
5362 CHECK_NE(mDoviProfile, 0u);
5363
5364 uint32_t type = 0;
5365 const void *data = nullptr;
5366 size_t size = 0;
5367 // check to see which key has the configuration box.
5368 if (mMeta->findData(kKeyDVCC, &type, &data, &size) ||
5369 mMeta->findData(kKeyDVVC, &type, &data, &size) ||
5370 mMeta->findData(kKeyDVWC, &type, &data, &size)) {
5371
5372 // if this box is present we write the box, or
5373 // this mp4 will be interpreted as a backward
5374 // compatible stream.
5375 if (mDoviProfile > DolbyVisionProfileDvav110) {
5376 mOwner->beginBox("dvwC");
5377 } else if (mDoviProfile > DolbyVisionProfileDvheDtb) {
5378 mOwner->beginBox("dvvC");
5379 } else {
5380 mOwner->beginBox("dvcC");
5381 }
5382 mOwner->write(data, size);
5383 mOwner->endBox(); // dvwC/dvvC/dvcC
5384 }
5385 }
5386
writeD263Box()5387 void MPEG4Writer::Track::writeD263Box() {
5388 mOwner->beginBox("d263");
5389 mOwner->writeInt32(0); // vendor
5390 mOwner->writeInt8(0); // decoder version
5391 mOwner->writeInt8(10); // level: 10
5392 mOwner->writeInt8(0); // profile: 0
5393 mOwner->endBox(); // d263
5394 }
5395
5396 // This is useful if the pixel is not square
writePaspBox()5397 void MPEG4Writer::Track::writePaspBox() {
5398 // Do not write 'pasp' box unless the track format specifies it.
5399 // According to ISO/IEC 14496-12 (ISO base media file format), 'pasp' box
5400 // is optional. If present, it overrides the SAR from the video CSD. Only
5401 // set it if the track format specifically requests that.
5402 int32_t hSpacing, vSpacing;
5403 if (mMeta->findInt32(kKeySARWidth, &hSpacing) && (hSpacing > 0)
5404 && mMeta->findInt32(kKeySARHeight, &vSpacing) && (vSpacing > 0)) {
5405 mOwner->beginBox("pasp");
5406 mOwner->writeInt32(hSpacing); // hspacing
5407 mOwner->writeInt32(vSpacing); // vspacing
5408 mOwner->endBox(); // pasp
5409 }
5410 }
5411
getStartTimeOffsetTimeUs() const5412 int64_t MPEG4Writer::Track::getStartTimeOffsetTimeUs() const {
5413 int64_t trackStartTimeOffsetUs = 0;
5414 int64_t moovStartTimeUs = mOwner->getStartTimestampUs();
5415 if (mStartTimestampUs != -1 && mStartTimestampUs != moovStartTimeUs) {
5416 CHECK_GT(mStartTimestampUs, moovStartTimeUs);
5417 trackStartTimeOffsetUs = mStartTimestampUs - moovStartTimeUs;
5418 }
5419 return trackStartTimeOffsetUs;
5420 }
5421
getStartTimeOffsetScaledTime() const5422 int32_t MPEG4Writer::Track::getStartTimeOffsetScaledTime() const {
5423 return (getStartTimeOffsetTimeUs() * mTimeScale + 500000LL) / 1000000LL;
5424 }
5425
writeSttsBox()5426 void MPEG4Writer::Track::writeSttsBox() {
5427 mOwner->beginBox("stts");
5428 mOwner->writeInt32(0); // version=0, flags=0
5429 mSttsTableEntries->write(mOwner);
5430 mOwner->endBox(); // stts
5431 }
5432
writeCttsBox()5433 void MPEG4Writer::Track::writeCttsBox() {
5434 // There is no B frame at all
5435 if (mMinCttsOffsetTicks == mMaxCttsOffsetTicks) {
5436 return;
5437 }
5438
5439 // Do not write ctts box when there is no need to have it.
5440 if (mCttsTableEntries->count() == 0) {
5441 return;
5442 }
5443
5444 ALOGV("ctts box has %d entries with range [%" PRId64 ", %" PRId64 "]",
5445 mCttsTableEntries->count(), mMinCttsOffsetTicks, mMaxCttsOffsetTicks);
5446
5447 mOwner->beginBox("ctts");
5448 mOwner->writeInt32(0); // version=0, flags=0
5449 // Adjust ctts entries to have only offset needed for reordering frames.
5450 int64_t deltaTimeUs = mMinCttsOffsetTimeUs;
5451 ALOGV("ctts deltaTimeUs:%" PRId64, deltaTimeUs);
5452 int64_t delta = (deltaTimeUs * mTimeScale + 500000LL) / 1000000LL;
5453 mCttsTableEntries->adjustEntries([delta](size_t /* ix */, uint32_t (&value)[2]) {
5454 // entries are <count, ctts> pairs; adjust only ctts
5455 uint32_t duration = htonl(value[1]); // back to host byte order
5456 // Prevent overflow and underflow
5457 if (delta > duration) {
5458 duration = 0;
5459 } else if (delta < 0 && UINT32_MAX + delta < duration) {
5460 duration = UINT32_MAX;
5461 } else {
5462 duration -= delta;
5463 }
5464 value[1] = htonl(duration);
5465 });
5466 mCttsTableEntries->write(mOwner);
5467 mOwner->endBox(); // ctts
5468 }
5469
writeStssBox()5470 void MPEG4Writer::Track::writeStssBox() {
5471 mOwner->beginBox("stss");
5472 mOwner->writeInt32(0); // version=0, flags=0
5473 mStssTableEntries->write(mOwner);
5474 mOwner->endBox(); // stss
5475 }
5476
writeStszBox()5477 void MPEG4Writer::Track::writeStszBox() {
5478 mOwner->beginBox("stsz");
5479 mOwner->writeInt32(0); // version=0, flags=0
5480 mOwner->writeInt32(0);
5481 mStszTableEntries->write(mOwner);
5482 mOwner->endBox(); // stsz
5483 }
5484
writeStscBox()5485 void MPEG4Writer::Track::writeStscBox() {
5486 mOwner->beginBox("stsc");
5487 mOwner->writeInt32(0); // version=0, flags=0
5488 mStscTableEntries->write(mOwner);
5489 mOwner->endBox(); // stsc
5490 }
5491
writeCo64Box()5492 void MPEG4Writer::Track::writeCo64Box() {
5493 mOwner->beginBox("co64");
5494 mOwner->writeInt32(0); // version=0, flags=0
5495 mCo64TableEntries->write(mOwner);
5496 mOwner->endBox(); // stco or co64
5497 }
5498
writeUdtaBox()5499 void MPEG4Writer::writeUdtaBox() {
5500 beginBox("udta");
5501 writeGeoDataBox();
5502 endBox();
5503 }
5504
writeHdlr(const char * handlerType)5505 void MPEG4Writer::writeHdlr(const char *handlerType) {
5506 beginBox("hdlr");
5507 writeInt32(0); // Version, Flags
5508 writeInt32(0); // Predefined
5509 writeFourcc(handlerType);
5510 writeInt32(0); // Reserved[0]
5511 writeInt32(0); // Reserved[1]
5512 writeInt32(0); // Reserved[2]
5513 writeInt8(0); // Name (empty)
5514 endBox();
5515 }
5516
writeKeys()5517 void MPEG4Writer::writeKeys() {
5518 size_t count = mMetaKeys->countEntries();
5519
5520 beginBox("keys");
5521 writeInt32(0); // Version, Flags
5522 writeInt32(count); // Entry_count
5523 for (size_t i = 0; i < count; i++) {
5524 AMessage::Type type;
5525 const char *key = mMetaKeys->getEntryNameAt(i, &type);
5526 size_t n = strlen(key);
5527 writeInt32(n + 8);
5528 writeFourcc("mdta");
5529 write(key, n); // write without the \0
5530 }
5531 endBox();
5532 }
5533
writeIlst()5534 void MPEG4Writer::writeIlst() {
5535 size_t count = mMetaKeys->countEntries();
5536
5537 beginBox("ilst");
5538 for (size_t i = 0; i < count; i++) {
5539 beginBox(i + 1); // key id (1-based)
5540 beginBox("data");
5541 AMessage::Type type;
5542 const char *key = mMetaKeys->getEntryNameAt(i, &type);
5543 switch (type) {
5544 case AMessage::kTypeString:
5545 {
5546 AString val;
5547 CHECK(mMetaKeys->findString(key, &val));
5548 writeInt32(1); // type = UTF8
5549 writeInt32(0); // default country/language
5550 write(val.c_str(), strlen(val.c_str())); // write without \0
5551 break;
5552 }
5553
5554 case AMessage::kTypeFloat:
5555 {
5556 float val;
5557 CHECK(mMetaKeys->findFloat(key, &val));
5558 writeInt32(23); // type = float32
5559 writeInt32(0); // default country/language
5560 writeInt32(*reinterpret_cast<int32_t *>(&val));
5561 break;
5562 }
5563
5564 case AMessage::kTypeInt32:
5565 {
5566 int32_t val;
5567 CHECK(mMetaKeys->findInt32(key, &val));
5568 writeInt32(67); // type = signed int32
5569 writeInt32(0); // default country/language
5570 writeInt32(val);
5571 break;
5572 }
5573
5574 default:
5575 {
5576 ALOGW("Unsupported key type, writing 0 instead");
5577 writeInt32(77); // type = unsigned int32
5578 writeInt32(0); // default country/language
5579 writeInt32(0);
5580 break;
5581 }
5582 }
5583 endBox(); // data
5584 endBox(); // key id
5585 }
5586 endBox(); // ilst
5587 }
5588
writeMoovLevelMetaBox()5589 void MPEG4Writer::writeMoovLevelMetaBox() {
5590 size_t count = mMetaKeys->countEntries();
5591 if (count == 0) {
5592 return;
5593 }
5594
5595 beginBox("meta");
5596 writeHdlr("mdta");
5597 writeKeys();
5598 writeIlst();
5599 endBox();
5600 }
5601
writeIlocBox()5602 void MPEG4Writer::writeIlocBox() {
5603 beginBox("iloc");
5604 // Use version 1 to allow construction method 1 that refers to
5605 // data in idat box inside meta box.
5606 writeInt32(0x01000000); // Version = 1, Flags = 0
5607 writeInt16(0x4400); // offset_size = length_size = 4
5608 // base_offset_size = index_size = 0
5609
5610 // 16-bit item_count
5611 size_t itemCount = mItems.size();
5612 if (itemCount > 65535) {
5613 ALOGW("Dropping excess items: itemCount %zu", itemCount);
5614 itemCount = 65535;
5615 }
5616 writeInt16((uint16_t)itemCount);
5617
5618 for (auto it = mItems.begin(); it != mItems.end(); it++) {
5619 ItemInfo &item = it->second;
5620
5621 writeInt16(item.itemId);
5622 bool isGrid = item.isGrid();
5623
5624 writeInt16(isGrid ? 1 : 0); // construction_method
5625 writeInt16(0); // data_reference_index = 0
5626 writeInt16(1); // extent_count = 1
5627
5628 if (isGrid) {
5629 // offset into the 'idat' box
5630 writeInt32(mNumGrids++ * 8);
5631 writeInt32(8);
5632 } else {
5633 writeInt32(item.offset);
5634 writeInt32(item.size);
5635 }
5636 }
5637 endBox();
5638 }
5639
writeInfeBox(uint16_t itemId,const char * itemType,uint32_t flags)5640 void MPEG4Writer::writeInfeBox(
5641 uint16_t itemId, const char *itemType, uint32_t flags) {
5642 beginBox("infe");
5643 writeInt32(0x02000000 | flags); // Version = 2, Flags = 0
5644 writeInt16(itemId);
5645 writeInt16(0); //item_protection_index = 0
5646 writeFourcc(itemType);
5647 writeCString(""); // item_name
5648 endBox();
5649 }
5650
writeIinfBox()5651 void MPEG4Writer::writeIinfBox() {
5652 beginBox("iinf");
5653 writeInt32(0); // Version = 0, Flags = 0
5654
5655 // 16-bit item_count
5656 size_t itemCount = mItems.size();
5657 if (itemCount > 65535) {
5658 ALOGW("Dropping excess items: itemCount %zu", itemCount);
5659 itemCount = 65535;
5660 }
5661
5662 writeInt16((uint16_t)itemCount);
5663 for (auto it = mItems.begin(); it != mItems.end(); it++) {
5664 ItemInfo &item = it->second;
5665
5666 writeInfeBox(item.itemId, item.itemType,
5667 (item.isImage() && item.isHidden) ? 1 : 0);
5668 }
5669
5670 endBox();
5671 }
5672
writeIdatBox()5673 void MPEG4Writer::writeIdatBox() {
5674 beginBox("idat");
5675
5676 for (auto it = mItems.begin(); it != mItems.end(); it++) {
5677 ItemInfo &item = it->second;
5678
5679 if (item.isGrid()) {
5680 writeInt8(0); // version
5681 // flags == 1 means 32-bit width,height
5682 int8_t flags = (item.width > 65535 || item.height > 65535);
5683 writeInt8(flags);
5684 writeInt8(item.rows - 1);
5685 writeInt8(item.cols - 1);
5686 if (flags) {
5687 writeInt32(item.width);
5688 writeInt32(item.height);
5689 } else {
5690 writeInt16((uint16_t)item.width);
5691 writeInt16((uint16_t)item.height);
5692 }
5693 }
5694 }
5695
5696 endBox();
5697 }
5698
writeIrefBox()5699 void MPEG4Writer::writeIrefBox() {
5700 beginBox("iref");
5701 writeInt32(0); // Version = 0, Flags = 0
5702 {
5703 for (auto it = mItems.begin(); it != mItems.end(); it++) {
5704 ItemInfo &item = it->second;
5705
5706 for (size_t r = 0; r < item.refsList.size(); r++) {
5707 const ItemRefs &refs = item.refsList[r];
5708 beginBox(refs.key);
5709 writeInt16(item.itemId);
5710 size_t refCount = refs.value.size();
5711 if (refCount > 65535) {
5712 ALOGW("too many entries in %s", refs.key);
5713 refCount = 65535;
5714 }
5715 writeInt16((uint16_t)refCount);
5716 for (size_t refIndex = 0; refIndex < refCount; refIndex++) {
5717 writeInt16(refs.value[refIndex]);
5718 }
5719 endBox();
5720 }
5721 }
5722 }
5723 endBox();
5724 }
5725
writePitmBox()5726 void MPEG4Writer::writePitmBox() {
5727 beginBox("pitm");
5728 writeInt32(0); // Version = 0, Flags = 0
5729 writeInt16(mPrimaryItemId);
5730 endBox();
5731 }
5732
writeGrplBox(const Vector<uint16_t> & items)5733 void MPEG4Writer::writeGrplBox(const Vector<uint16_t> &items) {
5734 if (flags_camera::camera_heif_gainmap()) {
5735 beginBox("grpl");
5736 beginBox("altr");
5737 writeInt32(0); // Version = 0, Flags = 0
5738 writeInt32(1); // Group Id
5739 writeInt32(items.size());// Number of entities
5740 for (size_t i = 0; i < items.size(); i++) {
5741 writeInt32(items[i]);// Item Id
5742 }
5743 endBox();
5744 endBox();
5745 }
5746 }
5747
writeIpcoBox()5748 void MPEG4Writer::writeIpcoBox() {
5749 beginBox("ipco");
5750 size_t numProperties = mProperties.size();
5751 if (numProperties > 32767) {
5752 ALOGW("Dropping excess properties: numProperties %zu", numProperties);
5753 numProperties = 32767;
5754 }
5755 for (size_t propIndex = 0; propIndex < numProperties; propIndex++) {
5756 switch (mProperties[propIndex].type) {
5757 case FOURCC('h', 'v', 'c', 'C'):
5758 {
5759 beginBox("hvcC");
5760 sp<ABuffer> hvcc = mProperties[propIndex].data;
5761 // Patch avcc's lengthSize field to match the number
5762 // of bytes we use to indicate the size of a nal unit.
5763 uint8_t *ptr = (uint8_t *)hvcc->data();
5764 ptr[21] = (ptr[21] & 0xfc) | (useNalLengthFour() ? 3 : 1);
5765 write(hvcc->data(), hvcc->size());
5766 endBox();
5767 break;
5768 }
5769 case FOURCC('a', 'v', '1', 'C'):
5770 {
5771 beginBox("av1C");
5772 sp<ABuffer> av1c = mProperties[propIndex].data;
5773 write(av1c->data(), av1c->size());
5774 endBox();
5775 break;
5776 }
5777 case FOURCC('i', 's', 'p', 'e'):
5778 {
5779 beginBox("ispe");
5780 writeInt32(0); // Version = 0, Flags = 0
5781 writeInt32(mProperties[propIndex].width);
5782 writeInt32(mProperties[propIndex].height);
5783 endBox();
5784 break;
5785 }
5786 case FOURCC('i', 'r', 'o', 't'):
5787 {
5788 beginBox("irot");
5789 writeInt8(mProperties[propIndex].rotation);
5790 endBox();
5791 break;
5792 }
5793 case FOURCC('c', 'o', 'l', 'r'):
5794 {
5795 if (flags_camera::camera_heif_gainmap()) {
5796 beginBox("colr");
5797 writeFourcc("nclx");
5798 writeInt16(mProperties[propIndex].colorPrimaries);
5799 writeInt16(mProperties[propIndex].colorTransfer);
5800 writeInt16(mProperties[propIndex].colorMatrix);
5801 writeInt8(int8_t(mProperties[propIndex].colorRange ? 0x80 : 0x0));
5802 endBox();
5803 }
5804 break;
5805 }
5806 case FOURCC('p', 'i', 'x', 'i'):
5807 {
5808 if (flags_camera::camera_heif_gainmap()) {
5809 beginBox("pixi");
5810 writeInt32(0); // Version = 0, Flags = 0
5811 writeInt8(mProperties[propIndex].bitsPerChannel.size()); // Number of channels
5812 for (size_t i = 0; i < mProperties[propIndex].bitsPerChannel.size(); i++) {
5813 writeInt8(mProperties[propIndex].bitsPerChannel[i]); // Channel bit depth
5814 }
5815 endBox();
5816 }
5817 break;
5818 }
5819 default:
5820 ALOGW("Skipping unrecognized property: type 0x%08x",
5821 mProperties[propIndex].type);
5822 }
5823 }
5824 endBox();
5825 }
5826
writeIpmaBox()5827 void MPEG4Writer::writeIpmaBox() {
5828 beginBox("ipma");
5829 uint32_t flags = (mProperties.size() > 127) ? 1 : 0;
5830 writeInt32(flags); // Version = 0
5831
5832 writeInt32(mAssociationEntryCount);
5833 for (auto it = mItems.begin(); it != mItems.end(); it++) {
5834 ItemInfo &item = it->second;
5835
5836 const Vector<uint16_t> &properties = item.properties;
5837 if (properties.empty()) {
5838 continue;
5839 }
5840 writeInt16(item.itemId);
5841
5842 size_t entryCount = properties.size();
5843 if (entryCount > 255) {
5844 ALOGW("Dropping excess associations: entryCount %zu", entryCount);
5845 entryCount = 255;
5846 }
5847 writeInt8((uint8_t)entryCount);
5848 for (size_t propIndex = 0; propIndex < entryCount; propIndex++) {
5849 if (flags & 1) {
5850 writeInt16((1 << 15) | properties[propIndex]);
5851 } else {
5852 writeInt8((1 << 7) | properties[propIndex]);
5853 }
5854 }
5855 }
5856 endBox();
5857 }
5858
writeIprpBox()5859 void MPEG4Writer::writeIprpBox() {
5860 beginBox("iprp");
5861 writeIpcoBox();
5862 writeIpmaBox();
5863 endBox();
5864 }
5865
writeFileLevelMetaBox()5866 void MPEG4Writer::writeFileLevelMetaBox() {
5867 // patch up the mPrimaryItemId and count items with prop associations
5868 uint16_t firstVisibleItemId = 0;
5869 uint16_t firstImageItemId = 0;
5870 for (auto it = mItems.begin(); it != mItems.end(); it++) {
5871 ItemInfo &item = it->second;
5872
5873 if (item.isGainmapMeta() && !item.properties.empty() &&
5874 flags_camera::camera_heif_gainmap()) {
5875 mAssociationEntryCount++;
5876 continue;
5877 }
5878
5879 if (!item.isImage()) continue;
5880
5881 if (item.isPrimary) {
5882 mPrimaryItemId = item.itemId;
5883 }
5884 if (!firstImageItemId) {
5885 firstImageItemId = item.itemId;
5886 }
5887 if (!firstVisibleItemId && !item.isHidden) {
5888 firstVisibleItemId = item.itemId;
5889 }
5890 if (!item.properties.empty()) {
5891 mAssociationEntryCount++;
5892 }
5893 }
5894
5895 if (!firstImageItemId) {
5896 ALOGE("no valid image was found");
5897 return;
5898 }
5899
5900 if (mPrimaryItemId == 0) {
5901 if (firstVisibleItemId > 0) {
5902 ALOGW("didn't find primary, using first visible image");
5903 mPrimaryItemId = firstVisibleItemId;
5904 } else {
5905 ALOGW("no primary and no visible item, using first image");
5906 mPrimaryItemId = firstImageItemId;
5907 }
5908 }
5909
5910 uint16_t gainmapItemId = 0;
5911 uint16_t gainmapMetaItemId = 0;
5912 for (List<Track *>::iterator it = mTracks.begin();
5913 it != mTracks.end(); ++it) {
5914 if ((*it)->isHeif()) {
5915 (*it)->flushItemRefs();
5916 }
5917 if (flags_camera::camera_heif_gainmap()) {
5918 if ((*it)->getGainmapItemId() > 0) {
5919 gainmapItemId = (*it)->getGainmapItemId();
5920 }
5921 if ((*it)->getGainmapMetaItemId() > 0) {
5922 gainmapMetaItemId = (*it)->getGainmapMetaItemId();
5923 }
5924 }
5925 }
5926 if ((gainmapItemId > 0) && (gainmapMetaItemId > 0) && flags_camera::camera_heif_gainmap()) {
5927 ItemRefs gainmapRefs("dimg");
5928 gainmapRefs.value.push_back(mPrimaryItemId);
5929 gainmapRefs.value.push_back(gainmapItemId);
5930 addRefs_l(gainmapMetaItemId, gainmapRefs);
5931 }
5932
5933 beginBox("meta");
5934 writeInt32(0); // Version = 0, Flags = 0
5935 writeHdlr("pict");
5936 writeIlocBox();
5937 writeIinfBox();
5938 writePitmBox();
5939 writeIprpBox();
5940 if (mNumGrids > 0) {
5941 writeIdatBox();
5942 }
5943 if (mHasRefs) {
5944 writeIrefBox();
5945 }
5946 if ((gainmapItemId > 0) && (gainmapMetaItemId > 0) && flags_camera::camera_heif_gainmap()) {
5947 Vector<uint16_t> itemIds;
5948 itemIds.push_back(gainmapMetaItemId);
5949 itemIds.push_back(mPrimaryItemId);
5950 writeGrplBox(itemIds);
5951 }
5952 endBox();
5953 }
5954
addProperty_l(const ItemProperty & prop)5955 uint16_t MPEG4Writer::addProperty_l(const ItemProperty &prop) {
5956 char typeStr[5];
5957 MakeFourCCString(prop.type, typeStr);
5958 ALOGV("addProperty_l: %s", typeStr);
5959
5960 mProperties.push_back(prop);
5961
5962 // returning 1-based property index
5963 return mProperties.size();
5964 }
5965
reserveItemId_l(size_t numItems,uint16_t * itemIdBase)5966 status_t MPEG4Writer::reserveItemId_l(size_t numItems, uint16_t *itemIdBase) {
5967 if (numItems > UINT16_MAX - mNextItemId) {
5968 ALOGE("couldn't reserve item ids for %zu items", numItems);
5969 return ERROR_OUT_OF_RANGE;
5970 }
5971 *itemIdBase = mNextItemId;
5972 mNextItemId += numItems;
5973 return OK;
5974 }
5975
addItem_l(const ItemInfo & info)5976 uint16_t MPEG4Writer::addItem_l(const ItemInfo &info) {
5977 ALOGV("addItem_l: type %s, offset %u, size %u",
5978 info.itemType, info.offset, info.size);
5979
5980 if (info.itemId < kItemIdBase || info.itemId >= mNextItemId) {
5981 ALOGW("Item id %u is used without reservation!", info.itemId);
5982 }
5983
5984 mItems[info.itemId] = info;
5985
5986 #if (LOG_NDEBUG==0)
5987 if (!info.properties.empty()) {
5988 AString str;
5989 for (size_t i = 0; i < info.properties.size(); i++) {
5990 if (i > 0) {
5991 str.append(", ");
5992 }
5993 str.append(info.properties[i]);
5994 }
5995 ALOGV("addItem_l: id %d, properties: %s", info.itemId, str.c_str());
5996 }
5997 #endif // (LOG_NDEBUG==0)
5998
5999 return info.itemId;
6000 }
6001
addRefs_l(uint16_t itemId,const ItemRefs & refs)6002 void MPEG4Writer::addRefs_l(uint16_t itemId, const ItemRefs &refs) {
6003 if (refs.value.empty()) {
6004 return;
6005 }
6006 if (itemId < kItemIdBase || itemId >= mNextItemId) {
6007 ALOGW("itemId %u for ref is invalid!", itemId);
6008 return;
6009 }
6010
6011 auto it = mItems.find(itemId);
6012 if (it == mItems.end()) {
6013 ALOGW("itemId %u was not added yet", itemId);
6014 return;
6015 }
6016 it->second.refsList.push_back(refs);
6017 mHasRefs = true;
6018 }
6019
6020 /*
6021 * Geodata is stored according to ISO-6709 standard.
6022 */
writeGeoDataBox()6023 void MPEG4Writer::writeGeoDataBox() {
6024 beginBox("\xA9xyz");
6025 /*
6026 * For historical reasons, any user data start
6027 * with "\0xA9", must be followed by its assoicated
6028 * language code.
6029 * 0x0012: text string length
6030 * 0x15c7: lang (locale) code: en
6031 */
6032 writeInt32(0x001215c7);
6033 writeLatitude(mLatitudex10000);
6034 writeLongitude(mLongitudex10000);
6035 writeInt8(0x2F);
6036 endBox();
6037 }
6038
6039 } // namespace android
6040