xref: /aosp_15_r20/development/tools/winscope/src/parsers/screen_recording/parser_screen_recording.ts (revision 90c8c64db3049935a07c6143d7fd006e26f8ecca) 
1/*
2 * Copyright (C) 2022 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17import {ArrayUtils} from 'common/array_utils';
18import {Timestamp} from 'common/time';
19import {ParserTimestampConverter} from 'common/timestamp_converter';
20import {TIME_UNIT_TO_NANO} from 'common/time_units';
21import {UserNotifier} from 'common/user_notifier';
22import {MonotonicScreenRecording} from 'messaging/user_warnings';
23import * as MP4Box from 'mp4box';
24import {AbstractParser} from 'parsers/legacy/abstract_parser';
25import {CoarseVersion} from 'trace/coarse_version';
26import {MediaBasedTraceEntry} from 'trace/media_based_trace_entry';
27import {ScreenRecordingUtils} from 'trace/screen_recording_utils';
28import {TraceFile} from 'trace/trace_file';
29import {ScreenRecordingOffsets, TraceMetadata} from 'trace/trace_metadata';
30import {TraceType} from 'trace/trace_type';
31
32class ParserScreenRecording extends AbstractParser {
33  private realToBootTimeOffsetNs: bigint | undefined;
34
35  constructor(
36    trace: TraceFile,
37    timestampConverter: ParserTimestampConverter,
38    metadata: TraceMetadata,
39  ) {
40    super(trace, timestampConverter, metadata);
41  }
42
43  override getTraceType(): TraceType {
44    return TraceType.SCREEN_RECORDING;
45  }
46
47  override getCoarseVersion(): CoarseVersion {
48    return CoarseVersion.LATEST;
49  }
50
51  override getMagicNumber(): number[] {
52    return ParserScreenRecording.MPEG4_MAGIC_NUMBER;
53  }
54
55  override getRealToMonotonicTimeOffsetNs(): bigint | undefined {
56    return undefined;
57  }
58
59  override getRealToBootTimeOffsetNs(): bigint | undefined {
60    return this.realToBootTimeOffsetNs;
61  }
62
63  override async decodeTrace(videoData: Uint8Array): Promise<Array<bigint>> {
64    const posVersion = this.searchMagicString(videoData);
65    if (posVersion !== undefined) {
66      return this.parseTimestampsUsingEmbeddedMetadata(videoData, posVersion);
67    } else if (this.metadata?.screenRecordingOffsets !== undefined) {
68      return await this.parseTimestampsUsingExternalMetadata(
69        videoData,
70        this.metadata.screenRecordingOffsets,
71      );
72    }
73    throw new TypeError(
74      "video data doesn't contain winscope magic string and metadata json not provided",
75    );
76  }
77
78  protected override getTimestamp(decodedEntry: bigint): Timestamp {
79    return this.timestampConverter.makeTimestampFromBootTimeNs(decodedEntry);
80  }
81
82  override processDecodedEntry(
83    index: number,
84    entry: bigint,
85  ): MediaBasedTraceEntry {
86    const videoTimeSeconds = ScreenRecordingUtils.timestampToVideoTimeSeconds(
87      this.decodedEntries[0],
88      entry,
89    );
90    const videoData = this.traceFile.file;
91    return new MediaBasedTraceEntry(videoTimeSeconds, videoData);
92  }
93
94  private searchMagicString(videoData: Uint8Array): number | undefined {
95    let pos = ArrayUtils.searchSubarray(
96      videoData,
97      ParserScreenRecording.WINSCOPE_META_MAGIC_STRING,
98    );
99    if (pos === undefined) {
100      return undefined;
101    }
102    pos += ParserScreenRecording.WINSCOPE_META_MAGIC_STRING.length;
103    return pos;
104  }
105
106  private parseTimestampsUsingEmbeddedMetadata(
107    videoData: Uint8Array,
108    posVersion: number,
109  ): Array<bigint> {
110    const [posCount, timeOffsetNs] = this.getOffsetAndCountFromPosVersion(
111      videoData,
112      posVersion,
113    );
114    const [posTimestamps, count] = this.parseFramesCount(videoData, posCount);
115    this.realToBootTimeOffsetNs = timeOffsetNs;
116    const timestampsElapsedNs = this.parseTimestampsElapsedNs(
117      videoData,
118      posTimestamps,
119      count,
120    );
121    return timestampsElapsedNs;
122  }
123
124  private getOffsetAndCountFromPosVersion(
125    videoData: Uint8Array,
126    posVersion: number,
127  ): [number, bigint] {
128    const [posTimeOffset, metadataVersion] = this.parseMetadataVersion(
129      videoData,
130      posVersion,
131    );
132
133    if (metadataVersion !== 1 && metadataVersion !== 2) {
134      throw new TypeError(
135        `Metadata version "${metadataVersion}" not supported`,
136      );
137    }
138
139    if (metadataVersion === 1) {
140      // UI traces contain "elapsed" timestamps (SYSTEM_TIME_BOOTTIME), whereas
141      // metadata Version 1 contains SYSTEM_TIME_MONOTONIC timestamps.
142      //
143      // Here we are pretending that metadata Version 1 contains "elapsed"
144      // timestamps as well, in order to synchronize with the other traces.
145      //
146      // If no device suspensions are involved, SYSTEM_TIME_MONOTONIC should
147      // indeed correspond to SYSTEM_TIME_BOOTTIME and things will work as
148      // expected.
149      UserNotifier.add(new MonotonicScreenRecording());
150    }
151
152    return this.parseRealToBootTimeOffsetNs(videoData, posTimeOffset);
153  }
154
155  private parseMetadataVersion(
156    videoData: Uint8Array,
157    pos: number,
158  ): [number, number] {
159    if (pos + 4 > videoData.length) {
160      throw new TypeError(
161        'Failed to parse metadata version. Video data is too short.',
162      );
163    }
164    const version = Number(
165      ArrayUtils.toUintLittleEndian(videoData, pos, pos + 4),
166    );
167    pos += 4;
168    return [pos, version];
169  }
170
171  private parseRealToBootTimeOffsetNs(
172    videoData: Uint8Array,
173    pos: number,
174  ): [number, bigint] {
175    if (pos + 8 > videoData.length) {
176      throw new TypeError(
177        'Failed to parse realtime-to-elapsed time offset. Video data is too short.',
178      );
179    }
180    const offset = ArrayUtils.toIntLittleEndian(videoData, pos, pos + 8);
181    pos += 8;
182    return [pos, offset];
183  }
184
185  private parseFramesCount(
186    videoData: Uint8Array,
187    pos: number,
188  ): [number, number] {
189    if (pos + 4 > videoData.length) {
190      throw new TypeError(
191        'Failed to parse frames count. Video data is too short.',
192      );
193    }
194    const count = Number(
195      ArrayUtils.toUintLittleEndian(videoData, pos, pos + 4),
196    );
197    pos += 4;
198    return [pos, count];
199  }
200
201  private parseTimestampsElapsedNs(
202    videoData: Uint8Array,
203    pos: number,
204    count: number,
205  ): Array<bigint> {
206    if (pos + count * 8 > videoData.length) {
207      throw new TypeError(
208        'Failed to parse timestamps. Video data is too short.',
209      );
210    }
211    const timestamps: Array<bigint> = [];
212    for (let i = 0; i < count; ++i) {
213      const timestamp = ArrayUtils.toUintLittleEndian(videoData, pos, pos + 8);
214      pos += 8;
215      timestamps.push(timestamp);
216    }
217    return timestamps;
218  }
219
220  private async parseTimestampsUsingExternalMetadata(
221    videoData: Uint8Array,
222    metadata: ScreenRecordingOffsets,
223  ): Promise<Array<bigint>> {
224    this.realToBootTimeOffsetNs = metadata.realToElapsedTimeOffsetNanos;
225    const timestampsElapsedNs = await this.parseTimestampsFromMp4(
226      videoData.buffer.slice(
227        videoData.byteOffset,
228        videoData.byteLength + videoData.byteOffset,
229      ),
230      metadata.elapsedRealTimeNanos,
231    );
232    return timestampsElapsedNs;
233  }
234
235  private async parseTimestampsFromMp4(
236    arrayBuffer: ArrayBuffer,
237    elapsedRealTimeNanos: bigint,
238  ): Promise<Array<bigint>> {
239    const timestamps: Array<bigint> = [];
240    const mp4File: MP4Box.MP4File = MP4Box.createFile();
241    await new Promise<void>((resolve) => {
242      mp4File.onReady = (info) => {
243        mp4File.onSamples = (id, user, samples) => {
244          let curr = elapsedRealTimeNanos;
245          samples.forEach((sample) => {
246            const timeSeconds = sample.duration / sample.timescale;
247            const timeNs = BigInt(
248              Math.floor(TIME_UNIT_TO_NANO.s * timeSeconds),
249            );
250            curr += timeNs;
251            timestamps.push(curr);
252          });
253          resolve();
254        };
255        mp4File.setExtractionOptions(info.tracks[0].id);
256      };
257      const buffer = arrayBuffer as MP4Box.MP4ArrayBuffer;
258      buffer.fileStart = 0;
259      mp4File.appendBuffer(buffer);
260      mp4File.start();
261    });
262    return timestamps;
263  }
264
265  private static readonly MPEG4_MAGIC_NUMBER = [
266    0x00, 0x00, 0x00, 0x18, 0x66, 0x74, 0x79, 0x70, 0x6d, 0x70, 0x34, 0x32,
267  ]; // ....ftypmp42
268  private static readonly WINSCOPE_META_MAGIC_STRING = [
269    0x23, 0x56, 0x56, 0x31, 0x4e, 0x53, 0x43, 0x30, 0x50, 0x45, 0x54, 0x31,
270    0x4d, 0x45, 0x32, 0x23,
271  ]; // #VV1NSC0PET1ME2#
272}
273
274export {ParserScreenRecording};
275