1 /*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/video_coding/frame_buffer.h"
12
13 #include <string.h>
14
15 #include "api/video/encoded_image.h"
16 #include "api/video/video_timing.h"
17 #include "modules/video_coding/include/video_codec_interface.h"
18 #include "modules/video_coding/packet.h"
19 #include "rtc_base/checks.h"
20 #include "rtc_base/logging.h"
21 #include "rtc_base/trace_event.h"
22
23 namespace webrtc {
24
VCMFrameBuffer()25 VCMFrameBuffer::VCMFrameBuffer()
26 : _state(kStateEmpty), _nackCount(0), _latestPacketTimeMs(-1) {}
27
~VCMFrameBuffer()28 VCMFrameBuffer::~VCMFrameBuffer() {}
29
FrameType() const30 webrtc::VideoFrameType VCMFrameBuffer::FrameType() const {
31 return _sessionInfo.FrameType();
32 }
33
GetLowSeqNum() const34 int32_t VCMFrameBuffer::GetLowSeqNum() const {
35 return _sessionInfo.LowSequenceNumber();
36 }
37
GetHighSeqNum() const38 int32_t VCMFrameBuffer::GetHighSeqNum() const {
39 return _sessionInfo.HighSequenceNumber();
40 }
41
PictureId() const42 int VCMFrameBuffer::PictureId() const {
43 return _sessionInfo.PictureId();
44 }
45
TemporalId() const46 int VCMFrameBuffer::TemporalId() const {
47 return _sessionInfo.TemporalId();
48 }
49
LayerSync() const50 bool VCMFrameBuffer::LayerSync() const {
51 return _sessionInfo.LayerSync();
52 }
53
Tl0PicId() const54 int VCMFrameBuffer::Tl0PicId() const {
55 return _sessionInfo.Tl0PicId();
56 }
57
GetNaluInfos() const58 std::vector<NaluInfo> VCMFrameBuffer::GetNaluInfos() const {
59 return _sessionInfo.GetNaluInfos();
60 }
61
SetGofInfo(const GofInfoVP9 & gof_info,size_t idx)62 void VCMFrameBuffer::SetGofInfo(const GofInfoVP9& gof_info, size_t idx) {
63 TRACE_EVENT0("webrtc", "VCMFrameBuffer::SetGofInfo");
64 _sessionInfo.SetGofInfo(gof_info, idx);
65 // TODO(asapersson): Consider adding hdr->VP9.ref_picture_id for testing.
66 _codecSpecificInfo.codecSpecific.VP9.temporal_idx =
67 gof_info.temporal_idx[idx];
68 _codecSpecificInfo.codecSpecific.VP9.temporal_up_switch =
69 gof_info.temporal_up_switch[idx];
70 }
71
72 // Insert packet
InsertPacket(const VCMPacket & packet,int64_t timeInMs,const FrameData & frame_data)73 VCMFrameBufferEnum VCMFrameBuffer::InsertPacket(const VCMPacket& packet,
74 int64_t timeInMs,
75 const FrameData& frame_data) {
76 TRACE_EVENT0("webrtc", "VCMFrameBuffer::InsertPacket");
77 RTC_DCHECK(!(NULL == packet.dataPtr && packet.sizeBytes > 0));
78 if (packet.dataPtr != NULL) {
79 _payloadType = packet.payloadType;
80 }
81
82 if (kStateEmpty == _state) {
83 // First packet (empty and/or media) inserted into this frame.
84 // store some info and set some initial values.
85 SetTimestamp(packet.timestamp);
86 // We only take the ntp timestamp of the first packet of a frame.
87 ntp_time_ms_ = packet.ntp_time_ms_;
88 _codec = packet.codec();
89 if (packet.video_header.frame_type != VideoFrameType::kEmptyFrame) {
90 // first media packet
91 SetState(kStateIncomplete);
92 }
93 }
94
95 size_t oldSize = encoded_image_buffer_ ? encoded_image_buffer_->size() : 0;
96 uint32_t requiredSizeBytes =
97 size() + packet.sizeBytes +
98 (packet.insertStartCode ? kH264StartCodeLengthBytes : 0);
99 if (requiredSizeBytes > oldSize) {
100 const uint8_t* prevBuffer = data();
101 const uint32_t increments =
102 requiredSizeBytes / kBufferIncStepSizeBytes +
103 (requiredSizeBytes % kBufferIncStepSizeBytes > 0);
104 const uint32_t newSize = oldSize + increments * kBufferIncStepSizeBytes;
105 if (newSize > kMaxJBFrameSizeBytes) {
106 RTC_LOG(LS_ERROR) << "Failed to insert packet due to frame being too "
107 "big.";
108 return kSizeError;
109 }
110 if (data() == nullptr) {
111 encoded_image_buffer_ = EncodedImageBuffer::Create(newSize);
112 SetEncodedData(encoded_image_buffer_);
113 set_size(0);
114 } else {
115 RTC_CHECK(encoded_image_buffer_ != nullptr);
116 RTC_DCHECK_EQ(encoded_image_buffer_->data(), data());
117 encoded_image_buffer_->Realloc(newSize);
118 }
119 _sessionInfo.UpdateDataPointers(prevBuffer, data());
120 }
121
122 if (packet.width() > 0 && packet.height() > 0) {
123 _encodedWidth = packet.width();
124 _encodedHeight = packet.height();
125 }
126
127 // Don't copy payload specific data for empty packets (e.g padding packets).
128 if (packet.sizeBytes > 0)
129 CopyCodecSpecific(&packet.video_header);
130
131 int retVal = _sessionInfo.InsertPacket(
132 packet, encoded_image_buffer_ ? encoded_image_buffer_->data() : nullptr,
133 frame_data);
134 if (retVal == -1) {
135 return kSizeError;
136 } else if (retVal == -2) {
137 return kDuplicatePacket;
138 } else if (retVal == -3) {
139 return kOutOfBoundsPacket;
140 }
141 // update size
142 set_size(size() + static_cast<uint32_t>(retVal));
143
144 _latestPacketTimeMs = timeInMs;
145
146 // http://www.etsi.org/deliver/etsi_ts/126100_126199/126114/12.07.00_60/
147 // ts_126114v120700p.pdf Section 7.4.5.
148 // The MTSI client shall add the payload bytes as defined in this clause
149 // onto the last RTP packet in each group of packets which make up a key
150 // frame (I-frame or IDR frame in H.264 (AVC), or an IRAP picture in H.265
151 // (HEVC)).
152 if (packet.markerBit) {
153 rotation_ = packet.video_header.rotation;
154 content_type_ = packet.video_header.content_type;
155 if (packet.video_header.video_timing.flags != VideoSendTiming::kInvalid) {
156 timing_.encode_start_ms =
157 ntp_time_ms_ + packet.video_header.video_timing.encode_start_delta_ms;
158 timing_.encode_finish_ms =
159 ntp_time_ms_ +
160 packet.video_header.video_timing.encode_finish_delta_ms;
161 timing_.packetization_finish_ms =
162 ntp_time_ms_ +
163 packet.video_header.video_timing.packetization_finish_delta_ms;
164 timing_.pacer_exit_ms =
165 ntp_time_ms_ + packet.video_header.video_timing.pacer_exit_delta_ms;
166 timing_.network_timestamp_ms =
167 ntp_time_ms_ +
168 packet.video_header.video_timing.network_timestamp_delta_ms;
169 timing_.network2_timestamp_ms =
170 ntp_time_ms_ +
171 packet.video_header.video_timing.network2_timestamp_delta_ms;
172 }
173 timing_.flags = packet.video_header.video_timing.flags;
174 }
175
176 if (packet.is_first_packet_in_frame()) {
177 playout_delay_ = packet.video_header.playout_delay;
178 }
179
180 if (_sessionInfo.complete()) {
181 SetState(kStateComplete);
182 return kCompleteSession;
183 }
184 return kIncomplete;
185 }
186
LatestPacketTimeMs() const187 int64_t VCMFrameBuffer::LatestPacketTimeMs() const {
188 TRACE_EVENT0("webrtc", "VCMFrameBuffer::LatestPacketTimeMs");
189 return _latestPacketTimeMs;
190 }
191
IncrementNackCount()192 void VCMFrameBuffer::IncrementNackCount() {
193 TRACE_EVENT0("webrtc", "VCMFrameBuffer::IncrementNackCount");
194 _nackCount++;
195 }
196
GetNackCount() const197 int16_t VCMFrameBuffer::GetNackCount() const {
198 TRACE_EVENT0("webrtc", "VCMFrameBuffer::GetNackCount");
199 return _nackCount;
200 }
201
HaveFirstPacket() const202 bool VCMFrameBuffer::HaveFirstPacket() const {
203 TRACE_EVENT0("webrtc", "VCMFrameBuffer::HaveFirstPacket");
204 return _sessionInfo.HaveFirstPacket();
205 }
206
NumPackets() const207 int VCMFrameBuffer::NumPackets() const {
208 TRACE_EVENT0("webrtc", "VCMFrameBuffer::NumPackets");
209 return _sessionInfo.NumPackets();
210 }
211
Reset()212 void VCMFrameBuffer::Reset() {
213 TRACE_EVENT0("webrtc", "VCMFrameBuffer::Reset");
214 set_size(0);
215 _sessionInfo.Reset();
216 _payloadType = 0;
217 _nackCount = 0;
218 _latestPacketTimeMs = -1;
219 _state = kStateEmpty;
220 VCMEncodedFrame::Reset();
221 }
222
223 // Set state of frame
SetState(VCMFrameBufferStateEnum state)224 void VCMFrameBuffer::SetState(VCMFrameBufferStateEnum state) {
225 TRACE_EVENT0("webrtc", "VCMFrameBuffer::SetState");
226 if (_state == state) {
227 return;
228 }
229 switch (state) {
230 case kStateIncomplete:
231 // we can go to this state from state kStateEmpty
232 RTC_DCHECK_EQ(_state, kStateEmpty);
233
234 // Do nothing, we received a packet
235 break;
236
237 case kStateComplete:
238 RTC_DCHECK(_state == kStateEmpty || _state == kStateIncomplete);
239
240 break;
241
242 case kStateEmpty:
243 // Should only be set to empty through Reset().
244 RTC_DCHECK_NOTREACHED();
245 break;
246 }
247 _state = state;
248 }
249
250 // Get current state of frame
GetState() const251 VCMFrameBufferStateEnum VCMFrameBuffer::GetState() const {
252 return _state;
253 }
254
PrepareForDecode(bool continuous)255 void VCMFrameBuffer::PrepareForDecode(bool continuous) {
256 TRACE_EVENT0("webrtc", "VCMFrameBuffer::PrepareForDecode");
257 size_t bytes_removed = _sessionInfo.MakeDecodable();
258 set_size(size() - bytes_removed);
259 // Transfer frame information to EncodedFrame and create any codec
260 // specific information.
261 _frameType = _sessionInfo.FrameType();
262 _missingFrame = !continuous;
263 }
264
265 } // namespace webrtc
266