1 /*
2 * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/video_coding/h264_packet_buffer.h"
12
13 #include <algorithm>
14 #include <cstdint>
15 #include <utility>
16 #include <vector>
17
18 #include "api/array_view.h"
19 #include "api/rtp_packet_info.h"
20 #include "api/video/video_frame_type.h"
21 #include "common_video/h264/h264_common.h"
22 #include "modules/rtp_rtcp/source/rtp_header_extensions.h"
23 #include "modules/rtp_rtcp/source/rtp_packet_received.h"
24 #include "modules/rtp_rtcp/source/rtp_video_header.h"
25 #include "modules/video_coding/codecs/h264/include/h264_globals.h"
26 #include "rtc_base/checks.h"
27 #include "rtc_base/copy_on_write_buffer.h"
28 #include "rtc_base/logging.h"
29 #include "rtc_base/numerics/sequence_number_util.h"
30
31 namespace webrtc {
32 namespace {
EuclideanMod(int64_t n,int64_t div)33 int64_t EuclideanMod(int64_t n, int64_t div) {
34 RTC_DCHECK_GT(div, 0);
35 return (n %= div) < 0 ? n + div : n;
36 }
37
GetNaluInfos(const RTPVideoHeaderH264 & h264_header)38 rtc::ArrayView<const NaluInfo> GetNaluInfos(
39 const RTPVideoHeaderH264& h264_header) {
40 if (h264_header.nalus_length > kMaxNalusPerPacket) {
41 return {};
42 }
43
44 return rtc::MakeArrayView(h264_header.nalus, h264_header.nalus_length);
45 }
46
IsFirstPacketOfFragment(const RTPVideoHeaderH264 & h264_header)47 bool IsFirstPacketOfFragment(const RTPVideoHeaderH264& h264_header) {
48 return h264_header.nalus_length > 0;
49 }
50
BeginningOfIdr(const H264PacketBuffer::Packet & packet)51 bool BeginningOfIdr(const H264PacketBuffer::Packet& packet) {
52 const auto& h264_header =
53 absl::get<RTPVideoHeaderH264>(packet.video_header.video_type_header);
54 const bool contains_idr_nalu =
55 absl::c_any_of(GetNaluInfos(h264_header), [](const auto& nalu_info) {
56 return nalu_info.type == H264::NaluType::kIdr;
57 });
58 switch (h264_header.packetization_type) {
59 case kH264StapA:
60 case kH264SingleNalu: {
61 return contains_idr_nalu;
62 }
63 case kH264FuA: {
64 return contains_idr_nalu && IsFirstPacketOfFragment(h264_header);
65 }
66 }
67 }
68
HasSps(const H264PacketBuffer::Packet & packet)69 bool HasSps(const H264PacketBuffer::Packet& packet) {
70 auto& h264_header =
71 absl::get<RTPVideoHeaderH264>(packet.video_header.video_type_header);
72 return absl::c_any_of(GetNaluInfos(h264_header), [](const auto& nalu_info) {
73 return nalu_info.type == H264::NaluType::kSps;
74 });
75 }
76
77 // TODO(bugs.webrtc.org/13157): Update the H264 depacketizer so we don't have to
78 // fiddle with the payload at this point.
FixVideoPayload(rtc::ArrayView<const uint8_t> payload,const RTPVideoHeader & video_header)79 rtc::CopyOnWriteBuffer FixVideoPayload(rtc::ArrayView<const uint8_t> payload,
80 const RTPVideoHeader& video_header) {
81 constexpr uint8_t kStartCode[] = {0, 0, 0, 1};
82
83 const auto& h264_header =
84 absl::get<RTPVideoHeaderH264>(video_header.video_type_header);
85
86 rtc::CopyOnWriteBuffer result;
87 switch (h264_header.packetization_type) {
88 case kH264StapA: {
89 const uint8_t* payload_end = payload.data() + payload.size();
90 const uint8_t* nalu_ptr = payload.data() + 1;
91 while (nalu_ptr < payload_end - 1) {
92 // The first two bytes describe the length of the segment, where a
93 // segment is the nalu type plus nalu payload.
94 uint16_t segment_length = nalu_ptr[0] << 8 | nalu_ptr[1];
95 nalu_ptr += 2;
96
97 if (nalu_ptr + segment_length <= payload_end) {
98 result.AppendData(kStartCode);
99 result.AppendData(nalu_ptr, segment_length);
100 }
101 nalu_ptr += segment_length;
102 }
103 return result;
104 }
105
106 case kH264FuA: {
107 if (IsFirstPacketOfFragment(h264_header)) {
108 result.AppendData(kStartCode);
109 }
110 result.AppendData(payload);
111 return result;
112 }
113
114 case kH264SingleNalu: {
115 result.AppendData(kStartCode);
116 result.AppendData(payload);
117 return result;
118 }
119 }
120
121 RTC_DCHECK_NOTREACHED();
122 return result;
123 }
124
125 } // namespace
126
H264PacketBuffer(bool idr_only_keyframes_allowed)127 H264PacketBuffer::H264PacketBuffer(bool idr_only_keyframes_allowed)
128 : idr_only_keyframes_allowed_(idr_only_keyframes_allowed) {}
129
InsertPacket(std::unique_ptr<Packet> packet)130 H264PacketBuffer::InsertResult H264PacketBuffer::InsertPacket(
131 std::unique_ptr<Packet> packet) {
132 RTC_DCHECK(packet->video_header.codec == kVideoCodecH264);
133
134 InsertResult result;
135 if (!absl::holds_alternative<RTPVideoHeaderH264>(
136 packet->video_header.video_type_header)) {
137 return result;
138 }
139
140 int64_t unwrapped_seq_num = seq_num_unwrapper_.Unwrap(packet->seq_num);
141 auto& packet_slot = GetPacket(unwrapped_seq_num);
142 if (packet_slot != nullptr &&
143 AheadOrAt(packet_slot->timestamp, packet->timestamp)) {
144 // The incoming `packet` is old or a duplicate.
145 return result;
146 } else {
147 packet_slot = std::move(packet);
148 }
149
150 result.packets = FindFrames(unwrapped_seq_num);
151 return result;
152 }
153
GetPacket(int64_t unwrapped_seq_num)154 std::unique_ptr<H264PacketBuffer::Packet>& H264PacketBuffer::GetPacket(
155 int64_t unwrapped_seq_num) {
156 return buffer_[EuclideanMod(unwrapped_seq_num, kBufferSize)];
157 }
158
BeginningOfStream(const H264PacketBuffer::Packet & packet) const159 bool H264PacketBuffer::BeginningOfStream(
160 const H264PacketBuffer::Packet& packet) const {
161 return HasSps(packet) ||
162 (idr_only_keyframes_allowed_ && BeginningOfIdr(packet));
163 }
164
165 std::vector<std::unique_ptr<H264PacketBuffer::Packet>>
FindFrames(int64_t unwrapped_seq_num)166 H264PacketBuffer::FindFrames(int64_t unwrapped_seq_num) {
167 std::vector<std::unique_ptr<Packet>> found_frames;
168
169 Packet* packet = GetPacket(unwrapped_seq_num).get();
170 RTC_CHECK(packet != nullptr);
171
172 // Check if the packet is continuous or the beginning of a new coded video
173 // sequence.
174 if (unwrapped_seq_num - 1 != last_continuous_unwrapped_seq_num_) {
175 if (unwrapped_seq_num <= last_continuous_unwrapped_seq_num_ ||
176 !BeginningOfStream(*packet)) {
177 return found_frames;
178 }
179
180 last_continuous_unwrapped_seq_num_ = unwrapped_seq_num;
181 }
182
183 for (int64_t seq_num = unwrapped_seq_num;
184 seq_num < unwrapped_seq_num + kBufferSize;) {
185 RTC_DCHECK_GE(seq_num, *last_continuous_unwrapped_seq_num_);
186
187 // Packets that were never assembled into a completed frame will stay in
188 // the 'buffer_'. Check that the `packet` sequence number match the expected
189 // unwrapped sequence number.
190 if (static_cast<uint16_t>(seq_num) != packet->seq_num) {
191 return found_frames;
192 }
193
194 last_continuous_unwrapped_seq_num_ = seq_num;
195 // Last packet of the frame, try to assemble the frame.
196 if (packet->marker_bit) {
197 uint32_t rtp_timestamp = packet->timestamp;
198
199 // Iterate backwards to find where the frame starts.
200 for (int64_t seq_num_start = seq_num;
201 seq_num_start > seq_num - kBufferSize; --seq_num_start) {
202 auto& prev_packet = GetPacket(seq_num_start - 1);
203
204 if (prev_packet == nullptr || prev_packet->timestamp != rtp_timestamp) {
205 if (MaybeAssembleFrame(seq_num_start, seq_num, found_frames)) {
206 // Frame was assembled, continue to look for more frames.
207 break;
208 } else {
209 // Frame was not assembled, no subsequent frame will be continuous.
210 return found_frames;
211 }
212 }
213 }
214 }
215
216 seq_num++;
217 packet = GetPacket(seq_num).get();
218 if (packet == nullptr) {
219 return found_frames;
220 }
221 }
222
223 return found_frames;
224 }
225
MaybeAssembleFrame(int64_t start_seq_num_unwrapped,int64_t end_sequence_number_unwrapped,std::vector<std::unique_ptr<Packet>> & frames)226 bool H264PacketBuffer::MaybeAssembleFrame(
227 int64_t start_seq_num_unwrapped,
228 int64_t end_sequence_number_unwrapped,
229 std::vector<std::unique_ptr<Packet>>& frames) {
230 bool has_sps = false;
231 bool has_pps = false;
232 bool has_idr = false;
233
234 int width = -1;
235 int height = -1;
236
237 for (int64_t seq_num = start_seq_num_unwrapped;
238 seq_num <= end_sequence_number_unwrapped; ++seq_num) {
239 const auto& packet = GetPacket(seq_num);
240 const auto& h264_header =
241 absl::get<RTPVideoHeaderH264>(packet->video_header.video_type_header);
242 for (const auto& nalu : GetNaluInfos(h264_header)) {
243 has_idr |= nalu.type == H264::NaluType::kIdr;
244 has_sps |= nalu.type == H264::NaluType::kSps;
245 has_pps |= nalu.type == H264::NaluType::kPps;
246 }
247
248 width = std::max<int>(packet->video_header.width, width);
249 height = std::max<int>(packet->video_header.height, height);
250 }
251
252 if (has_idr) {
253 if (!idr_only_keyframes_allowed_ && (!has_sps || !has_pps)) {
254 return false;
255 }
256 }
257
258 for (int64_t seq_num = start_seq_num_unwrapped;
259 seq_num <= end_sequence_number_unwrapped; ++seq_num) {
260 auto& packet = GetPacket(seq_num);
261
262 packet->video_header.is_first_packet_in_frame =
263 (seq_num == start_seq_num_unwrapped);
264 packet->video_header.is_last_packet_in_frame =
265 (seq_num == end_sequence_number_unwrapped);
266
267 if (packet->video_header.is_first_packet_in_frame) {
268 if (width > 0 && height > 0) {
269 packet->video_header.width = width;
270 packet->video_header.height = height;
271 }
272
273 packet->video_header.frame_type = has_idr
274 ? VideoFrameType::kVideoFrameKey
275 : VideoFrameType::kVideoFrameDelta;
276 }
277
278 packet->video_payload =
279 FixVideoPayload(packet->video_payload, packet->video_header);
280
281 frames.push_back(std::move(packet));
282 }
283
284 return true;
285 }
286
287 } // namespace webrtc
288