1 /*
2 * Intel License
3 */
4
5 #include <string.h>
6
7 #include "absl/types/optional.h"
8 #include "absl/types/variant.h"
9
10 #include "common_video/h264/h264_common.h"
11 #include "common_video/h265/h265_common.h"
12 #include "common_video/h265/h265_pps_parser.h"
13 #include "common_video/h265/h265_sps_parser.h"
14 #include "common_video/h265/h265_vps_parser.h"
15 #include "modules/include/module_common_types.h"
16 #include "modules/rtp_rtcp/source/byte_io.h"
17 #include "modules/rtp_rtcp/source/rtp_format_h265.h"
18 #include "modules/rtp_rtcp/source/rtp_packet_to_send.h"
19 #include "rtc_base/logging.h"
20
21 using namespace rtc;
22
23 namespace webrtc {
24 namespace {
25
26 enum NaluType {
27 kTrailN = 0,
28 kTrailR = 1,
29 kTsaN = 2,
30 kTsaR = 3,
31 kStsaN = 4,
32 kStsaR = 5,
33 kRadlN = 6,
34 kRadlR = 7,
35 kBlaWLp = 16,
36 kBlaWRadl = 17,
37 kBlaNLp = 18,
38 kIdrWRadl = 19,
39 kIdrNLp = 20,
40 kCra = 21,
41 kVps = 32,
42 kHevcSps = 33,
43 kHevcPps = 34,
44 kHevcAud = 35,
45 kPrefixSei = 39,
46 kSuffixSei = 40,
47 kHevcAp = 48,
48 kHevcFu = 49
49 };
50
51 /*
52 0 1 2 3
53 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
54 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
55 | PayloadHdr (Type=49) | FU header | DONL (cond) |
56 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-|
57 */
58 // Unlike H.264, HEVC NAL header is 2-bytes.
59 static const size_t kHevcNalHeaderSize = 2;
60 // H.265's FU is constructed of 2-byte payload header, and 1-byte FU header
61 static const size_t kHevcFuHeaderSize = 1;
62 static const size_t kHevcLengthFieldSize = 2;
63
64 enum HevcNalHdrMasks {
65 kHevcFBit = 0x80,
66 kHevcTypeMask = 0x7E,
67 kHevcLayerIDHMask = 0x1,
68 kHevcLayerIDLMask = 0xF8,
69 kHevcTIDMask = 0x7,
70 kHevcTypeMaskN = 0x81,
71 kHevcTypeMaskInFuHeader = 0x3F
72 };
73
74 // Bit masks for FU headers.
75 enum HevcFuDefs { kHevcSBit = 0x80, kHevcEBit = 0x40, kHevcFuTypeBit = 0x3F };
76
77 } // namespace
78
RtpPacketizerH265(rtc::ArrayView<const uint8_t> payload,PayloadSizeLimits limits,H265PacketizationMode packetization_mode)79 RtpPacketizerH265::RtpPacketizerH265(
80 rtc::ArrayView<const uint8_t> payload,
81 PayloadSizeLimits limits,
82 H265PacketizationMode packetization_mode)
83 : limits_(limits),
84 num_packets_left_(0) {
85 // Guard against uninitialized memory in packetization_mode.
86 RTC_CHECK(packetization_mode == H265PacketizationMode::NonInterleaved ||
87 packetization_mode == H265PacketizationMode::SingleNalUnit);
88
89 for (const auto& nalu :
90 H265::FindNaluIndices(payload.data(), payload.size())) {
91 input_fragments_.push_back(Fragment(payload.data() + nalu.payload_start_offset, nalu.payload_size));
92 }
93
94 if (!GeneratePackets(packetization_mode)) {
95 // If failed to generate all the packets, discard already generated
96 // packets in case the caller would ignore return value and still try to
97 // call NextPacket().
98 num_packets_left_ = 0;
99 while (!packets_.empty()) {
100 packets_.pop();
101 }
102 }
103 }
104
~RtpPacketizerH265()105 RtpPacketizerH265::~RtpPacketizerH265() {}
106
NumPackets() const107 size_t RtpPacketizerH265::NumPackets() const {
108 return num_packets_left_;
109 }
110
Fragment(const uint8_t * buffer,size_t length)111 RtpPacketizerH265::Fragment::Fragment(const uint8_t* buffer, size_t length)
112 : buffer(buffer), length(length) {}
Fragment(const Fragment & fragment)113 RtpPacketizerH265::Fragment::Fragment(const Fragment& fragment)
114 : buffer(fragment.buffer), length(fragment.length) {}
115
116
GeneratePackets(H265PacketizationMode packetization_mode)117 bool RtpPacketizerH265::GeneratePackets(
118 H265PacketizationMode packetization_mode) {
119 // For HEVC we follow non-interleaved mode for the packetization,
120 // and don't support single-nalu mode at present.
121 for (size_t i = 0; i < input_fragments_.size();) {
122 int fragment_len = input_fragments_[i].length;
123 int single_packet_capacity = limits_.max_payload_len;
124 if (input_fragments_.size() == 1)
125 single_packet_capacity -= limits_.single_packet_reduction_len;
126 else if (i == 0)
127 single_packet_capacity -= limits_.first_packet_reduction_len;
128 else if (i + 1 == input_fragments_.size()) {
129 // Pretend that last fragment is larger instead of making last packet
130 // smaller.
131 single_packet_capacity -= limits_.last_packet_reduction_len;
132 }
133 if (fragment_len > single_packet_capacity) {
134 PacketizeFu(i);
135 ++i;
136 } else {
137 PacketizeSingleNalu(i);
138 ++i;
139 }
140 }
141 return true;
142 }
143
PacketizeFu(size_t fragment_index)144 bool RtpPacketizerH265::PacketizeFu(size_t fragment_index) {
145 // Fragment payload into packets (FU).
146 // Strip out the original header and leave room for the FU header.
147 const Fragment& fragment = input_fragments_[fragment_index];
148 PayloadSizeLimits limits = limits_;
149 limits.max_payload_len -= kHevcFuHeaderSize + kHevcNalHeaderSize;
150
151 // Update single/first/last packet reductions unless it is single/first/last
152 // fragment.
153 if (input_fragments_.size() != 1) {
154 // if this fragment is put into a single packet, it might still be the
155 // first or the last packet in the whole sequence of packets.
156 if (fragment_index == input_fragments_.size() - 1) {
157 limits.single_packet_reduction_len = limits_.last_packet_reduction_len;
158 } else if (fragment_index == 0) {
159 limits.single_packet_reduction_len = limits_.first_packet_reduction_len;
160 } else {
161 limits.single_packet_reduction_len = 0;
162 }
163 }
164 if (fragment_index != 0)
165 limits.first_packet_reduction_len = 0;
166 if (fragment_index != input_fragments_.size() - 1)
167 limits.last_packet_reduction_len = 0;
168
169 // Strip out the original header.
170 size_t payload_left = fragment.length - kHevcNalHeaderSize;
171 int offset = kHevcNalHeaderSize;
172
173 std::vector<int> payload_sizes = SplitAboutEqually(payload_left, limits);
174 if (payload_sizes.empty())
175 return false;
176
177 for (size_t i = 0; i < payload_sizes.size(); ++i) {
178 int packet_length = payload_sizes[i];
179 RTC_CHECK_GT(packet_length, 0);
180 uint16_t header = (fragment.buffer[0] << 8) | fragment.buffer[1];
181 packets_.push(PacketUnit(Fragment(fragment.buffer + offset, packet_length),
182 /*first_fragment=*/i == 0,
183 /*last_fragment=*/i == payload_sizes.size() - 1,
184 false, header));
185 offset += packet_length;
186 payload_left -= packet_length;
187 }
188 num_packets_left_ += payload_sizes.size();
189 RTC_CHECK_EQ(0, payload_left);
190 return true;
191 }
192
193
PacketizeSingleNalu(size_t fragment_index)194 bool RtpPacketizerH265::PacketizeSingleNalu(size_t fragment_index) {
195 // Add a single NALU to the queue, no aggregation.
196 size_t payload_size_left = limits_.max_payload_len;
197 if (input_fragments_.size() == 1)
198 payload_size_left -= limits_.single_packet_reduction_len;
199 else if (fragment_index == 0)
200 payload_size_left -= limits_.first_packet_reduction_len;
201 else if (fragment_index + 1 == input_fragments_.size())
202 payload_size_left -= limits_.last_packet_reduction_len;
203 const Fragment* fragment = &input_fragments_[fragment_index];
204 if (payload_size_left < fragment->length) {
205 RTC_LOG(LS_ERROR) << "Failed to fit a fragment to packet in SingleNalu "
206 "packetization mode. Payload size left "
207 << payload_size_left << ", fragment length "
208 << fragment->length << ", packet capacity "
209 << limits_.max_payload_len;
210 return false;
211 }
212 RTC_CHECK_GT(fragment->length, 0u);
213 packets_.push(PacketUnit(*fragment, true /* first */, true /* last */,
214 false /* aggregated */, fragment->buffer[0]));
215 ++num_packets_left_;
216 return true;
217 }
218
PacketizeAp(size_t fragment_index)219 int RtpPacketizerH265::PacketizeAp(size_t fragment_index) {
220 // Aggregate fragments into one packet (STAP-A).
221 size_t payload_size_left = limits_.max_payload_len;
222 if (input_fragments_.size() == 1)
223 payload_size_left -= limits_.single_packet_reduction_len;
224 else if (fragment_index == 0)
225 payload_size_left -= limits_.first_packet_reduction_len;
226 int aggregated_fragments = 0;
227 size_t fragment_headers_length = 0;
228 const Fragment* fragment = &input_fragments_[fragment_index];
229 RTC_CHECK_GE(payload_size_left, fragment->length);
230 ++num_packets_left_;
231
232 auto payload_size_needed = [&] {
233 size_t fragment_size = fragment->length + fragment_headers_length;
234 if (input_fragments_.size() == 1) {
235 // Single fragment, single packet, payload_size_left already adjusted
236 // with limits_.single_packet_reduction_len.
237 return fragment_size;
238 }
239 if (fragment_index == input_fragments_.size() - 1) {
240 // Last fragment, so StrapA might be the last packet.
241 return fragment_size + limits_.last_packet_reduction_len;
242 }
243 return fragment_size;
244 };
245
246 while (payload_size_left >= payload_size_needed()) {
247 RTC_CHECK_GT(fragment->length, 0);
248 packets_.push(PacketUnit(*fragment, aggregated_fragments == 0, false, true,
249 fragment->buffer[0]));
250 payload_size_left -= fragment->length;
251 payload_size_left -= fragment_headers_length;
252
253 fragment_headers_length = kHevcLengthFieldSize;
254 // If we are going to try to aggregate more fragments into this packet
255 // we need to add the STAP-A NALU header and a length field for the first
256 // NALU of this packet.
257 if (aggregated_fragments == 0)
258 fragment_headers_length += kHevcNalHeaderSize + kHevcLengthFieldSize;
259 ++aggregated_fragments;
260
261 // Next fragment.
262 ++fragment_index;
263 if (fragment_index == input_fragments_.size())
264 break;
265 fragment = &input_fragments_[fragment_index];
266 }
267 RTC_CHECK_GT(aggregated_fragments, 0);
268 packets_.back().last_fragment = true;
269 return fragment_index;
270 }
271
NextPacket(RtpPacketToSend * rtp_packet)272 bool RtpPacketizerH265::NextPacket(RtpPacketToSend* rtp_packet) {
273 RTC_DCHECK(rtp_packet);
274
275 if (packets_.empty()) {
276 return false;
277 }
278
279 PacketUnit packet = packets_.front();
280
281 if (packet.first_fragment && packet.last_fragment) {
282 // Single NAL unit packet.
283 size_t bytes_to_send = packet.source_fragment.length;
284 uint8_t* buffer = rtp_packet->AllocatePayload(bytes_to_send);
285 memcpy(buffer, packet.source_fragment.buffer, bytes_to_send);
286 packets_.pop();
287 input_fragments_.pop_front();
288 } else if (packet.aggregated) {
289 bool is_last_packet = num_packets_left_ == 1;
290 NextAggregatePacket(rtp_packet, is_last_packet);
291 } else {
292 NextFragmentPacket(rtp_packet);
293 }
294 rtp_packet->SetMarker(packets_.empty());
295 --num_packets_left_;
296 return true;
297 }
298
NextAggregatePacket(RtpPacketToSend * rtp_packet,bool last)299 void RtpPacketizerH265::NextAggregatePacket(RtpPacketToSend* rtp_packet,
300 bool last) {
301 size_t payload_capacity = rtp_packet->FreeCapacity();
302 RTC_CHECK_GE(payload_capacity, kHevcNalHeaderSize);
303 uint8_t* buffer = rtp_packet->AllocatePayload(payload_capacity);
304
305 PacketUnit* packet = &packets_.front();
306 RTC_CHECK(packet->first_fragment);
307 uint8_t payload_hdr_h = packet->header >> 8;
308 uint8_t payload_hdr_l = packet->header & 0xFF;
309 uint8_t layer_id_h = payload_hdr_h & kHevcLayerIDHMask;
310
311 payload_hdr_h =
312 (payload_hdr_h & kHevcTypeMaskN) | (kHevcAp << 1) | layer_id_h;
313
314 buffer[0] = payload_hdr_h;
315 buffer[1] = payload_hdr_l;
316 int index = kHevcNalHeaderSize;
317 bool is_last_fragment = packet->last_fragment;
318 while (packet->aggregated) {
319 // Add NAL unit length field.
320 const Fragment& fragment = packet->source_fragment;
321 ByteWriter<uint16_t>::WriteBigEndian(&buffer[index], fragment.length);
322 index += kHevcLengthFieldSize;
323 // Add NAL unit.
324 memcpy(&buffer[index], fragment.buffer, fragment.length);
325 index += fragment.length;
326 packets_.pop();
327 input_fragments_.pop_front();
328 if (is_last_fragment)
329 break;
330 packet = &packets_.front();
331 is_last_fragment = packet->last_fragment;
332 }
333 RTC_CHECK(is_last_fragment);
334 rtp_packet->SetPayloadSize(index);
335 }
336
NextFragmentPacket(RtpPacketToSend * rtp_packet)337 void RtpPacketizerH265::NextFragmentPacket(RtpPacketToSend* rtp_packet) {
338 PacketUnit* packet = &packets_.front();
339 // NAL unit fragmented over multiple packets (FU).
340 // We do not send original NALU header, so it will be replaced by the
341 // PayloadHdr of the first packet.
342 uint8_t payload_hdr_h =
343 packet->header >> 8; // 1-bit F, 6-bit type, 1-bit layerID highest-bit
344 uint8_t payload_hdr_l = packet->header & 0xFF;
345 uint8_t layer_id_h = payload_hdr_h & kHevcLayerIDHMask;
346 uint8_t fu_header = 0;
347 // S | E |6 bit type.
348 fu_header |= (packet->first_fragment ? kHevcSBit : 0);
349 fu_header |= (packet->last_fragment ? kHevcEBit : 0);
350 uint8_t type = (payload_hdr_h & kHevcTypeMask) >> 1;
351 fu_header |= type;
352 // Now update payload_hdr_h with FU type.
353 payload_hdr_h =
354 (payload_hdr_h & kHevcTypeMaskN) | (kHevcFu << 1) | layer_id_h;
355 const Fragment& fragment = packet->source_fragment;
356 uint8_t* buffer = rtp_packet->AllocatePayload(
357 kHevcFuHeaderSize + kHevcNalHeaderSize + fragment.length);
358 RTC_CHECK(buffer);
359 buffer[0] = payload_hdr_h;
360 buffer[1] = payload_hdr_l;
361 buffer[2] = fu_header;
362
363 if (packet->last_fragment) {
364 memcpy(buffer + kHevcFuHeaderSize + kHevcNalHeaderSize, fragment.buffer,
365 fragment.length);
366 } else {
367 memcpy(buffer + kHevcFuHeaderSize + kHevcNalHeaderSize, fragment.buffer,
368 fragment.length);
369 }
370 packets_.pop();
371 }
372
373 } // namespace webrtc
374