1 /*
2  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #include "modules/video_coding/frame_buffer.h"
12 
13 #include <assert.h>
14 #include <string.h>
15 
16 #include "modules/video_coding/packet.h"
17 #include "rtc_base/checks.h"
18 #include "rtc_base/logging.h"
19 #include "rtc_base/trace_event.h"
20 
21 namespace webrtc {
22 
VCMFrameBuffer()23 VCMFrameBuffer::VCMFrameBuffer()
24     : _state(kStateEmpty), _nackCount(0), _latestPacketTimeMs(-1) {}
25 
~VCMFrameBuffer()26 VCMFrameBuffer::~VCMFrameBuffer() {}
27 
VCMFrameBuffer(const VCMFrameBuffer & rhs)28 VCMFrameBuffer::VCMFrameBuffer(const VCMFrameBuffer& rhs)
29     : VCMEncodedFrame(rhs),
30       _state(rhs._state),
31       _sessionInfo(),
32       _nackCount(rhs._nackCount),
33       _latestPacketTimeMs(rhs._latestPacketTimeMs) {
34   _sessionInfo = rhs._sessionInfo;
35   _sessionInfo.UpdateDataPointers(rhs._buffer, _buffer);
36 }
37 
FrameType() const38 webrtc::FrameType VCMFrameBuffer::FrameType() const {
39   return _sessionInfo.FrameType();
40 }
41 
GetLowSeqNum() const42 int32_t VCMFrameBuffer::GetLowSeqNum() const {
43   return _sessionInfo.LowSequenceNumber();
44 }
45 
GetHighSeqNum() const46 int32_t VCMFrameBuffer::GetHighSeqNum() const {
47   return _sessionInfo.HighSequenceNumber();
48 }
49 
PictureId() const50 int VCMFrameBuffer::PictureId() const {
51   return _sessionInfo.PictureId();
52 }
53 
TemporalId() const54 int VCMFrameBuffer::TemporalId() const {
55   return _sessionInfo.TemporalId();
56 }
57 
LayerSync() const58 bool VCMFrameBuffer::LayerSync() const {
59   return _sessionInfo.LayerSync();
60 }
61 
Tl0PicId() const62 int VCMFrameBuffer::Tl0PicId() const {
63   return _sessionInfo.Tl0PicId();
64 }
65 
NonReference() const66 bool VCMFrameBuffer::NonReference() const {
67   return _sessionInfo.NonReference();
68 }
69 
GetNaluInfos() const70 std::vector<NaluInfo> VCMFrameBuffer::GetNaluInfos() const {
71   return _sessionInfo.GetNaluInfos();
72 }
73 
SetGofInfo(const GofInfoVP9 & gof_info,size_t idx)74 void VCMFrameBuffer::SetGofInfo(const GofInfoVP9& gof_info, size_t idx) {
75   TRACE_EVENT0("webrtc", "VCMFrameBuffer::SetGofInfo");
76   _sessionInfo.SetGofInfo(gof_info, idx);
77   // TODO(asapersson): Consider adding hdr->VP9.ref_picture_id for testing.
78   _codecSpecificInfo.codecSpecific.VP9.temporal_idx =
79       gof_info.temporal_idx[idx];
80   _codecSpecificInfo.codecSpecific.VP9.temporal_up_switch =
81       gof_info.temporal_up_switch[idx];
82 }
83 
IsSessionComplete() const84 bool VCMFrameBuffer::IsSessionComplete() const {
85   TRACE_EVENT0("webrtc", "VCMFrameBuffer::IsSessionComplete");
86   return _sessionInfo.complete();
87 }
88 
89 // Insert packet
InsertPacket(const VCMPacket & packet,int64_t timeInMs,VCMDecodeErrorMode decode_error_mode,const FrameData & frame_data)90 VCMFrameBufferEnum VCMFrameBuffer::InsertPacket(
91     const VCMPacket& packet,
92     int64_t timeInMs,
93     VCMDecodeErrorMode decode_error_mode,
94     const FrameData& frame_data) {
95   TRACE_EVENT0("webrtc", "VCMFrameBuffer::InsertPacket");
96   assert(!(NULL == packet.dataPtr && packet.sizeBytes > 0));
97   if (packet.dataPtr != NULL) {
98     _payloadType = packet.payloadType;
99   }
100 
101   if (kStateEmpty == _state) {
102     // First packet (empty and/or media) inserted into this frame.
103     // store some info and set some initial values.
104     _timeStamp = packet.timestamp;
105     // We only take the ntp timestamp of the first packet of a frame.
106     ntp_time_ms_ = packet.ntp_time_ms_;
107     _codec = packet.codec;
108     if (packet.frameType != kEmptyFrame) {
109       // first media packet
110       SetState(kStateIncomplete);
111     }
112   }
113 
114   // add safety margin because STAP-A packets can cause it to expand by
115   // ~two bytes per NAL
116   uint32_t requiredSizeBytes =
117       Length() + packet.sizeBytes +
118       (packet.insertStartCode ? kH264StartCodeLengthBytes : 0) +
119       kBufferSafetyMargin +
120       EncodedImage::GetBufferPaddingBytes(packet.codec);
121   if (requiredSizeBytes >= _size) {
122     const uint8_t* prevBuffer = _buffer;
123     const uint32_t increments =
124         requiredSizeBytes / kBufferIncStepSizeBytes +
125         (requiredSizeBytes % kBufferIncStepSizeBytes > 0);
126     const uint32_t newSize = _size + increments * kBufferIncStepSizeBytes;
127     if (newSize > kMaxJBFrameSizeBytes) {
128       RTC_LOG(LS_ERROR) << "Failed to insert packet due to frame being too "
129                            "big.";
130       return kSizeError;
131     }
132     VerifyAndAllocate(newSize);
133     _sessionInfo.UpdateDataPointers(prevBuffer, _buffer);
134   }
135 
136   if (packet.width > 0 && packet.height > 0) {
137     _encodedWidth = packet.width;
138     _encodedHeight = packet.height;
139   }
140 
141   // Don't copy payload specific data for empty packets (e.g padding packets).
142   if (packet.sizeBytes > 0)
143     CopyCodecSpecific(&packet.video_header);
144 
145   int retVal =
146       _sessionInfo.InsertPacket(packet, _buffer, decode_error_mode, frame_data);
147   if (retVal == -1) {
148     return kSizeError;
149   } else if (retVal == -2) {
150     return kDuplicatePacket;
151   } else if (retVal == -3) {
152     return kOutOfBoundsPacket;
153   }
154   // update length
155   _length = Length() + static_cast<uint32_t>(retVal);
156 
157   _latestPacketTimeMs = timeInMs;
158 
159   // http://www.etsi.org/deliver/etsi_ts/126100_126199/126114/12.07.00_60/
160   // ts_126114v120700p.pdf Section 7.4.5.
161   // The MTSI client shall add the payload bytes as defined in this clause
162   // onto the last RTP packet in each group of packets which make up a key
163   // frame (I-frame or IDR frame in H.264 (AVC), or an IRAP picture in H.265
164   // (HEVC)).
165   if (packet.markerBit) {
166     RTC_DCHECK(!_rotation_set);
167     rotation_ = packet.video_header.rotation;
168     _rotation_set = true;
169     content_type_ = packet.video_header.content_type;
170     if (packet.video_header.video_timing.flags != TimingFrameFlags::kInvalid) {
171       timing_.encode_start_ms =
172           ntp_time_ms_ + packet.video_header.video_timing.encode_start_delta_ms;
173       timing_.encode_finish_ms =
174           ntp_time_ms_ +
175           packet.video_header.video_timing.encode_finish_delta_ms;
176       timing_.packetization_finish_ms =
177           ntp_time_ms_ +
178           packet.video_header.video_timing.packetization_finish_delta_ms;
179       timing_.pacer_exit_ms =
180           ntp_time_ms_ + packet.video_header.video_timing.pacer_exit_delta_ms;
181       timing_.network_timestamp_ms =
182           ntp_time_ms_ +
183           packet.video_header.video_timing.network_timestamp_delta_ms;
184       timing_.network2_timestamp_ms =
185           ntp_time_ms_ +
186           packet.video_header.video_timing.network2_timestamp_delta_ms;
187     }
188     timing_.flags = packet.video_header.video_timing.flags;
189   }
190 
191   if (packet.is_first_packet_in_frame) {
192     playout_delay_ = packet.video_header.playout_delay;
193   }
194 
195   if (_sessionInfo.complete()) {
196     SetState(kStateComplete);
197     return kCompleteSession;
198   } else if (_sessionInfo.decodable()) {
199     SetState(kStateDecodable);
200     return kDecodableSession;
201   }
202   return kIncomplete;
203 }
204 
LatestPacketTimeMs() const205 int64_t VCMFrameBuffer::LatestPacketTimeMs() const {
206   TRACE_EVENT0("webrtc", "VCMFrameBuffer::LatestPacketTimeMs");
207   return _latestPacketTimeMs;
208 }
209 
IncrementNackCount()210 void VCMFrameBuffer::IncrementNackCount() {
211   TRACE_EVENT0("webrtc", "VCMFrameBuffer::IncrementNackCount");
212   _nackCount++;
213 }
214 
GetNackCount() const215 int16_t VCMFrameBuffer::GetNackCount() const {
216   TRACE_EVENT0("webrtc", "VCMFrameBuffer::GetNackCount");
217   return _nackCount;
218 }
219 
HaveFirstPacket() const220 bool VCMFrameBuffer::HaveFirstPacket() const {
221   TRACE_EVENT0("webrtc", "VCMFrameBuffer::HaveFirstPacket");
222   return _sessionInfo.HaveFirstPacket();
223 }
224 
HaveLastPacket() const225 bool VCMFrameBuffer::HaveLastPacket() const {
226   TRACE_EVENT0("webrtc", "VCMFrameBuffer::HaveLastPacket");
227   return _sessionInfo.HaveLastPacket();
228 }
229 
NumPackets() const230 int VCMFrameBuffer::NumPackets() const {
231   TRACE_EVENT0("webrtc", "VCMFrameBuffer::NumPackets");
232   return _sessionInfo.NumPackets();
233 }
234 
Reset()235 void VCMFrameBuffer::Reset() {
236   TRACE_EVENT0("webrtc", "VCMFrameBuffer::Reset");
237   _length = 0;
238   _timeStamp = 0;
239   _sessionInfo.Reset();
240   _payloadType = 0;
241   _nackCount = 0;
242   _latestPacketTimeMs = -1;
243   _state = kStateEmpty;
244   VCMEncodedFrame::Reset();
245 }
246 
247 // Set state of frame
SetState(VCMFrameBufferStateEnum state)248 void VCMFrameBuffer::SetState(VCMFrameBufferStateEnum state) {
249   TRACE_EVENT0("webrtc", "VCMFrameBuffer::SetState");
250   if (_state == state) {
251     return;
252   }
253   switch (state) {
254     case kStateIncomplete:
255       // we can go to this state from state kStateEmpty
256       assert(_state == kStateEmpty);
257 
258       // Do nothing, we received a packet
259       break;
260 
261     case kStateComplete:
262       assert(_state == kStateEmpty || _state == kStateIncomplete ||
263              _state == kStateDecodable);
264 
265       break;
266 
267     case kStateEmpty:
268       // Should only be set to empty through Reset().
269       assert(false);
270       break;
271 
272     case kStateDecodable:
273       assert(_state == kStateEmpty || _state == kStateIncomplete);
274       break;
275   }
276   _state = state;
277 }
278 
279 // Get current state of frame
GetState() const280 VCMFrameBufferStateEnum VCMFrameBuffer::GetState() const {
281   return _state;
282 }
283 
284 // Get current state of frame
GetState(uint32_t & timeStamp) const285 VCMFrameBufferStateEnum VCMFrameBuffer::GetState(uint32_t& timeStamp) const {
286   TRACE_EVENT0("webrtc", "VCMFrameBuffer::GetState");
287   timeStamp = TimeStamp();
288   return GetState();
289 }
290 
IsRetransmitted() const291 bool VCMFrameBuffer::IsRetransmitted() const {
292   return _sessionInfo.session_nack();
293 }
294 
PrepareForDecode(bool continuous)295 void VCMFrameBuffer::PrepareForDecode(bool continuous) {
296   TRACE_EVENT0("webrtc", "VCMFrameBuffer::PrepareForDecode");
297   size_t bytes_removed = _sessionInfo.MakeDecodable();
298   _length -= bytes_removed;
299   // Transfer frame information to EncodedFrame and create any codec
300   // specific information.
301   _frameType = _sessionInfo.FrameType();
302   _completeFrame = _sessionInfo.complete();
303   _missingFrame = !continuous;
304 }
305 
306 }  // namespace webrtc
307