1 /*
2  * Copyright 2019 The libgav1 Authors
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #ifndef LIBGAV1_SRC_DECODER_IMPL_H_
18 #define LIBGAV1_SRC_DECODER_IMPL_H_
19 
20 #include <array>
21 #include <condition_variable>  // NOLINT (unapproved c++11 header)
22 #include <cstddef>
23 #include <cstdint>
24 #include <memory>
25 #include <mutex>  // NOLINT (unapproved c++11 header)
26 
27 #include "src/buffer_pool.h"
28 #include "src/decoder_state.h"
29 #include "src/dsp/constants.h"
30 #include "src/frame_scratch_buffer.h"
31 #include "src/gav1/decoder_buffer.h"
32 #include "src/gav1/decoder_settings.h"
33 #include "src/gav1/status_code.h"
34 #include "src/obu_parser.h"
35 #include "src/quantizer.h"
36 #include "src/residual_buffer_pool.h"
37 #include "src/symbol_decoder_context.h"
38 #include "src/tile.h"
39 #include "src/utils/array_2d.h"
40 #include "src/utils/block_parameters_holder.h"
41 #include "src/utils/compiler_attributes.h"
42 #include "src/utils/constants.h"
43 #include "src/utils/memory.h"
44 #include "src/utils/queue.h"
45 #include "src/utils/segmentation_map.h"
46 #include "src/utils/types.h"
47 
48 namespace libgav1 {
49 
50 struct TemporalUnit;
51 
52 struct EncodedFrame {
EncodedFrameEncodedFrame53   EncodedFrame(ObuParser* const obu, const DecoderState& state,
54                const RefCountedBufferPtr& frame, int position_in_temporal_unit)
55       : sequence_header(obu->sequence_header()),
56         frame_header(obu->frame_header()),
57         state(state),
58         temporal_unit(nullptr),
59         frame(frame),
60         position_in_temporal_unit(position_in_temporal_unit) {
61     obu->MoveTileBuffers(&tile_buffers);
62     frame->MarkFrameAsStarted();
63   }
64 
65   const ObuSequenceHeader sequence_header;
66   const ObuFrameHeader frame_header;
67   Vector<TileBuffer> tile_buffers;
68   DecoderState state;
69   TemporalUnit* temporal_unit;
70   RefCountedBufferPtr frame;
71   const int position_in_temporal_unit;
72 };
73 
74 struct TemporalUnit : public Allocable {
75   // The default constructor is invoked by the Queue<TemporalUnit>::Init()
76   // method. Queue<> does not use the default-constructed elements, so it is
77   // safe for the default constructor to not initialize the members.
78   TemporalUnit() = default;
TemporalUnitTemporalUnit79   TemporalUnit(const uint8_t* data, size_t size, int64_t user_private_data,
80                void* buffer_private_data)
81       : data(data),
82         size(size),
83         user_private_data(user_private_data),
84         buffer_private_data(buffer_private_data),
85         decoded(false),
86         status(kStatusOk),
87         has_displayable_frame(false),
88         output_frame_position(-1),
89         decoded_count(0),
90         output_layer_count(0),
91         released_input_buffer(false) {}
92 
93   const uint8_t* data;
94   size_t size;
95   int64_t user_private_data;
96   void* buffer_private_data;
97 
98   // The following members are used only in frame parallel mode.
99   bool decoded;
100   StatusCode status;
101   bool has_displayable_frame;
102   int output_frame_position;
103 
104   Vector<EncodedFrame> frames;
105   size_t decoded_count;
106 
107   // The struct (and the counter) is used to support output of multiple layers
108   // within a single temporal unit. The decoding process will store the output
109   // frames in |output_layers| in the order they are finished decoding. At the
110   // end of the decoding process, this array will be sorted in reverse order of
111   // |position_in_temporal_unit|. DequeueFrame() will then return the frames in
112   // reverse order (so that the entire process can run with a single counter
113   // variable).
114   struct OutputLayer {
115     // Used by std::sort to sort |output_layers| in reverse order of
116     // |position_in_temporal_unit|.
117     bool operator<(const OutputLayer& rhs) const {
118       return position_in_temporal_unit > rhs.position_in_temporal_unit;
119     }
120 
121     RefCountedBufferPtr frame;
122     int position_in_temporal_unit = 0;
123   } output_layers[kMaxLayers];
124   // Number of entries in |output_layers|.
125   int output_layer_count;
126   // Flag to ensure that we release the input buffer only once if there are
127   // multiple output layers.
128   bool released_input_buffer;
129 };
130 
131 class DecoderImpl : public Allocable {
132  public:
133   // The constructor saves a const reference to |*settings|. Therefore
134   // |*settings| must outlive the DecoderImpl object. On success, |*output|
135   // contains a pointer to the newly-created DecoderImpl object. On failure,
136   // |*output| is not modified.
137   static StatusCode Create(const DecoderSettings* settings,
138                            std::unique_ptr<DecoderImpl>* output);
139   ~DecoderImpl();
140   StatusCode EnqueueFrame(const uint8_t* data, size_t size,
141                           int64_t user_private_data, void* buffer_private_data);
142   StatusCode DequeueFrame(const DecoderBuffer** out_ptr);
GetMaxBitdepth()143   static constexpr int GetMaxBitdepth() {
144     static_assert(LIBGAV1_MAX_BITDEPTH == 8 || LIBGAV1_MAX_BITDEPTH == 10,
145                   "LIBGAV1_MAX_BITDEPTH must be 8 or 10.");
146     return LIBGAV1_MAX_BITDEPTH;
147   }
148 
149  private:
150   explicit DecoderImpl(const DecoderSettings* settings);
151   StatusCode Init();
152   // Called when the first frame is enqueued. It does the OBU parsing for one
153   // temporal unit to retrieve the tile configuration and sets up the frame
154   // threading if frame parallel mode is allowed. It also initializes the
155   // |temporal_units_| queue based on the number of frame threads.
156   //
157   // The following are the limitations of the current implementation:
158   //  * It assumes that all frames in the video have the same tile
159   //    configuration. The frame parallel threading model will not be updated
160   //    based on tile configuration changes mid-stream.
161   //  * The above assumption holds true even when there is a new coded video
162   //    sequence (i.e.) a new sequence header.
163   StatusCode InitializeFrameThreadPoolAndTemporalUnitQueue(const uint8_t* data,
164                                                            size_t size);
165   // Used only in frame parallel mode. Signals failure and waits until the
166   // worker threads are aborted if |status| is a failure status. If |status| is
167   // equal to kStatusOk or kStatusTryAgain, this function does not do anything.
168   // Always returns the input parameter |status| as the return value.
169   //
170   // This function is called only from the application thread (from
171   // EnqueueFrame() and DequeueFrame()).
172   StatusCode SignalFailure(StatusCode status);
173 
174   void ReleaseOutputFrame();
175 
176   // Decodes all the frames contained in the given temporal unit. Used only in
177   // non frame parallel mode.
178   StatusCode DecodeTemporalUnit(const TemporalUnit& temporal_unit,
179                                 const DecoderBuffer** out_ptr);
180   // Used only in frame parallel mode. Does the OBU parsing for |data| and
181   // schedules the individual frames for decoding in the |frame_thread_pool_|.
182   StatusCode ParseAndSchedule(const uint8_t* data, size_t size,
183                               int64_t user_private_data,
184                               void* buffer_private_data);
185   // Decodes the |encoded_frame| and updates the
186   // |encoded_frame->temporal_unit|'s parameters if the decoded frame is a
187   // displayable frame. Used only in frame parallel mode.
188   StatusCode DecodeFrame(EncodedFrame* encoded_frame);
189 
190   // Populates |buffer_| with values from |frame|. Adds a reference to |frame|
191   // in |output_frame_|.
192   StatusCode CopyFrameToOutputBuffer(const RefCountedBufferPtr& frame);
193   StatusCode DecodeTiles(const ObuSequenceHeader& sequence_header,
194                          const ObuFrameHeader& frame_header,
195                          const Vector<TileBuffer>& tile_buffers,
196                          const DecoderState& state,
197                          FrameScratchBuffer* frame_scratch_buffer,
198                          RefCountedBuffer* current_frame);
199   // Applies film grain synthesis to the |displayable_frame| and stores the film
200   // grain applied frame into |film_grain_frame|. Returns kStatusOk on success.
201   StatusCode ApplyFilmGrain(const ObuSequenceHeader& sequence_header,
202                             const ObuFrameHeader& frame_header,
203                             const RefCountedBufferPtr& displayable_frame,
204                             RefCountedBufferPtr* film_grain_frame,
205                             ThreadPool* thread_pool);
206 
207   bool IsNewSequenceHeader(const ObuParser& obu);
208 
HasFailure()209   bool HasFailure() {
210     std::lock_guard<std::mutex> lock(mutex_);
211     return failure_status_ != kStatusOk;
212   }
213 
214   // Initializes the |quantizer_matrix_| if necessary and sets
215   // |quantizer_matrix_initialized_| to true.
216   bool MaybeInitializeQuantizerMatrix(const ObuFrameHeader& frame_header);
217 
218   // Allocates and generates the |wedge_masks_| if necessary and sets
219   // |wedge_masks_initialized_| to true.
220   bool MaybeInitializeWedgeMasks(FrameType frame_type);
221 
222   // Elements in this queue cannot be moved with std::move since the
223   // |EncodedFrame.temporal_unit| stores a pointer to elements in this queue.
224   Queue<TemporalUnit> temporal_units_;
225   DecoderState state_;
226 
227   DecoderBuffer buffer_ = {};
228   // |output_frame_| holds a reference to the output frame on behalf of
229   // |buffer_|.
230   RefCountedBufferPtr output_frame_;
231 
232   // Queue of output frames that are to be returned in the DequeueFrame() calls.
233   // If |settings_.output_all_layers| is false, this queue will never contain
234   // more than 1 element. This queue is used only when |is_frame_parallel_| is
235   // false.
236   Queue<RefCountedBufferPtr> output_frame_queue_;
237 
238   BufferPool buffer_pool_;
239   WedgeMaskArray wedge_masks_;
240   bool wedge_masks_initialized_ = false;
241   QuantizerMatrix quantizer_matrix_;
242   bool quantizer_matrix_initialized_ = false;
243   FrameScratchBufferPool frame_scratch_buffer_pool_;
244 
245   // Used to synchronize the accesses into |temporal_units_| in order to update
246   // the "decoded" state of an temporal unit.
247   std::mutex mutex_;
248   std::condition_variable decoded_condvar_;
249   bool is_frame_parallel_;
250   std::unique_ptr<ThreadPool> frame_thread_pool_;
251 
252   // In frame parallel mode, there are two primary points of failure:
253   //  1) ParseAndSchedule()
254   //  2) DecodeTiles()
255   // Both of these functions have to respond to the other one failing by
256   // aborting whatever they are doing. This variable is used to accomplish that.
257   // If |failure_status_| is not kStatusOk, then the two functions will try to
258   // abort as early as they can.
259   StatusCode failure_status_ = kStatusOk LIBGAV1_GUARDED_BY(mutex_);
260 
261   ObuSequenceHeader sequence_header_ = {};
262   // If true, sequence_header is valid.
263   bool has_sequence_header_ = false;
264 
265   const DecoderSettings& settings_;
266   bool seen_first_frame_ = false;
267 };
268 
269 }  // namespace libgav1
270 
271 #endif  // LIBGAV1_SRC_DECODER_IMPL_H_
272