1 /* -----------------------------------------------------------------------------
2 The copyright in this software is being made available under the BSD
3 License, included below. No patent rights, trademark rights and/or
4 other Intellectual Property Rights other than the copyrights concerning
5 the Software are granted under this license.
6 
7 For any license concerning other Intellectual Property rights than the software,
8 especially patent licenses, a separate Agreement needs to be closed.
9 For more information please contact:
10 
11 Fraunhofer Heinrich Hertz Institute
12 Einsteinufer 37
13 10587 Berlin, Germany
14 www.hhi.fraunhofer.de/vvc
15 vvc@hhi.fraunhofer.de
16 
17 Copyright (c) 2018-2021, Fraunhofer-Gesellschaft zur Förderung der angewandten Forschung e.V.
18 All rights reserved.
19 
20 Redistribution and use in source and binary forms, with or without
21 modification, are permitted provided that the following conditions are met:
22 
23  * Redistributions of source code must retain the above copyright notice,
24    this list of conditions and the following disclaimer.
25  * Redistributions in binary form must reproduce the above copyright notice,
26    this list of conditions and the following disclaimer in the documentation
27    and/or other materials provided with the distribution.
28  * Neither the name of Fraunhofer nor the names of its contributors may
29    be used to endorse or promote products derived from this software without
30    specific prior written permission.
31 
32 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
33 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
34 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
35 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS
36 BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
37 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
38 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
39 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
40 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
41 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
42 THE POSSIBILITY OF SUCH DAMAGE.
43 
44 
45 ------------------------------------------------------------------------------------------- */
46 
47 /** \file     Picture.h
48  *  \brief    Description of a coded picture
49  */
50 
51 #pragma once
52 
53 #include "CommonDef.h"
54 
55 #include "Common.h"
56 #include "Unit.h"
57 #include "Buffer.h"
58 #include "Unit.h"
59 #include "Slice.h"
60 #include "CodingStructure.h"
61 #include <deque>
62 
63 #include "InterpolationFilter.h"
64 
65 #include "vvdec/sei.h"
66 
67 namespace vvdec
68 {
69 
70 struct Picture : public UnitArea
71 {
72   Picture() = default;
73   ~Picture() = default;
74 
75   void create(const ChromaFormat &_chromaFormat, const Size &size, const unsigned _maxCUSize, const unsigned margin, const int layerId);
76   void resetForUse();
77   void destroy();
78 
79          Pel*      getRecoBufPtr   (const ComponentID compID, bool wrap=false)       { return m_bufs[wrap ? PIC_RECON_WRAP : PIC_RECONSTRUCTION].bufs[compID].buf; }
80   const  Pel*      getRecoBufPtr   (const ComponentID compID, bool wrap=false) const { return m_bufs[wrap ? PIC_RECON_WRAP : PIC_RECONSTRUCTION].bufs[compID].buf; }
81          ptrdiff_t getRecoBufStride(const ComponentID compID, bool wrap=false)       { return m_bufs[wrap ? PIC_RECON_WRAP : PIC_RECONSTRUCTION].bufs[compID].stride; }
82   const  ptrdiff_t getRecoBufStride(const ComponentID compID, bool wrap=false) const { return m_bufs[wrap ? PIC_RECON_WRAP : PIC_RECONSTRUCTION].bufs[compID].stride; }
83          PelBuf     getRecoBuf(const ComponentID compID, bool wrap=false);
84   const CPelBuf     getRecoBuf(const ComponentID compID, bool wrap=false) const;
85          PelBuf     getRecoBuf(const CompArea &blk,      bool wrap=false);
86   const CPelBuf     getRecoBuf(const CompArea &blk,      bool wrap=false) const;
87          PelUnitBuf getRecoBuf(const UnitArea &unit,     bool wrap=false);
88   const CPelUnitBuf getRecoBuf(const UnitArea &unit,     bool wrap=false) const;
89          PelUnitBuf getRecoBuf(bool wrap=false);
90   const CPelUnitBuf getRecoBuf(bool wrap=false) const;
91 
92   // This returns a CPelBuf, with the origin at the picture origin (0,0), but the actual storage size of the sub picture.
93   // It can be used the same way as the full picture buffer, but you should only reference the data within the actual sub picture.
94   // Also, handle with care, because stride < width.
95   const CPelBuf getSubPicBuf( int subPicIdx, const ComponentID compID, bool wrap = false ) const
96   {
97     CHECK( wrap, "wraparound for subpics not supported yet" );
98 
99     Position subPicPos( subPictures[subPicIdx].getSubPicLeft() >> getComponentScaleX( compID, m_subPicRefBufs[subPicIdx].chromaFormat ),
100                         subPictures[subPicIdx].getSubPicTop()  >> getComponentScaleY( compID, m_subPicRefBufs[subPicIdx].chromaFormat ) );
101 
102     Size targetSize( m_bufs[PIC_RECONSTRUCTION].get( compID ) );
103 
104     const auto& subPicComp = m_subPicRefBufs[subPicIdx].bufs[compID];
105     return CPelBuf( subPicComp.bufAt( -subPicPos.x, -subPicPos.y ), subPicComp.stride, targetSize );
106   }
107   const Pel*      getSubPicBufPtr   ( int subPicIdx, const ComponentID compID, bool wrap = false ) const { return getSubPicBuf( subPicIdx, compID, wrap ).buf;    }
108   const ptrdiff_t getSubPicBufStride( int subPicIdx, const ComponentID compID, bool wrap = false ) const { return getSubPicBuf( subPicIdx, compID, wrap ).stride; }
109 
110 private:
getBufPicture111          PelBuf     getBuf(const ComponentID compID, const PictureType &type)       { return m_bufs[type].bufs[ compID ]; }
getBufPicture112   const CPelBuf     getBuf(const ComponentID compID, const PictureType &type) const { return m_bufs[type].bufs[ compID ]; }
113          PelBuf     getBuf(const CompArea &blk,      const PictureType &type);
114   const CPelBuf     getBuf(const CompArea &blk,      const PictureType &type) const;
115          PelUnitBuf getBuf(const UnitArea &unit,     const PictureType &type);
116   const CPelUnitBuf getBuf(const UnitArea &unit,     const PictureType &type) const;
117 
118 public:
119   void extendPicBorder    (                  bool top = true, bool bottom = true, bool leftrightT = true, bool leftrightB = true, ChannelType chType = MAX_NUM_CHANNEL_TYPE );
120   void extendPicBorderBuf ( PelStorage& buf, bool top = true, bool bottom = true, bool leftrightT = true, bool leftrightB = true, ChannelType chType = MAX_NUM_CHANNEL_TYPE );
121   void extendPicBorderWrap(                  bool top = true, bool bottom = true, bool leftrightT = true, bool leftrightB = true, ChannelType chType = MAX_NUM_CHANNEL_TYPE );
122 
123   void (*paddPicBorderBot) (Pel *pi, ptrdiff_t stride,int width,int xmargin,int ymargin);
124   void (*paddPicBorderTop) (Pel *pi, ptrdiff_t stride,int width,int xmargin,int ymargin);
125   void (*paddPicBorderLeftRight) (Pel *pi, ptrdiff_t stride,int width,int xmargin,int height);
126 
127   void finalInit( const SPS * sps, const PPS * pps, PicHeader *picHeader, APS* alfApss[ALF_CTB_MAX_NUM_APS], APS * lmcsAps, APS* scalingListAps, bool phPSupdate = true );
getPOCPicture128   int      getPOC()                           const { return poc; }
getCtsPicture129   uint64_t getCts()                           const { return cts; }
getDtsPicture130   uint64_t getDts()                           const { return dts; }
getTLayerPicture131   uint32_t getTLayer()                        const { return layer; }
getNaluBitsPicture132   uint64_t getNaluBits()                      const { return bits; }
getRapPicture133   bool     getRap()                           const { return rap; }
134 
135   Pel*   getOrigin( const PictureType &type, const ComponentID compID ) const;
136   PelBuf getOriginBuf( const PictureType &type, const ComponentID compID );
137 
getDecodingOrderNumberPicture138   int  getDecodingOrderNumber()               const { return decodingOrderNumber; }
setDecodingOrderNumberPicture139   void setDecodingOrderNumber(const int val)        { decodingOrderNumber = val;  }
140 
getMixedNaluTypesInPicFlagPicture141   bool getMixedNaluTypesInPicFlag()           const { return slices[0]->getPPS()->getMixedNaluTypesInPicFlag(); }
142 
143 public:
144   void startProcessingTimer();
145   void stopProcessingTimer();
resetProcessingTimePicture146   void resetProcessingTime() { m_dProcessingTime = 0; }
getProcessingTimePicture147   double getProcessingTime() const { return m_dProcessingTime; }
148 
149   std::chrono::time_point<std::chrono::steady_clock> m_processingStartTime;
150   double                                             m_dProcessingTime = 0;
151 
152   bool subPicExtStarted               = false;
153   bool borderExtStarted               = false;
154   bool wrapAroundValid                = false;
155   unsigned wrapAroundOffset           = 0;
156   bool referenced                     = false;
157   bool reconstructed                  = false;
158   bool inProgress                     = false;
159   bool neededForOutput                = false;
160   bool wasLost                        = false;
161   bool longTerm                       = false;
162   bool topField                       = false;
163   bool fieldPic                       = false;
164   int  skippedDecCount                = 0;
165   int  m_prevQP[MAX_NUM_CHANNEL_TYPE] = { -1, -1 };
166   bool nonReferencePictureFlag        = false;
167   bool              picCheckedDPH     = false;
168   std::vector<bool> subpicsCheckedDPH;
169 
170   // As long as this field is true, the picture will not be reused or deleted.
171   // An external application needs to call DecLib::releasePicture(), when it is done using the picture buffer.
172   bool lockedByApplication = false;
173 
174   int         poc          = 0;
175   uint64_t    cts          = 0;   // composition time stamp
176   uint64_t    dts          = 0;   // decoding time stamp
177   uint32_t    layer        = std::numeric_limits<uint32_t>::max();
178   uint32_t    depth        = 0;
179   int         layerId      = NOT_VALID;
180   NalUnitType eNalUnitType = NAL_UNIT_INVALID;
181   uint64_t    bits         = 0;   // input nal bit count
182   bool        rap          = 0;   // random access point flag
183   int         decodingOrderNumber = 0;
184 
185   std::vector<int>        sliceSubpicIdx;
186   std::vector<SubPic>     subPictures;
187   std::vector<PelStorage> m_subPicRefBufs;   // used as reference for subpictures, that are treated as pictures
188 
189   bool subLayerNonReferencePictureDueToSTSA = 0;
190 
191   PelStorage     m_bufs[NUM_PIC_TYPES];
192   uint32_t       margin = 0;
193   const Picture* unscaledPic;
194 
195   WaitCounter     m_ctuTaskCounter;
196   WaitCounter     m_dmvrTaskCounter;
197   WaitCounter     m_borderExtTaskCounter;
198   Barrier         m_copyWrapBufDone;
199   BlockingBarrier done;
200 #if RECO_WHILE_PARSE
201   std::vector<Barrier> ctuParsedBarrier;
202 #endif
203 #if ALLOW_MIDER_LF_DURING_PICEXT
204   CBarrierVec     refPicExtDepBarriers;
205 #endif
206   Barrier         parseDone;
207 
208   CodingStructure*    cs = nullptr;
209   std::vector<Slice*> slices;
210   unsigned int        numSlices = 1;
211 
212   seiMessages        seiMessageList;
213 
isRefScaledPicture214   bool               isRefScaled( const PPS* pps ) const
215   {
216     const PPS*  pps0     = slices[0]->getPPS();
217     const Size& recoSize = m_bufs[PIC_RECONSTRUCTION].bufs[COMPONENT_Y];
218     return ( recoSize.width  != pps->getPicWidthInLumaSamples()    ||
219              recoSize.height != pps->getPicHeightInLumaSamples() ) ||
220            ( ( pps0->getScalingWindow().getWindowEnabledFlag()   || pps->getScalingWindow().getWindowEnabledFlag() ) && (
221                pps0->getScalingWindow().getWindowLeftOffset()    != pps->getScalingWindow().getWindowLeftOffset()  ||
222                pps0->getScalingWindow().getWindowRightOffset()   != pps->getScalingWindow().getWindowRightOffset() ||
223                pps0->getScalingWindow().getWindowTopOffset()     != pps->getScalingWindow().getWindowTopOffset()   ||
224                pps0->getScalingWindow().getWindowBottomOffset()  != pps->getScalingWindow().getWindowBottomOffset() ) );
225   }
226 
227   std::shared_ptr<PicHeader> picHeader;
228   void                       setPicHead( const std::shared_ptr<PicHeader>& ph );
229 
isWrapAroundEnabledPicture230   bool         isWrapAroundEnabled( const PPS* pps ) const  { return  pps->getUseWrapAround() && !isRefScaled( pps ); }
231 
232   Slice*       allocateNewSlice( Slice** pilot = nullptr );
233   void         clearSliceBuffer();
234 
235 #if TRACE_ENABLE_ITT
236   __itt_domain* m_itt_decLibInst;
237 #endif
238 
239 public:
240   std::vector<uint8_t>  m_ccAlfFilterControl[2];
getccAlfFilterControlPicture241         uint8_t*        getccAlfFilterControl( int compIdx )       { return m_ccAlfFilterControl[compIdx].data(); }
getccAlfFilterControlPicture242   const uint8_t*        getccAlfFilterControl( int compIdx ) const { return m_ccAlfFilterControl[compIdx].data(); }
getccAlfFilterControlPicture243   std::vector<uint8_t>* getccAlfFilterControl()                    { return m_ccAlfFilterControl; }
resizeccAlfFilterControlPicture244   void                  resizeccAlfFilterControl( int numEntries )
245   {
246     for( int compIdx = 0; compIdx < 2; compIdx++ )
247     {
248       m_ccAlfFilterControl[compIdx].resize( numEntries );
249       m_ccAlfFilterControl[compIdx].assign( numEntries, 0 );
250     }
251   }
252 
253   std::vector<uint8_t>  m_alfCtuEnableFlag[MAX_NUM_COMPONENT];
getAlfCtuEnableFlagPicture254   uint8_t*              getAlfCtuEnableFlag( int compIdx ) { return m_alfCtuEnableFlag[compIdx].data(); }
getAlfCtuEnableFlagPicture255   std::vector<uint8_t>* getAlfCtuEnableFlag()              { return m_alfCtuEnableFlag; }
resizeAlfCtuEnableFlagPicture256   void                  resizeAlfCtuEnableFlag( int numEntries )
257   {
258     for( int compIdx = 0; compIdx < MAX_NUM_COMPONENT; compIdx++ )
259     {
260       m_alfCtuEnableFlag[compIdx].resize( numEntries );
261       m_alfCtuEnableFlag[compIdx].assign( numEntries, 0 );
262     }
263   }
264 
265   std::vector<short>  m_alfCtbFilterIndex;
getAlfCtbFilterIndexPicture266         short*        getAlfCtbFilterIndex()       { return m_alfCtbFilterIndex.data(); }
getAlfCtbFilterIndexPicture267   const short*        getAlfCtbFilterIndex() const { return m_alfCtbFilterIndex.data(); }
getAlfCtbFilterIndexVecPicture268   std::vector<short>& getAlfCtbFilterIndexVec() { return m_alfCtbFilterIndex; }
resizeAlfCtbFilterIndexPicture269   void                resizeAlfCtbFilterIndex( int numEntries )
270   {
271     m_alfCtbFilterIndex.resize( numEntries );
272     m_alfCtbFilterIndex.assign( numEntries, 0 );
273   }
274 
275   std::vector<uint8_t>  m_alfCtuAlternative[MAX_NUM_COMPONENT-1];
getAlfCtuAlternativeDataPicture276   uint8_t*              getAlfCtuAlternativeData( int compIdx ) { return m_alfCtuAlternative[compIdx].data(); }
getAlfCtuAlternativePicture277   std::vector<uint8_t>* getAlfCtuAlternative()                 { return m_alfCtuAlternative; }
resizeAlfCtuAlternativePicture278   void resizeAlfCtuAlternative( int numEntries )
279   {
280     for( int compIdx = 0; compIdx < MAX_NUM_COMPONENT-1; compIdx++ )
281     {
282       m_alfCtuAlternative[compIdx].resize( numEntries );
283       m_alfCtuAlternative[compIdx].assign( numEntries, 0 );
284     }
285   }
286 
287 #if  ENABLE_SIMD_OPT_PICTURE && defined( TARGET_SIMD_X86 )
288   void initPictureX86();
289   template <X86_VEXT vext>
290   void _initPictureX86();
291 #endif
292 };
293 
294 int calcAndPrintHashStatus(const CPelUnitBuf& pic, const vvdecSEIDecodedPictureHash* pictureHashSEI, const BitDepths &bitDepths, const MsgLevel msgl);
295 
296 }
297