1 /******************************************************************************
2  *
3  * Project:  FlatGeobuf driver
4  * Purpose:  Declaration of classes for OGR FlatGeobuf driver.
5  * Author:   Björn Harrtell <bjorn at wololo dot org>
6  *
7  ******************************************************************************
8  * Copyright (c) 2018-2020, Björn Harrtell <bjorn at wololo dot org>
9  *
10  * Permission is hereby granted, free of charge, to any person obtaining a
11  * copy of this software and associated documentation files (the "Software"),
12  * to deal in the Software without restriction, including without limitation
13  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
14  * and/or sell copies of the Software, and to permit persons to whom the
15  * Software is furnished to do so, subject to the following conditions:
16  *
17  * The above copyright notice and this permission notice shall be included
18  * in all copies or substantial portions of the Software.
19  *
20  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
21  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
25  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
26  * DEALINGS IN THE SOFTWARE.
27  ****************************************************************************/
28 
29 #ifndef OGR_FLATGEOBUF_H_INCLUDED
30 #define OGR_FLATGEOBUF_H_INCLUDED
31 
32 #include "ogrsf_frmts.h"
33 #include "ogr_p.h"
34 #include "ogreditablelayer.h"
35 
36 #include "header_generated.h"
37 #include "feature_generated.h"
38 #include "packedrtree.h"
39 
40 #include <limits>
41 
42 class OGRFlatGeobufDataset;
43 
44 static constexpr uint8_t magicbytes[8] = { 0x66, 0x67, 0x62, 0x03, 0x66, 0x67, 0x62, 0x00 };
45 
46 static constexpr uint32_t header_max_buffer_size = 1048576 * 10;
47 
48 // Cannot be larger than that, due to a x2 logic done in ensureFeatureBuf()
49 static constexpr uint32_t feature_max_buffer_size = static_cast<uint32_t>(std::numeric_limits<int32_t>::max());
50 
51 // holds feature meta needed to build spatial index
52 struct FeatureItem : FlatGeobuf::Item {
53     uint32_t size;
54     uint64_t offset;
55 };
56 
57 class OGRFlatGeobufBaseLayerInterface CPL_NON_FINAL
58 {
59     public:
60         virtual ~OGRFlatGeobufBaseLayerInterface();
61 
62         virtual const std::string& GetFilename() const = 0;
63         virtual OGRLayer* GetLayer() = 0;
64 };
65 
66 class OGRFlatGeobufLayer final : public OGRLayer, public OGRFlatGeobufBaseLayerInterface
67 {
68     private:
69         std::string m_osFilename;
70         std::string m_osLayerName;
71 
72         VSILFILE *m_poFp = nullptr;
73         vsi_l_offset m_nFileSize = 0;
74 
75         const FlatGeobuf::Header *m_poHeader = nullptr;
76         GByte *m_headerBuf = nullptr;
77         OGRwkbGeometryType m_eGType;
78         FlatGeobuf::GeometryType m_geometryType;
79         bool m_hasM = false;
80         bool m_hasZ = false;
81         bool m_hasT = false;
82         bool m_hasTM = false;
83         uint64_t m_featuresCount = 0;
84         OGREnvelope m_sExtent;
85         OGRFeatureDefn *m_poFeatureDefn = nullptr;
86         OGRSpatialReference *m_poSRS = nullptr;
87 
88         // iteration
89         size_t m_featuresPos = 0; // current iteration position
90         uint64_t m_offset = 0; // current read offset
91         uint64_t m_offsetFeatures = 0; // offset of feature data
92         std::vector<FlatGeobuf::SearchResultItem> m_foundItems; // found node items in spatial index search
93         bool m_queriedSpatialIndex = false;
94         bool m_ignoreSpatialFilter = false;
95         bool m_ignoreAttributeFilter = false;
96 
97         // creation
98         bool m_create = false;
99         bool m_update = false;
100         std::vector<std::shared_ptr<FlatGeobuf::Item>> m_featureItems; // feature item description used to create spatial index
101         bool m_bCreateSpatialIndexAtClose = true;
102         bool m_bVerifyBuffers = true;
103         bool m_bCanCreate = true;
104         VSILFILE *m_poFpWrite = nullptr;
105         uint64_t m_writeOffset = 0; // current write offset
106         uint16_t m_indexNodeSize = 0;
107         std::string m_osTempFile; // holds generated temp file name for two pass writing
108         uint32_t m_maxFeatureSize  = 0;
109 
110         // shared
111         GByte *m_featureBuf = nullptr; // reusable/resizable feature data buffer
112         uint32_t m_featureBufSize = 0; // current feature buffer size
113 
114         // deserialize
115         void ensurePadfBuffers(size_t count);
116         OGRErr ensureFeatureBuf(uint32_t featureSize);
117         OGRErr parseFeature(OGRFeature *poFeature);
118         const std::vector<flatbuffers::Offset<FlatGeobuf::Column>> writeColumns(flatbuffers::FlatBufferBuilder &fbb);
119         void readColumns();
120         OGRErr readIndex();
121         OGRErr readFeatureOffset(uint64_t index, uint64_t &featureOffset);
122 
123         // serialize
124         void Create();
125         void writeHeader(VSILFILE *poFp, uint64_t featuresCount, std::vector<double> *extentVector);
126 
127         // construction
128         OGRFlatGeobufLayer(const FlatGeobuf::Header *, GByte *headerBuf, const char *pszFilename, VSILFILE *poFp, uint64_t offset, bool update);
129         OGRFlatGeobufLayer(const char *pszLayerName, const char *pszFilename, OGRSpatialReference *poSpatialRef, OGRwkbGeometryType eGType, bool bCreateSpatialIndexAtClose, VSILFILE *poFpWrite, std::string &osTempFile);
130 
131     public:
132         virtual ~OGRFlatGeobufLayer();
133 
134         static OGRFlatGeobufLayer *Open(const FlatGeobuf::Header *, GByte *headerBuf, const char *pszFilename, VSILFILE *poFp, uint64_t offset, bool update);
135         static OGRFlatGeobufLayer *Open(const char* pszFilename, VSILFILE *fp, bool bVerifyBuffers, bool update);
136         static OGRFlatGeobufLayer *Create(const char *pszLayerName, const char *pszFilename, OGRSpatialReference *poSpatialRef, OGRwkbGeometryType eGType, bool bCreateSpatialIndexAtClose, char **papszOptions);
137 
138         virtual OGRFeature *GetFeature(GIntBig nFeatureId) override;
139         virtual OGRFeature *GetNextFeature() override;
140         virtual OGRErr CreateField(OGRFieldDefn *poField, int bApproxOK = true) override;
141         virtual OGRErr ICreateFeature(OGRFeature *poFeature) override;
142         virtual int TestCapability(const char *) override;
143 
144         virtual void ResetReading() override;
GetLayerDefn()145         virtual OGRFeatureDefn *GetLayerDefn() override { return m_poFeatureDefn; }
146         virtual GIntBig GetFeatureCount(int bForce) override;
147         virtual OGRErr GetExtent(OGREnvelope* psExtent, int bForce) override;
GetExtent(int iGeomField,OGREnvelope * psExtent,int bForce)148         virtual OGRErr GetExtent( int iGeomField, OGREnvelope *psExtent,
149                                        int bForce ) override
150                 { return OGRLayer::GetExtent(iGeomField, psExtent, bForce); }
151 
VerifyBuffers(int bFlag)152         void VerifyBuffers( int bFlag ) { m_bVerifyBuffers = CPL_TO_BOOL(bFlag); }
153 
GetFilename()154         const std::string& GetFilename() const override { return m_osFilename; }
GetLayer()155         OGRLayer* GetLayer() override { return this; }
156 
157         static std::string GetTempFilePath(const CPLString &fileName, CSLConstList papszOptions);
158         static VSILFILE *CreateOutputFile(const CPLString &pszFilename, CSLConstList papszOptions, bool isTemp);
159 
GetIndexNodeSize()160         uint16_t GetIndexNodeSize() const { return m_indexNodeSize; }
161         OGRwkbGeometryType getOGRwkbGeometryType();
162 };
163 
164 class OGRFlatGeobufEditableLayer final: public OGREditableLayer, public OGRFlatGeobufBaseLayerInterface
165 {
166     public:
167         OGRFlatGeobufEditableLayer(OGRFlatGeobufLayer *poFlatGeobufLayer, char **papszOpenOptions);
168 
169         virtual GIntBig     GetFeatureCount( int bForce = TRUE ) override;
170 
GetFilename()171         const std::string&  GetFilename() const override {
172             return static_cast<OGRFlatGeobufLayer *>(m_poDecoratedLayer)->GetFilename();
173         }
GetLayer()174         OGRLayer* GetLayer() override { return this; }
175 };
176 
177 class OGRFlatGeobufDataset final: public GDALDataset
178 {
179     private:
180         std::vector<std::unique_ptr<OGRFlatGeobufBaseLayerInterface>> m_apoLayers;
181         bool m_bCreate = false;
182         bool m_bUpdate = false;
183         bool m_bIsDir = false;
184 
185         bool OpenFile(const char* pszFilename, VSILFILE* fp, bool bVerifyBuffers);
186 
187     public:
188         OGRFlatGeobufDataset(const char *pszName, bool bIsDir, bool bCreate, bool bUpdate);
189         ~OGRFlatGeobufDataset();
190 
191         static GDALDataset *Open(GDALOpenInfo*);
192         static GDALDataset *Create( const char *pszName,
193                                         CPL_UNUSED int nBands,
194                                         CPL_UNUSED int nXSize,
195                                         CPL_UNUSED int nYSize,
196                                         CPL_UNUSED GDALDataType eDT,
197                                         char **papszOptions );
198         virtual OGRLayer *GetLayer( int ) override;
199         int TestCapability( const char *pszCap ) override;
200         virtual OGRLayer *ICreateLayer( const char *pszName,
201                                      OGRSpatialReference *poSpatialRef = nullptr,
202                                      OGRwkbGeometryType eGType = wkbUnknown,
203                                      char **papszOptions = nullptr ) override;
204 
GetLayerCount()205         virtual int GetLayerCount() override { return static_cast<int>(m_apoLayers.size()); }
206         char** GetFileList() override;
207 };
208 
209 #endif /* ndef OGR_FLATGEOBUF_H_INCLUDED */
210 
211