1 /***************************************************************************
2   qgsvectortilewriter.cpp
3   --------------------------------------
4   Date                 : April 2020
5   Copyright            : (C) 2020 by Martin Dobias
6   Email                : wonder dot sk at gmail dot com
7  ***************************************************************************
8  *                                                                         *
9  *   This program is free software; you can redistribute it and/or modify  *
10  *   it under the terms of the GNU General Public License as published by  *
11  *   the Free Software Foundation; either version 2 of the License, or     *
12  *   (at your option) any later version.                                   *
13  *                                                                         *
14  ***************************************************************************/
15 
16 #include "qgsvectortilewriter.h"
17 
18 #include "qgsdatasourceuri.h"
19 #include "qgsfeedback.h"
20 #include "qgsjsonutils.h"
21 #include "qgslogger.h"
22 #include "qgsmbtiles.h"
23 #include "qgstiles.h"
24 #include "qgsvectorlayer.h"
25 #include "qgsvectortilemvtencoder.h"
26 #include "qgsvectortileutils.h"
27 
28 #include <nlohmann/json.hpp>
29 
30 #include <QDir>
31 #include <QFile>
32 #include <QFileInfo>
33 #include <QUrl>
34 
35 
QgsVectorTileWriter()36 QgsVectorTileWriter::QgsVectorTileWriter()
37 {
38 }
39 
40 
writeTiles(QgsFeedback * feedback)41 bool QgsVectorTileWriter::writeTiles( QgsFeedback *feedback )
42 {
43   if ( mMinZoom < 0 )
44   {
45     mErrorMessage = tr( "Invalid min. zoom level" );
46     return false;
47   }
48   if ( mMaxZoom > 24 )
49   {
50     mErrorMessage = tr( "Invalid max. zoom level" );
51     return false;
52   }
53 
54   std::unique_ptr<QgsMbTiles> mbtiles;
55 
56   QgsDataSourceUri dsUri;
57   dsUri.setEncodedUri( mDestinationUri );
58 
59   QString sourceType = dsUri.param( QStringLiteral( "type" ) );
60   QString sourcePath = dsUri.param( QStringLiteral( "url" ) );
61   if ( sourceType == QLatin1String( "xyz" ) )
62   {
63     // remove the initial file:// scheme
64     sourcePath = QUrl( sourcePath ).toLocalFile();
65 
66     if ( !QgsVectorTileUtils::checkXYZUrlTemplate( sourcePath ) )
67     {
68       mErrorMessage = tr( "Invalid template for XYZ: " ) + sourcePath;
69       return false;
70     }
71   }
72   else if ( sourceType == QLatin1String( "mbtiles" ) )
73   {
74     mbtiles.reset( new QgsMbTiles( sourcePath ) );
75   }
76   else
77   {
78     mErrorMessage = tr( "Unsupported source type for writing: " ) + sourceType;
79     return false;
80   }
81 
82   QgsRectangle outputExtent = mExtent;
83   if ( outputExtent.isEmpty() )
84   {
85     outputExtent = fullExtent();
86     if ( outputExtent.isEmpty() )
87     {
88       mErrorMessage = tr( "Failed to calculate output extent" );
89       return false;
90     }
91   }
92 
93   // figure out how many tiles we will need to do
94   int tilesToCreate = 0;
95   for ( int zoomLevel = mMinZoom; zoomLevel <= mMaxZoom; ++zoomLevel )
96   {
97     QgsTileMatrix tileMatrix = QgsTileMatrix::fromWebMercator( zoomLevel );
98 
99     QgsTileRange tileRange = tileMatrix.tileRangeFromExtent( outputExtent );
100     tilesToCreate += ( tileRange.endRow() - tileRange.startRow() + 1 ) *
101                      ( tileRange.endColumn() - tileRange.startColumn() + 1 );
102   }
103 
104   if ( tilesToCreate == 0 )
105   {
106     mErrorMessage = tr( "No tiles to generate" );
107     return false;
108   }
109 
110   if ( mbtiles )
111   {
112     if ( !mbtiles->create() )
113     {
114       mErrorMessage = tr( "Failed to create MBTiles file: " ) + sourcePath;
115       return false;
116     }
117 
118     // required metadata
119     mbtiles->setMetadataValue( "format", "pbf" );
120     mbtiles->setMetadataValue( "json", mbtilesJsonSchema() );
121 
122     // metadata specified by the client
123     const QStringList metaKeys = mMetadata.keys();
124     for ( const QString &key : metaKeys )
125     {
126       mbtiles->setMetadataValue( key, mMetadata[key].toString() );
127     }
128 
129     // default metadata that we always write (if not written by the client)
130     if ( !mMetadata.contains( "name" ) )
131       mbtiles->setMetadataValue( "name",  "unnamed" );  // required by the spec
132     if ( !mMetadata.contains( "minzoom" ) )
133       mbtiles->setMetadataValue( "minzoom", QString::number( mMinZoom ) );
134     if ( !mMetadata.contains( "maxzoom" ) )
135       mbtiles->setMetadataValue( "maxzoom", QString::number( mMaxZoom ) );
136     if ( !mMetadata.contains( "bounds" ) )
137     {
138       try
139       {
140         QgsCoordinateTransform ct( QgsCoordinateReferenceSystem( "EPSG:3857" ), QgsCoordinateReferenceSystem( "EPSG:4326" ), mTransformContext );
141         QgsRectangle wgsExtent = ct.transform( outputExtent );
142         QString boundsStr = QString( "%1,%2,%3,%4" )
143                             .arg( wgsExtent.xMinimum() ).arg( wgsExtent.yMinimum() )
144                             .arg( wgsExtent.xMaximum() ).arg( wgsExtent.yMaximum() );
145         mbtiles->setMetadataValue( "bounds", boundsStr );
146       }
147       catch ( const QgsCsException & )
148       {
149         // bounds won't be written (not a problem - it is an optional value)
150       }
151     }
152   }
153 
154   int tilesCreated = 0;
155   for ( int zoomLevel = mMinZoom; zoomLevel <= mMaxZoom; ++zoomLevel )
156   {
157     QgsTileMatrix tileMatrix = QgsTileMatrix::fromWebMercator( zoomLevel );
158 
159     QgsTileRange tileRange = tileMatrix.tileRangeFromExtent( outputExtent );
160     for ( int row = tileRange.startRow(); row <= tileRange.endRow(); ++row )
161     {
162       for ( int col = tileRange.startColumn(); col <= tileRange.endColumn(); ++col )
163       {
164         QgsTileXYZ tileID( col, row, zoomLevel );
165         QgsVectorTileMVTEncoder encoder( tileID );
166         encoder.setTransformContext( mTransformContext );
167 
168         for ( const Layer &layer : qgis::as_const( mLayers ) )
169         {
170           if ( ( layer.minZoom() >= 0 && zoomLevel < layer.minZoom() ) ||
171                ( layer.maxZoom() >= 0 && zoomLevel > layer.maxZoom() ) )
172             continue;
173 
174           encoder.addLayer( layer.layer(), feedback, layer.filterExpression(), layer.layerName() );
175         }
176 
177         if ( feedback && feedback->isCanceled() )
178         {
179           mErrorMessage = tr( "Operation has been canceled" );
180           return false;
181         }
182 
183         QByteArray tileData = encoder.encode();
184 
185         ++tilesCreated;
186         if ( feedback )
187         {
188           feedback->setProgress( static_cast<double>( tilesCreated ) / tilesToCreate * 100 );
189         }
190 
191         if ( tileData.isEmpty() )
192         {
193           // skipping empty tile - no need to write it
194           continue;
195         }
196 
197         if ( sourceType == QLatin1String( "xyz" ) )
198         {
199           if ( !writeTileFileXYZ( sourcePath, tileID, tileMatrix, tileData ) )
200             return false;  // error message already set
201         }
202         else  // mbtiles
203         {
204           QByteArray gzipTileData;
205           QgsMbTiles::encodeGzip( tileData, gzipTileData );
206           int rowTMS = pow( 2, tileID.zoomLevel() ) - tileID.row() - 1;
207           mbtiles->setTileData( tileID.zoomLevel(), tileID.column(), rowTMS, gzipTileData );
208         }
209       }
210     }
211   }
212 
213   return true;
214 }
215 
fullExtent() const216 QgsRectangle QgsVectorTileWriter::fullExtent() const
217 {
218   QgsRectangle extent;
219   QgsCoordinateReferenceSystem destCrs( "EPSG:3857" );
220 
221   for ( const Layer &layer : mLayers )
222   {
223     QgsVectorLayer *vl = layer.layer();
224     QgsCoordinateTransform ct( vl->crs(), destCrs, mTransformContext );
225     try
226     {
227       QgsRectangle r = ct.transformBoundingBox( vl->extent() );
228       extent.combineExtentWith( r );
229     }
230     catch ( const QgsCsException & )
231     {
232       QgsDebugMsg( "Failed to reproject layer extent to destination CRS" );
233     }
234   }
235   return extent;
236 }
237 
writeTileFileXYZ(const QString & sourcePath,QgsTileXYZ tileID,const QgsTileMatrix & tileMatrix,const QByteArray & tileData)238 bool QgsVectorTileWriter::writeTileFileXYZ( const QString &sourcePath, QgsTileXYZ tileID, const QgsTileMatrix &tileMatrix, const QByteArray &tileData )
239 {
240   QString filePath = QgsVectorTileUtils::formatXYZUrlTemplate( sourcePath, tileID, tileMatrix );
241 
242   // make dirs if needed
243   QFileInfo fi( filePath );
244   QDir fileDir = fi.dir();
245   if ( !fileDir.exists() )
246   {
247     if ( !fileDir.mkpath( "." ) )
248     {
249       mErrorMessage = tr( "Cannot create directory " ) + fileDir.path();
250       return false;
251     }
252   }
253 
254   QFile f( filePath );
255   if ( !f.open( QIODevice::WriteOnly ) )
256   {
257     mErrorMessage = tr( "Cannot open file for writing " ) + filePath;
258     return false;
259   }
260 
261   f.write( tileData );
262   f.close();
263   return true;
264 }
265 
266 
mbtilesJsonSchema()267 QString QgsVectorTileWriter::mbtilesJsonSchema()
268 {
269   QVariantList arrayLayers;
270   for ( const Layer &layer : qgis::as_const( mLayers ) )
271   {
272     QgsVectorLayer *vl = layer.layer();
273     const QgsFields fields = vl->fields();
274 
275     QVariantMap fieldsObj;
276     for ( const QgsField &field : fields )
277     {
278       QString fieldTypeStr;
279       if ( field.type() == QVariant::Bool )
280         fieldTypeStr = QStringLiteral( "Boolean" );
281       else if ( field.type() == QVariant::Int || field.type() == QVariant::Double )
282         fieldTypeStr = QStringLiteral( "Number" );
283       else
284         fieldTypeStr = QStringLiteral( "String" );
285 
286       fieldsObj[field.name()] = fieldTypeStr;
287     }
288 
289     QVariantMap layerObj;
290     layerObj["id"] = vl->name();
291     layerObj["fields"] = fieldsObj;
292     arrayLayers.append( layerObj );
293   }
294 
295   QVariantMap rootObj;
296   rootObj["vector_layers"] = arrayLayers;
297   return QString::fromStdString( QgsJsonUtils::jsonFromVariant( rootObj ).dump() );
298 }
299