1 /***************************************************************************
2 qgsvectortileloader.cpp
3 --------------------------------------
4 Date : March 2020
5 Copyright : (C) 2020 by Martin Dobias
6 Email : wonder dot sk at gmail dot com
7 ***************************************************************************
8 * *
9 * This program is free software; you can redistribute it and/or modify *
10 * it under the terms of the GNU General Public License as published by *
11 * the Free Software Foundation; either version 2 of the License, or *
12 * (at your option) any later version. *
13 * *
14 ***************************************************************************/
15
16 #include "qgsvectortileloader.h"
17
18 #include <QEventLoop>
19
20 #include "qgsblockingnetworkrequest.h"
21 #include "qgslogger.h"
22 #include "qgsmbtiles.h"
23 #include "qgsnetworkaccessmanager.h"
24 #include "qgsvectortileutils.h"
25 #include "qgsapplication.h"
26 #include "qgsauthmanager.h"
27 #include "qgsmessagelog.h"
28
29 #include "qgstiledownloadmanager.h"
30
QgsVectorTileLoader(const QString & uri,const QgsTileMatrix & tileMatrix,const QgsTileRange & range,const QPointF & viewCenter,const QString & authid,const QString & referer,QgsFeedback * feedback)31 QgsVectorTileLoader::QgsVectorTileLoader( const QString &uri, const QgsTileMatrix &tileMatrix, const QgsTileRange &range, const QPointF &viewCenter, const QString &authid, const QString &referer, QgsFeedback *feedback )
32 : mEventLoop( new QEventLoop )
33 , mFeedback( feedback )
34 , mAuthCfg( authid )
35 , mReferer( referer )
36 {
37 if ( feedback )
38 {
39 connect( feedback, &QgsFeedback::canceled, this, &QgsVectorTileLoader::canceled, Qt::QueuedConnection );
40
41 // rendering could have been canceled before we started to listen to canceled() signal
42 // so let's check before doing the download and maybe quit prematurely
43 if ( feedback->isCanceled() )
44 return;
45 }
46
47 QgsDebugMsgLevel( QStringLiteral( "Starting network loader" ), 2 );
48 QVector<QgsTileXYZ> tiles = QgsVectorTileUtils::tilesInRange( range, tileMatrix.zoomLevel() );
49 QgsVectorTileUtils::sortTilesByDistanceFromCenter( tiles, viewCenter );
50 for ( QgsTileXYZ id : std::as_const( tiles ) )
51 {
52 loadFromNetworkAsync( id, tileMatrix, uri );
53 }
54 }
55
~QgsVectorTileLoader()56 QgsVectorTileLoader::~QgsVectorTileLoader()
57 {
58 QgsDebugMsgLevel( QStringLiteral( "Terminating network loader" ), 2 );
59
60 if ( !mReplies.isEmpty() )
61 {
62 // this can happen when the loader is terminated without getting requests finalized
63 // (e.g. downloadBlocking() was not called)
64 canceled();
65 }
66 }
67
downloadBlocking()68 void QgsVectorTileLoader::downloadBlocking()
69 {
70 if ( mFeedback && mFeedback->isCanceled() )
71 {
72 QgsDebugMsgLevel( QStringLiteral( "downloadBlocking - not staring event loop - canceled" ), 2 );
73 return; // nothing to do
74 }
75
76 QgsDebugMsgLevel( QStringLiteral( "Starting event loop with %1 requests" ).arg( mReplies.count() ), 2 );
77
78 mEventLoop->exec( QEventLoop::ExcludeUserInputEvents );
79
80 QgsDebugMsgLevel( QStringLiteral( "downloadBlocking finished" ), 2 );
81
82 Q_ASSERT( mReplies.isEmpty() );
83 }
84
loadFromNetworkAsync(const QgsTileXYZ & id,const QgsTileMatrix & tileMatrix,const QString & requestUrl)85 void QgsVectorTileLoader::loadFromNetworkAsync( const QgsTileXYZ &id, const QgsTileMatrix &tileMatrix, const QString &requestUrl )
86 {
87 QString url = QgsVectorTileUtils::formatXYZUrlTemplate( requestUrl, id, tileMatrix );
88 QNetworkRequest request( url );
89 QgsSetRequestInitiatorClass( request, QStringLiteral( "QgsVectorTileLoader" ) );
90 QgsSetRequestInitiatorId( request, id.toString() );
91
92 request.setAttribute( static_cast<QNetworkRequest::Attribute>( QNetworkRequest::User + 1 ), id.column() );
93 request.setAttribute( static_cast<QNetworkRequest::Attribute>( QNetworkRequest::User + 2 ), id.row() );
94 request.setAttribute( static_cast<QNetworkRequest::Attribute>( QNetworkRequest::User + 3 ), id.zoomLevel() );
95
96 request.setAttribute( QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::PreferCache );
97 request.setAttribute( QNetworkRequest::CacheSaveControlAttribute, true );
98
99 if ( !mReferer.isEmpty() )
100 request.setRawHeader( "Referer", mReferer.toUtf8() );
101
102 if ( !mAuthCfg.isEmpty() && !QgsApplication::authManager()->updateNetworkRequest( request, mAuthCfg ) )
103 {
104 QgsMessageLog::logMessage( tr( "network request update failed for authentication config" ), tr( "Network" ) );
105 }
106
107 QgsTileDownloadManagerReply *reply = QgsApplication::tileDownloadManager()->get( request );
108 connect( reply, &QgsTileDownloadManagerReply::finished, this, &QgsVectorTileLoader::tileReplyFinished );
109 mReplies << reply;
110 }
111
tileReplyFinished()112 void QgsVectorTileLoader::tileReplyFinished()
113 {
114 QgsTileDownloadManagerReply *reply = qobject_cast<QgsTileDownloadManagerReply *>( sender() );
115
116 int reqX = reply->request().attribute( static_cast<QNetworkRequest::Attribute>( QNetworkRequest::User + 1 ) ).toInt();
117 int reqY = reply->request().attribute( static_cast<QNetworkRequest::Attribute>( QNetworkRequest::User + 2 ) ).toInt();
118 int reqZ = reply->request().attribute( static_cast<QNetworkRequest::Attribute>( QNetworkRequest::User + 3 ) ).toInt();
119 QgsTileXYZ tileID( reqX, reqY, reqZ );
120
121 if ( reply->error() == QNetworkReply::NoError )
122 {
123 // TODO: handle redirections?
124
125 QgsDebugMsgLevel( QStringLiteral( "Tile download successful: " ) + tileID.toString(), 2 );
126 QByteArray rawData = reply->data();
127 mReplies.removeOne( reply );
128 reply->deleteLater();
129
130 emit tileRequestFinished( QgsVectorTileRawData( tileID, rawData ) );
131 }
132 else
133 {
134 QgsDebugMsg( QStringLiteral( "Tile download failed! " ) + reply->errorString() );
135 mReplies.removeOne( reply );
136 reply->deleteLater();
137
138 emit tileRequestFinished( QgsVectorTileRawData( tileID, QByteArray() ) );
139 }
140
141 if ( mReplies.isEmpty() )
142 {
143 // exist the event loop
144 QMetaObject::invokeMethod( mEventLoop.get(), "quit", Qt::QueuedConnection );
145 }
146 }
147
canceled()148 void QgsVectorTileLoader::canceled()
149 {
150 QgsDebugMsgLevel( QStringLiteral( "Canceling %1 pending requests" ).arg( mReplies.count() ), 2 );
151 qDeleteAll( mReplies );
152 mReplies.clear();
153
154 // stop blocking download
155 mEventLoop->quit();
156
157 }
158
159 //////
160
blockingFetchTileRawData(const QString & sourceType,const QString & sourcePath,const QgsTileMatrix & tileMatrix,const QPointF & viewCenter,const QgsTileRange & range,const QString & authid,const QString & referer)161 QList<QgsVectorTileRawData> QgsVectorTileLoader::blockingFetchTileRawData( const QString &sourceType, const QString &sourcePath, const QgsTileMatrix &tileMatrix, const QPointF &viewCenter, const QgsTileRange &range, const QString &authid, const QString &referer )
162 {
163 QList<QgsVectorTileRawData> rawTiles;
164
165 QgsMbTiles mbReader( sourcePath );
166 bool isUrl = ( sourceType == QLatin1String( "xyz" ) );
167 if ( !isUrl )
168 {
169 bool res = mbReader.open();
170 Q_UNUSED( res );
171 Q_ASSERT( res );
172 }
173
174 QVector<QgsTileXYZ> tiles = QgsVectorTileUtils::tilesInRange( range, tileMatrix.zoomLevel() );
175 QgsVectorTileUtils::sortTilesByDistanceFromCenter( tiles, viewCenter );
176 for ( QgsTileXYZ id : std::as_const( tiles ) )
177 {
178 QByteArray rawData = isUrl ? loadFromNetwork( id, tileMatrix, sourcePath, authid, referer ) : loadFromMBTiles( id, mbReader );
179 if ( !rawData.isEmpty() )
180 {
181 rawTiles.append( QgsVectorTileRawData( id, rawData ) );
182 }
183 }
184 return rawTiles;
185 }
186
loadFromNetwork(const QgsTileXYZ & id,const QgsTileMatrix & tileMatrix,const QString & requestUrl,const QString & authid,const QString & referer)187 QByteArray QgsVectorTileLoader::loadFromNetwork( const QgsTileXYZ &id, const QgsTileMatrix &tileMatrix, const QString &requestUrl, const QString &authid, const QString &referer )
188 {
189 QString url = QgsVectorTileUtils::formatXYZUrlTemplate( requestUrl, id, tileMatrix );
190 QNetworkRequest nr;
191 nr.setUrl( QUrl( url ) );
192
193 if ( !referer.isEmpty() )
194 nr.setRawHeader( "Referer", referer.toUtf8() );
195
196 QgsBlockingNetworkRequest req;
197 req.setAuthCfg( authid );
198 QgsDebugMsgLevel( QStringLiteral( "Blocking request: " ) + url, 2 );
199 QgsBlockingNetworkRequest::ErrorCode errCode = req.get( nr );
200 if ( errCode != QgsBlockingNetworkRequest::NoError )
201 {
202 QgsDebugMsg( QStringLiteral( "Request failed: " ) + url );
203 return QByteArray();
204 }
205 QgsNetworkReplyContent reply = req.reply();
206 QgsDebugMsgLevel( QStringLiteral( "Request successful, content size %1" ).arg( reply.content().size() ), 2 );
207 return reply.content();
208 }
209
210
loadFromMBTiles(const QgsTileXYZ & id,QgsMbTiles & mbTileReader)211 QByteArray QgsVectorTileLoader::loadFromMBTiles( const QgsTileXYZ &id, QgsMbTiles &mbTileReader )
212 {
213 // MBTiles uses TMS specs with Y starting at the bottom while XYZ uses Y starting at the top
214 int rowTMS = pow( 2, id.zoomLevel() ) - id.row() - 1;
215 QByteArray gzippedTileData = mbTileReader.tileData( id.zoomLevel(), id.column(), rowTMS );
216 if ( gzippedTileData.isEmpty() )
217 {
218 QgsDebugMsg( QStringLiteral( "Failed to get tile " ) + id.toString() );
219 return QByteArray();
220 }
221
222 QByteArray data;
223 if ( !QgsMbTiles::decodeGzip( gzippedTileData, data ) )
224 {
225 QgsDebugMsg( QStringLiteral( "Failed to decompress tile " ) + id.toString() );
226 return QByteArray();
227 }
228
229 QgsDebugMsgLevel( QStringLiteral( "Tile blob size %1 -> uncompressed size %2" ).arg( gzippedTileData.size() ).arg( data.size() ), 2 );
230 return data;
231 }
232