1 // SPDX-License-Identifier: LGPL-2.1-or-later
2 //
3 // SPDX-FileCopyrightText: 2011 Dennis Nienhüser <nienhueser@kde.org>
4 //
5
6 #include "job.h"
7 #include "logger.h"
8 #include "upload.h"
9
10 #include <QDebug>
11 #include <QDateTime>
12 #include <QProcess>
13
Job(const Region & region,const JobParameters & parameters,QObject * parent)14 Job::Job(const Region ®ion, const JobParameters ¶meters, QObject *parent) :
15 QObject(parent), m_status(Waiting), m_region(region), m_parameters(parameters)
16 {
17 // nothing to do
18 }
19
status() const20 Job::Status Job::status() const
21 {
22 return m_status;
23 }
24
statusMessage() const25 QString Job::statusMessage() const
26 {
27 return m_statusMessage;
28 }
29
region() const30 Region Job::region() const
31 {
32 return m_region;
33 }
34
setTransport(const QString & transport)35 void Job::setTransport(const QString &transport)
36 {
37 m_transport = transport;
38 }
39
transport() const40 QString Job::transport() const
41 {
42 return m_transport;
43 }
44
setProfile(const QString & profile)45 void Job::setProfile(const QString &profile)
46 {
47 m_profile = profile;
48 }
49
setMonavSettings(const QString & filename)50 void Job::setMonavSettings(const QString &filename)
51 {
52 m_monavSettings = filename;
53 }
54
operator ==(const Job & other) const55 bool Job::operator ==(const Job &other) const
56 {
57 return m_transport == other.m_transport && m_region == other.m_region;
58 }
59
run()60 void Job::run()
61 {
62 if (download() && monav() && search() && package() && upload()) {
63 // Nothing to do.
64 }
65
66 cleanup();
67 emit finished(this);
68 }
69
changeStatus(Job::Status status,const QString & message)70 void Job::changeStatus(Job::Status status, const QString &message)
71 {
72 QString statusType;
73 switch (status) {
74 case Waiting: statusType = "waiting"; break;
75 case Downloading: statusType = "downloading"; break;
76 case Routing: statusType = "routing"; break;
77 case Search: statusType = "search"; break;
78 case Packaging: statusType = "packaging"; break;
79 case Uploading: statusType = "uploading"; break;
80 case Finished: statusType = "finished"; break;
81 case Error: statusType = "error"; break;
82 }
83
84 Logger::instance().setStatus(m_region.id() + QLatin1Char('_') + m_transport,
85 m_region.name() + QLatin1String(" (") + m_transport + QLatin1Char(')'), statusType, message);
86 m_statusMessage = message;
87 m_status = status;
88 }
89
download()90 bool Job::download()
91 {
92 changeStatus(Downloading, "Downloading data.");
93 qDebug() << "Saving file to " << osmFile().absoluteFilePath();
94 if (osmFile().exists()) {
95 QDateTime now = QDateTime::currentDateTime();
96 if (osmFile().lastModified().daysTo(now) > 7) {
97 qDebug() << "Old file is outdated, re-downloading " << osmFile().absoluteFilePath();
98 QFile::remove(osmFile().absoluteFilePath());
99 } else {
100 qDebug() << "Old file is still ok, reusing" << osmFile().absoluteFilePath();
101 return true;
102 }
103 }
104
105 QProcess wget;
106 QStringList arguments;
107 QString url = m_region.pbfFile();
108 arguments << "-O" << osmFile().absoluteFilePath() << url;
109 qDebug() << "Downloading " << url;
110 wget.start("wget", arguments);
111 wget.waitForFinished(1000 * 60 * 60 * 12); // wait up to 12 hours for download to complete
112 if (wget.exitStatus() == QProcess::NormalExit && wget.exitCode() == 0) {
113 return true;
114 } else {
115 qDebug() << "Failed to download " << url;
116 QFile::remove(osmFile().absoluteFilePath());
117 changeStatus(Error, QLatin1String("Error downloading .osm.pbf file: ") + wget.readAllStandardError());
118 return false;
119 }
120 }
121
122 //bool Job::marble()
123 //{
124 // changeStatus(Routing, "Extracting bounding box.");
125 // QStringList arguments;
126 // arguments << "--name" << m_region.name();
127 // arguments << "--version" << "0.2";
128 // arguments << "--date" << QDateTime::currentDateTime().toString("yyyy/dd/MM");
129 // arguments << "--transport" << m_transport;
130 // arguments << "--payload" << targetFile().fileName();
131 // arguments << m_parameters.base().absoluteFilePath("poly/" + m_region.polyFile());
132 // arguments << monavDir().absoluteFilePath() + QLatin1String("/marble.kml");
133 // QProcess poly2kml;
134 // poly2kml.start("poly2kml", arguments);
135 // poly2kml.waitForFinished(1000 * 60 * 30); // wait up to half an hour for poly2kml to convert the data
136 // if (poly2kml.exitStatus() == QProcess::NormalExit && poly2kml.exitCode() == 0) {
137 // qDebug() << "Processed kml file for marble";
138 // return true;
139 // } else {
140 // qDebug() << "poly2kml exiting with status " << poly2kml.exitCode();
141 // changeStatus(Error, "Error creating marble.kml: " + poly2kml.readAllStandardError());
142 // return false;
143 // }
144 //}
145
monav()146 bool Job::monav()
147 {
148 QString const status = QString("Generating offline routing map from %1 (%2).").arg(osmFile().fileName()).arg(Region::fileSize(osmFile()));
149 changeStatus(Routing, status);
150 QStringList arguments;
151 arguments << QLatin1String("-s=") + m_monavSettings;
152 arguments << QLatin1String("-i=") + osmFile().absoluteFilePath();
153 arguments << QLatin1String("-o=") + monavDir().absoluteFilePath();
154 arguments << "-pi=OpenStreetMap Importer" << "-pro=Contraction Hierarchies";
155 arguments << "-pg=GPS Grid" << "-di";
156 arguments << QLatin1String("-dro=") + m_transport;
157 arguments << QLatin1String("--profile=") + m_profile;
158 arguments << "-dd" /*<< "-dc"*/;
159 QProcess monav;
160 monav.start("monav-preprocessor", arguments);
161 monav.waitForFinished(1000 * 60 * 60 * 6); // wait up to 6 hours for monav to convert the data
162 if (monav.exitStatus() == QProcess::NormalExit && monav.exitCode() == 0) {
163 qDebug() << "Processed osm file for monav";
164 } else {
165 qDebug() << "monav exiting with status " << monav.exitCode();
166 changeStatus(Error, QLatin1String("Routing map conversion failed: ") + monav.readAllStandardError());
167 return false;
168 }
169
170 QFile pluginsFile(monavDir().absoluteFilePath() + QLatin1String("/plugins.ini"));
171 pluginsFile.open(QFile::WriteOnly | QFile::Truncate);
172 QTextStream pluginsStream(&pluginsFile);
173 pluginsStream << "[General]\nrouter=Contraction Hierarchies\nrenderer=Mapnik Renderer\ngpsLookup=GPS Grid\naddressLookup=Unicode Tournament Trie\n";
174 pluginsFile.close();
175
176 QFileInfo subdir = QFileInfo(monavDir().absoluteFilePath() + QLatin1String("/routing_") + m_transport.toLower());
177 if (subdir.exists() && subdir.isDir()) {
178 QFileInfoList files = QDir(subdir.absoluteFilePath()).entryInfoList(QDir::Files);
179 for(const QFileInfo &file: files) {
180 if (!QFile::rename(file.absoluteFilePath(), monavDir().absoluteFilePath() + QLatin1Char('/') + file.fileName())) {
181 changeStatus(Error, "Unable to move monav files to target directory.");
182 return false;
183 }
184 }
185 QDir("/").rmdir(subdir.absoluteFilePath());
186 } else {
187 changeStatus(Error, "Unable to find files created by monav");
188 return false;
189 }
190
191 return true;
192 }
193
search()194 bool Job::search()
195 {
196 QString const status = QString("Generating offline search database from %1 (%2).").arg(osmFile().fileName()).arg(Region::fileSize(osmFile()));
197 changeStatus(Search, status);
198 QStringList arguments;
199 arguments << "--name" << m_region.name();
200 arguments << "--version" << "0.3";
201 arguments << "--date" << QDateTime::currentDateTime().toString("MM/dd/yy");
202 arguments << "--transport" << m_transport;
203 arguments << "--payload" << targetFile().fileName();
204 arguments << osmFile().absoluteFilePath();
205 arguments << searchFile().absoluteFilePath();
206 QFileInfo kmlFile(monavDir().absoluteFilePath() + QLatin1String("/marble.kml"));
207 arguments << kmlFile.absoluteFilePath();
208 QProcess osmAddresses;
209 osmAddresses.start("osm-addresses", arguments);
210 osmAddresses.waitForFinished(1000 * 60 * 60 * 18); // wait up to 18 hours for osm-addresses to convert the data
211 if (osmAddresses.exitStatus() == QProcess::NormalExit && osmAddresses.exitCode() == 0) {
212 searchFile().refresh();
213 if (!searchFile().exists()) {
214 qDebug() << "osm-addresses did not create the .sqlite file";
215 changeStatus(Error, "Unknown error when creating the search database");
216 return false;
217 } else if (searchFile().size() < 8000) {
218 qDebug() << "The .sqlite database has a suspiciously small size.";
219 changeStatus(Error, "Search database is too small. Too little memory?");
220 return false;
221 }
222
223 kmlFile.refresh();
224 if (!kmlFile.exists()) {
225 qDebug() << "File marble.kml has not been generated.";
226 changeStatus(Error, "Failed to generate marble.kml. Too little memory?");
227 return false;
228 }
229
230 return true;
231 } else {
232 qDebug() << "osm-addresses exiting with status " << osmAddresses.exitCode();
233 changeStatus(Error, QLatin1String("Error creating search database: ") + osmAddresses.readAllStandardError());
234 return false;
235 }
236 }
237
package()238 bool Job::package()
239 {
240 changeStatus(Packaging, "Creating archive.");
241 QStringList arguments;
242 arguments << "czf" << targetFile().absoluteFilePath() << "earth/monav/" << "earth/placemarks";
243 QProcess tar;
244 tar.setWorkingDirectory(m_parameters.base().absolutePath() + QLatin1String("/data/") + m_region.id());
245 tar.start("tar", arguments);
246 tar.waitForFinished(1000 * 60 * 60); // wait up to 1 hour for tar to package things
247 if (tar.exitStatus() == QProcess::NormalExit && tar.exitCode() == 0) {
248 qDebug() << "Packaged tar file";
249 return true;
250 } else {
251 changeStatus(Error, QLatin1String("Packaging failed: ") + tar.readAllStandardError());
252 return false;
253 }
254 }
255
upload()256 bool Job::upload()
257 {
258 changeStatus(Uploading, "Uploading file");
259 if (targetFile().exists()) {
260 Upload::instance().uploadAndDelete(m_region, targetFile(), m_transport);
261 return true;
262 }
263
264 changeStatus(Error, "Target file does not exist.");
265 return false;
266 }
267
cleanup()268 bool Job::cleanup()
269 {
270 if (!m_parameters.cacheData()) {
271 QFile::remove(osmFile().absoluteFilePath());
272 }
273
274 QFileInfo subdir = QFileInfo(monavDir().absoluteFilePath());
275 if (subdir.exists() && subdir.isDir()) {
276 QFileInfoList files = QDir(subdir.absoluteFilePath()).entryInfoList(QDir::Files);
277 for(const QFileInfo &file: files) {
278 QFile::remove(file.absoluteFilePath());
279 }
280 }
281
282 QFile::remove(searchFile().absoluteFilePath());
283 return true;
284 }
285
osmFile()286 QFileInfo Job::osmFile()
287 {
288 m_parameters.base().mkdir("download");
289 QFileInfo result(m_parameters.base(), QLatin1String("download/") + m_region.id() + QLatin1String(".osm.pbf"));
290 return result;
291 }
292
monavDir()293 QFileInfo Job::monavDir()
294 {
295 QString const subdir = QLatin1String("data/") + m_region.id() + QLatin1String("/earth/monav/") + m_transport.toLower() + QLatin1Char('/') + m_region.path();
296 m_parameters.base().mkpath(subdir);
297 QFileInfo result(m_parameters.base(), subdir);
298 return result;
299 }
300
targetFile()301 QFileInfo Job::targetFile()
302 {
303 m_parameters.base().mkdir("finished");
304 QFileInfo result(m_parameters.base(), QLatin1String("finished/") + m_region.id() + QLatin1Char('_') + m_transport.toLower() + QLatin1String(".tar.gz"));
305 return result;
306 }
307
searchFile()308 QFileInfo Job::searchFile()
309 {
310 QString const subdir = QLatin1String("data/") + m_region.id() + QLatin1String("/earth/placemarks/") + QFileInfo(m_region.path()).path();
311 m_parameters.base().mkpath(subdir);
312 QFileInfo result(m_parameters.base(), subdir + QLatin1Char('/') + m_region.id() + QLatin1String(".sqlite"));
313 return result;
314 }
315
316 #include "moc_job.cpp"
317