1 /* OnlineSolver, StellarSolver Internal Library developed by Robert Lancaster, 2020
2
3 This application is free software; you can redistribute it and/or
4 modify it under the terms of the GNU General Public
5 License as published by the Free Software Foundation; either
6 version 2 of the License, or (at your option) any later version.
7 */
8 #include "onlinesolver.h"
9 #include <QTimer>
10 #include <QEventLoop>
11
OnlineSolver(ProcessType type,ExtractorType sexType,SolverType solType,FITSImage::Statistic imagestats,uint8_t const * imageBuffer,QObject * parent)12 OnlineSolver::OnlineSolver(ProcessType type, ExtractorType sexType, SolverType solType, FITSImage::Statistic imagestats,
13 uint8_t const *imageBuffer, QObject *parent) : ExternalSextractorSolver(type, sexType, solType, imagestats, imageBuffer,
14 parent)
15 {
16 connect(this, &OnlineSolver::timeToCheckJobs, this, &OnlineSolver::checkJobs);
17 connect(this, &OnlineSolver::startupOnlineSolver, this, &OnlineSolver::authenticate);
18
19 networkManager = new QNetworkAccessManager(this);
20 connect(networkManager, &QNetworkAccessManager::finished, this, &OnlineSolver::onResult);
21 }
22
execute()23 void OnlineSolver::execute()
24 {
25 if(m_ActiveParameters.multiAlgorithm != NOT_MULTI)
26 emit logOutput("The Online solver option does not support multithreading, since the server already does this internally, ignoring this option");
27
28 if(m_ExtractorType == EXTRACTOR_BUILTIN)
29 runOnlineSolver();
30 else
31 {
32 delete xcol;
33 delete ycol;
34 xcol = strdup("X"); //This is the column for the x-coordinates, it doesn't accept X_IMAGE like the other one
35 ycol = strdup("Y"); //This is the column for the y-coordinates, it doesn't accept Y_IMAGE like the other one
36 int fail = 0;
37 if(m_ExtractorType == EXTRACTOR_INTERNAL)
38 fail = runSEPSextractor();
39 else if(m_ExtractorType == EXTRACTOR_EXTERNAL)
40 fail = runExternalSextractor();
41 if(fail != 0)
42 {
43 emit finished(fail);
44 return;
45 }
46 if(m_ExtractedStars.size() == 0)
47 {
48 emit logOutput("No stars were found, so the image cannot be solved");
49 emit finished(-1);
50 return;
51 }
52 if((fail = writeSextractorTable()) != 0)
53 {
54 emit finished(fail);
55 return;
56 }
57 runOnlineSolver();
58 }
59 }
60
runOnlineSolver()61 void OnlineSolver::runOnlineSolver()
62 {
63 emit logOutput("+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++");
64 emit logOutput("Configuring Online Solver");
65
66 if(m_LogToFile && m_AstrometryLogLevel != LOG_NONE)
67 {
68 if(m_LogFileName == "")
69 m_LogFileName = m_BasePath + "/" + m_BaseName + ".log.txt";
70 if(QFile(m_LogFileName).exists())
71 QFile(m_LogFileName).remove();
72 }
73
74 solverTimer.start();
75
76 emit startupOnlineSolver(); //Go to FIRST STAGE
77 start(); //Start the other thread, which will monitor everything.
78 }
79
80 //This method will monitor the processes of the Online solver
81 //Once it the job requests are sent, it will keep checking to see when they are done
82 //It is important that this thread keeps running as long as the online solver is doing anything
83 //That way it will behave like the other solvers and isRunning produces the right result
run()84 void OnlineSolver::run()
85 {
86 bool timedOut = false;
87
88 while(!m_HasSolved && !aborted && !timedOut && workflowStage != JOB_PROCESSING_STAGE)
89 {
90 msleep(200);
91 timedOut = solverTimer.elapsed() / 1000.0 > m_ActiveParameters.solverTimeLimit;
92 }
93
94 while(!m_HasSolved && !aborted && !timedOut && (workflowStage == JOB_PROCESSING_STAGE || workflowStage == JOB_QUEUE_STAGE))
95 {
96 msleep(JOB_RETRY_DURATION);
97 if (job_retries++ > JOB_RETRY_ATTEMPTS)
98 {
99 emit logOutput(("Failed to retrieve job ID, it appears to be lost in the queue."));
100 abort();
101 }
102 else
103 {
104 emit timeToCheckJobs();
105 timedOut = solverTimer.elapsed() / 1000.0 > m_ActiveParameters.solverTimeLimit;
106 }
107 }
108
109 if(!m_HasSolved && !aborted && !timedOut)
110 {
111 emit logOutput("+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++");
112 emit logOutput("Starting Online Solver with the " + m_ActiveParameters.listName + " profile . . .");
113
114 }
115
116 while(!m_HasSolved && !aborted && !timedOut && workflowStage == JOB_MONITORING_STAGE)
117 {
118 msleep(STATUS_CHECK_INTERVAL);
119 emit timeToCheckJobs();
120 timedOut = solverTimer.elapsed() / 1000.0 > m_ActiveParameters.solverTimeLimit;
121 }
122
123 if(aborted)
124 return;
125
126 if(timedOut)
127 {
128 disconnect(networkManager, &QNetworkAccessManager::finished, this, &OnlineSolver::onResult);
129 emit logOutput("Solver timed out");
130 emit finished(-1);
131 return;
132 }
133
134 //Note, if it already has solved,
135 //It may or may not have gotten the Stars, Log, and WCS data yet.
136 //We want to wait for a little bit, but not too long.
137 bool starsAndWCSTimedOut = false;
138 double starsAndWCSTimeLimit = 10.0;
139
140 if(!aborted && !m_HasWCS)
141 {
142 emit logOutput("Waiting for Stars and WCS. . .");
143 solverTimer.start(); //Restart the timer for the Stars and WCS download
144 }
145
146 //This will wait for stars and WCS until the time limit is reached
147 //If it does get the file, whether or not it can read it, the stage changes to NO_STAGE and this quits
148 while(!aborted && !starsAndWCSTimedOut && (workflowStage == LOG_LOADING_STAGE || workflowStage == WCS_LOADING_STAGE))
149 {
150 msleep(STATUS_CHECK_INTERVAL);
151 starsAndWCSTimedOut = solverTimer.elapsed() / 1000.0 > starsAndWCSTimeLimit; //Wait 10 seconds for STARS and WCS, NO LONGER!
152 }
153
154 disconnect(networkManager, &QNetworkAccessManager::finished, this, &OnlineSolver::onResult);
155
156 if(starsAndWCSTimedOut)
157 {
158 emit logOutput("WCS download timed out");
159 emit finished(0); //Note: It DID solve and we have results, just not WCS data, that is ok.
160 }
161 }
162
163
abort()164 void OnlineSolver::abort()
165 {
166 disconnect(networkManager, &QNetworkAccessManager::finished, this, &OnlineSolver::onResult);
167 workflowStage = NO_STAGE;
168 emit logOutput("Online Solver aborted.");
169 emit finished(-1);
170 aborted = true;
171 }
172
173 //This will start up the first stage, Authentication
authenticate()174 void OnlineSolver::authenticate()
175 {
176 QNetworkRequest request;
177 request.setHeader(QNetworkRequest::ContentTypeHeader, "application/x-www-form-urlencoded");
178
179 // If pure IP, add http to it.
180 if (!astrometryAPIURL.startsWith("http"))
181 astrometryAPIURL = "http://" + astrometryAPIURL;
182
183 QUrl url(astrometryAPIURL);
184 url.setPath("/api/login");
185 request.setUrl(url);
186
187 QVariantMap apiReq;
188 apiReq.insert("apikey", astrometryAPIKey);
189 QJsonObject json = QJsonObject::fromVariantMap(apiReq);
190 QJsonDocument json_doc(json);
191
192 QString json_request = QString("request-json=%1").arg(QString(json_doc.toJson(QJsonDocument::Compact)));
193 networkManager->post(request, json_request.toUtf8());
194
195 workflowStage = AUTH_STAGE;
196 emit logOutput("Authenticating. . .");
197
198 }
199
200 //This will start up the second stage, uploading the file
uploadFile()201 void OnlineSolver::uploadFile()
202 {
203 QNetworkRequest request;
204
205 QFile *fitsFile;
206 if(m_ExtractorType == EXTRACTOR_BUILTIN)
207 fitsFile = new QFile(fileToProcess);
208 else
209 fitsFile = new QFile(sextractorFilePath);
210 bool rc = fitsFile->open(QIODevice::ReadOnly);
211 if (rc == false)
212 {
213 emit logOutput(QString("Failed to open the file %1: %2").arg( fileToProcess).arg( fitsFile->errorString()));
214 delete (fitsFile);
215 emit finished(-1);
216 return;
217 }
218
219 QUrl url(astrometryAPIURL);
220 url.setPath("/api/upload");
221 request.setUrl(url);
222
223 QHttpMultiPart *reqEntity = new QHttpMultiPart(QHttpMultiPart::FormDataType);
224
225 QVariantMap uploadReq;
226 uploadReq.insert("publicly_visible", "n");
227 uploadReq.insert("allow_modifications", "n");
228 uploadReq.insert("session", sessionKey);
229 uploadReq.insert("allow_commercial_use", "n");
230
231 if(m_ExtractorType != EXTRACTOR_BUILTIN)
232 {
233 uploadReq.insert("image_width", m_Statistics.width);
234 uploadReq.insert("image_height", m_Statistics.height);
235 }
236
237 if (m_UseScale)
238 {
239 uploadReq.insert("scale_type", "ul");
240 uploadReq.insert("scale_units", getScaleUnitString());
241 uploadReq.insert("scale_lower", scalelo);
242 uploadReq.insert("scale_upper", scalehi);
243 }
244
245 if (m_UsePosition)
246 {
247 uploadReq.insert("center_ra", search_ra);
248 uploadReq.insert("center_dec", search_dec);
249 uploadReq.insert("radius", m_ActiveParameters.search_radius);
250 }
251
252 //We would like the Coordinates found to be the center of the image
253 uploadReq.insert("crpix_center", true);
254
255 if (m_ActiveParameters.downsample != 1)
256 uploadReq.insert("downsample_factor", m_ActiveParameters.downsample);
257
258 uploadReq.insert("parity", m_ActiveParameters.search_parity);
259
260 QJsonObject json = QJsonObject::fromVariantMap(uploadReq);
261 QJsonDocument json_doc(json);
262
263 QHttpPart jsonPart;
264
265 jsonPart.setHeader(QNetworkRequest::ContentTypeHeader, "application/text/plain");
266 jsonPart.setHeader(QNetworkRequest::ContentDispositionHeader, "form-data; name=\"request-json\"");
267 jsonPart.setBody(json_doc.toJson(QJsonDocument::Compact));
268
269 QHttpPart filePart;
270
271 filePart.setHeader(QNetworkRequest::ContentTypeHeader, "application/octet-stream");
272 filePart.setHeader(QNetworkRequest::ContentDispositionHeader,
273 QString("form-data; name=\"file\"; filename=\"%1\"").arg(fileToProcess));
274 filePart.setBodyDevice(fitsFile);
275
276 // Re-parent so that it get deleted later
277 fitsFile->setParent(reqEntity);
278
279 reqEntity->append(jsonPart);
280 reqEntity->append(filePart);
281
282 QNetworkReply *reply = networkManager->post(request, reqEntity);
283 reqEntity->setParent(reply); //So that it can be deleted later
284
285 workflowStage = UPLOAD_STAGE;
286 emit logOutput(("Uploading file..."));
287 }
288
289 //This will start up the third stage, waiting till processing is done
waitForProcessing()290 void OnlineSolver::waitForProcessing()
291 {
292 workflowStage = JOB_PROCESSING_STAGE;
293 emit logOutput(("Waiting for Processing to complete..."));
294 }
295
296 //This will start up the fourth stage, getting the Job ID, essentially waiting in the Job Queue
getJobID()297 void OnlineSolver::getJobID()
298 {
299 workflowStage = JOB_QUEUE_STAGE;
300 emit logOutput(("Waiting for the Job to Start..."));
301 }
302
303 //This will start the fifth stage, monitoring the job to see when it's done
startMonitoring()304 void OnlineSolver::startMonitoring()
305 {
306 workflowStage = JOB_MONITORING_STAGE;
307 emit logOutput(("Starting Job Monitoring..."));
308 }
309
310 //This will start the sixth stage, checking the results
checkJobCalibration()311 void OnlineSolver::checkJobCalibration()
312 {
313 QNetworkRequest request;
314 QUrl getCablirationResult = QUrl(QString("%1/api/jobs/%2/calibration").arg(astrometryAPIURL).arg(jobID));
315 request.setUrl(getCablirationResult);
316 networkManager->get(request);
317
318 workflowStage = JOB_CALIBRATION_STAGE;
319 emit logOutput(("Requesting the results..."));
320 }
321
322 //This will start the seventh stage, getting the Job LOG file and loading it (optional).
getJobLogFile()323 void OnlineSolver::getJobLogFile()
324 {
325 QString URL = QString("%1/joblog/%2").arg(astrometryAPIURL).arg(jobID);
326 networkManager->get(QNetworkRequest(QUrl(URL)));
327
328 workflowStage = LOG_LOADING_STAGE;
329 emit logOutput(("Downloading the Log file..."));
330 }
331
332 //This will start the eighth stage, getting the WCS File and loading it (optional).
getJobWCSFile()333 void OnlineSolver::getJobWCSFile()
334 {
335 QString URL = QString("%1/wcs_file/%2").arg(astrometryAPIURL).arg(jobID);
336 networkManager->get(QNetworkRequest(QUrl(URL)));
337
338 workflowStage = WCS_LOADING_STAGE;
339 emit logOutput(("Downloading the WCS file..."));
340 }
341
342 //This will check on the job status during the fourth stage, as it is solving
343 //It gets called by the other thread, which is monitoring what is happening.
checkJobs()344 void OnlineSolver::checkJobs()
345 {
346 if(workflowStage == JOB_PROCESSING_STAGE || workflowStage == JOB_QUEUE_STAGE)
347 {
348 QNetworkRequest request;
349 QUrl getJobID = QUrl(QString("%1/api/submissions/%2").arg(astrometryAPIURL).arg(subID));
350 request.setUrl(getJobID);
351 networkManager->get(request);
352 }
353 if(workflowStage == JOB_MONITORING_STAGE)
354 {
355 QNetworkRequest request;
356 QUrl getJobStatus = QUrl(QString("%1/api/jobs/%2").arg(astrometryAPIURL).arg(jobID));
357 request.setUrl(getJobStatus);
358 networkManager->get(request);
359 }
360 }
361
362 //This handles the replies from the server
onResult(QNetworkReply * reply)363 void OnlineSolver::onResult(QNetworkReply *reply)
364 {
365 bool ok = false;
366 QJsonParseError parseError;
367 QString status;
368 QList<QVariant> jsonArray;
369
370 if(m_SSLogLevel != LOG_OFF)
371 emit logOutput("Reply Received");
372
373 if (workflowStage == NO_STAGE)
374 {
375 reply->abort();
376 return;
377 }
378
379 if (reply->error() != QNetworkReply::NoError)
380 {
381 emit logOutput(reply->errorString());
382 emit finished(-1);
383 return;
384 }
385 QString json;
386 QJsonDocument json_doc;
387 QVariant json_result;
388 QVariantMap result;
389 if(workflowStage != LOG_LOADING_STAGE && workflowStage != WCS_LOADING_STAGE)
390 {
391 json = (QString)reply->readAll();
392
393 json_doc = QJsonDocument::fromJson(json.toUtf8(), &parseError);
394
395 if (parseError.error != QJsonParseError::NoError)
396 {
397 emit logOutput(QString("JSON error during parsing (%1).").arg(parseError.errorString()));
398 emit finished(-1);
399 return;
400 }
401
402 json_result = json_doc.toVariant();
403 result = json_result.toMap();
404
405 if(m_SSLogLevel != LOG_OFF)
406 emit logOutput(json_doc.toJson(QJsonDocument::Compact));
407 }
408 switch (workflowStage)
409 {
410 case AUTH_STAGE:
411 status = result["status"].toString();
412 if (status != "success")
413 {
414 emit logOutput(QString("%1 authentication failed. Check the validity of the API Key.").arg(astrometryAPIURL));
415 abort();
416 return;
417 }
418
419 sessionKey = result["session"].toString();
420
421 if(m_SSLogLevel != LOG_OFF)
422 emit logOutput(QString("Authentication to %1 is successful. Session: %2").arg(astrometryAPIURL).arg(sessionKey));
423
424 uploadFile(); //Go to NEXT STAGE
425 break;
426
427 case UPLOAD_STAGE:
428 status = result["status"].toString();
429 if (status != "success")
430 {
431 emit logOutput(("Upload failed."));
432 abort();
433 return;
434 }
435
436 subID = result["subid"].toInt(&ok);
437
438 if (ok == false)
439 {
440 emit logOutput(("Parsing submission ID failed."));
441 abort();
442 return;
443 }
444
445 emit logOutput(QString("Upload complete. Waiting for %1 solver to complete...").arg(astrometryAPIURL));
446 waitForProcessing(); //Go to the NEXT STAGE
447 break;
448
449 case JOB_PROCESSING_STAGE:
450 {
451 QString finished;
452 finished = result["processing_finished"].toString();
453
454 if (finished == "None" || finished == "")
455 return;
456
457 getJobID(); //Go to the NEXT STAGE
458 }
459 break;
460
461 case JOB_QUEUE_STAGE:
462 jsonArray = result["jobs"].toList();
463
464 if (jsonArray.isEmpty())
465 jobID = 0;
466 else
467 jobID = jsonArray.first().toInt(&ok);
468
469 if (jobID == 0 || !ok)
470 return;
471
472 startMonitoring(); //Go to the NEXT STAGE
473 break;
474
475 case JOB_MONITORING_STAGE:
476 status = result["status"].toString();
477 if (status == "success")
478 checkJobCalibration(); // Go to the NEXT STAGE
479 else if (status == "solving" || status == "processing")
480 {
481 return;
482 }
483 else if (status == "failure")
484 {
485 emit logOutput("Solver Failed");
486 abort();
487 return;
488 }
489 break;
490
491 case JOB_CALIBRATION_STAGE:
492 {
493 double fieldw = result["width_arcsec"].toDouble(&ok) / 60.0;
494 if (ok == false)
495 {
496 emit logOutput(("Error parsing width."));
497 abort();
498 return;
499 }
500 double fieldh = result["height_arcsec"].toDouble(&ok) / 60.0;
501 if (ok == false)
502 {
503 emit logOutput(("Error parsing width."));
504 abort();
505 return;
506 }
507 int parity = result["parity"].toInt(&ok);
508 if (ok == false)
509 {
510 emit logOutput(("Error parsing parity."));
511 abort();
512 return;
513 }
514 double orientation = result["orientation"].toDouble(&ok);
515 if (ok == false)
516 {
517 emit logOutput(("Error parsing orientation."));
518 abort();
519 return;
520 }
521 orientation *= parity;
522 double ra = result["ra"].toDouble(&ok);
523 if (ok == false)
524 {
525 emit logOutput(("Error parsing RA."));
526 abort();
527 return;
528 }
529 double dec = result["dec"].toDouble(&ok);
530 if (ok == false)
531 {
532 emit logOutput(("Error parsing DEC."));
533 abort();
534 return;
535 }
536
537 double pixscale = result["pixscale"].toDouble(&ok);
538 if (ok == false)
539 {
540 emit logOutput(("Error parsing DEC."));
541 abort();
542 return;
543 }
544
545 float raErr = 0;
546 float decErr = 0;
547 if(m_UsePosition)
548 {
549 raErr = (search_ra - ra) * 3600;
550 decErr = (search_dec - dec) * 3600;
551 }
552 QString par = (parity > 0) ? "neg" : "pos";
553 m_Solution = {fieldw, fieldh, ra, dec, orientation, pixscale, par, raErr, decErr};
554 m_HasSolved = true;
555
556 if(m_AstrometryLogLevel != LOG_NONE || m_LogToFile)
557 getJobLogFile(); //Go to next stage
558 else
559 getJobWCSFile(); //Go to Last Stage
560 }
561 break;
562
563 case LOG_LOADING_STAGE:
564 {
565 QByteArray responseData = reply->readAll();
566
567 if(m_AstrometryLogLevel != LOG_NONE && !m_LogToFile)
568 emit logOutput(responseData);
569
570 if(m_LogToFile)
571 {
572 QFile file(m_LogFileName);
573 if (!file.open(QIODevice::WriteOnly))
574 {
575 emit logOutput(("Log File Write Error"));
576 }
577 file.write(responseData.data(), responseData.size());
578 file.close();
579 }
580 getJobWCSFile(); //Go to Last Stage
581 }
582 break;
583
584 case WCS_LOADING_STAGE:
585 {
586 QByteArray responseData = reply->readAll();
587 QString solutionFile = m_BasePath + "/" + m_BaseName + ".wcs";
588 QFile file(solutionFile);
589 if (!file.open(QIODevice::WriteOnly))
590 {
591 emit logOutput(("WCS File Write Error"));
592 emit finished(0); //We still have the solution, this is not a failure!
593 return;
594 }
595 file.write(responseData.data(), responseData.size());
596 file.close();
597 loadWCS(); //Attempt to load WCS from the file
598 emit finished(0); //Success! We are completely done, whether or not the WCS loading was successful
599 workflowStage = NO_STAGE;
600 }
601 break;
602
603 default:
604 break;
605 }
606 }
607
608