1 #include "LotWUsers.hpp"
2 
3 #include <future>
4 #include <chrono>
5 
6 #include <QHash>
7 #include <QString>
8 #include <QDate>
9 #include <QFile>
10 #include <QTextStream>
11 #include <QDir>
12 #include <QFileInfo>
13 #include <QPointer>
14 #include <QSaveFile>
15 #include <QUrl>
16 #include <QNetworkAccessManager>
17 #include <QNetworkReply>
18 #include <QDebug>
19 
20 #include "pimpl_impl.hpp"
21 
22 #include "moc_LotWUsers.cpp"
23 
24 namespace
25 {
26   // Dictionary mapping call sign to date of last upload to LotW
27   using dictionary = QHash<QString, QDate>;
28 }
29 
30 class LotWUsers::impl final
31   : public QObject
32 {
33   Q_OBJECT
34 
35 public:
impl(LotWUsers * self,QNetworkAccessManager * network_manager)36   impl (LotWUsers * self, QNetworkAccessManager * network_manager)
37     : self_ {self}
38     , network_manager_ {network_manager}
39     , url_valid_ {false}
40     , redirect_count_ {0}
41     , age_constraint_ {365}
42   {
43   }
44 
load(QString const & url,bool fetch,bool forced_fetch)45   void load (QString const& url, bool fetch, bool forced_fetch)
46   {
47     abort ();                   // abort any active download
48     auto csv_file_name = csv_file_.fileName ();
49     auto exists = QFileInfo::exists (csv_file_name);
50     if (fetch && (!exists || forced_fetch))
51       {
52         current_url_.setUrl (url);
53         if (current_url_.isValid () && !QSslSocket::supportsSsl ())
54           {
55             current_url_.setScheme ("http");
56           }
57         redirect_count_ = 0;
58         download (current_url_);
59       }
60     else
61       {
62         if (exists)
63           {
64             // load the database asynchronously
65             future_load_ = std::async (std::launch::async, &LotWUsers::impl::load_dictionary, this, csv_file_name);
66           }
67       }
68   }
69 
download(QUrl url)70   void download (QUrl url)
71   {
72 #if QT_VERSION < QT_VERSION_CHECK(5, 15, 0)
73     if (QNetworkAccessManager::Accessible != network_manager_->networkAccessible ())
74       {
75         // try and recover network access for QNAM
76         network_manager_->setNetworkAccessible (QNetworkAccessManager::Accessible);
77       }
78 #endif
79 
80     QNetworkRequest request {url};
81     request.setRawHeader ("User-Agent", "WSJT LotW User Downloader");
82     request.setOriginatingObject (this);
83 
84     // this blocks for a second or two the first time it is used on
85     // Windows - annoying
86     if (!url_valid_)
87       {
88         reply_ = network_manager_->head (request);
89       }
90     else
91       {
92         reply_ = network_manager_->get (request);
93       }
94 
95     connect (reply_.data (), &QNetworkReply::finished, this, &LotWUsers::impl::reply_finished);
96     connect (reply_.data (), &QNetworkReply::readyRead, this, &LotWUsers::impl::store);
97   }
98 
reply_finished()99   void reply_finished ()
100   {
101     if (!reply_)
102       {
103         Q_EMIT self_->load_finished ();
104         return;           // we probably deleted it in an earlier call
105       }
106     QUrl redirect_url {reply_->attribute (QNetworkRequest::RedirectionTargetAttribute).toUrl ()};
107     if (reply_->error () == QNetworkReply::NoError && !redirect_url.isEmpty ())
108       {
109         if ("https" == redirect_url.scheme () && !QSslSocket::supportsSsl ())
110           {
111             Q_EMIT self_->LotW_users_error (tr ("Network Error - SSL/TLS support not installed, cannot fetch:\n\'%1\'")
112                                             .arg (redirect_url.toDisplayString ()));
113             url_valid_ = false; // reset
114             Q_EMIT self_->load_finished ();
115           }
116         else if (++redirect_count_ < 10) // maintain sanity
117           {
118             // follow redirect
119             download (reply_->url ().resolved (redirect_url));
120           }
121         else
122           {
123             Q_EMIT self_->LotW_users_error (tr ("Network Error - Too many redirects:\n\'%1\'")
124                                             .arg (redirect_url.toDisplayString ()));
125             url_valid_ = false; // reset
126             Q_EMIT self_->load_finished ();
127           }
128       }
129     else if (reply_->error () != QNetworkReply::NoError)
130       {
131         csv_file_.cancelWriting ();
132         csv_file_.commit ();
133         url_valid_ = false;     // reset
134         // report errors that are not due to abort
135         if (QNetworkReply::OperationCanceledError != reply_->error ())
136           {
137             Q_EMIT self_->LotW_users_error (tr ("Network Error:\n%1")
138                                             .arg (reply_->errorString ()));
139           }
140         Q_EMIT self_->load_finished ();
141       }
142     else
143       {
144         if (url_valid_ && !csv_file_.commit ())
145           {
146             Q_EMIT self_->LotW_users_error (tr ("File System Error - Cannot commit changes to:\n\"%1\"")
147                                             .arg (csv_file_.fileName ()));
148             url_valid_ = false; // reset
149             Q_EMIT self_->load_finished ();
150           }
151         else
152           {
153             if (!url_valid_)
154               {
155                 // now get the body content
156                 url_valid_ = true;
157                 download (reply_->url ().resolved (redirect_url));
158               }
159             else
160               {
161                 url_valid_ = false; // reset
162                 // load the database asynchronously
163                 future_load_ = std::async (std::launch::async, &LotWUsers::impl::load_dictionary, this, csv_file_.fileName ());
164               }
165           }
166       }
167     if (reply_ && reply_->isFinished ())
168       {
169         reply_->deleteLater ();
170       }
171   }
172 
store()173   void store ()
174   {
175     if (url_valid_)
176       {
177         if (!csv_file_.isOpen ())
178           {
179             // create temporary file in the final location
180             if (!csv_file_.open (QSaveFile::WriteOnly))
181               {
182                 abort ();
183                 Q_EMIT self_->LotW_users_error (tr ("File System Error - Cannot open file:\n\"%1\"\nError(%2): %3")
184                                                 .arg (csv_file_.fileName ())
185                                                 .arg (csv_file_.error ())
186                                                 .arg (csv_file_.errorString ()));
187               }
188           }
189         if (csv_file_.write (reply_->read (reply_->bytesAvailable ())) < 0)
190           {
191             abort ();
192             Q_EMIT self_->LotW_users_error (tr ("File System Error - Cannot write to file:\n\"%1\"\nError(%2): %3")
193                                             .arg (csv_file_.fileName ())
194                                             .arg (csv_file_.error ())
195                                             .arg (csv_file_.errorString ()));
196           }
197       }
198   }
199 
abort()200   void abort ()
201   {
202     if (reply_ && reply_->isRunning ())
203       {
204         reply_->abort ();
205       }
206   }
207 
208   // Load the database from the given file name
209   //
210   // Expects the file to be in CSV format with no header with one
211   // record per line. Record fields are call sign followed by upload
212   // date in yyyy-MM-dd format followed by upload time (ignored)
load_dictionary(QString const & lotw_csv_file)213   dictionary load_dictionary (QString const& lotw_csv_file)
214   {
215     dictionary result;
216     QFile f {lotw_csv_file};
217     if (f.open (QFile::ReadOnly | QFile::Text))
218       {
219         QTextStream s {&f};
220         for (auto l = s.readLine (); !l.isNull (); l = s.readLine ())
221           {
222             auto pos = l.indexOf (',');
223             result[l.left (pos)] = QDate::fromString (l.mid (pos + 1, l.indexOf (',', pos + 1) - pos - 1), "yyyy-MM-dd");
224           }
225 //        qDebug () << "LotW User Data Loaded";
226       }
227     else
228       {
229         throw std::runtime_error {QObject::tr ("Failed to open LotW users CSV file: '%1'").arg (f.fileName ()).toStdString ()};
230       }
231     return result;
232   }
233 
234   LotWUsers * self_;
235   QNetworkAccessManager * network_manager_;
236   QSaveFile csv_file_;
237   bool url_valid_;
238   QUrl current_url_;            // may be a redirect
239   int redirect_count_;
240   QPointer<QNetworkReply> reply_;
241   std::future<dictionary> future_load_;
242   dictionary last_uploaded_;
243   qint64 age_constraint_;       // days
244 };
245 
246 #include "LotWUsers.moc"
247 
LotWUsers(QNetworkAccessManager * network_manager,QObject * parent)248 LotWUsers::LotWUsers (QNetworkAccessManager * network_manager, QObject * parent)
249   : QObject {parent}
250   , m_ {this, network_manager}
251 {
252 }
253 
~LotWUsers()254 LotWUsers::~LotWUsers ()
255 {
256 }
257 
set_local_file_path(QString const & path)258 void LotWUsers::set_local_file_path (QString const& path)
259 {
260   m_->csv_file_.setFileName (path);
261 }
262 
load(QString const & url,bool fetch,bool force_download)263 void LotWUsers::load (QString const& url, bool fetch, bool force_download)
264 {
265   m_->load (url, fetch, force_download);
266 }
267 
set_age_constraint(qint64 uploaded_since_days)268 void LotWUsers::set_age_constraint (qint64 uploaded_since_days)
269 {
270   m_->age_constraint_ = uploaded_since_days;
271 }
272 
user(QString const & call) const273 bool LotWUsers::user (QString const& call) const
274 {
275   // check if a pending asynchronous load is ready
276   if (m_->future_load_.valid ()
277       && std::future_status::ready == m_->future_load_.wait_for (std::chrono::seconds {0}))
278     {
279       try
280         {
281           // wait for the load to finish if necessary
282           const_cast<dictionary&> (m_->last_uploaded_) = const_cast<std::future<dictionary>&> (m_->future_load_).get ();
283         }
284       catch (std::exception const& e)
285         {
286           Q_EMIT LotW_users_error (e.what ());
287         }
288       Q_EMIT load_finished ();
289     }
290   if (m_->last_uploaded_.size ())
291     {
292       auto p = m_->last_uploaded_.constFind (call);
293       if (p != m_->last_uploaded_.end ())
294         {
295           return p.value ().daysTo (QDate::currentDate ()) <= m_->age_constraint_;
296         }
297     }
298   return false;
299 }
300