1 /* cclive
2 * Copyright (C) 2010-2013 Toni Gundogdu <legatvs@gmail.com>
3 *
4 * This program is free software: you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation, either version 3 of the License, or
7 * (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program. If not, see <http://www.gnu.org/licenses/>.
16 */
17
18 #include <ccinternal>
19
20 #include <stdexcept>
21 #include <fstream>
22 #include <sstream>
23 #include <iomanip>
24
25 #ifdef HAVE_UNISTD_H
26 #include <unistd.h>
27 #endif
28
29 #ifdef HAVE_SIGNAL_H
30 #include <signal.h>
31 #endif
32
33 #if defined (HAVE_SIGNAL_H) && defined (HAVE_SIGNAL)
34 #define WITH_SIGNAL
35 #endif
36
37 #include <boost/program_options/variables_map.hpp>
38 #include <boost/filesystem.hpp>
39 #include <boost/foreach.hpp>
40 #include <boost/format.hpp>
41
42 #ifndef foreach
43 #define foreach BOOST_FOREACH
44 #endif
45
46 #include <curl/curl.h>
47 #include <pcrecpp.h>
48
49 #include <ccquvi>
50 #include <ccoptions>
51 #include <ccprogressbar>
52 #include <ccre>
53 #include <ccutil>
54 #include <cclog>
55 #include <ccfile>
56
57 namespace cc
58 {
59
60 namespace po = boost::program_options;
61
file(const quvi::media & media)62 file::file(const quvi::media& media)
63 : _initial_length(0), _nothing_todo(false)
64 {
65 try
66 {
67 _init(media);
68 }
69 catch (const cc::nothing_todo_error&)
70 {
71 _nothing_todo = true;
72 }
73 }
74
75 #define E "server response code %ld, expected 200 or 206 (conn_code=%ld)"
76
format_unexpected_http_error(const long resp_code,const long conn_code)77 static std::string format_unexpected_http_error(
78 const long resp_code,
79 const long conn_code)
80 {
81 return (boost::format(E) % resp_code % conn_code).str();
82 }
83
84 #undef E
85
86 #define E "%s (curl_code=%ld, resp_code=%ld, conn_code=%ld)"
87
format_error(const CURLcode curl_code,const long resp_code,const long conn_code)88 static std::string format_error(const CURLcode curl_code,
89 const long resp_code,
90 const long conn_code)
91 {
92 const std::string e = curl_easy_strerror(curl_code);
93 return (boost::format(E) % e % curl_code % resp_code % conn_code).str();
94 }
95
96 #undef E
97
io_error(const std::string & fpath)98 static std::string io_error(const std::string& fpath)
99 {
100 std::string s = fpath + ": ";
101 if (errno)
102 s += cc::perror();
103 else
104 s += "unknown i/o error";
105 return (s);
106 }
107
io_error(const cc::file & f)108 static std::string io_error(const cc::file& f)
109 {
110 return io_error(f.path());
111 }
112
113 class write_data
114 {
115 public:
write_data(cc::file * f)116 inline write_data(cc::file *f):o(NULL), f(f) { }
~write_data()117 inline ~write_data()
118 {
119 if (o == NULL)
120 return;
121
122 o->flush();
123 o->close();
124
125 delete o;
126 o = NULL;
127 }
open_file()128 inline void open_file()
129 {
130 std::ios_base::openmode mode = std::ofstream::binary;
131
132 if (cc::opts.flags.overwrite)
133 mode |= std::ofstream::trunc;
134 else
135 {
136 if (f->should_continue())
137 mode |= std::ofstream::app;
138 }
139
140 o = new std::ofstream(f->path().c_str(), mode);
141 if (o->fail())
142 throw std::runtime_error(io_error(*f));
143 }
144 public:
145 std::ofstream *o;
146 cc::file *f;
147 };
148
write_cb(void * ptr,size_t size,size_t nmemb,void * userdata)149 static size_t write_cb(void *ptr, size_t size, size_t nmemb, void *userdata)
150 {
151 write_data *w = reinterpret_cast<write_data*>(userdata);
152 const size_t rsize = size*nmemb;
153
154 w->o->write(static_cast<char*>(ptr), rsize);
155 if (w->o->fail())
156 return w->f->set_errmsg(io_error(*w->f));
157
158 w->o->flush();
159 if (w->o->fail())
160 return w->f->set_errmsg(io_error(*w->f));
161
162 return rsize;
163 }
164
165 #ifdef WITH_SIGNAL
166 static volatile sig_atomic_t recv_usr1;
167
handle_usr1(int s)168 static void handle_usr1(int s)
169 {
170 if (s == SIGUSR1)
171 recv_usr1 = 1;
172 }
173 #endif
174
progress_cb(void * ptr,double,double now,double,double)175 static int progress_cb(void *ptr, double, double now, double, double)
176 {
177 #ifdef WITH_SIGNAL
178 if (recv_usr1)
179 {
180 recv_usr1 = 0;
181 return 1; // Return a non-zero value to abort this transfer.
182 }
183 #endif
184 return reinterpret_cast<progressbar*>(ptr)->update(now);
185 }
186
_set(write_data * w,const quvi::media & m,CURL * c,progressbar * pb,const double initial_length)187 static void _set(write_data *w, const quvi::media& m, CURL *c,
188 progressbar *pb, const double initial_length)
189 {
190 curl_easy_setopt(c, CURLOPT_URL, m.stream_url().c_str());
191
192 curl_easy_setopt(c, CURLOPT_WRITEFUNCTION, write_cb);
193 curl_easy_setopt(c, CURLOPT_WRITEDATA, w);
194
195 curl_easy_setopt(c, CURLOPT_PROGRESSFUNCTION, progress_cb);
196 curl_easy_setopt(c, CURLOPT_PROGRESSDATA, pb);
197 curl_easy_setopt(c, CURLOPT_NOPROGRESS, 0L);
198
199 curl_easy_setopt(c, CURLOPT_ENCODING, "identity");
200 curl_easy_setopt(c, CURLOPT_HEADER, 0L);
201
202 if (cc::opts.flags.timestamp)
203 curl_easy_setopt(c, CURLOPT_FILETIME, 1L);
204
205 const po::variables_map map = cc::opts.map();
206
207 curl_easy_setopt(c, CURLOPT_MAX_RECV_SPEED_LARGE,
208 static_cast<curl_off_t>(map["throttle"].as<int>()*1024));
209
210 curl_easy_setopt(c, CURLOPT_RESUME_FROM_LARGE,
211 static_cast<curl_off_t>(initial_length));
212 }
213
_restore(CURL * c)214 static void _restore(CURL *c)
215 {
216 curl_easy_setopt(c, CURLOPT_RESUME_FROM_LARGE, 0L);
217 curl_easy_setopt(c, CURLOPT_NOPROGRESS, 1L);
218 curl_easy_setopt(c, CURLOPT_HEADER, 1L);
219
220 curl_easy_setopt(c, CURLOPT_MAX_RECV_SPEED_LARGE,
221 static_cast<curl_off_t>(0L));
222 }
223
_handle_error(const long resp_code,const CURLcode rc,std::string & errmsg)224 static bool _handle_error(const long resp_code, const CURLcode rc,
225 std::string& errmsg)
226 {
227 cc::log << std::endl;
228
229 // If an unrecoverable error then do not attempt to retry.
230 if (resp_code >= 400 && resp_code <= 500)
231 throw std::runtime_error(errmsg);
232
233 // Otherwise.
234 bool r = false; // Attempt to retry by default.
235 #ifdef WITH_SIGNAL
236 if (rc == 42) // 42=Operation aborted by callback (libcurl).
237 {
238 errmsg = "sigusr1 received: interrupt current download";
239 r = true; // Skip - do not attempt to retry.
240 }
241 #endif
242 cc::log << "error: " << errmsg << std::endl;
243 return r;
244 }
245
246 namespace fs = boost::filesystem;
247
write(const quvi::media & m,CURL * curl) const248 bool file::write(const quvi::media& m, CURL *curl) const
249 {
250 write_data w(const_cast<cc::file*>(this));
251 w.open_file();
252
253 progressbar pb(*this, m);
254 _set(&w, m, curl, &pb, _initial_length);
255
256 #ifdef WITH_SIGNAL
257 recv_usr1 = 0;
258 if (signal(SIGUSR1, handle_usr1) == SIG_ERR)
259 {
260 cc::log << "warning: ";
261 if (errno)
262 cc::log << cc::perror();
263 else
264 cc::log << "unable to catch SIGUSR1";
265 cc::log << std::endl;
266 }
267 #endif
268
269 const CURLcode rc = curl_easy_perform(curl);
270 _restore(curl);
271
272 // Restore curl settings.
273
274 curl_easy_setopt(curl, CURLOPT_HEADER, 1L);
275 curl_easy_setopt(curl, CURLOPT_FILETIME, 0L);
276 curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1L);
277 curl_easy_setopt(curl, CURLOPT_RESUME_FROM_LARGE, 0L);
278 curl_easy_setopt(curl,
279 CURLOPT_MAX_RECV_SPEED_LARGE,
280 static_cast<curl_off_t>(0L));
281
282 long resp_code = 0;
283 long conn_code = 0;
284
285 curl_easy_getinfo(curl, CURLINFO_HTTP_CONNECTCODE, &conn_code);
286 curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &resp_code);
287
288 std::string error;
289
290 if (CURLE_OK == rc)
291 {
292 if (resp_code != 200 && resp_code != 206)
293 error = format_unexpected_http_error(resp_code, conn_code);
294 }
295 else
296 {
297 if (CURLE_WRITE_ERROR == rc) // write_cb returned != rsize
298 error = _errmsg;
299 else
300 error = format_error(rc, resp_code, conn_code);
301 }
302
303 if (!error.empty())
304 return _handle_error(resp_code, rc, error);
305
306 pb.finish();
307 cc::log << std::endl;
308
309 if (cc::opts.flags.timestamp)
310 {
311 long ft = -1;
312 curl_easy_getinfo(curl, CURLINFO_FILETIME, &ft);
313 if (ft >=0)
314 fs::last_write_time(_path, ft);
315 }
316
317 return true;
318 }
319
to_mb(const double bytes)320 static double to_mb(const double bytes)
321 {
322 return bytes/(1024*1024);
323 }
324
to_s(const quvi::media & m) const325 std::string file::to_s(const quvi::media& m) const
326 {
327 const double length = to_mb(m.content_length());
328
329 boost::format fmt = boost::format("%s %.2fM [%s]")
330 % _name % length % m.content_type();
331
332 return fmt.str();
333 }
334
output_dir(const po::variables_map & map)335 static fs::path output_dir(const po::variables_map& map)
336 {
337 fs::path dir;
338 if (map.count("output-dir"))
339 dir = map["output-dir"].as<std::string>();
340 return fs::system_complete(dir);
341 }
342
343 typedef std::vector<std::string> vst;
344
_init(const quvi::media & media)345 void file::_init(const quvi::media& media)
346 {
347 _title = media.title();
348
349 const po::variables_map map = cc::opts.map();
350
351 if (map.count("output-file"))
352 {
353 // Overrides --filename-format.
354
355 fs::path p = output_dir(map);
356
357 p /= map["output-file"].as<std::string>();
358
359 #if BOOST_FILESYSTEM_VERSION > 2
360 _name = p.filename().string();
361 #else
362 _name = p.filename();
363 #endif
364 _path = p.string();
365 _initial_length = file::exists(_path);
366
367 if ( _initial_length >= media.content_length() && ! opts.flags.overwrite)
368 throw cc::nothing_todo_error();
369 }
370
371 else
372 {
373 // Cleanup media title.
374
375 std::string title = media.title();
376 vst tr;
377
378 if (map.count("tr"))
379 tr = map["tr"].as<vst>();
380 else // Use built-in default value.
381 {
382 if (map.count("regexp")) // Deprecated.
383 cc::re::capture(map["regexp"].as<std::string>(), title);
384 else
385 tr.push_back("/(\\w|\\pL|\\s)/g");
386 }
387
388 foreach (const std::string r, tr)
389 {
390 cc::re::tr(r, title);
391 }
392 cc::re::trim(title);
393
394 // --filename-format
395
396 std::string fname_format = map["filename-format"].as<std::string>();
397
398 pcrecpp::RE("%i").GlobalReplace(media.id(), &fname_format);
399 pcrecpp::RE("%t").GlobalReplace(title, &fname_format);
400 pcrecpp::RE("%h").GlobalReplace("nohostseq", &fname_format);
401 pcrecpp::RE("%s").GlobalReplace(media.file_ext(), &fname_format);
402
403 if (map.count("subst")) // Deprecated.
404 {
405 std::istringstream iss(map["subst"].as<std::string>());
406 vst v;
407
408 std::copy(
409 std::istream_iterator<std::string >(iss),
410 std::istream_iterator<std::string >(),
411 std::back_inserter<vst>(v)
412 );
413
414 foreach (const std::string s, v)
415 {
416 cc::re::subst(s, fname_format);
417 }
418 }
419
420 std::stringstream b;
421
422 b << fname_format;
423
424 // Output dir.
425
426 const fs::path out_dir = output_dir(map);
427 fs::path templ_path = out_dir;
428
429 templ_path /= b.str();
430
431 // Path, name.
432
433 fs::path p = fs::system_complete(templ_path);
434
435 #if BOOST_FILESYSTEM_VERSION > 2
436 _name = p.filename().string();
437 #else
438 _name = p.filename();
439 #endif
440 _path = p.string();
441
442 if (! opts.flags.overwrite)
443 {
444 for (int i=0; i<INT_MAX; ++i)
445 {
446 _initial_length = file::exists(_path);
447
448 if (_initial_length == 0)
449 break;
450
451 else if (_initial_length >= media.content_length())
452 throw cc::nothing_todo_error();
453
454 else
455 {
456 if (opts.flags.cont)
457 break;
458 }
459
460 boost::format fmt =
461 boost::format("%1%.%2%") % templ_path.string() % i;
462
463 p = fs::system_complete(fmt.str());
464
465 #if BOOST_FILESYSTEM_VERSION > 2
466 _name = p.filename().string();
467 #else
468 _name = p.filename();
469 #endif
470 _path = p.string();
471 }
472 }
473 }
474
475 if (opts.flags.overwrite)
476 _initial_length = 0;
477 }
478
exists(const std::string & path)479 double file::exists(const std::string& path)
480 {
481 fs::path p( fs::system_complete(path) );
482
483 double size = 0;
484
485 if (fs::exists(p))
486 size = static_cast<double>(fs::file_size(p));
487
488 return size;
489 }
490
491 } // namespace cc
492
493 // vim: set ts=2 sw=2 tw=72 expandtab:
494