1 /*
2 This file is part of the metalink program
3 Copyright (C) 2008 A. Bram Neijt <bneijt@gmail.com>
4
5 This program is free software: you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published by
7 the Free Software Foundation, either version 3 of the License, or
8 (at your option) any later version.
9
10 This program is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 GNU General Public License for more details.
14
15 You should have received a copy of the GNU General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 */
19
20
21 /** \file The main program
22
23 The main program creates the hashes, interprets the commandline, fills the bneijt::Globals class
24 and then opens and handles all files.
25
26 All hash classes are part of a vector created at the beginning.
27
28 As the list of hashes used is still small, we'll just learn to live with it.
29
30 */
31
32 #include <iostream>
33 #include <fstream>
34 #include <string>
35 #include <utility>
36 #include <algorithm>
37 #include <vector>
38 #include <set>
39 #include <glibmm/init.h>
40 #include <glibmm/optioncontext.h>
41 //#include <boost/program_options.hpp>
42 //#include <boost/filesystem/operations.hpp>
43 //#include <boost/filesystem/convenience.hpp>
44
45
46 #include "Options/Options.hh"
47 #include "Mirror/Mirror.hh"
48 #include "MirrorList/MirrorList.hh"
49 #include "Metalink/Metalink.hh"
50 #include "MetalinkFile/MetalinkFile.hh"
51 #include "MD5File/MD5File.hh"
52 #include "HashList/HashList.hh"
53
54 #include "Hash/GCrypt/GCrypt.hh"
55 #include "Hash/HashED2K/HashED2K.hh"
56 #include "Hash/HashGNUnet/HashGNUnet.hh"
57 #include "Hash/HashPieces/HashPieces.hh"
58
59 #include "Globals/Globals.hh"
60 #include "String/String.hh"
61 //#define DEBUGLEVEL 3
62 #include "Preprocessor/debug.hh"
63 #include "Preprocessor/os_win.hh"
64
65 #include <typeinfo>
66 #include <cassert>
67 #include <sys/stat.h> //mkdir(2)
68 #include <sys/types.h> //mkdir(2)
69
70 using namespace std;
71 using namespace bneijt;
72 //namespace po = boost::program_options;
73
74 namespace {
file_size(std::string const & fname)75 unsigned long long file_size(std::string const &fname)
76 {
77 struct stat info;
78
79 if(stat(fname.c_str(),&info) == 0)
80 return info.st_size;
81 //Should through error
82 return 0;
83 }
84 }
85
86
main(int argc,char * argv[])87 int main(int argc, char *argv[])
88 try
89 {
90 // po::variables_map variableMap;
91 bool allDigests(false), readMirrors(true), hashList(false);
92 vector<string> inputFiles, linkFiles;
93 set<string> digests;
94
95 //Program option handling
96 Glib::init();
97 Glib::OptionContext context;
98 context.set_help_enabled(false);
99 bneijt::Options options;
100 Glib::OptionGroup::vecustrings &md5Files = options.opt.md5files;
101 Glib::ustring &baseUrl = options.opt.addpath;
102 Glib::ustring &headerFile = options.opt.headerfile;
103 Glib::ustring &metalinkDescription = options.opt.desc;
104
105 context.set_main_group(options);
106
107
108
109 #ifdef GLIBMM_EXCEPTIONS_ENABLED
110 try
111 {
112 context.parse(argc, argv);
113 }
114 catch(const Glib::Error& ex)
115 {
116 std::cout << "Exception: " << ex.what() << std::endl;
117 }
118 #else
119 std::auto_ptr<Glib::Error> ex;
120 context.parse(argc, argv, ex);
121 if(ex.get())
122 {
123 std::cout << "Exception: " << ex->what() << std::endl;
124 }
125 #endif //GLIBMM_EXCEPTIONS_ENABLED
126
127
128 //Input files
129 for(int i = 1; i < argc; ++ i)
130 inputFiles.push_back(argv[i]);
131
132 //Handle options
133 if(options.opt.help)
134 {
135 cout << Globals::programName << " - " << Globals::programDescription << "\n";
136 cout << "Version " << Globals::version[0] << "." << Globals::version[1] << "." << Globals::version[2];
137 cout << ", Copyright (C) 2005 A. Bram Neijt <bneijt@gmail.com>\n";
138 cout << Globals::programName << " comes with ABSOLUTELY NO WARRANTY and is licensed under GPLv3+\n";
139 cout << "Usage:\n " << Globals::programName << " [options] (input files or --md5) < (mirror list) > (metalinkfile)\n";
140 ///TODO show help cout << helpOptions << "\n";
141 cout << context.get_help();
142 cout << "Supported algorithms are (-d options):\n"
143 << " md4 md5 sha1 sha256 sha384 sha512 rmd160 tiger crc32 ed2k gnunet sha1pieces"
144 << "\n";
145 cout << "\nMirror lists are single line definitions according to:\n"
146 << " [location [preference] [type] % ] <mirror base url>\n";
147 cout << "\nExamples:\n";
148 cout << "http://example.com/ as a mirror:\n echo http://example.com | "
149 << Globals::programName << " -d md5 -d sha1 *\n";
150 cout << "\nhttp://example.com/ as a mirror with preference and location:\n "
151 << "echo us 10 % http://example.com | " << Globals::programName << " -d md5 -d sha1 *\n";
152 cout << "\nhttp://example.com/ as a mirror with preference only:\n "
153 << "echo 0 10 % http://example.com | " << Globals::programName << " -d md5 *\n";
154 cout << "\nOnly P2P links:\n "
155 << Globals::programName << " --nomirrors -d sha1 -d ed2k -d gnunet *\n";
156 return 1;
157 }
158 if(options.opt.version)
159 {
160 cout << Globals::programName << " version "
161 << Globals::version[0] << "."
162 << Globals::version[1] << "."
163 << Globals::version[2] << "\n";
164 return 1;
165 }
166 //Verify input files
167 if(inputFiles.size() < 1 && md5Files.size() < 0)
168 {
169 cerr << "No input files or md5 flag given\nSee: " << argv[0] << " --help\n";
170 return 1;
171 }
172
173 __foreach(i, options.opt.digests)
174 digests.insert(*i);
175
176 if(options.opt.mindigests)
177 {
178 digests.insert("md5");
179 digests.insert("sha1");
180 }
181 if(options.opt.somedigests)
182 {
183 digests.insert("md5");
184 digests.insert("sha1");
185 digests.insert("ed2k");
186 }
187 allDigests = options.opt.alldigests;
188 readMirrors = options.opt.nomirrors == false;
189
190 //Simple boolean options
191 hashList = options.opt.hashlist;
192 if(hashList)
193 {
194 if(digests.size() < 1)
195 cerr << "metalink: Warning: No digests given, you probably forgot the -d option." << endl;
196 readMirrors = false;
197 }
198
199
200 //Read paths from stdin
201 MirrorList const mirrorList(cin, baseUrl, readMirrors);
202
203 //Generate records
204 std::vector< MetalinkFile > records;
205
206 _foreach(md5, md5Files)
207 {
208 //Open and read the file
209 MD5File file(*md5);
210
211 pair<string, string> r;
212
213 //Add the records for all paths
214 while(file.record(&r))
215 {
216 MetalinkFile record(r.second, &mirrorList);
217 record.addVerification("md5", r.first);
218 records.push_back(record);
219 }
220
221 }
222
223 //For each file, create a record from it and add it to records.
224 _foreach(it, inputFiles)
225 {
226 String filename(*it);
227 #ifdef OS_WIN
228 filename = filename.translated('\\', '/');
229 #endif
230 //Silently skip metalink files
231 if(filename.endsIn(Globals::metalinkExtension))
232 continue;
233
234 ifstream file(filename.c_str(), ios::binary);
235 if(!file.is_open())
236 {
237 cerr << "Unable to open '" << filename << "'\n";
238 continue;
239 }
240
241 _debugLevel2("\nstring------------------: " << targetFile.string()
242 << "\nnative_directory_string-: " << targetFile.native_directory_string()
243 << "\nnative_file_string------: " << targetFile.native_file_string()
244 << '\n');
245
246 MetalinkFile record(filename, &mirrorList);
247 //Add matching links
248 //foearch(links[filename], link) addpath basename(links)
249 cerr << "Hashing '" << filename << "' ... ";
250
251 HashList hl;
252 //Add needed hashes
253 //Known hashes: md4 md5
254 if(allDigests || digests.count("md4") > 0)
255 hl.push_back(new GCrypt(GCRY_MD_MD4));
256 if(allDigests || digests.count("md5") > 0)
257 hl.push_back(new GCrypt(GCRY_MD_MD5));
258
259 //Known hashes: sha1 sha256 sha384 sha512
260 if(allDigests || digests.count("sha1") > 0)
261 hl.push_back(new GCrypt(GCRY_MD_SHA1));
262 if(allDigests || digests.count("sha256") > 0)
263 hl.push_back(new GCrypt(GCRY_MD_SHA256));
264 if(allDigests || digests.count("sha384") > 0)
265 hl.push_back(new GCrypt(GCRY_MD_SHA384));
266 if(allDigests || digests.count("sha512") > 0)
267 hl.push_back(new GCrypt(GCRY_MD_SHA512));
268 //Known hashes: rmd160 tiger haval
269 if(allDigests || digests.count("rmd160") > 0)
270 hl.push_back(new GCrypt(GCRY_MD_RMD160));
271 if(allDigests || digests.count("tiger") > 0)
272 hl.push_back(new GCrypt(GCRY_MD_TIGER));
273
274 //Known hashes: crc32
275 if(allDigests || digests.count("crc32") > 0)
276 hl.push_back(new GCrypt(GCRY_MD_CRC32));
277
278 //Known hashes: ed2k
279 if(allDigests || digests.count("ed2k") > 0)
280 hl.push_back(new HashED2K());
281
282 //Known hashes: gnunet
283 if(allDigests || digests.count("gnunet") > 0)
284 hl.push_back(new HashGNUnet());
285
286 //Known hashes: pieces
287 if(allDigests || digests.count("sha1pieces") > 0)
288 {
289 unsigned long long filesize = file_size(filename);
290
291 //Small pieces => 256 kB in size.
292 unsigned long long psize = 256*1024;
293
294 //Larger file sizes typically have larger pieces.
295 // For example, a 4.37-GB file may have a piece size of 4 MB (4096 kB)
296 // To keep this while loop from going mad, we max at 16 loops
297 unsigned i = 0;
298 while(filesize / psize > 255 && ++i < 16)
299 psize += psize;
300
301 hl.push_back(new HashPieces(psize));
302 }
303
304 //Fill hashes
305 static unsigned const blockSize(10240);
306 char data[blockSize];
307 unsigned read(0);
308 char const * spinner = "\\|/\\|/-";
309 unsigned spini(0), spinii(0);
310 unsigned long long size(0);
311 cerr << " "; //Spinner space
312 while(true)
313 {
314 file.read(&data[0], blockSize);
315 read = file.gcount();
316
317 if(read == 0)
318 break;
319 size += read;
320
321 if(++spini % 150 == 0)
322 cerr << "\b" << spinner[++spinii%7];
323
324 //Update hashes
325 hl.update(&data[0], read);
326 }
327 cerr << "\b"; //Spinner removal
328
329 record.setSize(size);
330
331 //FINALIZE
332 hl.finalize();
333
334 //Add hashes and P2P paths
335 _foreach(hp, hl)
336 {
337 if(hashList)
338 {
339 String h((*hp)->name());
340 h.toUpper();
341 if(h.endsIn(String("PIECES")))
342 {
343 //Try down cast, exceptions are just wrong, so no try block here to keep them fatal.
344 HashPieces *pieces = dynamic_cast<HashPieces*>(*hp);
345 cout << h << "_SIZE(" << filename << ")= " << pieces->size() << "\n";
346 }
347 cout << h << "(" << filename << ")= " << (*hp)->value() << "\n";
348 continue;
349 }
350 //Only P2P link, not verify
351 if((*hp)->name() == "gnunet")
352 {
353 record.addPath("gnunet", "gnunet://ecrs/chk/" + (*hp)->value() + "." + record.size());
354 continue;
355 }
356
357 //Either make pieces a special name or add a new system for piece based hashing
358 record.addVerification((*hp)->xml());
359
360 //Add P2P specials
361 if((*hp)->name() == "sha1")
362 record.addPath("magnet", "magnet:?xt=urn:sha1:" + (*hp)->value() + "&dn=" + filename.translated(' ', '+'));
363 if((*hp)->name() == "ed2k")
364 record.addPath("ed2k", "ed2k://|file|" + filename.translated('|', '_') + "|" + record.size() + "|" + (*hp)->value() + "|/");
365 }
366
367 //Mirror list already added
368
369 records.push_back(record);
370 hl.destroyMembers();
371 cerr << "\n";
372 }//Foreach
373
374 if(!hashList)
375 cout << Metalink::from(records, headerFile, metalinkDescription);
376
377 return 0;
378 }
379 catch(std::string const &e)
380 {
381 cerr << "Exiting with ERRORS: " << e << endl;
382 }
383 catch(const char*e)
384 {
385 cerr << "Exiting with ERRORS: " << e << endl;
386 }
387 catch(const std::exception &e)
388 {
389 cerr << "Caught std::exception (" << typeid(e).name() << "): " << e.what() << endl;
390 }
391