1 // ==============================================================
2 // This file is part of Glest Shared Library (www.glest.org)
3 //
4 // Copyright (C) 2001-2008 Martiño Figueroa
5 //
6 // You can redistribute this code and/or modify it under
7 // the terms of the GNU General Public License as published
8 // by the Free Software Foundation; either version 2 of the
9 // License, or (at your option) any later version
10 // ==============================================================
11
12 #include "checksum.h"
13
14 #include <cassert>
15 #include <stdexcept>
16 #include <fcntl.h> // for open()
17
18 #ifdef WIN32
19 #include <io.h> // for open()
20 #endif
21
22 #include <sys/stat.h> // for open()
23
24 #include "util.h"
25 #include "platform_common.h"
26 #include "conversion.h"
27 #include "platform_util.h"
28 #include "leak_dumper.h"
29
30 using namespace std;
31 using namespace Shared::PlatformCommon;
32 using namespace Shared::Util;
33
34 namespace Shared{ namespace Util{
35
36 // =====================================================
37 // class Checksum
38 // =====================================================
39
40 Mutex Checksum::fileListCacheSynchAccessor;
41 std::map<string,uint32> Checksum::fileListCache;
42
43 unsigned int crc_table[256] =
44 {
45 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3,
46 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91,
47 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7,
48 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5,
49 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b,
50 0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59,
51 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f,
52 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d,
53 0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433,
54 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01,
55 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457,
56 0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65,
57 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb,
58 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9,
59 0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f,
60 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad,
61 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683,
62 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1,
63 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, 0x196c3671, 0x6e6b06e7,
64 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5,
65 0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b,
66 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79,
67 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f,
68 0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d,
69 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713,
70 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21,
71 0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777,
72 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45,
73 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db,
74 0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9,
75 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf,
76 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d
77 };
78
Checksum()79 Checksum::Checksum() {
80 sum= 0;
81 r= 55665;
82 c1= 52845;
83 c2= 22719;
84 }
85
addByte(const char value)86 uint32 Checksum::addByte(const char value) {
87 // int32 cipher= (value ^ (r >> 8));
88 //
89 // r= (cipher + r) * c1 + c2;
90 // sum += cipher;
91 //
92 // return cipher;
93
94 const unsigned char *rVal = reinterpret_cast<const unsigned char *>(&value);
95 sum = ~sum;
96 sum = (sum >> 8) ^ crc_table[*rVal ^ (sum & 0xff)];
97 sum = ~sum;
98
99 return sum;
100 }
101
addBytes(const void * _data,size_t _size)102 uint32 Checksum::addBytes(const void *_data, size_t _size) {
103 const unsigned char *rVal = reinterpret_cast<const unsigned char *>(_data);
104 sum = ~sum;
105 while (_size--) {
106 sum = (sum >> 8) ^ crc_table[*rVal++ ^ (sum & 0xff)];
107 }
108 sum = ~sum;
109
110 return sum;
111 }
112
113
addSum(uint32 value)114 void Checksum::addSum(uint32 value) {
115 sum += value;
116 }
117
addInt(const int32 & value)118 uint32 Checksum::addInt(const int32 &value) {
119 int8 byte = (value >> 0) & 0xFF;
120 addByte(byte);
121 byte = (value >> 8) & 0xFF;
122 addByte(byte);
123 byte = (value >> 16) & 0xFF;
124 addByte(byte);
125 byte = (value >> 24) & 0xFF;
126 addByte(byte);
127
128 return sum;
129 }
130
addUInt(const uint32 & value)131 uint32 Checksum::addUInt(const uint32 &value) {
132 int8 byte = (value >> 0) & 0xFF;
133 addByte(byte);
134 byte = (value >> 8) & 0xFF;
135 addByte(byte);
136 byte = (value >> 16) & 0xFF;
137 addByte(byte);
138 byte = (value >> 24) & 0xFF;
139 addByte(byte);
140
141 return sum;
142 }
143
addInt64(const int64 & value)144 uint32 Checksum::addInt64(const int64 &value) {
145 int8 byte = (value >> 0) & 0xFF;
146 addByte(byte);
147 byte = (value >> 8) & 0xFF;
148 addByte(byte);
149 byte = (value >> 16) & 0xFF;
150 addByte(byte);
151 byte = (value >> 24) & 0xFF;
152 addByte(byte);
153 byte = (value >> 32) & 0xFF;
154 addByte(byte);
155 byte = (value >> 40) & 0xFF;
156 addByte(byte);
157 byte = (value >> 48) & 0xFF;
158 addByte(byte);
159 byte = (value >> 56) & 0xFF;
160 addByte(byte);
161
162 return sum;
163 }
164
addString(const string & value)165 void Checksum::addString(const string &value) {
166 for(unsigned int i = 0; i < value.size(); ++i) {
167 addByte(value[i]);
168 }
169 }
170
addFile(const string & path)171 void Checksum::addFile(const string &path) {
172 if(path != "") {
173 fileList[path] = 0;
174 }
175 }
176
addFileToSum(const string & path)177 bool Checksum::addFileToSum(const string &path) {
178
179 // OLD SLOW FILE I/O
180 /*
181 FILE* file= fopen(path.c_str(), "rb");
182 if(file!=NULL){
183
184 addString(lastFile(path));
185
186 while(!feof(file)){
187 int8 byte= 0;
188
189 size_t readBytes = fread(&byte, 1, 1, file);
190 addByte(byte);
191 }
192 }
193 else
194 {
195 throw megaglest_runtime_error("Can not open file: " + path);
196 }
197 fclose(file);
198 */
199
200
201
202 /*
203 const double MAX_CRC_FILESIZE = 100000000;
204 int fd=0;
205 size_t bytes_read, bytes_expected = MAX_CRC_FILESIZE * sizeof(int8);
206 int8 *data=0;
207 const char *infile = path.c_str();
208
209 if ((fd = open(infile,O_RDONLY)) < 0)
210 throw megaglest_runtime_error("Can not open file: " + path);
211
212 if ((data = (int8 *)malloc(bytes_expected)) == NULL)
213 throw megaglest_runtime_error("malloc failed, Can not open file: " + path);
214
215 bytes_read = read(fd, data, bytes_expected);
216
217 //if (bytes_read != bytes_expected)
218 // throw megaglest_runtime_error("read failed, Can not open file: " + path);
219
220 for(int i = 0; i < bytes_read; i++) {
221 addByte(data[i]);
222 }
223 free(data);
224 */
225
226 bool fileExists = false;
227
228 /*
229 #ifdef WIN32
230 FILE* file= _wfopen(utf8_decode(path).c_str(), L"rb");
231 #else
232 FILE* file= fopen(path.c_str(), "rb");
233 #endif
234 if(file != NULL) {
235 fileExists = true;
236 addString(lastFile(path));
237
238 bool isXMLFile = (EndsWith(path, ".xml") == true);
239 bool inCommentTag=false;
240 char buf[4096]=""; // Should be large enough.
241 int bufSize = sizeof buf;
242 while(!feof(file)) {
243 //int8 byte= 0;
244
245 //size_t readBytes = fread(&byte, 1, 1, file);
246 memset(buf,0,bufSize);
247 if(fgets(buf, bufSize, file) != NULL) {
248 //addByte(byte);
249 for(int i = 0; i < bufSize && buf[i] != 0; i++) {
250 // Ignore Spaces in XML files as they are
251 // ONLY for formatting
252 if(isXMLFile == true) {
253 if(inCommentTag == true) {
254 if(buf[i] == '>' && i >= 3 && buf[i-1] == '-' && buf[i-2] == '-') {
255 inCommentTag = false;
256 //printf("TURNING OFF comment TAG, i = %d [%c]",i,buf[i]);
257 }
258 else {
259 //printf("SKIPPING XML comment character, i = %d [%c]",i,buf[i]);
260 }
261 continue;
262 }
263 //else if(buf[i] == '-' && i >= 4 && buf[i-1] == '-' && buf[i-2] == '!' && buf[i-3] == '<') {
264 else if(buf[i] == '<' && i+4 < bufSize && buf[i+1] == '!' && buf[i+2] == '-' && buf[i+3] == '-') {
265 inCommentTag = true;
266 //printf("TURNING ON comment TAG, i = %d [%c]",i,buf[i]);
267 continue;
268 }
269 else if(buf[i] == ' ' || buf[i] == '\t' || buf[i] == '\n' || buf[i] == '\r') {
270 //printf("SKIPPING special character, i = %d [%c]",i,buf[i]);
271 continue;
272 }
273 }
274 addByte(buf[i]);
275 }
276 }
277 }
278 }
279 else {
280 throw megaglest_runtime_error("Can not open file: " + path);
281 }
282 fclose(file);
283 */
284
285 #if defined(WIN32) && !defined(__MINGW32__)
286 wstring wstr = utf8_decode(path);
287 FILE *fp = _wfopen(wstr.c_str(), L"rb");
288 ifstream ifs(fp);
289 #else
290 ifstream ifs(path.c_str());
291 #endif
292
293 if (ifs) {
294 fileExists = true;
295 addString(lastFile(path));
296
297 bool isXMLFile = (EndsWith(path, ".xml") == true);
298
299 // Determine the file length
300 ifs.seekg(0, ios::end);
301 std::streamoff size=ifs.tellg();
302 ifs.seekg(0, ios::beg);
303
304 unsigned int bufSize = (unsigned int)size / sizeof(char);
305 // Create a vector to store the data
306 std::vector<char> buf(bufSize);
307 // Load the data
308 ifs.read((char*)&buf[0], buf.size());
309
310 if(SystemFlags::getSystemSettingType(SystemFlags::debugSystem).enabled) SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d] buf.size() = %d, path [%s], isXMLFile = %d\n",__FILE__,__FUNCTION__,__LINE__,buf.size(), path.c_str(),isXMLFile);
311
312 if(isXMLFile == true) {
313 bool inCommentTag=false;
314 for(std::size_t i = 0; i < buf.size(); ++i) {
315 // Ignore Spaces in XML files as they are
316 // ONLY for formatting
317 //if(isXMLFile == true) {
318 if(inCommentTag == true) {
319 if(buf[i] == '>' && i >= 3 && buf[i-1] == '-' && buf[i-2] == '-') {
320 inCommentTag = false;
321 //printf("TURNING OFF comment TAG, i = %d [%c]",i,buf[i]);
322 if(SystemFlags::getSystemSettingType(SystemFlags::debugSystem).enabled) SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d] i = %d\n",__FILE__,__FUNCTION__,__LINE__,i);
323 }
324 else {
325 //printf("SKIPPING XML comment character, i = %d [%c]",i,buf[i]);
326 if(SystemFlags::getSystemSettingType(SystemFlags::debugSystem).enabled) SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d] i = %d\n",__FILE__,__FUNCTION__,__LINE__,i);
327 }
328 continue;
329 }
330 //else if(buf[i] == '-' && i >= 4 && buf[i-1] == '-' && buf[i-2] == '!' && buf[i-3] == '<') {
331 else if(buf[i] == '<' && i+4 < bufSize && buf[i+1] == '!' && buf[i+2] == '-' && buf[i+3] == '-') {
332 inCommentTag = true;
333 //printf("TURNING ON comment TAG, i = %d [%c]",i,buf[i]);
334 if(SystemFlags::getSystemSettingType(SystemFlags::debugSystem).enabled) SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d] i = %d\n",__FILE__,__FUNCTION__,__LINE__,i);
335 continue;
336 }
337 else if(buf[i] == ' ' || buf[i] == '\t' || buf[i] == '\n' || buf[i] == '\r') {
338 //printf("SKIPPING special character, i = %d [%c]",i,buf[i]);
339 if(SystemFlags::getSystemSettingType(SystemFlags::debugSystem).enabled) SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d] i = %d\n",__FILE__,__FUNCTION__,__LINE__,i);
340 continue;
341 }
342 //}
343 uint32 cipher = addByte(buf[i]);
344
345 if(SystemFlags::getSystemSettingType(SystemFlags::debugSystem).enabled) SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d] %d / %d, cipher = %u\n",__FILE__,__FUNCTION__,__LINE__,i,buf.size(), cipher);
346 }
347 }
348 else {
349 uint32 cipher = addBytes(&buf[0],buf.size());
350 if(SystemFlags::getSystemSettingType(SystemFlags::debugSystem).enabled) SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d] %d, cipher = %u\n",__FILE__,__FUNCTION__,__LINE__,buf.size(), cipher);
351 }
352
353 // Close the file
354 ifs.close();
355 }
356 #if defined(WIN32) && !defined(__MINGW32__)
357 if(fp) {
358 fclose(fp);
359 }
360 #endif
361
362 return fileExists;
363 }
364
getSum()365 uint32 Checksum::getSum() {
366 //printf("Getting checksum for files [%d]\n",fileList.size());
367 if(fileList.size() > 0) {
368 if(SystemFlags::getSystemSettingType(SystemFlags::debugSystem).enabled) SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d] fileList.size() = %d\n",__FILE__,__FUNCTION__,__LINE__,fileList.size());
369
370 Checksum newResult;
371
372 {
373
374 for(std::map<string,uint32>::iterator iterMap = fileList.begin();
375 iterMap != fileList.end(); ++iterMap) {
376
377 MutexSafeWrapper safeMutexSocketDestructorFlag(&Checksum::fileListCacheSynchAccessor,string(__FILE__) + "_" + intToStr(__LINE__));
378 if(Checksum::fileListCache.find(iterMap->first) == Checksum::fileListCache.end()) {
379 Checksum fileResult;
380 //bool fileAddedOk = fileResult.addFileToSum(iterMap->first);
381 fileResult.addFileToSum(iterMap->first);
382 Checksum::fileListCache[iterMap->first] = fileResult.getSum();
383 //printf("fileAddedOk = %d for file [%s] CRC [%d]\n",fileAddedOk,iterMap->first.c_str(),Checksum::fileListCache[iterMap->first]);
384 }
385 else {
386 //printf("Getting checksum from CACHE for file [%s] CRC [%d]\n",iterMap->first.c_str(),Checksum::fileListCache[iterMap->first]);
387 }
388 newResult.addSum(Checksum::fileListCache[iterMap->first]);
389 }
390 }
391
392 if(SystemFlags::getSystemSettingType(SystemFlags::debugSystem).enabled) SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d] fileList.size() = %d\n",__FILE__,__FUNCTION__,__LINE__,fileList.size());
393
394 return newResult.getSum();
395 }
396 return sum;
397 }
398
getFinalFileListSum()399 uint32 Checksum::getFinalFileListSum() {
400 sum = 0;
401 return getSum();
402 }
403
404 //uint32 Checksum::getFinalFileListSum() {
405 // if(SystemFlags::getSystemSettingType(SystemFlags::debugSystem).enabled) SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d] sum = %u\n",extractFileFromDirectoryPath(__FILE__).c_str(),__FUNCTION__,__LINE__,sum);
406 //
407 // sum = 0;
408 // //return getSum();
409 // sum = getSum();
410 //
411 // if(SystemFlags::getSystemSettingType(SystemFlags::debugSystem).enabled) SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d] sum = %u\n",extractFileFromDirectoryPath(__FILE__).c_str(),__FUNCTION__,__LINE__,sum);
412 //
413 // return sum;
414 //}
415
getFileCount()416 uint32 Checksum::getFileCount() {
417 return (uint32)fileList.size();
418 }
419
removeFileFromCache(const string file)420 void Checksum::removeFileFromCache(const string file) {
421 MutexSafeWrapper safeMutexSocketDestructorFlag(&Checksum::fileListCacheSynchAccessor,string(__FILE__) + "_" + intToStr(__LINE__));
422 if(Checksum::fileListCache.find(file) != Checksum::fileListCache.end()) {
423 Checksum::fileListCache.erase(file);
424 }
425 }
426
clearFileCache()427 void Checksum::clearFileCache() {
428 MutexSafeWrapper safeMutexSocketDestructorFlag(&Checksum::fileListCacheSynchAccessor,string(__FILE__) + "_" + intToStr(__LINE__));
429 Checksum::fileListCache.clear();
430 }
431
432 }}//end namespace
433