1 // miniz_tester.cpp
2 // Note: This module is not intended to make a good example, or be used for anything other than testing.
3 // It's something quick I put together last year to help regression test miniz/tinfl under Linux/Win32/Mac. It's derived from LZHAM's test module.
4 #ifdef _MSC_VER
5 #pragma warning (disable:4127) // warning C4127: conditional expression is constant
6 #endif
7
8 #if defined(__GNUC__)
9 // Ensure we get the 64-bit variants of the CRT's file I/O calls
10 #ifndef _FILE_OFFSET_BITS
11 #define _FILE_OFFSET_BITS 64
12 #endif
13 #ifndef _LARGEFILE64_SOURCE
14 #define _LARGEFILE64_SOURCE 1
15 #endif
16 #endif
17
18 #include "miniz.h"
19 #include "miniz_zip.h"
20
21 #include <stdlib.h>
22 #include <stdio.h>
23 #include <assert.h>
24 #include <memory.h>
25 #include <stdarg.h>
26 #include <malloc.h>
27 #include <vector>
28 #include <string>
29 #include <limits.h>
30 #include <sys/stat.h>
31
32 #include "timer.h"
33
34 #define my_max(a,b) (((a) > (b)) ? (a) : (b))
35 #define my_min(a,b) (((a) < (b)) ? (a) : (b))
36
37 typedef unsigned char uint8;
38 typedef unsigned short uint16;
39 typedef unsigned int uint;
40
41 #define TDEFL_PRINT_OUTPUT_PROGRESS
42
43 #if defined(WIN32)
44 #define WIN32_LEAN_AND_MEAN
45 #include <windows.h>
46 #define FILE_STAT_STRUCT _stat
47 #define FILE_STAT _stat
48 #else
49 #include <unistd.h>
50 #define Sleep(ms) usleep(ms*1000)
51 #define _aligned_malloc(size, alignment) memalign(alignment, size)
52 #define _aligned_free free
53 #define fopen fopen64
54 #define _fseeki64 fseeko64
55 #define _ftelli64 ftello64
56 #define _stricmp strcasecmp
57 #define FILE_STAT_STRUCT stat64
58 #define FILE_STAT stat64
59 #endif
60
61 #ifdef WIN32
62 #define QUAD_INT_FMT "%I64u"
63 #else
64 #define QUAD_INT_FMT "%llu"
65 #endif
66
67 #ifdef _DEBUG
68 const bool g_is_debug = true;
69 #else
70 const bool g_is_debug = false;
71 #endif
72
73 typedef unsigned char uint8;
74 typedef unsigned int uint;
75 typedef unsigned int uint32;
76 typedef unsigned long long uint64;
77 typedef long long int64;
78
79 #define TDEFLTEST_COMP_INPUT_BUFFER_SIZE 1024*1024*2
80 #define TDEFLTEST_COMP_OUTPUT_BUFFER_SIZE 1024*1024*2
81 #define TDEFLTEST_DECOMP_INPUT_BUFFER_SIZE 1024*1024*2
82
83 static float s_max_small_comp_ratio, s_max_large_comp_ratio;
84
85 struct comp_options
86 {
comp_optionscomp_options87 comp_options() :
88 m_level(7),
89 m_unbuffered_decompression(false),
90 m_verify_compressed_data(false),
91 m_randomize_params(false),
92 m_randomize_buffer_sizes(false),
93 m_z_strat(Z_DEFAULT_STRATEGY),
94 m_random_z_flushing(false),
95 m_write_zlib_header(true),
96 m_archive_test(false),
97 m_write_archives(false)
98 {
99 }
100
printcomp_options101 void print()
102 {
103 printf("Level: %u\n", m_level);
104 printf("Write zlib header: %u\n", (uint)m_write_zlib_header);
105 printf("Unbuffered decompression: %u\n", (uint)m_unbuffered_decompression);
106 printf("Verify compressed data: %u\n", (uint)m_verify_compressed_data);
107 printf("Randomize parameters: %u\n", m_randomize_params);
108 printf("Randomize buffer sizes: %u\n", m_randomize_buffer_sizes);
109 printf("Deflate strategy: %u\n", m_z_strat);
110 printf("Random Z stream flushing: %u\n", m_random_z_flushing);
111 printf("Archive test: %u\n", m_archive_test);
112 printf("Write archives: %u\n", m_write_archives);
113 }
114
115 uint m_level;
116 bool m_unbuffered_decompression;
117 bool m_verify_compressed_data;
118 bool m_randomize_params;
119 bool m_randomize_buffer_sizes;
120 uint m_z_strat;
121 bool m_random_z_flushing;
122 bool m_write_zlib_header;
123 bool m_archive_test;
124 bool m_write_archives;
125 };
126
127 #define RND_SHR3(x) (x ^= (x << 17), x ^= (x >> 13), x ^= (x << 5))
128
129 #if 0
130 static void random_fill(uint8 *pDst, size_t len, uint32 x)
131 {
132 x ^= (x << 16);
133 if (!x) x++;
134
135 while (len)
136 {
137 RND_SHR3(x); uint64 l0 = x & 0xFFF;
138 RND_SHR3(x); uint64 l1 = x & 0xFFF;
139 RND_SHR3(x); uint64 l2 = x & 0xFFF;
140 RND_SHR3(x); uint c = x;
141
142 uint l = (uint)(((l0*l1*l2)/(16769025ULL) * 32) / 4095);
143 l = (uint)my_max(1,my_min(l, len));
144 len -= l;
145
146 while (l--)
147 {
148 *pDst++ = (uint8)c;
149 }
150
151 if (((int)x < 0) && len)
152 {
153 *pDst++ = 0;
154 len--;
155 }
156 }
157 }
158 #endif
159
print_usage()160 static void print_usage()
161 {
162 printf("Usage: [options] [mode] inpath/infile [outfile]\n");
163 printf("\n");
164 printf("Modes:\n");
165 printf("c - Compress \"infile\" to \"outfile\"\n");
166 printf("d - Decompress \"infile\" to \"outfile\"\n");
167 printf("a - Recursively compress all files under \"inpath\"\n");
168 printf("r - Archive decompression test\n");
169 printf("\n");
170 printf("Options:\n");
171 printf("-m[0-10] - Compression level: 0=fastest (Huffman only), 9=best (10=uber)\n");
172 printf("-u - Use unbuffered decompression on files that can fit into memory.\n");
173 printf(" Unbuffered decompression is faster, but may have more I/O overhead.\n");
174 printf("-v - Immediately decompress compressed file after compression for verification.\n");
175 printf("-z - Do not write zlib header\n");
176 printf("-r - Randomize parameters during recursive testing\n");
177 printf("-b - Randomize input/output buffer sizes\n");
178 printf("-h - Use random z_flushing\n");
179 printf("-x# - Set rand() seed to value\n");
180 printf("-t# - Set z_strategy to value [0-4]\n");
181 printf("-a - Create single-file archives instead of files during testing\n");
182 printf("-w - Test archive cloning\n");
183 }
184
print_error(const char * pMsg,...)185 static void print_error(const char *pMsg, ...)
186 {
187 char buf[1024];
188
189 va_list args;
190 va_start(args, pMsg);
191 vsnprintf(buf, sizeof(buf), pMsg, args);
192 va_end(args);
193
194 buf[sizeof(buf) - 1] = '\0';
195
196 fprintf(stderr, "Error: %s", buf);
197 }
198
open_file_with_retries(const char * pFilename,const char * pMode)199 static FILE* open_file_with_retries(const char *pFilename, const char* pMode)
200 {
201 const uint cNumRetries = 8;
202 for (uint i = 0; i < cNumRetries; i++)
203 {
204 FILE* pFile = fopen(pFilename, pMode);
205 if (pFile)
206 return pFile;
207 Sleep(250);
208 }
209 return NULL;
210 }
211
ensure_file_exists_and_is_readable(const char * pFilename)212 static bool ensure_file_exists_and_is_readable(const char *pFilename)
213 {
214 FILE *p = fopen(pFilename, "rb");
215 if (!p)
216 return false;
217
218 _fseeki64(p, 0, SEEK_END);
219 uint64 src_file_size = _ftelli64(p);
220 _fseeki64(p, 0, SEEK_SET);
221
222 if (src_file_size)
223 {
224 char buf[1];
225 if (fread(buf, 1, 1, p) != 1)
226 {
227 fclose(p);
228 return false;
229 }
230 }
231 fclose(p);
232 return true;
233 }
234
ensure_file_is_writable(const char * pFilename)235 static bool ensure_file_is_writable(const char *pFilename)
236 {
237 const int cNumRetries = 8;
238 for (int i = 0; i < cNumRetries; i++)
239 {
240 FILE *pFile = fopen(pFilename, "wb");
241 if (pFile)
242 {
243 fclose(pFile);
244 return true;
245 }
246 Sleep(250);
247 }
248 return false;
249 }
250
simple_test1(const comp_options & options)251 static int simple_test1(const comp_options &options)
252 {
253 (void)options;
254
255 size_t cmp_len = 0;
256
257 const char *p = "This is a test.This is a test.This is a test.1234567This is a test.This is a test.123456";
258 size_t uncomp_len = strlen(p);
259
260 void *pComp_data = tdefl_compress_mem_to_heap(p, uncomp_len, &cmp_len, TDEFL_WRITE_ZLIB_HEADER);
261 if (!pComp_data)
262 {
263 free(pComp_data);
264 print_error("Compression test failed!\n");
265 return EXIT_FAILURE;
266 }
267
268 printf("Uncompressed size: %u\nCompressed size: %u\n", (uint)uncomp_len, (uint)cmp_len);
269
270 size_t decomp_len = 0;
271 void *pDecomp_data = tinfl_decompress_mem_to_heap(pComp_data, cmp_len, &decomp_len, TINFL_FLAG_PARSE_ZLIB_HEADER);
272
273 if ((!pDecomp_data) || (decomp_len != uncomp_len) || (memcmp(pDecomp_data, p, uncomp_len)))
274 {
275 free(pComp_data);
276 free(pDecomp_data);
277 print_error("Compression test failed!\n");
278 return EXIT_FAILURE;
279 }
280
281 printf("Low-level API compression test succeeded.\n");
282
283 free(pComp_data);
284 free(pDecomp_data);
285
286 return EXIT_SUCCESS;
287 }
288
simple_test2(const comp_options & options)289 static int simple_test2(const comp_options &options)
290 {
291 (void)options;
292
293 uint8 cmp_buf[1024], decomp_buf[1024];
294 uLong cmp_len = sizeof(cmp_buf);
295
296 const char *p = "This is a test.This is a test.This is a test.1234567This is a test.This is a test.123456";
297 uLong uncomp_len = (uLong)strlen(p);
298
299 int status = compress(cmp_buf, &cmp_len, (const uint8*)p, uncomp_len);
300 if (status != Z_OK)
301 {
302 print_error("Compression test failed!\n");
303 return EXIT_FAILURE;
304 }
305
306 printf("Uncompressed size: %u\nCompressed size: %u\n", (uint)uncomp_len, (uint)cmp_len);
307
308 if (cmp_len > compressBound(uncomp_len))
309 {
310 print_error("compressBound() returned bogus result\n");
311 return EXIT_FAILURE;
312 }
313
314 uLong decomp_len = sizeof(decomp_buf);
315 status = uncompress(decomp_buf, &decomp_len, cmp_buf, cmp_len);;
316
317 if ((status != Z_OK) || (decomp_len != uncomp_len) || (memcmp(decomp_buf, p, uncomp_len)))
318 {
319 print_error("Compression test failed!\n");
320 return EXIT_FAILURE;
321 }
322
323 printf("zlib API compression test succeeded.\n");
324
325 return EXIT_SUCCESS;
326 }
327
compress_file_zlib(const char * pSrc_filename,const char * pDst_filename,const comp_options & options)328 static bool compress_file_zlib(const char* pSrc_filename, const char *pDst_filename, const comp_options &options)
329 {
330 printf("Testing: Streaming zlib compression\n");
331
332 FILE *pInFile = fopen(pSrc_filename, "rb");
333 if (!pInFile)
334 {
335 print_error("Unable to read file: %s\n", pSrc_filename);
336 return false;
337 }
338
339 FILE *pOutFile = fopen(pDst_filename, "wb");
340 if (!pOutFile)
341 {
342 print_error("Unable to create file: %s\n", pDst_filename);
343 return false;
344 }
345
346 _fseeki64(pInFile, 0, SEEK_END);
347 uint64 src_file_size = _ftelli64(pInFile);
348 _fseeki64(pInFile, 0, SEEK_SET);
349
350 fputc('D', pOutFile); fputc('E', pOutFile); fputc('F', pOutFile); fputc('0', pOutFile);
351 fputc(options.m_write_zlib_header, pOutFile);
352
353 for (uint i = 0; i < 8; i++)
354 fputc(static_cast<int>((src_file_size >> (i * 8)) & 0xFF), pOutFile);
355
356 uint cInBufSize = TDEFLTEST_COMP_INPUT_BUFFER_SIZE;
357 uint cOutBufSize = TDEFLTEST_COMP_OUTPUT_BUFFER_SIZE;
358 if (options.m_randomize_buffer_sizes)
359 {
360 cInBufSize = 1 + (rand() % 4096);
361 cOutBufSize = 1 + (rand() % 4096);
362 }
363 printf("Input buffer size: %u, Output buffer size: %u\n", cInBufSize, cOutBufSize);
364
365 uint8 *in_file_buf = static_cast<uint8*>(_aligned_malloc(cInBufSize, 16));
366 uint8 *out_file_buf = static_cast<uint8*>(_aligned_malloc(cOutBufSize, 16));
367 if ((!in_file_buf) || (!out_file_buf))
368 {
369 print_error("Out of memory!\n");
370 _aligned_free(in_file_buf);
371 _aligned_free(out_file_buf);
372 fclose(pInFile);
373 fclose(pOutFile);
374 return false;
375 }
376
377 uint64 src_bytes_left = src_file_size;
378
379 uint in_file_buf_size = 0;
380 uint in_file_buf_ofs = 0;
381
382 uint64 total_output_bytes = 0;
383
384 timer_ticks start_time = timer::get_ticks();
385
386 z_stream zstream;
387 memset(&zstream, 0, sizeof(zstream));
388
389 timer_ticks init_start_time = timer::get_ticks();
390 int status = deflateInit2(&zstream, options.m_level, Z_DEFLATED, options.m_write_zlib_header ? Z_DEFAULT_WINDOW_BITS : -Z_DEFAULT_WINDOW_BITS, 9, options.m_z_strat);
391 timer_ticks total_init_time = timer::get_ticks() - init_start_time;
392
393 if (status != Z_OK)
394 {
395 print_error("Failed initializing compressor!\n");
396 _aligned_free(in_file_buf);
397 _aligned_free(out_file_buf);
398 fclose(pInFile);
399 fclose(pOutFile);
400 return false;
401 }
402
403 printf("deflateInit2() took %3.3fms\n", timer::ticks_to_secs(total_init_time)*1000.0f);
404
405 uint32 x = my_max(1, (uint32)(src_file_size ^ (src_file_size >> 32)));
406
407 for ( ; ; )
408 {
409 if (src_file_size)
410 {
411 double total_elapsed_time = timer::ticks_to_secs(timer::get_ticks() - start_time);
412 double total_bytes_processed = static_cast<double>(src_file_size - src_bytes_left);
413 double comp_rate = (total_elapsed_time > 0.0f) ? total_bytes_processed / total_elapsed_time : 0.0f;
414
415 #ifdef TDEFL_PRINT_OUTPUT_PROGRESS
416 for (int i = 0; i < 15; i++)
417 printf("\b\b\b\b");
418 printf("Progress: %3.1f%%, Bytes Remaining: %3.1fMB, %3.3fMB/sec", (1.0f - (static_cast<float>(src_bytes_left) / src_file_size)) * 100.0f, src_bytes_left / 1048576.0f, comp_rate / (1024.0f * 1024.0f));
419 printf(" \b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b");
420 #endif
421 }
422
423 if (in_file_buf_ofs == in_file_buf_size)
424 {
425 in_file_buf_size = static_cast<uint>(my_min(cInBufSize, src_bytes_left));
426
427 if (fread(in_file_buf, 1, in_file_buf_size, pInFile) != in_file_buf_size)
428 {
429 printf("\n");
430 print_error("Failure reading from source file!\n");
431 _aligned_free(in_file_buf);
432 _aligned_free(out_file_buf);
433 fclose(pInFile);
434 fclose(pOutFile);
435 deflateEnd(&zstream);
436 return false;
437 }
438
439 src_bytes_left -= in_file_buf_size;
440
441 in_file_buf_ofs = 0;
442 }
443
444 zstream.next_in = &in_file_buf[in_file_buf_ofs];
445 zstream.avail_in = in_file_buf_size - in_file_buf_ofs;
446 zstream.next_out = out_file_buf;
447 zstream.avail_out = cOutBufSize;
448
449 int flush = !src_bytes_left ? Z_FINISH : Z_NO_FLUSH;
450 if ((flush == Z_NO_FLUSH) && (options.m_random_z_flushing))
451 {
452 RND_SHR3(x);
453 if ((x & 15) == 0)
454 {
455 RND_SHR3(x);
456 flush = (x & 31) ? Z_SYNC_FLUSH : Z_FULL_FLUSH;
457 }
458 }
459 status = deflate(&zstream, flush);
460
461 uint num_in_bytes = (in_file_buf_size - in_file_buf_ofs) - zstream.avail_in;
462 uint num_out_bytes = cOutBufSize - zstream.avail_out;
463 if (num_in_bytes)
464 {
465 in_file_buf_ofs += (uint)num_in_bytes;
466 assert(in_file_buf_ofs <= in_file_buf_size);
467 }
468
469 if (num_out_bytes)
470 {
471 if (fwrite(out_file_buf, 1, static_cast<uint>(num_out_bytes), pOutFile) != num_out_bytes)
472 {
473 printf("\n");
474 print_error("Failure writing to destination file!\n");
475 _aligned_free(in_file_buf);
476 _aligned_free(out_file_buf);
477 fclose(pInFile);
478 fclose(pOutFile);
479 deflateEnd(&zstream);
480 return false;
481 }
482
483 total_output_bytes += num_out_bytes;
484 }
485
486 if (status != Z_OK)
487 break;
488 }
489
490 #ifdef TDEFL_PRINT_OUTPUT_PROGRESS
491 for (int i = 0; i < 15; i++)
492 {
493 printf("\b\b\b\b \b\b\b\b");
494 }
495 #endif
496
497 src_bytes_left += (in_file_buf_size - in_file_buf_ofs);
498
499 uint32 adler32 = zstream.adler;
500 deflateEnd(&zstream);
501
502 timer_ticks end_time = timer::get_ticks();
503 double total_time = timer::ticks_to_secs(my_max(1, end_time - start_time));
504
505 uint64 cmp_file_size = _ftelli64(pOutFile);
506
507 _aligned_free(in_file_buf);
508 in_file_buf = NULL;
509 _aligned_free(out_file_buf);
510 out_file_buf = NULL;
511
512 fclose(pInFile);
513 pInFile = NULL;
514 fclose(pOutFile);
515 pOutFile = NULL;
516
517 if (status != Z_STREAM_END)
518 {
519 print_error("Compression failed with status %i\n", status);
520 return false;
521 }
522
523 if (src_bytes_left)
524 {
525 print_error("Compressor failed to consume entire input file!\n");
526 return false;
527 }
528
529 printf("Success\n");
530 printf("Input file size: " QUAD_INT_FMT ", Compressed file size: " QUAD_INT_FMT ", Ratio: %3.2f%%\n", src_file_size, cmp_file_size, src_file_size ? ((1.0f - (static_cast<float>(cmp_file_size) / src_file_size)) * 100.0f) : 0.0f);
531 printf("Compression time: %3.6f\nConsumption rate: %9.1f bytes/sec, Emission rate: %9.1f bytes/sec\n", total_time, src_file_size / total_time, cmp_file_size / total_time);
532 printf("Input file adler32: 0x%08X\n", adler32);
533 if (src_file_size)
534 {
535 if (src_file_size >= 256)
536 s_max_large_comp_ratio = my_max(s_max_large_comp_ratio, cmp_file_size / (float)src_file_size);
537 else
538 s_max_small_comp_ratio = my_max(s_max_small_comp_ratio, cmp_file_size / (float)src_file_size);
539 }
540 //printf("Max small comp ratio: %f, Max large comp ratio: %f\n", s_max_small_comp_ratio, s_max_large_comp_ratio);
541
542 return true;
543 }
544
decompress_file_zlib(const char * pSrc_filename,const char * pDst_filename,comp_options options)545 static bool decompress_file_zlib(const char* pSrc_filename, const char *pDst_filename, comp_options options)
546 {
547 FILE *pInFile = fopen(pSrc_filename, "rb");
548 if (!pInFile)
549 {
550 print_error("Unable to read file: %s\n", pSrc_filename);
551 return false;
552 }
553
554 _fseeki64(pInFile, 0, SEEK_END);
555 uint64 src_file_size = _ftelli64(pInFile);
556 _fseeki64(pInFile, 0, SEEK_SET);
557 if (src_file_size < (5+9))
558 {
559 print_error("Compressed file is too small!\n");
560 fclose(pInFile);
561 return false;
562 }
563
564 int h0 = fgetc(pInFile);
565 int h1 = fgetc(pInFile);
566 int h2 = fgetc(pInFile);
567 int h3 = fgetc(pInFile);
568 int zlib_header = fgetc(pInFile);
569 if ((h0 != 'D') | (h1 != 'E') || (h2 != 'F') || (h3 != '0'))
570 {
571 print_error("Unrecognized/invalid header in file: %s\n", pSrc_filename);
572 fclose(pInFile);
573 return false;
574 }
575
576 FILE *pOutFile = fopen(pDst_filename, "wb");
577 if (!pOutFile)
578 {
579 print_error("Unable to create file: %s\n", pDst_filename);
580 fclose(pInFile);
581 return false;
582 }
583
584 uint64 orig_file_size = 0;
585 for (uint i = 0; i < 8; i++)
586 orig_file_size |= (static_cast<uint64>(fgetc(pInFile)) << (i * 8));
587
588 int total_header_bytes = ftell(pInFile);
589
590 // Avoid running out of memory on large files when using unbuffered decompression.
591 if ((options.m_unbuffered_decompression) && (orig_file_size > 768*1024*1024))
592 {
593 printf("Output file is too large for unbuffered decompression - switching to streaming decompression.\n");
594 options.m_unbuffered_decompression = false;
595 }
596
597 if (options.m_unbuffered_decompression)
598 printf("Testing: Unbuffered decompression\n");
599 else
600 printf("Testing: Streaming decompression\n");
601
602 uint cInBufSize = options.m_unbuffered_decompression ? static_cast<uint>(src_file_size) : TDEFLTEST_DECOMP_INPUT_BUFFER_SIZE;
603 uint out_buf_size = options.m_unbuffered_decompression ? static_cast<uint>(orig_file_size) : TINFL_LZ_DICT_SIZE;
604
605 if ((options.m_randomize_buffer_sizes) && (!options.m_unbuffered_decompression))
606 {
607 cInBufSize = 1 + (rand() % 4096);
608 }
609
610 printf("Input buffer size: %u, Output buffer size: %u\n", cInBufSize, out_buf_size);
611
612 uint8 *in_file_buf = static_cast<uint8*>(_aligned_malloc(cInBufSize, 16));
613 uint8 *out_file_buf = static_cast<uint8*>(_aligned_malloc(out_buf_size, 16));
614 if ((!in_file_buf) || (!out_file_buf))
615 {
616 print_error("Failed allocating output buffer!\n");
617 _aligned_free(in_file_buf);
618 fclose(pInFile);
619 fclose(pOutFile);
620 return false;
621 }
622
623 uint64 src_bytes_left = src_file_size - total_header_bytes;
624 uint64 dst_bytes_left = orig_file_size;
625
626 uint in_file_buf_size = 0;
627 uint in_file_buf_ofs = 0;
628 uint out_file_buf_ofs = 0;
629
630 timer_ticks start_time = timer::get_ticks();
631 double decomp_only_time = 0;
632
633 z_stream zstream;
634 memset(&zstream, 0, sizeof(zstream));
635
636 timer_ticks init_start_time = timer::get_ticks();
637 int status = zlib_header ? inflateInit(&zstream) : inflateInit2(&zstream, -Z_DEFAULT_WINDOW_BITS);
638 timer_ticks total_init_time = timer::get_ticks() - init_start_time;
639 if (status != Z_OK)
640 {
641 print_error("Failed initializing decompressor!\n");
642 _aligned_free(in_file_buf);
643 _aligned_free(out_file_buf);
644 fclose(pInFile);
645 fclose(pOutFile);
646 return false;
647 }
648
649 printf("inflateInit() took %3.3fms\n", timer::ticks_to_secs(total_init_time)*1000.0f);
650
651 for ( ; ; )
652 {
653 if (in_file_buf_ofs == in_file_buf_size)
654 {
655 in_file_buf_size = static_cast<uint>(my_min(cInBufSize, src_bytes_left));
656
657 if (fread(in_file_buf, 1, in_file_buf_size, pInFile) != in_file_buf_size)
658 {
659 print_error("Failure reading from source file!\n");
660 _aligned_free(in_file_buf);
661 _aligned_free(out_file_buf);
662 deflateEnd(&zstream);
663 fclose(pInFile);
664 fclose(pOutFile);
665 return false;
666 }
667
668 src_bytes_left -= in_file_buf_size;
669
670 in_file_buf_ofs = 0;
671 }
672
673 uint num_in_bytes = (in_file_buf_size - in_file_buf_ofs);
674 uint num_out_bytes = (out_buf_size - out_file_buf_ofs);
675 zstream.next_in = in_file_buf + in_file_buf_ofs;
676 zstream.avail_in = num_in_bytes;
677 zstream.next_out = out_file_buf + out_file_buf_ofs;
678 zstream.avail_out = num_out_bytes;
679
680 {
681 timer decomp_only_timer;
682 decomp_only_timer.start();
683 status = inflate(&zstream, options.m_unbuffered_decompression ? Z_FINISH : Z_SYNC_FLUSH);
684 decomp_only_time += decomp_only_timer.get_elapsed_secs();
685 }
686 num_in_bytes -= zstream.avail_in;
687 num_out_bytes -= zstream.avail_out;
688
689 if (num_in_bytes)
690 {
691 in_file_buf_ofs += (uint)num_in_bytes;
692 assert(in_file_buf_ofs <= in_file_buf_size);
693 }
694
695 out_file_buf_ofs += (uint)num_out_bytes;
696
697 if ((out_file_buf_ofs == out_buf_size) || (status == Z_STREAM_END))
698 {
699 if (fwrite(out_file_buf, 1, static_cast<uint>(out_file_buf_ofs), pOutFile) != out_file_buf_ofs)
700 {
701 print_error("Failure writing to destination file!\n");
702 _aligned_free(in_file_buf);
703 _aligned_free(out_file_buf);
704 inflateEnd(&zstream);
705 fclose(pInFile);
706 fclose(pOutFile);
707 return false;
708 }
709 out_file_buf_ofs = 0;
710 }
711
712 if (num_out_bytes > dst_bytes_left)
713 {
714 print_error("Decompressor wrote too many bytes to destination file!\n");
715 _aligned_free(in_file_buf);
716 _aligned_free(out_file_buf);
717 inflateEnd(&zstream);
718 fclose(pInFile);
719 fclose(pOutFile);
720 return false;
721 }
722 dst_bytes_left -= num_out_bytes;
723
724 if (status != Z_OK)
725 break;
726 }
727 _aligned_free(in_file_buf);
728 in_file_buf = NULL;
729
730 _aligned_free(out_file_buf);
731 out_file_buf = NULL;
732
733 src_bytes_left += (in_file_buf_size - in_file_buf_ofs);
734
735 uint32 adler32 = zstream.adler;
736 inflateEnd(&zstream);
737
738 timer_ticks end_time = timer::get_ticks();
739 double total_time = timer::ticks_to_secs(my_max(1, end_time - start_time));
740
741 fclose(pInFile);
742 pInFile = NULL;
743
744 fclose(pOutFile);
745 pOutFile = NULL;
746
747 if (status != Z_STREAM_END)
748 {
749 print_error("Decompression FAILED with status %i\n", status);
750 return false;
751 }
752
753 if ((src_file_size < UINT_MAX) && (orig_file_size < UINT_MAX))
754 {
755 if ((((size_t)zstream.total_in + total_header_bytes) != src_file_size) || (zstream.total_out != orig_file_size))
756 {
757 print_error("Decompression FAILED to consume all input or write all expected output!\n");
758 return false;
759 }
760 }
761
762 if (dst_bytes_left)
763 {
764 print_error("Decompressor FAILED to output the entire output file!\n");
765 return false;
766 }
767
768 if (src_bytes_left)
769 {
770 print_error("Decompressor FAILED to read " QUAD_INT_FMT " bytes from input buffer\n", src_bytes_left);
771 }
772
773 printf("Success\n");
774 printf("Source file size: " QUAD_INT_FMT ", Decompressed file size: " QUAD_INT_FMT "\n", src_file_size, orig_file_size);
775 if (zlib_header) printf("Decompressed adler32: 0x%08X\n", adler32);
776 printf("Overall decompression time (decompression init+I/O+decompression): %3.6f\n Consumption rate: %9.1f bytes/sec, Decompression rate: %9.1f bytes/sec\n", total_time, src_file_size / total_time, orig_file_size / total_time);
777 printf("Decompression only time (not counting decompression init or I/O): %3.6f\n Consumption rate: %9.1f bytes/sec, Decompression rate: %9.1f bytes/sec\n", decomp_only_time, src_file_size / decomp_only_time, orig_file_size / decomp_only_time);
778
779 return true;
780 }
781
compare_files(const char * pFilename1,const char * pFilename2)782 static bool compare_files(const char *pFilename1, const char* pFilename2)
783 {
784 FILE* pFile1 = open_file_with_retries(pFilename1, "rb");
785 if (!pFile1)
786 {
787 print_error("Failed opening file: %s\n", pFilename1);
788 return false;
789 }
790
791 FILE* pFile2 = open_file_with_retries(pFilename2, "rb");
792 if (!pFile2)
793 {
794 print_error("Failed opening file: %s\n", pFilename2);
795 fclose(pFile1);
796 return false;
797 }
798
799 _fseeki64(pFile1, 0, SEEK_END);
800 int64 fileSize1 = _ftelli64(pFile1);
801 _fseeki64(pFile1, 0, SEEK_SET);
802
803 _fseeki64(pFile2, 0, SEEK_END);
804 int64 fileSize2 = _ftelli64(pFile2);
805 _fseeki64(pFile2, 0, SEEK_SET);
806
807 if (fileSize1 != fileSize2)
808 {
809 print_error("Files to compare are not the same size: %I64i vs. %I64i.\n", fileSize1, fileSize2);
810 fclose(pFile1);
811 fclose(pFile2);
812 return false;
813 }
814
815 const uint cBufSize = 1024 * 1024;
816 std::vector<uint8> buf1(cBufSize);
817 std::vector<uint8> buf2(cBufSize);
818
819 int64 bytes_remaining = fileSize1;
820 while (bytes_remaining)
821 {
822 const uint bytes_to_read = static_cast<uint>(my_min(cBufSize, bytes_remaining));
823
824 if (fread(&buf1.front(), bytes_to_read, 1, pFile1) != 1)
825 {
826 print_error("Failed reading from file: %s\n", pFilename1);
827 fclose(pFile1);
828 fclose(pFile2);
829 return false;
830 }
831
832 if (fread(&buf2.front(), bytes_to_read, 1, pFile2) != 1)
833 {
834 print_error("Failed reading from file: %s\n", pFilename2);
835 fclose(pFile1);
836 fclose(pFile2);
837 return false;
838 }
839
840 if (memcmp(&buf1.front(), &buf2.front(), bytes_to_read) != 0)
841 {
842 print_error("File data comparison failed!\n");
843 fclose(pFile1);
844 fclose(pFile2);
845 return false;
846 }
847
848 bytes_remaining -= bytes_to_read;
849 }
850
851 fclose(pFile1);
852 fclose(pFile2);
853 return true;
854 }
855
zip_create(const char * pZip_filename,const char * pSrc_filename)856 static bool zip_create(const char *pZip_filename, const char *pSrc_filename)
857 {
858 mz_zip_archive zip;
859 memset(&zip, 0, sizeof(zip));
860 if ((rand() % 100) >= 10)
861 zip.m_file_offset_alignment = 1 << (rand() & 15);
862 if (!mz_zip_writer_init_file(&zip, pZip_filename, 65537))
863 {
864 print_error("Failed creating zip archive \"%s\" (1)!\n", pZip_filename);
865 return false;
866 }
867
868 mz_bool success = MZ_TRUE;
869
870 const char *pStr = "This is a test!This is a test!This is a test!\n";
871 size_t comp_size;
872 void *pComp_data = tdefl_compress_mem_to_heap(pStr, strlen(pStr), &comp_size, 256);
873 success &= mz_zip_writer_add_mem_ex(&zip, "precomp.txt", pComp_data, comp_size, "Comment", (uint16)strlen("Comment"), MZ_ZIP_FLAG_COMPRESSED_DATA, strlen(pStr), mz_crc32(MZ_CRC32_INIT, (const uint8 *)pStr, strlen(pStr)));
874
875 /* Free the buffer created by tdefl_compress_mem_to_heap */
876 /* NOTE: Make sure this matches what tdefl_compress_mem_to_heap uses */
877 miniz_def_free_func(NULL, pComp_data);
878
879 success &= mz_zip_writer_add_mem(&zip, "cool/", NULL, 0, 0);
880
881 success &= mz_zip_writer_add_mem(&zip, "1.txt", pStr, strlen(pStr), 9);
882 int n = rand() & 4095;
883 for (int i = 0; i < n; i++)
884 {
885 char name[256], buf[256], comment[256];
886 sprintf(name, "t%u.txt", i);
887 sprintf(buf, "%u\n", i*5377);
888 sprintf(comment, "comment: %u\n", i);
889 success &= mz_zip_writer_add_mem_ex(&zip, name, buf, strlen(buf), comment, (uint16)strlen(comment), i % 10, 0, 0);
890 }
891
892 const char *pTestComment = "test comment";
893 success &= mz_zip_writer_add_file(&zip, "test.bin", pSrc_filename, pTestComment, (uint16)strlen(pTestComment), 9);
894
895 if (ensure_file_exists_and_is_readable("changelog.txt"))
896 success &= mz_zip_writer_add_file(&zip, "changelog.txt", "changelog.txt", "This is a comment", (uint16)strlen("This is a comment"), 9);
897
898 if (!success)
899 {
900 mz_zip_writer_end(&zip);
901 remove(pZip_filename);
902 print_error("Failed creating zip archive \"%s\" (2)!\n", pZip_filename);
903 return false;
904 }
905
906 if (!mz_zip_writer_finalize_archive(&zip))
907 {
908 mz_zip_writer_end(&zip);
909 remove(pZip_filename);
910 print_error("Failed creating zip archive \"%s\" (3)!\n", pZip_filename);
911 return false;
912 }
913
914 mz_zip_writer_end(&zip);
915
916 struct FILE_STAT_STRUCT stat_buf;
917 FILE_STAT(pZip_filename, &stat_buf);
918 uint64 actual_file_size = stat_buf.st_size;
919 if (zip.m_archive_size != actual_file_size)
920 {
921 print_error("Archive's actual size and zip archive object's size differ for file \"%s\"!\n", pZip_filename);
922 return false;
923 }
924
925 printf("Created zip file \"%s\", file size: " QUAD_INT_FMT "\n", pZip_filename, (long long unsigned int)zip.m_archive_size);
926 return true;
927 }
928
zip_write_callback(void * pOpaque,mz_uint64 ofs,const void * pBuf,size_t n)929 static size_t zip_write_callback(void *pOpaque, mz_uint64 ofs, const void *pBuf, size_t n)
930 {
931 (void)pOpaque, (void)ofs, (void)pBuf, (void)n;
932 return n;
933 }
934
zip_extract(const char * pZip_filename,const char * pDst_filename)935 static bool zip_extract(const char *pZip_filename, const char *pDst_filename)
936 {
937 mz_zip_archive zip;
938 memset(&zip, 0, sizeof(zip));
939 if (!mz_zip_reader_init_file(&zip, pZip_filename, 0))
940 {
941 print_error("Failed opening zip archive \"%s\"!\n", pZip_filename);
942 return false;
943 }
944
945 int file_index = mz_zip_reader_locate_file(&zip, "test.bin", "test Comment", 0);
946 int alt_file_index = mz_zip_reader_locate_file(&zip, "test.bin", "test Comment e", 0);
947 if ((file_index < 0) || (alt_file_index >= 0))
948 {
949 print_error("Archive \"%s\" is missing test.bin file!\n", pZip_filename);
950 mz_zip_reader_end(&zip);
951 return false;
952 }
953
954 alt_file_index = mz_zip_reader_locate_file(&zip, "test.bin", NULL, 0);
955 if (alt_file_index != file_index)
956 {
957 print_error("mz_zip_reader_locate_file() failed!\n", pZip_filename);
958 mz_zip_reader_end(&zip);
959 return false;
960 }
961
962 if (!mz_zip_reader_extract_to_file(&zip, file_index, pDst_filename, 0))
963 {
964 print_error("Failed extracting test.bin from archive \"%s\" err: %s!\n", pZip_filename, mz_zip_get_error_string(mz_zip_get_last_error(&zip)));
965 mz_zip_reader_end(&zip);
966 return false;
967 }
968
969 for (uint i = 0; i < mz_zip_reader_get_num_files(&zip); i++)
970 {
971 mz_zip_archive_file_stat stat;
972 if (!mz_zip_reader_file_stat(&zip, i, &stat))
973 {
974 print_error("Failed testing archive -1 \"%s\" err: %s!\n", pZip_filename, mz_zip_get_error_string(mz_zip_get_last_error(&zip)));
975 mz_zip_reader_end(&zip);
976 return false;
977 }
978 //printf("\"%s\" %I64u %I64u\n", stat.m_filename, stat.m_comp_size, stat.m_uncomp_size);
979 size_t size = 0;
980
981 mz_bool status = mz_zip_reader_extract_to_callback(&zip, i, zip_write_callback, NULL, 0);
982 if (!status)
983 {
984 print_error("Failed testing archive -2 \"%s\" err: %s!\n", pZip_filename, mz_zip_get_error_string(mz_zip_get_last_error(&zip)));
985 mz_zip_reader_end(&zip);
986 return false;
987 }
988
989 if (stat.m_uncomp_size<100*1024*1024)
990 {
991 void *p = mz_zip_reader_extract_to_heap(&zip, i, &size, 0);
992 if (!p)
993 {
994 print_error("Failed testing archive -3 \"%s\" err: %s!\n", pZip_filename, mz_zip_get_error_string(mz_zip_get_last_error(&zip)));
995 mz_zip_reader_end(&zip);
996 return false;
997 }
998 free(p);
999 }
1000 }
1001 printf("Verified %u files\n", mz_zip_reader_get_num_files(&zip));
1002
1003 mz_zip_reader_end(&zip);
1004
1005 printf("Extracted file \"%s\"\n", pDst_filename);
1006 return true;
1007 }
1008
1009 typedef std::vector< std::string > string_array;
1010
1011 #if defined(WIN32)
find_files(std::string pathname,const std::string & filename,string_array & files,bool recursive,int depth=0)1012 static bool find_files(std::string pathname, const std::string &filename, string_array &files, bool recursive, int depth = 0)
1013 {
1014 if (!pathname.empty())
1015 {
1016 char c = pathname[pathname.size() - 1];
1017 if ((c != ':') && (c != '\\') && (c != '/'))
1018 pathname += "\\";
1019 }
1020
1021 WIN32_FIND_DATAA find_data;
1022
1023 HANDLE findHandle = FindFirstFileA((pathname + filename).c_str(), &find_data);
1024 if (findHandle == INVALID_HANDLE_VALUE)
1025 {
1026 HRESULT hres = GetLastError();
1027 if ((!depth) && (hres != NO_ERROR) && (hres != ERROR_FILE_NOT_FOUND))
1028 return false;
1029 }
1030 else
1031 {
1032 do
1033 {
1034 const bool is_directory = (find_data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0;
1035 const bool is_system = (find_data.dwFileAttributes & FILE_ATTRIBUTE_SYSTEM) != 0;
1036 const bool is_hidden = false;//(find_data.dwFileAttributes & FILE_ATTRIBUTE_HIDDEN) != 0;
1037
1038 std::string filename(find_data.cFileName);
1039
1040 if ((!is_directory) && (!is_system) && (!is_hidden))
1041 files.push_back(pathname + filename);
1042
1043 } while (FindNextFileA(findHandle, &find_data));
1044
1045 FindClose(findHandle);
1046 }
1047
1048 if (recursive)
1049 {
1050 string_array paths;
1051
1052 HANDLE findHandle = FindFirstFileA((pathname + "*").c_str(), &find_data);
1053 if (findHandle != INVALID_HANDLE_VALUE)
1054 {
1055 do
1056 {
1057 const bool is_directory = (find_data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0;
1058 const bool is_system = (find_data.dwFileAttributes & FILE_ATTRIBUTE_SYSTEM) != 0;
1059 const bool is_hidden = false;//(find_data.dwFileAttributes & FILE_ATTRIBUTE_HIDDEN) != 0;
1060
1061 std::string filename(find_data.cFileName);
1062
1063 if ((is_directory) && (!is_hidden) && (!is_system))
1064 paths.push_back(filename);
1065
1066 } while (FindNextFileA(findHandle, &find_data));
1067
1068 FindClose(findHandle);
1069
1070 for (uint i = 0; i < paths.size(); i++)
1071 {
1072 const std::string &path = paths[i];
1073 if (path[0] == '.')
1074 continue;
1075
1076 if (!find_files(pathname + path, filename, files, true, depth + 1))
1077 return false;
1078 }
1079 }
1080 }
1081
1082 return true;
1083 }
1084 #else
1085 #include <dirent.h>
1086 #include <fnmatch.h>
find_files(std::string pathname,const std::string & pattern,string_array & files,bool recursive,int depth=0)1087 static bool find_files(std::string pathname, const std::string &pattern, string_array &files, bool recursive, int depth = 0)
1088 {
1089 if (!pathname.empty())
1090 {
1091 char c = pathname[pathname.size() - 1];
1092 if ((c != ':') && (c != '\\') && (c != '/'))
1093 pathname += "/";
1094 }
1095
1096 DIR *dp = opendir(pathname.c_str());
1097
1098 if (!dp) {
1099 return depth ? true : false;
1100 }
1101
1102 string_array paths;
1103
1104 for ( ; ; )
1105 {
1106 struct dirent *ep = readdir(dp);
1107 if (!ep)
1108 break;
1109
1110 const bool is_directory = (ep->d_type & DT_DIR) != 0;
1111 const bool is_file = (ep->d_type & DT_REG) != 0;
1112
1113 if (ep->d_name[0] == '.')
1114 continue;
1115
1116 std::string filename(ep->d_name);
1117
1118 if (is_directory)
1119 {
1120 if (recursive)
1121 paths.push_back(filename);
1122 }
1123 else if (is_file)
1124 {
1125 if (0 == fnmatch(pattern.c_str(), filename.c_str(), 0))
1126 files.push_back(pathname + filename);
1127 }
1128 }
1129
1130 closedir(dp);
1131 dp = NULL;
1132
1133 if (recursive)
1134 {
1135 for (uint i = 0; i < paths.size(); i++)
1136 {
1137 const std::string &path = paths[i];
1138 if (!find_files(pathname + path, pattern, files, true, depth + 1))
1139 return false;
1140 }
1141 }
1142
1143 return true;
1144 }
1145 #endif
1146
test_recursive(const char * pPath,comp_options options)1147 static bool test_recursive(const char *pPath, comp_options options)
1148 {
1149 string_array files;
1150 if (!find_files(pPath, "*", files, true))
1151 {
1152 print_error("Failed finding files under path \"%s\"!\n", pPath);
1153 return false;
1154 }
1155
1156 uint total_files_compressed = 0;
1157 uint64 total_source_size = 0;
1158 uint64 total_comp_size = 0;
1159
1160 #ifdef WIN32
1161 MEMORYSTATUS initial_mem_status;
1162 GlobalMemoryStatus(&initial_mem_status);
1163 #endif
1164
1165 timer_ticks start_tick_count = timer::get_ticks();
1166
1167 const int first_file_index = 0;
1168
1169 uint unique_id = static_cast<uint>(timer::get_init_ticks());
1170 char cmp_file[256], decomp_file[256];
1171
1172 sprintf(cmp_file, "__comp_temp_%u__.tmp", unique_id);
1173 sprintf(decomp_file, "__decomp_temp_%u__.tmp", unique_id);
1174
1175 for (uint file_index = first_file_index; file_index < files.size(); file_index++)
1176 {
1177 const std::string &src_file = files[file_index];
1178
1179 printf("***** [%u of %u] Compressing file \"%s\" to \"%s\"\n", 1 + file_index, (uint)files.size(), src_file.c_str(), cmp_file);
1180
1181 if ((strstr(src_file.c_str(), "__comp_temp") != NULL) || (strstr(src_file.c_str(), "__decomp_temp") != NULL))
1182 {
1183 printf("Skipping temporary file \"%s\"\n", src_file.c_str());
1184 continue;
1185 }
1186
1187 FILE *pFile = fopen(src_file.c_str(), "rb");
1188 if (!pFile)
1189 {
1190 printf("Skipping unreadable file \"%s\"\n", src_file.c_str());
1191 continue;
1192 }
1193 _fseeki64(pFile, 0, SEEK_END);
1194 int64 src_file_size = _ftelli64(pFile);
1195
1196 if (src_file_size)
1197 {
1198 _fseeki64(pFile, 0, SEEK_SET);
1199 if (fgetc(pFile) == EOF)
1200 {
1201 printf("Skipping unreadable file \"%s\"\n", src_file.c_str());
1202 fclose(pFile);
1203 continue;
1204 }
1205 }
1206
1207 fclose(pFile);
1208
1209 if (!ensure_file_is_writable(cmp_file))
1210 {
1211 print_error("Unable to create file \"%s\"!\n", cmp_file);
1212 return false;
1213 }
1214
1215 comp_options file_options(options);
1216 if (options.m_randomize_params)
1217 {
1218 file_options.m_level = rand() % 11;
1219 file_options.m_unbuffered_decompression = (rand() & 1) != 0;
1220 file_options.m_z_strat = rand() % (Z_FIXED + 1);
1221 file_options.m_write_zlib_header = (rand() & 1) != 0;
1222 file_options.m_random_z_flushing = (rand() & 1) != 0;
1223 file_options.print();
1224 }
1225
1226 bool status;
1227 if (file_options.m_archive_test)
1228 {
1229 printf("Creating test archive with file \"%s\", size " QUAD_INT_FMT "\n", src_file.c_str(), src_file_size);
1230 status = zip_create(cmp_file, src_file.c_str());
1231 }
1232 else
1233 status = compress_file_zlib(src_file.c_str(), cmp_file, file_options);
1234 if (!status)
1235 {
1236 print_error("Failed compressing file \"%s\" to \"%s\"\n", src_file.c_str(), cmp_file);
1237 return false;
1238 }
1239
1240 if (file_options.m_verify_compressed_data)
1241 {
1242 printf("Decompressing file \"%s\" to \"%s\"\n", cmp_file, decomp_file);
1243
1244 if (!ensure_file_is_writable(decomp_file))
1245 {
1246 print_error("Unable to create file \"%s\"!\n", decomp_file);
1247 return false;
1248 }
1249
1250 if (file_options.m_archive_test)
1251 status = zip_extract(cmp_file, decomp_file);
1252 else
1253 status = decompress_file_zlib(cmp_file, decomp_file, file_options);
1254
1255 if (!status)
1256 {
1257 print_error("Failed decompressing file \"%s\" to \"%s\"\n", src_file.c_str(), decomp_file);
1258 return false;
1259 }
1260
1261 printf("Comparing file \"%s\" to \"%s\"\n", decomp_file, src_file.c_str());
1262
1263 if (!compare_files(decomp_file, src_file.c_str()))
1264 {
1265 print_error("Failed comparing decompressed file data while compressing \"%s\" to \"%s\"\n", src_file.c_str(), cmp_file);
1266 return false;
1267 }
1268 else
1269 {
1270 printf("Decompressed file compared OK to original file.\n");
1271 }
1272 }
1273
1274 int64 cmp_file_size = 0;
1275 pFile = fopen(cmp_file, "rb");
1276 if (pFile)
1277 {
1278 _fseeki64(pFile, 0, SEEK_END);
1279 cmp_file_size = _ftelli64(pFile);
1280 fclose(pFile);
1281 }
1282
1283 total_files_compressed++;
1284 total_source_size += src_file_size;
1285 total_comp_size += cmp_file_size;
1286
1287 #ifdef WIN32
1288 MEMORYSTATUS mem_status;
1289 GlobalMemoryStatus(&mem_status);
1290
1291 const int64 bytes_allocated = initial_mem_status.dwAvailVirtual- mem_status.dwAvailVirtual;
1292
1293 printf("Memory allocated relative to first file: %I64i\n", bytes_allocated);
1294 #endif
1295
1296 printf("\n");
1297 }
1298
1299 timer_ticks end_tick_count = timer::get_ticks();
1300
1301 double total_elapsed_time = timer::ticks_to_secs(end_tick_count - start_tick_count);
1302
1303 printf("Test successful: %f secs\n", total_elapsed_time);
1304 printf("Total files processed: %u\n", total_files_compressed);
1305 printf("Total source size: " QUAD_INT_FMT "\n", total_source_size);
1306 printf("Total compressed size: " QUAD_INT_FMT "\n", total_comp_size);
1307 printf("Max small comp ratio: %f, Max large comp ratio: %f\n", s_max_small_comp_ratio, s_max_large_comp_ratio);
1308
1309 remove(cmp_file);
1310 remove(decomp_file);
1311
1312 return true;
1313 }
1314
dummy_zip_file_write_callback(void * pOpaque,mz_uint64 ofs,const void * pBuf,size_t n)1315 static size_t dummy_zip_file_write_callback(void *pOpaque, mz_uint64 ofs, const void *pBuf, size_t n)
1316 {
1317 (void)ofs; (void)pBuf;
1318 uint32 *pCRC = (uint32*)pOpaque;
1319 *pCRC = mz_crc32(*pCRC, (const uint8*)pBuf, n);
1320 return n;
1321 }
1322
test_archives(const char * pPath,comp_options options)1323 static bool test_archives(const char *pPath, comp_options options)
1324 {
1325 (void)options;
1326
1327 string_array files;
1328 if (!find_files(pPath, "*.zip", files, true))
1329 {
1330 print_error("Failed finding files under path \"%s\"!\n", pPath);
1331 return false;
1332 }
1333
1334 uint total_archives = 0;
1335 uint64 total_bytes_processed = 0;
1336 uint64 total_files_processed = 0;
1337 uint total_errors = 0;
1338
1339 #ifdef WIN32
1340 MEMORYSTATUS initial_mem_status;
1341 GlobalMemoryStatus(&initial_mem_status);
1342 #endif
1343
1344 const int first_file_index = 0;
1345 uint unique_id = static_cast<uint>(timer::get_init_ticks());
1346 char cmp_file[256], decomp_file[256];
1347
1348 sprintf(decomp_file, "__decomp_temp_%u__.tmp", unique_id);
1349
1350 string_array failed_archives;
1351
1352 for (uint file_index = first_file_index; file_index < files.size(); file_index++)
1353 {
1354 const std::string &src_file = files[file_index];
1355
1356 printf("***** [%u of %u] Testing archive file \"%s\"\n", 1 + file_index, (uint)files.size(), src_file.c_str());
1357
1358 if ((strstr(src_file.c_str(), "__comp_temp") != NULL) || (strstr(src_file.c_str(), "__decomp_temp") != NULL))
1359 {
1360 printf("Skipping temporary file \"%s\"\n", src_file.c_str());
1361 continue;
1362 }
1363
1364 FILE *pFile = fopen(src_file.c_str(), "rb");
1365 if (!pFile)
1366 {
1367 printf("Skipping unreadable file \"%s\"\n", src_file.c_str());
1368 continue;
1369 }
1370 _fseeki64(pFile, 0, SEEK_END);
1371 int64 src_file_size = _ftelli64(pFile);
1372 fclose(pFile);
1373
1374 (void)src_file_size;
1375
1376 sprintf(cmp_file, "__comp_temp_%u__.zip", file_index);
1377
1378 mz_zip_archive src_archive;
1379 memset(&src_archive, 0, sizeof(src_archive));
1380
1381 if (!mz_zip_reader_init_file(&src_archive, src_file.c_str(), 0))
1382 {
1383 failed_archives.push_back(src_file);
1384
1385 print_error("Failed opening archive \"%s\"!\n", src_file.c_str());
1386 total_errors++;
1387 continue;
1388 }
1389
1390 mz_zip_archive dst_archive;
1391 memset(&dst_archive, 0, sizeof(dst_archive));
1392 if (options.m_write_archives)
1393 {
1394 if (!ensure_file_is_writable(cmp_file))
1395 {
1396 print_error("Unable to create file \"%s\"!\n", cmp_file);
1397 return false;
1398 }
1399
1400 if (!mz_zip_writer_init_file(&dst_archive, cmp_file, 0))
1401 {
1402 print_error("Failed creating archive \"%s\"!\n", cmp_file);
1403 total_errors++;
1404 continue;
1405 }
1406 }
1407
1408 int i;
1409 //for (i = 0; i < mz_zip_reader_get_num_files(&src_archive); i++)
1410 for (i = mz_zip_reader_get_num_files(&src_archive) - 1; i >= 0; --i)
1411 {
1412 if (mz_zip_reader_is_file_encrypted(&src_archive, i))
1413 continue;
1414
1415 mz_zip_archive_file_stat file_stat;
1416 bool status = mz_zip_reader_file_stat(&src_archive, i, &file_stat) != 0;
1417
1418 int locate_file_index = mz_zip_reader_locate_file(&src_archive, file_stat.m_filename, NULL, 0);
1419 if (locate_file_index != i)
1420 {
1421 mz_zip_archive_file_stat locate_file_stat;
1422 mz_zip_reader_file_stat(&src_archive, locate_file_index, &locate_file_stat);
1423 if (_stricmp(locate_file_stat.m_filename, file_stat.m_filename) != 0)
1424 {
1425 print_error("mz_zip_reader_locate_file() failed!\n");
1426 return false;
1427 }
1428 else
1429 {
1430 printf("Warning: Duplicate filenames in archive!\n");
1431 }
1432 }
1433
1434 if ((file_stat.m_method) && (file_stat.m_method != MZ_DEFLATED))
1435 continue;
1436
1437 if (status)
1438 {
1439 char name[260];
1440 mz_zip_reader_get_filename(&src_archive, i, name, sizeof(name));
1441
1442 size_t extracted_size = 0;
1443 void *p = mz_zip_reader_extract_file_to_heap(&src_archive, name, &extracted_size, 0);
1444 if (!p)
1445 status = false;
1446
1447 uint32 extracted_crc32 = MZ_CRC32_INIT;
1448 if (!mz_zip_reader_extract_file_to_callback(&src_archive, name, dummy_zip_file_write_callback, &extracted_crc32, 0))
1449 status = false;
1450
1451 if (mz_crc32(MZ_CRC32_INIT, (const uint8*)p, extracted_size) != extracted_crc32)
1452 status = false;
1453
1454 free(p);
1455
1456 if (options.m_write_archives)
1457 {
1458 if ((status) && (!mz_zip_writer_add_from_zip_reader(&dst_archive, &src_archive, i)))
1459 {
1460 print_error("Failed adding new file to archive \"%s\"!\n", cmp_file);
1461 status = false;
1462 }
1463 }
1464
1465 total_bytes_processed += file_stat.m_uncomp_size;
1466 total_files_processed++;
1467 }
1468
1469 if (!status)
1470 break;
1471 }
1472
1473 mz_zip_reader_end(&src_archive);
1474
1475 //if (i < mz_zip_reader_get_num_files(&src_archive))
1476 if (i >= 0)
1477 {
1478 failed_archives.push_back(src_file);
1479
1480 print_error("Failed processing archive \"%s\"!\n", src_file.c_str());
1481 total_errors++;
1482 }
1483
1484 if (options.m_write_archives)
1485 {
1486 if (!mz_zip_writer_finalize_archive(&dst_archive) || !mz_zip_writer_end(&dst_archive))
1487 {
1488 failed_archives.push_back(src_file);
1489
1490 print_error("Failed finalizing archive \"%s\"!\n", cmp_file);
1491 total_errors++;
1492 }
1493 }
1494
1495 total_archives++;
1496
1497 #ifdef WIN32
1498 MEMORYSTATUS mem_status;
1499 GlobalMemoryStatus(&mem_status);
1500
1501 const int64 bytes_allocated = initial_mem_status.dwAvailVirtual- mem_status.dwAvailVirtual;
1502
1503 printf("Memory allocated relative to first file: %I64i\n", bytes_allocated);
1504 #endif
1505
1506 printf("\n");
1507 }
1508
1509 printf("Total archives processed: %u\n", total_archives);
1510 printf("Total errors: %u\n", total_errors);
1511 printf("Total bytes processed: " QUAD_INT_FMT "\n", total_bytes_processed);
1512 printf("Total archive files processed: " QUAD_INT_FMT "\n", total_files_processed);
1513
1514 printf("List of failed archives:\n");
1515 for (uint i = 0; i < failed_archives.size(); ++i)
1516 printf("%s\n", failed_archives[i].c_str());
1517
1518 remove(cmp_file);
1519 remove(decomp_file);
1520
1521 return true;
1522 }
1523
main_internal(string_array cmd_line)1524 int main_internal(string_array cmd_line)
1525 {
1526 comp_options options;
1527
1528 if (!cmd_line.size())
1529 {
1530 print_usage();
1531 if (simple_test1(options) || simple_test2(options))
1532 return EXIT_FAILURE;
1533 return EXIT_SUCCESS;
1534 }
1535
1536 enum op_mode_t
1537 {
1538 OP_MODE_INVALID = -1,
1539 OP_MODE_COMPRESS = 0,
1540 OP_MODE_DECOMPRESS = 1,
1541 OP_MODE_ALL = 2,
1542 OP_MODE_ARCHIVES = 3
1543 };
1544
1545 op_mode_t op_mode = OP_MODE_INVALID;
1546
1547 for (int i = 0; i < (int)cmd_line.size(); i++)
1548 {
1549 const std::string &str = cmd_line[i];
1550 if (str[0] == '-')
1551 {
1552 if (str.size() < 2)
1553 {
1554 print_error("Invalid option: %s\n", str.c_str());
1555 return EXIT_FAILURE;
1556 }
1557 switch (tolower(str[1]))
1558 {
1559 case 'u':
1560 {
1561 options.m_unbuffered_decompression = true;
1562 break;
1563 }
1564 case 'm':
1565 {
1566 int comp_level = atoi(str.c_str() + 2);
1567 if ((comp_level < 0) || (comp_level > (int)10))
1568 {
1569 print_error("Invalid compression level: %s\n", str.c_str());
1570 return EXIT_FAILURE;
1571 }
1572
1573 options.m_level = comp_level;
1574 break;
1575 }
1576 case 'v':
1577 {
1578 options.m_verify_compressed_data = true;
1579 break;
1580 }
1581 case 'r':
1582 {
1583 options.m_randomize_params = true;
1584 break;
1585 }
1586 case 'b':
1587 {
1588 options.m_randomize_buffer_sizes = true;
1589 break;
1590 }
1591 case 'h':
1592 {
1593 options.m_random_z_flushing = true;
1594 break;
1595 }
1596 case 'x':
1597 {
1598 int seed = atoi(str.c_str() + 2);
1599 srand(seed);
1600 printf("Using random seed: %i\n", seed);
1601 break;
1602 }
1603 case 't':
1604 {
1605 options.m_z_strat = my_min(Z_FIXED, my_max(0, atoi(str.c_str() + 2)));
1606 break;
1607 }
1608 case 'z':
1609 {
1610 options.m_write_zlib_header = false;
1611 break;
1612 }
1613 case 'a':
1614 {
1615 options.m_archive_test = true;
1616 break;
1617 }
1618 case 'w':
1619 {
1620 options.m_write_archives = true;
1621 break;
1622 }
1623 default:
1624 {
1625 print_error("Invalid option: %s\n", str.c_str());
1626 return EXIT_FAILURE;
1627 }
1628 }
1629
1630 cmd_line.erase(cmd_line.begin() + i);
1631 i--;
1632
1633 continue;
1634 }
1635
1636 if (str.size() != 1)
1637 {
1638 print_error("Invalid mode: %s\n", str.c_str());
1639 return EXIT_FAILURE;
1640 }
1641 switch (tolower(str[0]))
1642 {
1643 case 'c':
1644 {
1645 op_mode = OP_MODE_COMPRESS;
1646 break;
1647 }
1648 case 'd':
1649 {
1650 op_mode = OP_MODE_DECOMPRESS;
1651 break;
1652 }
1653 case 'a':
1654 {
1655 op_mode = OP_MODE_ALL;
1656 break;
1657 }
1658 case 'r':
1659 {
1660 op_mode = OP_MODE_ARCHIVES;
1661 break;
1662 }
1663 default:
1664 {
1665 print_error("Invalid mode: %s\n", str.c_str());
1666 return EXIT_FAILURE;
1667 }
1668 }
1669 cmd_line.erase(cmd_line.begin() + i);
1670 break;
1671 }
1672
1673 if (op_mode == OP_MODE_INVALID)
1674 {
1675 print_error("No mode specified!\n");
1676 print_usage();
1677 return EXIT_FAILURE;
1678 }
1679
1680 printf("Using options:\n");
1681 options.print();
1682 printf("\n");
1683
1684 int exit_status = EXIT_FAILURE;
1685
1686 switch (op_mode)
1687 {
1688 case OP_MODE_COMPRESS:
1689 {
1690 if (cmd_line.size() < 2)
1691 {
1692 print_error("Must specify input and output filenames!\n");
1693 return EXIT_FAILURE;
1694 }
1695 else if (cmd_line.size() > 2)
1696 {
1697 print_error("Too many filenames!\n");
1698 return EXIT_FAILURE;
1699 }
1700
1701 const std::string &src_file = cmd_line[0];
1702 const std::string &cmp_file = cmd_line[1];
1703
1704 bool comp_result = compress_file_zlib(src_file.c_str(), cmp_file.c_str(), options);
1705 if (comp_result)
1706 exit_status = EXIT_SUCCESS;
1707
1708 if ((comp_result) && (options.m_verify_compressed_data))
1709 {
1710 char decomp_file[256];
1711
1712 sprintf(decomp_file, "__decomp_temp_%u__.tmp", (uint)timer::get_ms());
1713
1714 if (!decompress_file_zlib(cmp_file.c_str(), decomp_file, options))
1715 {
1716 print_error("Failed decompressing file \"%s\" to \"%s\"\n", cmp_file.c_str(), decomp_file);
1717 return EXIT_FAILURE;
1718 }
1719
1720 printf("Comparing file \"%s\" to \"%s\"\n", decomp_file, src_file.c_str());
1721
1722 if (!compare_files(decomp_file, src_file.c_str()))
1723 {
1724 print_error("Failed comparing decompressed file data while compressing \"%s\" to \"%s\"\n", src_file.c_str(), cmp_file.c_str());
1725 return EXIT_FAILURE;
1726 }
1727 else
1728 {
1729 printf("Decompressed file compared OK to original file.\n");
1730 }
1731
1732 remove(decomp_file);
1733 }
1734
1735 break;
1736 }
1737 case OP_MODE_DECOMPRESS:
1738 {
1739 if (cmd_line.size() < 2)
1740 {
1741 print_error("Must specify input and output filenames!\n");
1742 return EXIT_FAILURE;
1743 }
1744 else if (cmd_line.size() > 2)
1745 {
1746 print_error("Too many filenames!\n");
1747 return EXIT_FAILURE;
1748 }
1749 if (decompress_file_zlib(cmd_line[0].c_str(), cmd_line[1].c_str(), options))
1750 exit_status = EXIT_SUCCESS;
1751 break;
1752 }
1753 case OP_MODE_ALL:
1754 {
1755 if (!cmd_line.size())
1756 {
1757 print_error("No directory specified!\n");
1758 return EXIT_FAILURE;
1759 }
1760 else if (cmd_line.size() != 1)
1761 {
1762 print_error("Too many filenames!\n");
1763 return EXIT_FAILURE;
1764 }
1765 if (test_recursive(cmd_line[0].c_str(), options))
1766 exit_status = EXIT_SUCCESS;
1767 break;
1768 }
1769 case OP_MODE_ARCHIVES:
1770 {
1771 if (!cmd_line.size())
1772 {
1773 print_error("No directory specified!\n");
1774 return EXIT_FAILURE;
1775 }
1776 else if (cmd_line.size() != 1)
1777 {
1778 print_error("Too many filenames!\n");
1779 return EXIT_FAILURE;
1780 }
1781 if (test_archives(cmd_line[0].c_str(), options))
1782 exit_status = EXIT_SUCCESS;
1783 break;
1784 }
1785 default:
1786 {
1787 print_error("No mode specified!\n");
1788 print_usage();
1789 return EXIT_FAILURE;
1790 }
1791 }
1792
1793 return exit_status;
1794 }
1795
main(int argc,char * argv[])1796 int main(int argc, char *argv[])
1797 {
1798 #if defined(_WIN64) || defined(__LP64__) || defined(_LP64)
1799 printf("miniz.c x64 Command Line Test App - Compiled %s %s\n", __DATE__, __TIME__);
1800 #else
1801 printf("miniz.c x86 Command Line Test App - Compiled %s %s\n", __DATE__, __TIME__);
1802 #endif
1803 timer::get_ticks();
1804
1805 string_array cmd_line;
1806 for (int i = 1; i < argc; i++)
1807 cmd_line.push_back(std::string(argv[i]));
1808
1809 int exit_status = main_internal(cmd_line);
1810
1811 return exit_status;
1812 }
1813