1 /* Lziprecover - Data recovery tool for the lzip format
2    Copyright (C) 2009-2021 Antonio Diaz Diaz.
3 
4    This program is free software: you can redistribute it and/or modify
5    it under the terms of the GNU General Public License as published by
6    the Free Software Foundation, either version 2 of the License, or
7    (at your option) any later version.
8 
9    This program is distributed in the hope that it will be useful,
10    but WITHOUT ANY WARRANTY; without even the implied warranty of
11    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12    GNU General Public License for more details.
13 
14    You should have received a copy of the GNU General Public License
15    along with this program.  If not, see <http://www.gnu.org/licenses/>.
16 */
17 
18 class State
19   {
20   int st;
21 
22 public:
23   enum { states = 12 };
State()24   State() : st( 0 ) {}
operator()25   int operator()() const { return st; }
is_char()26   bool is_char() const { return st < 7; }
27 
set_char()28   void set_char()
29     {
30     static const int next[states] = { 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 4, 5 };
31     st = next[st];
32     }
is_char_set_char()33   bool is_char_set_char()
34     {
35     if( st < 7 ) { st -= ( st < 4 ) ? st : 3; return true; }
36     else { st -= ( st < 10 ) ? 3 : 6; return false; }
37     }
set_match()38   void set_match()     { st = ( st < 7 ) ? 7 : 10; }
set_rep()39   void set_rep()       { st = ( st < 7 ) ? 8 : 11; }
set_short_rep()40   void set_short_rep() { st = ( st < 7 ) ? 9 : 11; }
41   };
42 
43 
44 enum {
45   min_dictionary_bits = 12,
46   min_dictionary_size = 1 << min_dictionary_bits,	// >= modeled_distances
47   max_dictionary_bits = 29,
48   max_dictionary_size = 1 << max_dictionary_bits,
49   min_member_size = 36,
50   literal_context_bits = 3,
51   literal_pos_state_bits = 0,				// not used
52   pos_state_bits = 2,
53   pos_states = 1 << pos_state_bits,
54   pos_state_mask = pos_states - 1,
55 
56   len_states = 4,
57   dis_slot_bits = 6,
58   start_dis_model = 4,
59   end_dis_model = 14,
60   modeled_distances = 1 << (end_dis_model / 2),		// 128
61   dis_align_bits = 4,
62   dis_align_size = 1 << dis_align_bits,
63 
64   len_low_bits = 3,
65   len_mid_bits = 3,
66   len_high_bits = 8,
67   len_low_symbols = 1 << len_low_bits,
68   len_mid_symbols = 1 << len_mid_bits,
69   len_high_symbols = 1 << len_high_bits,
70   max_len_symbols = len_low_symbols + len_mid_symbols + len_high_symbols,
71 
72   min_match_len = 2,					// must be 2
73   max_match_len = min_match_len + max_len_symbols - 1,	// 273
74   min_match_len_limit = 5 };
75 
get_len_state(const int len)76 inline int get_len_state( const int len )
77   { return std::min( len - min_match_len, len_states - 1 ); }
78 
get_lit_state(const uint8_t prev_byte)79 inline int get_lit_state( const uint8_t prev_byte )
80   { return prev_byte >> ( 8 - literal_context_bits ); }
81 
82 
83 enum { bit_model_move_bits = 5,
84        bit_model_total_bits = 11,
85        bit_model_total = 1 << bit_model_total_bits };
86 
87 struct Bit_model
88   {
89   int probability;
Bit_modelBit_model90   Bit_model() : probability( bit_model_total / 2 ) {}
91   };
92 
93 struct Len_model
94   {
95   Bit_model choice1;
96   Bit_model choice2;
97   Bit_model bm_low[pos_states][len_low_symbols];
98   Bit_model bm_mid[pos_states][len_mid_symbols];
99   Bit_model bm_high[len_high_symbols];
100   };
101 
102 
103 // defined in main.cc
104 extern int verbosity;
105 
106 class Pretty_print		// requires global var 'int verbosity'
107   {
108   std::string name_;
109   std::string padded_name;
110   const char * const stdin_name;
111   unsigned longest_name;
112   mutable bool first_post;
113 
114 public:
Pretty_print(const std::vector<std::string> & filenames)115   Pretty_print( const std::vector< std::string > & filenames )
116     : stdin_name( "(stdin)" ), longest_name( 0 ), first_post( false )
117     {
118     if( verbosity <= 0 ) return;
119     const unsigned stdin_name_len = std::strlen( stdin_name );
120     for( unsigned i = 0; i < filenames.size(); ++i )
121       {
122       const std::string & s = filenames[i];
123       const unsigned len = ( s == "-" ) ? stdin_name_len : s.size();
124       if( longest_name < len ) longest_name = len;
125       }
126     if( longest_name == 0 ) longest_name = stdin_name_len;
127     }
128 
Pretty_print(const std::string & filename)129   Pretty_print( const std::string & filename )
130     : stdin_name( "(stdin)" ), first_post( false )
131     {
132     const unsigned stdin_name_len = std::strlen( stdin_name );
133     longest_name = ( filename == "-" ) ? stdin_name_len : filename.size();
134     if( longest_name == 0 ) longest_name = stdin_name_len;
135     set_name( filename );
136     }
137 
set_name(const std::string & filename)138   void set_name( const std::string & filename )
139     {
140     if( filename.size() && filename != "-" ) name_ = filename;
141     else name_ = stdin_name;
142     padded_name = "  "; padded_name += name_; padded_name += ": ";
143     if( longest_name > name_.size() )
144       padded_name.append( longest_name - name_.size(), ' ' );
145     first_post = true;
146     }
147 
reset()148   void reset() const { if( name_.size() ) first_post = true; }
name()149   const char * name() const { return name_.c_str(); }
150   void operator()( const char * const msg = 0, FILE * const f = stderr ) const;
151   };
152 
153 
154 class CRC32
155   {
156   uint32_t data[256];		// Table of CRCs of all 8-bit messages.
157 
158 public:
CRC32()159   CRC32()
160     {
161     for( unsigned n = 0; n < 256; ++n )
162       {
163       unsigned c = n;
164       for( int k = 0; k < 8; ++k )
165         { if( c & 1 ) c = 0xEDB88320U ^ ( c >> 1 ); else c >>= 1; }
166       data[n] = c;
167       }
168     }
169 
170   uint32_t operator[]( const uint8_t byte ) const { return data[byte]; }
171 
update_byte(uint32_t & crc,const uint8_t byte)172   void update_byte( uint32_t & crc, const uint8_t byte ) const
173     { crc = data[(crc^byte)&0xFF] ^ ( crc >> 8 ); }
174 
update_buf(uint32_t & crc,const uint8_t * const buffer,const int size)175   void update_buf( uint32_t & crc, const uint8_t * const buffer,
176                    const int size ) const
177     {
178     uint32_t c = crc;
179     for( int i = 0; i < size; ++i )
180       c = data[(c^buffer[i])&0xFF] ^ ( c >> 8 );
181     crc = c;
182     }
183 
compute_crc(const uint8_t * const buffer,const long long size)184   uint32_t compute_crc( const uint8_t * const buffer,
185                         const long long size ) const
186     {
187     uint32_t crc = 0xFFFFFFFFU;
188     for( long long i = 0; i < size; ++i )
189       crc = data[(crc^buffer[i])&0xFF] ^ ( crc >> 8 );
190     return crc ^ 0xFFFFFFFFU;
191     }
192   };
193 
194 extern const CRC32 crc32;
195 
196 
isvalid_ds(const unsigned dictionary_size)197 inline bool isvalid_ds( const unsigned dictionary_size )
198   { return ( dictionary_size >= min_dictionary_size &&
199              dictionary_size <= max_dictionary_size ); }
200 
201 
real_bits(unsigned value)202 inline int real_bits( unsigned value )
203   {
204   int bits = 0;
205   while( value > 0 ) { value >>= 1; ++bits; }
206   return bits;
207   }
208 
209 
210 const uint8_t lzip_magic[4] = { 0x4C, 0x5A, 0x49, 0x50 };	// "LZIP"
211 
212 struct Lzip_header
213   {
214   uint8_t data[6];			// 0-3 magic bytes
215 					//   4 version
216 					//   5 coded dictionary size
217   enum { size = 6 };
218 
set_magicLzip_header219   void set_magic() { std::memcpy( data, lzip_magic, 4 ); data[4] = 1; }
verify_magicLzip_header220   bool verify_magic() const
221     { return ( std::memcmp( data, lzip_magic, 4 ) == 0 ); }
222 
verify_prefixLzip_header223   bool verify_prefix( const int sz ) const	// detect (truncated) header
224     {
225     for( int i = 0; i < sz && i < 4; ++i )
226       if( data[i] != lzip_magic[i] ) return false;
227     return ( sz > 0 );
228     }
verify_corruptLzip_header229   bool verify_corrupt() const			// detect corrupt header
230     {
231     int matches = 0;
232     for( int i = 0; i < 4; ++i )
233       if( data[i] == lzip_magic[i] ) ++matches;
234     return ( matches > 1 && matches < 4 );
235     }
236 
versionLzip_header237   uint8_t version() const { return data[4]; }
verify_versionLzip_header238   bool verify_version() const { return ( data[4] == 1 ); }
239 
dictionary_sizeLzip_header240   unsigned dictionary_size() const
241     {
242     unsigned sz = ( 1 << ( data[5] & 0x1F ) );
243     if( sz > min_dictionary_size )
244       sz -= ( sz / 16 ) * ( ( data[5] >> 5 ) & 7 );
245     return sz;
246     }
247 
dictionary_sizeLzip_header248   bool dictionary_size( const unsigned sz )
249     {
250     if( !isvalid_ds( sz ) ) return false;
251     data[5] = real_bits( sz - 1 );
252     if( sz > min_dictionary_size )
253       {
254       const unsigned base_size = 1 << data[5];
255       const unsigned fraction = base_size / 16;
256       for( unsigned i = 7; i >= 1; --i )
257         if( base_size - ( i * fraction ) >= sz )
258           { data[5] |= ( i << 5 ); break; }
259       }
260     return true;
261     }
262 
verifyLzip_header263   bool verify( const bool ignore_bad_ds ) const
264     { return verify_magic() && verify_version() &&
265              ( ignore_bad_ds || isvalid_ds( dictionary_size() ) ); }
266   };
267 
268 
269 struct Lzip_trailer
270   {
271   uint8_t data[20];	//  0-3  CRC32 of the uncompressed data
272 			//  4-11 size of the uncompressed data
273 			// 12-19 member size including header and trailer
274   enum { size = 20 };
275 
data_crcLzip_trailer276   unsigned data_crc() const
277     {
278     unsigned tmp = 0;
279     for( int i = 3; i >= 0; --i ) { tmp <<= 8; tmp += data[i]; }
280     return tmp;
281     }
282 
data_crcLzip_trailer283   void data_crc( unsigned crc )
284     { for( int i = 0; i <= 3; ++i ) { data[i] = (uint8_t)crc; crc >>= 8; } }
285 
data_sizeLzip_trailer286   unsigned long long data_size() const
287     {
288     unsigned long long tmp = 0;
289     for( int i = 11; i >= 4; --i ) { tmp <<= 8; tmp += data[i]; }
290     return tmp;
291     }
292 
data_sizeLzip_trailer293   void data_size( unsigned long long sz )
294     { for( int i = 4; i <= 11; ++i ) { data[i] = (uint8_t)sz; sz >>= 8; } }
295 
member_sizeLzip_trailer296   unsigned long long member_size() const
297     {
298     unsigned long long tmp = 0;
299     for( int i = 19; i >= 12; --i ) { tmp <<= 8; tmp += data[i]; }
300     return tmp;
301     }
302 
member_sizeLzip_trailer303   void member_size( unsigned long long sz )
304     { for( int i = 12; i <= 19; ++i ) { data[i] = (uint8_t)sz; sz >>= 8; } }
305 
verify_consistencyLzip_trailer306   bool verify_consistency() const	// check internal consistency
307     {
308     const unsigned crc = data_crc();
309     const unsigned long long dsize = data_size();
310     if( ( crc == 0 ) != ( dsize == 0 ) ) return false;
311     const unsigned long long msize = member_size();
312     if( msize < min_member_size ) return false;
313     const unsigned long long mlimit = ( 9 * dsize + 7 ) / 8 + min_member_size;
314     if( mlimit > dsize && msize > mlimit ) return false;
315     const unsigned long long dlimit = 7090 * ( msize - 26 ) - 1;
316     if( dlimit > msize && dsize > dlimit ) return false;
317     return true;
318     }
319   };
320 
321 
322 struct Bad_byte
323   {
324   enum Mode { literal, delta, flip };
325   long long pos;
326   Mode mode;
327   uint8_t value;
328 
Bad_byteBad_byte329   Bad_byte() : pos( -1 ), mode( literal ), value( 0 ) {}
operatorBad_byte330   uint8_t operator()( const uint8_t old_value ) const
331     {
332     if( mode == delta ) return old_value + value;
333     if( mode == flip ) return old_value ^ value;
334     return value;
335     }
336   };
337 
338 
339 #ifndef INT64_MAX
340 #define INT64_MAX  0x7FFFFFFFFFFFFFFFLL
341 #endif
342 
343 class Block
344   {
345   long long pos_, size_;		// pos + size <= INT64_MAX
346 
347 public:
Block(const long long p,const long long s)348   Block( const long long p, const long long s ) : pos_( p ), size_( s ) {}
349 
pos()350   long long pos() const { return pos_; }
size()351   long long size() const { return size_; }
end()352   long long end() const { return pos_ + size_; }
353 
pos(const long long p)354   void pos( const long long p ) { pos_ = p; }
size(const long long s)355   void size( const long long s ) { size_ = s; }
356 
357   bool operator==( const Block & b ) const
358     { return pos_ == b.pos_ && size_ == b.size_; }
359   bool operator!=( const Block & b ) const
360     { return pos_ != b.pos_ || size_ != b.size_; }
361 
362   bool operator<( const Block & b ) const { return pos_ < b.pos_; }
363 
includes(const long long pos)364   bool includes( const long long pos ) const
365     { return ( pos_ <= pos && end() > pos ); }
overlaps(const Block & b)366   bool overlaps( const Block & b ) const
367     { return ( pos_ < b.end() && b.pos_ < end() ); }
overlaps(const long long pos,const long long size)368   bool overlaps( const long long pos, const long long size ) const
369     { return ( pos_ < pos + size && pos < end() ); }
370 
shift(Block & b)371   void shift( Block & b ) { ++size_; ++b.pos_; --b.size_; }
372   Block split( const long long pos );
373   };
374 
375 
376 struct Member_list	// members/gaps/tdata to be dumped/removed/stripped
377   {
378   bool damaged;
379   bool tdata;
380   bool in, rin;
381   std::vector< Block > range_vector, rrange_vector;
382 
Member_listMember_list383   Member_list() : damaged( false ), tdata( false ), in( true ), rin( true ) {}
384   void parse( const char * p );
385 
rangeMember_list386   bool range() const { return range_vector.size() || rrange_vector.size(); }
387 
388   // blocks is the sum of members + gaps, excluding trailing data
includesMember_list389   bool includes( const long i, const long blocks ) const
390     {
391     for( unsigned j = 0; j < range_vector.size(); ++j )
392       {
393       if( range_vector[j].pos() > i ) break;
394       if( range_vector[j].end() > i ) return in;
395       }
396     if( i >= 0 && i < blocks )
397       for( unsigned j = 0; j < rrange_vector.size(); ++j )
398         {
399         if( rrange_vector[j].pos() > blocks - i - 1 ) break;
400         if( rrange_vector[j].end() > blocks - i - 1 ) return rin;
401         }
402     return !in || !rin;
403     }
404   };
405 
406 
407 struct Error
408   {
409   const char * const msg;
ErrorError410   explicit Error( const char * const s ) : msg( s ) {}
411   };
412 
positive_diff(const unsigned long long x,const unsigned long long y)413 inline unsigned long long positive_diff( const unsigned long long x,
414                                          const unsigned long long y )
415   { return ( ( x > y ) ? x - y : 0 ); }
416 
set_retval(int & retval,const int new_val)417 inline void set_retval( int & retval, const int new_val )
418   { if( retval < new_val ) retval = new_val; }
419 
420 const char * const bad_magic_msg = "Bad magic number (file not in lzip format).";
421 const char * const bad_dict_msg = "Invalid dictionary size in member header.";
422 const char * const corrupt_mm_msg = "Corrupt header in multimember file.";
423 const char * const trailing_msg = "Trailing data not allowed.";
424 
425 // defined in alone_to_lz.cc
426 int alone_to_lz( const int infd, const Pretty_print & pp );
427 
428 // defined in decoder.cc
429 long long readblock( const int fd, uint8_t * const buf, const long long size );
430 long long writeblock( const int fd, const uint8_t * const buf,
431                       const long long size );
432 
433 // defined in dump_remove.cc
434 int dump_members( const std::vector< std::string > & filenames,
435                   const std::string & default_output_filename,
436                   const Member_list & member_list, const bool force,
437                   bool ignore_errors, bool ignore_trailing,
438                   const bool loose_trailing, const bool strip,
439                   const bool to_stdout );
440 int remove_members( const std::vector< std::string > & filenames,
441                     const Member_list & member_list, bool ignore_errors,
442                     bool ignore_trailing, const bool loose_trailing );
443 
444 // defined in list.cc
445 int list_files( const std::vector< std::string > & filenames,
446                 const bool ignore_errors,
447                 const bool ignore_trailing, const bool loose_trailing );
448 
449 // defined in lzip_index.cc
450 int seek_read( const int fd, uint8_t * const buf, const int size,
451                const long long pos );
452 
453 // defined in lunzcrash.cc
454 int lunzcrash( const std::string & input_filename );
455 int md5sum_files( const std::vector< std::string > & filenames );
456 
457 // defined in main.cc
458 extern const char * const program_name;
459 extern std::string output_filename;	// global vars for output file
460 extern int outfd;
461 struct stat;
462 const char * bad_version( const unsigned version );
463 const char * format_ds( const unsigned dictionary_size );
464 void show_header( const unsigned dictionary_size );
465 int open_instream( const char * const name, struct stat * const in_statsp,
466                    const bool one_to_one, const bool reg_only = false );
467 int open_truncable_stream( const char * const name,
468                            struct stat * const in_statsp );
469 bool open_outstream( const bool force, const bool protect,
470                      const bool rw = false, const bool skipping = true );
471 bool file_exists( const std::string & filename );
472 void cleanup_and_fail( const int retval );
473 void set_signal_handler();
474 int close_outstream( const struct stat * const in_statsp );
475 std::string insert_fixed( std::string name );
476 void show_error( const char * const msg, const int errcode = 0,
477                  const bool help = false );
478 void show_file_error( const char * const filename, const char * const msg,
479                       const int errcode = 0 );
480 void internal_error( const char * const msg );
481 void show_2file_error( const char * const msg1, const char * const name1,
482                        const char * const name2, const char * const msg2 );
483 class Range_decoder;
484 void show_dprogress( const unsigned long long cfile_size = 0,
485                      const unsigned long long partial_size = 0,
486                      const Range_decoder * const d = 0,
487                      const Pretty_print * const p = 0 );
488 
489 // defined in merge.cc
490 bool copy_file( const int infd, const int outfd,
491                 const long long max_size = -1 );
492 int test_member_from_file( const int infd, const unsigned long long msize,
493                            long long * const failure_posp = 0 );
494 int merge_files( const std::vector< std::string > & filenames,
495                  const std::string & default_output_filename,
496                  const char terminator, const bool force );
497 
498 // defined in nrep_stats.cc
499 int print_nrep_stats( const std::vector< std::string > & filenames,
500                       const int repeated_byte, const bool ignore_errors,
501                       const bool ignore_trailing, const bool loose_trailing );
502 
503 // defined in range_dec.cc
504 const char * format_num( unsigned long long num,
505                          unsigned long long limit = -1ULL,
506                          const int set_prefix = 0 );
507 bool safe_seek( const int fd, const long long pos );
508 int range_decompress( const std::string & input_filename,
509                       const std::string & default_output_filename,
510                       Block range, const bool force, const bool ignore_errors,
511                       const bool ignore_trailing, const bool loose_trailing,
512                       const bool to_stdout );
513 
514 // defined in repair.cc
515 long long seek_write( const int fd, const uint8_t * const buf,
516                       const long long size, const long long pos );
517 uint8_t * read_member( const int infd, const long long mpos,
518                        const long long msize );
519 int repair_file( const std::string & input_filename,
520                  const std::string & default_output_filename,
521                  const char terminator, const bool force );
522 int debug_delay( const std::string & input_filename, Block range,
523                  const char terminator );
524 int debug_repair( const std::string & input_filename,
525                   const Bad_byte & bad_byte, const char terminator );
526 int debug_decompress( const std::string & input_filename,
527                       const Bad_byte & bad_byte, const bool show_packets );
528 
529 // defined in reproduce.cc
530 int reproduce_file( const std::string & input_filename,
531                     const std::string & default_output_filename,
532                     const char * const lzip_name,
533                     const char * const reference_filename,
534                     const int lzip_level, const char terminator,
535                     const bool force );
536 int debug_reproduce_file( const std::string & input_filename,
537                           const char * const lzip_name,
538                           const char * const reference_filename,
539                           const Block & range, const int sector_size,
540                           const int lzip_level );
541 
542 // defined in split.cc
543 int split_file( const std::string & input_filename,
544                 const std::string & default_output_filename, const bool force );
545