1
2 /***************************************************************************
3 * __ __ _ ___________ *
4 * \ \ / /| |____ ____| *
5 * \ \ / / | | | | *
6 * \ \ /\ / / | | | | *
7 * \ \/ \/ / | | | | *
8 * \ /\ / | | | | *
9 * \/ \/ |_| |_| *
10 * *
11 * Wiimms ISO Tools *
12 * http://wit.wiimm.de/ *
13 * *
14 ***************************************************************************
15 * *
16 * This file is part of the WIT project. *
17 * Visit http://wit.wiimm.de/ for project details and sources. *
18 * *
19 * Copyright (c) 2009-2013 by Dirk Clemens <wiimm@wiimm.de> *
20 * *
21 ***************************************************************************
22 * *
23 * This program is free software; you can redistribute it and/or modify *
24 * it under the terms of the GNU General Public License as published by *
25 * the Free Software Foundation; either version 2 of the License, or *
26 * (at your option) any later version. *
27 * *
28 * This program is distributed in the hope that it will be useful, *
29 * but WITHOUT ANY WARRANTY; without even the implied warranty of *
30 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
31 * GNU General Public License for more details. *
32 * *
33 * See file gpl-2.0.txt or http://www.gnu.org/licenses/gpl-2.0.txt *
34 * *
35 ***************************************************************************/
36
37 #define _GNU_SOURCE 1
38
39 #include "lib-lzma.h"
40 #include "lzma/LzmaEnc.h"
41 #include "lzma/LzmaDec.h"
42 #include "lzma/Lzma2Enc.h"
43 #include "lzma/Lzma2Dec.h"
44
45 /***********************************************
46 ** LZMA SDK: http://www.7-zip.org/sdk.html **
47 ***********************************************/
48
49 //
50 ///////////////////////////////////////////////////////////////////////////////
51 /////////////// debugging & logging ///////////////
52 ///////////////////////////////////////////////////////////////////////////////
53
54 #if defined(TEST) && defined(DEBUG)
55 #define LOG_ALLOC 0
56 // 0: off
57 // 1: show only max value
58 // 2: show all values
59 #else
60 #define LOG_ALLOC 0
61 #endif
62
63 //
64 ///////////////////////////////////////////////////////////////////////////////
65 /////////////// LZMA helpers ///////////////
66 ///////////////////////////////////////////////////////////////////////////////
67
GetMessageLZMA(int err,ccp unkown_error)68 ccp GetMessageLZMA
69 (
70 int err, // error code
71 ccp unkown_error // result for unkown error codes
72 )
73 {
74 switch (err)
75 {
76 case SZ_OK: return "OK";
77
78 case SZ_ERROR_ARCHIVE: return "Archive error";
79 case SZ_ERROR_CRC: return "CRC error";
80 case SZ_ERROR_DATA: return "Data error";
81 case SZ_ERROR_FAIL: return "Operation failed";
82 case SZ_ERROR_INPUT_EOF: return "Need more input";
83 case SZ_ERROR_MEM: return "Memory allocation error";
84 case SZ_ERROR_NO_ARCHIVE: return "No archive";
85 case SZ_ERROR_OUTPUT_EOF: return "Putput buffer overflow";
86 case SZ_ERROR_PARAM: return "Incorrect parameter";
87 case SZ_ERROR_PROGRESS: return "Some break from progress callback";
88 case SZ_ERROR_READ: return "Reading error";
89 case SZ_ERROR_THREAD: return "Errors in multithreading functions";
90 case SZ_ERROR_UNSUPPORTED: return "Unsupported properties";
91 case SZ_ERROR_WRITE: return "Writing error";
92 }
93
94 return unkown_error;
95 };
96
97 ///////////////////////////////////////////////////////////////////////////////
98
CalcCompressionLevelLZMA(int compr_level)99 int CalcCompressionLevelLZMA
100 (
101 int compr_level // valid are 1..9 / 0: use default value
102 )
103 {
104 return compr_level < 1
105 ? 5
106 : compr_level < 7
107 ? compr_level
108 : 7;
109 }
110
111 ///////////////////////////////////////////////////////////////////////////////
112
CalcMemoryUsageLZMA(int compr_level,bool is_writing)113 u32 CalcMemoryUsageLZMA
114 (
115 int compr_level, // valid are 1..9 / 0: use default value
116 bool is_writing // false: reading mode, true: writing mode
117 )
118 {
119 static u32 read_size[] =
120 {
121 0, // not used
122 97496, // level 1
123 294104, // level 2
124 1080536, // level 3
125 4226264, // level 4
126 16809176, // level 5
127 33586392, // level 6
128 67140824, // level 7
129 };
130
131 static u32 write_size[] =
132 {
133 0, // not used
134 1831198, // level 1
135 3174686, // level 2
136 9072926, // level 3
137 32665886, // level 4
138 194146594, // level 5
139 387084578, // level 6
140 705851730, // level 7
141 };
142
143 DASSERT( sizeof(write_size) == sizeof(read_size) );
144
145
146 compr_level = CalcCompressionLevelLZMA(compr_level);
147 if ( compr_level <= 0 )
148 compr_level = 1;
149 else
150 {
151 const size_t N = sizeof(write_size) / sizeof(*write_size);
152 if ( compr_level >= N )
153 compr_level = N-1;
154 }
155
156 u32 * tab = is_writing ? write_size : read_size;
157 return tab[compr_level];
158 }
159
160 ///////////////////////////////////////////////////////////////////////////////
161
CalcMemoryUsageLZMA2(int compr_level,bool is_writing)162 u32 CalcMemoryUsageLZMA2
163 (
164 int compr_level, // valid are 1..9 / 0: use default value
165 bool is_writing // false: reading mode, true: writing mode
166 )
167 {
168 static u32 read_size[] =
169 {
170 0, // not used
171 122072, // level 1
172 318680, // level 2
173 1105112, // level 3
174 4250840, // level 4
175 16833752, // level 5
176 33610968, // level 6
177 67165400, // level 7
178 };
179
180 static u32 write_size[] =
181 {
182 0, // not used
183 4938158, // level 1
184 5986734, // level 2
185 10705326, // level 3
186 32731566, // level 4
187 194212274, // level 5
188 387150258, // level 6
189 705917410, // level 7
190 };
191
192 DASSERT( sizeof(write_size) == sizeof(read_size) );
193
194
195 compr_level = CalcCompressionLevelLZMA(compr_level);
196 if ( compr_level <= 0 )
197 compr_level = 1;
198 else
199 {
200 const size_t N = sizeof(write_size) / sizeof(*write_size);
201 if ( compr_level >= N )
202 compr_level = N-1;
203 }
204
205 u32 * tab = is_writing ? write_size : read_size;
206 return tab[compr_level];
207 }
208
209 ///////////////////////////////////////////////////////////////////////////////
210
211 #if LOG_ALLOC
212 static u32 alloc_count = 0;
213 static u32 free_count = 0;
214 static u32 alloc_size = 0;
215 #endif
216
AllocLZMA(void * p,size_t size)217 static void * AllocLZMA ( void *p, size_t size )
218 {
219 #if LOG_ALLOC
220 void * ptr = MALLOC(size);
221 alloc_count++;
222 alloc_size += size;
223 #if LOG_ALLOC > 1
224 PRINT("$$$ LZMA ALLOC [%p]: %zu/%u [%u-%u=%u]\n",
225 ptr, size, alloc_size, alloc_count, free_count, alloc_count-free_count );
226 #endif
227 return ptr;
228 #else
229 return MALLOC(size);
230 #endif
231 }
232
FreeLZMA(void * p,void * ptr)233 static void FreeLZMA ( void *p, void *ptr )
234 {
235 #if LOG_ALLOC
236 if (ptr)
237 {
238 free_count++;
239 if ( LOG_ALLOC > 1 || free_count == 1 )
240 PRINT("$$$ LZMA FREE [%p]: %u [%u-%u=%u]\n",
241 ptr, alloc_size, alloc_count, free_count, alloc_count-free_count );
242 }
243 #endif
244 FREE(ptr);
245 }
246
247 static ISzAlloc lzma_alloc = { AllocLZMA, FreeLZMA };
248
249 //
250 ///////////////////////////////////////////////////////////////////////////////
251 /////////////// static LZMA write helpers ///////////////
252 ///////////////////////////////////////////////////////////////////////////////
253
254 typedef struct sz_inbuf_t
255 {
256 ISeqInStream func; // pounter to read function
257 DataList_t * data; // pointer to data list
258 u32 bytes_read; // total read
259
260 } sz_inbuf_t;
261
262 //-----------------------------------------------------------------------------
263
sz_read_buf(void * pp,void * buf,size_t * size)264 static SRes sz_read_buf ( void *pp, void *buf, size_t *size )
265 {
266 DASSERT(pp);
267 DASSERT(size);
268 sz_inbuf_t * ibuf = pp;
269 noPRINT("$$$ sz_read_buf(%p,%p,%zx=%zu)\n",ibuf,buf,*size,*size);
270
271 ibuf->bytes_read += *size = ReadDataList(ibuf->data,buf,*size);
272 noPRINT("sz_read_buf() size = %zu\n",*size);
273 return SZ_OK;
274 }
275
276 ///////////////////////////////////////////////////////////////////////////////
277
278 typedef struct sz_outfile_t
279 {
280 ISeqOutStream func;
281 File_t * file;
282 u32 bytes_written;
283
284 } sz_outfile_t;
285
286 //-----------------------------------------------------------------------------
287
sz_write_file(void * pp,const void * data,size_t size)288 static size_t sz_write_file ( void *pp, const void *data, size_t size )
289 {
290 DASSERT(pp);
291 sz_outfile_t * obuf = pp;
292 noPRINT("$$$ sz_write_file(%p->%p,%p,%zx=%zu)\n",obuf,obuf->file,data,size,size);
293 obuf->bytes_written += size;
294 obuf->file->bytes_written -= size;
295 return SIGINT_level>1 || WriteF(obuf->file,data,size) ? 0 : size;
296 }
297
298 ///////////////////////////////////////////////////////////////////////////////
299
300 typedef struct sz_progress_t
301 {
302 ICompressProgress func; // progress function
303 SuperFile_t * file; // file for progress output
304
305 } sz_progress_t;
306
307 //-----------------------------------------------------------------------------
308
sz_progress(void * pp,UInt64 in_size,UInt64 out_size)309 SRes sz_progress ( void *pp, UInt64 in_size, UInt64 out_size )
310 {
311 // *_size (UInt64)(Int64)-1 for size means unknown value.
312
313 if ( in_size != ~(UInt64)0 )
314 {
315 sz_progress_t * prog = (sz_progress_t*)pp;
316 DASSERT( prog );
317 DASSERT( prog->func.Progress == sz_progress );
318 DASSERT( prog->file );
319
320 noPRINT("$$$ sz_progress(%10llu,%10llu) => %10llu +%10llu =%10llu /%10llu\n",
321 in_size, out_size,
322 f->bytes_read, f->bytes_written, f->bytes_read + f->bytes_written,
323 prog->file->progress_last_total );
324
325 PrintProgressChunkSF(prog->file,in_size);
326 }
327
328 return SZ_OK; // other than SZ_OK means: terminate compression
329 }
330
331 //
332 ///////////////////////////////////////////////////////////////////////////////
333 /////////////// LZMA encoding (compression) ///////////////
334 ///////////////////////////////////////////////////////////////////////////////
335
EncLZMA_Open(EncLZMA_t * lzma,ccp error_object,int compr_level,bool write_endmark)336 enumError EncLZMA_Open
337 (
338 EncLZMA_t * lzma, // object, will be initialized
339 ccp error_object, // object name for error messages
340 int compr_level, // valid are 1..9 / 0: use default value
341 bool write_endmark // true: write end marker at end of stream
342 )
343 {
344 DASSERT(lzma);
345 memset(lzma,0,sizeof(*lzma));
346 lzma->error_object = error_object ? error_object : "?";
347
348 #if LOG_ALLOC
349 alloc_count = 0;
350 free_count = 0;
351 alloc_size = 0;
352 #endif
353
354
355 //----- create handle
356
357 lzma->handle = LzmaEnc_Create(&lzma_alloc);
358 if (!lzma->handle)
359 return ERROR0(ERR_LZMA,
360 "Error while opening LZMA stream: %s\n-> LZMA error: %s\n",
361 lzma->error_object, GetMessageLZMA(SZ_ERROR_MEM,"?") );
362
363
364 //----- compression method and properties
365
366 CLzmaEncProps props;
367 LzmaEncProps_Init(&props);
368 lzma->compr_level = CalcCompressionLevelLZMA(compr_level);
369 props.level = lzma->compr_level;
370 props.writeEndMark = write_endmark;
371
372 SRes res = LzmaEnc_SetProps(lzma->handle,&props);
373 if ( res != SZ_OK )
374 {
375 EncLZMA_Close(lzma);
376 return ERROR0(ERR_LZMA,
377 "Error while setup LZMA properties: %s\n-> LZMA error: %s\n",
378 lzma->error_object, GetMessageLZMA(res,"?") );
379 }
380
381
382 //----- store encoded properties
383
384 DASSERT( sizeof(lzma->enc_props) >= LZMA_PROPS_SIZE );
385 lzma->enc_props_len = sizeof(lzma->enc_props);
386 res = LzmaEnc_WriteProperties(lzma->handle,lzma->enc_props,&lzma->enc_props_len);
387 if ( res != SZ_OK )
388 {
389 EncLZMA_Close(lzma);
390 return ERROR0(ERR_LZMA,
391 "Error while writing LZMA properties: %s\n-> LZMA error: %s\n",
392 lzma->error_object, GetMessageLZMA(res,"?") );
393 }
394
395 noPRINT("EncLZMA_Open() done: prop-size=%zd\n", lzma->enc_props_len );
396
397 return ERR_OK;
398 };
399
400 ///////////////////////////////////////////////////////////////////////////////
401
EncLZMA_WriteDataToFile(EncLZMA_t * lzma,SuperFile_t * file,bool write_props,const void * data,size_t data_size,u32 * bytes_written)402 enumError EncLZMA_WriteDataToFile
403 (
404 EncLZMA_t * lzma, // valid pointer, opened with EncLZMA_Open()
405 SuperFile_t * file, // destination file and progress support
406 // -> write to 'file->f' at current offset
407 bool write_props, // true: write encoding properties
408 const void * data, // data to write
409 size_t data_size, // size of data to write
410 u32 * bytes_written // not NULL: store written bytes
411 )
412 {
413 DataArea_t area[2];
414 memset(&area,0,sizeof(area));
415 area->data = data;
416 area->size = data_size;
417
418 DataList_t list;
419 SetupDataList(&list,area);
420
421 return EncLZMA_WriteList2File(lzma,file,write_props,&list,bytes_written);
422 }
423
424 ///////////////////////////////////////////////////////////////////////////////
425
EncLZMA_WriteList2File(EncLZMA_t * lzma,SuperFile_t * file,bool write_props,DataList_t * data_list,u32 * bytes_written)426 enumError EncLZMA_WriteList2File
427 (
428 EncLZMA_t * lzma, // valid pointer, opened with EncLZMA_Open()
429 SuperFile_t * file, // destination file and progress support
430 // -> write to 'file->f' at current offset
431 bool write_props, // true: write encoding properties
432 DataList_t * data_list, // NULL or data list (modified)
433 u32 * bytes_written // not NULL: store written bytes
434 )
435 {
436 DASSERT(lzma);
437 DASSERT(lzma->handle);
438 DASSERT(file);
439
440 if (write_props)
441 {
442 enumError err = WriteF(&file->f,lzma->enc_props,lzma->enc_props_len);
443 if (err)
444 return err;
445 }
446
447 sz_inbuf_t inbuf;
448 inbuf.func.Read = sz_read_buf;
449 inbuf.data = data_list;
450 inbuf.bytes_read = 0;
451
452 sz_outfile_t outfile;
453 outfile.func.Write = sz_write_file;
454 outfile.file = &file->f;
455 outfile.bytes_written = write_props ? lzma->enc_props_len : 0;
456
457 sz_progress_t progress;
458 progress.func.Progress = sz_progress;
459 progress.file = file;
460
461 SRes res = LzmaEnc_Encode( lzma->handle,
462 (ISeqOutStream*)&outfile,
463 (ISeqInStream*)&inbuf,
464 (ICompressProgress*)&progress,
465 &lzma_alloc, &lzma_alloc );
466 if ( res != SZ_OK )
467 {
468 EncLZMA_Close(lzma);
469 return ERROR0(ERR_LZMA,
470 "Error while writing LZMA stream: %s\n-> LZMA error: %s\n",
471 lzma->error_object, GetMessageLZMA(res,"?") );
472 }
473
474 // count only uncomressed size -> separate operations because u32/u64 handling
475 file->f.bytes_written += inbuf.bytes_read;
476
477 if (bytes_written)
478 *bytes_written = outfile.bytes_written;
479 return ERR_OK;
480 }
481
482 ///////////////////////////////////////////////////////////////////////////////
483
EncLZMA_Close(EncLZMA_t * lzma)484 enumError EncLZMA_Close
485 (
486 EncLZMA_t * lzma // valid pointer
487 )
488 {
489 DASSERT(lzma);
490 if ( lzma->handle )
491 {
492 LzmaEnc_Destroy(lzma->handle,&lzma_alloc,&lzma_alloc);
493 lzma->handle = 0;
494 }
495
496 return ERR_OK;
497 }
498
499 ///////////////////////////////////////////////////////////////////////////////
500
EncLZMA_Data2File(EncLZMA_t * lzma,SuperFile_t * file,int compr_level,bool write_props,bool write_endmark,const void * data,size_t data_size,u32 * bytes_written)501 enumError EncLZMA_Data2File // open + write + close lzma stream
502 (
503 EncLZMA_t * lzma, // if NULL: use internal structure
504 SuperFile_t * file, // destination file and progress support
505 // -> write to 'file->f' at current offset
506 int compr_level, // valid are 1..9 / 0: use default value
507 bool write_props, // true: write encoding properties
508 bool write_endmark, // true: write end marker at end of stream
509 const void * data, // data to write
510 size_t data_size, // size of data to write
511 u32 * bytes_written // not NULL: store written bytes
512 )
513 {
514 DataArea_t area[2];
515 memset(&area,0,sizeof(area));
516 area->data = data;
517 area->size = data_size;
518
519 DataList_t list;
520 SetupDataList(&list,area);
521
522 return EncLZMA_List2File(lzma,file,compr_level,
523 write_props,write_endmark,&list,bytes_written);
524 }
525
526 ///////////////////////////////////////////////////////////////////////////////
527
EncLZMA_List2File(EncLZMA_t * lzma,SuperFile_t * file,int compr_level,bool write_props,bool write_endmark,DataList_t * data_list,u32 * bytes_written)528 enumError EncLZMA_List2File // open + write + close lzma stream
529 (
530 EncLZMA_t * lzma, // if NULL: use internal structure
531 SuperFile_t * file, // destination file and progress support
532 // -> write to 'file->f' at current offset
533 int compr_level, // valid are 1..9 / 0: use default value
534 bool write_props, // true: write encoding properties
535 bool write_endmark, // true: write end marker at end of stream
536 DataList_t * data_list, // NULL or data list (modified)
537 u32 * bytes_written // not NULL: store written bytes
538 )
539 {
540 DASSERT(file);
541
542 EncLZMA_t internal_lzma;
543 if (!lzma)
544 lzma = &internal_lzma;
545
546 enumError err = EncLZMA_Open(lzma,file->f.fname,compr_level,write_endmark);
547 if (err)
548 return err;
549
550 err = EncLZMA_WriteList2File(lzma,file,write_props,data_list,bytes_written);
551 if (err)
552 return err;
553
554 return EncLZMA_Close(lzma);
555 }
556
557 //
558 ///////////////////////////////////////////////////////////////////////////////
559 /////////////// LZMA decoding (decompression) ///////////////
560 ///////////////////////////////////////////////////////////////////////////////
561
DecLZMA_File2Buf(SuperFile_t * file,size_t read_count,void * buf,size_t buf_size,u32 * bytes_written,u8 * enc_props)562 enumError DecLZMA_File2Buf // open + read + close lzma stream
563 (
564 SuperFile_t * file, // source file and progress support
565 // -> read from 'file->f' at current offset
566 size_t read_count, // not NULL: max bytes to read from file
567 void * buf, // destination buffer
568 size_t buf_size, // size of destination buffer
569 u32 * bytes_written,// not NULL: store bytes written to buf
570 u8 * enc_props // Encoding properties
571 // If NULL: read it from file
572 )
573 {
574 DASSERT(buf);
575 DASSERT(file);
576 File_t * f = &file->f;
577
578 #if LOG_ALLOC
579 alloc_count = 0;
580 free_count = 0;
581 alloc_size = 0;
582 #endif
583
584 u8 prop_buf[LZMA_PROPS_SIZE];
585 if (!enc_props)
586 {
587 const enumError err = ReadF(&file->f,prop_buf,sizeof(prop_buf));
588 if (err)
589 return err;
590 enc_props = prop_buf;
591 }
592
593 CLzmaDec lzma;
594 LzmaDec_Construct(&lzma);
595 SRes res = LzmaDec_Allocate(&lzma,enc_props,LZMA_PROPS_SIZE,&lzma_alloc);
596 if ( res != SZ_OK )
597 return ERROR0(ERR_LZMA,
598 "Error while setup LZMA properties: %s\n-> LZMA error: %s\n",
599 f->fname, GetMessageLZMA(res,"?") );
600
601 enumError err = ERR_OK;
602 u8 in_buf[0x10000];
603 size_t in_buf_len = 0;
604
605 const int read_behind_eof = f->read_behind_eof;
606 if ( !read_count && f->seek_allowed && f->st.st_size )
607 read_count = f->st.st_size - f->cur_off;
608 const bool have_max_read = read_count > 0;
609 if (!have_max_read)
610 f->read_behind_eof = 2;
611
612 u8 * dest = buf;
613 u32 written = 0;
614
615 LzmaDec_Init(&lzma);
616 for(;;)
617 {
618 // [[2do]] [progress]
619
620 //--- fill input buffer
621
622 ELzmaFinishMode finish = LZMA_FINISH_ANY;
623
624 DASSERT( in_buf_len >= 0 && in_buf_len < sizeof(in_buf) );
625 size_t read_size = sizeof(in_buf) - in_buf_len;
626 if ( have_max_read && read_size > read_count )
627 {
628 read_size = read_count;
629 finish = LZMA_FINISH_END;
630 }
631 noPRINT("READ off=%llx, size=%zx=%zu, to=%zu\n",
632 (u64)f->cur_off, read_size, read_size, in_buf_len );
633 err = ReadF(f,in_buf+in_buf_len,read_size);
634 if (err)
635 return err;
636 in_buf_len += read_size;
637 read_count -= read_size;
638 f->bytes_read -= read_size; // count only decompressed data
639
640 if (!in_buf_len)
641 break;
642
643
644 //--- decode
645
646 size_t in_len = in_buf_len;
647 size_t out_len = buf_size;
648 ELzmaStatus status;
649
650 res = LzmaDec_DecodeToBuf(&lzma,dest,&out_len,in_buf,&in_len,finish,&status);
651 noPRINT("DECODED, res=%s, stat=%d, in=%zu/%zu out=%zu/%zu\n",
652 GetMessageLZMA(res,"?"), status,
653 in_len, in_buf_len, out_len, buf_size );
654
655 if ( res == SZ_OK && !in_len && !out_len && status != LZMA_STATUS_FINISHED_WITH_MARK )
656 res = SZ_ERROR_DATA;
657
658 if ( res != SZ_OK )
659 return ERROR0(ERR_LZMA,
660 "Error while reading LZMA stream: %s\n-> LZMA error: %s\n",
661 f->fname, GetMessageLZMA(res,"?") );
662
663 written += out_len;
664 dest += out_len;
665 buf_size -= out_len;
666 f->bytes_read += out_len; // count only decompressed data
667
668 if ( in_len < in_buf_len )
669 {
670 in_buf_len -= in_len;
671 memmove(in_buf,in_buf+in_len,in_buf_len);
672 }
673 else
674 in_buf_len = 0;
675
676 if ( status == LZMA_STATUS_FINISHED_WITH_MARK )
677 break;
678 }
679
680 f->read_behind_eof = read_behind_eof;
681 LzmaDec_Free(&lzma,&lzma_alloc);
682
683 if (bytes_written)
684 *bytes_written = written;
685 return err;
686 }
687
688 //
689 ///////////////////////////////////////////////////////////////////////////////
690 /////////////// LZMA2 encoding (compression) ///////////////
691 ///////////////////////////////////////////////////////////////////////////////
692
EncLZMA2_Open(EncLZMA_t * lzma,ccp error_object,int compr_level,bool write_endmark)693 enumError EncLZMA2_Open
694 (
695 EncLZMA_t * lzma, // object, will be initialized
696 ccp error_object, // object name for error messages
697 int compr_level, // valid are 1..9 / 0: use default value
698 bool write_endmark // true: write end marker at end of stream
699 )
700 {
701 DASSERT(lzma);
702 memset(lzma,0,sizeof(*lzma));
703 lzma->error_object = error_object ? error_object : "?";
704
705 #if LOG_ALLOC
706 alloc_count = 0;
707 free_count = 0;
708 alloc_size = 0;
709 #endif
710
711
712 //----- create handle
713
714 lzma->handle = Lzma2Enc_Create(&lzma_alloc,&lzma_alloc);
715 if (!lzma->handle)
716 return ERROR0(ERR_LZMA,
717 "Error while opening LZMA2 stream: %s\n-> LZMA2 error: %s\n",
718 lzma->error_object, GetMessageLZMA(SZ_ERROR_MEM,"?") );
719
720
721 //----- compression method and properties
722
723 CLzma2EncProps props;
724 Lzma2EncProps_Init(&props);
725 lzma->compr_level = CalcCompressionLevelLZMA(compr_level);
726 props.lzmaProps.level = lzma->compr_level;
727 props.lzmaProps.writeEndMark = write_endmark;
728
729 SRes res = Lzma2Enc_SetProps(lzma->handle,&props);
730 if ( res != SZ_OK )
731 {
732 EncLZMA2_Close(lzma);
733 return ERROR0(ERR_LZMA,
734 "Error while setup LZMA2 properties: %s\n-> LZMA2 error: %s\n",
735 lzma->error_object, GetMessageLZMA(res,"?") );
736 }
737
738
739 //----- store encoded properties
740
741 DASSERT( sizeof(lzma->enc_props) >= LZMA_PROPS_SIZE );
742 lzma->enc_props_len = sizeof(lzma->enc_props);
743 lzma->enc_props[0] = Lzma2Enc_WriteProperties(lzma->handle);
744 lzma->enc_props_len = 1;
745
746 noPRINT("EncLZMA2_Open() done: prop-size=%zd, byte=%02x\n",
747 lzma->enc_props_len, lzma->enc_props[0] );
748
749 return ERR_OK;
750 };
751
752 ///////////////////////////////////////////////////////////////////////////////
753
EncLZMA2_WriteDataToFile(EncLZMA_t * lzma,SuperFile_t * file,bool write_props,const void * data,size_t data_size,u32 * bytes_written)754 enumError EncLZMA2_WriteDataToFile
755 (
756 EncLZMA_t * lzma, // valid pointer, opened with EncLZMA2_Open()
757 SuperFile_t * file, // destination file and progress support
758 // -> write to 'file->f' at current offset
759 bool write_props, // true: write encoding properties
760 const void * data, // data to write
761 size_t data_size, // size of data to write
762 u32 * bytes_written // not NULL: store written bytes
763 )
764 {
765 DataArea_t area[2];
766 memset(&area,0,sizeof(area));
767 area->data = data;
768 area->size = data_size;
769
770 DataList_t list;
771 SetupDataList(&list,area);
772
773 return EncLZMA2_WriteList2File(lzma,file,write_props,&list,bytes_written);
774 }
775
776 ///////////////////////////////////////////////////////////////////////////////
777
EncLZMA2_WriteList2File(EncLZMA_t * lzma,SuperFile_t * file,bool write_props,DataList_t * data_list,u32 * bytes_written)778 enumError EncLZMA2_WriteList2File
779 (
780 EncLZMA_t * lzma, // valid pointer, opened with EncLZMA2_Open()
781 SuperFile_t * file, // destination file and progress support
782 // -> write to 'file->f' at current offset
783 bool write_props, // true: write encoding properties
784 DataList_t * data_list, // NULL or data list (modified)
785 u32 * bytes_written // not NULL: store written bytes
786 )
787 {
788 DASSERT(lzma);
789 DASSERT(lzma->handle);
790 DASSERT(file);
791
792 if (write_props)
793 {
794 enumError err = WriteF(&file->f,lzma->enc_props,lzma->enc_props_len);
795 if (err)
796 return err;
797 }
798
799 sz_inbuf_t inbuf;
800 inbuf.func.Read = sz_read_buf;
801 inbuf.data = data_list;
802 inbuf.bytes_read = 0;
803
804 sz_outfile_t outfile;
805 outfile.func.Write = sz_write_file;
806 outfile.file = &file->f;
807 outfile.bytes_written = write_props ? lzma->enc_props_len : 0;
808
809 sz_progress_t progress;
810 progress.func.Progress = sz_progress;
811 progress.file = file;
812
813 SRes res = Lzma2Enc_Encode( lzma->handle,
814 (ISeqOutStream*)&outfile,
815 (ISeqInStream*)&inbuf,
816 (ICompressProgress*)&progress );
817 if ( res != SZ_OK )
818 {
819 EncLZMA2_Close(lzma);
820 return ERROR0(ERR_LZMA,
821 "Error while writing LZMA2 stream: %s\n-> LZMA2 error: %s\n",
822 lzma->error_object, GetMessageLZMA(res,"?") );
823 }
824
825 // count only uncomressed size -> separate operations because u32/u64 handling
826 file->f.bytes_written += inbuf.bytes_read;
827
828 if (bytes_written)
829 *bytes_written = outfile.bytes_written;
830 return ERR_OK;
831 }
832
833 ///////////////////////////////////////////////////////////////////////////////
834
EncLZMA2_Close(EncLZMA_t * lzma)835 enumError EncLZMA2_Close
836 (
837 EncLZMA_t * lzma // valid pointer
838 )
839 {
840 DASSERT(lzma);
841 if ( lzma->handle )
842 {
843 Lzma2Enc_Destroy(lzma->handle);
844 lzma->handle = 0;
845 }
846
847 return ERR_OK;
848 }
849
850 ///////////////////////////////////////////////////////////////////////////////
851
EncLZMA2_Data2File(EncLZMA_t * lzma,SuperFile_t * file,int compr_level,bool write_props,bool write_endmark,const void * data,size_t data_size,u32 * bytes_written)852 enumError EncLZMA2_Data2File // open + write + close lzma stream
853 (
854 EncLZMA_t * lzma, // if NULL: use internal structure
855 SuperFile_t * file, // destination file and progress support
856 // -> write to 'file->f' at current offset
857 int compr_level, // valid are 1..9 / 0: use default value
858 bool write_props, // true: write encoding properties
859 bool write_endmark, // true: write end marker at end of stream
860 const void * data, // data to write
861 size_t data_size, // size of data to write
862 u32 * bytes_written // not NULL: store written bytes
863 )
864 {
865 DataArea_t area[2];
866 memset(&area,0,sizeof(area));
867 area->data = data;
868 area->size = data_size;
869
870 DataList_t list;
871 SetupDataList(&list,area);
872
873 return EncLZMA2_List2File(lzma,file,compr_level,
874 write_props,write_endmark,&list,bytes_written);
875 }
876
877 ///////////////////////////////////////////////////////////////////////////////
878
EncLZMA2_List2File(EncLZMA_t * lzma,SuperFile_t * file,int compr_level,bool write_props,bool write_endmark,DataList_t * data_list,u32 * bytes_written)879 enumError EncLZMA2_List2File // open + write + close lzma stream
880 (
881 EncLZMA_t * lzma, // if NULL: use internal structure
882 SuperFile_t * file, // destination file and progress support
883 // -> write to 'file->f' at current offset
884 int compr_level, // valid are 1..9 / 0: use default value
885 bool write_props, // true: write encoding properties
886 bool write_endmark, // true: write end marker at end of stream
887 DataList_t * data_list, // NULL or data list (modified)
888 u32 * bytes_written // not NULL: store written bytes
889 )
890 {
891 DASSERT(file);
892
893 EncLZMA_t internal_lzma;
894 if (!lzma)
895 lzma = &internal_lzma;
896
897 enumError err = EncLZMA2_Open(lzma,file->f.fname,compr_level,write_endmark);
898 if (err)
899 return err;
900
901 err = EncLZMA2_WriteList2File(lzma,file,write_props,data_list,bytes_written);
902 if (err)
903 return err;
904
905 return EncLZMA2_Close(lzma);
906 }
907
908 //
909 ///////////////////////////////////////////////////////////////////////////////
910 /////////////// LZMA2 decoding (decompression) ///////////////
911 ///////////////////////////////////////////////////////////////////////////////
912
DecLZMA2_File2Buf(SuperFile_t * file,size_t read_count,void * buf,size_t buf_size,u32 * bytes_written,u8 * enc_props)913 enumError DecLZMA2_File2Buf // open + read + close lzma stream
914 (
915 SuperFile_t * file, // source file and progress support
916 // -> read from 'file->f' at current offset
917 size_t read_count, // not NULL: max bytes to read from file
918 void * buf, // destination buffer
919 size_t buf_size, // size of destination buffer
920 u32 * bytes_written,// not NULL: store bytes written to buf
921 u8 * enc_props // Encoding properties
922 // If NULL: read it from file
923 )
924 {
925 DASSERT(buf);
926 DASSERT(file);
927 File_t * f = &file->f;
928
929 #if LOG_ALLOC
930 alloc_count = 0;
931 free_count = 0;
932 alloc_size = 0;
933 #endif
934
935 u8 prop_buf[1];
936 if (!enc_props)
937 {
938 const enumError err = ReadF(f,prop_buf,sizeof(prop_buf));
939 if (err)
940 return err;
941 enc_props = prop_buf;
942 }
943
944 CLzma2Dec lzma;
945 Lzma2Dec_Construct(&lzma);
946 SRes res = Lzma2Dec_Allocate(&lzma,*enc_props,&lzma_alloc);
947 if ( res != SZ_OK )
948 return ERROR0(ERR_LZMA,
949 "Error while setup LZMA properties: %s\n-> LZMA error: %s\n",
950 f->fname, GetMessageLZMA(res,"?") );
951
952 enumError err = ERR_OK;
953 u8 in_buf[0x10000];
954 size_t in_buf_len = 0;
955
956 const int read_behind_eof = f->read_behind_eof;
957 if ( !read_count && f->seek_allowed && f->st.st_size )
958 read_count = f->st.st_size - f->cur_off;
959 const bool have_max_read = read_count > 0;
960 if (!have_max_read)
961 f->read_behind_eof = 2;
962
963 u8 * dest = buf;
964 u32 written = 0;
965
966 Lzma2Dec_Init(&lzma);
967 for(;;)
968 {
969 // [[2do]] [progress]
970
971 //--- fill input buffer
972
973 ELzmaFinishMode finish = LZMA_FINISH_ANY;
974
975 DASSERT( in_buf_len >= 0 && in_buf_len < sizeof(in_buf) );
976 size_t read_size = sizeof(in_buf) - in_buf_len;
977 if ( have_max_read && read_size > read_count )
978 {
979 read_size = read_count;
980 finish = LZMA_FINISH_END;
981 }
982 noPRINT("READ off=%llx, size=%zx=%zu, to=%zu\n",
983 (u64)f->cur_off, read_size, read_size, in_buf_len );
984 err = ReadF(f,in_buf+in_buf_len,read_size);
985 if (err)
986 return err;
987 in_buf_len += read_size;
988 read_count -= read_size;
989 f->bytes_read -= read_size; // count only decompressed data
990
991 if (!in_buf_len)
992 break;
993
994
995 //--- decode
996
997 size_t in_len = in_buf_len;
998 size_t out_len = buf_size;
999 ELzmaStatus status;
1000
1001 res = Lzma2Dec_DecodeToBuf(&lzma,dest,&out_len,in_buf,&in_len,finish,&status);
1002 noPRINT("DECODED, res=%s, stat=%d, in=%zu/%zu out=%zu/%zu\n",
1003 GetMessageLZMA(res,"?"), status,
1004 in_len, in_buf_len, out_len, buf_size );
1005
1006 if ( res == SZ_OK && !in_len && !out_len && status != LZMA_STATUS_FINISHED_WITH_MARK )
1007 res = SZ_ERROR_DATA;
1008
1009 if ( res != SZ_OK )
1010 return ERROR0(ERR_LZMA,
1011 "Error while reading LZMA stream: %s\n-> LZMA error: %s\n",
1012 f->fname, GetMessageLZMA(res,"?") );
1013
1014 written += out_len;
1015 dest += out_len;
1016 buf_size -= out_len;
1017 f->bytes_read += out_len; // count only decompressed data
1018
1019 if ( in_len < in_buf_len )
1020 {
1021 in_buf_len -= in_len;
1022 memmove(in_buf,in_buf+in_len,in_buf_len);
1023 }
1024 else
1025 in_buf_len = 0;
1026
1027 if ( status == LZMA_STATUS_FINISHED_WITH_MARK )
1028 break;
1029 }
1030
1031 f->read_behind_eof = read_behind_eof;
1032 Lzma2Dec_Free(&lzma,&lzma_alloc);
1033
1034 if (bytes_written)
1035 *bytes_written = written;
1036 return err;
1037 }
1038
1039 //
1040 ///////////////////////////////////////////////////////////////////////////////
1041 /////////////// END ///////////////
1042 ///////////////////////////////////////////////////////////////////////////////
1043
1044