1 ///////////////////////////////////////////////////////////////////////////////
2 //
3 /// \file coder.c
4 /// \brief Compresses or uncompresses a file
5 //
6 // Author: Lasse Collin
7 //
8 // This file has been put into the public domain.
9 // You can do whatever you want with this file.
10 //
11 ///////////////////////////////////////////////////////////////////////////////
12
13 #include "private.h"
14
15
16 /// Return value type for coder_init().
17 enum coder_init_ret {
18 CODER_INIT_NORMAL,
19 CODER_INIT_PASSTHRU,
20 CODER_INIT_ERROR,
21 };
22
23
24 enum operation_mode opt_mode = MODE_COMPRESS;
25 enum format_type opt_format = FORMAT_AUTO;
26 bool opt_auto_adjust = true;
27 bool opt_single_stream = false;
28 uint64_t opt_block_size = 0;
29 uint64_t *opt_block_list = NULL;
30
31
32 /// Stream used to communicate with liblzma
33 static lzma_stream strm = LZMA_STREAM_INIT;
34
35 /// Filters needed for all encoding all formats, and also decoding in raw data
36 static lzma_filter filters[LZMA_FILTERS_MAX + 1];
37
38 /// Input and output buffers
39 static io_buf in_buf;
40 static io_buf out_buf;
41
42 /// Number of filters. Zero indicates that we are using a preset.
43 static uint32_t filters_count = 0;
44
45 /// Number of the preset (0-9)
46 static uint32_t preset_number = LZMA_PRESET_DEFAULT;
47
48 /// Integrity check type
49 static lzma_check check;
50
51 /// This becomes false if the --check=CHECK option is used.
52 static bool check_default = true;
53
54 #if defined(HAVE_ENCODERS) && defined(MYTHREAD_ENABLED)
55 static lzma_mt mt_options = {
56 .flags = 0,
57 .timeout = 300,
58 .filters = filters,
59 };
60 #endif
61
62
63 extern void
coder_set_check(lzma_check new_check)64 coder_set_check(lzma_check new_check)
65 {
66 check = new_check;
67 check_default = false;
68 return;
69 }
70
71
72 static void
forget_filter_chain(void)73 forget_filter_chain(void)
74 {
75 // Setting a preset makes us forget a possibly defined custom
76 // filter chain.
77 while (filters_count > 0) {
78 --filters_count;
79 free(filters[filters_count].options);
80 filters[filters_count].options = NULL;
81 }
82
83 return;
84 }
85
86
87 extern void
coder_set_preset(uint32_t new_preset)88 coder_set_preset(uint32_t new_preset)
89 {
90 preset_number &= ~LZMA_PRESET_LEVEL_MASK;
91 preset_number |= new_preset;
92 forget_filter_chain();
93 return;
94 }
95
96
97 extern void
coder_set_extreme(void)98 coder_set_extreme(void)
99 {
100 preset_number |= LZMA_PRESET_EXTREME;
101 forget_filter_chain();
102 return;
103 }
104
105
106 extern void
coder_add_filter(lzma_vli id,void * options)107 coder_add_filter(lzma_vli id, void *options)
108 {
109 if (filters_count == LZMA_FILTERS_MAX)
110 message_fatal(_("Maximum number of filters is four"));
111
112 filters[filters_count].id = id;
113 filters[filters_count].options = options;
114 ++filters_count;
115
116 // Setting a custom filter chain makes us forget the preset options.
117 // This makes a difference if one specifies e.g. "xz -9 --lzma2 -e"
118 // where the custom filter chain resets the preset level back to
119 // the default 6, making the example equivalent to "xz -6e".
120 preset_number = LZMA_PRESET_DEFAULT;
121
122 return;
123 }
124
125
126 static void lzma_attribute((__noreturn__))
memlimit_too_small(uint64_t memory_usage)127 memlimit_too_small(uint64_t memory_usage)
128 {
129 message(V_ERROR, _("Memory usage limit is too low for the given "
130 "filter setup."));
131 message_mem_needed(V_ERROR, memory_usage);
132 tuklib_exit(E_ERROR, E_ERROR, false);
133 }
134
135
136 extern void
coder_set_compression_settings(void)137 coder_set_compression_settings(void)
138 {
139 // The default check type is CRC64, but fallback to CRC32
140 // if CRC64 isn't supported by the copy of liblzma we are
141 // using. CRC32 is always supported.
142 if (check_default) {
143 check = LZMA_CHECK_CRC64;
144 if (!lzma_check_is_supported(check))
145 check = LZMA_CHECK_CRC32;
146 }
147
148 // Options for LZMA1 or LZMA2 in case we are using a preset.
149 static lzma_options_lzma opt_lzma;
150
151 if (filters_count == 0) {
152 // We are using a preset. This is not a good idea in raw mode
153 // except when playing around with things. Different versions
154 // of this software may use different options in presets, and
155 // thus make uncompressing the raw data difficult.
156 if (opt_format == FORMAT_RAW) {
157 // The message is shown only if warnings are allowed
158 // but the exit status isn't changed.
159 message(V_WARNING, _("Using a preset in raw mode "
160 "is discouraged."));
161 message(V_WARNING, _("The exact options of the "
162 "presets may vary between software "
163 "versions."));
164 }
165
166 // Get the preset for LZMA1 or LZMA2.
167 if (lzma_lzma_preset(&opt_lzma, preset_number))
168 message_bug();
169
170 // Use LZMA2 except with --format=lzma we use LZMA1.
171 filters[0].id = opt_format == FORMAT_LZMA
172 ? LZMA_FILTER_LZMA1 : LZMA_FILTER_LZMA2;
173 filters[0].options = &opt_lzma;
174 filters_count = 1;
175 }
176
177 // Terminate the filter options array.
178 filters[filters_count].id = LZMA_VLI_UNKNOWN;
179
180 // If we are using the .lzma format, allow exactly one filter
181 // which has to be LZMA1.
182 if (opt_format == FORMAT_LZMA && (filters_count != 1
183 || filters[0].id != LZMA_FILTER_LZMA1))
184 message_fatal(_("The .lzma format supports only "
185 "the LZMA1 filter"));
186
187 // If we are using the .xz format, make sure that there is no LZMA1
188 // filter to prevent LZMA_PROG_ERROR.
189 if (opt_format == FORMAT_XZ)
190 for (size_t i = 0; i < filters_count; ++i)
191 if (filters[i].id == LZMA_FILTER_LZMA1)
192 message_fatal(_("LZMA1 cannot be used "
193 "with the .xz format"));
194
195 // Print the selected filter chain.
196 message_filters_show(V_DEBUG, filters);
197
198 // The --flush-timeout option requires LZMA_SYNC_FLUSH support
199 // from the filter chain. Currently threaded encoder doesn't support
200 // LZMA_SYNC_FLUSH so single-threaded mode must be used.
201 if (opt_mode == MODE_COMPRESS && opt_flush_timeout != 0) {
202 for (size_t i = 0; i < filters_count; ++i) {
203 switch (filters[i].id) {
204 case LZMA_FILTER_LZMA2:
205 case LZMA_FILTER_DELTA:
206 break;
207
208 default:
209 message_fatal(_("The filter chain is "
210 "incompatible with --flush-timeout"));
211 }
212 }
213
214 if (hardware_threads_get() > 1) {
215 message(V_WARNING, _("Switching to single-threaded "
216 "mode due to --flush-timeout"));
217 hardware_threads_set(1);
218 }
219 }
220
221 // Get the memory usage. Note that if --format=raw was used,
222 // we can be decompressing.
223 const uint64_t memory_limit = hardware_memlimit_get(opt_mode);
224 uint64_t memory_usage = UINT64_MAX;
225 if (opt_mode == MODE_COMPRESS) {
226 #ifdef HAVE_ENCODERS
227 # ifdef MYTHREAD_ENABLED
228 if (opt_format == FORMAT_XZ && hardware_threads_get() > 1) {
229 mt_options.threads = hardware_threads_get();
230 mt_options.block_size = opt_block_size;
231 mt_options.check = check;
232 memory_usage = lzma_stream_encoder_mt_memusage(
233 &mt_options);
234 if (memory_usage != UINT64_MAX)
235 message(V_DEBUG, _("Using up to %" PRIu32
236 " threads."),
237 mt_options.threads);
238 } else
239 # endif
240 {
241 memory_usage = lzma_raw_encoder_memusage(filters);
242 }
243 #endif
244 } else {
245 #ifdef HAVE_DECODERS
246 memory_usage = lzma_raw_decoder_memusage(filters);
247 #endif
248 }
249
250 if (memory_usage == UINT64_MAX)
251 message_fatal(_("Unsupported filter chain or filter options"));
252
253 // Print memory usage info before possible dictionary
254 // size auto-adjusting.
255 //
256 // NOTE: If only encoder support was built, we cannot show the
257 // what the decoder memory usage will be.
258 message_mem_needed(V_DEBUG, memory_usage);
259 #ifdef HAVE_DECODERS
260 if (opt_mode == MODE_COMPRESS) {
261 const uint64_t decmem = lzma_raw_decoder_memusage(filters);
262 if (decmem != UINT64_MAX)
263 message(V_DEBUG, _("Decompression will need "
264 "%s MiB of memory."), uint64_to_str(
265 round_up_to_mib(decmem), 0));
266 }
267 #endif
268
269 if (memory_usage <= memory_limit)
270 return;
271
272 // If --no-adjust was used or we didn't find LZMA1 or
273 // LZMA2 as the last filter, give an error immediately.
274 // --format=raw implies --no-adjust.
275 if (!opt_auto_adjust || opt_format == FORMAT_RAW)
276 memlimit_too_small(memory_usage);
277
278 assert(opt_mode == MODE_COMPRESS);
279
280 #ifdef HAVE_ENCODERS
281 # ifdef MYTHREAD_ENABLED
282 if (opt_format == FORMAT_XZ && mt_options.threads > 1) {
283 // Try to reduce the number of threads before
284 // adjusting the compression settings down.
285 do {
286 // FIXME? The real single-threaded mode has
287 // lower memory usage, but it's not comparable
288 // because it doesn't write the size info
289 // into Block Headers.
290 if (--mt_options.threads == 0)
291 memlimit_too_small(memory_usage);
292
293 memory_usage = lzma_stream_encoder_mt_memusage(
294 &mt_options);
295 if (memory_usage == UINT64_MAX)
296 message_bug();
297
298 } while (memory_usage > memory_limit);
299
300 message(V_WARNING, _("Adjusted the number of threads "
301 "from %s to %s to not exceed "
302 "the memory usage limit of %s MiB"),
303 uint64_to_str(hardware_threads_get(), 0),
304 uint64_to_str(mt_options.threads, 1),
305 uint64_to_str(round_up_to_mib(
306 memory_limit), 2));
307 }
308 # endif
309
310 if (memory_usage <= memory_limit)
311 return;
312
313 // Look for the last filter if it is LZMA2 or LZMA1, so we can make
314 // it use less RAM. With other filters we don't know what to do.
315 size_t i = 0;
316 while (filters[i].id != LZMA_FILTER_LZMA2
317 && filters[i].id != LZMA_FILTER_LZMA1) {
318 if (filters[i].id == LZMA_VLI_UNKNOWN)
319 memlimit_too_small(memory_usage);
320
321 ++i;
322 }
323
324 // Decrease the dictionary size until we meet the memory
325 // usage limit. First round down to full mebibytes.
326 lzma_options_lzma *opt = filters[i].options;
327 const uint32_t orig_dict_size = opt->dict_size;
328 opt->dict_size &= ~((UINT32_C(1) << 20) - 1);
329 while (true) {
330 // If it is below 1 MiB, auto-adjusting failed. We could be
331 // more sophisticated and scale it down even more, but let's
332 // see if many complain about this version.
333 //
334 // FIXME: Displays the scaled memory usage instead
335 // of the original.
336 if (opt->dict_size < (UINT32_C(1) << 20))
337 memlimit_too_small(memory_usage);
338
339 memory_usage = lzma_raw_encoder_memusage(filters);
340 if (memory_usage == UINT64_MAX)
341 message_bug();
342
343 // Accept it if it is low enough.
344 if (memory_usage <= memory_limit)
345 break;
346
347 // Otherwise 1 MiB down and try again. I hope this
348 // isn't too slow method for cases where the original
349 // dict_size is very big.
350 opt->dict_size -= UINT32_C(1) << 20;
351 }
352
353 // Tell the user that we decreased the dictionary size.
354 message(V_WARNING, _("Adjusted LZMA%c dictionary size "
355 "from %s MiB to %s MiB to not exceed "
356 "the memory usage limit of %s MiB"),
357 filters[i].id == LZMA_FILTER_LZMA2
358 ? '2' : '1',
359 uint64_to_str(orig_dict_size >> 20, 0),
360 uint64_to_str(opt->dict_size >> 20, 1),
361 uint64_to_str(round_up_to_mib(memory_limit), 2));
362 #endif
363
364 return;
365 }
366
367
368 #ifdef HAVE_DECODERS
369 /// Return true if the data in in_buf seems to be in the .xz format.
370 static bool
is_format_xz(void)371 is_format_xz(void)
372 {
373 // Specify the magic as hex to be compatible with EBCDIC systems.
374 static const uint8_t magic[6] = { 0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00 };
375 return strm.avail_in >= sizeof(magic)
376 && memcmp(in_buf.u8, magic, sizeof(magic)) == 0;
377 }
378
379
380 /// Return true if the data in in_buf seems to be in the .lzma format.
381 static bool
is_format_lzma(void)382 is_format_lzma(void)
383 {
384 // The .lzma header is 13 bytes.
385 if (strm.avail_in < 13)
386 return false;
387
388 // Decode the LZMA1 properties.
389 lzma_filter filter = { .id = LZMA_FILTER_LZMA1 };
390 if (lzma_properties_decode(&filter, NULL, in_buf.u8, 5) != LZMA_OK)
391 return false;
392
393 // A hack to ditch tons of false positives: We allow only dictionary
394 // sizes that are 2^n or 2^n + 2^(n-1) or UINT32_MAX. LZMA_Alone
395 // created only files with 2^n, but accepts any dictionary size.
396 // If someone complains, this will be reconsidered.
397 lzma_options_lzma *opt = filter.options;
398 const uint32_t dict_size = opt->dict_size;
399 free(opt);
400
401 if (dict_size != UINT32_MAX) {
402 uint32_t d = dict_size - 1;
403 d |= d >> 2;
404 d |= d >> 3;
405 d |= d >> 4;
406 d |= d >> 8;
407 d |= d >> 16;
408 ++d;
409 if (d != dict_size || dict_size == 0)
410 return false;
411 }
412
413 // Another hack to ditch false positives: Assume that if the
414 // uncompressed size is known, it must be less than 256 GiB.
415 // Again, if someone complains, this will be reconsidered.
416 uint64_t uncompressed_size = 0;
417 for (size_t i = 0; i < 8; ++i)
418 uncompressed_size |= (uint64_t)(in_buf.u8[5 + i]) << (i * 8);
419
420 if (uncompressed_size != UINT64_MAX
421 && uncompressed_size > (UINT64_C(1) << 38))
422 return false;
423
424 return true;
425 }
426 #endif
427
428
429 /// Detect the input file type (for now, this done only when decompressing),
430 /// and initialize an appropriate coder. Return value indicates if a normal
431 /// liblzma-based coder was initialized (CODER_INIT_NORMAL), if passthru
432 /// mode should be used (CODER_INIT_PASSTHRU), or if an error occurred
433 /// (CODER_INIT_ERROR).
434 static enum coder_init_ret
coder_init(file_pair * pair)435 coder_init(file_pair *pair)
436 {
437 lzma_ret ret = LZMA_PROG_ERROR;
438
439 if (opt_mode == MODE_COMPRESS) {
440 #ifdef HAVE_ENCODERS
441 switch (opt_format) {
442 case FORMAT_AUTO:
443 // args.c ensures this.
444 assert(0);
445 break;
446
447 case FORMAT_XZ:
448 # ifdef MYTHREAD_ENABLED
449 if (hardware_threads_get() > 1)
450 ret = lzma_stream_encoder_mt(
451 &strm, &mt_options);
452 else
453 # endif
454 ret = lzma_stream_encoder(
455 &strm, filters, check);
456 break;
457
458 case FORMAT_LZMA:
459 ret = lzma_alone_encoder(&strm, filters[0].options);
460 break;
461
462 case FORMAT_RAW:
463 ret = lzma_raw_encoder(&strm, filters);
464 break;
465 }
466 #endif
467 } else {
468 #ifdef HAVE_DECODERS
469 uint32_t flags = 0;
470
471 // It seems silly to warn about unsupported check if the
472 // check won't be verified anyway due to --ignore-check.
473 if (opt_ignore_check)
474 flags |= LZMA_IGNORE_CHECK;
475 else
476 flags |= LZMA_TELL_UNSUPPORTED_CHECK;
477
478 if (!opt_single_stream)
479 flags |= LZMA_CONCATENATED;
480
481 // We abuse FORMAT_AUTO to indicate unknown file format,
482 // for which we may consider passthru mode.
483 enum format_type init_format = FORMAT_AUTO;
484
485 switch (opt_format) {
486 case FORMAT_AUTO:
487 if (is_format_xz())
488 init_format = FORMAT_XZ;
489 else if (is_format_lzma())
490 init_format = FORMAT_LZMA;
491 break;
492
493 case FORMAT_XZ:
494 if (is_format_xz())
495 init_format = FORMAT_XZ;
496 break;
497
498 case FORMAT_LZMA:
499 if (is_format_lzma())
500 init_format = FORMAT_LZMA;
501 break;
502
503 case FORMAT_RAW:
504 init_format = FORMAT_RAW;
505 break;
506 }
507
508 switch (init_format) {
509 case FORMAT_AUTO:
510 // Unknown file format. If --decompress --stdout
511 // --force have been given, then we copy the input
512 // as is to stdout. Checking for MODE_DECOMPRESS
513 // is needed, because we don't want to do use
514 // passthru mode with --test.
515 if (opt_mode == MODE_DECOMPRESS
516 && opt_stdout && opt_force)
517 return CODER_INIT_PASSTHRU;
518
519 ret = LZMA_FORMAT_ERROR;
520 break;
521
522 case FORMAT_XZ:
523 ret = lzma_stream_decoder(&strm,
524 hardware_memlimit_get(
525 MODE_DECOMPRESS), flags);
526 break;
527
528 case FORMAT_LZMA:
529 ret = lzma_alone_decoder(&strm,
530 hardware_memlimit_get(
531 MODE_DECOMPRESS));
532 break;
533
534 case FORMAT_RAW:
535 // Memory usage has already been checked in
536 // coder_set_compression_settings().
537 ret = lzma_raw_decoder(&strm, filters);
538 break;
539 }
540
541 // Try to decode the headers. This will catch too low
542 // memory usage limit in case it happens in the first
543 // Block of the first Stream, which is where it very
544 // probably will happen if it is going to happen.
545 if (ret == LZMA_OK && init_format != FORMAT_RAW) {
546 strm.next_out = NULL;
547 strm.avail_out = 0;
548 ret = lzma_code(&strm, LZMA_RUN);
549 }
550 #endif
551 }
552
553 if (ret != LZMA_OK) {
554 message_error("%s: %s", pair->src_name, message_strm(ret));
555 if (ret == LZMA_MEMLIMIT_ERROR)
556 message_mem_needed(V_ERROR, lzma_memusage(&strm));
557
558 return CODER_INIT_ERROR;
559 }
560
561 return CODER_INIT_NORMAL;
562 }
563
564
565 /// Resolve conflicts between opt_block_size and opt_block_list in single
566 /// threaded mode. We want to default to opt_block_list, except when it is
567 /// larger than opt_block_size. If this is the case for the current Block
568 /// at *list_pos, then we break into smaller Blocks. Otherwise advance
569 /// to the next Block in opt_block_list, and break apart if needed.
570 static void
split_block(uint64_t * block_remaining,uint64_t * next_block_remaining,size_t * list_pos)571 split_block(uint64_t *block_remaining,
572 uint64_t *next_block_remaining,
573 size_t *list_pos)
574 {
575 if (*next_block_remaining > 0) {
576 // The Block at *list_pos has previously been split up.
577 assert(hardware_threads_get() == 1);
578 assert(opt_block_size > 0);
579 assert(opt_block_list != NULL);
580
581 if (*next_block_remaining > opt_block_size) {
582 // We have to split the current Block at *list_pos
583 // into another opt_block_size length Block.
584 *block_remaining = opt_block_size;
585 } else {
586 // This is the last remaining split Block for the
587 // Block at *list_pos.
588 *block_remaining = *next_block_remaining;
589 }
590
591 *next_block_remaining -= *block_remaining;
592
593 } else {
594 // The Block at *list_pos has been finished. Go to the next
595 // entry in the list. If the end of the list has been reached,
596 // reuse the size of the last Block.
597 if (opt_block_list[*list_pos + 1] != 0)
598 ++*list_pos;
599
600 *block_remaining = opt_block_list[*list_pos];
601
602 // If in single-threaded mode, split up the Block if needed.
603 // This is not needed in multi-threaded mode because liblzma
604 // will do this due to how threaded encoding works.
605 if (hardware_threads_get() == 1 && opt_block_size > 0
606 && *block_remaining > opt_block_size) {
607 *next_block_remaining
608 = *block_remaining - opt_block_size;
609 *block_remaining = opt_block_size;
610 }
611 }
612 }
613
614
615 static bool
coder_write_output(file_pair * pair)616 coder_write_output(file_pair *pair)
617 {
618 if (opt_mode != MODE_TEST) {
619 if (io_write(pair, &out_buf, IO_BUFFER_SIZE - strm.avail_out))
620 return true;
621 }
622
623 strm.next_out = out_buf.u8;
624 strm.avail_out = IO_BUFFER_SIZE;
625 return false;
626 }
627
628
629 /// Compress or decompress using liblzma.
630 static bool
coder_normal(file_pair * pair)631 coder_normal(file_pair *pair)
632 {
633 // Encoder needs to know when we have given all the input to it.
634 // The decoders need to know it too when we are using
635 // LZMA_CONCATENATED. We need to check for src_eof here, because
636 // the first input chunk has been already read if decompressing,
637 // and that may have been the only chunk we will read.
638 lzma_action action = pair->src_eof ? LZMA_FINISH : LZMA_RUN;
639
640 lzma_ret ret;
641
642 // Assume that something goes wrong.
643 bool success = false;
644
645 // block_remaining indicates how many input bytes to encode before
646 // finishing the current .xz Block. The Block size is set with
647 // --block-size=SIZE and --block-list. They have an effect only when
648 // compressing to the .xz format. If block_remaining == UINT64_MAX,
649 // only a single block is created.
650 uint64_t block_remaining = UINT64_MAX;
651
652 // next_block_remaining for when we are in single-threaded mode and
653 // the Block in --block-list is larger than the --block-size=SIZE.
654 uint64_t next_block_remaining = 0;
655
656 // Position in opt_block_list. Unused if --block-list wasn't used.
657 size_t list_pos = 0;
658
659 // Handle --block-size for single-threaded mode and the first step
660 // of --block-list.
661 if (opt_mode == MODE_COMPRESS && opt_format == FORMAT_XZ) {
662 // --block-size doesn't do anything here in threaded mode,
663 // because the threaded encoder will take care of splitting
664 // to fixed-sized Blocks.
665 if (hardware_threads_get() == 1 && opt_block_size > 0)
666 block_remaining = opt_block_size;
667
668 // If --block-list was used, start with the first size.
669 //
670 // For threaded case, --block-size specifies how big Blocks
671 // the encoder needs to be prepared to create at maximum
672 // and --block-list will simultaneously cause new Blocks
673 // to be started at specified intervals. To keep things
674 // logical, the same is done in single-threaded mode. The
675 // output is still not identical because in single-threaded
676 // mode the size info isn't written into Block Headers.
677 if (opt_block_list != NULL) {
678 if (block_remaining < opt_block_list[list_pos]) {
679 assert(hardware_threads_get() == 1);
680 next_block_remaining = opt_block_list[list_pos]
681 - block_remaining;
682 } else {
683 block_remaining = opt_block_list[list_pos];
684 }
685 }
686 }
687
688 strm.next_out = out_buf.u8;
689 strm.avail_out = IO_BUFFER_SIZE;
690
691 while (!user_abort) {
692 // Fill the input buffer if it is empty and we aren't
693 // flushing or finishing.
694 if (strm.avail_in == 0 && action == LZMA_RUN) {
695 strm.next_in = in_buf.u8;
696 strm.avail_in = io_read(pair, &in_buf,
697 my_min(block_remaining,
698 IO_BUFFER_SIZE));
699
700 if (strm.avail_in == SIZE_MAX)
701 break;
702
703 if (pair->src_eof) {
704 action = LZMA_FINISH;
705
706 } else if (block_remaining != UINT64_MAX) {
707 // Start a new Block after every
708 // opt_block_size bytes of input.
709 block_remaining -= strm.avail_in;
710 if (block_remaining == 0)
711 action = LZMA_FULL_BARRIER;
712 }
713
714 if (action == LZMA_RUN && pair->flush_needed)
715 action = LZMA_SYNC_FLUSH;
716 }
717
718 // Let liblzma do the actual work.
719 ret = lzma_code(&strm, action);
720
721 // Write out if the output buffer became full.
722 if (strm.avail_out == 0) {
723 if (coder_write_output(pair))
724 break;
725 }
726
727 if (ret == LZMA_STREAM_END && (action == LZMA_SYNC_FLUSH
728 || action == LZMA_FULL_BARRIER)) {
729 if (action == LZMA_SYNC_FLUSH) {
730 // Flushing completed. Write the pending data
731 // out immediately so that the reading side
732 // can decompress everything compressed so far.
733 if (coder_write_output(pair))
734 break;
735
736 // Mark that we haven't seen any new input
737 // since the previous flush.
738 pair->src_has_seen_input = false;
739 pair->flush_needed = false;
740 } else {
741 // Start a new Block after LZMA_FULL_BARRIER.
742 if (opt_block_list == NULL) {
743 assert(hardware_threads_get() == 1);
744 assert(opt_block_size > 0);
745 block_remaining = opt_block_size;
746 } else {
747 split_block(&block_remaining,
748 &next_block_remaining,
749 &list_pos);
750 }
751 }
752
753 // Start a new Block after LZMA_FULL_FLUSH or continue
754 // the same block after LZMA_SYNC_FLUSH.
755 action = LZMA_RUN;
756
757 } else if (ret != LZMA_OK) {
758 // Determine if the return value indicates that we
759 // won't continue coding.
760 const bool stop = ret != LZMA_NO_CHECK
761 && ret != LZMA_UNSUPPORTED_CHECK;
762
763 if (stop) {
764 // Write the remaining bytes even if something
765 // went wrong, because that way the user gets
766 // as much data as possible, which can be good
767 // when trying to get at least some useful
768 // data out of damaged files.
769 if (coder_write_output(pair))
770 break;
771 }
772
773 if (ret == LZMA_STREAM_END) {
774 if (opt_single_stream) {
775 io_fix_src_pos(pair, strm.avail_in);
776 success = true;
777 break;
778 }
779
780 // Check that there is no trailing garbage.
781 // This is needed for LZMA_Alone and raw
782 // streams.
783 if (strm.avail_in == 0 && !pair->src_eof) {
784 // Try reading one more byte.
785 // Hopefully we don't get any more
786 // input, and thus pair->src_eof
787 // becomes true.
788 strm.avail_in = io_read(
789 pair, &in_buf, 1);
790 if (strm.avail_in == SIZE_MAX)
791 break;
792
793 assert(strm.avail_in == 0
794 || strm.avail_in == 1);
795 }
796
797 if (strm.avail_in == 0) {
798 assert(pair->src_eof);
799 success = true;
800 break;
801 }
802
803 // We hadn't reached the end of the file.
804 ret = LZMA_DATA_ERROR;
805 assert(stop);
806 }
807
808 // If we get here and stop is true, something went
809 // wrong and we print an error. Otherwise it's just
810 // a warning and coding can continue.
811 if (stop) {
812 message_error("%s: %s", pair->src_name,
813 message_strm(ret));
814 } else {
815 message_warning("%s: %s", pair->src_name,
816 message_strm(ret));
817
818 // When compressing, all possible errors set
819 // stop to true.
820 assert(opt_mode != MODE_COMPRESS);
821 }
822
823 if (ret == LZMA_MEMLIMIT_ERROR) {
824 // Display how much memory it would have
825 // actually needed.
826 message_mem_needed(V_ERROR,
827 lzma_memusage(&strm));
828 }
829
830 if (stop)
831 break;
832 }
833
834 // Show progress information under certain conditions.
835 message_progress_update();
836 }
837
838 return success;
839 }
840
841
842 /// Copy from input file to output file without processing the data in any
843 /// way. This is used only when trying to decompress unrecognized files
844 /// with --decompress --stdout --force, so the output is always stdout.
845 static bool
coder_passthru(file_pair * pair)846 coder_passthru(file_pair *pair)
847 {
848 while (strm.avail_in != 0) {
849 if (user_abort)
850 return false;
851
852 if (io_write(pair, &in_buf, strm.avail_in))
853 return false;
854
855 strm.total_in += strm.avail_in;
856 strm.total_out = strm.total_in;
857 message_progress_update();
858
859 strm.avail_in = io_read(pair, &in_buf, IO_BUFFER_SIZE);
860 if (strm.avail_in == SIZE_MAX)
861 return false;
862 }
863
864 return true;
865 }
866
867
868 extern void
coder_run(const char * filename)869 coder_run(const char *filename)
870 {
871 // Set and possibly print the filename for the progress message.
872 message_filename(filename);
873
874 // Try to open the input file.
875 file_pair *pair = io_open_src(filename);
876 if (pair == NULL)
877 return;
878
879 // Assume that something goes wrong.
880 bool success = false;
881
882 if (opt_mode == MODE_COMPRESS) {
883 strm.next_in = NULL;
884 strm.avail_in = 0;
885 } else {
886 // Read the first chunk of input data. This is needed
887 // to detect the input file type.
888 strm.next_in = in_buf.u8;
889 strm.avail_in = io_read(pair, &in_buf, IO_BUFFER_SIZE);
890 }
891
892 if (strm.avail_in != SIZE_MAX) {
893 // Initialize the coder. This will detect the file format
894 // and, in decompression or testing mode, check the memory
895 // usage of the first Block too. This way we don't try to
896 // open the destination file if we see that coding wouldn't
897 // work at all anyway. This also avoids deleting the old
898 // "target" file if --force was used.
899 const enum coder_init_ret init_ret = coder_init(pair);
900
901 if (init_ret != CODER_INIT_ERROR && !user_abort) {
902 // Don't open the destination file when --test
903 // is used.
904 if (opt_mode == MODE_TEST || !io_open_dest(pair)) {
905 // Remember the current time. It is needed
906 // for progress indicator.
907 mytime_set_start_time();
908
909 // Initialize the progress indicator.
910 const bool is_passthru = init_ret
911 == CODER_INIT_PASSTHRU;
912 const uint64_t in_size
913 = pair->src_st.st_size <= 0
914 ? 0 : (uint64_t)(pair->src_st.st_size);
915 message_progress_start(&strm,
916 is_passthru, in_size);
917
918 // Do the actual coding or passthru.
919 if (is_passthru)
920 success = coder_passthru(pair);
921 else
922 success = coder_normal(pair);
923
924 message_progress_end(success);
925 }
926 }
927 }
928
929 // Close the file pair. It needs to know if coding was successful to
930 // know if the source or target file should be unlinked.
931 io_close(pair, success);
932
933 return;
934 }
935
936
937 #ifndef NDEBUG
938 extern void
coder_free(void)939 coder_free(void)
940 {
941 lzma_end(&strm);
942 return;
943 }
944 #endif
945