1 /*- 2 * Copyright (c) 2003-2010 Tim Kientzle 3 * Copyright (c) 2009-2012 Michihiro NAKAJIMA 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 */ 26 27 #include "archive_platform.h" 28 29 #ifdef HAVE_ERRNO_H 30 #include <errno.h> 31 #endif 32 #ifdef HAVE_STDLIB_H 33 #include <stdlib.h> 34 #endif 35 #ifdef HAVE_STRING_H 36 #include <string.h> 37 #endif 38 #include <time.h> 39 #ifdef HAVE_LZMA_H 40 #include <lzma.h> 41 #endif 42 43 #include "archive.h" 44 #include "archive_endian.h" 45 #include "archive_private.h" 46 #include "archive_write_private.h" 47 48 #if ARCHIVE_VERSION_NUMBER < 4000000 49 int 50 archive_write_set_compression_lzip(struct archive *a) 51 { 52 __archive_write_filters_free(a); 53 return (archive_write_add_filter_lzip(a)); 54 } 55 56 int 57 archive_write_set_compression_lzma(struct archive *a) 58 { 59 __archive_write_filters_free(a); 60 return (archive_write_add_filter_lzma(a)); 61 } 62 63 int 64 archive_write_set_compression_xz(struct archive *a) 65 { 66 __archive_write_filters_free(a); 67 return (archive_write_add_filter_xz(a)); 68 } 69 70 #endif 71 72 #ifndef HAVE_LZMA_H 73 int 74 archive_write_add_filter_xz(struct archive *a) 75 { 76 archive_set_error(a, ARCHIVE_ERRNO_MISC, 77 "xz compression not supported on this platform"); 78 return (ARCHIVE_FATAL); 79 } 80 81 int 82 archive_write_add_filter_lzma(struct archive *a) 83 { 84 archive_set_error(a, ARCHIVE_ERRNO_MISC, 85 "lzma compression not supported on this platform"); 86 return (ARCHIVE_FATAL); 87 } 88 89 int 90 archive_write_add_filter_lzip(struct archive *a) 91 { 92 archive_set_error(a, ARCHIVE_ERRNO_MISC, 93 "lzma compression not supported on this platform"); 94 return (ARCHIVE_FATAL); 95 } 96 #else 97 /* Don't compile this if we don't have liblzma. */ 98 99 struct private_data { 100 int compression_level; 101 uint32_t threads; 102 lzma_stream stream; 103 lzma_filter lzmafilters[2]; 104 lzma_options_lzma lzma_opt; 105 int64_t total_in; 106 unsigned char *compressed; 107 size_t compressed_buffer_size; 108 int64_t total_out; 109 /* the CRC32 value of uncompressed data for lzip */ 110 uint32_t crc32; 111 }; 112 113 static int archive_compressor_xz_options(struct archive_write_filter *, 114 const char *, const char *); 115 static int archive_compressor_xz_open(struct archive_write_filter *); 116 static int archive_compressor_xz_write(struct archive_write_filter *, 117 const void *, size_t); 118 static int archive_compressor_xz_close(struct archive_write_filter *); 119 static int archive_compressor_xz_free(struct archive_write_filter *); 120 static int drive_compressor(struct archive_write_filter *, 121 struct private_data *, int finishing); 122 123 struct option_value { 124 uint32_t dict_size; 125 uint32_t nice_len; 126 lzma_match_finder mf; 127 }; 128 static const struct option_value option_values[] = { 129 { 1 << 16, 32, LZMA_MF_HC3}, 130 { 1 << 20, 32, LZMA_MF_HC3}, 131 { 3 << 19, 32, LZMA_MF_HC4}, 132 { 1 << 21, 32, LZMA_MF_BT4}, 133 { 3 << 20, 32, LZMA_MF_BT4}, 134 { 1 << 22, 32, LZMA_MF_BT4}, 135 { 1 << 23, 64, LZMA_MF_BT4}, 136 { 1 << 24, 64, LZMA_MF_BT4}, 137 { 3 << 23, 64, LZMA_MF_BT4}, 138 { 1 << 25, 64, LZMA_MF_BT4} 139 }; 140 141 static int 142 common_setup(struct archive_write_filter *f) 143 { 144 struct private_data *data; 145 struct archive_write *a = (struct archive_write *)f->archive; 146 data = calloc(1, sizeof(*data)); 147 if (data == NULL) { 148 archive_set_error(&a->archive, ENOMEM, "Out of memory"); 149 return (ARCHIVE_FATAL); 150 } 151 f->data = data; 152 data->compression_level = LZMA_PRESET_DEFAULT; 153 data->threads = 1; 154 f->open = &archive_compressor_xz_open; 155 f->close = archive_compressor_xz_close; 156 f->free = archive_compressor_xz_free; 157 f->options = &archive_compressor_xz_options; 158 return (ARCHIVE_OK); 159 } 160 161 /* 162 * Add an xz compression filter to this write handle. 163 */ 164 int 165 archive_write_add_filter_xz(struct archive *_a) 166 { 167 struct archive_write_filter *f; 168 int r; 169 170 archive_check_magic(_a, ARCHIVE_WRITE_MAGIC, 171 ARCHIVE_STATE_NEW, "archive_write_add_filter_xz"); 172 f = __archive_write_allocate_filter(_a); 173 r = common_setup(f); 174 if (r == ARCHIVE_OK) { 175 f->code = ARCHIVE_FILTER_XZ; 176 f->name = "xz"; 177 } 178 return (r); 179 } 180 181 /* LZMA is handled identically, we just need a different compression 182 * code set. (The liblzma setup looks at the code to determine 183 * the one place that XZ and LZMA require different handling.) */ 184 int 185 archive_write_add_filter_lzma(struct archive *_a) 186 { 187 struct archive_write_filter *f; 188 int r; 189 190 archive_check_magic(_a, ARCHIVE_WRITE_MAGIC, 191 ARCHIVE_STATE_NEW, "archive_write_add_filter_lzma"); 192 f = __archive_write_allocate_filter(_a); 193 r = common_setup(f); 194 if (r == ARCHIVE_OK) { 195 f->code = ARCHIVE_FILTER_LZMA; 196 f->name = "lzma"; 197 } 198 return (r); 199 } 200 201 int 202 archive_write_add_filter_lzip(struct archive *_a) 203 { 204 struct archive_write_filter *f; 205 int r; 206 207 archive_check_magic(_a, ARCHIVE_WRITE_MAGIC, 208 ARCHIVE_STATE_NEW, "archive_write_add_filter_lzip"); 209 f = __archive_write_allocate_filter(_a); 210 r = common_setup(f); 211 if (r == ARCHIVE_OK) { 212 f->code = ARCHIVE_FILTER_LZIP; 213 f->name = "lzip"; 214 } 215 return (r); 216 } 217 218 static int 219 archive_compressor_xz_init_stream(struct archive_write_filter *f, 220 struct private_data *data) 221 { 222 static const lzma_stream lzma_stream_init_data = LZMA_STREAM_INIT; 223 int ret; 224 #ifdef HAVE_LZMA_STREAM_ENCODER_MT 225 lzma_mt mt_options; 226 #endif 227 228 data->stream = lzma_stream_init_data; 229 data->stream.next_out = data->compressed; 230 data->stream.avail_out = data->compressed_buffer_size; 231 if (f->code == ARCHIVE_FILTER_XZ) { 232 #ifdef HAVE_LZMA_STREAM_ENCODER_MT 233 if (data->threads != 1) { 234 memset(&mt_options, 0, sizeof(mt_options)); 235 mt_options.threads = data->threads; 236 mt_options.timeout = 300; 237 mt_options.filters = data->lzmafilters; 238 mt_options.check = LZMA_CHECK_CRC64; 239 ret = lzma_stream_encoder_mt(&(data->stream), 240 &mt_options); 241 } else 242 #endif 243 ret = lzma_stream_encoder(&(data->stream), 244 data->lzmafilters, LZMA_CHECK_CRC64); 245 } else if (f->code == ARCHIVE_FILTER_LZMA) { 246 ret = lzma_alone_encoder(&(data->stream), &data->lzma_opt); 247 } else { /* ARCHIVE_FILTER_LZIP */ 248 int dict_size = data->lzma_opt.dict_size; 249 int ds, log2dic, wedges; 250 251 /* Calculate a coded dictionary size */ 252 if (dict_size < (1 << 12) || dict_size > (1 << 29)) { 253 archive_set_error(f->archive, ARCHIVE_ERRNO_MISC, 254 "Unacceptable dictionary size for lzip: %d", 255 dict_size); 256 return (ARCHIVE_FATAL); 257 } 258 for (log2dic = 29; log2dic >= 12; log2dic--) { 259 if (dict_size & (1 << log2dic)) 260 break; 261 } 262 if (dict_size > (1 << log2dic)) { 263 log2dic++; 264 wedges = 265 ((1 << log2dic) - dict_size) / (1 << (log2dic - 4)); 266 } else 267 wedges = 0; 268 ds = ((wedges << 5) & 0xe0) | (log2dic & 0x1f); 269 270 data->crc32 = 0; 271 /* Make a header */ 272 data->compressed[0] = 0x4C; 273 data->compressed[1] = 0x5A; 274 data->compressed[2] = 0x49; 275 data->compressed[3] = 0x50; 276 data->compressed[4] = 1;/* Version */ 277 data->compressed[5] = (unsigned char)ds; 278 data->stream.next_out += 6; 279 data->stream.avail_out -= 6; 280 281 ret = lzma_raw_encoder(&(data->stream), data->lzmafilters); 282 } 283 if (ret == LZMA_OK) 284 return (ARCHIVE_OK); 285 286 switch (ret) { 287 case LZMA_MEM_ERROR: 288 archive_set_error(f->archive, ENOMEM, 289 "Internal error initializing compression library: " 290 "Cannot allocate memory"); 291 break; 292 default: 293 archive_set_error(f->archive, ARCHIVE_ERRNO_MISC, 294 "Internal error initializing compression library: " 295 "It's a bug in liblzma"); 296 break; 297 } 298 return (ARCHIVE_FATAL); 299 } 300 301 /* 302 * Setup callback. 303 */ 304 static int 305 archive_compressor_xz_open(struct archive_write_filter *f) 306 { 307 struct private_data *data = f->data; 308 int ret; 309 310 if (data->compressed == NULL) { 311 size_t bs = 65536, bpb; 312 if (f->archive->magic == ARCHIVE_WRITE_MAGIC) { 313 /* Buffer size should be a multiple number of the of bytes 314 * per block for performance. */ 315 bpb = archive_write_get_bytes_per_block(f->archive); 316 if (bpb > bs) 317 bs = bpb; 318 else if (bpb != 0) 319 bs -= bs % bpb; 320 } 321 data->compressed_buffer_size = bs; 322 data->compressed 323 = (unsigned char *)malloc(data->compressed_buffer_size); 324 if (data->compressed == NULL) { 325 archive_set_error(f->archive, ENOMEM, 326 "Can't allocate data for compression buffer"); 327 return (ARCHIVE_FATAL); 328 } 329 } 330 331 f->write = archive_compressor_xz_write; 332 333 /* Initialize compression library. */ 334 if (f->code == ARCHIVE_FILTER_LZIP) { 335 const struct option_value *val = 336 &option_values[data->compression_level]; 337 338 data->lzma_opt.dict_size = val->dict_size; 339 data->lzma_opt.preset_dict = NULL; 340 data->lzma_opt.preset_dict_size = 0; 341 data->lzma_opt.lc = LZMA_LC_DEFAULT; 342 data->lzma_opt.lp = LZMA_LP_DEFAULT; 343 data->lzma_opt.pb = LZMA_PB_DEFAULT; 344 data->lzma_opt.mode = 345 data->compression_level<= 2? LZMA_MODE_FAST:LZMA_MODE_NORMAL; 346 data->lzma_opt.nice_len = val->nice_len; 347 data->lzma_opt.mf = val->mf; 348 data->lzma_opt.depth = 0; 349 data->lzmafilters[0].id = LZMA_FILTER_LZMA1; 350 data->lzmafilters[0].options = &data->lzma_opt; 351 data->lzmafilters[1].id = LZMA_VLI_UNKNOWN;/* Terminate */ 352 } else { 353 if (lzma_lzma_preset(&data->lzma_opt, data->compression_level)) { 354 archive_set_error(f->archive, ARCHIVE_ERRNO_MISC, 355 "Internal error initializing compression library"); 356 } 357 data->lzmafilters[0].id = LZMA_FILTER_LZMA2; 358 data->lzmafilters[0].options = &data->lzma_opt; 359 data->lzmafilters[1].id = LZMA_VLI_UNKNOWN;/* Terminate */ 360 } 361 ret = archive_compressor_xz_init_stream(f, data); 362 if (ret == LZMA_OK) { 363 f->data = data; 364 return (0); 365 } 366 return (ARCHIVE_FATAL); 367 } 368 369 /* 370 * Set write options. 371 */ 372 static int 373 archive_compressor_xz_options(struct archive_write_filter *f, 374 const char *key, const char *value) 375 { 376 struct private_data *data = (struct private_data *)f->data; 377 378 if (strcmp(key, "compression-level") == 0) { 379 if (value == NULL || !(value[0] >= '0' && value[0] <= '9') || 380 value[1] != '\0') 381 return (ARCHIVE_WARN); 382 data->compression_level = value[0] - '0'; 383 if (data->compression_level > 9) 384 data->compression_level = 9; 385 return (ARCHIVE_OK); 386 } else if (strcmp(key, "threads") == 0) { 387 char *endptr; 388 389 if (value == NULL) 390 return (ARCHIVE_WARN); 391 errno = 0; 392 data->threads = (int)strtoul(value, &endptr, 10); 393 if (errno != 0 || *endptr != '\0') { 394 data->threads = 1; 395 return (ARCHIVE_WARN); 396 } 397 if (data->threads == 0) { 398 #ifdef HAVE_LZMA_STREAM_ENCODER_MT 399 data->threads = lzma_cputhreads(); 400 #else 401 data->threads = 1; 402 #endif 403 } 404 return (ARCHIVE_OK); 405 } 406 407 /* Note: The "warn" return is just to inform the options 408 * supervisor that we didn't handle it. It will generate 409 * a suitable error if no one used this option. */ 410 return (ARCHIVE_WARN); 411 } 412 413 /* 414 * Write data to the compressed stream. 415 */ 416 static int 417 archive_compressor_xz_write(struct archive_write_filter *f, 418 const void *buff, size_t length) 419 { 420 struct private_data *data = (struct private_data *)f->data; 421 int ret; 422 423 /* Update statistics */ 424 data->total_in += length; 425 if (f->code == ARCHIVE_FILTER_LZIP) 426 data->crc32 = lzma_crc32(buff, length, data->crc32); 427 428 /* Compress input data to output buffer */ 429 data->stream.next_in = buff; 430 data->stream.avail_in = length; 431 if ((ret = drive_compressor(f, data, 0)) != ARCHIVE_OK) 432 return (ret); 433 434 return (ARCHIVE_OK); 435 } 436 437 438 /* 439 * Finish the compression... 440 */ 441 static int 442 archive_compressor_xz_close(struct archive_write_filter *f) 443 { 444 struct private_data *data = (struct private_data *)f->data; 445 int ret; 446 447 ret = drive_compressor(f, data, 1); 448 if (ret == ARCHIVE_OK) { 449 data->total_out += 450 data->compressed_buffer_size - data->stream.avail_out; 451 ret = __archive_write_filter(f->next_filter, 452 data->compressed, 453 data->compressed_buffer_size - data->stream.avail_out); 454 if (f->code == ARCHIVE_FILTER_LZIP && ret == ARCHIVE_OK) { 455 archive_le32enc(data->compressed, data->crc32); 456 archive_le64enc(data->compressed+4, data->total_in); 457 archive_le64enc(data->compressed+12, data->total_out + 20); 458 ret = __archive_write_filter(f->next_filter, 459 data->compressed, 20); 460 } 461 } 462 lzma_end(&(data->stream)); 463 return ret; 464 } 465 466 static int 467 archive_compressor_xz_free(struct archive_write_filter *f) 468 { 469 struct private_data *data = (struct private_data *)f->data; 470 free(data->compressed); 471 free(data); 472 f->data = NULL; 473 return (ARCHIVE_OK); 474 } 475 476 /* 477 * Utility function to push input data through compressor, 478 * writing full output blocks as necessary. 479 * 480 * Note that this handles both the regular write case (finishing == 481 * false) and the end-of-archive case (finishing == true). 482 */ 483 static int 484 drive_compressor(struct archive_write_filter *f, 485 struct private_data *data, int finishing) 486 { 487 int ret; 488 489 for (;;) { 490 if (data->stream.avail_out == 0) { 491 data->total_out += data->compressed_buffer_size; 492 ret = __archive_write_filter(f->next_filter, 493 data->compressed, 494 data->compressed_buffer_size); 495 if (ret != ARCHIVE_OK) 496 return (ARCHIVE_FATAL); 497 data->stream.next_out = data->compressed; 498 data->stream.avail_out = data->compressed_buffer_size; 499 } 500 501 /* If there's nothing to do, we're done. */ 502 if (!finishing && data->stream.avail_in == 0) 503 return (ARCHIVE_OK); 504 505 ret = lzma_code(&(data->stream), 506 finishing ? LZMA_FINISH : LZMA_RUN ); 507 508 switch (ret) { 509 case LZMA_OK: 510 /* In non-finishing case, check if compressor 511 * consumed everything */ 512 if (!finishing && data->stream.avail_in == 0) 513 return (ARCHIVE_OK); 514 /* In finishing case, this return always means 515 * there's more work */ 516 break; 517 case LZMA_STREAM_END: 518 /* This return can only occur in finishing case. */ 519 if (finishing) 520 return (ARCHIVE_OK); 521 archive_set_error(f->archive, ARCHIVE_ERRNO_MISC, 522 "lzma compression data error"); 523 return (ARCHIVE_FATAL); 524 case LZMA_MEMLIMIT_ERROR: 525 archive_set_error(f->archive, ENOMEM, 526 "lzma compression error: " 527 "%ju MiB would have been needed", 528 (uintmax_t)((lzma_memusage(&(data->stream)) 529 + 1024 * 1024 -1) 530 / (1024 * 1024))); 531 return (ARCHIVE_FATAL); 532 default: 533 /* Any other return value indicates an error. */ 534 archive_set_error(f->archive, ARCHIVE_ERRNO_MISC, 535 "lzma compression failed:" 536 " lzma_code() call returned status %d", 537 ret); 538 return (ARCHIVE_FATAL); 539 } 540 } 541 } 542 543 #endif /* HAVE_LZMA_H */ 544