1 /*- 2 * Copyright (c) 2003-2010 Tim Kientzle 3 * Copyright (c) 2009-2012 Michihiro NAKAJIMA 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 */ 26 27 #include "archive_platform.h" 28 29 __FBSDID("$FreeBSD: head/lib/libarchive/archive_write_set_compression_xz.c 201108 2009-12-28 03:28:21Z kientzle $"); 30 31 #ifdef HAVE_ERRNO_H 32 #include <errno.h> 33 #endif 34 #ifdef HAVE_STDLIB_H 35 #include <stdlib.h> 36 #endif 37 #ifdef HAVE_STRING_H 38 #include <string.h> 39 #endif 40 #include <time.h> 41 #ifdef HAVE_LZMA_H 42 #include <lzma.h> 43 #endif 44 45 #include "archive.h" 46 #include "archive_endian.h" 47 #include "archive_private.h" 48 #include "archive_write_private.h" 49 50 #if ARCHIVE_VERSION_NUMBER < 4000000 51 int 52 archive_write_set_compression_lzip(struct archive *a) 53 { 54 __archive_write_filters_free(a); 55 return (archive_write_add_filter_lzip(a)); 56 } 57 58 int 59 archive_write_set_compression_lzma(struct archive *a) 60 { 61 __archive_write_filters_free(a); 62 return (archive_write_add_filter_lzma(a)); 63 } 64 65 int 66 archive_write_set_compression_xz(struct archive *a) 67 { 68 __archive_write_filters_free(a); 69 return (archive_write_add_filter_xz(a)); 70 } 71 72 #endif 73 74 #ifndef HAVE_LZMA_H 75 int 76 archive_write_add_filter_xz(struct archive *a) 77 { 78 archive_set_error(a, ARCHIVE_ERRNO_MISC, 79 "xz compression not supported on this platform"); 80 return (ARCHIVE_FATAL); 81 } 82 83 int 84 archive_write_add_filter_lzma(struct archive *a) 85 { 86 archive_set_error(a, ARCHIVE_ERRNO_MISC, 87 "lzma compression not supported on this platform"); 88 return (ARCHIVE_FATAL); 89 } 90 91 int 92 archive_write_add_filter_lzip(struct archive *a) 93 { 94 archive_set_error(a, ARCHIVE_ERRNO_MISC, 95 "lzma compression not supported on this platform"); 96 return (ARCHIVE_FATAL); 97 } 98 #else 99 /* Don't compile this if we don't have liblzma. */ 100 101 struct private_data { 102 int compression_level; 103 uint32_t threads; 104 lzma_stream stream; 105 lzma_filter lzmafilters[2]; 106 lzma_options_lzma lzma_opt; 107 int64_t total_in; 108 unsigned char *compressed; 109 size_t compressed_buffer_size; 110 int64_t total_out; 111 /* the CRC32 value of uncompressed data for lzip */ 112 uint32_t crc32; 113 }; 114 115 static int archive_compressor_xz_options(struct archive_write_filter *, 116 const char *, const char *); 117 static int archive_compressor_xz_open(struct archive_write_filter *); 118 static int archive_compressor_xz_write(struct archive_write_filter *, 119 const void *, size_t); 120 static int archive_compressor_xz_close(struct archive_write_filter *); 121 static int archive_compressor_xz_free(struct archive_write_filter *); 122 static int drive_compressor(struct archive_write_filter *, 123 struct private_data *, int finishing); 124 125 struct option_value { 126 uint32_t dict_size; 127 uint32_t nice_len; 128 lzma_match_finder mf; 129 }; 130 static const struct option_value option_values[] = { 131 { 1 << 16, 32, LZMA_MF_HC3}, 132 { 1 << 20, 32, LZMA_MF_HC3}, 133 { 3 << 19, 32, LZMA_MF_HC4}, 134 { 1 << 21, 32, LZMA_MF_BT4}, 135 { 3 << 20, 32, LZMA_MF_BT4}, 136 { 1 << 22, 32, LZMA_MF_BT4}, 137 { 1 << 23, 64, LZMA_MF_BT4}, 138 { 1 << 24, 64, LZMA_MF_BT4}, 139 { 3 << 23, 64, LZMA_MF_BT4}, 140 { 1 << 25, 64, LZMA_MF_BT4} 141 }; 142 143 static int 144 common_setup(struct archive_write_filter *f) 145 { 146 struct private_data *data; 147 struct archive_write *a = (struct archive_write *)f->archive; 148 data = calloc(1, sizeof(*data)); 149 if (data == NULL) { 150 archive_set_error(&a->archive, ENOMEM, "Out of memory"); 151 return (ARCHIVE_FATAL); 152 } 153 f->data = data; 154 data->compression_level = LZMA_PRESET_DEFAULT; 155 data->threads = 1; 156 f->open = &archive_compressor_xz_open; 157 f->close = archive_compressor_xz_close; 158 f->free = archive_compressor_xz_free; 159 f->options = &archive_compressor_xz_options; 160 return (ARCHIVE_OK); 161 } 162 163 /* 164 * Add an xz compression filter to this write handle. 165 */ 166 int 167 archive_write_add_filter_xz(struct archive *_a) 168 { 169 struct archive_write_filter *f; 170 int r; 171 172 archive_check_magic(_a, ARCHIVE_WRITE_MAGIC, 173 ARCHIVE_STATE_NEW, "archive_write_add_filter_xz"); 174 f = __archive_write_allocate_filter(_a); 175 r = common_setup(f); 176 if (r == ARCHIVE_OK) { 177 f->code = ARCHIVE_FILTER_XZ; 178 f->name = "xz"; 179 } 180 return (r); 181 } 182 183 /* LZMA is handled identically, we just need a different compression 184 * code set. (The liblzma setup looks at the code to determine 185 * the one place that XZ and LZMA require different handling.) */ 186 int 187 archive_write_add_filter_lzma(struct archive *_a) 188 { 189 struct archive_write_filter *f; 190 int r; 191 192 archive_check_magic(_a, ARCHIVE_WRITE_MAGIC, 193 ARCHIVE_STATE_NEW, "archive_write_add_filter_lzma"); 194 f = __archive_write_allocate_filter(_a); 195 r = common_setup(f); 196 if (r == ARCHIVE_OK) { 197 f->code = ARCHIVE_FILTER_LZMA; 198 f->name = "lzma"; 199 } 200 return (r); 201 } 202 203 int 204 archive_write_add_filter_lzip(struct archive *_a) 205 { 206 struct archive_write_filter *f; 207 int r; 208 209 archive_check_magic(_a, ARCHIVE_WRITE_MAGIC, 210 ARCHIVE_STATE_NEW, "archive_write_add_filter_lzip"); 211 f = __archive_write_allocate_filter(_a); 212 r = common_setup(f); 213 if (r == ARCHIVE_OK) { 214 f->code = ARCHIVE_FILTER_LZIP; 215 f->name = "lzip"; 216 } 217 return (r); 218 } 219 220 static int 221 archive_compressor_xz_init_stream(struct archive_write_filter *f, 222 struct private_data *data) 223 { 224 static const lzma_stream lzma_stream_init_data = LZMA_STREAM_INIT; 225 int ret; 226 #ifdef HAVE_LZMA_STREAM_ENCODER_MT 227 lzma_mt mt_options; 228 #endif 229 230 data->stream = lzma_stream_init_data; 231 data->stream.next_out = data->compressed; 232 data->stream.avail_out = data->compressed_buffer_size; 233 if (f->code == ARCHIVE_FILTER_XZ) { 234 #ifdef HAVE_LZMA_STREAM_ENCODER_MT 235 if (data->threads != 1) { 236 memset(&mt_options, 0, sizeof(mt_options)); 237 mt_options.threads = data->threads; 238 mt_options.timeout = 300; 239 mt_options.filters = data->lzmafilters; 240 mt_options.check = LZMA_CHECK_CRC64; 241 ret = lzma_stream_encoder_mt(&(data->stream), 242 &mt_options); 243 } else 244 #endif 245 ret = lzma_stream_encoder(&(data->stream), 246 data->lzmafilters, LZMA_CHECK_CRC64); 247 } else if (f->code == ARCHIVE_FILTER_LZMA) { 248 ret = lzma_alone_encoder(&(data->stream), &data->lzma_opt); 249 } else { /* ARCHIVE_FILTER_LZIP */ 250 int dict_size = data->lzma_opt.dict_size; 251 int ds, log2dic, wedges; 252 253 /* Calculate a coded dictionary size */ 254 if (dict_size < (1 << 12) || dict_size > (1 << 29)) { 255 archive_set_error(f->archive, ARCHIVE_ERRNO_MISC, 256 "Unacceptable dictionary size for lzip: %d", 257 dict_size); 258 return (ARCHIVE_FATAL); 259 } 260 for (log2dic = 29; log2dic >= 12; log2dic--) { 261 if (dict_size & (1 << log2dic)) 262 break; 263 } 264 if (dict_size > (1 << log2dic)) { 265 log2dic++; 266 wedges = 267 ((1 << log2dic) - dict_size) / (1 << (log2dic - 4)); 268 } else 269 wedges = 0; 270 ds = ((wedges << 5) & 0xe0) | (log2dic & 0x1f); 271 272 data->crc32 = 0; 273 /* Make a header */ 274 data->compressed[0] = 0x4C; 275 data->compressed[1] = 0x5A; 276 data->compressed[2] = 0x49; 277 data->compressed[3] = 0x50; 278 data->compressed[4] = 1;/* Version */ 279 data->compressed[5] = (unsigned char)ds; 280 data->stream.next_out += 6; 281 data->stream.avail_out -= 6; 282 283 ret = lzma_raw_encoder(&(data->stream), data->lzmafilters); 284 } 285 if (ret == LZMA_OK) 286 return (ARCHIVE_OK); 287 288 switch (ret) { 289 case LZMA_MEM_ERROR: 290 archive_set_error(f->archive, ENOMEM, 291 "Internal error initializing compression library: " 292 "Cannot allocate memory"); 293 break; 294 default: 295 archive_set_error(f->archive, ARCHIVE_ERRNO_MISC, 296 "Internal error initializing compression library: " 297 "It's a bug in liblzma"); 298 break; 299 } 300 return (ARCHIVE_FATAL); 301 } 302 303 /* 304 * Setup callback. 305 */ 306 static int 307 archive_compressor_xz_open(struct archive_write_filter *f) 308 { 309 struct private_data *data = f->data; 310 int ret; 311 312 if (data->compressed == NULL) { 313 size_t bs = 65536, bpb; 314 if (f->archive->magic == ARCHIVE_WRITE_MAGIC) { 315 /* Buffer size should be a multiple number of the of bytes 316 * per block for performance. */ 317 bpb = archive_write_get_bytes_per_block(f->archive); 318 if (bpb > bs) 319 bs = bpb; 320 else if (bpb != 0) 321 bs -= bs % bpb; 322 } 323 data->compressed_buffer_size = bs; 324 data->compressed 325 = (unsigned char *)malloc(data->compressed_buffer_size); 326 if (data->compressed == NULL) { 327 archive_set_error(f->archive, ENOMEM, 328 "Can't allocate data for compression buffer"); 329 return (ARCHIVE_FATAL); 330 } 331 } 332 333 f->write = archive_compressor_xz_write; 334 335 /* Initialize compression library. */ 336 if (f->code == ARCHIVE_FILTER_LZIP) { 337 const struct option_value *val = 338 &option_values[data->compression_level]; 339 340 data->lzma_opt.dict_size = val->dict_size; 341 data->lzma_opt.preset_dict = NULL; 342 data->lzma_opt.preset_dict_size = 0; 343 data->lzma_opt.lc = LZMA_LC_DEFAULT; 344 data->lzma_opt.lp = LZMA_LP_DEFAULT; 345 data->lzma_opt.pb = LZMA_PB_DEFAULT; 346 data->lzma_opt.mode = 347 data->compression_level<= 2? LZMA_MODE_FAST:LZMA_MODE_NORMAL; 348 data->lzma_opt.nice_len = val->nice_len; 349 data->lzma_opt.mf = val->mf; 350 data->lzma_opt.depth = 0; 351 data->lzmafilters[0].id = LZMA_FILTER_LZMA1; 352 data->lzmafilters[0].options = &data->lzma_opt; 353 data->lzmafilters[1].id = LZMA_VLI_UNKNOWN;/* Terminate */ 354 } else { 355 if (lzma_lzma_preset(&data->lzma_opt, data->compression_level)) { 356 archive_set_error(f->archive, ARCHIVE_ERRNO_MISC, 357 "Internal error initializing compression library"); 358 } 359 data->lzmafilters[0].id = LZMA_FILTER_LZMA2; 360 data->lzmafilters[0].options = &data->lzma_opt; 361 data->lzmafilters[1].id = LZMA_VLI_UNKNOWN;/* Terminate */ 362 } 363 ret = archive_compressor_xz_init_stream(f, data); 364 if (ret == LZMA_OK) { 365 f->data = data; 366 return (0); 367 } 368 return (ARCHIVE_FATAL); 369 } 370 371 /* 372 * Set write options. 373 */ 374 static int 375 archive_compressor_xz_options(struct archive_write_filter *f, 376 const char *key, const char *value) 377 { 378 struct private_data *data = (struct private_data *)f->data; 379 380 if (strcmp(key, "compression-level") == 0) { 381 if (value == NULL || !(value[0] >= '0' && value[0] <= '9') || 382 value[1] != '\0') 383 return (ARCHIVE_WARN); 384 data->compression_level = value[0] - '0'; 385 if (data->compression_level > 9) 386 data->compression_level = 9; 387 return (ARCHIVE_OK); 388 } else if (strcmp(key, "threads") == 0) { 389 char *endptr; 390 391 if (value == NULL) 392 return (ARCHIVE_WARN); 393 errno = 0; 394 data->threads = (int)strtoul(value, &endptr, 10); 395 if (errno != 0 || *endptr != '\0') { 396 data->threads = 1; 397 return (ARCHIVE_WARN); 398 } 399 if (data->threads == 0) { 400 #ifdef HAVE_LZMA_STREAM_ENCODER_MT 401 data->threads = lzma_cputhreads(); 402 #else 403 data->threads = 1; 404 #endif 405 } 406 return (ARCHIVE_OK); 407 } 408 409 /* Note: The "warn" return is just to inform the options 410 * supervisor that we didn't handle it. It will generate 411 * a suitable error if no one used this option. */ 412 return (ARCHIVE_WARN); 413 } 414 415 /* 416 * Write data to the compressed stream. 417 */ 418 static int 419 archive_compressor_xz_write(struct archive_write_filter *f, 420 const void *buff, size_t length) 421 { 422 struct private_data *data = (struct private_data *)f->data; 423 int ret; 424 425 /* Update statistics */ 426 data->total_in += length; 427 if (f->code == ARCHIVE_FILTER_LZIP) 428 data->crc32 = lzma_crc32(buff, length, data->crc32); 429 430 /* Compress input data to output buffer */ 431 data->stream.next_in = buff; 432 data->stream.avail_in = length; 433 if ((ret = drive_compressor(f, data, 0)) != ARCHIVE_OK) 434 return (ret); 435 436 return (ARCHIVE_OK); 437 } 438 439 440 /* 441 * Finish the compression... 442 */ 443 static int 444 archive_compressor_xz_close(struct archive_write_filter *f) 445 { 446 struct private_data *data = (struct private_data *)f->data; 447 int ret; 448 449 ret = drive_compressor(f, data, 1); 450 if (ret == ARCHIVE_OK) { 451 data->total_out += 452 data->compressed_buffer_size - data->stream.avail_out; 453 ret = __archive_write_filter(f->next_filter, 454 data->compressed, 455 data->compressed_buffer_size - data->stream.avail_out); 456 if (f->code == ARCHIVE_FILTER_LZIP && ret == ARCHIVE_OK) { 457 archive_le32enc(data->compressed, data->crc32); 458 archive_le64enc(data->compressed+4, data->total_in); 459 archive_le64enc(data->compressed+12, data->total_out + 20); 460 ret = __archive_write_filter(f->next_filter, 461 data->compressed, 20); 462 } 463 } 464 lzma_end(&(data->stream)); 465 return ret; 466 } 467 468 static int 469 archive_compressor_xz_free(struct archive_write_filter *f) 470 { 471 struct private_data *data = (struct private_data *)f->data; 472 free(data->compressed); 473 free(data); 474 f->data = NULL; 475 return (ARCHIVE_OK); 476 } 477 478 /* 479 * Utility function to push input data through compressor, 480 * writing full output blocks as necessary. 481 * 482 * Note that this handles both the regular write case (finishing == 483 * false) and the end-of-archive case (finishing == true). 484 */ 485 static int 486 drive_compressor(struct archive_write_filter *f, 487 struct private_data *data, int finishing) 488 { 489 int ret; 490 491 for (;;) { 492 if (data->stream.avail_out == 0) { 493 data->total_out += data->compressed_buffer_size; 494 ret = __archive_write_filter(f->next_filter, 495 data->compressed, 496 data->compressed_buffer_size); 497 if (ret != ARCHIVE_OK) 498 return (ARCHIVE_FATAL); 499 data->stream.next_out = data->compressed; 500 data->stream.avail_out = data->compressed_buffer_size; 501 } 502 503 /* If there's nothing to do, we're done. */ 504 if (!finishing && data->stream.avail_in == 0) 505 return (ARCHIVE_OK); 506 507 ret = lzma_code(&(data->stream), 508 finishing ? LZMA_FINISH : LZMA_RUN ); 509 510 switch (ret) { 511 case LZMA_OK: 512 /* In non-finishing case, check if compressor 513 * consumed everything */ 514 if (!finishing && data->stream.avail_in == 0) 515 return (ARCHIVE_OK); 516 /* In finishing case, this return always means 517 * there's more work */ 518 break; 519 case LZMA_STREAM_END: 520 /* This return can only occur in finishing case. */ 521 if (finishing) 522 return (ARCHIVE_OK); 523 archive_set_error(f->archive, ARCHIVE_ERRNO_MISC, 524 "lzma compression data error"); 525 return (ARCHIVE_FATAL); 526 case LZMA_MEMLIMIT_ERROR: 527 archive_set_error(f->archive, ENOMEM, 528 "lzma compression error: " 529 "%ju MiB would have been needed", 530 (uintmax_t)((lzma_memusage(&(data->stream)) 531 + 1024 * 1024 -1) 532 / (1024 * 1024))); 533 return (ARCHIVE_FATAL); 534 default: 535 /* Any other return value indicates an error. */ 536 archive_set_error(f->archive, ARCHIVE_ERRNO_MISC, 537 "lzma compression failed:" 538 " lzma_code() call returned status %d", 539 ret); 540 return (ARCHIVE_FATAL); 541 } 542 } 543 } 544 545 #endif /* HAVE_LZMA_H */ 546