1 /*- 2 * Copyright (c) 2017 Sean Purcell 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR 15 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 16 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 17 * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, 18 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 19 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 20 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 21 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 23 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 */ 25 26 #include "archive_platform.h" 27 28 __FBSDID("$FreeBSD$"); 29 30 31 #ifdef HAVE_ERRNO_H 32 #include <errno.h> 33 #endif 34 #ifdef HAVE_STDLIB_H 35 #include <stdlib.h> 36 #endif 37 #ifdef HAVE_STRING_H 38 #include <string.h> 39 #endif 40 #ifdef HAVE_ZSTD_H 41 #include <zstd.h> 42 #endif 43 44 #include "archive.h" 45 #include "archive_private.h" 46 #include "archive_string.h" 47 #include "archive_write_private.h" 48 49 /* Don't compile this if we don't have zstd.h */ 50 51 struct private_data { 52 int compression_level; 53 int threads; 54 #if HAVE_ZSTD_H && HAVE_LIBZSTD_COMPRESSOR 55 ZSTD_CStream *cstream; 56 int64_t total_in; 57 ZSTD_outBuffer out; 58 #else 59 struct archive_write_program_data *pdata; 60 #endif 61 }; 62 63 /* If we don't have the library use default range values (zstdcli.c v1.4.0) */ 64 #define CLEVEL_MIN -99 65 #define CLEVEL_STD_MIN 0 /* prior to 1.3.4 and more recent without using --fast */ 66 #define CLEVEL_DEFAULT 3 67 #define CLEVEL_STD_MAX 19 /* without using --ultra */ 68 #define CLEVEL_MAX 22 69 70 #define MINVER_NEGCLEVEL 10304 71 #define MINVER_MINCLEVEL 10306 72 73 static int archive_compressor_zstd_options(struct archive_write_filter *, 74 const char *, const char *); 75 static int archive_compressor_zstd_open(struct archive_write_filter *); 76 static int archive_compressor_zstd_write(struct archive_write_filter *, 77 const void *, size_t); 78 static int archive_compressor_zstd_close(struct archive_write_filter *); 79 static int archive_compressor_zstd_free(struct archive_write_filter *); 80 #if HAVE_ZSTD_H && HAVE_LIBZSTD_COMPRESSOR 81 static int drive_compressor(struct archive_write_filter *, 82 struct private_data *, int, const void *, size_t); 83 #endif 84 85 86 /* 87 * Add a zstd compression filter to this write handle. 88 */ 89 int 90 archive_write_add_filter_zstd(struct archive *_a) 91 { 92 struct archive_write *a = (struct archive_write *)_a; 93 struct archive_write_filter *f = __archive_write_allocate_filter(_a); 94 struct private_data *data; 95 archive_check_magic(&a->archive, ARCHIVE_WRITE_MAGIC, 96 ARCHIVE_STATE_NEW, "archive_write_add_filter_zstd"); 97 98 data = calloc(1, sizeof(*data)); 99 if (data == NULL) { 100 archive_set_error(&a->archive, ENOMEM, "Out of memory"); 101 return (ARCHIVE_FATAL); 102 } 103 f->data = data; 104 f->open = &archive_compressor_zstd_open; 105 f->options = &archive_compressor_zstd_options; 106 f->close = &archive_compressor_zstd_close; 107 f->free = &archive_compressor_zstd_free; 108 f->code = ARCHIVE_FILTER_ZSTD; 109 f->name = "zstd"; 110 data->compression_level = CLEVEL_DEFAULT; 111 data->threads = 0; 112 #if HAVE_ZSTD_H && HAVE_LIBZSTD_COMPRESSOR 113 data->cstream = ZSTD_createCStream(); 114 if (data->cstream == NULL) { 115 free(data); 116 archive_set_error(&a->archive, ENOMEM, 117 "Failed to allocate zstd compressor object"); 118 return (ARCHIVE_FATAL); 119 } 120 121 return (ARCHIVE_OK); 122 #else 123 data->pdata = __archive_write_program_allocate("zstd"); 124 if (data->pdata == NULL) { 125 free(data); 126 archive_set_error(&a->archive, ENOMEM, "Out of memory"); 127 return (ARCHIVE_FATAL); 128 } 129 archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, 130 "Using external zstd program"); 131 return (ARCHIVE_WARN); 132 #endif 133 } 134 135 static int 136 archive_compressor_zstd_free(struct archive_write_filter *f) 137 { 138 struct private_data *data = (struct private_data *)f->data; 139 #if HAVE_ZSTD_H && HAVE_LIBZSTD_COMPRESSOR 140 ZSTD_freeCStream(data->cstream); 141 free(data->out.dst); 142 #else 143 __archive_write_program_free(data->pdata); 144 #endif 145 free(data); 146 f->data = NULL; 147 return (ARCHIVE_OK); 148 } 149 150 static int string_is_numeric (const char* value) 151 { 152 size_t len = strlen(value); 153 size_t i; 154 155 if (len == 0) { 156 return (ARCHIVE_WARN); 157 } 158 else if (len == 1 && !(value[0] >= '0' && value[0] <= '9')) { 159 return (ARCHIVE_WARN); 160 } 161 else if (!(value[0] >= '0' && value[0] <= '9') && 162 value[0] != '-' && value[0] != '+') { 163 return (ARCHIVE_WARN); 164 } 165 166 for (i = 1; i < len; i++) { 167 if (!(value[i] >= '0' && value[i] <= '9')) { 168 return (ARCHIVE_WARN); 169 } 170 } 171 172 return (ARCHIVE_OK); 173 } 174 175 /* 176 * Set write options. 177 */ 178 static int 179 archive_compressor_zstd_options(struct archive_write_filter *f, const char *key, 180 const char *value) 181 { 182 struct private_data *data = (struct private_data *)f->data; 183 184 if (strcmp(key, "compression-level") == 0) { 185 int level = atoi(value); 186 /* If we don't have the library, hard-code the max level */ 187 int minimum = CLEVEL_MIN; 188 int maximum = CLEVEL_MAX; 189 if (string_is_numeric(value) != ARCHIVE_OK) { 190 return (ARCHIVE_WARN); 191 } 192 #if HAVE_ZSTD_H && HAVE_LIBZSTD_COMPRESSOR 193 maximum = ZSTD_maxCLevel(); 194 #if ZSTD_VERSION_NUMBER >= MINVER_MINCLEVEL 195 if (ZSTD_versionNumber() >= MINVER_MINCLEVEL) { 196 minimum = ZSTD_minCLevel(); 197 } 198 else 199 #endif 200 if (ZSTD_versionNumber() < MINVER_NEGCLEVEL) { 201 minimum = CLEVEL_STD_MIN; 202 } 203 #endif 204 if (level < minimum || level > maximum) { 205 return (ARCHIVE_WARN); 206 } 207 data->compression_level = level; 208 return (ARCHIVE_OK); 209 } else if (strcmp(key, "threads") == 0) { 210 int threads = atoi(value); 211 if (string_is_numeric(value) != ARCHIVE_OK) { 212 return (ARCHIVE_WARN); 213 } 214 215 int minimum = 0; 216 217 if (threads < minimum) { 218 return (ARCHIVE_WARN); 219 } 220 221 data->threads = threads; 222 return (ARCHIVE_OK); 223 } 224 225 /* Note: The "warn" return is just to inform the options 226 * supervisor that we didn't handle it. It will generate 227 * a suitable error if no one used this option. */ 228 return (ARCHIVE_WARN); 229 } 230 231 #if HAVE_ZSTD_H && HAVE_LIBZSTD_COMPRESSOR 232 /* 233 * Setup callback. 234 */ 235 static int 236 archive_compressor_zstd_open(struct archive_write_filter *f) 237 { 238 struct private_data *data = (struct private_data *)f->data; 239 240 if (data->out.dst == NULL) { 241 size_t bs = ZSTD_CStreamOutSize(), bpb; 242 if (f->archive->magic == ARCHIVE_WRITE_MAGIC) { 243 /* Buffer size should be a multiple number of 244 * the of bytes per block for performance. */ 245 bpb = archive_write_get_bytes_per_block(f->archive); 246 if (bpb > bs) 247 bs = bpb; 248 else if (bpb != 0) 249 bs -= bs % bpb; 250 } 251 data->out.size = bs; 252 data->out.pos = 0; 253 data->out.dst 254 = (unsigned char *)malloc(data->out.size); 255 if (data->out.dst == NULL) { 256 archive_set_error(f->archive, ENOMEM, 257 "Can't allocate data for compression buffer"); 258 return (ARCHIVE_FATAL); 259 } 260 } 261 262 f->write = archive_compressor_zstd_write; 263 264 if (ZSTD_isError(ZSTD_initCStream(data->cstream, 265 data->compression_level))) { 266 archive_set_error(f->archive, ARCHIVE_ERRNO_MISC, 267 "Internal error initializing zstd compressor object"); 268 return (ARCHIVE_FATAL); 269 } 270 271 ZSTD_CCtx_setParameter(data->cstream, ZSTD_c_nbWorkers, data->threads); 272 273 return (ARCHIVE_OK); 274 } 275 276 /* 277 * Write data to the compressed stream. 278 */ 279 static int 280 archive_compressor_zstd_write(struct archive_write_filter *f, const void *buff, 281 size_t length) 282 { 283 struct private_data *data = (struct private_data *)f->data; 284 int ret; 285 286 /* Update statistics */ 287 data->total_in += length; 288 289 if ((ret = drive_compressor(f, data, 0, buff, length)) != ARCHIVE_OK) 290 return (ret); 291 292 return (ARCHIVE_OK); 293 } 294 295 /* 296 * Finish the compression... 297 */ 298 static int 299 archive_compressor_zstd_close(struct archive_write_filter *f) 300 { 301 struct private_data *data = (struct private_data *)f->data; 302 303 /* Finish zstd frame */ 304 return drive_compressor(f, data, 1, NULL, 0); 305 } 306 307 /* 308 * Utility function to push input data through compressor, 309 * writing full output blocks as necessary. 310 * 311 * Note that this handles both the regular write case (finishing == 312 * false) and the end-of-archive case (finishing == true). 313 */ 314 static int 315 drive_compressor(struct archive_write_filter *f, 316 struct private_data *data, int finishing, const void *src, size_t length) 317 { 318 ZSTD_inBuffer in = (ZSTD_inBuffer) { src, length, 0 }; 319 320 for (;;) { 321 if (data->out.pos == data->out.size) { 322 const int ret = __archive_write_filter(f->next_filter, 323 data->out.dst, data->out.size); 324 if (ret != ARCHIVE_OK) 325 return (ARCHIVE_FATAL); 326 data->out.pos = 0; 327 } 328 329 /* If there's nothing to do, we're done. */ 330 if (!finishing && in.pos == in.size) 331 return (ARCHIVE_OK); 332 333 { 334 const size_t zstdret = !finishing ? 335 ZSTD_compressStream(data->cstream, &data->out, &in) 336 : ZSTD_endStream(data->cstream, &data->out); 337 338 if (ZSTD_isError(zstdret)) { 339 archive_set_error(f->archive, 340 ARCHIVE_ERRNO_MISC, 341 "Zstd compression failed: %s", 342 ZSTD_getErrorName(zstdret)); 343 return (ARCHIVE_FATAL); 344 } 345 346 /* If we're finishing, 0 means nothing left to flush */ 347 if (finishing && zstdret == 0) { 348 const int ret = __archive_write_filter(f->next_filter, 349 data->out.dst, data->out.pos); 350 return (ret); 351 } 352 } 353 } 354 } 355 356 #else /* HAVE_ZSTD_H && HAVE_LIBZSTD_COMPRESSOR */ 357 358 static int 359 archive_compressor_zstd_open(struct archive_write_filter *f) 360 { 361 struct private_data *data = (struct private_data *)f->data; 362 struct archive_string as; 363 int r; 364 365 archive_string_init(&as); 366 /* --no-check matches library default */ 367 archive_strcpy(&as, "zstd --no-check"); 368 369 if (data->compression_level < CLEVEL_STD_MIN) { 370 struct archive_string as2; 371 archive_string_init(&as2); 372 archive_string_sprintf(&as2, " --fast=%d", -data->compression_level); 373 archive_string_concat(&as, &as2); 374 archive_string_free(&as2); 375 } else { 376 struct archive_string as2; 377 archive_string_init(&as2); 378 archive_string_sprintf(&as2, " -%d", data->compression_level); 379 archive_string_concat(&as, &as2); 380 archive_string_free(&as2); 381 } 382 383 if (data->compression_level > CLEVEL_STD_MAX) { 384 archive_strcat(&as, " --ultra"); 385 } 386 387 if (data->threads != 0) { 388 struct archive_string as2; 389 archive_string_init(&as2); 390 archive_string_sprintf(&as2, " --threads=%d", data->threads); 391 archive_string_concat(&as, &as2); 392 archive_string_free(&as2); 393 } 394 395 f->write = archive_compressor_zstd_write; 396 r = __archive_write_program_open(f, data->pdata, as.s); 397 archive_string_free(&as); 398 return (r); 399 } 400 401 static int 402 archive_compressor_zstd_write(struct archive_write_filter *f, const void *buff, 403 size_t length) 404 { 405 struct private_data *data = (struct private_data *)f->data; 406 407 return __archive_write_program_write(f, data->pdata, buff, length); 408 } 409 410 static int 411 archive_compressor_zstd_close(struct archive_write_filter *f) 412 { 413 struct private_data *data = (struct private_data *)f->data; 414 415 return __archive_write_program_close(f, data->pdata); 416 } 417 418 #endif /* HAVE_ZSTD_H && HAVE_LIBZSTD_COMPRESSOR */ 419