1 /*-
2 * Copyright (c) 2017 Sean Purcell
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
15 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
16 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
17 * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
18 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
19 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
20 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
21 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
23 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26 #include "archive_platform.h"
27
28 __FBSDID("$FreeBSD$");
29
30
31 #ifdef HAVE_ERRNO_H
32 #include <errno.h>
33 #endif
34 #ifdef HAVE_STDLIB_H
35 #include <stdlib.h>
36 #endif
37 #ifdef HAVE_STRING_H
38 #include <string.h>
39 #endif
40 #ifdef HAVE_ZSTD_H
41 #include <zstd.h>
42 #endif
43
44 #include "archive.h"
45 #include "archive_private.h"
46 #include "archive_string.h"
47 #include "archive_write_private.h"
48
49 /* Don't compile this if we don't have zstd.h */
50
51 struct private_data {
52 int compression_level;
53 int threads;
54 #if HAVE_ZSTD_H && HAVE_LIBZSTD_COMPRESSOR
55 ZSTD_CStream *cstream;
56 int64_t total_in;
57 ZSTD_outBuffer out;
58 #else
59 struct archive_write_program_data *pdata;
60 #endif
61 };
62
63 /* If we don't have the library use default range values (zstdcli.c v1.4.0) */
64 #define CLEVEL_MIN -99
65 #define CLEVEL_STD_MIN 0 /* prior to 1.3.4 and more recent without using --fast */
66 #define CLEVEL_DEFAULT 3
67 #define CLEVEL_STD_MAX 19 /* without using --ultra */
68 #define CLEVEL_MAX 22
69
70 #define MINVER_NEGCLEVEL 10304
71 #define MINVER_MINCLEVEL 10306
72
73 static int archive_compressor_zstd_options(struct archive_write_filter *,
74 const char *, const char *);
75 static int archive_compressor_zstd_open(struct archive_write_filter *);
76 static int archive_compressor_zstd_write(struct archive_write_filter *,
77 const void *, size_t);
78 static int archive_compressor_zstd_close(struct archive_write_filter *);
79 static int archive_compressor_zstd_free(struct archive_write_filter *);
80 #if HAVE_ZSTD_H && HAVE_LIBZSTD_COMPRESSOR
81 static int drive_compressor(struct archive_write_filter *,
82 struct private_data *, int, const void *, size_t);
83 #endif
84
85
86 /*
87 * Add a zstd compression filter to this write handle.
88 */
89 int
archive_write_add_filter_zstd(struct archive * _a)90 archive_write_add_filter_zstd(struct archive *_a)
91 {
92 struct archive_write *a = (struct archive_write *)_a;
93 struct archive_write_filter *f = __archive_write_allocate_filter(_a);
94 struct private_data *data;
95 archive_check_magic(&a->archive, ARCHIVE_WRITE_MAGIC,
96 ARCHIVE_STATE_NEW, "archive_write_add_filter_zstd");
97
98 data = calloc(1, sizeof(*data));
99 if (data == NULL) {
100 archive_set_error(&a->archive, ENOMEM, "Out of memory");
101 return (ARCHIVE_FATAL);
102 }
103 f->data = data;
104 f->open = &archive_compressor_zstd_open;
105 f->options = &archive_compressor_zstd_options;
106 f->close = &archive_compressor_zstd_close;
107 f->free = &archive_compressor_zstd_free;
108 f->code = ARCHIVE_FILTER_ZSTD;
109 f->name = "zstd";
110 data->compression_level = CLEVEL_DEFAULT;
111 data->threads = 0;
112 #if HAVE_ZSTD_H && HAVE_LIBZSTD_COMPRESSOR
113 data->cstream = ZSTD_createCStream();
114 if (data->cstream == NULL) {
115 free(data);
116 archive_set_error(&a->archive, ENOMEM,
117 "Failed to allocate zstd compressor object");
118 return (ARCHIVE_FATAL);
119 }
120
121 return (ARCHIVE_OK);
122 #else
123 data->pdata = __archive_write_program_allocate("zstd");
124 if (data->pdata == NULL) {
125 free(data);
126 archive_set_error(&a->archive, ENOMEM, "Out of memory");
127 return (ARCHIVE_FATAL);
128 }
129 archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC,
130 "Using external zstd program");
131 return (ARCHIVE_WARN);
132 #endif
133 }
134
135 static int
archive_compressor_zstd_free(struct archive_write_filter * f)136 archive_compressor_zstd_free(struct archive_write_filter *f)
137 {
138 struct private_data *data = (struct private_data *)f->data;
139 #if HAVE_ZSTD_H && HAVE_LIBZSTD_COMPRESSOR
140 ZSTD_freeCStream(data->cstream);
141 free(data->out.dst);
142 #else
143 __archive_write_program_free(data->pdata);
144 #endif
145 free(data);
146 f->data = NULL;
147 return (ARCHIVE_OK);
148 }
149
string_is_numeric(const char * value)150 static int string_is_numeric (const char* value)
151 {
152 size_t len = strlen(value);
153 size_t i;
154
155 if (len == 0) {
156 return (ARCHIVE_WARN);
157 }
158 else if (len == 1 && !(value[0] >= '0' && value[0] <= '9')) {
159 return (ARCHIVE_WARN);
160 }
161 else if (!(value[0] >= '0' && value[0] <= '9') &&
162 value[0] != '-' && value[0] != '+') {
163 return (ARCHIVE_WARN);
164 }
165
166 for (i = 1; i < len; i++) {
167 if (!(value[i] >= '0' && value[i] <= '9')) {
168 return (ARCHIVE_WARN);
169 }
170 }
171
172 return (ARCHIVE_OK);
173 }
174
175 /*
176 * Set write options.
177 */
178 static int
archive_compressor_zstd_options(struct archive_write_filter * f,const char * key,const char * value)179 archive_compressor_zstd_options(struct archive_write_filter *f, const char *key,
180 const char *value)
181 {
182 struct private_data *data = (struct private_data *)f->data;
183
184 if (strcmp(key, "compression-level") == 0) {
185 int level = atoi(value);
186 /* If we don't have the library, hard-code the max level */
187 int minimum = CLEVEL_MIN;
188 int maximum = CLEVEL_MAX;
189 if (string_is_numeric(value) != ARCHIVE_OK) {
190 return (ARCHIVE_WARN);
191 }
192 #if HAVE_ZSTD_H && HAVE_LIBZSTD_COMPRESSOR
193 maximum = ZSTD_maxCLevel();
194 #if ZSTD_VERSION_NUMBER >= MINVER_MINCLEVEL
195 if (ZSTD_versionNumber() >= MINVER_MINCLEVEL) {
196 minimum = ZSTD_minCLevel();
197 }
198 else
199 #endif
200 if (ZSTD_versionNumber() < MINVER_NEGCLEVEL) {
201 minimum = CLEVEL_STD_MIN;
202 }
203 #endif
204 if (level < minimum || level > maximum) {
205 return (ARCHIVE_WARN);
206 }
207 data->compression_level = level;
208 return (ARCHIVE_OK);
209 } else if (strcmp(key, "threads") == 0) {
210 int threads = atoi(value);
211 if (string_is_numeric(value) != ARCHIVE_OK) {
212 return (ARCHIVE_WARN);
213 }
214
215 int minimum = 0;
216
217 if (threads < minimum) {
218 return (ARCHIVE_WARN);
219 }
220
221 data->threads = threads;
222 return (ARCHIVE_OK);
223 }
224
225 /* Note: The "warn" return is just to inform the options
226 * supervisor that we didn't handle it. It will generate
227 * a suitable error if no one used this option. */
228 return (ARCHIVE_WARN);
229 }
230
231 #if HAVE_ZSTD_H && HAVE_LIBZSTD_COMPRESSOR
232 /*
233 * Setup callback.
234 */
235 static int
archive_compressor_zstd_open(struct archive_write_filter * f)236 archive_compressor_zstd_open(struct archive_write_filter *f)
237 {
238 struct private_data *data = (struct private_data *)f->data;
239
240 if (data->out.dst == NULL) {
241 size_t bs = ZSTD_CStreamOutSize(), bpb;
242 if (f->archive->magic == ARCHIVE_WRITE_MAGIC) {
243 /* Buffer size should be a multiple number of
244 * the of bytes per block for performance. */
245 bpb = archive_write_get_bytes_per_block(f->archive);
246 if (bpb > bs)
247 bs = bpb;
248 else if (bpb != 0)
249 bs -= bs % bpb;
250 }
251 data->out.size = bs;
252 data->out.pos = 0;
253 data->out.dst
254 = (unsigned char *)malloc(data->out.size);
255 if (data->out.dst == NULL) {
256 archive_set_error(f->archive, ENOMEM,
257 "Can't allocate data for compression buffer");
258 return (ARCHIVE_FATAL);
259 }
260 }
261
262 f->write = archive_compressor_zstd_write;
263
264 if (ZSTD_isError(ZSTD_initCStream(data->cstream,
265 data->compression_level))) {
266 archive_set_error(f->archive, ARCHIVE_ERRNO_MISC,
267 "Internal error initializing zstd compressor object");
268 return (ARCHIVE_FATAL);
269 }
270
271 ZSTD_CCtx_setParameter(data->cstream, ZSTD_c_nbWorkers, data->threads);
272
273 return (ARCHIVE_OK);
274 }
275
276 /*
277 * Write data to the compressed stream.
278 */
279 static int
archive_compressor_zstd_write(struct archive_write_filter * f,const void * buff,size_t length)280 archive_compressor_zstd_write(struct archive_write_filter *f, const void *buff,
281 size_t length)
282 {
283 struct private_data *data = (struct private_data *)f->data;
284 int ret;
285
286 /* Update statistics */
287 data->total_in += length;
288
289 if ((ret = drive_compressor(f, data, 0, buff, length)) != ARCHIVE_OK)
290 return (ret);
291
292 return (ARCHIVE_OK);
293 }
294
295 /*
296 * Finish the compression...
297 */
298 static int
archive_compressor_zstd_close(struct archive_write_filter * f)299 archive_compressor_zstd_close(struct archive_write_filter *f)
300 {
301 struct private_data *data = (struct private_data *)f->data;
302
303 /* Finish zstd frame */
304 return drive_compressor(f, data, 1, NULL, 0);
305 }
306
307 /*
308 * Utility function to push input data through compressor,
309 * writing full output blocks as necessary.
310 *
311 * Note that this handles both the regular write case (finishing ==
312 * false) and the end-of-archive case (finishing == true).
313 */
314 static int
drive_compressor(struct archive_write_filter * f,struct private_data * data,int finishing,const void * src,size_t length)315 drive_compressor(struct archive_write_filter *f,
316 struct private_data *data, int finishing, const void *src, size_t length)
317 {
318 ZSTD_inBuffer in = (ZSTD_inBuffer) { src, length, 0 };
319
320 for (;;) {
321 if (data->out.pos == data->out.size) {
322 const int ret = __archive_write_filter(f->next_filter,
323 data->out.dst, data->out.size);
324 if (ret != ARCHIVE_OK)
325 return (ARCHIVE_FATAL);
326 data->out.pos = 0;
327 }
328
329 /* If there's nothing to do, we're done. */
330 if (!finishing && in.pos == in.size)
331 return (ARCHIVE_OK);
332
333 {
334 const size_t zstdret = !finishing ?
335 ZSTD_compressStream(data->cstream, &data->out, &in)
336 : ZSTD_endStream(data->cstream, &data->out);
337
338 if (ZSTD_isError(zstdret)) {
339 archive_set_error(f->archive,
340 ARCHIVE_ERRNO_MISC,
341 "Zstd compression failed: %s",
342 ZSTD_getErrorName(zstdret));
343 return (ARCHIVE_FATAL);
344 }
345
346 /* If we're finishing, 0 means nothing left to flush */
347 if (finishing && zstdret == 0) {
348 const int ret = __archive_write_filter(f->next_filter,
349 data->out.dst, data->out.pos);
350 return (ret);
351 }
352 }
353 }
354 }
355
356 #else /* HAVE_ZSTD_H && HAVE_LIBZSTD_COMPRESSOR */
357
358 static int
archive_compressor_zstd_open(struct archive_write_filter * f)359 archive_compressor_zstd_open(struct archive_write_filter *f)
360 {
361 struct private_data *data = (struct private_data *)f->data;
362 struct archive_string as;
363 int r;
364
365 archive_string_init(&as);
366 /* --no-check matches library default */
367 archive_strcpy(&as, "zstd --no-check");
368
369 if (data->compression_level < CLEVEL_STD_MIN) {
370 struct archive_string as2;
371 archive_string_init(&as2);
372 archive_string_sprintf(&as2, " --fast=%d", -data->compression_level);
373 archive_string_concat(&as, &as2);
374 archive_string_free(&as2);
375 } else {
376 struct archive_string as2;
377 archive_string_init(&as2);
378 archive_string_sprintf(&as2, " -%d", data->compression_level);
379 archive_string_concat(&as, &as2);
380 archive_string_free(&as2);
381 }
382
383 if (data->compression_level > CLEVEL_STD_MAX) {
384 archive_strcat(&as, " --ultra");
385 }
386
387 if (data->threads != 0) {
388 struct archive_string as2;
389 archive_string_init(&as2);
390 archive_string_sprintf(&as2, " --threads=%d", data->threads);
391 archive_string_concat(&as, &as2);
392 archive_string_free(&as2);
393 }
394
395 f->write = archive_compressor_zstd_write;
396 r = __archive_write_program_open(f, data->pdata, as.s);
397 archive_string_free(&as);
398 return (r);
399 }
400
401 static int
archive_compressor_zstd_write(struct archive_write_filter * f,const void * buff,size_t length)402 archive_compressor_zstd_write(struct archive_write_filter *f, const void *buff,
403 size_t length)
404 {
405 struct private_data *data = (struct private_data *)f->data;
406
407 return __archive_write_program_write(f, data->pdata, buff, length);
408 }
409
410 static int
archive_compressor_zstd_close(struct archive_write_filter * f)411 archive_compressor_zstd_close(struct archive_write_filter *f)
412 {
413 struct private_data *data = (struct private_data *)f->data;
414
415 return __archive_write_program_close(f, data->pdata);
416 }
417
418 #endif /* HAVE_ZSTD_H && HAVE_LIBZSTD_COMPRESSOR */
419