xref: /freebsd/sys/kern/subr_compressor.c (revision e17f5b1d) 
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2014, 2017 Mark Johnston <markj@FreeBSD.org>
5  * Copyright (c) 2017 Conrad Meyer <cem@FreeBSD.org>
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions are
9  * met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in
14  *    the documentation and/or other materials provided with the
15  *    distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27  * SUCH DAMAGE.
28  */
29 
30 /*
31  * Subroutines used for writing compressed user process and kernel core dumps.
32  */
33 
34 #include <sys/cdefs.h>
35 __FBSDID("$FreeBSD$");
36 
37 #include "opt_gzio.h"
38 #include "opt_zstdio.h"
39 
40 #include <sys/param.h>
41 #include <sys/systm.h>
42 
43 #include <sys/compressor.h>
44 #include <sys/kernel.h>
45 #include <sys/linker_set.h>
46 #include <sys/malloc.h>
47 
48 MALLOC_DEFINE(M_COMPRESS, "compressor", "kernel compression subroutines");
49 
50 struct compressor_methods {
51 	int format;
52 	void *(* const init)(size_t, int);
53 	void (* const reset)(void *);
54 	int (* const write)(void *, void *, size_t, compressor_cb_t, void *);
55 	void (* const fini)(void *);
56 };
57 
58 struct compressor {
59 	const struct compressor_methods *methods;
60 	compressor_cb_t cb;
61 	void *priv;
62 	void *arg;
63 };
64 
65 SET_DECLARE(compressors, struct compressor_methods);
66 
67 #ifdef GZIO
68 
69 #include <contrib/zlib/zutil.h>
70 
71 struct gz_stream {
72 	uint8_t		*gz_buffer;	/* output buffer */
73 	size_t		gz_bufsz;	/* output buffer size */
74 	off_t		gz_off;		/* offset into the output stream */
75 	uint32_t	gz_crc;		/* stream CRC32 */
76 	z_stream	gz_stream;	/* zlib state */
77 };
78 
79 static void 	*gz_init(size_t maxiosize, int level);
80 static void	gz_reset(void *stream);
81 static int	gz_write(void *stream, void *data, size_t len, compressor_cb_t,
82 		    void *);
83 static void	gz_fini(void *stream);
84 
85 static void *
86 gz_alloc(void *arg __unused, u_int n, u_int sz)
87 {
88 
89 	/*
90 	 * Memory for zlib state is allocated using M_NODUMP since it may be
91 	 * used to compress a kernel dump, and we don't want zlib to attempt to
92 	 * compress its own state.
93 	 */
94 	return (malloc(n * sz, M_COMPRESS, M_WAITOK | M_ZERO | M_NODUMP));
95 }
96 
97 static void
98 gz_free(void *arg __unused, void *ptr)
99 {
100 
101 	free(ptr, M_COMPRESS);
102 }
103 
104 static void *
105 gz_init(size_t maxiosize, int level)
106 {
107 	struct gz_stream *s;
108 	int error;
109 
110 	s = gz_alloc(NULL, 1, roundup2(sizeof(*s), PAGE_SIZE));
111 	s->gz_buffer = gz_alloc(NULL, 1, maxiosize);
112 	s->gz_bufsz = maxiosize;
113 
114 	s->gz_stream.zalloc = gz_alloc;
115 	s->gz_stream.zfree = gz_free;
116 	s->gz_stream.opaque = NULL;
117 	s->gz_stream.next_in = Z_NULL;
118 	s->gz_stream.avail_in = 0;
119 
120 	if (level != Z_DEFAULT_COMPRESSION) {
121 		if (level < Z_BEST_SPEED)
122 			level = Z_BEST_SPEED;
123 		else if (level > Z_BEST_COMPRESSION)
124 			level = Z_BEST_COMPRESSION;
125 	}
126 
127 	error = deflateInit2(&s->gz_stream, level, Z_DEFLATED, -MAX_WBITS,
128 	    DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY);
129 	if (error != 0)
130 		goto fail;
131 
132 	gz_reset(s);
133 
134 	return (s);
135 
136 fail:
137 	gz_free(NULL, s);
138 	return (NULL);
139 }
140 
141 static void
142 gz_reset(void *stream)
143 {
144 	struct gz_stream *s;
145 	uint8_t *hdr;
146 	const size_t hdrlen = 10;
147 
148 	s = stream;
149 	s->gz_off = 0;
150 	s->gz_crc = crc32(0L, Z_NULL, 0);
151 
152 	(void)deflateReset(&s->gz_stream);
153 	s->gz_stream.avail_out = s->gz_bufsz;
154 	s->gz_stream.next_out = s->gz_buffer;
155 
156 	/* Write the gzip header to the output buffer. */
157 	hdr = s->gz_buffer;
158 	memset(hdr, 0, hdrlen);
159 	hdr[0] = 0x1f;
160 	hdr[1] = 0x8b;
161 	hdr[2] = Z_DEFLATED;
162 	hdr[9] = OS_CODE;
163 	s->gz_stream.next_out += hdrlen;
164 	s->gz_stream.avail_out -= hdrlen;
165 }
166 
167 static int
168 gz_write(void *stream, void *data, size_t len, compressor_cb_t cb,
169     void *arg)
170 {
171 	struct gz_stream *s;
172 	uint8_t trailer[8];
173 	size_t room;
174 	int error, zerror, zflag;
175 
176 	s = stream;
177 	zflag = data == NULL ? Z_FINISH : Z_NO_FLUSH;
178 
179 	if (len > 0) {
180 		s->gz_stream.avail_in = len;
181 		s->gz_stream.next_in = data;
182 		s->gz_crc = crc32(s->gz_crc, data, len);
183 	}
184 
185 	error = 0;
186 	do {
187 		zerror = deflate(&s->gz_stream, zflag);
188 		if (zerror != Z_OK && zerror != Z_STREAM_END) {
189 			error = EIO;
190 			break;
191 		}
192 
193 		if (s->gz_stream.avail_out == 0 || zerror == Z_STREAM_END) {
194 			/*
195 			 * Our output buffer is full or there's nothing left
196 			 * to produce, so we're flushing the buffer.
197 			 */
198 			len = s->gz_bufsz - s->gz_stream.avail_out;
199 			if (zerror == Z_STREAM_END) {
200 				/*
201 				 * Try to pack as much of the trailer into the
202 				 * output buffer as we can.
203 				 */
204 				((uint32_t *)trailer)[0] = s->gz_crc;
205 				((uint32_t *)trailer)[1] =
206 				    s->gz_stream.total_in;
207 				room = MIN(sizeof(trailer),
208 				    s->gz_bufsz - len);
209 				memcpy(s->gz_buffer + len, trailer, room);
210 				len += room;
211 			}
212 
213 			error = cb(s->gz_buffer, len, s->gz_off, arg);
214 			if (error != 0)
215 				break;
216 
217 			s->gz_off += len;
218 			s->gz_stream.next_out = s->gz_buffer;
219 			s->gz_stream.avail_out = s->gz_bufsz;
220 
221 			/*
222 			 * If we couldn't pack the trailer into the output
223 			 * buffer, write it out now.
224 			 */
225 			if (zerror == Z_STREAM_END && room < sizeof(trailer))
226 				error = cb(trailer + room,
227 				    sizeof(trailer) - room, s->gz_off, arg);
228 		}
229 	} while (zerror != Z_STREAM_END &&
230 	    (zflag == Z_FINISH || s->gz_stream.avail_in > 0));
231 
232 	return (error);
233 }
234 
235 static void
236 gz_fini(void *stream)
237 {
238 	struct gz_stream *s;
239 
240 	s = stream;
241 	(void)deflateEnd(&s->gz_stream);
242 	gz_free(NULL, s->gz_buffer);
243 	gz_free(NULL, s);
244 }
245 
246 struct compressor_methods gzip_methods = {
247 	.format = COMPRESS_GZIP,
248 	.init = gz_init,
249 	.reset = gz_reset,
250 	.write = gz_write,
251 	.fini = gz_fini,
252 };
253 DATA_SET(compressors, gzip_methods);
254 
255 #endif /* GZIO */
256 
257 #ifdef ZSTDIO
258 
259 #define	ZSTD_STATIC_LINKING_ONLY
260 #include <contrib/zstd/lib/zstd.h>
261 
262 struct zstdio_stream {
263 	ZSTD_CCtx	*zst_stream;
264 	ZSTD_inBuffer	zst_inbuffer;
265 	ZSTD_outBuffer	zst_outbuffer;
266 	uint8_t *	zst_buffer;	/* output buffer */
267 	size_t		zst_maxiosz;	/* Max output IO size */
268 	off_t		zst_off;	/* offset into the output stream */
269 	void *		zst_static_wkspc;
270 };
271 
272 static void 	*zstdio_init(size_t maxiosize, int level);
273 static void	zstdio_reset(void *stream);
274 static int	zstdio_write(void *stream, void *data, size_t len,
275 		    compressor_cb_t, void *);
276 static void	zstdio_fini(void *stream);
277 
278 static void *
279 zstdio_init(size_t maxiosize, int level)
280 {
281 	ZSTD_CCtx *dump_compressor;
282 	struct zstdio_stream *s;
283 	void *wkspc, *owkspc, *buffer;
284 	size_t wkspc_size, buf_size, rc;
285 
286 	s = NULL;
287 	wkspc_size = ZSTD_estimateCStreamSize(level);
288 	owkspc = wkspc = malloc(wkspc_size + 8, M_COMPRESS,
289 	    M_WAITOK | M_NODUMP);
290 	/* Zstd API requires 8-byte alignment. */
291 	if ((uintptr_t)wkspc % 8 != 0)
292 		wkspc = (void *)roundup2((uintptr_t)wkspc, 8);
293 
294 	dump_compressor = ZSTD_initStaticCCtx(wkspc, wkspc_size);
295 	if (dump_compressor == NULL) {
296 		printf("%s: workspace too small.\n", __func__);
297 		goto out;
298 	}
299 
300 	rc = ZSTD_CCtx_setParameter(dump_compressor, ZSTD_c_checksumFlag, 1);
301 	if (ZSTD_isError(rc)) {
302 		printf("%s: error setting checksumFlag: %s\n", __func__,
303 		    ZSTD_getErrorName(rc));
304 		goto out;
305 	}
306 	rc = ZSTD_CCtx_setParameter(dump_compressor, ZSTD_c_compressionLevel,
307 	    level);
308 	if (ZSTD_isError(rc)) {
309 		printf("%s: error setting compressLevel: %s\n", __func__,
310 		    ZSTD_getErrorName(rc));
311 		goto out;
312 	}
313 
314 	buf_size = ZSTD_CStreamOutSize() * 2;
315 	buffer = malloc(buf_size, M_COMPRESS, M_WAITOK | M_NODUMP);
316 
317 	s = malloc(sizeof(*s), M_COMPRESS, M_NODUMP | M_WAITOK);
318 	s->zst_buffer = buffer;
319 	s->zst_outbuffer.dst = buffer;
320 	s->zst_outbuffer.size = buf_size;
321 	s->zst_maxiosz = maxiosize;
322 	s->zst_stream = dump_compressor;
323 	s->zst_static_wkspc = owkspc;
324 
325 	zstdio_reset(s);
326 
327 out:
328 	if (s == NULL)
329 		free(owkspc, M_COMPRESS);
330 	return (s);
331 }
332 
333 static void
334 zstdio_reset(void *stream)
335 {
336 	struct zstdio_stream *s;
337 	size_t res;
338 
339 	s = stream;
340 	res = ZSTD_resetCStream(s->zst_stream, 0);
341 	if (ZSTD_isError(res))
342 		panic("%s: could not reset stream %p: %s\n", __func__, s,
343 		    ZSTD_getErrorName(res));
344 
345 	s->zst_off = 0;
346 	s->zst_inbuffer.src = NULL;
347 	s->zst_inbuffer.size = 0;
348 	s->zst_inbuffer.pos = 0;
349 	s->zst_outbuffer.pos = 0;
350 }
351 
352 static int
353 zst_flush_intermediate(struct zstdio_stream *s, compressor_cb_t cb, void *arg)
354 {
355 	size_t bytes_to_dump;
356 	int error;
357 
358 	/* Flush as many full output blocks as possible. */
359 	/* XXX: 4096 is arbitrary safe HDD block size for kernel dumps */
360 	while (s->zst_outbuffer.pos >= 4096) {
361 		bytes_to_dump = rounddown(s->zst_outbuffer.pos, 4096);
362 
363 		if (bytes_to_dump > s->zst_maxiosz)
364 			bytes_to_dump = s->zst_maxiosz;
365 
366 		error = cb(s->zst_buffer, bytes_to_dump, s->zst_off, arg);
367 		if (error != 0)
368 			return (error);
369 
370 		/*
371 		 * Shift any non-full blocks up to the front of the output
372 		 * buffer.
373 		 */
374 		s->zst_outbuffer.pos -= bytes_to_dump;
375 		memmove(s->zst_outbuffer.dst,
376 		    (char *)s->zst_outbuffer.dst + bytes_to_dump,
377 		    s->zst_outbuffer.pos);
378 		s->zst_off += bytes_to_dump;
379 	}
380 	return (0);
381 }
382 
383 static int
384 zstdio_flush(struct zstdio_stream *s, compressor_cb_t cb, void *arg)
385 {
386 	size_t rc, lastpos;
387 	int error;
388 
389 	/*
390 	 * Positive return indicates unflushed data remaining; need to call
391 	 * endStream again after clearing out room in output buffer.
392 	 */
393 	rc = 1;
394 	lastpos = s->zst_outbuffer.pos;
395 	while (rc > 0) {
396 		rc = ZSTD_endStream(s->zst_stream, &s->zst_outbuffer);
397 		if (ZSTD_isError(rc)) {
398 			printf("%s: ZSTD_endStream failed (%s)\n", __func__,
399 			    ZSTD_getErrorName(rc));
400 			return (EIO);
401 		}
402 		if (lastpos == s->zst_outbuffer.pos) {
403 			printf("%s: did not make forward progress endStream %zu\n",
404 			    __func__, lastpos);
405 			return (EIO);
406 		}
407 
408 		error = zst_flush_intermediate(s, cb, arg);
409 		if (error != 0)
410 			return (error);
411 
412 		lastpos = s->zst_outbuffer.pos;
413 	}
414 
415 	/*
416 	 * We've already done an intermediate flush, so all full blocks have
417 	 * been written.  Only a partial block remains.  Padding happens in a
418 	 * higher layer.
419 	 */
420 	if (s->zst_outbuffer.pos != 0) {
421 		error = cb(s->zst_buffer, s->zst_outbuffer.pos, s->zst_off,
422 		    arg);
423 		if (error != 0)
424 			return (error);
425 	}
426 
427 	return (0);
428 }
429 
430 static int
431 zstdio_write(void *stream, void *data, size_t len, compressor_cb_t cb,
432     void *arg)
433 {
434 	struct zstdio_stream *s;
435 	size_t lastpos, rc;
436 	int error;
437 
438 	s = stream;
439 	if (data == NULL)
440 		return (zstdio_flush(s, cb, arg));
441 
442 	s->zst_inbuffer.src = data;
443 	s->zst_inbuffer.size = len;
444 	s->zst_inbuffer.pos = 0;
445 	lastpos = 0;
446 
447 	while (s->zst_inbuffer.pos < s->zst_inbuffer.size) {
448 		rc = ZSTD_compressStream(s->zst_stream, &s->zst_outbuffer,
449 		    &s->zst_inbuffer);
450 		if (ZSTD_isError(rc)) {
451 			printf("%s: Compress failed on %p! (%s)\n",
452 			    __func__, data, ZSTD_getErrorName(rc));
453 			return (EIO);
454 		}
455 
456 		if (lastpos == s->zst_inbuffer.pos) {
457 			/*
458 			 * XXX: May need flushStream to make forward progress
459 			 */
460 			printf("ZSTD: did not make forward progress @pos %zu\n",
461 			    lastpos);
462 			return (EIO);
463 		}
464 		lastpos = s->zst_inbuffer.pos;
465 
466 		error = zst_flush_intermediate(s, cb, arg);
467 		if (error != 0)
468 			return (error);
469 	}
470 	return (0);
471 }
472 
473 static void
474 zstdio_fini(void *stream)
475 {
476 	struct zstdio_stream *s;
477 
478 	s = stream;
479 	if (s->zst_static_wkspc != NULL)
480 		free(s->zst_static_wkspc, M_COMPRESS);
481 	else
482 		ZSTD_freeCCtx(s->zst_stream);
483 	free(s->zst_buffer, M_COMPRESS);
484 	free(s, M_COMPRESS);
485 }
486 
487 static struct compressor_methods zstd_methods = {
488 	.format = COMPRESS_ZSTD,
489 	.init = zstdio_init,
490 	.reset = zstdio_reset,
491 	.write = zstdio_write,
492 	.fini = zstdio_fini,
493 };
494 DATA_SET(compressors, zstd_methods);
495 
496 #endif /* ZSTDIO */
497 
498 bool
499 compressor_avail(int format)
500 {
501 	struct compressor_methods **iter;
502 
503 	SET_FOREACH(iter, compressors) {
504 		if ((*iter)->format == format)
505 			return (true);
506 	}
507 	return (false);
508 }
509 
510 struct compressor *
511 compressor_init(compressor_cb_t cb, int format, size_t maxiosize, int level,
512     void *arg)
513 {
514 	struct compressor_methods **iter;
515 	struct compressor *s;
516 	void *priv;
517 
518 	SET_FOREACH(iter, compressors) {
519 		if ((*iter)->format == format)
520 			break;
521 	}
522 	if (iter == SET_LIMIT(compressors))
523 		return (NULL);
524 
525 	priv = (*iter)->init(maxiosize, level);
526 	if (priv == NULL)
527 		return (NULL);
528 
529 	s = malloc(sizeof(*s), M_COMPRESS, M_WAITOK | M_ZERO);
530 	s->methods = (*iter);
531 	s->priv = priv;
532 	s->cb = cb;
533 	s->arg = arg;
534 	return (s);
535 }
536 
537 void
538 compressor_reset(struct compressor *stream)
539 {
540 
541 	stream->methods->reset(stream->priv);
542 }
543 
544 int
545 compressor_write(struct compressor *stream, void *data, size_t len)
546 {
547 
548 	return (stream->methods->write(stream->priv, data, len, stream->cb,
549 	    stream->arg));
550 }
551 
552 int
553 compressor_flush(struct compressor *stream)
554 {
555 
556 	return (stream->methods->write(stream->priv, NULL, 0, stream->cb,
557 	    stream->arg));
558 }
559 
560 void
561 compressor_fini(struct compressor *stream)
562 {
563 
564 	stream->methods->fini(stream->priv);
565 }
566