xref: /freebsd/sys/kern/subr_compressor.c (revision 206b73d0) 
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2014, 2017 Mark Johnston <markj@FreeBSD.org>
5  * Copyright (c) 2017 Conrad Meyer <cem@FreeBSD.org>
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions are
9  * met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in
14  *    the documentation and/or other materials provided with the
15  *    distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27  * SUCH DAMAGE.
28  */
29 
30 /*
31  * Subroutines used for writing compressed user process and kernel core dumps.
32  */
33 
34 #include <sys/cdefs.h>
35 __FBSDID("$FreeBSD$");
36 
37 #include "opt_gzio.h"
38 #include "opt_zstdio.h"
39 
40 #include <sys/param.h>
41 #include <sys/systm.h>
42 
43 #include <sys/compressor.h>
44 #include <sys/kernel.h>
45 #include <sys/linker_set.h>
46 #include <sys/malloc.h>
47 
48 MALLOC_DEFINE(M_COMPRESS, "compressor", "kernel compression subroutines");
49 
50 struct compressor_methods {
51 	int format;
52 	void *(* const init)(size_t, int);
53 	void (* const reset)(void *);
54 	int (* const write)(void *, void *, size_t, compressor_cb_t, void *);
55 	void (* const fini)(void *);
56 };
57 
58 struct compressor {
59 	const struct compressor_methods *methods;
60 	compressor_cb_t cb;
61 	void *priv;
62 	void *arg;
63 };
64 
65 SET_DECLARE(compressors, struct compressor_methods);
66 
67 #ifdef GZIO
68 
69 #include <sys/gsb_crc32.h>
70 #include <sys/zutil.h>
71 
72 struct gz_stream {
73 	uint8_t		*gz_buffer;	/* output buffer */
74 	size_t		gz_bufsz;	/* output buffer size */
75 	off_t		gz_off;		/* offset into the output stream */
76 	uint32_t	gz_crc;		/* stream CRC32 */
77 	z_stream	gz_stream;	/* zlib state */
78 };
79 
80 static void 	*gz_init(size_t maxiosize, int level);
81 static void	gz_reset(void *stream);
82 static int	gz_write(void *stream, void *data, size_t len, compressor_cb_t,
83 		    void *);
84 static void	gz_fini(void *stream);
85 
86 static void *
87 gz_alloc(void *arg __unused, u_int n, u_int sz)
88 {
89 
90 	/*
91 	 * Memory for zlib state is allocated using M_NODUMP since it may be
92 	 * used to compress a kernel dump, and we don't want zlib to attempt to
93 	 * compress its own state.
94 	 */
95 	return (malloc(n * sz, M_COMPRESS, M_WAITOK | M_ZERO | M_NODUMP));
96 }
97 
98 static void
99 gz_free(void *arg __unused, void *ptr)
100 {
101 
102 	free(ptr, M_COMPRESS);
103 }
104 
105 static void *
106 gz_init(size_t maxiosize, int level)
107 {
108 	struct gz_stream *s;
109 	int error;
110 
111 	s = gz_alloc(NULL, 1, roundup2(sizeof(*s), PAGE_SIZE));
112 	s->gz_buffer = gz_alloc(NULL, 1, maxiosize);
113 	s->gz_bufsz = maxiosize;
114 
115 	s->gz_stream.zalloc = gz_alloc;
116 	s->gz_stream.zfree = gz_free;
117 	s->gz_stream.opaque = NULL;
118 	s->gz_stream.next_in = Z_NULL;
119 	s->gz_stream.avail_in = 0;
120 
121 	error = deflateInit2(&s->gz_stream, level, Z_DEFLATED, -MAX_WBITS,
122 	    DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY);
123 	if (error != 0)
124 		goto fail;
125 
126 	gz_reset(s);
127 
128 	return (s);
129 
130 fail:
131 	gz_free(NULL, s);
132 	return (NULL);
133 }
134 
135 static void
136 gz_reset(void *stream)
137 {
138 	struct gz_stream *s;
139 	uint8_t *hdr;
140 	const size_t hdrlen = 10;
141 
142 	s = stream;
143 	s->gz_off = 0;
144 	s->gz_crc = ~0U;
145 
146 	(void)deflateReset(&s->gz_stream);
147 	s->gz_stream.avail_out = s->gz_bufsz;
148 	s->gz_stream.next_out = s->gz_buffer;
149 
150 	/* Write the gzip header to the output buffer. */
151 	hdr = s->gz_buffer;
152 	memset(hdr, 0, hdrlen);
153 	hdr[0] = 0x1f;
154 	hdr[1] = 0x8b;
155 	hdr[2] = Z_DEFLATED;
156 	hdr[9] = OS_CODE;
157 	s->gz_stream.next_out += hdrlen;
158 	s->gz_stream.avail_out -= hdrlen;
159 }
160 
161 static int
162 gz_write(void *stream, void *data, size_t len, compressor_cb_t cb,
163     void *arg)
164 {
165 	struct gz_stream *s;
166 	uint8_t trailer[8];
167 	size_t room;
168 	int error, zerror, zflag;
169 
170 	s = stream;
171 	zflag = data == NULL ? Z_FINISH : Z_NO_FLUSH;
172 
173 	if (len > 0) {
174 		s->gz_stream.avail_in = len;
175 		s->gz_stream.next_in = data;
176 		s->gz_crc = crc32_raw(data, len, s->gz_crc);
177 	} else
178 		s->gz_crc ^= ~0U;
179 
180 	error = 0;
181 	do {
182 		zerror = deflate(&s->gz_stream, zflag);
183 		if (zerror != Z_OK && zerror != Z_STREAM_END) {
184 			error = EIO;
185 			break;
186 		}
187 
188 		if (s->gz_stream.avail_out == 0 || zerror == Z_STREAM_END) {
189 			/*
190 			 * Our output buffer is full or there's nothing left
191 			 * to produce, so we're flushing the buffer.
192 			 */
193 			len = s->gz_bufsz - s->gz_stream.avail_out;
194 			if (zerror == Z_STREAM_END) {
195 				/*
196 				 * Try to pack as much of the trailer into the
197 				 * output buffer as we can.
198 				 */
199 				((uint32_t *)trailer)[0] = s->gz_crc;
200 				((uint32_t *)trailer)[1] =
201 				    s->gz_stream.total_in;
202 				room = MIN(sizeof(trailer),
203 				    s->gz_bufsz - len);
204 				memcpy(s->gz_buffer + len, trailer, room);
205 				len += room;
206 			}
207 
208 			error = cb(s->gz_buffer, len, s->gz_off, arg);
209 			if (error != 0)
210 				break;
211 
212 			s->gz_off += len;
213 			s->gz_stream.next_out = s->gz_buffer;
214 			s->gz_stream.avail_out = s->gz_bufsz;
215 
216 			/*
217 			 * If we couldn't pack the trailer into the output
218 			 * buffer, write it out now.
219 			 */
220 			if (zerror == Z_STREAM_END && room < sizeof(trailer))
221 				error = cb(trailer + room,
222 				    sizeof(trailer) - room, s->gz_off, arg);
223 		}
224 	} while (zerror != Z_STREAM_END &&
225 	    (zflag == Z_FINISH || s->gz_stream.avail_in > 0));
226 
227 	return (error);
228 }
229 
230 static void
231 gz_fini(void *stream)
232 {
233 	struct gz_stream *s;
234 
235 	s = stream;
236 	(void)deflateEnd(&s->gz_stream);
237 	gz_free(NULL, s->gz_buffer);
238 	gz_free(NULL, s);
239 }
240 
241 struct compressor_methods gzip_methods = {
242 	.format = COMPRESS_GZIP,
243 	.init = gz_init,
244 	.reset = gz_reset,
245 	.write = gz_write,
246 	.fini = gz_fini,
247 };
248 DATA_SET(compressors, gzip_methods);
249 
250 #endif /* GZIO */
251 
252 #ifdef ZSTDIO
253 
254 #define	ZSTD_STATIC_LINKING_ONLY
255 #include <contrib/zstd/lib/zstd.h>
256 
257 struct zstdio_stream {
258 	ZSTD_CCtx	*zst_stream;
259 	ZSTD_inBuffer	zst_inbuffer;
260 	ZSTD_outBuffer	zst_outbuffer;
261 	uint8_t *	zst_buffer;	/* output buffer */
262 	size_t		zst_maxiosz;	/* Max output IO size */
263 	off_t		zst_off;	/* offset into the output stream */
264 	void *		zst_static_wkspc;
265 };
266 
267 static void 	*zstdio_init(size_t maxiosize, int level);
268 static void	zstdio_reset(void *stream);
269 static int	zstdio_write(void *stream, void *data, size_t len,
270 		    compressor_cb_t, void *);
271 static void	zstdio_fini(void *stream);
272 
273 static void *
274 zstdio_init(size_t maxiosize, int level)
275 {
276 	ZSTD_CCtx *dump_compressor;
277 	struct zstdio_stream *s;
278 	void *wkspc, *owkspc, *buffer;
279 	size_t wkspc_size, buf_size, rc;
280 
281 	s = NULL;
282 	wkspc_size = ZSTD_estimateCStreamSize(level);
283 	owkspc = wkspc = malloc(wkspc_size + 8, M_COMPRESS,
284 	    M_WAITOK | M_NODUMP);
285 	/* Zstd API requires 8-byte alignment. */
286 	if ((uintptr_t)wkspc % 8 != 0)
287 		wkspc = (void *)roundup2((uintptr_t)wkspc, 8);
288 
289 	dump_compressor = ZSTD_initStaticCCtx(wkspc, wkspc_size);
290 	if (dump_compressor == NULL) {
291 		printf("%s: workspace too small.\n", __func__);
292 		goto out;
293 	}
294 
295 	rc = ZSTD_CCtx_setParameter(dump_compressor, ZSTD_c_checksumFlag, 1);
296 	if (ZSTD_isError(rc)) {
297 		printf("%s: error setting checksumFlag: %s\n", __func__,
298 		    ZSTD_getErrorName(rc));
299 		goto out;
300 	}
301 	rc = ZSTD_CCtx_setParameter(dump_compressor, ZSTD_c_compressionLevel,
302 	    level);
303 	if (ZSTD_isError(rc)) {
304 		printf("%s: error setting compressLevel: %s\n", __func__,
305 		    ZSTD_getErrorName(rc));
306 		goto out;
307 	}
308 
309 	buf_size = ZSTD_CStreamOutSize() * 2;
310 	buffer = malloc(buf_size, M_COMPRESS, M_WAITOK | M_NODUMP);
311 
312 	s = malloc(sizeof(*s), M_COMPRESS, M_NODUMP | M_WAITOK);
313 	s->zst_buffer = buffer;
314 	s->zst_outbuffer.dst = buffer;
315 	s->zst_outbuffer.size = buf_size;
316 	s->zst_maxiosz = maxiosize;
317 	s->zst_stream = dump_compressor;
318 	s->zst_static_wkspc = owkspc;
319 
320 	zstdio_reset(s);
321 
322 out:
323 	if (s == NULL)
324 		free(owkspc, M_COMPRESS);
325 	return (s);
326 }
327 
328 static void
329 zstdio_reset(void *stream)
330 {
331 	struct zstdio_stream *s;
332 	size_t res;
333 
334 	s = stream;
335 	res = ZSTD_resetCStream(s->zst_stream, 0);
336 	if (ZSTD_isError(res))
337 		panic("%s: could not reset stream %p: %s\n", __func__, s,
338 		    ZSTD_getErrorName(res));
339 
340 	s->zst_off = 0;
341 	s->zst_inbuffer.src = NULL;
342 	s->zst_inbuffer.size = 0;
343 	s->zst_inbuffer.pos = 0;
344 	s->zst_outbuffer.pos = 0;
345 }
346 
347 static int
348 zst_flush_intermediate(struct zstdio_stream *s, compressor_cb_t cb, void *arg)
349 {
350 	size_t bytes_to_dump;
351 	int error;
352 
353 	/* Flush as many full output blocks as possible. */
354 	/* XXX: 4096 is arbitrary safe HDD block size for kernel dumps */
355 	while (s->zst_outbuffer.pos >= 4096) {
356 		bytes_to_dump = rounddown(s->zst_outbuffer.pos, 4096);
357 
358 		if (bytes_to_dump > s->zst_maxiosz)
359 			bytes_to_dump = s->zst_maxiosz;
360 
361 		error = cb(s->zst_buffer, bytes_to_dump, s->zst_off, arg);
362 		if (error != 0)
363 			return (error);
364 
365 		/*
366 		 * Shift any non-full blocks up to the front of the output
367 		 * buffer.
368 		 */
369 		s->zst_outbuffer.pos -= bytes_to_dump;
370 		memmove(s->zst_outbuffer.dst,
371 		    (char *)s->zst_outbuffer.dst + bytes_to_dump,
372 		    s->zst_outbuffer.pos);
373 		s->zst_off += bytes_to_dump;
374 	}
375 	return (0);
376 }
377 
378 static int
379 zstdio_flush(struct zstdio_stream *s, compressor_cb_t cb, void *arg)
380 {
381 	size_t rc, lastpos;
382 	int error;
383 
384 	/*
385 	 * Positive return indicates unflushed data remaining; need to call
386 	 * endStream again after clearing out room in output buffer.
387 	 */
388 	rc = 1;
389 	lastpos = s->zst_outbuffer.pos;
390 	while (rc > 0) {
391 		rc = ZSTD_endStream(s->zst_stream, &s->zst_outbuffer);
392 		if (ZSTD_isError(rc)) {
393 			printf("%s: ZSTD_endStream failed (%s)\n", __func__,
394 			    ZSTD_getErrorName(rc));
395 			return (EIO);
396 		}
397 		if (lastpos == s->zst_outbuffer.pos) {
398 			printf("%s: did not make forward progress endStream %zu\n",
399 			    __func__, lastpos);
400 			return (EIO);
401 		}
402 
403 		error = zst_flush_intermediate(s, cb, arg);
404 		if (error != 0)
405 			return (error);
406 
407 		lastpos = s->zst_outbuffer.pos;
408 	}
409 
410 	/*
411 	 * We've already done an intermediate flush, so all full blocks have
412 	 * been written.  Only a partial block remains.  Padding happens in a
413 	 * higher layer.
414 	 */
415 	if (s->zst_outbuffer.pos != 0) {
416 		error = cb(s->zst_buffer, s->zst_outbuffer.pos, s->zst_off,
417 		    arg);
418 		if (error != 0)
419 			return (error);
420 	}
421 
422 	return (0);
423 }
424 
425 static int
426 zstdio_write(void *stream, void *data, size_t len, compressor_cb_t cb,
427     void *arg)
428 {
429 	struct zstdio_stream *s;
430 	size_t lastpos, rc;
431 	int error;
432 
433 	s = stream;
434 	if (data == NULL)
435 		return (zstdio_flush(s, cb, arg));
436 
437 	s->zst_inbuffer.src = data;
438 	s->zst_inbuffer.size = len;
439 	s->zst_inbuffer.pos = 0;
440 	lastpos = 0;
441 
442 	while (s->zst_inbuffer.pos < s->zst_inbuffer.size) {
443 		rc = ZSTD_compressStream(s->zst_stream, &s->zst_outbuffer,
444 		    &s->zst_inbuffer);
445 		if (ZSTD_isError(rc)) {
446 			printf("%s: Compress failed on %p! (%s)\n",
447 			    __func__, data, ZSTD_getErrorName(rc));
448 			return (EIO);
449 		}
450 
451 		if (lastpos == s->zst_inbuffer.pos) {
452 			/*
453 			 * XXX: May need flushStream to make forward progress
454 			 */
455 			printf("ZSTD: did not make forward progress @pos %zu\n",
456 			    lastpos);
457 			return (EIO);
458 		}
459 		lastpos = s->zst_inbuffer.pos;
460 
461 		error = zst_flush_intermediate(s, cb, arg);
462 		if (error != 0)
463 			return (error);
464 	}
465 	return (0);
466 }
467 
468 static void
469 zstdio_fini(void *stream)
470 {
471 	struct zstdio_stream *s;
472 
473 	s = stream;
474 	if (s->zst_static_wkspc != NULL)
475 		free(s->zst_static_wkspc, M_COMPRESS);
476 	else
477 		ZSTD_freeCCtx(s->zst_stream);
478 	free(s->zst_buffer, M_COMPRESS);
479 	free(s, M_COMPRESS);
480 }
481 
482 static struct compressor_methods zstd_methods = {
483 	.format = COMPRESS_ZSTD,
484 	.init = zstdio_init,
485 	.reset = zstdio_reset,
486 	.write = zstdio_write,
487 	.fini = zstdio_fini,
488 };
489 DATA_SET(compressors, zstd_methods);
490 
491 #endif /* ZSTDIO */
492 
493 bool
494 compressor_avail(int format)
495 {
496 	struct compressor_methods **iter;
497 
498 	SET_FOREACH(iter, compressors) {
499 		if ((*iter)->format == format)
500 			return (true);
501 	}
502 	return (false);
503 }
504 
505 struct compressor *
506 compressor_init(compressor_cb_t cb, int format, size_t maxiosize, int level,
507     void *arg)
508 {
509 	struct compressor_methods **iter;
510 	struct compressor *s;
511 	void *priv;
512 
513 	SET_FOREACH(iter, compressors) {
514 		if ((*iter)->format == format)
515 			break;
516 	}
517 	if (iter == SET_LIMIT(compressors))
518 		return (NULL);
519 
520 	priv = (*iter)->init(maxiosize, level);
521 	if (priv == NULL)
522 		return (NULL);
523 
524 	s = malloc(sizeof(*s), M_COMPRESS, M_WAITOK | M_ZERO);
525 	s->methods = (*iter);
526 	s->priv = priv;
527 	s->cb = cb;
528 	s->arg = arg;
529 	return (s);
530 }
531 
532 void
533 compressor_reset(struct compressor *stream)
534 {
535 
536 	stream->methods->reset(stream->priv);
537 }
538 
539 int
540 compressor_write(struct compressor *stream, void *data, size_t len)
541 {
542 
543 	return (stream->methods->write(stream->priv, data, len, stream->cb,
544 	    stream->arg));
545 }
546 
547 int
548 compressor_flush(struct compressor *stream)
549 {
550 
551 	return (stream->methods->write(stream->priv, NULL, 0, stream->cb,
552 	    stream->arg));
553 }
554 
555 void
556 compressor_fini(struct compressor *stream)
557 {
558 
559 	stream->methods->fini(stream->priv);
560 }
561