xref: /freebsd/sys/kern/subr_compressor.c (revision 4d846d26) 
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause
3  *
4  * Copyright (c) 2014, 2017 Mark Johnston <markj@FreeBSD.org>
5  * Copyright (c) 2017 Conrad Meyer <cem@FreeBSD.org>
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions are
9  * met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in
14  *    the documentation and/or other materials provided with the
15  *    distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27  * SUCH DAMAGE.
28  */
29 
30 /*
31  * Subroutines used for writing compressed user process and kernel core dumps.
32  */
33 
34 #include <sys/cdefs.h>
35 __FBSDID("$FreeBSD$");
36 
37 #include "opt_gzio.h"
38 #include "opt_zstdio.h"
39 
40 #include <sys/param.h>
41 #include <sys/systm.h>
42 
43 #include <sys/compressor.h>
44 #include <sys/endian.h>
45 #include <sys/kernel.h>
46 #include <sys/linker_set.h>
47 #include <sys/malloc.h>
48 
49 MALLOC_DEFINE(M_COMPRESS, "compressor", "kernel compression subroutines");
50 
51 struct compressor_methods {
52 	int format;
53 	void *(* const init)(size_t, int);
54 	void (* const reset)(void *);
55 	int (* const write)(void *, void *, size_t, compressor_cb_t, void *);
56 	void (* const fini)(void *);
57 };
58 
59 struct compressor {
60 	const struct compressor_methods *methods;
61 	compressor_cb_t cb;
62 	void *priv;
63 	void *arg;
64 };
65 
66 SET_DECLARE(compressors, struct compressor_methods);
67 
68 #ifdef GZIO
69 
70 #include <contrib/zlib/zutil.h>
71 
72 struct gz_stream {
73 	uint8_t		*gz_buffer;	/* output buffer */
74 	size_t		gz_bufsz;	/* output buffer size */
75 	off_t		gz_off;		/* offset into the output stream */
76 	uint32_t	gz_crc;		/* stream CRC32 */
77 	z_stream	gz_stream;	/* zlib state */
78 };
79 
80 static void 	*gz_init(size_t maxiosize, int level);
81 static void	gz_reset(void *stream);
82 static int	gz_write(void *stream, void *data, size_t len, compressor_cb_t,
83 		    void *);
84 static void	gz_fini(void *stream);
85 
86 static void *
87 gz_alloc(void *arg __unused, u_int n, u_int sz)
88 {
89 
90 	/*
91 	 * Memory for zlib state is allocated using M_NODUMP since it may be
92 	 * used to compress a kernel dump, and we don't want zlib to attempt to
93 	 * compress its own state.
94 	 */
95 	return (malloc(n * sz, M_COMPRESS, M_WAITOK | M_ZERO | M_NODUMP));
96 }
97 
98 static void
99 gz_free(void *arg __unused, void *ptr)
100 {
101 
102 	free(ptr, M_COMPRESS);
103 }
104 
105 static void *
106 gz_init(size_t maxiosize, int level)
107 {
108 	struct gz_stream *s;
109 	int error;
110 
111 	s = gz_alloc(NULL, 1, roundup2(sizeof(*s), PAGE_SIZE));
112 	s->gz_buffer = gz_alloc(NULL, 1, maxiosize);
113 	s->gz_bufsz = maxiosize;
114 
115 	s->gz_stream.zalloc = gz_alloc;
116 	s->gz_stream.zfree = gz_free;
117 	s->gz_stream.opaque = NULL;
118 	s->gz_stream.next_in = Z_NULL;
119 	s->gz_stream.avail_in = 0;
120 
121 	if (level != Z_DEFAULT_COMPRESSION) {
122 		if (level < Z_BEST_SPEED)
123 			level = Z_BEST_SPEED;
124 		else if (level > Z_BEST_COMPRESSION)
125 			level = Z_BEST_COMPRESSION;
126 	}
127 
128 	error = deflateInit2(&s->gz_stream, level, Z_DEFLATED, -MAX_WBITS,
129 	    DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY);
130 	if (error != 0)
131 		goto fail;
132 
133 	gz_reset(s);
134 
135 	return (s);
136 
137 fail:
138 	gz_free(NULL, s);
139 	return (NULL);
140 }
141 
142 static void
143 gz_reset(void *stream)
144 {
145 	struct gz_stream *s;
146 	uint8_t *hdr;
147 	const size_t hdrlen = 10;
148 
149 	s = stream;
150 	s->gz_off = 0;
151 	s->gz_crc = crc32(0L, Z_NULL, 0);
152 
153 	(void)deflateReset(&s->gz_stream);
154 	s->gz_stream.avail_out = s->gz_bufsz;
155 	s->gz_stream.next_out = s->gz_buffer;
156 
157 	/* Write the gzip header to the output buffer. */
158 	hdr = s->gz_buffer;
159 	memset(hdr, 0, hdrlen);
160 	hdr[0] = 0x1f;
161 	hdr[1] = 0x8b;
162 	hdr[2] = Z_DEFLATED;
163 	hdr[9] = OS_CODE;
164 	s->gz_stream.next_out += hdrlen;
165 	s->gz_stream.avail_out -= hdrlen;
166 }
167 
168 static int
169 gz_write(void *stream, void *data, size_t len, compressor_cb_t cb,
170     void *arg)
171 {
172 	struct gz_stream *s;
173 	uint8_t trailer[8];
174 	size_t room;
175 	int error, zerror, zflag;
176 
177 	s = stream;
178 	zflag = data == NULL ? Z_FINISH : Z_NO_FLUSH;
179 
180 	if (len > 0) {
181 		s->gz_stream.avail_in = len;
182 		s->gz_stream.next_in = data;
183 		s->gz_crc = crc32(s->gz_crc, data, len);
184 	}
185 
186 	error = 0;
187 	do {
188 		zerror = deflate(&s->gz_stream, zflag);
189 		if (zerror != Z_OK && zerror != Z_STREAM_END) {
190 			error = EIO;
191 			break;
192 		}
193 
194 		if (s->gz_stream.avail_out == 0 || zerror == Z_STREAM_END) {
195 			/*
196 			 * Our output buffer is full or there's nothing left
197 			 * to produce, so we're flushing the buffer.
198 			 */
199 			len = s->gz_bufsz - s->gz_stream.avail_out;
200 			if (zerror == Z_STREAM_END) {
201 				/*
202 				 * Try to pack as much of the trailer into the
203 				 * output buffer as we can.
204 				 */
205 				((uint32_t *)trailer)[0] = htole32(s->gz_crc);
206 				((uint32_t *)trailer)[1] =
207 				    htole32(s->gz_stream.total_in);
208 				room = MIN(sizeof(trailer),
209 				    s->gz_bufsz - len);
210 				memcpy(s->gz_buffer + len, trailer, room);
211 				len += room;
212 			}
213 
214 			error = cb(s->gz_buffer, len, s->gz_off, arg);
215 			if (error != 0)
216 				break;
217 
218 			s->gz_off += len;
219 			s->gz_stream.next_out = s->gz_buffer;
220 			s->gz_stream.avail_out = s->gz_bufsz;
221 
222 			/*
223 			 * If we couldn't pack the trailer into the output
224 			 * buffer, write it out now.
225 			 */
226 			if (zerror == Z_STREAM_END && room < sizeof(trailer))
227 				error = cb(trailer + room,
228 				    sizeof(trailer) - room, s->gz_off, arg);
229 		}
230 	} while (zerror != Z_STREAM_END &&
231 	    (zflag == Z_FINISH || s->gz_stream.avail_in > 0));
232 
233 	return (error);
234 }
235 
236 static void
237 gz_fini(void *stream)
238 {
239 	struct gz_stream *s;
240 
241 	s = stream;
242 	(void)deflateEnd(&s->gz_stream);
243 	gz_free(NULL, s->gz_buffer);
244 	gz_free(NULL, s);
245 }
246 
247 struct compressor_methods gzip_methods = {
248 	.format = COMPRESS_GZIP,
249 	.init = gz_init,
250 	.reset = gz_reset,
251 	.write = gz_write,
252 	.fini = gz_fini,
253 };
254 DATA_SET(compressors, gzip_methods);
255 
256 #endif /* GZIO */
257 
258 #ifdef ZSTDIO
259 
260 #define	ZSTD_STATIC_LINKING_ONLY
261 #include <contrib/zstd/lib/zstd.h>
262 
263 struct zstdio_stream {
264 	ZSTD_CCtx	*zst_stream;
265 	ZSTD_inBuffer	zst_inbuffer;
266 	ZSTD_outBuffer	zst_outbuffer;
267 	uint8_t *	zst_buffer;	/* output buffer */
268 	size_t		zst_maxiosz;	/* Max output IO size */
269 	off_t		zst_off;	/* offset into the output stream */
270 	void *		zst_static_wkspc;
271 };
272 
273 static void 	*zstdio_init(size_t maxiosize, int level);
274 static void	zstdio_reset(void *stream);
275 static int	zstdio_write(void *stream, void *data, size_t len,
276 		    compressor_cb_t, void *);
277 static void	zstdio_fini(void *stream);
278 
279 static void *
280 zstdio_init(size_t maxiosize, int level)
281 {
282 	ZSTD_CCtx *dump_compressor;
283 	struct zstdio_stream *s;
284 	void *wkspc, *owkspc, *buffer;
285 	size_t wkspc_size, buf_size, rc;
286 
287 	s = NULL;
288 	wkspc_size = ZSTD_estimateCStreamSize(level);
289 	owkspc = wkspc = malloc(wkspc_size + 8, M_COMPRESS,
290 	    M_WAITOK | M_NODUMP);
291 	/* Zstd API requires 8-byte alignment. */
292 	if ((uintptr_t)wkspc % 8 != 0)
293 		wkspc = (void *)roundup2((uintptr_t)wkspc, 8);
294 
295 	dump_compressor = ZSTD_initStaticCCtx(wkspc, wkspc_size);
296 	if (dump_compressor == NULL) {
297 		printf("%s: workspace too small.\n", __func__);
298 		goto out;
299 	}
300 
301 	rc = ZSTD_CCtx_setParameter(dump_compressor, ZSTD_c_checksumFlag, 1);
302 	if (ZSTD_isError(rc)) {
303 		printf("%s: error setting checksumFlag: %s\n", __func__,
304 		    ZSTD_getErrorName(rc));
305 		goto out;
306 	}
307 	rc = ZSTD_CCtx_setParameter(dump_compressor, ZSTD_c_compressionLevel,
308 	    level);
309 	if (ZSTD_isError(rc)) {
310 		printf("%s: error setting compressLevel: %s\n", __func__,
311 		    ZSTD_getErrorName(rc));
312 		goto out;
313 	}
314 
315 	buf_size = ZSTD_CStreamOutSize() * 2;
316 	buffer = malloc(buf_size, M_COMPRESS, M_WAITOK | M_NODUMP);
317 
318 	s = malloc(sizeof(*s), M_COMPRESS, M_NODUMP | M_WAITOK);
319 	s->zst_buffer = buffer;
320 	s->zst_outbuffer.dst = buffer;
321 	s->zst_outbuffer.size = buf_size;
322 	s->zst_maxiosz = maxiosize;
323 	s->zst_stream = dump_compressor;
324 	s->zst_static_wkspc = owkspc;
325 
326 	zstdio_reset(s);
327 
328 out:
329 	if (s == NULL)
330 		free(owkspc, M_COMPRESS);
331 	return (s);
332 }
333 
334 static void
335 zstdio_reset(void *stream)
336 {
337 	struct zstdio_stream *s;
338 	size_t res;
339 
340 	s = stream;
341 	res = ZSTD_CCtx_reset(s->zst_stream, ZSTD_reset_session_only);
342 	if (ZSTD_isError(res))
343 		panic("%s: could not reset stream %p: %s\n", __func__, s,
344 		    ZSTD_getErrorName(res));
345 	res = ZSTD_CCtx_setPledgedSrcSize(s->zst_stream,
346 	    ZSTD_CONTENTSIZE_UNKNOWN);
347 	if (ZSTD_isError(res))
348 		panic("%s: could not set src size on %p: %s\n", __func__, s,
349 		    ZSTD_getErrorName(res));
350 
351 	s->zst_off = 0;
352 	s->zst_inbuffer.src = NULL;
353 	s->zst_inbuffer.size = 0;
354 	s->zst_inbuffer.pos = 0;
355 	s->zst_outbuffer.pos = 0;
356 }
357 
358 static int
359 zst_flush_intermediate(struct zstdio_stream *s, compressor_cb_t cb, void *arg)
360 {
361 	size_t bytes_to_dump;
362 	int error;
363 
364 	/* Flush as many full output blocks as possible. */
365 	/* XXX: 4096 is arbitrary safe HDD block size for kernel dumps */
366 	while (s->zst_outbuffer.pos >= 4096) {
367 		bytes_to_dump = rounddown(s->zst_outbuffer.pos, 4096);
368 
369 		if (bytes_to_dump > s->zst_maxiosz)
370 			bytes_to_dump = s->zst_maxiosz;
371 
372 		error = cb(s->zst_buffer, bytes_to_dump, s->zst_off, arg);
373 		if (error != 0)
374 			return (error);
375 
376 		/*
377 		 * Shift any non-full blocks up to the front of the output
378 		 * buffer.
379 		 */
380 		s->zst_outbuffer.pos -= bytes_to_dump;
381 		memmove(s->zst_outbuffer.dst,
382 		    (char *)s->zst_outbuffer.dst + bytes_to_dump,
383 		    s->zst_outbuffer.pos);
384 		s->zst_off += bytes_to_dump;
385 	}
386 	return (0);
387 }
388 
389 static int
390 zstdio_flush(struct zstdio_stream *s, compressor_cb_t cb, void *arg)
391 {
392 	size_t rc, lastpos;
393 	int error;
394 
395 	/*
396 	 * Positive return indicates unflushed data remaining; need to call
397 	 * endStream again after clearing out room in output buffer.
398 	 */
399 	rc = 1;
400 	lastpos = s->zst_outbuffer.pos;
401 	while (rc > 0) {
402 		rc = ZSTD_endStream(s->zst_stream, &s->zst_outbuffer);
403 		if (ZSTD_isError(rc)) {
404 			printf("%s: ZSTD_endStream failed (%s)\n", __func__,
405 			    ZSTD_getErrorName(rc));
406 			return (EIO);
407 		}
408 		if (lastpos == s->zst_outbuffer.pos) {
409 			printf("%s: did not make forward progress endStream %zu\n",
410 			    __func__, lastpos);
411 			return (EIO);
412 		}
413 
414 		error = zst_flush_intermediate(s, cb, arg);
415 		if (error != 0)
416 			return (error);
417 
418 		lastpos = s->zst_outbuffer.pos;
419 	}
420 
421 	/*
422 	 * We've already done an intermediate flush, so all full blocks have
423 	 * been written.  Only a partial block remains.  Padding happens in a
424 	 * higher layer.
425 	 */
426 	if (s->zst_outbuffer.pos != 0) {
427 		error = cb(s->zst_buffer, s->zst_outbuffer.pos, s->zst_off,
428 		    arg);
429 		if (error != 0)
430 			return (error);
431 	}
432 
433 	return (0);
434 }
435 
436 static int
437 zstdio_write(void *stream, void *data, size_t len, compressor_cb_t cb,
438     void *arg)
439 {
440 	struct zstdio_stream *s;
441 	size_t lastpos, rc;
442 	int error;
443 
444 	s = stream;
445 	if (data == NULL)
446 		return (zstdio_flush(s, cb, arg));
447 
448 	s->zst_inbuffer.src = data;
449 	s->zst_inbuffer.size = len;
450 	s->zst_inbuffer.pos = 0;
451 	lastpos = 0;
452 
453 	while (s->zst_inbuffer.pos < s->zst_inbuffer.size) {
454 		rc = ZSTD_compressStream(s->zst_stream, &s->zst_outbuffer,
455 		    &s->zst_inbuffer);
456 		if (ZSTD_isError(rc)) {
457 			printf("%s: Compress failed on %p! (%s)\n",
458 			    __func__, data, ZSTD_getErrorName(rc));
459 			return (EIO);
460 		}
461 
462 		if (lastpos == s->zst_inbuffer.pos) {
463 			/*
464 			 * XXX: May need flushStream to make forward progress
465 			 */
466 			printf("ZSTD: did not make forward progress @pos %zu\n",
467 			    lastpos);
468 			return (EIO);
469 		}
470 		lastpos = s->zst_inbuffer.pos;
471 
472 		error = zst_flush_intermediate(s, cb, arg);
473 		if (error != 0)
474 			return (error);
475 	}
476 	return (0);
477 }
478 
479 static void
480 zstdio_fini(void *stream)
481 {
482 	struct zstdio_stream *s;
483 
484 	s = stream;
485 	if (s->zst_static_wkspc != NULL)
486 		free(s->zst_static_wkspc, M_COMPRESS);
487 	else
488 		ZSTD_freeCCtx(s->zst_stream);
489 	free(s->zst_buffer, M_COMPRESS);
490 	free(s, M_COMPRESS);
491 }
492 
493 static struct compressor_methods zstd_methods = {
494 	.format = COMPRESS_ZSTD,
495 	.init = zstdio_init,
496 	.reset = zstdio_reset,
497 	.write = zstdio_write,
498 	.fini = zstdio_fini,
499 };
500 DATA_SET(compressors, zstd_methods);
501 
502 #endif /* ZSTDIO */
503 
504 bool
505 compressor_avail(int format)
506 {
507 	struct compressor_methods **iter;
508 
509 	SET_FOREACH(iter, compressors) {
510 		if ((*iter)->format == format)
511 			return (true);
512 	}
513 	return (false);
514 }
515 
516 struct compressor *
517 compressor_init(compressor_cb_t cb, int format, size_t maxiosize, int level,
518     void *arg)
519 {
520 	struct compressor_methods **iter;
521 	struct compressor *s;
522 	void *priv;
523 
524 	SET_FOREACH(iter, compressors) {
525 		if ((*iter)->format == format)
526 			break;
527 	}
528 	if (iter == SET_LIMIT(compressors))
529 		return (NULL);
530 
531 	priv = (*iter)->init(maxiosize, level);
532 	if (priv == NULL)
533 		return (NULL);
534 
535 	s = malloc(sizeof(*s), M_COMPRESS, M_WAITOK | M_ZERO);
536 	s->methods = (*iter);
537 	s->priv = priv;
538 	s->cb = cb;
539 	s->arg = arg;
540 	return (s);
541 }
542 
543 void
544 compressor_reset(struct compressor *stream)
545 {
546 
547 	stream->methods->reset(stream->priv);
548 }
549 
550 int
551 compressor_write(struct compressor *stream, void *data, size_t len)
552 {
553 
554 	return (stream->methods->write(stream->priv, data, len, stream->cb,
555 	    stream->arg));
556 }
557 
558 int
559 compressor_flush(struct compressor *stream)
560 {
561 
562 	return (stream->methods->write(stream->priv, NULL, 0, stream->cb,
563 	    stream->arg));
564 }
565 
566 void
567 compressor_fini(struct compressor *stream)
568 {
569 
570 	stream->methods->fini(stream->priv);
571 }
572