1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2014, 2017 Mark Johnston <markj@FreeBSD.org> 5 * Copyright (c) 2017 Conrad Meyer <cem@FreeBSD.org> 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions are 9 * met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in 14 * the documentation and/or other materials provided with the 15 * distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 /* 31 * Subroutines used for writing compressed user process and kernel core dumps. 32 */ 33 34 #include <sys/cdefs.h> 35 __FBSDID("$FreeBSD$"); 36 37 #include "opt_gzio.h" 38 #include "opt_zstdio.h" 39 40 #include <sys/param.h> 41 #include <sys/systm.h> 42 43 #include <sys/compressor.h> 44 #include <sys/kernel.h> 45 #include <sys/linker_set.h> 46 #include <sys/malloc.h> 47 48 MALLOC_DEFINE(M_COMPRESS, "compressor", "kernel compression subroutines"); 49 50 struct compressor_methods { 51 int format; 52 void *(* const init)(size_t, int); 53 void (* const reset)(void *); 54 int (* const write)(void *, void *, size_t, compressor_cb_t, void *); 55 void (* const fini)(void *); 56 }; 57 58 struct compressor { 59 const struct compressor_methods *methods; 60 compressor_cb_t cb; 61 void *priv; 62 void *arg; 63 }; 64 65 SET_DECLARE(compressors, struct compressor_methods); 66 67 #ifdef GZIO 68 69 #include <sys/zutil.h> 70 71 struct gz_stream { 72 uint8_t *gz_buffer; /* output buffer */ 73 size_t gz_bufsz; /* output buffer size */ 74 off_t gz_off; /* offset into the output stream */ 75 uint32_t gz_crc; /* stream CRC32 */ 76 z_stream gz_stream; /* zlib state */ 77 }; 78 79 static void *gz_init(size_t maxiosize, int level); 80 static void gz_reset(void *stream); 81 static int gz_write(void *stream, void *data, size_t len, compressor_cb_t, 82 void *); 83 static void gz_fini(void *stream); 84 85 static void * 86 gz_alloc(void *arg __unused, u_int n, u_int sz) 87 { 88 89 /* 90 * Memory for zlib state is allocated using M_NODUMP since it may be 91 * used to compress a kernel dump, and we don't want zlib to attempt to 92 * compress its own state. 93 */ 94 return (malloc(n * sz, M_COMPRESS, M_WAITOK | M_ZERO | M_NODUMP)); 95 } 96 97 static void 98 gz_free(void *arg __unused, void *ptr) 99 { 100 101 free(ptr, M_COMPRESS); 102 } 103 104 static void * 105 gz_init(size_t maxiosize, int level) 106 { 107 struct gz_stream *s; 108 int error; 109 110 s = gz_alloc(NULL, 1, roundup2(sizeof(*s), PAGE_SIZE)); 111 s->gz_buffer = gz_alloc(NULL, 1, maxiosize); 112 s->gz_bufsz = maxiosize; 113 114 s->gz_stream.zalloc = gz_alloc; 115 s->gz_stream.zfree = gz_free; 116 s->gz_stream.opaque = NULL; 117 s->gz_stream.next_in = Z_NULL; 118 s->gz_stream.avail_in = 0; 119 120 error = deflateInit2(&s->gz_stream, level, Z_DEFLATED, -MAX_WBITS, 121 DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY); 122 if (error != 0) 123 goto fail; 124 125 gz_reset(s); 126 127 return (s); 128 129 fail: 130 gz_free(NULL, s); 131 return (NULL); 132 } 133 134 static void 135 gz_reset(void *stream) 136 { 137 struct gz_stream *s; 138 uint8_t *hdr; 139 const size_t hdrlen = 10; 140 141 s = stream; 142 s->gz_off = 0; 143 s->gz_crc = ~0U; 144 145 (void)deflateReset(&s->gz_stream); 146 s->gz_stream.avail_out = s->gz_bufsz; 147 s->gz_stream.next_out = s->gz_buffer; 148 149 /* Write the gzip header to the output buffer. */ 150 hdr = s->gz_buffer; 151 memset(hdr, 0, hdrlen); 152 hdr[0] = 0x1f; 153 hdr[1] = 0x8b; 154 hdr[2] = Z_DEFLATED; 155 hdr[9] = OS_CODE; 156 s->gz_stream.next_out += hdrlen; 157 s->gz_stream.avail_out -= hdrlen; 158 } 159 160 static int 161 gz_write(void *stream, void *data, size_t len, compressor_cb_t cb, 162 void *arg) 163 { 164 struct gz_stream *s; 165 uint8_t trailer[8]; 166 size_t room; 167 int error, zerror, zflag; 168 169 s = stream; 170 zflag = data == NULL ? Z_FINISH : Z_NO_FLUSH; 171 172 if (len > 0) { 173 s->gz_stream.avail_in = len; 174 s->gz_stream.next_in = data; 175 s->gz_crc = crc32_raw(data, len, s->gz_crc); 176 } else 177 s->gz_crc ^= ~0U; 178 179 error = 0; 180 do { 181 zerror = deflate(&s->gz_stream, zflag); 182 if (zerror != Z_OK && zerror != Z_STREAM_END) { 183 error = EIO; 184 break; 185 } 186 187 if (s->gz_stream.avail_out == 0 || zerror == Z_STREAM_END) { 188 /* 189 * Our output buffer is full or there's nothing left 190 * to produce, so we're flushing the buffer. 191 */ 192 len = s->gz_bufsz - s->gz_stream.avail_out; 193 if (zerror == Z_STREAM_END) { 194 /* 195 * Try to pack as much of the trailer into the 196 * output buffer as we can. 197 */ 198 ((uint32_t *)trailer)[0] = s->gz_crc; 199 ((uint32_t *)trailer)[1] = 200 s->gz_stream.total_in; 201 room = MIN(sizeof(trailer), 202 s->gz_bufsz - len); 203 memcpy(s->gz_buffer + len, trailer, room); 204 len += room; 205 } 206 207 error = cb(s->gz_buffer, len, s->gz_off, arg); 208 if (error != 0) 209 break; 210 211 s->gz_off += len; 212 s->gz_stream.next_out = s->gz_buffer; 213 s->gz_stream.avail_out = s->gz_bufsz; 214 215 /* 216 * If we couldn't pack the trailer into the output 217 * buffer, write it out now. 218 */ 219 if (zerror == Z_STREAM_END && room < sizeof(trailer)) 220 error = cb(trailer + room, 221 sizeof(trailer) - room, s->gz_off, arg); 222 } 223 } while (zerror != Z_STREAM_END && 224 (zflag == Z_FINISH || s->gz_stream.avail_in > 0)); 225 226 return (error); 227 } 228 229 static void 230 gz_fini(void *stream) 231 { 232 struct gz_stream *s; 233 234 s = stream; 235 (void)deflateEnd(&s->gz_stream); 236 gz_free(NULL, s->gz_buffer); 237 gz_free(NULL, s); 238 } 239 240 struct compressor_methods gzip_methods = { 241 .format = COMPRESS_GZIP, 242 .init = gz_init, 243 .reset = gz_reset, 244 .write = gz_write, 245 .fini = gz_fini, 246 }; 247 DATA_SET(compressors, gzip_methods); 248 249 #endif /* GZIO */ 250 251 #ifdef ZSTDIO 252 253 #define ZSTD_STATIC_LINKING_ONLY 254 #include <contrib/zstd/lib/zstd.h> 255 256 struct zstdio_stream { 257 ZSTD_CCtx *zst_stream; 258 ZSTD_inBuffer zst_inbuffer; 259 ZSTD_outBuffer zst_outbuffer; 260 uint8_t * zst_buffer; /* output buffer */ 261 size_t zst_maxiosz; /* Max output IO size */ 262 off_t zst_off; /* offset into the output stream */ 263 void * zst_static_wkspc; 264 }; 265 266 static void *zstdio_init(size_t maxiosize, int level); 267 static void zstdio_reset(void *stream); 268 static int zstdio_write(void *stream, void *data, size_t len, 269 compressor_cb_t, void *); 270 static void zstdio_fini(void *stream); 271 272 static void * 273 zstdio_init(size_t maxiosize, int level) 274 { 275 ZSTD_CCtx *dump_compressor; 276 struct zstdio_stream *s; 277 void *wkspc, *owkspc, *buffer; 278 size_t wkspc_size, buf_size; 279 280 wkspc_size = ZSTD_estimateCStreamSize(level); 281 owkspc = wkspc = malloc(wkspc_size + 8, M_COMPRESS, 282 M_WAITOK | M_NODUMP); 283 /* Zstd API requires 8-byte alignment. */ 284 if ((uintptr_t)wkspc % 8 != 0) 285 wkspc = (void *)roundup2((uintptr_t)wkspc, 8); 286 287 dump_compressor = ZSTD_initStaticCCtx(wkspc, wkspc_size); 288 if (dump_compressor == NULL) { 289 free(owkspc, M_COMPRESS); 290 printf("%s: workspace too small.\n", __func__); 291 return (NULL); 292 } 293 294 (void)ZSTD_CCtx_setParameter(dump_compressor, ZSTD_p_checksumFlag, 1); 295 296 buf_size = ZSTD_CStreamOutSize() * 2; 297 buffer = malloc(buf_size, M_COMPRESS, M_WAITOK | M_NODUMP); 298 299 s = malloc(sizeof(*s), M_COMPRESS, M_NODUMP | M_WAITOK); 300 s->zst_buffer = buffer; 301 s->zst_outbuffer.dst = buffer; 302 s->zst_outbuffer.size = buf_size; 303 s->zst_maxiosz = maxiosize; 304 s->zst_stream = dump_compressor; 305 s->zst_static_wkspc = owkspc; 306 307 zstdio_reset(s); 308 309 return (s); 310 } 311 312 static void 313 zstdio_reset(void *stream) 314 { 315 struct zstdio_stream *s; 316 size_t res; 317 318 s = stream; 319 res = ZSTD_resetCStream(s->zst_stream, 0); 320 if (ZSTD_isError(res)) 321 panic("%s: could not reset stream %p: %s\n", __func__, s, 322 ZSTD_getErrorName(res)); 323 324 s->zst_off = 0; 325 s->zst_inbuffer.src = NULL; 326 s->zst_inbuffer.size = 0; 327 s->zst_inbuffer.pos = 0; 328 s->zst_outbuffer.pos = 0; 329 } 330 331 static int 332 zst_flush_intermediate(struct zstdio_stream *s, compressor_cb_t cb, void *arg) 333 { 334 size_t bytes_to_dump; 335 int error; 336 337 /* Flush as many full output blocks as possible. */ 338 /* XXX: 4096 is arbitrary safe HDD block size for kernel dumps */ 339 while (s->zst_outbuffer.pos >= 4096) { 340 bytes_to_dump = rounddown(s->zst_outbuffer.pos, 4096); 341 342 if (bytes_to_dump > s->zst_maxiosz) 343 bytes_to_dump = s->zst_maxiosz; 344 345 error = cb(s->zst_buffer, bytes_to_dump, s->zst_off, arg); 346 if (error != 0) 347 return (error); 348 349 /* 350 * Shift any non-full blocks up to the front of the output 351 * buffer. 352 */ 353 s->zst_outbuffer.pos -= bytes_to_dump; 354 memmove(s->zst_outbuffer.dst, 355 (char *)s->zst_outbuffer.dst + bytes_to_dump, 356 s->zst_outbuffer.pos); 357 s->zst_off += bytes_to_dump; 358 } 359 return (0); 360 } 361 362 static int 363 zstdio_flush(struct zstdio_stream *s, compressor_cb_t cb, void *arg) 364 { 365 size_t rc, lastpos; 366 int error; 367 368 /* 369 * Positive return indicates unflushed data remaining; need to call 370 * endStream again after clearing out room in output buffer. 371 */ 372 rc = 1; 373 lastpos = s->zst_outbuffer.pos; 374 while (rc > 0) { 375 rc = ZSTD_endStream(s->zst_stream, &s->zst_outbuffer); 376 if (ZSTD_isError(rc)) { 377 printf("%s: ZSTD_endStream failed (%s)\n", __func__, 378 ZSTD_getErrorName(rc)); 379 return (EIO); 380 } 381 if (lastpos == s->zst_outbuffer.pos) { 382 printf("%s: did not make forward progress endStream %zu\n", 383 __func__, lastpos); 384 return (EIO); 385 } 386 387 error = zst_flush_intermediate(s, cb, arg); 388 if (error != 0) 389 return (error); 390 391 lastpos = s->zst_outbuffer.pos; 392 } 393 394 /* 395 * We've already done an intermediate flush, so all full blocks have 396 * been written. Only a partial block remains. Padding happens in a 397 * higher layer. 398 */ 399 if (s->zst_outbuffer.pos != 0) { 400 error = cb(s->zst_buffer, s->zst_outbuffer.pos, s->zst_off, 401 arg); 402 if (error != 0) 403 return (error); 404 } 405 406 return (0); 407 } 408 409 static int 410 zstdio_write(void *stream, void *data, size_t len, compressor_cb_t cb, 411 void *arg) 412 { 413 struct zstdio_stream *s; 414 size_t lastpos, rc; 415 int error; 416 417 s = stream; 418 if (data == NULL) 419 return (zstdio_flush(s, cb, arg)); 420 421 s->zst_inbuffer.src = data; 422 s->zst_inbuffer.size = len; 423 s->zst_inbuffer.pos = 0; 424 lastpos = 0; 425 426 while (s->zst_inbuffer.pos < s->zst_inbuffer.size) { 427 rc = ZSTD_compressStream(s->zst_stream, &s->zst_outbuffer, 428 &s->zst_inbuffer); 429 if (ZSTD_isError(rc)) { 430 printf("%s: Compress failed on %p! (%s)\n", 431 __func__, data, ZSTD_getErrorName(rc)); 432 return (EIO); 433 } 434 435 if (lastpos == s->zst_inbuffer.pos) { 436 /* 437 * XXX: May need flushStream to make forward progress 438 */ 439 printf("ZSTD: did not make forward progress @pos %zu\n", 440 lastpos); 441 return (EIO); 442 } 443 lastpos = s->zst_inbuffer.pos; 444 445 error = zst_flush_intermediate(s, cb, arg); 446 if (error != 0) 447 return (error); 448 } 449 return (0); 450 } 451 452 static void 453 zstdio_fini(void *stream) 454 { 455 struct zstdio_stream *s; 456 457 s = stream; 458 if (s->zst_static_wkspc != NULL) 459 free(s->zst_static_wkspc, M_COMPRESS); 460 else 461 ZSTD_freeCCtx(s->zst_stream); 462 free(s->zst_buffer, M_COMPRESS); 463 free(s, M_COMPRESS); 464 } 465 466 static struct compressor_methods zstd_methods = { 467 .format = COMPRESS_ZSTD, 468 .init = zstdio_init, 469 .reset = zstdio_reset, 470 .write = zstdio_write, 471 .fini = zstdio_fini, 472 }; 473 DATA_SET(compressors, zstd_methods); 474 475 #endif /* ZSTDIO */ 476 477 bool 478 compressor_avail(int format) 479 { 480 struct compressor_methods **iter; 481 482 SET_FOREACH(iter, compressors) { 483 if ((*iter)->format == format) 484 return (true); 485 } 486 return (false); 487 } 488 489 struct compressor * 490 compressor_init(compressor_cb_t cb, int format, size_t maxiosize, int level, 491 void *arg) 492 { 493 struct compressor_methods **iter; 494 struct compressor *s; 495 void *priv; 496 497 SET_FOREACH(iter, compressors) { 498 if ((*iter)->format == format) 499 break; 500 } 501 if (iter == NULL) 502 return (NULL); 503 504 priv = (*iter)->init(maxiosize, level); 505 if (priv == NULL) 506 return (NULL); 507 508 s = malloc(sizeof(*s), M_COMPRESS, M_WAITOK | M_ZERO); 509 s->methods = (*iter); 510 s->priv = priv; 511 s->cb = cb; 512 s->arg = arg; 513 return (s); 514 } 515 516 void 517 compressor_reset(struct compressor *stream) 518 { 519 520 stream->methods->reset(stream->priv); 521 } 522 523 int 524 compressor_write(struct compressor *stream, void *data, size_t len) 525 { 526 527 return (stream->methods->write(stream->priv, data, len, stream->cb, 528 stream->arg)); 529 } 530 531 int 532 compressor_flush(struct compressor *stream) 533 { 534 535 return (stream->methods->write(stream->priv, NULL, 0, stream->cb, 536 stream->arg)); 537 } 538 539 void 540 compressor_fini(struct compressor *stream) 541 { 542 543 stream->methods->fini(stream->priv); 544 } 545