1From 57a3cf95b276946559f9e044c7352c11303bb9c1 Mon Sep 17 00:00:00 2001 2From: Sean Purcell <me@seanp.xyz> 3Date: Thu, 3 Aug 2017 17:47:03 -0700 4Subject: [PATCH v6] squashfs-tools: Add zstd support 5 6This patch adds zstd support to squashfs-tools. It works with zstd 7versions >= 1.0.0. It was originally written by Sean Purcell. 8 9Signed-off-by: Sean Purcell <me@seanp.xyz> 10Signed-off-by: Nick Terrell <terrelln@fb.com> 11--- 12v4 -> v5: 13- Fix patch documentation to reflect that Sean Purcell is the author 14- Don't strip trailing whitespace of unrelated code 15- Make zstd_display_options() static 16 17v5 -> v6: 18- Fix build instructions in Makefile 19 20 squashfs-tools/Makefile | 20 ++++ 21 squashfs-tools/compressor.c | 8 ++ 22 squashfs-tools/squashfs_fs.h | 1 + 23 squashfs-tools/zstd_wrapper.c | 254 ++++++++++++++++++++++++++++++++++++++++++ 24 squashfs-tools/zstd_wrapper.h | 48 ++++++++ 25 5 files changed, 331 insertions(+) 26 create mode 100644 squashfs-tools/zstd_wrapper.c 27 create mode 100644 squashfs-tools/zstd_wrapper.h 28 29diff --git a/squashfs-tools/Makefile b/squashfs-tools/Makefile 30index 52d2582..22fc559 100644 31--- a/squashfs-tools/Makefile 32+++ b/squashfs-tools/Makefile 33@@ -75,6 +75,18 @@ GZIP_SUPPORT = 1 34 #LZMA_SUPPORT = 1 35 #LZMA_DIR = ../../../../LZMA/lzma465 36 37+ 38+########### Building ZSTD support ############ 39+# 40+# The ZSTD library is supported 41+# ZSTD homepage: http://zstd.net 42+# ZSTD source repository: https://github.com/facebook/zstd 43+# 44+# To build using the ZSTD library - install the library and uncomment the 45+# ZSTD_SUPPORT line below. 46+# 47+#ZSTD_SUPPORT = 1 48+ 49 ######## Specifying default compression ######## 50 # 51 # The next line specifies which compression algorithm is used by default 52@@ -177,6 +189,14 @@ LIBS += -llz4 53 COMPRESSORS += lz4 54 endif 55 56+ifeq ($(ZSTD_SUPPORT),1) 57+CFLAGS += -DZSTD_SUPPORT 58+MKSQUASHFS_OBJS += zstd_wrapper.o 59+UNSQUASHFS_OBJS += zstd_wrapper.o 60+LIBS += -lzstd 61+COMPRESSORS += zstd 62+endif 63+ 64 ifeq ($(XATTR_SUPPORT),1) 65 ifeq ($(XATTR_DEFAULT),1) 66 CFLAGS += -DXATTR_SUPPORT -DXATTR_DEFAULT 67diff --git a/squashfs-tools/compressor.c b/squashfs-tools/compressor.c 68index 525e316..02b5e90 100644 69--- a/squashfs-tools/compressor.c 70+++ b/squashfs-tools/compressor.c 71@@ -65,6 +65,13 @@ static struct compressor xz_comp_ops = { 72 extern struct compressor xz_comp_ops; 73 #endif 74 75+#ifndef ZSTD_SUPPORT 76+static struct compressor zstd_comp_ops = { 77+ ZSTD_COMPRESSION, "zstd" 78+}; 79+#else 80+extern struct compressor zstd_comp_ops; 81+#endif 82 83 static struct compressor unknown_comp_ops = { 84 0, "unknown" 85@@ -77,6 +84,7 @@ struct compressor *compressor[] = { 86 &lzo_comp_ops, 87 &lz4_comp_ops, 88 &xz_comp_ops, 89+ &zstd_comp_ops, 90 &unknown_comp_ops 91 }; 92 93diff --git a/squashfs-tools/squashfs_fs.h b/squashfs-tools/squashfs_fs.h 94index 791fe12..afca918 100644 95--- a/squashfs-tools/squashfs_fs.h 96+++ b/squashfs-tools/squashfs_fs.h 97@@ -277,6 +277,7 @@ typedef long long squashfs_inode; 98 #define LZO_COMPRESSION 3 99 #define XZ_COMPRESSION 4 100 #define LZ4_COMPRESSION 5 101+#define ZSTD_COMPRESSION 6 102 103 struct squashfs_super_block { 104 unsigned int s_magic; 105diff --git a/squashfs-tools/zstd_wrapper.c b/squashfs-tools/zstd_wrapper.c 106new file mode 100644 107index 0000000..dcab75a 108--- /dev/null 109+++ b/squashfs-tools/zstd_wrapper.c 110@@ -0,0 +1,254 @@ 111+/* 112+ * Copyright (c) 2017 113+ * Phillip Lougher <phillip@squashfs.org.uk> 114+ * 115+ * This program is free software; you can redistribute it and/or 116+ * modify it under the terms of the GNU General Public License 117+ * as published by the Free Software Foundation; either version 2, 118+ * or (at your option) any later version. 119+ * 120+ * This program is distributed in the hope that it will be useful, 121+ * but WITHOUT ANY WARRANTY; without even the implied warranty of 122+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 123+ * GNU General Public License for more details. 124+ * 125+ * zstd_wrapper.c 126+ * 127+ * Support for ZSTD compression http://zstd.net 128+ */ 129+ 130+#include <stdio.h> 131+#include <string.h> 132+#include <stdlib.h> 133+#include <zstd.h> 134+#include <zstd_errors.h> 135+ 136+#include "squashfs_fs.h" 137+#include "zstd_wrapper.h" 138+#include "compressor.h" 139+ 140+static int compression_level = ZSTD_DEFAULT_COMPRESSION_LEVEL; 141+ 142+/* 143+ * This function is called by the options parsing code in mksquashfs.c 144+ * to parse any -X compressor option. 145+ * 146+ * This function returns: 147+ * >=0 (number of additional args parsed) on success 148+ * -1 if the option was unrecognised, or 149+ * -2 if the option was recognised, but otherwise bad in 150+ * some way (e.g. invalid parameter) 151+ * 152+ * Note: this function sets internal compressor state, but does not 153+ * pass back the results of the parsing other than success/failure. 154+ * The zstd_dump_options() function is called later to get the options in 155+ * a format suitable for writing to the filesystem. 156+ */ 157+static int zstd_options(char *argv[], int argc) 158+{ 159+ if (strcmp(argv[0], "-Xcompression-level") == 0) { 160+ if (argc < 2) { 161+ fprintf(stderr, "zstd: -Xcompression-level missing " 162+ "compression level\n"); 163+ fprintf(stderr, "zstd: -Xcompression-level it should " 164+ "be 1 <= n <= %d\n", ZSTD_maxCLevel()); 165+ goto failed; 166+ } 167+ 168+ compression_level = atoi(argv[1]); 169+ if (compression_level < 1 || 170+ compression_level > ZSTD_maxCLevel()) { 171+ fprintf(stderr, "zstd: -Xcompression-level invalid, it " 172+ "should be 1 <= n <= %d\n", ZSTD_maxCLevel()); 173+ goto failed; 174+ } 175+ 176+ return 1; 177+ } 178+ 179+ return -1; 180+failed: 181+ return -2; 182+} 183+ 184+/* 185+ * This function is called by mksquashfs to dump the parsed 186+ * compressor options in a format suitable for writing to the 187+ * compressor options field in the filesystem (stored immediately 188+ * after the superblock). 189+ * 190+ * This function returns a pointer to the compression options structure 191+ * to be stored (and the size), or NULL if there are no compression 192+ * options. 193+ */ 194+static void *zstd_dump_options(int block_size, int *size) 195+{ 196+ static struct zstd_comp_opts comp_opts; 197+ 198+ /* don't return anything if the options are all default */ 199+ if (compression_level == ZSTD_DEFAULT_COMPRESSION_LEVEL) 200+ return NULL; 201+ 202+ comp_opts.compression_level = compression_level; 203+ 204+ SQUASHFS_INSWAP_COMP_OPTS(&comp_opts); 205+ 206+ *size = sizeof(comp_opts); 207+ return &comp_opts; 208+} 209+ 210+/* 211+ * This function is a helper specifically for the append mode of 212+ * mksquashfs. Its purpose is to set the internal compressor state 213+ * to the stored compressor options in the passed compressor options 214+ * structure. 215+ * 216+ * In effect this function sets up the compressor options 217+ * to the same state they were when the filesystem was originally 218+ * generated, this is to ensure on appending, the compressor uses 219+ * the same compression options that were used to generate the 220+ * original filesystem. 221+ * 222+ * Note, even if there are no compressor options, this function is still 223+ * called with an empty compressor structure (size == 0), to explicitly 224+ * set the default options, this is to ensure any user supplied 225+ * -X options on the appending mksquashfs command line are over-ridden. 226+ * 227+ * This function returns 0 on successful extraction of options, and -1 on error. 228+ */ 229+static int zstd_extract_options(int block_size, void *buffer, int size) 230+{ 231+ struct zstd_comp_opts *comp_opts = buffer; 232+ 233+ if (size == 0) { 234+ /* Set default values */ 235+ compression_level = ZSTD_DEFAULT_COMPRESSION_LEVEL; 236+ return 0; 237+ } 238+ 239+ /* we expect a comp_opts structure of sufficient size to be present */ 240+ if (size < sizeof(*comp_opts)) 241+ goto failed; 242+ 243+ SQUASHFS_INSWAP_COMP_OPTS(comp_opts); 244+ 245+ if (comp_opts->compression_level < 1 || 246+ comp_opts->compression_level > ZSTD_maxCLevel()) { 247+ fprintf(stderr, "zstd: bad compression level in compression " 248+ "options structure\n"); 249+ goto failed; 250+ } 251+ 252+ compression_level = comp_opts->compression_level; 253+ 254+ return 0; 255+ 256+failed: 257+ fprintf(stderr, "zstd: error reading stored compressor options from " 258+ "filesystem!\n"); 259+ 260+ return -1; 261+} 262+ 263+static void zstd_display_options(void *buffer, int size) 264+{ 265+ struct zstd_comp_opts *comp_opts = buffer; 266+ 267+ /* we expect a comp_opts structure of sufficient size to be present */ 268+ if (size < sizeof(*comp_opts)) 269+ goto failed; 270+ 271+ SQUASHFS_INSWAP_COMP_OPTS(comp_opts); 272+ 273+ if (comp_opts->compression_level < 1 || 274+ comp_opts->compression_level > ZSTD_maxCLevel()) { 275+ fprintf(stderr, "zstd: bad compression level in compression " 276+ "options structure\n"); 277+ goto failed; 278+ } 279+ 280+ printf("\tcompression-level %d\n", comp_opts->compression_level); 281+ 282+ return; 283+ 284+failed: 285+ fprintf(stderr, "zstd: error reading stored compressor options from " 286+ "filesystem!\n"); 287+} 288+ 289+/* 290+ * This function is called by mksquashfs to initialise the 291+ * compressor, before compress() is called. 292+ * 293+ * This function returns 0 on success, and -1 on error. 294+ */ 295+static int zstd_init(void **strm, int block_size, int datablock) 296+{ 297+ ZSTD_CCtx *cctx = ZSTD_createCCtx(); 298+ 299+ if (!cctx) { 300+ fprintf(stderr, "zstd: failed to allocate compression " 301+ "context!\n"); 302+ return -1; 303+ } 304+ 305+ *strm = cctx; 306+ return 0; 307+} 308+ 309+static int zstd_compress(void *strm, void *dest, void *src, int size, 310+ int block_size, int *error) 311+{ 312+ const size_t res = ZSTD_compressCCtx((ZSTD_CCtx*)strm, dest, block_size, 313+ src, size, compression_level); 314+ 315+ if (ZSTD_isError(res)) { 316+ /* FIXME: 317+ * zstd does not expose stable error codes. The error enum may 318+ * change between versions. Until upstream zstd stablizes the 319+ * error codes, we have no way of knowing why the error occurs. 320+ * zstd shouldn't fail to compress any input unless there isn't 321+ * enough output space. We assume that is the cause and return 322+ * the special error code for not enough output space. 323+ */ 324+ return 0; 325+ } 326+ 327+ return (int)res; 328+} 329+ 330+static int zstd_uncompress(void *dest, void *src, int size, int outsize, 331+ int *error) 332+{ 333+ const size_t res = ZSTD_decompress(dest, outsize, src, size); 334+ 335+ if (ZSTD_isError(res)) { 336+ fprintf(stderr, "\t%d %d\n", outsize, size); 337+ 338+ *error = (int)ZSTD_getErrorCode(res); 339+ return -1; 340+ } 341+ 342+ return (int)res; 343+} 344+ 345+static void zstd_usage(void) 346+{ 347+ fprintf(stderr, "\t -Xcompression-level <compression-level>\n"); 348+ fprintf(stderr, "\t\t<compression-level> should be 1 .. %d (default " 349+ "%d)\n", ZSTD_maxCLevel(), ZSTD_DEFAULT_COMPRESSION_LEVEL); 350+} 351+ 352+struct compressor zstd_comp_ops = { 353+ .init = zstd_init, 354+ .compress = zstd_compress, 355+ .uncompress = zstd_uncompress, 356+ .options = zstd_options, 357+ .dump_options = zstd_dump_options, 358+ .extract_options = zstd_extract_options, 359+ .display_options = zstd_display_options, 360+ .usage = zstd_usage, 361+ .id = ZSTD_COMPRESSION, 362+ .name = "zstd", 363+ .supported = 1 364+}; 365diff --git a/squashfs-tools/zstd_wrapper.h b/squashfs-tools/zstd_wrapper.h 366new file mode 100644 367index 0000000..4fbef0a 368--- /dev/null 369+++ b/squashfs-tools/zstd_wrapper.h 370@@ -0,0 +1,48 @@ 371+#ifndef ZSTD_WRAPPER_H 372+#define ZSTD_WRAPPER_H 373+/* 374+ * Squashfs 375+ * 376+ * Copyright (c) 2017 377+ * Phillip Lougher <phillip@squashfs.org.uk> 378+ * 379+ * This program is free software; you can redistribute it and/or 380+ * modify it under the terms of the GNU General Public License 381+ * as published by the Free Software Foundation; either version 2, 382+ * or (at your option) any later version. 383+ * 384+ * This program is distributed in the hope that it will be useful, 385+ * but WITHOUT ANY WARRANTY; without even the implied warranty of 386+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 387+ * GNU General Public License for more details. 388+ * 389+ * zstd_wrapper.h 390+ * 391+ */ 392+ 393+#ifndef linux 394+#define __BYTE_ORDER BYTE_ORDER 395+#define __BIG_ENDIAN BIG_ENDIAN 396+#define __LITTLE_ENDIAN LITTLE_ENDIAN 397+#else 398+#include <endian.h> 399+#endif 400+ 401+#if __BYTE_ORDER == __BIG_ENDIAN 402+extern unsigned int inswap_le16(unsigned short); 403+extern unsigned int inswap_le32(unsigned int); 404+ 405+#define SQUASHFS_INSWAP_COMP_OPTS(s) { \ 406+ (s)->compression_level = inswap_le32((s)->compression_level); \ 407+} 408+#else 409+#define SQUASHFS_INSWAP_COMP_OPTS(s) 410+#endif 411+ 412+/* Default compression */ 413+#define ZSTD_DEFAULT_COMPRESSION_LEVEL 15 414+ 415+struct zstd_comp_opts { 416+ int compression_level; 417+}; 418+#endif 419-- 4202.9.5 421