1 /***********************************************************************************************************************************
2 Test Compression
3 ***********************************************************************************************************************************/
4 #include "common/io/filter/group.h"
5 #include "common/io/bufferRead.h"
6 #include "common/io/bufferWrite.h"
7 #include "common/io/io.h"
8 #include "storage/posix/storage.h"
9 
10 /***********************************************************************************************************************************
11 Compress data
12 ***********************************************************************************************************************************/
13 static Buffer *
testCompress(IoFilter * compress,Buffer * decompressed,size_t inputSize,size_t outputSize)14 testCompress(IoFilter *compress, Buffer *decompressed, size_t inputSize, size_t outputSize)
15 {
16     Buffer *compressed = bufNew(1024 * 1024);
17     size_t inputTotal = 0;
18     ioBufferSizeSet(outputSize);
19 
20     IoWrite *write = ioBufferWriteNew(compressed);
21     ioFilterGroupAdd(ioWriteFilterGroup(write), compress);
22     ioWriteOpen(write);
23 
24     // Compress input data
25     while (inputTotal < bufSize(decompressed))
26     {
27         // Generate the input buffer based on input size.  This breaks the data up into chunks as it would be in a real scenario.
28         Buffer *input = bufNewC(
29             bufPtr(decompressed) + inputTotal,
30             inputSize > bufSize(decompressed) - inputTotal ? bufSize(decompressed) - inputTotal : inputSize);
31 
32         ioWrite(write, input);
33 
34         inputTotal += bufUsed(input);
35         bufFree(input);
36     }
37 
38     ioWriteClose(write);
39     ioFilterFree(compress);
40 
41     return compressed;
42 }
43 
44 /***********************************************************************************************************************************
45 Decompress data
46 ***********************************************************************************************************************************/
47 static Buffer *
testDecompress(IoFilter * decompress,Buffer * compressed,size_t inputSize,size_t outputSize)48 testDecompress(IoFilter *decompress, Buffer *compressed, size_t inputSize, size_t outputSize)
49 {
50     Buffer *decompressed = bufNew(1024 * 1024);
51     Buffer *output = bufNew(outputSize);
52     ioBufferSizeSet(inputSize);
53 
54     IoRead *read = ioBufferReadNew(compressed);
55     ioFilterGroupAdd(ioReadFilterGroup(read), decompress);
56     ioReadOpen(read);
57 
58     while (!ioReadEof(read))
59     {
60         ioRead(read, output);
61         bufCat(decompressed, output);
62         bufUsedZero(output);
63     }
64 
65     ioReadClose(read);
66     bufFree(output);
67     ioFilterFree(decompress);
68 
69     return decompressed;
70 }
71 
72 /***********************************************************************************************************************************
73 Standard test suite to be applied to all compression types
74 ***********************************************************************************************************************************/
75 static void
testSuite(CompressType type,const char * decompressCmd)76 testSuite(CompressType type, const char *decompressCmd)
77 {
78     const char *simpleData = "A simple string";
79     Buffer *compressed = NULL;
80     Buffer *decompressed = bufNewC(simpleData, strlen(simpleData));
81 
82     VariantList *compressParamList = varLstNew();
83     varLstAdd(compressParamList, varNewUInt(1));
84 
85     // Create default storage object for testing
86     Storage *storageTest = storagePosixNewP(TEST_PATH_STR, .write = true);
87 
88     TEST_TITLE("simple data");
89 
90     TEST_ASSIGN(
91         compressed,
92         testCompress(
93             compressFilterVar(strNewFmt("%sCompress", strZ(compressTypeStr(type))), compressParamList), decompressed, 1024,
94             256 * 1024 * 1024),
95         "simple data - compress large in/large out buffer");
96 
97     // -------------------------------------------------------------------------------------------------------------------------
98     TEST_TITLE("compressed output can be decompressed with command-line tool");
99 
100     storagePutP(storageNewWriteP(storageTest, STRDEF("test.cmp")), compressed);
101     HRN_SYSTEM_FMT("%s " TEST_PATH "/test.cmp > " TEST_PATH "/test.out", decompressCmd);
102     TEST_RESULT_BOOL(bufEq(decompressed, storageGetP(storageNewReadP(storageTest, STRDEF("test.out")))), true, "check output");
103 
104     TEST_RESULT_BOOL(
105         bufEq(compressed, testCompress(compressFilter(type, 1), decompressed, 1024, 1)), true,
106         "simple data - compress large in/small out buffer");
107 
108     TEST_RESULT_BOOL(
109         bufEq(compressed, testCompress(compressFilter(type, 1), decompressed, 1, 1024)), true,
110         "simple data - compress small in/large out buffer");
111 
112     TEST_RESULT_BOOL(
113         bufEq(compressed, testCompress(compressFilter(type, 1), decompressed, 1, 1)), true,
114         "simple data - compress small in/small out buffer");
115 
116     TEST_RESULT_BOOL(
117         bufEq(
118             decompressed,
119             testDecompress(
120                 compressFilterVar(strNewFmt("%sDecompress", strZ(compressTypeStr(type))), NULL), compressed, 1024, 1024)),
121         true, "simple data - decompress large in/large out buffer");
122 
123     TEST_RESULT_BOOL(
124         bufEq(decompressed, testDecompress(decompressFilter(type), compressed, 1024, 1)), true,
125         "simple data - decompress large in/small out buffer");
126 
127     TEST_RESULT_BOOL(
128         bufEq(decompressed, testDecompress(decompressFilter(type), compressed, 1, 1024)), true,
129         "simple data - decompress small in/large out buffer");
130 
131     TEST_RESULT_BOOL(
132         bufEq(decompressed, testDecompress(decompressFilter(type), compressed, 1, 1)), true,
133         "simple data - decompress small in/small out buffer");
134 
135     // -------------------------------------------------------------------------------------------------------------------------
136     TEST_TITLE("error on no compression data");
137 
138     TEST_ERROR(testDecompress(decompressFilter(type), bufNew(0), 1, 1), FormatError, "unexpected eof in compressed data");
139 
140     // -------------------------------------------------------------------------------------------------------------------------
141     TEST_TITLE("error on truncated compression data");
142 
143     Buffer *truncated = bufNew(0);
144     bufCatSub(truncated, compressed, 0, bufUsed(compressed) - 1);
145 
146     TEST_RESULT_UINT(bufUsed(truncated), bufUsed(compressed) - 1, "check truncated buffer size");
147     TEST_ERROR(testDecompress(decompressFilter(type), truncated, 512, 512), FormatError, "unexpected eof in compressed data");
148 
149     // -------------------------------------------------------------------------------------------------------------------------
150     TEST_TITLE("compress a large non-zero input buffer into small output buffer");
151 
152     decompressed = bufNew(1024 * 1024 - 1);
153     unsigned char *chr = bufPtr(decompressed);
154 
155     // Step through the buffer, setting the individual bytes in a simple pattern (visible ASCII characters, DEC 32 - 126), to make
156     // sure that we fill the compression library's small output buffer
157     for (size_t chrIdx = 0; chrIdx < bufSize(decompressed); chrIdx++)
158         chr[chrIdx] = (unsigned char)(chrIdx % 94 + 32);
159 
160     bufUsedSet(decompressed, bufSize(decompressed));
161 
162     TEST_ASSIGN(
163         compressed, testCompress(compressFilter(type, 3), decompressed, bufSize(decompressed), 32),
164         "non-zero data - compress large in/small out buffer");
165 
166     TEST_RESULT_BOOL(
167         bufEq(decompressed, testDecompress(decompressFilter(type), compressed, bufSize(compressed), 1024 * 256)), true,
168         "non-zero data - decompress large in/small out buffer");
169 }
170 
171 /***********************************************************************************************************************************
172 Test Run
173 ***********************************************************************************************************************************/
174 void
testRun(void)175 testRun(void)
176 {
177     FUNCTION_HARNESS_VOID();
178 
179     // *****************************************************************************************************************************
180     if (testBegin("gz"))
181     {
182         // Run standard test suite
183         testSuite(compressTypeGz, "gzip -dc");
184 
185         // -------------------------------------------------------------------------------------------------------------------------
186         TEST_TITLE("gzError()");
187 
188         TEST_RESULT_INT(gzError(Z_OK), Z_OK, "check ok");
189         TEST_RESULT_INT(gzError(Z_STREAM_END), Z_STREAM_END, "check stream end");
190         TEST_ERROR(gzError(Z_NEED_DICT), AssertError, "zlib threw error: [2] need dictionary");
191         TEST_ERROR(gzError(Z_ERRNO), AssertError, "zlib threw error: [-1] file error");
192         TEST_ERROR(gzError(Z_STREAM_ERROR), FormatError, "zlib threw error: [-2] stream error");
193         TEST_ERROR(gzError(Z_DATA_ERROR), FormatError, "zlib threw error: [-3] data error");
194         TEST_ERROR(gzError(Z_MEM_ERROR), MemoryError, "zlib threw error: [-4] insufficient memory");
195         TEST_ERROR(gzError(Z_BUF_ERROR), AssertError, "zlib threw error: [-5] no space in buffer");
196         TEST_ERROR(gzError(Z_VERSION_ERROR), FormatError, "zlib threw error: [-6] incompatible version");
197         TEST_ERROR(gzError(999), AssertError, "zlib threw error: [999] unknown error");
198 
199         // -------------------------------------------------------------------------------------------------------------------------
200         TEST_TITLE("gzDecompressToLog() and gzCompressToLog()");
201 
202         GzDecompress *decompress = (GzDecompress *)ioFilterDriver(gzDecompressNew());
203 
204         TEST_RESULT_STR_Z(gzDecompressToLog(decompress), "{inputSame: false, done: false, availIn: 0}", "format object");
205 
206         decompress->inputSame = true;
207         decompress->done = true;
208 
209         TEST_RESULT_STR_Z(gzDecompressToLog(decompress), "{inputSame: true, done: true, availIn: 0}", "format object");
210     }
211 
212     // *****************************************************************************************************************************
213     if (testBegin("bz2"))
214     {
215         // Run standard test suite
216         testSuite(compressTypeBz2, "bzip2 -dc");
217 
218         // -------------------------------------------------------------------------------------------------------------------------
219         TEST_TITLE("bz2Error()");
220 
221         TEST_RESULT_INT(bz2Error(BZ_OK), BZ_OK, "check ok");
222         TEST_RESULT_INT(bz2Error(BZ_RUN_OK), BZ_RUN_OK, "check run ok");
223         TEST_RESULT_INT(bz2Error(BZ_FLUSH_OK), BZ_FLUSH_OK, "check flush ok");
224         TEST_RESULT_INT(bz2Error(BZ_FINISH_OK), BZ_FINISH_OK, "check finish ok");
225         TEST_RESULT_INT(bz2Error(BZ_STREAM_END), BZ_STREAM_END, "check stream end");
226         TEST_ERROR(bz2Error(BZ_SEQUENCE_ERROR), AssertError, "bz2 error: [-1] sequence error");
227         TEST_ERROR(bz2Error(BZ_PARAM_ERROR), AssertError, "bz2 error: [-2] parameter error");
228         TEST_ERROR(bz2Error(BZ_MEM_ERROR), MemoryError, "bz2 error: [-3] memory error");
229         TEST_ERROR(bz2Error(BZ_DATA_ERROR), FormatError, "bz2 error: [-4] data error");
230         TEST_ERROR(bz2Error(BZ_DATA_ERROR_MAGIC), FormatError, "bz2 error: [-5] data error magic");
231         TEST_ERROR(bz2Error(BZ_IO_ERROR), AssertError, "bz2 error: [-6] io error");
232         TEST_ERROR(bz2Error(BZ_UNEXPECTED_EOF), AssertError, "bz2 error: [-7] unexpected eof");
233         TEST_ERROR(bz2Error(BZ_OUTBUFF_FULL), AssertError, "bz2 error: [-8] outbuff full");
234         TEST_ERROR(bz2Error(BZ_CONFIG_ERROR), AssertError, "bz2 error: [-9] config error");
235         TEST_ERROR(bz2Error(-999), AssertError, "bz2 error: [-999] unknown error");
236 
237         // -------------------------------------------------------------------------------------------------------------------------
238         TEST_TITLE("bz2DecompressToLog() and bz2CompressToLog()");
239 
240         Bz2Compress *compress = (Bz2Compress *)ioFilterDriver(bz2CompressNew(1));
241 
242         compress->stream.avail_in = 999;
243 
244         TEST_RESULT_STR_Z(
245             bz2CompressToLog(compress), "{inputSame: false, done: false, flushing: false, avail_in: 999}", "format object");
246 
247         Bz2Decompress *decompress = (Bz2Decompress *)ioFilterDriver(bz2DecompressNew());
248 
249         decompress->inputSame = true;
250         decompress->done = true;
251 
252         TEST_RESULT_STR_Z(bz2DecompressToLog(decompress), "{inputSame: true, done: true, avail_in: 0}", "format object");
253     }
254 
255     // *****************************************************************************************************************************
256     if (testBegin("lz4"))
257     {
258 #ifdef HAVE_LIBLZ4
259         // Run standard test suite
260         testSuite(compressTypeLz4, "lz4 -dc");
261 
262         // -------------------------------------------------------------------------------------------------------------------------
263         TEST_TITLE("lz4Error()");
264 
265         TEST_RESULT_UINT(lz4Error(0), 0, "check success");
266         TEST_ERROR(lz4Error((size_t)-2), FormatError, "lz4 error: [-2] ERROR_maxBlockSize_invalid");
267 
268         // -------------------------------------------------------------------------------------------------------------------------
269         TEST_TITLE("lz4DecompressToLog() and lz4CompressToLog()");
270 
271         Lz4Compress *compress = (Lz4Compress *)ioFilterDriver(lz4CompressNew(7));
272 
273         compress->inputSame = true;
274         compress->flushing = true;
275 
276         TEST_RESULT_STR_Z(
277             lz4CompressToLog(compress), "{level: 7, first: true, inputSame: true, flushing: true}", "format object");
278 
279         Lz4Decompress *decompress = (Lz4Decompress *)ioFilterDriver(lz4DecompressNew());
280 
281         decompress->inputSame = true;
282         decompress->done = true;
283         decompress->inputOffset = 999;
284 
285         TEST_RESULT_STR_Z(
286             lz4DecompressToLog(decompress), "{inputSame: true, inputOffset: 999, frameDone false, done: true}",
287             "format object");
288 #else
289         TEST_ERROR(compressTypePresent(compressTypeLz4), OptionInvalidValueError, "pgBackRest not compiled with lz4 support");
290 #endif // HAVE_LIBLZ4
291     }
292 
293     // *****************************************************************************************************************************
294     if (testBegin("zst"))
295     {
296 #ifdef HAVE_LIBZST
297         // Run standard test suite
298         testSuite(compressTypeZst, "zstd -dc");
299 
300         // -------------------------------------------------------------------------------------------------------------------------
301         TEST_TITLE("zstError()");
302 
303         TEST_RESULT_UINT(zstError(0), 0, "check success");
304         TEST_ERROR(zstError((size_t)-12), FormatError, "zst error: [-12] Version not supported");
305 
306         // -------------------------------------------------------------------------------------------------------------------------
307         TEST_TITLE("zstDecompressToLog() and zstCompressToLog()");
308 
309         ZstCompress *compress = (ZstCompress *)ioFilterDriver(zstCompressNew(14));
310 
311         compress->inputSame = true;
312         compress->inputOffset = 49;
313         compress->flushing = true;
314 
315         TEST_RESULT_STR_Z(
316             zstCompressToLog(compress), "{level: 14, inputSame: true, inputOffset: 49, flushing: true}", "format object");
317 
318         ZstDecompress *decompress = (ZstDecompress *)ioFilterDriver(zstDecompressNew());
319 
320         decompress->inputSame = true;
321         decompress->done = true;
322         decompress->inputOffset = 999;
323 
324         TEST_RESULT_STR_Z(
325             zstDecompressToLog(decompress), "{inputSame: true, inputOffset: 999, frameDone false, done: true}",
326             "format object");
327 #else
328         TEST_ERROR(compressTypePresent(compressTypeZst), OptionInvalidValueError, "pgBackRest not compiled with zst support");
329 #endif // HAVE_LIBZST
330     }
331 
332     // Test everything in the helper that is not tested in the individual compression type tests
333     // *****************************************************************************************************************************
334     if (testBegin("helper"))
335     {
336         TEST_TITLE("compressTypeEnum()");
337 
338         TEST_RESULT_UINT(compressTypeEnum(STRDEF("none")), compressTypeNone, "none enum");
339         TEST_RESULT_UINT(compressTypeEnum(STRDEF("gz")), compressTypeGz, "gz enum");
340         TEST_ERROR(compressTypeEnum(STRDEF(BOGUS_STR)), AssertError, "invalid compression type 'BOGUS'");
341 
342         // -------------------------------------------------------------------------------------------------------------------------
343         TEST_TITLE("compressTypePresent()");
344 
345         TEST_RESULT_VOID(compressTypePresent(compressTypeNone), "type none always present");
346         TEST_ERROR(compressTypePresent(compressTypeXz), OptionInvalidValueError, "pgBackRest not compiled with xz support");
347 
348         // -------------------------------------------------------------------------------------------------------------------------
349         TEST_TITLE("compressTypeFromName()");
350 
351         TEST_RESULT_UINT(compressTypeFromName(STRDEF("file")), compressTypeNone, "type from name");
352         TEST_RESULT_UINT(compressTypeFromName(STRDEF("file.gz")), compressTypeGz, "type from name");
353 
354         // -------------------------------------------------------------------------------------------------------------------------
355         TEST_TITLE("compressFilterVar()");
356 
357         TEST_RESULT_PTR(compressFilterVar(STRDEF("BOGUS"), 0), NULL, "no filter match");
358 
359         // -------------------------------------------------------------------------------------------------------------------------
360         TEST_TITLE("compressExtStr()");
361 
362         TEST_RESULT_STR_Z(compressExtStr(compressTypeNone), "", "one ext");
363         TEST_RESULT_STR_Z(compressExtStr(compressTypeGz), ".gz", "gz ext");
364 
365         // -------------------------------------------------------------------------------------------------------------------------
366         TEST_TITLE("compressExtCat()");
367 
368         String *file = strNewZ("file");
369         TEST_RESULT_VOID(compressExtCat(file, compressTypeGz), "cat gz ext");
370         TEST_RESULT_STR_Z(file, "file.gz", "    check gz ext");
371 
372         // -------------------------------------------------------------------------------------------------------------------------
373         TEST_TITLE("compressExtStrip()");
374 
375         TEST_ERROR(compressExtStrip(STRDEF("file"), compressTypeGz), FormatError, "'file' must have '.gz' extension");
376         TEST_RESULT_STR_Z(compressExtStrip(STRDEF("file"), compressTypeNone), "file", "nothing to strip");
377         TEST_RESULT_STR_Z(compressExtStrip(STRDEF("file.gz"), compressTypeGz), "file", "strip gz");
378 
379         // -------------------------------------------------------------------------------------------------------------------------
380         TEST_TITLE("compressLevelDefault()");
381 
382         TEST_RESULT_INT(compressLevelDefault(compressTypeNone), 0, "none level=0");
383         TEST_RESULT_INT(compressLevelDefault(compressTypeGz), 6, "gz level=6");
384     }
385 
386     FUNCTION_HARNESS_RETURN_VOID();
387 }
388