1 /***********************************************************************************************************************************
2 Test Compression
3 ***********************************************************************************************************************************/
4 #include "common/io/filter/group.h"
5 #include "common/io/bufferRead.h"
6 #include "common/io/bufferWrite.h"
7 #include "common/io/io.h"
8 #include "storage/posix/storage.h"
9
10 /***********************************************************************************************************************************
11 Compress data
12 ***********************************************************************************************************************************/
13 static Buffer *
testCompress(IoFilter * compress,Buffer * decompressed,size_t inputSize,size_t outputSize)14 testCompress(IoFilter *compress, Buffer *decompressed, size_t inputSize, size_t outputSize)
15 {
16 Buffer *compressed = bufNew(1024 * 1024);
17 size_t inputTotal = 0;
18 ioBufferSizeSet(outputSize);
19
20 IoWrite *write = ioBufferWriteNew(compressed);
21 ioFilterGroupAdd(ioWriteFilterGroup(write), compress);
22 ioWriteOpen(write);
23
24 // Compress input data
25 while (inputTotal < bufSize(decompressed))
26 {
27 // Generate the input buffer based on input size. This breaks the data up into chunks as it would be in a real scenario.
28 Buffer *input = bufNewC(
29 bufPtr(decompressed) + inputTotal,
30 inputSize > bufSize(decompressed) - inputTotal ? bufSize(decompressed) - inputTotal : inputSize);
31
32 ioWrite(write, input);
33
34 inputTotal += bufUsed(input);
35 bufFree(input);
36 }
37
38 ioWriteClose(write);
39 ioFilterFree(compress);
40
41 return compressed;
42 }
43
44 /***********************************************************************************************************************************
45 Decompress data
46 ***********************************************************************************************************************************/
47 static Buffer *
testDecompress(IoFilter * decompress,Buffer * compressed,size_t inputSize,size_t outputSize)48 testDecompress(IoFilter *decompress, Buffer *compressed, size_t inputSize, size_t outputSize)
49 {
50 Buffer *decompressed = bufNew(1024 * 1024);
51 Buffer *output = bufNew(outputSize);
52 ioBufferSizeSet(inputSize);
53
54 IoRead *read = ioBufferReadNew(compressed);
55 ioFilterGroupAdd(ioReadFilterGroup(read), decompress);
56 ioReadOpen(read);
57
58 while (!ioReadEof(read))
59 {
60 ioRead(read, output);
61 bufCat(decompressed, output);
62 bufUsedZero(output);
63 }
64
65 ioReadClose(read);
66 bufFree(output);
67 ioFilterFree(decompress);
68
69 return decompressed;
70 }
71
72 /***********************************************************************************************************************************
73 Standard test suite to be applied to all compression types
74 ***********************************************************************************************************************************/
75 static void
testSuite(CompressType type,const char * decompressCmd)76 testSuite(CompressType type, const char *decompressCmd)
77 {
78 const char *simpleData = "A simple string";
79 Buffer *compressed = NULL;
80 Buffer *decompressed = bufNewC(simpleData, strlen(simpleData));
81
82 VariantList *compressParamList = varLstNew();
83 varLstAdd(compressParamList, varNewUInt(1));
84
85 // Create default storage object for testing
86 Storage *storageTest = storagePosixNewP(TEST_PATH_STR, .write = true);
87
88 TEST_TITLE("simple data");
89
90 TEST_ASSIGN(
91 compressed,
92 testCompress(
93 compressFilterVar(strNewFmt("%sCompress", strZ(compressTypeStr(type))), compressParamList), decompressed, 1024,
94 256 * 1024 * 1024),
95 "simple data - compress large in/large out buffer");
96
97 // -------------------------------------------------------------------------------------------------------------------------
98 TEST_TITLE("compressed output can be decompressed with command-line tool");
99
100 storagePutP(storageNewWriteP(storageTest, STRDEF("test.cmp")), compressed);
101 HRN_SYSTEM_FMT("%s " TEST_PATH "/test.cmp > " TEST_PATH "/test.out", decompressCmd);
102 TEST_RESULT_BOOL(bufEq(decompressed, storageGetP(storageNewReadP(storageTest, STRDEF("test.out")))), true, "check output");
103
104 TEST_RESULT_BOOL(
105 bufEq(compressed, testCompress(compressFilter(type, 1), decompressed, 1024, 1)), true,
106 "simple data - compress large in/small out buffer");
107
108 TEST_RESULT_BOOL(
109 bufEq(compressed, testCompress(compressFilter(type, 1), decompressed, 1, 1024)), true,
110 "simple data - compress small in/large out buffer");
111
112 TEST_RESULT_BOOL(
113 bufEq(compressed, testCompress(compressFilter(type, 1), decompressed, 1, 1)), true,
114 "simple data - compress small in/small out buffer");
115
116 TEST_RESULT_BOOL(
117 bufEq(
118 decompressed,
119 testDecompress(
120 compressFilterVar(strNewFmt("%sDecompress", strZ(compressTypeStr(type))), NULL), compressed, 1024, 1024)),
121 true, "simple data - decompress large in/large out buffer");
122
123 TEST_RESULT_BOOL(
124 bufEq(decompressed, testDecompress(decompressFilter(type), compressed, 1024, 1)), true,
125 "simple data - decompress large in/small out buffer");
126
127 TEST_RESULT_BOOL(
128 bufEq(decompressed, testDecompress(decompressFilter(type), compressed, 1, 1024)), true,
129 "simple data - decompress small in/large out buffer");
130
131 TEST_RESULT_BOOL(
132 bufEq(decompressed, testDecompress(decompressFilter(type), compressed, 1, 1)), true,
133 "simple data - decompress small in/small out buffer");
134
135 // -------------------------------------------------------------------------------------------------------------------------
136 TEST_TITLE("error on no compression data");
137
138 TEST_ERROR(testDecompress(decompressFilter(type), bufNew(0), 1, 1), FormatError, "unexpected eof in compressed data");
139
140 // -------------------------------------------------------------------------------------------------------------------------
141 TEST_TITLE("error on truncated compression data");
142
143 Buffer *truncated = bufNew(0);
144 bufCatSub(truncated, compressed, 0, bufUsed(compressed) - 1);
145
146 TEST_RESULT_UINT(bufUsed(truncated), bufUsed(compressed) - 1, "check truncated buffer size");
147 TEST_ERROR(testDecompress(decompressFilter(type), truncated, 512, 512), FormatError, "unexpected eof in compressed data");
148
149 // -------------------------------------------------------------------------------------------------------------------------
150 TEST_TITLE("compress a large non-zero input buffer into small output buffer");
151
152 decompressed = bufNew(1024 * 1024 - 1);
153 unsigned char *chr = bufPtr(decompressed);
154
155 // Step through the buffer, setting the individual bytes in a simple pattern (visible ASCII characters, DEC 32 - 126), to make
156 // sure that we fill the compression library's small output buffer
157 for (size_t chrIdx = 0; chrIdx < bufSize(decompressed); chrIdx++)
158 chr[chrIdx] = (unsigned char)(chrIdx % 94 + 32);
159
160 bufUsedSet(decompressed, bufSize(decompressed));
161
162 TEST_ASSIGN(
163 compressed, testCompress(compressFilter(type, 3), decompressed, bufSize(decompressed), 32),
164 "non-zero data - compress large in/small out buffer");
165
166 TEST_RESULT_BOOL(
167 bufEq(decompressed, testDecompress(decompressFilter(type), compressed, bufSize(compressed), 1024 * 256)), true,
168 "non-zero data - decompress large in/small out buffer");
169 }
170
171 /***********************************************************************************************************************************
172 Test Run
173 ***********************************************************************************************************************************/
174 void
testRun(void)175 testRun(void)
176 {
177 FUNCTION_HARNESS_VOID();
178
179 // *****************************************************************************************************************************
180 if (testBegin("gz"))
181 {
182 // Run standard test suite
183 testSuite(compressTypeGz, "gzip -dc");
184
185 // -------------------------------------------------------------------------------------------------------------------------
186 TEST_TITLE("gzError()");
187
188 TEST_RESULT_INT(gzError(Z_OK), Z_OK, "check ok");
189 TEST_RESULT_INT(gzError(Z_STREAM_END), Z_STREAM_END, "check stream end");
190 TEST_ERROR(gzError(Z_NEED_DICT), AssertError, "zlib threw error: [2] need dictionary");
191 TEST_ERROR(gzError(Z_ERRNO), AssertError, "zlib threw error: [-1] file error");
192 TEST_ERROR(gzError(Z_STREAM_ERROR), FormatError, "zlib threw error: [-2] stream error");
193 TEST_ERROR(gzError(Z_DATA_ERROR), FormatError, "zlib threw error: [-3] data error");
194 TEST_ERROR(gzError(Z_MEM_ERROR), MemoryError, "zlib threw error: [-4] insufficient memory");
195 TEST_ERROR(gzError(Z_BUF_ERROR), AssertError, "zlib threw error: [-5] no space in buffer");
196 TEST_ERROR(gzError(Z_VERSION_ERROR), FormatError, "zlib threw error: [-6] incompatible version");
197 TEST_ERROR(gzError(999), AssertError, "zlib threw error: [999] unknown error");
198
199 // -------------------------------------------------------------------------------------------------------------------------
200 TEST_TITLE("gzDecompressToLog() and gzCompressToLog()");
201
202 GzDecompress *decompress = (GzDecompress *)ioFilterDriver(gzDecompressNew());
203
204 TEST_RESULT_STR_Z(gzDecompressToLog(decompress), "{inputSame: false, done: false, availIn: 0}", "format object");
205
206 decompress->inputSame = true;
207 decompress->done = true;
208
209 TEST_RESULT_STR_Z(gzDecompressToLog(decompress), "{inputSame: true, done: true, availIn: 0}", "format object");
210 }
211
212 // *****************************************************************************************************************************
213 if (testBegin("bz2"))
214 {
215 // Run standard test suite
216 testSuite(compressTypeBz2, "bzip2 -dc");
217
218 // -------------------------------------------------------------------------------------------------------------------------
219 TEST_TITLE("bz2Error()");
220
221 TEST_RESULT_INT(bz2Error(BZ_OK), BZ_OK, "check ok");
222 TEST_RESULT_INT(bz2Error(BZ_RUN_OK), BZ_RUN_OK, "check run ok");
223 TEST_RESULT_INT(bz2Error(BZ_FLUSH_OK), BZ_FLUSH_OK, "check flush ok");
224 TEST_RESULT_INT(bz2Error(BZ_FINISH_OK), BZ_FINISH_OK, "check finish ok");
225 TEST_RESULT_INT(bz2Error(BZ_STREAM_END), BZ_STREAM_END, "check stream end");
226 TEST_ERROR(bz2Error(BZ_SEQUENCE_ERROR), AssertError, "bz2 error: [-1] sequence error");
227 TEST_ERROR(bz2Error(BZ_PARAM_ERROR), AssertError, "bz2 error: [-2] parameter error");
228 TEST_ERROR(bz2Error(BZ_MEM_ERROR), MemoryError, "bz2 error: [-3] memory error");
229 TEST_ERROR(bz2Error(BZ_DATA_ERROR), FormatError, "bz2 error: [-4] data error");
230 TEST_ERROR(bz2Error(BZ_DATA_ERROR_MAGIC), FormatError, "bz2 error: [-5] data error magic");
231 TEST_ERROR(bz2Error(BZ_IO_ERROR), AssertError, "bz2 error: [-6] io error");
232 TEST_ERROR(bz2Error(BZ_UNEXPECTED_EOF), AssertError, "bz2 error: [-7] unexpected eof");
233 TEST_ERROR(bz2Error(BZ_OUTBUFF_FULL), AssertError, "bz2 error: [-8] outbuff full");
234 TEST_ERROR(bz2Error(BZ_CONFIG_ERROR), AssertError, "bz2 error: [-9] config error");
235 TEST_ERROR(bz2Error(-999), AssertError, "bz2 error: [-999] unknown error");
236
237 // -------------------------------------------------------------------------------------------------------------------------
238 TEST_TITLE("bz2DecompressToLog() and bz2CompressToLog()");
239
240 Bz2Compress *compress = (Bz2Compress *)ioFilterDriver(bz2CompressNew(1));
241
242 compress->stream.avail_in = 999;
243
244 TEST_RESULT_STR_Z(
245 bz2CompressToLog(compress), "{inputSame: false, done: false, flushing: false, avail_in: 999}", "format object");
246
247 Bz2Decompress *decompress = (Bz2Decompress *)ioFilterDriver(bz2DecompressNew());
248
249 decompress->inputSame = true;
250 decompress->done = true;
251
252 TEST_RESULT_STR_Z(bz2DecompressToLog(decompress), "{inputSame: true, done: true, avail_in: 0}", "format object");
253 }
254
255 // *****************************************************************************************************************************
256 if (testBegin("lz4"))
257 {
258 #ifdef HAVE_LIBLZ4
259 // Run standard test suite
260 testSuite(compressTypeLz4, "lz4 -dc");
261
262 // -------------------------------------------------------------------------------------------------------------------------
263 TEST_TITLE("lz4Error()");
264
265 TEST_RESULT_UINT(lz4Error(0), 0, "check success");
266 TEST_ERROR(lz4Error((size_t)-2), FormatError, "lz4 error: [-2] ERROR_maxBlockSize_invalid");
267
268 // -------------------------------------------------------------------------------------------------------------------------
269 TEST_TITLE("lz4DecompressToLog() and lz4CompressToLog()");
270
271 Lz4Compress *compress = (Lz4Compress *)ioFilterDriver(lz4CompressNew(7));
272
273 compress->inputSame = true;
274 compress->flushing = true;
275
276 TEST_RESULT_STR_Z(
277 lz4CompressToLog(compress), "{level: 7, first: true, inputSame: true, flushing: true}", "format object");
278
279 Lz4Decompress *decompress = (Lz4Decompress *)ioFilterDriver(lz4DecompressNew());
280
281 decompress->inputSame = true;
282 decompress->done = true;
283 decompress->inputOffset = 999;
284
285 TEST_RESULT_STR_Z(
286 lz4DecompressToLog(decompress), "{inputSame: true, inputOffset: 999, frameDone false, done: true}",
287 "format object");
288 #else
289 TEST_ERROR(compressTypePresent(compressTypeLz4), OptionInvalidValueError, "pgBackRest not compiled with lz4 support");
290 #endif // HAVE_LIBLZ4
291 }
292
293 // *****************************************************************************************************************************
294 if (testBegin("zst"))
295 {
296 #ifdef HAVE_LIBZST
297 // Run standard test suite
298 testSuite(compressTypeZst, "zstd -dc");
299
300 // -------------------------------------------------------------------------------------------------------------------------
301 TEST_TITLE("zstError()");
302
303 TEST_RESULT_UINT(zstError(0), 0, "check success");
304 TEST_ERROR(zstError((size_t)-12), FormatError, "zst error: [-12] Version not supported");
305
306 // -------------------------------------------------------------------------------------------------------------------------
307 TEST_TITLE("zstDecompressToLog() and zstCompressToLog()");
308
309 ZstCompress *compress = (ZstCompress *)ioFilterDriver(zstCompressNew(14));
310
311 compress->inputSame = true;
312 compress->inputOffset = 49;
313 compress->flushing = true;
314
315 TEST_RESULT_STR_Z(
316 zstCompressToLog(compress), "{level: 14, inputSame: true, inputOffset: 49, flushing: true}", "format object");
317
318 ZstDecompress *decompress = (ZstDecompress *)ioFilterDriver(zstDecompressNew());
319
320 decompress->inputSame = true;
321 decompress->done = true;
322 decompress->inputOffset = 999;
323
324 TEST_RESULT_STR_Z(
325 zstDecompressToLog(decompress), "{inputSame: true, inputOffset: 999, frameDone false, done: true}",
326 "format object");
327 #else
328 TEST_ERROR(compressTypePresent(compressTypeZst), OptionInvalidValueError, "pgBackRest not compiled with zst support");
329 #endif // HAVE_LIBZST
330 }
331
332 // Test everything in the helper that is not tested in the individual compression type tests
333 // *****************************************************************************************************************************
334 if (testBegin("helper"))
335 {
336 TEST_TITLE("compressTypeEnum()");
337
338 TEST_RESULT_UINT(compressTypeEnum(STRDEF("none")), compressTypeNone, "none enum");
339 TEST_RESULT_UINT(compressTypeEnum(STRDEF("gz")), compressTypeGz, "gz enum");
340 TEST_ERROR(compressTypeEnum(STRDEF(BOGUS_STR)), AssertError, "invalid compression type 'BOGUS'");
341
342 // -------------------------------------------------------------------------------------------------------------------------
343 TEST_TITLE("compressTypePresent()");
344
345 TEST_RESULT_VOID(compressTypePresent(compressTypeNone), "type none always present");
346 TEST_ERROR(compressTypePresent(compressTypeXz), OptionInvalidValueError, "pgBackRest not compiled with xz support");
347
348 // -------------------------------------------------------------------------------------------------------------------------
349 TEST_TITLE("compressTypeFromName()");
350
351 TEST_RESULT_UINT(compressTypeFromName(STRDEF("file")), compressTypeNone, "type from name");
352 TEST_RESULT_UINT(compressTypeFromName(STRDEF("file.gz")), compressTypeGz, "type from name");
353
354 // -------------------------------------------------------------------------------------------------------------------------
355 TEST_TITLE("compressFilterVar()");
356
357 TEST_RESULT_PTR(compressFilterVar(STRDEF("BOGUS"), 0), NULL, "no filter match");
358
359 // -------------------------------------------------------------------------------------------------------------------------
360 TEST_TITLE("compressExtStr()");
361
362 TEST_RESULT_STR_Z(compressExtStr(compressTypeNone), "", "one ext");
363 TEST_RESULT_STR_Z(compressExtStr(compressTypeGz), ".gz", "gz ext");
364
365 // -------------------------------------------------------------------------------------------------------------------------
366 TEST_TITLE("compressExtCat()");
367
368 String *file = strNewZ("file");
369 TEST_RESULT_VOID(compressExtCat(file, compressTypeGz), "cat gz ext");
370 TEST_RESULT_STR_Z(file, "file.gz", " check gz ext");
371
372 // -------------------------------------------------------------------------------------------------------------------------
373 TEST_TITLE("compressExtStrip()");
374
375 TEST_ERROR(compressExtStrip(STRDEF("file"), compressTypeGz), FormatError, "'file' must have '.gz' extension");
376 TEST_RESULT_STR_Z(compressExtStrip(STRDEF("file"), compressTypeNone), "file", "nothing to strip");
377 TEST_RESULT_STR_Z(compressExtStrip(STRDEF("file.gz"), compressTypeGz), "file", "strip gz");
378
379 // -------------------------------------------------------------------------------------------------------------------------
380 TEST_TITLE("compressLevelDefault()");
381
382 TEST_RESULT_INT(compressLevelDefault(compressTypeNone), 0, "none level=0");
383 TEST_RESULT_INT(compressLevelDefault(compressTypeGz), 6, "gz level=6");
384 }
385
386 FUNCTION_HARNESS_RETURN_VOID();
387 }
388