1 // LZ4 API example : Dictionary Random Access
2
3 #if defined(_MSC_VER) && (_MSC_VER <= 1800) /* Visual Studio <= 2013 */
4 # define _CRT_SECURE_NO_WARNINGS
5 # define snprintf sprintf_s
6 #endif
7 #include "lz4.h"
8
9 #include <stdio.h>
10 #include <stdint.h>
11 #include <stdlib.h>
12 #include <string.h>
13
14 #define MIN(x, y) ((x) < (y) ? (x) : (y))
15
16 enum {
17 BLOCK_BYTES = 1024, /* 1 KiB of uncompressed data in a block */
18 DICTIONARY_BYTES = 1024, /* Load a 1 KiB dictionary */
19 MAX_BLOCKS = 1024 /* For simplicity of implementation */
20 };
21
22 /**
23 * Magic bytes for this test case.
24 * This is not a great magic number because it is a common word in ASCII.
25 * However, it is important to have some versioning system in your format.
26 */
27 const char kTestMagic[] = { 'T', 'E', 'S', 'T' };
28
29
write_int(FILE * fp,int i)30 void write_int(FILE* fp, int i) {
31 size_t written = fwrite(&i, sizeof(i), 1, fp);
32 if (written != 1) { exit(10); }
33 }
34
write_bin(FILE * fp,const void * array,size_t arrayBytes)35 void write_bin(FILE* fp, const void* array, size_t arrayBytes) {
36 size_t written = fwrite(array, 1, arrayBytes, fp);
37 if (written != arrayBytes) { exit(11); }
38 }
39
read_int(FILE * fp,int * i)40 void read_int(FILE* fp, int* i) {
41 size_t read = fread(i, sizeof(*i), 1, fp);
42 if (read != 1) { exit(12); }
43 }
44
read_bin(FILE * fp,void * array,size_t arrayBytes)45 size_t read_bin(FILE* fp, void* array, size_t arrayBytes) {
46 size_t read = fread(array, 1, arrayBytes, fp);
47 if (ferror(fp)) { exit(12); }
48 return read;
49 }
50
seek_bin(FILE * fp,long offset,int origin)51 void seek_bin(FILE* fp, long offset, int origin) {
52 if (fseek(fp, offset, origin)) { exit(14); }
53 }
54
55
test_compress(FILE * outFp,FILE * inpFp,void * dict,int dictSize)56 void test_compress(FILE* outFp, FILE* inpFp, void *dict, int dictSize)
57 {
58 LZ4_stream_t lz4Stream_body;
59 LZ4_stream_t* lz4Stream = &lz4Stream_body;
60
61 char inpBuf[BLOCK_BYTES];
62 int offsets[MAX_BLOCKS];
63 int *offsetsEnd = offsets;
64
65
66 LZ4_initStream(lz4Stream, sizeof(*lz4Stream));
67
68 /* Write header magic */
69 write_bin(outFp, kTestMagic, sizeof(kTestMagic));
70
71 *offsetsEnd++ = sizeof(kTestMagic);
72 /* Write compressed data blocks. Each block contains BLOCK_BYTES of plain
73 data except possibly the last. */
74 for(;;) {
75 const int inpBytes = (int) read_bin(inpFp, inpBuf, BLOCK_BYTES);
76 if(0 == inpBytes) {
77 break;
78 }
79
80 /* Forget previously compressed data and load the dictionary */
81 LZ4_loadDict(lz4Stream, dict, dictSize);
82 {
83 char cmpBuf[LZ4_COMPRESSBOUND(BLOCK_BYTES)];
84 const int cmpBytes = LZ4_compress_fast_continue(
85 lz4Stream, inpBuf, cmpBuf, inpBytes, sizeof(cmpBuf), 1);
86 if(cmpBytes <= 0) { exit(1); }
87 write_bin(outFp, cmpBuf, (size_t)cmpBytes);
88 /* Keep track of the offsets */
89 *offsetsEnd = *(offsetsEnd - 1) + cmpBytes;
90 ++offsetsEnd;
91 }
92 if (offsetsEnd - offsets > MAX_BLOCKS) { exit(2); }
93 }
94 /* Write the tailing jump table */
95 {
96 int *ptr = offsets;
97 while (ptr != offsetsEnd) {
98 write_int(outFp, *ptr++);
99 }
100 write_int(outFp, offsetsEnd - offsets);
101 }
102 }
103
104
test_decompress(FILE * outFp,FILE * inpFp,void * dict,int dictSize,int offset,int length)105 void test_decompress(FILE* outFp, FILE* inpFp, void *dict, int dictSize, int offset, int length)
106 {
107 LZ4_streamDecode_t lz4StreamDecode_body;
108 LZ4_streamDecode_t* lz4StreamDecode = &lz4StreamDecode_body;
109
110 /* The blocks [currentBlock, endBlock) contain the data we want */
111 int currentBlock = offset / BLOCK_BYTES;
112 int endBlock = ((offset + length - 1) / BLOCK_BYTES) + 1;
113
114 char decBuf[BLOCK_BYTES];
115 int offsets[MAX_BLOCKS];
116
117 /* Special cases */
118 if (length == 0) { return; }
119
120 /* Read the magic bytes */
121 {
122 char magic[sizeof(kTestMagic)];
123 size_t read = read_bin(inpFp, magic, sizeof(magic));
124 if (read != sizeof(magic)) { exit(1); }
125 if (memcmp(kTestMagic, magic, sizeof(magic))) { exit(2); }
126 }
127
128 /* Read the offsets tail */
129 {
130 int numOffsets;
131 int block;
132 int *offsetsPtr = offsets;
133 seek_bin(inpFp, -4, SEEK_END);
134 read_int(inpFp, &numOffsets);
135 if (numOffsets <= endBlock) { exit(3); }
136 seek_bin(inpFp, -4 * (numOffsets + 1), SEEK_END);
137 for (block = 0; block <= endBlock; ++block) {
138 read_int(inpFp, offsetsPtr++);
139 }
140 }
141 /* Seek to the first block to read */
142 seek_bin(inpFp, offsets[currentBlock], SEEK_SET);
143 offset = offset % BLOCK_BYTES;
144
145 /* Start decoding */
146 for(; currentBlock < endBlock; ++currentBlock) {
147 char cmpBuf[LZ4_COMPRESSBOUND(BLOCK_BYTES)];
148 /* The difference in offsets is the size of the block */
149 int cmpBytes = offsets[currentBlock + 1] - offsets[currentBlock];
150 {
151 const size_t read = read_bin(inpFp, cmpBuf, (size_t)cmpBytes);
152 if(read != (size_t)cmpBytes) { exit(4); }
153 }
154
155 /* Load the dictionary */
156 LZ4_setStreamDecode(lz4StreamDecode, dict, dictSize);
157 {
158 const int decBytes = LZ4_decompress_safe_continue(
159 lz4StreamDecode, cmpBuf, decBuf, cmpBytes, BLOCK_BYTES);
160 if(decBytes <= 0) { exit(5); }
161 {
162 /* Write out the part of the data we care about */
163 int blockLength = MIN(length, (decBytes - offset));
164 write_bin(outFp, decBuf + offset, (size_t)blockLength);
165 offset = 0;
166 length -= blockLength;
167 }
168 }
169 }
170 }
171
172
compare(FILE * fp0,FILE * fp1,int length)173 int compare(FILE* fp0, FILE* fp1, int length)
174 {
175 int result = 0;
176
177 while(0 == result) {
178 char b0[4096];
179 char b1[4096];
180 const size_t r0 = read_bin(fp0, b0, MIN(length, (int)sizeof(b0)));
181 const size_t r1 = read_bin(fp1, b1, MIN(length, (int)sizeof(b1)));
182
183 result = (int) r0 - (int) r1;
184
185 if(0 == r0 || 0 == r1) {
186 break;
187 }
188 if(0 == result) {
189 result = memcmp(b0, b1, r0);
190 }
191 length -= r0;
192 }
193
194 return result;
195 }
196
197
main(int argc,char * argv[])198 int main(int argc, char* argv[])
199 {
200 char inpFilename[256] = { 0 };
201 char lz4Filename[256] = { 0 };
202 char decFilename[256] = { 0 };
203 char dictFilename[256] = { 0 };
204 int offset;
205 int length;
206 char dict[DICTIONARY_BYTES];
207 int dictSize;
208
209 if(argc < 5) {
210 printf("Usage: %s input dictionary offset length", argv[0]);
211 return 0;
212 }
213
214 snprintf(inpFilename, 256, "%s", argv[1]);
215 snprintf(lz4Filename, 256, "%s.lz4s-%d", argv[1], BLOCK_BYTES);
216 snprintf(decFilename, 256, "%s.lz4s-%d.dec", argv[1], BLOCK_BYTES);
217 snprintf(dictFilename, 256, "%s", argv[2]);
218 offset = atoi(argv[3]);
219 length = atoi(argv[4]);
220
221 printf("inp = [%s]\n", inpFilename);
222 printf("lz4 = [%s]\n", lz4Filename);
223 printf("dec = [%s]\n", decFilename);
224 printf("dict = [%s]\n", dictFilename);
225 printf("offset = [%d]\n", offset);
226 printf("length = [%d]\n", length);
227
228 /* Load dictionary */
229 {
230 FILE* dictFp = fopen(dictFilename, "rb");
231 dictSize = (int)read_bin(dictFp, dict, DICTIONARY_BYTES);
232 fclose(dictFp);
233 }
234
235 /* compress */
236 {
237 FILE* inpFp = fopen(inpFilename, "rb");
238 FILE* outFp = fopen(lz4Filename, "wb");
239
240 printf("compress : %s -> %s\n", inpFilename, lz4Filename);
241 test_compress(outFp, inpFp, dict, dictSize);
242 printf("compress : done\n");
243
244 fclose(outFp);
245 fclose(inpFp);
246 }
247
248 /* decompress */
249 {
250 FILE* inpFp = fopen(lz4Filename, "rb");
251 FILE* outFp = fopen(decFilename, "wb");
252
253 printf("decompress : %s -> %s\n", lz4Filename, decFilename);
254 test_decompress(outFp, inpFp, dict, DICTIONARY_BYTES, offset, length);
255 printf("decompress : done\n");
256
257 fclose(outFp);
258 fclose(inpFp);
259 }
260
261 /* verify */
262 {
263 FILE* inpFp = fopen(inpFilename, "rb");
264 FILE* decFp = fopen(decFilename, "rb");
265 seek_bin(inpFp, offset, SEEK_SET);
266
267 printf("verify : %s <-> %s\n", inpFilename, decFilename);
268 const int cmp = compare(inpFp, decFp, length);
269 if(0 == cmp) {
270 printf("verify : OK\n");
271 } else {
272 printf("verify : NG\n");
273 }
274
275 fclose(decFp);
276 fclose(inpFp);
277 }
278
279 return 0;
280 }
281