1 #include <string.h>
2 #include <stdlib.h>
3 #include <hdf5.h>
4
5 #include "H5Zlzo.h"
6 #include "tables.h"
7
8 #ifdef HAVE_LZO_LIB
9 # include "lzo1x.h"
10 #endif
11 #ifdef HAVE_LZO2_LIB
12 # include "lzo/lzo1x.h"
13 # define HAVE_LZO_LIB /* The API for LZO and LZO2 is mostly identical */
14 #endif
15
16 /* #undef DEBUG */
17
18 /* Activate the checksum. It is safer and takes only a 1% more of
19 space and a 2% more of CPU (but sometimes is faster than without
20 checksum, which is almost negligible. F. Alted 2003/07/22
21
22 Added code for pytables 0.5 backward compatibility.
23 F. Alted 2003/07/28
24
25 Added code for saving the uncompressed length buffer as well.
26 F. Alted 2003/07/29
27
28 */
29
30 /* From pytables 0.8 on I decided to let the user select the
31 fletcher32 checksum provided in HDF5 1.6 or higher. So, even though
32 the CHECKSUM support here seems pretty stable it will be disabled.
33 F. Alted 2004/01/02 */
34 #undef CHECKSUM
35
36 size_t lzo_deflate (unsigned flags, size_t cd_nelmts,
37 const unsigned cd_values[], size_t nbytes,
38 size_t *buf_size, void **buf);
39
40
register_lzo(char ** version,char ** date)41 int register_lzo(char **version, char **date) {
42
43 #ifdef HAVE_LZO_LIB
44
45 H5Z_class_t filter_class = {
46 H5Z_CLASS_T_VERS, /* H5Z_class_t version */
47 (H5Z_filter_t)(FILTER_LZO), /* filter_id */
48 1, 1, /* Encoding and decoding enabled */
49 "lzo", /* comment */
50 NULL, /* can_apply_func */
51 NULL, /* set_local_func */
52 (H5Z_func_t)(lzo_deflate) /* filter_func */
53 };
54
55 /* Init the LZO library */
56 if (lzo_init()!=LZO_E_OK) {
57 fprintf(stderr, "Problems initializing LZO library\n");
58 *version = NULL;
59 *date = NULL;
60 return 0; /* lib is not available */
61 }
62
63 /* Register the lzo compressor */
64 H5Zregister(&filter_class);
65
66 *version = strdup(LZO_VERSION_STRING);
67 *date = strdup(LZO_VERSION_DATE);
68 return 1; /* lib is available */
69
70 #else
71 *version = NULL;
72 *date = NULL;
73 return 0; /* lib is not available */
74 #endif /* HAVE_LZO_LIB */
75
76 }
77
78
lzo_deflate(unsigned flags,size_t cd_nelmts,const unsigned cd_values[],size_t nbytes,size_t * buf_size,void ** buf)79 size_t lzo_deflate (unsigned flags, size_t cd_nelmts,
80 const unsigned cd_values[], size_t nbytes,
81 size_t *buf_size, void **buf)
82 {
83 size_t ret_value = 0;
84 #ifdef HAVE_LZO_LIB
85 void *outbuf = NULL, *wrkmem = NULL;
86 int status;
87 size_t nalloc = *buf_size;
88 lzo_uint out_len = (lzo_uint) nalloc;
89 /* max_len_buffer will keep the likely output buffer size
90 after processing the first chunk */
91 static unsigned int max_len_buffer = 0;
92 /* int complevel = 1; */
93 #if (defined CHECKSUM || defined DEBUG)
94 int object_version = 10; /* Default version 1.0 */
95 int object_type = Table; /* Default object type */
96 #endif
97 #ifdef CHECKSUM
98 lzo_uint32 checksum;
99 #endif
100
101 /* Check arguments */
102 /* For Table versions < 20, there were no parameters */
103 if (cd_nelmts==1 ) {
104 /* complevel = cd_values[0]; */ /* This do nothing right now */
105 }
106 else if (cd_nelmts==2 ) {
107 /* complevel = cd_values[0]; */ /* This do nothing right now */
108 #if (defined CHECKSUM || defined DEBUG)
109 object_version = cd_values[1]; /* The table VERSION attribute */
110 #endif
111 }
112 else if (cd_nelmts==3 ) {
113 /* complevel = cd_values[0]; */ /* This do nothing right now */
114 #if (defined CHECKSUM || defined DEBUG)
115 object_version = cd_values[1]; /* The table VERSION attribute */
116 object_type = cd_values[2]; /* A tag for identifying the object
117 (see tables.h) */
118 #endif
119 }
120
121 #ifdef DEBUG
122 printf("Object type: %d. ", object_type);
123 printf("object_version:%d\n", object_version);
124 #endif
125
126 if (flags & H5Z_FLAG_REVERSE) {
127 /* Input */
128
129 /* printf("Decompressing chunk with LZO\n"); */
130 #ifdef CHECKSUM
131 if ((object_type == Table && object_version >= 20) ||
132 object_type != Table) {
133 nbytes -= 4; /* Point to uncompressed buffer length */
134 memcpy(&nalloc, ((unsigned char *)(*buf)+nbytes), 4);
135 out_len = nalloc;
136 nbytes -= 4; /* Point to the checksum */
137 #ifdef DEBUG
138 printf("Compressed bytes: %d. Uncompressed bytes: %d\n", nbytes, nalloc);
139 #endif
140 }
141 #endif
142
143 /* Only allocate the bytes for the outbuf */
144 if (max_len_buffer == 0) {
145 if (NULL==(outbuf = (void *)malloc(nalloc)))
146 fprintf(stderr, "Memory allocation failed for lzo uncompression.\n");
147 }
148 else {
149 if (NULL==(outbuf = (void *)malloc(max_len_buffer)))
150 fprintf(stderr, "Memory allocation failed for lzo uncompression.\n");
151 out_len = max_len_buffer;
152 nalloc = max_len_buffer;
153 }
154
155 while(1) {
156
157 #ifdef DEBUG
158 printf("nbytes -->%d\n", nbytes);
159 printf("nalloc -->%d\n", nalloc);
160 printf("max_len_buffer -->%d\n", max_len_buffer);
161 #endif /* DEBUG */
162
163 /* The assembler version is a 10% slower than the C version with
164 gcc 3.2.2 and gcc 3.3.3 */
165 /* status = lzo1x_decompress_asm_safe(*buf, (lzo_uint)nbytes, outbuf, */
166 /* &out_len, NULL); */
167 /* The safe and unsafe versions have the same speed more or less */
168 status = lzo1x_decompress_safe(*buf, (lzo_uint)nbytes, outbuf,
169 &out_len, NULL);
170
171 if (status == LZO_E_OK) {
172 #ifdef DEBUG
173 printf("decompressed %lu bytes back into %lu bytes\n",
174 (long) nbytes, (long) out_len);
175 #endif
176 max_len_buffer = out_len;
177 break; /* done */
178 }
179 else if (status == LZO_E_OUTPUT_OVERRUN) {
180 nalloc *= 2;
181 out_len = (lzo_uint) nalloc;
182 if (NULL==(outbuf = realloc(outbuf, nalloc))) {
183 fprintf(stderr, "Memory allocation failed for lzo uncompression\n");
184 }
185 }
186 else {
187 /* this should NEVER happen */
188 fprintf(stderr, "internal error - decompression failed: %d\n", status);
189 ret_value = 0; /* fail */
190 goto done;
191 }
192 }
193
194 #ifdef CHECKSUM
195 if ((object_type == Table && object_version >= 20) ||
196 object_type != Table) {
197 #ifdef DEBUG
198 printf("Checksum uncompressing...");
199 #endif
200 /* Compute the checksum */
201 checksum=lzo_adler32(lzo_adler32(0,NULL,0), outbuf, out_len);
202
203 /* Compare */
204 if (memcmp(&checksum, (unsigned char*)(*buf)+nbytes, 4)) {
205 ret_value = 0; /*fail*/
206 fprintf(stderr,"Checksum failed!.\n");
207 goto done;
208 }
209 }
210 #endif /* CHECKSUM */
211
212 free(*buf);
213 *buf = outbuf;
214 outbuf = NULL;
215 *buf_size = nalloc;
216 ret_value = out_len;
217
218 } else {
219 /*
220 * Output; compress but fail if the result would be larger than the
221 * input. The library doesn't provide in-place compression, so we
222 * must allocate a separate buffer for the result.
223 */
224 lzo_byte *z_src = (lzo_byte*)(*buf);
225 lzo_byte *z_dst; /*destination buffer */
226 lzo_uint z_src_nbytes = (lzo_uint)(nbytes);
227 /* The next was the original computation for worst-case expansion */
228 /* I don't know why the difference with LZO1*. Perhaps some wrong docs in
229 LZO package? */
230 /* lzo_uint z_dst_nbytes = (lzo_uint)(nbytes + (nbytes / 64) + 16 + 3); */
231 /* The next is for LZO1* algorithms */
232 /* lzo_uint z_dst_nbytes = (lzo_uint)(nbytes + (nbytes / 16) + 64 + 3); */
233 /* The next is for LZO2* algorithms. This will be the default */
234 lzo_uint z_dst_nbytes = (lzo_uint)(nbytes + (nbytes / 8) + 128 + 3);
235
236 #ifdef CHECKSUM
237 if ((object_type == Table && object_version >= 20) ||
238 object_type != Table) {
239 z_dst_nbytes += 4+4; /* Checksum + buffer size */
240 }
241 #endif
242
243 if (NULL==(z_dst=outbuf=(void *)malloc(z_dst_nbytes))) {
244 fprintf(stderr, "Unable to allocate lzo destination buffer.\n");
245 ret_value = 0; /* fail */
246 goto done;
247 }
248
249 /* Compress this buffer */
250 wrkmem = malloc(LZO1X_1_MEM_COMPRESS);
251 if (wrkmem == NULL) {
252 fprintf(stderr, "Memory allocation failed for lzo compression\n");
253 ret_value = 0;
254 goto done;
255 }
256
257 status = lzo1x_1_compress (z_src, z_src_nbytes, z_dst, &z_dst_nbytes,
258 wrkmem);
259
260 free(wrkmem);
261 wrkmem = NULL;
262
263 #ifdef CHECKSUM
264 if ((object_type == Table && object_version >= 20) ||
265 object_type != Table) {
266 #ifdef DEBUG
267 printf("Checksum compressing ...");
268 printf("src_nbytes: %d, dst_nbytes: %d\n", z_src_nbytes, z_dst_nbytes);
269 #endif
270 /* Append checksum of *uncompressed* data at the end */
271 checksum = lzo_adler32(lzo_adler32(0,NULL,0), *buf, nbytes);
272 memcpy((unsigned char*)(z_dst)+z_dst_nbytes, &checksum, 4);
273 memcpy((unsigned char*)(z_dst)+z_dst_nbytes+4, &nbytes, 4);
274 z_dst_nbytes += (lzo_uint)4+4;
275 nbytes += 4+4;
276 }
277 #endif
278
279 if (z_dst_nbytes >= nbytes) {
280 #ifdef DEBUG
281 printf("The compressed buffer takes more space than uncompressed!.\n");
282 #endif
283 ret_value = 0; /* fail */
284 goto done;
285 } else if (LZO_E_OK != status) {
286 fprintf(stderr,"lzo library error in compression\n");
287 ret_value = 0; /* fail */
288 goto done;
289 } else {
290 free(*buf);
291 *buf = outbuf;
292 outbuf = NULL;
293 *buf_size = z_dst_nbytes;
294 ret_value = z_dst_nbytes;
295 }
296 }
297
298 done:
299 if(outbuf)
300 free(outbuf);
301
302 #endif /* HAVE_LZO_LIB */
303
304 return ret_value;
305 }
306