1 /*
2
3 Copyright 2016 Christian Hoene, Symonics GmbH
4
5 */
6
7 #include "reader.h"
8 #include <errno.h>
9 #include <inttypes.h>
10 #include <stdlib.h>
11 #include <string.h>
12
13 /*
14 *
15 00000370 42 54 4c 46 00 08 00 5b 01 00 00 00 2d 00 00 07 |BTLF...[....-...|
16 00000380 00 00 00 f8 ea 72 15 00 c9 03 00 00 00 26 00 00 |.....r.......&..|
17 00000390 14 00 00 00 32 32 7c 17 00 22 02 00 00 00 32 00 |....22|.."....2.|
18 000003a0 00 0b 00 00 00 07 ef 9c 26 00 bb 01 00 00 00 46 |........&......F|
19 000003b0 00 00 09 00 00 00 e5 f6 ba 26 00 45 03 00 00 00 |.........&.E....|
20 000003c0 34 00 00 11 00 00 00 f6 71 f0 2e 00 a3 02 00 00 |4.......q.......|
21 000003d0 00 3e 00 00 0d 00 00 00 61 36 dc 36 00 79 03 00 |.>......a6.6.y..|
22 000003e0 00 00 35 00 00 12 00 00 00 97 1b 4e 45 00 88 01 |..5........NE...|
23 000003f0 00 00 00 33 00 00 08 00 00 00 56 d7 d0 47 00 ae |...3......V..G..|
24 00000400 03 00 00 00 1b 00 00 13 00 00 00 2f 03 50 5a 00 |.........../.PZ.|
25 00000410 22 01 00 00 00 39 00 00 06 00 00 00 b7 88 37 66 |"....9........7f|
26 00000420 00 01 03 00 00 00 28 00 00 0f 00 00 00 dc aa 47 |......(........G|
27 00000430 66 00 16 04 00 00 00 2c 00 00 15 00 00 00 6b 54 |f......,......kT|
28 00000440 7d 77 00 fd 00 00 00 00 25 00 00 05 00 00 00 7d |}w......%......}|
29 00000450 0c 8c 9e 00 29 03 00 00 00 1c 00 00 10 00 00 00 |....)...........|
30 00000460 4c f3 0e a0 00 16 00 00 00 00 25 00 00 00 00 00 |L.........%.....|
31 00000470 00 e7 30 2d ab 00 01 02 00 00 00 21 00 00 0a 00 |..0-.......!....|
32 00000480 00 00 35 b5 69 b0 00 e1 02 00 00 00 20 00 00 0e |..5.i....... ...|
33 00000490 00 00 00 2b c5 8b c4 00 3b 00 00 00 00 20 00 00 |...+....;.... ..|
34 000004a0 01 00 00 00 09 a0 74 cc 00 93 00 00 00 00 2f 00 |......t......./.|
35 000004b0 00 03 00 00 00 3f 48 ef d6 00 5b 00 00 00 00 38 |.....?H...[....8|
36 000004c0 00 00 02 00 00 00 f1 7e 7d dd 00 54 02 00 00 00 |.......~}..T....|
37 000004d0 4f 00 00 0c 00 00 00 48 35 ff f5 00 c2 00 00 00 |O......H5.......|
38 000004e0 00 3b 00 00 04 00 00 00 ad 61 4e ff 63 42 f7 73 |.;.......aN.cB.s|
39 000004f0 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................|
40 *
41
42 00000570 42 54 4c 46 00 09 00 16 00 00 00 00 25 00 00 00 |BTLF........%...|
43 00000580 00 00 00 00 3b 00 00 00 00 20 00 00 01 00 00 00 |....;.... ......|
44 00000590 00 5b 00 00 00 00 38 00 00 02 00 00 00 00 93 00 |.[....8.........|
45 000005a0 00 00 00 2f 00 00 03 00 00 00 00 c2 00 00 00 00 |.../............|
46 000005b0 3b 00 00 04 00 00 00 00 fd 00 00 00 00 25 00 00 |;............%..|
47 000005c0 05 00 00 00 00 22 01 00 00 00 39 00 00 06 00 00 |....."....9.....|
48 000005d0 00 00 5b 01 00 00 00 2d 00 00 07 00 00 00 00 88 |..[....-........|
49 000005e0 01 00 00 00 33 00 00 08 00 00 00 00 bb 01 00 00 |....3...........|
50 000005f0 00 46 00 00 09 00 00 00 00 01 02 00 00 00 21 00 |.F............!.|
51 00000600 00 0a 00 00 00 00 22 02 00 00 00 32 00 00 0b 00 |......"....2....|
52 00000610 00 00 00 54 02 00 00 00 4f 00 00 0c 00 00 00 00 |...T....O.......|
53 00000620 a3 02 00 00 00 3e 00 00 0d 00 00 00 00 e1 02 00 |.....>..........|
54 00000630 00 00 20 00 00 0e 00 00 00 00 01 03 00 00 00 28 |.. ............(|
55 00000640 00 00 0f 00 00 00 00 29 03 00 00 00 1c 00 00 10 |.......)........|
56 00000650 00 00 00 00 45 03 00 00 00 34 00 00 11 00 00 00 |....E....4......|
57 00000660 00 79 03 00 00 00 35 00 00 12 00 00 00 00 ae 03 |.y....5.........|
58 00000670 00 00 00 1b 00 00 13 00 00 00 00 c9 03 00 00 00 |................|
59 00000680 26 00 00 14 00 00 00 00 16 04 00 00 00 2c 00 00 |&............,..|
60 00000690 15 00 00 00 d3 c7 19 a0 00 00 00 00 00 00 00 00 |................|
61 000006a0 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................|
62
63 */
64
65 // LCOV_EXCL_START
66
readBTLF(struct READER * reader,struct BTREE * btree,int number_of_records,union RECORD * records)67 static int readBTLF(struct READER *reader, struct BTREE *btree,
68 int number_of_records, union RECORD *records) {
69
70 int i;
71
72 uint8_t type, message_flags;
73 uint32_t creation_order, hash_of_name;
74 uint64_t heap_id;
75
76 char buf[5];
77
78 UNUSED(heap_id);
79 UNUSED(hash_of_name);
80 UNUSED(creation_order);
81 UNUSED(message_flags);
82
83 /* read signature */
84 if (fread(buf, 1, 4, reader->fhd) != 4 || strncmp(buf, "BTLF", 4)) {
85 mylog("cannot read signature of BTLF\n"); // LCOV_EXCL_LINE
86 return MYSOFA_INVALID_FORMAT; // LCOV_EXCL_LINE
87 }
88 buf[4] = 0;
89 mylog("%08" PRIX64 " %.4s\n", (uint64_t)ftell(reader->fhd) - 4, buf);
90
91 if (fgetc(reader->fhd) != 0) {
92 mylog("object BTLF must have version 0\n"); // LCOV_EXCL_LINE
93 return MYSOFA_INVALID_FORMAT; // LCOV_EXCL_LINE
94 }
95
96 type = (uint8_t)fgetc(reader->fhd);
97
98 for (i = 0; i < number_of_records; i++) {
99
100 switch (type) {
101 case 5:
102 records->type5.hash_of_name = (uint32_t)readValue(reader, 4);
103 records->type5.heap_id = readValue(reader, 7);
104 mylog(" type5 %08X %14" PRIX64 "\n", records->type5.hash_of_name,
105 records->type5.heap_id);
106 records++;
107 break;
108
109 case 6:
110 /*creation_order = */
111 readValue(reader, 8);
112 /*heap_id = */
113 readValue(reader, 7);
114 break;
115
116 case 8:
117 /*heap_id = */
118 readValue(reader, 8);
119 /*message_flags = */
120 fgetc(reader->fhd);
121 /*creation_order = */
122 readValue(reader, 4);
123 /*hash_of_name = */
124 readValue(reader, 4);
125 break;
126
127 case 9:
128 /*heap_id = */
129 readValue(reader, 8);
130 /*message_flags = */
131 fgetc(reader->fhd);
132 /*creation_order = */
133 readValue(reader, 4);
134 break;
135
136 default:
137 mylog("object BTLF has unknown type %d\n", type);
138 return MYSOFA_INVALID_FORMAT;
139 }
140 }
141
142 /* fseeko(reader->fhd, bthd->root_node_address + bthd->node_size,
143 * SEEK_SET); skip checksum */
144
145 return MYSOFA_OK;
146 }
147
148 /* III.A.2. Disk Format: Level 1A2 - Version 2 B-trees
149
150 000002d0 32 1d 42 54 48 44 00 08 00 02 00 00 11 00 00 00 |2.BTHD..........|
151 000002e0 64 28 70 03 00 00 00 00 00 00 16 00 16 00 00 00 |d(p.............|
152 000002f0 00 00 00 00 30 12 d9 6e 42 54 48 44 00 09 00 02 |....0..nBTHD....|
153 00000300 00 00 0d 00 00 00 64 28 70 05 00 00 00 00 00 00 |......d(p.......|
154 00000310 16 00 16 00 00 00 00 00 00 00 e2 0d 76 5c 46 53 |............v\FS|
155
156 */
157
btreeRead(struct READER * reader,struct BTREE * btree)158 int btreeRead(struct READER *reader, struct BTREE *btree) {
159 char buf[5];
160
161 /* read signature */
162 if (fread(buf, 1, 4, reader->fhd) != 4 || strncmp(buf, "BTHD", 4)) {
163 mylog("cannot read signature of BTHD\n");
164 return MYSOFA_INVALID_FORMAT;
165 }
166 buf[4] = 0;
167 mylog("%08" PRIX64 " %.4s\n", (uint64_t)ftell(reader->fhd) - 4, buf);
168
169 if (fgetc(reader->fhd) != 0) {
170 mylog("object BTHD must have version 0\n");
171 return MYSOFA_INVALID_FORMAT;
172 }
173
174 btree->type = (uint8_t)fgetc(reader->fhd);
175 btree->node_size = (uint32_t)readValue(reader, 4);
176 btree->record_size = (uint16_t)readValue(reader, 2);
177 btree->depth = (uint16_t)readValue(reader, 2);
178
179 btree->split_percent = (uint8_t)fgetc(reader->fhd);
180 btree->merge_percent = (uint8_t)fgetc(reader->fhd);
181 btree->root_node_address =
182 (uint64_t)readValue(reader, reader->superblock.size_of_offsets);
183 btree->number_of_records = (uint16_t)readValue(reader, 2);
184 if (btree->number_of_records > 0x1000)
185 return MYSOFA_UNSUPPORTED_FORMAT;
186 btree->total_number =
187 (uint64_t)readValue(reader, reader->superblock.size_of_lengths);
188
189 /* fseek(reader->fhd, 4, SEEK_CUR); skip checksum */
190
191 if (btree->total_number > 0x10000000)
192 return MYSOFA_NO_MEMORY;
193 btree->records = malloc(sizeof(btree->records[0]) * btree->total_number);
194 if (!btree->records)
195 return MYSOFA_NO_MEMORY;
196 memset(btree->records, 0, sizeof(btree->records[0]) * btree->total_number);
197
198 /* read records */
199 if (fseek(reader->fhd, btree->root_node_address, SEEK_SET) < 0)
200 return errno;
201 return readBTLF(reader, btree, btree->number_of_records, btree->records);
202 }
203
204 // LCOV_EXCL_STOP
205
btreeFree(struct BTREE * btree)206 void btreeFree(struct BTREE *btree) { free(btree->records); }
207
208 /* III.A.1. Disk Format: Level 1A1 - Version 1 B-trees
209 *
210 */
211
treeRead(struct READER * reader,struct DATAOBJECT * data)212 int treeRead(struct READER *reader, struct DATAOBJECT *data) {
213
214 int i, j, err, olen, elements, size, x, y, z, b, e, dy, dz, sx, sy, sz, dzy,
215 szy;
216 char *input, *output;
217
218 uint8_t node_type, node_level;
219 uint16_t entries_used;
220 uint32_t size_of_chunk;
221 uint32_t filter_mask;
222 uint64_t address_of_left_sibling, address_of_right_sibling, start[4],
223 child_pointer, key, store;
224
225 char buf[5];
226
227 UNUSED(node_level);
228 UNUSED(address_of_right_sibling);
229 UNUSED(address_of_left_sibling);
230 UNUSED(key);
231
232 if (data->ds.dimensionality > 3) {
233 mylog("TREE dimensions > 3"); // LCOV_EXCL_LINE
234 return MYSOFA_INVALID_FORMAT; // LCOV_EXCL_LINE
235 }
236
237 /* read signature */
238 if (fread(buf, 1, 4, reader->fhd) != 4 || strncmp(buf, "TREE", 4)) {
239 mylog("cannot read signature of TREE\n"); // LCOV_EXCL_LINE
240 return MYSOFA_INVALID_FORMAT; // LCOV_EXCL_LINE
241 }
242 buf[4] = 0;
243 mylog("%08" PRIX64 " %.4s\n", (uint64_t)ftell(reader->fhd) - 4, buf);
244
245 node_type = (uint8_t)fgetc(reader->fhd);
246 node_level = (uint8_t)fgetc(reader->fhd);
247 entries_used = (uint16_t)readValue(reader, 2);
248 if (entries_used > 0x1000)
249 return MYSOFA_UNSUPPORTED_FORMAT; // LCOV_EXCL_LINE
250 address_of_left_sibling =
251 readValue(reader, reader->superblock.size_of_offsets);
252 address_of_right_sibling =
253 readValue(reader, reader->superblock.size_of_offsets);
254
255 elements = 1;
256 for (j = 0; j < data->ds.dimensionality; j++)
257 elements *= data->datalayout_chunk[j];
258 dy = data->datalayout_chunk[1];
259 dz = data->datalayout_chunk[2];
260 sx = data->ds.dimension_size[0];
261 sy = data->ds.dimension_size[1];
262 sz = data->ds.dimension_size[2];
263 dzy = dz * dy;
264 szy = sz * sy;
265 size = data->datalayout_chunk[data->ds.dimensionality];
266
267 mylog("elements %d size %d\n", elements, size);
268
269 if (elements <= 0 || size <= 0 || elements >= 0x100000 || size > 0x10)
270 return MYSOFA_INVALID_FORMAT; // LCOV_EXCL_LINE
271 if (!(output = malloc(elements * size))) {
272 return MYSOFA_NO_MEMORY; // LCOV_EXCL_LINE
273 }
274
275 for (e = 0; e < entries_used * 2; e++) {
276 if (node_type == 0) {
277 key = readValue(reader, reader->superblock.size_of_lengths);
278 } else {
279 size_of_chunk = (uint32_t)readValue(reader, 4);
280 filter_mask = (uint32_t)readValue(reader, 4);
281 if (filter_mask) {
282 mylog("TREE all filters must be enabled\n"); // LCOV_EXCL_LINE
283 free(output); // LCOV_EXCL_LINE
284 return MYSOFA_INVALID_FORMAT; // LCOV_EXCL_LINE
285 }
286
287 for (j = 0; j < data->ds.dimensionality; j++) {
288 start[j] = readValue(reader, 8);
289 mylog("start %d %" PRIu64 "\n", j, start[j]);
290 }
291
292 if (readValue(reader, 8)) {
293 break;
294 }
295
296 child_pointer = readValue(reader, reader->superblock.size_of_offsets);
297 mylog(" data at %" PRIX64 " len %u\n", child_pointer, size_of_chunk);
298
299 /* read data */
300 store = ftell(reader->fhd);
301 if (fseek(reader->fhd, child_pointer, SEEK_SET) < 0) {
302 free(output); // LCOV_EXCL_LINE
303 return errno; // LCOV_EXCL_LINE
304 }
305
306 if (!(input = malloc(size_of_chunk))) {
307 free(output); // LCOV_EXCL_LINE
308 return MYSOFA_NO_MEMORY; // LCOV_EXCL_LINE
309 }
310 if (fread(input, 1, size_of_chunk, reader->fhd) != size_of_chunk) {
311 free(output); // LCOV_EXCL_LINE
312 free(input); // LCOV_EXCL_LINE
313 return MYSOFA_INVALID_FORMAT; // LCOV_EXCL_LINE
314 }
315
316 olen = elements * size;
317 err = gunzip(size_of_chunk, input, &olen, output);
318 free(input);
319
320 mylog(" gunzip %d %d %d\n", err, olen, elements * size);
321 if (err || olen != elements * size) {
322 free(output); // LCOV_EXCL_LINE
323 return MYSOFA_INVALID_FORMAT; // LCOV_EXCL_LINE
324 }
325
326 switch (data->ds.dimensionality) {
327 case 1:
328 for (i = 0; i < olen; i++) {
329 b = i / elements;
330 x = i % elements + start[0];
331 if (x < sx) {
332
333 j = x * size + b;
334 if (j >= 0 && j < data->data_len) {
335 ((char *)data->data)[j] = output[i];
336 }
337 }
338 }
339 break;
340 case 2:
341 for (i = 0; i < olen; i++) {
342 b = i / elements;
343 x = i % elements;
344 y = x % dy + start[1];
345 x = x / dy + start[0];
346 if (y < sy && x < sx) {
347 j = ((x * sy + y) * size) + b;
348 if (j >= 0 && j < data->data_len) {
349 ((char *)data->data)[j] = output[i];
350 }
351 }
352 }
353 break;
354 case 3:
355 for (i = 0; i < olen; i++) {
356 b = i / elements;
357 x = i % elements;
358 z = x % dz + start[2];
359 y = (x / dz) % dy + start[1];
360 x = (x / dzy) + start[0];
361 if (z < sz && y < sy && x < sx) {
362 j = (x * szy + y * sz + z) * size + b;
363 if (j >= 0 && j < data->data_len) {
364 ((char *)data->data)[j] = output[i];
365 }
366 }
367 }
368 break;
369 default:
370 mylog("invalid dim\n"); // LCOV_EXCL_LINE
371 return MYSOFA_INTERNAL_ERROR; // LCOV_EXCL_LINE
372 }
373
374 if (fseek(reader->fhd, store, SEEK_SET) < 0) {
375 free(output); // LCOV_EXCL_LINE
376 return errno; // LCOV_EXCL_LINE
377 }
378 }
379 }
380
381 free(output);
382 if (fseek(reader->fhd, 4, SEEK_CUR) < 0) /* skip checksum */
383 return errno; // LCOV_EXCL_LINE
384
385 return MYSOFA_OK;
386 }
387