1 /*
2 
3  Copyright 2016 Christian Hoene, Symonics GmbH
4 
5  */
6 
7 #include "reader.h"
8 #include <errno.h>
9 #include <inttypes.h>
10 #include <stdlib.h>
11 #include <string.h>
12 
13 /*
14  *
15  00000370  42 54 4c 46 00 08 00 5b  01 00 00 00 2d 00 00 07  |BTLF...[....-...|
16  00000380  00 00 00 f8 ea 72 15 00  c9 03 00 00 00 26 00 00  |.....r.......&..|
17  00000390  14 00 00 00 32 32 7c 17  00 22 02 00 00 00 32 00  |....22|.."....2.|
18  000003a0  00 0b 00 00 00 07 ef 9c  26 00 bb 01 00 00 00 46  |........&......F|
19  000003b0  00 00 09 00 00 00 e5 f6  ba 26 00 45 03 00 00 00  |.........&.E....|
20  000003c0  34 00 00 11 00 00 00 f6  71 f0 2e 00 a3 02 00 00  |4.......q.......|
21  000003d0  00 3e 00 00 0d 00 00 00  61 36 dc 36 00 79 03 00  |.>......a6.6.y..|
22  000003e0  00 00 35 00 00 12 00 00  00 97 1b 4e 45 00 88 01  |..5........NE...|
23  000003f0  00 00 00 33 00 00 08 00  00 00 56 d7 d0 47 00 ae  |...3......V..G..|
24  00000400  03 00 00 00 1b 00 00 13  00 00 00 2f 03 50 5a 00  |.........../.PZ.|
25  00000410  22 01 00 00 00 39 00 00  06 00 00 00 b7 88 37 66  |"....9........7f|
26  00000420  00 01 03 00 00 00 28 00  00 0f 00 00 00 dc aa 47  |......(........G|
27  00000430  66 00 16 04 00 00 00 2c  00 00 15 00 00 00 6b 54  |f......,......kT|
28  00000440  7d 77 00 fd 00 00 00 00  25 00 00 05 00 00 00 7d  |}w......%......}|
29  00000450  0c 8c 9e 00 29 03 00 00  00 1c 00 00 10 00 00 00  |....)...........|
30  00000460  4c f3 0e a0 00 16 00 00  00 00 25 00 00 00 00 00  |L.........%.....|
31  00000470  00 e7 30 2d ab 00 01 02  00 00 00 21 00 00 0a 00  |..0-.......!....|
32  00000480  00 00 35 b5 69 b0 00 e1  02 00 00 00 20 00 00 0e  |..5.i....... ...|
33  00000490  00 00 00 2b c5 8b c4 00  3b 00 00 00 00 20 00 00  |...+....;.... ..|
34  000004a0  01 00 00 00 09 a0 74 cc  00 93 00 00 00 00 2f 00  |......t......./.|
35  000004b0  00 03 00 00 00 3f 48 ef  d6 00 5b 00 00 00 00 38  |.....?H...[....8|
36  000004c0  00 00 02 00 00 00 f1 7e  7d dd 00 54 02 00 00 00  |.......~}..T....|
37  000004d0  4f 00 00 0c 00 00 00 48  35 ff f5 00 c2 00 00 00  |O......H5.......|
38  000004e0  00 3b 00 00 04 00 00 00  ad 61 4e ff 63 42 f7 73  |.;.......aN.cB.s|
39  000004f0  00 00 00 00 00 00 00 00  00 00 00 00 00 00 00 00  |................|
40  *
41 
42  00000570  42 54 4c 46 00 09 00 16  00 00 00 00 25 00 00 00  |BTLF........%...|
43  00000580  00 00 00 00 3b 00 00 00  00 20 00 00 01 00 00 00  |....;.... ......|
44  00000590  00 5b 00 00 00 00 38 00  00 02 00 00 00 00 93 00  |.[....8.........|
45  000005a0  00 00 00 2f 00 00 03 00  00 00 00 c2 00 00 00 00  |.../............|
46  000005b0  3b 00 00 04 00 00 00 00  fd 00 00 00 00 25 00 00  |;............%..|
47  000005c0  05 00 00 00 00 22 01 00  00 00 39 00 00 06 00 00  |....."....9.....|
48  000005d0  00 00 5b 01 00 00 00 2d  00 00 07 00 00 00 00 88  |..[....-........|
49  000005e0  01 00 00 00 33 00 00 08  00 00 00 00 bb 01 00 00  |....3...........|
50  000005f0  00 46 00 00 09 00 00 00  00 01 02 00 00 00 21 00  |.F............!.|
51  00000600  00 0a 00 00 00 00 22 02  00 00 00 32 00 00 0b 00  |......"....2....|
52  00000610  00 00 00 54 02 00 00 00  4f 00 00 0c 00 00 00 00  |...T....O.......|
53  00000620  a3 02 00 00 00 3e 00 00  0d 00 00 00 00 e1 02 00  |.....>..........|
54  00000630  00 00 20 00 00 0e 00 00  00 00 01 03 00 00 00 28  |.. ............(|
55  00000640  00 00 0f 00 00 00 00 29  03 00 00 00 1c 00 00 10  |.......)........|
56  00000650  00 00 00 00 45 03 00 00  00 34 00 00 11 00 00 00  |....E....4......|
57  00000660  00 79 03 00 00 00 35 00  00 12 00 00 00 00 ae 03  |.y....5.........|
58  00000670  00 00 00 1b 00 00 13 00  00 00 00 c9 03 00 00 00  |................|
59  00000680  26 00 00 14 00 00 00 00  16 04 00 00 00 2c 00 00  |&............,..|
60  00000690  15 00 00 00 d3 c7 19 a0  00 00 00 00 00 00 00 00  |................|
61  000006a0  00 00 00 00 00 00 00 00  00 00 00 00 00 00 00 00  |................|
62 
63  */
64 
65 // LCOV_EXCL_START
66 
readBTLF(struct READER * reader,struct BTREE * btree,int number_of_records,union RECORD * records)67 static int readBTLF(struct READER *reader, struct BTREE *btree,
68                     int number_of_records, union RECORD *records) {
69 
70   int i;
71 
72   uint8_t type, message_flags;
73   uint32_t creation_order, hash_of_name;
74   uint64_t heap_id;
75 
76   char buf[5];
77 
78   UNUSED(heap_id);
79   UNUSED(hash_of_name);
80   UNUSED(creation_order);
81   UNUSED(message_flags);
82 
83   /* read signature */
84   if (fread(buf, 1, 4, reader->fhd) != 4 || strncmp(buf, "BTLF", 4)) {
85     mylog("cannot read signature of BTLF\n"); // LCOV_EXCL_LINE
86     return MYSOFA_INVALID_FORMAT;             // LCOV_EXCL_LINE
87   }
88   buf[4] = 0;
89   mylog("%08" PRIX64 " %.4s\n", (uint64_t)ftell(reader->fhd) - 4, buf);
90 
91   if (fgetc(reader->fhd) != 0) {
92     mylog("object BTLF must have version 0\n"); // LCOV_EXCL_LINE
93     return MYSOFA_INVALID_FORMAT;               // LCOV_EXCL_LINE
94   }
95 
96   type = (uint8_t)fgetc(reader->fhd);
97 
98   for (i = 0; i < number_of_records; i++) {
99 
100     switch (type) {
101     case 5:
102       records->type5.hash_of_name = (uint32_t)readValue(reader, 4);
103       records->type5.heap_id = readValue(reader, 7);
104       mylog(" type5 %08X %14" PRIX64 "\n", records->type5.hash_of_name,
105             records->type5.heap_id);
106       records++;
107       break;
108 
109     case 6:
110       /*creation_order = */
111       readValue(reader, 8);
112       /*heap_id = */
113       readValue(reader, 7);
114       break;
115 
116     case 8:
117       /*heap_id = */
118       readValue(reader, 8);
119       /*message_flags = */
120       fgetc(reader->fhd);
121       /*creation_order = */
122       readValue(reader, 4);
123       /*hash_of_name = */
124       readValue(reader, 4);
125       break;
126 
127     case 9:
128       /*heap_id = */
129       readValue(reader, 8);
130       /*message_flags = */
131       fgetc(reader->fhd);
132       /*creation_order = */
133       readValue(reader, 4);
134       break;
135 
136     default:
137       mylog("object BTLF has unknown type %d\n", type);
138       return MYSOFA_INVALID_FORMAT;
139     }
140   }
141 
142   /*	fseeko(reader->fhd, bthd->root_node_address + bthd->node_size,
143    * SEEK_SET); skip checksum */
144 
145   return MYSOFA_OK;
146 }
147 
148 /*  III.A.2. Disk Format: Level 1A2 - Version 2 B-trees
149 
150  000002d0  32 1d 42 54 48 44 00 08  00 02 00 00 11 00 00 00  |2.BTHD..........|
151  000002e0  64 28 70 03 00 00 00 00  00 00 16 00 16 00 00 00  |d(p.............|
152  000002f0  00 00 00 00 30 12 d9 6e  42 54 48 44 00 09 00 02  |....0..nBTHD....|
153  00000300  00 00 0d 00 00 00 64 28  70 05 00 00 00 00 00 00  |......d(p.......|
154  00000310  16 00 16 00 00 00 00 00  00 00 e2 0d 76 5c 46 53  |............v\FS|
155 
156  */
157 
btreeRead(struct READER * reader,struct BTREE * btree)158 int btreeRead(struct READER *reader, struct BTREE *btree) {
159   char buf[5];
160 
161   /* read signature */
162   if (fread(buf, 1, 4, reader->fhd) != 4 || strncmp(buf, "BTHD", 4)) {
163     mylog("cannot read signature of BTHD\n");
164     return MYSOFA_INVALID_FORMAT;
165   }
166   buf[4] = 0;
167   mylog("%08" PRIX64 " %.4s\n", (uint64_t)ftell(reader->fhd) - 4, buf);
168 
169   if (fgetc(reader->fhd) != 0) {
170     mylog("object BTHD must have version 0\n");
171     return MYSOFA_INVALID_FORMAT;
172   }
173 
174   btree->type = (uint8_t)fgetc(reader->fhd);
175   btree->node_size = (uint32_t)readValue(reader, 4);
176   btree->record_size = (uint16_t)readValue(reader, 2);
177   btree->depth = (uint16_t)readValue(reader, 2);
178 
179   btree->split_percent = (uint8_t)fgetc(reader->fhd);
180   btree->merge_percent = (uint8_t)fgetc(reader->fhd);
181   btree->root_node_address =
182       (uint64_t)readValue(reader, reader->superblock.size_of_offsets);
183   btree->number_of_records = (uint16_t)readValue(reader, 2);
184   if (btree->number_of_records > 0x1000)
185     return MYSOFA_UNSUPPORTED_FORMAT;
186   btree->total_number =
187       (uint64_t)readValue(reader, reader->superblock.size_of_lengths);
188 
189   /*	fseek(reader->fhd, 4, SEEK_CUR);  skip checksum */
190 
191   if (btree->total_number > 0x10000000)
192     return MYSOFA_NO_MEMORY;
193   btree->records = malloc(sizeof(btree->records[0]) * btree->total_number);
194   if (!btree->records)
195     return MYSOFA_NO_MEMORY;
196   memset(btree->records, 0, sizeof(btree->records[0]) * btree->total_number);
197 
198   /* read records */
199   if (fseek(reader->fhd, btree->root_node_address, SEEK_SET) < 0)
200     return errno;
201   return readBTLF(reader, btree, btree->number_of_records, btree->records);
202 }
203 
204 // LCOV_EXCL_STOP
205 
btreeFree(struct BTREE * btree)206 void btreeFree(struct BTREE *btree) { free(btree->records); }
207 
208 /*  III.A.1. Disk Format: Level 1A1 - Version 1 B-trees
209  *
210  */
211 
treeRead(struct READER * reader,struct DATAOBJECT * data)212 int treeRead(struct READER *reader, struct DATAOBJECT *data) {
213 
214   int i, j, err, olen, elements, size, x, y, z, b, e, dy, dz, sx, sy, sz, dzy,
215       szy;
216   char *input, *output;
217 
218   uint8_t node_type, node_level;
219   uint16_t entries_used;
220   uint32_t size_of_chunk;
221   uint32_t filter_mask;
222   uint64_t address_of_left_sibling, address_of_right_sibling, start[4],
223       child_pointer, key, store;
224 
225   char buf[5];
226 
227   UNUSED(node_level);
228   UNUSED(address_of_right_sibling);
229   UNUSED(address_of_left_sibling);
230   UNUSED(key);
231 
232   if (data->ds.dimensionality > 3) {
233     mylog("TREE dimensions > 3"); // LCOV_EXCL_LINE
234     return MYSOFA_INVALID_FORMAT; // LCOV_EXCL_LINE
235   }
236 
237   /* read signature */
238   if (fread(buf, 1, 4, reader->fhd) != 4 || strncmp(buf, "TREE", 4)) {
239     mylog("cannot read signature of TREE\n"); // LCOV_EXCL_LINE
240     return MYSOFA_INVALID_FORMAT;             // LCOV_EXCL_LINE
241   }
242   buf[4] = 0;
243   mylog("%08" PRIX64 " %.4s\n", (uint64_t)ftell(reader->fhd) - 4, buf);
244 
245   node_type = (uint8_t)fgetc(reader->fhd);
246   node_level = (uint8_t)fgetc(reader->fhd);
247   entries_used = (uint16_t)readValue(reader, 2);
248   if (entries_used > 0x1000)
249     return MYSOFA_UNSUPPORTED_FORMAT; // LCOV_EXCL_LINE
250   address_of_left_sibling =
251       readValue(reader, reader->superblock.size_of_offsets);
252   address_of_right_sibling =
253       readValue(reader, reader->superblock.size_of_offsets);
254 
255   elements = 1;
256   for (j = 0; j < data->ds.dimensionality; j++)
257     elements *= data->datalayout_chunk[j];
258   dy = data->datalayout_chunk[1];
259   dz = data->datalayout_chunk[2];
260   sx = data->ds.dimension_size[0];
261   sy = data->ds.dimension_size[1];
262   sz = data->ds.dimension_size[2];
263   dzy = dz * dy;
264   szy = sz * sy;
265   size = data->datalayout_chunk[data->ds.dimensionality];
266 
267   mylog("elements %d size %d\n", elements, size);
268 
269   if (elements <= 0 || size <= 0 || elements >= 0x100000 || size > 0x10)
270     return MYSOFA_INVALID_FORMAT; // LCOV_EXCL_LINE
271   if (!(output = malloc(elements * size))) {
272     return MYSOFA_NO_MEMORY; // LCOV_EXCL_LINE
273   }
274 
275   for (e = 0; e < entries_used * 2; e++) {
276     if (node_type == 0) {
277       key = readValue(reader, reader->superblock.size_of_lengths);
278     } else {
279       size_of_chunk = (uint32_t)readValue(reader, 4);
280       filter_mask = (uint32_t)readValue(reader, 4);
281       if (filter_mask) {
282         mylog("TREE all filters must be enabled\n"); // LCOV_EXCL_LINE
283         free(output);                                // LCOV_EXCL_LINE
284         return MYSOFA_INVALID_FORMAT;                // LCOV_EXCL_LINE
285       }
286 
287       for (j = 0; j < data->ds.dimensionality; j++) {
288         start[j] = readValue(reader, 8);
289         mylog("start %d %" PRIu64 "\n", j, start[j]);
290       }
291 
292       if (readValue(reader, 8)) {
293         break;
294       }
295 
296       child_pointer = readValue(reader, reader->superblock.size_of_offsets);
297       mylog(" data at %" PRIX64 " len %u\n", child_pointer, size_of_chunk);
298 
299       /* read data */
300       store = ftell(reader->fhd);
301       if (fseek(reader->fhd, child_pointer, SEEK_SET) < 0) {
302         free(output); // LCOV_EXCL_LINE
303         return errno; // LCOV_EXCL_LINE
304       }
305 
306       if (!(input = malloc(size_of_chunk))) {
307         free(output);            // LCOV_EXCL_LINE
308         return MYSOFA_NO_MEMORY; // LCOV_EXCL_LINE
309       }
310       if (fread(input, 1, size_of_chunk, reader->fhd) != size_of_chunk) {
311         free(output);                 // LCOV_EXCL_LINE
312         free(input);                  // LCOV_EXCL_LINE
313         return MYSOFA_INVALID_FORMAT; // LCOV_EXCL_LINE
314       }
315 
316       olen = elements * size;
317       err = gunzip(size_of_chunk, input, &olen, output);
318       free(input);
319 
320       mylog("   gunzip %d %d %d\n", err, olen, elements * size);
321       if (err || olen != elements * size) {
322         free(output);                 // LCOV_EXCL_LINE
323         return MYSOFA_INVALID_FORMAT; // LCOV_EXCL_LINE
324       }
325 
326       switch (data->ds.dimensionality) {
327       case 1:
328         for (i = 0; i < olen; i++) {
329           b = i / elements;
330           x = i % elements + start[0];
331           if (x < sx) {
332 
333             j = x * size + b;
334             if (j >= 0 && j < data->data_len) {
335               ((char *)data->data)[j] = output[i];
336             }
337           }
338         }
339         break;
340       case 2:
341         for (i = 0; i < olen; i++) {
342           b = i / elements;
343           x = i % elements;
344           y = x % dy + start[1];
345           x = x / dy + start[0];
346           if (y < sy && x < sx) {
347             j = ((x * sy + y) * size) + b;
348             if (j >= 0 && j < data->data_len) {
349               ((char *)data->data)[j] = output[i];
350             }
351           }
352         }
353         break;
354       case 3:
355         for (i = 0; i < olen; i++) {
356           b = i / elements;
357           x = i % elements;
358           z = x % dz + start[2];
359           y = (x / dz) % dy + start[1];
360           x = (x / dzy) + start[0];
361           if (z < sz && y < sy && x < sx) {
362             j = (x * szy + y * sz + z) * size + b;
363             if (j >= 0 && j < data->data_len) {
364               ((char *)data->data)[j] = output[i];
365             }
366           }
367         }
368         break;
369       default:
370         mylog("invalid dim\n");       // LCOV_EXCL_LINE
371         return MYSOFA_INTERNAL_ERROR; // LCOV_EXCL_LINE
372       }
373 
374       if (fseek(reader->fhd, store, SEEK_SET) < 0) {
375         free(output); // LCOV_EXCL_LINE
376         return errno; // LCOV_EXCL_LINE
377       }
378     }
379   }
380 
381   free(output);
382   if (fseek(reader->fhd, 4, SEEK_CUR) < 0) /* skip checksum */
383     return errno;                          // LCOV_EXCL_LINE
384 
385   return MYSOFA_OK;
386 }
387