1 /*
2 * Copyright (c) Tony Bybell 2006-2014.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version 2
7 * of the License, or (at your option) any later version.
8 */
9
10 /* this code implements generic vlists. (see the original paper
11 from Phil Bagwell in 2002.) the original idea was to
12 clean up histents by using vlist_alloc() to create a growable
13 array that doesn't require next pointers per-element, however
14 that doesn't seem necessary given the space savings that
15 gzipped dormant vlist entries buys you.
16
17 the vlists have been modified since the original version in
18 two ways: (1) only half as many bytes are allocated as needed
19 and when vlist_alloc() reaches the halfway point the struct
20 is finally reallocated with the rest, (2) if vlist_spill is
21 set to "on" in the rc file, vlist entries spill to a tempfile
22 which can reduce memory usage dramatically.
23 */
24
25 #include <config.h>
26 #include "globals.h"
27 #include "vlist.h"
28 #include <zlib.h>
29 #include <string.h>
30
31
vlist_init_spillfile(void)32 void vlist_init_spillfile(void)
33 {
34 if(GLOBALS->use_fastload)
35 {
36 char *fname = malloc_2(strlen(GLOBALS->loaded_file_name) + 4 + 1);
37 sprintf(fname, "%s.idx", GLOBALS->loaded_file_name);
38
39 GLOBALS->vlist_handle = fopen(fname, "w+b");
40
41 free_2(fname);
42
43 fputc('!', GLOBALS->vlist_handle);
44 GLOBALS->vlist_bytes_written = 1;
45 }
46 else
47 {
48 #if defined _MSC_VER || defined __MINGW32__
49 GLOBALS->vlist_handle = tmpfile();
50 fputc('!', GLOBALS->vlist_handle);
51 GLOBALS->vlist_bytes_written = 1;
52 #else
53 int fd_dummy;
54 char *nam = tmpnam_2(NULL, &fd_dummy);
55
56 GLOBALS->vlist_handle = fopen(nam, "w+b");
57
58 unlink(nam);
59 if(fd_dummy >=0) { close(fd_dummy); free_2(nam); }
60
61 fputc('!', GLOBALS->vlist_handle);
62 GLOBALS->vlist_bytes_written = 1;
63 #endif
64 }
65 }
66
vlist_kill_spillfile(void)67 void vlist_kill_spillfile(void)
68 {
69 if(GLOBALS->vlist_handle)
70 {
71 fclose(GLOBALS->vlist_handle);
72 GLOBALS->vlist_handle = NULL;
73 }
74 }
75
76
77 /* machine-independent header i/o
78 */
vlist_fread_hdr(struct vlist_t * vl,FILE * f)79 static int vlist_fread_hdr(struct vlist_t *vl, FILE *f)
80 {
81 uintptr_t val;
82 unsigned int vali;
83 int ch, shamt, rc = 0;
84
85 val = 0; shamt = 0;
86 do
87 {
88 ch = fgetc(f);
89 if(ch == EOF) goto bail;
90
91 val |= ((unsigned long)(ch & 0x7f)) << shamt;
92 shamt += 7;
93 } while(!(ch & 0x80));
94 vl->next = (struct vlist_t *)val;
95
96 vali = 0; shamt = 0;
97 do
98 {
99 ch = fgetc(f);
100 if(ch == EOF) goto bail;
101
102 vali |= ((unsigned int)(ch & 0x7f)) << shamt;
103 shamt += 7;
104 } while(!(ch & 0x80));
105 vl->siz = (unsigned int)vali;
106
107 vali = 0; shamt = 0;
108 do
109 {
110 ch = fgetc(f);
111 if(ch == EOF) goto bail;
112
113 vali |= ((unsigned int)(ch & 0x7f)) << shamt;
114 shamt += 7;
115 } while(!(ch & 0x80));
116 vl->offs = (vali & 1) ? (unsigned int)(-(int)(vali >> 1)) : (vali >> 1);
117
118 vali = 0; shamt = 0;
119 do
120 {
121 ch = fgetc(f);
122 if(ch == EOF) goto bail;
123
124 vali |= ((unsigned int)(ch & 0x7f)) << shamt;
125 shamt += 7;
126 } while(!(ch & 0x80));
127 vl->elem_siz = (unsigned int)vali;
128
129 rc = 1;
130
131 bail:
132 return(rc);
133 }
134
135
vlist_fwrite(struct vlist_t * vl,unsigned int rsiz,FILE * f)136 static int vlist_fwrite(struct vlist_t *vl, unsigned int rsiz, FILE *f)
137 {
138 unsigned char mem[ 4 * sizeof(long) * 2];
139 unsigned char *pnt = mem;
140 uintptr_t val, nxt;
141 unsigned int vali, nxti;
142 int offs_as_int;
143 int rc;
144 int len = 0;
145
146 val = (uintptr_t)(vl->next);
147 while((nxt = val>>7))
148 {
149 *(pnt++) = (val&0x7f);
150 val = nxt;
151 }
152 *(pnt++) = (val&0x7f) | 0x80;
153
154
155 vali = (vl->siz);
156 while((nxti = vali>>7))
157 {
158 *(pnt++) = (vali&0x7f);
159 vali = nxti;
160 }
161 *(pnt++) = (vali&0x7f) | 0x80;
162
163 offs_as_int = (int)(vl->offs);
164 if(offs_as_int < 0)
165 {
166 offs_as_int = -offs_as_int; /* reduce number of one bits propagating left by making sign bit the lsb */
167 offs_as_int <<= 1;
168 offs_as_int |= 1;
169 }
170 else
171 {
172 offs_as_int <<= 1;
173 }
174
175 vali = (unsigned int)(offs_as_int);
176 while((nxti = vali>>7))
177 {
178 *(pnt++) = (vali&0x7f);
179 vali = nxti;
180 }
181 *(pnt++) = (vali&0x7f) | 0x80;
182
183 vali = (unsigned int)(vl->elem_siz);
184 while((nxti = vali>>7))
185 {
186 *(pnt++) = (vali&0x7f);
187 vali = nxti;
188 }
189 *(pnt++) = (vali&0x7f) | 0x80;
190
191 rc = fwrite(mem, 1, (len = (pnt - mem)), f);
192 if(rc)
193 {
194 unsigned int wrlen = (rsiz - sizeof(struct vlist_t));
195 len += wrlen;
196
197 rc = fwrite(vl + 1, 1, wrlen, f);
198 if(rc)
199 {
200 rc = len;
201 }
202 }
203
204 return(rc);
205 }
206
207
208 /* create / destroy
209 */
vlist_create(unsigned int elem_siz)210 struct vlist_t *vlist_create(unsigned int elem_siz)
211 {
212 struct vlist_t *v;
213
214 v = calloc_2(1, sizeof(struct vlist_t) + elem_siz);
215 v->siz = 1;
216 v->elem_siz = elem_siz;
217
218 return(v);
219 }
220
221
vlist_destroy(struct vlist_t * v)222 void vlist_destroy(struct vlist_t *v)
223 {
224 struct vlist_t *vt;
225
226 while(v)
227 {
228 vt = v->next;
229 free_2(v);
230 v = vt;
231 }
232 }
233
234
235 /* realtime compression/decompression of bytewise vlists
236 * this can obviously be extended if elem_siz > 1, but
237 * the viewer doesn't need that feature
238 */
vlist_compress_block(struct vlist_t * v,unsigned int * rsiz)239 struct vlist_t *vlist_compress_block(struct vlist_t *v, unsigned int *rsiz)
240 {
241 if(v->siz > 32)
242 {
243 struct vlist_t *vz;
244 unsigned int *ipnt;
245 char *dmem = malloc_2(compressBound(v->siz));
246 unsigned long destlen = v->siz;
247 int rc;
248
249 rc = compress2((unsigned char *)dmem, &destlen, (unsigned char *)(v+1), v->siz, GLOBALS->vlist_compression_depth);
250 if( (rc == Z_OK) && ((destlen + sizeof(int)) < v->siz) )
251 {
252 /* printf("siz: %d, dest: %d rc: %d\n", v->siz, (int)destlen, rc); */
253
254 vz = malloc_2(*rsiz = sizeof(struct vlist_t) + sizeof(int) + destlen);
255 memcpy(vz, v, sizeof(struct vlist_t));
256
257 ipnt = (unsigned int *)(vz + 1);
258 ipnt[0] = destlen;
259 memcpy(&ipnt[1], dmem, destlen);
260 vz->offs = (unsigned int)(-(int)v->offs); /* neg value signified compression */
261 free_2(v);
262 v = vz;
263 }
264
265 free_2(dmem);
266 }
267
268 return(v);
269 }
270
271
vlist_uncompress(struct vlist_t ** v)272 void vlist_uncompress(struct vlist_t **v)
273 {
274 struct vlist_t *vl = *v;
275 struct vlist_t *vprev = NULL;
276
277 if(GLOBALS->vlist_handle)
278 {
279 while(vl)
280 {
281 struct vlist_t vhdr;
282 struct vlist_t *vrebuild;
283 uintptr_t vl_offs = (uintptr_t)vl;
284 int rc;
285
286 off_t seekpos = (off_t) vl_offs; /* possible overflow conflicts were already handled in the writer */
287
288 fseeko(GLOBALS->vlist_handle, seekpos, SEEK_SET);
289
290 if(GLOBALS->use_fastload)
291 {
292 rc = vlist_fread_hdr(&vhdr, GLOBALS->vlist_handle);
293 }
294 else
295 {
296 rc = fread(&vhdr, sizeof(struct vlist_t), 1, GLOBALS->vlist_handle);
297 }
298
299 if(!rc)
300 {
301 printf("Error in reading from VList spill file!\n");
302 exit(255);
303 }
304
305 /* args are reversed to fread (compared to above) to handle short read at end of file! */
306 /* (this can happen because of how we write out only the used size of a block) */
307 vrebuild = malloc_2(sizeof(struct vlist_t) + vhdr.siz);
308 memcpy(vrebuild, &vhdr, sizeof(struct vlist_t));
309 rc = fread(vrebuild+1, 1, vrebuild->siz, GLOBALS->vlist_handle);
310 if(!rc)
311 {
312 printf("Error in reading from VList spill file!\n");
313 exit(255);
314 }
315
316 if(vprev)
317 {
318 vprev->next = vrebuild;
319 }
320 else
321 {
322 *v = vrebuild;
323 }
324
325 vprev = vrebuild;
326 vl = vhdr.next;
327 }
328
329 vl = *v;
330 vprev = NULL;
331 }
332
333 while(vl)
334 {
335 if((int)vl->offs < 0)
336 {
337 struct vlist_t *vz = malloc_2(sizeof(struct vlist_t) + vl->siz);
338 unsigned int *ipnt;
339 unsigned long sourcelen, destlen;
340 int rc;
341
342 memcpy(vz, vl, sizeof(struct vlist_t));
343 vz->offs = (unsigned int)(-(int)vl->offs);
344
345 ipnt = (unsigned int *)(vl + 1);
346 sourcelen = (unsigned long)ipnt[0];
347 destlen = (unsigned long)vl->siz;
348
349 rc = uncompress((unsigned char *)(vz+1), &destlen, (unsigned char *)&ipnt[1], sourcelen);
350 if(rc != Z_OK)
351 {
352 fprintf(stderr, "Error in vlist uncompress(), rc=%d/destlen=%d exiting!\n", rc, (int)destlen);
353 exit(255);
354 }
355
356 free_2(vl);
357 vl = vz;
358
359 if(vprev)
360 {
361 vprev->next = vz;
362 }
363 else
364 {
365 *v = vz;
366 }
367 }
368
369 vprev = vl;
370 vl = vl->next;
371 }
372
373 }
374
375
376 /* get pointer to one unit of space
377 */
vlist_alloc(struct vlist_t ** v,int compressable)378 void *vlist_alloc(struct vlist_t **v, int compressable)
379 {
380 struct vlist_t *vl = *v;
381 char *px;
382 struct vlist_t *v2;
383
384 if(vl->offs == vl->siz)
385 {
386 unsigned int siz, rsiz;
387
388 /* 2 times versions are the growable, indexable vlists */
389 siz = 2 * vl->siz;
390
391 rsiz = sizeof(struct vlist_t) + (vl->siz * vl->elem_siz);
392
393 if((compressable)&&(vl->elem_siz == 1))
394 {
395 if(GLOBALS->vlist_compression_depth>=0)
396 {
397 vl = vlist_compress_block(vl, &rsiz);
398 }
399 }
400
401 if(compressable && GLOBALS->vlist_handle)
402 {
403 size_t rc;
404 intptr_t write_cnt;
405
406 fseeko(GLOBALS->vlist_handle, GLOBALS->vlist_bytes_written, SEEK_SET);
407
408 if(GLOBALS->use_fastload)
409 {
410 rc = vlist_fwrite(vl, rsiz, GLOBALS->vlist_handle);
411 }
412 else
413 {
414 rc = fwrite(vl, rsiz, 1, GLOBALS->vlist_handle);
415 }
416
417 if(!rc)
418 {
419 fprintf(stderr, "Error in writing to VList spill file!\n");
420 perror("Why");
421 exit(255);
422 }
423
424 write_cnt = GLOBALS->vlist_bytes_written;
425 if(sizeof(long) != sizeof(off_t)) /* optimizes in or out at compile time */
426 {
427 if(write_cnt != GLOBALS->vlist_bytes_written)
428 {
429 fprintf(stderr, "VList spill file pointer-file overflow!\n");
430 exit(255);
431 }
432 }
433
434 v2 = calloc_2(1, sizeof(struct vlist_t) + (vl->siz * vl->elem_siz));
435 v2->siz = siz;
436 v2->elem_siz = vl->elem_siz;
437 v2->next = (struct vlist_t *)write_cnt;
438 free_2(vl);
439
440 *v = v2;
441 vl = *v;
442
443 if(GLOBALS->use_fastload)
444 {
445 GLOBALS->vlist_bytes_written += rc;
446 }
447 else
448 {
449 GLOBALS->vlist_bytes_written += rsiz;
450 }
451 }
452 else
453 {
454 v2 = calloc_2(1, sizeof(struct vlist_t) + (vl->siz * vl->elem_siz));
455 v2->siz = siz;
456 v2->elem_siz = vl->elem_siz;
457 v2->next = vl;
458 *v = v2;
459 vl = *v;
460 }
461 }
462 else
463 if(vl->offs*2 == vl->siz)
464 {
465 v2 = calloc_2(1, sizeof(struct vlist_t) + (vl->siz * vl->elem_siz));
466 memcpy(v2, vl, sizeof(struct vlist_t) + (vl->siz/2 * vl->elem_siz));
467 free_2(vl);
468
469 *v = v2;
470 vl = *v;
471 }
472
473 px =(((char *)(vl)) + sizeof(struct vlist_t) + ((vl->offs++) * vl->elem_siz));
474 return((void *)px);
475 }
476
477
478 /* vlist_size() and vlist_locate() do not work properly on
479 compressed lists...you'll have to call vlist_uncompress() first!
480 */
vlist_size(struct vlist_t * v)481 unsigned int vlist_size(struct vlist_t *v)
482 {
483 return(v->siz - 1 + v->offs);
484 }
485
486
vlist_locate(struct vlist_t * v,unsigned int idx)487 void *vlist_locate(struct vlist_t *v, unsigned int idx)
488 {
489 unsigned int here = v->siz - 1;
490 unsigned int siz = here + v->offs; /* siz is the same as vlist_size() */
491
492 if((!siz)||(idx>=siz)) return(NULL);
493
494 while (idx < here)
495 {
496 v = v->next;
497 here = v->siz - 1;
498 }
499
500 idx -= here;
501
502 return((void *)(((char *)(v)) + sizeof(struct vlist_t) + (idx * v->elem_siz)));
503 }
504
505
506 /* calling this if you don't plan on adding any more elements will free
507 up unused space as well as compress final blocks (if enabled)
508 */
vlist_freeze(struct vlist_t ** v)509 void vlist_freeze(struct vlist_t **v)
510 {
511 struct vlist_t *vl = *v;
512 unsigned int siz = vl->offs;
513 unsigned int rsiz = sizeof(struct vlist_t) + (siz * vl->elem_siz);
514
515 if((vl->elem_siz == 1)&&(siz))
516 {
517 struct vlist_t *w, *v2;
518
519 if(vl->offs*2 <= vl->siz) /* Electric Fence, change < to <= */
520 {
521 v2 = calloc_2(1, sizeof(struct vlist_t) + (vl->siz /* * vl->elem_siz */)); /* scan-build */
522 memcpy(v2, vl, sizeof(struct vlist_t) + (vl->siz/2 /* * vl->elem_siz */)); /* scan-build */
523 free_2(vl);
524
525 *v = v2;
526 vl = *v;
527 }
528
529 w = vlist_compress_block(vl, &rsiz);
530 *v = w;
531 }
532 else
533 if((siz != vl->siz)&&(!GLOBALS->vlist_handle))
534 {
535 struct vlist_t *w = malloc_2(rsiz);
536 memcpy(w, vl, rsiz);
537 free_2(vl);
538 *v = w;
539 }
540
541 if(GLOBALS->vlist_handle)
542 {
543 size_t rc;
544 intptr_t write_cnt;
545
546 vl = *v;
547 fseeko(GLOBALS->vlist_handle, GLOBALS->vlist_bytes_written, SEEK_SET);
548
549 if(GLOBALS->use_fastload)
550 {
551 rc = vlist_fwrite(vl, rsiz, GLOBALS->vlist_handle);
552 }
553 else
554 {
555 rc = fwrite(vl, rsiz, 1, GLOBALS->vlist_handle);
556 }
557
558 if(!rc)
559 {
560 fprintf(stderr, "Error in writing to VList spill file!\n");
561 perror("Why");
562 exit(255);
563 }
564
565 write_cnt = GLOBALS->vlist_bytes_written;
566 if(sizeof(long) != sizeof(off_t)) /* optimizes in or out at compile time */
567 {
568 if(write_cnt != GLOBALS->vlist_bytes_written)
569 {
570 fprintf(stderr, "VList spill file pointer-file overflow!\n");
571 exit(255);
572 }
573 }
574
575 *v = (struct vlist_t *)write_cnt;
576
577 if(GLOBALS->use_fastload)
578 {
579 GLOBALS->vlist_bytes_written += rc;
580 }
581 else
582 {
583 GLOBALS->vlist_bytes_written += rsiz;
584 }
585
586 free_2(vl);
587 }
588 }
589
590
591 /* this code implements an LZ-based filter that can sit on top
592 of the vlists. it uses a generic escape value of 0xff as
593 that is one that statistically occurs infrequently in value
594 change data fed into vlists.
595 */
596
vlist_packer_emit_out(struct vlist_packer_t * p,unsigned char byt)597 void vlist_packer_emit_out(struct vlist_packer_t *p, unsigned char byt)
598 {
599 char *pnt;
600
601 #ifdef WAVE_VLIST_PACKER_STATS
602 p->packed_bytes++;
603 #endif
604
605 pnt = vlist_alloc(&p->v, 1);
606 *pnt = byt;
607 }
608
609
vlist_packer_emit_uv32(struct vlist_packer_t * p,unsigned int v)610 void vlist_packer_emit_uv32(struct vlist_packer_t *p, unsigned int v)
611 {
612 unsigned int nxt;
613
614 while((nxt = v>>7))
615 {
616 vlist_packer_emit_out(p, v&0x7f);
617 v = nxt;
618 }
619
620 vlist_packer_emit_out(p, (v&0x7f) | 0x80);
621 }
622
623
vlist_packer_emit_uv32rvs(struct vlist_packer_t * p,unsigned int v)624 void vlist_packer_emit_uv32rvs(struct vlist_packer_t *p, unsigned int v)
625 {
626 unsigned int nxt;
627 unsigned char buf[2 * sizeof(int)];
628 unsigned int idx = 0;
629 int i;
630
631 while((nxt = v>>7))
632 {
633 buf[idx++] = v&0x7f;
634 v = nxt;
635 }
636
637 buf[idx] = (v&0x7f) | 0x80;
638
639 for(i = idx; i >= 0; i--)
640 {
641 vlist_packer_emit_out(p, buf[i]);
642 }
643 }
644
645
vlist_packer_alloc(struct vlist_packer_t * p,unsigned char byt)646 void vlist_packer_alloc(struct vlist_packer_t *p, unsigned char byt)
647 {
648 int i, j, k, l;
649
650 p->unpacked_bytes++;
651
652 if(!p->repcnt)
653 {
654 top:
655 for(i=0;i<WAVE_ZIVSRCH;i++)
656 {
657 if(p->buf[(p->bufpnt-i) & WAVE_ZIVMASK] == byt)
658 {
659 p->repdist = i;
660 p->repcnt = 1;
661
662 p->repdist2 = p->repdist3 = p->repdist4 = 0;
663
664 for(j=i+WAVE_ZIVSKIP;j<WAVE_ZIVSRCH;j++)
665 {
666 if(p->buf[(p->bufpnt-j) & WAVE_ZIVMASK] == byt)
667 {
668 p->repdist2 = j;
669 p->repcnt2 = 1;
670
671 for(k=j+WAVE_ZIVSKIP;k<WAVE_ZIVSRCH;k++)
672 {
673 if(p->buf[(p->bufpnt-k) & WAVE_ZIVMASK] == byt)
674 {
675 p->repdist3 = k;
676 p->repcnt3 = 1;
677
678 for(l=k+WAVE_ZIVSKIP;l<WAVE_ZIVSRCH;l++)
679 {
680 if(p->buf[(p->bufpnt-l) & WAVE_ZIVMASK] == byt)
681 {
682 p->repdist4 = l;
683 p->repcnt4 = 1;
684 break;
685 }
686 }
687 break;
688 }
689 }
690 break;
691 }
692 }
693
694 p->bufpnt++;
695 p->bufpnt &= WAVE_ZIVMASK;
696 p->buf[p->bufpnt] = byt;
697
698 return;
699 }
700 }
701
702 p->bufpnt++;
703 p->bufpnt &= WAVE_ZIVMASK;
704 p->buf[p->bufpnt] = byt;
705 vlist_packer_emit_out(p, byt);
706 if(byt==WAVE_ZIVFLAG)
707 {
708 vlist_packer_emit_uv32(p, 0);
709 }
710 }
711 else
712 {
713 attempt2:
714 if(p->buf[(p->bufpnt - p->repdist) & WAVE_ZIVMASK] == byt)
715 {
716 p->repcnt++;
717
718 if(p->repcnt2)
719 {
720 p->repcnt2 = ((p->buf[(p->bufpnt - p->repdist2) & WAVE_ZIVMASK] == byt)) ? p->repcnt2+1 : 0;
721 }
722 if(p->repcnt3)
723 {
724 p->repcnt3 = ((p->buf[(p->bufpnt - p->repdist3) & WAVE_ZIVMASK] == byt)) ? p->repcnt3+1 : 0;
725 }
726 if(p->repcnt4)
727 {
728 p->repcnt4 = ((p->buf[(p->bufpnt - p->repdist4) & WAVE_ZIVMASK] == byt)) ? p->repcnt4+1 : 0;
729 }
730
731 p->bufpnt++;
732 p->bufpnt &= WAVE_ZIVMASK;
733 p->buf[p->bufpnt] = byt;
734 }
735 else
736 {
737 if(p->repcnt2)
738 {
739 p->repcnt = p->repcnt2;
740 p->repdist = p->repdist2;
741
742 p->repcnt2 = p->repcnt3;
743 p->repdist2 = p->repdist3;
744
745 p->repcnt3 = p->repcnt4;
746 p->repdist3 = p->repdist4;
747
748 p->repcnt4 = 0;
749 p->repdist4 = 0;
750
751 goto attempt2;
752 }
753
754 if(p->repcnt3)
755 {
756 p->repcnt = p->repcnt3;
757 p->repdist = p->repdist3;
758
759 p->repcnt2 = p->repcnt4;
760 p->repdist2 = p->repdist4;
761
762 p->repcnt3 = 0;
763 p->repdist3 = 0;
764 p->repcnt4 = 0;
765 p->repdist4 = 0;
766
767 goto attempt2;
768 }
769
770 if(p->repcnt4)
771 {
772 p->repcnt = p->repcnt4;
773 p->repdist = p->repdist4;
774
775 p->repcnt4 = 0;
776 p->repdist4 = 0;
777
778 goto attempt2;
779 }
780
781 if(p->repcnt > 2)
782 {
783 vlist_packer_emit_out(p, WAVE_ZIVFLAG);
784 vlist_packer_emit_uv32(p, p->repcnt); p->repcnt = 0;
785 vlist_packer_emit_uv32(p, p->repdist);
786 }
787 else
788 {
789 if(p->repcnt == 2)
790 {
791 vlist_packer_emit_out(p, p->buf[(p->bufpnt-1) & WAVE_ZIVMASK]);
792 if(p->buf[(p->bufpnt-1) & WAVE_ZIVMASK]==WAVE_ZIVFLAG)
793 {
794 vlist_packer_emit_uv32(p, 0);
795 }
796 }
797
798 vlist_packer_emit_out(p, p->buf[p->bufpnt & WAVE_ZIVMASK]);
799 p->repcnt = 0;
800 if(p->buf[p->bufpnt & WAVE_ZIVMASK]==WAVE_ZIVFLAG)
801 {
802 vlist_packer_emit_uv32(p, 0);
803 }
804 }
805 goto top;
806 }
807 }
808 }
809
810
vlist_packer_finalize(struct vlist_packer_t * p)811 void vlist_packer_finalize(struct vlist_packer_t *p)
812 {
813 #ifdef WAVE_VLIST_PACKER_STATS
814 static guint64 pp = 0, upp = 0;
815 #endif
816
817 if(p->repcnt)
818 {
819 if(p->repcnt > 2)
820 {
821 vlist_packer_emit_out(p, WAVE_ZIVFLAG);
822 vlist_packer_emit_uv32(p, p->repcnt); p->repcnt = 0;
823 vlist_packer_emit_uv32(p, p->repdist);
824 }
825 else
826 {
827 if(p->repcnt == 2)
828 {
829 vlist_packer_emit_out(p, p->buf[(p->bufpnt-1) & WAVE_ZIVMASK]);
830 if(p->buf[(p->bufpnt-1) & WAVE_ZIVMASK]==WAVE_ZIVFLAG)
831 {
832 vlist_packer_emit_uv32(p, 0);
833 }
834 }
835
836 vlist_packer_emit_out(p, p->buf[p->bufpnt & WAVE_ZIVMASK]);
837 p->repcnt = 0;
838 if(p->buf[p->bufpnt & WAVE_ZIVMASK]==WAVE_ZIVFLAG)
839 {
840 vlist_packer_emit_uv32(p, 0);
841 }
842 }
843 }
844
845 vlist_packer_emit_uv32rvs(p, p->unpacked_bytes); /* for malloc later during decompress */
846
847 #ifdef WAVE_VLIST_PACKER_STATS
848 pp += p->packed_bytes;
849 upp += p->unpacked_bytes;
850
851 printf("pack:%d orig:%d (%lld %lld %f)\n", p->packed_bytes, p->unpacked_bytes, pp, upp, (float)pp / (float)upp);
852 #endif
853 }
854
855
vlist_packer_create(void)856 struct vlist_packer_t *vlist_packer_create(void)
857 {
858 struct vlist_packer_t *vp = calloc_2(1, sizeof(struct vlist_packer_t));
859 vp->v = vlist_create(sizeof(char));
860
861 return(vp);
862 }
863
864
vlist_packer_decompress(struct vlist_t * v,unsigned int * declen)865 unsigned char *vlist_packer_decompress(struct vlist_t *v, unsigned int *declen)
866 {
867 unsigned int list_size = vlist_size(v);
868 unsigned int top_of_packed_size = list_size-1;
869 unsigned char *chp;
870 unsigned int dec_size = 0;
871 unsigned int dec_size_cmp;
872 unsigned int shamt = 0;
873 unsigned char *mem, *dpnt;
874 unsigned int i, j, repcnt, dist;
875
876 for(;;)
877 {
878 chp = vlist_locate(v, top_of_packed_size);
879
880 dec_size |= ((unsigned int)(*chp & 0x7f)) << shamt;
881
882 if(*chp & 0x80)
883 {
884 break;
885 }
886
887 shamt+=7;
888 top_of_packed_size--;
889 }
890
891 mem = calloc_2(1, WAVE_ZIVWRAP + dec_size);
892 dpnt = mem + WAVE_ZIVWRAP;
893 for(i=0;i<top_of_packed_size;i++)
894 {
895 chp = vlist_locate(v, i);
896 if(*chp != WAVE_ZIVFLAG)
897 {
898 *(dpnt++) = *chp;
899 continue;
900 }
901
902 i++;
903 repcnt = shamt = 0;
904 for(;;)
905 {
906 chp = vlist_locate(v, i);
907
908 repcnt |= ((unsigned int)(*chp & 0x7f)) << shamt;
909
910 if(*chp & 0x80)
911 {
912 break;
913 }
914 i++;
915
916 shamt+=7;
917 }
918 if(repcnt == 0)
919 {
920 *(dpnt++) = WAVE_ZIVFLAG;
921 continue;
922 }
923
924 i++;
925 dist = shamt = 0;
926 for(;;)
927 {
928 chp = vlist_locate(v, i);
929
930 dist |= ((unsigned int)(*chp & 0x7f)) << shamt;
931
932 if(*chp & 0x80)
933 {
934 break;
935 }
936 i++;
937
938 shamt+=7;
939 }
940
941 for(j=0;j<repcnt;j++)
942 {
943 *dpnt = *(dpnt - dist - 1);
944 dpnt++;
945 }
946 }
947
948 *declen = dec_size;
949
950 dec_size_cmp = dpnt - mem - WAVE_ZIVWRAP;
951 if(dec_size != dec_size_cmp)
952 {
953 fprintf(stderr, "miscompare: decompressed vlist_packer length: %d vs %d bytes, exiting.\n", dec_size, dec_size_cmp);
954 exit(255);
955 }
956
957 return(mem + WAVE_ZIVWRAP);
958 }
959
960
vlist_packer_decompress_destroy(char * mem)961 void vlist_packer_decompress_destroy(char *mem)
962 {
963 free_2(mem - WAVE_ZIVWRAP);
964 }
965
966
967