1 /*
2  * Copyright (c) Tony Bybell 2006-2014.
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public License
6  * as published by the Free Software Foundation; either version 2
7  * of the License, or (at your option) any later version.
8  */
9 
10 /* this code implements generic vlists.  (see the original paper
11    from Phil Bagwell in 2002.)  the original idea was to
12    clean up histents by using vlist_alloc() to create a growable
13    array that doesn't require next pointers per-element, however
14    that doesn't seem necessary given the space savings that
15    gzipped dormant vlist entries buys you.
16 
17    the vlists have been modified since the original version in
18    two ways: (1) only half as many bytes are allocated as needed
19    and when vlist_alloc() reaches the halfway point the struct
20    is finally reallocated with the rest, (2) if vlist_spill is
21    set to "on" in the rc file, vlist entries spill to a tempfile
22    which can reduce memory usage dramatically.
23  */
24 
25 #include <config.h>
26 #include "globals.h"
27 #include "vlist.h"
28 #include <zlib.h>
29 #include <string.h>
30 
31 
vlist_init_spillfile(void)32 void vlist_init_spillfile(void)
33 {
34 if(GLOBALS->use_fastload)
35 	{
36 	char *fname = malloc_2(strlen(GLOBALS->loaded_file_name) + 4 + 1);
37 	sprintf(fname, "%s.idx", GLOBALS->loaded_file_name);
38 
39 	GLOBALS->vlist_handle = fopen(fname, "w+b");
40 
41 	free_2(fname);
42 
43 	fputc('!', GLOBALS->vlist_handle);
44 	GLOBALS->vlist_bytes_written = 1;
45 	}
46 else
47 	{
48 #if defined _MSC_VER || defined __MINGW32__
49 	GLOBALS->vlist_handle = tmpfile();
50 	fputc('!', GLOBALS->vlist_handle);
51 	GLOBALS->vlist_bytes_written = 1;
52 #else
53 	int fd_dummy;
54 	char *nam = tmpnam_2(NULL, &fd_dummy);
55 
56 	GLOBALS->vlist_handle = fopen(nam, "w+b");
57 
58 	unlink(nam);
59 	if(fd_dummy >=0) { close(fd_dummy); free_2(nam); }
60 
61 	fputc('!', GLOBALS->vlist_handle);
62 	GLOBALS->vlist_bytes_written = 1;
63 #endif
64 	}
65 }
66 
vlist_kill_spillfile(void)67 void vlist_kill_spillfile(void)
68 {
69 if(GLOBALS->vlist_handle)
70 	{
71 	fclose(GLOBALS->vlist_handle);
72 	GLOBALS->vlist_handle = NULL;
73 	}
74 }
75 
76 
77 /* machine-independent header i/o
78  */
vlist_fread_hdr(struct vlist_t * vl,FILE * f)79 static int vlist_fread_hdr(struct vlist_t *vl, FILE *f)
80 {
81 uintptr_t val;
82 unsigned int vali;
83 int ch, shamt, rc = 0;
84 
85 val = 0; shamt = 0;
86 do
87 	{
88 	ch = fgetc(f);
89 	if(ch == EOF) goto bail;
90 
91 	val |= ((unsigned long)(ch & 0x7f)) << shamt;
92 	shamt += 7;
93 	} while(!(ch & 0x80));
94 vl->next = (struct vlist_t *)val;
95 
96 vali = 0; shamt = 0;
97 do
98 	{
99 	ch = fgetc(f);
100 	if(ch == EOF) goto bail;
101 
102 	vali |= ((unsigned int)(ch & 0x7f)) << shamt;
103 	shamt += 7;
104 	} while(!(ch & 0x80));
105 vl->siz = (unsigned int)vali;
106 
107 vali = 0; shamt = 0;
108 do
109 	{
110 	ch = fgetc(f);
111 	if(ch == EOF) goto bail;
112 
113 	vali |= ((unsigned int)(ch & 0x7f)) << shamt;
114 	shamt += 7;
115 	} while(!(ch & 0x80));
116 vl->offs = (vali & 1) ? (unsigned int)(-(int)(vali >> 1)) : (vali >> 1);
117 
118 vali = 0; shamt = 0;
119 do
120 	{
121 	ch = fgetc(f);
122 	if(ch == EOF) goto bail;
123 
124 	vali |= ((unsigned int)(ch & 0x7f)) << shamt;
125 	shamt += 7;
126 	} while(!(ch & 0x80));
127 vl->elem_siz = (unsigned int)vali;
128 
129 rc = 1;
130 
131 bail:
132 return(rc);
133 }
134 
135 
vlist_fwrite(struct vlist_t * vl,unsigned int rsiz,FILE * f)136 static int vlist_fwrite(struct vlist_t *vl, unsigned int rsiz, FILE *f)
137 {
138 unsigned char mem[ 4 * sizeof(long) * 2];
139 unsigned char *pnt = mem;
140 uintptr_t val, nxt;
141 unsigned int vali, nxti;
142 int offs_as_int;
143 int rc;
144 int len = 0;
145 
146 val = (uintptr_t)(vl->next);
147 while((nxt = val>>7))
148         {
149         *(pnt++) = (val&0x7f);
150         val = nxt;
151         }
152 *(pnt++) = (val&0x7f) | 0x80;
153 
154 
155 vali = (vl->siz);
156 while((nxti = vali>>7))
157         {
158         *(pnt++) = (vali&0x7f);
159         vali = nxti;
160         }
161 *(pnt++) = (vali&0x7f) | 0x80;
162 
163 offs_as_int = (int)(vl->offs);
164 if(offs_as_int < 0)
165 	{
166 	offs_as_int = -offs_as_int;	/* reduce number of one bits propagating left by making sign bit the lsb */
167 	offs_as_int <<= 1;
168 	offs_as_int |= 1;
169 	}
170 	else
171 	{
172 	offs_as_int <<= 1;
173 	}
174 
175 vali = (unsigned int)(offs_as_int);
176 while((nxti = vali>>7))
177         {
178         *(pnt++) = (vali&0x7f);
179         vali = nxti;
180         }
181 *(pnt++) = (vali&0x7f) | 0x80;
182 
183 vali = (unsigned int)(vl->elem_siz);
184 while((nxti = vali>>7))
185         {
186         *(pnt++) = (vali&0x7f);
187         vali = nxti;
188         }
189 *(pnt++) = (vali&0x7f) | 0x80;
190 
191 rc = fwrite(mem, 1, (len = (pnt - mem)), f);
192 if(rc)
193 	{
194         unsigned int wrlen = (rsiz - sizeof(struct vlist_t));
195         len += wrlen;
196 
197 	rc = fwrite(vl + 1, 1, wrlen, f);
198 	if(rc)
199 		{
200 		rc = len;
201 		}
202 	}
203 
204 return(rc);
205 }
206 
207 
208 /* create / destroy
209  */
vlist_create(unsigned int elem_siz)210 struct vlist_t *vlist_create(unsigned int elem_siz)
211 {
212 struct vlist_t *v;
213 
214 v = calloc_2(1, sizeof(struct vlist_t) + elem_siz);
215 v->siz = 1;
216 v->elem_siz = elem_siz;
217 
218 return(v);
219 }
220 
221 
vlist_destroy(struct vlist_t * v)222 void vlist_destroy(struct vlist_t *v)
223 {
224 struct vlist_t *vt;
225 
226 while(v)
227 	{
228 	vt = v->next;
229 	free_2(v);
230 	v = vt;
231 	}
232 }
233 
234 
235 /* realtime compression/decompression of bytewise vlists
236  * this can obviously be extended if elem_siz > 1, but
237  * the viewer doesn't need that feature
238  */
vlist_compress_block(struct vlist_t * v,unsigned int * rsiz)239 struct vlist_t *vlist_compress_block(struct vlist_t *v, unsigned int *rsiz)
240 {
241 if(v->siz > 32)
242 	{
243 	struct vlist_t *vz;
244 	unsigned int *ipnt;
245 	char *dmem = malloc_2(compressBound(v->siz));
246 	unsigned long destlen = v->siz;
247 	int rc;
248 
249 	rc = compress2((unsigned char *)dmem, &destlen, (unsigned char *)(v+1), v->siz, GLOBALS->vlist_compression_depth);
250 	if( (rc == Z_OK) && ((destlen + sizeof(int)) < v->siz) )
251 		{
252 		/* printf("siz: %d, dest: %d rc: %d\n", v->siz, (int)destlen, rc); */
253 
254 		vz = malloc_2(*rsiz = sizeof(struct vlist_t) + sizeof(int) + destlen);
255 		memcpy(vz, v, sizeof(struct vlist_t));
256 
257 		ipnt = (unsigned int *)(vz + 1);
258 		ipnt[0] = destlen;
259 		memcpy(&ipnt[1], dmem, destlen);
260 		vz->offs = (unsigned int)(-(int)v->offs); /* neg value signified compression */
261 		free_2(v);
262 		v = vz;
263 		}
264 
265 	free_2(dmem);
266 	}
267 
268 return(v);
269 }
270 
271 
vlist_uncompress(struct vlist_t ** v)272 void vlist_uncompress(struct vlist_t **v)
273 {
274 struct vlist_t *vl = *v;
275 struct vlist_t *vprev = NULL;
276 
277 if(GLOBALS->vlist_handle)
278 	{
279 	while(vl)
280 		{
281 		struct vlist_t vhdr;
282 		struct vlist_t *vrebuild;
283 		uintptr_t vl_offs = (uintptr_t)vl;
284 		int rc;
285 
286 		off_t seekpos = (off_t) vl_offs;	/* possible overflow conflicts were already handled in the writer */
287 
288 		fseeko(GLOBALS->vlist_handle, seekpos, SEEK_SET);
289 
290 		if(GLOBALS->use_fastload)
291 			{
292 			rc = vlist_fread_hdr(&vhdr, GLOBALS->vlist_handle);
293 			}
294 			else
295 			{
296 			rc = fread(&vhdr, sizeof(struct vlist_t), 1, GLOBALS->vlist_handle);
297 			}
298 
299 		if(!rc)
300 			{
301 			printf("Error in reading from VList spill file!\n");
302 			exit(255);
303 			}
304 
305 		/* args are reversed to fread (compared to above) to handle short read at end of file! */
306 		/* (this can happen because of how we write out only the used size of a block)         */
307 		vrebuild = malloc_2(sizeof(struct vlist_t) + vhdr.siz);
308 		memcpy(vrebuild, &vhdr, sizeof(struct vlist_t));
309 		rc = fread(vrebuild+1, 1, vrebuild->siz, GLOBALS->vlist_handle);
310 		if(!rc)
311 			{
312 			printf("Error in reading from VList spill file!\n");
313 			exit(255);
314 			}
315 
316 		if(vprev)
317 			{
318 			vprev->next = vrebuild;
319 			}
320 			else
321 			{
322 			*v = vrebuild;
323 			}
324 
325 		vprev = vrebuild;
326 		vl = vhdr.next;
327 		}
328 
329 	vl = *v;
330 	vprev = NULL;
331 	}
332 
333 while(vl)
334 	{
335 	if((int)vl->offs < 0)
336 		{
337 		struct vlist_t *vz = malloc_2(sizeof(struct vlist_t) + vl->siz);
338 		unsigned int *ipnt;
339 		unsigned long sourcelen, destlen;
340 		int rc;
341 
342 		memcpy(vz, vl, sizeof(struct vlist_t));
343 		vz->offs = (unsigned int)(-(int)vl->offs);
344 
345 		ipnt = (unsigned int *)(vl + 1);
346 		sourcelen = (unsigned long)ipnt[0];
347 		destlen = (unsigned long)vl->siz;
348 
349 		rc = uncompress((unsigned char *)(vz+1), &destlen, (unsigned char *)&ipnt[1], sourcelen);
350 		if(rc != Z_OK)
351 			{
352 			fprintf(stderr, "Error in vlist uncompress(), rc=%d/destlen=%d exiting!\n", rc, (int)destlen);
353 			exit(255);
354 			}
355 
356 		free_2(vl);
357 		vl = vz;
358 
359 		if(vprev)
360 			{
361 			vprev->next = vz;
362 			}
363 			else
364 			{
365 			*v = vz;
366 			}
367 		}
368 
369 	vprev = vl;
370 	vl = vl->next;
371 	}
372 
373 }
374 
375 
376 /* get pointer to one unit of space
377  */
vlist_alloc(struct vlist_t ** v,int compressable)378 void *vlist_alloc(struct vlist_t **v, int compressable)
379 {
380 struct vlist_t *vl = *v;
381 char *px;
382 struct vlist_t *v2;
383 
384 if(vl->offs == vl->siz)
385 	{
386 	unsigned int siz, rsiz;
387 
388 	/* 2 times versions are the growable, indexable vlists */
389 	siz = 2 * vl->siz;
390 
391 	rsiz = sizeof(struct vlist_t) + (vl->siz * vl->elem_siz);
392 
393 	if((compressable)&&(vl->elem_siz == 1))
394 		{
395 		if(GLOBALS->vlist_compression_depth>=0)
396 			{
397 			vl = vlist_compress_block(vl, &rsiz);
398 			}
399 		}
400 
401 	if(compressable && GLOBALS->vlist_handle)
402 		{
403 		size_t rc;
404 		intptr_t write_cnt;
405 
406 		fseeko(GLOBALS->vlist_handle, GLOBALS->vlist_bytes_written, SEEK_SET);
407 
408 		if(GLOBALS->use_fastload)
409 			{
410 			rc = vlist_fwrite(vl, rsiz, GLOBALS->vlist_handle);
411 			}
412 			else
413 			{
414 			rc = fwrite(vl, rsiz, 1, GLOBALS->vlist_handle);
415 			}
416 
417 		if(!rc)
418 			{
419 			fprintf(stderr, "Error in writing to VList spill file!\n");
420 			perror("Why");
421 			exit(255);
422 			}
423 
424 		write_cnt = GLOBALS->vlist_bytes_written;
425 		if(sizeof(long) != sizeof(off_t))	/* optimizes in or out at compile time */
426 			{
427 			if(write_cnt != GLOBALS->vlist_bytes_written)
428 				{
429 				fprintf(stderr, "VList spill file pointer-file overflow!\n");
430 				exit(255);
431 				}
432 			}
433 
434 		v2 = calloc_2(1, sizeof(struct vlist_t) + (vl->siz * vl->elem_siz));
435 		v2->siz = siz;
436 		v2->elem_siz = vl->elem_siz;
437 		v2->next = (struct vlist_t *)write_cnt;
438 		free_2(vl);
439 
440 		*v = v2;
441 		vl = *v;
442 
443 		if(GLOBALS->use_fastload)
444 			{
445 			GLOBALS->vlist_bytes_written += rc;
446 			}
447 			else
448 			{
449 			GLOBALS->vlist_bytes_written += rsiz;
450 			}
451 		}
452 		else
453 		{
454 		v2 = calloc_2(1, sizeof(struct vlist_t) + (vl->siz * vl->elem_siz));
455 		v2->siz = siz;
456 		v2->elem_siz = vl->elem_siz;
457 		v2->next = vl;
458 		*v = v2;
459 		vl = *v;
460 		}
461 	}
462 else
463 if(vl->offs*2 == vl->siz)
464 	{
465 	v2 = calloc_2(1, sizeof(struct vlist_t) + (vl->siz * vl->elem_siz));
466 	memcpy(v2, vl, sizeof(struct vlist_t) + (vl->siz/2 * vl->elem_siz));
467 	free_2(vl);
468 
469 	*v = v2;
470 	vl = *v;
471 	}
472 
473 px =(((char *)(vl)) + sizeof(struct vlist_t) + ((vl->offs++) * vl->elem_siz));
474 return((void *)px);
475 }
476 
477 
478 /* vlist_size() and vlist_locate() do not work properly on
479    compressed lists...you'll have to call vlist_uncompress() first!
480  */
vlist_size(struct vlist_t * v)481 unsigned int vlist_size(struct vlist_t *v)
482 {
483 return(v->siz - 1 + v->offs);
484 }
485 
486 
vlist_locate(struct vlist_t * v,unsigned int idx)487 void *vlist_locate(struct vlist_t *v, unsigned int idx)
488 {
489 unsigned int here = v->siz - 1;
490 unsigned int siz = here + v->offs; /* siz is the same as vlist_size() */
491 
492 if((!siz)||(idx>=siz)) return(NULL);
493 
494 while (idx < here)
495 	{
496 	v = v->next;
497 	here = v->siz - 1;
498 	}
499 
500 idx -= here;
501 
502 return((void *)(((char *)(v)) + sizeof(struct vlist_t) + (idx * v->elem_siz)));
503 }
504 
505 
506 /* calling this if you don't plan on adding any more elements will free
507    up unused space as well as compress final blocks (if enabled)
508  */
vlist_freeze(struct vlist_t ** v)509 void vlist_freeze(struct vlist_t **v)
510 {
511 struct vlist_t *vl = *v;
512 unsigned int siz = vl->offs;
513 unsigned int rsiz = sizeof(struct vlist_t) + (siz * vl->elem_siz);
514 
515 if((vl->elem_siz == 1)&&(siz))
516 	{
517 	struct vlist_t *w, *v2;
518 
519 	if(vl->offs*2 <= vl->siz) /* Electric Fence, change < to <= */
520 		{
521 		v2 = calloc_2(1, sizeof(struct vlist_t) + (vl->siz /* * vl->elem_siz */)); /* scan-build */
522 		memcpy(v2, vl, sizeof(struct vlist_t) + (vl->siz/2 /* * vl->elem_siz */)); /* scan-build */
523 		free_2(vl);
524 
525 		*v = v2;
526 		vl = *v;
527 		}
528 
529 	w = vlist_compress_block(vl, &rsiz);
530 	*v = w;
531 	}
532 else
533 if((siz != vl->siz)&&(!GLOBALS->vlist_handle))
534 	{
535 	struct vlist_t *w = malloc_2(rsiz);
536 	memcpy(w, vl, rsiz);
537 	free_2(vl);
538 	*v = w;
539 	}
540 
541 if(GLOBALS->vlist_handle)
542 	{
543 	size_t rc;
544 	intptr_t write_cnt;
545 
546 	vl = *v;
547 	fseeko(GLOBALS->vlist_handle, GLOBALS->vlist_bytes_written, SEEK_SET);
548 
549 	if(GLOBALS->use_fastload)
550 		{
551 		rc = vlist_fwrite(vl, rsiz, GLOBALS->vlist_handle);
552 		}
553 		else
554 		{
555 		rc = fwrite(vl, rsiz, 1, GLOBALS->vlist_handle);
556 		}
557 
558 	if(!rc)
559 		{
560 		fprintf(stderr, "Error in writing to VList spill file!\n");
561 		perror("Why");
562 		exit(255);
563 		}
564 
565 	write_cnt = GLOBALS->vlist_bytes_written;
566 	if(sizeof(long) != sizeof(off_t))	/* optimizes in or out at compile time */
567 		{
568 		if(write_cnt != GLOBALS->vlist_bytes_written)
569 			{
570 			fprintf(stderr, "VList spill file pointer-file overflow!\n");
571 			exit(255);
572 			}
573 		}
574 
575 	*v = (struct vlist_t *)write_cnt;
576 
577 	if(GLOBALS->use_fastload)
578 		{
579 		GLOBALS->vlist_bytes_written += rc;
580 		}
581 		else
582 		{
583 		GLOBALS->vlist_bytes_written += rsiz;
584 		}
585 
586 	free_2(vl);
587 	}
588 }
589 
590 
591 /* this code implements an LZ-based filter that can sit on top
592    of the vlists.  it uses a generic escape value of 0xff as
593    that is one that statistically occurs infrequently in value
594    change data fed into vlists.
595  */
596 
vlist_packer_emit_out(struct vlist_packer_t * p,unsigned char byt)597 void vlist_packer_emit_out(struct vlist_packer_t *p, unsigned char byt)
598 {
599 char *pnt;
600 
601 #ifdef WAVE_VLIST_PACKER_STATS
602 p->packed_bytes++;
603 #endif
604 
605 pnt = vlist_alloc(&p->v, 1);
606 *pnt = byt;
607 }
608 
609 
vlist_packer_emit_uv32(struct vlist_packer_t * p,unsigned int v)610 void vlist_packer_emit_uv32(struct vlist_packer_t *p, unsigned int v)
611 {
612 unsigned int nxt;
613 
614 while((nxt = v>>7))
615         {
616         vlist_packer_emit_out(p, v&0x7f);
617         v = nxt;
618         }
619 
620 vlist_packer_emit_out(p, (v&0x7f) | 0x80);
621 }
622 
623 
vlist_packer_emit_uv32rvs(struct vlist_packer_t * p,unsigned int v)624 void vlist_packer_emit_uv32rvs(struct vlist_packer_t *p, unsigned int v)
625 {
626 unsigned int nxt;
627 unsigned char buf[2 * sizeof(int)];
628 unsigned int idx = 0;
629 int i;
630 
631 while((nxt = v>>7))
632         {
633 	buf[idx++] = v&0x7f;
634         v = nxt;
635         }
636 
637 buf[idx] = (v&0x7f) | 0x80;
638 
639 for(i = idx; i >= 0; i--)
640 	{
641 	vlist_packer_emit_out(p, buf[i]);
642 	}
643 }
644 
645 
vlist_packer_alloc(struct vlist_packer_t * p,unsigned char byt)646 void vlist_packer_alloc(struct vlist_packer_t *p, unsigned char byt)
647 {
648 int i, j, k, l;
649 
650 p->unpacked_bytes++;
651 
652 if(!p->repcnt)
653 	{
654 top:
655 	for(i=0;i<WAVE_ZIVSRCH;i++)
656 		{
657 		if(p->buf[(p->bufpnt-i) & WAVE_ZIVMASK] == byt)
658 			{
659 			p->repdist = i;
660 			p->repcnt = 1;
661 
662 			p->repdist2 = p->repdist3 = p->repdist4 = 0;
663 
664 			for(j=i+WAVE_ZIVSKIP;j<WAVE_ZIVSRCH;j++)
665 				{
666 				if(p->buf[(p->bufpnt-j) & WAVE_ZIVMASK] == byt)
667 					{
668 					p->repdist2 = j;
669 					p->repcnt2 = 1;
670 
671 					for(k=j+WAVE_ZIVSKIP;k<WAVE_ZIVSRCH;k++)
672 						{
673 						if(p->buf[(p->bufpnt-k) & WAVE_ZIVMASK] == byt)
674 							{
675 							p->repdist3 = k;
676 							p->repcnt3 = 1;
677 
678 							for(l=k+WAVE_ZIVSKIP;l<WAVE_ZIVSRCH;l++)
679 								{
680 								if(p->buf[(p->bufpnt-l) & WAVE_ZIVMASK] == byt)
681 									{
682 									p->repdist4 = l;
683 									p->repcnt4 = 1;
684 									break;
685 									}
686 								}
687 							break;
688 							}
689 						}
690 					break;
691 					}
692 				}
693 
694 			p->bufpnt++;
695 			p->bufpnt &= WAVE_ZIVMASK;
696 			p->buf[p->bufpnt] = byt;
697 
698 			return;
699 			}
700 		}
701 
702 	p->bufpnt++;
703 	p->bufpnt &= WAVE_ZIVMASK;
704 	p->buf[p->bufpnt] = byt;
705 	vlist_packer_emit_out(p, byt);
706 	if(byt==WAVE_ZIVFLAG)
707 		{
708 		vlist_packer_emit_uv32(p, 0);
709 		}
710 	}
711 	else
712 	{
713 attempt2:
714 	if(p->buf[(p->bufpnt - p->repdist) & WAVE_ZIVMASK] == byt)
715 		{
716 		p->repcnt++;
717 
718 		if(p->repcnt2)
719 			{
720 			p->repcnt2 = ((p->buf[(p->bufpnt - p->repdist2) & WAVE_ZIVMASK] == byt)) ? p->repcnt2+1 : 0;
721 			}
722 		if(p->repcnt3)
723 			{
724 			p->repcnt3 = ((p->buf[(p->bufpnt - p->repdist3) & WAVE_ZIVMASK] == byt)) ? p->repcnt3+1 : 0;
725 			}
726 		if(p->repcnt4)
727 			{
728 			p->repcnt4 = ((p->buf[(p->bufpnt - p->repdist4) & WAVE_ZIVMASK] == byt)) ? p->repcnt4+1 : 0;
729 			}
730 
731 		p->bufpnt++;
732 		p->bufpnt &= WAVE_ZIVMASK;
733 		p->buf[p->bufpnt] = byt;
734 		}
735 		else
736 		{
737 		if(p->repcnt2)
738 			{
739 			p->repcnt = p->repcnt2;
740 			p->repdist = p->repdist2;
741 
742 			p->repcnt2 = p->repcnt3;
743 			p->repdist2 = p->repdist3;
744 
745 			p->repcnt3 = p->repcnt4;
746 			p->repdist3 = p->repdist4;
747 
748 			p->repcnt4 = 0;
749 			p->repdist4 = 0;
750 
751 			goto attempt2;
752 			}
753 
754 		if(p->repcnt3)
755 			{
756 			p->repcnt = p->repcnt3;
757 			p->repdist = p->repdist3;
758 
759 			p->repcnt2 = p->repcnt4;
760 			p->repdist2 = p->repdist4;
761 
762 			p->repcnt3 = 0;
763 			p->repdist3 = 0;
764 			p->repcnt4 = 0;
765 			p->repdist4 = 0;
766 
767 			goto attempt2;
768 			}
769 
770 		if(p->repcnt4)
771 			{
772 			p->repcnt = p->repcnt4;
773 			p->repdist = p->repdist4;
774 
775 			p->repcnt4 = 0;
776 			p->repdist4 = 0;
777 
778 			goto attempt2;
779 			}
780 
781 		if(p->repcnt > 2)
782 			{
783 			vlist_packer_emit_out(p, WAVE_ZIVFLAG);
784 			vlist_packer_emit_uv32(p, p->repcnt);  p->repcnt = 0;
785 			vlist_packer_emit_uv32(p, p->repdist);
786 			}
787 			else
788 			{
789 			if(p->repcnt == 2)
790 				{
791 				vlist_packer_emit_out(p, p->buf[(p->bufpnt-1) & WAVE_ZIVMASK]);
792 				if(p->buf[(p->bufpnt-1) & WAVE_ZIVMASK]==WAVE_ZIVFLAG)
793 					{
794 					vlist_packer_emit_uv32(p, 0);
795 					}
796 				}
797 
798 			vlist_packer_emit_out(p, p->buf[p->bufpnt & WAVE_ZIVMASK]);
799 			p->repcnt = 0;
800 			if(p->buf[p->bufpnt & WAVE_ZIVMASK]==WAVE_ZIVFLAG)
801 				{
802 				vlist_packer_emit_uv32(p, 0);
803 				}
804 			}
805 		goto top;
806 		}
807 	}
808 }
809 
810 
vlist_packer_finalize(struct vlist_packer_t * p)811 void vlist_packer_finalize(struct vlist_packer_t *p)
812 {
813 #ifdef WAVE_VLIST_PACKER_STATS
814 static guint64 pp = 0, upp = 0;
815 #endif
816 
817 if(p->repcnt)
818 	{
819 	if(p->repcnt > 2)
820 		{
821 		vlist_packer_emit_out(p, WAVE_ZIVFLAG);
822 		vlist_packer_emit_uv32(p, p->repcnt);  p->repcnt = 0;
823 		vlist_packer_emit_uv32(p, p->repdist);
824 		}
825 		else
826 		{
827 		if(p->repcnt == 2)
828 			{
829 			vlist_packer_emit_out(p, p->buf[(p->bufpnt-1) & WAVE_ZIVMASK]);
830 			if(p->buf[(p->bufpnt-1) & WAVE_ZIVMASK]==WAVE_ZIVFLAG)
831 				{
832 				vlist_packer_emit_uv32(p, 0);
833 				}
834 			}
835 
836 		vlist_packer_emit_out(p, p->buf[p->bufpnt & WAVE_ZIVMASK]);
837 		p->repcnt = 0;
838 		if(p->buf[p->bufpnt & WAVE_ZIVMASK]==WAVE_ZIVFLAG)
839 			{
840 			vlist_packer_emit_uv32(p, 0);
841 			}
842 		}
843 	}
844 
845 vlist_packer_emit_uv32rvs(p, p->unpacked_bytes); /* for malloc later during decompress */
846 
847 #ifdef WAVE_VLIST_PACKER_STATS
848 pp += p->packed_bytes;
849 upp += p->unpacked_bytes;
850 
851 printf("pack:%d orig:%d (%lld %lld %f)\n", p->packed_bytes, p->unpacked_bytes, pp, upp, (float)pp / (float)upp);
852 #endif
853 }
854 
855 
vlist_packer_create(void)856 struct vlist_packer_t *vlist_packer_create(void)
857 {
858 struct vlist_packer_t *vp = calloc_2(1, sizeof(struct vlist_packer_t));
859 vp->v = vlist_create(sizeof(char));
860 
861 return(vp);
862 }
863 
864 
vlist_packer_decompress(struct vlist_t * v,unsigned int * declen)865 unsigned char *vlist_packer_decompress(struct vlist_t *v, unsigned int *declen)
866 {
867 unsigned int list_size = vlist_size(v);
868 unsigned int top_of_packed_size = list_size-1;
869 unsigned char *chp;
870 unsigned int dec_size = 0;
871 unsigned int dec_size_cmp;
872 unsigned int shamt = 0;
873 unsigned char *mem, *dpnt;
874 unsigned int i, j, repcnt, dist;
875 
876 for(;;)
877 	{
878 	chp = vlist_locate(v, top_of_packed_size);
879 
880 	dec_size |= ((unsigned int)(*chp & 0x7f)) << shamt;
881 
882 	if(*chp & 0x80)
883 		{
884 		break;
885 		}
886 
887 	shamt+=7;
888 	top_of_packed_size--;
889 	}
890 
891 mem = calloc_2(1, WAVE_ZIVWRAP + dec_size);
892 dpnt = mem + WAVE_ZIVWRAP;
893 for(i=0;i<top_of_packed_size;i++)
894 	{
895 	chp = vlist_locate(v, i);
896 	if(*chp != WAVE_ZIVFLAG)
897 		{
898 		*(dpnt++) = *chp;
899 		continue;
900 		}
901 
902 	i++;
903 	repcnt = shamt = 0;
904 	for(;;)
905 		{
906 		chp = vlist_locate(v, i);
907 
908 		repcnt |= ((unsigned int)(*chp & 0x7f)) << shamt;
909 
910 		if(*chp & 0x80)
911 			{
912 			break;
913 			}
914 		i++;
915 
916 		shamt+=7;
917 		}
918 	if(repcnt == 0)
919 		{
920 		*(dpnt++) = WAVE_ZIVFLAG;
921 		continue;
922 		}
923 
924 	i++;
925 	dist = shamt = 0;
926 	for(;;)
927 		{
928 		chp = vlist_locate(v, i);
929 
930 		dist |= ((unsigned int)(*chp & 0x7f)) << shamt;
931 
932 		if(*chp & 0x80)
933 			{
934 			break;
935 			}
936 		i++;
937 
938 		shamt+=7;
939 		}
940 
941 	for(j=0;j<repcnt;j++)
942 		{
943 		*dpnt = *(dpnt - dist - 1);
944 		dpnt++;
945 		}
946 	}
947 
948 *declen = dec_size;
949 
950 dec_size_cmp = dpnt - mem - WAVE_ZIVWRAP;
951 if(dec_size != dec_size_cmp)
952 	{
953 	fprintf(stderr, "miscompare: decompressed vlist_packer length: %d vs %d bytes, exiting.\n", dec_size, dec_size_cmp);
954 	exit(255);
955 	}
956 
957 return(mem + WAVE_ZIVWRAP);
958 }
959 
960 
vlist_packer_decompress_destroy(char * mem)961 void vlist_packer_decompress_destroy(char *mem)
962 {
963 free_2(mem - WAVE_ZIVWRAP);
964 }
965 
966 
967