1 #include "stdinc.h"
2 #include "dat.h"
3 #include "fns.h"
4
5 /*
6 * disk structure conversion routines
7 */
8 #define U8GET(p) ((p)[0])
9 #define U16GET(p) (((p)[0]<<8)|(p)[1])
10 #define U32GET(p) ((u32int)(((p)[0]<<24)|((p)[1]<<16)|((p)[2]<<8)|(p)[3]))
11 #define U64GET(p) (((u64int)U32GET(p)<<32)|(u64int)U32GET((p)+4))
12
13 #define U8PUT(p,v) (p)[0]=(v)&0xFF
14 #define U16PUT(p,v) (p)[0]=((v)>>8)&0xFF;(p)[1]=(v)&0xFF
15 #define U32PUT(p,v) (p)[0]=((v)>>24)&0xFF;(p)[1]=((v)>>16)&0xFF;(p)[2]=((v)>>8)&0xFF;(p)[3]=(v)&0xFF
16 #define U64PUT(p,v,t32) t32=(v)>>32;U32PUT(p,t32);t32=(v);U32PUT((p)+4,t32)
17
18 int debugarena = -1; /* hack to improve error reporting */
19
20 static struct {
21 u32int m;
22 char *s;
23 } magics[] = {
24 ArenaPartMagic, "ArenaPartMagic",
25 ArenaHeadMagic, "ArenaHeadMagic",
26 ArenaMagic, "ArenaMagic",
27 ISectMagic, "ISectMagic",
28 BloomMagic, "BloomMagic",
29 };
30
31 static char*
fmtmagic(char * s,u32int m)32 fmtmagic(char *s, u32int m)
33 {
34 int i;
35
36 for(i=0; i<nelem(magics); i++)
37 if(magics[i].m == m)
38 return magics[i].s;
39 sprint(s, "%#08ux", m);
40 return s;
41 }
42
43 u32int
unpackmagic(u8int * buf)44 unpackmagic(u8int *buf)
45 {
46 return U32GET(buf);
47 }
48
49 void
packmagic(u32int magic,u8int * buf)50 packmagic(u32int magic, u8int *buf)
51 {
52 U32PUT(buf, magic);
53 }
54
55 int
unpackarenapart(ArenaPart * ap,u8int * buf)56 unpackarenapart(ArenaPart *ap, u8int *buf)
57 {
58 u8int *p;
59 u32int m;
60 char fbuf[20];
61
62 p = buf;
63
64 m = U32GET(p);
65 if(m != ArenaPartMagic){
66 seterr(ECorrupt, "arena set has wrong magic number: %s expected ArenaPartMagic (%#lux)", fmtmagic(fbuf, m), ArenaPartMagic);
67 return -1;
68 }
69 p += U32Size;
70 ap->version = U32GET(p);
71 p += U32Size;
72 ap->blocksize = U32GET(p);
73 p += U32Size;
74 ap->arenabase = U32GET(p);
75 p += U32Size;
76
77 if(buf + ArenaPartSize != p)
78 sysfatal("unpackarenapart unpacked wrong amount");
79
80 return 0;
81 }
82
83 int
packarenapart(ArenaPart * ap,u8int * buf)84 packarenapart(ArenaPart *ap, u8int *buf)
85 {
86 u8int *p;
87
88 p = buf;
89
90 U32PUT(p, ArenaPartMagic);
91 p += U32Size;
92 U32PUT(p, ap->version);
93 p += U32Size;
94 U32PUT(p, ap->blocksize);
95 p += U32Size;
96 U32PUT(p, ap->arenabase);
97 p += U32Size;
98
99 if(buf + ArenaPartSize != p)
100 sysfatal("packarenapart packed wrong amount");
101
102 return 0;
103 }
104
105 int
unpackarena(Arena * arena,u8int * buf)106 unpackarena(Arena *arena, u8int *buf)
107 {
108 int sz;
109 u8int *p;
110 u32int m;
111 char fbuf[20];
112
113 p = buf;
114
115 m = U32GET(p);
116 if(m != ArenaMagic){
117 seterr(ECorrupt, "arena %d has wrong magic number: %s "
118 "expected ArenaMagic (%#lux)", debugarena,
119 fmtmagic(fbuf, m), ArenaMagic);
120 return -1;
121 }
122 p += U32Size;
123 arena->version = U32GET(p);
124 p += U32Size;
125 namecp(arena->name, (char*)p);
126 p += ANameSize;
127 arena->diskstats.clumps = U32GET(p);
128 p += U32Size;
129 arena->diskstats.cclumps = U32GET(p);
130 p += U32Size;
131 arena->ctime = U32GET(p);
132 p += U32Size;
133 arena->wtime = U32GET(p);
134 p += U32Size;
135 if(arena->version == ArenaVersion5){
136 arena->clumpmagic = U32GET(p);
137 p += U32Size;
138 }
139 arena->diskstats.used = U64GET(p);
140 p += U64Size;
141 arena->diskstats.uncsize = U64GET(p);
142 p += U64Size;
143 arena->diskstats.sealed = U8GET(p);
144 p += U8Size;
145 switch(arena->version){
146 case ArenaVersion4:
147 sz = ArenaSize4;
148 arena->clumpmagic = _ClumpMagic;
149 break;
150 case ArenaVersion5:
151 sz = ArenaSize5;
152 break;
153 default:
154 seterr(ECorrupt, "arena has bad version number %d", arena->version);
155 return -1;
156 }
157 /*
158 * Additional fields for the memstats version of the stats.
159 * Diskstats reflects what is committed to the index.
160 * Memstats reflects what is in the arena. Originally intended
161 * this to be a version 5 extension, but might as well use for
162 * all the existing version 4 arenas too.
163 *
164 * To maintain backwards compatibility with existing venti
165 * installations using the older format, we define that if
166 * memstats == diskstats, then the extension fields are not
167 * included (see packarena below). That is, only partially
168 * indexed arenas have these fields. Fully indexed arenas
169 * (in particular, sealed arenas) do not.
170 */
171 if(U8GET(p) == 1){
172 sz += ArenaSize5a-ArenaSize5;
173 p += U8Size;
174 arena->memstats.clumps = U32GET(p);
175 p += U32Size;
176 arena->memstats.cclumps = U32GET(p);
177 p += U32Size;
178 arena->memstats.used = U64GET(p);
179 p += U64Size;
180 arena->memstats.uncsize = U64GET(p);
181 p += U64Size;
182 arena->memstats.sealed = U8GET(p);
183 p += U8Size;
184
185 /*
186 * 2008/4/2
187 * Packarena (below) used to have a bug in which it would
188 * not zero out any existing extension fields when writing
189 * the arena metadata. This would manifest itself as arenas
190 * with arena->diskstats.sealed == 1 but arena->memstats.sealed == 0
191 * after a server restart. Because arena->memstats.sealed wouldn't
192 * be set, the server might try to fit another block into the arena
193 * (and succeed), violating the append-only structure of the log
194 * and invalidating any already-computed seal on the arena.
195 *
196 * It might end up that other fields in arena->memstats end up
197 * behind arena->diskstats too, but that would be considerably
198 * more rare, and the bug is fixed now. The case we need to
199 * handle is just the sealed mismatch.
200 *
201 * If we encounter such a bogus arena, fix the sealed field.
202 */
203 if(arena->diskstats.sealed)
204 arena->memstats.sealed = 1;
205 }else
206 arena->memstats = arena->diskstats;
207 if(buf + sz != p)
208 sysfatal("unpackarena unpacked wrong amount");
209
210 return 0;
211 }
212
213 int
packarena(Arena * arena,u8int * buf)214 packarena(Arena *arena, u8int *buf)
215 {
216 return _packarena(arena, buf, 0);
217 }
218
219 int
_packarena(Arena * arena,u8int * buf,int forceext)220 _packarena(Arena *arena, u8int *buf, int forceext)
221 {
222 int sz;
223 u8int *p;
224 u32int t32;
225
226 switch(arena->version){
227 case ArenaVersion4:
228 sz = ArenaSize4;
229 if(arena->clumpmagic != _ClumpMagic)
230 fprint(2, "warning: writing old arena tail loses clump magic 0x%lux != 0x%lux\n",
231 (ulong)arena->clumpmagic, (ulong)_ClumpMagic);
232 break;
233 case ArenaVersion5:
234 sz = ArenaSize5;
235 break;
236 default:
237 sysfatal("packarena unknown version %d", arena->version);
238 return -1;
239 }
240
241 p = buf;
242
243 U32PUT(p, ArenaMagic);
244 p += U32Size;
245 U32PUT(p, arena->version);
246 p += U32Size;
247 namecp((char*)p, arena->name);
248 p += ANameSize;
249 U32PUT(p, arena->diskstats.clumps);
250 p += U32Size;
251 U32PUT(p, arena->diskstats.cclumps);
252 p += U32Size;
253 U32PUT(p, arena->ctime);
254 p += U32Size;
255 U32PUT(p, arena->wtime);
256 p += U32Size;
257 if(arena->version == ArenaVersion5){
258 U32PUT(p, arena->clumpmagic);
259 p += U32Size;
260 }
261 U64PUT(p, arena->diskstats.used, t32);
262 p += U64Size;
263 U64PUT(p, arena->diskstats.uncsize, t32);
264 p += U64Size;
265 U8PUT(p, arena->diskstats.sealed);
266 p += U8Size;
267
268 /*
269 * Extension fields; see above.
270 */
271 if(forceext
272 || arena->memstats.clumps != arena->diskstats.clumps
273 || arena->memstats.cclumps != arena->diskstats.cclumps
274 || arena->memstats.used != arena->diskstats.used
275 || arena->memstats.uncsize != arena->diskstats.uncsize
276 || arena->memstats.sealed != arena->diskstats.sealed){
277 sz += ArenaSize5a - ArenaSize5;
278 U8PUT(p, 1);
279 p += U8Size;
280 U32PUT(p, arena->memstats.clumps);
281 p += U32Size;
282 U32PUT(p, arena->memstats.cclumps);
283 p += U32Size;
284 U64PUT(p, arena->memstats.used, t32);
285 p += U64Size;
286 U64PUT(p, arena->memstats.uncsize, t32);
287 p += U64Size;
288 U8PUT(p, arena->memstats.sealed);
289 p += U8Size;
290 }else{
291 /* Clear any extension fields already on disk. */
292 memset(p, 0, ArenaSize5a - ArenaSize5);
293 p += ArenaSize5a - ArenaSize5;
294 sz += ArenaSize5a - ArenaSize5;
295 }
296
297 if(buf + sz != p)
298 sysfatal("packarena packed wrong amount");
299
300 return 0;
301 }
302
303 int
unpackarenahead(ArenaHead * head,u8int * buf)304 unpackarenahead(ArenaHead *head, u8int *buf)
305 {
306 u8int *p;
307 u32int m;
308 int sz;
309 char fbuf[20];
310
311 p = buf;
312
313 m = U32GET(p);
314 if(m != ArenaHeadMagic){
315 seterr(ECorrupt, "arena %d head has wrong magic number: %s "
316 "expected ArenaHeadMagic (%#lux)", debugarena,
317 fmtmagic(fbuf, m), ArenaHeadMagic);
318 return -1;
319 }
320
321 p += U32Size;
322 head->version = U32GET(p);
323 p += U32Size;
324 namecp(head->name, (char*)p);
325 p += ANameSize;
326 head->blocksize = U32GET(p);
327 p += U32Size;
328 head->size = U64GET(p);
329 p += U64Size;
330 if(head->version == ArenaVersion5){
331 head->clumpmagic = U32GET(p);
332 p += U32Size;
333 }
334
335 switch(head->version){
336 case ArenaVersion4:
337 sz = ArenaHeadSize4;
338 head->clumpmagic = _ClumpMagic;
339 break;
340 case ArenaVersion5:
341 sz = ArenaHeadSize5;
342 break;
343 default:
344 seterr(ECorrupt, "arena head has unexpected version %d", head->version);
345 return -1;
346 }
347
348 if(buf + sz != p)
349 sysfatal("unpackarenahead unpacked wrong amount");
350
351 return 0;
352 }
353
354 int
packarenahead(ArenaHead * head,u8int * buf)355 packarenahead(ArenaHead *head, u8int *buf)
356 {
357 u8int *p;
358 int sz;
359 u32int t32;
360
361 switch(head->version){
362 case ArenaVersion4:
363 sz = ArenaHeadSize4;
364 if(head->clumpmagic != _ClumpMagic)
365 fprint(2, "warning: writing old arena header loses clump magic 0x%lux != 0x%lux\n",
366 (ulong)head->clumpmagic, (ulong)_ClumpMagic);
367 break;
368 case ArenaVersion5:
369 sz = ArenaHeadSize5;
370 break;
371 default:
372 sysfatal("packarenahead unknown version %d", head->version);
373 return -1;
374 }
375
376 p = buf;
377
378 U32PUT(p, ArenaHeadMagic);
379 p += U32Size;
380 U32PUT(p, head->version);
381 p += U32Size;
382 namecp((char*)p, head->name);
383 p += ANameSize;
384 U32PUT(p, head->blocksize);
385 p += U32Size;
386 U64PUT(p, head->size, t32);
387 p += U64Size;
388 if(head->version == ArenaVersion5){
389 U32PUT(p, head->clumpmagic);
390 p += U32Size;
391 }
392 if(buf + sz != p)
393 sysfatal("packarenahead packed wrong amount");
394
395 return 0;
396 }
397
398 static int
checkclump(Clump * w)399 checkclump(Clump *w)
400 {
401 if(w->encoding == ClumpENone){
402 if(w->info.size != w->info.uncsize){
403 seterr(ECorrupt, "uncompressed wad size mismatch");
404 return -1;
405 }
406 }else if(w->encoding == ClumpECompress){
407 if(w->info.size >= w->info.uncsize){
408 seterr(ECorrupt, "compressed lump has inconsistent block sizes %d %d", w->info.size, w->info.uncsize);
409 return -1;
410 }
411 }else{
412 seterr(ECorrupt, "clump has illegal encoding");
413 return -1;
414 }
415
416 return 0;
417 }
418
419 int
unpackclump(Clump * c,u8int * buf,u32int cmagic)420 unpackclump(Clump *c, u8int *buf, u32int cmagic)
421 {
422 u8int *p;
423 u32int magic;
424
425 p = buf;
426 magic = U32GET(p);
427 if(magic != cmagic){
428 seterr(ECorrupt, "clump has bad magic number=%#8.8ux != %#8.8ux", magic, cmagic);
429 return -1;
430 }
431 p += U32Size;
432
433 c->info.type = vtfromdisktype(U8GET(p));
434 p += U8Size;
435 c->info.size = U16GET(p);
436 p += U16Size;
437 c->info.uncsize = U16GET(p);
438 p += U16Size;
439 scorecp(c->info.score, p);
440 p += VtScoreSize;
441
442 c->encoding = U8GET(p);
443 p += U8Size;
444 c->creator = U32GET(p);
445 p += U32Size;
446 c->time = U32GET(p);
447 p += U32Size;
448
449 if(buf + ClumpSize != p)
450 sysfatal("unpackclump unpacked wrong amount");
451
452 return checkclump(c);
453 }
454
455 int
packclump(Clump * c,u8int * buf,u32int magic)456 packclump(Clump *c, u8int *buf, u32int magic)
457 {
458 u8int *p;
459
460 p = buf;
461 U32PUT(p, magic);
462 p += U32Size;
463
464 U8PUT(p, vttodisktype(c->info.type));
465 p += U8Size;
466 U16PUT(p, c->info.size);
467 p += U16Size;
468 U16PUT(p, c->info.uncsize);
469 p += U16Size;
470 scorecp(p, c->info.score);
471 p += VtScoreSize;
472
473 U8PUT(p, c->encoding);
474 p += U8Size;
475 U32PUT(p, c->creator);
476 p += U32Size;
477 U32PUT(p, c->time);
478 p += U32Size;
479
480 if(buf + ClumpSize != p)
481 sysfatal("packclump packed wrong amount");
482
483 return checkclump(c);
484 }
485
486 void
unpackclumpinfo(ClumpInfo * ci,u8int * buf)487 unpackclumpinfo(ClumpInfo *ci, u8int *buf)
488 {
489 u8int *p;
490
491 p = buf;
492 ci->type = vtfromdisktype(U8GET(p));
493 p += U8Size;
494 ci->size = U16GET(p);
495 p += U16Size;
496 ci->uncsize = U16GET(p);
497 p += U16Size;
498 scorecp(ci->score, p);
499 p += VtScoreSize;
500
501 if(buf + ClumpInfoSize != p)
502 sysfatal("unpackclumpinfo unpacked wrong amount");
503 }
504
505 void
packclumpinfo(ClumpInfo * ci,u8int * buf)506 packclumpinfo(ClumpInfo *ci, u8int *buf)
507 {
508 u8int *p;
509
510 p = buf;
511 U8PUT(p, vttodisktype(ci->type));
512 p += U8Size;
513 U16PUT(p, ci->size);
514 p += U16Size;
515 U16PUT(p, ci->uncsize);
516 p += U16Size;
517 scorecp(p, ci->score);
518 p += VtScoreSize;
519
520 if(buf + ClumpInfoSize != p)
521 sysfatal("packclumpinfo packed wrong amount");
522 }
523
524 int
unpackisect(ISect * is,u8int * buf)525 unpackisect(ISect *is, u8int *buf)
526 {
527 u8int *p;
528 u32int m;
529 char fbuf[20];
530
531 p = buf;
532
533
534 m = U32GET(p);
535 if(m != ISectMagic){
536 seterr(ECorrupt, "index section has wrong magic number: %s expected ISectMagic (%#lux)",
537 fmtmagic(fbuf, m), ISectMagic);
538 return -1;
539 }
540 p += U32Size;
541 is->version = U32GET(p);
542 p += U32Size;
543 namecp(is->name, (char*)p);
544 p += ANameSize;
545 namecp(is->index, (char*)p);
546 p += ANameSize;
547 is->blocksize = U32GET(p);
548 p += U32Size;
549 is->blockbase = U32GET(p);
550 p += U32Size;
551 is->blocks = U32GET(p);
552 p += U32Size;
553 is->start = U32GET(p);
554 p += U32Size;
555 is->stop = U32GET(p);
556 p += U32Size;
557 if(buf + ISectSize1 != p)
558 sysfatal("unpackisect unpacked wrong amount");
559 is->bucketmagic = 0;
560 if(is->version == ISectVersion2){
561 is->bucketmagic = U32GET(p);
562 p += U32Size;
563 if(buf + ISectSize2 != p)
564 sysfatal("unpackisect unpacked wrong amount");
565 }
566
567 return 0;
568 }
569
570 int
packisect(ISect * is,u8int * buf)571 packisect(ISect *is, u8int *buf)
572 {
573 u8int *p;
574
575 p = buf;
576
577 U32PUT(p, ISectMagic);
578 p += U32Size;
579 U32PUT(p, is->version);
580 p += U32Size;
581 namecp((char*)p, is->name);
582 p += ANameSize;
583 namecp((char*)p, is->index);
584 p += ANameSize;
585 U32PUT(p, is->blocksize);
586 p += U32Size;
587 U32PUT(p, is->blockbase);
588 p += U32Size;
589 U32PUT(p, is->blocks);
590 p += U32Size;
591 U32PUT(p, is->start);
592 p += U32Size;
593 U32PUT(p, is->stop);
594 p += U32Size;
595 if(buf + ISectSize1 != p)
596 sysfatal("packisect packed wrong amount");
597 if(is->version == ISectVersion2){
598 U32PUT(p, is->bucketmagic);
599 p += U32Size;
600 if(buf + ISectSize2 != p)
601 sysfatal("packisect packed wrong amount");
602 }
603
604 return 0;
605 }
606
607 void
unpackientry(IEntry * ie,u8int * buf)608 unpackientry(IEntry *ie, u8int *buf)
609 {
610 u8int *p;
611
612 p = buf;
613
614 scorecp(ie->score, p);
615 p += VtScoreSize;
616 /* ie->wtime = U32GET(p); */
617 p += U32Size;
618 /* ie->train = U16GET(p); */
619 p += U16Size;
620 if(p - buf != IEntryAddrOff)
621 sysfatal("unpackentry bad IEntryAddrOff amount");
622 ie->ia.addr = U64GET(p);
623 if(ie->ia.addr>>56) print("%.8H => %llux\n", p, ie->ia.addr);
624 p += U64Size;
625 ie->ia.size = U16GET(p);
626 p += U16Size;
627 if(p - buf != IEntryTypeOff)
628 sysfatal("unpackientry bad IEntryTypeOff amount");
629 ie->ia.type = vtfromdisktype(U8GET(p));
630 p += U8Size;
631 ie->ia.blocks = U8GET(p);
632 p += U8Size;
633
634 if(p - buf != IEntrySize)
635 sysfatal("unpackientry unpacked wrong amount");
636 }
637
638 void
packientry(IEntry * ie,u8int * buf)639 packientry(IEntry *ie, u8int *buf)
640 {
641 u32int t32;
642 u8int *p;
643
644 p = buf;
645
646 scorecp(p, ie->score);
647 p += VtScoreSize;
648 U32PUT(p, 0); /* wtime */
649 p += U32Size;
650 U16PUT(p, 0); /* train */
651 p += U16Size;
652 U64PUT(p, ie->ia.addr, t32);
653 p += U64Size;
654 U16PUT(p, ie->ia.size);
655 p += U16Size;
656 U8PUT(p, vttodisktype(ie->ia.type));
657 p += U8Size;
658 U8PUT(p, ie->ia.blocks);
659 p += U8Size;
660
661 if(p - buf != IEntrySize)
662 sysfatal("packientry packed wrong amount");
663 }
664
665 void
unpackibucket(IBucket * b,u8int * buf,u32int magic)666 unpackibucket(IBucket *b, u8int *buf, u32int magic)
667 {
668 b->n = U16GET(buf);
669 b->data = buf + IBucketSize;
670 if(magic && magic != U32GET(buf+U16Size))
671 b->n = 0;
672 }
673
674 void
packibucket(IBucket * b,u8int * buf,u32int magic)675 packibucket(IBucket *b, u8int *buf, u32int magic)
676 {
677 U16PUT(buf, b->n);
678 U32PUT(buf+U16Size, magic);
679 }
680
681 void
packbloomhead(Bloom * b,u8int * buf)682 packbloomhead(Bloom *b, u8int *buf)
683 {
684 u8int *p;
685
686 p = buf;
687 U32PUT(p, BloomMagic);
688 U32PUT(p+4, BloomVersion);
689 U32PUT(p+8, b->nhash);
690 U32PUT(p+12, b->size);
691 }
692
693 int
unpackbloomhead(Bloom * b,u8int * buf)694 unpackbloomhead(Bloom *b, u8int *buf)
695 {
696 u8int *p;
697 u32int m;
698 char fbuf[20];
699
700 p = buf;
701
702 m = U32GET(p);
703 if(m != BloomMagic){
704 seterr(ECorrupt, "bloom filter has wrong magic number: %s expected BloomMagic (%#lux)", fmtmagic(fbuf, m), (ulong)BloomMagic);
705 return -1;
706 }
707 p += U32Size;
708
709 m = U32GET(p);
710 if(m != BloomVersion){
711 seterr(ECorrupt, "bloom filter has wrong version %ud expected %ud", (uint)m, (uint)BloomVersion);
712 return -1;
713 }
714 p += U32Size;
715
716 b->nhash = U32GET(p);
717 p += U32Size;
718
719 b->size = U32GET(p);
720 p += U32Size;
721 if(b->size < BloomHeadSize || b->size > MaxBloomSize || (b->size&(b->size-1))){
722 seterr(ECorrupt, "bloom filter has invalid size %#lux", b->size);
723 return -1;
724 }
725
726 if(buf + BloomHeadSize != p)
727 sysfatal("unpackarena unpacked wrong amount");
728
729 return 0;
730 }
731