1 /*\
2 |*| Parity Archive - A way to restore missing files in a set.
3 |*|
4 |*| Copyright (C) 2001 Willem Monsuwe (willem@stack.nl)
5 |*|
6 |*| File format by Stefan Wehlus -
7 |*| initial idea by Tobias Rieper, further suggestions by Kilroy Balore
8 |*|
9 |*| Read and write PAR files
10 \*/
11
12 #include <stdlib.h>
13 #include <stdio.h>
14 #include <ctype.h>
15 #include <errno.h>
16 #include <string.h>
17 #include "util.h"
18 #include "rwpar.h"
19 #include "fileops.h"
20 #include "rs.h"
21 #include "readoldpar.h"
22 #include "md5.h"
23 #include "backend.h"
24
25 /*\ Endianless fixing code \*/
26
27 static i64
read_i64(void * data)28 read_i64(void *data)
29 {
30 int i;
31 i64 r = 0;
32 u8 *ptr = data;
33
34 for (i = sizeof(i64); --i >= 0; ) {
35 r <<= 8;
36 r += (i64)ptr[i];
37 }
38 return r;
39 }
40
41 static u32
read_u32(void * data)42 read_u32(void *data)
43 {
44 int i;
45 u32 r = 0;
46 u8 *ptr = data;
47
48 for (i = sizeof(u32); --i >= 0; ) {
49 r <<= 8;
50 r += (u32)ptr[i];
51 }
52 return r;
53 }
54
55 /*\ Read N bytes of little-endian u16s \*/
56 static void
read_u16s(u16 * str,void * data,i64 n)57 read_u16s(u16 *str, void *data, i64 n)
58 {
59 u8 *ptr = data;
60 while (--n >= 0) {
61 *str = ptr[0] + (ptr[1] << 8);
62 str++;
63 ptr += 2;
64 }
65 }
66
67 static void
write_i64(i64 v,void * data)68 write_i64(i64 v, void *data)
69 {
70 size_t i;
71 u8 *ptr = data;
72
73 for (i = 0; i < sizeof(i64); i++) {
74 ptr[i] = v & 0xFF;
75 v >>= 8;
76 }
77 }
78
79 static void
write_u32(u32 v,void * data)80 write_u32(u32 v, void *data)
81 {
82 size_t i;
83 u8 *ptr = data;
84
85 for (i = 0; i < sizeof(u32); i++) {
86 ptr[i] = v & 0xFF;
87 v >>= 8;
88 }
89 }
90
91 /*\ Write N bytes of little-endian u16s \*/
92 static void
write_u16s(u16 * str,void * data,i64 n)93 write_u16s(u16 *str, void *data, i64 n)
94 {
95 u8 *ptr = data;
96 while (--n >= 0) {
97 ptr[0] = (*str) & 0xff;
98 ptr[1] = ((*str) >> 8) & 0xff;
99 str++;
100 ptr += 2;
101 }
102 }
103
104 /*\ Change endianness to host byte order
105 |*| NB: This is a fix in place. Don't call this twice!
106 \*/
107 static void
par_endian_read(par_t * par)108 par_endian_read(par_t *par)
109 {
110 par->version = read_u32(&par->version);
111 par->client = read_u32(&par->client);
112 par->vol_number = read_i64(&par->vol_number);
113 par->num_files = read_i64(&par->num_files);
114 par->file_list = read_i64(&par->file_list);
115 par->file_list_size = read_i64(&par->file_list_size);
116 par->data = read_i64(&par->data);
117 par->data_size = read_i64(&par->data_size);
118 }
119
120 static void
par_endian_write(par_t * par,void * data)121 par_endian_write(par_t *par, void *data)
122 {
123 par_t *p = (par_t *)data;
124 memcpy(p, par, PAR_FIX_HEAD_SIZE);
125 write_u32(par->version, &p->version);
126 write_u32(par->client, &p->client);
127 write_i64(par->vol_number, &p->vol_number);
128 write_i64(par->num_files, &p->num_files);
129 write_i64(par->file_list, &p->file_list);
130 write_i64(par->file_list_size, &p->file_list_size);
131 write_i64(par->data, &p->data);
132 write_i64(par->data_size, &p->data_size);
133 }
134
135 static u16 uni_empty[] = { 0 };
136
137 static i64
uni_sizeof(u16 * str)138 uni_sizeof(u16 *str)
139 {
140 i64 l;
141 for (l = 0; str[l]; l++)
142 ;
143 return (2 * l);
144 }
145
146 /*\
147 |*| Return a pointer just past the last occurrence of '/' in a unicode string
148 |*| (somewhat like strrchr)
149 \*/
150 static u16 *
uni_strip(u16 * str)151 uni_strip(u16 *str)
152 {
153 u16 *ret;
154
155 for (ret = str; *str; str++)
156 if (*str == DIR_SEP)
157 ret = str + 1;
158 return ret;
159 }
160
161 /*\
162 |*| Debugging output functions
163 \*/
164 static void
dump_file(pfile_t * file)165 dump_file(pfile_t *file)
166 {
167 fprintf(stderr,
168 " status: 0x%llx\n"
169 " file size: %lld\n"
170 " hash: %s\n",
171 file->status,
172 file->file_size,
173 stmd5(file->hash));
174 fprintf(stderr,
175 " 16k hash: %s\n",
176 stmd5(file->hash_16k));
177 fprintf(stderr,
178 " filename: %s\n",
179 stuni(file->filename));
180 }
181
182 void
dump_par(par_t * par)183 dump_par(par_t *par)
184 {
185 pfile_t *p;
186
187 fprintf(stderr, "PAR file dump:\n"
188 " filename: %s\n"
189 " version: 0x%04x\n"
190 " client: 0x%04x\n"
191 " control hash: %s\n",
192 stuni(par->filename),
193 par->version,
194 par->client,
195 stmd5(par->control_hash));
196 fprintf(stderr,
197 " set hash: %s\n",
198 stmd5(par->set_hash));
199 fprintf(stderr,
200 " volume number: %lld\n"
201 " number of files: %lld\n"
202 " file list: 0x%llx\n"
203 " file list size: 0x%llx\n"
204 " data: 0x%llx\n"
205 " data size: 0x%llx\n",
206 par->vol_number,
207 par->num_files,
208 par->file_list,
209 par->file_list_size,
210 par->data,
211 par->data_size);
212 if (!par->vol_number)
213 fprintf(stderr,
214 " comment: %s\n",
215 stuni(par->comment));
216 fprintf(stderr, "\nFiles:\n\n");
217 for (p = par->files; p; p = p->next)
218 dump_file(p);
219 }
220
221 /*\
222 |*| Read in a PAR file entry to a file struct
223 \*/
224 static i64
read_pfile(pfile_t * file,u8 * ptr,u16 * path,i64 pl)225 read_pfile(pfile_t *file, u8 *ptr, u16 *path, i64 pl)
226 {
227 i64 i, l;
228 pfile_entr_t *pf;
229
230 pf = ((pfile_entr_t *)ptr);
231
232 i = read_i64(&pf->size);
233 file->status = read_i64(&pf->status);
234 file->file_size = read_i64(&pf->file_size);
235 COPY(file->hash, pf->hash, sizeof(md5));
236 COPY(file->hash_16k, pf->hash_16k, sizeof(md5));
237 l = (i - FILE_ENTRY_FIX_SIZE) / 2;
238 NEW(file->filename, pl + l + 1);
239 COPY(file->filename, path, pl);
240 read_u16s(file->filename + pl, &pf->filename, l);
241 file->filename[l + pl] = 0;
242
243 return i;
244 }
245
246 /*\
247 |*| Make a list of pointers into a list of file entries
248 \*/
249 static pfile_t *
read_pfiles(file_t f,i64 size,u16 * path)250 read_pfiles(file_t f, i64 size, u16 *path)
251 {
252 pfile_t *files = 0, **fptr = &files;
253 u8 *buf;
254 i64 i, pl;
255
256 for (pl = i = 0; path[i]; i++)
257 if (path[i] == DIR_SEP)
258 pl = i + 1;
259
260 NEW(buf, size);
261 size = file_read(f, buf, size);
262
263 /*\ The list size is at the start of the block \*/
264 i = 0;
265 /*\ Loop over the entries; the size of an entry is at the start \*/
266 while (i < size) {
267 CNEW(*fptr, 1);
268 i += read_pfile(*fptr, buf + i, path, pl);
269 fptr = &((*fptr)->next);
270 }
271 free(buf);
272 return files;
273 }
274
275 /*\
276 |*| Create a new PAR file struct
277 \*/
278 par_t *
create_par_header(u16 * file,i64 vol)279 create_par_header(u16 *file, i64 vol)
280 {
281 par_t *par;
282
283 CNEW(par, 1);
284 par->magic = PAR_MAGIC;
285 par->version = 0x00010000;
286 par->client = 0x02000900;
287 par->vol_number = vol;
288 par->filename = unicode_copy(file);
289 par->comment = uni_empty;
290 par->control_hash_offset = 0x20;
291
292 return par;
293 }
294
295 /*\
296 |*| Read in a PAR file, and return it into a newly allocated struct
297 |*| (to be freed with free_par())
298 \*/
299 par_t *
read_par_header(u16 * file,int create,i64 vol,int silent)300 read_par_header(u16 *file, int create, i64 vol, int silent)
301 {
302 par_t par, *r;
303 char *path;
304
305 memset(&par, 0, sizeof(par));
306
307 hash_directory(stuni(file));
308 path = complete_path(stuni(file));
309
310 par.f = file_open(file, 0);
311 /*\ Read in the first part of the struct, it fits directly on top \*/
312 if (file_read(par.f, &par, PAR_FIX_HEAD_SIZE) < PAR_FIX_HEAD_SIZE) {
313 if (!create || (errno != ENOENT)) {
314 if (!silent)
315 perror("Error reading PAR file");
316 file_close(par.f);
317 return 0;
318 }
319 if (!vol) {
320 /*\ Guess volume number from last digits \*/
321 u16 *p;
322 for (p = file; *p; p++)
323 ;
324 while ((--p >= file) && (*p >= '0') && (*p <= '9'))
325 ;
326 while (*++p)
327 vol = vol * 10 + (*p - '0');
328 }
329 return create_par_header(file, vol);
330 }
331 /*\ Is it the right file type ? \*/
332 if (!IS_PAR(par)) {
333 if (is_old_par(&par))
334 if (file_seek(par.f, 0) >= 0)
335 return read_old_par(par.f, file, silent);
336 if (!silent)
337 fprintf(stderr, "%s: Not a PAR file\n", basename(file));
338 file_close(par.f);
339 return 0;
340 }
341 par_endian_read(&par);
342
343 par.control_hash_offset = 0x20;
344 par.filename = file;
345
346 if (!silent && !par_control_check(&par)) {
347 file_close(par.f);
348 return 0;
349 }
350
351 file_seek(par.f, par.file_list);
352
353 par.filename = make_uni_str(path);
354
355 /*\ Read in the filelist. \*/
356 par.files = read_pfiles(par.f, par.file_list_size, par.filename);
357
358 file_seek(par.f, par.data);
359
360 if (par.vol_number == 0) {
361 CNEW(par.comment, (par.data_size / 2) + 1);
362 file_read(par.f, par.comment, par.data_size);
363 file_close(par.f);
364 par.f = 0;
365 }
366
367 par.volumes = 0;
368
369 NEW(r, 1);
370 COPY(r, &par, 1);
371 if (cmd.loglevel > 1)
372 dump_par(r);
373 return r;
374 }
375
376 void
free_file_list(pfile_t * list)377 free_file_list(pfile_t *list)
378 {
379 pfile_t *next;
380
381 while (list) {
382 if (list->f) file_close(list->f);
383 if (list->fnrs) free(list->fnrs);
384 next = list->next;
385 free(list);
386 list = next;
387 }
388 }
389
390 void
free_par(par_t * par)391 free_par(par_t *par)
392 {
393 free_file_list(par->files);
394 free_file_list(par->volumes);
395 free(par->filename);
396 if (par->f) file_close(par->f);
397 free(par);
398 }
399
400 /*\
401 |*| Write out a PAR file entry from a file struct
402 \*/
403 static i64
write_pfile(pfile_t * file,pfile_entr_t * pf)404 write_pfile(pfile_t *file, pfile_entr_t *pf)
405 {
406 u16 *name;
407 i64 i;
408
409 name = uni_strip(file->filename);
410 i = uni_sizeof(name);
411
412 write_i64(FILE_ENTRY_FIX_SIZE + i, &pf->size);
413 write_i64(file->status, &pf->status);
414 write_i64(file->file_size, &pf->file_size);
415 COPY(pf->hash, file->hash, sizeof(md5));
416 COPY(pf->hash_16k, file->hash_16k, sizeof(md5));
417 write_u16s(name, &pf->filename, i / 2);
418 return FILE_ENTRY_FIX_SIZE + i;
419 }
420
421 /*\
422 |*| Write a list of file entries to a file
423 \*/
424 static i64
write_file_entries(file_t f,pfile_t * files)425 write_file_entries(file_t f, pfile_t *files)
426 {
427 i64 tot, t, m;
428 pfile_t *p;
429 pfile_entr_t *pfe;
430
431 tot = m = 0;
432 for (p = files; p; p = p->next) {
433 t = FILE_ENTRY_FIX_SIZE + uni_sizeof(uni_strip(p->filename));
434 tot += t;
435 if (m < t) m = t;
436 }
437 pfe = (pfile_entr_t *)malloc(m);
438 if (f) {
439 for (p = files; p; p = p->next) {
440 t = write_pfile(p, pfe);
441 file_write(f, pfe, t);
442 }
443 }
444 free(pfe);
445 return tot;
446 }
447
448 /*\
449 |*| Write out a PAR volume header
450 \*/
451 file_t
write_par_header(par_t * par)452 write_par_header(par_t *par)
453 {
454 file_t f;
455 par_t data;
456 pfile_t *p;
457 int i;
458 md5 *hashes;
459
460 /*\ Open output file, but check so we don't overwrite anything \*/
461 if (move_away(par->filename, ".old")) {
462 fprintf(stderr, " WRITE ERROR: %s: ",
463 basename(par->filename));
464 fprintf(stderr, "File exists\n");
465 return 0;
466 }
467 f = file_open(par->filename, 1);
468 if (!f) {
469 fprintf(stderr, " WRITE ERROR: %s: ",
470 basename(par->filename));
471 perror("");
472 return 0;
473 }
474 par->file_list = PAR_FIX_HEAD_SIZE;
475 par->file_list_size = write_file_entries(0, par->files);
476 par->data = par->file_list + par->file_list_size;
477
478 if (par->vol_number == 0) {
479 par->data_size = uni_sizeof(par->comment);
480 } else {
481 for (i = 0, p = par->files; p; p = p->next, i++) {
482 if (par->data_size < p->file_size)
483 par->data_size = p->file_size;
484 }
485 }
486 /*\ Calculate set hash \*/
487 par->num_files = 0;
488 for (i = 0, p = par->files; p; p = p->next) {
489 par->num_files++;
490 if (USE_FILE(p))
491 i++;
492 }
493 NEW(hashes, i);
494 for (i = 0, p = par->files; p; p = p->next) {
495 if (!USE_FILE(p))
496 continue;
497 COPY(hashes[i], p->hash, sizeof(md5));
498 i++;
499 }
500 md5_buffer((char *)hashes, i * sizeof(md5), par->set_hash);
501 free(hashes);
502
503 if (cmd.loglevel > 1)
504 dump_par(par);
505
506 par_endian_write(par, &data);
507
508 file_write(f, &data, PAR_FIX_HEAD_SIZE);
509 write_file_entries(f, par->files);
510
511 if (par->vol_number == 0) {
512 file_write(f, par->comment, par->data_size);
513 if (cmd.ctrl) {
514 if (!file_add_md5(f, 0x0010, 0x0020,
515 par->data + par->data_size))
516 {
517 fprintf(stderr, " ERROR: %s:",
518 basename(par->filename));
519 perror("");
520 fprintf(stderr, " %-40s - FAILED\n",
521 basename(par->filename));
522 file_close(f);
523 f = 0;
524 if (!cmd.keep) file_delete(par->filename);
525 }
526 }
527 if (f) file_close(f);
528 }
529 return f;
530 }
531
532 /*\
533 |*| Restore missing files with recovery volumes
534 \*/
535 int
restore_files(pfile_t * files,pfile_t * volumes,sub_t * sub)536 restore_files(pfile_t *files, pfile_t *volumes, sub_t *sub)
537 {
538 int N, M, i;
539 xfile_t *in, *out;
540 pfile_t *p, *v, **pp, **qq;
541 int fail = 0;
542 i64 size;
543 pfile_t *mis_f, *mis_v;
544 u16 *path;
545
546 /*\ Separate out missing files \*/
547 p = files;
548 size = 0;
549 pp = &files;
550 qq = &mis_f;
551 *pp = *qq = 0;
552 for (i = 1; p; p = p->next, i++) {
553 p->vol_number = i;
554 if (!USE_FILE(p))
555 continue;
556 if (p->file_size > size)
557 size = p->file_size;
558 if (!find_file(p, 0)) {
559 NEW(*qq, 1);
560 COPY(*qq, p, 1);
561 qq = &((*qq)->next);
562 *qq = 0;
563 } else {
564 NEW(*pp, 1);
565 COPY(*pp, p, 1);
566 (*pp)->next = 0;
567 pp = &((*pp)->next);
568 *pp = 0;
569 }
570 }
571
572 /*\ Separate out missing volumes \*/
573 p = volumes;
574 pp = &volumes;
575 qq = &mis_v;
576 *pp = *qq = 0;
577 for (; p; p = p->next) {
578 if (p->vol_number && !(p->f)) {
579 NEW(*qq, 1);
580 COPY(*qq, p, 1);
581 qq = &((*qq)->next);
582 *qq = 0;
583 } else {
584 NEW(*pp, 1);
585 COPY(*pp, p, 1);
586 pp = &((*pp)->next);
587 *pp = 0;
588 }
589 }
590
591 /*\ Count existing files and volumes \*/
592 for (N = 0, p = files; p; p = p->next)
593 N++;
594 for (v = volumes; v; v = v->next, N++)
595 N++;
596
597 /*\ Count missing files and volumes \*/
598 for (M = 0, p = mis_f; p; p = p->next)
599 M++;
600 for (v = mis_v; v; v = v->next, N++)
601 M++;
602
603 NEW(in, N + 1);
604 NEW(out, M + 1);
605
606 /*\ Fill in input files \*/
607 for (i = 0, p = files; p; p = p->next) {
608 p->f = file_open(p->match->filename, 0);
609 if (!p->f) {
610 fprintf(stderr, " ERROR: %s:",
611 basename(p->match->filename));
612 perror("");
613 continue;
614 }
615 in[i].filenr = p->vol_number;
616 in[i].files = 0;
617 in[i].size = p->file_size;
618 in[i].f = p->f;
619 i++;
620 }
621 /*\ Fill in input volumes \*/
622 for (v = volumes; v; v = v->next) {
623 in[i].filenr = v->vol_number;
624 in[i].files = v->fnrs;
625 in[i].size = v->file_size;
626 in[i].f = v->f;
627 i++;
628 }
629 in[i].filenr = 0;
630
631 /*\ Fill in output files \*/
632 for (i = 0, p = mis_f; p; p = p->next) {
633 path = do_sub(p->filename, sub);
634 /*\ Open output file, but check we don't overwrite anything \*/
635 if (move_away(path, ".bad")) {
636 fprintf(stderr, " ERROR: %s: ",
637 basename(path));
638 fprintf(stderr, "File exists\n");
639 fprintf(stderr, " %-40s - NOT RESTORED\n",
640 basename(path));
641 continue;
642 }
643 p->f = file_open(path, 1);
644 if (!p->f) {
645 fprintf(stderr, " ERROR: %s: ",
646 basename(path));
647 perror("");
648 fprintf(stderr, " %-40s - NOT RESTORED\n",
649 basename(path));
650 continue;
651 }
652 out[i].size = p->file_size;
653 out[i].filenr = p->vol_number;
654 out[i].files = 0;
655 out[i].f = p->f;
656 i++;
657 }
658
659 /*\ Fill in output volumes \*/
660 for (v = mis_v; v; v = v->next) {
661 par_t *par;
662
663 par = create_par_header(v->filename, v->vol_number);
664 if (!par) {
665 fprintf(stderr, " %-40s - FAILED\n",
666 basename(v->match->filename));
667 continue;
668 }
669 /*\ Copy file list into par file \*/
670 par->files = files;
671 par->data_size = size;
672 v->f = write_par_header(par);
673 par->files = 0;
674 if (!v->f) {
675 fprintf(stderr, " %-40s - FAILED\n",
676 basename(par->filename));
677 fail |= 1;
678 free_par(par);
679 continue;
680 }
681 v->match = hfile_add(par->filename);
682 v->filename = v->match->filename;
683 v->file_size = par->data + par->data_size;
684 out[i].size = par->data_size;
685 out[i].filenr = v->vol_number;
686 out[i].files = v->fnrs;
687 out[i].f = v->f;
688 free_par(par);
689 i++;
690 }
691 out[i].filenr = 0;
692
693 if (!recreate(in, out))
694 fail |= 1;
695
696 free(in);
697 free(out);
698
699 /*\ Check resulting data files \*/
700 for (p = mis_f; p; p = p->next) {
701 if (!p->f) continue;
702 file_close(p->f);
703 p->f = 0;
704 path = do_sub(p->filename, sub);
705 p->match = hfile_add(path);
706 if (!hash_file(p->match, HASH)) {
707 fprintf(stderr, " ERROR: %s:",
708 basename(path));
709 perror("");
710 fprintf(stderr, " %-40s - NOT RESTORED\n",
711 basename(path));
712 fail |= 1;
713 if (!cmd.keep) file_delete(path);
714 continue;
715 }
716 if ((p->match->file_size == 0) && (p->file_size != 0)) {
717 fprintf(stderr, " %-40s - NOT RESTORED\n",
718 basename(path));
719 fail |= 1;
720 if (!cmd.keep) file_delete(path);
721 continue;
722 }
723 if (!CMP_MD5(p->match->hash, p->hash)) {
724 fprintf(stderr, " ERROR: %s: Failed md5 check\n",
725 basename(path));
726 fprintf(stderr, " %-40s - NOT RESTORED\n",
727 basename(path));
728 fail |= 1;
729 if (!cmd.keep) file_delete(path);
730 continue;
731 }
732 fprintf(stderr, " %-40s - RECOVERED\n",
733 basename(path));
734 }
735
736 /*\ Check resulting volumes \*/
737 for (v = mis_v; v; v = v->next) {
738 if (!v->f) continue;
739 if (!file_add_md5(v->f, 0x0010, 0x0020, v->file_size)) {
740 fprintf(stderr, " %-40s - FAILED\n",
741 basename(v->filename));
742 fail |= 1;
743 file_close(v->f);
744 v->f = 0;
745 if (!cmd.keep) file_delete(v->filename);
746 continue;
747 }
748 fprintf(stderr, " %-40s - OK\n", basename(v->filename));
749 }
750
751 while ((p = files)) {
752 files = p->next;
753 free(p);
754 }
755 while ((p = volumes)) {
756 volumes = p->next;
757 free(p);
758 }
759 while ((p = mis_f)) {
760 mis_f = p->next;
761 free(p);
762 }
763 while ((p = mis_v)) {
764 mis_v = p->next;
765 free(p);
766 }
767
768 if (fail) {
769 fprintf(stderr, "\nErrors occurred.\n\n");
770 return -1;
771 }
772 return 1;
773 }
774