1 /*
2 * Copyright (C) 2011 Andrea Mazzoleni
3 *
4 * This program is free software: you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation, either version 3 of the License, or
7 * (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program. If not, see <http://www.gnu.org/licenses/>.
16 */
17
18 #include "portable.h"
19
20 #include "elem.h"
21 #include "import.h"
22 #include "search.h"
23 #include "state.h"
24 #include "support.h"
25 #include "parity.h"
26 #include "stream.h"
27 #include "handle.h"
28 #include "io.h"
29 #include "raid/raid.h"
30 #include "raid/cpu.h"
31
32 /**
33 * Configure the multithread support.
34 *
35 * Multi thread for write could be either faster or slower, depending
36 * on the specific conditions. With multithreads it's likely faster
37 * writing to disk, but you'll need to access multiple times the same data,
38 * being potentially slower.
39 *
40 * Multi thread for verify is instead always generally faster,
41 * so we enable it if possible.
42 */
43 #if HAVE_THREAD
44 /* #define HAVE_MT_WRITE 1 */
45 #define HAVE_MT_VERIFY 1
46 #endif
47
lev_name(unsigned l)48 const char* lev_name(unsigned l)
49 {
50 switch (l) {
51 case 0 : return "Parity";
52 case 1 : return "2-Parity";
53 case 2 : return "3-Parity";
54 case 3 : return "4-Parity";
55 case 4 : return "5-Parity";
56 case 5 : return "6-Parity";
57 }
58
59 return 0;
60 }
61
lev_config_name(unsigned l)62 const char* lev_config_name(unsigned l)
63 {
64 switch (l) {
65 case 0 : return "parity";
66 case 1 : return "2-parity";
67 case 2 : return "3-parity";
68 case 3 : return "4-parity";
69 case 4 : return "5-parity";
70 case 5 : return "6-parity";
71 }
72
73 return 0;
74 }
75
lev_config_scan(const char * s,unsigned * level,unsigned * mode)76 static int lev_config_scan(const char* s, unsigned* level, unsigned* mode)
77 {
78 if (strcmp(s, "parity") == 0 || strcmp(s, "1-parity") == 0) {
79 *level = 0;
80 return 0;
81 }
82
83 if (strcmp(s, "q-parity") == 0 || strcmp(s, "2-parity") == 0) {
84 *level = 1;
85 return 0;
86 }
87
88 if (strcmp(s, "r-parity") == 0 || strcmp(s, "3-parity") == 0) {
89 *level = 2;
90 return 0;
91 }
92
93 if (strcmp(s, "4-parity") == 0) {
94 *level = 3;
95 return 0;
96 }
97
98 if (strcmp(s, "5-parity") == 0) {
99 *level = 4;
100 return 0;
101 }
102
103 if (strcmp(s, "6-parity") == 0) {
104 *level = 5;
105 return 0;
106 }
107
108 if (strcmp(s, "z-parity") == 0) {
109 *level = 2;
110 if (mode)
111 *mode = RAID_MODE_VANDERMONDE;
112 return 0;
113 }
114
115 return -1;
116 }
117
lev_raid_name(unsigned mode,unsigned n)118 const char* lev_raid_name(unsigned mode, unsigned n)
119 {
120 switch (n) {
121 case 1 : return "par1";
122 case 2 : return "par2";
123 case 3 : if (mode == RAID_MODE_CAUCHY)
124 return "par3";
125 else
126 return "parz";
127 case 4 : return "par4";
128 case 5 : return "par5";
129 case 6 : return "par6";
130 }
131
132 return 0;
133 }
134
state_init(struct snapraid_state * state)135 void state_init(struct snapraid_state* state)
136 {
137 unsigned l, s;
138
139 memset(&state->opt, 0, sizeof(state->opt));
140 state->filter_hidden = 0;
141 state->autosave = 0;
142 state->need_write = 0;
143 state->checked_read = 0;
144 state->block_size = 256 * KIBI; /* default 256 KiB */
145 state->raid_mode = RAID_MODE_CAUCHY;
146 state->file_mode = ADVISE_DEFAULT;
147 for (l = 0; l < LEV_MAX; ++l) {
148 state->parity[l].split_mac = 0;
149 for (s = 0; s < SPLIT_MAX; ++s) {
150 state->parity[l].split_map[s].path[0] = 0;
151 state->parity[l].split_map[s].uuid[0] = 0;
152 state->parity[l].split_map[s].size = PARITY_SIZE_INVALID;
153 state->parity[l].split_map[s].device = 0;
154 }
155 state->parity[l].smartctl[0] = 0;
156 state->parity[l].total_blocks = 0;
157 state->parity[l].free_blocks = 0;
158 state->parity[l].skip_access = 0;
159 state->parity[l].tick = 0;
160 state->parity[l].cached_blocks = 0;
161 state->parity[l].is_excluded_by_filter = 0;
162 }
163 state->tick_io = 0;
164 state->tick_misc = 0;
165 state->tick_sched = 0;
166 state->tick_raid = 0;
167 state->tick_hash = 0;
168 state->tick_last = tick();
169 state->share[0] = 0;
170 state->pool[0] = 0;
171 state->pool_device = 0;
172 state->lockfile[0] = 0;
173 state->level = 1; /* default is the lowest protection */
174 state->clear_past_hash = 0;
175 state->no_conf = 0;
176
177 tommy_list_init(&state->disklist);
178 tommy_list_init(&state->maplist);
179 tommy_list_init(&state->contentlist);
180 tommy_list_init(&state->filterlist);
181 tommy_list_init(&state->importlist);
182 tommy_hashdyn_init(&state->importset);
183 tommy_hashdyn_init(&state->previmportset);
184 tommy_hashdyn_init(&state->searchset);
185 tommy_arrayblkof_init(&state->infoarr, sizeof(snapraid_info));
186 }
187
state_done(struct snapraid_state * state)188 void state_done(struct snapraid_state* state)
189 {
190 tommy_list_foreach(&state->disklist, (tommy_foreach_func*)disk_free);
191 tommy_list_foreach(&state->maplist, (tommy_foreach_func*)map_free);
192 tommy_list_foreach(&state->contentlist, (tommy_foreach_func*)content_free);
193 tommy_list_foreach(&state->filterlist, (tommy_foreach_func*)filter_free);
194 tommy_list_foreach(&state->importlist, (tommy_foreach_func*)import_file_free);
195 tommy_hashdyn_foreach(&state->searchset, (tommy_foreach_func*)search_file_free);
196 tommy_hashdyn_done(&state->importset);
197 tommy_hashdyn_done(&state->previmportset);
198 tommy_hashdyn_done(&state->searchset);
199 tommy_arrayblkof_done(&state->infoarr);
200 }
201
202 /**
203 * Check the configuration.
204 */
state_config_check(struct snapraid_state * state,const char * path,tommy_list * filterlist_disk)205 static void state_config_check(struct snapraid_state* state, const char* path, tommy_list* filterlist_disk)
206 {
207 tommy_node* i;
208 unsigned l, s;
209
210 /* check for parity level */
211 if (state->raid_mode == RAID_MODE_VANDERMONDE) {
212 if (state->level > 3) {
213 /* LCOV_EXCL_START */
214 log_fatal("If you use the z-parity you cannot have more than 3 parities.\n");
215 exit(EXIT_FAILURE);
216 /* LCOV_EXCL_STIO */
217 }
218 }
219
220 for (l = 0; l < state->level; ++l) {
221 if (state->parity[l].split_mac == 0) {
222 /* LCOV_EXCL_START */
223 log_fatal("No '%s' specification in '%s'\n", lev_config_name(l), path);
224 exit(EXIT_FAILURE);
225 /* LCOV_EXCL_STOP */
226 }
227 }
228
229 if (tommy_list_empty(&state->contentlist)) {
230 /* LCOV_EXCL_START */
231 log_fatal("No 'content' specification in '%s'\n", path);
232 exit(EXIT_FAILURE);
233 /* LCOV_EXCL_STOP */
234 }
235
236 /* check for equal paths */
237 for (i = state->contentlist; i != 0; i = i->next) {
238 struct snapraid_content* content = i->data;
239
240 for (l = 0; l < state->level; ++l) {
241 for (s = 0; s < state->parity[l].split_mac; ++s) {
242 if (pathcmp(state->parity[l].split_map[s].path, content->content) == 0) {
243 /* LCOV_EXCL_START */
244 log_fatal("Same path used for '%s' and 'content' as '%s'\n", lev_config_name(l), content->content);
245 exit(EXIT_FAILURE);
246 /* LCOV_EXCL_STOP */
247 }
248 }
249 }
250 }
251
252 /* check device of data disks */
253 if (!state->opt.skip_device && !state->opt.skip_disk_access) {
254 for (i = state->disklist; i != 0; i = i->next) {
255 tommy_node* j;
256 struct snapraid_disk* disk = i->data;
257
258 /* skip data disks that are not accessible */
259 if (disk->skip_access)
260 continue;
261
262 #ifdef _WIN32
263 if (disk->device == 0) {
264 /* LCOV_EXCL_START */
265 log_fatal("Disk '%s' has a zero serial number.\n", disk->dir);
266 log_fatal("This is not necessarily wrong, but for using SnapRAID\n");
267 log_fatal("it's better to change the serial number of the disk.\n");
268 log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'.\n");
269 exit(EXIT_FAILURE);
270 /* LCOV_EXCL_STOP */
271 }
272 #endif
273
274 for (j = i->next; j != 0; j = j->next) {
275 struct snapraid_disk* other = j->data;
276 if (disk->device == other->device) {
277 if (state->opt.force_device) {
278 /* note that we just ignore the issue */
279 /* and we DON'T mark the disk to be skipped */
280 /* because we want to use these disks */
281 if (!state->opt.no_warnings)
282 log_fatal("DANGER! Ignoring that disks '%s' and '%s' are on the same device\n", disk->name, other->name);
283 } else {
284 /* LCOV_EXCL_START */
285 log_fatal("Disks '%s' and '%s' are on the same device.\n", disk->dir, other->dir);
286 #ifdef _WIN32
287 log_fatal("Both have the serial number '%" PRIx64 "'.\n", disk->device);
288 log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'\n");
289 log_fatal("to change one of the disk serial.\n");
290 #endif
291 /* in "fix" we allow to continue anyway */
292 if (strcmp(state->command, "fix") == 0) {
293 log_fatal("You can '%s' anyway, using 'snapraid --force-device %s'.\n", state->command, state->command);
294 }
295 exit(EXIT_FAILURE);
296 /* LCOV_EXCL_STOP */
297 }
298 }
299 }
300
301 /* skip data disks that are not accessible */
302 if (disk->skip_access)
303 continue;
304
305 if (!state->opt.skip_parity_access) {
306 for (l = 0; l < state->level; ++l) {
307 for (s = 0; s < state->parity[l].split_mac; ++s) {
308 if (disk->device == state->parity[l].split_map[s].device) {
309 if (state->opt.force_device) {
310 /* note that we just ignore the issue */
311 /* and we DON'T mark the disk to be skipped */
312 /* because we want to use these disks */
313 if (!state->opt.no_warnings)
314 log_fatal("DANGER! Ignoring that disks '%s' and %s '%s' are on the same device\n", disk->dir, lev_name(l), state->parity[l].split_map[s].path);
315 } else {
316 /* LCOV_EXCL_START */
317 log_fatal("Disk '%s' and %s '%s' are on the same device.\n", disk->dir, lev_name(l), state->parity[l].split_map[s].path);
318 #ifdef _WIN32
319 log_fatal("Both have the serial number '%" PRIx64 "'.\n", disk->device);
320 log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'\n");
321 log_fatal("to change one of the disk serial.\n");
322 #endif
323 exit(EXIT_FAILURE);
324 /* LCOV_EXCL_STOP */
325 }
326 }
327 }
328 }
329 }
330
331 if (state->pool[0] != 0 && disk->device == state->pool_device) {
332 /* LCOV_EXCL_START */
333 log_fatal("Disk '%s' and pool '%s' are on the same device.\n", disk->dir, state->pool);
334 #ifdef _WIN32
335 log_fatal("Both have the serial number '%" PRIx64 "'.\n", disk->device);
336 log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'\n");
337 log_fatal("to change one of the disk serial.\n");
338 #endif
339 exit(EXIT_FAILURE);
340 /* LCOV_EXCL_STOP */
341 }
342 }
343 }
344
345 /* check device of parity disks */
346 if (!state->opt.skip_device && !state->opt.skip_parity_access) {
347 for (l = 0; l < state->level; ++l) {
348 for (s = 0; s < state->parity[l].split_mac; ++s) {
349 unsigned j, t;
350
351 /* skip parity disks that are not accessible */
352 if (state->parity[l].skip_access)
353 continue;
354
355 #ifdef _WIN32
356 if (state->parity[l].split_map[s].device == 0) {
357 /* LCOV_EXCL_START */
358 log_fatal("Disk '%s' has a zero serial number.\n", state->parity[l].split_map[s].path);
359 log_fatal("This is not necessarily wrong, but for using SnapRAID\n");
360 log_fatal("it's better to change the serial number of the disk.\n");
361 log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'.\n");
362 exit(EXIT_FAILURE);
363 /* LCOV_EXCL_STOP */
364 }
365 #endif
366
367 for (j = l + 1; j < state->level; ++j) {
368 for (t = 0; t < state->parity[j].split_mac; ++t) {
369 if (state->parity[l].split_map[s].device == state->parity[j].split_map[t].device) {
370 if (state->opt.force_device) {
371 /* note that we just ignore the issue */
372 /* and we DON'T mark the disk to be skipped */
373 /* because we want to use these disks */
374 if (!state->opt.no_warnings)
375 log_fatal("DANGER! Skipping parities '%s' and '%s' on the same device\n", lev_config_name(l), lev_config_name(j));
376 } else {
377 /* LCOV_EXCL_START */
378 log_fatal("Parity '%s' and '%s' are on the same device.\n", state->parity[l].split_map[s].path, state->parity[j].split_map[t].path);
379 #ifdef _WIN32
380 log_fatal("Both have the serial number '%" PRIx64 "'.\n", state->parity[l].split_map[s].device);
381 log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'\n");
382 log_fatal("to change one of the disk serial.\n");
383 #endif
384 /* in "fix" we allow to continue anyway */
385 if (strcmp(state->command, "fix") == 0) {
386 log_fatal("You can '%s' anyway, using 'snapraid --force-device %s'.\n", state->command, state->command);
387 }
388 exit(EXIT_FAILURE);
389 /* LCOV_EXCL_STOP */
390 }
391 }
392 }
393 }
394
395 if (state->pool[0] != 0 && state->pool_device == state->parity[l].split_map[s].device) {
396 /* LCOV_EXCL_START */
397 log_fatal("Pool '%s' and parity '%s' are on the same device.\n", state->pool, state->parity[l].split_map[s].path);
398 #ifdef _WIN32
399 log_fatal("Both have the serial number '%" PRIx64 "'.\n", state->pool_device);
400 log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'\n");
401 log_fatal("to change one of the disk serial.\n");
402 #endif
403 exit(EXIT_FAILURE);
404 /* LCOV_EXCL_STOP */
405 }
406 }
407 }
408 }
409
410 /* check device of pool disk */
411 #ifdef _WIN32
412 if (!state->opt.skip_device) {
413 if (state->pool[0] != 0 && state->pool_device == 0) {
414 /* LCOV_EXCL_START */
415 log_fatal("Disk '%s' has a zero serial number.\n", state->pool);
416 log_fatal("This is not necessarily wrong, but for using SnapRAID\n");
417 log_fatal("it's better to change the serial number of the disk.\n");
418 log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'.\n");
419 exit(EXIT_FAILURE);
420 /* LCOV_EXCL_STOP */
421 }
422 }
423 #endif
424
425 /* count the content files */
426 if (!state->opt.skip_device && !state->opt.skip_content_access) {
427 unsigned content_count;
428
429 content_count = 0;
430 for (i = state->contentlist; i != 0; i = i->next) {
431 tommy_node* j;
432 struct snapraid_content* content = i->data;
433
434 /* check if there are others in the same disk */
435 for (j = i->next; j != 0; j = j->next) {
436 struct snapraid_content* other = j->data;
437 if (content->device == other->device) {
438 log_fatal("WARNING! Content files on the same disk: '%s' and '%s'.\n", content->content, other->content);
439 break;
440 }
441 }
442 if (j != 0) {
443 /* skip it */
444 continue;
445 }
446
447 ++content_count;
448 }
449
450 if (content_count < state->level + 1) {
451 /* LCOV_EXCL_START */
452 log_fatal("You must have at least %d 'content' files in different disks.\n", state->level + 1);
453 exit(EXIT_FAILURE);
454 /* LCOV_EXCL_STOP */
455 }
456 }
457
458 /* check for speed */
459 #ifdef CONFIG_X86
460 if (!raid_cpu_has_ssse3())
461 #endif
462 if (state->raid_mode == RAID_MODE_CAUCHY && !state->opt.no_warnings) {
463 if (state->level == 3) {
464 log_fatal("WARNING! Your CPU doesn't have a fast implementation for triple parity.\n");
465 log_fatal("WARNING! It's recommended to switch to 'z-parity' instead than '3-parity'.\n");
466 } else if (state->level > 3) {
467 log_fatal("WARNING! Your CPU doesn't have a fast implementation beyond triple parity.\n");
468 log_fatal("WARNING! It's recommended to reduce the parity levels to triple parity.\n");
469 }
470 }
471
472 /* ensure that specified filter disks are valid ones */
473 for (i = tommy_list_head(filterlist_disk); i != 0; i = i->next) {
474 tommy_node* j;
475 struct snapraid_filter* filter = i->data;
476 for (j = state->disklist; j != 0; j = j->next) {
477 struct snapraid_disk* disk = j->data;
478 if (fnmatch(filter->pattern, disk->name, FNM_CASEINSENSITIVE_FOR_WIN) == 0)
479 break;
480 }
481 if (j == 0) {
482 /* check matching with parity disks */
483 for (l = 0; l < state->level; ++l)
484 if (fnmatch(filter->pattern, lev_config_name(l), FNM_CASEINSENSITIVE_FOR_WIN) == 0)
485 break;
486 if (l == state->level) {
487 /* LCOV_EXCL_START */
488 log_fatal("Option -d, --filter-disk %s doesn't match any data or parity disk.\n", filter->pattern);
489 exit(EXIT_FAILURE);
490 /* LCOV_EXCL_STOP */
491 }
492 }
493 }
494 }
495
496 /**
497 * Validate the smartctl command.
498 *
499 * It must contains only one %s string, and not other % chars.
500 */
validate_smartctl(const char * custom)501 static int validate_smartctl(const char* custom)
502 {
503 const char* s = custom;
504 int arg = 0;
505
506 while (*s) {
507 if (s[0] == '%' && s[1] == 's') {
508 if (arg) {
509 /* LCOV_EXCL_START */
510 return -1;
511 /* LCOV_EXCL_STOP */
512 }
513 arg = 1;
514 } else if (s[0] == '%') {
515 /* LCOV_EXCL_START */
516 return -1;
517 /* LCOV_EXCL_STOP */
518 }
519
520 ++s;
521 }
522
523 return 0;
524 }
525
state_config(struct snapraid_state * state,const char * path,const char * command,struct snapraid_option * opt,tommy_list * filterlist_disk)526 void state_config(struct snapraid_state* state, const char* path, const char* command, struct snapraid_option* opt, tommy_list* filterlist_disk)
527 {
528 STREAM* f;
529 unsigned line;
530 tommy_node* i;
531 unsigned l, s;
532
533 /* copy the options */
534 state->opt = *opt;
535
536 /* if unset, sort by physical order */
537 if (!state->opt.force_order)
538 state->opt.force_order = SORT_PHYSICAL;
539
540 /* adjust file mode */
541 if (state->opt.file_mode != ADVISE_DEFAULT) {
542 state->file_mode = state->opt.file_mode;
543 } else {
544 /* default mode, if nothing is specified */
545 state->file_mode = ADVISE_DISCARD;
546 }
547
548 /* store current command */
549 state->command = command;
550
551 log_tag("conf:file:%s\n", path);
552
553 f = sopen_read(path);
554 if (!f) {
555 /* LCOV_EXCL_START */
556 if (errno == ENOENT) {
557 if (state->opt.auto_conf) {
558 log_tag("conf:missing:\n");
559 /* mark that we are without a configuration file */
560 state->no_conf = 1;
561 state->level = 0;
562 return;
563 }
564
565 log_fatal("No configuration file found at '%s'\n", path);
566 } else if (errno == EACCES) {
567 log_fatal("You do not have rights to access the configuration file '%s'\n", path);
568 } else {
569 log_fatal("Error opening the configuration file '%s'. %s.\n", path, strerror(errno));
570 }
571 exit(EXIT_FAILURE);
572 /* LCOV_EXCL_STOP */
573 }
574
575 line = 1;
576 while (1) {
577 char tag[PATH_MAX];
578 char buffer[PATH_MAX];
579 int ret;
580 int c;
581 unsigned level;
582
583 /* skip initial spaces */
584 sgetspace(f);
585
586 /* read the command */
587 ret = sgettok(f, tag, sizeof(tag));
588 if (ret < 0) {
589 /* LCOV_EXCL_START */
590 log_fatal("Error reading the configuration file '%s' at line %u\n", path, line);
591 exit(EXIT_FAILURE);
592 /* LCOV_EXCL_STOP */
593 }
594
595 /* skip spaces after the command */
596 sgetspace(f);
597
598 if (strcmp(tag, "blocksize") == 0
599 /* block_size is the old format of the option */
600 || strcmp(tag, "block_size") == 0) {
601
602 ret = sgetu32(f, &state->block_size);
603 if (ret < 0) {
604 /* LCOV_EXCL_START */
605 log_fatal("Invalid 'blocksize' specification in '%s' at line %u\n", path, line);
606 exit(EXIT_FAILURE);
607 /* LCOV_EXCL_STOP */
608 }
609 if (state->block_size < 1) {
610 /* LCOV_EXCL_START */
611 log_fatal("Too small 'blocksize' specification in '%s' at line %u\n", path, line);
612 exit(EXIT_FAILURE);
613 /* LCOV_EXCL_STOP */
614 }
615 if (state->block_size > 16 * KIBI) {
616 /* LCOV_EXCL_START */
617 log_fatal("Too big 'blocksize' specification in '%s' at line %u\n", path, line);
618 exit(EXIT_FAILURE);
619 /* LCOV_EXCL_STOP */
620 }
621 /* check if it's a power of 2 */
622 if ((state->block_size & (state->block_size - 1)) != 0) {
623 /* LCOV_EXCL_START */
624 log_fatal("Not power of 2 'blocksize' specification in '%s' at line %u\n", path, line);
625 exit(EXIT_FAILURE);
626 /* LCOV_EXCL_STOP */
627 }
628 state->block_size *= KIBI;
629 } else if (strcmp(tag, "hashsize") == 0
630 || strcmp(tag, "hash_size") == 0 /* v11.0 used incorrectly this one, kept now for backward compatibility */
631 ) {
632 uint32_t hash_size;
633
634 ret = sgetu32(f, &hash_size);
635 if (ret < 0) {
636 /* LCOV_EXCL_START */
637 log_fatal("Invalid 'hashsize' specification in '%s' at line %u\n", path, line);
638 exit(EXIT_FAILURE);
639 /* LCOV_EXCL_STOP */
640 }
641 if (hash_size < 2) {
642 /* LCOV_EXCL_START */
643 log_fatal("Too small 'hashsize' specification in '%s' at line %u\n", path, line);
644 exit(EXIT_FAILURE);
645 /* LCOV_EXCL_STOP */
646 }
647 if (hash_size > HASH_MAX) {
648 /* LCOV_EXCL_START */
649 log_fatal("Too big 'hashsize' specification in '%s' at line %u\n", path, line);
650 exit(EXIT_FAILURE);
651 /* LCOV_EXCL_STOP */
652 }
653 /* check if it's a power of 2 */
654 if ((hash_size & (hash_size - 1)) != 0) {
655 /* LCOV_EXCL_START */
656 log_fatal("Not power of 2 'hashsize' specification in '%s' at line %u\n", path, line);
657 exit(EXIT_FAILURE);
658 /* LCOV_EXCL_STOP */
659 }
660
661 BLOCK_HASH_SIZE = hash_size;
662 } else if (lev_config_scan(tag, &level, &state->raid_mode) == 0) {
663 char device[PATH_MAX];
664 char* split_map[SPLIT_MAX + 1];
665 unsigned split_mac;
666 char* slash;
667 uint64_t dev;
668 int skip_access;
669
670 if (state->parity[level].split_mac != 0) {
671 /* LCOV_EXCL_START */
672 log_fatal("Multiple '%s' specification in '%s' at line %u\n", tag, path, line);
673 exit(EXIT_FAILURE);
674 /* LCOV_EXCL_STOP */
675 }
676
677 ret = sgetlasttok(f, buffer, sizeof(buffer));
678 if (ret < 0) {
679 /* LCOV_EXCL_START */
680 log_fatal("Invalid '%s' specification in '%s' at line %u\n", tag, path, line);
681 exit(EXIT_FAILURE);
682 /* LCOV_EXCL_STOP */
683 }
684
685 if (!*buffer) {
686 /* LCOV_EXCL_START */
687 log_fatal("Empty '%s' specification in '%s' at line %u\n", tag, path, line);
688 exit(EXIT_FAILURE);
689 /* LCOV_EXCL_STOP */
690 }
691
692 split_mac = strsplit(split_map, SPLIT_MAX + 1, buffer, ",");
693
694 if (split_mac > SPLIT_MAX) {
695 /* LCOV_EXCL_START */
696 log_fatal("Too many files in '%s' specification in '%s' at line %u\n", tag, path, line);
697 exit(EXIT_FAILURE);
698 /* LCOV_EXCL_STOP */
699 }
700
701 skip_access = 0;
702 state->parity[level].split_mac = split_mac;
703 for (s = 0; s < split_mac; ++s) {
704 pathimport(state->parity[level].split_map[s].path, sizeof(state->parity[level].split_map[s].path), split_map[s]);
705
706 if (!state->opt.skip_parity_access) {
707 struct stat st;
708
709 /* get the device of the directory containing the parity file */
710 pathimport(device, sizeof(device), split_map[s]);
711 slash = strrchr(device, '/');
712 if (slash)
713 *slash = 0;
714 else
715 pathcpy(device, sizeof(device), ".");
716
717 if (stat(device, &st) == 0) {
718 dev = st.st_dev;
719 } else {
720 /* if the disk can be skipped */
721 if (state->opt.force_device) {
722 /* use a fake device, and mark the disk to be skipped */
723 dev = 0;
724 skip_access = 1;
725 log_fatal("DANGER! Skipping inaccessible parity disk '%s'...\n", tag);
726 } else {
727 /* LCOV_EXCL_START */
728 log_fatal("Error accessing 'parity' dir '%s' specification in '%s' at line %u\n", device, path, line);
729
730 /* in "fix" we allow to continue anyway */
731 if (strcmp(state->command, "fix") == 0) {
732 log_fatal("You can '%s' anyway, using 'snapraid --force-device %s'.\n", state->command, state->command);
733 }
734 exit(EXIT_FAILURE);
735 /* LCOV_EXCL_STOP */
736 }
737 }
738 } else {
739 /* use a fake device */
740 dev = 0;
741 }
742
743 state->parity[level].split_map[s].device = dev;
744 }
745
746 /* store the global parity skip_access */
747 state->parity[level].skip_access = skip_access;
748
749 /* adjust the level */
750 if (state->level < level + 1)
751 state->level = level + 1;
752 } else if (strcmp(tag, "share") == 0) {
753 if (*state->share) {
754 /* LCOV_EXCL_START */
755 log_fatal("Multiple 'share' specification in '%s' at line %u\n", path, line);
756 exit(EXIT_FAILURE);
757 /* LCOV_EXCL_STOP */
758 }
759
760 ret = sgetlasttok(f, buffer, sizeof(buffer));
761 if (ret < 0) {
762 /* LCOV_EXCL_START */
763 log_fatal("Invalid 'share' specification in '%s' at line %u\n", path, line);
764 exit(EXIT_FAILURE);
765 /* LCOV_EXCL_STOP */
766 }
767
768 if (!*buffer) {
769 /* LCOV_EXCL_START */
770 log_fatal("Empty 'share' specification in '%s' at line %u\n", path, line);
771 exit(EXIT_FAILURE);
772 /* LCOV_EXCL_STOP */
773 }
774
775 pathimport(state->share, sizeof(state->share), buffer);
776 } else if (strcmp(tag, "pool") == 0) {
777 struct stat st;
778
779 if (*state->pool) {
780 /* LCOV_EXCL_START */
781 log_fatal("Multiple 'pool' specification in '%s' at line %u\n", path, line);
782 exit(EXIT_FAILURE);
783 /* LCOV_EXCL_STOP */
784 }
785
786 ret = sgetlasttok(f, buffer, sizeof(buffer));
787 if (ret < 0) {
788 /* LCOV_EXCL_START */
789 log_fatal("Invalid 'pool' specification in '%s' at line %u\n", path, line);
790 exit(EXIT_FAILURE);
791 /* LCOV_EXCL_STOP */
792 }
793
794 if (!*buffer) {
795 /* LCOV_EXCL_START */
796 log_fatal("Empty 'pool' specification in '%s' at line %u\n", path, line);
797 exit(EXIT_FAILURE);
798 /* LCOV_EXCL_STOP */
799 }
800
801 pathimport(state->pool, sizeof(state->pool), buffer);
802
803 /* get the device of the directory containing the pool tree */
804 if (stat(buffer, &st) != 0) {
805 /* LCOV_EXCL_START */
806 log_fatal("Error accessing 'pool' dir '%s' specification in '%s' at line %u\n", buffer, path, line);
807 exit(EXIT_FAILURE);
808 /* LCOV_EXCL_STOP */
809 }
810
811 state->pool_device = st.st_dev;
812 } else if (strcmp(tag, "content") == 0) {
813 struct snapraid_content* content;
814 char device[PATH_MAX];
815 char* slash;
816 struct stat st;
817 uint64_t dev;
818
819 ret = sgetlasttok(f, buffer, sizeof(buffer));
820 if (ret < 0) {
821 /* LCOV_EXCL_START */
822 log_fatal("Invalid 'content' specification in '%s' at line %u\n", path, line);
823 exit(EXIT_FAILURE);
824 /* LCOV_EXCL_STOP */
825 }
826
827 if (pathcmp(buffer, "/dev/null") == 0 || pathcmp(buffer, "NUL") == 0) {
828 /* LCOV_EXCL_START */
829 log_fatal("You cannot use the null device as 'content' specification in '%s' at line %u\n", path, line);
830 exit(EXIT_FAILURE);
831 /* LCOV_EXCL_STOP */
832 }
833
834 if (!*buffer) {
835 /* LCOV_EXCL_START */
836 log_fatal("Empty 'content' specification in '%s' at line %u\n", path, line);
837 exit(EXIT_FAILURE);
838 /* LCOV_EXCL_STOP */
839 }
840
841 /* check if the content file is already specified */
842 for (i = state->contentlist; i != 0; i = i->next) {
843 content = i->data;
844 if (pathcmp(content->content, buffer) == 0)
845 break;
846 }
847 if (i) {
848 /* LCOV_EXCL_START */
849 log_fatal("Duplicate 'content' specification in '%s' at line %u\n", path, line);
850 exit(EXIT_FAILURE);
851 /* LCOV_EXCL_STOP */
852 }
853
854 /* get the device of the directory containing the content file */
855 pathimport(device, sizeof(device), buffer);
856 slash = strrchr(device, '/');
857 if (slash)
858 *slash = 0;
859 else
860 pathcpy(device, sizeof(device), ".");
861 if (stat(device, &st) == 0) {
862 dev = st.st_dev;
863 } else {
864 if (state->opt.skip_content_access) {
865 /* use a fake device */
866 dev = 0;
867 log_fatal("WARNING! Skipping inaccessible content file '%s'...\n", buffer);
868 } else {
869 /* LCOV_EXCL_START */
870 log_fatal("Error accessing 'content' dir '%s' specification in '%s' at line %u\n", device, path, line);
871 exit(EXIT_FAILURE);
872 /* LCOV_EXCL_STOP */
873 }
874 }
875
876 /* set the lock file at the first accessible content file */
877 if (state->lockfile[0] == 0 && dev != 0) {
878 pathcpy(state->lockfile, sizeof(state->lockfile), buffer);
879 pathcat(state->lockfile, sizeof(state->lockfile), ".lock");
880 }
881
882 content = content_alloc(buffer, dev);
883
884 tommy_list_insert_tail(&state->contentlist, &content->node, content);
885 } else if (strcmp(tag, "data") == 0 || strcmp(tag, "disk") == 0) {
886 /* "disk" is the deprecated name up to SnapRAID 9.x */
887 char dir[PATH_MAX];
888 char device[PATH_MAX];
889 char uuid[UUID_MAX];
890 struct snapraid_disk* disk;
891 uint64_t dev;
892 int skip_access;
893
894 ret = sgettok(f, buffer, sizeof(buffer));
895 if (ret < 0) {
896 /* LCOV_EXCL_START */
897 log_fatal("Invalid 'data' name specification in '%s' at line %u\n", path, line);
898 exit(EXIT_FAILURE);
899 /* LCOV_EXCL_STOP */
900 }
901
902 if (!*buffer) {
903 /* LCOV_EXCL_START */
904 log_fatal("Empty 'data' name specification in '%s' at line %u\n", path, line);
905 exit(EXIT_FAILURE);
906 /* LCOV_EXCL_STOP */
907 }
908
909 sgetspace(f);
910
911 ret = sgetlasttok(f, dir, sizeof(dir));
912 if (ret < 0) {
913 /* LCOV_EXCL_START */
914 log_fatal("Invalid 'data' dir specification in '%s' at line %u\n", path, line);
915 exit(EXIT_FAILURE);
916 /* LCOV_EXCL_STOP */
917 }
918
919 if (!*dir) {
920 /* LCOV_EXCL_START */
921 log_fatal("Empty 'data' dir specification in '%s' at line %u\n", path, line);
922 exit(EXIT_FAILURE);
923 /* LCOV_EXCL_STOP */
924 }
925
926 /* get the device of the dir */
927 pathimport(device, sizeof(device), dir);
928
929 /* check if the disk name already exists */
930 for (i = state->disklist; i != 0; i = i->next) {
931 disk = i->data;
932 if (strcmp(disk->name, buffer) == 0)
933 break;
934 }
935 if (i) {
936 /* LCOV_EXCL_START */
937 log_fatal("Duplicate 'data' name '%s' at line %u\n", buffer, line);
938 exit(EXIT_FAILURE);
939 /* LCOV_EXCL_STOP */
940 }
941
942 /* if the disk has to be present */
943 skip_access = 0;
944 if (!state->opt.skip_disk_access) {
945 struct stat st;
946
947 if (stat(device, &st) == 0) {
948 dev = st.st_dev;
949
950 /* read the uuid, if unsupported use an empty one */
951 if (devuuid(dev, uuid, sizeof(uuid)) != 0) {
952 *uuid = 0;
953 }
954
955 /* fake a different UUID when testing */
956 if (state->opt.fake_uuid) {
957 snprintf(uuid, sizeof(uuid), "fake-uuid-%d", state->opt.fake_uuid);
958 --state->opt.fake_uuid;
959 }
960 } else {
961 /* if the disk can be skipped */
962 if (state->opt.force_device) {
963 /* use a fake device, and mark the disk to be skipped */
964 dev = 0;
965 *uuid = 0;
966 skip_access = 1;
967 log_fatal("DANGER! Skipping inaccessible data disk '%s'...\n", buffer);
968 } else {
969 /* LCOV_EXCL_START */
970 log_fatal("Error accessing 'disk' '%s' specification in '%s' at line %u\n", dir, device, line);
971
972 /* in "fix" we allow to continue anyway */
973 if (strcmp(state->command, "fix") == 0) {
974 log_fatal("You can '%s' anyway, using 'snapraid --force-device %s'.\n", state->command, state->command);
975 }
976 exit(EXIT_FAILURE);
977 /* LCOV_EXCL_STOP */
978 }
979 }
980 } else {
981 /* use a fake device */
982 dev = 0;
983 *uuid = 0;
984 }
985
986 disk = disk_alloc(buffer, dir, dev, uuid, skip_access);
987
988 tommy_list_insert_tail(&state->disklist, &disk->node, disk);
989 } else if (strcmp(tag, "smartctl") == 0) {
990 char custom[PATH_MAX];
991
992 ret = sgettok(f, buffer, sizeof(buffer));
993 if (ret < 0) {
994 /* LCOV_EXCL_START */
995 log_fatal("Invalid 'smartctl' name specification in '%s' at line %u\n", path, line);
996 exit(EXIT_FAILURE);
997 /* LCOV_EXCL_STOP */
998 }
999
1000 if (!*buffer) {
1001 /* LCOV_EXCL_START */
1002 log_fatal("Empty 'smartctl' name specification in '%s' at line %u\n", path, line);
1003 exit(EXIT_FAILURE);
1004 /* LCOV_EXCL_STOP */
1005 }
1006
1007 sgetspace(f);
1008
1009 ret = sgetlasttok(f, custom, sizeof(custom));
1010 if (ret < 0) {
1011 /* LCOV_EXCL_START */
1012 log_fatal("Invalid 'smartctl' option specification in '%s' at line %u\n", path, line);
1013 exit(EXIT_FAILURE);
1014 /* LCOV_EXCL_STOP */
1015 }
1016
1017 if (!*custom) {
1018 /* LCOV_EXCL_START */
1019 log_fatal("Empty 'smartctl' option specification in '%s' at line %u\n", path, line);
1020 exit(EXIT_FAILURE);
1021 /* LCOV_EXCL_STOP */
1022 }
1023
1024 if (validate_smartctl(custom) != 0) {
1025 /* LCOV_EXCL_START */
1026 log_fatal("Invalid 'smartctl' option specification in '%s' at line %u\n", path, line);
1027 exit(EXIT_FAILURE);
1028 /* LCOV_EXCL_STOP */
1029 }
1030
1031 /* search for parity */
1032 if (lev_config_scan(buffer, &level, 0) == 0) {
1033 if (state->parity[level].smartctl[0] != 0) {
1034 /* LCOV_EXCL_START */
1035 log_fatal("Duplicate parity smartctl '%s' at line %u\n", buffer, line);
1036 exit(EXIT_FAILURE);
1037 /* LCOV_EXCL_STOP */
1038 }
1039
1040 pathcpy(state->parity[level].smartctl, sizeof(state->parity[level].smartctl), custom);
1041 } else {
1042 struct snapraid_disk* disk;
1043
1044 /* search the disk */
1045 disk = 0;
1046 for (i = state->disklist; i != 0; i = i->next) {
1047 disk = i->data;
1048 if (strcmp(disk->name, buffer) == 0)
1049 break;
1050 }
1051 if (!disk) {
1052 /* LCOV_EXCL_START */
1053 log_fatal("Missing disk smartctl '%s' at line %u\n", buffer, line);
1054 exit(EXIT_FAILURE);
1055 /* LCOV_EXCL_STOP */
1056 }
1057 if (disk->smartctl[0] != 0) {
1058 /* LCOV_EXCL_START */
1059 log_fatal("Duplicate disk name '%s' at line %u\n", buffer, line);
1060 exit(EXIT_FAILURE);
1061 /* LCOV_EXCL_STOP */
1062 }
1063
1064 pathcpy(disk->smartctl, sizeof(disk->smartctl), custom);
1065 }
1066 } else if (strcmp(tag, "nohidden") == 0) {
1067 state->filter_hidden = 1;
1068 } else if (strcmp(tag, "exclude") == 0) {
1069 struct snapraid_filter* filter;
1070
1071 ret = sgetlasttok(f, buffer, sizeof(buffer));
1072 if (ret < 0) {
1073 /* LCOV_EXCL_START */
1074 log_fatal("Invalid 'exclude' specification in '%s' at line %u\n", path, line);
1075 exit(EXIT_FAILURE);
1076 /* LCOV_EXCL_STOP */
1077 }
1078
1079 if (!*buffer) {
1080 /* LCOV_EXCL_START */
1081 log_fatal("Empty 'exclude' specification in '%s' at line %u\n", path, line);
1082 exit(EXIT_FAILURE);
1083 /* LCOV_EXCL_STOP */
1084 }
1085
1086 filter = filter_alloc_file(-1, buffer);
1087 if (!filter) {
1088 /* LCOV_EXCL_START */
1089 log_fatal("Invalid 'exclude' specification '%s' in '%s' at line %u\n", buffer, path, line);
1090 log_fatal("Filters using relative paths are not supported. Ensure to add an initial slash\n");
1091 exit(EXIT_FAILURE);
1092 /* LCOV_EXCL_STOP */
1093 }
1094 tommy_list_insert_tail(&state->filterlist, &filter->node, filter);
1095 } else if (strcmp(tag, "include") == 0) {
1096 struct snapraid_filter* filter;
1097
1098 ret = sgetlasttok(f, buffer, sizeof(buffer));
1099 if (ret < 0) {
1100 /* LCOV_EXCL_START */
1101 log_fatal("Invalid 'include' specification in '%s' at line %u\n", path, line);
1102 exit(EXIT_FAILURE);
1103 /* LCOV_EXCL_STOP */
1104 }
1105
1106 if (!*buffer) {
1107 /* LCOV_EXCL_START */
1108 log_fatal("Empty 'include' specification in '%s' at line %u\n", path, line);
1109 exit(EXIT_FAILURE);
1110 /* LCOV_EXCL_STOP */
1111 }
1112
1113 filter = filter_alloc_file(1, buffer);
1114 if (!filter) {
1115 /* LCOV_EXCL_START */
1116 log_fatal("Invalid 'include' specification '%s' in '%s' at line %u\n", buffer, path, line);
1117 log_fatal("Filters using relative paths are not supported. Ensure to add an initial slash\n");
1118 exit(EXIT_FAILURE);
1119 /* LCOV_EXCL_STOP */
1120 }
1121 tommy_list_insert_tail(&state->filterlist, &filter->node, filter);
1122 } else if (strcmp(tag, "autosave") == 0) {
1123 char* e;
1124
1125 ret = sgetlasttok(f, buffer, sizeof(buffer));
1126 if (ret < 0) {
1127 /* LCOV_EXCL_START */
1128 log_fatal("Invalid 'autosave' specification in '%s' at line %u\n", path, line);
1129 exit(EXIT_FAILURE);
1130 /* LCOV_EXCL_STOP */
1131 }
1132
1133 if (!*buffer) {
1134 /* LCOV_EXCL_START */
1135 log_fatal("Empty 'autosave' specification in '%s' at line %u\n", path, line);
1136 exit(EXIT_FAILURE);
1137 /* LCOV_EXCL_STOP */
1138 }
1139
1140 state->autosave = strtoul(buffer, &e, 0);
1141
1142 if (!e || *e) {
1143 /* LCOV_EXCL_START */
1144 log_fatal("Invalid 'autosave' specification in '%s' at line %u\n", path, line);
1145 exit(EXIT_FAILURE);
1146 /* LCOV_EXCL_STOP */
1147 }
1148
1149 /* convert to GB */
1150 state->autosave *= GIGA;
1151 } else if (tag[0] == 0) {
1152 /* allow empty lines */
1153 } else if (tag[0] == '#') {
1154 ret = sgetline(f, buffer, sizeof(buffer));
1155 if (ret < 0) {
1156 /* LCOV_EXCL_START */
1157 log_fatal("Invalid comment in '%s' at line %u\n", path, line);
1158 exit(EXIT_FAILURE);
1159 /* LCOV_EXCL_STOP */
1160 }
1161 } else {
1162 /* LCOV_EXCL_START */
1163 log_fatal("Invalid command '%s' in '%s' at line %u\n", tag, path, line);
1164 exit(EXIT_FAILURE);
1165 /* LCOV_EXCL_STOP */
1166 }
1167
1168 /* skip final spaces */
1169 sgetspace(f);
1170
1171 /* next line */
1172 c = sgeteol(f);
1173 if (c == EOF) {
1174 break;
1175 }
1176 if (c != '\n') {
1177 /* LCOV_EXCL_START */
1178 log_fatal("Extra data in '%s' at line %u\n", path, line);
1179 exit(EXIT_FAILURE);
1180 /* LCOV_EXCL_STOP */
1181 }
1182 ++line;
1183 }
1184
1185 if (serror(f)) {
1186 /* LCOV_EXCL_START */
1187 log_fatal("Error reading the configuration file '%s' at line %u\n", path, line);
1188 exit(EXIT_FAILURE);
1189 /* LCOV_EXCL_STOP */
1190 }
1191
1192 sclose(f);
1193
1194 state_config_check(state, path, filterlist_disk);
1195
1196 /* select the default hash */
1197 if (state->opt.force_murmur3) {
1198 state->besthash = HASH_MURMUR3;
1199 } else if (state->opt.force_spooky2) {
1200 state->besthash = HASH_SPOOKY2;
1201 } else {
1202 #ifdef CONFIG_X86
1203 if (sizeof(void*) == 4 && !raid_cpu_has_slowmult())
1204 state->besthash = HASH_MURMUR3;
1205 else
1206 state->besthash = HASH_SPOOKY2;
1207 #else
1208 if (sizeof(void*) == 4)
1209 state->besthash = HASH_MURMUR3;
1210 else
1211 state->besthash = HASH_SPOOKY2;
1212 #endif
1213 }
1214
1215 /* by default use the best hash */
1216 state->hash = state->besthash;
1217
1218 /* by default use a random hash seed */
1219 if (randomize(state->hashseed, HASH_MAX) != 0) {
1220 /* LCOV_EXCL_START */
1221 log_fatal("Failed to get random values.\n");
1222 exit(EXIT_FAILURE);
1223 /* LCOV_EXCL_STOP */
1224 }
1225
1226 /* no previous hash by default */
1227 state->prevhash = HASH_UNDEFINED;
1228
1229 /* intentionally not set the prevhashseed, if used valgrind will warn about it */
1230
1231 log_tag("blocksize:%u\n", state->block_size);
1232 for (i = state->disklist; i != 0; i = i->next) {
1233 struct snapraid_disk* disk = i->data;
1234 log_tag("data:%s:%s\n", disk->name, disk->dir);
1235 }
1236
1237 log_tag("mode:%s\n", lev_raid_name(state->raid_mode, state->level));
1238 for (l = 0; l < state->level; ++l)
1239 for (s = 0; s < state->parity[l].split_mac; ++s)
1240 log_tag("%s:%u:%s\n", lev_config_name(l), s, state->parity[l].split_map[s].path);
1241 if (state->pool[0] != 0)
1242 log_tag("pool:%s\n", state->pool);
1243 if (state->share[0] != 0)
1244 log_tag("share:%s\n", state->share);
1245 if (state->autosave != 0)
1246 log_tag("autosave:%" PRIu64 "\n", state->autosave);
1247 for (i = tommy_list_head(&state->filterlist); i != 0; i = i->next) {
1248 char out[PATH_MAX];
1249 struct snapraid_filter* filter = i->data;
1250 log_tag("filter:%s\n", filter_type(filter, out, sizeof(out)));
1251 }
1252 if (state->filter_hidden)
1253 log_tag("filter:nohidden:\n");
1254 log_flush();
1255 }
1256
1257 /**
1258 * Find a disk by name.
1259 */
find_disk_by_name(struct snapraid_state * state,const char * name)1260 static struct snapraid_disk* find_disk_by_name(struct snapraid_state* state, const char* name)
1261 {
1262 tommy_node* i;
1263
1264 for (i = state->disklist; i != 0; i = i->next) {
1265 struct snapraid_disk* disk = i->data;
1266 if (strcmp(disk->name, name) == 0)
1267 return disk;
1268 }
1269
1270 if (state->no_conf) {
1271 /* without a configuration file, add disks automatically */
1272 struct snapraid_disk* disk;
1273
1274 disk = disk_alloc(name, "DUMMY/", -1, "", 0);
1275
1276 tommy_list_insert_tail(&state->disklist, &disk->node, disk);
1277
1278 return disk;
1279 }
1280
1281 return 0;
1282 }
1283
1284 /**
1285 * Find a disk by UUID.
1286 */
find_disk_by_uuid(struct snapraid_state * state,const char * uuid)1287 static struct snapraid_disk* find_disk_by_uuid(struct snapraid_state* state, const char* uuid)
1288 {
1289 tommy_node* i;
1290 struct snapraid_disk* found = 0;
1291
1292 /* special test case to find the first matching UUID */
1293 /* when testing UUID are all equal or not supported */
1294 /* and we should handle this case specifically */
1295 if (state->opt.match_first_uuid)
1296 return state->disklist->data;
1297
1298 /* LCOV_EXCL_START */
1299 /* never find an empty uuid */
1300 if (!*uuid)
1301 return 0;
1302
1303 for (i = state->disklist; i != 0; i = i->next) {
1304 struct snapraid_disk* disk = i->data;
1305 if (strcmp(disk->uuid, uuid) == 0) {
1306 /* never match duplicate UUID */
1307 if (found)
1308 return 0;
1309 found = disk;
1310 }
1311 }
1312
1313 return found;
1314 /* LCOV_EXCL_STOP */
1315 }
1316
1317 /**
1318 * Update the disk mapping if required.
1319 */
state_map(struct snapraid_state * state)1320 static void state_map(struct snapraid_state* state)
1321 {
1322 unsigned hole;
1323 tommy_node* i;
1324 unsigned uuid_mismatch;
1325 unsigned diskcount;
1326 unsigned l, s;
1327
1328 /* remove all the mapping without a disk */
1329 /* this happens when a disk is removed from the configuration file */
1330 /* From SnapRAID 4.0 mappings are automatically removed if a disk is not used */
1331 /* when saving the content file, but we keep this code to import older content files. */
1332 for (i = state->maplist; i != 0; ) {
1333 struct snapraid_map* map = i->data;
1334 struct snapraid_disk* disk;
1335
1336 disk = find_disk_by_name(state, map->name);
1337
1338 /* go to the next mapping before removing */
1339 i = i->next;
1340
1341 if (disk == 0) {
1342 /* disk not found, remove the mapping */
1343 tommy_list_remove_existing(&state->maplist, &map->node);
1344 map_free(map);
1345 }
1346 }
1347
1348 /* maps each unmapped disk present in the configuration file in the first available hole */
1349 /* this happens when you add disks for the first time in the configuration file */
1350 hole = 0; /* first position to try */
1351 for (i = state->disklist; i != 0; i = i->next) {
1352 struct snapraid_disk* disk = i->data;
1353 struct snapraid_map* map;
1354 tommy_node* j;
1355
1356 /* check if the disk is already mapped */
1357 for (j = state->maplist; j != 0; j = j->next) {
1358 map = j->data;
1359 if (strcmp(disk->name, map->name) == 0) {
1360 /* mapping found */
1361 break;
1362 }
1363 }
1364 if (j != 0) {
1365 /* mapping is present, then copy the free blocks into to disk */
1366 disk->total_blocks = map->total_blocks;
1367 disk->free_blocks = map->free_blocks;
1368 continue;
1369 }
1370
1371 /* mapping not found, search for an hole */
1372 while (1) {
1373 for (j = state->maplist; j != 0; j = j->next) {
1374 map = j->data;
1375 if (map->position == hole) {
1376 /* position already used */
1377 break;
1378 }
1379 }
1380 if (j == 0) {
1381 /* hole found */
1382 break;
1383 }
1384
1385 /* try with the next one */
1386 ++hole;
1387 }
1388
1389 /* insert the new mapping */
1390 map = map_alloc(disk->name, hole, 0, 0, "");
1391
1392 tommy_list_insert_tail(&state->maplist, &map->node, map);
1393 }
1394
1395 /* without configuration don't check for number of data disks or uuid changes */
1396 if (state->no_conf)
1397 return;
1398
1399 /* counter for the number of UUID mismatches */
1400 uuid_mismatch = 0;
1401
1402 /* check if mapping match the disk uuid */
1403 if (!state->opt.skip_disk_access) {
1404 for (i = state->maplist; i != 0; i = i->next) {
1405 struct snapraid_map* map = i->data;
1406 struct snapraid_disk* disk;
1407
1408 disk = find_disk_by_name(state, map->name);
1409 if (disk == 0) {
1410 /* LCOV_EXCL_START */
1411 log_fatal("Internal inconsistency for mapping '%s'\n", map->name);
1412 os_abort();
1413 /* LCOV_EXCL_STOP */
1414 }
1415
1416 if (disk->has_unsupported_uuid) {
1417 /* if uuid is not available, skip this one */
1418 continue;
1419 }
1420
1421 /* if the uuid is changed */
1422 if (strcmp(disk->uuid, map->uuid) != 0) {
1423 /* mark the disk as with an UUID change */
1424 disk->has_different_uuid = 1;
1425
1426 /* if the previous uuid is available */
1427 if (map->uuid[0] != 0) {
1428 /* count the number of uuid change */
1429 ++uuid_mismatch;
1430 log_fatal("UUID change for disk '%s' from '%s' to '%s'\n", disk->name, map->uuid, disk->uuid);
1431 } else {
1432 /* no message here, because having a disk without */
1433 /* UUID is the normal state of an empty disk */
1434 disk->had_empty_uuid = 1;
1435 }
1436
1437 /* update the uuid in the mapping, */
1438 pathcpy(map->uuid, sizeof(map->uuid), disk->uuid);
1439
1440 /* write the new state with the new uuid */
1441 state->need_write = 1;
1442 }
1443 }
1444 }
1445
1446 /* check the parity uuid */
1447 if (!state->opt.skip_parity_access) {
1448 for (l = 0; l < state->level; ++l) {
1449 for (s = 0; s < state->parity[l].split_mac; ++s) {
1450 char uuid[UUID_MAX];
1451 int ret;
1452
1453 ret = devuuid(state->parity[l].split_map[s].device, uuid, sizeof(uuid));
1454 if (ret != 0) {
1455 /* uuid not available, just ignore */
1456 continue;
1457 }
1458
1459 /* if the uuid is changed */
1460 if (strcmp(uuid, state->parity[l].split_map[s].uuid) != 0) {
1461 /* if the previous uuid is available */
1462 if (state->parity[l].split_map[s].uuid[0] != 0) {
1463 /* count the number of uuid change */
1464 ++uuid_mismatch;
1465 log_fatal("UUID change for parity '%s[%u]' from '%s' to '%s'\n", lev_config_name(l), s, state->parity[l].split_map[s].uuid, uuid);
1466 }
1467
1468 /* update the uuid */
1469 pathcpy(state->parity[l].split_map[s].uuid, sizeof(state->parity[l].split_map[s].uuid), uuid);
1470
1471 /* write the new state with the new uuid */
1472 state->need_write = 1;
1473 }
1474 }
1475 }
1476 }
1477
1478 if (!state->opt.force_uuid && uuid_mismatch > state->level) {
1479 /* LCOV_EXCL_START */
1480 log_fatal("Too many disks have UUID changed from the latest 'sync'.\n");
1481 log_fatal("If this happens because you really replaced them,\n");
1482 log_fatal("you can '%s' anyway, using 'snapraid --force-uuid %s'.\n", state->command, state->command);
1483 log_fatal("Instead, it's possible that you messed up the disk mount points,\n");
1484 log_fatal("and you have to restore the mount points at the state of the latest sync.\n");
1485 exit(EXIT_FAILURE);
1486 /* LCOV_EXCL_STOP */
1487 }
1488
1489 /* count the number of data disks, including holes left after removing some */
1490 diskcount = 0;
1491 for (i = state->maplist; i != 0; i = i->next) {
1492 struct snapraid_map* map = i->data;
1493
1494 if (map->position + 1 > diskcount)
1495 diskcount = map->position + 1;
1496 }
1497
1498 /* ensure to don't go over the limit of the RAID engine */
1499 if (diskcount > RAID_DATA_MAX) {
1500 /* LCOV_EXCL_START */
1501 log_fatal("Too many data disks. No more than %u.\n", RAID_DATA_MAX);
1502 exit(EXIT_FAILURE);
1503 /* LCOV_EXCL_STOP */
1504 }
1505
1506 /* now count the real number of data disks, excluding holes left after removing some */
1507 diskcount = tommy_list_count(&state->maplist);
1508
1509 /* recommend number of parities */
1510 if (!state->opt.no_warnings) {
1511 /* intentionally use log_fatal() instead of log_error() to give more visibility at the warning */
1512 if (diskcount >= 36 && state->level < 6) {
1513 log_fatal("WARNING! With %u disks it's recommended to use six parity levels.\n", diskcount);
1514 } else if (diskcount >= 29 && state->level < 5) {
1515 log_fatal("WARNING! With %u disks it's recommended to use five parity levels.\n", diskcount);
1516 } else if (diskcount >= 22 && state->level < 4) {
1517 log_fatal("WARNING! With %u disks it's recommended to use four parity levels.\n", diskcount);
1518 } else if (diskcount >= 15 && state->level < 3) {
1519 log_fatal("WARNING! With %u disks it's recommended to use three parity levels.\n", diskcount);
1520 } else if (diskcount >= 5 && state->level < 2) {
1521 log_fatal("WARNING! With %u disks it's recommended to use two parity levels.\n", diskcount);
1522 }
1523 }
1524 }
1525
state_refresh(struct snapraid_state * state)1526 void state_refresh(struct snapraid_state* state)
1527 {
1528 tommy_node* i;
1529 unsigned l, s;
1530
1531 /* for all disks */
1532 for (i = state->maplist; i != 0; i = i->next) {
1533 struct snapraid_map* map = i->data;
1534 struct snapraid_disk* disk;
1535 uint64_t total_space;
1536 uint64_t free_space;
1537 int ret;
1538
1539 disk = find_disk_by_name(state, map->name);
1540 if (disk == 0) {
1541 /* LCOV_EXCL_START */
1542 log_fatal("Internal inconsistency for mapping '%s'\n", map->name);
1543 os_abort();
1544 /* LCOV_EXCL_STOP */
1545 }
1546
1547 ret = fsinfo(disk->dir, 0, 0, &total_space, &free_space);
1548 if (ret != 0) {
1549 /* LCOV_EXCL_START */
1550 log_fatal("Error accessing disk '%s' to get file-system info. %s.\n", disk->dir, strerror(errno));
1551 exit(EXIT_FAILURE);
1552 /* LCOV_EXCL_STOP */
1553 }
1554
1555 /* set the new free blocks */
1556 map->total_blocks = total_space / state->block_size;
1557 map->free_blocks = free_space / state->block_size;
1558
1559 /* also update the disk info */
1560 disk->total_blocks = map->total_blocks;
1561 disk->free_blocks = map->free_blocks;
1562 }
1563
1564 /* for all parities */
1565 for (l = 0; l < state->level; ++l) {
1566 /* set the new free blocks */
1567 state->parity[l].total_blocks = 0;
1568 state->parity[l].free_blocks = 0;
1569
1570 for (s = 0; s < state->parity[l].split_mac; ++s) {
1571 uint64_t total_space;
1572 uint64_t free_space;
1573 int ret;
1574
1575 ret = fsinfo(state->parity[l].split_map[s].path, 0, 0, &total_space, &free_space);
1576 if (ret != 0) {
1577 /* LCOV_EXCL_START */
1578 log_fatal("Error accessing file '%s' to get file-system info. %s.\n", state->parity[l].split_map[s].path, strerror(errno));
1579 exit(EXIT_FAILURE);
1580 /* LCOV_EXCL_STOP */
1581 }
1582
1583 /* add the new free blocks */
1584 state->parity[l].total_blocks += total_space / state->block_size;
1585 state->parity[l].free_blocks += free_space / state->block_size;
1586 }
1587 }
1588
1589 /* note what we don't set need_write = 1, because we don't want */
1590 /* to update the content file only for the free space info. */
1591 }
1592
1593 /**
1594 * Check the content.
1595 */
state_content_check(struct snapraid_state * state,const char * path)1596 static void state_content_check(struct snapraid_state* state, const char* path)
1597 {
1598 tommy_node* i;
1599
1600 /* check that any map has different name and position */
1601 for (i = state->maplist; i != 0; i = i->next) {
1602 struct snapraid_map* map = i->data;
1603 tommy_node* j;
1604 for (j = i->next; j != 0; j = j->next) {
1605 struct snapraid_map* other = j->data;
1606 if (strcmp(map->name, other->name) == 0) {
1607 /* LCOV_EXCL_START */
1608 log_fatal("Colliding 'map' disk specification in '%s'\n", path);
1609 exit(EXIT_FAILURE);
1610 /* LCOV_EXCL_STOP */
1611 }
1612 if (map->position == other->position) {
1613 /* LCOV_EXCL_START */
1614 log_fatal("Colliding 'map' index specification in '%s'\n", path);
1615 exit(EXIT_FAILURE);
1616 /* LCOV_EXCL_STOP */
1617 }
1618 }
1619 }
1620 }
1621
1622 /**
1623 * Check if the position is REQUIRED, or we can completely clear it from the state.
1624 *
1625 * Note that position with only DELETED blocks are discharged.
1626 */
fs_position_is_required(struct snapraid_state * state,block_off_t pos)1627 static int fs_position_is_required(struct snapraid_state* state, block_off_t pos)
1628 {
1629 tommy_node* i;
1630
1631 /* check for each disk */
1632 for (i = state->disklist; i != 0; i = i->next) {
1633 struct snapraid_disk* disk = i->data;
1634 struct snapraid_block* block = fs_par2block_find(disk, pos);
1635
1636 /* if we have at least one file, the position is needed */
1637 if (block_has_file(block))
1638 return 1;
1639 }
1640
1641 return 0;
1642 }
1643
1644 /**
1645 * Check if the info block is REQUIREQ.
1646 *
1647 * This is used to ensure that we keep the last check used for scrubbing.
1648 * and that we add it when importing old context files.
1649 *
1650 * Note that you can have position without info blocks, for example
1651 * if all the blocks are not synced.
1652 *
1653 * Note also that not requiring an info block, doesn't mean that if present it
1654 * can be discarded.
1655 */
fs_info_is_required(struct snapraid_state * state,block_off_t pos)1656 static int fs_info_is_required(struct snapraid_state* state, block_off_t pos)
1657 {
1658 tommy_node* i;
1659
1660 /* check for each disk */
1661 for (i = state->disklist; i != 0; i = i->next) {
1662 struct snapraid_disk* disk = i->data;
1663 struct snapraid_block* block = fs_par2block_find(disk, pos);
1664
1665 /* if we have at least one synced file, the info is required */
1666 if (block_state_get(block) == BLOCK_STATE_BLK)
1667 return 1;
1668 }
1669
1670 return 0;
1671 }
1672
fs_position_clear_deleted(struct snapraid_state * state,block_off_t pos)1673 static void fs_position_clear_deleted(struct snapraid_state* state, block_off_t pos)
1674 {
1675 tommy_node* i;
1676
1677 /* check for each disk if block is really used */
1678 for (i = state->disklist; i != 0; i = i->next) {
1679 struct snapraid_disk* disk = i->data;
1680 struct snapraid_block* block = fs_par2block_find(disk, pos);
1681
1682 /* if the block is deleted */
1683 if (block_state_get(block) == BLOCK_STATE_DELETED) {
1684 /* set it to empty */
1685 fs_deallocate(disk, pos);
1686 }
1687 }
1688 }
1689
1690 /**
1691 * Check if a block position in a disk is deleted.
1692 */
fs_is_block_deleted(struct snapraid_disk * disk,block_off_t pos)1693 static int fs_is_block_deleted(struct snapraid_disk* disk, block_off_t pos)
1694 {
1695 struct snapraid_block* block = fs_par2block_find(disk, pos);
1696
1697 return block_state_get(block) == BLOCK_STATE_DELETED;
1698 }
1699
1700 /**
1701 * Flush the file checking the final CRC.
1702 * We exploit the fact that the CRC is always stored in the last 4 bytes.
1703 */
decoding_error(const char * path,STREAM * f)1704 static void decoding_error(const char* path, STREAM* f)
1705 {
1706 unsigned char buf[4];
1707 uint32_t crc_stored;
1708 uint32_t crc_computed;
1709
1710 if (seof(f)) {
1711 /* LCOV_EXCL_START */
1712 log_fatal("Unexpected end of content file '%s' at offset %" PRIi64 "\n", path, stell(f));
1713 log_fatal("This content file is truncated. Use an alternate copy.\n");
1714 exit(EXIT_FAILURE);
1715 /* LCOV_EXCL_STOP */
1716 }
1717
1718 if (serror(f)) {
1719 /* LCOV_EXCL_START */
1720 log_fatal("Error reading the content file '%s' at offset %" PRIi64 "\n", path, stell(f));
1721 exit(EXIT_FAILURE);
1722 /* LCOV_EXCL_STOP */
1723 }
1724
1725 log_fatal("Decoding error in '%s' at offset %" PRIi64 "\n", path, stell(f));
1726
1727 if (sdeplete(f, buf) != 0) {
1728 /* LCOV_EXCL_START */
1729 log_fatal("Error flushing the content file '%s' at offset %" PRIi64 "\n", path, stell(f));
1730 exit(EXIT_FAILURE);
1731 /* LCOV_EXCL_STOP */
1732 }
1733
1734 /* get the stored crc from the last four bytes */
1735 crc_stored = buf[0] | (uint32_t)buf[1] << 8 | (uint32_t)buf[2] << 16 | (uint32_t)buf[3] << 24;
1736
1737 /* get the computed crc */
1738 crc_computed = scrc(f);
1739
1740 /* adjust the stored crc to include itself */
1741 crc_stored = crc32c(crc_stored, buf, 4);
1742
1743 if (crc_computed != crc_stored) {
1744 log_fatal("Mismatching CRC in '%s'\n", path);
1745 log_fatal("This content file is damaged! Use an alternate copy.\n");
1746 exit(EXIT_FAILURE);
1747 } else {
1748 log_fatal("The file CRC is correct!\n");
1749 }
1750 }
1751
state_read_content(struct snapraid_state * state,const char * path,STREAM * f)1752 static void state_read_content(struct snapraid_state* state, const char* path, STREAM* f)
1753 {
1754 block_off_t blockmax;
1755 unsigned count_file;
1756 unsigned count_hardlink;
1757 unsigned count_symlink;
1758 unsigned count_dir;
1759 int crc_checked;
1760 char buffer[PATH_MAX];
1761 int ret;
1762 tommy_array disk_mapping;
1763 uint32_t mapping_max;
1764
1765 blockmax = 0;
1766 count_file = 0;
1767 count_hardlink = 0;
1768 count_symlink = 0;
1769 count_dir = 0;
1770 crc_checked = 0;
1771 mapping_max = 0;
1772 tommy_array_init(&disk_mapping);
1773
1774 ret = sread(f, buffer, 12);
1775 if (ret < 0) {
1776 /* LCOV_EXCL_START */
1777 decoding_error(path, f);
1778 log_fatal("Invalid header!\n");
1779 os_abort();
1780 /* LCOV_EXCL_STOP */
1781 }
1782
1783 /*
1784 * File format versions:
1785 * - SNAPCNT1/SnapRAID 4.0 First version.
1786 * - SNAPCNT2/SnapRAID 7.0 Adds entries 'M' and 'P', to add free_blocks support.
1787 * The previous 'm' entry is now deprecated, but supported for importing.
1788 * Similarly for text file, we add 'mapping' and 'parity' deprecating 'map'.
1789 * - SNAPCNT3/SnapRAID 11.0 Adds entry 'y' for hash size.
1790 * - SNAPCNT3/SnapRAID 11.0 Adds entry 'Q' for multi parity file.
1791 * The previous 'P' entry is now deprecated, but supported for importing.
1792 */
1793 if (memcmp(buffer, "SNAPCNT1\n\3\0\0", 12) != 0
1794 && memcmp(buffer, "SNAPCNT2\n\3\0\0", 12) != 0
1795 && memcmp(buffer, "SNAPCNT3\n\3\0\0", 12) != 0
1796 ) {
1797 /* LCOV_EXCL_START */
1798 if (memcmp(buffer, "SNAPCNT", 7) != 0) {
1799 decoding_error(path, f);
1800 log_fatal("Invalid header!\n");
1801 os_abort();
1802 } else {
1803 log_fatal("The content file '%s' was generated with a newer version of SnapRAID!\n", path);
1804 exit(EXIT_FAILURE);
1805 }
1806 /* LCOV_EXCL_STOP */
1807 }
1808
1809 while (1) {
1810 int c;
1811
1812 /* read the command */
1813 c = sgetc(f);
1814 if (c == EOF) {
1815 break;
1816 }
1817
1818 if (c == 'f') {
1819 /* file */
1820 char sub[PATH_MAX];
1821 uint64_t v_size;
1822 uint64_t v_mtime_sec;
1823 uint32_t v_mtime_nsec;
1824 uint64_t v_inode;
1825 uint32_t v_idx;
1826 struct snapraid_file* file;
1827 struct snapraid_disk* disk;
1828 uint32_t mapping;
1829
1830 ret = sgetb32(f, &mapping);
1831 if (ret < 0 || mapping >= mapping_max) {
1832 /* LCOV_EXCL_START */
1833 decoding_error(path, f);
1834 log_fatal("Internal inconsistency in mapping index!\n");
1835 os_abort();
1836 /* LCOV_EXCL_STOP */
1837 }
1838 disk = tommy_array_get(&disk_mapping, mapping);
1839
1840 ret = sgetb64(f, &v_size);
1841 if (ret < 0) {
1842 /* LCOV_EXCL_START */
1843 decoding_error(path, f);
1844 os_abort();
1845 /* LCOV_EXCL_STOP */
1846 }
1847
1848 if (state->block_size == 0) {
1849 /* LCOV_EXCL_START */
1850 decoding_error(path, f);
1851 log_fatal("Internal inconsistency due zero blocksize!\n");
1852 exit(EXIT_FAILURE);
1853 /* LCOV_EXCL_STOP */
1854 }
1855
1856 /* check for impossible file size to avoid to crash for a too big allocation */
1857 if (v_size / state->block_size > blockmax) {
1858 /* LCOV_EXCL_START */
1859 decoding_error(path, f);
1860 log_fatal("Internal inconsistency in file size too big!\n");
1861 os_abort();
1862 /* LCOV_EXCL_STOP */
1863 }
1864
1865 ret = sgetb64(f, &v_mtime_sec);
1866 if (ret < 0) {
1867 /* LCOV_EXCL_START */
1868 decoding_error(path, f);
1869 os_abort();
1870 /* LCOV_EXCL_STOP */
1871 }
1872
1873 ret = sgetb32(f, &v_mtime_nsec);
1874 if (ret < 0) {
1875 /* LCOV_EXCL_START */
1876 decoding_error(path, f);
1877 os_abort();
1878 /* LCOV_EXCL_STOP */
1879 }
1880
1881 /* STAT_NSEC_INVALID is encoded as 0 */
1882 if (v_mtime_nsec == 0)
1883 v_mtime_nsec = STAT_NSEC_INVALID;
1884 else
1885 --v_mtime_nsec;
1886
1887 ret = sgetb64(f, &v_inode);
1888 if (ret < 0) {
1889 /* LCOV_EXCL_START */
1890 decoding_error(path, f);
1891 os_abort();
1892 /* LCOV_EXCL_STOP */
1893 }
1894
1895 ret = sgetbs(f, sub, sizeof(sub));
1896 if (ret < 0) {
1897 /* LCOV_EXCL_START */
1898 decoding_error(path, f);
1899 os_abort();
1900 /* LCOV_EXCL_STOP */
1901 }
1902 if (!*sub) {
1903 /* LCOV_EXCL_START */
1904 decoding_error(path, f);
1905 log_fatal("Internal inconsistency for null file!\n");
1906 os_abort();
1907 /* LCOV_EXCL_STOP */
1908 }
1909
1910 /* allocate the file */
1911 file = file_alloc(state->block_size, sub, v_size, v_mtime_sec, v_mtime_nsec, v_inode, 0);
1912
1913 /* insert the file in the file containers */
1914 tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode));
1915 tommy_hashdyn_insert(&disk->pathset, &file->pathset, file, file_path_hash(file->sub));
1916 tommy_hashdyn_insert(&disk->stampset, &file->stampset, file, file_stamp_hash(file->size, file->mtime_sec, file->mtime_nsec));
1917 tommy_list_insert_tail(&disk->filelist, &file->nodelist, file);
1918
1919 /* read all the blocks */
1920 v_idx = 0;
1921 while (v_idx < file->blockmax) {
1922 block_off_t v_pos;
1923 uint32_t v_count;
1924
1925 /* get the "subcommand */
1926 c = sgetc(f);
1927
1928 ret = sgetb32(f, &v_pos);
1929 if (ret < 0) {
1930 /* LCOV_EXCL_START */
1931 decoding_error(path, f);
1932 os_abort();
1933 /* LCOV_EXCL_STOP */
1934 }
1935
1936 ret = sgetb32(f, &v_count);
1937 if (ret < 0) {
1938 /* LCOV_EXCL_START */
1939 decoding_error(path, f);
1940 os_abort();
1941 /* LCOV_EXCL_STOP */
1942 }
1943
1944 if (v_idx + v_count > file->blockmax) {
1945 /* LCOV_EXCL_START */
1946 decoding_error(path, f);
1947 log_fatal("Internal inconsistency in block number!\n");
1948 os_abort();
1949 /* LCOV_EXCL_STOP */
1950 }
1951
1952 if (v_pos + v_count > blockmax) {
1953 /* LCOV_EXCL_START */
1954 decoding_error(path, f);
1955 log_fatal("Internal inconsistency in block size %u/%u!\n", blockmax, v_pos + v_count);
1956 os_abort();
1957 /* LCOV_EXCL_START */
1958 }
1959
1960 /* fill the blocks in the run */
1961 while (v_count) {
1962 struct snapraid_block* block = fs_file2block_get(file, v_idx);
1963
1964 switch (c) {
1965 case 'b' :
1966 block_state_set(block, BLOCK_STATE_BLK);
1967 break;
1968 case 'n' :
1969 /* deprecated NEW blocks are converted to CHG ones */
1970 block_state_set(block, BLOCK_STATE_CHG);
1971 break;
1972 case 'g' :
1973 block_state_set(block, BLOCK_STATE_CHG);
1974 break;
1975 case 'p' :
1976 block_state_set(block, BLOCK_STATE_REP);
1977 break;
1978 default :
1979 /* LCOV_EXCL_START */
1980 decoding_error(path, f);
1981 log_fatal("Invalid block type!\n");
1982 os_abort();
1983 /* LCOV_EXCL_STOP */
1984 }
1985
1986 /* read the hash only for 'blk/chg/rep', and not for 'new' */
1987 if (c != 'n') {
1988 ret = sread(f, block->hash, BLOCK_HASH_SIZE);
1989 if (ret < 0) {
1990 /* LCOV_EXCL_START */
1991 decoding_error(path, f);
1992 os_abort();
1993 /* LCOV_EXCL_STOP */
1994 }
1995 } else {
1996 /* set the ZERO hash for deprecated NEW blocks */
1997 hash_zero_set(block->hash);
1998 }
1999
2000 /* if the block contains a hash of past data */
2001 /* and we are clearing such indeterminate hashes */
2002 if (state->clear_past_hash
2003 && block_has_past_hash(block)
2004 ) {
2005 /* set the hash value to INVALID */
2006 hash_invalid_set(block->hash);
2007 }
2008
2009 /* if we are disabling the copy optimization */
2010 /* we want also to clear any already previously stored information */
2011 /* in other sync commands */
2012 /* note that this is required only in sync, and we detect */
2013 /* this using the clear_past_hash flag */
2014 if (state->clear_past_hash
2015 && state->opt.force_nocopy
2016 && block_state_get(block) == BLOCK_STATE_REP
2017 ) {
2018 /* set the hash value to INVALID */
2019 hash_invalid_set(block->hash);
2020 /* convert from REP to CHG block */
2021 block_state_set(block, BLOCK_STATE_CHG);
2022 }
2023
2024 /* if we want a full reallocation, marks block as invalid parity */
2025 /* note that we do this after the force_nocopy option */
2026 /* to avoid to mixup the two things */
2027 if (state->opt.force_realloc
2028 && block_state_get(block) == BLOCK_STATE_BLK) {
2029 /* convert from BLK to REP */
2030 block_state_set(block, BLOCK_STATE_REP);
2031 }
2032
2033 /* set the parity association */
2034 fs_allocate(disk, v_pos, file, v_idx);
2035
2036 /* go to the next block */
2037 ++v_idx;
2038 ++v_pos;
2039 --v_count;
2040 }
2041 }
2042
2043 /* stat */
2044 ++count_file;
2045 } else if (c == 'i') {
2046 /* "inf" command */
2047 snapraid_info info;
2048 uint32_t v_pos;
2049 uint32_t v_oldest;
2050
2051 ret = sgetb32(f, &v_oldest);
2052 if (ret < 0) {
2053 /* LCOV_EXCL_START */
2054 decoding_error(path, f);
2055 os_abort();
2056 /* LCOV_EXCL_STOP */
2057 }
2058
2059 v_pos = 0;
2060 while (v_pos < blockmax) {
2061 int bad;
2062 int rehash;
2063 int justsynced;
2064 uint32_t t;
2065 uint32_t flag;
2066 uint32_t v_count;
2067
2068 ret = sgetb32(f, &v_count);
2069 if (ret < 0) {
2070 /* LCOV_EXCL_START */
2071 decoding_error(path, f);
2072 os_abort();
2073 /* LCOV_EXCL_STOP */
2074 }
2075
2076 if (v_pos + v_count > blockmax) {
2077 /* LCOV_EXCL_START */
2078 decoding_error(path, f);
2079 log_fatal("Internal inconsistency in info size %u/%u!\n", blockmax, v_pos + v_count);
2080 os_abort();
2081 /* LCOV_EXCL_STOP */
2082 }
2083
2084 ret = sgetb32(f, &flag);
2085 if (ret < 0) {
2086 /* LCOV_EXCL_START */
2087 decoding_error(path, f);
2088 os_abort();
2089 /* LCOV_EXCL_STOP */
2090 }
2091
2092 /* if there is an info */
2093 if ((flag & 1) != 0) {
2094 /* read the time */
2095 ret = sgetb32(f, &t);
2096 if (ret < 0) {
2097 /* LCOV_EXCL_START */
2098 decoding_error(path, f);
2099 os_abort();
2100 /* LCOV_EXCL_STOP */
2101 }
2102
2103 /* analyze the flags */
2104 bad = (flag & 2) != 0;
2105 rehash = (flag & 4) != 0;
2106 justsynced = (flag & 8) != 0;
2107
2108 if (rehash && state->prevhash == HASH_UNDEFINED) {
2109 /* LCOV_EXCL_START */
2110 decoding_error(path, f);
2111 log_fatal("Internal inconsistency for missing previous checksum!\n");
2112 os_abort();
2113 /* LCOV_EXCL_STOP */
2114 }
2115
2116 info = info_make(t + v_oldest, bad, rehash, justsynced);
2117 } else {
2118 info = 0;
2119 }
2120
2121 while (v_count) {
2122 /* insert the info in the array */
2123 info_set(&state->infoarr, v_pos, info);
2124
2125 /* ensure that an info is present only for used positions */
2126 if (fs_info_is_required(state, v_pos)) {
2127 if (!info) {
2128 /* LCOV_EXCL_START */
2129 decoding_error(path, f);
2130 log_fatal("Internal inconsistency for missing info!\n");
2131 os_abort();
2132 /* LCOV_EXCL_STOP */
2133 }
2134 } else {
2135 /* extra info are accepted for backward compatibility */
2136 /* they are discarded at the first write */
2137 }
2138
2139 /* go to next block */
2140 ++v_pos;
2141 --v_count;
2142 }
2143 }
2144 } else if (c == 'h') {
2145 /* hole */
2146 uint32_t v_pos;
2147 struct snapraid_disk* disk;
2148 uint32_t mapping;
2149
2150 ret = sgetb32(f, &mapping);
2151 if (ret < 0 || mapping >= mapping_max) {
2152 /* LCOV_EXCL_START */
2153 decoding_error(path, f);
2154 log_fatal("Internal inconsistency in mapping index!\n");
2155 os_abort();
2156 /* LCOV_EXCL_STOP */
2157 }
2158 disk = tommy_array_get(&disk_mapping, mapping);
2159
2160 v_pos = 0;
2161 while (v_pos < blockmax) {
2162 uint32_t v_idx;
2163 uint32_t v_count;
2164 struct snapraid_file* deleted;
2165
2166 ret = sgetb32(f, &v_count);
2167 if (ret < 0) {
2168 /* LCOV_EXCL_START */
2169 decoding_error(path, f);
2170 os_abort();
2171 /* LCOV_EXCL_STOP */
2172 }
2173
2174 if (v_pos + v_count > blockmax) {
2175 /* LCOV_EXCL_START */
2176 decoding_error(path, f);
2177 log_fatal("Internal inconsistency in hole size %u/%u!\n", blockmax, v_pos + v_count);
2178 os_abort();
2179 /* LCOV_EXCL_STOP */
2180 }
2181
2182 /* get the sub-command */
2183 c = sgetc(f);
2184
2185 switch (c) {
2186 case 'o' :
2187 /* if it's a run of deleted blocks */
2188
2189 /* allocate a fake deleted file */
2190 deleted = file_alloc(state->block_size, "<deleted>", v_count * (data_off_t)state->block_size, 0, 0, 0, 0);
2191
2192 /* mark the file as deleted */
2193 file_flag_set(deleted, FILE_IS_DELETED);
2194
2195 /* insert it in the list of deleted files */
2196 tommy_list_insert_tail(&disk->deletedlist, &deleted->nodelist, deleted);
2197
2198 /* process all blocks */
2199 v_idx = 0;
2200 while (v_count) {
2201 struct snapraid_block* block = fs_file2block_get(deleted, v_idx);
2202
2203 /* set the block as deleted */
2204 block_state_set(block, BLOCK_STATE_DELETED);
2205
2206 /* read the hash */
2207 ret = sread(f, block->hash, BLOCK_HASH_SIZE);
2208 if (ret < 0) {
2209 /* LCOV_EXCL_START */
2210 decoding_error(path, f);
2211 os_abort();
2212 /* LCOV_EXCL_STOP */
2213 }
2214
2215 /* if we are clearing indeterminate hashes */
2216 if (state->clear_past_hash) {
2217 /* set the hash value to INVALID */
2218 hash_invalid_set(block->hash);
2219 }
2220
2221 /* insert the block in the block array */
2222 fs_allocate(disk, v_pos, deleted, v_idx);
2223
2224 /* go to next block */
2225 ++v_pos;
2226 ++v_idx;
2227 --v_count;
2228 }
2229 break;
2230 case 'O' :
2231 /* go to the next run */
2232 v_pos += v_count;
2233 break;
2234 default :
2235 /* LCOV_EXCL_START */
2236 decoding_error(path, f);
2237 log_fatal("Invalid hole type!\n");
2238 os_abort();
2239 /* LCOV_EXCL_STOP */
2240 }
2241 }
2242 } else if (c == 's') {
2243 /* symlink */
2244 char sub[PATH_MAX];
2245 char linkto[PATH_MAX];
2246 struct snapraid_link* slink;
2247 struct snapraid_disk* disk;
2248 uint32_t mapping;
2249
2250 ret = sgetb32(f, &mapping);
2251 if (ret < 0 || mapping >= mapping_max) {
2252 /* LCOV_EXCL_START */
2253 decoding_error(path, f);
2254 log_fatal("Internal inconsistency in mapping index!\n");
2255 os_abort();
2256 /* LCOV_EXCL_STOP */
2257 }
2258 disk = tommy_array_get(&disk_mapping, mapping);
2259
2260 ret = sgetbs(f, sub, sizeof(sub));
2261 if (ret < 0) {
2262 /* LCOV_EXCL_START */
2263 decoding_error(path, f);
2264 os_abort();
2265 /* LCOV_EXCL_STOP */
2266 }
2267
2268 if (!*sub) {
2269 /* LCOV_EXCL_START */
2270 decoding_error(path, f);
2271 log_fatal("Internal inconsistency for null symlink!\n");
2272 os_abort();
2273 /* LCOV_EXCL_STOP */
2274 }
2275
2276 ret = sgetbs(f, linkto, sizeof(linkto));
2277 if (ret < 0) {
2278 /* LCOV_EXCL_START */
2279 decoding_error(path, f);
2280 os_abort();
2281 /* LCOV_EXCL_STOP */
2282 }
2283
2284 /* allocate the link as symbolic link */
2285 slink = link_alloc(sub, linkto, FILE_IS_SYMLINK);
2286
2287 /* insert the link in the link containers */
2288 tommy_hashdyn_insert(&disk->linkset, &slink->nodeset, slink, link_name_hash(slink->sub));
2289 tommy_list_insert_tail(&disk->linklist, &slink->nodelist, slink);
2290
2291 /* stat */
2292 ++count_symlink;
2293 } else if (c == 'a') {
2294 /* hardlink */
2295 char sub[PATH_MAX];
2296 char linkto[PATH_MAX];
2297 struct snapraid_link* slink;
2298 struct snapraid_disk* disk;
2299 uint32_t mapping;
2300
2301 ret = sgetb32(f, &mapping);
2302 if (ret < 0 || mapping >= mapping_max) {
2303 /* LCOV_EXCL_START */
2304 decoding_error(path, f);
2305 log_fatal("Internal inconsistency in mapping index!\n");
2306 os_abort();
2307 /* LCOV_EXCL_STOP */
2308 }
2309 disk = tommy_array_get(&disk_mapping, mapping);
2310
2311 ret = sgetbs(f, sub, sizeof(sub));
2312 if (ret < 0) {
2313 /* LCOV_EXCL_START */
2314 decoding_error(path, f);
2315 os_abort();
2316 /* LCOV_EXCL_STOP */
2317 }
2318
2319 if (!*sub) {
2320 /* LCOV_EXCL_START */
2321 decoding_error(path, f);
2322 log_fatal("Internal inconsistency for null hardlink!\n");
2323 os_abort();
2324 /* LCOV_EXCL_STOP */
2325 }
2326
2327 ret = sgetbs(f, linkto, sizeof(linkto));
2328 if (ret < 0) {
2329 /* LCOV_EXCL_START */
2330 decoding_error(path, f);
2331 os_abort();
2332 /* LCOV_EXCL_STOP */
2333 }
2334
2335 if (!*linkto) {
2336 /* LCOV_EXCL_START */
2337 decoding_error(path, f);
2338 log_fatal("Internal inconsistency for empty hardlink '%s'!\n", sub);
2339 os_abort();
2340 /* LCOV_EXCL_STOP */
2341 }
2342
2343 /* allocate the link as hard link */
2344 slink = link_alloc(sub, linkto, FILE_IS_HARDLINK);
2345
2346 /* insert the link in the link containers */
2347 tommy_hashdyn_insert(&disk->linkset, &slink->nodeset, slink, link_name_hash(slink->sub));
2348 tommy_list_insert_tail(&disk->linklist, &slink->nodelist, slink);
2349
2350 /* stat */
2351 ++count_hardlink;
2352 } else if (c == 'r') {
2353 /* dir */
2354 char sub[PATH_MAX];
2355 struct snapraid_dir* dir;
2356 struct snapraid_disk* disk;
2357 uint32_t mapping;
2358
2359 ret = sgetb32(f, &mapping);
2360 if (ret < 0 || mapping >= mapping_max) {
2361 /* LCOV_EXCL_START */
2362 decoding_error(path, f);
2363 log_fatal("Internal inconsistency in mapping index!\n");
2364 os_abort();
2365 /* LCOV_EXCL_STOP */
2366 }
2367 disk = tommy_array_get(&disk_mapping, mapping);
2368
2369 ret = sgetbs(f, sub, sizeof(sub));
2370 if (ret < 0) {
2371 /* LCOV_EXCL_START */
2372 decoding_error(path, f);
2373 os_abort();
2374 /* LCOV_EXCL_STOP */
2375 }
2376
2377 if (!*sub) {
2378 /* LCOV_EXCL_START */
2379 decoding_error(path, f);
2380 log_fatal("Internal inconsistency for null dir!\n");
2381 os_abort();
2382 /* LCOV_EXCL_STOP */
2383 }
2384
2385 /* allocate the dir */
2386 dir = dir_alloc(sub);
2387
2388 /* insert the dir in the dir containers */
2389 tommy_hashdyn_insert(&disk->dirset, &dir->nodeset, dir, dir_name_hash(dir->sub));
2390 tommy_list_insert_tail(&disk->dirlist, &dir->nodelist, dir);
2391
2392 /* stat */
2393 ++count_dir;
2394 } else if (c == 'c') {
2395 /* get the subcommand */
2396 c = sgetc(f);
2397
2398 switch (c) {
2399 case 'u' :
2400 state->hash = HASH_MURMUR3;
2401 break;
2402 case 'k' :
2403 state->hash = HASH_SPOOKY2;
2404 break;
2405 case 'm' :
2406 state->hash = HASH_METRO;
2407 break;
2408 default :
2409 /* LCOV_EXCL_START */
2410 decoding_error(path, f);
2411 log_fatal("Invalid checksum!\n");
2412 os_abort();
2413 /* LCOV_EXCL_STOP */
2414 }
2415
2416 /* read the seed */
2417 ret = sread(f, state->hashseed, HASH_MAX);
2418 if (ret < 0) {
2419 /* LCOV_EXCL_START */
2420 decoding_error(path, f);
2421 os_abort();
2422 /* LCOV_EXCL_STOP */
2423 }
2424 } else if (c == 'C') {
2425 /* get the sub-command */
2426 c = sgetc(f);
2427
2428 switch (c) {
2429 case 'u' :
2430 state->prevhash = HASH_MURMUR3;
2431 break;
2432 case 'k' :
2433 state->prevhash = HASH_SPOOKY2;
2434 break;
2435 case 'm' :
2436 state->prevhash = HASH_METRO;
2437 break;
2438 default :
2439 /* LCOV_EXCL_START */
2440 decoding_error(path, f);
2441 log_fatal("Invalid checksum!\n");
2442 os_abort();
2443 /* LCOV_EXCL_STOP */
2444 }
2445
2446 /* read the seed */
2447 ret = sread(f, state->prevhashseed, HASH_MAX);
2448 if (ret < 0) {
2449 /* LCOV_EXCL_START */
2450 decoding_error(path, f);
2451 os_abort();
2452 /* LCOV_EXCL_STOP */
2453 }
2454 } else if (c == 'z') {
2455 uint32_t block_size;
2456
2457 ret = sgetb32(f, &block_size);
2458 if (ret < 0) {
2459 /* LCOV_EXCL_START */
2460 decoding_error(path, f);
2461 os_abort();
2462 /* LCOV_EXCL_STOP */
2463 }
2464
2465 if (block_size == 0) {
2466 /* LCOV_EXCL_START */
2467 decoding_error(path, f);
2468 log_fatal("Zero 'blocksize' specification in the content file!\n");
2469 exit(EXIT_FAILURE);
2470 /* LCOV_EXCL_STOP */
2471 }
2472
2473 /* without configuration, auto assign the block size */
2474 if (state->no_conf) {
2475 state->block_size = block_size;
2476 }
2477
2478 if (block_size != state->block_size) {
2479 /* LCOV_EXCL_START */
2480 decoding_error(path, f);
2481 log_fatal("Mismatching 'blocksize' specification in the content file!\n");
2482 log_fatal("Please restore the 'blocksize' value in the configuration file to '%u'\n", block_size / KIBI);
2483 exit(EXIT_FAILURE);
2484 /* LCOV_EXCL_STOP */
2485 }
2486 } else if (c == 'y') {
2487 uint32_t hash_size;
2488
2489 ret = sgetb32(f, &hash_size);
2490 if (ret < 0) {
2491 /* LCOV_EXCL_START */
2492 decoding_error(path, f);
2493 os_abort();
2494 /* LCOV_EXCL_STOP */
2495 }
2496
2497 if (hash_size < 2 || hash_size > HASH_MAX) {
2498 /* LCOV_EXCL_START */
2499 decoding_error(path, f);
2500 log_fatal("Invalid 'hashsize' specification in the content file!\n");
2501 exit(EXIT_FAILURE);
2502 /* LCOV_EXCL_STOP */
2503 }
2504
2505 /* without configuration, auto assign the block size */
2506 if (state->no_conf) {
2507 BLOCK_HASH_SIZE = hash_size;
2508 }
2509
2510 if ((int)hash_size != BLOCK_HASH_SIZE) {
2511 /* LCOV_EXCL_START */
2512 decoding_error(path, f);
2513 log_fatal("Mismatching 'hashsize' specification in the content file!\n");
2514 log_fatal("Please restore the 'hashsize' value in the configuration file to '%u'\n", hash_size);
2515 exit(EXIT_FAILURE);
2516 /* LCOV_EXCL_STOP */
2517 }
2518 } else if (c == 'x') {
2519 ret = sgetb32(f, &blockmax);
2520 if (ret < 0) {
2521 /* LCOV_EXCL_START */
2522 decoding_error(path, f);
2523 os_abort();
2524 /* LCOV_EXCL_STOP */
2525 }
2526 } else if (c == 'm' || c == 'M') {
2527 struct snapraid_map* map;
2528 char uuid[UUID_MAX];
2529 uint32_t v_pos;
2530 uint32_t v_total_blocks;
2531 uint32_t v_free_blocks;
2532 struct snapraid_disk* disk;
2533
2534 ret = sgetbs(f, buffer, sizeof(buffer));
2535 if (ret < 0) {
2536 /* LCOV_EXCL_START */
2537 decoding_error(path, f);
2538 os_abort();
2539 /* LCOV_EXCL_STOP */
2540 }
2541
2542 ret = sgetb32(f, &v_pos);
2543 if (ret < 0) {
2544 /* LCOV_EXCL_START */
2545 decoding_error(path, f);
2546 os_abort();
2547 /* LCOV_EXCL_STOP */
2548 }
2549
2550 /* from SnapRAID 7.0 the 'M' command includes the free space */
2551 if (c == 'M') {
2552 ret = sgetb32(f, &v_total_blocks);
2553 if (ret < 0) {
2554 /* LCOV_EXCL_START */
2555 decoding_error(path, f);
2556 os_abort();
2557 /* LCOV_EXCL_STOP */
2558 }
2559
2560 ret = sgetb32(f, &v_free_blocks);
2561 if (ret < 0) {
2562 /* LCOV_EXCL_START */
2563 decoding_error(path, f);
2564 os_abort();
2565 /* LCOV_EXCL_STOP */
2566 }
2567 } else {
2568 v_total_blocks = 0;
2569 v_free_blocks = 0;
2570 }
2571
2572 /* read the uuid */
2573 ret = sgetbs(f, uuid, sizeof(uuid));
2574 if (ret < 0) {
2575 /* LCOV_EXCL_START */
2576 decoding_error(path, f);
2577 os_abort();
2578 /* LCOV_EXCL_STOP */
2579 }
2580
2581 /* find the disk */
2582 disk = find_disk_by_name(state, buffer);
2583 if (!disk) {
2584 /* search by UUID if renamed */
2585 disk = find_disk_by_uuid(state, uuid);
2586 if (disk) {
2587 log_fatal("WARNING! Renaming disk '%s' to '%s'\n", buffer, disk->name);
2588
2589 /* write the new state with the new name */
2590 state->need_write = 1;
2591 }
2592 }
2593 if (!disk) {
2594 /* LCOV_EXCL_START */
2595 decoding_error(path, f);
2596 log_fatal("Disk '%s' with uuid '%s' not present in the configuration file!\n", buffer, uuid);
2597 log_fatal("If you have removed it from the configuration file, please restore it\n");
2598 /* if it's a command without UUID, it cannot autorename using UUID */
2599 if (state->opt.skip_disk_access)
2600 log_fatal("If you have renamed it, run 'sync' to update the new name\n");
2601 exit(EXIT_FAILURE);
2602 /* LCOV_EXCL_STOP */
2603 }
2604
2605 map = map_alloc(disk->name, v_pos, v_total_blocks, v_free_blocks, uuid);
2606
2607 tommy_list_insert_tail(&state->maplist, &map->node, map);
2608
2609 /* insert in the mapping vector */
2610 tommy_array_grow(&disk_mapping, mapping_max + 1);
2611 tommy_array_set(&disk_mapping, mapping_max, disk);
2612 ++mapping_max;
2613 } else if (c == 'P') {
2614 /* from SnapRAID 7.0 the 'P' command includes the free space */
2615 /* from SnapRAID 11.0 the 'P' command is deprecated by 'Q' */
2616 char v_uuid[UUID_MAX];
2617 uint32_t v_level;
2618 uint32_t v_total_blocks;
2619 uint32_t v_free_blocks;
2620
2621 ret = sgetb32(f, &v_level);
2622 if (ret < 0) {
2623 /* LCOV_EXCL_START */
2624 decoding_error(path, f);
2625 os_abort();
2626 /* LCOV_EXCL_STOP */
2627 }
2628
2629 ret = sgetb32(f, &v_total_blocks);
2630 if (ret < 0) {
2631 /* LCOV_EXCL_START */
2632 decoding_error(path, f);
2633 os_abort();
2634 /* LCOV_EXCL_STOP */
2635 }
2636
2637 ret = sgetb32(f, &v_free_blocks);
2638 if (ret < 0) {
2639 /* LCOV_EXCL_START */
2640 decoding_error(path, f);
2641 os_abort();
2642 /* LCOV_EXCL_STOP */
2643 }
2644
2645 ret = sgetbs(f, v_uuid, sizeof(v_uuid));
2646 if (ret < 0) {
2647 /* LCOV_EXCL_START */
2648 decoding_error(path, f);
2649 os_abort();
2650 /* LCOV_EXCL_STOP */
2651 }
2652
2653 if (v_level >= LEV_MAX) {
2654 /* LCOV_EXCL_START */
2655 decoding_error(path, f);
2656 log_fatal("Invalid parity level '%u' in the configuration file!\n", v_level);
2657 exit(EXIT_FAILURE);
2658 /* LCOV_EXCL_STOP */
2659 }
2660
2661 /* auto configure if configuration is missing */
2662 if (state->no_conf) {
2663 if (v_level >= state->level)
2664 state->level = v_level + 1;
2665 }
2666
2667 /* if we use this parity entry */
2668 if (v_level < state->level) {
2669 /* if the configuration has more splits, keep them */
2670 if (state->parity[v_level].split_mac < 1)
2671 state->parity[v_level].split_mac = 1;
2672 /* set the parity info */
2673 pathcpy(state->parity[v_level].split_map[0].uuid, sizeof(state->parity[v_level].split_map[0].uuid), v_uuid);
2674 state->parity[v_level].total_blocks = v_total_blocks;
2675 state->parity[v_level].free_blocks = v_free_blocks;
2676 }
2677 } else if (c == 'Q') {
2678 /* from SnapRAID 11.0 the 'Q' command include size info and multi file support */
2679 uint32_t v_level;
2680 uint32_t v_total_blocks;
2681 uint32_t v_free_blocks;
2682 uint32_t v_split_mac;
2683 unsigned s;
2684
2685 ret = sgetb32(f, &v_level);
2686 if (ret < 0) {
2687 /* LCOV_EXCL_START */
2688 decoding_error(path, f);
2689 os_abort();
2690 /* LCOV_EXCL_STOP */
2691 }
2692
2693 ret = sgetb32(f, &v_total_blocks);
2694 if (ret < 0) {
2695 /* LCOV_EXCL_START */
2696 decoding_error(path, f);
2697 os_abort();
2698 /* LCOV_EXCL_STOP */
2699 }
2700
2701 ret = sgetb32(f, &v_free_blocks);
2702 if (ret < 0) {
2703 /* LCOV_EXCL_START */
2704 decoding_error(path, f);
2705 os_abort();
2706 /* LCOV_EXCL_STOP */
2707 }
2708
2709 ret = sgetb32(f, &v_split_mac);
2710 if (ret < 0) {
2711 /* LCOV_EXCL_START */
2712 decoding_error(path, f);
2713 os_abort();
2714 /* LCOV_EXCL_STOP */
2715 }
2716
2717 if (v_level >= LEV_MAX) {
2718 /* LCOV_EXCL_START */
2719 decoding_error(path, f);
2720 log_fatal("Invalid parity level '%u' in the configuration file!\n", v_level);
2721 exit(EXIT_FAILURE);
2722 /* LCOV_EXCL_STOP */
2723 }
2724
2725 /* auto configure if configuration is missing */
2726 if (state->no_conf) {
2727 if (v_level >= state->level)
2728 state->level = v_level + 1;
2729 if (state->parity[v_level].split_mac < v_split_mac)
2730 state->parity[v_level].split_mac = v_split_mac;
2731 }
2732
2733 /* if we use this parity entry */
2734 if (v_level < state->level) {
2735 /* set the parity info */
2736 state->parity[v_level].total_blocks = v_total_blocks;
2737 state->parity[v_level].free_blocks = v_free_blocks;
2738 }
2739
2740 for (s = 0; s < v_split_mac; ++s) {
2741 char v_path[PATH_MAX];
2742 char v_uuid[UUID_MAX];
2743 uint64_t v_size;
2744
2745 ret = sgetbs(f, v_path, sizeof(v_path));
2746 if (ret < 0) {
2747 /* LCOV_EXCL_START */
2748 decoding_error(path, f);
2749 os_abort();
2750 /* LCOV_EXCL_STOP */
2751 }
2752
2753 ret = sgetbs(f, v_uuid, sizeof(v_uuid));
2754 if (ret < 0) {
2755 /* LCOV_EXCL_START */
2756 decoding_error(path, f);
2757 os_abort();
2758 /* LCOV_EXCL_STOP */
2759 }
2760
2761 ret = sgetb64(f, &v_size);
2762 if (ret < 0) {
2763 /* LCOV_EXCL_START */
2764 decoding_error(path, f);
2765 os_abort();
2766 /* LCOV_EXCL_STOP */
2767 }
2768
2769 /* if we use this parity entry */
2770 if (v_level < state->level) {
2771 /* if this split was removed from the configuration */
2772 if (s >= state->parity[v_level].split_mac) {
2773 /* if the file is used, we really need it */
2774 if (v_size != 0) {
2775 /* LCOV_EXCL_START */
2776 decoding_error(path, f);
2777 log_fatal("Parity '%s' misses used file '%u'!\n", lev_config_name(v_level), s);
2778 log_fatal("If you have removed it from the configuration file, please restore it\n");
2779 exit(EXIT_FAILURE);
2780 /* LCOV_EXCL_STOP */
2781 }
2782
2783 /* otherwise we can drop it */
2784 log_fatal("WARNING! Dropping from '%s' unused split '%u'\n", lev_config_name(v_level), s);
2785 } else {
2786 /* we copy the path only if without configuration file */
2787 if (state->no_conf)
2788 pathcpy(state->parity[v_level].split_map[s].path, sizeof(state->parity[v_level].split_map[s].path), v_path);
2789
2790 /* set the split info */
2791 pathcpy(state->parity[v_level].split_map[s].uuid, sizeof(state->parity[v_level].split_map[s].uuid), v_uuid);
2792 state->parity[v_level].split_map[s].size = v_size;
2793
2794 /* log the info read from the content file */
2795 log_tag("content:%s:%u:%s:%s:%" PRIi64 "\n", lev_config_name(v_level), s,
2796 state->parity[v_level].split_map[s].path,
2797 state->parity[v_level].split_map[s].uuid,
2798 state->parity[v_level].split_map[s].size);
2799 }
2800 }
2801 }
2802 } else if (c == 'N') {
2803 uint32_t crc_stored;
2804 uint32_t crc_computed;
2805
2806 /* get the crc before reading it from the file */
2807 crc_computed = scrc(f);
2808
2809 ret = sgetble32(f, &crc_stored);
2810 if (ret < 0) {
2811 /* LCOV_EXCL_START */
2812 /* here don't call decoding_error() because it's too late to get the crc */
2813 log_fatal("Error reading the CRC in '%s' at offset %" PRIi64 "\n", path, stell(f));
2814 log_fatal("This content file is damaged! Use an alternate copy.\n");
2815 exit(EXIT_FAILURE);
2816 /* LCOV_EXCL_STOP */
2817 }
2818
2819 if (crc_stored != crc_computed) {
2820 /* LCOV_EXCL_START */
2821 /* here don't call decoding_error() because it's too late to get the crc */
2822 log_fatal("Mismatching CRC in '%s'\n", path);
2823 log_fatal("This content file is damaged! Use an alternate copy.\n");
2824 exit(EXIT_FAILURE);
2825 /* LCOV_EXCL_STOP */
2826 }
2827
2828 crc_checked = 1;
2829 } else {
2830 /* LCOV_EXCL_START */
2831 decoding_error(path, f);
2832 log_fatal("Invalid command '%c'!\n", (char)c);
2833 os_abort();
2834 /* LCOV_EXCL_STOP */
2835 }
2836 }
2837
2838 tommy_array_done(&disk_mapping);
2839
2840 if (serror(f)) {
2841 /* LCOV_EXCL_START */
2842 log_fatal("Error reading the content file '%s' at offset %" PRIi64 "\n", path, stell(f));
2843 exit(EXIT_FAILURE);
2844 /* LCOV_EXCL_STOP */
2845 }
2846
2847 if (!crc_checked) {
2848 /* LCOV_EXCL_START */
2849 log_fatal("Finished reading '%s' without finding the CRC\n", path);
2850 log_fatal("This content file is truncated or damaged! Use an alternate copy.\n");
2851 exit(EXIT_FAILURE);
2852 /* LCOV_EXCL_STOP */
2853 }
2854
2855 /* check the file-system on all disks */
2856 state_fscheck(state, "after read");
2857
2858 /* check that the stored parity size matches the loaded state */
2859 if (blockmax != parity_allocated_size(state)) {
2860 /* LCOV_EXCL_START */
2861 log_fatal("Internal inconsistency in parity size %u/%u in '%s' at offset %" PRIi64 "\n", blockmax, parity_allocated_size(state), path, stell(f));
2862 if (state->opt.skip_content_check) {
2863 log_fatal("Overriding.\n");
2864 blockmax = parity_allocated_size(state);
2865 } else {
2866 exit(EXIT_FAILURE);
2867 }
2868 /* LCOV_EXCL_STOP */
2869 }
2870
2871 msg_verbose("%8u files\n", count_file);
2872 msg_verbose("%8u hardlinks\n", count_hardlink);
2873 msg_verbose("%8u symlinks\n", count_symlink);
2874 msg_verbose("%8u empty dirs\n", count_dir);
2875 }
2876
2877 struct state_write_thread_context {
2878 struct snapraid_state* state;
2879 #if HAVE_MT_WRITE
2880 thread_id_t thread;
2881 #endif
2882 /* input */
2883 block_off_t blockmax;
2884 time_t info_oldest;
2885 time_t info_now;
2886 int info_has_rehash;
2887 STREAM* f;
2888 /* output */
2889 uint32_t crc;
2890 unsigned count_file;
2891 unsigned count_hardlink;
2892 unsigned count_symlink;
2893 unsigned count_dir;
2894 };
2895
state_write_thread(void * arg)2896 static void* state_write_thread(void* arg)
2897 {
2898 struct state_write_thread_context* context = arg;
2899 struct snapraid_state* state = context->state;
2900 block_off_t blockmax = context->blockmax;
2901 time_t info_oldest = context->info_oldest;
2902 time_t info_now = context->info_now;
2903 int info_has_rehash = context->info_has_rehash;
2904 STREAM* f = context->f;
2905 uint32_t crc;
2906 unsigned count_file;
2907 unsigned count_hardlink;
2908 unsigned count_symlink;
2909 unsigned count_dir;
2910 tommy_node* i;
2911 block_off_t idx;
2912 block_off_t begin;
2913 unsigned l, s;
2914 int version;
2915
2916 count_file = 0;
2917 count_hardlink = 0;
2918 count_symlink = 0;
2919 count_dir = 0;
2920
2921 /* check what version to use */
2922 version = 2;
2923 for (l = 0; l < state->level; ++l) {
2924 if (state->parity[l].split_mac > 1)
2925 version = 3;
2926 }
2927 if (BLOCK_HASH_SIZE != 16)
2928 version = 3;
2929
2930 /* write header */
2931 if (version == 3)
2932 swrite("SNAPCNT3\n\3\0\0", 12, f);
2933 else
2934 swrite("SNAPCNT2\n\3\0\0", 12, f);
2935
2936 /* write block size and block max */
2937 sputc('z', f);
2938 sputb32(state->block_size, f);
2939 sputc('x', f);
2940 sputb32(blockmax, f);
2941
2942 /* hash size */
2943 if (version == 3) {
2944 sputc('y', f);
2945 sputb32(BLOCK_HASH_SIZE, f);
2946 }
2947
2948 if (serror(f)) {
2949 /* LCOV_EXCL_START */
2950 log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
2951 return context;
2952 /* LCOV_EXCL_STOP */
2953 }
2954
2955 sputc('c', f);
2956 if (state->hash == HASH_MURMUR3) {
2957 sputc('u', f);
2958 } else if (state->hash == HASH_SPOOKY2) {
2959 sputc('k', f);
2960 } else if (state->hash == HASH_METRO) {
2961 sputc('m', f);
2962 } else {
2963 /* LCOV_EXCL_START */
2964 log_fatal("Unexpected hash when writing the content file '%s'.\n", serrorfile(f));
2965 return context;
2966 /* LCOV_EXCL_STOP */
2967 }
2968 swrite(state->hashseed, HASH_MAX, f);
2969 if (serror(f)) {
2970 /* LCOV_EXCL_START */
2971 log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
2972 return context;
2973 /* LCOV_EXCL_STOP */
2974 }
2975
2976 /* previous hash only present */
2977 if (state->prevhash != HASH_UNDEFINED) {
2978 /* if at least one rehash tag found, we have to save the previous hash */
2979 if (info_has_rehash) {
2980 sputc('C', f);
2981 if (state->prevhash == HASH_MURMUR3) {
2982 sputc('u', f);
2983 } else if (state->prevhash == HASH_SPOOKY2) {
2984 sputc('k', f);
2985 } else if (state->prevhash == HASH_METRO) {
2986 sputc('m', f);
2987 } else {
2988 /* LCOV_EXCL_START */
2989 log_fatal("Unexpected prevhash when writing the content file '%s'.\n", serrorfile(f));
2990 return context;
2991 /* LCOV_EXCL_STOP */
2992 }
2993 swrite(state->prevhashseed, HASH_MAX, f);
2994 if (serror(f)) {
2995 /* LCOV_EXCL_START */
2996 log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
2997 return context;
2998 /* LCOV_EXCL_STOP */
2999 }
3000 }
3001 }
3002
3003 /* for each map */
3004 for (i = state->maplist; i != 0; i = i->next) {
3005 struct snapraid_map* map = i->data;
3006 struct snapraid_disk* disk;
3007
3008 /* find the disk for this mapping */
3009 disk = find_disk_by_name(state, map->name);
3010 if (!disk) {
3011 /* LCOV_EXCL_START */
3012 log_fatal("Internal inconsistency for unmapped disk '%s'\n", map->name);
3013 return context;
3014 /* LCOV_EXCL_STOP */
3015 }
3016
3017 /* save the mapping only if disk is mapped */
3018 if (disk->mapping_idx != -1) {
3019 sputc('M', f);
3020 sputbs(map->name, f);
3021 sputb32(map->position, f);
3022 sputb32(map->total_blocks, f);
3023 sputb32(map->free_blocks, f);
3024 sputbs(map->uuid, f);
3025 if (serror(f)) {
3026 /* LCOV_EXCL_START */
3027 log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3028 return context;
3029 /* LCOV_EXCL_STOP */
3030 }
3031 }
3032 }
3033
3034 /* for each parity */
3035 for (l = 0; l < state->level; ++l) {
3036 if (version == 3) {
3037 sputc('Q', f);
3038 sputb32(l, f);
3039 sputb32(state->parity[l].total_blocks, f);
3040 sputb32(state->parity[l].free_blocks, f);
3041 sputb32(state->parity[l].split_mac, f);
3042 for (s = 0; s < state->parity[l].split_mac; ++s) {
3043 sputbs(state->parity[l].split_map[s].path, f);
3044 sputbs(state->parity[l].split_map[s].uuid, f);
3045 sputb64(state->parity[l].split_map[s].size, f);
3046 }
3047 } else {
3048 sputc('P', f);
3049 sputb32(l, f);
3050 sputb32(state->parity[l].total_blocks, f);
3051 sputb32(state->parity[l].free_blocks, f);
3052 sputbs(state->parity[l].split_map[0].uuid, f);
3053 }
3054 if (serror(f)) {
3055 /* LCOV_EXCL_START */
3056 log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3057 return context;
3058 /* LCOV_EXCL_STOP */
3059 }
3060 }
3061
3062 /* for each disk */
3063 for (i = state->disklist; i != 0; i = i->next) {
3064 tommy_node* j;
3065 struct snapraid_disk* disk = i->data;
3066
3067 /* if the disk is not mapped, skip it */
3068 if (disk->mapping_idx < 0)
3069 continue;
3070
3071 /* for each file */
3072 for (j = disk->filelist; j != 0; j = j->next) {
3073 struct snapraid_file* file = j->data;
3074 uint64_t size;
3075 uint64_t mtime_sec;
3076 int32_t mtime_nsec;
3077 uint64_t inode;
3078
3079 size = file->size;
3080 mtime_sec = file->mtime_sec;
3081 mtime_nsec = file->mtime_nsec;
3082 inode = file->inode;
3083
3084 sputc('f', f);
3085 sputb32(disk->mapping_idx, f);
3086 sputb64(size, f);
3087 sputb64(mtime_sec, f);
3088 /* encode STAT_NSEC_INVALID as 0 */
3089 if (mtime_nsec == STAT_NSEC_INVALID)
3090 sputb32(0, f);
3091 else
3092 sputb32(mtime_nsec + 1, f);
3093 sputb64(inode, f);
3094 sputbs(file->sub, f);
3095 if (serror(f)) {
3096 /* LCOV_EXCL_START */
3097 log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3098 return context;
3099 /* LCOV_EXCL_STOP */
3100 }
3101
3102 /* for all the blocks of the file */
3103 begin = 0;
3104 while (begin < file->blockmax) {
3105 unsigned v_state = block_state_get(fs_file2block_get(file, begin));
3106 block_off_t v_pos = fs_file2par_get(disk, file, begin);
3107 uint32_t v_count;
3108
3109 block_off_t end;
3110
3111 /* find the end of run of blocks */
3112 end = begin + 1;
3113 while (end < file->blockmax) {
3114 if (v_state != block_state_get(fs_file2block_get(file, end)))
3115 break;
3116 if (v_pos + (end - begin) != fs_file2par_get(disk, file, end))
3117 break;
3118 ++end;
3119 }
3120
3121 switch (v_state) {
3122 case BLOCK_STATE_BLK :
3123 sputc('b', f);
3124 break;
3125 case BLOCK_STATE_CHG :
3126 sputc('g', f);
3127 break;
3128 case BLOCK_STATE_REP :
3129 sputc('p', f);
3130 break;
3131 default :
3132 /* LCOV_EXCL_START */
3133 log_fatal("Internal inconsistency in state for block %u state %u\n", v_pos, v_state);
3134 return context;
3135 /* LCOV_EXCL_STOP */
3136 }
3137
3138 sputb32(v_pos, f);
3139
3140 v_count = end - begin;
3141 sputb32(v_count, f);
3142
3143 /* write hashes */
3144 for (idx = begin; idx < end; ++idx) {
3145 struct snapraid_block* block = fs_file2block_get(file, idx);
3146
3147 swrite(block->hash, BLOCK_HASH_SIZE, f);
3148 }
3149
3150 if (serror(f)) {
3151 /* LCOV_EXCL_START */
3152 log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3153 return context;
3154 /* LCOV_EXCL_STOP */
3155 }
3156
3157 /* next begin position */
3158 begin = end;
3159 }
3160
3161 ++count_file;
3162 }
3163
3164 /* for each link */
3165 for (j = disk->linklist; j != 0; j = j->next) {
3166 struct snapraid_link* slink = j->data;
3167
3168 switch (link_flag_get(slink, FILE_IS_LINK_MASK)) {
3169 case FILE_IS_HARDLINK :
3170 sputc('a', f);
3171 ++count_hardlink;
3172 break;
3173 case FILE_IS_SYMLINK :
3174 sputc('s', f);
3175 ++count_symlink;
3176 break;
3177 }
3178
3179 sputb32(disk->mapping_idx, f);
3180 sputbs(slink->sub, f);
3181 sputbs(slink->linkto, f);
3182 if (serror(f)) {
3183 /* LCOV_EXCL_START */
3184 log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3185 return context;
3186 /* LCOV_EXCL_STOP */
3187 }
3188 }
3189
3190 /* for each dir */
3191 for (j = disk->dirlist; j != 0; j = j->next) {
3192 struct snapraid_dir* dir = j->data;
3193
3194 sputc('r', f);
3195 sputb32(disk->mapping_idx, f);
3196 sputbs(dir->sub, f);
3197 if (serror(f)) {
3198 /* LCOV_EXCL_START */
3199 log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3200 return context;
3201 /* LCOV_EXCL_STOP */
3202 }
3203
3204 ++count_dir;
3205 }
3206
3207 /* deleted blocks of the disk */
3208 sputc('h', f);
3209 sputb32(disk->mapping_idx, f);
3210 if (serror(f)) {
3211 /* LCOV_EXCL_START */
3212 log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3213 return context;
3214 /* LCOV_EXCL_STOP */
3215 }
3216 begin = 0;
3217 while (begin < blockmax) {
3218 int is_deleted;
3219 block_off_t end;
3220
3221 is_deleted = fs_is_block_deleted(disk, begin);
3222
3223 /* find the end of run of blocks */
3224 end = begin + 1;
3225 while (end < blockmax
3226 && is_deleted == fs_is_block_deleted(disk, end)
3227 ) {
3228 ++end;
3229 }
3230
3231 sputb32(end - begin, f);
3232
3233 if (is_deleted) {
3234 /* write the run of deleted blocks with hash */
3235 sputc('o', f);
3236
3237 /* write all the hash */
3238 while (begin < end) {
3239 struct snapraid_block* block = fs_par2block_get(disk, begin);
3240
3241 swrite(block->hash, BLOCK_HASH_SIZE, f);
3242
3243 ++begin;
3244 }
3245 } else {
3246 /* write the run of blocks without hash */
3247 /* they can be either used or empty blocks */
3248 sputc('O', f);
3249
3250 /* next begin position */
3251 begin = end;
3252 }
3253
3254 if (serror(f)) {
3255 /* LCOV_EXCL_START */
3256 log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3257 return context;
3258 /* LCOV_EXCL_STOP */
3259 }
3260 }
3261 }
3262
3263 /* write the info for each block */
3264 sputc('i', f);
3265 sputb32(info_oldest, f);
3266 begin = 0;
3267 while (begin < blockmax) {
3268 snapraid_info info;
3269 block_off_t end;
3270 time_t t;
3271 unsigned flag;
3272
3273 info = info_get(&state->infoarr, begin);
3274
3275 /* find the end of run of blocks */
3276 end = begin + 1;
3277 while (end < blockmax
3278 && info == info_get(&state->infoarr, end)
3279 ) {
3280 ++end;
3281 }
3282
3283 sputb32(end - begin, f);
3284
3285 /* if there is info */
3286 if (info) {
3287 /* other flags */
3288 flag = 1; /* info is present */
3289 if (info_get_bad(info))
3290 flag |= 2;
3291 if (info_get_rehash(info))
3292 flag |= 4;
3293 if (info_get_justsynced(info))
3294 flag |= 8;
3295 sputb32(flag, f);
3296
3297 t = info_get_time(info);
3298
3299 /* truncate any time that is in the future */
3300 if (t > info_now)
3301 t = info_now;
3302
3303 /* the oldest info is computed only on required blocks, so it may not be the absolute oldest */
3304 if (t < info_oldest)
3305 t = 0;
3306 else
3307 t -= info_oldest;
3308
3309 sputb32(t, f);
3310 } else {
3311 /* write a special 0 flag to mark missing info */
3312 sputb32(0, f);
3313 }
3314
3315 if (serror(f)) {
3316 /* LCOV_EXCL_START */
3317 log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3318 return context;
3319 /* LCOV_EXCL_STOP */
3320 }
3321
3322 /* next begin position */
3323 begin = end;
3324 }
3325
3326 sputc('N', f);
3327
3328 /* flush data written to the disk */
3329 if (sflush(f)) {
3330 /* LCOV_EXCL_START */
3331 log_fatal("Error writing the content file '%s' (in flush before crc). %s.\n", serrorfile(f), strerror(errno));
3332 return context;
3333 /* LCOV_EXCL_STOP */
3334 }
3335
3336 /* get the file crc */
3337 crc = scrc(f);
3338
3339 /* compare the crc of the data written to file */
3340 /* with the one of the data written to the stream */
3341 if (crc != scrc_stream(f)) {
3342 /* LCOV_EXCL_START */
3343 log_fatal("CRC mismatch writing the content stream.\n");
3344 log_fatal("DANGER! Your RAM memory is broken! DO NOT PROCEED UNTIL FIXED!\n");
3345 log_fatal("Try running a memory test like http://www.memtest86.com/\n");
3346 return context;
3347 /* LCOV_EXCL_STOP */
3348 }
3349
3350 sputble32(crc, f);
3351 if (serror(f)) {
3352 /* LCOV_EXCL_START */
3353 log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3354 return context;
3355 /* LCOV_EXCL_STOP */
3356 }
3357
3358 /* set output variables */
3359 context->crc = crc;
3360 context->count_file = count_file;
3361 context->count_hardlink = count_hardlink;
3362 context->count_symlink = count_symlink;
3363 context->count_dir = count_dir;
3364
3365 return 0;
3366 }
3367
state_write_content(struct snapraid_state * state,uint32_t * out_crc)3368 static void state_write_content(struct snapraid_state* state, uint32_t* out_crc)
3369 {
3370 #if HAVE_MT_WRITE
3371 int fail;
3372 int first;
3373 #else
3374 STREAM* f;
3375 unsigned count_content;
3376 unsigned k;
3377 struct state_write_thread_context* context;
3378 void* retval;
3379 #endif
3380 tommy_node* i;
3381 block_off_t blockmax;
3382 time_t info_oldest;
3383 time_t info_now;
3384 int info_has_rehash;
3385 int mapping_idx;
3386 block_off_t idx;
3387 uint32_t crc;
3388 unsigned count_file;
3389 unsigned count_hardlink;
3390 unsigned count_symlink;
3391 unsigned count_dir;
3392
3393 /* blocks of all array */
3394 blockmax = parity_allocated_size(state);
3395
3396 /* check the file-system on all disks */
3397 state_fscheck(state, "before write");
3398
3399 /* clear the info for unused blocks */
3400 /* and get some other info */
3401 info_oldest = 0; /* oldest time in info */
3402 info_now = time(0); /* get the present time */
3403 info_has_rehash = 0; /* if there is a rehash info */
3404 for (idx = 0; idx < blockmax; ++idx) {
3405 /* if the position is used */
3406 if (fs_position_is_required(state, idx)) {
3407 snapraid_info info = info_get(&state->infoarr, idx);
3408
3409 /* only if there is some info to store */
3410 if (info) {
3411 time_t info_time = info_get_time(info);
3412
3413 if (!info_oldest || info_time < info_oldest)
3414 info_oldest = info_time;
3415
3416 if (info_get_rehash(info))
3417 info_has_rehash = 1;
3418 }
3419 } else {
3420 /* clear any previous info */
3421 info_set(&state->infoarr, idx, 0);
3422
3423 /* and clear any deleted blocks */
3424 fs_position_clear_deleted(state, idx);
3425 }
3426 }
3427
3428 /* map disks */
3429 mapping_idx = 0;
3430 for (i = state->maplist; i != 0; i = i->next) {
3431 struct snapraid_map* map = i->data;
3432 struct snapraid_disk* disk;
3433
3434 /* find the disk for this mapping */
3435 disk = find_disk_by_name(state, map->name);
3436 if (!disk) {
3437 /* LCOV_EXCL_START */
3438 log_fatal("Internal inconsistency for unmapped disk '%s'\n", map->name);
3439 os_abort();
3440 /* LCOV_EXCL_STOP */
3441 }
3442
3443 /* save the mapping only for not empty disks */
3444 if (!fs_is_empty(disk, blockmax)) {
3445 /* assign the mapping index used to identify disks */
3446 disk->mapping_idx = mapping_idx;
3447 ++mapping_idx;
3448 } else {
3449 /* mark the disk as without mapping */
3450 disk->mapping_idx = -1;
3451 }
3452 }
3453
3454 #if HAVE_MT_WRITE
3455 /* start all writing threads */
3456 i = tommy_list_head(&state->contentlist);
3457 while (i) {
3458 struct snapraid_content* content = i->data;
3459 struct state_write_thread_context* context;
3460 char tmp[PATH_MAX];
3461 STREAM* f;
3462
3463 msg_progress("Saving state to %s...\n", content->content);
3464
3465 pathprint(tmp, sizeof(tmp), "%s.tmp", content->content);
3466
3467 /* ensure to delete a previous stale file */
3468 if (remove(tmp) != 0) {
3469 if (errno != ENOENT) {
3470 /* LCOV_EXCL_START */
3471 log_fatal("Error removing the stale content file '%s'. %s.\n", tmp, strerror(errno));
3472 exit(EXIT_FAILURE);
3473 /* LCOV_EXCL_STOP */
3474 }
3475 }
3476
3477 f = sopen_write(tmp);
3478 if (f == 0) {
3479 /* LCOV_EXCL_START */
3480 log_fatal("Error opening the temporary content file '%s'. %s.\n", tmp, strerror(errno));
3481 exit(EXIT_FAILURE);
3482 /* LCOV_EXCL_STOP */
3483 }
3484
3485 /* allocate the thread context */
3486 context = malloc_nofail(sizeof(struct state_write_thread_context));
3487 content->context = context;
3488
3489 /* initialize */
3490 context->state = state;
3491 context->blockmax = blockmax;
3492 context->info_oldest = info_oldest;
3493 context->info_now = info_now;
3494 context->info_has_rehash = info_has_rehash;
3495 context->f = f;
3496
3497 thread_create(&context->thread, state_write_thread, context);
3498
3499 i = i->next;
3500 }
3501
3502 /* join all thread */
3503 fail = 0;
3504 first = 1;
3505 crc = 0;
3506 count_file = 0;
3507 count_hardlink = 0;
3508 count_symlink = 0;
3509 count_dir = 0;
3510 i = tommy_list_head(&state->contentlist);
3511 while (i) {
3512 struct snapraid_content* content = i->data;
3513 struct state_write_thread_context* context = content->context;
3514 void* retval;
3515
3516 thread_join(context->thread, &retval);
3517
3518 if (retval) {
3519 /* LCOV_EXCL_START */
3520 fail = 1;
3521 /* LCOV_EXCL_STOP */
3522 } else {
3523 STREAM* f = context->f;
3524
3525 /* Use the sequence fflush() -> fsync() -> fclose() -> rename() to ensure */
3526 /* than even in a system crash event we have one valid copy of the file. */
3527 if (sflush(f) != 0) {
3528 /* LCOV_EXCL_START */
3529 log_fatal("Error writing the content file '%s', in flush(). %s.\n", serrorfile(f), strerror(errno));
3530 exit(EXIT_FAILURE);
3531 /* LCOV_EXCL_STOP */
3532 }
3533
3534 #if HAVE_FSYNC
3535 if (ssync(f) != 0) {
3536 /* LCOV_EXCL_START */
3537 log_fatal("Error writing the content file '%s' in sync(). %s.\n", serrorfile(f), strerror(errno));
3538 exit(EXIT_FAILURE);
3539 /* LCOV_EXCL_STOP */
3540 }
3541 #endif
3542
3543 if (sclose(f) != 0) {
3544 /* LCOV_EXCL_START */
3545 log_fatal("Error closing the content file. %s.\n", strerror(errno));
3546 exit(EXIT_FAILURE);
3547 /* LCOV_EXCL_STOP */
3548 }
3549
3550 if (first) {
3551 first = 0;
3552 crc = context->crc;
3553 count_file = context->count_file;
3554 count_hardlink = context->count_hardlink;
3555 count_symlink = context->count_symlink;
3556 count_dir = context->count_dir;
3557 } else {
3558 if (crc != context->crc) {
3559 /* LCOV_EXCL_START */
3560 log_fatal("Different CRCs writing content streams.\n");
3561 log_fatal("DANGER! Your RAM memory is broken! DO NOT PROCEED UNTIL FIXED!\n");
3562 log_fatal("Try running a memory test like http://www.memtest86.com/\n");
3563 exit(EXIT_FAILURE);
3564 /* LCOV_EXCL_STOP */
3565 }
3566 }
3567 }
3568
3569 free(context);
3570
3571 i = i->next;
3572 }
3573
3574 /* abort on failure */
3575 if (fail) {
3576 /* LCOV_EXCL_START */
3577 exit(EXIT_FAILURE);
3578 /* LCOV_EXCL_STOP */
3579 }
3580 #else
3581 /* count the content files */
3582 count_content = 0;
3583 i = tommy_list_head(&state->contentlist);
3584 while (i) {
3585 struct snapraid_content* content = i->data;
3586 msg_progress("Saving state to %s...\n", content->content);
3587 ++count_content;
3588 i = i->next;
3589 }
3590
3591 /* open all the content files */
3592 f = sopen_multi_write(count_content);
3593 if (!f) {
3594 /* LCOV_EXCL_START */
3595 log_fatal("Error opening the content files.\n");
3596 exit(EXIT_FAILURE);
3597 /* LCOV_EXCL_STOP */
3598 }
3599
3600 k = 0;
3601 i = tommy_list_head(&state->contentlist);
3602 while (i) {
3603 struct snapraid_content* content = i->data;
3604 char tmp[PATH_MAX];
3605 pathprint(tmp, sizeof(tmp), "%s.tmp", content->content);
3606
3607 /* ensure to delete a previous stale file */
3608 if (remove(tmp) != 0) {
3609 if (errno != ENOENT) {
3610 /* LCOV_EXCL_START */
3611 log_fatal("Error removing the stale content file '%s'. %s.\n", tmp, strerror(errno));
3612 exit(EXIT_FAILURE);
3613 /* LCOV_EXCL_STOP */
3614 }
3615 }
3616
3617 if (sopen_multi_file(f, k, tmp) != 0) {
3618 /* LCOV_EXCL_START */
3619 log_fatal("Error opening the temporary content file '%s'. %s.\n", tmp, strerror(errno));
3620 exit(EXIT_FAILURE);
3621 /* LCOV_EXCL_STOP */
3622 }
3623
3624 ++k;
3625 i = i->next;
3626 }
3627
3628 /* allocate the thread context */
3629 context = malloc_nofail(sizeof(struct state_write_thread_context));
3630
3631 /* initialize */
3632 context->state = state;
3633 context->blockmax = blockmax;
3634 context->info_oldest = info_oldest;
3635 context->info_now = info_now;
3636 context->info_has_rehash = info_has_rehash;
3637 context->f = f;
3638
3639 retval = state_write_thread(context);
3640
3641 /* abort on failure */
3642 if (retval) {
3643 /* LCOV_EXCL_START */
3644 exit(EXIT_FAILURE);
3645 /* LCOV_EXCL_STOP */
3646 }
3647
3648 /* Use the sequence fflush() -> fsync() -> fclose() -> rename() to ensure */
3649 /* than even in a system crash event we have one valid copy of the file. */
3650 if (sflush(f) != 0) {
3651 /* LCOV_EXCL_START */
3652 log_fatal("Error writing the content file '%s', in flush(). %s.\n", serrorfile(f), strerror(errno));
3653 exit(EXIT_FAILURE);
3654 /* LCOV_EXCL_STOP */
3655 }
3656
3657 #if HAVE_FSYNC
3658 if (ssync(f) != 0) {
3659 /* LCOV_EXCL_START */
3660 log_fatal("Error writing the content file '%s' in sync(). %s.\n", serrorfile(f), strerror(errno));
3661 exit(EXIT_FAILURE);
3662 /* LCOV_EXCL_STOP */
3663 }
3664 #endif
3665
3666 if (sclose(f) != 0) {
3667 /* LCOV_EXCL_START */
3668 log_fatal("Error closing the content file. %s.\n", strerror(errno));
3669 exit(EXIT_FAILURE);
3670 /* LCOV_EXCL_STOP */
3671 }
3672
3673 crc = context->crc;
3674 count_file = context->count_file;
3675 count_hardlink = context->count_hardlink;
3676 count_symlink = context->count_symlink;
3677 count_dir = context->count_dir;
3678
3679 free(context);
3680 #endif
3681
3682 msg_verbose("%8u files\n", count_file);
3683 msg_verbose("%8u hardlinks\n", count_hardlink);
3684 msg_verbose("%8u symlinks\n", count_symlink);
3685 msg_verbose("%8u empty dirs\n", count_dir);
3686
3687 *out_crc = crc;
3688 }
3689
state_read(struct snapraid_state * state)3690 void state_read(struct snapraid_state* state)
3691 {
3692 STREAM* f;
3693 char path[PATH_MAX];
3694 struct stat st;
3695 tommy_node* node;
3696 int ret;
3697 int c;
3698
3699 /* iterate over all the available content files and load the first one present */
3700 f = 0;
3701 node = tommy_list_head(&state->contentlist);
3702 while (node) {
3703 struct snapraid_content* content = node->data;
3704 pathcpy(path, sizeof(path), content->content);
3705
3706 if (!state->no_conf) {
3707 log_tag("content:%s\n", path);
3708 log_flush();
3709 }
3710 msg_progress("Loading state from %s...\n", path);
3711
3712 f = sopen_read(path);
3713 if (f != 0) {
3714 /* if opened stop the search */
3715 break;
3716 } else {
3717 /* if it's real error of an existing file, abort */
3718 if (errno != ENOENT) {
3719 /* LCOV_EXCL_START */
3720 log_fatal("Error opening the content file '%s'. %s.\n", path, strerror(errno));
3721 exit(EXIT_FAILURE);
3722 /* LCOV_EXCL_STOP */
3723 }
3724
3725 /* otherwise continue */
3726 if (node->next) {
3727 log_fatal("WARNING! Content file '%s' not found, trying with another copy...\n", path);
3728
3729 /* ensure to rewrite all the content files */
3730 state->need_write = 1;
3731 }
3732 }
3733
3734 /* next content file */
3735 node = node->next;
3736 }
3737
3738 /* if not found, assume empty */
3739 if (!f) {
3740 log_fatal("No content file found. Assuming empty.\n");
3741
3742 /* create the initial mapping */
3743 state_map(state);
3744 return;
3745 }
3746
3747 /* get the stat of the content file */
3748 ret = fstat(shandle(f), &st);
3749 if (ret != 0) {
3750 /* LCOV_EXCL_START */
3751 log_fatal("Error stating the content file '%s'. %s.\n", path, strerror(errno));
3752 exit(EXIT_FAILURE);
3753 /* LCOV_EXCL_STOP */
3754 }
3755
3756 /* go further to check other content files */
3757 while (node) {
3758 char other_path[PATH_MAX];
3759 struct stat other_st;
3760 struct snapraid_content* content = node->data;
3761 pathcpy(other_path, sizeof(other_path), content->content);
3762
3763 ret = stat(other_path, &other_st);
3764 if (ret != 0) {
3765 /* allow missing content files, but not any other kind of error */
3766 if (errno != ENOENT) {
3767 /* LCOV_EXCL_START */
3768 log_fatal("Error stating the content file '%s'. %s.\n", other_path, strerror(errno));
3769 exit(EXIT_FAILURE);
3770 /* LCOV_EXCL_STOP */
3771 }
3772
3773 /* ensure to rewrite all the content files */
3774 state->need_write = 1;
3775 } else {
3776 /* if the size is different */
3777 if (other_st.st_size != st.st_size) {
3778 log_fatal("WARNING! Content files '%s' and '%s' have a different size!\n", path, other_path);
3779 log_fatal("Likely one of the two is broken!\n");
3780
3781 /* ensure to rewrite all the content files */
3782 state->need_write = 1;
3783 }
3784 }
3785
3786 /* next content file */
3787 node = node->next;
3788 }
3789
3790 /* start with a undefined default. */
3791 /* it's for compatibility with version 1.0 where MD5 was implicit. */
3792 state->hash = HASH_UNDEFINED;
3793
3794 /* start with a zero seed, it was the default in old versions */
3795 memset(state->hashseed, 0, HASH_MAX);
3796
3797 /* previous hash, start with an undefined value */
3798 state->prevhash = HASH_UNDEFINED;
3799
3800 /* intentionally not set the prevhashseed, if used valgrind will warn about it */
3801
3802 /* get the first char to detect the file type */
3803 c = sgetc(f);
3804 sungetc(c, f);
3805
3806 /* guess the file type from the first char */
3807 if (c == 'S') {
3808 state_read_content(state, path, f);
3809 } else {
3810 /* LCOV_EXCL_START */
3811 log_fatal("From SnapRAID v9.0 the text content file is not supported anymore.\n");
3812 log_fatal("You have first to upgrade to SnapRAID v8.1 to convert it to binary format.\n");
3813 exit(EXIT_FAILURE);
3814 /* LCOV_EXCL_STOP */
3815 }
3816
3817 sclose(f);
3818
3819 if (state->hash == HASH_UNDEFINED) {
3820 /* LCOV_EXCL_START */
3821 log_fatal("The checksum to use is not specified.\n");
3822 log_fatal("This happens because you are likely upgrading from SnapRAID 1.0.\n");
3823 log_fatal("To use a new SnapRAID you must restart from scratch,\n");
3824 log_fatal("deleting all the content and parity files.\n");
3825 exit(EXIT_FAILURE);
3826 /* LCOV_EXCL_STOP */
3827 }
3828
3829 /* update the mapping */
3830 state_map(state);
3831
3832 state_content_check(state, path);
3833
3834 /* mark that we read the content file, and it passed all the checks */
3835 state->checked_read = 1;
3836 }
3837
3838 struct state_verify_thread_context {
3839 struct snapraid_state* state;
3840 struct snapraid_content* content;
3841 #if HAVE_MT_VERIFY
3842 thread_id_t thread;
3843 #else
3844 void* retval;
3845 #endif
3846 /* input */
3847 uint32_t crc;
3848 STREAM* f;
3849 };
3850
state_verify_thread(void * arg)3851 static void* state_verify_thread(void* arg)
3852 {
3853 struct state_verify_thread_context* context = arg;
3854 struct snapraid_content* content = context->content;
3855 STREAM* f = context->f;
3856 unsigned char buf[4];
3857 uint32_t crc_stored;
3858 uint32_t crc_computed;
3859 uint64_t start;
3860
3861 start = tick_ms();
3862
3863 if (sdeplete(f, buf) != 0) {
3864 /* LCOV_EXCL_START */
3865 log_fatal("Error flushing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3866 return context;
3867 /* LCOV_EXCL_STOP */
3868 }
3869
3870 /* get the stored crc from the last four bytes */
3871 crc_stored = buf[0] | (uint32_t)buf[1] << 8 | (uint32_t)buf[2] << 16 | (uint32_t)buf[3] << 24;
3872
3873 if (crc_stored != context->crc) {
3874 /* LCOV_EXCL_START */
3875 log_fatal("DANGER! Wrong stored CRC in '%s'\n", serrorfile(f));
3876 return context;
3877 /* LCOV_EXCL_STOP */
3878 }
3879
3880 /* get the computed crc */
3881 crc_computed = scrc(f);
3882
3883 /* adjust the stored crc to include itself */
3884 crc_stored = crc32c(crc_stored, buf, 4);
3885
3886 if (crc_computed != crc_stored) {
3887 /* LCOV_EXCL_START */
3888 log_fatal("DANGER! Wrong file CRC in '%s'\n", serrorfile(f));
3889 return context;
3890 /* LCOV_EXCL_STOP */
3891 }
3892
3893 msg_progress("Verified %s in %" PRIu64 " seconds\n", content->content, (tick_ms() - start) / 1000);
3894
3895 return 0;
3896 }
3897
state_verify_content(struct snapraid_state * state,uint32_t crc)3898 static void state_verify_content(struct snapraid_state* state, uint32_t crc)
3899 {
3900 tommy_node* i;
3901 int fail;
3902
3903 msg_progress("Verifying...\n");
3904
3905 /* start all reading threads */
3906 i = tommy_list_head(&state->contentlist);
3907 while (i) {
3908 struct snapraid_content* content = i->data;
3909 struct state_verify_thread_context* context;
3910 char tmp[PATH_MAX];
3911 STREAM* f;
3912
3913 pathprint(tmp, sizeof(tmp), "%s.tmp", content->content);
3914 f = sopen_read(tmp);
3915 if (f == 0) {
3916 /* LCOV_EXCL_START */
3917 log_fatal("Error reopening the temporary content file '%s'. %s.\n", tmp, strerror(errno));
3918 exit(EXIT_FAILURE);
3919 /* LCOV_EXCL_STOP */
3920 }
3921
3922 /* allocate the thread context */
3923 context = malloc_nofail(sizeof(struct state_verify_thread_context));
3924 content->context = context;
3925
3926 /* initialize */
3927 context->state = state;
3928 context->content = content;
3929 context->crc = crc;
3930 context->f = f;
3931
3932 #if HAVE_MT_VERIFY
3933 thread_create(&context->thread, state_verify_thread, context);
3934 #else
3935 context->retval = state_verify_thread(context);
3936 #endif
3937
3938 i = i->next;
3939 }
3940
3941 /* join all thread */
3942 fail = 0;
3943 i = tommy_list_head(&state->contentlist);
3944 while (i) {
3945 struct snapraid_content* content = i->data;
3946 struct state_verify_thread_context* context = content->context;
3947 void* retval;
3948
3949 #if HAVE_MT_VERIFY
3950 thread_join(context->thread, &retval);
3951 #else
3952 retval = context->retval;
3953 #endif
3954 if (retval) {
3955 /* LCOV_EXCL_START */
3956 fail = 1;
3957 /* LCOV_EXCL_STOP */
3958 } else {
3959 STREAM* f = context->f;
3960
3961 if (sclose(f) != 0) {
3962 /* LCOV_EXCL_START */
3963 log_fatal("Error closing the content file. %s.\n", strerror(errno));
3964 exit(EXIT_FAILURE);
3965 /* LCOV_EXCL_STOP */
3966 }
3967 }
3968
3969 free(context);
3970
3971 i = i->next;
3972 }
3973
3974 /* abort on failure */
3975 if (fail) {
3976 /* LCOV_EXCL_START */
3977 exit(EXIT_FAILURE);
3978 /* LCOV_EXCL_STOP */
3979 }
3980 }
3981
state_rename_content(struct snapraid_state * state)3982 static void state_rename_content(struct snapraid_state* state)
3983 {
3984 tommy_node* i;
3985
3986 #if defined(_linux) /* this sequence is linux specific */
3987 i = tommy_list_head(&state->contentlist);
3988 while (i) {
3989 struct snapraid_content* content = i->data;
3990 char tmp[PATH_MAX];
3991 char dir[PATH_MAX];
3992 char* slash;
3993 int handle;
3994
3995 pathcpy(dir, sizeof(dir), content->content);
3996
3997 slash = strrchr(tmp, '/');
3998 if (slash)
3999 *slash = 0;
4000 else
4001 pathcpy(dir, sizeof(dir), ".");
4002
4003 /* open the directory to get the handle */
4004 handle = open(dir, O_RDONLY | O_DIRECTORY);
4005 if (handle < 0) {
4006 /* LCOV_EXCL_START */
4007 log_fatal("Error opening the directory '%s'. %s.\n", dir, strerror(errno));
4008 exit(EXIT_FAILURE);
4009 /* LCOV_EXCL_STOP */
4010 }
4011
4012 /* now rename the just written copy with the correct name */
4013 pathprint(tmp, sizeof(tmp), "%s.tmp", content->content);
4014 if (rename(tmp, content->content) != 0) {
4015 /* LCOV_EXCL_START */
4016 log_fatal("Error renaming the content file '%s' to '%s'. %s.\n", tmp, content->content, strerror(errno));
4017 exit(EXIT_FAILURE);
4018 /* LCOV_EXCL_STOP */
4019 }
4020
4021 /* sync the directory */
4022 if (fsync(handle) != 0) {
4023 /* LCOV_EXCL_START */
4024 log_fatal("Error syncing the directory '%s'. %s.\n", dir, strerror(errno));
4025 exit(EXIT_FAILURE);
4026 /* LCOV_EXCL_STOP */
4027 }
4028
4029 if (close(handle) != 0) {
4030 /* LCOV_EXCL_START */
4031 log_fatal("Error closing the directory '%s'. %s.\n", dir, strerror(errno));
4032 exit(EXIT_FAILURE);
4033 /* LCOV_EXCL_STOP */
4034 }
4035
4036 i = i->next;
4037 }
4038 #else
4039 i = tommy_list_head(&state->contentlist);
4040 while (i) {
4041 struct snapraid_content* content = i->data;
4042 char tmp[PATH_MAX];
4043
4044 /* now renames the just written copy with the correct name */
4045 pathprint(tmp, sizeof(tmp), "%s.tmp", content->content);
4046 if (rename(tmp, content->content) != 0) {
4047 /* LCOV_EXCL_START */
4048 log_fatal("Error renaming the content file '%s' to '%s' in rename(). %s.\n", tmp, content->content, strerror(errno));
4049 exit(EXIT_FAILURE);
4050 /* LCOV_EXCL_STOP */
4051 }
4052
4053 i = i->next;
4054 }
4055 #endif
4056 }
4057
state_write(struct snapraid_state * state)4058 void state_write(struct snapraid_state* state)
4059 {
4060 uint32_t crc;
4061
4062 /* write all the content files */
4063 state_write_content(state, &crc);
4064
4065 /* verify the just written files */
4066 state_verify_content(state, crc);
4067
4068 /* rename the new files, over the old ones */
4069 state_rename_content(state);
4070
4071 state->need_write = 0; /* no write needed anymore */
4072 state->checked_read = 0; /* what we wrote is not checked in read */
4073 }
4074
state_skip(struct snapraid_state * state)4075 void state_skip(struct snapraid_state* state)
4076 {
4077 tommy_node* i;
4078
4079 /* for each disk */
4080 for (i = state->disklist; i != 0; i = i->next) {
4081 tommy_node* j;
4082 struct snapraid_disk* disk = i->data;
4083
4084 if (!disk->skip_access)
4085 continue;
4086
4087 /* for each file */
4088 for (j = tommy_list_head(&disk->filelist); j != 0; j = j->next) {
4089 struct snapraid_file* file = j->data;
4090 file_flag_set(file, FILE_IS_EXCLUDED);
4091 }
4092
4093 /* for each link */
4094 for (j = tommy_list_head(&disk->linklist); j != 0; j = j->next) {
4095 struct snapraid_link* slink = j->data;
4096 link_flag_set(slink, FILE_IS_EXCLUDED);
4097 }
4098
4099 /* for each dir */
4100 for (j = tommy_list_head(&disk->dirlist); j != 0; j = j->next) {
4101 struct snapraid_dir* dir = j->data;
4102 dir_flag_set(dir, FILE_IS_EXCLUDED);
4103 }
4104 }
4105 }
4106
state_filter(struct snapraid_state * state,tommy_list * filterlist_file,tommy_list * filterlist_disk,int filter_missing,int filter_error)4107 void state_filter(struct snapraid_state* state, tommy_list* filterlist_file, tommy_list* filterlist_disk, int filter_missing, int filter_error)
4108 {
4109 tommy_node* i;
4110 unsigned l;
4111
4112 /* if no filter, include all */
4113 if (!filter_missing && !filter_error && tommy_list_empty(filterlist_file) && tommy_list_empty(filterlist_disk))
4114 return;
4115
4116 msg_progress("Selecting...\n");
4117
4118 for (i = tommy_list_head(filterlist_disk); i != 0; i = i->next) {
4119 struct snapraid_filter* filter = i->data;
4120 msg_verbose("\t%s%s\n", filter->pattern, filter->is_disk ? "//" : "");
4121 }
4122 for (i = tommy_list_head(filterlist_file); i != 0; i = i->next) {
4123 struct snapraid_filter* filter = i->data;
4124 msg_verbose("\t%s%s\n", filter->pattern, filter->is_dir ? "/" : "");
4125 }
4126 if (filter_missing)
4127 msg_verbose("\t<missing>\n");
4128 if (filter_error)
4129 msg_verbose("\t<error>\n");
4130
4131 /* for each disk */
4132 for (i = state->disklist; i != 0; i = i->next) {
4133 tommy_node* j;
4134 struct snapraid_disk* disk = i->data;
4135
4136 /* if we filter for presence, we have to access the disk, so better to print something */
4137 if (filter_missing)
4138 msg_progress("Scanning disk %s...\n", disk->name);
4139
4140 /* for each file */
4141 for (j = tommy_list_head(&disk->filelist); j != 0; j = j->next) {
4142 struct snapraid_file* file = j->data;
4143
4144 if (filter_path(filterlist_disk, 0, disk->name, file->sub) != 0
4145 || filter_path(filterlist_file, 0, disk->name, file->sub) != 0
4146 || filter_existence(filter_missing, disk->dir, file->sub) != 0
4147 || filter_correctness(filter_error, &state->infoarr, disk, file) != 0
4148 ) {
4149 file_flag_set(file, FILE_IS_EXCLUDED);
4150 }
4151 }
4152
4153 /* for each link */
4154 for (j = tommy_list_head(&disk->linklist); j != 0; j = j->next) {
4155 struct snapraid_link* slink = j->data;
4156
4157 if (filter_path(filterlist_disk, 0, disk->name, slink->sub) != 0
4158 || filter_path(filterlist_file, 0, disk->name, slink->sub) != 0
4159 || filter_existence(filter_missing, disk->dir, slink->sub) != 0
4160 ) {
4161 link_flag_set(slink, FILE_IS_EXCLUDED);
4162 }
4163 }
4164
4165 /* for each empty dir */
4166 for (j = tommy_list_head(&disk->dirlist); j != 0; j = j->next) {
4167 struct snapraid_dir* dir = j->data;
4168
4169 if (filter_emptydir(filterlist_disk, 0, disk->name, dir->sub) != 0
4170 || filter_emptydir(filterlist_file, 0, disk->name, dir->sub) != 0
4171 || filter_existence(filter_missing, disk->dir, dir->sub) != 0
4172 ) {
4173 dir_flag_set(dir, FILE_IS_EXCLUDED);
4174 }
4175 }
4176 }
4177
4178 /* if we are filtering by disk, exclude any parity not explicitly included */
4179 if (!tommy_list_empty(filterlist_disk)) {
4180 /* for each parity disk */
4181 for (l = 0; l < state->level; ++l) {
4182 /* check if the parity is excluded by name */
4183 if (filter_path(filterlist_disk, 0, lev_config_name(l), 0) != 0) {
4184 /* excluded the parity from further operation */
4185 state->parity[l].is_excluded_by_filter = 1;
4186 }
4187 }
4188 } else {
4189 /* if we are filtering by file, exclude all parity */
4190 if (filter_missing || !tommy_list_empty(filterlist_file)) {
4191 /* for each parity disk */
4192 for (l = 0; l < state->level; ++l) {
4193 state->parity[l].is_excluded_by_filter = 1;
4194 }
4195 }
4196 }
4197 }
4198
state_progress_begin(struct snapraid_state * state,block_off_t blockstart,block_off_t blockmax,block_off_t countmax)4199 int state_progress_begin(struct snapraid_state* state, block_off_t blockstart, block_off_t blockmax, block_off_t countmax)
4200 {
4201 time_t now;
4202
4203 if (state->opt.gui) {
4204 log_tag("run:begin:%u:%u:%u\n", blockstart, blockmax, countmax);
4205 log_flush();
4206 }
4207
4208 now = time(0);
4209
4210 state->progress_whole_start = now;
4211
4212 state->progress_tick = 0;
4213 state->progress_ptr = 0;
4214 state->progress_wasted = 0;
4215
4216 /* stop if requested */
4217 if (global_interrupt) {
4218 /* LCOV_EXCL_START */
4219 if (!state->opt.gui) {
4220 msg_status("Not starting for interruption\n");
4221 }
4222 log_tag("sigint:0: SIGINT received\n");
4223 log_flush();
4224 return 0;
4225 /* LCOV_EXCL_STOP */
4226 }
4227
4228 return 1;
4229 }
4230
state_progress_end(struct snapraid_state * state,block_off_t countpos,block_off_t countmax,data_off_t countsize)4231 void state_progress_end(struct snapraid_state* state, block_off_t countpos, block_off_t countmax, data_off_t countsize)
4232 {
4233 if (state->opt.gui) {
4234 log_tag("run:end\n");
4235 log_flush();
4236 } else if (countmax == 0) {
4237 msg_status("Nothing to do\n");
4238 } else {
4239 time_t now;
4240 time_t elapsed;
4241
4242 unsigned countsize_MB = (countsize + MEGA - 1) / MEGA;
4243
4244 now = time(0);
4245
4246 elapsed = now - state->progress_whole_start - state->progress_wasted;
4247
4248 msg_bar("%u%% completed, %u MB accessed", countpos * 100 / countmax, countsize_MB);
4249
4250 msg_bar(" in %u:%02u", (unsigned)(elapsed / 3600), (unsigned)((elapsed % 3600) / 60));
4251
4252 /* some extra spaces to ensure to overwrite the latest status line */
4253 msg_bar(" ");
4254
4255 msg_bar("\n");
4256 msg_flush();
4257 }
4258 }
4259
state_progress_stop(struct snapraid_state * state)4260 void state_progress_stop(struct snapraid_state* state)
4261 {
4262 time_t now;
4263
4264 now = time(0);
4265
4266 if (!state->opt.gui) {
4267 msg_bar("\n");
4268 msg_flush();
4269 }
4270
4271 state->progress_interruption = now;
4272 }
4273
state_progress_restart(struct snapraid_state * state)4274 void state_progress_restart(struct snapraid_state* state)
4275 {
4276 time_t now;
4277
4278 now = time(0);
4279
4280 /* reset the progress counter */
4281 state->progress_tick = 0;
4282 state->progress_ptr = 0;
4283
4284 if (now >= state->progress_interruption) /* avoid degenerated cases when the clock is manually adjusted */
4285 state->progress_wasted += now - state->progress_interruption;
4286 }
4287
4288 #define PROGRESS_CLEAR " "
4289
4290 /**
4291 * Set the latest tick data in the progress vector.
4292 */
state_progress_latest(struct snapraid_state * state)4293 static void state_progress_latest(struct snapraid_state* state)
4294 {
4295 tommy_node* i;
4296 unsigned l;
4297
4298 state->progress_tick_misc[state->progress_ptr] = state->tick_misc;
4299 state->progress_tick_sched[state->progress_ptr] = state->tick_sched;
4300 state->progress_tick_raid[state->progress_ptr] = state->tick_raid;
4301 state->progress_tick_hash[state->progress_ptr] = state->tick_hash;
4302 state->progress_tick_io[state->progress_ptr] = state->tick_io;
4303 for (i = state->disklist; i != 0; i = i->next) {
4304 struct snapraid_disk* disk = i->data;
4305 disk->progress_tick[state->progress_ptr] = disk->tick;
4306 }
4307 for (l = 0; l < state->level; ++l)
4308 state->parity[l].progress_tick[state->progress_ptr] = state->parity[l].tick;
4309 }
4310
4311 /**
4312 * Number of columns
4313 */
4314 #define GRAPH_COLUMN 68
4315
4316 /**
4317 * Get the reference value, 0 if index is PROGRESS_MAX
4318 */
4319 #define ref(map, index) (index < PROGRESS_MAX ? map[index] : 0)
4320
state_progress_graph(struct snapraid_state * state,struct snapraid_io * io,unsigned current,unsigned oldest)4321 static void state_progress_graph(struct snapraid_state* state, struct snapraid_io* io, unsigned current, unsigned oldest)
4322 {
4323 uint64_t v;
4324 uint64_t tick_total;
4325 unsigned bar;
4326 tommy_node* i;
4327 unsigned l;
4328 size_t pad;
4329
4330 tick_total = 0;
4331
4332 tick_total += state->progress_tick_misc[current] - ref(state->progress_tick_misc, oldest);
4333 tick_total += state->progress_tick_sched[current] - ref(state->progress_tick_sched, oldest);
4334 tick_total += state->progress_tick_raid[current] - ref(state->progress_tick_raid, oldest);
4335 tick_total += state->progress_tick_hash[current] - ref(state->progress_tick_hash, oldest);
4336 tick_total += state->progress_tick_io[current] - ref(state->progress_tick_io, oldest);
4337 if (!tick_total)
4338 return;
4339
4340 pad = 4;
4341 for (i = state->disklist; i != 0; i = i->next) {
4342 struct snapraid_disk* disk = i->data;
4343 size_t len = strlen(disk->name);
4344 if (pad < len)
4345 pad = len;
4346 }
4347 for (l = 0; l < state->level; ++l) {
4348 size_t len = strlen(lev_config_name(l));
4349 if (pad < len)
4350 pad = len;
4351 }
4352
4353 /* extra space */
4354 pad += 1;
4355
4356 if (pad + 30 < GRAPH_COLUMN)
4357 bar = GRAPH_COLUMN - pad;
4358 else
4359 bar = 30;
4360
4361 if (io) {
4362 const char* legend = "cached blocks (instant, more is better)";
4363
4364 /* refresh the cached blocks info */
4365 io_refresh(io);
4366
4367 printf("\n");
4368
4369 /* search for the slowest */
4370 for (i = state->disklist; i != 0; i = i->next) {
4371 struct snapraid_disk* disk = i->data;
4372 v = disk->cached_blocks;
4373 printr(disk->name, pad);
4374 printf("%4" PRIu64 " | ", v);
4375
4376 if (disk->progress_file && disk->progress_file->sub)
4377 printf("%s", disk->progress_file->sub);
4378 else
4379 printf("-");
4380
4381 printf("\n");
4382 }
4383
4384 for (l = 0; l < state->level; ++l) {
4385 v = state->parity[l].cached_blocks;
4386 printr(lev_config_name(l), pad);
4387 printf("%4" PRIu64 " | ", v);
4388 printc('o', v * bar / io->io_max);
4389 printf("\n");
4390 }
4391
4392 printc(' ', pad);
4393 printf(" |_");
4394 printc('_', bar);
4395 printf("\n");
4396
4397 printc(' ', 5 + pad + 1 + bar / 2 - strlen(legend) / 2);
4398 printf("%s", legend);
4399 printf("\n");
4400 }
4401
4402 printf("\n");
4403
4404 for (i = state->disklist; i != 0; i = i->next) {
4405 struct snapraid_disk* disk = i->data;
4406 v = disk->progress_tick[current] - ref(disk->progress_tick, oldest);
4407 printr(disk->name, pad);
4408 printf("%3" PRIu64 "%% | ", v * 100 / tick_total);
4409 printc('*', v * bar / tick_total);
4410 printf("\n");
4411
4412 /* clear the file in progress */
4413 disk->progress_file = 0;
4414 }
4415
4416 for (l = 0; l < state->level; ++l) {
4417 v = state->parity[l].progress_tick[current] - ref(state->parity[l].progress_tick, oldest);
4418 printr(lev_config_name(l), pad);
4419 printf("%3" PRIu64 "%% | ", v * 100 / tick_total);
4420 printc('*', v * bar / tick_total);
4421 printf("\n");
4422 }
4423
4424 v = state->progress_tick_raid[current] - ref(state->progress_tick_raid, oldest);
4425 printr("raid", pad);
4426 printf("%3" PRIu64 "%% | ", v * 100 / tick_total);
4427 printc('*', v * bar / tick_total);
4428 printf("\n");
4429
4430 v = state->progress_tick_hash[current] - ref(state->progress_tick_hash, oldest);
4431 printr("hash", pad);
4432 printf("%3" PRIu64 "%% | ", v * 100 / tick_total);
4433 printc('*', v * bar / tick_total);
4434 printf("\n");
4435
4436 v = state->progress_tick_sched[current] - ref(state->progress_tick_sched, oldest);
4437 printr("sched", pad);
4438 printf("%3" PRIu64 "%% | ", v * 100 / tick_total);
4439 printc('*', v * bar / tick_total);
4440 printf("\n");
4441
4442 v = state->progress_tick_misc[current] - ref(state->progress_tick_misc, oldest);
4443 printr("misc", pad);
4444 printf("%3" PRIu64 "%% | ", v * 100 / tick_total);
4445 printc('*', v * bar / tick_total);
4446 printf("\n");
4447
4448 printc(' ', pad);
4449 printf(" |_");
4450 printc('_', bar);
4451 printf("\n");
4452
4453 if (oldest == PROGRESS_MAX) {
4454 const char* legend = "wait time (total, less is better)";
4455 printc(' ', 5 + pad + 1 + bar / 2 - strlen(legend) / 2);
4456 printf("%s", legend);
4457 printf("\n");
4458 } else {
4459 const char* legend_d = "wait time (last %d secs, less is better)";
4460 printc(' ', 5 + pad + 1 + bar / 2 - strlen(legend_d) / 2);
4461 printf(legend_d, PROGRESS_MAX);
4462 printf("\n");
4463 }
4464
4465 printf("\n");
4466 }
4467
state_progress(struct snapraid_state * state,struct snapraid_io * io,block_off_t blockpos,block_off_t countpos,block_off_t countmax,data_off_t countsize)4468 int state_progress(struct snapraid_state* state, struct snapraid_io* io, block_off_t blockpos, block_off_t countpos, block_off_t countmax, data_off_t countsize)
4469 {
4470 time_t now;
4471 int pred;
4472
4473 now = time(0);
4474
4475 /* previous position */
4476 pred = state->progress_ptr + PROGRESS_MAX - 1;
4477 if (pred >= PROGRESS_MAX)
4478 pred -= PROGRESS_MAX;
4479
4480 /* if the previous measure is different */
4481 if (state->progress_tick == 0
4482 || state->progress_time[pred] != now
4483 ) {
4484 time_t elapsed;
4485 unsigned out_perc = 0;
4486 unsigned out_size_speed = 0;
4487 unsigned out_block_speed = 0;
4488 unsigned out_cpu = 0;
4489 unsigned out_eta = 0;
4490 int out_computed = 0;
4491
4492 /* store the new measure */
4493 state->progress_time[state->progress_ptr] = now;
4494 state->progress_pos[state->progress_ptr] = countpos;
4495 state->progress_size[state->progress_ptr] = countsize;
4496 state_progress_latest(state);
4497
4498 elapsed = now - state->progress_whole_start - state->progress_wasted;
4499
4500 /* completion percentage */
4501 if (countmax)
4502 out_perc = countpos * 100 / countmax;
4503
4504 /* if we have at least 5 measures */
4505 if (state->progress_tick >= 5
4506 /* or if we are running in test mode, with at least one measure */
4507 || (state->opt.force_progress && state->progress_tick >= 1)
4508 ) {
4509 int oldest;
4510 int past;
4511 time_t delta_time;
4512 block_off_t delta_pos;
4513 data_off_t delta_size;
4514 uint64_t tick_cpu;
4515 uint64_t tick_total;
4516 uint64_t delta_tick_cpu;
4517 uint64_t delta_tick_total;
4518 uint64_t oldest_tick_cpu;
4519 uint64_t oldest_tick_total;
4520
4521 /* number of past measures */
4522 past = state->progress_tick;
4523
4524 /* drop the oldest ones, to promptly */
4525 /* skip the startup phase */
4526 past -= past / 5;
4527
4528 /* check how much we can go in the past */
4529 if (past >= PROGRESS_MAX - 1) {
4530 /* the vector is filled, so we are already in position */
4531 /* to get the possible oldest one */
4532 oldest = state->progress_ptr + 1;
4533 } else {
4534 /* go backward the number of positions selected */
4535 oldest = state->progress_ptr + PROGRESS_MAX - past;
4536 }
4537 if (oldest >= PROGRESS_MAX)
4538 oldest -= PROGRESS_MAX;
4539
4540 tick_cpu = state->progress_tick_misc[state->progress_ptr]
4541 + state->progress_tick_sched[state->progress_ptr]
4542 + state->progress_tick_raid[state->progress_ptr]
4543 + state->progress_tick_hash[state->progress_ptr];
4544 tick_total = tick_cpu + state->progress_tick_io[state->progress_ptr];
4545
4546 oldest_tick_cpu = state->progress_tick_misc[oldest]
4547 + state->progress_tick_sched[oldest]
4548 + state->progress_tick_raid[oldest]
4549 + state->progress_tick_hash[oldest];
4550 oldest_tick_total = oldest_tick_cpu + state->progress_tick_io[oldest];
4551
4552 delta_time = now - state->progress_time[oldest];
4553 delta_pos = countpos - state->progress_pos[oldest];
4554 delta_size = countsize - state->progress_size[oldest];
4555 delta_tick_cpu = tick_cpu - oldest_tick_cpu;
4556 delta_tick_total = tick_total - oldest_tick_total;
4557
4558 /* estimate the speed in MB/s */
4559 if (delta_time != 0)
4560 out_size_speed = (unsigned)(delta_size / MEGA / delta_time);
4561
4562 /* estimate the speed in block/s */
4563 if (delta_time != 0)
4564 out_block_speed = (unsigned)(delta_pos / delta_time);
4565
4566 /* estimate the cpu usage percentage */
4567 if (delta_tick_total != 0)
4568 out_cpu = (unsigned)(delta_tick_cpu * 100U / delta_tick_total);
4569
4570 /* estimate the remaining time in minutes */
4571 if (delta_pos != 0)
4572 out_eta = (countmax - countpos) * delta_time / (60 * delta_pos);
4573
4574 if (state->opt.force_stats) {
4575 os_clear();
4576 state_progress_graph(state, io, state->progress_ptr, oldest);
4577 }
4578
4579 /* we have the output value */
4580 out_computed = 1;
4581 }
4582
4583 if (state->opt.gui) {
4584 log_tag("run:pos:%u:%u:%" PRIu64 ":%u:%u:%u:%u:%" PRIu64 "\n", blockpos, countpos, countsize, out_perc, out_eta, out_size_speed, out_cpu, (uint64_t)elapsed);
4585 log_flush();
4586 } else {
4587 msg_bar("%u%%, %u MB", out_perc, (unsigned)(countsize / MEGA));
4588 if (out_computed) {
4589 msg_bar(", %u MB/s", out_size_speed);
4590 msg_bar(", %u stripe/s", out_block_speed);
4591 msg_bar(", CPU %u%%", out_cpu);
4592 msg_bar(", %u:%02u ETA", out_eta / 60, out_eta % 60);
4593 }
4594 msg_bar("%s\r", PROGRESS_CLEAR);
4595 msg_flush();
4596 }
4597
4598 /* next position to fill */
4599 ++state->progress_ptr;
4600 if (state->progress_ptr >= PROGRESS_MAX)
4601 state->progress_ptr -= PROGRESS_MAX;
4602
4603 /* one more measure */
4604 ++state->progress_tick;
4605 }
4606
4607 /* stop if requested */
4608 if (global_interrupt) {
4609 /* LCOV_EXCL_START */
4610 if (!state->opt.gui) {
4611 log_fatal("\n");
4612 log_fatal("Stopping for interruption at block %u\n", blockpos);
4613 }
4614 log_tag("sigint:%u: SIGINT received\n", blockpos);
4615 log_flush();
4616 return 1;
4617 /* LCOV_EXCL_STOP */
4618 }
4619
4620 return 0;
4621 }
4622
state_usage_waste(struct snapraid_state * state)4623 void state_usage_waste(struct snapraid_state* state)
4624 {
4625 uint64_t now = tick();
4626
4627 state->tick_last = now;
4628 }
4629
state_usage_misc(struct snapraid_state * state)4630 void state_usage_misc(struct snapraid_state* state)
4631 {
4632 uint64_t now = tick();
4633 uint64_t delta = now - state->tick_last;
4634
4635 /* increment the time spent in computations */
4636 state->tick_misc += delta;
4637
4638 state->tick_last = now;
4639 }
4640
state_usage_sched(struct snapraid_state * state)4641 void state_usage_sched(struct snapraid_state* state)
4642 {
4643 uint64_t now = tick();
4644 uint64_t delta = now - state->tick_last;
4645
4646 /* increment the time spent in computations */
4647 state->tick_sched += delta;
4648
4649 state->tick_last = now;
4650 }
4651
state_usage_raid(struct snapraid_state * state)4652 void state_usage_raid(struct snapraid_state* state)
4653 {
4654 uint64_t now = tick();
4655 uint64_t delta = now - state->tick_last;
4656
4657 /* increment the time spent in computations */
4658 state->tick_raid += delta;
4659
4660 state->tick_last = now;
4661 }
4662
state_usage_hash(struct snapraid_state * state)4663 void state_usage_hash(struct snapraid_state* state)
4664 {
4665 uint64_t now = tick();
4666 uint64_t delta = now - state->tick_last;
4667
4668 /* increment the time spent in computations */
4669 state->tick_hash += delta;
4670
4671 state->tick_last = now;
4672 }
4673
state_usage_file(struct snapraid_state * state,struct snapraid_disk * disk,struct snapraid_file * file)4674 void state_usage_file(struct snapraid_state* state, struct snapraid_disk* disk, struct snapraid_file* file)
4675 {
4676 (void)state;
4677
4678 disk->progress_file = file;
4679 }
4680
state_usage_disk(struct snapraid_state * state,struct snapraid_handle * handle_map,unsigned * waiting_map,unsigned waiting_mac)4681 void state_usage_disk(struct snapraid_state* state, struct snapraid_handle* handle_map, unsigned* waiting_map, unsigned waiting_mac)
4682 {
4683 uint64_t now = tick();
4684 uint64_t delta = now - state->tick_last;
4685 unsigned i;
4686
4687 /* increment the time spent in the data disks */
4688 for (i = 0; i < waiting_mac; ++i) {
4689 struct snapraid_disk* disk = handle_map[waiting_map[i]].disk;
4690
4691 if (!disk)
4692 continue;
4693
4694 disk->tick += delta;
4695 }
4696 state->tick_io += delta;
4697
4698 state->tick_last = now;
4699 }
4700
state_usage_parity(struct snapraid_state * state,unsigned * waiting_map,unsigned waiting_mac)4701 void state_usage_parity(struct snapraid_state* state, unsigned* waiting_map, unsigned waiting_mac)
4702 {
4703 uint64_t now = tick();
4704 uint64_t delta = now - state->tick_last;
4705 unsigned i;
4706
4707 /* increment the time spent in the parity disk */
4708 for (i = 0; i < waiting_mac; ++i)
4709 state->parity[waiting_map[i]].tick += delta;
4710 state->tick_io += delta;
4711
4712 state->tick_last = now;
4713 }
4714
state_usage_print(struct snapraid_state * state)4715 void state_usage_print(struct snapraid_state* state)
4716 {
4717 /* set the latest data */
4718 state_progress_latest(state);
4719
4720 if (msg_level < MSG_PROGRESS)
4721 return;
4722
4723 /* print a graph for it */
4724 state_progress_graph(state, 0, state->progress_ptr, PROGRESS_MAX);
4725 }
4726
state_fscheck(struct snapraid_state * state,const char * ope)4727 void state_fscheck(struct snapraid_state* state, const char* ope)
4728 {
4729 tommy_node* i;
4730
4731 /* check the file-system on all disks */
4732 for (i = state->disklist; i != 0; i = i->next) {
4733 struct snapraid_disk* disk = i->data;
4734
4735 if (fs_check(disk) != 0) {
4736 /* LCOV_EXCL_START */
4737 log_fatal("Internal inconsistency in file-system for disk '%s' %s\n", disk->name, ope);
4738 os_abort();
4739 /* LCOV_EXCL_STOP */
4740 }
4741 }
4742 }
4743
generate_configuration(const char * path)4744 void generate_configuration(const char* path)
4745 {
4746 struct snapraid_state state;
4747 struct snapraid_content* content;
4748 char esc_buffer[ESC_MAX];
4749 unsigned l, s;
4750 tommy_node* j;
4751
4752 state_init(&state);
4753
4754 /* mark that we are without a configuration file */
4755 state.no_conf = 1;
4756
4757 /* create the dummy content entry */
4758 content = content_alloc(path, -1);
4759
4760 /* adds the content entry */
4761 tommy_list_insert_tail(&state.contentlist, &content->node, content);
4762
4763 /* read the content file */
4764 state_read(&state);
4765
4766 /* output a dummy configuration file */
4767 printf("# Configuration file generated from %s\n", path);
4768 printf("\n");
4769 printf("# Use this blocksize\n");
4770 printf("blocksize %u\n", state.block_size / KIBI);
4771 printf("\n");
4772 printf("# Use this hashsize\n");
4773 printf("hashsize %u\n", BLOCK_HASH_SIZE);
4774 printf("\n");
4775 for (l = 0; l < state.level; ++l) {
4776 printf("# Set the correct path for the %s files\n", lev_name(l));
4777 printf("# You had %u of them:\n", state.parity[l].split_mac);
4778 for (s = 0; s < state.parity[l].split_mac; ++s) {
4779 printf("# %u:\n", s);
4780 printf("# PATH:");
4781 if (state.parity[l].split_map[s].path[0])
4782 printf("%s", state.parity[l].split_map[s].path);
4783 else
4784 printf("?");
4785 printf("\n");
4786 printf("# SIZE:");
4787 if (state.parity[l].split_map[s].size != PARITY_SIZE_INVALID)
4788 printf("%" PRIu64, state.parity[l].split_map[s].size);
4789 else
4790 printf("?");
4791 printf("\n");
4792 printf("# UUID:");
4793 if (state.parity[l].split_map[s].uuid[0])
4794 printf("%s", state.parity[l].split_map[s].uuid);
4795 else
4796 printf("?");
4797 printf("\n");
4798 printf("#\n");
4799 }
4800 printf("%s ENTER_HERE_THE_PARITY_FILES_COMMA_SEPARATED\n", lev_config_name(l));
4801 printf("\n");
4802 }
4803 printf("# Add any other content file\n");
4804 printf("content %s\n", path);
4805 printf("\n");
4806 for (j = state.maplist; j; j = j->next) {
4807 struct snapraid_map* map = j->data;
4808 struct snapraid_disk* disk;
4809 printf("# Set the correct dir for disk '%s'\n", map->name);
4810 if (map->uuid[0])
4811 printf("# Disk '%s' is the one with id '%s'\n", map->name, map->uuid);
4812 disk = find_disk_by_name(&state, map->name);
4813 if (disk && disk->filelist) {
4814 struct snapraid_file* file = disk->filelist->data;
4815 if (file) {
4816 printf("# and containing: %s\n", fmt_poll(disk, file->sub, esc_buffer));
4817 }
4818 }
4819 printf("data %s ENTER_HERE_THE_DIR\n", map->name);
4820 printf("\n");
4821 }
4822
4823 state_done(&state);
4824 }
4825
4826