1 #include <check.h>
2 #include <stdint.h>
3 #include <stdlib.h>
4 #include <stdio.h>
5 #include "../../../test.h"
6 #include "../../../builders/server/protocol2/champ_chooser/build_dindex.h"
7 #include "../../../../src/alloc.h"
8 #include "../../../../src/fsops.h"
9 #include "../../../../src/hexmap.h"
10 #include "../../../../src/prepend.h"
11 #include "../../../../src/server/protocol2/champ_chooser/dindex.h"
12 #include "../../../../src/server/sdirs.h"
13
14 #define CNAME "utestclient"
15 #define BASE "utest_dindex"
16
tear_down(struct sdirs ** sdirs)17 static void tear_down(struct sdirs **sdirs)
18 {
19 fail_unless(recursive_delete(BASE)==0);
20 sdirs_free(sdirs);
21 alloc_check();
22 }
23
do_sdirs_init(void)24 static struct sdirs *do_sdirs_init(void)
25 {
26 struct sdirs *sdirs;
27 fail_unless((sdirs=sdirs_alloc())!=NULL);
28 fail_unless(!sdirs_init(sdirs, PROTO_2,
29 BASE, // directory
30 CNAME,
31 NULL, // client_lockdir
32 "a_group", // dedup_group
33 NULL // manual_delete
34 ));
35 return sdirs;
36 }
37
setup(void)38 static struct sdirs *setup(void)
39 {
40 struct sdirs *sdirs;
41 sdirs=do_sdirs_init();
42 fail_unless(recursive_delete(BASE)==0);
43 fail_unless(!build_path_w(sdirs->data));
44 fail_unless(!build_path_w(sdirs->clients));
45 fail_unless(!mkdir(sdirs->data, 0777));
46 fail_unless(!mkdir(sdirs->clients, 0777));
47 return sdirs;
48 }
49
get_path(const char * dir,uint64_t savepath)50 static char *get_path(const char *dir, uint64_t savepath)
51 {
52 char *path=NULL;
53 const char *savepath_str;
54 savepath_str=uint64_to_savepathstr(savepath);
55 fail_unless((path=prepend_s(dir, savepath_str))!=NULL);
56 return path;
57 }
58
create_path(const char * path)59 static void create_path(const char *path)
60 {
61 FILE *fp=NULL;
62 fail_unless(!build_path_w(path));
63 fail_unless((fp=fopen(path, "wb"))!=NULL);
64 fail_unless(!fclose(fp));
65 }
66
create_data_file(const char * dir,uint64_t savepath)67 static void create_data_file(const char *dir, uint64_t savepath)
68 {
69 char *path=NULL;
70 path=get_path(dir, savepath);
71 create_path(path);
72 free_w(&path);
73 }
74
create_data_files(const char * dir,uint64_t * arr,size_t len)75 static void create_data_files(const char *dir, uint64_t *arr, size_t len)
76 {
77 size_t l;
78 for(l=0; l<len; l++)
79 create_data_file(dir, arr[l]);
80 }
81
assert_existence(const char * dir,uint64_t savepath,int exists)82 static void assert_existence(const char *dir,
83 uint64_t savepath, int exists)
84 {
85 char *path=NULL;
86 struct stat statp;
87 path=get_path(dir, savepath);
88 if(lstat(path, &statp))
89 fail_unless(exists==0);
90 else
91 fail_unless(exists==1);
92 free_w(&path);
93 }
94
assert_existences(const char * dir,uint64_t * arr,size_t len,int exists)95 static void assert_existences(const char *dir,
96 uint64_t *arr, size_t len, int exists)
97 {
98 size_t l;
99 for(l=0; l<len; l++)
100 assert_existence(dir, arr[l], exists);
101 }
102
common_di(uint64_t * dold,size_t dolen,uint64_t * dnew,size_t dnlen,uint64_t * deleted,size_t deletedlen)103 static void common_di(uint64_t *dold, size_t dolen,
104 uint64_t *dnew, size_t dnlen,
105 uint64_t *deleted, size_t deletedlen)
106 {
107 struct sdirs *sdirs;
108 char *dold_path;
109 char *dnew_path;
110 sdirs=setup();
111 fail_unless((dold_path=prepend_s(sdirs->data, "dindex.old"))!=NULL);
112 fail_unless((dnew_path=prepend_s(sdirs->data, "dindex.new"))!=NULL);
113 build_dindex(dold, dolen, dold_path);
114 build_dindex(dnew, dnlen, dnew_path);
115
116 create_data_files(sdirs->data, dold, dolen);
117 create_data_files(sdirs->data, dnew, dnlen);
118
119 fail_unless(!compare_dindexes_and_unlink_datafiles(dold_path,
120 dnew_path, sdirs->data));
121 assert_existences(sdirs->data,
122 deleted, deletedlen, 0 /* does not exist */);
123 assert_existences(sdirs->data,
124 dnew, dnlen, 1 /* does exist */);
125
126 free_w(&dold_path);
127 free_w(&dnew_path);
128 tear_down(&sdirs);
129 }
130
131 static uint64_t din1[3]={
132 0x1111222233330000,
133 0x1111222244440000,
134 0x2222222266660000
135 };
136
137 static uint64_t din2[2]={
138 0x1111222233330000,
139 0x2222222266660000
140 };
141
142 static uint64_t del1[1]={
143 0x1111222244440000
144 };
145
146 static uint64_t din3[1]={
147 0x1111222233330000
148 };
149
150 static uint64_t del2[1]={
151 0x1111222266660000
152 };
153
START_TEST(test_dindex)154 START_TEST(test_dindex)
155 {
156 hexmap_init();
157 common_di(NULL,0, NULL,0, NULL,0);
158 common_di(din1,ARR_LEN(din1), din1,ARR_LEN(din1), NULL,0);
159 common_di(din1,ARR_LEN(din1), din2,ARR_LEN(din2), del1,ARR_LEN(del1));
160 common_di(NULL,0, din1,ARR_LEN(din1), NULL,0);
161 common_di(din2,ARR_LEN(din2), din1,ARR_LEN(din1), NULL,0);
162 common_di(din2,ARR_LEN(din2), din3,ARR_LEN(din3), del2,ARR_LEN(del2));
163 }
164 END_TEST
165
START_TEST(test_delete_unused_data_files_error)166 START_TEST(test_delete_unused_data_files_error)
167 {
168 fail_unless(delete_unused_data_files(NULL, 0)==-1);
169 alloc_check();
170 }
171 END_TEST
172
173 static uint64_t din0[0]={
174 };
175
create_client_path(const char * client,const char * fname)176 static void create_client_path(const char *client, const char *fname)
177 {
178 char *tmp;
179 fail_unless((tmp=prepend_s(client, fname))!=NULL);
180 create_path(tmp);
181 free_w(&tmp);
182 }
183
do_delete_unused_data_files(int resume,int old_dindex,int c1w,int c1f,int c2w,int c2f,uint64_t * dfiles,size_t dfileslen,uint64_t * c1cf,size_t c1cflen,uint64_t * c2cf,size_t c2cflen,uint64_t * c1df,size_t c1dflen,uint64_t * c2df,size_t c2dflen,uint64_t * exists,size_t existslen,uint64_t * deleted,size_t deletedlen)184 static void do_delete_unused_data_files(
185 int resume, int old_dindex,
186 int c1w, int c1f, int c2w, int c2f,
187 uint64_t *dfiles, size_t dfileslen,
188 uint64_t *c1cf, size_t c1cflen,
189 uint64_t *c2cf, size_t c2cflen,
190 uint64_t *c1df, size_t c1dflen,
191 uint64_t *c2df, size_t c2dflen,
192 uint64_t *exists, size_t existslen,
193 uint64_t *deleted, size_t deletedlen)
194 {
195 char *client1;
196 char *client2;
197 char *c1cfiles;
198 char *c2cfiles;
199 char *c1dfiles;
200 char *c2dfiles;
201 char *dindex_old=NULL;
202 struct sdirs *sdirs;
203
204 hexmap_init();
205 sdirs=setup();
206
207 fail_unless((c1cfiles=prepend_s(sdirs->cfiles, CNAME "-rand"))!=NULL);
208 fail_unless((c2cfiles=prepend_s(sdirs->cfiles, "client2-rand"))!=NULL);
209 fail_unless(!build_path_w(c1cfiles));
210
211 client1=sdirs->client;
212 c1dfiles=sdirs->dfiles;
213 fail_unless((client2=prepend_s(sdirs->clients, "client2"))!=NULL);
214 fail_unless((c2dfiles=prepend_s(client2, "dfiles"))!=NULL);
215
216 fail_unless(!build_path_w(c1dfiles));
217 fail_unless(!build_path_w(c2dfiles));
218
219 // Data files created.
220 build_dindex(c1cf, c1cflen, c1cfiles);
221 build_dindex(c2cf, c2cflen, c2cfiles);
222
223 // Data files in use by finished backups.
224 build_dindex(c1df, c1dflen, c1dfiles);
225 build_dindex(c2df, c2dflen, c2dfiles);
226
227 if(old_dindex)
228 {
229 // FIX THIS: Just leaving the old dindex empty.
230 fail_unless((dindex_old=
231 prepend_s(sdirs->data, "dindex"))!=NULL);
232 build_dindex(0, 0, dindex_old);
233 }
234
235 if(c1w) create_client_path(client1, "working");
236 if(c1f) create_client_path(client1, "finishing");
237 if(c2w) create_client_path(client2, "working");
238 if(c2f) create_client_path(client2, "finishing");
239
240 create_data_files(sdirs->data, dfiles, dfileslen);
241
242 fail_unless(!delete_unused_data_files(sdirs, resume));
243
244 assert_existences(sdirs->data,
245 exists, existslen, 1 /* does exist */);
246 assert_existences(sdirs->data,
247 deleted, deletedlen, 0 /* does not exist */);
248
249 free_w(&client2);
250 free_w(&c1cfiles);
251 free_w(&c2cfiles);
252 free_w(&c2dfiles);
253 free_w(&dindex_old);
254
255 tear_down(&sdirs);
256 }
257
do_in_progress_test(int resume,int old_dindex,int c1w,int c1f,int c2w,int c2f,uint64_t * exists,size_t existslen,uint64_t * deleted,size_t deletedlen)258 static void do_in_progress_test(int resume, int old_dindex,
259 int c1w, int c1f, int c2w, int c2f,
260 uint64_t *exists, size_t existslen,
261 uint64_t *deleted, size_t deletedlen)
262 {
263 do_delete_unused_data_files(
264 resume, old_dindex, c1w, c1f, c2w, c2f,
265 din1, ARR_LEN(din1), // data files on disk
266 din1, ARR_LEN(din1), // client1 created cfiles
267 din0, ARR_LEN(din0), // client2 created cfiles
268 din0, ARR_LEN(din0), // client1 dfiles in backups
269 din0, ARR_LEN(din0), // client2 dfiles in backups
270 exists, existslen,
271 deleted, deletedlen
272 );
273 }
274
do_in_progress_test_deleted(int resume,int old_dindex,int c1w,int c1f,int c2w,int c2f)275 static void do_in_progress_test_deleted(int resume, int old_dindex,
276 int c1w, int c1f, int c2w, int c2f)
277 {
278 do_in_progress_test(resume, old_dindex, c1w, c1f, c2w, c2f,
279 din0, ARR_LEN(din0), // expected exists
280 din1, ARR_LEN(din1) // expected deleted
281 );
282 }
283
do_in_progress_test_existed(int resume,int old_dindex,int c1w,int c1f,int c2w,int c2f)284 static void do_in_progress_test_existed(int resume, int old_dindex,
285 int c1w, int c1f, int c2w, int c2f)
286 {
287 do_in_progress_test(resume, old_dindex, c1w, c1f, c2w, c2f,
288 din1, ARR_LEN(din1), // expected exists
289 din0, ARR_LEN(din0) // expected deleted
290 );
291 }
292
START_TEST(test_delete_unused_data_files_in_progress)293 START_TEST(test_delete_unused_data_files_in_progress)
294 {
295 // resume, old_dindex, c1w, c1f, c2w, c2f
296 do_in_progress_test_deleted(0, 0, 0, 0, 0, 0);
297 do_in_progress_test_existed(1, 0, 0, 0, 0, 0);
298
299 // Having client1 with working/finished files will end up deleting
300 // data files because the 'in progress' code will skip the client
301 // kicking off the delete. The client1 cfile is then valid to process.
302 do_in_progress_test_deleted(0, 0, 1, 0, 0, 0);
303 do_in_progress_test_deleted(0, 0, 0, 1, 0, 0);
304 do_in_progress_test_existed(0, 0, 0, 0, 1, 0);
305 do_in_progress_test_existed(0, 0, 0, 0, 0, 1);
306 do_in_progress_test_existed(0, 0, 1, 0, 1, 0);
307 do_in_progress_test_existed(0, 0, 0, 1, 0, 1);
308 // In the resume case, nothing gets deleted.
309 do_in_progress_test_existed(1, 0, 1, 0, 0, 0);
310 do_in_progress_test_existed(1, 0, 0, 1, 0, 0);
311 do_in_progress_test_existed(1, 0, 0, 0, 1, 0);
312 do_in_progress_test_existed(1, 0, 0, 0, 0, 1);
313 do_in_progress_test_existed(1, 0, 1, 0, 1, 0);
314 do_in_progress_test_existed(1, 0, 0, 1, 0, 1);
315 }
316 END_TEST
317
do_delete_unused_data_files_existed(uint64_t * c1cf,size_t c1cflen,uint64_t * c2cf,size_t c2cflen,uint64_t * c1df,size_t c1dflen,uint64_t * c2df,size_t c2dflen)318 static void do_delete_unused_data_files_existed(
319 uint64_t *c1cf, size_t c1cflen,
320 uint64_t *c2cf, size_t c2cflen,
321 uint64_t *c1df, size_t c1dflen,
322 uint64_t *c2df, size_t c2dflen)
323 {
324 do_delete_unused_data_files(
325 0, 0, 0, 0, 0, 0,
326 din1, ARR_LEN(din1), // data files on disk
327 c1cf, c1cflen,
328 c2cf, c2cflen,
329 c1df, c1dflen,
330 c2df, c2dflen,
331 din1, ARR_LEN(din1), // expected exists
332 din0, ARR_LEN(din0) // expected deleted
333 );
334 }
335
do_delete_unused_data_files_deleted(uint64_t * c1cf,size_t c1cflen,uint64_t * c2cf,size_t c2cflen,uint64_t * c1df,size_t c1dflen,uint64_t * c2df,size_t c2dflen)336 static void do_delete_unused_data_files_deleted(
337 uint64_t *c1cf, size_t c1cflen,
338 uint64_t *c2cf, size_t c2cflen,
339 uint64_t *c1df, size_t c1dflen,
340 uint64_t *c2df, size_t c2dflen)
341 {
342 do_delete_unused_data_files(
343 0, 0, 0, 0, 0, 0,
344 din1, ARR_LEN(din1), // data files on disk
345 c1cf, c1cflen,
346 c2cf, c2cflen,
347 c1df, c1dflen,
348 c2df, c2dflen,
349 din0, ARR_LEN(din0), // expected exists
350 din1, ARR_LEN(din1) // expected deleted
351 );
352 }
353
START_TEST(test_delete_unused_data_files)354 START_TEST(test_delete_unused_data_files)
355 {
356 do_delete_unused_data_files_deleted(
357 din1, ARR_LEN(din1), // client1 created cfiles
358 din0, ARR_LEN(din0), // client2 created cfiles
359 din0, ARR_LEN(din0), // client1 dfiles in backups
360 din0, ARR_LEN(din0) // client2 dfiles in backups
361 );
362 do_delete_unused_data_files_deleted(
363 din0, ARR_LEN(din0), // client1 created cfiles
364 din1, ARR_LEN(din1), // client2 created cfiles
365 din0, ARR_LEN(din0), // client1 dfiles in backups
366 din0, ARR_LEN(din0) // client2 dfiles in backups
367 );
368 do_delete_unused_data_files_deleted(
369 din1, ARR_LEN(din1), // client1 created cfiles
370 din1, ARR_LEN(din1), // client2 created cfiles
371 din0, ARR_LEN(din0), // client1 dfiles in backups
372 din0, ARR_LEN(din0) // client2 dfiles in backups
373 );
374 do_delete_unused_data_files_existed(
375 din0, ARR_LEN(din0), // client1 created cfiles
376 din0, ARR_LEN(din0), // client2 created cfiles
377 din1, ARR_LEN(din1), // client1 dfiles in backups
378 din0, ARR_LEN(din0) // client2 dfiles in backups
379 );
380 do_delete_unused_data_files_existed(
381 din0, ARR_LEN(din0), // client1 created cfiles
382 din0, ARR_LEN(din0), // client2 created cfiles
383 din0, ARR_LEN(din0), // client1 dfiles in backups
384 din1, ARR_LEN(din1) // client2 dfiles in backups
385 );
386 do_delete_unused_data_files_existed(
387 din0, ARR_LEN(din0), // client1 created cfiles
388 din0, ARR_LEN(din0), // client2 created cfiles
389 din1, ARR_LEN(din1), // client1 dfiles in backups
390 din1, ARR_LEN(din1) // client2 dfiles in backups
391 );
392 do_delete_unused_data_files_existed(
393 din1, ARR_LEN(din1), // client1 created cfiles
394 din1, ARR_LEN(din1), // client2 created cfiles
395 din1, ARR_LEN(din1), // client1 dfiles in backups
396 din0, ARR_LEN(din0) // client2 dfiles in backups
397 );
398 do_delete_unused_data_files_existed(
399 din1, ARR_LEN(din1), // client1 created cfiles
400 din0, ARR_LEN(din0), // client2 created cfiles
401 din1, ARR_LEN(din1), // client1 dfiles in backups
402 din0, ARR_LEN(din0) // client2 dfiles in backups
403 );
404 do_delete_unused_data_files_existed(
405 din0, ARR_LEN(din0), // client1 created cfiles
406 din1, ARR_LEN(din1), // client2 created cfiles
407 din0, ARR_LEN(din0), // client1 dfiles in backups
408 din1, ARR_LEN(din1) // client2 dfiles in backups
409 );
410 do_delete_unused_data_files_existed(
411 din1, ARR_LEN(din1), // client1 created cfiles
412 din1, ARR_LEN(din1), // client2 created cfiles
413 din0, ARR_LEN(din0), // client1 dfiles in backups
414 din1, ARR_LEN(din1) // client2 dfiles in backups
415 );
416 do_delete_unused_data_files_existed(
417 din0, ARR_LEN(din1), // client1 created cfiles
418 din1, ARR_LEN(din0), // client2 created cfiles
419 din1, ARR_LEN(din1), // client1 dfiles in backups
420 din1, ARR_LEN(din1) // client2 dfiles in backups
421 );
422 do_delete_unused_data_files_existed(
423 din1, ARR_LEN(din1), // client1 created cfiles
424 din0, ARR_LEN(din0), // client2 created cfiles
425 din1, ARR_LEN(din1), // client1 dfiles in backups
426 din1, ARR_LEN(din1) // client2 dfiles in backups
427 );
428 do_delete_unused_data_files_existed(
429 din1, ARR_LEN(din1), // client1 created cfiles
430 din1, ARR_LEN(din1), // client2 created cfiles
431 din1, ARR_LEN(din1), // client1 dfiles in backups
432 din1, ARR_LEN(din1) // client2 dfiles in backups
433 );
434 }
435 END_TEST
436
START_TEST(test_delete_unused_data_files_differ)437 START_TEST(test_delete_unused_data_files_differ)
438 {
439 do_delete_unused_data_files(
440 0, 1, 0, 0, 0, 0,
441 din1, ARR_LEN(din1), // data files on disk
442 din0, ARR_LEN(din0), // client1 created cfiles
443 din1, ARR_LEN(din1), // client2 created cfiles
444 din2, ARR_LEN(din2), // client1 dfiles in backups
445 din3, ARR_LEN(din3), // client2 dfiles in backups
446 din2, ARR_LEN(din2), // expected exists
447 del1, ARR_LEN(del1) // expected deleted
448 );
449 do_delete_unused_data_files(
450 0, 1, 0, 0, 0, 0,
451 din1, ARR_LEN(din1), // data files on disk
452 din0, ARR_LEN(din0), // client1 created cfiles
453 din1, ARR_LEN(din1), // client2 created cfiles
454 din2, ARR_LEN(din2), // client1 dfiles in backups
455 din3, ARR_LEN(din3), // client2 dfiles in backups
456 din2, ARR_LEN(din2), // expected exists
457 del1, ARR_LEN(del1) // expected deleted
458 );
459 do_delete_unused_data_files(
460 0, 1, 0, 0, 0, 0,
461 din1, ARR_LEN(din1), // data files on disk
462 din0, ARR_LEN(din0), // client1 created cfiles
463 din1, ARR_LEN(din1), // client2 created cfiles
464 din3, ARR_LEN(din3), // client1 dfiles in backups
465 din2, ARR_LEN(din2), // client2 dfiles in backups
466 din2, ARR_LEN(din2), // expected exists
467 del1, ARR_LEN(del1) // expected deleted
468 );
469 do_delete_unused_data_files(
470 0, 1, 0, 0, 0, 0,
471 din1, ARR_LEN(din1), // data files on disk
472 din1, ARR_LEN(din1), // client1 created cfiles
473 din0, ARR_LEN(din0), // client2 created cfiles
474 din3, ARR_LEN(din3), // client1 dfiles in backups
475 din2, ARR_LEN(din2), // client2 dfiles in backups
476 din2, ARR_LEN(din2), // expected exists
477 del1, ARR_LEN(del1) // expected deleted
478 );
479 }
480 END_TEST
481
suite_server_protocol2_champ_chooser_dindex(void)482 Suite *suite_server_protocol2_champ_chooser_dindex(void)
483 {
484 Suite *s;
485 TCase *tc_core;
486
487 s=suite_create("server_protocol2_champ_chooser_dindex");
488
489 tc_core=tcase_create("Core");
490 tcase_set_timeout(tc_core, 60);
491
492 tcase_add_test(tc_core, test_dindex);
493 tcase_add_test(tc_core, test_delete_unused_data_files_error);
494 tcase_add_test(tc_core, test_delete_unused_data_files_in_progress);
495 tcase_add_test(tc_core, test_delete_unused_data_files);
496 tcase_add_test(tc_core, test_delete_unused_data_files_differ);
497 suite_add_tcase(s, tc_core);
498
499 return s;
500 }
501